@sagentix-inc/evidence-collector 0.3.1 → 0.3.3

package/collector/src/deliveries.js

@@ -0,0 +1,244 @@
+// What sos-app may deliver to the vault (apps/sos-app/docs/COLLECTOR_CHANNEL.md, section 8), and the vault's summary the
+// check-in reports. A check-in answer may carry `deliveries`; each is one of four things, and nothing else:
+//
+//   set         a value for a name the config's `credentials` lists, with that name's kind: a secret or file sealed in
+//               the browser to this collector's key (sos-app cannot open it), or a setting in clear
+//   remove      the vault's value for such a name
+//   import-env  move such names' values from sagentix-collector.env into the vault, then drop those lines
+//   test        run one connector of the config dry (resident mode only), and report its counts
+//
+// A delivery for a name the config does not list, or with another kind, is refused and reported, never applied: what
+// sos-app says cannot widen what this collector reads. No delivery can name a file, a command or a source option.
+// The outcome of each goes back in a signed POST /api/collect/vault/ack, with the vault's new summary.
+import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync } from "node:fs";
+import { dirname, isAbsolute, join } from "node:path";
+import { signedPost, writePrivate } from "./channel.js";
+import { scrubText } from "./report.js";
+import { KINDS, MAX_VALUE, NAME, Vault, canonicalBind } from "./vault.js";
+export const ACK_PATH = "/api/collect/vault/ack";
+export const ENV_FILE = "sagentix-collector.env";
+export const MAX_DELIVERIES = 50;
+const ID = /^[A-Za-z0-9-]{1,64}$/;
+/** Each name a connector reads, with its kind: the only names a delivery may touch. */
+export function knownNames(creds) {
+    const m = new Map();
+    for (const c of Object.values(creds)) {
+        for (const n of c.secrets ?? [])
+            m.set(n, "secret");
+        for (const n of c.files ?? [])
+            m.set(n, "file");
+        for (const n of Object.values(c.instead ?? {}))
+            if (!m.has(n))
+                m.set(n, n.endsWith("_FILE") ? "file" : "secret");
+        for (const n of [...(c.settings ?? []), ...Object.keys(c.optional ?? {})])
+            if (!m.has(n))
+                m.set(n, "setting");
+    }
+    return m;
+}
+/** The settings of the connector a secret belongs to, as the vault holds them now: what a sealed value binds to. */
+export function bindFor(creds, name, v) {
+    const out = {};
+    for (const k of settingsOf(creds, name)) {
+        const val = v.get(k);
+        if (val !== undefined)
+            out[k] = val;
+    }
+    return out;
+}
+/** The settings (required and optional) of every connector that reads this secret or file. */
+export function settingsOf(creds, name) {
+    const out = new Set();
+    for (const c of Object.values(creds)) {
+        if (![...(c.secrets ?? []), ...(c.files ?? []), ...Object.values(c.instead ?? {})].includes(name))
+            continue;
+        for (const k of [...(c.settings ?? []), ...Object.keys(c.optional ?? {})])
+            out.add(k);
+    }
+    return [...out];
+}
+/** The config's credentials, or none when the file cannot be read. */
+export function credentialsIn(configPath) {
+    try {
+        return JSON.parse(readFileSync(configPath, "utf8").replace(/^\uFEFF/, "")).credentials ?? {};
+    }
+    catch {
+        return {};
+    }
+}
+/** A file's text when it exists and is at most MAX_VALUE bytes; one open, so the size checked is the file read. */
+function readSmall(path) {
+    let fd;
+    try {
+        fd = openSync(path, "r");
+    }
+    catch {
+        return null;
+    }
+    try {
+        if (fstatSync(fd).size > MAX_VALUE)
+            return null;
+        const buf = Buffer.alloc(MAX_VALUE + 1);
+        const n = readSync(fd, buf, 0, buf.length, 0);
+        return n > MAX_VALUE ? null : buf.subarray(0, n).toString("utf8");
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/** The vault beside a config, as the check-in sees it. `test`: the agent's dry run of some sources (absent in `run`). */
+export function vaultBridge(o) {
+    const dir = dirname(o.configPath), envPath = join(dir, ENV_FILE);
+    let vault;
+    const v = () => (vault ??= new Vault(dir, o.holder));
+    const creds = () => {
+        try {
+            return JSON.parse(readFileSync(o.configPath, "utf8").replace(/^/, "")).credentials ?? {};
+        }
+        catch {
+            return {};
+        }
+    };
+    const envLines = (known) => !existsSync(envPath) ? [] : readFileSync(envPath, "utf8").split(/\r?\n/)
+        .map((l) => /^(?:export\s+)?([A-Z][A-Z0-9_]*)=(.*)$/.exec(l.trim()))
+        .filter((m) => !!m && known.has(m[1]) && m[2].length > 0)
+        .map((m) => ({ name: m[1], value: m[2].replace(/^(['"])(.*)\1$/, "$2") }));
+    function summary() {
+        const c = creds(), known = knownNames(c);
+        let sealKey = null, entries = [], error = null, protection = null;
+        try {
+            protection = v().protection;
+            // A setting's value goes along (a URL, a tenant id), so sos-app shows it and the browser binds a secret to it.
+            entries = v().list().map((e) => e.kind === "setting" ? { ...e, value: v().get(e.name) } : e);
+            sealKey = Object.keys(c).length ? v().sealPublicKey() : null;
+        }
+        catch (e) {
+            error = String(e?.message ?? e).slice(0, 300);
+        }
+        return { sealKey, protection, error, entries, envFile: [...new Set(envLines(known).map((l) => l.name))], credentials: c };
+    }
+    function importEnv(known) {
+        const moved = [], leftFiles = [], problems = [], c = creds();
+        // Settings first, so each secret binds to the settings it came with.
+        for (const l of envLines(known).sort((x, y) => Number(known.get(x.name) !== "setting") - Number(known.get(y.name) !== "setting"))) {
+            const kind = known.get(l.name);
+            try {
+                if (kind === "file") {
+                    const path = isAbsolute(l.value) ? l.value : join(dir, l.value);
+                    const text = readSmall(path);
+                    if (text === null) {
+                        problems.push(`${l.name}: the file it names is missing or too large`);
+                        continue;
+                    }
+                    v().set(l.name, text, { kind, now: o.now(), bind: bindFor(c, l.name, v()) });
+                    leftFiles.push(l.value);
+                }
+                else
+                    v().set(l.name, l.value, { kind, now: o.now(), ...(kind !== "setting" ? { bind: bindFor(c, l.name, v()) } : {}) });
+                moved.push(l.name);
+            }
+            catch (e) {
+                problems.push(`${l.name}: ${String(e?.message ?? e)}`);
+            }
+        }
+        if (moved.length) {
+            const keep = readFileSync(envPath, "utf8").split(/\r?\n/).filter((l) => !moved.includes(/^(?:export\s+)?([A-Z][A-Z0-9_]*)=/.exec(l.trim())?.[1] ?? ""));
+            writePrivate(envPath, keep.join("\n").replace(/\n*$/, "\n"));
+        }
+        return { moved, leftFiles, ...(problems.length ? { error: problems.join("; ") } : {}) };
+    }
+    async function apply(raw, a) {
+        if (!Array.isArray(raw))
+            return [];
+        const c = creds(), known = knownNames(c), out = [];
+        for (const d of raw.slice(0, MAX_DELIVERIES)) {
+            if (typeof d !== "object" || d === null || typeof d.id !== "string" || !ID.test(d.id))
+                continue;
+            const refuse = (error) => { out.push({ id: d.id, ok: false, error }); o.log(`vault: refused delivery ${d.id}: ${error}`); };
+            try {
+                if (d.op === "set" || d.op === "remove") {
+                    const name = String(d.name ?? ""), kind = known.get(name);
+                    if (!NAME.test(name) || !kind) {
+                        refuse(`${name.slice(0, 100)} is not a name this collector's connectors read`);
+                        continue;
+                    }
+                    if (d.op === "remove") {
+                        v().remove(name);
+                        o.log(`vault: ${name} removed`);
+                        out.push({ id: d.id, ok: true });
+                        continue;
+                    }
+                    if (d.kind !== kind || !KINDS.includes(d.kind)) {
+                        refuse(`${name} is a ${kind}, not a ${String(d.kind)}`);
+                        continue;
+                    }
+                    let value;
+                    if (kind === "setting") {
+                        if (typeof d.value !== "string" || !d.value.trim()) {
+                            refuse(`${name}: no value`);
+                            continue;
+                        }
+                        value = d.value.trim();
+                    }
+                    else {
+                        if (!d.sealed || d.sealed.name !== name || d.sealed.kind !== kind) {
+                            refuse(`${name}: a ${kind} must come sealed, for this name and kind`);
+                            continue;
+                        }
+                        // The connector's settings as they stand now must be the ones the person saw when sealing.
+                        const bind = d.sealed.bind ?? {}, want = bindFor(c, name, v());
+                        const missing = Object.values(c).filter((x) => [...(x.secrets ?? []), ...(x.files ?? []), ...Object.values(x.instead ?? {})].includes(name))
+                            .flatMap((x) => x.settings ?? []).filter((k) => want[k] === undefined);
+                        if (missing.length) {
+                            refuse(`${name}: set ${missing.join(", ")} first, so the secret is bound to where it goes`);
+                            continue;
+                        }
+                        if (canonicalBind(bind) !== canonicalBind(want)) {
+                            refuse(`${name}: the connector's settings changed since it was sealed; enter it again`);
+                            continue;
+                        }
+                        value = v().openSealed(o.keyId, d.sealed);
+                    }
+                    v().set(name, value, { kind, now: o.now(), ...(kind !== "setting" ? { bind: d.sealed.bind ?? {} } : {}) });
+                    o.log(`vault: ${name} set`);
+                    out.push({ id: d.id, ok: true });
+                }
+                else if (d.op === "import-env") {
+                    const r = importEnv(known);
+                    o.log(`vault: ${r.moved?.length ?? 0} value(s) moved from ${ENV_FILE}`);
+                    out.push({ id: d.id, ok: !r.error, ...r });
+                }
+                else if (d.op === "test") {
+                    if (!a.agent || !o.test)
+                        continue; // left for the resident agent, which can run it now
+                    const id = String(d.connector ?? ""), cred = c[id];
+                    if (!cred) {
+                        refuse(`${id.slice(0, 40)} is not a connector of this collector`);
+                        continue;
+                    }
+                    const t = await o.test(cred.sources);
+                    const secrets = v().all().filter((e) => e.kind !== "setting").map((e) => e.value).filter((x) => x.length >= 6);
+                    out.push({ id: d.id, ok: true, test: { ...t, errors: t.errors.slice(0, 10).map((e) => scrubText(e, secrets).slice(0, 500)) } });
+                }
+                else
+                    refuse(`unknown operation ${String(d.op).slice(0, 20)}`);
+            }
+            catch (e) {
+                refuse(String(e?.message ?? e).slice(0, 300));
+            }
+        }
+        return out;
+    }
+    return { summary, apply };
+}
+/** Reports what the deliveries did, with the vault's summary after them. Never throws. */
+export async function ack(c, results, s, log) {
+    try {
+        const r = await signedPost(c, ACK_PATH, new TextEncoder().encode(JSON.stringify({ format: "sagentix-vault-ack/1", results, vault: s })));
+        if (r.status !== 200)
+            log(`warning: vault ack: HTTP ${r.status}`);
+    }
+    catch (e) {
+        log(`warning: vault ack failed: ${String(e?.message ?? e)}`);
+    }
+}

package/collector/src/report.js

@@ -187,6 +187,7 @@ export class RunRecorder {
     }
 }
 // ── Building, trimming, sending ───────────────────────────────────────────────────────────────────
+/** `secrets`: values to cut whatever their variable is called (the vault's secrets and files). */
 export function buildReport(rec, o) {
     const report = {
         format: REPORT_FORMAT,
@@ -205,7 +206,8 @@ export function buildReport(rec, o) {
     };
     if (rec.log.length > MAX_LOG_LINES)
         report.log[MAX_LOG_LINES - 1] = `(log cut: ${rec.log.length} lines, the first ${MAX_LOG_LINES - 1} kept)`;
-    return trimReport(scrubDeep(report, envSecrets(o.env)));
+    const secrets = [...new Set([...envSecrets(o.env), ...(o.secrets ?? []).filter((v) => v.length >= 6)])].sort((a, b) => b.length - a.length);
+    return trimReport(scrubDeep(report, secrets));
 }
 const bytes = (r) => Buffer.byteLength(JSON.stringify(r, null, 2));
 /** At most MAX_REPORT_BYTES: drops log lines first, then calls, then stacks, and says so in a last log line. */

package/collector/src/sources/prowler.js

@@ -31,6 +31,7 @@ export const REFUSED_ARGS = [
 ];
 /** Environment variables whose values never leave the machine, even if Prowler echoes them in a detail. */
 const SECRET_ENVS = [
+    "GITHUB_TOKEN", "CLOUDFLARE_API_TOKEN_READONLY", "ENTRA_CLIENT_SECRET", "M365_CERTIFICATE_CONTENT",
     "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AZURE_CLIENT_SECRET", "GITHUB_PERSONAL_ACCESS_TOKEN", "GITHUB_OAUTH_APP_TOKEN",
     "GITHUB_APP_KEY", "OKTA_PRIVATE_KEY", "CLOUDFLARE_API_TOKEN", "CLOUDFLARE_API_KEY", "GOOGLEWORKSPACE_CREDENTIALS_CONTENT",
 ];
@@ -43,7 +44,12 @@ function catalogIds(dir = DEFAULT_CATALOG_DIR) {
         return [];
     }
 }
-const one = (v) => (typeof v === "string" && v.trim() ? v.trim() : undefined);
+const one = (v) => {
+    const x = Array.isArray(v) ? v[0] : v;
+    return typeof x === "string" && x.trim() ? x.trim() : undefined;
+};
+/** --prowler-provider aws,azure or repeated, in order, without repeats. */
+const providersOf = (v) => [...new Set((Array.isArray(v) ? v : typeof v === "string" ? [v] : []).flatMap((x) => String(x).split(",")).map((x) => x.trim()).filter(Boolean))];
 /** Splits --prowler-args like a shell would for plain words and '…' or "…" quoting (no expansion). */
 export function splitArgs(s) {
     const out = [];
@@ -55,11 +61,22 @@ export function splitArgs(s) {
 }
 /** The provider's authentication flag, chosen from what the environment holds. Everything else Prowler reads from env itself. */
 export function authArgs(provider, env) {
+    // The generated workflow names what the collector's own source reads, so Prowler reads the same scope
+    // (Prowler 5.47.0: aws --region, azure --subscription-ids, gcp --project-ids).
+    const scope = (name, flag) => { const v = (env[name] ?? "").split(/\s+/).filter(Boolean); return v.length ? [flag, ...v] : []; };
     switch (provider) {
-        case "azure":
+        case "aws": return scope("SAGENTIX_PROWLER_AWS_REGIONS", "--region");
+        case "gcp": return scope("SAGENTIX_PROWLER_GCP_PROJECTS", "--project-ids");
         case "m365":
+            // A certificate signs in every Microsoft 365 service, Exchange and Teams included; a secret covers Graph only.
+            // --init-modules installs the PowerShell modules those services need (an ephemeral runner has none).
+            if (env.M365_CERTIFICATE_CONTENT)
+                return ["--certificate-auth", "--init-modules"];
+            return [...(env.AZURE_CLIENT_SECRET ? ["--sp-env-auth"] : ["--az-cli-auth"]), "--init-modules"];
+        case "azure":
             // A client secret in env: service principal; otherwise the session `azure/login` (OIDC) left for the az CLI.
-            return env.AZURE_CLIENT_SECRET ? ["--sp-env-auth"] : ["--az-cli-auth"];
+            return [...(env.AZURE_CLIENT_SECRET ? ["--sp-env-auth"] : ["--az-cli-auth"]),
+                ...(provider === "azure" ? scope("SAGENTIX_PROWLER_AZURE_SUBSCRIPTIONS", "--subscription-ids") : [])];
         case "kubernetes":
             return env.KUBECONFIG ? ["--kubeconfig-file", env.KUBECONFIG] : [];
         default:
@@ -68,6 +85,25 @@ export function authArgs(provider, env) {
             return [];
     }
 }
+/**
+ * Prowler reads its own variable names; the collector's native sources read theirs. So the same read-only credential
+ * the client already stored serves both ways (2026-11-01: each control collected at least two ways): the collector's
+ * name is copied to Prowler's when Prowler's is unset. Names from prowler/providers/<provider> in Prowler 5.47.0.
+ */
+export function prowlerEnv(provider, env) {
+    const alias = {
+        github: { GITHUB_PERSONAL_ACCESS_TOKEN: "GITHUB_TOKEN" },
+        cloudflare: { CLOUDFLARE_API_TOKEN: "CLOUDFLARE_API_TOKEN_READONLY" },
+        googleworkspace: { GOOGLEWORKSPACE_CREDENTIALS_FILE: "GOOGLE_SA_KEY_FILE", GOOGLEWORKSPACE_DELEGATED_USER: "GOOGLE_ADMIN_EMAIL" },
+        m365: { AZURE_TENANT_ID: "ENTRA_TENANT_ID", AZURE_CLIENT_ID: "ENTRA_CLIENT_ID", AZURE_CLIENT_SECRET: "ENTRA_CLIENT_SECRET" },
+        okta: { OKTA_ORG_DOMAIN: "OKTA_DOMAIN" },
+    };
+    const out = { ...env };
+    for (const [theirs, ours] of Object.entries(alias[provider] ?? {}))
+        if (!out[theirs] && env[ours])
+            out[theirs] = env[ours];
+    return out;
+}
 const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 const s = (v) => (typeof v === "string" ? v : "");
 const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
@@ -184,44 +220,63 @@ export async function runProwler(provider, o) {
 export const prowler = {
     name: "prowler",
     options: {
-        "prowler-provider": { type: "string", help: `Prowler provider to run: ${PROWLER_PROVIDERS.join(", ")}` },
+        "prowler-provider": { type: "string", multiple: true, help: `Prowler providers to run, comma-separated: ${PROWLER_PROVIDERS.join(", ")} (one failing never stops the others)` },
         "prowler-bin": { type: "string", help: "the prowler executable (default prowler on PATH)" },
         "prowler-args": { type: "string", help: "extra Prowler options, written --prowler-args=\"--region us-east-1 ca-central-1\" (never output, send or fix options)" },
         "prowler-output": { type: "string", help: "read this json-ocsf file instead of running Prowler" },
     },
     emits: catalogIds(),
     describe(opts) {
-        const p = one(opts["prowler-provider"]) ?? "(no provider)";
+        const p = providersOf(opts["prowler-provider"]).join(", ") || "(no provider)";
         return one(opts["prowler-output"]) ? `Prowler ${p} findings from a json-ocsf file` : `Prowler ${PROWLER_VERSION} ${p} scan`;
     },
     async run(ctx) {
-        const provider = one(ctx.opts["prowler-provider"]);
-        if (!provider || !PROWLER_PROVIDERS.includes(provider)) {
-            throw new Error(`--prowler-provider is required: one of ${PROWLER_PROVIDERS.join(", ")}.`);
-        }
-        const known = new Set(catalogIds(one(ctx.opts.catalog) ?? DEFAULT_CATALOG_DIR));
-        if (![...known].some((id) => id.startsWith(`prowler.${provider}.`))) {
-            ctx.warn(`prowler: sos-app's catalogue lists no ${provider} check (Prowler maps none to a KSI or SP 800-53 control); every finding will be dropped.`);
+        const providers = providersOf(ctx.opts["prowler-provider"]);
+        const bad = providers.filter((p) => !PROWLER_PROVIDERS.includes(p));
+        if (!providers.length || bad.length) {
+            throw new Error(`--prowler-provider is required: one or more of ${PROWLER_PROVIDERS.join(", ")}${bad.length ? ` (not: ${bad.join(", ")})` : ""}.`);
         }
         const file = one(ctx.opts["prowler-output"]);
-        let text;
-        if (file) {
+        if (file && providers.length > 1)
+            throw new Error("--prowler-output reads one provider's file; name one --prowler-provider with it.");
+        const known = new Set(catalogIds(one(ctx.opts.catalog) ?? DEFAULT_CATALOG_DIR));
+        const results = [];
+        const failures = [];
+        for (const provider of providers) {
+            if (![...known].some((id) => id.startsWith(`prowler.${provider}.`))) {
+                ctx.warn(`prowler: sos-app's catalogue lists no ${provider} check; every finding will be dropped.`);
+            }
+            let text;
             try {
-                text = readFileSync(file, "utf8");
+                if (file) {
+                    try {
+                        text = readFileSync(file, "utf8");
+                    }
+                    catch (e) {
+                        throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
+                    }
+                }
+                else {
+                    const spawner = ctx.clients.prowler ?? spawnProwler;
+                    const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
+                    text = await runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: prowlerEnv(provider, ctx.env), spawner });
+                }
             }
             catch (e) {
-                throw new Error(`cannot read --prowler-output ${file}: ${e.code ?? String(e)}`);
+                // A second way that fails leaves the control to its other ways: say so and go on with the next provider.
+                failures.push(e);
+                if (providers.length > 1)
+                    ctx.warn(`prowler: ${provider} did not run (${e.message}); its controls keep their other ways.`);
+                continue;
             }
+            const { results: r, summary } = parseOcsf(text, provider, known);
+            results.push(...r);
+            ctx.warn(`prowler: ${summary.findings} findings; ${summary.kept} results (${summary.merged} merged into the same check and resource, ` +
+                `${summary.muted} muted and ${summary.manual} manual sent as skipped); ${summary.dropped} findings dropped from ${summary.droppedChecks.length} checks not in the catalogue` +
+                (summary.droppedChecks.length ? `: ${summary.droppedChecks.slice(0, 20).join(", ")}${summary.droppedChecks.length > 20 ? ", …" : ""}` : ""));
         }
-        else {
-            const spawner = ctx.clients.prowler ?? spawnProwler;
-            const extra = one(ctx.opts["prowler-args"]) ? splitArgs(one(ctx.opts["prowler-args"])) : [];
-            text = await runProwler(provider, { bin: one(ctx.opts["prowler-bin"]) ?? "prowler", extra, env: ctx.env, spawner });
-        }
-        const { results, summary } = parseOcsf(text, provider, known);
-        ctx.warn(`prowler: ${summary.findings} findings; ${summary.kept} results (${summary.merged} merged into the same check and resource, ` +
-            `${summary.muted} muted and ${summary.manual} manual sent as skipped); ${summary.dropped} findings dropped from ${summary.droppedChecks.length} checks not in the catalogue` +
-            (summary.droppedChecks.length ? `: ${summary.droppedChecks.slice(0, 20).join(", ")}${summary.droppedChecks.length > 20 ? ", …" : ""}` : ""));
+        if (failures.length === providers.length)
+            throw failures[0];
         return scrub(results, SECRET_ENVS.map((n) => ctx.env[n]));
     },
 };