@sagentix-inc/evidence-collector 0.2.0 → 0.3.0

package/collector/src/agent.js

@@ -0,0 +1,312 @@
+// `collector agent --config <file>`: resident mode (apps/sos-app/docs/COLLECTOR_CHANNEL.md, section 5). Keeps one
+// outbound WebSocket open to sos-app's /api/collect/connect so sos-app can ask for a run now instead of waiting for
+// the schedule. The channel carries four words and nothing else: sos-app may say run, checkin or ping; the agent says
+// hello, status and pong. A run is the same `run --config <file>` the schedule starts, as a child process, never two
+// at once. No message can carry a command, a file or a configuration, so the collector stays read-only whatever the
+// server says; anything else received is logged and ignored.
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, extname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { CONNECT_PATH, EXIT_UPDATE, MODE_ENV, checkin, collectorInfo, detectRunner } from "./channel.js";
+import { loadConfig } from "./config.js";
+import { signBody } from "./keys.js";
+export const PING_MS = 50_000;
+export const CHECKIN_MS = 6 * 3600_000;
+export const BACKOFF_MIN_MS = 1000;
+export const BACKOFF_MAX_MS = 5 * 60_000;
+// cli.ts beside this file in the repository, cli.js in the npm package (tsc output): the same extension as this one.
+const CLI_PATH = fileURLToPath(new URL(`./cli${extname(fileURLToPath(import.meta.url))}`, import.meta.url));
+/** The next reconnect delay: 1 s doubling to 5 min. */
+export const nextBackoff = (ms) => Math.min(BACKOFF_MAX_MS, Math.max(BACKOFF_MIN_MS, ms * 2));
+/** wss://<origin>/api/collect/connect?key=&ts=&sig= with sig the base64url Ed25519 signature over "sagentix-connect/1\n<key id>\n<ts>". */
+export function connectUrl(endpoint, keyId, pem, nowMs) {
+    const u = new URL(CONNECT_PATH, new URL(endpoint).origin);
+    u.protocol = u.protocol === "https:" ? "wss:" : "ws:";
+    const ts = String(Math.floor(nowMs));
+    const sig = Buffer.from(signBody(pem, new TextEncoder().encode(`sagentix-connect/1\n${keyId}\n${ts}`)), "base64").toString("base64url");
+    u.search = new URLSearchParams({ key: keyId, ts, sig }).toString();
+    return u.toString();
+}
+/** The agent's side of the protocol, without a socket: what to send for each message received. */
+export class AgentSession {
+    h;
+    busy;
+    /** A run's final status that found the socket closed; sent after the next hello. */
+    unsent;
+    constructor(hooks) { this.h = hooks; }
+    get running() { return this.busy?.id; }
+    onOpen() {
+        this.h.send(JSON.stringify({ type: "hello", ...this.h.hello }));
+        if (this.unsent && this.h.send(this.unsent))
+            this.unsent = undefined;
+    }
+    onMessage(data) {
+        if (typeof data !== "string") {
+            this.h.log("agent: ignored a message that is not text");
+            return;
+        }
+        let m;
+        try {
+            m = JSON.parse(data);
+        }
+        catch {
+            this.h.log("agent: ignored a message that is not JSON");
+            return;
+        }
+        if (typeof m !== "object" || m === null) {
+            this.h.log("agent: ignored a message that is not an object");
+            return;
+        }
+        switch (m.type) {
+            case "ping":
+                this.h.send(JSON.stringify({ type: "pong" }));
+                return;
+            case "pong": return; // the answer to our own ping
+            case "checkin":
+                this.h.checkin().catch((e) => this.h.log(`agent: check-in failed: ${String(e?.message ?? e)}`));
+                return;
+            case "run":
+                this.run(m.id);
+                return;
+            default: this.h.log(`agent: ignored a message of type ${JSON.stringify(String(m.type ?? "")).slice(0, 60)}`);
+        }
+    }
+    status(id, state, exitCode) {
+        const text = JSON.stringify({ type: "status", id, state, exitCode });
+        if (!this.h.send(text) && state !== "started")
+            this.unsent = text;
+    }
+    run(id) {
+        if (typeof id !== "string" || !id || id.length > 100) {
+            this.h.log("agent: ignored a run without a usable id");
+            return;
+        }
+        if (this.busy) {
+            this.h.log(`agent: run ${id} refused: run ${this.busy.id} is still going`);
+            this.status(id, "failed", null);
+            return;
+        }
+        let child;
+        try {
+            child = this.h.startRun();
+        }
+        catch (e) {
+            this.h.log(`agent: cannot start run ${id}: ${String(e?.message ?? e)}`);
+            this.status(id, "failed", null);
+            return;
+        }
+        this.busy = { id, child };
+        this.h.log(`agent: run ${id} started`);
+        this.status(id, "started", null);
+        child.exit.then((code) => {
+            this.h.log(`agent: run ${id} ended with exit code ${code}`);
+            this.status(id, code === 0 ? "done" : "failed", code);
+        }, (e) => {
+            this.h.log(`agent: run ${id} failed: ${String(e?.message ?? e)}`);
+            this.status(id, "failed", null);
+        }).finally(() => { this.busy = undefined; this.h.onIdle?.(); });
+    }
+    /** Stops a run in progress (the agent is exiting). */
+    stop() { this.busy?.child.kill(); }
+}
+/** What a run started from sos-app executes (exported for the tests). */
+export function runCommand(configPath, exists, platform = process.platform) {
+    const dir = dirname(resolve(configPath));
+    const ps1 = resolve(dir, "sagentix-collector.ps1"), sh = resolve(dir, "sagentix-collector.sh");
+    if (platform === "win32" && exists(ps1))
+        return ["powershell.exe", ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", ps1]];
+    if (platform !== "win32" && exists(sh))
+        return ["/bin/bash", [sh]];
+    return [process.execPath, [...process.execArgv, CLI_PATH, "run", "--config", configPath]];
+}
+export function realAgentIo() {
+    return {
+        socket: (url) => new WebSocket(url),
+        spawnRun(configPath, env) {
+            // The generated script beside the config when there is one, so a run started from sos-app does what the daily one
+            // does (the update checkout, the application scans, then the collector); else this file's own cli.ts, with the
+            // same Node and flags this agent runs with (process.execArgv). Inside the runner image there is no script.
+            const [cmd, args] = runCommand(configPath, existsSync);
+            const child = spawn(cmd, args, { stdio: "inherit", cwd: dirname(resolve(configPath)), env: { ...env, [MODE_ENV]: "agent" } });
+            const exit = new Promise((res, rej) => { child.once("error", rej); child.once("exit", (code) => res(code)); });
+            return { exit, kill: () => { child.kill("SIGTERM"); } };
+        },
+        onStop(stop) { let done = false; const once = () => { if (!done) {
+            done = true;
+            stop();
+        } }; process.once("SIGINT", once); process.once("SIGTERM", once); },
+        setTimeout: (f, ms) => setTimeout(f, ms),
+        clearTimeout: (t) => clearTimeout(t),
+        setInterval: (f, ms) => setInterval(f, ms),
+        clearInterval: (t) => clearInterval(t),
+    };
+}
+const OPEN = 1;
+export async function agent(argv, deps, io = realAgentIo()) {
+    let values;
+    try {
+        ({ values } = parseArgs({ args: argv, options: { config: { type: "string" } }, strict: true }));
+    }
+    catch (e) {
+        deps.err(`${e.message}\nusage: collector agent --config collector.config.json`);
+        return 2;
+    }
+    if (!values.config) {
+        deps.err("usage: collector agent --config collector.config.json");
+        return 2;
+    }
+    const configPath = resolve(values.config);
+    const c = loadConfig(configPath, deps.sources);
+    if ("problems" in c) {
+        deps.err(`${values.config} is not a valid collector config:\n  ${c.problems.join("\n  ")}`);
+        return 2;
+    }
+    const endpoint = c.values.endpoint, keyId = c.values["key-id"];
+    let pem;
+    if (typeof c.values.key === "string") {
+        try {
+            pem = readFileSync(resolve(c.values.key), "utf8");
+        }
+        catch (e) {
+            deps.err(`cannot read keyFile ${c.values.key}: ${e.code ?? String(e)}`);
+            return 1;
+        }
+    }
+    else
+        pem = deps.env.SAGENTIX_COLLECTOR_KEY;
+    if (typeof keyId !== "string" || !pem) {
+        deps.err("the agent needs keyId and keyFile in the config (or SAGENTIX_COLLECTOR_KEY).");
+        return 2;
+    }
+    let origin;
+    try {
+        const u = new URL(endpoint);
+        const local = u.hostname === "localhost" || u.hostname === "127.0.0.1";
+        if (u.protocol !== "https:" && !(local && u.protocol === "http:"))
+            throw new Error("the endpoint must be https (http only for localhost).");
+        origin = u.origin;
+    }
+    catch (e) {
+        deps.err(e.message);
+        return 2;
+    }
+    const runner = detectRunner(deps.env);
+    const channel = { origin, keyId, pem, fetch: deps.channel ?? deps.fetch, dir: dirname(configPath), runner, mode: "agent", env: deps.env, now: deps.now };
+    const log = (s) => { try {
+        deps.err(s);
+    }
+    catch { /* never let logging end the agent */ } };
+    // A signed update written by a check-in ends the agent with 75 once no run is going: Docker's wrapper loop restarts
+    // the container on the new digest, a service manager restarts the script, which checks out the ref on start.
+    let restartPending = false;
+    const doCheckin = async () => {
+        try {
+            const out = await checkin(channel, { write: true, log });
+            if (out.wrote && !restartPending) {
+                restartPending = true;
+                log(`agent: exiting with ${EXIT_UPDATE} to restart on the update${session.running ? ` once run ${session.running} ends` : ""}`);
+                if (!session.running)
+                    finish(EXIT_UPDATE);
+            }
+        }
+        catch (e) {
+            log(`agent: check-in failed: ${String(e?.message ?? e)}`);
+        }
+    };
+    let ws, ping, retry, stopping = false, backoff = BACKOFF_MIN_MS;
+    const send = (t) => { try {
+        if (ws && ws.readyState === OPEN) {
+            ws.send(t);
+            return true;
+        }
+    }
+    catch (e) {
+        log(`agent: send failed: ${String(e)}`);
+    } return false; };
+    const session = new AgentSession({
+        send,
+        startRun: () => io.spawnRun(configPath, deps.env),
+        checkin: doCheckin,
+        log,
+        hello: { ...collectorInfo(deps.env), runner },
+        onIdle: () => { if (restartPending)
+            finish(EXIT_UPDATE); },
+    });
+    const connect = () => {
+        retry = undefined;
+        if (stopping)
+            return;
+        let s;
+        try {
+            s = io.socket(connectUrl(origin, keyId, pem, deps.now().getTime()));
+        }
+        catch (e) {
+            log(`agent: cannot connect: ${String(e?.message ?? e)}`);
+            return later();
+        }
+        ws = s;
+        s.onopen = () => {
+            log(`agent: connected to ${origin}`);
+            backoff = BACKOFF_MIN_MS;
+            try {
+                session.onOpen();
+            }
+            catch (e) {
+                log(`agent: ${String(e)}`);
+            }
+            void doCheckin();
+            ping = io.setInterval(() => send(JSON.stringify({ type: "ping" })), PING_MS);
+        };
+        s.onmessage = (ev) => { try {
+            session.onMessage(ev.data);
+        }
+        catch (e) {
+            log(`agent: ${String(e)}`);
+        } };
+        s.onerror = () => { };
+        s.onclose = (ev) => {
+            if (ping !== undefined) {
+                io.clearInterval(ping);
+                ping = undefined;
+            }
+            if (ws === s)
+                ws = undefined;
+            if (stopping)
+                return;
+            log(`agent: disconnected${ev?.code ? ` (${ev.code}${ev.reason ? ` ${ev.reason}` : ""})` : ""}; retrying in ${Math.round(backoff / 1000)} s`);
+            later();
+        };
+    };
+    const later = () => {
+        if (stopping || retry !== undefined)
+            return;
+        retry = io.setTimeout(connect, backoff);
+        backoff = nextBackoff(backoff);
+    };
+    let resolveDone = () => { };
+    function finish(code) {
+        if (stopping)
+            return;
+        stopping = true;
+        log(code === EXIT_UPDATE ? "agent: stopping for the update" : "agent: stopping");
+        io.clearInterval(periodic);
+        if (ping !== undefined)
+            io.clearInterval(ping);
+        if (retry !== undefined)
+            io.clearTimeout(retry);
+        session.stop();
+        try {
+            ws?.close(1000, "agent stopping");
+        }
+        catch { /* already closed */ }
+        resolveDone(code);
+    }
+    const periodic = io.setInterval(() => { void doCheckin(); }, CHECKIN_MS);
+    return new Promise((done) => {
+        resolveDone = done;
+        io.onStop(() => finish(0));
+        connect();
+    });
+}

package/collector/src/channel.js

@@ -0,0 +1,220 @@
+// The collector's channel to sos-app beside the evidence push (apps/sos-app/docs/COLLECTOR_CHANNEL.md): who this
+// collector is (version, commit, runner), the signed POST every message uses, and the check-in that may name a
+// signed update. Every connection goes out; nothing here listens, and nothing the server answers can make the
+// collector run a command or change a configuration. The most it can do is have `sagentix-collector.ref` (or
+// `.image`) written beside the config, which the generated script reads on its next start.
+import { spawnSync } from "node:child_process";
+import { chmodSync, existsSync, writeFileSync } from "node:fs";
+import { arch, platform, release } from "node:os";
+import { dirname, join } from "node:path";
+import { createPublicKey, verify } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import { signBody } from "./keys.js";
+import { COLLECTOR_VERSION } from "./payload.js";
+import { RELEASE_PUBLIC_KEYS } from "./release_keys.js";
+export const CHECKIN_PATH = "/api/collect/checkin";
+export const REPORT_PATH = "/api/collect/report";
+export const CONNECT_PATH = "/api/collect/connect";
+export const REF_FILE = "sagentix-collector.ref";
+export const IMAGE_FILE = "sagentix-collector.image";
+/** The agent's exit code after it wrote a signed update: the wrapper loop (Docker) or the service manager restarts it
+ *  on the new image or commit (EX_TEMPFAIL, 75, from sysexits.h). `run` never exits with it. */
+export const EXIT_UPDATE = 75;
+export const RUNNERS = ["github", "server", "docker", "mac", "windows", "unknown"];
+/** Set by the agent on the `run` it starts, so that run's report says `mode: "agent"`. */
+export const MODE_ENV = "SAGENTIX_COLLECTOR_MODE";
+const HEX40 = /^[0-9a-f]{40}$/;
+/** The only image a Docker runner accepts (section 3). */
+export const RUNNER_IMAGE_RE = /^ghcr\.io\/sagentix-advisors-inc\/sagentix-collector-runner(:[A-Za-z0-9._-]{1,128})?@sha256:[0-9a-f]{64}$/;
+const COLLECTOR_DIR = dirname(dirname(fileURLToPath(import.meta.url)));
+let commitCache;
+/**
+ * The commit of the collector's own checkout, or "unknown". Asked of git only when this file is tracked there, so a
+ * copy inside node_modules of someone else's repository never reports that repository's commit. The runner image
+ * has no .git; it may name its commit in SAGENTIX_COLLECTOR_COMMIT.
+ */
+export function collectorCommit(env = process.env) {
+    const given = env.SAGENTIX_COLLECTOR_COMMIT?.trim().toLowerCase();
+    if (given && HEX40.test(given))
+        return given;
+    if (commitCache !== undefined)
+        return commitCache;
+    commitCache = "unknown";
+    try {
+        const git = (...args) => spawnSync("git", ["-C", COLLECTOR_DIR, ...args], { encoding: "utf8", timeout: 3000, stdio: ["ignore", "pipe", "ignore"] });
+        if (git("ls-files", "--error-unmatch", "src/cli.ts").status === 0) {
+            const head = git("rev-parse", "HEAD");
+            const c = head.status === 0 ? head.stdout.trim() : "";
+            if (HEX40.test(c))
+                commitCache = c;
+        }
+    }
+    catch { /* no git: unknown */ }
+    return commitCache;
+}
+export function collectorInfo(env = process.env) {
+    return { version: COLLECTOR_VERSION, commit: collectorCommit(env) };
+}
+/** Where this run happens: SAGENTIX_RUNNER when the generated files set it, else what the machine shows. */
+export function detectRunner(env, plat = platform(), hasDockerEnv = () => existsSync("/.dockerenv")) {
+    const given = env.SAGENTIX_RUNNER?.trim().toLowerCase();
+    if (given && RUNNERS.includes(given))
+        return given;
+    if (env.GITHUB_ACTIONS === "true")
+        return "github";
+    if (plat === "darwin")
+        return "mac";
+    if (plat === "win32")
+        return "windows";
+    if (hasDockerEnv())
+        return "docker";
+    return "server";
+}
+export function modeOf(env) {
+    return env[MODE_ENV] === "agent" ? "agent" : "run";
+}
+export const osInfo = () => ({ platform: platform(), release: release(), arch: arch() });
+/** POSTs exact bytes signed with the collector's key, as a push to /api/collect is. Throws on a network failure. */
+export async function signedPost(c, path, body) {
+    const res = await c.fetch(new URL(path, c.origin).toString(), {
+        method: "POST",
+        headers: { "content-type": "application/json", "x-sagentix-key-id": c.keyId, "x-sagentix-signature": signBody(c.pem, body) },
+        body,
+    });
+    return { status: res.status, text: await res.text() };
+}
+/** Writes a file only its owner can read, also when it already existed with a wider mode. */
+export function writePrivate(path, text) {
+    writeFileSync(path, text, { mode: 0o600 });
+    try {
+        chmodSync(path, 0o600);
+    }
+    catch { /* Windows: no POSIX modes */ }
+}
+/** The bytes the release key signs (section 4). */
+export function releaseMessage(t) {
+    return new TextEncoder().encode(`sagentix-collector-release/1\n${t.version}\n${t.commit}\n${t.image ?? ""}\n${t.date}`);
+}
+const SPKI_ED25519_PREFIX = Buffer.from("302a300506032b6570032100", "hex");
+/** True when the target's signature verifies against one of the release public keys. */
+export function verifyTarget(t, keys = RELEASE_PUBLIC_KEYS) {
+    const sig = Buffer.from(t.signature, "base64");
+    if (sig.length !== 64)
+        return false;
+    const msg = releaseMessage(t);
+    return keys.some((k) => {
+        const raw = Buffer.from(k, "base64");
+        if (raw.length !== 32)
+            return false;
+        try {
+            return verify(null, msg, createPublicKey({ key: Buffer.concat([SPKI_ED25519_PREFIX, raw]), format: "der", type: "spki" }), sig);
+        }
+        catch {
+            return false;
+        }
+    });
+}
+function targetOf(v) {
+    if (typeof v !== "object" || v === null)
+        return "the answer names no target";
+    const t = v;
+    for (const k of ["version", "commit", "date", "signature"])
+        if (typeof t[k] !== "string" || !t[k])
+            return `the target has no ${k}`;
+    if (t.image !== null && t.image !== undefined && typeof t.image !== "string")
+        return "the target's image is not a string";
+    return { version: t.version, commit: t.commit, image: t.image ?? null, date: t.date, signature: t.signature };
+}
+/**
+ * POST /api/collect/checkin, and act on the answer. Never throws and never stops a run: a network failure, a 404
+ * from an sos-app older than this collector or a malformed answer is a warning. `write: false` (a dry run) reads the
+ * answer and writes nothing.
+ */
+export async function checkin(c, o) {
+    const warnings = [];
+    const warn = (s) => { warnings.push(s); o.log(`warning: check-in: ${s}`); };
+    const body = new TextEncoder().encode(JSON.stringify({
+        format: "sagentix-checkin/1", issuedAt: c.now().toISOString(), collector: collectorInfo(c.env),
+        runner: c.runner, mode: c.mode, os: osInfo(), node: process.version,
+    }));
+    let res;
+    try {
+        res = await signedPost(c, CHECKIN_PATH, body);
+    }
+    catch (e) {
+        warn(`POST ${CHECKIN_PATH} failed: ${String(e?.message ?? e)}; the run goes on`);
+        return { action: "failed", warnings };
+    }
+    if (res.status !== 200) {
+        warn(`HTTP ${res.status}${res.status === 404 ? " (this sos-app does not take check-ins yet)" : ""}; the run goes on`);
+        return { action: "failed", warnings };
+    }
+    let a;
+    try {
+        a = JSON.parse(res.text);
+    }
+    catch {
+        warn("the answer is not JSON");
+        return { action: "failed", warnings };
+    }
+    const reason = typeof a.reason === "string" ? a.reason.slice(0, 500) : "";
+    switch (a.action) {
+        case "none": return { action: "none", warnings };
+        case "notify":
+            o.log(`check-in: a newer collector is available, and this runner cannot update itself${reason ? `: ${reason}` : ""}`);
+            return { action: "notify", warnings };
+        case "approval-needed":
+            o.log(`check-in: an update waits for your organization's approval in sos-app${reason ? `: ${reason}` : ""}`);
+            return { action: "approval-needed", warnings };
+        case "update": break;
+        default:
+            warn(`unknown action ${JSON.stringify(a.action)}; nothing done`);
+            return { action: "failed", warnings };
+    }
+    const t = targetOf(a.target);
+    if (typeof t === "string") {
+        warn(`update refused: ${t}; nothing written`);
+        return { action: "update", warnings };
+    }
+    if (!HEX40.test(t.commit)) {
+        warn(`update refused: target commit ${JSON.stringify(t.commit.slice(0, 60))} is not 40 hex; nothing written`);
+        return { action: "update", warnings };
+    }
+    if (!verifyTarget(t, o.keys)) {
+        warn(`update to ${t.version} refused: its signature does not verify against a Sagentix release key; nothing written`);
+        return { action: "update", warnings };
+    }
+    const info = collectorInfo(c.env);
+    if (t.commit === info.commit)
+        return { action: "update", warnings };
+    let file, text;
+    if (c.runner === "docker") {
+        if (!t.image || !RUNNER_IMAGE_RE.test(t.image)) {
+            warn(`update to ${t.version} refused: the target names no runner image with a digest; nothing written`);
+            return { action: "update", warnings };
+        }
+        file = join(c.dir, IMAGE_FILE);
+        text = `${t.image}\n`;
+    }
+    else if (c.runner === "server" || c.runner === "mac" || c.runner === "windows") {
+        file = join(c.dir, REF_FILE);
+        text = `${t.commit}\n${t.version}\n`;
+    }
+    else {
+        warn(`update to ${t.version} offered, but a ${c.runner} runner pins its own commit; nothing written`);
+        return { action: "update", warnings };
+    }
+    if (!o.write) {
+        o.log(`check-in: signed update to ${t.version} (${t.commit}); dry run, so ${file} is not written`);
+        return { action: "update", warnings };
+    }
+    try {
+        writePrivate(file, text);
+    }
+    catch (e) {
+        warn(`cannot write ${file}: ${e.code ?? String(e)}`);
+        return { action: "update", warnings };
+    }
+    o.log(`check-in: signed update to ${t.version} (${t.commit}) written to ${file}; it takes effect on the next run${reason ? ` (${reason})` : ""}`);
+    return { action: "update", wrote: file, warnings };
+}