@runbooks/supervise 0.1.0

package/dist/emit.test.js

@@ -0,0 +1,187 @@
+import { describe, it, expect } from "vitest";
+import { readFileSync, readdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadCorpus } from "@runbooks/fixtures";
+import { derivePolicy } from "./policy.js";
+import { startRun, propose, advance, observe } from "./run.js";
+import { buildRunReport, durationBucket, failureClassFor, FAILURE_CLASS_BY_CODE, UnmappedReasonError, UnclassifiableFailureError, } from "./emit.js";
+const context = {
+    runbook: "std/k8s-node-not-ready-drain",
+    version: "1.0.0",
+    contentHash: `sha256:${"a".repeat(64)}`,
+    profile: "P1",
+    runtimeProfile: "R2",
+    executorKind: "autonomous",
+    envClass: "staging",
+    reportedAt: "2026-09-03",
+};
+function envelopeFor(name) {
+    return derivePolicy(loadCorpus().find((r) => r.name.startsWith(name)).doc);
+}
+function begin(name, inputs, options = {}) {
+    const started = startRun(envelopeFor(name), { agentIdentity: "agent-7", inputs, ...options });
+    if (!started.ok)
+        throw new Error(JSON.stringify(started.refusals));
+    return started.state;
+}
+describe("the report says what the run recorded", () => {
+    it("reports a run still going as partial rather than as anything decided", () => {
+        const state = begin("k8s-node", { node: "node-1" });
+        expect(buildRunReport(state, [], context).outcome).toBe("partial");
+    });
+    it("carries the deviation count, which is the field no other surface provides", () => {
+        let state = begin("disk-pressure", { mount: "/tmp" }, {
+            escalation: {
+                responses: Object.fromEntries(Object.keys(FAILURE_CLASS_BY_CODE).length > 0
+                    ? [["undeclared-tool", "block-and-continue"], ["out-of-scope-tool", "block-and-continue"],
+                        ["out-of-order-step", "block-and-continue"], ["unchecked-advance", "block-and-continue"],
+                        ["budget-exhausted", "block-and-continue"], ["unrouted-failure", "block-and-continue"],
+                        ["terminal-not-reached", "block-and-continue"]]
+                    : []),
+                maxTotal: 99,
+            },
+        });
+        state = propose(envelopeFor("disk-pressure"), state, { tool: "cli:nope" }).state;
+        expect(buildRunReport(state, [], context).deviation_count).toBe(1);
+    });
+    it("names the step a failure happened at", () => {
+        const envelope = envelopeFor("k8s-node");
+        const state = begin("k8s-node", { node: "node-1" }, { evaluatePostconditions: true });
+        const observed = observe(envelope, state, { exitCode: 0, stdout: "True" });
+        const report = buildRunReport(observed.state, observed.events, context);
+        expect(report.failed_step_id).toBe("s1");
+        expect(report.failure_class).toBe("expectation_mismatch");
+    });
+    /** A run that recovered and then failed elsewhere failed at the second place. */
+    it("takes the last cause, not the first", () => {
+        const events = [
+            { kind: "deviation", code: "run/expectation-failed", message: "", at: "s1" },
+            { kind: "deviation", code: "run/wall-clock-exhausted", message: "", at: "s4" },
+        ];
+        const state = { ...begin("k8s-node", { node: "node-1" }), status: "ended", outcome: "timed-out" };
+        const report = buildRunReport(state, events, context);
+        expect(report.failed_step_id).toBe("s4");
+        expect(report.failure_class).toBe("timeout");
+    });
+});
+/**
+ * The rule R-07 states outright: map onto the closed vocabulary, and fail loudly where
+ * there is no mapping rather than inventing a bucket.
+ */
+describe("every internal reason maps, or the emission stops", () => {
+    /**
+     * Against the source rather than against a list someone maintains by hand: a code
+     * added next month is one this test fails on until somebody decides what it means.
+     */
+    const codes = (() => {
+        const dir = dirname(fileURLToPath(import.meta.url));
+        const found = new Set();
+        for (const file of readdirSync(dir).filter((f) => f.endsWith(".ts") && !f.endsWith(".test.ts"))) {
+            const source = readFileSync(join(dir, file), "utf8");
+            for (const [, code] of source.matchAll(/code: "((?:run|runtime)\/[a-z-]+)"/g))
+                found.add(code);
+        }
+        return [...found].sort();
+    })();
+    it("finds the codes the supervisor actually emits", () => {
+        expect(codes.length).toBeGreaterThan(15);
+    });
+    it.each(codes)("%s is classified", (code) => {
+        expect(() => failureClassFor(code)).not.toThrow();
+    });
+    it("throws on a reason nobody has classified", () => {
+        expect(() => failureClassFor("run/something-new")).toThrow(UnmappedReasonError);
+    });
+    it("says why it would rather stop than guess", () => {
+        expect(() => failureClassFor("run/something-new")).toThrow(/reads as evidence/);
+    });
+    it("refuses to build a failed report it cannot classify", () => {
+        const state = {
+            ...begin("k8s-node", { node: "node-1" }),
+            status: "ended",
+            outcome: "failed",
+        };
+        expect(() => buildRunReport(state, [], context)).toThrow(UnclassifiableFailureError);
+    });
+    it("takes the operator's classification when the run did not explain itself", () => {
+        const state = {
+            ...begin("k8s-node", { node: "node-1" }),
+            status: "ended",
+            outcome: "failed",
+        };
+        const report = buildRunReport(state, [], { ...context, failureClass: "human_abort" });
+        expect(report.failure_class).toBe("human_abort");
+    });
+});
+/** Second-level precision plus submission time plus a rare runbook identifies a company. */
+describe("the duration is bucketed and never exact", () => {
+    it.each([
+        [0, "<1m"], [59, "<1m"], [60, "1-5m"], [299, "1-5m"],
+        [300, "5-30m"], [1799, "5-30m"], [1800, ">30m"], [86_400, ">30m"],
+    ])("%is is %s", (seconds, bucket) => {
+        expect(durationBucket(seconds)).toBe(bucket);
+    });
+    it("does not treat an unknown duration as a short one", () => {
+        expect(durationBucket(undefined)).toBe(">30m");
+        expect(durationBucket(Number.NaN)).toBe(">30m");
+    });
+    it("puts no precise duration in the report by any name", () => {
+        const state = { ...begin("k8s-node", { node: "node-1" }, { now: 100 }) };
+        const report = buildRunReport(state, [], { ...context, now: 100 + 137 });
+        expect(JSON.stringify(report)).not.toContain("137");
+        expect(report.duration_bucket).toBe("1-5m");
+    });
+});
+/**
+ * The property that matters most. Runs whose commands, tools and errors are
+ * secret-shaped, and a report that contains none of it — because there is nowhere in the
+ * format for it to go.
+ */
+describe("no free text reaches the report by any path", () => {
+    const secrets = [
+        "hunter2", "sk-live-4242424242", "eyJhbGciOiJIUzI1NiJ9", "wJalrXUtnFEMI",
+        "db-3.prod.internal", "postgres://user:pass@host/db",
+    ];
+    const permissive = {
+        responses: {
+            "undeclared-tool": "block-and-continue", "out-of-scope-tool": "block-and-continue",
+            "out-of-order-step": "block-and-continue", "unchecked-advance": "block-and-continue",
+            "budget-exhausted": "block-and-continue", "unrouted-failure": "block-and-continue",
+            "terminal-not-reached": "block-and-continue",
+        },
+        maxTotal: 999,
+    };
+    it.each(secrets)("keeps %s out of a report from a run that saw it", (secret) => {
+        const envelope = envelopeFor("disk-pressure");
+        let state = begin("disk-pressure", { mount: "/tmp" }, { escalation: permissive });
+        const events = [];
+        for (const attempt of [`cli:${secret}`, `mcp:${secret}`]) {
+            const decided = propose(envelope, state, { tool: attempt });
+            state = decided.state;
+            events.push(...decided.events);
+        }
+        // The events themselves carry the reason, which is local. The report must not.
+        expect(JSON.stringify(events)).toContain(secret);
+        const report = buildRunReport(state, events, context);
+        expect(JSON.stringify(report)).not.toContain(secret);
+    });
+    it("keeps a secret-shaped step title out, since only the id is reported", () => {
+        const envelope = envelopeFor("k8s-node");
+        const state = begin("k8s-node", { node: "node-1" }, { evaluatePostconditions: true });
+        const observed = observe(envelope, state, { exitCode: 0, stdout: "password=hunter2" });
+        const report = buildRunReport(observed.state, observed.events, context);
+        expect(JSON.stringify(report)).not.toContain("hunter2");
+    });
+    it("emits only the fields §14 declares, whatever the run did", () => {
+        const envelope = envelopeFor("disk-pressure");
+        let state = begin("disk-pressure", { mount: "/tmp" }, { escalation: permissive });
+        state = advance(envelope, state, "s2").state;
+        const report = buildRunReport(state, [], context);
+        expect(Object.keys(report).sort()).toEqual([
+            "content_hash", "deviation_count", "duration_bucket", "env_class", "executor_kind",
+            "outcome", "profile", "report_schema", "reported_at", "runbook", "runtime_profile",
+            "schema_version", "version",
+        ]);
+    });
+});

package/dist/enforce.d.ts

@@ -0,0 +1,29 @@
+import type { PolicyEnvelope } from "./policy.js";
+import type { Decided, RunState } from "./run.js";
+export interface StartCheck {
+    readonly ok: boolean;
+    readonly missing: readonly string[];
+    readonly message?: string;
+}
+/**
+ * Whether the client can satisfy the whole allowlist before the run begins.
+ *
+ * Refusing up front rather than failing at the step that needs the missing capability
+ * is normative (§13.4): a run that stops halfway has already done part of the work, and
+ * part of a procedure is a state nobody designed.
+ */
+export declare function canSatisfyAllowlist(envelope: PolicyEnvelope, available: readonly string[]): StartCheck;
+export interface CommandProposal {
+    readonly command: string;
+}
+export interface CommandDecision extends Decided {
+    /** What the command was found to need. */
+    readonly required: readonly string[];
+}
+/**
+ * Authorize a command by what it actually invokes, not by what the step claims.
+ *
+ * A step declaring `cli:kubectl` and running `psql` is the mismatch this exists to
+ * catch, and it is the runtime half of AST03/AST04.
+ */
+export declare function proposeCommand(envelope: PolicyEnvelope, state: RunState, proposal: CommandProposal): CommandDecision;

package/dist/enforce.js

@@ -0,0 +1,100 @@
+/**
+ * Allowlist and per-step scope enforcement.
+ *
+ * Where `capabilities[]` stops being documentation and becomes an allowlist. The rule
+ * that does the most work is that scope is **per step, not per run**: granting an agent
+ * the union of a procedure's capabilities for the whole run is how a diagnostic session
+ * ends in a deletion.
+ *
+ * Capability extraction is imported from `@runbooks/schema`, the same code the linter's
+ * consistency gate uses. A second parser here would mean a document that passes T2 fails
+ * at run time.
+ */
+import { capabilitiesInCommand } from "@runbooks/schema";
+import { recordDeviation } from "./deviations.js";
+import { propose } from "./run.js";
+/**
+ * Whether the client can satisfy the whole allowlist before the run begins.
+ *
+ * Refusing up front rather than failing at the step that needs the missing capability
+ * is normative (§13.4): a run that stops halfway has already done part of the work, and
+ * part of a procedure is a state nobody designed.
+ */
+export function canSatisfyAllowlist(envelope, available) {
+    const have = new Set(available);
+    const missing = envelope.allowlist.filter((c) => !have.has(c));
+    if (missing.length === 0)
+        return { ok: true, missing: [] };
+    return {
+        ok: false,
+        missing,
+        message: `This runbook requires ${missing.join(", ")}, which this client cannot provide. Connect them, or pick a procedure that does not need them.`,
+    };
+}
+/**
+ * Authorize a command by what it actually invokes, not by what the step claims.
+ *
+ * A step declaring `cli:kubectl` and running `psql` is the mismatch this exists to
+ * catch, and it is the runtime half of AST03/AST04.
+ */
+export function proposeCommand(envelope, state, proposal) {
+    const extraction = capabilitiesInCommand(proposal.command);
+    if (extraction.unanalyzable) {
+        // A gate that treats "I could not tell" as "consistent" is decoration.
+        const authorization = {
+            verdict: "block",
+            reason: extraction.reason ??
+                "This command cannot be attributed to a capability, so it cannot be bounded.",
+            deviation: "undeclared-tool",
+        };
+        const events = [
+            {
+                kind: "blocked",
+                code: "run/unanalyzable-command",
+                message: authorization.reason,
+                ...(state.current ? { at: state.current } : {}),
+            },
+        ];
+        const recorded = recordDeviation(state, { class: "undeclared-tool" });
+        return {
+            state: recorded.state,
+            authorization,
+            events: [...events, ...recorded.events],
+            required: [],
+        };
+    }
+    if (extraction.capabilities.length === 0) {
+        return {
+            state,
+            authorization: { verdict: "permit", reason: "The command invokes nothing." },
+            events: [],
+            required: [],
+        };
+    }
+    // Every capability the command needs must pass. The first refusal decides, because a
+    // partially authorized command is not a narrower command.
+    let current = state;
+    const events = [];
+    for (const capability of extraction.capabilities) {
+        const decided = propose(envelope, current, { tool: capability });
+        current = decided.state;
+        events.push(...decided.events);
+        if (decided.authorization.verdict !== "permit") {
+            return {
+                state: current,
+                authorization: decided.authorization,
+                events,
+                required: extraction.capabilities,
+            };
+        }
+    }
+    return {
+        state: current,
+        authorization: {
+            verdict: "permit",
+            reason: `Step ${state.current} declares ${extraction.capabilities.join(", ")}.`,
+        },
+        events,
+        required: extraction.capabilities,
+    };
+}

package/dist/enforce.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/enforce.test.js

@@ -0,0 +1,112 @@
+import { describe, it, expect } from "vitest";
+import { readdirSync, readFileSync } from "node:fs";
+import { join, dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { derivePolicy, scopeFor, dangerousSteps } from "./policy.js";
+import { startRun, propose, applyGrant } from "./run.js";
+import { canSatisfyAllowlist, proposeCommand } from "./enforce.js";
+const CORPUS = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", "schema", "fixtures", "p1-valid");
+const corpus = readdirSync(CORPUS).map((name) => ({
+    name,
+    doc: JSON.parse(readFileSync(join(CORPUS, name), "utf8")),
+}));
+const runbook = (r) => ({ runbook: r });
+const twoStep = runbook({
+    capabilities: ["cli:kubectl", "mcp:postgres"],
+    steps: [
+        { id: "s3", kind: "check", title: "Look", risk: "read-only", tool: "cli:kubectl", expect: "ok", on_fail: "s5", next: "s5" },
+        { id: "s5", kind: "action", title: "Write", risk: "reversible-write", tool: "mcp:postgres", next: "end:success" },
+    ],
+});
+function start(doc, inputs = {}) {
+    const env = derivePolicy(doc);
+    const r = startRun(env, inputs);
+    if (!r.ok)
+        throw new Error("expected a start");
+    return { env, state: r.state };
+}
+describe("the allowlist is checked before the run begins", () => {
+    it("refuses when the client cannot provide a declared capability", () => {
+        const { env } = start(twoStep);
+        const check = canSatisfyAllowlist(env, ["cli:kubectl"]);
+        expect(check.ok).toBe(false);
+        expect(check.missing).toEqual(["mcp:postgres"]);
+        expect(check.message).toMatch(/Connect them/);
+    });
+    it("accepts a client that provides everything", () => {
+        const { env } = start(twoStep);
+        expect(canSatisfyAllowlist(env, ["cli:kubectl", "mcp:postgres", "cli:extra"]).ok).toBe(true);
+    });
+});
+describe("a tool permitted at one step is blocked at another", () => {
+    it("blocks s5's capability while the run is at s3", () => {
+        const { env, state } = start(twoStep);
+        const decided = propose(env, state, { tool: "mcp:postgres" });
+        expect(decided.authorization.verdict).toBe("block");
+        expect(decided.authorization.deviation).toBe("out-of-scope-tool");
+        expect(decided.authorization.reason).toMatch(/not the union/);
+    });
+});
+describe("commands are authorized by what they invoke", () => {
+    it("permits a command matching the step's declared tool", () => {
+        const { env, state } = start(twoStep);
+        const decided = proposeCommand(env, state, { command: "kubectl get pods" });
+        expect(decided.authorization.verdict).toBe("permit");
+        expect(decided.required).toEqual(["cli:kubectl"]);
+    });
+    // A step declaring kubectl and running psql is the runtime half of AST03/AST04.
+    it("blocks a command the step did not declare, whatever the step says", () => {
+        const { env, state } = start(twoStep);
+        const decided = proposeCommand(env, state, { command: "psql -c 'drop table x'" });
+        expect(decided.authorization.verdict).toBe("block");
+        expect(decided.authorization.deviation).toBe("undeclared-tool");
+    });
+    it("blocks a pipeline whose second stage is undeclared", () => {
+        const { env, state } = start(twoStep);
+        expect(proposeCommand(env, state, { command: "kubectl get pods | psql" }).authorization.verdict).toBe("block");
+    });
+    it("blocks rather than guessing when a command cannot be analysed", () => {
+        const { env, state } = start(twoStep);
+        const decided = proposeCommand(env, state, {
+            command: "kubectl delete pod $(kubectl get pods -o name)",
+        });
+        expect(decided.authorization.verdict).toBe("block");
+        expect(decided.events[0].code).toBe("run/unanalyzable-command");
+    });
+});
+/**
+ * R-02's property: across every corpus record and every reachable state, a destructive
+ * capability is never in scope before its gate has been passed. Checked over the state
+ * machine rather than the envelope alone, because the envelope is a description and the
+ * run is what actually happens.
+ */
+describe("no reachable state grants a destructive capability before its gate", () => {
+    /** Guards the guard: with no corpus this property holds over nothing, and passes. */
+    it("has records to check the property over", () => {
+        expect(corpus.length).toBeGreaterThan(0);
+    });
+    it.each(corpus)("holds for $name", ({ doc }) => {
+        const env = derivePolicy(doc);
+        const inputs = {};
+        for (const [name, decl] of Object.entries(env.inputs)) {
+            inputs[name] = decl.type === "boolean" ? true : decl.type === "number" ? 1 : "x";
+        }
+        const started = startRun(env, inputs);
+        expect(started.ok).toBe(true);
+        if (!started.ok)
+            return;
+        for (const scope of dangerousSteps(env)) {
+            // Ungated, the capability must not exist...
+            expect(scopeFor(env, scope.id, new Set())).toEqual([]);
+            const at = { ...started.state, current: scope.id };
+            for (const tool of scope.tools) {
+                expect(propose(env, at, { tool }).authorization.verdict).not.toBe("permit");
+            }
+            // ...and it must appear only once a decision is recorded.
+            const approved = applyGrant(at, { requestId: "r1", stepId: scope.id, decidedBy: "operator" }).state;
+            for (const tool of scope.tools) {
+                expect(propose(env, approved, { tool }).authorization.verdict).toBe("permit");
+            }
+        }
+    });
+});

package/dist/expect.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Postcondition evaluation (R-04).
+ *
+ * R2's distinguishing behaviour: `expect` is not a suggestion the model grades itself
+ * against. The supervisor decides whether the step did what the procedure said it would,
+ * and takes the `on_fail` route itself.
+ *
+ * Q12 settled how: prose is never the thing evaluated. `expect` stays prose for the
+ * reader; a step an R2 supervisor must decide carries `assert`, a small closed set of
+ * predicates over the observation. A step with only prose is **unevaluable** — a first
+ * class outcome, distinct from pass and from fail, in which the supervisor halts and
+ * asks a person rather than guessing. A supervisor that reads "I could not tell" as
+ * "passed" is worse than one that does not check at all.
+ *
+ * Nothing here interprets meaning. Every predicate is decidable by inspection, which is
+ * the property that makes a supervised claim worth more than a plausible one.
+ */
+/** What an adapter reports back after a step ran. Fields it did not capture are absent. */
+export interface Observation {
+    readonly exitCode?: number;
+    readonly stdout?: string;
+    readonly stderr?: string;
+}
+export interface Assertion {
+    readonly exit_code?: number;
+    readonly stdout_matches?: string;
+    readonly stdout_not_matches?: string;
+    readonly stderr_empty?: boolean;
+    readonly json_path?: {
+        readonly path: string;
+        readonly equals: unknown;
+    };
+}
+export type PredicateName = keyof Assertion;
+export interface PredicateResult {
+    readonly predicate: PredicateName;
+    readonly verdict: "pass" | "fail" | "unevaluable";
+    /** Why, in the terms a person needs to act: what was expected, what was seen. */
+    readonly detail: string;
+}
+export type Evaluation = {
+    readonly outcome: "pass";
+    readonly predicates: readonly PredicateResult[];
+} | {
+    readonly outcome: "fail";
+    readonly predicates: readonly PredicateResult[];
+    readonly why: string;
+} | {
+    readonly outcome: "unevaluable";
+    readonly predicates: readonly PredicateResult[];
+    readonly why: string;
+};
+/**
+ * Beyond this, a regular expression over the output is not something we are willing to
+ * run inside a decision path. Refusing to decide is the honest answer; deciding slowly
+ * enough to hang an incident is not.
+ */
+export declare const MAX_OUTPUT_BYTES = 1048576;
+/**
+ * Evaluate a step's postcondition.
+ *
+ * Every predicate is evaluated rather than short-circuited, so a person sees everything
+ * that was wrong instead of the first thing. A definite failure outranks an undecided
+ * predicate — the step demonstrably did not do what it said — but nothing undecided ever
+ * becomes a pass.
+ */
+export declare function evaluate(assertion: Assertion | undefined, observation: Observation): Evaluation;