@runbooks/supervise 0.1.0

package/dist/deviations.test.js

@@ -0,0 +1,241 @@
+import { describe, it, expect } from "vitest";
+import { loadCorpus } from "@runbooks/fixtures";
+import { derivePolicy } from "./policy.js";
+import { startRun, propose, advance, observe, conclude } from "./run.js";
+import { proposeCommand } from "./enforce.js";
+import { DEFAULT_ESCALATION, capabilityClassOf, countByClass, countByStep, recordDeviation, } from "./deviations.js";
+import { report } from "./report.js";
+function envelopeFor(name) {
+    const record = loadCorpus().find((r) => r.name.startsWith(name));
+    return derivePolicy(record.doc);
+}
+function begin(envelope, options = {}) {
+    const started = startRun(envelope, { agentIdentity: "agent-7", ...options });
+    if (!started.ok)
+        throw new Error(JSON.stringify(started.refusals));
+    return started.state;
+}
+const disk = () => envelopeFor("disk-pressure");
+const k8s = () => envelopeFor("k8s-node");
+const flaky = () => envelopeFor("flaky-check");
+const permissive = {
+    responses: Object.fromEntries(Object.keys(DEFAULT_ESCALATION.responses).map((c) => [c, "block-and-continue"])),
+    maxTotal: 99,
+};
+/**
+ * Every class in the taxonomy, each produced by a run doing the thing that class names.
+ * A taxonomy with a member nothing can produce is a taxonomy that has drifted from the
+ * supervisor it describes.
+ */
+describe("every deviation class has a fixture that produces exactly it", () => {
+    const produced = {
+        "undeclared-tool": () => {
+            const state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+            return propose(disk(), state, { tool: "cli:nothing-declares-this" })
+                .state.deviations.map((d) => d.class);
+        },
+        "out-of-scope-tool": () => {
+            const envelope = disk();
+            const state = begin(envelope, { inputs: { mount: "/tmp" }, escalation: permissive });
+            const later = Object.values(envelope.steps)
+                .flatMap((s) => s.tools)
+                .find((t) => !envelope.steps[state.current].tools.includes(t));
+            return propose(envelope, state, { tool: later }).state.deviations.map((d) => d.class);
+        },
+        "out-of-order-step": () => {
+            const state = begin(k8s(), { inputs: { node: "node-1" }, escalation: permissive });
+            return advance(k8s(), state, "s5").state.deviations.map((d) => d.class);
+        },
+        "unchecked-advance": () => {
+            const state = begin(k8s(), {
+                inputs: { node: "node-1" }, escalation: permissive, evaluatePostconditions: true,
+            });
+            return advance(k8s(), state, "s2").state.deviations.map((d) => d.class);
+        },
+        "budget-exhausted": () => {
+            const envelope = flaky();
+            const [from, to] = Object.keys(envelope.budgets)[0].split("->");
+            let state = begin(envelope, { escalation: permissive });
+            for (let i = 0; i < 20 && state.status === "running"; i++) {
+                state = advance(envelope, state, state.current === to ? from : to).state;
+            }
+            return state.deviations.map((d) => d.class);
+        },
+        "unrouted-failure": () => {
+            const envelope = disk();
+            const { onFail: _none, ...unrouted } = envelope.steps[envelope.entry];
+            const patched = {
+                ...envelope,
+                steps: { ...envelope.steps, [envelope.entry]: { ...unrouted, assert: { exit_code: 0 } } },
+            };
+            const state = begin(patched, { inputs: { mount: "/tmp" }, escalation: permissive });
+            return observe(patched, state, { exitCode: 1 }).state.deviations.map((d) => d.class);
+        },
+        "terminal-not-reached": () => {
+            const state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+            return conclude(disk(), state).state.deviations.map((d) => d.class);
+        },
+    };
+    it("covers the taxonomy with no member left over", () => {
+        expect(Object.keys(produced).sort()).toEqual(Object.keys(DEFAULT_ESCALATION.responses).sort());
+    });
+    it.each(Object.keys(produced))("%s", (cls) => {
+        expect(produced[cls]()).toEqual([cls]);
+    });
+});
+/**
+ * §14's rule applies to everything that might leave the operator's environment, and a
+ * deviation record is exactly that. The class, the step, the capability class — never
+ * the payload.
+ */
+describe("a deviation record carries no payload", () => {
+    const secrets = [
+        "cli:psql --password=hunter2",
+        "cli:curl https://api.example.com?token=sk-live-4242424242",
+        "cli:kubectl --token=eyJhbGciOiJIUzI1NiJ9.secret",
+        "cli:aws --secret-access-key wJalrXUtnFEMI/K7MDENG",
+    ];
+    it.each(secrets)("keeps nothing of %s but its class", (tool) => {
+        const state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+        const decided = propose(disk(), state, { tool });
+        const serialized = JSON.stringify(decided.state.deviations);
+        expect(serialized).not.toMatch(/hunter2|sk-live|eyJ|wJalr/);
+        expect(serialized).not.toContain(tool);
+        expect(decided.state.deviations[0].capabilityClass).toBe("cli");
+    });
+    it("keeps nothing of a command that could not be attributed", () => {
+        const state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+        const decided = proposeCommand(disk(), state, {
+            command: "df -h $(cat /etc/shadow) --password=hunter2",
+        });
+        expect(JSON.stringify(decided.state.deviations)).not.toMatch(/shadow|hunter2|df/);
+    });
+    it("reduces a capability to its class and nothing finer", () => {
+        expect(capabilityClassOf("cli:kubectl")).toBe("cli");
+        expect(capabilityClassOf("mcp:github/create_issue")).toBe("mcp");
+        expect(capabilityClassOf("Bearer sk-live-1")).toBeUndefined();
+        expect(capabilityClassOf(undefined)).toBeUndefined();
+    });
+    it("orders records without reading a clock", () => {
+        let state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+        state = propose(disk(), state, { tool: "cli:a" }).state;
+        state = propose(disk(), state, { tool: "cli:b" }).state;
+        expect(state.deviations.map((d) => d.sequence)).toEqual([0, 1]);
+        expect(JSON.stringify(state.deviations)).not.toMatch(/\d{10}/);
+    });
+});
+/** A refusal that a second attempt turns into a permission is not a refusal. */
+describe("a blocked deviation never becomes a permitted call", () => {
+    it("blocks the same call however many times it is proposed", () => {
+        let state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+        for (let attempt = 0; attempt < 5; attempt++) {
+            const decided = propose(disk(), state, { tool: "cli:nothing-declares-this" });
+            expect(decided.authorization.verdict).toBe("block");
+            state = decided.state;
+        }
+        expect(state.deviations).toHaveLength(5);
+    });
+    it("does not let a blocked tool arrive through the command path either", () => {
+        const state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+        const decided = proposeCommand(disk(), state, { command: "rm -rf /var/lib/postgresql" });
+        expect(decided.authorization.verdict).toBe("block");
+    });
+});
+/**
+ * Escalation is the operator's policy with a conservative default: anything that breaks
+ * control flow stops the run, because every later authorization would otherwise be
+ * decided against the wrong step.
+ */
+describe("escalation policy", () => {
+    it("aborts the run on a control-flow deviation by default", () => {
+        const state = begin(k8s(), { inputs: { node: "node-1" } });
+        const moved = advance(k8s(), state, "s5");
+        expect(moved.state.status).toBe("ended");
+        expect(moved.events.some((e) => e.kind === "escalated")).toBe(true);
+    });
+    it("blocks and continues on a tool the runbook declares for another step", () => {
+        const envelope = disk();
+        const state = begin(envelope, { inputs: { mount: "/tmp" } });
+        const later = Object.values(envelope.steps)
+            .flatMap((s) => s.tools)
+            .find((t) => !envelope.steps[state.current].tools.includes(t));
+        const decided = propose(envelope, state, { tool: later });
+        expect(decided.authorization.verdict).toBe("block");
+        expect(decided.state.status).toBe("running");
+    });
+    it("stops once a run has accumulated more deviations than the operator allows", () => {
+        const envelope = disk();
+        const later = Object.values(envelope.steps).flatMap((s) => s.tools).find((t) => !envelope.steps[envelope.entry].tools.includes(t));
+        let state = begin(envelope, { inputs: { mount: "/tmp" } });
+        for (let i = 0; i <= DEFAULT_ESCALATION.maxTotal; i++) {
+            state = propose(envelope, state, { tool: later }).state;
+        }
+        expect(state.status).toBe("ended");
+        expect(state.deviations.length).toBe(DEFAULT_ESCALATION.maxTotal + 1);
+    });
+    it("lets an operator say otherwise", () => {
+        const state = begin(k8s(), { inputs: { node: "node-1" }, escalation: permissive });
+        expect(advance(k8s(), state, "s5").state.status).toBe("running");
+    });
+    it("names what stopped the run without naming what was being attempted", () => {
+        const state = begin(k8s(), { inputs: { node: "node-1" } });
+        const escalated = advance(k8s(), state, "s5").events.find((e) => e.kind === "escalated");
+        expect(escalated.message).toMatch(/out-of-order-step/);
+        expect(escalated.at).toBe("s1");
+    });
+});
+/** An escalation the runbook anticipated is where the procedure meant to end. */
+describe("an anticipated escalation is not a deviation", () => {
+    it("records nothing when a failed check routes to the escalate node the author wrote", () => {
+        const state = begin(k8s(), { inputs: { node: "node-1" }, evaluatePostconditions: true });
+        const observed = observe(k8s(), state, { exitCode: 0, stdout: "True" });
+        expect(observed.state.current).toBe("s9");
+        expect(observed.state.deviations).toEqual([]);
+    });
+    it("concludes a run sitting on that escalate node as ended, not as incomplete", () => {
+        const state = begin(k8s(), { inputs: { node: "node-1" }, evaluatePostconditions: true });
+        const escalated = observe(k8s(), state, { exitCode: 0, stdout: "True" }).state;
+        // The escalate node is a terminal, so the run already ended there — handing off to a
+        // person is where this procedure meant to go, and it cost no deviation.
+        expect(escalated.status).toBe("ended");
+        const concluded = conclude(k8s(), escalated);
+        expect(concluded.state.deviations).toEqual([]);
+        expect(report(concluded.state).complete).toBe(true);
+    });
+    it("records terminal-not-reached only when the run stopped somewhere else", () => {
+        const state = begin(k8s(), { inputs: { node: "node-1" } });
+        expect(conclude(k8s(), state).state.deviations.map((d) => d.class))
+            .toEqual(["terminal-not-reached"]);
+    });
+});
+/**
+ * The diagnostic half. A procedure that constantly provokes deviations is a procedure
+ * whose graph does not match reality, and an author needs that per step.
+ */
+describe("the aggregate is what an author reads", () => {
+    it("counts by class and by step", () => {
+        const records = [
+            { class: "out-of-scope-tool", stepId: "s4", sequence: 0 },
+            { class: "out-of-scope-tool", stepId: "s4", sequence: 1 },
+            { class: "undeclared-tool", stepId: "s1", sequence: 2 },
+        ];
+        expect(countByClass(records)).toEqual({ "out-of-scope-tool": 2, "undeclared-tool": 1 });
+        expect(countByStep(records)).toEqual({ s4: 2, s1: 1 });
+    });
+    it("surfaces the count and both breakdowns in the run report", () => {
+        let state = begin(disk(), { inputs: { mount: "/tmp" }, escalation: permissive });
+        state = propose(disk(), state, { tool: "cli:nope" }).state;
+        state = propose(disk(), state, { tool: "cli:nope" }).state;
+        const r = report(state);
+        expect(r.deviations.count).toBe(2);
+        expect(r.deviations.byClass["undeclared-tool"]).toBe(2);
+        expect(Object.values(r.deviations.byStep)[0]).toBe(2);
+        expect(r.caveats.join(" ")).toMatch(/graph may not match reality/);
+    });
+    it("records a deviation even once the run has ended, without resurrecting it", () => {
+        const ended = { ...begin(disk(), { inputs: { mount: "/tmp" } }), status: "ended" };
+        const after = recordDeviation(ended, { class: "out-of-order-step" });
+        expect(after.state.status).toBe("ended");
+        expect(after.state.deviations).toHaveLength(1);
+    });
+});

package/dist/emit.d.ts

@@ -0,0 +1,97 @@
+/**
+ * The §14 run report, assembled (R-07).
+ *
+ * Not to be confused with `./report.js`, which is the operator's own account of a run
+ * and stays on their machine. This is the thing that might leave, and everything about
+ * it is shaped by that: a closed vocabulary, a bucketed duration, and no field into
+ * which a message could be written.
+ *
+ * Assembly is pure, so it can be reasoned about; validation against the published schema
+ * happens at the emitting edge, where the filesystem lives.
+ */
+import type { RunEvent, RunState } from "./run.js";
+export type Outcome = "success" | "failed" | "aborted" | "partial";
+export type FailureClass = "precondition_unmet" | "tool_missing" | "permission_denied" | "expectation_mismatch" | "timeout" | "human_abort" | "upstream_changed";
+export type ExecutorKind = "human" | "agent-assisted" | "autonomous";
+export type DurationBucket = "<1m" | "1-5m" | "5-30m" | ">30m";
+export type EnvClass = "local" | "ci" | "staging" | "prod";
+export interface RunReportV1 {
+    readonly report_schema: "runbook-run-report/v1";
+    readonly schema_version: "v1";
+    readonly runbook: string;
+    readonly version: string;
+    readonly content_hash: string;
+    readonly profile: "P0" | "P1";
+    readonly outcome: Outcome;
+    readonly failed_step_id?: string;
+    readonly failure_class?: FailureClass;
+    readonly executor_kind: ExecutorKind;
+    readonly runtime_profile: "R0" | "R1" | "R2";
+    readonly deviation_count: number;
+    readonly duration_bucket: DurationBucket;
+    readonly env_class: EnvClass;
+    readonly reported_at: string;
+}
+/**
+ * Which §14 class an internal reason belongs to.
+ *
+ * `null` means "this reason is not a failure": the run was stopped, or it ended cleanly,
+ * and inventing a failure class for it would put a cause into an aggregate for runs that
+ * did not fail. Control-flow deviations are the interesting case — a run that went off
+ * its graph did not fail at a step, it was halted, and §14's vocabulary has no member for
+ * "the run left the procedure" because that is not a fact about the procedure.
+ */
+export declare const FAILURE_CLASS_BY_CODE: Readonly<Record<string, FailureClass | null>>;
+export declare class UnclassifiableFailureError extends Error {
+    constructor(stepId: string | undefined);
+}
+export declare class UnmappedReasonError extends Error {
+    constructor(code: string);
+}
+/** Loudly, per R-07: an unmapped reason throws rather than picking something adjacent. */
+export declare function failureClassFor(code: string): FailureClass | null;
+/** Bucketed, never exact (§14). An unknown duration is not a short one. */
+export declare function durationBucket(seconds: number | undefined): DurationBucket;
+export interface ReportContext {
+    /** publisher/slug, from the record that ran. */
+    readonly runbook: string;
+    readonly version: string;
+    readonly contentHash: string;
+    readonly profile: "P0" | "P1";
+    readonly runtimeProfile: "R0" | "R1" | "R2";
+    /**
+     * Both of these are the operator's to state and the supervisor cannot infer either.
+     * There is no default: guessing `local` for a run that was in production would put a
+     * prod failure into the wrong aggregate, and an aggregate is the whole point.
+     */
+    readonly executorKind: ExecutorKind;
+    readonly envClass: EnvClass;
+    /** The day, not the moment (§14). */
+    readonly reportedAt: string;
+    /** For the duration bucket. The core has no clock. */
+    readonly now?: number;
+    /**
+     * The operator's own classification, for a failure the run did not explain.
+     *
+     * A runbook that routes to `end:failed` has declared a failing path without saying
+     * why, and at R0 or R1 there may be no recorded cause at all. Rather than file it
+     * under something adjacent, assembly refuses and asks — the person who was there
+     * knows, and a wrong class is not a smaller error than a missing report.
+     */
+    readonly failureClass?: FailureClass;
+}
+/**
+ * Assemble the report from what the run actually recorded.
+ *
+ * Every value here is derived from run state or supplied by the operator. Nothing is
+ * copied out of a message, because there is nowhere in the format for a message to go.
+ */
+export declare function buildRunReport(state: RunState, events: readonly RunEvent[], context: ReportContext): RunReportV1;
+/**
+ * How a run ended, in §14's four words.
+ *
+ * Exported because a run attestation (R-09) states the same fact and must state it the
+ * same way: a report saying `failed` and an attestation saying `aborted` about one run
+ * would be two claims about the same thing, and whichever a reader saw first would win.
+ */
+export declare function outcomeOf(state: RunState): Outcome;

package/dist/emit.js

@@ -0,0 +1,184 @@
+import { SCHEMA_VERSION } from "@runbooks/schema";
+/**
+ * Which §14 class an internal reason belongs to.
+ *
+ * `null` means "this reason is not a failure": the run was stopped, or it ended cleanly,
+ * and inventing a failure class for it would put a cause into an aggregate for runs that
+ * did not fail. Control-flow deviations are the interesting case — a run that went off
+ * its graph did not fail at a step, it was halted, and §14's vocabulary has no member for
+ * "the run left the procedure" because that is not a fact about the procedure.
+ */
+export const FAILURE_CLASS_BY_CODE = {
+    // Nothing went wrong.
+    "run/started": null,
+    "run/entered": null,
+    "run/permitted": null,
+    "run/authorized": null,
+    "run/approved": null,
+    "run/adjudicated": null,
+    "run/ended": null,
+    "run/expectation-passed": null,
+    "run/approval-required": null,
+    "run/nothing-to-adjudicate": null,
+    // The procedure did not do what it said it would.
+    "run/expectation-failed": "expectation_mismatch",
+    "run/expectation-unevaluable": "expectation_mismatch",
+    "run/unrouted-failure": "expectation_mismatch",
+    // The supervisor refused.
+    "run/undeclared-tool": "permission_denied",
+    "run/out-of-scope-tool": "permission_denied",
+    "run/unanalyzable-command": "permission_denied",
+    "runtime/coverage-insufficient": "permission_denied",
+    "runtime/risk-above-ceiling": "permission_denied",
+    "runtime/profile-too-low": "permission_denied",
+    "runtime/autonomous-unsupervised": "permission_denied",
+    // The environment could not supply what the procedure needs.
+    "runtime/capability-unavailable": "tool_missing",
+    // The run was not set up to start.
+    "run/input-missing": "precondition_unmet",
+    "run/input-type": "precondition_unmet",
+    "run/input-undeclared": "precondition_unmet",
+    "run/no-entry": "precondition_unmet",
+    /**
+     * A budget is a statement about time the operator was willing to spend. A loop that
+     * exhausted its retries did not settle inside the allowance it was given, which is the
+     * same fact as a wall clock running out, told from the other end.
+     */
+    "run/budget-exhausted": "timeout",
+    "run/wall-clock-exhausted": "timeout",
+    "run/deviation-limit": null,
+    // A person stopped it.
+    "run/aborted-by-human": "human_abort",
+    "run/approval-denied": "human_abort",
+    // What ran is not what was published.
+    "run/content-hash-mismatch": "upstream_changed",
+    // Control flow: halted, not failed.
+    "run/out-of-order-step": null,
+    "run/unchecked-advance": null,
+    "run/terminal-not-reached": null,
+    "run/after-end": null,
+    "run/escalated-undeclared-tool": null,
+    "run/escalated-out-of-scope-tool": null,
+    "run/escalated-out-of-order-step": null,
+    "run/escalated-unchecked-advance": null,
+    "run/escalated-budget-exhausted": null,
+    "run/escalated-unrouted-failure": null,
+    "run/escalated-terminal-not-reached": null,
+};
+export class UnclassifiableFailureError extends Error {
+    constructor(stepId) {
+        super(`This run failed${stepId ? ` at ${stepId}` : ""} and recorded no cause a §14 class ` +
+            `covers. Supply failureClass explicitly: the person who was there knows, and a ` +
+            `wrong class is not a smaller error than a missing report.`);
+        this.name = "UnclassifiableFailureError";
+    }
+}
+export class UnmappedReasonError extends Error {
+    constructor(code) {
+        super(`${code} has no §14 failure class. Refusing to emit rather than choosing a bucket: ` +
+            `a report that files an unknown cause under a plausible heading is worse than no ` +
+            `report, because the aggregate then reads as evidence.`);
+        this.name = "UnmappedReasonError";
+    }
+}
+/** Loudly, per R-07: an unmapped reason throws rather than picking something adjacent. */
+export function failureClassFor(code) {
+    const known = FAILURE_CLASS_BY_CODE[code];
+    if (known === undefined)
+        throw new UnmappedReasonError(code);
+    return known;
+}
+/** Bucketed, never exact (§14). An unknown duration is not a short one. */
+export function durationBucket(seconds) {
+    if (seconds === undefined || !Number.isFinite(seconds))
+        return ">30m";
+    if (seconds < 60)
+        return "<1m";
+    if (seconds < 300)
+        return "1-5m";
+    if (seconds < 1800)
+        return "5-30m";
+    return ">30m";
+}
+/**
+ * Assemble the report from what the run actually recorded.
+ *
+ * Every value here is derived from run state or supplied by the operator. Nothing is
+ * copied out of a message, because there is nowhere in the format for a message to go.
+ */
+export function buildRunReport(state, events, context) {
+    const outcome = outcomeOf(state);
+    const derived = failureOf(state, events, outcome);
+    const failure = derived && context.failureClass
+        ? { ...derived, failureClass: context.failureClass }
+        : derived;
+    if (outcome === "failed" && !failure?.failureClass) {
+        throw new UnclassifiableFailureError(failure?.stepId);
+    }
+    const elapsed = context.now !== undefined && state.startedAt !== undefined
+        ? context.now - state.startedAt
+        : undefined;
+    return {
+        report_schema: "runbook-run-report/v1",
+        schema_version: SCHEMA_VERSION,
+        runbook: context.runbook,
+        version: context.version,
+        content_hash: context.contentHash,
+        profile: context.profile,
+        outcome,
+        ...(failure?.stepId ? { failed_step_id: failure.stepId } : {}),
+        ...(failure?.failureClass ? { failure_class: failure.failureClass } : {}),
+        executor_kind: context.executorKind,
+        runtime_profile: context.runtimeProfile,
+        deviation_count: state.deviations.length,
+        duration_bucket: durationBucket(elapsed),
+        env_class: context.envClass,
+        reported_at: context.reportedAt,
+    };
+}
+/**
+ * How a run ended, in §14's four words.
+ *
+ * Exported because a run attestation (R-09) states the same fact and must state it the
+ * same way: a report saying `failed` and an attestation saying `aborted` about one run
+ * would be two claims about the same thing, and whichever a reader saw first would win.
+ */
+export function outcomeOf(state) {
+    if (state.status !== "ended")
+        return "partial";
+    switch (state.outcome) {
+        case "success":
+            return "success";
+        case "failed":
+            return "failed";
+        case "timed-out":
+        case "budget-exhausted":
+            return "failed";
+        default:
+            return "aborted";
+    }
+}
+/**
+ * What failed and why, taken from the last event that carries a cause.
+ *
+ * The last, not the first: a run that recovered from one failure and then hit another
+ * failed at the second, and reporting the first would point an author at the step that
+ * worked.
+ */
+function failureOf(state, events, outcome) {
+    if (outcome === "success")
+        return undefined;
+    for (let i = events.length - 1; i >= 0; i--) {
+        const event = events[i];
+        const failureClass = failureClassFor(event.code);
+        if (failureClass === null)
+            continue;
+        return { ...(event.at ? { stepId: event.at } : {}), failureClass };
+    }
+    // A run stopped without a cause anyone recorded: where it stopped, and nothing more.
+    // Not for a run still going — a step it happens to be sitting on has not failed, and
+    // reporting it as the failed step would point an author at a step that was working.
+    if (outcome === "partial" || !state.current)
+        return undefined;
+    return { stepId: state.current };
+}

package/dist/emit.test.d.ts

@@ -0,0 +1 @@
+export {};