@runbooks/supervise 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +47 -0
- package/dist/approval.d.ts +132 -0
- package/dist/approval.js +148 -0
- package/dist/approval.test.d.ts +1 -0
- package/dist/approval.test.js +190 -0
- package/dist/budgets.test.d.ts +1 -0
- package/dist/budgets.test.js +148 -0
- package/dist/contract.d.ts +38 -0
- package/dist/contract.js +77 -0
- package/dist/contract.test.d.ts +1 -0
- package/dist/contract.test.js +131 -0
- package/dist/deviations.d.ts +77 -0
- package/dist/deviations.js +81 -0
- package/dist/deviations.test.d.ts +1 -0
- package/dist/deviations.test.js +241 -0
- package/dist/emit.d.ts +97 -0
- package/dist/emit.js +184 -0
- package/dist/emit.test.d.ts +1 -0
- package/dist/emit.test.js +187 -0
- package/dist/enforce.d.ts +29 -0
- package/dist/enforce.js +100 -0
- package/dist/enforce.test.d.ts +1 -0
- package/dist/enforce.test.js +112 -0
- package/dist/expect.d.ts +67 -0
- package/dist/expect.js +187 -0
- package/dist/expect.test.d.ts +1 -0
- package/dist/expect.test.js +120 -0
- package/dist/index.d.ts +33 -0
- package/dist/index.js +34 -0
- package/dist/index.test.d.ts +1 -0
- package/dist/index.test.js +30 -0
- package/dist/observe.test.d.ts +1 -0
- package/dist/observe.test.js +246 -0
- package/dist/policy.d.ts +122 -0
- package/dist/policy.js +147 -0
- package/dist/policy.test.d.ts +1 -0
- package/dist/policy.test.js +124 -0
- package/dist/purity.test.d.ts +1 -0
- package/dist/purity.test.js +191 -0
- package/dist/report.d.ts +72 -0
- package/dist/report.js +67 -0
- package/dist/run.d.ts +216 -0
- package/dist/run.js +445 -0
- package/dist/run.test.d.ts +1 -0
- package/dist/run.test.js +198 -0
- package/package.json +42 -0
package/dist/policy.d.ts
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The policy envelope: what a runbook authorizes at run time.
|
|
3
|
+
*
|
|
4
|
+
* Derived from the document, never authored separately (RUNBOOK.md 13.2). That is the
|
|
5
|
+
* load-bearing decision of this file: a policy that can drift from the procedure it
|
|
6
|
+
* governs is worse than no policy, because it is trusted while being wrong. An author
|
|
7
|
+
* who writes a valid P1 runbook has already written the policy.
|
|
8
|
+
*
|
|
9
|
+
* Derivation is a pure function. Two implementations must produce the same envelope for
|
|
10
|
+
* the same document, or the contract is not portable and "any client can enforce this"
|
|
11
|
+
* becomes "our client can enforce this".
|
|
12
|
+
*/
|
|
13
|
+
import type { Risk, RuntimeProfile } from "@runbooks/schema";
|
|
14
|
+
import { type RunbookDocument } from "@runbooks/graph";
|
|
15
|
+
import { type Execution } from "./contract.js";
|
|
16
|
+
import type { DeviationClass } from "./index.js";
|
|
17
|
+
import type { Assertion } from "./expect.js";
|
|
18
|
+
export declare const ENVELOPE_SCHEMA: "runbook-policy-envelope/v1";
|
|
19
|
+
export interface StepScope {
|
|
20
|
+
readonly id: string;
|
|
21
|
+
readonly kind: string;
|
|
22
|
+
readonly risk: Risk;
|
|
23
|
+
/**
|
|
24
|
+
* What this step alone may invoke. Not the union of the runbook: granting an agent
|
|
25
|
+
* every capability a procedure will eventually need is how a diagnostic session ends
|
|
26
|
+
* in a deletion.
|
|
27
|
+
*/
|
|
28
|
+
readonly tools: readonly string[];
|
|
29
|
+
/** Arrival at this step passes an approval gate. */
|
|
30
|
+
readonly gated: boolean;
|
|
31
|
+
/** Steps this step may hand control to. */
|
|
32
|
+
readonly transitions: readonly string[];
|
|
33
|
+
/**
|
|
34
|
+
* Tools the step names that the manifest does not declare. Never granted - the
|
|
35
|
+
* envelope carries no more authority than the document - and reported so the linter
|
|
36
|
+
* can fail the record (P-06).
|
|
37
|
+
*/
|
|
38
|
+
readonly undeclared: readonly string[];
|
|
39
|
+
/** The command template, for rendering an approval request. */
|
|
40
|
+
readonly command?: string;
|
|
41
|
+
readonly title: string;
|
|
42
|
+
readonly rollbackTo?: string;
|
|
43
|
+
/**
|
|
44
|
+
* The machine-checkable postcondition, if the step has one. Its absence is what makes
|
|
45
|
+
* a step unevaluable at R2 (Q12); the envelope carries the absence rather than
|
|
46
|
+
* inventing a predicate from the prose.
|
|
47
|
+
*/
|
|
48
|
+
readonly assert?: Assertion;
|
|
49
|
+
/** Where a failed postcondition sends the run. Chosen by the document, not the agent. */
|
|
50
|
+
readonly onFail?: string;
|
|
51
|
+
/** Where a passed postcondition sends the run, when the graph leaves no choice. */
|
|
52
|
+
readonly onPass?: string;
|
|
53
|
+
}
|
|
54
|
+
export interface InputDeclaration {
|
|
55
|
+
readonly type: "string" | "number" | "boolean";
|
|
56
|
+
readonly required: boolean;
|
|
57
|
+
readonly default?: unknown;
|
|
58
|
+
/** Redacted wherever the run is shown to a human, an approval request above all. */
|
|
59
|
+
readonly secret: boolean;
|
|
60
|
+
}
|
|
61
|
+
export interface PolicyEnvelope {
|
|
62
|
+
readonly schema: typeof ENVELOPE_SCHEMA;
|
|
63
|
+
readonly allowlist: readonly string[];
|
|
64
|
+
/** Declared parameters. Bound and validated before the graph is entered, never
|
|
65
|
+
* lazily at first use: interpolating an empty string is how a scoped operation
|
|
66
|
+
* silently becomes an unscoped one. */
|
|
67
|
+
readonly inputs: Readonly<Record<string, InputDeclaration>>;
|
|
68
|
+
readonly maxRisk: Risk;
|
|
69
|
+
readonly minRuntimeProfile: RuntimeProfile;
|
|
70
|
+
readonly entry: string | undefined;
|
|
71
|
+
readonly terminals: readonly string[];
|
|
72
|
+
readonly steps: Readonly<Record<string, StepScope>>;
|
|
73
|
+
/** Retry limits, keyed `from->to`. Absent means the transition is not a loop. */
|
|
74
|
+
readonly budgets: Readonly<Record<string, number>>;
|
|
75
|
+
/**
|
|
76
|
+
* What the document's `duration` suggests a whole-run limit could be, in seconds.
|
|
77
|
+
*
|
|
78
|
+
* A suggestion and nothing more. `duration` is a coarse bucket an author wrote to set
|
|
79
|
+
* a reader's expectation, and enforcing it would turn a hint into a kill switch nobody
|
|
80
|
+
* agreed to. `>30m` has no upper bound at all, so it suggests nothing. Whether to set
|
|
81
|
+
* a budget, and what it should be, is the operator's decision.
|
|
82
|
+
*/
|
|
83
|
+
readonly suggestedWallClockSeconds?: number;
|
|
84
|
+
}
|
|
85
|
+
interface Doc extends RunbookDocument {
|
|
86
|
+
runbook?: RunbookDocument["runbook"] & {
|
|
87
|
+
capabilities?: string[];
|
|
88
|
+
execution?: Execution;
|
|
89
|
+
min_runtime_profile?: RuntimeProfile;
|
|
90
|
+
duration?: string;
|
|
91
|
+
inputs?: Record<string, {
|
|
92
|
+
type?: string;
|
|
93
|
+
required?: boolean;
|
|
94
|
+
default?: unknown;
|
|
95
|
+
secret?: boolean;
|
|
96
|
+
}>;
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
export declare function derivePolicy(doc: Doc): PolicyEnvelope;
|
|
100
|
+
/**
|
|
101
|
+
* What may be invoked right now.
|
|
102
|
+
*
|
|
103
|
+
* The gate is the whole point: a gated step's capabilities do not exist until a human
|
|
104
|
+
* has decided. So the destructive capability is in scope only while the destructive step
|
|
105
|
+
* is current *and* its gate has been passed - never before, and never after the step
|
|
106
|
+
* hands control on.
|
|
107
|
+
*/
|
|
108
|
+
export declare function scopeFor(envelope: PolicyEnvelope, stepId: string, approved?: ReadonlySet<string>): readonly string[];
|
|
109
|
+
export interface AuthorizationRequest {
|
|
110
|
+
readonly stepId: string;
|
|
111
|
+
readonly tool: string;
|
|
112
|
+
readonly approved?: ReadonlySet<string>;
|
|
113
|
+
}
|
|
114
|
+
export interface Authorization {
|
|
115
|
+
readonly verdict: "permit" | "block" | "require-approval";
|
|
116
|
+
readonly reason: string;
|
|
117
|
+
readonly deviation?: DeviationClass;
|
|
118
|
+
}
|
|
119
|
+
export declare function authorize(envelope: PolicyEnvelope, req: AuthorizationRequest): Authorization;
|
|
120
|
+
/** Steps whose risk forces an approval gate (RUNBOOK.md 12). */
|
|
121
|
+
export declare function dangerousSteps(envelope: PolicyEnvelope): readonly StepScope[];
|
|
122
|
+
export {};
|
package/dist/policy.js
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { isDangerous, RISK_ORDER } from "@runbooks/schema";
|
|
2
|
+
import { toGraph } from "@runbooks/graph";
|
|
3
|
+
import { impliedMinimum } from "./contract.js";
|
|
4
|
+
export const ENVELOPE_SCHEMA = "runbook-policy-envelope/v1";
|
|
5
|
+
function highestRisk(risks) {
|
|
6
|
+
let worst = "read-only";
|
|
7
|
+
for (const r of risks) {
|
|
8
|
+
if (r && RISK_ORDER.indexOf(r) > RISK_ORDER.indexOf(worst))
|
|
9
|
+
worst = r;
|
|
10
|
+
}
|
|
11
|
+
return worst;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* `document -> envelope`. Deterministic: keys are emitted in document order, and
|
|
15
|
+
* nothing sorts by anything else.
|
|
16
|
+
*/
|
|
17
|
+
/** The document's own failure route, resolved to a node id. */
|
|
18
|
+
function onFailTarget(graph, id) {
|
|
19
|
+
return graph.edges.find((e) => e.from === id && e.kind === "on_fail")?.to;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The success route, but only where the graph leaves exactly one.
|
|
23
|
+
*
|
|
24
|
+
* A branching step has no single successor, and a supervisor that picked one would be
|
|
25
|
+
* making the author's decision for them. Where there is a choice, the run stays put and
|
|
26
|
+
* the choice is made where it belongs.
|
|
27
|
+
*/
|
|
28
|
+
function onPassTarget(graph, id) {
|
|
29
|
+
const forward = graph.edges.filter((e) => e.from === id && e.kind === "next");
|
|
30
|
+
return forward.length === 1 ? forward[0].to : undefined;
|
|
31
|
+
}
|
|
32
|
+
/** The upper edge of each duration bucket. `>30m` is open, so it suggests nothing. */
|
|
33
|
+
const SUGGESTED_WALL_CLOCK = {
|
|
34
|
+
"<5m": 300,
|
|
35
|
+
"5-30m": 1800,
|
|
36
|
+
};
|
|
37
|
+
export function derivePolicy(doc) {
|
|
38
|
+
const rb = doc.runbook;
|
|
39
|
+
const allowlist = [...(rb?.capabilities ?? [])];
|
|
40
|
+
const declared = new Set(allowlist);
|
|
41
|
+
const graph = toGraph(doc);
|
|
42
|
+
const steps = rb?.steps ?? [];
|
|
43
|
+
const gatedIds = new Set(graph.edges.filter((e) => e.kind === "approval").map((e) => e.to));
|
|
44
|
+
const scopes = {};
|
|
45
|
+
for (const step of steps) {
|
|
46
|
+
const named = step.tool ? [step.tool] : [];
|
|
47
|
+
scopes[step.id] = {
|
|
48
|
+
id: step.id,
|
|
49
|
+
kind: step.kind,
|
|
50
|
+
risk: step.risk ?? "read-only",
|
|
51
|
+
tools: named.filter((t) => declared.has(t)),
|
|
52
|
+
undeclared: named.filter((t) => !declared.has(t)),
|
|
53
|
+
gated: gatedIds.has(step.id),
|
|
54
|
+
title: step.title,
|
|
55
|
+
...(step.command ? { command: step.command } : {}),
|
|
56
|
+
...(step.rollback_ref ? { rollbackTo: step.rollback_ref } : {}),
|
|
57
|
+
transitions: graph.edges
|
|
58
|
+
.filter((e) => e.from === step.id && e.kind !== "approval")
|
|
59
|
+
.map((e) => e.to),
|
|
60
|
+
...(step.assert ? { assert: step.assert } : {}),
|
|
61
|
+
...(onFailTarget(graph, step.id) ? { onFail: onFailTarget(graph, step.id) } : {}),
|
|
62
|
+
...(onPassTarget(graph, step.id) ? { onPass: onPassTarget(graph, step.id) } : {}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
const budgets = {};
|
|
66
|
+
for (const e of graph.edges) {
|
|
67
|
+
if (e.kind === "retry" && typeof e.max === "number") {
|
|
68
|
+
budgets[`${e.from}->${e.to}`] = e.max;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const maxRisk = highestRisk(steps.map((s) => s.risk));
|
|
72
|
+
const inputs = {};
|
|
73
|
+
for (const [name, decl] of Object.entries(rb?.inputs ?? {})) {
|
|
74
|
+
inputs[name] = {
|
|
75
|
+
type: decl.type ?? "string",
|
|
76
|
+
required: decl.required === true,
|
|
77
|
+
secret: decl.secret === true,
|
|
78
|
+
...(decl.default !== undefined ? { default: decl.default } : {}),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
schema: ENVELOPE_SCHEMA,
|
|
83
|
+
allowlist,
|
|
84
|
+
inputs,
|
|
85
|
+
maxRisk,
|
|
86
|
+
minRuntimeProfile: rb?.min_runtime_profile ??
|
|
87
|
+
impliedMinimum({ maxRisk, execution: rb?.execution ?? "human-only" }),
|
|
88
|
+
entry: steps[0]?.id,
|
|
89
|
+
terminals: graph.nodes.filter((n) => n.kind === "escalate" || n.kind === "end").map((n) => n.id),
|
|
90
|
+
steps: scopes,
|
|
91
|
+
budgets,
|
|
92
|
+
...(SUGGESTED_WALL_CLOCK[rb?.duration ?? ""] !== undefined
|
|
93
|
+
? { suggestedWallClockSeconds: SUGGESTED_WALL_CLOCK[rb?.duration ?? ""] }
|
|
94
|
+
: {}),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* What may be invoked right now.
|
|
99
|
+
*
|
|
100
|
+
* The gate is the whole point: a gated step's capabilities do not exist until a human
|
|
101
|
+
* has decided. So the destructive capability is in scope only while the destructive step
|
|
102
|
+
* is current *and* its gate has been passed - never before, and never after the step
|
|
103
|
+
* hands control on.
|
|
104
|
+
*/
|
|
105
|
+
export function scopeFor(envelope, stepId, approved = new Set()) {
|
|
106
|
+
const scope = envelope.steps[stepId];
|
|
107
|
+
if (!scope)
|
|
108
|
+
return [];
|
|
109
|
+
if (scope.gated && !approved.has(stepId))
|
|
110
|
+
return [];
|
|
111
|
+
return scope.tools;
|
|
112
|
+
}
|
|
113
|
+
export function authorize(envelope, req) {
|
|
114
|
+
const scope = envelope.steps[req.stepId];
|
|
115
|
+
if (!scope) {
|
|
116
|
+
return {
|
|
117
|
+
verdict: "block",
|
|
118
|
+
reason: `Step ${req.stepId} is not part of this runbook.`,
|
|
119
|
+
deviation: "out-of-order-step",
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
if (!envelope.allowlist.includes(req.tool)) {
|
|
123
|
+
return {
|
|
124
|
+
verdict: "block",
|
|
125
|
+
reason: `${req.tool} is not declared in this runbook's capabilities. Nothing outside the allowlist may be invoked for the duration of the run.`,
|
|
126
|
+
deviation: "undeclared-tool",
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
if (!scope.tools.includes(req.tool)) {
|
|
130
|
+
return {
|
|
131
|
+
verdict: "block",
|
|
132
|
+
reason: `${req.tool} is declared by this runbook but not by step ${req.stepId}. Scope is the current step, not the union of the procedure.`,
|
|
133
|
+
deviation: "out-of-scope-tool",
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
if (scope.gated && !(req.approved?.has(req.stepId) ?? false)) {
|
|
137
|
+
return {
|
|
138
|
+
verdict: "require-approval",
|
|
139
|
+
reason: `Step ${req.stepId} is ${scope.risk} and needs a human decision before it runs.`,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
return { verdict: "permit", reason: `Step ${req.stepId} declares ${req.tool}.` };
|
|
143
|
+
}
|
|
144
|
+
/** Steps whose risk forces an approval gate (RUNBOOK.md 12). */
|
|
145
|
+
export function dangerousSteps(envelope) {
|
|
146
|
+
return Object.values(envelope.steps).filter((s) => isDangerous(s.risk));
|
|
147
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join, dirname, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { derivePolicy, scopeFor, authorize, dangerousSteps } from "./policy.js";
|
|
6
|
+
/** Typed at the boundary so a fixture that drifts from the model is a compile error
|
|
7
|
+
* rather than a test that passes against a shape the code will never see. */
|
|
8
|
+
const runbook = (r) => ({ runbook: r });
|
|
9
|
+
const CORPUS = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", "schema", "fixtures", "p1-valid");
|
|
10
|
+
const corpus = readdirSync(CORPUS).map((name) => ({
|
|
11
|
+
name,
|
|
12
|
+
doc: JSON.parse(readFileSync(join(CORPUS, name), "utf8")),
|
|
13
|
+
}));
|
|
14
|
+
describe("derivation is a pure function", () => {
|
|
15
|
+
it.each(corpus)("$name derives identically twice", ({ doc }) => {
|
|
16
|
+
expect(JSON.stringify(derivePolicy(doc))).toBe(JSON.stringify(derivePolicy(doc)));
|
|
17
|
+
});
|
|
18
|
+
it.each(corpus)("$name derivation does not mutate the document", ({ doc }) => {
|
|
19
|
+
const before = JSON.stringify(doc);
|
|
20
|
+
derivePolicy(doc);
|
|
21
|
+
expect(JSON.stringify(doc)).toBe(before);
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* S-15's central invariant, asserted over the derivation rather than over examples:
|
|
26
|
+
* an envelope may never grant a capability the document does not declare. Stated as a
|
|
27
|
+
* property because the failure it prevents - an envelope quietly wider than the runbook
|
|
28
|
+
* that authorized it - is invisible in any single example.
|
|
29
|
+
*/
|
|
30
|
+
describe("the envelope carries no more authority than the document", () => {
|
|
31
|
+
it.each(corpus)("$name grants nothing outside capabilities[]", ({ doc }) => {
|
|
32
|
+
const env = derivePolicy(doc);
|
|
33
|
+
const declared = new Set(env.allowlist);
|
|
34
|
+
for (const scope of Object.values(env.steps)) {
|
|
35
|
+
for (const tool of scope.tools)
|
|
36
|
+
expect(declared.has(tool)).toBe(true);
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
it("moves an undeclared tool out of scope rather than granting it", () => {
|
|
40
|
+
const env = derivePolicy(runbook({
|
|
41
|
+
capabilities: ["cli:kubectl"],
|
|
42
|
+
steps: [
|
|
43
|
+
{ id: "s1", kind: "action", title: "Drop it", risk: "read-only", tool: "mcp:postgres" },
|
|
44
|
+
],
|
|
45
|
+
}));
|
|
46
|
+
expect(env.steps.s1.tools).toEqual([]);
|
|
47
|
+
expect(env.steps.s1.undeclared).toEqual(["mcp:postgres"]);
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
/**
|
|
51
|
+
* The other half of S-15: there is no reachable state in which a destructive capability
|
|
52
|
+
* is in scope and the approval gate has not been passed.
|
|
53
|
+
*/
|
|
54
|
+
describe("a destructive capability is never in scope before its gate", () => {
|
|
55
|
+
const destructive = runbook({
|
|
56
|
+
capabilities: ["mcp:postgres"],
|
|
57
|
+
execution: "human-with-agent",
|
|
58
|
+
steps: [
|
|
59
|
+
{
|
|
60
|
+
id: "s1", kind: "action", title: "Drop the slot", risk: "destructive",
|
|
61
|
+
tool: "mcp:postgres", requires_approval: true, next: "end:success",
|
|
62
|
+
},
|
|
63
|
+
],
|
|
64
|
+
});
|
|
65
|
+
it("holds for every step of every corpus record", () => {
|
|
66
|
+
for (const { doc } of corpus) {
|
|
67
|
+
const env = derivePolicy(doc);
|
|
68
|
+
for (const scope of dangerousSteps(env)) {
|
|
69
|
+
expect(scopeFor(env, scope.id, new Set()), `${scope.id} is ${scope.risk} and had capabilities in scope with no approval`).toEqual([]);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
it("withholds the capability until approval and grants it after", () => {
|
|
74
|
+
const env = derivePolicy(destructive);
|
|
75
|
+
expect(scopeFor(env, "s1", new Set())).toEqual([]);
|
|
76
|
+
expect(scopeFor(env, "s1", new Set(["s1"]))).toEqual(["mcp:postgres"]);
|
|
77
|
+
});
|
|
78
|
+
it("asks for approval rather than blocking outright", () => {
|
|
79
|
+
const env = derivePolicy(destructive);
|
|
80
|
+
expect(authorize(env, { stepId: "s1", tool: "mcp:postgres" }).verdict)
|
|
81
|
+
.toBe("require-approval");
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
describe("scope is the current step, not the union of the runbook", () => {
|
|
85
|
+
const env = derivePolicy(runbook({
|
|
86
|
+
capabilities: ["cli:kubectl", "mcp:postgres"],
|
|
87
|
+
steps: [
|
|
88
|
+
{ id: "s1", kind: "check", title: "Look", risk: "read-only", tool: "cli:kubectl", expect: "ok", on_fail: "s2", next: "s2" },
|
|
89
|
+
{ id: "s2", kind: "action", title: "Write", risk: "reversible-write", tool: "mcp:postgres", next: "end:success" },
|
|
90
|
+
],
|
|
91
|
+
}));
|
|
92
|
+
// The rule that does the most work: granting the union for the whole run is how a
|
|
93
|
+
// diagnostic session ends in a deletion.
|
|
94
|
+
it("blocks a capability the runbook declares but this step does not", () => {
|
|
95
|
+
const d = authorize(env, { stepId: "s1", tool: "mcp:postgres" });
|
|
96
|
+
expect(d.verdict).toBe("block");
|
|
97
|
+
expect(d.deviation).toBe("out-of-scope-tool");
|
|
98
|
+
expect(d.reason).toMatch(/not the union/);
|
|
99
|
+
});
|
|
100
|
+
it("permits the capability this step declares", () => {
|
|
101
|
+
expect(authorize(env, { stepId: "s1", tool: "cli:kubectl" }).verdict).toBe("permit");
|
|
102
|
+
});
|
|
103
|
+
it("distinguishes undeclared from out-of-scope", () => {
|
|
104
|
+
expect(authorize(env, { stepId: "s1", tool: "cli:rm" }).deviation).toBe("undeclared-tool");
|
|
105
|
+
});
|
|
106
|
+
it("blocks an invocation attributed to a step that is not in the runbook", () => {
|
|
107
|
+
expect(authorize(env, { stepId: "s99", tool: "cli:kubectl" }).deviation)
|
|
108
|
+
.toBe("out-of-order-step");
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
describe("budgets and terminals come from the graph", () => {
|
|
112
|
+
it("records a finite retry budget per edge", () => {
|
|
113
|
+
const env = derivePolicy(runbook({
|
|
114
|
+
capabilities: [],
|
|
115
|
+
steps: [
|
|
116
|
+
{ id: "s1", kind: "wait", title: "Wait", duration: "30s", retry: { max: 3, target: "s1" }, next: "end:success" },
|
|
117
|
+
],
|
|
118
|
+
}));
|
|
119
|
+
expect(env.budgets["s1->s1"]).toBe(3);
|
|
120
|
+
});
|
|
121
|
+
it.each(corpus)("$name has at least one terminal", ({ doc }) => {
|
|
122
|
+
expect(derivePolicy(doc).terminals.length).toBeGreaterThan(0);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { readFileSync, existsSync, readdirSync } from "node:fs";
|
|
3
|
+
import { join, dirname, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { builtinModules } from "node:module";
|
|
6
|
+
import { specifiers, moduleOf, reachesNetwork } from "@runbooks/fixtures";
|
|
7
|
+
/**
|
|
8
|
+
* R-01: the supervisor core must acquire no filesystem or network dependency anywhere
|
|
9
|
+
* in its tree — "check the dependency tree, not just the source". That is what keeps
|
|
10
|
+
* the core auditable, and it is exactly the kind of rule that erodes through a
|
|
11
|
+
* transitive import nobody looked at, so it is enforced rather than trusted.
|
|
12
|
+
*
|
|
13
|
+
* The walk follows real imports from the entry point. Scanning every file in a package
|
|
14
|
+
* would be wrong in both directions: it flags a Node-only entry point nothing imports
|
|
15
|
+
* (`@runbooks/schema/node`), and it would miss a dependency reached through one.
|
|
16
|
+
*/
|
|
17
|
+
function repoRoot() {
|
|
18
|
+
let dir = dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
for (let i = 0; i < 8; i++) {
|
|
20
|
+
if (existsSync(join(dir, "pnpm-workspace.yaml")))
|
|
21
|
+
return dir;
|
|
22
|
+
dir = resolve(dir, "..");
|
|
23
|
+
}
|
|
24
|
+
throw new Error("repo root not found");
|
|
25
|
+
}
|
|
26
|
+
const ROOT = repoRoot();
|
|
27
|
+
/**
|
|
28
|
+
* The extractor lives in `@runbooks/fixtures`, because five test files each had a copy of
|
|
29
|
+
* this rule and every copy recognised `from "node:fs"` and nothing else. Probed by
|
|
30
|
+
* mutation — one import at the top of the supervisor's entry point, then this file — five
|
|
31
|
+
* of seven forms passed:
|
|
32
|
+
*
|
|
33
|
+
* | put in the core | before |
|
|
34
|
+
* |---|---|
|
|
35
|
+
* | `import { readFileSync } from "fs"` | passed |
|
|
36
|
+
* | `import "node:fs"` | passed |
|
|
37
|
+
* | `const fs = await import("node:fs")` | passed |
|
|
38
|
+
* | `import { createRequire } from "node:module"` | passed |
|
|
39
|
+
* | `import { Worker } from "node:worker_threads"` | passed |
|
|
40
|
+
* | `import { readFileSync } from "node:fs/promises"` | caught |
|
|
41
|
+
* | `import { connect } from "node:http2"` | caught, and by accident |
|
|
42
|
+
*
|
|
43
|
+
* The last one is the tell: `node:http2` was caught by the *other* rule below, whose
|
|
44
|
+
* pattern happened to lack a word boundary after `node:http`. A check that passes for the
|
|
45
|
+
* reason it was written to fail is the shape this repository keeps finding.
|
|
46
|
+
*/
|
|
47
|
+
/**
|
|
48
|
+
* The list comes from the runtime, not from here.
|
|
49
|
+
*
|
|
50
|
+
* The rule it enforces is also stronger than the one it replaces: the supervisor's tree
|
|
51
|
+
* imports **no Node builtin at all**, which is true of all 28 modules in it today. A
|
|
52
|
+
* hand-typed set of eight I/O modules is a list somebody has to remember to extend —
|
|
53
|
+
* `worker_threads` and `module` were not on it, and either one reaches the filesystem in
|
|
54
|
+
* two lines.
|
|
55
|
+
*/
|
|
56
|
+
const BUILTIN = new Set(builtinModules);
|
|
57
|
+
function entry(spec) {
|
|
58
|
+
const ws = /^@runbooks\/([a-z-]+)(\/(.+))?$/.exec(spec);
|
|
59
|
+
if (!ws)
|
|
60
|
+
return undefined;
|
|
61
|
+
const sub = ws[3] ? `${ws[3]}.ts` : "index.ts";
|
|
62
|
+
const file = join(ROOT, "packages", ws[1], "src", sub);
|
|
63
|
+
return existsSync(file) ? file : undefined;
|
|
64
|
+
}
|
|
65
|
+
/** Every module reachable from `start`, with the Node builtins each one imports. */
|
|
66
|
+
function reachable(start) {
|
|
67
|
+
const seen = new Map();
|
|
68
|
+
const queue = [start];
|
|
69
|
+
while (queue.length) {
|
|
70
|
+
const file = queue.pop();
|
|
71
|
+
if (seen.has(file) || !existsSync(file))
|
|
72
|
+
continue;
|
|
73
|
+
const source = readFileSync(file, "utf8");
|
|
74
|
+
const builtins = [];
|
|
75
|
+
for (const specifier of specifiers(source)) {
|
|
76
|
+
if (BUILTIN.has(moduleOf(specifier))) {
|
|
77
|
+
builtins.push(specifier);
|
|
78
|
+
}
|
|
79
|
+
else if (specifier.startsWith(".")) {
|
|
80
|
+
queue.push(resolve(dirname(file), specifier.replace(/\.js$/, ".ts")));
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
const next = entry(specifier);
|
|
84
|
+
if (next)
|
|
85
|
+
queue.push(next);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
seen.set(file, builtins);
|
|
89
|
+
}
|
|
90
|
+
return seen;
|
|
91
|
+
}
|
|
92
|
+
describe("supervisor core purity (R-01)", () => {
|
|
93
|
+
const modules = reachable(join(ROOT, "packages", "supervise", "src", "index.ts"));
|
|
94
|
+
it("reaches more than its own entry point", () => {
|
|
95
|
+
// Guards the guard: a walk that resolves nothing would pass vacuously.
|
|
96
|
+
expect(modules.size).toBeGreaterThan(1);
|
|
97
|
+
});
|
|
98
|
+
it("imports no Node builtin anywhere in its tree", () => {
|
|
99
|
+
const offenders = [...modules]
|
|
100
|
+
.filter(([, builtins]) => builtins.length > 0)
|
|
101
|
+
.map(([file, builtins]) => `${file.slice(ROOT.length + 1)} imports ${builtins.join(", ")}`);
|
|
102
|
+
expect(offenders, "anything that fetches, reads or spawns belongs in apps/cli, not in the supervisor's tree").toEqual([]);
|
|
103
|
+
});
|
|
104
|
+
/**
|
|
105
|
+
* The extractor, checked against the forms it exists to see. Without this the walk above
|
|
106
|
+
* is a rule whose reach nobody states, which is how it came to see one form of five.
|
|
107
|
+
*/
|
|
108
|
+
it.each([
|
|
109
|
+
['import { readFileSync } from "fs";', "fs"],
|
|
110
|
+
["import { readFileSync } from 'node:fs';", "node:fs"],
|
|
111
|
+
['import "node:fs";', "node:fs"],
|
|
112
|
+
['const fs = await import("node:fs");', "node:fs"],
|
|
113
|
+
['const fs = require("node:fs");', "node:fs"],
|
|
114
|
+
['import { readFileSync } from "node:fs/promises";', "node:fs/promises"],
|
|
115
|
+
['export { x } from "node:fs";', "node:fs"],
|
|
116
|
+
])("sees %s", (line, expected) => {
|
|
117
|
+
expect(specifiers(line)).toContain(expected);
|
|
118
|
+
});
|
|
119
|
+
});
|
|
120
|
+
/**
|
|
121
|
+
* R-10 acceptance: every network call in the runtime chain is in the resolver, and
|
|
122
|
+
* nowhere else. Checked across the repository rather than by inspection, because the
|
|
123
|
+
* rule is only worth stating if something notices when it stops being true.
|
|
124
|
+
*/
|
|
125
|
+
describe("network access is confined to the resolver", () => {
|
|
126
|
+
/**
|
|
127
|
+
* `dgram` and `http2` count now. Neither did: `http2` was caught only because the
|
|
128
|
+
* pattern this replaces had no word boundary after `node:http`, and `dgram` was on the
|
|
129
|
+
* *other* rule's list and not on this one. Two hand-written lists of one rule
|
|
130
|
+
* disagreeing with each other is why the list is not written here any more.
|
|
131
|
+
*/
|
|
132
|
+
/**
|
|
133
|
+
* Two files, each named, each with a reason. Not a directory and not a pattern: an
|
|
134
|
+
* allowlist that admits a whole app is not confinement, it is a rule that has been
|
|
135
|
+
* turned off politely.
|
|
136
|
+
*
|
|
137
|
+
* - the resolver, which is how a runbook is fetched at run time (R-10);
|
|
138
|
+
* - the reindex command, which asks a source whether it still says what it said (I-03).
|
|
139
|
+
* Ingest exists to read other people's repositories, so it cannot be network-free;
|
|
140
|
+
* what it can be is a single file, with the decision about what a fetch *means* kept
|
|
141
|
+
* somewhere that never makes one.
|
|
142
|
+
* - the Microsoft triage command (C-03), for exactly that reason: the adapter takes an
|
|
143
|
+
* injected `list()` and cannot fetch, the classifier is a pure function over text, and
|
|
144
|
+
* the socket lives in the CLI a person runs on purpose. Nothing in the build or the
|
|
145
|
+
* serving path calls it.
|
|
146
|
+
*
|
|
147
|
+
* The property this protects is unchanged: nothing the supervisor depends on, and
|
|
148
|
+
* nothing in the serving path, reaches the network.
|
|
149
|
+
*/
|
|
150
|
+
const ALLOWED = [
|
|
151
|
+
"apps/cli/src/resolve.ts",
|
|
152
|
+
"apps/ingest/src/reindex.cli.ts",
|
|
153
|
+
"apps/ingest/src/microsoft.cli.ts",
|
|
154
|
+
];
|
|
155
|
+
function sources(dir) {
|
|
156
|
+
const root = join(ROOT, dir);
|
|
157
|
+
if (!existsSync(root))
|
|
158
|
+
return [];
|
|
159
|
+
const out = [];
|
|
160
|
+
const walk = (d) => {
|
|
161
|
+
for (const entry of readdirSync(d, { withFileTypes: true })) {
|
|
162
|
+
const full = join(d, entry.name);
|
|
163
|
+
if (entry.isDirectory()) {
|
|
164
|
+
if (entry.name !== "node_modules" && entry.name !== "dist")
|
|
165
|
+
walk(full);
|
|
166
|
+
}
|
|
167
|
+
else if (entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) {
|
|
168
|
+
out.push(full);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
walk(root);
|
|
173
|
+
return out;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* The half of the ingest rule that keeps the allowance honest: the file that fetches
|
|
177
|
+
* makes no decisions, and the file that decides never fetches. Otherwise "one file"
|
|
178
|
+
* becomes one very large file.
|
|
179
|
+
*/
|
|
180
|
+
it("the reindex command decides nothing about what a fetch means", () => {
|
|
181
|
+
const decider = readFileSync(join(ROOT, "apps", "ingest", "src", "upstream.ts"), "utf8");
|
|
182
|
+
expect(reachesNetwork(decider), "the decision about a fetch must not make one").toBe(false);
|
|
183
|
+
});
|
|
184
|
+
it("no package or app reaches the network outside the resolver", () => {
|
|
185
|
+
const offenders = [...sources("packages"), ...sources("apps")]
|
|
186
|
+
.map((file) => ({ file: file.slice(ROOT.length + 1), src: readFileSync(file, "utf8") }))
|
|
187
|
+
.filter(({ file, src }) => reachesNetwork(src) && !ALLOWED.includes(file))
|
|
188
|
+
.map(({ file }) => file);
|
|
189
|
+
expect(offenders, `fetching belongs in one of ${ALLOWED.join(" or ")}, so that the supervisor's tree can be checked for having none`).toEqual([]);
|
|
190
|
+
});
|
|
191
|
+
});
|
package/dist/report.d.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The run report.
|
|
3
|
+
*
|
|
4
|
+
* What a person reads afterwards, and what an attestation would be built from. Its one
|
|
5
|
+
* job is to say what actually happened, which means never rounding an undecided
|
|
6
|
+
* postcondition off to a decided one: `unevaluable` appears here as its own count, and a
|
|
7
|
+
* check a person decided is marked as theirs rather than as something the supervisor
|
|
8
|
+
* verified.
|
|
9
|
+
*/
|
|
10
|
+
import type { CheckRecord, RunEvent, RunState } from "./run.js";
|
|
11
|
+
import type { DeviationRecord } from "./deviations.js";
|
|
12
|
+
export interface RunReport {
|
|
13
|
+
readonly status: RunState["status"];
|
|
14
|
+
readonly outcome?: RunState["outcome"];
|
|
15
|
+
readonly complete: boolean;
|
|
16
|
+
readonly stepsVisited: readonly string[];
|
|
17
|
+
readonly checks: {
|
|
18
|
+
readonly passed: number;
|
|
19
|
+
readonly failed: number;
|
|
20
|
+
/** Never folded into either of the above. */
|
|
21
|
+
readonly unevaluable: number;
|
|
22
|
+
/** Of the decided ones, how many a person decided rather than the supervisor. */
|
|
23
|
+
readonly adjudicated: number;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* What deviated, aggregated. The count is the headline §13.4 asks for; the breakdowns
|
|
27
|
+
* are what tell an author that agents keep trying something their step 4 forbids.
|
|
28
|
+
*/
|
|
29
|
+
readonly deviations: {
|
|
30
|
+
readonly count: number;
|
|
31
|
+
readonly byClass: Readonly<Record<string, number>>;
|
|
32
|
+
readonly byStep: Readonly<Record<string, number>>;
|
|
33
|
+
/** The records themselves: class, step, capability class. No payload, ever. */
|
|
34
|
+
readonly records: readonly DeviationRecord[];
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Both budgets, so a step that chronically exhausts one is visible in aggregate rather
|
|
38
|
+
* than only in the moment it stopped a run.
|
|
39
|
+
*/
|
|
40
|
+
readonly budgets: {
|
|
41
|
+
/** Traversals per retry edge, against the limit the document set. */
|
|
42
|
+
readonly retries: readonly {
|
|
43
|
+
readonly edge: string;
|
|
44
|
+
readonly used: number;
|
|
45
|
+
readonly max?: number;
|
|
46
|
+
}[];
|
|
47
|
+
readonly wallClock: {
|
|
48
|
+
readonly limitSeconds?: number;
|
|
49
|
+
readonly elapsedSeconds?: number;
|
|
50
|
+
readonly exhausted: boolean;
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
/** Set while a postcondition is waiting on a person. */
|
|
54
|
+
readonly awaiting?: {
|
|
55
|
+
readonly stepId: string;
|
|
56
|
+
readonly why: string;
|
|
57
|
+
};
|
|
58
|
+
readonly detail: readonly CheckRecord[];
|
|
59
|
+
/**
|
|
60
|
+
* What this report does and does not establish, stated in the report rather than in
|
|
61
|
+
* documentation someone may not read (§13.5, §19).
|
|
62
|
+
*/
|
|
63
|
+
readonly caveats: readonly string[];
|
|
64
|
+
}
|
|
65
|
+
export interface ReportOptions {
|
|
66
|
+
readonly envelope?: {
|
|
67
|
+
readonly budgets: Readonly<Record<string, number>>;
|
|
68
|
+
};
|
|
69
|
+
/** The clock, supplied rather than read. Without it, elapsed time is simply unknown. */
|
|
70
|
+
readonly now?: number;
|
|
71
|
+
}
|
|
72
|
+
export declare function report(state: RunState, events?: readonly RunEvent[], options?: ReportOptions): RunReport;
|