@runbooks/supervise 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +47 -0
- package/dist/approval.d.ts +132 -0
- package/dist/approval.js +148 -0
- package/dist/approval.test.d.ts +1 -0
- package/dist/approval.test.js +190 -0
- package/dist/budgets.test.d.ts +1 -0
- package/dist/budgets.test.js +148 -0
- package/dist/contract.d.ts +38 -0
- package/dist/contract.js +77 -0
- package/dist/contract.test.d.ts +1 -0
- package/dist/contract.test.js +131 -0
- package/dist/deviations.d.ts +77 -0
- package/dist/deviations.js +81 -0
- package/dist/deviations.test.d.ts +1 -0
- package/dist/deviations.test.js +241 -0
- package/dist/emit.d.ts +97 -0
- package/dist/emit.js +184 -0
- package/dist/emit.test.d.ts +1 -0
- package/dist/emit.test.js +187 -0
- package/dist/enforce.d.ts +29 -0
- package/dist/enforce.js +100 -0
- package/dist/enforce.test.d.ts +1 -0
- package/dist/enforce.test.js +112 -0
- package/dist/expect.d.ts +67 -0
- package/dist/expect.js +187 -0
- package/dist/expect.test.d.ts +1 -0
- package/dist/expect.test.js +120 -0
- package/dist/index.d.ts +33 -0
- package/dist/index.js +34 -0
- package/dist/index.test.d.ts +1 -0
- package/dist/index.test.js +30 -0
- package/dist/observe.test.d.ts +1 -0
- package/dist/observe.test.js +246 -0
- package/dist/policy.d.ts +122 -0
- package/dist/policy.js +147 -0
- package/dist/policy.test.d.ts +1 -0
- package/dist/policy.test.js +124 -0
- package/dist/purity.test.d.ts +1 -0
- package/dist/purity.test.js +191 -0
- package/dist/report.d.ts +72 -0
- package/dist/report.js +67 -0
- package/dist/run.d.ts +216 -0
- package/dist/run.js +445 -0
- package/dist/run.test.d.ts +1 -0
- package/dist/run.test.js +198 -0
- package/package.json +42 -0
package/dist/run.js
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
import { authorize } from "./policy.js";
|
|
2
|
+
import { evaluate } from "./expect.js";
|
|
3
|
+
import { recordDeviation, DEFAULT_ESCALATION } from "./deviations.js";
|
|
4
|
+
import { plural } from "@runbooks/schema";
|
|
5
|
+
export function startRun(envelope, options = {}) {
|
|
6
|
+
const opts = "inputs" in options || "agentIdentity" in options
|
|
7
|
+
? options
|
|
8
|
+
: { inputs: options };
|
|
9
|
+
const provided = opts.inputs ?? {};
|
|
10
|
+
const refusals = [];
|
|
11
|
+
const bound = {};
|
|
12
|
+
for (const [name, decl] of Object.entries(envelope.inputs)) {
|
|
13
|
+
const value = provided[name] ?? decl.default;
|
|
14
|
+
if (value === undefined) {
|
|
15
|
+
if (decl.required) {
|
|
16
|
+
refusals.push({
|
|
17
|
+
code: "run/input-missing",
|
|
18
|
+
message: `Input "${name}" is required and was not supplied. Provide it, or run a procedure that does not need it.`,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
if (typeof value !== decl.type) {
|
|
24
|
+
refusals.push({
|
|
25
|
+
code: "run/input-type",
|
|
26
|
+
message: `Input "${name}" must be a ${decl.type}; got ${typeof value}.`,
|
|
27
|
+
});
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
bound[name] = value;
|
|
31
|
+
}
|
|
32
|
+
for (const name of Object.keys(provided)) {
|
|
33
|
+
if (!(name in envelope.inputs)) {
|
|
34
|
+
refusals.push({
|
|
35
|
+
code: "run/input-undeclared",
|
|
36
|
+
message: `Input "${name}" is not declared by this runbook. Nothing would bind it, so it is more likely a typo than an extra.`,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (!envelope.entry) {
|
|
41
|
+
refusals.push({
|
|
42
|
+
code: "run/no-entry",
|
|
43
|
+
message: "This runbook has no steps, so there is nothing to supervise.",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
if (refusals.length > 0)
|
|
47
|
+
return { ok: false, refusals };
|
|
48
|
+
return {
|
|
49
|
+
ok: true,
|
|
50
|
+
state: {
|
|
51
|
+
status: "running",
|
|
52
|
+
current: envelope.entry,
|
|
53
|
+
visited: [envelope.entry],
|
|
54
|
+
agentIdentity: opts.agentIdentity ?? "agent",
|
|
55
|
+
retries: {},
|
|
56
|
+
deviations: [],
|
|
57
|
+
escalation: opts.escalation ?? DEFAULT_ESCALATION,
|
|
58
|
+
checks: [],
|
|
59
|
+
mustCheck: opts.evaluatePostconditions === true,
|
|
60
|
+
...(opts.now !== undefined ? { startedAt: opts.now } : {}),
|
|
61
|
+
...(opts.wallClockSeconds !== undefined ? { wallClockSeconds: opts.wallClockSeconds } : {}),
|
|
62
|
+
inputs: bound,
|
|
63
|
+
},
|
|
64
|
+
events: [
|
|
65
|
+
{ kind: "started", code: "run/started", message: `Entering ${envelope.entry}.`, at: envelope.entry },
|
|
66
|
+
],
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Has the run outlasted its wall-clock budget?
|
|
71
|
+
*
|
|
72
|
+
* A procedure can be slow without looping, so this is counted separately from retries.
|
|
73
|
+
* With no budget set, or no clock supplied, the answer is no — an absent limit is not a
|
|
74
|
+
* zero one, and the report says plainly that nothing was bounding the run in time.
|
|
75
|
+
*/
|
|
76
|
+
export function outOfTime(state, now) {
|
|
77
|
+
if (now === undefined || state.startedAt === undefined || state.wallClockSeconds === undefined) {
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
return now - state.startedAt > state.wallClockSeconds;
|
|
81
|
+
}
|
|
82
|
+
function timeOut(state) {
|
|
83
|
+
return {
|
|
84
|
+
state: { ...state, status: "ended", outcome: "timed-out" },
|
|
85
|
+
events: [
|
|
86
|
+
{
|
|
87
|
+
kind: "budget-exhausted",
|
|
88
|
+
code: "run/wall-clock-exhausted",
|
|
89
|
+
message: `The run passed its wall-clock budget of ${state.wallClockSeconds}s. Stopping: a procedure that has run long past what the operator allowed is not one to keep authorizing calls for.`,
|
|
90
|
+
...(state.current ? { at: state.current } : {}),
|
|
91
|
+
},
|
|
92
|
+
],
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/** Authorize a proposed invocation against the current step. */
|
|
96
|
+
export function propose(envelope, state, proposal) {
|
|
97
|
+
if (state.status === "ended" || !state.current) {
|
|
98
|
+
const authorization = {
|
|
99
|
+
verdict: "block",
|
|
100
|
+
reason: "The run has ended; nothing further is authorized.",
|
|
101
|
+
deviation: "out-of-order-step",
|
|
102
|
+
};
|
|
103
|
+
const recorded = recordDeviation(state, { class: "out-of-order-step" });
|
|
104
|
+
return {
|
|
105
|
+
state: recorded.state,
|
|
106
|
+
authorization,
|
|
107
|
+
events: [
|
|
108
|
+
{ kind: "deviation", code: "run/after-end", message: authorization.reason },
|
|
109
|
+
...recorded.events,
|
|
110
|
+
],
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
if (outOfTime(state, proposal.now)) {
|
|
114
|
+
const timedOut = timeOut(state);
|
|
115
|
+
return {
|
|
116
|
+
state: timedOut.state,
|
|
117
|
+
authorization: {
|
|
118
|
+
verdict: "block",
|
|
119
|
+
reason: timedOut.events[0].message,
|
|
120
|
+
deviation: "budget-exhausted",
|
|
121
|
+
},
|
|
122
|
+
events: timedOut.events,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
const approved = new Set(state.grant && state.grant.stepId === state.current ? [state.current] : []);
|
|
126
|
+
const authorization = authorize(envelope, {
|
|
127
|
+
stepId: state.current,
|
|
128
|
+
tool: proposal.tool,
|
|
129
|
+
approved,
|
|
130
|
+
});
|
|
131
|
+
if (authorization.verdict === "permit") {
|
|
132
|
+
return {
|
|
133
|
+
state,
|
|
134
|
+
authorization,
|
|
135
|
+
events: [
|
|
136
|
+
{ kind: "authorized", code: "run/permitted", message: authorization.reason, at: state.current },
|
|
137
|
+
],
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
if (authorization.verdict === "require-approval") {
|
|
141
|
+
return {
|
|
142
|
+
state: { ...state, status: "awaiting-approval" },
|
|
143
|
+
authorization,
|
|
144
|
+
events: [
|
|
145
|
+
{ kind: "approval-requested", code: "run/approval-required", message: authorization.reason, at: state.current },
|
|
146
|
+
],
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
const deviation = authorization.deviation ?? "undeclared-tool";
|
|
150
|
+
const recorded = recordDeviation(state, { class: deviation, capability: proposal.tool });
|
|
151
|
+
return {
|
|
152
|
+
state: recorded.state,
|
|
153
|
+
authorization,
|
|
154
|
+
events: [
|
|
155
|
+
{ kind: "deviation", code: `run/${deviation}`, message: authorization.reason, at: state.current },
|
|
156
|
+
...recorded.events,
|
|
157
|
+
],
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Attach a decision to the run.
|
|
162
|
+
*
|
|
163
|
+
* The grant is produced by `grant()` in ./approval.js, which is where self-approval is
|
|
164
|
+
* refused — the core never mints one itself, so there is no path by which a run
|
|
165
|
+
* approves itself.
|
|
166
|
+
*/
|
|
167
|
+
export function applyGrant(state, granted) {
|
|
168
|
+
return {
|
|
169
|
+
state: {
|
|
170
|
+
...state,
|
|
171
|
+
grant: granted,
|
|
172
|
+
status: state.status === "awaiting-approval" ? "running" : state.status,
|
|
173
|
+
},
|
|
174
|
+
authorization: { verdict: "permit", reason: `Step ${granted.stepId} approved by ${granted.decidedBy}.` },
|
|
175
|
+
events: [
|
|
176
|
+
{
|
|
177
|
+
kind: "approved",
|
|
178
|
+
code: "run/approved",
|
|
179
|
+
message: `Step ${granted.stepId} approved by ${granted.decidedBy}.`,
|
|
180
|
+
at: granted.stepId,
|
|
181
|
+
},
|
|
182
|
+
],
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
/** Move to the next step, counting retry budgets and detecting terminals. */
|
|
186
|
+
export function advance(envelope, state, to, now) {
|
|
187
|
+
if (outOfTime(state, now))
|
|
188
|
+
return timeOut(state);
|
|
189
|
+
if (!state.current || state.status === "ended") {
|
|
190
|
+
return {
|
|
191
|
+
state,
|
|
192
|
+
events: [{ kind: "deviation", code: "run/after-end", message: "The run has already ended." }],
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
const from = envelope.steps[state.current];
|
|
196
|
+
/**
|
|
197
|
+
* A step whose postcondition has not been decided cannot be left.
|
|
198
|
+
*
|
|
199
|
+
* Without this, "the supervisor routes" is true only of the routes it was asked
|
|
200
|
+
* about: an agent could simply advance past a check along a declared edge and never
|
|
201
|
+
* report a result. The edge is legal; taking it before the check is not.
|
|
202
|
+
*/
|
|
203
|
+
if (state.mustCheck && from?.assert && state.decided !== state.current) {
|
|
204
|
+
const recorded = recordDeviation(state, { class: "unchecked-advance" });
|
|
205
|
+
return {
|
|
206
|
+
state: recorded.state,
|
|
207
|
+
events: [
|
|
208
|
+
{
|
|
209
|
+
kind: "deviation",
|
|
210
|
+
code: "run/unchecked-advance",
|
|
211
|
+
message: `${state.current} declares a postcondition that has not been decided. At R2 a check is left by its result, not by proposal.`,
|
|
212
|
+
at: state.current,
|
|
213
|
+
},
|
|
214
|
+
...recorded.events,
|
|
215
|
+
],
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
const scope = envelope.steps[state.current];
|
|
219
|
+
if (!scope || !scope.transitions.includes(to)) {
|
|
220
|
+
const recorded = recordDeviation(state, { class: "out-of-order-step" });
|
|
221
|
+
return {
|
|
222
|
+
state: recorded.state,
|
|
223
|
+
events: [
|
|
224
|
+
{
|
|
225
|
+
kind: "deviation",
|
|
226
|
+
code: "run/out-of-order-step",
|
|
227
|
+
message: `Step ${state.current} does not lead to ${to}. The graph decides where a run goes, not the agent.`,
|
|
228
|
+
at: state.current,
|
|
229
|
+
},
|
|
230
|
+
...recorded.events,
|
|
231
|
+
],
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const edge = `${state.current}->${to}`;
|
|
235
|
+
const budget = envelope.budgets[edge];
|
|
236
|
+
if (budget !== undefined) {
|
|
237
|
+
const used = (state.retries[edge] ?? 0) + 1;
|
|
238
|
+
if (used > budget) {
|
|
239
|
+
return {
|
|
240
|
+
state: {
|
|
241
|
+
...recordDeviation(state, { class: "budget-exhausted" }).state,
|
|
242
|
+
status: "ended",
|
|
243
|
+
outcome: "budget-exhausted",
|
|
244
|
+
},
|
|
245
|
+
events: [
|
|
246
|
+
{
|
|
247
|
+
kind: "budget-exhausted",
|
|
248
|
+
code: "run/budget-exhausted",
|
|
249
|
+
message: `Retry budget of ${budget} on ${edge} is exhausted after ${plural(used - 1, "traversal")}. Stopping rather than looping: an unbounded loop in an autonomous agent costs money and causes incidents. This is not a failure of ${state.current} — the step decided nothing; the run gave up on it.`,
|
|
250
|
+
at: state.current,
|
|
251
|
+
},
|
|
252
|
+
],
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
state = { ...state, retries: { ...state.retries, [edge]: used } };
|
|
256
|
+
}
|
|
257
|
+
// A grant covers one entry into one step. Leaving discards it, and a retry that
|
|
258
|
+
// returns here is a new entry needing a new decision. The same is true of a decided
|
|
259
|
+
// postcondition: coming back means checking again.
|
|
260
|
+
const { grant: _spent, decided: _checked, ...rest } = state;
|
|
261
|
+
state = rest;
|
|
262
|
+
if (envelope.terminals.includes(to)) {
|
|
263
|
+
const outcome = to.startsWith("__end_")
|
|
264
|
+
? to.slice("__end_".length)
|
|
265
|
+
: "failed";
|
|
266
|
+
return {
|
|
267
|
+
state: { ...state, status: "ended", current: to, visited: [...state.visited, to], outcome },
|
|
268
|
+
events: [{ kind: "ended", code: "run/ended", message: `Run ended: ${outcome}.`, at: to }],
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
return {
|
|
272
|
+
state: { ...state, current: to, visited: [...state.visited, to] },
|
|
273
|
+
events: [{ kind: "entered", code: "run/entered", message: `Entering ${to}.`, at: to }],
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
/** A run that stopped anywhere but a terminal is incomplete, and says so (13.4). */
|
|
277
|
+
export function isComplete(state) {
|
|
278
|
+
return state.status === "ended" && state.outcome !== undefined;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Decide whether the current step did what the procedure said it would, and route.
|
|
282
|
+
*
|
|
283
|
+
* This is the behaviour that separates R2 from R1. The route on failure is the
|
|
284
|
+
* document's `on_fail`, taken by the supervisor: at no point does the agent's own
|
|
285
|
+
* account of what happened choose the next node. Where the postcondition cannot be
|
|
286
|
+
* decided the run stops and waits for a person — it does not proceed, and it does not
|
|
287
|
+
* fail.
|
|
288
|
+
*/
|
|
289
|
+
export function observe(envelope, state, observation, now) {
|
|
290
|
+
if (!state.current || state.status === "ended") {
|
|
291
|
+
return {
|
|
292
|
+
state,
|
|
293
|
+
events: [{ kind: "deviation", code: "run/after-end", message: "The run has already ended." }],
|
|
294
|
+
evaluation: { outcome: "unevaluable", predicates: [], why: "The run has already ended." },
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
const stepId = state.current;
|
|
298
|
+
const scope = envelope.steps[stepId];
|
|
299
|
+
const evaluation = evaluate(scope?.assert, observation);
|
|
300
|
+
const why = evaluation.outcome === "pass" ? "The postcondition holds." : evaluation.why;
|
|
301
|
+
const record = { stepId, outcome: evaluation.outcome, why };
|
|
302
|
+
const withCheck = { ...state, checks: [...state.checks, record], decided: stepId };
|
|
303
|
+
if (evaluation.outcome === "unevaluable") {
|
|
304
|
+
return {
|
|
305
|
+
state: { ...withCheck, status: "awaiting-adjudication", awaiting: { stepId, why } },
|
|
306
|
+
events: [
|
|
307
|
+
{
|
|
308
|
+
kind: "expectation-unevaluable",
|
|
309
|
+
code: "run/expectation-unevaluable",
|
|
310
|
+
message: `${stepId}: ${why} The run stops here for a person to decide.`,
|
|
311
|
+
at: stepId,
|
|
312
|
+
},
|
|
313
|
+
],
|
|
314
|
+
evaluation,
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
const routed = route(envelope, withCheck, evaluation.outcome, why, now);
|
|
318
|
+
return { ...routed, evaluation };
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* Where a decided postcondition sends the run.
|
|
322
|
+
*
|
|
323
|
+
* Routing goes through `advance`, so a retry edge is still counted, a terminal still
|
|
324
|
+
* ends the run, and a grant is still discarded on the way out. A second mover here
|
|
325
|
+
* would be a second set of rules about the same graph.
|
|
326
|
+
*/
|
|
327
|
+
function route(envelope, state, outcome, why, now) {
|
|
328
|
+
const stepId = state.current;
|
|
329
|
+
const scope = envelope.steps[stepId];
|
|
330
|
+
const target = outcome === "pass" ? scope?.onPass : scope?.onFail;
|
|
331
|
+
const kind = outcome === "pass" ? "expectation-passed" : "expectation-failed";
|
|
332
|
+
const decided = {
|
|
333
|
+
kind,
|
|
334
|
+
code: `run/expectation-${outcome === "pass" ? "passed" : "failed"}`,
|
|
335
|
+
message: `${stepId}: ${why}`,
|
|
336
|
+
at: stepId,
|
|
337
|
+
};
|
|
338
|
+
if (!target) {
|
|
339
|
+
// A check whose failure routes nowhere is a defect in the document (invariant 4).
|
|
340
|
+
// Continuing past it would be the supervisor deciding what the author did not.
|
|
341
|
+
if (outcome === "fail") {
|
|
342
|
+
return {
|
|
343
|
+
state: {
|
|
344
|
+
...recordDeviation(state, { class: "unrouted-failure", stepId }).state,
|
|
345
|
+
status: "ended",
|
|
346
|
+
outcome: "aborted",
|
|
347
|
+
},
|
|
348
|
+
events: [
|
|
349
|
+
decided,
|
|
350
|
+
{
|
|
351
|
+
kind: "deviation",
|
|
352
|
+
code: "run/unrouted-failure",
|
|
353
|
+
message: `${stepId} failed its postcondition and declares no on_fail. Aborting: where to go next is the author's decision, and it was not made.`,
|
|
354
|
+
at: stepId,
|
|
355
|
+
},
|
|
356
|
+
],
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
// A pass with more than one way forward is a choice the graph left open.
|
|
360
|
+
return { state, events: [decided] };
|
|
361
|
+
}
|
|
362
|
+
const moved = advance(envelope, state, target, now);
|
|
363
|
+
return { state: moved.state, events: [decided, ...moved.events] };
|
|
364
|
+
}
|
|
365
|
+
export class SelfAdjudicationError extends Error {
|
|
366
|
+
constructor(identity) {
|
|
367
|
+
super(`${identity} is running this procedure and cannot also decide whether its own step ` +
|
|
368
|
+
`succeeded. A supervisor that asks the agent how it did has stopped supervising.`);
|
|
369
|
+
this.name = "SelfAdjudicationError";
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* A person decides what the supervisor could not, and the run resumes on that decision.
|
|
374
|
+
*
|
|
375
|
+
* The decision is recorded as theirs. A run report that showed an adjudicated check as
|
|
376
|
+
* simply "passed" would be claiming the supervisor verified something it did not.
|
|
377
|
+
*/
|
|
378
|
+
export function adjudicate(envelope, state, decision) {
|
|
379
|
+
if (state.status !== "awaiting-adjudication" || !state.awaiting) {
|
|
380
|
+
return {
|
|
381
|
+
state,
|
|
382
|
+
events: [
|
|
383
|
+
{
|
|
384
|
+
kind: "deviation",
|
|
385
|
+
code: "run/nothing-to-adjudicate",
|
|
386
|
+
message: "No postcondition is waiting on a decision.",
|
|
387
|
+
},
|
|
388
|
+
],
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
if (decision.decidedBy === state.agentIdentity)
|
|
392
|
+
throw new SelfAdjudicationError(decision.decidedBy);
|
|
393
|
+
const stepId = state.awaiting.stepId;
|
|
394
|
+
const why = `${decision.decidedBy} decided the step ${decision.verdict === "pass" ? "succeeded" : "failed"}.${decision.note ? ` ${decision.note}` : ""}`;
|
|
395
|
+
const checks = state.checks.map((check, index) => index === state.checks.length - 1 && check.stepId === stepId
|
|
396
|
+
? { ...check, outcome: decision.verdict, why, adjudicatedBy: decision.decidedBy }
|
|
397
|
+
: check);
|
|
398
|
+
const { awaiting: _resolved, ...rest } = state;
|
|
399
|
+
const resumed = { ...rest, status: "running", checks, decided: stepId };
|
|
400
|
+
const routed = route(envelope, resumed, decision.verdict, why);
|
|
401
|
+
return {
|
|
402
|
+
state: routed.state,
|
|
403
|
+
events: [
|
|
404
|
+
{
|
|
405
|
+
kind: "adjudicated",
|
|
406
|
+
code: "run/adjudicated",
|
|
407
|
+
message: `${stepId}: ${why}`,
|
|
408
|
+
at: stepId,
|
|
409
|
+
},
|
|
410
|
+
...routed.events,
|
|
411
|
+
],
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Close out a run that stopped somewhere other than a terminal.
|
|
416
|
+
*
|
|
417
|
+
* §13.4: an incomplete run says so. An adapter that loses its client, or an operator who
|
|
418
|
+
* walks away, leaves a run whose outcome is genuinely unknown — and "unknown" recorded
|
|
419
|
+
* as success is the one reading that would be a lie. Reaching an escalate node is not
|
|
420
|
+
* this case: an escalation the runbook anticipated is where the procedure meant to end.
|
|
421
|
+
*/
|
|
422
|
+
export function conclude(envelope, state) {
|
|
423
|
+
if (state.status === "ended")
|
|
424
|
+
return { state, events: [] };
|
|
425
|
+
const atTerminal = state.current !== undefined && envelope.terminals.includes(state.current);
|
|
426
|
+
if (atTerminal) {
|
|
427
|
+
return {
|
|
428
|
+
state: { ...state, status: "ended", outcome: "success" },
|
|
429
|
+
events: [{ kind: "ended", code: "run/ended", message: "Run ended: success.", at: state.current }],
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
const recorded = recordDeviation(state, { class: "terminal-not-reached" });
|
|
433
|
+
return {
|
|
434
|
+
state: { ...recorded.state, status: "ended", outcome: "aborted" },
|
|
435
|
+
events: [
|
|
436
|
+
{
|
|
437
|
+
kind: "deviation",
|
|
438
|
+
code: "run/terminal-not-reached",
|
|
439
|
+
message: `The run stopped at ${state.current ?? "no step"}, which is not a terminal. What happened after that is not something this supervisor saw.`,
|
|
440
|
+
...(state.current ? { at: state.current } : {}),
|
|
441
|
+
},
|
|
442
|
+
...recorded.events,
|
|
443
|
+
],
|
|
444
|
+
};
|
|
445
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/run.test.js
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join, dirname, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { derivePolicy } from "./policy.js";
|
|
6
|
+
import { startRun, propose, applyGrant, advance, isComplete } from "./run.js";
|
|
7
|
+
const CORPUS = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", "schema", "fixtures", "p1-valid");
|
|
8
|
+
const corpus = readdirSync(CORPUS).map((name) => ({
|
|
9
|
+
name,
|
|
10
|
+
doc: JSON.parse(readFileSync(join(CORPUS, name), "utf8")),
|
|
11
|
+
}));
|
|
12
|
+
const runbook = (r) => ({ runbook: r });
|
|
13
|
+
const started = (doc, inputs = {}) => {
|
|
14
|
+
const env = derivePolicy(doc);
|
|
15
|
+
const result = startRun(env, inputs);
|
|
16
|
+
if (!result.ok)
|
|
17
|
+
throw new Error("expected the run to start");
|
|
18
|
+
return { env, state: result.state };
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* R-01 acceptance: inputs are bound and validated before the graph is entered. Not
|
|
22
|
+
* lazily at first use — interpolating an empty string does not make a command narrower,
|
|
23
|
+
* it makes it a different one.
|
|
24
|
+
*/
|
|
25
|
+
describe("inputs are bound before the first step", () => {
|
|
26
|
+
const needsGroup = runbook({
|
|
27
|
+
capabilities: ["cli:kafka"],
|
|
28
|
+
inputs: { group: { type: "string", required: true } },
|
|
29
|
+
steps: [{ id: "s1", kind: "action", title: "Restart", risk: "read-only", tool: "cli:kafka", next: "end:success" }],
|
|
30
|
+
});
|
|
31
|
+
it("aborts before entering the graph when a required input is missing", () => {
|
|
32
|
+
const result = startRun(derivePolicy(needsGroup), {});
|
|
33
|
+
expect(result.ok).toBe(false);
|
|
34
|
+
if (result.ok)
|
|
35
|
+
return;
|
|
36
|
+
expect(result.refusals[0].code).toBe("run/input-missing");
|
|
37
|
+
expect(result.refusals[0].message).toContain("group");
|
|
38
|
+
});
|
|
39
|
+
it("starts once the input is supplied", () => {
|
|
40
|
+
const result = startRun(derivePolicy(needsGroup), { group: "checkout" });
|
|
41
|
+
expect(result.ok).toBe(true);
|
|
42
|
+
if (result.ok)
|
|
43
|
+
expect(result.state.inputs.group).toBe("checkout");
|
|
44
|
+
});
|
|
45
|
+
it("applies a declared default", () => {
|
|
46
|
+
const doc = runbook({
|
|
47
|
+
inputs: { dry_run: { type: "boolean", required: false, default: true } },
|
|
48
|
+
steps: [{ id: "s1", kind: "action", title: "Go", risk: "read-only", next: "end:success" }],
|
|
49
|
+
});
|
|
50
|
+
const result = startRun(derivePolicy(doc), {});
|
|
51
|
+
expect(result.ok && result.state.inputs.dry_run).toBe(true);
|
|
52
|
+
});
|
|
53
|
+
it("refuses a value of the wrong type", () => {
|
|
54
|
+
const result = startRun(derivePolicy(needsGroup), { group: 42 });
|
|
55
|
+
expect(result.ok).toBe(false);
|
|
56
|
+
if (!result.ok)
|
|
57
|
+
expect(result.refusals[0].code).toBe("run/input-type");
|
|
58
|
+
});
|
|
59
|
+
// More likely a typo than an extra, and a typo that silently does nothing is how a
|
|
60
|
+
// run proceeds against the wrong target.
|
|
61
|
+
it("refuses an input the runbook does not declare", () => {
|
|
62
|
+
const result = startRun(derivePolicy(needsGroup), { group: "checkout", gruop: "x" });
|
|
63
|
+
expect(result.ok).toBe(false);
|
|
64
|
+
if (!result.ok)
|
|
65
|
+
expect(result.refusals[0].code).toBe("run/input-undeclared");
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
describe("authorization follows the current step", () => {
|
|
69
|
+
const doc = runbook({
|
|
70
|
+
capabilities: ["cli:kubectl", "mcp:postgres"],
|
|
71
|
+
steps: [
|
|
72
|
+
{ id: "s1", kind: "check", title: "Look", risk: "read-only", tool: "cli:kubectl", expect: "ok", on_fail: "s2", next: "s2" },
|
|
73
|
+
{ id: "s2", kind: "action", title: "Write", risk: "reversible-write", tool: "mcp:postgres", next: "end:success" },
|
|
74
|
+
],
|
|
75
|
+
});
|
|
76
|
+
it("permits what the current step declares", () => {
|
|
77
|
+
const { env, state } = started(doc);
|
|
78
|
+
expect(propose(env, state, { tool: "cli:kubectl" }).authorization.verdict).toBe("permit");
|
|
79
|
+
});
|
|
80
|
+
it("blocks a capability that belongs to a later step", () => {
|
|
81
|
+
const { env, state } = started(doc);
|
|
82
|
+
const decided = propose(env, state, { tool: "mcp:postgres" });
|
|
83
|
+
expect(decided.authorization.verdict).toBe("block");
|
|
84
|
+
expect(decided.state.deviations.map((d) => d.class)).toEqual(["out-of-scope-tool"]);
|
|
85
|
+
});
|
|
86
|
+
it("permits it once the run has reached that step", () => {
|
|
87
|
+
const { env, state } = started(doc);
|
|
88
|
+
const { state: at2 } = advance(env, state, "s2");
|
|
89
|
+
expect(propose(env, at2, { tool: "mcp:postgres" }).authorization.verdict).toBe("permit");
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
describe("approval gates", () => {
|
|
93
|
+
const doc = runbook({
|
|
94
|
+
capabilities: ["mcp:postgres"],
|
|
95
|
+
steps: [
|
|
96
|
+
{
|
|
97
|
+
id: "s1", kind: "action", title: "Drop it", risk: "destructive",
|
|
98
|
+
tool: "mcp:postgres", requires_approval: true, next: "end:success",
|
|
99
|
+
},
|
|
100
|
+
],
|
|
101
|
+
});
|
|
102
|
+
it("asks rather than blocking, and holds the run", () => {
|
|
103
|
+
const { env, state } = started(doc);
|
|
104
|
+
const decided = propose(env, state, { tool: "mcp:postgres" });
|
|
105
|
+
expect(decided.authorization.verdict).toBe("require-approval");
|
|
106
|
+
expect(decided.state.status).toBe("awaiting-approval");
|
|
107
|
+
});
|
|
108
|
+
it("permits only after the decision is recorded", () => {
|
|
109
|
+
const { env, state } = started(doc);
|
|
110
|
+
const held = propose(env, state, { tool: "mcp:postgres" }).state;
|
|
111
|
+
const resumed = applyGrant(held, { requestId: "r1", stepId: "s1", decidedBy: "operator" }).state;
|
|
112
|
+
expect(resumed.status).toBe("running");
|
|
113
|
+
expect(propose(env, resumed, { tool: "mcp:postgres" }).authorization.verdict).toBe("permit");
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
describe("the graph decides where a run goes, not the agent", () => {
|
|
117
|
+
const doc = runbook({
|
|
118
|
+
capabilities: [],
|
|
119
|
+
steps: [
|
|
120
|
+
{ id: "s1", kind: "check", title: "Look", risk: "read-only", expect: "ok", on_fail: "s2", next: "s2" },
|
|
121
|
+
{ id: "s2", kind: "action", title: "Act", risk: "read-only", next: "end:success" },
|
|
122
|
+
{ id: "s3", kind: "action", title: "Elsewhere", risk: "read-only", next: "end:success" },
|
|
123
|
+
],
|
|
124
|
+
});
|
|
125
|
+
it("records a transition the graph does not permit as a deviation", () => {
|
|
126
|
+
const { env, state } = started(doc);
|
|
127
|
+
const moved = advance(env, state, "s3");
|
|
128
|
+
expect(moved.state.deviations.map((d) => d.class)).toEqual(["out-of-order-step"]);
|
|
129
|
+
expect(moved.state.current).toBe("s1");
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
describe("retry budgets are counted and terminate the run", () => {
|
|
133
|
+
const doc = runbook({
|
|
134
|
+
capabilities: [],
|
|
135
|
+
steps: [
|
|
136
|
+
{ id: "s1", kind: "wait", title: "Wait", duration: "5s", retry: { max: 2, target: "s1" }, next: "end:success" },
|
|
137
|
+
],
|
|
138
|
+
});
|
|
139
|
+
it("aborts once the budget is exhausted rather than looping", () => {
|
|
140
|
+
const { env } = started(doc);
|
|
141
|
+
let s = started(doc).state;
|
|
142
|
+
s = advance(env, s, "s1").state;
|
|
143
|
+
s = advance(env, s, "s1").state;
|
|
144
|
+
const third = advance(env, s, "s1");
|
|
145
|
+
expect(third.state.status).toBe("ended");
|
|
146
|
+
// Its own outcome, not a generic abort: the step decided nothing, the run gave up.
|
|
147
|
+
expect(third.state.outcome).toBe("budget-exhausted");
|
|
148
|
+
expect(third.events[0].code).toBe("run/budget-exhausted");
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
describe("the core is deterministic and side-effect free", () => {
|
|
152
|
+
it.each(corpus)("$name decides identically for the same sequence", ({ doc }) => {
|
|
153
|
+
const env = derivePolicy(doc);
|
|
154
|
+
const once = startRun(env, sampleInputs(doc));
|
|
155
|
+
const twice = startRun(env, sampleInputs(doc));
|
|
156
|
+
expect(JSON.stringify(once)).toBe(JSON.stringify(twice));
|
|
157
|
+
});
|
|
158
|
+
it("does not mutate the state it is given", () => {
|
|
159
|
+
const { env, state } = started(runbook({
|
|
160
|
+
capabilities: ["cli:x"],
|
|
161
|
+
steps: [{ id: "s1", kind: "action", title: "Go", risk: "read-only", tool: "cli:x", next: "end:success" }],
|
|
162
|
+
}));
|
|
163
|
+
const before = JSON.stringify({ ...state });
|
|
164
|
+
propose(env, state, { tool: "cli:x" });
|
|
165
|
+
advance(env, state, "__end_success");
|
|
166
|
+
expect(JSON.stringify({ ...state })).toBe(before);
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
describe("every event carries a code that resolves to a spec anchor", () => {
|
|
170
|
+
it("codes are namespaced and lowercase", () => {
|
|
171
|
+
const { env, state } = started(runbook({
|
|
172
|
+
capabilities: [],
|
|
173
|
+
steps: [{ id: "s1", kind: "action", title: "Go", risk: "read-only", next: "end:success" }],
|
|
174
|
+
}));
|
|
175
|
+
const events = [
|
|
176
|
+
...propose(env, state, { tool: "cli:nope" }).events,
|
|
177
|
+
...advance(env, state, "__end_success").events,
|
|
178
|
+
];
|
|
179
|
+
expect(events.length).toBeGreaterThan(0);
|
|
180
|
+
for (const e of events)
|
|
181
|
+
expect(e.code).toMatch(/^run\/[a-z-]+$/);
|
|
182
|
+
});
|
|
183
|
+
it("a run that ended is complete; one that stopped elsewhere is not", () => {
|
|
184
|
+
const { env, state } = started(runbook({
|
|
185
|
+
capabilities: [],
|
|
186
|
+
steps: [{ id: "s1", kind: "action", title: "Go", risk: "read-only", next: "end:success" }],
|
|
187
|
+
}));
|
|
188
|
+
expect(isComplete(state)).toBe(false);
|
|
189
|
+
expect(isComplete(advance(env, state, "__end_success").state)).toBe(true);
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
function sampleInputs(doc) {
|
|
193
|
+
const out = {};
|
|
194
|
+
for (const [name, decl] of Object.entries(doc.runbook?.inputs ?? {})) {
|
|
195
|
+
out[name] = decl.type === "boolean" ? true : decl.type === "number" ? 1 : "x";
|
|
196
|
+
}
|
|
197
|
+
return out;
|
|
198
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@runbooks/supervise",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"type": "module",
|
|
6
|
+
"description": "Supervisor core: pure decision logic. No I/O, no network.",
|
|
7
|
+
"main": "./dist/index.js",
|
|
8
|
+
"types": "./dist/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"dependencies": {
|
|
16
|
+
"@runbooks/schema": "^0.1.0",
|
|
17
|
+
"@runbooks/graph": "^0.1.0"
|
|
18
|
+
},
|
|
19
|
+
"license": "MIT",
|
|
20
|
+
"repository": {
|
|
21
|
+
"type": "git",
|
|
22
|
+
"url": "https://github.com/runbooks-directory/runbooks.directory"
|
|
23
|
+
},
|
|
24
|
+
"engines": {
|
|
25
|
+
"node": ">=20.11"
|
|
26
|
+
},
|
|
27
|
+
"publishConfig": {
|
|
28
|
+
"access": "public"
|
|
29
|
+
},
|
|
30
|
+
"files": [
|
|
31
|
+
"dist",
|
|
32
|
+
"README.md"
|
|
33
|
+
],
|
|
34
|
+
"devDependencies": {
|
|
35
|
+
"@runbooks/fixtures": "0.0.0"
|
|
36
|
+
},
|
|
37
|
+
"scripts": {
|
|
38
|
+
"build": "tsc -b",
|
|
39
|
+
"test": "vitest run --passWithNoTests",
|
|
40
|
+
"lint": "eslint src"
|
|
41
|
+
}
|
|
42
|
+
}
|