@intentius/chant 0.94.0 → 0.95.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/handlers/operator.d.ts.map +1 -1
- package/dist/cli/handlers/run.d.ts.map +1 -1
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/mcp/server.d.ts +10 -4
- package/dist/cli/mcp/server.d.ts.map +1 -1
- package/dist/cli/registry.d.ts +2 -0
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/op/builders.d.ts +14 -3
- package/dist/op/builders.d.ts.map +1 -1
- package/dist/op/index.d.ts +6 -3
- package/dist/op/index.d.ts.map +1 -1
- package/dist/op/operator.d.ts +90 -0
- package/dist/op/operator.d.ts.map +1 -1
- package/dist/op/steward-beside.d.ts +84 -0
- package/dist/op/steward-beside.d.ts.map +1 -0
- package/dist/op/steward.d.ts +87 -2
- package/dist/op/steward.d.ts.map +1 -1
- package/dist/workspace/box-services.d.ts +31 -0
- package/dist/workspace/box-services.d.ts.map +1 -0
- package/dist/workspace/compose-graph.d.ts +11 -0
- package/dist/workspace/compose-graph.d.ts.map +1 -1
- package/dist/workspace/composites.d.ts +5 -1
- package/dist/workspace/composites.d.ts.map +1 -1
- package/dist/workspace/declaration.d.ts +37 -0
- package/dist/workspace/declaration.d.ts.map +1 -1
- package/dist/workspace/declaration.schema.json +57 -1
- package/dist/workspace/graph-cache.d.ts +168 -0
- package/dist/workspace/graph-cache.d.ts.map +1 -0
- package/dist/workspace/graph-cli.d.ts +11 -5
- package/dist/workspace/graph-cli.d.ts.map +1 -1
- package/dist/workspace/kind-readers.d.ts +39 -0
- package/dist/workspace/kind-readers.d.ts.map +1 -0
- package/dist/workspace/kinds.d.ts +29 -0
- package/dist/workspace/kinds.d.ts.map +1 -1
- package/dist/workspace/member-commands.d.ts +15 -1
- package/dist/workspace/member-commands.d.ts.map +1 -1
- package/dist/workspace/member-run.d.ts +2 -0
- package/dist/workspace/member-run.d.ts.map +1 -1
- package/dist/workspace/reason-codes.d.ts +3 -1
- package/dist/workspace/reason-codes.d.ts.map +1 -1
- package/dist/workspace/records-cli.d.ts +10 -1
- package/dist/workspace/records-cli.d.ts.map +1 -1
- package/dist/workspace/records-write.d.ts +4 -2
- package/dist/workspace/records-write.d.ts.map +1 -1
- package/dist/workspace/records.d.ts +8 -3
- package/dist/workspace/records.d.ts.map +1 -1
- package/dist/workspace/status-stewards.d.ts +23 -7
- package/dist/workspace/status-stewards.d.ts.map +1 -1
- package/dist/workspace/status.d.ts +12 -0
- package/dist/workspace/status.d.ts.map +1 -1
- package/dist/workspace/work-evidence.d.ts +1 -1
- package/dist/workspace/work-evidence.d.ts.map +1 -1
- package/dist/workspace/workspace-kinds.schema.json +26 -0
- package/package.json +1 -1
- package/src/cli/commands/carve-bridge.test.ts +7 -3
- package/src/cli/handlers/operator-steward-signal.e2e.test.ts +97 -0
- package/src/cli/handlers/operator.ts +53 -11
- package/src/cli/handlers/run.test.ts +71 -0
- package/src/cli/handlers/run.ts +65 -7
- package/src/cli/main.ts +17 -6
- package/src/cli/mcp/server.test.ts +14 -0
- package/src/cli/mcp/server.ts +10 -4
- package/src/cli/mcp/workspace-tools.ts +1 -1
- package/src/cli/registry.ts +2 -0
- package/src/cli/static-config-read.test.ts +8 -2
- package/src/meta/source-is-text.test.ts +21 -3
- package/src/okf.test.ts +6 -1
- package/src/op/builders.ts +14 -3
- package/src/op/index.ts +8 -2
- package/src/op/operator.ts +264 -16
- package/src/op/steward-beside.test.ts +267 -0
- package/src/op/steward-beside.ts +219 -0
- package/src/op/steward-points.test.ts +61 -1
- package/src/op/steward.ts +135 -3
- package/src/workspace/box-services.test.ts +129 -0
- package/src/workspace/box-services.ts +51 -0
- package/src/workspace/checks/boxes.test.ts +1 -0
- package/src/workspace/compose-graph.test.ts +1 -0
- package/src/workspace/compose-graph.ts +11 -0
- package/src/workspace/composites.test.ts +1 -1
- package/src/workspace/composites.ts +12 -5
- package/src/workspace/conformance/__fixture__/decisions/decision.kind.mjs +4 -2
- package/src/workspace/declaration.schema.json +57 -1
- package/src/workspace/declaration.ts +104 -0
- package/src/workspace/graph-cache.test.ts +343 -0
- package/src/workspace/graph-cache.ts +409 -0
- package/src/workspace/graph-cli.ts +93 -26
- package/src/workspace/graph-contract.test.ts +129 -6
- package/src/workspace/graph.schema.json +21 -1
- package/src/workspace/kind-readers.e2e.test.ts +68 -0
- package/src/workspace/kind-readers.test.ts +134 -0
- package/src/workspace/kind-readers.ts +111 -0
- package/src/workspace/kinds.test.ts +49 -0
- package/src/workspace/kinds.ts +59 -2
- package/src/workspace/member-commands.test.ts +15 -0
- package/src/workspace/member-commands.ts +47 -7
- package/src/workspace/member-run.ts +10 -2
- package/src/workspace/reason-codes.ts +3 -1
- package/src/workspace/records-amend.schema.json +1 -0
- package/src/workspace/records-cli.ts +20 -12
- package/src/workspace/records-contract.test.ts +2 -1
- package/src/workspace/records-new.schema.json +1 -0
- package/src/workspace/records-quorum.test.ts +10 -3
- package/src/workspace/records-sessions-write.test.ts +2 -1
- package/src/workspace/records-write.test.ts +101 -1
- package/src/workspace/records-write.ts +50 -6
- package/src/workspace/records.ts +22 -5
- package/src/workspace/status-contract.test.ts +31 -1
- package/src/workspace/status-stewards.ts +39 -6
- package/src/workspace/status.schema.json +49 -4
- package/src/workspace/status.ts +22 -0
- package/src/workspace/trust/record-seal.test.ts +26 -2
- package/src/workspace/work-evidence.schema.json +1 -0
- package/src/workspace/workspace-kinds.schema.json +26 -0
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An Op a steward runs beside its turns (#2861): the declaration, a long run
|
|
3
|
+
* that holds none of the steward's turns up, a hand run that takes the Op's
|
|
4
|
+
* lease and not the turn, and the operator stopping the runs it started.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, test, expect, vi } from "vitest";
|
|
7
|
+
import { withTestDir } from "@intentius/chant-test-utils";
|
|
8
|
+
import { spawnSync } from "node:child_process";
|
|
9
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import type { ActivityFn, ActivityProfile } from "./activity-registry";
|
|
12
|
+
import type { ActivityStep, OpConfig } from "./types";
|
|
13
|
+
import { declareSteward, readinessKeys, stewardBesideFor, stewardTurnOps, stewardTurnLeaseName } from "./steward";
|
|
14
|
+
import { createBesideState, formatRoundLine, runOperatorForever, runOperatorRound, waitForBesideRuns, type OperatorTickEvent } from "./operator";
|
|
15
|
+
import { askReady, holdBesideLease, inProcessBesideLauncher, type BesideLauncher } from "./steward-beside";
|
|
16
|
+
import { stepOutput } from "./step-output-ref";
|
|
17
|
+
import { readLease } from "../lifecycle/lease";
|
|
18
|
+
import { readRunLedger } from "../lifecycle/run-ledger";
|
|
19
|
+
|
|
20
|
+
const PROFILES: Record<string, ActivityProfile> = {};
|
|
21
|
+
|
|
22
|
+
function op(name: string, cron?: string, fn = "tick"): OpConfig {
|
|
23
|
+
return {
|
|
24
|
+
name,
|
|
25
|
+
overview: `${name} fixture`,
|
|
26
|
+
phases: [{ name: "Run", steps: [{ kind: "activity", fn, args: {} }] }],
|
|
27
|
+
...(cron ? { schedule: { cron, overlap: "skip" as const } } : {}),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const ready = (fn = "ready"): ActivityStep => ({ kind: "activity", fn, args: {} });
|
|
32
|
+
|
|
33
|
+
function git(args: string[], cwd: string): void {
|
|
34
|
+
spawnSync("git", args, { cwd, encoding: "utf-8" });
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function initRepo(dir: string): void {
|
|
38
|
+
git(["init", "-q", "-b", "main"], dir);
|
|
39
|
+
git(["config", "user.email", "test@chant.dev"], dir);
|
|
40
|
+
git(["config", "user.name", "Test"], dir);
|
|
41
|
+
mkdirSync(join(dir, "app"), { recursive: true });
|
|
42
|
+
writeFileSync(join(dir, "app", "index.ts"), "export const hello = 1;\n");
|
|
43
|
+
git(["add", "app/index.ts"], dir);
|
|
44
|
+
git(["commit", "-q", "-m", "init"], dir);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const kinds = (events: OperatorTickEvent[]) => events.map((e) => [e.kind, e.op]);
|
|
48
|
+
|
|
49
|
+
describe("declareSteward's beside (#2861)", () => {
|
|
50
|
+
test("lists the Ops beside the turns after its turns' Ops, and names them", () => {
|
|
51
|
+
const steward = declareSteward({
|
|
52
|
+
name: "box-steward",
|
|
53
|
+
ops: [op("converge", "* * * * *")],
|
|
54
|
+
beside: [{ op: op("dispatch", undefined, "build"), ready: ready() }, op("rebuild", "0 3 * * *")],
|
|
55
|
+
});
|
|
56
|
+
expect(steward.ops.map((o) => o.name)).toEqual(["converge", "dispatch", "rebuild"]);
|
|
57
|
+
expect(steward.beside).toEqual([{ op: "dispatch", ready: ready() }, { op: "rebuild", ready: null }]);
|
|
58
|
+
expect(stewardTurnOps(steward).map((o) => o.name)).toEqual(["converge"]);
|
|
59
|
+
expect(stewardBesideFor(steward, "dispatch")?.ready).toEqual(ready());
|
|
60
|
+
expect(stewardBesideFor(steward, "converge")).toBeUndefined();
|
|
61
|
+
// A declaration from before #2861 has no beside, and every Op is a turn.
|
|
62
|
+
const { beside: _dropped, ...older } = steward;
|
|
63
|
+
expect(stewardTurnOps(older as typeof steward).map((o) => o.name)).toEqual(["converge", "dispatch", "rebuild"]);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("refuses an Op both in ops and beside, a ready that is not one activity step, and one that reads a step's output", () => {
|
|
67
|
+
const converge = op("converge", "* * * * *");
|
|
68
|
+
expect(() => declareSteward({ name: "s", ops: [converge], beside: [converge] })).toThrow(/listed twice/);
|
|
69
|
+
expect(() =>
|
|
70
|
+
declareSteward({ name: "s", ops: [], beside: [{ op: op("dispatch"), ready: { kind: "gate", gate: "g" } as unknown as ActivityStep }] }),
|
|
71
|
+
).toThrow(/ready is one activity step/);
|
|
72
|
+
expect(() =>
|
|
73
|
+
declareSteward({ name: "s", ops: [], beside: [{ op: op("dispatch"), ready: { kind: "activity", fn: "shellCmd", args: { cmd: stepOutput("pick", "json") } } }] }),
|
|
74
|
+
).toThrow(/reads another step's output/);
|
|
75
|
+
// Started on its own, it can't be handed the work item a run names.
|
|
76
|
+
const leased: OpConfig = { ...op("dispatch"), changesCheckout: true, workLease: {} };
|
|
77
|
+
expect(() => declareSteward({ name: "s", ops: [], beside: [{ op: leased, ready: ready() }] })).toThrow(/workLease names no item/);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test("a ready step's answer: a list of keys, a string, true, or no work", () => {
|
|
81
|
+
expect(readinessKeys({ stdout: "", json: ["W-1:0", "intent:ws-1"] })).toEqual({ ready: true, keys: ["W-1:0", "intent:ws-1"] });
|
|
82
|
+
expect(readinessKeys([{ item: "W-2" }])).toEqual({ ready: true, keys: ['{"item":"W-2"}'] });
|
|
83
|
+
expect(readinessKeys("W-3")).toEqual({ ready: true, keys: ["W-3"] });
|
|
84
|
+
expect(readinessKeys(true)).toEqual({ ready: true, keys: [] });
|
|
85
|
+
for (const none of [false, null, "", [], { json: null }]) expect(readinessKeys(none)).toEqual({ ready: false, keys: [] });
|
|
86
|
+
expect(readinessKeys({ ready: [] })).toBeNull();
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("askReady reports a step that fails, answers in another shape, or isn't loaded", async () => {
|
|
90
|
+
const acts = new Map<string, ActivityFn>([
|
|
91
|
+
["boom", async () => { throw new Error("no factory here"); }],
|
|
92
|
+
["odd", async () => ({ ready: [{ item: "W-1" }], held: null })],
|
|
93
|
+
]);
|
|
94
|
+
expect(await askReady(ready("boom"), acts)).toEqual({ error: "no factory here" });
|
|
95
|
+
expect(await askReady(ready("odd"), acts)).toMatchObject({ error: expect.stringContaining("neither true") });
|
|
96
|
+
expect(await askReady(ready("missing"), acts)).toMatchObject({ error: expect.stringContaining('"missing" is not loaded') });
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
describe("an Op beside a steward's turns (#2861)", () => {
|
|
101
|
+
test("a long run beside a per-minute converge: converge ticks every minute while it runs, and no work is started twice", async () => {
|
|
102
|
+
await withTestDir(async (dir) => {
|
|
103
|
+
initRepo(dir);
|
|
104
|
+
let finish: () => void = () => {};
|
|
105
|
+
const building = new Promise<void>((resolve) => { finish = resolve; });
|
|
106
|
+
const log: string[] = [];
|
|
107
|
+
const activities = new Map<string, ActivityFn>([
|
|
108
|
+
["tick", async () => { log.push("converge"); return { ok: true }; }],
|
|
109
|
+
["build", async () => { log.push("build:start"); await building; log.push("build:end"); return { built: true }; }],
|
|
110
|
+
["ready", async () => ({ stdout: "", json: ["W-1:0"] })],
|
|
111
|
+
]);
|
|
112
|
+
const steward = declareSteward({
|
|
113
|
+
name: "box-steward",
|
|
114
|
+
ops: [op("converge", "* * * * *")],
|
|
115
|
+
beside: [{ op: op("dispatch", undefined, "build"), ready: ready() }],
|
|
116
|
+
});
|
|
117
|
+
const besideState = createBesideState();
|
|
118
|
+
const scheduleState = new Map<string, Date>();
|
|
119
|
+
const launchBeside = inProcessBesideLauncher(activities, PROFILES);
|
|
120
|
+
// Cron minutes are local time, so the rounds are too. The leases use the
|
|
121
|
+
// wall clock, so a fixed `now` is only handed to the cron.
|
|
122
|
+
let minute = 1;
|
|
123
|
+
const round = () => {
|
|
124
|
+
const at = new Date(2026, 8, 25, 10, minute++, 0);
|
|
125
|
+
return runOperatorRound({ cwd: dir, holder: "op1", steward, activities, profiles: PROFILES, besideState, scheduleState, launchBeside, now: () => at });
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
// 10:01: converge ticks as a turn; the ready step names work, and dispatch starts beside it.
|
|
129
|
+
const first = await round();
|
|
130
|
+
expect(kinds(first)).toEqual([["ticked", "converge"], ["beside-started", "dispatch"]]);
|
|
131
|
+
expect(first[1]).toMatchObject({ why: "ready", keys: ["W-1:0"], holder: "box-steward/dispatch@op1" });
|
|
132
|
+
expect(formatRoundLine(first[1])).toBe('operator: dispatch@local started=1(beside:ready) holder="box-steward/dispatch@op1" keys="W-1:0"');
|
|
133
|
+
await vi.waitFor(() => expect(log).toContain("build:start"));
|
|
134
|
+
|
|
135
|
+
// Mid-build the run holds the Op's own lease, and not the steward's turn.
|
|
136
|
+
expect((await readLease("dispatch", { cwd: dir })).record?.holder).toBe("box-steward/dispatch@op1");
|
|
137
|
+
expect((await readLease(stewardTurnLeaseName("box-steward"), { cwd: dir })).record).toBeUndefined();
|
|
138
|
+
|
|
139
|
+
// 10:02 and 10:03: converge ticks each minute while the build runs, and no second run starts.
|
|
140
|
+
for (let i = 0; i < 2; i++) {
|
|
141
|
+
const mid = await round();
|
|
142
|
+
expect(kinds(mid)).toEqual([["ticked", "converge"], ["beside-running", "dispatch"]]);
|
|
143
|
+
}
|
|
144
|
+
expect(log.filter((l) => l === "converge")).toHaveLength(3);
|
|
145
|
+
expect(log).not.toContain("build:end");
|
|
146
|
+
|
|
147
|
+
finish();
|
|
148
|
+
await waitForBesideRuns(besideState);
|
|
149
|
+
expect((await readLease("dispatch", { cwd: dir })).record).toBeUndefined();
|
|
150
|
+
const run = (await readRunLedger("local", "dispatch", { cwd: dir })).records.at(-1)!;
|
|
151
|
+
expect(run).toMatchObject({ status: "ok", steward: "box-steward" });
|
|
152
|
+
|
|
153
|
+
// 10:04: the end is reported, and the same work still named is not started again.
|
|
154
|
+
const after = await round();
|
|
155
|
+
expect(after).toEqual([
|
|
156
|
+
{ kind: "beside-ended", op: "dispatch", env: "local", code: 0 },
|
|
157
|
+
expect.objectContaining({ kind: "ticked", op: "converge" }),
|
|
158
|
+
{ kind: "skipped-not-ready", op: "dispatch", env: "local", already: ["W-1:0"] },
|
|
159
|
+
]);
|
|
160
|
+
expect(log.filter((l) => l === "build:start")).toHaveLength(1);
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("a hand run holds the Op's lease and not the turn: converge still ticks, and the operator starts no second run", async () => {
|
|
165
|
+
await withTestDir(async (dir) => {
|
|
166
|
+
initRepo(dir);
|
|
167
|
+
const log: string[] = [];
|
|
168
|
+
const activities = new Map<string, ActivityFn>([
|
|
169
|
+
["tick", async () => { log.push("converge"); return { ok: true }; }],
|
|
170
|
+
["ready", async () => true],
|
|
171
|
+
]);
|
|
172
|
+
const steward = declareSteward({
|
|
173
|
+
name: "box-steward",
|
|
174
|
+
ops: [op("converge", "* * * * *")],
|
|
175
|
+
beside: [{ op: op("dispatch", undefined, "build"), ready: ready() }],
|
|
176
|
+
});
|
|
177
|
+
const launched: string[] = [];
|
|
178
|
+
const launchBeside: BesideLauncher = (start) => {
|
|
179
|
+
launched.push(start.op.name);
|
|
180
|
+
return { done: Promise.resolve({ code: 0 }), stop: () => {} };
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
// What `chant run dispatch` takes for an Op beside the turns.
|
|
184
|
+
const hand = await holdBesideLease("dispatch", "hand-run", { cwd: dir });
|
|
185
|
+
expect(hand.acquired).toBe(true);
|
|
186
|
+
expect((await readLease(stewardTurnLeaseName("box-steward"), { cwd: dir })).record).toBeUndefined();
|
|
187
|
+
|
|
188
|
+
const at = new Date(2026, 8, 25, 10, 1, 0);
|
|
189
|
+
const events = await runOperatorRound({ cwd: dir, holder: "op1", steward, activities, profiles: PROFILES, launchBeside, now: () => at });
|
|
190
|
+
expect(events.map((e) => e.kind)).toEqual(["ticked", "beside-running"]);
|
|
191
|
+
expect(events[1]).toEqual({ kind: "beside-running", op: "dispatch", env: "local", heldBy: "hand-run" });
|
|
192
|
+
expect(log).toEqual(["converge"]);
|
|
193
|
+
expect(launched).toEqual([]);
|
|
194
|
+
|
|
195
|
+
// A second hand run is refused while the first holds the lease.
|
|
196
|
+
expect(await holdBesideLease("dispatch", "hand-run-2", { cwd: dir })).toEqual({ acquired: false, heldBy: "hand-run" });
|
|
197
|
+
if (hand.acquired) await hand.release();
|
|
198
|
+
expect((await readLease("dispatch", { cwd: dir })).record).toBeUndefined();
|
|
199
|
+
const next = await runOperatorRound({ cwd: dir, holder: "op1", steward, activities, profiles: PROFILES, launchBeside, now: () => at });
|
|
200
|
+
expect(next.map((e) => e.kind)).toContain("beside-started");
|
|
201
|
+
expect(launched).toEqual(["dispatch"]);
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("the lease a long run holds is renewed while it runs", async () => {
|
|
206
|
+
await withTestDir(async (dir) => {
|
|
207
|
+
initRepo(dir);
|
|
208
|
+
const held = await holdBesideLease("dispatch", "h", { cwd: dir, ttlMs: 900 });
|
|
209
|
+
expect(held.acquired).toBe(true);
|
|
210
|
+
if (!held.acquired) return;
|
|
211
|
+
const first = held.lease.expiresAt;
|
|
212
|
+
await vi.waitFor(async () => {
|
|
213
|
+
const now = (await readLease("dispatch", { cwd: dir })).record;
|
|
214
|
+
expect(now?.token).toBe(held.lease.token);
|
|
215
|
+
expect(now!.expiresAt > first).toBe(true);
|
|
216
|
+
}, { timeout: 3000 });
|
|
217
|
+
await held.release();
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("a cron fire starts it, and one that fires while a run is in flight is dropped", async () => {
|
|
222
|
+
await withTestDir(async (dir) => {
|
|
223
|
+
initRepo(dir);
|
|
224
|
+
const steward = declareSteward({ name: "box-steward", ops: [], beside: [op("nightly", "0 3 * * *")] });
|
|
225
|
+
let end: () => void = () => {};
|
|
226
|
+
const launchBeside: BesideLauncher = () => ({ done: new Promise((resolve) => { end = () => resolve({ code: 0 }); }), stop: () => {} });
|
|
227
|
+
const besideState = createBesideState();
|
|
228
|
+
const scheduleState = new Map<string, Date>();
|
|
229
|
+
const round = (h: number, m: number) =>
|
|
230
|
+
runOperatorRound({ cwd: dir, holder: "op1", steward, activities: new Map(), profiles: PROFILES, besideState, scheduleState, launchBeside, now: () => new Date(2026, 8, 25, h, m, 0) });
|
|
231
|
+
expect(kinds(await round(2, 59))).toEqual([["skipped-not-due", "nightly"]]);
|
|
232
|
+
expect(await round(3, 0)).toEqual([expect.objectContaining({ kind: "beside-started", op: "nightly", why: "cron" })]);
|
|
233
|
+
expect(kinds(await round(3, 1))).toEqual([["beside-running", "nightly"]]);
|
|
234
|
+
end();
|
|
235
|
+
await waitForBesideRuns(besideState);
|
|
236
|
+
expect(kinds(await round(3, 2))).toEqual([["beside-ended", "nightly"], ["skipped-not-due", "nightly"]]);
|
|
237
|
+
});
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
test("the operator stops the runs it started when it stops", async () => {
|
|
241
|
+
await withTestDir(async (dir) => {
|
|
242
|
+
initRepo(dir);
|
|
243
|
+
const steward = declareSteward({ name: "box-steward", ops: [], beside: [{ op: op("dispatch"), ready: ready() }] });
|
|
244
|
+
const controller = new AbortController();
|
|
245
|
+
const stopped: string[] = [];
|
|
246
|
+
const launchBeside: BesideLauncher = (start) => {
|
|
247
|
+
let end: (code: number) => void = () => {};
|
|
248
|
+
const done = new Promise<{ code: number }>((resolve) => { end = (code) => resolve({ code }); });
|
|
249
|
+
return { done, stop: () => { stopped.push(start.op.name); end(1); } };
|
|
250
|
+
};
|
|
251
|
+
await runOperatorForever({
|
|
252
|
+
cwd: dir,
|
|
253
|
+
holder: "op1",
|
|
254
|
+
steward,
|
|
255
|
+
activities: new Map<string, ActivityFn>([["ready", async () => true]]),
|
|
256
|
+
profiles: PROFILES,
|
|
257
|
+
launchBeside,
|
|
258
|
+
intervalMs: 10,
|
|
259
|
+
signal: controller.signal,
|
|
260
|
+
onRound: (events) => {
|
|
261
|
+
if (events.some((e) => e.kind === "beside-started")) controller.abort();
|
|
262
|
+
},
|
|
263
|
+
});
|
|
264
|
+
expect(stopped).toEqual(["dispatch"]);
|
|
265
|
+
});
|
|
266
|
+
});
|
|
267
|
+
});
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An Op a steward runs beside its turns (#2861): how a run of it is started,
|
|
3
|
+
* held and asked for.
|
|
4
|
+
*
|
|
5
|
+
* A local steward runs the Ops it lists one turn at a time (#2750). An Op
|
|
6
|
+
* listed under `beside` (`./steward.ts`) is the steward's too, but a run of it
|
|
7
|
+
* is not a turn: the operator (`./operator.ts`) starts it as a `chant run
|
|
8
|
+
* <op>` process of its own and goes on with its rounds, so converge keeps its
|
|
9
|
+
* cadence while a build of half an hour runs.
|
|
10
|
+
*
|
|
11
|
+
* - The run holds the Op's own lease (`refs/chant/lease/<op>`, the lease a
|
|
12
|
+
* round's tick of any Op takes), renewed every third of its time to live
|
|
13
|
+
* for as long as the run lasts, and released when it ends. That is what
|
|
14
|
+
* makes it one run at a time, whoever starts it: the operator, or a person's
|
|
15
|
+
* `chant run <op>`, which takes this lease and not the steward's turn.
|
|
16
|
+
* - The process the operator starts carries `CHANT_STEWARD`, so its run record
|
|
17
|
+
* names the steward, a decision point it asks is the steward's, and the
|
|
18
|
+
* operator resumes it once the question is answered, by starting another
|
|
19
|
+
* such process. Its work lease holder is `<steward>/<op>@<operator>`, which
|
|
20
|
+
* is how `workspace status` finds it under the steward.
|
|
21
|
+
*/
|
|
22
|
+
import { spawn } from "node:child_process";
|
|
23
|
+
import type { ActivityFn, ActivityProfile } from "./activity-registry";
|
|
24
|
+
import { runOpLocally, OpRunFailure } from "./local-executor";
|
|
25
|
+
import type { ActivityStep, OpConfig } from "./types";
|
|
26
|
+
import { parseDuration } from "./duration";
|
|
27
|
+
import { acquireLease, releaseLease, DEFAULT_LEASE_TTL_MS, type LeaseRecord } from "../lifecycle/lease";
|
|
28
|
+
import { readinessKeys } from "./steward";
|
|
29
|
+
|
|
30
|
+
/** Why the operator started a run beside the turns. */
|
|
31
|
+
export type BesideWhy = "cron" | "ready" | "resumed" | "approved";
|
|
32
|
+
|
|
33
|
+
/** What the operator asks a launcher to start. */
|
|
34
|
+
export interface BesideStart {
|
|
35
|
+
/** The Op. */
|
|
36
|
+
op: OpConfig;
|
|
37
|
+
/** The steward whose run it is. */
|
|
38
|
+
steward: string;
|
|
39
|
+
/** The operator's own `--env`, passed on so the run reads the steward's form the same way. */
|
|
40
|
+
env?: string;
|
|
41
|
+
/** The holder the run claims the Op's lease and any work lease as: `<steward>/<op>@<operator>`. */
|
|
42
|
+
holder: string;
|
|
43
|
+
/** The project directory. */
|
|
44
|
+
cwd: string;
|
|
45
|
+
/** How long the Op's lease lasts unless renewed. */
|
|
46
|
+
leaseTtlMs?: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** How a started run ended: its exit code (0 ok, 3 gated or waiting, 1 failed), or the error that kept it from running. */
|
|
50
|
+
export interface BesideExit {
|
|
51
|
+
code: number | null;
|
|
52
|
+
error?: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** A run started beside the turns. */
|
|
56
|
+
export interface BesideHandle {
|
|
57
|
+
/** Settles when the run ends; never rejects. */
|
|
58
|
+
done: Promise<BesideExit>;
|
|
59
|
+
/** Ask the run to stop, as Ctrl-C does: it stops its steps and releases its leases. */
|
|
60
|
+
stop(): void;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Starts a run beside the turns. The default is {@link spawnBesideRun}. */
|
|
64
|
+
export type BesideLauncher = (start: BesideStart) => BesideHandle;
|
|
65
|
+
|
|
66
|
+
/** The exit code `chant run` gives a run that stopped at a gate or on an open decision point. */
|
|
67
|
+
const WAITING_EXIT = 3;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Start `chant run <op>` as a process of its own: the same node, loader and
|
|
71
|
+
* entry script this process runs as, in the project directory, with
|
|
72
|
+
* `CHANT_STEWARD` naming the steward and `--holder` naming the holder.
|
|
73
|
+
*/
|
|
74
|
+
export function spawnBesideRun(start: BesideStart): BesideHandle {
|
|
75
|
+
const args = [
|
|
76
|
+
...process.execArgv,
|
|
77
|
+
process.argv[1],
|
|
78
|
+
"run",
|
|
79
|
+
start.op.name,
|
|
80
|
+
"--holder",
|
|
81
|
+
start.holder,
|
|
82
|
+
...(start.env ? ["--env", start.env] : []),
|
|
83
|
+
];
|
|
84
|
+
const child = spawn(process.execPath, args, {
|
|
85
|
+
cwd: start.cwd,
|
|
86
|
+
env: { ...process.env, CHANT_STEWARD: start.steward },
|
|
87
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
88
|
+
});
|
|
89
|
+
const done = new Promise<BesideExit>((resolve) => {
|
|
90
|
+
child.once("error", (err) => resolve({ code: null, error: err.message }));
|
|
91
|
+
child.once("exit", (code, signal) => resolve(signal ? { code: null, error: `stopped by ${signal}` } : { code }));
|
|
92
|
+
});
|
|
93
|
+
return {
|
|
94
|
+
done,
|
|
95
|
+
// `chant run` stops its Op and releases its leases on SIGINT.
|
|
96
|
+
stop: () => {
|
|
97
|
+
if (child.exitCode === null && child.signalCode === null) child.kill("SIGINT");
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* A launcher that runs the Op in this process, as an async task, under the
|
|
104
|
+
* same lease a `chant run` of it takes. For tests, and for an embedder whose
|
|
105
|
+
* activities are its own: the run does not enter the steward's turn (that is
|
|
106
|
+
* a property of the process, and the operator's own turns share it), so its
|
|
107
|
+
* record names the steward and a decision point it asks does not.
|
|
108
|
+
*/
|
|
109
|
+
export function inProcessBesideLauncher(
|
|
110
|
+
activities: Map<string, ActivityFn>,
|
|
111
|
+
profiles: Record<string, ActivityProfile>,
|
|
112
|
+
): BesideLauncher {
|
|
113
|
+
return (start) => {
|
|
114
|
+
const controller = new AbortController();
|
|
115
|
+
const done = (async (): Promise<BesideExit> => {
|
|
116
|
+
const held = await holdBesideLease(start.op.name, start.holder, { cwd: start.cwd, ttlMs: start.leaseTtlMs });
|
|
117
|
+
if (!held.acquired) return { code: 1, error: `the lease of "${start.op.name}" is held by ${held.heldBy ?? "another holder"}` };
|
|
118
|
+
try {
|
|
119
|
+
const result = await runOpLocally(start.op, activities, profiles, controller.signal, {
|
|
120
|
+
steward: start.steward,
|
|
121
|
+
ledger: { cwd: start.cwd },
|
|
122
|
+
work: { holder: start.holder },
|
|
123
|
+
});
|
|
124
|
+
return { code: result.status === "gated" || result.status === "waiting" ? WAITING_EXIT : 0 };
|
|
125
|
+
} catch (err) {
|
|
126
|
+
return { code: 1, error: err instanceof OpRunFailure ? `Op "${start.op.name}" failed` : err instanceof Error ? err.message : String(err) };
|
|
127
|
+
} finally {
|
|
128
|
+
await held.release();
|
|
129
|
+
}
|
|
130
|
+
})();
|
|
131
|
+
return { done, stop: () => controller.abort() };
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The Op's own lease, held for a run beside the turns. */
|
|
136
|
+
export type HeldBesideLease =
|
|
137
|
+
| { acquired: true; lease: LeaseRecord; release: () => Promise<void> }
|
|
138
|
+
| { acquired: false; heldBy?: string };
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Take the Op's own lease for one run beside the steward's turns, and renew
|
|
142
|
+
* it every third of its time to live until `release` is called. A run of half
|
|
143
|
+
* an hour keeps it; a process that dies stops renewing and the lease expires.
|
|
144
|
+
* Refused when another holder has it: a run of the Op is in flight.
|
|
145
|
+
*/
|
|
146
|
+
export async function holdBesideLease(
|
|
147
|
+
op: string,
|
|
148
|
+
holder: string,
|
|
149
|
+
opts: { cwd?: string; ttlMs?: number } = {},
|
|
150
|
+
): Promise<HeldBesideLease> {
|
|
151
|
+
const ttlMs = opts.ttlMs ?? DEFAULT_LEASE_TTL_MS;
|
|
152
|
+
const first = await acquireLease(op, holder, { cwd: opts.cwd, ttlMs });
|
|
153
|
+
if (!first.acquired) return { acquired: false, ...(first.heldBy?.holder ? { heldBy: first.heldBy.holder } : {}) };
|
|
154
|
+
const lease = first.lease!;
|
|
155
|
+
// A renewal that fails is retried on the next beat; one the lease refuses
|
|
156
|
+
// means another holder took it after it expired, and there is nothing to renew.
|
|
157
|
+
const timer = setInterval(() => {
|
|
158
|
+
void acquireLease(op, holder, { cwd: opts.cwd, ttlMs, mode: "renew", token: lease.token }).catch(() => undefined);
|
|
159
|
+
}, Math.max(1, Math.floor(ttlMs / 3)));
|
|
160
|
+
timer.unref?.();
|
|
161
|
+
let released = false;
|
|
162
|
+
return {
|
|
163
|
+
acquired: true,
|
|
164
|
+
lease,
|
|
165
|
+
release: async () => {
|
|
166
|
+
if (released) return;
|
|
167
|
+
released = true;
|
|
168
|
+
clearInterval(timer);
|
|
169
|
+
await releaseLease(op, holder, lease.token, { cwd: opts.cwd }).catch(() => false);
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** How long a ready step may run when it names no `timeout`. */
|
|
175
|
+
export const DEFAULT_READY_TIMEOUT_MS = 120_000;
|
|
176
|
+
|
|
177
|
+
/** What one ready step said: whether there is work and its keys, or why it could not say. */
|
|
178
|
+
export type ReadyAnswer = { ready: boolean; keys: string[] } | { error: string };
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Run a steward's `ready` step for an Op beside its turns (#2861) and read
|
|
182
|
+
* its answer with {@link readinessKeys}. The step is one activity call, with
|
|
183
|
+
* its own `timeout` or {@link DEFAULT_READY_TIMEOUT_MS}; it runs outside any
|
|
184
|
+
* run, so it writes no record, and it is the author's to keep it a read.
|
|
185
|
+
*/
|
|
186
|
+
export async function askReady(
|
|
187
|
+
step: ActivityStep,
|
|
188
|
+
activities: Map<string, ActivityFn>,
|
|
189
|
+
signal?: AbortSignal,
|
|
190
|
+
): Promise<ReadyAnswer> {
|
|
191
|
+
const fn = activities.get(step.fn);
|
|
192
|
+
if (!fn) return { error: `its ready step's activity "${step.fn}" is not loaded` };
|
|
193
|
+
const timeoutMs = step.timeout ? parseDuration(step.timeout) : DEFAULT_READY_TIMEOUT_MS;
|
|
194
|
+
const controller = new AbortController();
|
|
195
|
+
const onAbort = () => controller.abort();
|
|
196
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
197
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
198
|
+
try {
|
|
199
|
+
const call = Promise.resolve(fn({ ...(step.args ?? {}) }, controller.signal));
|
|
200
|
+
call.catch(() => {});
|
|
201
|
+
const result = await Promise.race([
|
|
202
|
+
call,
|
|
203
|
+
new Promise<never>((_, reject) => {
|
|
204
|
+
timer = setTimeout(() => {
|
|
205
|
+
controller.abort();
|
|
206
|
+
reject(new Error(`timed out after ${timeoutMs}ms`));
|
|
207
|
+
}, timeoutMs);
|
|
208
|
+
}),
|
|
209
|
+
]);
|
|
210
|
+
const answer = readinessKeys(result);
|
|
211
|
+
if (!answer) return { error: "its ready step answered neither true, false, null, a string nor a list" };
|
|
212
|
+
return answer;
|
|
213
|
+
} catch (err) {
|
|
214
|
+
return { error: err instanceof Error ? err.message : String(err) };
|
|
215
|
+
} finally {
|
|
216
|
+
if (timer) clearTimeout(timer);
|
|
217
|
+
signal?.removeEventListener("abort", onAbort);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
@@ -19,7 +19,8 @@ import { readMemberStewards } from "../workspace/status-stewards";
|
|
|
19
19
|
import { readRunLedger } from "../lifecycle/run-ledger";
|
|
20
20
|
import type { ActivityFn, ActivityProfile } from "./activity-registry";
|
|
21
21
|
import { runOpLocally } from "./local-executor";
|
|
22
|
-
import { formatRoundLine, runOperatorRound } from "./operator";
|
|
22
|
+
import { createBesideState, formatRoundLine, runOperatorRound, waitForBesideRuns } from "./operator";
|
|
23
|
+
import { inProcessBesideLauncher } from "./steward-beside";
|
|
23
24
|
import { declareSteward } from "./steward";
|
|
24
25
|
import { askPointInRun, isPointWait, PointWait, type BrokeredModelAsk } from "./steward-points";
|
|
25
26
|
import { enterStewardTurn, resetStewardTurn, STEWARD_ENV } from "./steward-turn";
|
|
@@ -362,3 +363,62 @@ describe("a waiting run of an Op the steward does not list (studio#137)", () =>
|
|
|
362
363
|
expect(listed).not.toContain("factory-stranger");
|
|
363
364
|
});
|
|
364
365
|
});
|
|
366
|
+
|
|
367
|
+
describe("a waiting run of an Op beside the steward's turns (#2861)", () => {
|
|
368
|
+
test("status lists it under the steward with its lease, and the operator resumes it beside the turns once a person answers", async () => {
|
|
369
|
+
const build = shipOp("box-build", "r-30");
|
|
370
|
+
const steward = declareSteward({
|
|
371
|
+
name: "build-steward",
|
|
372
|
+
ops: [],
|
|
373
|
+
beside: [{ op: build, ready: { kind: "activity", fn: "readyShip", args: {} } }],
|
|
374
|
+
capabilities: ["inference"],
|
|
375
|
+
});
|
|
376
|
+
const log: string[] = [];
|
|
377
|
+
const acts = new Map<string, ActivityFn>([
|
|
378
|
+
...shipActivities(log),
|
|
379
|
+
["askShip", async (args) => (await askPointInRun({ cwd: root, point: "ship-now", inputs: { release: args.release }, subject: String(args.release), on })).answer],
|
|
380
|
+
["readyShip", async () => ["r-30"]],
|
|
381
|
+
]);
|
|
382
|
+
const besideState = createBesideState();
|
|
383
|
+
const launchBeside = inProcessBesideLauncher(acts, PROFILES);
|
|
384
|
+
const round = () => runOperatorRound({ cwd: root, steward, activities: acts, profiles: PROFILES, holder: "box", besideState, launchBeside });
|
|
385
|
+
|
|
386
|
+
// The ready step names work: the run starts beside the turns, asks, and waits.
|
|
387
|
+
const first = await round();
|
|
388
|
+
expect(first).toEqual([expect.objectContaining({ kind: "beside-started", op: "box-build", why: "ready", keys: ["r-30"], holder: "build-steward/box-build@box" })]);
|
|
389
|
+
await waitForBesideRuns(besideState);
|
|
390
|
+
const waiting = (await readRunLedger("local", "box-build", { cwd: root })).records.at(-1)!;
|
|
391
|
+
expect(waiting).toMatchObject({ status: "waiting", steward: "build-steward" });
|
|
392
|
+
const question = waiting.point!.id;
|
|
393
|
+
|
|
394
|
+
// workspace status lists it under the steward, as one of its Ops, with its wait.
|
|
395
|
+
mkdirSync(join(root, "ops"), { recursive: true });
|
|
396
|
+
writeFileSync(join(root, "chant.config.json"), "{}\n");
|
|
397
|
+
writeFileSync(join(root, "ops", "build-steward.op.ts"), `export const steward = ${JSON.stringify(steward)};\n`);
|
|
398
|
+
const status = await readMemberStewards(root, "local", "2027-01-01T00:01:00Z");
|
|
399
|
+
const entry = status.stewards.find((s) => s.name === "build-steward")!;
|
|
400
|
+
const stewardShape = contract({ $schema: statusSchema.$schema, $id: "urn:test:status-steward-beside", $defs: statusSchema.$defs, $ref: "#/$defs/steward" });
|
|
401
|
+
stewardShape.expectValid(entry);
|
|
402
|
+
expect(entry.ops.map((o) => [o.name, o.beside])).toEqual([["box-build", { ready: true, lease: null }]]);
|
|
403
|
+
expect(entry.waiting).toEqual([
|
|
404
|
+
{ op: "box-build", run: waiting.id, id: question, point: "ship-now", state: "escalated", path: waiting.point!.path, subject: "r-30", since: waiting.point!.since },
|
|
405
|
+
]);
|
|
406
|
+
|
|
407
|
+
// Still open: the same work is not started again, and the round says what it waits on.
|
|
408
|
+
const second = await round();
|
|
409
|
+
expect(second).toEqual([
|
|
410
|
+
{ kind: "beside-ended", op: "box-build", env: "local", code: 3 },
|
|
411
|
+
{ kind: "waiting-on-point", op: "box-build", env: "local", point: "ship-now", question, state: "escalated" },
|
|
412
|
+
]);
|
|
413
|
+
|
|
414
|
+
// A person answers; the next round starts the run again beside the turns.
|
|
415
|
+
const answered = await answerPoint({ cwd: root, id: question, answer: "yes", by: ["alice"], on });
|
|
416
|
+
expect("error" in answered).toBe(false);
|
|
417
|
+
const third = await round();
|
|
418
|
+
expect(third).toEqual([expect.objectContaining({ kind: "beside-started", op: "box-build", why: "resumed", resumed: question })]);
|
|
419
|
+
await waitForBesideRuns(besideState);
|
|
420
|
+
expect(log).toEqual(["ship"]);
|
|
421
|
+
expect((await readRunLedger("local", "box-build", { cwd: root })).records.at(-1)).toMatchObject({ status: "ok", steward: "build-steward" });
|
|
422
|
+
expect((await readMemberStewards(root, "local", "2027-01-01T00:11:00Z")).stewards.find((s) => s.name === "build-steward")!.waiting).toEqual([]);
|
|
423
|
+
});
|
|
424
|
+
});
|