@stigmer/runner 3.14.0-dev.20260910084630 → 3.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/.build-fingerprint +1 -1
  2. package/dist/__test-utils__/hermetic-activity.d.ts +245 -0
  3. package/dist/__test-utils__/hermetic-activity.js +369 -0
  4. package/dist/__test-utils__/hermetic-activity.js.map +1 -0
  5. package/dist/__test-utils__/mock-client.d.ts +13 -0
  6. package/dist/__test-utils__/mock-client.js +45 -0
  7. package/dist/__test-utils__/mock-client.js.map +1 -0
  8. package/dist/activities/execute-cursor/__test-utils__/hermetic-cursor.d.ts +172 -0
  9. package/dist/activities/execute-cursor/__test-utils__/hermetic-cursor.js +331 -0
  10. package/dist/activities/execute-cursor/__test-utils__/hermetic-cursor.js.map +1 -0
  11. package/dist/activities/execute-cursor/__test-utils__/scripted-agent.d.ts +167 -0
  12. package/dist/activities/execute-cursor/__test-utils__/scripted-agent.js +239 -0
  13. package/dist/activities/execute-cursor/__test-utils__/scripted-agent.js.map +1 -0
  14. package/dist/activities/execute-cursor/__test-utils__/scripted-sdk.d.ts +97 -0
  15. package/dist/activities/execute-cursor/__test-utils__/scripted-sdk.js +132 -0
  16. package/dist/activities/execute-cursor/__test-utils__/scripted-sdk.js.map +1 -0
  17. package/dist/harness/capabilities.d.ts +71 -0
  18. package/dist/harness/capabilities.js +36 -0
  19. package/dist/harness/capabilities.js.map +1 -0
  20. package/dist/harness/registry.d.ts +67 -0
  21. package/dist/harness/registry.js +112 -0
  22. package/dist/harness/registry.js.map +1 -0
  23. package/dist/harness/types.d.ts +268 -0
  24. package/dist/harness/types.js +55 -0
  25. package/dist/harness/types.js.map +1 -0
  26. package/package.json +4 -4
  27. package/src/__test-utils__/__tests__/harness-contract-self-check.test.ts +229 -0
  28. package/src/__test-utils__/config-fixture.ts +63 -0
  29. package/src/__test-utils__/harness-contract/contract.ts +536 -0
  30. package/src/__test-utils__/harness-contract/recording-sink.ts +96 -0
  31. package/src/__test-utils__/harness-contract/scripted-adapter.ts +289 -0
  32. package/src/__test-utils__/harness-contract/types.ts +100 -0
  33. package/src/__test-utils__/hermetic-activity.ts +477 -0
  34. package/src/__test-utils__/proto-helpers.ts +25 -0
  35. package/src/__tests__/harness-contract.test.ts +25 -0
  36. package/src/activities/execute-cursor/__test-utils__/hermetic-cursor.ts +422 -0
  37. package/src/activities/execute-cursor/__test-utils__/scripted-agent.ts +342 -0
  38. package/src/activities/execute-cursor/__test-utils__/scripted-sdk.ts +166 -0
  39. package/src/activities/execute-cursor/__tests__/hermetic/deny-and-retry.test.ts +228 -0
  40. package/src/activities/execute-cursor/__tests__/hermetic/file-review-capture.test.ts +180 -0
  41. package/src/activities/execute-cursor/__tests__/hermetic/goldens/deny-and-retry.turn1.status.json +55 -0
  42. package/src/activities/execute-cursor/__tests__/hermetic/goldens/deny-and-retry.turn2.status.json +77 -0
  43. package/src/activities/execute-cursor/__tests__/hermetic/goldens/file-review-capture.status.json +126 -0
  44. package/src/activities/execute-cursor/__tests__/hermetic/goldens/pause.status.json +45 -0
  45. package/src/activities/execute-cursor/__tests__/hermetic/goldens/plain-turn.status.json +48 -0
  46. package/src/activities/execute-cursor/__tests__/hermetic/goldens/recovery-fresh-agent.status.json +53 -0
  47. package/src/activities/execute-cursor/__tests__/hermetic/goldens/tool-call.status.json +68 -0
  48. package/src/activities/execute-cursor/__tests__/hermetic/goldens/worker-shutdown.status.json +47 -0
  49. package/src/activities/execute-cursor/__tests__/hermetic/pause-vs-shutdown.test.ts +201 -0
  50. package/src/activities/execute-cursor/__tests__/hermetic/plain-turn.test.ts +171 -0
  51. package/src/activities/execute-cursor/__tests__/hermetic/recovery-fresh-agent.test.ts +156 -0
  52. package/src/activities/execute-cursor/__tests__/hermetic/tool-call.test.ts +137 -0
  53. package/src/harness/__tests__/registry.test.ts +167 -0
  54. package/src/harness/capabilities.ts +75 -0
  55. package/src/harness/registry.ts +123 -0
  56. package/src/harness/types.ts +278 -0
@@ -0,0 +1,536 @@
1
+ /**
2
+ * The harness adapter contract kit — the single authoritative statement of
3
+ * "these are THE things every `HarnessAdapter` must do," runnable against any
4
+ * {@link HarnessContractSubject}.
5
+ *
6
+ * Two halves exist in the program's design; this file is the ADAPTER-SIDE
7
+ * half: what an adapter owes the runtime through `runTurn`, `boot`,
8
+ * `shutdown` and `releaseSession`. The runtime-side half (the throw-vs-return
9
+ * table end to end, the whole-activity heartbeat, the single persist
10
+ * chokepoint, the byte-pinned copy) is proven against the real runtime once
11
+ * it exists, through the hermetic activity driver.
12
+ *
13
+ * The kit IS the runtime stand-in. {@link ExecutionDriver} does what the
14
+ * runtime and the server do around a turn: builds the `TurnInput`, threads
15
+ * the engine's state id (empty on an engine-minted harness's first turn, then
16
+ * the id the adapter bound; one fixed id for a deterministic harness),
17
+ * advances `turnSeq`, records a user's decision the way `SubmitApproval` does
18
+ * (on the persisted row's `approvalAction`, the one copy) and derives the
19
+ * decisions map from those rows on the next invocation, hands each
20
+ * reinvocation a CLONE of the previous status (the runtime persists and reads
21
+ * back; nothing survives by object identity), and owns the sink. The subject
22
+ * owns the engine.
23
+ *
24
+ * ── Invariant catalog ───────────────────────────────────────────────────────
25
+ * 1. Every exit is a `TurnOutcome`. `runTurn` resolves, never rejects, under
26
+ * every scenario kind and under a pre-aborted signal; a `CancelledFailure`
27
+ * never escapes (the runtime, not the adapter, throws it).
28
+ * 2. A proposal is a WAITING_APPROVAL row on `sink.status` when
29
+ * `awaiting_approval` resolves, and never executes in that turn.
30
+ * 3. Reinvoked with APPROVE for that id → executes exactly once; with REJECT
31
+ * or SKIP → never; reinvoked again after the approval → still once.
32
+ * 4. Aborting `stopSignal` mid-turn settles `runTurn` as `interrupted` within
33
+ * {@link INTERRUPT_SETTLE_BOUND_MS} and nothing after the stop executes; a
34
+ * signal aborted BEFORE `runTurn` yields `interrupted` with nothing done.
35
+ * 5. Usage reaches the sink as non-negative deltas summing to what the engine
36
+ * emitted.
37
+ * 6. Capability and behaviour agree on the state id: an `engine-minted`
38
+ * adapter binds before its first persist and resumes by the bound id, and
39
+ * surfaces a rejected bind as `failed` with nothing done after it; a
40
+ * `deterministic` adapter never binds.
41
+ * 7. Lifetimes: `boot` then `shutdown` resolve; `releaseSession` for a served
42
+ * session and for an unknown one both resolve; `shutdown` after a release
43
+ * resolves.
44
+ * 8. One adapter object serves concurrent turns independently: a hanging turn
45
+ * and a completing turn on the same adapter never cross outcomes or
46
+ * execution counts.
47
+ *
48
+ * Every invariant is an exported plain async function first and an `it`
49
+ * block second, so `__tests__/harness-contract-self-check.test.ts` can run
50
+ * each one against a deliberately broken adapter and prove it fires — a kit
51
+ * that cannot fail proves nothing. Every message names the subject.
52
+ *
53
+ * Above the contract line the pause primitives are indistinguishable: both
54
+ * end a turn `awaiting_approval`, both take the decisions on reinvocation.
55
+ * The kit therefore never branches on `capabilities.pausePrimitive`, and the
56
+ * runner test runs the fake under both to prove it.
57
+ */
58
+
59
+ import { describe, it, expect } from "vitest";
60
+ import { CancelledFailure } from "@temporalio/activity";
61
+ import { clone } from "@bufbuild/protobuf";
62
+ import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
63
+ import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
64
+ import { ApprovalAction, ToolCallStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
65
+
66
+ import type { TurnInput, TurnOutcome, UsageDelta } from "../../harness/types.js";
67
+ import type { ProposedAction } from "../approval-contract/types.js";
68
+ import { emptyStatus, findToolCallRow } from "../proto-helpers.js";
69
+ import { RecordingTurnSink } from "./recording-sink.js";
70
+ import { scenario } from "./types.js";
71
+ import type { HarnessContractSubject, TurnScenario } from "./types.js";
72
+
73
+ /**
74
+ * How long a turn may take to settle `interrupted` after its signal aborts.
75
+ * Generous against a saturated CI runner (the fake settles in microtasks; a
76
+ * real adapter cancels an SDK run), tight enough that an adapter parked on
77
+ * anything but the signal fails here rather than at Temporal's heartbeat
78
+ * timeout in production. A timeout race, never a sleep.
79
+ */
80
+ export const INTERRUPT_SETTLE_BOUND_MS = 5_000;
81
+
82
+ const OUTCOME_KINDS = ["completed", "awaiting_approval", "failed", "interrupted"] as const satisfies readonly TurnOutcome["kind"][];
83
+
84
+ /** Representative gated action shared by the invariants. */
85
+ const WRITE_ALPHA: ProposedAction = { kind: "write", resource: "/work/alpha.txt" };
86
+
87
+ // ── The runtime (and server) stand-in ───────────────────────────────────────
88
+
89
+ export interface TurnRun {
90
+ readonly input: TurnInput;
91
+ readonly sink: RecordingTurnSink;
92
+ readonly outcome: TurnOutcome;
93
+ }
94
+
95
+ export interface TurnOptions {
96
+ /** A prepared sink (e.g. one whose bind rejects); defaults to a fresh one over the threaded status. */
97
+ readonly sink?: RecordingTurnSink;
98
+ /** Called synchronously once `runTurn` has been entered, with the live sink — the hook that stops a turn mid-flight. */
99
+ readonly onStarted?: (sink: RecordingTurnSink) => void;
100
+ }
101
+
102
+ export interface TurnInFlight {
103
+ readonly input: TurnInput;
104
+ readonly sink: RecordingTurnSink;
105
+ readonly settled: Promise<TurnOutcome>;
106
+ }
107
+
108
+ /**
109
+ * The runtime's one reader of approval decisions, as the two harnesses agree
110
+ * on it today: every tool-call row still WAITING_APPROVAL whose
111
+ * `approvalAction` the server has set. The runtime extraction exports this
112
+ * from `src/harness/` and this kit imports it from there; until then the kit
113
+ * carries the definition so the data flow it drives is the real one.
114
+ */
115
+ function approvalDecisionsOf(status: AgentExecutionStatus): ReadonlyMap<string, ApprovalAction> {
116
+ const decisions = new Map<string, ApprovalAction>();
117
+ for (const message of status.messages) {
118
+ for (const row of message.toolCalls) {
119
+ if (row.status === ToolCallStatus.TOOL_CALL_WAITING_APPROVAL && row.approvalAction !== ApprovalAction.UNSPECIFIED) {
120
+ decisions.set(row.id, row.approvalAction);
121
+ }
122
+ }
123
+ }
124
+ return decisions;
125
+ }
126
+
127
+ /**
128
+ * One execution's worth of turns against one subject, doing the runtime's
129
+ * and the server's bookkeeping between them. A new driver per test: it
130
+ * carries the threaded state id and the persisted status across
131
+ * reinvocations.
132
+ */
133
+ export class ExecutionDriver {
134
+ readonly executionId: string;
135
+ readonly sessionId: string;
136
+ private threadId: string;
137
+ private turnSeq = 0;
138
+ private persisted: AgentExecutionStatus | undefined;
139
+
140
+ constructor(private readonly subject: HarnessContractSubject, label: string) {
141
+ this.executionId = `exec-${label}`;
142
+ this.sessionId = `ses-${label}`;
143
+ // A deterministic harness's id is the runtime's, known before any engine
144
+ // exists (native's `thread-{sessionId}`); an engine-minted one has nothing
145
+ // until the adapter binds.
146
+ this.threadId = subject.adapter.capabilities.stateIdSource === "deterministic" ? `thread-${this.sessionId}` : "";
147
+ }
148
+
149
+ /** Start a turn without awaiting it, for invariants about turns in flight. */
150
+ begin(turn: TurnScenario, options: TurnOptions = {}): TurnInFlight {
151
+ this.subject.arrange(turn);
152
+ const sink = options.sink ?? new RecordingTurnSink({ status: this.seedStatus() });
153
+ const input: TurnInput = {
154
+ executionId: this.executionId,
155
+ threadId: this.threadId,
156
+ turnSeq: this.turnSeq,
157
+ sessionId: this.sessionId,
158
+ approvalDecisions: approvalDecisionsOf(sink.status),
159
+ };
160
+ const settled = this.subject.adapter.runTurn(input, sink);
161
+ options.onStarted?.(sink);
162
+ return { input, sink, settled };
163
+ }
164
+
165
+ /** Run a turn to its outcome and do the runtime's bookkeeping after it. */
166
+ async turn(turn: TurnScenario, options: TurnOptions = {}): Promise<TurnRun> {
167
+ const { input, sink, settled } = this.begin(turn, options);
168
+ const outcome = await settled;
169
+ this.recordTurn(sink);
170
+ return { input, sink, outcome };
171
+ }
172
+
173
+ /** The runtime's post-turn bookkeeping: persist the status, adopt a bound id, advance the cycle. */
174
+ recordTurn(sink: RecordingTurnSink): void {
175
+ this.persisted = sink.status;
176
+ const bound = sink.boundStateIds.at(-1);
177
+ if (bound !== undefined) this.threadId = bound;
178
+ this.turnSeq += 1;
179
+ }
180
+
181
+ /**
182
+ * The server's act between invocations: `SubmitApproval` writes the user's
183
+ * verdict onto the WAITING row it owns. A decision on a row that does not
184
+ * exist or is not waiting is a test bug, not a contract case.
185
+ */
186
+ decide(toolCallId: string, action: ApprovalAction): void {
187
+ const row = this.persisted ? findToolCallRow(this.persisted, toolCallId) : undefined;
188
+ if (!row || row.status !== ToolCallStatus.TOOL_CALL_WAITING_APPROVAL) {
189
+ throw new Error(`${this.subject.name}: kit bug — decide('${toolCallId}') but no WAITING row was persisted`);
190
+ }
191
+ row.approvalAction = action;
192
+ }
193
+
194
+ /** The status a reinvocation is seeded with: a clone of what was persisted. */
195
+ seedStatus(): AgentExecutionStatus {
196
+ return this.persisted ? clone(AgentExecutionStatusSchema, this.persisted) : emptyStatus();
197
+ }
198
+ }
199
+
200
+ // ── Shared assertion helpers ────────────────────────────────────────────────
201
+
202
+ /**
203
+ * Await a turn, translating a rejection into a contract violation that names
204
+ * the subject and the kind of throw. A `CancelledFailure` gets named
205
+ * specially because throwing it is the one thing an adapter is most tempted
206
+ * to do and must never do.
207
+ */
208
+ async function settleAsOutcome(subject: HarnessContractSubject, settled: Promise<TurnOutcome>, when: string): Promise<TurnOutcome> {
209
+ let outcome: TurnOutcome;
210
+ try {
211
+ outcome = await settled;
212
+ } catch (err) {
213
+ const kind = err instanceof CancelledFailure ? "a CancelledFailure" : `an exception (${err instanceof Error ? err.message : String(err)})`;
214
+ throw new Error(`${subject.name}: runTurn rejected with ${kind} ${when}; every exit must be a TurnOutcome`);
215
+ }
216
+ expect(OUTCOME_KINDS, `${subject.name}: runTurn settled with an unknown outcome kind ${when}`).toContain(outcome.kind);
217
+ return outcome;
218
+ }
219
+
220
+ /** Race a settlement against the interrupt bound; the timer is cleared on settle so a passing test holds nothing. */
221
+ async function settleWithinBound<T>(subject: HarnessContractSubject, settled: Promise<T>, when: string, boundMs: number): Promise<T> {
222
+ let timer: NodeJS.Timeout | undefined;
223
+ const bound = new Promise<never>((_, reject) => {
224
+ timer = setTimeout(
225
+ () => reject(new Error(`${subject.name}: runTurn did not settle within ${boundMs}ms ${when}; every adapter call must be bounded by stopSignal`)),
226
+ boundMs,
227
+ );
228
+ });
229
+ try {
230
+ return await Promise.race([settled, bound]);
231
+ } finally {
232
+ if (timer) clearTimeout(timer);
233
+ }
234
+ }
235
+
236
+ /** Stop the turn on the next macrotask, after the adapter has reached whatever it was going to park on. */
237
+ function stopSoon(reason: string): (sink: RecordingTurnSink) => void {
238
+ return (sink) => {
239
+ setImmediate(() => sink.abort(reason));
240
+ };
241
+ }
242
+
243
+ async function expectResolves(subject: HarnessContractSubject, call: Promise<void>, what: string): Promise<void> {
244
+ try {
245
+ await call;
246
+ } catch (err) {
247
+ throw new Error(`${subject.name}: ${what} rejected (${err instanceof Error ? err.message : String(err)}); it must resolve`);
248
+ }
249
+ }
250
+
251
+ // ── Invariant 1 ─────────────────────────────────────────────────────────────
252
+
253
+ /**
254
+ * Invariant 1: every exit is a `TurnOutcome`. Each scenario kind is played
255
+ * once, the hanging one is stopped from the outside, and one turn is entered
256
+ * with an already-aborted signal; none may reject, none may settle with an
257
+ * unknown kind.
258
+ */
259
+ export async function assertEveryExitIsAnOutcome(subject: HarnessContractSubject, boundMs = INTERRUPT_SETTLE_BOUND_MS): Promise<void> {
260
+ const driver = new ExecutionDriver(subject, "inv1");
261
+ const plays: ReadonlyArray<{ readonly when: string; readonly turn: TurnScenario; readonly options?: TurnOptions }> = [
262
+ { when: "on a plain text turn", turn: [scenario.say("hello")] },
263
+ { when: "on a usage-only turn", turn: [scenario.usage({ inputTokens: 1, outputTokens: 1 })] },
264
+ { when: "on an undecided proposal", turn: [scenario.propose("inv1-call", WRITE_ALPHA)] },
265
+ { when: "on an engine failure", turn: [scenario.fail("engine exploded")] },
266
+ { when: "on a hang stopped from the outside", turn: [scenario.hang()], options: { onStarted: stopSoon("kit: stop") } },
267
+ ];
268
+ for (const play of plays) {
269
+ const { sink, settled } = driver.begin(play.turn, play.options);
270
+ await settleAsOutcome(subject, settleWithinBound(subject, settled, play.when, boundMs), play.when);
271
+ driver.recordTurn(sink);
272
+ }
273
+
274
+ const preAborted = new RecordingTurnSink({ status: driver.seedStatus() });
275
+ preAborted.abort("kit: aborted before the turn");
276
+ const { settled } = driver.begin([scenario.say("never")], { sink: preAborted });
277
+ await settleAsOutcome(subject, settleWithinBound(subject, settled, "under a pre-aborted signal", boundMs), "under a pre-aborted signal");
278
+ }
279
+
280
+ // ── Invariant 2 ─────────────────────────────────────────────────────────────
281
+
282
+ /**
283
+ * Invariant 2: a proposal is a WAITING_APPROVAL row and never executes in its
284
+ * own turn. The row must carry `requiresApproval` and an UNSPECIFIED
285
+ * `approvalAction` (the server's field, untouched by the adapter), and the
286
+ * turn must end `awaiting_approval` — not `completed`, which would tell the
287
+ * workflow nothing is pending.
288
+ */
289
+ export async function assertProposalIsWaitingAndUnexecuted(subject: HarnessContractSubject): Promise<void> {
290
+ const driver = new ExecutionDriver(subject, "inv2");
291
+ const id = "inv2-write";
292
+ const { outcome, sink } = await driver.turn([scenario.say("about to write"), scenario.propose(id, WRITE_ALPHA), scenario.say("after")]);
293
+
294
+ expect(outcome.kind, `${subject.name}: a turn that proposes an undecided action must end awaiting_approval`).toBe("awaiting_approval");
295
+ const row = findToolCallRow(sink.status, id);
296
+ expect(row, `${subject.name}: the proposal must be a tool-call row on sink.status when awaiting_approval resolves`).toBeDefined();
297
+ expect(row?.status, `${subject.name}: the proposal's row must be WAITING_APPROVAL`).toBe(ToolCallStatus.TOOL_CALL_WAITING_APPROVAL);
298
+ expect(row?.requiresApproval, `${subject.name}: the proposal's row must carry requiresApproval`).toBe(true);
299
+ expect(row?.approvalAction, `${subject.name}: the adapter must never write the server's approvalAction`).toBe(ApprovalAction.UNSPECIFIED);
300
+ expect(subject.executionCount(id), `${subject.name}: an undecided proposal must not execute in its own turn`).toBe(0);
301
+ }
302
+
303
+ // ── Invariant 3 ─────────────────────────────────────────────────────────────
304
+
305
+ /**
306
+ * Invariant 3: the decision is honoured exactly. APPROVE executes once and
307
+ * only once, however many times the same decided row is seen again; REJECT
308
+ * and SKIP never execute and carry the row to SKIPPED (the enum's own
309
+ * transition). The reinvocation sees a CLONE of the persisted status, as it
310
+ * would from the server.
311
+ */
312
+ export async function assertDecisionsExecuteExactlyOnce(subject: HarnessContractSubject): Promise<void> {
313
+ const driver = new ExecutionDriver(subject, "inv3");
314
+
315
+ const approved = "inv3-approve";
316
+ const first = await driver.turn([scenario.propose(approved, WRITE_ALPHA)]);
317
+ expect(first.outcome.kind, `${subject.name}: proposal must end awaiting_approval`).toBe("awaiting_approval");
318
+ driver.decide(approved, ApprovalAction.APPROVE);
319
+
320
+ const resumed = await driver.turn([scenario.propose(approved, WRITE_ALPHA), scenario.say("done")]);
321
+ expect(resumed.outcome.kind, `${subject.name}: after APPROVE the turn must run to completion`).toBe("completed");
322
+ expect(subject.executionCount(approved), `${subject.name}: an approved action must execute exactly once`).toBe(1);
323
+ expect(findToolCallRow(resumed.sink.status, approved)?.status, `${subject.name}: the approved row must be carried to COMPLETED`).toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
324
+
325
+ const again = await driver.turn([scenario.propose(approved, WRITE_ALPHA), scenario.say("again")]);
326
+ expect(again.outcome.kind, `${subject.name}: a settled proposal seen again must not re-gate`).toBe("completed");
327
+ expect(subject.executionCount(approved), `${subject.name}: reinvoked again after the approval, the action must still have executed exactly once`).toBe(1);
328
+
329
+ for (const [label, action] of [["REJECT", ApprovalAction.REJECT], ["SKIP", ApprovalAction.SKIP]] as const) {
330
+ const id = `inv3-${label.toLowerCase()}`;
331
+ const proposed = await driver.turn([scenario.propose(id, WRITE_ALPHA)]);
332
+ expect(proposed.outcome.kind, `${subject.name}: proposal must end awaiting_approval`).toBe("awaiting_approval");
333
+ driver.decide(id, action);
334
+ const decided = await driver.turn([scenario.propose(id, WRITE_ALPHA), scenario.say("moving on")]);
335
+ expect(decided.outcome.kind, `${subject.name}: after ${label} the turn must continue to completion`).toBe("completed");
336
+ expect(subject.executionCount(id), `${subject.name}: a ${label}-ed action must never execute`).toBe(0);
337
+ expect(findToolCallRow(decided.sink.status, id)?.status, `${subject.name}: a ${label}-ed row must be carried to SKIPPED`).toBe(ToolCallStatus.TOOL_CALL_SKIPPED);
338
+ }
339
+ }
340
+
341
+ // ── Invariant 4 ─────────────────────────────────────────────────────────────
342
+
343
+ /**
344
+ * Invariant 4: the stop signal is honoured promptly and completely. A turn
345
+ * stopped while hanging settles `interrupted` within the bound and executes
346
+ * nothing that came after the hang, even an already-approved action; a turn
347
+ * entered under an aborted signal does nothing at all.
348
+ */
349
+ export async function assertStopSignalInterrupts(subject: HarnessContractSubject, boundMs = INTERRUPT_SETTLE_BOUND_MS): Promise<void> {
350
+ const driver = new ExecutionDriver(subject, "inv4");
351
+ const id = "inv4-approved-after-hang";
352
+
353
+ const proposed = await driver.turn([scenario.propose(id, WRITE_ALPHA)]);
354
+ expect(proposed.outcome.kind, `${subject.name}: proposal must end awaiting_approval`).toBe("awaiting_approval");
355
+ driver.decide(id, ApprovalAction.APPROVE);
356
+
357
+ const hanging = driver.begin([scenario.say("working"), scenario.hang(), scenario.propose(id, WRITE_ALPHA)], { onStarted: stopSoon("kit: user pause") });
358
+ const outcome = await settleWithinBound(subject, hanging.settled, "after stopSignal aborted mid-hang", boundMs);
359
+ expect(outcome.kind, `${subject.name}: a turn stopped mid-hang must settle interrupted`).toBe("interrupted");
360
+ expect(subject.executionCount(id), `${subject.name}: nothing after the stop may execute, even an approved action`).toBe(0);
361
+ driver.recordTurn(hanging.sink);
362
+
363
+ const preAborted = new RecordingTurnSink({ status: driver.seedStatus() });
364
+ preAborted.abort("kit: aborted before the turn");
365
+ const early = driver.begin([scenario.propose(id, WRITE_ALPHA)], { sink: preAborted });
366
+ const earlyOutcome = await settleWithinBound(subject, early.settled, "under a pre-aborted signal", boundMs);
367
+ expect(earlyOutcome.kind, `${subject.name}: a turn entered under an aborted signal must settle interrupted`).toBe("interrupted");
368
+ expect(subject.executionCount(id), `${subject.name}: a turn entered under an aborted signal must do no work`).toBe(0);
369
+ expect(preAborted.persistRequests, `${subject.name}: a turn entered under an aborted signal must not ask to persist`).toBe(0);
370
+ }
371
+
372
+ // ── Invariant 5 ─────────────────────────────────────────────────────────────
373
+
374
+ function sumUsage(deltas: readonly UsageDelta[]): Required<UsageDelta> {
375
+ const total = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
376
+ for (const d of deltas) {
377
+ total.inputTokens += d.inputTokens ?? 0;
378
+ total.outputTokens += d.outputTokens ?? 0;
379
+ total.cacheReadTokens += d.cacheReadTokens ?? 0;
380
+ total.cacheWriteTokens += d.cacheWriteTokens ?? 0;
381
+ }
382
+ return total;
383
+ }
384
+
385
+ /**
386
+ * Invariant 5: usage reaches the sink as deltas — non-negative, and summing
387
+ * to what the engine emitted. An adapter that reports cumulative totals
388
+ * instead of deltas would double-count in the runtime's accumulator and trip
389
+ * the cost cap early; one that reports a negative number would credit it.
390
+ */
391
+ export async function assertUsageReachesSinkAsDeltas(subject: HarnessContractSubject): Promise<void> {
392
+ const driver = new ExecutionDriver(subject, "inv5");
393
+ const emitted: readonly UsageDelta[] = [
394
+ { inputTokens: 10, outputTokens: 5 },
395
+ { inputTokens: 3, cacheReadTokens: 2 },
396
+ { outputTokens: 4, cacheWriteTokens: 1 },
397
+ ];
398
+ const { outcome, sink } = await driver.turn([...emitted.map((d) => scenario.usage(d)), scenario.say("done")]);
399
+ expect(outcome.kind, `${subject.name}: a usage-reporting turn must complete`).toBe("completed");
400
+
401
+ for (const delta of sink.usageDeltas) {
402
+ for (const [field, value] of Object.entries(delta)) {
403
+ expect(value, `${subject.name}: usage delta field ${field} must be non-negative`).toBeGreaterThanOrEqual(0);
404
+ }
405
+ }
406
+ expect(sumUsage(sink.usageDeltas), `${subject.name}: the usage deltas must sum to what the engine emitted`).toEqual(sumUsage(emitted));
407
+ }
408
+
409
+ // ── Invariant 6 ─────────────────────────────────────────────────────────────
410
+
411
+ /**
412
+ * Invariant 6: capability and behaviour agree on the state id. Engine-minted:
413
+ * the bind precedes the first persist (so a crash mid-turn still resumes),
414
+ * the id threaded back on the next invocation is accepted, and a rejected
415
+ * bind ends the turn `failed` with nothing done after it. Deterministic: the
416
+ * adapter never binds and accepts the runtime's id on every turn.
417
+ */
418
+ export async function assertStateIdCapabilityAgrees(subject: HarnessContractSubject): Promise<void> {
419
+ const driver = new ExecutionDriver(subject, "inv6");
420
+ const first = await driver.turn([scenario.say("first turn")]);
421
+ expect(first.outcome.kind, `${subject.name}: a plain turn must complete`).toBe("completed");
422
+
423
+ if (subject.adapter.capabilities.stateIdSource === "engine-minted") {
424
+ expect(first.input.threadId, `${subject.name}: an engine-minted harness has no state id before its first turn`).toBe("");
425
+ const events = first.sink.events;
426
+ const firstBind = events.findIndex((e) => e.kind === "bind");
427
+ const firstPersist = events.findIndex((e) => e.kind === "persist");
428
+ expect(firstBind, `${subject.name}: an engine-minted adapter must bind its state id on the first turn`).toBeGreaterThanOrEqual(0);
429
+ expect(first.sink.boundStateIds[0], `${subject.name}: the bound state id must be non-empty`).toBeTruthy();
430
+ if (firstPersist >= 0) {
431
+ expect(firstBind, `${subject.name}: the state id must be bound BEFORE the first persist request, or a crash mid-turn cannot resume`).toBeLessThan(firstPersist);
432
+ }
433
+
434
+ const second = await driver.turn([scenario.say("second turn")]);
435
+ expect(second.input.threadId, `${subject.name}: the kit threads the bound id back as threadId`).toBe(first.sink.boundStateIds[0]);
436
+ expect(second.outcome.kind, `${subject.name}: resuming by the id the adapter bound must complete`).toBe("completed");
437
+ expect(second.sink.boundStateIds, `${subject.name}: a resumed turn must not bind a new state id`).toHaveLength(0);
438
+
439
+ const failing = new ExecutionDriver(subject, "inv6-bind-rejects");
440
+ const rejecting = new RecordingTurnSink({ bindRejectsWith: new Error("session write failed") });
441
+ const rejected = await failing.turn([scenario.say("never persisted")], { sink: rejecting });
442
+ expect(rejected.outcome.kind, `${subject.name}: a rejected bind must end the turn failed, not reject runTurn`).toBe("failed");
443
+ expect(rejecting.persistRequests, `${subject.name}: nothing may be persisted after a rejected bind`).toBe(0);
444
+ expect(rejecting.status.messages, `${subject.name}: nothing may be folded into the status after a rejected bind`).toHaveLength(0);
445
+ } else {
446
+ expect(first.input.threadId, `${subject.name}: a deterministic harness is handed the runtime's id on its first turn`).not.toBe("");
447
+ expect(first.sink.boundStateIds, `${subject.name}: a deterministic adapter must never bind a state id`).toHaveLength(0);
448
+ const second = await driver.turn([scenario.say("second turn")]);
449
+ expect(second.input.threadId, `${subject.name}: a deterministic id is the same on every turn`).toBe(first.input.threadId);
450
+ expect(second.outcome.kind, `${subject.name}: a deterministic resume must complete`).toBe("completed");
451
+ expect(second.sink.boundStateIds, `${subject.name}: a deterministic adapter must never bind a state id`).toHaveLength(0);
452
+ }
453
+ }
454
+
455
+ // ── Invariant 7 ─────────────────────────────────────────────────────────────
456
+
457
+ /**
458
+ * Invariant 7: the three lifetimes resolve. Boot with the subject's config,
459
+ * serve a turn, release the served session and an unknown one, shut down.
460
+ * Each must resolve — a rejection in any would fail a worker's boot, leak a
461
+ * session's resources, or hang a drain.
462
+ */
463
+ export async function assertLifetimesResolve(subject: HarnessContractSubject): Promise<void> {
464
+ const { adapter } = subject;
465
+ await expectResolves(subject, adapter.boot(subject.config), "boot(config)");
466
+
467
+ const driver = new ExecutionDriver(subject, "inv7");
468
+ const served = await driver.turn([scenario.say("served")]);
469
+ expect(served.outcome.kind, `${subject.name}: a plain turn must complete`).toBe("completed");
470
+
471
+ await expectResolves(subject, adapter.releaseSession(served.input.sessionId), `releaseSession('${served.input.sessionId}') for a served session`);
472
+ await expectResolves(subject, adapter.releaseSession("ses-never-served"), "releaseSession for an unknown session");
473
+ await expectResolves(subject, adapter.shutdown(), "shutdown() after release");
474
+ }
475
+
476
+ // ── Invariant 8 ─────────────────────────────────────────────────────────────
477
+
478
+ /**
479
+ * Invariant 8: one adapter object, many turns. While one execution's turn
480
+ * hangs, another execution's turns propose, get decided and execute on the
481
+ * SAME adapter; stopping the hanging turn settles only it; nothing crosses.
482
+ * This is "the adapter holds no per-turn state" made observable — the
483
+ * property `maxConcurrentActivities` relies on.
484
+ */
485
+ export async function assertConcurrentTurnsAreIndependent(subject: HarnessContractSubject, boundMs = INTERRUPT_SETTLE_BOUND_MS): Promise<void> {
486
+ const hangingExecution = new ExecutionDriver(subject, "inv8-hanging");
487
+ const busyExecution = new ExecutionDriver(subject, "inv8-busy");
488
+ const id = "inv8-busy-write";
489
+
490
+ const hanging = hangingExecution.begin([scenario.say("parked"), scenario.hang()]);
491
+
492
+ const proposed = await settleWithinBound(subject, busyExecution.turn([scenario.propose(id, WRITE_ALPHA)]), "on a second execution while another turn hangs", boundMs);
493
+ expect(proposed.outcome.kind, `${subject.name}: a second execution's proposal must settle while another turn hangs`).toBe("awaiting_approval");
494
+ busyExecution.decide(id, ApprovalAction.APPROVE);
495
+ const resumed = await settleWithinBound(subject, busyExecution.turn([scenario.propose(id, WRITE_ALPHA)]), "on a second execution's reinvocation while another turn hangs", boundMs);
496
+ expect(resumed.outcome.kind, `${subject.name}: a second execution's approved turn must complete while another turn hangs`).toBe("completed");
497
+ expect(subject.executionCount(id), `${subject.name}: the busy execution's action must execute exactly once`).toBe(1);
498
+
499
+ hanging.sink.abort("kit: stop the parked turn");
500
+ const stopped = await settleWithinBound(subject, hanging.settled, "after the hanging turn was stopped", boundMs);
501
+ expect(stopped.kind, `${subject.name}: the hanging turn must settle interrupted, and only it`).toBe("interrupted");
502
+ expect(subject.executionCount(id), `${subject.name}: stopping one execution must not touch another's execution count`).toBe(1);
503
+ expect(findToolCallRow(hanging.sink.status, id), `${subject.name}: one execution's rows must never appear on another's status`).toBeUndefined();
504
+ }
505
+
506
+ // ── The runnable suite ──────────────────────────────────────────────────────
507
+
508
+ /** Register the adapter-side contract against one subject. */
509
+ export function describeHarnessContract(subject: HarnessContractSubject): void {
510
+ describe(`harness contract — ${subject.name}`, () => {
511
+ it("settles every turn with a TurnOutcome and never rejects (invariant 1)", async () => {
512
+ await assertEveryExitIsAnOutcome(subject);
513
+ });
514
+ it("surfaces a proposal as a WAITING row and never executes it in its turn (invariant 2)", async () => {
515
+ await assertProposalIsWaitingAndUnexecuted(subject);
516
+ });
517
+ it("executes an approval exactly once and a rejection or skip never (invariant 3)", async () => {
518
+ await assertDecisionsExecuteExactlyOnce(subject);
519
+ });
520
+ it("settles interrupted promptly when stopSignal aborts and does nothing after it (invariant 4)", async () => {
521
+ await assertStopSignalInterrupts(subject);
522
+ });
523
+ it("reports usage as non-negative deltas that sum to what the engine emitted (invariant 5)", async () => {
524
+ await assertUsageReachesSinkAsDeltas(subject);
525
+ });
526
+ it(`binds its state id as its ${subject.adapter.capabilities.stateIdSource} capability declares (invariant 6)`, async () => {
527
+ await assertStateIdCapabilityAgrees(subject);
528
+ });
529
+ it("boots, serves, releases and shuts down without a rejection (invariant 7)", async () => {
530
+ await assertLifetimesResolve(subject);
531
+ });
532
+ it("serves concurrent turns on one adapter object independently (invariant 8)", async () => {
533
+ await assertConcurrentTurnsAreIndependent(subject);
534
+ });
535
+ });
536
+ }
@@ -0,0 +1,96 @@
1
+ /**
2
+ * A `TurnSink` that records everything an adapter asks of the runtime.
3
+ *
4
+ * The kit is the runtime stand-in, and this is the runtime's face during one
5
+ * turn: it owns the `AbortController` behind `stopSignal` (so a test can stop
6
+ * the turn from the outside, exactly as the runtime's watchdog, accounting or
7
+ * heartbeat would), and it keeps ONE ordered log of every call the adapter
8
+ * made. Counts and lists are derived from the log on read rather than kept
9
+ * as parallel counters, because the invariants care about ORDER as much as
10
+ * about counts ("bind before the first persist" is an ordering claim).
11
+ *
12
+ * `bindHarnessState` can be made to reject, standing in for a failed session
13
+ * write, so the kit can prove an adapter surfaces a runtime-side failure as
14
+ * `failed` instead of letting it escape `runTurn`.
15
+ *
16
+ * One sink per turn, like the runtime. A reinvocation gets a NEW sink whose
17
+ * `status` is a clone of the previous turn's — the runtime persists and reads
18
+ * back, so nothing survives between turns by object identity.
19
+ */
20
+
21
+ import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
22
+
23
+ import type { TurnSink, UsageDelta } from "../../harness/types.js";
24
+ import { emptyStatus } from "../proto-helpers.js";
25
+
26
+ /** One call the adapter made on the sink, in the order it made it. */
27
+ export type SinkEvent =
28
+ | { readonly kind: "persist" }
29
+ | { readonly kind: "activity" }
30
+ | { readonly kind: "usage"; readonly delta: UsageDelta }
31
+ | { readonly kind: "bind"; readonly harnessStateId: string };
32
+
33
+ export interface RecordingTurnSinkOptions {
34
+ /** The status this turn folds into; a fresh empty one when omitted. */
35
+ readonly status?: AgentExecutionStatus;
36
+ /** When set, `bindHarnessState` rejects with this error (a failed session write). */
37
+ readonly bindRejectsWith?: Error;
38
+ }
39
+
40
+ export class RecordingTurnSink implements TurnSink {
41
+ readonly status: AgentExecutionStatus;
42
+ readonly stopSignal: AbortSignal;
43
+
44
+ private readonly controller = new AbortController();
45
+ private readonly log: SinkEvent[] = [];
46
+ private readonly bindRejectsWith: Error | undefined;
47
+
48
+ constructor(options: RecordingTurnSinkOptions = {}) {
49
+ this.status = options.status ?? emptyStatus();
50
+ this.stopSignal = this.controller.signal;
51
+ this.bindRejectsWith = options.bindRejectsWith;
52
+ }
53
+
54
+ /** Stop the turn from the outside, the way the runtime would. */
55
+ abort(reason?: unknown): void {
56
+ this.controller.abort(reason);
57
+ }
58
+
59
+ requestPersist(): void {
60
+ this.log.push({ kind: "persist" });
61
+ }
62
+
63
+ recordActivity(): void {
64
+ this.log.push({ kind: "activity" });
65
+ }
66
+
67
+ reportUsage(delta: UsageDelta): void {
68
+ this.log.push({ kind: "usage", delta });
69
+ }
70
+
71
+ async bindHarnessState(harnessStateId: string): Promise<void> {
72
+ if (this.bindRejectsWith) throw this.bindRejectsWith;
73
+ this.log.push({ kind: "bind", harnessStateId });
74
+ }
75
+
76
+ /** Every call, in order. */
77
+ get events(): readonly SinkEvent[] {
78
+ return this.log;
79
+ }
80
+
81
+ get persistRequests(): number {
82
+ return this.log.filter((e) => e.kind === "persist").length;
83
+ }
84
+
85
+ get activityMarks(): number {
86
+ return this.log.filter((e) => e.kind === "activity").length;
87
+ }
88
+
89
+ get usageDeltas(): readonly UsageDelta[] {
90
+ return this.log.flatMap((e) => (e.kind === "usage" ? [e.delta] : []));
91
+ }
92
+
93
+ get boundStateIds(): readonly string[] {
94
+ return this.log.flatMap((e) => (e.kind === "bind" ? [e.harnessStateId] : []));
95
+ }
96
+ }