@jr2/orchestrator 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/bin/server.ts +7 -4
  2. package/console/canvas.ts +27 -14
  3. package/console/components/app.ts +10 -1
  4. package/console/components/fleet.ts +39 -6
  5. package/console/components/machine-pane.ts +89 -2
  6. package/console/components/nav.ts +1 -1
  7. package/console/components/schema-form.ts +12 -4
  8. package/console/json-field.ts +23 -0
  9. package/console/main.ts +110 -39
  10. package/console/store.ts +221 -17
  11. package/console/style.css +90 -0
  12. package/package.json +4 -4
  13. package/src/actor.ts +502 -49
  14. package/src/ambient.ts +2 -0
  15. package/src/assembly-checks.ts +165 -0
  16. package/src/assembly-tar.ts +494 -0
  17. package/src/assembly.ts +721 -0
  18. package/src/chart.ts +2050 -0
  19. package/src/charts.ts +720 -0
  20. package/src/config.ts +152 -46
  21. package/src/content-hash.ts +105 -0
  22. package/src/custodian.lua +6 -5
  23. package/src/custodian.ts +71 -19
  24. package/src/customize.ts +216 -25
  25. package/src/effect-checks.ts +557 -0
  26. package/src/effect.ts +905 -0
  27. package/src/effector-wire.ts +525 -0
  28. package/src/fingerprint.ts +4 -3
  29. package/src/gate.ts +8 -0
  30. package/src/git-clone.ts +65 -0
  31. package/src/git-push.ts +336 -0
  32. package/src/harness-client.ts +245 -23
  33. package/src/held-secrets.ts +82 -26
  34. package/src/holds.ts +156 -0
  35. package/src/http-effect.ts +981 -0
  36. package/src/http.ts +496 -104
  37. package/src/images.ts +47 -21
  38. package/src/index.ts +43 -2
  39. package/src/instance.ts +444 -58
  40. package/src/kube-client.ts +333 -0
  41. package/src/machine-doc.ts +88 -6
  42. package/src/mount-digest.ts +53 -0
  43. package/src/names.ts +66 -8
  44. package/src/notices.ts +88 -0
  45. package/src/open-body.ts +1444 -0
  46. package/src/open-effect.ts +144 -0
  47. package/src/open.ts +4 -4
  48. package/src/parts.ts +530 -33
  49. package/src/party.ts +468 -0
  50. package/src/pick.ts +309 -0
  51. package/src/pool.ts +1 -1
  52. package/src/registration.ts +301 -17
  53. package/src/repo-fetch.ts +60 -85
  54. package/src/repos.ts +63 -74
  55. package/src/rest.ts +133 -0
  56. package/src/run-host.ts +1832 -98
  57. package/src/sandbox-effect.ts +891 -0
  58. package/src/sandbox-effector.ts +111 -0
  59. package/src/sandbox-kube.ts +1719 -0
  60. package/src/sandbox-watch.ts +260 -0
  61. package/src/secret-placement.ts +494 -0
  62. package/src/server.ts +281 -84
  63. package/src/setup.ts +589 -70
  64. package/src/size.ts +342 -0
  65. package/src/snapshot-store.ts +64 -13
  66. package/src/stub-harness.ts +5 -0
  67. package/src/sync-client.ts +54 -0
  68. package/src/timers.ts +178 -0
  69. package/src/tokens.ts +35 -4
  70. package/src/vocabulary.ts +66 -1
  71. package/src/wire.ts +156 -2
  72. package/src/workspace.ts +529 -194
  73. package/src/sandbox-kubectl.ts +0 -1108
package/src/run-host.ts CHANGED
@@ -21,29 +21,82 @@
21
21
 
22
22
  import { randomUUID } from "node:crypto";
23
23
  import { z } from "zod";
24
- import { createActor, type AnyActor, type AnyActorLogic, type AnyActorRef, type AnyStateMachine } from "xstate";
25
- import type { EventSemantics } from "@jr2/agent-protocol";
24
+ import {
25
+ createActor,
26
+ type AnyActor,
27
+ type AnyActorLogic,
28
+ type AnyActorRef,
29
+ type AnyEventObject,
30
+ type AnyMachineSnapshot,
31
+ type AnyStateMachine,
32
+ } from "xstate";
33
+ import { SENDER_KEY, type EventDef, type EventSemantics, type Sender } from "@jr2/agent-protocol";
34
+ import { partiesAt, seatOf, senderText } from "./party.ts";
35
+ import { placeAnswers, type AnswerSeat } from "./pick.ts";
26
36
  import { inputSchemaOf } from "./vocabulary.ts";
27
- import type { EchoEvent, EchoStatusChild } from "./wire.ts";
37
+ import type { EchoEvent, EchoStatusChild, LiveConversation, PickAnswer } from "./wire.ts";
38
+ import { answerOf, childErrorOf, replyOf, type Hold } from "./holds.ts";
39
+ import { restOf, runningBelow } from "./rest.ts";
28
40
  import {
41
+ acceptsNow,
29
42
  agentAddress,
43
+ allowedPicks,
30
44
  bindRun,
31
45
  EventValidationError,
32
46
  gateAddress,
33
- mayMove,
47
+ NotTakenError,
48
+ OutstandingError,
34
49
  RegistrationTable,
50
+ refuseSenderField,
35
51
  UnknownAddressError,
36
52
  wouldMove,
53
+ targetsOf,
54
+ leftAgain,
55
+ type Registration,
37
56
  type RetryTelemetry,
38
57
  type RunBinding,
39
- type TurnMarker,
58
+ type RunMarker,
40
59
  } from "./registration.ts";
41
- import type { SandboxPort } from "./workspace.ts";
60
+ import { NoticeLedger, type NoticeLedgerState } from "./notices.ts";
61
+ import { pendingTimers, startAndRearm, watchTimers, type PendingTimer } from "./timers.ts";
62
+ import {
63
+ DEFAULT_LEASE_INTERVAL_MS,
64
+ WORKSPACE_WAIT_KEY,
65
+ leaseDelay,
66
+ workspaceName,
67
+ type SandboxPort,
68
+ type WorkspaceWait,
69
+ } from "./workspace.ts";
70
+ import { INSTANCE_HARNESS_SERVICE } from "./names.ts";
71
+ import { composesSandbox } from "./parts.ts";
42
72
  import { fingerprintOf } from "./fingerprint.ts";
43
- import { serializeMachine, type MachineDoc } from "./machine-doc.ts";
73
+ import {
74
+ serializeMachine,
75
+ type ChildMachineDoc,
76
+ type MachineBodyDoc,
77
+ type MachineDoc,
78
+ type MachineStateDoc,
79
+ } from "./machine-doc.ts";
44
80
  import type { SnapshotStore } from "./snapshot-store.ts";
45
81
  import type { AgentAdmission } from "./actor.ts";
82
+ import { dropSavedTails, type EffectRow } from "./effect.ts";
83
+ import type { EffectorPlacement, EffectorTarget } from "./http-effect.ts";
46
84
  import { reattachAgentRuns, serializeSnapshot } from "./durability.ts";
85
+ import { admitChart, chartBriefingOf, chartHash, ChartRefusal, isChartSchema, type ChartText } from "./chart.ts";
86
+ import type { ChartOrigin, NamedChart } from "./charts.ts";
87
+ import {
88
+ admitBodyChart,
89
+ admitWrapperChart,
90
+ boundChartRoutesOf,
91
+ chartWrapperMachineOf,
92
+ boundMachineOf,
93
+ compileChart,
94
+ isOpenBody,
95
+ lentIn,
96
+ openBodiesOf,
97
+ openBodyDeclarationOf,
98
+ type BoundChartRoute,
99
+ } from "./open-body.ts";
47
100
 
48
101
  /** The actors filled into a run's named slots — `.provide()` is the unit-test seam (ADR-0015);
49
102
  * production instances inject nothing. Built fresh at start and at restore. */
@@ -56,8 +109,111 @@ export type WorkflowDef = {
56
109
  machine: AnyStateMachine;
57
110
  /** Build this run's live providers. A test seam (ADR-0015): discovery injects nothing. */
58
111
  provide: (ctx: { instanceId: string }) => RunProviders;
112
+ /**
113
+ * The hash of the Assembly this Machine came from (ADR-0069): set at register for the current
114
+ * Assembly, and by `resolveDef` for a held one at restore. A run holds the Assembly it started
115
+ * under, so every save of the run stamps this hash. Absent only for a def that no Assembly
116
+ * loaded: an in-process test host, which stamps nothing.
117
+ */
118
+ assembly?: string;
119
+ /**
120
+ * The def is a Chart wrapper (ADR-0070): `body` is its (first) Open body's slot key, and
121
+ * `chartFields` are the door's fields that are `chartSchema` — empty for a wrapper whose Charts
122
+ * only its own states supply. `register` reads both off the Machine when the caller leaves this
123
+ * out.
124
+ */
125
+ wrapper?: { body: string; chartFields: string[] };
126
+ /**
127
+ * The def is a named Chart (ADR-0070): the Workflow `<wrapper>/<name>`, whose `machine` is its
128
+ * wrapper's. A start supplies every Chart field of the wrapper's door from `text` and takes the
129
+ * rest from the caller.
130
+ */
131
+ chart?: { wrapper: string; name: string; text: ChartText; hash: string; origin: ChartOrigin };
59
132
  };
60
133
 
134
+ /**
135
+ * One entry of `GET /workflows` (ADR-0070): a Workflow and where it came from. A Chart wrapper's
136
+ * entry carries its Open body, whether its door takes a Chart (`oneShot`), and its named Charts.
137
+ */
138
+ export type WorkflowListing = {
139
+ name: string;
140
+ origin: "machine" | ChartOrigin;
141
+ wrapper?: {
142
+ body: string;
143
+ oneShot: boolean;
144
+ charts: Array<{ name: string; origin: ChartOrigin; unrunnable?: string }>;
145
+ };
146
+ unrunnable?: string;
147
+ };
148
+
149
+ /** What makes a Machine a Chart wrapper (ADR-0070), read off it: its first Open body's slot key and
150
+ * the door's fields that are `chartSchema`. The bodies are seen through jr2's transparent wrappers
151
+ * (a Chart wrapper that is a `workspace()`'s body), and the door is the registered Machine's.
152
+ * Undefined for a Machine with no Open body. */
153
+ function wrapperOf(machine: AnyStateMachine): WorkflowDef["wrapper"] | undefined {
154
+ const bodies = openBodiesOf(chartWrapperMachineOf(machine).machine);
155
+ if (!bodies.length) return undefined;
156
+ const shape = (inputSchemaOf(machine)?.shape ?? {}) as Record<string, unknown>;
157
+ return { body: bodies[0]!.slot, chartFields: Object.keys(shape).filter((k) => isChartSchema(shape[k])) };
158
+ }
159
+
160
+ /** A pick one of whose parameters is a Chart (by schema identity, ADR-0070) is marked with that
161
+ * grammar's briefing, for the Turn part. */
162
+ function briefingOf(input: z.ZodObject): { briefing?: string } {
163
+ const shape = input.shape as Record<string, unknown>;
164
+ for (const key of Object.keys(shape)) {
165
+ const briefing = chartBriefingOf(shape[key]);
166
+ if (briefing !== undefined) return { briefing };
167
+ }
168
+ return {};
169
+ }
170
+
171
+ /** The short wrapper name of a Workflow name: `<wrapper>` of `<wrapper>/<name>`, else the name. */
172
+ function wrapperName(workflow: string): string {
173
+ const at = workflow.indexOf("/");
174
+ return at > 0 ? workflow.slice(0, at) : workflow;
175
+ }
176
+
177
+ /**
178
+ * Why a run's bound Charts no longer read under `machine` (ADR-0070, D-B), or undefined when each
179
+ * does: every bound Chart is admitted again against the Open body slot it binds, resolved on the
180
+ * Machine through `implementations.actors` (a lent child Machine's own slots included), with the
181
+ * lent set that Machine has now. A text whose hash does not match is drift too: a hand-edited
182
+ * volume.
183
+ */
184
+ function chartDrift(machine: AnyStateMachine, routes: readonly BoundChartRoute[]): string | undefined {
185
+ for (const route of routes) {
186
+ const where = `chart "${route.path.join("/")}" under body "${route.src}"`;
187
+ let at: AnyStateMachine = machine;
188
+ let resolved = true;
189
+ for (const src of route.srcs.slice(0, -1)) {
190
+ const logic = (at.implementations.actors as Record<string, unknown>)[src];
191
+ // The lent parts a bound body's Chart invokes are its wrapper's: the next hop is resolved there.
192
+ if (isOpenBody(logic)) continue;
193
+ const child = (logic as AnyStateMachine | undefined)?.root ? (logic as AnyStateMachine) : undefined;
194
+ if (!child) {
195
+ resolved = false;
196
+ break;
197
+ }
198
+ at = child;
199
+ }
200
+ const logic = resolved ? (at.implementations.actors as Record<string, unknown>)[route.src] : undefined;
201
+ if (!isOpenBody(logic)) return `${where}: the Machine has no Open body "${route.src}" now`;
202
+ const hash = chartHash(route.chart);
203
+ if (hash !== route.chartHash) {
204
+ return `${where}: its hash ${route.chartHash} does not match its text (${hash}); the stored run was edited`;
205
+ }
206
+ const decl = openBodyDeclarationOf(logic);
207
+ try {
208
+ admitBodyChart(route.chart, decl, lentIn(at.implementations as never, decl));
209
+ } catch (err) {
210
+ if (!(err instanceof ChartRefusal)) throw err;
211
+ return `${where}: ${err.report.join("; ")}`;
212
+ }
213
+ }
214
+ return undefined;
215
+ }
216
+
61
217
  /** The serializable identity of a run — what reconcile sees and what restore rebuilds from. */
62
218
  export type RunRecord = { runId: string; workflow: string; instanceId: string };
63
219
 
@@ -101,6 +257,191 @@ function runChildren(snapshot: unknown): RunChild[] {
101
257
  return out;
102
258
  }
103
259
 
260
+ /**
261
+ * One Workspace of a run waiting for capacity (ADR-0064): its Sandbox has no node. `child` is the
262
+ * Workspace's actor path below the run root — the {@link RunChild} ids down to it, joined by `/`,
263
+ * and `""` when the run's root IS the Workspace. `on` is what it waits for: a node from the
264
+ * scheduler, or room under the cluster owner's quota; `message` is the scheduler's or the API
265
+ * server's own words; `since` (ISO) is when the wait began, kept across a changed reason and a
266
+ * restart.
267
+ */
268
+ export type RunWaiting = { child: string; on: "node" | "quota"; message: string; since: string };
269
+
270
+ /**
271
+ * Every Workspace in `placing` with a reason, at any depth, read off the snapshot tree in the two
272
+ * shapes {@link runChildren} walks — the wait rides each Workspace's own persisted context, so a
273
+ * restored run reports it before its watch has said a word. A live child is held to being a
274
+ * `workspace()` by its logic; a persisted one has none, and the state key and the context's shape
275
+ * are the test. Only an ACTIVE run and an active Workspace wait: a run that ended — faulted,
276
+ * cancelled, done — has deleted what never ran, so a `placing` left in its snapshot waits for nothing.
277
+ */
278
+ function runWaiting(snapshot: unknown, logic?: unknown, path: string[] = []): RunWaiting[] {
279
+ const out: RunWaiting[] = [];
280
+ const snap = snapshot as { status?: unknown; value?: unknown; context?: Record<string, unknown> } | undefined;
281
+ if (snap?.status !== undefined && snap.status !== "active") return out;
282
+ const wait = snap?.context?.[WORKSPACE_WAIT_KEY] as WorkspaceWait | undefined;
283
+ const isWorkspace = logic == null || composesSandbox(logic as AnyStateMachine);
284
+ if (isWorkspace && snap?.value === "placing" && (wait?.on === "node" || wait?.on === "quota")) {
285
+ out.push({ child: path.join("/"), on: wait.on, message: String(wait.message), since: String(wait.since) });
286
+ }
287
+ const children = (snapshot as { children?: Record<string, unknown> } | undefined)?.children ?? {};
288
+ for (const [id, entry] of Object.entries(children)) {
289
+ const child = entry as { snapshot?: unknown; getSnapshot?: () => unknown; logic?: unknown };
290
+ const live = typeof child.getSnapshot === "function";
291
+ const childSnap = live ? child.getSnapshot!() : child.snapshot;
292
+ if ((childSnap as { value?: unknown } | undefined)?.value === undefined) continue;
293
+ out.push(...runWaiting(childSnap, live ? child.logic : undefined, [...path, id]));
294
+ }
295
+ return out;
296
+ }
297
+
298
+ /**
299
+ * The Sandbox of every live Workspace under `actor`, at any depth, by CR name — what a cancel
300
+ * destroys (ADR-0025). Read off the run's own live actor tree: each Workspace holds its `wsId` in
301
+ * context, and the name is a function of the run and that id, the same one its `teardown` deletes.
302
+ * Correct after a restart by construction: `restore()` rebuilds this tree from the persisted
303
+ * snapshot, so a restored run names its Sandboxes before its watch has listed. The Sandbox watch's
304
+ * set, filtered by the run's label, is not: before the watch has listed it holds nothing, and
305
+ * unknown is never absent (ADR-0063).
306
+ *
307
+ * Left out: a Workspace in `placing`, which deletes its own Sandbox when its wait is aborted — and
308
+ * only once any write in flight has settled, so a delete from here could race the apply and leave
309
+ * the CR behind (ADR-0064); and one that is `done` or ended, whose `teardown` already ran.
310
+ */
311
+ function runSandboxes(actor: AnyActorRef, runId: string): string[] {
312
+ const out: string[] = [];
313
+ const snap = actor.getSnapshot() as
314
+ | { status?: string; value?: unknown; context?: { wsId?: unknown }; children?: Record<string, AnyActorRef> }
315
+ | undefined;
316
+ if (snap?.status !== "active") return out;
317
+ const logic = (actor as AnyActorRef & { logic?: unknown }).logic;
318
+ if (
319
+ logic != null &&
320
+ composesSandbox(logic as AnyStateMachine) &&
321
+ snap.value !== "placing" &&
322
+ typeof snap.context?.wsId === "string"
323
+ ) {
324
+ out.push(workspaceName(runId, snap.context.wsId));
325
+ }
326
+ for (const child of Object.values(snap.children ?? {})) out.push(...runSandboxes(child, runId));
327
+ return out;
328
+ }
329
+
330
+ /**
331
+ * Stop every actor under `actor`, at any depth, deepest first. A machine that ended in `error` has
332
+ * stopped processing — its own stop is a no-op — so its children are reached through its snapshot
333
+ * rather than asked to stop through it.
334
+ */
335
+ function stopTree(actor: AnyActorRef): void {
336
+ const children = (actor.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
337
+ for (const child of Object.values(children)) {
338
+ stopTree(child);
339
+ // `_stop` is what xstate's own `stopChild` calls: a child cannot be stopped through `stop()`.
340
+ (child as AnyActorRef & { _stop?: () => void })._stop?.();
341
+ }
342
+ }
343
+
344
+ /** One outstanding pick, as `RunStatus.held` reports it: the Agent slot, its pick, and the ISO time
345
+ * it was delivered. */
346
+ export type RunHeld = { agent: string; pick: string; since: string };
347
+
348
+ /** The pick being delivered now (ADR-0024): its registration, every reply from below its Agent's
349
+ * state inside its own send, and what an `answer` action set there (ADR-0067). */
350
+ type Sending = {
351
+ reg: Registration;
352
+ pick: string;
353
+ chain: PickAnswer[];
354
+ shaped?: { value: unknown };
355
+ };
356
+
357
+ /**
358
+ * Whose pick started a child (ADR-0067): the Agent's slot, and the invoke id of the Agent's state the
359
+ * pick was made at, which tells one slot's two regions apart. `invoke` is absent when the event that
360
+ * started the child named the Agent and not where (a pick taken outside its own send).
361
+ */
362
+ type StartedBy = { agent: string; invoke?: string };
363
+
364
+ /**
365
+ * The key of the child `childId` of `parent` in the run's record of which Agent started each child
366
+ * (`startedBy`, ADR-0067): the invoke ids from below the root down to `parent`, then `childId`, each
367
+ * encoded and joined by `/`. An invoke id is local to its Machine — two Machines of one tree, and
368
+ * every worker of a pool, may each have a child `job` — and the chain is not. A child of the root
369
+ * keeps its bare id. The ids are the snapshot's, so a restore gives the same key.
370
+ */
371
+ function childKey(parent: AnyActorRef, childId: string): string {
372
+ const ids = [childId];
373
+ for (let at: AnyActorRef | undefined = parent; at?._parent !== undefined; at = at._parent) ids.unshift(at.id);
374
+ return ids.map(encodeURIComponent).join("/");
375
+ }
376
+
377
+ /**
378
+ * The Agent whose event `event` is, as `taker` takes it (ADR-0024, ADR-0067): a pick's sender, or
379
+ * the Agent whose pick started the child of `taker` whose done or error event it is (`startedBy`).
380
+ * Undefined when the event names no Agent: a timer, a Gate's answer, an outside send, a child no
381
+ * Agent's pick started. `sending`'s invoke is where a pick in its own send was made.
382
+ */
383
+ function agentOfEvent(
384
+ event: AnyEventObject,
385
+ startedBy: Record<string, StartedBy>,
386
+ taker: AnyActorRef,
387
+ sending?: Sending,
388
+ ): StartedBy | undefined {
389
+ const sender = (event as { [SENDER_KEY]?: Sender })[SENDER_KEY];
390
+ if (sender?.party === "agent") {
391
+ const invoke = sending?.reg.sender?.id === sender.id ? sending.reg.invokeId : undefined;
392
+ return invoke !== undefined ? { agent: sender.id, invoke } : { agent: sender.id };
393
+ }
394
+ const child = /^xstate\.(?:done|error)\.actor\.(.+)$/.exec(event.type)?.[1];
395
+ if (child === undefined) return undefined;
396
+ const key = childKey(taker, child);
397
+ return Object.hasOwn(startedBy, key) ? startedBy[key] : undefined;
398
+ }
399
+
400
+ /** How many of `snapshot`'s active states invoke an Agent. With two or more, an event that names no
401
+ * Agent cannot say whose pick an `answer` below no Agent's state shapes (ADR-0024). */
402
+ function activeAgentStates(snapshot: AnyMachineSnapshot): number {
403
+ const machine = snapshot.machine as AnyStateMachine;
404
+ const active = (snapshot as { _nodes?: AnyStateMachine["root"][] })._nodes ?? [];
405
+ return active.filter((node) => partiesAt(node, machine).some((party) => party.party === "agent")).length;
406
+ }
407
+
408
+ /** The `childKey` of every child in `actor`'s tree, at any depth. */
409
+ function liveChildKeys(actor: AnyActorRef, into = new Set<string>()): Set<string> {
410
+ const children = (actor.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
411
+ for (const child of Object.values(children)) {
412
+ if (!child) continue;
413
+ into.add(childKey(actor, child.id));
414
+ liveChildKeys(child, into);
415
+ }
416
+ return into;
417
+ }
418
+
419
+ /** `startedBy` as the save keeps it: only children still in the run's tree (ADR-0067). Read at the
420
+ * save, a microtask after the snapshot, so a done event the Machine took has run its route. */
421
+ function liveStartedBy(startedBy: Record<string, StartedBy>, actor: AnyActorRef): Record<string, StartedBy> {
422
+ const live = liveChildKeys(actor);
423
+ for (const id of Object.keys(startedBy)) if (!live.has(id)) delete startedBy[id];
424
+ return startedBy;
425
+ }
426
+
427
+ /** An `answer` action's value as the hold keeps it: JSON, because it rides the run blob and the
428
+ * wire (ADR-0067). `undefined`, or a value with no JSON form, sets no answer — the default stands. */
429
+ function jsonOf(value: unknown): { value: unknown } | undefined {
430
+ try {
431
+ const text = JSON.stringify(value);
432
+ return text === undefined ? undefined : { value: JSON.parse(text) as unknown };
433
+ } catch {
434
+ return undefined;
435
+ }
436
+ }
437
+
438
+ /** The outstanding picks among a run's holds. */
439
+ function heldOf(holds: Iterable<Hold>): RunHeld[] {
440
+ return [...holds]
441
+ .filter((hold) => hold.answer === undefined)
442
+ .map((hold) => ({ agent: hold.agent, pick: hold.pick, since: hold.since }));
443
+ }
444
+
104
445
  /** A run's current observable state. `fault` carries the error message when status is "error"
105
446
  * (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). */
106
447
  export type RunStatus = RunRecord & {
@@ -109,6 +450,14 @@ export type RunStatus = RunRecord & {
109
450
  context: unknown;
110
451
  /** The live child machines beneath the root — where most of a run actually is. */
111
452
  children: RunChild[];
453
+ /** Every Workspace waiting for capacity, and why (ADR-0064). Behind the token: the reason is
454
+ * the scheduler's or the quota's own words, which can name nodes and taints, so {@link observe}
455
+ * never carries it — the open band sees only that a Workspace is in `placing`. */
456
+ waiting: RunWaiting[];
457
+ /** Every pick of this run that is outstanding (ADR-0024): the Agent, its pick, and since when —
458
+ * the Console's "held since". There is no default time bound on a held Turn; this is how a
459
+ * human sees one. Behind the token, like `context`: a pick names what the Agent did. */
460
+ held: RunHeld[];
112
461
  fault?: string;
113
462
  /**
114
463
  * Why the HOST set this status, for the statuses the Machine did not choose — today `drifted`
@@ -117,6 +466,15 @@ export type RunStatus = RunRecord & {
117
466
  * route could read it.
118
467
  */
119
468
  reason?: string;
469
+ /** The hash of the Assembly the run started under (ADR-0069). Absent for a run no Assembly
470
+ * started. */
471
+ assembly?: string;
472
+ /** The run binds at least one Chart (ADR-0070): its Machine is the bound one, `runMachine`. */
473
+ bound?: true;
474
+ /** The hashes of the Charts the run binds now, distinct and sorted (ADR-0070). A run can bind a
475
+ * second Chart in its life (a planner whose first Chart reaches a ceiling picks again), so these,
476
+ * not the run id, say which bound Machine `runMachine` draws. Absent when it binds none. */
477
+ chartHashes?: string[];
120
478
  };
121
479
 
122
480
  /**
@@ -142,6 +500,12 @@ export type RunObservation = {
142
500
  status: string;
143
501
  value: unknown;
144
502
  children: RunChild[];
503
+ /** The run binds a Chart (ADR-0070): the Console draws `GET /workflows/:name/runs/:runId/machine`.
504
+ * Structure, like `value`. */
505
+ bound?: true;
506
+ /** The hashes of the Charts it binds now (see {@link RunStatus.chartHashes}). Structure: a hash
507
+ * of a Chart's text, which `runMachine` already serves on the open band. */
508
+ chartHashes?: string[];
145
509
  };
146
510
 
147
511
  /** Project a full status down to what an observer may see. The one place the line is drawn. */
@@ -152,9 +516,31 @@ export function observe(status: RunStatus): RunObservation {
152
516
  status: status.status,
153
517
  value: status.value,
154
518
  children: status.children,
519
+ ...(status.bound ? { bound: true as const } : {}),
520
+ ...(status.chartHashes ? { chartHashes: status.chartHashes } : {}),
155
521
  };
156
522
  }
157
523
 
524
+ /** An Agent registration's Menu (ADR-0029): the one it registered, or — registered with none, by a
525
+ * plain `setup()` machine — its derived set. */
526
+ function menuOf(reg: Registration): Map<string, EventDef> {
527
+ return reg.menu ?? reg.defs;
528
+ }
529
+
530
+ /** The invoking Machine's state value, as one comparable string; undefined when it cannot be read. */
531
+ function stateValueOf(invoker: AnyActorRef | undefined): string | undefined {
532
+ try {
533
+ return invoker ? JSON.stringify(invoker.getSnapshot().value) : undefined;
534
+ } catch {
535
+ return undefined;
536
+ }
537
+ }
538
+
539
+ /** The Allowed picks (ADR-0029), as the Agent's nudge and wake read them too. */
540
+ function allowedOf(reg: Registration): string[] {
541
+ return allowedPicks(reg);
542
+ }
543
+
158
544
  /** One open gate as external callers discover it (`GET /runs/:id` — ADR-0011): the accepted
159
545
  * events with their input schemas as JSON Schema (what drives a form or a `jr2 send` prompt),
160
546
  * plus the workflow-supplied `meta` (what a UI renders and a webhook translator matches on). */
@@ -165,15 +551,35 @@ export type GateView = {
165
551
  * id was authored ("F-12") or derived. Never parse `gate` for this. */
166
552
  path: string[];
167
553
  accepts: Array<{ name: string; description?: string; input: unknown }>;
554
+ /** The names of `accepts` the run would take from this Gate NOW, payload-blind (ADR-0011,
555
+ * ADR-0029): a send of another name is refused 409. `accepts` stays whole — it drives the forms. */
556
+ acceptsNow: string[];
168
557
  meta?: Record<string, unknown>;
169
558
  };
170
559
 
560
+ /** One event on an Agent's Menu, as the surface serves it: a tool, its `input` schema the tool's
561
+ * parameters. `semantics` rides along so the Harness refuses a reserved one (ADR-0013). */
562
+ /** One pick on the surface. `briefing` marks a pick whose parameter is a Chart: jr2's text on how a
563
+ * Chart runs (`CHART_BRIEFING`, ADR-0070), which the Harness says in the Turn part when the pick is
564
+ * Allowed, so only the Turns that may write a Chart carry it. */
565
+ export type SurfaceEvent = {
566
+ name: string;
567
+ description?: string;
568
+ input: unknown;
569
+ semantics: EventSemantics;
570
+ briefing?: string;
571
+ };
572
+
171
573
  /**
172
574
  * One live agent surface, as the Harness reads it through its Custodian (`GET /agents/:iid/surface`
173
- * — ADR-0013). The Harness presents it to its model as the Menu: each accepted event becomes a
174
- * tool, its `input` schema the tool's parameters. `semantics` rides along so the Harness can tell
175
- * an awaiting tool from a fire-and-forget one — the room ADR-0006's deferred results will land in,
176
- * unbuilt today.
575
+ * — ADR-0013, ADR-0029). Two answers, because they change at different rates:
576
+ *
577
+ * - `menu` is the Agent's MENU — every pick any state invoking this Agent offers, in a fixed
578
+ * order. The Harness presents it as the tools block, which is then byte-identical for every
579
+ * Turn of the conversation and never breaks the provider's prompt cache.
580
+ * - `allowed` is this Turn's ALLOWED PICKS — the invoking state's derived set, narrowed to what its
581
+ * guards would accept now, in Menu order. The Harness states it in the Briefing's Turn part,
582
+ * after the Frame's prompt (ADR-0062). Enforced at delivery, not here.
177
583
  *
178
584
  * `sandbox` is the Sandbox that may deliver here. It is not a secret from that pod (the pod IS the
179
585
  * sandbox), and serving it lets a caller fail loudly on a surface that is not its own.
@@ -182,7 +588,8 @@ export type AgentSurfaceView = {
182
588
  instanceId: string;
183
589
  runId: string;
184
590
  sandbox?: string;
185
- accepts: Array<{ name: string; description?: string; input: unknown; semantics: EventSemantics }>;
591
+ menu: SurfaceEvent[];
592
+ allowed: string[];
186
593
  };
187
594
 
188
595
  /**
@@ -195,10 +602,30 @@ export type AgentSurfaceView = {
195
602
  * turn has stopped waiting. It fails conservatively — anything that made the read unreliable reads
196
603
  * `false`, which is today's behavior, never a false claim.
197
604
  *
198
- * `deliveryId` is unchanged from ADR-0013: an outcome stays ADDRESSABLE after the fact, the room a
199
- * deferred result needs when it lands. Nothing polls it today, deliberately.
605
+ * `deliveryId` keeps the outcome ADDRESSABLE after the fact (ADR-0013): a pick the workflow is still
606
+ * running when the POST answers is OUTSTANDING (`outstanding: true`), and its answer is read at
607
+ * `GET /agents/:iid/deliveries/:id` — the same shape, `answered: true`, once the workflow is back at
608
+ * Rest in the Agent's state or has left it (ADR-0024).
200
609
  */
201
610
  export type AgentDeliveryReceipt = {
611
+ /** The workflow is running the pick: the answer comes on the delivery's poll. `allowed` is empty:
612
+ * no other pick is taken until it answers. */
613
+ outstanding?: true;
614
+ /** ISO time the pick was delivered — with `outstanding`. */
615
+ since?: string;
616
+ /** This receipt is a pick's answer, read at Rest (or once the Turn left the Agent's state): an outstanding
617
+ * pick's, a reply inside the pick's own send, or a guard straight back with no reply. */
618
+ answered?: true;
619
+ /** With `answered`: a guard straight back (ADR-0024) — the pick entered a state below the Agent's
620
+ * and came back inside its own send, nothing replying. The one answer that says "moved; nothing
621
+ * ran"; an outstanding pick's answer never carries it. */
622
+ straightBack?: true;
623
+ /** The last reply since the pick (ADR-0024); absent with `answered`: no reply came. */
624
+ answer?: PickAnswer;
625
+ /** With `answered` and `turnComplete`: something other than the pick's route left the Agent's
626
+ * state while the pick's work ran, and that exit stopped the work before it finished. `answer`,
627
+ * if any, is an earlier link's reply, kept (ADR-0024). */
628
+ stopped?: true;
202
629
  delivered: true;
203
630
  /** The event delivered — the Agent reads its own pick back, by name. */
204
631
  event: string;
@@ -213,6 +640,12 @@ export type AgentDeliveryReceipt = {
213
640
  moved: boolean;
214
641
  /** The invoking state stopped waiting: this Agent's turn is over. */
215
642
  turnComplete: boolean;
643
+ /**
644
+ * The Allowed picks read AFTER the delivery (ADR-0029): what this Turn may pick now — narrowed by
645
+ * a pick that moved the Machine within the invoking state, empty once the Turn is over. The
646
+ * Harness renders it ("Allowed now: `finish`"); it does not pre-check against it.
647
+ */
648
+ allowed: string[];
216
649
  deliveryId: string;
217
650
  };
218
651
 
@@ -227,7 +660,7 @@ export type AgentDeliveryReceipt = {
227
660
  export type RunFeedEvent =
228
661
  | { kind: "status"; status: RunStatus }
229
662
  | { kind: "emit"; event: { type: string } & Record<string, unknown> }
230
- | TurnMarker
663
+ | RunMarker
231
664
  // Absorbed-retry telemetry (ADR-0016): `{ child, attempt }` is state-key-class data — no iids
232
665
  // ride the feed (ADR-0014). `reason` is mechanism text, guarded like `fault`.
233
666
  | RetryTelemetry
@@ -235,9 +668,10 @@ export type RunFeedEvent =
235
668
 
236
669
  /**
237
670
  * The feed-so-far cap. The buffer exists so a Workspace attaching mid-run can open its log with
238
- * the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST frames, which
239
- * is the same trade kubelet's log rotation already makes on the printed side — the feed remains
240
- * the record, the buffer serves a courtesy view.
671
+ * the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST markers and
672
+ * Emits, which is the same trade kubelet's log rotation already makes on the printed side — the
673
+ * feed remains the record, the buffer serves a courtesy view. Statuses do not count against it:
674
+ * the buffer keeps only the latest (a status is a level — ADR-0023).
241
675
  */
242
676
  const FEED_SO_FAR_CAP = 1000;
243
677
 
@@ -265,6 +699,51 @@ function echoEventOf(event: RunFeedEvent, target: string): EchoEvent | undefined
265
699
  return undefined;
266
700
  }
267
701
 
702
+ /**
703
+ * The echo's outbox (ADR-0023): one push in flight per attachment, and whatever the feed produces
704
+ * meanwhile rides the NEXT push, together — a burst of transitions costs the Harness one push per
705
+ * flush, not one per transition.
706
+ *
707
+ * A status is a level, so a pending status is REPLACED by a newer one, never queued behind it. The
708
+ * ordering rule, exactly: markers and Emits keep feed order among themselves, and the one pending
709
+ * status sits where the NEWEST status arrived — after every marker and Emit fed before it, before
710
+ * every one fed after it. A superseded status drops out; nothing else moves. So a Harness that
711
+ * answers slowly holds at most one status plus the markers since its last push.
712
+ *
713
+ * Fire-and-forget: a failed push goes to `report` and the outbox drains on — nothing here can
714
+ * fail a turn, a state, or a run. Exported for its unit test; {@link RunHost} is its one caller.
715
+ */
716
+ export function echoOutbox(
717
+ push: (events: EchoEvent[]) => Promise<void>,
718
+ report: (err: unknown) => void,
719
+ ): (events: EchoEvent[]) => void {
720
+ let pending: EchoEvent[] = [];
721
+ let flushing = false;
722
+ const flush = async (): Promise<void> => {
723
+ while (pending.length > 0) {
724
+ const batch = pending;
725
+ pending = [];
726
+ try {
727
+ await push(batch);
728
+ } catch (err) {
729
+ report(err);
730
+ }
731
+ }
732
+ flushing = false;
733
+ };
734
+ return (events) => {
735
+ for (const event of events) {
736
+ if (event.kind === "status") pending = pending.filter((p) => p.kind !== "status");
737
+ pending.push(event);
738
+ }
739
+ if (flushing || pending.length === 0) return;
740
+ flushing = true;
741
+ // A microtask, so a synchronous burst (the replay; one transition's Emits and status) leaves
742
+ // as one push rather than as its first event alone.
743
+ queueMicrotask(() => void flush());
744
+ };
745
+ }
746
+
268
747
  /** The child-machine tree for the echo's status: spawn ids and state values alone — a root's
269
748
  * value says almost nothing about where a run is, and a {@link RunChild} is already context-free
270
749
  * by construction. `src`/`status` stay behind: they are join keys for the Console, not story. */
@@ -305,6 +784,19 @@ export type RunHostOptions = {
305
784
  reconcile?: (run: RunRecord) => boolean | Promise<boolean>;
306
785
  /** Injectable id generator (deterministic ids in tests). Default: `crypto.randomUUID`. */
307
786
  newId?: () => string;
787
+ /**
788
+ * Resolve a stamped blob's Workflow inside the Assembly it names (ADR-0069): the Machine that
789
+ * Assembly's `workflows/<name>.ts` exports, as a def stamped with that hash, or why it cannot be
790
+ * read (`drifted`: not on the volume, published for another Kit version, does not import). The
791
+ * host never learns the volume layout; `startInstance` supplies this over `assemblies/<hash>`.
792
+ * A def it returns is NOT registered: the registry is the current Assembly's alone. A throw is a
793
+ * fault, not a verdict: the row stays live for the next boot (`failed`).
794
+ *
795
+ * Absent (an in-process host): a stamped blob resolves to the registered def of its name only
796
+ * when that def came from the same Assembly hash, since one hash names one tree and so one
797
+ * Machine; otherwise the run is `drifted`.
798
+ */
799
+ resolveDef?: (blob: { workflow: string; assembly: string }) => Promise<WorkflowDef | { drifted: string }>;
308
800
  /** Report a run that threw during restore (ADR-0030). The row is left `live` for the next boot,
309
801
  * so this is the only account of why — the entrypoint logs it. Default: silent. */
310
802
  onRestoreError?: (runId: string, err: unknown) => void;
@@ -317,6 +809,9 @@ export type RunHostOptions = {
317
809
  * signing key. Handed to every run's binding, and to the echo pusher below. Absent, no Harness
318
810
  * call bears one: only the stub Harness answers such a host. */
319
811
  harnessBearer?: (placement: string) => string;
812
+ /** Where an Effector placement is, and its key (ADR-0068) — `startInstance` derives both, the key
813
+ * from the signing key. Handed to every run's binding; absent, an http Effect faults pointedly. */
814
+ effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
320
815
  /**
321
816
  * Build the run-narrative echo pusher for one Workspace's Harness (ADR-0023) — the seam a fake
322
817
  * echo server rides in tests; `startInstance` binds the real wire push (harness-client.ts).
@@ -325,10 +820,22 @@ export type RunHostOptions = {
325
820
  * never a dependency of the run.
326
821
  */
327
822
  echo?: (endpoint: string, bearer: string | undefined) => (events: EchoEvent[]) => Promise<void>;
823
+ /**
824
+ * Build the live-set statement to the Instance Harness (ADR-0031) — the seam a fake rides in
825
+ * tests; `startInstance` binds the real wire push (harness-client.ts). `bearer` is the Instance
826
+ * Harness's (ADR-0058). Absent, or with no `instanceHarness`, nothing is stated.
827
+ */
828
+ liveSet?: (endpoint: string, bearer: string | undefined) => (live: LiveConversation[]) => Promise<unknown>;
829
+ /** How often the live set is stated, before jitter. Default: the Lease's cadence (ADR-0021).
830
+ * Tests shrink it. */
831
+ liveSetIntervalMs?: number;
832
+ /** The wall clock a run's timer table is written and read on (ADR-0007, timers.ts): each pending
833
+ * timer's deadline is a time on it. Default: `Date.now`. Tests pass a clock that does not step. */
834
+ now?: () => number;
328
835
  };
329
836
 
330
837
  /** What we persist per run: the machine snapshot wrapped with the run metadata restore needs.
331
- * `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent actor
838
+ * `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent's Actor
332
839
  * through the run binding and saved in the same blob (same store, same atomicity). */
333
840
  type RunBlob = {
334
841
  workflow: string;
@@ -338,6 +845,10 @@ type RunBlob = {
338
845
  * Absent means written before the stamp existed, which restore treats as drift — the point is to
339
846
  * never interpret a snapshot whose Machine cannot be vouched for. */
340
847
  machine?: string;
848
+ /** The hash of the Assembly the run started under (ADR-0069), stamped on every save. A blob
849
+ * without one cannot be vouched for, so restore treats it as drift, as it treats a blob with no
850
+ * `machine` stamp. */
851
+ assembly?: string;
341
852
  snapshot: unknown;
342
853
  agents?: Record<string, AgentAdmission>;
343
854
  /**
@@ -353,6 +864,47 @@ type RunBlob = {
353
864
  * `<id>/<epoch>`), so one conversation has exactly one counter however often it burns.
354
865
  */
355
866
  epochs?: Record<string, number>;
867
+ /**
868
+ * The pending NOTICES (ADR-0062): what the next Turn in a scope must be told — a memory kill in
869
+ * its Workspace, a conversation that is new. The Orchestrator is their only keeper, so they ride
870
+ * this blob beside the two ledgers, for the same reasons: the run's exact lifetime, and the
871
+ * snapshot's atomicity. A notice leaves it in the save that ledgers the admission that carried
872
+ * it, so a restart before that save delivers it again and a restart after does not.
873
+ */
874
+ notices?: NoticeLedgerState;
875
+ /**
876
+ * The pending TIMERS (`after`, named delays, delayed `raise` and `sendTo`), each with its
877
+ * absolute deadline (ADR-0007, timers.ts). xstate 5 keeps them on the actor system, not in the
878
+ * snapshot, so a restore without this table loses every one. Same blob for the snapshot's
879
+ * atomicity: the timers saved are exactly the ones the saved states scheduled.
880
+ */
881
+ timers?: PendingTimer[];
882
+ /**
883
+ * The outstanding picks (ADR-0024, holds.ts): each conversation's latest pick that the workflow ran, its
884
+ * delivery id, the last reply since it, and its answer once read at Rest. Same blob for the
885
+ * snapshot's atomicity: a restored host answers the same poll from the same Machine.
886
+ */
887
+ holds?: Hold[];
888
+ /**
889
+ * The Effect LEDGER (ADR-0067): one row per Effect, by its id — `intent` before the call,
890
+ * `settled` with its output, `aborted` when its state exited while it ran. Same blob for the
891
+ * snapshot's atomicity: an intent row is saved with the snapshot that holds its Effect, so a restore
892
+ * reads which Effects may have acted and runs none of them twice. A row is never dropped: the
893
+ * settled and aborted rows keep id, kind and outcome, the record of what the run's Effects did; a
894
+ * settled row's output tail is dropped once a save carries its transition (`dropSavedTails`).
895
+ */
896
+ effects?: Record<string, EffectRow>;
897
+ /**
898
+ * Which Agent started each live child (ADR-0067), by the child's key — the invoke ids from below
899
+ * the root down to the child, encoded and joined by `/` (`childKey`), so a child of one Machine is
900
+ * never another Machine's child of the same local id: the Agent whose pick's
901
+ * own send started it, or whose event the Machine was taking when it started. A done event of that
902
+ * child names that Agent, so an `answer` on a transition below no Agent's state that the done event
903
+ * takes shapes that Agent's pick and no other — whether the pick that started it is still
904
+ * outstanding or not. Same blob, so a restore names the same Agent. A child no longer in the tree
905
+ * is dropped at the save.
906
+ */
907
+ startedBy?: Record<string, StartedBy>;
356
908
  fault?: string;
357
909
  };
358
910
 
@@ -367,15 +919,27 @@ type LiveRun = {
367
919
  agents: Record<string, AgentAdmission>;
368
920
  /** The live epoch ledger (ADR-0057): persisted as `RunBlob.epochs`, seeded on restore. */
369
921
  epochs: Record<string, number>;
922
+ /** The pending notices (ADR-0062): persisted as `RunBlob.notices`, seeded on restore. */
923
+ notices: NoticeLedger;
924
+ /** The outstanding picks by Agent address (ADR-0024): persisted as `RunBlob.holds`, seeded on restore. */
925
+ holds: Map<string, Hold>;
926
+ /** The Effect ledger (ADR-0067): persisted as `RunBlob.effects`, seeded on restore. */
927
+ effects: Record<string, EffectRow>;
928
+ /** Which Agent started each live child (ADR-0067): persisted as `RunBlob.startedBy`, seeded on restore. */
929
+ startedBy: Record<string, StartedBy>;
370
930
  /** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
371
931
  * message is captured here at the observer and persisted onto the blob for `read`). */
372
932
  fault?: string;
373
933
  /** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
374
934
  listeners: Set<(e: RunFeedEvent) => void>;
375
935
  /** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
376
- * log's preamble. In-memory and this-boot only — a restored run's preamble starts at restore,
377
- * the same live-only contract the printed log already has. Capped ({@link FEED_SO_FAR_CAP}). */
936
+ * log's preamble — its markers and Emits, in feed order. In-memory and this-boot only — a
937
+ * restored run's preamble starts at restore, the same live-only contract the printed log already
938
+ * has. Capped ({@link FEED_SO_FAR_CAP}). */
378
939
  feedSoFar: RunFeedEvent[];
940
+ /** The preamble's last line: where the run stands now. A status is a level (ADR-0023), so the
941
+ * buffer keeps the latest alone — old statuses are not interleaved between old markers. */
942
+ latestStatus?: RunFeedEvent & { kind: "status" };
379
943
  };
380
944
 
381
945
  export class RunHost {
@@ -387,15 +951,30 @@ export class RunHost {
387
951
  private readonly reconcile: (run: RunRecord) => boolean | Promise<boolean>;
388
952
  private readonly newId: () => string;
389
953
  private readonly onRestoreError?: (runId: string, err: unknown) => void;
954
+ private readonly resolveDef?: RunHostOptions["resolveDef"];
390
955
  private readonly sandbox?: SandboxPort;
391
956
  private readonly instanceHarness?: string;
392
957
  private readonly harnessBearer?: (placement: string) => string;
958
+ private readonly effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
393
959
  private readonly echoFactory?: (
394
960
  endpoint: string,
395
961
  bearer: string | undefined,
396
962
  ) => (events: EchoEvent[]) => Promise<void>;
963
+ private readonly liveSetPush?: (live: LiveConversation[]) => Promise<unknown>;
964
+ private readonly liveSetIntervalMs: number;
965
+ private readonly now: () => number;
966
+ private liveSetTimer?: ReturnType<typeof setTimeout>;
967
+ /** Whether the last live-set statement failed — a failing streak logs one line (ADR-0031). */
968
+ private liveSetFailing = false;
397
969
  private readonly workflowDefs = new Map<string, WorkflowDef>();
398
970
  private readonly runs = new Map<string, LiveRun>();
971
+ /**
972
+ * The admission ledgers of the runs restore could not host this boot (a `reconcile` that threw).
973
+ * Their rows stay `live` for the next boot to retry, so their conversations on the Instance
974
+ * Harness are still held: the live set names them from here (ADR-0031), or the retry would find
975
+ * nothing to re-attach to.
976
+ */
977
+ private readonly unhosted = new Map<string, Record<string, AgentAdmission>>();
399
978
  /**
400
979
  * Workflow name → its observers. Deliberately on the HOST and not on `LiveRun`: a workflow's
401
980
  * watcher outlives every individual run it watches, and `persist()` clears a settled run's own
@@ -403,16 +982,58 @@ export class RunHost {
403
982
  * than a comment asking the next reader not to clear the wrong Set.
404
983
  */
405
984
  private readonly workflowListeners = new Map<string, Set<WorkflowListener>>();
985
+ /** Every timer a run's actor system has armed and not yet fired or cleared: an `after`, a Chart's
986
+ * wall clock (ADR-0070). `close()` unrefs them, so a timer of hours never holds the process. */
987
+ private readonly timers = new Set<ReturnType<typeof setTimeout>>();
988
+ /** The clock every run's actor system arms its timers on: xstate's own, with each timer kept in
989
+ * {@link timers} and unref'd once the host is closing. The scheduled events stay in the system's
990
+ * snapshot, so a restore arms them again. */
991
+ private readonly clock = {
992
+ setTimeout: (fn: (...args: unknown[]) => void, ms: number) => {
993
+ const timer = setTimeout(() => {
994
+ this.timers.delete(timer);
995
+ fn();
996
+ }, ms);
997
+ this.timers.add(timer);
998
+ if (this.#stopping) (timer as { unref?: () => void }).unref?.();
999
+ return timer;
1000
+ },
1001
+ clearTimeout: (timer: ReturnType<typeof setTimeout>) => {
1002
+ this.timers.delete(timer);
1003
+ clearTimeout(timer);
1004
+ },
1005
+ };
1006
+ /** The registration each hold was opened under, this boot: the same address re-registered by
1007
+ * another invocation (`continue: true`) is another Turn. A restored hold adopts the first. */
1008
+ private readonly holdRegs = new WeakMap<Hold, Registration>();
1009
+ /** The long polls waiting on each open hold (ADR-0024). */
1010
+ private readonly holdWaiters = new Map<Hold, Set<() => void>>();
1011
+ /** The pick being delivered now, and the last reply from below its Agent's state inside its own
1012
+ * send (ADR-0024). A child that settles in the same macrostep as the pick — a decision Machine
1013
+ * that is final as it starts — replies before a hold can exist, so the chain records it here. */
1014
+ private sending: Sending | undefined;
1015
+ /** `<iid> <deliveryId>` of every outstanding pick of a run this host parked (`stop`). */
1016
+ private readonly parked = new Set<string>();
1017
+ #stopping = false;
1018
+ #restorePending = false;
406
1019
 
407
1020
  constructor(opts: RunHostOptions) {
408
1021
  this.store = opts.store;
409
1022
  this.reconcile = opts.reconcile ?? (() => true);
410
1023
  this.newId = opts.newId ?? (() => randomUUID());
411
1024
  this.onRestoreError = opts.onRestoreError;
1025
+ this.resolveDef = opts.resolveDef;
412
1026
  this.sandbox = opts.sandbox;
413
1027
  this.instanceHarness = opts.instanceHarness;
414
1028
  this.harnessBearer = opts.harnessBearer;
1029
+ this.effector = opts.effector;
415
1030
  this.echoFactory = opts.echo;
1031
+ this.liveSetPush =
1032
+ opts.instanceHarness && opts.liveSet
1033
+ ? opts.liveSet(opts.instanceHarness, opts.harnessBearer?.(INSTANCE_HARNESS_SERVICE))
1034
+ : undefined;
1035
+ this.liveSetIntervalMs = opts.liveSetIntervalMs ?? DEFAULT_LEASE_INTERVAL_MS;
1036
+ this.now = opts.now ?? Date.now;
416
1037
  }
417
1038
 
418
1039
  /** Register a workflow so `start`/`restore` can run it. Re-registering replaces (dev reload).
@@ -420,7 +1041,56 @@ export class RunHost {
420
1041
  * declared them and resolved at invoke time off the invoking Machine (ADR-0011, ADR-0049), so
421
1042
  * a nested Machine's defs are never the root's — or this host's — to hold. */
422
1043
  register(def: WorkflowDef): void {
423
- this.workflowDefs.set(def.name, def);
1044
+ const wrapper = def.wrapper ?? wrapperOf(def.machine);
1045
+ this.workflowDefs.set(def.name, wrapper ? { ...def, wrapper } : def);
1046
+ }
1047
+
1048
+ /**
1049
+ * Register a named Chart as the Workflow `<wrapper>/<name>` (ADR-0070). The registry of named
1050
+ * Charts (charts.ts) calls this for each runnable one; it keeps where the Charts are stored, and
1051
+ * the unrunnable ones, itself. Replacing a Chart a live run holds is fine: the run holds its text.
1052
+ */
1053
+ registerChart(def: WorkflowDef & { chart: NonNullable<WorkflowDef["chart"]> }): void {
1054
+ this.register(def);
1055
+ }
1056
+
1057
+ /** Drop a named Chart's Workflow. A live run of it keeps its text and runs on. */
1058
+ unregisterChart(wrapper: string, name: string): void {
1059
+ const key = `${wrapper}/${name}`;
1060
+ if (this.workflowDefs.get(key)?.chart) this.workflowDefs.delete(key);
1061
+ }
1062
+
1063
+ /** The registered named Chart `<wrapper>/<name>`, or undefined. */
1064
+ namedChart(wrapper: string, name: string): NamedChart | undefined {
1065
+ const chart = this.workflowDefs.get(`${wrapper}/${name}`)?.chart;
1066
+ return (
1067
+ chart && { wrapper: chart.wrapper, name: chart.name, text: chart.text, hash: chart.hash, origin: chart.origin }
1068
+ );
1069
+ }
1070
+
1071
+ /**
1072
+ * Every registered Workflow with its origin (ADR-0070): a Machine, or a named Chart from a file or
1073
+ * at run time. A Chart wrapper's entry carries its Open body, whether its door takes a Chart, and
1074
+ * its named Charts: a named Chart lists inside its wrapper's entry, not beside it, and has an
1075
+ * entry of its own only when its wrapper is not registered.
1076
+ */
1077
+ workflowList(): WorkflowListing[] {
1078
+ const defs = [...this.workflowDefs.values()];
1079
+ const wrappers = new Set(defs.filter((d) => !d.chart && d.wrapper).map((d) => d.name));
1080
+ return defs.flatMap((def): WorkflowListing[] => {
1081
+ if (def.chart) return wrappers.has(def.chart.wrapper) ? [] : [{ name: def.name, origin: def.chart.origin }];
1082
+ if (!def.wrapper) return [{ name: def.name, origin: "machine" }];
1083
+ const charts = defs
1084
+ .filter((d) => d.chart?.wrapper === def.name)
1085
+ .map((d) => ({ name: d.chart!.name, origin: d.chart!.origin }));
1086
+ return [
1087
+ {
1088
+ name: def.name,
1089
+ origin: "machine",
1090
+ wrapper: { body: def.wrapper.body, oneShot: def.wrapper.chartFields.length > 0, charts },
1091
+ },
1092
+ ];
1093
+ });
424
1094
  }
425
1095
 
426
1096
  /** Drop a workflow's registration (a workflow the host no longer discovers). In-flight
@@ -438,12 +1108,79 @@ export class RunHost {
438
1108
  return [...this.workflowDefs.keys()];
439
1109
  }
440
1110
 
1111
+ /** The runs this process hosts, by the hash of the Assembly each started under (ADR-0069). A
1112
+ * run from a def with no Assembly holds none. The store's live rows are the hold
1113
+ * (`SnapshotStore.liveAssemblies`); this adds a run started a moment ago, before its first save. */
1114
+ assemblyHolds(): Map<string, string[]> {
1115
+ const holds = new Map<string, string[]>();
1116
+ for (const [runId, run] of this.runs) {
1117
+ const hash = run.def.assembly;
1118
+ if (hash === undefined) continue;
1119
+ holds.set(hash, [...(holds.get(hash) ?? []), runId]);
1120
+ }
1121
+ return holds;
1122
+ }
1123
+
441
1124
  /** The registered template Machine's serialized structure (`GET /workflows/:name/machine`). */
442
1125
  machine(name: string): MachineDoc | undefined {
443
1126
  const def = this.workflowDefs.get(name);
444
1127
  return def && serializeMachine(name, def.machine);
445
1128
  }
446
1129
 
1130
+ /**
1131
+ * The run's bound Machine (ADR-0070): the Workflow's Machine with each bound Open body's subgraph
1132
+ * the compiled Chart's (its `src` stays the slot key, so the live child joins it as before) —
1133
+ * from the live actor's bound Machine, or, for a stored run, compiled again from the text its
1134
+ * blob holds. Undefined for an unknown run, and for a stored run whose Workflow is not registered.
1135
+ */
1136
+ async runMachine(runId: string): Promise<MachineDoc | undefined> {
1137
+ const live = this.runs.get(runId);
1138
+ let workflow: string;
1139
+ let machine: AnyStateMachine;
1140
+ let bodies: Array<{ route: BoundChartRoute; compiled: AnyStateMachine | string }>;
1141
+ if (live) {
1142
+ workflow = live.record.workflow;
1143
+ machine = live.def.machine;
1144
+ const refs = new Map<string, AnyActorRef>();
1145
+ const collect = (ref: AnyActorRef, path: string[]): void => {
1146
+ for (const [id, child] of Object.entries(childrenOfRef(ref))) {
1147
+ refs.set([...path, id].join("/"), child);
1148
+ collect(child, [...path, id]);
1149
+ }
1150
+ };
1151
+ collect(live.actor, []);
1152
+ bodies = boundChartRoutesOf(live.actor.getSnapshot()).map((route) => {
1153
+ const ref = refs.get(route.path.join("/"));
1154
+ return { route, compiled: (ref && boundMachineOf(ref)) ?? "the live body holds no compiled Chart" };
1155
+ });
1156
+ } else {
1157
+ const stored = await this.store.load(runId);
1158
+ const blob = stored?.snapshot as RunBlob | null | undefined;
1159
+ if (!blob) return undefined;
1160
+ const def = this.workflowDefs.get(blob.workflow) ?? this.workflowDefs.get(wrapperName(blob.workflow));
1161
+ if (!def) return undefined;
1162
+ workflow = blob.workflow;
1163
+ machine = def.machine;
1164
+ bodies = boundChartRoutesOf(blob.snapshot).map((route) => ({
1165
+ route,
1166
+ compiled: compileStored(def.machine, route),
1167
+ }));
1168
+ }
1169
+ const doc = serializeMachine(workflow, machine);
1170
+ const notDrawn: NonNullable<MachineDoc["chartsNotDrawn"]> = [];
1171
+ for (const { route, compiled } of bodies) {
1172
+ if (typeof compiled === "string") {
1173
+ // The doc keeps the slot's own subgraph, and says so: a doc that drew the wrapper's body
1174
+ // with no word would disagree with the run's `bound` (ADR-0070, D-B).
1175
+ notDrawn.push({ path: route.path.join("/"), src: route.src, reason: compiled });
1176
+ continue;
1177
+ }
1178
+ const { workflow: _w, opaqueStates: _o, ...body } = serializeMachine(route.src, compiled);
1179
+ placeBody(doc, route.srcs, body);
1180
+ }
1181
+ return notDrawn.length ? { ...doc, chartsNotDrawn: notDrawn } : doc;
1182
+ }
1183
+
447
1184
  /**
448
1185
  * The workflow's declared run-input contract as JSON Schema (ADR-0033) — what the workflow
449
1186
  * detail serves and the Console's start form generates from. `null` for a machine that
@@ -453,7 +1190,11 @@ export class RunHost {
453
1190
  inputSchema(name: string): Record<string, unknown> | null | undefined {
454
1191
  const def = this.workflowDefs.get(name);
455
1192
  if (!def) return undefined;
456
- const schema = inputSchemaOf(def.machine);
1193
+ let schema = inputSchemaOf(def.machine);
1194
+ // A named Chart's door is its wrapper's minus the Chart fields, which the Chart supplies (ADR-0070).
1195
+ if (schema && def.chart && def.wrapper?.chartFields.length) {
1196
+ schema = schema.omit(Object.fromEntries(def.wrapper.chartFields.map((f) => [f, true])) as never);
1197
+ }
457
1198
  // `io: "input"`: this schema describes what a caller SENDS — a defaulted field is optional
458
1199
  // at the door (the default is applied by `start`'s parse), not required of the form.
459
1200
  return schema ? (z.toJSONSchema(schema, { io: "input" }) as Record<string, unknown>) : null;
@@ -464,6 +1205,13 @@ export class RunHost {
464
1205
  const def = this.workflowDefs.get(workflow);
465
1206
  if (!def) throw new Error(`no workflow registered as "${workflow}"`);
466
1207
 
1208
+ // A named Chart supplies every Chart field of its wrapper's door, and the caller the rest
1209
+ // (ADR-0070): what the caller sent under a Chart field is not the run's.
1210
+ if (def.chart && def.wrapper) {
1211
+ input = { ...input };
1212
+ for (const field of def.wrapper.chartFields) input[field] = def.chart.text;
1213
+ }
1214
+
467
1215
  // A declared schema is enforced at the door (ADR-0033), and what starts the run is the
468
1216
  // PARSED shape (defaults applied, unknown keys stripped) — exactly what a gate delivery
469
1217
  // lands as, and the same error class: `EventValidationError`, which the wire maps to a 400
@@ -483,6 +1231,10 @@ export class RunHost {
483
1231
  const record: RunRecord = { runId, workflow, instanceId };
484
1232
 
485
1233
  const machine = this.assemble(def, instanceId);
1234
+ // Every Chart field of a Chart wrapper's door is admitted BEFORE the run exists (ADR-0070): a
1235
+ // refusal throws `ChartRefusal`, which the wire answers `422 { error: "refused", report }`. The
1236
+ // lent set is the run's own Machine's, with its providers in.
1237
+ this.admitDoorCharts(def, machine, runInput);
486
1238
  // The root machine gets the parsed door PLUS what the host injects beside it —
487
1239
  // `HostInjectedInput` (vocabulary.ts): the seed Instance ID, added after the parse because no
488
1240
  // caller sends it and nothing serves it (ADR-0033). Only the root: a machine invoked below is
@@ -493,6 +1245,46 @@ export class RunHost {
493
1245
  return { runId, instanceId };
494
1246
  }
495
1247
 
1248
+ /**
1249
+ * Admit every Chart field present in a Chart wrapper's parsed door (ADR-0070): the whole door,
1250
+ * the compile included, against each Open body (the door cannot tell which body a field feeds).
1251
+ * `input.*` references are the body's to check at its start: the Chart's `input` is what the
1252
+ * wrapper's invoke sends, which is not the door. Throws ChartRefusal.
1253
+ */
1254
+ private admitDoorCharts(def: WorkflowDef, machine: AnyStateMachine, input: Record<string, unknown>): void {
1255
+ for (const field of def.wrapper?.chartFields ?? []) {
1256
+ if (input[field] === undefined) continue;
1257
+ admitWrapperChart(machine, input[field]);
1258
+ }
1259
+ }
1260
+
1261
+ /**
1262
+ * The def that reads `blob` (ADR-0069), or the verdict when none can. A stamped blob resolves
1263
+ * inside the Assembly it names, through `resolveDef`; an unstamped blob resolves by name in the
1264
+ * registry, and only on a host that loads no Assembly. A blob that names no Workflow the
1265
+ * registry holds is `lost`: there is nothing it could ever be read by.
1266
+ */
1267
+ private async defFor(blob: RunBlob): Promise<WorkflowDef | { drifted: string } | { lost: string }> {
1268
+ const registered = this.workflowDefs.get(blob.workflow);
1269
+ if (blob.assembly !== undefined) {
1270
+ const ref = { workflow: blob.workflow, assembly: blob.assembly };
1271
+ if (this.resolveDef) {
1272
+ const found = await this.resolveDef(ref);
1273
+ return "drifted" in found ? found : { ...found, assembly: blob.assembly };
1274
+ }
1275
+ if (registered?.assembly === blob.assembly) return registered;
1276
+ return {
1277
+ drifted: `workflow "${blob.workflow}" saved under Assembly ${blob.assembly}, which this host does not hold`,
1278
+ };
1279
+ }
1280
+ // A blob that names no Assembly cannot be vouched for on a host that loads Assemblies:
1281
+ // greenfield, no migration (ADR-0069).
1282
+ if (this.resolveDef || registered?.assembly !== undefined) {
1283
+ return { drifted: `workflow "${blob.workflow}": saved under no Assembly` };
1284
+ }
1285
+ return registered ?? { lost: `no workflow "${blob.workflow}"` };
1286
+ }
1287
+
496
1288
  /**
497
1289
  * Restore every persisted live run: hydrate, reconcile against the live world, and either
498
1290
  * re-attach (re-spawn from the rewritten child input), mark lost, or refuse as drifted. Returns
@@ -504,8 +1296,10 @@ export class RunHost {
504
1296
  * later run in the store never restored at all. One run that cannot be resumed is one run, not an
505
1297
  * outage.
506
1298
  *
507
- * `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — the
508
- * Machine changed, and it will still have changed on the next boot — so it is written to the row.
1299
+ * `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — this
1300
+ * Orchestrator cannot read what it saved (the Assembly is gone, at another Kit version, does not
1301
+ * load, or its Machine's shape does not match), and the next boot will not read it either — so it
1302
+ * is written to the row.
509
1303
  * A throw is not: `reconcile` talks to a cluster, and a kubectl blip must not permanently condemn
510
1304
  * a run. Those rows are left `live` to be retried on the next boot, and reported every time until
511
1305
  * they stop failing.
@@ -516,30 +1310,72 @@ export class RunHost {
516
1310
  const drifted: string[] = [];
517
1311
  const failed: string[] = [];
518
1312
 
519
- for (const stored of await this.store.list()) {
520
- if (stored.status !== "live") continue;
1313
+ for (const stored of await this.store.live()) {
1314
+ // A run this process hosts already is not restored a second time. A deferred restore
1315
+ // (ADR-0069) that throws as a whole stays pending, and the next publish tries it again; a run
1316
+ // started under the current Assembly between those two publishes has a live row and a live
1317
+ // actor, and must not get a second actor.
1318
+ if (this.runs.has(stored.runId)) continue;
521
1319
  const blob = stored.snapshot as RunBlob | null;
522
- const def = blob ? this.workflowDefs.get(blob.workflow) : undefined;
523
- if (!blob || !def) {
524
- await this.store.markLost(stored.runId, blob ? `no workflow "${blob.workflow}"` : "empty snapshot");
1320
+ if (!blob) {
1321
+ await this.store.markLost(stored.runId, "empty snapshot");
525
1322
  lost.push(stored.runId);
526
1323
  continue;
527
1324
  }
528
1325
 
529
- // The workflow name resolved a def; this asks whether that def is still the Machine this
530
- // snapshot was written by (ADR-0030). Refused BEFORE `reconcile`, which talks to the cluster:
531
- // there is no point proving a Sandbox is alive for a run that cannot be read.
1326
+ // Which Machine reads this snapshot (ADR-0069). One rule: a stamped blob restores from its
1327
+ // Assembly; an unstamped blob restores from the registry; a stamped host never has unstamped
1328
+ // blobs it can vouch for. So a publish strands nothing: the def comes from the Assembly the
1329
+ // run started under, never from the current one. Drift is only "I can no longer read what I
1330
+ // saved", and it keeps the snapshot (`markDrifted`), so `jr2 status` still reads the run.
1331
+ let def: WorkflowDef;
1332
+ try {
1333
+ const found = await this.defFor(blob);
1334
+ if ("lost" in found) {
1335
+ await this.store.markLost(stored.runId, found.lost);
1336
+ lost.push(stored.runId);
1337
+ continue;
1338
+ }
1339
+ if ("drifted" in found) {
1340
+ await this.store.markDrifted(stored.runId, found.drifted);
1341
+ drifted.push(stored.runId);
1342
+ continue;
1343
+ }
1344
+ def = found;
1345
+ } catch (err) {
1346
+ // A fault while reading the Assembly, not a verdict on it: left live, as below.
1347
+ this.onRestoreError?.(stored.runId, err);
1348
+ failed.push(stored.runId);
1349
+ this.unhosted.set(stored.runId, blob.agents ?? {});
1350
+ continue;
1351
+ }
1352
+
1353
+ // The def is the Machine this snapshot names; this asks whether it is still the shape this
1354
+ // snapshot was written by (ADR-0030). Inside one Assembly the same bytes give the same
1355
+ // Machine, so this guards a hash collision or a hand-edited volume. Refused BEFORE
1356
+ // `reconcile`, which talks to the cluster: there is no point proving a Sandbox is alive for
1357
+ // a run that cannot be read.
532
1358
  const expected = fingerprintOf(def.machine);
533
1359
  if (blob.machine !== expected) {
534
1360
  await this.store.markDrifted(
535
1361
  stored.runId,
536
- `workflow "${blob.workflow}" changed shape since this run was saved ` +
1362
+ `workflow "${blob.workflow}"${def.assembly !== undefined ? ` in Assembly ${def.assembly}` : ""} changed shape since this run was saved ` +
537
1363
  `(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`,
538
1364
  );
539
1365
  drifted.push(stored.runId);
540
1366
  continue;
541
1367
  }
542
1368
 
1369
+ // The Charts the run binds (ADR-0070, D-B): the shape stamp is the wrapper's, and each Chart's
1370
+ // identity is its text, kept in its body's snapshot. Each is admitted again against the Open
1371
+ // body it binds and the lent set the Machine has now; one that no longer admits is drift.
1372
+ const charts = chartDrift(def.machine, boundChartRoutesOf(blob.snapshot));
1373
+ if (charts !== undefined) {
1374
+ await this.store.markDrifted(stored.runId, charts);
1375
+ drifted.push(stored.runId);
1376
+ continue;
1377
+ }
1378
+
543
1379
  const record: RunRecord = { runId: stored.runId, workflow: blob.workflow, instanceId: blob.instanceId };
544
1380
 
545
1381
  try {
@@ -558,37 +1394,124 @@ export class RunHost {
558
1394
  this.assemble(def, blob.instanceId),
559
1395
  { snapshot: hydrated as never },
560
1396
  record,
1397
+ // The def of the Assembly the run started under (ADR-0069), so every later save names
1398
+ // that hash. The config the run provisions under is the CURRENT Assembly's all the same:
1399
+ // the ports read it through the binding a publish replaces, and a held Assembly lends
1400
+ // only its Machines (ADR-0069).
561
1401
  def,
562
1402
  agents,
563
1403
  // The burned conversations come back with the admissions (ADR-0057): a restored run's
564
1404
  // next `continue` must land where this run's last one did, not on a dead conversation.
565
1405
  blob.epochs ?? {},
1406
+ // …and so do the undelivered notices (ADR-0062), reservations forgotten: a notice whose
1407
+ // admission never reached the ledger is delivered again.
1408
+ blob.notices,
1409
+ // …and the outstanding picks (ADR-0024): the restored Machine says when each is answered.
1410
+ blob.holds,
1411
+ // …and the Effect ledger (ADR-0067): a restored Effect reads its row and never runs twice.
1412
+ blob.effects ?? {},
1413
+ // …and which Agent started each child (ADR-0067): its done event names the same Agent.
1414
+ blob.startedBy ?? {},
566
1415
  );
567
- actor.start();
1416
+ // `start()`, then the pending timers again (ADR-0007, see `startAndRearm`). The next save
1417
+ // records them again.
1418
+ startAndRearm(actor, blob.timers ?? [], this.now());
1419
+ // A run an earlier restore left live to retry is hosted now.
1420
+ this.unhosted.delete(stored.runId);
568
1421
  reattached.push(stored.runId);
569
1422
  } catch (err) {
570
1423
  // Left `live` on purpose — see the note above. The row is unchanged, so the next boot tries
571
1424
  // again; what must not happen is this taking the remaining runs down with it.
572
1425
  this.onRestoreError?.(stored.runId, err);
573
1426
  failed.push(stored.runId);
1427
+ this.unhosted.set(stored.runId, blob.agents ?? {});
574
1428
  }
575
1429
  }
576
1430
 
1431
+ // Every run this boot will host is live now, so the set is whole: state it, and keep stating it
1432
+ // (ADR-0031). Not before — a statement ahead of restore would free the conversations of the
1433
+ // runs about to be re-attached.
1434
+ this.stateLiveSet();
1435
+ this.scheduleLiveSet();
1436
+
1437
+ this.#restorePending = false;
577
1438
  return { reattached, lost, drifted, failed };
578
1439
  }
579
1440
 
1441
+ /**
1442
+ * The conversations the live runs hold on the Instance Harness (ADR-0031): every admission in
1443
+ * their ledgers whose endpoint is the Instance Harness, as the `(agent, instanceId)` pair it runs
1444
+ * under. From MEMORY, never the store: the store also holds runs this process does not host, and
1445
+ * a run this process hosts is exactly one that may still continue its conversation — and so is a
1446
+ * run restore left `live` to retry (`unhosted`). A Sandbox's Harness is not in it — its
1447
+ * Workspace's teardown frees everything there.
1448
+ */
1449
+ liveConversations(): LiveConversation[] {
1450
+ const live = new Map<string, LiveConversation>();
1451
+ const ledgers = [...[...this.runs.values()].map((run) => run.agents), ...this.unhosted.values()];
1452
+ for (const agents of ledgers) {
1453
+ for (const [iid, admission] of Object.entries(agents)) {
1454
+ if (admission.endpoint !== this.instanceHarness || admission.agent === undefined) continue;
1455
+ const instanceId = admission.instanceId ?? iid;
1456
+ live.set(JSON.stringify([admission.agent, instanceId]), { agent: admission.agent, instanceId });
1457
+ }
1458
+ }
1459
+ return [...live.values()];
1460
+ }
1461
+
1462
+ /**
1463
+ * State the live set to the Instance Harness (ADR-0031), which frees every other conversation.
1464
+ * FIRE-AND-FORGET: level-triggered and idempotent, so a missed statement costs nothing and the
1465
+ * next one says it all again. A failing streak logs one line; nothing here can touch a run.
1466
+ */
1467
+ private stateLiveSet(): void {
1468
+ const push = this.liveSetPush;
1469
+ if (!push || this.#stopping) return;
1470
+ push(this.liveConversations()).then(
1471
+ () => {
1472
+ this.liveSetFailing = false;
1473
+ },
1474
+ (err: unknown) => {
1475
+ if (this.liveSetFailing) return;
1476
+ this.liveSetFailing = true;
1477
+ console.error(
1478
+ `live-set statement to ${this.instanceHarness} failed (the Instance Harness keeps what it holds): ` +
1479
+ (err instanceof Error ? err.message : String(err)),
1480
+ );
1481
+ },
1482
+ );
1483
+ }
1484
+
1485
+ /** Every Lease interval, ±20% (ADR-0021's cadence, which ADR-0031 names). The timer holds no
1486
+ * process alive, and `close()` clears it. */
1487
+ private scheduleLiveSet(): void {
1488
+ if (!this.liveSetPush || this.liveSetTimer !== undefined || this.#stopping) return;
1489
+ const next = () => {
1490
+ this.liveSetTimer = setTimeout(
1491
+ () => {
1492
+ if (this.#stopping) return;
1493
+ this.stateLiveSet();
1494
+ next();
1495
+ },
1496
+ leaseDelay(this.liveSetIntervalMs, "next", Math.random()),
1497
+ );
1498
+ this.liveSetTimer.unref?.();
1499
+ };
1500
+ next();
1501
+ }
1502
+
580
1503
  /**
581
1504
  * An agent instance's LIVE surface (`GET /agents/:iid/surface` — ADR-0013): what the state that
582
1505
  * invoked this Agent accepts, right now. Undefined once nothing is registered (the state exited,
583
1506
  * the run settled, the iid is unknown) — the one catch point, and the reason the Harness never
584
1507
  * has to learn which turn is live: it asks, per turn, and the answer IS the turn.
585
1508
  *
586
- * "Right now" is load-bearing (ADR-0029): the registered defs are the state's VOCABULARY, derived
587
- * statically from its transitions, and the guards on those transitions are asked here — so an
588
- * event the Machine cannot currently accept is not offered. The pick has not happened yet, so the
589
- * question is `mayMove`, not `wouldMove`: a guard that would have judged the Agent's arguments is
590
- * left on the menu and settled at delivery. The Harness reads this afresh per Submission, so the filter lands at turn boundaries and never moves under
591
- * an Agent mid-turn.
1509
+ * "Right now" is the Allowed picks' half (ADR-0029): the registered defs are the state's
1510
+ * VOCABULARY, derived statically from its transitions, and the guards on those transitions are
1511
+ * asked here, as this Agent's sender (`allowedPicks`, ADR-0015). The pick has not happened yet, so
1512
+ * the question is payload-blind: a guard that would have judged the Agent's arguments is left
1513
+ * allowed and settled at delivery. The Menu does not move with them — it is the conversation's,
1514
+ * fixed at the Machine's build.
592
1515
  */
593
1516
  agentSurface(instanceId: string): AgentSurfaceView | undefined {
594
1517
  const reg = this.table.lookup(agentAddress(instanceId));
@@ -597,30 +1520,79 @@ export class RunHost {
597
1520
  instanceId,
598
1521
  runId: reg.runId,
599
1522
  sandbox: reg.sandbox,
600
- accepts: [...reg.defs.values()]
601
- .filter((def) => mayMove(reg.invoker, def.name))
602
- .map((def) => ({
603
- name: def.name,
604
- description: def.description,
605
- input: z.toJSONSchema(def.input),
606
- semantics: def.semantics,
607
- })),
1523
+ menu: [...menuOf(reg).values()].map((def) => ({
1524
+ name: def.name,
1525
+ description: def.description,
1526
+ input: z.toJSONSchema(def.input),
1527
+ semantics: def.semantics,
1528
+ ...briefingOf(def.input),
1529
+ })),
1530
+ allowed: allowedOf(reg),
608
1531
  };
609
1532
  }
610
1533
 
1534
+ /** The Harness read this Turn's surface and was answered (ADR-0026): the notices its admission
1535
+ * carried reach a model now, so an end to the Turn no longer raises them again (ADR-0062). */
1536
+ agentSurfaceServed(instanceId: string): void {
1537
+ this.table.lookup(agentAddress(instanceId))?.served?.();
1538
+ }
1539
+
611
1540
  /**
612
1541
  * Deliver one event from an Agent's Harness (`POST /agents/:iid/events` — ADR-0013). Validation
613
1542
  * and delivery are the table's; this only agent-scopes the address and mints the receipt. (A
614
- * `deferred` event never reaches here: `jr2Setup` refuses to build a Machine that declares one —
1543
+ * `poll` event never reaches here: `jr2Setup` refuses to build a Machine that declares one —
615
1544
  * ADR-0013.)
616
1545
  */
617
1546
  sendToAgent(instanceId: string, event: { type?: unknown } & Record<string, unknown>): AgentDeliveryReceipt {
618
1547
  const { type, ...payload } = event;
619
1548
  if (typeof type !== "string" || !type) {
620
- throw new EventValidationError(`event body must carry a string "type" (one of the surface's accepted names)`);
1549
+ throw new EventValidationError(`event body must carry a string "type" (one of the surface's Allowed picks)`);
621
1550
  }
1551
+ refuseSenderField(payload);
622
1552
  const address = agentAddress(instanceId);
623
1553
  const invoking = this.table.lookup(address);
1554
+ const run = invoking ? this.runs.get(invoking.runId) : undefined;
1555
+ // One pick at a time per Turn (ADR-0024): while a pick is outstanding, or the Agent's state is
1556
+ // busy with work it started in, every pick is refused — `finish` included, and before any other
1557
+ // judgment. Decided from the Machine (the Rest rule) and the run's holds, both of which a
1558
+ // restore brings back, so it holds after a restart. xstate itself would take the pick, and one
1559
+ // the Agent's state handles would exit the child state and stop the work running there.
1560
+ if (invoking && run) {
1561
+ this.settleHolds(run);
1562
+ const hold = run.holds.get(instanceId);
1563
+ const out = hold?.answer === undefined ? hold?.pick : undefined;
1564
+ if (out !== undefined || restOf(invoking) === "busy") {
1565
+ throw new OutstandingError(
1566
+ out !== undefined
1567
+ ? `"${type}" was not delivered: the pick "${out}" is still running in the workflow, and no other pick is taken until it answers`
1568
+ : `"${type}" was not delivered: the workflow is still running work in the state that asked for this Turn, and no pick is taken until it is back at rest`,
1569
+ out,
1570
+ );
1571
+ }
1572
+ }
1573
+ if (invoking && !invoking.defs.has(type)) {
1574
+ // A pick on the Menu that another state of this Agent handles (ADR-0029): answered, not
1575
+ // refused — `moved: false` is the correction receipt, whatever its arguments, so the payload
1576
+ // is not read. It never reaches the Machine: the invoking state does not handle it, so it
1577
+ // could move nothing of this Turn's, and only a state the Agent does not own (a nested
1578
+ // Agent's, a Gate's) could act on it — a widening the audience rule forbids (ADR-0015).
1579
+ if (menuOf(invoking).has(type)) {
1580
+ return {
1581
+ delivered: true,
1582
+ event: type,
1583
+ moved: false,
1584
+ turnComplete: false,
1585
+ allowed: allowedOf(invoking),
1586
+ deliveryId: this.newId(),
1587
+ };
1588
+ }
1589
+ // A name off the Menu is refused HERE, in the Agent's own terms: the table's refusal lists the
1590
+ // derived set, bare and unnarrowed, where the Harness is about to write the Allowed picks in
1591
+ // the model's tool names (ADR-0029, ADR-0062) — two lists that disagree.
1592
+ const refused = new EventValidationError(`the workflow's current state does not accept "${type}"`);
1593
+ refused.allowed = allowedOf(invoking);
1594
+ throw refused;
1595
+ }
624
1596
 
625
1597
  // Ask the guard question BEFORE delivering, and ask it with the VALIDATED payload — this is the
626
1598
  // exact form of the check the surface build can only approximate payload-blind (ADR-0029).
@@ -630,9 +1602,45 @@ export class RunHost {
630
1602
  // makes this unreliable and `deliver` throws on the next line anyway — so it fails open.
631
1603
  const def = invoking?.defs.get(type);
632
1604
  const parsed = def?.input.safeParse(payload ?? {});
633
- const moved = wouldMove(invoking?.invoker, parsed?.success ? { type, ...parsed.data } : { type, ...payload });
634
-
635
- this.table.deliver(address, type, payload);
1605
+ const probe = {
1606
+ ...(parsed?.success ? { type, ...parsed.data } : { type, ...payload }),
1607
+ ...(invoking?.sender ? { [SENDER_KEY]: invoking.sender } : {}),
1608
+ };
1609
+ const moved = wouldMove(invoking?.invoker, probe);
1610
+ // The states the pick names as targets, and the state value it leaves: after the send, the same
1611
+ // value with a target no longer active means it entered a state and came straight back
1612
+ // (ADR-0024); a new value, an in-Turn mode move.
1613
+ const targets = moved ? targetsOf(invoking?.invoker, probe) : [];
1614
+ const before = targets.length > 0 ? stateValueOf(invoking?.invoker) : undefined;
1615
+
1616
+ const outer = this.sending;
1617
+ const sending: Sending | undefined = invoking && { reg: invoking, pick: type, chain: [] };
1618
+ this.sending = sending;
1619
+ try {
1620
+ this.table.deliver(address, type, payload);
1621
+ } catch (err) {
1622
+ // The writing xstate would run for this pick is another party's (ADR-0015, ADR-0029): the
1623
+ // pick is ANSWERED, never delivered — `moved: false`, and the Harness picks the receipt by
1624
+ // whether the name is in `allowed`. Not in it: the correction receipt. In it: a guard that
1625
+ // read the payload turned the selection outward, and the receipt says to change the arguments.
1626
+ if (err instanceof NotTakenError && invoking) {
1627
+ return {
1628
+ delivered: true,
1629
+ event: type,
1630
+ moved: false,
1631
+ turnComplete: false,
1632
+ allowed: allowedOf(invoking),
1633
+ deliveryId: this.newId(),
1634
+ };
1635
+ }
1636
+ // A refused pick names the Allowed picks, read after the refusal (ADR-0029): one refusal
1637
+ // naming them is what brings a wrong pick back — the statement in the Briefing carries the
1638
+ // everyday case, and this is the net under it.
1639
+ if (err instanceof EventValidationError && invoking) err.allowed = allowedOf(invoking);
1640
+ throw err;
1641
+ } finally {
1642
+ this.sending = outer;
1643
+ }
636
1644
  // Read AFTER the delivery, off the SAME table the ADR-0024 guarantee uses — so the receipt
637
1645
  // reports what happened rather than what was hoped. `deliver` reached the invoking state's
638
1646
  // `sendBack` synchronously, so a pick that moved the Machine out of that state has already
@@ -640,15 +1648,331 @@ export class RunHost {
640
1648
  //
641
1649
  // IDENTITY, not existence: under `continue: true` the next state re-registers the SAME
642
1650
  // address for its own turn, and that is a new turn — this one still ended.
1651
+ const turnComplete = this.table.lookup(address) !== invoking;
1652
+ // A reply from below the Agent's state inside the pick's own send (ADR-0024): a child that
1653
+ // settled in the same macrostep. It is the answer's default as any later reply would be.
1654
+ const replied = moved && sending ? answerOf(sending) : undefined;
1655
+ // Where the answer goes if the route leaves the Agent's state: the conversation the picking
1656
+ // Turn continues. A fresh Turn has none — its conversation is never heard again, and a later
1657
+ // continued Turn of the same slot never made this pick.
1658
+ const conversation = invoking?.conversation;
1659
+ // The pick moved the workflow to work inside the Agent's state (ADR-0024): it is OUTSTANDING,
1660
+ // answered once the workflow is back at Rest there — read after the mailbox drains, by the
1661
+ // persist that follows every transition (`settleHolds`). The POST answers now, with the delivery
1662
+ // id the Harness polls; no single request waits on the work (ADR-0013).
1663
+ if (!turnComplete && invoking && run && moved && restOf(invoking) === "busy") {
1664
+ const deliveryId = this.newId();
1665
+ const since = new Date(this.now()).toISOString();
1666
+ const hold: Hold = {
1667
+ instanceId,
1668
+ deliveryId,
1669
+ pick: type,
1670
+ agent: invoking.sender?.id ?? "",
1671
+ since,
1672
+ ...(invoking.sandbox !== undefined ? { sandbox: invoking.sandbox } : {}),
1673
+ ...(invoking.invokeId !== undefined ? { invokeId: invoking.invokeId } : {}),
1674
+ ...(conversation !== undefined ? { conversation } : {}),
1675
+ ...(sending && sending.chain.length > 0 ? { chain: [...sending.chain] } : {}),
1676
+ ...(sending?.shaped !== undefined ? { shaped: sending.shaped } : {}),
1677
+ };
1678
+ this.closeHold(run.holds.get(instanceId));
1679
+ run.holds.set(instanceId, hold);
1680
+ this.holdRegs.set(hold, invoking);
1681
+ this.persist(run);
1682
+ return {
1683
+ delivered: true,
1684
+ event: type,
1685
+ moved: true,
1686
+ turnComplete: false,
1687
+ allowed: [],
1688
+ deliveryId,
1689
+ outstanding: true,
1690
+ since,
1691
+ };
1692
+ }
1693
+ // The child replied and the route ran inside the send: the pick is answered on its own POST,
1694
+ // as its poll would have answered it — back at Rest with the Allowed picks read after the
1695
+ // route, or out of the Agent's state with the answer as a notice to the conversation the picking
1696
+ // Turn continued (none for a fresh Turn).
1697
+ if (replied !== undefined && invoking) {
1698
+ if (turnComplete && conversation !== undefined && run) {
1699
+ run.notices.raise(
1700
+ { kind: "pick-answer", scope: "conversation", pick: type, cut: false, answer: replied },
1701
+ conversation,
1702
+ );
1703
+ }
1704
+ return {
1705
+ delivered: true,
1706
+ event: type,
1707
+ moved: true,
1708
+ turnComplete,
1709
+ allowed: turnComplete ? [] : allowedOf(invoking),
1710
+ deliveryId: this.newId(),
1711
+ answered: true,
1712
+ answer: replied,
1713
+ };
1714
+ }
1715
+ // It entered a state below the Agent's and came straight back inside its own send — a guard
1716
+ // back (an `always`), nothing replying: answered, "moved; nothing ran" (ADR-0024). A pick that
1717
+ // changed the state value moved to a mode, and one whose target is still active (the state it
1718
+ // was taken in) entered nothing it left: both get the plain receipt below.
1719
+ if (
1720
+ !turnComplete &&
1721
+ invoking &&
1722
+ before !== undefined &&
1723
+ stateValueOf(invoking.invoker) === before &&
1724
+ leftAgain(invoking.invoker, targets)
1725
+ ) {
1726
+ return {
1727
+ delivered: true,
1728
+ event: type,
1729
+ moved: true,
1730
+ turnComplete: false,
1731
+ allowed: allowedOf(invoking),
1732
+ deliveryId: this.newId(),
1733
+ answered: true,
1734
+ straightBack: true,
1735
+ };
1736
+ }
643
1737
  return {
644
1738
  delivered: true,
645
1739
  event: type,
646
1740
  moved,
647
- turnComplete: this.table.lookup(address) !== invoking,
1741
+ turnComplete,
1742
+ // Read off the SAME registration, after the delivery: a pick that moved the Machine within
1743
+ // the invoking state may have changed its guards. A Turn that is over may pick nothing.
1744
+ allowed: turnComplete || !invoking ? [] : allowedOf(invoking),
648
1745
  deliveryId: this.newId(),
649
1746
  };
650
1747
  }
651
1748
 
1749
+ /**
1750
+ * Answer every outstanding pick of `run` that can be answered now (ADR-0024). Called by the persist that
1751
+ * follows every transition — a microtask after the snapshot, so the mailbox has drained — and
1752
+ * before any read of a hold, so no reader sees a hold the Machine already answered.
1753
+ *
1754
+ * - The Agent's state is busy: still outstanding.
1755
+ * - Back at Rest in it: answered with the last reply since the pick and the Allowed picks read
1756
+ * AFTER the route — the guards read the context the route wrote (ADR-0029).
1757
+ * - It left the Agent's state (the registration is gone, or is another invocation's): the Turn
1758
+ * is over, and the answer goes as a notice to the next Turn of the conversation the picking
1759
+ * Turn continued; a fresh picking Turn's answer goes nowhere (ADR-0062).
1760
+ */
1761
+ private settleHolds(run: LiveRun): void {
1762
+ for (const hold of run.holds.values()) {
1763
+ if (hold.answer !== undefined) continue;
1764
+ const reg = this.table.lookup(agentAddress(hold.instanceId));
1765
+ let same = reg !== undefined && reg.runId === run.record.runId && reg.invokeId === hold.invokeId;
1766
+ if (same) {
1767
+ const opened = this.holdRegs.get(hold);
1768
+ if (opened === undefined) this.holdRegs.set(hold, reg!);
1769
+ else same = opened === reg;
1770
+ }
1771
+ const reading = same ? restOf(reg!) : "gone";
1772
+ if (reading === "busy") continue;
1773
+ // The answer's default is the last reply since the pick, whatever moved the Machine after it
1774
+ // (ADR-0024); with none, the answer carries no reply. Never `straightBack`: an outstanding
1775
+ // pick's work was running, so "nothing ran" is not this answer's to say.
1776
+ const answered = answerOf(hold);
1777
+ const answer = answered ? { answer: answered } : {};
1778
+ if (reading === "gone") {
1779
+ // Left by the route of the last reply — its own events and the events it sent itself at once
1780
+ // included — or by something else while work below still ran (a timer the Machine set, an
1781
+ // outside event): then the exit stopped that work, and the answer says so. It keeps the last
1782
+ // reply an earlier link gave.
1783
+ const stopped = hold.cutShort === true ? { stopped: true as const } : {};
1784
+ hold.answer = { turnComplete: true, allowed: [], ...answer, ...stopped };
1785
+ if (hold.conversation !== undefined) {
1786
+ run.notices.raise(
1787
+ {
1788
+ kind: "pick-answer",
1789
+ scope: "conversation",
1790
+ pick: hold.pick,
1791
+ cut: false,
1792
+ ...answer,
1793
+ ...stopped,
1794
+ },
1795
+ hold.conversation,
1796
+ );
1797
+ }
1798
+ } else {
1799
+ hold.answer = { turnComplete: false, allowed: allowedOf(reg!), ...answer };
1800
+ }
1801
+ // The answer is fixed: nothing reads the chain or the shaped value again, so the next save
1802
+ // drops them. A loop below the Agent's state grows the blob by its tails only while one pick
1803
+ // is outstanding (ADR-0067, ADR-0065).
1804
+ delete hold.chain;
1805
+ delete hold.shaped;
1806
+ this.closeHold(hold);
1807
+ }
1808
+ }
1809
+
1810
+ /**
1811
+ * The pick an `answer` action in `machine` shapes (ADR-0067), from `seat`, the one place it runs
1812
+ * from (`placeAnswers`). At an Agent's state: only a pick of the Agent seated there — the one in
1813
+ * its own send, else the one outstanding. On a transition below no Agent's state (`null`): the
1814
+ * event that took the transition names the Agent — the sender of a pick, or the Agent whose pick
1815
+ * started the child whose done or error event it is — and that Agent's pick is the one shaped, and
1816
+ * no other; none when that Agent has no pick being answered. When one slot is invoked in two
1817
+ * regions, the event names an invoke too, and only the pick at that invoke is shaped: the sending
1818
+ * pick, or the pick at the invoke whose pick started the child; none when that pick is not being
1819
+ * answered (its Turn ended, and its child runs on). An event that names no Agent (a timer,
1820
+ * a Gate's answer, an outside send) shapes the pick in its own send. Outside one, it shapes the one
1821
+ * outstanding pick only while one Agent's state is active; while two Agents' states are active it
1822
+ * shapes none, whatever is outstanding: the event cannot say whose pick it is, and the other
1823
+ * Agent's pick keeps its default answer.
1824
+ */
1825
+ private answeringIn(
1826
+ machine: AnyActorRef,
1827
+ seat: AnswerSeat,
1828
+ runId: string,
1829
+ holds: Map<string, Hold>,
1830
+ startedBy: Record<string, StartedBy>,
1831
+ event?: AnyEventObject,
1832
+ ): Sending | Hold | undefined {
1833
+ const seatedFor = (reg: Registration | undefined): boolean => {
1834
+ if (reg?.invoker !== machine) return false;
1835
+ if (seat === null) return true;
1836
+ const snapshot = machine.getSnapshot() as AnyMachineSnapshot;
1837
+ return reg.invokeId !== undefined && seatOf(snapshot, reg.invokeId) === seat;
1838
+ };
1839
+ const regOf = (hold: Hold): Registration | undefined =>
1840
+ this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
1841
+ const sending = this.sending && this.sending.reg.runId === runId ? this.sending : undefined;
1842
+ const open = [...holds.values()].filter((hold) => hold.answer === undefined && seatedFor(regOf(hold)));
1843
+ if (seat === null && event !== undefined) {
1844
+ const named = agentOfEvent(event, startedBy, machine, sending);
1845
+ if (named !== undefined) {
1846
+ const { agent } = named;
1847
+ const own = sending && sending.reg.sender?.id === agent ? sending : undefined;
1848
+ // That Agent's picks being answered: the one in its own send, and the ones outstanding at
1849
+ // its other invokes (one slot invoked in two regions). The event names the invoke of the
1850
+ // pick that sent it or started its child, and only the pick at that invoke is shaped; none
1851
+ // when that pick is not being answered, and each other pick keeps its default answer.
1852
+ const held = open.filter((hold) => hold.agent === agent && hold.instanceId !== own?.reg.id);
1853
+ const picks = [...(own ? [own] : []), ...held];
1854
+ const at = (pick: Sending | Hold): string | undefined =>
1855
+ pick === own ? own.reg.invokeId : (pick as Hold).invokeId;
1856
+ const same = named.invoke === undefined ? picks : picks.filter((pick) => at(pick) === named.invoke);
1857
+ if (same.length !== 1) return undefined;
1858
+ const pick = same[0]!;
1859
+ if (pick === own) return seatedFor(own.reg) ? own : undefined;
1860
+ return pick;
1861
+ }
1862
+ }
1863
+ if (sending && seatedFor(sending.reg)) return sending;
1864
+ if (seat === null && activeAgentStates(machine.getSnapshot() as AnyMachineSnapshot) > 1) return undefined;
1865
+ return open.length === 1 ? open[0] : undefined;
1866
+ }
1867
+
1868
+ /**
1869
+ * The Agent whose outstanding pick holds the child `childId` that `machine` just started, or
1870
+ * undefined: the child is invoked on a state strictly below that Agent's state, so by the Rest rule
1871
+ * it is that pick's work (ADR-0024). Below two such states (one Agent's state inside another's),
1872
+ * it is the work of the nearer one's pick. `snapshot` is `machine`'s snapshot after the step.
1873
+ */
1874
+ private outstandingAbove(
1875
+ machine: AnyActorRef,
1876
+ snapshot: AnyMachineSnapshot,
1877
+ childId: string,
1878
+ holds: Map<string, Hold>,
1879
+ ): StartedBy | undefined {
1880
+ const node = seatOf(snapshot, childId);
1881
+ if (node === undefined) return undefined;
1882
+ const byState = new Map<unknown, StartedBy>();
1883
+ for (const hold of holds.values()) {
1884
+ if (hold.answer !== undefined) continue;
1885
+ const reg = this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
1886
+ if (reg?.invoker !== machine || reg.invokeId === undefined) continue;
1887
+ const agentState = seatOf(snapshot, reg.invokeId);
1888
+ if (agentState !== undefined) byState.set(agentState, { agent: hold.agent, invoke: reg.invokeId });
1889
+ }
1890
+ // The nearest Agent's state above the child: with B's state inside A's, the child is B's work.
1891
+ for (let n = node.parent; n; n = n.parent) {
1892
+ const starter = byState.get(n);
1893
+ if (starter !== undefined) return starter;
1894
+ }
1895
+ return undefined;
1896
+ }
1897
+
1898
+ /** Wake every long poll on a hold: it was answered, replaced, or the host is going away. */
1899
+ private closeHold(hold: Hold | undefined): void {
1900
+ if (hold === undefined) return;
1901
+ const waiters = this.holdWaiters.get(hold);
1902
+ this.holdWaiters.delete(hold);
1903
+ for (const wake of waiters ?? []) wake();
1904
+ }
1905
+
1906
+ /** An outstanding pick's delivery: its run and hold, by Agent address and delivery id. */
1907
+ private holdAt(instanceId: string, deliveryId: string): { run: LiveRun; hold: Hold } | undefined {
1908
+ for (const run of this.runs.values()) {
1909
+ const hold = run.holds.get(instanceId);
1910
+ if (hold?.deliveryId === deliveryId) return { run, hold };
1911
+ }
1912
+ return undefined;
1913
+ }
1914
+
1915
+ /** Is this delivery an outstanding pick of a run this host parked? Its poll is asked again. */
1916
+ parkedDelivery(instanceId: string, deliveryId: string): boolean {
1917
+ return this.parked.has(`${instanceId} ${deliveryId}`);
1918
+ }
1919
+
1920
+ /** The Sandbox that may read a delivery's answer (ADR-0013), or undefined for an unknown one. */
1921
+ deliveryScope(instanceId: string, deliveryId: string): { sandbox?: string } | undefined {
1922
+ const found = this.holdAt(instanceId, deliveryId);
1923
+ return found && { ...(found.hold.sandbox !== undefined ? { sandbox: found.hold.sandbox } : {}) };
1924
+ }
1925
+
1926
+ /**
1927
+ * One long poll of an outstanding pick's answer (`GET /agents/:iid/deliveries/:id` — ADR-0013, ADR-0024):
1928
+ * the answer when the workflow is back at Rest in the Agent's state or has left it, else
1929
+ * `outstanding` after at most `waitMs`, and the Harness asks again. Undefined for a delivery this
1930
+ * host does not hold — a pick that was never outstanding, or a run that ended.
1931
+ *
1932
+ * `signal` is the poll's connection. A poll whose caller went away (a killed Harness) ends then,
1933
+ * `outstanding`, and never marks the answer read: no model read it, so the re-admission after
1934
+ * the restart must still carry it (ADR-0031).
1935
+ */
1936
+ async awaitDelivery(
1937
+ instanceId: string,
1938
+ deliveryId: string,
1939
+ waitMs = 25_000,
1940
+ signal?: AbortSignal,
1941
+ ): Promise<AgentDeliveryReceipt | undefined> {
1942
+ const found = this.holdAt(instanceId, deliveryId);
1943
+ if (!found) return undefined;
1944
+ const { run, hold } = found;
1945
+ this.settleHolds(run);
1946
+ if (hold.answer === undefined && !this.#stopping && !signal?.aborted) {
1947
+ await new Promise<void>((resolve) => {
1948
+ let waiters = this.holdWaiters.get(hold);
1949
+ if (!waiters) this.holdWaiters.set(hold, (waiters = new Set()));
1950
+ const timer = setTimeout(wake, waitMs);
1951
+ signal?.addEventListener("abort", wake, { once: true });
1952
+ function wake(): void {
1953
+ clearTimeout(timer);
1954
+ signal?.removeEventListener("abort", wake);
1955
+ waiters!.delete(wake);
1956
+ resolve();
1957
+ }
1958
+ waiters.add(wake);
1959
+ });
1960
+ }
1961
+ const base = { delivered: true as const, event: hold.pick, moved: true, deliveryId };
1962
+ if (hold.answer === undefined || signal?.aborted) {
1963
+ return { ...base, turnComplete: false, allowed: [], outstanding: true, since: hold.since };
1964
+ }
1965
+ hold.fetched = true;
1966
+ return {
1967
+ ...base,
1968
+ turnComplete: hold.answer.turnComplete,
1969
+ allowed: hold.answer.allowed,
1970
+ answered: true,
1971
+ ...(hold.answer.answer !== undefined ? { answer: hold.answer.answer } : {}),
1972
+ ...(hold.answer.stopped ? { stopped: true as const } : {}),
1973
+ };
1974
+ }
1975
+
652
1976
  /** A run's open gates as callers discover them (`GET /runs/:id` — ADR-0011). Settled/unknown
653
1977
  * run → empty: a gate is a LIVE surface, it does not outlive its state. */
654
1978
  gates(runId: string): GateView[] {
@@ -663,20 +1987,50 @@ export class RunHost {
663
1987
  description: def.description,
664
1988
  input: z.toJSONSchema(def.input),
665
1989
  })),
1990
+ acceptsNow: acceptsNow(reg),
666
1991
  meta: reg.meta,
667
1992
  }));
668
1993
  }
669
1994
 
670
- /** Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
671
- * Validation and delivery are the table's — this only run-scopes the address. */
1995
+ /**
1996
+ * Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
1997
+ * Validation and delivery are the table's — this run-scopes the address, and returns only when
1998
+ * the run TOOK the event (ADR-0013): one that would run another party's writing, or that no
1999
+ * writing would take with this payload, throws {@link NotTakenError} naming what the Gate takes
2000
+ * now, and nothing is delivered.
2001
+ */
672
2002
  sendToGate(runId: string, gate: string, event: { type?: unknown } & Record<string, unknown>): void {
673
2003
  const { type, ...payload } = event;
674
2004
  if (typeof type !== "string" || !type) {
675
2005
  throw new EventValidationError(`event body must carry a string "type" (one of the gate's accepted names)`);
676
2006
  }
2007
+ const reg = this.table.lookup(gateAddress(runId, gate));
2008
+ const refused = (why: string): NotTakenError =>
2009
+ new NotTakenError(
2010
+ `gate "${gate}" is open, and the run does not take "${type}" now: ${why} — not delivered ` +
2011
+ `(takes now: ${reg ? acceptsNow(reg).join(", ") || "nothing" : "nothing"})`,
2012
+ );
677
2013
  try {
2014
+ // Exact, with the validated payload: a send no writing would move on is refused, never
2015
+ // answered `{ ok: true }`. A bad payload or an unaccepted name falls through to the table,
2016
+ // which refuses it with its own words.
2017
+ const def = reg?.defs.get(type);
2018
+ const parsed = def && !(payload && SENDER_KEY in payload) ? def.input.safeParse(payload ?? {}) : undefined;
2019
+ if (reg && parsed?.success && !wouldMove(reg.invoker, { type, ...parsed.data, [SENDER_KEY]: reg.sender })) {
2020
+ throw refused("no writing of it would move the run with this payload");
2021
+ }
678
2022
  this.table.deliver(gateAddress(runId, gate), type, payload);
679
2023
  } catch (err) {
2024
+ if (err instanceof NotTakenError && err.writings.length) {
2025
+ throw refused(
2026
+ err.writings
2027
+ .map(
2028
+ (w) =>
2029
+ `it would run the writing on "${w.state}"${w.owners.length ? `, which is ${w.owners.map(senderText).join(" and ")}'s` : ""}`,
2030
+ )
2031
+ .join("; "),
2032
+ );
2033
+ }
680
2034
  if (err instanceof UnknownAddressError) {
681
2035
  const open = this.gates(runId).map((g) => g.gate);
682
2036
  throw new UnknownAddressError(
@@ -699,7 +2053,10 @@ export class RunHost {
699
2053
  const run = this.runs.get(runId);
700
2054
  if (!run) return () => {};
701
2055
  run.listeners.add(listener);
702
- listener({ kind: "status", status: this.liveStatus(run) });
2056
+ // The status the feed last carried, the SAME object for every attach, so http.ts renders the
2057
+ // replay once however many readers attach — a reconnect after a close is the common attach
2058
+ // (ADR-0022). A transition not yet persisted follows on the feed within the microtask.
2059
+ listener(run.latestStatus ?? { kind: "status", status: this.liveStatus(run) });
703
2060
  return () => run.listeners.delete(listener);
704
2061
  }
705
2062
 
@@ -754,7 +2111,8 @@ export class RunHost {
754
2111
  * Read a run's status, **reading through to the store** when it is no longer live (ADR-0009). A
755
2112
  * completed run's final snapshot is persisted before `persist()` drops it from the registry, so a
756
2113
  * terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
757
- * undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared).
2114
+ * undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared) —
2115
+ * and a swept one is unknown: a finished run's row lives a week (ADR-0065).
758
2116
  */
759
2117
  async read(runId: string): Promise<RunStatus | undefined> {
760
2118
  const live = this.status(runId);
@@ -763,19 +2121,24 @@ export class RunHost {
763
2121
  const blob = stored?.snapshot as RunBlob | null | undefined;
764
2122
  if (!stored || !blob) return undefined;
765
2123
  const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown };
2124
+ // The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
2125
+ // cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
2126
+ // (ADR-0025). While the row still says "live", the Machine's own status is the answer.
2127
+ const status = stored.status === "live" ? (snap.status ?? stored.status) : stored.status;
766
2128
  return {
767
2129
  runId,
768
2130
  workflow: blob.workflow,
769
2131
  instanceId: blob.instanceId,
770
- // The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
771
- // cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
772
- // (ADR-0025). While the row still says "live", the Machine's own status is the answer.
773
- status: stored.status === "live" ? (snap.status ?? stored.status) : stored.status,
2132
+ status,
774
2133
  value: snap.value,
775
2134
  context: snap.context,
776
2135
  children: runChildren(snap), // the persisted `children` map — same tree, off the store
2136
+ waiting: status === "active" ? runWaiting(snap) : [],
2137
+ held: status === "active" ? heldOf(blob.holds ?? []) : [],
777
2138
  fault: blob.fault,
778
2139
  reason: stored.reason,
2140
+ ...(blob.assembly !== undefined ? { assembly: blob.assembly } : {}),
2141
+ ...boundFields(snap),
779
2142
  };
780
2143
  }
781
2144
 
@@ -805,6 +2168,25 @@ export class RunHost {
805
2168
  .map(observe);
806
2169
  }
807
2170
 
2171
+ /** The host is going away (`close()` has begun): its Turns are not over, and the Agents' surface
2172
+ * must not say they are (ADR-0026). */
2173
+ get stopping(): boolean {
2174
+ return this.#stopping;
2175
+ }
2176
+
2177
+ /** The live rows of a prior process are not restored yet: a boot with no current Assembly
2178
+ * deferred their restore to the first publish (ADR-0069). Those Turns are not over, so the
2179
+ * Agents' surface must not say they are (ADR-0026). Set by `deferRestore`, cleared when a
2180
+ * `restore()` completes; a restore that throws as a whole leaves it set. */
2181
+ get restorePending(): boolean {
2182
+ return this.#restorePending;
2183
+ }
2184
+
2185
+ /** Mark the restore deferred: until a `restore()` completes, `restorePending` is true. */
2186
+ deferRestore(): void {
2187
+ this.#restorePending = true;
2188
+ }
2189
+
808
2190
  /**
809
2191
  * End every open feed because the host is going away — the shutdown counterpart to `subscribe`.
810
2192
  *
@@ -815,6 +2197,15 @@ export class RunHost {
815
2197
  * OBSERVATION, not the work — the snapshots are already durable, and `restore()` picks them up.
816
2198
  */
817
2199
  async close(): Promise<void> {
2200
+ // From here on the Orchestrator is going away, not its Turns: `/agents/*` answers 503, which a
2201
+ // Harness re-asks on ADR-0042's ladder, and never 404, which says a Turn is over (ADR-0026).
2202
+ this.#stopping = true;
2203
+ clearTimeout(this.liveSetTimer);
2204
+ // A run's timers stay armed (a run is untouched by a close), but none holds the process alive.
2205
+ for (const timer of this.timers) (timer as { unref?: () => void }).unref?.();
2206
+ // Every long poll answers `outstanding` now, so the server can close; the Harness asks again,
2207
+ // and a restarted Orchestrator answers it from the same hold (ADR-0024).
2208
+ for (const hold of [...this.holdWaiters.keys()]) this.closeHold(hold);
818
2209
  for (const run of this.runs.values()) {
819
2210
  for (const listener of run.listeners) listener({ kind: "closed" });
820
2211
  run.listeners.clear();
@@ -841,6 +2232,14 @@ export class RunHost {
841
2232
  const run = this.runs.get(runId);
842
2233
  if (!run) return;
843
2234
  run.binding.hostStopping = true;
2235
+ // A parked run's outstanding picks are not over (ADR-0024): their polls are answered `503` —
2236
+ // ask again — never `404`, which tells the Harness its Turn ended. The host that restores the
2237
+ // run answers them from the saved holds.
2238
+ for (const hold of run.holds.values()) {
2239
+ if (hold.answer !== undefined) continue;
2240
+ this.parked.add(`${hold.instanceId} ${hold.deliveryId}`);
2241
+ this.closeHold(hold);
2242
+ }
844
2243
  // Say so BEFORE the actor stops, while there is still a status to read. A stopped run is not a
845
2244
  // settled one — `persist()` never runs for it (the tracked-run guard drops the scheduled save,
846
2245
  // keeping the stored status "live" for restore — see `spawn`), so nothing else on this path
@@ -859,14 +2258,37 @@ export class RunHost {
859
2258
  * Ending the turns and refusing to restore are one decision, not two: a cancelled run that came
860
2259
  * back would re-attach to submissions that settled `aborted`, and `settle`'s rejection would
861
2260
  * fault a run whose Agents were stopped on purpose.
2261
+ *
2262
+ * It then destroys every Sandbox the run owns, unless `keep` (`jr2 send <run> --event CANCEL
2263
+ * --keep`): kept, they age out at the operator's idle timeout, and `jr2 ssh` reaches them until
2264
+ * then. A faulted run never comes through here, and keeps its Workspaces (ADR-0012).
862
2265
  */
863
- async cancel(runId: string): Promise<void> {
2266
+ async cancel(runId: string, opts: { keep?: boolean } = {}): Promise<void> {
864
2267
  const run = this.runs.get(runId);
865
2268
  if (!run) return;
2269
+ // Named BEFORE the stop, while the tree still says which Workspaces are live.
2270
+ const sandboxes = opts.keep ? [] : runSandboxes(run.actor, runId);
866
2271
  run.actor.stop();
867
2272
  // Persist AFTER the stop: the snapshot is final, and `persist` fans out the last status,
868
2273
  // announces `gone`, and untracks — the same terminal path a run that settled on its own takes.
869
2274
  this.persist(run, "cancelled");
2275
+ // The run is over, so nothing can use its Workspaces, and each holds its whole Size (ADR-0025).
2276
+ // An actor stop skips the wrapper's `teardown` state, so the host runs the same idempotent
2277
+ // delete here, after the stop, so no lease renews what was deleted. `keep` leaves them to the
2278
+ // operator's idle timeout, for inspection. A delete that fails is the idle GC's, as in
2279
+ // `teardown` (ADR-0012 backstop): the cancel has already happened.
2280
+ const port = this.sandbox;
2281
+ if (!port) return;
2282
+ await Promise.all(
2283
+ sandboxes.map((name) =>
2284
+ port.destroy(name).catch((err: unknown) => {
2285
+ console.error(
2286
+ `run ${runId}: cancel could not delete Sandbox ${name} (the idle timeout will): ` +
2287
+ (err instanceof Error ? err.message : String(err)),
2288
+ );
2289
+ }),
2290
+ ),
2291
+ );
870
2292
  }
871
2293
 
872
2294
  /** Feed a run's final word to both granularities: where it ended, then that it is gone. */
@@ -879,19 +2301,25 @@ export class RunHost {
879
2301
 
880
2302
  /** Put one event on the run's feed: buffer it for ADR-0023's backfill, then fan it out. The
881
2303
  * buffer and the fan-out share one seat so an attached echo and a later attach see the SAME
882
- * feed — a projection cannot drift from a record it is read out of. */
2304
+ * feed — a projection cannot drift from a record it is read out of. Every listener is handed the
2305
+ * SAME event object, which is what lets http.ts render each frame once (ADR-0022). */
883
2306
  private feed(run: LiveRun, event: RunFeedEvent): void {
884
- run.feedSoFar.push(event);
885
- if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
2307
+ if (event.kind === "status") {
2308
+ run.latestStatus = event;
2309
+ } else {
2310
+ run.feedSoFar.push(event);
2311
+ if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
2312
+ }
886
2313
  for (const listener of run.listeners) listener(event);
887
2314
  }
888
2315
 
889
2316
  /**
890
2317
  * Attach the run-narrative echo (ADR-0023): replay the run's feed-so-far to the Workspace's
891
- * Harness at `endpoint` — the backfilled preamble, why this Workspace exists — then tee live
892
- * until detached. FIRE-AND-FORGET is this method's contract: pushes are chained so events
893
- * arrive in feed order, and a failure is logged once and swallowed — the feed remains the
894
- * record, the log is a courtesy view, and nothing here can fail a turn, a state, or a run.
2318
+ * Harness at `endpoint` — the backfilled preamble, why this Workspace exists: its markers and
2319
+ * Emits in feed order, then where the run stands now — then tee live until detached, through an
2320
+ * {@link echoOutbox}, which coalesces statuses. FIRE-AND-FORGET is this method's contract: a
2321
+ * failure is logged once and swallowed — the feed remains the record, the log is a courtesy
2322
+ * view, and nothing here can fail a turn, a state, or a run.
895
2323
  * Scoped to the OWNING run's lineage by construction: it reads one run's buffer and listeners
896
2324
  * and nothing else — a sibling run's events cannot reach this endpoint through here.
897
2325
  */
@@ -899,25 +2327,19 @@ export class RunHost {
899
2327
  const run = this.runs.get(runId);
900
2328
  const factory = this.echoFactory;
901
2329
  if (!run || !factory) return () => {};
902
- const push = factory(endpoint, this.harnessBearer?.(placement));
903
- let chain = Promise.resolve();
904
2330
  let reported = false;
905
- const enqueue = (events: EchoEvent[]): void => {
906
- if (events.length === 0) return;
907
- chain = chain
908
- .then(() => push(events))
909
- .catch((err) => {
910
- // Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
911
- // transition into an error line, and must not surface anywhere a run could trip on.
912
- if (reported) return;
913
- reported = true;
914
- console.error(
915
- `run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
916
- (err instanceof Error ? err.message : String(err)),
917
- );
918
- });
919
- };
920
- enqueue(run.feedSoFar.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
2331
+ const enqueue = echoOutbox(factory(endpoint, this.harnessBearer?.(placement)), (err) => {
2332
+ // Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
2333
+ // transition into an error line, and must not surface anywhere a run could trip on.
2334
+ if (reported) return;
2335
+ reported = true;
2336
+ console.error(
2337
+ `run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
2338
+ (err instanceof Error ? err.message : String(err)),
2339
+ );
2340
+ });
2341
+ const preamble = run.latestStatus ? [...run.feedSoFar, run.latestStatus] : run.feedSoFar;
2342
+ enqueue(preamble.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
921
2343
  const listener = (ev: RunFeedEvent): void => {
922
2344
  const projected = echoEventOf(ev, endpoint);
923
2345
  if (projected) enqueue([projected]);
@@ -935,7 +2357,11 @@ export class RunHost {
935
2357
  value: snap.value,
936
2358
  context: snap.context,
937
2359
  children: runChildren(snap),
2360
+ waiting: runWaiting(snap, run.actor.logic),
2361
+ held: heldOf(run.holds.values()),
938
2362
  fault: run.fault,
2363
+ ...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
2364
+ ...boundFields(snap),
939
2365
  };
940
2366
  }
941
2367
 
@@ -960,7 +2386,13 @@ export class RunHost {
960
2386
  def: WorkflowDef,
961
2387
  agents: Record<string, AgentAdmission> = {},
962
2388
  epochs: Record<string, number> = {},
2389
+ noticeState?: NoticeLedgerState,
2390
+ holdState: Hold[] = [],
2391
+ effects: Record<string, EffectRow> = {},
2392
+ startedBy: Record<string, StartedBy> = {},
963
2393
  ): AnyActor {
2394
+ const notices = new NoticeLedger(noticeState, this.newId);
2395
+ const holds = new Map(holdState.map((hold) => [hold.instanceId, hold]));
964
2396
  let live: LiveRun | undefined;
965
2397
  let scheduled = false;
966
2398
  const schedule = () => {
@@ -974,30 +2406,47 @@ export class RunHost {
974
2406
  const binding: RunBinding = {
975
2407
  runId: record.runId,
976
2408
  workflow: record.workflow,
2409
+ ...(def.assembly !== undefined ? { assembly: def.assembly } : {}),
977
2410
  table: this.table,
978
2411
  sandbox: this.sandbox,
979
2412
  instanceHarness: this.instanceHarness,
980
2413
  ...(this.harnessBearer ? { harnessBearer: this.harnessBearer } : {}),
981
- // The admission ledger's write half (ADR-0016): the Agent actor reports the durable handle the
2414
+ ...(this.effector ? { effector: this.effector } : {}),
2415
+ // The admission ledger's write half (ADR-0016): the Agent's Actor reports the durable handle the
982
2416
  // moment the Harness admits it, and the ledger hits the store in the same RunBlob save. An
983
2417
  // admission arriving around stop/untrack still lands in `agents` but skips the save,
984
2418
  // exactly like the persist scheduler's tracked-run guard.
985
- recordAdmission: (instanceId, admission) => {
2419
+ recordAdmission: (instanceId, admission, delivered = []) => {
986
2420
  agents[instanceId] = admission;
2421
+ // The notices this admission carried are delivered by THIS save (ADR-0062).
2422
+ notices.delivered(delivered);
987
2423
  const run = this.runs.get(record.runId);
988
2424
  if (run && run.agents === agents) this.persist(run);
989
2425
  },
2426
+ admissionOf: (instanceId) => agents[instanceId],
990
2427
  // The epoch ledger's two halves (ADR-0057). The read is what the input mapper mints from;
991
2428
  // the write is the terminal `agent.fault` burning the conversation it ended, saved in the
992
2429
  // same RunBlob as the admission ledger — and, like it, saved AT THE WRITE rather than
993
2430
  // waiting for the next transition, because a crash between the fault and the next state's
994
2431
  // invoke would otherwise restore into the burned conversation.
995
2432
  epochOf: (conversation) => epochs[conversation] ?? 0,
996
- bumpEpoch: (conversation) => {
2433
+ bumpEpoch: (conversation, reason) => {
997
2434
  epochs[conversation] = (epochs[conversation] ?? 0) + 1;
2435
+ // The next `continue` lands on a virgin conversation and is told so, with the fault's own
2436
+ // words (ADR-0062) — raised in the SAME save as the bump, so no restart can split them.
2437
+ notices.raise({ kind: "conversation-new", scope: "conversation", reason }, conversation);
998
2438
  const run = this.runs.get(record.runId);
999
2439
  if (run && run.epochs === epochs) this.persist(run);
1000
2440
  },
2441
+ // The notices' other halves (ADR-0062). A raise is saved at the write, like the ledgers; a
2442
+ // take is a reservation in memory, delivered by `recordAdmission` or released on failure.
2443
+ raiseNotice: (notice, to, source) => {
2444
+ if (!notices.raise(notice, to, source)) return;
2445
+ const run = this.runs.get(record.runId);
2446
+ if (run && run.notices === notices) this.persist(run);
2447
+ },
2448
+ takeNotices: (scope) => notices.take(scope),
2449
+ releaseNotices: (ids) => notices.release(ids),
1001
2450
  // Absorbed-retry attempts go straight to the run's observers (SSE/CLI watch) — they are
1002
2451
  // feed events, not machine events (ADR-0016: the workflow sees only the terminal fault).
1003
2452
  telemetry: (event) => {
@@ -1007,15 +2456,83 @@ export class RunHost {
1007
2456
  // Turn markers (ADR-0023) ride the per-run feed alone — the prompt is Instance-token-class
1008
2457
  // data, so they never touch the workflow feed (ADR-0014's open band strips even Emit
1009
2458
  // payloads there).
2459
+ // An Effect's line that comes after its run ended — a stop's answer on a cancel or a final
2460
+ // state (ADR-0067) — has no feed left to ride, so it goes to the Orchestrator's log.
1010
2461
  marker: (event) => {
1011
2462
  const run = this.runs.get(record.runId);
1012
2463
  if (run && run.binding === binding) this.feed(run, event);
2464
+ else if (!run && event.kind === "effect") {
2465
+ console.error(`run ${record.runId}: effect ${event.slot} (${event.effect}) ${event.stage}: ${event.text}`);
2466
+ }
1013
2467
  },
1014
2468
  // The run-narrative echo attach (ADR-0023), called from `workspace()`'s registrar.
1015
2469
  echo: (endpoint, placement) => this.attachEcho(record.runId, endpoint, placement),
2470
+ // The outstanding pick of the Turn now at this address, as the Agent's Actor reads it when its
2471
+ // Harness restarted (ADR-0024/0031). Settled first, so a hold whose Turn is over reads
2472
+ // `turnComplete`: that is an EARLIER Turn's pick (a route left its state, and the answer went as
2473
+ // a notice to the next Turn, or nowhere) — never this Turn's, whatever it continues.
2474
+ outstandingOf: (instanceId) => {
2475
+ const run = this.runs.get(record.runId);
2476
+ if (run && run.holds === holds) this.settleHolds(run);
2477
+ const hold = holds.get(instanceId);
2478
+ if (hold?.answer?.turnComplete) return undefined;
2479
+ return (
2480
+ hold && {
2481
+ pick: hold.pick,
2482
+ answered: hold.answer !== undefined,
2483
+ turnComplete: hold.answer?.turnComplete ?? false,
2484
+ ...(hold.answer?.answer !== undefined ? { answer: hold.answer.answer } : {}),
2485
+ fetched: hold.fetched === true,
2486
+ }
2487
+ );
2488
+ },
2489
+ // The `answer` action's two halves (ADR-0067): the chain of the one pick being answered of an
2490
+ // Agent whose state the action is written below — the one in its own send first, else the one
2491
+ // outstanding — and the value that replaces its default answer.
2492
+ chainOf: (machine, seat, event) => {
2493
+ const pick = this.answeringIn(machine, seat, record.runId, holds, startedBy, event);
2494
+ if (pick === undefined) return undefined;
2495
+ return "reg" in pick ? [...pick.chain] : [...(pick.chain ?? [])];
2496
+ },
2497
+ setAnswer: (machine, seat, value, event) => {
2498
+ const pick = this.answeringIn(machine, seat, record.runId, holds, startedBy, event);
2499
+ if (pick === undefined) return;
2500
+ // Not saved here: the action runs inside a transition, and the persist that follows it carries
2501
+ // the value — a save now would read the Machine before its route, and answer too early.
2502
+ pick.shaped = jsonOf(value);
2503
+ },
2504
+ // The Effect ledger's two halves (ADR-0067). A write is saved AT THE WRITE, like an admission:
2505
+ // an intent row must be on disk, with the snapshot that holds its Effect, before the call
2506
+ // leaves. A row written inside a transition (an abort) waits for the save that carries that
2507
+ // transition — a cancel's final save included, which untracks the run — and is saved after
2508
+ // it only when nothing else did. A row written around a park lands in memory and skips the save.
2509
+ effectRow: (effectId) => effects[effectId],
2510
+ effectRows: () => effects,
2511
+ recordEffect: (effectId, row, save = "now") => {
2512
+ effects[effectId] = row;
2513
+ const persist = (): void => {
2514
+ const run = this.runs.get(record.runId);
2515
+ if (run && run.effects === effects) this.persist(run);
2516
+ };
2517
+ if (save === "now") persist();
2518
+ else queueMicrotask(persist);
2519
+ },
2520
+ heldAnswerDelivered: (instanceId) => {
2521
+ const hold = holds.get(instanceId);
2522
+ if (hold) hold.fetched = true;
2523
+ },
2524
+ // A Chart by name (ADR-0070, D-C): among the Charts of the wrapper that is the run's Workflow.
2525
+ chartNamed: (name) => this.namedChart(wrapperName(record.workflow), name)?.text,
1016
2526
  };
1017
2527
  let bound = false;
2528
+ let firedTimer: ((source: unknown, event: unknown) => boolean) | undefined;
2529
+ // The children each Machine of the tree had after its last microstep: what the next one started
2530
+ // is the difference (ADR-0067's attribution of a child to the pick whose work started it).
2531
+ const childrenAfter = new WeakMap<AnyActorRef, Set<AnyActorRef>>();
2532
+ const childRefsOf = (snapshot: unknown): Set<AnyActorRef> =>
2533
+ new Set(Object.values((snapshot as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {}));
1018
2534
  const actor = createActor(machine, {
2535
+ clock: this.clock,
1019
2536
  ...options,
1020
2537
  inspect: (ev) => {
1021
2538
  // Bind the run's actor SYSTEM on the ROOT's creation event — the first inspection event,
@@ -1026,12 +2543,64 @@ export class RunHost {
1026
2543
  if (!bound && ev.type === "@xstate.actor") {
1027
2544
  bound = true;
1028
2545
  bindRun((ev.actorRef as AnyActorRef).system, binding);
2546
+ // …and watch its scheduler before anything is armed: whether an event the Machine took
2547
+ // was a timer it set, or an event an actor sent at once, decides `cutShort` below.
2548
+ firedTimer = watchTimers((ev.actorRef as AnyActorRef).system);
2549
+ }
2550
+ // A restored hold is the Turn of the FIRST registration of its address in this tree: bind
2551
+ // it as soon as one exists, before an event can end that Turn and start another at the same
2552
+ // address (`continue: true` re-registers it). Every inspection event, so it is bound before
2553
+ // the restored Machine processes anything: the root's snapshot comes after its children
2554
+ // start and before its mailbox drains.
2555
+ for (const hold of holds.values()) {
2556
+ if (hold.answer !== undefined || this.holdRegs.has(hold)) continue;
2557
+ const reg = this.table.lookup(agentAddress(hold.instanceId));
2558
+ if (reg && reg.runId === record.runId && reg.invokeId === hold.invokeId) this.holdRegs.set(hold, reg);
1029
2559
  }
1030
2560
  // Forward the workflow author's `emit({...})` onto the feeds, from EVERY actor in the
1031
2561
  // tree as it is created: xstate scopes emitted events to the actor that emits them — no
1032
2562
  // bubbling — and a run's Emits mostly happen in child machines (a `workspace()` body IS
1033
2563
  // one). A root-only subscription silently dropped exactly the Emits ADR-0022/0023 exist
1034
2564
  // to surface. The subscription dies with its actor; restore re-creates both.
2565
+ // The children a pick's own send starts are its Agent's (ADR-0067): their done events name
2566
+ // the Agent an `answer` below no Agent's state shapes. Each place an `answer` is written runs
2567
+ // as its own seat: each Machine of the tree is placed as its actor is created, before its
2568
+ // first state is entered (`placeAnswers`).
2569
+ if (ev.type === "@xstate.actor") {
2570
+ const sending = this.sending;
2571
+ const child = ev.actorRef as AnyActorRef;
2572
+ const logic = (child as { logic?: unknown }).logic as AnyStateMachine | undefined;
2573
+ if (logic && typeof logic === "object" && "root" in logic) placeAnswers(logic);
2574
+ if (sending && child._parent !== undefined && child._parent === sending.reg.invoker) {
2575
+ const agent = sending.reg.sender?.id;
2576
+ const invoke = sending.reg.invokeId;
2577
+ const key = childKey(child._parent, child.id);
2578
+ if (agent !== undefined) startedBy[key] = invoke !== undefined ? { agent, invoke } : { agent };
2579
+ else delete startedBy[key];
2580
+ }
2581
+ }
2582
+ // …and so are the children its later work starts: a child started while the Machine takes an
2583
+ // event that is already that Agent's — a done event of a child it started, or its own event —
2584
+ // is that Agent's too, so the work its pick began stays its own (ADR-0067). A child started
2585
+ // on an event that names no Agent (a timer) below the state of an Agent whose pick is
2586
+ // outstanding is that pick's work by the Rest rule, so it is that Agent's (ADR-0024); any
2587
+ // other child started on such an event is no Agent's.
2588
+ if (ev.type === "@xstate.microstep") {
2589
+ const taker = ev.actorRef as AnyActorRef;
2590
+ const before = childrenAfter.get(taker) ?? childRefsOf(taker.getSnapshot());
2591
+ const snapshot = (ev as { snapshot?: unknown }).snapshot;
2592
+ const after = childRefsOf(snapshot);
2593
+ childrenAfter.set(taker, after);
2594
+ const started = [...after].filter((c) => !before.has(c)).map((c) => c.id);
2595
+ if (started.length > 0 && !this.sending) {
2596
+ const agent = agentOfEvent(ev.event as AnyEventObject, startedBy, taker);
2597
+ for (const id of started) {
2598
+ const owner = agent ?? this.outstandingAbove(taker, snapshot as AnyMachineSnapshot, id, holds);
2599
+ if (owner !== undefined) startedBy[childKey(taker, id)] = owner;
2600
+ else delete startedBy[childKey(taker, id)];
2601
+ }
2602
+ }
2603
+ }
1035
2604
  if (ev.type === "@xstate.actor") {
1036
2605
  const ref = ev.actorRef as AnyActorRef & { on?: (type: string, handler: (e: unknown) => void) => unknown };
1037
2606
  ref.on?.("*", (emitted) => {
@@ -1042,10 +2611,61 @@ export class RunHost {
1042
2611
  this.announce(record.workflow, { kind: "emit", runId: record.runId, event });
1043
2612
  });
1044
2613
  }
2614
+ // The chain (ADR-0024): every reply from below the Agent's state while one of its picks is
2615
+ // outstanding — inspected as it is relayed, before the Machine processes it, so the states
2616
+ // it came from are still active — and the last is the answer's default. The Machine's own
2617
+ // events, and events from outside the states below the Agent's, are skipped (`replyOf`).
2618
+ // A reply inside the pick's own send comes before any hold exists: `sending` records it.
2619
+ if (ev.type === "@xstate.event") {
2620
+ const replyTo = (reg: Registration | undefined): PickAnswer | undefined => {
2621
+ if (!reg?.invoker || !reg.invokeId || reg.invoker !== ev.actorRef) return undefined;
2622
+ const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
2623
+ return seat && replyOf(ev.event, ev.sourceRef as AnyActorRef | undefined, reg.invoker, seat);
2624
+ };
2625
+ // A child's error from below the Agent's state is no reply: the pick's answer carries none,
2626
+ // and the operator reads it on the run's feed (ADR-0024).
2627
+ const errorTo = (reg: Registration | undefined, pick: string): void => {
2628
+ if (!reg?.invoker || !reg.invokeId || reg.invoker !== ev.actorRef) return;
2629
+ const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
2630
+ const failed = seat && childErrorOf(ev.event, ev.sourceRef as AnyActorRef | undefined, reg.invoker, seat);
2631
+ const run = this.runs.get(record.runId);
2632
+ if (!failed || !run || run.binding !== binding) return;
2633
+ this.feed(run, { kind: "child-error", agent: reg.sender?.id ?? "", pick, ...failed });
2634
+ };
2635
+ // Every relayed event is asked, so the watch forgets each timer as it fires.
2636
+ const timer = firedTimer?.(ev.sourceRef, ev.event) === true;
2637
+ for (const hold of holds.values()) {
2638
+ if (hold.answer !== undefined) continue;
2639
+ const live = this.table.lookup(agentAddress(hold.instanceId));
2640
+ const reg = this.holdRegs.get(hold) ?? live;
2641
+ const reply = replyTo(reg);
2642
+ if (reply) hold.chain = [...(hold.chain ?? []), reply];
2643
+ errorTo(reg, hold.pick);
2644
+ // While the Agent's state is still the hold's, each event the Machine takes says who would
2645
+ // move it out (`cutShort`), when the Machine that invokes the Agent or one above it takes
2646
+ // it — events between the children below do not move it; once the state is gone, the next
2647
+ // state's events are not the hold's:
2648
+ // - a reply: its route;
2649
+ // - an event a Machine sent itself at once (`sendTo(self)`; a `raise` is never relayed):
2650
+ // part of the route of the event whose actions sent it — no change;
2651
+ // - anything else — a timer the Machine set, an event from outside the states below, a
2652
+ // party's event, the Agent's own end: it stops the work below, if any still runs.
2653
+ const taker = ev.actorRef as AnyActorRef;
2654
+ if (reg === undefined || reg !== live || !reg.invoker || !atOrBelow(reg.invoker, taker)) continue;
2655
+ if (reply) delete hold.cutShort;
2656
+ else if (!timer && ev.sourceRef === taker) continue;
2657
+ else if (stillRunning(reg)) hold.cutShort = true;
2658
+ else delete hold.cutShort;
2659
+ }
2660
+ const sending = this.sending;
2661
+ const reply = sending && replyTo(sending.reg);
2662
+ if (sending && reply) sending.chain.push(reply);
2663
+ if (sending) errorTo(sending.reg, sending.pick);
2664
+ }
1045
2665
  if (ev.type === "@xstate.snapshot") schedule();
1046
2666
  },
1047
2667
  });
1048
- live = this.track(record, actor, def, agents, epochs, binding);
2668
+ live = this.track(record, actor, def, agents, epochs, notices, binding, holds, effects, startedBy);
1049
2669
  return actor;
1050
2670
  }
1051
2671
 
@@ -1055,9 +2675,26 @@ export class RunHost {
1055
2675
  def: WorkflowDef,
1056
2676
  agents: Record<string, AgentAdmission>,
1057
2677
  epochs: Record<string, number>,
2678
+ notices: NoticeLedger,
1058
2679
  binding: RunBinding,
2680
+ holds: Map<string, Hold>,
2681
+ effects: Record<string, EffectRow>,
2682
+ startedBy: Record<string, StartedBy>,
1059
2683
  ): LiveRun {
1060
- const run: LiveRun = { record, actor, def, agents, epochs, binding, listeners: new Set(), feedSoFar: [] };
2684
+ const run: LiveRun = {
2685
+ record,
2686
+ actor,
2687
+ def,
2688
+ agents,
2689
+ epochs,
2690
+ notices,
2691
+ holds,
2692
+ effects,
2693
+ startedBy,
2694
+ binding,
2695
+ listeners: new Set(),
2696
+ feedSoFar: [],
2697
+ };
1061
2698
  this.runs.set(record.runId, run);
1062
2699
  // Ordinary persistence rides the inspection stream (see `spawn`); the subscription exists
1063
2700
  // for the ERROR channel: an errored actor (an invoke threw — e.g. ADR-0011's invoke-time
@@ -1067,9 +2704,13 @@ export class RunHost {
1067
2704
  error: (err) => {
1068
2705
  run.fault = err instanceof Error ? err.message : String(err);
1069
2706
  this.persist(run);
1070
- // Nothing to release: a faulted run stops its actors, and each workspace's lease is one
1071
- // of them (ADR-0021). The pod stays up for inspection (ADR-0012's destroy-less terminal)
1072
- // and ages out of the operator's idle timeout on its own.
2707
+ // A faulted run stops its actors — each workspace's lease among them (ADR-0021), and a
2708
+ // Workspace still `placing` deletes the Sandbox that never ran (ADR-0064). Said here, not
2709
+ // assumed: xstate ends a machine on an unhandled child error WITHOUT stopping its other
2710
+ // children, so a lease beside the actor that failed would renew for nobody, forever. The
2711
+ // pod that was placed stays up for inspection (ADR-0012's destroy-less terminal) and ages
2712
+ // out of the operator's idle timeout on its own.
2713
+ stopTree(actor);
1073
2714
  },
1074
2715
  });
1075
2716
  // (The author's `emit({...})` forwarding lives in `spawn`'s inspect handler — per actor,
@@ -1098,6 +2739,8 @@ export class RunHost {
1098
2739
  * status and the one the feeds report.
1099
2740
  */
1100
2741
  private persist(run: LiveRun, terminal?: string): void {
2742
+ // The holds first: an answer read at Rest, and a notice a route out raised, ride this same save.
2743
+ this.settleHolds(run);
1101
2744
  const snapshot = run.actor.getPersistedSnapshot();
1102
2745
  const machineStatus = (snapshot as { status?: string }).status ?? "active";
1103
2746
  const serialized = serializeSnapshot(snapshot, {
@@ -1110,13 +2753,31 @@ export class RunHost {
1110
2753
  // Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
1111
2754
  // providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
1112
2755
  machine: fingerprintOf(run.def.machine),
2756
+ // The Assembly the run started under (ADR-0069): the def it was spawned with, never the
2757
+ // current one, so a publish mid-run does not restamp it, and a restore spawns it with the
2758
+ // hash its blob named.
2759
+ ...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
1113
2760
  snapshot: serialized,
1114
2761
  agents: run.agents,
1115
2762
  epochs: run.epochs,
2763
+ notices: run.notices.state(),
2764
+ timers: pendingTimers(run.actor, this.now()),
2765
+ holds: [...run.holds.values()],
2766
+ effects: run.effects,
2767
+ startedBy: liveStartedBy(run.startedBy, run.actor),
1116
2768
  fault: run.fault,
1117
2769
  };
1118
2770
  const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
1119
- void this.store.save(run.record.runId, blob, status);
2771
+ // A save that fails is this run's fault, never the process's: an unhandled rejection would end
2772
+ // the Orchestrator and every run on it. The run keeps its last saved snapshot.
2773
+ this.store.save(run.record.runId, blob, status).catch((err: unknown) => {
2774
+ console.error(
2775
+ `run ${run.record.runId}: its snapshot was not saved: ${err instanceof Error ? err.message : String(err)}`,
2776
+ );
2777
+ });
2778
+ // The save is on disk (the store writes synchronously, ADR-0065): a settled row whose transition
2779
+ // it carries keeps no tail in the next one (ADR-0067).
2780
+ dropSavedTails(run.effects, snapshot);
1120
2781
 
1121
2782
  // Feed per-run observers (SSE/CLI watch). On the terminal transition emit the final status
1122
2783
  // BEFORE untrack drops the run from the registry, then drop the now-useless listener set.
@@ -1138,3 +2799,76 @@ export class RunHost {
1138
2799
  }
1139
2800
  }
1140
2801
  }
2802
+
2803
+ /** Does work still run below the Agent's state of `reg` — a live child of an active state under it?
2804
+ * A timer that holds Rest is no work: when the state exits, nothing it started is stopped. */
2805
+ function stillRunning(reg: Registration): boolean {
2806
+ if (!reg.invoker || !reg.invokeId) return false;
2807
+ const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
2808
+ return seat !== undefined && runningBelow(reg.invoker, seat).length > 0;
2809
+ }
2810
+
2811
+ /** `actor` is `ref`, or `ref` is its parent, its parent's parent, and so on up to the root. */
2812
+ function atOrBelow(actor: AnyActorRef, ref: AnyActorRef): boolean {
2813
+ for (let at: AnyActorRef | undefined = actor; at !== undefined; at = at._parent) if (at === ref) return true;
2814
+ return false;
2815
+ }
2816
+
2817
+ /** A status's bound fields (ADR-0070): `bound`, and the hashes of the Charts bound, distinct and
2818
+ * sorted. Both absent when the snapshot binds no Chart. */
2819
+ function boundFields(snap: unknown): { bound?: true; chartHashes?: string[] } {
2820
+ const hashes = [...new Set(boundChartRoutesOf(snap).map((route) => route.chartHash))].sort();
2821
+ return hashes.length ? { bound: true, chartHashes: hashes } : {};
2822
+ }
2823
+
2824
+ /** A live actor's children, by id. */
2825
+ function childrenOfRef(ref: AnyActorRef): Record<string, AnyActorRef> {
2826
+ return (ref.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
2827
+ }
2828
+
2829
+ /** A stored run's bound Chart, compiled again from its text under the Open body it binds, for the
2830
+ * run's doc — or why it is not: the doc then keeps the slot's subgraph and names the reason. */
2831
+ function compileStored(machine: AnyStateMachine, route: BoundChartRoute): AnyStateMachine | string {
2832
+ let at: AnyStateMachine = machine;
2833
+ for (const src of route.srcs.slice(0, -1)) {
2834
+ const logic = (at.implementations.actors as Record<string, unknown>)[src];
2835
+ if (isOpenBody(logic)) continue;
2836
+ if (!(logic as AnyStateMachine | undefined)?.root) return `no Machine at slot "${src}"`;
2837
+ at = logic as AnyStateMachine;
2838
+ }
2839
+ const logic = (at.implementations.actors as Record<string, unknown>)[route.src];
2840
+ if (!isOpenBody(logic)) return `slot "${route.src}" is no Open body`;
2841
+ const decl = openBodyDeclarationOf(logic);
2842
+ const lent = lentIn(at.implementations as never, decl);
2843
+ try {
2844
+ return compileChart(admitChart(route.chart, decl, lent), lent, decl, { id: route.src });
2845
+ } catch (err) {
2846
+ return err instanceof ChartRefusal
2847
+ ? `the door refuses the Chart: ${err.report.join("; ")}`
2848
+ : err instanceof Error
2849
+ ? err.message
2850
+ : String(err);
2851
+ }
2852
+ }
2853
+
2854
+ /** Put a compiled Chart's body under the states of `doc` that invoke the Open body at the end of
2855
+ * `srcs`, descending through the child Machine docs the earlier slots name. */
2856
+ function placeBody(doc: MachineBodyDoc, srcs: readonly string[], body: MachineBodyDoc): void {
2857
+ const [src, ...rest] = srcs;
2858
+ if (src === undefined) return;
2859
+ const visit = (state: MachineStateDoc): void => {
2860
+ if (rest.length === 0) {
2861
+ for (const inv of state.invoke) {
2862
+ if (inv.src !== src) continue;
2863
+ const child: ChildMachineDoc = { src, label: inv.id, via: "invoke", machine: body };
2864
+ const at = state.children.findIndex((c) => c.src === src);
2865
+ if (at >= 0) state.children[at] = child;
2866
+ else state.children.push(child);
2867
+ }
2868
+ } else {
2869
+ for (const child of state.children) if (child.src === src && child.machine) placeBody(child.machine, rest, body);
2870
+ }
2871
+ for (const nested of state.states) visit(nested);
2872
+ };
2873
+ visit(doc.root);
2874
+ }