@jr2/orchestrator 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +23 -0
  3. package/bin/server.ts +23 -0
  4. package/console/canvas.ts +843 -0
  5. package/console/components/app.ts +79 -0
  6. package/console/components/drawer.ts +131 -0
  7. package/console/components/fleet.ts +117 -0
  8. package/console/components/machine-pane.ts +85 -0
  9. package/console/components/nav.ts +81 -0
  10. package/console/components/schema-form.ts +137 -0
  11. package/console/main.ts +383 -0
  12. package/console/page.html +28 -0
  13. package/console/store.ts +336 -0
  14. package/console/style.css +700 -0
  15. package/console/tsconfig.json +18 -0
  16. package/package.json +61 -0
  17. package/src/actor.ts +562 -0
  18. package/src/agent.ts +124 -0
  19. package/src/ambient.ts +50 -0
  20. package/src/config.ts +297 -0
  21. package/src/customize.ts +348 -0
  22. package/src/durability.ts +135 -0
  23. package/src/fingerprint.ts +92 -0
  24. package/src/gate.ts +76 -0
  25. package/src/harness-client.ts +503 -0
  26. package/src/http.ts +753 -0
  27. package/src/images.ts +303 -0
  28. package/src/index.ts +40 -0
  29. package/src/instance.ts +294 -0
  30. package/src/machine-doc.ts +334 -0
  31. package/src/names.ts +78 -0
  32. package/src/open.ts +17 -0
  33. package/src/parts.ts +500 -0
  34. package/src/pool.ts +284 -0
  35. package/src/registration.ts +340 -0
  36. package/src/repo-fetch.ts +259 -0
  37. package/src/repo-identity.ts +145 -0
  38. package/src/repos.ts +330 -0
  39. package/src/run-host.ts +1095 -0
  40. package/src/sandbox-kubectl.ts +1136 -0
  41. package/src/server.ts +220 -0
  42. package/src/setup.ts +360 -0
  43. package/src/snapshot-store.ts +150 -0
  44. package/src/stub-harness.ts +217 -0
  45. package/src/tokens.ts +126 -0
  46. package/src/vocabulary.ts +99 -0
  47. package/src/wire.ts +103 -0
  48. package/src/workspace.ts +874 -0
  49. package/tsconfig.instance.json +26 -0
@@ -0,0 +1,1095 @@
1
+ // The Machine host: runs an xstate Machine as a durable run and wires the three slice-1 modules
2
+ // together (ADR-0002/0007/0011). It is the integration seam the modules left open.
3
+ //
4
+ // Wiring (ADR-0011 registration table — no routing layer):
5
+ // - The host owns ONE RegistrationTable and binds each run's actor system to it at track time;
6
+ // `gate` and the Agent slots register their invocation's event surface there, with deliver
7
+ // closures over their own `sendBack` — so delivery lands on the invoking state at any
8
+ // nesting depth and the host routes nothing.
9
+ // - Two thin surfaces sit on that table, and NEITHER is MCP (ADR-0013 — the Orchestrator does
10
+ // not speak it): `agentSurface` / `sendToAgent` serve the Agent's Adapter (`/agents/:iid/*`),
11
+ // `gates` / `sendToGate` serve humans, webhooks and CI (`/runs/:id/gates/*`). Lookup,
12
+ // validation, delivery and lifecycle stay implemented once, in the table.
13
+ // - The run's Agent children report their durable admissions through the run binding into
14
+ // the host LEDGER (`RunBlob.agents` — ADR-0016), persisted in the same save as the snapshot.
15
+ // (the Harness wire = lifecycle; the agent surface = domain events.)
16
+ //
17
+ // Durability (ADR-0007): a snapshot is persisted after every transition. Live infrastructure (the
18
+ // wire-client-backed Agent slot) is injected via `.provide()` at start AND restore, never
19
+ // persisted — so the snapshot is JSON-safe and restore re-attaches by rewriting the child's
20
+ // persisted input (drop `prompt`, set `attach` from the ledger) rather than re-POSTing the prompt.
21
+
22
+ import { randomUUID } from "node:crypto";
23
+ import { z } from "zod";
24
+ import { createActor, type AnyActor, type AnyActorLogic, type AnyActorRef, type AnyStateMachine } from "xstate";
25
+ import type { EventSemantics } from "@jr2/agent-protocol";
26
+ import { inputSchemaOf } from "./vocabulary.ts";
27
+ import type { EchoEvent, EchoStatusChild } from "./wire.ts";
28
+ import {
29
+ agentAddress,
30
+ bindRun,
31
+ EventValidationError,
32
+ gateAddress,
33
+ mayMove,
34
+ RegistrationTable,
35
+ UnknownAddressError,
36
+ wouldMove,
37
+ type RetryTelemetry,
38
+ type RunBinding,
39
+ type TurnMarker,
40
+ } from "./registration.ts";
41
+ import type { SandboxPort } from "./workspace.ts";
42
+ import { fingerprintOf } from "./fingerprint.ts";
43
+ import { serializeMachine, type MachineDoc } from "./machine-doc.ts";
44
+ import type { SnapshotStore } from "./snapshot-store.ts";
45
+ import type { AgentAdmission } from "./actor.ts";
46
+ import { reattachAgentRuns, serializeSnapshot } from "./durability.ts";
47
+
48
+ /** The actors filled into a run's named slots — `.provide()` is the unit-test seam (ADR-0015);
49
+ * production instances inject nothing. Built fresh at start and at restore. */
50
+ export type RunProviders = { actors?: Record<string, AnyActorLogic> };
51
+
52
+ /** A registered workflow: a template Machine plus how to fill its live slots for one run. */
53
+ export type WorkflowDef = {
54
+ name: string;
55
+ /** The template; slots (e.g. an Agent) are referenced by name and filled by `provide`. */
56
+ machine: AnyStateMachine;
57
+ /** Build this run's live providers. A test seam (ADR-0015): discovery injects nothing. */
58
+ provide: (ctx: { instanceId: string }) => RunProviders;
59
+ };
60
+
61
+ /** The serializable identity of a run — what reconcile sees and what restore rebuilds from. */
62
+ export type RunRecord = { runId: string; workflow: string; instanceId: string };
63
+
64
+ /**
65
+ * One live child MACHINE of a run, and its own children below it. A workflow's root machine is
66
+ * usually a coordinator — `coding` spawns a `featureWorkspace` per feature and the real work happens
67
+ * in there — so the root's `value` alone says almost nothing about where a run IS. This is the rest.
68
+ *
69
+ * Context-free BY CONSTRUCTION: there is no context field to forget to redact, which is what lets
70
+ * `observe()` pass the whole tree through to unauthenticated observers untouched. `src` is the join
71
+ * key ({@link ChildMachineDoc}); `id` is the spawn id (`"F-1"`), which is how two live instances of
72
+ * the same child machine are told apart.
73
+ */
74
+ export type RunChild = { id: string; src: string; status: string; value: unknown; children: RunChild[] };
75
+
76
+ /**
77
+ * The child-machine tree under a snapshot, in the TWO shapes it arrives in: a live actor's
78
+ * `children` are actorRefs (`status()`), a persisted snapshot's are `{src, snapshot}` records
79
+ * (`read()`, off the store). Both carry `src`, so one walk serves both.
80
+ *
81
+ * Machine actors only — a promise/callback/observable child has no state `value`, so there is
82
+ * nothing to light up and nothing to nest.
83
+ */
84
+ function runChildren(snapshot: unknown): RunChild[] {
85
+ const children = (snapshot as { children?: Record<string, unknown> } | undefined)?.children ?? {};
86
+ const out: RunChild[] = [];
87
+ for (const [id, entry] of Object.entries(children)) {
88
+ const child = entry as { src?: unknown; snapshot?: unknown; getSnapshot?: () => unknown };
89
+ const snap = (typeof child.getSnapshot === "function" ? child.getSnapshot() : child.snapshot) as
90
+ | { status?: string; value?: unknown }
91
+ | undefined;
92
+ if (snap?.value === undefined) continue;
93
+ out.push({
94
+ id,
95
+ src: typeof child.src === "string" ? child.src : "",
96
+ status: snap.status ?? "active",
97
+ value: snap.value,
98
+ children: runChildren(snap),
99
+ });
100
+ }
101
+ return out;
102
+ }
103
+
104
+ /** A run's current observable state. `fault` carries the error message when status is "error"
105
+ * (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). */
106
+ export type RunStatus = RunRecord & {
107
+ status: string;
108
+ value: unknown;
109
+ context: unknown;
110
+ /** The live child machines beneath the root — where most of a run actually is. */
111
+ children: RunChild[];
112
+ fault?: string;
113
+ /**
114
+ * Why the HOST set this status, for the statuses the Machine did not choose — today `drifted`
115
+ * (ADR-0030). Distinct from `fault`, which is a running actor's own error; this is the store's
116
+ * account of a run it declined to resume, and until it was surfaced here nothing on any HTTP
117
+ * route could read it.
118
+ */
119
+ reason?: string;
120
+ };
121
+
122
+ /**
123
+ * A run as an UNAUTHENTICATED observer may see it: which run, of what workflow, and where in the
124
+ * Machine it is. That is the whole set — enough to light up a state in the Console, and nothing
125
+ * more.
126
+ *
127
+ * What is absent is the point. `context` is the workflow's working data (branch names, ticket
128
+ * bodies, review verdicts, Harness endpoints) and `instanceId`/`fault` name live infrastructure and
129
+ * leak error text; all of it is STATE, which ADR-0013 guards behind the Instance token. `value` is
130
+ * a tree of state KEYS — it is structure, and structure is already public (`/workflows/:name/machine`
131
+ * serves the whole Machine). So an observer learns nothing here it could not read from the Machine
132
+ * doc, except which states are lit.
133
+ *
134
+ * `children` extends that to the child machines (which is where a run mostly lives), and it does NOT
135
+ * widen the line: a {@link RunChild} is keys and ids by construction, with no context field at any
136
+ * depth. The one thing it adds is the spawn ids, which are the workflow's own labels for its
137
+ * parallel work (`"F-1"`) — the same class of thing as a state key.
138
+ */
139
+ export type RunObservation = {
140
+ runId: string;
141
+ workflow: string;
142
+ status: string;
143
+ value: unknown;
144
+ children: RunChild[];
145
+ };
146
+
147
+ /** Project a full status down to what an observer may see. The one place the line is drawn. */
148
+ export function observe(status: RunStatus): RunObservation {
149
+ return {
150
+ runId: status.runId,
151
+ workflow: status.workflow,
152
+ status: status.status,
153
+ value: status.value,
154
+ children: status.children,
155
+ };
156
+ }
157
+
158
+ /** One open gate as external callers discover it (`GET /runs/:id` — ADR-0011): the accepted
159
+ * events with their input schemas as JSON Schema (what drives a form or a `jr2 send` prompt),
160
+ * plus the workflow-supplied `meta` (what a UI renders and a webhook translator matches on). */
161
+ export type GateView = {
162
+ gate: string;
163
+ /** The invoking state's actor path below the run root — where the gate lives in the Machine
164
+ * (what the Console's "parked here" pin resolves against its scope tree), stable whether the
165
+ * id was authored ("F-12") or derived. Never parse `gate` for this. */
166
+ path: string[];
167
+ accepts: Array<{ name: string; description?: string; input: unknown }>;
168
+ meta?: Record<string, unknown>;
169
+ };
170
+
171
+ /**
172
+ * One live agent surface, as its Adapter reads it (`GET /agents/:iid/surface` — ADR-0013). The
173
+ * Adapter renders this as `tools/list`: each accepted event becomes a tool, its `input` schema the
174
+ * tool's input schema. `semantics` rides along so the Adapter can tell an awaiting tool from a
175
+ * fire-and-forget one — the room ADR-0006's deferred results will land in, unbuilt today.
176
+ *
177
+ * `sandbox` is the Sandbox that may deliver here. It is not a secret from the Adapter (that pod IS
178
+ * the sandbox), and serving it lets the Adapter fail loudly on a surface that is not its own.
179
+ */
180
+ export type AgentSurfaceView = {
181
+ instanceId: string;
182
+ runId: string;
183
+ sandbox?: string;
184
+ accepts: Array<{ name: string; description?: string; input: unknown; semantics: EventSemantics }>;
185
+ };
186
+
187
+ /**
188
+ * The answer to one Agent delivery (`POST /agents/:iid/events`) — a receipt that DESCRIBES ITSELF
189
+ * (ADR-0024). The Adapter renders it as prose, because a bare `deliveryId` told the Agent nothing
190
+ * about whether it was finished, and the model answered that silence by calling again.
191
+ *
192
+ * `turnComplete` is the HINT, never the guarantee (the abort on invocation end is): it is read off
193
+ * the registration table right after delivery, so it says whether the state that asked for this
194
+ * turn has stopped waiting. It fails conservatively — anything that made the read unreliable reads
195
+ * `false`, which is today's behavior, never a false claim.
196
+ *
197
+ * `deliveryId` is unchanged from ADR-0013: an outcome stays ADDRESSABLE after the fact, the room a
198
+ * deferred result needs when it lands. Nothing polls it today, deliberately.
199
+ */
200
+ export type AgentDeliveryReceipt = {
201
+ delivered: true;
202
+ /** The event delivered — the Agent reads its own pick back, by name. */
203
+ event: string;
204
+ /**
205
+ * A transition accepted it. False means the pick was well-formed, arrived, and moved nothing —
206
+ * every transition for it was guarded false in the current state (ADR-0029). Distinct from
207
+ * {@link turnComplete}: a pick can move the Machine WITHIN the invoking state, which is
208
+ * `moved: true, turnComplete: false`. Before this existed the two were indistinguishable, so an
209
+ * Agent whose pick a guard rejected was told the workflow was "still in the state that asked",
210
+ * and its only move was to call again.
211
+ */
212
+ moved: boolean;
213
+ /** The invoking state stopped waiting: this Agent's turn is over. */
214
+ turnComplete: boolean;
215
+ deliveryId: string;
216
+ };
217
+
218
+ /**
219
+ * One item on a run's observation feed (the `GET /runs/:id/events` SSE stream — ADR-0009). A
220
+ * `status` snapshot delta (emitted on every transition, and replayed once on attach); an `emit` — a
221
+ * message the workflow author surfaced via xstate `emit({...})` for whoever is watching; a Turn's
222
+ * `admission`/`pick` markers (ADR-0023 — what the run-narrative echo projects for remotely-hosted
223
+ * Turns); absorbed-retry telemetry; and `closed`, the host going away underneath a feed that would
224
+ * never end. This feed is the Instance token's (http.ts); markers never reach ADR-0014's open band.
225
+ */
226
+ export type RunFeedEvent =
227
+ | { kind: "status"; status: RunStatus }
228
+ | { kind: "emit"; event: { type: string } & Record<string, unknown> }
229
+ | TurnMarker
230
+ // Absorbed-retry telemetry (ADR-0016): `{ child, attempt }` is state-key-class data — no iids
231
+ // ride the feed (ADR-0014). `reason` is mechanism text, guarded like `fault`.
232
+ | RetryTelemetry
233
+ | { kind: "closed" };
234
+
235
+ /**
236
+ * The feed-so-far cap. The buffer exists so a Workspace attaching mid-run can open its log with
237
+ * the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST frames, which
238
+ * is the same trade kubelet's log rotation already makes on the printed side — the feed remains
239
+ * the record, the buffer serves a courtesy view.
240
+ */
241
+ const FEED_SO_FAR_CAP = 1000;
242
+
243
+ /** Project one feed event for the echo at `target` (ADR-0023), or nothing: markers of Turns
244
+ * hosted AT the target are dropped (the transcript prints there — markers, not mirrors), a status
245
+ * sheds everything but where the run stands, and retries/`closed` are mechanism, not narrative. */
246
+ function echoEventOf(event: RunFeedEvent, target: string): EchoEvent | undefined {
247
+ if (event.kind === "status") {
248
+ const children = echoChildrenOf(event.status.children);
249
+ return {
250
+ kind: "status",
251
+ status: event.status.status,
252
+ value: event.status.value,
253
+ ...(children.length ? { children } : {}),
254
+ };
255
+ }
256
+ if (event.kind === "emit") return { kind: "emit", event: event.event };
257
+ if (event.kind === "admission" && event.endpoint !== target) {
258
+ return { kind: "admission", agent: event.agent, prompt: event.prompt };
259
+ }
260
+ if (event.kind === "pick" && event.endpoint !== target) {
261
+ const { agent, payload } = event;
262
+ return { kind: "pick", agent, event: event.event, ...(payload ? { payload } : {}) };
263
+ }
264
+ return undefined;
265
+ }
266
+
267
+ /** The child-machine tree for the echo's status: spawn ids and state values alone — a root's
268
+ * value says almost nothing about where a run is, and a {@link RunChild} is already context-free
269
+ * by construction. `src`/`status` stay behind: they are join keys for the Console, not story. */
270
+ function echoChildrenOf(children: RunChild[]): EchoStatusChild[] {
271
+ return children.map((child) => {
272
+ const nested = echoChildrenOf(child.children);
273
+ return { id: child.id, value: child.value, ...(nested.length ? { children: nested } : {}) };
274
+ });
275
+ }
276
+
277
+ /**
278
+ * One item on a WORKFLOW's observation feed (the `GET /workflows/:name/events` SSE — ADR-0022).
279
+ *
280
+ * The same vocabulary as {@link RunFeedEvent} at a coarser granularity, plus `gone`. Every run-
281
+ * scoped item carries `runId` even where the per-run route makes it redundant, so a consumer can
282
+ * parse both feeds with one reader.
283
+ *
284
+ * `gone` is its own fact rather than an inference from a terminal status, because a run can leave
285
+ * the live set WITHOUT one: `stop()` deliberately leaves the stored status "live" so a later
286
+ * `restore()` picks the run back up (see `spawn`). A watcher that inferred departure from
287
+ * `status === "done"` would show a stopped run as live forever.
288
+ *
289
+ * The opening `runs` snapshot is not an arm here: {@link RunHost.observeWorkflow} returns it, so it
290
+ * cannot be missed by a listener that attached a moment too late. The wire re-frames it as `runs`.
291
+ */
292
+ export type WorkflowFeedEvent =
293
+ | { kind: "status"; status: RunStatus }
294
+ | { kind: "gone"; runId: string }
295
+ | { kind: "emit"; runId: string; event: { type: string } & Record<string, unknown> }
296
+ | ({ runId: string } & RetryTelemetry)
297
+ | { kind: "closed" };
298
+
299
+ type WorkflowListener = (e: WorkflowFeedEvent) => void;
300
+
301
+ export type RunHostOptions = {
302
+ store: SnapshotStore;
303
+ /** Probe the live world before re-attaching on restore (ADR-0007). Default: always present. */
304
+ reconcile?: (run: RunRecord) => boolean | Promise<boolean>;
305
+ /** Injectable id generator (deterministic ids in tests). Default: `crypto.randomUUID`. */
306
+ newId?: () => string;
307
+ /** Report a run that threw during restore (ADR-0030). The row is left `live` for the next boot,
308
+ * so this is the only account of why — the entrypoint logs it. Default: silent. */
309
+ onRestoreError?: (runId: string, err: unknown) => void;
310
+ /** The Sandbox backend `workspace()` provisions through (ADR-0012). Absent = no cluster:
311
+ * workspace-less workflows run fine; a `workspace()` invocation faults its run pointedly. */
312
+ sandbox?: SandboxPort;
313
+ /** The Instance Harness base URL (ADR-0031) — where `workspace: "none"` Turns are admitted. */
314
+ instanceHarness?: string;
315
+ /**
316
+ * Build the run-narrative echo pusher for one Workspace's Harness (ADR-0023) — the seam a fake
317
+ * echo server rides in tests; `startInstance` binds the real wire push (harness-client.ts),
318
+ * closed over the Instance token. Absent = no echo: a host without it runs identically, because
319
+ * the echo is a courtesy view of the feed, never a dependency of the run.
320
+ */
321
+ echo?: (endpoint: string) => (events: EchoEvent[]) => Promise<void>;
322
+ };
323
+
324
+ /** What we persist per run: the machine snapshot wrapped with the run metadata restore needs.
325
+ * `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent actor
326
+ * through the run binding and saved in the same blob (same store, same atomicity). */
327
+ type RunBlob = {
328
+ workflow: string;
329
+ instanceId: string;
330
+ /** The shape of the Machine that WROTE this snapshot (ADR-0030). `workflow` says which def to
331
+ * look up; this says whether the def found there is still the one this snapshot can be read by.
332
+ * Absent means written before the stamp existed, which restore treats as drift — the point is to
333
+ * never interpret a snapshot whose Machine cannot be vouched for. */
334
+ machine?: string;
335
+ snapshot: unknown;
336
+ agents?: Record<string, AgentAdmission>;
337
+ fault?: string;
338
+ };
339
+
340
+ type LiveRun = {
341
+ record: RunRecord;
342
+ actor: AnyActor;
343
+ def: WorkflowDef;
344
+ /** What this run's callback actors resolve off the actor system — kept so `stop()` can tell
345
+ * them the HOST is the one ending the run (ADR-0024's one exception). */
346
+ binding: RunBinding;
347
+ /** The live admission ledger (ADR-0016): persisted as `RunBlob.agents`, seeded on restore. */
348
+ agents: Record<string, AgentAdmission>;
349
+ /** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
350
+ * message is captured here at the observer and persisted onto the blob for `read`). */
351
+ fault?: string;
352
+ /** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
353
+ listeners: Set<(e: RunFeedEvent) => void>;
354
+ /** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
355
+ * log's preamble. In-memory and this-boot only — a restored run's preamble starts at restore,
356
+ * the same live-only contract the printed log already has. Capped ({@link FEED_SO_FAR_CAP}). */
357
+ feedSoFar: RunFeedEvent[];
358
+ };
359
+
360
+ export class RunHost {
361
+ /** The internal registration table both delivery surfaces share (ADR-0011). Callback actors
362
+ * (`gate`, the Agent slots) reach it via the run binding, not this field. */
363
+ private readonly table = new RegistrationTable();
364
+
365
+ private readonly store: SnapshotStore;
366
+ private readonly reconcile: (run: RunRecord) => boolean | Promise<boolean>;
367
+ private readonly newId: () => string;
368
+ private readonly onRestoreError?: (runId: string, err: unknown) => void;
369
+ private readonly sandbox?: SandboxPort;
370
+ private readonly instanceHarness?: string;
371
+ private readonly echoFactory?: (endpoint: string) => (events: EchoEvent[]) => Promise<void>;
372
+ private readonly workflowDefs = new Map<string, WorkflowDef>();
373
+ private readonly runs = new Map<string, LiveRun>();
374
+ /**
375
+ * Workflow name → its observers. Deliberately on the HOST and not on `LiveRun`: a workflow's
376
+ * watcher outlives every individual run it watches, and `persist()` clears a settled run's own
377
+ * listener set. Keeping it here is what makes that structurally impossible to get wrong, rather
378
+ * than a comment asking the next reader not to clear the wrong Set.
379
+ */
380
+ private readonly workflowListeners = new Map<string, Set<WorkflowListener>>();
381
+
382
+ constructor(opts: RunHostOptions) {
383
+ this.store = opts.store;
384
+ this.reconcile = opts.reconcile ?? (() => true);
385
+ this.newId = opts.newId ?? (() => randomUUID());
386
+ this.onRestoreError = opts.onRestoreError;
387
+ this.sandbox = opts.sandbox;
388
+ this.instanceHarness = opts.instanceHarness;
389
+ this.echoFactory = opts.echo;
390
+ }
391
+
392
+ /** Register a workflow so `start`/`restore` can run it. Re-registering replaces (dev reload).
393
+ * Nothing about the vocabulary is copied here: event names are scoped to the Machine that
394
+ * declared them and resolved at invoke time off the invoking Machine (ADR-0011, ADR-0049), so
395
+ * a nested Machine's defs are never the root's — or this host's — to hold. */
396
+ register(def: WorkflowDef): void {
397
+ this.workflowDefs.set(def.name, def);
398
+ }
399
+
400
+ /** Drop a workflow's registration (a workflow the host no longer discovers). In-flight
401
+ * runs keep their already-assembled definition; only future `start`s are affected.
402
+ *
403
+ * Observers stay ATTACHED, deliberately. Their runs are still running, so a feed that ended here
404
+ * would be reporting that the work stopped when it did not — and the file may well come back on
405
+ * the next reload, which the still-open feed then picks up with no reconnect. */
406
+ unregister(name: string): void {
407
+ this.workflowDefs.delete(name);
408
+ }
409
+
410
+ /** The names of every registered workflow (the `GET /workflows` listing — ADR-0009). */
411
+ workflows(): string[] {
412
+ return [...this.workflowDefs.keys()];
413
+ }
414
+
415
+ /** The registered template Machine's serialized structure (`GET /workflows/:name/machine`). */
416
+ machine(name: string): MachineDoc | undefined {
417
+ const def = this.workflowDefs.get(name);
418
+ return def && serializeMachine(name, def.machine);
419
+ }
420
+
421
+ /**
422
+ * The workflow's declared run-input contract as JSON Schema (ADR-0033) — what the workflow
423
+ * detail serves and the Console's start form generates from. `null` for a machine that
424
+ * declares none (a run of it starts with anything), undefined for an unknown workflow. A
425
+ * schema is STRUCTURE, the same class of thing as the Machine doc — open band (ADR-0014).
426
+ */
427
+ inputSchema(name: string): Record<string, unknown> | null | undefined {
428
+ const def = this.workflowDefs.get(name);
429
+ if (!def) return undefined;
430
+ const schema = inputSchemaOf(def.machine);
431
+ // `io: "input"`: this schema describes what a caller SENDS — a defaulted field is optional
432
+ // at the door (the default is applied by `start`'s parse), not required of the form.
433
+ return schema ? (z.toJSONSchema(schema, { io: "input" }) as Record<string, unknown>) : null;
434
+ }
435
+
436
+ /** Start a fresh run of a registered workflow; returns its durable ids. */
437
+ async start(workflow: string, input: Record<string, unknown> = {}): Promise<{ runId: string; instanceId: string }> {
438
+ const def = this.workflowDefs.get(workflow);
439
+ if (!def) throw new Error(`no workflow registered as "${workflow}"`);
440
+
441
+ // A declared schema is enforced at the door (ADR-0033), and what starts the run is the
442
+ // PARSED shape (defaults applied, unknown keys stripped) — exactly what a gate delivery
443
+ // lands as, and the same error class: `EventValidationError`, which the wire maps to a 400
444
+ // naming what is accepted. No schema → accept anything (the permissive door the wire was).
445
+ const declared = inputSchemaOf(def.machine);
446
+ let runInput = input;
447
+ if (declared) {
448
+ const parsed = declared.safeParse(input);
449
+ if (!parsed.success) {
450
+ throw new EventValidationError(`invalid input for workflow "${workflow}": ${parsed.error.message}`);
451
+ }
452
+ runInput = parsed.data;
453
+ }
454
+
455
+ const runId = this.newId();
456
+ const instanceId = this.newId();
457
+ const record: RunRecord = { runId, workflow, instanceId };
458
+
459
+ const machine = this.assemble(def, instanceId);
460
+ // The root machine gets the parsed door PLUS what the host injects beside it —
461
+ // `HostInjectedInput` (vocabulary.ts): the seed Instance ID, added after the parse because no
462
+ // caller sends it and nothing serves it (ADR-0033). Only the root: a machine invoked below is
463
+ // fed by its parent, which is why the type is placement-dependent and `workspace()`'s body
464
+ // guard counts those keys as provided rather than claiming to know where the wrapper sits.
465
+ const actor = this.spawn(machine, { input: { ...runInput, instanceId } }, record, def);
466
+ actor.start();
467
+ return { runId, instanceId };
468
+ }
469
+
470
+ /**
471
+ * Restore every persisted live run: hydrate, reconcile against the live world, and either
472
+ * re-attach (re-spawn from the rewritten child input), mark lost, or refuse as drifted. Returns
473
+ * the run ids handled, by outcome — the caller announces it, because a boot that silently skipped
474
+ * work is the failure this whole path exists to avoid.
475
+ *
476
+ * Every run is handled INDEPENDENTLY (ADR-0030). The loop is sequential and one throw used to
477
+ * reject the whole method, which rejects `startInstance`, which crash-loops the pod — and every
478
+ * later run in the store never restored at all. One run that cannot be resumed is one run, not an
479
+ * outage.
480
+ *
481
+ * `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — the
482
+ * Machine changed, and it will still have changed on the next boot — so it is written to the row.
483
+ * A throw is not: `reconcile` talks to a cluster, and a kubectl blip must not permanently condemn
484
+ * a run. Those rows are left `live` to be retried on the next boot, and reported every time until
485
+ * they stop failing.
486
+ */
487
+ async restore(): Promise<{ reattached: string[]; lost: string[]; drifted: string[]; failed: string[] }> {
488
+ const reattached: string[] = [];
489
+ const lost: string[] = [];
490
+ const drifted: string[] = [];
491
+ const failed: string[] = [];
492
+
493
+ for (const stored of await this.store.list()) {
494
+ if (stored.status !== "live") continue;
495
+ const blob = stored.snapshot as RunBlob | null;
496
+ const def = blob ? this.workflowDefs.get(blob.workflow) : undefined;
497
+ if (!blob || !def) {
498
+ await this.store.markLost(stored.runId, blob ? `no workflow "${blob.workflow}"` : "empty snapshot");
499
+ lost.push(stored.runId);
500
+ continue;
501
+ }
502
+
503
+ // The workflow name resolved a def; this asks whether that def is still the Machine this
504
+ // snapshot was written by (ADR-0030). Refused BEFORE `reconcile`, which talks to the cluster:
505
+ // there is no point proving a Sandbox is alive for a run that cannot be read.
506
+ const expected = fingerprintOf(def.machine);
507
+ if (blob.machine !== expected) {
508
+ await this.store.markDrifted(
509
+ stored.runId,
510
+ `workflow "${blob.workflow}" changed shape since this run was saved ` +
511
+ `(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`,
512
+ );
513
+ drifted.push(stored.runId);
514
+ continue;
515
+ }
516
+
517
+ const record: RunRecord = { runId: stored.runId, workflow: blob.workflow, instanceId: blob.instanceId };
518
+
519
+ try {
520
+ if (!(await this.reconcile(record))) {
521
+ await this.store.markLost(stored.runId, "reconcile: live world absent");
522
+ lost.push(stored.runId);
523
+ continue;
524
+ }
525
+
526
+ // Re-attach every persisted Agent input in the TREE from the admission ledger
527
+ // (ADR-0016): iids are globally unique, so one flat map covers every nesting depth.
528
+ const agents = blob.agents ?? {};
529
+ const hydrated = reattachAgentRuns(blob.snapshot, agents);
530
+
531
+ const actor = this.spawn(
532
+ this.assemble(def, blob.instanceId),
533
+ { snapshot: hydrated as never },
534
+ record,
535
+ def,
536
+ agents,
537
+ );
538
+ actor.start();
539
+ reattached.push(stored.runId);
540
+ } catch (err) {
541
+ // Left `live` on purpose — see the note above. The row is unchanged, so the next boot tries
542
+ // again; what must not happen is this taking the remaining runs down with it.
543
+ this.onRestoreError?.(stored.runId, err);
544
+ failed.push(stored.runId);
545
+ }
546
+ }
547
+
548
+ return { reattached, lost, drifted, failed };
549
+ }
550
+
551
+ /**
552
+ * An agent instance's LIVE surface (`GET /agents/:iid/surface` — ADR-0013): what the state that
553
+ * invoked this Agent accepts, right now. Undefined once nothing is registered (the state exited,
554
+ * the run settled, the iid is unknown) — the one catch point, and the reason the Adapter never
555
+ * has to learn which turn is live: it asks, per turn, and the answer IS the turn.
556
+ *
557
+ * "Right now" is load-bearing (ADR-0029): the registered defs are the state's VOCABULARY, derived
558
+ * statically from its transitions, and the guards on those transitions are asked here — so an
559
+ * event the Machine cannot currently accept is not offered. The pick has not happened yet, so the
560
+ * question is `mayMove`, not `wouldMove`: a guard that would have judged the Agent's arguments is
561
+ * left on the menu and settled at delivery. The Adapter rebuilds this per MCP connection and the
562
+ * Harness re-lists per Submission, so the filter lands at turn boundaries and never moves under
563
+ * an Agent mid-turn.
564
+ */
565
+ agentSurface(instanceId: string): AgentSurfaceView | undefined {
566
+ const reg = this.table.lookup(agentAddress(instanceId));
567
+ if (!reg) return undefined;
568
+ return {
569
+ instanceId,
570
+ runId: reg.runId,
571
+ sandbox: reg.sandbox,
572
+ accepts: [...reg.defs.values()]
573
+ .filter((def) => mayMove(reg.invoker, def.name))
574
+ .map((def) => ({
575
+ name: def.name,
576
+ description: def.description,
577
+ input: z.toJSONSchema(def.input),
578
+ semantics: def.semantics,
579
+ })),
580
+ };
581
+ }
582
+
583
+ /**
584
+ * Deliver one event from an Agent's Adapter (`POST /agents/:iid/events` — ADR-0013). Validation
585
+ * and delivery are the table's; this only agent-scopes the address and mints the receipt.
586
+ */
587
+ sendToAgent(instanceId: string, event: { type?: unknown } & Record<string, unknown>): AgentDeliveryReceipt {
588
+ const { type, ...payload } = event;
589
+ if (typeof type !== "string" || !type) {
590
+ throw new EventValidationError(`event body must carry a string "type" (one of the surface's accepted names)`);
591
+ }
592
+ const address = agentAddress(instanceId);
593
+ const invoking = this.table.lookup(address);
594
+
595
+ // Ask the guard question BEFORE delivering, and ask it with the VALIDATED payload — this is the
596
+ // exact form of the check the surface build can only approximate payload-blind (ADR-0029).
597
+ // Parsed here rather than read back out of `deliver` because `deliver` is void by design (it is
598
+ // the one behavior behind both dialects); a zod default applied there but not here would leave
599
+ // a guard reading that field answering on `undefined`. An unaccepted name or a bad payload
600
+ // makes this unreliable and `deliver` throws on the next line anyway — so it fails open.
601
+ const def = invoking?.defs.get(type);
602
+ const parsed = def?.input.safeParse(payload ?? {});
603
+ const moved = wouldMove(invoking?.invoker, parsed?.success ? { type, ...parsed.data } : { type, ...payload });
604
+
605
+ this.table.deliver(address, type, payload);
606
+ // Read AFTER the delivery, off the SAME table the ADR-0024 guarantee uses — so the receipt
607
+ // reports what happened rather than what was hoped. `deliver` reached the invoking state's
608
+ // `sendBack` synchronously, so a pick that moved the Machine out of that state has already
609
+ // destroyed this registration by now.
610
+ //
611
+ // IDENTITY, not existence: under `session: "continue"` the next state re-registers the SAME
612
+ // address for its own turn, and that is a new turn — this one still ended.
613
+ return {
614
+ delivered: true,
615
+ event: type,
616
+ moved,
617
+ turnComplete: this.table.lookup(address) !== invoking,
618
+ deliveryId: this.newId(),
619
+ };
620
+ }
621
+
622
+ /** A run's open gates as callers discover them (`GET /runs/:id` — ADR-0011). Settled/unknown
623
+ * run → empty: a gate is a LIVE surface, it does not outlive its state. */
624
+ gates(runId: string): GateView[] {
625
+ return this.table
626
+ .byRun(runId)
627
+ .filter((reg) => reg.kind === "gate")
628
+ .map((reg) => ({
629
+ gate: reg.id,
630
+ path: reg.path ?? [],
631
+ accepts: [...reg.defs.values()].map((def) => ({
632
+ name: def.name,
633
+ description: def.description,
634
+ input: z.toJSONSchema(def.input),
635
+ })),
636
+ meta: reg.meta,
637
+ }));
638
+ }
639
+
640
+ /** Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
641
+ * Validation and delivery are the table's — this only run-scopes the address. */
642
+ sendToGate(runId: string, gate: string, event: { type?: unknown } & Record<string, unknown>): void {
643
+ const { type, ...payload } = event;
644
+ if (typeof type !== "string" || !type) {
645
+ throw new EventValidationError(`event body must carry a string "type" (one of the gate's accepted names)`);
646
+ }
647
+ try {
648
+ this.table.deliver(gateAddress(runId, gate), type, payload);
649
+ } catch (err) {
650
+ if (err instanceof UnknownAddressError) {
651
+ const open = this.gates(runId).map((g) => g.gate);
652
+ throw new UnknownAddressError(
653
+ `no open gate "${gate}" on run "${runId}"${open.length ? ` (open: ${open.join(", ")})` : ""}`,
654
+ );
655
+ }
656
+ throw err;
657
+ }
658
+ }
659
+
660
+ /**
661
+ * Observe a live run's feed (the `GET /runs/:id/events` SSE — ADR-0009): status deltas after every
662
+ * transition, plus the author's `emit`s. Returns an unsubscribe fn. The current status is **replayed
663
+ * immediately** on attach, so a freshly-attached watcher sees where the run is now (e.g. parked on an
664
+ * approval) rather than waiting for the next transition. The final status is emitted on the terminal
665
+ * transition right before the run is dropped from the registry. Attaching to an unknown/settled run
666
+ * is a no-op — read its terminal status via {@link read} instead.
667
+ */
668
+ subscribe(runId: string, listener: (e: RunFeedEvent) => void): () => void {
669
+ const run = this.runs.get(runId);
670
+ if (!run) return () => {};
671
+ run.listeners.add(listener);
672
+ listener({ kind: "status", status: this.liveStatus(run) });
673
+ return () => run.listeners.delete(listener);
674
+ }
675
+
676
+ /**
677
+ * Observe a whole WORKFLOW (the `GET /workflows/:name/events` SSE — ADR-0022): every run of it
678
+ * appearing, moving, emitting and leaving, for as long as the caller stays attached.
679
+ *
680
+ * The current set is RETURNED rather than replayed through the listener, and the subscription is
681
+ * registered in the same synchronous call. That is the whole point of the signature: split into
682
+ * a list-then-subscribe pair, a run starting between the two calls appears in neither, and the
683
+ * watcher is quietly wrong until something else happens to move it.
684
+ *
685
+ * Subscribing to a NAME, not to a registration — an unknown workflow attaches to an empty set
686
+ * (a later registration may supply it, and the feed should just start working).
687
+ */
688
+ observeWorkflow(workflow: string, listener: WorkflowListener): { runs: RunStatus[]; unsubscribe: () => void } {
689
+ let listeners = this.workflowListeners.get(workflow);
690
+ if (!listeners) this.workflowListeners.set(workflow, (listeners = new Set()));
691
+ listeners.add(listener);
692
+ return {
693
+ runs: this.list().filter((s) => s.workflow === workflow),
694
+ unsubscribe: () => {
695
+ listeners.delete(listener);
696
+ // Drop the empty Set, but ONLY if the map still holds this one. A stale unsubscribe (called
697
+ // twice, or after `close()`) would otherwise evict whatever Set replaced it under the same
698
+ // name — silently orphaning a watcher that has nothing to do with this one.
699
+ if (listeners.size === 0 && this.workflowListeners.get(workflow) === listeners) {
700
+ this.workflowListeners.delete(workflow);
701
+ }
702
+ },
703
+ };
704
+ }
705
+
706
+ /** How many observers a workflow's feed currently has. Exists so a test can prove that a client
707
+ * going away actually DETACHES — a leak here is invisible until the process runs out of memory. */
708
+ observerCount(workflow: string): number {
709
+ return this.workflowListeners.get(workflow)?.size ?? 0;
710
+ }
711
+
712
+ /** Feed one workflow's observers. The projection to the wire happens in http.ts, not here. */
713
+ private announce(workflow: string, event: WorkflowFeedEvent): void {
714
+ for (const listener of this.workflowListeners.get(workflow) ?? []) listener(event);
715
+ }
716
+
717
+ /** A run's LIVE status — undefined once it settles and is dropped from the registry. Sync. */
718
+ status(runId: string): RunStatus | undefined {
719
+ const run = this.runs.get(runId);
720
+ return run && this.liveStatus(run);
721
+ }
722
+
723
+ /**
724
+ * Read a run's status, **reading through to the store** when it is no longer live (ADR-0009). A
725
+ * completed run's final snapshot is persisted before `persist()` drops it from the registry, so a
726
+ * terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
727
+ * undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared).
728
+ */
729
+ async read(runId: string): Promise<RunStatus | undefined> {
730
+ const live = this.status(runId);
731
+ if (live) return live;
732
+ const stored = await this.store.load(runId);
733
+ const blob = stored?.snapshot as RunBlob | null | undefined;
734
+ if (!stored || !blob) return undefined;
735
+ const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown };
736
+ return {
737
+ runId,
738
+ workflow: blob.workflow,
739
+ instanceId: blob.instanceId,
740
+ // The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
741
+ // cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
742
+ // (ADR-0025). While the row still says "live", the Machine's own status is the answer.
743
+ status: stored.status === "live" ? (snap.status ?? stored.status) : stored.status,
744
+ value: snap.value,
745
+ context: snap.context,
746
+ children: runChildren(snap), // the persisted `children` map — same tree, off the store
747
+ fault: blob.fault,
748
+ reason: stored.reason,
749
+ };
750
+ }
751
+
752
+ list(): RunStatus[] {
753
+ return [...this.runs.keys()].map((id) => this.status(id)).filter((s): s is RunStatus => s !== undefined);
754
+ }
755
+
756
+ /**
757
+ * Run ids sharing a prefix — the live registry unioned with the store — backing abbreviated run
758
+ * ids in the CLI (ADR-0009). The live half is not belt-and-braces: `persist()` is scheduled on a
759
+ * microtask, so a just-started run is in `runs` before it is anywhere in the store. The store
760
+ * half is what makes a settled run abbreviate-able. A run past its first transition sits in both,
761
+ * hence the `Set`.
762
+ */
763
+ async candidates(prefix: string, limit: number): Promise<string[]> {
764
+ const live = [...this.runs.keys()].filter((id) => id.startsWith(prefix));
765
+ const stored = await this.store.findIdsByPrefix(prefix, limit);
766
+ return [...new Set([...live, ...stored])].sort().slice(0, limit);
767
+ }
768
+
769
+ /** The live runs of ONE workflow, projected for observers (the Console's run list). Scoped
770
+ * server-side: an observer asks about a workflow it can already name, and gets back only runs of
771
+ * it — never a listing of everything this orchestrator happens to be running. */
772
+ observations(workflow: string): RunObservation[] {
773
+ return this.list()
774
+ .filter((s) => s.workflow === workflow)
775
+ .map(observe);
776
+ }
777
+
778
+ /**
779
+ * End every open feed because the host is going away — the shutdown counterpart to `subscribe`.
780
+ *
781
+ * A feed has no natural end: a run parked on a gate transitions for hours, and its watchers hold
782
+ * an in-flight HTTP request the whole time. `server.close()` (instance.ts) waits for in-flight
783
+ * requests, so without this a single attached `jr2 run` wedges shutdown indefinitely. `closed` is
784
+ * the frame that lets those handlers exit. Runs themselves are untouched: this ends the
785
+ * OBSERVATION, not the work — the snapshots are already durable, and `restore()` picks them up.
786
+ */
787
+ async close(): Promise<void> {
788
+ for (const run of this.runs.values()) {
789
+ for (const listener of run.listeners) listener({ kind: "closed" });
790
+ run.listeners.clear();
791
+ }
792
+ for (const listeners of this.workflowListeners.values()) {
793
+ for (const listener of listeners) listener({ kind: "closed" });
794
+ }
795
+ this.workflowListeners.clear();
796
+ }
797
+
798
+ /**
799
+ * TEARDOWN: stop hosting a run in this process, leaving it RESTORABLE (ADR-0007/0025). The run
800
+ * keeps its stored status "live", so the next `restore()` picks it up where it left off — which
801
+ * is why this is ADR-0024's one exception: the Agents' submissions must stay alive for that
802
+ * re-attach, and the actor cannot infer "the host did this", so the binding is told before the
803
+ * stop.
804
+ *
805
+ * This is NOT the user's CANCEL — that is {@link cancel}, which ends the work. Nothing in a
806
+ * deployed Orchestrator calls this today (process shutdown stops no actors: `instance.ts` ends
807
+ * observation and nothing else); it is the teardown primitive, and the seam an orchestrator
808
+ * restart is simulated through.
809
+ */
810
+ async stop(runId: string): Promise<void> {
811
+ const run = this.runs.get(runId);
812
+ if (!run) return;
813
+ run.binding.hostStopping = true;
814
+ // Say so BEFORE the actor stops, while there is still a status to read. A stopped run is not a
815
+ // settled one — `persist()` never runs for it (the tracked-run guard drops the scheduled save,
816
+ // keeping the stored status "live" for restore — see `spawn`), so nothing else on this path
817
+ // would ever tell a watcher the run left. Without it the page shows it live forever.
818
+ this.announceGone(run);
819
+ run.actor.stop();
820
+ this.untrack(run.record);
821
+ }
822
+
823
+ /**
824
+ * CANCEL: the human's "abandon this run" (`jr2 send <run> --event CANCEL` — ADR-0025). It ends
825
+ * the work rather than parking it: stopping the actor with no `hostStopping` flag ends every
826
+ * live Agent invocation, and each one ends its Agent's turn remotely (ADR-0024). The run
827
+ * is then persisted TERMINAL, so `restore()` leaves it alone and `read()` reports how it ended.
828
+ *
829
+ * Ending the turns and refusing to restore are one decision, not two: a cancelled run that came
830
+ * back would re-attach to submissions that settled `aborted`, and `settle`'s rejection would
831
+ * fault a run whose Agents were stopped on purpose.
832
+ */
833
+ async cancel(runId: string): Promise<void> {
834
+ const run = this.runs.get(runId);
835
+ if (!run) return;
836
+ run.actor.stop();
837
+ // Persist AFTER the stop: the snapshot is final, and `persist` fans out the last status,
838
+ // announces `gone`, and untracks — the same terminal path a run that settled on its own takes.
839
+ this.persist(run, "cancelled");
840
+ }
841
+
842
+ /** Feed a run's final word to both granularities: where it ended, then that it is gone. */
843
+ private announceGone(run: LiveRun, status: RunStatus = this.liveStatus(run)): void {
844
+ this.announce(run.record.workflow, { kind: "status", status });
845
+ this.announce(run.record.workflow, { kind: "gone", runId: run.record.runId });
846
+ }
847
+
848
+ // --- internals ---
849
+
850
+ /** Put one event on the run's feed: buffer it for ADR-0023's backfill, then fan it out. The
851
+ * buffer and the fan-out share one seat so an attached echo and a later attach see the SAME
852
+ * feed — a projection cannot drift from a record it is read out of. */
853
+ private feed(run: LiveRun, event: RunFeedEvent): void {
854
+ run.feedSoFar.push(event);
855
+ if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
856
+ for (const listener of run.listeners) listener(event);
857
+ }
858
+
859
+ /**
860
+ * Attach the run-narrative echo (ADR-0023): replay the run's feed-so-far to the Workspace's
861
+ * Harness at `endpoint` — the backfilled preamble, why this Workspace exists — then tee live
862
+ * until detached. FIRE-AND-FORGET is this method's contract: pushes are chained so events
863
+ * arrive in feed order, and a failure is logged once and swallowed — the feed remains the
864
+ * record, the log is a courtesy view, and nothing here can fail a turn, a state, or a run.
865
+ * Scoped to the OWNING run's lineage by construction: it reads one run's buffer and listeners
866
+ * and nothing else — a sibling run's events cannot reach this endpoint through here.
867
+ */
868
+ private attachEcho(runId: string, endpoint: string): () => void {
869
+ const run = this.runs.get(runId);
870
+ const factory = this.echoFactory;
871
+ if (!run || !factory) return () => {};
872
+ const push = factory(endpoint);
873
+ let chain = Promise.resolve();
874
+ let reported = false;
875
+ const enqueue = (events: EchoEvent[]): void => {
876
+ if (events.length === 0) return;
877
+ chain = chain
878
+ .then(() => push(events))
879
+ .catch((err) => {
880
+ // Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
881
+ // transition into an error line, and must not surface anywhere a run could trip on.
882
+ if (reported) return;
883
+ reported = true;
884
+ console.error(
885
+ `run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
886
+ (err instanceof Error ? err.message : String(err)),
887
+ );
888
+ });
889
+ };
890
+ enqueue(run.feedSoFar.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
891
+ const listener = (ev: RunFeedEvent): void => {
892
+ const projected = echoEventOf(ev, endpoint);
893
+ if (projected) enqueue([projected]);
894
+ };
895
+ run.listeners.add(listener);
896
+ return () => run.listeners.delete(listener);
897
+ }
898
+
899
+ /** A live run's status, read off its actor — the one place the shape is built. */
900
+ private liveStatus(run: LiveRun): RunStatus {
901
+ const snap = run.actor.getSnapshot();
902
+ return {
903
+ ...run.record,
904
+ status: snap.status,
905
+ value: snap.value,
906
+ context: snap.context,
907
+ children: runChildren(snap),
908
+ fault: run.fault,
909
+ };
910
+ }
911
+
912
+ /** Fill the machine's named actor slots for one run (`.provide()`, the ADR-0015 test seam). */
913
+ private assemble(def: WorkflowDef, instanceId: string): AnyStateMachine {
914
+ const providers = def.provide({ instanceId });
915
+ return def.machine.provide(providers as Parameters<AnyStateMachine["provide"]>[0]);
916
+ }
917
+
918
+ /**
919
+ * Create + track a run's root actor. Persistence is driven by the actor system's INSPECTION
920
+ * stream, not the root subscription: a nested body's transition never notifies root
921
+ * subscribers, but it must hit the store, or a crash would restore a stale tree. Snapshot
922
+ * events within one macrostep are coalesced per microtask, and a persist scheduled around
923
+ * stop/untrack is dropped by the tracked-run guard (so `stop()` keeps the stored status
924
+ * "live" for restore).
925
+ */
926
+ private spawn(
927
+ machine: AnyStateMachine,
928
+ options: { input?: Record<string, unknown>; snapshot?: never },
929
+ record: RunRecord,
930
+ def: WorkflowDef,
931
+ agents: Record<string, AgentAdmission> = {},
932
+ ): AnyActor {
933
+ let live: LiveRun | undefined;
934
+ let scheduled = false;
935
+ const schedule = () => {
936
+ if (scheduled) return;
937
+ scheduled = true;
938
+ queueMicrotask(() => {
939
+ scheduled = false;
940
+ if (live && this.runs.get(record.runId) === live) this.persist(live);
941
+ });
942
+ };
943
+ const binding: RunBinding = {
944
+ runId: record.runId,
945
+ workflow: record.workflow,
946
+ table: this.table,
947
+ sandbox: this.sandbox,
948
+ instanceHarness: this.instanceHarness,
949
+ // The admission ledger's write half (ADR-0016): the Agent actor reports the durable handle the
950
+ // moment the Harness admits it, and the ledger hits the store in the same RunBlob save. An
951
+ // admission arriving around stop/untrack still lands in `agents` but skips the save,
952
+ // exactly like the persist scheduler's tracked-run guard.
953
+ recordAdmission: (instanceId, admission) => {
954
+ agents[instanceId] = admission;
955
+ const run = this.runs.get(record.runId);
956
+ if (run && run.agents === agents) this.persist(run);
957
+ },
958
+ // Absorbed-retry attempts go straight to the run's observers (SSE/CLI watch) — they are
959
+ // feed events, not machine events (ADR-0016: the workflow sees only the terminal fault).
960
+ telemetry: (event) => {
961
+ for (const listener of this.runs.get(record.runId)?.listeners ?? []) listener(event);
962
+ this.announce(record.workflow, { ...event, runId: record.runId });
963
+ },
964
+ // Turn markers (ADR-0023) ride the per-run feed alone — the prompt is Instance-token-class
965
+ // data, so they never touch the workflow feed (ADR-0014's open band strips even Emit
966
+ // payloads there).
967
+ marker: (event) => {
968
+ const run = this.runs.get(record.runId);
969
+ if (run && run.binding === binding) this.feed(run, event);
970
+ },
971
+ // The run-narrative echo attach (ADR-0023), called from `workspace()`'s registrar.
972
+ echo: (endpoint) => this.attachEcho(record.runId, endpoint),
973
+ };
974
+ let bound = false;
975
+ const actor = createActor(machine, {
976
+ ...options,
977
+ inspect: (ev) => {
978
+ // Bind the run's actor SYSTEM on the ROOT's creation event — the first inspection event,
979
+ // fired inside createActor BEFORE any child of the initial state is constructed. That
980
+ // ordering matters: jr2Setup's wrapped invoke inputs (iid minting — ADR-0016) run at child
981
+ // construction and must already see the run identity. The system is shared by every actor
982
+ // in the tree, which is what run-scopes gate ids with zero workflow plumbing (ADR-0011).
983
+ if (!bound && ev.type === "@xstate.actor") {
984
+ bound = true;
985
+ bindRun((ev.actorRef as AnyActorRef).system, binding);
986
+ }
987
+ // Forward the workflow author's `emit({...})` onto the feeds, from EVERY actor in the
988
+ // tree as it is created: xstate scopes emitted events to the actor that emits them — no
989
+ // bubbling — and a run's Emits mostly happen in child machines (a `workspace()` body IS
990
+ // one). A root-only subscription silently dropped exactly the Emits ADR-0022/0023 exist
991
+ // to surface. The subscription dies with its actor; restore re-creates both.
992
+ if (ev.type === "@xstate.actor") {
993
+ const ref = ev.actorRef as AnyActorRef & { on?: (type: string, handler: (e: unknown) => void) => unknown };
994
+ ref.on?.("*", (emitted) => {
995
+ const event = emitted as { type: string } & Record<string, unknown>;
996
+ const run = this.runs.get(record.runId);
997
+ if (!run || run.binding !== binding) return;
998
+ this.feed(run, { kind: "emit", event });
999
+ this.announce(record.workflow, { kind: "emit", runId: record.runId, event });
1000
+ });
1001
+ }
1002
+ if (ev.type === "@xstate.snapshot") schedule();
1003
+ },
1004
+ });
1005
+ live = this.track(record, actor, def, agents, binding);
1006
+ return actor;
1007
+ }
1008
+
1009
+ private track(
1010
+ record: RunRecord,
1011
+ actor: AnyActor,
1012
+ def: WorkflowDef,
1013
+ agents: Record<string, AgentAdmission>,
1014
+ binding: RunBinding,
1015
+ ): LiveRun {
1016
+ const run: LiveRun = { record, actor, def, agents, binding, listeners: new Set(), feedSoFar: [] };
1017
+ this.runs.set(record.runId, run);
1018
+ // Ordinary persistence rides the inspection stream (see `spawn`); the subscription exists
1019
+ // for the ERROR channel: an errored actor (an invoke threw — e.g. ADR-0011's invoke-time
1020
+ // manifest check) reports here. Capture the message (xstate serializes the Error itself to
1021
+ // `{}`), then persist: the snapshot's "error" status stores the run and untracks it.
1022
+ actor.subscribe({
1023
+ error: (err) => {
1024
+ run.fault = err instanceof Error ? err.message : String(err);
1025
+ this.persist(run);
1026
+ // Nothing to release: a faulted run stops its actors, and each workspace's lease is one
1027
+ // of them (ADR-0021). The pod stays up for inspection (ADR-0012's destroy-less terminal)
1028
+ // and ages out of the operator's idle timeout on its own.
1029
+ },
1030
+ });
1031
+ // (The author's `emit({...})` forwarding lives in `spawn`'s inspect handler — per actor,
1032
+ // because emitted events do not bubble — not here on the root alone.)
1033
+ // The run has appeared. This is the convergence point of `start()` and `restore()` — both reach
1034
+ // the live set through here, so one fan-out covers a fresh run and a resumed one alike.
1035
+ //
1036
+ // It runs BEFORE `actor.start()` (see the call in `spawn`), so this first frame is the pre-start
1037
+ // snapshot, corrected on the next microtask by the first `persist()`. That is fine because the
1038
+ // feed is level-triggered — every frame is a whole status, so a momentarily-early one is simply
1039
+ // overwritten rather than accumulated. Do NOT "fix" this by moving the fan-out into `start()`:
1040
+ // `restore()` does not go through it, and resumed runs would silently stop appearing.
1041
+ this.announce(record.workflow, { kind: "status", status: this.liveStatus(run) });
1042
+ return run;
1043
+ }
1044
+
1045
+ private untrack(record: RunRecord): void {
1046
+ this.runs.delete(record.runId);
1047
+ }
1048
+
1049
+ /**
1050
+ * Persist the run's snapshot after a transition; drop it from the registry once final.
1051
+ *
1052
+ * `terminal` is the run-lifecycle verdict the MACHINE cannot supply: `cancel()` passes
1053
+ * "cancelled", because xstate only knows its actor was stopped (ADR-0025). It is the stored
1054
+ * status and the one the feeds report.
1055
+ */
1056
+ private persist(run: LiveRun, terminal?: string): void {
1057
+ const snapshot = run.actor.getPersistedSnapshot();
1058
+ const machineStatus = (snapshot as { status?: string }).status ?? "active";
1059
+ const serialized = serializeSnapshot(snapshot, {
1060
+ stripContext: (ctx) => ctx, // see ADR-0007: live infra lives in `.provide` closures, not context
1061
+ stripChildInput: (input) => input,
1062
+ });
1063
+ const blob: RunBlob = {
1064
+ workflow: run.record.workflow,
1065
+ instanceId: run.record.instanceId,
1066
+ // Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
1067
+ // providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
1068
+ machine: fingerprintOf(run.def.machine),
1069
+ snapshot: serialized,
1070
+ agents: run.agents,
1071
+ fault: run.fault,
1072
+ };
1073
+ const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
1074
+ void this.store.save(run.record.runId, blob, status);
1075
+
1076
+ // Feed per-run observers (SSE/CLI watch). On the terminal transition emit the final status
1077
+ // BEFORE untrack drops the run from the registry, then drop the now-useless listener set.
1078
+ //
1079
+ // Persistence rides the INSPECTION stream (see `spawn`), which fires on a child's transitions
1080
+ // too — so this frame lands on child movement, and its `children` tree carries the new state.
1081
+ // That is the entire live half of the Console's child diagrams: no extra subscription.
1082
+ const runStatus = terminal ? { ...this.liveStatus(run), status: terminal } : this.liveStatus(run);
1083
+ this.feed(run, { kind: "status", status: runStatus });
1084
+
1085
+ if (status !== "live") {
1086
+ // The workflow's watchers get the same final status, then `gone` — the run's last word on
1087
+ // both feeds, still before untrack, while there is a status to read.
1088
+ this.announceGone(run, runStatus);
1089
+ this.untrack(run.record);
1090
+ run.listeners.clear();
1091
+ } else {
1092
+ this.announce(run.record.workflow, { kind: "status", status: runStatus });
1093
+ }
1094
+ }
1095
+ }