@jr2/orchestrator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +23 -0
- package/bin/server.ts +23 -0
- package/console/canvas.ts +843 -0
- package/console/components/app.ts +79 -0
- package/console/components/drawer.ts +131 -0
- package/console/components/fleet.ts +117 -0
- package/console/components/machine-pane.ts +85 -0
- package/console/components/nav.ts +81 -0
- package/console/components/schema-form.ts +137 -0
- package/console/main.ts +383 -0
- package/console/page.html +28 -0
- package/console/store.ts +336 -0
- package/console/style.css +700 -0
- package/console/tsconfig.json +18 -0
- package/package.json +61 -0
- package/src/actor.ts +562 -0
- package/src/agent.ts +124 -0
- package/src/ambient.ts +50 -0
- package/src/config.ts +297 -0
- package/src/customize.ts +348 -0
- package/src/durability.ts +135 -0
- package/src/fingerprint.ts +92 -0
- package/src/gate.ts +76 -0
- package/src/harness-client.ts +503 -0
- package/src/http.ts +753 -0
- package/src/images.ts +303 -0
- package/src/index.ts +40 -0
- package/src/instance.ts +294 -0
- package/src/machine-doc.ts +334 -0
- package/src/names.ts +78 -0
- package/src/open.ts +17 -0
- package/src/parts.ts +500 -0
- package/src/pool.ts +284 -0
- package/src/registration.ts +340 -0
- package/src/repo-fetch.ts +259 -0
- package/src/repo-identity.ts +145 -0
- package/src/repos.ts +330 -0
- package/src/run-host.ts +1095 -0
- package/src/sandbox-kubectl.ts +1136 -0
- package/src/server.ts +220 -0
- package/src/setup.ts +360 -0
- package/src/snapshot-store.ts +150 -0
- package/src/stub-harness.ts +217 -0
- package/src/tokens.ts +126 -0
- package/src/vocabulary.ts +99 -0
- package/src/wire.ts +103 -0
- package/src/workspace.ts +874 -0
- package/tsconfig.instance.json +26 -0
package/src/run-host.ts
ADDED
|
@@ -0,0 +1,1095 @@
|
|
|
1
|
+
// The Machine host: runs an xstate Machine as a durable run and wires the three slice-1 modules
|
|
2
|
+
// together (ADR-0002/0007/0011). It is the integration seam the modules left open.
|
|
3
|
+
//
|
|
4
|
+
// Wiring (ADR-0011 registration table — no routing layer):
|
|
5
|
+
// - The host owns ONE RegistrationTable and binds each run's actor system to it at track time;
|
|
6
|
+
// `gate` and the Agent slots register their invocation's event surface there, with deliver
|
|
7
|
+
// closures over their own `sendBack` — so delivery lands on the invoking state at any
|
|
8
|
+
// nesting depth and the host routes nothing.
|
|
9
|
+
// - Two thin surfaces sit on that table, and NEITHER is MCP (ADR-0013 — the Orchestrator does
|
|
10
|
+
// not speak it): `agentSurface` / `sendToAgent` serve the Agent's Adapter (`/agents/:iid/*`),
|
|
11
|
+
// `gates` / `sendToGate` serve humans, webhooks and CI (`/runs/:id/gates/*`). Lookup,
|
|
12
|
+
// validation, delivery and lifecycle stay implemented once, in the table.
|
|
13
|
+
// - The run's Agent children report their durable admissions through the run binding into
|
|
14
|
+
// the host LEDGER (`RunBlob.agents` — ADR-0016), persisted in the same save as the snapshot.
|
|
15
|
+
// (the Harness wire = lifecycle; the agent surface = domain events.)
|
|
16
|
+
//
|
|
17
|
+
// Durability (ADR-0007): a snapshot is persisted after every transition. Live infrastructure (the
|
|
18
|
+
// wire-client-backed Agent slot) is injected via `.provide()` at start AND restore, never
|
|
19
|
+
// persisted — so the snapshot is JSON-safe and restore re-attaches by rewriting the child's
|
|
20
|
+
// persisted input (drop `prompt`, set `attach` from the ledger) rather than re-POSTing the prompt.
|
|
21
|
+
|
|
22
|
+
import { randomUUID } from "node:crypto";
|
|
23
|
+
import { z } from "zod";
|
|
24
|
+
import { createActor, type AnyActor, type AnyActorLogic, type AnyActorRef, type AnyStateMachine } from "xstate";
|
|
25
|
+
import type { EventSemantics } from "@jr2/agent-protocol";
|
|
26
|
+
import { inputSchemaOf } from "./vocabulary.ts";
|
|
27
|
+
import type { EchoEvent, EchoStatusChild } from "./wire.ts";
|
|
28
|
+
import {
|
|
29
|
+
agentAddress,
|
|
30
|
+
bindRun,
|
|
31
|
+
EventValidationError,
|
|
32
|
+
gateAddress,
|
|
33
|
+
mayMove,
|
|
34
|
+
RegistrationTable,
|
|
35
|
+
UnknownAddressError,
|
|
36
|
+
wouldMove,
|
|
37
|
+
type RetryTelemetry,
|
|
38
|
+
type RunBinding,
|
|
39
|
+
type TurnMarker,
|
|
40
|
+
} from "./registration.ts";
|
|
41
|
+
import type { SandboxPort } from "./workspace.ts";
|
|
42
|
+
import { fingerprintOf } from "./fingerprint.ts";
|
|
43
|
+
import { serializeMachine, type MachineDoc } from "./machine-doc.ts";
|
|
44
|
+
import type { SnapshotStore } from "./snapshot-store.ts";
|
|
45
|
+
import type { AgentAdmission } from "./actor.ts";
|
|
46
|
+
import { reattachAgentRuns, serializeSnapshot } from "./durability.ts";
|
|
47
|
+
|
|
48
|
+
/** The actors filled into a run's named slots — `.provide()` is the unit-test seam (ADR-0015);
|
|
49
|
+
* production instances inject nothing. Built fresh at start and at restore. */
|
|
50
|
+
export type RunProviders = { actors?: Record<string, AnyActorLogic> };
|
|
51
|
+
|
|
52
|
+
/** A registered workflow: a template Machine plus how to fill its live slots for one run. */
|
|
53
|
+
export type WorkflowDef = {
|
|
54
|
+
name: string;
|
|
55
|
+
/** The template; slots (e.g. an Agent) are referenced by name and filled by `provide`. */
|
|
56
|
+
machine: AnyStateMachine;
|
|
57
|
+
/** Build this run's live providers. A test seam (ADR-0015): discovery injects nothing. */
|
|
58
|
+
provide: (ctx: { instanceId: string }) => RunProviders;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
/** The serializable identity of a run — what reconcile sees and what restore rebuilds from. */
|
|
62
|
+
export type RunRecord = { runId: string; workflow: string; instanceId: string };
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* One live child MACHINE of a run, and its own children below it. A workflow's root machine is
|
|
66
|
+
* usually a coordinator — `coding` spawns a `featureWorkspace` per feature and the real work happens
|
|
67
|
+
* in there — so the root's `value` alone says almost nothing about where a run IS. This is the rest.
|
|
68
|
+
*
|
|
69
|
+
* Context-free BY CONSTRUCTION: there is no context field to forget to redact, which is what lets
|
|
70
|
+
* `observe()` pass the whole tree through to unauthenticated observers untouched. `src` is the join
|
|
71
|
+
* key ({@link ChildMachineDoc}); `id` is the spawn id (`"F-1"`), which is how two live instances of
|
|
72
|
+
* the same child machine are told apart.
|
|
73
|
+
*/
|
|
74
|
+
export type RunChild = { id: string; src: string; status: string; value: unknown; children: RunChild[] };
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The child-machine tree under a snapshot, in the TWO shapes it arrives in: a live actor's
|
|
78
|
+
* `children` are actorRefs (`status()`), a persisted snapshot's are `{src, snapshot}` records
|
|
79
|
+
* (`read()`, off the store). Both carry `src`, so one walk serves both.
|
|
80
|
+
*
|
|
81
|
+
* Machine actors only — a promise/callback/observable child has no state `value`, so there is
|
|
82
|
+
* nothing to light up and nothing to nest.
|
|
83
|
+
*/
|
|
84
|
+
function runChildren(snapshot: unknown): RunChild[] {
|
|
85
|
+
const children = (snapshot as { children?: Record<string, unknown> } | undefined)?.children ?? {};
|
|
86
|
+
const out: RunChild[] = [];
|
|
87
|
+
for (const [id, entry] of Object.entries(children)) {
|
|
88
|
+
const child = entry as { src?: unknown; snapshot?: unknown; getSnapshot?: () => unknown };
|
|
89
|
+
const snap = (typeof child.getSnapshot === "function" ? child.getSnapshot() : child.snapshot) as
|
|
90
|
+
| { status?: string; value?: unknown }
|
|
91
|
+
| undefined;
|
|
92
|
+
if (snap?.value === undefined) continue;
|
|
93
|
+
out.push({
|
|
94
|
+
id,
|
|
95
|
+
src: typeof child.src === "string" ? child.src : "",
|
|
96
|
+
status: snap.status ?? "active",
|
|
97
|
+
value: snap.value,
|
|
98
|
+
children: runChildren(snap),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** A run's current observable state. `fault` carries the error message when status is "error"
|
|
105
|
+
* (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). */
|
|
106
|
+
export type RunStatus = RunRecord & {
|
|
107
|
+
status: string;
|
|
108
|
+
value: unknown;
|
|
109
|
+
context: unknown;
|
|
110
|
+
/** The live child machines beneath the root — where most of a run actually is. */
|
|
111
|
+
children: RunChild[];
|
|
112
|
+
fault?: string;
|
|
113
|
+
/**
|
|
114
|
+
* Why the HOST set this status, for the statuses the Machine did not choose — today `drifted`
|
|
115
|
+
* (ADR-0030). Distinct from `fault`, which is a running actor's own error; this is the store's
|
|
116
|
+
* account of a run it declined to resume, and until it was surfaced here nothing on any HTTP
|
|
117
|
+
* route could read it.
|
|
118
|
+
*/
|
|
119
|
+
reason?: string;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* A run as an UNAUTHENTICATED observer may see it: which run, of what workflow, and where in the
|
|
124
|
+
* Machine it is. That is the whole set — enough to light up a state in the Console, and nothing
|
|
125
|
+
* more.
|
|
126
|
+
*
|
|
127
|
+
* What is absent is the point. `context` is the workflow's working data (branch names, ticket
|
|
128
|
+
* bodies, review verdicts, Harness endpoints) and `instanceId`/`fault` name live infrastructure and
|
|
129
|
+
* leak error text; all of it is STATE, which ADR-0013 guards behind the Instance token. `value` is
|
|
130
|
+
* a tree of state KEYS — it is structure, and structure is already public (`/workflows/:name/machine`
|
|
131
|
+
* serves the whole Machine). So an observer learns nothing here it could not read from the Machine
|
|
132
|
+
* doc, except which states are lit.
|
|
133
|
+
*
|
|
134
|
+
* `children` extends that to the child machines (which is where a run mostly lives), and it does NOT
|
|
135
|
+
* widen the line: a {@link RunChild} is keys and ids by construction, with no context field at any
|
|
136
|
+
* depth. The one thing it adds is the spawn ids, which are the workflow's own labels for its
|
|
137
|
+
* parallel work (`"F-1"`) — the same class of thing as a state key.
|
|
138
|
+
*/
|
|
139
|
+
export type RunObservation = {
|
|
140
|
+
runId: string;
|
|
141
|
+
workflow: string;
|
|
142
|
+
status: string;
|
|
143
|
+
value: unknown;
|
|
144
|
+
children: RunChild[];
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/** Project a full status down to what an observer may see. The one place the line is drawn. */
|
|
148
|
+
export function observe(status: RunStatus): RunObservation {
|
|
149
|
+
return {
|
|
150
|
+
runId: status.runId,
|
|
151
|
+
workflow: status.workflow,
|
|
152
|
+
status: status.status,
|
|
153
|
+
value: status.value,
|
|
154
|
+
children: status.children,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** One open gate as external callers discover it (`GET /runs/:id` — ADR-0011): the accepted
|
|
159
|
+
* events with their input schemas as JSON Schema (what drives a form or a `jr2 send` prompt),
|
|
160
|
+
* plus the workflow-supplied `meta` (what a UI renders and a webhook translator matches on). */
|
|
161
|
+
export type GateView = {
|
|
162
|
+
gate: string;
|
|
163
|
+
/** The invoking state's actor path below the run root — where the gate lives in the Machine
|
|
164
|
+
* (what the Console's "parked here" pin resolves against its scope tree), stable whether the
|
|
165
|
+
* id was authored ("F-12") or derived. Never parse `gate` for this. */
|
|
166
|
+
path: string[];
|
|
167
|
+
accepts: Array<{ name: string; description?: string; input: unknown }>;
|
|
168
|
+
meta?: Record<string, unknown>;
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* One live agent surface, as its Adapter reads it (`GET /agents/:iid/surface` — ADR-0013). The
|
|
173
|
+
* Adapter renders this as `tools/list`: each accepted event becomes a tool, its `input` schema the
|
|
174
|
+
* tool's input schema. `semantics` rides along so the Adapter can tell an awaiting tool from a
|
|
175
|
+
* fire-and-forget one — the room ADR-0006's deferred results will land in, unbuilt today.
|
|
176
|
+
*
|
|
177
|
+
* `sandbox` is the Sandbox that may deliver here. It is not a secret from the Adapter (that pod IS
|
|
178
|
+
* the sandbox), and serving it lets the Adapter fail loudly on a surface that is not its own.
|
|
179
|
+
*/
|
|
180
|
+
export type AgentSurfaceView = {
|
|
181
|
+
instanceId: string;
|
|
182
|
+
runId: string;
|
|
183
|
+
sandbox?: string;
|
|
184
|
+
accepts: Array<{ name: string; description?: string; input: unknown; semantics: EventSemantics }>;
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The answer to one Agent delivery (`POST /agents/:iid/events`) — a receipt that DESCRIBES ITSELF
|
|
189
|
+
* (ADR-0024). The Adapter renders it as prose, because a bare `deliveryId` told the Agent nothing
|
|
190
|
+
* about whether it was finished, and the model answered that silence by calling again.
|
|
191
|
+
*
|
|
192
|
+
* `turnComplete` is the HINT, never the guarantee (the abort on invocation end is): it is read off
|
|
193
|
+
* the registration table right after delivery, so it says whether the state that asked for this
|
|
194
|
+
* turn has stopped waiting. It fails conservatively — anything that made the read unreliable reads
|
|
195
|
+
* `false`, which is today's behavior, never a false claim.
|
|
196
|
+
*
|
|
197
|
+
* `deliveryId` is unchanged from ADR-0013: an outcome stays ADDRESSABLE after the fact, the room a
|
|
198
|
+
* deferred result needs when it lands. Nothing polls it today, deliberately.
|
|
199
|
+
*/
|
|
200
|
+
export type AgentDeliveryReceipt = {
|
|
201
|
+
delivered: true;
|
|
202
|
+
/** The event delivered — the Agent reads its own pick back, by name. */
|
|
203
|
+
event: string;
|
|
204
|
+
/**
|
|
205
|
+
* A transition accepted it. False means the pick was well-formed, arrived, and moved nothing —
|
|
206
|
+
* every transition for it was guarded false in the current state (ADR-0029). Distinct from
|
|
207
|
+
* {@link turnComplete}: a pick can move the Machine WITHIN the invoking state, which is
|
|
208
|
+
* `moved: true, turnComplete: false`. Before this existed the two were indistinguishable, so an
|
|
209
|
+
* Agent whose pick a guard rejected was told the workflow was "still in the state that asked",
|
|
210
|
+
* and its only move was to call again.
|
|
211
|
+
*/
|
|
212
|
+
moved: boolean;
|
|
213
|
+
/** The invoking state stopped waiting: this Agent's turn is over. */
|
|
214
|
+
turnComplete: boolean;
|
|
215
|
+
deliveryId: string;
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* One item on a run's observation feed (the `GET /runs/:id/events` SSE stream — ADR-0009). A
|
|
220
|
+
* `status` snapshot delta (emitted on every transition, and replayed once on attach); an `emit` — a
|
|
221
|
+
* message the workflow author surfaced via xstate `emit({...})` for whoever is watching; a Turn's
|
|
222
|
+
* `admission`/`pick` markers (ADR-0023 — what the run-narrative echo projects for remotely-hosted
|
|
223
|
+
* Turns); absorbed-retry telemetry; and `closed`, the host going away underneath a feed that would
|
|
224
|
+
* never end. This feed is the Instance token's (http.ts); markers never reach ADR-0014's open band.
|
|
225
|
+
*/
|
|
226
|
+
export type RunFeedEvent =
|
|
227
|
+
| { kind: "status"; status: RunStatus }
|
|
228
|
+
| { kind: "emit"; event: { type: string } & Record<string, unknown> }
|
|
229
|
+
| TurnMarker
|
|
230
|
+
// Absorbed-retry telemetry (ADR-0016): `{ child, attempt }` is state-key-class data — no iids
|
|
231
|
+
// ride the feed (ADR-0014). `reason` is mechanism text, guarded like `fault`.
|
|
232
|
+
| RetryTelemetry
|
|
233
|
+
| { kind: "closed" };
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* The feed-so-far cap. The buffer exists so a Workspace attaching mid-run can open its log with
|
|
237
|
+
* the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST frames, which
|
|
238
|
+
* is the same trade kubelet's log rotation already makes on the printed side — the feed remains
|
|
239
|
+
* the record, the buffer serves a courtesy view.
|
|
240
|
+
*/
|
|
241
|
+
const FEED_SO_FAR_CAP = 1000;
|
|
242
|
+
|
|
243
|
+
/** Project one feed event for the echo at `target` (ADR-0023), or nothing: markers of Turns
|
|
244
|
+
* hosted AT the target are dropped (the transcript prints there — markers, not mirrors), a status
|
|
245
|
+
* sheds everything but where the run stands, and retries/`closed` are mechanism, not narrative. */
|
|
246
|
+
function echoEventOf(event: RunFeedEvent, target: string): EchoEvent | undefined {
|
|
247
|
+
if (event.kind === "status") {
|
|
248
|
+
const children = echoChildrenOf(event.status.children);
|
|
249
|
+
return {
|
|
250
|
+
kind: "status",
|
|
251
|
+
status: event.status.status,
|
|
252
|
+
value: event.status.value,
|
|
253
|
+
...(children.length ? { children } : {}),
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
if (event.kind === "emit") return { kind: "emit", event: event.event };
|
|
257
|
+
if (event.kind === "admission" && event.endpoint !== target) {
|
|
258
|
+
return { kind: "admission", agent: event.agent, prompt: event.prompt };
|
|
259
|
+
}
|
|
260
|
+
if (event.kind === "pick" && event.endpoint !== target) {
|
|
261
|
+
const { agent, payload } = event;
|
|
262
|
+
return { kind: "pick", agent, event: event.event, ...(payload ? { payload } : {}) };
|
|
263
|
+
}
|
|
264
|
+
return undefined;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** The child-machine tree for the echo's status: spawn ids and state values alone — a root's
|
|
268
|
+
* value says almost nothing about where a run is, and a {@link RunChild} is already context-free
|
|
269
|
+
* by construction. `src`/`status` stay behind: they are join keys for the Console, not story. */
|
|
270
|
+
function echoChildrenOf(children: RunChild[]): EchoStatusChild[] {
|
|
271
|
+
return children.map((child) => {
|
|
272
|
+
const nested = echoChildrenOf(child.children);
|
|
273
|
+
return { id: child.id, value: child.value, ...(nested.length ? { children: nested } : {}) };
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* One item on a WORKFLOW's observation feed (the `GET /workflows/:name/events` SSE — ADR-0022).
|
|
279
|
+
*
|
|
280
|
+
* The same vocabulary as {@link RunFeedEvent} at a coarser granularity, plus `gone`. Every run-
|
|
281
|
+
* scoped item carries `runId` even where the per-run route makes it redundant, so a consumer can
|
|
282
|
+
* parse both feeds with one reader.
|
|
283
|
+
*
|
|
284
|
+
* `gone` is its own fact rather than an inference from a terminal status, because a run can leave
|
|
285
|
+
* the live set WITHOUT one: `stop()` deliberately leaves the stored status "live" so a later
|
|
286
|
+
* `restore()` picks the run back up (see `spawn`). A watcher that inferred departure from
|
|
287
|
+
* `status === "done"` would show a stopped run as live forever.
|
|
288
|
+
*
|
|
289
|
+
* The opening `runs` snapshot is not an arm here: {@link RunHost.observeWorkflow} returns it, so it
|
|
290
|
+
* cannot be missed by a listener that attached a moment too late. The wire re-frames it as `runs`.
|
|
291
|
+
*/
|
|
292
|
+
export type WorkflowFeedEvent =
|
|
293
|
+
| { kind: "status"; status: RunStatus }
|
|
294
|
+
| { kind: "gone"; runId: string }
|
|
295
|
+
| { kind: "emit"; runId: string; event: { type: string } & Record<string, unknown> }
|
|
296
|
+
| ({ runId: string } & RetryTelemetry)
|
|
297
|
+
| { kind: "closed" };
|
|
298
|
+
|
|
299
|
+
type WorkflowListener = (e: WorkflowFeedEvent) => void;
|
|
300
|
+
|
|
301
|
+
export type RunHostOptions = {
|
|
302
|
+
store: SnapshotStore;
|
|
303
|
+
/** Probe the live world before re-attaching on restore (ADR-0007). Default: always present. */
|
|
304
|
+
reconcile?: (run: RunRecord) => boolean | Promise<boolean>;
|
|
305
|
+
/** Injectable id generator (deterministic ids in tests). Default: `crypto.randomUUID`. */
|
|
306
|
+
newId?: () => string;
|
|
307
|
+
/** Report a run that threw during restore (ADR-0030). The row is left `live` for the next boot,
|
|
308
|
+
* so this is the only account of why — the entrypoint logs it. Default: silent. */
|
|
309
|
+
onRestoreError?: (runId: string, err: unknown) => void;
|
|
310
|
+
/** The Sandbox backend `workspace()` provisions through (ADR-0012). Absent = no cluster:
|
|
311
|
+
* workspace-less workflows run fine; a `workspace()` invocation faults its run pointedly. */
|
|
312
|
+
sandbox?: SandboxPort;
|
|
313
|
+
/** The Instance Harness base URL (ADR-0031) — where `workspace: "none"` Turns are admitted. */
|
|
314
|
+
instanceHarness?: string;
|
|
315
|
+
/**
|
|
316
|
+
* Build the run-narrative echo pusher for one Workspace's Harness (ADR-0023) — the seam a fake
|
|
317
|
+
* echo server rides in tests; `startInstance` binds the real wire push (harness-client.ts),
|
|
318
|
+
* closed over the Instance token. Absent = no echo: a host without it runs identically, because
|
|
319
|
+
* the echo is a courtesy view of the feed, never a dependency of the run.
|
|
320
|
+
*/
|
|
321
|
+
echo?: (endpoint: string) => (events: EchoEvent[]) => Promise<void>;
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
/** What we persist per run: the machine snapshot wrapped with the run metadata restore needs.
|
|
325
|
+
* `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent actor
|
|
326
|
+
* through the run binding and saved in the same blob (same store, same atomicity). */
|
|
327
|
+
type RunBlob = {
|
|
328
|
+
workflow: string;
|
|
329
|
+
instanceId: string;
|
|
330
|
+
/** The shape of the Machine that WROTE this snapshot (ADR-0030). `workflow` says which def to
|
|
331
|
+
* look up; this says whether the def found there is still the one this snapshot can be read by.
|
|
332
|
+
* Absent means written before the stamp existed, which restore treats as drift — the point is to
|
|
333
|
+
* never interpret a snapshot whose Machine cannot be vouched for. */
|
|
334
|
+
machine?: string;
|
|
335
|
+
snapshot: unknown;
|
|
336
|
+
agents?: Record<string, AgentAdmission>;
|
|
337
|
+
fault?: string;
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
type LiveRun = {
|
|
341
|
+
record: RunRecord;
|
|
342
|
+
actor: AnyActor;
|
|
343
|
+
def: WorkflowDef;
|
|
344
|
+
/** What this run's callback actors resolve off the actor system — kept so `stop()` can tell
|
|
345
|
+
* them the HOST is the one ending the run (ADR-0024's one exception). */
|
|
346
|
+
binding: RunBinding;
|
|
347
|
+
/** The live admission ledger (ADR-0016): persisted as `RunBlob.agents`, seeded on restore. */
|
|
348
|
+
agents: Record<string, AgentAdmission>;
|
|
349
|
+
/** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
|
|
350
|
+
* message is captured here at the observer and persisted onto the blob for `read`). */
|
|
351
|
+
fault?: string;
|
|
352
|
+
/** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
|
|
353
|
+
listeners: Set<(e: RunFeedEvent) => void>;
|
|
354
|
+
/** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
|
|
355
|
+
* log's preamble. In-memory and this-boot only — a restored run's preamble starts at restore,
|
|
356
|
+
* the same live-only contract the printed log already has. Capped ({@link FEED_SO_FAR_CAP}). */
|
|
357
|
+
feedSoFar: RunFeedEvent[];
|
|
358
|
+
};
|
|
359
|
+
|
|
360
|
+
export class RunHost {
|
|
361
|
+
/** The internal registration table both delivery surfaces share (ADR-0011). Callback actors
|
|
362
|
+
* (`gate`, the Agent slots) reach it via the run binding, not this field. */
|
|
363
|
+
private readonly table = new RegistrationTable();
|
|
364
|
+
|
|
365
|
+
private readonly store: SnapshotStore;
|
|
366
|
+
private readonly reconcile: (run: RunRecord) => boolean | Promise<boolean>;
|
|
367
|
+
private readonly newId: () => string;
|
|
368
|
+
private readonly onRestoreError?: (runId: string, err: unknown) => void;
|
|
369
|
+
private readonly sandbox?: SandboxPort;
|
|
370
|
+
private readonly instanceHarness?: string;
|
|
371
|
+
private readonly echoFactory?: (endpoint: string) => (events: EchoEvent[]) => Promise<void>;
|
|
372
|
+
private readonly workflowDefs = new Map<string, WorkflowDef>();
|
|
373
|
+
private readonly runs = new Map<string, LiveRun>();
|
|
374
|
+
/**
|
|
375
|
+
* Workflow name → its observers. Deliberately on the HOST and not on `LiveRun`: a workflow's
|
|
376
|
+
* watcher outlives every individual run it watches, and `persist()` clears a settled run's own
|
|
377
|
+
* listener set. Keeping it here is what makes that structurally impossible to get wrong, rather
|
|
378
|
+
* than a comment asking the next reader not to clear the wrong Set.
|
|
379
|
+
*/
|
|
380
|
+
private readonly workflowListeners = new Map<string, Set<WorkflowListener>>();
|
|
381
|
+
|
|
382
|
+
constructor(opts: RunHostOptions) {
|
|
383
|
+
this.store = opts.store;
|
|
384
|
+
this.reconcile = opts.reconcile ?? (() => true);
|
|
385
|
+
this.newId = opts.newId ?? (() => randomUUID());
|
|
386
|
+
this.onRestoreError = opts.onRestoreError;
|
|
387
|
+
this.sandbox = opts.sandbox;
|
|
388
|
+
this.instanceHarness = opts.instanceHarness;
|
|
389
|
+
this.echoFactory = opts.echo;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** Register a workflow so `start`/`restore` can run it. Re-registering replaces (dev reload).
|
|
393
|
+
* Nothing about the vocabulary is copied here: event names are scoped to the Machine that
|
|
394
|
+
* declared them and resolved at invoke time off the invoking Machine (ADR-0011, ADR-0049), so
|
|
395
|
+
* a nested Machine's defs are never the root's — or this host's — to hold. */
|
|
396
|
+
register(def: WorkflowDef): void {
|
|
397
|
+
this.workflowDefs.set(def.name, def);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** Drop a workflow's registration (a workflow the host no longer discovers). In-flight
|
|
401
|
+
* runs keep their already-assembled definition; only future `start`s are affected.
|
|
402
|
+
*
|
|
403
|
+
* Observers stay ATTACHED, deliberately. Their runs are still running, so a feed that ended here
|
|
404
|
+
* would be reporting that the work stopped when it did not — and the file may well come back on
|
|
405
|
+
* the next reload, which the still-open feed then picks up with no reconnect. */
|
|
406
|
+
unregister(name: string): void {
|
|
407
|
+
this.workflowDefs.delete(name);
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/** The names of every registered workflow (the `GET /workflows` listing — ADR-0009). */
|
|
411
|
+
workflows(): string[] {
|
|
412
|
+
return [...this.workflowDefs.keys()];
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/** The registered template Machine's serialized structure (`GET /workflows/:name/machine`). */
|
|
416
|
+
machine(name: string): MachineDoc | undefined {
|
|
417
|
+
const def = this.workflowDefs.get(name);
|
|
418
|
+
return def && serializeMachine(name, def.machine);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* The workflow's declared run-input contract as JSON Schema (ADR-0033) — what the workflow
|
|
423
|
+
* detail serves and the Console's start form generates from. `null` for a machine that
|
|
424
|
+
* declares none (a run of it starts with anything), undefined for an unknown workflow. A
|
|
425
|
+
* schema is STRUCTURE, the same class of thing as the Machine doc — open band (ADR-0014).
|
|
426
|
+
*/
|
|
427
|
+
inputSchema(name: string): Record<string, unknown> | null | undefined {
|
|
428
|
+
const def = this.workflowDefs.get(name);
|
|
429
|
+
if (!def) return undefined;
|
|
430
|
+
const schema = inputSchemaOf(def.machine);
|
|
431
|
+
// `io: "input"`: this schema describes what a caller SENDS — a defaulted field is optional
|
|
432
|
+
// at the door (the default is applied by `start`'s parse), not required of the form.
|
|
433
|
+
return schema ? (z.toJSONSchema(schema, { io: "input" }) as Record<string, unknown>) : null;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/** Start a fresh run of a registered workflow; returns its durable ids. */
|
|
437
|
+
async start(workflow: string, input: Record<string, unknown> = {}): Promise<{ runId: string; instanceId: string }> {
|
|
438
|
+
const def = this.workflowDefs.get(workflow);
|
|
439
|
+
if (!def) throw new Error(`no workflow registered as "${workflow}"`);
|
|
440
|
+
|
|
441
|
+
// A declared schema is enforced at the door (ADR-0033), and what starts the run is the
|
|
442
|
+
// PARSED shape (defaults applied, unknown keys stripped) — exactly what a gate delivery
|
|
443
|
+
// lands as, and the same error class: `EventValidationError`, which the wire maps to a 400
|
|
444
|
+
// naming what is accepted. No schema → accept anything (the permissive door the wire was).
|
|
445
|
+
const declared = inputSchemaOf(def.machine);
|
|
446
|
+
let runInput = input;
|
|
447
|
+
if (declared) {
|
|
448
|
+
const parsed = declared.safeParse(input);
|
|
449
|
+
if (!parsed.success) {
|
|
450
|
+
throw new EventValidationError(`invalid input for workflow "${workflow}": ${parsed.error.message}`);
|
|
451
|
+
}
|
|
452
|
+
runInput = parsed.data;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const runId = this.newId();
|
|
456
|
+
const instanceId = this.newId();
|
|
457
|
+
const record: RunRecord = { runId, workflow, instanceId };
|
|
458
|
+
|
|
459
|
+
const machine = this.assemble(def, instanceId);
|
|
460
|
+
// The root machine gets the parsed door PLUS what the host injects beside it —
|
|
461
|
+
// `HostInjectedInput` (vocabulary.ts): the seed Instance ID, added after the parse because no
|
|
462
|
+
// caller sends it and nothing serves it (ADR-0033). Only the root: a machine invoked below is
|
|
463
|
+
// fed by its parent, which is why the type is placement-dependent and `workspace()`'s body
|
|
464
|
+
// guard counts those keys as provided rather than claiming to know where the wrapper sits.
|
|
465
|
+
const actor = this.spawn(machine, { input: { ...runInput, instanceId } }, record, def);
|
|
466
|
+
actor.start();
|
|
467
|
+
return { runId, instanceId };
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* Restore every persisted live run: hydrate, reconcile against the live world, and either
|
|
472
|
+
* re-attach (re-spawn from the rewritten child input), mark lost, or refuse as drifted. Returns
|
|
473
|
+
* the run ids handled, by outcome — the caller announces it, because a boot that silently skipped
|
|
474
|
+
* work is the failure this whole path exists to avoid.
|
|
475
|
+
*
|
|
476
|
+
* Every run is handled INDEPENDENTLY (ADR-0030). The loop is sequential and one throw used to
|
|
477
|
+
* reject the whole method, which rejects `startInstance`, which crash-loops the pod — and every
|
|
478
|
+
* later run in the store never restored at all. One run that cannot be resumed is one run, not an
|
|
479
|
+
* outage.
|
|
480
|
+
*
|
|
481
|
+
* `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — the
|
|
482
|
+
* Machine changed, and it will still have changed on the next boot — so it is written to the row.
|
|
483
|
+
* A throw is not: `reconcile` talks to a cluster, and a kubectl blip must not permanently condemn
|
|
484
|
+
* a run. Those rows are left `live` to be retried on the next boot, and reported every time until
|
|
485
|
+
* they stop failing.
|
|
486
|
+
*/
|
|
487
|
+
async restore(): Promise<{ reattached: string[]; lost: string[]; drifted: string[]; failed: string[] }> {
|
|
488
|
+
const reattached: string[] = [];
|
|
489
|
+
const lost: string[] = [];
|
|
490
|
+
const drifted: string[] = [];
|
|
491
|
+
const failed: string[] = [];
|
|
492
|
+
|
|
493
|
+
for (const stored of await this.store.list()) {
|
|
494
|
+
if (stored.status !== "live") continue;
|
|
495
|
+
const blob = stored.snapshot as RunBlob | null;
|
|
496
|
+
const def = blob ? this.workflowDefs.get(blob.workflow) : undefined;
|
|
497
|
+
if (!blob || !def) {
|
|
498
|
+
await this.store.markLost(stored.runId, blob ? `no workflow "${blob.workflow}"` : "empty snapshot");
|
|
499
|
+
lost.push(stored.runId);
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// The workflow name resolved a def; this asks whether that def is still the Machine this
|
|
504
|
+
// snapshot was written by (ADR-0030). Refused BEFORE `reconcile`, which talks to the cluster:
|
|
505
|
+
// there is no point proving a Sandbox is alive for a run that cannot be read.
|
|
506
|
+
const expected = fingerprintOf(def.machine);
|
|
507
|
+
if (blob.machine !== expected) {
|
|
508
|
+
await this.store.markDrifted(
|
|
509
|
+
stored.runId,
|
|
510
|
+
`workflow "${blob.workflow}" changed shape since this run was saved ` +
|
|
511
|
+
`(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`,
|
|
512
|
+
);
|
|
513
|
+
drifted.push(stored.runId);
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
const record: RunRecord = { runId: stored.runId, workflow: blob.workflow, instanceId: blob.instanceId };
|
|
518
|
+
|
|
519
|
+
try {
|
|
520
|
+
if (!(await this.reconcile(record))) {
|
|
521
|
+
await this.store.markLost(stored.runId, "reconcile: live world absent");
|
|
522
|
+
lost.push(stored.runId);
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// Re-attach every persisted Agent input in the TREE from the admission ledger
|
|
527
|
+
// (ADR-0016): iids are globally unique, so one flat map covers every nesting depth.
|
|
528
|
+
const agents = blob.agents ?? {};
|
|
529
|
+
const hydrated = reattachAgentRuns(blob.snapshot, agents);
|
|
530
|
+
|
|
531
|
+
const actor = this.spawn(
|
|
532
|
+
this.assemble(def, blob.instanceId),
|
|
533
|
+
{ snapshot: hydrated as never },
|
|
534
|
+
record,
|
|
535
|
+
def,
|
|
536
|
+
agents,
|
|
537
|
+
);
|
|
538
|
+
actor.start();
|
|
539
|
+
reattached.push(stored.runId);
|
|
540
|
+
} catch (err) {
|
|
541
|
+
// Left `live` on purpose — see the note above. The row is unchanged, so the next boot tries
|
|
542
|
+
// again; what must not happen is this taking the remaining runs down with it.
|
|
543
|
+
this.onRestoreError?.(stored.runId, err);
|
|
544
|
+
failed.push(stored.runId);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
return { reattached, lost, drifted, failed };
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* An agent instance's LIVE surface (`GET /agents/:iid/surface` — ADR-0013): what the state that
|
|
553
|
+
* invoked this Agent accepts, right now. Undefined once nothing is registered (the state exited,
|
|
554
|
+
* the run settled, the iid is unknown) — the one catch point, and the reason the Adapter never
|
|
555
|
+
* has to learn which turn is live: it asks, per turn, and the answer IS the turn.
|
|
556
|
+
*
|
|
557
|
+
* "Right now" is load-bearing (ADR-0029): the registered defs are the state's VOCABULARY, derived
|
|
558
|
+
* statically from its transitions, and the guards on those transitions are asked here — so an
|
|
559
|
+
* event the Machine cannot currently accept is not offered. The pick has not happened yet, so the
|
|
560
|
+
* question is `mayMove`, not `wouldMove`: a guard that would have judged the Agent's arguments is
|
|
561
|
+
* left on the menu and settled at delivery. The Adapter rebuilds this per MCP connection and the
|
|
562
|
+
* Harness re-lists per Submission, so the filter lands at turn boundaries and never moves under
|
|
563
|
+
* an Agent mid-turn.
|
|
564
|
+
*/
|
|
565
|
+
agentSurface(instanceId: string): AgentSurfaceView | undefined {
|
|
566
|
+
const reg = this.table.lookup(agentAddress(instanceId));
|
|
567
|
+
if (!reg) return undefined;
|
|
568
|
+
return {
|
|
569
|
+
instanceId,
|
|
570
|
+
runId: reg.runId,
|
|
571
|
+
sandbox: reg.sandbox,
|
|
572
|
+
accepts: [...reg.defs.values()]
|
|
573
|
+
.filter((def) => mayMove(reg.invoker, def.name))
|
|
574
|
+
.map((def) => ({
|
|
575
|
+
name: def.name,
|
|
576
|
+
description: def.description,
|
|
577
|
+
input: z.toJSONSchema(def.input),
|
|
578
|
+
semantics: def.semantics,
|
|
579
|
+
})),
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Deliver one event from an Agent's Adapter (`POST /agents/:iid/events` — ADR-0013). Validation
|
|
585
|
+
* and delivery are the table's; this only agent-scopes the address and mints the receipt.
|
|
586
|
+
*/
|
|
587
|
+
sendToAgent(instanceId: string, event: { type?: unknown } & Record<string, unknown>): AgentDeliveryReceipt {
|
|
588
|
+
const { type, ...payload } = event;
|
|
589
|
+
if (typeof type !== "string" || !type) {
|
|
590
|
+
throw new EventValidationError(`event body must carry a string "type" (one of the surface's accepted names)`);
|
|
591
|
+
}
|
|
592
|
+
const address = agentAddress(instanceId);
|
|
593
|
+
const invoking = this.table.lookup(address);
|
|
594
|
+
|
|
595
|
+
// Ask the guard question BEFORE delivering, and ask it with the VALIDATED payload — this is the
|
|
596
|
+
// exact form of the check the surface build can only approximate payload-blind (ADR-0029).
|
|
597
|
+
// Parsed here rather than read back out of `deliver` because `deliver` is void by design (it is
|
|
598
|
+
// the one behavior behind both dialects); a zod default applied there but not here would leave
|
|
599
|
+
// a guard reading that field answering on `undefined`. An unaccepted name or a bad payload
|
|
600
|
+
// makes this unreliable and `deliver` throws on the next line anyway — so it fails open.
|
|
601
|
+
const def = invoking?.defs.get(type);
|
|
602
|
+
const parsed = def?.input.safeParse(payload ?? {});
|
|
603
|
+
const moved = wouldMove(invoking?.invoker, parsed?.success ? { type, ...parsed.data } : { type, ...payload });
|
|
604
|
+
|
|
605
|
+
this.table.deliver(address, type, payload);
|
|
606
|
+
// Read AFTER the delivery, off the SAME table the ADR-0024 guarantee uses — so the receipt
|
|
607
|
+
// reports what happened rather than what was hoped. `deliver` reached the invoking state's
|
|
608
|
+
// `sendBack` synchronously, so a pick that moved the Machine out of that state has already
|
|
609
|
+
// destroyed this registration by now.
|
|
610
|
+
//
|
|
611
|
+
// IDENTITY, not existence: under `session: "continue"` the next state re-registers the SAME
|
|
612
|
+
// address for its own turn, and that is a new turn — this one still ended.
|
|
613
|
+
return {
|
|
614
|
+
delivered: true,
|
|
615
|
+
event: type,
|
|
616
|
+
moved,
|
|
617
|
+
turnComplete: this.table.lookup(address) !== invoking,
|
|
618
|
+
deliveryId: this.newId(),
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/** A run's open gates as callers discover them (`GET /runs/:id` — ADR-0011). Settled/unknown
|
|
623
|
+
* run → empty: a gate is a LIVE surface, it does not outlive its state. */
|
|
624
|
+
gates(runId: string): GateView[] {
|
|
625
|
+
return this.table
|
|
626
|
+
.byRun(runId)
|
|
627
|
+
.filter((reg) => reg.kind === "gate")
|
|
628
|
+
.map((reg) => ({
|
|
629
|
+
gate: reg.id,
|
|
630
|
+
path: reg.path ?? [],
|
|
631
|
+
accepts: [...reg.defs.values()].map((def) => ({
|
|
632
|
+
name: def.name,
|
|
633
|
+
description: def.description,
|
|
634
|
+
input: z.toJSONSchema(def.input),
|
|
635
|
+
})),
|
|
636
|
+
meta: reg.meta,
|
|
637
|
+
}));
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
|
|
641
|
+
* Validation and delivery are the table's — this only run-scopes the address. */
|
|
642
|
+
sendToGate(runId: string, gate: string, event: { type?: unknown } & Record<string, unknown>): void {
|
|
643
|
+
const { type, ...payload } = event;
|
|
644
|
+
if (typeof type !== "string" || !type) {
|
|
645
|
+
throw new EventValidationError(`event body must carry a string "type" (one of the gate's accepted names)`);
|
|
646
|
+
}
|
|
647
|
+
try {
|
|
648
|
+
this.table.deliver(gateAddress(runId, gate), type, payload);
|
|
649
|
+
} catch (err) {
|
|
650
|
+
if (err instanceof UnknownAddressError) {
|
|
651
|
+
const open = this.gates(runId).map((g) => g.gate);
|
|
652
|
+
throw new UnknownAddressError(
|
|
653
|
+
`no open gate "${gate}" on run "${runId}"${open.length ? ` (open: ${open.join(", ")})` : ""}`,
|
|
654
|
+
);
|
|
655
|
+
}
|
|
656
|
+
throw err;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* Observe a live run's feed (the `GET /runs/:id/events` SSE — ADR-0009): status deltas after every
|
|
662
|
+
* transition, plus the author's `emit`s. Returns an unsubscribe fn. The current status is **replayed
|
|
663
|
+
* immediately** on attach, so a freshly-attached watcher sees where the run is now (e.g. parked on an
|
|
664
|
+
* approval) rather than waiting for the next transition. The final status is emitted on the terminal
|
|
665
|
+
* transition right before the run is dropped from the registry. Attaching to an unknown/settled run
|
|
666
|
+
* is a no-op — read its terminal status via {@link read} instead.
|
|
667
|
+
*/
|
|
668
|
+
subscribe(runId: string, listener: (e: RunFeedEvent) => void): () => void {
|
|
669
|
+
const run = this.runs.get(runId);
|
|
670
|
+
if (!run) return () => {};
|
|
671
|
+
run.listeners.add(listener);
|
|
672
|
+
listener({ kind: "status", status: this.liveStatus(run) });
|
|
673
|
+
return () => run.listeners.delete(listener);
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
/**
|
|
677
|
+
* Observe a whole WORKFLOW (the `GET /workflows/:name/events` SSE — ADR-0022): every run of it
|
|
678
|
+
* appearing, moving, emitting and leaving, for as long as the caller stays attached.
|
|
679
|
+
*
|
|
680
|
+
* The current set is RETURNED rather than replayed through the listener, and the subscription is
|
|
681
|
+
* registered in the same synchronous call. That is the whole point of the signature: split into
|
|
682
|
+
* a list-then-subscribe pair, a run starting between the two calls appears in neither, and the
|
|
683
|
+
* watcher is quietly wrong until something else happens to move it.
|
|
684
|
+
*
|
|
685
|
+
* Subscribing to a NAME, not to a registration — an unknown workflow attaches to an empty set
|
|
686
|
+
* (a later registration may supply it, and the feed should just start working).
|
|
687
|
+
*/
|
|
688
|
+
observeWorkflow(workflow: string, listener: WorkflowListener): { runs: RunStatus[]; unsubscribe: () => void } {
|
|
689
|
+
let listeners = this.workflowListeners.get(workflow);
|
|
690
|
+
if (!listeners) this.workflowListeners.set(workflow, (listeners = new Set()));
|
|
691
|
+
listeners.add(listener);
|
|
692
|
+
return {
|
|
693
|
+
runs: this.list().filter((s) => s.workflow === workflow),
|
|
694
|
+
unsubscribe: () => {
|
|
695
|
+
listeners.delete(listener);
|
|
696
|
+
// Drop the empty Set, but ONLY if the map still holds this one. A stale unsubscribe (called
|
|
697
|
+
// twice, or after `close()`) would otherwise evict whatever Set replaced it under the same
|
|
698
|
+
// name — silently orphaning a watcher that has nothing to do with this one.
|
|
699
|
+
if (listeners.size === 0 && this.workflowListeners.get(workflow) === listeners) {
|
|
700
|
+
this.workflowListeners.delete(workflow);
|
|
701
|
+
}
|
|
702
|
+
},
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/** How many observers a workflow's feed currently has. Exists so a test can prove that a client
|
|
707
|
+
* going away actually DETACHES — a leak here is invisible until the process runs out of memory. */
|
|
708
|
+
observerCount(workflow: string): number {
|
|
709
|
+
return this.workflowListeners.get(workflow)?.size ?? 0;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/** Feed one workflow's observers. The projection to the wire happens in http.ts, not here. */
|
|
713
|
+
private announce(workflow: string, event: WorkflowFeedEvent): void {
|
|
714
|
+
for (const listener of this.workflowListeners.get(workflow) ?? []) listener(event);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/** A run's LIVE status — undefined once it settles and is dropped from the registry. Sync. */
|
|
718
|
+
status(runId: string): RunStatus | undefined {
|
|
719
|
+
const run = this.runs.get(runId);
|
|
720
|
+
return run && this.liveStatus(run);
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* Read a run's status, **reading through to the store** when it is no longer live (ADR-0009). A
|
|
725
|
+
* completed run's final snapshot is persisted before `persist()` drops it from the registry, so a
|
|
726
|
+
* terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
|
|
727
|
+
* undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared).
|
|
728
|
+
*/
|
|
729
|
+
async read(runId: string): Promise<RunStatus | undefined> {
|
|
730
|
+
const live = this.status(runId);
|
|
731
|
+
if (live) return live;
|
|
732
|
+
const stored = await this.store.load(runId);
|
|
733
|
+
const blob = stored?.snapshot as RunBlob | null | undefined;
|
|
734
|
+
if (!stored || !blob) return undefined;
|
|
735
|
+
const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown };
|
|
736
|
+
return {
|
|
737
|
+
runId,
|
|
738
|
+
workflow: blob.workflow,
|
|
739
|
+
instanceId: blob.instanceId,
|
|
740
|
+
// The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
|
|
741
|
+
// cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
|
|
742
|
+
// (ADR-0025). While the row still says "live", the Machine's own status is the answer.
|
|
743
|
+
status: stored.status === "live" ? (snap.status ?? stored.status) : stored.status,
|
|
744
|
+
value: snap.value,
|
|
745
|
+
context: snap.context,
|
|
746
|
+
children: runChildren(snap), // the persisted `children` map — same tree, off the store
|
|
747
|
+
fault: blob.fault,
|
|
748
|
+
reason: stored.reason,
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
list(): RunStatus[] {
|
|
753
|
+
return [...this.runs.keys()].map((id) => this.status(id)).filter((s): s is RunStatus => s !== undefined);
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
/**
|
|
757
|
+
* Run ids sharing a prefix — the live registry unioned with the store — backing abbreviated run
|
|
758
|
+
* ids in the CLI (ADR-0009). The live half is not belt-and-braces: `persist()` is scheduled on a
|
|
759
|
+
* microtask, so a just-started run is in `runs` before it is anywhere in the store. The store
|
|
760
|
+
* half is what makes a settled run abbreviate-able. A run past its first transition sits in both,
|
|
761
|
+
* hence the `Set`.
|
|
762
|
+
*/
|
|
763
|
+
async candidates(prefix: string, limit: number): Promise<string[]> {
|
|
764
|
+
const live = [...this.runs.keys()].filter((id) => id.startsWith(prefix));
|
|
765
|
+
const stored = await this.store.findIdsByPrefix(prefix, limit);
|
|
766
|
+
return [...new Set([...live, ...stored])].sort().slice(0, limit);
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/** The live runs of ONE workflow, projected for observers (the Console's run list). Scoped
|
|
770
|
+
* server-side: an observer asks about a workflow it can already name, and gets back only runs of
|
|
771
|
+
* it — never a listing of everything this orchestrator happens to be running. */
|
|
772
|
+
observations(workflow: string): RunObservation[] {
|
|
773
|
+
return this.list()
|
|
774
|
+
.filter((s) => s.workflow === workflow)
|
|
775
|
+
.map(observe);
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/**
|
|
779
|
+
* End every open feed because the host is going away — the shutdown counterpart to `subscribe`.
|
|
780
|
+
*
|
|
781
|
+
* A feed has no natural end: a run parked on a gate transitions for hours, and its watchers hold
|
|
782
|
+
* an in-flight HTTP request the whole time. `server.close()` (instance.ts) waits for in-flight
|
|
783
|
+
* requests, so without this a single attached `jr2 run` wedges shutdown indefinitely. `closed` is
|
|
784
|
+
* the frame that lets those handlers exit. Runs themselves are untouched: this ends the
|
|
785
|
+
* OBSERVATION, not the work — the snapshots are already durable, and `restore()` picks them up.
|
|
786
|
+
*/
|
|
787
|
+
async close(): Promise<void> {
|
|
788
|
+
for (const run of this.runs.values()) {
|
|
789
|
+
for (const listener of run.listeners) listener({ kind: "closed" });
|
|
790
|
+
run.listeners.clear();
|
|
791
|
+
}
|
|
792
|
+
for (const listeners of this.workflowListeners.values()) {
|
|
793
|
+
for (const listener of listeners) listener({ kind: "closed" });
|
|
794
|
+
}
|
|
795
|
+
this.workflowListeners.clear();
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* TEARDOWN: stop hosting a run in this process, leaving it RESTORABLE (ADR-0007/0025). The run
|
|
800
|
+
* keeps its stored status "live", so the next `restore()` picks it up where it left off — which
|
|
801
|
+
* is why this is ADR-0024's one exception: the Agents' submissions must stay alive for that
|
|
802
|
+
* re-attach, and the actor cannot infer "the host did this", so the binding is told before the
|
|
803
|
+
* stop.
|
|
804
|
+
*
|
|
805
|
+
* This is NOT the user's CANCEL — that is {@link cancel}, which ends the work. Nothing in a
|
|
806
|
+
* deployed Orchestrator calls this today (process shutdown stops no actors: `instance.ts` ends
|
|
807
|
+
* observation and nothing else); it is the teardown primitive, and the seam an orchestrator
|
|
808
|
+
* restart is simulated through.
|
|
809
|
+
*/
|
|
810
|
+
async stop(runId: string): Promise<void> {
|
|
811
|
+
const run = this.runs.get(runId);
|
|
812
|
+
if (!run) return;
|
|
813
|
+
run.binding.hostStopping = true;
|
|
814
|
+
// Say so BEFORE the actor stops, while there is still a status to read. A stopped run is not a
|
|
815
|
+
// settled one — `persist()` never runs for it (the tracked-run guard drops the scheduled save,
|
|
816
|
+
// keeping the stored status "live" for restore — see `spawn`), so nothing else on this path
|
|
817
|
+
// would ever tell a watcher the run left. Without it the page shows it live forever.
|
|
818
|
+
this.announceGone(run);
|
|
819
|
+
run.actor.stop();
|
|
820
|
+
this.untrack(run.record);
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* CANCEL: the human's "abandon this run" (`jr2 send <run> --event CANCEL` — ADR-0025). It ends
|
|
825
|
+
* the work rather than parking it: stopping the actor with no `hostStopping` flag ends every
|
|
826
|
+
* live Agent invocation, and each one ends its Agent's turn remotely (ADR-0024). The run
|
|
827
|
+
* is then persisted TERMINAL, so `restore()` leaves it alone and `read()` reports how it ended.
|
|
828
|
+
*
|
|
829
|
+
* Ending the turns and refusing to restore are one decision, not two: a cancelled run that came
|
|
830
|
+
* back would re-attach to submissions that settled `aborted`, and `settle`'s rejection would
|
|
831
|
+
* fault a run whose Agents were stopped on purpose.
|
|
832
|
+
*/
|
|
833
|
+
async cancel(runId: string): Promise<void> {
|
|
834
|
+
const run = this.runs.get(runId);
|
|
835
|
+
if (!run) return;
|
|
836
|
+
run.actor.stop();
|
|
837
|
+
// Persist AFTER the stop: the snapshot is final, and `persist` fans out the last status,
|
|
838
|
+
// announces `gone`, and untracks — the same terminal path a run that settled on its own takes.
|
|
839
|
+
this.persist(run, "cancelled");
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Feed a run's final word to both granularities: where it ended, then that it is gone. */
|
|
843
|
+
private announceGone(run: LiveRun, status: RunStatus = this.liveStatus(run)): void {
|
|
844
|
+
this.announce(run.record.workflow, { kind: "status", status });
|
|
845
|
+
this.announce(run.record.workflow, { kind: "gone", runId: run.record.runId });
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
// --- internals ---
|
|
849
|
+
|
|
850
|
+
/** Put one event on the run's feed: buffer it for ADR-0023's backfill, then fan it out. The
|
|
851
|
+
* buffer and the fan-out share one seat so an attached echo and a later attach see the SAME
|
|
852
|
+
* feed — a projection cannot drift from a record it is read out of. */
|
|
853
|
+
private feed(run: LiveRun, event: RunFeedEvent): void {
|
|
854
|
+
run.feedSoFar.push(event);
|
|
855
|
+
if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
|
|
856
|
+
for (const listener of run.listeners) listener(event);
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* Attach the run-narrative echo (ADR-0023): replay the run's feed-so-far to the Workspace's
|
|
861
|
+
* Harness at `endpoint` — the backfilled preamble, why this Workspace exists — then tee live
|
|
862
|
+
* until detached. FIRE-AND-FORGET is this method's contract: pushes are chained so events
|
|
863
|
+
* arrive in feed order, and a failure is logged once and swallowed — the feed remains the
|
|
864
|
+
* record, the log is a courtesy view, and nothing here can fail a turn, a state, or a run.
|
|
865
|
+
* Scoped to the OWNING run's lineage by construction: it reads one run's buffer and listeners
|
|
866
|
+
* and nothing else — a sibling run's events cannot reach this endpoint through here.
|
|
867
|
+
*/
|
|
868
|
+
private attachEcho(runId: string, endpoint: string): () => void {
|
|
869
|
+
const run = this.runs.get(runId);
|
|
870
|
+
const factory = this.echoFactory;
|
|
871
|
+
if (!run || !factory) return () => {};
|
|
872
|
+
const push = factory(endpoint);
|
|
873
|
+
let chain = Promise.resolve();
|
|
874
|
+
let reported = false;
|
|
875
|
+
const enqueue = (events: EchoEvent[]): void => {
|
|
876
|
+
if (events.length === 0) return;
|
|
877
|
+
chain = chain
|
|
878
|
+
.then(() => push(events))
|
|
879
|
+
.catch((err) => {
|
|
880
|
+
// Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
|
|
881
|
+
// transition into an error line, and must not surface anywhere a run could trip on.
|
|
882
|
+
if (reported) return;
|
|
883
|
+
reported = true;
|
|
884
|
+
console.error(
|
|
885
|
+
`run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
|
|
886
|
+
(err instanceof Error ? err.message : String(err)),
|
|
887
|
+
);
|
|
888
|
+
});
|
|
889
|
+
};
|
|
890
|
+
enqueue(run.feedSoFar.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
|
|
891
|
+
const listener = (ev: RunFeedEvent): void => {
|
|
892
|
+
const projected = echoEventOf(ev, endpoint);
|
|
893
|
+
if (projected) enqueue([projected]);
|
|
894
|
+
};
|
|
895
|
+
run.listeners.add(listener);
|
|
896
|
+
return () => run.listeners.delete(listener);
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/** A live run's status, read off its actor — the one place the shape is built. */
|
|
900
|
+
private liveStatus(run: LiveRun): RunStatus {
|
|
901
|
+
const snap = run.actor.getSnapshot();
|
|
902
|
+
return {
|
|
903
|
+
...run.record,
|
|
904
|
+
status: snap.status,
|
|
905
|
+
value: snap.value,
|
|
906
|
+
context: snap.context,
|
|
907
|
+
children: runChildren(snap),
|
|
908
|
+
fault: run.fault,
|
|
909
|
+
};
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
/** Fill the machine's named actor slots for one run (`.provide()`, the ADR-0015 test seam). */
|
|
913
|
+
private assemble(def: WorkflowDef, instanceId: string): AnyStateMachine {
|
|
914
|
+
const providers = def.provide({ instanceId });
|
|
915
|
+
return def.machine.provide(providers as Parameters<AnyStateMachine["provide"]>[0]);
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
/**
|
|
919
|
+
* Create + track a run's root actor. Persistence is driven by the actor system's INSPECTION
|
|
920
|
+
* stream, not the root subscription: a nested body's transition never notifies root
|
|
921
|
+
* subscribers, but it must hit the store, or a crash would restore a stale tree. Snapshot
|
|
922
|
+
* events within one macrostep are coalesced per microtask, and a persist scheduled around
|
|
923
|
+
* stop/untrack is dropped by the tracked-run guard (so `stop()` keeps the stored status
|
|
924
|
+
* "live" for restore).
|
|
925
|
+
*/
|
|
926
|
+
private spawn(
|
|
927
|
+
machine: AnyStateMachine,
|
|
928
|
+
options: { input?: Record<string, unknown>; snapshot?: never },
|
|
929
|
+
record: RunRecord,
|
|
930
|
+
def: WorkflowDef,
|
|
931
|
+
agents: Record<string, AgentAdmission> = {},
|
|
932
|
+
): AnyActor {
|
|
933
|
+
let live: LiveRun | undefined;
|
|
934
|
+
let scheduled = false;
|
|
935
|
+
const schedule = () => {
|
|
936
|
+
if (scheduled) return;
|
|
937
|
+
scheduled = true;
|
|
938
|
+
queueMicrotask(() => {
|
|
939
|
+
scheduled = false;
|
|
940
|
+
if (live && this.runs.get(record.runId) === live) this.persist(live);
|
|
941
|
+
});
|
|
942
|
+
};
|
|
943
|
+
const binding: RunBinding = {
|
|
944
|
+
runId: record.runId,
|
|
945
|
+
workflow: record.workflow,
|
|
946
|
+
table: this.table,
|
|
947
|
+
sandbox: this.sandbox,
|
|
948
|
+
instanceHarness: this.instanceHarness,
|
|
949
|
+
// The admission ledger's write half (ADR-0016): the Agent actor reports the durable handle the
|
|
950
|
+
// moment the Harness admits it, and the ledger hits the store in the same RunBlob save. An
|
|
951
|
+
// admission arriving around stop/untrack still lands in `agents` but skips the save,
|
|
952
|
+
// exactly like the persist scheduler's tracked-run guard.
|
|
953
|
+
recordAdmission: (instanceId, admission) => {
|
|
954
|
+
agents[instanceId] = admission;
|
|
955
|
+
const run = this.runs.get(record.runId);
|
|
956
|
+
if (run && run.agents === agents) this.persist(run);
|
|
957
|
+
},
|
|
958
|
+
// Absorbed-retry attempts go straight to the run's observers (SSE/CLI watch) — they are
|
|
959
|
+
// feed events, not machine events (ADR-0016: the workflow sees only the terminal fault).
|
|
960
|
+
telemetry: (event) => {
|
|
961
|
+
for (const listener of this.runs.get(record.runId)?.listeners ?? []) listener(event);
|
|
962
|
+
this.announce(record.workflow, { ...event, runId: record.runId });
|
|
963
|
+
},
|
|
964
|
+
// Turn markers (ADR-0023) ride the per-run feed alone — the prompt is Instance-token-class
|
|
965
|
+
// data, so they never touch the workflow feed (ADR-0014's open band strips even Emit
|
|
966
|
+
// payloads there).
|
|
967
|
+
marker: (event) => {
|
|
968
|
+
const run = this.runs.get(record.runId);
|
|
969
|
+
if (run && run.binding === binding) this.feed(run, event);
|
|
970
|
+
},
|
|
971
|
+
// The run-narrative echo attach (ADR-0023), called from `workspace()`'s registrar.
|
|
972
|
+
echo: (endpoint) => this.attachEcho(record.runId, endpoint),
|
|
973
|
+
};
|
|
974
|
+
let bound = false;
|
|
975
|
+
const actor = createActor(machine, {
|
|
976
|
+
...options,
|
|
977
|
+
inspect: (ev) => {
|
|
978
|
+
// Bind the run's actor SYSTEM on the ROOT's creation event — the first inspection event,
|
|
979
|
+
// fired inside createActor BEFORE any child of the initial state is constructed. That
|
|
980
|
+
// ordering matters: jr2Setup's wrapped invoke inputs (iid minting — ADR-0016) run at child
|
|
981
|
+
// construction and must already see the run identity. The system is shared by every actor
|
|
982
|
+
// in the tree, which is what run-scopes gate ids with zero workflow plumbing (ADR-0011).
|
|
983
|
+
if (!bound && ev.type === "@xstate.actor") {
|
|
984
|
+
bound = true;
|
|
985
|
+
bindRun((ev.actorRef as AnyActorRef).system, binding);
|
|
986
|
+
}
|
|
987
|
+
// Forward the workflow author's `emit({...})` onto the feeds, from EVERY actor in the
|
|
988
|
+
// tree as it is created: xstate scopes emitted events to the actor that emits them — no
|
|
989
|
+
// bubbling — and a run's Emits mostly happen in child machines (a `workspace()` body IS
|
|
990
|
+
// one). A root-only subscription silently dropped exactly the Emits ADR-0022/0023 exist
|
|
991
|
+
// to surface. The subscription dies with its actor; restore re-creates both.
|
|
992
|
+
if (ev.type === "@xstate.actor") {
|
|
993
|
+
const ref = ev.actorRef as AnyActorRef & { on?: (type: string, handler: (e: unknown) => void) => unknown };
|
|
994
|
+
ref.on?.("*", (emitted) => {
|
|
995
|
+
const event = emitted as { type: string } & Record<string, unknown>;
|
|
996
|
+
const run = this.runs.get(record.runId);
|
|
997
|
+
if (!run || run.binding !== binding) return;
|
|
998
|
+
this.feed(run, { kind: "emit", event });
|
|
999
|
+
this.announce(record.workflow, { kind: "emit", runId: record.runId, event });
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
if (ev.type === "@xstate.snapshot") schedule();
|
|
1003
|
+
},
|
|
1004
|
+
});
|
|
1005
|
+
live = this.track(record, actor, def, agents, binding);
|
|
1006
|
+
return actor;
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
private track(
|
|
1010
|
+
record: RunRecord,
|
|
1011
|
+
actor: AnyActor,
|
|
1012
|
+
def: WorkflowDef,
|
|
1013
|
+
agents: Record<string, AgentAdmission>,
|
|
1014
|
+
binding: RunBinding,
|
|
1015
|
+
): LiveRun {
|
|
1016
|
+
const run: LiveRun = { record, actor, def, agents, binding, listeners: new Set(), feedSoFar: [] };
|
|
1017
|
+
this.runs.set(record.runId, run);
|
|
1018
|
+
// Ordinary persistence rides the inspection stream (see `spawn`); the subscription exists
|
|
1019
|
+
// for the ERROR channel: an errored actor (an invoke threw — e.g. ADR-0011's invoke-time
|
|
1020
|
+
// manifest check) reports here. Capture the message (xstate serializes the Error itself to
|
|
1021
|
+
// `{}`), then persist: the snapshot's "error" status stores the run and untracks it.
|
|
1022
|
+
actor.subscribe({
|
|
1023
|
+
error: (err) => {
|
|
1024
|
+
run.fault = err instanceof Error ? err.message : String(err);
|
|
1025
|
+
this.persist(run);
|
|
1026
|
+
// Nothing to release: a faulted run stops its actors, and each workspace's lease is one
|
|
1027
|
+
// of them (ADR-0021). The pod stays up for inspection (ADR-0012's destroy-less terminal)
|
|
1028
|
+
// and ages out of the operator's idle timeout on its own.
|
|
1029
|
+
},
|
|
1030
|
+
});
|
|
1031
|
+
// (The author's `emit({...})` forwarding lives in `spawn`'s inspect handler — per actor,
|
|
1032
|
+
// because emitted events do not bubble — not here on the root alone.)
|
|
1033
|
+
// The run has appeared. This is the convergence point of `start()` and `restore()` — both reach
|
|
1034
|
+
// the live set through here, so one fan-out covers a fresh run and a resumed one alike.
|
|
1035
|
+
//
|
|
1036
|
+
// It runs BEFORE `actor.start()` (see the call in `spawn`), so this first frame is the pre-start
|
|
1037
|
+
// snapshot, corrected on the next microtask by the first `persist()`. That is fine because the
|
|
1038
|
+
// feed is level-triggered — every frame is a whole status, so a momentarily-early one is simply
|
|
1039
|
+
// overwritten rather than accumulated. Do NOT "fix" this by moving the fan-out into `start()`:
|
|
1040
|
+
// `restore()` does not go through it, and resumed runs would silently stop appearing.
|
|
1041
|
+
this.announce(record.workflow, { kind: "status", status: this.liveStatus(run) });
|
|
1042
|
+
return run;
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
private untrack(record: RunRecord): void {
|
|
1046
|
+
this.runs.delete(record.runId);
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
/**
|
|
1050
|
+
* Persist the run's snapshot after a transition; drop it from the registry once final.
|
|
1051
|
+
*
|
|
1052
|
+
* `terminal` is the run-lifecycle verdict the MACHINE cannot supply: `cancel()` passes
|
|
1053
|
+
* "cancelled", because xstate only knows its actor was stopped (ADR-0025). It is the stored
|
|
1054
|
+
* status and the one the feeds report.
|
|
1055
|
+
*/
|
|
1056
|
+
private persist(run: LiveRun, terminal?: string): void {
|
|
1057
|
+
const snapshot = run.actor.getPersistedSnapshot();
|
|
1058
|
+
const machineStatus = (snapshot as { status?: string }).status ?? "active";
|
|
1059
|
+
const serialized = serializeSnapshot(snapshot, {
|
|
1060
|
+
stripContext: (ctx) => ctx, // see ADR-0007: live infra lives in `.provide` closures, not context
|
|
1061
|
+
stripChildInput: (input) => input,
|
|
1062
|
+
});
|
|
1063
|
+
const blob: RunBlob = {
|
|
1064
|
+
workflow: run.record.workflow,
|
|
1065
|
+
instanceId: run.record.instanceId,
|
|
1066
|
+
// Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
|
|
1067
|
+
// providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
|
|
1068
|
+
machine: fingerprintOf(run.def.machine),
|
|
1069
|
+
snapshot: serialized,
|
|
1070
|
+
agents: run.agents,
|
|
1071
|
+
fault: run.fault,
|
|
1072
|
+
};
|
|
1073
|
+
const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
|
|
1074
|
+
void this.store.save(run.record.runId, blob, status);
|
|
1075
|
+
|
|
1076
|
+
// Feed per-run observers (SSE/CLI watch). On the terminal transition emit the final status
|
|
1077
|
+
// BEFORE untrack drops the run from the registry, then drop the now-useless listener set.
|
|
1078
|
+
//
|
|
1079
|
+
// Persistence rides the INSPECTION stream (see `spawn`), which fires on a child's transitions
|
|
1080
|
+
// too — so this frame lands on child movement, and its `children` tree carries the new state.
|
|
1081
|
+
// That is the entire live half of the Console's child diagrams: no extra subscription.
|
|
1082
|
+
const runStatus = terminal ? { ...this.liveStatus(run), status: terminal } : this.liveStatus(run);
|
|
1083
|
+
this.feed(run, { kind: "status", status: runStatus });
|
|
1084
|
+
|
|
1085
|
+
if (status !== "live") {
|
|
1086
|
+
// The workflow's watchers get the same final status, then `gone` — the run's last word on
|
|
1087
|
+
// both feeds, still before untrack, while there is a status to read.
|
|
1088
|
+
this.announceGone(run, runStatus);
|
|
1089
|
+
this.untrack(run.record);
|
|
1090
|
+
run.listeners.clear();
|
|
1091
|
+
} else {
|
|
1092
|
+
this.announce(run.record.workflow, { kind: "status", status: runStatus });
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
}
|