@jr2/orchestrator 0.2.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/server.ts +7 -4
- package/console/canvas.ts +27 -14
- package/console/components/app.ts +10 -1
- package/console/components/fleet.ts +39 -6
- package/console/components/machine-pane.ts +89 -2
- package/console/components/nav.ts +1 -1
- package/console/components/schema-form.ts +12 -4
- package/console/json-field.ts +23 -0
- package/console/main.ts +110 -39
- package/console/store.ts +221 -17
- package/console/style.css +90 -0
- package/package.json +4 -5
- package/src/actor.ts +507 -54
- package/src/agent.ts +2 -2
- package/src/ambient.ts +2 -0
- package/src/assembly-checks.ts +165 -0
- package/src/assembly-tar.ts +494 -0
- package/src/assembly.ts +721 -0
- package/src/chart.ts +2050 -0
- package/src/charts.ts +720 -0
- package/src/config.ts +233 -22
- package/src/content-hash.ts +105 -0
- package/src/custodian.lua +313 -0
- package/src/custodian.ts +951 -0
- package/src/customize.ts +216 -25
- package/src/effect-checks.ts +557 -0
- package/src/effect.ts +905 -0
- package/src/effector-wire.ts +525 -0
- package/src/fingerprint.ts +4 -3
- package/src/gate.ts +8 -0
- package/src/git-clone.ts +65 -0
- package/src/git-push.ts +336 -0
- package/src/harness-client.ts +247 -25
- package/src/held-secrets.ts +643 -0
- package/src/holds.ts +156 -0
- package/src/http-effect.ts +981 -0
- package/src/http.ts +514 -119
- package/src/images.ts +58 -31
- package/src/index.ts +49 -4
- package/src/instance.ts +445 -59
- package/src/kube-client.ts +333 -0
- package/src/machine-doc.ts +88 -6
- package/src/model-keys.ts +92 -0
- package/src/mount-digest.ts +53 -0
- package/src/names.ts +89 -7
- package/src/notices.ts +88 -0
- package/src/open-body.ts +1444 -0
- package/src/open-effect.ts +144 -0
- package/src/open.ts +4 -4
- package/src/parts.ts +530 -33
- package/src/party.ts +468 -0
- package/src/pick.ts +309 -0
- package/src/pool.ts +1 -1
- package/src/registration.ts +301 -17
- package/src/repo-fetch.ts +61 -86
- package/src/repos.ts +63 -74
- package/src/rest.ts +133 -0
- package/src/run-host.ts +1841 -105
- package/src/sandbox-effect.ts +891 -0
- package/src/sandbox-effector.ts +111 -0
- package/src/sandbox-kube.ts +1719 -0
- package/src/sandbox-watch.ts +260 -0
- package/src/secret-placement.ts +494 -0
- package/src/server.ts +285 -89
- package/src/setup.ts +589 -70
- package/src/size.ts +342 -0
- package/src/snapshot-store.ts +64 -13
- package/src/stub-harness.ts +5 -0
- package/src/sync-client.ts +54 -0
- package/src/timers.ts +178 -0
- package/src/tokens.ts +44 -12
- package/src/vocabulary.ts +66 -1
- package/src/wire.ts +156 -2
- package/src/workspace.ts +529 -194
- package/src/sandbox-kubectl.ts +0 -1144
package/src/run-host.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// closures over their own `sendBack` — so delivery lands on the invoking state at any
|
|
8
8
|
// nesting depth and the host routes nothing.
|
|
9
9
|
// - Two thin surfaces sit on that table, and NEITHER is MCP (ADR-0013 — the Orchestrator does
|
|
10
|
-
// not speak it): `agentSurface` / `sendToAgent` serve the Agent's
|
|
10
|
+
// not speak it): `agentSurface` / `sendToAgent` serve the Agent's Harness (`/agents/:iid/*`),
|
|
11
11
|
// `gates` / `sendToGate` serve humans, webhooks and CI (`/runs/:id/gates/*`). Lookup,
|
|
12
12
|
// validation, delivery and lifecycle stay implemented once, in the table.
|
|
13
13
|
// - The run's Agent children report their durable admissions through the run binding into
|
|
@@ -21,29 +21,82 @@
|
|
|
21
21
|
|
|
22
22
|
import { randomUUID } from "node:crypto";
|
|
23
23
|
import { z } from "zod";
|
|
24
|
-
import {
|
|
25
|
-
|
|
24
|
+
import {
|
|
25
|
+
createActor,
|
|
26
|
+
type AnyActor,
|
|
27
|
+
type AnyActorLogic,
|
|
28
|
+
type AnyActorRef,
|
|
29
|
+
type AnyEventObject,
|
|
30
|
+
type AnyMachineSnapshot,
|
|
31
|
+
type AnyStateMachine,
|
|
32
|
+
} from "xstate";
|
|
33
|
+
import { SENDER_KEY, type EventDef, type EventSemantics, type Sender } from "@jr2/agent-protocol";
|
|
34
|
+
import { partiesAt, seatOf, senderText } from "./party.ts";
|
|
35
|
+
import { placeAnswers, type AnswerSeat } from "./pick.ts";
|
|
26
36
|
import { inputSchemaOf } from "./vocabulary.ts";
|
|
27
|
-
import type { EchoEvent, EchoStatusChild } from "./wire.ts";
|
|
37
|
+
import type { EchoEvent, EchoStatusChild, LiveConversation, PickAnswer } from "./wire.ts";
|
|
38
|
+
import { answerOf, childErrorOf, replyOf, type Hold } from "./holds.ts";
|
|
39
|
+
import { restOf, runningBelow } from "./rest.ts";
|
|
28
40
|
import {
|
|
41
|
+
acceptsNow,
|
|
29
42
|
agentAddress,
|
|
43
|
+
allowedPicks,
|
|
30
44
|
bindRun,
|
|
31
45
|
EventValidationError,
|
|
32
46
|
gateAddress,
|
|
33
|
-
|
|
47
|
+
NotTakenError,
|
|
48
|
+
OutstandingError,
|
|
34
49
|
RegistrationTable,
|
|
50
|
+
refuseSenderField,
|
|
35
51
|
UnknownAddressError,
|
|
36
52
|
wouldMove,
|
|
53
|
+
targetsOf,
|
|
54
|
+
leftAgain,
|
|
55
|
+
type Registration,
|
|
37
56
|
type RetryTelemetry,
|
|
38
57
|
type RunBinding,
|
|
39
|
-
type
|
|
58
|
+
type RunMarker,
|
|
40
59
|
} from "./registration.ts";
|
|
41
|
-
import type
|
|
60
|
+
import { NoticeLedger, type NoticeLedgerState } from "./notices.ts";
|
|
61
|
+
import { pendingTimers, startAndRearm, watchTimers, type PendingTimer } from "./timers.ts";
|
|
62
|
+
import {
|
|
63
|
+
DEFAULT_LEASE_INTERVAL_MS,
|
|
64
|
+
WORKSPACE_WAIT_KEY,
|
|
65
|
+
leaseDelay,
|
|
66
|
+
workspaceName,
|
|
67
|
+
type SandboxPort,
|
|
68
|
+
type WorkspaceWait,
|
|
69
|
+
} from "./workspace.ts";
|
|
70
|
+
import { INSTANCE_HARNESS_SERVICE } from "./names.ts";
|
|
71
|
+
import { composesSandbox } from "./parts.ts";
|
|
42
72
|
import { fingerprintOf } from "./fingerprint.ts";
|
|
43
|
-
import {
|
|
73
|
+
import {
|
|
74
|
+
serializeMachine,
|
|
75
|
+
type ChildMachineDoc,
|
|
76
|
+
type MachineBodyDoc,
|
|
77
|
+
type MachineDoc,
|
|
78
|
+
type MachineStateDoc,
|
|
79
|
+
} from "./machine-doc.ts";
|
|
44
80
|
import type { SnapshotStore } from "./snapshot-store.ts";
|
|
45
81
|
import type { AgentAdmission } from "./actor.ts";
|
|
82
|
+
import { dropSavedTails, type EffectRow } from "./effect.ts";
|
|
83
|
+
import type { EffectorPlacement, EffectorTarget } from "./http-effect.ts";
|
|
46
84
|
import { reattachAgentRuns, serializeSnapshot } from "./durability.ts";
|
|
85
|
+
import { admitChart, chartBriefingOf, chartHash, ChartRefusal, isChartSchema, type ChartText } from "./chart.ts";
|
|
86
|
+
import type { ChartOrigin, NamedChart } from "./charts.ts";
|
|
87
|
+
import {
|
|
88
|
+
admitBodyChart,
|
|
89
|
+
admitWrapperChart,
|
|
90
|
+
boundChartRoutesOf,
|
|
91
|
+
chartWrapperMachineOf,
|
|
92
|
+
boundMachineOf,
|
|
93
|
+
compileChart,
|
|
94
|
+
isOpenBody,
|
|
95
|
+
lentIn,
|
|
96
|
+
openBodiesOf,
|
|
97
|
+
openBodyDeclarationOf,
|
|
98
|
+
type BoundChartRoute,
|
|
99
|
+
} from "./open-body.ts";
|
|
47
100
|
|
|
48
101
|
/** The actors filled into a run's named slots — `.provide()` is the unit-test seam (ADR-0015);
|
|
49
102
|
* production instances inject nothing. Built fresh at start and at restore. */
|
|
@@ -56,8 +109,111 @@ export type WorkflowDef = {
|
|
|
56
109
|
machine: AnyStateMachine;
|
|
57
110
|
/** Build this run's live providers. A test seam (ADR-0015): discovery injects nothing. */
|
|
58
111
|
provide: (ctx: { instanceId: string }) => RunProviders;
|
|
112
|
+
/**
|
|
113
|
+
* The hash of the Assembly this Machine came from (ADR-0069): set at register for the current
|
|
114
|
+
* Assembly, and by `resolveDef` for a held one at restore. A run holds the Assembly it started
|
|
115
|
+
* under, so every save of the run stamps this hash. Absent only for a def that no Assembly
|
|
116
|
+
* loaded: an in-process test host, which stamps nothing.
|
|
117
|
+
*/
|
|
118
|
+
assembly?: string;
|
|
119
|
+
/**
|
|
120
|
+
* The def is a Chart wrapper (ADR-0070): `body` is its (first) Open body's slot key, and
|
|
121
|
+
* `chartFields` are the door's fields that are `chartSchema` — empty for a wrapper whose Charts
|
|
122
|
+
* only its own states supply. `register` reads both off the Machine when the caller leaves this
|
|
123
|
+
* out.
|
|
124
|
+
*/
|
|
125
|
+
wrapper?: { body: string; chartFields: string[] };
|
|
126
|
+
/**
|
|
127
|
+
* The def is a named Chart (ADR-0070): the Workflow `<wrapper>/<name>`, whose `machine` is its
|
|
128
|
+
* wrapper's. A start supplies every Chart field of the wrapper's door from `text` and takes the
|
|
129
|
+
* rest from the caller.
|
|
130
|
+
*/
|
|
131
|
+
chart?: { wrapper: string; name: string; text: ChartText; hash: string; origin: ChartOrigin };
|
|
59
132
|
};
|
|
60
133
|
|
|
134
|
+
/**
|
|
135
|
+
* One entry of `GET /workflows` (ADR-0070): a Workflow and where it came from. A Chart wrapper's
|
|
136
|
+
* entry carries its Open body, whether its door takes a Chart (`oneShot`), and its named Charts.
|
|
137
|
+
*/
|
|
138
|
+
export type WorkflowListing = {
|
|
139
|
+
name: string;
|
|
140
|
+
origin: "machine" | ChartOrigin;
|
|
141
|
+
wrapper?: {
|
|
142
|
+
body: string;
|
|
143
|
+
oneShot: boolean;
|
|
144
|
+
charts: Array<{ name: string; origin: ChartOrigin; unrunnable?: string }>;
|
|
145
|
+
};
|
|
146
|
+
unrunnable?: string;
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
/** What makes a Machine a Chart wrapper (ADR-0070), read off it: its first Open body's slot key and
|
|
150
|
+
* the door's fields that are `chartSchema`. The bodies are seen through jr2's transparent wrappers
|
|
151
|
+
* (a Chart wrapper that is a `workspace()`'s body), and the door is the registered Machine's.
|
|
152
|
+
* Undefined for a Machine with no Open body. */
|
|
153
|
+
function wrapperOf(machine: AnyStateMachine): WorkflowDef["wrapper"] | undefined {
|
|
154
|
+
const bodies = openBodiesOf(chartWrapperMachineOf(machine).machine);
|
|
155
|
+
if (!bodies.length) return undefined;
|
|
156
|
+
const shape = (inputSchemaOf(machine)?.shape ?? {}) as Record<string, unknown>;
|
|
157
|
+
return { body: bodies[0]!.slot, chartFields: Object.keys(shape).filter((k) => isChartSchema(shape[k])) };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** A pick one of whose parameters is a Chart (by schema identity, ADR-0070) is marked with that
|
|
161
|
+
* grammar's briefing, for the Turn part. */
|
|
162
|
+
function briefingOf(input: z.ZodObject): { briefing?: string } {
|
|
163
|
+
const shape = input.shape as Record<string, unknown>;
|
|
164
|
+
for (const key of Object.keys(shape)) {
|
|
165
|
+
const briefing = chartBriefingOf(shape[key]);
|
|
166
|
+
if (briefing !== undefined) return { briefing };
|
|
167
|
+
}
|
|
168
|
+
return {};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** The short wrapper name of a Workflow name: `<wrapper>` of `<wrapper>/<name>`, else the name. */
|
|
172
|
+
function wrapperName(workflow: string): string {
|
|
173
|
+
const at = workflow.indexOf("/");
|
|
174
|
+
return at > 0 ? workflow.slice(0, at) : workflow;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Why a run's bound Charts no longer read under `machine` (ADR-0070, D-B), or undefined when each
|
|
179
|
+
* does: every bound Chart is admitted again against the Open body slot it binds, resolved on the
|
|
180
|
+
* Machine through `implementations.actors` (a lent child Machine's own slots included), with the
|
|
181
|
+
* lent set that Machine has now. A text whose hash does not match is drift too: a hand-edited
|
|
182
|
+
* volume.
|
|
183
|
+
*/
|
|
184
|
+
function chartDrift(machine: AnyStateMachine, routes: readonly BoundChartRoute[]): string | undefined {
|
|
185
|
+
for (const route of routes) {
|
|
186
|
+
const where = `chart "${route.path.join("/")}" under body "${route.src}"`;
|
|
187
|
+
let at: AnyStateMachine = machine;
|
|
188
|
+
let resolved = true;
|
|
189
|
+
for (const src of route.srcs.slice(0, -1)) {
|
|
190
|
+
const logic = (at.implementations.actors as Record<string, unknown>)[src];
|
|
191
|
+
// The lent parts a bound body's Chart invokes are its wrapper's: the next hop is resolved there.
|
|
192
|
+
if (isOpenBody(logic)) continue;
|
|
193
|
+
const child = (logic as AnyStateMachine | undefined)?.root ? (logic as AnyStateMachine) : undefined;
|
|
194
|
+
if (!child) {
|
|
195
|
+
resolved = false;
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
at = child;
|
|
199
|
+
}
|
|
200
|
+
const logic = resolved ? (at.implementations.actors as Record<string, unknown>)[route.src] : undefined;
|
|
201
|
+
if (!isOpenBody(logic)) return `${where}: the Machine has no Open body "${route.src}" now`;
|
|
202
|
+
const hash = chartHash(route.chart);
|
|
203
|
+
if (hash !== route.chartHash) {
|
|
204
|
+
return `${where}: its hash ${route.chartHash} does not match its text (${hash}); the stored run was edited`;
|
|
205
|
+
}
|
|
206
|
+
const decl = openBodyDeclarationOf(logic);
|
|
207
|
+
try {
|
|
208
|
+
admitBodyChart(route.chart, decl, lentIn(at.implementations as never, decl));
|
|
209
|
+
} catch (err) {
|
|
210
|
+
if (!(err instanceof ChartRefusal)) throw err;
|
|
211
|
+
return `${where}: ${err.report.join("; ")}`;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return undefined;
|
|
215
|
+
}
|
|
216
|
+
|
|
61
217
|
/** The serializable identity of a run — what reconcile sees and what restore rebuilds from. */
|
|
62
218
|
export type RunRecord = { runId: string; workflow: string; instanceId: string };
|
|
63
219
|
|
|
@@ -101,6 +257,191 @@ function runChildren(snapshot: unknown): RunChild[] {
|
|
|
101
257
|
return out;
|
|
102
258
|
}
|
|
103
259
|
|
|
260
|
+
/**
|
|
261
|
+
* One Workspace of a run waiting for capacity (ADR-0064): its Sandbox has no node. `child` is the
|
|
262
|
+
* Workspace's actor path below the run root — the {@link RunChild} ids down to it, joined by `/`,
|
|
263
|
+
* and `""` when the run's root IS the Workspace. `on` is what it waits for: a node from the
|
|
264
|
+
* scheduler, or room under the cluster owner's quota; `message` is the scheduler's or the API
|
|
265
|
+
* server's own words; `since` (ISO) is when the wait began, kept across a changed reason and a
|
|
266
|
+
* restart.
|
|
267
|
+
*/
|
|
268
|
+
export type RunWaiting = { child: string; on: "node" | "quota"; message: string; since: string };
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Every Workspace in `placing` with a reason, at any depth, read off the snapshot tree in the two
|
|
272
|
+
* shapes {@link runChildren} walks — the wait rides each Workspace's own persisted context, so a
|
|
273
|
+
* restored run reports it before its watch has said a word. A live child is held to being a
|
|
274
|
+
* `workspace()` by its logic; a persisted one has none, and the state key and the context's shape
|
|
275
|
+
* are the test. Only an ACTIVE run and an active Workspace wait: a run that ended — faulted,
|
|
276
|
+
* cancelled, done — has deleted what never ran, so a `placing` left in its snapshot waits for nothing.
|
|
277
|
+
*/
|
|
278
|
+
function runWaiting(snapshot: unknown, logic?: unknown, path: string[] = []): RunWaiting[] {
|
|
279
|
+
const out: RunWaiting[] = [];
|
|
280
|
+
const snap = snapshot as { status?: unknown; value?: unknown; context?: Record<string, unknown> } | undefined;
|
|
281
|
+
if (snap?.status !== undefined && snap.status !== "active") return out;
|
|
282
|
+
const wait = snap?.context?.[WORKSPACE_WAIT_KEY] as WorkspaceWait | undefined;
|
|
283
|
+
const isWorkspace = logic == null || composesSandbox(logic as AnyStateMachine);
|
|
284
|
+
if (isWorkspace && snap?.value === "placing" && (wait?.on === "node" || wait?.on === "quota")) {
|
|
285
|
+
out.push({ child: path.join("/"), on: wait.on, message: String(wait.message), since: String(wait.since) });
|
|
286
|
+
}
|
|
287
|
+
const children = (snapshot as { children?: Record<string, unknown> } | undefined)?.children ?? {};
|
|
288
|
+
for (const [id, entry] of Object.entries(children)) {
|
|
289
|
+
const child = entry as { snapshot?: unknown; getSnapshot?: () => unknown; logic?: unknown };
|
|
290
|
+
const live = typeof child.getSnapshot === "function";
|
|
291
|
+
const childSnap = live ? child.getSnapshot!() : child.snapshot;
|
|
292
|
+
if ((childSnap as { value?: unknown } | undefined)?.value === undefined) continue;
|
|
293
|
+
out.push(...runWaiting(childSnap, live ? child.logic : undefined, [...path, id]));
|
|
294
|
+
}
|
|
295
|
+
return out;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* The Sandbox of every live Workspace under `actor`, at any depth, by CR name — what a cancel
|
|
300
|
+
* destroys (ADR-0025). Read off the run's own live actor tree: each Workspace holds its `wsId` in
|
|
301
|
+
* context, and the name is a function of the run and that id, the same one its `teardown` deletes.
|
|
302
|
+
* Correct after a restart by construction: `restore()` rebuilds this tree from the persisted
|
|
303
|
+
* snapshot, so a restored run names its Sandboxes before its watch has listed. The Sandbox watch's
|
|
304
|
+
* set, filtered by the run's label, is not: before the watch has listed it holds nothing, and
|
|
305
|
+
* unknown is never absent (ADR-0063).
|
|
306
|
+
*
|
|
307
|
+
* Left out: a Workspace in `placing`, which deletes its own Sandbox when its wait is aborted — and
|
|
308
|
+
* only once any write in flight has settled, so a delete from here could race the apply and leave
|
|
309
|
+
* the CR behind (ADR-0064); and one that is `done` or ended, whose `teardown` already ran.
|
|
310
|
+
*/
|
|
311
|
+
function runSandboxes(actor: AnyActorRef, runId: string): string[] {
|
|
312
|
+
const out: string[] = [];
|
|
313
|
+
const snap = actor.getSnapshot() as
|
|
314
|
+
| { status?: string; value?: unknown; context?: { wsId?: unknown }; children?: Record<string, AnyActorRef> }
|
|
315
|
+
| undefined;
|
|
316
|
+
if (snap?.status !== "active") return out;
|
|
317
|
+
const logic = (actor as AnyActorRef & { logic?: unknown }).logic;
|
|
318
|
+
if (
|
|
319
|
+
logic != null &&
|
|
320
|
+
composesSandbox(logic as AnyStateMachine) &&
|
|
321
|
+
snap.value !== "placing" &&
|
|
322
|
+
typeof snap.context?.wsId === "string"
|
|
323
|
+
) {
|
|
324
|
+
out.push(workspaceName(runId, snap.context.wsId));
|
|
325
|
+
}
|
|
326
|
+
for (const child of Object.values(snap.children ?? {})) out.push(...runSandboxes(child, runId));
|
|
327
|
+
return out;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Stop every actor under `actor`, at any depth, deepest first. A machine that ended in `error` has
|
|
332
|
+
* stopped processing — its own stop is a no-op — so its children are reached through its snapshot
|
|
333
|
+
* rather than asked to stop through it.
|
|
334
|
+
*/
|
|
335
|
+
function stopTree(actor: AnyActorRef): void {
|
|
336
|
+
const children = (actor.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
|
|
337
|
+
for (const child of Object.values(children)) {
|
|
338
|
+
stopTree(child);
|
|
339
|
+
// `_stop` is what xstate's own `stopChild` calls: a child cannot be stopped through `stop()`.
|
|
340
|
+
(child as AnyActorRef & { _stop?: () => void })._stop?.();
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** One outstanding pick, as `RunStatus.held` reports it: the Agent slot, its pick, and the ISO time
|
|
345
|
+
* it was delivered. */
|
|
346
|
+
export type RunHeld = { agent: string; pick: string; since: string };
|
|
347
|
+
|
|
348
|
+
/** The pick being delivered now (ADR-0024): its registration, every reply from below its Agent's
|
|
349
|
+
* state inside its own send, and what an `answer` action set there (ADR-0067). */
|
|
350
|
+
type Sending = {
|
|
351
|
+
reg: Registration;
|
|
352
|
+
pick: string;
|
|
353
|
+
chain: PickAnswer[];
|
|
354
|
+
shaped?: { value: unknown };
|
|
355
|
+
};
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Whose pick started a child (ADR-0067): the Agent's slot, and the invoke id of the Agent's state the
|
|
359
|
+
* pick was made at, which tells one slot's two regions apart. `invoke` is absent when the event that
|
|
360
|
+
* started the child named the Agent and not where (a pick taken outside its own send).
|
|
361
|
+
*/
|
|
362
|
+
type StartedBy = { agent: string; invoke?: string };
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* The key of the child `childId` of `parent` in the run's record of which Agent started each child
|
|
366
|
+
* (`startedBy`, ADR-0067): the invoke ids from below the root down to `parent`, then `childId`, each
|
|
367
|
+
* encoded and joined by `/`. An invoke id is local to its Machine — two Machines of one tree, and
|
|
368
|
+
* every worker of a pool, may each have a child `job` — and the chain is not. A child of the root
|
|
369
|
+
* keeps its bare id. The ids are the snapshot's, so a restore gives the same key.
|
|
370
|
+
*/
|
|
371
|
+
function childKey(parent: AnyActorRef, childId: string): string {
|
|
372
|
+
const ids = [childId];
|
|
373
|
+
for (let at: AnyActorRef | undefined = parent; at?._parent !== undefined; at = at._parent) ids.unshift(at.id);
|
|
374
|
+
return ids.map(encodeURIComponent).join("/");
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* The Agent whose event `event` is, as `taker` takes it (ADR-0024, ADR-0067): a pick's sender, or
|
|
379
|
+
* the Agent whose pick started the child of `taker` whose done or error event it is (`startedBy`).
|
|
380
|
+
* Undefined when the event names no Agent: a timer, a Gate's answer, an outside send, a child no
|
|
381
|
+
* Agent's pick started. `sending`'s invoke is where a pick in its own send was made.
|
|
382
|
+
*/
|
|
383
|
+
function agentOfEvent(
|
|
384
|
+
event: AnyEventObject,
|
|
385
|
+
startedBy: Record<string, StartedBy>,
|
|
386
|
+
taker: AnyActorRef,
|
|
387
|
+
sending?: Sending,
|
|
388
|
+
): StartedBy | undefined {
|
|
389
|
+
const sender = (event as { [SENDER_KEY]?: Sender })[SENDER_KEY];
|
|
390
|
+
if (sender?.party === "agent") {
|
|
391
|
+
const invoke = sending?.reg.sender?.id === sender.id ? sending.reg.invokeId : undefined;
|
|
392
|
+
return invoke !== undefined ? { agent: sender.id, invoke } : { agent: sender.id };
|
|
393
|
+
}
|
|
394
|
+
const child = /^xstate\.(?:done|error)\.actor\.(.+)$/.exec(event.type)?.[1];
|
|
395
|
+
if (child === undefined) return undefined;
|
|
396
|
+
const key = childKey(taker, child);
|
|
397
|
+
return Object.hasOwn(startedBy, key) ? startedBy[key] : undefined;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** How many of `snapshot`'s active states invoke an Agent. With two or more, an event that names no
|
|
401
|
+
* Agent cannot say whose pick an `answer` below no Agent's state shapes (ADR-0024). */
|
|
402
|
+
function activeAgentStates(snapshot: AnyMachineSnapshot): number {
|
|
403
|
+
const machine = snapshot.machine as AnyStateMachine;
|
|
404
|
+
const active = (snapshot as { _nodes?: AnyStateMachine["root"][] })._nodes ?? [];
|
|
405
|
+
return active.filter((node) => partiesAt(node, machine).some((party) => party.party === "agent")).length;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** The `childKey` of every child in `actor`'s tree, at any depth. */
|
|
409
|
+
function liveChildKeys(actor: AnyActorRef, into = new Set<string>()): Set<string> {
|
|
410
|
+
const children = (actor.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
|
|
411
|
+
for (const child of Object.values(children)) {
|
|
412
|
+
if (!child) continue;
|
|
413
|
+
into.add(childKey(actor, child.id));
|
|
414
|
+
liveChildKeys(child, into);
|
|
415
|
+
}
|
|
416
|
+
return into;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/** `startedBy` as the save keeps it: only children still in the run's tree (ADR-0067). Read at the
|
|
420
|
+
* save, a microtask after the snapshot, so a done event the Machine took has run its route. */
|
|
421
|
+
function liveStartedBy(startedBy: Record<string, StartedBy>, actor: AnyActorRef): Record<string, StartedBy> {
|
|
422
|
+
const live = liveChildKeys(actor);
|
|
423
|
+
for (const id of Object.keys(startedBy)) if (!live.has(id)) delete startedBy[id];
|
|
424
|
+
return startedBy;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** An `answer` action's value as the hold keeps it: JSON, because it rides the run blob and the
|
|
428
|
+
* wire (ADR-0067). `undefined`, or a value with no JSON form, sets no answer — the default stands. */
|
|
429
|
+
function jsonOf(value: unknown): { value: unknown } | undefined {
|
|
430
|
+
try {
|
|
431
|
+
const text = JSON.stringify(value);
|
|
432
|
+
return text === undefined ? undefined : { value: JSON.parse(text) as unknown };
|
|
433
|
+
} catch {
|
|
434
|
+
return undefined;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** The outstanding picks among a run's holds. */
|
|
439
|
+
function heldOf(holds: Iterable<Hold>): RunHeld[] {
|
|
440
|
+
return [...holds]
|
|
441
|
+
.filter((hold) => hold.answer === undefined)
|
|
442
|
+
.map((hold) => ({ agent: hold.agent, pick: hold.pick, since: hold.since }));
|
|
443
|
+
}
|
|
444
|
+
|
|
104
445
|
/** A run's current observable state. `fault` carries the error message when status is "error"
|
|
105
446
|
* (e.g. a gate invoked with a name outside the workflow's manifest — ADR-0011). */
|
|
106
447
|
export type RunStatus = RunRecord & {
|
|
@@ -109,6 +450,14 @@ export type RunStatus = RunRecord & {
|
|
|
109
450
|
context: unknown;
|
|
110
451
|
/** The live child machines beneath the root — where most of a run actually is. */
|
|
111
452
|
children: RunChild[];
|
|
453
|
+
/** Every Workspace waiting for capacity, and why (ADR-0064). Behind the token: the reason is
|
|
454
|
+
* the scheduler's or the quota's own words, which can name nodes and taints, so {@link observe}
|
|
455
|
+
* never carries it — the open band sees only that a Workspace is in `placing`. */
|
|
456
|
+
waiting: RunWaiting[];
|
|
457
|
+
/** Every pick of this run that is outstanding (ADR-0024): the Agent, its pick, and since when —
|
|
458
|
+
* the Console's "held since". There is no default time bound on a held Turn; this is how a
|
|
459
|
+
* human sees one. Behind the token, like `context`: a pick names what the Agent did. */
|
|
460
|
+
held: RunHeld[];
|
|
112
461
|
fault?: string;
|
|
113
462
|
/**
|
|
114
463
|
* Why the HOST set this status, for the statuses the Machine did not choose — today `drifted`
|
|
@@ -117,6 +466,15 @@ export type RunStatus = RunRecord & {
|
|
|
117
466
|
* route could read it.
|
|
118
467
|
*/
|
|
119
468
|
reason?: string;
|
|
469
|
+
/** The hash of the Assembly the run started under (ADR-0069). Absent for a run no Assembly
|
|
470
|
+
* started. */
|
|
471
|
+
assembly?: string;
|
|
472
|
+
/** The run binds at least one Chart (ADR-0070): its Machine is the bound one, `runMachine`. */
|
|
473
|
+
bound?: true;
|
|
474
|
+
/** The hashes of the Charts the run binds now, distinct and sorted (ADR-0070). A run can bind a
|
|
475
|
+
* second Chart in its life (a planner whose first Chart reaches a ceiling picks again), so these,
|
|
476
|
+
* not the run id, say which bound Machine `runMachine` draws. Absent when it binds none. */
|
|
477
|
+
chartHashes?: string[];
|
|
120
478
|
};
|
|
121
479
|
|
|
122
480
|
/**
|
|
@@ -142,6 +500,12 @@ export type RunObservation = {
|
|
|
142
500
|
status: string;
|
|
143
501
|
value: unknown;
|
|
144
502
|
children: RunChild[];
|
|
503
|
+
/** The run binds a Chart (ADR-0070): the Console draws `GET /workflows/:name/runs/:runId/machine`.
|
|
504
|
+
* Structure, like `value`. */
|
|
505
|
+
bound?: true;
|
|
506
|
+
/** The hashes of the Charts it binds now (see {@link RunStatus.chartHashes}). Structure: a hash
|
|
507
|
+
* of a Chart's text, which `runMachine` already serves on the open band. */
|
|
508
|
+
chartHashes?: string[];
|
|
145
509
|
};
|
|
146
510
|
|
|
147
511
|
/** Project a full status down to what an observer may see. The one place the line is drawn. */
|
|
@@ -152,9 +516,31 @@ export function observe(status: RunStatus): RunObservation {
|
|
|
152
516
|
status: status.status,
|
|
153
517
|
value: status.value,
|
|
154
518
|
children: status.children,
|
|
519
|
+
...(status.bound ? { bound: true as const } : {}),
|
|
520
|
+
...(status.chartHashes ? { chartHashes: status.chartHashes } : {}),
|
|
155
521
|
};
|
|
156
522
|
}
|
|
157
523
|
|
|
524
|
+
/** An Agent registration's Menu (ADR-0029): the one it registered, or — registered with none, by a
|
|
525
|
+
* plain `setup()` machine — its derived set. */
|
|
526
|
+
function menuOf(reg: Registration): Map<string, EventDef> {
|
|
527
|
+
return reg.menu ?? reg.defs;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/** The invoking Machine's state value, as one comparable string; undefined when it cannot be read. */
|
|
531
|
+
function stateValueOf(invoker: AnyActorRef | undefined): string | undefined {
|
|
532
|
+
try {
|
|
533
|
+
return invoker ? JSON.stringify(invoker.getSnapshot().value) : undefined;
|
|
534
|
+
} catch {
|
|
535
|
+
return undefined;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** The Allowed picks (ADR-0029), as the Agent's nudge and wake read them too. */
|
|
540
|
+
function allowedOf(reg: Registration): string[] {
|
|
541
|
+
return allowedPicks(reg);
|
|
542
|
+
}
|
|
543
|
+
|
|
158
544
|
/** One open gate as external callers discover it (`GET /runs/:id` — ADR-0011): the accepted
|
|
159
545
|
* events with their input schemas as JSON Schema (what drives a form or a `jr2 send` prompt),
|
|
160
546
|
* plus the workflow-supplied `meta` (what a UI renders and a webhook translator matches on). */
|
|
@@ -165,28 +551,50 @@ export type GateView = {
|
|
|
165
551
|
* id was authored ("F-12") or derived. Never parse `gate` for this. */
|
|
166
552
|
path: string[];
|
|
167
553
|
accepts: Array<{ name: string; description?: string; input: unknown }>;
|
|
554
|
+
/** The names of `accepts` the run would take from this Gate NOW, payload-blind (ADR-0011,
|
|
555
|
+
* ADR-0029): a send of another name is refused 409. `accepts` stays whole — it drives the forms. */
|
|
556
|
+
acceptsNow: string[];
|
|
168
557
|
meta?: Record<string, unknown>;
|
|
169
558
|
};
|
|
170
559
|
|
|
560
|
+
/** One event on an Agent's Menu, as the surface serves it: a tool, its `input` schema the tool's
|
|
561
|
+
* parameters. `semantics` rides along so the Harness refuses a reserved one (ADR-0013). */
|
|
562
|
+
/** One pick on the surface. `briefing` marks a pick whose parameter is a Chart: jr2's text on how a
|
|
563
|
+
* Chart runs (`CHART_BRIEFING`, ADR-0070), which the Harness says in the Turn part when the pick is
|
|
564
|
+
* Allowed, so only the Turns that may write a Chart carry it. */
|
|
565
|
+
export type SurfaceEvent = {
|
|
566
|
+
name: string;
|
|
567
|
+
description?: string;
|
|
568
|
+
input: unknown;
|
|
569
|
+
semantics: EventSemantics;
|
|
570
|
+
briefing?: string;
|
|
571
|
+
};
|
|
572
|
+
|
|
171
573
|
/**
|
|
172
|
-
* One live agent surface, as
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
574
|
+
* One live agent surface, as the Harness reads it through its Custodian (`GET /agents/:iid/surface`
|
|
575
|
+
* — ADR-0013, ADR-0029). Two answers, because they change at different rates:
|
|
576
|
+
*
|
|
577
|
+
* - `menu` is the Agent's MENU — every pick any state invoking this Agent offers, in a fixed
|
|
578
|
+
* order. The Harness presents it as the tools block, which is then byte-identical for every
|
|
579
|
+
* Turn of the conversation and never breaks the provider's prompt cache.
|
|
580
|
+
* - `allowed` is this Turn's ALLOWED PICKS — the invoking state's derived set, narrowed to what its
|
|
581
|
+
* guards would accept now, in Menu order. The Harness states it in the Briefing's Turn part,
|
|
582
|
+
* after the Frame's prompt (ADR-0062). Enforced at delivery, not here.
|
|
176
583
|
*
|
|
177
|
-
* `sandbox` is the Sandbox that may deliver here. It is not a secret from
|
|
178
|
-
*
|
|
584
|
+
* `sandbox` is the Sandbox that may deliver here. It is not a secret from that pod (the pod IS the
|
|
585
|
+
* sandbox), and serving it lets a caller fail loudly on a surface that is not its own.
|
|
179
586
|
*/
|
|
180
587
|
export type AgentSurfaceView = {
|
|
181
588
|
instanceId: string;
|
|
182
589
|
runId: string;
|
|
183
590
|
sandbox?: string;
|
|
184
|
-
|
|
591
|
+
menu: SurfaceEvent[];
|
|
592
|
+
allowed: string[];
|
|
185
593
|
};
|
|
186
594
|
|
|
187
595
|
/**
|
|
188
596
|
* The answer to one Agent delivery (`POST /agents/:iid/events`) — a receipt that DESCRIBES ITSELF
|
|
189
|
-
* (ADR-0024). The
|
|
597
|
+
* (ADR-0024). The Harness renders it as prose, because a bare `deliveryId` told the Agent nothing
|
|
190
598
|
* about whether it was finished, and the model answered that silence by calling again.
|
|
191
599
|
*
|
|
192
600
|
* `turnComplete` is the HINT, never the guarantee (the abort on invocation end is): it is read off
|
|
@@ -194,10 +602,30 @@ export type AgentSurfaceView = {
|
|
|
194
602
|
* turn has stopped waiting. It fails conservatively — anything that made the read unreliable reads
|
|
195
603
|
* `false`, which is today's behavior, never a false claim.
|
|
196
604
|
*
|
|
197
|
-
* `deliveryId`
|
|
198
|
-
*
|
|
605
|
+
* `deliveryId` keeps the outcome ADDRESSABLE after the fact (ADR-0013): a pick the workflow is still
|
|
606
|
+
* running when the POST answers is OUTSTANDING (`outstanding: true`), and its answer is read at
|
|
607
|
+
* `GET /agents/:iid/deliveries/:id` — the same shape, `answered: true`, once the workflow is back at
|
|
608
|
+
* Rest in the Agent's state or has left it (ADR-0024).
|
|
199
609
|
*/
|
|
200
610
|
export type AgentDeliveryReceipt = {
|
|
611
|
+
/** The workflow is running the pick: the answer comes on the delivery's poll. `allowed` is empty:
|
|
612
|
+
* no other pick is taken until it answers. */
|
|
613
|
+
outstanding?: true;
|
|
614
|
+
/** ISO time the pick was delivered — with `outstanding`. */
|
|
615
|
+
since?: string;
|
|
616
|
+
/** This receipt is a pick's answer, read at Rest (or once the Turn left the Agent's state): an outstanding
|
|
617
|
+
* pick's, a reply inside the pick's own send, or a guard straight back with no reply. */
|
|
618
|
+
answered?: true;
|
|
619
|
+
/** With `answered`: a guard straight back (ADR-0024) — the pick entered a state below the Agent's
|
|
620
|
+
* and came back inside its own send, nothing replying. The one answer that says "moved; nothing
|
|
621
|
+
* ran"; an outstanding pick's answer never carries it. */
|
|
622
|
+
straightBack?: true;
|
|
623
|
+
/** The last reply since the pick (ADR-0024); absent with `answered`: no reply came. */
|
|
624
|
+
answer?: PickAnswer;
|
|
625
|
+
/** With `answered` and `turnComplete`: something other than the pick's route left the Agent's
|
|
626
|
+
* state while the pick's work ran, and that exit stopped the work before it finished. `answer`,
|
|
627
|
+
* if any, is an earlier link's reply, kept (ADR-0024). */
|
|
628
|
+
stopped?: true;
|
|
201
629
|
delivered: true;
|
|
202
630
|
/** The event delivered — the Agent reads its own pick back, by name. */
|
|
203
631
|
event: string;
|
|
@@ -212,6 +640,12 @@ export type AgentDeliveryReceipt = {
|
|
|
212
640
|
moved: boolean;
|
|
213
641
|
/** The invoking state stopped waiting: this Agent's turn is over. */
|
|
214
642
|
turnComplete: boolean;
|
|
643
|
+
/**
|
|
644
|
+
* The Allowed picks read AFTER the delivery (ADR-0029): what this Turn may pick now — narrowed by
|
|
645
|
+
* a pick that moved the Machine within the invoking state, empty once the Turn is over. The
|
|
646
|
+
* Harness renders it ("Allowed now: `finish`"); it does not pre-check against it.
|
|
647
|
+
*/
|
|
648
|
+
allowed: string[];
|
|
215
649
|
deliveryId: string;
|
|
216
650
|
};
|
|
217
651
|
|
|
@@ -226,7 +660,7 @@ export type AgentDeliveryReceipt = {
|
|
|
226
660
|
export type RunFeedEvent =
|
|
227
661
|
| { kind: "status"; status: RunStatus }
|
|
228
662
|
| { kind: "emit"; event: { type: string } & Record<string, unknown> }
|
|
229
|
-
|
|
|
663
|
+
| RunMarker
|
|
230
664
|
// Absorbed-retry telemetry (ADR-0016): `{ child, attempt }` is state-key-class data — no iids
|
|
231
665
|
// ride the feed (ADR-0014). `reason` is mechanism text, guarded like `fault`.
|
|
232
666
|
| RetryTelemetry
|
|
@@ -234,9 +668,10 @@ export type RunFeedEvent =
|
|
|
234
668
|
|
|
235
669
|
/**
|
|
236
670
|
* The feed-so-far cap. The buffer exists so a Workspace attaching mid-run can open its log with
|
|
237
|
-
* the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST
|
|
238
|
-
* is the same trade kubelet's log rotation already makes on the printed side — the
|
|
239
|
-
* the record, the buffer serves a courtesy view.
|
|
671
|
+
* the run's preamble (ADR-0023); a run long enough to blow past it loses its OLDEST markers and
|
|
672
|
+
* Emits, which is the same trade kubelet's log rotation already makes on the printed side — the
|
|
673
|
+
* feed remains the record, the buffer serves a courtesy view. Statuses do not count against it:
|
|
674
|
+
* the buffer keeps only the latest (a status is a level — ADR-0023).
|
|
240
675
|
*/
|
|
241
676
|
const FEED_SO_FAR_CAP = 1000;
|
|
242
677
|
|
|
@@ -264,6 +699,51 @@ function echoEventOf(event: RunFeedEvent, target: string): EchoEvent | undefined
|
|
|
264
699
|
return undefined;
|
|
265
700
|
}
|
|
266
701
|
|
|
702
|
+
/**
|
|
703
|
+
* The echo's outbox (ADR-0023): one push in flight per attachment, and whatever the feed produces
|
|
704
|
+
* meanwhile rides the NEXT push, together — a burst of transitions costs the Harness one push per
|
|
705
|
+
* flush, not one per transition.
|
|
706
|
+
*
|
|
707
|
+
* A status is a level, so a pending status is REPLACED by a newer one, never queued behind it. The
|
|
708
|
+
* ordering rule, exactly: markers and Emits keep feed order among themselves, and the one pending
|
|
709
|
+
* status sits where the NEWEST status arrived — after every marker and Emit fed before it, before
|
|
710
|
+
* every one fed after it. A superseded status drops out; nothing else moves. So a Harness that
|
|
711
|
+
* answers slowly holds at most one status plus the markers since its last push.
|
|
712
|
+
*
|
|
713
|
+
* Fire-and-forget: a failed push goes to `report` and the outbox drains on — nothing here can
|
|
714
|
+
* fail a turn, a state, or a run. Exported for its unit test; {@link RunHost} is its one caller.
|
|
715
|
+
*/
|
|
716
|
+
export function echoOutbox(
|
|
717
|
+
push: (events: EchoEvent[]) => Promise<void>,
|
|
718
|
+
report: (err: unknown) => void,
|
|
719
|
+
): (events: EchoEvent[]) => void {
|
|
720
|
+
let pending: EchoEvent[] = [];
|
|
721
|
+
let flushing = false;
|
|
722
|
+
const flush = async (): Promise<void> => {
|
|
723
|
+
while (pending.length > 0) {
|
|
724
|
+
const batch = pending;
|
|
725
|
+
pending = [];
|
|
726
|
+
try {
|
|
727
|
+
await push(batch);
|
|
728
|
+
} catch (err) {
|
|
729
|
+
report(err);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
flushing = false;
|
|
733
|
+
};
|
|
734
|
+
return (events) => {
|
|
735
|
+
for (const event of events) {
|
|
736
|
+
if (event.kind === "status") pending = pending.filter((p) => p.kind !== "status");
|
|
737
|
+
pending.push(event);
|
|
738
|
+
}
|
|
739
|
+
if (flushing || pending.length === 0) return;
|
|
740
|
+
flushing = true;
|
|
741
|
+
// A microtask, so a synchronous burst (the replay; one transition's Emits and status) leaves
|
|
742
|
+
// as one push rather than as its first event alone.
|
|
743
|
+
queueMicrotask(() => void flush());
|
|
744
|
+
};
|
|
745
|
+
}
|
|
746
|
+
|
|
267
747
|
/** The child-machine tree for the echo's status: spawn ids and state values alone — a root's
|
|
268
748
|
* value says almost nothing about where a run is, and a {@link RunChild} is already context-free
|
|
269
749
|
* by construction. `src`/`status` stay behind: they are join keys for the Console, not story. */
|
|
@@ -304,6 +784,19 @@ export type RunHostOptions = {
|
|
|
304
784
|
reconcile?: (run: RunRecord) => boolean | Promise<boolean>;
|
|
305
785
|
/** Injectable id generator (deterministic ids in tests). Default: `crypto.randomUUID`. */
|
|
306
786
|
newId?: () => string;
|
|
787
|
+
/**
|
|
788
|
+
* Resolve a stamped blob's Workflow inside the Assembly it names (ADR-0069): the Machine that
|
|
789
|
+
* Assembly's `workflows/<name>.ts` exports, as a def stamped with that hash, or why it cannot be
|
|
790
|
+
* read (`drifted`: not on the volume, published for another Kit version, does not import). The
|
|
791
|
+
* host never learns the volume layout; `startInstance` supplies this over `assemblies/<hash>`.
|
|
792
|
+
* A def it returns is NOT registered: the registry is the current Assembly's alone. A throw is a
|
|
793
|
+
* fault, not a verdict: the row stays live for the next boot (`failed`).
|
|
794
|
+
*
|
|
795
|
+
* Absent (an in-process host): a stamped blob resolves to the registered def of its name only
|
|
796
|
+
* when that def came from the same Assembly hash, since one hash names one tree and so one
|
|
797
|
+
* Machine; otherwise the run is `drifted`.
|
|
798
|
+
*/
|
|
799
|
+
resolveDef?: (blob: { workflow: string; assembly: string }) => Promise<WorkflowDef | { drifted: string }>;
|
|
307
800
|
/** Report a run that threw during restore (ADR-0030). The row is left `live` for the next boot,
|
|
308
801
|
* so this is the only account of why — the entrypoint logs it. Default: silent. */
|
|
309
802
|
onRestoreError?: (runId: string, err: unknown) => void;
|
|
@@ -316,6 +809,9 @@ export type RunHostOptions = {
|
|
|
316
809
|
* signing key. Handed to every run's binding, and to the echo pusher below. Absent, no Harness
|
|
317
810
|
* call bears one: only the stub Harness answers such a host. */
|
|
318
811
|
harnessBearer?: (placement: string) => string;
|
|
812
|
+
/** Where an Effector placement is, and its key (ADR-0068) — `startInstance` derives both, the key
|
|
813
|
+
* from the signing key. Handed to every run's binding; absent, an http Effect faults pointedly. */
|
|
814
|
+
effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
|
|
319
815
|
/**
|
|
320
816
|
* Build the run-narrative echo pusher for one Workspace's Harness (ADR-0023) — the seam a fake
|
|
321
817
|
* echo server rides in tests; `startInstance` binds the real wire push (harness-client.ts).
|
|
@@ -324,10 +820,22 @@ export type RunHostOptions = {
|
|
|
324
820
|
* never a dependency of the run.
|
|
325
821
|
*/
|
|
326
822
|
echo?: (endpoint: string, bearer: string | undefined) => (events: EchoEvent[]) => Promise<void>;
|
|
823
|
+
/**
|
|
824
|
+
* Build the live-set statement to the Instance Harness (ADR-0031) — the seam a fake rides in
|
|
825
|
+
* tests; `startInstance` binds the real wire push (harness-client.ts). `bearer` is the Instance
|
|
826
|
+
* Harness's (ADR-0058). Absent, or with no `instanceHarness`, nothing is stated.
|
|
827
|
+
*/
|
|
828
|
+
liveSet?: (endpoint: string, bearer: string | undefined) => (live: LiveConversation[]) => Promise<unknown>;
|
|
829
|
+
/** How often the live set is stated, before jitter. Default: the Lease's cadence (ADR-0021).
|
|
830
|
+
* Tests shrink it. */
|
|
831
|
+
liveSetIntervalMs?: number;
|
|
832
|
+
/** The wall clock a run's timer table is written and read on (ADR-0007, timers.ts): each pending
|
|
833
|
+
* timer's deadline is a time on it. Default: `Date.now`. Tests pass a clock that does not step. */
|
|
834
|
+
now?: () => number;
|
|
327
835
|
};
|
|
328
836
|
|
|
329
837
|
/** What we persist per run: the machine snapshot wrapped with the run metadata restore needs.
|
|
330
|
-
* `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent
|
|
838
|
+
* `agents` is the admission LEDGER (ADR-0016) — iid → durable admission, reported by the Agent's Actor
|
|
331
839
|
* through the run binding and saved in the same blob (same store, same atomicity). */
|
|
332
840
|
type RunBlob = {
|
|
333
841
|
workflow: string;
|
|
@@ -337,6 +845,10 @@ type RunBlob = {
|
|
|
337
845
|
* Absent means written before the stamp existed, which restore treats as drift — the point is to
|
|
338
846
|
* never interpret a snapshot whose Machine cannot be vouched for. */
|
|
339
847
|
machine?: string;
|
|
848
|
+
/** The hash of the Assembly the run started under (ADR-0069), stamped on every save. A blob
|
|
849
|
+
* without one cannot be vouched for, so restore treats it as drift, as it treats a blob with no
|
|
850
|
+
* `machine` stamp. */
|
|
851
|
+
assembly?: string;
|
|
340
852
|
snapshot: unknown;
|
|
341
853
|
agents?: Record<string, AgentAdmission>;
|
|
342
854
|
/**
|
|
@@ -352,6 +864,47 @@ type RunBlob = {
|
|
|
352
864
|
* `<id>/<epoch>`), so one conversation has exactly one counter however often it burns.
|
|
353
865
|
*/
|
|
354
866
|
epochs?: Record<string, number>;
|
|
867
|
+
/**
|
|
868
|
+
* The pending NOTICES (ADR-0062): what the next Turn in a scope must be told — a memory kill in
|
|
869
|
+
* its Workspace, a conversation that is new. The Orchestrator is their only keeper, so they ride
|
|
870
|
+
* this blob beside the two ledgers, for the same reasons: the run's exact lifetime, and the
|
|
871
|
+
* snapshot's atomicity. A notice leaves it in the save that ledgers the admission that carried
|
|
872
|
+
* it, so a restart before that save delivers it again and a restart after does not.
|
|
873
|
+
*/
|
|
874
|
+
notices?: NoticeLedgerState;
|
|
875
|
+
/**
|
|
876
|
+
* The pending TIMERS (`after`, named delays, delayed `raise` and `sendTo`), each with its
|
|
877
|
+
* absolute deadline (ADR-0007, timers.ts). xstate 5 keeps them on the actor system, not in the
|
|
878
|
+
* snapshot, so a restore without this table loses every one. Same blob for the snapshot's
|
|
879
|
+
* atomicity: the timers saved are exactly the ones the saved states scheduled.
|
|
880
|
+
*/
|
|
881
|
+
timers?: PendingTimer[];
|
|
882
|
+
/**
|
|
883
|
+
* The outstanding picks (ADR-0024, holds.ts): each conversation's latest pick that the workflow ran, its
|
|
884
|
+
* delivery id, the last reply since it, and its answer once read at Rest. Same blob for the
|
|
885
|
+
* snapshot's atomicity: a restored host answers the same poll from the same Machine.
|
|
886
|
+
*/
|
|
887
|
+
holds?: Hold[];
|
|
888
|
+
/**
|
|
889
|
+
* The Effect LEDGER (ADR-0067): one row per Effect, by its id — `intent` before the call,
|
|
890
|
+
* `settled` with its output, `aborted` when its state exited while it ran. Same blob for the
|
|
891
|
+
* snapshot's atomicity: an intent row is saved with the snapshot that holds its Effect, so a restore
|
|
892
|
+
* reads which Effects may have acted and runs none of them twice. A row is never dropped: the
|
|
893
|
+
* settled and aborted rows keep id, kind and outcome, the record of what the run's Effects did; a
|
|
894
|
+
* settled row's output tail is dropped once a save carries its transition (`dropSavedTails`).
|
|
895
|
+
*/
|
|
896
|
+
effects?: Record<string, EffectRow>;
|
|
897
|
+
/**
|
|
898
|
+
* Which Agent started each live child (ADR-0067), by the child's key — the invoke ids from below
|
|
899
|
+
* the root down to the child, encoded and joined by `/` (`childKey`), so a child of one Machine is
|
|
900
|
+
* never another Machine's child of the same local id: the Agent whose pick's
|
|
901
|
+
* own send started it, or whose event the Machine was taking when it started. A done event of that
|
|
902
|
+
* child names that Agent, so an `answer` on a transition below no Agent's state that the done event
|
|
903
|
+
* takes shapes that Agent's pick and no other — whether the pick that started it is still
|
|
904
|
+
* outstanding or not. Same blob, so a restore names the same Agent. A child no longer in the tree
|
|
905
|
+
* is dropped at the save.
|
|
906
|
+
*/
|
|
907
|
+
startedBy?: Record<string, StartedBy>;
|
|
355
908
|
fault?: string;
|
|
356
909
|
};
|
|
357
910
|
|
|
@@ -366,15 +919,27 @@ type LiveRun = {
|
|
|
366
919
|
agents: Record<string, AgentAdmission>;
|
|
367
920
|
/** The live epoch ledger (ADR-0057): persisted as `RunBlob.epochs`, seeded on restore. */
|
|
368
921
|
epochs: Record<string, number>;
|
|
922
|
+
/** The pending notices (ADR-0062): persisted as `RunBlob.notices`, seeded on restore. */
|
|
923
|
+
notices: NoticeLedger;
|
|
924
|
+
/** The outstanding picks by Agent address (ADR-0024): persisted as `RunBlob.holds`, seeded on restore. */
|
|
925
|
+
holds: Map<string, Hold>;
|
|
926
|
+
/** The Effect ledger (ADR-0067): persisted as `RunBlob.effects`, seeded on restore. */
|
|
927
|
+
effects: Record<string, EffectRow>;
|
|
928
|
+
/** Which Agent started each live child (ADR-0067): persisted as `RunBlob.startedBy`, seeded on restore. */
|
|
929
|
+
startedBy: Record<string, StartedBy>;
|
|
369
930
|
/** The error that killed the run, if it errored (xstate serializes Error to `{}`, so the
|
|
370
931
|
* message is captured here at the observer and persisted onto the blob for `read`). */
|
|
371
932
|
fault?: string;
|
|
372
933
|
/** Per-run observers fed by `persist()` (status) and the actor's `emit` (emit) — SSE/CLI watch. */
|
|
373
934
|
listeners: Set<(e: RunFeedEvent) => void>;
|
|
374
935
|
/** The run's feed-so-far (ADR-0023): what a Workspace attaching mid-run gets replayed as its
|
|
375
|
-
* log's preamble. In-memory and this-boot only — a
|
|
376
|
-
* the same live-only contract the printed log already
|
|
936
|
+
* log's preamble — its markers and Emits, in feed order. In-memory and this-boot only — a
|
|
937
|
+
* restored run's preamble starts at restore, the same live-only contract the printed log already
|
|
938
|
+
* has. Capped ({@link FEED_SO_FAR_CAP}). */
|
|
377
939
|
feedSoFar: RunFeedEvent[];
|
|
940
|
+
/** The preamble's last line: where the run stands now. A status is a level (ADR-0023), so the
|
|
941
|
+
* buffer keeps the latest alone — old statuses are not interleaved between old markers. */
|
|
942
|
+
latestStatus?: RunFeedEvent & { kind: "status" };
|
|
378
943
|
};
|
|
379
944
|
|
|
380
945
|
export class RunHost {
|
|
@@ -386,15 +951,30 @@ export class RunHost {
|
|
|
386
951
|
private readonly reconcile: (run: RunRecord) => boolean | Promise<boolean>;
|
|
387
952
|
private readonly newId: () => string;
|
|
388
953
|
private readonly onRestoreError?: (runId: string, err: unknown) => void;
|
|
954
|
+
private readonly resolveDef?: RunHostOptions["resolveDef"];
|
|
389
955
|
private readonly sandbox?: SandboxPort;
|
|
390
956
|
private readonly instanceHarness?: string;
|
|
391
957
|
private readonly harnessBearer?: (placement: string) => string;
|
|
958
|
+
private readonly effector?: (placement: EffectorPlacement) => EffectorTarget | undefined;
|
|
392
959
|
private readonly echoFactory?: (
|
|
393
960
|
endpoint: string,
|
|
394
961
|
bearer: string | undefined,
|
|
395
962
|
) => (events: EchoEvent[]) => Promise<void>;
|
|
963
|
+
private readonly liveSetPush?: (live: LiveConversation[]) => Promise<unknown>;
|
|
964
|
+
private readonly liveSetIntervalMs: number;
|
|
965
|
+
private readonly now: () => number;
|
|
966
|
+
private liveSetTimer?: ReturnType<typeof setTimeout>;
|
|
967
|
+
/** Whether the last live-set statement failed — a failing streak logs one line (ADR-0031). */
|
|
968
|
+
private liveSetFailing = false;
|
|
396
969
|
private readonly workflowDefs = new Map<string, WorkflowDef>();
|
|
397
970
|
private readonly runs = new Map<string, LiveRun>();
|
|
971
|
+
/**
|
|
972
|
+
* The admission ledgers of the runs restore could not host this boot (a `reconcile` that threw).
|
|
973
|
+
* Their rows stay `live` for the next boot to retry, so their conversations on the Instance
|
|
974
|
+
* Harness are still held: the live set names them from here (ADR-0031), or the retry would find
|
|
975
|
+
* nothing to re-attach to.
|
|
976
|
+
*/
|
|
977
|
+
private readonly unhosted = new Map<string, Record<string, AgentAdmission>>();
|
|
398
978
|
/**
|
|
399
979
|
* Workflow name → its observers. Deliberately on the HOST and not on `LiveRun`: a workflow's
|
|
400
980
|
* watcher outlives every individual run it watches, and `persist()` clears a settled run's own
|
|
@@ -402,16 +982,58 @@ export class RunHost {
|
|
|
402
982
|
* than a comment asking the next reader not to clear the wrong Set.
|
|
403
983
|
*/
|
|
404
984
|
private readonly workflowListeners = new Map<string, Set<WorkflowListener>>();
|
|
985
|
+
/** Every timer a run's actor system has armed and not yet fired or cleared: an `after`, a Chart's
|
|
986
|
+
* wall clock (ADR-0070). `close()` unrefs them, so a timer of hours never holds the process. */
|
|
987
|
+
private readonly timers = new Set<ReturnType<typeof setTimeout>>();
|
|
988
|
+
/** The clock every run's actor system arms its timers on: xstate's own, with each timer kept in
|
|
989
|
+
* {@link timers} and unref'd once the host is closing. The scheduled events stay in the system's
|
|
990
|
+
* snapshot, so a restore arms them again. */
|
|
991
|
+
private readonly clock = {
|
|
992
|
+
setTimeout: (fn: (...args: unknown[]) => void, ms: number) => {
|
|
993
|
+
const timer = setTimeout(() => {
|
|
994
|
+
this.timers.delete(timer);
|
|
995
|
+
fn();
|
|
996
|
+
}, ms);
|
|
997
|
+
this.timers.add(timer);
|
|
998
|
+
if (this.#stopping) (timer as { unref?: () => void }).unref?.();
|
|
999
|
+
return timer;
|
|
1000
|
+
},
|
|
1001
|
+
clearTimeout: (timer: ReturnType<typeof setTimeout>) => {
|
|
1002
|
+
this.timers.delete(timer);
|
|
1003
|
+
clearTimeout(timer);
|
|
1004
|
+
},
|
|
1005
|
+
};
|
|
1006
|
+
/** The registration each hold was opened under, this boot: the same address re-registered by
|
|
1007
|
+
* another invocation (`continue: true`) is another Turn. A restored hold adopts the first. */
|
|
1008
|
+
private readonly holdRegs = new WeakMap<Hold, Registration>();
|
|
1009
|
+
/** The long polls waiting on each open hold (ADR-0024). */
|
|
1010
|
+
private readonly holdWaiters = new Map<Hold, Set<() => void>>();
|
|
1011
|
+
/** The pick being delivered now, and the last reply from below its Agent's state inside its own
|
|
1012
|
+
* send (ADR-0024). A child that settles in the same macrostep as the pick — a decision Machine
|
|
1013
|
+
* that is final as it starts — replies before a hold can exist, so the chain records it here. */
|
|
1014
|
+
private sending: Sending | undefined;
|
|
1015
|
+
/** `<iid> <deliveryId>` of every outstanding pick of a run this host parked (`stop`). */
|
|
1016
|
+
private readonly parked = new Set<string>();
|
|
1017
|
+
#stopping = false;
|
|
1018
|
+
#restorePending = false;
|
|
405
1019
|
|
|
406
1020
|
constructor(opts: RunHostOptions) {
|
|
407
1021
|
this.store = opts.store;
|
|
408
1022
|
this.reconcile = opts.reconcile ?? (() => true);
|
|
409
1023
|
this.newId = opts.newId ?? (() => randomUUID());
|
|
410
1024
|
this.onRestoreError = opts.onRestoreError;
|
|
1025
|
+
this.resolveDef = opts.resolveDef;
|
|
411
1026
|
this.sandbox = opts.sandbox;
|
|
412
1027
|
this.instanceHarness = opts.instanceHarness;
|
|
413
1028
|
this.harnessBearer = opts.harnessBearer;
|
|
1029
|
+
this.effector = opts.effector;
|
|
414
1030
|
this.echoFactory = opts.echo;
|
|
1031
|
+
this.liveSetPush =
|
|
1032
|
+
opts.instanceHarness && opts.liveSet
|
|
1033
|
+
? opts.liveSet(opts.instanceHarness, opts.harnessBearer?.(INSTANCE_HARNESS_SERVICE))
|
|
1034
|
+
: undefined;
|
|
1035
|
+
this.liveSetIntervalMs = opts.liveSetIntervalMs ?? DEFAULT_LEASE_INTERVAL_MS;
|
|
1036
|
+
this.now = opts.now ?? Date.now;
|
|
415
1037
|
}
|
|
416
1038
|
|
|
417
1039
|
/** Register a workflow so `start`/`restore` can run it. Re-registering replaces (dev reload).
|
|
@@ -419,7 +1041,56 @@ export class RunHost {
|
|
|
419
1041
|
* declared them and resolved at invoke time off the invoking Machine (ADR-0011, ADR-0049), so
|
|
420
1042
|
* a nested Machine's defs are never the root's — or this host's — to hold. */
|
|
421
1043
|
register(def: WorkflowDef): void {
|
|
422
|
-
|
|
1044
|
+
const wrapper = def.wrapper ?? wrapperOf(def.machine);
|
|
1045
|
+
this.workflowDefs.set(def.name, wrapper ? { ...def, wrapper } : def);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/**
|
|
1049
|
+
* Register a named Chart as the Workflow `<wrapper>/<name>` (ADR-0070). The registry of named
|
|
1050
|
+
* Charts (charts.ts) calls this for each runnable one; it keeps where the Charts are stored, and
|
|
1051
|
+
* the unrunnable ones, itself. Replacing a Chart a live run holds is fine: the run holds its text.
|
|
1052
|
+
*/
|
|
1053
|
+
registerChart(def: WorkflowDef & { chart: NonNullable<WorkflowDef["chart"]> }): void {
|
|
1054
|
+
this.register(def);
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
/** Drop a named Chart's Workflow. A live run of it keeps its text and runs on. */
|
|
1058
|
+
unregisterChart(wrapper: string, name: string): void {
|
|
1059
|
+
const key = `${wrapper}/${name}`;
|
|
1060
|
+
if (this.workflowDefs.get(key)?.chart) this.workflowDefs.delete(key);
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/** The registered named Chart `<wrapper>/<name>`, or undefined. */
|
|
1064
|
+
namedChart(wrapper: string, name: string): NamedChart | undefined {
|
|
1065
|
+
const chart = this.workflowDefs.get(`${wrapper}/${name}`)?.chart;
|
|
1066
|
+
return (
|
|
1067
|
+
chart && { wrapper: chart.wrapper, name: chart.name, text: chart.text, hash: chart.hash, origin: chart.origin }
|
|
1068
|
+
);
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/**
|
|
1072
|
+
* Every registered Workflow with its origin (ADR-0070): a Machine, or a named Chart from a file or
|
|
1073
|
+
* at run time. A Chart wrapper's entry carries its Open body, whether its door takes a Chart, and
|
|
1074
|
+
* its named Charts: a named Chart lists inside its wrapper's entry, not beside it, and has an
|
|
1075
|
+
* entry of its own only when its wrapper is not registered.
|
|
1076
|
+
*/
|
|
1077
|
+
workflowList(): WorkflowListing[] {
|
|
1078
|
+
const defs = [...this.workflowDefs.values()];
|
|
1079
|
+
const wrappers = new Set(defs.filter((d) => !d.chart && d.wrapper).map((d) => d.name));
|
|
1080
|
+
return defs.flatMap((def): WorkflowListing[] => {
|
|
1081
|
+
if (def.chart) return wrappers.has(def.chart.wrapper) ? [] : [{ name: def.name, origin: def.chart.origin }];
|
|
1082
|
+
if (!def.wrapper) return [{ name: def.name, origin: "machine" }];
|
|
1083
|
+
const charts = defs
|
|
1084
|
+
.filter((d) => d.chart?.wrapper === def.name)
|
|
1085
|
+
.map((d) => ({ name: d.chart!.name, origin: d.chart!.origin }));
|
|
1086
|
+
return [
|
|
1087
|
+
{
|
|
1088
|
+
name: def.name,
|
|
1089
|
+
origin: "machine",
|
|
1090
|
+
wrapper: { body: def.wrapper.body, oneShot: def.wrapper.chartFields.length > 0, charts },
|
|
1091
|
+
},
|
|
1092
|
+
];
|
|
1093
|
+
});
|
|
423
1094
|
}
|
|
424
1095
|
|
|
425
1096
|
/** Drop a workflow's registration (a workflow the host no longer discovers). In-flight
|
|
@@ -437,12 +1108,79 @@ export class RunHost {
|
|
|
437
1108
|
return [...this.workflowDefs.keys()];
|
|
438
1109
|
}
|
|
439
1110
|
|
|
1111
|
+
/** The runs this process hosts, by the hash of the Assembly each started under (ADR-0069). A
|
|
1112
|
+
* run from a def with no Assembly holds none. The store's live rows are the hold
|
|
1113
|
+
* (`SnapshotStore.liveAssemblies`); this adds a run started a moment ago, before its first save. */
|
|
1114
|
+
assemblyHolds(): Map<string, string[]> {
|
|
1115
|
+
const holds = new Map<string, string[]>();
|
|
1116
|
+
for (const [runId, run] of this.runs) {
|
|
1117
|
+
const hash = run.def.assembly;
|
|
1118
|
+
if (hash === undefined) continue;
|
|
1119
|
+
holds.set(hash, [...(holds.get(hash) ?? []), runId]);
|
|
1120
|
+
}
|
|
1121
|
+
return holds;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
440
1124
|
/** The registered template Machine's serialized structure (`GET /workflows/:name/machine`). */
|
|
441
1125
|
machine(name: string): MachineDoc | undefined {
|
|
442
1126
|
const def = this.workflowDefs.get(name);
|
|
443
1127
|
return def && serializeMachine(name, def.machine);
|
|
444
1128
|
}
|
|
445
1129
|
|
|
1130
|
+
/**
|
|
1131
|
+
* The run's bound Machine (ADR-0070): the Workflow's Machine with each bound Open body's subgraph
|
|
1132
|
+
* the compiled Chart's (its `src` stays the slot key, so the live child joins it as before) —
|
|
1133
|
+
* from the live actor's bound Machine, or, for a stored run, compiled again from the text its
|
|
1134
|
+
* blob holds. Undefined for an unknown run, and for a stored run whose Workflow is not registered.
|
|
1135
|
+
*/
|
|
1136
|
+
async runMachine(runId: string): Promise<MachineDoc | undefined> {
|
|
1137
|
+
const live = this.runs.get(runId);
|
|
1138
|
+
let workflow: string;
|
|
1139
|
+
let machine: AnyStateMachine;
|
|
1140
|
+
let bodies: Array<{ route: BoundChartRoute; compiled: AnyStateMachine | string }>;
|
|
1141
|
+
if (live) {
|
|
1142
|
+
workflow = live.record.workflow;
|
|
1143
|
+
machine = live.def.machine;
|
|
1144
|
+
const refs = new Map<string, AnyActorRef>();
|
|
1145
|
+
const collect = (ref: AnyActorRef, path: string[]): void => {
|
|
1146
|
+
for (const [id, child] of Object.entries(childrenOfRef(ref))) {
|
|
1147
|
+
refs.set([...path, id].join("/"), child);
|
|
1148
|
+
collect(child, [...path, id]);
|
|
1149
|
+
}
|
|
1150
|
+
};
|
|
1151
|
+
collect(live.actor, []);
|
|
1152
|
+
bodies = boundChartRoutesOf(live.actor.getSnapshot()).map((route) => {
|
|
1153
|
+
const ref = refs.get(route.path.join("/"));
|
|
1154
|
+
return { route, compiled: (ref && boundMachineOf(ref)) ?? "the live body holds no compiled Chart" };
|
|
1155
|
+
});
|
|
1156
|
+
} else {
|
|
1157
|
+
const stored = await this.store.load(runId);
|
|
1158
|
+
const blob = stored?.snapshot as RunBlob | null | undefined;
|
|
1159
|
+
if (!blob) return undefined;
|
|
1160
|
+
const def = this.workflowDefs.get(blob.workflow) ?? this.workflowDefs.get(wrapperName(blob.workflow));
|
|
1161
|
+
if (!def) return undefined;
|
|
1162
|
+
workflow = blob.workflow;
|
|
1163
|
+
machine = def.machine;
|
|
1164
|
+
bodies = boundChartRoutesOf(blob.snapshot).map((route) => ({
|
|
1165
|
+
route,
|
|
1166
|
+
compiled: compileStored(def.machine, route),
|
|
1167
|
+
}));
|
|
1168
|
+
}
|
|
1169
|
+
const doc = serializeMachine(workflow, machine);
|
|
1170
|
+
const notDrawn: NonNullable<MachineDoc["chartsNotDrawn"]> = [];
|
|
1171
|
+
for (const { route, compiled } of bodies) {
|
|
1172
|
+
if (typeof compiled === "string") {
|
|
1173
|
+
// The doc keeps the slot's own subgraph, and says so: a doc that drew the wrapper's body
|
|
1174
|
+
// with no word would disagree with the run's `bound` (ADR-0070, D-B).
|
|
1175
|
+
notDrawn.push({ path: route.path.join("/"), src: route.src, reason: compiled });
|
|
1176
|
+
continue;
|
|
1177
|
+
}
|
|
1178
|
+
const { workflow: _w, opaqueStates: _o, ...body } = serializeMachine(route.src, compiled);
|
|
1179
|
+
placeBody(doc, route.srcs, body);
|
|
1180
|
+
}
|
|
1181
|
+
return notDrawn.length ? { ...doc, chartsNotDrawn: notDrawn } : doc;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
446
1184
|
/**
|
|
447
1185
|
* The workflow's declared run-input contract as JSON Schema (ADR-0033) — what the workflow
|
|
448
1186
|
* detail serves and the Console's start form generates from. `null` for a machine that
|
|
@@ -452,7 +1190,11 @@ export class RunHost {
|
|
|
452
1190
|
inputSchema(name: string): Record<string, unknown> | null | undefined {
|
|
453
1191
|
const def = this.workflowDefs.get(name);
|
|
454
1192
|
if (!def) return undefined;
|
|
455
|
-
|
|
1193
|
+
let schema = inputSchemaOf(def.machine);
|
|
1194
|
+
// A named Chart's door is its wrapper's minus the Chart fields, which the Chart supplies (ADR-0070).
|
|
1195
|
+
if (schema && def.chart && def.wrapper?.chartFields.length) {
|
|
1196
|
+
schema = schema.omit(Object.fromEntries(def.wrapper.chartFields.map((f) => [f, true])) as never);
|
|
1197
|
+
}
|
|
456
1198
|
// `io: "input"`: this schema describes what a caller SENDS — a defaulted field is optional
|
|
457
1199
|
// at the door (the default is applied by `start`'s parse), not required of the form.
|
|
458
1200
|
return schema ? (z.toJSONSchema(schema, { io: "input" }) as Record<string, unknown>) : null;
|
|
@@ -463,6 +1205,13 @@ export class RunHost {
|
|
|
463
1205
|
const def = this.workflowDefs.get(workflow);
|
|
464
1206
|
if (!def) throw new Error(`no workflow registered as "${workflow}"`);
|
|
465
1207
|
|
|
1208
|
+
// A named Chart supplies every Chart field of its wrapper's door, and the caller the rest
|
|
1209
|
+
// (ADR-0070): what the caller sent under a Chart field is not the run's.
|
|
1210
|
+
if (def.chart && def.wrapper) {
|
|
1211
|
+
input = { ...input };
|
|
1212
|
+
for (const field of def.wrapper.chartFields) input[field] = def.chart.text;
|
|
1213
|
+
}
|
|
1214
|
+
|
|
466
1215
|
// A declared schema is enforced at the door (ADR-0033), and what starts the run is the
|
|
467
1216
|
// PARSED shape (defaults applied, unknown keys stripped) — exactly what a gate delivery
|
|
468
1217
|
// lands as, and the same error class: `EventValidationError`, which the wire maps to a 400
|
|
@@ -482,6 +1231,10 @@ export class RunHost {
|
|
|
482
1231
|
const record: RunRecord = { runId, workflow, instanceId };
|
|
483
1232
|
|
|
484
1233
|
const machine = this.assemble(def, instanceId);
|
|
1234
|
+
// Every Chart field of a Chart wrapper's door is admitted BEFORE the run exists (ADR-0070): a
|
|
1235
|
+
// refusal throws `ChartRefusal`, which the wire answers `422 { error: "refused", report }`. The
|
|
1236
|
+
// lent set is the run's own Machine's, with its providers in.
|
|
1237
|
+
this.admitDoorCharts(def, machine, runInput);
|
|
485
1238
|
// The root machine gets the parsed door PLUS what the host injects beside it —
|
|
486
1239
|
// `HostInjectedInput` (vocabulary.ts): the seed Instance ID, added after the parse because no
|
|
487
1240
|
// caller sends it and nothing serves it (ADR-0033). Only the root: a machine invoked below is
|
|
@@ -492,6 +1245,46 @@ export class RunHost {
|
|
|
492
1245
|
return { runId, instanceId };
|
|
493
1246
|
}
|
|
494
1247
|
|
|
1248
|
+
/**
|
|
1249
|
+
* Admit every Chart field present in a Chart wrapper's parsed door (ADR-0070): the whole door,
|
|
1250
|
+
* the compile included, against each Open body (the door cannot tell which body a field feeds).
|
|
1251
|
+
* `input.*` references are the body's to check at its start: the Chart's `input` is what the
|
|
1252
|
+
* wrapper's invoke sends, which is not the door. Throws ChartRefusal.
|
|
1253
|
+
*/
|
|
1254
|
+
private admitDoorCharts(def: WorkflowDef, machine: AnyStateMachine, input: Record<string, unknown>): void {
|
|
1255
|
+
for (const field of def.wrapper?.chartFields ?? []) {
|
|
1256
|
+
if (input[field] === undefined) continue;
|
|
1257
|
+
admitWrapperChart(machine, input[field]);
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
/**
|
|
1262
|
+
* The def that reads `blob` (ADR-0069), or the verdict when none can. A stamped blob resolves
|
|
1263
|
+
* inside the Assembly it names, through `resolveDef`; an unstamped blob resolves by name in the
|
|
1264
|
+
* registry, and only on a host that loads no Assembly. A blob that names no Workflow the
|
|
1265
|
+
* registry holds is `lost`: there is nothing it could ever be read by.
|
|
1266
|
+
*/
|
|
1267
|
+
private async defFor(blob: RunBlob): Promise<WorkflowDef | { drifted: string } | { lost: string }> {
|
|
1268
|
+
const registered = this.workflowDefs.get(blob.workflow);
|
|
1269
|
+
if (blob.assembly !== undefined) {
|
|
1270
|
+
const ref = { workflow: blob.workflow, assembly: blob.assembly };
|
|
1271
|
+
if (this.resolveDef) {
|
|
1272
|
+
const found = await this.resolveDef(ref);
|
|
1273
|
+
return "drifted" in found ? found : { ...found, assembly: blob.assembly };
|
|
1274
|
+
}
|
|
1275
|
+
if (registered?.assembly === blob.assembly) return registered;
|
|
1276
|
+
return {
|
|
1277
|
+
drifted: `workflow "${blob.workflow}" saved under Assembly ${blob.assembly}, which this host does not hold`,
|
|
1278
|
+
};
|
|
1279
|
+
}
|
|
1280
|
+
// A blob that names no Assembly cannot be vouched for on a host that loads Assemblies:
|
|
1281
|
+
// greenfield, no migration (ADR-0069).
|
|
1282
|
+
if (this.resolveDef || registered?.assembly !== undefined) {
|
|
1283
|
+
return { drifted: `workflow "${blob.workflow}": saved under no Assembly` };
|
|
1284
|
+
}
|
|
1285
|
+
return registered ?? { lost: `no workflow "${blob.workflow}"` };
|
|
1286
|
+
}
|
|
1287
|
+
|
|
495
1288
|
/**
|
|
496
1289
|
* Restore every persisted live run: hydrate, reconcile against the live world, and either
|
|
497
1290
|
* re-attach (re-spawn from the rewritten child input), mark lost, or refuse as drifted. Returns
|
|
@@ -503,8 +1296,10 @@ export class RunHost {
|
|
|
503
1296
|
* later run in the store never restored at all. One run that cannot be resumed is one run, not an
|
|
504
1297
|
* outage.
|
|
505
1298
|
*
|
|
506
|
-
* `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict —
|
|
507
|
-
*
|
|
1299
|
+
* `drifted` and `failed` are different claims, deliberately. Drift is a DURABLE verdict — this
|
|
1300
|
+
* Orchestrator cannot read what it saved (the Assembly is gone, at another Kit version, does not
|
|
1301
|
+
* load, or its Machine's shape does not match), and the next boot will not read it either — so it
|
|
1302
|
+
* is written to the row.
|
|
508
1303
|
* A throw is not: `reconcile` talks to a cluster, and a kubectl blip must not permanently condemn
|
|
509
1304
|
* a run. Those rows are left `live` to be retried on the next boot, and reported every time until
|
|
510
1305
|
* they stop failing.
|
|
@@ -515,30 +1310,72 @@ export class RunHost {
|
|
|
515
1310
|
const drifted: string[] = [];
|
|
516
1311
|
const failed: string[] = [];
|
|
517
1312
|
|
|
518
|
-
for (const stored of await this.store.
|
|
519
|
-
|
|
1313
|
+
for (const stored of await this.store.live()) {
|
|
1314
|
+
// A run this process hosts already is not restored a second time. A deferred restore
|
|
1315
|
+
// (ADR-0069) that throws as a whole stays pending, and the next publish tries it again; a run
|
|
1316
|
+
// started under the current Assembly between those two publishes has a live row and a live
|
|
1317
|
+
// actor, and must not get a second actor.
|
|
1318
|
+
if (this.runs.has(stored.runId)) continue;
|
|
520
1319
|
const blob = stored.snapshot as RunBlob | null;
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
await this.store.markLost(stored.runId, blob ? `no workflow "${blob.workflow}"` : "empty snapshot");
|
|
1320
|
+
if (!blob) {
|
|
1321
|
+
await this.store.markLost(stored.runId, "empty snapshot");
|
|
524
1322
|
lost.push(stored.runId);
|
|
525
1323
|
continue;
|
|
526
1324
|
}
|
|
527
1325
|
|
|
528
|
-
//
|
|
529
|
-
//
|
|
530
|
-
//
|
|
1326
|
+
// Which Machine reads this snapshot (ADR-0069). One rule: a stamped blob restores from its
|
|
1327
|
+
// Assembly; an unstamped blob restores from the registry; a stamped host never has unstamped
|
|
1328
|
+
// blobs it can vouch for. So a publish strands nothing: the def comes from the Assembly the
|
|
1329
|
+
// run started under, never from the current one. Drift is only "I can no longer read what I
|
|
1330
|
+
// saved", and it keeps the snapshot (`markDrifted`), so `jr2 status` still reads the run.
|
|
1331
|
+
let def: WorkflowDef;
|
|
1332
|
+
try {
|
|
1333
|
+
const found = await this.defFor(blob);
|
|
1334
|
+
if ("lost" in found) {
|
|
1335
|
+
await this.store.markLost(stored.runId, found.lost);
|
|
1336
|
+
lost.push(stored.runId);
|
|
1337
|
+
continue;
|
|
1338
|
+
}
|
|
1339
|
+
if ("drifted" in found) {
|
|
1340
|
+
await this.store.markDrifted(stored.runId, found.drifted);
|
|
1341
|
+
drifted.push(stored.runId);
|
|
1342
|
+
continue;
|
|
1343
|
+
}
|
|
1344
|
+
def = found;
|
|
1345
|
+
} catch (err) {
|
|
1346
|
+
// A fault while reading the Assembly, not a verdict on it: left live, as below.
|
|
1347
|
+
this.onRestoreError?.(stored.runId, err);
|
|
1348
|
+
failed.push(stored.runId);
|
|
1349
|
+
this.unhosted.set(stored.runId, blob.agents ?? {});
|
|
1350
|
+
continue;
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
// The def is the Machine this snapshot names; this asks whether it is still the shape this
|
|
1354
|
+
// snapshot was written by (ADR-0030). Inside one Assembly the same bytes give the same
|
|
1355
|
+
// Machine, so this guards a hash collision or a hand-edited volume. Refused BEFORE
|
|
1356
|
+
// `reconcile`, which talks to the cluster: there is no point proving a Sandbox is alive for
|
|
1357
|
+
// a run that cannot be read.
|
|
531
1358
|
const expected = fingerprintOf(def.machine);
|
|
532
1359
|
if (blob.machine !== expected) {
|
|
533
1360
|
await this.store.markDrifted(
|
|
534
1361
|
stored.runId,
|
|
535
|
-
`workflow "${blob.workflow}" changed shape since this run was saved ` +
|
|
1362
|
+
`workflow "${blob.workflow}"${def.assembly !== undefined ? ` in Assembly ${def.assembly}` : ""} changed shape since this run was saved ` +
|
|
536
1363
|
`(saved under ${blob.machine ?? "an unstamped Machine"}, now ${expected})`,
|
|
537
1364
|
);
|
|
538
1365
|
drifted.push(stored.runId);
|
|
539
1366
|
continue;
|
|
540
1367
|
}
|
|
541
1368
|
|
|
1369
|
+
// The Charts the run binds (ADR-0070, D-B): the shape stamp is the wrapper's, and each Chart's
|
|
1370
|
+
// identity is its text, kept in its body's snapshot. Each is admitted again against the Open
|
|
1371
|
+
// body it binds and the lent set the Machine has now; one that no longer admits is drift.
|
|
1372
|
+
const charts = chartDrift(def.machine, boundChartRoutesOf(blob.snapshot));
|
|
1373
|
+
if (charts !== undefined) {
|
|
1374
|
+
await this.store.markDrifted(stored.runId, charts);
|
|
1375
|
+
drifted.push(stored.runId);
|
|
1376
|
+
continue;
|
|
1377
|
+
}
|
|
1378
|
+
|
|
542
1379
|
const record: RunRecord = { runId: stored.runId, workflow: blob.workflow, instanceId: blob.instanceId };
|
|
543
1380
|
|
|
544
1381
|
try {
|
|
@@ -557,38 +1394,124 @@ export class RunHost {
|
|
|
557
1394
|
this.assemble(def, blob.instanceId),
|
|
558
1395
|
{ snapshot: hydrated as never },
|
|
559
1396
|
record,
|
|
1397
|
+
// The def of the Assembly the run started under (ADR-0069), so every later save names
|
|
1398
|
+
// that hash. The config the run provisions under is the CURRENT Assembly's all the same:
|
|
1399
|
+
// the ports read it through the binding a publish replaces, and a held Assembly lends
|
|
1400
|
+
// only its Machines (ADR-0069).
|
|
560
1401
|
def,
|
|
561
1402
|
agents,
|
|
562
1403
|
// The burned conversations come back with the admissions (ADR-0057): a restored run's
|
|
563
1404
|
// next `continue` must land where this run's last one did, not on a dead conversation.
|
|
564
1405
|
blob.epochs ?? {},
|
|
1406
|
+
// …and so do the undelivered notices (ADR-0062), reservations forgotten: a notice whose
|
|
1407
|
+
// admission never reached the ledger is delivered again.
|
|
1408
|
+
blob.notices,
|
|
1409
|
+
// …and the outstanding picks (ADR-0024): the restored Machine says when each is answered.
|
|
1410
|
+
blob.holds,
|
|
1411
|
+
// …and the Effect ledger (ADR-0067): a restored Effect reads its row and never runs twice.
|
|
1412
|
+
blob.effects ?? {},
|
|
1413
|
+
// …and which Agent started each child (ADR-0067): its done event names the same Agent.
|
|
1414
|
+
blob.startedBy ?? {},
|
|
565
1415
|
);
|
|
566
|
-
|
|
1416
|
+
// `start()`, then the pending timers again (ADR-0007, see `startAndRearm`). The next save
|
|
1417
|
+
// records them again.
|
|
1418
|
+
startAndRearm(actor, blob.timers ?? [], this.now());
|
|
1419
|
+
// A run an earlier restore left live to retry is hosted now.
|
|
1420
|
+
this.unhosted.delete(stored.runId);
|
|
567
1421
|
reattached.push(stored.runId);
|
|
568
1422
|
} catch (err) {
|
|
569
1423
|
// Left `live` on purpose — see the note above. The row is unchanged, so the next boot tries
|
|
570
1424
|
// again; what must not happen is this taking the remaining runs down with it.
|
|
571
1425
|
this.onRestoreError?.(stored.runId, err);
|
|
572
1426
|
failed.push(stored.runId);
|
|
1427
|
+
this.unhosted.set(stored.runId, blob.agents ?? {});
|
|
573
1428
|
}
|
|
574
1429
|
}
|
|
575
1430
|
|
|
1431
|
+
// Every run this boot will host is live now, so the set is whole: state it, and keep stating it
|
|
1432
|
+
// (ADR-0031). Not before — a statement ahead of restore would free the conversations of the
|
|
1433
|
+
// runs about to be re-attached.
|
|
1434
|
+
this.stateLiveSet();
|
|
1435
|
+
this.scheduleLiveSet();
|
|
1436
|
+
|
|
1437
|
+
this.#restorePending = false;
|
|
576
1438
|
return { reattached, lost, drifted, failed };
|
|
577
1439
|
}
|
|
578
1440
|
|
|
1441
|
+
/**
|
|
1442
|
+
* The conversations the live runs hold on the Instance Harness (ADR-0031): every admission in
|
|
1443
|
+
* their ledgers whose endpoint is the Instance Harness, as the `(agent, instanceId)` pair it runs
|
|
1444
|
+
* under. From MEMORY, never the store: the store also holds runs this process does not host, and
|
|
1445
|
+
* a run this process hosts is exactly one that may still continue its conversation — and so is a
|
|
1446
|
+
* run restore left `live` to retry (`unhosted`). A Sandbox's Harness is not in it — its
|
|
1447
|
+
* Workspace's teardown frees everything there.
|
|
1448
|
+
*/
|
|
1449
|
+
liveConversations(): LiveConversation[] {
|
|
1450
|
+
const live = new Map<string, LiveConversation>();
|
|
1451
|
+
const ledgers = [...[...this.runs.values()].map((run) => run.agents), ...this.unhosted.values()];
|
|
1452
|
+
for (const agents of ledgers) {
|
|
1453
|
+
for (const [iid, admission] of Object.entries(agents)) {
|
|
1454
|
+
if (admission.endpoint !== this.instanceHarness || admission.agent === undefined) continue;
|
|
1455
|
+
const instanceId = admission.instanceId ?? iid;
|
|
1456
|
+
live.set(JSON.stringify([admission.agent, instanceId]), { agent: admission.agent, instanceId });
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
return [...live.values()];
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
/**
|
|
1463
|
+
* State the live set to the Instance Harness (ADR-0031), which frees every other conversation.
|
|
1464
|
+
* FIRE-AND-FORGET: level-triggered and idempotent, so a missed statement costs nothing and the
|
|
1465
|
+
* next one says it all again. A failing streak logs one line; nothing here can touch a run.
|
|
1466
|
+
*/
|
|
1467
|
+
private stateLiveSet(): void {
|
|
1468
|
+
const push = this.liveSetPush;
|
|
1469
|
+
if (!push || this.#stopping) return;
|
|
1470
|
+
push(this.liveConversations()).then(
|
|
1471
|
+
() => {
|
|
1472
|
+
this.liveSetFailing = false;
|
|
1473
|
+
},
|
|
1474
|
+
(err: unknown) => {
|
|
1475
|
+
if (this.liveSetFailing) return;
|
|
1476
|
+
this.liveSetFailing = true;
|
|
1477
|
+
console.error(
|
|
1478
|
+
`live-set statement to ${this.instanceHarness} failed (the Instance Harness keeps what it holds): ` +
|
|
1479
|
+
(err instanceof Error ? err.message : String(err)),
|
|
1480
|
+
);
|
|
1481
|
+
},
|
|
1482
|
+
);
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
/** Every Lease interval, ±20% (ADR-0021's cadence, which ADR-0031 names). The timer holds no
|
|
1486
|
+
* process alive, and `close()` clears it. */
|
|
1487
|
+
private scheduleLiveSet(): void {
|
|
1488
|
+
if (!this.liveSetPush || this.liveSetTimer !== undefined || this.#stopping) return;
|
|
1489
|
+
const next = () => {
|
|
1490
|
+
this.liveSetTimer = setTimeout(
|
|
1491
|
+
() => {
|
|
1492
|
+
if (this.#stopping) return;
|
|
1493
|
+
this.stateLiveSet();
|
|
1494
|
+
next();
|
|
1495
|
+
},
|
|
1496
|
+
leaseDelay(this.liveSetIntervalMs, "next", Math.random()),
|
|
1497
|
+
);
|
|
1498
|
+
this.liveSetTimer.unref?.();
|
|
1499
|
+
};
|
|
1500
|
+
next();
|
|
1501
|
+
}
|
|
1502
|
+
|
|
579
1503
|
/**
|
|
580
1504
|
* An agent instance's LIVE surface (`GET /agents/:iid/surface` — ADR-0013): what the state that
|
|
581
1505
|
* invoked this Agent accepts, right now. Undefined once nothing is registered (the state exited,
|
|
582
|
-
* the run settled, the iid is unknown) — the one catch point, and the reason the
|
|
1506
|
+
* the run settled, the iid is unknown) — the one catch point, and the reason the Harness never
|
|
583
1507
|
* has to learn which turn is live: it asks, per turn, and the answer IS the turn.
|
|
584
1508
|
*
|
|
585
|
-
* "Right now" is
|
|
586
|
-
* statically from its transitions, and the guards on those transitions are
|
|
587
|
-
*
|
|
588
|
-
* question is
|
|
589
|
-
*
|
|
590
|
-
*
|
|
591
|
-
* an Agent mid-turn.
|
|
1509
|
+
* "Right now" is the Allowed picks' half (ADR-0029): the registered defs are the state's
|
|
1510
|
+
* VOCABULARY, derived statically from its transitions, and the guards on those transitions are
|
|
1511
|
+
* asked here, as this Agent's sender (`allowedPicks`, ADR-0015). The pick has not happened yet, so
|
|
1512
|
+
* the question is payload-blind: a guard that would have judged the Agent's arguments is left
|
|
1513
|
+
* allowed and settled at delivery. The Menu does not move with them — it is the conversation's,
|
|
1514
|
+
* fixed at the Machine's build.
|
|
592
1515
|
*/
|
|
593
1516
|
agentSurface(instanceId: string): AgentSurfaceView | undefined {
|
|
594
1517
|
const reg = this.table.lookup(agentAddress(instanceId));
|
|
@@ -597,28 +1520,79 @@ export class RunHost {
|
|
|
597
1520
|
instanceId,
|
|
598
1521
|
runId: reg.runId,
|
|
599
1522
|
sandbox: reg.sandbox,
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
.
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
1523
|
+
menu: [...menuOf(reg).values()].map((def) => ({
|
|
1524
|
+
name: def.name,
|
|
1525
|
+
description: def.description,
|
|
1526
|
+
input: z.toJSONSchema(def.input),
|
|
1527
|
+
semantics: def.semantics,
|
|
1528
|
+
...briefingOf(def.input),
|
|
1529
|
+
})),
|
|
1530
|
+
allowed: allowedOf(reg),
|
|
608
1531
|
};
|
|
609
1532
|
}
|
|
610
1533
|
|
|
1534
|
+
/** The Harness read this Turn's surface and was answered (ADR-0026): the notices its admission
|
|
1535
|
+
* carried reach a model now, so an end to the Turn no longer raises them again (ADR-0062). */
|
|
1536
|
+
agentSurfaceServed(instanceId: string): void {
|
|
1537
|
+
this.table.lookup(agentAddress(instanceId))?.served?.();
|
|
1538
|
+
}
|
|
1539
|
+
|
|
611
1540
|
/**
|
|
612
|
-
* Deliver one event from an Agent's
|
|
613
|
-
* and delivery are the table's; this only agent-scopes the address and mints the receipt.
|
|
1541
|
+
* Deliver one event from an Agent's Harness (`POST /agents/:iid/events` — ADR-0013). Validation
|
|
1542
|
+
* and delivery are the table's; this only agent-scopes the address and mints the receipt. (A
|
|
1543
|
+
* `poll` event never reaches here: `jr2Setup` refuses to build a Machine that declares one —
|
|
1544
|
+
* ADR-0013.)
|
|
614
1545
|
*/
|
|
615
1546
|
sendToAgent(instanceId: string, event: { type?: unknown } & Record<string, unknown>): AgentDeliveryReceipt {
|
|
616
1547
|
const { type, ...payload } = event;
|
|
617
1548
|
if (typeof type !== "string" || !type) {
|
|
618
|
-
throw new EventValidationError(`event body must carry a string "type" (one of the surface's
|
|
1549
|
+
throw new EventValidationError(`event body must carry a string "type" (one of the surface's Allowed picks)`);
|
|
619
1550
|
}
|
|
1551
|
+
refuseSenderField(payload);
|
|
620
1552
|
const address = agentAddress(instanceId);
|
|
621
1553
|
const invoking = this.table.lookup(address);
|
|
1554
|
+
const run = invoking ? this.runs.get(invoking.runId) : undefined;
|
|
1555
|
+
// One pick at a time per Turn (ADR-0024): while a pick is outstanding, or the Agent's state is
|
|
1556
|
+
// busy with work it started in, every pick is refused — `finish` included, and before any other
|
|
1557
|
+
// judgment. Decided from the Machine (the Rest rule) and the run's holds, both of which a
|
|
1558
|
+
// restore brings back, so it holds after a restart. xstate itself would take the pick, and one
|
|
1559
|
+
// the Agent's state handles would exit the child state and stop the work running there.
|
|
1560
|
+
if (invoking && run) {
|
|
1561
|
+
this.settleHolds(run);
|
|
1562
|
+
const hold = run.holds.get(instanceId);
|
|
1563
|
+
const out = hold?.answer === undefined ? hold?.pick : undefined;
|
|
1564
|
+
if (out !== undefined || restOf(invoking) === "busy") {
|
|
1565
|
+
throw new OutstandingError(
|
|
1566
|
+
out !== undefined
|
|
1567
|
+
? `"${type}" was not delivered: the pick "${out}" is still running in the workflow, and no other pick is taken until it answers`
|
|
1568
|
+
: `"${type}" was not delivered: the workflow is still running work in the state that asked for this Turn, and no pick is taken until it is back at rest`,
|
|
1569
|
+
out,
|
|
1570
|
+
);
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
if (invoking && !invoking.defs.has(type)) {
|
|
1574
|
+
// A pick on the Menu that another state of this Agent handles (ADR-0029): answered, not
|
|
1575
|
+
// refused — `moved: false` is the correction receipt, whatever its arguments, so the payload
|
|
1576
|
+
// is not read. It never reaches the Machine: the invoking state does not handle it, so it
|
|
1577
|
+
// could move nothing of this Turn's, and only a state the Agent does not own (a nested
|
|
1578
|
+
// Agent's, a Gate's) could act on it — a widening the audience rule forbids (ADR-0015).
|
|
1579
|
+
if (menuOf(invoking).has(type)) {
|
|
1580
|
+
return {
|
|
1581
|
+
delivered: true,
|
|
1582
|
+
event: type,
|
|
1583
|
+
moved: false,
|
|
1584
|
+
turnComplete: false,
|
|
1585
|
+
allowed: allowedOf(invoking),
|
|
1586
|
+
deliveryId: this.newId(),
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
// A name off the Menu is refused HERE, in the Agent's own terms: the table's refusal lists the
|
|
1590
|
+
// derived set, bare and unnarrowed, where the Harness is about to write the Allowed picks in
|
|
1591
|
+
// the model's tool names (ADR-0029, ADR-0062) — two lists that disagree.
|
|
1592
|
+
const refused = new EventValidationError(`the workflow's current state does not accept "${type}"`);
|
|
1593
|
+
refused.allowed = allowedOf(invoking);
|
|
1594
|
+
throw refused;
|
|
1595
|
+
}
|
|
622
1596
|
|
|
623
1597
|
// Ask the guard question BEFORE delivering, and ask it with the VALIDATED payload — this is the
|
|
624
1598
|
// exact form of the check the surface build can only approximate payload-blind (ADR-0029).
|
|
@@ -628,9 +1602,45 @@ export class RunHost {
|
|
|
628
1602
|
// makes this unreliable and `deliver` throws on the next line anyway — so it fails open.
|
|
629
1603
|
const def = invoking?.defs.get(type);
|
|
630
1604
|
const parsed = def?.input.safeParse(payload ?? {});
|
|
631
|
-
const
|
|
632
|
-
|
|
633
|
-
|
|
1605
|
+
const probe = {
|
|
1606
|
+
...(parsed?.success ? { type, ...parsed.data } : { type, ...payload }),
|
|
1607
|
+
...(invoking?.sender ? { [SENDER_KEY]: invoking.sender } : {}),
|
|
1608
|
+
};
|
|
1609
|
+
const moved = wouldMove(invoking?.invoker, probe);
|
|
1610
|
+
// The states the pick names as targets, and the state value it leaves: after the send, the same
|
|
1611
|
+
// value with a target no longer active means it entered a state and came straight back
|
|
1612
|
+
// (ADR-0024); a new value, an in-Turn mode move.
|
|
1613
|
+
const targets = moved ? targetsOf(invoking?.invoker, probe) : [];
|
|
1614
|
+
const before = targets.length > 0 ? stateValueOf(invoking?.invoker) : undefined;
|
|
1615
|
+
|
|
1616
|
+
const outer = this.sending;
|
|
1617
|
+
const sending: Sending | undefined = invoking && { reg: invoking, pick: type, chain: [] };
|
|
1618
|
+
this.sending = sending;
|
|
1619
|
+
try {
|
|
1620
|
+
this.table.deliver(address, type, payload);
|
|
1621
|
+
} catch (err) {
|
|
1622
|
+
// The writing xstate would run for this pick is another party's (ADR-0015, ADR-0029): the
|
|
1623
|
+
// pick is ANSWERED, never delivered — `moved: false`, and the Harness picks the receipt by
|
|
1624
|
+
// whether the name is in `allowed`. Not in it: the correction receipt. In it: a guard that
|
|
1625
|
+
// read the payload turned the selection outward, and the receipt says to change the arguments.
|
|
1626
|
+
if (err instanceof NotTakenError && invoking) {
|
|
1627
|
+
return {
|
|
1628
|
+
delivered: true,
|
|
1629
|
+
event: type,
|
|
1630
|
+
moved: false,
|
|
1631
|
+
turnComplete: false,
|
|
1632
|
+
allowed: allowedOf(invoking),
|
|
1633
|
+
deliveryId: this.newId(),
|
|
1634
|
+
};
|
|
1635
|
+
}
|
|
1636
|
+
// A refused pick names the Allowed picks, read after the refusal (ADR-0029): one refusal
|
|
1637
|
+
// naming them is what brings a wrong pick back — the statement in the Briefing carries the
|
|
1638
|
+
// everyday case, and this is the net under it.
|
|
1639
|
+
if (err instanceof EventValidationError && invoking) err.allowed = allowedOf(invoking);
|
|
1640
|
+
throw err;
|
|
1641
|
+
} finally {
|
|
1642
|
+
this.sending = outer;
|
|
1643
|
+
}
|
|
634
1644
|
// Read AFTER the delivery, off the SAME table the ADR-0024 guarantee uses — so the receipt
|
|
635
1645
|
// reports what happened rather than what was hoped. `deliver` reached the invoking state's
|
|
636
1646
|
// `sendBack` synchronously, so a pick that moved the Machine out of that state has already
|
|
@@ -638,15 +1648,331 @@ export class RunHost {
|
|
|
638
1648
|
//
|
|
639
1649
|
// IDENTITY, not existence: under `continue: true` the next state re-registers the SAME
|
|
640
1650
|
// address for its own turn, and that is a new turn — this one still ended.
|
|
1651
|
+
const turnComplete = this.table.lookup(address) !== invoking;
|
|
1652
|
+
// A reply from below the Agent's state inside the pick's own send (ADR-0024): a child that
|
|
1653
|
+
// settled in the same macrostep. It is the answer's default as any later reply would be.
|
|
1654
|
+
const replied = moved && sending ? answerOf(sending) : undefined;
|
|
1655
|
+
// Where the answer goes if the route leaves the Agent's state: the conversation the picking
|
|
1656
|
+
// Turn continues. A fresh Turn has none — its conversation is never heard again, and a later
|
|
1657
|
+
// continued Turn of the same slot never made this pick.
|
|
1658
|
+
const conversation = invoking?.conversation;
|
|
1659
|
+
// The pick moved the workflow to work inside the Agent's state (ADR-0024): it is OUTSTANDING,
|
|
1660
|
+
// answered once the workflow is back at Rest there — read after the mailbox drains, by the
|
|
1661
|
+
// persist that follows every transition (`settleHolds`). The POST answers now, with the delivery
|
|
1662
|
+
// id the Harness polls; no single request waits on the work (ADR-0013).
|
|
1663
|
+
if (!turnComplete && invoking && run && moved && restOf(invoking) === "busy") {
|
|
1664
|
+
const deliveryId = this.newId();
|
|
1665
|
+
const since = new Date(this.now()).toISOString();
|
|
1666
|
+
const hold: Hold = {
|
|
1667
|
+
instanceId,
|
|
1668
|
+
deliveryId,
|
|
1669
|
+
pick: type,
|
|
1670
|
+
agent: invoking.sender?.id ?? "",
|
|
1671
|
+
since,
|
|
1672
|
+
...(invoking.sandbox !== undefined ? { sandbox: invoking.sandbox } : {}),
|
|
1673
|
+
...(invoking.invokeId !== undefined ? { invokeId: invoking.invokeId } : {}),
|
|
1674
|
+
...(conversation !== undefined ? { conversation } : {}),
|
|
1675
|
+
...(sending && sending.chain.length > 0 ? { chain: [...sending.chain] } : {}),
|
|
1676
|
+
...(sending?.shaped !== undefined ? { shaped: sending.shaped } : {}),
|
|
1677
|
+
};
|
|
1678
|
+
this.closeHold(run.holds.get(instanceId));
|
|
1679
|
+
run.holds.set(instanceId, hold);
|
|
1680
|
+
this.holdRegs.set(hold, invoking);
|
|
1681
|
+
this.persist(run);
|
|
1682
|
+
return {
|
|
1683
|
+
delivered: true,
|
|
1684
|
+
event: type,
|
|
1685
|
+
moved: true,
|
|
1686
|
+
turnComplete: false,
|
|
1687
|
+
allowed: [],
|
|
1688
|
+
deliveryId,
|
|
1689
|
+
outstanding: true,
|
|
1690
|
+
since,
|
|
1691
|
+
};
|
|
1692
|
+
}
|
|
1693
|
+
// The child replied and the route ran inside the send: the pick is answered on its own POST,
|
|
1694
|
+
// as its poll would have answered it — back at Rest with the Allowed picks read after the
|
|
1695
|
+
// route, or out of the Agent's state with the answer as a notice to the conversation the picking
|
|
1696
|
+
// Turn continued (none for a fresh Turn).
|
|
1697
|
+
if (replied !== undefined && invoking) {
|
|
1698
|
+
if (turnComplete && conversation !== undefined && run) {
|
|
1699
|
+
run.notices.raise(
|
|
1700
|
+
{ kind: "pick-answer", scope: "conversation", pick: type, cut: false, answer: replied },
|
|
1701
|
+
conversation,
|
|
1702
|
+
);
|
|
1703
|
+
}
|
|
1704
|
+
return {
|
|
1705
|
+
delivered: true,
|
|
1706
|
+
event: type,
|
|
1707
|
+
moved: true,
|
|
1708
|
+
turnComplete,
|
|
1709
|
+
allowed: turnComplete ? [] : allowedOf(invoking),
|
|
1710
|
+
deliveryId: this.newId(),
|
|
1711
|
+
answered: true,
|
|
1712
|
+
answer: replied,
|
|
1713
|
+
};
|
|
1714
|
+
}
|
|
1715
|
+
// It entered a state below the Agent's and came straight back inside its own send — a guard
|
|
1716
|
+
// back (an `always`), nothing replying: answered, "moved; nothing ran" (ADR-0024). A pick that
|
|
1717
|
+
// changed the state value moved to a mode, and one whose target is still active (the state it
|
|
1718
|
+
// was taken in) entered nothing it left: both get the plain receipt below.
|
|
1719
|
+
if (
|
|
1720
|
+
!turnComplete &&
|
|
1721
|
+
invoking &&
|
|
1722
|
+
before !== undefined &&
|
|
1723
|
+
stateValueOf(invoking.invoker) === before &&
|
|
1724
|
+
leftAgain(invoking.invoker, targets)
|
|
1725
|
+
) {
|
|
1726
|
+
return {
|
|
1727
|
+
delivered: true,
|
|
1728
|
+
event: type,
|
|
1729
|
+
moved: true,
|
|
1730
|
+
turnComplete: false,
|
|
1731
|
+
allowed: allowedOf(invoking),
|
|
1732
|
+
deliveryId: this.newId(),
|
|
1733
|
+
answered: true,
|
|
1734
|
+
straightBack: true,
|
|
1735
|
+
};
|
|
1736
|
+
}
|
|
641
1737
|
return {
|
|
642
1738
|
delivered: true,
|
|
643
1739
|
event: type,
|
|
644
1740
|
moved,
|
|
645
|
-
turnComplete
|
|
1741
|
+
turnComplete,
|
|
1742
|
+
// Read off the SAME registration, after the delivery: a pick that moved the Machine within
|
|
1743
|
+
// the invoking state may have changed its guards. A Turn that is over may pick nothing.
|
|
1744
|
+
allowed: turnComplete || !invoking ? [] : allowedOf(invoking),
|
|
646
1745
|
deliveryId: this.newId(),
|
|
647
1746
|
};
|
|
648
1747
|
}
|
|
649
1748
|
|
|
1749
|
+
/**
|
|
1750
|
+
* Answer every outstanding pick of `run` that can be answered now (ADR-0024). Called by the persist that
|
|
1751
|
+
* follows every transition — a microtask after the snapshot, so the mailbox has drained — and
|
|
1752
|
+
* before any read of a hold, so no reader sees a hold the Machine already answered.
|
|
1753
|
+
*
|
|
1754
|
+
* - The Agent's state is busy: still outstanding.
|
|
1755
|
+
* - Back at Rest in it: answered with the last reply since the pick and the Allowed picks read
|
|
1756
|
+
* AFTER the route — the guards read the context the route wrote (ADR-0029).
|
|
1757
|
+
* - It left the Agent's state (the registration is gone, or is another invocation's): the Turn
|
|
1758
|
+
* is over, and the answer goes as a notice to the next Turn of the conversation the picking
|
|
1759
|
+
* Turn continued; a fresh picking Turn's answer goes nowhere (ADR-0062).
|
|
1760
|
+
*/
|
|
1761
|
+
private settleHolds(run: LiveRun): void {
|
|
1762
|
+
for (const hold of run.holds.values()) {
|
|
1763
|
+
if (hold.answer !== undefined) continue;
|
|
1764
|
+
const reg = this.table.lookup(agentAddress(hold.instanceId));
|
|
1765
|
+
let same = reg !== undefined && reg.runId === run.record.runId && reg.invokeId === hold.invokeId;
|
|
1766
|
+
if (same) {
|
|
1767
|
+
const opened = this.holdRegs.get(hold);
|
|
1768
|
+
if (opened === undefined) this.holdRegs.set(hold, reg!);
|
|
1769
|
+
else same = opened === reg;
|
|
1770
|
+
}
|
|
1771
|
+
const reading = same ? restOf(reg!) : "gone";
|
|
1772
|
+
if (reading === "busy") continue;
|
|
1773
|
+
// The answer's default is the last reply since the pick, whatever moved the Machine after it
|
|
1774
|
+
// (ADR-0024); with none, the answer carries no reply. Never `straightBack`: an outstanding
|
|
1775
|
+
// pick's work was running, so "nothing ran" is not this answer's to say.
|
|
1776
|
+
const answered = answerOf(hold);
|
|
1777
|
+
const answer = answered ? { answer: answered } : {};
|
|
1778
|
+
if (reading === "gone") {
|
|
1779
|
+
// Left by the route of the last reply — its own events and the events it sent itself at once
|
|
1780
|
+
// included — or by something else while work below still ran (a timer the Machine set, an
|
|
1781
|
+
// outside event): then the exit stopped that work, and the answer says so. It keeps the last
|
|
1782
|
+
// reply an earlier link gave.
|
|
1783
|
+
const stopped = hold.cutShort === true ? { stopped: true as const } : {};
|
|
1784
|
+
hold.answer = { turnComplete: true, allowed: [], ...answer, ...stopped };
|
|
1785
|
+
if (hold.conversation !== undefined) {
|
|
1786
|
+
run.notices.raise(
|
|
1787
|
+
{
|
|
1788
|
+
kind: "pick-answer",
|
|
1789
|
+
scope: "conversation",
|
|
1790
|
+
pick: hold.pick,
|
|
1791
|
+
cut: false,
|
|
1792
|
+
...answer,
|
|
1793
|
+
...stopped,
|
|
1794
|
+
},
|
|
1795
|
+
hold.conversation,
|
|
1796
|
+
);
|
|
1797
|
+
}
|
|
1798
|
+
} else {
|
|
1799
|
+
hold.answer = { turnComplete: false, allowed: allowedOf(reg!), ...answer };
|
|
1800
|
+
}
|
|
1801
|
+
// The answer is fixed: nothing reads the chain or the shaped value again, so the next save
|
|
1802
|
+
// drops them. A loop below the Agent's state grows the blob by its tails only while one pick
|
|
1803
|
+
// is outstanding (ADR-0067, ADR-0065).
|
|
1804
|
+
delete hold.chain;
|
|
1805
|
+
delete hold.shaped;
|
|
1806
|
+
this.closeHold(hold);
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
/**
|
|
1811
|
+
* The pick an `answer` action in `machine` shapes (ADR-0067), from `seat`, the one place it runs
|
|
1812
|
+
* from (`placeAnswers`). At an Agent's state: only a pick of the Agent seated there — the one in
|
|
1813
|
+
* its own send, else the one outstanding. On a transition below no Agent's state (`null`): the
|
|
1814
|
+
* event that took the transition names the Agent — the sender of a pick, or the Agent whose pick
|
|
1815
|
+
* started the child whose done or error event it is — and that Agent's pick is the one shaped, and
|
|
1816
|
+
* no other; none when that Agent has no pick being answered. When one slot is invoked in two
|
|
1817
|
+
* regions, the event names an invoke too, and only the pick at that invoke is shaped: the sending
|
|
1818
|
+
* pick, or the pick at the invoke whose pick started the child; none when that pick is not being
|
|
1819
|
+
* answered (its Turn ended, and its child runs on). An event that names no Agent (a timer,
|
|
1820
|
+
* a Gate's answer, an outside send) shapes the pick in its own send. Outside one, it shapes the one
|
|
1821
|
+
* outstanding pick only while one Agent's state is active; while two Agents' states are active it
|
|
1822
|
+
* shapes none, whatever is outstanding: the event cannot say whose pick it is, and the other
|
|
1823
|
+
* Agent's pick keeps its default answer.
|
|
1824
|
+
*/
|
|
1825
|
+
private answeringIn(
|
|
1826
|
+
machine: AnyActorRef,
|
|
1827
|
+
seat: AnswerSeat,
|
|
1828
|
+
runId: string,
|
|
1829
|
+
holds: Map<string, Hold>,
|
|
1830
|
+
startedBy: Record<string, StartedBy>,
|
|
1831
|
+
event?: AnyEventObject,
|
|
1832
|
+
): Sending | Hold | undefined {
|
|
1833
|
+
const seatedFor = (reg: Registration | undefined): boolean => {
|
|
1834
|
+
if (reg?.invoker !== machine) return false;
|
|
1835
|
+
if (seat === null) return true;
|
|
1836
|
+
const snapshot = machine.getSnapshot() as AnyMachineSnapshot;
|
|
1837
|
+
return reg.invokeId !== undefined && seatOf(snapshot, reg.invokeId) === seat;
|
|
1838
|
+
};
|
|
1839
|
+
const regOf = (hold: Hold): Registration | undefined =>
|
|
1840
|
+
this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
|
|
1841
|
+
const sending = this.sending && this.sending.reg.runId === runId ? this.sending : undefined;
|
|
1842
|
+
const open = [...holds.values()].filter((hold) => hold.answer === undefined && seatedFor(regOf(hold)));
|
|
1843
|
+
if (seat === null && event !== undefined) {
|
|
1844
|
+
const named = agentOfEvent(event, startedBy, machine, sending);
|
|
1845
|
+
if (named !== undefined) {
|
|
1846
|
+
const { agent } = named;
|
|
1847
|
+
const own = sending && sending.reg.sender?.id === agent ? sending : undefined;
|
|
1848
|
+
// That Agent's picks being answered: the one in its own send, and the ones outstanding at
|
|
1849
|
+
// its other invokes (one slot invoked in two regions). The event names the invoke of the
|
|
1850
|
+
// pick that sent it or started its child, and only the pick at that invoke is shaped; none
|
|
1851
|
+
// when that pick is not being answered, and each other pick keeps its default answer.
|
|
1852
|
+
const held = open.filter((hold) => hold.agent === agent && hold.instanceId !== own?.reg.id);
|
|
1853
|
+
const picks = [...(own ? [own] : []), ...held];
|
|
1854
|
+
const at = (pick: Sending | Hold): string | undefined =>
|
|
1855
|
+
pick === own ? own.reg.invokeId : (pick as Hold).invokeId;
|
|
1856
|
+
const same = named.invoke === undefined ? picks : picks.filter((pick) => at(pick) === named.invoke);
|
|
1857
|
+
if (same.length !== 1) return undefined;
|
|
1858
|
+
const pick = same[0]!;
|
|
1859
|
+
if (pick === own) return seatedFor(own.reg) ? own : undefined;
|
|
1860
|
+
return pick;
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
if (sending && seatedFor(sending.reg)) return sending;
|
|
1864
|
+
if (seat === null && activeAgentStates(machine.getSnapshot() as AnyMachineSnapshot) > 1) return undefined;
|
|
1865
|
+
return open.length === 1 ? open[0] : undefined;
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
/**
|
|
1869
|
+
* The Agent whose outstanding pick holds the child `childId` that `machine` just started, or
|
|
1870
|
+
* undefined: the child is invoked on a state strictly below that Agent's state, so by the Rest rule
|
|
1871
|
+
* it is that pick's work (ADR-0024). Below two such states (one Agent's state inside another's),
|
|
1872
|
+
* it is the work of the nearer one's pick. `snapshot` is `machine`'s snapshot after the step.
|
|
1873
|
+
*/
|
|
1874
|
+
private outstandingAbove(
|
|
1875
|
+
machine: AnyActorRef,
|
|
1876
|
+
snapshot: AnyMachineSnapshot,
|
|
1877
|
+
childId: string,
|
|
1878
|
+
holds: Map<string, Hold>,
|
|
1879
|
+
): StartedBy | undefined {
|
|
1880
|
+
const node = seatOf(snapshot, childId);
|
|
1881
|
+
if (node === undefined) return undefined;
|
|
1882
|
+
const byState = new Map<unknown, StartedBy>();
|
|
1883
|
+
for (const hold of holds.values()) {
|
|
1884
|
+
if (hold.answer !== undefined) continue;
|
|
1885
|
+
const reg = this.holdRegs.get(hold) ?? this.table.lookup(agentAddress(hold.instanceId));
|
|
1886
|
+
if (reg?.invoker !== machine || reg.invokeId === undefined) continue;
|
|
1887
|
+
const agentState = seatOf(snapshot, reg.invokeId);
|
|
1888
|
+
if (agentState !== undefined) byState.set(agentState, { agent: hold.agent, invoke: reg.invokeId });
|
|
1889
|
+
}
|
|
1890
|
+
// The nearest Agent's state above the child: with B's state inside A's, the child is B's work.
|
|
1891
|
+
for (let n = node.parent; n; n = n.parent) {
|
|
1892
|
+
const starter = byState.get(n);
|
|
1893
|
+
if (starter !== undefined) return starter;
|
|
1894
|
+
}
|
|
1895
|
+
return undefined;
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
/** Wake every long poll on a hold: it was answered, replaced, or the host is going away. */
|
|
1899
|
+
private closeHold(hold: Hold | undefined): void {
|
|
1900
|
+
if (hold === undefined) return;
|
|
1901
|
+
const waiters = this.holdWaiters.get(hold);
|
|
1902
|
+
this.holdWaiters.delete(hold);
|
|
1903
|
+
for (const wake of waiters ?? []) wake();
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
/** An outstanding pick's delivery: its run and hold, by Agent address and delivery id. */
|
|
1907
|
+
private holdAt(instanceId: string, deliveryId: string): { run: LiveRun; hold: Hold } | undefined {
|
|
1908
|
+
for (const run of this.runs.values()) {
|
|
1909
|
+
const hold = run.holds.get(instanceId);
|
|
1910
|
+
if (hold?.deliveryId === deliveryId) return { run, hold };
|
|
1911
|
+
}
|
|
1912
|
+
return undefined;
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
/** Is this delivery an outstanding pick of a run this host parked? Its poll is asked again. */
|
|
1916
|
+
parkedDelivery(instanceId: string, deliveryId: string): boolean {
|
|
1917
|
+
return this.parked.has(`${instanceId} ${deliveryId}`);
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
/** The Sandbox that may read a delivery's answer (ADR-0013), or undefined for an unknown one. */
|
|
1921
|
+
deliveryScope(instanceId: string, deliveryId: string): { sandbox?: string } | undefined {
|
|
1922
|
+
const found = this.holdAt(instanceId, deliveryId);
|
|
1923
|
+
return found && { ...(found.hold.sandbox !== undefined ? { sandbox: found.hold.sandbox } : {}) };
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
/**
|
|
1927
|
+
* One long poll of an outstanding pick's answer (`GET /agents/:iid/deliveries/:id` — ADR-0013, ADR-0024):
|
|
1928
|
+
* the answer when the workflow is back at Rest in the Agent's state or has left it, else
|
|
1929
|
+
* `outstanding` after at most `waitMs`, and the Harness asks again. Undefined for a delivery this
|
|
1930
|
+
* host does not hold — a pick that was never outstanding, or a run that ended.
|
|
1931
|
+
*
|
|
1932
|
+
* `signal` is the poll's connection. A poll whose caller went away (a killed Harness) ends then,
|
|
1933
|
+
* `outstanding`, and never marks the answer read: no model read it, so the re-admission after
|
|
1934
|
+
* the restart must still carry it (ADR-0031).
|
|
1935
|
+
*/
|
|
1936
|
+
async awaitDelivery(
|
|
1937
|
+
instanceId: string,
|
|
1938
|
+
deliveryId: string,
|
|
1939
|
+
waitMs = 25_000,
|
|
1940
|
+
signal?: AbortSignal,
|
|
1941
|
+
): Promise<AgentDeliveryReceipt | undefined> {
|
|
1942
|
+
const found = this.holdAt(instanceId, deliveryId);
|
|
1943
|
+
if (!found) return undefined;
|
|
1944
|
+
const { run, hold } = found;
|
|
1945
|
+
this.settleHolds(run);
|
|
1946
|
+
if (hold.answer === undefined && !this.#stopping && !signal?.aborted) {
|
|
1947
|
+
await new Promise<void>((resolve) => {
|
|
1948
|
+
let waiters = this.holdWaiters.get(hold);
|
|
1949
|
+
if (!waiters) this.holdWaiters.set(hold, (waiters = new Set()));
|
|
1950
|
+
const timer = setTimeout(wake, waitMs);
|
|
1951
|
+
signal?.addEventListener("abort", wake, { once: true });
|
|
1952
|
+
function wake(): void {
|
|
1953
|
+
clearTimeout(timer);
|
|
1954
|
+
signal?.removeEventListener("abort", wake);
|
|
1955
|
+
waiters!.delete(wake);
|
|
1956
|
+
resolve();
|
|
1957
|
+
}
|
|
1958
|
+
waiters.add(wake);
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
const base = { delivered: true as const, event: hold.pick, moved: true, deliveryId };
|
|
1962
|
+
if (hold.answer === undefined || signal?.aborted) {
|
|
1963
|
+
return { ...base, turnComplete: false, allowed: [], outstanding: true, since: hold.since };
|
|
1964
|
+
}
|
|
1965
|
+
hold.fetched = true;
|
|
1966
|
+
return {
|
|
1967
|
+
...base,
|
|
1968
|
+
turnComplete: hold.answer.turnComplete,
|
|
1969
|
+
allowed: hold.answer.allowed,
|
|
1970
|
+
answered: true,
|
|
1971
|
+
...(hold.answer.answer !== undefined ? { answer: hold.answer.answer } : {}),
|
|
1972
|
+
...(hold.answer.stopped ? { stopped: true as const } : {}),
|
|
1973
|
+
};
|
|
1974
|
+
}
|
|
1975
|
+
|
|
650
1976
|
/** A run's open gates as callers discover them (`GET /runs/:id` — ADR-0011). Settled/unknown
|
|
651
1977
|
* run → empty: a gate is a LIVE surface, it does not outlive its state. */
|
|
652
1978
|
gates(runId: string): GateView[] {
|
|
@@ -661,20 +1987,50 @@ export class RunHost {
|
|
|
661
1987
|
description: def.description,
|
|
662
1988
|
input: z.toJSONSchema(def.input),
|
|
663
1989
|
})),
|
|
1990
|
+
acceptsNow: acceptsNow(reg),
|
|
664
1991
|
meta: reg.meta,
|
|
665
1992
|
}));
|
|
666
1993
|
}
|
|
667
1994
|
|
|
668
|
-
/**
|
|
669
|
-
*
|
|
1995
|
+
/**
|
|
1996
|
+
* Deliver one external event to a run's open gate (`POST /runs/:id/gates/:gate/events`).
|
|
1997
|
+
* Validation and delivery are the table's — this run-scopes the address, and returns only when
|
|
1998
|
+
* the run TOOK the event (ADR-0013): one that would run another party's writing, or that no
|
|
1999
|
+
* writing would take with this payload, throws {@link NotTakenError} naming what the Gate takes
|
|
2000
|
+
* now, and nothing is delivered.
|
|
2001
|
+
*/
|
|
670
2002
|
sendToGate(runId: string, gate: string, event: { type?: unknown } & Record<string, unknown>): void {
|
|
671
2003
|
const { type, ...payload } = event;
|
|
672
2004
|
if (typeof type !== "string" || !type) {
|
|
673
2005
|
throw new EventValidationError(`event body must carry a string "type" (one of the gate's accepted names)`);
|
|
674
2006
|
}
|
|
2007
|
+
const reg = this.table.lookup(gateAddress(runId, gate));
|
|
2008
|
+
const refused = (why: string): NotTakenError =>
|
|
2009
|
+
new NotTakenError(
|
|
2010
|
+
`gate "${gate}" is open, and the run does not take "${type}" now: ${why} — not delivered ` +
|
|
2011
|
+
`(takes now: ${reg ? acceptsNow(reg).join(", ") || "nothing" : "nothing"})`,
|
|
2012
|
+
);
|
|
675
2013
|
try {
|
|
2014
|
+
// Exact, with the validated payload: a send no writing would move on is refused, never
|
|
2015
|
+
// answered `{ ok: true }`. A bad payload or an unaccepted name falls through to the table,
|
|
2016
|
+
// which refuses it with its own words.
|
|
2017
|
+
const def = reg?.defs.get(type);
|
|
2018
|
+
const parsed = def && !(payload && SENDER_KEY in payload) ? def.input.safeParse(payload ?? {}) : undefined;
|
|
2019
|
+
if (reg && parsed?.success && !wouldMove(reg.invoker, { type, ...parsed.data, [SENDER_KEY]: reg.sender })) {
|
|
2020
|
+
throw refused("no writing of it would move the run with this payload");
|
|
2021
|
+
}
|
|
676
2022
|
this.table.deliver(gateAddress(runId, gate), type, payload);
|
|
677
2023
|
} catch (err) {
|
|
2024
|
+
if (err instanceof NotTakenError && err.writings.length) {
|
|
2025
|
+
throw refused(
|
|
2026
|
+
err.writings
|
|
2027
|
+
.map(
|
|
2028
|
+
(w) =>
|
|
2029
|
+
`it would run the writing on "${w.state}"${w.owners.length ? `, which is ${w.owners.map(senderText).join(" and ")}'s` : ""}`,
|
|
2030
|
+
)
|
|
2031
|
+
.join("; "),
|
|
2032
|
+
);
|
|
2033
|
+
}
|
|
678
2034
|
if (err instanceof UnknownAddressError) {
|
|
679
2035
|
const open = this.gates(runId).map((g) => g.gate);
|
|
680
2036
|
throw new UnknownAddressError(
|
|
@@ -697,7 +2053,10 @@ export class RunHost {
|
|
|
697
2053
|
const run = this.runs.get(runId);
|
|
698
2054
|
if (!run) return () => {};
|
|
699
2055
|
run.listeners.add(listener);
|
|
700
|
-
|
|
2056
|
+
// The status the feed last carried, the SAME object for every attach, so http.ts renders the
|
|
2057
|
+
// replay once however many readers attach — a reconnect after a close is the common attach
|
|
2058
|
+
// (ADR-0022). A transition not yet persisted follows on the feed within the microtask.
|
|
2059
|
+
listener(run.latestStatus ?? { kind: "status", status: this.liveStatus(run) });
|
|
701
2060
|
return () => run.listeners.delete(listener);
|
|
702
2061
|
}
|
|
703
2062
|
|
|
@@ -752,7 +2111,8 @@ export class RunHost {
|
|
|
752
2111
|
* Read a run's status, **reading through to the store** when it is no longer live (ADR-0009). A
|
|
753
2112
|
* completed run's final snapshot is persisted before `persist()` drops it from the registry, so a
|
|
754
2113
|
* terminal run reports its `done`/`error` status + final context here rather than 404-ing. Returns
|
|
755
|
-
* undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared)
|
|
2114
|
+
* undefined only for a genuinely unknown run (or one marked `lost`, whose snapshot was cleared) —
|
|
2115
|
+
* and a swept one is unknown: a finished run's row lives a week (ADR-0065).
|
|
756
2116
|
*/
|
|
757
2117
|
async read(runId: string): Promise<RunStatus | undefined> {
|
|
758
2118
|
const live = this.status(runId);
|
|
@@ -761,19 +2121,24 @@ export class RunHost {
|
|
|
761
2121
|
const blob = stored?.snapshot as RunBlob | null | undefined;
|
|
762
2122
|
if (!stored || !blob) return undefined;
|
|
763
2123
|
const snap = (blob.snapshot ?? {}) as { status?: string; value?: unknown; context?: unknown };
|
|
2124
|
+
// The STORE row is the authority on the run's lifecycle, the snapshot on the Machine's: a
|
|
2125
|
+
// cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
|
|
2126
|
+
// (ADR-0025). While the row still says "live", the Machine's own status is the answer.
|
|
2127
|
+
const status = stored.status === "live" ? (snap.status ?? stored.status) : stored.status;
|
|
764
2128
|
return {
|
|
765
2129
|
runId,
|
|
766
2130
|
workflow: blob.workflow,
|
|
767
2131
|
instanceId: blob.instanceId,
|
|
768
|
-
|
|
769
|
-
// cancelled run's actor reports xstate's "stopped", which is mechanism, not an outcome
|
|
770
|
-
// (ADR-0025). While the row still says "live", the Machine's own status is the answer.
|
|
771
|
-
status: stored.status === "live" ? (snap.status ?? stored.status) : stored.status,
|
|
2132
|
+
status,
|
|
772
2133
|
value: snap.value,
|
|
773
2134
|
context: snap.context,
|
|
774
2135
|
children: runChildren(snap), // the persisted `children` map — same tree, off the store
|
|
2136
|
+
waiting: status === "active" ? runWaiting(snap) : [],
|
|
2137
|
+
held: status === "active" ? heldOf(blob.holds ?? []) : [],
|
|
775
2138
|
fault: blob.fault,
|
|
776
2139
|
reason: stored.reason,
|
|
2140
|
+
...(blob.assembly !== undefined ? { assembly: blob.assembly } : {}),
|
|
2141
|
+
...boundFields(snap),
|
|
777
2142
|
};
|
|
778
2143
|
}
|
|
779
2144
|
|
|
@@ -803,6 +2168,25 @@ export class RunHost {
|
|
|
803
2168
|
.map(observe);
|
|
804
2169
|
}
|
|
805
2170
|
|
|
2171
|
+
/** The host is going away (`close()` has begun): its Turns are not over, and the Agents' surface
|
|
2172
|
+
* must not say they are (ADR-0026). */
|
|
2173
|
+
get stopping(): boolean {
|
|
2174
|
+
return this.#stopping;
|
|
2175
|
+
}
|
|
2176
|
+
|
|
2177
|
+
/** The live rows of a prior process are not restored yet: a boot with no current Assembly
|
|
2178
|
+
* deferred their restore to the first publish (ADR-0069). Those Turns are not over, so the
|
|
2179
|
+
* Agents' surface must not say they are (ADR-0026). Set by `deferRestore`, cleared when a
|
|
2180
|
+
* `restore()` completes; a restore that throws as a whole leaves it set. */
|
|
2181
|
+
get restorePending(): boolean {
|
|
2182
|
+
return this.#restorePending;
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
/** Mark the restore deferred: until a `restore()` completes, `restorePending` is true. */
|
|
2186
|
+
deferRestore(): void {
|
|
2187
|
+
this.#restorePending = true;
|
|
2188
|
+
}
|
|
2189
|
+
|
|
806
2190
|
/**
|
|
807
2191
|
* End every open feed because the host is going away — the shutdown counterpart to `subscribe`.
|
|
808
2192
|
*
|
|
@@ -813,6 +2197,15 @@ export class RunHost {
|
|
|
813
2197
|
* OBSERVATION, not the work — the snapshots are already durable, and `restore()` picks them up.
|
|
814
2198
|
*/
|
|
815
2199
|
async close(): Promise<void> {
|
|
2200
|
+
// From here on the Orchestrator is going away, not its Turns: `/agents/*` answers 503, which a
|
|
2201
|
+
// Harness re-asks on ADR-0042's ladder, and never 404, which says a Turn is over (ADR-0026).
|
|
2202
|
+
this.#stopping = true;
|
|
2203
|
+
clearTimeout(this.liveSetTimer);
|
|
2204
|
+
// A run's timers stay armed (a run is untouched by a close), but none holds the process alive.
|
|
2205
|
+
for (const timer of this.timers) (timer as { unref?: () => void }).unref?.();
|
|
2206
|
+
// Every long poll answers `outstanding` now, so the server can close; the Harness asks again,
|
|
2207
|
+
// and a restarted Orchestrator answers it from the same hold (ADR-0024).
|
|
2208
|
+
for (const hold of [...this.holdWaiters.keys()]) this.closeHold(hold);
|
|
816
2209
|
for (const run of this.runs.values()) {
|
|
817
2210
|
for (const listener of run.listeners) listener({ kind: "closed" });
|
|
818
2211
|
run.listeners.clear();
|
|
@@ -839,6 +2232,14 @@ export class RunHost {
|
|
|
839
2232
|
const run = this.runs.get(runId);
|
|
840
2233
|
if (!run) return;
|
|
841
2234
|
run.binding.hostStopping = true;
|
|
2235
|
+
// A parked run's outstanding picks are not over (ADR-0024): their polls are answered `503` —
|
|
2236
|
+
// ask again — never `404`, which tells the Harness its Turn ended. The host that restores the
|
|
2237
|
+
// run answers them from the saved holds.
|
|
2238
|
+
for (const hold of run.holds.values()) {
|
|
2239
|
+
if (hold.answer !== undefined) continue;
|
|
2240
|
+
this.parked.add(`${hold.instanceId} ${hold.deliveryId}`);
|
|
2241
|
+
this.closeHold(hold);
|
|
2242
|
+
}
|
|
842
2243
|
// Say so BEFORE the actor stops, while there is still a status to read. A stopped run is not a
|
|
843
2244
|
// settled one — `persist()` never runs for it (the tracked-run guard drops the scheduled save,
|
|
844
2245
|
// keeping the stored status "live" for restore — see `spawn`), so nothing else on this path
|
|
@@ -857,14 +2258,37 @@ export class RunHost {
|
|
|
857
2258
|
* Ending the turns and refusing to restore are one decision, not two: a cancelled run that came
|
|
858
2259
|
* back would re-attach to submissions that settled `aborted`, and `settle`'s rejection would
|
|
859
2260
|
* fault a run whose Agents were stopped on purpose.
|
|
2261
|
+
*
|
|
2262
|
+
* It then destroys every Sandbox the run owns, unless `keep` (`jr2 send <run> --event CANCEL
|
|
2263
|
+
* --keep`): kept, they age out at the operator's idle timeout, and `jr2 ssh` reaches them until
|
|
2264
|
+
* then. A faulted run never comes through here, and keeps its Workspaces (ADR-0012).
|
|
860
2265
|
*/
|
|
861
|
-
async cancel(runId: string): Promise<void> {
|
|
2266
|
+
async cancel(runId: string, opts: { keep?: boolean } = {}): Promise<void> {
|
|
862
2267
|
const run = this.runs.get(runId);
|
|
863
2268
|
if (!run) return;
|
|
2269
|
+
// Named BEFORE the stop, while the tree still says which Workspaces are live.
|
|
2270
|
+
const sandboxes = opts.keep ? [] : runSandboxes(run.actor, runId);
|
|
864
2271
|
run.actor.stop();
|
|
865
2272
|
// Persist AFTER the stop: the snapshot is final, and `persist` fans out the last status,
|
|
866
2273
|
// announces `gone`, and untracks — the same terminal path a run that settled on its own takes.
|
|
867
2274
|
this.persist(run, "cancelled");
|
|
2275
|
+
// The run is over, so nothing can use its Workspaces, and each holds its whole Size (ADR-0025).
|
|
2276
|
+
// An actor stop skips the wrapper's `teardown` state, so the host runs the same idempotent
|
|
2277
|
+
// delete here, after the stop, so no lease renews what was deleted. `keep` leaves them to the
|
|
2278
|
+
// operator's idle timeout, for inspection. A delete that fails is the idle GC's, as in
|
|
2279
|
+
// `teardown` (ADR-0012 backstop): the cancel has already happened.
|
|
2280
|
+
const port = this.sandbox;
|
|
2281
|
+
if (!port) return;
|
|
2282
|
+
await Promise.all(
|
|
2283
|
+
sandboxes.map((name) =>
|
|
2284
|
+
port.destroy(name).catch((err: unknown) => {
|
|
2285
|
+
console.error(
|
|
2286
|
+
`run ${runId}: cancel could not delete Sandbox ${name} (the idle timeout will): ` +
|
|
2287
|
+
(err instanceof Error ? err.message : String(err)),
|
|
2288
|
+
);
|
|
2289
|
+
}),
|
|
2290
|
+
),
|
|
2291
|
+
);
|
|
868
2292
|
}
|
|
869
2293
|
|
|
870
2294
|
/** Feed a run's final word to both granularities: where it ended, then that it is gone. */
|
|
@@ -877,19 +2301,25 @@ export class RunHost {
|
|
|
877
2301
|
|
|
878
2302
|
/** Put one event on the run's feed: buffer it for ADR-0023's backfill, then fan it out. The
|
|
879
2303
|
* buffer and the fan-out share one seat so an attached echo and a later attach see the SAME
|
|
880
|
-
* feed — a projection cannot drift from a record it is read out of.
|
|
2304
|
+
* feed — a projection cannot drift from a record it is read out of. Every listener is handed the
|
|
2305
|
+
* SAME event object, which is what lets http.ts render each frame once (ADR-0022). */
|
|
881
2306
|
private feed(run: LiveRun, event: RunFeedEvent): void {
|
|
882
|
-
|
|
883
|
-
|
|
2307
|
+
if (event.kind === "status") {
|
|
2308
|
+
run.latestStatus = event;
|
|
2309
|
+
} else {
|
|
2310
|
+
run.feedSoFar.push(event);
|
|
2311
|
+
if (run.feedSoFar.length > FEED_SO_FAR_CAP) run.feedSoFar.shift();
|
|
2312
|
+
}
|
|
884
2313
|
for (const listener of run.listeners) listener(event);
|
|
885
2314
|
}
|
|
886
2315
|
|
|
887
2316
|
/**
|
|
888
2317
|
* Attach the run-narrative echo (ADR-0023): replay the run's feed-so-far to the Workspace's
|
|
889
|
-
* Harness at `endpoint` — the backfilled preamble, why this Workspace exists
|
|
890
|
-
*
|
|
891
|
-
*
|
|
892
|
-
*
|
|
2318
|
+
* Harness at `endpoint` — the backfilled preamble, why this Workspace exists: its markers and
|
|
2319
|
+
* Emits in feed order, then where the run stands now — then tee live until detached, through an
|
|
2320
|
+
* {@link echoOutbox}, which coalesces statuses. FIRE-AND-FORGET is this method's contract: a
|
|
2321
|
+
* failure is logged once and swallowed — the feed remains the record, the log is a courtesy
|
|
2322
|
+
* view, and nothing here can fail a turn, a state, or a run.
|
|
893
2323
|
* Scoped to the OWNING run's lineage by construction: it reads one run's buffer and listeners
|
|
894
2324
|
* and nothing else — a sibling run's events cannot reach this endpoint through here.
|
|
895
2325
|
*/
|
|
@@ -897,25 +2327,19 @@ export class RunHost {
|
|
|
897
2327
|
const run = this.runs.get(runId);
|
|
898
2328
|
const factory = this.echoFactory;
|
|
899
2329
|
if (!run || !factory) return () => {};
|
|
900
|
-
const push = factory(endpoint, this.harnessBearer?.(placement));
|
|
901
|
-
let chain = Promise.resolve();
|
|
902
2330
|
let reported = false;
|
|
903
|
-
const enqueue = (
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
);
|
|
916
|
-
});
|
|
917
|
-
};
|
|
918
|
-
enqueue(run.feedSoFar.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
|
|
2331
|
+
const enqueue = echoOutbox(factory(endpoint, this.harnessBearer?.(placement)), (err) => {
|
|
2332
|
+
// Log-and-continue, ONCE per attachment — an unreachable Harness must not turn every
|
|
2333
|
+
// transition into an error line, and must not surface anywhere a run could trip on.
|
|
2334
|
+
if (reported) return;
|
|
2335
|
+
reported = true;
|
|
2336
|
+
console.error(
|
|
2337
|
+
`run ${runId}: echo to ${endpoint} failed (log only — the run is unaffected): ` +
|
|
2338
|
+
(err instanceof Error ? err.message : String(err)),
|
|
2339
|
+
);
|
|
2340
|
+
});
|
|
2341
|
+
const preamble = run.latestStatus ? [...run.feedSoFar, run.latestStatus] : run.feedSoFar;
|
|
2342
|
+
enqueue(preamble.map((ev) => echoEventOf(ev, endpoint)).filter((ev): ev is EchoEvent => ev !== undefined));
|
|
919
2343
|
const listener = (ev: RunFeedEvent): void => {
|
|
920
2344
|
const projected = echoEventOf(ev, endpoint);
|
|
921
2345
|
if (projected) enqueue([projected]);
|
|
@@ -933,7 +2357,11 @@ export class RunHost {
|
|
|
933
2357
|
value: snap.value,
|
|
934
2358
|
context: snap.context,
|
|
935
2359
|
children: runChildren(snap),
|
|
2360
|
+
waiting: runWaiting(snap, run.actor.logic),
|
|
2361
|
+
held: heldOf(run.holds.values()),
|
|
936
2362
|
fault: run.fault,
|
|
2363
|
+
...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
|
|
2364
|
+
...boundFields(snap),
|
|
937
2365
|
};
|
|
938
2366
|
}
|
|
939
2367
|
|
|
@@ -958,7 +2386,13 @@ export class RunHost {
|
|
|
958
2386
|
def: WorkflowDef,
|
|
959
2387
|
agents: Record<string, AgentAdmission> = {},
|
|
960
2388
|
epochs: Record<string, number> = {},
|
|
2389
|
+
noticeState?: NoticeLedgerState,
|
|
2390
|
+
holdState: Hold[] = [],
|
|
2391
|
+
effects: Record<string, EffectRow> = {},
|
|
2392
|
+
startedBy: Record<string, StartedBy> = {},
|
|
961
2393
|
): AnyActor {
|
|
2394
|
+
const notices = new NoticeLedger(noticeState, this.newId);
|
|
2395
|
+
const holds = new Map(holdState.map((hold) => [hold.instanceId, hold]));
|
|
962
2396
|
let live: LiveRun | undefined;
|
|
963
2397
|
let scheduled = false;
|
|
964
2398
|
const schedule = () => {
|
|
@@ -972,30 +2406,47 @@ export class RunHost {
|
|
|
972
2406
|
const binding: RunBinding = {
|
|
973
2407
|
runId: record.runId,
|
|
974
2408
|
workflow: record.workflow,
|
|
2409
|
+
...(def.assembly !== undefined ? { assembly: def.assembly } : {}),
|
|
975
2410
|
table: this.table,
|
|
976
2411
|
sandbox: this.sandbox,
|
|
977
2412
|
instanceHarness: this.instanceHarness,
|
|
978
2413
|
...(this.harnessBearer ? { harnessBearer: this.harnessBearer } : {}),
|
|
979
|
-
|
|
2414
|
+
...(this.effector ? { effector: this.effector } : {}),
|
|
2415
|
+
// The admission ledger's write half (ADR-0016): the Agent's Actor reports the durable handle the
|
|
980
2416
|
// moment the Harness admits it, and the ledger hits the store in the same RunBlob save. An
|
|
981
2417
|
// admission arriving around stop/untrack still lands in `agents` but skips the save,
|
|
982
2418
|
// exactly like the persist scheduler's tracked-run guard.
|
|
983
|
-
recordAdmission: (instanceId, admission) => {
|
|
2419
|
+
recordAdmission: (instanceId, admission, delivered = []) => {
|
|
984
2420
|
agents[instanceId] = admission;
|
|
2421
|
+
// The notices this admission carried are delivered by THIS save (ADR-0062).
|
|
2422
|
+
notices.delivered(delivered);
|
|
985
2423
|
const run = this.runs.get(record.runId);
|
|
986
2424
|
if (run && run.agents === agents) this.persist(run);
|
|
987
2425
|
},
|
|
2426
|
+
admissionOf: (instanceId) => agents[instanceId],
|
|
988
2427
|
// The epoch ledger's two halves (ADR-0057). The read is what the input mapper mints from;
|
|
989
2428
|
// the write is the terminal `agent.fault` burning the conversation it ended, saved in the
|
|
990
2429
|
// same RunBlob as the admission ledger — and, like it, saved AT THE WRITE rather than
|
|
991
2430
|
// waiting for the next transition, because a crash between the fault and the next state's
|
|
992
2431
|
// invoke would otherwise restore into the burned conversation.
|
|
993
2432
|
epochOf: (conversation) => epochs[conversation] ?? 0,
|
|
994
|
-
bumpEpoch: (conversation) => {
|
|
2433
|
+
bumpEpoch: (conversation, reason) => {
|
|
995
2434
|
epochs[conversation] = (epochs[conversation] ?? 0) + 1;
|
|
2435
|
+
// The next `continue` lands on a virgin conversation and is told so, with the fault's own
|
|
2436
|
+
// words (ADR-0062) — raised in the SAME save as the bump, so no restart can split them.
|
|
2437
|
+
notices.raise({ kind: "conversation-new", scope: "conversation", reason }, conversation);
|
|
996
2438
|
const run = this.runs.get(record.runId);
|
|
997
2439
|
if (run && run.epochs === epochs) this.persist(run);
|
|
998
2440
|
},
|
|
2441
|
+
// The notices' other halves (ADR-0062). A raise is saved at the write, like the ledgers; a
|
|
2442
|
+
// take is a reservation in memory, delivered by `recordAdmission` or released on failure.
|
|
2443
|
+
raiseNotice: (notice, to, source) => {
|
|
2444
|
+
if (!notices.raise(notice, to, source)) return;
|
|
2445
|
+
const run = this.runs.get(record.runId);
|
|
2446
|
+
if (run && run.notices === notices) this.persist(run);
|
|
2447
|
+
},
|
|
2448
|
+
takeNotices: (scope) => notices.take(scope),
|
|
2449
|
+
releaseNotices: (ids) => notices.release(ids),
|
|
999
2450
|
// Absorbed-retry attempts go straight to the run's observers (SSE/CLI watch) — they are
|
|
1000
2451
|
// feed events, not machine events (ADR-0016: the workflow sees only the terminal fault).
|
|
1001
2452
|
telemetry: (event) => {
|
|
@@ -1005,15 +2456,83 @@ export class RunHost {
|
|
|
1005
2456
|
// Turn markers (ADR-0023) ride the per-run feed alone — the prompt is Instance-token-class
|
|
1006
2457
|
// data, so they never touch the workflow feed (ADR-0014's open band strips even Emit
|
|
1007
2458
|
// payloads there).
|
|
2459
|
+
// An Effect's line that comes after its run ended — a stop's answer on a cancel or a final
|
|
2460
|
+
// state (ADR-0067) — has no feed left to ride, so it goes to the Orchestrator's log.
|
|
1008
2461
|
marker: (event) => {
|
|
1009
2462
|
const run = this.runs.get(record.runId);
|
|
1010
2463
|
if (run && run.binding === binding) this.feed(run, event);
|
|
2464
|
+
else if (!run && event.kind === "effect") {
|
|
2465
|
+
console.error(`run ${record.runId}: effect ${event.slot} (${event.effect}) ${event.stage}: ${event.text}`);
|
|
2466
|
+
}
|
|
1011
2467
|
},
|
|
1012
2468
|
// The run-narrative echo attach (ADR-0023), called from `workspace()`'s registrar.
|
|
1013
2469
|
echo: (endpoint, placement) => this.attachEcho(record.runId, endpoint, placement),
|
|
2470
|
+
// The outstanding pick of the Turn now at this address, as the Agent's Actor reads it when its
|
|
2471
|
+
// Harness restarted (ADR-0024/0031). Settled first, so a hold whose Turn is over reads
|
|
2472
|
+
// `turnComplete`: that is an EARLIER Turn's pick (a route left its state, and the answer went as
|
|
2473
|
+
// a notice to the next Turn, or nowhere) — never this Turn's, whatever it continues.
|
|
2474
|
+
outstandingOf: (instanceId) => {
|
|
2475
|
+
const run = this.runs.get(record.runId);
|
|
2476
|
+
if (run && run.holds === holds) this.settleHolds(run);
|
|
2477
|
+
const hold = holds.get(instanceId);
|
|
2478
|
+
if (hold?.answer?.turnComplete) return undefined;
|
|
2479
|
+
return (
|
|
2480
|
+
hold && {
|
|
2481
|
+
pick: hold.pick,
|
|
2482
|
+
answered: hold.answer !== undefined,
|
|
2483
|
+
turnComplete: hold.answer?.turnComplete ?? false,
|
|
2484
|
+
...(hold.answer?.answer !== undefined ? { answer: hold.answer.answer } : {}),
|
|
2485
|
+
fetched: hold.fetched === true,
|
|
2486
|
+
}
|
|
2487
|
+
);
|
|
2488
|
+
},
|
|
2489
|
+
// The `answer` action's two halves (ADR-0067): the chain of the one pick being answered of an
|
|
2490
|
+
// Agent whose state the action is written below — the one in its own send first, else the one
|
|
2491
|
+
// outstanding — and the value that replaces its default answer.
|
|
2492
|
+
chainOf: (machine, seat, event) => {
|
|
2493
|
+
const pick = this.answeringIn(machine, seat, record.runId, holds, startedBy, event);
|
|
2494
|
+
if (pick === undefined) return undefined;
|
|
2495
|
+
return "reg" in pick ? [...pick.chain] : [...(pick.chain ?? [])];
|
|
2496
|
+
},
|
|
2497
|
+
setAnswer: (machine, seat, value, event) => {
|
|
2498
|
+
const pick = this.answeringIn(machine, seat, record.runId, holds, startedBy, event);
|
|
2499
|
+
if (pick === undefined) return;
|
|
2500
|
+
// Not saved here: the action runs inside a transition, and the persist that follows it carries
|
|
2501
|
+
// the value — a save now would read the Machine before its route, and answer too early.
|
|
2502
|
+
pick.shaped = jsonOf(value);
|
|
2503
|
+
},
|
|
2504
|
+
// The Effect ledger's two halves (ADR-0067). A write is saved AT THE WRITE, like an admission:
|
|
2505
|
+
// an intent row must be on disk, with the snapshot that holds its Effect, before the call
|
|
2506
|
+
// leaves. A row written inside a transition (an abort) waits for the save that carries that
|
|
2507
|
+
// transition — a cancel's final save included, which untracks the run — and is saved after
|
|
2508
|
+
// it only when nothing else did. A row written around a park lands in memory and skips the save.
|
|
2509
|
+
effectRow: (effectId) => effects[effectId],
|
|
2510
|
+
effectRows: () => effects,
|
|
2511
|
+
recordEffect: (effectId, row, save = "now") => {
|
|
2512
|
+
effects[effectId] = row;
|
|
2513
|
+
const persist = (): void => {
|
|
2514
|
+
const run = this.runs.get(record.runId);
|
|
2515
|
+
if (run && run.effects === effects) this.persist(run);
|
|
2516
|
+
};
|
|
2517
|
+
if (save === "now") persist();
|
|
2518
|
+
else queueMicrotask(persist);
|
|
2519
|
+
},
|
|
2520
|
+
heldAnswerDelivered: (instanceId) => {
|
|
2521
|
+
const hold = holds.get(instanceId);
|
|
2522
|
+
if (hold) hold.fetched = true;
|
|
2523
|
+
},
|
|
2524
|
+
// A Chart by name (ADR-0070, D-C): among the Charts of the wrapper that is the run's Workflow.
|
|
2525
|
+
chartNamed: (name) => this.namedChart(wrapperName(record.workflow), name)?.text,
|
|
1014
2526
|
};
|
|
1015
2527
|
let bound = false;
|
|
2528
|
+
let firedTimer: ((source: unknown, event: unknown) => boolean) | undefined;
|
|
2529
|
+
// The children each Machine of the tree had after its last microstep: what the next one started
|
|
2530
|
+
// is the difference (ADR-0067's attribution of a child to the pick whose work started it).
|
|
2531
|
+
const childrenAfter = new WeakMap<AnyActorRef, Set<AnyActorRef>>();
|
|
2532
|
+
const childRefsOf = (snapshot: unknown): Set<AnyActorRef> =>
|
|
2533
|
+
new Set(Object.values((snapshot as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {}));
|
|
1016
2534
|
const actor = createActor(machine, {
|
|
2535
|
+
clock: this.clock,
|
|
1017
2536
|
...options,
|
|
1018
2537
|
inspect: (ev) => {
|
|
1019
2538
|
// Bind the run's actor SYSTEM on the ROOT's creation event — the first inspection event,
|
|
@@ -1024,12 +2543,64 @@ export class RunHost {
|
|
|
1024
2543
|
if (!bound && ev.type === "@xstate.actor") {
|
|
1025
2544
|
bound = true;
|
|
1026
2545
|
bindRun((ev.actorRef as AnyActorRef).system, binding);
|
|
2546
|
+
// …and watch its scheduler before anything is armed: whether an event the Machine took
|
|
2547
|
+
// was a timer it set, or an event an actor sent at once, decides `cutShort` below.
|
|
2548
|
+
firedTimer = watchTimers((ev.actorRef as AnyActorRef).system);
|
|
2549
|
+
}
|
|
2550
|
+
// A restored hold is the Turn of the FIRST registration of its address in this tree: bind
|
|
2551
|
+
// it as soon as one exists, before an event can end that Turn and start another at the same
|
|
2552
|
+
// address (`continue: true` re-registers it). Every inspection event, so it is bound before
|
|
2553
|
+
// the restored Machine processes anything: the root's snapshot comes after its children
|
|
2554
|
+
// start and before its mailbox drains.
|
|
2555
|
+
for (const hold of holds.values()) {
|
|
2556
|
+
if (hold.answer !== undefined || this.holdRegs.has(hold)) continue;
|
|
2557
|
+
const reg = this.table.lookup(agentAddress(hold.instanceId));
|
|
2558
|
+
if (reg && reg.runId === record.runId && reg.invokeId === hold.invokeId) this.holdRegs.set(hold, reg);
|
|
1027
2559
|
}
|
|
1028
2560
|
// Forward the workflow author's `emit({...})` onto the feeds, from EVERY actor in the
|
|
1029
2561
|
// tree as it is created: xstate scopes emitted events to the actor that emits them — no
|
|
1030
2562
|
// bubbling — and a run's Emits mostly happen in child machines (a `workspace()` body IS
|
|
1031
2563
|
// one). A root-only subscription silently dropped exactly the Emits ADR-0022/0023 exist
|
|
1032
2564
|
// to surface. The subscription dies with its actor; restore re-creates both.
|
|
2565
|
+
// The children a pick's own send starts are its Agent's (ADR-0067): their done events name
|
|
2566
|
+
// the Agent an `answer` below no Agent's state shapes. Each place an `answer` is written runs
|
|
2567
|
+
// as its own seat: each Machine of the tree is placed as its actor is created, before its
|
|
2568
|
+
// first state is entered (`placeAnswers`).
|
|
2569
|
+
if (ev.type === "@xstate.actor") {
|
|
2570
|
+
const sending = this.sending;
|
|
2571
|
+
const child = ev.actorRef as AnyActorRef;
|
|
2572
|
+
const logic = (child as { logic?: unknown }).logic as AnyStateMachine | undefined;
|
|
2573
|
+
if (logic && typeof logic === "object" && "root" in logic) placeAnswers(logic);
|
|
2574
|
+
if (sending && child._parent !== undefined && child._parent === sending.reg.invoker) {
|
|
2575
|
+
const agent = sending.reg.sender?.id;
|
|
2576
|
+
const invoke = sending.reg.invokeId;
|
|
2577
|
+
const key = childKey(child._parent, child.id);
|
|
2578
|
+
if (agent !== undefined) startedBy[key] = invoke !== undefined ? { agent, invoke } : { agent };
|
|
2579
|
+
else delete startedBy[key];
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
// …and so are the children its later work starts: a child started while the Machine takes an
|
|
2583
|
+
// event that is already that Agent's — a done event of a child it started, or its own event —
|
|
2584
|
+
// is that Agent's too, so the work its pick began stays its own (ADR-0067). A child started
|
|
2585
|
+
// on an event that names no Agent (a timer) below the state of an Agent whose pick is
|
|
2586
|
+
// outstanding is that pick's work by the Rest rule, so it is that Agent's (ADR-0024); any
|
|
2587
|
+
// other child started on such an event is no Agent's.
|
|
2588
|
+
if (ev.type === "@xstate.microstep") {
|
|
2589
|
+
const taker = ev.actorRef as AnyActorRef;
|
|
2590
|
+
const before = childrenAfter.get(taker) ?? childRefsOf(taker.getSnapshot());
|
|
2591
|
+
const snapshot = (ev as { snapshot?: unknown }).snapshot;
|
|
2592
|
+
const after = childRefsOf(snapshot);
|
|
2593
|
+
childrenAfter.set(taker, after);
|
|
2594
|
+
const started = [...after].filter((c) => !before.has(c)).map((c) => c.id);
|
|
2595
|
+
if (started.length > 0 && !this.sending) {
|
|
2596
|
+
const agent = agentOfEvent(ev.event as AnyEventObject, startedBy, taker);
|
|
2597
|
+
for (const id of started) {
|
|
2598
|
+
const owner = agent ?? this.outstandingAbove(taker, snapshot as AnyMachineSnapshot, id, holds);
|
|
2599
|
+
if (owner !== undefined) startedBy[childKey(taker, id)] = owner;
|
|
2600
|
+
else delete startedBy[childKey(taker, id)];
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
1033
2604
|
if (ev.type === "@xstate.actor") {
|
|
1034
2605
|
const ref = ev.actorRef as AnyActorRef & { on?: (type: string, handler: (e: unknown) => void) => unknown };
|
|
1035
2606
|
ref.on?.("*", (emitted) => {
|
|
@@ -1040,10 +2611,61 @@ export class RunHost {
|
|
|
1040
2611
|
this.announce(record.workflow, { kind: "emit", runId: record.runId, event });
|
|
1041
2612
|
});
|
|
1042
2613
|
}
|
|
2614
|
+
// The chain (ADR-0024): every reply from below the Agent's state while one of its picks is
|
|
2615
|
+
// outstanding — inspected as it is relayed, before the Machine processes it, so the states
|
|
2616
|
+
// it came from are still active — and the last is the answer's default. The Machine's own
|
|
2617
|
+
// events, and events from outside the states below the Agent's, are skipped (`replyOf`).
|
|
2618
|
+
// A reply inside the pick's own send comes before any hold exists: `sending` records it.
|
|
2619
|
+
if (ev.type === "@xstate.event") {
|
|
2620
|
+
const replyTo = (reg: Registration | undefined): PickAnswer | undefined => {
|
|
2621
|
+
if (!reg?.invoker || !reg.invokeId || reg.invoker !== ev.actorRef) return undefined;
|
|
2622
|
+
const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
|
|
2623
|
+
return seat && replyOf(ev.event, ev.sourceRef as AnyActorRef | undefined, reg.invoker, seat);
|
|
2624
|
+
};
|
|
2625
|
+
// A child's error from below the Agent's state is no reply: the pick's answer carries none,
|
|
2626
|
+
// and the operator reads it on the run's feed (ADR-0024).
|
|
2627
|
+
const errorTo = (reg: Registration | undefined, pick: string): void => {
|
|
2628
|
+
if (!reg?.invoker || !reg.invokeId || reg.invoker !== ev.actorRef) return;
|
|
2629
|
+
const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
|
|
2630
|
+
const failed = seat && childErrorOf(ev.event, ev.sourceRef as AnyActorRef | undefined, reg.invoker, seat);
|
|
2631
|
+
const run = this.runs.get(record.runId);
|
|
2632
|
+
if (!failed || !run || run.binding !== binding) return;
|
|
2633
|
+
this.feed(run, { kind: "child-error", agent: reg.sender?.id ?? "", pick, ...failed });
|
|
2634
|
+
};
|
|
2635
|
+
// Every relayed event is asked, so the watch forgets each timer as it fires.
|
|
2636
|
+
const timer = firedTimer?.(ev.sourceRef, ev.event) === true;
|
|
2637
|
+
for (const hold of holds.values()) {
|
|
2638
|
+
if (hold.answer !== undefined) continue;
|
|
2639
|
+
const live = this.table.lookup(agentAddress(hold.instanceId));
|
|
2640
|
+
const reg = this.holdRegs.get(hold) ?? live;
|
|
2641
|
+
const reply = replyTo(reg);
|
|
2642
|
+
if (reply) hold.chain = [...(hold.chain ?? []), reply];
|
|
2643
|
+
errorTo(reg, hold.pick);
|
|
2644
|
+
// While the Agent's state is still the hold's, each event the Machine takes says who would
|
|
2645
|
+
// move it out (`cutShort`), when the Machine that invokes the Agent or one above it takes
|
|
2646
|
+
// it — events between the children below do not move it; once the state is gone, the next
|
|
2647
|
+
// state's events are not the hold's:
|
|
2648
|
+
// - a reply: its route;
|
|
2649
|
+
// - an event a Machine sent itself at once (`sendTo(self)`; a `raise` is never relayed):
|
|
2650
|
+
// part of the route of the event whose actions sent it — no change;
|
|
2651
|
+
// - anything else — a timer the Machine set, an event from outside the states below, a
|
|
2652
|
+
// party's event, the Agent's own end: it stops the work below, if any still runs.
|
|
2653
|
+
const taker = ev.actorRef as AnyActorRef;
|
|
2654
|
+
if (reg === undefined || reg !== live || !reg.invoker || !atOrBelow(reg.invoker, taker)) continue;
|
|
2655
|
+
if (reply) delete hold.cutShort;
|
|
2656
|
+
else if (!timer && ev.sourceRef === taker) continue;
|
|
2657
|
+
else if (stillRunning(reg)) hold.cutShort = true;
|
|
2658
|
+
else delete hold.cutShort;
|
|
2659
|
+
}
|
|
2660
|
+
const sending = this.sending;
|
|
2661
|
+
const reply = sending && replyTo(sending.reg);
|
|
2662
|
+
if (sending && reply) sending.chain.push(reply);
|
|
2663
|
+
if (sending) errorTo(sending.reg, sending.pick);
|
|
2664
|
+
}
|
|
1043
2665
|
if (ev.type === "@xstate.snapshot") schedule();
|
|
1044
2666
|
},
|
|
1045
2667
|
});
|
|
1046
|
-
live = this.track(record, actor, def, agents, epochs, binding);
|
|
2668
|
+
live = this.track(record, actor, def, agents, epochs, notices, binding, holds, effects, startedBy);
|
|
1047
2669
|
return actor;
|
|
1048
2670
|
}
|
|
1049
2671
|
|
|
@@ -1053,9 +2675,26 @@ export class RunHost {
|
|
|
1053
2675
|
def: WorkflowDef,
|
|
1054
2676
|
agents: Record<string, AgentAdmission>,
|
|
1055
2677
|
epochs: Record<string, number>,
|
|
2678
|
+
notices: NoticeLedger,
|
|
1056
2679
|
binding: RunBinding,
|
|
2680
|
+
holds: Map<string, Hold>,
|
|
2681
|
+
effects: Record<string, EffectRow>,
|
|
2682
|
+
startedBy: Record<string, StartedBy>,
|
|
1057
2683
|
): LiveRun {
|
|
1058
|
-
const run: LiveRun = {
|
|
2684
|
+
const run: LiveRun = {
|
|
2685
|
+
record,
|
|
2686
|
+
actor,
|
|
2687
|
+
def,
|
|
2688
|
+
agents,
|
|
2689
|
+
epochs,
|
|
2690
|
+
notices,
|
|
2691
|
+
holds,
|
|
2692
|
+
effects,
|
|
2693
|
+
startedBy,
|
|
2694
|
+
binding,
|
|
2695
|
+
listeners: new Set(),
|
|
2696
|
+
feedSoFar: [],
|
|
2697
|
+
};
|
|
1059
2698
|
this.runs.set(record.runId, run);
|
|
1060
2699
|
// Ordinary persistence rides the inspection stream (see `spawn`); the subscription exists
|
|
1061
2700
|
// for the ERROR channel: an errored actor (an invoke threw — e.g. ADR-0011's invoke-time
|
|
@@ -1065,9 +2704,13 @@ export class RunHost {
|
|
|
1065
2704
|
error: (err) => {
|
|
1066
2705
|
run.fault = err instanceof Error ? err.message : String(err);
|
|
1067
2706
|
this.persist(run);
|
|
1068
|
-
//
|
|
1069
|
-
//
|
|
1070
|
-
//
|
|
2707
|
+
// A faulted run stops its actors — each workspace's lease among them (ADR-0021), and a
|
|
2708
|
+
// Workspace still `placing` deletes the Sandbox that never ran (ADR-0064). Said here, not
|
|
2709
|
+
// assumed: xstate ends a machine on an unhandled child error WITHOUT stopping its other
|
|
2710
|
+
// children, so a lease beside the actor that failed would renew for nobody, forever. The
|
|
2711
|
+
// pod that was placed stays up for inspection (ADR-0012's destroy-less terminal) and ages
|
|
2712
|
+
// out of the operator's idle timeout on its own.
|
|
2713
|
+
stopTree(actor);
|
|
1071
2714
|
},
|
|
1072
2715
|
});
|
|
1073
2716
|
// (The author's `emit({...})` forwarding lives in `spawn`'s inspect handler — per actor,
|
|
@@ -1096,6 +2739,8 @@ export class RunHost {
|
|
|
1096
2739
|
* status and the one the feeds report.
|
|
1097
2740
|
*/
|
|
1098
2741
|
private persist(run: LiveRun, terminal?: string): void {
|
|
2742
|
+
// The holds first: an answer read at Rest, and a notice a route out raised, ride this same save.
|
|
2743
|
+
this.settleHolds(run);
|
|
1099
2744
|
const snapshot = run.actor.getPersistedSnapshot();
|
|
1100
2745
|
const machineStatus = (snapshot as { status?: string }).status ?? "active";
|
|
1101
2746
|
const serialized = serializeSnapshot(snapshot, {
|
|
@@ -1108,13 +2753,31 @@ export class RunHost {
|
|
|
1108
2753
|
// Stamped on every save, off the registered TEMPLATE (never the per-run `.provide()` result —
|
|
1109
2754
|
// providers do not change shape, and `fingerprintOf` memoizes per machine object). ADR-0030.
|
|
1110
2755
|
machine: fingerprintOf(run.def.machine),
|
|
2756
|
+
// The Assembly the run started under (ADR-0069): the def it was spawned with, never the
|
|
2757
|
+
// current one, so a publish mid-run does not restamp it, and a restore spawns it with the
|
|
2758
|
+
// hash its blob named.
|
|
2759
|
+
...(run.def.assembly !== undefined ? { assembly: run.def.assembly } : {}),
|
|
1111
2760
|
snapshot: serialized,
|
|
1112
2761
|
agents: run.agents,
|
|
1113
2762
|
epochs: run.epochs,
|
|
2763
|
+
notices: run.notices.state(),
|
|
2764
|
+
timers: pendingTimers(run.actor, this.now()),
|
|
2765
|
+
holds: [...run.holds.values()],
|
|
2766
|
+
effects: run.effects,
|
|
2767
|
+
startedBy: liveStartedBy(run.startedBy, run.actor),
|
|
1114
2768
|
fault: run.fault,
|
|
1115
2769
|
};
|
|
1116
2770
|
const status = terminal ?? (machineStatus === "active" ? "live" : machineStatus);
|
|
1117
|
-
|
|
2771
|
+
// A save that fails is this run's fault, never the process's: an unhandled rejection would end
|
|
2772
|
+
// the Orchestrator and every run on it. The run keeps its last saved snapshot.
|
|
2773
|
+
this.store.save(run.record.runId, blob, status).catch((err: unknown) => {
|
|
2774
|
+
console.error(
|
|
2775
|
+
`run ${run.record.runId}: its snapshot was not saved: ${err instanceof Error ? err.message : String(err)}`,
|
|
2776
|
+
);
|
|
2777
|
+
});
|
|
2778
|
+
// The save is on disk (the store writes synchronously, ADR-0065): a settled row whose transition
|
|
2779
|
+
// it carries keeps no tail in the next one (ADR-0067).
|
|
2780
|
+
dropSavedTails(run.effects, snapshot);
|
|
1118
2781
|
|
|
1119
2782
|
// Feed per-run observers (SSE/CLI watch). On the terminal transition emit the final status
|
|
1120
2783
|
// BEFORE untrack drops the run from the registry, then drop the now-useless listener set.
|
|
@@ -1136,3 +2799,76 @@ export class RunHost {
|
|
|
1136
2799
|
}
|
|
1137
2800
|
}
|
|
1138
2801
|
}
|
|
2802
|
+
|
|
2803
|
+
/** Does work still run below the Agent's state of `reg` — a live child of an active state under it?
|
|
2804
|
+
* A timer that holds Rest is no work: when the state exits, nothing it started is stopped. */
|
|
2805
|
+
function stillRunning(reg: Registration): boolean {
|
|
2806
|
+
if (!reg.invoker || !reg.invokeId) return false;
|
|
2807
|
+
const seat = seatOf(reg.invoker.getSnapshot() as AnyMachineSnapshot, reg.invokeId);
|
|
2808
|
+
return seat !== undefined && runningBelow(reg.invoker, seat).length > 0;
|
|
2809
|
+
}
|
|
2810
|
+
|
|
2811
|
+
/** `actor` is `ref`, or `ref` is its parent, its parent's parent, and so on up to the root. */
|
|
2812
|
+
function atOrBelow(actor: AnyActorRef, ref: AnyActorRef): boolean {
|
|
2813
|
+
for (let at: AnyActorRef | undefined = actor; at !== undefined; at = at._parent) if (at === ref) return true;
|
|
2814
|
+
return false;
|
|
2815
|
+
}
|
|
2816
|
+
|
|
2817
|
+
/** A status's bound fields (ADR-0070): `bound`, and the hashes of the Charts bound, distinct and
|
|
2818
|
+
* sorted. Both absent when the snapshot binds no Chart. */
|
|
2819
|
+
function boundFields(snap: unknown): { bound?: true; chartHashes?: string[] } {
|
|
2820
|
+
const hashes = [...new Set(boundChartRoutesOf(snap).map((route) => route.chartHash))].sort();
|
|
2821
|
+
return hashes.length ? { bound: true, chartHashes: hashes } : {};
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
/** A live actor's children, by id. */
|
|
2825
|
+
function childrenOfRef(ref: AnyActorRef): Record<string, AnyActorRef> {
|
|
2826
|
+
return (ref.getSnapshot() as { children?: Record<string, AnyActorRef> } | undefined)?.children ?? {};
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
/** A stored run's bound Chart, compiled again from its text under the Open body it binds, for the
|
|
2830
|
+
* run's doc — or why it is not: the doc then keeps the slot's subgraph and names the reason. */
|
|
2831
|
+
function compileStored(machine: AnyStateMachine, route: BoundChartRoute): AnyStateMachine | string {
|
|
2832
|
+
let at: AnyStateMachine = machine;
|
|
2833
|
+
for (const src of route.srcs.slice(0, -1)) {
|
|
2834
|
+
const logic = (at.implementations.actors as Record<string, unknown>)[src];
|
|
2835
|
+
if (isOpenBody(logic)) continue;
|
|
2836
|
+
if (!(logic as AnyStateMachine | undefined)?.root) return `no Machine at slot "${src}"`;
|
|
2837
|
+
at = logic as AnyStateMachine;
|
|
2838
|
+
}
|
|
2839
|
+
const logic = (at.implementations.actors as Record<string, unknown>)[route.src];
|
|
2840
|
+
if (!isOpenBody(logic)) return `slot "${route.src}" is no Open body`;
|
|
2841
|
+
const decl = openBodyDeclarationOf(logic);
|
|
2842
|
+
const lent = lentIn(at.implementations as never, decl);
|
|
2843
|
+
try {
|
|
2844
|
+
return compileChart(admitChart(route.chart, decl, lent), lent, decl, { id: route.src });
|
|
2845
|
+
} catch (err) {
|
|
2846
|
+
return err instanceof ChartRefusal
|
|
2847
|
+
? `the door refuses the Chart: ${err.report.join("; ")}`
|
|
2848
|
+
: err instanceof Error
|
|
2849
|
+
? err.message
|
|
2850
|
+
: String(err);
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2853
|
+
|
|
2854
|
+
/** Put a compiled Chart's body under the states of `doc` that invoke the Open body at the end of
|
|
2855
|
+
* `srcs`, descending through the child Machine docs the earlier slots name. */
|
|
2856
|
+
function placeBody(doc: MachineBodyDoc, srcs: readonly string[], body: MachineBodyDoc): void {
|
|
2857
|
+
const [src, ...rest] = srcs;
|
|
2858
|
+
if (src === undefined) return;
|
|
2859
|
+
const visit = (state: MachineStateDoc): void => {
|
|
2860
|
+
if (rest.length === 0) {
|
|
2861
|
+
for (const inv of state.invoke) {
|
|
2862
|
+
if (inv.src !== src) continue;
|
|
2863
|
+
const child: ChildMachineDoc = { src, label: inv.id, via: "invoke", machine: body };
|
|
2864
|
+
const at = state.children.findIndex((c) => c.src === src);
|
|
2865
|
+
if (at >= 0) state.children[at] = child;
|
|
2866
|
+
else state.children.push(child);
|
|
2867
|
+
}
|
|
2868
|
+
} else {
|
|
2869
|
+
for (const child of state.children) if (child.src === src && child.machine) placeBody(child.machine, rest, body);
|
|
2870
|
+
}
|
|
2871
|
+
for (const nested of state.states) visit(nested);
|
|
2872
|
+
};
|
|
2873
|
+
visit(doc.root);
|
|
2874
|
+
}
|