@zachwill/pi-orchestrate 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,49 +1,53 @@
1
1
  import type { Api, Model } from "@earendil-works/pi-ai";
2
2
  import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
3
- import { Effect, Option } from "effect";
3
+ import {
4
+ Cause,
5
+ Clock,
6
+ Context,
7
+ Deferred,
8
+ Effect,
9
+ Exit,
10
+ Fiber,
11
+ FiberMap,
12
+ FiberSet,
13
+ Layer,
14
+ Schema,
15
+ } from "effect";
4
16
  import {
5
17
  CANCELLATION_GRACE_MS,
6
18
  EMPTY_WORKER_USAGE,
7
19
  MAX_WORKER_INSTRUCTIONS_LENGTH,
8
20
  MAX_WORKER_TITLE_LENGTH,
21
+ OrchestrateTaskInput,
22
+ RunId,
23
+ WorkerId,
9
24
  createRandomIdFactories,
10
25
  findWorkerByName,
11
26
  isTerminalWorkerStatus,
12
27
  transitionWorkerStatus,
13
28
  type OrchestrateIdFactories,
14
- type OrchestrateTaskInput,
15
- type RunId,
16
29
  type RunMode,
17
30
  type RunRecord,
18
31
  type WorkerCatalog,
19
32
  type WorkerDefinition,
20
- type WorkerId,
21
33
  type WorkerOutcome,
22
34
  type WorkerRecord,
23
35
  type WorkerUsage,
24
36
  } from "./domain.js";
25
37
  import {
26
- createWorkflowScheduler,
27
- type WorkflowScheduler,
28
- } from "./scheduler.js";
29
- import {
30
- resolveWorkerModel,
31
- type WorkerSessionFactory,
38
+ ChildSessions,
39
+ type ChildSessionsService,
40
+ type WorkerSessionAbortError,
32
41
  type WorkerSessionHandle,
42
+ type WorkerSessionObservation,
33
43
  } from "./worker-session.js";
34
44
  import type {
35
45
  SettlementFailureStage,
36
46
  WorkerSettlement,
37
47
  } from "./worker-settlement.js";
38
48
 
39
- export type {
40
- SettlementFailureStage,
41
- WorkerSettlement,
42
- } from "./worker-settlement.js";
43
-
44
49
  export const MAX_TERMINAL_WORKER_HISTORY = 100;
45
50
  export const MAX_COMPLETED_RUN_HISTORY = 100;
46
- /** Shutdown waits this long for interrupted bootstrap/prompt promises, then returns best-effort. */
47
51
  export const SHUTDOWN_CLEANUP_GRACE_MS = CANCELLATION_GRACE_MS;
48
52
 
49
53
  export interface OrchestrationContext {
@@ -92,433 +96,699 @@ export interface RuntimeSnapshot {
92
96
 
93
97
  export type SettlementListener = (settlement: WorkerSettlement) => void;
94
98
  export type UnsubscribeSettlement = () => void;
95
- export type StateListener = (ownerSessionId: string) => void;
96
-
97
- export type AbortTarget =
98
- | {
99
- readonly workerIds: readonly WorkerId[];
100
- readonly all?: never;
101
- }
102
- | {
103
- readonly all: true;
104
- readonly workerIds?: never;
105
- };
99
+ export type StateListener = (snapshot: RuntimeSnapshot) => void;
106
100
 
107
- export type DeadlineResult = "settled" | "timed-out";
108
-
109
- export interface BestEffortDeadline {
110
- wait(promise: Promise<unknown>, timeoutMs: number): Promise<DeadlineResult>;
101
+ export interface AbortTarget {
102
+ readonly workerIds?: readonly string[];
103
+ readonly all?: boolean;
111
104
  }
112
105
 
113
- export interface OrchestratorRuntimeOptions {
114
- readonly workerSessionFactory: WorkerSessionFactory;
106
+ export interface OrchestrationLayerOptions {
115
107
  readonly idFactories?: OrchestrateIdFactories;
116
- readonly clock?: () => number;
117
- readonly scheduler?: WorkflowScheduler<WorkerId>;
118
- readonly bestEffortDeadline?: BestEffortDeadline;
119
108
  }
120
109
 
121
- export interface OrchestratorRuntime {
110
+ const OrchestrationOperation = Schema.Literals([
111
+ "orchestrate",
112
+ "sendInteractive",
113
+ "abort",
114
+ "closeInteractive",
115
+ "snapshot",
116
+ ]);
117
+ type OrchestrationOperation = typeof OrchestrationOperation.Type;
118
+
119
+ const OrchestrationRejectionReason = Schema.Literals([
120
+ "shutdown",
121
+ "validation",
122
+ "ownership",
123
+ "target",
124
+ "worker-state",
125
+ "unknown-worker",
126
+ "model-unavailable",
127
+ ]);
128
+ type OrchestrationRejectionReason = typeof OrchestrationRejectionReason.Type;
129
+
130
+ export class OrchestrationActionRejected extends Schema.TaggedError<OrchestrationActionRejected>()(
131
+ "Orchestration.ActionRejected",
132
+ {
133
+ operation: OrchestrationOperation,
134
+ reason: OrchestrationRejectionReason,
135
+ message: Schema.String,
136
+ },
137
+ ) {}
138
+
139
+ export interface OrchestrationService {
122
140
  orchestrate(
123
141
  context: OrchestrationContext,
124
142
  task: OrchestrateTaskInput,
125
143
  mode: "async",
126
- signal?: AbortSignal,
127
144
  onSettlement?: SettlementListener,
128
- ): Promise<AcceptedRun>;
145
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
129
146
  orchestrate(
130
147
  context: OrchestrationContext,
131
148
  task: OrchestrateTaskInput,
132
149
  mode: "inline",
133
- signal?: AbortSignal,
134
150
  onSettlement?: SettlementListener,
135
- ): Promise<CompletedRun>;
151
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
136
152
  orchestrate(
137
153
  context: OrchestrationContext,
138
154
  task: OrchestrateTaskInput,
139
155
  mode: RunMode,
140
- signal?: AbortSignal,
141
156
  onSettlement?: SettlementListener,
142
- ): Promise<AcceptedRun | CompletedRun>;
157
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
143
158
  sendInteractive(
144
159
  context: OrchestrationContext,
145
- workerId: WorkerId,
160
+ workerId: string,
146
161
  instructions: string,
147
162
  mode: "async",
148
- signal?: AbortSignal,
149
163
  onSettlement?: SettlementListener,
150
- ): Promise<AcceptedRun>;
164
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
151
165
  sendInteractive(
152
166
  context: OrchestrationContext,
153
- workerId: WorkerId,
167
+ workerId: string,
154
168
  instructions: string,
155
169
  mode: "inline",
156
- signal?: AbortSignal,
157
170
  onSettlement?: SettlementListener,
158
- ): Promise<CompletedRun>;
171
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
159
172
  sendInteractive(
160
173
  context: OrchestrationContext,
161
- workerId: WorkerId,
174
+ workerId: string,
162
175
  instructions: string,
163
176
  mode: RunMode,
164
- signal?: AbortSignal,
165
177
  onSettlement?: SettlementListener,
166
- ): Promise<AcceptedRun | CompletedRun>;
167
- abort(ownerSessionId: string, target: AbortTarget): Promise<void>;
168
- closeInteractive(ownerSessionId: string, workerId: WorkerId): Promise<void>;
169
- snapshot(ownerSessionId: string): Promise<RuntimeSnapshot>;
170
- subscribeSettlement(listener: SettlementListener): UnsubscribeSettlement;
171
- subscribeState(listener: StateListener): () => void;
172
- shutdown(): Promise<void>;
178
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
179
+ readonly abort: (
180
+ ownerSessionId: string,
181
+ target: AbortTarget,
182
+ ) => Effect.Effect<void, OrchestrationActionRejected>;
183
+ readonly closeInteractive: (
184
+ ownerSessionId: string,
185
+ workerId: string,
186
+ ) => Effect.Effect<void, OrchestrationActionRejected>;
187
+ readonly snapshot: (
188
+ ownerSessionId: string,
189
+ ) => Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected>;
190
+ readonly subscribeSettlement: (
191
+ listener: SettlementListener,
192
+ ) => UnsubscribeSettlement;
193
+ readonly subscribeState: (
194
+ ownerSessionId: string,
195
+ listener: StateListener,
196
+ ) => () => void;
197
+ readonly shutdown: () => Effect.Effect<void>;
173
198
  }
174
199
 
175
- interface RuntimeEntry {
200
+ export class Orchestration extends Context.Service<Orchestration, OrchestrationService>()(
201
+ "@zachwill/pi-orchestrate/Orchestration",
202
+ ) {}
203
+
204
+ interface RuntimeWorker {
205
+ readonly record: WorkerRecord;
176
206
  readonly context: OrchestrationContext;
177
207
  readonly definition: WorkerDefinition;
178
- generation: number;
179
- session?: WorkerSessionHandle;
180
- unsubscribeUsage?: () => void;
181
- unsubscribeActivity?: () => void;
182
- unsubscribeMessageDirection?: () => void;
208
+ readonly generation: number;
209
+ readonly session?: WorkerSessionHandle;
210
+ readonly observationRelease?: () => void;
211
+ readonly cancellation?: Deferred.Deferred<void>;
183
212
  }
184
213
 
185
- interface RunWaiter {
186
- readonly promise: Promise<CompletedRun>;
187
- settled: boolean;
188
- onSettled?: () => void;
189
- resolve(run: CompletedRun): void;
214
+ interface RunningRuntimeRun {
215
+ readonly _tag: "running";
216
+ readonly record: RunRecord;
217
+ readonly completion: Deferred.Deferred<CompletedRun>;
218
+ readonly settlementListener?: SettlementListener;
190
219
  }
191
220
 
192
- const defaultBestEffortDeadline: BestEffortDeadline = {
193
- wait(promise, timeoutMs) {
194
- const settled = Effect.promise(() => promise.then(
195
- () => "settled" as const,
196
- () => "settled" as const,
197
- ));
198
- return Effect.runPromise(
199
- settled.pipe(
200
- Effect.timeoutOption(timeoutMs),
201
- Effect.map((result) => Option.getOrElse(result, () => "timed-out" as const)),
202
- ),
203
- );
204
- },
205
- };
206
-
207
- class DefaultOrchestratorRuntime implements OrchestratorRuntime {
208
- private readonly workerSessionFactory: WorkerSessionFactory;
209
- private readonly idFactories: OrchestrateIdFactories;
210
- private readonly clock: () => number;
211
- private readonly scheduler: WorkflowScheduler<WorkerId>;
212
- private readonly bestEffortDeadline: BestEffortDeadline;
213
- private readonly workers = new Map<WorkerId, WorkerRecord>();
214
- private readonly runs = new Map<RunId, RunRecord>();
215
- private readonly entries = new Map<WorkerId, RuntimeEntry>();
216
- private readonly runWaiters = new Map<RunId, RunWaiter>();
217
- private readonly completedRuns = new Map<RunId, CompletedRun>();
218
- private readonly cancellationPromises = new Map<WorkerId, Promise<void>>();
219
- private readonly cleanupOperations = new Set<Promise<void>>();
220
- private readonly terminalWorkerOrder: WorkerId[] = [];
221
- private readonly completedRunOrder: RunId[] = [];
221
+ type RuntimeRun = RunningRuntimeRun | RunRecord;
222
+ type RuntimeLifecycle = "open" | "shutting-down" | "shutdown";
223
+
224
+ interface RuntimeState {
225
+ workers: Map<WorkerId, RuntimeWorker>;
226
+ runs: Map<RunId, RuntimeRun>;
227
+ terminalWorkerOrder: WorkerId[];
228
+ completedRunOrder: RunId[];
229
+ settlementSequence: number;
230
+ lifecycle: RuntimeLifecycle;
231
+ }
232
+
233
+ type CommittedAction = () => void;
234
+ type PostCommitAction = (
235
+ state: RuntimeState,
236
+ settlementListeners: ReadonlySet<SettlementListener>,
237
+ stateListeners: ReadonlyMap<string, ReadonlySet<StateListener>>,
238
+ ) => CommittedAction;
239
+
240
+ interface TransactionMutation<A> {
241
+ readonly value: A;
242
+ readonly actions?: readonly PostCommitAction[];
243
+ }
244
+
245
+ type Decision<A> =
246
+ | {
247
+ readonly _tag: "accepted";
248
+ readonly value: A;
249
+ }
250
+ | {
251
+ readonly _tag: "rejected";
252
+ readonly error: OrchestrationActionRejected;
253
+ };
254
+
255
+ class StatefulOrchestration implements OrchestrationService {
256
+ private readonly actionQueue: CommittedAction[] = [];
222
257
  private readonly settlementListeners = new Set<SettlementListener>();
223
- private readonly runSettlementListeners = new Map<RunId, SettlementListener>();
224
- private readonly stateListeners = new Set<StateListener>();
225
- private settlementSequence = 0;
226
- private readonly disposedSessions = new WeakSet<WorkerSessionHandle>();
227
- private shuttingDown = false;
228
- private shutdownPromise: Promise<void> | undefined;
229
-
230
- constructor(options: OrchestratorRuntimeOptions) {
231
- this.workerSessionFactory = options.workerSessionFactory;
232
- this.idFactories = options.idFactories ?? createRandomIdFactories();
233
- this.clock = options.clock ?? Date.now;
234
- this.scheduler = options.scheduler ?? createWorkflowScheduler<WorkerId>();
235
- this.bestEffortDeadline = options.bestEffortDeadline ?? defaultBestEffortDeadline;
258
+ private readonly stateListeners = new Map<string, Set<StateListener>>();
259
+ private drainingActions = false;
260
+ private state: RuntimeState;
261
+
262
+ constructor(
263
+ private readonly childSessions: ChildSessionsService,
264
+ private readonly generations: FiberMap.FiberMap<WorkerId, void, never>,
265
+ private readonly runGeneration: (
266
+ key: WorkerId,
267
+ effect: Effect.Effect<void, never>,
268
+ ) => Fiber.Fiber<void, never>,
269
+ private readonly cancellations: FiberSet.FiberSet<void, never>,
270
+ private readonly runCancellation: (
271
+ effect: Effect.Effect<void, never>,
272
+ ) => Fiber.Fiber<void, never>,
273
+ private readonly cleanups: FiberSet.FiberSet<void, never>,
274
+ private readonly runCleanup: (
275
+ effect: Effect.Effect<void, never>,
276
+ ) => Fiber.Fiber<void, never>,
277
+ private readonly clock: Clock.Clock,
278
+ private readonly idFactories: OrchestrateIdFactories,
279
+ private readonly shutdownCompletion: Deferred.Deferred<void>,
280
+ ) {
281
+ this.state = initialState();
236
282
  }
237
283
 
238
284
  orchestrate(
239
285
  context: OrchestrationContext,
240
286
  task: OrchestrateTaskInput,
241
287
  mode: "async",
242
- signal?: AbortSignal,
243
288
  onSettlement?: SettlementListener,
244
- ): Promise<AcceptedRun>;
289
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
245
290
  orchestrate(
246
291
  context: OrchestrationContext,
247
292
  task: OrchestrateTaskInput,
248
293
  mode: "inline",
249
- signal?: AbortSignal,
250
294
  onSettlement?: SettlementListener,
251
- ): Promise<CompletedRun>;
295
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
252
296
  orchestrate(
253
297
  context: OrchestrationContext,
254
298
  task: OrchestrateTaskInput,
255
299
  mode: RunMode,
256
- signal?: AbortSignal,
257
300
  onSettlement?: SettlementListener,
258
- ): Promise<AcceptedRun | CompletedRun>;
259
- async orchestrate(
301
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
302
+ orchestrate(
260
303
  context: OrchestrationContext,
261
304
  task: OrchestrateTaskInput,
262
305
  mode: RunMode,
263
- signal?: AbortSignal,
264
306
  onSettlement?: SettlementListener,
265
- ): Promise<AcceptedRun | CompletedRun> {
266
- this.assertOpen();
267
- throwIfAborted(signal);
268
- const definition = this.validateTask(context, task, mode);
269
-
270
- const runId = this.idFactories.runId();
271
- const workerId = this.idFactories.workerId();
272
- this.assertFreshIds(runId, workerId);
273
-
274
- const run: RunRecord = {
275
- id: runId,
276
- ownerSessionId: context.ownerSessionId,
277
- workerId,
278
- mode,
279
- state: "running",
280
- createdAt: this.clock(),
281
- ...(context.synthesisGroup
282
- ? {
283
- synthesisGroupId: context.synthesisGroup.id,
284
- synthesisGroupSize: context.synthesisGroup.size,
285
- }
286
- : {}),
287
- };
288
- const record: WorkerRecord = {
289
- id: workerId,
290
- worker: definition.name,
291
- ownerSessionId: context.ownerSessionId,
292
- runId,
293
- title: task.title,
294
- instructions: task.instructions,
295
- lifecycle: definition.lifecycle,
296
- status: "starting",
297
- usage: copyUsage(EMPTY_WORKER_USAGE),
298
- messageDirection: "to-model",
299
- startedAt: this.clock(),
300
- };
307
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
308
+ return Effect.fn("Orchestration.orchestrate")(function* (
309
+ this: StatefulOrchestration,
310
+ ) {
311
+ const validated = yield* this.validateTask(context, task, mode);
312
+ const preflight = this.preflightOpen("orchestrate");
313
+ if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
314
+ // Keep user-supplied ID factories outside the transaction: a reentrant
315
+ // factory cannot commit an outer stale draft. Admission rechecks all
316
+ // authority below, so a race may burn an ID but cannot create a run.
317
+ const runId = this.idFactories.runId();
318
+ const workerId = this.idFactories.workerId();
319
+ const completion = yield* Deferred.make<CompletedRun>();
320
+ const now = this.clock.currentTimeMillisUnsafe();
321
+ const runRecord = makeRunRecord(runId, workerId, context, mode, now);
322
+ const workerRecord = makeWorkerRecord(
323
+ workerId,
324
+ runId,
325
+ context.ownerSessionId,
326
+ validated.definition,
327
+ validated.task,
328
+ now,
329
+ );
301
330
 
302
- const waiter = makeRunWaiter();
303
- this.runs.set(runId, run);
304
- this.runWaiters.set(runId, waiter);
305
- if (onSettlement) this.runSettlementListeners.set(runId, onSettlement);
306
- this.workers.set(workerId, record);
307
- this.entries.set(workerId, { context, definition, generation: 1 });
308
- this.emitState(context.ownerSessionId);
309
- this.launchBootstrap(workerId, 1);
310
-
311
- if (mode === "inline") {
312
- return this.awaitInlineRun(run, waiter, signal);
313
- }
314
- return freezeAcceptedRun(runId, workerId);
331
+ const admission = this.transact((draft) => {
332
+ const open = openDecision(draft, "orchestrate");
333
+ if (open._tag === "rejected") return { value: open };
334
+ if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
335
+ if (draft.workers.has(workerId)) {
336
+ throw new Error(`Duplicate worker ID: ${workerId}`);
337
+ }
338
+
339
+ draft.runs.set(runId, {
340
+ _tag: "running",
341
+ record: runRecord,
342
+ completion,
343
+ ...(onSettlement ? { settlementListener: onSettlement } : {}),
344
+ });
345
+ draft.workers.set(workerId, {
346
+ record: workerRecord,
347
+ context,
348
+ definition: validated.definition,
349
+ generation: 1,
350
+ });
351
+
352
+ return {
353
+ value: accepted(undefined),
354
+ actions: [
355
+ publishState(context.ownerSessionId),
356
+ runAction(() => this.launchBootstrap(workerId, 1)),
357
+ ],
358
+ };
359
+ });
360
+ if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
361
+
362
+ if (mode === "inline") {
363
+ return yield* this.awaitInlineRun(runRecord, completion);
364
+ }
365
+ return freezeAcceptedRun(runId, workerId);
366
+ }).call(this);
315
367
  }
316
368
 
317
369
  sendInteractive(
318
370
  context: OrchestrationContext,
319
- workerId: WorkerId,
371
+ workerId: string,
320
372
  instructions: string,
321
373
  mode: "async",
322
- signal?: AbortSignal,
323
374
  onSettlement?: SettlementListener,
324
- ): Promise<AcceptedRun>;
375
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
325
376
  sendInteractive(
326
377
  context: OrchestrationContext,
327
- workerId: WorkerId,
378
+ workerId: string,
328
379
  instructions: string,
329
380
  mode: "inline",
330
- signal?: AbortSignal,
331
381
  onSettlement?: SettlementListener,
332
- ): Promise<CompletedRun>;
382
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
333
383
  sendInteractive(
334
384
  context: OrchestrationContext,
335
- workerId: WorkerId,
385
+ workerId: string,
336
386
  instructions: string,
337
387
  mode: RunMode,
338
- signal?: AbortSignal,
339
388
  onSettlement?: SettlementListener,
340
- ): Promise<AcceptedRun | CompletedRun>;
341
- async sendInteractive(
389
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
390
+ sendInteractive(
342
391
  context: OrchestrationContext,
343
- workerId: WorkerId,
392
+ workerId: string,
344
393
  instructions: string,
345
394
  mode: RunMode,
346
- signal?: AbortSignal,
347
395
  onSettlement?: SettlementListener,
348
- ): Promise<AcceptedRun | CompletedRun> {
349
- this.assertOpen();
350
- throwIfAborted(signal);
351
- validateContextOwner(context.ownerSessionId);
352
- validateMode(mode);
353
- validateText("instructions", instructions, MAX_WORKER_INSTRUCTIONS_LENGTH);
354
-
355
- const current = this.ownedWorker(context.ownerSessionId, workerId);
356
- if (current.lifecycle !== "interactive" || current.status !== "ready") {
357
- throw new Error("interactive_send requires an owned ready interactive worker");
358
- }
359
- const entry = this.entries.get(workerId);
360
- if (!entry?.session) throw new Error("Ready interactive worker has no session handle");
361
-
362
- const runId = this.idFactories.runId();
363
- if (this.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
364
- const run: RunRecord = {
365
- id: runId,
366
- ownerSessionId: context.ownerSessionId,
367
- workerId,
368
- mode,
369
- state: "running",
370
- createdAt: this.clock(),
371
- };
372
- const running: WorkerRecord = {
373
- ...transitionWorkerStatus(current, "running"),
374
- runId,
375
- instructions,
376
- activity: undefined,
377
- messageDirection: "to-model",
378
- startedAt: this.clock(),
379
- settledAt: undefined,
380
- };
381
- const waiter = makeRunWaiter();
382
-
383
- entry.generation += 1;
384
- const generation = entry.generation;
385
- this.runs.set(runId, run);
386
- this.runWaiters.set(runId, waiter);
387
- if (onSettlement) this.runSettlementListeners.set(runId, onSettlement);
388
- this.workers.set(workerId, running);
389
- this.subscribeEntryObservability(workerId, entry, entry.session, generation);
390
- this.emitState(context.ownerSessionId);
391
- this.launchPrompt(workerId, generation, entry.session, instructions);
392
-
393
- if (mode === "inline") {
394
- return this.awaitInlineRun(run, waiter, signal);
395
- }
396
- return freezeAcceptedRun(runId, workerId);
396
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
397
+ return Effect.fn("Orchestration.sendInteractive")(function* (
398
+ this: StatefulOrchestration,
399
+ ) {
400
+ yield* validateContextOwner("sendInteractive", context.ownerSessionId);
401
+ yield* validateMode("sendInteractive", mode);
402
+ yield* validateText(
403
+ "sendInteractive",
404
+ "instructions",
405
+ instructions,
406
+ MAX_WORKER_INSTRUCTIONS_LENGTH,
407
+ );
408
+ const validatedWorkerId = yield* validateWorkerId(
409
+ "sendInteractive",
410
+ workerId,
411
+ );
412
+ const preflight = this.preflightReadyInteractive(
413
+ context.ownerSessionId,
414
+ validatedWorkerId,
415
+ );
416
+ if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
417
+ // See orchestrate: allocation remains outside copy-on-write reducers so
418
+ // reentrant factories cannot invalidate transaction atomicity.
419
+ const runId = this.idFactories.runId();
420
+ const completion = yield* Deferred.make<CompletedRun>();
421
+ const now = this.clock.currentTimeMillisUnsafe();
422
+ const runRecord: RunRecord = {
423
+ id: runId,
424
+ ownerSessionId: context.ownerSessionId,
425
+ workerId: validatedWorkerId,
426
+ mode,
427
+ state: "running",
428
+ createdAt: now,
429
+ };
430
+
431
+ const admission = this.transact((draft) => {
432
+ const open = openDecision(draft, "sendInteractive");
433
+ if (open._tag === "rejected") return { value: open };
434
+ const ownership = ownedWorkerDecision(
435
+ draft,
436
+ "sendInteractive",
437
+ context.ownerSessionId,
438
+ validatedWorkerId,
439
+ );
440
+ if (ownership._tag === "rejected") return { value: ownership };
441
+
442
+ const worker = ownership.value;
443
+ if (
444
+ worker.record.lifecycle !== "interactive" ||
445
+ worker.record.status !== "ready" ||
446
+ !worker.session
447
+ ) {
448
+ return {
449
+ value: rejected(
450
+ "sendInteractive",
451
+ "worker-state",
452
+ "interactive_send requires an owned ready interactive worker",
453
+ ),
454
+ };
455
+ }
456
+ if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
457
+
458
+ const generation = worker.generation + 1;
459
+ const session = worker.session;
460
+ const runningRecord: WorkerRecord = {
461
+ ...transitionWorkerStatus(worker.record, "running"),
462
+ runId,
463
+ instructions,
464
+ activity: undefined,
465
+ messageDirection: "to-model",
466
+ startedAt: now,
467
+ settledAt: undefined,
468
+ };
469
+ draft.runs.set(runId, {
470
+ _tag: "running",
471
+ record: runRecord,
472
+ completion,
473
+ ...(onSettlement ? { settlementListener: onSettlement } : {}),
474
+ });
475
+ draft.workers.set(validatedWorkerId, {
476
+ ...worker,
477
+ record: runningRecord,
478
+ generation,
479
+ observationRelease: undefined,
480
+ cancellation: undefined,
481
+ });
482
+
483
+ return {
484
+ value: accepted(undefined),
485
+ actions: [
486
+ runAction(() => {
487
+ safelyCall(worker.observationRelease);
488
+ this.subscribeObservation(validatedWorkerId, generation, session);
489
+ }),
490
+ publishState(context.ownerSessionId),
491
+ runAction(() => {
492
+ this.launchPrompt(
493
+ validatedWorkerId,
494
+ generation,
495
+ session,
496
+ instructions,
497
+ );
498
+ }),
499
+ ],
500
+ };
501
+ });
502
+ if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
503
+
504
+ if (mode === "inline") {
505
+ return yield* this.awaitInlineRun(runRecord, completion);
506
+ }
507
+ return freezeAcceptedRun(runId, validatedWorkerId);
508
+ }).call(this);
397
509
  }
398
510
 
399
- async abort(ownerSessionId: string, target: AbortTarget): Promise<void> {
400
- this.assertOpen();
401
- validateContextOwner(ownerSessionId);
402
- const targets = this.resolveAbortTargets(ownerSessionId, target);
403
- await this.cancelWorkers(targets);
511
+ abort(
512
+ ownerSessionId: string,
513
+ target: AbortTarget,
514
+ ): Effect.Effect<void, OrchestrationActionRejected> {
515
+ return Effect.fn("Orchestration.abort")(function* (
516
+ this: StatefulOrchestration,
517
+ ) {
518
+ yield* validateContextOwner("abort", ownerSessionId);
519
+ const validatedTarget = yield* validateAbortTarget(target);
520
+ const candidateIds = validatedTarget._tag === "ids"
521
+ ? validatedTarget.workerIds
522
+ : activeWorkerIds(this.current(), ownerSessionId);
523
+ const candidates = yield* makeCancellationCandidates(candidateIds);
524
+ const cancellation = this.beginCancellation(
525
+ "abort",
526
+ ownerSessionId,
527
+ validatedTarget,
528
+ candidates,
529
+ );
530
+ if (cancellation._tag === "rejected") {
531
+ return yield* Effect.fail(cancellation.error);
532
+ }
533
+ yield* awaitAll(cancellation.value);
534
+ }).call(this);
404
535
  }
405
536
 
406
- async closeInteractive(ownerSessionId: string, workerId: WorkerId): Promise<void> {
407
- this.assertOpen();
408
- validateContextOwner(ownerSessionId);
409
- const current = this.ownedWorker(ownerSessionId, workerId);
410
- if (current.lifecycle !== "interactive" || current.status !== "ready") {
411
- throw new Error("interactive_close requires an owned ready interactive worker");
412
- }
413
- this.closeReadyInteractiveWorker(current);
537
+ closeInteractive(
538
+ ownerSessionId: string,
539
+ workerId: string,
540
+ ): Effect.Effect<void, OrchestrationActionRejected> {
541
+ return Effect.fn("Orchestration.closeInteractive")(function* (
542
+ this: StatefulOrchestration,
543
+ ) {
544
+ yield* validateContextOwner("closeInteractive", ownerSessionId);
545
+ const id = yield* validateWorkerId("closeInteractive", workerId);
546
+ const now = this.clock.currentTimeMillisUnsafe();
547
+ const decision = this.transact((draft) => {
548
+ const open = openDecision(draft, "closeInteractive");
549
+ if (open._tag === "rejected") return { value: open };
550
+ const ownership = ownedWorkerDecision(
551
+ draft,
552
+ "closeInteractive",
553
+ ownerSessionId,
554
+ id,
555
+ );
556
+ if (ownership._tag === "rejected") return { value: ownership };
557
+ if (
558
+ ownership.value.record.lifecycle !== "interactive" ||
559
+ ownership.value.record.status !== "ready"
560
+ ) {
561
+ return {
562
+ value: rejected(
563
+ "closeInteractive",
564
+ "worker-state",
565
+ "interactive_close requires an owned ready interactive worker",
566
+ ),
567
+ };
568
+ }
569
+ const actions = this.closeReadyWorker(draft, ownership.value, now);
570
+ actions.push(...stateActionsAfterPrune(draft, ownerSessionId));
571
+ return {
572
+ value: accepted(undefined),
573
+ actions,
574
+ };
575
+ });
576
+ if (decision._tag === "rejected") yield* Effect.fail(decision.error);
577
+ }).call(this);
414
578
  }
415
579
 
416
- async snapshot(ownerSessionId: string): Promise<RuntimeSnapshot> {
417
- validateContextOwner(ownerSessionId);
418
- const runs = [...this.runs.values()]
419
- .filter((run) => run.ownerSessionId === ownerSessionId)
420
- .map(copyRunRecord);
421
- const workers = [...this.workers.values()]
422
- .filter((worker) => worker.ownerSessionId === ownerSessionId)
423
- .map(copyWorkerRecord);
424
- return Object.freeze({
425
- runs: Object.freeze(runs),
426
- workers: Object.freeze(workers),
427
- });
580
+ snapshot(
581
+ ownerSessionId: string,
582
+ ): Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected> {
583
+ return validateContextOwner("snapshot", ownerSessionId).pipe(
584
+ Effect.andThen(Effect.sync(() => snapshotFor(this.current(), ownerSessionId))),
585
+ );
428
586
  }
429
587
 
430
588
  subscribeSettlement(listener: SettlementListener): UnsubscribeSettlement {
431
- if (typeof listener !== "function") throw new Error("Settlement listener must be a function");
589
+ if (typeof listener !== "function") {
590
+ throw new Error("Settlement listener must be a function");
591
+ }
592
+ if (this.state.lifecycle !== "open") return noOp;
432
593
  this.settlementListeners.add(listener);
433
- let subscribed = true;
594
+
595
+ let active = true;
434
596
  return () => {
435
- if (!subscribed) return;
436
- subscribed = false;
597
+ if (!active) return;
598
+ active = false;
437
599
  this.settlementListeners.delete(listener);
438
600
  };
439
601
  }
440
602
 
441
- subscribeState(listener: StateListener): () => void {
442
- if (typeof listener !== "function") throw new Error("State listener must be a function");
443
- this.stateListeners.add(listener);
444
- let subscribed = true;
603
+ subscribeState(ownerSessionId: string, listener: StateListener): () => void {
604
+ if (typeof ownerSessionId !== "string" || ownerSessionId.trim() === "") {
605
+ throw new Error("ownerSessionId must not be blank");
606
+ }
607
+ if (typeof listener !== "function") {
608
+ throw new Error("State listener must be a function");
609
+ }
610
+
611
+ if (this.state.lifecycle !== "open") return noOp;
612
+ const ownerListeners = this.stateListeners.get(ownerSessionId) ?? new Set();
613
+ ownerListeners.add(listener);
614
+ this.stateListeners.set(ownerSessionId, ownerListeners);
615
+ this.enqueueActions([
616
+ publishStateTo(ownerSessionId, listener)(
617
+ this.state,
618
+ this.settlementListeners,
619
+ this.stateListeners,
620
+ ),
621
+ ]);
622
+
623
+ let active = true;
445
624
  return () => {
446
- if (!subscribed) return;
447
- subscribed = false;
448
- this.stateListeners.delete(listener);
625
+ if (!active) return;
626
+ active = false;
627
+ ownerListeners.delete(listener);
628
+ if (ownerListeners.size === 0) this.stateListeners.delete(ownerSessionId);
449
629
  };
450
630
  }
451
631
 
452
- shutdown(): Promise<void> {
453
- if (this.shutdownPromise) return this.shutdownPromise;
454
- this.shuttingDown = true;
455
- this.shutdownPromise = this.performShutdown();
456
- return this.shutdownPromise;
632
+ /** Calling shutdown closes admission synchronously, before the returned Effect runs. */
633
+ shutdown(): Effect.Effect<void> {
634
+ const first = this.transact((draft) => {
635
+ if (draft.lifecycle !== "open") return { value: false };
636
+ draft.lifecycle = "shutting-down";
637
+ return { value: true };
638
+ });
639
+ if (!first) return Deferred.await(this.shutdownCompletion);
640
+
641
+ return this.performShutdown().pipe(
642
+ Effect.ensuring(
643
+ Effect.sync(() => {
644
+ this.transact((draft) => {
645
+ draft.lifecycle = "shutdown";
646
+ for (const [runId, run] of draft.runs) {
647
+ if (isRunningRuntimeRun(run) && run.settlementListener) {
648
+ draft.runs.set(runId, { ...run, settlementListener: undefined });
649
+ }
650
+ }
651
+ return { value: undefined };
652
+ });
653
+ this.settlementListeners.clear();
654
+ this.stateListeners.clear();
655
+ Deferred.doneUnsafe(this.shutdownCompletion, Effect.void);
656
+ }),
657
+ ),
658
+ );
457
659
  }
458
660
 
459
- private async performShutdown(): Promise<void> {
460
- try {
461
- this.closeReadyInteractiveWorkersForShutdown();
462
- const active = [...this.workers.values()]
463
- .filter((worker) => isActiveWorkerStatus(worker.status))
464
- .map((worker) => worker.id);
465
- await this.cancelWorkers(active);
466
- } finally {
467
- try {
468
- await this.scheduler.close();
469
- } catch {
470
- // Scheduler closure is best-effort during process shutdown.
471
- } finally {
472
- await this.awaitTrackedCleanupBestEffort();
473
- this.runSettlementListeners.clear();
474
- this.settlementListeners.clear();
475
- this.stateListeners.clear();
476
- }
477
- }
661
+ private performShutdown(): Effect.Effect<void> {
662
+ return Effect.fn("Orchestration.shutdown")(function* (
663
+ this: StatefulOrchestration,
664
+ ) {
665
+ const now = this.clock.currentTimeMillisUnsafe();
666
+ this.transact((draft) => {
667
+ const owners = new Set<string>();
668
+ const actions: PostCommitAction[] = [];
669
+ for (const worker of draft.workers.values()) {
670
+ if (
671
+ worker.record.lifecycle === "interactive" &&
672
+ worker.record.status === "ready"
673
+ ) {
674
+ this.closeReadyWorker(draft, worker, now, actions);
675
+ owners.add(worker.record.ownerSessionId);
676
+ }
677
+ }
678
+ addAll(owners, pruneHistory(draft));
679
+ actions.push(...publishOwners(owners));
680
+ return { value: undefined, actions };
681
+ });
682
+
683
+ const activeIds = activeWorkerIds(this.current());
684
+ const candidates = yield* makeCancellationCandidates(activeIds);
685
+ const cancellations = this.beginShutdownCancellation(candidates);
686
+ yield* awaitAll(cancellations);
687
+ yield* this.childSessions.shutdown().pipe(Effect.catchCause(() => Effect.void));
688
+ yield* FiberSet.awaitEmpty(this.cleanups).pipe(
689
+ Effect.timeoutOption(SHUTDOWN_CLEANUP_GRACE_MS),
690
+ Effect.ignore,
691
+ );
692
+ }).call(this);
478
693
  }
479
694
 
480
695
  private validateTask(
481
696
  context: OrchestrationContext,
482
697
  task: OrchestrateTaskInput,
483
698
  mode: RunMode,
484
- ): WorkerDefinition {
485
- validateContextOwner(context.ownerSessionId);
486
- validateMode(mode);
487
- if (!task || typeof task !== "object" || Array.isArray(task)) {
488
- throw new Error("orchestrate requires one task object");
489
- }
490
- if (context.synthesisGroup) {
491
- validateText("synthesis group ID", context.synthesisGroup.id, MAX_WORKER_TITLE_LENGTH);
492
- if (mode !== "async") throw new Error("Sibling synthesis requires an async task");
493
- if (!Number.isSafeInteger(context.synthesisGroup.size) || context.synthesisGroup.size < 2) {
494
- throw new Error("Synthesis group size must be an integer of at least 2");
699
+ ): Effect.Effect<
700
+ { definition: WorkerDefinition; task: OrchestrateTaskInput },
701
+ OrchestrationActionRejected
702
+ > {
703
+ return Effect.gen(function* () {
704
+ yield* validateContextOwner("orchestrate", context.ownerSessionId);
705
+ yield* validateMode("orchestrate", mode);
706
+ if (!task || typeof task !== "object" || Array.isArray(task)) {
707
+ return yield* rejectAction(
708
+ "orchestrate",
709
+ "validation",
710
+ "orchestrate requires one task object",
711
+ );
495
712
  }
496
- }
497
-
498
- validateText("worker", task.worker, MAX_WORKER_TITLE_LENGTH);
499
- validateText("title", task.title, MAX_WORKER_TITLE_LENGTH);
500
- validateText("instructions", task.instructions, MAX_WORKER_INSTRUCTIONS_LENGTH);
501
- const definition = findWorkerByName(context.catalog, task.worker);
502
- if (!definition) throw new Error(`Unknown worker: ${task.worker}`);
503
- resolveWorkerModel(definition, context.parentModel, context.modelRegistry);
504
- return definition;
505
- }
506
-
507
- private assertFreshIds(runId: RunId, workerId: WorkerId): void {
508
- if (this.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
509
- if (this.workers.has(workerId)) throw new Error(`Duplicate worker ID: ${workerId}`);
713
+ if (context.synthesisGroup) {
714
+ yield* validateText(
715
+ "orchestrate",
716
+ "synthesis group ID",
717
+ context.synthesisGroup.id,
718
+ MAX_WORKER_TITLE_LENGTH,
719
+ );
720
+ if (mode !== "async") {
721
+ return yield* rejectAction(
722
+ "orchestrate",
723
+ "validation",
724
+ "Sibling synthesis requires an async task",
725
+ );
726
+ }
727
+ if (
728
+ !Number.isSafeInteger(context.synthesisGroup.size) ||
729
+ context.synthesisGroup.size < 2
730
+ ) {
731
+ return yield* rejectAction(
732
+ "orchestrate",
733
+ "validation",
734
+ "Synthesis group size must be an integer of at least 2",
735
+ );
736
+ }
737
+ }
738
+ yield* validateText(
739
+ "orchestrate",
740
+ "worker",
741
+ task.worker,
742
+ MAX_WORKER_TITLE_LENGTH,
743
+ );
744
+ yield* validateText(
745
+ "orchestrate",
746
+ "title",
747
+ task.title,
748
+ MAX_WORKER_TITLE_LENGTH,
749
+ );
750
+ yield* validateText(
751
+ "orchestrate",
752
+ "instructions",
753
+ task.instructions,
754
+ MAX_WORKER_INSTRUCTIONS_LENGTH,
755
+ );
756
+ const definition = findWorkerByName(context.catalog, task.worker);
757
+ if (!definition) {
758
+ return yield* rejectAction(
759
+ "orchestrate",
760
+ "unknown-worker",
761
+ `Unknown worker: ${task.worker}`,
762
+ );
763
+ }
764
+ const configured = definition.model;
765
+ if (!configured && !context.parentModel) {
766
+ return yield* rejectAction(
767
+ "orchestrate",
768
+ "model-unavailable",
769
+ `Worker "${definition.name}" has no configured model and no parent model is available`,
770
+ );
771
+ }
772
+ if (
773
+ configured &&
774
+ !context.modelRegistry.find(configured.provider, configured.modelId)
775
+ ) {
776
+ return yield* rejectAction(
777
+ "orchestrate",
778
+ "model-unavailable",
779
+ `Worker "${definition.name}" configured model "${configured.provider}/${configured.modelId}" was not found`,
780
+ );
781
+ }
782
+ return { definition, task };
783
+ });
510
784
  }
511
785
 
512
786
  private launchBootstrap(workerId: WorkerId, generation: number): void {
513
- try {
514
- this.scheduler.start(
515
- workerId,
516
- this.bootstrapAndPrompt(workerId, generation),
517
- (error) => this.settleWorkflowDefect(workerId, generation, error),
518
- );
519
- } catch (error) {
520
- this.settleWorkflowDefect(workerId, generation, error);
521
- }
787
+ this.launchGeneration(
788
+ workerId,
789
+ generation,
790
+ this.bootstrapAndPrompt(workerId, generation),
791
+ );
522
792
  }
523
793
 
524
794
  private launchPrompt(
@@ -526,29 +796,73 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
526
796
  generation: number,
527
797
  session: WorkerSessionHandle,
528
798
  instructions: string,
799
+ ): void {
800
+ this.launchGeneration(
801
+ workerId,
802
+ generation,
803
+ this.executePrompt(workerId, generation, session, instructions),
804
+ );
805
+ }
806
+
807
+ private launchGeneration(
808
+ workerId: WorkerId,
809
+ generation: number,
810
+ workflow: Effect.Effect<void, never>,
529
811
  ): void {
530
812
  try {
531
- this.scheduler.start(
813
+ const fiber = this.runGeneration(
532
814
  workerId,
533
- this.executePrompt(workerId, generation, session, instructions),
534
- (error) => this.settleWorkflowDefect(workerId, generation, error),
815
+ workflow.pipe(
816
+ Effect.catchCause((cause) => {
817
+ if (Cause.hasInterruptsOnly(cause)) return Effect.void;
818
+ return Effect.sync(() => {
819
+ this.settleWorkflowDefect(workerId, generation, Cause.squash(cause));
820
+ });
821
+ }),
822
+ ),
535
823
  );
824
+ const exit = fiber.pollUnsafe();
825
+ if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
826
+ this.settleGenerationLaunchFailure(workerId, generation);
827
+ }
536
828
  } catch (error) {
537
829
  this.settleWorkflowDefect(workerId, generation, error);
538
830
  }
539
831
  }
540
832
 
833
+ private settleGenerationLaunchFailure(
834
+ workerId: WorkerId,
835
+ generation: number,
836
+ ): void {
837
+ this.settleActiveWorker(
838
+ workerId,
839
+ generation,
840
+ ["starting", "running"],
841
+ "failed",
842
+ {
843
+ status: "failed",
844
+ message: "Worker generation could not start because orchestration is closed",
845
+ },
846
+ "workflow",
847
+ true,
848
+ );
849
+ }
850
+
541
851
  private bootstrapAndPrompt(
542
852
  workerId: WorkerId,
543
853
  generation: number,
544
854
  ): Effect.Effect<void, never> {
545
- const runtime = this;
546
- return Effect.gen(function* () {
547
- const session = yield* runtime.bootstrap(workerId, generation);
855
+ return Effect.gen({ self: this }, function* () {
856
+ const session = yield* this.bootstrap(workerId, generation);
548
857
  if (!session) return;
549
- const current = runtime.workers.get(workerId);
550
- if (!current) return;
551
- yield* runtime.executePrompt(workerId, generation, session, current.instructions);
858
+ const worker = this.current().workers.get(workerId);
859
+ if (!worker) return;
860
+ yield* this.executePrompt(
861
+ workerId,
862
+ generation,
863
+ session,
864
+ worker.record.instructions,
865
+ );
552
866
  });
553
867
  }
554
868
 
@@ -557,89 +871,109 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
557
871
  generation: number,
558
872
  ): Effect.Effect<WorkerSessionHandle | undefined, never> {
559
873
  return Effect.suspend(() => {
560
- const entry = this.entries.get(workerId);
561
- if (!entry) return Effect.succeed(undefined);
562
-
563
- let creation: Promise<WorkerSessionHandle>;
564
- try {
565
- creation = this.workerSessionFactory.create({
566
- cwd: entry.context.cwd,
567
- agentDir: entry.context.agentDir,
568
- parentSessionFile: entry.context.parentSessionFile,
569
- projectTrusted: entry.context.projectTrusted,
570
- definition: entry.definition,
571
- parentModel: entry.context.parentModel,
572
- modelRegistry: entry.context.modelRegistry,
573
- });
574
- } catch (error) {
575
- this.settleCreationFailure(workerId, generation, error);
874
+ const expected = this.current().workers.get(workerId);
875
+ if (!expected || expected.generation !== generation) {
576
876
  return Effect.succeed(undefined);
577
877
  }
578
-
579
- this.trackCleanup(creation.then(() => undefined, () => undefined));
580
- void creation.then((session) => {
581
- if (!this.canAdoptCreatedSession(workerId, generation, entry)) {
582
- this.disposeSession(session);
583
- }
584
- }, () => undefined);
585
-
586
- return Effect.tryPromise({
587
- try: () => creation,
588
- catch: (error) => error,
589
- }).pipe(
878
+ return this.childSessions.acquire(
879
+ {
880
+ cwd: expected.context.cwd,
881
+ agentDir: expected.context.agentDir,
882
+ parentSessionFile: expected.context.parentSessionFile,
883
+ projectTrusted: expected.context.projectTrusted,
884
+ definition: expected.definition,
885
+ parentModel: expected.context.parentModel,
886
+ modelRegistry: expected.context.modelRegistry,
887
+ },
888
+ (session) => this.adoptCreatedSession(workerId, generation, session),
889
+ ).pipe(
590
890
  Effect.match({
591
891
  onFailure: (error) => {
592
892
  this.settleCreationFailure(workerId, generation, error);
593
893
  return undefined;
594
894
  },
595
- onSuccess: (session) => this.adoptCreatedSession(
596
- workerId,
597
- generation,
598
- entry,
599
- session,
600
- ),
895
+ onSuccess: (session) => session,
601
896
  }),
602
897
  );
603
898
  });
604
899
  }
605
900
 
606
- private canAdoptCreatedSession(
607
- workerId: WorkerId,
608
- generation: number,
609
- entry: RuntimeEntry,
610
- ): boolean {
611
- const current = this.workers.get(workerId);
612
- return !this.shuttingDown &&
613
- current?.status === "starting" &&
614
- entry.generation === generation;
615
- }
616
-
617
901
  private adoptCreatedSession(
618
902
  workerId: WorkerId,
619
903
  generation: number,
620
- entry: RuntimeEntry,
621
904
  session: WorkerSessionHandle,
622
905
  ): WorkerSessionHandle | undefined {
623
- const current = this.workers.get(workerId);
624
- if (!this.canAdoptCreatedSession(workerId, generation, entry) || !current) {
625
- this.disposeSession(session);
906
+ let release: () => void;
907
+ try {
908
+ release = session.subscribeObservation((observation) => {
909
+ this.updateObservation(workerId, generation, session, observation);
910
+ });
911
+ } catch (error) {
912
+ this.settleCreationFailure(workerId, generation, error);
626
913
  return undefined;
627
914
  }
628
915
 
916
+ const adopted = this.transact((draft) => {
917
+ const worker = draft.workers.get(workerId);
918
+ if (
919
+ draft.lifecycle !== "open" ||
920
+ !worker ||
921
+ worker.generation !== generation ||
922
+ worker.record.status !== "starting"
923
+ ) {
924
+ return {
925
+ value: false,
926
+ actions: [runAction(() => safelyCall(release))],
927
+ };
928
+ }
929
+ draft.workers.set(workerId, {
930
+ ...worker,
931
+ session,
932
+ observationRelease: release,
933
+ record: {
934
+ ...transitionWorkerStatus(worker.record, "running"),
935
+ sessionFile: session.sessionFile,
936
+ },
937
+ });
938
+ return {
939
+ value: true,
940
+ actions: [publishState(worker.record.ownerSessionId)],
941
+ };
942
+ });
943
+ return adopted ? session : undefined;
944
+ }
945
+
946
+ private subscribeObservation(
947
+ workerId: WorkerId,
948
+ generation: number,
949
+ session: WorkerSessionHandle,
950
+ ): void {
951
+ let release: () => void;
629
952
  try {
630
- entry.session = session;
631
- this.subscribeEntryObservability(workerId, entry, session, generation);
632
- this.workers.set(workerId, {
633
- ...transitionWorkerStatus(current, "running"),
634
- sessionFile: session.sessionFile,
953
+ release = session.subscribeObservation((observation) => {
954
+ this.updateObservation(workerId, generation, session, observation);
635
955
  });
636
- this.emitState(current.ownerSessionId);
637
- return session;
638
956
  } catch (error) {
639
- this.disposeEntrySession(entry);
640
- this.settleCreationFailure(workerId, generation, error);
641
- return undefined;
957
+ this.settleWorkflowDefect(workerId, generation, error);
958
+ return;
642
959
  }
960
+
961
+ this.transact((draft) => {
962
+ const worker = draft.workers.get(workerId);
963
+ if (
964
+ !worker ||
965
+ worker.generation !== generation ||
966
+ worker.session !== session ||
967
+ worker.record.status !== "running"
968
+ ) {
969
+ return {
970
+ value: undefined,
971
+ actions: [runAction(() => safelyCall(release))],
972
+ };
973
+ }
974
+ draft.workers.set(workerId, { ...worker, observationRelease: release });
975
+ return { value: undefined };
976
+ });
643
977
  }
644
978
 
645
979
  private executePrompt(
@@ -649,34 +983,16 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
649
983
  instructions: string,
650
984
  ): Effect.Effect<void, never> {
651
985
  return Effect.suspend(() => {
652
- const before = this.workers.get(workerId);
653
- const entry = this.entries.get(workerId);
986
+ const worker = this.current().workers.get(workerId);
654
987
  if (
655
- !before ||
656
- before.status !== "running" ||
657
- !entry ||
658
- entry.generation !== generation ||
659
- entry.session !== session
988
+ !worker ||
989
+ worker.record.status !== "running" ||
990
+ worker.generation !== generation ||
991
+ worker.session !== session
660
992
  ) {
661
993
  return Effect.void;
662
994
  }
663
-
664
- let prompt: Promise<WorkerOutcome>;
665
- try {
666
- prompt = session.prompt(instructions);
667
- } catch (error) {
668
- this.settleOutcome(workerId, generation, session, {
669
- status: "failed",
670
- message: describeError(error, "Worker prompt failed"),
671
- });
672
- return Effect.void;
673
- }
674
- this.trackCleanup(prompt.then(() => undefined, () => undefined));
675
-
676
- return Effect.tryPromise({
677
- try: () => prompt,
678
- catch: (error) => error,
679
- }).pipe(
995
+ return session.prompt(instructions).pipe(
680
996
  Effect.match({
681
997
  onFailure: (error): WorkerOutcome => ({
682
998
  status: "failed",
@@ -697,13 +1013,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
697
1013
  generation: number,
698
1014
  error: unknown,
699
1015
  ): void {
700
- const current = this.workers.get(workerId);
701
- const entry = this.entries.get(workerId);
702
- if (!current || current.status !== "starting" || entry?.generation !== generation) return;
703
- this.settleTerminalWorker(current, "failed", {
704
- status: "failed",
705
- message: describeError(error, "Worker session creation failed"),
706
- }, "startup");
1016
+ this.settleActiveWorker(
1017
+ workerId,
1018
+ generation,
1019
+ ["starting"],
1020
+ "failed",
1021
+ {
1022
+ status: "failed",
1023
+ message: describeError(error, "Worker session creation failed"),
1024
+ },
1025
+ "startup",
1026
+ false,
1027
+ );
707
1028
  }
708
1029
 
709
1030
  private settleWorkflowDefect(
@@ -711,19 +1032,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
711
1032
  generation: number,
712
1033
  error: unknown,
713
1034
  ): void {
714
- const current = this.workers.get(workerId);
715
- const entry = this.entries.get(workerId);
716
- if (!current || entry?.generation !== generation) return;
717
- if (!isActiveWorkerStatus(current.status)) {
718
- this.maybeCompleteRun(current.runId);
719
- return;
720
- }
721
-
722
- this.disposeEntrySession(entry);
723
- this.settleTerminalWorker(current, "failed", {
724
- status: "failed",
725
- message: describeError(error, "Worker workflow failed"),
726
- }, "workflow");
1035
+ this.settleActiveWorker(
1036
+ workerId,
1037
+ generation,
1038
+ ["starting", "running", "stopping"],
1039
+ "failed",
1040
+ {
1041
+ status: "failed",
1042
+ message: describeError(error, "Worker workflow failed"),
1043
+ },
1044
+ "workflow",
1045
+ true,
1046
+ );
727
1047
  }
728
1048
 
729
1049
  private settleOutcome(
@@ -732,14 +1052,12 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
732
1052
  session: WorkerSessionHandle,
733
1053
  outcome: WorkerOutcome,
734
1054
  ): void {
735
- const current = this.workers.get(workerId);
736
- const entry = this.entries.get(workerId);
1055
+ const worker = this.current().workers.get(workerId);
737
1056
  if (
738
- !current ||
739
- current.status !== "running" ||
740
- !entry ||
741
- entry.generation !== generation ||
742
- entry.session !== session
1057
+ !worker ||
1058
+ worker.generation !== generation ||
1059
+ worker.session !== session ||
1060
+ worker.record.status !== "running"
743
1061
  ) {
744
1062
  return;
745
1063
  }
@@ -748,549 +1066,1026 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
748
1066
  let status: "ready" | "completed" | "failed" | "aborted";
749
1067
  if (outcome.status === "failed" || outcome.status === "aborted") {
750
1068
  status = outcome.status;
751
- } else if (current.lifecycle === "interactive" && outcome.status === "ready") {
1069
+ } else if (
1070
+ worker.record.lifecycle === "interactive" &&
1071
+ outcome.status === "ready"
1072
+ ) {
752
1073
  status = "ready";
753
- } else if (current.lifecycle === "one-shot" && outcome.status === "completed") {
1074
+ } else if (
1075
+ worker.record.lifecycle === "one-shot" &&
1076
+ outcome.status === "completed"
1077
+ ) {
754
1078
  status = "completed";
755
1079
  } else {
756
1080
  status = "failed";
757
1081
  settledOutcome = {
758
1082
  status: "failed",
759
- message: `Worker session returned ${outcome.status} for a ${current.lifecycle} worker`,
1083
+ message: `Worker session returned ${outcome.status} for a ${worker.record.lifecycle} worker`,
760
1084
  };
761
1085
  }
762
-
763
- const settledAt = this.clock();
764
- this.workers.set(workerId, {
765
- ...transitionWorkerStatus(current, status),
766
- activity: undefined,
767
- outcome: copyOutcome(settledOutcome),
768
- settledAt,
769
- });
770
- if (status !== "ready") this.disposeEntrySession(entry);
771
- this.maybeCompleteRun(current.runId);
772
- const affectedOwners = new Set([current.ownerSessionId]);
773
- this.emitSettlement(
1086
+ this.settleActiveWorker(
774
1087
  workerId,
775
1088
  generation,
776
- settledAt,
1089
+ ["running"],
1090
+ status,
1091
+ settledOutcome,
777
1092
  status === "failed" ? "prompt" : undefined,
1093
+ status !== "ready",
778
1094
  );
779
- if (isTerminalWorkerStatus(status)) {
780
- addAll(affectedOwners, this.rememberTerminalWorker(workerId));
781
- } else {
782
- addAll(affectedOwners, this.pruneHistory());
783
- }
784
- this.emitStateForOwners(affectedOwners);
785
1095
  }
786
1096
 
787
- private settleTerminalWorker(
788
- current: WorkerRecord,
789
- status: "failed" | "aborted",
1097
+ private settleActiveWorker(
1098
+ workerId: WorkerId,
1099
+ generation: number,
1100
+ expectedStatuses: readonly WorkerRecord["status"][],
1101
+ status: "ready" | "completed" | "failed" | "aborted",
790
1102
  outcome: WorkerOutcome,
791
- failureStage?: SettlementFailureStage,
1103
+ failureStage: SettlementFailureStage | undefined,
1104
+ dispose: boolean,
792
1105
  ): void {
793
- const settledAt = this.clock();
794
- this.workers.set(current.id, {
795
- ...transitionWorkerStatus(current, status),
796
- activity: undefined,
797
- outcome: copyOutcome(outcome),
798
- settledAt,
1106
+ const settledAt = this.clock.currentTimeMillisUnsafe();
1107
+ this.transact((draft) => {
1108
+ const worker = draft.workers.get(workerId);
1109
+ if (
1110
+ !worker ||
1111
+ worker.generation !== generation ||
1112
+ !expectedStatuses.includes(worker.record.status)
1113
+ ) {
1114
+ return { value: undefined };
1115
+ }
1116
+
1117
+ const settledRecord: WorkerRecord = {
1118
+ ...transitionWorkerStatus(worker.record, status),
1119
+ activity: undefined,
1120
+ outcome: copyOutcome(outcome),
1121
+ settledAt,
1122
+ };
1123
+ const actions: PostCommitAction[] = [];
1124
+ let settledWorker: RuntimeWorker = { ...worker, record: settledRecord };
1125
+ if (dispose) {
1126
+ actions.push(...this.releaseWorkerResources(settledWorker));
1127
+ settledWorker = {
1128
+ ...settledWorker,
1129
+ session: undefined,
1130
+ observationRelease: undefined,
1131
+ };
1132
+ }
1133
+ draft.workers.set(workerId, settledWorker);
1134
+ const settlement = makeSettlement(
1135
+ draft,
1136
+ settledWorker,
1137
+ settledAt,
1138
+ failureStage,
1139
+ );
1140
+ actions.push(...completeRun(draft, workerId));
1141
+ if (settlement) actions.push(settlement);
1142
+ if (isTerminalWorkerStatus(status)) rememberTerminalWorker(draft, workerId);
1143
+ actions.push(...stateActionsAfterPrune(draft, worker.record.ownerSessionId));
1144
+ return { value: undefined, actions };
799
1145
  });
800
- this.maybeCompleteRun(current.runId);
801
- const affectedOwners = new Set([current.ownerSessionId]);
802
- const generation = this.entries.get(current.id)?.generation;
803
- if (generation !== undefined) {
804
- this.emitSettlement(current.id, generation, settledAt, failureStage);
805
- }
806
- addAll(affectedOwners, this.rememberTerminalWorker(current.id));
807
- this.emitStateForOwners(affectedOwners);
808
1146
  }
809
1147
 
810
- private emitSettlement(
1148
+ private updateObservation(
811
1149
  workerId: WorkerId,
812
1150
  generation: number,
813
- settledAt: number,
814
- failureStage?: SettlementFailureStage,
1151
+ session: WorkerSessionHandle,
1152
+ observation: WorkerSessionObservation,
815
1153
  ): void {
816
- const worker = this.workers.get(workerId);
817
- const run = worker ? this.runs.get(worker.runId) : undefined;
818
- if (!worker || !run || !worker.outcome || !isSettledWorkerStatus(worker.status)) return;
819
- if (worker.outcome.status === "closed") return;
820
-
821
- const sequence = ++this.settlementSequence;
822
- const settlement: WorkerSettlement = Object.freeze({
823
- eventId: `${sequence}:${run.id}:${workerId}:${generation}`,
824
- sequence,
825
- ownerSessionId: worker.ownerSessionId,
826
- runId: run.id,
827
- workerId,
828
- generation,
829
- mode: run.mode,
830
- worker: worker.worker,
831
- title: worker.title,
832
- lifecycle: worker.lifecycle,
833
- status: worker.status,
834
- outcome: Object.freeze(copyOutcome(worker.outcome)),
835
- ...(failureStage ? { failureStage } : {}),
836
- usage: Object.freeze(copyUsage(worker.usage)),
837
- startedAt: worker.startedAt,
838
- settledAt,
839
- ...(run.synthesisGroupId && run.synthesisGroupSize
840
- ? {
841
- synthesisGroupId: run.synthesisGroupId,
842
- synthesisGroupSize: run.synthesisGroupSize,
843
- }
844
- : {}),
845
- ...(worker.sessionFile !== undefined ? { sessionFile: worker.sessionFile } : {}),
1154
+ this.transact((draft) => {
1155
+ const worker = draft.workers.get(workerId);
1156
+ if (
1157
+ !worker ||
1158
+ worker.generation !== generation ||
1159
+ worker.session !== session ||
1160
+ (worker.record.status !== "starting" && worker.record.status !== "running")
1161
+ ) {
1162
+ return { value: undefined };
1163
+ }
1164
+ const nextRecord: WorkerRecord = {
1165
+ ...worker.record,
1166
+ usage: copyUsage(observation.usage),
1167
+ activity: observation.activity,
1168
+ messageDirection: observation.messageDirection,
1169
+ };
1170
+ if (observationsEqual(worker.record, nextRecord)) {
1171
+ return { value: undefined };
1172
+ }
1173
+ draft.workers.set(workerId, { ...worker, record: nextRecord });
1174
+ return {
1175
+ value: undefined,
1176
+ actions: [publishState(worker.record.ownerSessionId)],
1177
+ };
846
1178
  });
847
-
848
- const localListener = this.runSettlementListeners.get(run.id);
849
- this.runSettlementListeners.delete(run.id);
850
- notifySettlementListener(localListener, settlement);
851
- for (const listener of [...this.settlementListeners]) {
852
- notifySettlementListener(listener, settlement);
853
- }
854
1179
  }
855
1180
 
856
- private maybeCompleteRun(runId: RunId): void {
857
- if (this.completedRuns.has(runId)) return;
858
- const run = this.runs.get(runId);
859
- if (!run) return;
860
- const worker = this.workers.get(run.workerId);
861
- if (!worker || worker.runId !== runId || !worker.outcome || isActiveWorkerStatus(worker.status)) {
862
- return;
863
- }
864
-
865
- const completed = freezeCompletedRun(run, worker);
866
- this.completedRuns.set(runId, completed);
867
- this.completedRunOrder.push(runId);
868
- this.runs.set(runId, { ...run, state: "complete" });
869
- this.runWaiters.get(runId)?.resolve(completed);
870
- this.runWaiters.delete(runId);
871
- }
1181
+ private beginCancellation(
1182
+ operation: "abort",
1183
+ ownerSessionId: string,
1184
+ target: ValidatedAbortTarget,
1185
+ candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
1186
+ ): Decision<readonly Deferred.Deferred<void>[]> {
1187
+ return this.transact((draft) => {
1188
+ const open = openDecision(draft, operation);
1189
+ if (open._tag === "rejected") return { value: open };
1190
+ const workers = target._tag === "all"
1191
+ ? [...draft.workers.values()].filter((worker) => (
1192
+ worker.record.ownerSessionId === ownerSessionId &&
1193
+ isActiveWorkerStatus(worker.record.status)
1194
+ ))
1195
+ : target.workerIds.map((id) => draft.workers.get(id));
1196
+
1197
+ if (target._tag === "ids") {
1198
+ for (let index = 0; index < target.workerIds.length; index += 1) {
1199
+ const worker = workers[index];
1200
+ if (!worker || worker.record.ownerSessionId !== ownerSessionId) {
1201
+ return {
1202
+ value: rejected(
1203
+ operation,
1204
+ "ownership",
1205
+ "Worker is not owned by this session",
1206
+ ),
1207
+ };
1208
+ }
1209
+ if (worker.record.status === "ready") {
1210
+ return {
1211
+ value: rejected(
1212
+ operation,
1213
+ "worker-state",
1214
+ "Ready interactive workers are not active; use interactive_close",
1215
+ ),
1216
+ };
1217
+ }
1218
+ if (!isActiveWorkerStatus(worker.record.status)) {
1219
+ return {
1220
+ value: rejected(
1221
+ operation,
1222
+ "worker-state",
1223
+ "worker_abort requires owned active workers",
1224
+ ),
1225
+ };
1226
+ }
1227
+ }
1228
+ }
872
1229
 
873
- private rememberTerminalWorker(workerId: WorkerId): Set<string> {
874
- if (!this.terminalWorkerOrder.includes(workerId)) {
875
- this.terminalWorkerOrder.push(workerId);
876
- }
877
- return this.pruneHistory();
1230
+ const activeWorkers = workers.filter(isRuntimeWorker);
1231
+ const marked = this.markWorkersStopping(draft, activeWorkers, candidates);
1232
+ return {
1233
+ value: accepted(marked.completions),
1234
+ actions: [
1235
+ ...marked.actions,
1236
+ ...publishOwners(new Set(
1237
+ activeWorkers.map((worker) => worker.record.ownerSessionId),
1238
+ )),
1239
+ ],
1240
+ };
1241
+ });
878
1242
  }
879
1243
 
880
- private pruneHistory(): Set<string> {
881
- const affectedOwners = new Set<string>();
882
- while (this.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
883
- const runId = this.completedRunOrder.shift();
884
- if (!runId) break;
885
- this.completedRuns.delete(runId);
886
- const run = this.runs.get(runId);
887
- if (run) affectedOwners.add(run.ownerSessionId);
888
- this.runs.delete(runId);
889
- }
890
-
891
- while (this.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
892
- const removableIndex = this.terminalWorkerOrder.findIndex((workerId) => {
893
- const worker = this.workers.get(workerId);
894
- if (!worker || !isTerminalWorkerStatus(worker.status)) return true;
895
- return this.runs.get(worker.runId)?.state !== "running";
896
- });
897
- if (removableIndex < 0) return affectedOwners;
898
- const [workerId] = this.terminalWorkerOrder.splice(removableIndex, 1);
899
- if (!workerId) return affectedOwners;
900
- const worker = this.workers.get(workerId);
901
- if (worker) affectedOwners.add(worker.ownerSessionId);
902
- this.workers.delete(workerId);
903
- this.entries.delete(workerId);
904
- }
905
-
906
- return affectedOwners;
1244
+ private beginShutdownCancellation(
1245
+ candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
1246
+ ): readonly Deferred.Deferred<void>[] {
1247
+ return this.transact((draft) => {
1248
+ const workers = [...draft.workers.values()].filter((worker) => (
1249
+ isActiveWorkerStatus(worker.record.status)
1250
+ ));
1251
+ const marked = this.markWorkersStopping(draft, workers, candidates);
1252
+ return {
1253
+ value: marked.completions,
1254
+ actions: [
1255
+ ...marked.actions,
1256
+ ...publishOwners(new Set(
1257
+ workers.map((worker) => worker.record.ownerSessionId),
1258
+ )),
1259
+ ],
1260
+ };
1261
+ });
907
1262
  }
908
1263
 
909
- private resolveAbortTargets(ownerSessionId: string, target: AbortTarget): WorkerId[] {
910
- if (!target || typeof target !== "object") throw new Error("Invalid abort target");
911
- const candidate = target as {
912
- workerIds?: readonly WorkerId[];
913
- all?: boolean;
914
- };
915
- const selected = [
916
- candidate.workerIds !== undefined,
917
- candidate.all !== undefined,
918
- ].filter(Boolean).length;
919
- if (selected !== 1 || (candidate.all !== undefined && candidate.all !== true)) {
920
- throw new Error("Abort target must specify exactly one of workerIds or all: true");
921
- }
922
-
923
- if (candidate.workerIds !== undefined) {
924
- if (!Array.isArray(candidate.workerIds) || candidate.workerIds.length === 0) {
925
- throw new Error("workerIds must contain at least one worker ID");
1264
+ private markWorkersStopping(
1265
+ draft: RuntimeState,
1266
+ workers: readonly RuntimeWorker[],
1267
+ candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
1268
+ ): {
1269
+ readonly completions: readonly Deferred.Deferred<void>[];
1270
+ readonly actions: readonly PostCommitAction[];
1271
+ } {
1272
+ const completions: Deferred.Deferred<void>[] = [];
1273
+ const actions: PostCommitAction[] = [];
1274
+ for (const worker of workers) {
1275
+ if (worker.cancellation) {
1276
+ completions.push(worker.cancellation);
1277
+ continue;
926
1278
  }
927
- const unique = [...new Set(candidate.workerIds)];
928
- for (const workerId of unique) {
929
- const worker = this.ownedWorker(ownerSessionId, workerId);
930
- if (worker.status === "ready") {
931
- throw new Error("Ready interactive workers are not active; use interactive_close");
932
- }
933
- if (!isActiveWorkerStatus(worker.status)) {
934
- throw new Error("worker_abort requires owned active workers");
935
- }
1279
+ const completion = candidates.get(worker.record.id);
1280
+ if (!completion) {
1281
+ throw new Error(`Missing cancellation candidate: ${worker.record.id}`);
936
1282
  }
937
- return unique;
1283
+ const stoppingRecord = worker.record.status === "stopping"
1284
+ ? worker.record
1285
+ : {
1286
+ ...transitionWorkerStatus(worker.record, "stopping"),
1287
+ activity: undefined,
1288
+ };
1289
+ draft.workers.set(worker.record.id, {
1290
+ ...worker,
1291
+ record: stoppingRecord,
1292
+ cancellation: completion,
1293
+ });
1294
+ completions.push(completion);
1295
+ actions.push(runAction(() => {
1296
+ this.launchCancellationOrSettleAfterClosure(worker.record.id, completion);
1297
+ }));
938
1298
  }
939
-
940
- return [...this.workers.values()]
941
- .filter(
942
- (worker) =>
943
- worker.ownerSessionId === ownerSessionId && isActiveWorkerStatus(worker.status),
944
- )
945
- .map((worker) => worker.id);
1299
+ return { completions, actions };
946
1300
  }
947
1301
 
948
- private async cancelWorkers(workerIds: readonly WorkerId[]): Promise<void> {
949
- const owners = new Set<string>();
950
- const cancellations: Promise<void>[] = [];
1302
+ private launchCancellationOrSettleAfterClosure(
1303
+ workerId: WorkerId,
1304
+ completion: Deferred.Deferred<void>,
1305
+ ): void {
1306
+ const fiber = this.runCancellation(this.cancelWorker(workerId, completion));
1307
+ const exit = fiber.pollUnsafe();
1308
+ if (!exit || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return;
951
1309
 
952
- for (const workerId of workerIds) {
953
- const current = this.workers.get(workerId);
954
- if (!current || !isActiveWorkerStatus(current.status)) continue;
955
- if (current.status !== "stopping") {
956
- this.workers.set(workerId, {
957
- ...transitionWorkerStatus(current, "stopping"),
958
- activity: undefined,
959
- });
960
- }
961
- owners.add(current.ownerSessionId);
962
- cancellations.push(this.cancellationFor(workerId));
1310
+ const worker = this.current().workers.get(workerId);
1311
+ if (worker?.record.status === "stopping") {
1312
+ this.settleActiveWorker(
1313
+ workerId,
1314
+ worker.generation,
1315
+ ["stopping"],
1316
+ "aborted",
1317
+ { status: "aborted" },
1318
+ "cancellation",
1319
+ true,
1320
+ );
963
1321
  }
964
-
965
- this.emitStateForOwners(owners);
966
- await Promise.all(cancellations);
967
- }
968
-
969
- private cancellationFor(workerId: WorkerId): Promise<void> {
970
- const existing = this.cancellationPromises.get(workerId);
971
- if (existing) return existing;
972
-
973
- const cancellation = Promise.resolve()
974
- .then(() => this.cancelWorker(workerId))
975
- .catch((error) => {
976
- this.settleCancellationFailure(workerId, error);
977
- });
978
- this.cancellationPromises.set(workerId, cancellation);
979
- void cancellation.then(() => {
980
- if (this.cancellationPromises.get(workerId) === cancellation) {
981
- this.cancellationPromises.delete(workerId);
982
- }
983
- });
984
- return cancellation;
1322
+ this.completeCancellation(workerId, completion);
985
1323
  }
986
1324
 
987
- private async cancelWorker(workerId: WorkerId): Promise<void> {
988
- const entry = this.entries.get(workerId);
989
- const session = entry?.session;
990
-
991
- try {
1325
+ private cancelWorker(
1326
+ workerId: WorkerId,
1327
+ completion: Deferred.Deferred<void>,
1328
+ ): Effect.Effect<void> {
1329
+ return Effect.gen({ self: this }, function* () {
1330
+ const session = this.current().workers.get(workerId)?.session;
992
1331
  if (session) {
993
- const abortOperation = Promise.resolve()
994
- .then(() => session.abort())
995
- .catch(() => undefined);
996
- await this.waitBestEffort(abortOperation, CANCELLATION_GRACE_MS);
997
- }
998
- try {
999
- await this.scheduler.remove(workerId);
1000
- } catch {
1001
- // Worker state still settles even if scheduler cleanup fails.
1332
+ yield* session.abort().pipe(
1333
+ Effect.catchTag(
1334
+ "WorkerSession.AbortError",
1335
+ (_: WorkerSessionAbortError) => Effect.void,
1336
+ ),
1337
+ Effect.timeoutOption(CANCELLATION_GRACE_MS),
1338
+ Effect.ignore,
1339
+ );
1002
1340
  }
1003
- } finally {
1004
- if (entry) this.disposeEntrySession(entry);
1005
- const current = this.workers.get(workerId);
1006
- if (current?.status === "stopping") {
1007
- this.settleTerminalWorker(
1008
- current,
1009
- "aborted",
1010
- { status: "aborted" },
1341
+ const removal = yield* FiberSet.run(
1342
+ this.cancellations,
1343
+ FiberMap.remove(this.generations, workerId).pipe(
1344
+ Effect.catchCause(() => Effect.void),
1345
+ ),
1346
+ );
1347
+ yield* Fiber.await(removal).pipe(
1348
+ Effect.timeoutOption(CANCELLATION_GRACE_MS),
1349
+ Effect.ignore,
1350
+ );
1351
+ this.settleActiveWorker(
1352
+ workerId,
1353
+ this.current().workers.get(workerId)?.generation ?? -1,
1354
+ ["stopping"],
1355
+ "aborted",
1356
+ { status: "aborted" },
1357
+ "cancellation",
1358
+ true,
1359
+ );
1360
+ }).pipe(
1361
+ Effect.catchCause((cause) => Effect.sync(() => {
1362
+ const generation = this.current().workers.get(workerId)?.generation;
1363
+ if (generation === undefined) return;
1364
+ this.settleActiveWorker(
1365
+ workerId,
1366
+ generation,
1367
+ ["stopping"],
1368
+ "failed",
1369
+ {
1370
+ status: "failed",
1371
+ message: describeError(
1372
+ Cause.squash(cause),
1373
+ "Worker cancellation failed",
1374
+ ),
1375
+ },
1011
1376
  "cancellation",
1377
+ true,
1012
1378
  );
1013
- }
1014
- }
1015
- }
1016
-
1017
- private settleCancellationFailure(workerId: WorkerId, error: unknown): void {
1018
- const current = this.workers.get(workerId);
1019
- if (!current || current.status !== "stopping") return;
1020
- const entry = this.entries.get(workerId);
1021
- if (entry) this.disposeEntrySession(entry);
1022
- this.settleTerminalWorker(current, "failed", {
1023
- status: "failed",
1024
- message: describeError(error, "Worker cancellation failed"),
1025
- }, "cancellation");
1379
+ })),
1380
+ Effect.ensuring(
1381
+ Effect.sync(() => this.completeCancellation(workerId, completion)),
1382
+ ),
1383
+ );
1026
1384
  }
1027
1385
 
1028
- private async cancelExactRun(run: RunRecord): Promise<void> {
1029
- const worker = this.workers.get(run.workerId);
1030
- const active = worker?.ownerSessionId === run.ownerSessionId &&
1031
- worker.runId === run.id &&
1032
- isActiveWorkerStatus(worker.status)
1033
- ? [worker.id]
1034
- : [];
1035
- await this.cancelWorkers(active);
1036
- this.maybeCompleteRun(run.id);
1386
+ private completeCancellation(
1387
+ workerId: WorkerId,
1388
+ completion: Deferred.Deferred<void>,
1389
+ ): void {
1390
+ this.transact((draft) => {
1391
+ const worker = draft.workers.get(workerId);
1392
+ if (worker?.cancellation === completion) {
1393
+ draft.workers.set(workerId, { ...worker, cancellation: undefined });
1394
+ }
1395
+ return { value: undefined };
1396
+ });
1397
+ Deferred.doneUnsafe(completion, Effect.void);
1037
1398
  }
1038
1399
 
1039
1400
  private awaitInlineRun(
1040
1401
  run: RunRecord,
1041
- waiter: RunWaiter,
1042
- signal: AbortSignal | undefined,
1043
- ): Promise<CompletedRun> {
1044
- if (!signal) return waiter.promise;
1045
-
1046
- return new Promise<CompletedRun>((resolve, reject) => {
1047
- let abortClaimed = false;
1048
- const removeAbortListener = () => signal.removeEventListener("abort", onAbort);
1049
- const onAbort = () => {
1050
- if (abortClaimed || waiter.settled) return;
1051
- abortClaimed = true;
1052
- removeAbortListener();
1053
- const reason = abortSignalReason(signal);
1054
- void this.cancelExactRun(run).then(
1055
- () => reject(reason),
1056
- () => reject(reason),
1057
- );
1058
- };
1059
-
1060
- waiter.onSettled = () => {
1061
- removeAbortListener();
1062
- if (!abortClaimed) waiter.promise.then(resolve, reject);
1063
- };
1064
- signal.addEventListener("abort", onAbort, { once: true });
1402
+ completion: Deferred.Deferred<CompletedRun>,
1403
+ ): Effect.Effect<CompletedRun> {
1404
+ return Deferred.await(completion).pipe(
1405
+ Effect.onInterrupt(() => this.cancelExactRun(run)),
1406
+ );
1407
+ }
1065
1408
 
1066
- if (signal.aborted) onAbort();
1067
- else if (waiter.settled) waiter.onSettled();
1409
+ private cancelExactRun(run: RunRecord): Effect.Effect<void> {
1410
+ return Effect.gen({ self: this }, function* () {
1411
+ const candidate = yield* Deferred.make<void>();
1412
+ const candidates = new Map([[run.workerId, candidate]]);
1413
+ const completions = this.transact((draft) => {
1414
+ const worker = draft.workers.get(run.workerId);
1415
+ if (
1416
+ !worker ||
1417
+ worker.record.ownerSessionId !== run.ownerSessionId ||
1418
+ worker.record.runId !== run.id ||
1419
+ !isActiveWorkerStatus(worker.record.status)
1420
+ ) {
1421
+ return { value: [] as readonly Deferred.Deferred<void>[] };
1422
+ }
1423
+ const marked = this.markWorkersStopping(draft, [worker], candidates);
1424
+ return {
1425
+ value: marked.completions,
1426
+ actions: [...marked.actions, publishState(run.ownerSessionId)],
1427
+ };
1428
+ });
1429
+ yield* awaitAll(completions);
1068
1430
  });
1069
1431
  }
1070
1432
 
1071
- private closeReadyInteractiveWorker(current: WorkerRecord): void {
1072
- const entry = this.entries.get(current.id);
1073
- if (entry) this.disposeEntrySession(entry);
1074
- this.workers.set(current.id, {
1075
- ...transitionWorkerStatus(current, "closed"),
1433
+ private closeReadyWorker(
1434
+ draft: RuntimeState,
1435
+ worker: RuntimeWorker,
1436
+ settledAt: number,
1437
+ actions: PostCommitAction[] = [],
1438
+ ): PostCommitAction[] {
1439
+ const closedRecord: WorkerRecord = {
1440
+ ...transitionWorkerStatus(worker.record, "closed"),
1076
1441
  activity: undefined,
1077
1442
  outcome: { status: "closed" },
1078
- settledAt: this.clock(),
1443
+ settledAt,
1444
+ };
1445
+ actions.push(...this.releaseWorkerResources(worker));
1446
+ draft.workers.set(worker.record.id, {
1447
+ ...worker,
1448
+ record: closedRecord,
1449
+ session: undefined,
1450
+ observationRelease: undefined,
1079
1451
  });
1080
- this.maybeCompleteRun(current.runId);
1081
- const affectedOwners = this.rememberTerminalWorker(current.id);
1082
- affectedOwners.add(current.ownerSessionId);
1083
- this.emitStateForOwners(affectedOwners);
1452
+ rememberTerminalWorker(draft, worker.record.id);
1453
+ return actions;
1084
1454
  }
1085
1455
 
1086
- private closeReadyInteractiveWorkersForShutdown(): void {
1087
- const readyInteractiveWorkers = [...this.workers.values()].filter(
1088
- (worker) => worker.lifecycle === "interactive" && worker.status === "ready",
1089
- );
1090
- for (const worker of readyInteractiveWorkers) {
1091
- this.closeReadyInteractiveWorker(worker);
1456
+ private releaseWorkerResources(worker: RuntimeWorker): PostCommitAction[] {
1457
+ const actions: PostCommitAction[] = [];
1458
+ if (worker.observationRelease) {
1459
+ actions.push(runAction(() => safelyCall(worker.observationRelease)));
1092
1460
  }
1461
+ const session = worker.session;
1462
+ if (session) {
1463
+ actions.push(runAction(() => {
1464
+ this.launchCleanupOrJoinAfterClosure(
1465
+ Effect.suspend(() => session.dispose()).pipe(
1466
+ Effect.catchCause(() => Effect.void),
1467
+ Effect.uninterruptible,
1468
+ ),
1469
+ );
1470
+ }));
1471
+ }
1472
+ return actions;
1093
1473
  }
1094
1474
 
1095
- private subscribeEntryObservability(
1096
- workerId: WorkerId,
1097
- entry: RuntimeEntry,
1098
- session: WorkerSessionHandle,
1099
- generation: number,
1100
- ): void {
1101
- this.unsubscribeEntryObservability(entry);
1102
- entry.unsubscribeUsage = session.subscribeUsage((usage) => {
1103
- const latest = this.workers.get(workerId);
1104
- if (!latest || entry.session !== session || entry.generation !== generation) return;
1105
- if (latest.status !== "starting" && latest.status !== "running") return;
1106
- this.workers.set(workerId, { ...latest, usage: copyUsage(usage) });
1107
- this.emitState(latest.ownerSessionId);
1108
- });
1109
- entry.unsubscribeActivity = session.subscribeActivity((activity) => {
1110
- const latest = this.workers.get(workerId);
1111
- if (!latest || entry.session !== session || entry.generation !== generation) return;
1112
- if (latest.status !== "starting" && latest.status !== "running") return;
1113
- if (latest.activity === activity) return;
1114
- this.workers.set(workerId, { ...latest, activity });
1115
- this.emitState(latest.ownerSessionId);
1116
- });
1117
- entry.unsubscribeMessageDirection = session.subscribeMessageDirection((messageDirection) => {
1118
- const latest = this.workers.get(workerId);
1119
- if (!latest || entry.session !== session || entry.generation !== generation) return;
1120
- if (latest.status !== "starting" && latest.status !== "running") return;
1121
- if (latest.messageDirection === messageDirection) return;
1122
- this.workers.set(workerId, { ...latest, messageDirection });
1123
- this.emitState(latest.ownerSessionId);
1124
- });
1125
- }
1126
-
1127
- private emitState(ownerSessionId: string): void {
1128
- for (const listener of [...this.stateListeners]) {
1129
- try {
1130
- listener(ownerSessionId);
1131
- } catch {
1132
- // One subscriber cannot prevent runtime mutations or other notifications.
1133
- }
1475
+ private launchCleanupOrJoinAfterClosure(cleanup: Effect.Effect<void>): void {
1476
+ if (this.cleanups.state._tag === "Closed") {
1477
+ Effect.runFork(cleanup);
1478
+ return;
1479
+ }
1480
+ const fiber = this.runCleanup(cleanup);
1481
+ const exit = fiber.pollUnsafe();
1482
+ if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
1483
+ // FiberSet.runtime returns an already-interrupted sentinel when closure
1484
+ // wins admission. If admission won and closure interrupted the real fiber,
1485
+ // repeating the cached uninterruptible disposal only joins that cleanup.
1486
+ Effect.runFork(cleanup);
1134
1487
  }
1135
1488
  }
1136
1489
 
1137
- private emitStateForOwners(ownerSessionIds: ReadonlySet<string>): void {
1138
- for (const ownerSessionId of ownerSessionIds) this.emitState(ownerSessionId);
1490
+ private preflightOpen(
1491
+ operation: OrchestrationOperation,
1492
+ ): Decision<void> {
1493
+ return this.transact((draft) => ({
1494
+ value: openDecision(draft, operation),
1495
+ }));
1139
1496
  }
1140
1497
 
1141
- private ownedWorker(ownerSessionId: string, workerId: WorkerId): WorkerRecord {
1142
- const worker = this.workers.get(workerId);
1143
- if (!worker || worker.ownerSessionId !== ownerSessionId) {
1144
- throw new Error("Worker is not owned by this session");
1145
- }
1146
- return worker;
1498
+ private preflightReadyInteractive(
1499
+ ownerSessionId: string,
1500
+ workerId: WorkerId,
1501
+ ): Decision<void> {
1502
+ return this.transact((draft) => {
1503
+ const open = openDecision(draft, "sendInteractive");
1504
+ if (open._tag === "rejected") return { value: open };
1505
+ const ownership = ownedWorkerDecision(
1506
+ draft,
1507
+ "sendInteractive",
1508
+ ownerSessionId,
1509
+ workerId,
1510
+ );
1511
+ if (ownership._tag === "rejected") return { value: ownership };
1512
+ const worker = ownership.value;
1513
+ if (
1514
+ worker.record.lifecycle !== "interactive" ||
1515
+ worker.record.status !== "ready" ||
1516
+ !worker.session
1517
+ ) {
1518
+ return {
1519
+ value: rejected(
1520
+ "sendInteractive",
1521
+ "worker-state",
1522
+ "interactive_send requires an owned ready interactive worker",
1523
+ ),
1524
+ };
1525
+ }
1526
+ return { value: accepted(undefined) };
1527
+ });
1147
1528
  }
1148
1529
 
1149
- private unsubscribeEntryObservability(entry: RuntimeEntry): void {
1150
- safelyCall(entry.unsubscribeUsage);
1151
- entry.unsubscribeUsage = undefined;
1152
- safelyCall(entry.unsubscribeActivity);
1153
- entry.unsubscribeActivity = undefined;
1154
- safelyCall(entry.unsubscribeMessageDirection);
1155
- entry.unsubscribeMessageDirection = undefined;
1530
+ private current(): RuntimeState {
1531
+ return this.state;
1156
1532
  }
1157
1533
 
1158
- private disposeEntrySession(entry: RuntimeEntry): void {
1159
- this.unsubscribeEntryObservability(entry);
1160
- if (entry.session) this.disposeSession(entry.session);
1161
- entry.session = undefined;
1534
+ private transact<A>(
1535
+ reducer: (draft: RuntimeState) => TransactionMutation<A>,
1536
+ ): A {
1537
+ const draft = makeDraft(this.state);
1538
+ const mutation = reducer(draft);
1539
+ this.state = Object.freeze(draft);
1540
+ this.enqueueActions((mutation.actions ?? []).map((action) => action(
1541
+ this.state,
1542
+ this.settlementListeners,
1543
+ this.stateListeners,
1544
+ )));
1545
+ return mutation.value;
1162
1546
  }
1163
1547
 
1164
- private disposeSession(session: WorkerSessionHandle): void {
1165
- if (this.disposedSessions.has(session)) return;
1166
- this.disposedSessions.add(session);
1548
+ private enqueueActions(actions: readonly CommittedAction[]): void {
1549
+ this.actionQueue.push(...actions);
1550
+ if (this.drainingActions) return;
1167
1551
 
1552
+ this.drainingActions = true;
1553
+ let hasFailure = false;
1554
+ let firstFailure: unknown;
1168
1555
  try {
1169
- this.trackCleanup(session.dispose());
1170
- } catch {
1171
- // Cleanup is best-effort and cannot leave lifecycle state unsettled.
1556
+ while (this.actionQueue.length > 0) {
1557
+ const action = this.actionQueue.shift();
1558
+ if (!action) continue;
1559
+ try {
1560
+ action();
1561
+ } catch (error) {
1562
+ if (!hasFailure) firstFailure = error;
1563
+ hasFailure = true;
1564
+ }
1565
+ }
1566
+ } finally {
1567
+ this.drainingActions = false;
1172
1568
  }
1569
+ if (hasFailure) throw firstFailure;
1173
1570
  }
1571
+ }
1572
+
1573
+ export function orchestrationLayer(
1574
+ options: OrchestrationLayerOptions = {},
1575
+ ): Layer.Layer<Orchestration, never, ChildSessions> {
1576
+ return Layer.effect(
1577
+ Orchestration,
1578
+ Effect.gen(function* () {
1579
+ const childSessions = yield* ChildSessions;
1580
+ // Scope finalizers run in reverse acquisition order. Keep cleanup open while
1581
+ // generation and cancellation interruption settle workers and enqueue disposal.
1582
+ const cleanups = yield* FiberSet.make<void, never>();
1583
+ const runCleanup = yield* FiberSet.runtime(cleanups)<never>();
1584
+ const cancellations = yield* FiberSet.make<void, never>();
1585
+ const runCancellation = yield* FiberSet.runtime(cancellations)<never>();
1586
+ const generations = yield* FiberMap.make<WorkerId, void, never>();
1587
+ const runGeneration = yield* FiberMap.runtime(generations)<never>();
1588
+ const clock = yield* Clock.Clock;
1589
+ const shutdownCompletion = yield* Deferred.make<void>();
1590
+ return Orchestration.of(new StatefulOrchestration(
1591
+ childSessions,
1592
+ generations,
1593
+ runGeneration,
1594
+ cancellations,
1595
+ runCancellation,
1596
+ cleanups,
1597
+ runCleanup,
1598
+ clock,
1599
+ options.idFactories ?? createRandomIdFactories(),
1600
+ shutdownCompletion,
1601
+ ));
1602
+ }),
1603
+ );
1604
+ }
1605
+
1606
+ function initialState(): RuntimeState {
1607
+ return Object.freeze({
1608
+ workers: new Map<WorkerId, RuntimeWorker>(),
1609
+ runs: new Map<RunId, RuntimeRun>(),
1610
+ terminalWorkerOrder: [],
1611
+ completedRunOrder: [],
1612
+ settlementSequence: 0,
1613
+ lifecycle: "open",
1614
+ });
1615
+ }
1616
+
1617
+ function makeDraft(state: RuntimeState): RuntimeState {
1618
+ return {
1619
+ ...state,
1620
+ workers: new Map(state.workers),
1621
+ runs: new Map(state.runs),
1622
+ terminalWorkerOrder: [...state.terminalWorkerOrder],
1623
+ completedRunOrder: [...state.completedRunOrder],
1624
+ };
1625
+ }
1174
1626
 
1175
- private trackCleanup(operation: Promise<void>): Promise<void> {
1176
- this.cleanupOperations.add(operation);
1177
- const forget = () => this.cleanupOperations.delete(operation);
1178
- void operation.then(forget, forget);
1179
- return operation;
1627
+ function completeRun(
1628
+ state: RuntimeState,
1629
+ workerId: WorkerId,
1630
+ ): PostCommitAction[] {
1631
+ const worker = state.workers.get(workerId);
1632
+ if (!worker) return [];
1633
+ const run = state.runs.get(worker.record.runId);
1634
+ if (!run || !isRunningRuntimeRun(run) || !isCompletedRunWorker(worker.record)) {
1635
+ return [];
1180
1636
  }
1181
1637
 
1182
- private async awaitTrackedCleanupBestEffort(): Promise<void> {
1183
- const operations = [...this.cleanupOperations];
1184
- if (operations.length === 0) return;
1185
- await this.waitBestEffort(
1186
- Promise.allSettled(operations).then(() => undefined),
1187
- SHUTDOWN_CLEANUP_GRACE_MS,
1188
- );
1638
+ const completed = freezeCompletedRun(run.record, worker.record);
1639
+ state.runs.set(run.record.id, { ...run.record, state: "complete" });
1640
+ state.completedRunOrder.push(run.record.id);
1641
+ return [runAction(() => {
1642
+ Deferred.doneUnsafe(run.completion, Effect.succeed(completed));
1643
+ })];
1644
+ }
1645
+
1646
+ function makeSettlement(
1647
+ draft: RuntimeState,
1648
+ worker: RuntimeWorker,
1649
+ settledAt: number,
1650
+ failureStage: SettlementFailureStage | undefined,
1651
+ ): PostCommitAction | undefined {
1652
+ const run = draft.runs.get(worker.record.runId);
1653
+ if (!run || !isCompletedRunWorker(worker.record)) {
1654
+ return undefined;
1189
1655
  }
1656
+ const runRecord = runtimeRunRecord(run);
1657
+ const sequence = draft.settlementSequence + 1;
1658
+ draft.settlementSequence = sequence;
1659
+ const settlement: WorkerSettlement = Object.freeze({
1660
+ eventId: `${sequence}:${runRecord.id}:${worker.record.id}:${worker.generation}`,
1661
+ sequence,
1662
+ ownerSessionId: worker.record.ownerSessionId,
1663
+ runId: runRecord.id,
1664
+ workerId: worker.record.id,
1665
+ generation: worker.generation,
1666
+ mode: runRecord.mode,
1667
+ worker: worker.record.worker,
1668
+ title: worker.record.title,
1669
+ lifecycle: worker.record.lifecycle,
1670
+ status: worker.record.status,
1671
+ outcome: Object.freeze(copyOutcome(worker.record.outcome)),
1672
+ ...(failureStage ? { failureStage } : {}),
1673
+ usage: Object.freeze(copyUsage(worker.record.usage)),
1674
+ startedAt: worker.record.startedAt,
1675
+ settledAt,
1676
+ ...(runRecord.synthesisGroupId && runRecord.synthesisGroupSize
1677
+ ? {
1678
+ synthesisGroupId: runRecord.synthesisGroupId,
1679
+ synthesisGroupSize: runRecord.synthesisGroupSize,
1680
+ }
1681
+ : {}),
1682
+ ...(worker.record.sessionFile !== undefined
1683
+ ? { sessionFile: worker.record.sessionFile }
1684
+ : {}),
1685
+ });
1686
+ return publishSettlement(
1687
+ settlement,
1688
+ isRunningRuntimeRun(run) ? run.settlementListener : undefined,
1689
+ );
1690
+ }
1190
1691
 
1191
- private async waitBestEffort(promise: Promise<unknown>, timeoutMs: number): Promise<void> {
1192
- try {
1193
- await this.bestEffortDeadline.wait(promise, timeoutMs);
1194
- } catch {
1195
- // A deadline implementation cannot prevent lifecycle settlement.
1196
- }
1692
+ function stateActionsAfterPrune(
1693
+ draft: RuntimeState,
1694
+ ownerSessionId: string,
1695
+ ): PostCommitAction[] {
1696
+ const owners = pruneHistory(draft);
1697
+ owners.add(ownerSessionId);
1698
+ return publishOwners(owners);
1699
+ }
1700
+
1701
+ function pruneHistory(draft: RuntimeState): Set<string> {
1702
+ const owners = new Set<string>();
1703
+ while (draft.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
1704
+ const runId = draft.completedRunOrder.shift();
1705
+ if (!runId) break;
1706
+ const run = draft.runs.get(runId);
1707
+ if (run) owners.add(runtimeRunRecord(run).ownerSessionId);
1708
+ draft.runs.delete(runId);
1709
+ }
1710
+ while (draft.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
1711
+ const index = draft.terminalWorkerOrder.findIndex((workerId) => {
1712
+ const worker = draft.workers.get(workerId);
1713
+ return !worker || isTerminalWorkerStatus(worker.record.status);
1714
+ });
1715
+ if (index < 0) break;
1716
+ const removed = draft.terminalWorkerOrder.splice(index, 1)[0];
1717
+ if (!removed) break;
1718
+ const worker = draft.workers.get(removed);
1719
+ if (worker) owners.add(worker.record.ownerSessionId);
1720
+ draft.workers.delete(removed);
1197
1721
  }
1722
+ return owners;
1723
+ }
1198
1724
 
1199
- private assertOpen(): void {
1200
- if (this.shuttingDown) throw new Error("Orchestrator runtime is shutting down");
1725
+ function rememberTerminalWorker(
1726
+ draft: RuntimeState,
1727
+ workerId: WorkerId,
1728
+ ): void {
1729
+ if (!draft.terminalWorkerOrder.includes(workerId)) {
1730
+ draft.terminalWorkerOrder.push(workerId);
1201
1731
  }
1202
1732
  }
1203
1733
 
1204
- export function createOrchestratorRuntime(
1205
- options: OrchestratorRuntimeOptions,
1206
- ): OrchestratorRuntime {
1207
- return new DefaultOrchestratorRuntime(options);
1734
+ function snapshotFor(
1735
+ state: RuntimeState,
1736
+ ownerSessionId: string,
1737
+ ): RuntimeSnapshot {
1738
+ return Object.freeze({
1739
+ runs: Object.freeze(
1740
+ [...state.runs.values()]
1741
+ .map(runtimeRunRecord)
1742
+ .filter((run) => run.ownerSessionId === ownerSessionId)
1743
+ .map(copyRunRecord),
1744
+ ),
1745
+ workers: Object.freeze(
1746
+ [...state.workers.values()]
1747
+ .map((worker) => worker.record)
1748
+ .filter((worker) => worker.ownerSessionId === ownerSessionId)
1749
+ .map(copyWorkerRecord),
1750
+ ),
1751
+ });
1752
+ }
1753
+
1754
+ function runtimeRunRecord(run: RuntimeRun): RunRecord {
1755
+ return isRunningRuntimeRun(run) ? run.record : run;
1756
+ }
1757
+
1758
+ function isRunningRuntimeRun(run: RuntimeRun): run is RunningRuntimeRun {
1759
+ return "_tag" in run;
1760
+ }
1761
+
1762
+ function makeRunRecord(
1763
+ runId: RunId,
1764
+ workerId: WorkerId,
1765
+ context: OrchestrationContext,
1766
+ mode: RunMode,
1767
+ createdAt: number,
1768
+ ): RunRecord {
1769
+ return {
1770
+ id: runId,
1771
+ ownerSessionId: context.ownerSessionId,
1772
+ workerId,
1773
+ mode,
1774
+ state: "running",
1775
+ createdAt,
1776
+ ...(context.synthesisGroup
1777
+ ? {
1778
+ synthesisGroupId: context.synthesisGroup.id,
1779
+ synthesisGroupSize: context.synthesisGroup.size,
1780
+ }
1781
+ : {}),
1782
+ };
1783
+ }
1784
+
1785
+ function makeWorkerRecord(
1786
+ workerId: WorkerId,
1787
+ runId: RunId,
1788
+ ownerSessionId: string,
1789
+ definition: WorkerDefinition,
1790
+ task: OrchestrateTaskInput,
1791
+ startedAt: number,
1792
+ ): WorkerRecord {
1793
+ return {
1794
+ id: workerId,
1795
+ worker: definition.name,
1796
+ ownerSessionId,
1797
+ runId,
1798
+ title: task.title,
1799
+ instructions: task.instructions,
1800
+ lifecycle: definition.lifecycle,
1801
+ status: "starting",
1802
+ usage: copyUsage(EMPTY_WORKER_USAGE),
1803
+ messageDirection: "to-model",
1804
+ startedAt,
1805
+ };
1806
+ }
1807
+
1808
+ function publishState(ownerSessionId: string): PostCommitAction {
1809
+ return (state, _settlementListeners, stateListeners) => {
1810
+ const snapshot = snapshotFor(state, ownerSessionId);
1811
+ const listeners = [...(stateListeners.get(ownerSessionId) ?? [])];
1812
+ return () => {
1813
+ for (const listener of listeners) safelyNotify(() => listener(snapshot));
1814
+ };
1815
+ };
1816
+ }
1817
+
1818
+ function publishStateTo(
1819
+ ownerSessionId: string,
1820
+ listener: StateListener,
1821
+ ): PostCommitAction {
1822
+ return (state) => {
1823
+ const snapshot = snapshotFor(state, ownerSessionId);
1824
+ return () => safelyNotify(() => listener(snapshot));
1825
+ };
1826
+ }
1827
+
1828
+ function publishSettlement(
1829
+ settlement: WorkerSettlement,
1830
+ localListener: SettlementListener | undefined,
1831
+ ): PostCommitAction {
1832
+ return (_state, settlementListeners) => {
1833
+ const listeners = [...settlementListeners];
1834
+ return () => {
1835
+ if (localListener) safelyNotify(() => localListener(settlement));
1836
+ for (const listener of listeners) safelyNotify(() => listener(settlement));
1837
+ };
1838
+ };
1839
+ }
1840
+
1841
+ function publishOwners(owners: ReadonlySet<string>): PostCommitAction[] {
1842
+ return [...owners].map(publishState);
1843
+ }
1844
+
1845
+ function runAction(run: () => void): PostCommitAction {
1846
+ return () => run;
1847
+ }
1848
+
1849
+ function openDecision(
1850
+ draft: RuntimeState,
1851
+ operation: OrchestrationOperation,
1852
+ ): Decision<void> {
1853
+ return draft.lifecycle === "open"
1854
+ ? accepted(undefined)
1855
+ : rejected(
1856
+ operation,
1857
+ "shutdown",
1858
+ "Orchestrator runtime is shutting down",
1859
+ );
1860
+ }
1861
+
1862
+ function ownedWorkerDecision(
1863
+ draft: RuntimeState,
1864
+ operation: OrchestrationOperation,
1865
+ ownerSessionId: string,
1866
+ workerId: WorkerId,
1867
+ ): Decision<RuntimeWorker> {
1868
+ const worker = draft.workers.get(workerId);
1869
+ return !worker || worker.record.ownerSessionId !== ownerSessionId
1870
+ ? rejected(
1871
+ operation,
1872
+ "ownership",
1873
+ "Worker is not owned by this session",
1874
+ )
1875
+ : accepted(worker);
1876
+ }
1877
+
1878
+ function accepted<A>(value: A): Decision<A> {
1879
+ return { _tag: "accepted", value };
1208
1880
  }
1209
1881
 
1210
- function validateContextOwner(ownerSessionId: string): void {
1211
- if (typeof ownerSessionId !== "string" || ownerSessionId.trim() === "") {
1212
- throw new Error("ownerSessionId must not be blank");
1882
+ function rejected(
1883
+ operation: OrchestrationOperation,
1884
+ reason: OrchestrationRejectionReason,
1885
+ message: string,
1886
+ ): Decision<never> {
1887
+ return {
1888
+ _tag: "rejected",
1889
+ error: actionRejection(operation, reason, message),
1890
+ };
1891
+ }
1892
+
1893
+ function actionRejection(
1894
+ operation: OrchestrationOperation,
1895
+ reason: OrchestrationRejectionReason,
1896
+ message: string,
1897
+ ): OrchestrationActionRejected {
1898
+ return new OrchestrationActionRejected({ operation, reason, message });
1899
+ }
1900
+
1901
+ function rejectAction(
1902
+ operation: OrchestrationOperation,
1903
+ reason: OrchestrationRejectionReason,
1904
+ message: string,
1905
+ ): Effect.Effect<never, OrchestrationActionRejected> {
1906
+ return Effect.fail(actionRejection(operation, reason, message));
1907
+ }
1908
+
1909
+ function validateContextOwner(
1910
+ operation: OrchestrationOperation,
1911
+ ownerSessionId: string,
1912
+ ): Effect.Effect<void, OrchestrationActionRejected> {
1913
+ return typeof ownerSessionId !== "string" || ownerSessionId.trim() === ""
1914
+ ? rejectAction(
1915
+ operation,
1916
+ "validation",
1917
+ "ownerSessionId must not be blank",
1918
+ )
1919
+ : Effect.void;
1920
+ }
1921
+
1922
+ function validateWorkerId(
1923
+ operation: OrchestrationOperation,
1924
+ workerId: string,
1925
+ ): Effect.Effect<WorkerId, OrchestrationActionRejected> {
1926
+ if (typeof workerId !== "string" || workerId.trim() === "") {
1927
+ return rejectAction(
1928
+ operation,
1929
+ "validation",
1930
+ "worker_id must not be blank",
1931
+ );
1213
1932
  }
1933
+ return Schema.decodeUnknownEffect(WorkerId)(workerId).pipe(
1934
+ Effect.mapError(() => actionRejection(
1935
+ operation,
1936
+ "validation",
1937
+ "worker_id must use the canonical worker- prefix",
1938
+ )),
1939
+ );
1214
1940
  }
1215
1941
 
1216
- function validateMode(mode: RunMode): void {
1217
- if (mode !== "async" && mode !== "inline") throw new Error("Invalid orchestration mode");
1942
+ function validateMode(
1943
+ operation: OrchestrationOperation,
1944
+ mode: RunMode,
1945
+ ): Effect.Effect<void, OrchestrationActionRejected> {
1946
+ return mode !== "async" && mode !== "inline"
1947
+ ? rejectAction(operation, "validation", "Invalid orchestration mode")
1948
+ : Effect.void;
1218
1949
  }
1219
1950
 
1220
- function validateText(name: string, value: string, maximumLength: number): void {
1951
+ function validateText(
1952
+ operation: OrchestrationOperation,
1953
+ name: string,
1954
+ value: string,
1955
+ maximumLength: number,
1956
+ ): Effect.Effect<void, OrchestrationActionRejected> {
1221
1957
  if (typeof value !== "string" || value.trim() === "") {
1222
- throw new Error(`${name} must not be blank`);
1223
- }
1224
- if (value.length > maximumLength) {
1225
- throw new Error(`${name} must be at most ${maximumLength} characters`);
1958
+ return rejectAction(operation, "validation", `${name} must not be blank`);
1226
1959
  }
1960
+ return value.length > maximumLength
1961
+ ? rejectAction(
1962
+ operation,
1963
+ "validation",
1964
+ `${name} must be at most ${maximumLength} characters`,
1965
+ )
1966
+ : Effect.void;
1227
1967
  }
1228
1968
 
1229
- function describeError(error: unknown, fallback: string): string {
1230
- if (error instanceof Error && error.message !== "") return error.message;
1231
- if (typeof error === "string" && error !== "") return error;
1232
- return fallback;
1969
+ type ValidatedAbortTarget =
1970
+ | {
1971
+ readonly _tag: "ids";
1972
+ readonly workerIds: readonly WorkerId[];
1973
+ }
1974
+ | {
1975
+ readonly _tag: "all";
1976
+ };
1977
+
1978
+ function validateAbortTarget(
1979
+ target: AbortTarget,
1980
+ ): Effect.Effect<ValidatedAbortTarget, OrchestrationActionRejected> {
1981
+ return Effect.gen(function* () {
1982
+ if (!target || typeof target !== "object") {
1983
+ return yield* rejectAction("abort", "target", "Invalid abort target");
1984
+ }
1985
+ const selected = [target.workerIds !== undefined, target.all !== undefined]
1986
+ .filter(Boolean).length;
1987
+ if (selected !== 1 || (target.all !== undefined && target.all !== true)) {
1988
+ return yield* rejectAction(
1989
+ "abort",
1990
+ "target",
1991
+ "Abort target must specify exactly one of workerIds or all: true",
1992
+ );
1993
+ }
1994
+ if (target.workerIds === undefined) return { _tag: "all" };
1995
+ if (!Array.isArray(target.workerIds) || target.workerIds.length === 0) {
1996
+ return yield* rejectAction(
1997
+ "abort",
1998
+ "target",
1999
+ "workerIds must contain at least one worker ID",
2000
+ );
2001
+ }
2002
+ const workerIds = yield* Effect.all(
2003
+ [...new Set(target.workerIds)].map((id) => validateWorkerId("abort", id)),
2004
+ );
2005
+ return { _tag: "ids", workerIds };
2006
+ });
1233
2007
  }
1234
2008
 
1235
- function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
1236
- return status === "starting" || status === "running" || status === "stopping";
2009
+ function makeCancellationCandidates(
2010
+ workerIds: readonly WorkerId[],
2011
+ ): Effect.Effect<ReadonlyMap<WorkerId, Deferred.Deferred<void>>> {
2012
+ return Effect.forEach(workerIds, (workerId) => Deferred.make<void>().pipe(
2013
+ Effect.map((completion) => [workerId, completion] as const),
2014
+ )).pipe(Effect.map((entries) => new Map(entries)));
1237
2015
  }
1238
2016
 
1239
- function isSettledWorkerStatus(
1240
- status: WorkerRecord["status"],
1241
- ): status is RunResult["status"] {
1242
- return status === "completed" || status === "ready" || status === "failed" || status === "aborted";
2017
+ function activeWorkerIds(
2018
+ state: RuntimeState,
2019
+ ownerSessionId?: string,
2020
+ ): WorkerId[] {
2021
+ return [...state.workers.values()]
2022
+ .filter((worker) => (
2023
+ (ownerSessionId === undefined ||
2024
+ worker.record.ownerSessionId === ownerSessionId) &&
2025
+ isActiveWorkerStatus(worker.record.status)
2026
+ ))
2027
+ .map((worker) => worker.record.id);
1243
2028
  }
1244
2029
 
1245
- function safelyCall(callback: (() => void) | undefined): void {
1246
- if (!callback) return;
1247
- try {
1248
- callback();
1249
- } catch {
1250
- // Session cleanup is idempotent best-effort and must not strand lifecycle state.
1251
- }
2030
+ function awaitAll(
2031
+ deferreds: readonly Deferred.Deferred<void>[],
2032
+ ): Effect.Effect<void> {
2033
+ return Effect.forEach(
2034
+ deferreds,
2035
+ (deferred) => Deferred.await(deferred),
2036
+ { concurrency: "unbounded", discard: true },
2037
+ );
1252
2038
  }
1253
2039
 
1254
- function throwIfAborted(signal: AbortSignal | undefined): void {
1255
- if (signal?.aborted) throw abortSignalReason(signal);
2040
+ function isRuntimeWorker(
2041
+ worker: RuntimeWorker | undefined,
2042
+ ): worker is RuntimeWorker {
2043
+ return worker !== undefined;
1256
2044
  }
1257
2045
 
1258
- function abortSignalReason(signal: AbortSignal): unknown {
1259
- if ("reason" in signal) return signal.reason;
1260
- return new DOMException("This operation was aborted", "AbortError");
2046
+ function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
2047
+ return status === "starting" || status === "running" || status === "stopping";
1261
2048
  }
1262
2049
 
1263
- function addAll(target: Set<string>, source: ReadonlySet<string>): void {
1264
- for (const value of source) target.add(value);
2050
+ function isCompletedRunWorker(
2051
+ record: WorkerRecord,
2052
+ ): record is WorkerRecord & {
2053
+ readonly status: RunResult["status"];
2054
+ readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
2055
+ readonly settledAt: number;
2056
+ } {
2057
+ return (
2058
+ (record.status === "completed" ||
2059
+ record.status === "ready" ||
2060
+ record.status === "failed" ||
2061
+ record.status === "aborted") &&
2062
+ record.outcome !== undefined &&
2063
+ record.outcome.status !== "closed" &&
2064
+ record.settledAt !== undefined
2065
+ );
1265
2066
  }
1266
2067
 
1267
- function notifySettlementListener(
1268
- listener: SettlementListener | undefined,
1269
- settlement: WorkerSettlement,
1270
- ): void {
1271
- if (!listener) return;
1272
- try {
1273
- listener(settlement);
1274
- } catch {
1275
- // One observer cannot prevent settlement or other observers from being notified.
1276
- }
2068
+ function observationsEqual(
2069
+ previous: WorkerRecord,
2070
+ next: WorkerRecord,
2071
+ ): boolean {
2072
+ return (
2073
+ previous.activity === next.activity &&
2074
+ previous.messageDirection === next.messageDirection &&
2075
+ usageEquals(previous.usage, next.usage)
2076
+ );
1277
2077
  }
1278
2078
 
1279
- function makeRunWaiter(): RunWaiter {
1280
- let complete!: (run: CompletedRun) => void;
1281
- const waiter: RunWaiter = {
1282
- promise: new Promise<CompletedRun>((resolve) => {
1283
- complete = resolve;
1284
- }),
1285
- settled: false,
1286
- resolve(run) {
1287
- if (waiter.settled) return;
1288
- waiter.settled = true;
1289
- waiter.onSettled?.();
1290
- complete(run);
1291
- },
1292
- };
1293
- return waiter;
2079
+ function usageEquals(left: WorkerUsage, right: WorkerUsage): boolean {
2080
+ return (
2081
+ left.input === right.input &&
2082
+ left.output === right.output &&
2083
+ left.cacheRead === right.cacheRead &&
2084
+ left.cacheWrite === right.cacheWrite &&
2085
+ left.cost === right.cost &&
2086
+ left.contextTokens === right.contextTokens &&
2087
+ left.turns === right.turns
2088
+ );
1294
2089
  }
1295
2090
 
1296
2091
  function copyUsage(usage: WorkerUsage): WorkerUsage {
@@ -1317,7 +2112,9 @@ function copyWorkerRecord(worker: WorkerRecord): WorkerRecord {
1317
2112
  return Object.freeze({
1318
2113
  ...worker,
1319
2114
  usage: Object.freeze(copyUsage(worker.usage)),
1320
- ...(worker.outcome ? { outcome: Object.freeze(copyOutcome(worker.outcome)) } : {}),
2115
+ ...(worker.outcome
2116
+ ? { outcome: Object.freeze(copyOutcome(worker.outcome)) }
2117
+ : {}),
1321
2118
  });
1322
2119
  }
1323
2120
 
@@ -1327,21 +2124,18 @@ function freezeAcceptedRun(id: RunId, workerId: WorkerId): AcceptedRun {
1327
2124
 
1328
2125
  function freezeCompletedRun(
1329
2126
  run: RunRecord,
1330
- record: WorkerRecord,
2127
+ record: WorkerRecord & {
2128
+ readonly status: RunResult["status"];
2129
+ readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
2130
+ readonly settledAt: number;
2131
+ },
1331
2132
  ): CompletedRun {
1332
- const outcome = record.outcome;
1333
- if (!outcome || outcome.status === "closed") {
1334
- throw new Error("Completed run requires a worker response outcome");
1335
- }
1336
- if (record.settledAt === undefined) {
1337
- throw new Error("Completed run requires a settlement timestamp");
1338
- }
1339
2133
  const result: RunResult = Object.freeze({
1340
2134
  workerId: record.id,
1341
2135
  worker: record.worker,
1342
2136
  title: record.title,
1343
- status: record.status as RunResult["status"],
1344
- outcome: Object.freeze(copyOutcome(outcome)),
2137
+ status: record.status,
2138
+ outcome: Object.freeze(copyOutcome(record.outcome)),
1345
2139
  usage: Object.freeze(copyUsage(record.usage)),
1346
2140
  startedAt: record.startedAt,
1347
2141
  settledAt: record.settledAt,
@@ -1354,3 +2148,27 @@ function freezeCompletedRun(
1354
2148
  result,
1355
2149
  });
1356
2150
  }
2151
+
2152
+ function describeError(error: unknown, fallback: string): string {
2153
+ if (error instanceof Error && error.message !== "") return error.message;
2154
+ if (typeof error === "string" && error !== "") return error;
2155
+ return fallback;
2156
+ }
2157
+
2158
+ function safelyCall(callback: (() => void) | undefined): void {
2159
+ if (callback) safelyNotify(callback);
2160
+ }
2161
+
2162
+ function safelyNotify(callback: () => void): void {
2163
+ try {
2164
+ callback();
2165
+ } catch {
2166
+ // Observers and best-effort cleanup cannot block committed state transitions.
2167
+ }
2168
+ }
2169
+
2170
+ function addAll(target: Set<string>, source: ReadonlySet<string>): void {
2171
+ for (const value of source) target.add(value);
2172
+ }
2173
+
2174
+ function noOp(): void {}