@zachwill/pi-orchestrate 0.8.0 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,49 +1,53 @@
1
1
  import type { Api, Model } from "@earendil-works/pi-ai";
2
2
  import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
3
- import { Effect, Option } from "effect";
3
+ import {
4
+ Cause,
5
+ Clock,
6
+ Context,
7
+ Deferred,
8
+ Effect,
9
+ Exit,
10
+ Fiber,
11
+ FiberMap,
12
+ FiberSet,
13
+ Layer,
14
+ Schema,
15
+ } from "effect";
4
16
  import {
5
17
  CANCELLATION_GRACE_MS,
6
18
  EMPTY_WORKER_USAGE,
7
19
  MAX_WORKER_INSTRUCTIONS_LENGTH,
8
20
  MAX_WORKER_TITLE_LENGTH,
21
+ OrchestrateTaskInput,
22
+ RunId,
23
+ WorkerId,
9
24
  createRandomIdFactories,
10
25
  findWorkerByName,
11
26
  isTerminalWorkerStatus,
12
27
  transitionWorkerStatus,
13
28
  type OrchestrateIdFactories,
14
- type OrchestrateTaskInput,
15
- type RunId,
16
29
  type RunMode,
17
30
  type RunRecord,
18
31
  type WorkerCatalog,
19
32
  type WorkerDefinition,
20
- type WorkerId,
21
33
  type WorkerOutcome,
22
34
  type WorkerRecord,
23
35
  type WorkerUsage,
24
36
  } from "./domain.js";
25
37
  import {
26
- createWorkflowScheduler,
27
- type WorkflowScheduler,
28
- } from "./scheduler.js";
29
- import {
30
- resolveWorkerModel,
31
- type WorkerSessionFactory,
38
+ ChildSessions,
39
+ type ChildSessionsService,
40
+ type WorkerSessionAbortError,
32
41
  type WorkerSessionHandle,
42
+ type WorkerSessionObservation,
33
43
  } from "./worker-session.js";
34
44
  import type {
35
45
  SettlementFailureStage,
36
46
  WorkerSettlement,
37
47
  } from "./worker-settlement.js";
38
48
 
39
- export type {
40
- SettlementFailureStage,
41
- WorkerSettlement,
42
- } from "./worker-settlement.js";
43
-
44
49
  export const MAX_TERMINAL_WORKER_HISTORY = 100;
45
50
  export const MAX_COMPLETED_RUN_HISTORY = 100;
46
- /** Shutdown waits this long for interrupted bootstrap/prompt promises, then returns best-effort. */
47
51
  export const SHUTDOWN_CLEANUP_GRACE_MS = CANCELLATION_GRACE_MS;
48
52
 
49
53
  export interface OrchestrationContext {
@@ -92,433 +96,679 @@ export interface RuntimeSnapshot {
92
96
 
93
97
  export type SettlementListener = (settlement: WorkerSettlement) => void;
94
98
  export type UnsubscribeSettlement = () => void;
95
- export type StateListener = (ownerSessionId: string) => void;
96
-
97
- export type AbortTarget =
98
- | {
99
- readonly workerIds: readonly WorkerId[];
100
- readonly all?: never;
101
- }
102
- | {
103
- readonly all: true;
104
- readonly workerIds?: never;
105
- };
99
+ export type StateListener = (snapshot: RuntimeSnapshot) => void;
106
100
 
107
- export type DeadlineResult = "settled" | "timed-out";
108
-
109
- export interface BestEffortDeadline {
110
- wait(promise: Promise<unknown>, timeoutMs: number): Promise<DeadlineResult>;
101
+ export interface AbortTarget {
102
+ readonly workerIds?: readonly string[];
103
+ readonly all?: boolean;
111
104
  }
112
105
 
113
- export interface OrchestratorRuntimeOptions {
114
- readonly workerSessionFactory: WorkerSessionFactory;
106
+ export interface OrchestrationLayerOptions {
115
107
  readonly idFactories?: OrchestrateIdFactories;
116
- readonly clock?: () => number;
117
- readonly scheduler?: WorkflowScheduler<WorkerId>;
118
- readonly bestEffortDeadline?: BestEffortDeadline;
119
108
  }
120
109
 
121
- export interface OrchestratorRuntime {
110
+ const OrchestrationOperation = Schema.Literals([
111
+ "orchestrate",
112
+ "sendInteractive",
113
+ "abort",
114
+ "closeInteractive",
115
+ "snapshot",
116
+ ]);
117
+ type OrchestrationOperation = typeof OrchestrationOperation.Type;
118
+
119
+ const OrchestrationRejectionReason = Schema.Literals([
120
+ "shutdown",
121
+ "validation",
122
+ "ownership",
123
+ "target",
124
+ "worker-state",
125
+ "unknown-worker",
126
+ "model-unavailable",
127
+ ]);
128
+ type OrchestrationRejectionReason = typeof OrchestrationRejectionReason.Type;
129
+
130
+ export class OrchestrationActionRejected extends Schema.TaggedError<OrchestrationActionRejected>()(
131
+ "Orchestration.ActionRejected",
132
+ {
133
+ operation: OrchestrationOperation,
134
+ reason: OrchestrationRejectionReason,
135
+ message: Schema.String,
136
+ },
137
+ ) {}
138
+
139
+ export interface OrchestrationService {
122
140
  orchestrate(
123
141
  context: OrchestrationContext,
124
142
  task: OrchestrateTaskInput,
125
143
  mode: "async",
126
- signal?: AbortSignal,
127
144
  onSettlement?: SettlementListener,
128
- ): Promise<AcceptedRun>;
145
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
129
146
  orchestrate(
130
147
  context: OrchestrationContext,
131
148
  task: OrchestrateTaskInput,
132
149
  mode: "inline",
133
- signal?: AbortSignal,
134
150
  onSettlement?: SettlementListener,
135
- ): Promise<CompletedRun>;
151
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
136
152
  orchestrate(
137
153
  context: OrchestrationContext,
138
154
  task: OrchestrateTaskInput,
139
155
  mode: RunMode,
140
- signal?: AbortSignal,
141
156
  onSettlement?: SettlementListener,
142
- ): Promise<AcceptedRun | CompletedRun>;
157
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
143
158
  sendInteractive(
144
159
  context: OrchestrationContext,
145
- workerId: WorkerId,
160
+ workerId: string,
146
161
  instructions: string,
147
162
  mode: "async",
148
- signal?: AbortSignal,
149
163
  onSettlement?: SettlementListener,
150
- ): Promise<AcceptedRun>;
164
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
151
165
  sendInteractive(
152
166
  context: OrchestrationContext,
153
- workerId: WorkerId,
167
+ workerId: string,
154
168
  instructions: string,
155
169
  mode: "inline",
156
- signal?: AbortSignal,
157
170
  onSettlement?: SettlementListener,
158
- ): Promise<CompletedRun>;
171
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
159
172
  sendInteractive(
160
173
  context: OrchestrationContext,
161
- workerId: WorkerId,
174
+ workerId: string,
162
175
  instructions: string,
163
176
  mode: RunMode,
164
- signal?: AbortSignal,
165
177
  onSettlement?: SettlementListener,
166
- ): Promise<AcceptedRun | CompletedRun>;
167
- abort(ownerSessionId: string, target: AbortTarget): Promise<void>;
168
- closeInteractive(ownerSessionId: string, workerId: WorkerId): Promise<void>;
169
- snapshot(ownerSessionId: string): Promise<RuntimeSnapshot>;
170
- subscribeSettlement(listener: SettlementListener): UnsubscribeSettlement;
171
- subscribeState(listener: StateListener): () => void;
172
- shutdown(): Promise<void>;
178
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
179
+ readonly abort: (
180
+ ownerSessionId: string,
181
+ target: AbortTarget,
182
+ ) => Effect.Effect<void, OrchestrationActionRejected>;
183
+ readonly closeInteractive: (
184
+ ownerSessionId: string,
185
+ workerId: string,
186
+ ) => Effect.Effect<void, OrchestrationActionRejected>;
187
+ readonly snapshot: (
188
+ ownerSessionId: string,
189
+ ) => Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected>;
190
+ readonly subscribeSettlement: (
191
+ listener: SettlementListener,
192
+ ) => UnsubscribeSettlement;
193
+ readonly subscribeState: (
194
+ ownerSessionId: string,
195
+ listener: StateListener,
196
+ ) => () => void;
197
+ readonly shutdown: () => Effect.Effect<void>;
173
198
  }
174
199
 
175
- interface RuntimeEntry {
200
+ export class Orchestration extends Context.Service<Orchestration, OrchestrationService>()(
201
+ "@zachwill/pi-orchestrate/Orchestration",
202
+ ) {}
203
+
204
+ interface RuntimeWorker {
205
+ readonly record: WorkerRecord;
176
206
  readonly context: OrchestrationContext;
177
207
  readonly definition: WorkerDefinition;
178
- generation: number;
179
- session?: WorkerSessionHandle;
180
- unsubscribeUsage?: () => void;
181
- unsubscribeActivity?: () => void;
182
- unsubscribeMessageDirection?: () => void;
208
+ // Worker-local authority fence; stale asynchronous callbacks must revalidate it.
209
+ readonly generation: number;
210
+ readonly session?: WorkerSessionHandle;
211
+ readonly observationRelease?: () => void;
212
+ readonly cancellation?: Deferred.Deferred<void>;
183
213
  }
184
214
 
185
- interface RunWaiter {
186
- readonly promise: Promise<CompletedRun>;
187
- settled: boolean;
188
- onSettled?: () => void;
189
- resolve(run: CompletedRun): void;
215
+ interface RunningRuntimeRun {
216
+ readonly _tag: "running";
217
+ readonly record: RunRecord;
218
+ readonly completion: Deferred.Deferred<CompletedRun>;
219
+ readonly settlementListener?: SettlementListener;
190
220
  }
191
221
 
192
- const defaultBestEffortDeadline: BestEffortDeadline = {
193
- wait(promise, timeoutMs) {
194
- const settled = Effect.promise(() => promise.then(
195
- () => "settled" as const,
196
- () => "settled" as const,
197
- ));
198
- return Effect.runPromise(
199
- settled.pipe(
200
- Effect.timeoutOption(timeoutMs),
201
- Effect.map((result) => Option.getOrElse(result, () => "timed-out" as const)),
202
- ),
203
- );
204
- },
205
- };
206
-
207
- class DefaultOrchestratorRuntime implements OrchestratorRuntime {
208
- private readonly workerSessionFactory: WorkerSessionFactory;
209
- private readonly idFactories: OrchestrateIdFactories;
210
- private readonly clock: () => number;
211
- private readonly scheduler: WorkflowScheduler<WorkerId>;
212
- private readonly bestEffortDeadline: BestEffortDeadline;
213
- private readonly workers = new Map<WorkerId, WorkerRecord>();
214
- private readonly runs = new Map<RunId, RunRecord>();
215
- private readonly entries = new Map<WorkerId, RuntimeEntry>();
216
- private readonly runWaiters = new Map<RunId, RunWaiter>();
217
- private readonly completedRuns = new Map<RunId, CompletedRun>();
218
- private readonly cancellationPromises = new Map<WorkerId, Promise<void>>();
219
- private readonly cleanupOperations = new Set<Promise<void>>();
220
- private readonly terminalWorkerOrder: WorkerId[] = [];
221
- private readonly completedRunOrder: RunId[] = [];
222
+ type RuntimeRun = RunningRuntimeRun | RunRecord;
223
+ type RuntimeLifecycle = "open" | "shutting-down" | "shutdown";
224
+
225
+ interface RuntimeState {
226
+ workers: Map<WorkerId, RuntimeWorker>;
227
+ runs: Map<RunId, RuntimeRun>;
228
+ terminalWorkerOrder: WorkerId[];
229
+ completedRunOrder: RunId[];
230
+ settlementSequence: number;
231
+ lifecycle: RuntimeLifecycle;
232
+ }
233
+
234
+ type CommittedAction = () => void;
235
+ type PostCommitAction = (
236
+ state: RuntimeState,
237
+ settlementListeners: ReadonlySet<SettlementListener>,
238
+ stateListeners: ReadonlyMap<string, ReadonlySet<StateListener>>,
239
+ ) => CommittedAction;
240
+
241
+ interface TransactionMutation<A> {
242
+ readonly value: A;
243
+ readonly actions?: readonly PostCommitAction[];
244
+ }
245
+
246
+ type Decision<A> =
247
+ | {
248
+ readonly _tag: "accepted";
249
+ readonly value: A;
250
+ }
251
+ | {
252
+ readonly _tag: "rejected";
253
+ readonly error: OrchestrationActionRejected;
254
+ };
255
+
256
+ class StatefulOrchestration implements OrchestrationService {
257
+ private readonly actionQueue: CommittedAction[] = [];
222
258
  private readonly settlementListeners = new Set<SettlementListener>();
223
- private readonly runSettlementListeners = new Map<RunId, SettlementListener>();
224
- private readonly stateListeners = new Set<StateListener>();
225
- private settlementSequence = 0;
226
- private readonly disposedSessions = new WeakSet<WorkerSessionHandle>();
227
- private shuttingDown = false;
228
- private shutdownPromise: Promise<void> | undefined;
229
-
230
- constructor(options: OrchestratorRuntimeOptions) {
231
- this.workerSessionFactory = options.workerSessionFactory;
232
- this.idFactories = options.idFactories ?? createRandomIdFactories();
233
- this.clock = options.clock ?? Date.now;
234
- this.scheduler = options.scheduler ?? createWorkflowScheduler<WorkerId>();
235
- this.bestEffortDeadline = options.bestEffortDeadline ?? defaultBestEffortDeadline;
259
+ private readonly stateListeners = new Map<string, Set<StateListener>>();
260
+ private drainingActions = false;
261
+ private state: RuntimeState;
262
+
263
+ constructor(
264
+ private readonly childSessions: ChildSessionsService,
265
+ private readonly generations: FiberMap.FiberMap<WorkerId, void, never>,
266
+ private readonly runGeneration: (
267
+ key: WorkerId,
268
+ effect: Effect.Effect<void, never>,
269
+ ) => Fiber.Fiber<void, never>,
270
+ private readonly cancellations: FiberSet.FiberSet<void, never>,
271
+ private readonly runCancellation: (
272
+ effect: Effect.Effect<void, never>,
273
+ ) => Fiber.Fiber<void, never>,
274
+ private readonly cleanups: FiberSet.FiberSet<void, never>,
275
+ private readonly clock: Clock.Clock,
276
+ private readonly idFactories: OrchestrateIdFactories,
277
+ private readonly shutdownCompletion: Deferred.Deferred<void>,
278
+ ) {
279
+ this.state = initialState();
236
280
  }
237
281
 
238
282
  orchestrate(
239
283
  context: OrchestrationContext,
240
284
  task: OrchestrateTaskInput,
241
285
  mode: "async",
242
- signal?: AbortSignal,
243
286
  onSettlement?: SettlementListener,
244
- ): Promise<AcceptedRun>;
287
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
245
288
  orchestrate(
246
289
  context: OrchestrationContext,
247
290
  task: OrchestrateTaskInput,
248
291
  mode: "inline",
249
- signal?: AbortSignal,
250
292
  onSettlement?: SettlementListener,
251
- ): Promise<CompletedRun>;
293
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
252
294
  orchestrate(
253
295
  context: OrchestrationContext,
254
296
  task: OrchestrateTaskInput,
255
297
  mode: RunMode,
256
- signal?: AbortSignal,
257
298
  onSettlement?: SettlementListener,
258
- ): Promise<AcceptedRun | CompletedRun>;
259
- async orchestrate(
299
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
300
+ orchestrate(
260
301
  context: OrchestrationContext,
261
302
  task: OrchestrateTaskInput,
262
303
  mode: RunMode,
263
- signal?: AbortSignal,
264
304
  onSettlement?: SettlementListener,
265
- ): Promise<AcceptedRun | CompletedRun> {
266
- this.assertOpen();
267
- throwIfAborted(signal);
268
- const definition = this.validateTask(context, task, mode);
269
-
270
- const runId = this.idFactories.runId();
271
- const workerId = this.idFactories.workerId();
272
- this.assertFreshIds(runId, workerId);
273
-
274
- const run: RunRecord = {
275
- id: runId,
276
- ownerSessionId: context.ownerSessionId,
277
- workerId,
278
- mode,
279
- state: "running",
280
- createdAt: this.clock(),
281
- ...(context.synthesisGroup
282
- ? {
283
- synthesisGroupId: context.synthesisGroup.id,
284
- synthesisGroupSize: context.synthesisGroup.size,
285
- }
286
- : {}),
287
- };
288
- const record: WorkerRecord = {
289
- id: workerId,
290
- worker: definition.name,
291
- ownerSessionId: context.ownerSessionId,
292
- runId,
293
- title: task.title,
294
- instructions: task.instructions,
295
- lifecycle: definition.lifecycle,
296
- status: "starting",
297
- usage: copyUsage(EMPTY_WORKER_USAGE),
298
- messageDirection: "to-model",
299
- startedAt: this.clock(),
300
- };
305
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
306
+ return Effect.fn("Orchestration.orchestrate")(function* (
307
+ this: StatefulOrchestration,
308
+ ) {
309
+ const validated = yield* this.validateTask(context, task, mode);
310
+ const preflight = this.preflightOpen("orchestrate");
311
+ if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
312
+ // Keep user-supplied ID factories outside the transaction: a reentrant
313
+ // factory cannot commit an outer stale draft. Admission rechecks all
314
+ // authority below, so a race may burn an ID but cannot create a run.
315
+ const runId = this.idFactories.runId();
316
+ const workerId = this.idFactories.workerId();
317
+ const completion = yield* Deferred.make<CompletedRun>();
318
+ const now = this.clock.currentTimeMillisUnsafe();
319
+ const runRecord = makeRunRecord(runId, workerId, context, mode, now);
320
+ const workerRecord = makeWorkerRecord(
321
+ workerId,
322
+ runId,
323
+ context.ownerSessionId,
324
+ validated.definition,
325
+ validated.task,
326
+ now,
327
+ );
301
328
 
302
- const waiter = makeRunWaiter();
303
- this.runs.set(runId, run);
304
- this.runWaiters.set(runId, waiter);
305
- if (onSettlement) this.runSettlementListeners.set(runId, onSettlement);
306
- this.workers.set(workerId, record);
307
- this.entries.set(workerId, { context, definition, generation: 1 });
308
- this.emitState(context.ownerSessionId);
309
- this.launchBootstrap(workerId, 1);
310
-
311
- if (mode === "inline") {
312
- return this.awaitInlineRun(run, waiter, signal);
313
- }
314
- return freezeAcceptedRun(runId, workerId);
329
+ const admission = this.transact((draft) => {
330
+ const open = openDecision(draft, "orchestrate");
331
+ if (open._tag === "rejected") return { value: open };
332
+ if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
333
+ if (draft.workers.has(workerId)) {
334
+ throw new Error(`Duplicate worker ID: ${workerId}`);
335
+ }
336
+
337
+ draft.runs.set(runId, {
338
+ _tag: "running",
339
+ record: runRecord,
340
+ completion,
341
+ ...(onSettlement ? { settlementListener: onSettlement } : {}),
342
+ });
343
+ draft.workers.set(workerId, {
344
+ record: workerRecord,
345
+ context,
346
+ definition: validated.definition,
347
+ generation: 1,
348
+ });
349
+
350
+ return {
351
+ value: accepted(undefined),
352
+ actions: [
353
+ publishState(context.ownerSessionId),
354
+ runAction(() => this.launchBootstrap(workerId, 1)),
355
+ ],
356
+ };
357
+ });
358
+ if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
359
+
360
+ if (mode === "inline") {
361
+ return yield* this.awaitInlineRun(runRecord, completion);
362
+ }
363
+ return freezeAcceptedRun(runId, workerId);
364
+ }).call(this);
315
365
  }
316
366
 
317
367
  sendInteractive(
318
368
  context: OrchestrationContext,
319
- workerId: WorkerId,
369
+ workerId: string,
320
370
  instructions: string,
321
371
  mode: "async",
322
- signal?: AbortSignal,
323
372
  onSettlement?: SettlementListener,
324
- ): Promise<AcceptedRun>;
373
+ ): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
325
374
  sendInteractive(
326
375
  context: OrchestrationContext,
327
- workerId: WorkerId,
376
+ workerId: string,
328
377
  instructions: string,
329
378
  mode: "inline",
330
- signal?: AbortSignal,
331
379
  onSettlement?: SettlementListener,
332
- ): Promise<CompletedRun>;
380
+ ): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
333
381
  sendInteractive(
334
382
  context: OrchestrationContext,
335
- workerId: WorkerId,
383
+ workerId: string,
336
384
  instructions: string,
337
385
  mode: RunMode,
338
- signal?: AbortSignal,
339
386
  onSettlement?: SettlementListener,
340
- ): Promise<AcceptedRun | CompletedRun>;
341
- async sendInteractive(
387
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
388
+ sendInteractive(
342
389
  context: OrchestrationContext,
343
- workerId: WorkerId,
390
+ workerId: string,
344
391
  instructions: string,
345
392
  mode: RunMode,
346
- signal?: AbortSignal,
347
393
  onSettlement?: SettlementListener,
348
- ): Promise<AcceptedRun | CompletedRun> {
349
- this.assertOpen();
350
- throwIfAborted(signal);
351
- validateContextOwner(context.ownerSessionId);
352
- validateMode(mode);
353
- validateText("instructions", instructions, MAX_WORKER_INSTRUCTIONS_LENGTH);
354
-
355
- const current = this.ownedWorker(context.ownerSessionId, workerId);
356
- if (current.lifecycle !== "interactive" || current.status !== "ready") {
357
- throw new Error("interactive_send requires an owned ready interactive worker");
358
- }
359
- const entry = this.entries.get(workerId);
360
- if (!entry?.session) throw new Error("Ready interactive worker has no session handle");
361
-
362
- const runId = this.idFactories.runId();
363
- if (this.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
364
- const run: RunRecord = {
365
- id: runId,
366
- ownerSessionId: context.ownerSessionId,
367
- workerId,
368
- mode,
369
- state: "running",
370
- createdAt: this.clock(),
371
- };
372
- const running: WorkerRecord = {
373
- ...transitionWorkerStatus(current, "running"),
374
- runId,
375
- instructions,
376
- activity: undefined,
377
- messageDirection: "to-model",
378
- startedAt: this.clock(),
379
- settledAt: undefined,
380
- };
381
- const waiter = makeRunWaiter();
382
-
383
- entry.generation += 1;
384
- const generation = entry.generation;
385
- this.runs.set(runId, run);
386
- this.runWaiters.set(runId, waiter);
387
- if (onSettlement) this.runSettlementListeners.set(runId, onSettlement);
388
- this.workers.set(workerId, running);
389
- this.subscribeEntryObservability(workerId, entry, entry.session, generation);
390
- this.emitState(context.ownerSessionId);
391
- this.launchPrompt(workerId, generation, entry.session, instructions);
392
-
393
- if (mode === "inline") {
394
- return this.awaitInlineRun(run, waiter, signal);
395
- }
396
- return freezeAcceptedRun(runId, workerId);
394
+ ): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
395
+ return Effect.fn("Orchestration.sendInteractive")(function* (
396
+ this: StatefulOrchestration,
397
+ ) {
398
+ yield* validateContextOwner("sendInteractive", context.ownerSessionId);
399
+ yield* validateMode("sendInteractive", mode);
400
+ yield* validateText(
401
+ "sendInteractive",
402
+ "instructions",
403
+ instructions,
404
+ MAX_WORKER_INSTRUCTIONS_LENGTH,
405
+ );
406
+ const validatedWorkerId = yield* validateWorkerId(
407
+ "sendInteractive",
408
+ workerId,
409
+ );
410
+ const preflight = this.preflightReadyInteractive(
411
+ context.ownerSessionId,
412
+ validatedWorkerId,
413
+ );
414
+ if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
415
+ // See orchestrate: allocation remains outside copy-on-write reducers so
416
+ // reentrant factories cannot invalidate transaction atomicity.
417
+ const runId = this.idFactories.runId();
418
+ const completion = yield* Deferred.make<CompletedRun>();
419
+ const now = this.clock.currentTimeMillisUnsafe();
420
+ const runRecord: RunRecord = {
421
+ id: runId,
422
+ ownerSessionId: context.ownerSessionId,
423
+ workerId: validatedWorkerId,
424
+ mode,
425
+ state: "running",
426
+ createdAt: now,
427
+ };
428
+
429
+ const admission = this.transact((draft) => {
430
+ const ready = readyInteractiveDecision(
431
+ draft,
432
+ context.ownerSessionId,
433
+ validatedWorkerId,
434
+ );
435
+ if (ready._tag === "rejected") return { value: ready };
436
+ if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
437
+
438
+ const { worker, session } = ready.value;
439
+ const generation = worker.generation + 1;
440
+ const runningRecord: WorkerRecord = {
441
+ ...transitionWorkerStatus(worker.record, "running"),
442
+ runId,
443
+ instructions,
444
+ activity: undefined,
445
+ messageDirection: "to-model",
446
+ startedAt: now,
447
+ settledAt: undefined,
448
+ };
449
+ draft.runs.set(runId, {
450
+ _tag: "running",
451
+ record: runRecord,
452
+ completion,
453
+ ...(onSettlement ? { settlementListener: onSettlement } : {}),
454
+ });
455
+ draft.workers.set(validatedWorkerId, {
456
+ ...worker,
457
+ record: runningRecord,
458
+ generation,
459
+ observationRelease: undefined,
460
+ cancellation: undefined,
461
+ });
462
+
463
+ return {
464
+ value: accepted(undefined),
465
+ actions: [
466
+ runAction(() => {
467
+ safelyCall(worker.observationRelease);
468
+ this.subscribeObservation(validatedWorkerId, generation, session);
469
+ }),
470
+ publishState(context.ownerSessionId),
471
+ runAction(() => {
472
+ this.launchPrompt(
473
+ validatedWorkerId,
474
+ generation,
475
+ session,
476
+ instructions,
477
+ );
478
+ }),
479
+ ],
480
+ };
481
+ });
482
+ if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
483
+
484
+ if (mode === "inline") {
485
+ return yield* this.awaitInlineRun(runRecord, completion);
486
+ }
487
+ return freezeAcceptedRun(runId, validatedWorkerId);
488
+ }).call(this);
397
489
  }
398
490
 
399
- async abort(ownerSessionId: string, target: AbortTarget): Promise<void> {
400
- this.assertOpen();
401
- validateContextOwner(ownerSessionId);
402
- const targets = this.resolveAbortTargets(ownerSessionId, target);
403
- await this.cancelWorkers(targets);
491
+ abort(
492
+ ownerSessionId: string,
493
+ target: AbortTarget,
494
+ ): Effect.Effect<void, OrchestrationActionRejected> {
495
+ return Effect.fn("Orchestration.abort")(function* (
496
+ this: StatefulOrchestration,
497
+ ) {
498
+ yield* validateContextOwner("abort", ownerSessionId);
499
+ const validatedTarget = yield* validateAbortTarget(target);
500
+ const candidateIds = validatedTarget._tag === "ids"
501
+ ? validatedTarget.workerIds
502
+ : activeWorkerIds(this.current(), ownerSessionId);
503
+ const candidates = yield* makeCancellationCandidates(candidateIds);
504
+ const cancellation = this.beginCancellation(
505
+ "abort",
506
+ ownerSessionId,
507
+ validatedTarget,
508
+ candidates,
509
+ );
510
+ if (cancellation._tag === "rejected") {
511
+ return yield* Effect.fail(cancellation.error);
512
+ }
513
+ yield* awaitAll(cancellation.value);
514
+ }).call(this);
404
515
  }
405
516
 
406
- async closeInteractive(ownerSessionId: string, workerId: WorkerId): Promise<void> {
407
- this.assertOpen();
408
- validateContextOwner(ownerSessionId);
409
- const current = this.ownedWorker(ownerSessionId, workerId);
410
- if (current.lifecycle !== "interactive" || current.status !== "ready") {
411
- throw new Error("interactive_close requires an owned ready interactive worker");
412
- }
413
- this.closeReadyInteractiveWorker(current);
517
+ closeInteractive(
518
+ ownerSessionId: string,
519
+ workerId: string,
520
+ ): Effect.Effect<void, OrchestrationActionRejected> {
521
+ return Effect.fn("Orchestration.closeInteractive")(function* (
522
+ this: StatefulOrchestration,
523
+ ) {
524
+ yield* validateContextOwner("closeInteractive", ownerSessionId);
525
+ const id = yield* validateWorkerId("closeInteractive", workerId);
526
+ const now = this.clock.currentTimeMillisUnsafe();
527
+ const decision = this.transact((draft) => {
528
+ const open = openDecision(draft, "closeInteractive");
529
+ if (open._tag === "rejected") return { value: open };
530
+ const ownership = ownedWorkerDecision(
531
+ draft,
532
+ "closeInteractive",
533
+ ownerSessionId,
534
+ id,
535
+ );
536
+ if (ownership._tag === "rejected") return { value: ownership };
537
+ if (
538
+ ownership.value.record.lifecycle !== "interactive" ||
539
+ ownership.value.record.status !== "ready"
540
+ ) {
541
+ return {
542
+ value: rejected(
543
+ "closeInteractive",
544
+ "worker-state",
545
+ "interactive_close requires an owned ready interactive worker",
546
+ ),
547
+ };
548
+ }
549
+ const actions = this.closeReadyWorker(draft, ownership.value, now);
550
+ actions.push(...stateActionsAfterPrune(draft, ownerSessionId));
551
+ return {
552
+ value: accepted(undefined),
553
+ actions,
554
+ };
555
+ });
556
+ if (decision._tag === "rejected") yield* Effect.fail(decision.error);
557
+ }).call(this);
414
558
  }
415
559
 
416
- async snapshot(ownerSessionId: string): Promise<RuntimeSnapshot> {
417
- validateContextOwner(ownerSessionId);
418
- const runs = [...this.runs.values()]
419
- .filter((run) => run.ownerSessionId === ownerSessionId)
420
- .map(copyRunRecord);
421
- const workers = [...this.workers.values()]
422
- .filter((worker) => worker.ownerSessionId === ownerSessionId)
423
- .map(copyWorkerRecord);
424
- return Object.freeze({
425
- runs: Object.freeze(runs),
426
- workers: Object.freeze(workers),
427
- });
560
+ snapshot(
561
+ ownerSessionId: string,
562
+ ): Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected> {
563
+ return validateContextOwner("snapshot", ownerSessionId).pipe(
564
+ Effect.andThen(Effect.sync(() => snapshotFor(this.current(), ownerSessionId))),
565
+ );
428
566
  }
429
567
 
430
568
  subscribeSettlement(listener: SettlementListener): UnsubscribeSettlement {
431
- if (typeof listener !== "function") throw new Error("Settlement listener must be a function");
569
+ if (typeof listener !== "function") {
570
+ throw new Error("Settlement listener must be a function");
571
+ }
572
+ if (this.state.lifecycle !== "open") return noOp;
432
573
  this.settlementListeners.add(listener);
433
- let subscribed = true;
574
+
575
+ let active = true;
434
576
  return () => {
435
- if (!subscribed) return;
436
- subscribed = false;
577
+ if (!active) return;
578
+ active = false;
437
579
  this.settlementListeners.delete(listener);
438
580
  };
439
581
  }
440
582
 
441
- subscribeState(listener: StateListener): () => void {
442
- if (typeof listener !== "function") throw new Error("State listener must be a function");
443
- this.stateListeners.add(listener);
444
- let subscribed = true;
583
+ subscribeState(ownerSessionId: string, listener: StateListener): () => void {
584
+ if (typeof ownerSessionId !== "string" || ownerSessionId.trim() === "") {
585
+ throw new Error("ownerSessionId must not be blank");
586
+ }
587
+ if (typeof listener !== "function") {
588
+ throw new Error("State listener must be a function");
589
+ }
590
+
591
+ if (this.state.lifecycle !== "open") return noOp;
592
+ const ownerListeners = this.stateListeners.get(ownerSessionId) ?? new Set();
593
+ ownerListeners.add(listener);
594
+ this.stateListeners.set(ownerSessionId, ownerListeners);
595
+ this.enqueueActions([
596
+ publishStateTo(ownerSessionId, listener)(
597
+ this.state,
598
+ this.settlementListeners,
599
+ this.stateListeners,
600
+ ),
601
+ ]);
602
+
603
+ let active = true;
445
604
  return () => {
446
- if (!subscribed) return;
447
- subscribed = false;
448
- this.stateListeners.delete(listener);
605
+ if (!active) return;
606
+ active = false;
607
+ ownerListeners.delete(listener);
608
+ if (ownerListeners.size === 0) this.stateListeners.delete(ownerSessionId);
449
609
  };
450
610
  }
451
611
 
452
- shutdown(): Promise<void> {
453
- if (this.shutdownPromise) return this.shutdownPromise;
454
- this.shuttingDown = true;
455
- this.shutdownPromise = this.performShutdown();
456
- return this.shutdownPromise;
612
+ /** Calling shutdown closes admission synchronously, before the returned Effect runs. */
613
+ shutdown(): Effect.Effect<void> {
614
+ const first = this.transact((draft) => {
615
+ if (draft.lifecycle !== "open") return { value: false };
616
+ draft.lifecycle = "shutting-down";
617
+ return { value: true };
618
+ });
619
+ if (!first) return Deferred.await(this.shutdownCompletion);
620
+
621
+ return this.performShutdown().pipe(
622
+ Effect.ensuring(
623
+ Effect.sync(() => {
624
+ this.transact((draft) => {
625
+ draft.lifecycle = "shutdown";
626
+ for (const [runId, run] of draft.runs) {
627
+ if (isRunningRuntimeRun(run) && run.settlementListener) {
628
+ draft.runs.set(runId, { ...run, settlementListener: undefined });
629
+ }
630
+ }
631
+ return { value: undefined };
632
+ });
633
+ this.settlementListeners.clear();
634
+ this.stateListeners.clear();
635
+ Deferred.doneUnsafe(this.shutdownCompletion, Effect.void);
636
+ }),
637
+ ),
638
+ );
457
639
  }
458
640
 
459
- private async performShutdown(): Promise<void> {
460
- try {
461
- this.closeReadyInteractiveWorkersForShutdown();
462
- const active = [...this.workers.values()]
463
- .filter((worker) => isActiveWorkerStatus(worker.status))
464
- .map((worker) => worker.id);
465
- await this.cancelWorkers(active);
466
- } finally {
467
- try {
468
- await this.scheduler.close();
469
- } catch {
470
- // Scheduler closure is best-effort during process shutdown.
471
- } finally {
472
- await this.awaitTrackedCleanupBestEffort();
473
- this.runSettlementListeners.clear();
474
- this.settlementListeners.clear();
475
- this.stateListeners.clear();
476
- }
477
- }
641
+ private performShutdown(): Effect.Effect<void> {
642
+ return Effect.fn("Orchestration.shutdown")(function* (
643
+ this: StatefulOrchestration,
644
+ ) {
645
+ const now = this.clock.currentTimeMillisUnsafe();
646
+ this.transact((draft) => {
647
+ const owners = new Set<string>();
648
+ const actions: PostCommitAction[] = [];
649
+ for (const worker of draft.workers.values()) {
650
+ if (
651
+ worker.record.lifecycle === "interactive" &&
652
+ worker.record.status === "ready"
653
+ ) {
654
+ this.closeReadyWorker(draft, worker, now, actions);
655
+ owners.add(worker.record.ownerSessionId);
656
+ }
657
+ }
658
+ addAll(owners, pruneHistory(draft));
659
+ actions.push(...publishOwners(owners));
660
+ return { value: undefined, actions };
661
+ });
662
+
663
+ const activeIds = activeWorkerIds(this.current());
664
+ const candidates = yield* makeCancellationCandidates(activeIds);
665
+ const cancellations = this.beginShutdownCancellation(candidates);
666
+ yield* awaitAll(cancellations);
667
+ yield* this.childSessions.shutdown().pipe(Effect.catchCause(() => Effect.void));
668
+ yield* FiberSet.awaitEmpty(this.cleanups).pipe(
669
+ Effect.timeoutOption(SHUTDOWN_CLEANUP_GRACE_MS),
670
+ Effect.ignore,
671
+ );
672
+ }).call(this);
478
673
  }
479
674
 
480
675
  private validateTask(
481
676
  context: OrchestrationContext,
482
677
  task: OrchestrateTaskInput,
483
678
  mode: RunMode,
484
- ): WorkerDefinition {
485
- validateContextOwner(context.ownerSessionId);
486
- validateMode(mode);
487
- if (!task || typeof task !== "object" || Array.isArray(task)) {
488
- throw new Error("orchestrate requires one task object");
489
- }
490
- if (context.synthesisGroup) {
491
- validateText("synthesis group ID", context.synthesisGroup.id, MAX_WORKER_TITLE_LENGTH);
492
- if (mode !== "async") throw new Error("Sibling synthesis requires an async task");
493
- if (!Number.isSafeInteger(context.synthesisGroup.size) || context.synthesisGroup.size < 2) {
494
- throw new Error("Synthesis group size must be an integer of at least 2");
679
+ ): Effect.Effect<
680
+ { definition: WorkerDefinition; task: OrchestrateTaskInput },
681
+ OrchestrationActionRejected
682
+ > {
683
+ return Effect.gen(function* () {
684
+ yield* validateContextOwner("orchestrate", context.ownerSessionId);
685
+ yield* validateMode("orchestrate", mode);
686
+ if (!task || typeof task !== "object" || Array.isArray(task)) {
687
+ return yield* rejectAction(
688
+ "orchestrate",
689
+ "validation",
690
+ "orchestrate requires one task object",
691
+ );
495
692
  }
496
- }
497
-
498
- validateText("worker", task.worker, MAX_WORKER_TITLE_LENGTH);
499
- validateText("title", task.title, MAX_WORKER_TITLE_LENGTH);
500
- validateText("instructions", task.instructions, MAX_WORKER_INSTRUCTIONS_LENGTH);
501
- const definition = findWorkerByName(context.catalog, task.worker);
502
- if (!definition) throw new Error(`Unknown worker: ${task.worker}`);
503
- resolveWorkerModel(definition, context.parentModel, context.modelRegistry);
504
- return definition;
505
- }
506
-
507
- private assertFreshIds(runId: RunId, workerId: WorkerId): void {
508
- if (this.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
509
- if (this.workers.has(workerId)) throw new Error(`Duplicate worker ID: ${workerId}`);
693
+ if (context.synthesisGroup) {
694
+ yield* validateText(
695
+ "orchestrate",
696
+ "synthesis group ID",
697
+ context.synthesisGroup.id,
698
+ MAX_WORKER_TITLE_LENGTH,
699
+ );
700
+ if (mode !== "async") {
701
+ return yield* rejectAction(
702
+ "orchestrate",
703
+ "validation",
704
+ "Sibling synthesis requires an async task",
705
+ );
706
+ }
707
+ if (
708
+ !Number.isSafeInteger(context.synthesisGroup.size) ||
709
+ context.synthesisGroup.size < 2
710
+ ) {
711
+ return yield* rejectAction(
712
+ "orchestrate",
713
+ "validation",
714
+ "Synthesis group size must be an integer of at least 2",
715
+ );
716
+ }
717
+ }
718
+ yield* validateText(
719
+ "orchestrate",
720
+ "worker",
721
+ task.worker,
722
+ MAX_WORKER_TITLE_LENGTH,
723
+ );
724
+ yield* validateText(
725
+ "orchestrate",
726
+ "title",
727
+ task.title,
728
+ MAX_WORKER_TITLE_LENGTH,
729
+ );
730
+ yield* validateText(
731
+ "orchestrate",
732
+ "instructions",
733
+ task.instructions,
734
+ MAX_WORKER_INSTRUCTIONS_LENGTH,
735
+ );
736
+ const definition = findWorkerByName(context.catalog, task.worker);
737
+ if (!definition) {
738
+ return yield* rejectAction(
739
+ "orchestrate",
740
+ "unknown-worker",
741
+ `Unknown worker: ${task.worker}`,
742
+ );
743
+ }
744
+ const configured = definition.model;
745
+ if (!configured && !context.parentModel) {
746
+ return yield* rejectAction(
747
+ "orchestrate",
748
+ "model-unavailable",
749
+ `Worker "${definition.name}" has no configured model and no parent model is available`,
750
+ );
751
+ }
752
+ if (
753
+ configured &&
754
+ !context.modelRegistry.find(configured.provider, configured.modelId)
755
+ ) {
756
+ return yield* rejectAction(
757
+ "orchestrate",
758
+ "model-unavailable",
759
+ `Worker "${definition.name}" configured model "${configured.provider}/${configured.modelId}" was not found`,
760
+ );
761
+ }
762
+ return { definition, task };
763
+ });
510
764
  }
511
765
 
512
766
  private launchBootstrap(workerId: WorkerId, generation: number): void {
513
- try {
514
- this.scheduler.start(
515
- workerId,
516
- this.bootstrapAndPrompt(workerId, generation),
517
- (error) => this.settleWorkflowDefect(workerId, generation, error),
518
- );
519
- } catch (error) {
520
- this.settleWorkflowDefect(workerId, generation, error);
521
- }
767
+ this.launchGeneration(
768
+ workerId,
769
+ generation,
770
+ this.bootstrapAndPrompt(workerId, generation),
771
+ );
522
772
  }
523
773
 
524
774
  private launchPrompt(
@@ -526,29 +776,74 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
526
776
  generation: number,
527
777
  session: WorkerSessionHandle,
528
778
  instructions: string,
779
+ ): void {
780
+ this.launchGeneration(
781
+ workerId,
782
+ generation,
783
+ this.executePrompt(workerId, generation, session, instructions),
784
+ );
785
+ }
786
+
787
+ private launchGeneration(
788
+ workerId: WorkerId,
789
+ generation: number,
790
+ workflow: Effect.Effect<void, never>,
529
791
  ): void {
530
792
  try {
531
- this.scheduler.start(
793
+ const fiber = this.runGeneration(
532
794
  workerId,
533
- this.executePrompt(workerId, generation, session, instructions),
534
- (error) => this.settleWorkflowDefect(workerId, generation, error),
795
+ workflow.pipe(
796
+ Effect.catchCause((cause) => {
797
+ if (Cause.hasInterruptsOnly(cause)) return Effect.void;
798
+ return Effect.sync(() => {
799
+ this.settleWorkflowDefect(workerId, generation, Cause.squash(cause));
800
+ });
801
+ }),
802
+ ),
535
803
  );
804
+ const exit = fiber.pollUnsafe();
805
+ // A closed FiberMap rejects admission with an interrupted sentinel.
806
+ if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
807
+ this.settleGenerationLaunchFailure(workerId, generation);
808
+ }
536
809
  } catch (error) {
537
810
  this.settleWorkflowDefect(workerId, generation, error);
538
811
  }
539
812
  }
540
813
 
814
+ private settleGenerationLaunchFailure(
815
+ workerId: WorkerId,
816
+ generation: number,
817
+ ): void {
818
+ this.settleActiveWorker(
819
+ workerId,
820
+ generation,
821
+ ["starting", "running"],
822
+ "failed",
823
+ {
824
+ status: "failed",
825
+ message: "Worker generation could not start because orchestration is closed",
826
+ },
827
+ "workflow",
828
+ true,
829
+ );
830
+ }
831
+
541
832
  private bootstrapAndPrompt(
542
833
  workerId: WorkerId,
543
834
  generation: number,
544
835
  ): Effect.Effect<void, never> {
545
- const runtime = this;
546
- return Effect.gen(function* () {
547
- const session = yield* runtime.bootstrap(workerId, generation);
836
+ return Effect.gen({ self: this }, function* () {
837
+ const session = yield* this.bootstrap(workerId, generation);
548
838
  if (!session) return;
549
- const current = runtime.workers.get(workerId);
550
- if (!current) return;
551
- yield* runtime.executePrompt(workerId, generation, session, current.instructions);
839
+ const worker = this.current().workers.get(workerId);
840
+ if (!worker) return;
841
+ yield* this.executePrompt(
842
+ workerId,
843
+ generation,
844
+ session,
845
+ worker.record.instructions,
846
+ );
552
847
  });
553
848
  }
554
849
 
@@ -557,89 +852,109 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
557
852
  generation: number,
558
853
  ): Effect.Effect<WorkerSessionHandle | undefined, never> {
559
854
  return Effect.suspend(() => {
560
- const entry = this.entries.get(workerId);
561
- if (!entry) return Effect.succeed(undefined);
562
-
563
- let creation: Promise<WorkerSessionHandle>;
564
- try {
565
- creation = this.workerSessionFactory.create({
566
- cwd: entry.context.cwd,
567
- agentDir: entry.context.agentDir,
568
- parentSessionFile: entry.context.parentSessionFile,
569
- projectTrusted: entry.context.projectTrusted,
570
- definition: entry.definition,
571
- parentModel: entry.context.parentModel,
572
- modelRegistry: entry.context.modelRegistry,
573
- });
574
- } catch (error) {
575
- this.settleCreationFailure(workerId, generation, error);
855
+ const expected = this.current().workers.get(workerId);
856
+ if (!expected || expected.generation !== generation) {
576
857
  return Effect.succeed(undefined);
577
858
  }
578
-
579
- this.trackCleanup(creation.then(() => undefined, () => undefined));
580
- void creation.then((session) => {
581
- if (!this.canAdoptCreatedSession(workerId, generation, entry)) {
582
- this.disposeSession(session);
583
- }
584
- }, () => undefined);
585
-
586
- return Effect.tryPromise({
587
- try: () => creation,
588
- catch: (error) => error,
589
- }).pipe(
859
+ return this.childSessions.acquire(
860
+ {
861
+ cwd: expected.context.cwd,
862
+ agentDir: expected.context.agentDir,
863
+ parentSessionFile: expected.context.parentSessionFile,
864
+ projectTrusted: expected.context.projectTrusted,
865
+ definition: expected.definition,
866
+ parentModel: expected.context.parentModel,
867
+ modelRegistry: expected.context.modelRegistry,
868
+ },
869
+ (session) => this.adoptCreatedSession(workerId, generation, session),
870
+ ).pipe(
590
871
  Effect.match({
591
872
  onFailure: (error) => {
592
873
  this.settleCreationFailure(workerId, generation, error);
593
874
  return undefined;
594
875
  },
595
- onSuccess: (session) => this.adoptCreatedSession(
596
- workerId,
597
- generation,
598
- entry,
599
- session,
600
- ),
876
+ onSuccess: (session) => session,
601
877
  }),
602
878
  );
603
879
  });
604
880
  }
605
881
 
606
- private canAdoptCreatedSession(
607
- workerId: WorkerId,
608
- generation: number,
609
- entry: RuntimeEntry,
610
- ): boolean {
611
- const current = this.workers.get(workerId);
612
- return !this.shuttingDown &&
613
- current?.status === "starting" &&
614
- entry.generation === generation;
615
- }
616
-
617
882
  private adoptCreatedSession(
618
883
  workerId: WorkerId,
619
884
  generation: number,
620
- entry: RuntimeEntry,
621
885
  session: WorkerSessionHandle,
622
886
  ): WorkerSessionHandle | undefined {
623
- const current = this.workers.get(workerId);
624
- if (!this.canAdoptCreatedSession(workerId, generation, entry) || !current) {
625
- this.disposeSession(session);
887
+ let release: () => void;
888
+ try {
889
+ release = session.subscribeObservation((observation) => {
890
+ this.updateObservation(workerId, generation, session, observation);
891
+ });
892
+ } catch (error) {
893
+ this.settleCreationFailure(workerId, generation, error);
626
894
  return undefined;
627
895
  }
628
896
 
897
+ const adopted = this.transact((draft) => {
898
+ const worker = draft.workers.get(workerId);
899
+ if (
900
+ draft.lifecycle !== "open" ||
901
+ !worker ||
902
+ worker.generation !== generation ||
903
+ worker.record.status !== "starting"
904
+ ) {
905
+ return {
906
+ value: false,
907
+ actions: [runAction(() => safelyCall(release))],
908
+ };
909
+ }
910
+ draft.workers.set(workerId, {
911
+ ...worker,
912
+ session,
913
+ observationRelease: release,
914
+ record: {
915
+ ...transitionWorkerStatus(worker.record, "running"),
916
+ sessionFile: session.sessionFile,
917
+ },
918
+ });
919
+ return {
920
+ value: true,
921
+ actions: [publishState(worker.record.ownerSessionId)],
922
+ };
923
+ });
924
+ return adopted ? session : undefined;
925
+ }
926
+
927
+ private subscribeObservation(
928
+ workerId: WorkerId,
929
+ generation: number,
930
+ session: WorkerSessionHandle,
931
+ ): void {
932
+ let release: () => void;
629
933
  try {
630
- entry.session = session;
631
- this.subscribeEntryObservability(workerId, entry, session, generation);
632
- this.workers.set(workerId, {
633
- ...transitionWorkerStatus(current, "running"),
634
- sessionFile: session.sessionFile,
934
+ release = session.subscribeObservation((observation) => {
935
+ this.updateObservation(workerId, generation, session, observation);
635
936
  });
636
- this.emitState(current.ownerSessionId);
637
- return session;
638
937
  } catch (error) {
639
- this.disposeEntrySession(entry);
640
- this.settleCreationFailure(workerId, generation, error);
641
- return undefined;
938
+ this.settleWorkflowDefect(workerId, generation, error);
939
+ return;
642
940
  }
941
+
942
+ this.transact((draft) => {
943
+ const worker = draft.workers.get(workerId);
944
+ if (
945
+ !worker ||
946
+ worker.generation !== generation ||
947
+ worker.session !== session ||
948
+ worker.record.status !== "running"
949
+ ) {
950
+ return {
951
+ value: undefined,
952
+ actions: [runAction(() => safelyCall(release))],
953
+ };
954
+ }
955
+ draft.workers.set(workerId, { ...worker, observationRelease: release });
956
+ return { value: undefined };
957
+ });
643
958
  }
644
959
 
645
960
  private executePrompt(
@@ -649,34 +964,16 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
649
964
  instructions: string,
650
965
  ): Effect.Effect<void, never> {
651
966
  return Effect.suspend(() => {
652
- const before = this.workers.get(workerId);
653
- const entry = this.entries.get(workerId);
967
+ const worker = this.current().workers.get(workerId);
654
968
  if (
655
- !before ||
656
- before.status !== "running" ||
657
- !entry ||
658
- entry.generation !== generation ||
659
- entry.session !== session
969
+ !worker ||
970
+ worker.record.status !== "running" ||
971
+ worker.generation !== generation ||
972
+ worker.session !== session
660
973
  ) {
661
974
  return Effect.void;
662
975
  }
663
-
664
- let prompt: Promise<WorkerOutcome>;
665
- try {
666
- prompt = session.prompt(instructions);
667
- } catch (error) {
668
- this.settleOutcome(workerId, generation, session, {
669
- status: "failed",
670
- message: describeError(error, "Worker prompt failed"),
671
- });
672
- return Effect.void;
673
- }
674
- this.trackCleanup(prompt.then(() => undefined, () => undefined));
675
-
676
- return Effect.tryPromise({
677
- try: () => prompt,
678
- catch: (error) => error,
679
- }).pipe(
976
+ return session.prompt(instructions).pipe(
680
977
  Effect.match({
681
978
  onFailure: (error): WorkerOutcome => ({
682
979
  status: "failed",
@@ -697,13 +994,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
697
994
  generation: number,
698
995
  error: unknown,
699
996
  ): void {
700
- const current = this.workers.get(workerId);
701
- const entry = this.entries.get(workerId);
702
- if (!current || current.status !== "starting" || entry?.generation !== generation) return;
703
- this.settleTerminalWorker(current, "failed", {
704
- status: "failed",
705
- message: describeError(error, "Worker session creation failed"),
706
- }, "startup");
997
+ this.settleActiveWorker(
998
+ workerId,
999
+ generation,
1000
+ ["starting"],
1001
+ "failed",
1002
+ {
1003
+ status: "failed",
1004
+ message: describeError(error, "Worker session creation failed"),
1005
+ },
1006
+ "startup",
1007
+ false,
1008
+ );
707
1009
  }
708
1010
 
709
1011
  private settleWorkflowDefect(
@@ -711,19 +1013,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
711
1013
  generation: number,
712
1014
  error: unknown,
713
1015
  ): void {
714
- const current = this.workers.get(workerId);
715
- const entry = this.entries.get(workerId);
716
- if (!current || entry?.generation !== generation) return;
717
- if (!isActiveWorkerStatus(current.status)) {
718
- this.maybeCompleteRun(current.runId);
719
- return;
720
- }
721
-
722
- this.disposeEntrySession(entry);
723
- this.settleTerminalWorker(current, "failed", {
724
- status: "failed",
725
- message: describeError(error, "Worker workflow failed"),
726
- }, "workflow");
1016
+ this.settleActiveWorker(
1017
+ workerId,
1018
+ generation,
1019
+ ["starting", "running", "stopping"],
1020
+ "failed",
1021
+ {
1022
+ status: "failed",
1023
+ message: describeError(error, "Worker workflow failed"),
1024
+ },
1025
+ "workflow",
1026
+ true,
1027
+ );
727
1028
  }
728
1029
 
729
1030
  private settleOutcome(
@@ -732,14 +1033,12 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
732
1033
  session: WorkerSessionHandle,
733
1034
  outcome: WorkerOutcome,
734
1035
  ): void {
735
- const current = this.workers.get(workerId);
736
- const entry = this.entries.get(workerId);
1036
+ const worker = this.current().workers.get(workerId);
737
1037
  if (
738
- !current ||
739
- current.status !== "running" ||
740
- !entry ||
741
- entry.generation !== generation ||
742
- entry.session !== session
1038
+ !worker ||
1039
+ worker.generation !== generation ||
1040
+ worker.session !== session ||
1041
+ worker.record.status !== "running"
743
1042
  ) {
744
1043
  return;
745
1044
  }
@@ -748,549 +1047,1033 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
748
1047
  let status: "ready" | "completed" | "failed" | "aborted";
749
1048
  if (outcome.status === "failed" || outcome.status === "aborted") {
750
1049
  status = outcome.status;
751
- } else if (current.lifecycle === "interactive" && outcome.status === "ready") {
1050
+ } else if (
1051
+ worker.record.lifecycle === "interactive" &&
1052
+ outcome.status === "ready"
1053
+ ) {
752
1054
  status = "ready";
753
- } else if (current.lifecycle === "one-shot" && outcome.status === "completed") {
1055
+ } else if (
1056
+ worker.record.lifecycle === "one-shot" &&
1057
+ outcome.status === "completed"
1058
+ ) {
754
1059
  status = "completed";
755
1060
  } else {
756
1061
  status = "failed";
757
1062
  settledOutcome = {
758
1063
  status: "failed",
759
- message: `Worker session returned ${outcome.status} for a ${current.lifecycle} worker`,
1064
+ message: `Worker session returned ${outcome.status} for a ${worker.record.lifecycle} worker`,
760
1065
  };
761
1066
  }
762
-
763
- const settledAt = this.clock();
764
- this.workers.set(workerId, {
765
- ...transitionWorkerStatus(current, status),
766
- activity: undefined,
767
- outcome: copyOutcome(settledOutcome),
768
- settledAt,
769
- });
770
- if (status !== "ready") this.disposeEntrySession(entry);
771
- this.maybeCompleteRun(current.runId);
772
- const affectedOwners = new Set([current.ownerSessionId]);
773
- this.emitSettlement(
1067
+ this.settleActiveWorker(
774
1068
  workerId,
775
1069
  generation,
776
- settledAt,
1070
+ ["running"],
1071
+ status,
1072
+ settledOutcome,
777
1073
  status === "failed" ? "prompt" : undefined,
1074
+ status !== "ready",
778
1075
  );
779
- if (isTerminalWorkerStatus(status)) {
780
- addAll(affectedOwners, this.rememberTerminalWorker(workerId));
781
- } else {
782
- addAll(affectedOwners, this.pruneHistory());
783
- }
784
- this.emitStateForOwners(affectedOwners);
785
1076
  }
786
1077
 
787
- private settleTerminalWorker(
788
- current: WorkerRecord,
789
- status: "failed" | "aborted",
1078
+ private settleActiveWorker(
1079
+ workerId: WorkerId,
1080
+ generation: number,
1081
+ expectedStatuses: readonly WorkerRecord["status"][],
1082
+ status: "ready" | "completed" | "failed" | "aborted",
790
1083
  outcome: WorkerOutcome,
791
- failureStage?: SettlementFailureStage,
1084
+ failureStage: SettlementFailureStage | undefined,
1085
+ dispose: boolean,
792
1086
  ): void {
793
- const settledAt = this.clock();
794
- this.workers.set(current.id, {
795
- ...transitionWorkerStatus(current, status),
796
- activity: undefined,
797
- outcome: copyOutcome(outcome),
798
- settledAt,
1087
+ const settledAt = this.clock.currentTimeMillisUnsafe();
1088
+ this.transact((draft) => {
1089
+ const worker = draft.workers.get(workerId);
1090
+ if (
1091
+ !worker ||
1092
+ worker.generation !== generation ||
1093
+ !expectedStatuses.includes(worker.record.status)
1094
+ ) {
1095
+ return { value: undefined };
1096
+ }
1097
+
1098
+ const settledRecord: WorkerRecord = {
1099
+ ...transitionWorkerStatus(worker.record, status),
1100
+ activity: undefined,
1101
+ outcome: copyOutcome(outcome),
1102
+ settledAt,
1103
+ };
1104
+ const actions: PostCommitAction[] = [];
1105
+ let settledWorker: RuntimeWorker = { ...worker, record: settledRecord };
1106
+ if (dispose) {
1107
+ actions.push(...this.releaseWorkerResources(settledWorker));
1108
+ settledWorker = {
1109
+ ...settledWorker,
1110
+ session: undefined,
1111
+ observationRelease: undefined,
1112
+ };
1113
+ }
1114
+ draft.workers.set(workerId, settledWorker);
1115
+ const settlement = makeSettlement(
1116
+ draft,
1117
+ settledWorker,
1118
+ settledAt,
1119
+ failureStage,
1120
+ );
1121
+ actions.push(...completeRun(draft, workerId));
1122
+ if (settlement) actions.push(settlement);
1123
+ if (isTerminalWorkerStatus(status)) rememberTerminalWorker(draft, workerId);
1124
+ actions.push(...stateActionsAfterPrune(draft, worker.record.ownerSessionId));
1125
+ return { value: undefined, actions };
799
1126
  });
800
- this.maybeCompleteRun(current.runId);
801
- const affectedOwners = new Set([current.ownerSessionId]);
802
- const generation = this.entries.get(current.id)?.generation;
803
- if (generation !== undefined) {
804
- this.emitSettlement(current.id, generation, settledAt, failureStage);
805
- }
806
- addAll(affectedOwners, this.rememberTerminalWorker(current.id));
807
- this.emitStateForOwners(affectedOwners);
808
1127
  }
809
1128
 
810
- private emitSettlement(
1129
+ private updateObservation(
811
1130
  workerId: WorkerId,
812
1131
  generation: number,
813
- settledAt: number,
814
- failureStage?: SettlementFailureStage,
1132
+ session: WorkerSessionHandle,
1133
+ observation: WorkerSessionObservation,
815
1134
  ): void {
816
- const worker = this.workers.get(workerId);
817
- const run = worker ? this.runs.get(worker.runId) : undefined;
818
- if (!worker || !run || !worker.outcome || !isSettledWorkerStatus(worker.status)) return;
819
- if (worker.outcome.status === "closed") return;
820
-
821
- const sequence = ++this.settlementSequence;
822
- const settlement: WorkerSettlement = Object.freeze({
823
- eventId: `${sequence}:${run.id}:${workerId}:${generation}`,
824
- sequence,
825
- ownerSessionId: worker.ownerSessionId,
826
- runId: run.id,
827
- workerId,
828
- generation,
829
- mode: run.mode,
830
- worker: worker.worker,
831
- title: worker.title,
832
- lifecycle: worker.lifecycle,
833
- status: worker.status,
834
- outcome: Object.freeze(copyOutcome(worker.outcome)),
835
- ...(failureStage ? { failureStage } : {}),
836
- usage: Object.freeze(copyUsage(worker.usage)),
837
- startedAt: worker.startedAt,
838
- settledAt,
839
- ...(run.synthesisGroupId && run.synthesisGroupSize
840
- ? {
841
- synthesisGroupId: run.synthesisGroupId,
842
- synthesisGroupSize: run.synthesisGroupSize,
843
- }
844
- : {}),
845
- ...(worker.sessionFile !== undefined ? { sessionFile: worker.sessionFile } : {}),
1135
+ this.transact((draft) => {
1136
+ const worker = draft.workers.get(workerId);
1137
+ if (
1138
+ !worker ||
1139
+ worker.generation !== generation ||
1140
+ worker.session !== session ||
1141
+ (worker.record.status !== "starting" && worker.record.status !== "running")
1142
+ ) {
1143
+ return { value: undefined };
1144
+ }
1145
+ const nextRecord: WorkerRecord = {
1146
+ ...worker.record,
1147
+ usage: copyUsage(observation.usage),
1148
+ activity: observation.activity,
1149
+ messageDirection: observation.messageDirection,
1150
+ };
1151
+ if (observationsEqual(worker.record, nextRecord)) {
1152
+ return { value: undefined };
1153
+ }
1154
+ draft.workers.set(workerId, { ...worker, record: nextRecord });
1155
+ return {
1156
+ value: undefined,
1157
+ actions: [publishState(worker.record.ownerSessionId)],
1158
+ };
846
1159
  });
847
-
848
- const localListener = this.runSettlementListeners.get(run.id);
849
- this.runSettlementListeners.delete(run.id);
850
- notifySettlementListener(localListener, settlement);
851
- for (const listener of [...this.settlementListeners]) {
852
- notifySettlementListener(listener, settlement);
853
- }
854
1160
  }
855
1161
 
856
- private maybeCompleteRun(runId: RunId): void {
857
- if (this.completedRuns.has(runId)) return;
858
- const run = this.runs.get(runId);
859
- if (!run) return;
860
- const worker = this.workers.get(run.workerId);
861
- if (!worker || worker.runId !== runId || !worker.outcome || isActiveWorkerStatus(worker.status)) {
862
- return;
863
- }
864
-
865
- const completed = freezeCompletedRun(run, worker);
866
- this.completedRuns.set(runId, completed);
867
- this.completedRunOrder.push(runId);
868
- this.runs.set(runId, { ...run, state: "complete" });
869
- this.runWaiters.get(runId)?.resolve(completed);
870
- this.runWaiters.delete(runId);
871
- }
1162
+ private beginCancellation(
1163
+ operation: "abort",
1164
+ ownerSessionId: string,
1165
+ target: ValidatedAbortTarget,
1166
+ candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
1167
+ ): Decision<readonly Deferred.Deferred<void>[]> {
1168
+ return this.transact((draft) => {
1169
+ const open = openDecision(draft, operation);
1170
+ if (open._tag === "rejected") return { value: open };
1171
+ const workers = target._tag === "all"
1172
+ ? [...draft.workers.values()].filter((worker) => (
1173
+ worker.record.ownerSessionId === ownerSessionId &&
1174
+ isActiveWorkerStatus(worker.record.status)
1175
+ ))
1176
+ : target.workerIds.map((id) => draft.workers.get(id));
1177
+
1178
+ if (target._tag === "ids") {
1179
+ for (let index = 0; index < target.workerIds.length; index += 1) {
1180
+ const worker = workers[index];
1181
+ if (!worker || worker.record.ownerSessionId !== ownerSessionId) {
1182
+ return {
1183
+ value: rejected(
1184
+ operation,
1185
+ "ownership",
1186
+ "Worker is not owned by this session",
1187
+ ),
1188
+ };
1189
+ }
1190
+ if (worker.record.status === "ready") {
1191
+ return {
1192
+ value: rejected(
1193
+ operation,
1194
+ "worker-state",
1195
+ "Ready interactive workers are not active; use interactive_close",
1196
+ ),
1197
+ };
1198
+ }
1199
+ if (!isActiveWorkerStatus(worker.record.status)) {
1200
+ return {
1201
+ value: rejected(
1202
+ operation,
1203
+ "worker-state",
1204
+ "worker_abort requires owned active workers",
1205
+ ),
1206
+ };
1207
+ }
1208
+ }
1209
+ }
872
1210
 
873
- private rememberTerminalWorker(workerId: WorkerId): Set<string> {
874
- if (!this.terminalWorkerOrder.includes(workerId)) {
875
- this.terminalWorkerOrder.push(workerId);
876
- }
877
- return this.pruneHistory();
1211
+ const activeWorkers = workers.filter(isRuntimeWorker);
1212
+ const marked = this.markWorkersStopping(draft, activeWorkers, candidates);
1213
+ return {
1214
+ value: accepted(marked.completions),
1215
+ actions: [
1216
+ ...marked.actions,
1217
+ ...publishOwners(new Set(
1218
+ activeWorkers.map((worker) => worker.record.ownerSessionId),
1219
+ )),
1220
+ ],
1221
+ };
1222
+ });
878
1223
  }
879
1224
 
880
- private pruneHistory(): Set<string> {
881
- const affectedOwners = new Set<string>();
882
- while (this.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
883
- const runId = this.completedRunOrder.shift();
884
- if (!runId) break;
885
- this.completedRuns.delete(runId);
886
- const run = this.runs.get(runId);
887
- if (run) affectedOwners.add(run.ownerSessionId);
888
- this.runs.delete(runId);
889
- }
890
-
891
- while (this.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
892
- const removableIndex = this.terminalWorkerOrder.findIndex((workerId) => {
893
- const worker = this.workers.get(workerId);
894
- if (!worker || !isTerminalWorkerStatus(worker.status)) return true;
895
- return this.runs.get(worker.runId)?.state !== "running";
896
- });
897
- if (removableIndex < 0) return affectedOwners;
898
- const [workerId] = this.terminalWorkerOrder.splice(removableIndex, 1);
899
- if (!workerId) return affectedOwners;
900
- const worker = this.workers.get(workerId);
901
- if (worker) affectedOwners.add(worker.ownerSessionId);
902
- this.workers.delete(workerId);
903
- this.entries.delete(workerId);
904
- }
905
-
906
- return affectedOwners;
1225
+ private beginShutdownCancellation(
1226
+ candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
1227
+ ): readonly Deferred.Deferred<void>[] {
1228
+ return this.transact((draft) => {
1229
+ const workers = [...draft.workers.values()].filter((worker) => (
1230
+ isActiveWorkerStatus(worker.record.status)
1231
+ ));
1232
+ const marked = this.markWorkersStopping(draft, workers, candidates);
1233
+ return {
1234
+ value: marked.completions,
1235
+ actions: [
1236
+ ...marked.actions,
1237
+ ...publishOwners(new Set(
1238
+ workers.map((worker) => worker.record.ownerSessionId),
1239
+ )),
1240
+ ],
1241
+ };
1242
+ });
907
1243
  }
908
1244
 
909
- private resolveAbortTargets(ownerSessionId: string, target: AbortTarget): WorkerId[] {
910
- if (!target || typeof target !== "object") throw new Error("Invalid abort target");
911
- const candidate = target as {
912
- workerIds?: readonly WorkerId[];
913
- all?: boolean;
914
- };
915
- const selected = [
916
- candidate.workerIds !== undefined,
917
- candidate.all !== undefined,
918
- ].filter(Boolean).length;
919
- if (selected !== 1 || (candidate.all !== undefined && candidate.all !== true)) {
920
- throw new Error("Abort target must specify exactly one of workerIds or all: true");
921
- }
922
-
923
- if (candidate.workerIds !== undefined) {
924
- if (!Array.isArray(candidate.workerIds) || candidate.workerIds.length === 0) {
925
- throw new Error("workerIds must contain at least one worker ID");
1245
+ // Commit stopping and one shared completion before post-commit physical abort
1246
+ // and generation interruption; every cancellation caller joins it.
1247
+ private markWorkersStopping(
1248
+ draft: RuntimeState,
1249
+ workers: readonly RuntimeWorker[],
1250
+ candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
1251
+ ): {
1252
+ readonly completions: readonly Deferred.Deferred<void>[];
1253
+ readonly actions: readonly PostCommitAction[];
1254
+ } {
1255
+ const completions: Deferred.Deferred<void>[] = [];
1256
+ const actions: PostCommitAction[] = [];
1257
+ for (const worker of workers) {
1258
+ if (worker.cancellation) {
1259
+ completions.push(worker.cancellation);
1260
+ continue;
926
1261
  }
927
- const unique = [...new Set(candidate.workerIds)];
928
- for (const workerId of unique) {
929
- const worker = this.ownedWorker(ownerSessionId, workerId);
930
- if (worker.status === "ready") {
931
- throw new Error("Ready interactive workers are not active; use interactive_close");
932
- }
933
- if (!isActiveWorkerStatus(worker.status)) {
934
- throw new Error("worker_abort requires owned active workers");
935
- }
1262
+ const completion = candidates.get(worker.record.id);
1263
+ if (!completion) {
1264
+ throw new Error(`Missing cancellation candidate: ${worker.record.id}`);
936
1265
  }
937
- return unique;
1266
+ const stoppingRecord = worker.record.status === "stopping"
1267
+ ? worker.record
1268
+ : {
1269
+ ...transitionWorkerStatus(worker.record, "stopping"),
1270
+ activity: undefined,
1271
+ };
1272
+ draft.workers.set(worker.record.id, {
1273
+ ...worker,
1274
+ record: stoppingRecord,
1275
+ cancellation: completion,
1276
+ });
1277
+ completions.push(completion);
1278
+ actions.push(runAction(() => {
1279
+ this.launchCancellationOrSettleAfterClosure(worker.record.id, completion);
1280
+ }));
938
1281
  }
939
-
940
- return [...this.workers.values()]
941
- .filter(
942
- (worker) =>
943
- worker.ownerSessionId === ownerSessionId && isActiveWorkerStatus(worker.status),
944
- )
945
- .map((worker) => worker.id);
1282
+ return { completions, actions };
946
1283
  }
947
1284
 
948
- private async cancelWorkers(workerIds: readonly WorkerId[]): Promise<void> {
949
- const owners = new Set<string>();
950
- const cancellations: Promise<void>[] = [];
951
-
952
- for (const workerId of workerIds) {
953
- const current = this.workers.get(workerId);
954
- if (!current || !isActiveWorkerStatus(current.status)) continue;
955
- if (current.status !== "stopping") {
956
- this.workers.set(workerId, {
957
- ...transitionWorkerStatus(current, "stopping"),
958
- activity: undefined,
959
- });
960
- }
961
- owners.add(current.ownerSessionId);
962
- cancellations.push(this.cancellationFor(workerId));
1285
+ private launchCancellationOrSettleAfterClosure(
1286
+ workerId: WorkerId,
1287
+ completion: Deferred.Deferred<void>,
1288
+ ): void {
1289
+ const fiber = this.runCancellation(this.cancelWorker(workerId, completion));
1290
+ const exit = fiber.pollUnsafe();
1291
+ // A closed FiberSet rejects admission with an interrupted sentinel.
1292
+ if (!exit || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return;
1293
+
1294
+ const worker = this.current().workers.get(workerId);
1295
+ if (worker?.record.status === "stopping") {
1296
+ this.settleActiveWorker(
1297
+ workerId,
1298
+ worker.generation,
1299
+ ["stopping"],
1300
+ "aborted",
1301
+ { status: "aborted" },
1302
+ "cancellation",
1303
+ true,
1304
+ );
963
1305
  }
964
-
965
- this.emitStateForOwners(owners);
966
- await Promise.all(cancellations);
1306
+ this.completeCancellation(workerId, completion);
967
1307
  }
968
1308
 
969
- private cancellationFor(workerId: WorkerId): Promise<void> {
970
- const existing = this.cancellationPromises.get(workerId);
971
- if (existing) return existing;
972
-
973
- const cancellation = Promise.resolve()
974
- .then(() => this.cancelWorker(workerId))
975
- .catch((error) => {
976
- this.settleCancellationFailure(workerId, error);
977
- });
978
- this.cancellationPromises.set(workerId, cancellation);
979
- void cancellation.then(() => {
980
- if (this.cancellationPromises.get(workerId) === cancellation) {
981
- this.cancellationPromises.delete(workerId);
982
- }
983
- });
984
- return cancellation;
985
- }
986
-
987
- private async cancelWorker(workerId: WorkerId): Promise<void> {
988
- const entry = this.entries.get(workerId);
989
- const session = entry?.session;
990
-
991
- try {
1309
+ private cancelWorker(
1310
+ workerId: WorkerId,
1311
+ completion: Deferred.Deferred<void>,
1312
+ ): Effect.Effect<void> {
1313
+ return Effect.gen({ self: this }, function* () {
1314
+ const session = this.current().workers.get(workerId)?.session;
992
1315
  if (session) {
993
- const abortOperation = Promise.resolve()
994
- .then(() => session.abort())
995
- .catch(() => undefined);
996
- await this.waitBestEffort(abortOperation, CANCELLATION_GRACE_MS);
997
- }
998
- try {
999
- await this.scheduler.remove(workerId);
1000
- } catch {
1001
- // Worker state still settles even if scheduler cleanup fails.
1316
+ yield* session.abort().pipe(
1317
+ Effect.catchTag(
1318
+ "WorkerSession.AbortError",
1319
+ (_: WorkerSessionAbortError) => Effect.void,
1320
+ ),
1321
+ Effect.timeoutOption(CANCELLATION_GRACE_MS),
1322
+ Effect.ignore,
1323
+ );
1002
1324
  }
1003
- } finally {
1004
- if (entry) this.disposeEntrySession(entry);
1005
- const current = this.workers.get(workerId);
1006
- if (current?.status === "stopping") {
1007
- this.settleTerminalWorker(
1008
- current,
1009
- "aborted",
1010
- { status: "aborted" },
1325
+ const removal = yield* FiberSet.run(
1326
+ this.cancellations,
1327
+ FiberMap.remove(this.generations, workerId).pipe(
1328
+ Effect.catchCause(() => Effect.void),
1329
+ ),
1330
+ );
1331
+ yield* Fiber.await(removal).pipe(
1332
+ Effect.timeoutOption(CANCELLATION_GRACE_MS),
1333
+ Effect.ignore,
1334
+ );
1335
+ this.settleActiveWorker(
1336
+ workerId,
1337
+ this.current().workers.get(workerId)?.generation ?? -1,
1338
+ ["stopping"],
1339
+ "aborted",
1340
+ { status: "aborted" },
1341
+ "cancellation",
1342
+ true,
1343
+ );
1344
+ }).pipe(
1345
+ Effect.catchCause((cause) => Effect.sync(() => {
1346
+ const generation = this.current().workers.get(workerId)?.generation;
1347
+ if (generation === undefined) return;
1348
+ this.settleActiveWorker(
1349
+ workerId,
1350
+ generation,
1351
+ ["stopping"],
1352
+ "failed",
1353
+ {
1354
+ status: "failed",
1355
+ message: describeError(
1356
+ Cause.squash(cause),
1357
+ "Worker cancellation failed",
1358
+ ),
1359
+ },
1011
1360
  "cancellation",
1361
+ true,
1012
1362
  );
1013
- }
1014
- }
1015
- }
1016
-
1017
- private settleCancellationFailure(workerId: WorkerId, error: unknown): void {
1018
- const current = this.workers.get(workerId);
1019
- if (!current || current.status !== "stopping") return;
1020
- const entry = this.entries.get(workerId);
1021
- if (entry) this.disposeEntrySession(entry);
1022
- this.settleTerminalWorker(current, "failed", {
1023
- status: "failed",
1024
- message: describeError(error, "Worker cancellation failed"),
1025
- }, "cancellation");
1363
+ })),
1364
+ Effect.ensuring(
1365
+ Effect.sync(() => this.completeCancellation(workerId, completion)),
1366
+ ),
1367
+ );
1026
1368
  }
1027
1369
 
1028
- private async cancelExactRun(run: RunRecord): Promise<void> {
1029
- const worker = this.workers.get(run.workerId);
1030
- const active = worker?.ownerSessionId === run.ownerSessionId &&
1031
- worker.runId === run.id &&
1032
- isActiveWorkerStatus(worker.status)
1033
- ? [worker.id]
1034
- : [];
1035
- await this.cancelWorkers(active);
1036
- this.maybeCompleteRun(run.id);
1370
+ private completeCancellation(
1371
+ workerId: WorkerId,
1372
+ completion: Deferred.Deferred<void>,
1373
+ ): void {
1374
+ this.transact((draft) => {
1375
+ const worker = draft.workers.get(workerId);
1376
+ if (worker?.cancellation === completion) {
1377
+ draft.workers.set(workerId, { ...worker, cancellation: undefined });
1378
+ }
1379
+ return { value: undefined };
1380
+ });
1381
+ Deferred.doneUnsafe(completion, Effect.void);
1037
1382
  }
1038
1383
 
1039
1384
  private awaitInlineRun(
1040
1385
  run: RunRecord,
1041
- waiter: RunWaiter,
1042
- signal: AbortSignal | undefined,
1043
- ): Promise<CompletedRun> {
1044
- if (!signal) return waiter.promise;
1045
-
1046
- return new Promise<CompletedRun>((resolve, reject) => {
1047
- let abortClaimed = false;
1048
- const removeAbortListener = () => signal.removeEventListener("abort", onAbort);
1049
- const onAbort = () => {
1050
- if (abortClaimed || waiter.settled) return;
1051
- abortClaimed = true;
1052
- removeAbortListener();
1053
- const reason = abortSignalReason(signal);
1054
- void this.cancelExactRun(run).then(
1055
- () => reject(reason),
1056
- () => reject(reason),
1057
- );
1058
- };
1059
-
1060
- waiter.onSettled = () => {
1061
- removeAbortListener();
1062
- if (!abortClaimed) waiter.promise.then(resolve, reject);
1063
- };
1064
- signal.addEventListener("abort", onAbort, { once: true });
1386
+ completion: Deferred.Deferred<CompletedRun>,
1387
+ ): Effect.Effect<CompletedRun> {
1388
+ return Deferred.await(completion).pipe(
1389
+ Effect.onInterrupt(() => this.cancelExactRun(run)),
1390
+ );
1391
+ }
1065
1392
 
1066
- if (signal.aborted) onAbort();
1067
- else if (waiter.settled) waiter.onSettled();
1393
+ private cancelExactRun(run: RunRecord): Effect.Effect<void> {
1394
+ return Effect.gen({ self: this }, function* () {
1395
+ const candidate = yield* Deferred.make<void>();
1396
+ const candidates = new Map([[run.workerId, candidate]]);
1397
+ const completions = this.transact((draft) => {
1398
+ const worker = draft.workers.get(run.workerId);
1399
+ if (
1400
+ !worker ||
1401
+ worker.record.ownerSessionId !== run.ownerSessionId ||
1402
+ worker.record.runId !== run.id ||
1403
+ !isActiveWorkerStatus(worker.record.status)
1404
+ ) {
1405
+ return { value: [] as readonly Deferred.Deferred<void>[] };
1406
+ }
1407
+ const marked = this.markWorkersStopping(draft, [worker], candidates);
1408
+ return {
1409
+ value: marked.completions,
1410
+ actions: [...marked.actions, publishState(run.ownerSessionId)],
1411
+ };
1412
+ });
1413
+ yield* awaitAll(completions);
1068
1414
  });
1069
1415
  }
1070
1416
 
1071
- private closeReadyInteractiveWorker(current: WorkerRecord): void {
1072
- const entry = this.entries.get(current.id);
1073
- if (entry) this.disposeEntrySession(entry);
1074
- this.workers.set(current.id, {
1075
- ...transitionWorkerStatus(current, "closed"),
1417
+ private closeReadyWorker(
1418
+ draft: RuntimeState,
1419
+ worker: RuntimeWorker,
1420
+ settledAt: number,
1421
+ actions: PostCommitAction[] = [],
1422
+ ): PostCommitAction[] {
1423
+ const closedRecord: WorkerRecord = {
1424
+ ...transitionWorkerStatus(worker.record, "closed"),
1076
1425
  activity: undefined,
1077
1426
  outcome: { status: "closed" },
1078
- settledAt: this.clock(),
1427
+ settledAt,
1428
+ };
1429
+ actions.push(...this.releaseWorkerResources(worker));
1430
+ draft.workers.set(worker.record.id, {
1431
+ ...worker,
1432
+ record: closedRecord,
1433
+ session: undefined,
1434
+ observationRelease: undefined,
1079
1435
  });
1080
- this.maybeCompleteRun(current.runId);
1081
- const affectedOwners = this.rememberTerminalWorker(current.id);
1082
- affectedOwners.add(current.ownerSessionId);
1083
- this.emitStateForOwners(affectedOwners);
1436
+ rememberTerminalWorker(draft, worker.record.id);
1437
+ return actions;
1084
1438
  }
1085
1439
 
1086
- private closeReadyInteractiveWorkersForShutdown(): void {
1087
- const readyInteractiveWorkers = [...this.workers.values()].filter(
1088
- (worker) => worker.lifecycle === "interactive" && worker.status === "ready",
1089
- );
1090
- for (const worker of readyInteractiveWorkers) {
1091
- this.closeReadyInteractiveWorker(worker);
1440
+ private releaseWorkerResources(worker: RuntimeWorker): PostCommitAction[] {
1441
+ const actions: PostCommitAction[] = [];
1442
+ if (worker.observationRelease) {
1443
+ actions.push(runAction(() => safelyCall(worker.observationRelease)));
1444
+ }
1445
+ const session = worker.session;
1446
+ if (session) {
1447
+ actions.push(runAction(() => {
1448
+ this.launchCleanup(
1449
+ Effect.suspend(() => session.dispose()).pipe(
1450
+ Effect.catchCause(() => Effect.void),
1451
+ Effect.uninterruptible,
1452
+ ),
1453
+ );
1454
+ }));
1092
1455
  }
1456
+ return actions;
1457
+ }
1458
+
1459
+ private launchCleanup(cleanup: Effect.Effect<void>): void {
1460
+ const fiber = Effect.runFork(cleanup);
1461
+ FiberSet.addUnsafe(this.cleanups, fiber);
1093
1462
  }
1094
1463
 
1095
- private subscribeEntryObservability(
1464
+ private preflightOpen(
1465
+ operation: OrchestrationOperation,
1466
+ ): Decision<void> {
1467
+ return this.transact((draft) => ({
1468
+ value: openDecision(draft, operation),
1469
+ }));
1470
+ }
1471
+
1472
+ private preflightReadyInteractive(
1473
+ ownerSessionId: string,
1096
1474
  workerId: WorkerId,
1097
- entry: RuntimeEntry,
1098
- session: WorkerSessionHandle,
1099
- generation: number,
1100
- ): void {
1101
- this.unsubscribeEntryObservability(entry);
1102
- entry.unsubscribeUsage = session.subscribeUsage((usage) => {
1103
- const latest = this.workers.get(workerId);
1104
- if (!latest || entry.session !== session || entry.generation !== generation) return;
1105
- if (latest.status !== "starting" && latest.status !== "running") return;
1106
- this.workers.set(workerId, { ...latest, usage: copyUsage(usage) });
1107
- this.emitState(latest.ownerSessionId);
1108
- });
1109
- entry.unsubscribeActivity = session.subscribeActivity((activity) => {
1110
- const latest = this.workers.get(workerId);
1111
- if (!latest || entry.session !== session || entry.generation !== generation) return;
1112
- if (latest.status !== "starting" && latest.status !== "running") return;
1113
- if (latest.activity === activity) return;
1114
- this.workers.set(workerId, { ...latest, activity });
1115
- this.emitState(latest.ownerSessionId);
1116
- });
1117
- entry.unsubscribeMessageDirection = session.subscribeMessageDirection((messageDirection) => {
1118
- const latest = this.workers.get(workerId);
1119
- if (!latest || entry.session !== session || entry.generation !== generation) return;
1120
- if (latest.status !== "starting" && latest.status !== "running") return;
1121
- if (latest.messageDirection === messageDirection) return;
1122
- this.workers.set(workerId, { ...latest, messageDirection });
1123
- this.emitState(latest.ownerSessionId);
1475
+ ): Decision<void> {
1476
+ return this.transact((draft) => {
1477
+ const ready = readyInteractiveDecision(draft, ownerSessionId, workerId);
1478
+ return {
1479
+ value: ready._tag === "accepted"
1480
+ ? accepted(undefined)
1481
+ : ready,
1482
+ };
1124
1483
  });
1125
1484
  }
1126
1485
 
1127
- private emitState(ownerSessionId: string): void {
1128
- for (const listener of [...this.stateListeners]) {
1129
- try {
1130
- listener(ownerSessionId);
1131
- } catch {
1132
- // One subscriber cannot prevent runtime mutations or other notifications.
1133
- }
1134
- }
1486
+ private current(): RuntimeState {
1487
+ return this.state;
1135
1488
  }
1136
1489
 
1137
- private emitStateForOwners(ownerSessionIds: ReadonlySet<string>): void {
1138
- for (const ownerSessionId of ownerSessionIds) this.emitState(ownerSessionId);
1490
+ private transact<A>(
1491
+ reducer: (draft: RuntimeState) => TransactionMutation<A>,
1492
+ ): A {
1493
+ const draft = makeDraft(this.state);
1494
+ const mutation = reducer(draft);
1495
+ // Commit before action factories capture the snapshot and callbacks can run.
1496
+ this.state = Object.freeze(draft);
1497
+ this.enqueueActions((mutation.actions ?? []).map((action) => action(
1498
+ this.state,
1499
+ this.settlementListeners,
1500
+ this.stateListeners,
1501
+ )));
1502
+ return mutation.value;
1139
1503
  }
1140
1504
 
1141
- private ownedWorker(ownerSessionId: string, workerId: WorkerId): WorkerRecord {
1142
- const worker = this.workers.get(workerId);
1143
- if (!worker || worker.ownerSessionId !== ownerSessionId) {
1144
- throw new Error("Worker is not owned by this session");
1505
+ private enqueueActions(actions: readonly CommittedAction[]): void {
1506
+ // Reentrant actions queue behind this drain; drain all before rethrowing the first failure.
1507
+ this.actionQueue.push(...actions);
1508
+ if (this.drainingActions) return;
1509
+
1510
+ this.drainingActions = true;
1511
+ let hasFailure = false;
1512
+ let firstFailure: unknown;
1513
+ try {
1514
+ while (this.actionQueue.length > 0) {
1515
+ const action = this.actionQueue.shift();
1516
+ if (!action) continue;
1517
+ try {
1518
+ action();
1519
+ } catch (error) {
1520
+ if (!hasFailure) firstFailure = error;
1521
+ hasFailure = true;
1522
+ }
1523
+ }
1524
+ } finally {
1525
+ this.drainingActions = false;
1145
1526
  }
1146
- return worker;
1527
+ if (hasFailure) throw firstFailure;
1147
1528
  }
1529
+ }
1530
+
1531
+ export function orchestrationLayer(
1532
+ options: OrchestrationLayerOptions = {},
1533
+ ): Layer.Layer<Orchestration, never, ChildSessions> {
1534
+ return Layer.effect(
1535
+ Orchestration,
1536
+ Effect.gen(function* () {
1537
+ const childSessions = yield* ChildSessions;
1538
+ // Scope finalizers run in reverse acquisition order. Keep cleanup open while
1539
+ // generation and cancellation interruption settle workers and enqueue disposal.
1540
+ const cleanups = yield* FiberSet.make<void, never>();
1541
+ const cancellations = yield* FiberSet.make<void, never>();
1542
+ const runCancellation = yield* FiberSet.runtime(cancellations)<never>();
1543
+ const generations = yield* FiberMap.make<WorkerId, void, never>();
1544
+ const runGeneration = yield* FiberMap.runtime(generations)<never>();
1545
+ const clock = yield* Clock.Clock;
1546
+ const shutdownCompletion = yield* Deferred.make<void>();
1547
+ return Orchestration.of(new StatefulOrchestration(
1548
+ childSessions,
1549
+ generations,
1550
+ runGeneration,
1551
+ cancellations,
1552
+ runCancellation,
1553
+ cleanups,
1554
+ clock,
1555
+ options.idFactories ?? createRandomIdFactories(),
1556
+ shutdownCompletion,
1557
+ ));
1558
+ }),
1559
+ );
1560
+ }
1561
+
1562
+ function initialState(): RuntimeState {
1563
+ return Object.freeze({
1564
+ workers: new Map<WorkerId, RuntimeWorker>(),
1565
+ runs: new Map<RunId, RuntimeRun>(),
1566
+ terminalWorkerOrder: [],
1567
+ completedRunOrder: [],
1568
+ settlementSequence: 0,
1569
+ lifecycle: "open",
1570
+ });
1571
+ }
1148
1572
 
1149
- private unsubscribeEntryObservability(entry: RuntimeEntry): void {
1150
- safelyCall(entry.unsubscribeUsage);
1151
- entry.unsubscribeUsage = undefined;
1152
- safelyCall(entry.unsubscribeActivity);
1153
- entry.unsubscribeActivity = undefined;
1154
- safelyCall(entry.unsubscribeMessageDirection);
1155
- entry.unsubscribeMessageDirection = undefined;
1573
+ function makeDraft(state: RuntimeState): RuntimeState {
1574
+ return {
1575
+ ...state,
1576
+ workers: new Map(state.workers),
1577
+ runs: new Map(state.runs),
1578
+ terminalWorkerOrder: [...state.terminalWorkerOrder],
1579
+ completedRunOrder: [...state.completedRunOrder],
1580
+ };
1581
+ }
1582
+
1583
+ function completeRun(
1584
+ state: RuntimeState,
1585
+ workerId: WorkerId,
1586
+ ): PostCommitAction[] {
1587
+ const worker = state.workers.get(workerId);
1588
+ if (!worker) return [];
1589
+ const run = state.runs.get(worker.record.runId);
1590
+ if (!run || !isRunningRuntimeRun(run) || !isCompletedRunWorker(worker.record)) {
1591
+ return [];
1156
1592
  }
1157
1593
 
1158
- private disposeEntrySession(entry: RuntimeEntry): void {
1159
- this.unsubscribeEntryObservability(entry);
1160
- if (entry.session) this.disposeSession(entry.session);
1161
- entry.session = undefined;
1594
+ const completed = freezeCompletedRun(run.record, worker.record);
1595
+ state.runs.set(run.record.id, { ...run.record, state: "complete" });
1596
+ state.completedRunOrder.push(run.record.id);
1597
+ return [runAction(() => {
1598
+ Deferred.doneUnsafe(run.completion, Effect.succeed(completed));
1599
+ })];
1600
+ }
1601
+
1602
+ function makeSettlement(
1603
+ draft: RuntimeState,
1604
+ worker: RuntimeWorker,
1605
+ settledAt: number,
1606
+ failureStage: SettlementFailureStage | undefined,
1607
+ ): PostCommitAction | undefined {
1608
+ const run = draft.runs.get(worker.record.runId);
1609
+ if (!run || !isCompletedRunWorker(worker.record)) {
1610
+ return undefined;
1162
1611
  }
1612
+ const runRecord = runtimeRunRecord(run);
1613
+ const sequence = draft.settlementSequence + 1;
1614
+ draft.settlementSequence = sequence;
1615
+ const settlement: WorkerSettlement = Object.freeze({
1616
+ eventId: `${sequence}:${runRecord.id}:${worker.record.id}:${worker.generation}`,
1617
+ sequence,
1618
+ ownerSessionId: worker.record.ownerSessionId,
1619
+ runId: runRecord.id,
1620
+ workerId: worker.record.id,
1621
+ generation: worker.generation,
1622
+ mode: runRecord.mode,
1623
+ worker: worker.record.worker,
1624
+ title: worker.record.title,
1625
+ lifecycle: worker.record.lifecycle,
1626
+ status: worker.record.status,
1627
+ outcome: Object.freeze(copyOutcome(worker.record.outcome)),
1628
+ ...(failureStage ? { failureStage } : {}),
1629
+ usage: Object.freeze(copyUsage(worker.record.usage)),
1630
+ startedAt: worker.record.startedAt,
1631
+ settledAt,
1632
+ ...(runRecord.synthesisGroupId && runRecord.synthesisGroupSize
1633
+ ? {
1634
+ synthesisGroupId: runRecord.synthesisGroupId,
1635
+ synthesisGroupSize: runRecord.synthesisGroupSize,
1636
+ }
1637
+ : {}),
1638
+ ...(worker.record.sessionFile !== undefined
1639
+ ? { sessionFile: worker.record.sessionFile }
1640
+ : {}),
1641
+ });
1642
+ return publishSettlement(
1643
+ settlement,
1644
+ isRunningRuntimeRun(run) ? run.settlementListener : undefined,
1645
+ );
1646
+ }
1163
1647
 
1164
- private disposeSession(session: WorkerSessionHandle): void {
1165
- if (this.disposedSessions.has(session)) return;
1166
- this.disposedSessions.add(session);
1648
+ function stateActionsAfterPrune(
1649
+ draft: RuntimeState,
1650
+ ownerSessionId: string,
1651
+ ): PostCommitAction[] {
1652
+ const owners = pruneHistory(draft);
1653
+ owners.add(ownerSessionId);
1654
+ return publishOwners(owners);
1655
+ }
1167
1656
 
1168
- try {
1169
- this.trackCleanup(session.dispose());
1170
- } catch {
1171
- // Cleanup is best-effort and cannot leave lifecycle state unsettled.
1172
- }
1657
+ function pruneHistory(draft: RuntimeState): Set<string> {
1658
+ const owners = new Set<string>();
1659
+ while (draft.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
1660
+ const runId = draft.completedRunOrder.shift();
1661
+ if (!runId) break;
1662
+ const run = draft.runs.get(runId);
1663
+ if (run) owners.add(runtimeRunRecord(run).ownerSessionId);
1664
+ draft.runs.delete(runId);
1173
1665
  }
1666
+ while (draft.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
1667
+ const index = draft.terminalWorkerOrder.findIndex((workerId) => {
1668
+ const worker = draft.workers.get(workerId);
1669
+ return !worker || isTerminalWorkerStatus(worker.record.status);
1670
+ });
1671
+ if (index < 0) break;
1672
+ const removed = draft.terminalWorkerOrder.splice(index, 1)[0];
1673
+ if (!removed) break;
1674
+ const worker = draft.workers.get(removed);
1675
+ if (worker) owners.add(worker.record.ownerSessionId);
1676
+ draft.workers.delete(removed);
1677
+ }
1678
+ return owners;
1679
+ }
1174
1680
 
1175
- private trackCleanup(operation: Promise<void>): Promise<void> {
1176
- this.cleanupOperations.add(operation);
1177
- const forget = () => this.cleanupOperations.delete(operation);
1178
- void operation.then(forget, forget);
1179
- return operation;
1681
+ function rememberTerminalWorker(
1682
+ draft: RuntimeState,
1683
+ workerId: WorkerId,
1684
+ ): void {
1685
+ if (!draft.terminalWorkerOrder.includes(workerId)) {
1686
+ draft.terminalWorkerOrder.push(workerId);
1180
1687
  }
1688
+ }
1689
+
1690
+ function snapshotFor(
1691
+ state: RuntimeState,
1692
+ ownerSessionId: string,
1693
+ ): RuntimeSnapshot {
1694
+ return Object.freeze({
1695
+ runs: Object.freeze(
1696
+ [...state.runs.values()]
1697
+ .map(runtimeRunRecord)
1698
+ .filter((run) => run.ownerSessionId === ownerSessionId)
1699
+ .map(copyRunRecord),
1700
+ ),
1701
+ workers: Object.freeze(
1702
+ [...state.workers.values()]
1703
+ .map((worker) => worker.record)
1704
+ .filter((worker) => worker.ownerSessionId === ownerSessionId)
1705
+ .map(copyWorkerRecord),
1706
+ ),
1707
+ });
1708
+ }
1709
+
1710
+ function runtimeRunRecord(run: RuntimeRun): RunRecord {
1711
+ return isRunningRuntimeRun(run) ? run.record : run;
1712
+ }
1181
1713
 
1182
- private async awaitTrackedCleanupBestEffort(): Promise<void> {
1183
- const operations = [...this.cleanupOperations];
1184
- if (operations.length === 0) return;
1185
- await this.waitBestEffort(
1186
- Promise.allSettled(operations).then(() => undefined),
1187
- SHUTDOWN_CLEANUP_GRACE_MS,
1714
+ function isRunningRuntimeRun(run: RuntimeRun): run is RunningRuntimeRun {
1715
+ return "_tag" in run;
1716
+ }
1717
+
1718
+ function makeRunRecord(
1719
+ runId: RunId,
1720
+ workerId: WorkerId,
1721
+ context: OrchestrationContext,
1722
+ mode: RunMode,
1723
+ createdAt: number,
1724
+ ): RunRecord {
1725
+ return {
1726
+ id: runId,
1727
+ ownerSessionId: context.ownerSessionId,
1728
+ workerId,
1729
+ mode,
1730
+ state: "running",
1731
+ createdAt,
1732
+ ...(context.synthesisGroup
1733
+ ? {
1734
+ synthesisGroupId: context.synthesisGroup.id,
1735
+ synthesisGroupSize: context.synthesisGroup.size,
1736
+ }
1737
+ : {}),
1738
+ };
1739
+ }
1740
+
1741
+ function makeWorkerRecord(
1742
+ workerId: WorkerId,
1743
+ runId: RunId,
1744
+ ownerSessionId: string,
1745
+ definition: WorkerDefinition,
1746
+ task: OrchestrateTaskInput,
1747
+ startedAt: number,
1748
+ ): WorkerRecord {
1749
+ return {
1750
+ id: workerId,
1751
+ worker: definition.name,
1752
+ ownerSessionId,
1753
+ runId,
1754
+ title: task.title,
1755
+ instructions: task.instructions,
1756
+ lifecycle: definition.lifecycle,
1757
+ status: "starting",
1758
+ usage: copyUsage(EMPTY_WORKER_USAGE),
1759
+ messageDirection: "to-model",
1760
+ startedAt,
1761
+ };
1762
+ }
1763
+
1764
+ function publishState(ownerSessionId: string): PostCommitAction {
1765
+ return (state, _settlementListeners, stateListeners) => {
1766
+ const snapshot = snapshotFor(state, ownerSessionId);
1767
+ const listeners = [...(stateListeners.get(ownerSessionId) ?? [])];
1768
+ return () => {
1769
+ for (const listener of listeners) safelyNotify(() => listener(snapshot));
1770
+ };
1771
+ };
1772
+ }
1773
+
1774
+ function publishStateTo(
1775
+ ownerSessionId: string,
1776
+ listener: StateListener,
1777
+ ): PostCommitAction {
1778
+ return (state) => {
1779
+ const snapshot = snapshotFor(state, ownerSessionId);
1780
+ return () => safelyNotify(() => listener(snapshot));
1781
+ };
1782
+ }
1783
+
1784
+ function publishSettlement(
1785
+ settlement: WorkerSettlement,
1786
+ localListener: SettlementListener | undefined,
1787
+ ): PostCommitAction {
1788
+ return (_state, settlementListeners) => {
1789
+ const listeners = [...settlementListeners];
1790
+ return () => {
1791
+ if (localListener) safelyNotify(() => localListener(settlement));
1792
+ for (const listener of listeners) safelyNotify(() => listener(settlement));
1793
+ };
1794
+ };
1795
+ }
1796
+
1797
+ function publishOwners(owners: ReadonlySet<string>): PostCommitAction[] {
1798
+ return [...owners].map(publishState);
1799
+ }
1800
+
1801
+ function runAction(run: () => void): PostCommitAction {
1802
+ return () => run;
1803
+ }
1804
+
1805
+ function openDecision(
1806
+ draft: RuntimeState,
1807
+ operation: OrchestrationOperation,
1808
+ ): Decision<void> {
1809
+ return draft.lifecycle === "open"
1810
+ ? accepted(undefined)
1811
+ : rejected(
1812
+ operation,
1813
+ "shutdown",
1814
+ "Orchestrator runtime is shutting down",
1815
+ );
1816
+ }
1817
+
1818
+ function ownedWorkerDecision(
1819
+ draft: RuntimeState,
1820
+ operation: OrchestrationOperation,
1821
+ ownerSessionId: string,
1822
+ workerId: WorkerId,
1823
+ ): Decision<RuntimeWorker> {
1824
+ const worker = draft.workers.get(workerId);
1825
+ return !worker || worker.record.ownerSessionId !== ownerSessionId
1826
+ ? rejected(
1827
+ operation,
1828
+ "ownership",
1829
+ "Worker is not owned by this session",
1830
+ )
1831
+ : accepted(worker);
1832
+ }
1833
+
1834
+ function readyInteractiveDecision(
1835
+ draft: RuntimeState,
1836
+ ownerSessionId: string,
1837
+ workerId: WorkerId,
1838
+ ): Decision<{
1839
+ readonly worker: RuntimeWorker;
1840
+ readonly session: WorkerSessionHandle;
1841
+ }> {
1842
+ const open = openDecision(draft, "sendInteractive");
1843
+ if (open._tag === "rejected") return open;
1844
+ const ownership = ownedWorkerDecision(
1845
+ draft,
1846
+ "sendInteractive",
1847
+ ownerSessionId,
1848
+ workerId,
1849
+ );
1850
+ if (ownership._tag === "rejected") return ownership;
1851
+ const worker = ownership.value;
1852
+ if (
1853
+ worker.record.lifecycle !== "interactive" ||
1854
+ worker.record.status !== "ready" ||
1855
+ !worker.session
1856
+ ) {
1857
+ return rejected(
1858
+ "sendInteractive",
1859
+ "worker-state",
1860
+ "interactive_send requires an owned ready interactive worker",
1188
1861
  );
1189
1862
  }
1863
+ return accepted({ worker, session: worker.session });
1864
+ }
1190
1865
 
1191
- private async waitBestEffort(promise: Promise<unknown>, timeoutMs: number): Promise<void> {
1192
- try {
1193
- await this.bestEffortDeadline.wait(promise, timeoutMs);
1194
- } catch {
1195
- // A deadline implementation cannot prevent lifecycle settlement.
1196
- }
1197
- }
1866
+ function accepted<A>(value: A): Decision<A> {
1867
+ return { _tag: "accepted", value };
1868
+ }
1198
1869
 
1199
- private assertOpen(): void {
1200
- if (this.shuttingDown) throw new Error("Orchestrator runtime is shutting down");
1201
- }
1870
+ function rejected(
1871
+ operation: OrchestrationOperation,
1872
+ reason: OrchestrationRejectionReason,
1873
+ message: string,
1874
+ ): Decision<never> {
1875
+ return {
1876
+ _tag: "rejected",
1877
+ error: actionRejection(operation, reason, message),
1878
+ };
1879
+ }
1880
+
1881
+ function actionRejection(
1882
+ operation: OrchestrationOperation,
1883
+ reason: OrchestrationRejectionReason,
1884
+ message: string,
1885
+ ): OrchestrationActionRejected {
1886
+ return new OrchestrationActionRejected({ operation, reason, message });
1202
1887
  }
1203
1888
 
1204
- export function createOrchestratorRuntime(
1205
- options: OrchestratorRuntimeOptions,
1206
- ): OrchestratorRuntime {
1207
- return new DefaultOrchestratorRuntime(options);
1889
+ function rejectAction(
1890
+ operation: OrchestrationOperation,
1891
+ reason: OrchestrationRejectionReason,
1892
+ message: string,
1893
+ ): Effect.Effect<never, OrchestrationActionRejected> {
1894
+ return Effect.fail(actionRejection(operation, reason, message));
1208
1895
  }
1209
1896
 
1210
- function validateContextOwner(ownerSessionId: string): void {
1211
- if (typeof ownerSessionId !== "string" || ownerSessionId.trim() === "") {
1212
- throw new Error("ownerSessionId must not be blank");
1897
+ function validateContextOwner(
1898
+ operation: OrchestrationOperation,
1899
+ ownerSessionId: string,
1900
+ ): Effect.Effect<void, OrchestrationActionRejected> {
1901
+ return typeof ownerSessionId !== "string" || ownerSessionId.trim() === ""
1902
+ ? rejectAction(
1903
+ operation,
1904
+ "validation",
1905
+ "ownerSessionId must not be blank",
1906
+ )
1907
+ : Effect.void;
1908
+ }
1909
+
1910
+ function validateWorkerId(
1911
+ operation: OrchestrationOperation,
1912
+ workerId: string,
1913
+ ): Effect.Effect<WorkerId, OrchestrationActionRejected> {
1914
+ if (typeof workerId !== "string" || workerId.trim() === "") {
1915
+ return rejectAction(
1916
+ operation,
1917
+ "validation",
1918
+ "worker_id must not be blank",
1919
+ );
1213
1920
  }
1921
+ return Schema.decodeUnknownEffect(WorkerId)(workerId).pipe(
1922
+ Effect.mapError(() => actionRejection(
1923
+ operation,
1924
+ "validation",
1925
+ "worker_id must use the canonical worker- prefix",
1926
+ )),
1927
+ );
1214
1928
  }
1215
1929
 
1216
- function validateMode(mode: RunMode): void {
1217
- if (mode !== "async" && mode !== "inline") throw new Error("Invalid orchestration mode");
1930
+ function validateMode(
1931
+ operation: OrchestrationOperation,
1932
+ mode: RunMode,
1933
+ ): Effect.Effect<void, OrchestrationActionRejected> {
1934
+ return mode !== "async" && mode !== "inline"
1935
+ ? rejectAction(operation, "validation", "Invalid orchestration mode")
1936
+ : Effect.void;
1218
1937
  }
1219
1938
 
1220
- function validateText(name: string, value: string, maximumLength: number): void {
1939
+ function validateText(
1940
+ operation: OrchestrationOperation,
1941
+ name: string,
1942
+ value: string,
1943
+ maximumLength: number,
1944
+ ): Effect.Effect<void, OrchestrationActionRejected> {
1221
1945
  if (typeof value !== "string" || value.trim() === "") {
1222
- throw new Error(`${name} must not be blank`);
1223
- }
1224
- if (value.length > maximumLength) {
1225
- throw new Error(`${name} must be at most ${maximumLength} characters`);
1946
+ return rejectAction(operation, "validation", `${name} must not be blank`);
1226
1947
  }
1948
+ return value.length > maximumLength
1949
+ ? rejectAction(
1950
+ operation,
1951
+ "validation",
1952
+ `${name} must be at most ${maximumLength} characters`,
1953
+ )
1954
+ : Effect.void;
1227
1955
  }
1228
1956
 
1229
- function describeError(error: unknown, fallback: string): string {
1230
- if (error instanceof Error && error.message !== "") return error.message;
1231
- if (typeof error === "string" && error !== "") return error;
1232
- return fallback;
1957
+ type ValidatedAbortTarget =
1958
+ | {
1959
+ readonly _tag: "ids";
1960
+ readonly workerIds: readonly WorkerId[];
1961
+ }
1962
+ | {
1963
+ readonly _tag: "all";
1964
+ };
1965
+
1966
+ function validateAbortTarget(
1967
+ target: AbortTarget,
1968
+ ): Effect.Effect<ValidatedAbortTarget, OrchestrationActionRejected> {
1969
+ return Effect.gen(function* () {
1970
+ if (!target || typeof target !== "object") {
1971
+ return yield* rejectAction("abort", "target", "Invalid abort target");
1972
+ }
1973
+ const selected = [target.workerIds !== undefined, target.all !== undefined]
1974
+ .filter(Boolean).length;
1975
+ if (selected !== 1 || (target.all !== undefined && target.all !== true)) {
1976
+ return yield* rejectAction(
1977
+ "abort",
1978
+ "target",
1979
+ "Abort target must specify exactly one of workerIds or all: true",
1980
+ );
1981
+ }
1982
+ if (target.workerIds === undefined) return { _tag: "all" };
1983
+ if (!Array.isArray(target.workerIds) || target.workerIds.length === 0) {
1984
+ return yield* rejectAction(
1985
+ "abort",
1986
+ "target",
1987
+ "workerIds must contain at least one worker ID",
1988
+ );
1989
+ }
1990
+ const workerIds = yield* Effect.all(
1991
+ [...new Set(target.workerIds)].map((id) => validateWorkerId("abort", id)),
1992
+ );
1993
+ return { _tag: "ids", workerIds };
1994
+ });
1233
1995
  }
1234
1996
 
1235
- function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
1236
- return status === "starting" || status === "running" || status === "stopping";
1997
+ function makeCancellationCandidates(
1998
+ workerIds: readonly WorkerId[],
1999
+ ): Effect.Effect<ReadonlyMap<WorkerId, Deferred.Deferred<void>>> {
2000
+ return Effect.forEach(workerIds, (workerId) => Deferred.make<void>().pipe(
2001
+ Effect.map((completion) => [workerId, completion] as const),
2002
+ )).pipe(Effect.map((entries) => new Map(entries)));
1237
2003
  }
1238
2004
 
1239
- function isSettledWorkerStatus(
1240
- status: WorkerRecord["status"],
1241
- ): status is RunResult["status"] {
1242
- return status === "completed" || status === "ready" || status === "failed" || status === "aborted";
2005
+ function activeWorkerIds(
2006
+ state: RuntimeState,
2007
+ ownerSessionId?: string,
2008
+ ): WorkerId[] {
2009
+ return [...state.workers.values()]
2010
+ .filter((worker) => (
2011
+ (ownerSessionId === undefined ||
2012
+ worker.record.ownerSessionId === ownerSessionId) &&
2013
+ isActiveWorkerStatus(worker.record.status)
2014
+ ))
2015
+ .map((worker) => worker.record.id);
1243
2016
  }
1244
2017
 
1245
- function safelyCall(callback: (() => void) | undefined): void {
1246
- if (!callback) return;
1247
- try {
1248
- callback();
1249
- } catch {
1250
- // Session cleanup is idempotent best-effort and must not strand lifecycle state.
1251
- }
2018
+ function awaitAll(
2019
+ deferreds: readonly Deferred.Deferred<void>[],
2020
+ ): Effect.Effect<void> {
2021
+ return Effect.forEach(
2022
+ deferreds,
2023
+ (deferred) => Deferred.await(deferred),
2024
+ { concurrency: "unbounded", discard: true },
2025
+ );
1252
2026
  }
1253
2027
 
1254
- function throwIfAborted(signal: AbortSignal | undefined): void {
1255
- if (signal?.aborted) throw abortSignalReason(signal);
2028
+ function isRuntimeWorker(
2029
+ worker: RuntimeWorker | undefined,
2030
+ ): worker is RuntimeWorker {
2031
+ return worker !== undefined;
1256
2032
  }
1257
2033
 
1258
- function abortSignalReason(signal: AbortSignal): unknown {
1259
- if ("reason" in signal) return signal.reason;
1260
- return new DOMException("This operation was aborted", "AbortError");
2034
+ function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
2035
+ return status === "starting" || status === "running" || status === "stopping";
1261
2036
  }
1262
2037
 
1263
- function addAll(target: Set<string>, source: ReadonlySet<string>): void {
1264
- for (const value of source) target.add(value);
2038
+ function isCompletedRunWorker(
2039
+ record: WorkerRecord,
2040
+ ): record is WorkerRecord & {
2041
+ readonly status: RunResult["status"];
2042
+ readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
2043
+ readonly settledAt: number;
2044
+ } {
2045
+ return (
2046
+ (record.status === "completed" ||
2047
+ record.status === "ready" ||
2048
+ record.status === "failed" ||
2049
+ record.status === "aborted") &&
2050
+ record.outcome !== undefined &&
2051
+ record.outcome.status !== "closed" &&
2052
+ record.settledAt !== undefined
2053
+ );
1265
2054
  }
1266
2055
 
1267
- function notifySettlementListener(
1268
- listener: SettlementListener | undefined,
1269
- settlement: WorkerSettlement,
1270
- ): void {
1271
- if (!listener) return;
1272
- try {
1273
- listener(settlement);
1274
- } catch {
1275
- // One observer cannot prevent settlement or other observers from being notified.
1276
- }
2056
+ function observationsEqual(
2057
+ previous: WorkerRecord,
2058
+ next: WorkerRecord,
2059
+ ): boolean {
2060
+ return (
2061
+ previous.activity === next.activity &&
2062
+ previous.messageDirection === next.messageDirection &&
2063
+ usageEquals(previous.usage, next.usage)
2064
+ );
1277
2065
  }
1278
2066
 
1279
- function makeRunWaiter(): RunWaiter {
1280
- let complete!: (run: CompletedRun) => void;
1281
- const waiter: RunWaiter = {
1282
- promise: new Promise<CompletedRun>((resolve) => {
1283
- complete = resolve;
1284
- }),
1285
- settled: false,
1286
- resolve(run) {
1287
- if (waiter.settled) return;
1288
- waiter.settled = true;
1289
- waiter.onSettled?.();
1290
- complete(run);
1291
- },
1292
- };
1293
- return waiter;
2067
+ function usageEquals(left: WorkerUsage, right: WorkerUsage): boolean {
2068
+ return (
2069
+ left.input === right.input &&
2070
+ left.output === right.output &&
2071
+ left.cacheRead === right.cacheRead &&
2072
+ left.cacheWrite === right.cacheWrite &&
2073
+ left.cost === right.cost &&
2074
+ left.contextTokens === right.contextTokens &&
2075
+ left.turns === right.turns
2076
+ );
1294
2077
  }
1295
2078
 
1296
2079
  function copyUsage(usage: WorkerUsage): WorkerUsage {
@@ -1317,7 +2100,9 @@ function copyWorkerRecord(worker: WorkerRecord): WorkerRecord {
1317
2100
  return Object.freeze({
1318
2101
  ...worker,
1319
2102
  usage: Object.freeze(copyUsage(worker.usage)),
1320
- ...(worker.outcome ? { outcome: Object.freeze(copyOutcome(worker.outcome)) } : {}),
2103
+ ...(worker.outcome
2104
+ ? { outcome: Object.freeze(copyOutcome(worker.outcome)) }
2105
+ : {}),
1321
2106
  });
1322
2107
  }
1323
2108
 
@@ -1327,21 +2112,18 @@ function freezeAcceptedRun(id: RunId, workerId: WorkerId): AcceptedRun {
1327
2112
 
1328
2113
  function freezeCompletedRun(
1329
2114
  run: RunRecord,
1330
- record: WorkerRecord,
2115
+ record: WorkerRecord & {
2116
+ readonly status: RunResult["status"];
2117
+ readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
2118
+ readonly settledAt: number;
2119
+ },
1331
2120
  ): CompletedRun {
1332
- const outcome = record.outcome;
1333
- if (!outcome || outcome.status === "closed") {
1334
- throw new Error("Completed run requires a worker response outcome");
1335
- }
1336
- if (record.settledAt === undefined) {
1337
- throw new Error("Completed run requires a settlement timestamp");
1338
- }
1339
2121
  const result: RunResult = Object.freeze({
1340
2122
  workerId: record.id,
1341
2123
  worker: record.worker,
1342
2124
  title: record.title,
1343
- status: record.status as RunResult["status"],
1344
- outcome: Object.freeze(copyOutcome(outcome)),
2125
+ status: record.status,
2126
+ outcome: Object.freeze(copyOutcome(record.outcome)),
1345
2127
  usage: Object.freeze(copyUsage(record.usage)),
1346
2128
  startedAt: record.startedAt,
1347
2129
  settledAt: record.settledAt,
@@ -1354,3 +2136,27 @@ function freezeCompletedRun(
1354
2136
  result,
1355
2137
  });
1356
2138
  }
2139
+
2140
+ function describeError(error: unknown, fallback: string): string {
2141
+ if (error instanceof Error && error.message !== "") return error.message;
2142
+ if (typeof error === "string" && error !== "") return error;
2143
+ return fallback;
2144
+ }
2145
+
2146
+ function safelyCall(callback: (() => void) | undefined): void {
2147
+ if (callback) safelyNotify(callback);
2148
+ }
2149
+
2150
+ function safelyNotify(callback: () => void): void {
2151
+ try {
2152
+ callback();
2153
+ } catch {
2154
+ // Observers and best-effort cleanup cannot block committed state transitions.
2155
+ }
2156
+ }
2157
+
2158
+ function addAll(target: Set<string>, source: ReadonlySet<string>): void {
2159
+ for (const value of source) target.add(value);
2160
+ }
2161
+
2162
+ function noOp(): void {}