@zachwill/pi-orchestrate 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extension/catalog/definition.ts +89 -0
- package/extension/{catalog.ts → catalog/discovery.ts} +4 -4
- package/extension/index.ts +22 -51
- package/extension/orchestration/admission.ts +297 -0
- package/extension/{domain.ts → orchestration/model.ts} +27 -89
- package/extension/{runtime.ts → orchestration/service.ts} +222 -523
- package/extension/{worker-settlement.ts → orchestration/settlement.ts} +67 -21
- package/extension/package-root.ts +3 -0
- package/extension/{contract.ts → parent/contract.ts} +75 -9
- package/extension/{delivery.ts → parent/delivery.ts} +19 -16
- package/extension/parent/dispatch-policy.ts +49 -0
- package/extension/{host.ts → parent/process-host.ts} +30 -31
- package/extension/{presentation.ts → pi/presentation.ts} +41 -38
- package/extension/pi/tool-renderer.ts +422 -0
- package/extension/pi/tools.ts +470 -0
- package/extension/worker/child-sessions.ts +291 -0
- package/extension/{worker-session.ts → worker/session.ts} +23 -287
- package/package.json +2 -1
- package/extension/tools.ts +0 -771
- /package/extension/{tui.ts → pi/tui.ts} +0 -0
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
|
-
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
3
1
|
import {
|
|
4
2
|
Cause,
|
|
5
3
|
Clock,
|
|
@@ -11,71 +9,75 @@ import {
|
|
|
11
9
|
FiberMap,
|
|
12
10
|
FiberSet,
|
|
13
11
|
Layer,
|
|
14
|
-
Schema,
|
|
15
12
|
} from "effect";
|
|
13
|
+
import type { WorkerDefinition } from "../catalog/definition.js";
|
|
16
14
|
import {
|
|
17
15
|
CANCELLATION_GRACE_MS,
|
|
18
16
|
EMPTY_WORKER_USAGE,
|
|
19
17
|
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
20
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
21
|
-
OrchestrateTaskInput,
|
|
22
18
|
RunId,
|
|
23
19
|
WorkerId,
|
|
24
20
|
createRandomIdFactories,
|
|
25
|
-
findWorkerByName,
|
|
26
21
|
isTerminalWorkerStatus,
|
|
27
22
|
transitionWorkerStatus,
|
|
28
23
|
type OrchestrateIdFactories,
|
|
24
|
+
type OrchestrateTaskInput,
|
|
29
25
|
type RunMode,
|
|
30
26
|
type RunRecord,
|
|
31
|
-
type
|
|
32
|
-
type
|
|
27
|
+
type SettledWorkerOutcome,
|
|
28
|
+
type SettledWorkerRecord,
|
|
29
|
+
type SettledWorkerStatus,
|
|
33
30
|
type WorkerOutcome,
|
|
34
31
|
type WorkerRecord,
|
|
35
32
|
type WorkerUsage,
|
|
36
|
-
} from "./
|
|
33
|
+
} from "./model.js";
|
|
34
|
+
import {
|
|
35
|
+
OrchestrationActionRejected,
|
|
36
|
+
accepted,
|
|
37
|
+
rejected,
|
|
38
|
+
validateAbortTarget,
|
|
39
|
+
validateContextOwner,
|
|
40
|
+
validateMode,
|
|
41
|
+
validateOrchestrateRequest,
|
|
42
|
+
validateText,
|
|
43
|
+
validateWorkerId,
|
|
44
|
+
type AbortTarget,
|
|
45
|
+
type Decision,
|
|
46
|
+
type OrchestrationContext,
|
|
47
|
+
type OrchestrationOperation,
|
|
48
|
+
type ValidatedAbortTarget,
|
|
49
|
+
} from "./admission.js";
|
|
50
|
+
import {
|
|
51
|
+
createWorkerSettlement,
|
|
52
|
+
type SettlementFailureStage,
|
|
53
|
+
type WorkerSettlement,
|
|
54
|
+
} from "./settlement.js";
|
|
37
55
|
import {
|
|
38
56
|
ChildSessions,
|
|
39
57
|
type ChildSessionsService,
|
|
40
|
-
|
|
41
|
-
type WorkerSessionHandle,
|
|
42
|
-
type WorkerSessionObservation,
|
|
43
|
-
} from "./worker-session.js";
|
|
58
|
+
} from "../worker/child-sessions.js";
|
|
44
59
|
import type {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
60
|
+
WorkerSessionAbortError,
|
|
61
|
+
WorkerSessionHandle,
|
|
62
|
+
WorkerSessionObservation,
|
|
63
|
+
} from "../worker/session.js";
|
|
48
64
|
|
|
49
65
|
export const MAX_TERMINAL_WORKER_HISTORY = 100;
|
|
50
66
|
export const MAX_COMPLETED_RUN_HISTORY = 100;
|
|
51
67
|
export const SHUTDOWN_CLEANUP_GRACE_MS = CANCELLATION_GRACE_MS;
|
|
52
68
|
|
|
53
|
-
export interface OrchestrationContext {
|
|
54
|
-
readonly ownerSessionId: string;
|
|
55
|
-
readonly cwd: string;
|
|
56
|
-
readonly agentDir: string;
|
|
57
|
-
readonly parentSessionFile: string | undefined;
|
|
58
|
-
readonly projectTrusted: boolean;
|
|
59
|
-
readonly catalog: WorkerCatalog;
|
|
60
|
-
readonly parentModel?: Model<Api>;
|
|
61
|
-
readonly modelRegistry: ModelRegistry;
|
|
62
|
-
readonly synthesisGroup?: {
|
|
63
|
-
readonly id: string;
|
|
64
|
-
readonly size: number;
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
|
|
68
69
|
export interface AcceptedRun {
|
|
69
70
|
readonly id: RunId;
|
|
70
71
|
readonly workerId: WorkerId;
|
|
71
72
|
}
|
|
72
73
|
|
|
73
|
-
|
|
74
|
+
/** One settled worker generation, as returned to the owner that requested it. */
|
|
75
|
+
export interface WorkerRunResult {
|
|
74
76
|
readonly workerId: WorkerId;
|
|
75
77
|
readonly worker: string;
|
|
76
78
|
readonly title: string;
|
|
77
|
-
readonly status:
|
|
78
|
-
readonly outcome:
|
|
79
|
+
readonly status: SettledWorkerStatus;
|
|
80
|
+
readonly outcome: SettledWorkerOutcome;
|
|
79
81
|
readonly usage: WorkerUsage;
|
|
80
82
|
readonly startedAt: number;
|
|
81
83
|
readonly settledAt: number;
|
|
@@ -86,56 +88,23 @@ export interface CompletedRun {
|
|
|
86
88
|
readonly id: RunId;
|
|
87
89
|
readonly ownerSessionId: string;
|
|
88
90
|
readonly mode: RunMode;
|
|
89
|
-
readonly result:
|
|
91
|
+
readonly result: WorkerRunResult;
|
|
90
92
|
}
|
|
91
93
|
|
|
92
|
-
|
|
94
|
+
/** Runs and workers visible to exactly one owner session. */
|
|
95
|
+
export interface OwnerSnapshot {
|
|
93
96
|
readonly runs: readonly RunRecord[];
|
|
94
97
|
readonly workers: readonly WorkerRecord[];
|
|
95
98
|
}
|
|
96
99
|
|
|
97
100
|
export type SettlementListener = (settlement: WorkerSettlement) => void;
|
|
98
101
|
export type UnsubscribeSettlement = () => void;
|
|
99
|
-
export type StateListener = (snapshot:
|
|
100
|
-
|
|
101
|
-
export interface AbortTarget {
|
|
102
|
-
readonly workerIds?: readonly string[];
|
|
103
|
-
readonly all?: boolean;
|
|
104
|
-
}
|
|
102
|
+
export type StateListener = (snapshot: OwnerSnapshot) => void;
|
|
105
103
|
|
|
106
104
|
export interface OrchestrationLayerOptions {
|
|
107
105
|
readonly idFactories?: OrchestrateIdFactories;
|
|
108
106
|
}
|
|
109
107
|
|
|
110
|
-
const OrchestrationOperation = Schema.Literals([
|
|
111
|
-
"orchestrate",
|
|
112
|
-
"sendInteractive",
|
|
113
|
-
"abort",
|
|
114
|
-
"closeInteractive",
|
|
115
|
-
"snapshot",
|
|
116
|
-
]);
|
|
117
|
-
type OrchestrationOperation = typeof OrchestrationOperation.Type;
|
|
118
|
-
|
|
119
|
-
const OrchestrationRejectionReason = Schema.Literals([
|
|
120
|
-
"shutdown",
|
|
121
|
-
"validation",
|
|
122
|
-
"ownership",
|
|
123
|
-
"target",
|
|
124
|
-
"worker-state",
|
|
125
|
-
"unknown-worker",
|
|
126
|
-
"model-unavailable",
|
|
127
|
-
]);
|
|
128
|
-
type OrchestrationRejectionReason = typeof OrchestrationRejectionReason.Type;
|
|
129
|
-
|
|
130
|
-
export class OrchestrationActionRejected extends Schema.TaggedError<OrchestrationActionRejected>()(
|
|
131
|
-
"Orchestration.ActionRejected",
|
|
132
|
-
{
|
|
133
|
-
operation: OrchestrationOperation,
|
|
134
|
-
reason: OrchestrationRejectionReason,
|
|
135
|
-
message: Schema.String,
|
|
136
|
-
},
|
|
137
|
-
) {}
|
|
138
|
-
|
|
139
108
|
export interface OrchestrationService {
|
|
140
109
|
orchestrate(
|
|
141
110
|
context: OrchestrationContext,
|
|
@@ -186,7 +155,7 @@ export interface OrchestrationService {
|
|
|
186
155
|
) => Effect.Effect<void, OrchestrationActionRejected>;
|
|
187
156
|
readonly snapshot: (
|
|
188
157
|
ownerSessionId: string,
|
|
189
|
-
) => Effect.Effect<
|
|
158
|
+
) => Effect.Effect<OwnerSnapshot, OrchestrationActionRejected>;
|
|
190
159
|
readonly subscribeSettlement: (
|
|
191
160
|
listener: SettlementListener,
|
|
192
161
|
) => UnsubscribeSettlement;
|
|
@@ -201,38 +170,39 @@ export class Orchestration extends Context.Service<Orchestration, OrchestrationS
|
|
|
201
170
|
"@zachwill/pi-orchestrate/Orchestration",
|
|
202
171
|
) {}
|
|
203
172
|
|
|
204
|
-
interface
|
|
173
|
+
interface WorkerEntry {
|
|
205
174
|
readonly record: WorkerRecord;
|
|
206
175
|
readonly context: OrchestrationContext;
|
|
207
176
|
readonly definition: WorkerDefinition;
|
|
208
|
-
|
|
177
|
+
// Worker-local authority fence; stale asynchronous callbacks must revalidate it.
|
|
178
|
+
readonly workerEpoch: number;
|
|
209
179
|
readonly session?: WorkerSessionHandle;
|
|
210
180
|
readonly observationRelease?: () => void;
|
|
211
181
|
readonly cancellation?: Deferred.Deferred<void>;
|
|
212
182
|
}
|
|
213
183
|
|
|
214
|
-
interface
|
|
184
|
+
interface ActiveRunEntry {
|
|
215
185
|
readonly _tag: "running";
|
|
216
186
|
readonly record: RunRecord;
|
|
217
187
|
readonly completion: Deferred.Deferred<CompletedRun>;
|
|
218
188
|
readonly settlementListener?: SettlementListener;
|
|
219
189
|
}
|
|
220
190
|
|
|
221
|
-
type
|
|
222
|
-
type
|
|
191
|
+
type RunEntry = ActiveRunEntry | RunRecord;
|
|
192
|
+
type OrchestrationLifecycle = "open" | "shutting-down" | "shutdown";
|
|
223
193
|
|
|
224
|
-
interface
|
|
225
|
-
workers: Map<WorkerId,
|
|
226
|
-
runs: Map<RunId,
|
|
194
|
+
interface OrchestrationState {
|
|
195
|
+
workers: Map<WorkerId, WorkerEntry>;
|
|
196
|
+
runs: Map<RunId, RunEntry>;
|
|
227
197
|
terminalWorkerOrder: WorkerId[];
|
|
228
198
|
completedRunOrder: RunId[];
|
|
229
199
|
settlementSequence: number;
|
|
230
|
-
lifecycle:
|
|
200
|
+
lifecycle: OrchestrationLifecycle;
|
|
231
201
|
}
|
|
232
202
|
|
|
233
203
|
type CommittedAction = () => void;
|
|
234
204
|
type PostCommitAction = (
|
|
235
|
-
state:
|
|
205
|
+
state: OrchestrationState,
|
|
236
206
|
settlementListeners: ReadonlySet<SettlementListener>,
|
|
237
207
|
stateListeners: ReadonlyMap<string, ReadonlySet<StateListener>>,
|
|
238
208
|
) => CommittedAction;
|
|
@@ -242,27 +212,17 @@ interface TransactionMutation<A> {
|
|
|
242
212
|
readonly actions?: readonly PostCommitAction[];
|
|
243
213
|
}
|
|
244
214
|
|
|
245
|
-
|
|
246
|
-
| {
|
|
247
|
-
readonly _tag: "accepted";
|
|
248
|
-
readonly value: A;
|
|
249
|
-
}
|
|
250
|
-
| {
|
|
251
|
-
readonly _tag: "rejected";
|
|
252
|
-
readonly error: OrchestrationActionRejected;
|
|
253
|
-
};
|
|
254
|
-
|
|
255
|
-
class StatefulOrchestration implements OrchestrationService {
|
|
215
|
+
class OrchestrationEngine implements OrchestrationService {
|
|
256
216
|
private readonly actionQueue: CommittedAction[] = [];
|
|
257
217
|
private readonly settlementListeners = new Set<SettlementListener>();
|
|
258
218
|
private readonly stateListeners = new Map<string, Set<StateListener>>();
|
|
259
219
|
private drainingActions = false;
|
|
260
|
-
private state:
|
|
220
|
+
private state: OrchestrationState;
|
|
261
221
|
|
|
262
222
|
constructor(
|
|
263
223
|
private readonly childSessions: ChildSessionsService,
|
|
264
|
-
private readonly
|
|
265
|
-
private readonly
|
|
224
|
+
private readonly workerWorkflows: FiberMap.FiberMap<WorkerId, void, never>,
|
|
225
|
+
private readonly runWorkerWorkflow: (
|
|
266
226
|
key: WorkerId,
|
|
267
227
|
effect: Effect.Effect<void, never>,
|
|
268
228
|
) => Fiber.Fiber<void, never>,
|
|
@@ -271,9 +231,6 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
271
231
|
effect: Effect.Effect<void, never>,
|
|
272
232
|
) => Fiber.Fiber<void, never>,
|
|
273
233
|
private readonly cleanups: FiberSet.FiberSet<void, never>,
|
|
274
|
-
private readonly runCleanup: (
|
|
275
|
-
effect: Effect.Effect<void, never>,
|
|
276
|
-
) => Fiber.Fiber<void, never>,
|
|
277
234
|
private readonly clock: Clock.Clock,
|
|
278
235
|
private readonly idFactories: OrchestrateIdFactories,
|
|
279
236
|
private readonly shutdownCompletion: Deferred.Deferred<void>,
|
|
@@ -306,9 +263,9 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
306
263
|
onSettlement?: SettlementListener,
|
|
307
264
|
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
308
265
|
return Effect.fn("Orchestration.orchestrate")(function* (
|
|
309
|
-
this:
|
|
266
|
+
this: OrchestrationEngine,
|
|
310
267
|
) {
|
|
311
|
-
const validated = yield*
|
|
268
|
+
const validated = yield* validateOrchestrateRequest(context, task, mode);
|
|
312
269
|
const preflight = this.preflightOpen("orchestrate");
|
|
313
270
|
if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
|
|
314
271
|
// Keep user-supplied ID factories outside the transaction: a reentrant
|
|
@@ -346,7 +303,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
346
303
|
record: workerRecord,
|
|
347
304
|
context,
|
|
348
305
|
definition: validated.definition,
|
|
349
|
-
|
|
306
|
+
workerEpoch: 1,
|
|
350
307
|
});
|
|
351
308
|
|
|
352
309
|
return {
|
|
@@ -395,7 +352,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
395
352
|
onSettlement?: SettlementListener,
|
|
396
353
|
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
397
354
|
return Effect.fn("Orchestration.sendInteractive")(function* (
|
|
398
|
-
this:
|
|
355
|
+
this: OrchestrationEngine,
|
|
399
356
|
) {
|
|
400
357
|
yield* validateContextOwner("sendInteractive", context.ownerSessionId);
|
|
401
358
|
yield* validateMode("sendInteractive", mode);
|
|
@@ -429,34 +386,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
429
386
|
};
|
|
430
387
|
|
|
431
388
|
const admission = this.transact((draft) => {
|
|
432
|
-
const
|
|
433
|
-
if (open._tag === "rejected") return { value: open };
|
|
434
|
-
const ownership = ownedWorkerDecision(
|
|
389
|
+
const ready = readyInteractiveDecision(
|
|
435
390
|
draft,
|
|
436
|
-
"sendInteractive",
|
|
437
391
|
context.ownerSessionId,
|
|
438
392
|
validatedWorkerId,
|
|
439
393
|
);
|
|
440
|
-
if (
|
|
441
|
-
|
|
442
|
-
const worker = ownership.value;
|
|
443
|
-
if (
|
|
444
|
-
worker.record.lifecycle !== "interactive" ||
|
|
445
|
-
worker.record.status !== "ready" ||
|
|
446
|
-
!worker.session
|
|
447
|
-
) {
|
|
448
|
-
return {
|
|
449
|
-
value: rejected(
|
|
450
|
-
"sendInteractive",
|
|
451
|
-
"worker-state",
|
|
452
|
-
"interactive_send requires an owned ready interactive worker",
|
|
453
|
-
),
|
|
454
|
-
};
|
|
455
|
-
}
|
|
394
|
+
if (ready._tag === "rejected") return { value: ready };
|
|
456
395
|
if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
|
|
457
396
|
|
|
458
|
-
const
|
|
459
|
-
const
|
|
397
|
+
const { worker, session } = ready.value;
|
|
398
|
+
const workerEpoch = worker.workerEpoch + 1;
|
|
460
399
|
const runningRecord: WorkerRecord = {
|
|
461
400
|
...transitionWorkerStatus(worker.record, "running"),
|
|
462
401
|
runId,
|
|
@@ -475,7 +414,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
475
414
|
draft.workers.set(validatedWorkerId, {
|
|
476
415
|
...worker,
|
|
477
416
|
record: runningRecord,
|
|
478
|
-
|
|
417
|
+
workerEpoch,
|
|
479
418
|
observationRelease: undefined,
|
|
480
419
|
cancellation: undefined,
|
|
481
420
|
});
|
|
@@ -485,13 +424,13 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
485
424
|
actions: [
|
|
486
425
|
runAction(() => {
|
|
487
426
|
safelyCall(worker.observationRelease);
|
|
488
|
-
this.subscribeObservation(validatedWorkerId,
|
|
427
|
+
this.subscribeObservation(validatedWorkerId, workerEpoch, session);
|
|
489
428
|
}),
|
|
490
429
|
publishState(context.ownerSessionId),
|
|
491
430
|
runAction(() => {
|
|
492
431
|
this.launchPrompt(
|
|
493
432
|
validatedWorkerId,
|
|
494
|
-
|
|
433
|
+
workerEpoch,
|
|
495
434
|
session,
|
|
496
435
|
instructions,
|
|
497
436
|
);
|
|
@@ -513,7 +452,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
513
452
|
target: AbortTarget,
|
|
514
453
|
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
515
454
|
return Effect.fn("Orchestration.abort")(function* (
|
|
516
|
-
this:
|
|
455
|
+
this: OrchestrationEngine,
|
|
517
456
|
) {
|
|
518
457
|
yield* validateContextOwner("abort", ownerSessionId);
|
|
519
458
|
const validatedTarget = yield* validateAbortTarget(target);
|
|
@@ -539,7 +478,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
539
478
|
workerId: string,
|
|
540
479
|
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
541
480
|
return Effect.fn("Orchestration.closeInteractive")(function* (
|
|
542
|
-
this:
|
|
481
|
+
this: OrchestrationEngine,
|
|
543
482
|
) {
|
|
544
483
|
yield* validateContextOwner("closeInteractive", ownerSessionId);
|
|
545
484
|
const id = yield* validateWorkerId("closeInteractive", workerId);
|
|
@@ -579,7 +518,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
579
518
|
|
|
580
519
|
snapshot(
|
|
581
520
|
ownerSessionId: string,
|
|
582
|
-
): Effect.Effect<
|
|
521
|
+
): Effect.Effect<OwnerSnapshot, OrchestrationActionRejected> {
|
|
583
522
|
return validateContextOwner("snapshot", ownerSessionId).pipe(
|
|
584
523
|
Effect.andThen(Effect.sync(() => snapshotFor(this.current(), ownerSessionId))),
|
|
585
524
|
);
|
|
@@ -644,7 +583,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
644
583
|
this.transact((draft) => {
|
|
645
584
|
draft.lifecycle = "shutdown";
|
|
646
585
|
for (const [runId, run] of draft.runs) {
|
|
647
|
-
if (
|
|
586
|
+
if (isActiveRunEntry(run) && run.settlementListener) {
|
|
648
587
|
draft.runs.set(runId, { ...run, settlementListener: undefined });
|
|
649
588
|
}
|
|
650
589
|
}
|
|
@@ -660,7 +599,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
660
599
|
|
|
661
600
|
private performShutdown(): Effect.Effect<void> {
|
|
662
601
|
return Effect.fn("Orchestration.shutdown")(function* (
|
|
663
|
-
this:
|
|
602
|
+
this: OrchestrationEngine,
|
|
664
603
|
) {
|
|
665
604
|
const now = this.clock.currentTimeMillisUnsafe();
|
|
666
605
|
this.transact((draft) => {
|
|
@@ -692,151 +631,61 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
692
631
|
}).call(this);
|
|
693
632
|
}
|
|
694
633
|
|
|
695
|
-
private
|
|
696
|
-
|
|
697
|
-
task: OrchestrateTaskInput,
|
|
698
|
-
mode: RunMode,
|
|
699
|
-
): Effect.Effect<
|
|
700
|
-
{ definition: WorkerDefinition; task: OrchestrateTaskInput },
|
|
701
|
-
OrchestrationActionRejected
|
|
702
|
-
> {
|
|
703
|
-
return Effect.gen(function* () {
|
|
704
|
-
yield* validateContextOwner("orchestrate", context.ownerSessionId);
|
|
705
|
-
yield* validateMode("orchestrate", mode);
|
|
706
|
-
if (!task || typeof task !== "object" || Array.isArray(task)) {
|
|
707
|
-
return yield* rejectAction(
|
|
708
|
-
"orchestrate",
|
|
709
|
-
"validation",
|
|
710
|
-
"orchestrate requires one task object",
|
|
711
|
-
);
|
|
712
|
-
}
|
|
713
|
-
if (context.synthesisGroup) {
|
|
714
|
-
yield* validateText(
|
|
715
|
-
"orchestrate",
|
|
716
|
-
"synthesis group ID",
|
|
717
|
-
context.synthesisGroup.id,
|
|
718
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
719
|
-
);
|
|
720
|
-
if (mode !== "async") {
|
|
721
|
-
return yield* rejectAction(
|
|
722
|
-
"orchestrate",
|
|
723
|
-
"validation",
|
|
724
|
-
"Sibling synthesis requires an async task",
|
|
725
|
-
);
|
|
726
|
-
}
|
|
727
|
-
if (
|
|
728
|
-
!Number.isSafeInteger(context.synthesisGroup.size) ||
|
|
729
|
-
context.synthesisGroup.size < 2
|
|
730
|
-
) {
|
|
731
|
-
return yield* rejectAction(
|
|
732
|
-
"orchestrate",
|
|
733
|
-
"validation",
|
|
734
|
-
"Synthesis group size must be an integer of at least 2",
|
|
735
|
-
);
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
yield* validateText(
|
|
739
|
-
"orchestrate",
|
|
740
|
-
"worker",
|
|
741
|
-
task.worker,
|
|
742
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
743
|
-
);
|
|
744
|
-
yield* validateText(
|
|
745
|
-
"orchestrate",
|
|
746
|
-
"title",
|
|
747
|
-
task.title,
|
|
748
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
749
|
-
);
|
|
750
|
-
yield* validateText(
|
|
751
|
-
"orchestrate",
|
|
752
|
-
"instructions",
|
|
753
|
-
task.instructions,
|
|
754
|
-
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
755
|
-
);
|
|
756
|
-
const definition = findWorkerByName(context.catalog, task.worker);
|
|
757
|
-
if (!definition) {
|
|
758
|
-
return yield* rejectAction(
|
|
759
|
-
"orchestrate",
|
|
760
|
-
"unknown-worker",
|
|
761
|
-
`Unknown worker: ${task.worker}`,
|
|
762
|
-
);
|
|
763
|
-
}
|
|
764
|
-
const configured = definition.model;
|
|
765
|
-
if (!configured && !context.parentModel) {
|
|
766
|
-
return yield* rejectAction(
|
|
767
|
-
"orchestrate",
|
|
768
|
-
"model-unavailable",
|
|
769
|
-
`Worker "${definition.name}" has no configured model and no parent model is available`,
|
|
770
|
-
);
|
|
771
|
-
}
|
|
772
|
-
if (
|
|
773
|
-
configured &&
|
|
774
|
-
!context.modelRegistry.find(configured.provider, configured.modelId)
|
|
775
|
-
) {
|
|
776
|
-
return yield* rejectAction(
|
|
777
|
-
"orchestrate",
|
|
778
|
-
"model-unavailable",
|
|
779
|
-
`Worker "${definition.name}" configured model "${configured.provider}/${configured.modelId}" was not found`,
|
|
780
|
-
);
|
|
781
|
-
}
|
|
782
|
-
return { definition, task };
|
|
783
|
-
});
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
private launchBootstrap(workerId: WorkerId, generation: number): void {
|
|
787
|
-
this.launchGeneration(
|
|
634
|
+
private launchBootstrap(workerId: WorkerId, workerEpoch: number): void {
|
|
635
|
+
this.launchWorkerWorkflow(
|
|
788
636
|
workerId,
|
|
789
|
-
|
|
790
|
-
this.bootstrapAndPrompt(workerId,
|
|
637
|
+
workerEpoch,
|
|
638
|
+
this.bootstrapAndPrompt(workerId, workerEpoch),
|
|
791
639
|
);
|
|
792
640
|
}
|
|
793
641
|
|
|
794
642
|
private launchPrompt(
|
|
795
643
|
workerId: WorkerId,
|
|
796
|
-
|
|
644
|
+
workerEpoch: number,
|
|
797
645
|
session: WorkerSessionHandle,
|
|
798
646
|
instructions: string,
|
|
799
647
|
): void {
|
|
800
|
-
this.
|
|
648
|
+
this.launchWorkerWorkflow(
|
|
801
649
|
workerId,
|
|
802
|
-
|
|
803
|
-
this.executePrompt(workerId,
|
|
650
|
+
workerEpoch,
|
|
651
|
+
this.executePrompt(workerId, workerEpoch, session, instructions),
|
|
804
652
|
);
|
|
805
653
|
}
|
|
806
654
|
|
|
807
|
-
private
|
|
655
|
+
private launchWorkerWorkflow(
|
|
808
656
|
workerId: WorkerId,
|
|
809
|
-
|
|
657
|
+
workerEpoch: number,
|
|
810
658
|
workflow: Effect.Effect<void, never>,
|
|
811
659
|
): void {
|
|
812
660
|
try {
|
|
813
|
-
const fiber = this.
|
|
661
|
+
const fiber = this.runWorkerWorkflow(
|
|
814
662
|
workerId,
|
|
815
663
|
workflow.pipe(
|
|
816
664
|
Effect.catchCause((cause) => {
|
|
817
665
|
if (Cause.hasInterruptsOnly(cause)) return Effect.void;
|
|
818
666
|
return Effect.sync(() => {
|
|
819
|
-
this.settleWorkflowDefect(workerId,
|
|
667
|
+
this.settleWorkflowDefect(workerId, workerEpoch, Cause.squash(cause));
|
|
820
668
|
});
|
|
821
669
|
}),
|
|
822
670
|
),
|
|
823
671
|
);
|
|
824
672
|
const exit = fiber.pollUnsafe();
|
|
673
|
+
// A closed FiberMap rejects admission with an interrupted sentinel.
|
|
825
674
|
if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
826
|
-
this.
|
|
675
|
+
this.settleWorkflowLaunchFailure(workerId, workerEpoch);
|
|
827
676
|
}
|
|
828
677
|
} catch (error) {
|
|
829
|
-
this.settleWorkflowDefect(workerId,
|
|
678
|
+
this.settleWorkflowDefect(workerId, workerEpoch, error);
|
|
830
679
|
}
|
|
831
680
|
}
|
|
832
681
|
|
|
833
|
-
private
|
|
682
|
+
private settleWorkflowLaunchFailure(
|
|
834
683
|
workerId: WorkerId,
|
|
835
|
-
|
|
684
|
+
workerEpoch: number,
|
|
836
685
|
): void {
|
|
837
686
|
this.settleActiveWorker(
|
|
838
687
|
workerId,
|
|
839
|
-
|
|
688
|
+
workerEpoch,
|
|
840
689
|
["starting", "running"],
|
|
841
690
|
"failed",
|
|
842
691
|
{
|
|
@@ -850,16 +699,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
850
699
|
|
|
851
700
|
private bootstrapAndPrompt(
|
|
852
701
|
workerId: WorkerId,
|
|
853
|
-
|
|
702
|
+
workerEpoch: number,
|
|
854
703
|
): Effect.Effect<void, never> {
|
|
855
704
|
return Effect.gen({ self: this }, function* () {
|
|
856
|
-
const session = yield* this.bootstrap(workerId,
|
|
705
|
+
const session = yield* this.bootstrap(workerId, workerEpoch);
|
|
857
706
|
if (!session) return;
|
|
858
707
|
const worker = this.current().workers.get(workerId);
|
|
859
708
|
if (!worker) return;
|
|
860
709
|
yield* this.executePrompt(
|
|
861
710
|
workerId,
|
|
862
|
-
|
|
711
|
+
workerEpoch,
|
|
863
712
|
session,
|
|
864
713
|
worker.record.instructions,
|
|
865
714
|
);
|
|
@@ -868,11 +717,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
868
717
|
|
|
869
718
|
private bootstrap(
|
|
870
719
|
workerId: WorkerId,
|
|
871
|
-
|
|
720
|
+
workerEpoch: number,
|
|
872
721
|
): Effect.Effect<WorkerSessionHandle | undefined, never> {
|
|
873
722
|
return Effect.suspend(() => {
|
|
874
723
|
const expected = this.current().workers.get(workerId);
|
|
875
|
-
if (!expected || expected.
|
|
724
|
+
if (!expected || expected.workerEpoch !== workerEpoch) {
|
|
876
725
|
return Effect.succeed(undefined);
|
|
877
726
|
}
|
|
878
727
|
return this.childSessions.acquire(
|
|
@@ -885,11 +734,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
885
734
|
parentModel: expected.context.parentModel,
|
|
886
735
|
modelRegistry: expected.context.modelRegistry,
|
|
887
736
|
},
|
|
888
|
-
(session) => this.adoptCreatedSession(workerId,
|
|
737
|
+
(session) => this.adoptCreatedSession(workerId, workerEpoch, session),
|
|
889
738
|
).pipe(
|
|
890
739
|
Effect.match({
|
|
891
740
|
onFailure: (error) => {
|
|
892
|
-
this.settleCreationFailure(workerId,
|
|
741
|
+
this.settleCreationFailure(workerId, workerEpoch, error);
|
|
893
742
|
return undefined;
|
|
894
743
|
},
|
|
895
744
|
onSuccess: (session) => session,
|
|
@@ -900,16 +749,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
900
749
|
|
|
901
750
|
private adoptCreatedSession(
|
|
902
751
|
workerId: WorkerId,
|
|
903
|
-
|
|
752
|
+
workerEpoch: number,
|
|
904
753
|
session: WorkerSessionHandle,
|
|
905
754
|
): WorkerSessionHandle | undefined {
|
|
906
755
|
let release: () => void;
|
|
907
756
|
try {
|
|
908
757
|
release = session.subscribeObservation((observation) => {
|
|
909
|
-
this.updateObservation(workerId,
|
|
758
|
+
this.updateObservation(workerId, workerEpoch, session, observation);
|
|
910
759
|
});
|
|
911
760
|
} catch (error) {
|
|
912
|
-
this.settleCreationFailure(workerId,
|
|
761
|
+
this.settleCreationFailure(workerId, workerEpoch, error);
|
|
913
762
|
return undefined;
|
|
914
763
|
}
|
|
915
764
|
|
|
@@ -918,7 +767,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
918
767
|
if (
|
|
919
768
|
draft.lifecycle !== "open" ||
|
|
920
769
|
!worker ||
|
|
921
|
-
worker.
|
|
770
|
+
worker.workerEpoch !== workerEpoch ||
|
|
922
771
|
worker.record.status !== "starting"
|
|
923
772
|
) {
|
|
924
773
|
return {
|
|
@@ -945,16 +794,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
945
794
|
|
|
946
795
|
private subscribeObservation(
|
|
947
796
|
workerId: WorkerId,
|
|
948
|
-
|
|
797
|
+
workerEpoch: number,
|
|
949
798
|
session: WorkerSessionHandle,
|
|
950
799
|
): void {
|
|
951
800
|
let release: () => void;
|
|
952
801
|
try {
|
|
953
802
|
release = session.subscribeObservation((observation) => {
|
|
954
|
-
this.updateObservation(workerId,
|
|
803
|
+
this.updateObservation(workerId, workerEpoch, session, observation);
|
|
955
804
|
});
|
|
956
805
|
} catch (error) {
|
|
957
|
-
this.settleWorkflowDefect(workerId,
|
|
806
|
+
this.settleWorkflowDefect(workerId, workerEpoch, error);
|
|
958
807
|
return;
|
|
959
808
|
}
|
|
960
809
|
|
|
@@ -962,7 +811,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
962
811
|
const worker = draft.workers.get(workerId);
|
|
963
812
|
if (
|
|
964
813
|
!worker ||
|
|
965
|
-
worker.
|
|
814
|
+
worker.workerEpoch !== workerEpoch ||
|
|
966
815
|
worker.session !== session ||
|
|
967
816
|
worker.record.status !== "running"
|
|
968
817
|
) {
|
|
@@ -978,7 +827,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
978
827
|
|
|
979
828
|
private executePrompt(
|
|
980
829
|
workerId: WorkerId,
|
|
981
|
-
|
|
830
|
+
workerEpoch: number,
|
|
982
831
|
session: WorkerSessionHandle,
|
|
983
832
|
instructions: string,
|
|
984
833
|
): Effect.Effect<void, never> {
|
|
@@ -987,7 +836,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
987
836
|
if (
|
|
988
837
|
!worker ||
|
|
989
838
|
worker.record.status !== "running" ||
|
|
990
|
-
worker.
|
|
839
|
+
worker.workerEpoch !== workerEpoch ||
|
|
991
840
|
worker.session !== session
|
|
992
841
|
) {
|
|
993
842
|
return Effect.void;
|
|
@@ -1001,7 +850,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1001
850
|
onSuccess: (outcome) => outcome,
|
|
1002
851
|
}),
|
|
1003
852
|
Effect.tap((outcome) => Effect.sync(() => {
|
|
1004
|
-
this.settleOutcome(workerId,
|
|
853
|
+
this.settleOutcome(workerId, workerEpoch, session, outcome);
|
|
1005
854
|
})),
|
|
1006
855
|
Effect.asVoid,
|
|
1007
856
|
);
|
|
@@ -1010,12 +859,12 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1010
859
|
|
|
1011
860
|
private settleCreationFailure(
|
|
1012
861
|
workerId: WorkerId,
|
|
1013
|
-
|
|
862
|
+
workerEpoch: number,
|
|
1014
863
|
error: unknown,
|
|
1015
864
|
): void {
|
|
1016
865
|
this.settleActiveWorker(
|
|
1017
866
|
workerId,
|
|
1018
|
-
|
|
867
|
+
workerEpoch,
|
|
1019
868
|
["starting"],
|
|
1020
869
|
"failed",
|
|
1021
870
|
{
|
|
@@ -1029,12 +878,12 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1029
878
|
|
|
1030
879
|
private settleWorkflowDefect(
|
|
1031
880
|
workerId: WorkerId,
|
|
1032
|
-
|
|
881
|
+
workerEpoch: number,
|
|
1033
882
|
error: unknown,
|
|
1034
883
|
): void {
|
|
1035
884
|
this.settleActiveWorker(
|
|
1036
885
|
workerId,
|
|
1037
|
-
|
|
886
|
+
workerEpoch,
|
|
1038
887
|
["starting", "running", "stopping"],
|
|
1039
888
|
"failed",
|
|
1040
889
|
{
|
|
@@ -1048,14 +897,14 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1048
897
|
|
|
1049
898
|
private settleOutcome(
|
|
1050
899
|
workerId: WorkerId,
|
|
1051
|
-
|
|
900
|
+
workerEpoch: number,
|
|
1052
901
|
session: WorkerSessionHandle,
|
|
1053
902
|
outcome: WorkerOutcome,
|
|
1054
903
|
): void {
|
|
1055
904
|
const worker = this.current().workers.get(workerId);
|
|
1056
905
|
if (
|
|
1057
906
|
!worker ||
|
|
1058
|
-
worker.
|
|
907
|
+
worker.workerEpoch !== workerEpoch ||
|
|
1059
908
|
worker.session !== session ||
|
|
1060
909
|
worker.record.status !== "running"
|
|
1061
910
|
) {
|
|
@@ -1085,7 +934,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1085
934
|
}
|
|
1086
935
|
this.settleActiveWorker(
|
|
1087
936
|
workerId,
|
|
1088
|
-
|
|
937
|
+
workerEpoch,
|
|
1089
938
|
["running"],
|
|
1090
939
|
status,
|
|
1091
940
|
settledOutcome,
|
|
@@ -1096,9 +945,9 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1096
945
|
|
|
1097
946
|
private settleActiveWorker(
|
|
1098
947
|
workerId: WorkerId,
|
|
1099
|
-
|
|
948
|
+
workerEpoch: number,
|
|
1100
949
|
expectedStatuses: readonly WorkerRecord["status"][],
|
|
1101
|
-
status:
|
|
950
|
+
status: SettledWorkerStatus,
|
|
1102
951
|
outcome: WorkerOutcome,
|
|
1103
952
|
failureStage: SettlementFailureStage | undefined,
|
|
1104
953
|
dispose: boolean,
|
|
@@ -1108,7 +957,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1108
957
|
const worker = draft.workers.get(workerId);
|
|
1109
958
|
if (
|
|
1110
959
|
!worker ||
|
|
1111
|
-
worker.
|
|
960
|
+
worker.workerEpoch !== workerEpoch ||
|
|
1112
961
|
!expectedStatuses.includes(worker.record.status)
|
|
1113
962
|
) {
|
|
1114
963
|
return { value: undefined };
|
|
@@ -1121,7 +970,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1121
970
|
settledAt,
|
|
1122
971
|
};
|
|
1123
972
|
const actions: PostCommitAction[] = [];
|
|
1124
|
-
let settledWorker:
|
|
973
|
+
let settledWorker: WorkerEntry = { ...worker, record: settledRecord };
|
|
1125
974
|
if (dispose) {
|
|
1126
975
|
actions.push(...this.releaseWorkerResources(settledWorker));
|
|
1127
976
|
settledWorker = {
|
|
@@ -1147,7 +996,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1147
996
|
|
|
1148
997
|
private updateObservation(
|
|
1149
998
|
workerId: WorkerId,
|
|
1150
|
-
|
|
999
|
+
workerEpoch: number,
|
|
1151
1000
|
session: WorkerSessionHandle,
|
|
1152
1001
|
observation: WorkerSessionObservation,
|
|
1153
1002
|
): void {
|
|
@@ -1155,7 +1004,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1155
1004
|
const worker = draft.workers.get(workerId);
|
|
1156
1005
|
if (
|
|
1157
1006
|
!worker ||
|
|
1158
|
-
worker.
|
|
1007
|
+
worker.workerEpoch !== workerEpoch ||
|
|
1159
1008
|
worker.session !== session ||
|
|
1160
1009
|
(worker.record.status !== "starting" && worker.record.status !== "running")
|
|
1161
1010
|
) {
|
|
@@ -1227,7 +1076,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1227
1076
|
}
|
|
1228
1077
|
}
|
|
1229
1078
|
|
|
1230
|
-
const activeWorkers = workers.filter(
|
|
1079
|
+
const activeWorkers = workers.filter(isWorkerEntry);
|
|
1231
1080
|
const marked = this.markWorkersStopping(draft, activeWorkers, candidates);
|
|
1232
1081
|
return {
|
|
1233
1082
|
value: accepted(marked.completions),
|
|
@@ -1261,9 +1110,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1261
1110
|
});
|
|
1262
1111
|
}
|
|
1263
1112
|
|
|
1113
|
+
// Commit stopping and one shared completion before post-commit physical abort
|
|
1114
|
+
// and worker-workflow interruption; every cancellation caller joins it.
|
|
1264
1115
|
private markWorkersStopping(
|
|
1265
|
-
draft:
|
|
1266
|
-
workers: readonly
|
|
1116
|
+
draft: OrchestrationState,
|
|
1117
|
+
workers: readonly WorkerEntry[],
|
|
1267
1118
|
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1268
1119
|
): {
|
|
1269
1120
|
readonly completions: readonly Deferred.Deferred<void>[];
|
|
@@ -1305,13 +1156,14 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1305
1156
|
): void {
|
|
1306
1157
|
const fiber = this.runCancellation(this.cancelWorker(workerId, completion));
|
|
1307
1158
|
const exit = fiber.pollUnsafe();
|
|
1159
|
+
// A closed FiberSet rejects admission with an interrupted sentinel.
|
|
1308
1160
|
if (!exit || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return;
|
|
1309
1161
|
|
|
1310
1162
|
const worker = this.current().workers.get(workerId);
|
|
1311
1163
|
if (worker?.record.status === "stopping") {
|
|
1312
1164
|
this.settleActiveWorker(
|
|
1313
1165
|
workerId,
|
|
1314
|
-
worker.
|
|
1166
|
+
worker.workerEpoch,
|
|
1315
1167
|
["stopping"],
|
|
1316
1168
|
"aborted",
|
|
1317
1169
|
{ status: "aborted" },
|
|
@@ -1340,7 +1192,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1340
1192
|
}
|
|
1341
1193
|
const removal = yield* FiberSet.run(
|
|
1342
1194
|
this.cancellations,
|
|
1343
|
-
FiberMap.remove(this.
|
|
1195
|
+
FiberMap.remove(this.workerWorkflows, workerId).pipe(
|
|
1344
1196
|
Effect.catchCause(() => Effect.void),
|
|
1345
1197
|
),
|
|
1346
1198
|
);
|
|
@@ -1350,7 +1202,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1350
1202
|
);
|
|
1351
1203
|
this.settleActiveWorker(
|
|
1352
1204
|
workerId,
|
|
1353
|
-
this.current().workers.get(workerId)?.
|
|
1205
|
+
this.current().workers.get(workerId)?.workerEpoch ?? -1,
|
|
1354
1206
|
["stopping"],
|
|
1355
1207
|
"aborted",
|
|
1356
1208
|
{ status: "aborted" },
|
|
@@ -1359,11 +1211,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1359
1211
|
);
|
|
1360
1212
|
}).pipe(
|
|
1361
1213
|
Effect.catchCause((cause) => Effect.sync(() => {
|
|
1362
|
-
const
|
|
1363
|
-
if (
|
|
1214
|
+
const workerEpoch = this.current().workers.get(workerId)?.workerEpoch;
|
|
1215
|
+
if (workerEpoch === undefined) return;
|
|
1364
1216
|
this.settleActiveWorker(
|
|
1365
1217
|
workerId,
|
|
1366
|
-
|
|
1218
|
+
workerEpoch,
|
|
1367
1219
|
["stopping"],
|
|
1368
1220
|
"failed",
|
|
1369
1221
|
{
|
|
@@ -1431,8 +1283,8 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1431
1283
|
}
|
|
1432
1284
|
|
|
1433
1285
|
private closeReadyWorker(
|
|
1434
|
-
draft:
|
|
1435
|
-
worker:
|
|
1286
|
+
draft: OrchestrationState,
|
|
1287
|
+
worker: WorkerEntry,
|
|
1436
1288
|
settledAt: number,
|
|
1437
1289
|
actions: PostCommitAction[] = [],
|
|
1438
1290
|
): PostCommitAction[] {
|
|
@@ -1453,7 +1305,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1453
1305
|
return actions;
|
|
1454
1306
|
}
|
|
1455
1307
|
|
|
1456
|
-
private releaseWorkerResources(worker:
|
|
1308
|
+
private releaseWorkerResources(worker: WorkerEntry): PostCommitAction[] {
|
|
1457
1309
|
const actions: PostCommitAction[] = [];
|
|
1458
1310
|
if (worker.observationRelease) {
|
|
1459
1311
|
actions.push(runAction(() => safelyCall(worker.observationRelease)));
|
|
@@ -1461,7 +1313,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1461
1313
|
const session = worker.session;
|
|
1462
1314
|
if (session) {
|
|
1463
1315
|
actions.push(runAction(() => {
|
|
1464
|
-
this.
|
|
1316
|
+
this.launchCleanup(
|
|
1465
1317
|
Effect.suspend(() => session.dispose()).pipe(
|
|
1466
1318
|
Effect.catchCause(() => Effect.void),
|
|
1467
1319
|
Effect.uninterruptible,
|
|
@@ -1472,19 +1324,9 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1472
1324
|
return actions;
|
|
1473
1325
|
}
|
|
1474
1326
|
|
|
1475
|
-
private
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
return;
|
|
1479
|
-
}
|
|
1480
|
-
const fiber = this.runCleanup(cleanup);
|
|
1481
|
-
const exit = fiber.pollUnsafe();
|
|
1482
|
-
if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
1483
|
-
// FiberSet.runtime returns an already-interrupted sentinel when closure
|
|
1484
|
-
// wins admission. If admission won and closure interrupted the real fiber,
|
|
1485
|
-
// repeating the cached uninterruptible disposal only joins that cleanup.
|
|
1486
|
-
Effect.runFork(cleanup);
|
|
1487
|
-
}
|
|
1327
|
+
private launchCleanup(cleanup: Effect.Effect<void>): void {
|
|
1328
|
+
const fiber = Effect.runFork(cleanup);
|
|
1329
|
+
FiberSet.addUnsafe(this.cleanups, fiber);
|
|
1488
1330
|
}
|
|
1489
1331
|
|
|
1490
1332
|
private preflightOpen(
|
|
@@ -1500,42 +1342,25 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1500
1342
|
workerId: WorkerId,
|
|
1501
1343
|
): Decision<void> {
|
|
1502
1344
|
return this.transact((draft) => {
|
|
1503
|
-
const
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
workerId,
|
|
1510
|
-
);
|
|
1511
|
-
if (ownership._tag === "rejected") return { value: ownership };
|
|
1512
|
-
const worker = ownership.value;
|
|
1513
|
-
if (
|
|
1514
|
-
worker.record.lifecycle !== "interactive" ||
|
|
1515
|
-
worker.record.status !== "ready" ||
|
|
1516
|
-
!worker.session
|
|
1517
|
-
) {
|
|
1518
|
-
return {
|
|
1519
|
-
value: rejected(
|
|
1520
|
-
"sendInteractive",
|
|
1521
|
-
"worker-state",
|
|
1522
|
-
"interactive_send requires an owned ready interactive worker",
|
|
1523
|
-
),
|
|
1524
|
-
};
|
|
1525
|
-
}
|
|
1526
|
-
return { value: accepted(undefined) };
|
|
1345
|
+
const ready = readyInteractiveDecision(draft, ownerSessionId, workerId);
|
|
1346
|
+
return {
|
|
1347
|
+
value: ready._tag === "accepted"
|
|
1348
|
+
? accepted(undefined)
|
|
1349
|
+
: ready,
|
|
1350
|
+
};
|
|
1527
1351
|
});
|
|
1528
1352
|
}
|
|
1529
1353
|
|
|
1530
|
-
private current():
|
|
1354
|
+
private current(): OrchestrationState {
|
|
1531
1355
|
return this.state;
|
|
1532
1356
|
}
|
|
1533
1357
|
|
|
1534
1358
|
private transact<A>(
|
|
1535
|
-
reducer: (draft:
|
|
1359
|
+
reducer: (draft: OrchestrationState) => TransactionMutation<A>,
|
|
1536
1360
|
): A {
|
|
1537
1361
|
const draft = makeDraft(this.state);
|
|
1538
1362
|
const mutation = reducer(draft);
|
|
1363
|
+
// Commit before action factories capture the snapshot and callbacks can run.
|
|
1539
1364
|
this.state = Object.freeze(draft);
|
|
1540
1365
|
this.enqueueActions((mutation.actions ?? []).map((action) => action(
|
|
1541
1366
|
this.state,
|
|
@@ -1546,6 +1371,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1546
1371
|
}
|
|
1547
1372
|
|
|
1548
1373
|
private enqueueActions(actions: readonly CommittedAction[]): void {
|
|
1374
|
+
// Reentrant actions queue behind this drain; drain all before rethrowing the first failure.
|
|
1549
1375
|
this.actionQueue.push(...actions);
|
|
1550
1376
|
if (this.drainingActions) return;
|
|
1551
1377
|
|
|
@@ -1578,23 +1404,21 @@ export function orchestrationLayer(
|
|
|
1578
1404
|
Effect.gen(function* () {
|
|
1579
1405
|
const childSessions = yield* ChildSessions;
|
|
1580
1406
|
// Scope finalizers run in reverse acquisition order. Keep cleanup open while
|
|
1581
|
-
//
|
|
1407
|
+
// worker-workflow and cancellation interruption settle workers and enqueue disposal.
|
|
1582
1408
|
const cleanups = yield* FiberSet.make<void, never>();
|
|
1583
|
-
const runCleanup = yield* FiberSet.runtime(cleanups)<never>();
|
|
1584
1409
|
const cancellations = yield* FiberSet.make<void, never>();
|
|
1585
1410
|
const runCancellation = yield* FiberSet.runtime(cancellations)<never>();
|
|
1586
|
-
const
|
|
1587
|
-
const
|
|
1411
|
+
const workerWorkflows = yield* FiberMap.make<WorkerId, void, never>();
|
|
1412
|
+
const runWorkerWorkflow = yield* FiberMap.runtime(workerWorkflows)<never>();
|
|
1588
1413
|
const clock = yield* Clock.Clock;
|
|
1589
1414
|
const shutdownCompletion = yield* Deferred.make<void>();
|
|
1590
|
-
return Orchestration.of(new
|
|
1415
|
+
return Orchestration.of(new OrchestrationEngine(
|
|
1591
1416
|
childSessions,
|
|
1592
|
-
|
|
1593
|
-
|
|
1417
|
+
workerWorkflows,
|
|
1418
|
+
runWorkerWorkflow,
|
|
1594
1419
|
cancellations,
|
|
1595
1420
|
runCancellation,
|
|
1596
1421
|
cleanups,
|
|
1597
|
-
runCleanup,
|
|
1598
1422
|
clock,
|
|
1599
1423
|
options.idFactories ?? createRandomIdFactories(),
|
|
1600
1424
|
shutdownCompletion,
|
|
@@ -1603,10 +1427,10 @@ export function orchestrationLayer(
|
|
|
1603
1427
|
);
|
|
1604
1428
|
}
|
|
1605
1429
|
|
|
1606
|
-
function initialState():
|
|
1430
|
+
function initialState(): OrchestrationState {
|
|
1607
1431
|
return Object.freeze({
|
|
1608
|
-
workers: new Map<WorkerId,
|
|
1609
|
-
runs: new Map<RunId,
|
|
1432
|
+
workers: new Map<WorkerId, WorkerEntry>(),
|
|
1433
|
+
runs: new Map<RunId, RunEntry>(),
|
|
1610
1434
|
terminalWorkerOrder: [],
|
|
1611
1435
|
completedRunOrder: [],
|
|
1612
1436
|
settlementSequence: 0,
|
|
@@ -1614,7 +1438,7 @@ function initialState(): RuntimeState {
|
|
|
1614
1438
|
});
|
|
1615
1439
|
}
|
|
1616
1440
|
|
|
1617
|
-
function makeDraft(state:
|
|
1441
|
+
function makeDraft(state: OrchestrationState): OrchestrationState {
|
|
1618
1442
|
return {
|
|
1619
1443
|
...state,
|
|
1620
1444
|
workers: new Map(state.workers),
|
|
@@ -1625,13 +1449,13 @@ function makeDraft(state: RuntimeState): RuntimeState {
|
|
|
1625
1449
|
}
|
|
1626
1450
|
|
|
1627
1451
|
function completeRun(
|
|
1628
|
-
state:
|
|
1452
|
+
state: OrchestrationState,
|
|
1629
1453
|
workerId: WorkerId,
|
|
1630
1454
|
): PostCommitAction[] {
|
|
1631
1455
|
const worker = state.workers.get(workerId);
|
|
1632
1456
|
if (!worker) return [];
|
|
1633
1457
|
const run = state.runs.get(worker.record.runId);
|
|
1634
|
-
if (!run || !
|
|
1458
|
+
if (!run || !isActiveRunEntry(run) || !isSettledWorkerRecord(worker.record)) {
|
|
1635
1459
|
return [];
|
|
1636
1460
|
}
|
|
1637
1461
|
|
|
@@ -1644,53 +1468,35 @@ function completeRun(
|
|
|
1644
1468
|
}
|
|
1645
1469
|
|
|
1646
1470
|
function makeSettlement(
|
|
1647
|
-
draft:
|
|
1648
|
-
worker:
|
|
1471
|
+
draft: OrchestrationState,
|
|
1472
|
+
worker: WorkerEntry,
|
|
1649
1473
|
settledAt: number,
|
|
1650
1474
|
failureStage: SettlementFailureStage | undefined,
|
|
1651
1475
|
): PostCommitAction | undefined {
|
|
1652
1476
|
const run = draft.runs.get(worker.record.runId);
|
|
1653
|
-
if (!run || !
|
|
1477
|
+
if (!run || !isSettledWorkerRecord(worker.record)) {
|
|
1654
1478
|
return undefined;
|
|
1655
1479
|
}
|
|
1656
|
-
const runRecord = runtimeRunRecord(run);
|
|
1657
1480
|
const sequence = draft.settlementSequence + 1;
|
|
1658
1481
|
draft.settlementSequence = sequence;
|
|
1659
|
-
|
|
1660
|
-
|
|
1482
|
+
// Persisted generation records this authority epoch for reconstruction and
|
|
1483
|
+
// diagnostics.
|
|
1484
|
+
const settlement = createWorkerSettlement({
|
|
1661
1485
|
sequence,
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
generation: worker.generation,
|
|
1666
|
-
mode: runRecord.mode,
|
|
1667
|
-
worker: worker.record.worker,
|
|
1668
|
-
title: worker.record.title,
|
|
1669
|
-
lifecycle: worker.record.lifecycle,
|
|
1670
|
-
status: worker.record.status,
|
|
1671
|
-
outcome: Object.freeze(copyOutcome(worker.record.outcome)),
|
|
1672
|
-
...(failureStage ? { failureStage } : {}),
|
|
1673
|
-
usage: Object.freeze(copyUsage(worker.record.usage)),
|
|
1674
|
-
startedAt: worker.record.startedAt,
|
|
1486
|
+
generation: worker.workerEpoch,
|
|
1487
|
+
run: runRecordFromEntry(run),
|
|
1488
|
+
worker: worker.record,
|
|
1675
1489
|
settledAt,
|
|
1676
|
-
...(
|
|
1677
|
-
? {
|
|
1678
|
-
synthesisGroupId: runRecord.synthesisGroupId,
|
|
1679
|
-
synthesisGroupSize: runRecord.synthesisGroupSize,
|
|
1680
|
-
}
|
|
1681
|
-
: {}),
|
|
1682
|
-
...(worker.record.sessionFile !== undefined
|
|
1683
|
-
? { sessionFile: worker.record.sessionFile }
|
|
1684
|
-
: {}),
|
|
1490
|
+
...(failureStage ? { failureStage } : {}),
|
|
1685
1491
|
});
|
|
1686
1492
|
return publishSettlement(
|
|
1687
1493
|
settlement,
|
|
1688
|
-
|
|
1494
|
+
isActiveRunEntry(run) ? run.settlementListener : undefined,
|
|
1689
1495
|
);
|
|
1690
1496
|
}
|
|
1691
1497
|
|
|
1692
1498
|
function stateActionsAfterPrune(
|
|
1693
|
-
draft:
|
|
1499
|
+
draft: OrchestrationState,
|
|
1694
1500
|
ownerSessionId: string,
|
|
1695
1501
|
): PostCommitAction[] {
|
|
1696
1502
|
const owners = pruneHistory(draft);
|
|
@@ -1698,13 +1504,13 @@ function stateActionsAfterPrune(
|
|
|
1698
1504
|
return publishOwners(owners);
|
|
1699
1505
|
}
|
|
1700
1506
|
|
|
1701
|
-
function pruneHistory(draft:
|
|
1507
|
+
function pruneHistory(draft: OrchestrationState): Set<string> {
|
|
1702
1508
|
const owners = new Set<string>();
|
|
1703
1509
|
while (draft.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
|
|
1704
1510
|
const runId = draft.completedRunOrder.shift();
|
|
1705
1511
|
if (!runId) break;
|
|
1706
1512
|
const run = draft.runs.get(runId);
|
|
1707
|
-
if (run) owners.add(
|
|
1513
|
+
if (run) owners.add(runRecordFromEntry(run).ownerSessionId);
|
|
1708
1514
|
draft.runs.delete(runId);
|
|
1709
1515
|
}
|
|
1710
1516
|
while (draft.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
|
|
@@ -1723,7 +1529,7 @@ function pruneHistory(draft: RuntimeState): Set<string> {
|
|
|
1723
1529
|
}
|
|
1724
1530
|
|
|
1725
1531
|
function rememberTerminalWorker(
|
|
1726
|
-
draft:
|
|
1532
|
+
draft: OrchestrationState,
|
|
1727
1533
|
workerId: WorkerId,
|
|
1728
1534
|
): void {
|
|
1729
1535
|
if (!draft.terminalWorkerOrder.includes(workerId)) {
|
|
@@ -1732,13 +1538,13 @@ function rememberTerminalWorker(
|
|
|
1732
1538
|
}
|
|
1733
1539
|
|
|
1734
1540
|
function snapshotFor(
|
|
1735
|
-
state:
|
|
1541
|
+
state: OrchestrationState,
|
|
1736
1542
|
ownerSessionId: string,
|
|
1737
|
-
):
|
|
1543
|
+
): OwnerSnapshot {
|
|
1738
1544
|
return Object.freeze({
|
|
1739
1545
|
runs: Object.freeze(
|
|
1740
1546
|
[...state.runs.values()]
|
|
1741
|
-
.map(
|
|
1547
|
+
.map(runRecordFromEntry)
|
|
1742
1548
|
.filter((run) => run.ownerSessionId === ownerSessionId)
|
|
1743
1549
|
.map(copyRunRecord),
|
|
1744
1550
|
),
|
|
@@ -1751,11 +1557,11 @@ function snapshotFor(
|
|
|
1751
1557
|
});
|
|
1752
1558
|
}
|
|
1753
1559
|
|
|
1754
|
-
function
|
|
1755
|
-
return
|
|
1560
|
+
function runRecordFromEntry(run: RunEntry): RunRecord {
|
|
1561
|
+
return isActiveRunEntry(run) ? run.record : run;
|
|
1756
1562
|
}
|
|
1757
1563
|
|
|
1758
|
-
function
|
|
1564
|
+
function isActiveRunEntry(run: RunEntry): run is ActiveRunEntry {
|
|
1759
1565
|
return "_tag" in run;
|
|
1760
1566
|
}
|
|
1761
1567
|
|
|
@@ -1847,7 +1653,7 @@ function runAction(run: () => void): PostCommitAction {
|
|
|
1847
1653
|
}
|
|
1848
1654
|
|
|
1849
1655
|
function openDecision(
|
|
1850
|
-
draft:
|
|
1656
|
+
draft: OrchestrationState,
|
|
1851
1657
|
operation: OrchestrationOperation,
|
|
1852
1658
|
): Decision<void> {
|
|
1853
1659
|
return draft.lifecycle === "open"
|
|
@@ -1855,16 +1661,16 @@ function openDecision(
|
|
|
1855
1661
|
: rejected(
|
|
1856
1662
|
operation,
|
|
1857
1663
|
"shutdown",
|
|
1858
|
-
"
|
|
1664
|
+
"Orchestration is shutting down",
|
|
1859
1665
|
);
|
|
1860
1666
|
}
|
|
1861
1667
|
|
|
1862
1668
|
function ownedWorkerDecision(
|
|
1863
|
-
draft:
|
|
1669
|
+
draft: OrchestrationState,
|
|
1864
1670
|
operation: OrchestrationOperation,
|
|
1865
1671
|
ownerSessionId: string,
|
|
1866
1672
|
workerId: WorkerId,
|
|
1867
|
-
): Decision<
|
|
1673
|
+
): Decision<WorkerEntry> {
|
|
1868
1674
|
const worker = draft.workers.get(workerId);
|
|
1869
1675
|
return !worker || worker.record.ownerSessionId !== ownerSessionId
|
|
1870
1676
|
? rejected(
|
|
@@ -1875,135 +1681,36 @@ function ownedWorkerDecision(
|
|
|
1875
1681
|
: accepted(worker);
|
|
1876
1682
|
}
|
|
1877
1683
|
|
|
1878
|
-
function
|
|
1879
|
-
|
|
1880
|
-
}
|
|
1881
|
-
|
|
1882
|
-
function rejected(
|
|
1883
|
-
operation: OrchestrationOperation,
|
|
1884
|
-
reason: OrchestrationRejectionReason,
|
|
1885
|
-
message: string,
|
|
1886
|
-
): Decision<never> {
|
|
1887
|
-
return {
|
|
1888
|
-
_tag: "rejected",
|
|
1889
|
-
error: actionRejection(operation, reason, message),
|
|
1890
|
-
};
|
|
1891
|
-
}
|
|
1892
|
-
|
|
1893
|
-
function actionRejection(
|
|
1894
|
-
operation: OrchestrationOperation,
|
|
1895
|
-
reason: OrchestrationRejectionReason,
|
|
1896
|
-
message: string,
|
|
1897
|
-
): OrchestrationActionRejected {
|
|
1898
|
-
return new OrchestrationActionRejected({ operation, reason, message });
|
|
1899
|
-
}
|
|
1900
|
-
|
|
1901
|
-
function rejectAction(
|
|
1902
|
-
operation: OrchestrationOperation,
|
|
1903
|
-
reason: OrchestrationRejectionReason,
|
|
1904
|
-
message: string,
|
|
1905
|
-
): Effect.Effect<never, OrchestrationActionRejected> {
|
|
1906
|
-
return Effect.fail(actionRejection(operation, reason, message));
|
|
1907
|
-
}
|
|
1908
|
-
|
|
1909
|
-
function validateContextOwner(
|
|
1910
|
-
operation: OrchestrationOperation,
|
|
1684
|
+
function readyInteractiveDecision(
|
|
1685
|
+
draft: OrchestrationState,
|
|
1911
1686
|
ownerSessionId: string,
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
workerId: string,
|
|
1925
|
-
): Effect.Effect<WorkerId, OrchestrationActionRejected> {
|
|
1926
|
-
if (typeof workerId !== "string" || workerId.trim() === "") {
|
|
1927
|
-
return rejectAction(
|
|
1928
|
-
operation,
|
|
1929
|
-
"validation",
|
|
1930
|
-
"worker_id must not be blank",
|
|
1931
|
-
);
|
|
1932
|
-
}
|
|
1933
|
-
return Schema.decodeUnknownEffect(WorkerId)(workerId).pipe(
|
|
1934
|
-
Effect.mapError(() => actionRejection(
|
|
1935
|
-
operation,
|
|
1936
|
-
"validation",
|
|
1937
|
-
"worker_id must use the canonical worker- prefix",
|
|
1938
|
-
)),
|
|
1687
|
+
workerId: WorkerId,
|
|
1688
|
+
): Decision<{
|
|
1689
|
+
readonly worker: WorkerEntry;
|
|
1690
|
+
readonly session: WorkerSessionHandle;
|
|
1691
|
+
}> {
|
|
1692
|
+
const open = openDecision(draft, "sendInteractive");
|
|
1693
|
+
if (open._tag === "rejected") return open;
|
|
1694
|
+
const ownership = ownedWorkerDecision(
|
|
1695
|
+
draft,
|
|
1696
|
+
"sendInteractive",
|
|
1697
|
+
ownerSessionId,
|
|
1698
|
+
workerId,
|
|
1939
1699
|
);
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
function validateText(
|
|
1952
|
-
operation: OrchestrationOperation,
|
|
1953
|
-
name: string,
|
|
1954
|
-
value: string,
|
|
1955
|
-
maximumLength: number,
|
|
1956
|
-
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1957
|
-
if (typeof value !== "string" || value.trim() === "") {
|
|
1958
|
-
return rejectAction(operation, "validation", `${name} must not be blank`);
|
|
1959
|
-
}
|
|
1960
|
-
return value.length > maximumLength
|
|
1961
|
-
? rejectAction(
|
|
1962
|
-
operation,
|
|
1963
|
-
"validation",
|
|
1964
|
-
`${name} must be at most ${maximumLength} characters`,
|
|
1965
|
-
)
|
|
1966
|
-
: Effect.void;
|
|
1967
|
-
}
|
|
1968
|
-
|
|
1969
|
-
type ValidatedAbortTarget =
|
|
1970
|
-
| {
|
|
1971
|
-
readonly _tag: "ids";
|
|
1972
|
-
readonly workerIds: readonly WorkerId[];
|
|
1973
|
-
}
|
|
1974
|
-
| {
|
|
1975
|
-
readonly _tag: "all";
|
|
1976
|
-
};
|
|
1977
|
-
|
|
1978
|
-
function validateAbortTarget(
|
|
1979
|
-
target: AbortTarget,
|
|
1980
|
-
): Effect.Effect<ValidatedAbortTarget, OrchestrationActionRejected> {
|
|
1981
|
-
return Effect.gen(function* () {
|
|
1982
|
-
if (!target || typeof target !== "object") {
|
|
1983
|
-
return yield* rejectAction("abort", "target", "Invalid abort target");
|
|
1984
|
-
}
|
|
1985
|
-
const selected = [target.workerIds !== undefined, target.all !== undefined]
|
|
1986
|
-
.filter(Boolean).length;
|
|
1987
|
-
if (selected !== 1 || (target.all !== undefined && target.all !== true)) {
|
|
1988
|
-
return yield* rejectAction(
|
|
1989
|
-
"abort",
|
|
1990
|
-
"target",
|
|
1991
|
-
"Abort target must specify exactly one of workerIds or all: true",
|
|
1992
|
-
);
|
|
1993
|
-
}
|
|
1994
|
-
if (target.workerIds === undefined) return { _tag: "all" };
|
|
1995
|
-
if (!Array.isArray(target.workerIds) || target.workerIds.length === 0) {
|
|
1996
|
-
return yield* rejectAction(
|
|
1997
|
-
"abort",
|
|
1998
|
-
"target",
|
|
1999
|
-
"workerIds must contain at least one worker ID",
|
|
2000
|
-
);
|
|
2001
|
-
}
|
|
2002
|
-
const workerIds = yield* Effect.all(
|
|
2003
|
-
[...new Set(target.workerIds)].map((id) => validateWorkerId("abort", id)),
|
|
1700
|
+
if (ownership._tag === "rejected") return ownership;
|
|
1701
|
+
const worker = ownership.value;
|
|
1702
|
+
if (
|
|
1703
|
+
worker.record.lifecycle !== "interactive" ||
|
|
1704
|
+
worker.record.status !== "ready" ||
|
|
1705
|
+
!worker.session
|
|
1706
|
+
) {
|
|
1707
|
+
return rejected(
|
|
1708
|
+
"sendInteractive",
|
|
1709
|
+
"worker-state",
|
|
1710
|
+
"interactive_send requires an owned ready interactive worker",
|
|
2004
1711
|
);
|
|
2005
|
-
|
|
2006
|
-
});
|
|
1712
|
+
}
|
|
1713
|
+
return accepted({ worker, session: worker.session });
|
|
2007
1714
|
}
|
|
2008
1715
|
|
|
2009
1716
|
function makeCancellationCandidates(
|
|
@@ -2015,7 +1722,7 @@ function makeCancellationCandidates(
|
|
|
2015
1722
|
}
|
|
2016
1723
|
|
|
2017
1724
|
function activeWorkerIds(
|
|
2018
|
-
state:
|
|
1725
|
+
state: OrchestrationState,
|
|
2019
1726
|
ownerSessionId?: string,
|
|
2020
1727
|
): WorkerId[] {
|
|
2021
1728
|
return [...state.workers.values()]
|
|
@@ -2037,9 +1744,9 @@ function awaitAll(
|
|
|
2037
1744
|
);
|
|
2038
1745
|
}
|
|
2039
1746
|
|
|
2040
|
-
function
|
|
2041
|
-
worker:
|
|
2042
|
-
): worker is
|
|
1747
|
+
function isWorkerEntry(
|
|
1748
|
+
worker: WorkerEntry | undefined,
|
|
1749
|
+
): worker is WorkerEntry {
|
|
2043
1750
|
return worker !== undefined;
|
|
2044
1751
|
}
|
|
2045
1752
|
|
|
@@ -2047,13 +1754,9 @@ function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
|
|
|
2047
1754
|
return status === "starting" || status === "running" || status === "stopping";
|
|
2048
1755
|
}
|
|
2049
1756
|
|
|
2050
|
-
function
|
|
1757
|
+
function isSettledWorkerRecord(
|
|
2051
1758
|
record: WorkerRecord,
|
|
2052
|
-
): record is
|
|
2053
|
-
readonly status: RunResult["status"];
|
|
2054
|
-
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2055
|
-
readonly settledAt: number;
|
|
2056
|
-
} {
|
|
1759
|
+
): record is SettledWorkerRecord {
|
|
2057
1760
|
return (
|
|
2058
1761
|
(record.status === "completed" ||
|
|
2059
1762
|
record.status === "ready" ||
|
|
@@ -2124,13 +1827,9 @@ function freezeAcceptedRun(id: RunId, workerId: WorkerId): AcceptedRun {
|
|
|
2124
1827
|
|
|
2125
1828
|
function freezeCompletedRun(
|
|
2126
1829
|
run: RunRecord,
|
|
2127
|
-
record:
|
|
2128
|
-
readonly status: RunResult["status"];
|
|
2129
|
-
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2130
|
-
readonly settledAt: number;
|
|
2131
|
-
},
|
|
1830
|
+
record: SettledWorkerRecord,
|
|
2132
1831
|
): CompletedRun {
|
|
2133
|
-
const result:
|
|
1832
|
+
const result: WorkerRunResult = Object.freeze({
|
|
2134
1833
|
workerId: record.id,
|
|
2135
1834
|
worker: record.worker,
|
|
2136
1835
|
title: record.title,
|