@zachwill/pi-orchestrate 0.9.2 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -108
- package/extension/catalog/definition.ts +89 -0
- package/extension/{catalog.ts → catalog/discovery.ts} +4 -4
- package/extension/index.ts +19 -36
- package/extension/orchestration/admission.ts +297 -0
- package/extension/{domain.ts → orchestration/model.ts} +27 -89
- package/extension/{runtime.ts → orchestration/service.ts} +179 -473
- package/extension/{worker-settlement.ts → orchestration/settlement.ts} +67 -21
- package/extension/package-root.ts +3 -0
- package/extension/parent/contract.ts +147 -0
- package/extension/{delivery.ts → parent/delivery.ts} +5 -8
- package/extension/parent/dispatch-policy.ts +49 -0
- package/extension/{host.ts → parent/process-host.ts} +33 -26
- package/extension/{presentation.ts → pi/presentation.ts} +39 -27
- package/extension/pi/tool-renderer.ts +422 -0
- package/extension/pi/tools.ts +470 -0
- package/extension/worker/child-sessions.ts +291 -0
- package/extension/{worker-session.ts → worker/session.ts} +33 -291
- package/package.json +2 -1
- package/extension/contract.ts +0 -144
- package/extension/tools.ts +0 -837
- /package/extension/{tui.ts → pi/tui.ts} +0 -0
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
|
-
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
3
1
|
import {
|
|
4
2
|
Cause,
|
|
5
3
|
Clock,
|
|
@@ -11,71 +9,75 @@ import {
|
|
|
11
9
|
FiberMap,
|
|
12
10
|
FiberSet,
|
|
13
11
|
Layer,
|
|
14
|
-
Schema,
|
|
15
12
|
} from "effect";
|
|
13
|
+
import type { WorkerDefinition } from "../catalog/definition.ts";
|
|
16
14
|
import {
|
|
17
15
|
CANCELLATION_GRACE_MS,
|
|
18
16
|
EMPTY_WORKER_USAGE,
|
|
19
17
|
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
20
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
21
|
-
OrchestrateTaskInput,
|
|
22
18
|
RunId,
|
|
23
19
|
WorkerId,
|
|
24
20
|
createRandomIdFactories,
|
|
25
|
-
findWorkerByName,
|
|
26
21
|
isTerminalWorkerStatus,
|
|
27
22
|
transitionWorkerStatus,
|
|
28
23
|
type OrchestrateIdFactories,
|
|
24
|
+
type OrchestrateTaskInput,
|
|
29
25
|
type RunMode,
|
|
30
26
|
type RunRecord,
|
|
31
|
-
type
|
|
32
|
-
type
|
|
27
|
+
type SettledWorkerOutcome,
|
|
28
|
+
type SettledWorkerRecord,
|
|
29
|
+
type SettledWorkerStatus,
|
|
33
30
|
type WorkerOutcome,
|
|
34
31
|
type WorkerRecord,
|
|
35
32
|
type WorkerUsage,
|
|
36
|
-
} from "./
|
|
33
|
+
} from "./model.ts";
|
|
34
|
+
import {
|
|
35
|
+
OrchestrationActionRejected,
|
|
36
|
+
accepted,
|
|
37
|
+
rejected,
|
|
38
|
+
validateAbortTarget,
|
|
39
|
+
validateContextOwner,
|
|
40
|
+
validateMode,
|
|
41
|
+
validateOrchestrateRequest,
|
|
42
|
+
validateText,
|
|
43
|
+
validateWorkerId,
|
|
44
|
+
type AbortTarget,
|
|
45
|
+
type Decision,
|
|
46
|
+
type OrchestrationContext,
|
|
47
|
+
type OrchestrationOperation,
|
|
48
|
+
type ValidatedAbortTarget,
|
|
49
|
+
} from "./admission.ts";
|
|
50
|
+
import {
|
|
51
|
+
createWorkerSettlement,
|
|
52
|
+
type SettlementFailureStage,
|
|
53
|
+
type WorkerSettlement,
|
|
54
|
+
} from "./settlement.ts";
|
|
37
55
|
import {
|
|
38
56
|
ChildSessions,
|
|
39
57
|
type ChildSessionsService,
|
|
40
|
-
|
|
41
|
-
type WorkerSessionHandle,
|
|
42
|
-
type WorkerSessionObservation,
|
|
43
|
-
} from "./worker-session.js";
|
|
58
|
+
} from "../worker/child-sessions.ts";
|
|
44
59
|
import type {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
60
|
+
WorkerSessionAbortError,
|
|
61
|
+
WorkerSessionHandle,
|
|
62
|
+
WorkerSessionObservation,
|
|
63
|
+
} from "../worker/session.ts";
|
|
48
64
|
|
|
49
65
|
export const MAX_TERMINAL_WORKER_HISTORY = 100;
|
|
50
66
|
export const MAX_COMPLETED_RUN_HISTORY = 100;
|
|
51
67
|
export const SHUTDOWN_CLEANUP_GRACE_MS = CANCELLATION_GRACE_MS;
|
|
52
68
|
|
|
53
|
-
export interface OrchestrationContext {
|
|
54
|
-
readonly ownerSessionId: string;
|
|
55
|
-
readonly cwd: string;
|
|
56
|
-
readonly agentDir: string;
|
|
57
|
-
readonly parentSessionFile: string | undefined;
|
|
58
|
-
readonly projectTrusted: boolean;
|
|
59
|
-
readonly catalog: WorkerCatalog;
|
|
60
|
-
readonly parentModel?: Model<Api>;
|
|
61
|
-
readonly modelRegistry: ModelRegistry;
|
|
62
|
-
readonly synthesisGroup?: {
|
|
63
|
-
readonly id: string;
|
|
64
|
-
readonly size: number;
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
|
|
68
69
|
export interface AcceptedRun {
|
|
69
70
|
readonly id: RunId;
|
|
70
71
|
readonly workerId: WorkerId;
|
|
71
72
|
}
|
|
72
73
|
|
|
73
|
-
|
|
74
|
+
/** One settled worker generation, as returned to the owner that requested it. */
|
|
75
|
+
export interface WorkerRunResult {
|
|
74
76
|
readonly workerId: WorkerId;
|
|
75
77
|
readonly worker: string;
|
|
76
78
|
readonly title: string;
|
|
77
|
-
readonly status:
|
|
78
|
-
readonly outcome:
|
|
79
|
+
readonly status: SettledWorkerStatus;
|
|
80
|
+
readonly outcome: SettledWorkerOutcome;
|
|
79
81
|
readonly usage: WorkerUsage;
|
|
80
82
|
readonly startedAt: number;
|
|
81
83
|
readonly settledAt: number;
|
|
@@ -86,56 +88,23 @@ export interface CompletedRun {
|
|
|
86
88
|
readonly id: RunId;
|
|
87
89
|
readonly ownerSessionId: string;
|
|
88
90
|
readonly mode: RunMode;
|
|
89
|
-
readonly result:
|
|
91
|
+
readonly result: WorkerRunResult;
|
|
90
92
|
}
|
|
91
93
|
|
|
92
|
-
|
|
94
|
+
/** Runs and workers visible to exactly one owner session. */
|
|
95
|
+
export interface OwnerSnapshot {
|
|
93
96
|
readonly runs: readonly RunRecord[];
|
|
94
97
|
readonly workers: readonly WorkerRecord[];
|
|
95
98
|
}
|
|
96
99
|
|
|
97
100
|
export type SettlementListener = (settlement: WorkerSettlement) => void;
|
|
98
101
|
export type UnsubscribeSettlement = () => void;
|
|
99
|
-
export type StateListener = (snapshot:
|
|
100
|
-
|
|
101
|
-
export interface AbortTarget {
|
|
102
|
-
readonly workerIds?: readonly string[];
|
|
103
|
-
readonly all?: boolean;
|
|
104
|
-
}
|
|
102
|
+
export type StateListener = (snapshot: OwnerSnapshot) => void;
|
|
105
103
|
|
|
106
104
|
export interface OrchestrationLayerOptions {
|
|
107
105
|
readonly idFactories?: OrchestrateIdFactories;
|
|
108
106
|
}
|
|
109
107
|
|
|
110
|
-
const OrchestrationOperation = Schema.Literals([
|
|
111
|
-
"orchestrate",
|
|
112
|
-
"sendInteractive",
|
|
113
|
-
"abort",
|
|
114
|
-
"closeInteractive",
|
|
115
|
-
"snapshot",
|
|
116
|
-
]);
|
|
117
|
-
type OrchestrationOperation = typeof OrchestrationOperation.Type;
|
|
118
|
-
|
|
119
|
-
const OrchestrationRejectionReason = Schema.Literals([
|
|
120
|
-
"shutdown",
|
|
121
|
-
"validation",
|
|
122
|
-
"ownership",
|
|
123
|
-
"target",
|
|
124
|
-
"worker-state",
|
|
125
|
-
"unknown-worker",
|
|
126
|
-
"model-unavailable",
|
|
127
|
-
]);
|
|
128
|
-
type OrchestrationRejectionReason = typeof OrchestrationRejectionReason.Type;
|
|
129
|
-
|
|
130
|
-
export class OrchestrationActionRejected extends Schema.TaggedError<OrchestrationActionRejected>()(
|
|
131
|
-
"Orchestration.ActionRejected",
|
|
132
|
-
{
|
|
133
|
-
operation: OrchestrationOperation,
|
|
134
|
-
reason: OrchestrationRejectionReason,
|
|
135
|
-
message: Schema.String,
|
|
136
|
-
},
|
|
137
|
-
) {}
|
|
138
|
-
|
|
139
108
|
export interface OrchestrationService {
|
|
140
109
|
orchestrate(
|
|
141
110
|
context: OrchestrationContext,
|
|
@@ -186,7 +155,7 @@ export interface OrchestrationService {
|
|
|
186
155
|
) => Effect.Effect<void, OrchestrationActionRejected>;
|
|
187
156
|
readonly snapshot: (
|
|
188
157
|
ownerSessionId: string,
|
|
189
|
-
) => Effect.Effect<
|
|
158
|
+
) => Effect.Effect<OwnerSnapshot, OrchestrationActionRejected>;
|
|
190
159
|
readonly subscribeSettlement: (
|
|
191
160
|
listener: SettlementListener,
|
|
192
161
|
) => UnsubscribeSettlement;
|
|
@@ -201,39 +170,39 @@ export class Orchestration extends Context.Service<Orchestration, OrchestrationS
|
|
|
201
170
|
"@zachwill/pi-orchestrate/Orchestration",
|
|
202
171
|
) {}
|
|
203
172
|
|
|
204
|
-
interface
|
|
173
|
+
interface WorkerEntry {
|
|
205
174
|
readonly record: WorkerRecord;
|
|
206
175
|
readonly context: OrchestrationContext;
|
|
207
176
|
readonly definition: WorkerDefinition;
|
|
208
177
|
// Worker-local authority fence; stale asynchronous callbacks must revalidate it.
|
|
209
|
-
readonly
|
|
178
|
+
readonly workerEpoch: number;
|
|
210
179
|
readonly session?: WorkerSessionHandle;
|
|
211
180
|
readonly observationRelease?: () => void;
|
|
212
181
|
readonly cancellation?: Deferred.Deferred<void>;
|
|
213
182
|
}
|
|
214
183
|
|
|
215
|
-
interface
|
|
184
|
+
interface ActiveRunEntry {
|
|
216
185
|
readonly _tag: "running";
|
|
217
186
|
readonly record: RunRecord;
|
|
218
187
|
readonly completion: Deferred.Deferred<CompletedRun>;
|
|
219
188
|
readonly settlementListener?: SettlementListener;
|
|
220
189
|
}
|
|
221
190
|
|
|
222
|
-
type
|
|
223
|
-
type
|
|
191
|
+
type RunEntry = ActiveRunEntry | RunRecord;
|
|
192
|
+
type OrchestrationLifecycle = "open" | "shutting-down" | "shutdown";
|
|
224
193
|
|
|
225
|
-
interface
|
|
226
|
-
workers: Map<WorkerId,
|
|
227
|
-
runs: Map<RunId,
|
|
194
|
+
interface OrchestrationState {
|
|
195
|
+
workers: Map<WorkerId, WorkerEntry>;
|
|
196
|
+
runs: Map<RunId, RunEntry>;
|
|
228
197
|
terminalWorkerOrder: WorkerId[];
|
|
229
198
|
completedRunOrder: RunId[];
|
|
230
199
|
settlementSequence: number;
|
|
231
|
-
lifecycle:
|
|
200
|
+
lifecycle: OrchestrationLifecycle;
|
|
232
201
|
}
|
|
233
202
|
|
|
234
203
|
type CommittedAction = () => void;
|
|
235
204
|
type PostCommitAction = (
|
|
236
|
-
state:
|
|
205
|
+
state: OrchestrationState,
|
|
237
206
|
settlementListeners: ReadonlySet<SettlementListener>,
|
|
238
207
|
stateListeners: ReadonlyMap<string, ReadonlySet<StateListener>>,
|
|
239
208
|
) => CommittedAction;
|
|
@@ -243,27 +212,17 @@ interface TransactionMutation<A> {
|
|
|
243
212
|
readonly actions?: readonly PostCommitAction[];
|
|
244
213
|
}
|
|
245
214
|
|
|
246
|
-
|
|
247
|
-
| {
|
|
248
|
-
readonly _tag: "accepted";
|
|
249
|
-
readonly value: A;
|
|
250
|
-
}
|
|
251
|
-
| {
|
|
252
|
-
readonly _tag: "rejected";
|
|
253
|
-
readonly error: OrchestrationActionRejected;
|
|
254
|
-
};
|
|
255
|
-
|
|
256
|
-
class StatefulOrchestration implements OrchestrationService {
|
|
215
|
+
class OrchestrationEngine implements OrchestrationService {
|
|
257
216
|
private readonly actionQueue: CommittedAction[] = [];
|
|
258
217
|
private readonly settlementListeners = new Set<SettlementListener>();
|
|
259
218
|
private readonly stateListeners = new Map<string, Set<StateListener>>();
|
|
260
219
|
private drainingActions = false;
|
|
261
|
-
private state:
|
|
220
|
+
private state: OrchestrationState;
|
|
262
221
|
|
|
263
222
|
constructor(
|
|
264
223
|
private readonly childSessions: ChildSessionsService,
|
|
265
|
-
private readonly
|
|
266
|
-
private readonly
|
|
224
|
+
private readonly workerWorkflows: FiberMap.FiberMap<WorkerId, void, never>,
|
|
225
|
+
private readonly runWorkerWorkflow: (
|
|
267
226
|
key: WorkerId,
|
|
268
227
|
effect: Effect.Effect<void, never>,
|
|
269
228
|
) => Fiber.Fiber<void, never>,
|
|
@@ -304,9 +263,9 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
304
263
|
onSettlement?: SettlementListener,
|
|
305
264
|
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
306
265
|
return Effect.fn("Orchestration.orchestrate")(function* (
|
|
307
|
-
this:
|
|
266
|
+
this: OrchestrationEngine,
|
|
308
267
|
) {
|
|
309
|
-
const validated = yield*
|
|
268
|
+
const validated = yield* validateOrchestrateRequest(context, task, mode);
|
|
310
269
|
const preflight = this.preflightOpen("orchestrate");
|
|
311
270
|
if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
|
|
312
271
|
// Keep user-supplied ID factories outside the transaction: a reentrant
|
|
@@ -344,7 +303,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
344
303
|
record: workerRecord,
|
|
345
304
|
context,
|
|
346
305
|
definition: validated.definition,
|
|
347
|
-
|
|
306
|
+
workerEpoch: 1,
|
|
348
307
|
});
|
|
349
308
|
|
|
350
309
|
return {
|
|
@@ -393,7 +352,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
393
352
|
onSettlement?: SettlementListener,
|
|
394
353
|
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
395
354
|
return Effect.fn("Orchestration.sendInteractive")(function* (
|
|
396
|
-
this:
|
|
355
|
+
this: OrchestrationEngine,
|
|
397
356
|
) {
|
|
398
357
|
yield* validateContextOwner("sendInteractive", context.ownerSessionId);
|
|
399
358
|
yield* validateMode("sendInteractive", mode);
|
|
@@ -436,7 +395,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
436
395
|
if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
|
|
437
396
|
|
|
438
397
|
const { worker, session } = ready.value;
|
|
439
|
-
const
|
|
398
|
+
const workerEpoch = worker.workerEpoch + 1;
|
|
440
399
|
const runningRecord: WorkerRecord = {
|
|
441
400
|
...transitionWorkerStatus(worker.record, "running"),
|
|
442
401
|
runId,
|
|
@@ -455,7 +414,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
455
414
|
draft.workers.set(validatedWorkerId, {
|
|
456
415
|
...worker,
|
|
457
416
|
record: runningRecord,
|
|
458
|
-
|
|
417
|
+
workerEpoch,
|
|
459
418
|
observationRelease: undefined,
|
|
460
419
|
cancellation: undefined,
|
|
461
420
|
});
|
|
@@ -465,13 +424,13 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
465
424
|
actions: [
|
|
466
425
|
runAction(() => {
|
|
467
426
|
safelyCall(worker.observationRelease);
|
|
468
|
-
this.subscribeObservation(validatedWorkerId,
|
|
427
|
+
this.subscribeObservation(validatedWorkerId, workerEpoch, session);
|
|
469
428
|
}),
|
|
470
429
|
publishState(context.ownerSessionId),
|
|
471
430
|
runAction(() => {
|
|
472
431
|
this.launchPrompt(
|
|
473
432
|
validatedWorkerId,
|
|
474
|
-
|
|
433
|
+
workerEpoch,
|
|
475
434
|
session,
|
|
476
435
|
instructions,
|
|
477
436
|
);
|
|
@@ -493,7 +452,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
493
452
|
target: AbortTarget,
|
|
494
453
|
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
495
454
|
return Effect.fn("Orchestration.abort")(function* (
|
|
496
|
-
this:
|
|
455
|
+
this: OrchestrationEngine,
|
|
497
456
|
) {
|
|
498
457
|
yield* validateContextOwner("abort", ownerSessionId);
|
|
499
458
|
const validatedTarget = yield* validateAbortTarget(target);
|
|
@@ -519,7 +478,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
519
478
|
workerId: string,
|
|
520
479
|
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
521
480
|
return Effect.fn("Orchestration.closeInteractive")(function* (
|
|
522
|
-
this:
|
|
481
|
+
this: OrchestrationEngine,
|
|
523
482
|
) {
|
|
524
483
|
yield* validateContextOwner("closeInteractive", ownerSessionId);
|
|
525
484
|
const id = yield* validateWorkerId("closeInteractive", workerId);
|
|
@@ -559,7 +518,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
559
518
|
|
|
560
519
|
snapshot(
|
|
561
520
|
ownerSessionId: string,
|
|
562
|
-
): Effect.Effect<
|
|
521
|
+
): Effect.Effect<OwnerSnapshot, OrchestrationActionRejected> {
|
|
563
522
|
return validateContextOwner("snapshot", ownerSessionId).pipe(
|
|
564
523
|
Effect.andThen(Effect.sync(() => snapshotFor(this.current(), ownerSessionId))),
|
|
565
524
|
);
|
|
@@ -624,7 +583,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
624
583
|
this.transact((draft) => {
|
|
625
584
|
draft.lifecycle = "shutdown";
|
|
626
585
|
for (const [runId, run] of draft.runs) {
|
|
627
|
-
if (
|
|
586
|
+
if (isActiveRunEntry(run) && run.settlementListener) {
|
|
628
587
|
draft.runs.set(runId, { ...run, settlementListener: undefined });
|
|
629
588
|
}
|
|
630
589
|
}
|
|
@@ -640,7 +599,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
640
599
|
|
|
641
600
|
private performShutdown(): Effect.Effect<void> {
|
|
642
601
|
return Effect.fn("Orchestration.shutdown")(function* (
|
|
643
|
-
this:
|
|
602
|
+
this: OrchestrationEngine,
|
|
644
603
|
) {
|
|
645
604
|
const now = this.clock.currentTimeMillisUnsafe();
|
|
646
605
|
this.transact((draft) => {
|
|
@@ -672,131 +631,40 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
672
631
|
}).call(this);
|
|
673
632
|
}
|
|
674
633
|
|
|
675
|
-
private
|
|
676
|
-
|
|
677
|
-
task: OrchestrateTaskInput,
|
|
678
|
-
mode: RunMode,
|
|
679
|
-
): Effect.Effect<
|
|
680
|
-
{ definition: WorkerDefinition; task: OrchestrateTaskInput },
|
|
681
|
-
OrchestrationActionRejected
|
|
682
|
-
> {
|
|
683
|
-
return Effect.gen(function* () {
|
|
684
|
-
yield* validateContextOwner("orchestrate", context.ownerSessionId);
|
|
685
|
-
yield* validateMode("orchestrate", mode);
|
|
686
|
-
if (!task || typeof task !== "object" || Array.isArray(task)) {
|
|
687
|
-
return yield* rejectAction(
|
|
688
|
-
"orchestrate",
|
|
689
|
-
"validation",
|
|
690
|
-
"orchestrate requires one task object",
|
|
691
|
-
);
|
|
692
|
-
}
|
|
693
|
-
if (context.synthesisGroup) {
|
|
694
|
-
yield* validateText(
|
|
695
|
-
"orchestrate",
|
|
696
|
-
"synthesis group ID",
|
|
697
|
-
context.synthesisGroup.id,
|
|
698
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
699
|
-
);
|
|
700
|
-
if (mode !== "async") {
|
|
701
|
-
return yield* rejectAction(
|
|
702
|
-
"orchestrate",
|
|
703
|
-
"validation",
|
|
704
|
-
"Sibling synthesis requires an async task",
|
|
705
|
-
);
|
|
706
|
-
}
|
|
707
|
-
if (
|
|
708
|
-
!Number.isSafeInteger(context.synthesisGroup.size) ||
|
|
709
|
-
context.synthesisGroup.size < 2
|
|
710
|
-
) {
|
|
711
|
-
return yield* rejectAction(
|
|
712
|
-
"orchestrate",
|
|
713
|
-
"validation",
|
|
714
|
-
"Synthesis group size must be an integer of at least 2",
|
|
715
|
-
);
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
yield* validateText(
|
|
719
|
-
"orchestrate",
|
|
720
|
-
"worker",
|
|
721
|
-
task.worker,
|
|
722
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
723
|
-
);
|
|
724
|
-
yield* validateText(
|
|
725
|
-
"orchestrate",
|
|
726
|
-
"title",
|
|
727
|
-
task.title,
|
|
728
|
-
MAX_WORKER_TITLE_LENGTH,
|
|
729
|
-
);
|
|
730
|
-
yield* validateText(
|
|
731
|
-
"orchestrate",
|
|
732
|
-
"instructions",
|
|
733
|
-
task.instructions,
|
|
734
|
-
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
735
|
-
);
|
|
736
|
-
const definition = findWorkerByName(context.catalog, task.worker);
|
|
737
|
-
if (!definition) {
|
|
738
|
-
return yield* rejectAction(
|
|
739
|
-
"orchestrate",
|
|
740
|
-
"unknown-worker",
|
|
741
|
-
`Unknown worker: ${task.worker}`,
|
|
742
|
-
);
|
|
743
|
-
}
|
|
744
|
-
const configured = definition.model;
|
|
745
|
-
if (!configured && !context.parentModel) {
|
|
746
|
-
return yield* rejectAction(
|
|
747
|
-
"orchestrate",
|
|
748
|
-
"model-unavailable",
|
|
749
|
-
`Worker "${definition.name}" has no configured model and no parent model is available`,
|
|
750
|
-
);
|
|
751
|
-
}
|
|
752
|
-
if (
|
|
753
|
-
configured &&
|
|
754
|
-
!context.modelRegistry.find(configured.provider, configured.modelId)
|
|
755
|
-
) {
|
|
756
|
-
return yield* rejectAction(
|
|
757
|
-
"orchestrate",
|
|
758
|
-
"model-unavailable",
|
|
759
|
-
`Worker "${definition.name}" configured model "${configured.provider}/${configured.modelId}" was not found`,
|
|
760
|
-
);
|
|
761
|
-
}
|
|
762
|
-
return { definition, task };
|
|
763
|
-
});
|
|
764
|
-
}
|
|
765
|
-
|
|
766
|
-
private launchBootstrap(workerId: WorkerId, generation: number): void {
|
|
767
|
-
this.launchGeneration(
|
|
634
|
+
private launchBootstrap(workerId: WorkerId, workerEpoch: number): void {
|
|
635
|
+
this.launchWorkerWorkflow(
|
|
768
636
|
workerId,
|
|
769
|
-
|
|
770
|
-
this.bootstrapAndPrompt(workerId,
|
|
637
|
+
workerEpoch,
|
|
638
|
+
this.bootstrapAndPrompt(workerId, workerEpoch),
|
|
771
639
|
);
|
|
772
640
|
}
|
|
773
641
|
|
|
774
642
|
private launchPrompt(
|
|
775
643
|
workerId: WorkerId,
|
|
776
|
-
|
|
644
|
+
workerEpoch: number,
|
|
777
645
|
session: WorkerSessionHandle,
|
|
778
646
|
instructions: string,
|
|
779
647
|
): void {
|
|
780
|
-
this.
|
|
648
|
+
this.launchWorkerWorkflow(
|
|
781
649
|
workerId,
|
|
782
|
-
|
|
783
|
-
this.executePrompt(workerId,
|
|
650
|
+
workerEpoch,
|
|
651
|
+
this.executePrompt(workerId, workerEpoch, session, instructions),
|
|
784
652
|
);
|
|
785
653
|
}
|
|
786
654
|
|
|
787
|
-
private
|
|
655
|
+
private launchWorkerWorkflow(
|
|
788
656
|
workerId: WorkerId,
|
|
789
|
-
|
|
657
|
+
workerEpoch: number,
|
|
790
658
|
workflow: Effect.Effect<void, never>,
|
|
791
659
|
): void {
|
|
792
660
|
try {
|
|
793
|
-
const fiber = this.
|
|
661
|
+
const fiber = this.runWorkerWorkflow(
|
|
794
662
|
workerId,
|
|
795
663
|
workflow.pipe(
|
|
796
664
|
Effect.catchCause((cause) => {
|
|
797
665
|
if (Cause.hasInterruptsOnly(cause)) return Effect.void;
|
|
798
666
|
return Effect.sync(() => {
|
|
799
|
-
this.settleWorkflowDefect(workerId,
|
|
667
|
+
this.settleWorkflowDefect(workerId, workerEpoch, Cause.squash(cause));
|
|
800
668
|
});
|
|
801
669
|
}),
|
|
802
670
|
),
|
|
@@ -804,20 +672,20 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
804
672
|
const exit = fiber.pollUnsafe();
|
|
805
673
|
// A closed FiberMap rejects admission with an interrupted sentinel.
|
|
806
674
|
if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
807
|
-
this.
|
|
675
|
+
this.settleWorkflowLaunchFailure(workerId, workerEpoch);
|
|
808
676
|
}
|
|
809
677
|
} catch (error) {
|
|
810
|
-
this.settleWorkflowDefect(workerId,
|
|
678
|
+
this.settleWorkflowDefect(workerId, workerEpoch, error);
|
|
811
679
|
}
|
|
812
680
|
}
|
|
813
681
|
|
|
814
|
-
private
|
|
682
|
+
private settleWorkflowLaunchFailure(
|
|
815
683
|
workerId: WorkerId,
|
|
816
|
-
|
|
684
|
+
workerEpoch: number,
|
|
817
685
|
): void {
|
|
818
686
|
this.settleActiveWorker(
|
|
819
687
|
workerId,
|
|
820
|
-
|
|
688
|
+
workerEpoch,
|
|
821
689
|
["starting", "running"],
|
|
822
690
|
"failed",
|
|
823
691
|
{
|
|
@@ -831,16 +699,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
831
699
|
|
|
832
700
|
private bootstrapAndPrompt(
|
|
833
701
|
workerId: WorkerId,
|
|
834
|
-
|
|
702
|
+
workerEpoch: number,
|
|
835
703
|
): Effect.Effect<void, never> {
|
|
836
704
|
return Effect.gen({ self: this }, function* () {
|
|
837
|
-
const session = yield* this.bootstrap(workerId,
|
|
705
|
+
const session = yield* this.bootstrap(workerId, workerEpoch);
|
|
838
706
|
if (!session) return;
|
|
839
707
|
const worker = this.current().workers.get(workerId);
|
|
840
708
|
if (!worker) return;
|
|
841
709
|
yield* this.executePrompt(
|
|
842
710
|
workerId,
|
|
843
|
-
|
|
711
|
+
workerEpoch,
|
|
844
712
|
session,
|
|
845
713
|
worker.record.instructions,
|
|
846
714
|
);
|
|
@@ -849,11 +717,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
849
717
|
|
|
850
718
|
private bootstrap(
|
|
851
719
|
workerId: WorkerId,
|
|
852
|
-
|
|
720
|
+
workerEpoch: number,
|
|
853
721
|
): Effect.Effect<WorkerSessionHandle | undefined, never> {
|
|
854
722
|
return Effect.suspend(() => {
|
|
855
723
|
const expected = this.current().workers.get(workerId);
|
|
856
|
-
if (!expected || expected.
|
|
724
|
+
if (!expected || expected.workerEpoch !== workerEpoch) {
|
|
857
725
|
return Effect.succeed(undefined);
|
|
858
726
|
}
|
|
859
727
|
return this.childSessions.acquire(
|
|
@@ -866,11 +734,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
866
734
|
parentModel: expected.context.parentModel,
|
|
867
735
|
modelRegistry: expected.context.modelRegistry,
|
|
868
736
|
},
|
|
869
|
-
(session) => this.adoptCreatedSession(workerId,
|
|
737
|
+
(session) => this.adoptCreatedSession(workerId, workerEpoch, session),
|
|
870
738
|
).pipe(
|
|
871
739
|
Effect.match({
|
|
872
740
|
onFailure: (error) => {
|
|
873
|
-
this.settleCreationFailure(workerId,
|
|
741
|
+
this.settleCreationFailure(workerId, workerEpoch, error);
|
|
874
742
|
return undefined;
|
|
875
743
|
},
|
|
876
744
|
onSuccess: (session) => session,
|
|
@@ -881,16 +749,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
881
749
|
|
|
882
750
|
private adoptCreatedSession(
|
|
883
751
|
workerId: WorkerId,
|
|
884
|
-
|
|
752
|
+
workerEpoch: number,
|
|
885
753
|
session: WorkerSessionHandle,
|
|
886
754
|
): WorkerSessionHandle | undefined {
|
|
887
755
|
let release: () => void;
|
|
888
756
|
try {
|
|
889
757
|
release = session.subscribeObservation((observation) => {
|
|
890
|
-
this.updateObservation(workerId,
|
|
758
|
+
this.updateObservation(workerId, workerEpoch, session, observation);
|
|
891
759
|
});
|
|
892
760
|
} catch (error) {
|
|
893
|
-
this.settleCreationFailure(workerId,
|
|
761
|
+
this.settleCreationFailure(workerId, workerEpoch, error);
|
|
894
762
|
return undefined;
|
|
895
763
|
}
|
|
896
764
|
|
|
@@ -899,7 +767,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
899
767
|
if (
|
|
900
768
|
draft.lifecycle !== "open" ||
|
|
901
769
|
!worker ||
|
|
902
|
-
worker.
|
|
770
|
+
worker.workerEpoch !== workerEpoch ||
|
|
903
771
|
worker.record.status !== "starting"
|
|
904
772
|
) {
|
|
905
773
|
return {
|
|
@@ -926,16 +794,16 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
926
794
|
|
|
927
795
|
private subscribeObservation(
|
|
928
796
|
workerId: WorkerId,
|
|
929
|
-
|
|
797
|
+
workerEpoch: number,
|
|
930
798
|
session: WorkerSessionHandle,
|
|
931
799
|
): void {
|
|
932
800
|
let release: () => void;
|
|
933
801
|
try {
|
|
934
802
|
release = session.subscribeObservation((observation) => {
|
|
935
|
-
this.updateObservation(workerId,
|
|
803
|
+
this.updateObservation(workerId, workerEpoch, session, observation);
|
|
936
804
|
});
|
|
937
805
|
} catch (error) {
|
|
938
|
-
this.settleWorkflowDefect(workerId,
|
|
806
|
+
this.settleWorkflowDefect(workerId, workerEpoch, error);
|
|
939
807
|
return;
|
|
940
808
|
}
|
|
941
809
|
|
|
@@ -943,7 +811,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
943
811
|
const worker = draft.workers.get(workerId);
|
|
944
812
|
if (
|
|
945
813
|
!worker ||
|
|
946
|
-
worker.
|
|
814
|
+
worker.workerEpoch !== workerEpoch ||
|
|
947
815
|
worker.session !== session ||
|
|
948
816
|
worker.record.status !== "running"
|
|
949
817
|
) {
|
|
@@ -959,7 +827,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
959
827
|
|
|
960
828
|
private executePrompt(
|
|
961
829
|
workerId: WorkerId,
|
|
962
|
-
|
|
830
|
+
workerEpoch: number,
|
|
963
831
|
session: WorkerSessionHandle,
|
|
964
832
|
instructions: string,
|
|
965
833
|
): Effect.Effect<void, never> {
|
|
@@ -968,7 +836,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
968
836
|
if (
|
|
969
837
|
!worker ||
|
|
970
838
|
worker.record.status !== "running" ||
|
|
971
|
-
worker.
|
|
839
|
+
worker.workerEpoch !== workerEpoch ||
|
|
972
840
|
worker.session !== session
|
|
973
841
|
) {
|
|
974
842
|
return Effect.void;
|
|
@@ -982,7 +850,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
982
850
|
onSuccess: (outcome) => outcome,
|
|
983
851
|
}),
|
|
984
852
|
Effect.tap((outcome) => Effect.sync(() => {
|
|
985
|
-
this.settleOutcome(workerId,
|
|
853
|
+
this.settleOutcome(workerId, workerEpoch, session, outcome);
|
|
986
854
|
})),
|
|
987
855
|
Effect.asVoid,
|
|
988
856
|
);
|
|
@@ -991,12 +859,12 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
991
859
|
|
|
992
860
|
private settleCreationFailure(
|
|
993
861
|
workerId: WorkerId,
|
|
994
|
-
|
|
862
|
+
workerEpoch: number,
|
|
995
863
|
error: unknown,
|
|
996
864
|
): void {
|
|
997
865
|
this.settleActiveWorker(
|
|
998
866
|
workerId,
|
|
999
|
-
|
|
867
|
+
workerEpoch,
|
|
1000
868
|
["starting"],
|
|
1001
869
|
"failed",
|
|
1002
870
|
{
|
|
@@ -1010,12 +878,12 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1010
878
|
|
|
1011
879
|
private settleWorkflowDefect(
|
|
1012
880
|
workerId: WorkerId,
|
|
1013
|
-
|
|
881
|
+
workerEpoch: number,
|
|
1014
882
|
error: unknown,
|
|
1015
883
|
): void {
|
|
1016
884
|
this.settleActiveWorker(
|
|
1017
885
|
workerId,
|
|
1018
|
-
|
|
886
|
+
workerEpoch,
|
|
1019
887
|
["starting", "running", "stopping"],
|
|
1020
888
|
"failed",
|
|
1021
889
|
{
|
|
@@ -1029,14 +897,14 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1029
897
|
|
|
1030
898
|
private settleOutcome(
|
|
1031
899
|
workerId: WorkerId,
|
|
1032
|
-
|
|
900
|
+
workerEpoch: number,
|
|
1033
901
|
session: WorkerSessionHandle,
|
|
1034
902
|
outcome: WorkerOutcome,
|
|
1035
903
|
): void {
|
|
1036
904
|
const worker = this.current().workers.get(workerId);
|
|
1037
905
|
if (
|
|
1038
906
|
!worker ||
|
|
1039
|
-
worker.
|
|
907
|
+
worker.workerEpoch !== workerEpoch ||
|
|
1040
908
|
worker.session !== session ||
|
|
1041
909
|
worker.record.status !== "running"
|
|
1042
910
|
) {
|
|
@@ -1066,7 +934,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1066
934
|
}
|
|
1067
935
|
this.settleActiveWorker(
|
|
1068
936
|
workerId,
|
|
1069
|
-
|
|
937
|
+
workerEpoch,
|
|
1070
938
|
["running"],
|
|
1071
939
|
status,
|
|
1072
940
|
settledOutcome,
|
|
@@ -1077,9 +945,9 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1077
945
|
|
|
1078
946
|
private settleActiveWorker(
|
|
1079
947
|
workerId: WorkerId,
|
|
1080
|
-
|
|
948
|
+
workerEpoch: number,
|
|
1081
949
|
expectedStatuses: readonly WorkerRecord["status"][],
|
|
1082
|
-
status:
|
|
950
|
+
status: SettledWorkerStatus,
|
|
1083
951
|
outcome: WorkerOutcome,
|
|
1084
952
|
failureStage: SettlementFailureStage | undefined,
|
|
1085
953
|
dispose: boolean,
|
|
@@ -1089,7 +957,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1089
957
|
const worker = draft.workers.get(workerId);
|
|
1090
958
|
if (
|
|
1091
959
|
!worker ||
|
|
1092
|
-
worker.
|
|
960
|
+
worker.workerEpoch !== workerEpoch ||
|
|
1093
961
|
!expectedStatuses.includes(worker.record.status)
|
|
1094
962
|
) {
|
|
1095
963
|
return { value: undefined };
|
|
@@ -1102,7 +970,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1102
970
|
settledAt,
|
|
1103
971
|
};
|
|
1104
972
|
const actions: PostCommitAction[] = [];
|
|
1105
|
-
let settledWorker:
|
|
973
|
+
let settledWorker: WorkerEntry = { ...worker, record: settledRecord };
|
|
1106
974
|
if (dispose) {
|
|
1107
975
|
actions.push(...this.releaseWorkerResources(settledWorker));
|
|
1108
976
|
settledWorker = {
|
|
@@ -1128,7 +996,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1128
996
|
|
|
1129
997
|
private updateObservation(
|
|
1130
998
|
workerId: WorkerId,
|
|
1131
|
-
|
|
999
|
+
workerEpoch: number,
|
|
1132
1000
|
session: WorkerSessionHandle,
|
|
1133
1001
|
observation: WorkerSessionObservation,
|
|
1134
1002
|
): void {
|
|
@@ -1136,7 +1004,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1136
1004
|
const worker = draft.workers.get(workerId);
|
|
1137
1005
|
if (
|
|
1138
1006
|
!worker ||
|
|
1139
|
-
worker.
|
|
1007
|
+
worker.workerEpoch !== workerEpoch ||
|
|
1140
1008
|
worker.session !== session ||
|
|
1141
1009
|
(worker.record.status !== "starting" && worker.record.status !== "running")
|
|
1142
1010
|
) {
|
|
@@ -1208,7 +1076,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1208
1076
|
}
|
|
1209
1077
|
}
|
|
1210
1078
|
|
|
1211
|
-
const activeWorkers = workers.filter(
|
|
1079
|
+
const activeWorkers = workers.filter(isWorkerEntry);
|
|
1212
1080
|
const marked = this.markWorkersStopping(draft, activeWorkers, candidates);
|
|
1213
1081
|
return {
|
|
1214
1082
|
value: accepted(marked.completions),
|
|
@@ -1243,10 +1111,10 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1243
1111
|
}
|
|
1244
1112
|
|
|
1245
1113
|
// Commit stopping and one shared completion before post-commit physical abort
|
|
1246
|
-
// and
|
|
1114
|
+
// and worker-workflow interruption; every cancellation caller joins it.
|
|
1247
1115
|
private markWorkersStopping(
|
|
1248
|
-
draft:
|
|
1249
|
-
workers: readonly
|
|
1116
|
+
draft: OrchestrationState,
|
|
1117
|
+
workers: readonly WorkerEntry[],
|
|
1250
1118
|
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1251
1119
|
): {
|
|
1252
1120
|
readonly completions: readonly Deferred.Deferred<void>[];
|
|
@@ -1295,7 +1163,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1295
1163
|
if (worker?.record.status === "stopping") {
|
|
1296
1164
|
this.settleActiveWorker(
|
|
1297
1165
|
workerId,
|
|
1298
|
-
worker.
|
|
1166
|
+
worker.workerEpoch,
|
|
1299
1167
|
["stopping"],
|
|
1300
1168
|
"aborted",
|
|
1301
1169
|
{ status: "aborted" },
|
|
@@ -1324,7 +1192,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1324
1192
|
}
|
|
1325
1193
|
const removal = yield* FiberSet.run(
|
|
1326
1194
|
this.cancellations,
|
|
1327
|
-
FiberMap.remove(this.
|
|
1195
|
+
FiberMap.remove(this.workerWorkflows, workerId).pipe(
|
|
1328
1196
|
Effect.catchCause(() => Effect.void),
|
|
1329
1197
|
),
|
|
1330
1198
|
);
|
|
@@ -1334,7 +1202,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1334
1202
|
);
|
|
1335
1203
|
this.settleActiveWorker(
|
|
1336
1204
|
workerId,
|
|
1337
|
-
this.current().workers.get(workerId)?.
|
|
1205
|
+
this.current().workers.get(workerId)?.workerEpoch ?? -1,
|
|
1338
1206
|
["stopping"],
|
|
1339
1207
|
"aborted",
|
|
1340
1208
|
{ status: "aborted" },
|
|
@@ -1343,11 +1211,11 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1343
1211
|
);
|
|
1344
1212
|
}).pipe(
|
|
1345
1213
|
Effect.catchCause((cause) => Effect.sync(() => {
|
|
1346
|
-
const
|
|
1347
|
-
if (
|
|
1214
|
+
const workerEpoch = this.current().workers.get(workerId)?.workerEpoch;
|
|
1215
|
+
if (workerEpoch === undefined) return;
|
|
1348
1216
|
this.settleActiveWorker(
|
|
1349
1217
|
workerId,
|
|
1350
|
-
|
|
1218
|
+
workerEpoch,
|
|
1351
1219
|
["stopping"],
|
|
1352
1220
|
"failed",
|
|
1353
1221
|
{
|
|
@@ -1415,8 +1283,8 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1415
1283
|
}
|
|
1416
1284
|
|
|
1417
1285
|
private closeReadyWorker(
|
|
1418
|
-
draft:
|
|
1419
|
-
worker:
|
|
1286
|
+
draft: OrchestrationState,
|
|
1287
|
+
worker: WorkerEntry,
|
|
1420
1288
|
settledAt: number,
|
|
1421
1289
|
actions: PostCommitAction[] = [],
|
|
1422
1290
|
): PostCommitAction[] {
|
|
@@ -1437,7 +1305,7 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1437
1305
|
return actions;
|
|
1438
1306
|
}
|
|
1439
1307
|
|
|
1440
|
-
private releaseWorkerResources(worker:
|
|
1308
|
+
private releaseWorkerResources(worker: WorkerEntry): PostCommitAction[] {
|
|
1441
1309
|
const actions: PostCommitAction[] = [];
|
|
1442
1310
|
if (worker.observationRelease) {
|
|
1443
1311
|
actions.push(runAction(() => safelyCall(worker.observationRelease)));
|
|
@@ -1483,12 +1351,12 @@ class StatefulOrchestration implements OrchestrationService {
|
|
|
1483
1351
|
});
|
|
1484
1352
|
}
|
|
1485
1353
|
|
|
1486
|
-
private current():
|
|
1354
|
+
private current(): OrchestrationState {
|
|
1487
1355
|
return this.state;
|
|
1488
1356
|
}
|
|
1489
1357
|
|
|
1490
1358
|
private transact<A>(
|
|
1491
|
-
reducer: (draft:
|
|
1359
|
+
reducer: (draft: OrchestrationState) => TransactionMutation<A>,
|
|
1492
1360
|
): A {
|
|
1493
1361
|
const draft = makeDraft(this.state);
|
|
1494
1362
|
const mutation = reducer(draft);
|
|
@@ -1536,18 +1404,18 @@ export function orchestrationLayer(
|
|
|
1536
1404
|
Effect.gen(function* () {
|
|
1537
1405
|
const childSessions = yield* ChildSessions;
|
|
1538
1406
|
// Scope finalizers run in reverse acquisition order. Keep cleanup open while
|
|
1539
|
-
//
|
|
1407
|
+
// worker-workflow and cancellation interruption settle workers and enqueue disposal.
|
|
1540
1408
|
const cleanups = yield* FiberSet.make<void, never>();
|
|
1541
1409
|
const cancellations = yield* FiberSet.make<void, never>();
|
|
1542
1410
|
const runCancellation = yield* FiberSet.runtime(cancellations)<never>();
|
|
1543
|
-
const
|
|
1544
|
-
const
|
|
1411
|
+
const workerWorkflows = yield* FiberMap.make<WorkerId, void, never>();
|
|
1412
|
+
const runWorkerWorkflow = yield* FiberMap.runtime(workerWorkflows)<never>();
|
|
1545
1413
|
const clock = yield* Clock.Clock;
|
|
1546
1414
|
const shutdownCompletion = yield* Deferred.make<void>();
|
|
1547
|
-
return Orchestration.of(new
|
|
1415
|
+
return Orchestration.of(new OrchestrationEngine(
|
|
1548
1416
|
childSessions,
|
|
1549
|
-
|
|
1550
|
-
|
|
1417
|
+
workerWorkflows,
|
|
1418
|
+
runWorkerWorkflow,
|
|
1551
1419
|
cancellations,
|
|
1552
1420
|
runCancellation,
|
|
1553
1421
|
cleanups,
|
|
@@ -1559,10 +1427,10 @@ export function orchestrationLayer(
|
|
|
1559
1427
|
);
|
|
1560
1428
|
}
|
|
1561
1429
|
|
|
1562
|
-
function initialState():
|
|
1430
|
+
function initialState(): OrchestrationState {
|
|
1563
1431
|
return Object.freeze({
|
|
1564
|
-
workers: new Map<WorkerId,
|
|
1565
|
-
runs: new Map<RunId,
|
|
1432
|
+
workers: new Map<WorkerId, WorkerEntry>(),
|
|
1433
|
+
runs: new Map<RunId, RunEntry>(),
|
|
1566
1434
|
terminalWorkerOrder: [],
|
|
1567
1435
|
completedRunOrder: [],
|
|
1568
1436
|
settlementSequence: 0,
|
|
@@ -1570,7 +1438,7 @@ function initialState(): RuntimeState {
|
|
|
1570
1438
|
});
|
|
1571
1439
|
}
|
|
1572
1440
|
|
|
1573
|
-
function makeDraft(state:
|
|
1441
|
+
function makeDraft(state: OrchestrationState): OrchestrationState {
|
|
1574
1442
|
return {
|
|
1575
1443
|
...state,
|
|
1576
1444
|
workers: new Map(state.workers),
|
|
@@ -1581,13 +1449,13 @@ function makeDraft(state: RuntimeState): RuntimeState {
|
|
|
1581
1449
|
}
|
|
1582
1450
|
|
|
1583
1451
|
function completeRun(
|
|
1584
|
-
state:
|
|
1452
|
+
state: OrchestrationState,
|
|
1585
1453
|
workerId: WorkerId,
|
|
1586
1454
|
): PostCommitAction[] {
|
|
1587
1455
|
const worker = state.workers.get(workerId);
|
|
1588
1456
|
if (!worker) return [];
|
|
1589
1457
|
const run = state.runs.get(worker.record.runId);
|
|
1590
|
-
if (!run || !
|
|
1458
|
+
if (!run || !isActiveRunEntry(run) || !isSettledWorkerRecord(worker.record)) {
|
|
1591
1459
|
return [];
|
|
1592
1460
|
}
|
|
1593
1461
|
|
|
@@ -1600,53 +1468,35 @@ function completeRun(
|
|
|
1600
1468
|
}
|
|
1601
1469
|
|
|
1602
1470
|
function makeSettlement(
|
|
1603
|
-
draft:
|
|
1604
|
-
worker:
|
|
1471
|
+
draft: OrchestrationState,
|
|
1472
|
+
worker: WorkerEntry,
|
|
1605
1473
|
settledAt: number,
|
|
1606
1474
|
failureStage: SettlementFailureStage | undefined,
|
|
1607
1475
|
): PostCommitAction | undefined {
|
|
1608
1476
|
const run = draft.runs.get(worker.record.runId);
|
|
1609
|
-
if (!run || !
|
|
1477
|
+
if (!run || !isSettledWorkerRecord(worker.record)) {
|
|
1610
1478
|
return undefined;
|
|
1611
1479
|
}
|
|
1612
|
-
const runRecord = runtimeRunRecord(run);
|
|
1613
1480
|
const sequence = draft.settlementSequence + 1;
|
|
1614
1481
|
draft.settlementSequence = sequence;
|
|
1615
|
-
|
|
1616
|
-
|
|
1482
|
+
// Persisted generation stores workerEpoch, distinguishes follow-up settlements
|
|
1483
|
+
// for one stable worker ID, and makes event IDs generation-specific.
|
|
1484
|
+
const settlement = createWorkerSettlement({
|
|
1617
1485
|
sequence,
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
generation: worker.generation,
|
|
1622
|
-
mode: runRecord.mode,
|
|
1623
|
-
worker: worker.record.worker,
|
|
1624
|
-
title: worker.record.title,
|
|
1625
|
-
lifecycle: worker.record.lifecycle,
|
|
1626
|
-
status: worker.record.status,
|
|
1627
|
-
outcome: Object.freeze(copyOutcome(worker.record.outcome)),
|
|
1628
|
-
...(failureStage ? { failureStage } : {}),
|
|
1629
|
-
usage: Object.freeze(copyUsage(worker.record.usage)),
|
|
1630
|
-
startedAt: worker.record.startedAt,
|
|
1486
|
+
generation: worker.workerEpoch,
|
|
1487
|
+
run: runRecordFromEntry(run),
|
|
1488
|
+
worker: worker.record,
|
|
1631
1489
|
settledAt,
|
|
1632
|
-
...(
|
|
1633
|
-
? {
|
|
1634
|
-
synthesisGroupId: runRecord.synthesisGroupId,
|
|
1635
|
-
synthesisGroupSize: runRecord.synthesisGroupSize,
|
|
1636
|
-
}
|
|
1637
|
-
: {}),
|
|
1638
|
-
...(worker.record.sessionFile !== undefined
|
|
1639
|
-
? { sessionFile: worker.record.sessionFile }
|
|
1640
|
-
: {}),
|
|
1490
|
+
...(failureStage ? { failureStage } : {}),
|
|
1641
1491
|
});
|
|
1642
1492
|
return publishSettlement(
|
|
1643
1493
|
settlement,
|
|
1644
|
-
|
|
1494
|
+
isActiveRunEntry(run) ? run.settlementListener : undefined,
|
|
1645
1495
|
);
|
|
1646
1496
|
}
|
|
1647
1497
|
|
|
1648
1498
|
function stateActionsAfterPrune(
|
|
1649
|
-
draft:
|
|
1499
|
+
draft: OrchestrationState,
|
|
1650
1500
|
ownerSessionId: string,
|
|
1651
1501
|
): PostCommitAction[] {
|
|
1652
1502
|
const owners = pruneHistory(draft);
|
|
@@ -1654,22 +1504,17 @@ function stateActionsAfterPrune(
|
|
|
1654
1504
|
return publishOwners(owners);
|
|
1655
1505
|
}
|
|
1656
1506
|
|
|
1657
|
-
function pruneHistory(draft:
|
|
1507
|
+
function pruneHistory(draft: OrchestrationState): Set<string> {
|
|
1658
1508
|
const owners = new Set<string>();
|
|
1659
1509
|
while (draft.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
|
|
1660
1510
|
const runId = draft.completedRunOrder.shift();
|
|
1661
1511
|
if (!runId) break;
|
|
1662
1512
|
const run = draft.runs.get(runId);
|
|
1663
|
-
if (run) owners.add(
|
|
1513
|
+
if (run) owners.add(runRecordFromEntry(run).ownerSessionId);
|
|
1664
1514
|
draft.runs.delete(runId);
|
|
1665
1515
|
}
|
|
1666
1516
|
while (draft.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
|
|
1667
|
-
const
|
|
1668
|
-
const worker = draft.workers.get(workerId);
|
|
1669
|
-
return !worker || isTerminalWorkerStatus(worker.record.status);
|
|
1670
|
-
});
|
|
1671
|
-
if (index < 0) break;
|
|
1672
|
-
const removed = draft.terminalWorkerOrder.splice(index, 1)[0];
|
|
1517
|
+
const removed = draft.terminalWorkerOrder.shift();
|
|
1673
1518
|
if (!removed) break;
|
|
1674
1519
|
const worker = draft.workers.get(removed);
|
|
1675
1520
|
if (worker) owners.add(worker.record.ownerSessionId);
|
|
@@ -1679,7 +1524,7 @@ function pruneHistory(draft: RuntimeState): Set<string> {
|
|
|
1679
1524
|
}
|
|
1680
1525
|
|
|
1681
1526
|
function rememberTerminalWorker(
|
|
1682
|
-
draft:
|
|
1527
|
+
draft: OrchestrationState,
|
|
1683
1528
|
workerId: WorkerId,
|
|
1684
1529
|
): void {
|
|
1685
1530
|
if (!draft.terminalWorkerOrder.includes(workerId)) {
|
|
@@ -1688,13 +1533,13 @@ function rememberTerminalWorker(
|
|
|
1688
1533
|
}
|
|
1689
1534
|
|
|
1690
1535
|
function snapshotFor(
|
|
1691
|
-
state:
|
|
1536
|
+
state: OrchestrationState,
|
|
1692
1537
|
ownerSessionId: string,
|
|
1693
|
-
):
|
|
1538
|
+
): OwnerSnapshot {
|
|
1694
1539
|
return Object.freeze({
|
|
1695
1540
|
runs: Object.freeze(
|
|
1696
1541
|
[...state.runs.values()]
|
|
1697
|
-
.map(
|
|
1542
|
+
.map(runRecordFromEntry)
|
|
1698
1543
|
.filter((run) => run.ownerSessionId === ownerSessionId)
|
|
1699
1544
|
.map(copyRunRecord),
|
|
1700
1545
|
),
|
|
@@ -1707,11 +1552,11 @@ function snapshotFor(
|
|
|
1707
1552
|
});
|
|
1708
1553
|
}
|
|
1709
1554
|
|
|
1710
|
-
function
|
|
1711
|
-
return
|
|
1555
|
+
function runRecordFromEntry(run: RunEntry): RunRecord {
|
|
1556
|
+
return isActiveRunEntry(run) ? run.record : run;
|
|
1712
1557
|
}
|
|
1713
1558
|
|
|
1714
|
-
function
|
|
1559
|
+
function isActiveRunEntry(run: RunEntry): run is ActiveRunEntry {
|
|
1715
1560
|
return "_tag" in run;
|
|
1716
1561
|
}
|
|
1717
1562
|
|
|
@@ -1803,7 +1648,7 @@ function runAction(run: () => void): PostCommitAction {
|
|
|
1803
1648
|
}
|
|
1804
1649
|
|
|
1805
1650
|
function openDecision(
|
|
1806
|
-
draft:
|
|
1651
|
+
draft: OrchestrationState,
|
|
1807
1652
|
operation: OrchestrationOperation,
|
|
1808
1653
|
): Decision<void> {
|
|
1809
1654
|
return draft.lifecycle === "open"
|
|
@@ -1811,16 +1656,16 @@ function openDecision(
|
|
|
1811
1656
|
: rejected(
|
|
1812
1657
|
operation,
|
|
1813
1658
|
"shutdown",
|
|
1814
|
-
"
|
|
1659
|
+
"Orchestration is shutting down",
|
|
1815
1660
|
);
|
|
1816
1661
|
}
|
|
1817
1662
|
|
|
1818
1663
|
function ownedWorkerDecision(
|
|
1819
|
-
draft:
|
|
1664
|
+
draft: OrchestrationState,
|
|
1820
1665
|
operation: OrchestrationOperation,
|
|
1821
1666
|
ownerSessionId: string,
|
|
1822
1667
|
workerId: WorkerId,
|
|
1823
|
-
): Decision<
|
|
1668
|
+
): Decision<WorkerEntry> {
|
|
1824
1669
|
const worker = draft.workers.get(workerId);
|
|
1825
1670
|
return !worker || worker.record.ownerSessionId !== ownerSessionId
|
|
1826
1671
|
? rejected(
|
|
@@ -1832,11 +1677,11 @@ function ownedWorkerDecision(
|
|
|
1832
1677
|
}
|
|
1833
1678
|
|
|
1834
1679
|
function readyInteractiveDecision(
|
|
1835
|
-
draft:
|
|
1680
|
+
draft: OrchestrationState,
|
|
1836
1681
|
ownerSessionId: string,
|
|
1837
1682
|
workerId: WorkerId,
|
|
1838
1683
|
): Decision<{
|
|
1839
|
-
readonly worker:
|
|
1684
|
+
readonly worker: WorkerEntry;
|
|
1840
1685
|
readonly session: WorkerSessionHandle;
|
|
1841
1686
|
}> {
|
|
1842
1687
|
const open = openDecision(draft, "sendInteractive");
|
|
@@ -1863,137 +1708,6 @@ function readyInteractiveDecision(
|
|
|
1863
1708
|
return accepted({ worker, session: worker.session });
|
|
1864
1709
|
}
|
|
1865
1710
|
|
|
1866
|
-
function accepted<A>(value: A): Decision<A> {
|
|
1867
|
-
return { _tag: "accepted", value };
|
|
1868
|
-
}
|
|
1869
|
-
|
|
1870
|
-
function rejected(
|
|
1871
|
-
operation: OrchestrationOperation,
|
|
1872
|
-
reason: OrchestrationRejectionReason,
|
|
1873
|
-
message: string,
|
|
1874
|
-
): Decision<never> {
|
|
1875
|
-
return {
|
|
1876
|
-
_tag: "rejected",
|
|
1877
|
-
error: actionRejection(operation, reason, message),
|
|
1878
|
-
};
|
|
1879
|
-
}
|
|
1880
|
-
|
|
1881
|
-
function actionRejection(
|
|
1882
|
-
operation: OrchestrationOperation,
|
|
1883
|
-
reason: OrchestrationRejectionReason,
|
|
1884
|
-
message: string,
|
|
1885
|
-
): OrchestrationActionRejected {
|
|
1886
|
-
return new OrchestrationActionRejected({ operation, reason, message });
|
|
1887
|
-
}
|
|
1888
|
-
|
|
1889
|
-
function rejectAction(
|
|
1890
|
-
operation: OrchestrationOperation,
|
|
1891
|
-
reason: OrchestrationRejectionReason,
|
|
1892
|
-
message: string,
|
|
1893
|
-
): Effect.Effect<never, OrchestrationActionRejected> {
|
|
1894
|
-
return Effect.fail(actionRejection(operation, reason, message));
|
|
1895
|
-
}
|
|
1896
|
-
|
|
1897
|
-
function validateContextOwner(
|
|
1898
|
-
operation: OrchestrationOperation,
|
|
1899
|
-
ownerSessionId: string,
|
|
1900
|
-
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1901
|
-
return typeof ownerSessionId !== "string" || ownerSessionId.trim() === ""
|
|
1902
|
-
? rejectAction(
|
|
1903
|
-
operation,
|
|
1904
|
-
"validation",
|
|
1905
|
-
"ownerSessionId must not be blank",
|
|
1906
|
-
)
|
|
1907
|
-
: Effect.void;
|
|
1908
|
-
}
|
|
1909
|
-
|
|
1910
|
-
function validateWorkerId(
|
|
1911
|
-
operation: OrchestrationOperation,
|
|
1912
|
-
workerId: string,
|
|
1913
|
-
): Effect.Effect<WorkerId, OrchestrationActionRejected> {
|
|
1914
|
-
if (typeof workerId !== "string" || workerId.trim() === "") {
|
|
1915
|
-
return rejectAction(
|
|
1916
|
-
operation,
|
|
1917
|
-
"validation",
|
|
1918
|
-
"worker_id must not be blank",
|
|
1919
|
-
);
|
|
1920
|
-
}
|
|
1921
|
-
return Schema.decodeUnknownEffect(WorkerId)(workerId).pipe(
|
|
1922
|
-
Effect.mapError(() => actionRejection(
|
|
1923
|
-
operation,
|
|
1924
|
-
"validation",
|
|
1925
|
-
"worker_id must use the canonical worker- prefix",
|
|
1926
|
-
)),
|
|
1927
|
-
);
|
|
1928
|
-
}
|
|
1929
|
-
|
|
1930
|
-
function validateMode(
|
|
1931
|
-
operation: OrchestrationOperation,
|
|
1932
|
-
mode: RunMode,
|
|
1933
|
-
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1934
|
-
return mode !== "async" && mode !== "inline"
|
|
1935
|
-
? rejectAction(operation, "validation", "Invalid orchestration mode")
|
|
1936
|
-
: Effect.void;
|
|
1937
|
-
}
|
|
1938
|
-
|
|
1939
|
-
function validateText(
|
|
1940
|
-
operation: OrchestrationOperation,
|
|
1941
|
-
name: string,
|
|
1942
|
-
value: string,
|
|
1943
|
-
maximumLength: number,
|
|
1944
|
-
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1945
|
-
if (typeof value !== "string" || value.trim() === "") {
|
|
1946
|
-
return rejectAction(operation, "validation", `${name} must not be blank`);
|
|
1947
|
-
}
|
|
1948
|
-
return value.length > maximumLength
|
|
1949
|
-
? rejectAction(
|
|
1950
|
-
operation,
|
|
1951
|
-
"validation",
|
|
1952
|
-
`${name} must be at most ${maximumLength} characters`,
|
|
1953
|
-
)
|
|
1954
|
-
: Effect.void;
|
|
1955
|
-
}
|
|
1956
|
-
|
|
1957
|
-
type ValidatedAbortTarget =
|
|
1958
|
-
| {
|
|
1959
|
-
readonly _tag: "ids";
|
|
1960
|
-
readonly workerIds: readonly WorkerId[];
|
|
1961
|
-
}
|
|
1962
|
-
| {
|
|
1963
|
-
readonly _tag: "all";
|
|
1964
|
-
};
|
|
1965
|
-
|
|
1966
|
-
function validateAbortTarget(
|
|
1967
|
-
target: AbortTarget,
|
|
1968
|
-
): Effect.Effect<ValidatedAbortTarget, OrchestrationActionRejected> {
|
|
1969
|
-
return Effect.gen(function* () {
|
|
1970
|
-
if (!target || typeof target !== "object") {
|
|
1971
|
-
return yield* rejectAction("abort", "target", "Invalid abort target");
|
|
1972
|
-
}
|
|
1973
|
-
const selected = [target.workerIds !== undefined, target.all !== undefined]
|
|
1974
|
-
.filter(Boolean).length;
|
|
1975
|
-
if (selected !== 1 || (target.all !== undefined && target.all !== true)) {
|
|
1976
|
-
return yield* rejectAction(
|
|
1977
|
-
"abort",
|
|
1978
|
-
"target",
|
|
1979
|
-
"Abort target must specify exactly one of workerIds or all: true",
|
|
1980
|
-
);
|
|
1981
|
-
}
|
|
1982
|
-
if (target.workerIds === undefined) return { _tag: "all" };
|
|
1983
|
-
if (!Array.isArray(target.workerIds) || target.workerIds.length === 0) {
|
|
1984
|
-
return yield* rejectAction(
|
|
1985
|
-
"abort",
|
|
1986
|
-
"target",
|
|
1987
|
-
"workerIds must contain at least one worker ID",
|
|
1988
|
-
);
|
|
1989
|
-
}
|
|
1990
|
-
const workerIds = yield* Effect.all(
|
|
1991
|
-
[...new Set(target.workerIds)].map((id) => validateWorkerId("abort", id)),
|
|
1992
|
-
);
|
|
1993
|
-
return { _tag: "ids", workerIds };
|
|
1994
|
-
});
|
|
1995
|
-
}
|
|
1996
|
-
|
|
1997
1711
|
function makeCancellationCandidates(
|
|
1998
1712
|
workerIds: readonly WorkerId[],
|
|
1999
1713
|
): Effect.Effect<ReadonlyMap<WorkerId, Deferred.Deferred<void>>> {
|
|
@@ -2003,7 +1717,7 @@ function makeCancellationCandidates(
|
|
|
2003
1717
|
}
|
|
2004
1718
|
|
|
2005
1719
|
function activeWorkerIds(
|
|
2006
|
-
state:
|
|
1720
|
+
state: OrchestrationState,
|
|
2007
1721
|
ownerSessionId?: string,
|
|
2008
1722
|
): WorkerId[] {
|
|
2009
1723
|
return [...state.workers.values()]
|
|
@@ -2025,9 +1739,9 @@ function awaitAll(
|
|
|
2025
1739
|
);
|
|
2026
1740
|
}
|
|
2027
1741
|
|
|
2028
|
-
function
|
|
2029
|
-
worker:
|
|
2030
|
-
): worker is
|
|
1742
|
+
function isWorkerEntry(
|
|
1743
|
+
worker: WorkerEntry | undefined,
|
|
1744
|
+
): worker is WorkerEntry {
|
|
2031
1745
|
return worker !== undefined;
|
|
2032
1746
|
}
|
|
2033
1747
|
|
|
@@ -2035,13 +1749,9 @@ function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
|
|
|
2035
1749
|
return status === "starting" || status === "running" || status === "stopping";
|
|
2036
1750
|
}
|
|
2037
1751
|
|
|
2038
|
-
function
|
|
1752
|
+
function isSettledWorkerRecord(
|
|
2039
1753
|
record: WorkerRecord,
|
|
2040
|
-
): record is
|
|
2041
|
-
readonly status: RunResult["status"];
|
|
2042
|
-
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2043
|
-
readonly settledAt: number;
|
|
2044
|
-
} {
|
|
1754
|
+
): record is SettledWorkerRecord {
|
|
2045
1755
|
return (
|
|
2046
1756
|
(record.status === "completed" ||
|
|
2047
1757
|
record.status === "ready" ||
|
|
@@ -2112,13 +1822,9 @@ function freezeAcceptedRun(id: RunId, workerId: WorkerId): AcceptedRun {
|
|
|
2112
1822
|
|
|
2113
1823
|
function freezeCompletedRun(
|
|
2114
1824
|
run: RunRecord,
|
|
2115
|
-
record:
|
|
2116
|
-
readonly status: RunResult["status"];
|
|
2117
|
-
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2118
|
-
readonly settledAt: number;
|
|
2119
|
-
},
|
|
1825
|
+
record: SettledWorkerRecord,
|
|
2120
1826
|
): CompletedRun {
|
|
2121
|
-
const result:
|
|
1827
|
+
const result: WorkerRunResult = Object.freeze({
|
|
2122
1828
|
workerId: record.id,
|
|
2123
1829
|
worker: record.worker,
|
|
2124
1830
|
title: record.title,
|