@zachwill/pi-orchestrate 0.7.2 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extension/catalog.ts +70 -75
- package/extension/contract.ts +2 -2
- package/extension/delivery.ts +49 -7
- package/extension/domain.ts +83 -58
- package/extension/host.ts +321 -31
- package/extension/index.ts +36 -29
- package/extension/presentation.ts +42 -89
- package/extension/runtime.ts +1705 -887
- package/extension/tools.ts +146 -246
- package/extension/tui.ts +50 -0
- package/extension/worker-session.ts +517 -191
- package/extension/worker-settlement.ts +127 -47
- package/package.json +2 -2
- package/extension/scheduler.ts +0 -85
package/extension/runtime.ts
CHANGED
|
@@ -1,49 +1,53 @@
|
|
|
1
1
|
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
2
|
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
Cause,
|
|
5
|
+
Clock,
|
|
6
|
+
Context,
|
|
7
|
+
Deferred,
|
|
8
|
+
Effect,
|
|
9
|
+
Exit,
|
|
10
|
+
Fiber,
|
|
11
|
+
FiberMap,
|
|
12
|
+
FiberSet,
|
|
13
|
+
Layer,
|
|
14
|
+
Schema,
|
|
15
|
+
} from "effect";
|
|
4
16
|
import {
|
|
5
17
|
CANCELLATION_GRACE_MS,
|
|
6
18
|
EMPTY_WORKER_USAGE,
|
|
7
19
|
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
8
20
|
MAX_WORKER_TITLE_LENGTH,
|
|
21
|
+
OrchestrateTaskInput,
|
|
22
|
+
RunId,
|
|
23
|
+
WorkerId,
|
|
9
24
|
createRandomIdFactories,
|
|
10
25
|
findWorkerByName,
|
|
11
26
|
isTerminalWorkerStatus,
|
|
12
27
|
transitionWorkerStatus,
|
|
13
28
|
type OrchestrateIdFactories,
|
|
14
|
-
type OrchestrateTaskInput,
|
|
15
|
-
type RunId,
|
|
16
29
|
type RunMode,
|
|
17
30
|
type RunRecord,
|
|
18
31
|
type WorkerCatalog,
|
|
19
32
|
type WorkerDefinition,
|
|
20
|
-
type WorkerId,
|
|
21
33
|
type WorkerOutcome,
|
|
22
34
|
type WorkerRecord,
|
|
23
35
|
type WorkerUsage,
|
|
24
36
|
} from "./domain.js";
|
|
25
37
|
import {
|
|
26
|
-
|
|
27
|
-
type
|
|
28
|
-
|
|
29
|
-
import {
|
|
30
|
-
resolveWorkerModel,
|
|
31
|
-
type WorkerSessionFactory,
|
|
38
|
+
ChildSessions,
|
|
39
|
+
type ChildSessionsService,
|
|
40
|
+
type WorkerSessionAbortError,
|
|
32
41
|
type WorkerSessionHandle,
|
|
42
|
+
type WorkerSessionObservation,
|
|
33
43
|
} from "./worker-session.js";
|
|
34
44
|
import type {
|
|
35
45
|
SettlementFailureStage,
|
|
36
46
|
WorkerSettlement,
|
|
37
47
|
} from "./worker-settlement.js";
|
|
38
48
|
|
|
39
|
-
export type {
|
|
40
|
-
SettlementFailureStage,
|
|
41
|
-
WorkerSettlement,
|
|
42
|
-
} from "./worker-settlement.js";
|
|
43
|
-
|
|
44
49
|
export const MAX_TERMINAL_WORKER_HISTORY = 100;
|
|
45
50
|
export const MAX_COMPLETED_RUN_HISTORY = 100;
|
|
46
|
-
/** Shutdown waits this long for interrupted bootstrap/prompt promises, then returns best-effort. */
|
|
47
51
|
export const SHUTDOWN_CLEANUP_GRACE_MS = CANCELLATION_GRACE_MS;
|
|
48
52
|
|
|
49
53
|
export interface OrchestrationContext {
|
|
@@ -66,7 +70,7 @@ export interface AcceptedRun {
|
|
|
66
70
|
readonly workerId: WorkerId;
|
|
67
71
|
}
|
|
68
72
|
|
|
69
|
-
export interface
|
|
73
|
+
export interface RunResult {
|
|
70
74
|
readonly workerId: WorkerId;
|
|
71
75
|
readonly worker: string;
|
|
72
76
|
readonly title: string;
|
|
@@ -82,7 +86,7 @@ export interface CompletedRun {
|
|
|
82
86
|
readonly id: RunId;
|
|
83
87
|
readonly ownerSessionId: string;
|
|
84
88
|
readonly mode: RunMode;
|
|
85
|
-
readonly result:
|
|
89
|
+
readonly result: RunResult;
|
|
86
90
|
}
|
|
87
91
|
|
|
88
92
|
export interface RuntimeSnapshot {
|
|
@@ -92,433 +96,699 @@ export interface RuntimeSnapshot {
|
|
|
92
96
|
|
|
93
97
|
export type SettlementListener = (settlement: WorkerSettlement) => void;
|
|
94
98
|
export type UnsubscribeSettlement = () => void;
|
|
95
|
-
export type StateListener = (
|
|
96
|
-
|
|
97
|
-
export type AbortTarget =
|
|
98
|
-
| {
|
|
99
|
-
readonly workerIds: readonly WorkerId[];
|
|
100
|
-
readonly all?: never;
|
|
101
|
-
}
|
|
102
|
-
| {
|
|
103
|
-
readonly all: true;
|
|
104
|
-
readonly workerIds?: never;
|
|
105
|
-
};
|
|
99
|
+
export type StateListener = (snapshot: RuntimeSnapshot) => void;
|
|
106
100
|
|
|
107
|
-
export
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
wait(promise: Promise<unknown>, timeoutMs: number): Promise<DeadlineResult>;
|
|
101
|
+
export interface AbortTarget {
|
|
102
|
+
readonly workerIds?: readonly string[];
|
|
103
|
+
readonly all?: boolean;
|
|
111
104
|
}
|
|
112
105
|
|
|
113
|
-
export interface
|
|
114
|
-
readonly workerSessionFactory: WorkerSessionFactory;
|
|
106
|
+
export interface OrchestrationLayerOptions {
|
|
115
107
|
readonly idFactories?: OrchestrateIdFactories;
|
|
116
|
-
readonly clock?: () => number;
|
|
117
|
-
readonly scheduler?: WorkflowScheduler<WorkerId>;
|
|
118
|
-
readonly bestEffortDeadline?: BestEffortDeadline;
|
|
119
108
|
}
|
|
120
109
|
|
|
121
|
-
|
|
110
|
+
const OrchestrationOperation = Schema.Literals([
|
|
111
|
+
"orchestrate",
|
|
112
|
+
"sendInteractive",
|
|
113
|
+
"abort",
|
|
114
|
+
"closeInteractive",
|
|
115
|
+
"snapshot",
|
|
116
|
+
]);
|
|
117
|
+
type OrchestrationOperation = typeof OrchestrationOperation.Type;
|
|
118
|
+
|
|
119
|
+
const OrchestrationRejectionReason = Schema.Literals([
|
|
120
|
+
"shutdown",
|
|
121
|
+
"validation",
|
|
122
|
+
"ownership",
|
|
123
|
+
"target",
|
|
124
|
+
"worker-state",
|
|
125
|
+
"unknown-worker",
|
|
126
|
+
"model-unavailable",
|
|
127
|
+
]);
|
|
128
|
+
type OrchestrationRejectionReason = typeof OrchestrationRejectionReason.Type;
|
|
129
|
+
|
|
130
|
+
export class OrchestrationActionRejected extends Schema.TaggedError<OrchestrationActionRejected>()(
|
|
131
|
+
"Orchestration.ActionRejected",
|
|
132
|
+
{
|
|
133
|
+
operation: OrchestrationOperation,
|
|
134
|
+
reason: OrchestrationRejectionReason,
|
|
135
|
+
message: Schema.String,
|
|
136
|
+
},
|
|
137
|
+
) {}
|
|
138
|
+
|
|
139
|
+
export interface OrchestrationService {
|
|
122
140
|
orchestrate(
|
|
123
141
|
context: OrchestrationContext,
|
|
124
142
|
task: OrchestrateTaskInput,
|
|
125
143
|
mode: "async",
|
|
126
|
-
signal?: AbortSignal,
|
|
127
144
|
onSettlement?: SettlementListener,
|
|
128
|
-
):
|
|
145
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
129
146
|
orchestrate(
|
|
130
147
|
context: OrchestrationContext,
|
|
131
148
|
task: OrchestrateTaskInput,
|
|
132
149
|
mode: "inline",
|
|
133
|
-
signal?: AbortSignal,
|
|
134
150
|
onSettlement?: SettlementListener,
|
|
135
|
-
):
|
|
151
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
136
152
|
orchestrate(
|
|
137
153
|
context: OrchestrationContext,
|
|
138
154
|
task: OrchestrateTaskInput,
|
|
139
155
|
mode: RunMode,
|
|
140
|
-
signal?: AbortSignal,
|
|
141
156
|
onSettlement?: SettlementListener,
|
|
142
|
-
):
|
|
157
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
143
158
|
sendInteractive(
|
|
144
159
|
context: OrchestrationContext,
|
|
145
|
-
workerId:
|
|
160
|
+
workerId: string,
|
|
146
161
|
instructions: string,
|
|
147
162
|
mode: "async",
|
|
148
|
-
signal?: AbortSignal,
|
|
149
163
|
onSettlement?: SettlementListener,
|
|
150
|
-
):
|
|
164
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
151
165
|
sendInteractive(
|
|
152
166
|
context: OrchestrationContext,
|
|
153
|
-
workerId:
|
|
167
|
+
workerId: string,
|
|
154
168
|
instructions: string,
|
|
155
169
|
mode: "inline",
|
|
156
|
-
signal?: AbortSignal,
|
|
157
170
|
onSettlement?: SettlementListener,
|
|
158
|
-
):
|
|
171
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
159
172
|
sendInteractive(
|
|
160
173
|
context: OrchestrationContext,
|
|
161
|
-
workerId:
|
|
174
|
+
workerId: string,
|
|
162
175
|
instructions: string,
|
|
163
176
|
mode: RunMode,
|
|
164
|
-
signal?: AbortSignal,
|
|
165
177
|
onSettlement?: SettlementListener,
|
|
166
|
-
):
|
|
167
|
-
abort
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
178
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
179
|
+
readonly abort: (
|
|
180
|
+
ownerSessionId: string,
|
|
181
|
+
target: AbortTarget,
|
|
182
|
+
) => Effect.Effect<void, OrchestrationActionRejected>;
|
|
183
|
+
readonly closeInteractive: (
|
|
184
|
+
ownerSessionId: string,
|
|
185
|
+
workerId: string,
|
|
186
|
+
) => Effect.Effect<void, OrchestrationActionRejected>;
|
|
187
|
+
readonly snapshot: (
|
|
188
|
+
ownerSessionId: string,
|
|
189
|
+
) => Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected>;
|
|
190
|
+
readonly subscribeSettlement: (
|
|
191
|
+
listener: SettlementListener,
|
|
192
|
+
) => UnsubscribeSettlement;
|
|
193
|
+
readonly subscribeState: (
|
|
194
|
+
ownerSessionId: string,
|
|
195
|
+
listener: StateListener,
|
|
196
|
+
) => () => void;
|
|
197
|
+
readonly shutdown: () => Effect.Effect<void>;
|
|
173
198
|
}
|
|
174
199
|
|
|
175
|
-
|
|
200
|
+
export class Orchestration extends Context.Service<Orchestration, OrchestrationService>()(
|
|
201
|
+
"@zachwill/pi-orchestrate/Orchestration",
|
|
202
|
+
) {}
|
|
203
|
+
|
|
204
|
+
interface RuntimeWorker {
|
|
205
|
+
readonly record: WorkerRecord;
|
|
176
206
|
readonly context: OrchestrationContext;
|
|
177
207
|
readonly definition: WorkerDefinition;
|
|
178
|
-
generation: number;
|
|
179
|
-
session?: WorkerSessionHandle;
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
unsubscribeMessageDirection?: () => void;
|
|
208
|
+
readonly generation: number;
|
|
209
|
+
readonly session?: WorkerSessionHandle;
|
|
210
|
+
readonly observationRelease?: () => void;
|
|
211
|
+
readonly cancellation?: Deferred.Deferred<void>;
|
|
183
212
|
}
|
|
184
213
|
|
|
185
|
-
interface
|
|
186
|
-
readonly
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
214
|
+
interface RunningRuntimeRun {
|
|
215
|
+
readonly _tag: "running";
|
|
216
|
+
readonly record: RunRecord;
|
|
217
|
+
readonly completion: Deferred.Deferred<CompletedRun>;
|
|
218
|
+
readonly settlementListener?: SettlementListener;
|
|
190
219
|
}
|
|
191
220
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
221
|
+
type RuntimeRun = RunningRuntimeRun | RunRecord;
|
|
222
|
+
type RuntimeLifecycle = "open" | "shutting-down" | "shutdown";
|
|
223
|
+
|
|
224
|
+
interface RuntimeState {
|
|
225
|
+
workers: Map<WorkerId, RuntimeWorker>;
|
|
226
|
+
runs: Map<RunId, RuntimeRun>;
|
|
227
|
+
terminalWorkerOrder: WorkerId[];
|
|
228
|
+
completedRunOrder: RunId[];
|
|
229
|
+
settlementSequence: number;
|
|
230
|
+
lifecycle: RuntimeLifecycle;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
type CommittedAction = () => void;
|
|
234
|
+
type PostCommitAction = (
|
|
235
|
+
state: RuntimeState,
|
|
236
|
+
settlementListeners: ReadonlySet<SettlementListener>,
|
|
237
|
+
stateListeners: ReadonlyMap<string, ReadonlySet<StateListener>>,
|
|
238
|
+
) => CommittedAction;
|
|
239
|
+
|
|
240
|
+
interface TransactionMutation<A> {
|
|
241
|
+
readonly value: A;
|
|
242
|
+
readonly actions?: readonly PostCommitAction[];
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
type Decision<A> =
|
|
246
|
+
| {
|
|
247
|
+
readonly _tag: "accepted";
|
|
248
|
+
readonly value: A;
|
|
249
|
+
}
|
|
250
|
+
| {
|
|
251
|
+
readonly _tag: "rejected";
|
|
252
|
+
readonly error: OrchestrationActionRejected;
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
class StatefulOrchestration implements OrchestrationService {
|
|
256
|
+
private readonly actionQueue: CommittedAction[] = [];
|
|
222
257
|
private readonly settlementListeners = new Set<SettlementListener>();
|
|
223
|
-
private readonly
|
|
224
|
-
private
|
|
225
|
-
private
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
258
|
+
private readonly stateListeners = new Map<string, Set<StateListener>>();
|
|
259
|
+
private drainingActions = false;
|
|
260
|
+
private state: RuntimeState;
|
|
261
|
+
|
|
262
|
+
constructor(
|
|
263
|
+
private readonly childSessions: ChildSessionsService,
|
|
264
|
+
private readonly generations: FiberMap.FiberMap<WorkerId, void, never>,
|
|
265
|
+
private readonly runGeneration: (
|
|
266
|
+
key: WorkerId,
|
|
267
|
+
effect: Effect.Effect<void, never>,
|
|
268
|
+
) => Fiber.Fiber<void, never>,
|
|
269
|
+
private readonly cancellations: FiberSet.FiberSet<void, never>,
|
|
270
|
+
private readonly runCancellation: (
|
|
271
|
+
effect: Effect.Effect<void, never>,
|
|
272
|
+
) => Fiber.Fiber<void, never>,
|
|
273
|
+
private readonly cleanups: FiberSet.FiberSet<void, never>,
|
|
274
|
+
private readonly runCleanup: (
|
|
275
|
+
effect: Effect.Effect<void, never>,
|
|
276
|
+
) => Fiber.Fiber<void, never>,
|
|
277
|
+
private readonly clock: Clock.Clock,
|
|
278
|
+
private readonly idFactories: OrchestrateIdFactories,
|
|
279
|
+
private readonly shutdownCompletion: Deferred.Deferred<void>,
|
|
280
|
+
) {
|
|
281
|
+
this.state = initialState();
|
|
236
282
|
}
|
|
237
283
|
|
|
238
284
|
orchestrate(
|
|
239
285
|
context: OrchestrationContext,
|
|
240
286
|
task: OrchestrateTaskInput,
|
|
241
287
|
mode: "async",
|
|
242
|
-
signal?: AbortSignal,
|
|
243
288
|
onSettlement?: SettlementListener,
|
|
244
|
-
):
|
|
289
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
245
290
|
orchestrate(
|
|
246
291
|
context: OrchestrationContext,
|
|
247
292
|
task: OrchestrateTaskInput,
|
|
248
293
|
mode: "inline",
|
|
249
|
-
signal?: AbortSignal,
|
|
250
294
|
onSettlement?: SettlementListener,
|
|
251
|
-
):
|
|
295
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
252
296
|
orchestrate(
|
|
253
297
|
context: OrchestrationContext,
|
|
254
298
|
task: OrchestrateTaskInput,
|
|
255
299
|
mode: RunMode,
|
|
256
|
-
signal?: AbortSignal,
|
|
257
300
|
onSettlement?: SettlementListener,
|
|
258
|
-
):
|
|
259
|
-
|
|
301
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
302
|
+
orchestrate(
|
|
260
303
|
context: OrchestrationContext,
|
|
261
304
|
task: OrchestrateTaskInput,
|
|
262
305
|
mode: RunMode,
|
|
263
|
-
signal?: AbortSignal,
|
|
264
306
|
onSettlement?: SettlementListener,
|
|
265
|
-
):
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const record: WorkerRecord = {
|
|
289
|
-
id: workerId,
|
|
290
|
-
worker: definition.name,
|
|
291
|
-
ownerSessionId: context.ownerSessionId,
|
|
292
|
-
runId,
|
|
293
|
-
title: task.title,
|
|
294
|
-
instructions: task.instructions,
|
|
295
|
-
lifecycle: definition.lifecycle,
|
|
296
|
-
status: "starting",
|
|
297
|
-
usage: copyUsage(EMPTY_WORKER_USAGE),
|
|
298
|
-
messageDirection: "to-model",
|
|
299
|
-
startedAt: this.clock(),
|
|
300
|
-
};
|
|
307
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
308
|
+
return Effect.fn("Orchestration.orchestrate")(function* (
|
|
309
|
+
this: StatefulOrchestration,
|
|
310
|
+
) {
|
|
311
|
+
const validated = yield* this.validateTask(context, task, mode);
|
|
312
|
+
const preflight = this.preflightOpen("orchestrate");
|
|
313
|
+
if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
|
|
314
|
+
// Keep user-supplied ID factories outside the transaction: a reentrant
|
|
315
|
+
// factory cannot commit an outer stale draft. Admission rechecks all
|
|
316
|
+
// authority below, so a race may burn an ID but cannot create a run.
|
|
317
|
+
const runId = this.idFactories.runId();
|
|
318
|
+
const workerId = this.idFactories.workerId();
|
|
319
|
+
const completion = yield* Deferred.make<CompletedRun>();
|
|
320
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
321
|
+
const runRecord = makeRunRecord(runId, workerId, context, mode, now);
|
|
322
|
+
const workerRecord = makeWorkerRecord(
|
|
323
|
+
workerId,
|
|
324
|
+
runId,
|
|
325
|
+
context.ownerSessionId,
|
|
326
|
+
validated.definition,
|
|
327
|
+
validated.task,
|
|
328
|
+
now,
|
|
329
|
+
);
|
|
301
330
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
331
|
+
const admission = this.transact((draft) => {
|
|
332
|
+
const open = openDecision(draft, "orchestrate");
|
|
333
|
+
if (open._tag === "rejected") return { value: open };
|
|
334
|
+
if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
|
|
335
|
+
if (draft.workers.has(workerId)) {
|
|
336
|
+
throw new Error(`Duplicate worker ID: ${workerId}`);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
draft.runs.set(runId, {
|
|
340
|
+
_tag: "running",
|
|
341
|
+
record: runRecord,
|
|
342
|
+
completion,
|
|
343
|
+
...(onSettlement ? { settlementListener: onSettlement } : {}),
|
|
344
|
+
});
|
|
345
|
+
draft.workers.set(workerId, {
|
|
346
|
+
record: workerRecord,
|
|
347
|
+
context,
|
|
348
|
+
definition: validated.definition,
|
|
349
|
+
generation: 1,
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
return {
|
|
353
|
+
value: accepted(undefined),
|
|
354
|
+
actions: [
|
|
355
|
+
publishState(context.ownerSessionId),
|
|
356
|
+
runAction(() => this.launchBootstrap(workerId, 1)),
|
|
357
|
+
],
|
|
358
|
+
};
|
|
359
|
+
});
|
|
360
|
+
if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
|
|
361
|
+
|
|
362
|
+
if (mode === "inline") {
|
|
363
|
+
return yield* this.awaitInlineRun(runRecord, completion);
|
|
364
|
+
}
|
|
365
|
+
return freezeAcceptedRun(runId, workerId);
|
|
366
|
+
}).call(this);
|
|
315
367
|
}
|
|
316
368
|
|
|
317
369
|
sendInteractive(
|
|
318
370
|
context: OrchestrationContext,
|
|
319
|
-
workerId:
|
|
371
|
+
workerId: string,
|
|
320
372
|
instructions: string,
|
|
321
373
|
mode: "async",
|
|
322
|
-
signal?: AbortSignal,
|
|
323
374
|
onSettlement?: SettlementListener,
|
|
324
|
-
):
|
|
375
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
325
376
|
sendInteractive(
|
|
326
377
|
context: OrchestrationContext,
|
|
327
|
-
workerId:
|
|
378
|
+
workerId: string,
|
|
328
379
|
instructions: string,
|
|
329
380
|
mode: "inline",
|
|
330
|
-
signal?: AbortSignal,
|
|
331
381
|
onSettlement?: SettlementListener,
|
|
332
|
-
):
|
|
382
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
333
383
|
sendInteractive(
|
|
334
384
|
context: OrchestrationContext,
|
|
335
|
-
workerId:
|
|
385
|
+
workerId: string,
|
|
336
386
|
instructions: string,
|
|
337
387
|
mode: RunMode,
|
|
338
|
-
signal?: AbortSignal,
|
|
339
388
|
onSettlement?: SettlementListener,
|
|
340
|
-
):
|
|
341
|
-
|
|
389
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
390
|
+
sendInteractive(
|
|
342
391
|
context: OrchestrationContext,
|
|
343
|
-
workerId:
|
|
392
|
+
workerId: string,
|
|
344
393
|
instructions: string,
|
|
345
394
|
mode: RunMode,
|
|
346
|
-
signal?: AbortSignal,
|
|
347
395
|
onSettlement?: SettlementListener,
|
|
348
|
-
):
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
396
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
397
|
+
return Effect.fn("Orchestration.sendInteractive")(function* (
|
|
398
|
+
this: StatefulOrchestration,
|
|
399
|
+
) {
|
|
400
|
+
yield* validateContextOwner("sendInteractive", context.ownerSessionId);
|
|
401
|
+
yield* validateMode("sendInteractive", mode);
|
|
402
|
+
yield* validateText(
|
|
403
|
+
"sendInteractive",
|
|
404
|
+
"instructions",
|
|
405
|
+
instructions,
|
|
406
|
+
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
407
|
+
);
|
|
408
|
+
const validatedWorkerId = yield* validateWorkerId(
|
|
409
|
+
"sendInteractive",
|
|
410
|
+
workerId,
|
|
411
|
+
);
|
|
412
|
+
const preflight = this.preflightReadyInteractive(
|
|
413
|
+
context.ownerSessionId,
|
|
414
|
+
validatedWorkerId,
|
|
415
|
+
);
|
|
416
|
+
if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
|
|
417
|
+
// See orchestrate: allocation remains outside copy-on-write reducers so
|
|
418
|
+
// reentrant factories cannot invalidate transaction atomicity.
|
|
419
|
+
const runId = this.idFactories.runId();
|
|
420
|
+
const completion = yield* Deferred.make<CompletedRun>();
|
|
421
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
422
|
+
const runRecord: RunRecord = {
|
|
423
|
+
id: runId,
|
|
424
|
+
ownerSessionId: context.ownerSessionId,
|
|
425
|
+
workerId: validatedWorkerId,
|
|
426
|
+
mode,
|
|
427
|
+
state: "running",
|
|
428
|
+
createdAt: now,
|
|
429
|
+
};
|
|
430
|
+
|
|
431
|
+
const admission = this.transact((draft) => {
|
|
432
|
+
const open = openDecision(draft, "sendInteractive");
|
|
433
|
+
if (open._tag === "rejected") return { value: open };
|
|
434
|
+
const ownership = ownedWorkerDecision(
|
|
435
|
+
draft,
|
|
436
|
+
"sendInteractive",
|
|
437
|
+
context.ownerSessionId,
|
|
438
|
+
validatedWorkerId,
|
|
439
|
+
);
|
|
440
|
+
if (ownership._tag === "rejected") return { value: ownership };
|
|
441
|
+
|
|
442
|
+
const worker = ownership.value;
|
|
443
|
+
if (
|
|
444
|
+
worker.record.lifecycle !== "interactive" ||
|
|
445
|
+
worker.record.status !== "ready" ||
|
|
446
|
+
!worker.session
|
|
447
|
+
) {
|
|
448
|
+
return {
|
|
449
|
+
value: rejected(
|
|
450
|
+
"sendInteractive",
|
|
451
|
+
"worker-state",
|
|
452
|
+
"interactive_send requires an owned ready interactive worker",
|
|
453
|
+
),
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
|
|
457
|
+
|
|
458
|
+
const generation = worker.generation + 1;
|
|
459
|
+
const session = worker.session;
|
|
460
|
+
const runningRecord: WorkerRecord = {
|
|
461
|
+
...transitionWorkerStatus(worker.record, "running"),
|
|
462
|
+
runId,
|
|
463
|
+
instructions,
|
|
464
|
+
activity: undefined,
|
|
465
|
+
messageDirection: "to-model",
|
|
466
|
+
startedAt: now,
|
|
467
|
+
settledAt: undefined,
|
|
468
|
+
};
|
|
469
|
+
draft.runs.set(runId, {
|
|
470
|
+
_tag: "running",
|
|
471
|
+
record: runRecord,
|
|
472
|
+
completion,
|
|
473
|
+
...(onSettlement ? { settlementListener: onSettlement } : {}),
|
|
474
|
+
});
|
|
475
|
+
draft.workers.set(validatedWorkerId, {
|
|
476
|
+
...worker,
|
|
477
|
+
record: runningRecord,
|
|
478
|
+
generation,
|
|
479
|
+
observationRelease: undefined,
|
|
480
|
+
cancellation: undefined,
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
return {
|
|
484
|
+
value: accepted(undefined),
|
|
485
|
+
actions: [
|
|
486
|
+
runAction(() => {
|
|
487
|
+
safelyCall(worker.observationRelease);
|
|
488
|
+
this.subscribeObservation(validatedWorkerId, generation, session);
|
|
489
|
+
}),
|
|
490
|
+
publishState(context.ownerSessionId),
|
|
491
|
+
runAction(() => {
|
|
492
|
+
this.launchPrompt(
|
|
493
|
+
validatedWorkerId,
|
|
494
|
+
generation,
|
|
495
|
+
session,
|
|
496
|
+
instructions,
|
|
497
|
+
);
|
|
498
|
+
}),
|
|
499
|
+
],
|
|
500
|
+
};
|
|
501
|
+
});
|
|
502
|
+
if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
|
|
503
|
+
|
|
504
|
+
if (mode === "inline") {
|
|
505
|
+
return yield* this.awaitInlineRun(runRecord, completion);
|
|
506
|
+
}
|
|
507
|
+
return freezeAcceptedRun(runId, validatedWorkerId);
|
|
508
|
+
}).call(this);
|
|
397
509
|
}
|
|
398
510
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
511
|
+
abort(
|
|
512
|
+
ownerSessionId: string,
|
|
513
|
+
target: AbortTarget,
|
|
514
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
515
|
+
return Effect.fn("Orchestration.abort")(function* (
|
|
516
|
+
this: StatefulOrchestration,
|
|
517
|
+
) {
|
|
518
|
+
yield* validateContextOwner("abort", ownerSessionId);
|
|
519
|
+
const validatedTarget = yield* validateAbortTarget(target);
|
|
520
|
+
const candidateIds = validatedTarget._tag === "ids"
|
|
521
|
+
? validatedTarget.workerIds
|
|
522
|
+
: activeWorkerIds(this.current(), ownerSessionId);
|
|
523
|
+
const candidates = yield* makeCancellationCandidates(candidateIds);
|
|
524
|
+
const cancellation = this.beginCancellation(
|
|
525
|
+
"abort",
|
|
526
|
+
ownerSessionId,
|
|
527
|
+
validatedTarget,
|
|
528
|
+
candidates,
|
|
529
|
+
);
|
|
530
|
+
if (cancellation._tag === "rejected") {
|
|
531
|
+
return yield* Effect.fail(cancellation.error);
|
|
532
|
+
}
|
|
533
|
+
yield* awaitAll(cancellation.value);
|
|
534
|
+
}).call(this);
|
|
404
535
|
}
|
|
405
536
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
537
|
+
closeInteractive(
|
|
538
|
+
ownerSessionId: string,
|
|
539
|
+
workerId: string,
|
|
540
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
541
|
+
return Effect.fn("Orchestration.closeInteractive")(function* (
|
|
542
|
+
this: StatefulOrchestration,
|
|
543
|
+
) {
|
|
544
|
+
yield* validateContextOwner("closeInteractive", ownerSessionId);
|
|
545
|
+
const id = yield* validateWorkerId("closeInteractive", workerId);
|
|
546
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
547
|
+
const decision = this.transact((draft) => {
|
|
548
|
+
const open = openDecision(draft, "closeInteractive");
|
|
549
|
+
if (open._tag === "rejected") return { value: open };
|
|
550
|
+
const ownership = ownedWorkerDecision(
|
|
551
|
+
draft,
|
|
552
|
+
"closeInteractive",
|
|
553
|
+
ownerSessionId,
|
|
554
|
+
id,
|
|
555
|
+
);
|
|
556
|
+
if (ownership._tag === "rejected") return { value: ownership };
|
|
557
|
+
if (
|
|
558
|
+
ownership.value.record.lifecycle !== "interactive" ||
|
|
559
|
+
ownership.value.record.status !== "ready"
|
|
560
|
+
) {
|
|
561
|
+
return {
|
|
562
|
+
value: rejected(
|
|
563
|
+
"closeInteractive",
|
|
564
|
+
"worker-state",
|
|
565
|
+
"interactive_close requires an owned ready interactive worker",
|
|
566
|
+
),
|
|
567
|
+
};
|
|
568
|
+
}
|
|
569
|
+
const actions = this.closeReadyWorker(draft, ownership.value, now);
|
|
570
|
+
actions.push(...stateActionsAfterPrune(draft, ownerSessionId));
|
|
571
|
+
return {
|
|
572
|
+
value: accepted(undefined),
|
|
573
|
+
actions,
|
|
574
|
+
};
|
|
575
|
+
});
|
|
576
|
+
if (decision._tag === "rejected") yield* Effect.fail(decision.error);
|
|
577
|
+
}).call(this);
|
|
414
578
|
}
|
|
415
579
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
.
|
|
421
|
-
|
|
422
|
-
.filter((worker) => worker.ownerSessionId === ownerSessionId)
|
|
423
|
-
.map(copyWorkerRecord);
|
|
424
|
-
return Object.freeze({
|
|
425
|
-
runs: Object.freeze(runs),
|
|
426
|
-
workers: Object.freeze(workers),
|
|
427
|
-
});
|
|
580
|
+
snapshot(
|
|
581
|
+
ownerSessionId: string,
|
|
582
|
+
): Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected> {
|
|
583
|
+
return validateContextOwner("snapshot", ownerSessionId).pipe(
|
|
584
|
+
Effect.andThen(Effect.sync(() => snapshotFor(this.current(), ownerSessionId))),
|
|
585
|
+
);
|
|
428
586
|
}
|
|
429
587
|
|
|
430
588
|
subscribeSettlement(listener: SettlementListener): UnsubscribeSettlement {
|
|
431
|
-
if (typeof listener !== "function")
|
|
589
|
+
if (typeof listener !== "function") {
|
|
590
|
+
throw new Error("Settlement listener must be a function");
|
|
591
|
+
}
|
|
592
|
+
if (this.state.lifecycle !== "open") return noOp;
|
|
432
593
|
this.settlementListeners.add(listener);
|
|
433
|
-
|
|
594
|
+
|
|
595
|
+
let active = true;
|
|
434
596
|
return () => {
|
|
435
|
-
if (!
|
|
436
|
-
|
|
597
|
+
if (!active) return;
|
|
598
|
+
active = false;
|
|
437
599
|
this.settlementListeners.delete(listener);
|
|
438
600
|
};
|
|
439
601
|
}
|
|
440
602
|
|
|
441
|
-
subscribeState(listener: StateListener): () => void {
|
|
442
|
-
if (typeof
|
|
443
|
-
|
|
444
|
-
|
|
603
|
+
subscribeState(ownerSessionId: string, listener: StateListener): () => void {
|
|
604
|
+
if (typeof ownerSessionId !== "string" || ownerSessionId.trim() === "") {
|
|
605
|
+
throw new Error("ownerSessionId must not be blank");
|
|
606
|
+
}
|
|
607
|
+
if (typeof listener !== "function") {
|
|
608
|
+
throw new Error("State listener must be a function");
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
if (this.state.lifecycle !== "open") return noOp;
|
|
612
|
+
const ownerListeners = this.stateListeners.get(ownerSessionId) ?? new Set();
|
|
613
|
+
ownerListeners.add(listener);
|
|
614
|
+
this.stateListeners.set(ownerSessionId, ownerListeners);
|
|
615
|
+
this.enqueueActions([
|
|
616
|
+
publishStateTo(ownerSessionId, listener)(
|
|
617
|
+
this.state,
|
|
618
|
+
this.settlementListeners,
|
|
619
|
+
this.stateListeners,
|
|
620
|
+
),
|
|
621
|
+
]);
|
|
622
|
+
|
|
623
|
+
let active = true;
|
|
445
624
|
return () => {
|
|
446
|
-
if (!
|
|
447
|
-
|
|
448
|
-
|
|
625
|
+
if (!active) return;
|
|
626
|
+
active = false;
|
|
627
|
+
ownerListeners.delete(listener);
|
|
628
|
+
if (ownerListeners.size === 0) this.stateListeners.delete(ownerSessionId);
|
|
449
629
|
};
|
|
450
630
|
}
|
|
451
631
|
|
|
452
|
-
shutdown
|
|
453
|
-
|
|
454
|
-
this.
|
|
455
|
-
|
|
456
|
-
|
|
632
|
+
/** Calling shutdown closes admission synchronously, before the returned Effect runs. */
|
|
633
|
+
shutdown(): Effect.Effect<void> {
|
|
634
|
+
const first = this.transact((draft) => {
|
|
635
|
+
if (draft.lifecycle !== "open") return { value: false };
|
|
636
|
+
draft.lifecycle = "shutting-down";
|
|
637
|
+
return { value: true };
|
|
638
|
+
});
|
|
639
|
+
if (!first) return Deferred.await(this.shutdownCompletion);
|
|
640
|
+
|
|
641
|
+
return this.performShutdown().pipe(
|
|
642
|
+
Effect.ensuring(
|
|
643
|
+
Effect.sync(() => {
|
|
644
|
+
this.transact((draft) => {
|
|
645
|
+
draft.lifecycle = "shutdown";
|
|
646
|
+
for (const [runId, run] of draft.runs) {
|
|
647
|
+
if (isRunningRuntimeRun(run) && run.settlementListener) {
|
|
648
|
+
draft.runs.set(runId, { ...run, settlementListener: undefined });
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
return { value: undefined };
|
|
652
|
+
});
|
|
653
|
+
this.settlementListeners.clear();
|
|
654
|
+
this.stateListeners.clear();
|
|
655
|
+
Deferred.doneUnsafe(this.shutdownCompletion, Effect.void);
|
|
656
|
+
}),
|
|
657
|
+
),
|
|
658
|
+
);
|
|
457
659
|
}
|
|
458
660
|
|
|
459
|
-
private
|
|
460
|
-
|
|
461
|
-
this
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
661
|
+
private performShutdown(): Effect.Effect<void> {
|
|
662
|
+
return Effect.fn("Orchestration.shutdown")(function* (
|
|
663
|
+
this: StatefulOrchestration,
|
|
664
|
+
) {
|
|
665
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
666
|
+
this.transact((draft) => {
|
|
667
|
+
const owners = new Set<string>();
|
|
668
|
+
const actions: PostCommitAction[] = [];
|
|
669
|
+
for (const worker of draft.workers.values()) {
|
|
670
|
+
if (
|
|
671
|
+
worker.record.lifecycle === "interactive" &&
|
|
672
|
+
worker.record.status === "ready"
|
|
673
|
+
) {
|
|
674
|
+
this.closeReadyWorker(draft, worker, now, actions);
|
|
675
|
+
owners.add(worker.record.ownerSessionId);
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
addAll(owners, pruneHistory(draft));
|
|
679
|
+
actions.push(...publishOwners(owners));
|
|
680
|
+
return { value: undefined, actions };
|
|
681
|
+
});
|
|
682
|
+
|
|
683
|
+
const activeIds = activeWorkerIds(this.current());
|
|
684
|
+
const candidates = yield* makeCancellationCandidates(activeIds);
|
|
685
|
+
const cancellations = this.beginShutdownCancellation(candidates);
|
|
686
|
+
yield* awaitAll(cancellations);
|
|
687
|
+
yield* this.childSessions.shutdown().pipe(Effect.catchCause(() => Effect.void));
|
|
688
|
+
yield* FiberSet.awaitEmpty(this.cleanups).pipe(
|
|
689
|
+
Effect.timeoutOption(SHUTDOWN_CLEANUP_GRACE_MS),
|
|
690
|
+
Effect.ignore,
|
|
691
|
+
);
|
|
692
|
+
}).call(this);
|
|
478
693
|
}
|
|
479
694
|
|
|
480
695
|
private validateTask(
|
|
481
696
|
context: OrchestrationContext,
|
|
482
697
|
task: OrchestrateTaskInput,
|
|
483
698
|
mode: RunMode,
|
|
484
|
-
):
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
699
|
+
): Effect.Effect<
|
|
700
|
+
{ definition: WorkerDefinition; task: OrchestrateTaskInput },
|
|
701
|
+
OrchestrationActionRejected
|
|
702
|
+
> {
|
|
703
|
+
return Effect.gen(function* () {
|
|
704
|
+
yield* validateContextOwner("orchestrate", context.ownerSessionId);
|
|
705
|
+
yield* validateMode("orchestrate", mode);
|
|
706
|
+
if (!task || typeof task !== "object" || Array.isArray(task)) {
|
|
707
|
+
return yield* rejectAction(
|
|
708
|
+
"orchestrate",
|
|
709
|
+
"validation",
|
|
710
|
+
"orchestrate requires one task object",
|
|
711
|
+
);
|
|
495
712
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
713
|
+
if (context.synthesisGroup) {
|
|
714
|
+
yield* validateText(
|
|
715
|
+
"orchestrate",
|
|
716
|
+
"synthesis group ID",
|
|
717
|
+
context.synthesisGroup.id,
|
|
718
|
+
MAX_WORKER_TITLE_LENGTH,
|
|
719
|
+
);
|
|
720
|
+
if (mode !== "async") {
|
|
721
|
+
return yield* rejectAction(
|
|
722
|
+
"orchestrate",
|
|
723
|
+
"validation",
|
|
724
|
+
"Sibling synthesis requires an async task",
|
|
725
|
+
);
|
|
726
|
+
}
|
|
727
|
+
if (
|
|
728
|
+
!Number.isSafeInteger(context.synthesisGroup.size) ||
|
|
729
|
+
context.synthesisGroup.size < 2
|
|
730
|
+
) {
|
|
731
|
+
return yield* rejectAction(
|
|
732
|
+
"orchestrate",
|
|
733
|
+
"validation",
|
|
734
|
+
"Synthesis group size must be an integer of at least 2",
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
yield* validateText(
|
|
739
|
+
"orchestrate",
|
|
740
|
+
"worker",
|
|
741
|
+
task.worker,
|
|
742
|
+
MAX_WORKER_TITLE_LENGTH,
|
|
743
|
+
);
|
|
744
|
+
yield* validateText(
|
|
745
|
+
"orchestrate",
|
|
746
|
+
"title",
|
|
747
|
+
task.title,
|
|
748
|
+
MAX_WORKER_TITLE_LENGTH,
|
|
749
|
+
);
|
|
750
|
+
yield* validateText(
|
|
751
|
+
"orchestrate",
|
|
752
|
+
"instructions",
|
|
753
|
+
task.instructions,
|
|
754
|
+
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
755
|
+
);
|
|
756
|
+
const definition = findWorkerByName(context.catalog, task.worker);
|
|
757
|
+
if (!definition) {
|
|
758
|
+
return yield* rejectAction(
|
|
759
|
+
"orchestrate",
|
|
760
|
+
"unknown-worker",
|
|
761
|
+
`Unknown worker: ${task.worker}`,
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
const configured = definition.model;
|
|
765
|
+
if (!configured && !context.parentModel) {
|
|
766
|
+
return yield* rejectAction(
|
|
767
|
+
"orchestrate",
|
|
768
|
+
"model-unavailable",
|
|
769
|
+
`Worker "${definition.name}" has no configured model and no parent model is available`,
|
|
770
|
+
);
|
|
771
|
+
}
|
|
772
|
+
if (
|
|
773
|
+
configured &&
|
|
774
|
+
!context.modelRegistry.find(configured.provider, configured.modelId)
|
|
775
|
+
) {
|
|
776
|
+
return yield* rejectAction(
|
|
777
|
+
"orchestrate",
|
|
778
|
+
"model-unavailable",
|
|
779
|
+
`Worker "${definition.name}" configured model "${configured.provider}/${configured.modelId}" was not found`,
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
return { definition, task };
|
|
783
|
+
});
|
|
510
784
|
}
|
|
511
785
|
|
|
512
786
|
private launchBootstrap(workerId: WorkerId, generation: number): void {
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
);
|
|
519
|
-
} catch (error) {
|
|
520
|
-
this.settleWorkflowDefect(workerId, generation, error);
|
|
521
|
-
}
|
|
787
|
+
this.launchGeneration(
|
|
788
|
+
workerId,
|
|
789
|
+
generation,
|
|
790
|
+
this.bootstrapAndPrompt(workerId, generation),
|
|
791
|
+
);
|
|
522
792
|
}
|
|
523
793
|
|
|
524
794
|
private launchPrompt(
|
|
@@ -526,29 +796,73 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
526
796
|
generation: number,
|
|
527
797
|
session: WorkerSessionHandle,
|
|
528
798
|
instructions: string,
|
|
799
|
+
): void {
|
|
800
|
+
this.launchGeneration(
|
|
801
|
+
workerId,
|
|
802
|
+
generation,
|
|
803
|
+
this.executePrompt(workerId, generation, session, instructions),
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
private launchGeneration(
|
|
808
|
+
workerId: WorkerId,
|
|
809
|
+
generation: number,
|
|
810
|
+
workflow: Effect.Effect<void, never>,
|
|
529
811
|
): void {
|
|
530
812
|
try {
|
|
531
|
-
this.
|
|
813
|
+
const fiber = this.runGeneration(
|
|
532
814
|
workerId,
|
|
533
|
-
|
|
534
|
-
|
|
815
|
+
workflow.pipe(
|
|
816
|
+
Effect.catchCause((cause) => {
|
|
817
|
+
if (Cause.hasInterruptsOnly(cause)) return Effect.void;
|
|
818
|
+
return Effect.sync(() => {
|
|
819
|
+
this.settleWorkflowDefect(workerId, generation, Cause.squash(cause));
|
|
820
|
+
});
|
|
821
|
+
}),
|
|
822
|
+
),
|
|
535
823
|
);
|
|
824
|
+
const exit = fiber.pollUnsafe();
|
|
825
|
+
if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
826
|
+
this.settleGenerationLaunchFailure(workerId, generation);
|
|
827
|
+
}
|
|
536
828
|
} catch (error) {
|
|
537
829
|
this.settleWorkflowDefect(workerId, generation, error);
|
|
538
830
|
}
|
|
539
831
|
}
|
|
540
832
|
|
|
833
|
+
private settleGenerationLaunchFailure(
|
|
834
|
+
workerId: WorkerId,
|
|
835
|
+
generation: number,
|
|
836
|
+
): void {
|
|
837
|
+
this.settleActiveWorker(
|
|
838
|
+
workerId,
|
|
839
|
+
generation,
|
|
840
|
+
["starting", "running"],
|
|
841
|
+
"failed",
|
|
842
|
+
{
|
|
843
|
+
status: "failed",
|
|
844
|
+
message: "Worker generation could not start because orchestration is closed",
|
|
845
|
+
},
|
|
846
|
+
"workflow",
|
|
847
|
+
true,
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
|
|
541
851
|
private bootstrapAndPrompt(
|
|
542
852
|
workerId: WorkerId,
|
|
543
853
|
generation: number,
|
|
544
854
|
): Effect.Effect<void, never> {
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
const session = yield* runtime.bootstrap(workerId, generation);
|
|
855
|
+
return Effect.gen({ self: this }, function* () {
|
|
856
|
+
const session = yield* this.bootstrap(workerId, generation);
|
|
548
857
|
if (!session) return;
|
|
549
|
-
const
|
|
550
|
-
if (!
|
|
551
|
-
yield*
|
|
858
|
+
const worker = this.current().workers.get(workerId);
|
|
859
|
+
if (!worker) return;
|
|
860
|
+
yield* this.executePrompt(
|
|
861
|
+
workerId,
|
|
862
|
+
generation,
|
|
863
|
+
session,
|
|
864
|
+
worker.record.instructions,
|
|
865
|
+
);
|
|
552
866
|
});
|
|
553
867
|
}
|
|
554
868
|
|
|
@@ -557,89 +871,109 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
557
871
|
generation: number,
|
|
558
872
|
): Effect.Effect<WorkerSessionHandle | undefined, never> {
|
|
559
873
|
return Effect.suspend(() => {
|
|
560
|
-
const
|
|
561
|
-
if (!
|
|
562
|
-
|
|
563
|
-
let creation: Promise<WorkerSessionHandle>;
|
|
564
|
-
try {
|
|
565
|
-
creation = this.workerSessionFactory.create({
|
|
566
|
-
cwd: entry.context.cwd,
|
|
567
|
-
agentDir: entry.context.agentDir,
|
|
568
|
-
parentSessionFile: entry.context.parentSessionFile,
|
|
569
|
-
projectTrusted: entry.context.projectTrusted,
|
|
570
|
-
definition: entry.definition,
|
|
571
|
-
parentModel: entry.context.parentModel,
|
|
572
|
-
modelRegistry: entry.context.modelRegistry,
|
|
573
|
-
});
|
|
574
|
-
} catch (error) {
|
|
575
|
-
this.settleCreationFailure(workerId, generation, error);
|
|
874
|
+
const expected = this.current().workers.get(workerId);
|
|
875
|
+
if (!expected || expected.generation !== generation) {
|
|
576
876
|
return Effect.succeed(undefined);
|
|
577
877
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
878
|
+
return this.childSessions.acquire(
|
|
879
|
+
{
|
|
880
|
+
cwd: expected.context.cwd,
|
|
881
|
+
agentDir: expected.context.agentDir,
|
|
882
|
+
parentSessionFile: expected.context.parentSessionFile,
|
|
883
|
+
projectTrusted: expected.context.projectTrusted,
|
|
884
|
+
definition: expected.definition,
|
|
885
|
+
parentModel: expected.context.parentModel,
|
|
886
|
+
modelRegistry: expected.context.modelRegistry,
|
|
887
|
+
},
|
|
888
|
+
(session) => this.adoptCreatedSession(workerId, generation, session),
|
|
889
|
+
).pipe(
|
|
590
890
|
Effect.match({
|
|
591
891
|
onFailure: (error) => {
|
|
592
892
|
this.settleCreationFailure(workerId, generation, error);
|
|
593
893
|
return undefined;
|
|
594
894
|
},
|
|
595
|
-
onSuccess: (session) =>
|
|
596
|
-
workerId,
|
|
597
|
-
generation,
|
|
598
|
-
entry,
|
|
599
|
-
session,
|
|
600
|
-
),
|
|
895
|
+
onSuccess: (session) => session,
|
|
601
896
|
}),
|
|
602
897
|
);
|
|
603
898
|
});
|
|
604
899
|
}
|
|
605
900
|
|
|
606
|
-
private canAdoptCreatedSession(
|
|
607
|
-
workerId: WorkerId,
|
|
608
|
-
generation: number,
|
|
609
|
-
entry: RuntimeEntry,
|
|
610
|
-
): boolean {
|
|
611
|
-
const current = this.workers.get(workerId);
|
|
612
|
-
return !this.shuttingDown &&
|
|
613
|
-
current?.status === "starting" &&
|
|
614
|
-
entry.generation === generation;
|
|
615
|
-
}
|
|
616
|
-
|
|
617
901
|
private adoptCreatedSession(
|
|
618
902
|
workerId: WorkerId,
|
|
619
903
|
generation: number,
|
|
620
|
-
entry: RuntimeEntry,
|
|
621
904
|
session: WorkerSessionHandle,
|
|
622
905
|
): WorkerSessionHandle | undefined {
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
906
|
+
let release: () => void;
|
|
907
|
+
try {
|
|
908
|
+
release = session.subscribeObservation((observation) => {
|
|
909
|
+
this.updateObservation(workerId, generation, session, observation);
|
|
910
|
+
});
|
|
911
|
+
} catch (error) {
|
|
912
|
+
this.settleCreationFailure(workerId, generation, error);
|
|
626
913
|
return undefined;
|
|
627
914
|
}
|
|
628
915
|
|
|
916
|
+
const adopted = this.transact((draft) => {
|
|
917
|
+
const worker = draft.workers.get(workerId);
|
|
918
|
+
if (
|
|
919
|
+
draft.lifecycle !== "open" ||
|
|
920
|
+
!worker ||
|
|
921
|
+
worker.generation !== generation ||
|
|
922
|
+
worker.record.status !== "starting"
|
|
923
|
+
) {
|
|
924
|
+
return {
|
|
925
|
+
value: false,
|
|
926
|
+
actions: [runAction(() => safelyCall(release))],
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
draft.workers.set(workerId, {
|
|
930
|
+
...worker,
|
|
931
|
+
session,
|
|
932
|
+
observationRelease: release,
|
|
933
|
+
record: {
|
|
934
|
+
...transitionWorkerStatus(worker.record, "running"),
|
|
935
|
+
sessionFile: session.sessionFile,
|
|
936
|
+
},
|
|
937
|
+
});
|
|
938
|
+
return {
|
|
939
|
+
value: true,
|
|
940
|
+
actions: [publishState(worker.record.ownerSessionId)],
|
|
941
|
+
};
|
|
942
|
+
});
|
|
943
|
+
return adopted ? session : undefined;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
private subscribeObservation(
|
|
947
|
+
workerId: WorkerId,
|
|
948
|
+
generation: number,
|
|
949
|
+
session: WorkerSessionHandle,
|
|
950
|
+
): void {
|
|
951
|
+
let release: () => void;
|
|
629
952
|
try {
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
this.workers.set(workerId, {
|
|
633
|
-
...transitionWorkerStatus(current, "running"),
|
|
634
|
-
sessionFile: session.sessionFile,
|
|
953
|
+
release = session.subscribeObservation((observation) => {
|
|
954
|
+
this.updateObservation(workerId, generation, session, observation);
|
|
635
955
|
});
|
|
636
|
-
this.emitState(current.ownerSessionId);
|
|
637
|
-
return session;
|
|
638
956
|
} catch (error) {
|
|
639
|
-
this.
|
|
640
|
-
|
|
641
|
-
return undefined;
|
|
957
|
+
this.settleWorkflowDefect(workerId, generation, error);
|
|
958
|
+
return;
|
|
642
959
|
}
|
|
960
|
+
|
|
961
|
+
this.transact((draft) => {
|
|
962
|
+
const worker = draft.workers.get(workerId);
|
|
963
|
+
if (
|
|
964
|
+
!worker ||
|
|
965
|
+
worker.generation !== generation ||
|
|
966
|
+
worker.session !== session ||
|
|
967
|
+
worker.record.status !== "running"
|
|
968
|
+
) {
|
|
969
|
+
return {
|
|
970
|
+
value: undefined,
|
|
971
|
+
actions: [runAction(() => safelyCall(release))],
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
draft.workers.set(workerId, { ...worker, observationRelease: release });
|
|
975
|
+
return { value: undefined };
|
|
976
|
+
});
|
|
643
977
|
}
|
|
644
978
|
|
|
645
979
|
private executePrompt(
|
|
@@ -649,34 +983,16 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
649
983
|
instructions: string,
|
|
650
984
|
): Effect.Effect<void, never> {
|
|
651
985
|
return Effect.suspend(() => {
|
|
652
|
-
const
|
|
653
|
-
const entry = this.entries.get(workerId);
|
|
986
|
+
const worker = this.current().workers.get(workerId);
|
|
654
987
|
if (
|
|
655
|
-
!
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
entry.session !== session
|
|
988
|
+
!worker ||
|
|
989
|
+
worker.record.status !== "running" ||
|
|
990
|
+
worker.generation !== generation ||
|
|
991
|
+
worker.session !== session
|
|
660
992
|
) {
|
|
661
993
|
return Effect.void;
|
|
662
994
|
}
|
|
663
|
-
|
|
664
|
-
let prompt: Promise<WorkerOutcome>;
|
|
665
|
-
try {
|
|
666
|
-
prompt = session.prompt(instructions);
|
|
667
|
-
} catch (error) {
|
|
668
|
-
this.settleOutcome(workerId, generation, session, {
|
|
669
|
-
status: "failed",
|
|
670
|
-
message: describeError(error, "Worker prompt failed"),
|
|
671
|
-
});
|
|
672
|
-
return Effect.void;
|
|
673
|
-
}
|
|
674
|
-
this.trackCleanup(prompt.then(() => undefined, () => undefined));
|
|
675
|
-
|
|
676
|
-
return Effect.tryPromise({
|
|
677
|
-
try: () => prompt,
|
|
678
|
-
catch: (error) => error,
|
|
679
|
-
}).pipe(
|
|
995
|
+
return session.prompt(instructions).pipe(
|
|
680
996
|
Effect.match({
|
|
681
997
|
onFailure: (error): WorkerOutcome => ({
|
|
682
998
|
status: "failed",
|
|
@@ -697,13 +1013,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
697
1013
|
generation: number,
|
|
698
1014
|
error: unknown,
|
|
699
1015
|
): void {
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
1016
|
+
this.settleActiveWorker(
|
|
1017
|
+
workerId,
|
|
1018
|
+
generation,
|
|
1019
|
+
["starting"],
|
|
1020
|
+
"failed",
|
|
1021
|
+
{
|
|
1022
|
+
status: "failed",
|
|
1023
|
+
message: describeError(error, "Worker session creation failed"),
|
|
1024
|
+
},
|
|
1025
|
+
"startup",
|
|
1026
|
+
false,
|
|
1027
|
+
);
|
|
707
1028
|
}
|
|
708
1029
|
|
|
709
1030
|
private settleWorkflowDefect(
|
|
@@ -711,19 +1032,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
711
1032
|
generation: number,
|
|
712
1033
|
error: unknown,
|
|
713
1034
|
): void {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
}, "workflow");
|
|
1035
|
+
this.settleActiveWorker(
|
|
1036
|
+
workerId,
|
|
1037
|
+
generation,
|
|
1038
|
+
["starting", "running", "stopping"],
|
|
1039
|
+
"failed",
|
|
1040
|
+
{
|
|
1041
|
+
status: "failed",
|
|
1042
|
+
message: describeError(error, "Worker workflow failed"),
|
|
1043
|
+
},
|
|
1044
|
+
"workflow",
|
|
1045
|
+
true,
|
|
1046
|
+
);
|
|
727
1047
|
}
|
|
728
1048
|
|
|
729
1049
|
private settleOutcome(
|
|
@@ -732,14 +1052,12 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
732
1052
|
session: WorkerSessionHandle,
|
|
733
1053
|
outcome: WorkerOutcome,
|
|
734
1054
|
): void {
|
|
735
|
-
const
|
|
736
|
-
const entry = this.entries.get(workerId);
|
|
1055
|
+
const worker = this.current().workers.get(workerId);
|
|
737
1056
|
if (
|
|
738
|
-
!
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
entry.session !== session
|
|
1057
|
+
!worker ||
|
|
1058
|
+
worker.generation !== generation ||
|
|
1059
|
+
worker.session !== session ||
|
|
1060
|
+
worker.record.status !== "running"
|
|
743
1061
|
) {
|
|
744
1062
|
return;
|
|
745
1063
|
}
|
|
@@ -748,549 +1066,1026 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
748
1066
|
let status: "ready" | "completed" | "failed" | "aborted";
|
|
749
1067
|
if (outcome.status === "failed" || outcome.status === "aborted") {
|
|
750
1068
|
status = outcome.status;
|
|
751
|
-
} else if (
|
|
1069
|
+
} else if (
|
|
1070
|
+
worker.record.lifecycle === "interactive" &&
|
|
1071
|
+
outcome.status === "ready"
|
|
1072
|
+
) {
|
|
752
1073
|
status = "ready";
|
|
753
|
-
} else if (
|
|
1074
|
+
} else if (
|
|
1075
|
+
worker.record.lifecycle === "one-shot" &&
|
|
1076
|
+
outcome.status === "completed"
|
|
1077
|
+
) {
|
|
754
1078
|
status = "completed";
|
|
755
1079
|
} else {
|
|
756
1080
|
status = "failed";
|
|
757
1081
|
settledOutcome = {
|
|
758
1082
|
status: "failed",
|
|
759
|
-
message: `Worker session returned ${outcome.status} for a ${
|
|
1083
|
+
message: `Worker session returned ${outcome.status} for a ${worker.record.lifecycle} worker`,
|
|
760
1084
|
};
|
|
761
1085
|
}
|
|
762
|
-
|
|
763
|
-
const settledAt = this.clock();
|
|
764
|
-
this.workers.set(workerId, {
|
|
765
|
-
...transitionWorkerStatus(current, status),
|
|
766
|
-
activity: undefined,
|
|
767
|
-
outcome: copyOutcome(settledOutcome),
|
|
768
|
-
settledAt,
|
|
769
|
-
});
|
|
770
|
-
if (status !== "ready") this.disposeEntrySession(entry);
|
|
771
|
-
this.maybeCompleteRun(current.runId);
|
|
772
|
-
const affectedOwners = new Set([current.ownerSessionId]);
|
|
773
|
-
this.emitSettlement(
|
|
1086
|
+
this.settleActiveWorker(
|
|
774
1087
|
workerId,
|
|
775
1088
|
generation,
|
|
776
|
-
|
|
1089
|
+
["running"],
|
|
1090
|
+
status,
|
|
1091
|
+
settledOutcome,
|
|
777
1092
|
status === "failed" ? "prompt" : undefined,
|
|
1093
|
+
status !== "ready",
|
|
778
1094
|
);
|
|
779
|
-
if (isTerminalWorkerStatus(status)) {
|
|
780
|
-
addAll(affectedOwners, this.rememberTerminalWorker(workerId));
|
|
781
|
-
} else {
|
|
782
|
-
addAll(affectedOwners, this.pruneHistory());
|
|
783
|
-
}
|
|
784
|
-
this.emitStateForOwners(affectedOwners);
|
|
785
1095
|
}
|
|
786
1096
|
|
|
787
|
-
private
|
|
788
|
-
|
|
789
|
-
|
|
1097
|
+
private settleActiveWorker(
|
|
1098
|
+
workerId: WorkerId,
|
|
1099
|
+
generation: number,
|
|
1100
|
+
expectedStatuses: readonly WorkerRecord["status"][],
|
|
1101
|
+
status: "ready" | "completed" | "failed" | "aborted",
|
|
790
1102
|
outcome: WorkerOutcome,
|
|
791
|
-
failureStage
|
|
1103
|
+
failureStage: SettlementFailureStage | undefined,
|
|
1104
|
+
dispose: boolean,
|
|
792
1105
|
): void {
|
|
793
|
-
const settledAt = this.clock();
|
|
794
|
-
this.
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
1106
|
+
const settledAt = this.clock.currentTimeMillisUnsafe();
|
|
1107
|
+
this.transact((draft) => {
|
|
1108
|
+
const worker = draft.workers.get(workerId);
|
|
1109
|
+
if (
|
|
1110
|
+
!worker ||
|
|
1111
|
+
worker.generation !== generation ||
|
|
1112
|
+
!expectedStatuses.includes(worker.record.status)
|
|
1113
|
+
) {
|
|
1114
|
+
return { value: undefined };
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
const settledRecord: WorkerRecord = {
|
|
1118
|
+
...transitionWorkerStatus(worker.record, status),
|
|
1119
|
+
activity: undefined,
|
|
1120
|
+
outcome: copyOutcome(outcome),
|
|
1121
|
+
settledAt,
|
|
1122
|
+
};
|
|
1123
|
+
const actions: PostCommitAction[] = [];
|
|
1124
|
+
let settledWorker: RuntimeWorker = { ...worker, record: settledRecord };
|
|
1125
|
+
if (dispose) {
|
|
1126
|
+
actions.push(...this.releaseWorkerResources(settledWorker));
|
|
1127
|
+
settledWorker = {
|
|
1128
|
+
...settledWorker,
|
|
1129
|
+
session: undefined,
|
|
1130
|
+
observationRelease: undefined,
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
draft.workers.set(workerId, settledWorker);
|
|
1134
|
+
const settlement = makeSettlement(
|
|
1135
|
+
draft,
|
|
1136
|
+
settledWorker,
|
|
1137
|
+
settledAt,
|
|
1138
|
+
failureStage,
|
|
1139
|
+
);
|
|
1140
|
+
actions.push(...completeRun(draft, workerId));
|
|
1141
|
+
if (settlement) actions.push(settlement);
|
|
1142
|
+
if (isTerminalWorkerStatus(status)) rememberTerminalWorker(draft, workerId);
|
|
1143
|
+
actions.push(...stateActionsAfterPrune(draft, worker.record.ownerSessionId));
|
|
1144
|
+
return { value: undefined, actions };
|
|
799
1145
|
});
|
|
800
|
-
this.maybeCompleteRun(current.runId);
|
|
801
|
-
const affectedOwners = new Set([current.ownerSessionId]);
|
|
802
|
-
const generation = this.entries.get(current.id)?.generation;
|
|
803
|
-
if (generation !== undefined) {
|
|
804
|
-
this.emitSettlement(current.id, generation, settledAt, failureStage);
|
|
805
|
-
}
|
|
806
|
-
addAll(affectedOwners, this.rememberTerminalWorker(current.id));
|
|
807
|
-
this.emitStateForOwners(affectedOwners);
|
|
808
1146
|
}
|
|
809
1147
|
|
|
810
|
-
private
|
|
1148
|
+
private updateObservation(
|
|
811
1149
|
workerId: WorkerId,
|
|
812
1150
|
generation: number,
|
|
813
|
-
|
|
814
|
-
|
|
1151
|
+
session: WorkerSessionHandle,
|
|
1152
|
+
observation: WorkerSessionObservation,
|
|
815
1153
|
): void {
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
? {
|
|
841
|
-
synthesisGroupId: run.synthesisGroupId,
|
|
842
|
-
synthesisGroupSize: run.synthesisGroupSize,
|
|
843
|
-
}
|
|
844
|
-
: {}),
|
|
845
|
-
...(worker.sessionFile !== undefined ? { sessionFile: worker.sessionFile } : {}),
|
|
1154
|
+
this.transact((draft) => {
|
|
1155
|
+
const worker = draft.workers.get(workerId);
|
|
1156
|
+
if (
|
|
1157
|
+
!worker ||
|
|
1158
|
+
worker.generation !== generation ||
|
|
1159
|
+
worker.session !== session ||
|
|
1160
|
+
(worker.record.status !== "starting" && worker.record.status !== "running")
|
|
1161
|
+
) {
|
|
1162
|
+
return { value: undefined };
|
|
1163
|
+
}
|
|
1164
|
+
const nextRecord: WorkerRecord = {
|
|
1165
|
+
...worker.record,
|
|
1166
|
+
usage: copyUsage(observation.usage),
|
|
1167
|
+
activity: observation.activity,
|
|
1168
|
+
messageDirection: observation.messageDirection,
|
|
1169
|
+
};
|
|
1170
|
+
if (observationsEqual(worker.record, nextRecord)) {
|
|
1171
|
+
return { value: undefined };
|
|
1172
|
+
}
|
|
1173
|
+
draft.workers.set(workerId, { ...worker, record: nextRecord });
|
|
1174
|
+
return {
|
|
1175
|
+
value: undefined,
|
|
1176
|
+
actions: [publishState(worker.record.ownerSessionId)],
|
|
1177
|
+
};
|
|
846
1178
|
});
|
|
847
|
-
|
|
848
|
-
const localListener = this.runSettlementListeners.get(run.id);
|
|
849
|
-
this.runSettlementListeners.delete(run.id);
|
|
850
|
-
notifySettlementListener(localListener, settlement);
|
|
851
|
-
for (const listener of [...this.settlementListeners]) {
|
|
852
|
-
notifySettlementListener(listener, settlement);
|
|
853
|
-
}
|
|
854
1179
|
}
|
|
855
1180
|
|
|
856
|
-
private
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
1181
|
+
private beginCancellation(
|
|
1182
|
+
operation: "abort",
|
|
1183
|
+
ownerSessionId: string,
|
|
1184
|
+
target: ValidatedAbortTarget,
|
|
1185
|
+
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1186
|
+
): Decision<readonly Deferred.Deferred<void>[]> {
|
|
1187
|
+
return this.transact((draft) => {
|
|
1188
|
+
const open = openDecision(draft, operation);
|
|
1189
|
+
if (open._tag === "rejected") return { value: open };
|
|
1190
|
+
const workers = target._tag === "all"
|
|
1191
|
+
? [...draft.workers.values()].filter((worker) => (
|
|
1192
|
+
worker.record.ownerSessionId === ownerSessionId &&
|
|
1193
|
+
isActiveWorkerStatus(worker.record.status)
|
|
1194
|
+
))
|
|
1195
|
+
: target.workerIds.map((id) => draft.workers.get(id));
|
|
1196
|
+
|
|
1197
|
+
if (target._tag === "ids") {
|
|
1198
|
+
for (let index = 0; index < target.workerIds.length; index += 1) {
|
|
1199
|
+
const worker = workers[index];
|
|
1200
|
+
if (!worker || worker.record.ownerSessionId !== ownerSessionId) {
|
|
1201
|
+
return {
|
|
1202
|
+
value: rejected(
|
|
1203
|
+
operation,
|
|
1204
|
+
"ownership",
|
|
1205
|
+
"Worker is not owned by this session",
|
|
1206
|
+
),
|
|
1207
|
+
};
|
|
1208
|
+
}
|
|
1209
|
+
if (worker.record.status === "ready") {
|
|
1210
|
+
return {
|
|
1211
|
+
value: rejected(
|
|
1212
|
+
operation,
|
|
1213
|
+
"worker-state",
|
|
1214
|
+
"Ready interactive workers are not active; use interactive_close",
|
|
1215
|
+
),
|
|
1216
|
+
};
|
|
1217
|
+
}
|
|
1218
|
+
if (!isActiveWorkerStatus(worker.record.status)) {
|
|
1219
|
+
return {
|
|
1220
|
+
value: rejected(
|
|
1221
|
+
operation,
|
|
1222
|
+
"worker-state",
|
|
1223
|
+
"worker_abort requires owned active workers",
|
|
1224
|
+
),
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
872
1229
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
1230
|
+
const activeWorkers = workers.filter(isRuntimeWorker);
|
|
1231
|
+
const marked = this.markWorkersStopping(draft, activeWorkers, candidates);
|
|
1232
|
+
return {
|
|
1233
|
+
value: accepted(marked.completions),
|
|
1234
|
+
actions: [
|
|
1235
|
+
...marked.actions,
|
|
1236
|
+
...publishOwners(new Set(
|
|
1237
|
+
activeWorkers.map((worker) => worker.record.ownerSessionId),
|
|
1238
|
+
)),
|
|
1239
|
+
],
|
|
1240
|
+
};
|
|
1241
|
+
});
|
|
878
1242
|
}
|
|
879
1243
|
|
|
880
|
-
private
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
const [workerId] = this.terminalWorkerOrder.splice(removableIndex, 1);
|
|
899
|
-
if (!workerId) return affectedOwners;
|
|
900
|
-
const worker = this.workers.get(workerId);
|
|
901
|
-
if (worker) affectedOwners.add(worker.ownerSessionId);
|
|
902
|
-
this.workers.delete(workerId);
|
|
903
|
-
this.entries.delete(workerId);
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
return affectedOwners;
|
|
1244
|
+
private beginShutdownCancellation(
|
|
1245
|
+
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1246
|
+
): readonly Deferred.Deferred<void>[] {
|
|
1247
|
+
return this.transact((draft) => {
|
|
1248
|
+
const workers = [...draft.workers.values()].filter((worker) => (
|
|
1249
|
+
isActiveWorkerStatus(worker.record.status)
|
|
1250
|
+
));
|
|
1251
|
+
const marked = this.markWorkersStopping(draft, workers, candidates);
|
|
1252
|
+
return {
|
|
1253
|
+
value: marked.completions,
|
|
1254
|
+
actions: [
|
|
1255
|
+
...marked.actions,
|
|
1256
|
+
...publishOwners(new Set(
|
|
1257
|
+
workers.map((worker) => worker.record.ownerSessionId),
|
|
1258
|
+
)),
|
|
1259
|
+
],
|
|
1260
|
+
};
|
|
1261
|
+
});
|
|
907
1262
|
}
|
|
908
1263
|
|
|
909
|
-
private
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
]
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
if (candidate.workerIds !== undefined) {
|
|
924
|
-
if (!Array.isArray(candidate.workerIds) || candidate.workerIds.length === 0) {
|
|
925
|
-
throw new Error("workerIds must contain at least one worker ID");
|
|
1264
|
+
private markWorkersStopping(
|
|
1265
|
+
draft: RuntimeState,
|
|
1266
|
+
workers: readonly RuntimeWorker[],
|
|
1267
|
+
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1268
|
+
): {
|
|
1269
|
+
readonly completions: readonly Deferred.Deferred<void>[];
|
|
1270
|
+
readonly actions: readonly PostCommitAction[];
|
|
1271
|
+
} {
|
|
1272
|
+
const completions: Deferred.Deferred<void>[] = [];
|
|
1273
|
+
const actions: PostCommitAction[] = [];
|
|
1274
|
+
for (const worker of workers) {
|
|
1275
|
+
if (worker.cancellation) {
|
|
1276
|
+
completions.push(worker.cancellation);
|
|
1277
|
+
continue;
|
|
926
1278
|
}
|
|
927
|
-
const
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
if (worker.status === "ready") {
|
|
931
|
-
throw new Error("Ready interactive workers are not active; use interactive_close");
|
|
932
|
-
}
|
|
933
|
-
if (!isActiveWorkerStatus(worker.status)) {
|
|
934
|
-
throw new Error("worker_abort requires owned active workers");
|
|
935
|
-
}
|
|
1279
|
+
const completion = candidates.get(worker.record.id);
|
|
1280
|
+
if (!completion) {
|
|
1281
|
+
throw new Error(`Missing cancellation candidate: ${worker.record.id}`);
|
|
936
1282
|
}
|
|
937
|
-
|
|
1283
|
+
const stoppingRecord = worker.record.status === "stopping"
|
|
1284
|
+
? worker.record
|
|
1285
|
+
: {
|
|
1286
|
+
...transitionWorkerStatus(worker.record, "stopping"),
|
|
1287
|
+
activity: undefined,
|
|
1288
|
+
};
|
|
1289
|
+
draft.workers.set(worker.record.id, {
|
|
1290
|
+
...worker,
|
|
1291
|
+
record: stoppingRecord,
|
|
1292
|
+
cancellation: completion,
|
|
1293
|
+
});
|
|
1294
|
+
completions.push(completion);
|
|
1295
|
+
actions.push(runAction(() => {
|
|
1296
|
+
this.launchCancellationOrSettleAfterClosure(worker.record.id, completion);
|
|
1297
|
+
}));
|
|
938
1298
|
}
|
|
939
|
-
|
|
940
|
-
return [...this.workers.values()]
|
|
941
|
-
.filter(
|
|
942
|
-
(worker) =>
|
|
943
|
-
worker.ownerSessionId === ownerSessionId && isActiveWorkerStatus(worker.status),
|
|
944
|
-
)
|
|
945
|
-
.map((worker) => worker.id);
|
|
1299
|
+
return { completions, actions };
|
|
946
1300
|
}
|
|
947
1301
|
|
|
948
|
-
private
|
|
949
|
-
|
|
950
|
-
|
|
1302
|
+
private launchCancellationOrSettleAfterClosure(
|
|
1303
|
+
workerId: WorkerId,
|
|
1304
|
+
completion: Deferred.Deferred<void>,
|
|
1305
|
+
): void {
|
|
1306
|
+
const fiber = this.runCancellation(this.cancelWorker(workerId, completion));
|
|
1307
|
+
const exit = fiber.pollUnsafe();
|
|
1308
|
+
if (!exit || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return;
|
|
951
1309
|
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
1310
|
+
const worker = this.current().workers.get(workerId);
|
|
1311
|
+
if (worker?.record.status === "stopping") {
|
|
1312
|
+
this.settleActiveWorker(
|
|
1313
|
+
workerId,
|
|
1314
|
+
worker.generation,
|
|
1315
|
+
["stopping"],
|
|
1316
|
+
"aborted",
|
|
1317
|
+
{ status: "aborted" },
|
|
1318
|
+
"cancellation",
|
|
1319
|
+
true,
|
|
1320
|
+
);
|
|
963
1321
|
}
|
|
964
|
-
|
|
965
|
-
this.emitStateForOwners(owners);
|
|
966
|
-
await Promise.all(cancellations);
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
private cancellationFor(workerId: WorkerId): Promise<void> {
|
|
970
|
-
const existing = this.cancellationPromises.get(workerId);
|
|
971
|
-
if (existing) return existing;
|
|
972
|
-
|
|
973
|
-
const cancellation = Promise.resolve()
|
|
974
|
-
.then(() => this.cancelWorker(workerId))
|
|
975
|
-
.catch((error) => {
|
|
976
|
-
this.settleCancellationFailure(workerId, error);
|
|
977
|
-
});
|
|
978
|
-
this.cancellationPromises.set(workerId, cancellation);
|
|
979
|
-
void cancellation.then(() => {
|
|
980
|
-
if (this.cancellationPromises.get(workerId) === cancellation) {
|
|
981
|
-
this.cancellationPromises.delete(workerId);
|
|
982
|
-
}
|
|
983
|
-
});
|
|
984
|
-
return cancellation;
|
|
1322
|
+
this.completeCancellation(workerId, completion);
|
|
985
1323
|
}
|
|
986
1324
|
|
|
987
|
-
private
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
1325
|
+
private cancelWorker(
|
|
1326
|
+
workerId: WorkerId,
|
|
1327
|
+
completion: Deferred.Deferred<void>,
|
|
1328
|
+
): Effect.Effect<void> {
|
|
1329
|
+
return Effect.gen({ self: this }, function* () {
|
|
1330
|
+
const session = this.current().workers.get(workerId)?.session;
|
|
992
1331
|
if (session) {
|
|
993
|
-
|
|
994
|
-
.
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
// Worker state still settles even if scheduler cleanup fails.
|
|
1332
|
+
yield* session.abort().pipe(
|
|
1333
|
+
Effect.catchTag(
|
|
1334
|
+
"WorkerSession.AbortError",
|
|
1335
|
+
(_: WorkerSessionAbortError) => Effect.void,
|
|
1336
|
+
),
|
|
1337
|
+
Effect.timeoutOption(CANCELLATION_GRACE_MS),
|
|
1338
|
+
Effect.ignore,
|
|
1339
|
+
);
|
|
1002
1340
|
}
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1341
|
+
const removal = yield* FiberSet.run(
|
|
1342
|
+
this.cancellations,
|
|
1343
|
+
FiberMap.remove(this.generations, workerId).pipe(
|
|
1344
|
+
Effect.catchCause(() => Effect.void),
|
|
1345
|
+
),
|
|
1346
|
+
);
|
|
1347
|
+
yield* Fiber.await(removal).pipe(
|
|
1348
|
+
Effect.timeoutOption(CANCELLATION_GRACE_MS),
|
|
1349
|
+
Effect.ignore,
|
|
1350
|
+
);
|
|
1351
|
+
this.settleActiveWorker(
|
|
1352
|
+
workerId,
|
|
1353
|
+
this.current().workers.get(workerId)?.generation ?? -1,
|
|
1354
|
+
["stopping"],
|
|
1355
|
+
"aborted",
|
|
1356
|
+
{ status: "aborted" },
|
|
1357
|
+
"cancellation",
|
|
1358
|
+
true,
|
|
1359
|
+
);
|
|
1360
|
+
}).pipe(
|
|
1361
|
+
Effect.catchCause((cause) => Effect.sync(() => {
|
|
1362
|
+
const generation = this.current().workers.get(workerId)?.generation;
|
|
1363
|
+
if (generation === undefined) return;
|
|
1364
|
+
this.settleActiveWorker(
|
|
1365
|
+
workerId,
|
|
1366
|
+
generation,
|
|
1367
|
+
["stopping"],
|
|
1368
|
+
"failed",
|
|
1369
|
+
{
|
|
1370
|
+
status: "failed",
|
|
1371
|
+
message: describeError(
|
|
1372
|
+
Cause.squash(cause),
|
|
1373
|
+
"Worker cancellation failed",
|
|
1374
|
+
),
|
|
1375
|
+
},
|
|
1011
1376
|
"cancellation",
|
|
1377
|
+
true,
|
|
1012
1378
|
);
|
|
1013
|
-
}
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
const current = this.workers.get(workerId);
|
|
1019
|
-
if (!current || current.status !== "stopping") return;
|
|
1020
|
-
const entry = this.entries.get(workerId);
|
|
1021
|
-
if (entry) this.disposeEntrySession(entry);
|
|
1022
|
-
this.settleTerminalWorker(current, "failed", {
|
|
1023
|
-
status: "failed",
|
|
1024
|
-
message: describeError(error, "Worker cancellation failed"),
|
|
1025
|
-
}, "cancellation");
|
|
1379
|
+
})),
|
|
1380
|
+
Effect.ensuring(
|
|
1381
|
+
Effect.sync(() => this.completeCancellation(workerId, completion)),
|
|
1382
|
+
),
|
|
1383
|
+
);
|
|
1026
1384
|
}
|
|
1027
1385
|
|
|
1028
|
-
private
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1386
|
+
private completeCancellation(
|
|
1387
|
+
workerId: WorkerId,
|
|
1388
|
+
completion: Deferred.Deferred<void>,
|
|
1389
|
+
): void {
|
|
1390
|
+
this.transact((draft) => {
|
|
1391
|
+
const worker = draft.workers.get(workerId);
|
|
1392
|
+
if (worker?.cancellation === completion) {
|
|
1393
|
+
draft.workers.set(workerId, { ...worker, cancellation: undefined });
|
|
1394
|
+
}
|
|
1395
|
+
return { value: undefined };
|
|
1396
|
+
});
|
|
1397
|
+
Deferred.doneUnsafe(completion, Effect.void);
|
|
1037
1398
|
}
|
|
1038
1399
|
|
|
1039
1400
|
private awaitInlineRun(
|
|
1040
1401
|
run: RunRecord,
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
let abortClaimed = false;
|
|
1048
|
-
const removeAbortListener = () => signal.removeEventListener("abort", onAbort);
|
|
1049
|
-
const onAbort = () => {
|
|
1050
|
-
if (abortClaimed || waiter.settled) return;
|
|
1051
|
-
abortClaimed = true;
|
|
1052
|
-
removeAbortListener();
|
|
1053
|
-
const reason = abortSignalReason(signal);
|
|
1054
|
-
void this.cancelExactRun(run).then(
|
|
1055
|
-
() => reject(reason),
|
|
1056
|
-
() => reject(reason),
|
|
1057
|
-
);
|
|
1058
|
-
};
|
|
1059
|
-
|
|
1060
|
-
waiter.onSettled = () => {
|
|
1061
|
-
removeAbortListener();
|
|
1062
|
-
if (!abortClaimed) waiter.promise.then(resolve, reject);
|
|
1063
|
-
};
|
|
1064
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
1402
|
+
completion: Deferred.Deferred<CompletedRun>,
|
|
1403
|
+
): Effect.Effect<CompletedRun> {
|
|
1404
|
+
return Deferred.await(completion).pipe(
|
|
1405
|
+
Effect.onInterrupt(() => this.cancelExactRun(run)),
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1065
1408
|
|
|
1066
|
-
|
|
1067
|
-
|
|
1409
|
+
private cancelExactRun(run: RunRecord): Effect.Effect<void> {
|
|
1410
|
+
return Effect.gen({ self: this }, function* () {
|
|
1411
|
+
const candidate = yield* Deferred.make<void>();
|
|
1412
|
+
const candidates = new Map([[run.workerId, candidate]]);
|
|
1413
|
+
const completions = this.transact((draft) => {
|
|
1414
|
+
const worker = draft.workers.get(run.workerId);
|
|
1415
|
+
if (
|
|
1416
|
+
!worker ||
|
|
1417
|
+
worker.record.ownerSessionId !== run.ownerSessionId ||
|
|
1418
|
+
worker.record.runId !== run.id ||
|
|
1419
|
+
!isActiveWorkerStatus(worker.record.status)
|
|
1420
|
+
) {
|
|
1421
|
+
return { value: [] as readonly Deferred.Deferred<void>[] };
|
|
1422
|
+
}
|
|
1423
|
+
const marked = this.markWorkersStopping(draft, [worker], candidates);
|
|
1424
|
+
return {
|
|
1425
|
+
value: marked.completions,
|
|
1426
|
+
actions: [...marked.actions, publishState(run.ownerSessionId)],
|
|
1427
|
+
};
|
|
1428
|
+
});
|
|
1429
|
+
yield* awaitAll(completions);
|
|
1068
1430
|
});
|
|
1069
1431
|
}
|
|
1070
1432
|
|
|
1071
|
-
private
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1433
|
+
private closeReadyWorker(
|
|
1434
|
+
draft: RuntimeState,
|
|
1435
|
+
worker: RuntimeWorker,
|
|
1436
|
+
settledAt: number,
|
|
1437
|
+
actions: PostCommitAction[] = [],
|
|
1438
|
+
): PostCommitAction[] {
|
|
1439
|
+
const closedRecord: WorkerRecord = {
|
|
1440
|
+
...transitionWorkerStatus(worker.record, "closed"),
|
|
1076
1441
|
activity: undefined,
|
|
1077
1442
|
outcome: { status: "closed" },
|
|
1078
|
-
settledAt
|
|
1443
|
+
settledAt,
|
|
1444
|
+
};
|
|
1445
|
+
actions.push(...this.releaseWorkerResources(worker));
|
|
1446
|
+
draft.workers.set(worker.record.id, {
|
|
1447
|
+
...worker,
|
|
1448
|
+
record: closedRecord,
|
|
1449
|
+
session: undefined,
|
|
1450
|
+
observationRelease: undefined,
|
|
1079
1451
|
});
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
affectedOwners.add(current.ownerSessionId);
|
|
1083
|
-
this.emitStateForOwners(affectedOwners);
|
|
1452
|
+
rememberTerminalWorker(draft, worker.record.id);
|
|
1453
|
+
return actions;
|
|
1084
1454
|
}
|
|
1085
1455
|
|
|
1086
|
-
private
|
|
1087
|
-
const
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
for (const worker of readyInteractiveWorkers) {
|
|
1091
|
-
this.closeReadyInteractiveWorker(worker);
|
|
1456
|
+
private releaseWorkerResources(worker: RuntimeWorker): PostCommitAction[] {
|
|
1457
|
+
const actions: PostCommitAction[] = [];
|
|
1458
|
+
if (worker.observationRelease) {
|
|
1459
|
+
actions.push(runAction(() => safelyCall(worker.observationRelease)));
|
|
1092
1460
|
}
|
|
1461
|
+
const session = worker.session;
|
|
1462
|
+
if (session) {
|
|
1463
|
+
actions.push(runAction(() => {
|
|
1464
|
+
this.launchCleanupOrJoinAfterClosure(
|
|
1465
|
+
Effect.suspend(() => session.dispose()).pipe(
|
|
1466
|
+
Effect.catchCause(() => Effect.void),
|
|
1467
|
+
Effect.uninterruptible,
|
|
1468
|
+
),
|
|
1469
|
+
);
|
|
1470
|
+
}));
|
|
1471
|
+
}
|
|
1472
|
+
return actions;
|
|
1093
1473
|
}
|
|
1094
1474
|
|
|
1095
|
-
private
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
this.emitState(latest.ownerSessionId);
|
|
1108
|
-
});
|
|
1109
|
-
entry.unsubscribeActivity = session.subscribeActivity((activity) => {
|
|
1110
|
-
const latest = this.workers.get(workerId);
|
|
1111
|
-
if (!latest || entry.session !== session || entry.generation !== generation) return;
|
|
1112
|
-
if (latest.status !== "starting" && latest.status !== "running") return;
|
|
1113
|
-
if (latest.activity === activity) return;
|
|
1114
|
-
this.workers.set(workerId, { ...latest, activity });
|
|
1115
|
-
this.emitState(latest.ownerSessionId);
|
|
1116
|
-
});
|
|
1117
|
-
entry.unsubscribeMessageDirection = session.subscribeMessageDirection((messageDirection) => {
|
|
1118
|
-
const latest = this.workers.get(workerId);
|
|
1119
|
-
if (!latest || entry.session !== session || entry.generation !== generation) return;
|
|
1120
|
-
if (latest.status !== "starting" && latest.status !== "running") return;
|
|
1121
|
-
if (latest.messageDirection === messageDirection) return;
|
|
1122
|
-
this.workers.set(workerId, { ...latest, messageDirection });
|
|
1123
|
-
this.emitState(latest.ownerSessionId);
|
|
1124
|
-
});
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
|
-
private emitState(ownerSessionId: string): void {
|
|
1128
|
-
for (const listener of [...this.stateListeners]) {
|
|
1129
|
-
try {
|
|
1130
|
-
listener(ownerSessionId);
|
|
1131
|
-
} catch {
|
|
1132
|
-
// One subscriber cannot prevent runtime mutations or other notifications.
|
|
1133
|
-
}
|
|
1475
|
+
private launchCleanupOrJoinAfterClosure(cleanup: Effect.Effect<void>): void {
|
|
1476
|
+
if (this.cleanups.state._tag === "Closed") {
|
|
1477
|
+
Effect.runFork(cleanup);
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
const fiber = this.runCleanup(cleanup);
|
|
1481
|
+
const exit = fiber.pollUnsafe();
|
|
1482
|
+
if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
1483
|
+
// FiberSet.runtime returns an already-interrupted sentinel when closure
|
|
1484
|
+
// wins admission. If admission won and closure interrupted the real fiber,
|
|
1485
|
+
// repeating the cached uninterruptible disposal only joins that cleanup.
|
|
1486
|
+
Effect.runFork(cleanup);
|
|
1134
1487
|
}
|
|
1135
1488
|
}
|
|
1136
1489
|
|
|
1137
|
-
private
|
|
1138
|
-
|
|
1490
|
+
private preflightOpen(
|
|
1491
|
+
operation: OrchestrationOperation,
|
|
1492
|
+
): Decision<void> {
|
|
1493
|
+
return this.transact((draft) => ({
|
|
1494
|
+
value: openDecision(draft, operation),
|
|
1495
|
+
}));
|
|
1139
1496
|
}
|
|
1140
1497
|
|
|
1141
|
-
private
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1498
|
+
private preflightReadyInteractive(
|
|
1499
|
+
ownerSessionId: string,
|
|
1500
|
+
workerId: WorkerId,
|
|
1501
|
+
): Decision<void> {
|
|
1502
|
+
return this.transact((draft) => {
|
|
1503
|
+
const open = openDecision(draft, "sendInteractive");
|
|
1504
|
+
if (open._tag === "rejected") return { value: open };
|
|
1505
|
+
const ownership = ownedWorkerDecision(
|
|
1506
|
+
draft,
|
|
1507
|
+
"sendInteractive",
|
|
1508
|
+
ownerSessionId,
|
|
1509
|
+
workerId,
|
|
1510
|
+
);
|
|
1511
|
+
if (ownership._tag === "rejected") return { value: ownership };
|
|
1512
|
+
const worker = ownership.value;
|
|
1513
|
+
if (
|
|
1514
|
+
worker.record.lifecycle !== "interactive" ||
|
|
1515
|
+
worker.record.status !== "ready" ||
|
|
1516
|
+
!worker.session
|
|
1517
|
+
) {
|
|
1518
|
+
return {
|
|
1519
|
+
value: rejected(
|
|
1520
|
+
"sendInteractive",
|
|
1521
|
+
"worker-state",
|
|
1522
|
+
"interactive_send requires an owned ready interactive worker",
|
|
1523
|
+
),
|
|
1524
|
+
};
|
|
1525
|
+
}
|
|
1526
|
+
return { value: accepted(undefined) };
|
|
1527
|
+
});
|
|
1147
1528
|
}
|
|
1148
1529
|
|
|
1149
|
-
private
|
|
1150
|
-
|
|
1151
|
-
entry.unsubscribeUsage = undefined;
|
|
1152
|
-
safelyCall(entry.unsubscribeActivity);
|
|
1153
|
-
entry.unsubscribeActivity = undefined;
|
|
1154
|
-
safelyCall(entry.unsubscribeMessageDirection);
|
|
1155
|
-
entry.unsubscribeMessageDirection = undefined;
|
|
1530
|
+
private current(): RuntimeState {
|
|
1531
|
+
return this.state;
|
|
1156
1532
|
}
|
|
1157
1533
|
|
|
1158
|
-
private
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1534
|
+
private transact<A>(
|
|
1535
|
+
reducer: (draft: RuntimeState) => TransactionMutation<A>,
|
|
1536
|
+
): A {
|
|
1537
|
+
const draft = makeDraft(this.state);
|
|
1538
|
+
const mutation = reducer(draft);
|
|
1539
|
+
this.state = Object.freeze(draft);
|
|
1540
|
+
this.enqueueActions((mutation.actions ?? []).map((action) => action(
|
|
1541
|
+
this.state,
|
|
1542
|
+
this.settlementListeners,
|
|
1543
|
+
this.stateListeners,
|
|
1544
|
+
)));
|
|
1545
|
+
return mutation.value;
|
|
1162
1546
|
}
|
|
1163
1547
|
|
|
1164
|
-
private
|
|
1165
|
-
|
|
1166
|
-
this.
|
|
1548
|
+
private enqueueActions(actions: readonly CommittedAction[]): void {
|
|
1549
|
+
this.actionQueue.push(...actions);
|
|
1550
|
+
if (this.drainingActions) return;
|
|
1167
1551
|
|
|
1552
|
+
this.drainingActions = true;
|
|
1553
|
+
let hasFailure = false;
|
|
1554
|
+
let firstFailure: unknown;
|
|
1168
1555
|
try {
|
|
1169
|
-
this.
|
|
1170
|
-
|
|
1171
|
-
|
|
1556
|
+
while (this.actionQueue.length > 0) {
|
|
1557
|
+
const action = this.actionQueue.shift();
|
|
1558
|
+
if (!action) continue;
|
|
1559
|
+
try {
|
|
1560
|
+
action();
|
|
1561
|
+
} catch (error) {
|
|
1562
|
+
if (!hasFailure) firstFailure = error;
|
|
1563
|
+
hasFailure = true;
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
} finally {
|
|
1567
|
+
this.drainingActions = false;
|
|
1172
1568
|
}
|
|
1569
|
+
if (hasFailure) throw firstFailure;
|
|
1173
1570
|
}
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
export function orchestrationLayer(
|
|
1574
|
+
options: OrchestrationLayerOptions = {},
|
|
1575
|
+
): Layer.Layer<Orchestration, never, ChildSessions> {
|
|
1576
|
+
return Layer.effect(
|
|
1577
|
+
Orchestration,
|
|
1578
|
+
Effect.gen(function* () {
|
|
1579
|
+
const childSessions = yield* ChildSessions;
|
|
1580
|
+
// Scope finalizers run in reverse acquisition order. Keep cleanup open while
|
|
1581
|
+
// generation and cancellation interruption settle workers and enqueue disposal.
|
|
1582
|
+
const cleanups = yield* FiberSet.make<void, never>();
|
|
1583
|
+
const runCleanup = yield* FiberSet.runtime(cleanups)<never>();
|
|
1584
|
+
const cancellations = yield* FiberSet.make<void, never>();
|
|
1585
|
+
const runCancellation = yield* FiberSet.runtime(cancellations)<never>();
|
|
1586
|
+
const generations = yield* FiberMap.make<WorkerId, void, never>();
|
|
1587
|
+
const runGeneration = yield* FiberMap.runtime(generations)<never>();
|
|
1588
|
+
const clock = yield* Clock.Clock;
|
|
1589
|
+
const shutdownCompletion = yield* Deferred.make<void>();
|
|
1590
|
+
return Orchestration.of(new StatefulOrchestration(
|
|
1591
|
+
childSessions,
|
|
1592
|
+
generations,
|
|
1593
|
+
runGeneration,
|
|
1594
|
+
cancellations,
|
|
1595
|
+
runCancellation,
|
|
1596
|
+
cleanups,
|
|
1597
|
+
runCleanup,
|
|
1598
|
+
clock,
|
|
1599
|
+
options.idFactories ?? createRandomIdFactories(),
|
|
1600
|
+
shutdownCompletion,
|
|
1601
|
+
));
|
|
1602
|
+
}),
|
|
1603
|
+
);
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
function initialState(): RuntimeState {
|
|
1607
|
+
return Object.freeze({
|
|
1608
|
+
workers: new Map<WorkerId, RuntimeWorker>(),
|
|
1609
|
+
runs: new Map<RunId, RuntimeRun>(),
|
|
1610
|
+
terminalWorkerOrder: [],
|
|
1611
|
+
completedRunOrder: [],
|
|
1612
|
+
settlementSequence: 0,
|
|
1613
|
+
lifecycle: "open",
|
|
1614
|
+
});
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
function makeDraft(state: RuntimeState): RuntimeState {
|
|
1618
|
+
return {
|
|
1619
|
+
...state,
|
|
1620
|
+
workers: new Map(state.workers),
|
|
1621
|
+
runs: new Map(state.runs),
|
|
1622
|
+
terminalWorkerOrder: [...state.terminalWorkerOrder],
|
|
1623
|
+
completedRunOrder: [...state.completedRunOrder],
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1174
1626
|
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1627
|
+
function completeRun(
|
|
1628
|
+
state: RuntimeState,
|
|
1629
|
+
workerId: WorkerId,
|
|
1630
|
+
): PostCommitAction[] {
|
|
1631
|
+
const worker = state.workers.get(workerId);
|
|
1632
|
+
if (!worker) return [];
|
|
1633
|
+
const run = state.runs.get(worker.record.runId);
|
|
1634
|
+
if (!run || !isRunningRuntimeRun(run) || !isCompletedRunWorker(worker.record)) {
|
|
1635
|
+
return [];
|
|
1180
1636
|
}
|
|
1181
1637
|
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1638
|
+
const completed = freezeCompletedRun(run.record, worker.record);
|
|
1639
|
+
state.runs.set(run.record.id, { ...run.record, state: "complete" });
|
|
1640
|
+
state.completedRunOrder.push(run.record.id);
|
|
1641
|
+
return [runAction(() => {
|
|
1642
|
+
Deferred.doneUnsafe(run.completion, Effect.succeed(completed));
|
|
1643
|
+
})];
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
function makeSettlement(
|
|
1647
|
+
draft: RuntimeState,
|
|
1648
|
+
worker: RuntimeWorker,
|
|
1649
|
+
settledAt: number,
|
|
1650
|
+
failureStage: SettlementFailureStage | undefined,
|
|
1651
|
+
): PostCommitAction | undefined {
|
|
1652
|
+
const run = draft.runs.get(worker.record.runId);
|
|
1653
|
+
if (!run || !isCompletedRunWorker(worker.record)) {
|
|
1654
|
+
return undefined;
|
|
1189
1655
|
}
|
|
1656
|
+
const runRecord = runtimeRunRecord(run);
|
|
1657
|
+
const sequence = draft.settlementSequence + 1;
|
|
1658
|
+
draft.settlementSequence = sequence;
|
|
1659
|
+
const settlement: WorkerSettlement = Object.freeze({
|
|
1660
|
+
eventId: `${sequence}:${runRecord.id}:${worker.record.id}:${worker.generation}`,
|
|
1661
|
+
sequence,
|
|
1662
|
+
ownerSessionId: worker.record.ownerSessionId,
|
|
1663
|
+
runId: runRecord.id,
|
|
1664
|
+
workerId: worker.record.id,
|
|
1665
|
+
generation: worker.generation,
|
|
1666
|
+
mode: runRecord.mode,
|
|
1667
|
+
worker: worker.record.worker,
|
|
1668
|
+
title: worker.record.title,
|
|
1669
|
+
lifecycle: worker.record.lifecycle,
|
|
1670
|
+
status: worker.record.status,
|
|
1671
|
+
outcome: Object.freeze(copyOutcome(worker.record.outcome)),
|
|
1672
|
+
...(failureStage ? { failureStage } : {}),
|
|
1673
|
+
usage: Object.freeze(copyUsage(worker.record.usage)),
|
|
1674
|
+
startedAt: worker.record.startedAt,
|
|
1675
|
+
settledAt,
|
|
1676
|
+
...(runRecord.synthesisGroupId && runRecord.synthesisGroupSize
|
|
1677
|
+
? {
|
|
1678
|
+
synthesisGroupId: runRecord.synthesisGroupId,
|
|
1679
|
+
synthesisGroupSize: runRecord.synthesisGroupSize,
|
|
1680
|
+
}
|
|
1681
|
+
: {}),
|
|
1682
|
+
...(worker.record.sessionFile !== undefined
|
|
1683
|
+
? { sessionFile: worker.record.sessionFile }
|
|
1684
|
+
: {}),
|
|
1685
|
+
});
|
|
1686
|
+
return publishSettlement(
|
|
1687
|
+
settlement,
|
|
1688
|
+
isRunningRuntimeRun(run) ? run.settlementListener : undefined,
|
|
1689
|
+
);
|
|
1690
|
+
}
|
|
1190
1691
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1692
|
+
function stateActionsAfterPrune(
|
|
1693
|
+
draft: RuntimeState,
|
|
1694
|
+
ownerSessionId: string,
|
|
1695
|
+
): PostCommitAction[] {
|
|
1696
|
+
const owners = pruneHistory(draft);
|
|
1697
|
+
owners.add(ownerSessionId);
|
|
1698
|
+
return publishOwners(owners);
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
function pruneHistory(draft: RuntimeState): Set<string> {
|
|
1702
|
+
const owners = new Set<string>();
|
|
1703
|
+
while (draft.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
|
|
1704
|
+
const runId = draft.completedRunOrder.shift();
|
|
1705
|
+
if (!runId) break;
|
|
1706
|
+
const run = draft.runs.get(runId);
|
|
1707
|
+
if (run) owners.add(runtimeRunRecord(run).ownerSessionId);
|
|
1708
|
+
draft.runs.delete(runId);
|
|
1709
|
+
}
|
|
1710
|
+
while (draft.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
|
|
1711
|
+
const index = draft.terminalWorkerOrder.findIndex((workerId) => {
|
|
1712
|
+
const worker = draft.workers.get(workerId);
|
|
1713
|
+
return !worker || isTerminalWorkerStatus(worker.record.status);
|
|
1714
|
+
});
|
|
1715
|
+
if (index < 0) break;
|
|
1716
|
+
const removed = draft.terminalWorkerOrder.splice(index, 1)[0];
|
|
1717
|
+
if (!removed) break;
|
|
1718
|
+
const worker = draft.workers.get(removed);
|
|
1719
|
+
if (worker) owners.add(worker.record.ownerSessionId);
|
|
1720
|
+
draft.workers.delete(removed);
|
|
1197
1721
|
}
|
|
1722
|
+
return owners;
|
|
1723
|
+
}
|
|
1198
1724
|
|
|
1199
|
-
|
|
1200
|
-
|
|
1725
|
+
function rememberTerminalWorker(
|
|
1726
|
+
draft: RuntimeState,
|
|
1727
|
+
workerId: WorkerId,
|
|
1728
|
+
): void {
|
|
1729
|
+
if (!draft.terminalWorkerOrder.includes(workerId)) {
|
|
1730
|
+
draft.terminalWorkerOrder.push(workerId);
|
|
1201
1731
|
}
|
|
1202
1732
|
}
|
|
1203
1733
|
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1734
|
+
function snapshotFor(
|
|
1735
|
+
state: RuntimeState,
|
|
1736
|
+
ownerSessionId: string,
|
|
1737
|
+
): RuntimeSnapshot {
|
|
1738
|
+
return Object.freeze({
|
|
1739
|
+
runs: Object.freeze(
|
|
1740
|
+
[...state.runs.values()]
|
|
1741
|
+
.map(runtimeRunRecord)
|
|
1742
|
+
.filter((run) => run.ownerSessionId === ownerSessionId)
|
|
1743
|
+
.map(copyRunRecord),
|
|
1744
|
+
),
|
|
1745
|
+
workers: Object.freeze(
|
|
1746
|
+
[...state.workers.values()]
|
|
1747
|
+
.map((worker) => worker.record)
|
|
1748
|
+
.filter((worker) => worker.ownerSessionId === ownerSessionId)
|
|
1749
|
+
.map(copyWorkerRecord),
|
|
1750
|
+
),
|
|
1751
|
+
});
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
function runtimeRunRecord(run: RuntimeRun): RunRecord {
|
|
1755
|
+
return isRunningRuntimeRun(run) ? run.record : run;
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
function isRunningRuntimeRun(run: RuntimeRun): run is RunningRuntimeRun {
|
|
1759
|
+
return "_tag" in run;
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
function makeRunRecord(
|
|
1763
|
+
runId: RunId,
|
|
1764
|
+
workerId: WorkerId,
|
|
1765
|
+
context: OrchestrationContext,
|
|
1766
|
+
mode: RunMode,
|
|
1767
|
+
createdAt: number,
|
|
1768
|
+
): RunRecord {
|
|
1769
|
+
return {
|
|
1770
|
+
id: runId,
|
|
1771
|
+
ownerSessionId: context.ownerSessionId,
|
|
1772
|
+
workerId,
|
|
1773
|
+
mode,
|
|
1774
|
+
state: "running",
|
|
1775
|
+
createdAt,
|
|
1776
|
+
...(context.synthesisGroup
|
|
1777
|
+
? {
|
|
1778
|
+
synthesisGroupId: context.synthesisGroup.id,
|
|
1779
|
+
synthesisGroupSize: context.synthesisGroup.size,
|
|
1780
|
+
}
|
|
1781
|
+
: {}),
|
|
1782
|
+
};
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
function makeWorkerRecord(
|
|
1786
|
+
workerId: WorkerId,
|
|
1787
|
+
runId: RunId,
|
|
1788
|
+
ownerSessionId: string,
|
|
1789
|
+
definition: WorkerDefinition,
|
|
1790
|
+
task: OrchestrateTaskInput,
|
|
1791
|
+
startedAt: number,
|
|
1792
|
+
): WorkerRecord {
|
|
1793
|
+
return {
|
|
1794
|
+
id: workerId,
|
|
1795
|
+
worker: definition.name,
|
|
1796
|
+
ownerSessionId,
|
|
1797
|
+
runId,
|
|
1798
|
+
title: task.title,
|
|
1799
|
+
instructions: task.instructions,
|
|
1800
|
+
lifecycle: definition.lifecycle,
|
|
1801
|
+
status: "starting",
|
|
1802
|
+
usage: copyUsage(EMPTY_WORKER_USAGE),
|
|
1803
|
+
messageDirection: "to-model",
|
|
1804
|
+
startedAt,
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
function publishState(ownerSessionId: string): PostCommitAction {
|
|
1809
|
+
return (state, _settlementListeners, stateListeners) => {
|
|
1810
|
+
const snapshot = snapshotFor(state, ownerSessionId);
|
|
1811
|
+
const listeners = [...(stateListeners.get(ownerSessionId) ?? [])];
|
|
1812
|
+
return () => {
|
|
1813
|
+
for (const listener of listeners) safelyNotify(() => listener(snapshot));
|
|
1814
|
+
};
|
|
1815
|
+
};
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
function publishStateTo(
|
|
1819
|
+
ownerSessionId: string,
|
|
1820
|
+
listener: StateListener,
|
|
1821
|
+
): PostCommitAction {
|
|
1822
|
+
return (state) => {
|
|
1823
|
+
const snapshot = snapshotFor(state, ownerSessionId);
|
|
1824
|
+
return () => safelyNotify(() => listener(snapshot));
|
|
1825
|
+
};
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
function publishSettlement(
|
|
1829
|
+
settlement: WorkerSettlement,
|
|
1830
|
+
localListener: SettlementListener | undefined,
|
|
1831
|
+
): PostCommitAction {
|
|
1832
|
+
return (_state, settlementListeners) => {
|
|
1833
|
+
const listeners = [...settlementListeners];
|
|
1834
|
+
return () => {
|
|
1835
|
+
if (localListener) safelyNotify(() => localListener(settlement));
|
|
1836
|
+
for (const listener of listeners) safelyNotify(() => listener(settlement));
|
|
1837
|
+
};
|
|
1838
|
+
};
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
function publishOwners(owners: ReadonlySet<string>): PostCommitAction[] {
|
|
1842
|
+
return [...owners].map(publishState);
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
function runAction(run: () => void): PostCommitAction {
|
|
1846
|
+
return () => run;
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
function openDecision(
|
|
1850
|
+
draft: RuntimeState,
|
|
1851
|
+
operation: OrchestrationOperation,
|
|
1852
|
+
): Decision<void> {
|
|
1853
|
+
return draft.lifecycle === "open"
|
|
1854
|
+
? accepted(undefined)
|
|
1855
|
+
: rejected(
|
|
1856
|
+
operation,
|
|
1857
|
+
"shutdown",
|
|
1858
|
+
"Orchestrator runtime is shutting down",
|
|
1859
|
+
);
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
function ownedWorkerDecision(
|
|
1863
|
+
draft: RuntimeState,
|
|
1864
|
+
operation: OrchestrationOperation,
|
|
1865
|
+
ownerSessionId: string,
|
|
1866
|
+
workerId: WorkerId,
|
|
1867
|
+
): Decision<RuntimeWorker> {
|
|
1868
|
+
const worker = draft.workers.get(workerId);
|
|
1869
|
+
return !worker || worker.record.ownerSessionId !== ownerSessionId
|
|
1870
|
+
? rejected(
|
|
1871
|
+
operation,
|
|
1872
|
+
"ownership",
|
|
1873
|
+
"Worker is not owned by this session",
|
|
1874
|
+
)
|
|
1875
|
+
: accepted(worker);
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
function accepted<A>(value: A): Decision<A> {
|
|
1879
|
+
return { _tag: "accepted", value };
|
|
1208
1880
|
}
|
|
1209
1881
|
|
|
1210
|
-
function
|
|
1211
|
-
|
|
1212
|
-
|
|
1882
|
+
function rejected(
|
|
1883
|
+
operation: OrchestrationOperation,
|
|
1884
|
+
reason: OrchestrationRejectionReason,
|
|
1885
|
+
message: string,
|
|
1886
|
+
): Decision<never> {
|
|
1887
|
+
return {
|
|
1888
|
+
_tag: "rejected",
|
|
1889
|
+
error: actionRejection(operation, reason, message),
|
|
1890
|
+
};
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
function actionRejection(
|
|
1894
|
+
operation: OrchestrationOperation,
|
|
1895
|
+
reason: OrchestrationRejectionReason,
|
|
1896
|
+
message: string,
|
|
1897
|
+
): OrchestrationActionRejected {
|
|
1898
|
+
return new OrchestrationActionRejected({ operation, reason, message });
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
function rejectAction(
|
|
1902
|
+
operation: OrchestrationOperation,
|
|
1903
|
+
reason: OrchestrationRejectionReason,
|
|
1904
|
+
message: string,
|
|
1905
|
+
): Effect.Effect<never, OrchestrationActionRejected> {
|
|
1906
|
+
return Effect.fail(actionRejection(operation, reason, message));
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
function validateContextOwner(
|
|
1910
|
+
operation: OrchestrationOperation,
|
|
1911
|
+
ownerSessionId: string,
|
|
1912
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1913
|
+
return typeof ownerSessionId !== "string" || ownerSessionId.trim() === ""
|
|
1914
|
+
? rejectAction(
|
|
1915
|
+
operation,
|
|
1916
|
+
"validation",
|
|
1917
|
+
"ownerSessionId must not be blank",
|
|
1918
|
+
)
|
|
1919
|
+
: Effect.void;
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
function validateWorkerId(
|
|
1923
|
+
operation: OrchestrationOperation,
|
|
1924
|
+
workerId: string,
|
|
1925
|
+
): Effect.Effect<WorkerId, OrchestrationActionRejected> {
|
|
1926
|
+
if (typeof workerId !== "string" || workerId.trim() === "") {
|
|
1927
|
+
return rejectAction(
|
|
1928
|
+
operation,
|
|
1929
|
+
"validation",
|
|
1930
|
+
"worker_id must not be blank",
|
|
1931
|
+
);
|
|
1213
1932
|
}
|
|
1933
|
+
return Schema.decodeUnknownEffect(WorkerId)(workerId).pipe(
|
|
1934
|
+
Effect.mapError(() => actionRejection(
|
|
1935
|
+
operation,
|
|
1936
|
+
"validation",
|
|
1937
|
+
"worker_id must use the canonical worker- prefix",
|
|
1938
|
+
)),
|
|
1939
|
+
);
|
|
1214
1940
|
}
|
|
1215
1941
|
|
|
1216
|
-
function validateMode(
|
|
1217
|
-
|
|
1942
|
+
function validateMode(
|
|
1943
|
+
operation: OrchestrationOperation,
|
|
1944
|
+
mode: RunMode,
|
|
1945
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1946
|
+
return mode !== "async" && mode !== "inline"
|
|
1947
|
+
? rejectAction(operation, "validation", "Invalid orchestration mode")
|
|
1948
|
+
: Effect.void;
|
|
1218
1949
|
}
|
|
1219
1950
|
|
|
1220
|
-
function validateText(
|
|
1951
|
+
function validateText(
|
|
1952
|
+
operation: OrchestrationOperation,
|
|
1953
|
+
name: string,
|
|
1954
|
+
value: string,
|
|
1955
|
+
maximumLength: number,
|
|
1956
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1221
1957
|
if (typeof value !== "string" || value.trim() === "") {
|
|
1222
|
-
|
|
1223
|
-
}
|
|
1224
|
-
if (value.length > maximumLength) {
|
|
1225
|
-
throw new Error(`${name} must be at most ${maximumLength} characters`);
|
|
1958
|
+
return rejectAction(operation, "validation", `${name} must not be blank`);
|
|
1226
1959
|
}
|
|
1960
|
+
return value.length > maximumLength
|
|
1961
|
+
? rejectAction(
|
|
1962
|
+
operation,
|
|
1963
|
+
"validation",
|
|
1964
|
+
`${name} must be at most ${maximumLength} characters`,
|
|
1965
|
+
)
|
|
1966
|
+
: Effect.void;
|
|
1227
1967
|
}
|
|
1228
1968
|
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1969
|
+
type ValidatedAbortTarget =
|
|
1970
|
+
| {
|
|
1971
|
+
readonly _tag: "ids";
|
|
1972
|
+
readonly workerIds: readonly WorkerId[];
|
|
1973
|
+
}
|
|
1974
|
+
| {
|
|
1975
|
+
readonly _tag: "all";
|
|
1976
|
+
};
|
|
1977
|
+
|
|
1978
|
+
function validateAbortTarget(
|
|
1979
|
+
target: AbortTarget,
|
|
1980
|
+
): Effect.Effect<ValidatedAbortTarget, OrchestrationActionRejected> {
|
|
1981
|
+
return Effect.gen(function* () {
|
|
1982
|
+
if (!target || typeof target !== "object") {
|
|
1983
|
+
return yield* rejectAction("abort", "target", "Invalid abort target");
|
|
1984
|
+
}
|
|
1985
|
+
const selected = [target.workerIds !== undefined, target.all !== undefined]
|
|
1986
|
+
.filter(Boolean).length;
|
|
1987
|
+
if (selected !== 1 || (target.all !== undefined && target.all !== true)) {
|
|
1988
|
+
return yield* rejectAction(
|
|
1989
|
+
"abort",
|
|
1990
|
+
"target",
|
|
1991
|
+
"Abort target must specify exactly one of workerIds or all: true",
|
|
1992
|
+
);
|
|
1993
|
+
}
|
|
1994
|
+
if (target.workerIds === undefined) return { _tag: "all" };
|
|
1995
|
+
if (!Array.isArray(target.workerIds) || target.workerIds.length === 0) {
|
|
1996
|
+
return yield* rejectAction(
|
|
1997
|
+
"abort",
|
|
1998
|
+
"target",
|
|
1999
|
+
"workerIds must contain at least one worker ID",
|
|
2000
|
+
);
|
|
2001
|
+
}
|
|
2002
|
+
const workerIds = yield* Effect.all(
|
|
2003
|
+
[...new Set(target.workerIds)].map((id) => validateWorkerId("abort", id)),
|
|
2004
|
+
);
|
|
2005
|
+
return { _tag: "ids", workerIds };
|
|
2006
|
+
});
|
|
1233
2007
|
}
|
|
1234
2008
|
|
|
1235
|
-
function
|
|
1236
|
-
|
|
2009
|
+
function makeCancellationCandidates(
|
|
2010
|
+
workerIds: readonly WorkerId[],
|
|
2011
|
+
): Effect.Effect<ReadonlyMap<WorkerId, Deferred.Deferred<void>>> {
|
|
2012
|
+
return Effect.forEach(workerIds, (workerId) => Deferred.make<void>().pipe(
|
|
2013
|
+
Effect.map((completion) => [workerId, completion] as const),
|
|
2014
|
+
)).pipe(Effect.map((entries) => new Map(entries)));
|
|
1237
2015
|
}
|
|
1238
2016
|
|
|
1239
|
-
function
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
2017
|
+
function activeWorkerIds(
|
|
2018
|
+
state: RuntimeState,
|
|
2019
|
+
ownerSessionId?: string,
|
|
2020
|
+
): WorkerId[] {
|
|
2021
|
+
return [...state.workers.values()]
|
|
2022
|
+
.filter((worker) => (
|
|
2023
|
+
(ownerSessionId === undefined ||
|
|
2024
|
+
worker.record.ownerSessionId === ownerSessionId) &&
|
|
2025
|
+
isActiveWorkerStatus(worker.record.status)
|
|
2026
|
+
))
|
|
2027
|
+
.map((worker) => worker.record.id);
|
|
1243
2028
|
}
|
|
1244
2029
|
|
|
1245
|
-
function
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
2030
|
+
function awaitAll(
|
|
2031
|
+
deferreds: readonly Deferred.Deferred<void>[],
|
|
2032
|
+
): Effect.Effect<void> {
|
|
2033
|
+
return Effect.forEach(
|
|
2034
|
+
deferreds,
|
|
2035
|
+
(deferred) => Deferred.await(deferred),
|
|
2036
|
+
{ concurrency: "unbounded", discard: true },
|
|
2037
|
+
);
|
|
1252
2038
|
}
|
|
1253
2039
|
|
|
1254
|
-
function
|
|
1255
|
-
|
|
2040
|
+
function isRuntimeWorker(
|
|
2041
|
+
worker: RuntimeWorker | undefined,
|
|
2042
|
+
): worker is RuntimeWorker {
|
|
2043
|
+
return worker !== undefined;
|
|
1256
2044
|
}
|
|
1257
2045
|
|
|
1258
|
-
function
|
|
1259
|
-
|
|
1260
|
-
return new DOMException("This operation was aborted", "AbortError");
|
|
2046
|
+
function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
|
|
2047
|
+
return status === "starting" || status === "running" || status === "stopping";
|
|
1261
2048
|
}
|
|
1262
2049
|
|
|
1263
|
-
function
|
|
1264
|
-
|
|
2050
|
+
function isCompletedRunWorker(
|
|
2051
|
+
record: WorkerRecord,
|
|
2052
|
+
): record is WorkerRecord & {
|
|
2053
|
+
readonly status: RunResult["status"];
|
|
2054
|
+
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2055
|
+
readonly settledAt: number;
|
|
2056
|
+
} {
|
|
2057
|
+
return (
|
|
2058
|
+
(record.status === "completed" ||
|
|
2059
|
+
record.status === "ready" ||
|
|
2060
|
+
record.status === "failed" ||
|
|
2061
|
+
record.status === "aborted") &&
|
|
2062
|
+
record.outcome !== undefined &&
|
|
2063
|
+
record.outcome.status !== "closed" &&
|
|
2064
|
+
record.settledAt !== undefined
|
|
2065
|
+
);
|
|
1265
2066
|
}
|
|
1266
2067
|
|
|
1267
|
-
function
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
):
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
}
|
|
2068
|
+
function observationsEqual(
|
|
2069
|
+
previous: WorkerRecord,
|
|
2070
|
+
next: WorkerRecord,
|
|
2071
|
+
): boolean {
|
|
2072
|
+
return (
|
|
2073
|
+
previous.activity === next.activity &&
|
|
2074
|
+
previous.messageDirection === next.messageDirection &&
|
|
2075
|
+
usageEquals(previous.usage, next.usage)
|
|
2076
|
+
);
|
|
1277
2077
|
}
|
|
1278
2078
|
|
|
1279
|
-
function
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
waiter.onSettled?.();
|
|
1290
|
-
complete(run);
|
|
1291
|
-
},
|
|
1292
|
-
};
|
|
1293
|
-
return waiter;
|
|
2079
|
+
function usageEquals(left: WorkerUsage, right: WorkerUsage): boolean {
|
|
2080
|
+
return (
|
|
2081
|
+
left.input === right.input &&
|
|
2082
|
+
left.output === right.output &&
|
|
2083
|
+
left.cacheRead === right.cacheRead &&
|
|
2084
|
+
left.cacheWrite === right.cacheWrite &&
|
|
2085
|
+
left.cost === right.cost &&
|
|
2086
|
+
left.contextTokens === right.contextTokens &&
|
|
2087
|
+
left.turns === right.turns
|
|
2088
|
+
);
|
|
1294
2089
|
}
|
|
1295
2090
|
|
|
1296
2091
|
function copyUsage(usage: WorkerUsage): WorkerUsage {
|
|
@@ -1317,7 +2112,9 @@ function copyWorkerRecord(worker: WorkerRecord): WorkerRecord {
|
|
|
1317
2112
|
return Object.freeze({
|
|
1318
2113
|
...worker,
|
|
1319
2114
|
usage: Object.freeze(copyUsage(worker.usage)),
|
|
1320
|
-
...(worker.outcome
|
|
2115
|
+
...(worker.outcome
|
|
2116
|
+
? { outcome: Object.freeze(copyOutcome(worker.outcome)) }
|
|
2117
|
+
: {}),
|
|
1321
2118
|
});
|
|
1322
2119
|
}
|
|
1323
2120
|
|
|
@@ -1327,21 +2124,18 @@ function freezeAcceptedRun(id: RunId, workerId: WorkerId): AcceptedRun {
|
|
|
1327
2124
|
|
|
1328
2125
|
function freezeCompletedRun(
|
|
1329
2126
|
run: RunRecord,
|
|
1330
|
-
record: WorkerRecord
|
|
2127
|
+
record: WorkerRecord & {
|
|
2128
|
+
readonly status: RunResult["status"];
|
|
2129
|
+
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2130
|
+
readonly settledAt: number;
|
|
2131
|
+
},
|
|
1331
2132
|
): CompletedRun {
|
|
1332
|
-
const
|
|
1333
|
-
if (!outcome || outcome.status === "closed") {
|
|
1334
|
-
throw new Error("Completed run requires a worker response outcome");
|
|
1335
|
-
}
|
|
1336
|
-
if (record.settledAt === undefined) {
|
|
1337
|
-
throw new Error("Completed run requires a settlement timestamp");
|
|
1338
|
-
}
|
|
1339
|
-
const result: CompletedResult = Object.freeze({
|
|
2133
|
+
const result: RunResult = Object.freeze({
|
|
1340
2134
|
workerId: record.id,
|
|
1341
2135
|
worker: record.worker,
|
|
1342
2136
|
title: record.title,
|
|
1343
|
-
status: record.status
|
|
1344
|
-
outcome: Object.freeze(copyOutcome(outcome)),
|
|
2137
|
+
status: record.status,
|
|
2138
|
+
outcome: Object.freeze(copyOutcome(record.outcome)),
|
|
1345
2139
|
usage: Object.freeze(copyUsage(record.usage)),
|
|
1346
2140
|
startedAt: record.startedAt,
|
|
1347
2141
|
settledAt: record.settledAt,
|
|
@@ -1354,3 +2148,27 @@ function freezeCompletedRun(
|
|
|
1354
2148
|
result,
|
|
1355
2149
|
});
|
|
1356
2150
|
}
|
|
2151
|
+
|
|
2152
|
+
function describeError(error: unknown, fallback: string): string {
|
|
2153
|
+
if (error instanceof Error && error.message !== "") return error.message;
|
|
2154
|
+
if (typeof error === "string" && error !== "") return error;
|
|
2155
|
+
return fallback;
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
function safelyCall(callback: (() => void) | undefined): void {
|
|
2159
|
+
if (callback) safelyNotify(callback);
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
function safelyNotify(callback: () => void): void {
|
|
2163
|
+
try {
|
|
2164
|
+
callback();
|
|
2165
|
+
} catch {
|
|
2166
|
+
// Observers and best-effort cleanup cannot block committed state transitions.
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
function addAll(target: Set<string>, source: ReadonlySet<string>): void {
|
|
2171
|
+
for (const value of source) target.add(value);
|
|
2172
|
+
}
|
|
2173
|
+
|
|
2174
|
+
function noOp(): void {}
|