@zachwill/pi-orchestrate 0.8.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extension/catalog.ts +70 -75
- package/extension/contract.ts +73 -7
- package/extension/delivery.ts +65 -17
- package/extension/domain.ts +83 -58
- package/extension/host.ts +316 -30
- package/extension/index.ts +28 -31
- package/extension/presentation.ts +48 -98
- package/extension/runtime.ts +1690 -884
- package/extension/tools.ts +236 -270
- package/extension/tui.ts +50 -0
- package/extension/worker-session.ts +514 -188
- package/extension/worker-settlement.ts +105 -44
- package/package.json +1 -1
- package/extension/scheduler.ts +0 -85
package/extension/runtime.ts
CHANGED
|
@@ -1,49 +1,53 @@
|
|
|
1
1
|
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
2
|
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
Cause,
|
|
5
|
+
Clock,
|
|
6
|
+
Context,
|
|
7
|
+
Deferred,
|
|
8
|
+
Effect,
|
|
9
|
+
Exit,
|
|
10
|
+
Fiber,
|
|
11
|
+
FiberMap,
|
|
12
|
+
FiberSet,
|
|
13
|
+
Layer,
|
|
14
|
+
Schema,
|
|
15
|
+
} from "effect";
|
|
4
16
|
import {
|
|
5
17
|
CANCELLATION_GRACE_MS,
|
|
6
18
|
EMPTY_WORKER_USAGE,
|
|
7
19
|
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
8
20
|
MAX_WORKER_TITLE_LENGTH,
|
|
21
|
+
OrchestrateTaskInput,
|
|
22
|
+
RunId,
|
|
23
|
+
WorkerId,
|
|
9
24
|
createRandomIdFactories,
|
|
10
25
|
findWorkerByName,
|
|
11
26
|
isTerminalWorkerStatus,
|
|
12
27
|
transitionWorkerStatus,
|
|
13
28
|
type OrchestrateIdFactories,
|
|
14
|
-
type OrchestrateTaskInput,
|
|
15
|
-
type RunId,
|
|
16
29
|
type RunMode,
|
|
17
30
|
type RunRecord,
|
|
18
31
|
type WorkerCatalog,
|
|
19
32
|
type WorkerDefinition,
|
|
20
|
-
type WorkerId,
|
|
21
33
|
type WorkerOutcome,
|
|
22
34
|
type WorkerRecord,
|
|
23
35
|
type WorkerUsage,
|
|
24
36
|
} from "./domain.js";
|
|
25
37
|
import {
|
|
26
|
-
|
|
27
|
-
type
|
|
28
|
-
|
|
29
|
-
import {
|
|
30
|
-
resolveWorkerModel,
|
|
31
|
-
type WorkerSessionFactory,
|
|
38
|
+
ChildSessions,
|
|
39
|
+
type ChildSessionsService,
|
|
40
|
+
type WorkerSessionAbortError,
|
|
32
41
|
type WorkerSessionHandle,
|
|
42
|
+
type WorkerSessionObservation,
|
|
33
43
|
} from "./worker-session.js";
|
|
34
44
|
import type {
|
|
35
45
|
SettlementFailureStage,
|
|
36
46
|
WorkerSettlement,
|
|
37
47
|
} from "./worker-settlement.js";
|
|
38
48
|
|
|
39
|
-
export type {
|
|
40
|
-
SettlementFailureStage,
|
|
41
|
-
WorkerSettlement,
|
|
42
|
-
} from "./worker-settlement.js";
|
|
43
|
-
|
|
44
49
|
export const MAX_TERMINAL_WORKER_HISTORY = 100;
|
|
45
50
|
export const MAX_COMPLETED_RUN_HISTORY = 100;
|
|
46
|
-
/** Shutdown waits this long for interrupted bootstrap/prompt promises, then returns best-effort. */
|
|
47
51
|
export const SHUTDOWN_CLEANUP_GRACE_MS = CANCELLATION_GRACE_MS;
|
|
48
52
|
|
|
49
53
|
export interface OrchestrationContext {
|
|
@@ -92,433 +96,679 @@ export interface RuntimeSnapshot {
|
|
|
92
96
|
|
|
93
97
|
export type SettlementListener = (settlement: WorkerSettlement) => void;
|
|
94
98
|
export type UnsubscribeSettlement = () => void;
|
|
95
|
-
export type StateListener = (
|
|
96
|
-
|
|
97
|
-
export type AbortTarget =
|
|
98
|
-
| {
|
|
99
|
-
readonly workerIds: readonly WorkerId[];
|
|
100
|
-
readonly all?: never;
|
|
101
|
-
}
|
|
102
|
-
| {
|
|
103
|
-
readonly all: true;
|
|
104
|
-
readonly workerIds?: never;
|
|
105
|
-
};
|
|
99
|
+
export type StateListener = (snapshot: RuntimeSnapshot) => void;
|
|
106
100
|
|
|
107
|
-
export
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
wait(promise: Promise<unknown>, timeoutMs: number): Promise<DeadlineResult>;
|
|
101
|
+
export interface AbortTarget {
|
|
102
|
+
readonly workerIds?: readonly string[];
|
|
103
|
+
readonly all?: boolean;
|
|
111
104
|
}
|
|
112
105
|
|
|
113
|
-
export interface
|
|
114
|
-
readonly workerSessionFactory: WorkerSessionFactory;
|
|
106
|
+
export interface OrchestrationLayerOptions {
|
|
115
107
|
readonly idFactories?: OrchestrateIdFactories;
|
|
116
|
-
readonly clock?: () => number;
|
|
117
|
-
readonly scheduler?: WorkflowScheduler<WorkerId>;
|
|
118
|
-
readonly bestEffortDeadline?: BestEffortDeadline;
|
|
119
108
|
}
|
|
120
109
|
|
|
121
|
-
|
|
110
|
+
const OrchestrationOperation = Schema.Literals([
|
|
111
|
+
"orchestrate",
|
|
112
|
+
"sendInteractive",
|
|
113
|
+
"abort",
|
|
114
|
+
"closeInteractive",
|
|
115
|
+
"snapshot",
|
|
116
|
+
]);
|
|
117
|
+
type OrchestrationOperation = typeof OrchestrationOperation.Type;
|
|
118
|
+
|
|
119
|
+
const OrchestrationRejectionReason = Schema.Literals([
|
|
120
|
+
"shutdown",
|
|
121
|
+
"validation",
|
|
122
|
+
"ownership",
|
|
123
|
+
"target",
|
|
124
|
+
"worker-state",
|
|
125
|
+
"unknown-worker",
|
|
126
|
+
"model-unavailable",
|
|
127
|
+
]);
|
|
128
|
+
type OrchestrationRejectionReason = typeof OrchestrationRejectionReason.Type;
|
|
129
|
+
|
|
130
|
+
export class OrchestrationActionRejected extends Schema.TaggedError<OrchestrationActionRejected>()(
|
|
131
|
+
"Orchestration.ActionRejected",
|
|
132
|
+
{
|
|
133
|
+
operation: OrchestrationOperation,
|
|
134
|
+
reason: OrchestrationRejectionReason,
|
|
135
|
+
message: Schema.String,
|
|
136
|
+
},
|
|
137
|
+
) {}
|
|
138
|
+
|
|
139
|
+
export interface OrchestrationService {
|
|
122
140
|
orchestrate(
|
|
123
141
|
context: OrchestrationContext,
|
|
124
142
|
task: OrchestrateTaskInput,
|
|
125
143
|
mode: "async",
|
|
126
|
-
signal?: AbortSignal,
|
|
127
144
|
onSettlement?: SettlementListener,
|
|
128
|
-
):
|
|
145
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
129
146
|
orchestrate(
|
|
130
147
|
context: OrchestrationContext,
|
|
131
148
|
task: OrchestrateTaskInput,
|
|
132
149
|
mode: "inline",
|
|
133
|
-
signal?: AbortSignal,
|
|
134
150
|
onSettlement?: SettlementListener,
|
|
135
|
-
):
|
|
151
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
136
152
|
orchestrate(
|
|
137
153
|
context: OrchestrationContext,
|
|
138
154
|
task: OrchestrateTaskInput,
|
|
139
155
|
mode: RunMode,
|
|
140
|
-
signal?: AbortSignal,
|
|
141
156
|
onSettlement?: SettlementListener,
|
|
142
|
-
):
|
|
157
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
143
158
|
sendInteractive(
|
|
144
159
|
context: OrchestrationContext,
|
|
145
|
-
workerId:
|
|
160
|
+
workerId: string,
|
|
146
161
|
instructions: string,
|
|
147
162
|
mode: "async",
|
|
148
|
-
signal?: AbortSignal,
|
|
149
163
|
onSettlement?: SettlementListener,
|
|
150
|
-
):
|
|
164
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
151
165
|
sendInteractive(
|
|
152
166
|
context: OrchestrationContext,
|
|
153
|
-
workerId:
|
|
167
|
+
workerId: string,
|
|
154
168
|
instructions: string,
|
|
155
169
|
mode: "inline",
|
|
156
|
-
signal?: AbortSignal,
|
|
157
170
|
onSettlement?: SettlementListener,
|
|
158
|
-
):
|
|
171
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
159
172
|
sendInteractive(
|
|
160
173
|
context: OrchestrationContext,
|
|
161
|
-
workerId:
|
|
174
|
+
workerId: string,
|
|
162
175
|
instructions: string,
|
|
163
176
|
mode: RunMode,
|
|
164
|
-
signal?: AbortSignal,
|
|
165
177
|
onSettlement?: SettlementListener,
|
|
166
|
-
):
|
|
167
|
-
abort
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
178
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
179
|
+
readonly abort: (
|
|
180
|
+
ownerSessionId: string,
|
|
181
|
+
target: AbortTarget,
|
|
182
|
+
) => Effect.Effect<void, OrchestrationActionRejected>;
|
|
183
|
+
readonly closeInteractive: (
|
|
184
|
+
ownerSessionId: string,
|
|
185
|
+
workerId: string,
|
|
186
|
+
) => Effect.Effect<void, OrchestrationActionRejected>;
|
|
187
|
+
readonly snapshot: (
|
|
188
|
+
ownerSessionId: string,
|
|
189
|
+
) => Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected>;
|
|
190
|
+
readonly subscribeSettlement: (
|
|
191
|
+
listener: SettlementListener,
|
|
192
|
+
) => UnsubscribeSettlement;
|
|
193
|
+
readonly subscribeState: (
|
|
194
|
+
ownerSessionId: string,
|
|
195
|
+
listener: StateListener,
|
|
196
|
+
) => () => void;
|
|
197
|
+
readonly shutdown: () => Effect.Effect<void>;
|
|
173
198
|
}
|
|
174
199
|
|
|
175
|
-
|
|
200
|
+
export class Orchestration extends Context.Service<Orchestration, OrchestrationService>()(
|
|
201
|
+
"@zachwill/pi-orchestrate/Orchestration",
|
|
202
|
+
) {}
|
|
203
|
+
|
|
204
|
+
interface RuntimeWorker {
|
|
205
|
+
readonly record: WorkerRecord;
|
|
176
206
|
readonly context: OrchestrationContext;
|
|
177
207
|
readonly definition: WorkerDefinition;
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
208
|
+
// Worker-local authority fence; stale asynchronous callbacks must revalidate it.
|
|
209
|
+
readonly generation: number;
|
|
210
|
+
readonly session?: WorkerSessionHandle;
|
|
211
|
+
readonly observationRelease?: () => void;
|
|
212
|
+
readonly cancellation?: Deferred.Deferred<void>;
|
|
183
213
|
}
|
|
184
214
|
|
|
185
|
-
interface
|
|
186
|
-
readonly
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
215
|
+
interface RunningRuntimeRun {
|
|
216
|
+
readonly _tag: "running";
|
|
217
|
+
readonly record: RunRecord;
|
|
218
|
+
readonly completion: Deferred.Deferred<CompletedRun>;
|
|
219
|
+
readonly settlementListener?: SettlementListener;
|
|
190
220
|
}
|
|
191
221
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
+
type RuntimeRun = RunningRuntimeRun | RunRecord;
|
|
223
|
+
type RuntimeLifecycle = "open" | "shutting-down" | "shutdown";
|
|
224
|
+
|
|
225
|
+
interface RuntimeState {
|
|
226
|
+
workers: Map<WorkerId, RuntimeWorker>;
|
|
227
|
+
runs: Map<RunId, RuntimeRun>;
|
|
228
|
+
terminalWorkerOrder: WorkerId[];
|
|
229
|
+
completedRunOrder: RunId[];
|
|
230
|
+
settlementSequence: number;
|
|
231
|
+
lifecycle: RuntimeLifecycle;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
type CommittedAction = () => void;
|
|
235
|
+
type PostCommitAction = (
|
|
236
|
+
state: RuntimeState,
|
|
237
|
+
settlementListeners: ReadonlySet<SettlementListener>,
|
|
238
|
+
stateListeners: ReadonlyMap<string, ReadonlySet<StateListener>>,
|
|
239
|
+
) => CommittedAction;
|
|
240
|
+
|
|
241
|
+
interface TransactionMutation<A> {
|
|
242
|
+
readonly value: A;
|
|
243
|
+
readonly actions?: readonly PostCommitAction[];
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
type Decision<A> =
|
|
247
|
+
| {
|
|
248
|
+
readonly _tag: "accepted";
|
|
249
|
+
readonly value: A;
|
|
250
|
+
}
|
|
251
|
+
| {
|
|
252
|
+
readonly _tag: "rejected";
|
|
253
|
+
readonly error: OrchestrationActionRejected;
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
class StatefulOrchestration implements OrchestrationService {
|
|
257
|
+
private readonly actionQueue: CommittedAction[] = [];
|
|
222
258
|
private readonly settlementListeners = new Set<SettlementListener>();
|
|
223
|
-
private readonly
|
|
224
|
-
private
|
|
225
|
-
private
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
259
|
+
private readonly stateListeners = new Map<string, Set<StateListener>>();
|
|
260
|
+
private drainingActions = false;
|
|
261
|
+
private state: RuntimeState;
|
|
262
|
+
|
|
263
|
+
constructor(
|
|
264
|
+
private readonly childSessions: ChildSessionsService,
|
|
265
|
+
private readonly generations: FiberMap.FiberMap<WorkerId, void, never>,
|
|
266
|
+
private readonly runGeneration: (
|
|
267
|
+
key: WorkerId,
|
|
268
|
+
effect: Effect.Effect<void, never>,
|
|
269
|
+
) => Fiber.Fiber<void, never>,
|
|
270
|
+
private readonly cancellations: FiberSet.FiberSet<void, never>,
|
|
271
|
+
private readonly runCancellation: (
|
|
272
|
+
effect: Effect.Effect<void, never>,
|
|
273
|
+
) => Fiber.Fiber<void, never>,
|
|
274
|
+
private readonly cleanups: FiberSet.FiberSet<void, never>,
|
|
275
|
+
private readonly clock: Clock.Clock,
|
|
276
|
+
private readonly idFactories: OrchestrateIdFactories,
|
|
277
|
+
private readonly shutdownCompletion: Deferred.Deferred<void>,
|
|
278
|
+
) {
|
|
279
|
+
this.state = initialState();
|
|
236
280
|
}
|
|
237
281
|
|
|
238
282
|
orchestrate(
|
|
239
283
|
context: OrchestrationContext,
|
|
240
284
|
task: OrchestrateTaskInput,
|
|
241
285
|
mode: "async",
|
|
242
|
-
signal?: AbortSignal,
|
|
243
286
|
onSettlement?: SettlementListener,
|
|
244
|
-
):
|
|
287
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
245
288
|
orchestrate(
|
|
246
289
|
context: OrchestrationContext,
|
|
247
290
|
task: OrchestrateTaskInput,
|
|
248
291
|
mode: "inline",
|
|
249
|
-
signal?: AbortSignal,
|
|
250
292
|
onSettlement?: SettlementListener,
|
|
251
|
-
):
|
|
293
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
252
294
|
orchestrate(
|
|
253
295
|
context: OrchestrationContext,
|
|
254
296
|
task: OrchestrateTaskInput,
|
|
255
297
|
mode: RunMode,
|
|
256
|
-
signal?: AbortSignal,
|
|
257
298
|
onSettlement?: SettlementListener,
|
|
258
|
-
):
|
|
259
|
-
|
|
299
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
300
|
+
orchestrate(
|
|
260
301
|
context: OrchestrationContext,
|
|
261
302
|
task: OrchestrateTaskInput,
|
|
262
303
|
mode: RunMode,
|
|
263
|
-
signal?: AbortSignal,
|
|
264
304
|
onSettlement?: SettlementListener,
|
|
265
|
-
):
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const record: WorkerRecord = {
|
|
289
|
-
id: workerId,
|
|
290
|
-
worker: definition.name,
|
|
291
|
-
ownerSessionId: context.ownerSessionId,
|
|
292
|
-
runId,
|
|
293
|
-
title: task.title,
|
|
294
|
-
instructions: task.instructions,
|
|
295
|
-
lifecycle: definition.lifecycle,
|
|
296
|
-
status: "starting",
|
|
297
|
-
usage: copyUsage(EMPTY_WORKER_USAGE),
|
|
298
|
-
messageDirection: "to-model",
|
|
299
|
-
startedAt: this.clock(),
|
|
300
|
-
};
|
|
305
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
306
|
+
return Effect.fn("Orchestration.orchestrate")(function* (
|
|
307
|
+
this: StatefulOrchestration,
|
|
308
|
+
) {
|
|
309
|
+
const validated = yield* this.validateTask(context, task, mode);
|
|
310
|
+
const preflight = this.preflightOpen("orchestrate");
|
|
311
|
+
if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
|
|
312
|
+
// Keep user-supplied ID factories outside the transaction: a reentrant
|
|
313
|
+
// factory cannot commit an outer stale draft. Admission rechecks all
|
|
314
|
+
// authority below, so a race may burn an ID but cannot create a run.
|
|
315
|
+
const runId = this.idFactories.runId();
|
|
316
|
+
const workerId = this.idFactories.workerId();
|
|
317
|
+
const completion = yield* Deferred.make<CompletedRun>();
|
|
318
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
319
|
+
const runRecord = makeRunRecord(runId, workerId, context, mode, now);
|
|
320
|
+
const workerRecord = makeWorkerRecord(
|
|
321
|
+
workerId,
|
|
322
|
+
runId,
|
|
323
|
+
context.ownerSessionId,
|
|
324
|
+
validated.definition,
|
|
325
|
+
validated.task,
|
|
326
|
+
now,
|
|
327
|
+
);
|
|
301
328
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
329
|
+
const admission = this.transact((draft) => {
|
|
330
|
+
const open = openDecision(draft, "orchestrate");
|
|
331
|
+
if (open._tag === "rejected") return { value: open };
|
|
332
|
+
if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
|
|
333
|
+
if (draft.workers.has(workerId)) {
|
|
334
|
+
throw new Error(`Duplicate worker ID: ${workerId}`);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
draft.runs.set(runId, {
|
|
338
|
+
_tag: "running",
|
|
339
|
+
record: runRecord,
|
|
340
|
+
completion,
|
|
341
|
+
...(onSettlement ? { settlementListener: onSettlement } : {}),
|
|
342
|
+
});
|
|
343
|
+
draft.workers.set(workerId, {
|
|
344
|
+
record: workerRecord,
|
|
345
|
+
context,
|
|
346
|
+
definition: validated.definition,
|
|
347
|
+
generation: 1,
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
return {
|
|
351
|
+
value: accepted(undefined),
|
|
352
|
+
actions: [
|
|
353
|
+
publishState(context.ownerSessionId),
|
|
354
|
+
runAction(() => this.launchBootstrap(workerId, 1)),
|
|
355
|
+
],
|
|
356
|
+
};
|
|
357
|
+
});
|
|
358
|
+
if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
|
|
359
|
+
|
|
360
|
+
if (mode === "inline") {
|
|
361
|
+
return yield* this.awaitInlineRun(runRecord, completion);
|
|
362
|
+
}
|
|
363
|
+
return freezeAcceptedRun(runId, workerId);
|
|
364
|
+
}).call(this);
|
|
315
365
|
}
|
|
316
366
|
|
|
317
367
|
sendInteractive(
|
|
318
368
|
context: OrchestrationContext,
|
|
319
|
-
workerId:
|
|
369
|
+
workerId: string,
|
|
320
370
|
instructions: string,
|
|
321
371
|
mode: "async",
|
|
322
|
-
signal?: AbortSignal,
|
|
323
372
|
onSettlement?: SettlementListener,
|
|
324
|
-
):
|
|
373
|
+
): Effect.Effect<AcceptedRun, OrchestrationActionRejected>;
|
|
325
374
|
sendInteractive(
|
|
326
375
|
context: OrchestrationContext,
|
|
327
|
-
workerId:
|
|
376
|
+
workerId: string,
|
|
328
377
|
instructions: string,
|
|
329
378
|
mode: "inline",
|
|
330
|
-
signal?: AbortSignal,
|
|
331
379
|
onSettlement?: SettlementListener,
|
|
332
|
-
):
|
|
380
|
+
): Effect.Effect<CompletedRun, OrchestrationActionRejected>;
|
|
333
381
|
sendInteractive(
|
|
334
382
|
context: OrchestrationContext,
|
|
335
|
-
workerId:
|
|
383
|
+
workerId: string,
|
|
336
384
|
instructions: string,
|
|
337
385
|
mode: RunMode,
|
|
338
|
-
signal?: AbortSignal,
|
|
339
386
|
onSettlement?: SettlementListener,
|
|
340
|
-
):
|
|
341
|
-
|
|
387
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected>;
|
|
388
|
+
sendInteractive(
|
|
342
389
|
context: OrchestrationContext,
|
|
343
|
-
workerId:
|
|
390
|
+
workerId: string,
|
|
344
391
|
instructions: string,
|
|
345
392
|
mode: RunMode,
|
|
346
|
-
signal?: AbortSignal,
|
|
347
393
|
onSettlement?: SettlementListener,
|
|
348
|
-
):
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
394
|
+
): Effect.Effect<AcceptedRun | CompletedRun, OrchestrationActionRejected> {
|
|
395
|
+
return Effect.fn("Orchestration.sendInteractive")(function* (
|
|
396
|
+
this: StatefulOrchestration,
|
|
397
|
+
) {
|
|
398
|
+
yield* validateContextOwner("sendInteractive", context.ownerSessionId);
|
|
399
|
+
yield* validateMode("sendInteractive", mode);
|
|
400
|
+
yield* validateText(
|
|
401
|
+
"sendInteractive",
|
|
402
|
+
"instructions",
|
|
403
|
+
instructions,
|
|
404
|
+
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
405
|
+
);
|
|
406
|
+
const validatedWorkerId = yield* validateWorkerId(
|
|
407
|
+
"sendInteractive",
|
|
408
|
+
workerId,
|
|
409
|
+
);
|
|
410
|
+
const preflight = this.preflightReadyInteractive(
|
|
411
|
+
context.ownerSessionId,
|
|
412
|
+
validatedWorkerId,
|
|
413
|
+
);
|
|
414
|
+
if (preflight._tag === "rejected") yield* Effect.fail(preflight.error);
|
|
415
|
+
// See orchestrate: allocation remains outside copy-on-write reducers so
|
|
416
|
+
// reentrant factories cannot invalidate transaction atomicity.
|
|
417
|
+
const runId = this.idFactories.runId();
|
|
418
|
+
const completion = yield* Deferred.make<CompletedRun>();
|
|
419
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
420
|
+
const runRecord: RunRecord = {
|
|
421
|
+
id: runId,
|
|
422
|
+
ownerSessionId: context.ownerSessionId,
|
|
423
|
+
workerId: validatedWorkerId,
|
|
424
|
+
mode,
|
|
425
|
+
state: "running",
|
|
426
|
+
createdAt: now,
|
|
427
|
+
};
|
|
428
|
+
|
|
429
|
+
const admission = this.transact((draft) => {
|
|
430
|
+
const ready = readyInteractiveDecision(
|
|
431
|
+
draft,
|
|
432
|
+
context.ownerSessionId,
|
|
433
|
+
validatedWorkerId,
|
|
434
|
+
);
|
|
435
|
+
if (ready._tag === "rejected") return { value: ready };
|
|
436
|
+
if (draft.runs.has(runId)) throw new Error(`Duplicate run ID: ${runId}`);
|
|
437
|
+
|
|
438
|
+
const { worker, session } = ready.value;
|
|
439
|
+
const generation = worker.generation + 1;
|
|
440
|
+
const runningRecord: WorkerRecord = {
|
|
441
|
+
...transitionWorkerStatus(worker.record, "running"),
|
|
442
|
+
runId,
|
|
443
|
+
instructions,
|
|
444
|
+
activity: undefined,
|
|
445
|
+
messageDirection: "to-model",
|
|
446
|
+
startedAt: now,
|
|
447
|
+
settledAt: undefined,
|
|
448
|
+
};
|
|
449
|
+
draft.runs.set(runId, {
|
|
450
|
+
_tag: "running",
|
|
451
|
+
record: runRecord,
|
|
452
|
+
completion,
|
|
453
|
+
...(onSettlement ? { settlementListener: onSettlement } : {}),
|
|
454
|
+
});
|
|
455
|
+
draft.workers.set(validatedWorkerId, {
|
|
456
|
+
...worker,
|
|
457
|
+
record: runningRecord,
|
|
458
|
+
generation,
|
|
459
|
+
observationRelease: undefined,
|
|
460
|
+
cancellation: undefined,
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
return {
|
|
464
|
+
value: accepted(undefined),
|
|
465
|
+
actions: [
|
|
466
|
+
runAction(() => {
|
|
467
|
+
safelyCall(worker.observationRelease);
|
|
468
|
+
this.subscribeObservation(validatedWorkerId, generation, session);
|
|
469
|
+
}),
|
|
470
|
+
publishState(context.ownerSessionId),
|
|
471
|
+
runAction(() => {
|
|
472
|
+
this.launchPrompt(
|
|
473
|
+
validatedWorkerId,
|
|
474
|
+
generation,
|
|
475
|
+
session,
|
|
476
|
+
instructions,
|
|
477
|
+
);
|
|
478
|
+
}),
|
|
479
|
+
],
|
|
480
|
+
};
|
|
481
|
+
});
|
|
482
|
+
if (admission._tag === "rejected") return yield* Effect.fail(admission.error);
|
|
483
|
+
|
|
484
|
+
if (mode === "inline") {
|
|
485
|
+
return yield* this.awaitInlineRun(runRecord, completion);
|
|
486
|
+
}
|
|
487
|
+
return freezeAcceptedRun(runId, validatedWorkerId);
|
|
488
|
+
}).call(this);
|
|
397
489
|
}
|
|
398
490
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
491
|
+
abort(
|
|
492
|
+
ownerSessionId: string,
|
|
493
|
+
target: AbortTarget,
|
|
494
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
495
|
+
return Effect.fn("Orchestration.abort")(function* (
|
|
496
|
+
this: StatefulOrchestration,
|
|
497
|
+
) {
|
|
498
|
+
yield* validateContextOwner("abort", ownerSessionId);
|
|
499
|
+
const validatedTarget = yield* validateAbortTarget(target);
|
|
500
|
+
const candidateIds = validatedTarget._tag === "ids"
|
|
501
|
+
? validatedTarget.workerIds
|
|
502
|
+
: activeWorkerIds(this.current(), ownerSessionId);
|
|
503
|
+
const candidates = yield* makeCancellationCandidates(candidateIds);
|
|
504
|
+
const cancellation = this.beginCancellation(
|
|
505
|
+
"abort",
|
|
506
|
+
ownerSessionId,
|
|
507
|
+
validatedTarget,
|
|
508
|
+
candidates,
|
|
509
|
+
);
|
|
510
|
+
if (cancellation._tag === "rejected") {
|
|
511
|
+
return yield* Effect.fail(cancellation.error);
|
|
512
|
+
}
|
|
513
|
+
yield* awaitAll(cancellation.value);
|
|
514
|
+
}).call(this);
|
|
404
515
|
}
|
|
405
516
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
517
|
+
closeInteractive(
|
|
518
|
+
ownerSessionId: string,
|
|
519
|
+
workerId: string,
|
|
520
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
521
|
+
return Effect.fn("Orchestration.closeInteractive")(function* (
|
|
522
|
+
this: StatefulOrchestration,
|
|
523
|
+
) {
|
|
524
|
+
yield* validateContextOwner("closeInteractive", ownerSessionId);
|
|
525
|
+
const id = yield* validateWorkerId("closeInteractive", workerId);
|
|
526
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
527
|
+
const decision = this.transact((draft) => {
|
|
528
|
+
const open = openDecision(draft, "closeInteractive");
|
|
529
|
+
if (open._tag === "rejected") return { value: open };
|
|
530
|
+
const ownership = ownedWorkerDecision(
|
|
531
|
+
draft,
|
|
532
|
+
"closeInteractive",
|
|
533
|
+
ownerSessionId,
|
|
534
|
+
id,
|
|
535
|
+
);
|
|
536
|
+
if (ownership._tag === "rejected") return { value: ownership };
|
|
537
|
+
if (
|
|
538
|
+
ownership.value.record.lifecycle !== "interactive" ||
|
|
539
|
+
ownership.value.record.status !== "ready"
|
|
540
|
+
) {
|
|
541
|
+
return {
|
|
542
|
+
value: rejected(
|
|
543
|
+
"closeInteractive",
|
|
544
|
+
"worker-state",
|
|
545
|
+
"interactive_close requires an owned ready interactive worker",
|
|
546
|
+
),
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
const actions = this.closeReadyWorker(draft, ownership.value, now);
|
|
550
|
+
actions.push(...stateActionsAfterPrune(draft, ownerSessionId));
|
|
551
|
+
return {
|
|
552
|
+
value: accepted(undefined),
|
|
553
|
+
actions,
|
|
554
|
+
};
|
|
555
|
+
});
|
|
556
|
+
if (decision._tag === "rejected") yield* Effect.fail(decision.error);
|
|
557
|
+
}).call(this);
|
|
414
558
|
}
|
|
415
559
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
.
|
|
421
|
-
|
|
422
|
-
.filter((worker) => worker.ownerSessionId === ownerSessionId)
|
|
423
|
-
.map(copyWorkerRecord);
|
|
424
|
-
return Object.freeze({
|
|
425
|
-
runs: Object.freeze(runs),
|
|
426
|
-
workers: Object.freeze(workers),
|
|
427
|
-
});
|
|
560
|
+
snapshot(
|
|
561
|
+
ownerSessionId: string,
|
|
562
|
+
): Effect.Effect<RuntimeSnapshot, OrchestrationActionRejected> {
|
|
563
|
+
return validateContextOwner("snapshot", ownerSessionId).pipe(
|
|
564
|
+
Effect.andThen(Effect.sync(() => snapshotFor(this.current(), ownerSessionId))),
|
|
565
|
+
);
|
|
428
566
|
}
|
|
429
567
|
|
|
430
568
|
subscribeSettlement(listener: SettlementListener): UnsubscribeSettlement {
|
|
431
|
-
if (typeof listener !== "function")
|
|
569
|
+
if (typeof listener !== "function") {
|
|
570
|
+
throw new Error("Settlement listener must be a function");
|
|
571
|
+
}
|
|
572
|
+
if (this.state.lifecycle !== "open") return noOp;
|
|
432
573
|
this.settlementListeners.add(listener);
|
|
433
|
-
|
|
574
|
+
|
|
575
|
+
let active = true;
|
|
434
576
|
return () => {
|
|
435
|
-
if (!
|
|
436
|
-
|
|
577
|
+
if (!active) return;
|
|
578
|
+
active = false;
|
|
437
579
|
this.settlementListeners.delete(listener);
|
|
438
580
|
};
|
|
439
581
|
}
|
|
440
582
|
|
|
441
|
-
subscribeState(listener: StateListener): () => void {
|
|
442
|
-
if (typeof
|
|
443
|
-
|
|
444
|
-
|
|
583
|
+
subscribeState(ownerSessionId: string, listener: StateListener): () => void {
|
|
584
|
+
if (typeof ownerSessionId !== "string" || ownerSessionId.trim() === "") {
|
|
585
|
+
throw new Error("ownerSessionId must not be blank");
|
|
586
|
+
}
|
|
587
|
+
if (typeof listener !== "function") {
|
|
588
|
+
throw new Error("State listener must be a function");
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
if (this.state.lifecycle !== "open") return noOp;
|
|
592
|
+
const ownerListeners = this.stateListeners.get(ownerSessionId) ?? new Set();
|
|
593
|
+
ownerListeners.add(listener);
|
|
594
|
+
this.stateListeners.set(ownerSessionId, ownerListeners);
|
|
595
|
+
this.enqueueActions([
|
|
596
|
+
publishStateTo(ownerSessionId, listener)(
|
|
597
|
+
this.state,
|
|
598
|
+
this.settlementListeners,
|
|
599
|
+
this.stateListeners,
|
|
600
|
+
),
|
|
601
|
+
]);
|
|
602
|
+
|
|
603
|
+
let active = true;
|
|
445
604
|
return () => {
|
|
446
|
-
if (!
|
|
447
|
-
|
|
448
|
-
|
|
605
|
+
if (!active) return;
|
|
606
|
+
active = false;
|
|
607
|
+
ownerListeners.delete(listener);
|
|
608
|
+
if (ownerListeners.size === 0) this.stateListeners.delete(ownerSessionId);
|
|
449
609
|
};
|
|
450
610
|
}
|
|
451
611
|
|
|
452
|
-
shutdown
|
|
453
|
-
|
|
454
|
-
this.
|
|
455
|
-
|
|
456
|
-
|
|
612
|
+
/** Calling shutdown closes admission synchronously, before the returned Effect runs. */
|
|
613
|
+
shutdown(): Effect.Effect<void> {
|
|
614
|
+
const first = this.transact((draft) => {
|
|
615
|
+
if (draft.lifecycle !== "open") return { value: false };
|
|
616
|
+
draft.lifecycle = "shutting-down";
|
|
617
|
+
return { value: true };
|
|
618
|
+
});
|
|
619
|
+
if (!first) return Deferred.await(this.shutdownCompletion);
|
|
620
|
+
|
|
621
|
+
return this.performShutdown().pipe(
|
|
622
|
+
Effect.ensuring(
|
|
623
|
+
Effect.sync(() => {
|
|
624
|
+
this.transact((draft) => {
|
|
625
|
+
draft.lifecycle = "shutdown";
|
|
626
|
+
for (const [runId, run] of draft.runs) {
|
|
627
|
+
if (isRunningRuntimeRun(run) && run.settlementListener) {
|
|
628
|
+
draft.runs.set(runId, { ...run, settlementListener: undefined });
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
return { value: undefined };
|
|
632
|
+
});
|
|
633
|
+
this.settlementListeners.clear();
|
|
634
|
+
this.stateListeners.clear();
|
|
635
|
+
Deferred.doneUnsafe(this.shutdownCompletion, Effect.void);
|
|
636
|
+
}),
|
|
637
|
+
),
|
|
638
|
+
);
|
|
457
639
|
}
|
|
458
640
|
|
|
459
|
-
private
|
|
460
|
-
|
|
461
|
-
this
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
641
|
+
private performShutdown(): Effect.Effect<void> {
|
|
642
|
+
return Effect.fn("Orchestration.shutdown")(function* (
|
|
643
|
+
this: StatefulOrchestration,
|
|
644
|
+
) {
|
|
645
|
+
const now = this.clock.currentTimeMillisUnsafe();
|
|
646
|
+
this.transact((draft) => {
|
|
647
|
+
const owners = new Set<string>();
|
|
648
|
+
const actions: PostCommitAction[] = [];
|
|
649
|
+
for (const worker of draft.workers.values()) {
|
|
650
|
+
if (
|
|
651
|
+
worker.record.lifecycle === "interactive" &&
|
|
652
|
+
worker.record.status === "ready"
|
|
653
|
+
) {
|
|
654
|
+
this.closeReadyWorker(draft, worker, now, actions);
|
|
655
|
+
owners.add(worker.record.ownerSessionId);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
addAll(owners, pruneHistory(draft));
|
|
659
|
+
actions.push(...publishOwners(owners));
|
|
660
|
+
return { value: undefined, actions };
|
|
661
|
+
});
|
|
662
|
+
|
|
663
|
+
const activeIds = activeWorkerIds(this.current());
|
|
664
|
+
const candidates = yield* makeCancellationCandidates(activeIds);
|
|
665
|
+
const cancellations = this.beginShutdownCancellation(candidates);
|
|
666
|
+
yield* awaitAll(cancellations);
|
|
667
|
+
yield* this.childSessions.shutdown().pipe(Effect.catchCause(() => Effect.void));
|
|
668
|
+
yield* FiberSet.awaitEmpty(this.cleanups).pipe(
|
|
669
|
+
Effect.timeoutOption(SHUTDOWN_CLEANUP_GRACE_MS),
|
|
670
|
+
Effect.ignore,
|
|
671
|
+
);
|
|
672
|
+
}).call(this);
|
|
478
673
|
}
|
|
479
674
|
|
|
480
675
|
private validateTask(
|
|
481
676
|
context: OrchestrationContext,
|
|
482
677
|
task: OrchestrateTaskInput,
|
|
483
678
|
mode: RunMode,
|
|
484
|
-
):
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
679
|
+
): Effect.Effect<
|
|
680
|
+
{ definition: WorkerDefinition; task: OrchestrateTaskInput },
|
|
681
|
+
OrchestrationActionRejected
|
|
682
|
+
> {
|
|
683
|
+
return Effect.gen(function* () {
|
|
684
|
+
yield* validateContextOwner("orchestrate", context.ownerSessionId);
|
|
685
|
+
yield* validateMode("orchestrate", mode);
|
|
686
|
+
if (!task || typeof task !== "object" || Array.isArray(task)) {
|
|
687
|
+
return yield* rejectAction(
|
|
688
|
+
"orchestrate",
|
|
689
|
+
"validation",
|
|
690
|
+
"orchestrate requires one task object",
|
|
691
|
+
);
|
|
495
692
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
693
|
+
if (context.synthesisGroup) {
|
|
694
|
+
yield* validateText(
|
|
695
|
+
"orchestrate",
|
|
696
|
+
"synthesis group ID",
|
|
697
|
+
context.synthesisGroup.id,
|
|
698
|
+
MAX_WORKER_TITLE_LENGTH,
|
|
699
|
+
);
|
|
700
|
+
if (mode !== "async") {
|
|
701
|
+
return yield* rejectAction(
|
|
702
|
+
"orchestrate",
|
|
703
|
+
"validation",
|
|
704
|
+
"Sibling synthesis requires an async task",
|
|
705
|
+
);
|
|
706
|
+
}
|
|
707
|
+
if (
|
|
708
|
+
!Number.isSafeInteger(context.synthesisGroup.size) ||
|
|
709
|
+
context.synthesisGroup.size < 2
|
|
710
|
+
) {
|
|
711
|
+
return yield* rejectAction(
|
|
712
|
+
"orchestrate",
|
|
713
|
+
"validation",
|
|
714
|
+
"Synthesis group size must be an integer of at least 2",
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
yield* validateText(
|
|
719
|
+
"orchestrate",
|
|
720
|
+
"worker",
|
|
721
|
+
task.worker,
|
|
722
|
+
MAX_WORKER_TITLE_LENGTH,
|
|
723
|
+
);
|
|
724
|
+
yield* validateText(
|
|
725
|
+
"orchestrate",
|
|
726
|
+
"title",
|
|
727
|
+
task.title,
|
|
728
|
+
MAX_WORKER_TITLE_LENGTH,
|
|
729
|
+
);
|
|
730
|
+
yield* validateText(
|
|
731
|
+
"orchestrate",
|
|
732
|
+
"instructions",
|
|
733
|
+
task.instructions,
|
|
734
|
+
MAX_WORKER_INSTRUCTIONS_LENGTH,
|
|
735
|
+
);
|
|
736
|
+
const definition = findWorkerByName(context.catalog, task.worker);
|
|
737
|
+
if (!definition) {
|
|
738
|
+
return yield* rejectAction(
|
|
739
|
+
"orchestrate",
|
|
740
|
+
"unknown-worker",
|
|
741
|
+
`Unknown worker: ${task.worker}`,
|
|
742
|
+
);
|
|
743
|
+
}
|
|
744
|
+
const configured = definition.model;
|
|
745
|
+
if (!configured && !context.parentModel) {
|
|
746
|
+
return yield* rejectAction(
|
|
747
|
+
"orchestrate",
|
|
748
|
+
"model-unavailable",
|
|
749
|
+
`Worker "${definition.name}" has no configured model and no parent model is available`,
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
if (
|
|
753
|
+
configured &&
|
|
754
|
+
!context.modelRegistry.find(configured.provider, configured.modelId)
|
|
755
|
+
) {
|
|
756
|
+
return yield* rejectAction(
|
|
757
|
+
"orchestrate",
|
|
758
|
+
"model-unavailable",
|
|
759
|
+
`Worker "${definition.name}" configured model "${configured.provider}/${configured.modelId}" was not found`,
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
return { definition, task };
|
|
763
|
+
});
|
|
510
764
|
}
|
|
511
765
|
|
|
512
766
|
private launchBootstrap(workerId: WorkerId, generation: number): void {
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
);
|
|
519
|
-
} catch (error) {
|
|
520
|
-
this.settleWorkflowDefect(workerId, generation, error);
|
|
521
|
-
}
|
|
767
|
+
this.launchGeneration(
|
|
768
|
+
workerId,
|
|
769
|
+
generation,
|
|
770
|
+
this.bootstrapAndPrompt(workerId, generation),
|
|
771
|
+
);
|
|
522
772
|
}
|
|
523
773
|
|
|
524
774
|
private launchPrompt(
|
|
@@ -526,29 +776,74 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
526
776
|
generation: number,
|
|
527
777
|
session: WorkerSessionHandle,
|
|
528
778
|
instructions: string,
|
|
779
|
+
): void {
|
|
780
|
+
this.launchGeneration(
|
|
781
|
+
workerId,
|
|
782
|
+
generation,
|
|
783
|
+
this.executePrompt(workerId, generation, session, instructions),
|
|
784
|
+
);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
private launchGeneration(
|
|
788
|
+
workerId: WorkerId,
|
|
789
|
+
generation: number,
|
|
790
|
+
workflow: Effect.Effect<void, never>,
|
|
529
791
|
): void {
|
|
530
792
|
try {
|
|
531
|
-
this.
|
|
793
|
+
const fiber = this.runGeneration(
|
|
532
794
|
workerId,
|
|
533
|
-
|
|
534
|
-
|
|
795
|
+
workflow.pipe(
|
|
796
|
+
Effect.catchCause((cause) => {
|
|
797
|
+
if (Cause.hasInterruptsOnly(cause)) return Effect.void;
|
|
798
|
+
return Effect.sync(() => {
|
|
799
|
+
this.settleWorkflowDefect(workerId, generation, Cause.squash(cause));
|
|
800
|
+
});
|
|
801
|
+
}),
|
|
802
|
+
),
|
|
535
803
|
);
|
|
804
|
+
const exit = fiber.pollUnsafe();
|
|
805
|
+
// A closed FiberMap rejects admission with an interrupted sentinel.
|
|
806
|
+
if (exit && Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause)) {
|
|
807
|
+
this.settleGenerationLaunchFailure(workerId, generation);
|
|
808
|
+
}
|
|
536
809
|
} catch (error) {
|
|
537
810
|
this.settleWorkflowDefect(workerId, generation, error);
|
|
538
811
|
}
|
|
539
812
|
}
|
|
540
813
|
|
|
814
|
+
private settleGenerationLaunchFailure(
|
|
815
|
+
workerId: WorkerId,
|
|
816
|
+
generation: number,
|
|
817
|
+
): void {
|
|
818
|
+
this.settleActiveWorker(
|
|
819
|
+
workerId,
|
|
820
|
+
generation,
|
|
821
|
+
["starting", "running"],
|
|
822
|
+
"failed",
|
|
823
|
+
{
|
|
824
|
+
status: "failed",
|
|
825
|
+
message: "Worker generation could not start because orchestration is closed",
|
|
826
|
+
},
|
|
827
|
+
"workflow",
|
|
828
|
+
true,
|
|
829
|
+
);
|
|
830
|
+
}
|
|
831
|
+
|
|
541
832
|
private bootstrapAndPrompt(
|
|
542
833
|
workerId: WorkerId,
|
|
543
834
|
generation: number,
|
|
544
835
|
): Effect.Effect<void, never> {
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
const session = yield* runtime.bootstrap(workerId, generation);
|
|
836
|
+
return Effect.gen({ self: this }, function* () {
|
|
837
|
+
const session = yield* this.bootstrap(workerId, generation);
|
|
548
838
|
if (!session) return;
|
|
549
|
-
const
|
|
550
|
-
if (!
|
|
551
|
-
yield*
|
|
839
|
+
const worker = this.current().workers.get(workerId);
|
|
840
|
+
if (!worker) return;
|
|
841
|
+
yield* this.executePrompt(
|
|
842
|
+
workerId,
|
|
843
|
+
generation,
|
|
844
|
+
session,
|
|
845
|
+
worker.record.instructions,
|
|
846
|
+
);
|
|
552
847
|
});
|
|
553
848
|
}
|
|
554
849
|
|
|
@@ -557,89 +852,109 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
557
852
|
generation: number,
|
|
558
853
|
): Effect.Effect<WorkerSessionHandle | undefined, never> {
|
|
559
854
|
return Effect.suspend(() => {
|
|
560
|
-
const
|
|
561
|
-
if (!
|
|
562
|
-
|
|
563
|
-
let creation: Promise<WorkerSessionHandle>;
|
|
564
|
-
try {
|
|
565
|
-
creation = this.workerSessionFactory.create({
|
|
566
|
-
cwd: entry.context.cwd,
|
|
567
|
-
agentDir: entry.context.agentDir,
|
|
568
|
-
parentSessionFile: entry.context.parentSessionFile,
|
|
569
|
-
projectTrusted: entry.context.projectTrusted,
|
|
570
|
-
definition: entry.definition,
|
|
571
|
-
parentModel: entry.context.parentModel,
|
|
572
|
-
modelRegistry: entry.context.modelRegistry,
|
|
573
|
-
});
|
|
574
|
-
} catch (error) {
|
|
575
|
-
this.settleCreationFailure(workerId, generation, error);
|
|
855
|
+
const expected = this.current().workers.get(workerId);
|
|
856
|
+
if (!expected || expected.generation !== generation) {
|
|
576
857
|
return Effect.succeed(undefined);
|
|
577
858
|
}
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
859
|
+
return this.childSessions.acquire(
|
|
860
|
+
{
|
|
861
|
+
cwd: expected.context.cwd,
|
|
862
|
+
agentDir: expected.context.agentDir,
|
|
863
|
+
parentSessionFile: expected.context.parentSessionFile,
|
|
864
|
+
projectTrusted: expected.context.projectTrusted,
|
|
865
|
+
definition: expected.definition,
|
|
866
|
+
parentModel: expected.context.parentModel,
|
|
867
|
+
modelRegistry: expected.context.modelRegistry,
|
|
868
|
+
},
|
|
869
|
+
(session) => this.adoptCreatedSession(workerId, generation, session),
|
|
870
|
+
).pipe(
|
|
590
871
|
Effect.match({
|
|
591
872
|
onFailure: (error) => {
|
|
592
873
|
this.settleCreationFailure(workerId, generation, error);
|
|
593
874
|
return undefined;
|
|
594
875
|
},
|
|
595
|
-
onSuccess: (session) =>
|
|
596
|
-
workerId,
|
|
597
|
-
generation,
|
|
598
|
-
entry,
|
|
599
|
-
session,
|
|
600
|
-
),
|
|
876
|
+
onSuccess: (session) => session,
|
|
601
877
|
}),
|
|
602
878
|
);
|
|
603
879
|
});
|
|
604
880
|
}
|
|
605
881
|
|
|
606
|
-
private canAdoptCreatedSession(
|
|
607
|
-
workerId: WorkerId,
|
|
608
|
-
generation: number,
|
|
609
|
-
entry: RuntimeEntry,
|
|
610
|
-
): boolean {
|
|
611
|
-
const current = this.workers.get(workerId);
|
|
612
|
-
return !this.shuttingDown &&
|
|
613
|
-
current?.status === "starting" &&
|
|
614
|
-
entry.generation === generation;
|
|
615
|
-
}
|
|
616
|
-
|
|
617
882
|
private adoptCreatedSession(
|
|
618
883
|
workerId: WorkerId,
|
|
619
884
|
generation: number,
|
|
620
|
-
entry: RuntimeEntry,
|
|
621
885
|
session: WorkerSessionHandle,
|
|
622
886
|
): WorkerSessionHandle | undefined {
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
887
|
+
let release: () => void;
|
|
888
|
+
try {
|
|
889
|
+
release = session.subscribeObservation((observation) => {
|
|
890
|
+
this.updateObservation(workerId, generation, session, observation);
|
|
891
|
+
});
|
|
892
|
+
} catch (error) {
|
|
893
|
+
this.settleCreationFailure(workerId, generation, error);
|
|
626
894
|
return undefined;
|
|
627
895
|
}
|
|
628
896
|
|
|
897
|
+
const adopted = this.transact((draft) => {
|
|
898
|
+
const worker = draft.workers.get(workerId);
|
|
899
|
+
if (
|
|
900
|
+
draft.lifecycle !== "open" ||
|
|
901
|
+
!worker ||
|
|
902
|
+
worker.generation !== generation ||
|
|
903
|
+
worker.record.status !== "starting"
|
|
904
|
+
) {
|
|
905
|
+
return {
|
|
906
|
+
value: false,
|
|
907
|
+
actions: [runAction(() => safelyCall(release))],
|
|
908
|
+
};
|
|
909
|
+
}
|
|
910
|
+
draft.workers.set(workerId, {
|
|
911
|
+
...worker,
|
|
912
|
+
session,
|
|
913
|
+
observationRelease: release,
|
|
914
|
+
record: {
|
|
915
|
+
...transitionWorkerStatus(worker.record, "running"),
|
|
916
|
+
sessionFile: session.sessionFile,
|
|
917
|
+
},
|
|
918
|
+
});
|
|
919
|
+
return {
|
|
920
|
+
value: true,
|
|
921
|
+
actions: [publishState(worker.record.ownerSessionId)],
|
|
922
|
+
};
|
|
923
|
+
});
|
|
924
|
+
return adopted ? session : undefined;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
private subscribeObservation(
|
|
928
|
+
workerId: WorkerId,
|
|
929
|
+
generation: number,
|
|
930
|
+
session: WorkerSessionHandle,
|
|
931
|
+
): void {
|
|
932
|
+
let release: () => void;
|
|
629
933
|
try {
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
this.workers.set(workerId, {
|
|
633
|
-
...transitionWorkerStatus(current, "running"),
|
|
634
|
-
sessionFile: session.sessionFile,
|
|
934
|
+
release = session.subscribeObservation((observation) => {
|
|
935
|
+
this.updateObservation(workerId, generation, session, observation);
|
|
635
936
|
});
|
|
636
|
-
this.emitState(current.ownerSessionId);
|
|
637
|
-
return session;
|
|
638
937
|
} catch (error) {
|
|
639
|
-
this.
|
|
640
|
-
|
|
641
|
-
return undefined;
|
|
938
|
+
this.settleWorkflowDefect(workerId, generation, error);
|
|
939
|
+
return;
|
|
642
940
|
}
|
|
941
|
+
|
|
942
|
+
this.transact((draft) => {
|
|
943
|
+
const worker = draft.workers.get(workerId);
|
|
944
|
+
if (
|
|
945
|
+
!worker ||
|
|
946
|
+
worker.generation !== generation ||
|
|
947
|
+
worker.session !== session ||
|
|
948
|
+
worker.record.status !== "running"
|
|
949
|
+
) {
|
|
950
|
+
return {
|
|
951
|
+
value: undefined,
|
|
952
|
+
actions: [runAction(() => safelyCall(release))],
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
draft.workers.set(workerId, { ...worker, observationRelease: release });
|
|
956
|
+
return { value: undefined };
|
|
957
|
+
});
|
|
643
958
|
}
|
|
644
959
|
|
|
645
960
|
private executePrompt(
|
|
@@ -649,34 +964,16 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
649
964
|
instructions: string,
|
|
650
965
|
): Effect.Effect<void, never> {
|
|
651
966
|
return Effect.suspend(() => {
|
|
652
|
-
const
|
|
653
|
-
const entry = this.entries.get(workerId);
|
|
967
|
+
const worker = this.current().workers.get(workerId);
|
|
654
968
|
if (
|
|
655
|
-
!
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
entry.session !== session
|
|
969
|
+
!worker ||
|
|
970
|
+
worker.record.status !== "running" ||
|
|
971
|
+
worker.generation !== generation ||
|
|
972
|
+
worker.session !== session
|
|
660
973
|
) {
|
|
661
974
|
return Effect.void;
|
|
662
975
|
}
|
|
663
|
-
|
|
664
|
-
let prompt: Promise<WorkerOutcome>;
|
|
665
|
-
try {
|
|
666
|
-
prompt = session.prompt(instructions);
|
|
667
|
-
} catch (error) {
|
|
668
|
-
this.settleOutcome(workerId, generation, session, {
|
|
669
|
-
status: "failed",
|
|
670
|
-
message: describeError(error, "Worker prompt failed"),
|
|
671
|
-
});
|
|
672
|
-
return Effect.void;
|
|
673
|
-
}
|
|
674
|
-
this.trackCleanup(prompt.then(() => undefined, () => undefined));
|
|
675
|
-
|
|
676
|
-
return Effect.tryPromise({
|
|
677
|
-
try: () => prompt,
|
|
678
|
-
catch: (error) => error,
|
|
679
|
-
}).pipe(
|
|
976
|
+
return session.prompt(instructions).pipe(
|
|
680
977
|
Effect.match({
|
|
681
978
|
onFailure: (error): WorkerOutcome => ({
|
|
682
979
|
status: "failed",
|
|
@@ -697,13 +994,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
697
994
|
generation: number,
|
|
698
995
|
error: unknown,
|
|
699
996
|
): void {
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
997
|
+
this.settleActiveWorker(
|
|
998
|
+
workerId,
|
|
999
|
+
generation,
|
|
1000
|
+
["starting"],
|
|
1001
|
+
"failed",
|
|
1002
|
+
{
|
|
1003
|
+
status: "failed",
|
|
1004
|
+
message: describeError(error, "Worker session creation failed"),
|
|
1005
|
+
},
|
|
1006
|
+
"startup",
|
|
1007
|
+
false,
|
|
1008
|
+
);
|
|
707
1009
|
}
|
|
708
1010
|
|
|
709
1011
|
private settleWorkflowDefect(
|
|
@@ -711,19 +1013,18 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
711
1013
|
generation: number,
|
|
712
1014
|
error: unknown,
|
|
713
1015
|
): void {
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
}, "workflow");
|
|
1016
|
+
this.settleActiveWorker(
|
|
1017
|
+
workerId,
|
|
1018
|
+
generation,
|
|
1019
|
+
["starting", "running", "stopping"],
|
|
1020
|
+
"failed",
|
|
1021
|
+
{
|
|
1022
|
+
status: "failed",
|
|
1023
|
+
message: describeError(error, "Worker workflow failed"),
|
|
1024
|
+
},
|
|
1025
|
+
"workflow",
|
|
1026
|
+
true,
|
|
1027
|
+
);
|
|
727
1028
|
}
|
|
728
1029
|
|
|
729
1030
|
private settleOutcome(
|
|
@@ -732,14 +1033,12 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
732
1033
|
session: WorkerSessionHandle,
|
|
733
1034
|
outcome: WorkerOutcome,
|
|
734
1035
|
): void {
|
|
735
|
-
const
|
|
736
|
-
const entry = this.entries.get(workerId);
|
|
1036
|
+
const worker = this.current().workers.get(workerId);
|
|
737
1037
|
if (
|
|
738
|
-
!
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
entry.session !== session
|
|
1038
|
+
!worker ||
|
|
1039
|
+
worker.generation !== generation ||
|
|
1040
|
+
worker.session !== session ||
|
|
1041
|
+
worker.record.status !== "running"
|
|
743
1042
|
) {
|
|
744
1043
|
return;
|
|
745
1044
|
}
|
|
@@ -748,549 +1047,1033 @@ class DefaultOrchestratorRuntime implements OrchestratorRuntime {
|
|
|
748
1047
|
let status: "ready" | "completed" | "failed" | "aborted";
|
|
749
1048
|
if (outcome.status === "failed" || outcome.status === "aborted") {
|
|
750
1049
|
status = outcome.status;
|
|
751
|
-
} else if (
|
|
1050
|
+
} else if (
|
|
1051
|
+
worker.record.lifecycle === "interactive" &&
|
|
1052
|
+
outcome.status === "ready"
|
|
1053
|
+
) {
|
|
752
1054
|
status = "ready";
|
|
753
|
-
} else if (
|
|
1055
|
+
} else if (
|
|
1056
|
+
worker.record.lifecycle === "one-shot" &&
|
|
1057
|
+
outcome.status === "completed"
|
|
1058
|
+
) {
|
|
754
1059
|
status = "completed";
|
|
755
1060
|
} else {
|
|
756
1061
|
status = "failed";
|
|
757
1062
|
settledOutcome = {
|
|
758
1063
|
status: "failed",
|
|
759
|
-
message: `Worker session returned ${outcome.status} for a ${
|
|
1064
|
+
message: `Worker session returned ${outcome.status} for a ${worker.record.lifecycle} worker`,
|
|
760
1065
|
};
|
|
761
1066
|
}
|
|
762
|
-
|
|
763
|
-
const settledAt = this.clock();
|
|
764
|
-
this.workers.set(workerId, {
|
|
765
|
-
...transitionWorkerStatus(current, status),
|
|
766
|
-
activity: undefined,
|
|
767
|
-
outcome: copyOutcome(settledOutcome),
|
|
768
|
-
settledAt,
|
|
769
|
-
});
|
|
770
|
-
if (status !== "ready") this.disposeEntrySession(entry);
|
|
771
|
-
this.maybeCompleteRun(current.runId);
|
|
772
|
-
const affectedOwners = new Set([current.ownerSessionId]);
|
|
773
|
-
this.emitSettlement(
|
|
1067
|
+
this.settleActiveWorker(
|
|
774
1068
|
workerId,
|
|
775
1069
|
generation,
|
|
776
|
-
|
|
1070
|
+
["running"],
|
|
1071
|
+
status,
|
|
1072
|
+
settledOutcome,
|
|
777
1073
|
status === "failed" ? "prompt" : undefined,
|
|
1074
|
+
status !== "ready",
|
|
778
1075
|
);
|
|
779
|
-
if (isTerminalWorkerStatus(status)) {
|
|
780
|
-
addAll(affectedOwners, this.rememberTerminalWorker(workerId));
|
|
781
|
-
} else {
|
|
782
|
-
addAll(affectedOwners, this.pruneHistory());
|
|
783
|
-
}
|
|
784
|
-
this.emitStateForOwners(affectedOwners);
|
|
785
1076
|
}
|
|
786
1077
|
|
|
787
|
-
private
|
|
788
|
-
|
|
789
|
-
|
|
1078
|
+
private settleActiveWorker(
|
|
1079
|
+
workerId: WorkerId,
|
|
1080
|
+
generation: number,
|
|
1081
|
+
expectedStatuses: readonly WorkerRecord["status"][],
|
|
1082
|
+
status: "ready" | "completed" | "failed" | "aborted",
|
|
790
1083
|
outcome: WorkerOutcome,
|
|
791
|
-
failureStage
|
|
1084
|
+
failureStage: SettlementFailureStage | undefined,
|
|
1085
|
+
dispose: boolean,
|
|
792
1086
|
): void {
|
|
793
|
-
const settledAt = this.clock();
|
|
794
|
-
this.
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
1087
|
+
const settledAt = this.clock.currentTimeMillisUnsafe();
|
|
1088
|
+
this.transact((draft) => {
|
|
1089
|
+
const worker = draft.workers.get(workerId);
|
|
1090
|
+
if (
|
|
1091
|
+
!worker ||
|
|
1092
|
+
worker.generation !== generation ||
|
|
1093
|
+
!expectedStatuses.includes(worker.record.status)
|
|
1094
|
+
) {
|
|
1095
|
+
return { value: undefined };
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
const settledRecord: WorkerRecord = {
|
|
1099
|
+
...transitionWorkerStatus(worker.record, status),
|
|
1100
|
+
activity: undefined,
|
|
1101
|
+
outcome: copyOutcome(outcome),
|
|
1102
|
+
settledAt,
|
|
1103
|
+
};
|
|
1104
|
+
const actions: PostCommitAction[] = [];
|
|
1105
|
+
let settledWorker: RuntimeWorker = { ...worker, record: settledRecord };
|
|
1106
|
+
if (dispose) {
|
|
1107
|
+
actions.push(...this.releaseWorkerResources(settledWorker));
|
|
1108
|
+
settledWorker = {
|
|
1109
|
+
...settledWorker,
|
|
1110
|
+
session: undefined,
|
|
1111
|
+
observationRelease: undefined,
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
draft.workers.set(workerId, settledWorker);
|
|
1115
|
+
const settlement = makeSettlement(
|
|
1116
|
+
draft,
|
|
1117
|
+
settledWorker,
|
|
1118
|
+
settledAt,
|
|
1119
|
+
failureStage,
|
|
1120
|
+
);
|
|
1121
|
+
actions.push(...completeRun(draft, workerId));
|
|
1122
|
+
if (settlement) actions.push(settlement);
|
|
1123
|
+
if (isTerminalWorkerStatus(status)) rememberTerminalWorker(draft, workerId);
|
|
1124
|
+
actions.push(...stateActionsAfterPrune(draft, worker.record.ownerSessionId));
|
|
1125
|
+
return { value: undefined, actions };
|
|
799
1126
|
});
|
|
800
|
-
this.maybeCompleteRun(current.runId);
|
|
801
|
-
const affectedOwners = new Set([current.ownerSessionId]);
|
|
802
|
-
const generation = this.entries.get(current.id)?.generation;
|
|
803
|
-
if (generation !== undefined) {
|
|
804
|
-
this.emitSettlement(current.id, generation, settledAt, failureStage);
|
|
805
|
-
}
|
|
806
|
-
addAll(affectedOwners, this.rememberTerminalWorker(current.id));
|
|
807
|
-
this.emitStateForOwners(affectedOwners);
|
|
808
1127
|
}
|
|
809
1128
|
|
|
810
|
-
private
|
|
1129
|
+
private updateObservation(
|
|
811
1130
|
workerId: WorkerId,
|
|
812
1131
|
generation: number,
|
|
813
|
-
|
|
814
|
-
|
|
1132
|
+
session: WorkerSessionHandle,
|
|
1133
|
+
observation: WorkerSessionObservation,
|
|
815
1134
|
): void {
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
? {
|
|
841
|
-
synthesisGroupId: run.synthesisGroupId,
|
|
842
|
-
synthesisGroupSize: run.synthesisGroupSize,
|
|
843
|
-
}
|
|
844
|
-
: {}),
|
|
845
|
-
...(worker.sessionFile !== undefined ? { sessionFile: worker.sessionFile } : {}),
|
|
1135
|
+
this.transact((draft) => {
|
|
1136
|
+
const worker = draft.workers.get(workerId);
|
|
1137
|
+
if (
|
|
1138
|
+
!worker ||
|
|
1139
|
+
worker.generation !== generation ||
|
|
1140
|
+
worker.session !== session ||
|
|
1141
|
+
(worker.record.status !== "starting" && worker.record.status !== "running")
|
|
1142
|
+
) {
|
|
1143
|
+
return { value: undefined };
|
|
1144
|
+
}
|
|
1145
|
+
const nextRecord: WorkerRecord = {
|
|
1146
|
+
...worker.record,
|
|
1147
|
+
usage: copyUsage(observation.usage),
|
|
1148
|
+
activity: observation.activity,
|
|
1149
|
+
messageDirection: observation.messageDirection,
|
|
1150
|
+
};
|
|
1151
|
+
if (observationsEqual(worker.record, nextRecord)) {
|
|
1152
|
+
return { value: undefined };
|
|
1153
|
+
}
|
|
1154
|
+
draft.workers.set(workerId, { ...worker, record: nextRecord });
|
|
1155
|
+
return {
|
|
1156
|
+
value: undefined,
|
|
1157
|
+
actions: [publishState(worker.record.ownerSessionId)],
|
|
1158
|
+
};
|
|
846
1159
|
});
|
|
847
|
-
|
|
848
|
-
const localListener = this.runSettlementListeners.get(run.id);
|
|
849
|
-
this.runSettlementListeners.delete(run.id);
|
|
850
|
-
notifySettlementListener(localListener, settlement);
|
|
851
|
-
for (const listener of [...this.settlementListeners]) {
|
|
852
|
-
notifySettlementListener(listener, settlement);
|
|
853
|
-
}
|
|
854
1160
|
}
|
|
855
1161
|
|
|
856
|
-
private
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
1162
|
+
private beginCancellation(
|
|
1163
|
+
operation: "abort",
|
|
1164
|
+
ownerSessionId: string,
|
|
1165
|
+
target: ValidatedAbortTarget,
|
|
1166
|
+
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1167
|
+
): Decision<readonly Deferred.Deferred<void>[]> {
|
|
1168
|
+
return this.transact((draft) => {
|
|
1169
|
+
const open = openDecision(draft, operation);
|
|
1170
|
+
if (open._tag === "rejected") return { value: open };
|
|
1171
|
+
const workers = target._tag === "all"
|
|
1172
|
+
? [...draft.workers.values()].filter((worker) => (
|
|
1173
|
+
worker.record.ownerSessionId === ownerSessionId &&
|
|
1174
|
+
isActiveWorkerStatus(worker.record.status)
|
|
1175
|
+
))
|
|
1176
|
+
: target.workerIds.map((id) => draft.workers.get(id));
|
|
1177
|
+
|
|
1178
|
+
if (target._tag === "ids") {
|
|
1179
|
+
for (let index = 0; index < target.workerIds.length; index += 1) {
|
|
1180
|
+
const worker = workers[index];
|
|
1181
|
+
if (!worker || worker.record.ownerSessionId !== ownerSessionId) {
|
|
1182
|
+
return {
|
|
1183
|
+
value: rejected(
|
|
1184
|
+
operation,
|
|
1185
|
+
"ownership",
|
|
1186
|
+
"Worker is not owned by this session",
|
|
1187
|
+
),
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
if (worker.record.status === "ready") {
|
|
1191
|
+
return {
|
|
1192
|
+
value: rejected(
|
|
1193
|
+
operation,
|
|
1194
|
+
"worker-state",
|
|
1195
|
+
"Ready interactive workers are not active; use interactive_close",
|
|
1196
|
+
),
|
|
1197
|
+
};
|
|
1198
|
+
}
|
|
1199
|
+
if (!isActiveWorkerStatus(worker.record.status)) {
|
|
1200
|
+
return {
|
|
1201
|
+
value: rejected(
|
|
1202
|
+
operation,
|
|
1203
|
+
"worker-state",
|
|
1204
|
+
"worker_abort requires owned active workers",
|
|
1205
|
+
),
|
|
1206
|
+
};
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
872
1210
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
1211
|
+
const activeWorkers = workers.filter(isRuntimeWorker);
|
|
1212
|
+
const marked = this.markWorkersStopping(draft, activeWorkers, candidates);
|
|
1213
|
+
return {
|
|
1214
|
+
value: accepted(marked.completions),
|
|
1215
|
+
actions: [
|
|
1216
|
+
...marked.actions,
|
|
1217
|
+
...publishOwners(new Set(
|
|
1218
|
+
activeWorkers.map((worker) => worker.record.ownerSessionId),
|
|
1219
|
+
)),
|
|
1220
|
+
],
|
|
1221
|
+
};
|
|
1222
|
+
});
|
|
878
1223
|
}
|
|
879
1224
|
|
|
880
|
-
private
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
const [workerId] = this.terminalWorkerOrder.splice(removableIndex, 1);
|
|
899
|
-
if (!workerId) return affectedOwners;
|
|
900
|
-
const worker = this.workers.get(workerId);
|
|
901
|
-
if (worker) affectedOwners.add(worker.ownerSessionId);
|
|
902
|
-
this.workers.delete(workerId);
|
|
903
|
-
this.entries.delete(workerId);
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
return affectedOwners;
|
|
1225
|
+
private beginShutdownCancellation(
|
|
1226
|
+
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1227
|
+
): readonly Deferred.Deferred<void>[] {
|
|
1228
|
+
return this.transact((draft) => {
|
|
1229
|
+
const workers = [...draft.workers.values()].filter((worker) => (
|
|
1230
|
+
isActiveWorkerStatus(worker.record.status)
|
|
1231
|
+
));
|
|
1232
|
+
const marked = this.markWorkersStopping(draft, workers, candidates);
|
|
1233
|
+
return {
|
|
1234
|
+
value: marked.completions,
|
|
1235
|
+
actions: [
|
|
1236
|
+
...marked.actions,
|
|
1237
|
+
...publishOwners(new Set(
|
|
1238
|
+
workers.map((worker) => worker.record.ownerSessionId),
|
|
1239
|
+
)),
|
|
1240
|
+
],
|
|
1241
|
+
};
|
|
1242
|
+
});
|
|
907
1243
|
}
|
|
908
1244
|
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
throw new Error("workerIds must contain at least one worker ID");
|
|
1245
|
+
// Commit stopping and one shared completion before post-commit physical abort
|
|
1246
|
+
// and generation interruption; every cancellation caller joins it.
|
|
1247
|
+
private markWorkersStopping(
|
|
1248
|
+
draft: RuntimeState,
|
|
1249
|
+
workers: readonly RuntimeWorker[],
|
|
1250
|
+
candidates: ReadonlyMap<WorkerId, Deferred.Deferred<void>>,
|
|
1251
|
+
): {
|
|
1252
|
+
readonly completions: readonly Deferred.Deferred<void>[];
|
|
1253
|
+
readonly actions: readonly PostCommitAction[];
|
|
1254
|
+
} {
|
|
1255
|
+
const completions: Deferred.Deferred<void>[] = [];
|
|
1256
|
+
const actions: PostCommitAction[] = [];
|
|
1257
|
+
for (const worker of workers) {
|
|
1258
|
+
if (worker.cancellation) {
|
|
1259
|
+
completions.push(worker.cancellation);
|
|
1260
|
+
continue;
|
|
926
1261
|
}
|
|
927
|
-
const
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
if (worker.status === "ready") {
|
|
931
|
-
throw new Error("Ready interactive workers are not active; use interactive_close");
|
|
932
|
-
}
|
|
933
|
-
if (!isActiveWorkerStatus(worker.status)) {
|
|
934
|
-
throw new Error("worker_abort requires owned active workers");
|
|
935
|
-
}
|
|
1262
|
+
const completion = candidates.get(worker.record.id);
|
|
1263
|
+
if (!completion) {
|
|
1264
|
+
throw new Error(`Missing cancellation candidate: ${worker.record.id}`);
|
|
936
1265
|
}
|
|
937
|
-
|
|
1266
|
+
const stoppingRecord = worker.record.status === "stopping"
|
|
1267
|
+
? worker.record
|
|
1268
|
+
: {
|
|
1269
|
+
...transitionWorkerStatus(worker.record, "stopping"),
|
|
1270
|
+
activity: undefined,
|
|
1271
|
+
};
|
|
1272
|
+
draft.workers.set(worker.record.id, {
|
|
1273
|
+
...worker,
|
|
1274
|
+
record: stoppingRecord,
|
|
1275
|
+
cancellation: completion,
|
|
1276
|
+
});
|
|
1277
|
+
completions.push(completion);
|
|
1278
|
+
actions.push(runAction(() => {
|
|
1279
|
+
this.launchCancellationOrSettleAfterClosure(worker.record.id, completion);
|
|
1280
|
+
}));
|
|
938
1281
|
}
|
|
939
|
-
|
|
940
|
-
return [...this.workers.values()]
|
|
941
|
-
.filter(
|
|
942
|
-
(worker) =>
|
|
943
|
-
worker.ownerSessionId === ownerSessionId && isActiveWorkerStatus(worker.status),
|
|
944
|
-
)
|
|
945
|
-
.map((worker) => worker.id);
|
|
1282
|
+
return { completions, actions };
|
|
946
1283
|
}
|
|
947
1284
|
|
|
948
|
-
private
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
1285
|
+
private launchCancellationOrSettleAfterClosure(
|
|
1286
|
+
workerId: WorkerId,
|
|
1287
|
+
completion: Deferred.Deferred<void>,
|
|
1288
|
+
): void {
|
|
1289
|
+
const fiber = this.runCancellation(this.cancelWorker(workerId, completion));
|
|
1290
|
+
const exit = fiber.pollUnsafe();
|
|
1291
|
+
// A closed FiberSet rejects admission with an interrupted sentinel.
|
|
1292
|
+
if (!exit || Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return;
|
|
1293
|
+
|
|
1294
|
+
const worker = this.current().workers.get(workerId);
|
|
1295
|
+
if (worker?.record.status === "stopping") {
|
|
1296
|
+
this.settleActiveWorker(
|
|
1297
|
+
workerId,
|
|
1298
|
+
worker.generation,
|
|
1299
|
+
["stopping"],
|
|
1300
|
+
"aborted",
|
|
1301
|
+
{ status: "aborted" },
|
|
1302
|
+
"cancellation",
|
|
1303
|
+
true,
|
|
1304
|
+
);
|
|
963
1305
|
}
|
|
964
|
-
|
|
965
|
-
this.emitStateForOwners(owners);
|
|
966
|
-
await Promise.all(cancellations);
|
|
1306
|
+
this.completeCancellation(workerId, completion);
|
|
967
1307
|
}
|
|
968
1308
|
|
|
969
|
-
private
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
.catch((error) => {
|
|
976
|
-
this.settleCancellationFailure(workerId, error);
|
|
977
|
-
});
|
|
978
|
-
this.cancellationPromises.set(workerId, cancellation);
|
|
979
|
-
void cancellation.then(() => {
|
|
980
|
-
if (this.cancellationPromises.get(workerId) === cancellation) {
|
|
981
|
-
this.cancellationPromises.delete(workerId);
|
|
982
|
-
}
|
|
983
|
-
});
|
|
984
|
-
return cancellation;
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
private async cancelWorker(workerId: WorkerId): Promise<void> {
|
|
988
|
-
const entry = this.entries.get(workerId);
|
|
989
|
-
const session = entry?.session;
|
|
990
|
-
|
|
991
|
-
try {
|
|
1309
|
+
private cancelWorker(
|
|
1310
|
+
workerId: WorkerId,
|
|
1311
|
+
completion: Deferred.Deferred<void>,
|
|
1312
|
+
): Effect.Effect<void> {
|
|
1313
|
+
return Effect.gen({ self: this }, function* () {
|
|
1314
|
+
const session = this.current().workers.get(workerId)?.session;
|
|
992
1315
|
if (session) {
|
|
993
|
-
|
|
994
|
-
.
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
// Worker state still settles even if scheduler cleanup fails.
|
|
1316
|
+
yield* session.abort().pipe(
|
|
1317
|
+
Effect.catchTag(
|
|
1318
|
+
"WorkerSession.AbortError",
|
|
1319
|
+
(_: WorkerSessionAbortError) => Effect.void,
|
|
1320
|
+
),
|
|
1321
|
+
Effect.timeoutOption(CANCELLATION_GRACE_MS),
|
|
1322
|
+
Effect.ignore,
|
|
1323
|
+
);
|
|
1002
1324
|
}
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1325
|
+
const removal = yield* FiberSet.run(
|
|
1326
|
+
this.cancellations,
|
|
1327
|
+
FiberMap.remove(this.generations, workerId).pipe(
|
|
1328
|
+
Effect.catchCause(() => Effect.void),
|
|
1329
|
+
),
|
|
1330
|
+
);
|
|
1331
|
+
yield* Fiber.await(removal).pipe(
|
|
1332
|
+
Effect.timeoutOption(CANCELLATION_GRACE_MS),
|
|
1333
|
+
Effect.ignore,
|
|
1334
|
+
);
|
|
1335
|
+
this.settleActiveWorker(
|
|
1336
|
+
workerId,
|
|
1337
|
+
this.current().workers.get(workerId)?.generation ?? -1,
|
|
1338
|
+
["stopping"],
|
|
1339
|
+
"aborted",
|
|
1340
|
+
{ status: "aborted" },
|
|
1341
|
+
"cancellation",
|
|
1342
|
+
true,
|
|
1343
|
+
);
|
|
1344
|
+
}).pipe(
|
|
1345
|
+
Effect.catchCause((cause) => Effect.sync(() => {
|
|
1346
|
+
const generation = this.current().workers.get(workerId)?.generation;
|
|
1347
|
+
if (generation === undefined) return;
|
|
1348
|
+
this.settleActiveWorker(
|
|
1349
|
+
workerId,
|
|
1350
|
+
generation,
|
|
1351
|
+
["stopping"],
|
|
1352
|
+
"failed",
|
|
1353
|
+
{
|
|
1354
|
+
status: "failed",
|
|
1355
|
+
message: describeError(
|
|
1356
|
+
Cause.squash(cause),
|
|
1357
|
+
"Worker cancellation failed",
|
|
1358
|
+
),
|
|
1359
|
+
},
|
|
1011
1360
|
"cancellation",
|
|
1361
|
+
true,
|
|
1012
1362
|
);
|
|
1013
|
-
}
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
const current = this.workers.get(workerId);
|
|
1019
|
-
if (!current || current.status !== "stopping") return;
|
|
1020
|
-
const entry = this.entries.get(workerId);
|
|
1021
|
-
if (entry) this.disposeEntrySession(entry);
|
|
1022
|
-
this.settleTerminalWorker(current, "failed", {
|
|
1023
|
-
status: "failed",
|
|
1024
|
-
message: describeError(error, "Worker cancellation failed"),
|
|
1025
|
-
}, "cancellation");
|
|
1363
|
+
})),
|
|
1364
|
+
Effect.ensuring(
|
|
1365
|
+
Effect.sync(() => this.completeCancellation(workerId, completion)),
|
|
1366
|
+
),
|
|
1367
|
+
);
|
|
1026
1368
|
}
|
|
1027
1369
|
|
|
1028
|
-
private
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1370
|
+
private completeCancellation(
|
|
1371
|
+
workerId: WorkerId,
|
|
1372
|
+
completion: Deferred.Deferred<void>,
|
|
1373
|
+
): void {
|
|
1374
|
+
this.transact((draft) => {
|
|
1375
|
+
const worker = draft.workers.get(workerId);
|
|
1376
|
+
if (worker?.cancellation === completion) {
|
|
1377
|
+
draft.workers.set(workerId, { ...worker, cancellation: undefined });
|
|
1378
|
+
}
|
|
1379
|
+
return { value: undefined };
|
|
1380
|
+
});
|
|
1381
|
+
Deferred.doneUnsafe(completion, Effect.void);
|
|
1037
1382
|
}
|
|
1038
1383
|
|
|
1039
1384
|
private awaitInlineRun(
|
|
1040
1385
|
run: RunRecord,
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
let abortClaimed = false;
|
|
1048
|
-
const removeAbortListener = () => signal.removeEventListener("abort", onAbort);
|
|
1049
|
-
const onAbort = () => {
|
|
1050
|
-
if (abortClaimed || waiter.settled) return;
|
|
1051
|
-
abortClaimed = true;
|
|
1052
|
-
removeAbortListener();
|
|
1053
|
-
const reason = abortSignalReason(signal);
|
|
1054
|
-
void this.cancelExactRun(run).then(
|
|
1055
|
-
() => reject(reason),
|
|
1056
|
-
() => reject(reason),
|
|
1057
|
-
);
|
|
1058
|
-
};
|
|
1059
|
-
|
|
1060
|
-
waiter.onSettled = () => {
|
|
1061
|
-
removeAbortListener();
|
|
1062
|
-
if (!abortClaimed) waiter.promise.then(resolve, reject);
|
|
1063
|
-
};
|
|
1064
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
1386
|
+
completion: Deferred.Deferred<CompletedRun>,
|
|
1387
|
+
): Effect.Effect<CompletedRun> {
|
|
1388
|
+
return Deferred.await(completion).pipe(
|
|
1389
|
+
Effect.onInterrupt(() => this.cancelExactRun(run)),
|
|
1390
|
+
);
|
|
1391
|
+
}
|
|
1065
1392
|
|
|
1066
|
-
|
|
1067
|
-
|
|
1393
|
+
private cancelExactRun(run: RunRecord): Effect.Effect<void> {
|
|
1394
|
+
return Effect.gen({ self: this }, function* () {
|
|
1395
|
+
const candidate = yield* Deferred.make<void>();
|
|
1396
|
+
const candidates = new Map([[run.workerId, candidate]]);
|
|
1397
|
+
const completions = this.transact((draft) => {
|
|
1398
|
+
const worker = draft.workers.get(run.workerId);
|
|
1399
|
+
if (
|
|
1400
|
+
!worker ||
|
|
1401
|
+
worker.record.ownerSessionId !== run.ownerSessionId ||
|
|
1402
|
+
worker.record.runId !== run.id ||
|
|
1403
|
+
!isActiveWorkerStatus(worker.record.status)
|
|
1404
|
+
) {
|
|
1405
|
+
return { value: [] as readonly Deferred.Deferred<void>[] };
|
|
1406
|
+
}
|
|
1407
|
+
const marked = this.markWorkersStopping(draft, [worker], candidates);
|
|
1408
|
+
return {
|
|
1409
|
+
value: marked.completions,
|
|
1410
|
+
actions: [...marked.actions, publishState(run.ownerSessionId)],
|
|
1411
|
+
};
|
|
1412
|
+
});
|
|
1413
|
+
yield* awaitAll(completions);
|
|
1068
1414
|
});
|
|
1069
1415
|
}
|
|
1070
1416
|
|
|
1071
|
-
private
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1417
|
+
private closeReadyWorker(
|
|
1418
|
+
draft: RuntimeState,
|
|
1419
|
+
worker: RuntimeWorker,
|
|
1420
|
+
settledAt: number,
|
|
1421
|
+
actions: PostCommitAction[] = [],
|
|
1422
|
+
): PostCommitAction[] {
|
|
1423
|
+
const closedRecord: WorkerRecord = {
|
|
1424
|
+
...transitionWorkerStatus(worker.record, "closed"),
|
|
1076
1425
|
activity: undefined,
|
|
1077
1426
|
outcome: { status: "closed" },
|
|
1078
|
-
settledAt
|
|
1427
|
+
settledAt,
|
|
1428
|
+
};
|
|
1429
|
+
actions.push(...this.releaseWorkerResources(worker));
|
|
1430
|
+
draft.workers.set(worker.record.id, {
|
|
1431
|
+
...worker,
|
|
1432
|
+
record: closedRecord,
|
|
1433
|
+
session: undefined,
|
|
1434
|
+
observationRelease: undefined,
|
|
1079
1435
|
});
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
affectedOwners.add(current.ownerSessionId);
|
|
1083
|
-
this.emitStateForOwners(affectedOwners);
|
|
1436
|
+
rememberTerminalWorker(draft, worker.record.id);
|
|
1437
|
+
return actions;
|
|
1084
1438
|
}
|
|
1085
1439
|
|
|
1086
|
-
private
|
|
1087
|
-
const
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1440
|
+
private releaseWorkerResources(worker: RuntimeWorker): PostCommitAction[] {
|
|
1441
|
+
const actions: PostCommitAction[] = [];
|
|
1442
|
+
if (worker.observationRelease) {
|
|
1443
|
+
actions.push(runAction(() => safelyCall(worker.observationRelease)));
|
|
1444
|
+
}
|
|
1445
|
+
const session = worker.session;
|
|
1446
|
+
if (session) {
|
|
1447
|
+
actions.push(runAction(() => {
|
|
1448
|
+
this.launchCleanup(
|
|
1449
|
+
Effect.suspend(() => session.dispose()).pipe(
|
|
1450
|
+
Effect.catchCause(() => Effect.void),
|
|
1451
|
+
Effect.uninterruptible,
|
|
1452
|
+
),
|
|
1453
|
+
);
|
|
1454
|
+
}));
|
|
1092
1455
|
}
|
|
1456
|
+
return actions;
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
private launchCleanup(cleanup: Effect.Effect<void>): void {
|
|
1460
|
+
const fiber = Effect.runFork(cleanup);
|
|
1461
|
+
FiberSet.addUnsafe(this.cleanups, fiber);
|
|
1093
1462
|
}
|
|
1094
1463
|
|
|
1095
|
-
private
|
|
1464
|
+
private preflightOpen(
|
|
1465
|
+
operation: OrchestrationOperation,
|
|
1466
|
+
): Decision<void> {
|
|
1467
|
+
return this.transact((draft) => ({
|
|
1468
|
+
value: openDecision(draft, operation),
|
|
1469
|
+
}));
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
private preflightReadyInteractive(
|
|
1473
|
+
ownerSessionId: string,
|
|
1096
1474
|
workerId: WorkerId,
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
if (latest.status !== "starting" && latest.status !== "running") return;
|
|
1106
|
-
this.workers.set(workerId, { ...latest, usage: copyUsage(usage) });
|
|
1107
|
-
this.emitState(latest.ownerSessionId);
|
|
1108
|
-
});
|
|
1109
|
-
entry.unsubscribeActivity = session.subscribeActivity((activity) => {
|
|
1110
|
-
const latest = this.workers.get(workerId);
|
|
1111
|
-
if (!latest || entry.session !== session || entry.generation !== generation) return;
|
|
1112
|
-
if (latest.status !== "starting" && latest.status !== "running") return;
|
|
1113
|
-
if (latest.activity === activity) return;
|
|
1114
|
-
this.workers.set(workerId, { ...latest, activity });
|
|
1115
|
-
this.emitState(latest.ownerSessionId);
|
|
1116
|
-
});
|
|
1117
|
-
entry.unsubscribeMessageDirection = session.subscribeMessageDirection((messageDirection) => {
|
|
1118
|
-
const latest = this.workers.get(workerId);
|
|
1119
|
-
if (!latest || entry.session !== session || entry.generation !== generation) return;
|
|
1120
|
-
if (latest.status !== "starting" && latest.status !== "running") return;
|
|
1121
|
-
if (latest.messageDirection === messageDirection) return;
|
|
1122
|
-
this.workers.set(workerId, { ...latest, messageDirection });
|
|
1123
|
-
this.emitState(latest.ownerSessionId);
|
|
1475
|
+
): Decision<void> {
|
|
1476
|
+
return this.transact((draft) => {
|
|
1477
|
+
const ready = readyInteractiveDecision(draft, ownerSessionId, workerId);
|
|
1478
|
+
return {
|
|
1479
|
+
value: ready._tag === "accepted"
|
|
1480
|
+
? accepted(undefined)
|
|
1481
|
+
: ready,
|
|
1482
|
+
};
|
|
1124
1483
|
});
|
|
1125
1484
|
}
|
|
1126
1485
|
|
|
1127
|
-
private
|
|
1128
|
-
|
|
1129
|
-
try {
|
|
1130
|
-
listener(ownerSessionId);
|
|
1131
|
-
} catch {
|
|
1132
|
-
// One subscriber cannot prevent runtime mutations or other notifications.
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1486
|
+
private current(): RuntimeState {
|
|
1487
|
+
return this.state;
|
|
1135
1488
|
}
|
|
1136
1489
|
|
|
1137
|
-
private
|
|
1138
|
-
|
|
1490
|
+
private transact<A>(
|
|
1491
|
+
reducer: (draft: RuntimeState) => TransactionMutation<A>,
|
|
1492
|
+
): A {
|
|
1493
|
+
const draft = makeDraft(this.state);
|
|
1494
|
+
const mutation = reducer(draft);
|
|
1495
|
+
// Commit before action factories capture the snapshot and callbacks can run.
|
|
1496
|
+
this.state = Object.freeze(draft);
|
|
1497
|
+
this.enqueueActions((mutation.actions ?? []).map((action) => action(
|
|
1498
|
+
this.state,
|
|
1499
|
+
this.settlementListeners,
|
|
1500
|
+
this.stateListeners,
|
|
1501
|
+
)));
|
|
1502
|
+
return mutation.value;
|
|
1139
1503
|
}
|
|
1140
1504
|
|
|
1141
|
-
private
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1505
|
+
private enqueueActions(actions: readonly CommittedAction[]): void {
|
|
1506
|
+
// Reentrant actions queue behind this drain; drain all before rethrowing the first failure.
|
|
1507
|
+
this.actionQueue.push(...actions);
|
|
1508
|
+
if (this.drainingActions) return;
|
|
1509
|
+
|
|
1510
|
+
this.drainingActions = true;
|
|
1511
|
+
let hasFailure = false;
|
|
1512
|
+
let firstFailure: unknown;
|
|
1513
|
+
try {
|
|
1514
|
+
while (this.actionQueue.length > 0) {
|
|
1515
|
+
const action = this.actionQueue.shift();
|
|
1516
|
+
if (!action) continue;
|
|
1517
|
+
try {
|
|
1518
|
+
action();
|
|
1519
|
+
} catch (error) {
|
|
1520
|
+
if (!hasFailure) firstFailure = error;
|
|
1521
|
+
hasFailure = true;
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
} finally {
|
|
1525
|
+
this.drainingActions = false;
|
|
1145
1526
|
}
|
|
1146
|
-
|
|
1527
|
+
if (hasFailure) throw firstFailure;
|
|
1147
1528
|
}
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
export function orchestrationLayer(
|
|
1532
|
+
options: OrchestrationLayerOptions = {},
|
|
1533
|
+
): Layer.Layer<Orchestration, never, ChildSessions> {
|
|
1534
|
+
return Layer.effect(
|
|
1535
|
+
Orchestration,
|
|
1536
|
+
Effect.gen(function* () {
|
|
1537
|
+
const childSessions = yield* ChildSessions;
|
|
1538
|
+
// Scope finalizers run in reverse acquisition order. Keep cleanup open while
|
|
1539
|
+
// generation and cancellation interruption settle workers and enqueue disposal.
|
|
1540
|
+
const cleanups = yield* FiberSet.make<void, never>();
|
|
1541
|
+
const cancellations = yield* FiberSet.make<void, never>();
|
|
1542
|
+
const runCancellation = yield* FiberSet.runtime(cancellations)<never>();
|
|
1543
|
+
const generations = yield* FiberMap.make<WorkerId, void, never>();
|
|
1544
|
+
const runGeneration = yield* FiberMap.runtime(generations)<never>();
|
|
1545
|
+
const clock = yield* Clock.Clock;
|
|
1546
|
+
const shutdownCompletion = yield* Deferred.make<void>();
|
|
1547
|
+
return Orchestration.of(new StatefulOrchestration(
|
|
1548
|
+
childSessions,
|
|
1549
|
+
generations,
|
|
1550
|
+
runGeneration,
|
|
1551
|
+
cancellations,
|
|
1552
|
+
runCancellation,
|
|
1553
|
+
cleanups,
|
|
1554
|
+
clock,
|
|
1555
|
+
options.idFactories ?? createRandomIdFactories(),
|
|
1556
|
+
shutdownCompletion,
|
|
1557
|
+
));
|
|
1558
|
+
}),
|
|
1559
|
+
);
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
function initialState(): RuntimeState {
|
|
1563
|
+
return Object.freeze({
|
|
1564
|
+
workers: new Map<WorkerId, RuntimeWorker>(),
|
|
1565
|
+
runs: new Map<RunId, RuntimeRun>(),
|
|
1566
|
+
terminalWorkerOrder: [],
|
|
1567
|
+
completedRunOrder: [],
|
|
1568
|
+
settlementSequence: 0,
|
|
1569
|
+
lifecycle: "open",
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1148
1572
|
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1573
|
+
function makeDraft(state: RuntimeState): RuntimeState {
|
|
1574
|
+
return {
|
|
1575
|
+
...state,
|
|
1576
|
+
workers: new Map(state.workers),
|
|
1577
|
+
runs: new Map(state.runs),
|
|
1578
|
+
terminalWorkerOrder: [...state.terminalWorkerOrder],
|
|
1579
|
+
completedRunOrder: [...state.completedRunOrder],
|
|
1580
|
+
};
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
function completeRun(
|
|
1584
|
+
state: RuntimeState,
|
|
1585
|
+
workerId: WorkerId,
|
|
1586
|
+
): PostCommitAction[] {
|
|
1587
|
+
const worker = state.workers.get(workerId);
|
|
1588
|
+
if (!worker) return [];
|
|
1589
|
+
const run = state.runs.get(worker.record.runId);
|
|
1590
|
+
if (!run || !isRunningRuntimeRun(run) || !isCompletedRunWorker(worker.record)) {
|
|
1591
|
+
return [];
|
|
1156
1592
|
}
|
|
1157
1593
|
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1594
|
+
const completed = freezeCompletedRun(run.record, worker.record);
|
|
1595
|
+
state.runs.set(run.record.id, { ...run.record, state: "complete" });
|
|
1596
|
+
state.completedRunOrder.push(run.record.id);
|
|
1597
|
+
return [runAction(() => {
|
|
1598
|
+
Deferred.doneUnsafe(run.completion, Effect.succeed(completed));
|
|
1599
|
+
})];
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
function makeSettlement(
|
|
1603
|
+
draft: RuntimeState,
|
|
1604
|
+
worker: RuntimeWorker,
|
|
1605
|
+
settledAt: number,
|
|
1606
|
+
failureStage: SettlementFailureStage | undefined,
|
|
1607
|
+
): PostCommitAction | undefined {
|
|
1608
|
+
const run = draft.runs.get(worker.record.runId);
|
|
1609
|
+
if (!run || !isCompletedRunWorker(worker.record)) {
|
|
1610
|
+
return undefined;
|
|
1162
1611
|
}
|
|
1612
|
+
const runRecord = runtimeRunRecord(run);
|
|
1613
|
+
const sequence = draft.settlementSequence + 1;
|
|
1614
|
+
draft.settlementSequence = sequence;
|
|
1615
|
+
const settlement: WorkerSettlement = Object.freeze({
|
|
1616
|
+
eventId: `${sequence}:${runRecord.id}:${worker.record.id}:${worker.generation}`,
|
|
1617
|
+
sequence,
|
|
1618
|
+
ownerSessionId: worker.record.ownerSessionId,
|
|
1619
|
+
runId: runRecord.id,
|
|
1620
|
+
workerId: worker.record.id,
|
|
1621
|
+
generation: worker.generation,
|
|
1622
|
+
mode: runRecord.mode,
|
|
1623
|
+
worker: worker.record.worker,
|
|
1624
|
+
title: worker.record.title,
|
|
1625
|
+
lifecycle: worker.record.lifecycle,
|
|
1626
|
+
status: worker.record.status,
|
|
1627
|
+
outcome: Object.freeze(copyOutcome(worker.record.outcome)),
|
|
1628
|
+
...(failureStage ? { failureStage } : {}),
|
|
1629
|
+
usage: Object.freeze(copyUsage(worker.record.usage)),
|
|
1630
|
+
startedAt: worker.record.startedAt,
|
|
1631
|
+
settledAt,
|
|
1632
|
+
...(runRecord.synthesisGroupId && runRecord.synthesisGroupSize
|
|
1633
|
+
? {
|
|
1634
|
+
synthesisGroupId: runRecord.synthesisGroupId,
|
|
1635
|
+
synthesisGroupSize: runRecord.synthesisGroupSize,
|
|
1636
|
+
}
|
|
1637
|
+
: {}),
|
|
1638
|
+
...(worker.record.sessionFile !== undefined
|
|
1639
|
+
? { sessionFile: worker.record.sessionFile }
|
|
1640
|
+
: {}),
|
|
1641
|
+
});
|
|
1642
|
+
return publishSettlement(
|
|
1643
|
+
settlement,
|
|
1644
|
+
isRunningRuntimeRun(run) ? run.settlementListener : undefined,
|
|
1645
|
+
);
|
|
1646
|
+
}
|
|
1163
1647
|
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1648
|
+
function stateActionsAfterPrune(
|
|
1649
|
+
draft: RuntimeState,
|
|
1650
|
+
ownerSessionId: string,
|
|
1651
|
+
): PostCommitAction[] {
|
|
1652
|
+
const owners = pruneHistory(draft);
|
|
1653
|
+
owners.add(ownerSessionId);
|
|
1654
|
+
return publishOwners(owners);
|
|
1655
|
+
}
|
|
1167
1656
|
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1657
|
+
function pruneHistory(draft: RuntimeState): Set<string> {
|
|
1658
|
+
const owners = new Set<string>();
|
|
1659
|
+
while (draft.completedRunOrder.length > MAX_COMPLETED_RUN_HISTORY) {
|
|
1660
|
+
const runId = draft.completedRunOrder.shift();
|
|
1661
|
+
if (!runId) break;
|
|
1662
|
+
const run = draft.runs.get(runId);
|
|
1663
|
+
if (run) owners.add(runtimeRunRecord(run).ownerSessionId);
|
|
1664
|
+
draft.runs.delete(runId);
|
|
1173
1665
|
}
|
|
1666
|
+
while (draft.terminalWorkerOrder.length > MAX_TERMINAL_WORKER_HISTORY) {
|
|
1667
|
+
const index = draft.terminalWorkerOrder.findIndex((workerId) => {
|
|
1668
|
+
const worker = draft.workers.get(workerId);
|
|
1669
|
+
return !worker || isTerminalWorkerStatus(worker.record.status);
|
|
1670
|
+
});
|
|
1671
|
+
if (index < 0) break;
|
|
1672
|
+
const removed = draft.terminalWorkerOrder.splice(index, 1)[0];
|
|
1673
|
+
if (!removed) break;
|
|
1674
|
+
const worker = draft.workers.get(removed);
|
|
1675
|
+
if (worker) owners.add(worker.record.ownerSessionId);
|
|
1676
|
+
draft.workers.delete(removed);
|
|
1677
|
+
}
|
|
1678
|
+
return owners;
|
|
1679
|
+
}
|
|
1174
1680
|
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1681
|
+
function rememberTerminalWorker(
|
|
1682
|
+
draft: RuntimeState,
|
|
1683
|
+
workerId: WorkerId,
|
|
1684
|
+
): void {
|
|
1685
|
+
if (!draft.terminalWorkerOrder.includes(workerId)) {
|
|
1686
|
+
draft.terminalWorkerOrder.push(workerId);
|
|
1180
1687
|
}
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
function snapshotFor(
|
|
1691
|
+
state: RuntimeState,
|
|
1692
|
+
ownerSessionId: string,
|
|
1693
|
+
): RuntimeSnapshot {
|
|
1694
|
+
return Object.freeze({
|
|
1695
|
+
runs: Object.freeze(
|
|
1696
|
+
[...state.runs.values()]
|
|
1697
|
+
.map(runtimeRunRecord)
|
|
1698
|
+
.filter((run) => run.ownerSessionId === ownerSessionId)
|
|
1699
|
+
.map(copyRunRecord),
|
|
1700
|
+
),
|
|
1701
|
+
workers: Object.freeze(
|
|
1702
|
+
[...state.workers.values()]
|
|
1703
|
+
.map((worker) => worker.record)
|
|
1704
|
+
.filter((worker) => worker.ownerSessionId === ownerSessionId)
|
|
1705
|
+
.map(copyWorkerRecord),
|
|
1706
|
+
),
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
function runtimeRunRecord(run: RuntimeRun): RunRecord {
|
|
1711
|
+
return isRunningRuntimeRun(run) ? run.record : run;
|
|
1712
|
+
}
|
|
1181
1713
|
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1714
|
+
function isRunningRuntimeRun(run: RuntimeRun): run is RunningRuntimeRun {
|
|
1715
|
+
return "_tag" in run;
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
function makeRunRecord(
|
|
1719
|
+
runId: RunId,
|
|
1720
|
+
workerId: WorkerId,
|
|
1721
|
+
context: OrchestrationContext,
|
|
1722
|
+
mode: RunMode,
|
|
1723
|
+
createdAt: number,
|
|
1724
|
+
): RunRecord {
|
|
1725
|
+
return {
|
|
1726
|
+
id: runId,
|
|
1727
|
+
ownerSessionId: context.ownerSessionId,
|
|
1728
|
+
workerId,
|
|
1729
|
+
mode,
|
|
1730
|
+
state: "running",
|
|
1731
|
+
createdAt,
|
|
1732
|
+
...(context.synthesisGroup
|
|
1733
|
+
? {
|
|
1734
|
+
synthesisGroupId: context.synthesisGroup.id,
|
|
1735
|
+
synthesisGroupSize: context.synthesisGroup.size,
|
|
1736
|
+
}
|
|
1737
|
+
: {}),
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
function makeWorkerRecord(
|
|
1742
|
+
workerId: WorkerId,
|
|
1743
|
+
runId: RunId,
|
|
1744
|
+
ownerSessionId: string,
|
|
1745
|
+
definition: WorkerDefinition,
|
|
1746
|
+
task: OrchestrateTaskInput,
|
|
1747
|
+
startedAt: number,
|
|
1748
|
+
): WorkerRecord {
|
|
1749
|
+
return {
|
|
1750
|
+
id: workerId,
|
|
1751
|
+
worker: definition.name,
|
|
1752
|
+
ownerSessionId,
|
|
1753
|
+
runId,
|
|
1754
|
+
title: task.title,
|
|
1755
|
+
instructions: task.instructions,
|
|
1756
|
+
lifecycle: definition.lifecycle,
|
|
1757
|
+
status: "starting",
|
|
1758
|
+
usage: copyUsage(EMPTY_WORKER_USAGE),
|
|
1759
|
+
messageDirection: "to-model",
|
|
1760
|
+
startedAt,
|
|
1761
|
+
};
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
function publishState(ownerSessionId: string): PostCommitAction {
|
|
1765
|
+
return (state, _settlementListeners, stateListeners) => {
|
|
1766
|
+
const snapshot = snapshotFor(state, ownerSessionId);
|
|
1767
|
+
const listeners = [...(stateListeners.get(ownerSessionId) ?? [])];
|
|
1768
|
+
return () => {
|
|
1769
|
+
for (const listener of listeners) safelyNotify(() => listener(snapshot));
|
|
1770
|
+
};
|
|
1771
|
+
};
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
function publishStateTo(
|
|
1775
|
+
ownerSessionId: string,
|
|
1776
|
+
listener: StateListener,
|
|
1777
|
+
): PostCommitAction {
|
|
1778
|
+
return (state) => {
|
|
1779
|
+
const snapshot = snapshotFor(state, ownerSessionId);
|
|
1780
|
+
return () => safelyNotify(() => listener(snapshot));
|
|
1781
|
+
};
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
function publishSettlement(
|
|
1785
|
+
settlement: WorkerSettlement,
|
|
1786
|
+
localListener: SettlementListener | undefined,
|
|
1787
|
+
): PostCommitAction {
|
|
1788
|
+
return (_state, settlementListeners) => {
|
|
1789
|
+
const listeners = [...settlementListeners];
|
|
1790
|
+
return () => {
|
|
1791
|
+
if (localListener) safelyNotify(() => localListener(settlement));
|
|
1792
|
+
for (const listener of listeners) safelyNotify(() => listener(settlement));
|
|
1793
|
+
};
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
function publishOwners(owners: ReadonlySet<string>): PostCommitAction[] {
|
|
1798
|
+
return [...owners].map(publishState);
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
function runAction(run: () => void): PostCommitAction {
|
|
1802
|
+
return () => run;
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
function openDecision(
|
|
1806
|
+
draft: RuntimeState,
|
|
1807
|
+
operation: OrchestrationOperation,
|
|
1808
|
+
): Decision<void> {
|
|
1809
|
+
return draft.lifecycle === "open"
|
|
1810
|
+
? accepted(undefined)
|
|
1811
|
+
: rejected(
|
|
1812
|
+
operation,
|
|
1813
|
+
"shutdown",
|
|
1814
|
+
"Orchestrator runtime is shutting down",
|
|
1815
|
+
);
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
function ownedWorkerDecision(
|
|
1819
|
+
draft: RuntimeState,
|
|
1820
|
+
operation: OrchestrationOperation,
|
|
1821
|
+
ownerSessionId: string,
|
|
1822
|
+
workerId: WorkerId,
|
|
1823
|
+
): Decision<RuntimeWorker> {
|
|
1824
|
+
const worker = draft.workers.get(workerId);
|
|
1825
|
+
return !worker || worker.record.ownerSessionId !== ownerSessionId
|
|
1826
|
+
? rejected(
|
|
1827
|
+
operation,
|
|
1828
|
+
"ownership",
|
|
1829
|
+
"Worker is not owned by this session",
|
|
1830
|
+
)
|
|
1831
|
+
: accepted(worker);
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
function readyInteractiveDecision(
|
|
1835
|
+
draft: RuntimeState,
|
|
1836
|
+
ownerSessionId: string,
|
|
1837
|
+
workerId: WorkerId,
|
|
1838
|
+
): Decision<{
|
|
1839
|
+
readonly worker: RuntimeWorker;
|
|
1840
|
+
readonly session: WorkerSessionHandle;
|
|
1841
|
+
}> {
|
|
1842
|
+
const open = openDecision(draft, "sendInteractive");
|
|
1843
|
+
if (open._tag === "rejected") return open;
|
|
1844
|
+
const ownership = ownedWorkerDecision(
|
|
1845
|
+
draft,
|
|
1846
|
+
"sendInteractive",
|
|
1847
|
+
ownerSessionId,
|
|
1848
|
+
workerId,
|
|
1849
|
+
);
|
|
1850
|
+
if (ownership._tag === "rejected") return ownership;
|
|
1851
|
+
const worker = ownership.value;
|
|
1852
|
+
if (
|
|
1853
|
+
worker.record.lifecycle !== "interactive" ||
|
|
1854
|
+
worker.record.status !== "ready" ||
|
|
1855
|
+
!worker.session
|
|
1856
|
+
) {
|
|
1857
|
+
return rejected(
|
|
1858
|
+
"sendInteractive",
|
|
1859
|
+
"worker-state",
|
|
1860
|
+
"interactive_send requires an owned ready interactive worker",
|
|
1188
1861
|
);
|
|
1189
1862
|
}
|
|
1863
|
+
return accepted({ worker, session: worker.session });
|
|
1864
|
+
}
|
|
1190
1865
|
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
} catch {
|
|
1195
|
-
// A deadline implementation cannot prevent lifecycle settlement.
|
|
1196
|
-
}
|
|
1197
|
-
}
|
|
1866
|
+
function accepted<A>(value: A): Decision<A> {
|
|
1867
|
+
return { _tag: "accepted", value };
|
|
1868
|
+
}
|
|
1198
1869
|
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1870
|
+
function rejected(
|
|
1871
|
+
operation: OrchestrationOperation,
|
|
1872
|
+
reason: OrchestrationRejectionReason,
|
|
1873
|
+
message: string,
|
|
1874
|
+
): Decision<never> {
|
|
1875
|
+
return {
|
|
1876
|
+
_tag: "rejected",
|
|
1877
|
+
error: actionRejection(operation, reason, message),
|
|
1878
|
+
};
|
|
1879
|
+
}
|
|
1880
|
+
|
|
1881
|
+
function actionRejection(
|
|
1882
|
+
operation: OrchestrationOperation,
|
|
1883
|
+
reason: OrchestrationRejectionReason,
|
|
1884
|
+
message: string,
|
|
1885
|
+
): OrchestrationActionRejected {
|
|
1886
|
+
return new OrchestrationActionRejected({ operation, reason, message });
|
|
1202
1887
|
}
|
|
1203
1888
|
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1889
|
+
function rejectAction(
|
|
1890
|
+
operation: OrchestrationOperation,
|
|
1891
|
+
reason: OrchestrationRejectionReason,
|
|
1892
|
+
message: string,
|
|
1893
|
+
): Effect.Effect<never, OrchestrationActionRejected> {
|
|
1894
|
+
return Effect.fail(actionRejection(operation, reason, message));
|
|
1208
1895
|
}
|
|
1209
1896
|
|
|
1210
|
-
function validateContextOwner(
|
|
1211
|
-
|
|
1212
|
-
|
|
1897
|
+
function validateContextOwner(
|
|
1898
|
+
operation: OrchestrationOperation,
|
|
1899
|
+
ownerSessionId: string,
|
|
1900
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1901
|
+
return typeof ownerSessionId !== "string" || ownerSessionId.trim() === ""
|
|
1902
|
+
? rejectAction(
|
|
1903
|
+
operation,
|
|
1904
|
+
"validation",
|
|
1905
|
+
"ownerSessionId must not be blank",
|
|
1906
|
+
)
|
|
1907
|
+
: Effect.void;
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
function validateWorkerId(
|
|
1911
|
+
operation: OrchestrationOperation,
|
|
1912
|
+
workerId: string,
|
|
1913
|
+
): Effect.Effect<WorkerId, OrchestrationActionRejected> {
|
|
1914
|
+
if (typeof workerId !== "string" || workerId.trim() === "") {
|
|
1915
|
+
return rejectAction(
|
|
1916
|
+
operation,
|
|
1917
|
+
"validation",
|
|
1918
|
+
"worker_id must not be blank",
|
|
1919
|
+
);
|
|
1213
1920
|
}
|
|
1921
|
+
return Schema.decodeUnknownEffect(WorkerId)(workerId).pipe(
|
|
1922
|
+
Effect.mapError(() => actionRejection(
|
|
1923
|
+
operation,
|
|
1924
|
+
"validation",
|
|
1925
|
+
"worker_id must use the canonical worker- prefix",
|
|
1926
|
+
)),
|
|
1927
|
+
);
|
|
1214
1928
|
}
|
|
1215
1929
|
|
|
1216
|
-
function validateMode(
|
|
1217
|
-
|
|
1930
|
+
function validateMode(
|
|
1931
|
+
operation: OrchestrationOperation,
|
|
1932
|
+
mode: RunMode,
|
|
1933
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1934
|
+
return mode !== "async" && mode !== "inline"
|
|
1935
|
+
? rejectAction(operation, "validation", "Invalid orchestration mode")
|
|
1936
|
+
: Effect.void;
|
|
1218
1937
|
}
|
|
1219
1938
|
|
|
1220
|
-
function validateText(
|
|
1939
|
+
function validateText(
|
|
1940
|
+
operation: OrchestrationOperation,
|
|
1941
|
+
name: string,
|
|
1942
|
+
value: string,
|
|
1943
|
+
maximumLength: number,
|
|
1944
|
+
): Effect.Effect<void, OrchestrationActionRejected> {
|
|
1221
1945
|
if (typeof value !== "string" || value.trim() === "") {
|
|
1222
|
-
|
|
1223
|
-
}
|
|
1224
|
-
if (value.length > maximumLength) {
|
|
1225
|
-
throw new Error(`${name} must be at most ${maximumLength} characters`);
|
|
1946
|
+
return rejectAction(operation, "validation", `${name} must not be blank`);
|
|
1226
1947
|
}
|
|
1948
|
+
return value.length > maximumLength
|
|
1949
|
+
? rejectAction(
|
|
1950
|
+
operation,
|
|
1951
|
+
"validation",
|
|
1952
|
+
`${name} must be at most ${maximumLength} characters`,
|
|
1953
|
+
)
|
|
1954
|
+
: Effect.void;
|
|
1227
1955
|
}
|
|
1228
1956
|
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1957
|
+
type ValidatedAbortTarget =
|
|
1958
|
+
| {
|
|
1959
|
+
readonly _tag: "ids";
|
|
1960
|
+
readonly workerIds: readonly WorkerId[];
|
|
1961
|
+
}
|
|
1962
|
+
| {
|
|
1963
|
+
readonly _tag: "all";
|
|
1964
|
+
};
|
|
1965
|
+
|
|
1966
|
+
function validateAbortTarget(
|
|
1967
|
+
target: AbortTarget,
|
|
1968
|
+
): Effect.Effect<ValidatedAbortTarget, OrchestrationActionRejected> {
|
|
1969
|
+
return Effect.gen(function* () {
|
|
1970
|
+
if (!target || typeof target !== "object") {
|
|
1971
|
+
return yield* rejectAction("abort", "target", "Invalid abort target");
|
|
1972
|
+
}
|
|
1973
|
+
const selected = [target.workerIds !== undefined, target.all !== undefined]
|
|
1974
|
+
.filter(Boolean).length;
|
|
1975
|
+
if (selected !== 1 || (target.all !== undefined && target.all !== true)) {
|
|
1976
|
+
return yield* rejectAction(
|
|
1977
|
+
"abort",
|
|
1978
|
+
"target",
|
|
1979
|
+
"Abort target must specify exactly one of workerIds or all: true",
|
|
1980
|
+
);
|
|
1981
|
+
}
|
|
1982
|
+
if (target.workerIds === undefined) return { _tag: "all" };
|
|
1983
|
+
if (!Array.isArray(target.workerIds) || target.workerIds.length === 0) {
|
|
1984
|
+
return yield* rejectAction(
|
|
1985
|
+
"abort",
|
|
1986
|
+
"target",
|
|
1987
|
+
"workerIds must contain at least one worker ID",
|
|
1988
|
+
);
|
|
1989
|
+
}
|
|
1990
|
+
const workerIds = yield* Effect.all(
|
|
1991
|
+
[...new Set(target.workerIds)].map((id) => validateWorkerId("abort", id)),
|
|
1992
|
+
);
|
|
1993
|
+
return { _tag: "ids", workerIds };
|
|
1994
|
+
});
|
|
1233
1995
|
}
|
|
1234
1996
|
|
|
1235
|
-
function
|
|
1236
|
-
|
|
1997
|
+
function makeCancellationCandidates(
|
|
1998
|
+
workerIds: readonly WorkerId[],
|
|
1999
|
+
): Effect.Effect<ReadonlyMap<WorkerId, Deferred.Deferred<void>>> {
|
|
2000
|
+
return Effect.forEach(workerIds, (workerId) => Deferred.make<void>().pipe(
|
|
2001
|
+
Effect.map((completion) => [workerId, completion] as const),
|
|
2002
|
+
)).pipe(Effect.map((entries) => new Map(entries)));
|
|
1237
2003
|
}
|
|
1238
2004
|
|
|
1239
|
-
function
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
2005
|
+
function activeWorkerIds(
|
|
2006
|
+
state: RuntimeState,
|
|
2007
|
+
ownerSessionId?: string,
|
|
2008
|
+
): WorkerId[] {
|
|
2009
|
+
return [...state.workers.values()]
|
|
2010
|
+
.filter((worker) => (
|
|
2011
|
+
(ownerSessionId === undefined ||
|
|
2012
|
+
worker.record.ownerSessionId === ownerSessionId) &&
|
|
2013
|
+
isActiveWorkerStatus(worker.record.status)
|
|
2014
|
+
))
|
|
2015
|
+
.map((worker) => worker.record.id);
|
|
1243
2016
|
}
|
|
1244
2017
|
|
|
1245
|
-
function
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
2018
|
+
function awaitAll(
|
|
2019
|
+
deferreds: readonly Deferred.Deferred<void>[],
|
|
2020
|
+
): Effect.Effect<void> {
|
|
2021
|
+
return Effect.forEach(
|
|
2022
|
+
deferreds,
|
|
2023
|
+
(deferred) => Deferred.await(deferred),
|
|
2024
|
+
{ concurrency: "unbounded", discard: true },
|
|
2025
|
+
);
|
|
1252
2026
|
}
|
|
1253
2027
|
|
|
1254
|
-
function
|
|
1255
|
-
|
|
2028
|
+
function isRuntimeWorker(
|
|
2029
|
+
worker: RuntimeWorker | undefined,
|
|
2030
|
+
): worker is RuntimeWorker {
|
|
2031
|
+
return worker !== undefined;
|
|
1256
2032
|
}
|
|
1257
2033
|
|
|
1258
|
-
function
|
|
1259
|
-
|
|
1260
|
-
return new DOMException("This operation was aborted", "AbortError");
|
|
2034
|
+
function isActiveWorkerStatus(status: WorkerRecord["status"]): boolean {
|
|
2035
|
+
return status === "starting" || status === "running" || status === "stopping";
|
|
1261
2036
|
}
|
|
1262
2037
|
|
|
1263
|
-
function
|
|
1264
|
-
|
|
2038
|
+
function isCompletedRunWorker(
|
|
2039
|
+
record: WorkerRecord,
|
|
2040
|
+
): record is WorkerRecord & {
|
|
2041
|
+
readonly status: RunResult["status"];
|
|
2042
|
+
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2043
|
+
readonly settledAt: number;
|
|
2044
|
+
} {
|
|
2045
|
+
return (
|
|
2046
|
+
(record.status === "completed" ||
|
|
2047
|
+
record.status === "ready" ||
|
|
2048
|
+
record.status === "failed" ||
|
|
2049
|
+
record.status === "aborted") &&
|
|
2050
|
+
record.outcome !== undefined &&
|
|
2051
|
+
record.outcome.status !== "closed" &&
|
|
2052
|
+
record.settledAt !== undefined
|
|
2053
|
+
);
|
|
1265
2054
|
}
|
|
1266
2055
|
|
|
1267
|
-
function
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
):
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
}
|
|
2056
|
+
function observationsEqual(
|
|
2057
|
+
previous: WorkerRecord,
|
|
2058
|
+
next: WorkerRecord,
|
|
2059
|
+
): boolean {
|
|
2060
|
+
return (
|
|
2061
|
+
previous.activity === next.activity &&
|
|
2062
|
+
previous.messageDirection === next.messageDirection &&
|
|
2063
|
+
usageEquals(previous.usage, next.usage)
|
|
2064
|
+
);
|
|
1277
2065
|
}
|
|
1278
2066
|
|
|
1279
|
-
function
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
waiter.onSettled?.();
|
|
1290
|
-
complete(run);
|
|
1291
|
-
},
|
|
1292
|
-
};
|
|
1293
|
-
return waiter;
|
|
2067
|
+
function usageEquals(left: WorkerUsage, right: WorkerUsage): boolean {
|
|
2068
|
+
return (
|
|
2069
|
+
left.input === right.input &&
|
|
2070
|
+
left.output === right.output &&
|
|
2071
|
+
left.cacheRead === right.cacheRead &&
|
|
2072
|
+
left.cacheWrite === right.cacheWrite &&
|
|
2073
|
+
left.cost === right.cost &&
|
|
2074
|
+
left.contextTokens === right.contextTokens &&
|
|
2075
|
+
left.turns === right.turns
|
|
2076
|
+
);
|
|
1294
2077
|
}
|
|
1295
2078
|
|
|
1296
2079
|
function copyUsage(usage: WorkerUsage): WorkerUsage {
|
|
@@ -1317,7 +2100,9 @@ function copyWorkerRecord(worker: WorkerRecord): WorkerRecord {
|
|
|
1317
2100
|
return Object.freeze({
|
|
1318
2101
|
...worker,
|
|
1319
2102
|
usage: Object.freeze(copyUsage(worker.usage)),
|
|
1320
|
-
...(worker.outcome
|
|
2103
|
+
...(worker.outcome
|
|
2104
|
+
? { outcome: Object.freeze(copyOutcome(worker.outcome)) }
|
|
2105
|
+
: {}),
|
|
1321
2106
|
});
|
|
1322
2107
|
}
|
|
1323
2108
|
|
|
@@ -1327,21 +2112,18 @@ function freezeAcceptedRun(id: RunId, workerId: WorkerId): AcceptedRun {
|
|
|
1327
2112
|
|
|
1328
2113
|
function freezeCompletedRun(
|
|
1329
2114
|
run: RunRecord,
|
|
1330
|
-
record: WorkerRecord
|
|
2115
|
+
record: WorkerRecord & {
|
|
2116
|
+
readonly status: RunResult["status"];
|
|
2117
|
+
readonly outcome: Exclude<WorkerOutcome, { readonly status: "closed" }>;
|
|
2118
|
+
readonly settledAt: number;
|
|
2119
|
+
},
|
|
1331
2120
|
): CompletedRun {
|
|
1332
|
-
const outcome = record.outcome;
|
|
1333
|
-
if (!outcome || outcome.status === "closed") {
|
|
1334
|
-
throw new Error("Completed run requires a worker response outcome");
|
|
1335
|
-
}
|
|
1336
|
-
if (record.settledAt === undefined) {
|
|
1337
|
-
throw new Error("Completed run requires a settlement timestamp");
|
|
1338
|
-
}
|
|
1339
2121
|
const result: RunResult = Object.freeze({
|
|
1340
2122
|
workerId: record.id,
|
|
1341
2123
|
worker: record.worker,
|
|
1342
2124
|
title: record.title,
|
|
1343
|
-
status: record.status
|
|
1344
|
-
outcome: Object.freeze(copyOutcome(outcome)),
|
|
2125
|
+
status: record.status,
|
|
2126
|
+
outcome: Object.freeze(copyOutcome(record.outcome)),
|
|
1345
2127
|
usage: Object.freeze(copyUsage(record.usage)),
|
|
1346
2128
|
startedAt: record.startedAt,
|
|
1347
2129
|
settledAt: record.settledAt,
|
|
@@ -1354,3 +2136,27 @@ function freezeCompletedRun(
|
|
|
1354
2136
|
result,
|
|
1355
2137
|
});
|
|
1356
2138
|
}
|
|
2139
|
+
|
|
2140
|
+
function describeError(error: unknown, fallback: string): string {
|
|
2141
|
+
if (error instanceof Error && error.message !== "") return error.message;
|
|
2142
|
+
if (typeof error === "string" && error !== "") return error;
|
|
2143
|
+
return fallback;
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
function safelyCall(callback: (() => void) | undefined): void {
|
|
2147
|
+
if (callback) safelyNotify(callback);
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
function safelyNotify(callback: () => void): void {
|
|
2151
|
+
try {
|
|
2152
|
+
callback();
|
|
2153
|
+
} catch {
|
|
2154
|
+
// Observers and best-effort cleanup cannot block committed state transitions.
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
function addAll(target: Set<string>, source: ReadonlySet<string>): void {
|
|
2159
|
+
for (const value of source) target.add(value);
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
function noOp(): void {}
|