@agentswf/engine 0.0.0-stage → 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +39 -4
- package/src/accounting/format.ts +252 -0
- package/src/accounting/plans.ts +38 -0
- package/src/accounting/prices.ts +98 -0
- package/src/accounting/summary.ts +385 -0
- package/src/accounting/tokens.ts +34 -0
- package/src/agent-launcher.ts +127 -0
- package/src/allowance-command.ts +105 -0
- package/src/attempt-ending.ts +198 -0
- package/src/attempt-output.ts +132 -0
- package/src/attempt-view.ts +362 -0
- package/src/canonical-json.ts +12 -0
- package/src/control-plane.ts +424 -0
- package/src/deadlines.ts +106 -0
- package/src/decisions/directory.ts +432 -0
- package/src/decisions/fake.ts +60 -0
- package/src/decisions/openrouter.ts +161 -0
- package/src/decisions/seam.ts +70 -0
- package/src/display-path.ts +16 -0
- package/src/duration.ts +12 -0
- package/src/errors.ts +4 -0
- package/src/files.ts +78 -0
- package/src/herdr-placement.ts +182 -0
- package/src/herdr-run-session.ts +530 -0
- package/src/herdr-workspace-marks.ts +194 -0
- package/src/here.ts +289 -0
- package/src/index.ts +5 -0
- package/src/jsonl.ts +74 -0
- package/src/logs-command.ts +605 -0
- package/src/machine.ts +27 -0
- package/src/message-operation.ts +245 -0
- package/src/messaging/limits.ts +35 -0
- package/src/messaging/messaging.ts +1303 -0
- package/src/messaging/routes.ts +79 -0
- package/src/messaging/text.ts +251 -0
- package/src/operation-events.ts +182 -0
- package/src/operation-liveness.ts +674 -0
- package/src/operator-aliases.ts +10 -0
- package/src/operator-cli.ts +569 -0
- package/src/operator-runtime.ts +503 -0
- package/src/pane-layout.ts +125 -0
- package/src/progress-view.ts +568 -0
- package/src/result-slots.ts +325 -0
- package/src/result-validation.ts +47 -0
- package/src/run-command.ts +241 -0
- package/src/run-commands.ts +215 -0
- package/src/run-event-log.ts +72 -0
- package/src/run-prepare.ts +178 -0
- package/src/run-progress.ts +137 -0
- package/src/run-read.ts +425 -0
- package/src/run-state.ts +120 -0
- package/src/run-usage.ts +393 -0
- package/src/run-view.ts +225 -0
- package/src/runs-command.ts +356 -0
- package/src/runs.ts +645 -0
- package/src/sandbox-homes.ts +211 -0
- package/src/sandboxes.ts +516 -0
- package/src/skills/fetch.ts +305 -0
- package/src/skills/run-skills.ts +181 -0
- package/src/skills/sources.ts +87 -0
- package/src/skills/tree.ts +60 -0
- package/src/stage-ledger.ts +535 -0
- package/src/stage-plan.ts +127 -0
- package/src/stage-schema.ts +88 -0
- package/src/stopped.ts +36 -0
- package/src/test-command.ts +110 -0
- package/src/testing/operation-events.ts +89 -0
- package/src/testing.ts +164 -0
- package/src/workflow-loader.ts +113 -0
- package/src/workflow-runner.ts +3665 -0
- package/src/workflow-test-preload.ts +13 -0
- package/src/workflow-testing/decisions.ts +129 -0
- package/src/workflow-testing/host.ts +811 -0
- package/src/workflow-testing/index.ts +420 -0
- package/src/workflow-testing/script.ts +447 -0
- package/src/workflow-testing/workflow-testing.typecheck.ts +47 -0
- package/README.md +0 -3
|
@@ -0,0 +1,3665 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { mkdir, realpath, rm } from "node:fs/promises";
|
|
4
|
+
import { dirname, join, relative } from "node:path";
|
|
5
|
+
import { isDeepStrictEqual } from "node:util";
|
|
6
|
+
import { PARTICIPANT } from "@agentswf/contract/participant";
|
|
7
|
+
import type {
|
|
8
|
+
AgentPaneRecord,
|
|
9
|
+
AgentSkillsRecord,
|
|
10
|
+
RunAccounting,
|
|
11
|
+
SandboxRecord,
|
|
12
|
+
SettledDecision,
|
|
13
|
+
SettledOperation,
|
|
14
|
+
} from "@agentswf/contract/records";
|
|
15
|
+
import {
|
|
16
|
+
TURN_OPERATION_FIELDS,
|
|
17
|
+
TURN_RECORD_VERSION,
|
|
18
|
+
type TurnRecord,
|
|
19
|
+
} from "@agentswf/contract/records";
|
|
20
|
+
import {
|
|
21
|
+
formatErrors,
|
|
22
|
+
type JsonSchema,
|
|
23
|
+
parseJsonSchema,
|
|
24
|
+
validate,
|
|
25
|
+
} from "@agentswf/contract/schema";
|
|
26
|
+
import {
|
|
27
|
+
type AbsoluteDeadline,
|
|
28
|
+
type AgentExecution,
|
|
29
|
+
type AgentForkSpec,
|
|
30
|
+
type AgentKey,
|
|
31
|
+
type AgentOpenSpec,
|
|
32
|
+
type AgentRef,
|
|
33
|
+
type AgentRunStructuredSpec,
|
|
34
|
+
type AgentRunTextSpec,
|
|
35
|
+
type AgentStructuredTurnSpec,
|
|
36
|
+
type AgentTextTurnSpec,
|
|
37
|
+
type CompactionId,
|
|
38
|
+
type CompactSpec,
|
|
39
|
+
DeadlineExceededError,
|
|
40
|
+
isJsonValue,
|
|
41
|
+
type JsonObject,
|
|
42
|
+
type JsonValue,
|
|
43
|
+
type KeepPane,
|
|
44
|
+
type MessageAccess,
|
|
45
|
+
type OperationRecord,
|
|
46
|
+
type OutputSchema,
|
|
47
|
+
type PaneLayout,
|
|
48
|
+
type PlacementChoice,
|
|
49
|
+
placementOf,
|
|
50
|
+
type RunResult,
|
|
51
|
+
type RuntimeSelection,
|
|
52
|
+
type SettingsSpec,
|
|
53
|
+
type SkillSource,
|
|
54
|
+
type StageOptions,
|
|
55
|
+
type StageSummary,
|
|
56
|
+
type TurnOutcome,
|
|
57
|
+
type WorkflowContext,
|
|
58
|
+
type WorkflowDefinition,
|
|
59
|
+
} from "@agentswf/contract/workflow";
|
|
60
|
+
import {
|
|
61
|
+
type AgentSkills,
|
|
62
|
+
effortRefusal,
|
|
63
|
+
findHarness,
|
|
64
|
+
hostHome,
|
|
65
|
+
settingsRefusal,
|
|
66
|
+
skillsLayout,
|
|
67
|
+
} from "@agentswf/harness";
|
|
68
|
+
import type {
|
|
69
|
+
AgentRunHost,
|
|
70
|
+
AgentRuntimeConfig,
|
|
71
|
+
AuthoredMessage,
|
|
72
|
+
AuthoredTurn,
|
|
73
|
+
HarnessActivation,
|
|
74
|
+
HarnessReleaseDisposition,
|
|
75
|
+
HarnessSession,
|
|
76
|
+
HarnessTurn,
|
|
77
|
+
HarnessTurnOutcome,
|
|
78
|
+
NativeFork,
|
|
79
|
+
SessionCopy,
|
|
80
|
+
SessionSettings,
|
|
81
|
+
} from "@agentswf/harness/adapter";
|
|
82
|
+
import type { Occupant } from "@agentswf/sandbox";
|
|
83
|
+
import { PUBLISHED_PRICES } from "./accounting/prices";
|
|
84
|
+
import { summarizeRun } from "./accounting/summary";
|
|
85
|
+
import { buildAgentBundle, installAgentLauncher, installSandboxedDoor } from "./agent-launcher";
|
|
86
|
+
import {
|
|
87
|
+
type ResultChannel,
|
|
88
|
+
type ResultControlPlane,
|
|
89
|
+
startResultControlPlane,
|
|
90
|
+
} from "./control-plane";
|
|
91
|
+
import {
|
|
92
|
+
assertDeadline,
|
|
93
|
+
assertDeadlineValue,
|
|
94
|
+
deadlineWithin,
|
|
95
|
+
earlierDeadline,
|
|
96
|
+
runUntilStopped,
|
|
97
|
+
scheduleAt,
|
|
98
|
+
WorkflowCancelledError,
|
|
99
|
+
waitForDeadline,
|
|
100
|
+
} from "./deadlines";
|
|
101
|
+
import { RunDecisions } from "./decisions/directory";
|
|
102
|
+
import type { DecisionInstallation } from "./decisions/seam";
|
|
103
|
+
import { messageOf } from "./errors";
|
|
104
|
+
import { superviseMessageOperation } from "./message-operation";
|
|
105
|
+
import { canAwaitReply } from "./messaging/limits";
|
|
106
|
+
import { type AgentView, type Delivery, Messaging, type OperationEnd } from "./messaging/messaging";
|
|
107
|
+
import type { Scope as MessagingScope } from "./messaging/routes";
|
|
108
|
+
import { createOperationEvents } from "./operation-events";
|
|
109
|
+
import {
|
|
110
|
+
type LivenessEvent,
|
|
111
|
+
type LivenessPolicy,
|
|
112
|
+
type OperationMessages,
|
|
113
|
+
type OperationStop,
|
|
114
|
+
type SupervisedOutcome,
|
|
115
|
+
superviseOperation,
|
|
116
|
+
} from "./operation-liveness";
|
|
117
|
+
import { besideOf, checkPaneOptions, keepsPane, storedLayout } from "./pane-layout";
|
|
118
|
+
import { createResultSlotRegistry, type ResultSlotRegistry } from "./result-slots";
|
|
119
|
+
import { RunEventLog } from "./run-event-log";
|
|
120
|
+
import {
|
|
121
|
+
type AgentProgress,
|
|
122
|
+
type GroupProgress,
|
|
123
|
+
type MessageProgress,
|
|
124
|
+
RunProgress,
|
|
125
|
+
} from "./run-progress";
|
|
126
|
+
import {
|
|
127
|
+
type AccountedAgent,
|
|
128
|
+
type AgentLedger,
|
|
129
|
+
createRunLedger,
|
|
130
|
+
type OperationTags,
|
|
131
|
+
type RunLedger,
|
|
132
|
+
} from "./run-usage";
|
|
133
|
+
import { endTurnsLine, readStageRecords } from "./runs";
|
|
134
|
+
import { type CredentialLocks, placeCarried, seedHome } from "./sandbox-homes";
|
|
135
|
+
import {
|
|
136
|
+
RUN_SANDBOX_ONLY,
|
|
137
|
+
RunSandboxes,
|
|
138
|
+
type RunSandboxOptions,
|
|
139
|
+
type SeatedAgent,
|
|
140
|
+
} from "./sandboxes";
|
|
141
|
+
import { placeSkills, RunSkills, readSkillSources } from "./skills/run-skills";
|
|
142
|
+
import { StageLedger, type StageProgress } from "./stage-ledger";
|
|
143
|
+
import { FromStageUnreached } from "./stopped";
|
|
144
|
+
|
|
145
|
+
export { WorkflowCancelledError } from "./deadlines";
|
|
146
|
+
|
|
147
|
+
export type RunWorkflowOptions = {
|
|
148
|
+
/** Internal timing policy; workflow authors use the operation deadline. */
|
|
149
|
+
livenessPolicy?: LivenessPolicy;
|
|
150
|
+
runRoot: string;
|
|
151
|
+
/** The run this is an attempt of, whose folder the operator claimed under `runRoot`. */
|
|
152
|
+
run: {
|
|
153
|
+
dir: string;
|
|
154
|
+
id: string;
|
|
155
|
+
attempt: number;
|
|
156
|
+
/** How the attempt is named to a person: its Herdr workspace's label. */
|
|
157
|
+
label?: string;
|
|
158
|
+
/** `--from-stage`: the stage this attempt starts at, reusing those before it. */
|
|
159
|
+
fromStage?: string;
|
|
160
|
+
/** `--values`: the values of stages before `fromStage` that have no record to reuse. */
|
|
161
|
+
values?: ReadonlyMap<string, JsonValue>;
|
|
162
|
+
};
|
|
163
|
+
runtime: AgentRuntimeConfig;
|
|
164
|
+
deadline: AbsoluteDeadline;
|
|
165
|
+
cwd?: string;
|
|
166
|
+
onLog?: (message: string, fields?: JsonObject) => void;
|
|
167
|
+
/** The operator hands the session back once agents.stop has released it. */
|
|
168
|
+
onCallerStopped?: (reason: string) => Promise<void>;
|
|
169
|
+
signal?: AbortSignal;
|
|
170
|
+
/** The sandbox providers agents may run in; without them, a sandboxed agent is refused. */
|
|
171
|
+
sandboxes?: RunSandboxOptions;
|
|
172
|
+
/** Where public skills are fetched to, shared by runs; `$XDG_CACHE_HOME/awf/skills` by default. */
|
|
173
|
+
skillCache?: string;
|
|
174
|
+
/** The decision models workflows may ask; without them, `decisions.decide` is refused. */
|
|
175
|
+
decisions?: DecisionInstallation;
|
|
176
|
+
/**
|
|
177
|
+
* Where an agent may `--await-reply`: the engine's list by harness and placement unless given. A
|
|
178
|
+
* workflow's test allows it everywhere, as the fake harness can.
|
|
179
|
+
*/
|
|
180
|
+
awaitReply?: (execution: AgentExecution, confirmsDelivery: boolean) => boolean;
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
/** What a run is known by once it has ended, whether or not it succeeded. */
|
|
184
|
+
export type SettledRun = {
|
|
185
|
+
runId: string;
|
|
186
|
+
/** The stages entered, in order, each run or reused; absent when none was. */
|
|
187
|
+
stages?: StageSummary[];
|
|
188
|
+
/** The stage a run that didn't complete ended in; absent between stages. */
|
|
189
|
+
endedIn?: string;
|
|
190
|
+
/** Every operation's record, completed with the spend read when the run ended. */
|
|
191
|
+
usage: SettledOperation[];
|
|
192
|
+
/** ISO times the run started and its own work, cleanup included, ended. */
|
|
193
|
+
startedAt: string;
|
|
194
|
+
finishedAt: string;
|
|
195
|
+
/**
|
|
196
|
+
* Derived from `usage` and the two times, which are what a record keeps; `summarizeRun` prices
|
|
197
|
+
* them again with another table.
|
|
198
|
+
*/
|
|
199
|
+
accounting: RunAccounting;
|
|
200
|
+
/** Each sandbox the run opened, with its agents; absent when it opened none. */
|
|
201
|
+
sandboxes?: SandboxRecord[];
|
|
202
|
+
/** Each agent's skills; absent when no agent opened. */
|
|
203
|
+
skills?: AgentSkillsRecord[];
|
|
204
|
+
/** Where each pane agent's pane went; absent when the run placed none. */
|
|
205
|
+
panes?: AgentPaneRecord[];
|
|
206
|
+
/** Every decision the run asked, in the order asked; absent when it asked none. */
|
|
207
|
+
decisions?: SettledDecision[];
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
export type WorkflowRunResult<Result extends JsonValue> = SettledRun & { value: Result };
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* A run that failed after its host opened, carrying what it spent. `cause` is the run's own error:
|
|
214
|
+
* the body's, or an `AggregateError` with the cleanup's.
|
|
215
|
+
*/
|
|
216
|
+
export class WorkflowRunError extends Error implements SettledRun {
|
|
217
|
+
readonly runId: string;
|
|
218
|
+
readonly stages?: StageSummary[];
|
|
219
|
+
readonly endedIn?: string;
|
|
220
|
+
readonly usage: SettledOperation[];
|
|
221
|
+
readonly startedAt: string;
|
|
222
|
+
readonly finishedAt: string;
|
|
223
|
+
readonly accounting: RunAccounting;
|
|
224
|
+
readonly sandboxes?: SandboxRecord[];
|
|
225
|
+
readonly skills?: AgentSkillsRecord[];
|
|
226
|
+
readonly panes?: AgentPaneRecord[];
|
|
227
|
+
readonly decisions?: SettledDecision[];
|
|
228
|
+
|
|
229
|
+
constructor(cause: unknown, run: SettledRun) {
|
|
230
|
+
super(messageOf(cause), { cause });
|
|
231
|
+
this.name = "WorkflowRunError";
|
|
232
|
+
this.runId = run.runId;
|
|
233
|
+
if (run.stages) this.stages = run.stages;
|
|
234
|
+
if (run.endedIn !== undefined) this.endedIn = run.endedIn;
|
|
235
|
+
this.usage = run.usage;
|
|
236
|
+
this.startedAt = run.startedAt;
|
|
237
|
+
this.finishedAt = run.finishedAt;
|
|
238
|
+
this.accounting = run.accounting;
|
|
239
|
+
if (run.sandboxes) this.sandboxes = run.sandboxes;
|
|
240
|
+
if (run.skills) this.skills = run.skills;
|
|
241
|
+
if (run.panes) this.panes = run.panes;
|
|
242
|
+
if (run.decisions) this.decisions = run.decisions;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export type WorkflowRunHandle<Result extends JsonValue> = {
|
|
247
|
+
runId: string;
|
|
248
|
+
result: Promise<WorkflowRunResult<Result>>;
|
|
249
|
+
inspect(): WorkflowRunSnapshot;
|
|
250
|
+
stop(reason?: unknown): Promise<void>;
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
export type WorkflowRunSnapshot = {
|
|
254
|
+
state: "starting" | "running" | "closing" | "closed";
|
|
255
|
+
/** Labelled `parallel` calls, in the order they began. */
|
|
256
|
+
groups: readonly GroupProgress[];
|
|
257
|
+
/** The workflow stages this attempt entered, in order. */
|
|
258
|
+
stages: readonly StageProgress[];
|
|
259
|
+
/** Stages an earlier attempt recorded that this one hasn't entered yet. */
|
|
260
|
+
upcoming: readonly string[];
|
|
261
|
+
/** Messages admitted this attempt, in order. */
|
|
262
|
+
messages: readonly MessageProgress[];
|
|
263
|
+
agents: readonly (Partial<AgentProgress> & {
|
|
264
|
+
key: AgentKey;
|
|
265
|
+
/** Messages held for it and not yet shown; absent when none are. */
|
|
266
|
+
queued?: number;
|
|
267
|
+
execution: AgentExecution;
|
|
268
|
+
state:
|
|
269
|
+
| "starting"
|
|
270
|
+
| "idle"
|
|
271
|
+
| "working"
|
|
272
|
+
| "blocked"
|
|
273
|
+
| "dormant"
|
|
274
|
+
| "quarantined"
|
|
275
|
+
| "missing"
|
|
276
|
+
| "unknown";
|
|
277
|
+
observedAt: number;
|
|
278
|
+
detail?: string;
|
|
279
|
+
})[];
|
|
280
|
+
};
|
|
281
|
+
|
|
282
|
+
type AgentEntry = {
|
|
283
|
+
identity: AgentIdentity;
|
|
284
|
+
state: Promise<LogicalAgent>;
|
|
285
|
+
/**
|
|
286
|
+
* Closed with the agent: its socket is its authority, so it must not outlive it.
|
|
287
|
+
* `undefined` when the channel never opened; the agent state carries the reason.
|
|
288
|
+
*/
|
|
289
|
+
channel: Promise<ResultChannel | undefined>;
|
|
290
|
+
/** Why `agents.stop` ended it; its key is not opened again. */
|
|
291
|
+
stopped?: string;
|
|
292
|
+
stopping?: Promise<boolean>;
|
|
293
|
+
stopComplete?: boolean;
|
|
294
|
+
releasedAt?: number;
|
|
295
|
+
/** How it was opened, or found for the calling session, which its forks reuse. */
|
|
296
|
+
opening?: {
|
|
297
|
+
sessions: AgentSessions;
|
|
298
|
+
accounted: AccountedAgent;
|
|
299
|
+
/** Its skills as placed for its harness; absent when it has the operator's. */
|
|
300
|
+
skills: Promise<PlacedSkills | undefined>;
|
|
301
|
+
};
|
|
302
|
+
};
|
|
303
|
+
|
|
304
|
+
type AgentIdentity = {
|
|
305
|
+
execution: AgentExecution;
|
|
306
|
+
cwd: string;
|
|
307
|
+
/** As the agent was first opened with it: a ref, an inline spec, or absent. */
|
|
308
|
+
sandbox?: unknown;
|
|
309
|
+
/** Its skills' sources, checked; absent when it has the operator's. */
|
|
310
|
+
skills?: SkillSource[];
|
|
311
|
+
instructions?: string;
|
|
312
|
+
labels?: AgentOpenSpec["labels"];
|
|
313
|
+
/** The agent it was forked from, and the spec it was forked with. */
|
|
314
|
+
forkedFrom?: { parent: AgentKey; spec: AgentForkSpec };
|
|
315
|
+
/** As written; a reopen compares them so, never where the pane ended up. */
|
|
316
|
+
layout?: PaneLayout;
|
|
317
|
+
keepPane?: KeepPane;
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
const CLEANUP_GRACE_MILLISECONDS = 5_000;
|
|
321
|
+
/** How long after a pane's turn ends its receipt may still be read from the session's file. */
|
|
322
|
+
const RECEIPT_GRACE_MS = 10_000;
|
|
323
|
+
/** What a fork of the session the run was started from names as its parent. */
|
|
324
|
+
const CALLING = "the calling session";
|
|
325
|
+
/** A fork asks no model: it copies a session file, in seconds (F7). */
|
|
326
|
+
const FORK_TIMEOUT_MS = 60_000;
|
|
327
|
+
const scopes = new AsyncLocalStorage<ExecutionScope>();
|
|
328
|
+
|
|
329
|
+
export async function runWorkflow<Args extends JsonValue, Result extends JsonValue>(
|
|
330
|
+
definition: WorkflowDefinition<Args, Result>,
|
|
331
|
+
args: Args,
|
|
332
|
+
options: RunWorkflowOptions,
|
|
333
|
+
): Promise<WorkflowRunResult<Result>> {
|
|
334
|
+
return (await startWorkflow(definition, args, options)).result;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Opens the run and starts the workflow's body. It throws only when the run never started: it was
|
|
339
|
+
* cancelled, or its records, host or sandbox did not open; from the body on, every failure is the
|
|
340
|
+
* handle's `result`'s.
|
|
341
|
+
*/
|
|
342
|
+
export async function startWorkflow<Args extends JsonValue, Result extends JsonValue>(
|
|
343
|
+
definition: WorkflowDefinition<Args, Result>,
|
|
344
|
+
args: Args,
|
|
345
|
+
options: RunWorkflowOptions,
|
|
346
|
+
): Promise<WorkflowRunHandle<Result>> {
|
|
347
|
+
assertDeadline(options.deadline);
|
|
348
|
+
if (options.signal?.aborted) throw new WorkflowCancelledError(options.signal.reason);
|
|
349
|
+
const { dir: runDir, id: runId, attempt } = options.run;
|
|
350
|
+
const startedAt = Date.now();
|
|
351
|
+
const cwd = options.cwd ?? process.cwd();
|
|
352
|
+
// Before anything opens: a record this awf can't read refuses the attempt, and a crash may have
|
|
353
|
+
// torn the last line an earlier attempt appended.
|
|
354
|
+
const recorded = await readStageRecords(runDir);
|
|
355
|
+
// The log is observation: one that cannot be written says so as it is, and fails nothing.
|
|
356
|
+
await endTurnsLine(runDir).catch(() => undefined);
|
|
357
|
+
const slots = createResultSlotRegistry({
|
|
358
|
+
runDir,
|
|
359
|
+
...(options.livenessPolicy
|
|
360
|
+
? {
|
|
361
|
+
waitMinimumMs: options.livenessPolicy.quietMs,
|
|
362
|
+
waitDefaultMs: Math.max(1, options.livenessPolicy.quietMs * 4),
|
|
363
|
+
}
|
|
364
|
+
: {}),
|
|
365
|
+
});
|
|
366
|
+
const control = await startResultControlPlane({ slots });
|
|
367
|
+
let host: AgentRunHost;
|
|
368
|
+
const openingHost = Promise.resolve().then(() =>
|
|
369
|
+
options.runtime.host.openRun({
|
|
370
|
+
runId,
|
|
371
|
+
...(options.run.label === undefined ? {} : { label: options.run.label }),
|
|
372
|
+
cwd,
|
|
373
|
+
deadline: options.deadline,
|
|
374
|
+
}),
|
|
375
|
+
);
|
|
376
|
+
try {
|
|
377
|
+
host = await runUntilStopped(() => openingHost, options.signal, options.deadline);
|
|
378
|
+
} catch (error) {
|
|
379
|
+
void openingHost
|
|
380
|
+
.then((lateHost) => lateHost.close("run ended before host acquisition"))
|
|
381
|
+
.catch(() => undefined);
|
|
382
|
+
await control.close();
|
|
383
|
+
throw error;
|
|
384
|
+
}
|
|
385
|
+
// A failed write is said on stderr only: as a warning, it would be written to the log again.
|
|
386
|
+
const events = new RunEventLog({
|
|
387
|
+
runDir,
|
|
388
|
+
attempt,
|
|
389
|
+
failed: (message) => options.onLog?.(message),
|
|
390
|
+
});
|
|
391
|
+
const say =
|
|
392
|
+
(kind: "log" | "warning") =>
|
|
393
|
+
(text: string, fields?: JsonObject, stage = scopes.getStore()?.stage): void => {
|
|
394
|
+
options.onLog?.(text, fields);
|
|
395
|
+
events.event({
|
|
396
|
+
kind,
|
|
397
|
+
text,
|
|
398
|
+
...(fields === undefined ? {} : { fields }),
|
|
399
|
+
...(stage === undefined ? {} : { stage }),
|
|
400
|
+
});
|
|
401
|
+
};
|
|
402
|
+
const warn = say("warning");
|
|
403
|
+
// Each turn as it settles, so an attempt that dies keeps the sessions and times of its turns.
|
|
404
|
+
const turns: TurnRecord[] = [];
|
|
405
|
+
const ledger = createRunLedger({ accounting: options.runtime.host.accounting, startedAt });
|
|
406
|
+
const recordTurn: RecordTurn = (outcome, kind) => {
|
|
407
|
+
const turn: TurnRecord = {
|
|
408
|
+
version: TURN_RECORD_VERSION,
|
|
409
|
+
attempt,
|
|
410
|
+
kind,
|
|
411
|
+
outcome: outcome.kind,
|
|
412
|
+
callPath: outcome.usage.callPath,
|
|
413
|
+
...pick(outcome.usage, TURN_OPERATION_FIELDS),
|
|
414
|
+
};
|
|
415
|
+
turns.push(turn);
|
|
416
|
+
events.turn(turn);
|
|
417
|
+
};
|
|
418
|
+
const stages = new StageLedger({
|
|
419
|
+
runDir,
|
|
420
|
+
attempt,
|
|
421
|
+
records: recorded,
|
|
422
|
+
...(options.run.fromStage === undefined ? {} : { fromStage: options.run.fromStage }),
|
|
423
|
+
...(options.run.values === undefined ? {} : { values: options.run.values }),
|
|
424
|
+
...(definition.meta.version === undefined ? {} : { workflowVersion: definition.meta.version }),
|
|
425
|
+
turns: () => turns,
|
|
426
|
+
event: (body, at) => events.event(body, at),
|
|
427
|
+
});
|
|
428
|
+
const progress = new RunProgress();
|
|
429
|
+
// Shared by sandboxed agents' homes and a host agent's own: they may copy one credential.
|
|
430
|
+
const locks: CredentialLocks = new Map();
|
|
431
|
+
const environment = options.sandboxes?.environment ?? process.env;
|
|
432
|
+
const sandboxes = new RunSandboxes({
|
|
433
|
+
...(options.sandboxes ? { sandboxes: options.sandboxes } : {}),
|
|
434
|
+
locks,
|
|
435
|
+
runRoot: options.runRoot,
|
|
436
|
+
cwd,
|
|
437
|
+
deadline: options.deadline,
|
|
438
|
+
log: (message) => warn(message),
|
|
439
|
+
});
|
|
440
|
+
if (sandboxes.hasRunSandbox) {
|
|
441
|
+
try {
|
|
442
|
+
await runUntilStopped(() => sandboxes.openRunSandbox(), options.signal, options.deadline);
|
|
443
|
+
} catch (error) {
|
|
444
|
+
await sandboxes.close().catch(() => undefined);
|
|
445
|
+
await host.close("the run's sandbox did not open").catch(() => undefined);
|
|
446
|
+
await control.close();
|
|
447
|
+
if (error instanceof WorkflowCancelledError || error instanceof DeadlineExceededError) {
|
|
448
|
+
throw error;
|
|
449
|
+
}
|
|
450
|
+
throw new Error(`the run's sandbox did not open: ${messageOf(error)}`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
const decisions = new RunDecisions({
|
|
454
|
+
...(options.decisions ? { installation: options.decisions } : {}),
|
|
455
|
+
runDir,
|
|
456
|
+
event: (body) => events.event(body),
|
|
457
|
+
});
|
|
458
|
+
const skills = new RunSkills({
|
|
459
|
+
runDir,
|
|
460
|
+
...(options.skillCache === undefined ? {} : { cacheRoot: options.skillCache }),
|
|
461
|
+
environment,
|
|
462
|
+
});
|
|
463
|
+
const fatal = Promise.withResolvers<never>();
|
|
464
|
+
void fatal.promise.catch(() => undefined);
|
|
465
|
+
const owner = new WorkflowOwner({
|
|
466
|
+
failRun: (error) => fatal.reject(error),
|
|
467
|
+
decisions,
|
|
468
|
+
sandboxes,
|
|
469
|
+
skills,
|
|
470
|
+
runDir,
|
|
471
|
+
locks,
|
|
472
|
+
environment,
|
|
473
|
+
progress,
|
|
474
|
+
recordTurn,
|
|
475
|
+
stages,
|
|
476
|
+
runId,
|
|
477
|
+
attempt,
|
|
478
|
+
cwd,
|
|
479
|
+
deadline: options.deadline,
|
|
480
|
+
runtime: options.runtime,
|
|
481
|
+
host,
|
|
482
|
+
slots,
|
|
483
|
+
control,
|
|
484
|
+
ledger,
|
|
485
|
+
onLog: warn,
|
|
486
|
+
log: say("log"),
|
|
487
|
+
events,
|
|
488
|
+
onCallerStopped: options.onCallerStopped,
|
|
489
|
+
livenessPolicy: options.livenessPolicy,
|
|
490
|
+
awaitReply: options.awaitReply ?? ((execution, confirms) => canAwaitReply(execution, confirms)),
|
|
491
|
+
});
|
|
492
|
+
const stopped = new AbortController();
|
|
493
|
+
// Aborted by a stop that comes after the body ended; the one that ended it does not cut short the
|
|
494
|
+
// read of what it spent.
|
|
495
|
+
const reading = new AbortController();
|
|
496
|
+
let bodyEnded = false;
|
|
497
|
+
const signal = options.signal
|
|
498
|
+
? AbortSignal.any([options.signal, stopped.signal])
|
|
499
|
+
: stopped.signal;
|
|
500
|
+
const result = (async (): Promise<WorkflowRunResult<Result>> => {
|
|
501
|
+
let value: Result | undefined;
|
|
502
|
+
let failure: unknown;
|
|
503
|
+
let failed = false;
|
|
504
|
+
try {
|
|
505
|
+
value = await runUntilStopped(
|
|
506
|
+
() =>
|
|
507
|
+
Promise.race([
|
|
508
|
+
// Its value waits for the message operations its routes outside any stage opened.
|
|
509
|
+
definition.run(owner.context, args).then(async (returned) => {
|
|
510
|
+
await owner.settleMessages();
|
|
511
|
+
return returned;
|
|
512
|
+
}),
|
|
513
|
+
fatal.promise,
|
|
514
|
+
]),
|
|
515
|
+
signal,
|
|
516
|
+
options.deadline,
|
|
517
|
+
);
|
|
518
|
+
if (!isJsonValue(value)) throw new Error("workflow result must contain only JSON values");
|
|
519
|
+
const caught = stages.caught();
|
|
520
|
+
if (caught) throw caught;
|
|
521
|
+
if (stages.fromStageUnreached !== undefined) {
|
|
522
|
+
throw new FromStageUnreached(stages.fromStageUnreached, stages.recorded);
|
|
523
|
+
}
|
|
524
|
+
} catch (error) {
|
|
525
|
+
failed = true;
|
|
526
|
+
failure = error;
|
|
527
|
+
}
|
|
528
|
+
const ending = stages.settle(
|
|
529
|
+
failed ? failure : undefined,
|
|
530
|
+
signal.aborted || failure instanceof DeadlineExceededError,
|
|
531
|
+
);
|
|
532
|
+
if (ending !== undefined) {
|
|
533
|
+
failed = true;
|
|
534
|
+
failure = ending;
|
|
535
|
+
}
|
|
536
|
+
bodyEnded = true;
|
|
537
|
+
// What the body left running enters no stage; one left open is failed once its turns settle.
|
|
538
|
+
stages.seal(failed ? messageOf(failure) : "the workflow returned before the stage ended");
|
|
539
|
+
const cutReading = () => reading.abort();
|
|
540
|
+
if (!signal.aborted) signal.addEventListener("abort", cutReading, { once: true });
|
|
541
|
+
|
|
542
|
+
if (!failed) await ledger.letFinish(options.deadline, signal);
|
|
543
|
+
const cleanupErrors: unknown[] = [];
|
|
544
|
+
const ownerCleanupDeadline = {
|
|
545
|
+
unixMilliseconds: Date.now() + CLEANUP_GRACE_MILLISECONDS,
|
|
546
|
+
};
|
|
547
|
+
try {
|
|
548
|
+
cleanupErrors.push(...(await owner.close(ownerCleanupDeadline)));
|
|
549
|
+
} catch (error) {
|
|
550
|
+
cleanupErrors.push(error);
|
|
551
|
+
}
|
|
552
|
+
const controlCleanupDeadline = {
|
|
553
|
+
unixMilliseconds: Date.now() + CLEANUP_GRACE_MILLISECONDS,
|
|
554
|
+
};
|
|
555
|
+
try {
|
|
556
|
+
await waitForDeadline(control.close(), controlCleanupDeadline);
|
|
557
|
+
} catch (error) {
|
|
558
|
+
cleanupErrors.push(
|
|
559
|
+
error instanceof DeadlineExceededError
|
|
560
|
+
? new Error(
|
|
561
|
+
`control-plane cleanup exceeded ${CLEANUP_GRACE_MILLISECONDS}ms shutdown grace`,
|
|
562
|
+
)
|
|
563
|
+
: error,
|
|
564
|
+
);
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
// Before the wait for session files, which is bookkeeping rather than the run's own time.
|
|
568
|
+
const finishedAt = new Date().toISOString();
|
|
569
|
+
await stages
|
|
570
|
+
.close()
|
|
571
|
+
.catch((error) => warn(`awf: a stage record was not written: ${messageOf(error)}`));
|
|
572
|
+
await events.close();
|
|
573
|
+
const usage = await ledger.settle(reading.signal);
|
|
574
|
+
signal.removeEventListener("abort", cutReading);
|
|
575
|
+
const times = { startedAt: new Date(startedAt).toISOString(), finishedAt };
|
|
576
|
+
const opened = sandboxes.records();
|
|
577
|
+
const given = skills.records();
|
|
578
|
+
const placed = owner.panes();
|
|
579
|
+
const asked = decisions.records();
|
|
580
|
+
const endedIn = failed ? stages.endedIn(failure) : undefined;
|
|
581
|
+
const settled: SettledRun = {
|
|
582
|
+
runId,
|
|
583
|
+
...(endedIn === undefined ? {} : { endedIn }),
|
|
584
|
+
...(stages.summaries.length > 0 ? { stages: stages.summaries } : {}),
|
|
585
|
+
usage,
|
|
586
|
+
...times,
|
|
587
|
+
accounting: summarizeRun(usage, PUBLISHED_PRICES, times, asked, stages.summaries),
|
|
588
|
+
...(opened.length > 0 ? { sandboxes: opened } : {}),
|
|
589
|
+
...(given.length > 0 ? { skills: given } : {}),
|
|
590
|
+
...(placed.length > 0 ? { panes: placed } : {}),
|
|
591
|
+
...(asked.length > 0 ? { decisions: asked } : {}),
|
|
592
|
+
};
|
|
593
|
+
if (failed) {
|
|
594
|
+
throw new WorkflowRunError(
|
|
595
|
+
cleanupErrors.length > 0
|
|
596
|
+
? new AggregateError([failure, ...cleanupErrors], "workflow and cleanup failed")
|
|
597
|
+
: failure,
|
|
598
|
+
settled,
|
|
599
|
+
);
|
|
600
|
+
}
|
|
601
|
+
if (cleanupErrors.length > 0) {
|
|
602
|
+
throw new WorkflowRunError(
|
|
603
|
+
new AggregateError(cleanupErrors, "workflow cleanup failed"),
|
|
604
|
+
settled,
|
|
605
|
+
);
|
|
606
|
+
}
|
|
607
|
+
return { ...settled, value: value as Result };
|
|
608
|
+
})();
|
|
609
|
+
|
|
610
|
+
return {
|
|
611
|
+
runId,
|
|
612
|
+
result,
|
|
613
|
+
inspect: () => {
|
|
614
|
+
const { state, agents } = structuredClone(host.inspect());
|
|
615
|
+
const known = progress.snapshot();
|
|
616
|
+
return {
|
|
617
|
+
state,
|
|
618
|
+
groups: known.groups,
|
|
619
|
+
...stages.progress(),
|
|
620
|
+
messages: known.messages,
|
|
621
|
+
agents: agents.map((agent) => {
|
|
622
|
+
const queued = owner.held(agent.key);
|
|
623
|
+
return {
|
|
624
|
+
...known.agents.get(agent.key),
|
|
625
|
+
...agent,
|
|
626
|
+
...(queued > 0 ? { queued } : {}),
|
|
627
|
+
};
|
|
628
|
+
}),
|
|
629
|
+
};
|
|
630
|
+
},
|
|
631
|
+
async stop(reason) {
|
|
632
|
+
stopped.abort(reason);
|
|
633
|
+
if (bodyEnded) reading.abort();
|
|
634
|
+
try {
|
|
635
|
+
await result;
|
|
636
|
+
} catch (error) {
|
|
637
|
+
const cause = error instanceof WorkflowRunError ? error.cause : error;
|
|
638
|
+
if (!(cause instanceof WorkflowCancelledError)) throw error;
|
|
639
|
+
}
|
|
640
|
+
},
|
|
641
|
+
};
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
/**
|
|
645
|
+
* An agent's sessions, as its launcher reports them and as its harness session knows them, and its
|
|
646
|
+
* forks', whose copied context may reach its launcher with a fork's own session (ADR 0009).
|
|
647
|
+
*/
|
|
648
|
+
type AgentSessions = {
|
|
649
|
+
launcher: Set<string>;
|
|
650
|
+
harness?: HarnessSession;
|
|
651
|
+
forks: AgentSessions[];
|
|
652
|
+
/** A fork's parent's, whose session the fork's copied context can name (pi keeps its id). */
|
|
653
|
+
parent?: AgentSessions;
|
|
654
|
+
};
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* What the agent reported or its harness saw, less any session one of its forks runs, and less
|
|
658
|
+
* any its parent has: a fork answering under its parent's id is its parent's session, not its own.
|
|
659
|
+
*/
|
|
660
|
+
function reportedSessions(sessions: AgentSessions): string[] {
|
|
661
|
+
const own = sessions.harness?.sessions?.() ?? [];
|
|
662
|
+
const others = new Set(forkSessions(sessions));
|
|
663
|
+
for (let parent = sessions.parent; parent; parent = parent.parent) {
|
|
664
|
+
for (const id of [...parent.launcher, ...(parent.harness?.sessions?.() ?? [])]) {
|
|
665
|
+
// What its own harness saw is its own, even reported through its parent's channel.
|
|
666
|
+
if (!own.includes(id)) others.add(id);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
return [...new Set([...sessions.launcher, ...own])].filter((id) => !others.has(id));
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function forkSessions(sessions: AgentSessions): string[] {
|
|
673
|
+
return sessions.forks.flatMap((fork) => [
|
|
674
|
+
...(fork.harness?.sessions?.() ?? []),
|
|
675
|
+
...forkSessions(fork),
|
|
676
|
+
]);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
type PlacedSkills = { given: AgentSkills; writeBack?: () => Promise<void> };
|
|
680
|
+
|
|
681
|
+
/** What both ways of opening an agent share. */
|
|
682
|
+
type AgentOpening = {
|
|
683
|
+
spec: AgentOpenSpec;
|
|
684
|
+
execution: AgentExecution;
|
|
685
|
+
deadline: AbsoluteDeadline;
|
|
686
|
+
sessions: AgentSessions;
|
|
687
|
+
ledger: AgentLedger;
|
|
688
|
+
/** Its skills' sources, resolved when it opens; absent when it has the operator's. */
|
|
689
|
+
skills?: SkillSource[];
|
|
690
|
+
/** The host's activation for the agent, working in `cwd`, inside `occupant` if sandboxed. */
|
|
691
|
+
activation(
|
|
692
|
+
cwd: string,
|
|
693
|
+
occupant?: Occupant,
|
|
694
|
+
skills?: AgentSkills,
|
|
695
|
+
/** Its harness home: a sandbox's, or its own on the host. */
|
|
696
|
+
home?: string,
|
|
697
|
+
): HarnessActivation;
|
|
698
|
+
/**
|
|
699
|
+
* A fork whose parent has a harness home of its own: the parent's session, copied into `copy` by
|
|
700
|
+
* `forked`, is placed in this agent's home before it is admitted, in the parent's sandbox.
|
|
701
|
+
*/
|
|
702
|
+
carried?: { parent: AgentKey; forked: Promise<unknown>; copy: string };
|
|
703
|
+
/**
|
|
704
|
+
* Settles before the host places the pane: once a `beside` target has opened, with why the
|
|
705
|
+
* layout can't be used where the engine knows already.
|
|
706
|
+
*/
|
|
707
|
+
layoutFallback?: Promise<string | undefined>;
|
|
708
|
+
};
|
|
709
|
+
|
|
710
|
+
class WorkflowOwner {
|
|
711
|
+
readonly #cleanupFailures: Error[] = [];
|
|
712
|
+
readonly context: WorkflowContext;
|
|
713
|
+
readonly #agents = new Map<string, AgentEntry>();
|
|
714
|
+
/** Each agent once it has opened, by key: what messaging asks about, and what a ref must be. */
|
|
715
|
+
readonly #logical = new Map<string, LogicalAgent>();
|
|
716
|
+
readonly #messaging: Messaging;
|
|
717
|
+
/** A stage's execution scope, by its messaging scope: what owns the operations its routes open. */
|
|
718
|
+
readonly #stageScopes = new Map<MessagingScope, ExecutionScope>();
|
|
719
|
+
/** Message operations routes outside any stage opened, which the attempt owns. */
|
|
720
|
+
readonly #attemptOwned = new Set<Promise<unknown>>();
|
|
721
|
+
/** Resolves on any change to messaging, for a drain waiting on it. */
|
|
722
|
+
#messagingChange = Promise.withResolvers<void>();
|
|
723
|
+
readonly #inFlight = new Set<Promise<unknown>>();
|
|
724
|
+
#closed = false;
|
|
725
|
+
#closing: Promise<unknown[]> | undefined;
|
|
726
|
+
/** The bundled `wf`'s source, built once, at the first sandboxed agent. */
|
|
727
|
+
#bundle: Promise<string> | undefined;
|
|
728
|
+
/** The calling session, once the workflow has asked for it (ADR 0010). */
|
|
729
|
+
#caller: { key: string; state: Promise<LogicalAgent> } | undefined;
|
|
730
|
+
/** Stops a fork of the session the run was started from, whose process is not an agent's. */
|
|
731
|
+
readonly #closingRun = new AbortController();
|
|
732
|
+
|
|
733
|
+
constructor(
|
|
734
|
+
private readonly options: {
|
|
735
|
+
sandboxes: RunSandboxes;
|
|
736
|
+
skills: RunSkills;
|
|
737
|
+
failRun(error: Error): void;
|
|
738
|
+
runDir: string;
|
|
739
|
+
livenessPolicy?: LivenessPolicy;
|
|
740
|
+
locks: CredentialLocks;
|
|
741
|
+
environment: Readonly<Record<string, string | undefined>>;
|
|
742
|
+
decisions: RunDecisions;
|
|
743
|
+
stages: StageLedger;
|
|
744
|
+
runId: string;
|
|
745
|
+
attempt: number;
|
|
746
|
+
cwd: string;
|
|
747
|
+
deadline: AbsoluteDeadline;
|
|
748
|
+
runtime: AgentRuntimeConfig;
|
|
749
|
+
host: AgentRunHost;
|
|
750
|
+
slots: ResultSlotRegistry;
|
|
751
|
+
control: ResultControlPlane;
|
|
752
|
+
ledger: RunLedger;
|
|
753
|
+
progress: RunProgress;
|
|
754
|
+
recordTurn: RecordTurn;
|
|
755
|
+
/** Engine warnings, written to the event log as well, under `stage` or the scope's. */
|
|
756
|
+
onLog?: Warn;
|
|
757
|
+
/** The workflow's own log lines. */
|
|
758
|
+
log(message: string, fields?: JsonObject): void;
|
|
759
|
+
events: RunEventLog;
|
|
760
|
+
onCallerStopped?: RunWorkflowOptions["onCallerStopped"];
|
|
761
|
+
awaitReply: NonNullable<RunWorkflowOptions["awaitReply"]>;
|
|
762
|
+
},
|
|
763
|
+
) {
|
|
764
|
+
this.#messaging = new Messaging({
|
|
765
|
+
agent: (key) => this.#agentView(key),
|
|
766
|
+
canAwait: (key) => {
|
|
767
|
+
const agent = this.#logical.get(key);
|
|
768
|
+
return agent ? options.awaitReply(agent.execution, agent.confirmsDelivery) : false;
|
|
769
|
+
},
|
|
770
|
+
launcher: (key) => this.#logical.get(key)?.launcher ?? "wf",
|
|
771
|
+
changed: (key) => {
|
|
772
|
+
this.#messagesChanged();
|
|
773
|
+
this.#logical.get(key)?.messagesChanged();
|
|
774
|
+
},
|
|
775
|
+
schedule: (at, action) => scheduleAt({ unixMilliseconds: at }, action),
|
|
776
|
+
now: Date.now,
|
|
777
|
+
record: async (event) => {
|
|
778
|
+
this.#messagesChanged();
|
|
779
|
+
if (event.kind === "message-sent") {
|
|
780
|
+
options.progress.messageSent({
|
|
781
|
+
from: event.from,
|
|
782
|
+
to: event.to,
|
|
783
|
+
bytes: event.bytes,
|
|
784
|
+
at: Date.now(),
|
|
785
|
+
...(event.awaitsReply ? { asks: true as const } : {}),
|
|
786
|
+
...(event.replyTo === undefined ? {} : { answers: true as const }),
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
if (!(await options.events.event({ ...event, callPath: [] }))) {
|
|
790
|
+
throw new Error("the run's event log was not written");
|
|
791
|
+
}
|
|
792
|
+
},
|
|
793
|
+
});
|
|
794
|
+
options.control.attach(this.#messaging);
|
|
795
|
+
this.context = {
|
|
796
|
+
runId: options.runId,
|
|
797
|
+
attempt: options.attempt,
|
|
798
|
+
cwd: options.cwd,
|
|
799
|
+
deadline: options.deadline,
|
|
800
|
+
agents: {
|
|
801
|
+
open: (spec) => {
|
|
802
|
+
try {
|
|
803
|
+
return this.openAgent(spec);
|
|
804
|
+
} catch (error) {
|
|
805
|
+
return Promise.reject(error);
|
|
806
|
+
}
|
|
807
|
+
},
|
|
808
|
+
attach: (key, runtime) => {
|
|
809
|
+
if (this.#caller?.key !== key) return unavailable("agents.attach");
|
|
810
|
+
const stopped = this.#agents.get(key)?.stopped;
|
|
811
|
+
if (stopped !== undefined) return Promise.reject(stoppedError(key, stopped));
|
|
812
|
+
if (runtime !== undefined) {
|
|
813
|
+
return Promise.reject(
|
|
814
|
+
new Error(`agent ${key} is the calling session, whose runtime is the operator's`),
|
|
815
|
+
);
|
|
816
|
+
}
|
|
817
|
+
return this.#caller.state;
|
|
818
|
+
},
|
|
819
|
+
stop: (key, reason) => {
|
|
820
|
+
try {
|
|
821
|
+
return this.stopAgent(key, reason);
|
|
822
|
+
} catch (error) {
|
|
823
|
+
return Promise.reject(error);
|
|
824
|
+
}
|
|
825
|
+
},
|
|
826
|
+
caller: (spec) => {
|
|
827
|
+
try {
|
|
828
|
+
return this.caller(spec.key);
|
|
829
|
+
} catch (error) {
|
|
830
|
+
return Promise.reject(error);
|
|
831
|
+
}
|
|
832
|
+
},
|
|
833
|
+
forkCaller: (spec) => {
|
|
834
|
+
try {
|
|
835
|
+
return this.forkCaller(spec);
|
|
836
|
+
} catch (error) {
|
|
837
|
+
return Promise.reject(error);
|
|
838
|
+
}
|
|
839
|
+
},
|
|
840
|
+
},
|
|
841
|
+
sandboxes: {
|
|
842
|
+
open: (spec) => {
|
|
843
|
+
if (this.#closed || this.#cleanupFailures.length)
|
|
844
|
+
return Promise.reject(new Error("workflow context is closed"));
|
|
845
|
+
try {
|
|
846
|
+
this.options.stages.checkOperation();
|
|
847
|
+
} catch (error) {
|
|
848
|
+
return Promise.reject(error);
|
|
849
|
+
}
|
|
850
|
+
return this.track(options.sandboxes.open(spec));
|
|
851
|
+
},
|
|
852
|
+
},
|
|
853
|
+
decisions: {
|
|
854
|
+
decide: (spec) => {
|
|
855
|
+
if (this.#closed || this.#cleanupFailures.length)
|
|
856
|
+
return Promise.reject(new Error("workflow context is closed"));
|
|
857
|
+
const scope = scopes.getStore();
|
|
858
|
+
try {
|
|
859
|
+
scope?.assertAccepting();
|
|
860
|
+
this.options.stages.checkOperation();
|
|
861
|
+
} catch (error) {
|
|
862
|
+
return Promise.reject(error);
|
|
863
|
+
}
|
|
864
|
+
const decided = this.track(
|
|
865
|
+
options.decisions.decide(spec, {
|
|
866
|
+
deadline: scope?.deadline ?? options.deadline,
|
|
867
|
+
...(scope ? { add: (cancel) => scope.add((stop) => cancel(stop.reason)) } : {}),
|
|
868
|
+
...(scope?.stage === undefined ? {} : { stage: scope.stage }),
|
|
869
|
+
}),
|
|
870
|
+
);
|
|
871
|
+
scope?.track(decided);
|
|
872
|
+
return decided;
|
|
873
|
+
},
|
|
874
|
+
},
|
|
875
|
+
participants: {
|
|
876
|
+
connect: () => unavailable("participants.connect"),
|
|
877
|
+
get: () => unavailable("participants.get"),
|
|
878
|
+
},
|
|
879
|
+
messages: { allow: async (access) => this.allowRoute(access) },
|
|
880
|
+
steps: {
|
|
881
|
+
run: () => unavailable("steps.run"),
|
|
882
|
+
sleep: () => unavailable("steps.sleep"),
|
|
883
|
+
},
|
|
884
|
+
signals: { receive: () => unavailable("signals.receive") },
|
|
885
|
+
parallel: (items, operation, parallelOptions) => {
|
|
886
|
+
const inherited = scopes.getStore()?.deadline ?? options.deadline;
|
|
887
|
+
const deadline = parallelOptions?.deadline
|
|
888
|
+
? earlierDeadline(parallelOptions.deadline, inherited)
|
|
889
|
+
: inherited;
|
|
890
|
+
const label = parallelOptions?.label;
|
|
891
|
+
return runParallel(items, operation, deadline, parallelOptions?.concurrency, () =>
|
|
892
|
+
label === undefined ? undefined : options.progress.group(label, items.length),
|
|
893
|
+
);
|
|
894
|
+
},
|
|
895
|
+
stop: (reason: string): never => {
|
|
896
|
+
// The stage it is called in, by scope: not whichever one is open beside it.
|
|
897
|
+
throw this.options.stages.stop(String(reason), scopes.getStore()?.stage);
|
|
898
|
+
},
|
|
899
|
+
stage: ((name: string, ...rest: unknown[]) =>
|
|
900
|
+
this.runStage(name, rest)) as WorkflowContext["stage"],
|
|
901
|
+
call: () => unavailable("call"),
|
|
902
|
+
usage: () => options.ledger.records(),
|
|
903
|
+
log: (message, fields) => options.log(message, fields),
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
/** An agent as messaging sees it, from its entry and its operations. */
|
|
908
|
+
#agentView(key: string): AgentView | undefined {
|
|
909
|
+
const agent = this.#logical.get(key);
|
|
910
|
+
if (!agent) return undefined;
|
|
911
|
+
const caller = agent.execution.caller ? { caller: true as const } : {};
|
|
912
|
+
const stopped = this.#agents.get(key)?.stopped ?? agent.closedReason;
|
|
913
|
+
if (stopped !== undefined) return { state: "stopped", reason: stopped, ...caller };
|
|
914
|
+
return { state: agent.busy ? "working" : agent.hasRun ? "idle" : "not-started", ...caller };
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
#messagesChanged(): void {
|
|
918
|
+
const change = this.#messagingChange;
|
|
919
|
+
this.#messagingChange = Promise.withResolvers<void>();
|
|
920
|
+
change.resolve();
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
/**
|
|
924
|
+
* Grants a route in the current stage, or for the attempt outside any. Its ends must be the refs
|
|
925
|
+
* this run issued: a spread or cast of one names an agent it did not hand out.
|
|
926
|
+
*/
|
|
927
|
+
private allowRoute(access: MessageAccess): void {
|
|
928
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
929
|
+
const scope = scopes.getStore();
|
|
930
|
+
scope?.assertAccepting();
|
|
931
|
+
const end = (ref: unknown): string => {
|
|
932
|
+
const key =
|
|
933
|
+
typeof ref === "object" && ref !== null ? (ref as { key?: unknown }).key : undefined;
|
|
934
|
+
const agent = typeof key === "string" ? this.#logical.get(key) : undefined;
|
|
935
|
+
if (!agent || agent !== ref) {
|
|
936
|
+
throw new Error(
|
|
937
|
+
`a route's ends are agents this workflow opened; ${typeof key === "string" ? key : "this"} is not one`,
|
|
938
|
+
);
|
|
939
|
+
}
|
|
940
|
+
return agent.key;
|
|
941
|
+
};
|
|
942
|
+
if (typeof access !== "object" || access === null)
|
|
943
|
+
throw new Error("a route is { from, to } or { between }, with a purpose");
|
|
944
|
+
const shape = access as { from?: unknown; to?: unknown; between?: unknown };
|
|
945
|
+
let ends: { from: string; to: string } | { between: [string, string] };
|
|
946
|
+
if (shape.between !== undefined) {
|
|
947
|
+
if (shape.from !== undefined || shape.to !== undefined)
|
|
948
|
+
throw new Error("a route is between two agents, or from one to another, not both");
|
|
949
|
+
if (!Array.isArray(shape.between) || shape.between.length !== 2)
|
|
950
|
+
throw new Error("a route between agents names two of them");
|
|
951
|
+
ends = { between: [end(shape.between[0]), end(shape.between[1])] };
|
|
952
|
+
} else ends = { from: end(shape.from), to: end(shape.to) };
|
|
953
|
+
this.#messaging.grant(ends, {
|
|
954
|
+
purpose: access.purpose,
|
|
955
|
+
maxMessages: access.maxMessages,
|
|
956
|
+
scope: scope?.messages ?? this.#messaging.attempt,
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/** Who owns a message operation a route in `scope` opens: its stage, or the attempt. */
|
|
961
|
+
#messageOwner(
|
|
962
|
+
scope: MessagingScope,
|
|
963
|
+
): { stage?: ExecutionScope; adopt(operation: Promise<unknown>): void } | undefined {
|
|
964
|
+
if (scope.state === "closed") return undefined;
|
|
965
|
+
const stage = this.#stageScopes.get(scope);
|
|
966
|
+
if (stage) {
|
|
967
|
+
return {
|
|
968
|
+
stage,
|
|
969
|
+
adopt: (operation) => {
|
|
970
|
+
stage.adopt(operation);
|
|
971
|
+
this.#messagesChanged();
|
|
972
|
+
},
|
|
973
|
+
};
|
|
974
|
+
}
|
|
975
|
+
if (scope !== this.#messaging.attempt) return undefined;
|
|
976
|
+
return {
|
|
977
|
+
adopt: (operation) => {
|
|
978
|
+
this.#attemptOwned.add(operation);
|
|
979
|
+
this.#messagesChanged();
|
|
980
|
+
// The attempt owns it: one that times out or whose harness fails ends the run, even after
|
|
981
|
+
// the workflow returned a value.
|
|
982
|
+
operation.catch((error: unknown) => this.options.failRun(error as Error));
|
|
983
|
+
},
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/**
|
|
988
|
+
* Until nothing a scope owns is in flight: its operations settled, and no answer owed or message
|
|
989
|
+
* held on its routes. A message held for an idle agent opens an operation meanwhile, which the
|
|
990
|
+
* scope then owns too.
|
|
991
|
+
*/
|
|
992
|
+
private async drainMessages(scope: MessagingScope, settle: () => Promise<void>): Promise<void> {
|
|
993
|
+
for (;;) {
|
|
994
|
+
const change = this.#messagingChange.promise;
|
|
995
|
+
await settle();
|
|
996
|
+
if (!this.#messaging.busy(scope)) {
|
|
997
|
+
await settle();
|
|
998
|
+
if (!this.#messaging.busy(scope)) return;
|
|
999
|
+
}
|
|
1000
|
+
await change;
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/** Messages held for `key` and not yet shown. */
|
|
1005
|
+
held(key: AgentKey): number {
|
|
1006
|
+
return this.#messaging.held(key);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
/**
|
|
1010
|
+
* The workflow returned: its routes outside any stage take nothing new, and the message
|
|
1011
|
+
* operations they opened are waited for, as a stage waits for its own.
|
|
1012
|
+
*/
|
|
1013
|
+
async settleMessages(): Promise<void> {
|
|
1014
|
+
const attempt = this.#messaging.attempt;
|
|
1015
|
+
this.#messaging.finishing(attempt);
|
|
1016
|
+
await this.drainMessages(attempt, async () => {
|
|
1017
|
+
const settled = await Promise.allSettled([...this.#attemptOwned]);
|
|
1018
|
+
const failed = settled.find((each) => each.status === "rejected");
|
|
1019
|
+
if (failed?.status === "rejected") throw failed.reason;
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
async close(deadline: AbsoluteDeadline): Promise<unknown[]> {
|
|
1024
|
+
this.#closing ??= (async (): Promise<unknown[]> => {
|
|
1025
|
+
this.#closed = true;
|
|
1026
|
+
// Nothing more is delivered; what was held is recorded undelivered.
|
|
1027
|
+
this.#messaging.close();
|
|
1028
|
+
this.#closingRun.abort();
|
|
1029
|
+
// Each agent is closed by itself first, which knows whether its pane stays; then the host,
|
|
1030
|
+
// which closes whatever is left.
|
|
1031
|
+
const stopping = [...this.#agents.values()].map((agent) =>
|
|
1032
|
+
agent.state.then(
|
|
1033
|
+
(state) =>
|
|
1034
|
+
state
|
|
1035
|
+
.stopOperations({
|
|
1036
|
+
kind:
|
|
1037
|
+
Date.now() >= this.options.deadline.unixMilliseconds ? "timed-out" : "cancelled",
|
|
1038
|
+
reason: "workflow closing",
|
|
1039
|
+
})
|
|
1040
|
+
.then(() => state.close("workflow complete")),
|
|
1041
|
+
() => undefined,
|
|
1042
|
+
),
|
|
1043
|
+
);
|
|
1044
|
+
const cleanup = Promise.all([
|
|
1045
|
+
Promise.allSettled(stopping).then(async (stopped) => [
|
|
1046
|
+
...stopped,
|
|
1047
|
+
...(await Promise.allSettled([this.options.host.close("workflow complete")])),
|
|
1048
|
+
]),
|
|
1049
|
+
// A call nobody awaited any more is cancelled and recorded, not left to its deadline.
|
|
1050
|
+
this.options.decisions.close().then(() => Promise.allSettled([...this.#inFlight])),
|
|
1051
|
+
]).then(async ([settled]) => {
|
|
1052
|
+
// Once their sessions are closed: nothing of theirs runs inside any more.
|
|
1053
|
+
const released = await this.options.sandboxes.release();
|
|
1054
|
+
// Only now: an agent can still be submitting from inside its own close, and taking its
|
|
1055
|
+
// socket away first turns that into a connection error it cannot report.
|
|
1056
|
+
await Promise.allSettled(
|
|
1057
|
+
[...this.#agents.values()].map((agent) => agent.channel.then((c) => c?.close())),
|
|
1058
|
+
);
|
|
1059
|
+
// Last, after every agent in each: closing a sandbox ends what runs in it.
|
|
1060
|
+
const closed = await this.options.sandboxes.close();
|
|
1061
|
+
return [
|
|
1062
|
+
...settled.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])),
|
|
1063
|
+
...released,
|
|
1064
|
+
...closed,
|
|
1065
|
+
...this.#cleanupFailures,
|
|
1066
|
+
];
|
|
1067
|
+
});
|
|
1068
|
+
try {
|
|
1069
|
+
return await waitForDeadline(cleanup, deadline);
|
|
1070
|
+
} catch (error) {
|
|
1071
|
+
if (error instanceof DeadlineExceededError) {
|
|
1072
|
+
// A session that would not close still must not outlive its sandbox: its agents are
|
|
1073
|
+
// released and the sandboxes closed, which ends everything inside, on a grace of their own.
|
|
1074
|
+
const sandboxes = this.options.sandboxes;
|
|
1075
|
+
// A slow agent close may be what overran: the host still closes what it made, though
|
|
1076
|
+
// nothing waits on it here, as an agent that never closes would hold it too.
|
|
1077
|
+
void this.options.host.close("workflow complete").catch(() => undefined);
|
|
1078
|
+
const closed = await waitForDeadline(
|
|
1079
|
+
sandboxes
|
|
1080
|
+
.release()
|
|
1081
|
+
.then(async (released) => [...released, ...(await sandboxes.close())]),
|
|
1082
|
+
{ unixMilliseconds: Date.now() + CLEANUP_GRACE_MILLISECONDS },
|
|
1083
|
+
).catch((late: unknown) => [late]);
|
|
1084
|
+
return [
|
|
1085
|
+
new Error(`agent cleanup exceeded ${CLEANUP_GRACE_MILLISECONDS}ms shutdown grace`),
|
|
1086
|
+
...closed,
|
|
1087
|
+
];
|
|
1088
|
+
}
|
|
1089
|
+
throw error;
|
|
1090
|
+
}
|
|
1091
|
+
})();
|
|
1092
|
+
return this.#closing;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
/**
|
|
1096
|
+
* Runs one stage's work in a scope of its own, which cancels what is still open in it when the
|
|
1097
|
+
* work fails, then writes the stage's record.
|
|
1098
|
+
*/
|
|
1099
|
+
private async runStage(name: string, rest: unknown[]): Promise<JsonValue | undefined> {
|
|
1100
|
+
const [options, work] = (rest.length === 1 ? [undefined, rest[0]] : rest) as [
|
|
1101
|
+
StageOptions<JsonValue> | undefined,
|
|
1102
|
+
unknown,
|
|
1103
|
+
];
|
|
1104
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
1105
|
+
if (typeof work !== "function") throw new Error(`stage ${name} needs its work as a function`);
|
|
1106
|
+
// Before anything is spent on it.
|
|
1107
|
+
let result: JsonSchema | undefined;
|
|
1108
|
+
if (options) {
|
|
1109
|
+
try {
|
|
1110
|
+
result = parseJsonSchema(options.result);
|
|
1111
|
+
} catch (error) {
|
|
1112
|
+
throw new Error(`stage ${name}'s result: ${messageOf(error)}`);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
const parent = scopes.getStore();
|
|
1116
|
+
parent?.assertAccepting();
|
|
1117
|
+
const entered = await this.options.stages.enter(name, {
|
|
1118
|
+
...(result === undefined ? {} : { result }),
|
|
1119
|
+
summary: (value) => summaryOf(name, options, value, this.options.onLog),
|
|
1120
|
+
});
|
|
1121
|
+
if (entered.kind === "reuse") {
|
|
1122
|
+
entered.release();
|
|
1123
|
+
return entered.value;
|
|
1124
|
+
}
|
|
1125
|
+
const open = entered.stage;
|
|
1126
|
+
const messages = this.#messaging.scope(name);
|
|
1127
|
+
const scope = new ExecutionScope(
|
|
1128
|
+
parent?.deadline ?? this.options.deadline,
|
|
1129
|
+
parent?.group,
|
|
1130
|
+
name,
|
|
1131
|
+
messages,
|
|
1132
|
+
);
|
|
1133
|
+
this.#stageScopes.set(messages, scope);
|
|
1134
|
+
const removeFromParent = parent?.add((stop) => scope.cancel(stop));
|
|
1135
|
+
try {
|
|
1136
|
+
const returned: unknown = await scopes.run(scope, () => work());
|
|
1137
|
+
// Its routes take nothing new; what they carry still goes before the stage ends.
|
|
1138
|
+
this.#messaging.finishing(messages);
|
|
1139
|
+
scope.seal();
|
|
1140
|
+
await this.drainMessages(messages, () => scope.settleOwned());
|
|
1141
|
+
if (this.options.stages.stopped?.stage === name) throw this.options.stages.caught();
|
|
1142
|
+
const value = stageValue(name, options, returned);
|
|
1143
|
+
await open.succeed(value, summaryOf(name, options, value, this.options.onLog));
|
|
1144
|
+
return value;
|
|
1145
|
+
} catch (error) {
|
|
1146
|
+
// Its turns settle once cancelled, so its record names the sessions they ran on.
|
|
1147
|
+
await scope.cancel();
|
|
1148
|
+
await waitForDeadline(scope.settleOwned(), {
|
|
1149
|
+
unixMilliseconds: Date.now() + CLEANUP_GRACE_MILLISECONDS,
|
|
1150
|
+
}).catch(() => undefined);
|
|
1151
|
+
const stopped = this.options.stages.stopped;
|
|
1152
|
+
if (stopped && error === stopped && stopped.stage === name) await open.stop(stopped.reason);
|
|
1153
|
+
else {
|
|
1154
|
+
this.options.stages.failedIn(name, error);
|
|
1155
|
+
await open.fail(messageOf(error));
|
|
1156
|
+
}
|
|
1157
|
+
throw error;
|
|
1158
|
+
} finally {
|
|
1159
|
+
this.#messaging.closeScope(messages);
|
|
1160
|
+
this.#stageScopes.delete(messages);
|
|
1161
|
+
removeFromParent?.();
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
private openAgent(spec: AgentOpenSpec): Promise<AgentRef> {
|
|
1166
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
1167
|
+
scopes.getStore()?.assertAccepting();
|
|
1168
|
+
if (this.#caller?.key === spec.key) {
|
|
1169
|
+
throw new Error(`agent ${spec.key} is the calling session; it is not opened`);
|
|
1170
|
+
}
|
|
1171
|
+
const inRunSandbox = this.options.sandboxes.hasRunSandbox;
|
|
1172
|
+
if (inRunSandbox && spec.sandbox !== undefined)
|
|
1173
|
+
throw new Error(`agent ${spec.key}: ${RUN_SANDBOX_ONLY}`);
|
|
1174
|
+
const existing = this.#agents.get(spec.key);
|
|
1175
|
+
if (existing?.stopped !== undefined) throw stoppedError(spec.key, existing.stopped);
|
|
1176
|
+
const execution = existing
|
|
1177
|
+
? constrainExistingExecution(spec.runtime, existing.identity.execution)
|
|
1178
|
+
: withKnownEffort(resolveExecution(spec.runtime, this.options.runtime));
|
|
1179
|
+
checkPaneOptions(spec.key, spec, {
|
|
1180
|
+
headless: placementOf(execution) === "headless",
|
|
1181
|
+
sandboxed:
|
|
1182
|
+
spec.sandbox !== undefined || existing?.identity.sandbox !== undefined || inRunSandbox,
|
|
1183
|
+
});
|
|
1184
|
+
const identity: AgentIdentity = {
|
|
1185
|
+
execution,
|
|
1186
|
+
cwd: spec.cwd ?? this.options.cwd,
|
|
1187
|
+
...(spec.instructions === undefined ? {} : { instructions: spec.instructions }),
|
|
1188
|
+
...(spec.labels === undefined ? {} : { labels: structuredClone(spec.labels) }),
|
|
1189
|
+
...(spec.sandbox === undefined ? {} : { sandbox: spec.sandbox }),
|
|
1190
|
+
...(spec.skills === undefined ? {} : { skills: readSkillSources(spec.skills) }),
|
|
1191
|
+
...(spec.layout === undefined ? {} : { layout: storedLayout(spec.layout) }),
|
|
1192
|
+
...(spec.keepPane === undefined ? {} : { keepPane: spec.keepPane }),
|
|
1193
|
+
};
|
|
1194
|
+
const scope = scopes.getStore();
|
|
1195
|
+
const inheritedDeadline = scope?.deadline ?? this.options.deadline;
|
|
1196
|
+
const effectiveDeadline = spec.deadline
|
|
1197
|
+
? earlierDeadline(spec.deadline, inheritedDeadline)
|
|
1198
|
+
: inheritedDeadline;
|
|
1199
|
+
assertDeadline(effectiveDeadline);
|
|
1200
|
+
if (existing) {
|
|
1201
|
+
assertCompatibleAgent(spec.key, existing.identity, identity, spec);
|
|
1202
|
+
const seated =
|
|
1203
|
+
spec.sandbox === undefined
|
|
1204
|
+
? existing.state
|
|
1205
|
+
: this.assertSameSandbox(spec.key, existing.identity, spec.sandbox).then(
|
|
1206
|
+
() => existing.state,
|
|
1207
|
+
);
|
|
1208
|
+
const attached = this.track(waitForDeadline(seated, effectiveDeadline));
|
|
1209
|
+
scope?.track(attached);
|
|
1210
|
+
return attached;
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
this.options.progress.agentOpened(spec.key, scope?.group);
|
|
1214
|
+
// Opened before the session so registration below stays synchronous: two concurrent `agent()`
|
|
1215
|
+
// calls for one key must not each build an agent.
|
|
1216
|
+
const sessions: AgentSessions = { launcher: new Set(), forks: [] };
|
|
1217
|
+
const accounted: AccountedAgent = {
|
|
1218
|
+
key: spec.key,
|
|
1219
|
+
execution,
|
|
1220
|
+
cwd: identity.cwd,
|
|
1221
|
+
sessions: () => reportedSessions(sessions),
|
|
1222
|
+
};
|
|
1223
|
+
const opening: AgentOpening = {
|
|
1224
|
+
spec,
|
|
1225
|
+
execution,
|
|
1226
|
+
deadline: effectiveDeadline,
|
|
1227
|
+
sessions,
|
|
1228
|
+
ledger: this.options.ledger.agent(accounted),
|
|
1229
|
+
...(identity.skills ? { skills: identity.skills } : {}),
|
|
1230
|
+
activation: (cwd, occupant, skills, home) => ({
|
|
1231
|
+
key: spec.key,
|
|
1232
|
+
deadline: effectiveDeadline,
|
|
1233
|
+
cwd,
|
|
1234
|
+
execution,
|
|
1235
|
+
...(spec.instructions === undefined ? {} : { instructions: spec.instructions }),
|
|
1236
|
+
...(spec.labels === undefined ? {} : { labels: spec.labels }),
|
|
1237
|
+
...(occupant ? { occupant } : {}),
|
|
1238
|
+
...(skills ? { skills } : {}),
|
|
1239
|
+
...(home ? { home } : {}),
|
|
1240
|
+
...paneOptions(identity),
|
|
1241
|
+
}),
|
|
1242
|
+
layoutFallback: this.besideFallback(identity.layout, effectiveDeadline)(),
|
|
1243
|
+
};
|
|
1244
|
+
let skills: Promise<PlacedSkills | undefined> = Promise.resolve(undefined);
|
|
1245
|
+
let opened: { state: Promise<LogicalAgent>; channel: Promise<ResultChannel> };
|
|
1246
|
+
if (spec.sandbox === undefined && !inRunSandbox) {
|
|
1247
|
+
skills = this.placeHostSkills(spec.key, execution, identity.cwd, identity.skills);
|
|
1248
|
+
skills.catch(() => undefined);
|
|
1249
|
+
opened = this.openHostAgent(opening, identity.cwd, accounted, skills);
|
|
1250
|
+
} else {
|
|
1251
|
+
opened = this.openSandboxedAgent(opening, identity.cwd, accounted);
|
|
1252
|
+
}
|
|
1253
|
+
const { state, channel } = opened;
|
|
1254
|
+
const ownedState = this.track(state);
|
|
1255
|
+
this.agentOpened(ownedState, accounted);
|
|
1256
|
+
this.#agents.set(spec.key, {
|
|
1257
|
+
identity,
|
|
1258
|
+
opening: { sessions, accounted, skills },
|
|
1259
|
+
state: ownedState,
|
|
1260
|
+
// Never rejects: the failure is already carried by `state`, and a second copy with no
|
|
1261
|
+
// reader is an unhandled rejection that takes the process down with it.
|
|
1262
|
+
channel: channel.then(
|
|
1263
|
+
(opened) => opened,
|
|
1264
|
+
() => undefined,
|
|
1265
|
+
),
|
|
1266
|
+
});
|
|
1267
|
+
const activated = this.track(waitForDeadline(ownedState, effectiveDeadline));
|
|
1268
|
+
scope?.track(activated);
|
|
1269
|
+
return activated;
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
/**
|
|
1273
|
+
* Ends an agent the workflow has no more use for, as an operation that fails ends it: what it
|
|
1274
|
+
* is running is cancelled, and its pane closed or, where `keepPane` says, kept and released.
|
|
1275
|
+
*/
|
|
1276
|
+
private stopAgent(key: string, reason?: string): Promise<boolean> {
|
|
1277
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
1278
|
+
const scope = scopes.getStore();
|
|
1279
|
+
scope?.assertAccepting();
|
|
1280
|
+
this.options.stages.checkOperation();
|
|
1281
|
+
const deadline = scope?.deadline ?? this.options.deadline;
|
|
1282
|
+
assertDeadline(deadline);
|
|
1283
|
+
const wait = (pending: Promise<boolean>) => {
|
|
1284
|
+
const bounded = this.track(waitForDeadline(pending, deadline));
|
|
1285
|
+
scope?.track(bounded);
|
|
1286
|
+
return bounded;
|
|
1287
|
+
};
|
|
1288
|
+
const entry = this.#agents.get(key);
|
|
1289
|
+
if (!entry || entry.stopComplete) return Promise.resolve(false);
|
|
1290
|
+
if (entry.stopping) return wait(entry.stopping);
|
|
1291
|
+
entry.stopped ??= reason ?? "stopped by the workflow";
|
|
1292
|
+
const stopping = this.track(
|
|
1293
|
+
(async () => {
|
|
1294
|
+
const state = await entry.state.catch(() => undefined);
|
|
1295
|
+
if (!state) {
|
|
1296
|
+
delete entry.stopped;
|
|
1297
|
+
return false;
|
|
1298
|
+
}
|
|
1299
|
+
this.options.progress.agentStopped(key, entry.stopped!);
|
|
1300
|
+
try {
|
|
1301
|
+
await state.close(entry.stopped);
|
|
1302
|
+
} finally {
|
|
1303
|
+
await entry.channel.then((channel) => channel?.close());
|
|
1304
|
+
}
|
|
1305
|
+
if (this.#caller?.key === key) {
|
|
1306
|
+
entry.releasedAt = Date.now();
|
|
1307
|
+
try {
|
|
1308
|
+
await this.options.onCallerStopped?.(entry.stopped!);
|
|
1309
|
+
} catch (error) {
|
|
1310
|
+
this.options.onLog?.(
|
|
1311
|
+
`awf: the calling session was not handed back: ${messageOf(error)}`,
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
entry.stopComplete = true;
|
|
1316
|
+
return true;
|
|
1317
|
+
})(),
|
|
1318
|
+
);
|
|
1319
|
+
entry.stopping = stopping;
|
|
1320
|
+
stopping
|
|
1321
|
+
.finally(() => {
|
|
1322
|
+
if (entry.stopping === stopping) delete entry.stopping;
|
|
1323
|
+
})
|
|
1324
|
+
.catch(() => undefined);
|
|
1325
|
+
return wait(stopping);
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
/**
|
|
1329
|
+
* The session the run was started from, under the key the workflow names: `null` when there is
|
|
1330
|
+
* none. It is found, not opened: its harness, directory and skills are the operator's, and it
|
|
1331
|
+
* answers through a launcher by path as any pane agent does (ADR 0010).
|
|
1332
|
+
*/
|
|
1333
|
+
private caller(key: string): Promise<AgentRef | null> {
|
|
1334
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
1335
|
+
const scope = scopes.getStore();
|
|
1336
|
+
scope?.assertAccepting();
|
|
1337
|
+
const found = this.options.runtime.host.caller;
|
|
1338
|
+
if (!found) return Promise.resolve(null);
|
|
1339
|
+
if (this.#caller) {
|
|
1340
|
+
if (this.#caller.key !== key) {
|
|
1341
|
+
throw new Error(
|
|
1342
|
+
`the calling session is already agent ${this.#caller.key}; it cannot also be ${key}`,
|
|
1343
|
+
);
|
|
1344
|
+
}
|
|
1345
|
+
const stopped = this.#agents.get(key)?.stopped;
|
|
1346
|
+
if (stopped !== undefined) throw stoppedError(key, stopped);
|
|
1347
|
+
return this.#caller.state;
|
|
1348
|
+
}
|
|
1349
|
+
if (this.#agents.has(key))
|
|
1350
|
+
throw new Error(`agent ${key} is already open; the caller needs a key of its own`);
|
|
1351
|
+
const execution: AgentExecution = { harness: found.harness, model: "", caller: true };
|
|
1352
|
+
this.options.progress.agentOpened(key, scope?.group);
|
|
1353
|
+
const sessions: AgentSessions = { launcher: new Set(), forks: [] };
|
|
1354
|
+
const accounted: AccountedAgent = {
|
|
1355
|
+
key,
|
|
1356
|
+
execution,
|
|
1357
|
+
cwd: found.cwd,
|
|
1358
|
+
sessions: () => reportedSessions(sessions),
|
|
1359
|
+
promptedAt: () => sessions.harness?.promptedAt?.(),
|
|
1360
|
+
releasedAt: () => this.#agents.get(key)?.releasedAt,
|
|
1361
|
+
};
|
|
1362
|
+
const ledger = this.options.ledger.agent(accounted);
|
|
1363
|
+
this.options.skills.record(key, "operator");
|
|
1364
|
+
const opened = this.options.control.openChannel(key, (id) => sessions.launcher.add(id));
|
|
1365
|
+
const state = opened.then(async (channel) => {
|
|
1366
|
+
try {
|
|
1367
|
+
const launcher = await installAgentLauncher(
|
|
1368
|
+
dirname(channel.endpoint),
|
|
1369
|
+
channel.endpoint,
|
|
1370
|
+
findHarness(execution.harness)?.sessionEnv,
|
|
1371
|
+
);
|
|
1372
|
+
const session = await this.options.host.openAgent({
|
|
1373
|
+
key,
|
|
1374
|
+
deadline: this.options.deadline,
|
|
1375
|
+
cwd: found.cwd,
|
|
1376
|
+
execution,
|
|
1377
|
+
});
|
|
1378
|
+
sessions.harness = session;
|
|
1379
|
+
return this.logicalAgent(key, execution, session, channel.endpoint, launcher, ledger);
|
|
1380
|
+
} catch (error) {
|
|
1381
|
+
await channel.close().catch(() => undefined);
|
|
1382
|
+
throw error;
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
const owned = this.track(state);
|
|
1386
|
+
this.agentOpened(owned, accounted, { caller: true });
|
|
1387
|
+
this.#caller = { key, state: owned };
|
|
1388
|
+
this.#agents.set(key, {
|
|
1389
|
+
identity: { execution, cwd: found.cwd },
|
|
1390
|
+
opening: { sessions, accounted, skills: Promise.resolve(undefined) },
|
|
1391
|
+
state: owned,
|
|
1392
|
+
channel: opened.then(
|
|
1393
|
+
(channel) => channel,
|
|
1394
|
+
() => undefined,
|
|
1395
|
+
),
|
|
1396
|
+
});
|
|
1397
|
+
const ready = this.track(waitForDeadline(owned, scope?.deadline ?? this.options.deadline));
|
|
1398
|
+
scope?.track(ready);
|
|
1399
|
+
return ready;
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
/**
|
|
1403
|
+
* A fork of the session the run was started from (story 027). Handed over and held as the caller,
|
|
1404
|
+
* it is that agent's own fork, taken in its queue; otherwise it is taken now, from a parent the
|
|
1405
|
+
* run never opened. `null`, and the reason logged, where there is none to fork.
|
|
1406
|
+
*/
|
|
1407
|
+
private forkCaller(spec: AgentForkSpec): Promise<AgentRef | null> {
|
|
1408
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
1409
|
+
const scope = scopes.getStore();
|
|
1410
|
+
scope?.assertAccepting();
|
|
1411
|
+
this.options.stages.checkOperation();
|
|
1412
|
+
if (this.#caller) {
|
|
1413
|
+
const stopped = this.#agents.get(this.#caller.key)?.stopped;
|
|
1414
|
+
if (stopped !== undefined) throw stoppedError(this.#caller.key, stopped);
|
|
1415
|
+
}
|
|
1416
|
+
const calling = this.options.runtime.host.calling;
|
|
1417
|
+
if (calling === undefined || typeof calling === "string") {
|
|
1418
|
+
this.options.onLog?.(
|
|
1419
|
+
`awf: forkCaller: no session to fork${calling === undefined ? "" : `: ${calling}`}`,
|
|
1420
|
+
);
|
|
1421
|
+
return Promise.resolve(null);
|
|
1422
|
+
}
|
|
1423
|
+
if (this.#caller) return this.#caller.state.then((caller) => caller.fork(spec));
|
|
1424
|
+
const execution: AgentExecution = { harness: calling.harness, model: "", caller: true };
|
|
1425
|
+
const parent: AgentEntry = {
|
|
1426
|
+
identity: { execution, cwd: calling.cwd },
|
|
1427
|
+
opening: {
|
|
1428
|
+
sessions: { launcher: new Set(), forks: [] },
|
|
1429
|
+
accounted: { key: CALLING, execution, cwd: calling.cwd, sessions: () => [calling.session] },
|
|
1430
|
+
skills: Promise.resolve(undefined),
|
|
1431
|
+
},
|
|
1432
|
+
state: Promise.reject(new Error("the calling session is not an agent of this run")),
|
|
1433
|
+
channel: Promise.resolve(undefined),
|
|
1434
|
+
};
|
|
1435
|
+
parent.state.catch(() => undefined);
|
|
1436
|
+
const deadline = scope?.deadline ?? this.options.deadline;
|
|
1437
|
+
return this.forkAgent(
|
|
1438
|
+
CALLING,
|
|
1439
|
+
spec,
|
|
1440
|
+
(into) =>
|
|
1441
|
+
calling
|
|
1442
|
+
.fork(deadlineWithin(FORK_TIMEOUT_MS, deadline), into, this.#closingRun.signal)
|
|
1443
|
+
.then((native) => ({ native, after: null })),
|
|
1444
|
+
execution,
|
|
1445
|
+
parent,
|
|
1446
|
+
);
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
/**
|
|
1450
|
+
* An agent on the host: its skills placed first, since a source that will not resolve refuses the
|
|
1451
|
+
* agent before it holds anything, then its channel and session together.
|
|
1452
|
+
*/
|
|
1453
|
+
private openHostAgent(
|
|
1454
|
+
{ spec, execution, sessions, ledger, activation, layoutFallback }: AgentOpening,
|
|
1455
|
+
cwd: string,
|
|
1456
|
+
accounted: AccountedAgent,
|
|
1457
|
+
placed: Promise<PlacedSkills | undefined>,
|
|
1458
|
+
): { state: Promise<LogicalAgent>; channel: Promise<ResultChannel> } {
|
|
1459
|
+
const opened = placed.then((skills) => {
|
|
1460
|
+
if (skills?.given.ownHome) accounted.home = skills.given.ownHome;
|
|
1461
|
+
return this.options.control.openChannel(spec.key, (id) => sessions.launcher.add(id));
|
|
1462
|
+
});
|
|
1463
|
+
const reachable = opened.then(async (channel) => ({
|
|
1464
|
+
channel,
|
|
1465
|
+
// Beside the socket, in the directory the control plane made for this agent alone. Deriving
|
|
1466
|
+
// a directory from the key instead would let two keys that differ only in punctuation share
|
|
1467
|
+
// one, and the second install would point the first agent at the wrong socket.
|
|
1468
|
+
launcher: await installAgentLauncher(
|
|
1469
|
+
dirname(channel.endpoint),
|
|
1470
|
+
channel.endpoint,
|
|
1471
|
+
findHarness(execution.harness)?.sessionEnv,
|
|
1472
|
+
),
|
|
1473
|
+
}));
|
|
1474
|
+
// Read below only once the session exists; a failure before then is reported through `state`.
|
|
1475
|
+
reachable.catch(() => undefined);
|
|
1476
|
+
// An open socket authorizes an agent until something closes it, so every failure path does.
|
|
1477
|
+
const closeChannel = () => opened.then((channel) => channel.close()).catch(() => undefined);
|
|
1478
|
+
const state = placed
|
|
1479
|
+
.then(async (skills) => {
|
|
1480
|
+
const fallback = await layoutFallback;
|
|
1481
|
+
return this.options.host.openAgent({
|
|
1482
|
+
...activation(cwd, undefined, skills?.given, skills?.given.ownHome),
|
|
1483
|
+
...(fallback === undefined ? {} : { layoutFallback: fallback }),
|
|
1484
|
+
});
|
|
1485
|
+
})
|
|
1486
|
+
.catch(async (error: unknown) => {
|
|
1487
|
+
await closeChannel();
|
|
1488
|
+
throw error;
|
|
1489
|
+
})
|
|
1490
|
+
.then(async (session) => {
|
|
1491
|
+
sessions.harness = session;
|
|
1492
|
+
let channel: ResultChannel;
|
|
1493
|
+
let launcher: string;
|
|
1494
|
+
try {
|
|
1495
|
+
({ channel, launcher } = await reachable);
|
|
1496
|
+
} catch (error) {
|
|
1497
|
+
// A session with no way to answer is no agent at all; neither half outlives the other.
|
|
1498
|
+
await Promise.allSettled([session.close(messageOf(error)), closeChannel()]);
|
|
1499
|
+
throw error;
|
|
1500
|
+
}
|
|
1501
|
+
const writeBack = (await placed)?.writeBack;
|
|
1502
|
+
return this.logicalAgent(
|
|
1503
|
+
spec.key,
|
|
1504
|
+
execution,
|
|
1505
|
+
session,
|
|
1506
|
+
channel.endpoint,
|
|
1507
|
+
launcher,
|
|
1508
|
+
ledger,
|
|
1509
|
+
writeBack ? { writeBack } : undefined,
|
|
1510
|
+
);
|
|
1511
|
+
});
|
|
1512
|
+
return { state, channel: opened };
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
/**
|
|
1516
|
+
* Copies an agent's skills where its harness reads them on the host: a directory of the agent's
|
|
1517
|
+
* own, or, for a harness that finds skills only in its home, a home of its own seeded as a
|
|
1518
|
+
* sandboxed agent's is. An agent the workflow named none for keeps the operator's.
|
|
1519
|
+
*/
|
|
1520
|
+
private async placeHostSkills(
|
|
1521
|
+
key: string,
|
|
1522
|
+
execution: AgentExecution,
|
|
1523
|
+
cwd: string,
|
|
1524
|
+
sources: SkillSource[] | undefined,
|
|
1525
|
+
/** A forked session's files, placed in the home it is given with its skills. */
|
|
1526
|
+
carry?: string,
|
|
1527
|
+
): Promise<PlacedSkills | undefined> {
|
|
1528
|
+
if (!sources) {
|
|
1529
|
+
this.options.skills.record(key, "operator");
|
|
1530
|
+
return undefined;
|
|
1531
|
+
}
|
|
1532
|
+
const resolved = await this.options.skills.resolve(sources);
|
|
1533
|
+
const id = randomUUID();
|
|
1534
|
+
// Real paths: claude compares them with its working directory, codex keys its trust by them.
|
|
1535
|
+
const [runDir, realCwd] = await Promise.all([realpath(this.options.runDir), realpath(cwd)]);
|
|
1536
|
+
const given = skillsLayout(
|
|
1537
|
+
execution.harness,
|
|
1538
|
+
resolved.map((skill) => skill.name),
|
|
1539
|
+
{ bundle: join(runDir, "agents", id), cwd: realCwd },
|
|
1540
|
+
);
|
|
1541
|
+
this.options.skills.record(key, resolved, given.ownHome);
|
|
1542
|
+
const { ownHome } = given;
|
|
1543
|
+
if (!ownHome) {
|
|
1544
|
+
await placeSkills(resolved, given.directory);
|
|
1545
|
+
return { given };
|
|
1546
|
+
}
|
|
1547
|
+
await mkdir(dirname(ownHome), { recursive: true });
|
|
1548
|
+
const seeded = await seedHome(
|
|
1549
|
+
ownHome,
|
|
1550
|
+
join(runDir, "staging", id),
|
|
1551
|
+
hostHome(execution.harness, ownHome, this.options.environment),
|
|
1552
|
+
realCwd,
|
|
1553
|
+
this.options.locks,
|
|
1554
|
+
async (staged) => {
|
|
1555
|
+
await placeSkills(resolved, join(staged, relative(ownHome, given.directory)));
|
|
1556
|
+
if (carry) await placeCarried(carry, staged);
|
|
1557
|
+
},
|
|
1558
|
+
);
|
|
1559
|
+
return { given, writeBack: () => seeded.writeBack() };
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
/**
|
|
1563
|
+
* A sandboxed agent opens in order, each step before the next: its sandbox (a private one opened
|
|
1564
|
+
* now), its channel, its door, its home, its admission, then its session. Everything refused
|
|
1565
|
+
* about where it runs is refused before a channel opens.
|
|
1566
|
+
*/
|
|
1567
|
+
private openSandboxedAgent(
|
|
1568
|
+
{
|
|
1569
|
+
spec,
|
|
1570
|
+
execution,
|
|
1571
|
+
deadline,
|
|
1572
|
+
sessions,
|
|
1573
|
+
ledger,
|
|
1574
|
+
activation,
|
|
1575
|
+
skills: sources,
|
|
1576
|
+
carried,
|
|
1577
|
+
}: AgentOpening,
|
|
1578
|
+
cwd: string,
|
|
1579
|
+
accounted: AccountedAgent,
|
|
1580
|
+
): { state: Promise<LogicalAgent>; channel: Promise<ResultChannel> } {
|
|
1581
|
+
const scope = scopes.getStore();
|
|
1582
|
+
const opened = (async () => {
|
|
1583
|
+
// A fork is seated once its parent's session is copied, in its parent's sandbox.
|
|
1584
|
+
await carried?.forked;
|
|
1585
|
+
const skills = sources ? await this.options.skills.resolve(sources) : undefined;
|
|
1586
|
+
const seat = await this.options.sandboxes.seat({
|
|
1587
|
+
key: spec.key,
|
|
1588
|
+
sandbox: spec.sandbox,
|
|
1589
|
+
cwd,
|
|
1590
|
+
execution,
|
|
1591
|
+
...(skills ? { skills } : {}),
|
|
1592
|
+
...(carried ? { joins: carried.parent } : {}),
|
|
1593
|
+
});
|
|
1594
|
+
// Once it has a place: an agent refused one was given nothing. In a sandbox, none named is
|
|
1595
|
+
// none had, as its fresh home holds none of the operator's.
|
|
1596
|
+
this.options.skills.record(spec.key, skills ?? []);
|
|
1597
|
+
// What the agent really ran in, which is where accounting finds its sessions.
|
|
1598
|
+
accounted.cwd = seat.cwd;
|
|
1599
|
+
accounted.home = seat.home;
|
|
1600
|
+
let channel: ResultChannel | undefined;
|
|
1601
|
+
try {
|
|
1602
|
+
channel = await this.options.control.openChannel(spec.key, (id) =>
|
|
1603
|
+
sessions.launcher.add(id),
|
|
1604
|
+
);
|
|
1605
|
+
const door = await installSandboxedDoor(
|
|
1606
|
+
dirname(channel.endpoint),
|
|
1607
|
+
channel.endpoint,
|
|
1608
|
+
await this.bundle(),
|
|
1609
|
+
findHarness(execution.harness)?.sessionEnv,
|
|
1610
|
+
);
|
|
1611
|
+
// The last step before the agent holds anything inside; its bound may be gone by now.
|
|
1612
|
+
assertDeadline(deadline);
|
|
1613
|
+
scope?.assertActive();
|
|
1614
|
+
const seated = await seat.admit(door, carried?.copy);
|
|
1615
|
+
return { seat, seated, channel, launcher: door.launcher };
|
|
1616
|
+
} catch (error) {
|
|
1617
|
+
// The channel first, as at run end; neither undoing masks why the agent did not open.
|
|
1618
|
+
await channel?.close().catch(() => undefined);
|
|
1619
|
+
await seat.abandon();
|
|
1620
|
+
throw error;
|
|
1621
|
+
}
|
|
1622
|
+
})();
|
|
1623
|
+
const state = opened.then(async ({ seat, seated, channel, launcher }) => {
|
|
1624
|
+
let session: HarnessSession;
|
|
1625
|
+
try {
|
|
1626
|
+
session = await this.options.host.openAgent(
|
|
1627
|
+
activation(seat.cwd, seated.occupant, seat.skills, seat.home),
|
|
1628
|
+
);
|
|
1629
|
+
} catch (error) {
|
|
1630
|
+
await seated.release().catch(() => undefined);
|
|
1631
|
+
await channel.close().catch(() => undefined);
|
|
1632
|
+
await seat.abandon();
|
|
1633
|
+
throw error;
|
|
1634
|
+
}
|
|
1635
|
+
sessions.harness = session;
|
|
1636
|
+
return this.logicalAgent(
|
|
1637
|
+
spec.key,
|
|
1638
|
+
execution,
|
|
1639
|
+
session,
|
|
1640
|
+
channel.endpoint,
|
|
1641
|
+
launcher,
|
|
1642
|
+
ledger,
|
|
1643
|
+
seated,
|
|
1644
|
+
);
|
|
1645
|
+
});
|
|
1646
|
+
return { state, channel: opened.then(({ channel }) => channel) };
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
private logicalAgent(
|
|
1650
|
+
key: string,
|
|
1651
|
+
execution: AgentExecution,
|
|
1652
|
+
session: HarnessSession,
|
|
1653
|
+
endpoint: string,
|
|
1654
|
+
launcher: string,
|
|
1655
|
+
ledger: AgentLedger,
|
|
1656
|
+
seated?: Pick<SeatedAgent, "writeBack">,
|
|
1657
|
+
): LogicalAgent {
|
|
1658
|
+
const agent = new LogicalAgent({
|
|
1659
|
+
key,
|
|
1660
|
+
runDir: this.options.runDir,
|
|
1661
|
+
messaging: this.#messaging,
|
|
1662
|
+
messageOwner: (scope) => this.#messageOwner(scope),
|
|
1663
|
+
execution,
|
|
1664
|
+
session,
|
|
1665
|
+
slots: this.options.slots,
|
|
1666
|
+
endpoint,
|
|
1667
|
+
launcher,
|
|
1668
|
+
deadline: this.options.deadline,
|
|
1669
|
+
ledger,
|
|
1670
|
+
progress: this.options.progress,
|
|
1671
|
+
recordTurn: this.options.recordTurn,
|
|
1672
|
+
events: this.options.events,
|
|
1673
|
+
track: (promise) => this.track(promise),
|
|
1674
|
+
isRunClosing: () => this.#closed || this.#cleanupFailures.length > 0,
|
|
1675
|
+
checkOperation: () => this.options.stages.checkOperation(),
|
|
1676
|
+
cleanupUnresolved: (reason) => {
|
|
1677
|
+
const error = new Error(reason);
|
|
1678
|
+
this.#cleanupFailures.push(error);
|
|
1679
|
+
this.options.failRun(error);
|
|
1680
|
+
},
|
|
1681
|
+
livenessPolicy: this.options.livenessPolicy,
|
|
1682
|
+
fork: (spec, take, settings) => this.forkAgent(key, spec, take, settings),
|
|
1683
|
+
keepPane: () => this.#agents.get(key)?.identity.keepPane,
|
|
1684
|
+
// Its socket is its authority: a kept agent's harness lives on, and must not answer.
|
|
1685
|
+
revoke: () =>
|
|
1686
|
+
this.#agents
|
|
1687
|
+
.get(key)
|
|
1688
|
+
?.channel.then((channel) => channel?.close())
|
|
1689
|
+
.catch(() => undefined),
|
|
1690
|
+
...(seated
|
|
1691
|
+
? {
|
|
1692
|
+
// A turn may have refreshed the agent's credential; the operator's copy follows it.
|
|
1693
|
+
afterOperation: () =>
|
|
1694
|
+
seated
|
|
1695
|
+
.writeBack()
|
|
1696
|
+
.catch((error) => this.options.onLog?.(`agent ${key}: ${messageOf(error)}`)),
|
|
1697
|
+
}
|
|
1698
|
+
: {}),
|
|
1699
|
+
});
|
|
1700
|
+
this.#logical.set(key, agent);
|
|
1701
|
+
return agent;
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
/**
|
|
1705
|
+
* A new agent on a copy of `parentKey`'s session (ADR 0009). Its key is taken now, so two forks or
|
|
1706
|
+
* a fork and an open of one key never build two agents; `take` queues the native fork in the
|
|
1707
|
+
* parent's operations, and the child opens after it, so its opening never holds them.
|
|
1708
|
+
*/
|
|
1709
|
+
private forkAgent(
|
|
1710
|
+
parentKey: string,
|
|
1711
|
+
spec: AgentForkSpec,
|
|
1712
|
+
/** Queues the native fork, or its copy `into` another home; or why its host cannot fork. */
|
|
1713
|
+
take: ((into?: SessionCopy) => Promise<TakenFork>) | string,
|
|
1714
|
+
/** The parent's settings once every `set` queued before the fork has run. */
|
|
1715
|
+
parentSettings: AgentExecution,
|
|
1716
|
+
/** A parent the run never opened: the session it was started from. */
|
|
1717
|
+
unopened?: AgentEntry,
|
|
1718
|
+
): Promise<AgentRef> {
|
|
1719
|
+
if (this.#closed || this.#cleanupFailures.length) throw new Error("workflow context is closed");
|
|
1720
|
+
const scope = scopes.getStore();
|
|
1721
|
+
scope?.assertAccepting();
|
|
1722
|
+
this.options.stages.checkOperation();
|
|
1723
|
+
if (this.#caller?.key === spec.key) {
|
|
1724
|
+
throw new Error(`agent ${spec.key} is the calling session; it is not opened`);
|
|
1725
|
+
}
|
|
1726
|
+
const forkSpec = structuredClone(spec);
|
|
1727
|
+
if (forkSpec.layout !== undefined) forkSpec.layout = storedLayout(forkSpec.layout);
|
|
1728
|
+
const deadline = scope?.deadline ?? this.options.deadline;
|
|
1729
|
+
assertDeadline(deadline);
|
|
1730
|
+
// A fork of the calling session is one whichever way it was asked for.
|
|
1731
|
+
const from = unopened || this.#caller?.key === parentKey ? CALLING : parentKey;
|
|
1732
|
+
const existing = this.#agents.get(spec.key);
|
|
1733
|
+
if (existing) {
|
|
1734
|
+
if (existing.stopped !== undefined) throw stoppedError(spec.key, existing.stopped);
|
|
1735
|
+
const forked = existing.identity.forkedFrom;
|
|
1736
|
+
if (forked?.parent !== from || !isDeepStrictEqual(forked.spec, forkSpec)) {
|
|
1737
|
+
throw new Error(`agent ${spec.key} is already open, not as this fork of ${from}`);
|
|
1738
|
+
}
|
|
1739
|
+
const attached = this.track(waitForDeadline(existing.state, deadline));
|
|
1740
|
+
scope?.track(attached);
|
|
1741
|
+
return attached;
|
|
1742
|
+
}
|
|
1743
|
+
const parent = unopened ?? this.#agents.get(parentKey)!;
|
|
1744
|
+
const opening = parent.opening!;
|
|
1745
|
+
if (typeof take === "string") throw new Error(take);
|
|
1746
|
+
const sandboxed = parent.identity.sandbox !== undefined || this.options.sandboxes.hasRunSandbox;
|
|
1747
|
+
// A parent with a harness home of its own has its session copied out, and the fork made in the
|
|
1748
|
+
// child's home: anything added to the parent's would be read as the parent's.
|
|
1749
|
+
const copy =
|
|
1750
|
+
sandboxed || opening.accounted.home !== undefined
|
|
1751
|
+
? join(this.options.runDir, "forks", randomUUID())
|
|
1752
|
+
: undefined;
|
|
1753
|
+
const execution = withKnownEffort(forkExecution(parentSettings, forkSpec));
|
|
1754
|
+
// A fork of the calling session is an agent the run opened, on the model the session's files
|
|
1755
|
+
// show, which the fork reads: `""` until then (ADR 0010).
|
|
1756
|
+
const ofCaller = execution.caller === true;
|
|
1757
|
+
delete execution.caller;
|
|
1758
|
+
checkPaneOptions(spec.key, forkSpec, {
|
|
1759
|
+
headless: placementOf(execution) === "headless",
|
|
1760
|
+
sandboxed,
|
|
1761
|
+
});
|
|
1762
|
+
const identity: AgentIdentity = {
|
|
1763
|
+
execution,
|
|
1764
|
+
cwd: parent.identity.cwd,
|
|
1765
|
+
...(forkSpec.instructions === undefined ? {} : { instructions: forkSpec.instructions }),
|
|
1766
|
+
...(forkSpec.labels === undefined ? {} : { labels: forkSpec.labels }),
|
|
1767
|
+
...(parent.identity.skills ? { skills: parent.identity.skills } : {}),
|
|
1768
|
+
...(parent.identity.sandbox === undefined ? {} : { sandbox: parent.identity.sandbox }),
|
|
1769
|
+
forkedFrom: { parent: from, spec: forkSpec },
|
|
1770
|
+
...(forkSpec.layout === undefined ? {} : { layout: forkSpec.layout }),
|
|
1771
|
+
...(forkSpec.keepPane === undefined ? {} : { keepPane: forkSpec.keepPane }),
|
|
1772
|
+
};
|
|
1773
|
+
|
|
1774
|
+
this.options.progress.agentOpened(spec.key, scope?.group);
|
|
1775
|
+
const sessions: AgentSessions = { launcher: new Set(), forks: [], parent: opening.sessions };
|
|
1776
|
+
opening.sessions.forks.push(sessions);
|
|
1777
|
+
let forkedAt: number | undefined;
|
|
1778
|
+
const accounted: AccountedAgent = {
|
|
1779
|
+
key: spec.key,
|
|
1780
|
+
execution,
|
|
1781
|
+
cwd: identity.cwd,
|
|
1782
|
+
sessions: () => reportedSessions(sessions),
|
|
1783
|
+
forkedAt: () => forkedAt,
|
|
1784
|
+
};
|
|
1785
|
+
const besideAtFork = this.besideFallback(identity.layout, deadline);
|
|
1786
|
+
let continues: NativeFork | undefined;
|
|
1787
|
+
// The fork is asked for now, in the parent's queue; the child opens once it is made. One that
|
|
1788
|
+
// was not made leaves nothing behind: its key may be forked again, after the parent's turn.
|
|
1789
|
+
const forked = take(copy === undefined ? undefined : { directory: copy })
|
|
1790
|
+
.then(({ native: fork, after }) => {
|
|
1791
|
+
if (ofCaller) {
|
|
1792
|
+
if (!fork.model) throw new Error(`agent ${parentKey}'s fork named no model to run on`);
|
|
1793
|
+
// The one object its identity, ledger and activation all hold, before it activates.
|
|
1794
|
+
execution.model = fork.model;
|
|
1795
|
+
}
|
|
1796
|
+
continues = fork;
|
|
1797
|
+
forkedAt = Date.now();
|
|
1798
|
+
this.options.progress.agentForked(spec.key, from);
|
|
1799
|
+
this.options.events.event({
|
|
1800
|
+
kind: "agent-forked",
|
|
1801
|
+
agent: spec.key,
|
|
1802
|
+
from: from === CALLING ? (this.#caller?.key ?? null) : from,
|
|
1803
|
+
after,
|
|
1804
|
+
});
|
|
1805
|
+
return fork;
|
|
1806
|
+
})
|
|
1807
|
+
.catch((error: unknown) => {
|
|
1808
|
+
if (this.#agents.get(spec.key)?.opening?.sessions === sessions) {
|
|
1809
|
+
this.#agents.delete(spec.key);
|
|
1810
|
+
this.options.progress.agentDropped(spec.key);
|
|
1811
|
+
}
|
|
1812
|
+
opening.sessions.forks.splice(opening.sessions.forks.indexOf(sessions), 1);
|
|
1813
|
+
throw error;
|
|
1814
|
+
});
|
|
1815
|
+
const sources = parent.identity.skills;
|
|
1816
|
+
const forkOpening: AgentOpening = {
|
|
1817
|
+
spec: {
|
|
1818
|
+
key: spec.key,
|
|
1819
|
+
runtime: execution,
|
|
1820
|
+
...(parent.identity.sandbox === undefined
|
|
1821
|
+
? {}
|
|
1822
|
+
: { sandbox: parent.identity.sandbox as AgentOpenSpec["sandbox"] }),
|
|
1823
|
+
},
|
|
1824
|
+
execution,
|
|
1825
|
+
deadline,
|
|
1826
|
+
sessions,
|
|
1827
|
+
ledger: this.options.ledger.agent(accounted),
|
|
1828
|
+
...(sources ? { skills: sources } : {}),
|
|
1829
|
+
...(sandboxed && copy ? { carried: { parent: parentKey, forked, copy } } : {}),
|
|
1830
|
+
activation: (cwd, occupant, given, home) => ({
|
|
1831
|
+
key: spec.key,
|
|
1832
|
+
deadline,
|
|
1833
|
+
cwd,
|
|
1834
|
+
execution,
|
|
1835
|
+
...(forkSpec.instructions === undefined ? {} : { instructions: forkSpec.instructions }),
|
|
1836
|
+
...(forkSpec.labels === undefined ? {} : { labels: forkSpec.labels }),
|
|
1837
|
+
...(occupant ? { occupant } : {}),
|
|
1838
|
+
...(given ? { skills: given } : {}),
|
|
1839
|
+
...(home ? { home } : {}),
|
|
1840
|
+
...(continues ? { continues } : {}),
|
|
1841
|
+
...paneOptions(identity),
|
|
1842
|
+
}),
|
|
1843
|
+
// The target as it stands when `fork` is called, waited for once the fork is made.
|
|
1844
|
+
layoutFallback: forked.then(besideAtFork, () => undefined),
|
|
1845
|
+
};
|
|
1846
|
+
let skills: Promise<PlacedSkills | undefined> = Promise.resolve(undefined);
|
|
1847
|
+
let opened: { state: Promise<LogicalAgent>; channel: Promise<ResultChannel> };
|
|
1848
|
+
if (sandboxed) {
|
|
1849
|
+
forked.catch(() => undefined);
|
|
1850
|
+
opened = this.openSandboxedAgent(forkOpening, identity.cwd, accounted);
|
|
1851
|
+
} else {
|
|
1852
|
+
// On the host, its parent's copy of its skills, the same files at the same paths: its
|
|
1853
|
+
// context is its parent's, skills included, and so is its cache. A home of its own, codex's
|
|
1854
|
+
// for skills, holds the same skills and the parent's session copied in.
|
|
1855
|
+
skills = forked.then(async () => {
|
|
1856
|
+
if (copy) return this.placeHostSkills(spec.key, execution, identity.cwd, sources, copy);
|
|
1857
|
+
if (!sources) {
|
|
1858
|
+
this.options.skills.record(spec.key, "operator");
|
|
1859
|
+
return undefined;
|
|
1860
|
+
}
|
|
1861
|
+
this.options.skills.record(spec.key, await this.options.skills.resolve(sources));
|
|
1862
|
+
const placed = await opening.skills;
|
|
1863
|
+
return placed ? { given: placed.given } : undefined;
|
|
1864
|
+
});
|
|
1865
|
+
skills.catch(() => undefined);
|
|
1866
|
+
opened = this.openHostAgent(forkOpening, identity.cwd, accounted, skills);
|
|
1867
|
+
}
|
|
1868
|
+
const { state, channel } = opened;
|
|
1869
|
+
// Once the child's home holds it, or it never will, the copy has served.
|
|
1870
|
+
if (copy) {
|
|
1871
|
+
void state
|
|
1872
|
+
.catch(() => undefined)
|
|
1873
|
+
.finally(() => rm(copy, { recursive: true, force: true }).catch(() => undefined));
|
|
1874
|
+
}
|
|
1875
|
+
const ownedState = this.track(state);
|
|
1876
|
+
this.agentOpened(ownedState, accounted, { fork: true });
|
|
1877
|
+
this.#agents.set(spec.key, {
|
|
1878
|
+
identity,
|
|
1879
|
+
opening: { sessions, accounted, skills },
|
|
1880
|
+
state: ownedState,
|
|
1881
|
+
channel: channel.then(
|
|
1882
|
+
(opened) => opened,
|
|
1883
|
+
() => undefined,
|
|
1884
|
+
),
|
|
1885
|
+
});
|
|
1886
|
+
const activated = this.track(waitForDeadline(ownedState, deadline));
|
|
1887
|
+
scope?.track(activated);
|
|
1888
|
+
return activated;
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
/** Once it is open, where its native files are: a sandboxed agent's are known only then. */
|
|
1892
|
+
private agentOpened(
|
|
1893
|
+
state: Promise<LogicalAgent>,
|
|
1894
|
+
accounted: AccountedAgent,
|
|
1895
|
+
as?: { caller: true } | { fork: true },
|
|
1896
|
+
): void {
|
|
1897
|
+
state.then(
|
|
1898
|
+
() => {
|
|
1899
|
+
this.options.events.event({
|
|
1900
|
+
kind: "agent-opened",
|
|
1901
|
+
agent: accounted.key,
|
|
1902
|
+
callPath: [],
|
|
1903
|
+
harness: accounted.execution.harness,
|
|
1904
|
+
cwd: accounted.cwd,
|
|
1905
|
+
...(accounted.home === undefined ? {} : { home: accounted.home }),
|
|
1906
|
+
...(as && "caller" in as ? { caller: true } : {}),
|
|
1907
|
+
});
|
|
1908
|
+
// A fork's session is made before its first operation names it.
|
|
1909
|
+
if (!as || !("fork" in as)) return;
|
|
1910
|
+
for (const id of accounted.sessions()) {
|
|
1911
|
+
this.options.events.event({
|
|
1912
|
+
kind: "session-known",
|
|
1913
|
+
session: { harness: accounted.execution.harness, id },
|
|
1914
|
+
agent: accounted.key,
|
|
1915
|
+
});
|
|
1916
|
+
}
|
|
1917
|
+
},
|
|
1918
|
+
() => undefined,
|
|
1919
|
+
);
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
private bundle(): Promise<string> {
|
|
1923
|
+
this.#bundle ??= buildAgentBundle();
|
|
1924
|
+
const building = this.#bundle;
|
|
1925
|
+
// A build that failed is tried again by the next agent, not remembered.
|
|
1926
|
+
building.catch(() => {
|
|
1927
|
+
if (this.#bundle === building) this.#bundle = undefined;
|
|
1928
|
+
});
|
|
1929
|
+
return building;
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
/**
|
|
1933
|
+
* Why a pane can't go beside the agent `layout` names, where the engine knows: that agent
|
|
1934
|
+
* is not open now, or has no pane awf places. The wait it returns waits for a target still
|
|
1935
|
+
* opening, so its pane is there to split.
|
|
1936
|
+
*/
|
|
1937
|
+
private besideFallback(
|
|
1938
|
+
layout: PaneLayout | undefined,
|
|
1939
|
+
deadline: AbsoluteDeadline,
|
|
1940
|
+
): () => Promise<string | undefined> {
|
|
1941
|
+
const known = (reason: string | undefined) => () => Promise.resolve(reason);
|
|
1942
|
+
const target = besideOf(layout);
|
|
1943
|
+
if (target === undefined) return known(undefined);
|
|
1944
|
+
if (this.#caller?.key === target) {
|
|
1945
|
+
return known(`${target} is the calling session, whose pane is the operator's`);
|
|
1946
|
+
}
|
|
1947
|
+
const entry = this.#agents.get(target);
|
|
1948
|
+
if (!entry) return known(`${target} is not open`);
|
|
1949
|
+
if (entry.stopped !== undefined) return known(`${target} was stopped`);
|
|
1950
|
+
if (placementOf(entry.identity.execution) === "headless") return known(`${target} is headless`);
|
|
1951
|
+
if (entry.identity.sandbox !== undefined) {
|
|
1952
|
+
return known(`${target} runs in a sandbox, whose panes are in its own Herdr`);
|
|
1953
|
+
}
|
|
1954
|
+
// Registered before this agent, so it can't be waiting on this one in turn.
|
|
1955
|
+
return () =>
|
|
1956
|
+
waitForDeadline(entry.state, deadline).then(
|
|
1957
|
+
() => (entry.stopped === undefined ? undefined : `${target} was stopped`),
|
|
1958
|
+
() => `${target} did not open`,
|
|
1959
|
+
);
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
/** Where each pane agent's pane went, as its host reported it, in the order opened. */
|
|
1963
|
+
panes(): AgentPaneRecord[] {
|
|
1964
|
+
return [...this.#agents].flatMap(([agent, entry]) => {
|
|
1965
|
+
const placed = entry.opening?.sessions.harness?.pane?.();
|
|
1966
|
+
if (!placed) return [];
|
|
1967
|
+
const { layout, keepPane } = entry.identity;
|
|
1968
|
+
return [
|
|
1969
|
+
{
|
|
1970
|
+
callPath: [],
|
|
1971
|
+
agent,
|
|
1972
|
+
...(layout === undefined ? {} : { layout }),
|
|
1973
|
+
...(keepPane === undefined ? {} : { keepPane }),
|
|
1974
|
+
placed,
|
|
1975
|
+
},
|
|
1976
|
+
];
|
|
1977
|
+
});
|
|
1978
|
+
}
|
|
1979
|
+
|
|
1980
|
+
/** A reopened agent names the sandbox it runs in, or none: a different one is a conflict. */
|
|
1981
|
+
private async assertSameSandbox(
|
|
1982
|
+
key: string,
|
|
1983
|
+
existing: AgentIdentity,
|
|
1984
|
+
sandbox: unknown,
|
|
1985
|
+
): Promise<void> {
|
|
1986
|
+
if (!(await this.options.sandboxes.same(key, existing.sandbox, sandbox, existing.cwd))) {
|
|
1987
|
+
conflict(key, "sandbox");
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
private track<T>(promise: Promise<T>): Promise<T> {
|
|
1992
|
+
this.#inFlight.add(promise);
|
|
1993
|
+
promise.finally(() => this.#inFlight.delete(promise)).catch(() => undefined);
|
|
1994
|
+
return promise;
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
class LogicalAgent implements AgentRef {
|
|
1999
|
+
readonly [PARTICIPANT] = true as const;
|
|
2000
|
+
readonly #operationStops = new Set<(stop: OperationStop) => Promise<void>>();
|
|
2001
|
+
readonly #operations = new Map<
|
|
2002
|
+
string,
|
|
2003
|
+
{
|
|
2004
|
+
spec: AgentRunTextSpec | AgentRunStructuredSpec<JsonValue>;
|
|
2005
|
+
result: Promise<RunResult<JsonValue>>;
|
|
2006
|
+
}
|
|
2007
|
+
>();
|
|
2008
|
+
readonly #compactions = new Map<
|
|
2009
|
+
string,
|
|
2010
|
+
{ spec: CompactSpec; result: Promise<TurnOutcome<string>> }
|
|
2011
|
+
>();
|
|
2012
|
+
readonly #sets = new Map<string, { spec: SettingsSpec; result: Promise<TurnOutcome<null>> }>();
|
|
2013
|
+
/** Its settings now; see `AgentRef.execution`. */
|
|
2014
|
+
#execution: AgentExecution;
|
|
2015
|
+
/** What each `set` queued and not yet settled changes, in queue order. */
|
|
2016
|
+
readonly #pendingSets: SettingsChange[] = [];
|
|
2017
|
+
#tail: Promise<void> = Promise.resolve();
|
|
2018
|
+
/** How its last operation ended, which `keepPane: "on-failure"` reads. */
|
|
2019
|
+
#last: TurnOutcome<never>["kind"] | undefined;
|
|
2020
|
+
#closed = false;
|
|
2021
|
+
#closePromise: Promise<void> | undefined;
|
|
2022
|
+
/** Why it closed, once it has. */
|
|
2023
|
+
#closedReason: string | undefined;
|
|
2024
|
+
/** The operation delivered last, which a fork taken now is copied after. */
|
|
2025
|
+
#lastOperation: string | undefined;
|
|
2026
|
+
/** Operations queued or running: a message waits for them rather than waking it. */
|
|
2027
|
+
#queued = 0;
|
|
2028
|
+
/** An operation of its own has been delivered: a message to it continues its session. */
|
|
2029
|
+
#hasRun = false;
|
|
2030
|
+
/** Supervisors of its open operation, woken when something arrives for it. */
|
|
2031
|
+
readonly #watchers = new Set<() => void>();
|
|
2032
|
+
|
|
2033
|
+
constructor(
|
|
2034
|
+
private readonly options: {
|
|
2035
|
+
key: string;
|
|
2036
|
+
runDir: string;
|
|
2037
|
+
messaging: Messaging;
|
|
2038
|
+
/**
|
|
2039
|
+
* What owns a message operation a route in `scope` opens: its stage, or the attempt outside
|
|
2040
|
+
* any stage. Undefined once that scope has closed: nothing opens for it then.
|
|
2041
|
+
*/
|
|
2042
|
+
messageOwner(
|
|
2043
|
+
scope: MessagingScope,
|
|
2044
|
+
): { stage?: ExecutionScope; adopt(operation: Promise<unknown>): void } | undefined;
|
|
2045
|
+
execution: AgentExecution;
|
|
2046
|
+
session: HarnessSession;
|
|
2047
|
+
slots: ResultSlotRegistry;
|
|
2048
|
+
endpoint: string;
|
|
2049
|
+
/** The path this agent is told to run; see `installAgentLauncher`. */
|
|
2050
|
+
launcher: string;
|
|
2051
|
+
deadline: AbsoluteDeadline;
|
|
2052
|
+
ledger: AgentLedger;
|
|
2053
|
+
progress: RunProgress;
|
|
2054
|
+
recordTurn: RecordTurn;
|
|
2055
|
+
events: RunEventLog;
|
|
2056
|
+
track<T>(promise: Promise<T>): Promise<T>;
|
|
2057
|
+
isRunClosing(): boolean;
|
|
2058
|
+
/** Throws when no turn may start: the attempt is looking on for values it needs. */
|
|
2059
|
+
checkOperation(): void;
|
|
2060
|
+
cleanupUnresolved(reason: string): void;
|
|
2061
|
+
livenessPolicy?: LivenessPolicy;
|
|
2062
|
+
/** After each operation settles, whatever its outcome. */
|
|
2063
|
+
afterOperation?(): Promise<void>;
|
|
2064
|
+
/**
|
|
2065
|
+
* Opens `spec`'s agent on the native fork `take` queues in this agent's operations, at
|
|
2066
|
+
* `settings`, this agent's once the operations before the fork have run.
|
|
2067
|
+
*/
|
|
2068
|
+
fork(
|
|
2069
|
+
spec: AgentForkSpec,
|
|
2070
|
+
take: ((into?: SessionCopy) => Promise<TakenFork>) | string,
|
|
2071
|
+
settings: AgentExecution,
|
|
2072
|
+
): Promise<AgentRef>;
|
|
2073
|
+
/** As the workflow wrote it; decides, at close, whether its pane stays. */
|
|
2074
|
+
keepPane(): KeepPane | undefined;
|
|
2075
|
+
/** Closes its result channel, once its pane is kept. */
|
|
2076
|
+
revoke(): Promise<void> | undefined;
|
|
2077
|
+
},
|
|
2078
|
+
) {
|
|
2079
|
+
this.#execution = options.execution;
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
get key(): string {
|
|
2083
|
+
return this.options.key;
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
get execution(): AgentExecution {
|
|
2087
|
+
return this.#execution;
|
|
2088
|
+
}
|
|
2089
|
+
|
|
2090
|
+
get launcher(): string {
|
|
2091
|
+
return this.options.launcher;
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
/** It can confirm what a pane was sent, as waiting in a pane needs. */
|
|
2095
|
+
get confirmsDelivery(): boolean {
|
|
2096
|
+
return this.options.session.supportsWaiting === true;
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
get busy(): boolean {
|
|
2100
|
+
return this.#queued > 0;
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
get hasRun(): boolean {
|
|
2104
|
+
return this.#hasRun;
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
get closedReason(): string | undefined {
|
|
2108
|
+
return this.#closed ? (this.#closedReason ?? "closed") : undefined;
|
|
2109
|
+
}
|
|
2110
|
+
|
|
2111
|
+
/**
|
|
2112
|
+
* Something arrived for it: its open operation's supervisor looks again, and an agent with
|
|
2113
|
+
* nothing queued wakes for a held message, in a message operation.
|
|
2114
|
+
*/
|
|
2115
|
+
messagesChanged(): void {
|
|
2116
|
+
for (const wake of this.#watchers) wake();
|
|
2117
|
+
if (this.#queued > 0 || this.#closed || this.options.isRunClosing()) return;
|
|
2118
|
+
const opening = this.options.messaging.opening(this.key);
|
|
2119
|
+
if (opening) this.startMessageOperation(opening.scope);
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
enqueue(_spec: AgentTextTurnSpec): Promise<never>;
|
|
2123
|
+
enqueue<T extends JsonValue>(_spec: AgentStructuredTurnSpec<T>): Promise<never>;
|
|
2124
|
+
enqueue(): Promise<never> {
|
|
2125
|
+
return unavailable("AgentRef.enqueue");
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
run(spec: AgentRunTextSpec): Promise<RunResult<string>>;
|
|
2129
|
+
run<T extends JsonValue>(spec: AgentRunStructuredSpec<T>): Promise<RunResult<T>>;
|
|
2130
|
+
run<T extends JsonValue>(
|
|
2131
|
+
spec: AgentRunTextSpec | AgentRunStructuredSpec<T>,
|
|
2132
|
+
): Promise<RunResult<string | T>> {
|
|
2133
|
+
if (this.#closed) return Promise.reject(new Error("logical agent is closed"));
|
|
2134
|
+
const scope = scopes.getStore();
|
|
2135
|
+
try {
|
|
2136
|
+
scope?.assertAccepting();
|
|
2137
|
+
this.options.checkOperation();
|
|
2138
|
+
} catch (error) {
|
|
2139
|
+
return Promise.reject(error);
|
|
2140
|
+
}
|
|
2141
|
+
const id = spec.id ?? randomUUID();
|
|
2142
|
+
const completeSpec = structuredClone({ ...spec, id }) as
|
|
2143
|
+
| AgentRunTextSpec
|
|
2144
|
+
| AgentRunStructuredSpec<JsonValue>;
|
|
2145
|
+
const existing = this.#operations.get(id);
|
|
2146
|
+
if (existing) {
|
|
2147
|
+
if (!isDeepStrictEqual(existing.spec, completeSpec)) {
|
|
2148
|
+
const rejected = Promise.reject<RunResult<string | T>>(
|
|
2149
|
+
new Error(`turn id ${id} was reused with a different specification`),
|
|
2150
|
+
);
|
|
2151
|
+
scope?.track(rejected);
|
|
2152
|
+
return rejected;
|
|
2153
|
+
}
|
|
2154
|
+
scope?.track(existing.result);
|
|
2155
|
+
return existing.result as Promise<RunResult<string | T>>;
|
|
2156
|
+
}
|
|
2157
|
+
let deadline: AbsoluteDeadline;
|
|
2158
|
+
try {
|
|
2159
|
+
deadline = this.operationDeadline(spec, scope);
|
|
2160
|
+
const recoveryDeadline = spec.nudge ? spec.nudge.deadline : undefined;
|
|
2161
|
+
if (recoveryDeadline) {
|
|
2162
|
+
assertDeadlineValue(recoveryDeadline);
|
|
2163
|
+
deadline = earlierDeadline(deadline, recoveryDeadline);
|
|
2164
|
+
}
|
|
2165
|
+
} catch (error) {
|
|
2166
|
+
const rejected = Promise.reject<RunResult<string | T>>(error);
|
|
2167
|
+
scope?.track(rejected);
|
|
2168
|
+
return rejected;
|
|
2169
|
+
}
|
|
2170
|
+
const result = this.queue(async () => {
|
|
2171
|
+
if (this.#closed || this.options.isRunClosing()) {
|
|
2172
|
+
throw new Error("logical agent is closed");
|
|
2173
|
+
}
|
|
2174
|
+
scope?.assertActive();
|
|
2175
|
+
const tags = tagsOf(scope, completeSpec.label);
|
|
2176
|
+
// Drawn here, so a turn that throws before it is delivered is still written as failed.
|
|
2177
|
+
const operation = this.operationLog(randomUUID(), "turn", tags);
|
|
2178
|
+
this.options.progress.turnStarted(this.key, tags);
|
|
2179
|
+
try {
|
|
2180
|
+
const settled = await this.executeOperation(completeSpec, scope, deadline, operation);
|
|
2181
|
+
this.turnEnded(settled.outcome, "turn");
|
|
2182
|
+
return settled;
|
|
2183
|
+
} catch (error) {
|
|
2184
|
+
operation.failed(error);
|
|
2185
|
+
this.options.progress.turnSettled(this.key, "failed", messageOf(error));
|
|
2186
|
+
throw error;
|
|
2187
|
+
} finally {
|
|
2188
|
+
await this.options.afterOperation?.();
|
|
2189
|
+
}
|
|
2190
|
+
});
|
|
2191
|
+
const tracked = this.options.track(result);
|
|
2192
|
+
scope?.track(tracked);
|
|
2193
|
+
this.#operations.set(id, { spec: completeSpec, result: tracked });
|
|
2194
|
+
return tracked as Promise<RunResult<string | T>>;
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
compact(spec: CompactSpec): Promise<TurnOutcome<string>> {
|
|
2198
|
+
if (this.#closed) return Promise.reject(new Error("logical agent is closed"));
|
|
2199
|
+
const scope = scopes.getStore();
|
|
2200
|
+
try {
|
|
2201
|
+
scope?.assertAccepting();
|
|
2202
|
+
this.options.checkOperation();
|
|
2203
|
+
} catch (error) {
|
|
2204
|
+
return Promise.reject(error);
|
|
2205
|
+
}
|
|
2206
|
+
const id = spec.id ?? randomUUID();
|
|
2207
|
+
const completeSpec = structuredClone({ ...spec, id });
|
|
2208
|
+
const existing = this.#compactions.get(id);
|
|
2209
|
+
if (existing) {
|
|
2210
|
+
if (!isDeepStrictEqual(existing.spec, completeSpec)) {
|
|
2211
|
+
const rejected = Promise.reject<TurnOutcome<string>>(
|
|
2212
|
+
new Error(`compaction id ${id} was reused with a different specification`),
|
|
2213
|
+
);
|
|
2214
|
+
scope?.track(rejected);
|
|
2215
|
+
return rejected;
|
|
2216
|
+
}
|
|
2217
|
+
scope?.track(existing.result);
|
|
2218
|
+
return existing.result;
|
|
2219
|
+
}
|
|
2220
|
+
let deadline: AbsoluteDeadline;
|
|
2221
|
+
try {
|
|
2222
|
+
deadline = this.operationDeadline(spec, scope);
|
|
2223
|
+
} catch (error) {
|
|
2224
|
+
const rejected = Promise.reject<TurnOutcome<string>>(error);
|
|
2225
|
+
scope?.track(rejected);
|
|
2226
|
+
return rejected;
|
|
2227
|
+
}
|
|
2228
|
+
const result = this.queue(async () => {
|
|
2229
|
+
if (this.#closed || this.options.isRunClosing()) {
|
|
2230
|
+
throw new Error("logical agent is closed");
|
|
2231
|
+
}
|
|
2232
|
+
scope?.assertActive();
|
|
2233
|
+
const tags = tagsOf(scope);
|
|
2234
|
+
const operation = this.operationLog(randomUUID(), "compact", tags);
|
|
2235
|
+
this.options.progress.turnStarted(this.key, tags, "compact");
|
|
2236
|
+
try {
|
|
2237
|
+
const outcome = await this.executeCompaction(id, spec.prompt, deadline, scope, operation);
|
|
2238
|
+
this.turnEnded(outcome, "compact");
|
|
2239
|
+
return outcome;
|
|
2240
|
+
} catch (error) {
|
|
2241
|
+
operation.failed(error);
|
|
2242
|
+
this.options.progress.turnSettled(this.key, "failed", messageOf(error));
|
|
2243
|
+
throw error;
|
|
2244
|
+
} finally {
|
|
2245
|
+
await this.options.afterOperation?.();
|
|
2246
|
+
}
|
|
2247
|
+
});
|
|
2248
|
+
const tracked = this.options.track(result);
|
|
2249
|
+
scope?.track(tracked);
|
|
2250
|
+
this.#compactions.set(id, { spec: completeSpec, result: tracked });
|
|
2251
|
+
return tracked;
|
|
2252
|
+
}
|
|
2253
|
+
|
|
2254
|
+
/** Not a turn: progress counts none for it, and it asks the model nothing. */
|
|
2255
|
+
set(spec: SettingsSpec): Promise<TurnOutcome<null>> {
|
|
2256
|
+
if (this.#closed) return Promise.reject(new Error("logical agent is closed"));
|
|
2257
|
+
const scope = scopes.getStore();
|
|
2258
|
+
try {
|
|
2259
|
+
scope?.assertAccepting();
|
|
2260
|
+
} catch (error) {
|
|
2261
|
+
return Promise.reject(error);
|
|
2262
|
+
}
|
|
2263
|
+
const id = spec.id ?? randomUUID();
|
|
2264
|
+
const completeSpec = structuredClone({ ...spec, id });
|
|
2265
|
+
const existing = this.#sets.get(id);
|
|
2266
|
+
if (existing) {
|
|
2267
|
+
if (!isDeepStrictEqual(existing.spec, completeSpec)) {
|
|
2268
|
+
const rejected = Promise.reject<TurnOutcome<null>>(
|
|
2269
|
+
new Error(`settings id ${id} was reused with a different specification`),
|
|
2270
|
+
);
|
|
2271
|
+
scope?.track(rejected);
|
|
2272
|
+
return rejected;
|
|
2273
|
+
}
|
|
2274
|
+
scope?.track(existing.result);
|
|
2275
|
+
return existing.result;
|
|
2276
|
+
}
|
|
2277
|
+
let change: SettingsChange;
|
|
2278
|
+
let deadline: AbsoluteDeadline;
|
|
2279
|
+
try {
|
|
2280
|
+
change = this.settingsChange(spec);
|
|
2281
|
+
deadline = this.operationDeadline(spec, scope);
|
|
2282
|
+
} catch (error) {
|
|
2283
|
+
const rejected = Promise.reject<TurnOutcome<null>>(error);
|
|
2284
|
+
scope?.track(rejected);
|
|
2285
|
+
return rejected;
|
|
2286
|
+
}
|
|
2287
|
+
this.#pendingSets.push(change);
|
|
2288
|
+
const result = this.queue(async () => {
|
|
2289
|
+
try {
|
|
2290
|
+
if (this.#closed || this.options.isRunClosing()) {
|
|
2291
|
+
throw new Error("logical agent is closed");
|
|
2292
|
+
}
|
|
2293
|
+
scope?.assertActive();
|
|
2294
|
+
const outcome = await this.executeSet(change, deadline, scope);
|
|
2295
|
+
this.#last = outcome.kind;
|
|
2296
|
+
return outcome;
|
|
2297
|
+
} finally {
|
|
2298
|
+
this.#pendingSets.splice(this.#pendingSets.indexOf(change), 1);
|
|
2299
|
+
}
|
|
2300
|
+
});
|
|
2301
|
+
const tracked = this.options.track(result);
|
|
2302
|
+
scope?.track(tracked);
|
|
2303
|
+
this.#sets.set(id, { spec: completeSpec, result: tracked });
|
|
2304
|
+
return tracked;
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
/**
|
|
2308
|
+
* An operation's start and failure in the run's event log, under the id its record keeps. Its
|
|
2309
|
+
* settling is the turn line `recordTurn` writes.
|
|
2310
|
+
*/
|
|
2311
|
+
private operationLog(
|
|
2312
|
+
operationId: string,
|
|
2313
|
+
of: TurnRecord["kind"],
|
|
2314
|
+
tags: OperationTags,
|
|
2315
|
+
): OperationLog {
|
|
2316
|
+
let deliveredAt: number | undefined;
|
|
2317
|
+
/** Named by its host before delivery, for its start to carry. */
|
|
2318
|
+
const named: string[] = [];
|
|
2319
|
+
const fields = () => {
|
|
2320
|
+
const { callPath, sessions, ...record } = this.options.ledger.unsettled(
|
|
2321
|
+
operationId,
|
|
2322
|
+
this.#execution,
|
|
2323
|
+
tags,
|
|
2324
|
+
deliveredAt,
|
|
2325
|
+
);
|
|
2326
|
+
const harness = this.#execution.harness;
|
|
2327
|
+
const fresh = named.filter((id) => !sessions.some((known) => known.id === id));
|
|
2328
|
+
return {
|
|
2329
|
+
of,
|
|
2330
|
+
callPath,
|
|
2331
|
+
...record,
|
|
2332
|
+
sessions: [...sessions, ...fresh.map((id) => ({ harness, id }))],
|
|
2333
|
+
};
|
|
2334
|
+
};
|
|
2335
|
+
return {
|
|
2336
|
+
operationId,
|
|
2337
|
+
tags,
|
|
2338
|
+
delivered: (at: number) => {
|
|
2339
|
+
deliveredAt = at;
|
|
2340
|
+
this.#hasRun = true;
|
|
2341
|
+
this.#lastOperation = operationId;
|
|
2342
|
+
this.options.events.event({ kind: "operation-started", ...fields() }, at);
|
|
2343
|
+
},
|
|
2344
|
+
session: (id: string) => {
|
|
2345
|
+
if (deliveredAt === undefined) {
|
|
2346
|
+
named.push(id);
|
|
2347
|
+
return;
|
|
2348
|
+
}
|
|
2349
|
+
this.options.events.event({
|
|
2350
|
+
kind: "session-known",
|
|
2351
|
+
session: { harness: this.#execution.harness, id },
|
|
2352
|
+
operationId,
|
|
2353
|
+
});
|
|
2354
|
+
},
|
|
2355
|
+
failed: (error: unknown) =>
|
|
2356
|
+
this.options.events.event({
|
|
2357
|
+
kind: "operation-failed",
|
|
2358
|
+
...fields(),
|
|
2359
|
+
error: messageOf(error),
|
|
2360
|
+
}),
|
|
2361
|
+
};
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
/** A turn or compaction that settled, as the view shows it and `turns.jsonl` keeps it. */
|
|
2365
|
+
private turnEnded(outcome: TurnOutcome<JsonValue>, kind: TurnRecord["kind"]): void {
|
|
2366
|
+
this.#last = outcome.kind;
|
|
2367
|
+
const reason = "reason" in outcome ? outcome.reason : undefined;
|
|
2368
|
+
this.options.progress.turnSettled(this.key, outcome.kind, reason);
|
|
2369
|
+
this.options.recordTurn(outcome, kind);
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
fork(spec: AgentForkSpec): Promise<AgentRef> {
|
|
2373
|
+
if (this.#closed) return Promise.reject(new Error("logical agent is closed"));
|
|
2374
|
+
const scope = scopes.getStore();
|
|
2375
|
+
const deadline = this.deadlineCeiling(scope);
|
|
2376
|
+
const { session, execution } = this.options;
|
|
2377
|
+
const settings = this.queuedExecution();
|
|
2378
|
+
const fork = session.fork?.bind(session);
|
|
2379
|
+
if (!fork) {
|
|
2380
|
+
try {
|
|
2381
|
+
return this.options.fork(
|
|
2382
|
+
spec,
|
|
2383
|
+
execution.caller
|
|
2384
|
+
? `agent ${this.key} is the calling session, which this run cannot fork; forkCaller answers null and the run's output says why`
|
|
2385
|
+
: `agent ${this.key} cannot be forked: ${execution.harness} ${placementOf(execution)} agents have no fork yet`,
|
|
2386
|
+
settings,
|
|
2387
|
+
);
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
return Promise.reject(error);
|
|
2390
|
+
}
|
|
2391
|
+
}
|
|
2392
|
+
// The harness bounds its fork by this deadline and stops it there, so the session is idle again
|
|
2393
|
+
// when the fork rejects.
|
|
2394
|
+
const take = (into?: SessionCopy) =>
|
|
2395
|
+
this.queue(async () => {
|
|
2396
|
+
if (this.#closed || this.options.isRunClosing()) {
|
|
2397
|
+
throw new Error("logical agent is closed");
|
|
2398
|
+
}
|
|
2399
|
+
scope?.assertActive();
|
|
2400
|
+
// The fork was opened at the settings it would copy; a `set` before it that did not take
|
|
2401
|
+
// would leave it recording settings it never ran at.
|
|
2402
|
+
if (!isDeepStrictEqual(sessionSettings(this.#execution), sessionSettings(settings))) {
|
|
2403
|
+
throw new Error(
|
|
2404
|
+
`agent ${this.key} was not at the settings its fork was opened at: a set queued before the fork did not take`,
|
|
2405
|
+
);
|
|
2406
|
+
}
|
|
2407
|
+
const after = this.#lastOperation ?? null;
|
|
2408
|
+
return { native: await fork(deadlineWithin(FORK_TIMEOUT_MS, deadline), into), after };
|
|
2409
|
+
});
|
|
2410
|
+
try {
|
|
2411
|
+
return this.options.fork(spec, take, settings);
|
|
2412
|
+
} catch (error) {
|
|
2413
|
+
return Promise.reject(error);
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
|
|
2417
|
+
/** `deadline` or `timeoutMs`, never later than the scope's or the run's; else the earlier of those. */
|
|
2418
|
+
private operationDeadline(
|
|
2419
|
+
spec: { deadline?: AbsoluteDeadline; timeoutMs?: number },
|
|
2420
|
+
scope: ExecutionScope | undefined,
|
|
2421
|
+
): AbsoluteDeadline {
|
|
2422
|
+
if (spec.deadline && spec.timeoutMs !== undefined) {
|
|
2423
|
+
throw new Error("an operation cannot specify both deadline and timeoutMs");
|
|
2424
|
+
}
|
|
2425
|
+
const ceiling = this.deadlineCeiling(scope);
|
|
2426
|
+
if (spec.timeoutMs !== undefined) return deadlineWithin(spec.timeoutMs, ceiling);
|
|
2427
|
+
if (!spec.deadline) return ceiling;
|
|
2428
|
+
assertDeadlineValue(spec.deadline);
|
|
2429
|
+
return earlierDeadline(spec.deadline, ceiling);
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2432
|
+
private deadlineCeiling(scope: ExecutionScope | undefined): AbsoluteDeadline {
|
|
2433
|
+
return scope ? earlierDeadline(scope.deadline, this.options.deadline) : this.options.deadline;
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
/** Its settings once every `set` queued so far has run. */
|
|
2437
|
+
private queuedExecution(): AgentExecution {
|
|
2438
|
+
return Object.assign({}, this.#execution, ...this.#pendingSets);
|
|
2439
|
+
}
|
|
2440
|
+
|
|
2441
|
+
/** What `spec` changes, or why it is refused before it is queued. */
|
|
2442
|
+
private settingsChange(spec: SettingsSpec): SettingsChange {
|
|
2443
|
+
const { execution, session } = this.options;
|
|
2444
|
+
if (execution.caller) {
|
|
2445
|
+
throw new Error(
|
|
2446
|
+
`agent ${this.key} is the calling session, whose model and effort are the operator's`,
|
|
2447
|
+
);
|
|
2448
|
+
}
|
|
2449
|
+
// A workflow is untyped JavaScript at run time: a harness or placement asked for must not be
|
|
2450
|
+
// dropped in silence.
|
|
2451
|
+
const others = Object.keys(spec).filter((field) => !SETTINGS_FIELDS.has(field));
|
|
2452
|
+
if (others.length > 0) {
|
|
2453
|
+
throw new Error(
|
|
2454
|
+
`set changes an agent's model and effort, not its ${others.join(", ")}: another harness or placement is another agent`,
|
|
2455
|
+
);
|
|
2456
|
+
}
|
|
2457
|
+
if (spec.model === undefined && spec.effort === undefined) {
|
|
2458
|
+
throw new Error("set names neither a model nor an effort");
|
|
2459
|
+
}
|
|
2460
|
+
if (spec.model !== undefined && (typeof spec.model !== "string" || spec.model === "")) {
|
|
2461
|
+
throw new Error(`set names no model: ${JSON.stringify(spec.model)}`);
|
|
2462
|
+
}
|
|
2463
|
+
if (spec.effort !== undefined && typeof spec.effort !== "string") {
|
|
2464
|
+
throw new Error(`an effort is a level's name, not ${JSON.stringify(spec.effort)}`);
|
|
2465
|
+
}
|
|
2466
|
+
const refused =
|
|
2467
|
+
(spec.effort === undefined ? undefined : effortRefusal(execution.harness, spec.effort)) ??
|
|
2468
|
+
(session.set
|
|
2469
|
+
? undefined
|
|
2470
|
+
: (settingsRefusal(execution) ??
|
|
2471
|
+
`${execution.harness} ${placementOf(execution)} agents cannot switch model or effort yet`));
|
|
2472
|
+
if (refused) throw new Error(`agent ${this.key}: ${refused}`);
|
|
2473
|
+
return {
|
|
2474
|
+
...(spec.model === undefined ? {} : { model: spec.model }),
|
|
2475
|
+
...(spec.effort === undefined ? {} : { effort: spec.effort }),
|
|
2476
|
+
};
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
/**
|
|
2480
|
+
* Switches the session's settings. Nothing answers through a result slot: the host confirms the
|
|
2481
|
+
* switch by resolving. A pane's that did not is in settings nobody knows, so it is closed.
|
|
2482
|
+
*/
|
|
2483
|
+
private async executeSet(
|
|
2484
|
+
change: SettingsChange,
|
|
2485
|
+
deadline: AbsoluteDeadline,
|
|
2486
|
+
scope: ExecutionScope | undefined,
|
|
2487
|
+
): Promise<TurnOutcome<null>> {
|
|
2488
|
+
const entry = this.options.ledger.reserve(randomUUID(), () => this.#execution, tagsOf(scope));
|
|
2489
|
+
const usage = () =>
|
|
2490
|
+
entry.settle({ settledAt: Math.min(Date.now(), deadline.unixMilliseconds) }, []);
|
|
2491
|
+
if (Date.now() >= deadline.unixMilliseconds) {
|
|
2492
|
+
return { kind: "timed-out", reason: "settings deadline exceeded", usage: usage() };
|
|
2493
|
+
}
|
|
2494
|
+
const next: AgentExecution = { ...this.#execution, ...change };
|
|
2495
|
+
const pane = placementOf(next) === "pane";
|
|
2496
|
+
let cancelled: string | undefined;
|
|
2497
|
+
// A pane's switch waits on its screen; a scope cancelled meanwhile closes the agent, as a
|
|
2498
|
+
// half-made switch leaves it at settings nobody knows. A headless one takes no time.
|
|
2499
|
+
const removeCanceller = pane
|
|
2500
|
+
? scope?.add(async (stop) => {
|
|
2501
|
+
cancelled = stop.reason;
|
|
2502
|
+
this.#last = "cancelled";
|
|
2503
|
+
await this.close(stop.reason);
|
|
2504
|
+
})
|
|
2505
|
+
: undefined;
|
|
2506
|
+
try {
|
|
2507
|
+
await waitForDeadline(
|
|
2508
|
+
Promise.resolve().then(() => this.options.session.set!(sessionSettings(next), deadline)),
|
|
2509
|
+
deadline,
|
|
2510
|
+
);
|
|
2511
|
+
} catch (error) {
|
|
2512
|
+
const expired = error instanceof DeadlineExceededError;
|
|
2513
|
+
const reason = cancelled ?? (expired ? "settings deadline exceeded" : messageOf(error));
|
|
2514
|
+
if (pane) {
|
|
2515
|
+
this.closeAfterFailure(
|
|
2516
|
+
reason,
|
|
2517
|
+
cancelled !== undefined ? "cancelled" : expired ? "timed-out" : "failed",
|
|
2518
|
+
);
|
|
2519
|
+
}
|
|
2520
|
+
if (cancelled !== undefined) return { kind: "cancelled", reason, usage: usage() };
|
|
2521
|
+
return expired
|
|
2522
|
+
? { kind: "timed-out", reason, usage: usage() }
|
|
2523
|
+
: { kind: "failed", reason, retryable: false, usage: usage() };
|
|
2524
|
+
} finally {
|
|
2525
|
+
removeCanceller?.();
|
|
2526
|
+
}
|
|
2527
|
+
this.#execution = next;
|
|
2528
|
+
return { kind: "answered", value: null, usage: usage() };
|
|
2529
|
+
}
|
|
2530
|
+
|
|
2531
|
+
/**
|
|
2532
|
+
* An operation that failed, was cancelled or timed out ends a session the run owns. The calling
|
|
2533
|
+
* session is the operator's and goes on: its next turn waits for it to settle (ADR 0010).
|
|
2534
|
+
*/
|
|
2535
|
+
private closeAfterFailure(reason: string, ended: TurnOutcome<never>["kind"] = "failed"): void {
|
|
2536
|
+
if (this.options.execution.caller) return;
|
|
2537
|
+
// Before its outcome is recorded: whether its pane stays turns on how this one ended.
|
|
2538
|
+
this.#last = ended;
|
|
2539
|
+
void this.close(reason).catch(() => undefined);
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
close(reason?: string): Promise<void> {
|
|
2543
|
+
const first = !this.#closed;
|
|
2544
|
+
this.#closed = true;
|
|
2545
|
+
if (first) {
|
|
2546
|
+
this.#closedReason = reason ?? "closed";
|
|
2547
|
+
// Its routes close, what it asked is dropped and what it owed fails, before the operation it
|
|
2548
|
+
// is running ends, so an answer it asked back on still reaches its asker.
|
|
2549
|
+
this.options.messaging.agentStopped(this.key, this.#closedReason);
|
|
2550
|
+
}
|
|
2551
|
+
// An operation still going is cancelled by this close.
|
|
2552
|
+
const keep = keepsPane(
|
|
2553
|
+
this.options.keepPane(),
|
|
2554
|
+
this.#operationStops.size > 0 ? "cancelled" : this.#last,
|
|
2555
|
+
);
|
|
2556
|
+
const stopped = this.stopOperations({ kind: "cancelled", reason: reason ?? "agent closed" });
|
|
2557
|
+
if (!this.#closePromise) {
|
|
2558
|
+
let attempt: Promise<void>;
|
|
2559
|
+
try {
|
|
2560
|
+
attempt = Promise.resolve(
|
|
2561
|
+
this.options.session.close(reason, keep ? { keep: true } : undefined),
|
|
2562
|
+
).then(() => (keep ? this.options.revoke() : undefined));
|
|
2563
|
+
} catch (error) {
|
|
2564
|
+
attempt = Promise.reject(error);
|
|
2565
|
+
}
|
|
2566
|
+
this.#closePromise = attempt;
|
|
2567
|
+
void attempt.catch(() => {
|
|
2568
|
+
if (this.#closePromise === attempt) this.#closePromise = undefined;
|
|
2569
|
+
});
|
|
2570
|
+
}
|
|
2571
|
+
const closed = Promise.all([this.#closePromise, stopped]).then(() => undefined);
|
|
2572
|
+
// Once what it was running has ended, so its last operation's line comes first.
|
|
2573
|
+
if (first) {
|
|
2574
|
+
const { key, execution } = this.options;
|
|
2575
|
+
const stoppedLine = () =>
|
|
2576
|
+
this.options.events.event({
|
|
2577
|
+
kind: "agent-stopped",
|
|
2578
|
+
agent: key,
|
|
2579
|
+
callPath: [],
|
|
2580
|
+
harness: execution.harness,
|
|
2581
|
+
});
|
|
2582
|
+
closed.then(stoppedLine, stoppedLine);
|
|
2583
|
+
}
|
|
2584
|
+
return closed;
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
/**
|
|
2588
|
+
* The standard recovery, except for a calling session whose harness's interrupt cannot be told
|
|
2589
|
+
* from a turn that ended without answering: nudging it could be prompting an operator who just
|
|
2590
|
+
* stopped it (ADR 0010).
|
|
2591
|
+
*/
|
|
2592
|
+
async stopOperations(request: OperationStop): Promise<void> {
|
|
2593
|
+
// Before they settle, which records it later than a close that follows reads it.
|
|
2594
|
+
if (this.#operationStops.size > 0 || this.#pendingSets.length > 0) this.#last = request.kind;
|
|
2595
|
+
await Promise.all([...this.#operationStops].map((stop) => stop(request)));
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
private defaultNudge(): Exclude<AgentRunTextSpec["nudge"], false> {
|
|
2599
|
+
const { execution } = this.options;
|
|
2600
|
+
return execution.caller && !findHarness(execution.harness)?.interrupted ? undefined : {};
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2603
|
+
private queue<T>(operation: () => Promise<T>): Promise<T> {
|
|
2604
|
+
this.#queued += 1;
|
|
2605
|
+
const result = this.#tail.then(operation, operation);
|
|
2606
|
+
this.#tail = result.then(
|
|
2607
|
+
() => undefined,
|
|
2608
|
+
() => undefined,
|
|
2609
|
+
);
|
|
2610
|
+
void this.#tail.then(() => {
|
|
2611
|
+
this.#queued -= 1;
|
|
2612
|
+
// With nothing queued, a message held meanwhile opens the next operation.
|
|
2613
|
+
if (this.#queued === 0) this.messagesChanged();
|
|
2614
|
+
});
|
|
2615
|
+
return result;
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
private async executeOperation(
|
|
2619
|
+
spec: AgentRunTextSpec | AgentRunStructuredSpec<JsonValue>,
|
|
2620
|
+
scope: ExecutionScope | undefined,
|
|
2621
|
+
operationDeadline: AbsoluteDeadline,
|
|
2622
|
+
operation: OperationLog,
|
|
2623
|
+
): Promise<RunResult<JsonValue>> {
|
|
2624
|
+
const nudge = spec.nudge === false ? undefined : (spec.nudge ?? this.defaultNudge());
|
|
2625
|
+
scope?.assertActive();
|
|
2626
|
+
if (nudge?.deadline) assertDeadlineValue(nudge.deadline);
|
|
2627
|
+
return this.executeWaitingOperation(spec, scope, operationDeadline, operation, nudge);
|
|
2628
|
+
}
|
|
2629
|
+
|
|
2630
|
+
private async executeWaitingOperation(
|
|
2631
|
+
spec: AgentRunTextSpec | AgentRunStructuredSpec<JsonValue>,
|
|
2632
|
+
scope: ExecutionScope | undefined,
|
|
2633
|
+
deadline: AbsoluteDeadline,
|
|
2634
|
+
{ operationId, tags, delivered, session }: OperationLog,
|
|
2635
|
+
nudge: Exclude<AgentRunTextSpec["nudge"], false>,
|
|
2636
|
+
): Promise<RunResult<JsonValue>> {
|
|
2637
|
+
const schema = resultSchema(spec.schema);
|
|
2638
|
+
const binding = { endpoint: this.options.endpoint, operationId };
|
|
2639
|
+
const entry = this.options.ledger.reserve(operationId, () => this.#execution, tags);
|
|
2640
|
+
const { messaging } = this.options;
|
|
2641
|
+
messaging.opened({
|
|
2642
|
+
id: operationId,
|
|
2643
|
+
agent: this.key,
|
|
2644
|
+
kind: "workflow",
|
|
2645
|
+
deadline: deadline.unixMilliseconds,
|
|
2646
|
+
ceiling: this.options.deadline.unixMilliseconds,
|
|
2647
|
+
scope: scope?.messages ?? messaging.attempt,
|
|
2648
|
+
});
|
|
2649
|
+
// After its prompt: how to use its routes, then each message held for it, wrapped.
|
|
2650
|
+
const joined = messaging.take(operationId, "joins");
|
|
2651
|
+
const prompt = [spec.prompt, messaging.lesson(this.key), joined?.text]
|
|
2652
|
+
.filter(Boolean)
|
|
2653
|
+
.join("\n\n");
|
|
2654
|
+
const authored: AuthoredTurn = {
|
|
2655
|
+
prompt: spec.prompt,
|
|
2656
|
+
...(spec.label === undefined ? {} : { label: spec.label }),
|
|
2657
|
+
...(tags.stage === undefined ? {} : { stage: tags.stage }),
|
|
2658
|
+
...(spec.schema ? { schema: spec.schema } : {}),
|
|
2659
|
+
...(joined ? { messages: joined.messages } : {}),
|
|
2660
|
+
};
|
|
2661
|
+
const waitingSupported = this.options.session.supportsWaiting === true;
|
|
2662
|
+
const recoveryPrompt =
|
|
2663
|
+
nudge?.prompt ??
|
|
2664
|
+
(waitingSupported
|
|
2665
|
+
? "Report the result now, or use wf waiting with a reason if you still need time."
|
|
2666
|
+
: "You finished without reporting the requested result. Report it now.");
|
|
2667
|
+
// The first prompt carries only the result command; waiting is rare, so it is offered only
|
|
2668
|
+
// once a check-in finds the agent idle without an answer.
|
|
2669
|
+
const checkInPrompt = (text: string) =>
|
|
2670
|
+
operationPrompt(text, schema, this.options.launcher, operationId) +
|
|
2671
|
+
(waitingSupported
|
|
2672
|
+
? [
|
|
2673
|
+
"",
|
|
2674
|
+
"Finish or stop work needed for this answer before submitting the result.",
|
|
2675
|
+
`If waiting for work to finish, report it now using: ${this.options.launcher} waiting ${operationId} --reason 'what you are waiting for' [--timeout 2m]`,
|
|
2676
|
+
"Omit the square brackets when running the optional timeout flag. This command acknowledges your wait immediately; it does not sleep.",
|
|
2677
|
+
`The fixed answer deadline is ${new Date(deadline.unixMilliseconds).toISOString()}. Waiting never extends it.`,
|
|
2678
|
+
].join("\n")
|
|
2679
|
+
: "");
|
|
2680
|
+
const events = createOperationEvents(this.options.runDir, operationId);
|
|
2681
|
+
const messages = this.operationMessages(operationId, `${spec.id!}:message`, {
|
|
2682
|
+
deadline: () => deadline,
|
|
2683
|
+
authored,
|
|
2684
|
+
session,
|
|
2685
|
+
record: events.record,
|
|
2686
|
+
});
|
|
2687
|
+
const outcome = await superviseOperation({
|
|
2688
|
+
messages,
|
|
2689
|
+
event: events.record,
|
|
2690
|
+
operationId,
|
|
2691
|
+
agentId: this.key,
|
|
2692
|
+
question: spec.prompt,
|
|
2693
|
+
schema,
|
|
2694
|
+
deadline,
|
|
2695
|
+
slots: this.options.slots,
|
|
2696
|
+
waitingSupported,
|
|
2697
|
+
nudge: nudge !== undefined,
|
|
2698
|
+
policy: this.options.livenessPolicy,
|
|
2699
|
+
start: () =>
|
|
2700
|
+
this.options.session
|
|
2701
|
+
.start(
|
|
2702
|
+
{
|
|
2703
|
+
id: spec.id!,
|
|
2704
|
+
prompt: operationPrompt(prompt, schema, this.options.launcher, operationId),
|
|
2705
|
+
deadline,
|
|
2706
|
+
authored,
|
|
2707
|
+
onSession: session,
|
|
2708
|
+
},
|
|
2709
|
+
binding,
|
|
2710
|
+
)
|
|
2711
|
+
.then((turn) => this.confirmed(operationId, joined?.numbers ?? [], turn)),
|
|
2712
|
+
successor: (turn, sequence, deliverySignal) =>
|
|
2713
|
+
turn.nudge({
|
|
2714
|
+
id: `${spec.id!}:check-in:${sequence}`,
|
|
2715
|
+
deadline,
|
|
2716
|
+
deliverySignal,
|
|
2717
|
+
prompt: checkInPrompt(recoveryPrompt),
|
|
2718
|
+
// A check-in delivers no messages: those its first prompt carried are not its own.
|
|
2719
|
+
authored: { ...withoutMessages(authored), prompt: recoveryPrompt },
|
|
2720
|
+
}),
|
|
2721
|
+
onStop: (stop) => {
|
|
2722
|
+
this.#operationStops.add(stop);
|
|
2723
|
+
const remove = scope?.add(stop);
|
|
2724
|
+
return () => {
|
|
2725
|
+
this.#operationStops.delete(stop);
|
|
2726
|
+
remove?.();
|
|
2727
|
+
};
|
|
2728
|
+
},
|
|
2729
|
+
phase: (phase, reason, until) =>
|
|
2730
|
+
this.options.progress.turnPhase(this.key, phase, reason, until),
|
|
2731
|
+
delivered,
|
|
2732
|
+
})
|
|
2733
|
+
.catch(
|
|
2734
|
+
(error: unknown): SupervisedOutcome => ({
|
|
2735
|
+
kind: "failed",
|
|
2736
|
+
reason: messageOf(error),
|
|
2737
|
+
cleanupUnresolved: true,
|
|
2738
|
+
charges: [],
|
|
2739
|
+
settledAt: Date.now(),
|
|
2740
|
+
}),
|
|
2741
|
+
)
|
|
2742
|
+
.finally(() => events.close());
|
|
2743
|
+
// Its wait is dropped and what it still owed fails, before a failure closes the agent.
|
|
2744
|
+
messaging.ended(operationId, operationEnd(outcome.kind));
|
|
2745
|
+
if (outcome.cleanupUnresolved) {
|
|
2746
|
+
this.options.cleanupUnresolved(`cleanup-unresolved: ${this.key} operation ${operationId}`);
|
|
2747
|
+
this.closeAfterFailure("cleanup-unresolved");
|
|
2748
|
+
}
|
|
2749
|
+
if (outcome.kind === "timed-out" || outcome.kind === "failed")
|
|
2750
|
+
this.closeAfterFailure(outcome.reason, outcome.kind);
|
|
2751
|
+
const usage = entry.settle(
|
|
2752
|
+
{
|
|
2753
|
+
...(outcome.deliveredAt === undefined ? {} : { deliveredAt: outcome.deliveredAt }),
|
|
2754
|
+
settledAt: outcome.settledAt,
|
|
2755
|
+
},
|
|
2756
|
+
outcome.charges,
|
|
2757
|
+
);
|
|
2758
|
+
if (outcome.kind === "answered")
|
|
2759
|
+
return { outcome: { kind: "answered", value: outcome.value as JsonValue, usage } };
|
|
2760
|
+
if (outcome.kind === "failed") {
|
|
2761
|
+
const login = outcome.login ? { login: outcome.login } : {};
|
|
2762
|
+
return {
|
|
2763
|
+
outcome: { kind: "failed", reason: outcome.reason, retryable: false, ...login, usage },
|
|
2764
|
+
};
|
|
2765
|
+
}
|
|
2766
|
+
return { outcome: { kind: outcome.kind, reason: outcome.reason, usage } };
|
|
2767
|
+
}
|
|
2768
|
+
|
|
2769
|
+
/**
|
|
2770
|
+
* What an operation's supervisor asks of its messages: what is held, what it awaits, where its
|
|
2771
|
+
* turn's end goes, and the turn that carries a delivery or the reminder.
|
|
2772
|
+
*/
|
|
2773
|
+
private operationMessages(
|
|
2774
|
+
operationId: string,
|
|
2775
|
+
turnIds: string,
|
|
2776
|
+
turn: {
|
|
2777
|
+
deadline: () => AbsoluteDeadline;
|
|
2778
|
+
authored: AuthoredTurn;
|
|
2779
|
+
session: (id: string) => void;
|
|
2780
|
+
record: (event: LivenessEvent) => void;
|
|
2781
|
+
/** A message operation's deadline moved later, for the askers it now answers. */
|
|
2782
|
+
extended?: (deadline: number) => void;
|
|
2783
|
+
},
|
|
2784
|
+
): OperationMessages {
|
|
2785
|
+
const { messaging } = this.options;
|
|
2786
|
+
return {
|
|
2787
|
+
pending: () => messaging.pending(operationId),
|
|
2788
|
+
awaiting: () => messaging.awaiting(operationId),
|
|
2789
|
+
turnEnded: () => messaging.turnEnded(operationId),
|
|
2790
|
+
reminderExpired: () => messaging.reminderExpired(operationId),
|
|
2791
|
+
continueWith: async (previous, sequence, signal, reminder) => {
|
|
2792
|
+
const delivery =
|
|
2793
|
+
reminder === undefined ? messaging.take(operationId, "continues") : undefined;
|
|
2794
|
+
if (reminder === undefined && !delivery) {
|
|
2795
|
+
throw new Error("nothing was held to continue the operation with");
|
|
2796
|
+
}
|
|
2797
|
+
if (delivery?.deadline !== undefined) turn.extended?.(delivery.deadline);
|
|
2798
|
+
const prompt = reminder ?? delivery!.text;
|
|
2799
|
+
const messages = delivery?.messages ?? [];
|
|
2800
|
+
if (reminder === undefined) {
|
|
2801
|
+
turn.record({
|
|
2802
|
+
kind: "continued",
|
|
2803
|
+
at: Date.now(),
|
|
2804
|
+
sequence,
|
|
2805
|
+
reason: continuedBy(messages),
|
|
2806
|
+
});
|
|
2807
|
+
}
|
|
2808
|
+
const next = await previous.deliver({
|
|
2809
|
+
id: `${turnIds}:${sequence}`,
|
|
2810
|
+
prompt,
|
|
2811
|
+
deadline: turn.deadline(),
|
|
2812
|
+
deliverySignal: signal,
|
|
2813
|
+
authored: {
|
|
2814
|
+
...withoutMessages(turn.authored),
|
|
2815
|
+
prompt,
|
|
2816
|
+
...(messages.length ? { messages } : {}),
|
|
2817
|
+
},
|
|
2818
|
+
onSession: turn.session,
|
|
2819
|
+
});
|
|
2820
|
+
return this.confirmed(operationId, delivery?.numbers ?? [], next);
|
|
2821
|
+
},
|
|
2822
|
+
onChange: (wake) => {
|
|
2823
|
+
this.#watchers.add(wake);
|
|
2824
|
+
return () => this.#watchers.delete(wake);
|
|
2825
|
+
},
|
|
2826
|
+
};
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
/**
|
|
2830
|
+
* What the harness confirms of a delivery: received, or a session record showing the prompt, is
|
|
2831
|
+
* `observed`; a failure, or no evidence by the turn's end, `unconfirmed`.
|
|
2832
|
+
*/
|
|
2833
|
+
private confirmed(
|
|
2834
|
+
operationId: string,
|
|
2835
|
+
numbers: readonly number[],
|
|
2836
|
+
turn: HarnessTurn,
|
|
2837
|
+
): HarnessTurn {
|
|
2838
|
+
if (numbers.length === 0) return turn;
|
|
2839
|
+
const { messaging } = this.options;
|
|
2840
|
+
let said = false;
|
|
2841
|
+
const say = (state: "observed" | "unconfirmed") => {
|
|
2842
|
+
if (said) return;
|
|
2843
|
+
said = true;
|
|
2844
|
+
messaging.confirmed(operationId, numbers, state);
|
|
2845
|
+
};
|
|
2846
|
+
if (turn.delivery) {
|
|
2847
|
+
const { received, failed } = turn.delivery;
|
|
2848
|
+
void received.then(() => say("observed"));
|
|
2849
|
+
void failed.then(() => say("unconfirmed"));
|
|
2850
|
+
// A receipt read from the session's file may come just after the turn has ended.
|
|
2851
|
+
void turn.settled.then(() =>
|
|
2852
|
+
Promise.race([
|
|
2853
|
+
received,
|
|
2854
|
+
failed,
|
|
2855
|
+
new Promise((resolve) => setTimeout(resolve, RECEIPT_GRACE_MS).unref()),
|
|
2856
|
+
]).then(() => say("unconfirmed")),
|
|
2857
|
+
);
|
|
2858
|
+
return turn;
|
|
2859
|
+
}
|
|
2860
|
+
// Headless, the resumed session's own record is the evidence, read as the turn ends.
|
|
2861
|
+
void turn.settled.then((outcome) => say(outcome.observed ? "observed" : "unconfirmed"));
|
|
2862
|
+
return turn;
|
|
2863
|
+
}
|
|
2864
|
+
|
|
2865
|
+
/**
|
|
2866
|
+
* Wakes this idle agent in a message operation: its route's stage owns it, or the attempt outside
|
|
2867
|
+
* any stage. Queued as any operation is, so nothing else runs beside it.
|
|
2868
|
+
*/
|
|
2869
|
+
private startMessageOperation(scope: MessagingScope): void {
|
|
2870
|
+
const owner = this.options.messageOwner(scope);
|
|
2871
|
+
if (!owner) return;
|
|
2872
|
+
const result = this.queue(async () => {
|
|
2873
|
+
if (this.#closed || this.options.isRunClosing()) return;
|
|
2874
|
+
// What is first now: a message dropped since this was queued leaves another route's, which
|
|
2875
|
+
// waits for an operation of its own scope.
|
|
2876
|
+
const opening = this.options.messaging.opening(this.key);
|
|
2877
|
+
if (opening?.scope !== scope) return;
|
|
2878
|
+
await this.executeMessageOperation(opening.from, scope, owner.stage);
|
|
2879
|
+
});
|
|
2880
|
+
this.options.track(result).catch(() => undefined);
|
|
2881
|
+
owner.adopt(result);
|
|
2882
|
+
}
|
|
2883
|
+
|
|
2884
|
+
private async executeMessageOperation(
|
|
2885
|
+
from: AgentKey,
|
|
2886
|
+
scope: MessagingScope,
|
|
2887
|
+
stage: ExecutionScope | undefined,
|
|
2888
|
+
): Promise<void> {
|
|
2889
|
+
const { messaging } = this.options;
|
|
2890
|
+
const operationId = randomUUID();
|
|
2891
|
+
const ceiling = this.deadlineCeiling(stage).unixMilliseconds;
|
|
2892
|
+
let deadline = Math.min(Date.now() + messaging.limits.operationMs, ceiling);
|
|
2893
|
+
messaging.opened({
|
|
2894
|
+
id: operationId,
|
|
2895
|
+
agent: this.key,
|
|
2896
|
+
kind: "message",
|
|
2897
|
+
deadline,
|
|
2898
|
+
ceiling,
|
|
2899
|
+
scope,
|
|
2900
|
+
});
|
|
2901
|
+
/** How it ended; unset until its supervisor says, or if it never opened. */
|
|
2902
|
+
let end: SupervisedOutcome["kind"] | undefined;
|
|
2903
|
+
try {
|
|
2904
|
+
const opening = messaging.take(operationId, "opens");
|
|
2905
|
+
// Withdrawn before it could be shown: nothing to wake it for.
|
|
2906
|
+
if (!opening) return;
|
|
2907
|
+
if (opening.deadline !== undefined) deadline = opening.deadline;
|
|
2908
|
+
const cause = { message: { from } };
|
|
2909
|
+
const tags: OperationTags = {
|
|
2910
|
+
...(stage?.stage === undefined ? {} : { stage: stage.stage }),
|
|
2911
|
+
cause,
|
|
2912
|
+
};
|
|
2913
|
+
const operation = this.operationLog(operationId, "turn", tags);
|
|
2914
|
+
this.options.progress.turnStarted(this.key, tags);
|
|
2915
|
+
try {
|
|
2916
|
+
const outcome = await this.superviseMessage(operationId, deadline, opening, {
|
|
2917
|
+
cause,
|
|
2918
|
+
tags,
|
|
2919
|
+
stage,
|
|
2920
|
+
operation,
|
|
2921
|
+
});
|
|
2922
|
+
end = outcome.kind;
|
|
2923
|
+
if (outcome.kind === "timed-out" || outcome.kind === "failed") {
|
|
2924
|
+
throw new Error(
|
|
2925
|
+
`agent ${this.key}'s message operation ${outcome.kind}: ${outcome.reason}`,
|
|
2926
|
+
);
|
|
2927
|
+
}
|
|
2928
|
+
} catch (error) {
|
|
2929
|
+
if (end === undefined) {
|
|
2930
|
+
end = "failed";
|
|
2931
|
+
operation.failed(error);
|
|
2932
|
+
this.options.progress.turnSettled(this.key, "failed", messageOf(error));
|
|
2933
|
+
}
|
|
2934
|
+
throw error;
|
|
2935
|
+
} finally {
|
|
2936
|
+
await this.options.afterOperation?.();
|
|
2937
|
+
}
|
|
2938
|
+
} finally {
|
|
2939
|
+
// A message operation that simply ends settles; its wait is dropped and what it owed fails.
|
|
2940
|
+
messaging.ended(
|
|
2941
|
+
operationId,
|
|
2942
|
+
end === undefined ? "cancelled" : end === "answered" ? "settled" : operationEnd(end),
|
|
2943
|
+
);
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
|
|
2947
|
+
/** Supervises a message operation's turns, records its turn and usage, and says how it ended. */
|
|
2948
|
+
private async superviseMessage(
|
|
2949
|
+
operationId: string,
|
|
2950
|
+
initial: number,
|
|
2951
|
+
opening: Delivery,
|
|
2952
|
+
context: {
|
|
2953
|
+
cause: { message: { from: AgentKey } };
|
|
2954
|
+
tags: OperationTags;
|
|
2955
|
+
stage: ExecutionScope | undefined;
|
|
2956
|
+
operation: OperationLog;
|
|
2957
|
+
},
|
|
2958
|
+
): Promise<SupervisedOutcome> {
|
|
2959
|
+
const { messaging } = this.options;
|
|
2960
|
+
const { cause, tags, stage, operation } = context;
|
|
2961
|
+
let deadline = initial;
|
|
2962
|
+
const entry = this.options.ledger.reserve(operationId, () => this.#execution, tags);
|
|
2963
|
+
const events = createOperationEvents(this.options.runDir, operationId);
|
|
2964
|
+
const authored: AuthoredTurn = {
|
|
2965
|
+
prompt: opening.text,
|
|
2966
|
+
...(tags.stage === undefined ? {} : { stage: tags.stage }),
|
|
2967
|
+
cause,
|
|
2968
|
+
messages: opening.messages,
|
|
2969
|
+
};
|
|
2970
|
+
let outcome: SupervisedOutcome;
|
|
2971
|
+
try {
|
|
2972
|
+
outcome = await superviseMessageOperation({
|
|
2973
|
+
deadline: () => deadline,
|
|
2974
|
+
start: () =>
|
|
2975
|
+
this.options.session
|
|
2976
|
+
.start(
|
|
2977
|
+
{
|
|
2978
|
+
id: operationId,
|
|
2979
|
+
prompt: opening.text,
|
|
2980
|
+
deadline: { unixMilliseconds: deadline },
|
|
2981
|
+
authored,
|
|
2982
|
+
onSession: operation.session,
|
|
2983
|
+
},
|
|
2984
|
+
{ endpoint: this.options.endpoint, operationId },
|
|
2985
|
+
)
|
|
2986
|
+
.then((turn) => this.confirmed(operationId, opening.numbers, turn)),
|
|
2987
|
+
messages: this.operationMessages(operationId, `${operationId}:message`, {
|
|
2988
|
+
deadline: () => ({ unixMilliseconds: deadline }),
|
|
2989
|
+
authored,
|
|
2990
|
+
session: operation.session,
|
|
2991
|
+
record: events.record,
|
|
2992
|
+
extended: (later) => {
|
|
2993
|
+
if (later <= deadline) return;
|
|
2994
|
+
deadline = later;
|
|
2995
|
+
messaging.extended(operationId, later);
|
|
2996
|
+
events.record({ kind: "deadline-extended", at: Date.now(), sequence: 0, until: later });
|
|
2997
|
+
},
|
|
2998
|
+
}),
|
|
2999
|
+
onStop: (stop) => {
|
|
3000
|
+
this.#operationStops.add(stop);
|
|
3001
|
+
const remove = stage?.add(stop);
|
|
3002
|
+
return () => {
|
|
3003
|
+
this.#operationStops.delete(stop);
|
|
3004
|
+
remove?.();
|
|
3005
|
+
};
|
|
3006
|
+
},
|
|
3007
|
+
event: events.record,
|
|
3008
|
+
phase: (phase, reason, until) =>
|
|
3009
|
+
this.options.progress.turnPhase(this.key, phase, reason, until),
|
|
3010
|
+
delivered: operation.delivered,
|
|
3011
|
+
...(this.options.livenessPolicy ? { policy: this.options.livenessPolicy } : {}),
|
|
3012
|
+
});
|
|
3013
|
+
} finally {
|
|
3014
|
+
await events.close();
|
|
3015
|
+
}
|
|
3016
|
+
if (outcome.cleanupUnresolved) {
|
|
3017
|
+
this.options.cleanupUnresolved(`cleanup-unresolved: ${this.key} operation ${operationId}`);
|
|
3018
|
+
}
|
|
3019
|
+
// A timeout or a harness failure stops the agent, as any operation's does (Q10).
|
|
3020
|
+
if (outcome.kind === "timed-out" || outcome.kind === "failed")
|
|
3021
|
+
this.closeAfterFailure(outcome.reason, outcome.kind);
|
|
3022
|
+
const usage = entry.settle(
|
|
3023
|
+
{
|
|
3024
|
+
...(outcome.deliveredAt === undefined ? {} : { deliveredAt: outcome.deliveredAt }),
|
|
3025
|
+
settledAt: outcome.settledAt,
|
|
3026
|
+
},
|
|
3027
|
+
outcome.charges,
|
|
3028
|
+
);
|
|
3029
|
+
// Recorded `answered` with no value: it did what it was woken for, and takes no result.
|
|
3030
|
+
this.turnEnded(
|
|
3031
|
+
outcome.kind === "answered"
|
|
3032
|
+
? { kind: "answered", value: null, usage }
|
|
3033
|
+
: outcome.kind === "failed"
|
|
3034
|
+
? { kind: "failed", reason: outcome.reason, retryable: false, usage }
|
|
3035
|
+
: { kind: outcome.kind, reason: outcome.reason, usage },
|
|
3036
|
+
"turn",
|
|
3037
|
+
);
|
|
3038
|
+
return outcome;
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
/**
|
|
3042
|
+
* The harness's own compaction, with the spec's prompt as its focus (ADR 0007). Nothing answers
|
|
3043
|
+
* through a result slot: the harness confirms it compacted by setting `summary`. One past its
|
|
3044
|
+
* deadline is left to end on its own rather than stopped, which would close a pane's agent.
|
|
3045
|
+
*/
|
|
3046
|
+
private async executeCompaction(
|
|
3047
|
+
id: CompactionId,
|
|
3048
|
+
prompt: string,
|
|
3049
|
+
deadline: AbsoluteDeadline,
|
|
3050
|
+
scope: ExecutionScope | undefined,
|
|
3051
|
+
{ operationId, tags, delivered }: OperationLog,
|
|
3052
|
+
): Promise<TurnOutcome<string>> {
|
|
3053
|
+
const entry = this.options.ledger.reserve(operationId, () => this.#execution, tags);
|
|
3054
|
+
const times: { deliveredAt?: number } = {};
|
|
3055
|
+
const settle = (
|
|
3056
|
+
native: HarnessTurnOutcome | "expired",
|
|
3057
|
+
/** A compaction left to end on its own, whose charges come when it does. */
|
|
3058
|
+
later?: Promise<HarnessTurnOutcome>,
|
|
3059
|
+
): TurnOutcome<string> => {
|
|
3060
|
+
const usage = entry.settle(
|
|
3061
|
+
{
|
|
3062
|
+
...times,
|
|
3063
|
+
settledAt: Math.min(Date.now(), deadline.unixMilliseconds),
|
|
3064
|
+
},
|
|
3065
|
+
native === "expired" ? [] : native.chargesUsd,
|
|
3066
|
+
later,
|
|
3067
|
+
);
|
|
3068
|
+
if (native === "expired") {
|
|
3069
|
+
return { kind: "timed-out", reason: "compaction deadline exceeded", usage };
|
|
3070
|
+
}
|
|
3071
|
+
if (native.state === "completed") {
|
|
3072
|
+
return native.summary === undefined
|
|
3073
|
+
? {
|
|
3074
|
+
kind: "failed",
|
|
3075
|
+
reason: native.detail ?? "the harness did not confirm a compaction",
|
|
3076
|
+
retryable: false,
|
|
3077
|
+
usage,
|
|
3078
|
+
}
|
|
3079
|
+
: { kind: "answered", value: native.summary, usage };
|
|
3080
|
+
}
|
|
3081
|
+
return reconcile<string>(native, usage);
|
|
3082
|
+
};
|
|
3083
|
+
if (Date.now() >= deadline.unixMilliseconds) return settle("expired");
|
|
3084
|
+
let turn: HarnessTurn;
|
|
3085
|
+
const acquiring = Promise.resolve().then(() =>
|
|
3086
|
+
this.options.session.compact(id, prompt, deadline),
|
|
3087
|
+
);
|
|
3088
|
+
try {
|
|
3089
|
+
turn = await waitForDeadline(acquiring, deadline);
|
|
3090
|
+
} catch (error) {
|
|
3091
|
+
if (error instanceof DeadlineExceededError) {
|
|
3092
|
+
// A compaction that starts after all is left to end on its own, as one past its deadline.
|
|
3093
|
+
// The agent's next operation waits for it to start, or would race it for the session.
|
|
3094
|
+
const late = await waitForDeadline(acquiring, {
|
|
3095
|
+
unixMilliseconds: Date.now() + CLEANUP_GRACE_MILLISECONDS,
|
|
3096
|
+
}).catch(() => undefined);
|
|
3097
|
+
if (!late) {
|
|
3098
|
+
this.abandonTurnAcquisition(acquiring, "compaction deadline exceeded before it started");
|
|
3099
|
+
return settle("expired");
|
|
3100
|
+
}
|
|
3101
|
+
return settle("expired", (await this.leaveFinishing(late)).later);
|
|
3102
|
+
}
|
|
3103
|
+
return settle({
|
|
3104
|
+
state: "failed",
|
|
3105
|
+
detail: messageOf(error),
|
|
3106
|
+
resultEvidence: { kind: "unavailable" },
|
|
3107
|
+
chargesUsd: [],
|
|
3108
|
+
});
|
|
3109
|
+
}
|
|
3110
|
+
times.deliveredAt = Date.now();
|
|
3111
|
+
delivered(times.deliveredAt);
|
|
3112
|
+
const removeCanceller = scope?.add((stop) => releaseTurn(turn, stop.reason));
|
|
3113
|
+
try {
|
|
3114
|
+
let cancelTimer: (() => void) | undefined;
|
|
3115
|
+
const native = await Promise.race([
|
|
3116
|
+
turn.settled,
|
|
3117
|
+
new Promise<"expired">((resolve) => {
|
|
3118
|
+
cancelTimer = scheduleAt(deadline, () => resolve("expired"));
|
|
3119
|
+
}),
|
|
3120
|
+
]).finally(() => cancelTimer?.());
|
|
3121
|
+
if (native === "expired") return settle(native, (await this.leaveFinishing(turn)).later);
|
|
3122
|
+
return settle(native);
|
|
3123
|
+
} finally {
|
|
3124
|
+
removeCanceller?.();
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
|
|
3128
|
+
/** Releases a compaction past its deadline to end on its own; its end, when the host left it. */
|
|
3129
|
+
private async leaveFinishing(
|
|
3130
|
+
turn: HarnessTurn,
|
|
3131
|
+
): Promise<{ later?: Promise<HarnessTurnOutcome> }> {
|
|
3132
|
+
const disposition = await releaseTurn(turn, "compaction deadline exceeded", true);
|
|
3133
|
+
return disposition?.kind === "finishing" ? { later: turn.settled } : {};
|
|
3134
|
+
}
|
|
3135
|
+
|
|
3136
|
+
private abandonTurnAcquisition(acquiring: Promise<HarnessTurn>, reason: string): void {
|
|
3137
|
+
const lateRelease = acquiring.then(
|
|
3138
|
+
(turn) => releaseTurn(turn, reason),
|
|
3139
|
+
() => undefined,
|
|
3140
|
+
);
|
|
3141
|
+
this.options.track(lateRelease);
|
|
3142
|
+
this.closeAfterFailure(reason);
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
|
|
3146
|
+
class ExecutionScope {
|
|
3147
|
+
readonly #cancellers = new Set<(stop: OperationStop) => Promise<unknown>>();
|
|
3148
|
+
readonly #owned = new Set<Promise<unknown>>();
|
|
3149
|
+
#cancelled = false;
|
|
3150
|
+
#stop: OperationStop | undefined;
|
|
3151
|
+
#sealed = false;
|
|
3152
|
+
#cancellation: Promise<void> | undefined;
|
|
3153
|
+
|
|
3154
|
+
constructor(
|
|
3155
|
+
readonly deadline: AbsoluteDeadline,
|
|
3156
|
+
/** The labelled `parallel` it runs in, for progress. */
|
|
3157
|
+
readonly group?: GroupProgress,
|
|
3158
|
+
/** The workflow stage everything in this scope runs in. */
|
|
3159
|
+
readonly stage?: string,
|
|
3160
|
+
/** The stage's messaging scope, where routes granted in it live and close. */
|
|
3161
|
+
readonly messages?: MessagingScope,
|
|
3162
|
+
) {}
|
|
3163
|
+
|
|
3164
|
+
get cancelled(): boolean {
|
|
3165
|
+
return this.#cancelled;
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
assertActive(): void {
|
|
3169
|
+
if (this.#cancelled || Date.now() >= this.deadline.unixMilliseconds) {
|
|
3170
|
+
throw new DeadlineExceededError(this.deadline);
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3173
|
+
|
|
3174
|
+
assertAccepting(): void {
|
|
3175
|
+
this.assertActive();
|
|
3176
|
+
if (this.#sealed) throw new Error("parallel execution scope is closed");
|
|
3177
|
+
}
|
|
3178
|
+
|
|
3179
|
+
add(cancel: (stop: OperationStop) => Promise<unknown>): () => void {
|
|
3180
|
+
if (this.#cancelled) {
|
|
3181
|
+
try {
|
|
3182
|
+
void cancel(this.#stop!).catch(() => undefined);
|
|
3183
|
+
} catch {
|
|
3184
|
+
// Late registration keeps the stop fence synchronous, including a throwing canceller.
|
|
3185
|
+
}
|
|
3186
|
+
return () => undefined;
|
|
3187
|
+
}
|
|
3188
|
+
this.#cancellers.add(cancel);
|
|
3189
|
+
return () => this.#cancellers.delete(cancel);
|
|
3190
|
+
}
|
|
3191
|
+
|
|
3192
|
+
track(promise: Promise<unknown>): void {
|
|
3193
|
+
this.assertAccepting();
|
|
3194
|
+
this.#owned.add(promise);
|
|
3195
|
+
promise.catch(() => undefined);
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
/**
|
|
3199
|
+
* Owns a message operation one of the stage's routes opened, even once the stage's function has
|
|
3200
|
+
* returned: a message held before then can open one after.
|
|
3201
|
+
*/
|
|
3202
|
+
adopt(promise: Promise<unknown>): void {
|
|
3203
|
+
this.#owned.add(promise);
|
|
3204
|
+
promise.catch(() => undefined);
|
|
3205
|
+
}
|
|
3206
|
+
|
|
3207
|
+
/** Until everything it owns has settled, what was adopted meanwhile included. */
|
|
3208
|
+
async settleOwned(): Promise<void> {
|
|
3209
|
+
let seen = 0;
|
|
3210
|
+
while (this.#owned.size > seen) {
|
|
3211
|
+
seen = this.#owned.size;
|
|
3212
|
+
const settled = await Promise.allSettled([...this.#owned]);
|
|
3213
|
+
const rejected = settled.find((result) => result.status === "rejected");
|
|
3214
|
+
if (rejected?.status === "rejected") throw rejected.reason;
|
|
3215
|
+
}
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
seal(): void {
|
|
3219
|
+
this.#sealed = true;
|
|
3220
|
+
}
|
|
3221
|
+
|
|
3222
|
+
cancel(
|
|
3223
|
+
stop: OperationStop = { kind: "cancelled", reason: "parallel execution cancelled" },
|
|
3224
|
+
): Promise<void> {
|
|
3225
|
+
if (this.#cancellation) return this.#cancellation;
|
|
3226
|
+
this.#cancelled = true;
|
|
3227
|
+
this.#stop = stop;
|
|
3228
|
+
this.#cancellation = Promise.allSettled(
|
|
3229
|
+
[...this.#cancellers].map((cancel) => Promise.resolve().then(() => cancel(stop))),
|
|
3230
|
+
).then(() => undefined);
|
|
3231
|
+
return this.#cancellation;
|
|
3232
|
+
}
|
|
3233
|
+
}
|
|
3234
|
+
|
|
3235
|
+
function runParallel<Item, Result>(
|
|
3236
|
+
items: readonly Item[],
|
|
3237
|
+
operation: (item: Item, index: number) => Promise<Result>,
|
|
3238
|
+
deadline: AbsoluteDeadline,
|
|
3239
|
+
requestedConcurrency?: number,
|
|
3240
|
+
openGroup?: () => GroupProgress | undefined,
|
|
3241
|
+
): Promise<Result[]> {
|
|
3242
|
+
assertDeadline(deadline);
|
|
3243
|
+
const concurrency = requestedConcurrency ?? Math.max(1, items.length);
|
|
3244
|
+
if (!Number.isSafeInteger(concurrency) || concurrency <= 0) {
|
|
3245
|
+
throw new Error("parallel concurrency must be a positive safe integer");
|
|
3246
|
+
}
|
|
3247
|
+
const parent = scopes.getStore();
|
|
3248
|
+
parent?.assertAccepting();
|
|
3249
|
+
if (items.length === 0) return Promise.resolve([]);
|
|
3250
|
+
const execution = executeParallel(items, operation, deadline, concurrency, parent, openGroup?.());
|
|
3251
|
+
parent?.track(execution);
|
|
3252
|
+
return execution;
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
async function executeParallel<Item, Result>(
|
|
3256
|
+
items: readonly Item[],
|
|
3257
|
+
operation: (item: Item, index: number) => Promise<Result>,
|
|
3258
|
+
deadline: AbsoluteDeadline,
|
|
3259
|
+
concurrency: number,
|
|
3260
|
+
parent: ExecutionScope | undefined,
|
|
3261
|
+
group?: GroupProgress,
|
|
3262
|
+
): Promise<Result[]> {
|
|
3263
|
+
// An unlabelled parallel inside a labelled one stays part of it.
|
|
3264
|
+
const scope = new ExecutionScope(
|
|
3265
|
+
deadline,
|
|
3266
|
+
group ?? parent?.group,
|
|
3267
|
+
parent?.stage,
|
|
3268
|
+
parent?.messages,
|
|
3269
|
+
);
|
|
3270
|
+
const removeFromParent = parent?.add((stop) => scope.cancel(stop));
|
|
3271
|
+
const results = new Array<Result>(items.length);
|
|
3272
|
+
let next = 0;
|
|
3273
|
+
const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
3274
|
+
await scopes.run(scope, async () => {
|
|
3275
|
+
while (true) {
|
|
3276
|
+
const index = next;
|
|
3277
|
+
next += 1;
|
|
3278
|
+
if (index >= items.length) return;
|
|
3279
|
+
scope.assertActive();
|
|
3280
|
+
if (group) group.started += 1;
|
|
3281
|
+
try {
|
|
3282
|
+
results[index] = await operation(items[index]!, index);
|
|
3283
|
+
} finally {
|
|
3284
|
+
if (group) group.done += 1;
|
|
3285
|
+
}
|
|
3286
|
+
}
|
|
3287
|
+
});
|
|
3288
|
+
});
|
|
3289
|
+
const completion = Promise.all(workers).then(async () => {
|
|
3290
|
+
scope.seal();
|
|
3291
|
+
await scope.settleOwned();
|
|
3292
|
+
});
|
|
3293
|
+
let cancelTimer: (() => void) | undefined;
|
|
3294
|
+
const expiry = new Promise<never>((_resolve, reject) => {
|
|
3295
|
+
cancelTimer = scheduleAt(deadline, () => {
|
|
3296
|
+
void scope
|
|
3297
|
+
.cancel({ kind: "timed-out", reason: "parallel deadline exceeded" })
|
|
3298
|
+
.then(() => reject(new DeadlineExceededError(deadline)));
|
|
3299
|
+
});
|
|
3300
|
+
});
|
|
3301
|
+
try {
|
|
3302
|
+
await Promise.race([completion, expiry]);
|
|
3303
|
+
if (scope.cancelled || Date.now() >= deadline.unixMilliseconds) {
|
|
3304
|
+
await scope.cancel({ kind: "timed-out", reason: "parallel deadline exceeded" });
|
|
3305
|
+
throw new DeadlineExceededError(deadline);
|
|
3306
|
+
}
|
|
3307
|
+
return results;
|
|
3308
|
+
} catch (error) {
|
|
3309
|
+
await scope.cancel();
|
|
3310
|
+
completion.catch(() => undefined);
|
|
3311
|
+
throw error;
|
|
3312
|
+
} finally {
|
|
3313
|
+
if (group) group.endedAt = Date.now();
|
|
3314
|
+
cancelTimer?.();
|
|
3315
|
+
removeFromParent?.();
|
|
3316
|
+
}
|
|
3317
|
+
}
|
|
3318
|
+
|
|
3319
|
+
/**
|
|
3320
|
+
* Compaction keeps its existing bounded release behavior. Structured operations use the
|
|
3321
|
+
* supervisor, which requires confirmed native completion before returning an answer.
|
|
3322
|
+
*/
|
|
3323
|
+
async function releaseTurn(
|
|
3324
|
+
turn: HarnessTurn,
|
|
3325
|
+
reason: string,
|
|
3326
|
+
answered = false,
|
|
3327
|
+
): Promise<HarnessReleaseDisposition | undefined> {
|
|
3328
|
+
const deadline = { unixMilliseconds: Date.now() + CLEANUP_GRACE_MILLISECONDS };
|
|
3329
|
+
try {
|
|
3330
|
+
return await waitForDeadline(
|
|
3331
|
+
turn.release(reason, deadline, answered ? { answered } : undefined),
|
|
3332
|
+
deadline,
|
|
3333
|
+
);
|
|
3334
|
+
} catch {
|
|
3335
|
+
return undefined;
|
|
3336
|
+
}
|
|
3337
|
+
}
|
|
3338
|
+
|
|
3339
|
+
function reconcile<T extends JsonValue>(
|
|
3340
|
+
native: HarnessTurnOutcome,
|
|
3341
|
+
usage: OperationRecord,
|
|
3342
|
+
): TurnOutcome<T> {
|
|
3343
|
+
switch (native.state) {
|
|
3344
|
+
case "completed":
|
|
3345
|
+
return { kind: "unanswered", reason: "agent settled without an accepted result", usage };
|
|
3346
|
+
case "blocked":
|
|
3347
|
+
return { kind: "blocked", reason: native.detail ?? "agent is blocked", usage };
|
|
3348
|
+
case "timed-out":
|
|
3349
|
+
return { kind: "timed-out", reason: native.detail ?? "operation timed out", usage };
|
|
3350
|
+
case "failed":
|
|
3351
|
+
return {
|
|
3352
|
+
kind: "failed",
|
|
3353
|
+
reason: native.detail ?? "harness operation failed",
|
|
3354
|
+
retryable: false,
|
|
3355
|
+
usage,
|
|
3356
|
+
};
|
|
3357
|
+
case "cancelled":
|
|
3358
|
+
return { kind: "cancelled", reason: native.detail ?? "operation cancelled", usage };
|
|
3359
|
+
}
|
|
3360
|
+
}
|
|
3361
|
+
|
|
3362
|
+
/** A stage's one line, which a summary that throws leaves out rather than failing the stage. */
|
|
3363
|
+
function summaryOf(
|
|
3364
|
+
name: string,
|
|
3365
|
+
options: StageOptions<JsonValue> | undefined,
|
|
3366
|
+
value: JsonValue | undefined,
|
|
3367
|
+
log: Warn | undefined,
|
|
3368
|
+
): string | undefined {
|
|
3369
|
+
if (!options?.summary || value === undefined) return undefined;
|
|
3370
|
+
try {
|
|
3371
|
+
return String(options.summary(value));
|
|
3372
|
+
} catch (error) {
|
|
3373
|
+
log?.(`awf: stage ${name}'s summary failed: ${messageOf(error)}`, undefined, name);
|
|
3374
|
+
return undefined;
|
|
3375
|
+
}
|
|
3376
|
+
}
|
|
3377
|
+
|
|
3378
|
+
type Warn = (message: string, fields?: JsonObject, stage?: string) => void;
|
|
3379
|
+
|
|
3380
|
+
/** An operation's start and failure, written under the id its record keeps. */
|
|
3381
|
+
type OperationLog = {
|
|
3382
|
+
operationId: string;
|
|
3383
|
+
tags: OperationTags;
|
|
3384
|
+
delivered(at: number): void;
|
|
3385
|
+
/** A session its host named while it ran: on its start if that is still to come. */
|
|
3386
|
+
session(id: string): void;
|
|
3387
|
+
failed(error: unknown): void;
|
|
3388
|
+
};
|
|
3389
|
+
|
|
3390
|
+
/** A native fork, and the parent's operation it was copied after: null when it ran none. */
|
|
3391
|
+
type TakenFork = { native: NativeFork; after: string | null };
|
|
3392
|
+
|
|
3393
|
+
/** A settled turn or compaction, kept in the run's `turns.jsonl`. */
|
|
3394
|
+
type RecordTurn = (outcome: TurnOutcome<JsonValue>, kind: TurnRecord["kind"]) => void;
|
|
3395
|
+
|
|
3396
|
+
function pick<T extends object, K extends keyof T>(value: T, keys: readonly K[]): Pick<T, K> {
|
|
3397
|
+
return Object.fromEntries(
|
|
3398
|
+
keys.flatMap((key) => (value[key] === undefined ? [] : [[key, value[key]]])),
|
|
3399
|
+
) as Pick<T, K>;
|
|
3400
|
+
}
|
|
3401
|
+
|
|
3402
|
+
/** How an operation's end reads to messaging: a result, or a turn that simply ended, or worse. */
|
|
3403
|
+
function operationEnd(kind: SupervisedOutcome["kind"]): OperationEnd {
|
|
3404
|
+
switch (kind) {
|
|
3405
|
+
case "answered":
|
|
3406
|
+
return "result";
|
|
3407
|
+
case "unanswered":
|
|
3408
|
+
case "blocked":
|
|
3409
|
+
return "settled";
|
|
3410
|
+
default:
|
|
3411
|
+
return kind;
|
|
3412
|
+
}
|
|
3413
|
+
}
|
|
3414
|
+
|
|
3415
|
+
/** A turn as the workflow wrote it, without the messages an earlier turn of it delivered. */
|
|
3416
|
+
function withoutMessages(authored: AuthoredTurn): AuthoredTurn {
|
|
3417
|
+
const { messages: _messages, ...rest } = authored;
|
|
3418
|
+
return rest;
|
|
3419
|
+
}
|
|
3420
|
+
|
|
3421
|
+
/** What continued an operation, as its liveness record says. */
|
|
3422
|
+
function continuedBy(messages: readonly AuthoredMessage[]): string {
|
|
3423
|
+
if (messages.some((each) => each.kind === "no-reply")) return "no-reply";
|
|
3424
|
+
if (messages.some((each) => each.kind === "message" && each.answers)) return "reply";
|
|
3425
|
+
if (messages.some((each) => each.kind === "message" && each.asks)) return "request";
|
|
3426
|
+
return "messages";
|
|
3427
|
+
}
|
|
3428
|
+
|
|
3429
|
+
function tagsOf(scope: ExecutionScope | undefined, label?: string): OperationTags {
|
|
3430
|
+
return {
|
|
3431
|
+
...(scope?.stage === undefined ? {} : { stage: scope.stage }),
|
|
3432
|
+
...(label === undefined ? {} : { label }),
|
|
3433
|
+
};
|
|
3434
|
+
}
|
|
3435
|
+
|
|
3436
|
+
/**
|
|
3437
|
+
* A stage's value as it is recorded and handed back: through JSON, as a continue reads it, and
|
|
3438
|
+
* checked by its `result`. A stage without one returns nothing.
|
|
3439
|
+
*/
|
|
3440
|
+
function stageValue(
|
|
3441
|
+
name: string,
|
|
3442
|
+
options: StageOptions<JsonValue> | undefined,
|
|
3443
|
+
value: unknown,
|
|
3444
|
+
): JsonValue | undefined {
|
|
3445
|
+
if (!options) {
|
|
3446
|
+
if (value === undefined) return undefined;
|
|
3447
|
+
throw new Error(
|
|
3448
|
+
`stage ${name} has no result schema, so it returns nothing, and it returned a value`,
|
|
3449
|
+
);
|
|
3450
|
+
}
|
|
3451
|
+
let text: string | undefined;
|
|
3452
|
+
try {
|
|
3453
|
+
text = value === undefined ? undefined : JSON.stringify(value);
|
|
3454
|
+
} catch (error) {
|
|
3455
|
+
throw new Error(`stage ${name}'s value is not JSON: ${messageOf(error)}`);
|
|
3456
|
+
}
|
|
3457
|
+
if (text === undefined) {
|
|
3458
|
+
throw new Error(`stage ${name} returned nothing; its result expects a value`);
|
|
3459
|
+
}
|
|
3460
|
+
const recorded = JSON.parse(text) as JsonValue;
|
|
3461
|
+
const errors = validate(parseJsonSchema(options.result), recorded);
|
|
3462
|
+
if (errors.length > 0) {
|
|
3463
|
+
throw new Error(`stage ${name}'s value does not fit its result: ${formatErrors(errors)}`);
|
|
3464
|
+
}
|
|
3465
|
+
return recorded;
|
|
3466
|
+
}
|
|
3467
|
+
|
|
3468
|
+
function resultSchema<T extends JsonValue>(schema: OutputSchema<T> | undefined): JsonSchema {
|
|
3469
|
+
return schema ? parseJsonSchema(schema) : { type: "string" };
|
|
3470
|
+
}
|
|
3471
|
+
|
|
3472
|
+
function operationPrompt(
|
|
3473
|
+
prompt: string,
|
|
3474
|
+
schema: JsonSchema,
|
|
3475
|
+
launcher: string,
|
|
3476
|
+
callId: string,
|
|
3477
|
+
): string {
|
|
3478
|
+
// The full path, because the command is not on the agent's PATH and in some harnesses cannot be.
|
|
3479
|
+
// A quoted heredoc passes the JSON through untouched: a quoted argument broke on shell quoting
|
|
3480
|
+
// (story 002). Not indented: a copied terminator with leading spaces never closes the heredoc.
|
|
3481
|
+
// The schema itself, not a rendering of it: without its bounds, first answers were 0/160 valid
|
|
3482
|
+
// (E5). `wf` says what to do about a rejection, so the prompt does not.
|
|
3483
|
+
return [
|
|
3484
|
+
prompt,
|
|
3485
|
+
"",
|
|
3486
|
+
"When the answer is ready, return it by running:",
|
|
3487
|
+
"",
|
|
3488
|
+
`${launcher} result ${callId} <<'WF_JSON'`,
|
|
3489
|
+
"<json>",
|
|
3490
|
+
"WF_JSON",
|
|
3491
|
+
"",
|
|
3492
|
+
"The JSON must match this schema:",
|
|
3493
|
+
JSON.stringify(schema),
|
|
3494
|
+
].join("\n");
|
|
3495
|
+
}
|
|
3496
|
+
|
|
3497
|
+
function resolveExecution(
|
|
3498
|
+
selection: RuntimeSelection,
|
|
3499
|
+
runtime: AgentRuntimeConfig,
|
|
3500
|
+
): AgentExecution {
|
|
3501
|
+
if (typeof selection === "string") return resolveAlias(selection, runtime);
|
|
3502
|
+
if ("alias" in selection) {
|
|
3503
|
+
const resolved = resolveAlias(selection.alias, runtime);
|
|
3504
|
+
for (const field of ["harness", "model"] as const) {
|
|
3505
|
+
const required = selection[field];
|
|
3506
|
+
if (required !== undefined && !isDeepStrictEqual(required, resolved[field])) {
|
|
3507
|
+
throw new Error(`runtime alias ${selection.alias} does not satisfy required ${field}`);
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
3510
|
+
return {
|
|
3511
|
+
...resolved,
|
|
3512
|
+
...(selection.effort === undefined ? {} : { effort: selection.effort }),
|
|
3513
|
+
...storedPlacement(selection),
|
|
3514
|
+
};
|
|
3515
|
+
}
|
|
3516
|
+
const { harness, model, effort } = selection;
|
|
3517
|
+
return {
|
|
3518
|
+
harness,
|
|
3519
|
+
model,
|
|
3520
|
+
...(effort === undefined ? {} : { effort }),
|
|
3521
|
+
...storedPlacement(selection),
|
|
3522
|
+
};
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
/** An agent at an effort its harness does not list is refused before anything runs. */
|
|
3526
|
+
function withKnownEffort(execution: AgentExecution): AgentExecution {
|
|
3527
|
+
if (execution.effort === undefined) return execution;
|
|
3528
|
+
if (typeof execution.effort !== "string") {
|
|
3529
|
+
throw new Error(`an effort is a level's name, not ${JSON.stringify(execution.effort)}`);
|
|
3530
|
+
}
|
|
3531
|
+
const refused = effortRefusal(execution.harness, execution.effort);
|
|
3532
|
+
if (refused) throw new Error(refused);
|
|
3533
|
+
return execution;
|
|
3534
|
+
}
|
|
3535
|
+
|
|
3536
|
+
/** What `set` changes. */
|
|
3537
|
+
type SettingsChange = Partial<Pick<AgentExecution, "model" | "effort">>;
|
|
3538
|
+
|
|
3539
|
+
const SETTINGS_FIELDS = new Set(["id", "model", "effort", "deadline", "timeoutMs"]);
|
|
3540
|
+
|
|
3541
|
+
function sessionSettings({ model, effort }: AgentExecution): SessionSettings {
|
|
3542
|
+
return { model, ...(effort === undefined ? {} : { effort }) };
|
|
3543
|
+
}
|
|
3544
|
+
|
|
3545
|
+
/**
|
|
3546
|
+
* A pane is the default and is left unsaid, and `metered` is kept only where it means something,
|
|
3547
|
+
* so the same agent always has one identity.
|
|
3548
|
+
*/
|
|
3549
|
+
function storedPlacement(choice: PlacementChoice): PlacementChoice {
|
|
3550
|
+
// A workflow is untyped JavaScript at run time; a misspelt placement must not become a pane.
|
|
3551
|
+
if (
|
|
3552
|
+
choice.placement !== undefined &&
|
|
3553
|
+
choice.placement !== "pane" &&
|
|
3554
|
+
choice.placement !== "headless"
|
|
3555
|
+
) {
|
|
3556
|
+
throw new Error(
|
|
3557
|
+
`unknown placement ${JSON.stringify(choice.placement)}; expected pane or headless`,
|
|
3558
|
+
);
|
|
3559
|
+
}
|
|
3560
|
+
return placementOf(choice) === "headless"
|
|
3561
|
+
? { placement: "headless", ...(choice.metered === true ? { metered: true as const } : {}) }
|
|
3562
|
+
: {};
|
|
3563
|
+
}
|
|
3564
|
+
|
|
3565
|
+
/**
|
|
3566
|
+
* The parent's harness, model and effort, in the placement and at the effort the fork names, else
|
|
3567
|
+
* its parent's.
|
|
3568
|
+
*/
|
|
3569
|
+
function forkExecution(parent: AgentExecution, spec: AgentForkSpec): AgentExecution {
|
|
3570
|
+
const { placement: _placement, metered: _metered, ...target } = parent;
|
|
3571
|
+
const choice = spec.placement === undefined && spec.metered === undefined ? parent : spec;
|
|
3572
|
+
return {
|
|
3573
|
+
...target,
|
|
3574
|
+
...(spec.effort === undefined ? {} : { effort: spec.effort }),
|
|
3575
|
+
...storedPlacement(choice),
|
|
3576
|
+
};
|
|
3577
|
+
}
|
|
3578
|
+
|
|
3579
|
+
function constrainExistingExecution(
|
|
3580
|
+
selection: RuntimeSelection,
|
|
3581
|
+
existing: AgentExecution,
|
|
3582
|
+
): AgentExecution {
|
|
3583
|
+
if (typeof selection === "string") {
|
|
3584
|
+
return constrainExistingExecution({ alias: selection }, existing);
|
|
3585
|
+
}
|
|
3586
|
+
if ("alias" in selection) {
|
|
3587
|
+
if (existing.alias !== selection.alias) {
|
|
3588
|
+
throw new Error(`existing agent does not use runtime alias ${selection.alias}`);
|
|
3589
|
+
}
|
|
3590
|
+
for (const field of ["harness", "model"] as const) {
|
|
3591
|
+
const required = selection[field];
|
|
3592
|
+
if (required !== undefined && !isDeepStrictEqual(required, existing[field])) {
|
|
3593
|
+
throw new Error(`existing agent does not satisfy required ${field}`);
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
} else if (selection.harness !== existing.harness || selection.model !== existing.model) {
|
|
3597
|
+
throw new Error("existing agent uses a different runtime configuration");
|
|
3598
|
+
}
|
|
3599
|
+
// As opened: a `set` since does not change what reopening it compares with.
|
|
3600
|
+
if (selection.effort !== undefined && selection.effort !== existing.effort) {
|
|
3601
|
+
throw new Error("existing agent was opened at a different effort");
|
|
3602
|
+
}
|
|
3603
|
+
const placed = placementOf(existing);
|
|
3604
|
+
if (
|
|
3605
|
+
(selection.placement !== undefined && placementOf(selection) !== placed) ||
|
|
3606
|
+
(selection.metered !== undefined &&
|
|
3607
|
+
placed === "headless" &&
|
|
3608
|
+
selection.metered !== existing.metered)
|
|
3609
|
+
) {
|
|
3610
|
+
throw new Error("existing agent uses a different placement");
|
|
3611
|
+
}
|
|
3612
|
+
return existing;
|
|
3613
|
+
}
|
|
3614
|
+
|
|
3615
|
+
function resolveAlias(alias: string, runtime: AgentRuntimeConfig): AgentExecution {
|
|
3616
|
+
const selected = runtime.aliases[alias];
|
|
3617
|
+
if (!selected) throw new Error(`unknown runtime alias: ${alias}`);
|
|
3618
|
+
// Placement is the agent's to choose, never the operator's, so only the target is copied.
|
|
3619
|
+
return {
|
|
3620
|
+
harness: selected.harness,
|
|
3621
|
+
model: selected.model,
|
|
3622
|
+
...(selected.effort === undefined ? {} : { effort: selected.effort }),
|
|
3623
|
+
alias,
|
|
3624
|
+
};
|
|
3625
|
+
}
|
|
3626
|
+
|
|
3627
|
+
function assertCompatibleAgent(
|
|
3628
|
+
key: string,
|
|
3629
|
+
existing: AgentIdentity,
|
|
3630
|
+
requested: AgentIdentity,
|
|
3631
|
+
spec: AgentOpenSpec,
|
|
3632
|
+
): void {
|
|
3633
|
+
if (!isDeepStrictEqual(existing.execution, requested.execution)) conflict(key, "runtime");
|
|
3634
|
+
if (spec.cwd !== undefined && existing.cwd !== requested.cwd) conflict(key, "cwd");
|
|
3635
|
+
if (spec.skills !== undefined && !isDeepStrictEqual(existing.skills, requested.skills)) {
|
|
3636
|
+
conflict(key, "skills");
|
|
3637
|
+
}
|
|
3638
|
+
for (const field of ["instructions", "labels", "layout", "keepPane"] as const) {
|
|
3639
|
+
if (spec[field] !== undefined && !isDeepStrictEqual(existing[field], requested[field])) {
|
|
3640
|
+
conflict(key, field);
|
|
3641
|
+
}
|
|
3642
|
+
}
|
|
3643
|
+
}
|
|
3644
|
+
|
|
3645
|
+
function paneOptions({
|
|
3646
|
+
layout,
|
|
3647
|
+
keepPane,
|
|
3648
|
+
}: AgentIdentity): Pick<HarnessActivation, "layout" | "keepPane"> {
|
|
3649
|
+
return {
|
|
3650
|
+
...(layout === undefined ? {} : { layout }),
|
|
3651
|
+
...(keepPane === undefined ? {} : { keepPane }),
|
|
3652
|
+
};
|
|
3653
|
+
}
|
|
3654
|
+
|
|
3655
|
+
function conflict(key: string, field: string): never {
|
|
3656
|
+
throw new Error(`agent ${key} is already open with different ${field}`);
|
|
3657
|
+
}
|
|
3658
|
+
|
|
3659
|
+
function stoppedError(key: string, reason: string): Error {
|
|
3660
|
+
return new Error(`agent ${key} was stopped (${reason}); its key is not opened again`);
|
|
3661
|
+
}
|
|
3662
|
+
|
|
3663
|
+
function unavailable(name: string): Promise<never> {
|
|
3664
|
+
return Promise.reject(new Error(`${name} is not implemented by the minimum workflow runner`));
|
|
3665
|
+
}
|