@sema-agent/core 7.9.2 → 7.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/agents/child-model-seat.d.ts +31 -1
- package/dist/agents/child-model-seat.js +12 -0
- package/dist/agents/subagent.d.ts +2 -2
- package/dist/agents/subagent.js +6 -4
- package/dist/core/ask-unresolvable-notice.d.ts +52 -0
- package/dist/core/ask-unresolvable-notice.js +25 -0
- package/dist/core/auto-mode.d.ts +62 -3
- package/dist/core/auto-mode.js +31 -0
- package/dist/core/checkpoint-store.d.ts +14 -0
- package/dist/core/checkpoint-store.js +2 -1
- package/dist/core/engine-notice.d.ts +28 -7
- package/dist/core/gate-lanes.js +15 -0
- package/dist/core/governance-codes.d.ts +1 -1
- package/dist/core/governance-codes.js +4 -0
- package/dist/core/hooks.d.ts +24 -1
- package/dist/core/hooks.js +2 -0
- package/dist/core/permission-rule-model.d.ts +51 -16
- package/dist/core/permission-rule-model.js +55 -21
- package/dist/core/permission-rules.d.ts +6 -4
- package/dist/core/permission-rules.js +14 -14
- package/dist/core/runner/contracts.d.ts +26 -2
- package/dist/core/runner/denial-limit-arms.d.ts +14 -3
- package/dist/core/runner/denial-limit-arms.js +15 -5
- package/dist/core/runner/permission-rule-lanes.d.ts +7 -1
- package/dist/core/runner/permission-rule-lanes.js +9 -3
- package/dist/core/runner/prepare-caps-and-workflow.d.ts +1 -1
- package/dist/core/runner/prepare-caps-and-workflow.js +13 -3
- package/dist/core/runner/prepare-gate-stations.d.ts +3 -2
- package/dist/core/runner/prepare-gate-stations.js +3 -0
- package/dist/core/runner/prepare-policy-chain.js +7 -6
- package/dist/core/runner/prepare-wiring-manifest.d.ts +1 -1
- package/dist/core/runner/prepare-wiring-manifest.js +8 -1
- package/dist/core/runner/runtask.d.ts +33 -32
- package/dist/core/runner/runtask.js +63 -33
- package/dist/core/runner-deps.d.ts +9 -2
- package/dist/core/swappable-deps.d.ts +90 -0
- package/dist/core/swappable-deps.js +55 -0
- package/dist/core/tool-policy.d.ts +26 -0
- package/dist/core/tool-policy.js +5 -1
- package/dist/core/wiring-manifest.d.ts +15 -1
- package/dist/core/wiring-manifest.js +10 -2
- package/dist/core/workflow-journal-store.d.ts +21 -2
- package/dist/core/workflow-journal-store.js +1 -1
- package/dist/engine/execution-env/node-execution-env.d.ts +2 -0
- package/dist/engine/execution-env/node-execution-env.js +2 -1
- package/dist/engine/harness/types.d.ts +11 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.js +5 -3
- package/dist/orchestration/run-workflow-tool.d.ts +17 -0
- package/dist/orchestration/run-workflow-tool.js +12 -0
- package/dist/orchestration/workflow-observe.d.ts +1 -1
- package/dist/orchestration/workflow-observe.js +2 -0
- package/dist/orchestration/workflow-types.d.ts +37 -2
- package/dist/orchestration/workflow-types.js +16 -0
- package/dist/orchestration/workflow.d.ts +41 -2
- package/dist/orchestration/workflow.js +295 -44
- package/dist/stores/file/workflow-journal-store.js +10 -3
- package/package.json +1 -1
- package/test/export-surface.snapshot.json +51 -5
|
@@ -11,14 +11,14 @@ import { combinePolicies, createAllowDenyPolicy } from "../core/tool-policy.js";
|
|
|
11
11
|
import { createSafeNotifier } from "../core/safe-notify.js";
|
|
12
12
|
import { callKeyOrdinal, oversizeJournalResult, journalOversizeTombstone, JOURNAL_OVERSIZE_ERROR_CODE, MAX_JOURNAL_RESULT_BYTES } from "../core/workflow-journal-store.js";
|
|
13
13
|
import { isWorkflowRunActive, closeWorkflowChannel, markWorkflowActive, publishWorkflowEvent } from "./workflow-observe.js";
|
|
14
|
-
import { isDurablePause
|
|
14
|
+
import { isDurablePause } from "../agents/suspend-guard.js";
|
|
15
15
|
import { RUNNING_AGENT_OBSERVE_EVERY_BEATS } from "../config/defaults.js";
|
|
16
16
|
import { boundInputHashOf } from "../core/canonical-json.js";
|
|
17
17
|
import { boundedRedactedSummary } from "../core/untrusted-egress.js";
|
|
18
18
|
import { delimitUntrusted } from "../core/untrusted-text.js";
|
|
19
19
|
import { OUTPUT_TOOL_NAME } from "../core/runner/synthetic-tools.js";
|
|
20
20
|
import { compileOutputSchema } from "../core/runner/strict-output-schema.js";
|
|
21
|
-
import { WorkflowBudgetExceededError, WorkflowNestingError, WorkflowAgentBlockedError, WorkflowAgentSchemaError, WorkflowAgentStalledError, WorkflowMaxAgentsError, WORKFLOW_SPAWN_BLOCKED_ERROR_CODE } from "./workflow-types.js";
|
|
21
|
+
import { WorkflowBudgetExceededError, WorkflowNestingError, WorkflowAgentBlockedError, WorkflowAgentParkedError, WorkflowAgentSchemaError, WorkflowAgentStalledError, WorkflowMaxAgentsError, WORKFLOW_SPAWN_BLOCKED_ERROR_CODE } from "./workflow-types.js";
|
|
22
22
|
import { WorkflowScriptError } from "./workflow-meta.js";
|
|
23
23
|
export * from "./workflow-types.js";
|
|
24
24
|
const MAX_TRANSCRIPT_CHARS = 4000;
|
|
@@ -206,10 +206,9 @@ export class WorkflowJournalIncompatibleError extends Error {
|
|
|
206
206
|
fromRunId;
|
|
207
207
|
ordinal;
|
|
208
208
|
code = "workflow.journal_incompatible";
|
|
209
|
-
constructor(fromRunId, ordinal) {
|
|
210
|
-
super(`startWorkflow: resume from "${fromRunId}" was REFUSED — journal entry #${ordinal} carries no terminal cause ` +
|
|
211
|
-
"
|
|
212
|
-
"start a fresh run instead of resuming this journal.");
|
|
209
|
+
constructor(fromRunId, ordinal, reason) {
|
|
210
|
+
super(`startWorkflow: resume from "${fromRunId}" was REFUSED — ${reason ?? `journal entry #${ordinal} carries no terminal cause (a journal written before TaskResult.terminal, or a malformed entry)`}. ` +
|
|
211
|
+
"Completed work must not run again; start a fresh run instead of resuming this journal.");
|
|
213
212
|
this.fromRunId = fromRunId;
|
|
214
213
|
this.ordinal = ordinal;
|
|
215
214
|
this.name = "WorkflowJournalIncompatibleError";
|
|
@@ -448,6 +447,12 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
448
447
|
if (opts.resumeFromRunId !== undefined && opts.journalStore === undefined) {
|
|
449
448
|
throw new Error("runWorkflow: resumeFromRunId requires a journalStore to load the prior run's journal");
|
|
450
449
|
}
|
|
450
|
+
if (opts.defaultDurableApproval !== undefined && opts.store === undefined) {
|
|
451
|
+
throw new Error("runWorkflow: defaultDurableApproval requires a WorkflowRunStore — a workflow child's park is recorded on its wa* row, and without a store there is no durable row for a host to route the parked approval by");
|
|
452
|
+
}
|
|
453
|
+
if (opts.parkedResume !== undefined && opts.resumeFromRunId === undefined) {
|
|
454
|
+
throw new Error("runWorkflow: parkedResume drives a PARKED ordinal of a prior run and requires resumeFromRunId");
|
|
455
|
+
}
|
|
451
456
|
const now = opts.now ?? (() => Date.now());
|
|
452
457
|
const scope = opts.scope ?? "default";
|
|
453
458
|
const runId = opts.runId ?? randomUUID();
|
|
@@ -769,6 +774,60 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
769
774
|
let resumeClaim;
|
|
770
775
|
const replayByOrdinal = [];
|
|
771
776
|
let diverged = false;
|
|
777
|
+
let parkedLeg;
|
|
778
|
+
const parkedError = () => {
|
|
779
|
+
const p = parkedLeg;
|
|
780
|
+
return new WorkflowAgentParkedError(p.label, p.paused.terminal.gate.kind, p.paused.terminal.checkpointId);
|
|
781
|
+
};
|
|
782
|
+
const parkedResumeFor = (token) => opts.parkedResume?.find((d) => d.token === token);
|
|
783
|
+
const parkedCallChangedError = (ordinal) => Object.assign(new Error(`workflow resume: ordinal ${ordinal} is PARKED on an approval checkpoint, but the call at that ordinal changed (its call key differs from the journaled one) — ` +
|
|
784
|
+
"the script or its args were edited at a parked call, and a fresh spawn there would run beside the pinned child. Restore the call and resume again, or decide the checkpoint " +
|
|
785
|
+
"through the deployment's approval channel and start a fresh run."), { code: "workflow.parked_call_changed" });
|
|
786
|
+
let inFlightLegs = 0;
|
|
787
|
+
let legsIdleResolve;
|
|
788
|
+
const legStart = () => {
|
|
789
|
+
inFlightLegs += 1;
|
|
790
|
+
};
|
|
791
|
+
const legEnd = () => {
|
|
792
|
+
inFlightLegs -= 1;
|
|
793
|
+
if (inFlightLegs <= 0) {
|
|
794
|
+
inFlightLegs = 0;
|
|
795
|
+
const r = legsIdleResolve;
|
|
796
|
+
legsIdleResolve = undefined;
|
|
797
|
+
r?.();
|
|
798
|
+
}
|
|
799
|
+
};
|
|
800
|
+
const awaitLegsIdle = async (signal) => {
|
|
801
|
+
while (inFlightLegs > 0 && signal?.aborted !== true) {
|
|
802
|
+
await new Promise((resolve) => {
|
|
803
|
+
legsIdleResolve = resolve;
|
|
804
|
+
signal?.addEventListener("abort", () => resolve(), { once: true });
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
};
|
|
808
|
+
const writtenOrdinals = new Set();
|
|
809
|
+
const carryUnresolvedParks = async () => {
|
|
810
|
+
if (opts.resumeFromRunId === undefined || !journalStore)
|
|
811
|
+
return;
|
|
812
|
+
for (const e of replayByOrdinal) {
|
|
813
|
+
if (e?.parked === undefined)
|
|
814
|
+
continue;
|
|
815
|
+
const ord = callKeyOrdinal(e.callKey);
|
|
816
|
+
if (writtenOrdinals.has(ord))
|
|
817
|
+
continue;
|
|
818
|
+
if (workflowSignal.aborted) {
|
|
819
|
+
emitLogLine(`resume-journal: the prior run's PARKED leg at ordinal ${ord} was NOT carried into this run's journal (the run was cancelled or timed out first) — resume from the prior run, not from this one.`);
|
|
820
|
+
continue;
|
|
821
|
+
}
|
|
822
|
+
const outcome = await finalizeWithin(timers, journalAppendParked(e.callKey, e.parked).then(() => "ok", (err) => ({ err })), "timeout");
|
|
823
|
+
if (outcome !== "ok") {
|
|
824
|
+
const why = outcome === "timeout" ? `the append did not settle within ${WORKFLOW_RESUME_CLAIM_FINALIZE_TIMEOUT_MS}ms` : outcome.err instanceof Error ? outcome.err.message : String(outcome.err);
|
|
825
|
+
emitLogLine(`resume-journal: the prior run's PARKED leg at ordinal ${ord} could not be carried into this run's journal (${why}) — resume from the prior run, not from this one.`);
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
};
|
|
829
|
+
const driveNeverStarted = (r) => r.terminal.kind === "failed" && (r.terminal.code === "resume.reopened_unstarted" || r.terminal.code === "resume.reopen_failed_unstarted");
|
|
830
|
+
const isPausedResult = (r) => isDurablePause(r.terminal);
|
|
772
831
|
let isolationAdvised = false;
|
|
773
832
|
const adviseIsolationIfConcurrent = (opts) => {
|
|
774
833
|
if (isolationAdvised || opts.isolation || sem.active <= 1)
|
|
@@ -783,6 +842,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
783
842
|
};
|
|
784
843
|
let journalTail = Promise.resolve();
|
|
785
844
|
const JOURNAL_DRAIN_MAX_MS = 5_000;
|
|
845
|
+
const durableApprovalInherit = (s) => s.durableApproval === undefined && opts.defaultDurableApproval !== undefined ? { durableApproval: { ...opts.defaultDurableApproval } } : {};
|
|
786
846
|
const journalDurableResult = (result) => {
|
|
787
847
|
const t = result.terminal;
|
|
788
848
|
if (t === undefined || t.kind !== "failed" || typeof t.message !== "string" || t.message === "")
|
|
@@ -821,8 +881,10 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
821
881
|
if (dbg)
|
|
822
882
|
console.error(`[sema:wf-journal] runId=${runId} callKey=${callKey} TOMBSTONE-ERROR ${err instanceof Error ? err.message : String(err)}`);
|
|
823
883
|
}
|
|
884
|
+
writtenOrdinals.add(callKeyOrdinal(callKey));
|
|
824
885
|
return;
|
|
825
886
|
}
|
|
887
|
+
writtenOrdinals.add(callKeyOrdinal(callKey));
|
|
826
888
|
const p = journalStore.append(runId, scope, { callKey, result: stored });
|
|
827
889
|
journalTail = journalTail.then(() => p).catch(() => undefined);
|
|
828
890
|
try {
|
|
@@ -836,6 +898,56 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
836
898
|
throw err;
|
|
837
899
|
}
|
|
838
900
|
};
|
|
901
|
+
const journalAppendParked = async (callKey, paused) => {
|
|
902
|
+
if (!journalStore)
|
|
903
|
+
return;
|
|
904
|
+
writtenOrdinals.add(callKeyOrdinal(callKey));
|
|
905
|
+
let lastErr;
|
|
906
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
907
|
+
const p = journalStore.append(runId, scope, { callKey, parked: paused });
|
|
908
|
+
journalTail = journalTail.then(() => p).catch(() => undefined);
|
|
909
|
+
try {
|
|
910
|
+
await p;
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
catch (err) {
|
|
914
|
+
lastErr = err;
|
|
915
|
+
if (attempt < 2)
|
|
916
|
+
await new Promise((r) => timers.setTimeout(r, 25 * (attempt + 1)));
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
throw lastErr;
|
|
920
|
+
};
|
|
921
|
+
const parkAgentLeg = async (rec, paused, activityTail, chargeLive) => {
|
|
922
|
+
const s = paused.stats;
|
|
923
|
+
accumulateStats(paused, chargeLive);
|
|
924
|
+
rec.status = "parked";
|
|
925
|
+
rec.parkedCheckpointToken = paused.terminal.token;
|
|
926
|
+
rec.taskStatus = terminalProjection(paused.terminal).status;
|
|
927
|
+
if (paused.sessionId)
|
|
928
|
+
rec.sessionId = paused.sessionId;
|
|
929
|
+
setLegRunId(rec, paused);
|
|
930
|
+
rec.endedAt = now();
|
|
931
|
+
rec.stats = { tokens: s.tokens ?? 0, turns: s.turns ?? 0, costMicroUsd: s.costMicroUsd };
|
|
932
|
+
if (s.toolCalls !== undefined)
|
|
933
|
+
rec.toolCalls = s.toolCalls;
|
|
934
|
+
if (activityTail.length > 0)
|
|
935
|
+
rec.activity = activityTail.slice();
|
|
936
|
+
foldLegApprovalReport(rec, paused);
|
|
937
|
+
emit({ type: "agent_end", runId, label: rec.label, phase: rec.phase, ...(rec.groupId !== undefined ? { groupId: rec.groupId } : {}), status: "parked", ...(paused.model !== undefined ? { modelResolved: paused.model } : {}), ts: rec.endedAt });
|
|
938
|
+
void persist("update");
|
|
939
|
+
bceLive.delete(waIdOf(rec.callKey));
|
|
940
|
+
if (parkedLeg === undefined)
|
|
941
|
+
parkedLeg = { label: rec.label, callKey: rec.callKey, paused };
|
|
942
|
+
try {
|
|
943
|
+
await journalAppendParked(rec.callKey, paused);
|
|
944
|
+
}
|
|
945
|
+
catch (err) {
|
|
946
|
+
emitRunLog(`resume-journal: the PARKED leg at ordinal ${callKeyOrdinal(rec.callKey)} ("${rec.label.slice(0, 80)}") could not be recorded (${err instanceof Error ? err.message : String(err)}) — ` +
|
|
947
|
+
"a resume of this run will NOT find the park at its ordinal; decide the checkpoint through the deployment's approval channel and start a fresh run instead of resuming this one.");
|
|
948
|
+
}
|
|
949
|
+
throw parkedError();
|
|
950
|
+
};
|
|
839
951
|
const createActivityCapture = (callKey, label, groupId, rec) => {
|
|
840
952
|
const tail = [];
|
|
841
953
|
let beatCount = 0;
|
|
@@ -1191,9 +1303,34 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1191
1303
|
async agent(spec, agentOpts = {}) {
|
|
1192
1304
|
const effectiveSignal = assertAgentSpawnAllowed("ctx.agent", spec, agentOpts);
|
|
1193
1305
|
const { label, phase, phaseInstance, groupId, inheritedModelSnap, inheritedModelError, callKey, prompt, model } = prepareAgentCall(spec, agentOpts);
|
|
1306
|
+
let drive;
|
|
1307
|
+
const cached = opts.resumeFromRunId !== undefined ? replayByOrdinal[run.agents.length] : undefined;
|
|
1308
|
+
if (cached?.parked !== undefined) {
|
|
1309
|
+
if (cached.callKey !== callKey)
|
|
1310
|
+
throw parkedCallChangedError(run.agents.length);
|
|
1311
|
+
{
|
|
1312
|
+
const decision = parkedResumeFor(cached.parked.terminal.token);
|
|
1313
|
+
if (decision === undefined) {
|
|
1314
|
+
if (finalized)
|
|
1315
|
+
return cached.parked;
|
|
1316
|
+
const at = now();
|
|
1317
|
+
const reparkRec = { label, callKey, ...(groupId !== undefined ? { groupId } : {}), phase, prompt, ...(cached.parked.model !== undefined ? { model: cached.parked.model } : {}), replayed: true, status: "running", queuedAt: at, startedAt: at };
|
|
1318
|
+
if (run.resume)
|
|
1319
|
+
run.resume.replayed += 1;
|
|
1320
|
+
run.agents.push(reparkRec);
|
|
1321
|
+
if (phaseInstance)
|
|
1322
|
+
agentPhaseOf.set(reparkRec, phaseInstance);
|
|
1323
|
+
emit({ type: "agent_start", runId, label, phase, ...(groupId !== undefined ? { groupId } : {}), callKey, prompt, ...(cached.parked.model !== undefined ? { model: cached.parked.model } : {}), replayed: true, ts: at });
|
|
1324
|
+
bceSpawn(callKey, label, agentOpts.agentType, true, cached.parked.sessionId || undefined, cached.parked.model);
|
|
1325
|
+
return await parkAgentLeg(reparkRec, cached.parked, [], false);
|
|
1326
|
+
}
|
|
1327
|
+
drive = { token: decision.token, outcome: decision.outcome, ...(decision.inheritedGate !== undefined ? { inheritedGate: decision.inheritedGate } : {}), sessionId: cached.parked.sessionId || undefined, paused: cached.parked };
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
if (parkedLeg !== undefined && drive === undefined)
|
|
1331
|
+
throw parkedError();
|
|
1194
1332
|
if (opts.resumeFromRunId !== undefined && !diverged) {
|
|
1195
|
-
|
|
1196
|
-
if (cached && cached.callKey === callKey && TERMINAL_CAUSE_IS_REPLAYABLE[cached.result.terminal.kind]) {
|
|
1333
|
+
if (drive === undefined && cached && cached.callKey === callKey && cached.result !== undefined && TERMINAL_CAUSE_IS_REPLAYABLE[cached.result.terminal.kind]) {
|
|
1197
1334
|
if (finalized)
|
|
1198
1335
|
return cached.result;
|
|
1199
1336
|
const r = cached.result;
|
|
@@ -1237,18 +1374,20 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1237
1374
|
}
|
|
1238
1375
|
return r;
|
|
1239
1376
|
}
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
?
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1377
|
+
if (drive === undefined) {
|
|
1378
|
+
noteDivergence(run.agents.length, cached === undefined
|
|
1379
|
+
? run.resume?.journalEntries === 0
|
|
1380
|
+
? "the journal loaded no entries"
|
|
1381
|
+
: run.agents.length >= replayByOrdinal.length
|
|
1382
|
+
? "a new call past the journaled prefix — the prior run ended before this point (the intended extend-the-script resume shape)"
|
|
1383
|
+
: "no journal entry at this ordinal (never recorded, e.g. an oversize result under a pre-tombstone engine, or dropped)"
|
|
1384
|
+
: cached.callKey !== callKey
|
|
1385
|
+
? "the call key changed — the script or its args differ here"
|
|
1386
|
+
: cached.result?.terminal?.kind === "failed" && cached.result.terminal.code === JOURNAL_OVERSIZE_ERROR_CODE
|
|
1387
|
+
? "the prior result exceeded the journal size cap and was never cached"
|
|
1388
|
+
: `the journaled result was ${cached.result?.terminal?.kind ?? "(no terminal record)"}, not completed`);
|
|
1389
|
+
diverged = true;
|
|
1390
|
+
}
|
|
1252
1391
|
}
|
|
1253
1392
|
if (budgetTotal !== null && spent() >= budgetTotal) {
|
|
1254
1393
|
throw new WorkflowBudgetExceededError(spent(), budgetTotal);
|
|
@@ -1260,6 +1399,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1260
1399
|
agentPhaseOf.set(rec, phaseInstance);
|
|
1261
1400
|
emit({ type: "agent_start", runId, label, phase, ...(groupId !== undefined ? { groupId } : {}), callKey, prompt, ...(model !== undefined ? { model } : {}), queuedAt: rec.queuedAt, ts: rec.queuedAt });
|
|
1262
1401
|
void persist("update");
|
|
1402
|
+
legStart();
|
|
1263
1403
|
const release = await sem.acquire();
|
|
1264
1404
|
adviseIsolationIfConcurrent(agentOpts);
|
|
1265
1405
|
let result;
|
|
@@ -1268,11 +1408,13 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1268
1408
|
throw new Error("workflow aborted");
|
|
1269
1409
|
if (finalized)
|
|
1270
1410
|
throw new Error("workflow run already finalized — ctx.agent cannot spawn after the run ended");
|
|
1411
|
+
if (parkedLeg !== undefined && drive === undefined)
|
|
1412
|
+
throw parkedError();
|
|
1271
1413
|
if (budgetTotal !== null && spent() >= budgetTotal) {
|
|
1272
1414
|
throw new WorkflowBudgetExceededError(spent(), budgetTotal);
|
|
1273
1415
|
}
|
|
1274
1416
|
rec.startedAt = now();
|
|
1275
|
-
const bornChildSessionId = resolveChildSessionIdAtSpawn(spec);
|
|
1417
|
+
const bornChildSessionId = drive !== undefined ? drive.sessionId : resolveChildSessionIdAtSpawn(spec);
|
|
1276
1418
|
bceSpawn(callKey, label, agentOpts.agentType, false, bornChildSessionId, model);
|
|
1277
1419
|
if (inheritedModelError !== undefined)
|
|
1278
1420
|
throw inheritedModelError;
|
|
@@ -1280,9 +1422,10 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1280
1422
|
const typedSpec = typedSpec0.model === undefined && inheritedModelSnap !== undefined ? { ...typedSpec0, model: inheritedModelSnap } : typedSpec0;
|
|
1281
1423
|
const framedSpec = withWorkflowChildPersona(typedSpec, agentOpts.schema ?? typedSpec.outputSchema);
|
|
1282
1424
|
const authInherit = framedSpec.getApiKeyAndHeaders === undefined && opts.defaultGetApiKeyAndHeaders !== undefined ? { getApiKeyAndHeaders: opts.defaultGetApiKeyAndHeaders } : {};
|
|
1425
|
+
const parkInherit = durableApprovalInherit(framedSpec);
|
|
1283
1426
|
const preReviewSpec = agentOpts.schema
|
|
1284
|
-
? { ...framedSpec, ...authInherit, signal: effectiveSignal, outputSchema: agentOpts.schema }
|
|
1285
|
-
: { ...framedSpec, ...authInherit, signal: effectiveSignal };
|
|
1427
|
+
? { ...framedSpec, ...authInherit, ...parkInherit, signal: effectiveSignal, outputSchema: agentOpts.schema }
|
|
1428
|
+
: { ...framedSpec, ...authInherit, ...parkInherit, signal: effectiveSignal };
|
|
1286
1429
|
const runSpec = await reviewSpawnBeforeLaunch("ctx.agent", label, callKey, preReviewSpec, effectiveSignal);
|
|
1287
1430
|
const enrichedForward = opts.onForwardEvent !== undefined
|
|
1288
1431
|
? (e) => {
|
|
@@ -1319,7 +1462,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1319
1462
|
const onCallerAbort = () => attemptCtl.abort(new Error("workflow aborted"));
|
|
1320
1463
|
effectiveSignal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
1321
1464
|
armWatchdog();
|
|
1322
|
-
const attemptSessionId = attempts === 1 ? bornChildSessionId : resolveChildSessionIdAtSpawn(runSpec);
|
|
1465
|
+
const attemptSessionId = drive !== undefined ? drive.sessionId : attempts === 1 ? bornChildSessionId : resolveChildSessionIdAtSpawn(runSpec);
|
|
1323
1466
|
const attemptSpec = attemptSessionId !== undefined ? { ...runSpec, sessionId: attemptSessionId, signal: attemptCtl.signal } : { ...runSpec, signal: attemptCtl.signal };
|
|
1324
1467
|
if (attemptSessionId !== undefined && !finalized) {
|
|
1325
1468
|
rec.sessionId = attemptSessionId;
|
|
@@ -1373,10 +1516,15 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1373
1516
|
activeUsageBeats.add(activeBeat);
|
|
1374
1517
|
try {
|
|
1375
1518
|
attemptResult = await workflowDepthStore.run({ depth: depth + 1 }, async () => {
|
|
1376
|
-
if (typeof runner.runTaskStream !== "function") {
|
|
1519
|
+
if (drive === undefined && typeof runner.runTaskStream !== "function") {
|
|
1377
1520
|
return runner.runTask(attemptSpec, attemptInternals);
|
|
1378
1521
|
}
|
|
1379
|
-
const stream =
|
|
1522
|
+
const stream = drive === undefined
|
|
1523
|
+
? runner.runTaskStream(attemptSpec, undefined, attemptInternals)
|
|
1524
|
+
: await (() => {
|
|
1525
|
+
const { objective: _o, sessionId: _s, ...resumeConfig } = attemptSpec;
|
|
1526
|
+
return runner.resumeStream(drive.token, drive.outcome, resumeConfig, { ...attemptInternals, ...(drive.inheritedGate !== undefined ? { inheritedGate: drive.inheritedGate } : {}) });
|
|
1527
|
+
})();
|
|
1380
1528
|
for await (const ev of stream) {
|
|
1381
1529
|
if (ev.type === "text_delta" || ev.type === "reasoning_delta")
|
|
1382
1530
|
onStreamProgress();
|
|
@@ -1409,7 +1557,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1409
1557
|
}
|
|
1410
1558
|
throw new Error("workflow aborted");
|
|
1411
1559
|
}
|
|
1412
|
-
if (stallFired && attemptResult?.terminal.kind !== "completed") {
|
|
1560
|
+
if (drive === undefined && stallFired && attemptResult?.terminal.kind !== "completed") {
|
|
1413
1561
|
if (attemptResult !== undefined && !finalized)
|
|
1414
1562
|
accumulateStats(attemptResult, true);
|
|
1415
1563
|
if (attemptResult !== undefined && !finalized)
|
|
@@ -1438,7 +1586,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1438
1586
|
(result.stats.outputTokens ?? Number.POSITIVE_INFINITY) < 50 &&
|
|
1439
1587
|
lastStopReason === undefined &&
|
|
1440
1588
|
durationMs > stallMs * 0.5;
|
|
1441
|
-
if (degraded && attempts === 1 && !throttleRetried) {
|
|
1589
|
+
if (degraded && attempts === 1 && !throttleRetried && drive === undefined) {
|
|
1442
1590
|
throttleRetried = true;
|
|
1443
1591
|
lastAttemptReason = "throttled";
|
|
1444
1592
|
emitRunLog(`[${label}] throttled response (${(result.result ?? "").trim().length} chars in ${Math.round(durationMs / 1000)}s) — sleeping ${Math.round(throttleBackoffMs / 1000)}s before retry`);
|
|
@@ -1474,6 +1622,10 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1474
1622
|
}
|
|
1475
1623
|
}
|
|
1476
1624
|
catch (err) {
|
|
1625
|
+
if (drive !== undefined && !finalized && !(err instanceof WorkflowAgentParkedError)) {
|
|
1626
|
+
emitRunLog(`[park] the parked leg "${label}" could not be driven — the resume was refused before the checkpoint was consumed (${err instanceof Error ? err.message : String(err)}); the park stands.`);
|
|
1627
|
+
return await parkAgentLeg(rec, drive.paused, activityTail, false);
|
|
1628
|
+
}
|
|
1477
1629
|
if (!finalized) {
|
|
1478
1630
|
rec.status = "failed";
|
|
1479
1631
|
rec.endedAt = now();
|
|
@@ -1520,7 +1672,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1520
1672
|
},
|
|
1521
1673
|
stats: { turns: 0, tokens: 0, costMicroUsd: 0 },
|
|
1522
1674
|
}, label).catch(() => undefined);
|
|
1523
|
-
if (!(err instanceof WorkflowBudgetExceededError && rec.startedAt === undefined))
|
|
1675
|
+
if (!((err instanceof WorkflowBudgetExceededError || err instanceof WorkflowAgentParkedError) && rec.startedAt === undefined))
|
|
1524
1676
|
await journaled;
|
|
1525
1677
|
bceTerminal(callKey, "failed", rec.output ?? (err instanceof Error ? err.message : String(err)), rec.sessionId, rec.stats);
|
|
1526
1678
|
}
|
|
@@ -1528,11 +1680,16 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1528
1680
|
}
|
|
1529
1681
|
finally {
|
|
1530
1682
|
release();
|
|
1683
|
+
legEnd();
|
|
1531
1684
|
}
|
|
1532
1685
|
if (finalized)
|
|
1533
1686
|
return result;
|
|
1534
|
-
if (
|
|
1535
|
-
|
|
1687
|
+
if (isPausedResult(result)) {
|
|
1688
|
+
return await parkAgentLeg(rec, result, activityTail, true);
|
|
1689
|
+
}
|
|
1690
|
+
if (drive !== undefined && driveNeverStarted(result)) {
|
|
1691
|
+
emitRunLog(`[park] the parked leg "${label}" was driven but the approved action never started (${result.terminal.kind === "failed" ? result.terminal.code : ""}); the park stands.`);
|
|
1692
|
+
return await parkAgentLeg(rec, drive.paused, activityTail, false);
|
|
1536
1693
|
}
|
|
1537
1694
|
await settleAgentResult(rec, result, activityTail, agentOpts, "awaited");
|
|
1538
1695
|
return result;
|
|
@@ -1543,8 +1700,30 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1543
1700
|
throw new WorkflowBudgetExceededError(spent(), budgetTotal);
|
|
1544
1701
|
}
|
|
1545
1702
|
const { label, phase, phaseInstance, groupId, inheritedModelSnap, inheritedModelError, callKey, prompt, model } = prepareAgentCall(spec, agentOpts);
|
|
1546
|
-
|
|
1547
|
-
|
|
1703
|
+
let drive;
|
|
1704
|
+
const parkedHere = opts.resumeFromRunId !== undefined ? replayByOrdinal[run.agents.length] : undefined;
|
|
1705
|
+
if (parkedHere?.parked !== undefined) {
|
|
1706
|
+
if (parkedHere.callKey !== callKey)
|
|
1707
|
+
throw parkedCallChangedError(run.agents.length);
|
|
1708
|
+
const decision = parkedResumeFor(parkedHere.parked.terminal.token);
|
|
1709
|
+
if (decision === undefined) {
|
|
1710
|
+
const at = now();
|
|
1711
|
+
const reparkRec = { label, callKey, ...(groupId !== undefined ? { groupId } : {}), phase, prompt, ...(parkedHere.parked.model !== undefined ? { model: parkedHere.parked.model } : {}), replayed: true, status: "running", queuedAt: at, startedAt: at };
|
|
1712
|
+
if (run.resume)
|
|
1713
|
+
run.resume.replayed += 1;
|
|
1714
|
+
run.agents.push(reparkRec);
|
|
1715
|
+
if (phaseInstance)
|
|
1716
|
+
agentPhaseOf.set(reparkRec, phaseInstance);
|
|
1717
|
+
emit({ type: "agent_start", runId, label, phase, ...(groupId !== undefined ? { groupId } : {}), callKey, prompt, ...(parkedHere.parked.model !== undefined ? { model: parkedHere.parked.model } : {}), replayed: true, ts: at });
|
|
1718
|
+
bceSpawn(callKey, label, agentOpts.agentType, true, parkedHere.parked.sessionId || undefined, parkedHere.parked.model);
|
|
1719
|
+
return await parkAgentLeg(reparkRec, parkedHere.parked, [], false);
|
|
1720
|
+
}
|
|
1721
|
+
drive = { token: decision.token, outcome: decision.outcome, ...(decision.inheritedGate !== undefined ? { inheritedGate: decision.inheritedGate } : {}), sessionId: parkedHere.parked.sessionId || undefined, paused: parkedHere.parked };
|
|
1722
|
+
}
|
|
1723
|
+
if (drive === undefined) {
|
|
1724
|
+
noteDivergence(run.agents.length, "ctx.agentStream results are never replayed");
|
|
1725
|
+
diverged = true;
|
|
1726
|
+
}
|
|
1548
1727
|
const rec = { label, callKey, ...(groupId !== undefined ? { groupId } : {}), phase, prompt, ...(model !== undefined ? { model } : {}), status: "running", queuedAt: now() };
|
|
1549
1728
|
const { tail: activityTail, onActivity } = createActivityCapture(callKey, label, groupId, rec);
|
|
1550
1729
|
run.agents.push(rec);
|
|
@@ -1552,6 +1731,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1552
1731
|
agentPhaseOf.set(rec, phaseInstance);
|
|
1553
1732
|
emit({ type: "agent_start", runId, label, phase, ...(groupId !== undefined ? { groupId } : {}), callKey, prompt, ...(model !== undefined ? { model } : {}), queuedAt: rec.queuedAt, ts: rec.queuedAt });
|
|
1554
1733
|
void persist("update");
|
|
1734
|
+
legStart();
|
|
1555
1735
|
const release = await sem.acquire();
|
|
1556
1736
|
adviseIsolationIfConcurrent(agentOpts);
|
|
1557
1737
|
let released = false;
|
|
@@ -1569,17 +1749,34 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1569
1749
|
void persist("update");
|
|
1570
1750
|
}
|
|
1571
1751
|
};
|
|
1752
|
+
const releaseLeg = () => {
|
|
1753
|
+
if (!released)
|
|
1754
|
+
legEnd();
|
|
1755
|
+
releaseOnce();
|
|
1756
|
+
};
|
|
1572
1757
|
if (effectiveSignal?.aborted || finalized) {
|
|
1573
1758
|
recordFailed();
|
|
1574
|
-
|
|
1759
|
+
releaseLeg();
|
|
1575
1760
|
throw new Error(finalized ? "workflow run already finalized — ctx.agentStream cannot spawn after the run ended" : "workflow aborted");
|
|
1576
1761
|
}
|
|
1762
|
+
if (parkedLeg !== undefined && drive === undefined) {
|
|
1763
|
+
const parked = parkedError();
|
|
1764
|
+
rec.errorMessage = boundedRedactedSummary(parked.message, MAX_TRANSCRIPT_CHARS);
|
|
1765
|
+
recordFailed();
|
|
1766
|
+
releaseLeg();
|
|
1767
|
+
throw parked;
|
|
1768
|
+
}
|
|
1577
1769
|
if (budgetTotal !== null && spent() >= budgetTotal) {
|
|
1770
|
+
if (drive !== undefined) {
|
|
1771
|
+
emitRunLog(`[park] the parked leg "${label}" could not be driven — the run's token ceiling was reached before its turn; the park stands.`);
|
|
1772
|
+
releaseLeg();
|
|
1773
|
+
return await parkAgentLeg(rec, drive.paused, [], false);
|
|
1774
|
+
}
|
|
1578
1775
|
const refusal = new WorkflowBudgetExceededError(spent(), budgetTotal);
|
|
1579
1776
|
rec.errorCode = refusal.code;
|
|
1580
1777
|
rec.errorMessage = boundedRedactedSummary(refusal.message, MAX_TRANSCRIPT_CHARS);
|
|
1581
1778
|
recordFailed();
|
|
1582
|
-
|
|
1779
|
+
releaseLeg();
|
|
1583
1780
|
void journalAppend(callKey, {
|
|
1584
1781
|
taskId: callKey,
|
|
1585
1782
|
sessionId: "",
|
|
@@ -1590,7 +1787,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1590
1787
|
throw refusal;
|
|
1591
1788
|
}
|
|
1592
1789
|
rec.startedAt = now();
|
|
1593
|
-
const childSessionId = resolveChildSessionIdAtSpawn(spec);
|
|
1790
|
+
const childSessionId = drive !== undefined ? drive.sessionId : resolveChildSessionIdAtSpawn(spec);
|
|
1594
1791
|
bceSpawn(callKey, label, agentOpts.agentType, false, childSessionId, model);
|
|
1595
1792
|
let stream;
|
|
1596
1793
|
try {
|
|
@@ -1600,9 +1797,10 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1600
1797
|
const typedSpec = typedSpec0.model === undefined && inheritedModelSnap !== undefined ? { ...typedSpec0, model: inheritedModelSnap } : typedSpec0;
|
|
1601
1798
|
const framedSpec = withWorkflowChildPersona(typedSpec, agentOpts.schema ?? typedSpec.outputSchema);
|
|
1602
1799
|
const authInherit = framedSpec.getApiKeyAndHeaders === undefined && opts.defaultGetApiKeyAndHeaders !== undefined ? { getApiKeyAndHeaders: opts.defaultGetApiKeyAndHeaders } : {};
|
|
1800
|
+
const parkInherit = durableApprovalInherit(framedSpec);
|
|
1603
1801
|
const baseRunSpec = agentOpts.schema
|
|
1604
|
-
? { ...framedSpec, ...authInherit, signal: effectiveSignal, outputSchema: agentOpts.schema }
|
|
1605
|
-
: { ...framedSpec, ...authInherit, signal: effectiveSignal };
|
|
1802
|
+
? { ...framedSpec, ...authInherit, ...parkInherit, signal: effectiveSignal, outputSchema: agentOpts.schema }
|
|
1803
|
+
: { ...framedSpec, ...authInherit, ...parkInherit, signal: effectiveSignal };
|
|
1606
1804
|
const preReviewSpec = childSessionId !== undefined ? { ...baseRunSpec, sessionId: childSessionId } : baseRunSpec;
|
|
1607
1805
|
const runSpec = await reviewSpawnBeforeLaunch("ctx.agentStream", label, callKey, preReviewSpec, effectiveSignal);
|
|
1608
1806
|
const enrichedForwardS = opts.onForwardEvent !== undefined
|
|
@@ -1634,9 +1832,20 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1634
1832
|
}
|
|
1635
1833
|
: {}),
|
|
1636
1834
|
};
|
|
1637
|
-
stream =
|
|
1835
|
+
stream = drive === undefined
|
|
1836
|
+
? workflowDepthStore.run({ depth: depth + 1 }, () => runner.runTaskStream(runSpec, undefined, runInternals))
|
|
1837
|
+
: await workflowDepthStore.run({ depth: depth + 1 }, () => {
|
|
1838
|
+
const d = drive;
|
|
1839
|
+
const { objective: _o, sessionId: _s, ...resumeConfig } = runSpec;
|
|
1840
|
+
return runner.resumeStream(d.token, d.outcome, resumeConfig, { ...runInternals, ...(d.inheritedGate !== undefined ? { inheritedGate: d.inheritedGate } : {}) });
|
|
1841
|
+
});
|
|
1638
1842
|
}
|
|
1639
1843
|
catch (err) {
|
|
1844
|
+
if (drive !== undefined && !finalized) {
|
|
1845
|
+
emitRunLog(`[park] the parked leg "${label}" could not be driven — the resume was refused before the checkpoint was consumed (${err instanceof Error ? err.message : String(err)}); the park stands.`);
|
|
1846
|
+
releaseLeg();
|
|
1847
|
+
return await parkAgentLeg(rec, drive.paused, [], false);
|
|
1848
|
+
}
|
|
1640
1849
|
if (!finalized && rec.errorMessage === undefined && err instanceof Error && err.message !== "")
|
|
1641
1850
|
rec.errorMessage = boundedRedactedSummary(err.message, MAX_TRANSCRIPT_CHARS);
|
|
1642
1851
|
if (!finalized && err instanceof WorkflowAgentBlockedError) {
|
|
@@ -1651,7 +1860,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1651
1860
|
}).catch(() => undefined);
|
|
1652
1861
|
}
|
|
1653
1862
|
recordFailed();
|
|
1654
|
-
|
|
1863
|
+
releaseLeg();
|
|
1655
1864
|
bceTerminal(callKey, "failed", err instanceof Error ? err.message : String(err), undefined, undefined);
|
|
1656
1865
|
throw err;
|
|
1657
1866
|
}
|
|
@@ -1691,7 +1900,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1691
1900
|
accumulateStats({ stats: partialSpend }, true);
|
|
1692
1901
|
}
|
|
1693
1902
|
recordFailed();
|
|
1694
|
-
|
|
1903
|
+
releaseLeg();
|
|
1695
1904
|
void journalAppend(callKey, {
|
|
1696
1905
|
taskId: callKey,
|
|
1697
1906
|
sessionId: "",
|
|
@@ -1704,11 +1913,15 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1704
1913
|
}
|
|
1705
1914
|
rollbackUsageBeat();
|
|
1706
1915
|
activeUsageBeats.delete(activeBeat);
|
|
1707
|
-
|
|
1916
|
+
releaseLeg();
|
|
1708
1917
|
if (finalized)
|
|
1709
1918
|
return result;
|
|
1710
|
-
if (
|
|
1711
|
-
|
|
1919
|
+
if (isPausedResult(result)) {
|
|
1920
|
+
return await parkAgentLeg(rec, result, activityTail, true);
|
|
1921
|
+
}
|
|
1922
|
+
if (drive !== undefined && driveNeverStarted(result)) {
|
|
1923
|
+
emitRunLog(`[park] the parked leg "${label}" was driven but the approved action never started (${result.terminal.kind === "failed" ? result.terminal.code : ""}); the park stands.`);
|
|
1924
|
+
return await parkAgentLeg(rec, drive.paused, activityTail, false);
|
|
1712
1925
|
}
|
|
1713
1926
|
await settleAgentResult(rec, result, activityTail, agentOpts, "best-effort");
|
|
1714
1927
|
return result;
|
|
@@ -1866,12 +2079,40 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1866
2079
|
}
|
|
1867
2080
|
const entries = await journalStore.load(opts.resumeFromRunId, scope);
|
|
1868
2081
|
for (const e of entries) {
|
|
2082
|
+
const parked = e.parked;
|
|
2083
|
+
if (parked !== undefined) {
|
|
2084
|
+
const pt = parked?.terminal;
|
|
2085
|
+
if (pt === null || typeof pt !== "object" || pt.kind !== "paused" || typeof pt.token !== "string" || pt.token.length === 0) {
|
|
2086
|
+
throw new WorkflowJournalIncompatibleError(clampRunIdText(opts.resumeFromRunId), callKeyOrdinal(e.callKey));
|
|
2087
|
+
}
|
|
2088
|
+
replayByOrdinal[callKeyOrdinal(e.callKey)] = e;
|
|
2089
|
+
continue;
|
|
2090
|
+
}
|
|
1869
2091
|
const t = e.result?.terminal;
|
|
1870
2092
|
if (t === null || typeof t !== "object" || !isTerminalCauseKind(t.kind)) {
|
|
1871
2093
|
throw new WorkflowJournalIncompatibleError(clampRunIdText(opts.resumeFromRunId), callKeyOrdinal(e.callKey));
|
|
1872
2094
|
}
|
|
1873
2095
|
replayByOrdinal[callKeyOrdinal(e.callKey)] = e;
|
|
1874
2096
|
}
|
|
2097
|
+
if (opts.store !== undefined) {
|
|
2098
|
+
let prior;
|
|
2099
|
+
try {
|
|
2100
|
+
prior = await opts.store.get(opts.resumeFromRunId);
|
|
2101
|
+
}
|
|
2102
|
+
catch (err) {
|
|
2103
|
+
throw new WorkflowJournalIncompatibleError(clampRunIdText(opts.resumeFromRunId), -1, `the prior run's record could not be read from the run store (${err instanceof Error ? err.message : String(err)}), so a lost park cannot be ruled out`);
|
|
2104
|
+
}
|
|
2105
|
+
if (prior !== null && prior.scope === scope) {
|
|
2106
|
+
for (const row of prior.agents) {
|
|
2107
|
+
if (row.status !== "parked")
|
|
2108
|
+
continue;
|
|
2109
|
+
const ord = callKeyOrdinal(row.callKey);
|
|
2110
|
+
if (replayByOrdinal[ord]?.parked === undefined) {
|
|
2111
|
+
throw new WorkflowJournalIncompatibleError(clampRunIdText(opts.resumeFromRunId), ord, `the prior run's agent #${ord} ("${row.label.slice(0, 80)}") is PARKED on an approval checkpoint but its journal entry is missing (the park was not recorded) — a resume would run that call live beside the pinned child`);
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
}
|
|
2115
|
+
}
|
|
1875
2116
|
run.resume = {
|
|
1876
2117
|
fromRunId: clampRunIdText(opts.resumeFromRunId),
|
|
1877
2118
|
journalEntries: entries.length,
|
|
@@ -1912,6 +2153,11 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1912
2153
|
else {
|
|
1913
2154
|
result = await bodyPromise;
|
|
1914
2155
|
}
|
|
2156
|
+
if (parkedLeg !== undefined) {
|
|
2157
|
+
await awaitLegsIdle(workflowSignal);
|
|
2158
|
+
throw parkedError();
|
|
2159
|
+
}
|
|
2160
|
+
await carryUnresolvedParks();
|
|
1915
2161
|
if (maxResultChars !== undefined) {
|
|
1916
2162
|
let size = 0;
|
|
1917
2163
|
try {
|
|
@@ -1944,6 +2190,9 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1944
2190
|
return { result, runId, run };
|
|
1945
2191
|
}
|
|
1946
2192
|
catch (err) {
|
|
2193
|
+
if (parkedLeg !== undefined)
|
|
2194
|
+
await awaitLegsIdle(workflowSignal);
|
|
2195
|
+
await carryUnresolvedParks();
|
|
1947
2196
|
finalized = true;
|
|
1948
2197
|
const unsettledOnFailure = settleActiveUsageBeats();
|
|
1949
2198
|
stampBudgetOvershoot(unsettledOnFailure);
|
|
@@ -1952,6 +2201,8 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
|
|
|
1952
2201
|
run.status = "failed";
|
|
1953
2202
|
run.completionId ??= uuidv7();
|
|
1954
2203
|
run.endedAt = now();
|
|
2204
|
+
if (parkedLeg !== undefined && !(err instanceof WorkflowAgentParkedError))
|
|
2205
|
+
err = parkedError();
|
|
1955
2206
|
run.error = err instanceof Error ? err.message : String(err);
|
|
1956
2207
|
closeAbandonedAgents(run.endedAt);
|
|
1957
2208
|
restampPhaseFailures();
|
|
@@ -7,6 +7,11 @@ import { canonicalStoreKey, sanitizePathComponent, writeThenLink } from "./fs-at
|
|
|
7
7
|
import { assertAdoptionBootGate } from "./adoption/marker.js";
|
|
8
8
|
import { oversizeJournalResult } from "../../core/workflow-journal-store.js";
|
|
9
9
|
export { MAX_JOURNAL_RESULT_BYTES, oversizeJournalResult } from "../../core/workflow-journal-store.js";
|
|
10
|
+
function entryOf(rec) {
|
|
11
|
+
if (rec.parked !== undefined)
|
|
12
|
+
return { callKey: rec.callKey, parked: rec.parked };
|
|
13
|
+
return { callKey: rec.callKey, result: rec.result };
|
|
14
|
+
}
|
|
10
15
|
export const RESUME_CLAIM_TTL_MS = 60 * 60 * 1000;
|
|
11
16
|
export const RESUME_CLAIM_RELEASE_GRACE_MS = 5_000;
|
|
12
17
|
export const RESUME_CLAIM_SWEEP_INTERVAL_MS = 60_000;
|
|
@@ -82,12 +87,12 @@ export class FileWorkflowJournalStore {
|
|
|
82
87
|
scope = scope ?? rec.scope;
|
|
83
88
|
if (rec.scope !== scope)
|
|
84
89
|
continue;
|
|
85
|
-
byOrdinal.set(rec.ordinal,
|
|
90
|
+
byOrdinal.set(rec.ordinal, entryOf(rec));
|
|
86
91
|
}
|
|
87
92
|
return scope === undefined ? undefined : { scope, byOrdinal };
|
|
88
93
|
}
|
|
89
94
|
async append(runId, scope, entry) {
|
|
90
|
-
const serialized = JSON.stringify(entry.result);
|
|
95
|
+
const serialized = JSON.stringify(entry.parked !== undefined ? entry.parked : entry.result);
|
|
91
96
|
if (oversizeJournalResult(serialized))
|
|
92
97
|
return;
|
|
93
98
|
const committed = this.scopes.get(runId) ?? this.replay(runId)?.scope;
|
|
@@ -115,7 +120,9 @@ export class FileWorkflowJournalStore {
|
|
|
115
120
|
catch {
|
|
116
121
|
}
|
|
117
122
|
}
|
|
118
|
-
const line =
|
|
123
|
+
const line = entry.parked !== undefined
|
|
124
|
+
? { scope, ordinal: callKeyOrdinal(entry.callKey), callKey: entry.callKey, parked: entry.parked }
|
|
125
|
+
: { scope, ordinal: callKeyOrdinal(entry.callKey), callKey: entry.callKey, result: entry.result };
|
|
119
126
|
log.append(line, this.fsyncEnabled);
|
|
120
127
|
}
|
|
121
128
|
async load(runId, scope) {
|