@tea-agent/loop-agent 0.44.0-next.10 → 0.44.0-next.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/build-stamp.json +3 -3
- package/dist/executors/dag-pi/sessions/index.js +6 -0
- package/dist/executors/dag-pi/sessions/plan-batches.js +280 -0
- package/dist/executors/dag-pi/sessions/plan-prompts.js +367 -0
- package/dist/executors/dag-pi/sessions/scout-parallel.js +197 -0
- package/dist/executors/dag-pi/sessions/segmented-plan.js +1184 -0
- package/dist/executors/dag-pi/sessions/writer-evidence.js +60 -0
- package/dist/executors/dag-pi-executor.js +12 -2075
- package/dist/executors/shell-executor.js +39 -11
- package/dist/worker/console/routes.js +5 -0
- package/dist/worker/console/static/app-icon.svg +39 -0
- package/dist/worker/console/static/index.html +3 -0
- package/dist/worker/console/static/manifest.webmanifest +19 -0
- package/dist/worker/observe/static/console-theme.js +38 -0
- package/dist/workflows/dag/checkpoint.js +686 -0
- package/dist/workflows/dag/frontend-repair.js +24 -0
- package/dist/workflows/dag/frontend-review-scopes.js +10 -2
- package/dist/workflows/dag/frontend-test-execution-evidence.js +163 -67
- package/dist/workflows/dag/frontend-test-framework-adapters.js +309 -0
- package/dist/workflows/dag/hybrid/sources.js +1812 -0
- package/dist/workflows/dag/hybrid/templates/backend-test.js +1807 -0
- package/dist/workflows/dag/hybrid/templates/frontend-test.js +702 -0
- package/dist/workflows/dag/hybrid/templates/frontend.js +1529 -0
- package/dist/workflows/dag/hybrid/templates/index.js +8 -0
- package/dist/workflows/dag/hybrid/templates/kg-bootstrap.js +449 -0
- package/dist/workflows/dag/hybrid/templates/knowledge-sync.js +495 -0
- package/dist/workflows/dag/hybrid/templates/shared.js +101 -0
- package/dist/workflows/dag/hybrid/templates/standard.js +265 -0
- package/dist/workflows/dag/hybrid/types.js +32 -0
- package/dist/workflows/dag/init-hybrid.js +42 -7162
- package/dist/workflows/dag/runner.js +11 -1178
- package/dist/workflows/dag/terminal-status.js +502 -0
- package/docs/architecture/dag-execution.md +4 -4
- package/package.json +1 -1
|
@@ -3,46 +3,30 @@ import { sourceBudgetContractIdentity } from "../../application/evaluation/budge
|
|
|
3
3
|
import { summarizeFrozenSourceUsage } from "../../task/source-prepare/source-provider-budget.js";
|
|
4
4
|
import { sha256OfCanonicalJson } from "../../task/contract/hash.js";
|
|
5
5
|
import { readdir, readFile } from "node:fs/promises";
|
|
6
|
-
import { createHash } from "node:crypto";
|
|
7
6
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
8
7
|
import { hostname } from "node:os";
|
|
9
8
|
import path from "node:path";
|
|
10
|
-
import {
|
|
9
|
+
import { resolveEffectiveMaxConcurrent, } from "../../application/evaluation/budget.js";
|
|
11
10
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
12
11
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
13
|
-
import { assertFrozenBudget, initRunBudgetLedger,
|
|
14
|
-
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact,
|
|
15
|
-
import {
|
|
16
|
-
import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
17
|
-
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
18
|
-
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
19
|
-
import { executeDagPiNode } from "../../executors/dag-pi-executor.js";
|
|
12
|
+
import { assertFrozenBudget, initRunBudgetLedger, } from "./budget-enforcement.js";
|
|
13
|
+
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
14
|
+
import { prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
20
15
|
import { assertValidDagSpec } from "./validate.js";
|
|
21
16
|
import { assertRuntimeContractCompatible, DAG_CONTROLLER_CAPABILITIES, } from "./runtime-contract.js";
|
|
22
17
|
import { captureControllerIdentity, verifyControllerIdentityForResume, } from "./controller-identity.js";
|
|
23
18
|
import { resolveRunningControllerIdentity } from "../../shared/package-metadata.js";
|
|
24
19
|
import { normalizeDagPromptSources } from "./prompt-source.js";
|
|
25
|
-
import { relocateConvergenceArtifactPaths, relocateRunArtifactPaths, } from "./upstream-artifacts.js";
|
|
26
20
|
import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSnapshot, } from "./skill-snapshot.js";
|
|
27
|
-
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL,
|
|
28
|
-
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions,
|
|
29
|
-
import { runConvergencePassController, } from "./convergence/controller.js";
|
|
30
|
-
import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
21
|
+
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
|
|
22
|
+
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, } from "./node-execution.js";
|
|
31
23
|
import { topoSortToRanks } from "./topo.js";
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import { bindFrontendShapeStateToRunTopology, commitFrontendShapeTransitionCas, commitRuntimeFrontendShapeFact, isCanonicalFrontendShapeDag, } from "./frontend-shape-facts.js";
|
|
35
|
-
import { assertFrontendTopologyBinding, buildFrontendTopologyBinding, computeFrontendShapeSourceDigest, } from "./frontend-shape.js";
|
|
36
|
-
import { ensureFrontendShapeCapsuleRecord } from "./frontend-shape-capsule-store.js";
|
|
24
|
+
import { isCanonicalFrontendShapeDag, } from "./frontend-shape-facts.js";
|
|
25
|
+
import { assertFrontendTopologyBinding, buildFrontendTopologyBinding, } from "./frontend-shape.js";
|
|
37
26
|
import { parseDagSpec, resolveModelForTask, } from "./types.js";
|
|
38
|
-
import {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
|
|
42
|
-
import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
|
|
43
|
-
import { executeDynamicMapExpansion } from "./dynamic-runtime/map.js";
|
|
44
|
-
import { executeDynamicReduction } from "./dynamic-runtime/reduction.js";
|
|
45
|
-
import { installRunnerExitDiagnostics } from "./runner-exit-diagnostics.js";
|
|
27
|
+
import { executeDagCheckpoint, notifyRunObserver, } from "./checkpoint.js";
|
|
28
|
+
export { buildCandidateContractInvalidResult, buildWriterPartialWriteBlockedResult, buildRecoveryResultForTrigger, collectSupersededIntermediateFailures, finalizeTerminalRunStatus, isSuccessfulConvergenceTerminal, createRecoveryOperation, SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS } from "./terminal-status.js";
|
|
29
|
+
export { assertRecoveredFrontendLineage, finalizeRunOwnedInterrupt, buildRankAwareExecuteNode, executeDagCheckpoint, concurrentSiblingWriteSetsForNode, ensureFreshFrontendShapeCapsuleRecord, loadCanonicalFrontendTaskSourceBinding, notifyRunObserver } from "./checkpoint.js";
|
|
46
30
|
export { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions };
|
|
47
31
|
export async function loadDagSpecFromFile(dagPath) {
|
|
48
32
|
const raw = JSON.parse(await readFile(dagPath, "utf-8"));
|
|
@@ -58,51 +42,6 @@ export function toKebabSlug(text, maxLen = 40) {
|
|
|
58
42
|
.replace(/^-|-$/g, "")
|
|
59
43
|
.slice(0, maxLen) || "dag");
|
|
60
44
|
}
|
|
61
|
-
export async function loadCanonicalFrontendTaskSourceBinding(cwd, spec) {
|
|
62
|
-
const binding = spec.sourceBinding;
|
|
63
|
-
if (!binding?.taskId)
|
|
64
|
-
throw new Error("canonical frontend DAG requires sourceBinding.taskId");
|
|
65
|
-
const requirement = binding.sources.find((source) => source.kind === "requirement");
|
|
66
|
-
if (!requirement)
|
|
67
|
-
throw new Error("canonical frontend DAG requires a bound requirement source");
|
|
68
|
-
const constraint = binding.sources.find((source) => source.kind === "constraint");
|
|
69
|
-
const readFreshSource = async (source) => {
|
|
70
|
-
let bytes;
|
|
71
|
-
try {
|
|
72
|
-
bytes = await readFile(path.resolve(cwd, source.path));
|
|
73
|
-
}
|
|
74
|
-
catch (error) {
|
|
75
|
-
throw new Error(`canonical frontend ${source.kind} source freshness read failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
76
|
-
}
|
|
77
|
-
const actualSha256 = createHash("sha256").update(bytes).digest("hex");
|
|
78
|
-
if (actualSha256 !== source.sha256) {
|
|
79
|
-
throw new Error(`canonical frontend ${source.kind} source sha256 drifted: expected ${source.sha256}, received ${actualSha256}`);
|
|
80
|
-
}
|
|
81
|
-
return bytes.toString("utf8");
|
|
82
|
-
};
|
|
83
|
-
const requirementMarkdown = await readFreshSource(requirement);
|
|
84
|
-
const constraintMarkdown = constraint ? await readFreshSource(constraint) : "";
|
|
85
|
-
return {
|
|
86
|
-
taskId: binding.taskId,
|
|
87
|
-
requirementMarkdown,
|
|
88
|
-
constraintMarkdown,
|
|
89
|
-
sourceDigest: computeFrontendShapeSourceDigest({ requirementMarkdown, constraintMarkdown }),
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
export async function ensureFreshFrontendShapeCapsuleRecord(input) {
|
|
93
|
-
const freshSource = await loadCanonicalFrontendTaskSourceBinding(input.cwd, input.spec);
|
|
94
|
-
if (freshSource.taskId !== input.canonicalSource.taskId ||
|
|
95
|
-
freshSource.sourceDigest !== input.canonicalSource.sourceDigest) {
|
|
96
|
-
throw new Error("canonical frontend task source freshness binding drifted before transition commit");
|
|
97
|
-
}
|
|
98
|
-
await (input.ensureRecord ?? ensureFrontendShapeCapsuleRecord)({
|
|
99
|
-
repoRoot: input.cwd,
|
|
100
|
-
taskId: input.canonicalSource.taskId,
|
|
101
|
-
parentRunId: input.parentRunId,
|
|
102
|
-
terminalOutcome: input.terminalOutcome,
|
|
103
|
-
capsule: input.capsule,
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
45
|
export function extractTaskIdFromDagSpec(spec) {
|
|
107
46
|
if (spec.objective) {
|
|
108
47
|
const match = spec.objective.match(TASK_ID_IN_TEXT_PATTERN);
|
|
@@ -548,956 +487,12 @@ export async function resumeDagRun(opts) {
|
|
|
548
487
|
finally {
|
|
549
488
|
}
|
|
550
489
|
}
|
|
551
|
-
async function executeDagCheckpoint(input) {
|
|
552
|
-
const { spec, state, ranks, runDir: initialRunDir, cwd, maxConcurrent, completedRunDir, pausedRunDir, } = input;
|
|
553
|
-
bindFrontendShapeStateToRunTopology(state);
|
|
554
|
-
let runDir = initialRunDir;
|
|
555
|
-
let stateWriteQueue = Promise.resolve();
|
|
556
|
-
let shapeTransitionBarrier;
|
|
557
|
-
const persistState = async (options) => {
|
|
558
|
-
if (shapeTransitionBarrier)
|
|
559
|
-
await shapeTransitionBarrier;
|
|
560
|
-
stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, state, options));
|
|
561
|
-
await stateWriteQueue;
|
|
562
|
-
};
|
|
563
|
-
const runnerLivenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy);
|
|
564
|
-
const heartbeatTimer = setInterval(() => {
|
|
565
|
-
if (!state.runner)
|
|
566
|
-
return;
|
|
567
|
-
// Runner lease only — never counts as meaningful Pi progress.
|
|
568
|
-
state.runner.heartbeatAt = new Date().toISOString();
|
|
569
|
-
const tasksByIdForPolicy = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
570
|
-
for (const node of Object.values(state.nodes)) {
|
|
571
|
-
if (node.status !== "RUNNING")
|
|
572
|
-
continue;
|
|
573
|
-
// Lease clock is separate from meaningful activity.
|
|
574
|
-
node.lastLeaseAt = state.runner.heartbeatAt;
|
|
575
|
-
const task = tasksByIdForPolicy.get(node.id);
|
|
576
|
-
const policy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task?.livenessPolicy);
|
|
577
|
-
const evaluation = evaluateNodeLiveness({
|
|
578
|
-
node,
|
|
579
|
-
policy,
|
|
580
|
-
runnerHeartbeatAt: state.runner.heartbeatAt,
|
|
581
|
-
});
|
|
582
|
-
node.livenessStatus = evaluation.status;
|
|
583
|
-
// Intentionally do NOT refresh lastActivityAt / lastMeaningfulProgressAt
|
|
584
|
-
// from the runner lease timer.
|
|
585
|
-
}
|
|
586
|
-
void persistState().catch(() => { });
|
|
587
|
-
}, runnerLivenessPolicy.heartbeatIntervalMs);
|
|
588
|
-
heartbeatTimer.unref();
|
|
589
|
-
await persistState();
|
|
590
|
-
const interruptController = new AbortController();
|
|
591
|
-
const expectedRunnerIdentity = runnerIdentityFromState(state);
|
|
592
|
-
const interruptWatcher = expectedRunnerIdentity
|
|
593
|
-
? startInterruptWatcher({
|
|
594
|
-
runDir,
|
|
595
|
-
runId: state.runId,
|
|
596
|
-
expectedRunnerIdentity,
|
|
597
|
-
controller: interruptController,
|
|
598
|
-
})
|
|
599
|
-
: undefined;
|
|
600
|
-
const abortSignal = mergeAbortSignals(input.abortSignal, interruptController.signal);
|
|
601
|
-
// Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
|
|
602
|
-
// timeout, supervision layer, or shell wall-clock) must not leave the run
|
|
603
|
-
// orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
|
|
604
|
-
// state with an explicit terminalReason before exiting so downstream
|
|
605
|
-
// liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
|
|
606
|
-
let terminalSignal;
|
|
607
|
-
const shutdownOnSignal = (signal) => {
|
|
608
|
-
if (terminalSignal)
|
|
609
|
-
return;
|
|
610
|
-
terminalSignal = signal;
|
|
611
|
-
clearInterval(heartbeatTimer);
|
|
612
|
-
if (state.runner)
|
|
613
|
-
state.runner.heartbeatAt = new Date().toISOString();
|
|
614
|
-
state.status = "failed";
|
|
615
|
-
state.finishedAt = new Date().toISOString();
|
|
616
|
-
state.terminalReason = `runner terminated by ${signal} before run completion`;
|
|
617
|
-
console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
|
|
618
|
-
const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
|
|
619
|
-
// Best-effort atomic persist with a hard exit deadline so a stuck write
|
|
620
|
-
// queue cannot keep the process alive past the supervisor's kill window.
|
|
621
|
-
const hardExit = setTimeout(exit, 5000);
|
|
622
|
-
hardExit.unref();
|
|
623
|
-
void persistState()
|
|
624
|
-
.catch(() => { })
|
|
625
|
-
.finally(() => {
|
|
626
|
-
clearTimeout(hardExit);
|
|
627
|
-
exit();
|
|
628
|
-
});
|
|
629
|
-
};
|
|
630
|
-
const onSigTerm = () => shutdownOnSignal("SIGTERM");
|
|
631
|
-
const onSigInt = () => shutdownOnSignal("SIGINT");
|
|
632
|
-
// Keep listeners installed until persistence completes. Pi dependencies use
|
|
633
|
-
// signal-exit, which recounts listeners during signal dispatch and re-sends
|
|
634
|
-
// the signal if a once-listener has already removed itself. terminalSignal
|
|
635
|
-
// above makes repeated signals idempotent; normal cleanup removes listeners.
|
|
636
|
-
process.on("SIGTERM", onSigTerm);
|
|
637
|
-
process.on("SIGINT", onSigInt);
|
|
638
|
-
// Exit-time diagnostics (silent-death forensics): write an `armed` line at
|
|
639
|
-
// checkpoint start and append process-exit / uncaught-exception /
|
|
640
|
-
// unhandled-rejection lines with a live state snapshot so an abnormal death
|
|
641
|
-
// (supervisor hard kill, wedged event loop, crash with lost stderr) leaves
|
|
642
|
-
// a durable trace next to state.json. Removed on clean terminal return.
|
|
643
|
-
const exitDiagnostics = installRunnerExitDiagnostics({
|
|
644
|
-
runDir,
|
|
645
|
-
runId: state.runId,
|
|
646
|
-
title: state.title,
|
|
647
|
-
snapshot: () => {
|
|
648
|
-
const heartbeatAt = state.runner?.heartbeatAt;
|
|
649
|
-
return {
|
|
650
|
-
status: state.status,
|
|
651
|
-
terminalReason: state.terminalReason,
|
|
652
|
-
failureCategory: state.failureCategory,
|
|
653
|
-
heartbeatAt,
|
|
654
|
-
heartbeatAgeMs: heartbeatAt === undefined
|
|
655
|
-
? undefined
|
|
656
|
-
: Math.max(0, Date.now() - Date.parse(heartbeatAt)),
|
|
657
|
-
running: Object.values(state.nodes)
|
|
658
|
-
.filter((node) => node.status === "RUNNING")
|
|
659
|
-
.map((node) => ({
|
|
660
|
-
id: node.id,
|
|
661
|
-
attempt: node.currentAttempt,
|
|
662
|
-
startedAt: node.startedAt,
|
|
663
|
-
lastMeaningfulProgressAt: node.lastMeaningfulProgressAt,
|
|
664
|
-
})),
|
|
665
|
-
};
|
|
666
|
-
},
|
|
667
|
-
});
|
|
668
|
-
try {
|
|
669
|
-
const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
670
|
-
const baseExecuteNode = input.executeNode ??
|
|
671
|
-
createDagNodeExecutor({
|
|
672
|
-
runDir,
|
|
673
|
-
runId: state.runId,
|
|
674
|
-
spec,
|
|
675
|
-
});
|
|
676
|
-
const executeDynamicNode = async (dynamicInput) => {
|
|
677
|
-
const { task } = dynamicInput;
|
|
678
|
-
if (task.dynamicExpansion) {
|
|
679
|
-
return executeDynamicMapExpansion({
|
|
680
|
-
...dynamicInput,
|
|
681
|
-
expansion: task.dynamicExpansion,
|
|
682
|
-
executeDynamicNode,
|
|
683
|
-
});
|
|
684
|
-
}
|
|
685
|
-
if (task.dynamicCondition) {
|
|
686
|
-
return executeDynamicCondition({
|
|
687
|
-
task,
|
|
688
|
-
condition: task.dynamicCondition,
|
|
689
|
-
tasksById: dynamicInput.tasksById,
|
|
690
|
-
state: dynamicInput.state,
|
|
691
|
-
});
|
|
692
|
-
}
|
|
693
|
-
if (task.dynamicLoopUntil) {
|
|
694
|
-
return executeDynamicLoopUntil({
|
|
695
|
-
...dynamicInput,
|
|
696
|
-
loop: task.dynamicLoopUntil,
|
|
697
|
-
executeDynamicNode,
|
|
698
|
-
});
|
|
699
|
-
}
|
|
700
|
-
return executeDynamicReduction({
|
|
701
|
-
task,
|
|
702
|
-
reduction: task.dynamicReduction,
|
|
703
|
-
state: dynamicInput.state,
|
|
704
|
-
runDir: dynamicInput.runDir,
|
|
705
|
-
});
|
|
706
|
-
};
|
|
707
|
-
const deferTerminalCloseout = state.convergence !== undefined && tasksById.has("closeout-pi");
|
|
708
|
-
const convergenceRanks = deferTerminalCloseout
|
|
709
|
-
? ranks
|
|
710
|
-
.map((rank) => rank.filter((nodeId) => nodeId !== "closeout-pi"))
|
|
711
|
-
.filter((rank) => rank.length > 0)
|
|
712
|
-
: ranks;
|
|
713
|
-
const terminalCloseoutRanks = deferTerminalCloseout
|
|
714
|
-
? ranks
|
|
715
|
-
.map((rank) => rank.filter((nodeId) => nodeId === "closeout-pi"))
|
|
716
|
-
.filter((rank) => rank.length > 0)
|
|
717
|
-
: [];
|
|
718
|
-
const canonicalFrontendSource = isCanonicalFrontendShapeDag(tasksById)
|
|
719
|
-
? await loadCanonicalFrontendTaskSourceBinding(cwd, spec)
|
|
720
|
-
: undefined;
|
|
721
|
-
const executeRanks = (selectedRanks) => executeDagRanksOnce({
|
|
722
|
-
state,
|
|
723
|
-
ranks: selectedRanks,
|
|
724
|
-
tasksById,
|
|
725
|
-
maxConcurrent,
|
|
726
|
-
persistState,
|
|
727
|
-
commitFrontendShapeTransition: async (request) => {
|
|
728
|
-
let releaseBarrier;
|
|
729
|
-
shapeTransitionBarrier = new Promise((resolve) => {
|
|
730
|
-
releaseBarrier = resolve;
|
|
731
|
-
});
|
|
732
|
-
try {
|
|
733
|
-
if (!canonicalFrontendSource) {
|
|
734
|
-
throw new Error("frontend shape transition requires canonical task source binding");
|
|
735
|
-
}
|
|
736
|
-
return await commitFrontendShapeTransitionCas({
|
|
737
|
-
state,
|
|
738
|
-
tasksById,
|
|
739
|
-
request,
|
|
740
|
-
taskSourceDigest: canonicalFrontendSource.sourceDigest,
|
|
741
|
-
ensureRecord: async (capsule, terminalOutcome) => {
|
|
742
|
-
await ensureFreshFrontendShapeCapsuleRecord({
|
|
743
|
-
cwd,
|
|
744
|
-
spec,
|
|
745
|
-
canonicalSource: canonicalFrontendSource,
|
|
746
|
-
parentRunId: state.runId,
|
|
747
|
-
terminalOutcome,
|
|
748
|
-
capsule,
|
|
749
|
-
});
|
|
750
|
-
},
|
|
751
|
-
persist: async (nextState) => {
|
|
752
|
-
stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, nextState));
|
|
753
|
-
await stateWriteQueue;
|
|
754
|
-
},
|
|
755
|
-
});
|
|
756
|
-
}
|
|
757
|
-
finally {
|
|
758
|
-
releaseBarrier();
|
|
759
|
-
shapeTransitionBarrier = undefined;
|
|
760
|
-
}
|
|
761
|
-
},
|
|
762
|
-
runDir,
|
|
763
|
-
abortSignal: input.abortSignal,
|
|
764
|
-
createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
|
|
765
|
-
baseExecuteNode,
|
|
766
|
-
customExecuteNode: input.executeNode,
|
|
767
|
-
rankWriterNodeIds,
|
|
768
|
-
tasksById,
|
|
769
|
-
meta: { runDir, runId: state.runId, spec },
|
|
770
|
-
}),
|
|
771
|
-
executeScheduledNode: async (nodeId, executeNode, onPause) => {
|
|
772
|
-
if (abortSignal?.aborted)
|
|
773
|
-
return;
|
|
774
|
-
if (isHardBudgetBreached(state.budgetLedger))
|
|
775
|
-
return;
|
|
776
|
-
const preBreach = preflightBudgetOrBreach(state);
|
|
777
|
-
if (preBreach && isHardBudgetBreached(state.budgetLedger)) {
|
|
778
|
-
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
779
|
-
await persistState();
|
|
780
|
-
return;
|
|
781
|
-
}
|
|
782
|
-
await executeDagNode({
|
|
783
|
-
nodeId,
|
|
784
|
-
tasksById,
|
|
785
|
-
state,
|
|
786
|
-
spec,
|
|
787
|
-
cwd,
|
|
788
|
-
runDir,
|
|
789
|
-
executeNode,
|
|
790
|
-
executeDynamicNode,
|
|
791
|
-
observer: input.observer,
|
|
792
|
-
persistState,
|
|
793
|
-
onPause,
|
|
794
|
-
});
|
|
795
|
-
const node = state.nodes[nodeId];
|
|
796
|
-
if (node && (node.status === "FINISHED" || node.status === "ERROR")) {
|
|
797
|
-
if (node.status === "FINISHED") {
|
|
798
|
-
if (canonicalFrontendSource) {
|
|
799
|
-
await commitRuntimeFrontendShapeFact({
|
|
800
|
-
state,
|
|
801
|
-
tasksById,
|
|
802
|
-
sourceNodeId: nodeId,
|
|
803
|
-
taskId: canonicalFrontendSource.taskId,
|
|
804
|
-
requirementMarkdown: canonicalFrontendSource.requirementMarkdown,
|
|
805
|
-
constraintMarkdown: canonicalFrontendSource.constraintMarkdown,
|
|
806
|
-
});
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
recordFinishedNodeBudget(state, node);
|
|
810
|
-
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
811
|
-
await persistState();
|
|
812
|
-
}
|
|
813
|
-
},
|
|
814
|
-
});
|
|
815
|
-
let pausedByNodeId;
|
|
816
|
-
while (true) {
|
|
817
|
-
pausedByNodeId = await executeRanks(convergenceRanks);
|
|
818
|
-
if (state.frontendShapeState?.terminalOutcome) {
|
|
819
|
-
break;
|
|
820
|
-
}
|
|
821
|
-
if (pausedByNodeId) {
|
|
822
|
-
break;
|
|
823
|
-
}
|
|
824
|
-
if (isHardBudgetBreached(state.budgetLedger)) {
|
|
825
|
-
break;
|
|
826
|
-
}
|
|
827
|
-
const convergenceDecision = await runConvergencePassController({
|
|
828
|
-
spec,
|
|
829
|
-
state,
|
|
830
|
-
tasksById,
|
|
831
|
-
runDir,
|
|
832
|
-
cwd,
|
|
833
|
-
persistState,
|
|
834
|
-
});
|
|
835
|
-
if (convergenceDecision.pausedByNodeId) {
|
|
836
|
-
pausedByNodeId = convergenceDecision.pausedByNodeId;
|
|
837
|
-
break;
|
|
838
|
-
}
|
|
839
|
-
if (convergenceDecision.retry) {
|
|
840
|
-
continue;
|
|
841
|
-
}
|
|
842
|
-
break;
|
|
843
|
-
}
|
|
844
|
-
if (!pausedByNodeId &&
|
|
845
|
-
terminalCloseoutRanks.length > 0 &&
|
|
846
|
-
state.convergence?.terminalReason &&
|
|
847
|
-
!isHardBudgetBreached(state.budgetLedger)) {
|
|
848
|
-
// Convergence closeout is a terminal phase, not an ordinary descendant
|
|
849
|
-
// rank. Running it only after the controller records terminalReason avoids
|
|
850
|
-
// stale success handoffs on intermediate request-revision passes while
|
|
851
|
-
// still producing a truthful failure handoff for max-passes/non-retry.
|
|
852
|
-
pausedByNodeId = await executeRanks(terminalCloseoutRanks);
|
|
853
|
-
}
|
|
854
|
-
interruptWatcher?.stop();
|
|
855
|
-
// Stop heartbeats before terminal archive. A late heartbeat writing
|
|
856
|
-
// state.json under completed/ trips the completed-facts write guard and
|
|
857
|
-
// makes run-dag exit non-zero after every node already finished.
|
|
858
|
-
clearInterval(heartbeatTimer);
|
|
859
|
-
await stateWriteQueue;
|
|
860
|
-
state.finishedAt = new Date().toISOString();
|
|
861
|
-
const runDirBeforeTransfer = runDir;
|
|
862
|
-
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
863
|
-
if (pausedByNodeId) {
|
|
864
|
-
state.status = "paused";
|
|
865
|
-
await finalizeRunOwnedInterrupt({
|
|
866
|
-
runDir,
|
|
867
|
-
state,
|
|
868
|
-
paused: true,
|
|
869
|
-
interruptAborted: interruptController.signal.aborted,
|
|
870
|
-
});
|
|
871
|
-
await persistState();
|
|
872
|
-
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
873
|
-
runDir = await moveToPausedRunDir(runDir, pausedRunDir);
|
|
874
|
-
}
|
|
875
|
-
else {
|
|
876
|
-
const { recoveryPending, recovery: recoveryTrigger } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd, spec.frontendRecovery?.maxContinuations ?? 1);
|
|
877
|
-
await finalizeRunOwnedInterrupt({
|
|
878
|
-
runDir,
|
|
879
|
-
state,
|
|
880
|
-
paused: false,
|
|
881
|
-
interruptAborted: interruptController.signal.aborted,
|
|
882
|
-
});
|
|
883
|
-
await persistState();
|
|
884
|
-
if (recoveryPending && input.recovery && recoveryTrigger) {
|
|
885
|
-
// M6 recovery operation controller path: construct/reuse one recovery
|
|
886
|
-
// operation keyed by a deterministic clientRequestId and supervise it
|
|
887
|
-
// to terminal, instead of staging an in-process child run.
|
|
888
|
-
const created = await createRecoveryOperation({
|
|
889
|
-
cwd,
|
|
890
|
-
runDir,
|
|
891
|
-
spec,
|
|
892
|
-
state,
|
|
893
|
-
trigger: recoveryTrigger,
|
|
894
|
-
deps: input.recovery,
|
|
895
|
-
});
|
|
896
|
-
if (created.kind === "blocked") {
|
|
897
|
-
state.status = "failed";
|
|
898
|
-
state.terminalReason = created.reason;
|
|
899
|
-
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
|
|
900
|
-
}
|
|
901
|
-
else if (created.kind === "conflict") {
|
|
902
|
-
// Same clientRequestId with a different payload → fail closed; never
|
|
903
|
-
// create a second recovery operation (AC-004).
|
|
904
|
-
state.status = "failed";
|
|
905
|
-
state.terminalReason =
|
|
906
|
-
"frontend recovery request-id reuse conflict";
|
|
907
|
-
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
|
|
908
|
-
}
|
|
909
|
-
else if (created.kind === "lease-held") {
|
|
910
|
-
// A different recovery entrance has already durably claimed this
|
|
911
|
-
// parent. Do not "help" by running its operation: that would reopen
|
|
912
|
-
// the exact two-child-DAG race the lease prevents.
|
|
913
|
-
state.status = "failed";
|
|
914
|
-
state.terminalReason =
|
|
915
|
-
"frontend recovery parent lease already held";
|
|
916
|
-
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
|
|
917
|
-
}
|
|
918
|
-
else {
|
|
919
|
-
const { runRecoveryOperationController } = await import("./frontend-recovery-controller.js");
|
|
920
|
-
const result = await runRecoveryOperationController({
|
|
921
|
-
operationId: created.operation.operationId,
|
|
922
|
-
parentRunId: state.runId,
|
|
923
|
-
observeOnly: created.kind === "observed",
|
|
924
|
-
deps: {
|
|
925
|
-
...input.recovery,
|
|
926
|
-
budget: input.recovery.budget ?? state.budget,
|
|
927
|
-
readLedgers: input.recovery.readLedgers ?? (async () => {
|
|
928
|
-
if (!state.budget)
|
|
929
|
-
return [];
|
|
930
|
-
const location = await locateDagRun(cwd, state.runId);
|
|
931
|
-
let current = await readDagRunState(location?.runDir ?? runDir);
|
|
932
|
-
const visited = new Set([current.runId]);
|
|
933
|
-
while (current.frontendRecoveryState?.parentRunId === current.runId && current.frontendRecoveryState.childRunId && current.frontendRecoveryState.childRunId !== current.runId) {
|
|
934
|
-
if (["child-staging", "child-activating"].includes(current.frontendRecoveryState.phase))
|
|
935
|
-
break;
|
|
936
|
-
const childId = current.frontendRecoveryState.childRunId;
|
|
937
|
-
if (visited.has(childId))
|
|
938
|
-
throw Error("FRONTEND_PROVIDER_BUDGET_LINEAGE_INVALID: cyclic recovery");
|
|
939
|
-
visited.add(childId);
|
|
940
|
-
const childLocation = await locateDagRun(cwd, childId);
|
|
941
|
-
if (!childLocation)
|
|
942
|
-
throw Error("FRONTEND_PROVIDER_BUDGET_LINEAGE_INVALID: missing recovery child");
|
|
943
|
-
const child = await readDagRunState(childLocation.runDir);
|
|
944
|
-
validateCumulativeBudgetSuccessor(current, child);
|
|
945
|
-
current = child;
|
|
946
|
-
}
|
|
947
|
-
if (!current.budgetLedger)
|
|
948
|
-
throw Error("FRONTEND_PROVIDER_BUDGET_INVALID: recovery ledger missing");
|
|
949
|
-
return [current.budgetLedger];
|
|
950
|
-
}),
|
|
951
|
-
failureOwner: input.recovery.failureOwner ??
|
|
952
|
-
recoveryTrigger.failureOwner,
|
|
953
|
-
protocolFailureReason: input.recovery.protocolFailureReason ??
|
|
954
|
-
recoveryTrigger.protocolFailureReason,
|
|
955
|
-
recoveryMode: input.recovery.recoveryMode ?? recoveryTrigger.recoveryMode,
|
|
956
|
-
},
|
|
957
|
-
});
|
|
958
|
-
// The recovery CLI stages its child by writing a fresh
|
|
959
|
-
// `frontendRecoveryState` into the parent run on disk. A successful
|
|
960
|
-
// operation without that bound lineage is not a successful recovery:
|
|
961
|
-
// fail closed instead of archiving a false-positive `recovered` result.
|
|
962
|
-
let recoveryLineageError;
|
|
963
|
-
if (result.terminalReason === "succeeded") {
|
|
964
|
-
try {
|
|
965
|
-
const persistedParentState = await readDagRunState(runDir);
|
|
966
|
-
const persistedRecovery = persistedParentState.frontendRecoveryState;
|
|
967
|
-
if (!persistedRecovery ||
|
|
968
|
-
persistedRecovery.phase !== "child-running" ||
|
|
969
|
-
persistedRecovery.parentRunId !== state.runId ||
|
|
970
|
-
persistedRecovery.requestId !== recoveryTrigger.requestId ||
|
|
971
|
-
!persistedRecovery.childRunId) {
|
|
972
|
-
throw new Error("persisted recovery lineage is missing or unbound");
|
|
973
|
-
}
|
|
974
|
-
await assertRecoveredFrontendLineage({
|
|
975
|
-
cwd,
|
|
976
|
-
rootRunId: state.runId,
|
|
977
|
-
rootRequestId: recoveryTrigger.requestId,
|
|
978
|
-
recovery: persistedRecovery,
|
|
979
|
-
maxContinuations: spec.frontendRecovery?.maxContinuations ?? 1,
|
|
980
|
-
});
|
|
981
|
-
state.frontendRecoveryState = structuredClone(persistedRecovery);
|
|
982
|
-
}
|
|
983
|
-
catch (error) {
|
|
984
|
-
recoveryLineageError =
|
|
985
|
-
error instanceof Error ? error.message : String(error);
|
|
986
|
-
state.status = "failed";
|
|
987
|
-
state.terminalReason =
|
|
988
|
-
`frontend recovery lineage invalid: ${recoveryLineageError}`;
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, result.terminalReason === "succeeded" && !recoveryLineageError
|
|
992
|
-
? "recovered"
|
|
993
|
-
: "auto-recovery-blocked");
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
997
|
-
if (input.beforeTerminalArchive) {
|
|
998
|
-
await input.beforeTerminalArchive({ runDir, state });
|
|
999
|
-
await persistState();
|
|
1000
|
-
}
|
|
1001
|
-
// Parent is always archived to completed/ — no active/ retention for an
|
|
1002
|
-
// in-process child CAS anymore (recovery runs as a separate operation).
|
|
1003
|
-
try {
|
|
1004
|
-
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
1005
|
-
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
1006
|
-
}
|
|
1007
|
-
catch (error) {
|
|
1008
|
-
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
1009
|
-
}
|
|
1010
|
-
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
1011
|
-
}
|
|
1012
|
-
await relocateRunArtifactPaths({
|
|
1013
|
-
runDir,
|
|
1014
|
-
oldRunDir: runDirBeforeTransfer,
|
|
1015
|
-
nodes: state.nodes,
|
|
1016
|
-
});
|
|
1017
|
-
if (state.convergence) {
|
|
1018
|
-
relocateConvergenceArtifactPaths(state.convergence, runDirBeforeTransfer, runDir);
|
|
1019
|
-
}
|
|
1020
|
-
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
1021
|
-
await persistState({ allowCompletedFactsWrite: true });
|
|
1022
|
-
return {
|
|
1023
|
-
title: state.title,
|
|
1024
|
-
runId: state.runId,
|
|
1025
|
-
status: state.status,
|
|
1026
|
-
ranks,
|
|
1027
|
-
nodes: state.nodes,
|
|
1028
|
-
runDir,
|
|
1029
|
-
};
|
|
1030
|
-
}
|
|
1031
|
-
finally {
|
|
1032
|
-
clearInterval(heartbeatTimer);
|
|
1033
|
-
interruptWatcher?.stop();
|
|
1034
|
-
exitDiagnostics.stop();
|
|
1035
|
-
process.removeListener("SIGTERM", onSigTerm);
|
|
1036
|
-
process.removeListener("SIGINT", onSigInt);
|
|
1037
|
-
}
|
|
1038
|
-
}
|
|
1039
|
-
async function assertRecoveredFrontendLineage(input) {
|
|
1040
|
-
const maxContinuations = Math.max(1, Math.min(5, input.maxContinuations));
|
|
1041
|
-
let parentRunId = input.rootRunId;
|
|
1042
|
-
let recovery = input.recovery;
|
|
1043
|
-
const recoveryRootRunId = recovery.recoveryRootRunId;
|
|
1044
|
-
const visited = new Set();
|
|
1045
|
-
if (recoveryRootRunId !== input.rootRunId) {
|
|
1046
|
-
throw new Error("recovery root lineage binding mismatch");
|
|
1047
|
-
}
|
|
1048
|
-
for (let depth = 0; depth < maxContinuations; depth += 1) {
|
|
1049
|
-
// The persisted parent intent represents the edge about to be traversed,
|
|
1050
|
-
// so its counters equal the number of already-completed continuation edges.
|
|
1051
|
-
// Each completed child increments both counters exactly once; no edge may
|
|
1052
|
-
// exceed the frozen DAG quota.
|
|
1053
|
-
if (recovery.continuationCount !== depth ||
|
|
1054
|
-
recovery.attemptIndex !== depth) {
|
|
1055
|
-
throw new Error("recovery child lineage counters are not monotonic");
|
|
1056
|
-
}
|
|
1057
|
-
if (recovery.phase !== "child-running" ||
|
|
1058
|
-
recovery.parentRunId !== parentRunId ||
|
|
1059
|
-
recovery.recoveryRootRunId !== recoveryRootRunId ||
|
|
1060
|
-
!recovery.childRunId ||
|
|
1061
|
-
(depth === 0
|
|
1062
|
-
? recovery.requestId !== input.rootRequestId
|
|
1063
|
-
: recovery.requestId !== parentRunId)) {
|
|
1064
|
-
throw new Error("persisted recovery lineage is missing or unbound");
|
|
1065
|
-
}
|
|
1066
|
-
if (visited.has(recovery.childRunId)) {
|
|
1067
|
-
throw new Error("recovery child lineage contains a cycle");
|
|
1068
|
-
}
|
|
1069
|
-
visited.add(recovery.childRunId);
|
|
1070
|
-
const child = await locateDagRun(input.cwd, recovery.childRunId);
|
|
1071
|
-
if (!child || child.lifecycle !== "completed") {
|
|
1072
|
-
throw new Error("recovery child is not completed");
|
|
1073
|
-
}
|
|
1074
|
-
const childState = await readDagRunState(child.runDir);
|
|
1075
|
-
if (childState.runId !== recovery.childRunId) {
|
|
1076
|
-
throw new Error("recovery child state binding mismatch");
|
|
1077
|
-
}
|
|
1078
|
-
if (childState.status === "finished")
|
|
1079
|
-
return;
|
|
1080
|
-
if (childState.frontendRecoveryResult?.outcome !== "recovered" ||
|
|
1081
|
-
!childState.frontendRecoveryState) {
|
|
1082
|
-
throw new Error("recovery child terminal state is not successful");
|
|
1083
|
-
}
|
|
1084
|
-
parentRunId = childState.runId;
|
|
1085
|
-
recovery = childState.frontendRecoveryState;
|
|
1086
|
-
}
|
|
1087
|
-
throw new Error("recovery child lineage exceeds the continuation quota");
|
|
1088
|
-
}
|
|
1089
|
-
async function finalizeRunOwnedInterrupt(input) {
|
|
1090
|
-
const request = await readInterruptRequest(input.runDir);
|
|
1091
|
-
if (!request)
|
|
1092
|
-
return;
|
|
1093
|
-
if (request.status === "settled" || request.status === "needs-reconcile") {
|
|
1094
|
-
return;
|
|
1095
|
-
}
|
|
1096
|
-
const hasUnconfirmed = Object.values(input.state.nodes).some((node) => node.failureCategory === "termination-unconfirmed");
|
|
1097
|
-
if (hasUnconfirmed || input.paused) {
|
|
1098
|
-
await markInterruptNeedsReconcile({
|
|
1099
|
-
runDir: input.runDir,
|
|
1100
|
-
outcome: hasUnconfirmed
|
|
1101
|
-
? "termination-unconfirmed"
|
|
1102
|
-
: "paused-before-settle",
|
|
1103
|
-
});
|
|
1104
|
-
return;
|
|
1105
|
-
}
|
|
1106
|
-
if (input.interruptAborted &&
|
|
1107
|
-
request.status === "acknowledged" &&
|
|
1108
|
-
(input.state.status === "failed" ||
|
|
1109
|
-
input.state.status === "partial_failed" ||
|
|
1110
|
-
input.state.failureCategory === "controller-interrupted")) {
|
|
1111
|
-
await settleDagInterrupt({
|
|
1112
|
-
runDir: input.runDir,
|
|
1113
|
-
outcome: "controller-interrupted",
|
|
1114
|
-
});
|
|
1115
|
-
return;
|
|
1116
|
-
}
|
|
1117
|
-
if (request.status === "requested") {
|
|
1118
|
-
await markInterruptNeedsReconcile({
|
|
1119
|
-
runDir: input.runDir,
|
|
1120
|
-
outcome: "runner-did-not-acknowledge",
|
|
1121
|
-
});
|
|
1122
|
-
return;
|
|
1123
|
-
}
|
|
1124
|
-
await markInterruptNeedsReconcile({
|
|
1125
|
-
runDir: input.runDir,
|
|
1126
|
-
outcome: "run-exited-before-interrupt-settled",
|
|
1127
|
-
});
|
|
1128
|
-
}
|
|
1129
|
-
async function notifyRunObserver(observer, event, state) {
|
|
1130
|
-
try {
|
|
1131
|
-
await observer?.[event]?.(state);
|
|
1132
|
-
}
|
|
1133
|
-
catch {
|
|
1134
|
-
// Observers are derived views; they must not affect canonical DAG execution.
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
const SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS = new Set([
|
|
1138
|
-
"review-pass",
|
|
1139
|
-
"hard-verify-pass",
|
|
1140
|
-
]);
|
|
1141
490
|
/**
|
|
1142
491
|
* Intermediate soft-verify failures may be excluded from terminal error
|
|
1143
492
|
* aggregation only when repair + final hard verify succeeded and, when a review
|
|
1144
493
|
* gate is present, convergence ended in review-pass. Historical node status is
|
|
1145
494
|
* never rewritten.
|
|
1146
495
|
*/
|
|
1147
|
-
function collectSupersededIntermediateFailures(state) {
|
|
1148
|
-
const supersededIds = new Set();
|
|
1149
|
-
const supersededFailures = [];
|
|
1150
|
-
const soft = state.nodes["soft-verify-shell"];
|
|
1151
|
-
if (!soft || soft.status !== "ERROR") {
|
|
1152
|
-
return { supersededIds, supersededFailures };
|
|
1153
|
-
}
|
|
1154
|
-
const repair = state.nodes["repair-pi"];
|
|
1155
|
-
const hard = state.nodes["hard-verify-shell"];
|
|
1156
|
-
if (repair?.status !== "FINISHED" || hard?.status !== "FINISHED") {
|
|
1157
|
-
return { supersededIds, supersededFailures };
|
|
1158
|
-
}
|
|
1159
|
-
const observesReview = Boolean(state.nodes["review-gate-shell"]);
|
|
1160
|
-
const terminalReason = state.convergence?.terminalReason;
|
|
1161
|
-
if (observesReview) {
|
|
1162
|
-
if (terminalReason !== "review-pass") {
|
|
1163
|
-
return { supersededIds, supersededFailures };
|
|
1164
|
-
}
|
|
1165
|
-
}
|
|
1166
|
-
else if (terminalReason && terminalReason !== "hard-verify-pass") {
|
|
1167
|
-
return { supersededIds, supersededFailures };
|
|
1168
|
-
}
|
|
1169
|
-
// Evidence: at least one successful repair pass after soft failure, with hard verify finished in terminal state.
|
|
1170
|
-
const hasRepairEvidence = !state.convergence?.enabled ||
|
|
1171
|
-
state.convergence.passHistory.some((pass) => pass.reason === "hard-verify-failed" ||
|
|
1172
|
-
pass.reason === "review-request-revision" ||
|
|
1173
|
-
pass.status === "retrying") ||
|
|
1174
|
-
repair.status === "FINISHED";
|
|
1175
|
-
if (!hasRepairEvidence) {
|
|
1176
|
-
return { supersededIds, supersededFailures };
|
|
1177
|
-
}
|
|
1178
|
-
supersededIds.add("soft-verify-shell");
|
|
1179
|
-
supersededFailures.push({
|
|
1180
|
-
nodeId: "soft-verify-shell",
|
|
1181
|
-
preservedStatus: "ERROR",
|
|
1182
|
-
supersededBy: ["repair-pi", "hard-verify-shell"],
|
|
1183
|
-
pass: state.convergence?.currentPass,
|
|
1184
|
-
reason: "intermediate-soft-verify-covered-by-repair-and-final-hard",
|
|
1185
|
-
});
|
|
1186
|
-
return { supersededIds, supersededFailures };
|
|
1187
|
-
}
|
|
1188
|
-
function isSuccessfulConvergenceTerminal(state) {
|
|
1189
|
-
const reason = state.convergence?.terminalReason;
|
|
1190
|
-
if (!reason)
|
|
1191
|
-
return true;
|
|
1192
|
-
return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
|
|
1193
|
-
}
|
|
1194
|
-
function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
1195
|
-
return {
|
|
1196
|
-
schemaVersion: 1,
|
|
1197
|
-
outcome: "candidate-contract-invalid",
|
|
1198
|
-
origin: {
|
|
1199
|
-
kind: "frontend-prewrite-gate",
|
|
1200
|
-
parentRunId: state.runId,
|
|
1201
|
-
requestId: state.runId,
|
|
1202
|
-
failedNodeId: result.failureSource ?? "frontend-plan-pi",
|
|
1203
|
-
},
|
|
1204
|
-
failureClass: {
|
|
1205
|
-
code: "candidate-contract-invalid",
|
|
1206
|
-
classification: result.classification,
|
|
1207
|
-
reason: result.findings.map((finding) => `${finding.code}: ${finding.message}`).join("; ") ??
|
|
1208
|
-
"candidate contract invalid",
|
|
1209
|
-
},
|
|
1210
|
-
evidenceRefs: [
|
|
1211
|
-
{
|
|
1212
|
-
runId: state.runId,
|
|
1213
|
-
relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
1214
|
-
sha256: artifactHash,
|
|
1215
|
-
},
|
|
1216
|
-
],
|
|
1217
|
-
};
|
|
1218
|
-
}
|
|
1219
|
-
function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
1220
|
-
return {
|
|
1221
|
-
schemaVersion: 1,
|
|
1222
|
-
outcome: "auto-recovery-blocked",
|
|
1223
|
-
origin: {
|
|
1224
|
-
kind: "frontend-writer",
|
|
1225
|
-
parentRunId: state.runId,
|
|
1226
|
-
requestId: state.runId,
|
|
1227
|
-
failedNodeId,
|
|
1228
|
-
},
|
|
1229
|
-
failureClass: {
|
|
1230
|
-
code: "writer-transient-partial-write",
|
|
1231
|
-
classification: "transient",
|
|
1232
|
-
reason,
|
|
1233
|
-
},
|
|
1234
|
-
evidenceRefs: [],
|
|
1235
|
-
};
|
|
1236
|
-
}
|
|
1237
|
-
function buildRecoveryResultForTrigger(state, trigger, outcome) {
|
|
1238
|
-
return {
|
|
1239
|
-
schemaVersion: 1,
|
|
1240
|
-
outcome,
|
|
1241
|
-
origin: {
|
|
1242
|
-
kind: trigger.failureSource === "frontend-implement-pi"
|
|
1243
|
-
? "frontend-writer"
|
|
1244
|
-
: "frontend-prewrite-gate",
|
|
1245
|
-
parentRunId: state.runId,
|
|
1246
|
-
requestId: trigger.requestId,
|
|
1247
|
-
failedNodeId: trigger.failureSource,
|
|
1248
|
-
},
|
|
1249
|
-
evidenceRefs: [],
|
|
1250
|
-
};
|
|
1251
|
-
}
|
|
1252
|
-
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd, frontendRecoveryQuota = 1) {
|
|
1253
|
-
const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
|
|
1254
|
-
if (state.frontendShapeState?.terminalOutcome &&
|
|
1255
|
-
state.frontendShapeState.transitionCapsule) {
|
|
1256
|
-
state.status = "failed";
|
|
1257
|
-
return { recoveryPending: false };
|
|
1258
|
-
}
|
|
1259
|
-
// Terminal override: a denied frontend writer must never surface as a
|
|
1260
|
-
// mechanical partial_failed just because prewrite FINISHED while the writer
|
|
1261
|
-
// was SKIPPED. Fail closed on blocked before aggregation.
|
|
1262
|
-
let recoveryTrigger;
|
|
1263
|
-
const skipFrontendRecovery = state.failureCategory === "controller-interrupted" ||
|
|
1264
|
-
state.continuation?.writerTransaction !== undefined;
|
|
1265
|
-
const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
|
|
1266
|
-
if (hasFrontendWriter && !skipFrontendRecovery) {
|
|
1267
|
-
const admission = await readFrontendPrewriteResult(runDir);
|
|
1268
|
-
if (admission.ok) {
|
|
1269
|
-
// M8: admission uses the accepted/blocked state machine. A blocked admission whose
|
|
1270
|
-
// failureSource is frontend-plan-pi is the contract-invalid analog of
|
|
1271
|
-
// the old retryable-invalid path (candidate continuation may re-plan);
|
|
1272
|
-
// every other blocked admission is terminal.
|
|
1273
|
-
const planContractInvalid = admission.result.classification === "blocked" &&
|
|
1274
|
-
admission.result.failureSource === "frontend-plan-pi";
|
|
1275
|
-
// A blocked admission caused by the design review committing
|
|
1276
|
-
// `request_design_changes` is a definite finding (not a protocol or
|
|
1277
|
-
// environment error): the design was reviewed and rejected. It is the
|
|
1278
|
-
// design-review analog of review-request-changes — not terminal, but
|
|
1279
|
-
// re-enters the next plan round with the committed design findings.
|
|
1280
|
-
// The failureSource may be frontend-writer-admission-shell or
|
|
1281
|
-
// frontend-design-policy-shell; the committed design fact is the
|
|
1282
|
-
// authoritative signal.
|
|
1283
|
-
const designRequestFact = await readCommittedDesignRequestFact({
|
|
1284
|
-
runDir,
|
|
1285
|
-
});
|
|
1286
|
-
const designRejected = admission.result.classification === "blocked" &&
|
|
1287
|
-
designRequestFact !== undefined;
|
|
1288
|
-
const terminalAdmissionDenial = (admission.result.classification === "blocked" ||
|
|
1289
|
-
admission.result.classification === "stale") &&
|
|
1290
|
-
!planContractInvalid &&
|
|
1291
|
-
!designRejected;
|
|
1292
|
-
if (terminalAdmissionDenial) {
|
|
1293
|
-
state.status = "failed";
|
|
1294
|
-
return { recoveryPending: false };
|
|
1295
|
-
}
|
|
1296
|
-
if (designRejected && designRequestFact) {
|
|
1297
|
-
state.terminalReason = "design-request-changes";
|
|
1298
|
-
try {
|
|
1299
|
-
await writeFrontendReviewFindingsPrd({
|
|
1300
|
-
runDir,
|
|
1301
|
-
parentRunId: state.runId,
|
|
1302
|
-
fact: designRequestFact,
|
|
1303
|
-
});
|
|
1304
|
-
}
|
|
1305
|
-
catch (error) {
|
|
1306
|
-
console.warn(`[run-dag] warning: failed to write frontend design findings PRD: ${error instanceof Error ? error.message : String(error)}`);
|
|
1307
|
-
}
|
|
1308
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1309
|
-
if (continuationCount < frontendRecoveryQuota) {
|
|
1310
|
-
recoveryTrigger = {
|
|
1311
|
-
requestId: state.runId,
|
|
1312
|
-
// The rejected design is a plan input defect. Reset from the
|
|
1313
|
-
// planner so the next child can incorporate the committed
|
|
1314
|
-
// findings before design review runs again.
|
|
1315
|
-
failureSource: "frontend-design-review-pi",
|
|
1316
|
-
failureOwner: classifyReviewIssueCategoryFailureOwner(designRequestFact.issueCategory),
|
|
1317
|
-
protocolFailureReason: `frontend design review request_design_changes (issueCategory=${designRequestFact.issueCategory})`,
|
|
1318
|
-
recoveryMode: "new-dag",
|
|
1319
|
-
};
|
|
1320
|
-
}
|
|
1321
|
-
}
|
|
1322
|
-
if (planContractInvalid) {
|
|
1323
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1324
|
-
if (continuationCount >= frontendRecoveryQuota) {
|
|
1325
|
-
// Continuation quota exhausted: a second contract-invalid candidate
|
|
1326
|
-
// is authoritative candidate-contract-invalid → forced failed.
|
|
1327
|
-
state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
|
|
1328
|
-
state.status = "failed";
|
|
1329
|
-
return { recoveryPending: false };
|
|
1330
|
-
}
|
|
1331
|
-
// Root continuation still available: emit a recovery trigger for the
|
|
1332
|
-
// M6 operation controller. The parent no longer writes
|
|
1333
|
-
// frontendRecoveryState or stays in active/ for a child CAS.
|
|
1334
|
-
recoveryTrigger = {
|
|
1335
|
-
requestId: state.runId,
|
|
1336
|
-
failureSource: admission.result.failureSource ?? "frontend-plan-pi",
|
|
1337
|
-
failureOwner: "plan",
|
|
1338
|
-
protocolFailureReason: "frontend candidate contract invalid",
|
|
1339
|
-
recoveryMode: "new-dag",
|
|
1340
|
-
};
|
|
1341
|
-
}
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
// Phase 5: writer transient partial write → rollback + recovery trigger.
|
|
1345
|
-
// Only for frontend-implementation writers with a remaining root
|
|
1346
|
-
// continuation, and only when the rollback journal (captured before the
|
|
1347
|
-
// provider call) restores cleanly; otherwise the run stays failed with
|
|
1348
|
-
// auto-recovery-blocked.
|
|
1349
|
-
if (hasFrontendWriter && !recoveryTrigger && !skipFrontendRecovery) {
|
|
1350
|
-
const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
|
|
1351
|
-
const node = state.nodes[id];
|
|
1352
|
-
return (node &&
|
|
1353
|
-
node.status === "ERROR" &&
|
|
1354
|
-
isFrontendWriterTransientPartialWrite({
|
|
1355
|
-
failureCategory: node.failureCategory,
|
|
1356
|
-
}));
|
|
1357
|
-
});
|
|
1358
|
-
if (writerNodeId) {
|
|
1359
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1360
|
-
if (continuationCount < frontendRecoveryQuota) {
|
|
1361
|
-
const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
|
|
1362
|
-
const rollback = await rollbackFrontendWriter({
|
|
1363
|
-
cwd: repoRoot,
|
|
1364
|
-
parentRunId: state.runId,
|
|
1365
|
-
});
|
|
1366
|
-
if (rollback.ok) {
|
|
1367
|
-
recoveryTrigger = {
|
|
1368
|
-
requestId: state.runId,
|
|
1369
|
-
failureSource: "frontend-implement-pi",
|
|
1370
|
-
failureOwner: "implementation",
|
|
1371
|
-
protocolFailureReason: "frontend writer transient partial write",
|
|
1372
|
-
recoveryMode: "continuation",
|
|
1373
|
-
};
|
|
1374
|
-
}
|
|
1375
|
-
else {
|
|
1376
|
-
state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
|
|
1377
|
-
state.status = "failed";
|
|
1378
|
-
return { recoveryPending: false };
|
|
1379
|
-
}
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
}
|
|
1383
|
-
// Phase 6: verification failure → automatic recovery (re-implement +
|
|
1384
|
-
// re-verify). The verify shell is terminal by design so the failed run's
|
|
1385
|
-
// evidence stays clean; the M6 recovery operation re-runs implement and
|
|
1386
|
-
// verify in a fresh continuation, guarded by the same continuation quota,
|
|
1387
|
-
// budget fold and failure fingerprint as the other recovery paths.
|
|
1388
|
-
if (hasFrontendWriter && !recoveryTrigger && !skipFrontendRecovery) {
|
|
1389
|
-
const verifyFailed = state.nodes["frontend-verify-shell"]?.status === "ERROR";
|
|
1390
|
-
if (verifyFailed) {
|
|
1391
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1392
|
-
if (continuationCount < frontendRecoveryQuota) {
|
|
1393
|
-
let recoveryOwner = classifyFailureOwner({
|
|
1394
|
-
nodeId: "frontend-verify-shell",
|
|
1395
|
-
rawFailureCategory: state.nodes["frontend-verify-shell"]?.failureCategory,
|
|
1396
|
-
});
|
|
1397
|
-
try {
|
|
1398
|
-
const verifyFailure = JSON.parse(await readFile(path.join(runDir, "contracts/frontend-verify-failure.json"), "utf8"));
|
|
1399
|
-
const owner = failureOwnerSchema.safeParse(verifyFailure.failureOwner);
|
|
1400
|
-
if (verifyFailure.schemaVersion === 1 &&
|
|
1401
|
-
verifyFailure.schemaId === "frontend-verify-failure-v1" &&
|
|
1402
|
-
verifyFailure.nodeId === "frontend-verify-shell" &&
|
|
1403
|
-
verifyFailure.failureCategory === state.nodes["frontend-verify-shell"]?.failureCategory &&
|
|
1404
|
-
owner.success) {
|
|
1405
|
-
recoveryOwner = owner.data;
|
|
1406
|
-
}
|
|
1407
|
-
else {
|
|
1408
|
-
recoveryOwner = "unknown";
|
|
1409
|
-
}
|
|
1410
|
-
}
|
|
1411
|
-
catch {
|
|
1412
|
-
// The node's classified category remains the fallback; never turn
|
|
1413
|
-
// unknown/environment/configuration failures into writer work.
|
|
1414
|
-
}
|
|
1415
|
-
if (recoveryOwner === "implementation")
|
|
1416
|
-
recoveryTrigger = {
|
|
1417
|
-
requestId: state.runId,
|
|
1418
|
-
failureSource: "frontend-verify-shell",
|
|
1419
|
-
failureOwner: recoveryOwner,
|
|
1420
|
-
protocolFailureReason: "frontend verification failure",
|
|
1421
|
-
recoveryMode: "continuation",
|
|
1422
|
-
};
|
|
1423
|
-
else
|
|
1424
|
-
state.terminalReason = recoveryOwner === "environment"
|
|
1425
|
-
? "frontend-verification-wait-external"
|
|
1426
|
-
: `frontend-verification-${recoveryOwner}-requires-contract-review`;
|
|
1427
|
-
}
|
|
1428
|
-
}
|
|
1429
|
-
}
|
|
1430
|
-
// Phase 7: frontend review "request changes" → not passed. When the review
|
|
1431
|
-
// node FINISHED but committed `request_review_changes` (typed terminal
|
|
1432
|
-
// verdict), the implementation was reviewed and rejected. The run must not
|
|
1433
|
-
// surface as finished: terminal aggregation below stays partial_failed, a
|
|
1434
|
-
// terminal reason is recorded, the committed findings are rendered into a
|
|
1435
|
-
// downstream-task PRD draft (run-owned), and a recovery trigger carries the
|
|
1436
|
-
// issueCategory-derived owner so a later DAG round consumes the findings
|
|
1437
|
-
// instead of repeating the previous flow.
|
|
1438
|
-
const reviewRequestFact = await readCommittedReviewRequestFact({
|
|
1439
|
-
runDir,
|
|
1440
|
-
});
|
|
1441
|
-
if (reviewRequestFact) {
|
|
1442
|
-
state.terminalReason = "review-request-changes";
|
|
1443
|
-
try {
|
|
1444
|
-
await writeFrontendReviewFindingsPrd({
|
|
1445
|
-
runDir,
|
|
1446
|
-
parentRunId: state.runId,
|
|
1447
|
-
fact: reviewRequestFact,
|
|
1448
|
-
});
|
|
1449
|
-
}
|
|
1450
|
-
catch (error) {
|
|
1451
|
-
// The PRD draft is additive; a write failure must not mask the
|
|
1452
|
-
// not-passed verdict. The terminal aggregation below still applies.
|
|
1453
|
-
console.warn(`[run-dag] warning: failed to write frontend review findings PRD: ${error instanceof Error ? error.message : String(error)}`);
|
|
1454
|
-
}
|
|
1455
|
-
if (!recoveryTrigger && !skipFrontendRecovery) {
|
|
1456
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1457
|
-
if (continuationCount < frontendRecoveryQuota) {
|
|
1458
|
-
recoveryTrigger = {
|
|
1459
|
-
requestId: state.runId,
|
|
1460
|
-
failureSource: "frontend-review-pi",
|
|
1461
|
-
failureOwner: classifyReviewIssueCategoryFailureOwner(reviewRequestFact.issueCategory),
|
|
1462
|
-
protocolFailureReason: `frontend review request_review_changes (issueCategory=${reviewRequestFact.issueCategory})`,
|
|
1463
|
-
recoveryMode: "new-dag",
|
|
1464
|
-
};
|
|
1465
|
-
}
|
|
1466
|
-
}
|
|
1467
|
-
}
|
|
1468
|
-
const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
|
|
1469
|
-
const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
|
|
1470
|
-
if (state.convergence && supersededFailures.length > 0) {
|
|
1471
|
-
state.convergence.supersededFailures = supersededFailures;
|
|
1472
|
-
}
|
|
1473
|
-
// Prefer map keys: DagNodeRecord may not always mirror the map key on `.id`.
|
|
1474
|
-
const effectiveErrorCount = Object.entries(state.nodes).filter(([nodeId, n]) => n.status === "ERROR" && !supersededIds.has(nodeId)).length;
|
|
1475
|
-
const supersededErrorCount = Object.entries(state.nodes).filter(([nodeId, n]) => n.status === "ERROR" && supersededIds.has(nodeId)).length;
|
|
1476
|
-
const conditionSkippedCount = Object.values(state.nodes).filter((n) => n.status === "SKIPPED" && isConditionSkippedReason(n.skippedReason)).length;
|
|
1477
|
-
const successTerminal = isSuccessfulConvergenceTerminal(state);
|
|
1478
|
-
// Superseded intermediate ERRORs remain ERROR in history but count as settled
|
|
1479
|
-
// for terminal aggregation only (they do not inflate effectiveErrorCount).
|
|
1480
|
-
if (effectiveErrorCount === 0 &&
|
|
1481
|
-
successTerminal &&
|
|
1482
|
-
finishedCount + conditionSkippedCount + supersededErrorCount === taskCount) {
|
|
1483
|
-
// Phase 7 guard: a committed `request_review_changes` means the review
|
|
1484
|
-
// rejected the implementation; the run must never surface as finished
|
|
1485
|
-
// even though every node technically FINISHED.
|
|
1486
|
-
state.status =
|
|
1487
|
-
state.terminalReason === "review-request-changes"
|
|
1488
|
-
? "partial_failed"
|
|
1489
|
-
: "finished";
|
|
1490
|
-
}
|
|
1491
|
-
else if (finishedCount > 0) {
|
|
1492
|
-
state.status = "partial_failed";
|
|
1493
|
-
}
|
|
1494
|
-
else {
|
|
1495
|
-
state.status = "failed";
|
|
1496
|
-
}
|
|
1497
|
-
return recoveryTrigger
|
|
1498
|
-
? { recoveryPending: true, recovery: recoveryTrigger }
|
|
1499
|
-
: { recoveryPending: false };
|
|
1500
|
-
}
|
|
1501
496
|
/**
|
|
1502
497
|
* Construct (or reuse) the recovery operation for a failed frontend run.
|
|
1503
498
|
*
|
|
@@ -1511,165 +506,3 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd, f
|
|
|
1511
506
|
* parent feedback through the managed rerun path. `planHash` is the second
|
|
1512
507
|
* idempotency dimension of `actionParams`.
|
|
1513
508
|
*/
|
|
1514
|
-
async function createRecoveryOperation(input) {
|
|
1515
|
-
const { cwd, runDir, spec, state, trigger } = input;
|
|
1516
|
-
const { computeFrontendRecoveryPlanForSource } = await import("./frontend-recovery-plan.js");
|
|
1517
|
-
let plan;
|
|
1518
|
-
try {
|
|
1519
|
-
plan = computeFrontendRecoveryPlanForSource({
|
|
1520
|
-
spec,
|
|
1521
|
-
parentRunId: state.runId,
|
|
1522
|
-
requestId: trigger.requestId,
|
|
1523
|
-
failureSource: trigger.failureSource,
|
|
1524
|
-
failureOwner: trigger.failureOwner,
|
|
1525
|
-
});
|
|
1526
|
-
}
|
|
1527
|
-
catch (error) {
|
|
1528
|
-
return { kind: "blocked", reason: error instanceof Error ? error.message : String(error) };
|
|
1529
|
-
}
|
|
1530
|
-
const recoveryLease = await acquireDagRecoveryLease({
|
|
1531
|
-
cwd,
|
|
1532
|
-
parentRunId: state.runId,
|
|
1533
|
-
requestId: trigger.requestId,
|
|
1534
|
-
kind: "auto-frontend-recovery",
|
|
1535
|
-
});
|
|
1536
|
-
let effectiveLease = recoveryLease;
|
|
1537
|
-
let observeOnly = false;
|
|
1538
|
-
if (recoveryLease.kind === "held") {
|
|
1539
|
-
if (recoveryLease.lease.kind !== "auto-frontend-recovery" ||
|
|
1540
|
-
recoveryLease.lease.requestId !== trigger.requestId) {
|
|
1541
|
-
return { kind: "lease-held", lease: recoveryLease.lease };
|
|
1542
|
-
}
|
|
1543
|
-
const claimed = await claimDagRecoveryLease({
|
|
1544
|
-
leasePath: recoveryLease.leasePath,
|
|
1545
|
-
lease: recoveryLease.lease,
|
|
1546
|
-
});
|
|
1547
|
-
effectiveLease = claimed;
|
|
1548
|
-
if (claimed.kind === "held") {
|
|
1549
|
-
if (!claimed.lease.executionRef) {
|
|
1550
|
-
return { kind: "lease-held", lease: claimed.lease };
|
|
1551
|
-
}
|
|
1552
|
-
observeOnly = true;
|
|
1553
|
-
}
|
|
1554
|
-
}
|
|
1555
|
-
const planHash = createHash("sha256")
|
|
1556
|
-
.update(JSON.stringify(plan))
|
|
1557
|
-
.digest("hex");
|
|
1558
|
-
const continuationPath = path.join(runDir, ".runtime", "frontend-recovery", `${trigger.requestId}.json`);
|
|
1559
|
-
await writeJsonAtomic(continuationPath, {
|
|
1560
|
-
schemaVersion: 1,
|
|
1561
|
-
parentRunId: state.runId,
|
|
1562
|
-
requestId: trigger.requestId,
|
|
1563
|
-
failureSource: trigger.failureSource,
|
|
1564
|
-
failureOwner: plan.failureOwner,
|
|
1565
|
-
resetRootNodeId: plan.resetRootNodeId,
|
|
1566
|
-
recoveryMode: trigger.recoveryMode,
|
|
1567
|
-
planHash,
|
|
1568
|
-
plan,
|
|
1569
|
-
});
|
|
1570
|
-
const operationInput = {
|
|
1571
|
-
clientRequestId: `dag-recovery-${state.runId}`,
|
|
1572
|
-
action: "dagRunTask",
|
|
1573
|
-
actionParams: {
|
|
1574
|
-
parentRunId: state.runId,
|
|
1575
|
-
requestId: trigger.requestId,
|
|
1576
|
-
failureSource: trigger.failureSource,
|
|
1577
|
-
failureOwner: plan.failureOwner,
|
|
1578
|
-
resetRootNodeId: plan.resetRootNodeId,
|
|
1579
|
-
protocolFailureReason: trigger.protocolFailureReason,
|
|
1580
|
-
recoveryMode: trigger.recoveryMode,
|
|
1581
|
-
planHash,
|
|
1582
|
-
},
|
|
1583
|
-
cliArgs: [
|
|
1584
|
-
"dag",
|
|
1585
|
-
"rerun-task",
|
|
1586
|
-
"--run-id",
|
|
1587
|
-
state.runId,
|
|
1588
|
-
"--reason",
|
|
1589
|
-
trigger.protocolFailureReason ?? "frontend automatic recovery",
|
|
1590
|
-
"--request-id",
|
|
1591
|
-
trigger.requestId,
|
|
1592
|
-
"--frontend-recovery",
|
|
1593
|
-
"--json",
|
|
1594
|
-
],
|
|
1595
|
-
};
|
|
1596
|
-
let created;
|
|
1597
|
-
if (effectiveLease.lease.executionRef) {
|
|
1598
|
-
// Replay the exact operation bound by the durable lease. A missing record
|
|
1599
|
-
// cannot be replaced with a new operation id without violating the fence.
|
|
1600
|
-
const existing = await input.deps.store.get(effectiveLease.lease.executionRef);
|
|
1601
|
-
if (!existing) {
|
|
1602
|
-
throw new Error(`frontend recovery lease operation not found: ${effectiveLease.lease.executionRef}`);
|
|
1603
|
-
}
|
|
1604
|
-
if (existing.clientRequestId !== operationInput.clientRequestId ||
|
|
1605
|
-
existing.action !== operationInput.action ||
|
|
1606
|
-
existing.payloadHash !==
|
|
1607
|
-
computeActionPayloadHash(operationInput.action, operationInput.actionParams)) {
|
|
1608
|
-
throw new Error("frontend recovery lease execution identity drift: referenced operation payload does not match the recovery request");
|
|
1609
|
-
}
|
|
1610
|
-
created = { kind: "idempotent", operation: existing };
|
|
1611
|
-
}
|
|
1612
|
-
else {
|
|
1613
|
-
created = await input.deps.store.create(operationInput);
|
|
1614
|
-
}
|
|
1615
|
-
if (created.kind !== "conflict" && !observeOnly) {
|
|
1616
|
-
// The durable lease must name the operation before controller execution.
|
|
1617
|
-
// This also repairs a lease left behind by a crash before the original bind.
|
|
1618
|
-
await bindDagRecoveryLeaseExecution({
|
|
1619
|
-
leasePath: effectiveLease.leasePath,
|
|
1620
|
-
lease: effectiveLease.lease,
|
|
1621
|
-
executionRef: created.operation.operationId,
|
|
1622
|
-
});
|
|
1623
|
-
// Repair CLI args on both fresh and replayed operations. This closes the
|
|
1624
|
-
// crash window after operation creation but before args were persisted.
|
|
1625
|
-
await input.deps.store.update(created.operation.operationId, {
|
|
1626
|
-
cliArgs: [
|
|
1627
|
-
"dag",
|
|
1628
|
-
"rerun-task",
|
|
1629
|
-
"--run-id",
|
|
1630
|
-
state.runId,
|
|
1631
|
-
"--reason",
|
|
1632
|
-
trigger.protocolFailureReason ?? "frontend automatic recovery",
|
|
1633
|
-
"--request-id",
|
|
1634
|
-
trigger.requestId,
|
|
1635
|
-
"--frontend-recovery",
|
|
1636
|
-
"--recovery-operation-id",
|
|
1637
|
-
created.operation.operationId,
|
|
1638
|
-
"--json",
|
|
1639
|
-
],
|
|
1640
|
-
});
|
|
1641
|
-
}
|
|
1642
|
-
if (created.kind !== "conflict" && observeOnly) {
|
|
1643
|
-
return { kind: "observed", operation: created.operation };
|
|
1644
|
-
}
|
|
1645
|
-
return created;
|
|
1646
|
-
}
|
|
1647
|
-
function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
|
|
1648
|
-
if (rankWriterNodeIds.length <= 1)
|
|
1649
|
-
return [];
|
|
1650
|
-
return rankWriterNodeIds
|
|
1651
|
-
.filter((id) => id !== nodeId)
|
|
1652
|
-
.map((id) => tasksById.get(id)?.writeSet ?? [])
|
|
1653
|
-
.filter((writeSet) => writeSet.length > 0);
|
|
1654
|
-
}
|
|
1655
|
-
function buildRankAwareExecuteNode(input) {
|
|
1656
|
-
if (input.customExecuteNode) {
|
|
1657
|
-
return input.baseExecuteNode;
|
|
1658
|
-
}
|
|
1659
|
-
return async (nodeInput) => {
|
|
1660
|
-
const isPiWriter = nodeInput.task.executor === "pi" &&
|
|
1661
|
-
nodeInput.task.toolProfile === "write" &&
|
|
1662
|
-
Boolean(nodeInput.model);
|
|
1663
|
-
if (!isPiWriter) {
|
|
1664
|
-
return input.baseExecuteNode(nodeInput);
|
|
1665
|
-
}
|
|
1666
|
-
const siblingWriteSets = nodeInput.task.writePolicy === "exclusive"
|
|
1667
|
-
? concurrentSiblingWriteSetsForNode(input.rankWriterNodeIds, nodeInput.task.id, input.tasksById)
|
|
1668
|
-
: [];
|
|
1669
|
-
return executeDagPiNode({ ...nodeInput, model: nodeInput.model }, {
|
|
1670
|
-
...input.meta,
|
|
1671
|
-
concurrentSiblingWriteSets: siblingWriteSets.length > 0 ? siblingWriteSets : undefined,
|
|
1672
|
-
writeGuardAttribution: siblingWriteSets.length > 0 ? "best-effort" : "per-node",
|
|
1673
|
-
});
|
|
1674
|
-
};
|
|
1675
|
-
}
|