@tea-agent/loop-agent 0.35.1-beta.1 → 0.35.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -2
- package/dist/build-stamp.json +3 -3
- package/dist/executors/dag-pi-executor.js +44 -0
- package/dist/shared/package-metadata.js +10 -0
- package/dist/worker/console/static/assets/{index-fsjzREob.js → index-CvsQgALl.js} +26 -26
- package/dist/worker/console/static/index.html +1 -1
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +5 -0
- package/dist/worker/loop-agent/loop-agent-client.js +4 -3
- package/dist/worker/observability/read-model.js +20 -0
- package/dist/worker/preflight.js +2 -1
- package/dist/workflows/dag/backend-test-scenario-param.js +33 -23
- package/dist/workflows/dag/frontend-recovery-plan.js +73 -0
- package/dist/workflows/dag/frontend-recovery-root-manifest.js +123 -0
- package/dist/workflows/dag/frontend-recovery-run.js +539 -0
- package/dist/workflows/dag/frontend-writer-recovery.js +106 -0
- package/dist/workflows/dag/frontend-writer-rollback.js +821 -0
- package/dist/workflows/dag/recovery-recommendation.js +58 -0
- package/dist/workflows/dag/runner.js +201 -16
- package/dist/workflows/dag/scheduler.js +159 -0
- package/dist/workflows/dag/types.js +107 -0
- package/package.json +1 -1
|
@@ -269,3 +269,61 @@ export function planDagRecovery(input) {
|
|
|
269
269
|
}
|
|
270
270
|
return planByNormalizedCategory(input);
|
|
271
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Phase 6: route a frontend recovery outcome to a recovery CTA (read-only; never
|
|
274
|
+
* mutates a run's status). The six outcomes map to the existing recovery actions
|
|
275
|
+
* so the Console recovery page reuses the current CTA surface.
|
|
276
|
+
*/
|
|
277
|
+
export function recommendFrontendRecoveryOutcome(outcome) {
|
|
278
|
+
switch (outcome) {
|
|
279
|
+
case "none":
|
|
280
|
+
return {
|
|
281
|
+
action: "none",
|
|
282
|
+
summary: "未发生前端恢复;按普通终态展示。",
|
|
283
|
+
reason: "该 frontend-implementation 根没有触发自动恢复。",
|
|
284
|
+
humanRequired: false,
|
|
285
|
+
autoRetryEligible: false,
|
|
286
|
+
};
|
|
287
|
+
case "recovered":
|
|
288
|
+
return {
|
|
289
|
+
action: "none",
|
|
290
|
+
summary: "自动恢复成功;child 已重新实现并通过验证。",
|
|
291
|
+
reason: "parent 的 transient 失败已由 child 恢复,全链按 root 计一次成功。",
|
|
292
|
+
humanRequired: false,
|
|
293
|
+
autoRetryEligible: false,
|
|
294
|
+
};
|
|
295
|
+
case "candidate-contract-invalid":
|
|
296
|
+
return {
|
|
297
|
+
action: "manual-review",
|
|
298
|
+
summary: "方案输出无效:candidate JSON 无法解析/推导,重试用尽。",
|
|
299
|
+
reason: "建议人工检查需求或重新生成方案后再跑。",
|
|
300
|
+
humanRequired: true,
|
|
301
|
+
autoRetryEligible: false,
|
|
302
|
+
};
|
|
303
|
+
case "prewrite-blocked":
|
|
304
|
+
return {
|
|
305
|
+
action: "manual-review",
|
|
306
|
+
summary: "治理门禁阻断:source stale / writeSet 越界 / Mock policy / contract 违规。",
|
|
307
|
+
reason: "需人工修正任务边界/事实后重跑,不自动重试。",
|
|
308
|
+
humanRequired: true,
|
|
309
|
+
autoRetryEligible: false,
|
|
310
|
+
};
|
|
311
|
+
case "repair-exhausted":
|
|
312
|
+
return {
|
|
313
|
+
action: "rerun-after-fix",
|
|
314
|
+
summary: "自动修复未通过:verify 失败,1 次 repair 后 reverify 仍失败。",
|
|
315
|
+
reason: "建议 `dag rerun` 或人工排查。",
|
|
316
|
+
humanRequired: true,
|
|
317
|
+
autoRetryEligible: false,
|
|
318
|
+
commandHint: "loop-agent dag rerun --run-id <runId>",
|
|
319
|
+
};
|
|
320
|
+
case "auto-recovery-blocked":
|
|
321
|
+
return {
|
|
322
|
+
action: "manual-review",
|
|
323
|
+
summary: "自动恢复被阻止:rollback 无法证明完整或存在外部/越界修改。",
|
|
324
|
+
reason: "必须人工介入,禁止自动覆盖用户文件。",
|
|
325
|
+
humanRequired: true,
|
|
326
|
+
autoRetryEligible: false,
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
3
4
|
import { hostname } from "node:os";
|
|
4
5
|
import path from "node:path";
|
|
@@ -6,7 +7,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
|
|
|
6
7
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
7
8
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
8
9
|
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
9
|
-
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
10
|
+
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
10
11
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
11
12
|
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
12
13
|
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
@@ -21,8 +22,9 @@ import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSna
|
|
|
21
22
|
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
|
|
22
23
|
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
|
|
23
24
|
import { runConvergencePassController, } from "./convergence/controller.js";
|
|
24
|
-
import { executeDagRanksOnce, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
25
|
+
import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
25
26
|
import { topoSortToRanks } from "./topo.js";
|
|
27
|
+
import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
|
|
26
28
|
import { parseDagSpec, resolveModelForTask, } from "./types.js";
|
|
27
29
|
import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
|
|
28
30
|
import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
|
|
@@ -629,17 +631,54 @@ async function executeDagCheckpoint(input) {
|
|
|
629
631
|
runDir = await moveToPausedRunDir(runDir, pausedRunDir);
|
|
630
632
|
}
|
|
631
633
|
else {
|
|
632
|
-
await finalizeTerminalRunStatus(state, spec.tasks.length, runDir);
|
|
634
|
+
const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
|
|
633
635
|
await persistState();
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
636
|
+
if (recoveryPending) {
|
|
637
|
+
// Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
|
|
638
|
+
// child and run it synchronously. Dynamic import avoids a static
|
|
639
|
+
// runner ↔ frontend-recovery-run module cycle. stageRecoveryChild
|
|
640
|
+
// locates the parent via the canonical active/ layout, so recovery
|
|
641
|
+
// only fires when this run is still the canonical active run dir
|
|
642
|
+
// (direct runDagContinuation callers using an arbitrary runDir keep
|
|
643
|
+
// the previous scheduler-layer skip semantics).
|
|
644
|
+
const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
|
|
645
|
+
if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
|
|
646
|
+
const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
|
|
647
|
+
const staged = await stageRecoveryChild({
|
|
648
|
+
cwd,
|
|
649
|
+
parentRunId: state.runId,
|
|
650
|
+
});
|
|
651
|
+
// stageRecoveryChild CAS-advanced the parent's on-disk lineage
|
|
652
|
+
// (child-staging → child-running + childRunId). Refresh the in-memory
|
|
653
|
+
// copy so the terminal persistState below does not regress the parent
|
|
654
|
+
// back to stale child-staging.
|
|
655
|
+
const parentOnDisk = await readDagRunState(runDir);
|
|
656
|
+
state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
|
|
657
|
+
const childState = await readDagRunState(staged.childRunDir);
|
|
658
|
+
const childSpec = await readDagRunSpec(staged.childRunDir);
|
|
659
|
+
await runDagContinuation({
|
|
660
|
+
cwd,
|
|
661
|
+
spec: childSpec,
|
|
662
|
+
state: childState,
|
|
663
|
+
runDir: staged.childRunDir,
|
|
664
|
+
maxConcurrent,
|
|
665
|
+
executeNode: input.executeNode,
|
|
666
|
+
observer: input.observer,
|
|
667
|
+
abortSignal: input.abortSignal,
|
|
668
|
+
});
|
|
669
|
+
}
|
|
638
670
|
}
|
|
639
|
-
|
|
640
|
-
|
|
671
|
+
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
672
|
+
if (!recoveryPending) {
|
|
673
|
+
try {
|
|
674
|
+
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
675
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
676
|
+
}
|
|
677
|
+
catch (error) {
|
|
678
|
+
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
679
|
+
}
|
|
680
|
+
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
641
681
|
}
|
|
642
|
-
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
643
682
|
}
|
|
644
683
|
await relocateRunArtifactPaths({
|
|
645
684
|
runDir,
|
|
@@ -731,18 +770,151 @@ function isSuccessfulConvergenceTerminal(state) {
|
|
|
731
770
|
return true;
|
|
732
771
|
return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
|
|
733
772
|
}
|
|
734
|
-
|
|
773
|
+
function buildFrontendRecoveryIntent(state, failureSource) {
|
|
774
|
+
const existing = state.frontendRecoveryState;
|
|
775
|
+
const requestId = existing?.requestId ?? randomUUID();
|
|
776
|
+
return {
|
|
777
|
+
schemaVersion: 1,
|
|
778
|
+
phase: "child-staging",
|
|
779
|
+
requestId,
|
|
780
|
+
recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
|
|
781
|
+
parentRunId: state.runId,
|
|
782
|
+
...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
|
|
783
|
+
attemptId: existing?.attemptId ?? `recovery-${requestId}`,
|
|
784
|
+
attemptIndex: existing?.attemptIndex ?? 0,
|
|
785
|
+
continuationCount: existing?.continuationCount ?? 0,
|
|
786
|
+
...(failureSource ? { failureSource } : {}),
|
|
787
|
+
revision: (existing?.revision ?? 0) + 1,
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
791
|
+
const recovery = state.frontendRecoveryState;
|
|
792
|
+
return {
|
|
793
|
+
schemaVersion: 1,
|
|
794
|
+
outcome: "candidate-contract-invalid",
|
|
795
|
+
origin: {
|
|
796
|
+
kind: "frontend-prewrite-gate",
|
|
797
|
+
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
798
|
+
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
799
|
+
requestId: recovery?.requestId ?? "",
|
|
800
|
+
failedNodeId: result.selectedPlanNodeId,
|
|
801
|
+
},
|
|
802
|
+
failureClass: {
|
|
803
|
+
code: "candidate-contract-invalid",
|
|
804
|
+
classification: result.classification,
|
|
805
|
+
reason: result.failureReason ?? "candidate contract invalid",
|
|
806
|
+
},
|
|
807
|
+
evidenceRefs: [
|
|
808
|
+
{
|
|
809
|
+
runId: state.runId,
|
|
810
|
+
relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
811
|
+
sha256: artifactHash,
|
|
812
|
+
},
|
|
813
|
+
],
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
817
|
+
const recovery = state.frontendRecoveryState;
|
|
818
|
+
return {
|
|
819
|
+
schemaVersion: 1,
|
|
820
|
+
outcome: "auto-recovery-blocked",
|
|
821
|
+
origin: {
|
|
822
|
+
kind: "frontend-writer",
|
|
823
|
+
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
824
|
+
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
825
|
+
requestId: recovery?.requestId ?? "",
|
|
826
|
+
failedNodeId,
|
|
827
|
+
},
|
|
828
|
+
failureClass: {
|
|
829
|
+
code: "writer-transient-partial-write",
|
|
830
|
+
classification: "transient",
|
|
831
|
+
reason,
|
|
832
|
+
},
|
|
833
|
+
evidenceRefs: [],
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
function settleFrontendRecovery(state, outcome) {
|
|
837
|
+
const recovery = state.frontendRecoveryState;
|
|
838
|
+
if (!recovery)
|
|
839
|
+
return;
|
|
840
|
+
recovery.phase = "settled";
|
|
841
|
+
if (!state.frontendRecoveryResult) {
|
|
842
|
+
state.frontendRecoveryResult = {
|
|
843
|
+
schemaVersion: 1,
|
|
844
|
+
outcome,
|
|
845
|
+
origin: {
|
|
846
|
+
kind: "frontend-prewrite-gate",
|
|
847
|
+
parentRunId: recovery.parentRunId,
|
|
848
|
+
...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
849
|
+
requestId: recovery.requestId,
|
|
850
|
+
failedNodeId: "",
|
|
851
|
+
},
|
|
852
|
+
evidenceRefs: [],
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
857
|
+
const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
|
|
735
858
|
// Terminal override: a denied frontend writer must never surface as a
|
|
736
859
|
// mechanical partial_failed just because prewrite FINISHED while the writer
|
|
737
860
|
// was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
|
|
861
|
+
let recoveryPending = false;
|
|
738
862
|
const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
|
|
739
863
|
if (hasFrontendWriter) {
|
|
740
864
|
const admission = await readFrontendPrewriteResult(runDir);
|
|
741
|
-
if (admission.ok
|
|
742
|
-
(admission.result.classification === "
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
865
|
+
if (admission.ok) {
|
|
866
|
+
if (admission.result.classification === "blocked") {
|
|
867
|
+
state.status = "failed";
|
|
868
|
+
return { recoveryPending: false };
|
|
869
|
+
}
|
|
870
|
+
if (admission.result.classification === "retryable-invalid") {
|
|
871
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
872
|
+
if (continuationCount >= 1) {
|
|
873
|
+
// Continuation quota exhausted: a second retryable-invalid candidate
|
|
874
|
+
// is authoritative candidate-contract-invalid → forced failed.
|
|
875
|
+
state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
|
|
876
|
+
state.status = "failed";
|
|
877
|
+
return { recoveryPending: false };
|
|
878
|
+
}
|
|
879
|
+
// Root continuation still available: record the recovery intent and let
|
|
880
|
+
// mechanical aggregation settle partial_failed/failed. The parent keeps
|
|
881
|
+
// phase != settled (recovery in progress) and no final root manifest.
|
|
882
|
+
state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
|
|
883
|
+
recoveryPending = true;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
// Phase 5: writer transient partial write → rollback + recovery intent. Only
|
|
888
|
+
// for frontend-implementation writers with a remaining root continuation, and
|
|
889
|
+
// only when the rollback journal (captured before the provider call) restores
|
|
890
|
+
// cleanly; otherwise the run stays failed with auto-recovery-blocked.
|
|
891
|
+
if (hasFrontendWriter && !recoveryPending) {
|
|
892
|
+
const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
|
|
893
|
+
const node = state.nodes[id];
|
|
894
|
+
return (node &&
|
|
895
|
+
node.status === "ERROR" &&
|
|
896
|
+
isFrontendWriterTransientPartialWrite({
|
|
897
|
+
failureCategory: node.failureCategory,
|
|
898
|
+
}));
|
|
899
|
+
});
|
|
900
|
+
if (writerNodeId) {
|
|
901
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
902
|
+
if (continuationCount < 1) {
|
|
903
|
+
const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
|
|
904
|
+
const rollback = await rollbackFrontendWriter({
|
|
905
|
+
cwd: repoRoot,
|
|
906
|
+
parentRunId: state.runId,
|
|
907
|
+
});
|
|
908
|
+
if (rollback.ok) {
|
|
909
|
+
state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
|
|
910
|
+
recoveryPending = true;
|
|
911
|
+
}
|
|
912
|
+
else {
|
|
913
|
+
state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
|
|
914
|
+
state.status = "failed";
|
|
915
|
+
return { recoveryPending: false };
|
|
916
|
+
}
|
|
917
|
+
}
|
|
746
918
|
}
|
|
747
919
|
}
|
|
748
920
|
const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
|
|
@@ -768,6 +940,19 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir) {
|
|
|
768
940
|
else {
|
|
769
941
|
state.status = "failed";
|
|
770
942
|
}
|
|
943
|
+
// Terminal recovery bookkeeping for a child attempt: a child that finished
|
|
944
|
+
// recovered settles its own lineage; any other child terminal settles as
|
|
945
|
+
// auto-recovery-blocked (phase 3 performs no further recovery).
|
|
946
|
+
const recovery = state.frontendRecoveryState;
|
|
947
|
+
if (recovery && recovery.attemptIndex >= 1) {
|
|
948
|
+
if (state.status === "finished") {
|
|
949
|
+
settleFrontendRecovery(state, "recovered");
|
|
950
|
+
}
|
|
951
|
+
else if (!state.frontendRecoveryResult) {
|
|
952
|
+
settleFrontendRecovery(state, "auto-recovery-blocked");
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return { recoveryPending };
|
|
771
956
|
}
|
|
772
957
|
function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
|
|
773
958
|
if (rankWriterNodeIds.length <= 1)
|
|
@@ -4,6 +4,7 @@ import { isPauseOnHumanDecisionGate } from "./decision-envelope.js";
|
|
|
4
4
|
import { evaluateConditionExpression } from "./dynamic-runtime/condition.js";
|
|
5
5
|
import { sha256Hex } from "./frontend-implementation-contract.js";
|
|
6
6
|
import { frontendPrewriteResultV1Schema, FRONTEND_PREWRITE_RESULT_ARTIFACT_NAME, } from "./frontend-prewrite-gate.js";
|
|
7
|
+
import { FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH, FRONTEND_RECOVERY_INTENT_REL_DIR, } from "./frontend-recovery-plan.js";
|
|
7
8
|
export function isConditionSkippedReason(reason) {
|
|
8
9
|
return Boolean(reason?.startsWith("condition "));
|
|
9
10
|
}
|
|
@@ -178,8 +179,166 @@ export function markPendingNodesControllerInterrupted(state, reason = "run abort
|
|
|
178
179
|
}
|
|
179
180
|
return affected;
|
|
180
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* True when this run is a candidate-continuation child: it has a recovery
|
|
184
|
+
* lineage whose attemptIndex is >= 1 and whose recovery root is another run.
|
|
185
|
+
* The root/parent keeps `recoveryRootRunId === state.runId`.
|
|
186
|
+
*/
|
|
187
|
+
export function isFrontendRecoveryChild(state) {
|
|
188
|
+
const recovery = state.frontendRecoveryState;
|
|
189
|
+
return Boolean(recovery &&
|
|
190
|
+
recovery.attemptIndex >= 1 &&
|
|
191
|
+
recovery.recoveryRootRunId !== state.runId);
|
|
192
|
+
}
|
|
193
|
+
function isValidFrontendRecoveryActivationMarker(raw) {
|
|
194
|
+
return (raw.schemaVersion === 1 &&
|
|
195
|
+
typeof raw.requestId === "string" &&
|
|
196
|
+
raw.requestId.length > 0 &&
|
|
197
|
+
typeof raw.parentRunId === "string" &&
|
|
198
|
+
raw.parentRunId.length > 0 &&
|
|
199
|
+
typeof raw.recoveryRootRunId === "string" &&
|
|
200
|
+
raw.recoveryRootRunId.length > 0 &&
|
|
201
|
+
typeof raw.childRunId === "string" &&
|
|
202
|
+
raw.childRunId.length > 0 &&
|
|
203
|
+
typeof raw.importManifestSha256 === "string" &&
|
|
204
|
+
/^[a-f0-9]{64}$/.test(raw.importManifestSha256));
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Activation marker gate (phase 3c AC-1). Fail-closed: an active child is only
|
|
208
|
+
* executable when its parent is `child-running`, points at this child, carries
|
|
209
|
+
* a structurally valid activation marker whose lineage matches, and the marker
|
|
210
|
+
* hash matches the child's import manifest. Read-only and never throws.
|
|
211
|
+
*/
|
|
212
|
+
export async function checkFrontendRecoveryActivation(state, childRunDir) {
|
|
213
|
+
if (!isFrontendRecoveryChild(state)) {
|
|
214
|
+
return { applicable: false };
|
|
215
|
+
}
|
|
216
|
+
const recovery = state.frontendRecoveryState;
|
|
217
|
+
const parentRunId = recovery.parentRunId;
|
|
218
|
+
const parentRunDir = path.join(path.dirname(childRunDir), parentRunId);
|
|
219
|
+
let parentState;
|
|
220
|
+
try {
|
|
221
|
+
parentState = JSON.parse(await readFile(path.join(parentRunDir, "state.json"), "utf8"));
|
|
222
|
+
}
|
|
223
|
+
catch {
|
|
224
|
+
return { applicable: true, ok: false, reason: "parent state unreadable" };
|
|
225
|
+
}
|
|
226
|
+
const parentRecovery = parentState.frontendRecoveryState;
|
|
227
|
+
if (parentRecovery?.phase !== "child-running") {
|
|
228
|
+
return {
|
|
229
|
+
applicable: true,
|
|
230
|
+
ok: false,
|
|
231
|
+
reason: "parent phase not child-running",
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
if (parentRecovery.childRunId !== state.runId) {
|
|
235
|
+
return {
|
|
236
|
+
applicable: true,
|
|
237
|
+
ok: false,
|
|
238
|
+
reason: "parent childRunId mismatch",
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
const markerPath = path.join(parentRunDir, FRONTEND_RECOVERY_INTENT_REL_DIR, `${recovery.requestId}.json`);
|
|
242
|
+
let markerRaw;
|
|
243
|
+
try {
|
|
244
|
+
markerRaw = JSON.parse(await readFile(markerPath, "utf8"));
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return {
|
|
248
|
+
applicable: true,
|
|
249
|
+
ok: false,
|
|
250
|
+
reason: "activation marker missing",
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
if (typeof markerRaw !== "object" ||
|
|
254
|
+
markerRaw === null ||
|
|
255
|
+
!isValidFrontendRecoveryActivationMarker(markerRaw)) {
|
|
256
|
+
return {
|
|
257
|
+
applicable: true,
|
|
258
|
+
ok: false,
|
|
259
|
+
reason: "activation marker invalid",
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
const marker = markerRaw;
|
|
263
|
+
if (marker.requestId !== recovery.requestId) {
|
|
264
|
+
return {
|
|
265
|
+
applicable: true,
|
|
266
|
+
ok: false,
|
|
267
|
+
reason: "marker requestId mismatch",
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
if (marker.parentRunId !== parentRunId) {
|
|
271
|
+
return {
|
|
272
|
+
applicable: true,
|
|
273
|
+
ok: false,
|
|
274
|
+
reason: "marker parentRunId mismatch",
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
if (marker.recoveryRootRunId !== recovery.recoveryRootRunId) {
|
|
278
|
+
return {
|
|
279
|
+
applicable: true,
|
|
280
|
+
ok: false,
|
|
281
|
+
reason: "marker recoveryRootRunId mismatch",
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
if (marker.childRunId !== state.runId) {
|
|
285
|
+
return {
|
|
286
|
+
applicable: true,
|
|
287
|
+
ok: false,
|
|
288
|
+
reason: "marker childRunId mismatch",
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
let manifestSha256;
|
|
292
|
+
try {
|
|
293
|
+
manifestSha256 = sha256Hex(await readFile(path.join(childRunDir, FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH), "utf8"));
|
|
294
|
+
}
|
|
295
|
+
catch {
|
|
296
|
+
return {
|
|
297
|
+
applicable: true,
|
|
298
|
+
ok: false,
|
|
299
|
+
reason: "import manifest unreadable",
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
if (manifestSha256 !== marker.importManifestSha256) {
|
|
303
|
+
return {
|
|
304
|
+
applicable: true,
|
|
305
|
+
ok: false,
|
|
306
|
+
reason: "import manifest sha256 mismatch",
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
return {
|
|
310
|
+
applicable: true,
|
|
311
|
+
ok: true,
|
|
312
|
+
requestId: recovery.requestId,
|
|
313
|
+
childRunId: state.runId,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
181
316
|
export async function executeDagRanksOnce(input) {
|
|
182
317
|
let pausedByNodeId;
|
|
318
|
+
// Activation marker gate (phase 3c AC-1): a recovery child without a valid
|
|
319
|
+
// activation marker must never be scheduled or executed. Fail-closed before
|
|
320
|
+
// any executeScheduledNode call, so zero writer/provider invocations happen.
|
|
321
|
+
if (input.runDir) {
|
|
322
|
+
const hasPending = Object.values(input.state.nodes).some((node) => node.status === "PENDING");
|
|
323
|
+
if (hasPending) {
|
|
324
|
+
const activation = await checkFrontendRecoveryActivation(input.state, input.runDir);
|
|
325
|
+
if (activation.applicable && !activation.ok) {
|
|
326
|
+
const finishedAt = new Date().toISOString();
|
|
327
|
+
let affected = 0;
|
|
328
|
+
for (const node of Object.values(input.state.nodes)) {
|
|
329
|
+
if (node.status !== "PENDING")
|
|
330
|
+
continue;
|
|
331
|
+
node.status = "SKIPPED";
|
|
332
|
+
node.skippedReason = "frontend-recovery-child-not-activated";
|
|
333
|
+
node.finishedAt = finishedAt;
|
|
334
|
+
affected += 1;
|
|
335
|
+
}
|
|
336
|
+
if (affected > 0)
|
|
337
|
+
await input.persistState();
|
|
338
|
+
return undefined;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
183
342
|
for (const rank of input.ranks) {
|
|
184
343
|
if (input.abortSignal?.aborted) {
|
|
185
344
|
const marked = markPendingNodesControllerInterrupted(input.state, input.abortSignal.reason
|
|
@@ -895,6 +895,113 @@ export const dagSpecSchema = z
|
|
|
895
895
|
});
|
|
896
896
|
}
|
|
897
897
|
});
|
|
898
|
+
export const FRONTEND_RECOVERY_STATE_SCHEMA_VERSION = 1;
|
|
899
|
+
export const FRONTEND_RECOVERY_RESULT_SCHEMA_VERSION = 1;
|
|
900
|
+
/**
|
|
901
|
+
* Frontend candidate-continuation recovery phase (decision B, phase-3 subset).
|
|
902
|
+
* Phase 3a parents converge first, so `rollback-pending` is not produced and
|
|
903
|
+
* was removed by AC-1; the phase starts at `child-staging` and ends at
|
|
904
|
+
* `settled`.
|
|
905
|
+
*/
|
|
906
|
+
export const frontendRecoveryPhaseSchema = z.enum([
|
|
907
|
+
"child-staging",
|
|
908
|
+
"child-activating",
|
|
909
|
+
"child-running",
|
|
910
|
+
"settled",
|
|
911
|
+
]);
|
|
912
|
+
/**
|
|
913
|
+
* Single-writer recovery intent/lineage stored on `DagRunState`. The parent run
|
|
914
|
+
* owns the authoritative copy and advances it via revision-guarded CAS writes;
|
|
915
|
+
* the child carries a frozen lineage snapshot so the activation gate can prove
|
|
916
|
+
* it is the reserved child of a child-running parent.
|
|
917
|
+
*
|
|
918
|
+
* Invariants: `attemptIndex ∈ {0,1}`; `continuationCount ∈ {0,1}`; a child's
|
|
919
|
+
* `recoveryRootRunId` always equals the root runId; `revision` is the CAS
|
|
920
|
+
* pre-comparison counter. `revision` monotonicity is enforced at runtime by the
|
|
921
|
+
* runner's `(existing?.revision ?? 0) + 1` CAS write (the schema only bounds it
|
|
922
|
+
* non-negative). `childRunId` may be absent through `child-activating` and must
|
|
923
|
+
* be non-empty from `child-running` onward.
|
|
924
|
+
*/
|
|
925
|
+
export const frontendRecoveryStateSchema = z
|
|
926
|
+
.object({
|
|
927
|
+
schemaVersion: z.literal(FRONTEND_RECOVERY_STATE_SCHEMA_VERSION),
|
|
928
|
+
phase: frontendRecoveryPhaseSchema,
|
|
929
|
+
requestId: z.string().min(1),
|
|
930
|
+
recoveryRootRunId: z.string().min(1),
|
|
931
|
+
parentRunId: z.string().min(1),
|
|
932
|
+
childRunId: z.string().min(1).optional(),
|
|
933
|
+
attemptId: z.string().min(1),
|
|
934
|
+
attemptIndex: z.number().int().min(0).max(1),
|
|
935
|
+
continuationCount: z.number().int().min(0).max(1),
|
|
936
|
+
/** Node id of the reset-closure root: frontend-plan-pi /
|
|
937
|
+
* frontend-plan-revision-pi / frontend-prewrite-gate-shell /
|
|
938
|
+
* frontend-implement-pi (writer partial write). */
|
|
939
|
+
failureSource: z.string().min(1).optional(),
|
|
940
|
+
revision: z.number().int().nonnegative(),
|
|
941
|
+
})
|
|
942
|
+
.strict()
|
|
943
|
+
.superRefine((value, ctx) => {
|
|
944
|
+
const requiresChild = value.phase === "child-running" || value.phase === "settled";
|
|
945
|
+
if (requiresChild && !value.childRunId) {
|
|
946
|
+
ctx.addIssue({
|
|
947
|
+
code: z.ZodIssueCode.custom,
|
|
948
|
+
message: `phase ${value.phase} requires a non-empty childRunId`,
|
|
949
|
+
path: ["childRunId"],
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
});
|
|
953
|
+
export const frontendRecoveryOutcomeSchema = z.enum([
|
|
954
|
+
"none",
|
|
955
|
+
"recovered",
|
|
956
|
+
"candidate-contract-invalid",
|
|
957
|
+
"prewrite-blocked",
|
|
958
|
+
"repair-exhausted",
|
|
959
|
+
"auto-recovery-blocked",
|
|
960
|
+
]);
|
|
961
|
+
export const frontendRecoveryOriginSchema = z
|
|
962
|
+
.object({
|
|
963
|
+
kind: z.enum(["frontend-prewrite-gate", "frontend-writer"]),
|
|
964
|
+
parentRunId: z.string().min(1),
|
|
965
|
+
childRunId: z.string().min(1).optional(),
|
|
966
|
+
requestId: z.string().min(1),
|
|
967
|
+
failedNodeId: z.string(),
|
|
968
|
+
})
|
|
969
|
+
.strict();
|
|
970
|
+
export const frontendRecoveryFailureClassSchema = z
|
|
971
|
+
.object({
|
|
972
|
+
code: z.enum([
|
|
973
|
+
"candidate-contract-invalid",
|
|
974
|
+
"prewrite-blocked",
|
|
975
|
+
"staging-failed",
|
|
976
|
+
"writer-transient-partial-write",
|
|
977
|
+
]),
|
|
978
|
+
classification: z.string(),
|
|
979
|
+
reason: z.string(),
|
|
980
|
+
})
|
|
981
|
+
.strict();
|
|
982
|
+
export const frontendRecoveryEvidenceRefSchema = z
|
|
983
|
+
.object({
|
|
984
|
+
runId: z.string().min(1),
|
|
985
|
+
relativePath: z.string().min(1),
|
|
986
|
+
sha256: z.string().min(1),
|
|
987
|
+
})
|
|
988
|
+
.strict();
|
|
989
|
+
/**
|
|
990
|
+
* Terminal frontend recovery result. `failureClass` is absent for `none`,
|
|
991
|
+
* `recovered` and `auto-recovery-blocked` outcomes. The schema intentionally
|
|
992
|
+
* stays permissive (failureClass optional, evidenceRefs may be empty) so it
|
|
993
|
+
* accepts the runner's actual phase-3a products, which do not always carry a
|
|
994
|
+
* failure class or evidence reference.
|
|
995
|
+
*/
|
|
996
|
+
export const frontendRecoveryResultSchema = z
|
|
997
|
+
.object({
|
|
998
|
+
schemaVersion: z.literal(FRONTEND_RECOVERY_RESULT_SCHEMA_VERSION),
|
|
999
|
+
outcome: frontendRecoveryOutcomeSchema,
|
|
1000
|
+
origin: frontendRecoveryOriginSchema,
|
|
1001
|
+
failureClass: frontendRecoveryFailureClassSchema.optional(),
|
|
1002
|
+
evidenceRefs: z.array(frontendRecoveryEvidenceRefSchema),
|
|
1003
|
+
})
|
|
1004
|
+
.strict();
|
|
898
1005
|
export const DEFAULT_DAG_EXECUTOR_MODELS = {
|
|
899
1006
|
pi: {
|
|
900
1007
|
LOW: "gpt-5.3-codex-spark",
|