@tea-agent/loop-agent 0.42.0-next.1 → 0.42.0-next.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +319 -1
- package/dist/application/dag/generate-task-dag.js +75 -19
- package/dist/application/dag/run-dag.js +41 -0
- package/dist/application/task-lifecycle/advance.js +24 -5
- package/dist/application/task-lifecycle/observe.js +171 -17
- package/dist/application/task-lifecycle/plan-transitions.js +42 -7
- package/dist/application/task-lifecycle/recommendations.js +13 -2
- package/dist/build-stamp.json +3 -3
- package/dist/cli/command-definitions.js +7 -0
- package/dist/cli/program.js +6 -1
- package/dist/commands/client-recovery.js +3 -0
- package/dist/commands/dag-follow-up.js +138 -0
- package/dist/commands/dag-rerun-task.js +2 -0
- package/dist/commands/init.js +27 -1
- package/dist/commands/task-advance.js +19 -0
- package/dist/executors/dag-pi-executor.js +3976 -76
- package/dist/executors/pi-executor.js +15 -4
- package/dist/executors/pi-read-budget-policy.js +239 -0
- package/dist/executors/shell-executor.js +978 -188
- package/dist/executors/shell-write-guard.js +7 -0
- package/dist/infrastructure/console/operation-store.js +226 -6
- package/dist/shared/dag-failure-category.js +12 -0
- package/dist/shared/openspec-spec.js +70 -4
- package/dist/shared/operator/capabilities.js +7 -0
- package/dist/task/config-types.js +105 -5
- package/dist/task/contract/adopt.js +4 -0
- package/dist/task/contract/import-revision.js +4 -0
- package/dist/task/contract/project.js +3 -0
- package/dist/task/contract/schema.js +2 -1
- package/dist/task/frontend-project-capability.js +203 -20
- package/dist/task/runtime.js +5 -2
- package/dist/task/source-prepare/build-draft.js +3 -3
- package/dist/task/source-prepare/fragment-inventory.js +64 -25
- package/dist/task/source-prepare/prepare.js +126 -21
- package/dist/task/source-prepare/semantic-intake.js +6 -2
- package/dist/task/source-references.js +22 -1
- package/dist/worker/console/chat/chat-ui-policy.js +4 -2
- package/dist/worker/console/chat/model-resolver.js +7 -4
- package/dist/worker/console/chat/pi-runtime.js +258 -11
- package/dist/worker/console/chat/repo-browser.js +10 -3
- package/dist/worker/console/chat/routes.js +165 -71
- package/dist/worker/console/chat/sdd-data-alignment.js +222 -0
- package/dist/worker/console/chat/session-catalog.js +2 -0
- package/dist/worker/console/chat/session-store.js +159 -70
- package/dist/worker/console/chat/shortcuts.js +19 -3
- package/dist/worker/console/chat/turn-process.js +111 -56
- package/dist/worker/console/frontend-human-decision-adapter.js +19 -0
- package/dist/worker/console/frontend-split-operation-adapter.js +20 -0
- package/dist/worker/console/index.js +3 -0
- package/dist/worker/console/operator-actions.js +169 -7
- package/dist/worker/console/pi-readiness.js +3 -2
- package/dist/worker/console/server.js +23 -1
- package/dist/worker/console/static/assets/{abnfDiagram-N423BO3Z-DZZ8m3PO.js → abnfDiagram-N423BO3Z-CcS17TBr.js} +1 -1
- package/dist/worker/console/static/assets/{arc-D6PvaVd-.js → arc-COptKq2S.js} +1 -1
- package/dist/worker/console/static/assets/{architectureDiagram-T3A2C74G-B_OTOiI8.js → architectureDiagram-T3A2C74G-h4LKMHjP.js} +1 -1
- package/dist/worker/console/static/assets/{blockDiagram-VBNYF7ZC-Bv6rqHBg.js → blockDiagram-VBNYF7ZC-COA1MOH0.js} +1 -1
- package/dist/worker/console/static/assets/{c4Diagram-5PPSVZJV-B8eHr0oz.js → c4Diagram-5PPSVZJV-DJUf0QPm.js} +1 -1
- package/dist/worker/console/static/assets/channel-DdBCaOJ6.js +1 -0
- package/dist/worker/console/static/assets/{chunk-2GRJ4B5K-DYwR0im2.js → chunk-2GRJ4B5K-mtWfKrUX.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-2Q5K7J3B-D2WPGqXt.js → chunk-2Q5K7J3B-B8pXsxDQ.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-5RXB4S5H-CQISJ_I7.js → chunk-5RXB4S5H-ipKzByl1.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-5VM5RSS4-C0o2Du1e.js → chunk-5VM5RSS4-DYi3Ald_.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-6Q2QTUOP-4f8kr-U7.js → chunk-6Q2QTUOP-DQtGYoty.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-GF5L2VYU-D_OWvXzX.js → chunk-GF5L2VYU-BU0qS2YV.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-JWPE2WC7-CasdPz5X.js → chunk-JWPE2WC7-DVH9dIGE.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-KBJHAD2P-Cj8lRrla.js → chunk-KBJHAD2P-Dt60SmgM.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-RYQCIY6F-CUvD1FSd.js → chunk-RYQCIY6F-BoaUGuEY.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-XXDRQBXY-CqLw_eqB.js → chunk-XXDRQBXY-_WHFDTp2.js} +1 -1
- package/dist/worker/console/static/assets/classDiagram-JCYQIIEL-DZFra1GO.js +1 -0
- package/dist/worker/console/static/assets/classDiagram-v2-OCEON4UE-DZFra1GO.js +1 -0
- package/dist/worker/console/static/assets/{cose-bilkent-JH36ORCC-ku-WqJPx.js → cose-bilkent-JH36ORCC-BxkbTIRd.js} +1 -1
- package/dist/worker/console/static/assets/{cynefin-VYW2F7L2-Bq_PYDhl.js → cynefin-VYW2F7L2-Bf2UVnoG.js} +1 -1
- package/dist/worker/console/static/assets/{cynefinDiagram-MW4NZA55-BiGZV641.js → cynefinDiagram-MW4NZA55-Bo23q1J_.js} +1 -1
- package/dist/worker/console/static/assets/{dagre-VZM6K2ZE-C4JRlglF.js → dagre-VZM6K2ZE-Dd_UU49i.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-7IWD3JNH-B0dNeWYG.js → diagram-7IWD3JNH-ebUa1a9y.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-B4RE2ZJO-Cj35YvYM.js → diagram-B4RE2ZJO-BucthU8r.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-LBJQPF4R-uzQoJ2-8.js → diagram-LBJQPF4R-BgqNpAkm.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-Q27KOJAE-D1a-Buoz.js → diagram-Q27KOJAE-DC4q5NGa.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-UB23O5K3-XjRrLRSs.js → diagram-UB23O5K3-CBqCaMeb.js} +1 -1
- package/dist/worker/console/static/assets/{ebnfDiagram-BXEA7PRR-Da_O24cW.js → ebnfDiagram-BXEA7PRR-CGtnbQ3-.js} +1 -1
- package/dist/worker/console/static/assets/{erDiagram-JOGREHBK-BLJ8jrYU.js → erDiagram-JOGREHBK-CG3LUao5.js} +1 -1
- package/dist/worker/console/static/assets/{flowDiagram-UKHOOZJN-B9GrLjM0.js → flowDiagram-UKHOOZJN-D2ZVgoFS.js} +1 -1
- package/dist/worker/console/static/assets/{ganttDiagram-PKOTCBZU-CeJ0TqiK.js → ganttDiagram-PKOTCBZU-DoYFZiKt.js} +1 -1
- package/dist/worker/console/static/assets/{gitGraphDiagram-DS77QQ5N-Bm2eNNFX.js → gitGraphDiagram-DS77QQ5N-CKoP1s6j.js} +1 -1
- package/dist/worker/console/static/assets/index-CAZ2fC_X.css +1 -0
- package/dist/worker/console/static/assets/index-vbTcFnFs.js +449 -0
- package/dist/worker/console/static/assets/{infoDiagram-6WML65LV-DB26i3d8.js → infoDiagram-6WML65LV-Duofv8p2.js} +1 -1
- package/dist/worker/console/static/assets/{ishikawaDiagram-WSZJBQD7-BOGRyeZT.js → ishikawaDiagram-WSZJBQD7-D2nlCkA1.js} +1 -1
- package/dist/worker/console/static/assets/{journeyDiagram-NVQOT4AX-CZRoBO_6.js → journeyDiagram-NVQOT4AX-Dd4IHdus.js} +1 -1
- package/dist/worker/console/static/assets/{kanban-definition-27J2QSJJ-qlWhJyoB.js → kanban-definition-27J2QSJJ-Bi20AOdt.js} +1 -1
- package/dist/worker/console/static/assets/{linear-CfUiDDB3.js → linear-D59YJ9kB.js} +1 -1
- package/dist/worker/console/static/assets/{mermaid.core-BQe6fpqj.js → mermaid.core-Bl13LpNa.js} +5 -5
- package/dist/worker/console/static/assets/{mindmap-definition-FAOFIHXS-zFzWHw64.js → mindmap-definition-FAOFIHXS-C6DLP5eY.js} +1 -1
- package/dist/worker/console/static/assets/{pegDiagram-VL7TDLO6-DFGUqjZO.js → pegDiagram-VL7TDLO6-BjO4cy64.js} +1 -1
- package/dist/worker/console/static/assets/{pieDiagram-7S7Q4E2Y-BB5i0l8Z.js → pieDiagram-7S7Q4E2Y-D7i2qZTG.js} +1 -1
- package/dist/worker/console/static/assets/{quadrantDiagram-CIZ2JOQS-obbVZ_X5.js → quadrantDiagram-CIZ2JOQS-Dx8o_h0y.js} +1 -1
- package/dist/worker/console/static/assets/{railroadDiagram-AXF67PYL-BwdaNzc1.js → railroadDiagram-AXF67PYL-B55orI_t.js} +1 -1
- package/dist/worker/console/static/assets/{requirementDiagram-LRYGKXZP-CkTJaugh.js → requirementDiagram-LRYGKXZP-zWaehp6f.js} +1 -1
- package/dist/worker/console/static/assets/{sankeyDiagram-W5VNT64P-BwJ-hIgq.js → sankeyDiagram-W5VNT64P-CcA-pjvD.js} +1 -1
- package/dist/worker/console/static/assets/{sequenceDiagram-SI44F4Z6-BzSgmm7w.js → sequenceDiagram-SI44F4Z6-BOFbzNHI.js} +1 -1
- package/dist/worker/console/static/assets/{sizeCapture-X5ZJPWSS-Bn3oj5Q1.js → sizeCapture-X5ZJPWSS-BjAejah1.js} +1 -1
- package/dist/worker/console/static/assets/{stateDiagram-OKZ733FA-dxhGtL8K.js → stateDiagram-OKZ733FA-DXYwxJwZ.js} +1 -1
- package/dist/worker/console/static/assets/stateDiagram-v2-UEYNNEHI-Clg3V9t1.js +1 -0
- package/dist/worker/console/static/assets/{swimlanes-SLNWSIFB-BLpEInya.js → swimlanes-SLNWSIFB-OYiOch8n.js} +2 -2
- package/dist/worker/console/static/assets/swimlanesDiagram-ULZ7WXOC-DX1dxAAW.js +8 -0
- package/dist/worker/console/static/assets/{timeline-definition-Z64GVDOM-ChJGYcXy.js → timeline-definition-Z64GVDOM-cZfH3nmU.js} +1 -1
- package/dist/worker/console/static/assets/{vennDiagram-T6HMQDX7-DK-qjRer.js → vennDiagram-T6HMQDX7-BDE7b3E1.js} +1 -1
- package/dist/worker/console/static/assets/{wardleyDiagram-T6FBY63Y-3qeaWAg-.js → wardleyDiagram-T6FBY63Y-kyyGy9WJ.js} +1 -1
- package/dist/worker/console/static/assets/{xychartDiagram-ELKLHX3M-CootlyP9.js → xychartDiagram-ELKLHX3M-CJj6VTog.js} +1 -1
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/console/static-src/active-run-badge.js +17 -0
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +5 -5
- package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +19 -8
- package/dist/worker/console/static-src/operator-chat/input-history.js +8 -6
- package/dist/worker/console/static-src/operator-chat/open-preview-in-browser.js +15 -7
- package/dist/worker/console/static-src/operator-chat/runtime-snapshot-store.js +10 -0
- package/dist/worker/console/static-src/operator-chat/useChatSessions.js +258 -23
- package/dist/worker/console/static-src/operator-chat/useChatThread.js +112 -4
- package/dist/worker/console/static-src/operator-chat/useComposer.js +13 -4
- package/dist/worker/console/static-src/operator-chat/useRuntimeControls.js +29 -6
- package/dist/worker/console/static-src/operator-chat/useRuntimeSnapshot.js +8 -2
- package/dist/worker/console/static-src/shell/console-update-reload.js +36 -9
- package/dist/worker/console/static-src/shell/workspace-route.js +11 -0
- package/dist/worker/console/workspace-context.js +115 -1
- package/dist/worker/materialize/frontend-split-task-materializer.js +72 -0
- package/dist/worker/observe/routes.js +4 -0
- package/dist/worker/observe/static/dag-helpers.js +4 -2
- package/dist/worker/observe/static/dag-node-purpose.js +5 -0
- package/dist/worker/observe/static/inspect-workspace.js +23 -0
- package/dist/worker/observe/static/kpi.js +2 -2
- package/dist/worker/observe/static/operator-chrome.d.ts +10 -2
- package/dist/worker/observe/static/operator-chrome.js +37 -23
- package/dist/worker/observe/static/relations.js +2 -2
- package/dist/worker/observe/static/router.d.ts +12 -1
- package/dist/worker/observe/static/router.js +48 -6
- package/dist/worker/observe/static/run-processing.js +4 -2
- package/dist/worker/observe/static/shell-chrome.js +2 -2
- package/dist/worker/observe/static/state.js +5 -0
- package/dist/worker/observe/static/styles.css +95 -0
- package/dist/worker/observe/static/views/dag-inspector.js +14 -7
- package/dist/worker/observe/static/views/dag.js +10 -2
- package/dist/worker/observe/static/views/dags.js +3 -1
- package/dist/worker/observe/static/views/dashboard.js +14 -6
- package/dist/worker/observe/static/views/pool.js +7 -2
- package/dist/worker/observe/static/views/run.js +12 -3
- package/dist/worker/observe/static/views/session-timeline.js +69 -10
- package/dist/worker/observe/static/views/task.js +7 -2
- package/dist/workflows/dag/backend-test-case-coverage-analysis.js +215 -28
- package/dist/workflows/dag/backend-test-markdown-workflow.js +13 -1
- package/dist/workflows/dag/backend-test-plan-protocol.js +106 -0
- package/dist/workflows/dag/backend-test-scenario-param.js +172 -25
- package/dist/workflows/dag/backend-test-scenario-partitions.js +62 -1
- package/dist/workflows/dag/backend-test-writer-completeness.js +104 -12
- package/dist/workflows/dag/contract-validator-registrations.js +1 -2
- package/dist/workflows/dag/dag-retry-schema.js +138 -0
- package/dist/workflows/dag/frontend-closeout.js +221 -0
- package/dist/workflows/dag/frontend-design-policy.js +400 -0
- package/dist/workflows/dag/frontend-human-decision.js +182 -0
- package/dist/workflows/dag/frontend-implementation-contract.js +1237 -192
- package/dist/workflows/dag/frontend-plan-render.js +2 -1
- package/dist/workflows/dag/frontend-prewrite-gate.js +256 -350
- package/dist/workflows/dag/frontend-provider-capability-matrix.js +159 -0
- package/dist/workflows/dag/frontend-recovery-capsule.js +455 -0
- package/dist/workflows/dag/frontend-recovery-controller.js +226 -0
- package/dist/workflows/dag/frontend-recovery-lineage.js +178 -0
- package/dist/workflows/dag/frontend-recovery-plan.js +21 -10
- package/dist/workflows/dag/frontend-recovery-run.js +166 -34
- package/dist/workflows/dag/frontend-repair.js +1 -432
- package/dist/workflows/dag/frontend-review-context.js +261 -15
- package/dist/workflows/dag/frontend-review-findings.js +270 -0
- package/dist/workflows/dag/frontend-shadow-dual-write.js +941 -0
- package/dist/workflows/dag/frontend-shape-capsule-store.js +191 -0
- package/dist/workflows/dag/frontend-shape-facts.js +419 -0
- package/dist/workflows/dag/frontend-shape.js +435 -0
- package/dist/workflows/dag/frontend-source-fidelity-ledger.js +108 -0
- package/dist/workflows/dag/frontend-split-application-service.js +203 -0
- package/dist/workflows/dag/frontend-split-orchestrator.js +899 -0
- package/dist/workflows/dag/frontend-typed-event-store.js +452 -0
- package/dist/workflows/dag/frontend-typed-event-transaction.js +180 -0
- package/dist/workflows/dag/frontend-verification-trace.js +252 -25
- package/dist/workflows/dag/frontend-worktree-diff.js +250 -17
- package/dist/workflows/dag/frontend-writer-admission.js +319 -0
- package/dist/workflows/dag/frontend-writer-status.js +256 -0
- package/dist/workflows/dag/init-hybrid.js +1028 -562
- package/dist/workflows/dag/interrupt-request.js +7 -0
- package/dist/workflows/dag/node-execution.js +828 -4
- package/dist/workflows/dag/prompt.js +70 -3
- package/dist/workflows/dag/recovery-lease.js +170 -0
- package/dist/workflows/dag/report.js +37 -1
- package/dist/workflows/dag/rerun-feedback.js +315 -1
- package/dist/workflows/dag/rerun-plan.js +17 -6
- package/dist/workflows/dag/rerun-run.js +44 -6
- package/dist/workflows/dag/rerun-task.js +251 -13
- package/dist/workflows/dag/retry-policy.js +219 -104
- package/dist/workflows/dag/runner.js +580 -124
- package/dist/workflows/dag/scheduler.js +133 -20
- package/dist/workflows/dag/types.js +244 -16
- package/dist/workflows/dag/validate.js +28 -12
- package/docs/examples/README.md +5 -0
- package/docs/init-surface.manifest.json +30 -12
- package/docs/skills/vetted-skill-registry.md +4 -2
- package/docs/templates/README.md +2 -0
- package/docs/templates/agent-dag-report.schema.json +8 -2
- package/docs/templates/agent-dag.schema.json +1 -1
- package/docs/templates/backend-test-dag.json +23 -20
- package/docs/templates/frontend-implementation-contract.schema.json +4 -1
- package/docs/templates/frontend-implementation-dag.json +89 -0
- package/docs/templates/spec-registry.schema.json +45 -0
- package/harness.json +1 -1
- package/package.json +4 -3
- package/skills/frontend-bounded-implement/SKILL.md +15 -14
- package/skills/frontend-bounded-implement/references/code-standards.md +19 -0
- package/skills/frontend-contract/SKILL.md +23 -0
- package/skills/frontend-contract/references/contract-protocol.md +34 -0
- package/skills/frontend-design-review/SKILL.md +22 -41
- package/skills/frontend-plan/SKILL.md +26 -0
- package/skills/frontend-plan/references/decision-contract.md +37 -0
- package/skills/frontend-plan/references/design-decisions.md +17 -0
- package/skills/frontend-review/SKILL.md +20 -15
- package/skills/frontend-review/references/review-findings.md +6 -7
- package/skills/frontend-scout/SKILL.md +25 -0
- package/skills/frontend-scout/references/design-evidence.md +16 -0
- package/skills/frontend-scout/references/scout-evidence.md +23 -0
- package/skills/frontend-verification/SKILL.md +1 -1
- package/skills/loop-agent/references/command-reference.md +1 -0
- package/skills/loop-agent/references/hybrid-dag.md +2 -2
- package/dist/worker/console/static/assets/channel-BU5gOilw.js +0 -1
- package/dist/worker/console/static/assets/classDiagram-JCYQIIEL-DIzKJGHr.js +0 -1
- package/dist/worker/console/static/assets/classDiagram-v2-OCEON4UE-DIzKJGHr.js +0 -1
- package/dist/worker/console/static/assets/index-H9rFJiGL.css +0 -1
- package/dist/worker/console/static/assets/index-xwu9GxEc.js +0 -451
- package/dist/worker/console/static/assets/stateDiagram-v2-UEYNNEHI-Cld2qK9v.js +0 -1
- package/dist/worker/console/static/assets/swimlanesDiagram-ULZ7WXOC-TIHpiT7w.js +0 -8
- package/skills/frontend-implementation/SKILL.md +0 -52
- package/skills/frontend-implementation/references/code-standards.md +0 -33
- package/skills/frontend-implementation/references/design-spec.md +0 -56
- package/skills/frontend-implementation/references/node-contracts.md +0 -31
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
-
import {
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
3
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
4
4
|
import { hostname } from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
@@ -7,7 +7,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
|
|
|
7
7
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
8
8
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
9
9
|
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
10
|
-
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun,
|
|
10
|
+
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, readDagRunState, requireActiveDagRun, } from "./lifecycle.js";
|
|
11
11
|
import { markInterruptNeedsReconcile, mergeAbortSignals, readInterruptRequest, runnerIdentityFromState, settleDagInterrupt, startInterruptWatcher, } from "./interrupt-request.js";
|
|
12
12
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
13
13
|
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
@@ -26,7 +26,13 @@ import { runConvergencePassController, } from "./convergence/controller.js";
|
|
|
26
26
|
import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
27
27
|
import { topoSortToRanks } from "./topo.js";
|
|
28
28
|
import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
|
|
29
|
+
import { classifyReviewIssueCategoryFailureOwner, readCommittedDesignRequestFact, readCommittedReviewRequestFact, writeFrontendReviewFindingsPrd, } from "./frontend-review-findings.js";
|
|
30
|
+
import { bindFrontendShapeStateToRunTopology, commitFrontendShapeTransitionCas, commitRuntimeFrontendShapeFact, isCanonicalFrontendShapeDag, } from "./frontend-shape-facts.js";
|
|
31
|
+
import { assertFrontendTopologyBinding, buildFrontendTopologyBinding, computeFrontendShapeSourceDigest, } from "./frontend-shape.js";
|
|
32
|
+
import { ensureFrontendShapeCapsuleRecord } from "./frontend-shape-capsule-store.js";
|
|
29
33
|
import { parseDagSpec, resolveModelForTask, } from "./types.js";
|
|
34
|
+
import { computeActionPayloadHash, } from "../../infrastructure/console/operation-store.js";
|
|
35
|
+
import { acquireDagRecoveryLease, bindDagRecoveryLeaseExecution, claimDagRecoveryLease, } from "./recovery-lease.js";
|
|
30
36
|
import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
|
|
31
37
|
import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
|
|
32
38
|
import { executeDynamicMapExpansion } from "./dynamic-runtime/map.js";
|
|
@@ -46,6 +52,51 @@ export function toKebabSlug(text, maxLen = 40) {
|
|
|
46
52
|
.replace(/^-|-$/g, "")
|
|
47
53
|
.slice(0, maxLen) || "dag");
|
|
48
54
|
}
|
|
55
|
+
export async function loadCanonicalFrontendTaskSourceBinding(cwd, spec) {
|
|
56
|
+
const binding = spec.sourceBinding;
|
|
57
|
+
if (!binding?.taskId)
|
|
58
|
+
throw new Error("canonical frontend DAG requires sourceBinding.taskId");
|
|
59
|
+
const requirement = binding.sources.find((source) => source.kind === "requirement");
|
|
60
|
+
if (!requirement)
|
|
61
|
+
throw new Error("canonical frontend DAG requires a bound requirement source");
|
|
62
|
+
const constraint = binding.sources.find((source) => source.kind === "constraint");
|
|
63
|
+
const readFreshSource = async (source) => {
|
|
64
|
+
let bytes;
|
|
65
|
+
try {
|
|
66
|
+
bytes = await readFile(path.resolve(cwd, source.path));
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
throw new Error(`canonical frontend ${source.kind} source freshness read failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
70
|
+
}
|
|
71
|
+
const actualSha256 = createHash("sha256").update(bytes).digest("hex");
|
|
72
|
+
if (actualSha256 !== source.sha256) {
|
|
73
|
+
throw new Error(`canonical frontend ${source.kind} source sha256 drifted: expected ${source.sha256}, received ${actualSha256}`);
|
|
74
|
+
}
|
|
75
|
+
return bytes.toString("utf8");
|
|
76
|
+
};
|
|
77
|
+
const requirementMarkdown = await readFreshSource(requirement);
|
|
78
|
+
const constraintMarkdown = constraint ? await readFreshSource(constraint) : "";
|
|
79
|
+
return {
|
|
80
|
+
taskId: binding.taskId,
|
|
81
|
+
requirementMarkdown,
|
|
82
|
+
constraintMarkdown,
|
|
83
|
+
sourceDigest: computeFrontendShapeSourceDigest({ requirementMarkdown, constraintMarkdown }),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export async function ensureFreshFrontendShapeCapsuleRecord(input) {
|
|
87
|
+
const freshSource = await loadCanonicalFrontendTaskSourceBinding(input.cwd, input.spec);
|
|
88
|
+
if (freshSource.taskId !== input.canonicalSource.taskId ||
|
|
89
|
+
freshSource.sourceDigest !== input.canonicalSource.sourceDigest) {
|
|
90
|
+
throw new Error("canonical frontend task source freshness binding drifted before transition commit");
|
|
91
|
+
}
|
|
92
|
+
await (input.ensureRecord ?? ensureFrontendShapeCapsuleRecord)({
|
|
93
|
+
repoRoot: input.cwd,
|
|
94
|
+
taskId: input.canonicalSource.taskId,
|
|
95
|
+
parentRunId: input.parentRunId,
|
|
96
|
+
terminalOutcome: input.terminalOutcome,
|
|
97
|
+
capsule: input.capsule,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
49
100
|
export function extractTaskIdFromDagSpec(spec) {
|
|
50
101
|
if (spec.objective) {
|
|
51
102
|
const match = spec.objective.match(TASK_ID_IN_TEXT_PATTERN);
|
|
@@ -180,6 +231,15 @@ export function createInitialRunState(spec, opts, ranks, runId = opts.runId ?? "
|
|
|
180
231
|
...(opts.workerAssociation
|
|
181
232
|
? { workerAssociation: structuredClone(opts.workerAssociation) }
|
|
182
233
|
: {}),
|
|
234
|
+
...(isCanonicalFrontendShapeDag(new Map(spec.tasks.map((task) => [task.id, task])))
|
|
235
|
+
? {
|
|
236
|
+
frontendShapeState: {
|
|
237
|
+
frozenTopology: buildFrontendTopologyBinding(spec.tasks.map((task) => task.id)),
|
|
238
|
+
revision: 0,
|
|
239
|
+
committedFacts: {},
|
|
240
|
+
},
|
|
241
|
+
}
|
|
242
|
+
: {}),
|
|
183
243
|
};
|
|
184
244
|
initRunBudgetLedger(state, spec.budget);
|
|
185
245
|
return state;
|
|
@@ -301,6 +361,7 @@ export async function runDag(spec, opts) {
|
|
|
301
361
|
activeRunDir,
|
|
302
362
|
completedRunDir,
|
|
303
363
|
pausedRunDir,
|
|
364
|
+
recovery: opts.recovery,
|
|
304
365
|
});
|
|
305
366
|
}
|
|
306
367
|
finally {
|
|
@@ -322,6 +383,16 @@ export async function runDagContinuation(opts) {
|
|
|
322
383
|
await assertTaskContractBindingConsistent({ repoRoot: opts.cwd, spec: opts.spec });
|
|
323
384
|
const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), opts.spec.budget);
|
|
324
385
|
const state = opts.state;
|
|
386
|
+
const canonicalFrontend = isCanonicalFrontendShapeDag(new Map(opts.spec.tasks.map((task) => [task.id, task])));
|
|
387
|
+
if (canonicalFrontend) {
|
|
388
|
+
assertFrontendTopologyBinding(state.frontendShapeState?.frozenTopology, opts.spec.tasks.map((task) => task.id));
|
|
389
|
+
}
|
|
390
|
+
else if (state.frontendShapeState) {
|
|
391
|
+
throw new Error("non-frontend DAG unexpectedly contains frontend topology state");
|
|
392
|
+
}
|
|
393
|
+
if (state.frontendShapeState?.terminalOutcome) {
|
|
394
|
+
throw new Error(`dag run ${state.runId} is terminal due to frontend shape ${state.frontendShapeState.terminalOutcome}`);
|
|
395
|
+
}
|
|
325
396
|
state.status = "running";
|
|
326
397
|
const resumedAt = new Date().toISOString();
|
|
327
398
|
state.runner = {
|
|
@@ -347,6 +418,7 @@ export async function runDagContinuation(opts) {
|
|
|
347
418
|
activeRunDir,
|
|
348
419
|
completedRunDir,
|
|
349
420
|
pausedRunDir,
|
|
421
|
+
recovery: opts.recovery,
|
|
350
422
|
});
|
|
351
423
|
}
|
|
352
424
|
export async function resumeDagRun(opts) {
|
|
@@ -449,9 +521,13 @@ export async function resumeDagRun(opts) {
|
|
|
449
521
|
}
|
|
450
522
|
async function executeDagCheckpoint(input) {
|
|
451
523
|
const { spec, state, ranks, runDir: initialRunDir, cwd, maxConcurrent, completedRunDir, pausedRunDir, } = input;
|
|
524
|
+
bindFrontendShapeStateToRunTopology(state);
|
|
452
525
|
let runDir = initialRunDir;
|
|
453
526
|
let stateWriteQueue = Promise.resolve();
|
|
527
|
+
let shapeTransitionBarrier;
|
|
454
528
|
const persistState = async (options) => {
|
|
529
|
+
if (shapeTransitionBarrier)
|
|
530
|
+
await shapeTransitionBarrier;
|
|
455
531
|
stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, state, options));
|
|
456
532
|
await stateWriteQueue;
|
|
457
533
|
};
|
|
@@ -576,12 +652,50 @@ async function executeDagCheckpoint(input) {
|
|
|
576
652
|
.map((rank) => rank.filter((nodeId) => nodeId === "closeout-pi"))
|
|
577
653
|
.filter((rank) => rank.length > 0)
|
|
578
654
|
: [];
|
|
655
|
+
const canonicalFrontendSource = isCanonicalFrontendShapeDag(tasksById)
|
|
656
|
+
? await loadCanonicalFrontendTaskSourceBinding(cwd, spec)
|
|
657
|
+
: undefined;
|
|
579
658
|
const executeRanks = (selectedRanks) => executeDagRanksOnce({
|
|
580
659
|
state,
|
|
581
660
|
ranks: selectedRanks,
|
|
582
661
|
tasksById,
|
|
583
662
|
maxConcurrent,
|
|
584
663
|
persistState,
|
|
664
|
+
commitFrontendShapeTransition: async (request) => {
|
|
665
|
+
let releaseBarrier;
|
|
666
|
+
shapeTransitionBarrier = new Promise((resolve) => {
|
|
667
|
+
releaseBarrier = resolve;
|
|
668
|
+
});
|
|
669
|
+
try {
|
|
670
|
+
if (!canonicalFrontendSource) {
|
|
671
|
+
throw new Error("frontend shape transition requires canonical task source binding");
|
|
672
|
+
}
|
|
673
|
+
return await commitFrontendShapeTransitionCas({
|
|
674
|
+
state,
|
|
675
|
+
tasksById,
|
|
676
|
+
request,
|
|
677
|
+
taskSourceDigest: canonicalFrontendSource.sourceDigest,
|
|
678
|
+
ensureRecord: async (capsule, terminalOutcome) => {
|
|
679
|
+
await ensureFreshFrontendShapeCapsuleRecord({
|
|
680
|
+
cwd,
|
|
681
|
+
spec,
|
|
682
|
+
canonicalSource: canonicalFrontendSource,
|
|
683
|
+
parentRunId: state.runId,
|
|
684
|
+
terminalOutcome,
|
|
685
|
+
capsule,
|
|
686
|
+
});
|
|
687
|
+
},
|
|
688
|
+
persist: async (nextState) => {
|
|
689
|
+
stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, nextState));
|
|
690
|
+
await stateWriteQueue;
|
|
691
|
+
},
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
finally {
|
|
695
|
+
releaseBarrier();
|
|
696
|
+
shapeTransitionBarrier = undefined;
|
|
697
|
+
}
|
|
698
|
+
},
|
|
585
699
|
runDir,
|
|
586
700
|
abortSignal: input.abortSignal,
|
|
587
701
|
createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
|
|
@@ -617,6 +731,18 @@ async function executeDagCheckpoint(input) {
|
|
|
617
731
|
});
|
|
618
732
|
const node = state.nodes[nodeId];
|
|
619
733
|
if (node && (node.status === "FINISHED" || node.status === "ERROR")) {
|
|
734
|
+
if (node.status === "FINISHED") {
|
|
735
|
+
if (canonicalFrontendSource) {
|
|
736
|
+
await commitRuntimeFrontendShapeFact({
|
|
737
|
+
state,
|
|
738
|
+
tasksById,
|
|
739
|
+
sourceNodeId: nodeId,
|
|
740
|
+
taskId: canonicalFrontendSource.taskId,
|
|
741
|
+
requirementMarkdown: canonicalFrontendSource.requirementMarkdown,
|
|
742
|
+
constraintMarkdown: canonicalFrontendSource.constraintMarkdown,
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
}
|
|
620
746
|
recordFinishedNodeBudget(state, node);
|
|
621
747
|
await writeBudgetLedgerArtifacts(runDir, state.budgetLedger);
|
|
622
748
|
await persistState();
|
|
@@ -626,6 +752,9 @@ async function executeDagCheckpoint(input) {
|
|
|
626
752
|
let pausedByNodeId;
|
|
627
753
|
while (true) {
|
|
628
754
|
pausedByNodeId = await executeRanks(convergenceRanks);
|
|
755
|
+
if (state.frontendShapeState?.terminalOutcome) {
|
|
756
|
+
break;
|
|
757
|
+
}
|
|
629
758
|
if (pausedByNodeId) {
|
|
630
759
|
break;
|
|
631
760
|
}
|
|
@@ -681,7 +810,7 @@ async function executeDagCheckpoint(input) {
|
|
|
681
810
|
runDir = await moveToPausedRunDir(runDir, pausedRunDir);
|
|
682
811
|
}
|
|
683
812
|
else {
|
|
684
|
-
const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
|
|
813
|
+
const { recoveryPending, recovery: recoveryTrigger } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd, spec.frontendRecovery?.maxContinuations ?? 1);
|
|
685
814
|
await finalizeRunOwnedInterrupt({
|
|
686
815
|
runDir,
|
|
687
816
|
state,
|
|
@@ -689,56 +818,103 @@ async function executeDagCheckpoint(input) {
|
|
|
689
818
|
interruptAborted: interruptController.signal.aborted,
|
|
690
819
|
});
|
|
691
820
|
await persistState();
|
|
692
|
-
if (recoveryPending) {
|
|
693
|
-
//
|
|
694
|
-
//
|
|
695
|
-
//
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
821
|
+
if (recoveryPending && input.recovery && recoveryTrigger) {
|
|
822
|
+
// M6 recovery operation controller path: construct/reuse one recovery
|
|
823
|
+
// operation keyed by a deterministic clientRequestId and supervise it
|
|
824
|
+
// to terminal, instead of staging an in-process child run.
|
|
825
|
+
const created = await createRecoveryOperation({
|
|
826
|
+
cwd,
|
|
827
|
+
runDir,
|
|
828
|
+
spec,
|
|
829
|
+
state,
|
|
830
|
+
trigger: recoveryTrigger,
|
|
831
|
+
deps: input.recovery,
|
|
832
|
+
});
|
|
833
|
+
if (created.kind === "conflict") {
|
|
834
|
+
// Same clientRequestId with a different payload → fail closed; never
|
|
835
|
+
// create a second recovery operation (AC-004).
|
|
836
|
+
state.status = "failed";
|
|
837
|
+
state.terminalReason =
|
|
838
|
+
"frontend recovery request-id reuse conflict";
|
|
839
|
+
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
|
|
840
|
+
}
|
|
841
|
+
else if (created.kind === "lease-held") {
|
|
842
|
+
// A different recovery entrance has already durably claimed this
|
|
843
|
+
// parent. Do not "help" by running its operation: that would reopen
|
|
844
|
+
// the exact two-child-DAG race the lease prevents.
|
|
845
|
+
state.status = "failed";
|
|
846
|
+
state.terminalReason =
|
|
847
|
+
"frontend recovery parent lease already held";
|
|
848
|
+
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, "auto-recovery-blocked");
|
|
849
|
+
}
|
|
850
|
+
else {
|
|
851
|
+
const { runRecoveryOperationController } = await import("./frontend-recovery-controller.js");
|
|
852
|
+
const result = await runRecoveryOperationController({
|
|
853
|
+
operationId: created.operation.operationId,
|
|
705
854
|
parentRunId: state.runId,
|
|
855
|
+
observeOnly: created.kind === "observed",
|
|
856
|
+
deps: {
|
|
857
|
+
...input.recovery,
|
|
858
|
+
failureOwner: input.recovery.failureOwner ??
|
|
859
|
+
recoveryTrigger.failureOwner,
|
|
860
|
+
protocolFailureReason: input.recovery.protocolFailureReason ??
|
|
861
|
+
recoveryTrigger.protocolFailureReason,
|
|
862
|
+
recoveryMode: input.recovery.recoveryMode ?? recoveryTrigger.recoveryMode,
|
|
863
|
+
},
|
|
706
864
|
});
|
|
707
|
-
//
|
|
708
|
-
//
|
|
709
|
-
//
|
|
710
|
-
//
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
865
|
+
// The recovery CLI stages its child by writing a fresh
|
|
866
|
+
// `frontendRecoveryState` into the parent run on disk. A successful
|
|
867
|
+
// operation without that bound lineage is not a successful recovery:
|
|
868
|
+
// fail closed instead of archiving a false-positive `recovered` result.
|
|
869
|
+
let recoveryLineageError;
|
|
870
|
+
if (result.terminalReason === "succeeded") {
|
|
871
|
+
try {
|
|
872
|
+
const persistedParentState = await readDagRunState(runDir);
|
|
873
|
+
const persistedRecovery = persistedParentState.frontendRecoveryState;
|
|
874
|
+
if (!persistedRecovery ||
|
|
875
|
+
persistedRecovery.phase !== "child-running" ||
|
|
876
|
+
persistedRecovery.parentRunId !== state.runId ||
|
|
877
|
+
persistedRecovery.requestId !== recoveryTrigger.requestId ||
|
|
878
|
+
!persistedRecovery.childRunId) {
|
|
879
|
+
throw new Error("persisted recovery lineage is missing or unbound");
|
|
880
|
+
}
|
|
881
|
+
await assertRecoveredFrontendLineage({
|
|
882
|
+
cwd,
|
|
883
|
+
rootRunId: state.runId,
|
|
884
|
+
rootRequestId: recoveryTrigger.requestId,
|
|
885
|
+
recovery: persistedRecovery,
|
|
886
|
+
maxContinuations: spec.frontendRecovery?.maxContinuations ?? 1,
|
|
887
|
+
});
|
|
888
|
+
state.frontendRecoveryState = structuredClone(persistedRecovery);
|
|
889
|
+
}
|
|
890
|
+
catch (error) {
|
|
891
|
+
recoveryLineageError =
|
|
892
|
+
error instanceof Error ? error.message : String(error);
|
|
893
|
+
state.status = "failed";
|
|
894
|
+
state.terminalReason =
|
|
895
|
+
`frontend recovery lineage invalid: ${recoveryLineageError}`;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
state.frontendRecoveryResult = buildRecoveryResultForTrigger(state, recoveryTrigger, result.terminalReason === "succeeded" && !recoveryLineageError
|
|
899
|
+
? "recovered"
|
|
900
|
+
: "auto-recovery-blocked");
|
|
725
901
|
}
|
|
726
902
|
}
|
|
727
903
|
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
728
|
-
if (
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
await persistState();
|
|
732
|
-
}
|
|
733
|
-
try {
|
|
734
|
-
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
735
|
-
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
736
|
-
}
|
|
737
|
-
catch (error) {
|
|
738
|
-
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
739
|
-
}
|
|
740
|
-
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
904
|
+
if (input.beforeTerminalArchive) {
|
|
905
|
+
await input.beforeTerminalArchive({ runDir, state });
|
|
906
|
+
await persistState();
|
|
741
907
|
}
|
|
908
|
+
// Parent is always archived to completed/ — no active/ retention for an
|
|
909
|
+
// in-process child CAS anymore (recovery runs as a separate operation).
|
|
910
|
+
try {
|
|
911
|
+
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
912
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
913
|
+
}
|
|
914
|
+
catch (error) {
|
|
915
|
+
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
916
|
+
}
|
|
917
|
+
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
742
918
|
}
|
|
743
919
|
await relocateRunArtifactPaths({
|
|
744
920
|
runDir,
|
|
@@ -766,6 +942,56 @@ async function executeDagCheckpoint(input) {
|
|
|
766
942
|
process.removeListener("SIGINT", onSigInt);
|
|
767
943
|
}
|
|
768
944
|
}
|
|
945
|
+
async function assertRecoveredFrontendLineage(input) {
|
|
946
|
+
const maxContinuations = Math.max(1, Math.min(5, input.maxContinuations));
|
|
947
|
+
let parentRunId = input.rootRunId;
|
|
948
|
+
let recovery = input.recovery;
|
|
949
|
+
const recoveryRootRunId = recovery.recoveryRootRunId;
|
|
950
|
+
const visited = new Set();
|
|
951
|
+
if (recoveryRootRunId !== input.rootRunId) {
|
|
952
|
+
throw new Error("recovery root lineage binding mismatch");
|
|
953
|
+
}
|
|
954
|
+
for (let depth = 0; depth < maxContinuations; depth += 1) {
|
|
955
|
+
// The persisted parent intent represents the edge about to be traversed,
|
|
956
|
+
// so its counters equal the number of already-completed continuation edges.
|
|
957
|
+
// Each completed child increments both counters exactly once; no edge may
|
|
958
|
+
// exceed the frozen DAG quota.
|
|
959
|
+
if (recovery.continuationCount !== depth ||
|
|
960
|
+
recovery.attemptIndex !== depth) {
|
|
961
|
+
throw new Error("recovery child lineage counters are not monotonic");
|
|
962
|
+
}
|
|
963
|
+
if (recovery.phase !== "child-running" ||
|
|
964
|
+
recovery.parentRunId !== parentRunId ||
|
|
965
|
+
recovery.recoveryRootRunId !== recoveryRootRunId ||
|
|
966
|
+
!recovery.childRunId ||
|
|
967
|
+
(depth === 0
|
|
968
|
+
? recovery.requestId !== input.rootRequestId
|
|
969
|
+
: recovery.requestId !== parentRunId)) {
|
|
970
|
+
throw new Error("persisted recovery lineage is missing or unbound");
|
|
971
|
+
}
|
|
972
|
+
if (visited.has(recovery.childRunId)) {
|
|
973
|
+
throw new Error("recovery child lineage contains a cycle");
|
|
974
|
+
}
|
|
975
|
+
visited.add(recovery.childRunId);
|
|
976
|
+
const child = await locateDagRun(input.cwd, recovery.childRunId);
|
|
977
|
+
if (!child || child.lifecycle !== "completed") {
|
|
978
|
+
throw new Error("recovery child is not completed");
|
|
979
|
+
}
|
|
980
|
+
const childState = await readDagRunState(child.runDir);
|
|
981
|
+
if (childState.runId !== recovery.childRunId) {
|
|
982
|
+
throw new Error("recovery child state binding mismatch");
|
|
983
|
+
}
|
|
984
|
+
if (childState.status === "finished")
|
|
985
|
+
return;
|
|
986
|
+
if (childState.frontendRecoveryResult?.outcome !== "recovered" ||
|
|
987
|
+
!childState.frontendRecoveryState) {
|
|
988
|
+
throw new Error("recovery child terminal state is not successful");
|
|
989
|
+
}
|
|
990
|
+
parentRunId = childState.runId;
|
|
991
|
+
recovery = childState.frontendRecoveryState;
|
|
992
|
+
}
|
|
993
|
+
throw new Error("recovery child lineage exceeds the continuation quota");
|
|
994
|
+
}
|
|
769
995
|
async function finalizeRunOwnedInterrupt(input) {
|
|
770
996
|
const request = await readInterruptRequest(input.runDir);
|
|
771
997
|
if (!request)
|
|
@@ -871,39 +1097,21 @@ function isSuccessfulConvergenceTerminal(state) {
|
|
|
871
1097
|
return true;
|
|
872
1098
|
return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
|
|
873
1099
|
}
|
|
874
|
-
function buildFrontendRecoveryIntent(state, failureSource) {
|
|
875
|
-
const existing = state.frontendRecoveryState;
|
|
876
|
-
const requestId = existing?.requestId ?? randomUUID();
|
|
877
|
-
return {
|
|
878
|
-
schemaVersion: 1,
|
|
879
|
-
phase: "child-staging",
|
|
880
|
-
requestId,
|
|
881
|
-
recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
|
|
882
|
-
parentRunId: state.runId,
|
|
883
|
-
...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
|
|
884
|
-
attemptId: existing?.attemptId ?? `recovery-${requestId}`,
|
|
885
|
-
attemptIndex: existing?.attemptIndex ?? 0,
|
|
886
|
-
continuationCount: existing?.continuationCount ?? 0,
|
|
887
|
-
...(failureSource ? { failureSource } : {}),
|
|
888
|
-
revision: (existing?.revision ?? 0) + 1,
|
|
889
|
-
};
|
|
890
|
-
}
|
|
891
1100
|
function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
892
|
-
const recovery = state.frontendRecoveryState;
|
|
893
1101
|
return {
|
|
894
1102
|
schemaVersion: 1,
|
|
895
1103
|
outcome: "candidate-contract-invalid",
|
|
896
1104
|
origin: {
|
|
897
1105
|
kind: "frontend-prewrite-gate",
|
|
898
|
-
parentRunId:
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
failedNodeId: result.selectedPlanNodeId,
|
|
1106
|
+
parentRunId: state.runId,
|
|
1107
|
+
requestId: state.runId,
|
|
1108
|
+
failedNodeId: result.failureSource ?? "frontend-plan-pi",
|
|
902
1109
|
},
|
|
903
1110
|
failureClass: {
|
|
904
1111
|
code: "candidate-contract-invalid",
|
|
905
1112
|
classification: result.classification,
|
|
906
|
-
reason: result.
|
|
1113
|
+
reason: result.findings.map((finding) => `${finding.code}: ${finding.message}`).join("; ") ??
|
|
1114
|
+
"candidate contract invalid",
|
|
907
1115
|
},
|
|
908
1116
|
evidenceRefs: [
|
|
909
1117
|
{
|
|
@@ -915,15 +1123,13 @@ function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
|
915
1123
|
};
|
|
916
1124
|
}
|
|
917
1125
|
function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
918
|
-
const recovery = state.frontendRecoveryState;
|
|
919
1126
|
return {
|
|
920
1127
|
schemaVersion: 1,
|
|
921
1128
|
outcome: "auto-recovery-blocked",
|
|
922
1129
|
origin: {
|
|
923
1130
|
kind: "frontend-writer",
|
|
924
|
-
parentRunId:
|
|
925
|
-
|
|
926
|
-
requestId: recovery?.requestId ?? "",
|
|
1131
|
+
parentRunId: state.runId,
|
|
1132
|
+
requestId: state.runId,
|
|
927
1133
|
failedNodeId,
|
|
928
1134
|
},
|
|
929
1135
|
failureClass: {
|
|
@@ -934,64 +1140,119 @@ function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
|
934
1140
|
evidenceRefs: [],
|
|
935
1141
|
};
|
|
936
1142
|
}
|
|
937
|
-
function
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
failedNodeId: "",
|
|
952
|
-
},
|
|
953
|
-
evidenceRefs: [],
|
|
954
|
-
};
|
|
955
|
-
}
|
|
1143
|
+
function buildRecoveryResultForTrigger(state, trigger, outcome) {
|
|
1144
|
+
return {
|
|
1145
|
+
schemaVersion: 1,
|
|
1146
|
+
outcome,
|
|
1147
|
+
origin: {
|
|
1148
|
+
kind: trigger.failureSource === "frontend-implement-pi"
|
|
1149
|
+
? "frontend-writer"
|
|
1150
|
+
: "frontend-prewrite-gate",
|
|
1151
|
+
parentRunId: state.runId,
|
|
1152
|
+
requestId: trigger.requestId,
|
|
1153
|
+
failedNodeId: trigger.failureSource,
|
|
1154
|
+
},
|
|
1155
|
+
evidenceRefs: [],
|
|
1156
|
+
};
|
|
956
1157
|
}
|
|
957
|
-
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
1158
|
+
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd, frontendRecoveryQuota = 1) {
|
|
958
1159
|
const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
|
|
1160
|
+
if (state.frontendShapeState?.terminalOutcome &&
|
|
1161
|
+
state.frontendShapeState.transitionCapsule) {
|
|
1162
|
+
state.status = "failed";
|
|
1163
|
+
return { recoveryPending: false };
|
|
1164
|
+
}
|
|
959
1165
|
// Terminal override: a denied frontend writer must never surface as a
|
|
960
1166
|
// mechanical partial_failed just because prewrite FINISHED while the writer
|
|
961
|
-
// was SKIPPED. Fail closed on
|
|
962
|
-
let
|
|
1167
|
+
// was SKIPPED. Fail closed on blocked before aggregation.
|
|
1168
|
+
let recoveryTrigger;
|
|
963
1169
|
const skipFrontendRecovery = state.failureCategory === "controller-interrupted" ||
|
|
964
1170
|
state.continuation?.writerTransaction !== undefined;
|
|
965
1171
|
const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
|
|
966
1172
|
if (hasFrontendWriter && !skipFrontendRecovery) {
|
|
967
1173
|
const admission = await readFrontendPrewriteResult(runDir);
|
|
968
1174
|
if (admission.ok) {
|
|
969
|
-
|
|
1175
|
+
// M8: admission uses the accepted/blocked state machine. A blocked admission whose
|
|
1176
|
+
// failureSource is frontend-plan-pi is the contract-invalid analog of
|
|
1177
|
+
// the old retryable-invalid path (candidate continuation may re-plan);
|
|
1178
|
+
// every other blocked admission is terminal.
|
|
1179
|
+
const planContractInvalid = admission.result.classification === "blocked" &&
|
|
1180
|
+
admission.result.failureSource === "frontend-plan-pi";
|
|
1181
|
+
// A blocked admission caused by the design review committing
|
|
1182
|
+
// `request_design_changes` is a definite finding (not a protocol or
|
|
1183
|
+
// environment error): the design was reviewed and rejected. It is the
|
|
1184
|
+
// design-review analog of review-request-changes — not terminal, but
|
|
1185
|
+
// re-enters the next plan round with the committed design findings.
|
|
1186
|
+
// The failureSource may be frontend-writer-admission-shell or
|
|
1187
|
+
// frontend-design-policy-shell; the committed design fact is the
|
|
1188
|
+
// authoritative signal.
|
|
1189
|
+
const designRequestFact = await readCommittedDesignRequestFact({
|
|
1190
|
+
runDir,
|
|
1191
|
+
});
|
|
1192
|
+
const designRejected = admission.result.classification === "blocked" &&
|
|
1193
|
+
designRequestFact !== undefined;
|
|
1194
|
+
const terminalAdmissionDenial = (admission.result.classification === "blocked" ||
|
|
1195
|
+
admission.result.classification === "stale") &&
|
|
1196
|
+
!planContractInvalid &&
|
|
1197
|
+
!designRejected;
|
|
1198
|
+
if (terminalAdmissionDenial) {
|
|
970
1199
|
state.status = "failed";
|
|
971
1200
|
return { recoveryPending: false };
|
|
972
1201
|
}
|
|
973
|
-
if (
|
|
1202
|
+
if (designRejected && designRequestFact) {
|
|
1203
|
+
state.terminalReason = "design-request-changes";
|
|
1204
|
+
try {
|
|
1205
|
+
await writeFrontendReviewFindingsPrd({
|
|
1206
|
+
runDir,
|
|
1207
|
+
parentRunId: state.runId,
|
|
1208
|
+
fact: designRequestFact,
|
|
1209
|
+
});
|
|
1210
|
+
}
|
|
1211
|
+
catch (error) {
|
|
1212
|
+
console.warn(`[run-dag] warning: failed to write frontend design findings PRD: ${error instanceof Error ? error.message : String(error)}`);
|
|
1213
|
+
}
|
|
1214
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1215
|
+
if (continuationCount < frontendRecoveryQuota) {
|
|
1216
|
+
recoveryTrigger = {
|
|
1217
|
+
requestId: state.runId,
|
|
1218
|
+
// The rejected design is a plan input defect. Reset from the
|
|
1219
|
+
// planner so the next child can incorporate the committed
|
|
1220
|
+
// findings before design review runs again.
|
|
1221
|
+
failureSource: "frontend-plan-pi",
|
|
1222
|
+
failureOwner: classifyReviewIssueCategoryFailureOwner(designRequestFact.issueCategory),
|
|
1223
|
+
protocolFailureReason: `frontend design review request_design_changes (issueCategory=${designRequestFact.issueCategory})`,
|
|
1224
|
+
recoveryMode: "new-dag",
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
if (planContractInvalid) {
|
|
974
1229
|
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
975
|
-
if (continuationCount >=
|
|
976
|
-
// Continuation quota exhausted: a second
|
|
1230
|
+
if (continuationCount >= frontendRecoveryQuota) {
|
|
1231
|
+
// Continuation quota exhausted: a second contract-invalid candidate
|
|
977
1232
|
// is authoritative candidate-contract-invalid → forced failed.
|
|
978
1233
|
state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
|
|
979
1234
|
state.status = "failed";
|
|
980
1235
|
return { recoveryPending: false };
|
|
981
1236
|
}
|
|
982
|
-
// Root continuation still available:
|
|
983
|
-
//
|
|
984
|
-
//
|
|
985
|
-
|
|
986
|
-
|
|
1237
|
+
// Root continuation still available: emit a recovery trigger for the
|
|
1238
|
+
// M6 operation controller. The parent no longer writes
|
|
1239
|
+
// frontendRecoveryState or stays in active/ for a child CAS.
|
|
1240
|
+
recoveryTrigger = {
|
|
1241
|
+
requestId: state.runId,
|
|
1242
|
+
failureSource: admission.result.failureSource ?? "frontend-plan-pi",
|
|
1243
|
+
failureOwner: "plan",
|
|
1244
|
+
protocolFailureReason: "frontend candidate contract invalid",
|
|
1245
|
+
recoveryMode: "new-dag",
|
|
1246
|
+
};
|
|
987
1247
|
}
|
|
988
1248
|
}
|
|
989
1249
|
}
|
|
990
|
-
// Phase 5: writer transient partial write → rollback + recovery
|
|
991
|
-
// for frontend-implementation writers with a remaining root
|
|
992
|
-
// only when the rollback journal (captured before the
|
|
993
|
-
// cleanly; otherwise the run stays failed with
|
|
994
|
-
|
|
1250
|
+
// Phase 5: writer transient partial write → rollback + recovery trigger.
|
|
1251
|
+
// Only for frontend-implementation writers with a remaining root
|
|
1252
|
+
// continuation, and only when the rollback journal (captured before the
|
|
1253
|
+
// provider call) restores cleanly; otherwise the run stays failed with
|
|
1254
|
+
// auto-recovery-blocked.
|
|
1255
|
+
if (hasFrontendWriter && !recoveryTrigger && !skipFrontendRecovery) {
|
|
995
1256
|
const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
|
|
996
1257
|
const node = state.nodes[id];
|
|
997
1258
|
return (node &&
|
|
@@ -1002,15 +1263,20 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
|
1002
1263
|
});
|
|
1003
1264
|
if (writerNodeId) {
|
|
1004
1265
|
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1005
|
-
if (continuationCount <
|
|
1266
|
+
if (continuationCount < frontendRecoveryQuota) {
|
|
1006
1267
|
const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
|
|
1007
1268
|
const rollback = await rollbackFrontendWriter({
|
|
1008
1269
|
cwd: repoRoot,
|
|
1009
1270
|
parentRunId: state.runId,
|
|
1010
1271
|
});
|
|
1011
1272
|
if (rollback.ok) {
|
|
1012
|
-
|
|
1013
|
-
|
|
1273
|
+
recoveryTrigger = {
|
|
1274
|
+
requestId: state.runId,
|
|
1275
|
+
failureSource: "frontend-implement-pi",
|
|
1276
|
+
failureOwner: "implementation",
|
|
1277
|
+
protocolFailureReason: "frontend writer transient partial write",
|
|
1278
|
+
recoveryMode: "continuation",
|
|
1279
|
+
};
|
|
1014
1280
|
}
|
|
1015
1281
|
else {
|
|
1016
1282
|
state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
|
|
@@ -1020,6 +1286,64 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
|
1020
1286
|
}
|
|
1021
1287
|
}
|
|
1022
1288
|
}
|
|
1289
|
+
// Phase 6: verification failure → automatic recovery (re-implement +
|
|
1290
|
+
// re-verify). The verify shell is terminal by design so the failed run's
|
|
1291
|
+
// evidence stays clean; the M6 recovery operation re-runs implement and
|
|
1292
|
+
// verify in a fresh continuation, guarded by the same continuation quota,
|
|
1293
|
+
// budget fold and failure fingerprint as the other recovery paths.
|
|
1294
|
+
if (hasFrontendWriter && !recoveryTrigger && !skipFrontendRecovery) {
|
|
1295
|
+
const verifyFailed = state.nodes["frontend-verify-shell"]?.status === "ERROR";
|
|
1296
|
+
if (verifyFailed) {
|
|
1297
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1298
|
+
if (continuationCount < frontendRecoveryQuota) {
|
|
1299
|
+
recoveryTrigger = {
|
|
1300
|
+
requestId: state.runId,
|
|
1301
|
+
failureSource: "frontend-verify-shell",
|
|
1302
|
+
failureOwner: "implementation",
|
|
1303
|
+
protocolFailureReason: "frontend verification failure",
|
|
1304
|
+
recoveryMode: "continuation",
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
}
|
|
1309
|
+
// Phase 7: frontend review "request changes" → not passed. When the review
|
|
1310
|
+
// node FINISHED but committed `request_review_changes` (typed terminal
|
|
1311
|
+
// verdict), the implementation was reviewed and rejected. The run must not
|
|
1312
|
+
// surface as finished: terminal aggregation below stays partial_failed, a
|
|
1313
|
+
// terminal reason is recorded, the committed findings are rendered into a
|
|
1314
|
+
// downstream-task PRD draft (run-owned), and a recovery trigger carries the
|
|
1315
|
+
// issueCategory-derived owner so a later DAG round consumes the findings
|
|
1316
|
+
// instead of repeating the previous flow.
|
|
1317
|
+
const reviewRequestFact = await readCommittedReviewRequestFact({
|
|
1318
|
+
runDir,
|
|
1319
|
+
});
|
|
1320
|
+
if (reviewRequestFact) {
|
|
1321
|
+
state.terminalReason = "review-request-changes";
|
|
1322
|
+
try {
|
|
1323
|
+
await writeFrontendReviewFindingsPrd({
|
|
1324
|
+
runDir,
|
|
1325
|
+
parentRunId: state.runId,
|
|
1326
|
+
fact: reviewRequestFact,
|
|
1327
|
+
});
|
|
1328
|
+
}
|
|
1329
|
+
catch (error) {
|
|
1330
|
+
// The PRD draft is additive; a write failure must not mask the
|
|
1331
|
+
// not-passed verdict. The terminal aggregation below still applies.
|
|
1332
|
+
console.warn(`[run-dag] warning: failed to write frontend review findings PRD: ${error instanceof Error ? error.message : String(error)}`);
|
|
1333
|
+
}
|
|
1334
|
+
if (!recoveryTrigger && !skipFrontendRecovery) {
|
|
1335
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
1336
|
+
if (continuationCount < frontendRecoveryQuota) {
|
|
1337
|
+
recoveryTrigger = {
|
|
1338
|
+
requestId: state.runId,
|
|
1339
|
+
failureSource: "frontend-review-pi",
|
|
1340
|
+
failureOwner: classifyReviewIssueCategoryFailureOwner(reviewRequestFact.issueCategory),
|
|
1341
|
+
protocolFailureReason: `frontend review request_review_changes (issueCategory=${reviewRequestFact.issueCategory})`,
|
|
1342
|
+
recoveryMode: "new-dag",
|
|
1343
|
+
};
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1023
1347
|
const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
|
|
1024
1348
|
const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
|
|
1025
1349
|
if (state.convergence && supersededFailures.length > 0) {
|
|
@@ -1035,7 +1359,13 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
|
1035
1359
|
if (effectiveErrorCount === 0 &&
|
|
1036
1360
|
successTerminal &&
|
|
1037
1361
|
finishedCount + conditionSkippedCount + supersededErrorCount === taskCount) {
|
|
1038
|
-
|
|
1362
|
+
// Phase 7 guard: a committed `request_review_changes` means the review
|
|
1363
|
+
// rejected the implementation; the run must never surface as finished
|
|
1364
|
+
// even though every node technically FINISHED.
|
|
1365
|
+
state.status =
|
|
1366
|
+
state.terminalReason === "review-request-changes"
|
|
1367
|
+
? "partial_failed"
|
|
1368
|
+
: "finished";
|
|
1039
1369
|
}
|
|
1040
1370
|
else if (finishedCount > 0) {
|
|
1041
1371
|
state.status = "partial_failed";
|
|
@@ -1043,19 +1373,145 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
|
1043
1373
|
else {
|
|
1044
1374
|
state.status = "failed";
|
|
1045
1375
|
}
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1376
|
+
return recoveryTrigger
|
|
1377
|
+
? { recoveryPending: true, recovery: recoveryTrigger }
|
|
1378
|
+
: { recoveryPending: false };
|
|
1379
|
+
}
|
|
1380
|
+
/**
|
|
1381
|
+
* Construct (or reuse) the recovery operation for a failed frontend run.
|
|
1382
|
+
*
|
|
1383
|
+
* A parent-run atomic lease is acquired before `OperationStore.create`, then
|
|
1384
|
+
* its operation id is persisted into that lease before the controller may run.
|
|
1385
|
+
* `clientRequestId` remains a useful store-level replay key, but is not a
|
|
1386
|
+
* cross-process recovery admission boundary. The reset partition is frozen via
|
|
1387
|
+
* `computeFrontendRecoveryPlanForSource` and serialized under the run dir's
|
|
1388
|
+
* bounded `.runtime/` namespace for audit. The operation invokes
|
|
1389
|
+
* `dag rerun-task`, which regenerates a valid child DagSpec and carries typed
|
|
1390
|
+
* parent feedback through the managed rerun path. `planHash` is the second
|
|
1391
|
+
* idempotency dimension of `actionParams`.
|
|
1392
|
+
*/
|
|
1393
|
+
async function createRecoveryOperation(input) {
|
|
1394
|
+
const { cwd, runDir, spec, state, trigger } = input;
|
|
1395
|
+
const recoveryLease = await acquireDagRecoveryLease({
|
|
1396
|
+
cwd,
|
|
1397
|
+
parentRunId: state.runId,
|
|
1398
|
+
requestId: trigger.requestId,
|
|
1399
|
+
kind: "auto-frontend-recovery",
|
|
1400
|
+
});
|
|
1401
|
+
let effectiveLease = recoveryLease;
|
|
1402
|
+
let observeOnly = false;
|
|
1403
|
+
if (recoveryLease.kind === "held") {
|
|
1404
|
+
if (recoveryLease.lease.kind !== "auto-frontend-recovery" ||
|
|
1405
|
+
recoveryLease.lease.requestId !== trigger.requestId) {
|
|
1406
|
+
return { kind: "lease-held", lease: recoveryLease.lease };
|
|
1407
|
+
}
|
|
1408
|
+
const claimed = await claimDagRecoveryLease({
|
|
1409
|
+
leasePath: recoveryLease.leasePath,
|
|
1410
|
+
lease: recoveryLease.lease,
|
|
1411
|
+
});
|
|
1412
|
+
effectiveLease = claimed;
|
|
1413
|
+
if (claimed.kind === "held") {
|
|
1414
|
+
if (!claimed.lease.executionRef) {
|
|
1415
|
+
return { kind: "lease-held", lease: claimed.lease };
|
|
1416
|
+
}
|
|
1417
|
+
observeOnly = true;
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
const { computeFrontendRecoveryPlanForSource } = await import("./frontend-recovery-plan.js");
|
|
1421
|
+
const plan = computeFrontendRecoveryPlanForSource({
|
|
1422
|
+
spec,
|
|
1423
|
+
parentRunId: state.runId,
|
|
1424
|
+
requestId: trigger.requestId,
|
|
1425
|
+
failureSource: trigger.failureSource,
|
|
1426
|
+
});
|
|
1427
|
+
const planHash = createHash("sha256")
|
|
1428
|
+
.update(JSON.stringify(plan))
|
|
1429
|
+
.digest("hex");
|
|
1430
|
+
const continuationPath = path.join(runDir, ".runtime", "frontend-recovery", `${trigger.requestId}.json`);
|
|
1431
|
+
await writeJsonAtomic(continuationPath, {
|
|
1432
|
+
schemaVersion: 1,
|
|
1433
|
+
parentRunId: state.runId,
|
|
1434
|
+
requestId: trigger.requestId,
|
|
1435
|
+
failureSource: trigger.failureSource,
|
|
1436
|
+
recoveryMode: trigger.recoveryMode,
|
|
1437
|
+
planHash,
|
|
1438
|
+
plan,
|
|
1439
|
+
});
|
|
1440
|
+
const operationInput = {
|
|
1441
|
+
clientRequestId: `dag-recovery-${state.runId}`,
|
|
1442
|
+
action: "dagRunTask",
|
|
1443
|
+
actionParams: {
|
|
1444
|
+
parentRunId: state.runId,
|
|
1445
|
+
requestId: trigger.requestId,
|
|
1446
|
+
failureSource: trigger.failureSource,
|
|
1447
|
+
failureOwner: trigger.failureOwner,
|
|
1448
|
+
protocolFailureReason: trigger.protocolFailureReason,
|
|
1449
|
+
recoveryMode: trigger.recoveryMode,
|
|
1450
|
+
planHash,
|
|
1451
|
+
},
|
|
1452
|
+
cliArgs: [
|
|
1453
|
+
"dag",
|
|
1454
|
+
"rerun-task",
|
|
1455
|
+
"--run-id",
|
|
1456
|
+
state.runId,
|
|
1457
|
+
"--reason",
|
|
1458
|
+
trigger.protocolFailureReason ?? "frontend automatic recovery",
|
|
1459
|
+
"--request-id",
|
|
1460
|
+
trigger.requestId,
|
|
1461
|
+
"--frontend-recovery",
|
|
1462
|
+
"--json",
|
|
1463
|
+
],
|
|
1464
|
+
};
|
|
1465
|
+
let created;
|
|
1466
|
+
if (effectiveLease.lease.executionRef) {
|
|
1467
|
+
// Replay the exact operation bound by the durable lease. A missing record
|
|
1468
|
+
// cannot be replaced with a new operation id without violating the fence.
|
|
1469
|
+
const existing = await input.deps.store.get(effectiveLease.lease.executionRef);
|
|
1470
|
+
if (!existing) {
|
|
1471
|
+
throw new Error(`frontend recovery lease operation not found: ${effectiveLease.lease.executionRef}`);
|
|
1053
1472
|
}
|
|
1054
|
-
|
|
1055
|
-
|
|
1473
|
+
if (existing.clientRequestId !== operationInput.clientRequestId ||
|
|
1474
|
+
existing.action !== operationInput.action ||
|
|
1475
|
+
existing.payloadHash !==
|
|
1476
|
+
computeActionPayloadHash(operationInput.action, operationInput.actionParams)) {
|
|
1477
|
+
throw new Error("frontend recovery lease execution identity drift: referenced operation payload does not match the recovery request");
|
|
1056
1478
|
}
|
|
1479
|
+
created = { kind: "idempotent", operation: existing };
|
|
1480
|
+
}
|
|
1481
|
+
else {
|
|
1482
|
+
created = await input.deps.store.create(operationInput);
|
|
1483
|
+
}
|
|
1484
|
+
if (created.kind !== "conflict" && !observeOnly) {
|
|
1485
|
+
// The durable lease must name the operation before controller execution.
|
|
1486
|
+
// This also repairs a lease left behind by a crash before the original bind.
|
|
1487
|
+
await bindDagRecoveryLeaseExecution({
|
|
1488
|
+
leasePath: effectiveLease.leasePath,
|
|
1489
|
+
lease: effectiveLease.lease,
|
|
1490
|
+
executionRef: created.operation.operationId,
|
|
1491
|
+
});
|
|
1492
|
+
// Repair CLI args on both fresh and replayed operations. This closes the
|
|
1493
|
+
// crash window after operation creation but before args were persisted.
|
|
1494
|
+
await input.deps.store.update(created.operation.operationId, {
|
|
1495
|
+
cliArgs: [
|
|
1496
|
+
"dag",
|
|
1497
|
+
"rerun-task",
|
|
1498
|
+
"--run-id",
|
|
1499
|
+
state.runId,
|
|
1500
|
+
"--reason",
|
|
1501
|
+
trigger.protocolFailureReason ?? "frontend automatic recovery",
|
|
1502
|
+
"--request-id",
|
|
1503
|
+
trigger.requestId,
|
|
1504
|
+
"--frontend-recovery",
|
|
1505
|
+
"--recovery-operation-id",
|
|
1506
|
+
created.operation.operationId,
|
|
1507
|
+
"--json",
|
|
1508
|
+
],
|
|
1509
|
+
});
|
|
1510
|
+
}
|
|
1511
|
+
if (created.kind !== "conflict" && observeOnly) {
|
|
1512
|
+
return { kind: "observed", operation: created.operation };
|
|
1057
1513
|
}
|
|
1058
|
-
return
|
|
1514
|
+
return created;
|
|
1059
1515
|
}
|
|
1060
1516
|
function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
|
|
1061
1517
|
if (rankWriterNodeIds.length <= 1)
|