@tea-agent/loop-agent 0.42.0-next.9 → 0.42.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -45
- package/dist/application/dag/run-dag.js +8 -2
- package/dist/application/evaluation/budget.js +19 -1
- package/dist/application/task-lifecycle/advance.js +26 -8
- package/dist/application/task-lifecycle/observe.js +43 -29
- package/dist/application/task-lifecycle/plan-transitions.js +5 -4
- package/dist/build-stamp.json +3 -3
- package/dist/cli/program.js +1 -1
- package/dist/commands/dag-rerun-task.js +2 -0
- package/dist/commands/task-source-prepare.js +3 -1
- package/dist/executors/dag-pi-executor.js +2584 -600
- package/dist/executors/pi-executor.js +22 -1
- package/dist/executors/pi-extension-resolver.js +14 -2
- package/dist/executors/pi-sdk-executor.js +140 -39
- package/dist/executors/shell-executor.js +135 -55
- package/dist/shared/dag-failure-category.js +6 -0
- package/dist/shared/frontend-execution-policy.js +23 -0
- package/dist/task/config-types.js +4 -0
- package/dist/task/contract/apply.js +36 -2
- package/dist/task/source-prepare/ledger-reconciliation.js +2 -2
- package/dist/task/source-prepare/ledger-review.js +6 -9
- package/dist/task/source-prepare/parse-intent.js +7 -0
- package/dist/task/source-prepare/semantic-intake.js +16 -26
- package/dist/task/source-prepare/source-fidelity-pi.js +28 -9
- package/dist/task/source-references.js +48 -23
- package/dist/worker/console/chat/assistant-content.js +23 -2
- package/dist/worker/console/chat/browser-policy.js +143 -0
- package/dist/worker/console/chat/browser-routes.js +148 -0
- package/dist/worker/console/chat/chat-event-store.js +4 -2
- package/dist/worker/console/chat/explore-tools.js +13 -0
- package/dist/worker/console/chat/pi-runtime.js +173 -94
- package/dist/worker/console/chat/resource-loader.js +4 -1
- package/dist/worker/console/chat/routes.js +208 -66
- package/dist/worker/console/chat/scm-routes.js +217 -0
- package/dist/worker/console/chat/scm-service.js +283 -0
- package/dist/worker/console/chat/scm-tools.js +111 -0
- package/dist/worker/console/chat/sdd-data-alignment.js +78 -11
- package/dist/worker/console/chat/session-catalog.js +32 -0
- package/dist/worker/console/chat/session-mode-view.js +48 -0
- package/dist/worker/console/chat/session-mode.js +218 -0
- package/dist/worker/console/chat/session-store.js +27 -7
- package/dist/worker/console/chat/shortcuts.js +6 -0
- package/dist/worker/console/chat/subagents/agent-tool.js +62 -0
- package/dist/worker/console/chat/subagents/explore-agent.js +169 -0
- package/dist/worker/console/chat/subagents/index.js +5 -0
- package/dist/worker/console/chat/subagents/orchestrator.js +273 -0
- package/dist/worker/console/chat/subagents/tool-policy.js +53 -0
- package/dist/worker/console/chat/subagents/types.js +13 -0
- package/dist/worker/console/chat/terminal-routes.js +216 -0
- package/dist/worker/console/chat/terminal-sessions.js +284 -0
- package/dist/worker/console/chat/terminal-tools.js +199 -0
- package/dist/worker/console/chat/tool-preview.js +21 -0
- package/dist/worker/console/chat/tools.js +13 -1
- package/dist/worker/console/chat/turn-process.js +1 -0
- package/dist/worker/console/interview/tools.js +1 -0
- package/dist/worker/console/server.js +2 -28
- package/dist/worker/console/static/assets/{abnfDiagram-N423BO3Z-C6n9_m0P.js → abnfDiagram-N423BO3Z-C9eI7nEo.js} +1 -1
- package/dist/worker/console/static/assets/{arc-DQh-IfZ1.js → arc-VJWsxWhB.js} +1 -1
- package/dist/worker/console/static/assets/{architectureDiagram-T3A2C74G-54NnrwUC.js → architectureDiagram-T3A2C74G-BGcyiMSo.js} +1 -1
- package/dist/worker/console/static/assets/{blockDiagram-VBNYF7ZC-pivRALGK.js → blockDiagram-VBNYF7ZC-BScnFwyi.js} +1 -1
- package/dist/worker/console/static/assets/{c4Diagram-5PPSVZJV-BR7OV2NJ.js → c4Diagram-5PPSVZJV-Bl9_BsLi.js} +1 -1
- package/dist/worker/console/static/assets/channel-B6sQYuOE.js +1 -0
- package/dist/worker/console/static/assets/{chunk-2GRJ4B5K-B1Aq6BcK.js → chunk-2GRJ4B5K-F1uKxiUt.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-2Q5K7J3B-CbdU0rjo.js → chunk-2Q5K7J3B-rSRMynVu.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-5RXB4S5H-Dqi2GJeD.js → chunk-5RXB4S5H-DAsJm1WD.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-5VM5RSS4-BYn1Hu4R.js → chunk-5VM5RSS4-DQIdVI0a.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-6Q2QTUOP-DRLjDL0k.js → chunk-6Q2QTUOP-4A_8rJ-Q.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-GF5L2VYU-CY9Xx2jV.js → chunk-GF5L2VYU-BHnT-vjh.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-JWPE2WC7-BNCZs7_z.js → chunk-JWPE2WC7-Rq7QKBkn.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-KBJHAD2P-DZ8AStLL.js → chunk-KBJHAD2P-BXh-AI2u.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-RYQCIY6F-DsxjYrzz.js → chunk-RYQCIY6F-BblcLy8m.js} +1 -1
- package/dist/worker/console/static/assets/{chunk-XXDRQBXY-Ddx1KC1l.js → chunk-XXDRQBXY-zHZmVnB6.js} +1 -1
- package/dist/worker/console/static/assets/classDiagram-JCYQIIEL-Z6s5QoeI.js +1 -0
- package/dist/worker/console/static/assets/classDiagram-v2-OCEON4UE-Z6s5QoeI.js +1 -0
- package/dist/worker/console/static/assets/{cose-bilkent-JH36ORCC-W9TveCnK.js → cose-bilkent-JH36ORCC-Cmwz0Wi4.js} +1 -1
- package/dist/worker/console/static/assets/{cynefin-VYW2F7L2-l9j_ztZH.js → cynefin-VYW2F7L2-Dq76MHQU.js} +1 -1
- package/dist/worker/console/static/assets/{cynefinDiagram-MW4NZA55-BMJMKi4G.js → cynefinDiagram-MW4NZA55-zPahV1fM.js} +1 -1
- package/dist/worker/console/static/assets/{dagre-VZM6K2ZE-Bb4mG9pH.js → dagre-VZM6K2ZE-DQJ3xPc_.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-7IWD3JNH-BMgL_Qv0.js → diagram-7IWD3JNH-BwX7u6fv.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-B4RE2ZJO-Bvo2T4OQ.js → diagram-B4RE2ZJO-CodnDCco.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-LBJQPF4R-_5kWRN9c.js → diagram-LBJQPF4R-gAjtWkxG.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-Q27KOJAE-DqdMrltM.js → diagram-Q27KOJAE-CGCqz0cS.js} +1 -1
- package/dist/worker/console/static/assets/{diagram-UB23O5K3-CDDYsNkp.js → diagram-UB23O5K3-RHwYEywd.js} +1 -1
- package/dist/worker/console/static/assets/{ebnfDiagram-BXEA7PRR-c4nfnZUV.js → ebnfDiagram-BXEA7PRR-B6seb_p_.js} +1 -1
- package/dist/worker/console/static/assets/{erDiagram-JOGREHBK-DnkUcNMs.js → erDiagram-JOGREHBK-CmwCKYe8.js} +1 -1
- package/dist/worker/console/static/assets/{flowDiagram-UKHOOZJN-Dz0KGLZE.js → flowDiagram-UKHOOZJN-BfNsaZ0V.js} +1 -1
- package/dist/worker/console/static/assets/{ganttDiagram-PKOTCBZU-Cj1t1uka.js → ganttDiagram-PKOTCBZU-zmYgi_Z3.js} +1 -1
- package/dist/worker/console/static/assets/{gitGraphDiagram-DS77QQ5N-tp2FrBHd.js → gitGraphDiagram-DS77QQ5N-BDEzzSKF.js} +1 -1
- package/dist/worker/console/static/assets/index-DgenUAfc.js +468 -0
- package/dist/worker/console/static/assets/index-aF4u-Y__.css +1 -0
- package/dist/worker/console/static/assets/{infoDiagram-6WML65LV-DQwJS8DH.js → infoDiagram-6WML65LV-DeSz8l42.js} +1 -1
- package/dist/worker/console/static/assets/{ishikawaDiagram-WSZJBQD7-eimCQWD4.js → ishikawaDiagram-WSZJBQD7-B1LV29aH.js} +1 -1
- package/dist/worker/console/static/assets/{journeyDiagram-NVQOT4AX-Dd9Gbwgv.js → journeyDiagram-NVQOT4AX-CCw4W2oU.js} +1 -1
- package/dist/worker/console/static/assets/{kanban-definition-27J2QSJJ-Qve3cyft.js → kanban-definition-27J2QSJJ-BozaMoRJ.js} +1 -1
- package/dist/worker/console/static/assets/{linear-UXKSe36Z.js → linear-G1JR40SX.js} +1 -1
- package/dist/worker/console/static/assets/{mermaid.core-CWhj4JXN.js → mermaid.core-CCgPF0oA.js} +5 -5
- package/dist/worker/console/static/assets/{mindmap-definition-FAOFIHXS-yTcwf4SJ.js → mindmap-definition-FAOFIHXS-CEqnCHnD.js} +1 -1
- package/dist/worker/console/static/assets/{pegDiagram-VL7TDLO6-_7qToaZP.js → pegDiagram-VL7TDLO6-DpmQPKuI.js} +1 -1
- package/dist/worker/console/static/assets/{pieDiagram-7S7Q4E2Y-CXmrKmDm.js → pieDiagram-7S7Q4E2Y-BP501AxR.js} +1 -1
- package/dist/worker/console/static/assets/{quadrantDiagram-CIZ2JOQS-hMmrJyaS.js → quadrantDiagram-CIZ2JOQS-D1dBhl_A.js} +1 -1
- package/dist/worker/console/static/assets/{railroadDiagram-AXF67PYL-BXw8tCBe.js → railroadDiagram-AXF67PYL-DgZDS5pY.js} +1 -1
- package/dist/worker/console/static/assets/{requirementDiagram-LRYGKXZP-BmjgnLZl.js → requirementDiagram-LRYGKXZP-BzB3pyIq.js} +1 -1
- package/dist/worker/console/static/assets/{sankeyDiagram-W5VNT64P-Bj77SPpN.js → sankeyDiagram-W5VNT64P-B9ZprFFP.js} +1 -1
- package/dist/worker/console/static/assets/{sequenceDiagram-SI44F4Z6-B2FDVysL.js → sequenceDiagram-SI44F4Z6-CC2-_XrA.js} +1 -1
- package/dist/worker/console/static/assets/{sizeCapture-X5ZJPWSS-UChNY9ZZ.js → sizeCapture-X5ZJPWSS-UVtAUqgB.js} +1 -1
- package/dist/worker/console/static/assets/{stateDiagram-OKZ733FA-wbAEkbd7.js → stateDiagram-OKZ733FA-CWcS5JYG.js} +1 -1
- package/dist/worker/console/static/assets/stateDiagram-v2-UEYNNEHI-BooX8u1Q.js +1 -0
- package/dist/worker/console/static/assets/{swimlanes-SLNWSIFB-DCEkU8HR.js → swimlanes-SLNWSIFB-C4p-dn06.js} +2 -2
- package/dist/worker/console/static/assets/swimlanesDiagram-ULZ7WXOC-D64_eqqE.js +8 -0
- package/dist/worker/console/static/assets/{timeline-definition-Z64GVDOM-BGp7H06U.js → timeline-definition-Z64GVDOM-Dnz52OHX.js} +1 -1
- package/dist/worker/console/static/assets/{vennDiagram-T6HMQDX7-DsCIQuWR.js → vennDiagram-T6HMQDX7-CzTE0Nfu.js} +1 -1
- package/dist/worker/console/static/assets/{wardleyDiagram-T6FBY63Y-cRZLP4Xe.js → wardleyDiagram-T6FBY63Y-NAR9cjQQ.js} +1 -1
- package/dist/worker/console/static/assets/{xychartDiagram-ELKLHX3M-CP0xLR9Y.js → xychartDiagram-ELKLHX3M-DVMk8413.js} +1 -1
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/console/static-src/chat-view-types.js +1 -1
- package/dist/worker/console/static-src/operator-chat/chat-link.js +94 -0
- package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +142 -7
- package/dist/worker/console/static-src/operator-chat/pending-user-message.js +44 -0
- package/dist/worker/console/static-src/operator-chat/process-label.js +41 -0
- package/dist/worker/console/static-src/operator-chat/tools-catalog.js +21 -2
- package/dist/worker/console/static-src/operator-chat/turn-group-equality.js +13 -0
- package/dist/worker/console/static-src/operator-chat/turn-stream-controller.js +2 -0
- package/dist/worker/console/static-src/operator-chat/use-searchable-hidden.js +51 -0
- package/dist/worker/console/static-src/operator-chat/useChatSessions.js +33 -18
- package/dist/worker/console/static-src/operator-chat/useChatStream.js +48 -16
- package/dist/worker/console/static-src/operator-chat/useChatThread.js +170 -7
- package/dist/worker/console/static-src/operator-chat/useComposer.js +13 -7
- package/dist/worker/console/static-src/operator-chat/user-turn-anchor-equality.js +21 -0
- package/dist/worker/console/static-src/shell/workspace-route.js +4 -0
- package/dist/worker/console/workspace-context.js +15 -1
- package/dist/worker/observe/node-transparency.js +81 -72
- package/dist/worker/observe/routes.js +20 -1
- package/dist/worker/observe/static/constants.js +22 -22
- package/dist/worker/observe/static/dag-context-reason-labels.js +19 -0
- package/dist/worker/observe/static/dag-history-labels.js +1 -0
- package/dist/worker/observe/static/dag-inspector-humanize.d.ts +16 -0
- package/dist/worker/observe/static/dag-inspector-humanize.js +342 -0
- package/dist/worker/observe/static/dag-node-purpose.js +10 -10
- package/dist/worker/observe/static/dom.js +20 -1
- package/dist/worker/observe/static/format-pool.d.ts +2 -0
- package/dist/worker/observe/static/format-pool.js +6 -0
- package/dist/worker/observe/static/format.js +7 -0
- package/dist/worker/observe/static/index.html +4 -4
- package/dist/worker/observe/static/inspect-workspace.js +34 -7
- package/dist/worker/observe/static/inspector-submission.js +32 -0
- package/dist/worker/observe/static/kpi.js +1 -0
- package/dist/worker/observe/static/prompt-restart-candidates.js +4 -2
- package/dist/worker/observe/static/relations.js +7 -5
- package/dist/worker/observe/static/router.js +13 -0
- package/dist/worker/observe/static/run-processing.js +2 -0
- package/dist/worker/observe/static/shell-chrome.js +36 -3
- package/dist/worker/observe/static/state.js +35 -2
- package/dist/worker/observe/static/styles.css +431 -39
- package/dist/worker/observe/static/task-failure-labels.d.ts +4 -0
- package/dist/worker/observe/static/task-failure-labels.js +67 -0
- package/dist/worker/observe/static/task-history.js +12 -0
- package/dist/worker/observe/static/views/batch.js +6 -13
- package/dist/worker/observe/static/views/dag-graph.js +51 -4
- package/dist/worker/observe/static/views/dag-inspector.js +1069 -426
- package/dist/worker/observe/static/views/dag-trajectory.js +3 -0
- package/dist/worker/observe/static/views/dag.d.ts +6 -0
- package/dist/worker/observe/static/views/dag.js +64 -22
- package/dist/worker/observe/static/views/dags.js +2 -0
- package/dist/worker/observe/static/views/dashboard.js +21 -12
- package/dist/worker/observe/static/views/failures.js +21 -11
- package/dist/worker/observe/static/views/feature.js +11 -29
- package/dist/worker/observe/static/views/pool.js +37 -28
- package/dist/worker/observe/static/views/run.js +48 -5
- package/dist/worker/observe/static/views/session-timeline.js +194 -243
- package/dist/worker/observe/static/views/task.js +81 -62
- package/dist/workflows/dag/backend-test-plan-protocol.js +104 -0
- package/dist/workflows/dag/budget-enforcement.js +53 -3
- package/dist/workflows/dag/dag-retry-schema.js +3 -0
- package/dist/workflows/dag/frontend-capacity.js +9 -0
- package/dist/workflows/dag/frontend-contract-facts.js +130 -0
- package/dist/workflows/dag/frontend-design-policy.js +103 -17
- package/dist/workflows/dag/frontend-durable-tools.js +193 -0
- package/dist/workflows/dag/frontend-execution-groups.js +24 -0
- package/dist/workflows/dag/frontend-implementation-contract.js +423 -41
- package/dist/workflows/dag/frontend-input-projection.js +76 -0
- package/dist/workflows/dag/frontend-plan-completeness.js +186 -0
- package/dist/workflows/dag/frontend-plan-recovery-policy.js +18 -0
- package/dist/workflows/dag/frontend-plan-render.js +21 -3
- package/dist/workflows/dag/frontend-prewrite-gate.js +1 -1
- package/dist/workflows/dag/frontend-recovery-controller.js +32 -8
- package/dist/workflows/dag/frontend-recovery-lineage.js +13 -0
- package/dist/workflows/dag/frontend-recovery-plan.js +4 -1
- package/dist/workflows/dag/frontend-recovery-run.js +146 -16
- package/dist/workflows/dag/frontend-review-scopes.js +139 -0
- package/dist/workflows/dag/frontend-risk.js +2 -0
- package/dist/workflows/dag/frontend-session-budget.js +249 -0
- package/dist/workflows/dag/frontend-shadow-dual-write.js +37 -3
- package/dist/workflows/dag/frontend-shape-facts.js +12 -2
- package/dist/workflows/dag/frontend-shape.js +46 -7
- package/dist/workflows/dag/frontend-test-execution-evidence.js +49 -0
- package/dist/workflows/dag/frontend-typed-event-store.js +14 -0
- package/dist/workflows/dag/frontend-verification-trace.js +38 -4
- package/dist/workflows/dag/frontend-writer-admission.js +2 -2
- package/dist/workflows/dag/init-hybrid.js +70 -35
- package/dist/workflows/dag/node-execution.js +225 -226
- package/dist/workflows/dag/prompt.js +4 -0
- package/dist/workflows/dag/recovery-lease.js +106 -16
- package/dist/workflows/dag/rerun-feedback.js +1 -0
- package/dist/workflows/dag/rerun-plan.js +96 -4
- package/dist/workflows/dag/rerun-run.js +28 -6
- package/dist/workflows/dag/rerun-task.js +221 -18
- package/dist/workflows/dag/retry-policy.js +27 -10
- package/dist/workflows/dag/runner-exit-diagnostics.js +125 -0
- package/dist/workflows/dag/runner.js +246 -19
- package/dist/workflows/dag/structured-output-repair.js +4 -1
- package/dist/workflows/dag/types.js +10 -3
- package/dist/workflows/dag/validate.js +10 -8
- package/dist/workflows/dag/workspace-checkpoint.js +66 -0
- package/docs/operations/README.md +2 -0
- package/docs/templates/agent-dag.schema.json +2 -2
- package/docs/templates/backend-test-dag.json +6 -4
- package/docs/templates/frontend-design-contract.md +4 -4
- package/docs/templates/frontend-implementation-contract.schema.json +68 -4
- package/docs/templates/frontend-implementation-dag.json +5 -5
- package/harness.json +1 -1
- package/package.json +8 -6
- package/skills/frontend-contract/SKILL.md +2 -1
- package/skills/frontend-contract/references/contract-protocol.md +19 -3
- package/skills/frontend-design-review/SKILL.md +12 -11
- package/skills/frontend-plan/SKILL.md +22 -2
- package/skills/frontend-plan/references/decision-contract.md +114 -5
- package/skills/frontend-plan/references/design-decisions.md +32 -0
- package/skills/frontend-review/SKILL.md +10 -11
- package/skills/frontend-scout/references/scout-evidence.md +4 -0
- package/dist/worker/console/static/assets/channel-DsZxrgqe.js +0 -1
- package/dist/worker/console/static/assets/classDiagram-JCYQIIEL-CuToPeGV.js +0 -1
- package/dist/worker/console/static/assets/classDiagram-v2-OCEON4UE-CuToPeGV.js +0 -1
- package/dist/worker/console/static/assets/index-D9Sc0f0y.js +0 -449
- package/dist/worker/console/static/assets/index-DDQc5a50.css +0 -1
- package/dist/worker/console/static/assets/stateDiagram-v2-UEYNNEHI-yPoT19ft.js +0 -1
- package/dist/worker/console/static/assets/swimlanesDiagram-ULZ7WXOC-VWdarchG.js +0 -8
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { access, readFile } from "node:fs/promises";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { generateTaskDagUseCase } from "../../application/dag/generate-task-dag.js";
|
|
5
|
+
import { buildDefaultFrontendRecoveryDeps, } from "../../application/dag/run-dag.js";
|
|
6
|
+
import { runDagContinuation, } from "./runner.js";
|
|
7
|
+
import { stageRecoveryChild } from "./frontend-recovery-run.js";
|
|
4
8
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
5
9
|
import { DAG_RUNS_DIR, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, } from "./lifecycle.js";
|
|
6
10
|
import { deriveDagRerunFeedback } from "./rerun-feedback.js";
|
|
@@ -9,7 +13,7 @@ import { parseDagSpec, } from "./types.js";
|
|
|
9
13
|
export const RERUN_TASK_LINEAGE_REL_PATH = ".runtime/rerun-task-lineage.json";
|
|
10
14
|
export function parseDagRerunTaskArgs(args) {
|
|
11
15
|
if (args.length === 0) {
|
|
12
|
-
throw new Error("usage: dag rerun-task --run-id <id> --reason <text> --request-id <key> [--profile auto|minimal|standard|reviewed|supervised] [--task-id <id>] [--json]");
|
|
16
|
+
throw new Error("usage: dag rerun-task --run-id <id> --reason <text> --request-id <key> [--profile auto|minimal|standard|reviewed|supervised] [--task-id <id>] [--frontend-recovery --recovery-operation-id <id>] [--json]");
|
|
13
17
|
}
|
|
14
18
|
let parentRunId;
|
|
15
19
|
let reason;
|
|
@@ -18,6 +22,8 @@ export function parseDagRerunTaskArgs(args) {
|
|
|
18
22
|
let profileExplicit = false;
|
|
19
23
|
let taskIdOverride;
|
|
20
24
|
let execute = true;
|
|
25
|
+
let frontendRecovery = false;
|
|
26
|
+
let recoveryOperationId;
|
|
21
27
|
for (let index = 0; index < args.length; index += 1) {
|
|
22
28
|
const arg = args[index];
|
|
23
29
|
if (arg === "--run-id")
|
|
@@ -48,6 +54,12 @@ export function parseDagRerunTaskArgs(args) {
|
|
|
48
54
|
continue;
|
|
49
55
|
else if (arg === "--no-execute")
|
|
50
56
|
execute = false;
|
|
57
|
+
else if (arg === "--frontend-recovery")
|
|
58
|
+
frontendRecovery = true;
|
|
59
|
+
else if (arg === "--recovery-operation-id")
|
|
60
|
+
recoveryOperationId = args[++index];
|
|
61
|
+
else if (arg.startsWith("--recovery-operation-id="))
|
|
62
|
+
recoveryOperationId = arg.slice("--recovery-operation-id=".length);
|
|
51
63
|
else if (arg.startsWith("-"))
|
|
52
64
|
throw new Error(`unknown dag rerun-task flag: ${arg}`);
|
|
53
65
|
else
|
|
@@ -70,6 +82,48 @@ export function parseDagRerunTaskArgs(args) {
|
|
|
70
82
|
profileExplicit,
|
|
71
83
|
...(taskIdOverride?.trim() ? { taskIdOverride: taskIdOverride.trim() } : {}),
|
|
72
84
|
execute,
|
|
85
|
+
frontendRecovery,
|
|
86
|
+
...(recoveryOperationId ? { recoveryOperationId } : {}),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
const frontendRecoveryDescriptorPath = (runDir, requestId) => path.join(runDir, ".runtime", "frontend-recovery", `${requestId}.json`);
|
|
90
|
+
async function readFrontendRecoveryDescriptor(runDir, parentRunId, requestId) {
|
|
91
|
+
const raw = JSON.parse(await readFile(frontendRecoveryDescriptorPath(runDir, requestId), "utf8"));
|
|
92
|
+
const plan = raw.plan;
|
|
93
|
+
if (raw.schemaVersion !== 1 ||
|
|
94
|
+
raw.parentRunId !== parentRunId ||
|
|
95
|
+
raw.requestId !== requestId ||
|
|
96
|
+
typeof raw.failureSource !== "string" ||
|
|
97
|
+
typeof raw.recoveryMode !== "string" ||
|
|
98
|
+
typeof raw.planHash !== "string" ||
|
|
99
|
+
!plan ||
|
|
100
|
+
plan.schemaVersion !== 1 ||
|
|
101
|
+
plan.parentRunId !== parentRunId ||
|
|
102
|
+
plan.requestId !== requestId ||
|
|
103
|
+
!Array.isArray(plan.resetNodeIds) ||
|
|
104
|
+
!Array.isArray(plan.importedNodeIds)) {
|
|
105
|
+
throw new Error("frontend recovery descriptor is invalid or unbound");
|
|
106
|
+
}
|
|
107
|
+
const expectedHash = createHash("sha256")
|
|
108
|
+
.update(JSON.stringify(plan))
|
|
109
|
+
.digest("hex");
|
|
110
|
+
if (raw.planHash !== expectedHash) {
|
|
111
|
+
throw new Error("frontend recovery descriptor plan hash mismatch");
|
|
112
|
+
}
|
|
113
|
+
return raw;
|
|
114
|
+
}
|
|
115
|
+
function buildFrontendRecoveryIntent(input) {
|
|
116
|
+
return {
|
|
117
|
+
schemaVersion: 1,
|
|
118
|
+
phase: "child-staging",
|
|
119
|
+
requestId: input.requestId,
|
|
120
|
+
recoveryRootRunId: input.parentRecovery?.recoveryRootRunId ?? input.parentRunId,
|
|
121
|
+
parentRunId: input.parentRunId,
|
|
122
|
+
attemptId: input.requestId,
|
|
123
|
+
attemptIndex: input.parentRecovery?.attemptIndex ?? 0,
|
|
124
|
+
continuationCount: input.parentRecovery?.continuationCount ?? 0,
|
|
125
|
+
failureSource: input.failureSource,
|
|
126
|
+
revision: 0,
|
|
73
127
|
};
|
|
74
128
|
}
|
|
75
129
|
function parseDagRerunTaskProfile(value) {
|
|
@@ -136,6 +190,21 @@ function resolveTaskIdForRerun(spec, taskIdOverride) {
|
|
|
136
190
|
function operatorRequestPath(cwd, requestId) {
|
|
137
191
|
return path.join(cwd, DAG_RUNS_DIR, ".operator-requests", `${requestId}.json`);
|
|
138
192
|
}
|
|
193
|
+
function computeRerunTaskRequestHash(input) {
|
|
194
|
+
return createHash("sha256")
|
|
195
|
+
.update(JSON.stringify({
|
|
196
|
+
parentRunId: input.parentRunId,
|
|
197
|
+
reason: input.reason,
|
|
198
|
+
requestId: input.requestId,
|
|
199
|
+
profile: input.profile ?? "auto",
|
|
200
|
+
profileExplicit: input.profileExplicit ?? false,
|
|
201
|
+
taskIdOverride: input.taskIdOverride ?? null,
|
|
202
|
+
execute: input.execute ?? true,
|
|
203
|
+
frontendRecovery: input.frontendRecovery ?? false,
|
|
204
|
+
recoveryOperationId: input.recoveryOperationId ?? null,
|
|
205
|
+
}))
|
|
206
|
+
.digest("hex");
|
|
207
|
+
}
|
|
139
208
|
async function fileExists(filePath) {
|
|
140
209
|
try {
|
|
141
210
|
await access(filePath);
|
|
@@ -185,23 +254,45 @@ export async function assessGeneratedOutputConflict(spec, cwd) {
|
|
|
185
254
|
}
|
|
186
255
|
return conflictPaths;
|
|
187
256
|
}
|
|
188
|
-
async function readExistingOperatorRequest(cwd, requestId) {
|
|
257
|
+
async function readExistingOperatorRequest(cwd, requestId, expected) {
|
|
258
|
+
let raw;
|
|
189
259
|
try {
|
|
190
|
-
|
|
191
|
-
if (raw.result && typeof raw.result === "object") {
|
|
192
|
-
return { ...raw.result, idempotentReplay: true };
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
catch {
|
|
196
|
-
// not found or unreadable
|
|
260
|
+
raw = JSON.parse(await readFile(operatorRequestPath(cwd, requestId), "utf-8"));
|
|
197
261
|
}
|
|
198
|
-
|
|
262
|
+
catch (error) {
|
|
263
|
+
if (error &&
|
|
264
|
+
typeof error === "object" &&
|
|
265
|
+
"code" in error &&
|
|
266
|
+
error.code === "ENOENT") {
|
|
267
|
+
return undefined;
|
|
268
|
+
}
|
|
269
|
+
throw new Error(`invalid persisted rerun-task operator request: ${error instanceof Error ? error.message : String(error)}`);
|
|
270
|
+
}
|
|
271
|
+
if (raw.schemaVersion !== 1 ||
|
|
272
|
+
raw.kind !== "standalone-task-rerun-request" ||
|
|
273
|
+
raw.requestId !== requestId) {
|
|
274
|
+
throw new Error("REQUEST_ID_REUSE_CONFLICT: operator request id belongs to a different request kind");
|
|
275
|
+
}
|
|
276
|
+
if (raw.parentRunId !== expected.parentRunId ||
|
|
277
|
+
raw.requestHash !== expected.requestHash) {
|
|
278
|
+
throw new Error("REQUEST_ID_REUSE_CONFLICT: rerun-task operator request payload drifted");
|
|
279
|
+
}
|
|
280
|
+
const result = raw.result;
|
|
281
|
+
if (!result ||
|
|
282
|
+
typeof result !== "object" ||
|
|
283
|
+
Array.isArray(result) ||
|
|
284
|
+
result.parentRunId !== expected.parentRunId ||
|
|
285
|
+
result.requestId !== requestId) {
|
|
286
|
+
throw new Error("invalid persisted rerun-task operator request result");
|
|
287
|
+
}
|
|
288
|
+
return { ...result, idempotentReplay: true };
|
|
199
289
|
}
|
|
200
|
-
async function persistOperatorRequest(cwd, result) {
|
|
290
|
+
async function persistOperatorRequest(cwd, result, requestHash) {
|
|
201
291
|
await writeJsonAtomic(operatorRequestPath(cwd, result.requestId), {
|
|
202
292
|
schemaVersion: 1,
|
|
203
293
|
kind: "standalone-task-rerun-request",
|
|
204
294
|
parentRunId: result.parentRunId,
|
|
295
|
+
requestHash,
|
|
205
296
|
newRunId: result.newRunId,
|
|
206
297
|
taskId: result.taskId,
|
|
207
298
|
reason: result.reason,
|
|
@@ -264,8 +355,106 @@ function extractNewRunId(generateResult) {
|
|
|
264
355
|
const runSummary = extractRunSummary(generateResult);
|
|
265
356
|
return runSummary?.runId;
|
|
266
357
|
}
|
|
358
|
+
async function executeFrontendRecoveryContinuation(input) {
|
|
359
|
+
const located = await locateDagRun(input.cwd, input.parentRunId);
|
|
360
|
+
if (!located || located.lifecycle !== "active") {
|
|
361
|
+
throw new Error(`frontend recovery parent run must remain active: ${input.parentRunId}`);
|
|
362
|
+
}
|
|
363
|
+
const lease = await acquireDagRecoveryLease({
|
|
364
|
+
cwd: input.cwd,
|
|
365
|
+
parentRunId: input.parentRunId,
|
|
366
|
+
requestId: input.requestId,
|
|
367
|
+
kind: "standalone-task-rerun",
|
|
368
|
+
});
|
|
369
|
+
if (lease.kind !== "held" ||
|
|
370
|
+
lease.lease.kind !== "auto-frontend-recovery" ||
|
|
371
|
+
lease.lease.requestId !== input.requestId ||
|
|
372
|
+
!input.recoveryOperationId ||
|
|
373
|
+
lease.lease.executionRef !== input.recoveryOperationId) {
|
|
374
|
+
return blockedResult({
|
|
375
|
+
parentRunId: input.parentRunId,
|
|
376
|
+
reason: input.reason,
|
|
377
|
+
requestId: input.requestId,
|
|
378
|
+
blockedReason: "recovery-lease-held",
|
|
379
|
+
note: "frontend recovery operation is not the owner of the parent recovery lease",
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
const descriptor = await readFrontendRecoveryDescriptor(located.runDir, input.parentRunId, input.requestId);
|
|
383
|
+
const parentState = await readDagRunState(located.runDir);
|
|
384
|
+
const staged = await stageRecoveryChild({
|
|
385
|
+
cwd: input.cwd,
|
|
386
|
+
parentRunId: input.parentRunId,
|
|
387
|
+
recoveryIntent: buildFrontendRecoveryIntent({
|
|
388
|
+
parentRunId: input.parentRunId,
|
|
389
|
+
requestId: input.requestId,
|
|
390
|
+
failureSource: descriptor.failureSource,
|
|
391
|
+
parentRecovery: parentState.frontendRecoveryState,
|
|
392
|
+
}),
|
|
393
|
+
recoveryPlan: descriptor.plan,
|
|
394
|
+
});
|
|
395
|
+
const locatedChild = await locateDagRun(input.cwd, staged.childRunId);
|
|
396
|
+
if (!locatedChild || locatedChild.runDir !== staged.childRunDir) {
|
|
397
|
+
throw new Error(`frontend recovery child lifecycle drift: ${staged.childRunId}`);
|
|
398
|
+
}
|
|
399
|
+
const childSpec = await readDagRunSpec(locatedChild.runDir);
|
|
400
|
+
const childState = await readDagRunState(locatedChild.runDir);
|
|
401
|
+
let runSummary;
|
|
402
|
+
if (locatedChild.lifecycle === "completed" ||
|
|
403
|
+
locatedChild.lifecycle === "paused") {
|
|
404
|
+
runSummary = {
|
|
405
|
+
title: childState.title,
|
|
406
|
+
runId: childState.runId,
|
|
407
|
+
status: childState.status,
|
|
408
|
+
ranks: childState.ranks,
|
|
409
|
+
nodes: childState.nodes,
|
|
410
|
+
runDir: locatedChild.runDir,
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
else {
|
|
414
|
+
const childRecovery = childSpec.tasks.some((task) => task.id === "frontend-implement-pi")
|
|
415
|
+
? input.recovery ?? await buildDefaultFrontendRecoveryDeps(input.cwd)
|
|
416
|
+
: undefined;
|
|
417
|
+
runSummary = await runDagContinuation({
|
|
418
|
+
cwd: input.cwd,
|
|
419
|
+
spec: childSpec,
|
|
420
|
+
state: childState,
|
|
421
|
+
runDir: locatedChild.runDir,
|
|
422
|
+
...(childRecovery ? { recovery: childRecovery } : {}),
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
const settledChild = await locateDagRun(input.cwd, staged.childRunId);
|
|
426
|
+
const settledChildState = settledChild
|
|
427
|
+
? await readDagRunState(settledChild.runDir)
|
|
428
|
+
: childState;
|
|
429
|
+
const feedback = summarizeFeedback(childSpec.rerunFeedback, "no-unresolved-verify-review-feedback");
|
|
430
|
+
const result = {
|
|
431
|
+
ok: runSummary.status === "finished" ||
|
|
432
|
+
settledChildState.frontendRecoveryResult?.outcome === "recovered",
|
|
433
|
+
parentRunId: input.parentRunId,
|
|
434
|
+
newRunId: staged.childRunId,
|
|
435
|
+
taskId: childSpec.taskContractBinding?.taskId ??
|
|
436
|
+
childSpec.sourceBinding?.taskId,
|
|
437
|
+
reason: input.reason,
|
|
438
|
+
requestId: input.requestId,
|
|
439
|
+
mode: "generate+validate+execute",
|
|
440
|
+
feedback,
|
|
441
|
+
runSummary,
|
|
442
|
+
};
|
|
443
|
+
await writeRerunTaskLineage(input.cwd, staged.childRunId, {
|
|
444
|
+
parentRunId: input.parentRunId,
|
|
445
|
+
reason: input.reason,
|
|
446
|
+
requestId: input.requestId,
|
|
447
|
+
feedback,
|
|
448
|
+
});
|
|
449
|
+
await persistOperatorRequest(input.cwd, result, input.requestHash);
|
|
450
|
+
return result;
|
|
451
|
+
}
|
|
267
452
|
export async function rerunStandaloneTask(input) {
|
|
268
|
-
const
|
|
453
|
+
const requestHash = computeRerunTaskRequestHash(input);
|
|
454
|
+
const existing = await readExistingOperatorRequest(input.cwd, input.requestId, {
|
|
455
|
+
parentRunId: input.parentRunId,
|
|
456
|
+
requestHash,
|
|
457
|
+
});
|
|
269
458
|
if (existing) {
|
|
270
459
|
return existing;
|
|
271
460
|
}
|
|
@@ -298,6 +487,17 @@ export async function rerunStandaloneTask(input) {
|
|
|
298
487
|
note: eligibility.note,
|
|
299
488
|
});
|
|
300
489
|
}
|
|
490
|
+
if (input.frontendRecovery) {
|
|
491
|
+
return executeFrontendRecoveryContinuation({
|
|
492
|
+
cwd: input.cwd,
|
|
493
|
+
parentRunId: input.parentRunId,
|
|
494
|
+
reason: input.reason,
|
|
495
|
+
requestId: input.requestId,
|
|
496
|
+
requestHash,
|
|
497
|
+
recoveryOperationId: input.recoveryOperationId,
|
|
498
|
+
recovery: input.recovery,
|
|
499
|
+
});
|
|
500
|
+
}
|
|
301
501
|
let parentFeedback;
|
|
302
502
|
let missingFeedbackReason = "no-unresolved-verify-review-feedback";
|
|
303
503
|
try {
|
|
@@ -318,6 +518,9 @@ export async function rerunStandaloneTask(input) {
|
|
|
318
518
|
// eligible full rerun.
|
|
319
519
|
missingFeedbackReason = "feedback-unavailable";
|
|
320
520
|
}
|
|
521
|
+
if (state.terminalReason === "design-request-changes" && !parentFeedback) {
|
|
522
|
+
throw new Error("frontend design recovery feedback unavailable: no typed findings were found");
|
|
523
|
+
}
|
|
321
524
|
const execute = input.execute ?? true;
|
|
322
525
|
let effectiveFeedback;
|
|
323
526
|
let expectedRerunFeedbackDigest;
|
|
@@ -381,10 +584,7 @@ export async function rerunStandaloneTask(input) {
|
|
|
381
584
|
kind: "standalone-task-rerun",
|
|
382
585
|
})
|
|
383
586
|
: undefined;
|
|
384
|
-
|
|
385
|
-
recoveryLease.lease.kind === "auto-frontend-recovery" &&
|
|
386
|
-
recoveryLease.lease.requestId === input.requestId;
|
|
387
|
-
if (recoveryLease?.kind === "held" && !nestedAutoRecoveryLease) {
|
|
587
|
+
if (recoveryLease?.kind === "held") {
|
|
388
588
|
return blockedResult({
|
|
389
589
|
parentRunId: input.parentRunId,
|
|
390
590
|
reason: input.reason,
|
|
@@ -455,10 +655,13 @@ export async function rerunStandaloneTask(input) {
|
|
|
455
655
|
...(runSummary ? { runSummary } : {}),
|
|
456
656
|
};
|
|
457
657
|
if (await fileExists(operatorRequestPath(input.cwd, input.requestId))) {
|
|
458
|
-
const replay = await readExistingOperatorRequest(input.cwd, input.requestId
|
|
658
|
+
const replay = await readExistingOperatorRequest(input.cwd, input.requestId, {
|
|
659
|
+
parentRunId: input.parentRunId,
|
|
660
|
+
requestHash,
|
|
661
|
+
});
|
|
459
662
|
if (replay)
|
|
460
663
|
return replay;
|
|
461
664
|
}
|
|
462
|
-
await persistOperatorRequest(input.cwd, result);
|
|
665
|
+
await persistOperatorRequest(input.cwd, result, requestHash);
|
|
463
666
|
return result;
|
|
464
667
|
}
|
|
@@ -7,8 +7,8 @@ import { TYPED_EVENT_FACT_KINDS } from "./frontend-typed-event-store.js";
|
|
|
7
7
|
* is re-exported here unchanged so retry-policy stays the single import
|
|
8
8
|
* surface for retry policies and helpers.
|
|
9
9
|
*/
|
|
10
|
-
export { ALL_DAG_RETRY_CATEGORIES, CONTEXT_OVERFLOW_RETRY_CATEGORY, DEFAULT_DAG_RETRY_CATEGORIES, INCOMPLETE_WRITE_SET_RETRY_CATEGORY, PROTOCOL_AWARE_DAG_RETRY_CATEGORIES, PROTOCOL_INVALID_RETRY_CATEGORY, READ_BURST_RETRY_CATEGORY, REVIEW_TERMINAL_MISSING_RETRY_CATEGORY, STRUCTURED_ARTIFACT_INVALID_RETRY_CATEGORY, STRUCTURED_OUTPUT_RETRY_CATEGORY, STRUCTURED_OUTPUT_TRUNCATED_RETRY_CATEGORY, STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES, TARGET_TEMPLATE_TRANSIENT_RETRY_PROFILE, WRITER_CLEAN_TIMEOUT_RETRY_CATEGORY, WRITER_EMPTY_DIFF_RETRY_CATEGORY, dagRetryBackoffSchema, dagRetryCategorySchema, dagRetryPolicySchema, } from "./dag-retry-schema.js";
|
|
11
|
-
import { CONTEXT_OVERFLOW_RETRY_CATEGORY, DEFAULT_DAG_RETRY_CATEGORIES, INCOMPLETE_WRITE_SET_RETRY_CATEGORY, PROTOCOL_AWARE_DAG_RETRY_CATEGORIES, READ_BURST_RETRY_CATEGORY, REVIEW_TERMINAL_MISSING_RETRY_CATEGORY, STRUCTURED_ARTIFACT_INVALID_RETRY_CATEGORY, STRUCTURED_OUTPUT_RETRY_CATEGORY, STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES, TARGET_TEMPLATE_TRANSIENT_RETRY_PROFILE, WRITER_CLEAN_TIMEOUT_RETRY_CATEGORY, WRITER_EMPTY_DIFF_RETRY_CATEGORY, } from "./dag-retry-schema.js";
|
|
10
|
+
export { ALL_DAG_RETRY_CATEGORIES, CONTEXT_OVERFLOW_RETRY_CATEGORY, DEFAULT_DAG_RETRY_CATEGORIES, INCOMPLETE_WRITE_SET_RETRY_CATEGORY, OUTPUT_LIMIT_RETRY_CATEGORY, PROTOCOL_AWARE_DAG_RETRY_CATEGORIES, PROTOCOL_INVALID_RETRY_CATEGORY, READ_BURST_RETRY_CATEGORY, REVIEW_TERMINAL_MISSING_RETRY_CATEGORY, STRUCTURED_ARTIFACT_INVALID_RETRY_CATEGORY, STRUCTURED_OUTPUT_RETRY_CATEGORY, STRUCTURED_OUTPUT_TRUNCATED_RETRY_CATEGORY, STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES, TARGET_TEMPLATE_TRANSIENT_RETRY_PROFILE, WRITER_CLEAN_TIMEOUT_RETRY_CATEGORY, WRITER_EMPTY_DIFF_RETRY_CATEGORY, dagRetryBackoffSchema, dagRetryCategorySchema, dagRetryPolicySchema, } from "./dag-retry-schema.js";
|
|
11
|
+
import { CONTEXT_OVERFLOW_RETRY_CATEGORY, DEFAULT_DAG_RETRY_CATEGORIES, INCOMPLETE_WRITE_SET_RETRY_CATEGORY, OUTPUT_LIMIT_RETRY_CATEGORY, PROTOCOL_AWARE_DAG_RETRY_CATEGORIES, READ_BURST_RETRY_CATEGORY, REVIEW_TERMINAL_MISSING_RETRY_CATEGORY, STRUCTURED_ARTIFACT_INVALID_RETRY_CATEGORY, STRUCTURED_OUTPUT_RETRY_CATEGORY, STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES, TARGET_TEMPLATE_TRANSIENT_RETRY_PROFILE, WRITER_CLEAN_TIMEOUT_RETRY_CATEGORY, WRITER_EMPTY_DIFF_RETRY_CATEGORY, } from "./dag-retry-schema.js";
|
|
12
12
|
const RETRY_SAFE_PI_ROLES = new Set([
|
|
13
13
|
"planner",
|
|
14
14
|
"scout",
|
|
@@ -19,7 +19,7 @@ const RETRY_SAFE_PI_ROLES = new Set([
|
|
|
19
19
|
]);
|
|
20
20
|
/**
|
|
21
21
|
* The default retry policy applied to safe generated read-only Pi nodes.
|
|
22
|
-
* Total attempts:
|
|
22
|
+
* Total attempts: 5, exponential backoff with cap.
|
|
23
23
|
*
|
|
24
24
|
* Includes `context-overflow`: a read-only node that blew the context window
|
|
25
25
|
* (400 request too large, e.g. a review node accumulating too many reads) is a
|
|
@@ -29,7 +29,7 @@ const RETRY_SAFE_PI_ROLES = new Set([
|
|
|
29
29
|
* design-review/review fell straight to ERROR without a retry attempt.
|
|
30
30
|
*/
|
|
31
31
|
export const DEFAULT_READ_ONLY_PI_RETRY_POLICY = {
|
|
32
|
-
maxAttempts:
|
|
32
|
+
maxAttempts: 5,
|
|
33
33
|
backoff: "exponential",
|
|
34
34
|
initialDelayMs: 2000,
|
|
35
35
|
maxDelayMs: 30000,
|
|
@@ -80,6 +80,17 @@ export const PLANNER_OUTPUT_LIMIT_RETRY_POLICY = {
|
|
|
80
80
|
CONTEXT_OVERFLOW_RETRY_CATEGORY,
|
|
81
81
|
],
|
|
82
82
|
};
|
|
83
|
+
/** Backend-test N2 emits a mandatory run-owned Markdown protocol, so both
|
|
84
|
+
* provider length truncation and deterministic protocol validation failures
|
|
85
|
+
* are safe producer-local retries. */
|
|
86
|
+
export const BACKEND_TEST_MD_PLAN_RETRY_POLICY = {
|
|
87
|
+
...STRUCTURED_REQUIRED_PI_RETRY_POLICY,
|
|
88
|
+
retryCategories: [
|
|
89
|
+
...STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES,
|
|
90
|
+
READ_BURST_RETRY_CATEGORY,
|
|
91
|
+
CONTEXT_OVERFLOW_RETRY_CATEGORY,
|
|
92
|
+
],
|
|
93
|
+
};
|
|
83
94
|
/**
|
|
84
95
|
* Default retry for reviewer / recovery nodes that declare outputProtocol.
|
|
85
96
|
* Includes protocol-invalid so missing VERDICT lines are corrected in-node.
|
|
@@ -114,7 +125,10 @@ export const WRITER_EMPTY_DIFF_RETRY_POLICY = {
|
|
|
114
125
|
backoff: "exponential",
|
|
115
126
|
initialDelayMs: 2000,
|
|
116
127
|
maxDelayMs: 30000,
|
|
117
|
-
retryCategories: [
|
|
128
|
+
retryCategories: [
|
|
129
|
+
WRITER_EMPTY_DIFF_RETRY_CATEGORY,
|
|
130
|
+
OUTPUT_LIMIT_RETRY_CATEGORY,
|
|
131
|
+
],
|
|
118
132
|
};
|
|
119
133
|
/**
|
|
120
134
|
* Bounded transport retry for standard exclusive implementers when a provider
|
|
@@ -129,6 +143,7 @@ export const WRITER_TRANSPORT_RETRY_POLICY = {
|
|
|
129
143
|
retryCategories: [
|
|
130
144
|
WRITER_CLEAN_TIMEOUT_RETRY_CATEGORY,
|
|
131
145
|
CONTEXT_OVERFLOW_RETRY_CATEGORY,
|
|
146
|
+
OUTPUT_LIMIT_RETRY_CATEGORY,
|
|
132
147
|
],
|
|
133
148
|
};
|
|
134
149
|
/**
|
|
@@ -164,6 +179,7 @@ export const TARGET_TEMPLATE_WRITER_TRANSPORT_RETRY_POLICY = {
|
|
|
164
179
|
retryCategories: [
|
|
165
180
|
WRITER_CLEAN_TIMEOUT_RETRY_CATEGORY,
|
|
166
181
|
CONTEXT_OVERFLOW_RETRY_CATEGORY,
|
|
182
|
+
OUTPUT_LIMIT_RETRY_CATEGORY,
|
|
167
183
|
],
|
|
168
184
|
};
|
|
169
185
|
/**
|
|
@@ -171,13 +187,14 @@ export const TARGET_TEMPLATE_WRITER_TRANSPORT_RETRY_POLICY = {
|
|
|
171
187
|
* recovery attempts driven by Completeness Gate (missing/broken target files).
|
|
172
188
|
*/
|
|
173
189
|
export const BACKEND_TEST_WRITER_COMPLETENESS_RETRY_POLICY = {
|
|
174
|
-
maxAttempts:
|
|
190
|
+
maxAttempts: 5,
|
|
175
191
|
backoff: "exponential",
|
|
176
192
|
initialDelayMs: 2000,
|
|
177
193
|
maxDelayMs: 30000,
|
|
178
194
|
retryCategories: [
|
|
179
195
|
WRITER_EMPTY_DIFF_RETRY_CATEGORY,
|
|
180
196
|
INCOMPLETE_WRITE_SET_RETRY_CATEGORY,
|
|
197
|
+
OUTPUT_LIMIT_RETRY_CATEGORY,
|
|
181
198
|
],
|
|
182
199
|
};
|
|
183
200
|
/** Markdown shard writers use one full attempt plus at most one bounded binding repair. */
|
|
@@ -461,9 +478,9 @@ function reasonMandatedFrontendPlanRetryStep(reason) {
|
|
|
461
478
|
}
|
|
462
479
|
}
|
|
463
480
|
/**
|
|
464
|
-
* Opt-in retry policy mounted on the frontend-plan-pi node. maxAttempts=
|
|
465
|
-
*
|
|
466
|
-
*
|
|
481
|
+
* Opt-in retry policy mounted on the frontend-plan-pi node. maxAttempts=5
|
|
482
|
+
* lets the producer validate and repair each generated increment while still
|
|
483
|
+
* bounding the four-rung ladder (normal → bounded-tool-only →
|
|
467
484
|
* compact-terminal-first → backup-model). `invalid-output` (typed-fact schema
|
|
468
485
|
* violations, e.g. verification targets referencing undeclared UI states)
|
|
469
486
|
* retries at the `normal` rung with the contractCheck reason injected via
|
|
@@ -474,7 +491,7 @@ export const FRONTEND_PLAN_LADDER_RETRY_POLICY = {
|
|
|
474
491
|
// incremental record_* calls; dense call bursts hit provider rate limits
|
|
475
492
|
// whose windows exceed the old 30s cap. Longer backoff gives the limit
|
|
476
493
|
// window time to expire before the next attempt.
|
|
477
|
-
maxAttempts:
|
|
494
|
+
maxAttempts: 5,
|
|
478
495
|
backoff: "exponential",
|
|
479
496
|
initialDelayMs: 5000,
|
|
480
497
|
maxDelayMs: 60000,
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync, unlinkSync } from "node:fs";
|
|
2
|
+
import { hostname } from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
/**
|
|
5
|
+
* Runner exit-time diagnostics (silent-death forensics).
|
|
6
|
+
*
|
|
7
|
+
* A DAG runner can disappear without a crash report and without stderr (five
|
|
8
|
+
* reproductions during frontend recovery/long-plan runs, 2026-09-06): when a
|
|
9
|
+
* supervisor hard-kills the process or the event loop is wedged, no JS runs,
|
|
10
|
+
* so nothing is persisted and the run is only discoverable later as an
|
|
11
|
+
* orphaned RUNNING state with a frozen heartbeat. That leaves no evidence of
|
|
12
|
+
* WHERE the runner was when it died or whether the exit was even observable
|
|
13
|
+
* by Node.
|
|
14
|
+
*
|
|
15
|
+
* This module gives every abnormal death a durable trace next to state.json:
|
|
16
|
+
* - an `armed` line is written when the checkpoint starts executing;
|
|
17
|
+
* - a `process-exit` line is appended from the process `exit` hook with the
|
|
18
|
+
* exit code and a live state snapshot (status, terminalReason, RUNNING
|
|
19
|
+
* nodes, heartbeat staleness);
|
|
20
|
+
* - `uncaught-exception` / `unhandled-rejection` lines record the error
|
|
21
|
+
* before the process is allowed to crash exactly as it would have
|
|
22
|
+
* via uncaughtExceptionMonitor without changing the host's crash policy.
|
|
23
|
+
*
|
|
24
|
+
* On a clean terminal return the runner calls stop(), which removes every
|
|
25
|
+
* listener and deletes the journal, so completed runs carry no noise. A file
|
|
26
|
+
* left behind therefore means: diagnostics were armed, then the process went
|
|
27
|
+
* away through a path Node could not observe (`armed` line only → external
|
|
28
|
+
* SIGKILL/OOM/event-loop wedge) or through an observed abnormal exit (`armed`
|
|
29
|
+
* + one or more event lines → the reason is recorded in the file).
|
|
30
|
+
*
|
|
31
|
+
* Journal writes are strictly synchronous and best-effort: an append failure
|
|
32
|
+
* must never alter the runner's own crash/exit behavior.
|
|
33
|
+
*/
|
|
34
|
+
export const RUNNER_EXIT_DIAGNOSTICS_FILE = "runner-exit-diagnostics.jsonl";
|
|
35
|
+
function describeError(error) {
|
|
36
|
+
if (error instanceof Error) {
|
|
37
|
+
return { name: error.name, message: error.message };
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
return { message: String(error) };
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return { message: "<unstringifiable error>" };
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
function appendJournalLineSync(journalPath, record) {
|
|
47
|
+
try {
|
|
48
|
+
mkdirSync(path.dirname(journalPath), { recursive: true });
|
|
49
|
+
appendFileSync(journalPath, `${JSON.stringify(record)}\n`, "utf8");
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
// Best-effort: journaling must never crash or alter exit behavior.
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
function removeJournalSync(journalPath) {
|
|
56
|
+
try {
|
|
57
|
+
unlinkSync(journalPath);
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// Tolerate a missing journal (e.g. nothing was ever written).
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export function installRunnerExitDiagnostics(options) {
|
|
64
|
+
const journalPath = path.join(options.runDir, RUNNER_EXIT_DIAGNOSTICS_FILE);
|
|
65
|
+
const target = options.target ?? process;
|
|
66
|
+
const armedAt = Date.now();
|
|
67
|
+
let stopped = false;
|
|
68
|
+
const base = {
|
|
69
|
+
runId: options.runId,
|
|
70
|
+
title: options.title,
|
|
71
|
+
pid: process.pid,
|
|
72
|
+
hostname: hostname(),
|
|
73
|
+
};
|
|
74
|
+
const safeSnapshot = () => {
|
|
75
|
+
try {
|
|
76
|
+
return options.snapshot();
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return { status: "<snapshot-error>", running: [] };
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
const record = (kind, extra) => ({
|
|
83
|
+
ts: new Date().toISOString(),
|
|
84
|
+
kind,
|
|
85
|
+
...base,
|
|
86
|
+
...(extra ?? {}),
|
|
87
|
+
state: safeSnapshot(),
|
|
88
|
+
});
|
|
89
|
+
appendJournalLineSync(journalPath, {
|
|
90
|
+
...record("armed"),
|
|
91
|
+
armedAt: new Date(armedAt).toISOString(),
|
|
92
|
+
});
|
|
93
|
+
const onProcessExit = (code) => {
|
|
94
|
+
if (stopped)
|
|
95
|
+
return;
|
|
96
|
+
appendJournalLineSync(journalPath, record("process-exit", {
|
|
97
|
+
code: typeof code === "number" ? code : process.exitCode ?? 0,
|
|
98
|
+
elapsedMs: Date.now() - armedAt,
|
|
99
|
+
}));
|
|
100
|
+
};
|
|
101
|
+
const onUncaughtException = (error, origin) => {
|
|
102
|
+
if (stopped)
|
|
103
|
+
return;
|
|
104
|
+
appendJournalLineSync(journalPath, record(origin === "unhandledRejection" ? "unhandled-rejection" : "uncaught-exception", {
|
|
105
|
+
error: describeError(error),
|
|
106
|
+
elapsedMs: Date.now() - armedAt,
|
|
107
|
+
}));
|
|
108
|
+
// Monitor listeners neither suppress fatal errors nor force a crash
|
|
109
|
+
// when the host has its own handlers or a nonfatal rejection policy.
|
|
110
|
+
};
|
|
111
|
+
target.on("exit", onProcessExit);
|
|
112
|
+
target.on("uncaughtExceptionMonitor", onUncaughtException);
|
|
113
|
+
return {
|
|
114
|
+
journalPath,
|
|
115
|
+
stop: () => {
|
|
116
|
+
if (stopped)
|
|
117
|
+
return;
|
|
118
|
+
stopped = true;
|
|
119
|
+
target.removeListener("exit", onProcessExit);
|
|
120
|
+
target.removeListener("uncaughtExceptionMonitor", onUncaughtException);
|
|
121
|
+
// Clean completion: leave completed runs free of diagnostics noise.
|
|
122
|
+
removeJournalSync(journalPath);
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|