@tea-agent/loop-agent 0.35.2 → 0.35.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/AGENTS.md +2 -0
  2. package/CHANGELOG.md +31 -0
  3. package/README.md +1 -1
  4. package/bin/loop-agent.js +37 -1
  5. package/dist/build-stamp.json +6 -0
  6. package/dist/cli/program.js +2 -2
  7. package/dist/executors/dag-pi-executor.js +44 -0
  8. package/dist/shared/package-metadata.js +42 -0
  9. package/dist/worker/console/chat/assistant-content.js +11 -0
  10. package/dist/worker/console/chat/pi-runtime.js +6 -2
  11. package/dist/worker/console/chat/turn-process.js +17 -9
  12. package/dist/worker/console/chat/workspace-landing.js +1 -1
  13. package/dist/worker/console/static/assets/index-DuVLjCIT.js +57 -0
  14. package/dist/worker/console/static/index.html +1 -1
  15. package/dist/worker/console/static-src/app/useRecoveryConsole.js +5 -0
  16. package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +15 -3
  17. package/dist/worker/console/static-src/operator-chat/refs.js +3 -0
  18. package/dist/worker/console/static-src/operator-chat/useChatSessions.js +3 -0
  19. package/dist/worker/console/static-src/operator-chat/useChatThread.js +1 -0
  20. package/dist/worker/loop-agent/loop-agent-client.js +17 -3
  21. package/dist/worker/observability/read-model.js +20 -0
  22. package/dist/worker/observe/spec-evidence.js +3 -8
  23. package/dist/worker/observe/static/views/dag-inspector.js +6 -71
  24. package/dist/worker/preflight.js +2 -1
  25. package/dist/workflows/dag/backend-test-scenario-param.js +33 -23
  26. package/dist/workflows/dag/contract-output-registry.js +14 -0
  27. package/dist/workflows/dag/contract-validator-registrations.js +8 -0
  28. package/dist/workflows/dag/dynamic-runtime/shared.js +9 -1
  29. package/dist/workflows/dag/frontend-implementation-contract.js +233 -39
  30. package/dist/workflows/dag/frontend-prewrite-gate.js +364 -61
  31. package/dist/workflows/dag/frontend-recovery-plan.js +73 -0
  32. package/dist/workflows/dag/frontend-recovery-root-manifest.js +123 -0
  33. package/dist/workflows/dag/frontend-recovery-run.js +539 -0
  34. package/dist/workflows/dag/frontend-repair.js +219 -18
  35. package/dist/workflows/dag/frontend-verification-trace.js +47 -32
  36. package/dist/workflows/dag/frontend-writer-recovery.js +106 -0
  37. package/dist/workflows/dag/frontend-writer-rollback.js +821 -0
  38. package/dist/workflows/dag/init-hybrid.js +49 -24
  39. package/dist/workflows/dag/node-execution.js +89 -0
  40. package/dist/workflows/dag/recovery-recommendation.js +58 -0
  41. package/dist/workflows/dag/runner.js +245 -11
  42. package/dist/workflows/dag/scheduler.js +257 -3
  43. package/dist/workflows/dag/types.js +130 -2
  44. package/package.json +4 -3
  45. package/dist/worker/console/static/assets/index-gVHrlqI9.js +0 -56
@@ -2199,6 +2199,26 @@ function pruneFrontendTasksForRisk(tasks, risk) {
2199
2199
  return { ...task, depends_on };
2200
2200
  });
2201
2201
  }
2202
+ /**
2203
+ * AC-1: shared frontend writer node defaults. frontend-implement-pi and
2204
+ * frontend-repair-pi share one prompt/contract/writeSet-guard surface and an
2205
+ * identical writeSet; only id, depends_on, runIf, outputContract, and
2206
+ * subtask_prompt differ per phase.
2207
+ */
2208
+ function buildFrontendWriterNodeDefaults(input) {
2209
+ return {
2210
+ role: "implementer",
2211
+ executor: "pi",
2212
+ toolProfile: "write",
2213
+ complexity: input.complexity,
2214
+ writePolicy: "exclusive",
2215
+ writeSet: input.writeSet,
2216
+ allowedPaths: input.allowedPaths,
2217
+ forbiddenPaths: input.forbiddenPaths,
2218
+ skills: FRONTEND_BOUNDED_IMPLEMENT_SKILLS,
2219
+ writerOutcomePolicy: { type: "implementation-outcome-v1" },
2220
+ };
2221
+ }
2202
2222
  async function buildFrontendHybridDagFromTask(sources) {
2203
2223
  const { taskConfig } = sources;
2204
2224
  const mockCapability = sources.frontendMockCapability ?? {
@@ -2339,6 +2359,12 @@ async function buildFrontendHybridDagFromTask(sources) {
2339
2359
  `Every requirement MUST have a non-empty expectedOutcome. Every interaction MUST have non-empty trigger and expectedBehavior. UI states with applicable=true MUST have non-empty expectedBehavior. Empty strings or omitted fields for these will cause contract rejection.`,
2340
2360
  ].join("\n")
2341
2361
  : "";
2362
+ const verificationTargetFileInstruction = [
2363
+ `## Verification target file semantics`,
2364
+ `verificationTargets[].file is the code file that the target verifies (the file the writer changes), NOT where the command is defined.`,
2365
+ `Non-static targets (type unit/component/integration/mock) MUST set file to a concrete code file inside the implementation writeSet (task allowedPaths); the prewrite gate rejects any non-static target whose file falls outside the writeSet.`,
2366
+ `Command-level checks that run project-wide (all tests, typecheck, build, governance) MUST use type "static" and must NOT be bound as non-static targets with file=package.json/tsconfig.json/vite.config.ts/scripts/*. Static targets are exempt from the writeSet containment check.`,
2367
+ ].join("\n");
2342
2368
  const strategy = resolveDagVerifyStrategy(taskConfig);
2343
2369
  const readOnlyPaths = taskConfig.allowedPaths.length > 0 ? taskConfig.allowedPaths : ["**"];
2344
2370
  const behaviorPaths = deriveFrontendBehaviorPaths(taskConfig);
@@ -2532,6 +2558,10 @@ async function buildFrontendHybridDagFromTask(sources) {
2532
2558
  writePolicy: "read-only",
2533
2559
  outputMode: "structured-required",
2534
2560
  retryPolicy: STRUCTURED_REQUIRED_PI_RETRY_POLICY,
2561
+ structuredContractOutput: {
2562
+ schemaId: "frontend-implementation-contract-v1",
2563
+ retryOnInvalid: true,
2564
+ },
2535
2565
  allowedPaths: readOnlyPaths,
2536
2566
  forbiddenPaths,
2537
2567
  skills: FRONTEND_IMPLEMENTATION_SKILLS,
@@ -2545,6 +2575,8 @@ async function buildFrontendHybridDagFromTask(sources) {
2545
2575
  "End with exactly one fenced json object conforming to frontend-implementation-contract-v1. This node is the single contract JSON producer: the fenced block is the authoritative contract the prewrite gate materializes. Do not emit any raw JSON, JSON in prose, or a second fenced block anywhere in the response; the plan text must not contain other balanced JSON objects.",
2546
2576
  "Each requirement must state its user-observable or logic-observable expectedOutcome. Each interaction must state its trigger and expectedBehavior. IDs plus file paths are not sufficient behavior semantics.",
2547
2577
  requirementCoverageInstruction,
2578
+ "verificationTargets[].commandLabel MUST be one of the frozen command labels listed above. Any other value will be rejected at contract materialization.",
2579
+ verificationTargetFileInstruction,
2548
2580
  "Read-only: do not modify code, docs, artifacts, or repository files.",
2549
2581
  fixedVerificationContext,
2550
2582
  sourceContext,
@@ -2584,6 +2616,10 @@ async function buildFrontendHybridDagFromTask(sources) {
2584
2616
  writePolicy: "read-only",
2585
2617
  outputMode: "structured-required",
2586
2618
  retryPolicy: STRUCTURED_REQUIRED_PI_RETRY_POLICY,
2619
+ structuredContractOutput: {
2620
+ schemaId: "frontend-implementation-contract-v1",
2621
+ retryOnInvalid: true,
2622
+ },
2587
2623
  allowedPaths: readOnlyPaths,
2588
2624
  forbiddenPaths,
2589
2625
  skills: FRONTEND_IMPLEMENTATION_SKILLS,
@@ -2598,6 +2634,7 @@ async function buildFrontendHybridDagFromTask(sources) {
2598
2634
  "End the response with exactly one fenced json object conforming to frontend-implementation-contract-v1. This node is the single contract JSON producer when the design review requests revision: the fenced block is the authoritative contract the prewrite gate materializes. Do not emit any raw JSON, JSON in prose, or a second fenced block anywhere in the response. Bind it to the supplied task sources; map every requirement and applicable UI state to concrete implementation and verification targets or an explicit blocking evidence gap. Do not include secrets or unsafe paths.",
2599
2635
  "Preserve each requirement expectedOutcome and each interaction trigger/expectedBehavior in the revised contract; do not reduce behavior semantics to IDs and paths.",
2600
2636
  "verificationTargets[].commandLabel MUST be one of the frozen command labels listed above. Any other value will be rejected at contract materialization.",
2637
+ verificationTargetFileInstruction,
2601
2638
  fixedVerificationContext,
2602
2639
  sourceContext,
2603
2640
  frontendContractSchemaBlock,
@@ -2716,18 +2753,12 @@ async function buildFrontendHybridDagFromTask(sources) {
2716
2753
  ? ["frontend-lint-baseline-shell"]
2717
2754
  : []),
2718
2755
  ],
2719
- role: "implementer",
2720
- executor: "pi",
2721
- toolProfile: "write",
2722
- complexity: resolveWriterComplexity(taskConfig),
2723
- writePolicy: "exclusive",
2724
- writeSet: implementPaths.writeSet,
2725
- allowedPaths: implementPaths.allowedPaths,
2726
- forbiddenPaths,
2727
- skills: FRONTEND_BOUNDED_IMPLEMENT_SKILLS,
2728
- writerOutcomePolicy: {
2729
- type: "implementation-outcome-v1",
2730
- },
2756
+ ...buildFrontendWriterNodeDefaults({
2757
+ complexity: resolveWriterComplexity(taskConfig),
2758
+ writeSet: implementPaths.writeSet,
2759
+ allowedPaths: implementPaths.allowedPaths,
2760
+ forbiddenPaths,
2761
+ }),
2731
2762
  outputContract: "First non-empty line must be exactly one of: IMPLEMENTATION_OUTCOME: changed; IMPLEMENTATION_OUTCOME: already-satisfied; IMPLEMENTATION_OUTCOME: blocked. Then a Markdown delivery summary with Contract Ref (path/schema/hash), Changed Files, Requirements Implemented, UI States, Tests Changed, Verification Attempts, Deviations, and Residual Risks. Follow fixed stages: contract confirm → tests → component/state → API/Mock → focused checks → diff cleanup.",
2732
2763
  subtask_prompt: [
2733
2764
  "Implement against the validated run-owned Frontend Implementation Contract from frontend-prewrite-gate-shell (path/schema/hash). Do not rebuild the contract from Markdown alone.",
@@ -2781,18 +2812,12 @@ async function buildFrontendHybridDagFromTask(sources) {
2781
2812
  id: "frontend-repair-pi",
2782
2813
  depends_on: ["frontend-verify-assess-shell", implementId],
2783
2814
  runIf: "$.nodes['frontend-verify-assess-shell'].json.eligible == true",
2784
- role: "implementer",
2785
- executor: "pi",
2786
- toolProfile: "write",
2787
- complexity: resolveWriterComplexity(taskConfig),
2788
- writePolicy: "exclusive",
2789
- writeSet: implementPaths.writeSet,
2790
- allowedPaths: implementPaths.allowedPaths,
2791
- forbiddenPaths,
2792
- skills: FRONTEND_BOUNDED_IMPLEMENT_SKILLS,
2793
- writerOutcomePolicy: {
2794
- type: "implementation-outcome-v1",
2795
- },
2815
+ ...buildFrontendWriterNodeDefaults({
2816
+ complexity: resolveWriterComplexity(taskConfig),
2817
+ writeSet: implementPaths.writeSet,
2818
+ allowedPaths: implementPaths.allowedPaths,
2819
+ forbiddenPaths,
2820
+ }),
2796
2821
  outputContract: "First non-empty line must be exactly one of: IMPLEMENTATION_OUTCOME: changed; IMPLEMENTATION_OUTCOME: already-satisfied; IMPLEMENTATION_OUTCOME: blocked. Then a repair summary for an eligible repairable assessment. Must not expand writeSet, re-interpret requirements, skip tests, or enable Mock by default.",
2797
2822
  subtask_prompt: [
2798
2823
  "Read contracts/frontend-repair-assessment.json and the validated frontend implementation contract.",
@@ -4,6 +4,7 @@ import { readFile } from "node:fs/promises";
4
4
  import path from "node:path";
5
5
  import { pathMatchesPattern } from "../../shared/git-progress.js";
6
6
  import { recordDecisionEnvelopeForNode, shouldPauseOnHumanEscalation, writeHumanEscalationArtifacts, } from "./decision-envelope.js";
7
+ import { FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isFrontendWriterAuthorized, readFrontendPrewriteResult, } from "./scheduler.js";
7
8
  import { writeNodeRecord, writeNodeSkillArtifacts } from "./run-store.js";
8
9
  import { resolveContextPolicy } from "./context-policy.js";
9
10
  import { buildDagNodePromptEnvelope, formatConvergenceFeedbackBlock, } from "./prompt.js";
@@ -12,6 +13,8 @@ import { buildOutputLimitRecoverySection, loadBackendTestWriterProgressForRetry,
12
13
  import { computeBackoffDelayMs, isRetryablePiFailureCategory, isSafeReadOnlyPiRetryCandidate, isWriterEmptyDiffRetryCandidate, isWriterTransportRetryCandidate, } from "./retry-policy.js";
13
14
  import { applyNodeActivity, evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
14
15
  import { buildProtocolRetryInstruction, normalizeReviewVerdictAfterRetries, parseJsonReviewVerdict, validateOutputProtocol, } from "./output-protocol.js";
16
+ import { getStructuredContractValidator } from "./contract-output-registry.js";
17
+ import "./contract-validator-registrations.js";
15
18
  import { writeDagNodeJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
16
19
  import { buildProjectGovernanceContext, readCompletedWriterChangeManifests, writeProjectGovernanceContext, } from "./project-governance-context.js";
17
20
  import { assertSkillSnapshotCoversSpec, buildNodePromptFromSnapshot, isDagSkillSnapshotIntegrityError, readSkillSnapshot, } from "./skill-snapshot.js";
@@ -138,6 +141,20 @@ function buildAttemptPrompt(task, basePrompt, attemptNumber, previousFailureCate
138
141
  buildProtocolRetryInstruction(task.outputProtocol, previousProtocolReason),
139
142
  ].join("\n");
140
143
  }
144
+ if (previousFailureCategory === "invalid-output" &&
145
+ task.structuredContractOutput &&
146
+ previousProtocolReason) {
147
+ return [
148
+ basePrompt,
149
+ "",
150
+ "<retry_instruction>",
151
+ "Previous attempt produced an invalid frontend implementation contract:",
152
+ previousProtocolReason,
153
+ "Return exactly one fenced json block conforming to the frontend-implementation-contract-v1 schema.",
154
+ "Fix every reported field violation: do not emit null for optional fields, do not misspell field names, and match the required types exactly.",
155
+ "</retry_instruction>",
156
+ ].join("\n");
157
+ }
141
158
  if (previousFailureCategory === "writer-empty-diff") {
142
159
  const maxAttempts = task.retryPolicy?.maxAttempts ?? 3;
143
160
  // When a completeness progress exists for this writer, fold the concrete
@@ -395,6 +412,21 @@ export async function executeDagNode(input) {
395
412
  await notifyNodeObserver(input.observer, "onNodeFinish", nodeId, state);
396
413
  };
397
414
  const failSkillSnapshot = (error) => failBeforePrompt(error, "skill-snapshot-integrity");
415
+ const skipFrontendWriter = async (admission) => {
416
+ const skippedAt = new Date().toISOString();
417
+ node.startedAt ??= skippedAt;
418
+ node.frontendWriterAdmission = admission;
419
+ node.status = "SKIPPED";
420
+ node.skippedReason = "frontend-prewrite-not-authorized";
421
+ node.finishedAt = skippedAt;
422
+ node.lastActivityAt = skippedAt;
423
+ node.durationMs = durationBetween(node.startedAt, node.finishedAt);
424
+ node.timing = { retryBackoffMs: 0, settlementCleanupMs: 0 };
425
+ state.nodes[nodeId].nodeRecordPath = path.join(runDir, `${nodeId}.json`);
426
+ await writeNodeRecord(runDir, nodeId, state.nodes[nodeId]);
427
+ await input.persistState();
428
+ await notifyNodeObserver(input.observer, "onNodeFinish", nodeId, state);
429
+ };
398
430
  if (task.finalWriteSetApproval) {
399
431
  const authorization = parseAndValidateFinalWriteSetApproval({ task, spec, state });
400
432
  if (!authorization.ok) {
@@ -423,6 +455,30 @@ export async function executeDagNode(input) {
423
455
  effectiveWriteSet: [...authorization.effectiveWriteSet],
424
456
  };
425
457
  }
458
+ if (FRONTEND_WRITER_NODE_IDS.includes(nodeId)) {
459
+ const admission = await readFrontendPrewriteResult(runDir);
460
+ if (!admission.ok) {
461
+ await skipFrontendWriter(undefined);
462
+ return;
463
+ }
464
+ const decision = isFrontendWriterAuthorized(admission.result);
465
+ const record = {
466
+ schemaVersion: 1,
467
+ writerNodeId: nodeId,
468
+ decision,
469
+ sourceArtifact: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
470
+ artifactHash: admission.artifactHash,
471
+ checkedAt: new Date().toISOString(),
472
+ reason: decision === "denied"
473
+ ? `classification: ${admission.result.classification}`
474
+ : null,
475
+ };
476
+ if (decision === "denied") {
477
+ await skipFrontendWriter(record);
478
+ return;
479
+ }
480
+ node.frontendWriterAdmission = record;
481
+ }
426
482
  let projectGovernanceContext;
427
483
  if (task.governanceStandardReview) {
428
484
  try {
@@ -689,6 +745,39 @@ export async function executeDagNode(input) {
689
745
  previousProtocolReason = undefined;
690
746
  }
691
747
  }
748
+ // R1: structured contract nodes self-validate their output so schema,
749
+ // typo, and null violations surface as retryable invalid-output at the
750
+ // producing node instead of failing the whole run at the prewrite gate.
751
+ if (result.ok && task.structuredContractOutput) {
752
+ const validator = getStructuredContractValidator(task.structuredContractOutput.schemaId);
753
+ if (validator) {
754
+ const contractText = canonicalNodeOutput(result);
755
+ const contractCheck = await validator({
756
+ runDir,
757
+ text: contractText,
758
+ sourceBinding: spec.sourceBinding,
759
+ });
760
+ if (!contractCheck.ok) {
761
+ result = {
762
+ ...result,
763
+ ok: false,
764
+ failureCategory: "invalid-output",
765
+ stderr: [result.stderr, contractCheck.reason]
766
+ .filter(Boolean)
767
+ .join("\n"),
768
+ };
769
+ previousProtocolReason = contractCheck.reason;
770
+ }
771
+ else {
772
+ previousProtocolReason = undefined;
773
+ }
774
+ }
775
+ else {
776
+ // DAG spec validation rejects unknown schemaIds; this is a
777
+ // defensive fallback for a registry that has not been populated.
778
+ console.warn(`[run-dag] warning: no structured contract validator registered for schemaId ${task.structuredContractOutput.schemaId}; skipping node self-check`);
779
+ }
780
+ }
692
781
  const attemptFinishedAt = new Date().toISOString();
693
782
  const attemptWallDurationMs = durationBetween(attemptStartedAt, attemptFinishedAt);
694
783
  totalAttemptWallDurationMs += attemptWallDurationMs;
@@ -269,3 +269,61 @@ export function planDagRecovery(input) {
269
269
  }
270
270
  return planByNormalizedCategory(input);
271
271
  }
272
+ /**
273
+ * Phase 6: route a frontend recovery outcome to a recovery CTA (read-only; never
274
+ * mutates a run's status). The six outcomes map to the existing recovery actions
275
+ * so the Console recovery page reuses the current CTA surface.
276
+ */
277
+ export function recommendFrontendRecoveryOutcome(outcome) {
278
+ switch (outcome) {
279
+ case "none":
280
+ return {
281
+ action: "none",
282
+ summary: "未发生前端恢复;按普通终态展示。",
283
+ reason: "该 frontend-implementation 根没有触发自动恢复。",
284
+ humanRequired: false,
285
+ autoRetryEligible: false,
286
+ };
287
+ case "recovered":
288
+ return {
289
+ action: "none",
290
+ summary: "自动恢复成功;child 已重新实现并通过验证。",
291
+ reason: "parent 的 transient 失败已由 child 恢复,全链按 root 计一次成功。",
292
+ humanRequired: false,
293
+ autoRetryEligible: false,
294
+ };
295
+ case "candidate-contract-invalid":
296
+ return {
297
+ action: "manual-review",
298
+ summary: "方案输出无效:candidate JSON 无法解析/推导,重试用尽。",
299
+ reason: "建议人工检查需求或重新生成方案后再跑。",
300
+ humanRequired: true,
301
+ autoRetryEligible: false,
302
+ };
303
+ case "prewrite-blocked":
304
+ return {
305
+ action: "manual-review",
306
+ summary: "治理门禁阻断:source stale / writeSet 越界 / Mock policy / contract 违规。",
307
+ reason: "需人工修正任务边界/事实后重跑,不自动重试。",
308
+ humanRequired: true,
309
+ autoRetryEligible: false,
310
+ };
311
+ case "repair-exhausted":
312
+ return {
313
+ action: "rerun-after-fix",
314
+ summary: "自动修复未通过:verify 失败,1 次 repair 后 reverify 仍失败。",
315
+ reason: "建议 `dag rerun` 或人工排查。",
316
+ humanRequired: true,
317
+ autoRetryEligible: false,
318
+ commandHint: "loop-agent dag rerun --run-id <runId>",
319
+ };
320
+ case "auto-recovery-blocked":
321
+ return {
322
+ action: "manual-review",
323
+ summary: "自动恢复被阻止:rollback 无法证明完整或存在外部/越界修改。",
324
+ reason: "必须人工介入,禁止自动覆盖用户文件。",
325
+ humanRequired: true,
326
+ autoRetryEligible: false,
327
+ };
328
+ }
329
+ }
@@ -1,4 +1,5 @@
1
1
  import { readdir, readFile } from "node:fs/promises";
2
+ import { randomUUID } from "node:crypto";
2
3
  import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
3
4
  import { hostname } from "node:os";
4
5
  import path from "node:path";
@@ -6,7 +7,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
6
7
  import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
7
8
  import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
8
9
  import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
9
- import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
10
+ import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
10
11
  import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
11
12
  import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
12
13
  import { createDagNodeExecutor } from "./executor-registry.js";
@@ -21,8 +22,9 @@ import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSna
21
22
  import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
22
23
  import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
23
24
  import { runConvergencePassController, } from "./convergence/controller.js";
24
- import { executeDagRanksOnce, isConditionSkippedReason } from "./scheduler.js";
25
+ import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
25
26
  import { topoSortToRanks } from "./topo.js";
27
+ import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
26
28
  import { parseDagSpec, resolveModelForTask, } from "./types.js";
27
29
  import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
28
30
  import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
@@ -448,6 +450,39 @@ async function executeDagCheckpoint(input) {
448
450
  void persistState().catch(() => { });
449
451
  }, runnerLivenessPolicy.heartbeatIntervalMs);
450
452
  heartbeatTimer.unref();
453
+ // Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
454
+ // timeout, supervision layer, or shell wall-clock) must not leave the run
455
+ // orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
456
+ // state with an explicit terminalReason before exiting so downstream
457
+ // liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
458
+ let terminalSignal;
459
+ const shutdownOnSignal = (signal) => {
460
+ if (terminalSignal)
461
+ return;
462
+ terminalSignal = signal;
463
+ clearInterval(heartbeatTimer);
464
+ if (state.runner)
465
+ state.runner.heartbeatAt = new Date().toISOString();
466
+ state.status = "failed";
467
+ state.finishedAt = new Date().toISOString();
468
+ state.terminalReason = `runner terminated by ${signal} before run completion`;
469
+ console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
470
+ const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
471
+ // Best-effort atomic persist with a hard exit deadline so a stuck write
472
+ // queue cannot keep the process alive past the supervisor's kill window.
473
+ const hardExit = setTimeout(exit, 5000);
474
+ hardExit.unref();
475
+ void persistState()
476
+ .catch(() => { })
477
+ .finally(() => {
478
+ clearTimeout(hardExit);
479
+ exit();
480
+ });
481
+ };
482
+ const onSigTerm = () => shutdownOnSignal("SIGTERM");
483
+ const onSigInt = () => shutdownOnSignal("SIGINT");
484
+ process.once("SIGTERM", onSigTerm);
485
+ process.once("SIGINT", onSigInt);
451
486
  try {
452
487
  const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
453
488
  const baseExecuteNode = input.executeNode ??
@@ -504,6 +539,7 @@ async function executeDagCheckpoint(input) {
504
539
  tasksById,
505
540
  maxConcurrent,
506
541
  persistState,
542
+ runDir,
507
543
  abortSignal: input.abortSignal,
508
544
  createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
509
545
  baseExecuteNode,
@@ -595,17 +631,54 @@ async function executeDagCheckpoint(input) {
595
631
  runDir = await moveToPausedRunDir(runDir, pausedRunDir);
596
632
  }
597
633
  else {
598
- finalizeTerminalRunStatus(state, spec.tasks.length);
634
+ const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
599
635
  await persistState();
600
- await notifyRunObserver(input.observer, "onRunFinish", state);
601
- try {
602
- const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
603
- await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
636
+ if (recoveryPending) {
637
+ // Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
638
+ // child and run it synchronously. Dynamic import avoids a static
639
+ // runner frontend-recovery-run module cycle. stageRecoveryChild
640
+ // locates the parent via the canonical active/ layout, so recovery
641
+ // only fires when this run is still the canonical active run dir
642
+ // (direct runDagContinuation callers using an arbitrary runDir keep
643
+ // the previous scheduler-layer skip semantics).
644
+ const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
645
+ if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
646
+ const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
647
+ const staged = await stageRecoveryChild({
648
+ cwd,
649
+ parentRunId: state.runId,
650
+ });
651
+ // stageRecoveryChild CAS-advanced the parent's on-disk lineage
652
+ // (child-staging → child-running + childRunId). Refresh the in-memory
653
+ // copy so the terminal persistState below does not regress the parent
654
+ // back to stale child-staging.
655
+ const parentOnDisk = await readDagRunState(runDir);
656
+ state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
657
+ const childState = await readDagRunState(staged.childRunDir);
658
+ const childSpec = await readDagRunSpec(staged.childRunDir);
659
+ await runDagContinuation({
660
+ cwd,
661
+ spec: childSpec,
662
+ state: childState,
663
+ runDir: staged.childRunDir,
664
+ maxConcurrent,
665
+ executeNode: input.executeNode,
666
+ observer: input.observer,
667
+ abortSignal: input.abortSignal,
668
+ });
669
+ }
604
670
  }
605
- catch (error) {
606
- console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
671
+ await notifyRunObserver(input.observer, "onRunFinish", state);
672
+ if (!recoveryPending) {
673
+ try {
674
+ const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
675
+ await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
676
+ }
677
+ catch (error) {
678
+ console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
679
+ }
680
+ runDir = await moveToCompletedRunDir(runDir, completedRunDir);
607
681
  }
608
- runDir = await moveToCompletedRunDir(runDir, completedRunDir);
609
682
  }
610
683
  await relocateRunArtifactPaths({
611
684
  runDir,
@@ -628,6 +701,8 @@ async function executeDagCheckpoint(input) {
628
701
  }
629
702
  finally {
630
703
  clearInterval(heartbeatTimer);
704
+ process.removeListener("SIGTERM", onSigTerm);
705
+ process.removeListener("SIGINT", onSigInt);
631
706
  }
632
707
  }
633
708
  async function notifyRunObserver(observer, event, state) {
@@ -695,7 +770,153 @@ function isSuccessfulConvergenceTerminal(state) {
695
770
  return true;
696
771
  return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
697
772
  }
698
- function finalizeTerminalRunStatus(state, taskCount) {
773
+ function buildFrontendRecoveryIntent(state, failureSource) {
774
+ const existing = state.frontendRecoveryState;
775
+ const requestId = existing?.requestId ?? randomUUID();
776
+ return {
777
+ schemaVersion: 1,
778
+ phase: "child-staging",
779
+ requestId,
780
+ recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
781
+ parentRunId: state.runId,
782
+ ...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
783
+ attemptId: existing?.attemptId ?? `recovery-${requestId}`,
784
+ attemptIndex: existing?.attemptIndex ?? 0,
785
+ continuationCount: existing?.continuationCount ?? 0,
786
+ ...(failureSource ? { failureSource } : {}),
787
+ revision: (existing?.revision ?? 0) + 1,
788
+ };
789
+ }
790
+ function buildCandidateContractInvalidResult(state, result, artifactHash) {
791
+ const recovery = state.frontendRecoveryState;
792
+ return {
793
+ schemaVersion: 1,
794
+ outcome: "candidate-contract-invalid",
795
+ origin: {
796
+ kind: "frontend-prewrite-gate",
797
+ parentRunId: recovery?.parentRunId ?? state.runId,
798
+ ...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
799
+ requestId: recovery?.requestId ?? "",
800
+ failedNodeId: result.selectedPlanNodeId,
801
+ },
802
+ failureClass: {
803
+ code: "candidate-contract-invalid",
804
+ classification: result.classification,
805
+ reason: result.failureReason ?? "candidate contract invalid",
806
+ },
807
+ evidenceRefs: [
808
+ {
809
+ runId: state.runId,
810
+ relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
811
+ sha256: artifactHash,
812
+ },
813
+ ],
814
+ };
815
+ }
816
+ function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
817
+ const recovery = state.frontendRecoveryState;
818
+ return {
819
+ schemaVersion: 1,
820
+ outcome: "auto-recovery-blocked",
821
+ origin: {
822
+ kind: "frontend-writer",
823
+ parentRunId: recovery?.parentRunId ?? state.runId,
824
+ ...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
825
+ requestId: recovery?.requestId ?? "",
826
+ failedNodeId,
827
+ },
828
+ failureClass: {
829
+ code: "writer-transient-partial-write",
830
+ classification: "transient",
831
+ reason,
832
+ },
833
+ evidenceRefs: [],
834
+ };
835
+ }
836
+ function settleFrontendRecovery(state, outcome) {
837
+ const recovery = state.frontendRecoveryState;
838
+ if (!recovery)
839
+ return;
840
+ recovery.phase = "settled";
841
+ if (!state.frontendRecoveryResult) {
842
+ state.frontendRecoveryResult = {
843
+ schemaVersion: 1,
844
+ outcome,
845
+ origin: {
846
+ kind: "frontend-prewrite-gate",
847
+ parentRunId: recovery.parentRunId,
848
+ ...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
849
+ requestId: recovery.requestId,
850
+ failedNodeId: "",
851
+ },
852
+ evidenceRefs: [],
853
+ };
854
+ }
855
+ }
856
+ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
857
+ const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
858
+ // Terminal override: a denied frontend writer must never surface as a
859
+ // mechanical partial_failed just because prewrite FINISHED while the writer
860
+ // was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
861
+ let recoveryPending = false;
862
+ const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
863
+ if (hasFrontendWriter) {
864
+ const admission = await readFrontendPrewriteResult(runDir);
865
+ if (admission.ok) {
866
+ if (admission.result.classification === "blocked") {
867
+ state.status = "failed";
868
+ return { recoveryPending: false };
869
+ }
870
+ if (admission.result.classification === "retryable-invalid") {
871
+ const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
872
+ if (continuationCount >= 1) {
873
+ // Continuation quota exhausted: a second retryable-invalid candidate
874
+ // is authoritative candidate-contract-invalid → forced failed.
875
+ state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
876
+ state.status = "failed";
877
+ return { recoveryPending: false };
878
+ }
879
+ // Root continuation still available: record the recovery intent and let
880
+ // mechanical aggregation settle partial_failed/failed. The parent keeps
881
+ // phase != settled (recovery in progress) and no final root manifest.
882
+ state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
883
+ recoveryPending = true;
884
+ }
885
+ }
886
+ }
887
+ // Phase 5: writer transient partial write → rollback + recovery intent. Only
888
+ // for frontend-implementation writers with a remaining root continuation, and
889
+ // only when the rollback journal (captured before the provider call) restores
890
+ // cleanly; otherwise the run stays failed with auto-recovery-blocked.
891
+ if (hasFrontendWriter && !recoveryPending) {
892
+ const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
893
+ const node = state.nodes[id];
894
+ return (node &&
895
+ node.status === "ERROR" &&
896
+ isFrontendWriterTransientPartialWrite({
897
+ failureCategory: node.failureCategory,
898
+ }));
899
+ });
900
+ if (writerNodeId) {
901
+ const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
902
+ if (continuationCount < 1) {
903
+ const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
904
+ const rollback = await rollbackFrontendWriter({
905
+ cwd: repoRoot,
906
+ parentRunId: state.runId,
907
+ });
908
+ if (rollback.ok) {
909
+ state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
910
+ recoveryPending = true;
911
+ }
912
+ else {
913
+ state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
914
+ state.status = "failed";
915
+ return { recoveryPending: false };
916
+ }
917
+ }
918
+ }
919
+ }
699
920
  const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
700
921
  const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
701
922
  if (state.convergence && supersededFailures.length > 0) {
@@ -719,6 +940,19 @@ function finalizeTerminalRunStatus(state, taskCount) {
719
940
  else {
720
941
  state.status = "failed";
721
942
  }
943
+ // Terminal recovery bookkeeping for a child attempt: a child that finished
944
+ // recovered settles its own lineage; any other child terminal settles as
945
+ // auto-recovery-blocked (phase 3 performs no further recovery).
946
+ const recovery = state.frontendRecoveryState;
947
+ if (recovery && recovery.attemptIndex >= 1) {
948
+ if (state.status === "finished") {
949
+ settleFrontendRecovery(state, "recovered");
950
+ }
951
+ else if (!state.frontendRecoveryResult) {
952
+ settleFrontendRecovery(state, "auto-recovery-blocked");
953
+ }
954
+ }
955
+ return { recoveryPending };
722
956
  }
723
957
  function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
724
958
  if (rankWriterNodeIds.length <= 1)