@tea-agent/loop-agent 0.35.1-beta.4 → 0.35.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +0 -2
- package/CHANGELOG.md +43 -26
- package/bin/loop-agent.js +1 -37
- package/dist/application/dag/generate-task-dag.js +4 -1
- package/dist/application/task-lifecycle/advance.js +14 -0
- package/dist/cli/program.js +2 -2
- package/dist/commands/task-advance.js +1 -0
- package/dist/executors/dag-pi-executor.js +0 -44
- package/dist/shared/package-metadata.js +0 -42
- package/dist/task/config-types.js +2 -0
- package/dist/task/contract/project.js +3 -0
- package/dist/task/contract/schema.js +1 -0
- package/dist/task/source-prepare/build-draft.js +7 -0
- package/dist/task/source-prepare/semantic-intake.js +37 -10
- package/dist/task/task-demand-routing.js +10 -0
- package/dist/worker/console/chat/assistant-content.js +110 -0
- package/dist/worker/console/chat/pi-runtime.js +13 -32
- package/dist/worker/console/chat/routes.js +9 -4
- package/dist/worker/console/chat/session-store.js +3 -0
- package/dist/worker/console/chat/shortcuts.js +9 -7
- package/dist/worker/console/chat/turn-process.js +62 -15
- package/dist/worker/console/operator-actions.js +72 -6
- package/dist/worker/console/prd-intake-bridge.js +10 -3
- package/dist/worker/console/prd-reference-discovery.js +124 -0
- package/dist/worker/console/static/assets/index-DRqZiQ7J.css +1 -0
- package/dist/worker/console/static/assets/index-gVHrlqI9.js +56 -0
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/console/static-src/app/useOperatorActions.js +19 -1
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +0 -5
- package/dist/worker/console/static-src/app/useTaskWizard.js +12 -0
- package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +10 -1
- package/dist/worker/console/static-src/operator-chat/useChatSessions.js +11 -6
- package/dist/worker/console/static-src/operator-chat/useChatThread.js +11 -6
- package/dist/worker/console/static-src/operator-chat/useComposer.js +15 -1
- package/dist/worker/loop-agent/loop-agent-client.js +3 -17
- package/dist/worker/observability/read-model.js +0 -20
- package/dist/worker/observe/spec-evidence.js +8 -3
- package/dist/worker/observe/static/views/dag-inspector.js +71 -6
- package/dist/worker/preflight.js +1 -2
- package/dist/workflows/dag/backend-test-scenario-param.js +23 -33
- package/dist/workflows/dag/dynamic-runtime/shared.js +1 -9
- package/dist/workflows/dag/frontend-implementation-contract.js +39 -233
- package/dist/workflows/dag/frontend-prewrite-gate.js +61 -364
- package/dist/workflows/dag/frontend-repair.js +18 -219
- package/dist/workflows/dag/frontend-verification-trace.js +32 -47
- package/dist/workflows/dag/init-hybrid.js +26 -49
- package/dist/workflows/dag/node-execution.js +0 -89
- package/dist/workflows/dag/recovery-recommendation.js +0 -58
- package/dist/workflows/dag/runner.js +11 -245
- package/dist/workflows/dag/scheduler.js +3 -257
- package/dist/workflows/dag/types.js +2 -130
- package/harness.json +1 -1
- package/package.json +2 -2
- package/dist/build-stamp.json +0 -6
- package/dist/worker/console/static/assets/index-CvsQgALl.js +0 -56
- package/dist/worker/console/static/assets/index-hJqCPs_g.css +0 -1
- package/dist/workflows/dag/contract-output-registry.js +0 -14
- package/dist/workflows/dag/contract-validator-registrations.js +0 -8
- package/dist/workflows/dag/frontend-recovery-plan.js +0 -73
- package/dist/workflows/dag/frontend-recovery-root-manifest.js +0 -123
- package/dist/workflows/dag/frontend-recovery-run.js +0 -539
- package/dist/workflows/dag/frontend-writer-recovery.js +0 -106
- package/dist/workflows/dag/frontend-writer-rollback.js +0 -821
|
@@ -4,7 +4,6 @@ import { readFile } from "node:fs/promises";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { pathMatchesPattern } from "../../shared/git-progress.js";
|
|
6
6
|
import { recordDecisionEnvelopeForNode, shouldPauseOnHumanEscalation, writeHumanEscalationArtifacts, } from "./decision-envelope.js";
|
|
7
|
-
import { FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isFrontendWriterAuthorized, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
8
7
|
import { writeNodeRecord, writeNodeSkillArtifacts } from "./run-store.js";
|
|
9
8
|
import { resolveContextPolicy } from "./context-policy.js";
|
|
10
9
|
import { buildDagNodePromptEnvelope, formatConvergenceFeedbackBlock, } from "./prompt.js";
|
|
@@ -13,8 +12,6 @@ import { buildOutputLimitRecoverySection, loadBackendTestWriterProgressForRetry,
|
|
|
13
12
|
import { computeBackoffDelayMs, isRetryablePiFailureCategory, isSafeReadOnlyPiRetryCandidate, isWriterEmptyDiffRetryCandidate, isWriterTransportRetryCandidate, } from "./retry-policy.js";
|
|
14
13
|
import { applyNodeActivity, evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
15
14
|
import { buildProtocolRetryInstruction, normalizeReviewVerdictAfterRetries, parseJsonReviewVerdict, validateOutputProtocol, } from "./output-protocol.js";
|
|
16
|
-
import { getStructuredContractValidator } from "./contract-output-registry.js";
|
|
17
|
-
import "./contract-validator-registrations.js";
|
|
18
15
|
import { writeDagNodeJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
19
16
|
import { buildProjectGovernanceContext, readCompletedWriterChangeManifests, writeProjectGovernanceContext, } from "./project-governance-context.js";
|
|
20
17
|
import { assertSkillSnapshotCoversSpec, buildNodePromptFromSnapshot, isDagSkillSnapshotIntegrityError, readSkillSnapshot, } from "./skill-snapshot.js";
|
|
@@ -141,20 +138,6 @@ function buildAttemptPrompt(task, basePrompt, attemptNumber, previousFailureCate
|
|
|
141
138
|
buildProtocolRetryInstruction(task.outputProtocol, previousProtocolReason),
|
|
142
139
|
].join("\n");
|
|
143
140
|
}
|
|
144
|
-
if (previousFailureCategory === "invalid-output" &&
|
|
145
|
-
task.structuredContractOutput &&
|
|
146
|
-
previousProtocolReason) {
|
|
147
|
-
return [
|
|
148
|
-
basePrompt,
|
|
149
|
-
"",
|
|
150
|
-
"<retry_instruction>",
|
|
151
|
-
"Previous attempt produced an invalid frontend implementation contract:",
|
|
152
|
-
previousProtocolReason,
|
|
153
|
-
"Return exactly one fenced json block conforming to the frontend-implementation-contract-v1 schema.",
|
|
154
|
-
"Fix every reported field violation: do not emit null for optional fields, do not misspell field names, and match the required types exactly.",
|
|
155
|
-
"</retry_instruction>",
|
|
156
|
-
].join("\n");
|
|
157
|
-
}
|
|
158
141
|
if (previousFailureCategory === "writer-empty-diff") {
|
|
159
142
|
const maxAttempts = task.retryPolicy?.maxAttempts ?? 3;
|
|
160
143
|
// When a completeness progress exists for this writer, fold the concrete
|
|
@@ -412,21 +395,6 @@ export async function executeDagNode(input) {
|
|
|
412
395
|
await notifyNodeObserver(input.observer, "onNodeFinish", nodeId, state);
|
|
413
396
|
};
|
|
414
397
|
const failSkillSnapshot = (error) => failBeforePrompt(error, "skill-snapshot-integrity");
|
|
415
|
-
const skipFrontendWriter = async (admission) => {
|
|
416
|
-
const skippedAt = new Date().toISOString();
|
|
417
|
-
node.startedAt ??= skippedAt;
|
|
418
|
-
node.frontendWriterAdmission = admission;
|
|
419
|
-
node.status = "SKIPPED";
|
|
420
|
-
node.skippedReason = "frontend-prewrite-not-authorized";
|
|
421
|
-
node.finishedAt = skippedAt;
|
|
422
|
-
node.lastActivityAt = skippedAt;
|
|
423
|
-
node.durationMs = durationBetween(node.startedAt, node.finishedAt);
|
|
424
|
-
node.timing = { retryBackoffMs: 0, settlementCleanupMs: 0 };
|
|
425
|
-
state.nodes[nodeId].nodeRecordPath = path.join(runDir, `${nodeId}.json`);
|
|
426
|
-
await writeNodeRecord(runDir, nodeId, state.nodes[nodeId]);
|
|
427
|
-
await input.persistState();
|
|
428
|
-
await notifyNodeObserver(input.observer, "onNodeFinish", nodeId, state);
|
|
429
|
-
};
|
|
430
398
|
if (task.finalWriteSetApproval) {
|
|
431
399
|
const authorization = parseAndValidateFinalWriteSetApproval({ task, spec, state });
|
|
432
400
|
if (!authorization.ok) {
|
|
@@ -455,30 +423,6 @@ export async function executeDagNode(input) {
|
|
|
455
423
|
effectiveWriteSet: [...authorization.effectiveWriteSet],
|
|
456
424
|
};
|
|
457
425
|
}
|
|
458
|
-
if (FRONTEND_WRITER_NODE_IDS.includes(nodeId)) {
|
|
459
|
-
const admission = await readFrontendPrewriteResult(runDir);
|
|
460
|
-
if (!admission.ok) {
|
|
461
|
-
await skipFrontendWriter(undefined);
|
|
462
|
-
return;
|
|
463
|
-
}
|
|
464
|
-
const decision = isFrontendWriterAuthorized(admission.result);
|
|
465
|
-
const record = {
|
|
466
|
-
schemaVersion: 1,
|
|
467
|
-
writerNodeId: nodeId,
|
|
468
|
-
decision,
|
|
469
|
-
sourceArtifact: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
470
|
-
artifactHash: admission.artifactHash,
|
|
471
|
-
checkedAt: new Date().toISOString(),
|
|
472
|
-
reason: decision === "denied"
|
|
473
|
-
? `classification: ${admission.result.classification}`
|
|
474
|
-
: null,
|
|
475
|
-
};
|
|
476
|
-
if (decision === "denied") {
|
|
477
|
-
await skipFrontendWriter(record);
|
|
478
|
-
return;
|
|
479
|
-
}
|
|
480
|
-
node.frontendWriterAdmission = record;
|
|
481
|
-
}
|
|
482
426
|
let projectGovernanceContext;
|
|
483
427
|
if (task.governanceStandardReview) {
|
|
484
428
|
try {
|
|
@@ -745,39 +689,6 @@ export async function executeDagNode(input) {
|
|
|
745
689
|
previousProtocolReason = undefined;
|
|
746
690
|
}
|
|
747
691
|
}
|
|
748
|
-
// R1: structured contract nodes self-validate their output so schema,
|
|
749
|
-
// typo, and null violations surface as retryable invalid-output at the
|
|
750
|
-
// producing node instead of failing the whole run at the prewrite gate.
|
|
751
|
-
if (result.ok && task.structuredContractOutput) {
|
|
752
|
-
const validator = getStructuredContractValidator(task.structuredContractOutput.schemaId);
|
|
753
|
-
if (validator) {
|
|
754
|
-
const contractText = canonicalNodeOutput(result);
|
|
755
|
-
const contractCheck = await validator({
|
|
756
|
-
runDir,
|
|
757
|
-
text: contractText,
|
|
758
|
-
sourceBinding: spec.sourceBinding,
|
|
759
|
-
});
|
|
760
|
-
if (!contractCheck.ok) {
|
|
761
|
-
result = {
|
|
762
|
-
...result,
|
|
763
|
-
ok: false,
|
|
764
|
-
failureCategory: "invalid-output",
|
|
765
|
-
stderr: [result.stderr, contractCheck.reason]
|
|
766
|
-
.filter(Boolean)
|
|
767
|
-
.join("\n"),
|
|
768
|
-
};
|
|
769
|
-
previousProtocolReason = contractCheck.reason;
|
|
770
|
-
}
|
|
771
|
-
else {
|
|
772
|
-
previousProtocolReason = undefined;
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
else {
|
|
776
|
-
// DAG spec validation rejects unknown schemaIds; this is a
|
|
777
|
-
// defensive fallback for a registry that has not been populated.
|
|
778
|
-
console.warn(`[run-dag] warning: no structured contract validator registered for schemaId ${task.structuredContractOutput.schemaId}; skipping node self-check`);
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
692
|
const attemptFinishedAt = new Date().toISOString();
|
|
782
693
|
const attemptWallDurationMs = durationBetween(attemptStartedAt, attemptFinishedAt);
|
|
783
694
|
totalAttemptWallDurationMs += attemptWallDurationMs;
|
|
@@ -269,61 +269,3 @@ export function planDagRecovery(input) {
|
|
|
269
269
|
}
|
|
270
270
|
return planByNormalizedCategory(input);
|
|
271
271
|
}
|
|
272
|
-
/**
|
|
273
|
-
* Phase 6: route a frontend recovery outcome to a recovery CTA (read-only; never
|
|
274
|
-
* mutates a run's status). The six outcomes map to the existing recovery actions
|
|
275
|
-
* so the Console recovery page reuses the current CTA surface.
|
|
276
|
-
*/
|
|
277
|
-
export function recommendFrontendRecoveryOutcome(outcome) {
|
|
278
|
-
switch (outcome) {
|
|
279
|
-
case "none":
|
|
280
|
-
return {
|
|
281
|
-
action: "none",
|
|
282
|
-
summary: "未发生前端恢复;按普通终态展示。",
|
|
283
|
-
reason: "该 frontend-implementation 根没有触发自动恢复。",
|
|
284
|
-
humanRequired: false,
|
|
285
|
-
autoRetryEligible: false,
|
|
286
|
-
};
|
|
287
|
-
case "recovered":
|
|
288
|
-
return {
|
|
289
|
-
action: "none",
|
|
290
|
-
summary: "自动恢复成功;child 已重新实现并通过验证。",
|
|
291
|
-
reason: "parent 的 transient 失败已由 child 恢复,全链按 root 计一次成功。",
|
|
292
|
-
humanRequired: false,
|
|
293
|
-
autoRetryEligible: false,
|
|
294
|
-
};
|
|
295
|
-
case "candidate-contract-invalid":
|
|
296
|
-
return {
|
|
297
|
-
action: "manual-review",
|
|
298
|
-
summary: "方案输出无效:candidate JSON 无法解析/推导,重试用尽。",
|
|
299
|
-
reason: "建议人工检查需求或重新生成方案后再跑。",
|
|
300
|
-
humanRequired: true,
|
|
301
|
-
autoRetryEligible: false,
|
|
302
|
-
};
|
|
303
|
-
case "prewrite-blocked":
|
|
304
|
-
return {
|
|
305
|
-
action: "manual-review",
|
|
306
|
-
summary: "治理门禁阻断:source stale / writeSet 越界 / Mock policy / contract 违规。",
|
|
307
|
-
reason: "需人工修正任务边界/事实后重跑,不自动重试。",
|
|
308
|
-
humanRequired: true,
|
|
309
|
-
autoRetryEligible: false,
|
|
310
|
-
};
|
|
311
|
-
case "repair-exhausted":
|
|
312
|
-
return {
|
|
313
|
-
action: "rerun-after-fix",
|
|
314
|
-
summary: "自动修复未通过:verify 失败,1 次 repair 后 reverify 仍失败。",
|
|
315
|
-
reason: "建议 `dag rerun` 或人工排查。",
|
|
316
|
-
humanRequired: true,
|
|
317
|
-
autoRetryEligible: false,
|
|
318
|
-
commandHint: "loop-agent dag rerun --run-id <runId>",
|
|
319
|
-
};
|
|
320
|
-
case "auto-recovery-blocked":
|
|
321
|
-
return {
|
|
322
|
-
action: "manual-review",
|
|
323
|
-
summary: "自动恢复被阻止:rollback 无法证明完整或存在外部/越界修改。",
|
|
324
|
-
reason: "必须人工介入,禁止自动覆盖用户文件。",
|
|
325
|
-
humanRequired: true,
|
|
326
|
-
autoRetryEligible: false,
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
-
import { randomUUID } from "node:crypto";
|
|
3
2
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
4
3
|
import { hostname } from "node:os";
|
|
5
4
|
import path from "node:path";
|
|
@@ -7,7 +6,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
|
|
|
7
6
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
8
7
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
9
8
|
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
10
|
-
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun,
|
|
9
|
+
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
11
10
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
12
11
|
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
13
12
|
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
@@ -22,9 +21,8 @@ import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSna
|
|
|
22
21
|
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
|
|
23
22
|
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
|
|
24
23
|
import { runConvergencePassController, } from "./convergence/controller.js";
|
|
25
|
-
import { executeDagRanksOnce,
|
|
24
|
+
import { executeDagRanksOnce, isConditionSkippedReason } from "./scheduler.js";
|
|
26
25
|
import { topoSortToRanks } from "./topo.js";
|
|
27
|
-
import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
|
|
28
26
|
import { parseDagSpec, resolveModelForTask, } from "./types.js";
|
|
29
27
|
import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
|
|
30
28
|
import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
|
|
@@ -450,39 +448,6 @@ async function executeDagCheckpoint(input) {
|
|
|
450
448
|
void persistState().catch(() => { });
|
|
451
449
|
}, runnerLivenessPolicy.heartbeatIntervalMs);
|
|
452
450
|
heartbeatTimer.unref();
|
|
453
|
-
// Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
|
|
454
|
-
// timeout, supervision layer, or shell wall-clock) must not leave the run
|
|
455
|
-
// orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
|
|
456
|
-
// state with an explicit terminalReason before exiting so downstream
|
|
457
|
-
// liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
|
|
458
|
-
let terminalSignal;
|
|
459
|
-
const shutdownOnSignal = (signal) => {
|
|
460
|
-
if (terminalSignal)
|
|
461
|
-
return;
|
|
462
|
-
terminalSignal = signal;
|
|
463
|
-
clearInterval(heartbeatTimer);
|
|
464
|
-
if (state.runner)
|
|
465
|
-
state.runner.heartbeatAt = new Date().toISOString();
|
|
466
|
-
state.status = "failed";
|
|
467
|
-
state.finishedAt = new Date().toISOString();
|
|
468
|
-
state.terminalReason = `runner terminated by ${signal} before run completion`;
|
|
469
|
-
console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
|
|
470
|
-
const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
|
|
471
|
-
// Best-effort atomic persist with a hard exit deadline so a stuck write
|
|
472
|
-
// queue cannot keep the process alive past the supervisor's kill window.
|
|
473
|
-
const hardExit = setTimeout(exit, 5000);
|
|
474
|
-
hardExit.unref();
|
|
475
|
-
void persistState()
|
|
476
|
-
.catch(() => { })
|
|
477
|
-
.finally(() => {
|
|
478
|
-
clearTimeout(hardExit);
|
|
479
|
-
exit();
|
|
480
|
-
});
|
|
481
|
-
};
|
|
482
|
-
const onSigTerm = () => shutdownOnSignal("SIGTERM");
|
|
483
|
-
const onSigInt = () => shutdownOnSignal("SIGINT");
|
|
484
|
-
process.once("SIGTERM", onSigTerm);
|
|
485
|
-
process.once("SIGINT", onSigInt);
|
|
486
451
|
try {
|
|
487
452
|
const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
488
453
|
const baseExecuteNode = input.executeNode ??
|
|
@@ -539,7 +504,6 @@ async function executeDagCheckpoint(input) {
|
|
|
539
504
|
tasksById,
|
|
540
505
|
maxConcurrent,
|
|
541
506
|
persistState,
|
|
542
|
-
runDir,
|
|
543
507
|
abortSignal: input.abortSignal,
|
|
544
508
|
createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
|
|
545
509
|
baseExecuteNode,
|
|
@@ -631,54 +595,17 @@ async function executeDagCheckpoint(input) {
|
|
|
631
595
|
runDir = await moveToPausedRunDir(runDir, pausedRunDir);
|
|
632
596
|
}
|
|
633
597
|
else {
|
|
634
|
-
|
|
598
|
+
finalizeTerminalRunStatus(state, spec.tasks.length);
|
|
635
599
|
await persistState();
|
|
636
|
-
if (recoveryPending) {
|
|
637
|
-
// Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
|
|
638
|
-
// child and run it synchronously. Dynamic import avoids a static
|
|
639
|
-
// runner ↔ frontend-recovery-run module cycle. stageRecoveryChild
|
|
640
|
-
// locates the parent via the canonical active/ layout, so recovery
|
|
641
|
-
// only fires when this run is still the canonical active run dir
|
|
642
|
-
// (direct runDagContinuation callers using an arbitrary runDir keep
|
|
643
|
-
// the previous scheduler-layer skip semantics).
|
|
644
|
-
const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
|
|
645
|
-
if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
|
|
646
|
-
const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
|
|
647
|
-
const staged = await stageRecoveryChild({
|
|
648
|
-
cwd,
|
|
649
|
-
parentRunId: state.runId,
|
|
650
|
-
});
|
|
651
|
-
// stageRecoveryChild CAS-advanced the parent's on-disk lineage
|
|
652
|
-
// (child-staging → child-running + childRunId). Refresh the in-memory
|
|
653
|
-
// copy so the terminal persistState below does not regress the parent
|
|
654
|
-
// back to stale child-staging.
|
|
655
|
-
const parentOnDisk = await readDagRunState(runDir);
|
|
656
|
-
state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
|
|
657
|
-
const childState = await readDagRunState(staged.childRunDir);
|
|
658
|
-
const childSpec = await readDagRunSpec(staged.childRunDir);
|
|
659
|
-
await runDagContinuation({
|
|
660
|
-
cwd,
|
|
661
|
-
spec: childSpec,
|
|
662
|
-
state: childState,
|
|
663
|
-
runDir: staged.childRunDir,
|
|
664
|
-
maxConcurrent,
|
|
665
|
-
executeNode: input.executeNode,
|
|
666
|
-
observer: input.observer,
|
|
667
|
-
abortSignal: input.abortSignal,
|
|
668
|
-
});
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
600
|
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
676
|
-
}
|
|
677
|
-
catch (error) {
|
|
678
|
-
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
679
|
-
}
|
|
680
|
-
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
601
|
+
try {
|
|
602
|
+
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
603
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
681
604
|
}
|
|
605
|
+
catch (error) {
|
|
606
|
+
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
607
|
+
}
|
|
608
|
+
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
682
609
|
}
|
|
683
610
|
await relocateRunArtifactPaths({
|
|
684
611
|
runDir,
|
|
@@ -701,8 +628,6 @@ async function executeDagCheckpoint(input) {
|
|
|
701
628
|
}
|
|
702
629
|
finally {
|
|
703
630
|
clearInterval(heartbeatTimer);
|
|
704
|
-
process.removeListener("SIGTERM", onSigTerm);
|
|
705
|
-
process.removeListener("SIGINT", onSigInt);
|
|
706
631
|
}
|
|
707
632
|
}
|
|
708
633
|
async function notifyRunObserver(observer, event, state) {
|
|
@@ -770,153 +695,7 @@ function isSuccessfulConvergenceTerminal(state) {
|
|
|
770
695
|
return true;
|
|
771
696
|
return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
|
|
772
697
|
}
|
|
773
|
-
function
|
|
774
|
-
const existing = state.frontendRecoveryState;
|
|
775
|
-
const requestId = existing?.requestId ?? randomUUID();
|
|
776
|
-
return {
|
|
777
|
-
schemaVersion: 1,
|
|
778
|
-
phase: "child-staging",
|
|
779
|
-
requestId,
|
|
780
|
-
recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
|
|
781
|
-
parentRunId: state.runId,
|
|
782
|
-
...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
|
|
783
|
-
attemptId: existing?.attemptId ?? `recovery-${requestId}`,
|
|
784
|
-
attemptIndex: existing?.attemptIndex ?? 0,
|
|
785
|
-
continuationCount: existing?.continuationCount ?? 0,
|
|
786
|
-
...(failureSource ? { failureSource } : {}),
|
|
787
|
-
revision: (existing?.revision ?? 0) + 1,
|
|
788
|
-
};
|
|
789
|
-
}
|
|
790
|
-
function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
791
|
-
const recovery = state.frontendRecoveryState;
|
|
792
|
-
return {
|
|
793
|
-
schemaVersion: 1,
|
|
794
|
-
outcome: "candidate-contract-invalid",
|
|
795
|
-
origin: {
|
|
796
|
-
kind: "frontend-prewrite-gate",
|
|
797
|
-
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
798
|
-
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
799
|
-
requestId: recovery?.requestId ?? "",
|
|
800
|
-
failedNodeId: result.selectedPlanNodeId,
|
|
801
|
-
},
|
|
802
|
-
failureClass: {
|
|
803
|
-
code: "candidate-contract-invalid",
|
|
804
|
-
classification: result.classification,
|
|
805
|
-
reason: result.failureReason ?? "candidate contract invalid",
|
|
806
|
-
},
|
|
807
|
-
evidenceRefs: [
|
|
808
|
-
{
|
|
809
|
-
runId: state.runId,
|
|
810
|
-
relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
811
|
-
sha256: artifactHash,
|
|
812
|
-
},
|
|
813
|
-
],
|
|
814
|
-
};
|
|
815
|
-
}
|
|
816
|
-
function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
817
|
-
const recovery = state.frontendRecoveryState;
|
|
818
|
-
return {
|
|
819
|
-
schemaVersion: 1,
|
|
820
|
-
outcome: "auto-recovery-blocked",
|
|
821
|
-
origin: {
|
|
822
|
-
kind: "frontend-writer",
|
|
823
|
-
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
824
|
-
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
825
|
-
requestId: recovery?.requestId ?? "",
|
|
826
|
-
failedNodeId,
|
|
827
|
-
},
|
|
828
|
-
failureClass: {
|
|
829
|
-
code: "writer-transient-partial-write",
|
|
830
|
-
classification: "transient",
|
|
831
|
-
reason,
|
|
832
|
-
},
|
|
833
|
-
evidenceRefs: [],
|
|
834
|
-
};
|
|
835
|
-
}
|
|
836
|
-
function settleFrontendRecovery(state, outcome) {
|
|
837
|
-
const recovery = state.frontendRecoveryState;
|
|
838
|
-
if (!recovery)
|
|
839
|
-
return;
|
|
840
|
-
recovery.phase = "settled";
|
|
841
|
-
if (!state.frontendRecoveryResult) {
|
|
842
|
-
state.frontendRecoveryResult = {
|
|
843
|
-
schemaVersion: 1,
|
|
844
|
-
outcome,
|
|
845
|
-
origin: {
|
|
846
|
-
kind: "frontend-prewrite-gate",
|
|
847
|
-
parentRunId: recovery.parentRunId,
|
|
848
|
-
...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
849
|
-
requestId: recovery.requestId,
|
|
850
|
-
failedNodeId: "",
|
|
851
|
-
},
|
|
852
|
-
evidenceRefs: [],
|
|
853
|
-
};
|
|
854
|
-
}
|
|
855
|
-
}
|
|
856
|
-
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
857
|
-
const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
|
|
858
|
-
// Terminal override: a denied frontend writer must never surface as a
|
|
859
|
-
// mechanical partial_failed just because prewrite FINISHED while the writer
|
|
860
|
-
// was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
|
|
861
|
-
let recoveryPending = false;
|
|
862
|
-
const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
|
|
863
|
-
if (hasFrontendWriter) {
|
|
864
|
-
const admission = await readFrontendPrewriteResult(runDir);
|
|
865
|
-
if (admission.ok) {
|
|
866
|
-
if (admission.result.classification === "blocked") {
|
|
867
|
-
state.status = "failed";
|
|
868
|
-
return { recoveryPending: false };
|
|
869
|
-
}
|
|
870
|
-
if (admission.result.classification === "retryable-invalid") {
|
|
871
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
872
|
-
if (continuationCount >= 1) {
|
|
873
|
-
// Continuation quota exhausted: a second retryable-invalid candidate
|
|
874
|
-
// is authoritative candidate-contract-invalid → forced failed.
|
|
875
|
-
state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
|
|
876
|
-
state.status = "failed";
|
|
877
|
-
return { recoveryPending: false };
|
|
878
|
-
}
|
|
879
|
-
// Root continuation still available: record the recovery intent and let
|
|
880
|
-
// mechanical aggregation settle partial_failed/failed. The parent keeps
|
|
881
|
-
// phase != settled (recovery in progress) and no final root manifest.
|
|
882
|
-
state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
|
|
883
|
-
recoveryPending = true;
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
// Phase 5: writer transient partial write → rollback + recovery intent. Only
|
|
888
|
-
// for frontend-implementation writers with a remaining root continuation, and
|
|
889
|
-
// only when the rollback journal (captured before the provider call) restores
|
|
890
|
-
// cleanly; otherwise the run stays failed with auto-recovery-blocked.
|
|
891
|
-
if (hasFrontendWriter && !recoveryPending) {
|
|
892
|
-
const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
|
|
893
|
-
const node = state.nodes[id];
|
|
894
|
-
return (node &&
|
|
895
|
-
node.status === "ERROR" &&
|
|
896
|
-
isFrontendWriterTransientPartialWrite({
|
|
897
|
-
failureCategory: node.failureCategory,
|
|
898
|
-
}));
|
|
899
|
-
});
|
|
900
|
-
if (writerNodeId) {
|
|
901
|
-
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
902
|
-
if (continuationCount < 1) {
|
|
903
|
-
const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
|
|
904
|
-
const rollback = await rollbackFrontendWriter({
|
|
905
|
-
cwd: repoRoot,
|
|
906
|
-
parentRunId: state.runId,
|
|
907
|
-
});
|
|
908
|
-
if (rollback.ok) {
|
|
909
|
-
state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
|
|
910
|
-
recoveryPending = true;
|
|
911
|
-
}
|
|
912
|
-
else {
|
|
913
|
-
state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
|
|
914
|
-
state.status = "failed";
|
|
915
|
-
return { recoveryPending: false };
|
|
916
|
-
}
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
}
|
|
698
|
+
function finalizeTerminalRunStatus(state, taskCount) {
|
|
920
699
|
const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
|
|
921
700
|
const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
|
|
922
701
|
if (state.convergence && supersededFailures.length > 0) {
|
|
@@ -940,19 +719,6 @@ export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
|
940
719
|
else {
|
|
941
720
|
state.status = "failed";
|
|
942
721
|
}
|
|
943
|
-
// Terminal recovery bookkeeping for a child attempt: a child that finished
|
|
944
|
-
// recovered settles its own lineage; any other child terminal settles as
|
|
945
|
-
// auto-recovery-blocked (phase 3 performs no further recovery).
|
|
946
|
-
const recovery = state.frontendRecoveryState;
|
|
947
|
-
if (recovery && recovery.attemptIndex >= 1) {
|
|
948
|
-
if (state.status === "finished") {
|
|
949
|
-
settleFrontendRecovery(state, "recovered");
|
|
950
|
-
}
|
|
951
|
-
else if (!state.frontendRecoveryResult) {
|
|
952
|
-
settleFrontendRecovery(state, "auto-recovery-blocked");
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
return { recoveryPending };
|
|
956
722
|
}
|
|
957
723
|
function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
|
|
958
724
|
if (rankWriterNodeIds.length <= 1)
|