@tea-agent/loop-agent 0.35.2 → 0.35.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/CHANGELOG.md +31 -0
- package/README.md +1 -1
- package/bin/loop-agent.js +37 -1
- package/dist/build-stamp.json +6 -0
- package/dist/cli/program.js +2 -2
- package/dist/executors/dag-pi-executor.js +44 -0
- package/dist/shared/package-metadata.js +42 -0
- package/dist/worker/console/chat/assistant-content.js +11 -0
- package/dist/worker/console/chat/pi-runtime.js +6 -2
- package/dist/worker/console/chat/turn-process.js +17 -9
- package/dist/worker/console/chat/workspace-landing.js +1 -1
- package/dist/worker/console/static/assets/index-DuVLjCIT.js +57 -0
- package/dist/worker/console/static/index.html +1 -1
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +5 -0
- package/dist/worker/console/static-src/operator-chat/chat-sse-events.js +15 -3
- package/dist/worker/console/static-src/operator-chat/refs.js +3 -0
- package/dist/worker/console/static-src/operator-chat/useChatSessions.js +3 -0
- package/dist/worker/console/static-src/operator-chat/useChatThread.js +1 -0
- package/dist/worker/loop-agent/loop-agent-client.js +17 -3
- package/dist/worker/observability/read-model.js +20 -0
- package/dist/worker/observe/spec-evidence.js +3 -8
- package/dist/worker/observe/static/views/dag-inspector.js +6 -71
- package/dist/worker/preflight.js +2 -1
- package/dist/workflows/dag/backend-test-scenario-param.js +33 -23
- package/dist/workflows/dag/contract-output-registry.js +14 -0
- package/dist/workflows/dag/contract-validator-registrations.js +8 -0
- package/dist/workflows/dag/dynamic-runtime/shared.js +9 -1
- package/dist/workflows/dag/frontend-implementation-contract.js +233 -39
- package/dist/workflows/dag/frontend-prewrite-gate.js +364 -61
- package/dist/workflows/dag/frontend-recovery-plan.js +73 -0
- package/dist/workflows/dag/frontend-recovery-root-manifest.js +123 -0
- package/dist/workflows/dag/frontend-recovery-run.js +539 -0
- package/dist/workflows/dag/frontend-repair.js +219 -18
- package/dist/workflows/dag/frontend-verification-trace.js +47 -32
- package/dist/workflows/dag/frontend-writer-recovery.js +106 -0
- package/dist/workflows/dag/frontend-writer-rollback.js +821 -0
- package/dist/workflows/dag/init-hybrid.js +49 -24
- package/dist/workflows/dag/node-execution.js +89 -0
- package/dist/workflows/dag/recovery-recommendation.js +58 -0
- package/dist/workflows/dag/runner.js +245 -11
- package/dist/workflows/dag/scheduler.js +257 -3
- package/dist/workflows/dag/types.js +130 -2
- package/package.json +4 -3
- package/dist/worker/console/static/assets/index-gVHrlqI9.js +0 -56
|
@@ -2199,6 +2199,26 @@ function pruneFrontendTasksForRisk(tasks, risk) {
|
|
|
2199
2199
|
return { ...task, depends_on };
|
|
2200
2200
|
});
|
|
2201
2201
|
}
|
|
2202
|
+
/**
|
|
2203
|
+
* AC-1: shared frontend writer node defaults. frontend-implement-pi and
|
|
2204
|
+
* frontend-repair-pi share one prompt/contract/writeSet-guard surface and an
|
|
2205
|
+
* identical writeSet; only id, depends_on, runIf, outputContract, and
|
|
2206
|
+
* subtask_prompt differ per phase.
|
|
2207
|
+
*/
|
|
2208
|
+
function buildFrontendWriterNodeDefaults(input) {
|
|
2209
|
+
return {
|
|
2210
|
+
role: "implementer",
|
|
2211
|
+
executor: "pi",
|
|
2212
|
+
toolProfile: "write",
|
|
2213
|
+
complexity: input.complexity,
|
|
2214
|
+
writePolicy: "exclusive",
|
|
2215
|
+
writeSet: input.writeSet,
|
|
2216
|
+
allowedPaths: input.allowedPaths,
|
|
2217
|
+
forbiddenPaths: input.forbiddenPaths,
|
|
2218
|
+
skills: FRONTEND_BOUNDED_IMPLEMENT_SKILLS,
|
|
2219
|
+
writerOutcomePolicy: { type: "implementation-outcome-v1" },
|
|
2220
|
+
};
|
|
2221
|
+
}
|
|
2202
2222
|
async function buildFrontendHybridDagFromTask(sources) {
|
|
2203
2223
|
const { taskConfig } = sources;
|
|
2204
2224
|
const mockCapability = sources.frontendMockCapability ?? {
|
|
@@ -2339,6 +2359,12 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2339
2359
|
`Every requirement MUST have a non-empty expectedOutcome. Every interaction MUST have non-empty trigger and expectedBehavior. UI states with applicable=true MUST have non-empty expectedBehavior. Empty strings or omitted fields for these will cause contract rejection.`,
|
|
2340
2360
|
].join("\n")
|
|
2341
2361
|
: "";
|
|
2362
|
+
const verificationTargetFileInstruction = [
|
|
2363
|
+
`## Verification target file semantics`,
|
|
2364
|
+
`verificationTargets[].file is the code file that the target verifies (the file the writer changes), NOT where the command is defined.`,
|
|
2365
|
+
`Non-static targets (type unit/component/integration/mock) MUST set file to a concrete code file inside the implementation writeSet (task allowedPaths); the prewrite gate rejects any non-static target whose file falls outside the writeSet.`,
|
|
2366
|
+
`Command-level checks that run project-wide (all tests, typecheck, build, governance) MUST use type "static" and must NOT be bound as non-static targets with file=package.json/tsconfig.json/vite.config.ts/scripts/*. Static targets are exempt from the writeSet containment check.`,
|
|
2367
|
+
].join("\n");
|
|
2342
2368
|
const strategy = resolveDagVerifyStrategy(taskConfig);
|
|
2343
2369
|
const readOnlyPaths = taskConfig.allowedPaths.length > 0 ? taskConfig.allowedPaths : ["**"];
|
|
2344
2370
|
const behaviorPaths = deriveFrontendBehaviorPaths(taskConfig);
|
|
@@ -2532,6 +2558,10 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2532
2558
|
writePolicy: "read-only",
|
|
2533
2559
|
outputMode: "structured-required",
|
|
2534
2560
|
retryPolicy: STRUCTURED_REQUIRED_PI_RETRY_POLICY,
|
|
2561
|
+
structuredContractOutput: {
|
|
2562
|
+
schemaId: "frontend-implementation-contract-v1",
|
|
2563
|
+
retryOnInvalid: true,
|
|
2564
|
+
},
|
|
2535
2565
|
allowedPaths: readOnlyPaths,
|
|
2536
2566
|
forbiddenPaths,
|
|
2537
2567
|
skills: FRONTEND_IMPLEMENTATION_SKILLS,
|
|
@@ -2545,6 +2575,8 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2545
2575
|
"End with exactly one fenced json object conforming to frontend-implementation-contract-v1. This node is the single contract JSON producer: the fenced block is the authoritative contract the prewrite gate materializes. Do not emit any raw JSON, JSON in prose, or a second fenced block anywhere in the response; the plan text must not contain other balanced JSON objects.",
|
|
2546
2576
|
"Each requirement must state its user-observable or logic-observable expectedOutcome. Each interaction must state its trigger and expectedBehavior. IDs plus file paths are not sufficient behavior semantics.",
|
|
2547
2577
|
requirementCoverageInstruction,
|
|
2578
|
+
"verificationTargets[].commandLabel MUST be one of the frozen command labels listed above. Any other value will be rejected at contract materialization.",
|
|
2579
|
+
verificationTargetFileInstruction,
|
|
2548
2580
|
"Read-only: do not modify code, docs, artifacts, or repository files.",
|
|
2549
2581
|
fixedVerificationContext,
|
|
2550
2582
|
sourceContext,
|
|
@@ -2584,6 +2616,10 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2584
2616
|
writePolicy: "read-only",
|
|
2585
2617
|
outputMode: "structured-required",
|
|
2586
2618
|
retryPolicy: STRUCTURED_REQUIRED_PI_RETRY_POLICY,
|
|
2619
|
+
structuredContractOutput: {
|
|
2620
|
+
schemaId: "frontend-implementation-contract-v1",
|
|
2621
|
+
retryOnInvalid: true,
|
|
2622
|
+
},
|
|
2587
2623
|
allowedPaths: readOnlyPaths,
|
|
2588
2624
|
forbiddenPaths,
|
|
2589
2625
|
skills: FRONTEND_IMPLEMENTATION_SKILLS,
|
|
@@ -2598,6 +2634,7 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2598
2634
|
"End the response with exactly one fenced json object conforming to frontend-implementation-contract-v1. This node is the single contract JSON producer when the design review requests revision: the fenced block is the authoritative contract the prewrite gate materializes. Do not emit any raw JSON, JSON in prose, or a second fenced block anywhere in the response. Bind it to the supplied task sources; map every requirement and applicable UI state to concrete implementation and verification targets or an explicit blocking evidence gap. Do not include secrets or unsafe paths.",
|
|
2599
2635
|
"Preserve each requirement expectedOutcome and each interaction trigger/expectedBehavior in the revised contract; do not reduce behavior semantics to IDs and paths.",
|
|
2600
2636
|
"verificationTargets[].commandLabel MUST be one of the frozen command labels listed above. Any other value will be rejected at contract materialization.",
|
|
2637
|
+
verificationTargetFileInstruction,
|
|
2601
2638
|
fixedVerificationContext,
|
|
2602
2639
|
sourceContext,
|
|
2603
2640
|
frontendContractSchemaBlock,
|
|
@@ -2716,18 +2753,12 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2716
2753
|
? ["frontend-lint-baseline-shell"]
|
|
2717
2754
|
: []),
|
|
2718
2755
|
],
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
allowedPaths: implementPaths.allowedPaths,
|
|
2726
|
-
forbiddenPaths,
|
|
2727
|
-
skills: FRONTEND_BOUNDED_IMPLEMENT_SKILLS,
|
|
2728
|
-
writerOutcomePolicy: {
|
|
2729
|
-
type: "implementation-outcome-v1",
|
|
2730
|
-
},
|
|
2756
|
+
...buildFrontendWriterNodeDefaults({
|
|
2757
|
+
complexity: resolveWriterComplexity(taskConfig),
|
|
2758
|
+
writeSet: implementPaths.writeSet,
|
|
2759
|
+
allowedPaths: implementPaths.allowedPaths,
|
|
2760
|
+
forbiddenPaths,
|
|
2761
|
+
}),
|
|
2731
2762
|
outputContract: "First non-empty line must be exactly one of: IMPLEMENTATION_OUTCOME: changed; IMPLEMENTATION_OUTCOME: already-satisfied; IMPLEMENTATION_OUTCOME: blocked. Then a Markdown delivery summary with Contract Ref (path/schema/hash), Changed Files, Requirements Implemented, UI States, Tests Changed, Verification Attempts, Deviations, and Residual Risks. Follow fixed stages: contract confirm → tests → component/state → API/Mock → focused checks → diff cleanup.",
|
|
2732
2763
|
subtask_prompt: [
|
|
2733
2764
|
"Implement against the validated run-owned Frontend Implementation Contract from frontend-prewrite-gate-shell (path/schema/hash). Do not rebuild the contract from Markdown alone.",
|
|
@@ -2781,18 +2812,12 @@ async function buildFrontendHybridDagFromTask(sources) {
|
|
|
2781
2812
|
id: "frontend-repair-pi",
|
|
2782
2813
|
depends_on: ["frontend-verify-assess-shell", implementId],
|
|
2783
2814
|
runIf: "$.nodes['frontend-verify-assess-shell'].json.eligible == true",
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
allowedPaths: implementPaths.allowedPaths,
|
|
2791
|
-
forbiddenPaths,
|
|
2792
|
-
skills: FRONTEND_BOUNDED_IMPLEMENT_SKILLS,
|
|
2793
|
-
writerOutcomePolicy: {
|
|
2794
|
-
type: "implementation-outcome-v1",
|
|
2795
|
-
},
|
|
2815
|
+
...buildFrontendWriterNodeDefaults({
|
|
2816
|
+
complexity: resolveWriterComplexity(taskConfig),
|
|
2817
|
+
writeSet: implementPaths.writeSet,
|
|
2818
|
+
allowedPaths: implementPaths.allowedPaths,
|
|
2819
|
+
forbiddenPaths,
|
|
2820
|
+
}),
|
|
2796
2821
|
outputContract: "First non-empty line must be exactly one of: IMPLEMENTATION_OUTCOME: changed; IMPLEMENTATION_OUTCOME: already-satisfied; IMPLEMENTATION_OUTCOME: blocked. Then a repair summary for an eligible repairable assessment. Must not expand writeSet, re-interpret requirements, skip tests, or enable Mock by default.",
|
|
2797
2822
|
subtask_prompt: [
|
|
2798
2823
|
"Read contracts/frontend-repair-assessment.json and the validated frontend implementation contract.",
|
|
@@ -4,6 +4,7 @@ import { readFile } from "node:fs/promises";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { pathMatchesPattern } from "../../shared/git-progress.js";
|
|
6
6
|
import { recordDecisionEnvelopeForNode, shouldPauseOnHumanEscalation, writeHumanEscalationArtifacts, } from "./decision-envelope.js";
|
|
7
|
+
import { FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isFrontendWriterAuthorized, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
7
8
|
import { writeNodeRecord, writeNodeSkillArtifacts } from "./run-store.js";
|
|
8
9
|
import { resolveContextPolicy } from "./context-policy.js";
|
|
9
10
|
import { buildDagNodePromptEnvelope, formatConvergenceFeedbackBlock, } from "./prompt.js";
|
|
@@ -12,6 +13,8 @@ import { buildOutputLimitRecoverySection, loadBackendTestWriterProgressForRetry,
|
|
|
12
13
|
import { computeBackoffDelayMs, isRetryablePiFailureCategory, isSafeReadOnlyPiRetryCandidate, isWriterEmptyDiffRetryCandidate, isWriterTransportRetryCandidate, } from "./retry-policy.js";
|
|
13
14
|
import { applyNodeActivity, evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
14
15
|
import { buildProtocolRetryInstruction, normalizeReviewVerdictAfterRetries, parseJsonReviewVerdict, validateOutputProtocol, } from "./output-protocol.js";
|
|
16
|
+
import { getStructuredContractValidator } from "./contract-output-registry.js";
|
|
17
|
+
import "./contract-validator-registrations.js";
|
|
15
18
|
import { writeDagNodeJsonArtifact } from "../../infrastructure/harness/artifact-store.js";
|
|
16
19
|
import { buildProjectGovernanceContext, readCompletedWriterChangeManifests, writeProjectGovernanceContext, } from "./project-governance-context.js";
|
|
17
20
|
import { assertSkillSnapshotCoversSpec, buildNodePromptFromSnapshot, isDagSkillSnapshotIntegrityError, readSkillSnapshot, } from "./skill-snapshot.js";
|
|
@@ -138,6 +141,20 @@ function buildAttemptPrompt(task, basePrompt, attemptNumber, previousFailureCate
|
|
|
138
141
|
buildProtocolRetryInstruction(task.outputProtocol, previousProtocolReason),
|
|
139
142
|
].join("\n");
|
|
140
143
|
}
|
|
144
|
+
if (previousFailureCategory === "invalid-output" &&
|
|
145
|
+
task.structuredContractOutput &&
|
|
146
|
+
previousProtocolReason) {
|
|
147
|
+
return [
|
|
148
|
+
basePrompt,
|
|
149
|
+
"",
|
|
150
|
+
"<retry_instruction>",
|
|
151
|
+
"Previous attempt produced an invalid frontend implementation contract:",
|
|
152
|
+
previousProtocolReason,
|
|
153
|
+
"Return exactly one fenced json block conforming to the frontend-implementation-contract-v1 schema.",
|
|
154
|
+
"Fix every reported field violation: do not emit null for optional fields, do not misspell field names, and match the required types exactly.",
|
|
155
|
+
"</retry_instruction>",
|
|
156
|
+
].join("\n");
|
|
157
|
+
}
|
|
141
158
|
if (previousFailureCategory === "writer-empty-diff") {
|
|
142
159
|
const maxAttempts = task.retryPolicy?.maxAttempts ?? 3;
|
|
143
160
|
// When a completeness progress exists for this writer, fold the concrete
|
|
@@ -395,6 +412,21 @@ export async function executeDagNode(input) {
|
|
|
395
412
|
await notifyNodeObserver(input.observer, "onNodeFinish", nodeId, state);
|
|
396
413
|
};
|
|
397
414
|
const failSkillSnapshot = (error) => failBeforePrompt(error, "skill-snapshot-integrity");
|
|
415
|
+
const skipFrontendWriter = async (admission) => {
|
|
416
|
+
const skippedAt = new Date().toISOString();
|
|
417
|
+
node.startedAt ??= skippedAt;
|
|
418
|
+
node.frontendWriterAdmission = admission;
|
|
419
|
+
node.status = "SKIPPED";
|
|
420
|
+
node.skippedReason = "frontend-prewrite-not-authorized";
|
|
421
|
+
node.finishedAt = skippedAt;
|
|
422
|
+
node.lastActivityAt = skippedAt;
|
|
423
|
+
node.durationMs = durationBetween(node.startedAt, node.finishedAt);
|
|
424
|
+
node.timing = { retryBackoffMs: 0, settlementCleanupMs: 0 };
|
|
425
|
+
state.nodes[nodeId].nodeRecordPath = path.join(runDir, `${nodeId}.json`);
|
|
426
|
+
await writeNodeRecord(runDir, nodeId, state.nodes[nodeId]);
|
|
427
|
+
await input.persistState();
|
|
428
|
+
await notifyNodeObserver(input.observer, "onNodeFinish", nodeId, state);
|
|
429
|
+
};
|
|
398
430
|
if (task.finalWriteSetApproval) {
|
|
399
431
|
const authorization = parseAndValidateFinalWriteSetApproval({ task, spec, state });
|
|
400
432
|
if (!authorization.ok) {
|
|
@@ -423,6 +455,30 @@ export async function executeDagNode(input) {
|
|
|
423
455
|
effectiveWriteSet: [...authorization.effectiveWriteSet],
|
|
424
456
|
};
|
|
425
457
|
}
|
|
458
|
+
if (FRONTEND_WRITER_NODE_IDS.includes(nodeId)) {
|
|
459
|
+
const admission = await readFrontendPrewriteResult(runDir);
|
|
460
|
+
if (!admission.ok) {
|
|
461
|
+
await skipFrontendWriter(undefined);
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
const decision = isFrontendWriterAuthorized(admission.result);
|
|
465
|
+
const record = {
|
|
466
|
+
schemaVersion: 1,
|
|
467
|
+
writerNodeId: nodeId,
|
|
468
|
+
decision,
|
|
469
|
+
sourceArtifact: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
470
|
+
artifactHash: admission.artifactHash,
|
|
471
|
+
checkedAt: new Date().toISOString(),
|
|
472
|
+
reason: decision === "denied"
|
|
473
|
+
? `classification: ${admission.result.classification}`
|
|
474
|
+
: null,
|
|
475
|
+
};
|
|
476
|
+
if (decision === "denied") {
|
|
477
|
+
await skipFrontendWriter(record);
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
node.frontendWriterAdmission = record;
|
|
481
|
+
}
|
|
426
482
|
let projectGovernanceContext;
|
|
427
483
|
if (task.governanceStandardReview) {
|
|
428
484
|
try {
|
|
@@ -689,6 +745,39 @@ export async function executeDagNode(input) {
|
|
|
689
745
|
previousProtocolReason = undefined;
|
|
690
746
|
}
|
|
691
747
|
}
|
|
748
|
+
// R1: structured contract nodes self-validate their output so schema,
|
|
749
|
+
// typo, and null violations surface as retryable invalid-output at the
|
|
750
|
+
// producing node instead of failing the whole run at the prewrite gate.
|
|
751
|
+
if (result.ok && task.structuredContractOutput) {
|
|
752
|
+
const validator = getStructuredContractValidator(task.structuredContractOutput.schemaId);
|
|
753
|
+
if (validator) {
|
|
754
|
+
const contractText = canonicalNodeOutput(result);
|
|
755
|
+
const contractCheck = await validator({
|
|
756
|
+
runDir,
|
|
757
|
+
text: contractText,
|
|
758
|
+
sourceBinding: spec.sourceBinding,
|
|
759
|
+
});
|
|
760
|
+
if (!contractCheck.ok) {
|
|
761
|
+
result = {
|
|
762
|
+
...result,
|
|
763
|
+
ok: false,
|
|
764
|
+
failureCategory: "invalid-output",
|
|
765
|
+
stderr: [result.stderr, contractCheck.reason]
|
|
766
|
+
.filter(Boolean)
|
|
767
|
+
.join("\n"),
|
|
768
|
+
};
|
|
769
|
+
previousProtocolReason = contractCheck.reason;
|
|
770
|
+
}
|
|
771
|
+
else {
|
|
772
|
+
previousProtocolReason = undefined;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
else {
|
|
776
|
+
// DAG spec validation rejects unknown schemaIds; this is a
|
|
777
|
+
// defensive fallback for a registry that has not been populated.
|
|
778
|
+
console.warn(`[run-dag] warning: no structured contract validator registered for schemaId ${task.structuredContractOutput.schemaId}; skipping node self-check`);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
692
781
|
const attemptFinishedAt = new Date().toISOString();
|
|
693
782
|
const attemptWallDurationMs = durationBetween(attemptStartedAt, attemptFinishedAt);
|
|
694
783
|
totalAttemptWallDurationMs += attemptWallDurationMs;
|
|
@@ -269,3 +269,61 @@ export function planDagRecovery(input) {
|
|
|
269
269
|
}
|
|
270
270
|
return planByNormalizedCategory(input);
|
|
271
271
|
}
|
|
272
|
+
/**
|
|
273
|
+
* Phase 6: route a frontend recovery outcome to a recovery CTA (read-only; never
|
|
274
|
+
* mutates a run's status). The six outcomes map to the existing recovery actions
|
|
275
|
+
* so the Console recovery page reuses the current CTA surface.
|
|
276
|
+
*/
|
|
277
|
+
export function recommendFrontendRecoveryOutcome(outcome) {
|
|
278
|
+
switch (outcome) {
|
|
279
|
+
case "none":
|
|
280
|
+
return {
|
|
281
|
+
action: "none",
|
|
282
|
+
summary: "未发生前端恢复;按普通终态展示。",
|
|
283
|
+
reason: "该 frontend-implementation 根没有触发自动恢复。",
|
|
284
|
+
humanRequired: false,
|
|
285
|
+
autoRetryEligible: false,
|
|
286
|
+
};
|
|
287
|
+
case "recovered":
|
|
288
|
+
return {
|
|
289
|
+
action: "none",
|
|
290
|
+
summary: "自动恢复成功;child 已重新实现并通过验证。",
|
|
291
|
+
reason: "parent 的 transient 失败已由 child 恢复,全链按 root 计一次成功。",
|
|
292
|
+
humanRequired: false,
|
|
293
|
+
autoRetryEligible: false,
|
|
294
|
+
};
|
|
295
|
+
case "candidate-contract-invalid":
|
|
296
|
+
return {
|
|
297
|
+
action: "manual-review",
|
|
298
|
+
summary: "方案输出无效:candidate JSON 无法解析/推导,重试用尽。",
|
|
299
|
+
reason: "建议人工检查需求或重新生成方案后再跑。",
|
|
300
|
+
humanRequired: true,
|
|
301
|
+
autoRetryEligible: false,
|
|
302
|
+
};
|
|
303
|
+
case "prewrite-blocked":
|
|
304
|
+
return {
|
|
305
|
+
action: "manual-review",
|
|
306
|
+
summary: "治理门禁阻断:source stale / writeSet 越界 / Mock policy / contract 违规。",
|
|
307
|
+
reason: "需人工修正任务边界/事实后重跑,不自动重试。",
|
|
308
|
+
humanRequired: true,
|
|
309
|
+
autoRetryEligible: false,
|
|
310
|
+
};
|
|
311
|
+
case "repair-exhausted":
|
|
312
|
+
return {
|
|
313
|
+
action: "rerun-after-fix",
|
|
314
|
+
summary: "自动修复未通过:verify 失败,1 次 repair 后 reverify 仍失败。",
|
|
315
|
+
reason: "建议 `dag rerun` 或人工排查。",
|
|
316
|
+
humanRequired: true,
|
|
317
|
+
autoRetryEligible: false,
|
|
318
|
+
commandHint: "loop-agent dag rerun --run-id <runId>",
|
|
319
|
+
};
|
|
320
|
+
case "auto-recovery-blocked":
|
|
321
|
+
return {
|
|
322
|
+
action: "manual-review",
|
|
323
|
+
summary: "自动恢复被阻止:rollback 无法证明完整或存在外部/越界修改。",
|
|
324
|
+
reason: "必须人工介入,禁止自动覆盖用户文件。",
|
|
325
|
+
humanRequired: true,
|
|
326
|
+
autoRetryEligible: false,
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
3
4
|
import { hostname } from "node:os";
|
|
4
5
|
import path from "node:path";
|
|
@@ -6,7 +7,7 @@ import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../appl
|
|
|
6
7
|
import { readCandidateRecord } from "../../infrastructure/evaluation/candidate-store.js";
|
|
7
8
|
import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/runtime.js";
|
|
8
9
|
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
9
|
-
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
10
|
+
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
10
11
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
11
12
|
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
12
13
|
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
@@ -21,8 +22,9 @@ import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSna
|
|
|
21
22
|
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
|
|
22
23
|
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
|
|
23
24
|
import { runConvergencePassController, } from "./convergence/controller.js";
|
|
24
|
-
import { executeDagRanksOnce, isConditionSkippedReason } from "./scheduler.js";
|
|
25
|
+
import { executeDagRanksOnce, FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, FRONTEND_WRITER_NODE_IDS, isConditionSkippedReason, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
25
26
|
import { topoSortToRanks } from "./topo.js";
|
|
27
|
+
import { isFrontendWriterTransientPartialWrite } from "./frontend-writer-recovery.js";
|
|
26
28
|
import { parseDagSpec, resolveModelForTask, } from "./types.js";
|
|
27
29
|
import { executeDynamicCondition } from "./dynamic-runtime/condition.js";
|
|
28
30
|
import { executeDynamicLoopUntil } from "./dynamic-runtime/loop-until.js";
|
|
@@ -448,6 +450,39 @@ async function executeDagCheckpoint(input) {
|
|
|
448
450
|
void persistState().catch(() => { });
|
|
449
451
|
}, runnerLivenessPolicy.heartbeatIntervalMs);
|
|
450
452
|
heartbeatTimer.unref();
|
|
453
|
+
// Graceful terminal persistence: an outer SIGTERM/SIGINT (operator hard
|
|
454
|
+
// timeout, supervision layer, or shell wall-clock) must not leave the run
|
|
455
|
+
// orphaned in RUNNING with a dead heartbeat. Persist a terminal failed
|
|
456
|
+
// state with an explicit terminalReason before exiting so downstream
|
|
457
|
+
// liveness/doctor tooling sees a diagnosable terminal instead of an orphan.
|
|
458
|
+
let terminalSignal;
|
|
459
|
+
const shutdownOnSignal = (signal) => {
|
|
460
|
+
if (terminalSignal)
|
|
461
|
+
return;
|
|
462
|
+
terminalSignal = signal;
|
|
463
|
+
clearInterval(heartbeatTimer);
|
|
464
|
+
if (state.runner)
|
|
465
|
+
state.runner.heartbeatAt = new Date().toISOString();
|
|
466
|
+
state.status = "failed";
|
|
467
|
+
state.finishedAt = new Date().toISOString();
|
|
468
|
+
state.terminalReason = `runner terminated by ${signal} before run completion`;
|
|
469
|
+
console.warn(`[run-dag] received ${signal}; persisting terminal state and exiting`);
|
|
470
|
+
const exit = () => process.exit(signal === "SIGINT" ? 130 : 143);
|
|
471
|
+
// Best-effort atomic persist with a hard exit deadline so a stuck write
|
|
472
|
+
// queue cannot keep the process alive past the supervisor's kill window.
|
|
473
|
+
const hardExit = setTimeout(exit, 5000);
|
|
474
|
+
hardExit.unref();
|
|
475
|
+
void persistState()
|
|
476
|
+
.catch(() => { })
|
|
477
|
+
.finally(() => {
|
|
478
|
+
clearTimeout(hardExit);
|
|
479
|
+
exit();
|
|
480
|
+
});
|
|
481
|
+
};
|
|
482
|
+
const onSigTerm = () => shutdownOnSignal("SIGTERM");
|
|
483
|
+
const onSigInt = () => shutdownOnSignal("SIGINT");
|
|
484
|
+
process.once("SIGTERM", onSigTerm);
|
|
485
|
+
process.once("SIGINT", onSigInt);
|
|
451
486
|
try {
|
|
452
487
|
const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
453
488
|
const baseExecuteNode = input.executeNode ??
|
|
@@ -504,6 +539,7 @@ async function executeDagCheckpoint(input) {
|
|
|
504
539
|
tasksById,
|
|
505
540
|
maxConcurrent,
|
|
506
541
|
persistState,
|
|
542
|
+
runDir,
|
|
507
543
|
abortSignal: input.abortSignal,
|
|
508
544
|
createExecuteNodeForRank: (rankWriterNodeIds) => buildRankAwareExecuteNode({
|
|
509
545
|
baseExecuteNode,
|
|
@@ -595,17 +631,54 @@ async function executeDagCheckpoint(input) {
|
|
|
595
631
|
runDir = await moveToPausedRunDir(runDir, pausedRunDir);
|
|
596
632
|
}
|
|
597
633
|
else {
|
|
598
|
-
finalizeTerminalRunStatus(state, spec.tasks.length);
|
|
634
|
+
const { recoveryPending } = await finalizeTerminalRunStatus(state, spec.tasks.length, runDir, cwd);
|
|
599
635
|
await persistState();
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
636
|
+
if (recoveryPending) {
|
|
637
|
+
// Recovery coordinator (phase 3c AC-2/AC-3): materialize the reserved
|
|
638
|
+
// child and run it synchronously. Dynamic import avoids a static
|
|
639
|
+
// runner ↔ frontend-recovery-run module cycle. stageRecoveryChild
|
|
640
|
+
// locates the parent via the canonical active/ layout, so recovery
|
|
641
|
+
// only fires when this run is still the canonical active run dir
|
|
642
|
+
// (direct runDagContinuation callers using an arbitrary runDir keep
|
|
643
|
+
// the previous scheduler-layer skip semantics).
|
|
644
|
+
const canonicalActiveDir = getDagRunDir(cwd, "active", state.runId);
|
|
645
|
+
if (path.resolve(runDir) === path.resolve(canonicalActiveDir)) {
|
|
646
|
+
const { stageRecoveryChild } = await import("./frontend-recovery-run.js");
|
|
647
|
+
const staged = await stageRecoveryChild({
|
|
648
|
+
cwd,
|
|
649
|
+
parentRunId: state.runId,
|
|
650
|
+
});
|
|
651
|
+
// stageRecoveryChild CAS-advanced the parent's on-disk lineage
|
|
652
|
+
// (child-staging → child-running + childRunId). Refresh the in-memory
|
|
653
|
+
// copy so the terminal persistState below does not regress the parent
|
|
654
|
+
// back to stale child-staging.
|
|
655
|
+
const parentOnDisk = await readDagRunState(runDir);
|
|
656
|
+
state.frontendRecoveryState = parentOnDisk.frontendRecoveryState;
|
|
657
|
+
const childState = await readDagRunState(staged.childRunDir);
|
|
658
|
+
const childSpec = await readDagRunSpec(staged.childRunDir);
|
|
659
|
+
await runDagContinuation({
|
|
660
|
+
cwd,
|
|
661
|
+
spec: childSpec,
|
|
662
|
+
state: childState,
|
|
663
|
+
runDir: staged.childRunDir,
|
|
664
|
+
maxConcurrent,
|
|
665
|
+
executeNode: input.executeNode,
|
|
666
|
+
observer: input.observer,
|
|
667
|
+
abortSignal: input.abortSignal,
|
|
668
|
+
});
|
|
669
|
+
}
|
|
604
670
|
}
|
|
605
|
-
|
|
606
|
-
|
|
671
|
+
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
672
|
+
if (!recoveryPending) {
|
|
673
|
+
try {
|
|
674
|
+
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
675
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
676
|
+
}
|
|
677
|
+
catch (error) {
|
|
678
|
+
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
679
|
+
}
|
|
680
|
+
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
607
681
|
}
|
|
608
|
-
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
609
682
|
}
|
|
610
683
|
await relocateRunArtifactPaths({
|
|
611
684
|
runDir,
|
|
@@ -628,6 +701,8 @@ async function executeDagCheckpoint(input) {
|
|
|
628
701
|
}
|
|
629
702
|
finally {
|
|
630
703
|
clearInterval(heartbeatTimer);
|
|
704
|
+
process.removeListener("SIGTERM", onSigTerm);
|
|
705
|
+
process.removeListener("SIGINT", onSigInt);
|
|
631
706
|
}
|
|
632
707
|
}
|
|
633
708
|
async function notifyRunObserver(observer, event, state) {
|
|
@@ -695,7 +770,153 @@ function isSuccessfulConvergenceTerminal(state) {
|
|
|
695
770
|
return true;
|
|
696
771
|
return SUCCESSFUL_CONVERGENCE_TERMINAL_REASONS.has(reason);
|
|
697
772
|
}
|
|
698
|
-
function
|
|
773
|
+
function buildFrontendRecoveryIntent(state, failureSource) {
|
|
774
|
+
const existing = state.frontendRecoveryState;
|
|
775
|
+
const requestId = existing?.requestId ?? randomUUID();
|
|
776
|
+
return {
|
|
777
|
+
schemaVersion: 1,
|
|
778
|
+
phase: "child-staging",
|
|
779
|
+
requestId,
|
|
780
|
+
recoveryRootRunId: existing?.recoveryRootRunId ?? state.runId,
|
|
781
|
+
parentRunId: state.runId,
|
|
782
|
+
...(existing?.childRunId ? { childRunId: existing.childRunId } : {}),
|
|
783
|
+
attemptId: existing?.attemptId ?? `recovery-${requestId}`,
|
|
784
|
+
attemptIndex: existing?.attemptIndex ?? 0,
|
|
785
|
+
continuationCount: existing?.continuationCount ?? 0,
|
|
786
|
+
...(failureSource ? { failureSource } : {}),
|
|
787
|
+
revision: (existing?.revision ?? 0) + 1,
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
function buildCandidateContractInvalidResult(state, result, artifactHash) {
|
|
791
|
+
const recovery = state.frontendRecoveryState;
|
|
792
|
+
return {
|
|
793
|
+
schemaVersion: 1,
|
|
794
|
+
outcome: "candidate-contract-invalid",
|
|
795
|
+
origin: {
|
|
796
|
+
kind: "frontend-prewrite-gate",
|
|
797
|
+
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
798
|
+
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
799
|
+
requestId: recovery?.requestId ?? "",
|
|
800
|
+
failedNodeId: result.selectedPlanNodeId,
|
|
801
|
+
},
|
|
802
|
+
failureClass: {
|
|
803
|
+
code: "candidate-contract-invalid",
|
|
804
|
+
classification: result.classification,
|
|
805
|
+
reason: result.failureReason ?? "candidate contract invalid",
|
|
806
|
+
},
|
|
807
|
+
evidenceRefs: [
|
|
808
|
+
{
|
|
809
|
+
runId: state.runId,
|
|
810
|
+
relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
811
|
+
sha256: artifactHash,
|
|
812
|
+
},
|
|
813
|
+
],
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
function buildWriterPartialWriteBlockedResult(state, failedNodeId, reason) {
|
|
817
|
+
const recovery = state.frontendRecoveryState;
|
|
818
|
+
return {
|
|
819
|
+
schemaVersion: 1,
|
|
820
|
+
outcome: "auto-recovery-blocked",
|
|
821
|
+
origin: {
|
|
822
|
+
kind: "frontend-writer",
|
|
823
|
+
parentRunId: recovery?.parentRunId ?? state.runId,
|
|
824
|
+
...(recovery?.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
825
|
+
requestId: recovery?.requestId ?? "",
|
|
826
|
+
failedNodeId,
|
|
827
|
+
},
|
|
828
|
+
failureClass: {
|
|
829
|
+
code: "writer-transient-partial-write",
|
|
830
|
+
classification: "transient",
|
|
831
|
+
reason,
|
|
832
|
+
},
|
|
833
|
+
evidenceRefs: [],
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
function settleFrontendRecovery(state, outcome) {
|
|
837
|
+
const recovery = state.frontendRecoveryState;
|
|
838
|
+
if (!recovery)
|
|
839
|
+
return;
|
|
840
|
+
recovery.phase = "settled";
|
|
841
|
+
if (!state.frontendRecoveryResult) {
|
|
842
|
+
state.frontendRecoveryResult = {
|
|
843
|
+
schemaVersion: 1,
|
|
844
|
+
outcome,
|
|
845
|
+
origin: {
|
|
846
|
+
kind: "frontend-prewrite-gate",
|
|
847
|
+
parentRunId: recovery.parentRunId,
|
|
848
|
+
...(recovery.childRunId ? { childRunId: recovery.childRunId } : {}),
|
|
849
|
+
requestId: recovery.requestId,
|
|
850
|
+
failedNodeId: "",
|
|
851
|
+
},
|
|
852
|
+
evidenceRefs: [],
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
export async function finalizeTerminalRunStatus(state, taskCount, runDir, cwd) {
|
|
857
|
+
const repoRoot = cwd ?? path.resolve(runDir, "..", "..", "..", "..");
|
|
858
|
+
// Terminal override: a denied frontend writer must never surface as a
|
|
859
|
+
// mechanical partial_failed just because prewrite FINISHED while the writer
|
|
860
|
+
// was SKIPPED. Fail closed on retryable-invalid/blocked before aggregation.
|
|
861
|
+
let recoveryPending = false;
|
|
862
|
+
const hasFrontendWriter = Object.keys(state.nodes).some((id) => FRONTEND_WRITER_NODE_IDS.includes(id));
|
|
863
|
+
if (hasFrontendWriter) {
|
|
864
|
+
const admission = await readFrontendPrewriteResult(runDir);
|
|
865
|
+
if (admission.ok) {
|
|
866
|
+
if (admission.result.classification === "blocked") {
|
|
867
|
+
state.status = "failed";
|
|
868
|
+
return { recoveryPending: false };
|
|
869
|
+
}
|
|
870
|
+
if (admission.result.classification === "retryable-invalid") {
|
|
871
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
872
|
+
if (continuationCount >= 1) {
|
|
873
|
+
// Continuation quota exhausted: a second retryable-invalid candidate
|
|
874
|
+
// is authoritative candidate-contract-invalid → forced failed.
|
|
875
|
+
state.frontendRecoveryResult = buildCandidateContractInvalidResult(state, admission.result, admission.artifactHash);
|
|
876
|
+
state.status = "failed";
|
|
877
|
+
return { recoveryPending: false };
|
|
878
|
+
}
|
|
879
|
+
// Root continuation still available: record the recovery intent and let
|
|
880
|
+
// mechanical aggregation settle partial_failed/failed. The parent keeps
|
|
881
|
+
// phase != settled (recovery in progress) and no final root manifest.
|
|
882
|
+
state.frontendRecoveryState = buildFrontendRecoveryIntent(state);
|
|
883
|
+
recoveryPending = true;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
// Phase 5: writer transient partial write → rollback + recovery intent. Only
|
|
888
|
+
// for frontend-implementation writers with a remaining root continuation, and
|
|
889
|
+
// only when the rollback journal (captured before the provider call) restores
|
|
890
|
+
// cleanly; otherwise the run stays failed with auto-recovery-blocked.
|
|
891
|
+
if (hasFrontendWriter && !recoveryPending) {
|
|
892
|
+
const writerNodeId = FRONTEND_WRITER_NODE_IDS.find((id) => {
|
|
893
|
+
const node = state.nodes[id];
|
|
894
|
+
return (node &&
|
|
895
|
+
node.status === "ERROR" &&
|
|
896
|
+
isFrontendWriterTransientPartialWrite({
|
|
897
|
+
failureCategory: node.failureCategory,
|
|
898
|
+
}));
|
|
899
|
+
});
|
|
900
|
+
if (writerNodeId) {
|
|
901
|
+
const continuationCount = state.frontendRecoveryState?.continuationCount ?? 0;
|
|
902
|
+
if (continuationCount < 1) {
|
|
903
|
+
const { rollbackFrontendWriter } = await import("./frontend-writer-recovery.js");
|
|
904
|
+
const rollback = await rollbackFrontendWriter({
|
|
905
|
+
cwd: repoRoot,
|
|
906
|
+
parentRunId: state.runId,
|
|
907
|
+
});
|
|
908
|
+
if (rollback.ok) {
|
|
909
|
+
state.frontendRecoveryState = buildFrontendRecoveryIntent(state, "frontend-implement-pi");
|
|
910
|
+
recoveryPending = true;
|
|
911
|
+
}
|
|
912
|
+
else {
|
|
913
|
+
state.frontendRecoveryResult = buildWriterPartialWriteBlockedResult(state, writerNodeId, rollback.reason);
|
|
914
|
+
state.status = "failed";
|
|
915
|
+
return { recoveryPending: false };
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
}
|
|
699
920
|
const finishedCount = Object.values(state.nodes).filter((n) => n.status === "FINISHED").length;
|
|
700
921
|
const { supersededIds, supersededFailures } = collectSupersededIntermediateFailures(state);
|
|
701
922
|
if (state.convergence && supersededFailures.length > 0) {
|
|
@@ -719,6 +940,19 @@ function finalizeTerminalRunStatus(state, taskCount) {
|
|
|
719
940
|
else {
|
|
720
941
|
state.status = "failed";
|
|
721
942
|
}
|
|
943
|
+
// Terminal recovery bookkeeping for a child attempt: a child that finished
|
|
944
|
+
// recovered settles its own lineage; any other child terminal settles as
|
|
945
|
+
// auto-recovery-blocked (phase 3 performs no further recovery).
|
|
946
|
+
const recovery = state.frontendRecoveryState;
|
|
947
|
+
if (recovery && recovery.attemptIndex >= 1) {
|
|
948
|
+
if (state.status === "finished") {
|
|
949
|
+
settleFrontendRecovery(state, "recovered");
|
|
950
|
+
}
|
|
951
|
+
else if (!state.frontendRecoveryResult) {
|
|
952
|
+
settleFrontendRecovery(state, "auto-recovery-blocked");
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return { recoveryPending };
|
|
722
956
|
}
|
|
723
957
|
function concurrentSiblingWriteSetsForNode(rankWriterNodeIds, nodeId, tasksById) {
|
|
724
958
|
if (rankWriterNodeIds.length <= 1)
|