@tea-agent/loop-agent 0.35.1-beta.2 → 0.35.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +0 -2
- package/CHANGELOG.md +25 -24
- package/bin/loop-agent.js +1 -37
- package/dist/application/dag/generate-task-dag.js +4 -1
- package/dist/application/task-lifecycle/advance.js +14 -0
- package/dist/cli/program.js +2 -2
- package/dist/commands/task-advance.js +1 -0
- package/dist/executors/dag-pi-executor.js +0 -44
- package/dist/shared/package-metadata.js +0 -42
- package/dist/task/config-types.js +2 -0
- package/dist/task/contract/project.js +3 -0
- package/dist/task/contract/schema.js +1 -0
- package/dist/task/source-prepare/build-draft.js +7 -0
- package/dist/task/source-prepare/semantic-intake.js +37 -10
- package/dist/task/task-demand-routing.js +10 -0
- package/dist/worker/console/operator-actions.js +72 -6
- package/dist/worker/console/prd-intake-bridge.js +10 -3
- package/dist/worker/console/prd-reference-discovery.js +124 -0
- package/dist/worker/console/static/assets/{index-hJqCPs_g.css → index-HX1pbOyl.css} +1 -1
- package/dist/worker/console/static/assets/{index-CvsQgALl.js → index-M0BLEBfh.js} +25 -25
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/console/static-src/app/useOperatorActions.js +19 -1
- package/dist/worker/console/static-src/app/useRecoveryConsole.js +0 -5
- package/dist/worker/console/static-src/app/useTaskWizard.js +12 -0
- package/dist/worker/loop-agent/loop-agent-client.js +3 -17
- package/dist/worker/observability/read-model.js +0 -20
- package/dist/worker/preflight.js +1 -2
- package/dist/workflows/dag/backend-test-scenario-param.js +23 -33
- package/dist/workflows/dag/dynamic-runtime/shared.js +1 -9
- package/dist/workflows/dag/frontend-implementation-contract.js +39 -233
- package/dist/workflows/dag/frontend-prewrite-gate.js +61 -364
- package/dist/workflows/dag/frontend-repair.js +18 -219
- package/dist/workflows/dag/frontend-verification-trace.js +32 -47
- package/dist/workflows/dag/init-hybrid.js +26 -41
- package/dist/workflows/dag/node-execution.js +0 -89
- package/dist/workflows/dag/recovery-recommendation.js +0 -58
- package/dist/workflows/dag/runner.js +11 -245
- package/dist/workflows/dag/scheduler.js +3 -257
- package/dist/workflows/dag/types.js +2 -130
- package/package.json +2 -2
- package/dist/build-stamp.json +0 -6
- package/dist/workflows/dag/contract-output-registry.js +0 -14
- package/dist/workflows/dag/contract-validator-registrations.js +0 -8
- package/dist/workflows/dag/frontend-recovery-plan.js +0 -73
- package/dist/workflows/dag/frontend-recovery-root-manifest.js +0 -123
- package/dist/workflows/dag/frontend-recovery-run.js +0 -539
- package/dist/workflows/dag/frontend-writer-recovery.js +0 -106
- package/dist/workflows/dag/frontend-writer-rollback.js +0 -821
|
@@ -1,539 +0,0 @@
|
|
|
1
|
-
import { copyFile, mkdir, readdir, readFile, rm, access } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
4
|
-
import { sha256Hex } from "./frontend-implementation-contract.js";
|
|
5
|
-
import { FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH, FRONTEND_RECOVERY_INTENT_REL_DIR, FRONTEND_RECOVERY_STAGING_PREFIX, computeFrontendRecoveryPlan, computeFrontendRecoveryPlanForSource, } from "./frontend-recovery-plan.js";
|
|
6
|
-
import { dagRunDirExists, getDagRunDir, readDagRunSpec, readDagRunState, transferDagRunDir, writeDagRunState, } from "./lifecycle.js";
|
|
7
|
-
import { buildDagRunId } from "./runner.js";
|
|
8
|
-
import { prepareRunDir, writeNodeRecord } from "./run-store.js";
|
|
9
|
-
import { FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT, readFrontendPrewriteResult, } from "./scheduler.js";
|
|
10
|
-
import { topoSortToRanks } from "./topo.js";
|
|
11
|
-
/**
|
|
12
|
-
* Candidate-continuation child creation kernel (phase 3b).
|
|
13
|
-
*
|
|
14
|
-
* Parent terminal convergence (phase 3a) has already persisted a recovery
|
|
15
|
-
* intent on `DagRunState.frontendRecoveryState`. This module is the single
|
|
16
|
-
* writer that stages and commits one reserved child run per requestId, with a
|
|
17
|
-
* commit-point handoff: CAS-reserve childRunId → stage files (hash-verified) →
|
|
18
|
-
* rename into active/ → activation marker → three-way consistency check → CAS
|
|
19
|
-
* phase=child-running. Scheduling/execution of the child is phase 3c and out
|
|
20
|
-
* of scope here.
|
|
21
|
-
*/
|
|
22
|
-
/** Canonical contract artifact default location; overridden by the prewrite
|
|
23
|
-
* gate's configured outputDir/artifactName when present. */
|
|
24
|
-
const FRONTEND_CANONICAL_CONTRACT_DEFAULT_REL_PATH = "contracts/frontend-implementation-contract.json";
|
|
25
|
-
function resolveDeps(deps) {
|
|
26
|
-
return {
|
|
27
|
-
readParentState: deps?.readParentState ?? readDagRunState,
|
|
28
|
-
writeParentState: deps?.writeParentState ?? writeDagRunState,
|
|
29
|
-
writeMarker: deps?.writeMarker ??
|
|
30
|
-
((markerPath, marker) => writeJsonAtomic(markerPath, marker)),
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
function frontendRecoveryMarkerRelPath(parentRunDir, requestId) {
|
|
34
|
-
return path.join(parentRunDir, FRONTEND_RECOVERY_INTENT_REL_DIR, `${requestId}.json`);
|
|
35
|
-
}
|
|
36
|
-
async function hashFile(filePath) {
|
|
37
|
-
return sha256Hex(await readFile(filePath, "utf8"));
|
|
38
|
-
}
|
|
39
|
-
async function fileExists(filePath) {
|
|
40
|
-
try {
|
|
41
|
-
await access(filePath);
|
|
42
|
-
return true;
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
45
|
-
return false;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
function isValidFrontendRecoveryMarker(raw) {
|
|
49
|
-
return (raw.schemaVersion === 1 &&
|
|
50
|
-
typeof raw.recoveryRootRunId === "string" &&
|
|
51
|
-
raw.recoveryRootRunId.length > 0 &&
|
|
52
|
-
typeof raw.parentRunId === "string" &&
|
|
53
|
-
raw.parentRunId.length > 0 &&
|
|
54
|
-
typeof raw.childRunId === "string" &&
|
|
55
|
-
raw.childRunId.length > 0 &&
|
|
56
|
-
typeof raw.requestId === "string" &&
|
|
57
|
-
raw.requestId.length > 0 &&
|
|
58
|
-
typeof raw.importManifestSha256 === "string" &&
|
|
59
|
-
/^[a-f0-9]{64}$/.test(raw.importManifestSha256));
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Read + validate the activation marker. Missing or invalid markers return
|
|
63
|
-
* `undefined` so the phase-3c gate and tests can treat an unmarked active
|
|
64
|
-
* child as not executable (AC-2 / AC-5d).
|
|
65
|
-
*/
|
|
66
|
-
export async function readFrontendRecoveryMarker(parentRunDir, requestId) {
|
|
67
|
-
let raw;
|
|
68
|
-
try {
|
|
69
|
-
raw = JSON.parse(await readFile(frontendRecoveryMarkerRelPath(parentRunDir, requestId), "utf8"));
|
|
70
|
-
}
|
|
71
|
-
catch {
|
|
72
|
-
return undefined;
|
|
73
|
-
}
|
|
74
|
-
if (typeof raw !== "object" || raw === null)
|
|
75
|
-
return undefined;
|
|
76
|
-
if (!isValidFrontendRecoveryMarker(raw)) {
|
|
77
|
-
return undefined;
|
|
78
|
-
}
|
|
79
|
-
return raw;
|
|
80
|
-
}
|
|
81
|
-
function classifyFrontendFactKind(nodeId) {
|
|
82
|
-
if (/prewrite/i.test(nodeId))
|
|
83
|
-
return "prewrite";
|
|
84
|
-
if (/scout/i.test(nodeId))
|
|
85
|
-
return "scout";
|
|
86
|
-
if (/contract/i.test(nodeId))
|
|
87
|
-
return "contract";
|
|
88
|
-
if (/plan/i.test(nodeId))
|
|
89
|
-
return "plan";
|
|
90
|
-
return "plan";
|
|
91
|
-
}
|
|
92
|
-
function resolveCanonicalContractRelPath(spec) {
|
|
93
|
-
for (const task of spec.tasks) {
|
|
94
|
-
const gate = task.shell?.frontendPrewriteGate;
|
|
95
|
-
if (gate) {
|
|
96
|
-
return path.posix.join(gate.outputDir, gate.artifactName);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
return FRONTEND_CANONICAL_CONTRACT_DEFAULT_REL_PATH;
|
|
100
|
-
}
|
|
101
|
-
function normalizeRunRelativePath(relPath) {
|
|
102
|
-
return relPath.replace(/\\/g, "/");
|
|
103
|
-
}
|
|
104
|
-
function assertArtifactPathWithinParentRun(parentRunDir, artifactPath) {
|
|
105
|
-
if (!path.isAbsolute(artifactPath)) {
|
|
106
|
-
return normalizeRunRelativePath(artifactPath);
|
|
107
|
-
}
|
|
108
|
-
const resolved = path.resolve(artifactPath);
|
|
109
|
-
const parentResolved = path.resolve(parentRunDir);
|
|
110
|
-
if (resolved !== parentResolved &&
|
|
111
|
-
!resolved.startsWith(`${parentResolved}${path.sep}`)) {
|
|
112
|
-
throw new Error(`artifact path outside parent run directory: ${artifactPath}`);
|
|
113
|
-
}
|
|
114
|
-
return normalizeRunRelativePath(path.relative(parentResolved, resolved));
|
|
115
|
-
}
|
|
116
|
-
function collectNodeArtifactRelativePaths(record) {
|
|
117
|
-
const paths = new Set([`${record.id}.json`]);
|
|
118
|
-
const add = (value) => {
|
|
119
|
-
if (value?.trim())
|
|
120
|
-
paths.add(normalizeRunRelativePath(value.trim()));
|
|
121
|
-
};
|
|
122
|
-
add(record.stdoutArtifactPath);
|
|
123
|
-
add(record.assistantArtifactPath);
|
|
124
|
-
add(record.structuredArtifactPath);
|
|
125
|
-
add(record.nodeRecordPath);
|
|
126
|
-
add(record.escalationArtifactPath);
|
|
127
|
-
add(record.humanApprovalArtifactPath);
|
|
128
|
-
add(record.humanRejectionArtifactPath);
|
|
129
|
-
for (const attempt of record.attempts ?? []) {
|
|
130
|
-
add(attempt.artifactPath);
|
|
131
|
-
}
|
|
132
|
-
return [...paths];
|
|
133
|
-
}
|
|
134
|
-
async function listNodeDirectoryArtifacts(parentRunDir, nodeId) {
|
|
135
|
-
const nodeDir = path.join(parentRunDir, nodeId);
|
|
136
|
-
if (!(await fileExists(nodeDir)))
|
|
137
|
-
return [];
|
|
138
|
-
const entries = await readdir(nodeDir, { withFileTypes: true });
|
|
139
|
-
const artifacts = [];
|
|
140
|
-
for (const entry of entries) {
|
|
141
|
-
if (!entry.isFile())
|
|
142
|
-
continue;
|
|
143
|
-
artifacts.push(normalizeRunRelativePath(path.join(nodeId, entry.name)));
|
|
144
|
-
}
|
|
145
|
-
return artifacts;
|
|
146
|
-
}
|
|
147
|
-
async function copyArtifactVerified(input) {
|
|
148
|
-
const normalized = assertArtifactPathWithinParentRun(input.parentRunDir, input.relativePath);
|
|
149
|
-
const sourcePath = path.join(input.parentRunDir, ...normalized.split("/"));
|
|
150
|
-
if (!(await fileExists(sourcePath))) {
|
|
151
|
-
throw new Error(`missing imported artifact: ${normalized}`);
|
|
152
|
-
}
|
|
153
|
-
const expected = await hashFile(sourcePath);
|
|
154
|
-
const destinationPath = path.join(input.newRunDir, ...normalized.split("/"));
|
|
155
|
-
await mkdir(path.dirname(destinationPath), { recursive: true });
|
|
156
|
-
await copyFile(sourcePath, destinationPath);
|
|
157
|
-
const actual = await hashFile(destinationPath);
|
|
158
|
-
if (actual !== expected) {
|
|
159
|
-
throw new Error(`copy hash mismatch for artifact: ${normalized}`);
|
|
160
|
-
}
|
|
161
|
-
return { destinationRelativePath: normalized, sha256: actual };
|
|
162
|
-
}
|
|
163
|
-
function normalizeImportedArtifactPath(parentRunDir, newRunDir, artifactPath) {
|
|
164
|
-
if (!artifactPath?.trim())
|
|
165
|
-
return artifactPath;
|
|
166
|
-
const relative = assertArtifactPathWithinParentRun(parentRunDir, artifactPath);
|
|
167
|
-
return path.join(newRunDir, ...relative.split("/"));
|
|
168
|
-
}
|
|
169
|
-
function buildImportedNodeRecord(parentRecord, parentRunId, sourceNodeRecordSha256, parentRunDir, newRunDir) {
|
|
170
|
-
const { origin: _origin, ...rest } = parentRecord;
|
|
171
|
-
void _origin;
|
|
172
|
-
const imported = structuredClone(rest);
|
|
173
|
-
imported.nodeRecordPath = path.join(newRunDir, `${parentRecord.id}.json`);
|
|
174
|
-
for (const key of [
|
|
175
|
-
"stdoutArtifactPath",
|
|
176
|
-
"assistantArtifactPath",
|
|
177
|
-
"structuredArtifactPath",
|
|
178
|
-
"escalationArtifactPath",
|
|
179
|
-
"humanApprovalArtifactPath",
|
|
180
|
-
"humanRejectionArtifactPath",
|
|
181
|
-
]) {
|
|
182
|
-
imported[key] = normalizeImportedArtifactPath(parentRunDir, newRunDir, imported[key]);
|
|
183
|
-
}
|
|
184
|
-
return {
|
|
185
|
-
...imported,
|
|
186
|
-
origin: {
|
|
187
|
-
kind: "imported",
|
|
188
|
-
parentRunId,
|
|
189
|
-
sourceNodeRecordSha256,
|
|
190
|
-
},
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
function buildPendingNodeRecord(task) {
|
|
194
|
-
return {
|
|
195
|
-
id: task.id,
|
|
196
|
-
status: "PENDING",
|
|
197
|
-
executor: task.executor,
|
|
198
|
-
complexity: task.complexity,
|
|
199
|
-
origin: { kind: "executed" },
|
|
200
|
-
...(task.outputMode ? { outputMode: task.outputMode } : {}),
|
|
201
|
-
};
|
|
202
|
-
}
|
|
203
|
-
async function importNodeFacts(input) {
|
|
204
|
-
const nodeJsonRel = `${input.nodeId}.json`;
|
|
205
|
-
const parentNodeJsonPath = path.join(input.parentRunDir, nodeJsonRel);
|
|
206
|
-
const sourceNodeRecordSha256 = (await fileExists(parentNodeJsonPath))
|
|
207
|
-
? await hashFile(parentNodeJsonPath)
|
|
208
|
-
: sha256Hex(`${JSON.stringify(input.parentRecord, null, 2)}\n`);
|
|
209
|
-
const importedRecord = buildImportedNodeRecord(input.parentRecord, input.parentRunId, sourceNodeRecordSha256, input.parentRunDir, input.newRunDir);
|
|
210
|
-
await writeNodeRecord(input.newRunDir, input.nodeId, importedRecord);
|
|
211
|
-
const relativePaths = [
|
|
212
|
-
...new Set([
|
|
213
|
-
...collectNodeArtifactRelativePaths(input.parentRecord).filter((relativePath) => relativePath !== nodeJsonRel),
|
|
214
|
-
...(await listNodeDirectoryArtifacts(input.parentRunDir, input.nodeId)),
|
|
215
|
-
]),
|
|
216
|
-
];
|
|
217
|
-
const artifacts = [
|
|
218
|
-
{
|
|
219
|
-
sourceRelativePath: nodeJsonRel,
|
|
220
|
-
destinationRelativePath: nodeJsonRel,
|
|
221
|
-
sha256: await hashFile(path.join(input.newRunDir, nodeJsonRel)),
|
|
222
|
-
},
|
|
223
|
-
];
|
|
224
|
-
for (const relativePath of relativePaths.sort()) {
|
|
225
|
-
const copied = await copyArtifactVerified({
|
|
226
|
-
parentRunDir: input.parentRunDir,
|
|
227
|
-
newRunDir: input.newRunDir,
|
|
228
|
-
relativePath,
|
|
229
|
-
});
|
|
230
|
-
artifacts.push({
|
|
231
|
-
sourceRelativePath: relativePath,
|
|
232
|
-
destinationRelativePath: copied.destinationRelativePath,
|
|
233
|
-
sha256: copied.sha256,
|
|
234
|
-
});
|
|
235
|
-
}
|
|
236
|
-
return {
|
|
237
|
-
fact: {
|
|
238
|
-
nodeId: input.nodeId,
|
|
239
|
-
kind: classifyFrontendFactKind(input.nodeId),
|
|
240
|
-
artifacts,
|
|
241
|
-
},
|
|
242
|
-
importedRecord,
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
/** Write JSON atomically, then re-read the bytes and verify the hash matches
|
|
246
|
-
* the exact serialization we intended (AC-1 per-file hash verification). */
|
|
247
|
-
async function writeJsonVerified(runDir, relPath, value) {
|
|
248
|
-
const expectedJson = `${JSON.stringify(value, null, 2)}\n`;
|
|
249
|
-
const targetPath = path.join(runDir, ...relPath.split("/"));
|
|
250
|
-
await writeJsonAtomic(targetPath, value);
|
|
251
|
-
const actual = await hashFile(targetPath);
|
|
252
|
-
const expected = sha256Hex(expectedJson);
|
|
253
|
-
if (actual !== expected) {
|
|
254
|
-
throw new Error(`hash verification failed after writing ${relPath}`);
|
|
255
|
-
}
|
|
256
|
-
return actual;
|
|
257
|
-
}
|
|
258
|
-
async function casUpdateFrontendRecoveryState(parentRunDir, expectedRevision, deps, mutate) {
|
|
259
|
-
const state = await deps.readParentState(parentRunDir);
|
|
260
|
-
const recovery = state.frontendRecoveryState;
|
|
261
|
-
if (!recovery) {
|
|
262
|
-
throw new Error("parent frontend recovery intent missing during CAS update");
|
|
263
|
-
}
|
|
264
|
-
if (recovery.revision !== expectedRevision) {
|
|
265
|
-
throw new Error(`frontend recovery revision drifted (expected ${expectedRevision}, found ${recovery.revision}); refuse to continue`);
|
|
266
|
-
}
|
|
267
|
-
mutate(recovery);
|
|
268
|
-
recovery.revision = expectedRevision + 1;
|
|
269
|
-
state.frontendRecoveryState = recovery;
|
|
270
|
-
await deps.writeParentState(parentRunDir, state);
|
|
271
|
-
}
|
|
272
|
-
async function materializeChildStaging(input) {
|
|
273
|
-
const { cwd, parentRunDir, parentRunId, parentSpec, parentState, recovery, requestId, childRunId, stagingDir, createdAt, } = input;
|
|
274
|
-
await prepareRunDir(stagingDir);
|
|
275
|
-
await writeJsonVerified(stagingDir, "run.json", parentSpec);
|
|
276
|
-
// Phase 5: writer transient partial-write recovery resets the writer subtree
|
|
277
|
-
// (frontend-implement-pi) instead of the candidate producer; everything
|
|
278
|
-
// upstream is imported as verified parent facts.
|
|
279
|
-
let plan;
|
|
280
|
-
if (recovery.failureSource) {
|
|
281
|
-
plan = computeFrontendRecoveryPlanForSource({
|
|
282
|
-
spec: parentSpec,
|
|
283
|
-
parentRunId,
|
|
284
|
-
requestId,
|
|
285
|
-
failureSource: recovery.failureSource,
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
else {
|
|
289
|
-
const prewriteRead = await readFrontendPrewriteResult(parentRunDir);
|
|
290
|
-
if (!prewriteRead.ok) {
|
|
291
|
-
throw new Error(prewriteRead.reason);
|
|
292
|
-
}
|
|
293
|
-
plan = computeFrontendRecoveryPlan({
|
|
294
|
-
spec: parentSpec,
|
|
295
|
-
parentRunId,
|
|
296
|
-
requestId,
|
|
297
|
-
result: prewriteRead.result,
|
|
298
|
-
});
|
|
299
|
-
}
|
|
300
|
-
const { ranks } = topoSortToRanks(parentSpec);
|
|
301
|
-
const resetSet = new Set(plan.resetNodeIds);
|
|
302
|
-
const importedSet = new Set(plan.importedNodeIds);
|
|
303
|
-
const tasksById = new Map(parentSpec.tasks.map((task) => [task.id, task]));
|
|
304
|
-
const nodes = {};
|
|
305
|
-
const importedFacts = [];
|
|
306
|
-
for (const nodeId of plan.importedNodeIds) {
|
|
307
|
-
const parentRecord = parentState.nodes[nodeId];
|
|
308
|
-
if (!parentRecord) {
|
|
309
|
-
throw new Error(`parent node record missing for imported node: ${nodeId}`);
|
|
310
|
-
}
|
|
311
|
-
const { fact, importedRecord } = await importNodeFacts({
|
|
312
|
-
parentRunDir,
|
|
313
|
-
newRunDir: stagingDir,
|
|
314
|
-
parentRunId,
|
|
315
|
-
nodeId,
|
|
316
|
-
parentRecord,
|
|
317
|
-
});
|
|
318
|
-
importedFacts.push(fact);
|
|
319
|
-
nodes[nodeId] = importedRecord;
|
|
320
|
-
}
|
|
321
|
-
for (const nodeId of plan.resetNodeIds) {
|
|
322
|
-
const task = tasksById.get(nodeId);
|
|
323
|
-
if (!task) {
|
|
324
|
-
throw new Error(`reset node missing from spec: ${nodeId}`);
|
|
325
|
-
}
|
|
326
|
-
nodes[nodeId] = buildPendingNodeRecord(task);
|
|
327
|
-
}
|
|
328
|
-
for (const task of parentSpec.tasks) {
|
|
329
|
-
if (!resetSet.has(task.id) && !importedSet.has(task.id)) {
|
|
330
|
-
throw new Error(`node ${task.id} missing from frontend recovery plan partition`);
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
// Canonical contract + prewrite-result identity facts are copied alongside
|
|
334
|
-
// the per-node facts so reset nodes can re-read them (AC-4).
|
|
335
|
-
const canonicalContract = await copyArtifactVerified({
|
|
336
|
-
parentRunDir,
|
|
337
|
-
newRunDir: stagingDir,
|
|
338
|
-
relativePath: resolveCanonicalContractRelPath(parentSpec),
|
|
339
|
-
});
|
|
340
|
-
const prewriteResult = await copyArtifactVerified({
|
|
341
|
-
parentRunDir,
|
|
342
|
-
newRunDir: stagingDir,
|
|
343
|
-
relativePath: FRONTEND_PREWRITE_RESULT_SOURCE_ARTIFACT,
|
|
344
|
-
});
|
|
345
|
-
const childState = {
|
|
346
|
-
version: 1,
|
|
347
|
-
title: parentSpec.title,
|
|
348
|
-
runId: childRunId,
|
|
349
|
-
cwd,
|
|
350
|
-
startedAt: createdAt,
|
|
351
|
-
status: "pending",
|
|
352
|
-
ranks,
|
|
353
|
-
nodes,
|
|
354
|
-
frontendRecoveryState: {
|
|
355
|
-
schemaVersion: 1,
|
|
356
|
-
phase: "child-running",
|
|
357
|
-
requestId,
|
|
358
|
-
recoveryRootRunId: recovery.recoveryRootRunId,
|
|
359
|
-
parentRunId,
|
|
360
|
-
childRunId,
|
|
361
|
-
attemptId: recovery.attemptId,
|
|
362
|
-
attemptIndex: 1,
|
|
363
|
-
continuationCount: 1,
|
|
364
|
-
revision: 0,
|
|
365
|
-
},
|
|
366
|
-
...(parentState.evaluation
|
|
367
|
-
? { evaluation: structuredClone(parentState.evaluation) }
|
|
368
|
-
: {}),
|
|
369
|
-
};
|
|
370
|
-
await writeJsonVerified(stagingDir, "state.json", childState);
|
|
371
|
-
const importManifest = {
|
|
372
|
-
schemaVersion: 1,
|
|
373
|
-
parentRunId,
|
|
374
|
-
requestId,
|
|
375
|
-
createdAt,
|
|
376
|
-
canonicalContract: {
|
|
377
|
-
relativePath: canonicalContract.destinationRelativePath,
|
|
378
|
-
sha256: canonicalContract.sha256,
|
|
379
|
-
},
|
|
380
|
-
prewriteResult: {
|
|
381
|
-
relativePath: prewriteResult.destinationRelativePath,
|
|
382
|
-
sha256: prewriteResult.sha256,
|
|
383
|
-
},
|
|
384
|
-
importedFacts,
|
|
385
|
-
};
|
|
386
|
-
await writeJsonVerified(stagingDir, FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH, importManifest);
|
|
387
|
-
}
|
|
388
|
-
async function verifyThreeWayConsistency(input) {
|
|
389
|
-
const parentState = await input.deps.readParentState(input.parentRunDir);
|
|
390
|
-
const childState = await readDagRunState(input.activeChildDir);
|
|
391
|
-
const parentRecovery = parentState.frontendRecoveryState;
|
|
392
|
-
if (!parentRecovery || parentRecovery.childRunId !== input.childRunId) {
|
|
393
|
-
throw new Error("three-way consistency failure: parent childRunId mismatch");
|
|
394
|
-
}
|
|
395
|
-
if (childState.runId !== input.childRunId) {
|
|
396
|
-
throw new Error("three-way consistency failure: child state runId mismatch");
|
|
397
|
-
}
|
|
398
|
-
if (input.marker.childRunId !== input.childRunId) {
|
|
399
|
-
throw new Error("three-way consistency failure: marker childRunId mismatch");
|
|
400
|
-
}
|
|
401
|
-
if (input.marker.requestId !== input.requestId) {
|
|
402
|
-
throw new Error("three-way consistency failure: marker requestId mismatch");
|
|
403
|
-
}
|
|
404
|
-
if (input.marker.parentRunId !== input.parentRunId) {
|
|
405
|
-
throw new Error("three-way consistency failure: marker parentRunId mismatch");
|
|
406
|
-
}
|
|
407
|
-
if (input.marker.recoveryRootRunId !== input.recoveryRootRunId) {
|
|
408
|
-
throw new Error("three-way consistency failure: marker recoveryRootRunId mismatch");
|
|
409
|
-
}
|
|
410
|
-
const actualManifestSha256 = await hashFile(path.join(input.activeChildDir, FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH));
|
|
411
|
-
if (actualManifestSha256 !== input.marker.importManifestSha256) {
|
|
412
|
-
throw new Error("three-way consistency failure: import manifest sha256 mismatch");
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
/**
|
|
416
|
-
* Stage and commit one reserved child run for a parent frontend recovery
|
|
417
|
-
* intent. Single-writer: the recovery coordinator is the only caller.
|
|
418
|
-
* Fail-closed on revision drift, missing intent, unreadable prewrite result,
|
|
419
|
-
* hash mismatch, marker-write failure, or consistency failure.
|
|
420
|
-
*/
|
|
421
|
-
export async function stageRecoveryChild(options) {
|
|
422
|
-
const { cwd, parentRunId } = options;
|
|
423
|
-
const now = options.now ?? new Date();
|
|
424
|
-
const createdAt = now.toISOString();
|
|
425
|
-
const deps = resolveDeps(options.dependencies);
|
|
426
|
-
const parentRunDir = getDagRunDir(cwd, "active", parentRunId);
|
|
427
|
-
const parentState = await deps.readParentState(parentRunDir);
|
|
428
|
-
const parentSpec = await readDagRunSpec(parentRunDir);
|
|
429
|
-
const recovery = parentState.frontendRecoveryState;
|
|
430
|
-
if (!recovery) {
|
|
431
|
-
throw new Error(`parent run ${parentRunId} has no frontend recovery intent`);
|
|
432
|
-
}
|
|
433
|
-
const requestId = recovery.requestId;
|
|
434
|
-
if (!requestId) {
|
|
435
|
-
throw new Error(`parent run ${parentRunId} recovery intent has no requestId`);
|
|
436
|
-
}
|
|
437
|
-
// Idempotent short-circuit (D4.2): reserved childRunId + activated marker +
|
|
438
|
-
// existing child dir means the request was already fully committed.
|
|
439
|
-
if (recovery.childRunId) {
|
|
440
|
-
const existingMarker = await readFrontendRecoveryMarker(parentRunDir, requestId);
|
|
441
|
-
if (existingMarker) {
|
|
442
|
-
const existingChildDir = getDagRunDir(cwd, "active", existingMarker.childRunId);
|
|
443
|
-
if (await dagRunDirExists(existingChildDir)) {
|
|
444
|
-
return {
|
|
445
|
-
ok: true,
|
|
446
|
-
childRunId: existingMarker.childRunId,
|
|
447
|
-
childRunDir: existingChildDir,
|
|
448
|
-
marker: existingMarker,
|
|
449
|
-
idempotentReplay: true,
|
|
450
|
-
};
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
// Derive the childRunId exactly once; replay always reuses a reserved id.
|
|
455
|
-
const childRunId = recovery.childRunId ??
|
|
456
|
-
(await (options.buildChildRunId ?? buildDagRunId)(parentSpec, cwd));
|
|
457
|
-
// Orphan staging cleanup (D4.3): remove any in-flight staging dir for this id.
|
|
458
|
-
const activeRootDir = getDagRunDir(cwd, "active", "");
|
|
459
|
-
const stagingDir = path.join(activeRootDir, `${FRONTEND_RECOVERY_STAGING_PREFIX}${childRunId}`);
|
|
460
|
-
await rm(stagingDir, { recursive: true, force: true });
|
|
461
|
-
// CAS #1: reserve childRunId (phase stays child-staging) before any child
|
|
462
|
-
// files exist, so a crash can never produce two children for one requestId.
|
|
463
|
-
if (!recovery.childRunId) {
|
|
464
|
-
await casUpdateFrontendRecoveryState(parentRunDir, recovery.revision, deps, (rec) => {
|
|
465
|
-
rec.childRunId = childRunId;
|
|
466
|
-
rec.phase = "child-staging";
|
|
467
|
-
});
|
|
468
|
-
}
|
|
469
|
-
const parentAfterReserve = await deps.readParentState(parentRunDir);
|
|
470
|
-
const reserved = parentAfterReserve.frontendRecoveryState;
|
|
471
|
-
if (!reserved || reserved.childRunId !== childRunId) {
|
|
472
|
-
throw new Error("frontend recovery reservation drift after CAS; refuse to continue");
|
|
473
|
-
}
|
|
474
|
-
const activeChildDir = getDagRunDir(cwd, "active", childRunId);
|
|
475
|
-
const childDirExists = await dagRunDirExists(activeChildDir);
|
|
476
|
-
let importManifestSha256;
|
|
477
|
-
if (childDirExists) {
|
|
478
|
-
// Crash between rename and marker write: the child is already fully
|
|
479
|
-
// materialized. Recover by rewriting the marker instead of rebuilding.
|
|
480
|
-
importManifestSha256 = await hashFile(path.join(activeChildDir, FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH));
|
|
481
|
-
}
|
|
482
|
-
else {
|
|
483
|
-
await materializeChildStaging({
|
|
484
|
-
cwd,
|
|
485
|
-
parentRunDir,
|
|
486
|
-
parentRunId,
|
|
487
|
-
parentSpec,
|
|
488
|
-
parentState,
|
|
489
|
-
recovery,
|
|
490
|
-
requestId,
|
|
491
|
-
childRunId,
|
|
492
|
-
stagingDir,
|
|
493
|
-
createdAt,
|
|
494
|
-
});
|
|
495
|
-
await transferDagRunDir(stagingDir, activeChildDir);
|
|
496
|
-
importManifestSha256 = await hashFile(path.join(activeChildDir, FRONTEND_RECOVERY_IMPORT_MANIFEST_REL_PATH));
|
|
497
|
-
}
|
|
498
|
-
const marker = {
|
|
499
|
-
schemaVersion: 1,
|
|
500
|
-
requestId,
|
|
501
|
-
recoveryRootRunId: recovery.recoveryRootRunId,
|
|
502
|
-
parentRunId,
|
|
503
|
-
childRunId,
|
|
504
|
-
importManifestSha256,
|
|
505
|
-
createdAt,
|
|
506
|
-
activatedAt: createdAt,
|
|
507
|
-
};
|
|
508
|
-
const markerPath = frontendRecoveryMarkerRelPath(parentRunDir, requestId);
|
|
509
|
-
try {
|
|
510
|
-
await deps.writeMarker(markerPath, marker);
|
|
511
|
-
await verifyThreeWayConsistency({
|
|
512
|
-
parentRunDir,
|
|
513
|
-
activeChildDir,
|
|
514
|
-
marker,
|
|
515
|
-
parentRunId,
|
|
516
|
-
requestId,
|
|
517
|
-
recoveryRootRunId: recovery.recoveryRootRunId,
|
|
518
|
-
childRunId,
|
|
519
|
-
deps,
|
|
520
|
-
});
|
|
521
|
-
await casUpdateFrontendRecoveryState(parentRunDir, reserved.revision, deps, (rec) => {
|
|
522
|
-
rec.phase = "child-running";
|
|
523
|
-
});
|
|
524
|
-
}
|
|
525
|
-
catch (error) {
|
|
526
|
-
// Best-effort cleanup so no marker-less active child survives. If cleanup
|
|
527
|
-
// fails, the unmarked child remains non-executable (AC-2).
|
|
528
|
-
await rm(activeChildDir, { recursive: true, force: true }).catch(() => { });
|
|
529
|
-
await rm(stagingDir, { recursive: true, force: true }).catch(() => { });
|
|
530
|
-
throw error;
|
|
531
|
-
}
|
|
532
|
-
return {
|
|
533
|
-
ok: true,
|
|
534
|
-
childRunId,
|
|
535
|
-
childRunDir: activeChildDir,
|
|
536
|
-
marker,
|
|
537
|
-
idempotentReplay: false,
|
|
538
|
-
};
|
|
539
|
-
}
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import path from "node:path";
|
|
2
|
-
import { readFile } from "node:fs/promises";
|
|
3
|
-
import { getDagRunDir } from "./lifecycle.js";
|
|
4
|
-
import { captureWriterBaseline, readJournalFile, restoreJournal, sha256HexBytes, writeJournalFile, } from "./frontend-writer-rollback.js";
|
|
5
|
-
/**
|
|
6
|
-
* Phase 5: writer transient partial-write recovery — compose the phase-4 rollback
|
|
7
|
-
* journal with the phase-3 staged child handoff. A frontend writer attempt is
|
|
8
|
-
* snapshotted before the provider call so a transient partial write can be rolled
|
|
9
|
-
* back to the pre-attempt baseline, then recovered by a staged child that re-runs
|
|
10
|
-
* `frontend-implement-pi`.
|
|
11
|
-
*
|
|
12
|
-
* The runner remains the single-writer coordinator: it captures the baseline via
|
|
13
|
-
* {@link captureFrontendWriterAttemptIntent} before the writer provider call, and
|
|
14
|
-
* on a transient partial write runs {@link rollbackFrontendWriter} before writing
|
|
15
|
-
* the recovery intent and staging the child.
|
|
16
|
-
*/
|
|
17
|
-
/** Rollback intent path inside a run dir (written before the writer provider call). */
|
|
18
|
-
export const FRONTEND_WRITER_ROLLBACK_INTENT_REL = ".runtime/frontend-writer-rollback.json";
|
|
19
|
-
export function frontendWriterRollbackIntentAbsPath(runDir) {
|
|
20
|
-
return path.join(runDir, FRONTEND_WRITER_ROLLBACK_INTENT_REL);
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* AC/P1-9: capture the writeSet baseline and persist the attempt intent BEFORE
|
|
24
|
-
* the writer provider call. Returns the intent path. Callers must fail the
|
|
25
|
-
* attempt closed if this throws (no baseline → no rollback → no recovery).
|
|
26
|
-
*/
|
|
27
|
-
export async function captureFrontendWriterAttemptIntent(input) {
|
|
28
|
-
const baseline = await captureWriterBaseline(input.cwd, input.writeSet);
|
|
29
|
-
const intentPath = frontendWriterRollbackIntentAbsPath(input.runDir);
|
|
30
|
-
await writeJournalFile(intentPath, baseline);
|
|
31
|
-
return intentPath;
|
|
32
|
-
}
|
|
33
|
-
/**
|
|
34
|
-
* Record the writer attempt's changed paths and per-path attempt hashes into the
|
|
35
|
-
* rollback intent AFTER the provider call. This is what makes restoreJournal able
|
|
36
|
-
* to CAS-restore: it compares each changed path's current hash against the
|
|
37
|
-
* recorded attempt hash before writing back the baseline.
|
|
38
|
-
*/
|
|
39
|
-
export async function recordFrontendWriterAttempt(input) {
|
|
40
|
-
const intentPath = frontendWriterRollbackIntentAbsPath(input.runDir);
|
|
41
|
-
const snapshot = await readJournalFile(intentPath);
|
|
42
|
-
if (!snapshot)
|
|
43
|
-
return; // no baseline was captured; nothing to record
|
|
44
|
-
const changed = [...new Set(input.changedPaths)].sort((a, b) => a.localeCompare(b));
|
|
45
|
-
const fileHashes = {};
|
|
46
|
-
for (const rel of changed) {
|
|
47
|
-
try {
|
|
48
|
-
const bytes = await readFile(path.join(input.cwd, rel));
|
|
49
|
-
fileHashes[rel] = sha256HexBytes(bytes);
|
|
50
|
-
}
|
|
51
|
-
catch {
|
|
52
|
-
fileHashes[rel] = "absent";
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
snapshot.attemptChangedPaths = changed;
|
|
56
|
-
snapshot.attemptFileHashes = fileHashes;
|
|
57
|
-
await writeJournalFile(intentPath, snapshot);
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Restore the parent workspace to its pre-attempt baseline using the phase-4 CAS
|
|
61
|
-
* journal. This must run BEFORE the recovery intent is written and the child is
|
|
62
|
-
* staged (rollback commit point): a blocked rollback short-circuits to
|
|
63
|
-
* `auto-recovery-blocked` and never stages a child.
|
|
64
|
-
*/
|
|
65
|
-
export async function rollbackFrontendWriter(input) {
|
|
66
|
-
const parentRunDir = getDagRunDir(input.cwd, "active", input.parentRunId);
|
|
67
|
-
const snapshot = await readJournalFile(frontendWriterRollbackIntentAbsPath(parentRunDir));
|
|
68
|
-
if (!snapshot) {
|
|
69
|
-
return {
|
|
70
|
-
ok: false,
|
|
71
|
-
state: "missing-intent",
|
|
72
|
-
reason: "auto-recovery-blocked: writer rollback intent missing (baseline was not captured before the provider call)",
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
const result = await restoreJournal(snapshot, { repoRoot: input.cwd });
|
|
76
|
-
if (result.state !== "completed") {
|
|
77
|
-
return {
|
|
78
|
-
ok: false,
|
|
79
|
-
state: "blocked",
|
|
80
|
-
reason: result.blockedReason ?? "auto-recovery-blocked",
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
return {
|
|
84
|
-
ok: true,
|
|
85
|
-
state: "completed",
|
|
86
|
-
restoredPaths: result.restoredPaths,
|
|
87
|
-
snapshot,
|
|
88
|
-
};
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Transient failure categories that indicate a writer may have partially written
|
|
92
|
-
* before the attempt was interrupted. Clean-timeout (zero writes) is handled by
|
|
93
|
-
* the retry policy and never reaches terminal aggregation as a recovery trigger.
|
|
94
|
-
*/
|
|
95
|
-
const TRANSIENT_WRITER_FAILURE_CATEGORIES = new Set([
|
|
96
|
-
"timeout",
|
|
97
|
-
"network",
|
|
98
|
-
"rate-limit",
|
|
99
|
-
"unavailable",
|
|
100
|
-
"terminated",
|
|
101
|
-
"disconnect",
|
|
102
|
-
"provider-error",
|
|
103
|
-
]);
|
|
104
|
-
export function isFrontendWriterTransientPartialWrite(input) {
|
|
105
|
-
return TRANSIENT_WRITER_FAILURE_CATEGORIES.has(input.failureCategory ?? "");
|
|
106
|
-
}
|