@tea-agent/loop-agent 0.20.1-beta.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/dist/application/dag/args.js +29 -0
- package/dist/application/dag/run-dag.js +3 -1
- package/dist/cli/command-definitions.js +15 -1
- package/dist/cli/program.js +11 -1
- package/dist/commands/dag-rerun-task.js +19 -0
- package/dist/commands/dag-rerun.js +111 -0
- package/dist/commands/init.js +7 -0
- package/dist/executors/dag-pi-executor.js +24 -0
- package/dist/executors/pi-executor.js +111 -36
- package/dist/executors/pi-sdk-executor.js +105 -29
- package/dist/executors/shell-executor.js +54 -11
- package/dist/shared/operator/capabilities.js +54 -0
- package/dist/worker/console/index.js +1 -1
- package/dist/worker/console/inspect-split.js +82 -0
- package/dist/worker/console/operation-runner.js +3 -1
- package/dist/worker/console/operation-store.js +1 -0
- package/dist/worker/console/operator-actions.js +153 -2
- package/dist/worker/console/operator-user-error.js +10 -0
- package/dist/worker/console/pi-readiness.js +4 -0
- package/dist/worker/console/recovery-cta.js +116 -5
- package/dist/worker/console/recovery-selection.js +107 -0
- package/dist/worker/console/resolve-dag-run-for-task.js +50 -11
- package/dist/worker/console/routes.js +20 -0
- package/dist/worker/console/sibling-controller.js +12 -7
- package/dist/worker/console/static/assets/index-CUDke82y.js +18 -0
- package/dist/worker/console/static/assets/index-wSEksVSO.css +1 -0
- package/dist/worker/console/static/index.html +2 -2
- package/dist/worker/loop-agent/loop-agent-client.js +43 -9
- package/dist/worker/observability/read-model.js +67 -1
- package/dist/worker/observe/static/constants.js +5 -0
- package/dist/worker/observe/static/format-pool.js +22 -3
- package/dist/worker/observe/static/index.html +1 -1
- package/dist/worker/observe/static/styles.css +32 -3
- package/dist/worker/observe/static/views/dag-inspector.js +2 -2
- package/dist/worker/run-task/run-task.js +23 -6
- package/dist/workflows/dag/backend-test-markdown-workflow.js +291 -97
- package/dist/workflows/dag/backend-test-result-contract.js +10 -4
- package/dist/workflows/dag/frontend-test-result-contract.js +64 -0
- package/dist/workflows/dag/init-hybrid.js +117 -64
- package/dist/workflows/dag/lifecycle.js +60 -4
- package/dist/workflows/dag/liveness-policy.js +250 -0
- package/dist/workflows/dag/node-execution.js +89 -6
- package/dist/workflows/dag/output-protocol.js +76 -0
- package/dist/workflows/dag/rerun-plan.js +611 -0
- package/dist/workflows/dag/rerun-run.js +497 -0
- package/dist/workflows/dag/rerun-task.js +284 -0
- package/dist/workflows/dag/retry-policy.js +20 -1
- package/dist/workflows/dag/runner.js +71 -1
- package/dist/workflows/dag/skill-snapshot.js +22 -3
- package/dist/workflows/dag/types.js +12 -0
- package/dist/workflows/dag/validate.js +11 -0
- package/dist/workflows/dag/workspace-checkpoint.js +163 -0
- package/docs/README.md +5 -5
- package/docs/architecture/dag-execution.md +11 -0
- package/docs/architecture/facts-and-state.md +1 -0
- package/docs/architecture/worker-and-feature.md +10 -0
- package/docs/templates/agent-dag.schema.json +17 -2
- package/docs/templates/backend-test-dag.generate-pytest.prompt.md +5 -2
- package/docs/templates/backend-test-dag.json +15 -15
- package/docs/templates/frontend-test-case-checklist.md +16 -1
- package/docs/templates/frontend-test-dag.generate-cases.prompt.md +26 -2
- package/docs/templates/frontend-test-dag.json +65 -6
- package/docs/templates/frontend-test-dag.retrieve-context.prompt.md +4 -1
- package/harness.json +1 -1
- package/package.json +1 -1
- package/skills/loop-agent/references/command-reference.md +5 -0
- package/skills/loop-agent/references/hybrid-dag.md +2 -2
- package/skills/playwright-cli-case-generator/SKILL.md +35 -7
- package/dist/worker/console/static/assets/index-3vsjZJHq.js +0 -16
- package/dist/worker/console/static/assets/index-i1wV4LrY.css +0 -1
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { access, readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { generateTaskDagUseCase } from "../../application/dag/generate-task-dag.js";
|
|
4
|
+
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
5
|
+
import { DAG_RUNS_DIR, isTerminalDagRunStatus, locateDagRun, readDagRunSpec, readDagRunState, } from "./lifecycle.js";
|
|
6
|
+
export const RERUN_TASK_LINEAGE_REL_PATH = ".runtime/rerun-task-lineage.json";
|
|
7
|
+
export function parseDagRerunTaskArgs(args) {
|
|
8
|
+
if (args.length === 0) {
|
|
9
|
+
throw new Error("usage: dag rerun-task --run-id <id> --reason <text> --request-id <key> [--profile auto|minimal|standard|reviewed|supervised] [--task-id <id>] [--json]");
|
|
10
|
+
}
|
|
11
|
+
let parentRunId;
|
|
12
|
+
let reason;
|
|
13
|
+
let requestId;
|
|
14
|
+
let profile = "auto";
|
|
15
|
+
let profileExplicit = false;
|
|
16
|
+
let taskIdOverride;
|
|
17
|
+
let execute = true;
|
|
18
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
19
|
+
const arg = args[index];
|
|
20
|
+
if (arg === "--run-id")
|
|
21
|
+
parentRunId = args[++index];
|
|
22
|
+
else if (arg.startsWith("--run-id="))
|
|
23
|
+
parentRunId = arg.slice("--run-id=".length);
|
|
24
|
+
else if (arg === "--reason")
|
|
25
|
+
reason = args[++index];
|
|
26
|
+
else if (arg.startsWith("--reason="))
|
|
27
|
+
reason = arg.slice("--reason=".length);
|
|
28
|
+
else if (arg === "--request-id")
|
|
29
|
+
requestId = args[++index];
|
|
30
|
+
else if (arg.startsWith("--request-id="))
|
|
31
|
+
requestId = arg.slice("--request-id=".length);
|
|
32
|
+
else if (arg === "--profile") {
|
|
33
|
+
profile = parseDagRerunTaskProfile(args[++index]);
|
|
34
|
+
profileExplicit = true;
|
|
35
|
+
}
|
|
36
|
+
else if (arg.startsWith("--profile=")) {
|
|
37
|
+
profile = parseDagRerunTaskProfile(arg.slice("--profile=".length));
|
|
38
|
+
profileExplicit = true;
|
|
39
|
+
}
|
|
40
|
+
else if (arg === "--task-id")
|
|
41
|
+
taskIdOverride = args[++index];
|
|
42
|
+
else if (arg.startsWith("--task-id="))
|
|
43
|
+
taskIdOverride = arg.slice("--task-id=".length);
|
|
44
|
+
else if (arg === "--json")
|
|
45
|
+
continue;
|
|
46
|
+
else if (arg === "--no-execute")
|
|
47
|
+
execute = false;
|
|
48
|
+
else if (arg.startsWith("-"))
|
|
49
|
+
throw new Error(`unknown dag rerun-task flag: ${arg}`);
|
|
50
|
+
else
|
|
51
|
+
throw new Error(`unexpected positional argument: ${arg}`);
|
|
52
|
+
}
|
|
53
|
+
if (!parentRunId) {
|
|
54
|
+
throw new Error("dag rerun-task requires --run-id <id>");
|
|
55
|
+
}
|
|
56
|
+
if (!reason?.trim()) {
|
|
57
|
+
throw new Error("dag rerun-task requires --reason <text>");
|
|
58
|
+
}
|
|
59
|
+
if (!requestId?.trim()) {
|
|
60
|
+
throw new Error("dag rerun-task requires --request-id <key>");
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
parentRunId,
|
|
64
|
+
reason: reason.trim(),
|
|
65
|
+
requestId: requestId.trim(),
|
|
66
|
+
profile,
|
|
67
|
+
profileExplicit,
|
|
68
|
+
...(taskIdOverride?.trim() ? { taskIdOverride: taskIdOverride.trim() } : {}),
|
|
69
|
+
execute,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function parseDagRerunTaskProfile(value) {
|
|
73
|
+
if (value === "auto" ||
|
|
74
|
+
value === "minimal" ||
|
|
75
|
+
value === "standard" ||
|
|
76
|
+
value === "reviewed" ||
|
|
77
|
+
value === "supervised") {
|
|
78
|
+
return value;
|
|
79
|
+
}
|
|
80
|
+
throw new Error(`dag rerun-task --profile must be one of: auto, minimal, standard, reviewed, supervised; got ${value ?? ""}`);
|
|
81
|
+
}
|
|
82
|
+
function assessStandaloneTaskRerunEligibility(input) {
|
|
83
|
+
if (input.lifecycle === "paused") {
|
|
84
|
+
return {
|
|
85
|
+
eligible: false,
|
|
86
|
+
blockedReason: "parent-run-paused-use-resume",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
if (input.lifecycle === "active" &&
|
|
90
|
+
!isTerminalDagRunStatus(input.state.status)) {
|
|
91
|
+
return {
|
|
92
|
+
eligible: false,
|
|
93
|
+
blockedReason: "parent-run-still-active",
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
if (!isTerminalDagRunStatus(input.state.status)) {
|
|
97
|
+
return {
|
|
98
|
+
eligible: false,
|
|
99
|
+
blockedReason: "parent-run-not-terminal",
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
if (input.state.status === "superseded" || input.state.status === "abandoned") {
|
|
103
|
+
return {
|
|
104
|
+
eligible: false,
|
|
105
|
+
blockedReason: `parent-run-${input.state.status}`,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
if (input.state.status === "finished") {
|
|
109
|
+
return {
|
|
110
|
+
eligible: true,
|
|
111
|
+
note: "parent-run-succeeded-full-rerun-still-allowed",
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
return { eligible: true };
|
|
115
|
+
}
|
|
116
|
+
function resolveTaskIdForRerun(spec, taskIdOverride) {
|
|
117
|
+
if (taskIdOverride?.trim()) {
|
|
118
|
+
return { taskId: taskIdOverride.trim() };
|
|
119
|
+
}
|
|
120
|
+
const fromContract = spec.taskContractBinding?.taskId;
|
|
121
|
+
const fromSource = spec.sourceBinding?.taskId;
|
|
122
|
+
if (fromContract && fromSource && fromContract !== fromSource) {
|
|
123
|
+
return { blockedReason: "ambiguous-task-identity" };
|
|
124
|
+
}
|
|
125
|
+
const taskId = fromContract ?? fromSource;
|
|
126
|
+
if (!taskId) {
|
|
127
|
+
return {
|
|
128
|
+
blockedReason: "missing-task-identity-requires-task-id-override",
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
return { taskId };
|
|
132
|
+
}
|
|
133
|
+
function operatorRequestPath(cwd, requestId) {
|
|
134
|
+
return path.join(cwd, DAG_RUNS_DIR, ".operator-requests", `${requestId}.json`);
|
|
135
|
+
}
|
|
136
|
+
async function fileExists(filePath) {
|
|
137
|
+
try {
|
|
138
|
+
await access(filePath);
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async function readExistingOperatorRequest(cwd, requestId) {
|
|
146
|
+
try {
|
|
147
|
+
const raw = JSON.parse(await readFile(operatorRequestPath(cwd, requestId), "utf-8"));
|
|
148
|
+
if (raw.result && typeof raw.result === "object") {
|
|
149
|
+
return { ...raw.result, idempotentReplay: true };
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
// not found or unreadable
|
|
154
|
+
}
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
async function persistOperatorRequest(cwd, result) {
|
|
158
|
+
await writeJsonAtomic(operatorRequestPath(cwd, result.requestId), {
|
|
159
|
+
schemaVersion: 1,
|
|
160
|
+
kind: "standalone-task-rerun-request",
|
|
161
|
+
parentRunId: result.parentRunId,
|
|
162
|
+
newRunId: result.newRunId,
|
|
163
|
+
taskId: result.taskId,
|
|
164
|
+
reason: result.reason,
|
|
165
|
+
requestId: result.requestId,
|
|
166
|
+
createdAt: new Date().toISOString(),
|
|
167
|
+
result,
|
|
168
|
+
}, { repoRoot: cwd });
|
|
169
|
+
}
|
|
170
|
+
async function writeRerunTaskLineage(cwd, newRunId, input) {
|
|
171
|
+
const located = await locateDagRun(cwd, newRunId);
|
|
172
|
+
if (!located)
|
|
173
|
+
return;
|
|
174
|
+
await writeJsonAtomic(path.join(located.runDir, RERUN_TASK_LINEAGE_REL_PATH), {
|
|
175
|
+
schemaVersion: 1,
|
|
176
|
+
kind: "standalone-task-rerun",
|
|
177
|
+
parentRunId: input.parentRunId,
|
|
178
|
+
reason: input.reason,
|
|
179
|
+
requestId: input.requestId,
|
|
180
|
+
createdAt: new Date().toISOString(),
|
|
181
|
+
}, { repoRoot: cwd, allowCompletedFactsWrite: true });
|
|
182
|
+
}
|
|
183
|
+
function blockedResult(input) {
|
|
184
|
+
return {
|
|
185
|
+
ok: false,
|
|
186
|
+
parentRunId: input.parentRunId,
|
|
187
|
+
reason: input.reason,
|
|
188
|
+
requestId: input.requestId,
|
|
189
|
+
mode: "blocked",
|
|
190
|
+
blockedReason: input.blockedReason,
|
|
191
|
+
...(input.note ? { note: input.note } : {}),
|
|
192
|
+
...(input.taskId ? { taskId: input.taskId } : {}),
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
function extractRunSummary(generateResult) {
|
|
196
|
+
if (generateResult.mode === "generate+validate")
|
|
197
|
+
return undefined;
|
|
198
|
+
return generateResult.run;
|
|
199
|
+
}
|
|
200
|
+
function extractNewRunId(generateResult) {
|
|
201
|
+
const runSummary = extractRunSummary(generateResult);
|
|
202
|
+
return runSummary?.runId;
|
|
203
|
+
}
|
|
204
|
+
export async function rerunStandaloneTask(input) {
|
|
205
|
+
const existing = await readExistingOperatorRequest(input.cwd, input.requestId);
|
|
206
|
+
if (existing) {
|
|
207
|
+
return existing;
|
|
208
|
+
}
|
|
209
|
+
const located = await locateDagRun(input.cwd, input.parentRunId);
|
|
210
|
+
if (!located) {
|
|
211
|
+
throw new Error(`dag run not found: ${input.parentRunId}`);
|
|
212
|
+
}
|
|
213
|
+
const state = await readDagRunState(located.runDir);
|
|
214
|
+
const eligibility = assessStandaloneTaskRerunEligibility({
|
|
215
|
+
lifecycle: located.lifecycle,
|
|
216
|
+
state,
|
|
217
|
+
});
|
|
218
|
+
if (!eligibility.eligible) {
|
|
219
|
+
return blockedResult({
|
|
220
|
+
parentRunId: input.parentRunId,
|
|
221
|
+
reason: input.reason,
|
|
222
|
+
requestId: input.requestId,
|
|
223
|
+
blockedReason: eligibility.blockedReason,
|
|
224
|
+
note: eligibility.note,
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
const spec = await readDagRunSpec(located.runDir);
|
|
228
|
+
const taskResolution = resolveTaskIdForRerun(spec, input.taskIdOverride);
|
|
229
|
+
if (!taskResolution.taskId) {
|
|
230
|
+
return blockedResult({
|
|
231
|
+
parentRunId: input.parentRunId,
|
|
232
|
+
reason: input.reason,
|
|
233
|
+
requestId: input.requestId,
|
|
234
|
+
blockedReason: taskResolution.blockedReason,
|
|
235
|
+
note: eligibility.note,
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
const execute = input.execute ?? true;
|
|
239
|
+
const generateResult = await generateTaskDagUseCase({
|
|
240
|
+
repoRoot: input.cwd,
|
|
241
|
+
taskId: taskResolution.taskId,
|
|
242
|
+
cwd: input.cwd,
|
|
243
|
+
strictModels: true,
|
|
244
|
+
execute,
|
|
245
|
+
initOnly: false,
|
|
246
|
+
dryRun: false,
|
|
247
|
+
profile: input.profile ?? "auto",
|
|
248
|
+
profileExplicit: input.profileExplicit ?? false,
|
|
249
|
+
});
|
|
250
|
+
const newRunId = extractNewRunId(generateResult);
|
|
251
|
+
const runSummary = extractRunSummary(generateResult);
|
|
252
|
+
const completedSuccessfully = !execute ||
|
|
253
|
+
(runSummary !== undefined && "status" in runSummary && runSummary.status === "finished");
|
|
254
|
+
if (execute && newRunId) {
|
|
255
|
+
await writeRerunTaskLineage(input.cwd, newRunId, {
|
|
256
|
+
parentRunId: input.parentRunId,
|
|
257
|
+
reason: input.reason,
|
|
258
|
+
requestId: input.requestId,
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
const result = {
|
|
262
|
+
ok: completedSuccessfully,
|
|
263
|
+
parentRunId: input.parentRunId,
|
|
264
|
+
reason: input.reason,
|
|
265
|
+
requestId: input.requestId,
|
|
266
|
+
mode: "generate+validate+execute",
|
|
267
|
+
taskId: taskResolution.taskId,
|
|
268
|
+
...(newRunId ? { newRunId } : {}),
|
|
269
|
+
...(runSummary && !completedSuccessfully && "status" in runSummary
|
|
270
|
+
? { note: `child-run-${runSummary.status}` }
|
|
271
|
+
: eligibility.note
|
|
272
|
+
? { note: eligibility.note }
|
|
273
|
+
: {}),
|
|
274
|
+
generateResult,
|
|
275
|
+
...(runSummary ? { runSummary } : {}),
|
|
276
|
+
};
|
|
277
|
+
if (await fileExists(operatorRequestPath(input.cwd, input.requestId))) {
|
|
278
|
+
const replay = await readExistingOperatorRequest(input.cwd, input.requestId);
|
|
279
|
+
if (replay)
|
|
280
|
+
return replay;
|
|
281
|
+
}
|
|
282
|
+
await persistOperatorRequest(input.cwd, result);
|
|
283
|
+
return result;
|
|
284
|
+
}
|
|
@@ -17,10 +17,21 @@ export const DEFAULT_DAG_RETRY_CATEGORIES = [
|
|
|
17
17
|
"unavailable",
|
|
18
18
|
];
|
|
19
19
|
export const STRUCTURED_OUTPUT_RETRY_CATEGORY = "output-too-large";
|
|
20
|
+
export const PROTOCOL_INVALID_RETRY_CATEGORY = "protocol-invalid";
|
|
20
21
|
export const STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES = [
|
|
21
22
|
...DEFAULT_DAG_RETRY_CATEGORIES,
|
|
22
23
|
STRUCTURED_OUTPUT_RETRY_CATEGORY,
|
|
23
24
|
];
|
|
25
|
+
/** Categories allowed on nodes that declare a machine-readable outputProtocol. */
|
|
26
|
+
export const PROTOCOL_AWARE_DAG_RETRY_CATEGORIES = [
|
|
27
|
+
...DEFAULT_DAG_RETRY_CATEGORIES,
|
|
28
|
+
PROTOCOL_INVALID_RETRY_CATEGORY,
|
|
29
|
+
];
|
|
30
|
+
export const ALL_DAG_RETRY_CATEGORIES = [
|
|
31
|
+
...DEFAULT_DAG_RETRY_CATEGORIES,
|
|
32
|
+
STRUCTURED_OUTPUT_RETRY_CATEGORY,
|
|
33
|
+
PROTOCOL_INVALID_RETRY_CATEGORY,
|
|
34
|
+
];
|
|
24
35
|
const RETRY_SAFE_PI_ROLES = new Set([
|
|
25
36
|
"planner",
|
|
26
37
|
"scout",
|
|
@@ -28,7 +39,7 @@ const RETRY_SAFE_PI_ROLES = new Set([
|
|
|
28
39
|
"verifier",
|
|
29
40
|
"closeout",
|
|
30
41
|
]);
|
|
31
|
-
export const dagRetryCategorySchema = z.enum(
|
|
42
|
+
export const dagRetryCategorySchema = z.enum(ALL_DAG_RETRY_CATEGORIES);
|
|
32
43
|
export const dagRetryBackoffSchema = z.enum(["exponential"]);
|
|
33
44
|
/**
|
|
34
45
|
* Opt-in retry policy for a DagTask. Generated only for safe read-only Pi
|
|
@@ -86,6 +97,14 @@ export const STRUCTURED_REQUIRED_PI_RETRY_POLICY = {
|
|
|
86
97
|
...DEFAULT_READ_ONLY_PI_RETRY_POLICY,
|
|
87
98
|
retryCategories: [...STRUCTURED_REQUIRED_DAG_RETRY_CATEGORIES],
|
|
88
99
|
};
|
|
100
|
+
/**
|
|
101
|
+
* Default retry for reviewer / recovery nodes that declare outputProtocol.
|
|
102
|
+
* Includes protocol-invalid so missing VERDICT lines are corrected in-node.
|
|
103
|
+
*/
|
|
104
|
+
export const PROTOCOL_AWARE_PI_RETRY_POLICY = {
|
|
105
|
+
...DEFAULT_READ_ONLY_PI_RETRY_POLICY,
|
|
106
|
+
retryCategories: [...PROTOCOL_AWARE_DAG_RETRY_CATEGORIES],
|
|
107
|
+
};
|
|
89
108
|
/**
|
|
90
109
|
* Deterministic helper: is this raw failure category eligible for retry under
|
|
91
110
|
* the given policy? Pure function; executor never decides retry eligibility.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
2
3
|
import { hostname } from "node:os";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { isHardBudgetBreached, resolveEffectiveMaxConcurrent, } from "../../application/evaluation/budget.js";
|
|
@@ -7,6 +8,7 @@ import { CANONICAL_TASK_ID_PATTERN, formatLocalCompactDate, } from "../../task/r
|
|
|
7
8
|
import { assertFrozenBudget, initRunBudgetLedger, preflightBudgetOrBreach, recordFinishedNodeBudget, writeBudgetLedgerArtifacts, } from "./budget-enforcement.js";
|
|
8
9
|
import { getDagRunDir, isTerminalDagRunStatus, locateDagRun, readHumanApprovalArtifact, requireActiveDagRun, } from "./lifecycle.js";
|
|
9
10
|
import { moveToCompletedRunDir, moveToPausedRunDir, prepareActiveRunDir, writeRunSpec, writeRunState, } from "./run-store.js";
|
|
11
|
+
import { evaluateNodeLiveness, resolveLivenessPolicy, } from "./liveness-policy.js";
|
|
10
12
|
import { createDagNodeExecutor } from "./executor-registry.js";
|
|
11
13
|
import { executeDagPiNode } from "../../executors/dag-pi-executor.js";
|
|
12
14
|
import { assertValidDagSpec } from "./validate.js";
|
|
@@ -16,6 +18,7 @@ import { resolveRunningControllerIdentity } from "../../shared/package-metadata.
|
|
|
16
18
|
import { normalizeDagPromptSources } from "./prompt-source.js";
|
|
17
19
|
import { relocateConvergenceArtifactPaths, relocateRunArtifactPaths, } from "./upstream-artifacts.js";
|
|
18
20
|
import { createSkillSnapshot, prepareSkillSnapshotForContinuation, writeSkillSnapshot, } from "./skill-snapshot.js";
|
|
21
|
+
import { captureWorkspaceCheckpoint, WORKSPACE_CHECKPOINT_START_REL, WORKSPACE_CHECKPOINT_TERMINAL_REL, writeWorkspaceCheckpoint, } from "./workspace-checkpoint.js";
|
|
19
22
|
import { buildNodePrompt, buildNodePromptWithResolvedSkillInstructions, executeDagNode, } from "./node-execution.js";
|
|
20
23
|
import { runConvergencePassController, shouldEnableDagConvergence, } from "./convergence/controller.js";
|
|
21
24
|
import { executeDagRanksOnce, isConditionSkippedReason } from "./scheduler.js";
|
|
@@ -149,6 +152,9 @@ export function createInitialRunState(spec, opts, ranks, runId = opts.runId ?? "
|
|
|
149
152
|
},
|
|
150
153
|
}
|
|
151
154
|
: {}),
|
|
155
|
+
...(opts.workerAssociation
|
|
156
|
+
? { workerAssociation: structuredClone(opts.workerAssociation) }
|
|
157
|
+
: {}),
|
|
152
158
|
};
|
|
153
159
|
initRunBudgetLedger(state, spec.budget);
|
|
154
160
|
return state;
|
|
@@ -229,7 +235,17 @@ export async function runDag(spec, opts) {
|
|
|
229
235
|
throw new Error(`failed to create run-start skill snapshot: ${error instanceof Error ? error.message : String(error)}`);
|
|
230
236
|
}
|
|
231
237
|
await writeRunState(runDir, state);
|
|
238
|
+
if (opts.workerAssociation) {
|
|
239
|
+
await writeJsonAtomic(path.join(runDir, ".runtime", "worker-association.json"), opts.workerAssociation);
|
|
240
|
+
}
|
|
232
241
|
await notifyRunObserver(opts.observer, "onRunStart", state);
|
|
242
|
+
try {
|
|
243
|
+
const startCheckpoint = await captureWorkspaceCheckpoint(opts.cwd);
|
|
244
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_START_REL, startCheckpoint);
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
console.warn(`[run-dag] warning: failed to write workspace start checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
248
|
+
}
|
|
233
249
|
if (opts.dryRun || opts.initOnly) {
|
|
234
250
|
return {
|
|
235
251
|
title: spec.title,
|
|
@@ -258,6 +274,34 @@ export async function runDag(spec, opts) {
|
|
|
258
274
|
finally {
|
|
259
275
|
}
|
|
260
276
|
}
|
|
277
|
+
export async function runDagContinuation(opts) {
|
|
278
|
+
const { maxConcurrent } = resolveEffectiveMaxConcurrent(Math.max(1, opts.maxConcurrent ?? 4), opts.spec.budget);
|
|
279
|
+
const state = opts.state;
|
|
280
|
+
state.status = "running";
|
|
281
|
+
const resumedAt = new Date().toISOString();
|
|
282
|
+
state.runner = {
|
|
283
|
+
pid: process.pid,
|
|
284
|
+
hostname: hostname(),
|
|
285
|
+
startedAt: resumedAt,
|
|
286
|
+
heartbeatAt: resumedAt,
|
|
287
|
+
};
|
|
288
|
+
const activeRunDir = getDagRunDir(opts.cwd, "active", state.runId);
|
|
289
|
+
const completedRunDir = getDagRunDir(opts.cwd, "completed", state.runId);
|
|
290
|
+
const pausedRunDir = getDagRunDir(opts.cwd, "paused", state.runId);
|
|
291
|
+
return executeDagCheckpoint({
|
|
292
|
+
spec: opts.spec,
|
|
293
|
+
state,
|
|
294
|
+
ranks: state.ranks,
|
|
295
|
+
runDir: opts.runDir,
|
|
296
|
+
cwd: opts.cwd,
|
|
297
|
+
maxConcurrent,
|
|
298
|
+
executeNode: opts.executeNode,
|
|
299
|
+
observer: opts.observer,
|
|
300
|
+
activeRunDir,
|
|
301
|
+
completedRunDir,
|
|
302
|
+
pausedRunDir,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
261
305
|
export async function resumeDagRun(opts) {
|
|
262
306
|
const located = await locateDagRun(opts.cwd, opts.runId);
|
|
263
307
|
if (!located) {
|
|
@@ -363,12 +407,31 @@ async function executeDagCheckpoint(input) {
|
|
|
363
407
|
stateWriteQueue = stateWriteQueue.then(() => writeRunState(runDir, state, options));
|
|
364
408
|
await stateWriteQueue;
|
|
365
409
|
};
|
|
410
|
+
const runnerLivenessPolicy = resolveLivenessPolicy(spec.defaults?.livenessPolicy);
|
|
366
411
|
const heartbeatTimer = setInterval(() => {
|
|
367
412
|
if (!state.runner)
|
|
368
413
|
return;
|
|
414
|
+
// Runner lease only — never counts as meaningful Pi progress.
|
|
369
415
|
state.runner.heartbeatAt = new Date().toISOString();
|
|
416
|
+
const tasksByIdForPolicy = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
417
|
+
for (const node of Object.values(state.nodes)) {
|
|
418
|
+
if (node.status !== "RUNNING")
|
|
419
|
+
continue;
|
|
420
|
+
// Lease clock is separate from meaningful activity.
|
|
421
|
+
node.lastLeaseAt = state.runner.heartbeatAt;
|
|
422
|
+
const task = tasksByIdForPolicy.get(node.id);
|
|
423
|
+
const policy = resolveLivenessPolicy(spec.defaults?.livenessPolicy, task?.livenessPolicy);
|
|
424
|
+
const evaluation = evaluateNodeLiveness({
|
|
425
|
+
node,
|
|
426
|
+
policy,
|
|
427
|
+
runnerHeartbeatAt: state.runner.heartbeatAt,
|
|
428
|
+
});
|
|
429
|
+
node.livenessStatus = evaluation.status;
|
|
430
|
+
// Intentionally do NOT refresh lastActivityAt / lastMeaningfulProgressAt
|
|
431
|
+
// from the runner lease timer.
|
|
432
|
+
}
|
|
370
433
|
void persistState().catch(() => { });
|
|
371
|
-
},
|
|
434
|
+
}, runnerLivenessPolicy.heartbeatIntervalMs);
|
|
372
435
|
heartbeatTimer.unref();
|
|
373
436
|
try {
|
|
374
437
|
const tasksById = new Map(spec.tasks.map((task) => [task.id, task]));
|
|
@@ -496,6 +559,13 @@ async function executeDagCheckpoint(input) {
|
|
|
496
559
|
finalizeTerminalRunStatus(state, spec.tasks.length);
|
|
497
560
|
await persistState();
|
|
498
561
|
await notifyRunObserver(input.observer, "onRunFinish", state);
|
|
562
|
+
try {
|
|
563
|
+
const terminalCheckpoint = await captureWorkspaceCheckpoint(cwd);
|
|
564
|
+
await writeWorkspaceCheckpoint(runDir, WORKSPACE_CHECKPOINT_TERMINAL_REL, terminalCheckpoint);
|
|
565
|
+
}
|
|
566
|
+
catch (error) {
|
|
567
|
+
console.warn(`[run-dag] warning: failed to write workspace terminal checkpoint: ${error instanceof Error ? error.message : String(error)}`);
|
|
568
|
+
}
|
|
499
569
|
runDir = await moveToCompletedRunDir(runDir, completedRunDir);
|
|
500
570
|
}
|
|
501
571
|
await relocateRunArtifactPaths({
|
|
@@ -2,13 +2,14 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { readFile } from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
|
-
import { writeTextAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
5
|
+
import { writeJsonAtomic, writeTextAtomic, } from "../../infrastructure/harness/atomic-write.js";
|
|
6
6
|
import { resolveContextPolicy } from "./context-policy.js";
|
|
7
7
|
import { buildDagNodePromptEnvelope } from "./prompt.js";
|
|
8
8
|
import { resolveDagSkillInstructions, } from "./skill-instructions.js";
|
|
9
9
|
export const SKILL_SNAPSHOT_SCHEMA_VERSION = 1;
|
|
10
10
|
export const SKILL_SNAPSHOT_RESOLVER_VERSION = 1;
|
|
11
11
|
export const SKILL_SNAPSHOT_REL_PATH = ".runtime/skill-snapshot.json";
|
|
12
|
+
export const SKILL_SNAPSHOT_CONTINUATION_REL_PATH = ".runtime/skill-snapshot-continuation.json";
|
|
12
13
|
const skillReferenceSchema = z.object({
|
|
13
14
|
name: z.string().min(1),
|
|
14
15
|
path: z.string().min(1),
|
|
@@ -75,7 +76,7 @@ const skillSnapshotSchema = z.object({
|
|
|
75
76
|
resolverVersion: z.literal(SKILL_SNAPSHOT_RESOLVER_VERSION),
|
|
76
77
|
runId: z.string().min(1),
|
|
77
78
|
createdAt: z.string().datetime(),
|
|
78
|
-
mode: z.enum(["run-start", "legacy-resume-backfill"]),
|
|
79
|
+
mode: z.enum(["run-start", "legacy-resume-backfill", "continuation-from-parent"]),
|
|
79
80
|
initialTaskIds: z.array(z.string().min(1)),
|
|
80
81
|
bindings: z.array(skillSnapshotBindingSchema),
|
|
81
82
|
profiles: z.array(skillSnapshotProfileSchema),
|
|
@@ -85,7 +86,7 @@ const skillSnapshotRefSchema = z.object({
|
|
|
85
86
|
path: z.literal(SKILL_SNAPSHOT_REL_PATH),
|
|
86
87
|
sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
87
88
|
createdAt: z.string().datetime(),
|
|
88
|
-
mode: z.enum(["run-start", "legacy-resume-backfill"]),
|
|
89
|
+
mode: z.enum(["run-start", "legacy-resume-backfill", "continuation-from-parent"]),
|
|
89
90
|
}).strict();
|
|
90
91
|
export class DagSkillSnapshotIntegrityError extends Error {
|
|
91
92
|
constructor(message, options) {
|
|
@@ -329,6 +330,24 @@ function parseSnapshot(value) {
|
|
|
329
330
|
throw new DagSkillSnapshotIntegrityError(`skill snapshot schema validation failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
330
331
|
}
|
|
331
332
|
}
|
|
333
|
+
export async function cloneParentSkillSnapshotForContinuation(input) {
|
|
334
|
+
const parentSnapshot = await readSkillSnapshot(input.parentRunDir, input.parentRef, { expectedRunId: input.parentRunId });
|
|
335
|
+
const parentSnapshotSha256 = hashSkillSnapshot(parentSnapshot);
|
|
336
|
+
const cloned = {
|
|
337
|
+
...structuredClone(parentSnapshot),
|
|
338
|
+
runId: input.newRunId,
|
|
339
|
+
createdAt: (input.now ?? new Date()).toISOString(),
|
|
340
|
+
mode: "continuation-from-parent",
|
|
341
|
+
};
|
|
342
|
+
const ref = await writeSkillSnapshot(input.newRunDir, cloned);
|
|
343
|
+
await writeJsonAtomic(path.join(input.newRunDir, ...SKILL_SNAPSHOT_CONTINUATION_REL_PATH.split("/")), {
|
|
344
|
+
schemaVersion: 1,
|
|
345
|
+
mode: "continuation-from-parent",
|
|
346
|
+
parentRunId: input.parentRunId,
|
|
347
|
+
parentSnapshotSha256,
|
|
348
|
+
});
|
|
349
|
+
return ref;
|
|
350
|
+
}
|
|
332
351
|
export async function createSkillSnapshot(input) {
|
|
333
352
|
const initialTaskIds = input.initialTaskIds
|
|
334
353
|
?? input.spec.tasks.map((task) => task.id);
|
|
@@ -2,6 +2,8 @@ import { z } from "zod";
|
|
|
2
2
|
import { campaignBudgetSchema, } from "../../application/evaluation/budget.js";
|
|
3
3
|
import { assertDagPromptSourceRule } from "./prompt-source.js";
|
|
4
4
|
import { dagRetryPolicySchema } from "./retry-policy.js";
|
|
5
|
+
import { dagLivenessPolicySchema, } from "./liveness-policy.js";
|
|
6
|
+
import { dagOutputProtocolSchema } from "./output-protocol.js";
|
|
5
7
|
export const dagComplexitySchema = z.enum(["HIGH", "MED", "LOW"]);
|
|
6
8
|
export const dagNodeExecutorSchema = z.enum(["pi", "shell", "static"]);
|
|
7
9
|
export const CURSOR_DAG_EXECUTOR_REMOVED_ERROR = 'executor "cursor" is no longer supported; regenerate the DAG with Pi-only writers (implement-pi / repair-pi)';
|
|
@@ -193,6 +195,8 @@ export const dagDefaultsSchema = z
|
|
|
193
195
|
contextPolicyId: contextPolicyIdSchema.optional(),
|
|
194
196
|
skills: z.array(z.string()).optional(),
|
|
195
197
|
writePolicy: dagWritePolicySchema.optional(),
|
|
198
|
+
/** Adaptive liveness thresholds for Pi node supervision. */
|
|
199
|
+
livenessPolicy: dagLivenessPolicySchema,
|
|
196
200
|
})
|
|
197
201
|
.optional();
|
|
198
202
|
export const dagNodeStatusSchema = z.enum([
|
|
@@ -382,6 +386,12 @@ export const dagTaskSchema = z.object({
|
|
|
382
386
|
outputContract: z.string().optional(),
|
|
383
387
|
outputMode: dagOutputModeSchema.optional(),
|
|
384
388
|
firstProtocolLine: z.string().min(1).optional(),
|
|
389
|
+
/**
|
|
390
|
+
* Machine-readable output protocol (phase 1: first-line-enum).
|
|
391
|
+
* When set with retryOnInvalid, protocol-invalid failures may auto-retry
|
|
392
|
+
* on safe read-only Pi nodes before the node becomes ERROR.
|
|
393
|
+
*/
|
|
394
|
+
outputProtocol: dagOutputProtocolSchema.optional(),
|
|
385
395
|
/**
|
|
386
396
|
* Explicit opt-in for the deterministic project governance context resolver
|
|
387
397
|
* (AGENTS.md chain + referenced code standards). Only tasks that set this
|
|
@@ -393,6 +403,8 @@ export const dagTaskSchema = z.object({
|
|
|
393
403
|
forbiddenPaths: z.array(z.string()).optional().default([]),
|
|
394
404
|
decisionGate: dagDecisionGateSchema.optional(),
|
|
395
405
|
retryPolicy: dagRetryPolicySchema.optional(),
|
|
406
|
+
/** Optional per-node adaptive liveness override (merged over defaults). */
|
|
407
|
+
livenessPolicy: dagLivenessPolicySchema,
|
|
396
408
|
dynamicExpansion: dagDynamicExpansionSchema.optional(),
|
|
397
409
|
dynamicReduction: dagDynamicReductionSchema.optional(),
|
|
398
410
|
dynamicCondition: dagDynamicConditionSchema.optional(),
|
|
@@ -593,6 +593,16 @@ function validateRetryPolicyTaskConfig(task, issues) {
|
|
|
593
593
|
});
|
|
594
594
|
}
|
|
595
595
|
}
|
|
596
|
+
function validateOutputProtocolTaskConfig(task, issues) {
|
|
597
|
+
if (task.outputProtocol === undefined)
|
|
598
|
+
return;
|
|
599
|
+
if (!isSafeReadOnlyPiRetryCandidate(task)) {
|
|
600
|
+
issues.push({
|
|
601
|
+
type: "invalid-output-protocol",
|
|
602
|
+
message: `task ${task.id} declares outputProtocol but is not a safe read-only non-dynamic Pi node; output protocol retry is only allowed for read-only/none Pi planner/scout/reviewer/verifier/closeout nodes`,
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
}
|
|
596
606
|
function validateDecisionGateTaskConfig(task, issues) {
|
|
597
607
|
if (!task.decisionGate?.enabled) {
|
|
598
608
|
return;
|
|
@@ -692,6 +702,7 @@ export function validateDagSpec(spec) {
|
|
|
692
702
|
validateStaticTaskConfig(task, issues);
|
|
693
703
|
validateDecisionGateTaskConfig(task, issues);
|
|
694
704
|
validateRetryPolicyTaskConfig(task, issues);
|
|
705
|
+
validateOutputProtocolTaskConfig(task, issues);
|
|
695
706
|
validateProjectGovernanceTaskConfig(task, spec, issues);
|
|
696
707
|
}
|
|
697
708
|
validateSameRankWriteSetConflicts(spec, ranks, issues);
|