@tea-agent/loop-agent 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/CHANGELOG.md +51 -1
- package/README.md +20 -0
- package/dist/application/dag/args.js +9 -2
- package/dist/cli/command-definitions.js +7 -0
- package/dist/cli/program.js +6 -1
- package/dist/commands/dag-reconcile-run.js +118 -0
- package/dist/commands/init.js +12 -3
- package/dist/executors/shell-executor.js +74 -8
- package/dist/governance/manifest-types.js +4 -0
- package/dist/shared/reference-context.js +48 -22
- package/dist/task/config-types.js +1 -1
- package/dist/task/runtime.js +1 -1
- package/dist/worker/cli.js +216 -0
- package/dist/worker/closeout/apply.js +73 -0
- package/dist/worker/closeout/preview.js +30 -0
- package/dist/worker/delivery/final-verification.js +158 -0
- package/dist/worker/delivery/git-transaction.js +354 -0
- package/dist/worker/delivery/package.js +449 -0
- package/dist/worker/feature/decision-loader.js +68 -0
- package/dist/worker/feature/discover.js +14 -0
- package/dist/worker/feature/next-action.js +74 -0
- package/dist/worker/feature/reducer.js +133 -0
- package/dist/worker/feature/review.js +502 -0
- package/dist/worker/feature/run.js +313 -0
- package/dist/worker/feature/types.js +1 -0
- package/dist/worker/follow-up/approve.js +270 -0
- package/dist/worker/follow-up/factory.js +234 -0
- package/dist/worker/follow-up/paths.js +25 -0
- package/dist/worker/follow-up/policy.js +26 -0
- package/dist/worker/follow-up/schema.js +93 -0
- package/dist/worker/follow-up/store.js +96 -0
- package/dist/worker/loop-agent/loop-agent-client.js +51 -10
- package/dist/worker/metrics/projector.js +139 -0
- package/dist/worker/observability/read-model.js +256 -15
- package/dist/worker/observe/paths.js +17 -5
- package/dist/worker/observe/routes.js +78 -20
- package/dist/worker/observe/server.js +8 -6
- package/dist/worker/observe/static/app.js +1045 -177
- package/dist/worker/observe/static/index.html +70 -43
- package/dist/worker/observe/static/styles.css +553 -610
- package/dist/worker/pool/run-store.js +14 -2
- package/dist/worker/pool/validation.js +59 -0
- package/dist/worker/report/morning-report.js +41 -6
- package/dist/worker/run-task/run-task.js +1 -1
- package/dist/worker/runner/run-ready.js +19 -5
- package/dist/workflows/dag/init-hybrid.js +3 -1
- package/dist/workflows/dag/lifecycle.js +146 -0
- package/dist/workflows/dag/node-execution.js +3 -0
- package/dist/workflows/dag/prompt.js +16 -0
- package/dist/workflows/dag/report.js +2 -0
- package/dist/workflows/dag/runner.js +133 -104
- package/dist/workflows/dag/types.js +3 -0
- package/docs/README.md +21 -0
- package/docs/agent-dag-recovery-playbook.md +1 -1
- package/docs/architecture/runtime-boundaries.md +3 -2
- package/docs/design/README.md +13 -7
- package/docs/exec-plans/active/README.md +2 -2
- package/docs/exec-plans/completed/README.md +15 -0
- package/docs/loop-agent-harness.md +45 -2
- package/docs/progress/README.md +2 -0
- package/docs/reports/README.md +13 -0
- package/docs/templates/agent-dag-report.schema.json +5 -3
- package/docs/templates/harness.schema.json +7 -2
- package/docs/templates/init-evolution-review.md +4 -2
- package/docs/verification-matrix.md +7 -0
- package/harness.json +4 -3
- package/package.json +4 -2
- package/scripts/check-product-line-docs.sh +7 -3
- package/scripts/check-task-pool-root.sh +1 -1
- package/skills/init-capability-evolution/SKILL.md +1 -0
- package/skills/loop-agent/references/command-reference.md +21 -0
- package/skills/loop-agent/references/hybrid-dag.md +4 -3
- package/skills/loop-agent/references/verification-and-failure-handling.md +8 -3
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { appendFile, mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
export const TASK_POOL_RELATIVE_ROOT = ".harness/task-pool";
|
|
4
5
|
export function getTaskPoolRoot(repoRoot) {
|
|
@@ -19,6 +20,10 @@ export async function recordTaskPoolRun(input) {
|
|
|
19
20
|
await writeTaskPoolState(input.repoRoot, stateFromRun(input.run));
|
|
20
21
|
await appendJsonlFile(getEventsJsonlPath(input.repoRoot), eventFromRun(input.run));
|
|
21
22
|
}
|
|
23
|
+
export async function appendTaskPoolEvent(repoRoot, event) {
|
|
24
|
+
await ensurePoolDirs(repoRoot);
|
|
25
|
+
await appendJsonlFile(getEventsJsonlPath(repoRoot), event);
|
|
26
|
+
}
|
|
22
27
|
export async function findRunByWorkerRunId(repoRoot, workerRunId) {
|
|
23
28
|
const runs = await readJsonlFile(getRunsJsonlPath(repoRoot));
|
|
24
29
|
return runs.find((run) => run.workerRunId === workerRunId);
|
|
@@ -99,7 +104,14 @@ export async function readAllTaskPoolStates(repoRoot) {
|
|
|
99
104
|
export async function writeTaskPoolState(repoRoot, state) {
|
|
100
105
|
const statePath = getTaskStatePath(repoRoot, state.taskId);
|
|
101
106
|
await mkdir(path.dirname(statePath), { recursive: true });
|
|
102
|
-
|
|
107
|
+
const tempPath = path.join(path.dirname(statePath), `.${path.basename(statePath)}.${randomUUID()}.tmp`);
|
|
108
|
+
try {
|
|
109
|
+
await writeFile(tempPath, `${JSON.stringify({ schemaVersion: 1, ...state }, null, 2)}\n`, "utf-8");
|
|
110
|
+
await rename(tempPath, statePath);
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
await unlink(tempPath).catch(() => { });
|
|
114
|
+
}
|
|
103
115
|
}
|
|
104
116
|
export async function readJsonlFile(filePath) {
|
|
105
117
|
try {
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { readFile, readdir } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { getRunsJsonlPath, getTaskPoolRoot } from "./run-store.js";
|
|
5
|
+
export const taskPoolRunFactSchema = z.object({
|
|
6
|
+
schemaVersion: z.literal(1), batchRunId: z.string().min(1), workerRunId: z.string().min(1), taskId: z.string().min(1), featureId: z.string().min(1),
|
|
7
|
+
status: z.enum(["succeeded", "failed", "run-error"]), recordedAt: z.string().datetime(),
|
|
8
|
+
}).passthrough();
|
|
9
|
+
export const taskPoolStateFactSchema = z.object({
|
|
10
|
+
taskId: z.string().min(1), status: z.enum(["Draft", "Ready", "Queued", "Running", "AgentCompleted", "VerificationRunning", "HumanReview", "Done", "Failed", "Blocked", "Abandoned"]), updatedAt: z.string().datetime(),
|
|
11
|
+
}).passthrough();
|
|
12
|
+
export async function readValidTaskPoolRuns(repoRoot) {
|
|
13
|
+
const records = [];
|
|
14
|
+
const warnings = [];
|
|
15
|
+
try {
|
|
16
|
+
const raw = await readFile(getRunsJsonlPath(repoRoot), "utf-8");
|
|
17
|
+
for (const [index, line] of raw.split(/\r?\n/).filter(Boolean).entries())
|
|
18
|
+
try {
|
|
19
|
+
const parsed = taskPoolRunFactSchema.safeParse(JSON.parse(line));
|
|
20
|
+
if (parsed.success)
|
|
21
|
+
records.push(parsed.data);
|
|
22
|
+
else
|
|
23
|
+
warnings.push(`Task Pool runs.jsonl line ${index + 1} is semantically invalid`);
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
warnings.push(`Task Pool runs.jsonl line ${index + 1} is corrupt`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (!isNotFound(error))
|
|
31
|
+
warnings.push("Task Pool runs.jsonl is unreadable");
|
|
32
|
+
}
|
|
33
|
+
return { records, warnings };
|
|
34
|
+
}
|
|
35
|
+
export async function readValidTaskPoolStates(repoRoot) {
|
|
36
|
+
const records = {};
|
|
37
|
+
const warnings = [];
|
|
38
|
+
const stateDir = path.join(getTaskPoolRoot(repoRoot), "states");
|
|
39
|
+
try {
|
|
40
|
+
for (const entry of await readdir(stateDir))
|
|
41
|
+
if (entry.endsWith(".json"))
|
|
42
|
+
try {
|
|
43
|
+
const parsed = taskPoolStateFactSchema.safeParse(JSON.parse(await readFile(path.join(stateDir, entry), "utf-8")));
|
|
44
|
+
if (parsed.success && parsed.data.taskId === entry.slice(0, -5))
|
|
45
|
+
records[parsed.data.taskId] = parsed.data;
|
|
46
|
+
else
|
|
47
|
+
warnings.push(`Task Pool state is semantically invalid: ${entry}`);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
warnings.push(`Task Pool state is corrupt: ${entry}`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (!isNotFound(error))
|
|
55
|
+
warnings.push("Task Pool states are unreadable");
|
|
56
|
+
}
|
|
57
|
+
return { records, warnings };
|
|
58
|
+
}
|
|
59
|
+
function isNotFound(error) { return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT"); }
|
|
@@ -1,17 +1,30 @@
|
|
|
1
|
-
import { access, writeFile, mkdir } from "node:fs/promises";
|
|
1
|
+
import { access, writeFile, mkdir, readFile } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import
|
|
3
|
+
import YAML from "yaml";
|
|
4
|
+
import { followUpActionCardSchema } from "../follow-up/schema.js";
|
|
5
|
+
import { readFollowUpIndex, resolveRepoFile } from "../follow-up/store.js";
|
|
6
|
+
import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
|
|
7
|
+
import { readValidTaskPoolRuns, readValidTaskPoolStates } from "../pool/validation.js";
|
|
4
8
|
export async function renderMorningReport(options) {
|
|
5
|
-
const
|
|
9
|
+
const { features } = await loadFeatureDecisionModels(options.repoRoot);
|
|
10
|
+
const allRuns = (await readValidTaskPoolRuns(options.repoRoot)).records;
|
|
11
|
+
const runs = allRuns.filter((run) => !options.batchRunId || run.batchRunId === options.batchRunId);
|
|
6
12
|
const total = runs.length;
|
|
7
13
|
const succeeded = runs.filter((run) => run.status === "succeeded").length;
|
|
8
14
|
const failed = runs.filter((run) => run.status === "failed").length;
|
|
9
15
|
const followUps = runs.filter((run) => run.failure).length;
|
|
10
|
-
const states = await
|
|
16
|
+
const states = (await readValidTaskPoolStates(options.repoRoot)).records;
|
|
11
17
|
const blocked = Object.values(states).filter((state) => state.status === "Blocked").length;
|
|
12
18
|
const lines = [
|
|
13
19
|
"# Nightly Worker Report",
|
|
14
20
|
"",
|
|
21
|
+
"## Feature Decision Summary",
|
|
22
|
+
"",
|
|
23
|
+
...(features.length === 0 ? ["- Status: no Feature Packet discovered", "- Next Action: add or locate a Feature Packet", "- Why: no shared Feature read model is available", "- Evidence: none"] : features.flatMap((feature) => {
|
|
24
|
+
const why = feature.blockingItems[0]?.message ?? `${feature.summary.tasksSucceeded}/${feature.summary.tasksTotal} tasks; ${feature.summary.requiredAcCovered}/${feature.summary.requiredAcTotal} required AC`;
|
|
25
|
+
const evidence = [feature.evidence.delivery, feature.evidence.closeout, feature.evidence.morningReport, feature.evidence.observeSnapshot, ...feature.blockingItems.flatMap((item) => item.evidence)].find(Boolean) ?? "none";
|
|
26
|
+
return [`### ${feature.featureId}`, "", `- Status: ${feature.status}`, `- Next Action: ${feature.nextAction?.command ?? feature.nextAction?.label ?? "none"}`, `- Why: ${why}`, `- Evidence: ${evidence}`, ""];
|
|
27
|
+
})),
|
|
15
28
|
"## Summary",
|
|
16
29
|
"",
|
|
17
30
|
`- Total: ${total}`,
|
|
@@ -26,16 +39,38 @@ export async function renderMorningReport(options) {
|
|
|
26
39
|
"|---|---|---|---|---|---|",
|
|
27
40
|
];
|
|
28
41
|
for (const run of runs) {
|
|
29
|
-
|
|
42
|
+
const followUp = await followUpSummary(run, options.repoRoot);
|
|
43
|
+
lines.push(`| ${run.taskId} | ${run.status} | ${run.workerRunId} | ${run.failure?.category ?? "-"} | ${followUp ?? run.failure?.derivedFollowUpTaskId ?? "-"} | ${await artifactSummary(run)} |`);
|
|
30
44
|
}
|
|
31
45
|
if (followUps > 0) {
|
|
32
46
|
lines.push("", "## Human Actions", "");
|
|
33
47
|
for (const run of runs.filter((candidate) => candidate.failure)) {
|
|
34
|
-
|
|
48
|
+
const followUp = await followUpSummary(run, options.repoRoot);
|
|
49
|
+
lines.push(`- ${followUp ?? `${run.failure?.derivedFollowUpTaskId}: ${run.failure?.recommendedFollowUpKind} for ${run.taskId}`}`);
|
|
35
50
|
}
|
|
36
51
|
}
|
|
37
52
|
return `${lines.join("\n")}\n`;
|
|
38
53
|
}
|
|
54
|
+
async function followUpSummary(run, repoRoot) {
|
|
55
|
+
if (!run.failure)
|
|
56
|
+
return undefined;
|
|
57
|
+
try {
|
|
58
|
+
const index = await readFollowUpIndex(repoRoot, run.featureId);
|
|
59
|
+
const entry = index.entries.find((candidate) => candidate.workerRunId === run.workerRunId && candidate.status !== "Superseded" && candidate.status !== "Rejected");
|
|
60
|
+
if (!entry)
|
|
61
|
+
return undefined;
|
|
62
|
+
if (entry.kind === "TaskDraft") {
|
|
63
|
+
return entry.status === "Approved"
|
|
64
|
+
? `${entry.followUpId}: approved as ${entry.proposedTaskId}`
|
|
65
|
+
: `${entry.followUpId}: approve ${entry.proposedTaskId}`;
|
|
66
|
+
}
|
|
67
|
+
const card = followUpActionCardSchema.parse(YAML.parse(await readFile(await resolveRepoFile(repoRoot, entry.draftPath), "utf-8")));
|
|
68
|
+
return `${entry.followUpId}: ${card.command ?? card.label}`;
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
39
74
|
export async function writeMorningReport(input) {
|
|
40
75
|
const markdown = await renderMorningReport(input);
|
|
41
76
|
await mkdir(path.dirname(input.outputPath), { recursive: true });
|
|
@@ -156,7 +156,7 @@ export async function runTaskSpec(options) {
|
|
|
156
156
|
taskArtifactsDir,
|
|
157
157
|
eventCtx,
|
|
158
158
|
});
|
|
159
|
-
if (reportDecision.succeeded) {
|
|
159
|
+
if (reportDecision.succeeded && !options.skipSuccessFinalization) {
|
|
160
160
|
await runObservedStep(eventCtx, "promote-run", async () => {
|
|
161
161
|
progress.step("promote run");
|
|
162
162
|
await runRequiredCommand(options.repoRoot, client, "promote-run", ["promote-run", materializeManifest.harnessTaskId, "--run-id", workerRunId], true, undefined, eventCtx);
|
|
@@ -66,6 +66,7 @@ export async function runReadyTasks(options) {
|
|
|
66
66
|
status: "reused",
|
|
67
67
|
runRecordPath: existing.runRecordPath,
|
|
68
68
|
});
|
|
69
|
+
await options.onTaskFinalized?.({ status: "reused", taskSpec, workerRunId, ...(existing.runRecordPath ? { runRecordPath: existing.runRecordPath } : {}) });
|
|
69
70
|
continue;
|
|
70
71
|
}
|
|
71
72
|
}
|
|
@@ -106,10 +107,12 @@ export async function runReadyTasks(options) {
|
|
|
106
107
|
client: options.client,
|
|
107
108
|
now,
|
|
108
109
|
workerRunId,
|
|
109
|
-
preflight:
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
110
|
+
preflight: options.skipTaskPreflight
|
|
111
|
+
? false
|
|
112
|
+
: {
|
|
113
|
+
runCheckRepo: options.runCheckRepo,
|
|
114
|
+
...(options.checkRepoCommand ? { checkRepoCommand: options.checkRepoCommand } : {}),
|
|
115
|
+
},
|
|
113
116
|
progress,
|
|
114
117
|
...(options.piModel ? { piModel: options.piModel } : {}),
|
|
115
118
|
});
|
|
@@ -128,6 +131,7 @@ export async function runReadyTasks(options) {
|
|
|
128
131
|
status: "failed",
|
|
129
132
|
message,
|
|
130
133
|
});
|
|
134
|
+
let recorded = false;
|
|
131
135
|
try {
|
|
132
136
|
await recordTaskPoolRun({
|
|
133
137
|
repoRoot: options.repoRoot,
|
|
@@ -144,16 +148,20 @@ export async function runReadyTasks(options) {
|
|
|
144
148
|
...(retryOfWorkerRunId ? { retryOfWorkerRunId } : {}),
|
|
145
149
|
},
|
|
146
150
|
});
|
|
151
|
+
recorded = true;
|
|
147
152
|
}
|
|
148
153
|
catch (recordError) {
|
|
154
|
+
await options.onTaskFinalized?.({ status: "run-error", taskSpec, workerRunId });
|
|
149
155
|
tasks.push({
|
|
150
156
|
taskId,
|
|
151
157
|
workerRunId,
|
|
152
158
|
status: "record-error",
|
|
153
159
|
error: errorMessage(recordError),
|
|
154
160
|
});
|
|
155
|
-
|
|
161
|
+
break;
|
|
156
162
|
}
|
|
163
|
+
if (recorded)
|
|
164
|
+
await options.onTaskFinalized?.({ status: "run-error", taskSpec, workerRunId });
|
|
157
165
|
tasks.push({
|
|
158
166
|
taskId,
|
|
159
167
|
workerRunId,
|
|
@@ -207,8 +215,10 @@ export async function runReadyTasks(options) {
|
|
|
207
215
|
...(retryOfWorkerRunId ? { retryOfWorkerRunId } : {}),
|
|
208
216
|
...(result.failureArtifacts ? { failureArtifacts: result.failureArtifacts } : {}),
|
|
209
217
|
};
|
|
218
|
+
let recorded = false;
|
|
210
219
|
try {
|
|
211
220
|
await recordTaskPoolRun({ repoRoot: options.repoRoot, run });
|
|
221
|
+
recorded = true;
|
|
212
222
|
tasks.push({
|
|
213
223
|
taskId,
|
|
214
224
|
workerRunId: result.workerRunId,
|
|
@@ -217,6 +227,7 @@ export async function runReadyTasks(options) {
|
|
|
217
227
|
});
|
|
218
228
|
}
|
|
219
229
|
catch (error) {
|
|
230
|
+
await options.onTaskFinalized?.({ status: result.status, taskSpec, workerRunId: result.workerRunId, runRecordPath: result.runRecordPath });
|
|
220
231
|
tasks.push({
|
|
221
232
|
taskId,
|
|
222
233
|
workerRunId: result.workerRunId,
|
|
@@ -224,7 +235,10 @@ export async function runReadyTasks(options) {
|
|
|
224
235
|
error: errorMessage(error),
|
|
225
236
|
runRecordPath: result.runRecordPath,
|
|
226
237
|
});
|
|
238
|
+
break;
|
|
227
239
|
}
|
|
240
|
+
if (recorded)
|
|
241
|
+
await options.onTaskFinalized?.({ status: result.status, taskSpec, workerRunId: result.workerRunId, runRecordPath: result.runRecordPath });
|
|
228
242
|
}
|
|
229
243
|
const summary = summarize(tasks);
|
|
230
244
|
const batchRunPath = path.join(getTaskPoolRoot(options.repoRoot), "artifacts", batchRunId, "batch-run.json");
|
|
@@ -2,7 +2,7 @@ import { access, readdir, readFile, writeFile } from "node:fs/promises";
|
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { assertValidDagSpec } from "./validate.js";
|
|
5
|
-
import { DEFAULT_DAG_EXECUTOR_MODELS, parseDagSpec, } from "./types.js";
|
|
5
|
+
import { DEFAULT_DAG_OUTPUT_LANGUAGE, DEFAULT_DAG_EXECUTOR_MODELS, parseDagSpec, } from "./types.js";
|
|
6
6
|
import { pathMatchesPattern } from "../../shared/git-progress.js";
|
|
7
7
|
import { BASELINE_FORBIDDEN_PATHS } from "./governance-constants.js";
|
|
8
8
|
import { resolveAdapter } from "../../adapters/index.js";
|
|
@@ -448,6 +448,7 @@ export async function loadTaskHybridSources(repoRoot, taskId) {
|
|
|
448
448
|
constraintPath,
|
|
449
449
|
requirementMarkdown,
|
|
450
450
|
constraintMarkdown,
|
|
451
|
+
outputLanguage: manifest.workflowPolicy.dag.outputLanguage,
|
|
451
452
|
referenceDocuments,
|
|
452
453
|
taskConfig,
|
|
453
454
|
enabledExecutors: resolveEnabledExecutors(manifest.executors),
|
|
@@ -524,6 +525,7 @@ export function buildStandardHybridDagFromTask(sources) {
|
|
|
524
525
|
const spec = {
|
|
525
526
|
version: 2,
|
|
526
527
|
title: `Hybrid DAG: ${taskConfig.title}`,
|
|
528
|
+
outputLanguage: sources.outputLanguage ?? DEFAULT_DAG_OUTPUT_LANGUAGE,
|
|
527
529
|
objective: extractObjective(sources.requirementMarkdown, taskConfig.title),
|
|
528
530
|
successCriteria: extractSuccessCriteria(sources.requirementMarkdown, sources.taskId),
|
|
529
531
|
globalConstraints,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { access, mkdir, readFile, readdir, rename, } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { hostname as localHostname } from "node:os";
|
|
3
4
|
import { writeJsonAtomic, } from "../../infrastructure/harness/atomic-write.js";
|
|
4
5
|
import { parseDagSpec } from "./types.js";
|
|
5
6
|
import { normalizeDagFailureCategory, } from "./failure-category.js";
|
|
@@ -161,10 +162,98 @@ export async function humanApprovalArtifactExists(runDir, nodeId) {
|
|
|
161
162
|
return false;
|
|
162
163
|
}
|
|
163
164
|
}
|
|
165
|
+
export function assessDagRunLiveness(input) {
|
|
166
|
+
if (input.state.status !== "running")
|
|
167
|
+
return { status: "unknown" };
|
|
168
|
+
const runner = input.state.runner;
|
|
169
|
+
if (!runner)
|
|
170
|
+
return { status: "unknown" };
|
|
171
|
+
if (runner.hostname !== (input.hostname ?? localHostname()))
|
|
172
|
+
return { status: "unknown-host" };
|
|
173
|
+
const isAlive = input.isProcessAlive ?? ((pid) => {
|
|
174
|
+
try {
|
|
175
|
+
process.kill(pid, 0);
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
if (!isAlive(runner.pid))
|
|
183
|
+
return { status: "orphaned", runnerAlive: false };
|
|
184
|
+
const heartbeatMs = Date.parse(runner.heartbeatAt);
|
|
185
|
+
const nowMs = (input.now ?? new Date()).getTime();
|
|
186
|
+
if (!Number.isNaN(heartbeatMs) && nowMs - heartbeatMs > (input.staleThresholdMs ?? 90_000)) {
|
|
187
|
+
return { status: "stale", runnerAlive: true };
|
|
188
|
+
}
|
|
189
|
+
const activeNode = Object.values(input.state.nodes).find((node) => node.status === "RUNNING");
|
|
190
|
+
const nodeActivityMs = Date.parse(activeNode?.lastActivityAt ?? activeNode?.startedAt ?? "");
|
|
191
|
+
if (!Number.isNaN(nodeActivityMs) && nowMs - nodeActivityMs > (input.nodeQuietThresholdMs ?? 300_000)) {
|
|
192
|
+
return { status: "node-quiet", runnerAlive: true };
|
|
193
|
+
}
|
|
194
|
+
return { status: "active", runnerAlive: true };
|
|
195
|
+
}
|
|
196
|
+
export function deriveDagRunEffectiveStatus(input) {
|
|
197
|
+
if (input.state.status === "superseded")
|
|
198
|
+
return "superseded";
|
|
199
|
+
if (input.state.status === "abandoned")
|
|
200
|
+
return "abandoned";
|
|
201
|
+
if (input.lifecycle === "paused")
|
|
202
|
+
return "paused";
|
|
203
|
+
if (input.lifecycle === "completed") {
|
|
204
|
+
return input.state.status === "finished" ? "finished" : "failed";
|
|
205
|
+
}
|
|
206
|
+
if (input.state.status === "pending")
|
|
207
|
+
return "pending";
|
|
208
|
+
if (isTerminalDagRunStatus(input.state.status)) {
|
|
209
|
+
return input.state.status === "finished" ? "finished" : "failed";
|
|
210
|
+
}
|
|
211
|
+
if (input.liveness === "orphaned" || input.liveness === "stale")
|
|
212
|
+
return "interrupted";
|
|
213
|
+
if (input.liveness === "node-quiet")
|
|
214
|
+
return "running-quiet";
|
|
215
|
+
if (input.liveness === "unknown-host")
|
|
216
|
+
return "remote-unknown";
|
|
217
|
+
if (input.liveness === "active")
|
|
218
|
+
return "running";
|
|
219
|
+
return "unknown";
|
|
220
|
+
}
|
|
221
|
+
export function assessDagRunRecoveryEligibility(input) {
|
|
222
|
+
const reasons = [];
|
|
223
|
+
const canResume = input.lifecycle === "active"
|
|
224
|
+
&& input.state.status === "running"
|
|
225
|
+
&& Boolean(input.state.humanDecisionNodeId)
|
|
226
|
+
&& Boolean(input.hasHumanApproval);
|
|
227
|
+
if (!canResume)
|
|
228
|
+
reasons.push("standard-resume-preconditions-not-met");
|
|
229
|
+
let canReconcile = true;
|
|
230
|
+
if (input.lifecycle === "completed" || isTerminalDagRunStatus(input.state.status)) {
|
|
231
|
+
canReconcile = false;
|
|
232
|
+
reasons.push("run-already-terminal");
|
|
233
|
+
}
|
|
234
|
+
if (input.lifecycle === "active"
|
|
235
|
+
&& ["active", "node-quiet", "stale", "unknown-host", "unknown"].includes(input.liveness)) {
|
|
236
|
+
canReconcile = false;
|
|
237
|
+
reasons.push("runner-not-proven-dead-or-stopped");
|
|
238
|
+
}
|
|
239
|
+
if (input.lifecycle === "paused" && input.state.status !== "paused") {
|
|
240
|
+
reasons.push("lifecycle-status-mismatch");
|
|
241
|
+
}
|
|
242
|
+
if (!input.state.runner)
|
|
243
|
+
reasons.push("missing-runner-metadata");
|
|
244
|
+
return {
|
|
245
|
+
canResume,
|
|
246
|
+
canReconcile,
|
|
247
|
+
allowedActions: canReconcile ? ["supersede", "abandon"] : [],
|
|
248
|
+
reasons: [...new Set(reasons)],
|
|
249
|
+
};
|
|
250
|
+
}
|
|
164
251
|
export const TERMINAL_RUN_STATUSES = new Set([
|
|
165
252
|
"finished",
|
|
166
253
|
"failed",
|
|
167
254
|
"partial_failed",
|
|
255
|
+
"superseded",
|
|
256
|
+
"abandoned",
|
|
168
257
|
]);
|
|
169
258
|
export function isTerminalDagRunStatus(status) {
|
|
170
259
|
return TERMINAL_RUN_STATUSES.has(status);
|
|
@@ -248,6 +337,31 @@ export async function detectDagRunHealthIssues(input) {
|
|
|
248
337
|
advisoryAction: "Use dag doctor; approve/reject/resume may fail until lifecycle facts are consistent.",
|
|
249
338
|
});
|
|
250
339
|
}
|
|
340
|
+
const liveness = assessDagRunLiveness({ state });
|
|
341
|
+
if (liveness.status === "orphaned") {
|
|
342
|
+
issues.push({
|
|
343
|
+
code: "runner-process-missing",
|
|
344
|
+
severity: "error",
|
|
345
|
+
message: `Runner PID ${state.runner?.pid} is not alive on host ${state.runner?.hostname}`,
|
|
346
|
+
advisoryAction: "Treat this run as orphaned; inspect artifacts and start a new run instead of resuming it.",
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
else if (liveness.status === "stale") {
|
|
350
|
+
issues.push({
|
|
351
|
+
code: "runner-heartbeat-stale",
|
|
352
|
+
severity: "warning",
|
|
353
|
+
message: `Runner heartbeat is stale since ${state.runner?.heartbeatAt}`,
|
|
354
|
+
advisoryAction: "Inspect node artifacts and process liveness before stopping or retrying.",
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
else if (liveness.status === "node-quiet") {
|
|
358
|
+
issues.push({
|
|
359
|
+
code: "node-activity-quiet",
|
|
360
|
+
severity: "warning",
|
|
361
|
+
message: "Runner heartbeat is fresh but the current RUNNING node has produced no state activity for more than 5 minutes",
|
|
362
|
+
advisoryAction: "Inspect the node session events and executor logs before deciding whether to wait or abort.",
|
|
363
|
+
});
|
|
364
|
+
}
|
|
251
365
|
if (lifecycle === "active" &&
|
|
252
366
|
state.status === "running" &&
|
|
253
367
|
state.humanDecisionNodeId) {
|
|
@@ -340,6 +454,14 @@ export async function buildDagOperatorRunSummary(entry) {
|
|
|
340
454
|
pendingNodes: [],
|
|
341
455
|
finishedNodes: [],
|
|
342
456
|
healthIssues,
|
|
457
|
+
effectiveStatus: entry.lifecycle === "paused" ? "paused" : "unknown",
|
|
458
|
+
stateConsistent: false,
|
|
459
|
+
recoveryEligibility: {
|
|
460
|
+
canResume: false,
|
|
461
|
+
canReconcile: false,
|
|
462
|
+
allowedActions: [],
|
|
463
|
+
reasons: ["missing-state-json"],
|
|
464
|
+
},
|
|
343
465
|
nextRecommendedAction: deriveOperatorNextAction({
|
|
344
466
|
lifecycle: entry.lifecycle,
|
|
345
467
|
state: {
|
|
@@ -363,9 +485,22 @@ export async function buildDagOperatorRunSummary(entry) {
|
|
|
363
485
|
runDir: entry.runDir,
|
|
364
486
|
state,
|
|
365
487
|
});
|
|
488
|
+
const liveness = assessDagRunLiveness({ state });
|
|
366
489
|
const hasHumanApproval = state.humanDecisionNodeId
|
|
367
490
|
? await humanApprovalArtifactExists(entry.runDir, state.humanDecisionNodeId)
|
|
368
491
|
: false;
|
|
492
|
+
const effectiveStatus = deriveDagRunEffectiveStatus({
|
|
493
|
+
lifecycle: entry.lifecycle,
|
|
494
|
+
state,
|
|
495
|
+
liveness: liveness.status,
|
|
496
|
+
});
|
|
497
|
+
const stateConsistent = !healthIssues.some((issue) => ["lifecycle-status-mismatch", "non-terminal-in-completed"].includes(issue.code));
|
|
498
|
+
const recoveryEligibility = assessDagRunRecoveryEligibility({
|
|
499
|
+
lifecycle: entry.lifecycle,
|
|
500
|
+
state,
|
|
501
|
+
liveness: liveness.status,
|
|
502
|
+
hasHumanApproval,
|
|
503
|
+
});
|
|
369
504
|
return {
|
|
370
505
|
runId: state.runId,
|
|
371
506
|
title: state.title,
|
|
@@ -380,6 +515,10 @@ export async function buildDagOperatorRunSummary(entry) {
|
|
|
380
515
|
pendingNodes: listPendingNodeIds(state),
|
|
381
516
|
finishedNodes: listFinishedNodeIds(state),
|
|
382
517
|
healthIssues,
|
|
518
|
+
liveness: liveness.status,
|
|
519
|
+
effectiveStatus,
|
|
520
|
+
stateConsistent,
|
|
521
|
+
recoveryEligibility,
|
|
383
522
|
nextRecommendedAction: deriveOperatorNextAction({
|
|
384
523
|
lifecycle: entry.lifecycle,
|
|
385
524
|
state,
|
|
@@ -552,6 +691,13 @@ async function formatDagDoctorMarkdown(repoRoot, runId) {
|
|
|
552
691
|
"",
|
|
553
692
|
`- run id: ${runId}`,
|
|
554
693
|
`- lifecycle: ${located.lifecycle}`,
|
|
694
|
+
`- effective status: ${summary.effectiveStatus}`,
|
|
695
|
+
`- raw status: ${summary.status}`,
|
|
696
|
+
`- state consistent: ${summary.stateConsistent ? "yes" : "no"}`,
|
|
697
|
+
`- liveness: ${summary.liveness ?? "unknown"}`,
|
|
698
|
+
`- can resume: ${summary.recoveryEligibility.canResume ? "yes" : "no"}`,
|
|
699
|
+
`- can reconcile: ${summary.recoveryEligibility.canReconcile ? "yes" : "no"}`,
|
|
700
|
+
`- health issues: ${summary.healthIssues.length > 0 ? summary.healthIssues.map((issue) => `${issue.code}: ${issue.message}`).join("; ") : "none"}`,
|
|
555
701
|
`- failed node: ${failure.nodeId ?? "-"}`,
|
|
556
702
|
`- raw failure: ${rawFailureCategory ?? "-"}`,
|
|
557
703
|
`- normalized category: ${normalizedCategory}`,
|
|
@@ -113,6 +113,7 @@ export async function executeDagNode(input) {
|
|
|
113
113
|
const node = state.nodes[nodeId];
|
|
114
114
|
node.status = "RUNNING";
|
|
115
115
|
node.startedAt = new Date().toISOString();
|
|
116
|
+
node.lastActivityAt = node.startedAt;
|
|
116
117
|
if (task.shell?.verifyEvidence) {
|
|
117
118
|
node.verifyEvidence = task.shell.verifyEvidence;
|
|
118
119
|
}
|
|
@@ -140,6 +141,7 @@ export async function executeDagNode(input) {
|
|
|
140
141
|
node.stderr = result.stderr;
|
|
141
142
|
node.failureCategory = result.failureCategory;
|
|
142
143
|
node.finishedAt = new Date().toISOString();
|
|
144
|
+
node.lastActivityAt = node.finishedAt;
|
|
143
145
|
node.status = result.ok ? "FINISHED" : "ERROR";
|
|
144
146
|
}
|
|
145
147
|
catch (error) {
|
|
@@ -202,6 +204,7 @@ export async function executeDagNode(input) {
|
|
|
202
204
|
await notifyNodeObserver(input.observer, "onNodeOutput", nodeId, state, outputChunk);
|
|
203
205
|
}
|
|
204
206
|
node.finishedAt = new Date().toISOString();
|
|
207
|
+
node.lastActivityAt = node.finishedAt;
|
|
205
208
|
node.status = result.ok ? "FINISHED" : "ERROR";
|
|
206
209
|
if (result.ok) {
|
|
207
210
|
const decisionRecord = await recordDecisionEnvelopeForNode({
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DEFAULT_DAG_OUTPUT_LANGUAGE, } from "./types.js";
|
|
1
2
|
import { formatStdoutPreview, formatUpstreamArtifactPointerMap, } from "./upstream-artifacts.js";
|
|
2
3
|
export const MAX_UPSTREAM_CHARS = 2_000;
|
|
3
4
|
/** Shared bullets for DAG authoring templates, docs, and planner-node envelopes. */
|
|
@@ -41,6 +42,20 @@ function formatResolvedSkillInstructions(instructions) {
|
|
|
41
42
|
.map((instruction) => instruction.promptText)
|
|
42
43
|
.join("\n\n---\n\n");
|
|
43
44
|
}
|
|
45
|
+
export function formatOutputLanguageBlock(language = DEFAULT_DAG_OUTPUT_LANGUAGE) {
|
|
46
|
+
if (language === "en") {
|
|
47
|
+
return [
|
|
48
|
+
"Write prose, analysis, reports, summaries, and documentation in English.",
|
|
49
|
+
"Keep code, commands, paths, identifiers, JSON keys, exact protocol tokens, verdict lines, and output-contract literals unchanged.",
|
|
50
|
+
"If the node task explicitly requires another language, follow the explicit task requirement.",
|
|
51
|
+
].join("\n");
|
|
52
|
+
}
|
|
53
|
+
return [
|
|
54
|
+
"使用简体中文撰写说明、分析、报告、总结和文档正文。",
|
|
55
|
+
"代码、命令、路径、标识符、JSON 字段、精确协议 token、VERDICT 行以及 output contract 中要求的字面量保持原样,不要翻译。",
|
|
56
|
+
"如果当前节点任务明确要求其他语言,以节点的明确要求为准。",
|
|
57
|
+
].join("\n");
|
|
58
|
+
}
|
|
44
59
|
export function buildUpstreamContext(task, upstream, maxChars = MAX_UPSTREAM_CHARS) {
|
|
45
60
|
const sections = [];
|
|
46
61
|
for (const depId of task.depends_on) {
|
|
@@ -91,6 +106,7 @@ export function buildDagNodePromptEnvelope(input) {
|
|
|
91
106
|
`<dag_objective>\n${objective}\n</dag_objective>`,
|
|
92
107
|
`<success_criteria>\n${successCriteria}\n</success_criteria>`,
|
|
93
108
|
`<global_constraints>\n${globalConstraints}\n</global_constraints>`,
|
|
109
|
+
`<output_language>\n${formatOutputLanguageBlock(spec.outputLanguage)}\n</output_language>`,
|
|
94
110
|
[
|
|
95
111
|
"<node_contract>",
|
|
96
112
|
`Role: ${role}`,
|
|
@@ -20,6 +20,8 @@ const dagRunStatusSchema = z.enum([
|
|
|
20
20
|
"partial_failed",
|
|
21
21
|
"failed",
|
|
22
22
|
"paused",
|
|
23
|
+
"superseded",
|
|
24
|
+
"abandoned",
|
|
23
25
|
]);
|
|
24
26
|
const dagRecoveryActionSchema = z.enum(DAG_RECOVERY_ACTIONS);
|
|
25
27
|
const dagProductLineFailureCategorySchema = z.enum(dagProductLineFailureCategoryValues);
|