@tea-agent/loop-agent 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/CHANGELOG.md +51 -1
- package/README.md +20 -0
- package/dist/application/dag/args.js +9 -2
- package/dist/cli/command-definitions.js +7 -0
- package/dist/cli/program.js +6 -1
- package/dist/commands/dag-reconcile-run.js +118 -0
- package/dist/commands/init.js +12 -3
- package/dist/executors/shell-executor.js +74 -8
- package/dist/governance/manifest-types.js +4 -0
- package/dist/shared/reference-context.js +48 -22
- package/dist/task/config-types.js +1 -1
- package/dist/task/runtime.js +1 -1
- package/dist/worker/cli.js +216 -0
- package/dist/worker/closeout/apply.js +73 -0
- package/dist/worker/closeout/preview.js +30 -0
- package/dist/worker/delivery/final-verification.js +158 -0
- package/dist/worker/delivery/git-transaction.js +354 -0
- package/dist/worker/delivery/package.js +449 -0
- package/dist/worker/feature/decision-loader.js +68 -0
- package/dist/worker/feature/discover.js +14 -0
- package/dist/worker/feature/next-action.js +74 -0
- package/dist/worker/feature/reducer.js +133 -0
- package/dist/worker/feature/review.js +502 -0
- package/dist/worker/feature/run.js +313 -0
- package/dist/worker/feature/types.js +1 -0
- package/dist/worker/follow-up/approve.js +270 -0
- package/dist/worker/follow-up/factory.js +234 -0
- package/dist/worker/follow-up/paths.js +25 -0
- package/dist/worker/follow-up/policy.js +26 -0
- package/dist/worker/follow-up/schema.js +93 -0
- package/dist/worker/follow-up/store.js +96 -0
- package/dist/worker/loop-agent/loop-agent-client.js +51 -10
- package/dist/worker/metrics/projector.js +139 -0
- package/dist/worker/observability/read-model.js +256 -15
- package/dist/worker/observe/paths.js +17 -5
- package/dist/worker/observe/routes.js +78 -20
- package/dist/worker/observe/server.js +8 -6
- package/dist/worker/observe/static/app.js +1045 -177
- package/dist/worker/observe/static/index.html +70 -43
- package/dist/worker/observe/static/styles.css +553 -610
- package/dist/worker/pool/run-store.js +14 -2
- package/dist/worker/pool/validation.js +59 -0
- package/dist/worker/report/morning-report.js +41 -6
- package/dist/worker/run-task/run-task.js +1 -1
- package/dist/worker/runner/run-ready.js +19 -5
- package/dist/workflows/dag/init-hybrid.js +3 -1
- package/dist/workflows/dag/lifecycle.js +146 -0
- package/dist/workflows/dag/node-execution.js +3 -0
- package/dist/workflows/dag/prompt.js +16 -0
- package/dist/workflows/dag/report.js +2 -0
- package/dist/workflows/dag/runner.js +133 -104
- package/dist/workflows/dag/types.js +3 -0
- package/docs/README.md +21 -0
- package/docs/agent-dag-recovery-playbook.md +1 -1
- package/docs/architecture/runtime-boundaries.md +3 -2
- package/docs/design/README.md +13 -7
- package/docs/exec-plans/active/README.md +2 -2
- package/docs/exec-plans/completed/README.md +15 -0
- package/docs/loop-agent-harness.md +45 -2
- package/docs/progress/README.md +2 -0
- package/docs/reports/README.md +13 -0
- package/docs/templates/agent-dag-report.schema.json +5 -3
- package/docs/templates/harness.schema.json +7 -2
- package/docs/templates/init-evolution-review.md +4 -2
- package/docs/verification-matrix.md +7 -0
- package/harness.json +4 -3
- package/package.json +4 -2
- package/scripts/check-product-line-docs.sh +7 -3
- package/scripts/check-task-pool-root.sh +1 -1
- package/skills/init-capability-evolution/SKILL.md +1 -0
- package/skills/loop-agent/references/command-reference.md +21 -0
- package/skills/loop-agent/references/hybrid-dag.md +4 -3
- package/skills/loop-agent/references/verification-and-failure-handling.md +8 -3
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
|
|
5
|
+
import { readFollowUpIndex } from "../follow-up/store.js";
|
|
6
|
+
import { getTaskPoolRoot } from "../pool/run-store.js";
|
|
7
|
+
import { readValidTaskPoolRuns } from "../pool/validation.js";
|
|
8
|
+
export async function projectMonthlyMetrics(input) {
|
|
9
|
+
if (!/^\d{4}-\d{2}$/.test(input.month))
|
|
10
|
+
throw new Error("metrics month must be YYYY-MM");
|
|
11
|
+
const repoRoot = path.resolve(input.repoRoot);
|
|
12
|
+
const start = new Date(`${input.month}-01T00:00:00.000Z`);
|
|
13
|
+
const end = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth() + 1, 1));
|
|
14
|
+
if (!Number.isFinite(start.getTime()) || start.toISOString().slice(0, 7) !== input.month)
|
|
15
|
+
throw new Error("metrics month must be a valid YYYY-MM");
|
|
16
|
+
const { features, projectionWarnings } = await loadFeatureDecisionModels(repoRoot);
|
|
17
|
+
const validRuns = await readValidTaskPoolRuns(repoRoot);
|
|
18
|
+
const allRuns = validRuns.records;
|
|
19
|
+
projectionWarnings.push(...validRuns.warnings);
|
|
20
|
+
const runs = dedupeRuns(allRuns).filter((run) => inWindow(run.recordedAt, start, end));
|
|
21
|
+
const featureById = new Map(features.map((feature) => [feature.featureId, feature]));
|
|
22
|
+
const entered = new Set(runs.map((run) => run.featureId));
|
|
23
|
+
let closed = 0;
|
|
24
|
+
for (const id of entered) {
|
|
25
|
+
const feature = featureById.get(id);
|
|
26
|
+
if (feature?.status === "closed" && feature.evidence.closeout)
|
|
27
|
+
try {
|
|
28
|
+
const raw = (await import("yaml")).default.parse(await readFile(feature.evidence.closeout, "utf-8"));
|
|
29
|
+
if (raw.appliedAt && inWindow(raw.appliedAt, start, end))
|
|
30
|
+
closed += 1;
|
|
31
|
+
}
|
|
32
|
+
catch { }
|
|
33
|
+
}
|
|
34
|
+
const metricWarnings = [];
|
|
35
|
+
const followUps = new Map();
|
|
36
|
+
for (const id of new Set(allRuns.map((run) => run.featureId)))
|
|
37
|
+
try {
|
|
38
|
+
followUps.set(id, await readFollowUpIndex(repoRoot, id));
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
metricWarnings.push(`follow-up index unavailable for ${id}`);
|
|
42
|
+
}
|
|
43
|
+
const failed = runs.filter((run) => run.status !== "succeeded");
|
|
44
|
+
const handled = failed.filter((run) => followUps.get(run.featureId)?.entries.some((entry) => entry.workerRunId === run.workerRunId)).length;
|
|
45
|
+
const approved = [...followUps.entries()].flatMap(([featureId, index]) => index.entries.map((entry) => ({ ...entry, featureId }))).filter((entry) => entry.status === "Approved" && entry.approvedAt && inWindow(entry.approvedAt, start, end));
|
|
46
|
+
const executedApproved = approved.filter((entry) => allRuns.some((run) => run.featureId === entry.featureId && run.taskId === entry.proposedTaskId && Boolean(entry.approvedAt) && run.recordedAt >= entry.approvedAt));
|
|
47
|
+
const approvedSuccess = executedApproved.filter((entry) => allRuns.some((run) => run.featureId === entry.featureId && run.taskId === entry.proposedTaskId && run.status === "succeeded" && run.recordedAt >= entry.approvedAt)).length;
|
|
48
|
+
const taskTypes = new Map(features.flatMap((feature) => feature.tasks.map((task) => [`${feature.featureId}:${task.taskId}`, task.type])));
|
|
49
|
+
const successfulDev = new Set(runs.filter((run) => run.status === "succeeded" && isDevelopmentType(taskTypes.get(`${run.featureId}:${run.taskId}`))).map((run) => `${run.featureId}:${run.taskId}`));
|
|
50
|
+
const deliveryTasks = new Set();
|
|
51
|
+
for (const feature of features)
|
|
52
|
+
if (feature.evidence.delivery)
|
|
53
|
+
try {
|
|
54
|
+
const manifest = JSON.parse(await readFile(feature.evidence.delivery, "utf-8"));
|
|
55
|
+
if (manifest.createdAt && inWindow(manifest.createdAt, start, end))
|
|
56
|
+
for (const task of manifest.tasks ?? [])
|
|
57
|
+
if (task.taskId)
|
|
58
|
+
deliveryTasks.add(`${feature.featureId}:${task.taskId}`);
|
|
59
|
+
}
|
|
60
|
+
catch { }
|
|
61
|
+
const required = features.filter((feature) => entered.has(feature.featureId)).flatMap((feature) => feature.acceptanceCoverage.filter((item) => item.required));
|
|
62
|
+
const acComplete = required.filter((item) => item.status === "covered" || item.status === "waived").length;
|
|
63
|
+
const decisionLatencies = approved.flatMap((entry) => entry.approvedAt ? [new Date(entry.approvedAt).getTime() - new Date(entry.createdAt).getTime()] : []).filter((value) => value >= 0).sort((a, b) => a - b);
|
|
64
|
+
const recoveryRounds = [];
|
|
65
|
+
const recoveredTasks = new Set();
|
|
66
|
+
for (const taskId of new Set(runs.filter((run) => run.status !== "succeeded").map((run) => `${run.featureId}:${run.taskId}`))) {
|
|
67
|
+
const [featureId, id] = taskId.split(":");
|
|
68
|
+
const history = runs.filter((run) => run.featureId === featureId && run.taskId === id).sort((a, b) => a.recordedAt.localeCompare(b.recordedAt));
|
|
69
|
+
const firstFail = history.findIndex((run) => run.status !== "succeeded");
|
|
70
|
+
const success = history.findIndex((run, index) => index > firstFail && run.status === "succeeded");
|
|
71
|
+
if (firstFail >= 0 && success > firstFail) {
|
|
72
|
+
recoveryRounds.push(success - firstFail);
|
|
73
|
+
recoveredTasks.add(taskId);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
for (const entry of executedApproved) {
|
|
77
|
+
const parentKey = `${entry.featureId}:${entry.parentTaskId}`;
|
|
78
|
+
if (recoveredTasks.has(parentKey))
|
|
79
|
+
continue;
|
|
80
|
+
const sourceRun = allRuns.find((run) => run.featureId === entry.featureId && run.workerRunId === entry.workerRunId);
|
|
81
|
+
const replacementSucceeded = runs.some((run) => run.featureId === entry.featureId && run.taskId === entry.proposedTaskId && run.status === "succeeded" && run.recordedAt >= entry.approvedAt);
|
|
82
|
+
if (!sourceRun || !replacementSucceeded)
|
|
83
|
+
continue;
|
|
84
|
+
const failedAttempts = runs.filter((run) => run.featureId === entry.featureId && run.taskId === entry.parentTaskId && run.status !== "succeeded" && run.recordedAt <= sourceRun.recordedAt).length;
|
|
85
|
+
if (failedAttempts > 0) {
|
|
86
|
+
recoveryRounds.push(failedAttempts);
|
|
87
|
+
recoveredTasks.add(parentKey);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const boundary = await countBoundaryInterceptions(path.join(getTaskPoolRoot(repoRoot), "artifacts", "features"), start, end);
|
|
91
|
+
const latencySum = decisionLatencies.reduce((a, b) => a + b, 0);
|
|
92
|
+
const recoverySum = recoveryRounds.reduce((a, b) => a + b, 0);
|
|
93
|
+
const metrics = { schemaVersion: 1, month: input.month, window: { start: start.toISOString(), end: end.toISOString() }, generatedAt: (input.now ?? new Date()).toISOString(), metrics: {
|
|
94
|
+
featureClosureRate: ratio(closed, entered.size, [...(entered.size ? [] : ["no Feature entered execution in window"]), ...projectionWarnings]),
|
|
95
|
+
orphanFreeFailureRate: ratio(handled, failed.length, [...(failed.length ? [] : ["no failed runs in window"]), ...metricWarnings, ...projectionWarnings]),
|
|
96
|
+
approvedFollowUpSuccessRate: ratio(approvedSuccess, executedApproved.length, [...(executedApproved.length ? [] : approved.length ? ["approved Follow-up exists but has not executed"] : ["no approved Follow-up in window"]), ...metricWarnings, ...projectionWarnings]),
|
|
97
|
+
deliverableOutputRate: ratio([...successfulDev].filter((id) => deliveryTasks.has(id)).length, successfulDev.size, [...(successfulDev.size ? [] : ["no successful development task in window"]), ...projectionWarnings]),
|
|
98
|
+
requiredAcEvidenceRate: ratio(acComplete, required.length, [...(required.length ? ["formal Blocked decisions are not yet a distinct persisted fact"] : ["no required AC discovered"]), ...projectionWarnings]),
|
|
99
|
+
humanDecisionLatency: { numerator: latencySum, denominator: decisionLatencies.length, value: decisionLatencies.length ? latencySum / decisionLatencies.length : null, sampleSize: decisionLatencies.length, valuesMs: decisionLatencies, medianMs: median(decisionLatencies), missingData: [...(decisionLatencies.length ? [] : ["no completed approval decision in window"]), ...projectionWarnings] },
|
|
100
|
+
failureRecoveryRounds: { numerator: recoverySum, denominator: recoveryRounds.length, value: recoveryRounds.length ? recoverySum / recoveryRounds.length : null, sampleSize: recoveryRounds.length, values: recoveryRounds, average: recoveryRounds.length ? recoverySum / recoveryRounds.length : null, missingData: [...(recoveryRounds.length ? [] : ["no failed task recovered inside the requested window"]), ...projectionWarnings] },
|
|
101
|
+
writeBoundaryInterceptions: { numerator: boundary.interceptions, denominator: boundary.audited, value: boundary.audited ? boundary.interceptions / boundary.audited : null, count: boundary.interceptions, sampleSize: boundary.audited, missingData: [...(boundary.audited ? [] : ["no boundary audit artifacts in window"]), ...projectionWarnings] },
|
|
102
|
+
} };
|
|
103
|
+
const outputDir = path.join(getTaskPoolRoot(repoRoot), "artifacts", "metrics");
|
|
104
|
+
const jsonPath = path.join(outputDir, `monthly-${input.month}.json`);
|
|
105
|
+
const markdownPath = path.join(outputDir, `monthly-${input.month}.md`);
|
|
106
|
+
await mkdir(outputDir, { recursive: true });
|
|
107
|
+
await atomicWrite(jsonPath, `${JSON.stringify(metrics, null, 2)}\n`);
|
|
108
|
+
await atomicWrite(markdownPath, renderMetrics(metrics));
|
|
109
|
+
return { metrics, jsonPath, markdownPath };
|
|
110
|
+
}
|
|
111
|
+
function ratio(numerator, denominator, missingData) { return { numerator, denominator, value: denominator ? numerator / denominator : null, sampleSize: denominator, missingData }; }
|
|
112
|
+
function median(values) { if (!values.length)
|
|
113
|
+
return null; const middle = Math.floor(values.length / 2); return values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2; }
|
|
114
|
+
function inWindow(value, start, end) { const time = new Date(value).getTime(); return time >= start.getTime() && time < end.getTime(); }
|
|
115
|
+
function dedupeRuns(runs) { return [...new Map(runs.map((run) => [run.workerRunId, run])).values()]; }
|
|
116
|
+
async function countBoundaryInterceptions(root, start, end) { let interceptions = 0; let audited = 0; try {
|
|
117
|
+
for (const entry of await readdir(root, { withFileTypes: true })) {
|
|
118
|
+
const target = path.join(root, entry.name);
|
|
119
|
+
if (entry.isDirectory()) {
|
|
120
|
+
const child = await countBoundaryInterceptions(target, start, end);
|
|
121
|
+
interceptions += child.interceptions;
|
|
122
|
+
audited += child.audited;
|
|
123
|
+
}
|
|
124
|
+
else if (entry.name === "failure.json")
|
|
125
|
+
try {
|
|
126
|
+
const raw = JSON.parse(await readFile(target, "utf-8"));
|
|
127
|
+
if (typeof raw.writeBoundaryAudit?.ok === "boolean" && raw.capturedAt && inWindow(raw.capturedAt, start, end)) {
|
|
128
|
+
audited += 1;
|
|
129
|
+
if (!raw.writeBoundaryAudit.ok)
|
|
130
|
+
interceptions += 1;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
catch { }
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
catch { } return { interceptions, audited }; }
|
|
137
|
+
async function atomicWrite(filePath, content) { const temp = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${randomUUID()}.tmp`); await writeFile(temp, content); await rename(temp, filePath); }
|
|
138
|
+
function renderMetrics(metrics) { const rows = Object.entries(metrics.metrics).map(([name, value]) => `| ${name} | ${value.numerator} | ${value.denominator} | ${value.value ?? "n/a"} | ${value.sampleSize} | ${value.missingData.join("; ") || "none"} |`); return `# Monthly Metrics ${metrics.month}\n\nWindow: ${metrics.window.start} — ${metrics.window.end}\n\n| Metric | Numerator | Denominator | Value | Sample | Missing data |\n|---|---:|---:|---:|---:|---|\n${rows.join("\n")}\n`; }
|
|
139
|
+
function isDevelopmentType(type) { return Boolean(type && !type.startsWith("qa-") && !["architecture", "review"].includes(type)); }
|
|
@@ -4,6 +4,9 @@ import path from "node:path";
|
|
|
4
4
|
import { truncateUtf8Preview } from "../../shared/preview.js";
|
|
5
5
|
import { parseWorkerEventLine } from "./events.js";
|
|
6
6
|
import { getTaskPoolRoot } from "../pool/run-store.js";
|
|
7
|
+
import { assessDagRunLiveness, assessDagRunRecoveryEligibility, deriveDagRunEffectiveStatus, } from "../../workflows/dag/lifecycle.js";
|
|
8
|
+
import { parseDagSpec, resolveModelForTask, } from "../../workflows/dag/types.js";
|
|
9
|
+
import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
|
|
7
10
|
const ACTIVE_TASK_STATUSES = new Set(["running", "pending"]);
|
|
8
11
|
const ACTIVE_BATCH_STATUSES = new Set(["running", "pending"]);
|
|
9
12
|
export async function buildGlobalSnapshot(options) {
|
|
@@ -18,7 +21,7 @@ export async function buildGlobalSnapshot(options) {
|
|
|
18
21
|
loadLedgerBatches(repoRoot),
|
|
19
22
|
loadStateRecords(repoRoot),
|
|
20
23
|
loadFailureHandoffs(repoRoot),
|
|
21
|
-
loadDagRuns(repoRoot),
|
|
24
|
+
loadDagRuns(repoRoot, now()),
|
|
22
25
|
]);
|
|
23
26
|
const taskMap = new Map();
|
|
24
27
|
for (const state of states) {
|
|
@@ -41,7 +44,8 @@ export async function buildGlobalSnapshot(options) {
|
|
|
41
44
|
...batch,
|
|
42
45
|
tasks: batch.tasks.map((task) => normalizeTaskArtifactRefs(task, repoRoot)),
|
|
43
46
|
}));
|
|
44
|
-
const health = computeHealth(batches, tasks);
|
|
47
|
+
const health = computeHealth(batches, tasks, dagRuns);
|
|
48
|
+
const { features, projectionWarnings } = await loadFeatureDecisionModels(repoRoot);
|
|
45
49
|
return {
|
|
46
50
|
schemaVersion: 1,
|
|
47
51
|
generatedAt: now().toISOString(),
|
|
@@ -49,12 +53,31 @@ export async function buildGlobalSnapshot(options) {
|
|
|
49
53
|
batches,
|
|
50
54
|
tasks,
|
|
51
55
|
dagRuns,
|
|
56
|
+
features,
|
|
57
|
+
projectionWarnings,
|
|
52
58
|
health,
|
|
53
59
|
};
|
|
54
60
|
}
|
|
55
|
-
catch {
|
|
61
|
+
catch (error) {
|
|
56
62
|
const now = options.now ?? (() => new Date());
|
|
57
|
-
|
|
63
|
+
const safeNow = () => {
|
|
64
|
+
try {
|
|
65
|
+
return now();
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return new Date();
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
let repoRoot;
|
|
72
|
+
try {
|
|
73
|
+
repoRoot = path.resolve(options.repoRoot);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
repoRoot = String(options.repoRoot ?? "");
|
|
77
|
+
}
|
|
78
|
+
const snapshot = emptySnapshot(repoRoot, safeNow);
|
|
79
|
+
snapshot.projectionError = sanitizeProjectionError(error);
|
|
80
|
+
return snapshot;
|
|
58
81
|
}
|
|
59
82
|
}
|
|
60
83
|
function emptySnapshot(repoRoot, now) {
|
|
@@ -72,6 +95,25 @@ function emptySnapshot(repoRoot, now) {
|
|
|
72
95
|
staleCount: 0,
|
|
73
96
|
timeoutRiskCount: 0,
|
|
74
97
|
failuresCount: 0,
|
|
98
|
+
dag: {
|
|
99
|
+
activeRuns: 0,
|
|
100
|
+
runningNodes: 0,
|
|
101
|
+
pendingNodes: 0,
|
|
102
|
+
pausedRuns: 0,
|
|
103
|
+
failedRuns: 0,
|
|
104
|
+
staleRuns: 0,
|
|
105
|
+
interruptedRuns: 0,
|
|
106
|
+
inconsistentRuns: 0,
|
|
107
|
+
attentionRuns: 0,
|
|
108
|
+
},
|
|
109
|
+
worker: {
|
|
110
|
+
activeTasks: 0,
|
|
111
|
+
activeBatches: 0,
|
|
112
|
+
quietCount: 0,
|
|
113
|
+
staleCount: 0,
|
|
114
|
+
timeoutRiskCount: 0,
|
|
115
|
+
failuresCount: 0,
|
|
116
|
+
},
|
|
75
117
|
},
|
|
76
118
|
};
|
|
77
119
|
}
|
|
@@ -369,15 +411,168 @@ function summarizeTasks(tasks) {
|
|
|
369
411
|
}
|
|
370
412
|
return { total: tasks.length, succeeded, failed, reused };
|
|
371
413
|
}
|
|
372
|
-
function computeHealth(batches, tasks) {
|
|
373
|
-
|
|
374
|
-
activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status)).length,
|
|
414
|
+
function computeHealth(batches, tasks, dagRuns) {
|
|
415
|
+
const worker = {
|
|
375
416
|
activeTasks: tasks.filter((t) => ACTIVE_TASK_STATUSES.has(t.status)).length,
|
|
417
|
+
activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status)).length,
|
|
376
418
|
quietCount: tasks.filter((t) => t.liveness === "quiet").length,
|
|
377
419
|
staleCount: tasks.filter((t) => t.status === "stale").length,
|
|
378
420
|
timeoutRiskCount: tasks.filter((t) => t.liveness === "timeout-risk").length,
|
|
379
421
|
failuresCount: tasks.filter((t) => t.status === "failed").length,
|
|
380
422
|
};
|
|
423
|
+
const dag = computeDagHealth(dagRuns);
|
|
424
|
+
return {
|
|
425
|
+
// Flat aliases retained for back-compat; mirror worker.* plus activeBatches.
|
|
426
|
+
activeBatches: worker.activeBatches,
|
|
427
|
+
activeTasks: worker.activeTasks,
|
|
428
|
+
quietCount: worker.quietCount,
|
|
429
|
+
staleCount: worker.staleCount,
|
|
430
|
+
timeoutRiskCount: worker.timeoutRiskCount,
|
|
431
|
+
failuresCount: worker.failuresCount,
|
|
432
|
+
dag,
|
|
433
|
+
worker,
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
const ACTIVE_EFFECTIVE_STATUSES = new Set([
|
|
437
|
+
"running",
|
|
438
|
+
"running-quiet",
|
|
439
|
+
"remote-unknown",
|
|
440
|
+
"pending",
|
|
441
|
+
]);
|
|
442
|
+
const TERMINAL_RUN_STATUSES = new Set([
|
|
443
|
+
"finished",
|
|
444
|
+
"failed",
|
|
445
|
+
"partial_failed",
|
|
446
|
+
"superseded",
|
|
447
|
+
"abandoned",
|
|
448
|
+
]);
|
|
449
|
+
/**
|
|
450
|
+
* Server-side mirror of app.js isDagRunActive. Derives DAG-run activeness from
|
|
451
|
+
* already-loaded DagRunSummary facts without touching app.js or new files.
|
|
452
|
+
*/
|
|
453
|
+
function isDagRunActiveForHealth(dag) {
|
|
454
|
+
if (dag.effectiveStatus && ACTIVE_EFFECTIVE_STATUSES.has(dag.effectiveStatus)) {
|
|
455
|
+
return true;
|
|
456
|
+
}
|
|
457
|
+
// Only "unknown" (and absent) means liveness evidence is insufficient;
|
|
458
|
+
// it must NOT override raw lifecycle/status/node facts. Every other concrete
|
|
459
|
+
// effectiveStatus is authoritative-non-active and short-circuits to false.
|
|
460
|
+
if (dag.effectiveStatus && dag.effectiveStatus !== "unknown")
|
|
461
|
+
return false;
|
|
462
|
+
const status = (dag.status ?? "").toLowerCase();
|
|
463
|
+
if ((dag.lifecycle ?? "").toLowerCase() === "paused")
|
|
464
|
+
return false;
|
|
465
|
+
if (["orphaned", "stale"].includes((dag.liveness ?? "").toLowerCase()))
|
|
466
|
+
return false;
|
|
467
|
+
if (TERMINAL_RUN_STATUSES.has(status))
|
|
468
|
+
return false;
|
|
469
|
+
if (["running", "pending", "started"].includes(status))
|
|
470
|
+
return true;
|
|
471
|
+
return (dag.nodes ?? []).some((node) => isNodeStatusActive(node.status));
|
|
472
|
+
}
|
|
473
|
+
function isNodeStatusActive(status) {
|
|
474
|
+
const normalized = (status ?? "").toLowerCase();
|
|
475
|
+
if (["finished", "completed", "succeeded", "done", "error", "failed", "skipped", "partial_failed"].includes(normalized)) {
|
|
476
|
+
return false;
|
|
477
|
+
}
|
|
478
|
+
return true;
|
|
479
|
+
}
|
|
480
|
+
function computeDagHealth(dagRuns) {
|
|
481
|
+
let activeRuns = 0;
|
|
482
|
+
let runningNodes = 0;
|
|
483
|
+
let pendingNodes = 0;
|
|
484
|
+
let pausedRuns = 0;
|
|
485
|
+
let failedRuns = 0;
|
|
486
|
+
let staleRuns = 0;
|
|
487
|
+
let interruptedRuns = 0;
|
|
488
|
+
let inconsistentRuns = 0;
|
|
489
|
+
let attentionRuns = 0;
|
|
490
|
+
for (const dag of dagRuns) {
|
|
491
|
+
const active = isDagRunActiveForHealth(dag);
|
|
492
|
+
if (active) {
|
|
493
|
+
activeRuns++;
|
|
494
|
+
for (const node of dag.nodes ?? []) {
|
|
495
|
+
const status = (node.status ?? "").toLowerCase();
|
|
496
|
+
if (status === "running" || status === "started")
|
|
497
|
+
runningNodes++;
|
|
498
|
+
if (status === "pending" || status === "queued")
|
|
499
|
+
pendingNodes++;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
const lifecycle = (dag.lifecycle ?? "").toLowerCase();
|
|
503
|
+
const liveness = (dag.liveness ?? "").toLowerCase();
|
|
504
|
+
if (lifecycle === "paused")
|
|
505
|
+
pausedRuns++;
|
|
506
|
+
if (dag.effectiveStatus === "failed")
|
|
507
|
+
failedRuns++;
|
|
508
|
+
if (liveness === "stale" || liveness === "orphaned")
|
|
509
|
+
staleRuns++;
|
|
510
|
+
if (dag.effectiveStatus === "interrupted")
|
|
511
|
+
interruptedRuns++;
|
|
512
|
+
if (dag.stateConsistent === false)
|
|
513
|
+
inconsistentRuns++;
|
|
514
|
+
if (lifecycle !== "completed"
|
|
515
|
+
&& (lifecycle === "paused"
|
|
516
|
+
|| liveness === "stale"
|
|
517
|
+
|| liveness === "orphaned"
|
|
518
|
+
|| dag.effectiveStatus === "interrupted"
|
|
519
|
+
|| dag.effectiveStatus === "remote-unknown"
|
|
520
|
+
|| dag.stateConsistent === false)) {
|
|
521
|
+
attentionRuns++;
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return {
|
|
525
|
+
activeRuns,
|
|
526
|
+
runningNodes,
|
|
527
|
+
pendingNodes,
|
|
528
|
+
pausedRuns,
|
|
529
|
+
failedRuns,
|
|
530
|
+
staleRuns,
|
|
531
|
+
interruptedRuns,
|
|
532
|
+
inconsistentRuns,
|
|
533
|
+
attentionRuns,
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Whitelist a short safe projection-failure message. Never expose stack frames,
|
|
538
|
+
* absolute paths, env values, or secrets. Mirrors the bare catch that previously
|
|
539
|
+
* silently returned a zero-shape snapshot.
|
|
540
|
+
*/
|
|
541
|
+
function sanitizeProjectionError(error) {
|
|
542
|
+
const fallback = "投影失败 (projection failed)";
|
|
543
|
+
const at = new Date().toISOString();
|
|
544
|
+
if (!error || typeof error !== "object") {
|
|
545
|
+
return { message: fallback, at };
|
|
546
|
+
}
|
|
547
|
+
let name;
|
|
548
|
+
let message;
|
|
549
|
+
try {
|
|
550
|
+
name = typeof error.name === "string"
|
|
551
|
+
? error.name
|
|
552
|
+
: undefined;
|
|
553
|
+
message = typeof error.message === "string"
|
|
554
|
+
? error.message
|
|
555
|
+
: undefined;
|
|
556
|
+
}
|
|
557
|
+
catch {
|
|
558
|
+
return { message: fallback, at };
|
|
559
|
+
}
|
|
560
|
+
const candidate = message ?? name;
|
|
561
|
+
if (!candidate) {
|
|
562
|
+
return { message: fallback, at };
|
|
563
|
+
}
|
|
564
|
+
// Reject anything that looks like a stack frame, an absolute path, or a secret.
|
|
565
|
+
const raw = String(candidate);
|
|
566
|
+
if (raw.includes("\n")
|
|
567
|
+
|| /\bat \S+/i.test(raw)
|
|
568
|
+
|| /[A-Za-z]:\\/.test(raw)
|
|
569
|
+
|| raw.includes("/Users/")
|
|
570
|
+
|| raw.includes("/home/")
|
|
571
|
+
|| /SECRET|TOKEN|PASSWORD|API_KEY/i.test(raw)) {
|
|
572
|
+
return { message: name ? `${name}: 投影失败` : fallback, at };
|
|
573
|
+
}
|
|
574
|
+
const trimmed = raw.trim().slice(0, 200);
|
|
575
|
+
return { message: trimmed || fallback, at };
|
|
381
576
|
}
|
|
382
577
|
function mergeArtifactRefs(existing, incoming) {
|
|
383
578
|
if (!incoming)
|
|
@@ -643,11 +838,11 @@ async function loadFailureHandoffs(repoRoot) {
|
|
|
643
838
|
}
|
|
644
839
|
return handoffs;
|
|
645
840
|
}
|
|
646
|
-
async function loadDagRuns(repoRoot) {
|
|
841
|
+
async function loadDagRuns(repoRoot, now) {
|
|
647
842
|
const byId = new Map();
|
|
648
843
|
const statePaths = await findStateJsonFiles(path.join(repoRoot, ".harness", "dag-runs"));
|
|
649
844
|
for (const statePath of statePaths) {
|
|
650
|
-
const summary = await parseDagStateFile(statePath);
|
|
845
|
+
const summary = await parseDagStateFile(statePath, now);
|
|
651
846
|
if (summary)
|
|
652
847
|
byId.set(summary.dagRunId, mergeDagRun(byId.get(summary.dagRunId), summary));
|
|
653
848
|
}
|
|
@@ -670,6 +865,11 @@ function mergeDagRun(existing, incoming) {
|
|
|
670
865
|
return {
|
|
671
866
|
dagRunId: incoming.dagRunId,
|
|
672
867
|
status: incoming.status ?? existing.status,
|
|
868
|
+
lifecycle: incoming.lifecycle ?? existing.lifecycle,
|
|
869
|
+
liveness: incoming.liveness ?? existing.liveness,
|
|
870
|
+
effectiveStatus: incoming.effectiveStatus ?? existing.effectiveStatus,
|
|
871
|
+
stateConsistent: incoming.stateConsistent ?? existing.stateConsistent,
|
|
872
|
+
recoveryEligibility: incoming.recoveryEligibility ?? existing.recoveryEligibility,
|
|
673
873
|
title: incoming.title ?? existing.title,
|
|
674
874
|
startedAt,
|
|
675
875
|
finishedAt,
|
|
@@ -849,7 +1049,7 @@ async function walkForStateJson(dir, results) {
|
|
|
849
1049
|
// skip
|
|
850
1050
|
}
|
|
851
1051
|
}
|
|
852
|
-
async function parseDagStateFile(statePath) {
|
|
1052
|
+
async function parseDagStateFile(statePath, now) {
|
|
853
1053
|
try {
|
|
854
1054
|
if (!existsSync(statePath))
|
|
855
1055
|
return undefined;
|
|
@@ -858,6 +1058,10 @@ async function parseDagStateFile(statePath) {
|
|
|
858
1058
|
if (!parsed)
|
|
859
1059
|
return undefined;
|
|
860
1060
|
const runDir = path.dirname(statePath);
|
|
1061
|
+
const lifecycleName = path.basename(path.dirname(runDir));
|
|
1062
|
+
const lifecycle = lifecycleName === "active" || lifecycleName === "paused" || lifecycleName === "completed"
|
|
1063
|
+
? lifecycleName
|
|
1064
|
+
: undefined;
|
|
861
1065
|
const dagRunId = readString(parsed, "runId") ?? path.basename(runDir);
|
|
862
1066
|
const status = readString(parsed, "status");
|
|
863
1067
|
const title = readString(parsed, "title");
|
|
@@ -866,11 +1070,30 @@ async function parseDagStateFile(statePath) {
|
|
|
866
1070
|
const durationMs = computeDurationMs(startedAt, finishedAt);
|
|
867
1071
|
const ranks = readStringMatrix(parsed, "ranks");
|
|
868
1072
|
const rankByNode = buildRankIndex(ranks);
|
|
869
|
-
const
|
|
870
|
-
const
|
|
1073
|
+
const runSpecPath = path.join(runDir, "run.json");
|
|
1074
|
+
const modelByNode = await loadDagNodeModels(runSpecPath);
|
|
1075
|
+
const nodes = parseDagNodes(parsed, rankByNode, modelByNode);
|
|
1076
|
+
const edges = await parseDagEdges(runSpecPath, nodes);
|
|
1077
|
+
const state = parsed;
|
|
1078
|
+
const liveness = assessDagRunLiveness({ state, now });
|
|
1079
|
+
const effectiveStatus = lifecycle
|
|
1080
|
+
? deriveDagRunEffectiveStatus({ lifecycle, state, liveness: liveness.status })
|
|
1081
|
+
: "unknown";
|
|
1082
|
+
const stateConsistent = lifecycle === undefined
|
|
1083
|
+
? false
|
|
1084
|
+
: !((lifecycle === "paused" && state.status !== "paused")
|
|
1085
|
+
|| (lifecycle === "completed" && !["finished", "failed", "partial_failed", "superseded", "abandoned"].includes(state.status)));
|
|
1086
|
+
const recoveryEligibility = lifecycle
|
|
1087
|
+
? assessDagRunRecoveryEligibility({ lifecycle, state, liveness: liveness.status })
|
|
1088
|
+
: undefined;
|
|
871
1089
|
return {
|
|
872
1090
|
dagRunId,
|
|
873
1091
|
status,
|
|
1092
|
+
...(lifecycle ? { lifecycle } : {}),
|
|
1093
|
+
liveness: liveness.status,
|
|
1094
|
+
effectiveStatus,
|
|
1095
|
+
stateConsistent,
|
|
1096
|
+
...(recoveryEligibility ? { recoveryEligibility } : {}),
|
|
874
1097
|
...(title ? { title } : {}),
|
|
875
1098
|
...(startedAt ? { startedAt } : {}),
|
|
876
1099
|
...(finishedAt ? { finishedAt } : {}),
|
|
@@ -924,7 +1147,25 @@ function buildRankIndex(ranks) {
|
|
|
924
1147
|
}
|
|
925
1148
|
return index;
|
|
926
1149
|
}
|
|
927
|
-
function
|
|
1150
|
+
async function loadDagNodeModels(runPath) {
|
|
1151
|
+
const models = new Map();
|
|
1152
|
+
const raw = await safeReadJson(runPath);
|
|
1153
|
+
if (!raw)
|
|
1154
|
+
return models;
|
|
1155
|
+
try {
|
|
1156
|
+
const spec = parseDagSpec(raw);
|
|
1157
|
+
for (const task of spec.tasks) {
|
|
1158
|
+
const model = resolveModelForTask(task, spec.executorModels);
|
|
1159
|
+
if (model)
|
|
1160
|
+
models.set(task.id, model);
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
catch {
|
|
1164
|
+
// Historical specs may predate the current schema; state facts remain readable.
|
|
1165
|
+
}
|
|
1166
|
+
return models;
|
|
1167
|
+
}
|
|
1168
|
+
function parseDagNodes(parsed, rankByNode, modelByNode) {
|
|
928
1169
|
const nodes = [];
|
|
929
1170
|
const nodesValue = parsed.nodes;
|
|
930
1171
|
if (nodesValue && typeof nodesValue === "object" && !Array.isArray(nodesValue)) {
|
|
@@ -940,7 +1181,7 @@ function parseDagNodes(parsed, rankByNode) {
|
|
|
940
1181
|
nodeId: resolvedId,
|
|
941
1182
|
rank: rankByNode.get(nodeId) ?? rankByNode.get(resolvedId),
|
|
942
1183
|
executor: readString(node, "executor"),
|
|
943
|
-
model: readString(node, "model"),
|
|
1184
|
+
model: readString(node, "model") ?? modelByNode.get(resolvedId),
|
|
944
1185
|
status: nodeStatus,
|
|
945
1186
|
label: readString(node, "label") ?? readString(node, "title") ?? nodeId,
|
|
946
1187
|
durationMs: readNumber(node, "durationMs"),
|
|
@@ -962,7 +1203,7 @@ function parseDagNodes(parsed, rankByNode) {
|
|
|
962
1203
|
nodeId,
|
|
963
1204
|
rank: rankByNode.get(nodeId),
|
|
964
1205
|
executor: readString(node, "executor"),
|
|
965
|
-
model: readString(node, "model"),
|
|
1206
|
+
model: readString(node, "model") ?? modelByNode.get(nodeId),
|
|
966
1207
|
status: nodeStatus,
|
|
967
1208
|
label: readString(node, "label") ?? readString(node, "title") ?? nodeId,
|
|
968
1209
|
durationMs: readNumber(node, "durationMs"),
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
import { existsSync, realpathSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
export function resolveArtifactPath(repoRoot,
|
|
4
|
-
if (!
|
|
5
|
-
return null;
|
|
6
|
-
}
|
|
7
|
-
if (!isAllowedArtifactRelativePath(relative)) {
|
|
3
|
+
export function resolveArtifactPath(repoRoot, artifactPath) {
|
|
4
|
+
if (!artifactPath) {
|
|
8
5
|
return null;
|
|
9
6
|
}
|
|
10
7
|
const resolvedRepoRoot = path.resolve(repoRoot);
|
|
@@ -15,6 +12,21 @@ export function resolveArtifactPath(repoRoot, relative) {
|
|
|
15
12
|
catch {
|
|
16
13
|
realRepoRoot = resolvedRepoRoot;
|
|
17
14
|
}
|
|
15
|
+
let absoluteArtifact = path.resolve(artifactPath);
|
|
16
|
+
if (path.isAbsolute(artifactPath) && existsSync(absoluteArtifact)) {
|
|
17
|
+
try {
|
|
18
|
+
absoluteArtifact = realpathSync(absoluteArtifact);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
const relative = path.isAbsolute(artifactPath)
|
|
25
|
+
? path.relative(realRepoRoot, absoluteArtifact)
|
|
26
|
+
: artifactPath;
|
|
27
|
+
if (!relative || path.isAbsolute(relative) || !isAllowedArtifactRelativePath(relative)) {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
18
30
|
const candidate = path.resolve(realRepoRoot, relative);
|
|
19
31
|
if (!isPathInside(realRepoRoot, candidate)) {
|
|
20
32
|
return null;
|