@tea-agent/loop-agent 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +3 -2
- package/dist/application/dag/generate-task-dag.js +15 -0
- package/dist/application/dag/run-dag.js +10 -0
- package/dist/application/dag/validate-dag.js +11 -0
- package/dist/commands/init.js +74 -7
- package/dist/shared/package-metadata.js +135 -0
- package/dist/task/config-types.js +1 -0
- package/dist/worker/cli.js +3 -1
- package/dist/worker/observability/event-history.js +216 -0
- package/dist/worker/observability/read-model.js +312 -83
- package/dist/worker/observe/paths.js +17 -0
- package/dist/worker/observe/routes.js +165 -21
- package/dist/worker/observe/server.js +59 -1
- package/dist/worker/observe/static/api.js +27 -0
- package/dist/worker/observe/static/app.js +120 -2598
- package/dist/worker/observe/static/constants.js +148 -0
- package/dist/worker/observe/static/copy.js +67 -0
- package/dist/worker/observe/static/dag-helpers.js +172 -0
- package/dist/worker/observe/static/dag-model.js +72 -0
- package/dist/worker/observe/static/dom.js +61 -0
- package/dist/worker/observe/static/format-pool.js +67 -0
- package/dist/worker/observe/static/format.js +292 -0
- package/dist/worker/observe/static/index.html +300 -82
- package/dist/worker/observe/static/kpi.js +94 -0
- package/dist/worker/observe/static/relations.js +128 -0
- package/dist/worker/observe/static/router.js +85 -0
- package/dist/worker/observe/static/run-processing.js +148 -0
- package/dist/worker/observe/static/shell-chrome.js +68 -0
- package/dist/worker/observe/static/state.js +253 -0
- package/dist/worker/observe/static/styles.css +1719 -495
- package/dist/worker/observe/static/views/batch.js +226 -0
- package/dist/worker/observe/static/views/dag-graph.js +172 -0
- package/dist/worker/observe/static/views/dag-inspector.js +477 -0
- package/dist/worker/observe/static/views/dag.js +362 -0
- package/dist/worker/observe/static/views/dashboard.js +442 -0
- package/dist/worker/observe/static/views/failures.js +143 -0
- package/dist/worker/observe/static/views/feature.js +453 -0
- package/dist/worker/observe/static/views/pool.js +347 -0
- package/dist/worker/observe/static/views/run.js +453 -0
- package/dist/worker/observe/static/views/session-timeline.js +205 -0
- package/dist/worker/observe/static/views/shell.js +7 -0
- package/dist/worker/observe/static/views/task.js +260 -0
- package/dist/worker/observe/static/views/timeline.js +163 -0
- package/dist/workflows/dag/controller-identity.js +104 -0
- package/dist/workflows/dag/init-hybrid.js +396 -3
- package/dist/workflows/dag/node-execution.js +123 -29
- package/dist/workflows/dag/repair-artifact.js +91 -0
- package/dist/workflows/dag/report.js +50 -0
- package/dist/workflows/dag/retry-policy.js +138 -0
- package/dist/workflows/dag/runner.js +32 -0
- package/dist/workflows/dag/runtime-contract.js +87 -0
- package/dist/workflows/dag/skill-snapshot.js +2 -0
- package/dist/workflows/dag/types.js +44 -1
- package/dist/workflows/dag/validate.js +68 -4
- package/docs/agent-dag-runner.md +26 -1
- package/docs/architecture/dag-execution.md +6 -0
- package/docs/architecture/evolution.md +4 -3
- package/docs/architecture/facts-and-state.md +1 -1
- package/docs/design/README.md +4 -3
- package/docs/exec-plans/active/README.md +1 -3
- package/docs/exec-plans/completed/README.md +11 -0
- package/docs/feature-workflow.md +28 -0
- package/docs/progress/README.md +18 -0
- package/docs/reports/README.md +8 -2
- package/docs/templates/agent-dag-report.schema.json +17 -0
- package/docs/templates/agent-dag.schema.json +69 -1
- package/docs/templates/agent-dag.supervised-implementation.json +8 -2
- package/docs/templates/backend-test-dag.generate-pytest.prompt.md +139 -0
- package/docs/templates/backend-test-dag.json +276 -0
- package/docs/templates/backend-test-dag.retrospect.prompt.md +125 -0
- package/docs/templates/backend-test-dag.review-cases.prompt.md +81 -0
- package/package.json +1 -1
- package/skills/loop-agent/references/command-reference.md +1 -0
- package/skills/loop-agent/references/hybrid-dag.md +22 -3
- package/skills/loop-agent/references/verification-and-failure-handling.md +6 -0
|
@@ -3,10 +3,13 @@ import { readdir, readFile } from "node:fs/promises";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { truncateUtf8Preview } from "../../shared/preview.js";
|
|
5
5
|
import { parseWorkerEventLine } from "./events.js";
|
|
6
|
+
import { readBoundedJsonlTail } from "./event-history.js";
|
|
6
7
|
import { getTaskPoolRoot } from "../pool/run-store.js";
|
|
7
8
|
import { assessDagRunLiveness, assessDagRunRecoveryEligibility, deriveDagRunEffectiveStatus, } from "../../workflows/dag/lifecycle.js";
|
|
8
9
|
import { parseDagSpec, resolveModelForTask, } from "../../workflows/dag/types.js";
|
|
9
10
|
import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
|
|
11
|
+
export const TASK_RUN_HISTORY_DEFAULT_LIMIT = 20;
|
|
12
|
+
export const TASK_RUN_HISTORY_MAX_LIMIT = 100;
|
|
10
13
|
const ACTIVE_TASK_STATUSES = new Set(["running", "pending"]);
|
|
11
14
|
const ACTIVE_BATCH_STATUSES = new Set(["running", "pending"]);
|
|
12
15
|
export async function buildGlobalSnapshot(options) {
|
|
@@ -15,7 +18,8 @@ export async function buildGlobalSnapshot(options) {
|
|
|
15
18
|
const now = options.now ?? (() => new Date());
|
|
16
19
|
const staleThresholdMs = options.staleThresholdMs ?? 90_000;
|
|
17
20
|
const quietThresholdMs = options.quietThresholdMs ?? 60_000;
|
|
18
|
-
const
|
|
21
|
+
const taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
|
|
22
|
+
const [eventsLoad, ledgerLoad, ledgerBatches, states, handoffs, dagRuns] = await Promise.all([
|
|
19
23
|
loadObservabilityEvents(repoRoot),
|
|
20
24
|
loadLedgerRuns(repoRoot),
|
|
21
25
|
loadLedgerBatches(repoRoot),
|
|
@@ -23,6 +27,8 @@ export async function buildGlobalSnapshot(options) {
|
|
|
23
27
|
loadFailureHandoffs(repoRoot),
|
|
24
28
|
loadDagRuns(repoRoot, now()),
|
|
25
29
|
]);
|
|
30
|
+
const events = eventsLoad.events;
|
|
31
|
+
const ledgerRuns = ledgerLoad.runs;
|
|
26
32
|
const taskMap = new Map();
|
|
27
33
|
for (const state of states) {
|
|
28
34
|
mergeStateRecord(taskMap, state);
|
|
@@ -45,7 +51,12 @@ export async function buildGlobalSnapshot(options) {
|
|
|
45
51
|
tasks: batch.tasks.map((task) => normalizeTaskArtifactRefs(task, repoRoot)),
|
|
46
52
|
}));
|
|
47
53
|
const health = computeHealth(batches, tasks, dagRuns);
|
|
48
|
-
const
|
|
54
|
+
const featureLoad = await loadFeatureDecisionModels(repoRoot);
|
|
55
|
+
const projectionWarnings = [
|
|
56
|
+
...(eventsLoad.warnings ?? []),
|
|
57
|
+
...(ledgerLoad.warnings ?? []),
|
|
58
|
+
...(featureLoad.projectionWarnings ?? []),
|
|
59
|
+
];
|
|
49
60
|
return {
|
|
50
61
|
schemaVersion: 1,
|
|
51
62
|
generatedAt: now().toISOString(),
|
|
@@ -53,7 +64,8 @@ export async function buildGlobalSnapshot(options) {
|
|
|
53
64
|
batches,
|
|
54
65
|
tasks,
|
|
55
66
|
dagRuns,
|
|
56
|
-
features,
|
|
67
|
+
features: featureLoad.features,
|
|
68
|
+
taskPool: { present: taskPoolPresent },
|
|
57
69
|
projectionWarnings,
|
|
58
70
|
health,
|
|
59
71
|
};
|
|
@@ -80,7 +92,99 @@ export async function buildGlobalSnapshot(options) {
|
|
|
80
92
|
return snapshot;
|
|
81
93
|
}
|
|
82
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Bounded Task run history from Task Pool ledger facts (newest first).
|
|
97
|
+
* `before` is an exclusive upper bound on the sort key (`recordedAt` or empty).
|
|
98
|
+
*/
|
|
99
|
+
export async function listTaskRunHistory(repoRoot, taskId, options = {}) {
|
|
100
|
+
const limit = clampTaskRunHistoryLimit(options.limit);
|
|
101
|
+
const before = decodeTaskRunHistoryCursor(options.before);
|
|
102
|
+
const [ledgerLoad, eventsLoad] = await Promise.all([
|
|
103
|
+
loadLedgerRuns(path.resolve(repoRoot)),
|
|
104
|
+
loadObservabilityEvents(path.resolve(repoRoot)),
|
|
105
|
+
]);
|
|
106
|
+
const ledgerRuns = ledgerLoad.runs;
|
|
107
|
+
const events = eventsLoad.events;
|
|
108
|
+
const historical = buildHistoricalRunMap(ledgerRuns, events);
|
|
109
|
+
const items = [];
|
|
110
|
+
for (const run of ledgerRuns) {
|
|
111
|
+
if (run.taskId !== taskId)
|
|
112
|
+
continue;
|
|
113
|
+
const enriched = historical.get(run.workerRunId);
|
|
114
|
+
items.push({
|
|
115
|
+
workerRunId: run.workerRunId,
|
|
116
|
+
taskId: run.taskId,
|
|
117
|
+
status: enriched?.status ?? mapRunRecordStatus(run.status),
|
|
118
|
+
startedAt: enriched?.startedAt,
|
|
119
|
+
finishedAt: enriched?.finishedAt ?? run.recordedAt,
|
|
120
|
+
recordedAt: run.recordedAt,
|
|
121
|
+
batchRunId: run.batchRunId,
|
|
122
|
+
dagRunId: enriched?.dagRunId,
|
|
123
|
+
...(run.retryOfWorkerRunId
|
|
124
|
+
? { retryOfWorkerRunId: run.retryOfWorkerRunId }
|
|
125
|
+
: {}),
|
|
126
|
+
failureCategory: run.failureCategory ?? enriched?.failureCategory,
|
|
127
|
+
recommendedFollowUp: run.recommendedFollowUp ?? enriched?.recommendedFollowUp,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
items.sort((a, b) => {
|
|
131
|
+
const ta = a.recordedAt ?? a.finishedAt ?? "";
|
|
132
|
+
const tb = b.recordedAt ?? b.finishedAt ?? "";
|
|
133
|
+
if (ta !== tb)
|
|
134
|
+
return tb.localeCompare(ta);
|
|
135
|
+
return b.workerRunId.localeCompare(a.workerRunId);
|
|
136
|
+
});
|
|
137
|
+
const eligible = before
|
|
138
|
+
? items.filter((item) => taskRunSortKey(item).localeCompare(before) < 0)
|
|
139
|
+
: items;
|
|
140
|
+
const page = eligible.slice(0, limit);
|
|
141
|
+
const nextBefore = eligible.length > limit
|
|
142
|
+
? encodeTaskRunHistoryCursor(page[page.length - 1])
|
|
143
|
+
: null;
|
|
144
|
+
return {
|
|
145
|
+
taskId,
|
|
146
|
+
runs: page,
|
|
147
|
+
limit,
|
|
148
|
+
nextBefore,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
function taskRunSortKey(item) {
|
|
152
|
+
return `${item.recordedAt ?? item.finishedAt ?? ""}\u0000${item.workerRunId}`;
|
|
153
|
+
}
|
|
154
|
+
function encodeTaskRunHistoryCursor(item) {
|
|
155
|
+
if (!item)
|
|
156
|
+
return null;
|
|
157
|
+
return `v1.${Buffer.from(taskRunSortKey(item), "utf-8").toString("base64url")}`;
|
|
158
|
+
}
|
|
159
|
+
function decodeTaskRunHistoryCursor(cursor) {
|
|
160
|
+
const value = cursor?.trim();
|
|
161
|
+
if (!value)
|
|
162
|
+
return null;
|
|
163
|
+
if (!value.startsWith("v1.")) {
|
|
164
|
+
// Compatibility with the initial timestamp-only cursor shape.
|
|
165
|
+
return `${value}\u0000`;
|
|
166
|
+
}
|
|
167
|
+
try {
|
|
168
|
+
return Buffer.from(value.slice(3), "base64url").toString("utf-8");
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return `${value}\u0000`;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
export function clampTaskRunHistoryLimit(limit) {
|
|
175
|
+
if (limit === undefined || !Number.isFinite(limit) || limit <= 0) {
|
|
176
|
+
return TASK_RUN_HISTORY_DEFAULT_LIMIT;
|
|
177
|
+
}
|
|
178
|
+
return Math.min(Math.floor(limit), TASK_RUN_HISTORY_MAX_LIMIT);
|
|
179
|
+
}
|
|
83
180
|
function emptySnapshot(repoRoot, now) {
|
|
181
|
+
let taskPoolPresent = false;
|
|
182
|
+
try {
|
|
183
|
+
taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
taskPoolPresent = false;
|
|
187
|
+
}
|
|
84
188
|
return {
|
|
85
189
|
schemaVersion: 1,
|
|
86
190
|
generatedAt: now().toISOString(),
|
|
@@ -88,6 +192,7 @@ function emptySnapshot(repoRoot, now) {
|
|
|
88
192
|
batches: [],
|
|
89
193
|
tasks: [],
|
|
90
194
|
dagRuns: [],
|
|
195
|
+
taskPool: { present: taskPoolPresent },
|
|
91
196
|
health: {
|
|
92
197
|
activeBatches: 0,
|
|
93
198
|
activeTasks: 0,
|
|
@@ -118,15 +223,21 @@ function emptySnapshot(repoRoot, now) {
|
|
|
118
223
|
};
|
|
119
224
|
}
|
|
120
225
|
function toPublicTaskSummary(task) {
|
|
121
|
-
const {
|
|
226
|
+
const { timeoutMs: _to, commandStartedAt: _cs, ...rest } = task;
|
|
122
227
|
return rest;
|
|
123
228
|
}
|
|
124
229
|
function mergeStateRecord(taskMap, state) {
|
|
125
|
-
const existing = taskMap.get(state.taskId) ?? {
|
|
230
|
+
const existing = taskMap.get(state.taskId) ?? {
|
|
231
|
+
taskId: state.taskId,
|
|
232
|
+
status: "unknown",
|
|
233
|
+
};
|
|
126
234
|
taskMap.set(state.taskId, {
|
|
127
235
|
...existing,
|
|
128
|
-
status: existing.status !== "unknown"
|
|
236
|
+
status: existing.status !== "unknown"
|
|
237
|
+
? existing.status
|
|
238
|
+
: mapStateStatus(state.status),
|
|
129
239
|
workerRunId: existing.workerRunId ?? state.workerRunId,
|
|
240
|
+
updatedAt: state.updatedAt ?? existing.updatedAt,
|
|
130
241
|
artifactRefs: mergeArtifactRefs(existing.artifactRefs, {
|
|
131
242
|
runRecordPath: state.lastRunRecordPath,
|
|
132
243
|
}),
|
|
@@ -135,7 +246,10 @@ function mergeStateRecord(taskMap, state) {
|
|
|
135
246
|
});
|
|
136
247
|
}
|
|
137
248
|
function mergeLedgerRun(taskMap, run) {
|
|
138
|
-
const existing = taskMap.get(run.taskId) ?? {
|
|
249
|
+
const existing = taskMap.get(run.taskId) ?? {
|
|
250
|
+
taskId: run.taskId,
|
|
251
|
+
status: "unknown",
|
|
252
|
+
};
|
|
139
253
|
const sameRun = existing.workerRunId === run.workerRunId;
|
|
140
254
|
const runArtifactRefs = {
|
|
141
255
|
runRecordPath: run.runRecordPath,
|
|
@@ -158,7 +272,8 @@ function mergeLedgerRun(taskMap, run) {
|
|
|
158
272
|
harnessTaskId: run.harnessTaskId ?? existing.harnessTaskId,
|
|
159
273
|
finishedAt: run.recordedAt ?? (sameRun ? existing.finishedAt : undefined),
|
|
160
274
|
failureCategory: run.failureCategory ?? (sameRun ? existing.failureCategory : undefined),
|
|
161
|
-
recommendedFollowUp: run.recommendedFollowUp ??
|
|
275
|
+
recommendedFollowUp: run.recommendedFollowUp ??
|
|
276
|
+
(sameRun ? existing.recommendedFollowUp : undefined),
|
|
162
277
|
artifactRefs: mergeArtifactRefs(sameRun ? existing.artifactRefs : undefined, runArtifactRefs),
|
|
163
278
|
});
|
|
164
279
|
}
|
|
@@ -172,7 +287,10 @@ function mergeEvents(taskMap, events) {
|
|
|
172
287
|
}
|
|
173
288
|
function applyTaskEvent(taskMap, event) {
|
|
174
289
|
const taskId = event.taskId;
|
|
175
|
-
const existing = taskMap.get(taskId) ?? {
|
|
290
|
+
const existing = taskMap.get(taskId) ?? {
|
|
291
|
+
taskId,
|
|
292
|
+
status: "unknown",
|
|
293
|
+
};
|
|
176
294
|
taskMap.set(taskId, applyEventToSummary(existing, event));
|
|
177
295
|
}
|
|
178
296
|
function applyEventToSummary(existing, event) {
|
|
@@ -320,7 +438,9 @@ function computeLiveness(task, nowMs, staleThresholdMs, quietThresholdMs) {
|
|
|
320
438
|
task.timeoutAt = new Date(commandStartedMs + timeoutMs).toISOString();
|
|
321
439
|
}
|
|
322
440
|
}
|
|
323
|
-
const heartbeatMs = task.lastHeartbeatAt
|
|
441
|
+
const heartbeatMs = task.lastHeartbeatAt
|
|
442
|
+
? Date.parse(task.lastHeartbeatAt)
|
|
443
|
+
: NaN;
|
|
324
444
|
const outputMs = task.lastOutputAt ? Date.parse(task.lastOutputAt) : NaN;
|
|
325
445
|
// Priority 1: timeout-risk (elapsed > 0.8 * timeoutMs), only if timeoutMs known.
|
|
326
446
|
if (timeoutMs !== undefined &&
|
|
@@ -333,7 +453,9 @@ function computeLiveness(task, nowMs, staleThresholdMs, quietThresholdMs) {
|
|
|
333
453
|
// (a) Heartbeat present but older than staleThresholdMs.
|
|
334
454
|
// (b) No heartbeat but running longer than staleThresholdMs (startedAt known).
|
|
335
455
|
const heartbeatStale = !Number.isNaN(heartbeatMs) && nowMs - heartbeatMs > staleThresholdMs;
|
|
336
|
-
const noHeartbeatStale = Number.isNaN(heartbeatMs) &&
|
|
456
|
+
const noHeartbeatStale = Number.isNaN(heartbeatMs) &&
|
|
457
|
+
elapsedMs !== undefined &&
|
|
458
|
+
elapsedMs > staleThresholdMs;
|
|
337
459
|
if (heartbeatStale || noHeartbeatStale) {
|
|
338
460
|
task.liveness = "stale";
|
|
339
461
|
task.status = "stale";
|
|
@@ -414,7 +536,8 @@ function summarizeTasks(tasks) {
|
|
|
414
536
|
function computeHealth(batches, tasks, dagRuns) {
|
|
415
537
|
const worker = {
|
|
416
538
|
activeTasks: tasks.filter((t) => ACTIVE_TASK_STATUSES.has(t.status)).length,
|
|
417
|
-
activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status))
|
|
539
|
+
activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status))
|
|
540
|
+
.length,
|
|
418
541
|
quietCount: tasks.filter((t) => t.liveness === "quiet").length,
|
|
419
542
|
staleCount: tasks.filter((t) => t.status === "stale").length,
|
|
420
543
|
timeoutRiskCount: tasks.filter((t) => t.liveness === "timeout-risk").length,
|
|
@@ -451,7 +574,8 @@ const TERMINAL_RUN_STATUSES = new Set([
|
|
|
451
574
|
* already-loaded DagRunSummary facts without touching app.js or new files.
|
|
452
575
|
*/
|
|
453
576
|
function isDagRunActiveForHealth(dag) {
|
|
454
|
-
if (dag.effectiveStatus &&
|
|
577
|
+
if (dag.effectiveStatus &&
|
|
578
|
+
ACTIVE_EFFECTIVE_STATUSES.has(dag.effectiveStatus)) {
|
|
455
579
|
return true;
|
|
456
580
|
}
|
|
457
581
|
// Only "unknown" (and absent) means liveness evidence is insufficient;
|
|
@@ -472,7 +596,16 @@ function isDagRunActiveForHealth(dag) {
|
|
|
472
596
|
}
|
|
473
597
|
function isNodeStatusActive(status) {
|
|
474
598
|
const normalized = (status ?? "").toLowerCase();
|
|
475
|
-
if ([
|
|
599
|
+
if ([
|
|
600
|
+
"finished",
|
|
601
|
+
"completed",
|
|
602
|
+
"succeeded",
|
|
603
|
+
"done",
|
|
604
|
+
"error",
|
|
605
|
+
"failed",
|
|
606
|
+
"skipped",
|
|
607
|
+
"partial_failed",
|
|
608
|
+
].includes(normalized)) {
|
|
476
609
|
return false;
|
|
477
610
|
}
|
|
478
611
|
return true;
|
|
@@ -511,13 +644,13 @@ function computeDagHealth(dagRuns) {
|
|
|
511
644
|
interruptedRuns++;
|
|
512
645
|
if (dag.stateConsistent === false)
|
|
513
646
|
inconsistentRuns++;
|
|
514
|
-
if (lifecycle !== "completed"
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
647
|
+
if (lifecycle !== "completed" &&
|
|
648
|
+
(lifecycle === "paused" ||
|
|
649
|
+
liveness === "stale" ||
|
|
650
|
+
liveness === "orphaned" ||
|
|
651
|
+
dag.effectiveStatus === "interrupted" ||
|
|
652
|
+
dag.effectiveStatus === "remote-unknown" ||
|
|
653
|
+
dag.stateConsistent === false)) {
|
|
521
654
|
attentionRuns++;
|
|
522
655
|
}
|
|
523
656
|
}
|
|
@@ -547,12 +680,14 @@ function sanitizeProjectionError(error) {
|
|
|
547
680
|
let name;
|
|
548
681
|
let message;
|
|
549
682
|
try {
|
|
550
|
-
name =
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
683
|
+
name =
|
|
684
|
+
typeof error.name === "string"
|
|
685
|
+
? error.name
|
|
686
|
+
: undefined;
|
|
687
|
+
message =
|
|
688
|
+
typeof error.message === "string"
|
|
689
|
+
? error.message
|
|
690
|
+
: undefined;
|
|
556
691
|
}
|
|
557
692
|
catch {
|
|
558
693
|
return { message: fallback, at };
|
|
@@ -563,12 +698,12 @@ function sanitizeProjectionError(error) {
|
|
|
563
698
|
}
|
|
564
699
|
// Reject anything that looks like a stack frame, an absolute path, or a secret.
|
|
565
700
|
const raw = String(candidate);
|
|
566
|
-
if (raw.includes("\n")
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
701
|
+
if (raw.includes("\n") ||
|
|
702
|
+
/\bat \S+/i.test(raw) ||
|
|
703
|
+
/[A-Za-z]:\\/.test(raw) ||
|
|
704
|
+
raw.includes("/Users/") ||
|
|
705
|
+
raw.includes("/home/") ||
|
|
706
|
+
/SECRET|TOKEN|PASSWORD|API_KEY/i.test(raw)) {
|
|
572
707
|
return { message: name ? `${name}: 投影失败` : fallback, at };
|
|
573
708
|
}
|
|
574
709
|
const trimmed = raw.trim().slice(0, 200);
|
|
@@ -581,7 +716,11 @@ function mergeArtifactRefs(existing, incoming) {
|
|
|
581
716
|
for (const [key, value] of Object.entries(incoming)) {
|
|
582
717
|
if (!value)
|
|
583
718
|
continue;
|
|
584
|
-
if (key === "runRecordPath" ||
|
|
719
|
+
if (key === "runRecordPath" ||
|
|
720
|
+
key === "dagPath" ||
|
|
721
|
+
key === "reportMarkdown" ||
|
|
722
|
+
key === "doctorMarkdown" ||
|
|
723
|
+
key === "closeoutDraft") {
|
|
585
724
|
merged[key] = value;
|
|
586
725
|
}
|
|
587
726
|
}
|
|
@@ -668,49 +807,86 @@ function mapBatchStatus(status) {
|
|
|
668
807
|
return "unknown";
|
|
669
808
|
}
|
|
670
809
|
}
|
|
810
|
+
/** Per-file hard cap for projection merge (snapshot must not embed unbounded ledger). */
|
|
811
|
+
const SNAPSHOT_EVENTS_MAX_LINES_PER_FILE = 2_000;
|
|
812
|
+
const SNAPSHOT_EVENTS_MAX_BYTES_PER_FILE = 4 * 1024 * 1024;
|
|
813
|
+
/** Global hard cap after dedupe for snapshot projection merge. */
|
|
814
|
+
const SNAPSHOT_EVENTS_MAX_TOTAL = 8_000;
|
|
815
|
+
const SNAPSHOT_LEDGER_MAX_LINES = 10_000;
|
|
816
|
+
const SNAPSHOT_LEDGER_MAX_BYTES = 8 * 1024 * 1024;
|
|
671
817
|
async function loadObservabilityEvents(repoRoot) {
|
|
672
818
|
const events = [];
|
|
819
|
+
const warnings = [];
|
|
673
820
|
const obsRoot = path.join(getTaskPoolRoot(repoRoot), "observability");
|
|
674
|
-
await
|
|
821
|
+
await appendJsonlEventsBounded(path.join(obsRoot, "events.jsonl"), events, warnings);
|
|
675
822
|
const batchesDir = path.join(obsRoot, "batches");
|
|
676
|
-
await forEachSubdirJsonl(batchesDir, "events.jsonl", events);
|
|
823
|
+
await forEachSubdirJsonl(batchesDir, "events.jsonl", events, warnings);
|
|
677
824
|
const runsDir = path.join(obsRoot, "runs");
|
|
678
|
-
await forEachSubdirJsonl(runsDir, "events.jsonl", events);
|
|
679
|
-
|
|
825
|
+
await forEachSubdirJsonl(runsDir, "events.jsonl", events, warnings);
|
|
826
|
+
const deduped = [
|
|
827
|
+
...new Map(events.map((event) => [event.id, event])).values(),
|
|
828
|
+
].sort(compareWorkerEvents);
|
|
829
|
+
if (deduped.length > SNAPSHOT_EVENTS_MAX_TOTAL) {
|
|
830
|
+
warnings.push(`observability event projection truncated to ${SNAPSHOT_EVENTS_MAX_TOTAL} events (capacity bound)`);
|
|
831
|
+
return {
|
|
832
|
+
events: deduped.slice(-SNAPSHOT_EVENTS_MAX_TOTAL),
|
|
833
|
+
warnings,
|
|
834
|
+
};
|
|
835
|
+
}
|
|
836
|
+
return { events: deduped, warnings };
|
|
680
837
|
}
|
|
681
|
-
|
|
838
|
+
function compareWorkerEvents(a, b) {
|
|
839
|
+
const byTime = a.at.localeCompare(b.at);
|
|
840
|
+
if (byTime !== 0)
|
|
841
|
+
return byTime;
|
|
842
|
+
return a.id.localeCompare(b.id);
|
|
843
|
+
}
|
|
844
|
+
async function forEachSubdirJsonl(parentDir, filename, events, warnings) {
|
|
682
845
|
try {
|
|
683
846
|
const entries = await readdir(parentDir, { withFileTypes: true });
|
|
684
847
|
for (const entry of entries) {
|
|
685
848
|
if (!entry.isDirectory())
|
|
686
849
|
continue;
|
|
687
|
-
await
|
|
850
|
+
await appendJsonlEventsBounded(path.join(parentDir, entry.name, filename), events, warnings);
|
|
688
851
|
}
|
|
689
852
|
}
|
|
690
853
|
catch {
|
|
691
854
|
// skip missing dir
|
|
692
855
|
}
|
|
693
856
|
}
|
|
694
|
-
async function
|
|
695
|
-
const
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
857
|
+
async function appendJsonlEventsBounded(filePath, events, warnings) {
|
|
858
|
+
const tail = await readBoundedJsonlTail(filePath, {
|
|
859
|
+
maxBytes: SNAPSHOT_EVENTS_MAX_BYTES_PER_FILE,
|
|
860
|
+
maxLines: SNAPSHOT_EVENTS_MAX_LINES_PER_FILE,
|
|
861
|
+
});
|
|
862
|
+
let accepted = 0;
|
|
863
|
+
for (const line of tail.lines) {
|
|
864
|
+
const trimmed = line.text.trim();
|
|
700
865
|
if (!trimmed)
|
|
701
866
|
continue;
|
|
702
867
|
const event = parseWorkerEventLine(trimmed);
|
|
703
|
-
if (event)
|
|
868
|
+
if (event) {
|
|
704
869
|
events.push(event);
|
|
870
|
+
accepted += 1;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
if (tail.truncated) {
|
|
874
|
+
warnings.push("truncated event tail for " + path.basename(path.dirname(filePath)) + "/" + path.basename(filePath) + " to newest " + accepted + " valid events");
|
|
705
875
|
}
|
|
706
876
|
}
|
|
707
877
|
async function loadLedgerRuns(repoRoot) {
|
|
708
878
|
const runs = [];
|
|
709
|
-
const
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
879
|
+
const warnings = [];
|
|
880
|
+
const tail = await readBoundedJsonlTail(path.join(getTaskPoolRoot(repoRoot), "runs.jsonl"), { maxBytes: SNAPSHOT_LEDGER_MAX_BYTES, maxLines: SNAPSHOT_LEDGER_MAX_LINES });
|
|
881
|
+
if (tail.truncated) {
|
|
882
|
+
warnings.push("runs.jsonl projection truncated to newest " +
|
|
883
|
+
SNAPSHOT_LEDGER_MAX_LINES +
|
|
884
|
+
" lines / " +
|
|
885
|
+
SNAPSHOT_LEDGER_MAX_BYTES +
|
|
886
|
+
" bytes");
|
|
887
|
+
}
|
|
888
|
+
for (const line of tail.lines) {
|
|
889
|
+
const trimmed = line.text.trim();
|
|
714
890
|
if (!trimmed)
|
|
715
891
|
continue;
|
|
716
892
|
const parsed = safeParseJson(trimmed);
|
|
@@ -724,6 +900,7 @@ async function loadLedgerRuns(repoRoot) {
|
|
|
724
900
|
continue;
|
|
725
901
|
const failure = readObject(parsed, "failure");
|
|
726
902
|
const failureArtifacts = readObject(parsed, "failureArtifacts");
|
|
903
|
+
const retryOfWorkerRunId = readString(parsed, "retryOfWorkerRunId");
|
|
727
904
|
runs.push({
|
|
728
905
|
batchRunId,
|
|
729
906
|
workerRunId,
|
|
@@ -734,8 +911,11 @@ async function loadLedgerRuns(repoRoot) {
|
|
|
734
911
|
runRecordPath: readString(parsed, "runRecordPath"),
|
|
735
912
|
dagPath: readString(parsed, "dagPath"),
|
|
736
913
|
recordedAt: readString(parsed, "recordedAt"),
|
|
914
|
+
...(retryOfWorkerRunId ? { retryOfWorkerRunId } : {}),
|
|
737
915
|
failureCategory: failure ? readString(failure, "category") : undefined,
|
|
738
|
-
recommendedFollowUp: failure
|
|
916
|
+
recommendedFollowUp: failure
|
|
917
|
+
? readString(failure, "recommendedFollowUpKind")
|
|
918
|
+
: undefined,
|
|
739
919
|
failureArtifacts: failureArtifacts
|
|
740
920
|
? {
|
|
741
921
|
reportMarkdown: readString(failureArtifacts, "reportMarkdownArtifactPath"),
|
|
@@ -745,7 +925,7 @@ async function loadLedgerRuns(repoRoot) {
|
|
|
745
925
|
: undefined,
|
|
746
926
|
});
|
|
747
927
|
}
|
|
748
|
-
return runs;
|
|
928
|
+
return { runs, warnings };
|
|
749
929
|
}
|
|
750
930
|
async function loadLedgerBatches(repoRoot) {
|
|
751
931
|
const batches = [];
|
|
@@ -761,20 +941,26 @@ async function loadLedgerBatches(repoRoot) {
|
|
|
761
941
|
if (!parsed)
|
|
762
942
|
continue;
|
|
763
943
|
const summaryObj = readObject(parsed, "summary");
|
|
944
|
+
let summary;
|
|
945
|
+
if (summaryObj) {
|
|
946
|
+
const recordErrors = readNumber(summaryObj, "recordErrors");
|
|
947
|
+
const runErrors = readNumber(summaryObj, "runErrors");
|
|
948
|
+
summary = {
|
|
949
|
+
total: readNumber(summaryObj, "total") ?? 0,
|
|
950
|
+
succeeded: readNumber(summaryObj, "succeeded") ?? 0,
|
|
951
|
+
failed: readNumber(summaryObj, "failed") ?? 0,
|
|
952
|
+
reused: readNumber(summaryObj, "reused") ?? 0,
|
|
953
|
+
...(recordErrors !== undefined ? { recordErrors } : {}),
|
|
954
|
+
...(runErrors !== undefined ? { runErrors } : {}),
|
|
955
|
+
};
|
|
956
|
+
}
|
|
764
957
|
batches.push({
|
|
765
958
|
batchRunId: readString(parsed, "batchRunId") ?? batchRunId,
|
|
766
959
|
featureId: readString(parsed, "featureId"),
|
|
767
960
|
startedAt: readString(parsed, "startedAt"),
|
|
768
961
|
finishedAt: readString(parsed, "finishedAt"),
|
|
769
962
|
status: mapBatchStatus(readString(parsed, "status") ?? "unknown"),
|
|
770
|
-
summary
|
|
771
|
-
? {
|
|
772
|
-
total: readNumber(summaryObj, "total") ?? 0,
|
|
773
|
-
succeeded: readNumber(summaryObj, "succeeded") ?? 0,
|
|
774
|
-
failed: readNumber(summaryObj, "failed") ?? 0,
|
|
775
|
-
reused: readNumber(summaryObj, "reused") ?? 0,
|
|
776
|
-
}
|
|
777
|
-
: undefined,
|
|
963
|
+
summary,
|
|
778
964
|
tasks: [],
|
|
779
965
|
batchRunPath,
|
|
780
966
|
});
|
|
@@ -806,8 +992,11 @@ async function loadStateRecords(repoRoot) {
|
|
|
806
992
|
status,
|
|
807
993
|
workerRunId: readString(parsed, "workerRunId"),
|
|
808
994
|
lastRunRecordPath: readString(parsed, "lastRunRecordPath"),
|
|
995
|
+
updatedAt: readString(parsed, "updatedAt"),
|
|
809
996
|
failureCategory: failure ? readString(failure, "category") : undefined,
|
|
810
|
-
recommendedFollowUp: failure
|
|
997
|
+
recommendedFollowUp: failure
|
|
998
|
+
? readString(failure, "recommendedFollowUpKind")
|
|
999
|
+
: undefined,
|
|
811
1000
|
});
|
|
812
1001
|
}
|
|
813
1002
|
}
|
|
@@ -858,7 +1047,9 @@ function mergeDagRun(existing, incoming) {
|
|
|
858
1047
|
return incoming;
|
|
859
1048
|
const startedAt = incoming.startedAt ?? existing.startedAt;
|
|
860
1049
|
const finishedAt = incoming.finishedAt ?? existing.finishedAt;
|
|
861
|
-
const ranks = incoming.ranks && incoming.ranks.length > 0
|
|
1050
|
+
const ranks = incoming.ranks && incoming.ranks.length > 0
|
|
1051
|
+
? incoming.ranks
|
|
1052
|
+
: existing.ranks;
|
|
862
1053
|
const durationMs = incoming.durationMs ??
|
|
863
1054
|
existing.durationMs ??
|
|
864
1055
|
computeDurationMs(startedAt, finishedAt);
|
|
@@ -877,7 +1068,9 @@ function mergeDagRun(existing, incoming) {
|
|
|
877
1068
|
...(ranks && ranks.length > 0 ? { ranks } : {}),
|
|
878
1069
|
nodes: mergeDagNodes(existing.nodes, incoming.nodes),
|
|
879
1070
|
edges: incoming.edges.length > 0 ? incoming.edges : existing.edges,
|
|
880
|
-
...(incoming.dagPath ?? existing.dagPath
|
|
1071
|
+
...((incoming.dagPath ?? existing.dagPath)
|
|
1072
|
+
? { dagPath: incoming.dagPath ?? existing.dagPath }
|
|
1073
|
+
: {}),
|
|
881
1074
|
};
|
|
882
1075
|
}
|
|
883
1076
|
function mergeDagNodes(existing, incoming) {
|
|
@@ -921,7 +1114,9 @@ function isFailedNodeStatus(status) {
|
|
|
921
1114
|
if (!status)
|
|
922
1115
|
return false;
|
|
923
1116
|
const normalized = status.toLowerCase();
|
|
924
|
-
return normalized === "error" ||
|
|
1117
|
+
return (normalized === "error" ||
|
|
1118
|
+
normalized === "failed" ||
|
|
1119
|
+
normalized === "partial_failed");
|
|
925
1120
|
}
|
|
926
1121
|
function nodeOutputPreview(node) {
|
|
927
1122
|
const stdout = readString(node, "stdout") ?? readString(node, "assistantText");
|
|
@@ -997,13 +1192,19 @@ async function loadDagEventFiles(repoRoot) {
|
|
|
997
1192
|
const outputPreview = readString(parsed, "outputPreview");
|
|
998
1193
|
nodeMap.set(nodeId, {
|
|
999
1194
|
nodeId,
|
|
1000
|
-
...(rank ?? prev.rank ? { rank: rank ?? prev.rank } : {}),
|
|
1001
|
-
...(executor ?? prev.executor
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
...(
|
|
1005
|
-
...(
|
|
1006
|
-
...(
|
|
1195
|
+
...((rank ?? prev.rank) ? { rank: rank ?? prev.rank } : {}),
|
|
1196
|
+
...((executor ?? prev.executor)
|
|
1197
|
+
? { executor: executor ?? prev.executor }
|
|
1198
|
+
: {}),
|
|
1199
|
+
...((model ?? prev.model) ? { model: model ?? prev.model } : {}),
|
|
1200
|
+
...((label ?? prev.label) ? { label: label ?? prev.label } : {}),
|
|
1201
|
+
...((status ?? prev.status) ? { status: status ?? prev.status } : {}),
|
|
1202
|
+
...((durationMs ?? prev.durationMs)
|
|
1203
|
+
? { durationMs: durationMs ?? prev.durationMs }
|
|
1204
|
+
: {}),
|
|
1205
|
+
...((outputPreview ?? prev.outputPreview)
|
|
1206
|
+
? { outputPreview: outputPreview ?? prev.outputPreview }
|
|
1207
|
+
: {}),
|
|
1007
1208
|
});
|
|
1008
1209
|
byId.set(dagRunId, nodeMap);
|
|
1009
1210
|
}
|
|
@@ -1059,7 +1260,9 @@ async function parseDagStateFile(statePath, now) {
|
|
|
1059
1260
|
return undefined;
|
|
1060
1261
|
const runDir = path.dirname(statePath);
|
|
1061
1262
|
const lifecycleName = path.basename(path.dirname(runDir));
|
|
1062
|
-
const lifecycle = lifecycleName === "active" ||
|
|
1263
|
+
const lifecycle = lifecycleName === "active" ||
|
|
1264
|
+
lifecycleName === "paused" ||
|
|
1265
|
+
lifecycleName === "completed"
|
|
1063
1266
|
? lifecycleName
|
|
1064
1267
|
: undefined;
|
|
1065
1268
|
const dagRunId = readString(parsed, "runId") ?? path.basename(runDir);
|
|
@@ -1077,14 +1280,29 @@ async function parseDagStateFile(statePath, now) {
|
|
|
1077
1280
|
const state = parsed;
|
|
1078
1281
|
const liveness = assessDagRunLiveness({ state, now });
|
|
1079
1282
|
const effectiveStatus = lifecycle
|
|
1080
|
-
? deriveDagRunEffectiveStatus({
|
|
1283
|
+
? deriveDagRunEffectiveStatus({
|
|
1284
|
+
lifecycle,
|
|
1285
|
+
state,
|
|
1286
|
+
liveness: liveness.status,
|
|
1287
|
+
})
|
|
1081
1288
|
: "unknown";
|
|
1082
1289
|
const stateConsistent = lifecycle === undefined
|
|
1083
1290
|
? false
|
|
1084
|
-
: !((lifecycle === "paused" && state.status !== "paused")
|
|
1085
|
-
|
|
1291
|
+
: !((lifecycle === "paused" && state.status !== "paused") ||
|
|
1292
|
+
(lifecycle === "completed" &&
|
|
1293
|
+
![
|
|
1294
|
+
"finished",
|
|
1295
|
+
"failed",
|
|
1296
|
+
"partial_failed",
|
|
1297
|
+
"superseded",
|
|
1298
|
+
"abandoned",
|
|
1299
|
+
].includes(state.status)));
|
|
1086
1300
|
const recoveryEligibility = lifecycle
|
|
1087
|
-
? assessDagRunRecoveryEligibility({
|
|
1301
|
+
? assessDagRunRecoveryEligibility({
|
|
1302
|
+
lifecycle,
|
|
1303
|
+
state,
|
|
1304
|
+
liveness: liveness.status,
|
|
1305
|
+
})
|
|
1088
1306
|
: undefined;
|
|
1089
1307
|
return {
|
|
1090
1308
|
dagRunId,
|
|
@@ -1122,7 +1340,9 @@ async function parseDagEdges(runPath, nodes) {
|
|
|
1122
1340
|
if (!taskId || !knownNodeIds.has(taskId))
|
|
1123
1341
|
continue;
|
|
1124
1342
|
const dependencies = readStringArray(task, "depends_on");
|
|
1125
|
-
const compatibleDependencies = dependencies.length > 0
|
|
1343
|
+
const compatibleDependencies = dependencies.length > 0
|
|
1344
|
+
? dependencies
|
|
1345
|
+
: readStringArray(task, "dependsOn");
|
|
1126
1346
|
for (const dependencyId of compatibleDependencies) {
|
|
1127
1347
|
if (dependencyId === taskId || !knownNodeIds.has(dependencyId))
|
|
1128
1348
|
continue;
|
|
@@ -1169,7 +1389,10 @@ async function loadDagNodeModels(runPath) {
|
|
|
1169
1389
|
const legacy = raw;
|
|
1170
1390
|
const tasks = legacy.tasks;
|
|
1171
1391
|
const executorModels = legacy.executorModels;
|
|
1172
|
-
if (!Array.isArray(tasks) ||
|
|
1392
|
+
if (!Array.isArray(tasks) ||
|
|
1393
|
+
typeof executorModels !== "object" ||
|
|
1394
|
+
executorModels === null ||
|
|
1395
|
+
Array.isArray(executorModels)) {
|
|
1173
1396
|
return models;
|
|
1174
1397
|
}
|
|
1175
1398
|
for (const task of tasks) {
|
|
@@ -1194,7 +1417,9 @@ async function loadDagNodeModels(runPath) {
|
|
|
1194
1417
|
function parseDagNodes(parsed, rankByNode, modelByNode) {
|
|
1195
1418
|
const nodes = [];
|
|
1196
1419
|
const nodesValue = parsed.nodes;
|
|
1197
|
-
if (nodesValue &&
|
|
1420
|
+
if (nodesValue &&
|
|
1421
|
+
typeof nodesValue === "object" &&
|
|
1422
|
+
!Array.isArray(nodesValue)) {
|
|
1198
1423
|
for (const [nodeId, nodeVal] of Object.entries(nodesValue)) {
|
|
1199
1424
|
if (!nodeVal || typeof nodeVal !== "object" || Array.isArray(nodeVal))
|
|
1200
1425
|
continue;
|
|
@@ -1261,7 +1486,9 @@ async function safeReadJson(filePath) {
|
|
|
1261
1486
|
function safeParseJson(raw) {
|
|
1262
1487
|
try {
|
|
1263
1488
|
const parsed = JSON.parse(raw);
|
|
1264
|
-
if (parsed === null ||
|
|
1489
|
+
if (parsed === null ||
|
|
1490
|
+
typeof parsed !== "object" ||
|
|
1491
|
+
Array.isArray(parsed)) {
|
|
1265
1492
|
return undefined;
|
|
1266
1493
|
}
|
|
1267
1494
|
return parsed;
|
|
@@ -1304,7 +1531,9 @@ function readNumber(value, key) {
|
|
|
1304
1531
|
if (!value || typeof value !== "object")
|
|
1305
1532
|
return undefined;
|
|
1306
1533
|
const child = value[key];
|
|
1307
|
-
return typeof child === "number" && Number.isFinite(child)
|
|
1534
|
+
return typeof child === "number" && Number.isFinite(child)
|
|
1535
|
+
? child
|
|
1536
|
+
: undefined;
|
|
1308
1537
|
}
|
|
1309
1538
|
function readStringMatrix(value, key) {
|
|
1310
1539
|
if (!value || typeof value !== "object")
|