@tea-agent/loop-agent 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/README.md +3 -2
  3. package/dist/application/dag/generate-task-dag.js +15 -0
  4. package/dist/application/dag/run-dag.js +10 -0
  5. package/dist/application/dag/validate-dag.js +11 -0
  6. package/dist/commands/init.js +74 -7
  7. package/dist/shared/package-metadata.js +135 -0
  8. package/dist/task/config-types.js +1 -0
  9. package/dist/worker/cli.js +3 -1
  10. package/dist/worker/observability/event-history.js +216 -0
  11. package/dist/worker/observability/read-model.js +312 -83
  12. package/dist/worker/observe/paths.js +17 -0
  13. package/dist/worker/observe/routes.js +165 -21
  14. package/dist/worker/observe/server.js +59 -1
  15. package/dist/worker/observe/static/api.js +27 -0
  16. package/dist/worker/observe/static/app.js +120 -2598
  17. package/dist/worker/observe/static/constants.js +148 -0
  18. package/dist/worker/observe/static/copy.js +67 -0
  19. package/dist/worker/observe/static/dag-helpers.js +172 -0
  20. package/dist/worker/observe/static/dag-model.js +72 -0
  21. package/dist/worker/observe/static/dom.js +61 -0
  22. package/dist/worker/observe/static/format-pool.js +67 -0
  23. package/dist/worker/observe/static/format.js +292 -0
  24. package/dist/worker/observe/static/index.html +300 -82
  25. package/dist/worker/observe/static/kpi.js +94 -0
  26. package/dist/worker/observe/static/relations.js +128 -0
  27. package/dist/worker/observe/static/router.js +85 -0
  28. package/dist/worker/observe/static/run-processing.js +148 -0
  29. package/dist/worker/observe/static/shell-chrome.js +68 -0
  30. package/dist/worker/observe/static/state.js +253 -0
  31. package/dist/worker/observe/static/styles.css +1719 -495
  32. package/dist/worker/observe/static/views/batch.js +226 -0
  33. package/dist/worker/observe/static/views/dag-graph.js +172 -0
  34. package/dist/worker/observe/static/views/dag-inspector.js +477 -0
  35. package/dist/worker/observe/static/views/dag.js +362 -0
  36. package/dist/worker/observe/static/views/dashboard.js +442 -0
  37. package/dist/worker/observe/static/views/failures.js +143 -0
  38. package/dist/worker/observe/static/views/feature.js +453 -0
  39. package/dist/worker/observe/static/views/pool.js +347 -0
  40. package/dist/worker/observe/static/views/run.js +453 -0
  41. package/dist/worker/observe/static/views/session-timeline.js +205 -0
  42. package/dist/worker/observe/static/views/shell.js +7 -0
  43. package/dist/worker/observe/static/views/task.js +260 -0
  44. package/dist/worker/observe/static/views/timeline.js +163 -0
  45. package/dist/workflows/dag/controller-identity.js +104 -0
  46. package/dist/workflows/dag/init-hybrid.js +396 -3
  47. package/dist/workflows/dag/node-execution.js +123 -29
  48. package/dist/workflows/dag/repair-artifact.js +91 -0
  49. package/dist/workflows/dag/report.js +50 -0
  50. package/dist/workflows/dag/retry-policy.js +138 -0
  51. package/dist/workflows/dag/runner.js +32 -0
  52. package/dist/workflows/dag/runtime-contract.js +87 -0
  53. package/dist/workflows/dag/skill-snapshot.js +2 -0
  54. package/dist/workflows/dag/types.js +44 -1
  55. package/dist/workflows/dag/validate.js +68 -4
  56. package/docs/agent-dag-runner.md +26 -1
  57. package/docs/architecture/dag-execution.md +6 -0
  58. package/docs/architecture/evolution.md +4 -3
  59. package/docs/architecture/facts-and-state.md +1 -1
  60. package/docs/design/README.md +4 -3
  61. package/docs/exec-plans/active/README.md +1 -3
  62. package/docs/exec-plans/completed/README.md +11 -0
  63. package/docs/feature-workflow.md +28 -0
  64. package/docs/progress/README.md +18 -0
  65. package/docs/reports/README.md +8 -2
  66. package/docs/templates/agent-dag-report.schema.json +17 -0
  67. package/docs/templates/agent-dag.schema.json +69 -1
  68. package/docs/templates/agent-dag.supervised-implementation.json +8 -2
  69. package/docs/templates/backend-test-dag.generate-pytest.prompt.md +139 -0
  70. package/docs/templates/backend-test-dag.json +276 -0
  71. package/docs/templates/backend-test-dag.retrospect.prompt.md +125 -0
  72. package/docs/templates/backend-test-dag.review-cases.prompt.md +81 -0
  73. package/package.json +1 -1
  74. package/skills/loop-agent/references/command-reference.md +1 -0
  75. package/skills/loop-agent/references/hybrid-dag.md +22 -3
  76. package/skills/loop-agent/references/verification-and-failure-handling.md +6 -0
@@ -3,10 +3,13 @@ import { readdir, readFile } from "node:fs/promises";
3
3
  import path from "node:path";
4
4
  import { truncateUtf8Preview } from "../../shared/preview.js";
5
5
  import { parseWorkerEventLine } from "./events.js";
6
+ import { readBoundedJsonlTail } from "./event-history.js";
6
7
  import { getTaskPoolRoot } from "../pool/run-store.js";
7
8
  import { assessDagRunLiveness, assessDagRunRecoveryEligibility, deriveDagRunEffectiveStatus, } from "../../workflows/dag/lifecycle.js";
8
9
  import { parseDagSpec, resolveModelForTask, } from "../../workflows/dag/types.js";
9
10
  import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
11
+ export const TASK_RUN_HISTORY_DEFAULT_LIMIT = 20;
12
+ export const TASK_RUN_HISTORY_MAX_LIMIT = 100;
10
13
  const ACTIVE_TASK_STATUSES = new Set(["running", "pending"]);
11
14
  const ACTIVE_BATCH_STATUSES = new Set(["running", "pending"]);
12
15
  export async function buildGlobalSnapshot(options) {
@@ -15,7 +18,8 @@ export async function buildGlobalSnapshot(options) {
15
18
  const now = options.now ?? (() => new Date());
16
19
  const staleThresholdMs = options.staleThresholdMs ?? 90_000;
17
20
  const quietThresholdMs = options.quietThresholdMs ?? 60_000;
18
- const [events, ledgerRuns, ledgerBatches, states, handoffs, dagRuns] = await Promise.all([
21
+ const taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
22
+ const [eventsLoad, ledgerLoad, ledgerBatches, states, handoffs, dagRuns] = await Promise.all([
19
23
  loadObservabilityEvents(repoRoot),
20
24
  loadLedgerRuns(repoRoot),
21
25
  loadLedgerBatches(repoRoot),
@@ -23,6 +27,8 @@ export async function buildGlobalSnapshot(options) {
23
27
  loadFailureHandoffs(repoRoot),
24
28
  loadDagRuns(repoRoot, now()),
25
29
  ]);
30
+ const events = eventsLoad.events;
31
+ const ledgerRuns = ledgerLoad.runs;
26
32
  const taskMap = new Map();
27
33
  for (const state of states) {
28
34
  mergeStateRecord(taskMap, state);
@@ -45,7 +51,12 @@ export async function buildGlobalSnapshot(options) {
45
51
  tasks: batch.tasks.map((task) => normalizeTaskArtifactRefs(task, repoRoot)),
46
52
  }));
47
53
  const health = computeHealth(batches, tasks, dagRuns);
48
- const { features, projectionWarnings } = await loadFeatureDecisionModels(repoRoot);
54
+ const featureLoad = await loadFeatureDecisionModels(repoRoot);
55
+ const projectionWarnings = [
56
+ ...(eventsLoad.warnings ?? []),
57
+ ...(ledgerLoad.warnings ?? []),
58
+ ...(featureLoad.projectionWarnings ?? []),
59
+ ];
49
60
  return {
50
61
  schemaVersion: 1,
51
62
  generatedAt: now().toISOString(),
@@ -53,7 +64,8 @@ export async function buildGlobalSnapshot(options) {
53
64
  batches,
54
65
  tasks,
55
66
  dagRuns,
56
- features,
67
+ features: featureLoad.features,
68
+ taskPool: { present: taskPoolPresent },
57
69
  projectionWarnings,
58
70
  health,
59
71
  };
@@ -80,7 +92,99 @@ export async function buildGlobalSnapshot(options) {
80
92
  return snapshot;
81
93
  }
82
94
  }
95
+ /**
96
+ * Bounded Task run history from Task Pool ledger facts (newest first).
97
+ * `before` is an exclusive upper bound on the sort key (`recordedAt` or empty).
98
+ */
99
+ export async function listTaskRunHistory(repoRoot, taskId, options = {}) {
100
+ const limit = clampTaskRunHistoryLimit(options.limit);
101
+ const before = decodeTaskRunHistoryCursor(options.before);
102
+ const [ledgerLoad, eventsLoad] = await Promise.all([
103
+ loadLedgerRuns(path.resolve(repoRoot)),
104
+ loadObservabilityEvents(path.resolve(repoRoot)),
105
+ ]);
106
+ const ledgerRuns = ledgerLoad.runs;
107
+ const events = eventsLoad.events;
108
+ const historical = buildHistoricalRunMap(ledgerRuns, events);
109
+ const items = [];
110
+ for (const run of ledgerRuns) {
111
+ if (run.taskId !== taskId)
112
+ continue;
113
+ const enriched = historical.get(run.workerRunId);
114
+ items.push({
115
+ workerRunId: run.workerRunId,
116
+ taskId: run.taskId,
117
+ status: enriched?.status ?? mapRunRecordStatus(run.status),
118
+ startedAt: enriched?.startedAt,
119
+ finishedAt: enriched?.finishedAt ?? run.recordedAt,
120
+ recordedAt: run.recordedAt,
121
+ batchRunId: run.batchRunId,
122
+ dagRunId: enriched?.dagRunId,
123
+ ...(run.retryOfWorkerRunId
124
+ ? { retryOfWorkerRunId: run.retryOfWorkerRunId }
125
+ : {}),
126
+ failureCategory: run.failureCategory ?? enriched?.failureCategory,
127
+ recommendedFollowUp: run.recommendedFollowUp ?? enriched?.recommendedFollowUp,
128
+ });
129
+ }
130
+ items.sort((a, b) => {
131
+ const ta = a.recordedAt ?? a.finishedAt ?? "";
132
+ const tb = b.recordedAt ?? b.finishedAt ?? "";
133
+ if (ta !== tb)
134
+ return tb.localeCompare(ta);
135
+ return b.workerRunId.localeCompare(a.workerRunId);
136
+ });
137
+ const eligible = before
138
+ ? items.filter((item) => taskRunSortKey(item).localeCompare(before) < 0)
139
+ : items;
140
+ const page = eligible.slice(0, limit);
141
+ const nextBefore = eligible.length > limit
142
+ ? encodeTaskRunHistoryCursor(page[page.length - 1])
143
+ : null;
144
+ return {
145
+ taskId,
146
+ runs: page,
147
+ limit,
148
+ nextBefore,
149
+ };
150
+ }
151
+ function taskRunSortKey(item) {
152
+ return `${item.recordedAt ?? item.finishedAt ?? ""}\u0000${item.workerRunId}`;
153
+ }
154
+ function encodeTaskRunHistoryCursor(item) {
155
+ if (!item)
156
+ return null;
157
+ return `v1.${Buffer.from(taskRunSortKey(item), "utf-8").toString("base64url")}`;
158
+ }
159
+ function decodeTaskRunHistoryCursor(cursor) {
160
+ const value = cursor?.trim();
161
+ if (!value)
162
+ return null;
163
+ if (!value.startsWith("v1.")) {
164
+ // Compatibility with the initial timestamp-only cursor shape.
165
+ return `${value}\u0000`;
166
+ }
167
+ try {
168
+ return Buffer.from(value.slice(3), "base64url").toString("utf-8");
169
+ }
170
+ catch {
171
+ return `${value}\u0000`;
172
+ }
173
+ }
174
+ export function clampTaskRunHistoryLimit(limit) {
175
+ if (limit === undefined || !Number.isFinite(limit) || limit <= 0) {
176
+ return TASK_RUN_HISTORY_DEFAULT_LIMIT;
177
+ }
178
+ return Math.min(Math.floor(limit), TASK_RUN_HISTORY_MAX_LIMIT);
179
+ }
83
180
  function emptySnapshot(repoRoot, now) {
181
+ let taskPoolPresent = false;
182
+ try {
183
+ taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
184
+ }
185
+ catch {
186
+ taskPoolPresent = false;
187
+ }
84
188
  return {
85
189
  schemaVersion: 1,
86
190
  generatedAt: now().toISOString(),
@@ -88,6 +192,7 @@ function emptySnapshot(repoRoot, now) {
88
192
  batches: [],
89
193
  tasks: [],
90
194
  dagRuns: [],
195
+ taskPool: { present: taskPoolPresent },
91
196
  health: {
92
197
  activeBatches: 0,
93
198
  activeTasks: 0,
@@ -118,15 +223,21 @@ function emptySnapshot(repoRoot, now) {
118
223
  };
119
224
  }
120
225
  function toPublicTaskSummary(task) {
121
- const { batchRunId: _batch, timeoutMs: _to, commandStartedAt: _cs, ...rest } = task;
226
+ const { timeoutMs: _to, commandStartedAt: _cs, ...rest } = task;
122
227
  return rest;
123
228
  }
124
229
  function mergeStateRecord(taskMap, state) {
125
- const existing = taskMap.get(state.taskId) ?? { taskId: state.taskId, status: "unknown" };
230
+ const existing = taskMap.get(state.taskId) ?? {
231
+ taskId: state.taskId,
232
+ status: "unknown",
233
+ };
126
234
  taskMap.set(state.taskId, {
127
235
  ...existing,
128
- status: existing.status !== "unknown" ? existing.status : mapStateStatus(state.status),
236
+ status: existing.status !== "unknown"
237
+ ? existing.status
238
+ : mapStateStatus(state.status),
129
239
  workerRunId: existing.workerRunId ?? state.workerRunId,
240
+ updatedAt: state.updatedAt ?? existing.updatedAt,
130
241
  artifactRefs: mergeArtifactRefs(existing.artifactRefs, {
131
242
  runRecordPath: state.lastRunRecordPath,
132
243
  }),
@@ -135,7 +246,10 @@ function mergeStateRecord(taskMap, state) {
135
246
  });
136
247
  }
137
248
  function mergeLedgerRun(taskMap, run) {
138
- const existing = taskMap.get(run.taskId) ?? { taskId: run.taskId, status: "unknown" };
249
+ const existing = taskMap.get(run.taskId) ?? {
250
+ taskId: run.taskId,
251
+ status: "unknown",
252
+ };
139
253
  const sameRun = existing.workerRunId === run.workerRunId;
140
254
  const runArtifactRefs = {
141
255
  runRecordPath: run.runRecordPath,
@@ -158,7 +272,8 @@ function mergeLedgerRun(taskMap, run) {
158
272
  harnessTaskId: run.harnessTaskId ?? existing.harnessTaskId,
159
273
  finishedAt: run.recordedAt ?? (sameRun ? existing.finishedAt : undefined),
160
274
  failureCategory: run.failureCategory ?? (sameRun ? existing.failureCategory : undefined),
161
- recommendedFollowUp: run.recommendedFollowUp ?? (sameRun ? existing.recommendedFollowUp : undefined),
275
+ recommendedFollowUp: run.recommendedFollowUp ??
276
+ (sameRun ? existing.recommendedFollowUp : undefined),
162
277
  artifactRefs: mergeArtifactRefs(sameRun ? existing.artifactRefs : undefined, runArtifactRefs),
163
278
  });
164
279
  }
@@ -172,7 +287,10 @@ function mergeEvents(taskMap, events) {
172
287
  }
173
288
  function applyTaskEvent(taskMap, event) {
174
289
  const taskId = event.taskId;
175
- const existing = taskMap.get(taskId) ?? { taskId, status: "unknown" };
290
+ const existing = taskMap.get(taskId) ?? {
291
+ taskId,
292
+ status: "unknown",
293
+ };
176
294
  taskMap.set(taskId, applyEventToSummary(existing, event));
177
295
  }
178
296
  function applyEventToSummary(existing, event) {
@@ -320,7 +438,9 @@ function computeLiveness(task, nowMs, staleThresholdMs, quietThresholdMs) {
320
438
  task.timeoutAt = new Date(commandStartedMs + timeoutMs).toISOString();
321
439
  }
322
440
  }
323
- const heartbeatMs = task.lastHeartbeatAt ? Date.parse(task.lastHeartbeatAt) : NaN;
441
+ const heartbeatMs = task.lastHeartbeatAt
442
+ ? Date.parse(task.lastHeartbeatAt)
443
+ : NaN;
324
444
  const outputMs = task.lastOutputAt ? Date.parse(task.lastOutputAt) : NaN;
325
445
  // Priority 1: timeout-risk (elapsed > 0.8 * timeoutMs), only if timeoutMs known.
326
446
  if (timeoutMs !== undefined &&
@@ -333,7 +453,9 @@ function computeLiveness(task, nowMs, staleThresholdMs, quietThresholdMs) {
333
453
  // (a) Heartbeat present but older than staleThresholdMs.
334
454
  // (b) No heartbeat but running longer than staleThresholdMs (startedAt known).
335
455
  const heartbeatStale = !Number.isNaN(heartbeatMs) && nowMs - heartbeatMs > staleThresholdMs;
336
- const noHeartbeatStale = Number.isNaN(heartbeatMs) && elapsedMs !== undefined && elapsedMs > staleThresholdMs;
456
+ const noHeartbeatStale = Number.isNaN(heartbeatMs) &&
457
+ elapsedMs !== undefined &&
458
+ elapsedMs > staleThresholdMs;
337
459
  if (heartbeatStale || noHeartbeatStale) {
338
460
  task.liveness = "stale";
339
461
  task.status = "stale";
@@ -414,7 +536,8 @@ function summarizeTasks(tasks) {
414
536
  function computeHealth(batches, tasks, dagRuns) {
415
537
  const worker = {
416
538
  activeTasks: tasks.filter((t) => ACTIVE_TASK_STATUSES.has(t.status)).length,
417
- activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status)).length,
539
+ activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status))
540
+ .length,
418
541
  quietCount: tasks.filter((t) => t.liveness === "quiet").length,
419
542
  staleCount: tasks.filter((t) => t.status === "stale").length,
420
543
  timeoutRiskCount: tasks.filter((t) => t.liveness === "timeout-risk").length,
@@ -451,7 +574,8 @@ const TERMINAL_RUN_STATUSES = new Set([
451
574
  * already-loaded DagRunSummary facts without touching app.js or new files.
452
575
  */
453
576
  function isDagRunActiveForHealth(dag) {
454
- if (dag.effectiveStatus && ACTIVE_EFFECTIVE_STATUSES.has(dag.effectiveStatus)) {
577
+ if (dag.effectiveStatus &&
578
+ ACTIVE_EFFECTIVE_STATUSES.has(dag.effectiveStatus)) {
455
579
  return true;
456
580
  }
457
581
  // Only "unknown" (and absent) means liveness evidence is insufficient;
@@ -472,7 +596,16 @@ function isDagRunActiveForHealth(dag) {
472
596
  }
473
597
  function isNodeStatusActive(status) {
474
598
  const normalized = (status ?? "").toLowerCase();
475
- if (["finished", "completed", "succeeded", "done", "error", "failed", "skipped", "partial_failed"].includes(normalized)) {
599
+ if ([
600
+ "finished",
601
+ "completed",
602
+ "succeeded",
603
+ "done",
604
+ "error",
605
+ "failed",
606
+ "skipped",
607
+ "partial_failed",
608
+ ].includes(normalized)) {
476
609
  return false;
477
610
  }
478
611
  return true;
@@ -511,13 +644,13 @@ function computeDagHealth(dagRuns) {
511
644
  interruptedRuns++;
512
645
  if (dag.stateConsistent === false)
513
646
  inconsistentRuns++;
514
- if (lifecycle !== "completed"
515
- && (lifecycle === "paused"
516
- || liveness === "stale"
517
- || liveness === "orphaned"
518
- || dag.effectiveStatus === "interrupted"
519
- || dag.effectiveStatus === "remote-unknown"
520
- || dag.stateConsistent === false)) {
647
+ if (lifecycle !== "completed" &&
648
+ (lifecycle === "paused" ||
649
+ liveness === "stale" ||
650
+ liveness === "orphaned" ||
651
+ dag.effectiveStatus === "interrupted" ||
652
+ dag.effectiveStatus === "remote-unknown" ||
653
+ dag.stateConsistent === false)) {
521
654
  attentionRuns++;
522
655
  }
523
656
  }
@@ -547,12 +680,14 @@ function sanitizeProjectionError(error) {
547
680
  let name;
548
681
  let message;
549
682
  try {
550
- name = typeof error.name === "string"
551
- ? error.name
552
- : undefined;
553
- message = typeof error.message === "string"
554
- ? error.message
555
- : undefined;
683
+ name =
684
+ typeof error.name === "string"
685
+ ? error.name
686
+ : undefined;
687
+ message =
688
+ typeof error.message === "string"
689
+ ? error.message
690
+ : undefined;
556
691
  }
557
692
  catch {
558
693
  return { message: fallback, at };
@@ -563,12 +698,12 @@ function sanitizeProjectionError(error) {
563
698
  }
564
699
  // Reject anything that looks like a stack frame, an absolute path, or a secret.
565
700
  const raw = String(candidate);
566
- if (raw.includes("\n")
567
- || /\bat \S+/i.test(raw)
568
- || /[A-Za-z]:\\/.test(raw)
569
- || raw.includes("/Users/")
570
- || raw.includes("/home/")
571
- || /SECRET|TOKEN|PASSWORD|API_KEY/i.test(raw)) {
701
+ if (raw.includes("\n") ||
702
+ /\bat \S+/i.test(raw) ||
703
+ /[A-Za-z]:\\/.test(raw) ||
704
+ raw.includes("/Users/") ||
705
+ raw.includes("/home/") ||
706
+ /SECRET|TOKEN|PASSWORD|API_KEY/i.test(raw)) {
572
707
  return { message: name ? `${name}: 投影失败` : fallback, at };
573
708
  }
574
709
  const trimmed = raw.trim().slice(0, 200);
@@ -581,7 +716,11 @@ function mergeArtifactRefs(existing, incoming) {
581
716
  for (const [key, value] of Object.entries(incoming)) {
582
717
  if (!value)
583
718
  continue;
584
- if (key === "runRecordPath" || key === "dagPath" || key === "reportMarkdown" || key === "doctorMarkdown" || key === "closeoutDraft") {
719
+ if (key === "runRecordPath" ||
720
+ key === "dagPath" ||
721
+ key === "reportMarkdown" ||
722
+ key === "doctorMarkdown" ||
723
+ key === "closeoutDraft") {
585
724
  merged[key] = value;
586
725
  }
587
726
  }
@@ -668,49 +807,86 @@ function mapBatchStatus(status) {
668
807
  return "unknown";
669
808
  }
670
809
  }
810
+ /** Per-file hard cap for projection merge (snapshot must not embed unbounded ledger). */
811
+ const SNAPSHOT_EVENTS_MAX_LINES_PER_FILE = 2_000;
812
+ const SNAPSHOT_EVENTS_MAX_BYTES_PER_FILE = 4 * 1024 * 1024;
813
+ /** Global hard cap after dedupe for snapshot projection merge. */
814
+ const SNAPSHOT_EVENTS_MAX_TOTAL = 8_000;
815
+ const SNAPSHOT_LEDGER_MAX_LINES = 10_000;
816
+ const SNAPSHOT_LEDGER_MAX_BYTES = 8 * 1024 * 1024;
671
817
  async function loadObservabilityEvents(repoRoot) {
672
818
  const events = [];
819
+ const warnings = [];
673
820
  const obsRoot = path.join(getTaskPoolRoot(repoRoot), "observability");
674
- await appendJsonlEvents(path.join(obsRoot, "events.jsonl"), events);
821
+ await appendJsonlEventsBounded(path.join(obsRoot, "events.jsonl"), events, warnings);
675
822
  const batchesDir = path.join(obsRoot, "batches");
676
- await forEachSubdirJsonl(batchesDir, "events.jsonl", events);
823
+ await forEachSubdirJsonl(batchesDir, "events.jsonl", events, warnings);
677
824
  const runsDir = path.join(obsRoot, "runs");
678
- await forEachSubdirJsonl(runsDir, "events.jsonl", events);
679
- return [...new Map(events.map((event) => [event.id, event])).values()];
825
+ await forEachSubdirJsonl(runsDir, "events.jsonl", events, warnings);
826
+ const deduped = [
827
+ ...new Map(events.map((event) => [event.id, event])).values(),
828
+ ].sort(compareWorkerEvents);
829
+ if (deduped.length > SNAPSHOT_EVENTS_MAX_TOTAL) {
830
+ warnings.push(`observability event projection truncated to ${SNAPSHOT_EVENTS_MAX_TOTAL} events (capacity bound)`);
831
+ return {
832
+ events: deduped.slice(-SNAPSHOT_EVENTS_MAX_TOTAL),
833
+ warnings,
834
+ };
835
+ }
836
+ return { events: deduped, warnings };
680
837
  }
681
- async function forEachSubdirJsonl(parentDir, filename, events) {
838
+ function compareWorkerEvents(a, b) {
839
+ const byTime = a.at.localeCompare(b.at);
840
+ if (byTime !== 0)
841
+ return byTime;
842
+ return a.id.localeCompare(b.id);
843
+ }
844
+ async function forEachSubdirJsonl(parentDir, filename, events, warnings) {
682
845
  try {
683
846
  const entries = await readdir(parentDir, { withFileTypes: true });
684
847
  for (const entry of entries) {
685
848
  if (!entry.isDirectory())
686
849
  continue;
687
- await appendJsonlEvents(path.join(parentDir, entry.name, filename), events);
850
+ await appendJsonlEventsBounded(path.join(parentDir, entry.name, filename), events, warnings);
688
851
  }
689
852
  }
690
853
  catch {
691
854
  // skip missing dir
692
855
  }
693
856
  }
694
- async function appendJsonlEvents(filePath, events) {
695
- const raw = await safeReadText(filePath);
696
- if (!raw)
697
- return;
698
- for (const line of raw.split("\n")) {
699
- const trimmed = line.trim();
857
+ async function appendJsonlEventsBounded(filePath, events, warnings) {
858
+ const tail = await readBoundedJsonlTail(filePath, {
859
+ maxBytes: SNAPSHOT_EVENTS_MAX_BYTES_PER_FILE,
860
+ maxLines: SNAPSHOT_EVENTS_MAX_LINES_PER_FILE,
861
+ });
862
+ let accepted = 0;
863
+ for (const line of tail.lines) {
864
+ const trimmed = line.text.trim();
700
865
  if (!trimmed)
701
866
  continue;
702
867
  const event = parseWorkerEventLine(trimmed);
703
- if (event)
868
+ if (event) {
704
869
  events.push(event);
870
+ accepted += 1;
871
+ }
872
+ }
873
+ if (tail.truncated) {
874
+ warnings.push("truncated event tail for " + path.basename(path.dirname(filePath)) + "/" + path.basename(filePath) + " to newest " + accepted + " valid events");
705
875
  }
706
876
  }
707
877
  async function loadLedgerRuns(repoRoot) {
708
878
  const runs = [];
709
- const raw = await safeReadText(path.join(getTaskPoolRoot(repoRoot), "runs.jsonl"));
710
- if (!raw)
711
- return runs;
712
- for (const line of raw.split("\n")) {
713
- const trimmed = line.trim();
879
+ const warnings = [];
880
+ const tail = await readBoundedJsonlTail(path.join(getTaskPoolRoot(repoRoot), "runs.jsonl"), { maxBytes: SNAPSHOT_LEDGER_MAX_BYTES, maxLines: SNAPSHOT_LEDGER_MAX_LINES });
881
+ if (tail.truncated) {
882
+ warnings.push("runs.jsonl projection truncated to newest " +
883
+ SNAPSHOT_LEDGER_MAX_LINES +
884
+ " lines / " +
885
+ SNAPSHOT_LEDGER_MAX_BYTES +
886
+ " bytes");
887
+ }
888
+ for (const line of tail.lines) {
889
+ const trimmed = line.text.trim();
714
890
  if (!trimmed)
715
891
  continue;
716
892
  const parsed = safeParseJson(trimmed);
@@ -724,6 +900,7 @@ async function loadLedgerRuns(repoRoot) {
724
900
  continue;
725
901
  const failure = readObject(parsed, "failure");
726
902
  const failureArtifacts = readObject(parsed, "failureArtifacts");
903
+ const retryOfWorkerRunId = readString(parsed, "retryOfWorkerRunId");
727
904
  runs.push({
728
905
  batchRunId,
729
906
  workerRunId,
@@ -734,8 +911,11 @@ async function loadLedgerRuns(repoRoot) {
734
911
  runRecordPath: readString(parsed, "runRecordPath"),
735
912
  dagPath: readString(parsed, "dagPath"),
736
913
  recordedAt: readString(parsed, "recordedAt"),
914
+ ...(retryOfWorkerRunId ? { retryOfWorkerRunId } : {}),
737
915
  failureCategory: failure ? readString(failure, "category") : undefined,
738
- recommendedFollowUp: failure ? readString(failure, "recommendedFollowUpKind") : undefined,
916
+ recommendedFollowUp: failure
917
+ ? readString(failure, "recommendedFollowUpKind")
918
+ : undefined,
739
919
  failureArtifacts: failureArtifacts
740
920
  ? {
741
921
  reportMarkdown: readString(failureArtifacts, "reportMarkdownArtifactPath"),
@@ -745,7 +925,7 @@ async function loadLedgerRuns(repoRoot) {
745
925
  : undefined,
746
926
  });
747
927
  }
748
- return runs;
928
+ return { runs, warnings };
749
929
  }
750
930
  async function loadLedgerBatches(repoRoot) {
751
931
  const batches = [];
@@ -761,20 +941,26 @@ async function loadLedgerBatches(repoRoot) {
761
941
  if (!parsed)
762
942
  continue;
763
943
  const summaryObj = readObject(parsed, "summary");
944
+ let summary;
945
+ if (summaryObj) {
946
+ const recordErrors = readNumber(summaryObj, "recordErrors");
947
+ const runErrors = readNumber(summaryObj, "runErrors");
948
+ summary = {
949
+ total: readNumber(summaryObj, "total") ?? 0,
950
+ succeeded: readNumber(summaryObj, "succeeded") ?? 0,
951
+ failed: readNumber(summaryObj, "failed") ?? 0,
952
+ reused: readNumber(summaryObj, "reused") ?? 0,
953
+ ...(recordErrors !== undefined ? { recordErrors } : {}),
954
+ ...(runErrors !== undefined ? { runErrors } : {}),
955
+ };
956
+ }
764
957
  batches.push({
765
958
  batchRunId: readString(parsed, "batchRunId") ?? batchRunId,
766
959
  featureId: readString(parsed, "featureId"),
767
960
  startedAt: readString(parsed, "startedAt"),
768
961
  finishedAt: readString(parsed, "finishedAt"),
769
962
  status: mapBatchStatus(readString(parsed, "status") ?? "unknown"),
770
- summary: summaryObj
771
- ? {
772
- total: readNumber(summaryObj, "total") ?? 0,
773
- succeeded: readNumber(summaryObj, "succeeded") ?? 0,
774
- failed: readNumber(summaryObj, "failed") ?? 0,
775
- reused: readNumber(summaryObj, "reused") ?? 0,
776
- }
777
- : undefined,
963
+ summary,
778
964
  tasks: [],
779
965
  batchRunPath,
780
966
  });
@@ -806,8 +992,11 @@ async function loadStateRecords(repoRoot) {
806
992
  status,
807
993
  workerRunId: readString(parsed, "workerRunId"),
808
994
  lastRunRecordPath: readString(parsed, "lastRunRecordPath"),
995
+ updatedAt: readString(parsed, "updatedAt"),
809
996
  failureCategory: failure ? readString(failure, "category") : undefined,
810
- recommendedFollowUp: failure ? readString(failure, "recommendedFollowUpKind") : undefined,
997
+ recommendedFollowUp: failure
998
+ ? readString(failure, "recommendedFollowUpKind")
999
+ : undefined,
811
1000
  });
812
1001
  }
813
1002
  }
@@ -858,7 +1047,9 @@ function mergeDagRun(existing, incoming) {
858
1047
  return incoming;
859
1048
  const startedAt = incoming.startedAt ?? existing.startedAt;
860
1049
  const finishedAt = incoming.finishedAt ?? existing.finishedAt;
861
- const ranks = incoming.ranks && incoming.ranks.length > 0 ? incoming.ranks : existing.ranks;
1050
+ const ranks = incoming.ranks && incoming.ranks.length > 0
1051
+ ? incoming.ranks
1052
+ : existing.ranks;
862
1053
  const durationMs = incoming.durationMs ??
863
1054
  existing.durationMs ??
864
1055
  computeDurationMs(startedAt, finishedAt);
@@ -877,7 +1068,9 @@ function mergeDagRun(existing, incoming) {
877
1068
  ...(ranks && ranks.length > 0 ? { ranks } : {}),
878
1069
  nodes: mergeDagNodes(existing.nodes, incoming.nodes),
879
1070
  edges: incoming.edges.length > 0 ? incoming.edges : existing.edges,
880
- ...(incoming.dagPath ?? existing.dagPath ? { dagPath: incoming.dagPath ?? existing.dagPath } : {}),
1071
+ ...((incoming.dagPath ?? existing.dagPath)
1072
+ ? { dagPath: incoming.dagPath ?? existing.dagPath }
1073
+ : {}),
881
1074
  };
882
1075
  }
883
1076
  function mergeDagNodes(existing, incoming) {
@@ -921,7 +1114,9 @@ function isFailedNodeStatus(status) {
921
1114
  if (!status)
922
1115
  return false;
923
1116
  const normalized = status.toLowerCase();
924
- return normalized === "error" || normalized === "failed" || normalized === "partial_failed";
1117
+ return (normalized === "error" ||
1118
+ normalized === "failed" ||
1119
+ normalized === "partial_failed");
925
1120
  }
926
1121
  function nodeOutputPreview(node) {
927
1122
  const stdout = readString(node, "stdout") ?? readString(node, "assistantText");
@@ -997,13 +1192,19 @@ async function loadDagEventFiles(repoRoot) {
997
1192
  const outputPreview = readString(parsed, "outputPreview");
998
1193
  nodeMap.set(nodeId, {
999
1194
  nodeId,
1000
- ...(rank ?? prev.rank ? { rank: rank ?? prev.rank } : {}),
1001
- ...(executor ?? prev.executor ? { executor: executor ?? prev.executor } : {}),
1002
- ...(model ?? prev.model ? { model: model ?? prev.model } : {}),
1003
- ...(label ?? prev.label ? { label: label ?? prev.label } : {}),
1004
- ...(status ?? prev.status ? { status: status ?? prev.status } : {}),
1005
- ...(durationMs ?? prev.durationMs ? { durationMs: durationMs ?? prev.durationMs } : {}),
1006
- ...(outputPreview ?? prev.outputPreview ? { outputPreview: outputPreview ?? prev.outputPreview } : {}),
1195
+ ...((rank ?? prev.rank) ? { rank: rank ?? prev.rank } : {}),
1196
+ ...((executor ?? prev.executor)
1197
+ ? { executor: executor ?? prev.executor }
1198
+ : {}),
1199
+ ...((model ?? prev.model) ? { model: model ?? prev.model } : {}),
1200
+ ...((label ?? prev.label) ? { label: label ?? prev.label } : {}),
1201
+ ...((status ?? prev.status) ? { status: status ?? prev.status } : {}),
1202
+ ...((durationMs ?? prev.durationMs)
1203
+ ? { durationMs: durationMs ?? prev.durationMs }
1204
+ : {}),
1205
+ ...((outputPreview ?? prev.outputPreview)
1206
+ ? { outputPreview: outputPreview ?? prev.outputPreview }
1207
+ : {}),
1007
1208
  });
1008
1209
  byId.set(dagRunId, nodeMap);
1009
1210
  }
@@ -1059,7 +1260,9 @@ async function parseDagStateFile(statePath, now) {
1059
1260
  return undefined;
1060
1261
  const runDir = path.dirname(statePath);
1061
1262
  const lifecycleName = path.basename(path.dirname(runDir));
1062
- const lifecycle = lifecycleName === "active" || lifecycleName === "paused" || lifecycleName === "completed"
1263
+ const lifecycle = lifecycleName === "active" ||
1264
+ lifecycleName === "paused" ||
1265
+ lifecycleName === "completed"
1063
1266
  ? lifecycleName
1064
1267
  : undefined;
1065
1268
  const dagRunId = readString(parsed, "runId") ?? path.basename(runDir);
@@ -1077,14 +1280,29 @@ async function parseDagStateFile(statePath, now) {
1077
1280
  const state = parsed;
1078
1281
  const liveness = assessDagRunLiveness({ state, now });
1079
1282
  const effectiveStatus = lifecycle
1080
- ? deriveDagRunEffectiveStatus({ lifecycle, state, liveness: liveness.status })
1283
+ ? deriveDagRunEffectiveStatus({
1284
+ lifecycle,
1285
+ state,
1286
+ liveness: liveness.status,
1287
+ })
1081
1288
  : "unknown";
1082
1289
  const stateConsistent = lifecycle === undefined
1083
1290
  ? false
1084
- : !((lifecycle === "paused" && state.status !== "paused")
1085
- || (lifecycle === "completed" && !["finished", "failed", "partial_failed", "superseded", "abandoned"].includes(state.status)));
1291
+ : !((lifecycle === "paused" && state.status !== "paused") ||
1292
+ (lifecycle === "completed" &&
1293
+ ![
1294
+ "finished",
1295
+ "failed",
1296
+ "partial_failed",
1297
+ "superseded",
1298
+ "abandoned",
1299
+ ].includes(state.status)));
1086
1300
  const recoveryEligibility = lifecycle
1087
- ? assessDagRunRecoveryEligibility({ lifecycle, state, liveness: liveness.status })
1301
+ ? assessDagRunRecoveryEligibility({
1302
+ lifecycle,
1303
+ state,
1304
+ liveness: liveness.status,
1305
+ })
1088
1306
  : undefined;
1089
1307
  return {
1090
1308
  dagRunId,
@@ -1122,7 +1340,9 @@ async function parseDagEdges(runPath, nodes) {
1122
1340
  if (!taskId || !knownNodeIds.has(taskId))
1123
1341
  continue;
1124
1342
  const dependencies = readStringArray(task, "depends_on");
1125
- const compatibleDependencies = dependencies.length > 0 ? dependencies : readStringArray(task, "dependsOn");
1343
+ const compatibleDependencies = dependencies.length > 0
1344
+ ? dependencies
1345
+ : readStringArray(task, "dependsOn");
1126
1346
  for (const dependencyId of compatibleDependencies) {
1127
1347
  if (dependencyId === taskId || !knownNodeIds.has(dependencyId))
1128
1348
  continue;
@@ -1169,7 +1389,10 @@ async function loadDagNodeModels(runPath) {
1169
1389
  const legacy = raw;
1170
1390
  const tasks = legacy.tasks;
1171
1391
  const executorModels = legacy.executorModels;
1172
- if (!Array.isArray(tasks) || typeof executorModels !== "object" || executorModels === null || Array.isArray(executorModels)) {
1392
+ if (!Array.isArray(tasks) ||
1393
+ typeof executorModels !== "object" ||
1394
+ executorModels === null ||
1395
+ Array.isArray(executorModels)) {
1173
1396
  return models;
1174
1397
  }
1175
1398
  for (const task of tasks) {
@@ -1194,7 +1417,9 @@ async function loadDagNodeModels(runPath) {
1194
1417
  function parseDagNodes(parsed, rankByNode, modelByNode) {
1195
1418
  const nodes = [];
1196
1419
  const nodesValue = parsed.nodes;
1197
- if (nodesValue && typeof nodesValue === "object" && !Array.isArray(nodesValue)) {
1420
+ if (nodesValue &&
1421
+ typeof nodesValue === "object" &&
1422
+ !Array.isArray(nodesValue)) {
1198
1423
  for (const [nodeId, nodeVal] of Object.entries(nodesValue)) {
1199
1424
  if (!nodeVal || typeof nodeVal !== "object" || Array.isArray(nodeVal))
1200
1425
  continue;
@@ -1261,7 +1486,9 @@ async function safeReadJson(filePath) {
1261
1486
  function safeParseJson(raw) {
1262
1487
  try {
1263
1488
  const parsed = JSON.parse(raw);
1264
- if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
1489
+ if (parsed === null ||
1490
+ typeof parsed !== "object" ||
1491
+ Array.isArray(parsed)) {
1265
1492
  return undefined;
1266
1493
  }
1267
1494
  return parsed;
@@ -1304,7 +1531,9 @@ function readNumber(value, key) {
1304
1531
  if (!value || typeof value !== "object")
1305
1532
  return undefined;
1306
1533
  const child = value[key];
1307
- return typeof child === "number" && Number.isFinite(child) ? child : undefined;
1534
+ return typeof child === "number" && Number.isFinite(child)
1535
+ ? child
1536
+ : undefined;
1308
1537
  }
1309
1538
  function readStringMatrix(value, key) {
1310
1539
  if (!value || typeof value !== "object")