@tea-agent/loop-agent 0.11.0 → 0.13.0-alpha.0

package/dist/worker/observability/read-model.js

@@ -3,10 +3,44 @@ import { readdir, readFile } from "node:fs/promises";
 import path from "node:path";
 import { truncateUtf8Preview } from "../../shared/preview.js";
 import { parseWorkerEventLine } from "./events.js";
-import { getTaskPoolRoot } from "../pool/run-store.js";
+import { readBoundedJsonlTail } from "./event-history.js";
+import { getTaskPoolRoot, listLegacyStateFiles, listTaskPoolStates, } from "../pool/run-store.js";
 import { assessDagRunLiveness, assessDagRunRecoveryEligibility, deriveDagRunEffectiveStatus, } from "../../workflows/dag/lifecycle.js";
 import { parseDagSpec, resolveModelForTask, } from "../../workflows/dag/types.js";
 import { loadFeatureDecisionModels } from "../feature/decision-loader.js";
+/** Internal map key: featureId + NUL + taskId (stable, non-printable delimiter). */
+export function taskIdentityKey(featureId, taskId) {
+    return `${featureId}\0${taskId}`;
+}
+export function compareTaskIdentity(a, b) {
+    const fa = a.featureId ?? "";
+    const fb = b.featureId ?? "";
+    const byFeature = fa.localeCompare(fb);
+    if (byFeature !== 0)
+        return byFeature;
+    return a.taskId.localeCompare(b.taskId);
+}
+export function findTasksByTaskId(tasks, taskId) {
+    return tasks.filter((task) => task.taskId === taskId);
+}
+export function resolveLegacyTask(tasks, taskId) {
+    const matches = findTasksByTaskId(tasks, taskId);
+    const candidates = matches.map((task) => ({
+        featureId: task.featureId,
+        taskId: task.taskId,
+        status: task.status,
+        workerRunId: task.workerRunId,
+    }));
+    if (matches.length === 0) {
+        return { kind: "missing", candidates: [] };
+    }
+    if (matches.length === 1) {
+        return { kind: "unique", task: matches[0], candidates };
+    }
+    return { kind: "ambiguous", candidates };
+}
+export const TASK_RUN_HISTORY_DEFAULT_LIMIT = 20;
+export const TASK_RUN_HISTORY_MAX_LIMIT = 100;
 const ACTIVE_TASK_STATUSES = new Set(["running", "pending"]);
 const ACTIVE_BATCH_STATUSES = new Set(["running", "pending"]);
 export async function buildGlobalSnapshot(options) {
@@ -15,7 +49,8 @@ export async function buildGlobalSnapshot(options) {
         const now = options.now ?? (() => new Date());
         const staleThresholdMs = options.staleThresholdMs ?? 90_000;
         const quietThresholdMs = options.quietThresholdMs ?? 60_000;
-        const [events, ledgerRuns, ledgerBatches, states, handoffs, dagRuns] = await Promise.all([
+        const taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
+        const [eventsLoad, ledgerLoad, ledgerBatches, statesLoad, handoffs, dagRuns,] = await Promise.all([
             loadObservabilityEvents(repoRoot),
             loadLedgerRuns(repoRoot),
             loadLedgerBatches(repoRoot),
@@ -23,29 +58,45 @@ export async function buildGlobalSnapshot(options) {
             loadFailureHandoffs(repoRoot),
             loadDagRuns(repoRoot, now()),
         ]);
+        const events = eventsLoad.events;
+        const ledgerRuns = ledgerLoad.runs;
+        const identityWarnings = [];
         const taskMap = new Map();
-        for (const state of states) {
-            mergeStateRecord(taskMap, state);
-        }
+        // Ledger first so featureId-bearing rows seed composite identity before states/events.
         for (const run of ledgerRuns) {
-            mergeLedgerRun(taskMap, run);
+            mergeLedgerRun(taskMap, run, identityWarnings);
+        }
+        for (const state of statesLoad.states) {
+            mergeStateRecord(taskMap, state, identityWarnings);
+        }
+        // Attach uniquely-resolvable legacy flat states after owned rows exist.
+        for (const state of statesLoad.legacyCandidates) {
+            mergeLegacyStateRecord(taskMap, state, identityWarnings);
         }
-        mergeEvents(taskMap, events);
+        const workerRunFeatureIndex = buildWorkerRunFeatureIndex(ledgerRuns, taskMap);
+        mergeEvents(taskMap, events, workerRunFeatureIndex, identityWarnings);
         mergeHandoffs(taskMap, handoffs);
         applyStaleDetection(taskMap, now(), staleThresholdMs, quietThresholdMs);
-        const historicalRunMap = buildHistoricalRunMap(ledgerRuns, events);
+        const historicalRunMap = buildHistoricalRunMap(ledgerRuns, events, workerRunFeatureIndex, identityWarnings);
         mergeHandoffs(historicalRunMap, handoffs);
         applyStaleDetection(historicalRunMap, now(), staleThresholdMs, quietThresholdMs);
         const tasks = [...taskMap.values()]
             .map(toPublicTaskSummary)
             .map((task) => normalizeTaskArtifactRefs(task, repoRoot))
-            .sort((a, b) => a.taskId.localeCompare(b.taskId));
+            .sort(compareTaskIdentity);
         const batches = buildBatches(ledgerBatches, events, historicalRunMap).map((batch) => ({
             ...batch,
             tasks: batch.tasks.map((task) => normalizeTaskArtifactRefs(task, repoRoot)),
         }));
         const health = computeHealth(batches, tasks, dagRuns);
-        const { features, projectionWarnings } = await loadFeatureDecisionModels(repoRoot);
+        const featureLoad = await loadFeatureDecisionModels(repoRoot);
+        const projectionWarnings = [
+            ...(eventsLoad.warnings ?? []),
+            ...(ledgerLoad.warnings ?? []),
+            ...(statesLoad.warnings ?? []),
+            ...identityWarnings,
+            ...(featureLoad.projectionWarnings ?? []),
+        ];
         return {
             schemaVersion: 1,
             generatedAt: now().toISOString(),
@@ -53,7 +104,8 @@ export async function buildGlobalSnapshot(options) {
             batches,
             tasks,
             dagRuns,
-            features,
+            features: featureLoad.features,
+            taskPool: { present: taskPoolPresent },
             projectionWarnings,
             health,
         };
@@ -80,7 +132,105 @@ export async function buildGlobalSnapshot(options) {
         return snapshot;
     }
 }
+/**
+ * Bounded Task run history from Task Pool ledger facts (newest first).
+ * Filtered by composite `{featureId, taskId}`.
+ * `before` is an exclusive upper bound on the sort key (`recordedAt` or empty).
+ */
+export async function listTaskRunHistory(repoRoot, featureId, taskId, options = {}) {
+    const limit = clampTaskRunHistoryLimit(options.limit);
+    const before = decodeTaskRunHistoryCursor(options.before);
+    const [ledgerLoad, eventsLoad] = await Promise.all([
+        loadLedgerRuns(path.resolve(repoRoot)),
+        loadObservabilityEvents(path.resolve(repoRoot)),
+    ]);
+    const ledgerRuns = ledgerLoad.runs;
+    const events = eventsLoad.events;
+    const workerRunFeatureIndex = buildWorkerRunFeatureIndex(ledgerRuns);
+    const historical = buildHistoricalRunMap(ledgerRuns, events, workerRunFeatureIndex);
+    const items = [];
+    for (const run of ledgerRuns) {
+        if (run.taskId !== taskId)
+            continue;
+        if ((run.featureId ?? "") !== featureId)
+            continue;
+        const enriched = historical.get(run.workerRunId);
+        items.push({
+            workerRunId: run.workerRunId,
+            taskId: run.taskId,
+            featureId: run.featureId ?? featureId,
+            status: enriched?.status ?? mapRunRecordStatus(run.status),
+            startedAt: enriched?.startedAt,
+            finishedAt: enriched?.finishedAt ?? run.recordedAt,
+            recordedAt: run.recordedAt,
+            batchRunId: run.batchRunId,
+            dagRunId: enriched?.dagRunId,
+            ...(run.retryOfWorkerRunId
+                ? { retryOfWorkerRunId: run.retryOfWorkerRunId }
+                : {}),
+            failureCategory: run.failureCategory ?? enriched?.failureCategory,
+            recommendedFollowUp: run.recommendedFollowUp ?? enriched?.recommendedFollowUp,
+        });
+    }
+    items.sort((a, b) => {
+        const ta = a.recordedAt ?? a.finishedAt ?? "";
+        const tb = b.recordedAt ?? b.finishedAt ?? "";
+        if (ta !== tb)
+            return tb.localeCompare(ta);
+        return b.workerRunId.localeCompare(a.workerRunId);
+    });
+    const eligible = before
+        ? items.filter((item) => taskRunSortKey(item).localeCompare(before) < 0)
+        : items;
+    const page = eligible.slice(0, limit);
+    const nextBefore = eligible.length > limit
+        ? encodeTaskRunHistoryCursor(page[page.length - 1])
+        : null;
+    return {
+        featureId,
+        taskId,
+        runs: page,
+        limit,
+        nextBefore,
+    };
+}
+function taskRunSortKey(item) {
+    return `${item.recordedAt ?? item.finishedAt ?? ""}\u0000${item.workerRunId}`;
+}
+function encodeTaskRunHistoryCursor(item) {
+    if (!item)
+        return null;
+    return `v1.${Buffer.from(taskRunSortKey(item), "utf-8").toString("base64url")}`;
+}
+function decodeTaskRunHistoryCursor(cursor) {
+    const value = cursor?.trim();
+    if (!value)
+        return null;
+    if (!value.startsWith("v1.")) {
+        // Compatibility with the initial timestamp-only cursor shape.
+        return `${value}\u0000`;
+    }
+    try {
+        return Buffer.from(value.slice(3), "base64url").toString("utf-8");
+    }
+    catch {
+        return `${value}\u0000`;
+    }
+}
+export function clampTaskRunHistoryLimit(limit) {
+    if (limit === undefined || !Number.isFinite(limit) || limit <= 0) {
+        return TASK_RUN_HISTORY_DEFAULT_LIMIT;
+    }
+    return Math.min(Math.floor(limit), TASK_RUN_HISTORY_MAX_LIMIT);
+}
 function emptySnapshot(repoRoot, now) {
+    let taskPoolPresent = false;
+    try {
+        taskPoolPresent = existsSync(getTaskPoolRoot(repoRoot));
+    }
+    catch {
+        taskPoolPresent = false;
+    }
     return {
         schemaVersion: 1,
         generatedAt: now().toISOString(),
@@ -88,6 +238,7 @@ function emptySnapshot(repoRoot, now) {
         batches: [],
         tasks: [],
         dagRuns: [],
+        taskPool: { present: taskPoolPresent },
         health: {
             activeBatches: 0,
             activeTasks: 0,
@@ -118,15 +269,29 @@ function emptySnapshot(repoRoot, now) {
     };
 }
 function toPublicTaskSummary(task) {
-    const { batchRunId: _batch, timeoutMs: _to, commandStartedAt: _cs, ...rest } = task;
+    const { timeoutMs: _to, commandStartedAt: _cs, ...rest } = task;
     return rest;
 }
-function mergeStateRecord(taskMap, state) {
-    const existing = taskMap.get(state.taskId) ?? { taskId: state.taskId, status: "unknown" };
-    taskMap.set(state.taskId, {
+function mergeStateRecord(taskMap, state, warnings) {
+    const featureId = state.featureId?.trim();
+    if (!featureId) {
+        warnings?.push(`state for task ${state.taskId} missing featureId; excluded from composite projection`);
+        return;
+    }
+    const key = taskIdentityKey(featureId, state.taskId);
+    const existing = taskMap.get(key) ?? {
+        taskId: state.taskId,
+        featureId,
+        status: "unknown",
+    };
+    taskMap.set(key, {
         ...existing,
-        status: existing.status !== "unknown" ? existing.status : mapStateStatus(state.status),
+        featureId,
+        status: existing.status !== "unknown"
+            ? existing.status
+            : mapStateStatus(state.status),
         workerRunId: existing.workerRunId ?? state.workerRunId,
+        updatedAt: state.updatedAt ?? existing.updatedAt,
         artifactRefs: mergeArtifactRefs(existing.artifactRefs, {
             runRecordPath: state.lastRunRecordPath,
         }),
@@ -134,46 +299,140 @@ function mergeStateRecord(taskMap, state) {
         recommendedFollowUp: existing.recommendedFollowUp ?? state.recommendedFollowUp,
     });
 }
-function mergeLedgerRun(taskMap, run) {
-    const existing = taskMap.get(run.taskId) ?? { taskId: run.taskId, status: "unknown" };
+/**
+ * Legacy flat states/<taskId>.json: only merge when the taskId already maps to
+ * exactly one composite row in the snapshot (unique ownership). Otherwise warn+exclude.
+ */
+function mergeLegacyStateRecord(taskMap, state, warnings) {
+    const owners = [...taskMap.values()].filter((task) => task.taskId === state.taskId && Boolean(task.featureId));
+    if (owners.length === 1) {
+        const owner = owners[0];
+        mergeStateRecord(taskMap, { ...state, featureId: owner.featureId }, warnings);
+        warnings.push(`legacy flat state for task ${state.taskId} attached to unique feature ${owner.featureId}`);
+        return;
+    }
+    if (owners.length === 0) {
+        // No owned composite row yet: still surface as a warning and do not invent featureId.
+        warnings.push(`legacy flat state for task ${state.taskId} has no unique feature ownership; excluded from composite projection`);
+        return;
+    }
+    warnings.push(`legacy flat state for task ${state.taskId} is ambiguous across features; excluded from composite projection`);
+}
+function mergeLedgerRun(taskMap, run, warnings) {
+    const featureId = run.featureId?.trim();
+    if (!featureId) {
+        warnings?.push(`ledger run ${run.workerRunId} for task ${run.taskId} missing featureId; excluded from composite projection`);
+        return;
+    }
+    const key = taskIdentityKey(featureId, run.taskId);
+    const existing = taskMap.get(key) ?? {
+        taskId: run.taskId,
+        featureId,
+        status: "unknown",
+    };
     const sameRun = existing.workerRunId === run.workerRunId;
     const runArtifactRefs = {
         runRecordPath: run.runRecordPath,
         dagPath: run.dagPath,
         ...run.failureArtifacts,
     };
-    taskMap.set(run.taskId, {
+    taskMap.set(key, {
         ...(sameRun
             ? existing
             : {
                 taskId: run.taskId,
+                featureId,
                 status: "unknown",
-                featureId: existing.featureId,
                 harnessTaskId: existing.harnessTaskId,
             }),
         batchRunId: run.batchRunId,
-        featureId: run.featureId ?? existing.featureId,
+        featureId,
         status: mapRunRecordStatus(run.status),
         workerRunId: run.workerRunId,
         harnessTaskId: run.harnessTaskId ?? existing.harnessTaskId,
         finishedAt: run.recordedAt ?? (sameRun ? existing.finishedAt : undefined),
         failureCategory: run.failureCategory ?? (sameRun ? existing.failureCategory : undefined),
-        recommendedFollowUp: run.recommendedFollowUp ?? (sameRun ? existing.recommendedFollowUp : undefined),
+        recommendedFollowUp: run.recommendedFollowUp ??
+            (sameRun ? existing.recommendedFollowUp : undefined),
         artifactRefs: mergeArtifactRefs(sameRun ? existing.artifactRefs : undefined, runArtifactRefs),
     });
 }
-function mergeEvents(taskMap, events) {
+/**
+ * Build workerRunId → featureId unique index from ledger + already-owned task rows.
+ * Ambiguous (0 or N) mappings are omitted so callers treat them as unowned.
+ */
+function buildWorkerRunFeatureIndex(ledgerRuns, taskMap) {
+    const candidates = new Map();
+    const add = (workerRunId, featureId) => {
+        const wr = workerRunId?.trim();
+        const fid = featureId?.trim();
+        if (!wr || !fid)
+            return;
+        let set = candidates.get(wr);
+        if (!set) {
+            set = new Set();
+            candidates.set(wr, set);
+        }
+        set.add(fid);
+    };
+    for (const run of ledgerRuns) {
+        add(run.workerRunId, run.featureId);
+    }
+    if (taskMap) {
+        for (const task of taskMap.values()) {
+            add(task.workerRunId, task.featureId);
+        }
+    }
+    const unique = new Map();
+    for (const [workerRunId, features] of candidates) {
+        if (features.size === 1) {
+            unique.set(workerRunId, [...features][0]);
+        }
+    }
+    return unique;
+}
+function mergeEvents(taskMap, events, workerRunFeatureIndex, warnings) {
     const sorted = [...events].sort((a, b) => a.at.localeCompare(b.at));
     for (const event of sorted) {
         if (event.taskId) {
-            applyTaskEvent(taskMap, event);
+            applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings);
         }
     }
 }
-function applyTaskEvent(taskMap, event) {
+function applyTaskEvent(taskMap, event, workerRunFeatureIndex, warnings) {
+    const taskId = event.taskId;
+    const featureId = resolveEventFeatureId(event, taskMap, workerRunFeatureIndex, warnings);
+    if (!featureId)
+        return;
+    const key = taskIdentityKey(featureId, taskId);
+    const existing = taskMap.get(key) ?? {
+        taskId,
+        featureId,
+        status: "unknown",
+    };
+    taskMap.set(key, applyEventToSummary(existing, event));
+}
+function resolveEventFeatureId(event, taskMap, workerRunFeatureIndex, warnings) {
     const taskId = event.taskId;
-    const existing = taskMap.get(taskId) ?? { taskId, status: "unknown" };
-    taskMap.set(taskId, applyEventToSummary(existing, event));
+    if (!taskId)
+        return undefined;
+    if (event.workerRunId) {
+        const fromIndex = workerRunFeatureIndex.get(event.workerRunId);
+        if (fromIndex)
+            return fromIndex;
+        // Fall through: maybe a single existing composite row already owns this workerRunId.
+        const byWorkerRun = [...taskMap.values()].filter((task) => task.workerRunId === event.workerRunId && Boolean(task.featureId));
+        if (byWorkerRun.length === 1)
+            return byWorkerRun[0].featureId;
+    }
+    // Unique taskId ownership in current map (single Feature).
+    const byTaskId = [...taskMap.values()].filter((task) => task.taskId === taskId && Boolean(task.featureId));
+    if (byTaskId.length === 1)
+        return byTaskId[0].featureId;
+    warnings?.push(`unowned event ${event.type} for task ${taskId}` +
+        (event.workerRunId ? ` workerRunId ${event.workerRunId}` : "") +
+        ` cannot uniquely resolve featureId; excluded from task projection`);
+    return undefined;
 }
 function applyEventToSummary(existing, event) {
     const updated = {
@@ -224,9 +483,10 @@ function applyEventToSummary(existing, event) {
     }
     return updated;
 }
-function buildHistoricalRunMap(ledgerRuns, events) {
+function buildHistoricalRunMap(ledgerRuns, events, workerRunFeatureIndex = new Map(), warnings) {
     const runMap = new Map();
     for (const run of ledgerRuns) {
+        // History map is keyed by workerRunId (run-scoped); featureId still required when present.
         runMap.set(run.workerRunId, {
             taskId: run.taskId,
             batchRunId: run.batchRunId,
@@ -247,12 +507,23 @@ function buildHistoricalRunMap(ledgerRuns, events) {
     for (const event of [...events].sort((a, b) => a.at.localeCompare(b.at))) {
         if (!event.workerRunId || !event.taskId)
             continue;
-        const existing = runMap.get(event.workerRunId) ?? {
+        const existing = runMap.get(event.workerRunId);
+        if (existing) {
+            runMap.set(event.workerRunId, applyEventToSummary(existing, event));
+            continue;
+        }
+        // Only invent a historical row when workerRunId uniquely maps to a feature.
+        const featureId = workerRunFeatureIndex.get(event.workerRunId);
+        if (!featureId) {
+            warnings?.push(`unowned historical event ${event.type} for task ${event.taskId} workerRunId ${event.workerRunId} cannot uniquely resolve featureId; excluded from historical run map`);
+            continue;
+        }
+        runMap.set(event.workerRunId, applyEventToSummary({
             taskId: event.taskId,
+            featureId,
             workerRunId: event.workerRunId,
             status: "unknown",
-        };
-        runMap.set(event.workerRunId, applyEventToSummary(existing, event));
+        }, event));
     }
     return runMap;
 }
@@ -320,7 +591,9 @@ function computeLiveness(task, nowMs, staleThresholdMs, quietThresholdMs) {
             task.timeoutAt = new Date(commandStartedMs + timeoutMs).toISOString();
         }
     }
-    const heartbeatMs = task.lastHeartbeatAt ? Date.parse(task.lastHeartbeatAt) : NaN;
+    const heartbeatMs = task.lastHeartbeatAt
+        ? Date.parse(task.lastHeartbeatAt)
+        : NaN;
     const outputMs = task.lastOutputAt ? Date.parse(task.lastOutputAt) : NaN;
     // Priority 1: timeout-risk (elapsed > 0.8 * timeoutMs), only if timeoutMs known.
     if (timeoutMs !== undefined &&
@@ -333,7 +606,9 @@ function computeLiveness(task, nowMs, staleThresholdMs, quietThresholdMs) {
     // (a) Heartbeat present but older than staleThresholdMs.
     // (b) No heartbeat but running longer than staleThresholdMs (startedAt known).
     const heartbeatStale = !Number.isNaN(heartbeatMs) && nowMs - heartbeatMs > staleThresholdMs;
-    const noHeartbeatStale = Number.isNaN(heartbeatMs) && elapsedMs !== undefined && elapsedMs > staleThresholdMs;
+    const noHeartbeatStale = Number.isNaN(heartbeatMs) &&
+        elapsedMs !== undefined &&
+        elapsedMs > staleThresholdMs;
     if (heartbeatStale || noHeartbeatStale) {
         task.liveness = "stale";
         task.status = "stale";
@@ -390,7 +665,7 @@ function buildBatches(ledgerBatches, events, taskMap) {
         batch.tasks.push(toPublicTaskSummary(task));
     }
     for (const batch of batchMap.values()) {
-        batch.tasks.sort((a, b) => a.taskId.localeCompare(b.taskId));
+        batch.tasks.sort(compareTaskIdentity);
         if (batch.summary === undefined && batch.tasks.length > 0) {
             batch.summary = summarizeTasks(batch.tasks);
         }
@@ -414,7 +689,8 @@ function summarizeTasks(tasks) {
 function computeHealth(batches, tasks, dagRuns) {
     const worker = {
         activeTasks: tasks.filter((t) => ACTIVE_TASK_STATUSES.has(t.status)).length,
-        activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status)).length,
+        activeBatches: batches.filter((b) => ACTIVE_BATCH_STATUSES.has(b.status))
+            .length,
         quietCount: tasks.filter((t) => t.liveness === "quiet").length,
         staleCount: tasks.filter((t) => t.status === "stale").length,
         timeoutRiskCount: tasks.filter((t) => t.liveness === "timeout-risk").length,
@@ -451,7 +727,8 @@ const TERMINAL_RUN_STATUSES = new Set([
  * already-loaded DagRunSummary facts without touching app.js or new files.
  */
 function isDagRunActiveForHealth(dag) {
-    if (dag.effectiveStatus && ACTIVE_EFFECTIVE_STATUSES.has(dag.effectiveStatus)) {
+    if (dag.effectiveStatus &&
+        ACTIVE_EFFECTIVE_STATUSES.has(dag.effectiveStatus)) {
         return true;
     }
     // Only "unknown" (and absent) means liveness evidence is insufficient;
@@ -472,7 +749,16 @@ function isDagRunActiveForHealth(dag) {
 }
 function isNodeStatusActive(status) {
     const normalized = (status ?? "").toLowerCase();
-    if (["finished", "completed", "succeeded", "done", "error", "failed", "skipped", "partial_failed"].includes(normalized)) {
+    if ([
+        "finished",
+        "completed",
+        "succeeded",
+        "done",
+        "error",
+        "failed",
+        "skipped",
+        "partial_failed",
+    ].includes(normalized)) {
         return false;
     }
     return true;
@@ -511,13 +797,13 @@ function computeDagHealth(dagRuns) {
             interruptedRuns++;
         if (dag.stateConsistent === false)
             inconsistentRuns++;
-        if (lifecycle !== "completed"
-            && (lifecycle === "paused"
-                || liveness === "stale"
-                || liveness === "orphaned"
-                || dag.effectiveStatus === "interrupted"
-                || dag.effectiveStatus === "remote-unknown"
-                || dag.stateConsistent === false)) {
+        if (lifecycle !== "completed" &&
+            (lifecycle === "paused" ||
+                liveness === "stale" ||
+                liveness === "orphaned" ||
+                dag.effectiveStatus === "interrupted" ||
+                dag.effectiveStatus === "remote-unknown" ||
+                dag.stateConsistent === false)) {
             attentionRuns++;
         }
     }
@@ -547,12 +833,14 @@ function sanitizeProjectionError(error) {
     let name;
     let message;
     try {
-        name = typeof error.name === "string"
-            ? error.name
-            : undefined;
-        message = typeof error.message === "string"
-            ? error.message
-            : undefined;
+        name =
+            typeof error.name === "string"
+                ? error.name
+                : undefined;
+        message =
+            typeof error.message === "string"
+                ? error.message
+                : undefined;
     }
     catch {
         return { message: fallback, at };
@@ -563,12 +851,12 @@ function sanitizeProjectionError(error) {
     }
     // Reject anything that looks like a stack frame, an absolute path, or a secret.
     const raw = String(candidate);
-    if (raw.includes("\n")
-        || /\bat \S+/i.test(raw)
-        || /[A-Za-z]:\\/.test(raw)
-        || raw.includes("/Users/")
-        || raw.includes("/home/")
-        || /SECRET|TOKEN|PASSWORD|API_KEY/i.test(raw)) {
+    if (raw.includes("\n") ||
+        /\bat \S+/i.test(raw) ||
+        /[A-Za-z]:\\/.test(raw) ||
+        raw.includes("/Users/") ||
+        raw.includes("/home/") ||
+        /SECRET|TOKEN|PASSWORD|API_KEY/i.test(raw)) {
         return { message: name ? `${name}: 投影失败` : fallback, at };
     }
     const trimmed = raw.trim().slice(0, 200);
@@ -581,7 +869,11 @@ function mergeArtifactRefs(existing, incoming) {
     for (const [key, value] of Object.entries(incoming)) {
         if (!value)
             continue;
-        if (key === "runRecordPath" || key === "dagPath" || key === "reportMarkdown" || key === "doctorMarkdown" || key === "closeoutDraft") {
+        if (key === "runRecordPath" ||
+            key === "dagPath" ||
+            key === "reportMarkdown" ||
+            key === "doctorMarkdown" ||
+            key === "closeoutDraft") {
             merged[key] = value;
         }
     }
@@ -645,6 +937,7 @@ function mapStateStatus(status) {
         case "Running":
             return "running";
         case "Queued":
+        case "Ready":
             return "pending";
         case "Done":
             return "succeeded";
@@ -668,49 +961,86 @@ function mapBatchStatus(status) {
             return "unknown";
     }
 }
+/** Per-file hard cap for projection merge (snapshot must not embed unbounded ledger). */
+const SNAPSHOT_EVENTS_MAX_LINES_PER_FILE = 2_000;
+const SNAPSHOT_EVENTS_MAX_BYTES_PER_FILE = 4 * 1024 * 1024;
+/** Global hard cap after dedupe for snapshot projection merge. */
+const SNAPSHOT_EVENTS_MAX_TOTAL = 8_000;
+const SNAPSHOT_LEDGER_MAX_LINES = 10_000;
+const SNAPSHOT_LEDGER_MAX_BYTES = 8 * 1024 * 1024;
 async function loadObservabilityEvents(repoRoot) {
     const events = [];
+    const warnings = [];
     const obsRoot = path.join(getTaskPoolRoot(repoRoot), "observability");
-    await appendJsonlEvents(path.join(obsRoot, "events.jsonl"), events);
+    await appendJsonlEventsBounded(path.join(obsRoot, "events.jsonl"), events, warnings);
     const batchesDir = path.join(obsRoot, "batches");
-    await forEachSubdirJsonl(batchesDir, "events.jsonl", events);
+    await forEachSubdirJsonl(batchesDir, "events.jsonl", events, warnings);
     const runsDir = path.join(obsRoot, "runs");
-    await forEachSubdirJsonl(runsDir, "events.jsonl", events);
-    return [...new Map(events.map((event) => [event.id, event])).values()];
+    await forEachSubdirJsonl(runsDir, "events.jsonl", events, warnings);
+    const deduped = [
+        ...new Map(events.map((event) => [event.id, event])).values(),
+    ].sort(compareWorkerEvents);
+    if (deduped.length > SNAPSHOT_EVENTS_MAX_TOTAL) {
+        warnings.push(`observability event projection truncated to ${SNAPSHOT_EVENTS_MAX_TOTAL} events (capacity bound)`);
+        return {
+            events: deduped.slice(-SNAPSHOT_EVENTS_MAX_TOTAL),
+            warnings,
+        };
+    }
+    return { events: deduped, warnings };
+}
+function compareWorkerEvents(a, b) {
+    const byTime = a.at.localeCompare(b.at);
+    if (byTime !== 0)
+        return byTime;
+    return a.id.localeCompare(b.id);
 }
-async function forEachSubdirJsonl(parentDir, filename, events) {
+async function forEachSubdirJsonl(parentDir, filename, events, warnings) {
     try {
         const entries = await readdir(parentDir, { withFileTypes: true });
         for (const entry of entries) {
             if (!entry.isDirectory())
                 continue;
-            await appendJsonlEvents(path.join(parentDir, entry.name, filename), events);
+            await appendJsonlEventsBounded(path.join(parentDir, entry.name, filename), events, warnings);
         }
     }
     catch {
         // skip missing dir
     }
 }
-async function appendJsonlEvents(filePath, events) {
-    const raw = await safeReadText(filePath);
-    if (!raw)
-        return;
-    for (const line of raw.split("\n")) {
-        const trimmed = line.trim();
+async function appendJsonlEventsBounded(filePath, events, warnings) {
+    const tail = await readBoundedJsonlTail(filePath, {
+        maxBytes: SNAPSHOT_EVENTS_MAX_BYTES_PER_FILE,
+        maxLines: SNAPSHOT_EVENTS_MAX_LINES_PER_FILE,
+    });
+    let accepted = 0;
+    for (const line of tail.lines) {
+        const trimmed = line.text.trim();
         if (!trimmed)
             continue;
         const event = parseWorkerEventLine(trimmed);
-        if (event)
+        if (event) {
             events.push(event);
+            accepted += 1;
+        }
+    }
+    if (tail.truncated) {
+        warnings.push("truncated event tail for " + path.basename(path.dirname(filePath)) + "/" + path.basename(filePath) + " to newest " + accepted + " valid events");
     }
 }
 async function loadLedgerRuns(repoRoot) {
     const runs = [];
-    const raw = await safeReadText(path.join(getTaskPoolRoot(repoRoot), "runs.jsonl"));
-    if (!raw)
-        return runs;
-    for (const line of raw.split("\n")) {
-        const trimmed = line.trim();
+    const warnings = [];
+    const tail = await readBoundedJsonlTail(path.join(getTaskPoolRoot(repoRoot), "runs.jsonl"), { maxBytes: SNAPSHOT_LEDGER_MAX_BYTES, maxLines: SNAPSHOT_LEDGER_MAX_LINES });
+    if (tail.truncated) {
+        warnings.push("runs.jsonl projection truncated to newest " +
+            SNAPSHOT_LEDGER_MAX_LINES +
+            " lines / " +
+            SNAPSHOT_LEDGER_MAX_BYTES +
+            " bytes");
+    }
+    for (const line of tail.lines) {
+        const trimmed = line.text.trim();
         if (!trimmed)
             continue;
         const parsed = safeParseJson(trimmed);
@@ -724,6 +1054,7 @@ async function loadLedgerRuns(repoRoot) {
             continue;
         const failure = readObject(parsed, "failure");
         const failureArtifacts = readObject(parsed, "failureArtifacts");
+        const retryOfWorkerRunId = readString(parsed, "retryOfWorkerRunId");
         runs.push({
             batchRunId,
             workerRunId,
@@ -734,8 +1065,11 @@ async function loadLedgerRuns(repoRoot) {
             runRecordPath: readString(parsed, "runRecordPath"),
             dagPath: readString(parsed, "dagPath"),
             recordedAt: readString(parsed, "recordedAt"),
+            ...(retryOfWorkerRunId ? { retryOfWorkerRunId } : {}),
             failureCategory: failure ? readString(failure, "category") : undefined,
-            recommendedFollowUp: failure ? readString(failure, "recommendedFollowUpKind") : undefined,
+            recommendedFollowUp: failure
+                ? readString(failure, "recommendedFollowUpKind")
+                : undefined,
             failureArtifacts: failureArtifacts
                 ? {
                     reportMarkdown: readString(failureArtifacts, "reportMarkdownArtifactPath"),
@@ -745,7 +1079,7 @@ async function loadLedgerRuns(repoRoot) {
                 : undefined,
         });
     }
-    return runs;
+    return { runs, warnings };
 }
 async function loadLedgerBatches(repoRoot) {
     const batches = [];
@@ -761,20 +1095,26 @@ async function loadLedgerBatches(repoRoot) {
             if (!parsed)
                 continue;
             const summaryObj = readObject(parsed, "summary");
+            let summary;
+            if (summaryObj) {
+                const recordErrors = readNumber(summaryObj, "recordErrors");
+                const runErrors = readNumber(summaryObj, "runErrors");
+                summary = {
+                    total: readNumber(summaryObj, "total") ?? 0,
+                    succeeded: readNumber(summaryObj, "succeeded") ?? 0,
+                    failed: readNumber(summaryObj, "failed") ?? 0,
+                    reused: readNumber(summaryObj, "reused") ?? 0,
+                    ...(recordErrors !== undefined ? { recordErrors } : {}),
+                    ...(runErrors !== undefined ? { runErrors } : {}),
+                };
+            }
             batches.push({
                 batchRunId: readString(parsed, "batchRunId") ?? batchRunId,
                 featureId: readString(parsed, "featureId"),
                 startedAt: readString(parsed, "startedAt"),
                 finishedAt: readString(parsed, "finishedAt"),
                 status: mapBatchStatus(readString(parsed, "status") ?? "unknown"),
-                summary: summaryObj
-                    ? {
-                        total: readNumber(summaryObj, "total") ?? 0,
-                        succeeded: readNumber(summaryObj, "succeeded") ?? 0,
-                        failed: readNumber(summaryObj, "failed") ?? 0,
-                        reused: readNumber(summaryObj, "reused") ?? 0,
-                    }
-                    : undefined,
+                summary,
                 tasks: [],
                 batchRunPath,
             });
@@ -787,34 +1127,88 @@ async function loadLedgerBatches(repoRoot) {
 }
 async function loadStateRecords(repoRoot) {
     const states = [];
-    const stateDir = path.join(getTaskPoolRoot(repoRoot), "states");
+    const legacyCandidates = [];
+    const warnings = [];
+    // Primary path: v2 nested states via pool list API (read-only import).
     try {
-        const entries = await readdir(stateDir, { withFileTypes: true });
-        for (const entry of entries) {
-            if (!entry.isFile() || !entry.name.endsWith(".json"))
+        const v2 = await listTaskPoolStates(repoRoot);
+        for (const record of v2) {
+            if (!record.taskId || !record.status)
                 continue;
-            const parsed = await safeReadJson(path.join(stateDir, entry.name));
-            if (!parsed)
+            if (!record.featureId) {
+                warnings.push(`v2 state for task ${record.taskId} missing featureId; excluded from composite projection`);
                 continue;
-            const taskId = readString(parsed, "taskId") ?? entry.name.replace(/\.json$/, "");
+            }
+            states.push({
+                taskId: record.taskId,
+                featureId: record.featureId,
+                status: record.status,
+                workerRunId: record.workerRunId,
+                lastRunRecordPath: record.lastRunRecordPath,
+                updatedAt: record.updatedAt,
+                failureCategory: record.failure?.category,
+                recommendedFollowUp: record.failure?.recommendedFollowUpKind,
+            });
+        }
+    }
+    catch (error) {
+        warnings.push(`failed to load feature-scoped task pool states: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    // Compatibility: legacy flat states/<taskId>.json (warning + unique-only merge later).
+    try {
+        const legacyFiles = await listLegacyStateFiles(repoRoot);
+        for (const file of legacyFiles) {
+            const parsed = await safeReadJson(file.path);
+            if (!parsed) {
+                warnings.push(`legacy flat state file unreadable or corrupt: ${file.path}`);
+                continue;
+            }
+            const taskId = readString(parsed, "taskId") ?? file.taskId;
             const status = readString(parsed, "status");
-            if (!status)
+            if (!taskId || !status) {
+                warnings.push(`legacy flat state missing taskId/status: ${file.path}`);
                 continue;
+            }
             const failure = readObject(parsed, "failure");
-            states.push({
+            const embeddedFeatureId = readString(parsed, "featureId");
+            if (embeddedFeatureId) {
+                // Rare: flat file already carries featureId — treat as owned state.
+                states.push({
+                    taskId,
+                    featureId: embeddedFeatureId,
+                    status,
+                    workerRunId: readString(parsed, "workerRunId"),
+                    lastRunRecordPath: readString(parsed, "lastRunRecordPath"),
+                    updatedAt: readString(parsed, "updatedAt"),
+                    failureCategory: failure
+                        ? readString(failure, "category")
+                        : undefined,
+                    recommendedFollowUp: failure
+                        ? readString(failure, "recommendedFollowUpKind")
+                        : undefined,
+                    legacyFlat: true,
+                });
+                warnings.push(`legacy flat state for task ${taskId} carries featureId ${embeddedFeatureId}`);
+                continue;
+            }
+            legacyCandidates.push({
                 taskId,
                 status,
                 workerRunId: readString(parsed, "workerRunId"),
                 lastRunRecordPath: readString(parsed, "lastRunRecordPath"),
+                updatedAt: readString(parsed, "updatedAt"),
                 failureCategory: failure ? readString(failure, "category") : undefined,
-                recommendedFollowUp: failure ? readString(failure, "recommendedFollowUpKind") : undefined,
+                recommendedFollowUp: failure
+                    ? readString(failure, "recommendedFollowUpKind")
+                    : undefined,
+                legacyFlat: true,
             });
         }
     }
-    catch {
-        // skip
+    catch (error) {
+        warnings.push(`failed to scan legacy flat states: ${error instanceof Error ? error.message : String(error)}`);
     }
-    return states;
+    return { states, legacyCandidates, warnings };
 }
 async function loadFailureHandoffs(repoRoot) {
     const handoffs = [];
@@ -858,7 +1252,9 @@ function mergeDagRun(existing, incoming) {
         return incoming;
     const startedAt = incoming.startedAt ?? existing.startedAt;
     const finishedAt = incoming.finishedAt ?? existing.finishedAt;
-    const ranks = incoming.ranks && incoming.ranks.length > 0 ? incoming.ranks : existing.ranks;
+    const ranks = incoming.ranks && incoming.ranks.length > 0
+        ? incoming.ranks
+        : existing.ranks;
     const durationMs = incoming.durationMs ??
         existing.durationMs ??
         computeDurationMs(startedAt, finishedAt);
@@ -877,7 +1273,9 @@ function mergeDagRun(existing, incoming) {
         ...(ranks && ranks.length > 0 ? { ranks } : {}),
         nodes: mergeDagNodes(existing.nodes, incoming.nodes),
         edges: incoming.edges.length > 0 ? incoming.edges : existing.edges,
-        ...(incoming.dagPath ?? existing.dagPath ? { dagPath: incoming.dagPath ?? existing.dagPath } : {}),
+        ...((incoming.dagPath ?? existing.dagPath)
+            ? { dagPath: incoming.dagPath ?? existing.dagPath }
+            : {}),
     };
 }
 function mergeDagNodes(existing, incoming) {
@@ -921,7 +1319,9 @@ function isFailedNodeStatus(status) {
     if (!status)
         return false;
     const normalized = status.toLowerCase();
-    return normalized === "error" || normalized === "failed" || normalized === "partial_failed";
+    return (normalized === "error" ||
+        normalized === "failed" ||
+        normalized === "partial_failed");
 }
 function nodeOutputPreview(node) {
     const stdout = readString(node, "stdout") ?? readString(node, "assistantText");
@@ -997,13 +1397,19 @@ async function loadDagEventFiles(repoRoot) {
                 const outputPreview = readString(parsed, "outputPreview");
                 nodeMap.set(nodeId, {
                     nodeId,
-                    ...(rank ?? prev.rank ? { rank: rank ?? prev.rank } : {}),
-                    ...(executor ?? prev.executor ? { executor: executor ?? prev.executor } : {}),
-                    ...(model ?? prev.model ? { model: model ?? prev.model } : {}),
-                    ...(label ?? prev.label ? { label: label ?? prev.label } : {}),
-                    ...(status ?? prev.status ? { status: status ?? prev.status } : {}),
-                    ...(durationMs ?? prev.durationMs ? { durationMs: durationMs ?? prev.durationMs } : {}),
-                    ...(outputPreview ?? prev.outputPreview ? { outputPreview: outputPreview ?? prev.outputPreview } : {}),
+                    ...((rank ?? prev.rank) ? { rank: rank ?? prev.rank } : {}),
+                    ...((executor ?? prev.executor)
+                        ? { executor: executor ?? prev.executor }
+                        : {}),
+                    ...((model ?? prev.model) ? { model: model ?? prev.model } : {}),
+                    ...((label ?? prev.label) ? { label: label ?? prev.label } : {}),
+                    ...((status ?? prev.status) ? { status: status ?? prev.status } : {}),
+                    ...((durationMs ?? prev.durationMs)
+                        ? { durationMs: durationMs ?? prev.durationMs }
+                        : {}),
+                    ...((outputPreview ?? prev.outputPreview)
+                        ? { outputPreview: outputPreview ?? prev.outputPreview }
+                        : {}),
                 });
                 byId.set(dagRunId, nodeMap);
             }
@@ -1059,7 +1465,9 @@ async function parseDagStateFile(statePath, now) {
             return undefined;
         const runDir = path.dirname(statePath);
         const lifecycleName = path.basename(path.dirname(runDir));
-        const lifecycle = lifecycleName === "active" || lifecycleName === "paused" || lifecycleName === "completed"
+        const lifecycle = lifecycleName === "active" ||
+            lifecycleName === "paused" ||
+            lifecycleName === "completed"
             ? lifecycleName
             : undefined;
         const dagRunId = readString(parsed, "runId") ?? path.basename(runDir);
@@ -1077,14 +1485,29 @@ async function parseDagStateFile(statePath, now) {
         const state = parsed;
         const liveness = assessDagRunLiveness({ state, now });
         const effectiveStatus = lifecycle
-            ? deriveDagRunEffectiveStatus({ lifecycle, state, liveness: liveness.status })
+            ? deriveDagRunEffectiveStatus({
+                lifecycle,
+                state,
+                liveness: liveness.status,
+            })
             : "unknown";
         const stateConsistent = lifecycle === undefined
             ? false
-            : !((lifecycle === "paused" && state.status !== "paused")
-                || (lifecycle === "completed" && !["finished", "failed", "partial_failed", "superseded", "abandoned"].includes(state.status)));
+            : !((lifecycle === "paused" && state.status !== "paused") ||
+                (lifecycle === "completed" &&
+                    ![
+                        "finished",
+                        "failed",
+                        "partial_failed",
+                        "superseded",
+                        "abandoned",
+                    ].includes(state.status)));
         const recoveryEligibility = lifecycle
-            ? assessDagRunRecoveryEligibility({ lifecycle, state, liveness: liveness.status })
+            ? assessDagRunRecoveryEligibility({
+                lifecycle,
+                state,
+                liveness: liveness.status,
+            })
             : undefined;
         return {
             dagRunId,
@@ -1122,7 +1545,9 @@ async function parseDagEdges(runPath, nodes) {
         if (!taskId || !knownNodeIds.has(taskId))
             continue;
         const dependencies = readStringArray(task, "depends_on");
-        const compatibleDependencies = dependencies.length > 0 ? dependencies : readStringArray(task, "dependsOn");
+        const compatibleDependencies = dependencies.length > 0
+            ? dependencies
+            : readStringArray(task, "dependsOn");
         for (const dependencyId of compatibleDependencies) {
             if (dependencyId === taskId || !knownNodeIds.has(dependencyId))
                 continue;
@@ -1169,7 +1594,10 @@ async function loadDagNodeModels(runPath) {
         const legacy = raw;
         const tasks = legacy.tasks;
         const executorModels = legacy.executorModels;
-        if (!Array.isArray(tasks) || typeof executorModels !== "object" || executorModels === null || Array.isArray(executorModels)) {
+        if (!Array.isArray(tasks) ||
+            typeof executorModels !== "object" ||
+            executorModels === null ||
+            Array.isArray(executorModels)) {
             return models;
         }
         for (const task of tasks) {
@@ -1194,7 +1622,9 @@ async function loadDagNodeModels(runPath) {
 function parseDagNodes(parsed, rankByNode, modelByNode) {
     const nodes = [];
     const nodesValue = parsed.nodes;
-    if (nodesValue && typeof nodesValue === "object" && !Array.isArray(nodesValue)) {
+    if (nodesValue &&
+        typeof nodesValue === "object" &&
+        !Array.isArray(nodesValue)) {
         for (const [nodeId, nodeVal] of Object.entries(nodesValue)) {
             if (!nodeVal || typeof nodeVal !== "object" || Array.isArray(nodeVal))
                 continue;
@@ -1261,7 +1691,9 @@ async function safeReadJson(filePath) {
 function safeParseJson(raw) {
     try {
         const parsed = JSON.parse(raw);
-        if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+        if (parsed === null ||
+            typeof parsed !== "object" ||
+            Array.isArray(parsed)) {
             return undefined;
         }
         return parsed;
@@ -1304,7 +1736,9 @@ function readNumber(value, key) {
     if (!value || typeof value !== "object")
         return undefined;
     const child = value[key];
-    return typeof child === "number" && Number.isFinite(child) ? child : undefined;
+    return typeof child === "number" && Number.isFinite(child)
+        ? child
+        : undefined;
 }
 function readStringMatrix(value, key) {
     if (!value || typeof value !== "object")