@dorokuma/herdsman 0.12.1 → 0.13.2

package/dist/src/observability/agent-index-service.js

@@ -6,7 +6,7 @@ import { runSqliteTransaction } from "../db/client.js";
 import { normalizeHerdrSessionSnapshot } from "../herdr/session-snapshot.js";
 import { HerdrSocketClient } from "../herdr/socket-client.js";
 import { AgentContextService } from "../observability/agent-context-service.js";
-import { parseAgentStatus, } from "../observability/contracts.js";
+import { parseAgentStatus, VALID_AGENT_EVENT_TYPES, } from "../observability/contracts.js";
 import { TURN_SIGNAL_WAIT_MS } from "../observability/turn-completion.js";
 export function historyHasAdvanced(current, baseline, options) {
     if (current.lastAssistantMessage?.ref &&
@@ -33,6 +33,17 @@ export class PlanWaitingHistoryError extends Error {
         this.name = "PlanWaitingHistoryError";
     }
 }
+/**
+ * Invariant for the waiting-history retry ring: only a genuine
+ * `PlanWaitingHistoryError` may (re)arm the 10s timer. It is the single signal
+ * that the agent history legitimately has not advanced yet. Synthetic retry
+ * reasons such as `degraded` are still recorded on the row, but they never
+ * authorize another spin: the pending row is drained by the next refresh
+ * instead, so a degraded round cannot keep the ring alive on its own.
+ */
+function retryRingAuthorized(error) {
+    return error instanceof PlanWaitingHistoryError;
+}
 /**
  * Sentinel returned by #appendStatusEvents when the plan must be CANCELLED
  * rather than skipped (skip still maps to completed in #runPlanRow). The agent
@@ -51,18 +62,23 @@ export class AgentIndexService {
     #refreshInFlightBySession = new Map();
     #scheduleRetry;
     #sessionOperationTail = new Map();
+    #shutdownSignal;
     #sleep;
     #turnCompletions;
     #waitingHistoryTimers = new Map();
     #now;
+    #onAgentEvent;
     constructor(options) {
         this.#clearRetry = options.clearRetry ?? ((timer) => clearTimeout(timer));
         this.#clientFactory = options.clientFactory ?? ((input) => new HerdrSocketClient(input));
         this.#scheduleRetry = options.scheduleRetry ?? ((cb, delay) => setTimeout(cb, delay));
         this.#sleep = options.sleep ?? sleep;
+        this.#shutdownSignal = options.shutdownSignal;
+        this.#shutdownSignal?.addEventListener("abort", () => this.abortPendingWaits(), { once: true });
         this.#now = options.now ?? (() => Date.now());
         this.#stores = options.stores;
         this.#turnCompletions = options.turnCompletions;
+        this.#onAgentEvent = options.onAgentEvent ?? (() => { });
         if (options.context) {
             this.#context = options.context;
         }
@@ -133,6 +149,21 @@ export class AgentIndexService {
         }
         this.#waitingHistoryTimers.clear();
     }
+    /**
+     * Aborts every in-flight status wait (history advance, turn completion,
+     * readiness polling). Called on shutdown through the shutdown signal; the
+     * waiters treat an abort exactly like a closed pane, so no event is appended
+     * for a half-observed transition and the plan row stays retryable.
+     */
+    abortPendingWaits() {
+        for (const [key, controllers] of this.#activeWaiters) {
+            for (const controller of controllers) {
+                controller.abort();
+            }
+            controllers.clear();
+            this.#activeWaiters.delete(key);
+        }
+    }
     async drainInFlightPlans() {
         const tails = Array.from(this.#planTailByAgent.values());
         await Promise.allSettled(tails);
@@ -159,7 +190,29 @@ export class AgentIndexService {
                 await this.#enqueueAgentPlan(plan.agent.id, () => this.#retryWaitingPlanRow(planId, plan));
             }
             catch (error) {
-                console.debug("Herdsman waiting history retry callback error", error);
+                // A failed retry callback used to be a silent `console.debug`; the row it
+                // was supposed to advance can then sit in `pending` forever with no
+                // signal at all, so the failure is reported as a warning with the row
+                // identity attached.
+                console.warn("Herdsman waiting history retry callback failed", {
+                    planId,
+                    agentId: plan.agent.id,
+                    herdrSessionName: plan.agent.herdrSessionName,
+                    paneId: plan.agent.paneId,
+                    error,
+                });
+            }
+            finally {
+                const store = this.#stores.statusEventPlans;
+                if (store) {
+                    const current = store.get(planId);
+                    if (current.status === "completed" ||
+                        current.status === "cancelled" ||
+                        current.status === "failed" ||
+                        current.status === "discarded") {
+                        this.#clearWaitingTimer(planId);
+                    }
+                }
             }
         }, 10_000);
         this.#waitingHistoryTimers.set(planId, timer);
@@ -195,13 +248,20 @@ export class AgentIndexService {
                 agentId: plan.agent.id,
                 error: err,
             });
-            const updated = store.markRetry(planId, new PlanWaitingHistoryError());
+            const currentPlan = store.get(planId);
+            const retryError = currentPlan.lastError === "degraded"
+                ? new Error("degraded")
+                : new PlanWaitingHistoryError();
+            const updated = store.markRetry(planId, retryError);
             if (!updated) {
                 this.#clearWaitingTimer(planId);
                 return;
             }
             if (updated.status === "pending") {
-                this.#scheduleWaitingHistoryRetry(planId, plan);
+                // Invariant: a synthetic retry reason never re-arms the ring (see
+                // retryRingAuthorized); the pending row waits for the next refresh.
+                if (retryRingAuthorized(retryError))
+                    this.#scheduleWaitingHistoryRetry(planId, plan);
             }
             else if (updated.status === "discarded") {
                 this.#clearWaitingTimer(planId);
@@ -218,6 +278,7 @@ export class AgentIndexService {
                     attempts: updated.attempts,
                     compactHistory: plan.compactHistory,
                     from: plan.from,
+                    planCreatedAt: currentPlan.createdAt.getTime(),
                     planId,
                     reason: updated.lastError,
                     to: plan.to,
@@ -233,19 +294,26 @@ export class AgentIndexService {
                     to: plan.to,
                     herdrSessionName: plan.agent.herdrSessionName,
                 });
-                this.#appendPlanFailedEvent({
+                const failedEvent = this.#appendPlanFailedEvent({
                     agent: plan.agent,
                     attempts: updated.attempts,
                     compactHistory: plan.compactHistory,
                     from: plan.from,
+                    planCreatedAt: currentPlan.createdAt.getTime(),
                     planId,
                     reason: updated.lastError,
                     to: plan.to,
                 });
+                if (VALID_AGENT_EVENT_TYPES.has(failedEvent.type)) {
+                    this.#onAgentEvent?.(failedEvent);
+                }
             }
             return;
         }
-        await this.#runPlanRow(current, retryPlan);
+        const event = await this.#runPlanRow(current, retryPlan);
+        if (event && VALID_AGENT_EVENT_TYPES.has(event.type)) {
+            this.#onAgentEvent?.(event);
+        }
     }
     async executeStatusEventPlan(plan) {
         if (plan.from === plan.to)
@@ -273,7 +341,14 @@ export class AgentIndexService {
                 to: plan.to,
                 ...(plan.compactHistory ? { compactHistory: plan.compactHistory } : {}),
             });
-        return this.#enqueueAgentPlan(plan.agent.id, () => this.#runPlanRow(inserted, plan));
+        const result = await this.#enqueueAgentPlan(plan.agent.id, () => this.#runPlanRow(inserted, plan));
+        // Publication stays with the caller: every caller forwards this return value
+        // (`HerdrSessionWatchManager#submitPlan` -> `publishAgentEvent`, or the
+        // non-fast refresh/handle wrappers putting it into `events`). Pushing here as
+        // well would write the same `agent.event` twice on the socket, breaking the
+        // "exactly once" delivery contract. The retry-exhausted failed branch in
+        // `#retryWaitingPlanRow` keeps its own push because no caller sees its result.
+        return result;
     }
     async drainPendingPlans() {
         if (!this.#stores.statusEventPlans)
@@ -304,8 +379,18 @@ export class AgentIndexService {
             }
         });
         // Individual rows may reject (after markRetry) but the drain itself never
-        // rejects: every row is either drained, cancelled, or retried.
-        await Promise.allSettled(tasks);
+        // rejects: every row is either drained, cancelled, or retried. A rejected
+        // row is still an anomaly (its own drain path threw), so it is reported
+        // instead of being silently folded into the settled results.
+        const results = await Promise.allSettled(tasks);
+        const rejected = results.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
+        if (rejected.length > 0) {
+            console.error("Herdsman status event plan drain rejected rows", {
+                rejected: rejected.length,
+                total: results.length,
+                errors: rejected.map((reason) => reason instanceof Error ? reason.message : String(reason)),
+            });
+        }
         await this.#backfillFailedPlanEvents();
         await this.#backfillDiscardedPlanEvents();
     }
@@ -313,37 +398,62 @@ export class AgentIndexService {
         const store = this.#stores.statusEventPlans;
         if (!store)
             return;
+        let backfilled = 0;
+        let skipped = 0;
         for (const row of store.listFailed()) {
-            if (row.lastError === PLAN_WAITING_HISTORY)
-                continue;
-            const existing = this.#stores.sqlite
-                .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
-                .get(row.herdrSessionName, `agent.failed:plan:${row.id}`);
-            if (existing)
-                continue;
-            let agent;
+            // Per-row isolation: one unresolvable or unwritable row must not abort the
+            // whole backfill round. The legacy loop had no guard, so the first
+            // `FOREIGN KEY constraint failed` threw out of `drainPendingPlans` and no
+            // later row was ever attempted.
             try {
-                agent = this.#stores.agents.get(row.agentId);
-            }
-            catch {
-                agent = this.#minimalAgentFromLatestEvent(row.agentId, row.herdrSessionName);
+                if (row.lastError === PLAN_WAITING_HISTORY) {
+                    skipped += 1;
+                    continue;
+                }
+                const existing = this.#stores.sqlite
+                    .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
+                    .get(row.herdrSessionName, `agent.failed:plan:${row.id}`);
+                if (existing) {
+                    skipped += 1;
+                    continue;
+                }
+                const agent = this.#resolveBackfillAgent(row);
                 if (!agent) {
-                    console.warn("Herdsman skipping agent.failed backfill; no agent row or events", {
-                        agentId: row.agentId,
-                        herdrSessionName: row.herdrSessionName,
-                        planId: row.id,
-                    });
+                    skipped += 1;
                     continue;
                 }
+                const event = this.#appendPlanFailedEvent({
+                    agent,
+                    attempts: row.attempts,
+                    compactHistory: row.compactHistory,
+                    from: row.fromStatus,
+                    planCreatedAt: row.createdAt.getTime(),
+                    planId: row.id,
+                    reason: row.lastError,
+                    to: row.toStatus,
+                });
+                // Stock terminal results are ingested silently: the row is written, then
+                // immediately acknowledged/non-deliverable, so it stays queryable without
+                // waking the current owner with a long-past failure.
+                this.#stores.agentEvents.markAcked(event.id);
+                backfilled += 1;
             }
-            this.#appendPlanFailedEvent({
-                agent,
-                attempts: row.attempts,
-                compactHistory: row.compactHistory,
-                from: row.fromStatus,
-                planId: row.id,
-                reason: row.lastError,
-                to: row.toStatus,
+            catch (error) {
+                skipped += 1;
+                console.warn("Herdsman skipping agent.failed backfill row", {
+                    agentId: row.agentId,
+                    herdrSessionName: row.herdrSessionName,
+                    paneId: row.paneId,
+                    planId: row.id,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        if (backfilled > 0 || skipped > 0) {
+            console.info("Herdsman backfilled agent.failed rows", {
+                backfilled,
+                skipped,
+                total: backfilled + skipped,
             });
         }
     }
@@ -351,43 +461,110 @@ export class AgentIndexService {
         const store = this.#stores.statusEventPlans;
         if (!store)
             return;
+        let backfilled = 0;
+        let skipped = 0;
         for (const row of store.listDiscarded()) {
-            const existing = this.#stores.sqlite
-                .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
-                .get(row.herdrSessionName, `agent.discarded:plan:${row.id}`);
-            if (existing)
-                continue;
-            let agent;
             try {
-                agent = this.#stores.agents.get(row.agentId);
-            }
-            catch {
-                agent = this.#minimalAgentFromLatestEvent(row.agentId, row.herdrSessionName);
+                const existing = this.#stores.sqlite
+                    .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
+                    .get(row.herdrSessionName, `agent.discarded:plan:${row.id}`);
+                if (existing) {
+                    skipped += 1;
+                    continue;
+                }
+                const agent = this.#resolveBackfillAgent(row);
                 if (!agent) {
-                    console.warn("Herdsman skipping agent.discarded backfill; no agent row or events", {
-                        agentId: row.agentId,
-                        herdrSessionName: row.herdrSessionName,
-                        planId: row.id,
-                    });
+                    skipped += 1;
                     continue;
                 }
+                const event = this.#appendPlanDiscardedEvent({
+                    agent,
+                    attempts: row.attempts,
+                    compactHistory: row.compactHistory,
+                    from: row.fromStatus,
+                    planCreatedAt: row.createdAt.getTime(),
+                    planId: row.id,
+                    reason: row.lastError,
+                    to: row.toStatus,
+                });
+                // Same silent-ingest contract as the failed backfill.
+                this.#stores.agentEvents.markAcked(event.id);
+                backfilled += 1;
             }
-            this.#appendPlanDiscardedEvent({
-                agent,
-                attempts: row.attempts,
-                compactHistory: row.compactHistory,
-                from: row.fromStatus,
-                planId: row.id,
-                reason: row.lastError,
-                to: row.toStatus,
+            catch (error) {
+                skipped += 1;
+                console.warn("Herdsman skipping agent.discarded backfill row", {
+                    agentId: row.agentId,
+                    herdrSessionName: row.herdrSessionName,
+                    paneId: row.paneId,
+                    planId: row.id,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        if (backfilled > 0 || skipped > 0) {
+            console.info("Herdsman backfilled agent.discarded rows", {
+                backfilled,
+                skipped,
+                total: backfilled + skipped,
             });
         }
     }
-    #minimalAgentFromLatestEvent(agentId, herdrSessionName) {
+    /**
+     * Resolves the metadata a terminal plan outcome may use to reach an
+     * orchestrator: the live `agents` row when it still exists, otherwise a
+     * minimal record stitched from the pane's own event history.
+     */
+    #resolveBackfillAgent(row) {
+        try {
+            return this.#stores.agents.get(row.agentId);
+        }
+        catch {
+            const stitched = this.#minimalAgentFromLatestEvent({
+                agentId: row.agentId,
+                createdBefore: row.createdAt.getTime(),
+                herdrSessionName: row.herdrSessionName,
+                paneGeneration: row.paneGeneration,
+                paneId: row.paneId,
+            });
+            if (!stitched) {
+                console.warn("Herdsman skipping terminal event backfill; no agent row or pane events", {
+                    agentId: row.agentId,
+                    herdrSessionName: row.herdrSessionName,
+                    paneId: row.paneId,
+                    planId: row.id,
+                });
+            }
+            return stitched;
+        }
+    }
+    /**
+     * Reconstructs the minimum agent metadata a terminal event needs from the
+     * pane's own event history.
+     *
+     * The lookup is keyed by `(herdr_session_name, pane_id)` plus the plan's
+     * `pane_generation`, and is bounded by `createdBefore` (the plan row's
+     * `created_at`). Matching on `agent_id` alone cannot work here: physical
+     * deletion of the `agents` row runs `ON DELETE SET NULL` over every event, so
+     * the legacy stitch always came back empty, and a pane-id-only match would
+     * happily stitch an old failure onto a newer instance of the same pane. The
+     * generation and time bounds are what keep the two apart.
+     */
+    #minimalAgentFromLatestEvent(input) {
+        const generationClause = input.paneGeneration === null ? "pane_generation is null" : "pane_generation = ?";
+        const params = [
+            input.herdrSessionName,
+            input.paneId,
+            input.createdBefore,
+        ];
+        if (input.paneGeneration !== null)
+            params.push(input.paneGeneration);
         const latest = this.#stores.sqlite
             .prepare(`select pane_id, pane_generation, terminal_id, workspace_id, herdr_session_name
-         from agent_events where agent_id = ? order by id desc limit 1`)
-            .get(agentId);
+         from agent_events
+         where herdr_session_name = ? and pane_id = ? and created_at <= ? and ${generationClause}
+         order by id desc limit 1`)
+            .get(...params);
         if (!latest?.pane_id || !latest.workspace_id)
             return undefined;
         return {
@@ -398,8 +575,8 @@ export class AgentIndexService {
             firstSeenAt: new Date(0),
             focused: false,
             foregroundCwd: null,
-            herdrSessionName: latest.herdr_session_name || herdrSessionName,
-            id: agentId,
+            herdrSessionName: latest.herdr_session_name || input.herdrSessionName,
+            id: input.agentId,
             lastSeenAt: new Date(0),
             name: null,
             paneId: latest.pane_id,
@@ -416,16 +593,18 @@ export class AgentIndexService {
             return;
         let current;
         try {
-            current = store.get(row.id);
+            current = runSqliteTransaction(this.#stores.sqlite, () => {
+                const candidate = store.get(row.id);
+                if (candidate.status !== "pending")
+                    return null;
+                return candidate;
+            });
         }
         catch {
             this.#clearWaitingTimer(row.id);
             return;
         }
-        if (current.status === "completed" ||
-            current.status === "cancelled" ||
-            current.status === "failed" ||
-            current.status === "discarded") {
+        if (!current) {
             this.#clearWaitingTimer(row.id);
             return;
         }
@@ -462,7 +641,8 @@ export class AgentIndexService {
             activeCompact = refreshed.snapshot.compactHistory;
         }
         catch (err) {
-            const updated = store.markRetry(current.id, new PlanWaitingHistoryError());
+            const retryError = current.lastError === "degraded" ? new Error("degraded") : new PlanWaitingHistoryError();
+            const updated = store.markRetry(current.id, retryError);
             if (!updated) {
                 this.#clearWaitingTimer(current.id);
                 return;
@@ -473,13 +653,18 @@ export class AgentIndexService {
                     agentId: agent.id,
                     error: err,
                 });
-                this.#scheduleWaitingHistoryRetry(current.id, {
-                    agent,
-                    compactHistory: current.compactHistory,
-                    from: current.fromStatus,
-                    to: current.toStatus,
-                    ...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
-                });
+                // Invariant: the ring is only re-armed by a genuine
+                // PlanWaitingHistoryError; a degraded row stays pending for the next
+                // refresh to drain instead of spinning this callback again.
+                if (retryRingAuthorized(retryError)) {
+                    this.#scheduleWaitingHistoryRetry(current.id, {
+                        agent,
+                        compactHistory: current.compactHistory,
+                        from: current.fromStatus,
+                        to: current.toStatus,
+                        ...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
+                    });
+                }
             }
             else if (updated.status === "discarded") {
                 this.#clearWaitingTimer(current.id);
@@ -495,6 +680,7 @@ export class AgentIndexService {
                     attempts: updated.attempts,
                     compactHistory: current.compactHistory,
                     from: current.fromStatus,
+                    planCreatedAt: current.createdAt.getTime(),
                     planId: current.id,
                     reason: updated.lastError,
                     to: current.toStatus,
@@ -509,15 +695,19 @@ export class AgentIndexService {
                     to: current.toStatus,
                     reason: updated.lastError,
                 });
-                this.#appendPlanFailedEvent({
+                const failedEvent = this.#appendPlanFailedEvent({
                     agent,
                     attempts: updated.attempts,
                     compactHistory: current.compactHistory,
                     from: current.fromStatus,
+                    planCreatedAt: current.createdAt.getTime(),
                     planId: current.id,
                     reason: updated.lastError,
                     to: current.toStatus,
                 });
+                if (VALID_AGENT_EVENT_TYPES.has(failedEvent.type)) {
+                    this.#onAgentEvent?.(failedEvent);
+                }
             }
             return;
         }
@@ -529,7 +719,10 @@ export class AgentIndexService {
             to: current.toStatus,
             ...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
         };
-        await this.#runPlanRow(current, plan);
+        const event = await this.#runPlanRow(current, plan);
+        if (event && VALID_AGENT_EVENT_TYPES.has(event.type)) {
+            this.#onAgentEvent?.(event);
+        }
     }
     #enqueueAgentPlan(agentId, operation) {
         const previous = this.#planTailByAgent.get(agentId) ?? Promise.resolve();
@@ -555,10 +748,16 @@ export class AgentIndexService {
             this.#clearWaitingTimer(row.id);
             return undefined;
         }
-        store.markRunning(row.id);
+        if (!store.markRunning(row.id)) {
+            this.#clearWaitingTimer(row.id);
+            return undefined;
+        }
         let activePlan = {
             ...plan,
             attempts: row.attempts,
+            // Row correlation for observability (see `#logDegradedRelease`): the plan
+            // id is only known here, where the store row is in hand.
+            planId: row.id,
         };
         if (row.attempts > 0) {
             try {
@@ -571,6 +770,7 @@ export class AgentIndexService {
                     ...plan,
                     attempts: row.attempts,
                     compactHistory: refreshed.snapshot.compactHistory,
+                    planId: row.id,
                 };
             }
             catch (err) {
@@ -579,13 +779,22 @@ export class AgentIndexService {
                     agentId: plan.agent.id,
                     error: err,
                 });
-                const updated = store.markRetry(row.id, new PlanWaitingHistoryError());
+                // Keep the degraded reason when the plan was already degraded by an
+                // earlier round: markRetry decides discarded (silent) vs failed (wakes)
+                // from this message, so passing PLAN_WAITING_HISTORY here would let a
+                // degraded plan exhaust silently.
+                const retryError = row.lastError === "degraded" ? new Error("degraded") : new PlanWaitingHistoryError();
+                const updated = store.markRetry(row.id, retryError);
                 if (!updated) {
                     this.#clearWaitingTimer(row.id);
                     return undefined;
                 }
                 if (updated.status === "pending") {
-                    this.#scheduleWaitingHistoryRetry(row.id, plan);
+                    // Invariant: only a genuine PlanWaitingHistoryError authorizes the
+                    // ring; a degraded pre-refresh round records the synthetic reason and
+                    // waits for the next refresh instead.
+                    if (retryRingAuthorized(retryError))
+                        this.#scheduleWaitingHistoryRetry(row.id, plan);
                 }
                 else {
                     this.#clearWaitingTimer(row.id);
@@ -603,6 +812,7 @@ export class AgentIndexService {
                             attempts: updated.attempts,
                             compactHistory: plan.compactHistory,
                             from: row.fromStatus,
+                            planCreatedAt: row.createdAt.getTime(),
                             planId: row.id,
                             reason: updated.lastError,
                             to: row.toStatus,
@@ -622,6 +832,7 @@ export class AgentIndexService {
                             attempts: updated.attempts,
                             compactHistory: plan.compactHistory,
                             from: row.fromStatus,
+                            planCreatedAt: row.createdAt.getTime(),
                             planId: row.id,
                             reason: updated.lastError,
                             to: row.toStatus,
@@ -655,12 +866,22 @@ export class AgentIndexService {
                 eventPayload !== null &&
                 eventPayload.degraded === true;
             if (isDegraded) {
-                const updated = store.markRetry(row.id, new Error("degraded"));
+                if (event) {
+                    this.#stores.agentEvents.invalidateById(event.id, "degraded_retry");
+                    Object.assign(event, { status: "invalidated", deliverable: 0 });
+                }
+                const degradedError = new Error("degraded");
+                const updated = store.markRetry(row.id, degradedError);
                 if (!updated) {
-                    return event;
+                    return undefined;
                 }
                 if (updated.status === "pending") {
-                    this.#scheduleWaitingHistoryRetry(row.id, plan);
+                    // Invariant: `degraded` is a synthetic reason, so it never authorizes
+                    // the retry ring; the row stays pending and the next refresh
+                    // re-drains it.
+                    if (retryRingAuthorized(degradedError))
+                        this.#scheduleWaitingHistoryRetry(row.id, plan);
+                    return undefined;
                 }
                 else {
                     this.#clearWaitingTimer(row.id);
@@ -670,6 +891,7 @@ export class AgentIndexService {
                             attempts: updated.attempts,
                             compactHistory: activePlan.compactHistory,
                             from: row.fromStatus,
+                            planCreatedAt: row.createdAt.getTime(),
                             planId: row.id,
                             reason: updated.lastError,
                             to: row.toStatus,
@@ -681,26 +903,34 @@ export class AgentIndexService {
                             attempts: updated.attempts,
                             compactHistory: activePlan.compactHistory,
                             from: row.fromStatus,
+                            planCreatedAt: row.createdAt.getTime(),
                             planId: row.id,
                             reason: updated.lastError,
                             to: row.toStatus,
                         });
                     }
                 }
-                return event;
+                return undefined;
             }
             store.markCompleted(row.id);
             return event;
         }
         catch (error) {
             if (error instanceof PlanWaitingHistoryError) {
-                const updated = store.markRetry(row.id, error);
+                const retryError = row.lastError === "degraded" ? new Error("degraded") : error;
+                const updated = store.markRetry(row.id, retryError);
                 if (!updated) {
                     this.#clearWaitingTimer(row.id);
                     return undefined;
                 }
                 if (updated.status === "pending") {
-                    this.#scheduleWaitingHistoryRetry(row.id, plan);
+                    // Invariant: only a genuine PlanWaitingHistoryError authorizes the
+                    // ring. A row that already carries the synthetic `degraded` reason is
+                    // recorded as degraded again and waits for the next refresh instead of
+                    // re-arming the 10s timer, so this catch cannot be used as a back door
+                    // that lets a degraded round keep the ring alive.
+                    if (retryRingAuthorized(retryError))
+                        this.#scheduleWaitingHistoryRetry(row.id, plan);
                 }
                 else {
                     this.#clearWaitingTimer(row.id);
@@ -718,6 +948,7 @@ export class AgentIndexService {
                             attempts: updated.attempts,
                             compactHistory: activePlan.compactHistory,
                             from: row.fromStatus,
+                            planCreatedAt: row.createdAt.getTime(),
                             planId: row.id,
                             reason: updated.lastError,
                             to: row.toStatus,
@@ -737,6 +968,7 @@ export class AgentIndexService {
                             attempts: updated.attempts,
                             compactHistory: activePlan.compactHistory,
                             from: row.fromStatus,
+                            planCreatedAt: row.createdAt.getTime(),
                             planId: row.id,
                             reason: updated.lastError,
                             to: row.toStatus,
@@ -759,10 +991,15 @@ export class AgentIndexService {
                 return undefined;
             }
             this.#clearWaitingTimer(row.id);
-            const updated = store.markRetry(row.id, error);
+            const retryError = row.lastError === "degraded" ? new Error("degraded") : error;
+            const updated = store.markRetry(row.id, retryError);
             if (!updated) {
                 return undefined;
             }
+            // Invariant: only a genuine PlanWaitingHistoryError re-arms the retry ring,
+            // and that case was handled above. On this generic path `retryError` is
+            // either the synthetic `degraded` reason or an unrelated failure, so no
+            // spin is scheduled: a still-pending row is drained by the next refresh.
             if (updated.status === "discarded") {
                 console.info("Herdsman status event plan discarded after max attempts", {
                     planId: row.id,
@@ -777,6 +1014,7 @@ export class AgentIndexService {
                     attempts: updated.attempts,
                     compactHistory: activePlan.compactHistory,
                     from: row.fromStatus,
+                    planCreatedAt: row.createdAt.getTime(),
                     planId: row.id,
                     reason: updated.lastError,
                     to: row.toStatus,
@@ -796,6 +1034,7 @@ export class AgentIndexService {
                     attempts: updated.attempts,
                     compactHistory: activePlan.compactHistory,
                     from: row.fromStatus,
+                    planCreatedAt: row.createdAt.getTime(),
                     planId: row.id,
                     reason: updated.lastError,
                     to: row.toStatus,
@@ -806,10 +1045,12 @@ export class AgentIndexService {
     }
     async #waitForHistoryAdvance(input) {
         const maxAttempts = input.maxAttempts ?? 8;
-        const baseDelay = 500;
-        const maxDelay = 16000;
-        const maxTotalMs = 30000;
-        let delay = baseDelay;
+        // Fixed 200ms interval, no exponential backoff: the history cache either
+        // advances shortly after the turn signal or it does not, and a doubling
+        // delay only pushed the first deliverable terminal event tens of seconds
+        // away. The whole window is hard bounded by maxTotalMs.
+        const baseDelay = 200;
+        const maxTotalMs = 1500;
         let refreshed = input.initial
             ? { snapshot: { compactHistory: input.initial } }
             : await this.#context.refreshAgent({
@@ -824,7 +1065,13 @@ export class AgentIndexService {
             }) &&
             !input.controller.signal.aborted &&
             this.#now() - start < maxTotalMs; attempt += 1) {
-            await this.#sleep(delay, input.controller.signal);
+            // Clamp the last step to the remaining budget so the total wait never
+            // exceeds maxTotalMs even though the loop condition is checked before
+            // sleeping.
+            const remaining = maxTotalMs - (this.#now() - start);
+            if (remaining <= 0)
+                break;
+            await this.#sleep(Math.min(baseDelay, remaining), input.controller.signal);
             if (input.controller.signal.aborted) {
                 console.debug("Herdsman retrying status event plan because wait was aborted during history refresh", {
                     agentId: input.agent.id,
@@ -850,7 +1097,6 @@ export class AgentIndexService {
                 forceRefresh: true,
                 identityChanged: false,
             });
-            delay = Math.min(delay * 2, maxDelay);
         }
         return refreshed.snapshot.compactHistory;
     }
@@ -1205,8 +1451,23 @@ export class AgentIndexService {
             paneGeneration: paneGenerationOf(agent),
         });
     }
+    /**
+     * Distinguishes the two reasons an in-flight wait can be aborted. A closed pane
+     * makes the transition moot, so the plan is skipped; a daemon shutdown leaves
+     * the pane alive, so the plan must stay retryable instead of being completed
+     * without ever emitting the transition (it is drained on the next start).
+     */
+    #throwIfShutdownAbort(controller) {
+        if (controller.signal.aborted && this.#shutdownSignal?.aborted) {
+            throw new PlanWaitingHistoryError();
+        }
+    }
     #registerActiveWaiter(agent, controller) {
         const key = `${agent.herdrSessionName}\0${agent.paneId}\0${agent.paneGeneration ?? ""}`;
+        // A wait that starts after shutdown already aborted must not sleep at all:
+        // abort it up front so the caller short-circuits like a closed pane.
+        if (this.#shutdownSignal?.aborted)
+            controller.abort();
         let set = this.#activeWaiters.get(key);
         if (!set) {
             set = new Set();
@@ -1234,10 +1495,36 @@ export class AgentIndexService {
             }
         }
     }
+    /**
+     * Observability for a degraded release. The emitted status event carries the
+     * specific `degradedReason`, but the plan row itself only stores the synthetic
+     * `degraded` marker (`row.lastError`), so without this line the cause is
+     * invisible in the daemon log. It cannot ride along in the failed-event payload
+     * either: for most append sites only `row.lastError` is at hand, so handing them
+     * the specific reason would mean persisting a new plan field (schema change).
+     * That is not universal — at least three call sites in this file already have the
+     * specific reason in their own call stack (`:1074` the status event payload's
+     * `degradedReason`, `:1136` the `PlanWaitingHistoryError`, `:1203` the generic
+     * `Error`), so routing the reason into the payload does not necessarily require a
+     * schema change. Log-only on purpose — no behaviour change (see
+     * .agents/notes/20260930-terminal-event-delivery-open-items.md A1).
+     *
+     * `plan.planId` is the row correlation that already exists on
+     * `StatusEventPlan` (see `executeStatusEventPlan`); it is `null` for the
+     * row-less in-memory plans executed without the SQLite store.
+     */
+    #logDegradedRelease(plan, degradedReason) {
+        console.warn("Herdsman emitted degraded status event", {
+            agentId: plan.agent.id,
+            attempts: plan.attempts ?? 0,
+            degradedReason,
+            herdrSessionName: plan.agent.herdrSessionName,
+            paneId: plan.agent.paneId,
+            planId: plan.planId ?? null,
+        });
+    }
     async #appendStatusEvents(input) {
         // Invariant: compactHistory is always populated as an object before calling #appendStatusEvents; !input.compactHistory check below is defensive and unreachable in normal operation.
-        if (input.from === input.to || !input.compactHistory)
-            return undefined;
         if (this.#stores.agents.isPaneClosed({
             herdrSessionName: input.agent.herdrSessionName,
             paneId: input.agent.paneId,
@@ -1266,7 +1553,7 @@ export class AgentIndexService {
                 skipStatusChanged = true;
             }
             if (hasTerminalAfter && !input.herdrEventKey) {
-                const isAgyHistory = input.agent.agent === "agy" || input.compactHistory.source === "antigravity-sqlite";
+                const isAgyHistory = input.agent.agent === "agy" || input.compactHistory?.source === "antigravity-sqlite";
                 // For an agy/antigravity pane only a terminal event that already
                 // represented a completed turn may count as the delivered baseline: an
                 // `unknown -> idle` startup row would otherwise make the `working -> done`
@@ -1325,6 +1612,7 @@ export class AgentIndexService {
                         herdrSessionName: input.agent.herdrSessionName,
                         paneId: input.agent.paneId,
                     });
+                    this.#throwIfShutdownAbort(controller);
                     return undefined;
                 }
                 if (turn?.received) {
@@ -1349,7 +1637,40 @@ export class AgentIndexService {
                     const expectedText = turn.expectedText?.trim();
                     const lastAssistantText = fresh.lastAssistantMessage?.text?.trim() ?? "";
                     const textMatches = !expectedText || lastAssistantText.endsWith(expectedText);
-                    if (!isAlreadyEmittedInRetry &&
+                    // A client-confirmed turn is a deliverable terminal state: `confirmed`
+                    // means Pi verified the final assistant message is on disk, so a
+                    // confirmed round is delivered straight through and is never marked
+                    // degraded (which is what made #runPlanRow run
+                    // invalidateById(..., "degraded_retry") on already-written content).
+                    const confirmedTerminal = turn.confirmed === true;
+                    const degradeOrRelease = (degradedReason) => {
+                        if (confirmedTerminal) {
+                            // A confirmed round stays non-degraded (see above), but the reason
+                            // is no longer swallowed: this path used to release silently with
+                            // an emptied body, so an empty `agent.done` could only be
+                            // diagnosed by reading the pane by hand. `degraded: true` must
+                            // stay off here — #runPlanRow would `invalidateById(...,
+                            // "degraded_retry")` the row it has already written.
+                            console.warn("Herdsman released a confirmed pi status event with no deliverable text", {
+                                agentId: input.agent.id,
+                                degradedReason,
+                                herdrSessionName: input.agent.herdrSessionName,
+                                paneId: input.agent.paneId,
+                                planId: input.planId ?? null,
+                                terminalId: input.agent.terminalId,
+                            });
+                            return { staleSnapshot: false };
+                        }
+                        this.#logDegradedRelease(input, degradedReason);
+                        return { degraded: true, degradedReason, staleSnapshot: false };
+                    };
+                    if (confirmedTerminal &&
+                        freshIsTerminal &&
+                        textMatches &&
+                        hasNonEmptyAssistantMessage(fresh)) {
+                        compactHistory = fresh;
+                    }
+                    else if (!isAlreadyEmittedInRetry &&
                         freshIsTerminal &&
                         textMatches &&
                         hasNonEmptyAssistantMessage(fresh) &&
@@ -1372,33 +1693,49 @@ export class AgentIndexService {
                             !historyHasAdvanced(advanced, baseline, { requireAssistantChange: true })) {
                             throw new PlanWaitingHistoryError();
                         }
+                        // `historyHasAdvanced(..., { requireAssistantChange: true })` compares
+                        // the assistant ref/text against the plan baseline. That baseline can
+                        // hold this round's answer (a plan row's `compact_history_json` is
+                        // written once at plan creation and never updated, and creation
+                        // measured 131ms/308ms after the final assistant message reached
+                        // disk). It can also hold the answer of a *previous* round — the same
+                        // frozen baseline is exactly what the daemon last delivered — so a
+                        // round that writes nothing new would otherwise re-release the
+                        // previous round's body as if it were this one's. `latestTerminal` is
+                        // the latest non-invalidated terminal row read before this round's row
+                        // is written: usually the last delivery, possibly an undelivered idle.
+                        const staleBaselineDuplicate = sameTerminalAssistantContent(advanced, latestTerminal?.compactHistory, "pi");
+                        // A confirmed turn carrying a non-empty terminal assistant message is
+                        // trusted as-is, so `no_advance_from_input` stays reserved for rounds
+                        // with genuinely no deliverable assistant text (empty or non-terminal
+                        // tail).
+                        const confirmedDeliverable = confirmedTerminal &&
+                            isTerminalAssistant(advanced) &&
+                            hasNonEmptyAssistantMessage(advanced) &&
+                            !staleBaselineDuplicate;
                         if (!isRetry &&
                             !historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
-                            compactHistory = { ...advanced, lastAssistantMessage: null };
-                            payloadExtra = {
-                                degraded: true,
-                                degradedReason: "no_advance_from_input",
-                                staleSnapshot: false,
-                            };
+                            if (confirmedDeliverable) {
+                                // Trust the already-in-place answer: release it as a non-stale
+                                // snapshot and without a `degraded` marker.
+                                compactHistory = advanced;
+                                payloadExtra = { staleSnapshot: false };
+                            }
+                            else {
+                                compactHistory = { ...advanced, lastAssistantMessage: null };
+                                payloadExtra = degradeOrRelease(staleBaselineDuplicate ? "stale_baseline_duplicate" : "no_advance_from_input");
+                            }
                         }
                         else {
                             const advancedText = advanced.lastAssistantMessage?.text?.trim() ?? "";
                             const advancedMatchesExpected = !expectedText || advancedText.endsWith(expectedText);
                             if (!advancedMatchesExpected) {
                                 compactHistory = { ...advanced, lastAssistantMessage: null };
-                                payloadExtra = {
-                                    degraded: true,
-                                    degradedReason: "expected_text_mismatch",
-                                    staleSnapshot: false,
-                                };
+                                payloadExtra = degradeOrRelease("expected_text_mismatch");
                             }
-                            else if (!isTerminalAssistant(advanced)) {
+                            else if (!isTerminalAssistant(advanced) || !hasNonEmptyAssistantMessage(advanced)) {
                                 compactHistory = { ...advanced, lastAssistantMessage: null };
-                                payloadExtra = {
-                                    degraded: true,
-                                    degradedReason: "non_terminal_assistant",
-                                    staleSnapshot: false,
-                                };
+                                payloadExtra = degradeOrRelease("non_terminal_assistant");
                             }
                             else {
                                 compactHistory = advanced;
@@ -1428,14 +1765,16 @@ export class AgentIndexService {
                     if (!isRetry &&
                         !historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
                         compactHistory = { ...advanced, lastAssistantMessage: null };
+                        this.#logDegradedRelease(input, "no_advance_from_input");
                         payloadExtra = {
                             degraded: true,
                             degradedReason: "no_advance_from_input",
                             staleSnapshot: false,
                         };
                     }
-                    else if (!isTerminalAssistant(advanced)) {
+                    else if (!isTerminalAssistant(advanced) || !hasNonEmptyAssistantMessage(advanced)) {
                         compactHistory = { ...advanced, lastAssistantMessage: null };
+                        this.#logDegradedRelease(input, "non_terminal_assistant");
                         payloadExtra = {
                             degraded: true,
                             degradedReason: "non_terminal_assistant",
@@ -1493,6 +1832,7 @@ export class AgentIndexService {
                     compactHistory?.source !== "antigravity-sqlite" &&
                     input.herdrEventKey &&
                     latestTerminalForSkip &&
+                    compactHistory &&
                     hasNonEmptyAssistantMessage(compactHistory)) {
                     const currentMsg = compactHistory.lastAssistantMessage;
                     const prevMsg = latestTerminalForSkip.compactHistory?.lastAssistantMessage;
@@ -1518,6 +1858,7 @@ export class AgentIndexService {
                                 herdrSessionName: input.agent.herdrSessionName,
                                 paneId: input.agent.paneId,
                             });
+                            this.#throwIfShutdownAbort(controller);
                             return undefined;
                         }
                         try {
@@ -1562,6 +1903,7 @@ export class AgentIndexService {
                     herdrSessionName: input.agent.herdrSessionName,
                     paneId: input.agent.paneId,
                 });
+                this.#throwIfShutdownAbort(controller);
                 return undefined;
             }
             const currentAgent = this.#stores.agents.findByPane({
@@ -1616,7 +1958,7 @@ export class AgentIndexService {
                 ? undefined
                 : this.#appendAndAckSelfEvent({
                     agentId: targetAgent.id,
-                    compactHistory,
+                    compactHistory: compactHistory ?? null,
                     herdrSessionName: targetAgent.herdrSessionName,
                     idempotencyKey: idempotencyKey("agent.status.changed", targetAgent, input.from, input.to, `${observationId}:${input.herdrEventKey ?? "legacy"}:agent.status.changed`),
                     paneId: targetAgent.paneId,
@@ -1630,7 +1972,7 @@ export class AgentIndexService {
             if (statusType) {
                 return this.#appendAndAckSelfEvent({
                     agentId: targetAgent.id,
-                    compactHistory,
+                    compactHistory: compactHistory ?? null,
                     herdrSessionName: targetAgent.herdrSessionName,
                     idempotencyKey: idempotencyKey(statusType, targetAgent, input.from, input.to, `${observationId}:${input.herdrEventKey ?? "legacy"}:${statusType}`),
                     paneId: targetAgent.paneId,
@@ -1647,18 +1989,42 @@ export class AgentIndexService {
             unregister();
         }
     }
-    #appendPlanFailedEvent(input) {
-        let agent = input.agent;
+    /**
+     * Resolves the metadata a terminal plan outcome is attributed to and whether
+     * the event may keep its `agent_id` foreign key.
+     *
+     * A live `agents` row wins because it carries the current pane generation.
+     * Once the row is gone (pane retired / physically deleted) the plan's own
+     * pane/session/generation are the authority, refined by the newest event of
+     * the same pane + generation that predates the plan, so an old failure is
+     * never stitched onto a newer instance of the same pane. The event is still
+     * written in that case, but as an orphan (`agentRowPresent: false`).
+     */
+    #resolvePlanOutcomeAgent(agent, planCreatedAt) {
         try {
-            agent = this.#stores.agents.get(input.agent.id);
+            return { agent: this.#stores.agents.get(agent.id), agentRowPresent: true };
         }
         catch {
-            const stitched = this.#minimalAgentFromLatestEvent(input.agent.id, input.agent.herdrSessionName);
-            if (stitched)
-                agent = stitched;
+            const stitched = this.#minimalAgentFromLatestEvent({
+                agentId: agent.id,
+                // The plan row's own creation time is the upper bound: anything newer on
+                // the same pane belongs to a later instance.
+                createdBefore: planCreatedAt ?? Date.now(),
+                herdrSessionName: agent.herdrSessionName,
+                paneGeneration: agent.paneGeneration ?? null,
+                paneId: agent.paneId,
+            });
+            return { agent: stitched ?? agent, agentRowPresent: false };
         }
+    }
+    #appendPlanFailedEvent(input) {
+        const originalAgentId = input.agent.id;
+        const resolved = this.#resolvePlanOutcomeAgent(input.agent, input.planCreatedAt);
+        const agent = resolved.agent;
+        const fallbackOutcome = input.fallbackOutcome ??
+            ((input.to === "done" || input.to === "blocked") && input.reason === "degraded");
         return this.#appendAndAckSelfEvent({
-            agentId: agent.id,
+            agentId: resolved.agentRowPresent ? agent.id : null,
             compactHistory: input.compactHistory ?? null,
             herdrSessionName: agent.herdrSessionName,
             idempotencyKey: `agent.failed:plan:${input.planId}`,
@@ -1666,7 +2032,11 @@ export class AgentIndexService {
             paneGeneration: agent.paneGeneration ?? null,
             payload: {
                 agent: agent.agent,
+                // The original id survives in the payload even when the row had to be
+                // written as an orphan, so the failure stays traceable to its agent.
+                agentId: originalAgentId,
                 attempts: input.attempts,
+                fallbackOutcome,
                 from: input.from,
                 herdrSessionName: agent.herdrSessionName,
                 name: agent.name,
@@ -1682,17 +2052,11 @@ export class AgentIndexService {
         });
     }
     #appendPlanDiscardedEvent(input) {
-        let agent = input.agent;
-        try {
-            agent = this.#stores.agents.get(input.agent.id);
-        }
-        catch {
-            const stitched = this.#minimalAgentFromLatestEvent(input.agent.id, input.agent.herdrSessionName);
-            if (stitched)
-                agent = stitched;
-        }
+        const originalAgentId = input.agent.id;
+        const resolved = this.#resolvePlanOutcomeAgent(input.agent, input.planCreatedAt);
+        const agent = resolved.agent;
         return this.#appendAndAckSelfEvent({
-            agentId: agent.id,
+            agentId: resolved.agentRowPresent ? agent.id : null,
             compactHistory: input.compactHistory ?? null,
             herdrSessionName: agent.herdrSessionName,
             idempotencyKey: `agent.discarded:plan:${input.planId}`,
@@ -1700,7 +2064,7 @@ export class AgentIndexService {
             paneGeneration: agent.paneGeneration ?? null,
             payload: {
                 agent: agent.agent,
-                agentId: agent.id,
+                agentId: originalAgentId,
                 attempts: input.attempts,
                 from: input.from,
                 herdrSessionName: agent.herdrSessionName,