@dorokuma/herdsman 0.11.6 → 0.13.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +26 -13
  2. package/dist/src/agent-history/pi-reader.d.ts.map +1 -1
  3. package/dist/src/agent-history/pi-reader.js +10 -2
  4. package/dist/src/agent-history/pi-reader.js.map +1 -1
  5. package/dist/src/agent-history/readers.js +6 -1
  6. package/dist/src/agent-history/readers.js.map +1 -1
  7. package/dist/src/agent-history/text.d.ts.map +1 -1
  8. package/dist/src/agent-history/text.js +3 -0
  9. package/dist/src/agent-history/text.js.map +1 -1
  10. package/dist/src/cli/herdsman.d.ts +14 -1
  11. package/dist/src/cli/herdsman.d.ts.map +1 -1
  12. package/dist/src/cli/herdsman.js +49 -65
  13. package/dist/src/cli/herdsman.js.map +1 -1
  14. package/dist/src/config/schema.d.ts +4 -0
  15. package/dist/src/config/schema.d.ts.map +1 -1
  16. package/dist/src/config/schema.js +5 -0
  17. package/dist/src/config/schema.js.map +1 -1
  18. package/dist/src/daemon/agent-event-reconciler.d.ts.map +1 -1
  19. package/dist/src/daemon/agent-event-reconciler.js +53 -2
  20. package/dist/src/daemon/agent-event-reconciler.js.map +1 -1
  21. package/dist/src/daemon/herdr-session-watch-manager.d.ts +13 -1
  22. package/dist/src/daemon/herdr-session-watch-manager.d.ts.map +1 -1
  23. package/dist/src/daemon/herdr-session-watch-manager.js +52 -15
  24. package/dist/src/daemon/herdr-session-watch-manager.js.map +1 -1
  25. package/dist/src/daemon/observability-server.d.ts.map +1 -1
  26. package/dist/src/daemon/observability-server.js +17 -1
  27. package/dist/src/daemon/observability-server.js.map +1 -1
  28. package/dist/src/daemon/process-manager.d.ts +50 -49
  29. package/dist/src/daemon/process-manager.d.ts.map +1 -1
  30. package/dist/src/daemon/process-manager.js +246 -204
  31. package/dist/src/daemon/process-manager.js.map +1 -1
  32. package/dist/src/daemon/service-supervision.d.ts +68 -0
  33. package/dist/src/daemon/service-supervision.d.ts.map +1 -0
  34. package/dist/src/daemon/service-supervision.js +127 -0
  35. package/dist/src/daemon/service-supervision.js.map +1 -0
  36. package/dist/src/daemon/service.d.ts +35 -0
  37. package/dist/src/daemon/service.d.ts.map +1 -1
  38. package/dist/src/daemon/service.js +185 -27
  39. package/dist/src/daemon/service.js.map +1 -1
  40. package/dist/src/db/agent-events.d.ts +31 -0
  41. package/dist/src/db/agent-events.d.ts.map +1 -1
  42. package/dist/src/db/agent-events.js +207 -55
  43. package/dist/src/db/agent-events.js.map +1 -1
  44. package/dist/src/db/agents.js +1 -1
  45. package/dist/src/db/agents.js.map +1 -1
  46. package/dist/src/db/status-event-plans.d.ts +36 -4
  47. package/dist/src/db/status-event-plans.d.ts.map +1 -1
  48. package/dist/src/db/status-event-plans.js +96 -11
  49. package/dist/src/db/status-event-plans.js.map +1 -1
  50. package/dist/src/herdr/session-list.d.ts +11 -0
  51. package/dist/src/herdr/session-list.d.ts.map +1 -1
  52. package/dist/src/herdr/session-list.js +13 -0
  53. package/dist/src/herdr/session-list.js.map +1 -1
  54. package/dist/src/herdr/socket-client.d.ts +8 -0
  55. package/dist/src/herdr/socket-client.d.ts.map +1 -1
  56. package/dist/src/herdr/socket-client.js +43 -0
  57. package/dist/src/herdr/socket-client.js.map +1 -1
  58. package/dist/src/observability/agent-index-service.d.ts +16 -0
  59. package/dist/src/observability/agent-index-service.d.ts.map +1 -1
  60. package/dist/src/observability/agent-index-service.js +837 -114
  61. package/dist/src/observability/agent-index-service.js.map +1 -1
  62. package/dist/src/observability/agent-orchestrator-service.d.ts.map +1 -1
  63. package/dist/src/observability/agent-orchestrator-service.js +5 -1
  64. package/dist/src/observability/agent-orchestrator-service.js.map +1 -1
  65. package/dist/src/observability/contracts.d.ts +6 -1
  66. package/dist/src/observability/contracts.d.ts.map +1 -1
  67. package/dist/src/observability/contracts.js +8 -0
  68. package/dist/src/observability/contracts.js.map +1 -1
  69. package/dist/src/observability/schemas.d.ts +1 -0
  70. package/dist/src/observability/schemas.d.ts.map +1 -1
  71. package/dist/src/observability/schemas.js +1 -0
  72. package/dist/src/observability/schemas.js.map +1 -1
  73. package/dist/src/observability/turn-completion.d.ts +3 -0
  74. package/dist/src/observability/turn-completion.d.ts.map +1 -1
  75. package/dist/src/observability/turn-completion.js +12 -2
  76. package/dist/src/observability/turn-completion.js.map +1 -1
  77. package/drizzle/0010_legacy_degraded_cleanup.sql +1 -0
  78. package/drizzle/meta/0010_snapshot.json +975 -0
  79. package/drizzle/meta/_journal.json +7 -0
  80. package/package.json +3 -6
  81. package/README.ja.md +0 -145
@@ -6,7 +6,7 @@ import { runSqliteTransaction } from "../db/client.js";
6
6
  import { normalizeHerdrSessionSnapshot } from "../herdr/session-snapshot.js";
7
7
  import { HerdrSocketClient } from "../herdr/socket-client.js";
8
8
  import { AgentContextService } from "../observability/agent-context-service.js";
9
- import { parseAgentStatus, } from "../observability/contracts.js";
9
+ import { parseAgentStatus, VALID_AGENT_EVENT_TYPES, } from "../observability/contracts.js";
10
10
  import { TURN_SIGNAL_WAIT_MS } from "../observability/turn-completion.js";
11
11
  export function historyHasAdvanced(current, baseline, options) {
12
12
  if (current.lastAssistantMessage?.ref &&
@@ -22,6 +22,10 @@ export function historyHasAdvanced(current, baseline, options) {
22
22
  }
23
23
  return current.messageCount > (baseline?.messageCount ?? 0);
24
24
  }
25
+ function isTerminalAssistant(compact) {
26
+ const stopReason = compact?.lastAssistantMessage?.stopReason;
27
+ return stopReason === "stop" || stopReason === "length";
28
+ }
25
29
  export const PLAN_WAITING_HISTORY = "PLAN_WAITING_HISTORY";
26
30
  export class PlanWaitingHistoryError extends Error {
27
31
  constructor() {
@@ -29,6 +33,17 @@ export class PlanWaitingHistoryError extends Error {
29
33
  this.name = "PlanWaitingHistoryError";
30
34
  }
31
35
  }
36
+ /**
37
+ * Invariant for the waiting-history retry ring: only a genuine
38
+ * `PlanWaitingHistoryError` may (re)arm the 10s timer. It is the single signal
39
+ * that the agent history legitimately has not advanced yet. Synthetic retry
40
+ * reasons such as `degraded` are still recorded on the row, but they never
41
+ * authorize another spin: the pending row is drained by the next refresh
42
+ * instead, so a degraded round cannot keep the ring alive on its own.
43
+ */
44
+ function retryRingAuthorized(error) {
45
+ return error instanceof PlanWaitingHistoryError;
46
+ }
32
47
  /**
33
48
  * Sentinel returned by #appendStatusEvents when the plan must be CANCELLED
34
49
  * rather than skipped (skip still maps to completed in #runPlanRow). The agent
@@ -47,16 +62,23 @@ export class AgentIndexService {
47
62
  #refreshInFlightBySession = new Map();
48
63
  #scheduleRetry;
49
64
  #sessionOperationTail = new Map();
65
+ #shutdownSignal;
50
66
  #sleep;
51
67
  #turnCompletions;
52
68
  #waitingHistoryTimers = new Map();
69
+ #now;
70
+ #onAgentEvent;
53
71
  constructor(options) {
54
72
  this.#clearRetry = options.clearRetry ?? ((timer) => clearTimeout(timer));
55
73
  this.#clientFactory = options.clientFactory ?? ((input) => new HerdrSocketClient(input));
56
74
  this.#scheduleRetry = options.scheduleRetry ?? ((cb, delay) => setTimeout(cb, delay));
57
75
  this.#sleep = options.sleep ?? sleep;
76
+ this.#shutdownSignal = options.shutdownSignal;
77
+ this.#shutdownSignal?.addEventListener("abort", () => this.abortPendingWaits(), { once: true });
78
+ this.#now = options.now ?? (() => Date.now());
58
79
  this.#stores = options.stores;
59
80
  this.#turnCompletions = options.turnCompletions;
81
+ this.#onAgentEvent = options.onAgentEvent ?? (() => { });
60
82
  if (options.context) {
61
83
  this.#context = options.context;
62
84
  }
@@ -127,6 +149,21 @@ export class AgentIndexService {
127
149
  }
128
150
  this.#waitingHistoryTimers.clear();
129
151
  }
152
+ /**
153
+ * Aborts every in-flight status wait (history advance, turn completion,
154
+ * readiness polling). Called on shutdown through the shutdown signal; the
155
+ * waiters treat an abort exactly like a closed pane, so no event is appended
156
+ * for a half-observed transition and the plan row stays retryable.
157
+ */
158
+ abortPendingWaits() {
159
+ for (const [key, controllers] of this.#activeWaiters) {
160
+ for (const controller of controllers) {
161
+ controller.abort();
162
+ }
163
+ controllers.clear();
164
+ this.#activeWaiters.delete(key);
165
+ }
166
+ }
130
167
  async drainInFlightPlans() {
131
168
  const tails = Array.from(this.#planTailByAgent.values());
132
169
  await Promise.allSettled(tails);
@@ -146,14 +183,36 @@ export class AgentIndexService {
146
183
  if (!store)
147
184
  return;
148
185
  const current = store.get(planId);
149
- if (current.status !== "pending" || current.lastError !== PLAN_WAITING_HISTORY) {
150
- this.#waitingHistoryTimers.delete(planId);
186
+ if (current.status !== "pending") {
187
+ this.#clearWaitingTimer(planId);
151
188
  return;
152
189
  }
153
190
  await this.#enqueueAgentPlan(plan.agent.id, () => this.#retryWaitingPlanRow(planId, plan));
154
191
  }
155
192
  catch (error) {
156
- console.debug("Herdsman waiting history retry callback error", error);
193
+ // A failed retry callback used to be a silent `console.debug`; the row it
194
+ // was supposed to advance can then sit in `pending` forever with no
195
+ // signal at all, so the failure is reported as a warning with the row
196
+ // identity attached.
197
+ console.warn("Herdsman waiting history retry callback failed", {
198
+ planId,
199
+ agentId: plan.agent.id,
200
+ herdrSessionName: plan.agent.herdrSessionName,
201
+ paneId: plan.agent.paneId,
202
+ error,
203
+ });
204
+ }
205
+ finally {
206
+ const store = this.#stores.statusEventPlans;
207
+ if (store) {
208
+ const current = store.get(planId);
209
+ if (current.status === "completed" ||
210
+ current.status === "cancelled" ||
211
+ current.status === "failed" ||
212
+ current.status === "discarded") {
213
+ this.#clearWaitingTimer(planId);
214
+ }
215
+ }
157
216
  }
158
217
  }, 10_000);
159
218
  this.#waitingHistoryTimers.set(planId, timer);
@@ -167,11 +226,7 @@ export class AgentIndexService {
167
226
  current = store.get(planId);
168
227
  }
169
228
  catch {
170
- this.#waitingHistoryTimers.delete(planId);
171
- return;
172
- }
173
- if (current.status !== "pending" || current.lastError !== PLAN_WAITING_HISTORY) {
174
- this.#waitingHistoryTimers.delete(planId);
229
+ this.#clearWaitingTimer(planId);
175
230
  return;
176
231
  }
177
232
  let retryPlan;
@@ -193,12 +248,44 @@ export class AgentIndexService {
193
248
  agentId: plan.agent.id,
194
249
  error: err,
195
250
  });
196
- store.markRetry(planId, new PlanWaitingHistoryError());
197
- const updated = store.get(planId);
251
+ const currentPlan = store.get(planId);
252
+ const retryError = currentPlan.lastError === "degraded"
253
+ ? new Error("degraded")
254
+ : new PlanWaitingHistoryError();
255
+ const updated = store.markRetry(planId, retryError);
256
+ if (!updated) {
257
+ this.#clearWaitingTimer(planId);
258
+ return;
259
+ }
198
260
  if (updated.status === "pending") {
199
- this.#scheduleWaitingHistoryRetry(planId, plan);
261
+ // Invariant: a synthetic retry reason never re-arms the ring (see
262
+ // retryRingAuthorized); the pending row waits for the next refresh.
263
+ if (retryRingAuthorized(retryError))
264
+ this.#scheduleWaitingHistoryRetry(planId, plan);
265
+ }
266
+ else if (updated.status === "discarded") {
267
+ this.#clearWaitingTimer(planId);
268
+ console.info("Herdsman status event plan discarded after max attempts", {
269
+ planId,
270
+ agentId: plan.agent.id,
271
+ attempts: updated.attempts,
272
+ from: plan.from,
273
+ to: plan.to,
274
+ herdrSessionName: plan.agent.herdrSessionName,
275
+ });
276
+ this.#appendPlanDiscardedEvent({
277
+ agent: plan.agent,
278
+ attempts: updated.attempts,
279
+ compactHistory: plan.compactHistory,
280
+ from: plan.from,
281
+ planCreatedAt: currentPlan.createdAt.getTime(),
282
+ planId,
283
+ reason: updated.lastError,
284
+ to: plan.to,
285
+ });
200
286
  }
201
287
  else if (updated.status === "failed") {
288
+ this.#clearWaitingTimer(planId);
202
289
  console.warn("Herdsman status event plan failed after max attempts", {
203
290
  planId,
204
291
  agentId: plan.agent.id,
@@ -207,19 +294,26 @@ export class AgentIndexService {
207
294
  to: plan.to,
208
295
  herdrSessionName: plan.agent.herdrSessionName,
209
296
  });
210
- this.#appendPlanFailedEvent({
297
+ const failedEvent = this.#appendPlanFailedEvent({
211
298
  agent: plan.agent,
212
299
  attempts: updated.attempts,
213
300
  compactHistory: plan.compactHistory,
214
301
  from: plan.from,
302
+ planCreatedAt: currentPlan.createdAt.getTime(),
215
303
  planId,
216
304
  reason: updated.lastError,
217
305
  to: plan.to,
218
306
  });
307
+ if (VALID_AGENT_EVENT_TYPES.has(failedEvent.type)) {
308
+ this.#onAgentEvent?.(failedEvent);
309
+ }
219
310
  }
220
311
  return;
221
312
  }
222
- await this.#runPlanRow(current, retryPlan);
313
+ const event = await this.#runPlanRow(current, retryPlan);
314
+ if (event && VALID_AGENT_EVENT_TYPES.has(event.type)) {
315
+ this.#onAgentEvent?.(event);
316
+ }
223
317
  }
224
318
  async executeStatusEventPlan(plan) {
225
319
  if (plan.from === plan.to)
@@ -247,7 +341,14 @@ export class AgentIndexService {
247
341
  to: plan.to,
248
342
  ...(plan.compactHistory ? { compactHistory: plan.compactHistory } : {}),
249
343
  });
250
- return this.#enqueueAgentPlan(plan.agent.id, () => this.#runPlanRow(inserted, plan));
344
+ const result = await this.#enqueueAgentPlan(plan.agent.id, () => this.#runPlanRow(inserted, plan));
345
+ // Publication stays with the caller: every caller forwards this return value
346
+ // (`HerdrSessionWatchManager#submitPlan` -> `publishAgentEvent`, or the
347
+ // non-fast refresh/handle wrappers putting it into `events`). Pushing here as
348
+ // well would write the same `agent.event` twice on the socket, breaking the
349
+ // "exactly once" delivery contract. The retry-exhausted failed branch in
350
+ // `#retryWaitingPlanRow` keeps its own push because no caller sees its result.
351
+ return result;
251
352
  }
252
353
  async drainPendingPlans() {
253
354
  if (!this.#stores.statusEventPlans)
@@ -278,51 +379,192 @@ export class AgentIndexService {
278
379
  }
279
380
  });
280
381
  // Individual rows may reject (after markRetry) but the drain itself never
281
- // rejects: every row is either drained, cancelled, or retried.
282
- await Promise.allSettled(tasks);
382
+ // rejects: every row is either drained, cancelled, or retried. A rejected
383
+ // row is still an anomaly (its own drain path threw), so it is reported
384
+ // instead of being silently folded into the settled results.
385
+ const results = await Promise.allSettled(tasks);
386
+ const rejected = results.flatMap((result) => result.status === "rejected" ? [result.reason] : []);
387
+ if (rejected.length > 0) {
388
+ console.error("Herdsman status event plan drain rejected rows", {
389
+ rejected: rejected.length,
390
+ total: results.length,
391
+ errors: rejected.map((reason) => reason instanceof Error ? reason.message : String(reason)),
392
+ });
393
+ }
283
394
  await this.#backfillFailedPlanEvents();
395
+ await this.#backfillDiscardedPlanEvents();
284
396
  }
285
397
  async #backfillFailedPlanEvents() {
286
398
  const store = this.#stores.statusEventPlans;
287
399
  if (!store)
288
400
  return;
401
+ let backfilled = 0;
402
+ let skipped = 0;
289
403
  for (const row of store.listFailed()) {
290
- const existing = this.#stores.sqlite
291
- .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
292
- .get(row.herdrSessionName, `agent.failed:plan:${row.id}`);
293
- if (existing)
294
- continue;
295
- let agent;
404
+ // Per-row isolation: one unresolvable or unwritable row must not abort the
405
+ // whole backfill round. The legacy loop had no guard, so the first
406
+ // `FOREIGN KEY constraint failed` threw out of `drainPendingPlans` and no
407
+ // later row was ever attempted.
296
408
  try {
297
- agent = this.#stores.agents.get(row.agentId);
409
+ if (row.lastError === PLAN_WAITING_HISTORY) {
410
+ skipped += 1;
411
+ continue;
412
+ }
413
+ const existing = this.#stores.sqlite
414
+ .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
415
+ .get(row.herdrSessionName, `agent.failed:plan:${row.id}`);
416
+ if (existing) {
417
+ skipped += 1;
418
+ continue;
419
+ }
420
+ const agent = this.#resolveBackfillAgent(row);
421
+ if (!agent) {
422
+ skipped += 1;
423
+ continue;
424
+ }
425
+ const event = this.#appendPlanFailedEvent({
426
+ agent,
427
+ attempts: row.attempts,
428
+ compactHistory: row.compactHistory,
429
+ from: row.fromStatus,
430
+ planCreatedAt: row.createdAt.getTime(),
431
+ planId: row.id,
432
+ reason: row.lastError,
433
+ to: row.toStatus,
434
+ });
435
+ // Stock terminal results are ingested silently: the row is written, then
436
+ // immediately acknowledged/non-deliverable, so it stays queryable without
437
+ // waking the current owner with a long-past failure.
438
+ this.#stores.agentEvents.markAcked(event.id);
439
+ backfilled += 1;
298
440
  }
299
- catch {
300
- agent = this.#minimalAgentFromLatestEvent(row.agentId, row.herdrSessionName);
441
+ catch (error) {
442
+ skipped += 1;
443
+ console.warn("Herdsman skipping agent.failed backfill row", {
444
+ agentId: row.agentId,
445
+ herdrSessionName: row.herdrSessionName,
446
+ paneId: row.paneId,
447
+ planId: row.id,
448
+ error: error instanceof Error ? error.message : String(error),
449
+ });
450
+ }
451
+ }
452
+ if (backfilled > 0 || skipped > 0) {
453
+ console.info("Herdsman backfilled agent.failed rows", {
454
+ backfilled,
455
+ skipped,
456
+ total: backfilled + skipped,
457
+ });
458
+ }
459
+ }
460
+ async #backfillDiscardedPlanEvents() {
461
+ const store = this.#stores.statusEventPlans;
462
+ if (!store)
463
+ return;
464
+ let backfilled = 0;
465
+ let skipped = 0;
466
+ for (const row of store.listDiscarded()) {
467
+ try {
468
+ const existing = this.#stores.sqlite
469
+ .prepare("select 1 from agent_events where herdr_session_name = ? and idempotency_key = ?")
470
+ .get(row.herdrSessionName, `agent.discarded:plan:${row.id}`);
471
+ if (existing) {
472
+ skipped += 1;
473
+ continue;
474
+ }
475
+ const agent = this.#resolveBackfillAgent(row);
301
476
  if (!agent) {
302
- console.warn("Herdsman skipping agent.failed backfill; no agent row or events", {
303
- agentId: row.agentId,
304
- herdrSessionName: row.herdrSessionName,
305
- planId: row.id,
306
- });
477
+ skipped += 1;
307
478
  continue;
308
479
  }
480
+ const event = this.#appendPlanDiscardedEvent({
481
+ agent,
482
+ attempts: row.attempts,
483
+ compactHistory: row.compactHistory,
484
+ from: row.fromStatus,
485
+ planCreatedAt: row.createdAt.getTime(),
486
+ planId: row.id,
487
+ reason: row.lastError,
488
+ to: row.toStatus,
489
+ });
490
+ // Same silent-ingest contract as the failed backfill.
491
+ this.#stores.agentEvents.markAcked(event.id);
492
+ backfilled += 1;
309
493
  }
310
- this.#appendPlanFailedEvent({
311
- agent,
312
- attempts: row.attempts,
313
- compactHistory: row.compactHistory,
314
- from: row.fromStatus,
315
- planId: row.id,
316
- reason: row.lastError,
317
- to: row.toStatus,
494
+ catch (error) {
495
+ skipped += 1;
496
+ console.warn("Herdsman skipping agent.discarded backfill row", {
497
+ agentId: row.agentId,
498
+ herdrSessionName: row.herdrSessionName,
499
+ paneId: row.paneId,
500
+ planId: row.id,
501
+ error: error instanceof Error ? error.message : String(error),
502
+ });
503
+ }
504
+ }
505
+ if (backfilled > 0 || skipped > 0) {
506
+ console.info("Herdsman backfilled agent.discarded rows", {
507
+ backfilled,
508
+ skipped,
509
+ total: backfilled + skipped,
318
510
  });
319
511
  }
320
512
  }
321
- #minimalAgentFromLatestEvent(agentId, herdrSessionName) {
513
+ /**
514
+ * Resolves the metadata a terminal plan outcome may use to reach an
515
+ * orchestrator: the live `agents` row when it still exists, otherwise a
516
+ * minimal record stitched from the pane's own event history.
517
+ */
518
+ #resolveBackfillAgent(row) {
519
+ try {
520
+ return this.#stores.agents.get(row.agentId);
521
+ }
522
+ catch {
523
+ const stitched = this.#minimalAgentFromLatestEvent({
524
+ agentId: row.agentId,
525
+ createdBefore: row.createdAt.getTime(),
526
+ herdrSessionName: row.herdrSessionName,
527
+ paneGeneration: row.paneGeneration,
528
+ paneId: row.paneId,
529
+ });
530
+ if (!stitched) {
531
+ console.warn("Herdsman skipping terminal event backfill; no agent row or pane events", {
532
+ agentId: row.agentId,
533
+ herdrSessionName: row.herdrSessionName,
534
+ paneId: row.paneId,
535
+ planId: row.id,
536
+ });
537
+ }
538
+ return stitched;
539
+ }
540
+ }
541
+ /**
542
+ * Reconstructs the minimum agent metadata a terminal event needs from the
543
+ * pane's own event history.
544
+ *
545
+ * The lookup is keyed by `(herdr_session_name, pane_id)` plus the plan's
546
+ * `pane_generation`, and is bounded by `createdBefore` (the plan row's
547
+ * `created_at`). Matching on `agent_id` alone cannot work here: physical
548
+ * deletion of the `agents` row runs `ON DELETE SET NULL` over every event, so
549
+ * the legacy stitch always came back empty, and a pane-id-only match would
550
+ * happily stitch an old failure onto a newer instance of the same pane. The
551
+ * generation and time bounds are what keep the two apart.
552
+ */
553
+ #minimalAgentFromLatestEvent(input) {
554
+ const generationClause = input.paneGeneration === null ? "pane_generation is null" : "pane_generation = ?";
555
+ const params = [
556
+ input.herdrSessionName,
557
+ input.paneId,
558
+ input.createdBefore,
559
+ ];
560
+ if (input.paneGeneration !== null)
561
+ params.push(input.paneGeneration);
322
562
  const latest = this.#stores.sqlite
323
563
  .prepare(`select pane_id, pane_generation, terminal_id, workspace_id, herdr_session_name
324
- from agent_events where agent_id = ? order by id desc limit 1`)
325
- .get(agentId);
564
+ from agent_events
565
+ where herdr_session_name = ? and pane_id = ? and created_at <= ? and ${generationClause}
566
+ order by id desc limit 1`)
567
+ .get(...params);
326
568
  if (!latest?.pane_id || !latest.workspace_id)
327
569
  return undefined;
328
570
  return {
@@ -333,8 +575,8 @@ export class AgentIndexService {
333
575
  firstSeenAt: new Date(0),
334
576
  focused: false,
335
577
  foregroundCwd: null,
336
- herdrSessionName: latest.herdr_session_name || herdrSessionName,
337
- id: agentId,
578
+ herdrSessionName: latest.herdr_session_name || input.herdrSessionName,
579
+ id: input.agentId,
338
580
  lastSeenAt: new Date(0),
339
581
  name: null,
340
582
  paneId: latest.pane_id,
@@ -346,30 +588,50 @@ export class AgentIndexService {
346
588
  };
347
589
  }
348
590
  async #drainPlanRow(row) {
591
+ const store = this.#stores.statusEventPlans;
592
+ if (!store)
593
+ return;
594
+ let current;
595
+ try {
596
+ current = runSqliteTransaction(this.#stores.sqlite, () => {
597
+ const candidate = store.get(row.id);
598
+ if (candidate.status !== "pending")
599
+ return null;
600
+ return candidate;
601
+ });
602
+ }
603
+ catch {
604
+ this.#clearWaitingTimer(row.id);
605
+ return;
606
+ }
607
+ if (!current) {
608
+ this.#clearWaitingTimer(row.id);
609
+ return;
610
+ }
349
611
  let agent;
350
612
  try {
351
613
  agent =
352
614
  this.#stores.agents.findByPane({
353
- herdrSessionName: row.herdrSessionName,
354
- paneId: row.paneId,
355
- paneGeneration: row.paneGeneration,
356
- }) ?? this.#stores.agents.get(row.agentId);
615
+ herdrSessionName: current.herdrSessionName,
616
+ paneId: current.paneId,
617
+ paneGeneration: current.paneGeneration,
618
+ }) ?? this.#stores.agents.get(current.agentId);
357
619
  }
358
620
  catch {
359
621
  agent = undefined;
360
622
  }
361
623
  if (!agent) {
362
- this.#stores.statusEventPlans.markCancelled(row.id);
624
+ store.markCancelled(current.id);
363
625
  console.warn("Herdsman cancelling status event plan for missing agent", {
364
- agentId: row.agentId,
365
- herdrSessionName: row.herdrSessionName,
366
- paneId: row.paneId,
367
- from: row.fromStatus,
368
- to: row.toStatus,
626
+ agentId: current.agentId,
627
+ herdrSessionName: current.herdrSessionName,
628
+ paneId: current.paneId,
629
+ from: current.fromStatus,
630
+ to: current.toStatus,
369
631
  });
370
632
  return;
371
633
  }
372
- let activeCompact = row.compactHistory;
634
+ let activeCompact = current.compactHistory;
373
635
  try {
374
636
  const refreshed = await this.#context.refreshAgent({
375
637
  agent,
@@ -379,52 +641,88 @@ export class AgentIndexService {
379
641
  activeCompact = refreshed.snapshot.compactHistory;
380
642
  }
381
643
  catch (err) {
382
- this.#stores.statusEventPlans.markRetry(row.id, new PlanWaitingHistoryError());
383
- const updated = this.#stores.statusEventPlans.get(row.id);
644
+ const retryError = current.lastError === "degraded" ? new Error("degraded") : new PlanWaitingHistoryError();
645
+ const updated = store.markRetry(current.id, retryError);
646
+ if (!updated) {
647
+ this.#clearWaitingTimer(current.id);
648
+ return;
649
+ }
384
650
  if (updated.status === "pending") {
385
651
  console.warn("Herdsman failed to refresh agent during drain plan row, keeping waiting", {
386
- planId: row.id,
652
+ planId: current.id,
387
653
  agentId: agent.id,
388
654
  error: err,
389
655
  });
390
- this.#scheduleWaitingHistoryRetry(row.id, {
656
+ // Invariant: the ring is only re-armed by a genuine
657
+ // PlanWaitingHistoryError; a degraded row stays pending for the next
658
+ // refresh to drain instead of spinning this callback again.
659
+ if (retryRingAuthorized(retryError)) {
660
+ this.#scheduleWaitingHistoryRetry(current.id, {
661
+ agent,
662
+ compactHistory: current.compactHistory,
663
+ from: current.fromStatus,
664
+ to: current.toStatus,
665
+ ...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
666
+ });
667
+ }
668
+ }
669
+ else if (updated.status === "discarded") {
670
+ this.#clearWaitingTimer(current.id);
671
+ console.info("Herdsman plan marked discarded during drain", {
672
+ planId: current.id,
673
+ agentId: agent.id,
674
+ from: current.fromStatus,
675
+ to: current.toStatus,
676
+ reason: updated.lastError,
677
+ });
678
+ this.#appendPlanDiscardedEvent({
391
679
  agent,
392
- compactHistory: row.compactHistory,
393
- from: row.fromStatus,
394
- to: row.toStatus,
395
- ...(row.herdrEventKey ? { herdrEventKey: row.herdrEventKey } : {}),
680
+ attempts: updated.attempts,
681
+ compactHistory: current.compactHistory,
682
+ from: current.fromStatus,
683
+ planCreatedAt: current.createdAt.getTime(),
684
+ planId: current.id,
685
+ reason: updated.lastError,
686
+ to: current.toStatus,
396
687
  });
397
688
  }
398
689
  else if (updated.status === "failed") {
399
- this.#clearWaitingTimer(row.id);
690
+ this.#clearWaitingTimer(current.id);
400
691
  console.warn("Herdsman plan marked failed", {
401
- planId: row.id,
692
+ planId: current.id,
402
693
  agentId: agent.id,
403
- from: row.fromStatus,
404
- to: row.toStatus,
694
+ from: current.fromStatus,
695
+ to: current.toStatus,
405
696
  reason: updated.lastError,
406
697
  });
407
- this.#appendPlanFailedEvent({
698
+ const failedEvent = this.#appendPlanFailedEvent({
408
699
  agent,
409
700
  attempts: updated.attempts,
410
- compactHistory: row.compactHistory,
411
- from: row.fromStatus,
412
- planId: row.id,
701
+ compactHistory: current.compactHistory,
702
+ from: current.fromStatus,
703
+ planCreatedAt: current.createdAt.getTime(),
704
+ planId: current.id,
413
705
  reason: updated.lastError,
414
- to: row.toStatus,
706
+ to: current.toStatus,
415
707
  });
708
+ if (VALID_AGENT_EVENT_TYPES.has(failedEvent.type)) {
709
+ this.#onAgentEvent?.(failedEvent);
710
+ }
416
711
  }
417
712
  return;
418
713
  }
419
714
  const plan = {
420
715
  agent,
421
- attempts: row.attempts,
716
+ attempts: current.attempts,
422
717
  compactHistory: activeCompact,
423
- from: row.fromStatus,
424
- to: row.toStatus,
425
- ...(row.herdrEventKey ? { herdrEventKey: row.herdrEventKey } : {}),
718
+ from: current.fromStatus,
719
+ to: current.toStatus,
720
+ ...(current.herdrEventKey ? { herdrEventKey: current.herdrEventKey } : {}),
426
721
  };
427
- await this.#runPlanRow(row, plan);
722
+ const event = await this.#runPlanRow(current, plan);
723
+ if (event && VALID_AGENT_EVENT_TYPES.has(event.type)) {
724
+ this.#onAgentEvent?.(event);
725
+ }
428
726
  }
429
727
  #enqueueAgentPlan(agentId, operation) {
430
728
  const previous = this.#planTailByAgent.get(agentId) ?? Promise.resolve();
@@ -445,14 +743,21 @@ export class AgentIndexService {
445
743
  const current = store.get(row.id);
446
744
  if (current.status === "completed" ||
447
745
  current.status === "cancelled" ||
448
- current.status === "failed") {
746
+ current.status === "failed" ||
747
+ current.status === "discarded") {
748
+ this.#clearWaitingTimer(row.id);
749
+ return undefined;
750
+ }
751
+ if (!store.markRunning(row.id)) {
449
752
  this.#clearWaitingTimer(row.id);
450
753
  return undefined;
451
754
  }
452
- store.markRunning(row.id);
453
755
  let activePlan = {
454
756
  ...plan,
455
757
  attempts: row.attempts,
758
+ // Row correlation for observability (see `#logDegradedRelease`): the plan
759
+ // id is only known here, where the store row is in hand.
760
+ planId: row.id,
456
761
  };
457
762
  if (row.attempts > 0) {
458
763
  try {
@@ -465,6 +770,7 @@ export class AgentIndexService {
465
770
  ...plan,
466
771
  attempts: row.attempts,
467
772
  compactHistory: refreshed.snapshot.compactHistory,
773
+ planId: row.id,
468
774
  };
469
775
  }
470
776
  catch (err) {
@@ -473,13 +779,45 @@ export class AgentIndexService {
473
779
  agentId: plan.agent.id,
474
780
  error: err,
475
781
  });
476
- store.markRetry(row.id, new PlanWaitingHistoryError());
477
- const updated = store.get(row.id);
782
+ // Keep the degraded reason when the plan was already degraded by an
783
+ // earlier round: markRetry decides discarded (silent) vs failed (wakes)
784
+ // from this message, so passing PLAN_WAITING_HISTORY here would let a
785
+ // degraded plan exhaust silently.
786
+ const retryError = row.lastError === "degraded" ? new Error("degraded") : new PlanWaitingHistoryError();
787
+ const updated = store.markRetry(row.id, retryError);
788
+ if (!updated) {
789
+ this.#clearWaitingTimer(row.id);
790
+ return undefined;
791
+ }
478
792
  if (updated.status === "pending") {
479
- this.#scheduleWaitingHistoryRetry(row.id, plan);
793
+ // Invariant: only a genuine PlanWaitingHistoryError authorizes the
794
+ // ring; a degraded pre-refresh round records the synthetic reason and
795
+ // waits for the next refresh instead.
796
+ if (retryRingAuthorized(retryError))
797
+ this.#scheduleWaitingHistoryRetry(row.id, plan);
480
798
  }
481
799
  else {
482
800
  this.#clearWaitingTimer(row.id);
801
+ if (updated.status === "discarded") {
802
+ console.info("Herdsman status event plan discarded after max attempts", {
803
+ planId: row.id,
804
+ agentId: row.agentId,
805
+ attempts: updated.attempts,
806
+ from: row.fromStatus,
807
+ to: row.toStatus,
808
+ herdrSessionName: row.herdrSessionName,
809
+ });
810
+ return this.#appendPlanDiscardedEvent({
811
+ agent: plan.agent,
812
+ attempts: updated.attempts,
813
+ compactHistory: plan.compactHistory,
814
+ from: row.fromStatus,
815
+ planCreatedAt: row.createdAt.getTime(),
816
+ planId: row.id,
817
+ reason: updated.lastError,
818
+ to: row.toStatus,
819
+ });
820
+ }
483
821
  if (updated.status === "failed") {
484
822
  console.warn("Herdsman status event plan failed after max attempts", {
485
823
  planId: row.id,
@@ -494,6 +832,7 @@ export class AgentIndexService {
494
832
  attempts: updated.attempts,
495
833
  compactHistory: plan.compactHistory,
496
834
  from: row.fromStatus,
835
+ planCreatedAt: row.createdAt.getTime(),
497
836
  planId: row.id,
498
837
  reason: updated.lastError,
499
838
  to: row.toStatus,
@@ -521,18 +860,100 @@ export class AgentIndexService {
521
860
  return undefined;
522
861
  }
523
862
  this.#clearWaitingTimer(row.id);
863
+ const eventPayload = event?.payload;
864
+ const isDegraded = !!event &&
865
+ typeof eventPayload === "object" &&
866
+ eventPayload !== null &&
867
+ eventPayload.degraded === true;
868
+ if (isDegraded) {
869
+ if (event) {
870
+ this.#stores.agentEvents.invalidateById(event.id, "degraded_retry");
871
+ Object.assign(event, { status: "invalidated", deliverable: 0 });
872
+ }
873
+ const degradedError = new Error("degraded");
874
+ const updated = store.markRetry(row.id, degradedError);
875
+ if (!updated) {
876
+ return undefined;
877
+ }
878
+ if (updated.status === "pending") {
879
+ // Invariant: `degraded` is a synthetic reason, so it never authorizes
880
+ // the retry ring; the row stays pending and the next refresh
881
+ // re-drains it.
882
+ if (retryRingAuthorized(degradedError))
883
+ this.#scheduleWaitingHistoryRetry(row.id, plan);
884
+ return undefined;
885
+ }
886
+ else {
887
+ this.#clearWaitingTimer(row.id);
888
+ if (updated.status === "discarded") {
889
+ return this.#appendPlanDiscardedEvent({
890
+ agent: plan.agent,
891
+ attempts: updated.attempts,
892
+ compactHistory: activePlan.compactHistory,
893
+ from: row.fromStatus,
894
+ planCreatedAt: row.createdAt.getTime(),
895
+ planId: row.id,
896
+ reason: updated.lastError,
897
+ to: row.toStatus,
898
+ });
899
+ }
900
+ if (updated.status === "failed") {
901
+ return this.#appendPlanFailedEvent({
902
+ agent: plan.agent,
903
+ attempts: updated.attempts,
904
+ compactHistory: activePlan.compactHistory,
905
+ from: row.fromStatus,
906
+ planCreatedAt: row.createdAt.getTime(),
907
+ planId: row.id,
908
+ reason: updated.lastError,
909
+ to: row.toStatus,
910
+ });
911
+ }
912
+ }
913
+ return undefined;
914
+ }
524
915
  store.markCompleted(row.id);
525
916
  return event;
526
917
  }
527
918
  catch (error) {
528
919
  if (error instanceof PlanWaitingHistoryError) {
529
- store.markRetry(row.id, error);
530
- const updated = store.get(row.id);
920
+ const retryError = row.lastError === "degraded" ? new Error("degraded") : error;
921
+ const updated = store.markRetry(row.id, retryError);
922
+ if (!updated) {
923
+ this.#clearWaitingTimer(row.id);
924
+ return undefined;
925
+ }
531
926
  if (updated.status === "pending") {
532
- this.#scheduleWaitingHistoryRetry(row.id, plan);
927
+ // Invariant: only a genuine PlanWaitingHistoryError authorizes the
928
+ // ring. A row that already carries the synthetic `degraded` reason is
929
+ // recorded as degraded again and waits for the next refresh instead of
930
+ // re-arming the 10s timer, so this catch cannot be used as a back door
931
+ // that lets a degraded round keep the ring alive.
932
+ if (retryRingAuthorized(retryError))
933
+ this.#scheduleWaitingHistoryRetry(row.id, plan);
533
934
  }
534
935
  else {
535
936
  this.#clearWaitingTimer(row.id);
937
+ if (updated.status === "discarded") {
938
+ console.info("Herdsman status event plan discarded after max attempts", {
939
+ planId: row.id,
940
+ agentId: row.agentId,
941
+ attempts: updated.attempts,
942
+ from: row.fromStatus,
943
+ to: row.toStatus,
944
+ herdrSessionName: row.herdrSessionName,
945
+ });
946
+ return this.#appendPlanDiscardedEvent({
947
+ agent: plan.agent,
948
+ attempts: updated.attempts,
949
+ compactHistory: activePlan.compactHistory,
950
+ from: row.fromStatus,
951
+ planCreatedAt: row.createdAt.getTime(),
952
+ planId: row.id,
953
+ reason: updated.lastError,
954
+ to: row.toStatus,
955
+ });
956
+ }
536
957
  if (updated.status === "failed") {
537
958
  console.warn("Herdsman status event plan failed after max attempts", {
538
959
  planId: row.id,
@@ -547,6 +968,7 @@ export class AgentIndexService {
547
968
  attempts: updated.attempts,
548
969
  compactHistory: activePlan.compactHistory,
549
970
  from: row.fromStatus,
971
+ planCreatedAt: row.createdAt.getTime(),
550
972
  planId: row.id,
551
973
  reason: updated.lastError,
552
974
  to: row.toStatus,
@@ -569,8 +991,35 @@ export class AgentIndexService {
569
991
  return undefined;
570
992
  }
571
993
  this.#clearWaitingTimer(row.id);
572
- store.markRetry(row.id, error);
573
- const updated = store.get(row.id);
994
+ const retryError = row.lastError === "degraded" ? new Error("degraded") : error;
995
+ const updated = store.markRetry(row.id, retryError);
996
+ if (!updated) {
997
+ return undefined;
998
+ }
999
+ // Invariant: only a genuine PlanWaitingHistoryError re-arms the retry ring,
1000
+ // and that case was handled above. On this generic path `retryError` is
1001
+ // either the synthetic `degraded` reason or an unrelated failure, so no
1002
+ // spin is scheduled: a still-pending row is drained by the next refresh.
1003
+ if (updated.status === "discarded") {
1004
+ console.info("Herdsman status event plan discarded after max attempts", {
1005
+ planId: row.id,
1006
+ agentId: row.agentId,
1007
+ attempts: updated.attempts,
1008
+ from: row.fromStatus,
1009
+ to: row.toStatus,
1010
+ herdrSessionName: row.herdrSessionName,
1011
+ });
1012
+ return this.#appendPlanDiscardedEvent({
1013
+ agent: plan.agent,
1014
+ attempts: updated.attempts,
1015
+ compactHistory: activePlan.compactHistory,
1016
+ from: row.fromStatus,
1017
+ planCreatedAt: row.createdAt.getTime(),
1018
+ planId: row.id,
1019
+ reason: updated.lastError,
1020
+ to: row.toStatus,
1021
+ });
1022
+ }
574
1023
  if (updated.status === "failed") {
575
1024
  console.warn("Herdsman status event plan failed after max attempts", {
576
1025
  planId: row.id,
@@ -585,6 +1034,7 @@ export class AgentIndexService {
585
1034
  attempts: updated.attempts,
586
1035
  compactHistory: activePlan.compactHistory,
587
1036
  from: row.fromStatus,
1037
+ planCreatedAt: row.createdAt.getTime(),
588
1038
  planId: row.id,
589
1039
  reason: updated.lastError,
590
1040
  to: row.toStatus,
@@ -595,6 +1045,12 @@ export class AgentIndexService {
595
1045
  }
596
1046
  async #waitForHistoryAdvance(input) {
597
1047
  const maxAttempts = input.maxAttempts ?? 8;
1048
+ // Fixed 200ms interval, no exponential backoff: the history cache either
1049
+ // advances shortly after the turn signal or it does not, and a doubling
1050
+ // delay only pushed the first deliverable terminal event tens of seconds
1051
+ // away. The whole window is hard bounded by maxTotalMs.
1052
+ const baseDelay = 200;
1053
+ const maxTotalMs = 1500;
598
1054
  let refreshed = input.initial
599
1055
  ? { snapshot: { compactHistory: input.initial } }
600
1056
  : await this.#context.refreshAgent({
@@ -602,20 +1058,34 @@ export class AgentIndexService {
602
1058
  forceRefresh: true,
603
1059
  identityChanged: false,
604
1060
  });
1061
+ const start = this.#now();
605
1062
  for (let attempt = 0; attempt < maxAttempts &&
606
1063
  !historyHasAdvanced(refreshed.snapshot.compactHistory, input.baseline, {
607
1064
  requireAssistantChange: input.requireAssistantChange,
608
1065
  }) &&
609
- !input.controller.signal.aborted; attempt += 1) {
610
- await this.#sleep(500, input.controller.signal);
611
- if (input.controller.signal.aborted ||
612
- this.#stores.agents.isPaneClosed({
1066
+ !input.controller.signal.aborted &&
1067
+ this.#now() - start < maxTotalMs; attempt += 1) {
1068
+ // Clamp the last step to the remaining budget so the total wait never
1069
+ // exceeds maxTotalMs even though the loop condition is checked before
1070
+ // sleeping.
1071
+ const remaining = maxTotalMs - (this.#now() - start);
1072
+ if (remaining <= 0)
1073
+ break;
1074
+ await this.#sleep(Math.min(baseDelay, remaining), input.controller.signal);
1075
+ if (input.controller.signal.aborted) {
1076
+ console.debug("Herdsman retrying status event plan because wait was aborted during history refresh", {
1077
+ agentId: input.agent.id,
613
1078
  herdrSessionName: input.agent.herdrSessionName,
614
1079
  paneId: input.agent.paneId,
615
- paneGeneration: input.agent.paneGeneration ?? null,
616
- })) {
617
- console.debug("Herdsman skipping status event generation because wait was aborted during history refresh", {
618
- aborted: input.controller.signal.aborted,
1080
+ });
1081
+ throw new PlanWaitingHistoryError();
1082
+ }
1083
+ if (this.#stores.agents.isPaneClosed({
1084
+ herdrSessionName: input.agent.herdrSessionName,
1085
+ paneId: input.agent.paneId,
1086
+ paneGeneration: input.agent.paneGeneration ?? null,
1087
+ })) {
1088
+ console.debug("Herdsman skipping status event generation because pane closed during history refresh", {
619
1089
  agentId: input.agent.id,
620
1090
  herdrSessionName: input.agent.herdrSessionName,
621
1091
  paneId: input.agent.paneId,
@@ -981,8 +1451,23 @@ export class AgentIndexService {
981
1451
  paneGeneration: paneGenerationOf(agent),
982
1452
  });
983
1453
  }
1454
+ /**
1455
+ * Distinguishes the two reasons an in-flight wait can be aborted. A closed pane
1456
+ * makes the transition moot, so the plan is skipped; a daemon shutdown leaves
1457
+ * the pane alive, so the plan must stay retryable instead of being completed
1458
+ * without ever emitting the transition (it is drained on the next start).
1459
+ */
1460
+ #throwIfShutdownAbort(controller) {
1461
+ if (controller.signal.aborted && this.#shutdownSignal?.aborted) {
1462
+ throw new PlanWaitingHistoryError();
1463
+ }
1464
+ }
984
1465
  #registerActiveWaiter(agent, controller) {
985
1466
  const key = `${agent.herdrSessionName}\0${agent.paneId}\0${agent.paneGeneration ?? ""}`;
1467
+ // A wait that starts after shutdown already aborted must not sleep at all:
1468
+ // abort it up front so the caller short-circuits like a closed pane.
1469
+ if (this.#shutdownSignal?.aborted)
1470
+ controller.abort();
986
1471
  let set = this.#activeWaiters.get(key);
987
1472
  if (!set) {
988
1473
  set = new Set();
@@ -1010,10 +1495,36 @@ export class AgentIndexService {
1010
1495
  }
1011
1496
  }
1012
1497
  }
1498
+ /**
1499
+ * Observability for a degraded release. The emitted status event carries the
1500
+ * specific `degradedReason`, but the plan row itself only stores the synthetic
1501
+ * `degraded` marker (`row.lastError`), so without this line the cause is
1502
+ * invisible in the daemon log. It cannot ride along in the failed-event payload
1503
+ * either: for most append sites only `row.lastError` is at hand, so handing them
1504
+ * the specific reason would mean persisting a new plan field (schema change).
1505
+ * That is not universal — at least three call sites in this file already have the
1506
+ * specific reason in their own call stack (`:1074` the status event payload's
1507
+ * `degradedReason`, `:1136` the `PlanWaitingHistoryError`, `:1203` the generic
1508
+ * `Error`), so routing the reason into the payload does not necessarily require a
1509
+ * schema change. Log-only on purpose — no behaviour change (see
1510
+ * .agents/notes/20260930-terminal-event-delivery-open-items.md A1).
1511
+ *
1512
+ * `plan.planId` is the row correlation that already exists on
1513
+ * `StatusEventPlan` (see `executeStatusEventPlan`); it is `null` for the
1514
+ * row-less in-memory plans executed without the SQLite store.
1515
+ */
1516
+ #logDegradedRelease(plan, degradedReason) {
1517
+ console.warn("Herdsman emitted degraded status event", {
1518
+ agentId: plan.agent.id,
1519
+ attempts: plan.attempts ?? 0,
1520
+ degradedReason,
1521
+ herdrSessionName: plan.agent.herdrSessionName,
1522
+ paneId: plan.agent.paneId,
1523
+ planId: plan.planId ?? null,
1524
+ });
1525
+ }
1013
1526
  async #appendStatusEvents(input) {
1014
1527
  // Invariant: compactHistory is always populated as an object before calling #appendStatusEvents; !input.compactHistory check below is defensive and unreachable in normal operation.
1015
- if (input.from === input.to || !input.compactHistory)
1016
- return undefined;
1017
1528
  if (this.#stores.agents.isPaneClosed({
1018
1529
  herdrSessionName: input.agent.herdrSessionName,
1019
1530
  paneId: input.agent.paneId,
@@ -1042,9 +1553,18 @@ export class AgentIndexService {
1042
1553
  skipStatusChanged = true;
1043
1554
  }
1044
1555
  if (hasTerminalAfter && !input.herdrEventKey) {
1045
- const lastTerminal = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
1556
+ const isAgyHistory = input.agent.agent === "agy" || input.compactHistory?.source === "antigravity-sqlite";
1557
+ // For an agy/antigravity pane only a terminal event that already
1558
+ // represented a completed turn may count as the delivered baseline: an
1559
+ // `unknown -> idle` startup row would otherwise make the `working -> done`
1560
+ // row of the same round look like a duplicate.
1561
+ const lastTerminal = isAgyHistory
1562
+ ? this.#stores.agentEvents.latestCompletedTurnEvent(input.agent.id, input.agent.herdrSessionName)
1563
+ : this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
1046
1564
  if (sameTerminalAssistantContent(input.compactHistory, lastTerminal?.compactHistory, input.agent.agent)) {
1047
- if ((input.attempts ?? 0) > 0) {
1565
+ // An agy round whose ref already reached a completed turn is settled: a
1566
+ // retry must skip it instead of exhausting the budget into `discarded`.
1567
+ if ((input.attempts ?? 0) > 0 && !isAgyHistory) {
1048
1568
  throw new PlanWaitingHistoryError();
1049
1569
  }
1050
1570
  console.debug("Herdsman skipping legacy duplicate status transition event with identical history ref", {
@@ -1092,6 +1612,7 @@ export class AgentIndexService {
1092
1612
  herdrSessionName: input.agent.herdrSessionName,
1093
1613
  paneId: input.agent.paneId,
1094
1614
  });
1615
+ this.#throwIfShutdownAbort(controller);
1095
1616
  return undefined;
1096
1617
  }
1097
1618
  if (turn?.received) {
@@ -1100,7 +1621,61 @@ export class AgentIndexService {
1100
1621
  forceRefresh: true,
1101
1622
  identityChanged: false,
1102
1623
  })).snapshot.compactHistory;
1103
- if (!isAlreadyEmittedInRetry && hasNonEmptyAssistantMessage(fresh)) {
1624
+ if (this.#stores.agents.isPaneClosed({
1625
+ herdrSessionName: input.agent.herdrSessionName,
1626
+ paneId: input.agent.paneId,
1627
+ paneGeneration: input.agent.paneGeneration ?? null,
1628
+ })) {
1629
+ console.debug("Herdsman skipping status event generation because pane closed after turn signal", {
1630
+ agentId: input.agent.id,
1631
+ herdrSessionName: input.agent.herdrSessionName,
1632
+ paneId: input.agent.paneId,
1633
+ });
1634
+ return undefined;
1635
+ }
1636
+ const freshIsTerminal = isTerminalAssistant(fresh);
1637
+ const expectedText = turn.expectedText?.trim();
1638
+ const lastAssistantText = fresh.lastAssistantMessage?.text?.trim() ?? "";
1639
+ const textMatches = !expectedText || lastAssistantText.endsWith(expectedText);
1640
+ // A client-confirmed turn is a deliverable terminal state: `confirmed`
1641
+ // means Pi verified the final assistant message is on disk, so a
1642
+ // confirmed round is delivered straight through and is never marked
1643
+ // degraded (which is what made #runPlanRow run
1644
+ // invalidateById(..., "degraded_retry") on already-written content).
1645
+ const confirmedTerminal = turn.confirmed === true;
1646
+ const degradeOrRelease = (degradedReason) => {
1647
+ if (confirmedTerminal) {
1648
+ // A confirmed round stays non-degraded (see above), but the reason
1649
+ // is no longer swallowed: this path used to release silently with
1650
+ // an emptied body, so an empty `agent.done` could only be
1651
+ // diagnosed by reading the pane by hand. `degraded: true` must
1652
+ // stay off here — #runPlanRow would `invalidateById(...,
1653
+ // "degraded_retry")` the row it has already written.
1654
+ console.warn("Herdsman released a confirmed pi status event with no deliverable text", {
1655
+ agentId: input.agent.id,
1656
+ degradedReason,
1657
+ herdrSessionName: input.agent.herdrSessionName,
1658
+ paneId: input.agent.paneId,
1659
+ planId: input.planId ?? null,
1660
+ terminalId: input.agent.terminalId,
1661
+ });
1662
+ return { staleSnapshot: false };
1663
+ }
1664
+ this.#logDegradedRelease(input, degradedReason);
1665
+ return { degraded: true, degradedReason, staleSnapshot: false };
1666
+ };
1667
+ if (confirmedTerminal &&
1668
+ freshIsTerminal &&
1669
+ textMatches &&
1670
+ hasNonEmptyAssistantMessage(fresh)) {
1671
+ compactHistory = fresh;
1672
+ }
1673
+ else if (!isAlreadyEmittedInRetry &&
1674
+ freshIsTerminal &&
1675
+ textMatches &&
1676
+ hasNonEmptyAssistantMessage(fresh) &&
1677
+ (isTerminalAssistant(input.compactHistory) ||
1678
+ historyHasAdvanced(fresh, input.compactHistory))) {
1104
1679
  compactHistory = fresh;
1105
1680
  }
1106
1681
  else {
@@ -1118,7 +1693,54 @@ export class AgentIndexService {
1118
1693
  !historyHasAdvanced(advanced, baseline, { requireAssistantChange: true })) {
1119
1694
  throw new PlanWaitingHistoryError();
1120
1695
  }
1121
- compactHistory = advanced;
1696
+ // `historyHasAdvanced(..., { requireAssistantChange: true })` compares
1697
+ // the assistant ref/text against the plan baseline. That baseline can
1698
+ // hold this round's answer (a plan row's `compact_history_json` is
1699
+ // written once at plan creation and never updated, and creation
1700
+ // measured 131ms/308ms after the final assistant message reached
1701
+ // disk). It can also hold the answer of a *previous* round — the same
1702
+ // frozen baseline is exactly what the daemon last delivered — so a
1703
+ // round that writes nothing new would otherwise re-release the
1704
+ // previous round's body as if it were this one's. `latestTerminal` is
1705
+ // the latest non-invalidated terminal row read before this round's row
1706
+ // is written: usually the last delivery, possibly an undelivered idle.
1707
+ const staleBaselineDuplicate = sameTerminalAssistantContent(advanced, latestTerminal?.compactHistory, "pi");
1708
+ // A confirmed turn carrying a non-empty terminal assistant message is
1709
+ // trusted as-is, so `no_advance_from_input` stays reserved for rounds
1710
+ // with genuinely no deliverable assistant text (empty or non-terminal
1711
+ // tail).
1712
+ const confirmedDeliverable = confirmedTerminal &&
1713
+ isTerminalAssistant(advanced) &&
1714
+ hasNonEmptyAssistantMessage(advanced) &&
1715
+ !staleBaselineDuplicate;
1716
+ if (!isRetry &&
1717
+ !historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
1718
+ if (confirmedDeliverable) {
1719
+ // Trust the already-in-place answer: release it as a non-stale
1720
+ // snapshot and without a `degraded` marker.
1721
+ compactHistory = advanced;
1722
+ payloadExtra = { staleSnapshot: false };
1723
+ }
1724
+ else {
1725
+ compactHistory = { ...advanced, lastAssistantMessage: null };
1726
+ payloadExtra = degradeOrRelease(staleBaselineDuplicate ? "stale_baseline_duplicate" : "no_advance_from_input");
1727
+ }
1728
+ }
1729
+ else {
1730
+ const advancedText = advanced.lastAssistantMessage?.text?.trim() ?? "";
1731
+ const advancedMatchesExpected = !expectedText || advancedText.endsWith(expectedText);
1732
+ if (!advancedMatchesExpected) {
1733
+ compactHistory = { ...advanced, lastAssistantMessage: null };
1734
+ payloadExtra = degradeOrRelease("expected_text_mismatch");
1735
+ }
1736
+ else if (!isTerminalAssistant(advanced) || !hasNonEmptyAssistantMessage(advanced)) {
1737
+ compactHistory = { ...advanced, lastAssistantMessage: null };
1738
+ payloadExtra = degradeOrRelease("non_terminal_assistant");
1739
+ }
1740
+ else {
1741
+ compactHistory = advanced;
1742
+ }
1743
+ }
1122
1744
  }
1123
1745
  console.log(`Herdsman emitted pi agent.${input.to} after turn completion signal (confirmed=${turn.confirmed})`, {
1124
1746
  agentId: input.agent.id,
@@ -1143,7 +1765,21 @@ export class AgentIndexService {
1143
1765
  if (!isRetry &&
1144
1766
  !historyHasAdvanced(advanced, input.compactHistory, { requireAssistantChange: true })) {
1145
1767
  compactHistory = { ...advanced, lastAssistantMessage: null };
1146
- payloadExtra = { noAdvance: true, staleSnapshot: false };
1768
+ this.#logDegradedRelease(input, "no_advance_from_input");
1769
+ payloadExtra = {
1770
+ degraded: true,
1771
+ degradedReason: "no_advance_from_input",
1772
+ staleSnapshot: false,
1773
+ };
1774
+ }
1775
+ else if (!isTerminalAssistant(advanced) || !hasNonEmptyAssistantMessage(advanced)) {
1776
+ compactHistory = { ...advanced, lastAssistantMessage: null };
1777
+ this.#logDegradedRelease(input, "non_terminal_assistant");
1778
+ payloadExtra = {
1779
+ degraded: true,
1780
+ degradedReason: "non_terminal_assistant",
1781
+ staleSnapshot: false,
1782
+ };
1147
1783
  }
1148
1784
  else {
1149
1785
  compactHistory = advanced;
@@ -1157,15 +1793,21 @@ export class AgentIndexService {
1157
1793
  }
1158
1794
  else if (input.agent.agent !== "pi" && (input.to === "idle" || input.to === "done")) {
1159
1795
  // Note: blocked is an interactive intermediate state, exempt from ready gate and empty delivery gate.
1796
+ const isAgyHistory = (compact) => compact?.source === "antigravity-sqlite" || input.agent.agent === "agy";
1797
+ // Only a terminal event that already represented a completed turn may
1798
+ // consume the assistant ref: a late startup idle (`unknown -> idle`),
1799
+ // an `agent.status.changed` row or a discarded plan must not make the
1800
+ // following `working -> done` plan look like a duplicate round.
1801
+ const latestCompletedTurnRef = () => this.#stores.agentEvents.latestCompletedTurnEvent(input.agent.id, input.agent.herdrSessionName)?.compactHistory?.lastAssistantMessage?.ref ?? null;
1160
1802
  const isReadyNonPi = (compact) => {
1161
1803
  if (!compact || !hasNonEmptyAssistantMessage(compact))
1162
1804
  return false;
1163
- const latestTerminal = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
1164
- if (compact.source === "antigravity-sqlite" || input.agent.agent === "agy") {
1165
- const prevRef = latestTerminal?.compactHistory?.lastAssistantMessage?.ref ?? null;
1805
+ if (isAgyHistory(compact)) {
1806
+ const prevRef = latestCompletedTurnRef();
1166
1807
  const currentRef = compact.lastAssistantMessage?.ref ?? null;
1167
1808
  return currentRef !== null && currentRef !== prevRef;
1168
1809
  }
1810
+ const latestTerminal = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
1169
1811
  if (!latestTerminal)
1170
1812
  return true;
1171
1813
  const currentMsg = compact.lastAssistantMessage;
@@ -1175,11 +1817,22 @@ export class AgentIndexService {
1175
1817
  const same = sameTerminalAssistantContent(compact, latestTerminal.compactHistory, input.agent.agent);
1176
1818
  return !same || refChanged || textChanged;
1177
1819
  };
1820
+ // A non-empty assistant ref that was already delivered by a completed
1821
+ // turn is a genuine skip (the round was announced), never a reason to
1822
+ // exhaust the retry budget into `discarded`.
1823
+ const isAlreadyDeliveredCompletedTurn = (compact) => {
1824
+ if (!compact || !isAgyHistory(compact) || !hasNonEmptyAssistantMessage(compact)) {
1825
+ return false;
1826
+ }
1827
+ const currentRef = compact.lastAssistantMessage?.ref ?? null;
1828
+ return currentRef !== null && currentRef === latestCompletedTurnRef();
1829
+ };
1178
1830
  const latestTerminalForSkip = this.#stores.agentEvents.latestTerminalEvent(input.agent.id, input.agent.herdrSessionName);
1179
1831
  if (input.agent.agent !== "agy" &&
1180
1832
  compactHistory?.source !== "antigravity-sqlite" &&
1181
1833
  input.herdrEventKey &&
1182
1834
  latestTerminalForSkip &&
1835
+ compactHistory &&
1183
1836
  hasNonEmptyAssistantMessage(compactHistory)) {
1184
1837
  const currentMsg = compactHistory.lastAssistantMessage;
1185
1838
  const prevMsg = latestTerminalForSkip.compactHistory?.lastAssistantMessage;
@@ -1205,6 +1858,7 @@ export class AgentIndexService {
1205
1858
  herdrSessionName: input.agent.herdrSessionName,
1206
1859
  paneId: input.agent.paneId,
1207
1860
  });
1861
+ this.#throwIfShutdownAbort(controller);
1208
1862
  return undefined;
1209
1863
  }
1210
1864
  try {
@@ -1224,6 +1878,16 @@ export class AgentIndexService {
1224
1878
  }
1225
1879
  }
1226
1880
  if (!isReadyNonPi(compactHistory)) {
1881
+ if (isAlreadyDeliveredCompletedTurn(compactHistory)) {
1882
+ console.debug("Herdsman skipping non-pi terminal status event because the assistant ref was already delivered by a completed turn", {
1883
+ agent: input.agent.agent,
1884
+ agentId: input.agent.id,
1885
+ from: input.from,
1886
+ ref: compactHistory?.lastAssistantMessage?.ref ?? null,
1887
+ to: input.to,
1888
+ });
1889
+ return undefined;
1890
+ }
1227
1891
  throw new PlanWaitingHistoryError();
1228
1892
  }
1229
1893
  }
@@ -1239,6 +1903,7 @@ export class AgentIndexService {
1239
1903
  herdrSessionName: input.agent.herdrSessionName,
1240
1904
  paneId: input.agent.paneId,
1241
1905
  });
1906
+ this.#throwIfShutdownAbort(controller);
1242
1907
  return undefined;
1243
1908
  }
1244
1909
  const currentAgent = this.#stores.agents.findByPane({
@@ -1293,7 +1958,7 @@ export class AgentIndexService {
1293
1958
  ? undefined
1294
1959
  : this.#appendAndAckSelfEvent({
1295
1960
  agentId: targetAgent.id,
1296
- compactHistory,
1961
+ compactHistory: compactHistory ?? null,
1297
1962
  herdrSessionName: targetAgent.herdrSessionName,
1298
1963
  idempotencyKey: idempotencyKey("agent.status.changed", targetAgent, input.from, input.to, `${observationId}:${input.herdrEventKey ?? "legacy"}:agent.status.changed`),
1299
1964
  paneId: targetAgent.paneId,
@@ -1307,7 +1972,7 @@ export class AgentIndexService {
1307
1972
  if (statusType) {
1308
1973
  return this.#appendAndAckSelfEvent({
1309
1974
  agentId: targetAgent.id,
1310
- compactHistory,
1975
+ compactHistory: compactHistory ?? null,
1311
1976
  herdrSessionName: targetAgent.herdrSessionName,
1312
1977
  idempotencyKey: idempotencyKey(statusType, targetAgent, input.from, input.to, `${observationId}:${input.herdrEventKey ?? "legacy"}:${statusType}`),
1313
1978
  paneId: targetAgent.paneId,
@@ -1324,18 +1989,42 @@ export class AgentIndexService {
1324
1989
  unregister();
1325
1990
  }
1326
1991
  }
1327
- #appendPlanFailedEvent(input) {
1328
- let agent = input.agent;
1992
+ /**
1993
+ * Resolves the metadata a terminal plan outcome is attributed to and whether
1994
+ * the event may keep its `agent_id` foreign key.
1995
+ *
1996
+ * A live `agents` row wins because it carries the current pane generation.
1997
+ * Once the row is gone (pane retired / physically deleted) the plan's own
1998
+ * pane/session/generation are the authority, refined by the newest event of
1999
+ * the same pane + generation that predates the plan, so an old failure is
2000
+ * never stitched onto a newer instance of the same pane. The event is still
2001
+ * written in that case, but as an orphan (`agentRowPresent: false`).
2002
+ */
2003
+ #resolvePlanOutcomeAgent(agent, planCreatedAt) {
1329
2004
  try {
1330
- agent = this.#stores.agents.get(input.agent.id);
2005
+ return { agent: this.#stores.agents.get(agent.id), agentRowPresent: true };
1331
2006
  }
1332
2007
  catch {
1333
- const stitched = this.#minimalAgentFromLatestEvent(input.agent.id, input.agent.herdrSessionName);
1334
- if (stitched)
1335
- agent = stitched;
2008
+ const stitched = this.#minimalAgentFromLatestEvent({
2009
+ agentId: agent.id,
2010
+ // The plan row's own creation time is the upper bound: anything newer on
2011
+ // the same pane belongs to a later instance.
2012
+ createdBefore: planCreatedAt ?? Date.now(),
2013
+ herdrSessionName: agent.herdrSessionName,
2014
+ paneGeneration: agent.paneGeneration ?? null,
2015
+ paneId: agent.paneId,
2016
+ });
2017
+ return { agent: stitched ?? agent, agentRowPresent: false };
1336
2018
  }
2019
+ }
2020
+ #appendPlanFailedEvent(input) {
2021
+ const originalAgentId = input.agent.id;
2022
+ const resolved = this.#resolvePlanOutcomeAgent(input.agent, input.planCreatedAt);
2023
+ const agent = resolved.agent;
2024
+ const fallbackOutcome = input.fallbackOutcome ??
2025
+ ((input.to === "done" || input.to === "blocked") && input.reason === "degraded");
1337
2026
  return this.#appendAndAckSelfEvent({
1338
- agentId: agent.id,
2027
+ agentId: resolved.agentRowPresent ? agent.id : null,
1339
2028
  compactHistory: input.compactHistory ?? null,
1340
2029
  herdrSessionName: agent.herdrSessionName,
1341
2030
  idempotencyKey: `agent.failed:plan:${input.planId}`,
@@ -1343,7 +2032,11 @@ export class AgentIndexService {
1343
2032
  paneGeneration: agent.paneGeneration ?? null,
1344
2033
  payload: {
1345
2034
  agent: agent.agent,
2035
+ // The original id survives in the payload even when the row had to be
2036
+ // written as an orphan, so the failure stays traceable to its agent.
2037
+ agentId: originalAgentId,
1346
2038
  attempts: input.attempts,
2039
+ fallbackOutcome,
1347
2040
  from: input.from,
1348
2041
  herdrSessionName: agent.herdrSessionName,
1349
2042
  name: agent.name,
@@ -1358,6 +2051,36 @@ export class AgentIndexService {
1358
2051
  workspaceId: agent.workspaceId,
1359
2052
  });
1360
2053
  }
2054
+ #appendPlanDiscardedEvent(input) {
2055
+ const originalAgentId = input.agent.id;
2056
+ const resolved = this.#resolvePlanOutcomeAgent(input.agent, input.planCreatedAt);
2057
+ const agent = resolved.agent;
2058
+ return this.#appendAndAckSelfEvent({
2059
+ agentId: resolved.agentRowPresent ? agent.id : null,
2060
+ compactHistory: input.compactHistory ?? null,
2061
+ herdrSessionName: agent.herdrSessionName,
2062
+ idempotencyKey: `agent.discarded:plan:${input.planId}`,
2063
+ paneId: agent.paneId,
2064
+ paneGeneration: agent.paneGeneration ?? null,
2065
+ payload: {
2066
+ agent: agent.agent,
2067
+ agentId: originalAgentId,
2068
+ attempts: input.attempts,
2069
+ from: input.from,
2070
+ herdrSessionName: agent.herdrSessionName,
2071
+ name: agent.name,
2072
+ paneId: agent.paneId,
2073
+ planId: input.planId,
2074
+ reason: input.reason ?? "unknown",
2075
+ terminalId: agent.terminalId,
2076
+ to: input.to,
2077
+ workspaceId: agent.workspaceId,
2078
+ },
2079
+ terminalId: agent.terminalId,
2080
+ type: "agent.discarded",
2081
+ workspaceId: agent.workspaceId,
2082
+ });
2083
+ }
1361
2084
  #appendAndAckSelfEvent(input) {
1362
2085
  const event = this.#stores.agentEvents.append(input);
1363
2086
  const owner = this.#stores.agentOrchestratorScopes?.get({