@dorokuma/herdsman-pi 0.14.2 → 0.14.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/package.json +1 -1
  2. package/src/index.ts +662 -343
package/src/index.ts CHANGED
@@ -23,6 +23,7 @@ import {
23
23
  import {
24
24
  projectAgentOutcomes,
25
25
  formatAgentOutcomeUpdates,
26
+ type AgentOutcome,
26
27
  WAKE_SETTLE_MS,
27
28
  } from "./wake.js";
28
29
  import { loadWakeFilterConfig } from "./wake-filter-config.js";
@@ -121,30 +122,20 @@ type HerdsmanState = {
121
122
  */
122
123
  unackedDelivered: Map<number, AgentEventWireRecord>;
123
124
  wakeDeferredUntilSettled: boolean;
124
- /** Wall-clock start of the current bounded wake deferral, if any. */
125
- wakeDeferredSince: number | undefined;
126
125
  /**
127
- * Set once the hard deferral budget elapsed: the next pass injects the batch
128
- * from the current state instead of deferring again.
129
- */
130
- wakeForcedRelease: boolean;
131
- /**
132
- * Event ids handed to Pi as a *queued* (non-triggering) follow-up whose
133
- * content has not been seen entering the transcript yet.
126
+ * Event ids handed to Pi as a follow-up whose content has not been seen
127
+ * entering the transcript yet.
134
128
  *
135
- * A wake injected while the orchestrator streams is parked in the agent's
136
- * follow-up queue, and Pi only drains that queue when a run reaches its stop
137
- * point. When the run it rode on already passed that point, the update sits in
138
- * the queue until the next user message. This set is what keeps such a
139
- * delivery from being written off, and it carries three guarantees at once:
129
+ * A wake is only ever injected into an idle orchestrator, and it always
130
+ * triggers its own turn, so the copy Pi holds is a normal turning message
131
+ * rather than a queued follow-up nothing may drain. This set is what keeps such
132
+ * an in-flight delivery out of both the acknowledgement path and the
133
+ * transcript, and it carries three guarantees at once:
140
134
  *
141
- * - never lost: while it is non-empty the settlement drives a continuation
142
- * (bounded by MAX_WAKE_CONTINUATION_ATTEMPTS drives) so a later run drains
143
- * the queue and carries the update out. The bound is a real ceiling, not a
144
- * suggestion: once it is spent the ids are released from here and
145
- * acknowledged (`writeOffStrandedWakeDelivery`), because an update that no
146
- * turn will ever carry out must not keep a run loop — and the daemon's
147
- * redelivery of it — alive forever;
135
+ * - never lost: while it is non-empty the next settlement leaves the id
136
+ * pending, so the next `scheduleWake` re-presents it (with the real update
137
+ * content, never with an empty marker run) instead of parking it in a
138
+ * follow-up queue nothing drains;
148
139
  * - never acknowledged unseen: these ids are excluded from the acknowledgement
149
140
  * path, so the daemon keeps them pending and redelivers them when this
150
141
  * session never consumes them (the only way a delivery Pi itself dropped —
@@ -157,7 +148,10 @@ type HerdsmanState = {
157
148
  * evidence: the hidden wake message's `message_end`, which then moves them to
158
149
  * `wakeConsumptionObserved`) or when the event leaves the delivery queue for
159
150
  * good (acknowledged, covered by the acknowledgement watermark,
160
- * dead-lettered), and with the delivery queue on a role/scope reset.
151
+ * dead-lettered), and with the delivery queue on a role/scope reset. A
152
+ * settlement that saw no evidence dead-letters them: it lifts the injection
153
+ * barrier (via `wakeRetryableEventIds`) but not the watermark one, because the
154
+ * content was never confirmed.
161
155
  */
162
156
  wakeAwaitingConsumption: Set<number>;
163
157
  /**
@@ -205,24 +199,45 @@ type HerdsmanState = {
205
199
  */
206
200
  wakeSkipLogReasons: Map<number, string>;
207
201
  /**
208
- * Continuation drives already spent, keyed by event id.
202
+ * Ids whose delivery was dead-lettered and which a later wake must be allowed
203
+ * to present again.
204
+ *
205
+ * `wakeAwaitingConsumption` deliberately keeps holding these ids: unconfirmed
206
+ * content must still block the acknowledgement watermark, because the daemon
207
+ * confirms by `where id <= ?` and letting a larger id through would swallow the
208
+ * dead-lettered one for good. What the write-off changes is only the *other*
209
+ * role of that set: an id that is still awaiting consumption is also barred
210
+ * from every later injection, and a dead-lettered one has to be presented
211
+ * again — it never reached the transcript, so a redelivery is the only remedy.
209
212
  *
210
- * The bound is per event, never per delivery and never per session: an
211
- * unrelated event being injected must not hand a stuck one a fresh budget. A
212
- * session that keeps receiving new updates used to reset a shared counter at
213
- * every injection, so the one id no run ever drained drove on unboundedly and
214
- * `MAX_WAKE_CONTINUATION_ATTEMPTS` never fired — the loop that flooded the
215
- * orchestrator session with empty turns.
213
+ * An id leaves the set as soon as a wake hands it to Pi again (it is in flight
214
+ * once more, so the ordinary `awaiting` guard applies), when the daemon
215
+ * confirms it, or on a role/scope reset.
216
216
  *
217
- * `MAX_WAKE_CONTINUATION_ATTEMPTS` is therefore a hard ceiling per id: the
218
- * drive that reaches it writes that id off
219
- * (`writeOffStrandedWakeDelivery`), and an id that still has drives left keeps
220
- * both its copy and its count even when a sibling is written off next to it.
217
+ * Membership here does *not* by itself authorise a re-presentation: an
218
+ * unconditional pass turned every write-off into an immediate re-injection on
219
+ * the settlement that produced it, and an injection that fails the same way
220
+ * dead-letters the id again — an unbounded turn loop with no backoff and no
221
+ * limit. `representableWakeOutcomes` gates that instead (a wake carried by
222
+ * fresh content, or an explicit backoff with a bounded attempt budget).
223
+ */
224
+ wakeRetryableEventIds: Set<number>;
225
+ /**
226
+ * Automatic re-presentations already spent per dead-lettered event id.
221
227
  *
222
- * An entry leaves the map with the id itself: acknowledged, dead-lettered,
223
- * consumed (its copy reached the transcript), or a role/scope reset.
228
+ * Counted here rather than on the wire record because the budget is a
229
+ * property of *this* client's retry policy: the daemon's own `attempts`
230
+ * counter tracks acknowledgement failures, which are a different thing.
224
231
  */
225
- wakeContinuationDrives: Map<number, number>;
232
+ wakeDeadLetterAttempts: Map<number, number>;
233
+ /** Earliest timestamp at which an id's next automatic retry may fire. */
234
+ wakeDeadLetterRetryAt: Map<number, number>;
235
+ /**
236
+ * Consecutive ticks spent waiting for the orchestrator, i.e. the position in
237
+ * `WAKE_BUSY_BACKOFF_MS`. Reset when the orchestrator settles or the wake is
238
+ * cancelled, so a new busy period starts at the first rung again.
239
+ */
240
+ wakeBusySpinRung: number;
226
241
  wakeRequested: boolean;
227
242
  wakeRequestedThroughEventId: number;
228
243
  wakeTimer: ReturnType<typeof setTimeout> | undefined;
@@ -279,53 +294,93 @@ const RECONNECTING_MESSAGE = "Herdsman is reconnecting · try again shortly";
279
294
  export const MAX_ACK_ATTEMPTS = 5;
280
295
  export const ACK_BACKOFF_CAP_MS = 30_000;
281
296
  const KEEPALIVE_INTERVAL_MS = 30_000;
282
- /** Retry interval used while a wake cannot be injected (busy orchestrator). */
297
+ /**
298
+ * First tick of the busy poll ladder, and the first rung of
299
+ * `WAKE_BUSY_BACKOFF_MS`.
300
+ *
301
+ * Kept as its own export for callers/tests that only care about the latency of
302
+ * the first deferral; everything else about the wait is in the ladder.
303
+ */
283
304
  export const WAKE_BUSY_SPIN_MS = 100;
284
305
  /**
285
- * Hard upper bound for every deferred wake. Once it elapses the scheduler stops
286
- * waiting for the orchestrator to become idle or for a settlement to arrive and
287
- * forces the injection decision from the current state, so a wake can never be
288
- * parked forever.
306
+ * Ceiling the busy poll ladder settles on for an arbitrarily long turn.
307
+ *
308
+ * The ladder does not simply stop at its last rung: `scheduleDeferredWake`
309
+ * clamps every tick to this value (the same shape `ackBackoffMs` uses for
310
+ * `ACK_BACKOFF_CAP_MS`), so raising a rung above the ceiling cannot silently
311
+ * raise the real cap. The ladder's top rung is this constant, which keeps the
312
+ * exported value, the comment, the decision note and the timed behaviour one
313
+ * and the same number.
289
314
  */
290
- export const WAKE_DEFERRED_TIMEOUT_MS = 5_000;
315
+ export const WAKE_BUSY_BACKOFF_CAP_MS = 5_000;
291
316
  /**
292
- * Upper bound on the continuation drives spent on **one** unconsumed wake
293
- * event id.
317
+ * Poll-frequency ladder while a wake waits for the orchestrator (milliseconds).
294
318
  *
295
- * Every drive costs a full agent run, so the budget is deliberately small: three
296
- * attempts already cover the drive that follows the settlement which missed Pi's
297
- * follow-up queue plus two retries after intervening runs that also ended before
298
- * their stop point, and five adds margin for a run that spent its stop point on
299
- * a tool call. Beyond that the cause is systemic — the runs keep ending early, so
300
- * starting another one could only produce another empty turn — and that id is
301
- * written off instead (`writeOffStrandedWakeDelivery`): it leaves
302
- * `wakeAwaitingConsumption` and is acknowledged, which stops both the
303
- * continuation loop and the daemon's redelivery of the same event.
319
+ * Delivery does not branch on busy versus idle: every wake is queued and is
320
+ * injected once the orchestrator has settled (or a poll observes it idle), and
321
+ * a busy one cannot be interrupted anyway. `agent_settled` is therefore the
322
+ * primary signal that closes a deferral — it fires exactly when the turn ends
323
+ * and re-takes the wake decision — so the liveness bound is "until
324
+ * `agent_settled`, or until one of these ticks observes an idle orchestrator",
325
+ * never a deadline. The ladder exists to make that wait cheap, and it is a
326
+ * poll-frequency reduction, not a safety net: there is no deadline to miss. It
327
+ * used to tick at a flat 100ms forever, which made the extension re-evaluate
328
+ * the same deferral ten times a second for the whole turn and burn the turn's
329
+ * wall clock without ever being able to inject anything.
304
330
  *
305
- * The bound is per event id (`wakeContinuationDrives`) and is never reset by a
306
- * later injection: a session that keeps receiving new updates must not hand an
307
- * already-stuck id a fresh ceiling, or the loop it was meant to stop continues
308
- * for exactly the events that are stuck.
331
+ * Growing the gap keeps the correctness contract (nothing is injected until
332
+ * the orchestrator is idle, however long that takes) while making the poll
333
+ * cheap: the tick count per turn drops from ~10/s to at most ~2 in the first
334
+ * second and then ~1/s. The last rung is `WAKE_BUSY_BACKOFF_CAP_MS`, so a turn
335
+ * that outlives the ladder is re-evaluated at most once every 5s.
336
+ */
337
+ export const WAKE_BUSY_BACKOFF_MS = [
338
+ WAKE_BUSY_SPIN_MS,
339
+ 200,
340
+ 500,
341
+ 1_000,
342
+ 2_000,
343
+ WAKE_BUSY_BACKOFF_CAP_MS,
344
+ ] as const;
345
+ /**
346
+ * How many times a dead-lettered wake event may be re-presented automatically
347
+ * before the client stops driving turns for it.
309
348
  *
310
- * The trade-off is deliberate and asymmetric: losing one agent update is
311
- * recoverable (the agent is still there and its transcript can be read
312
- * directly), while an unbounded continuation loop floods the orchestrator
313
- * session with empty turns and makes it unusable. A missed update is therefore
314
- * strictly better than an endless one.
349
+ * Two, because a dead-letter means the content never reached the transcript and
350
+ * the usual causes (a turn that ended before its stop point, an abort, a
351
+ * message Pi dropped) leave no evidence that an immediate retry helps: one
352
+ * extra attempt covers a genuinely transient hiccup, the second covers a
353
+ * hiccup that also broke the first retry, and a third would only turn the
354
+ * retry into the unbounded turn-churn this cap exists to prevent. Each attempt
355
+ * is separated by `WAKE_DEAD_LETTER_RETRY_BACKOFF_MS`, so the bounded worst case
356
+ * is ~20s of automatic effort; after that the pending update is the human's
357
+ * (the write-off notify already says so), and the id stays unacked as the
358
+ * watermark barrier until a new external event merges it into a wake.
315
359
  */
316
- export const MAX_WAKE_CONTINUATION_ATTEMPTS = 5;
317
- /** `customType` of the hidden wake context this extension injects. */
318
- const WAKE_CONTEXT_CUSTOM_TYPE = "herdsman-wake-context";
319
- /** `customType` of the hidden marker that drives a missed wake continuation. */
320
- const WAKE_CONTINUATION_CUSTOM_TYPE = "herdsman-wake-continuation";
360
+ export const WAKE_DEAD_LETTER_RETRY_LIMIT = 2;
321
361
  /**
322
- * Content of the continuation marker. Its only job is to start a run
323
- * (`triggerTurn: true`) so the run's loop drains the queued follow-up that was
324
- * never delivered; the wake content itself is not repeated here, so the
325
- * evidence is not presented twice.
362
+ * Delay before the n-th automatic re-presentation of a dead-lettered event
363
+ * (milliseconds), i.e. `[0] === 5_000` for the first retry, `[1] === 15_000` for
364
+ * the second.
365
+ *
366
+ * Deliberately far from the 100ms busy tick: the retry must not read as a spin,
367
+ * and the settlement that produced the dead-letter has just finished, so an
368
+ * immediate re-injection would re-run exactly the turn that just failed.
326
369
  */
327
- const WAKE_CONTINUATION_CONTENT =
328
- "[HERDSMAN WAKE CONTINUATION]\nA queued Herdsman agent update was not delivered by the previous turn; it follows this message. Handle it, and do not start unrelated work.";
370
+ const WAKE_DEAD_LETTER_RETRY_BACKOFF_MS = [5_000, 15_000] as const;
371
+ /**
372
+ * Window after which an unchanged watermark hold-back is noted again
373
+ * (milliseconds).
374
+ *
375
+ * The hold-back note is deduplicated on its exact state (blocked id, reason,
376
+ * count), so a stable stall is logged once when it starts; this window is what
377
+ * keeps a stall that outlives that first line visible instead of silently
378
+ * freezing the queue forever, while still capping the noise at one line per
379
+ * minute per state.
380
+ */
381
+ const WAKE_BARRIER_NOTE_INTERVAL_MS = 60_000;
382
+ /** `customType` of the hidden wake context this extension injects. */
383
+ const WAKE_CONTEXT_CUSTOM_TYPE = "herdsman-wake-context";
329
384
 
330
385
  type AckFailureClass = "terminal" | "resync" | "transient";
331
386
 
@@ -409,13 +464,14 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
409
464
  subscriberId: undefined,
410
465
  unackedDelivered: new Map(),
411
466
  wakeDeferredUntilSettled: false,
412
- wakeDeferredSince: undefined,
413
- wakeForcedRelease: false,
414
467
  wakeAwaitingConsumption: new Set(),
415
468
  wakeConsumptionObserved: new Set(),
416
469
  wakeSuppressedEventIds: new Set(),
417
470
  wakeSkipLogReasons: new Map(),
418
- wakeContinuationDrives: new Map(),
471
+ wakeRetryableEventIds: new Set(),
472
+ wakeDeadLetterAttempts: new Map(),
473
+ wakeDeadLetterRetryAt: new Map(),
474
+ wakeBusySpinRung: 0,
419
475
  wakeRequested: false,
420
476
  wakeRequestedThroughEventId: 0,
421
477
  wakeTimer: undefined,
@@ -425,6 +481,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
425
481
  let activeContext: PiContext | undefined;
426
482
  let keepaliveTimer: ReturnType<typeof setInterval> | undefined;
427
483
  let wakeGeneration = 0;
484
+ // Throttle state for the watermark hold-back note (see
485
+ // `noteWakeWatermarkHoldBack`): the last noted barrier state and when it was
486
+ // noted, so a stable stall notes itself once and a long one re-notes itself
487
+ // per window instead of flooding the log.
488
+ let lastWakeBarrierNote: { at: number; key: string } | undefined;
428
489
 
429
490
  const stopKeepalive = () => {
430
491
  if (!keepaliveTimer) return;
@@ -442,15 +503,24 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
442
503
 
443
504
  const setHerdsmanUi = (ctx: PiContext | undefined) => {
444
505
  if (!ctx) return;
506
+ const pendingOutcomeCount = projectAgentOutcomes(state.pendingEvents, wakeFilter).outcomes.length;
445
507
  const footerState: HerdsmanFooterState = state.reconnectingFromOn
446
508
  ? { kind: "reconnecting" }
447
509
  : state.isOrchestrator
448
- ? {
449
- kind: "on",
450
- updateCount: projectAgentOutcomes(state.pendingEvents, wakeFilter).outcomes.length,
451
- }
510
+ ? { kind: "on", updateCount: pendingOutcomeCount }
452
511
  : { kind: "off" };
453
512
  ctx.ui.setStatus?.("herdsman", formatHerdsmanFooterStatus(footerState));
513
+ // A held-back wake is otherwise invisible during a long turn: updates are
514
+ // pending and nothing is injected, so the count alone reads as "Herdsman
515
+ // has work queued" instead of "the orchestrator is busy and this is what
516
+ // it is waiting for". The hint gets its own line (the pending count stays a
517
+ // count) and disappears the moment the wake is released or the queue drains.
518
+ ctx.ui.setStatus?.(
519
+ "herdsman-wake",
520
+ state.isOrchestrator && state.wakeDeferredUntilSettled && pendingOutcomeCount > 0
521
+ ? "Herdsman · waiting for the current turn to end"
522
+ : undefined,
523
+ );
454
524
  };
455
525
 
456
526
  const cancelWakeTimer = () => {
@@ -458,8 +528,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
458
528
  if (state.wakeTimer) clearTimeout(state.wakeTimer);
459
529
  state.wakeTimer = undefined;
460
530
  state.wakeDeferredUntilSettled = false;
461
- state.wakeDeferredSince = undefined;
462
- state.wakeForcedRelease = false;
531
+ state.wakeBusySpinRung = 0;
463
532
  };
464
533
 
465
534
  const cancelWake = () => {
@@ -512,15 +581,18 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
512
581
  * (and holding its batch open) for content that can never arrive. The other
513
582
  * two sets describe the same ids and follow the queue just as well: a
514
583
  * confirmed id needs no evidence flag, and an id that is gone from the queue
515
- * cannot be re-injected, so it needs no suppression either.
584
+ * cannot be re-injected, so it needs no suppression either. The dead-letter
585
+ * retry bookkeeping follows for the same reason: a confirmed event needs no
586
+ * backoff and no attempt budget.
516
587
  */
517
588
  const dropUnackedDelivered = (eventId: number): void => {
518
589
  state.unackedDelivered.delete(eventId);
519
590
  state.wakeAwaitingConsumption.delete(eventId);
520
591
  state.wakeConsumptionObserved.delete(eventId);
521
592
  state.wakeSuppressedEventIds.delete(eventId);
522
- // The id can no longer owe a drive, so its own budget leaves with it.
523
- state.wakeContinuationDrives.delete(eventId);
593
+ state.wakeRetryableEventIds.delete(eventId);
594
+ state.wakeDeadLetterAttempts.delete(eventId);
595
+ state.wakeDeadLetterRetryAt.delete(eventId);
524
596
  };
525
597
 
526
598
  /**
@@ -550,7 +622,22 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
550
622
  if (carriedOver.length > 0) {
551
623
  logHerdsmanPi(
552
624
  "info",
553
- `[herdsman-pi] keeping ${carriedOver.length} unconsumed wake event id(s) suppressed across the scope change eventIds=${carriedOver.join(",")} · daemon redelivery for them is traded away to keep the transcript single-copy`,
625
+ `[herdsman-pi] keeping ${carriedOver.length} unconsumed wake event id(s) suppressed on a delivery reset eventIds=${carriedOver.join(",")} · daemon redelivery for them is traded away to keep the transcript single-copy`,
626
+ );
627
+ }
628
+ // Written-off ids leave with the same reset: they are still in the delivery
629
+ // queue (unconfirmed content must keep blocking the watermark), so without
630
+ // this line nothing would say which dead letters stopped being retried here.
631
+ // This function only copies ids from the awaiting set (`wakeAwaitingConsumption`)
632
+ // into the suppression set, so a written-off id is *not* suppressed: if the
633
+ // daemon redelivers it the update may be injected again. What the reset costs is
634
+ // the local retry bookkeeping for it; this function does not assert whether the
635
+ // daemon still holds the update.
636
+ const writtenOff = [...state.wakeRetryableEventIds].sort((left, right) => left - right);
637
+ if (writtenOff.length > 0) {
638
+ logHerdsmanPi(
639
+ "info",
640
+ `[herdsman-pi] dropping ${writtenOff.length} written-off wake event id(s) on a delivery reset eventIds=${writtenOff.join(",")} · only the local dead-letter retry bookkeeping is dropped here; these ids do not join the suppression set, so if the daemon redelivers them the update may be injected again`,
554
641
  );
555
642
  }
556
643
  state.presentedEventIds.clear();
@@ -560,7 +647,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
560
647
  state.wakeAwaitingConsumption.clear();
561
648
  state.wakeConsumptionObserved.clear();
562
649
  state.wakeSkipLogReasons.clear();
563
- state.wakeContinuationDrives.clear();
650
+ state.wakeRetryableEventIds.clear();
651
+ // The automatic retry budget is a property of the id inside this scope: a
652
+ // scope that never held the id cannot owe it a retry, and keeping the
653
+ // counter would silently make the next scope's first write-off final.
654
+ state.wakeDeadLetterAttempts.clear();
655
+ state.wakeDeadLetterRetryAt.clear();
656
+ state.wakeBusySpinRung = 0;
564
657
  };
565
658
 
566
659
  const pruneAcknowledgedEvents = (ackedEventId: number | undefined) => {
@@ -590,6 +683,30 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
590
683
  for (const eventId of [...state.wakeSkipLogReasons.keys()]) {
591
684
  if (eventId <= ackedEventId) state.wakeSkipLogReasons.delete(eventId);
592
685
  }
686
+ // A confirmed event is neither re-presentable nor owed a retry, so the three
687
+ // fields that describe a dead letter for it are swept together, here and in
688
+ // `dropUnackedDelivered`: the flag (may a wake present it again?), the spent
689
+ // attempt counter (does it still owe automatic retries?) and the retry
690
+ // deadline (when is the next one due?). Keeping any of them for an id the
691
+ // daemon has confirmed would let a later redelivery treat a settled event as
692
+ // an unfinished one — `alreadyPresented` would wave the id through and
693
+ // `deadLetterRetryDue` would hand it a budget it no longer needs.
694
+ //
695
+ // Ids still in the delivery queue are swept by the queue loop above (which
696
+ // routes through `dropUnackedDelivered`); this loop is what keeps the same
697
+ // three fields consistent for an id the queue no longer holds.
698
+ for (const eventId of [...state.wakeRetryableEventIds]) {
699
+ if (eventId <= ackedEventId) state.wakeRetryableEventIds.delete(eventId);
700
+ }
701
+ // A confirmed event owes no retry, so the dead-letter bookkeeping for it is
702
+ // garbage from here on: the counter would otherwise make the id's next
703
+ // write-off (after a reset) start from a spent budget.
704
+ for (const eventId of [...state.wakeDeadLetterAttempts.keys()]) {
705
+ if (eventId <= ackedEventId) {
706
+ state.wakeDeadLetterAttempts.delete(eventId);
707
+ state.wakeDeadLetterRetryAt.delete(eventId);
708
+ }
709
+ }
593
710
  };
594
711
 
595
712
  /**
@@ -619,9 +736,9 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
619
736
  * Whether an event was already handed to the orchestrator in this session and
620
737
  * therefore must not be injected again (logging why, at most once per reason).
621
738
  *
622
- * - `awaiting`: a copy of it was handed over as a queued follow-up that no run
623
- * has drained yet, and the continuation is what carries it out (checked
624
- * first: for a queued copy this is the state that explains the redelivery);
739
+ * - `awaiting`: a copy of it was handed to Pi and has not been seen in the
740
+ * transcript yet (a dead-lettered one is re-presentable instead, see
741
+ * `wakeRetryableEventIds`, and is therefore checked first);
625
742
  * - `presented`: it was injected before (or its content has since been observed
626
743
  * in the transcript), so a redelivery would duplicate it;
627
744
  * - `suppressed`: its copy may still sit in Pi's process-wide follow-up queue
@@ -629,6 +746,14 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
629
746
  * only one that must not be shown (see `wakeSuppressedEventIds`).
630
747
  */
631
748
  const alreadyPresented = (eventId: number): boolean => {
749
+ if (state.wakeRetryableEventIds.has(eventId)) {
750
+ // Its delivery was dead-lettered, so the content never reached the
751
+ // transcript: a redelivery is the only remedy and must not be skipped.
752
+ // *When* that redelivery may happen is a separate question, answered by
753
+ // `representableWakeOutcomes`; this predicate only says "no copy of this
754
+ // id is rightfully in the transcript".
755
+ return false;
756
+ }
632
757
  if (state.wakeAwaitingConsumption.has(eventId)) {
633
758
  noteSkippedWakeInjection(eventId, "awaiting");
634
759
  return true;
@@ -644,6 +769,92 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
644
769
  return false;
645
770
  };
646
771
 
772
+ /**
773
+ * Backoff before the n-th automatic re-presentation of a dead-lettered id.
774
+ *
775
+ * The deadline and the budget are read together, so this maps "attempts
776
+ * already spent" to "how long this id must now stay quiet": 0 spent → 5s,
777
+ * 1 spent → 15s, anything beyond the budget is unreachable (the caller stops
778
+ * asking once `WAKE_DEAD_LETTER_RETRY_LIMIT` is reached).
779
+ */
780
+ const deadLetterRetryBackoffMs = (attempts: number): number => {
781
+ const ladder = WAKE_DEAD_LETTER_RETRY_BACKOFF_MS;
782
+ const rung = Math.min(Math.max(attempts, 0), ladder.length - 1);
783
+ // `noUncheckedIndexedAccess` types every index access as optional; `rung` is
784
+ // clamped into a non-empty ladder, so the fallback is unreachable.
785
+ return ladder[rung] ?? ladder[0];
786
+ };
787
+
788
+ /**
789
+ * Whether a dead-lettered id's automatic retry is due and still budgeted.
790
+ *
791
+ * This answers the question "may this id drive a turn of its own right
792
+ * now?", which is deliberately much narrower than "may it be presented
793
+ * again": a dead-lettered id may also ride a wake that fresh content leads
794
+ * (see `representableWakeOutcomes`), and that ride costs it nothing.
795
+ *
796
+ * The deadline is set when the id is written off (`writeOffStrandedWakeDelivery`)
797
+ * and the budget is `WAKE_DEAD_LETTER_RETRY_LIMIT`, so standing on its own
798
+ * means: not before the backoff has elapsed, and not after the last attempt
799
+ * has been spent. Once it is spent the id stops driving turns entirely — it
800
+ * stays in `unackedDelivered` as the watermark barrier and in the pending
801
+ * projection with its human-visible warning, and only fresh content can
802
+ * still carry it.
803
+ */
804
+ const deadLetterRetryDue = (eventId: number): boolean => {
805
+ if (!state.wakeRetryableEventIds.has(eventId)) return false;
806
+ if ((state.wakeDeadLetterAttempts.get(eventId) ?? 0) >= WAKE_DEAD_LETTER_RETRY_LIMIT) {
807
+ return false;
808
+ }
809
+ return (state.wakeDeadLetterRetryAt.get(eventId) ?? 0) <= Date.now();
810
+ };
811
+
812
+ /** Earliest moment a still-budgeted id's automatic retry may fire. */
813
+ const deadLetterRetryAt = (eventId: number): number | undefined => {
814
+ if (!state.wakeRetryableEventIds.has(eventId)) return undefined;
815
+ if ((state.wakeDeadLetterAttempts.get(eventId) ?? 0) >= WAKE_DEAD_LETTER_RETRY_LIMIT) {
816
+ return undefined;
817
+ }
818
+ return state.wakeDeadLetterRetryAt.get(eventId);
819
+ };
820
+
821
+ /**
822
+ * The outcomes a wake may present: everything presentable that is not
823
+ * dead-lettered, plus the dead-lettered ids that may join it.
824
+ *
825
+ * A dead-letter used to be an unconditional pass through `alreadyPresented`,
826
+ * and because the settlement that produced it immediately scheduled the next
827
+ * wake, that turned every failed wake turn into the next wake turn: an
828
+ * unbounded loop with no backoff and no ceiling. The remedy is to separate
829
+ * the two ways an id can come back:
830
+ *
831
+ * - *rides*: the wake is led by content that is not a dead letter — an id this
832
+ * session has not handed over yet, or a genuinely new external event. That
833
+ * turn belongs to the fresh content, and merging the dead-lettered id into
834
+ * it is exactly the recovery it needs (the batch merge invariant), so it
835
+ * costs none of the id's retry budget;
836
+ * - *retries*: no fresh content is left, and the id waits for its explicit
837
+ * backoff (`deadLetterRetryDue`) and then spends one automatic attempt.
838
+ *
839
+ * Everything else leaves the id alone: the settlement that wrote it off, a
840
+ * daemon redelivery of the same id, an acknowledgement sweep, a reconnect,
841
+ * or an exhausted budget. The id is not lost in that case — it keeps blocking
842
+ * the watermark, keeps its pending row and its human-visible warning, and is
843
+ * still picked up by the next wake that fresh content leads.
844
+ */
845
+ const representableWakeOutcomes = (
846
+ candidates: readonly AgentOutcome[],
847
+ ): { lead: readonly AgentOutcome[]; outcomes: readonly AgentOutcome[] } => {
848
+ const lead = candidates.filter(
849
+ (outcome) => !state.wakeRetryableEventIds.has(outcome.eventId),
850
+ );
851
+ if (lead.length > 0) return { lead, outcomes: candidates };
852
+ return {
853
+ lead,
854
+ outcomes: candidates.filter((outcome) => deadLetterRetryDue(outcome.eventId)),
855
+ };
856
+ };
857
+
647
858
  const isWakeableEvent = (event: AgentEventWireRecord | undefined) =>
648
859
  !event?.nextAttemptAt || event.nextAttemptAt <= Date.now();
649
860
 
@@ -666,8 +877,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
666
877
  }
667
878
  const ackResponse = (await state.client.request("agent.notifications.ack", {
668
879
  eventId: event.id,
669
- })) as { ackedEventId?: number; state?: { ackedEventId?: number } } | undefined;
670
- pruneAcknowledgedEvents(ackResponse?.ackedEventId ?? ackResponse?.state?.ackedEventId);
880
+ })) as { state?: { ackedEventId?: number } } | undefined;
881
+ // The ack contract has no top-level field: the daemon answers
882
+ // `{ acknowledged: true, state: toWireState(state) }` (see
883
+ // `src/daemon/observability-server.ts`), so reading only `state` here
884
+ // is also fail-closed — `pruneAcknowledgedEvents(undefined)` is a no-op.
885
+ pruneAcknowledgedEvents(ackResponse?.state?.ackedEventId);
671
886
  state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
672
887
  // The event is confirmed: it leaves the delivery queue for good, which
673
888
  // is what keeps the acknowledgement watermark monotonic (ids are acked
@@ -846,30 +1061,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
846
1061
  }, WAKE_SETTLE_MS);
847
1062
  };
848
1063
 
849
- /**
850
- * Ends the current wake-deferral episode without injecting.
851
- *
852
- * `wakeForcedRelease` and `wakeDeferredSince` describe *one* bounded
853
- * deferral: the released deadline is re-derived from `wakeDeferredSince`
854
- * every time `scheduleDeferredWake` runs, so a stale pair would let an
855
- * unrelated later wake bypass the busy gate. Every path that ends a wake
856
- * pass without injecting clears both, which gives the next deferral its own
857
- * full `WAKE_DEFERRED_TIMEOUT_MS` budget. The 5s hard deadline itself is
858
- * unchanged: it is measured inside a single episode, and an episode that
859
- * reaches it still force-releases the batch on its next pass.
860
- */
861
- const endWakeDeferral = () => {
862
- state.wakeForcedRelease = false;
863
- state.wakeDeferredSince = undefined;
864
- };
865
-
866
1064
  /**
867
1065
  * Event ids a hidden wake message proves to have reached the transcript.
868
1066
  *
869
1067
  * A wake injection names the outcomes it presents in
870
1068
  * `details.presentedEventIds`, and Pi writes those details onto the session
871
- * entry it emits on `message_end` (both for a triggered turn and when a run
872
- * finally drains the queued follow-up). That emission is the only consumption
1069
+ * entry it emits on `message_end`. That emission is the only consumption
873
1070
  * evidence there is: nothing else tells us that the content — not merely the
874
1071
  * request that carried it — reached the orchestrator.
875
1072
  *
@@ -909,6 +1106,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
909
1106
  * blocks the prefix: leaving it unacknowledged keeps it pending, which is what
910
1107
  * lets the daemon redeliver the one copy that never arrived.
911
1108
  *
1109
+ * An id awaiting consumption is the in-flight state of a delivery that has been
1110
+ * handed to Pi but whose `message_end` has not been seen yet. It blocks the
1111
+ * prefix, and the settlement that still finds it there dead-letters it
1112
+ * (`writeOffStrandedWakeDelivery`): the content never reached the transcript, so
1113
+ * it must stay pending in the daemon and be eligible for the next wake instead
1114
+ * of being confirmed by a heuristic.
1115
+ *
912
1116
  * An event whose own acknowledgement already failed is left out (the daemon's
913
1117
  * cursor advance sweeps it, and a retry would reset its attempt/backoff
914
1118
  * accounting), but it must not hold the prefix back: it stays in the queue until
@@ -927,7 +1131,19 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
927
1131
  const confirmablePrefix: AgentEventWireRecord[] = [];
928
1132
  let blockedByMissingTurn = false;
929
1133
  for (const event of deliveryQueue) {
930
- if (state.wakeAwaitingConsumption.has(event.id)) break;
1134
+ // An id that was handed to Pi and whose evidence has not arrived yet
1135
+ // blocks the prefix: it has no evidence at all, and the settlement still
1136
+ // owes it either that evidence (a run drains the copy) or a dead-letter.
1137
+ // A dead-lettered id keeps blocking it for the same reason — unconfirmed
1138
+ // content must never be swept by the watermark — while an id that is not
1139
+ // awaiting consumption and not observed otherwise looks like a delivered,
1140
+ // triggered prompt and would be confirmed by the turn heuristic.
1141
+ if (
1142
+ state.wakeAwaitingConsumption.has(event.id) ||
1143
+ state.wakeRetryableEventIds.has(event.id)
1144
+ ) {
1145
+ break;
1146
+ }
931
1147
  if (!state.wakeConsumptionObserved.has(event.id) && !turnProducedFinalResponse) {
932
1148
  blockedByMissingTurn = true;
933
1149
  break;
@@ -939,161 +1155,219 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
939
1155
  }
940
1156
  confirmablePrefix.push(event);
941
1157
  }
942
- // Any id still awaiting consumption keeps the batch in flight: it owns the
943
- // acknowledgement cursor, so the settlement of the run that finally drains
944
- // the queued copy (the continuation driven by the settle handler) confirms it
945
- // then. Reading the queue directly — instead of comparing two filtered lists —
946
- // keeps that decision independent of the `attempts` exclusion above, which
947
- // would otherwise hide an unconsumed id and drop the batch too early.
948
- const stillAwaitingConsumption = deliveryQueue.some((event) =>
949
- state.wakeAwaitingConsumption.has(event.id),
950
- );
951
- return { deliveryQueue, confirmablePrefix, blockedByMissingTurn, stillAwaitingConsumption };
1158
+ // Nothing else keeps the batch in flight once the settlement has decided:
1159
+ // any id that is still awaiting consumption has been dead-lettered above,
1160
+ // which lifts only the injection barrier — it still blocks the watermark.
1161
+ return { deliveryQueue, confirmablePrefix, blockedByMissingTurn };
952
1162
  };
953
1163
 
954
- const writeOffStrandedWakeDelivery = (eventIds: number[], ctx: PiContext): void => {
955
- // Release first, acknowledge second: the release is what breaks the loop
956
- // synchronously, while the acknowledgement is an RPC that may fail.
957
- for (const eventId of eventIds) {
958
- state.wakeAwaitingConsumption.delete(eventId);
959
- // This set only ever holds the id of a delivery that was already handed
960
- // to Pi, so releasing it is not permission to present it again: the
961
- // session-wide presented guard stays behind and keeps a daemon replay
962
- // of the same id out of the transcript.
963
- state.presentedEventIds.add(eventId);
1164
+ /**
1165
+ * The smallest delivery-queue id that still blocks the acknowledgement
1166
+ * watermark, plus why it blocks and how many ids it holds back.
1167
+ *
1168
+ * This is the barrier `confirmableDeliveryPrefix` walks (the `break` in its
1169
+ * loop), read on its own because the silent upstream-error acknowledgement
1170
+ * must respect it without a turn authorising a prefix: the daemon confirms
1171
+ * by watermark (`update agent_events set status = 'acked' where id <= ?`), so
1172
+ * an acknowledgement at or above this id would hand the daemon the authority
1173
+ * to mark it confirmed without its content ever reaching the transcript —
1174
+ * the "busy delivery never arrives" failure in its silent form, with no
1175
+ * redelivery and no warning anywhere afterwards.
1176
+ *
1177
+ * An id awaiting consumption has no evidence at all yet, and a dead-lettered
1178
+ * one never reached the transcript: exactly the two states that must not be
1179
+ * confirmed by an ack meant for someone else's id. Ids the queue no longer
1180
+ * holds are not barriers — a confirmed id is done, and an id the daemon
1181
+ * itself terminally dead-lettered was its decision, made visible when it
1182
+ * happened.
1183
+ */
1184
+ const unconfirmedWatermarkBarrier = ():
1185
+ | { blocked: number; eventId: number; reason: string }
1186
+ | undefined => {
1187
+ // How many ids the barrier sets hold (members of the two sets, not the
1188
+ // number of later ids the watermark is held back over): awaiting
1189
+ // consumption, or still owed a dead-letter recovery (the two sets this
1190
+ // walk checks).
1191
+ const blocked = new Set([
1192
+ ...state.wakeAwaitingConsumption,
1193
+ ...state.wakeRetryableEventIds,
1194
+ ]).size;
1195
+ for (const queued of unackedDeliveredAscending()) {
1196
+ if (state.wakeRetryableEventIds.has(queued.id)) {
1197
+ return {
1198
+ blocked,
1199
+ eventId: queued.id,
1200
+ reason:
1201
+ (state.wakeDeadLetterAttempts.get(queued.id) ?? 0) >= WAKE_DEAD_LETTER_RETRY_LIMIT
1202
+ ? "dead-letter-budget-exhausted"
1203
+ : "dead-letter-backoff",
1204
+ };
1205
+ }
1206
+ if (state.wakeAwaitingConsumption.has(queued.id)) {
1207
+ return { blocked, eventId: queued.id, reason: "awaiting-consumption" };
1208
+ }
964
1209
  }
965
- // The written-off ids can no longer owe a drive, so their budgets leave
966
- // with them. Every id that is still awaiting consumption keeps its own
967
- // budget: it has not spent its own MAX_WAKE_CONTINUATION_ATTEMPTS drives
968
- // yet, and resetting it here is what used to let a sibling's fresh budget
969
- // cover for an unrelated stuck id.
970
- for (const eventId of eventIds) state.wakeContinuationDrives.delete(eventId);
971
- const stranded = eventIds
972
- .map((eventId) => state.unackedDelivered.get(eventId))
973
- .filter((event): event is AgentEventWireRecord => event !== undefined)
974
- .sort((left, right) => left.id - right.id);
1210
+ return undefined;
1211
+ };
1212
+
1213
+ /**
1214
+ * Notes — once per state, and again after `WAKE_BARRIER_NOTE_INTERVAL_MS` —
1215
+ * that the acknowledgement watermark is being held back on purpose.
1216
+ *
1217
+ * The hold-back is invisible otherwise: the queue simply stops advancing and
1218
+ * the log shows nothing, so a stalled delivery cannot be told apart from an
1219
+ * idle one. The line names the smallest blocked id, why it blocks
1220
+ * (dead-letter backoff, spent dead-letter budget, or awaiting consumption)
1221
+ * and how many ids form the barrier, in the same field style as the write-off
1222
+ * warning and the injection-skip note. Deduplicating on the exact state
1223
+ * keeps a long stall from flooding the file; the window keeps a stall that
1224
+ * outlives its first line visible.
1225
+ *
1226
+ * "Long-term" cannot be decided from inside the extension — a barrier that
1227
+ * is about to be released looks exactly like one that never will be — so the
1228
+ * minimal observable contract is this: note the moment a due
1229
+ * acknowledgement is declined because of the barrier, not when a timer
1230
+ * merely fires.
1231
+ */
1232
+ const noteWakeWatermarkHoldBack = (barrier: {
1233
+ blocked: number;
1234
+ eventId: number;
1235
+ reason: string;
1236
+ }): void => {
1237
+ const key = `${barrier.eventId}:${barrier.reason}:${barrier.blocked}`;
1238
+ const now = Date.now();
1239
+ if (
1240
+ lastWakeBarrierNote !== undefined &&
1241
+ lastWakeBarrierNote.key === key &&
1242
+ now - lastWakeBarrierNote.at < WAKE_BARRIER_NOTE_INTERVAL_MS
1243
+ ) {
1244
+ return;
1245
+ }
1246
+ lastWakeBarrierNote = { at: now, key };
975
1247
  logHerdsmanPi(
976
- "warn",
977
- `[herdsman-pi] wake continuation gave up eventIds=${eventIds.join(",")} drives=${MAX_WAKE_CONTINUATION_ATTEMPTS} · released and acknowledged as consumed so the daemon stops redelivering them`,
1248
+ "info",
1249
+ `[herdsman-pi] wake acknowledgement held eventId=${barrier.eventId} reason=${barrier.reason} blocked=${barrier.blocked} · the daemon confirms by watermark (id <= ?), so no larger id may be acknowledged while this one is unconfirmed`,
978
1250
  );
979
- ctx.ui.notify?.(
980
- `Herdsman · ${eventIds.length} agent update${eventIds.length === 1 ? "" : "s"} could not be delivered by a wake turn · given up on (possibly never seen): read the agent directly for the details, or hand this workspace to another terminal so the daemon delivers it there`,
981
- "warning",
982
- );
983
- if (stranded.length === 0) return;
984
- void acknowledgeEventIds(stranded, { notify: false }, ctx).catch((error: unknown) => {
985
- logHerdsmanPi(
986
- "warn",
987
- `[herdsman-pi] wake write-off acknowledgement failed eventIds=${stranded.map((event) => event.id).join(",")} · ${String(error)}`,
988
- );
989
- });
990
1251
  };
991
1252
 
992
1253
  /**
993
- * Drives one continuation that carries out a wake Pi has not drained.
1254
+ * Dead-letters a wake delivery whose content never reached the transcript.
994
1255
  *
995
- * A wake injected while the orchestrator streams is delivered as a queued
996
- * follow-up (`triggerTurn: false`), and Pi only drains that queue when a run
997
- * reaches its stop point (`agent-loop` "Agent would stop here. Check for
998
- * follow-up messages."). When the run it rode on already passed that point,
999
- * the update sits in the queue until the next user message. At settlement the
1000
- * orchestrator is no longer streaming, so `triggerTurn: true` starts a real
1001
- * run (`_runAgentPrompt`) and that run's loop drains the queued follow-up.
1002
- * The marker carries no wake content: the queued follow-up is what delivers
1003
- * the evidence, exactly once.
1256
+ * The caller is a settlement that produced no consumption evidence for these
1257
+ * ids: the injected message was never carried out by any run (the turn ended
1258
+ * before its stop point, was aborted, or Pi dropped the message), so nothing
1259
+ * may be told to the daemon about them. An acknowledgement is a statement
1260
+ * that the orchestrator has the content, and here that statement would be a
1261
+ * lie: the daemon would mark the event consumed and never redeliver it.
1004
1262
  *
1005
- * Only consumption evidence writes a delivery off, so an intervening run that
1006
- * ends before its stop point (error, user abort, a refused tool) leaves the
1007
- * next settlement driving again. That bound is per event id and nothing
1008
- * resets it: starting runs cannot fix a cause that is not about the queue, so
1009
- * once an id's own budget is spent the id is written off (released and
1010
- * acknowledged) instead of looping forever — see
1011
- * `writeOffStrandedWakeDelivery` for the trade-off.
1263
+ * What the write-off does is local only: the ids stop being barred from every
1264
+ * later injection, so a later `scheduleWake` may re-present them with the real
1265
+ * update content. *May* is the whole point: `representableWakeOutcomes` decides
1266
+ * when, and until content that is not itself a dead letter leads a wake, the
1267
+ * only automatic resurface is a bounded, explicitly backed-off retry.
1268
+ * Re-presenting unconditionally on the settlement that had just written the id
1269
+ * off is what produced the turn loop this replaces. They deliberately stay in
1270
+ * `wakeAwaitingConsumption` (and therefore in the delivery queue): unconfirmed
1271
+ * content must keep blocking the acknowledgement watermark, because the daemon
1272
+ * confirms by `where id <= ?` and confirming a larger id would swallow them for
1273
+ * good. The events themselves stay pending in the daemon: nothing was
1274
+ * acknowledged, so a later wake is still eligible to present them, and a
1275
+ * replacement session can still receive them.
1276
+ *
1277
+ * The presentation guard is deliberately *not* extended here:
1278
+ * `presentedEventIds` is the "this content is in the transcript" set, and
1279
+ * these ids have no evidence of ever being there. Writing them off would
1280
+ * lock the update out of this session for good, which is exactly the
1281
+ * "busy delivery never arrives" failure this replaces — it is only the
1282
+ * *timing* of the next presentation that is now bounded.
1012
1283
  */
1013
- const driveWakeContinuation = (ctx: PiContext) => {
1014
- if (state.wakeAwaitingConsumption.size === 0) return;
1015
- if (!pi.sendMessage) return;
1016
- const eventIds = [...state.wakeAwaitingConsumption].sort((left, right) => left - right);
1017
- // Every id that is still waiting spends one drive of its own budget. The
1018
- // counters are keyed by event id and are never reset by an injection, so a
1019
- // delivery handed over later cannot buy an already-stuck id a fresh budget.
1020
- const spent: number[] = [];
1284
+ const writeOffStrandedWakeDelivery = (eventIds: number[], ctx: PiContext): void => {
1285
+ if (eventIds.length === 0) return;
1021
1286
  for (const eventId of eventIds) {
1022
- const drives = (state.wakeContinuationDrives.get(eventId) ?? 0) + 1;
1023
- state.wakeContinuationDrives.set(eventId, drives);
1024
- spent.push(drives);
1025
- }
1026
- try {
1027
- pi.sendMessage(
1028
- {
1029
- content: WAKE_CONTINUATION_CONTENT,
1030
- customType: WAKE_CONTINUATION_CUSTOM_TYPE,
1031
- display: false,
1032
- },
1033
- { deliverAs: "followUp", triggerTurn: true },
1034
- );
1035
- logHerdsmanPi(
1036
- "info",
1037
- `[herdsman-pi] wake continuation driven eventIds=${eventIds[0] ?? 0}-${eventIds.at(-1) ?? 0} count=${eventIds.length} drives=${spent.join(",")}`,
1038
- );
1039
- } catch {
1040
- logHerdsmanPi("warn", "[herdsman-pi] wake continuation refused by pi");
1287
+ state.wakeAwaitingConsumption.delete(eventId);
1288
+ // Unconfirmed content must still block the acknowledgement watermark: the
1289
+ // daemon confirms by `where id <= ?`, so letting a later id through would
1290
+ // swallow this one for good. The retry set is that barrier, and it is also
1291
+ // what lifts the injection barrier — a delivery that never reached the
1292
+ // transcript has to be presented again (see `alreadyPresented`), but only
1293
+ // on `representableWakeOutcomes`' terms.
1294
+ state.wakeRetryableEventIds.add(eventId);
1295
+ // The write-off happens *after* the turn that failed to carry the content
1296
+ // out, so this is where "quiet since" is pinned: until the backoff
1297
+ // elapses, nothing may present the id on its own. A budget that is already
1298
+ // spent gets no new deadline either — the id stays a barrier, not a turn
1299
+ // driver, and only content that is fresh can still carry it.
1300
+ const attempts = state.wakeDeadLetterAttempts.get(eventId) ?? 0;
1301
+ if (attempts < WAKE_DEAD_LETTER_RETRY_LIMIT) {
1302
+ state.wakeDeadLetterRetryAt.set(
1303
+ eventId,
1304
+ Date.now() + deadLetterRetryBackoffMs(attempts),
1305
+ );
1306
+ } else {
1307
+ state.wakeDeadLetterRetryAt.delete(eventId);
1308
+ }
1041
1309
  }
1042
- // Only the ids whose own budget this drive spent are written off. A sibling
1043
- // that still has drives left keeps its copy in the queue and its count, so
1044
- // it is neither lost nor granted a fresh ceiling.
1045
- const exhausted = eventIds.filter(
1046
- (_eventId, index) => (spent[index] ?? 0) >= MAX_WAKE_CONTINUATION_ATTEMPTS,
1310
+ logHerdsmanPi(
1311
+ "warn",
1312
+ `[herdsman-pi] wake delivery dead-lettered eventIds=${eventIds.join(",")} automaticRetries=${eventIds.map((eventId) => state.wakeDeadLetterAttempts.get(eventId) ?? 0).join(",")} limit=${WAKE_DEAD_LETTER_RETRY_LIMIT} · updates retained unacked in daemon: no consumption evidence, nothing acknowledged, and the ids stay eligible for a later wake`,
1313
+ );
1314
+ ctx.ui.notify?.(
1315
+ `Herdsman · ${eventIds.length} agent update${eventIds.length === 1 ? "" : "s"} could not be delivered by a wake turn · updates retained unacked in daemon: read the agent directly for the details, or hand this workspace to another terminal so the daemon delivers it there`,
1316
+ "warning",
1047
1317
  );
1048
- if (exhausted.length > 0) {
1049
- // This was the last drive the id's own budget allows, and the queued copy
1050
- // is still undrained: the id is written off instead of being left
1051
- // pending forever. The ids are released from
1052
- // `wakeAwaitingConsumption` (no further run is started for them) and
1053
- // acknowledged, which is what stops the daemon from redelivering them
1054
- // every freshness window — the loop that used to flood the orchestrator
1055
- // session until manual sqlite surgery. Losing one update is the accepted
1056
- // cost; see `writeOffStrandedWakeDelivery`.
1057
- writeOffStrandedWakeDelivery(exhausted, ctx);
1058
- }
1059
1318
  };
1060
1319
 
1320
+
1061
1321
  /**
1062
- * Arms the bounded deferral for a wake that cannot be injected right now.
1322
+ * Arms the deferral for a wake that cannot be injected right now.
1323
+ *
1324
+ * Delivery does not depend on whether the orchestrator is busy or idle:
1325
+ * every wake is queued and injected once the orchestrator has settled (or a
1326
+ * poll observes it idle). `agent_settled` is therefore the primary close
1327
+ * signal — it fires exactly when the turn ends and re-takes the wake
1328
+ * decision — and the timer here is only the poll that keeps the decision
1329
+ * alive while the turn runs. It backs off along `WAKE_BUSY_BACKOFF_MS` up to
1330
+ * `WAKE_BUSY_BACKOFF_CAP_MS` (a poll-frequency reduction, not a safety net):
1331
+ * the ladder used to tick at a flat `WAKE_BUSY_SPIN_MS`, so a long turn was
1332
+ * re-evaluated ten times a second forever.
1063
1333
  *
1064
- * The retry spins every `WAKE_BUSY_SPIN_MS` while the orchestrator is busy
1065
- * and flips `wakeForcedRelease` once `WAKE_DEFERRED_TIMEOUT_MS` has elapsed,
1066
- * so the next pass injects the batch from the current state (as a queued,
1067
- * non-triggering follow-up) instead of waiting for an idle signal or a
1068
- * settlement that may never arrive.
1334
+ * There is deliberately no deadline (nothing like a `WAKE_DEFERRED_TIMEOUT_MS`
1335
+ * that force-releases a busy orchestrator after N seconds): a wake injected
1336
+ * into a busy orchestrator either lands in a follow-up queue nothing drains
1337
+ * — the "busy delivery never arrives" failure this replaces — or interrupts
1338
+ * the user's own turn, so waiting is the only outcome that loses nothing.
1339
+ * The liveness bound is exactly "until `agent_settled`, or until one of these
1340
+ * ticks observes an idle orchestrator"; backing off only makes that wait
1341
+ * cheaper, never shorter.
1069
1342
  */
1070
1343
  const scheduleDeferredWake = (ctx: PiContext) => {
1071
1344
  state.wakeDeferredUntilSettled = true;
1072
- const since = state.wakeDeferredSince ?? Date.now();
1073
- state.wakeDeferredSince = since;
1074
- const remaining = WAKE_DEFERRED_TIMEOUT_MS - (Date.now() - since);
1075
- if (remaining <= 0) state.wakeForcedRelease = true;
1345
+ // The deferral is the state the user cannot see (nothing is injected, no
1346
+ // turn is started, and the pending count alone reads as idle work), so it
1347
+ // is announced once per arming rather than left to be guessed.
1348
+ setHerdsmanUi(ctx);
1076
1349
  if (state.wakeTimer) return;
1350
+ const rung = Math.min(state.wakeBusySpinRung, WAKE_BUSY_BACKOFF_MS.length - 1);
1351
+ state.wakeBusySpinRung = Math.min(rung + 1, WAKE_BUSY_BACKOFF_MS.length - 1);
1352
+ // Clamped to the cap rather than trusted to the ladder's last rung: a rung
1353
+ // edited above the ceiling would otherwise raise the real backoff while the
1354
+ // exported constant, the note and the tests still say 5s.
1355
+ const delay = Math.min(
1356
+ WAKE_BUSY_BACKOFF_MS[rung] ?? WAKE_BUSY_SPIN_MS,
1357
+ WAKE_BUSY_BACKOFF_CAP_MS,
1358
+ );
1077
1359
  state.wakeTimer = setTimeout(() => {
1078
1360
  state.wakeTimer = undefined;
1079
- if (state.wakeForcedRelease && state.deliveredBatch && ctx.isIdle?.() !== false) {
1080
- // The hard deadline only releases the delivery — it never discards an
1081
- // unconfirmed event. The batch record is dropped so a wake turn that
1082
- // is no longer running cannot gate later wakes, but its events stay in
1083
- // the delivery queue (`unackedDelivered`) and are re-attached to the
1084
- // batch injected right below, which acknowledges them once it settles.
1085
- state.deliveredBatch = undefined;
1086
- }
1087
1361
  scheduleWake(ctx);
1088
- }, Math.max(0, Math.min(WAKE_BUSY_SPIN_MS, remaining)));
1362
+ }, delay);
1089
1363
  };
1090
1364
 
1091
1365
  const scheduleWake = (ctx: PiContext | undefined) => {
1092
1366
  if (!ctx || !state.isOrchestrator || !state.currentScope || !pi.sendMessage) return;
1093
1367
  if (state.wakeTimer || state.wakeRequested) {
1094
1368
  // A pending wake owns the release; this branch is also hit re-entrantly by
1095
- // a running pass (its fired settle timer is still set), where clearing
1096
- // `wakeForcedRelease` would re-defer a batch the deadline just released.
1369
+ // a running pass (its fired settle timer is still set), and ending the
1370
+ // deferral there would let a second injection ride in behind it.
1097
1371
  return;
1098
1372
  }
1099
1373
  const projection = projectAgentOutcomes(state.pendingEvents, wakeFilter);
@@ -1116,11 +1390,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1116
1390
  const wakeable = outcomes.filter((outcome) =>
1117
1391
  isWakeableEvent(state.pendingEvents.find((pending) => pending.id === outcome.eventId)),
1118
1392
  );
1119
- if (wakeable.length === 0) {
1120
- // Nothing is wakeable right now, so the deferral that led here is over:
1121
- // its released deadline must not let a later, unrelated wake bypass the
1122
- // busy gate.
1123
- endWakeDeferral();
1393
+ // A dead-lettered id may only join a wake that fresh content leads, or, on
1394
+ // its own, wait for its explicit backoff (see `representableWakeOutcomes`).
1395
+ // Without this the settlement that wrote it off re-injects it in the same
1396
+ // cycle, which is the turn loop this replaces.
1397
+ const { outcomes: injectableOutcomes } = representableWakeOutcomes(wakeable);
1398
+ if (injectableOutcomes.length === 0) {
1124
1399
  // A suppressed upstream error that is now due must be silently
1125
1400
  // acknowledged before any backoff timer is planted: planting the timer
1126
1401
  // first would make scheduleSilentUpstreamErrorAck's entry guard
@@ -1128,19 +1403,48 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1128
1403
  // off its own timer and, once the backoff window has elapsed, the
1129
1404
  // 0ms-timer + blocked-ack loop never converges the queue.
1130
1405
  const dueSuppressed = suppressedEvents.filter(isWakeableEvent);
1131
- if (dueSuppressed.length > 0) {
1132
- scheduleSilentUpstreamErrorAck(ctx, dueSuppressed);
1406
+ // A silent acknowledgement is a watermark statement, though: the daemon
1407
+ // confirms every id at or below the acked one, so it may not cross an id
1408
+ // that is still unconfirmed locally. The barrier is the same one
1409
+ // `confirmableDeliveryPrefix` walks (`unconfirmedWatermarkBarrier`), and
1410
+ // crossing it would hand the daemon the authority to mark the blocked id
1411
+ // confirmed without its content ever reaching the transcript — the
1412
+ // "busy delivery never arrives" failure in its silent form, with no
1413
+ // redelivery and no warning afterwards. The held ids stay due and are
1414
+ // acknowledged by a later pass, once the barrier id is confirmed (or
1415
+ // recovered) and the watermark has moved past it.
1416
+ const watermarkBarrier = unconfirmedWatermarkBarrier();
1417
+ const ackableSuppressed =
1418
+ watermarkBarrier === undefined
1419
+ ? dueSuppressed
1420
+ : dueSuppressed.filter((event) => event.id < watermarkBarrier.eventId);
1421
+ if (ackableSuppressed.length > 0) {
1422
+ scheduleSilentUpstreamErrorAck(ctx, ackableSuppressed);
1133
1423
  return;
1134
1424
  }
1425
+ if (watermarkBarrier !== undefined && dueSuppressed.length > 0) {
1426
+ // Held back by the barrier rather than by a missing tick: say so, in
1427
+ // the throttled shape above, so the stall is observable.
1428
+ noteWakeWatermarkHoldBack(watermarkBarrier);
1429
+ }
1135
1430
  // No suppressed event is due, so plant a backoff timer for the next
1136
1431
  // future `nextAttemptAt`. Expired timestamps are excluded (strictly
1137
1432
  // greater than now) so an already-past window does not produce a
1138
- // zero-delay spin.
1433
+ // zero-delay spin. A dead-lettered id that still owes an automatic retry
1434
+ // contributes its own deadline here: without it the retry the write-off
1435
+ // scheduled would wait for an unrelated event to come along.
1139
1436
  const nextAttemptAt = [
1140
1437
  ...outcomes.map((outcome) => outcome.eventId),
1141
1438
  ...suppressedEvents.map((event) => event.id),
1142
1439
  ]
1143
- .map((eventId) => state.pendingEvents.find((event) => event.id === eventId)?.nextAttemptAt)
1440
+ .flatMap((eventId) => {
1441
+ const nextAttempt = state.pendingEvents.find((event) => event.id === eventId)
1442
+ ?.nextAttemptAt;
1443
+ const retryAt = state.wakeRetryableEventIds.has(eventId)
1444
+ ? deadLetterRetryAt(eventId)
1445
+ : undefined;
1446
+ return [nextAttempt, retryAt];
1447
+ })
1144
1448
  .filter((value): value is number => value !== undefined && value > Date.now())
1145
1449
  .sort((left, right) => left - right)[0];
1146
1450
  if (nextAttemptAt !== undefined) {
@@ -1153,10 +1457,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1153
1457
  }
1154
1458
 
1155
1459
  // An in-flight batch owns the ack cursor and a busy orchestrator must not
1156
- // be interrupted, so neither is woken immediately — but both are deferred
1157
- // on a bounded spin (never parked until an event that may never come).
1460
+ // be interrupted, so neither is woken immediately — the wake stays
1461
+ // deferred on the backed-off poll until the batch settles and the
1462
+ // orchestrator is idle, and nothing is released past that gate.
1158
1463
  const inFlight = state.deliveredBatch !== undefined || state.ackInFlight;
1159
- if (!state.wakeForcedRelease && (inFlight || ctx.isIdle?.() === false)) {
1464
+ if (inFlight || ctx.isIdle?.() === false) {
1160
1465
  scheduleDeferredWake(ctx);
1161
1466
  return;
1162
1467
  }
@@ -1165,8 +1470,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1165
1470
  const ownerTerminalId = state.currentScope.terminalId;
1166
1471
  const ownerWorkspaceId = state.currentScope.workspaceId;
1167
1472
  state.wakeTimer = setTimeout(() => {
1168
- const startWake = async () => {
1169
- if (
1473
+ const startWake = async () => { if (
1170
1474
  generation !== wakeGeneration ||
1171
1475
  !state.isOrchestrator ||
1172
1476
  state.currentScope?.herdrSessionName !== ownerHerdrSessionName ||
@@ -1174,16 +1478,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1174
1478
  state.currentScope?.workspaceId !== ownerWorkspaceId
1175
1479
  ) {
1176
1480
  state.wakeTimer = undefined;
1177
- // The pass belongs to a stale generation or scope, so its deferral
1178
- // episode ends here (a scope reset usually got there first through
1179
- // `cancelWakeTimer`).
1180
- endWakeDeferral();
1181
1481
  return;
1182
1482
  }
1183
- if (ctx.isIdle?.() === false && !state.wakeForcedRelease) {
1483
+ if (ctx.isIdle?.() === false) {
1184
1484
  state.wakeTimer = undefined;
1185
- // Still busy: this pass re-defers, so the deferral episode (and with
1186
- // it the 5s deadline) must keep running instead of restarting.
1485
+ // Still busy: this pass re-defers and keeps waiting, so nothing is
1486
+ // injected into (or parked for) a running turn.
1187
1487
  scheduleDeferredWake(ctx);
1188
1488
  return;
1189
1489
  }
@@ -1195,13 +1495,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1195
1495
  )) as ConnectionStateResponse | undefined;
1196
1496
  if (!response) {
1197
1497
  state.wakeTimer = undefined;
1198
- endWakeDeferral();
1199
1498
  return;
1200
1499
  }
1201
1500
  applyConnectionStateResponse(response, ctx);
1202
1501
  } catch {
1203
1502
  state.wakeTimer = undefined;
1204
- endWakeDeferral();
1205
1503
  // A failed load is only temporary: the batch stays pending and is
1206
1504
  // retried on the next wake instead of being permanently suppressed.
1207
1505
  ctx.ui.notify?.(
@@ -1219,14 +1517,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1219
1517
  state.currentScope?.workspaceId !== ownerWorkspaceId
1220
1518
  ) {
1221
1519
  state.wakeTimer = undefined;
1222
- // See the earlier generation re-check: the episode ends with it.
1223
- endWakeDeferral();
1224
1520
  return;
1225
1521
  }
1226
- if (ctx.isIdle?.() === false && !state.wakeForcedRelease) {
1522
+ if (ctx.isIdle?.() === false) {
1227
1523
  state.wakeTimer = undefined;
1228
- // See the earlier re-check: re-deferring keeps this episode's 5s
1229
- // deadline intact.
1524
+ // See the earlier re-check: a busy orchestrator keeps waiting.
1230
1525
  scheduleDeferredWake(ctx);
1231
1526
  return;
1232
1527
  }
@@ -1234,15 +1529,20 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1234
1529
  const batchEvents = [...state.pendingEvents].sort((left, right) => left.id - right.id);
1235
1530
  const batchProjection = projectAgentOutcomes(batchEvents, wakeFilter);
1236
1531
  const batchSuppressedIds = new Set(batchProjection.suppressedUpstreamErrorEventIds);
1237
- const batchOutcomes = batchProjection.outcomes.filter(
1532
+ const projectableBatchOutcomes = batchProjection.outcomes.filter(
1238
1533
  (outcome) =>
1239
1534
  outcome.eventId > state.failedWakeThroughEventId &&
1240
1535
  !alreadyPresented(outcome.eventId) &&
1241
1536
  isWakeableEvent(batchEvents.find((event) => event.id === outcome.eventId)),
1242
1537
  );
1538
+ // Same split as in `scheduleWake`, re-taken here because the pass
1539
+ // awaited an `agent.orchestrator.get` in between: a dead-lettered id
1540
+ // may only ride content that is fresh, or retry on its own once its
1541
+ // backoff is due (see `representableWakeOutcomes`).
1542
+ const { lead: batchLeadOutcomes, outcomes: batchOutcomes } =
1543
+ representableWakeOutcomes(projectableBatchOutcomes);
1243
1544
  if (batchOutcomes.length === 0) {
1244
1545
  state.wakeTimer = undefined;
1245
- endWakeDeferral();
1246
1546
  return;
1247
1547
  }
1248
1548
  const current = batchOutcomes;
@@ -1254,24 +1554,19 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1254
1554
  state.wakeRequested = true;
1255
1555
  state.wakeRequestedThroughEventId = current.at(-1)?.eventId ?? 0;
1256
1556
  // Single-track injection: the wake content is handed to Pi exactly
1257
- // once, as the follow-up message below. An idle orchestrator gets a
1258
- // triggered follow-up turn (immediate delivery); a busy one is not
1259
- // interrupted and receives the same content as a queued,
1260
- // non-triggering follow-up that a later run drains out of the
1261
- // follow-up queue. There is deliberately no second copy on the
1262
- // `context` hook: a run that is busy cannot act on an early copy
1263
- // anyway, while a second copy is exactly what puts the same update
1264
- // into the session twice. So the transcript holds one entry per event
1265
- // id, and the continuation drive (see `driveWakeContinuation`) is what
1266
- // carries a queued copy that no run drained into the transcript.
1267
- const orchestratorBusy = ctx.isIdle?.() === false;
1268
- const wakeContent = formatAgentOutcomeUpdates(batchOutcomes);
1269
- // Single line, injection path only: the decision that produced this
1270
- // batch plus the signals it came from, so a wake that still arrives
1271
- // late can be told apart from one parked by a stale gate.
1557
+ // once, as the follow-up message below, and only once the orchestrator
1558
+ // is idle — a busy one has been deferred (see `scheduleDeferredWake`),
1559
+ // because a follow-up parked in a running turn's queue may never be
1560
+ // drained. There is deliberately no second copy on the `context` hook
1561
+ // either: the update enters the transcript exactly once per event id,
1562
+ // through the one message this path sends.
1563
+ const wakeContent = formatAgentOutcomeUpdates([...batchOutcomes]);
1564
+ // Single line, injection path only: the state the batch was taken from,
1565
+ // so a wake that still arrives late can be told apart from one that was
1566
+ // held back by the busy gate.
1272
1567
  logHerdsmanPi(
1273
1568
  "info",
1274
- `[herdsman-pi] wake inject deliverAs=followUp triggerTurn=${String(!orchestratorBusy)} forced=${state.wakeForcedRelease} runActive=${String(state.runActive)} isIdle=${ctx.isIdle === undefined ? "unknown" : String(ctx.isIdle())} eventIds=${batchOutcomes[0]?.eventId ?? 0}-${batchOutcomes.at(-1)?.eventId ?? 0} count=${batchOutcomes.length}`,
1569
+ `[herdsman-pi] wake inject deliverAs=followUp triggerTurn=true runActive=${String(state.runActive)} isIdle=${ctx.isIdle === undefined ? "unknown" : String(ctx.isIdle())} eventIds=${batchOutcomes[0]?.eventId ?? 0}-${batchOutcomes.at(-1)?.eventId ?? 0} count=${batchOutcomes.length}`,
1275
1570
  );
1276
1571
  try {
1277
1572
  pi.sendMessage?.(
@@ -1293,35 +1588,43 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1293
1588
  },
1294
1589
  display: false,
1295
1590
  },
1296
- orchestratorBusy
1297
- ? { deliverAs: "followUp", triggerTurn: false }
1298
- : { deliverAs: "followUp", triggerTurn: true },
1591
+ // `triggerTurn: true` is what makes this a real turn: pi hands the
1592
+ // message to the agent core, which produces a regular `message_end`
1593
+ // for the hidden entry, and only that emission is the consumption
1594
+ // evidence the acknowledgement waits for. A `triggerTurn: false`
1595
+ // variant only parks the message in a follow-up queue that a run
1596
+ // drains at its stop point — the copy that made "busy delivery
1597
+ // failed" possible.
1598
+ { deliverAs: "followUp", triggerTurn: true },
1299
1599
  );
1300
- // A queued (non-triggering) delivery rides the running turn: Pi parks it
1301
- // in the agent's follow-up queue, which a run only drains when it reaches
1302
- // its stop point. If the run it rode on already passed that point, nothing
1303
- // drains it — the settlement drives a continuation instead (see
1304
- // `driveWakeContinuation`), and until the content is seen in the
1305
- // transcript the delivery is neither acknowledged nor injected again.
1306
- if (orchestratorBusy) {
1307
- // A freshly queued id starts its own continuation budget; an id that
1308
- // was already awaiting consumption (merged into this batch from an
1309
- // earlier injection) keeps the drives it already spent. Only a new
1310
- // id may get a fresh ceiling — never an unrelated one.
1311
- for (const outcome of batchOutcomes) {
1312
- state.wakeAwaitingConsumption.add(outcome.eventId);
1313
- if (!state.wakeContinuationDrives.has(outcome.eventId)) {
1314
- state.wakeContinuationDrives.set(outcome.eventId, 0);
1315
- }
1600
+ // A freshly delivered id is in flight until its content is seen in
1601
+ // the transcript: it can neither be acknowledged (the evidence is
1602
+ // what authorises that) nor injected again (the content must not
1603
+ // enter the transcript twice). An id merged into this batch from an
1604
+ // earlier delivery keeps that same state; a dead-lettered one is in
1605
+ // flight again and loses its retry flag.
1606
+ for (const outcome of batchOutcomes) {
1607
+ // A dead-lettered id that retried on its own spends one of its
1608
+ // automatic attempts; one that rode a wake fresh content led is
1609
+ // riding that update's turn, not its own, and rides for free.
1610
+ if (
1611
+ batchLeadOutcomes.length === 0 &&
1612
+ state.wakeRetryableEventIds.has(outcome.eventId)
1613
+ ) {
1614
+ state.wakeDeadLetterAttempts.set(
1615
+ outcome.eventId,
1616
+ (state.wakeDeadLetterAttempts.get(outcome.eventId) ?? 0) + 1,
1617
+ );
1316
1618
  }
1619
+ state.wakeAwaitingConsumption.add(outcome.eventId);
1620
+ state.wakeRetryableEventIds.delete(outcome.eventId);
1621
+ state.wakeDeadLetterRetryAt.delete(outcome.eventId);
1317
1622
  }
1318
- state.wakeForcedRelease = false;
1319
- state.wakeDeferredSince = undefined;
1320
1623
  // Only expose the batch after the hidden context was accepted by pi. This
1321
1624
  // keeps an injection failure eligible for daemon redelivery.
1322
1625
  //
1323
1626
  // The batch is the delivery queue plus this injection: previously
1324
- // unconfirmed events are merged (never replaced) so that a forced
1627
+ // unconfirmed events are merged (never replaced) so that a later
1325
1628
  // release always leaves both the old and the new events deliverable
1326
1629
  // and acknowledgeable in id order. The turn-consumption flags of a
1327
1630
  // still-running previous batch are carried over, because they
@@ -1330,39 +1633,40 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1330
1633
  // `hasSubstantiveWork` is the sole gate that decides whether an
1331
1634
  // ownership/scope change may abort the in-flight turn (see loseRole
1332
1635
  // and resetForScopeChange), and aborting is only ever allowed for a
1333
- // *pure* Herdsman wake turn. A busy orchestrator gets the batch as a
1334
- // non-triggering queued follow-up, which rides the user's own turn:
1335
- // that turn is not a Herdsman wake turn, so it must never be aborted
1336
- // on our behalf and the flag is set here.
1636
+ // *pure* Herdsman wake turn. The injected message starts one of those
1637
+ // (it is only ever sent to an idle orchestrator, and it triggers its
1638
+ // own turn), and nothing here has produced substance yet, so the
1639
+ // flag starts false and the tool/assistant hooks raise it.
1337
1640
  state.deliveredBatch = {
1338
1641
  abortedByUser: previousBatch?.abortedByUser ?? false,
1339
1642
  assistantFinalSucceeded: previousBatch?.assistantFinalSucceeded ?? false,
1340
1643
  events: mergeUnackedDelivered(incomingEvents),
1341
- hasSubstantiveWork: orchestratorBusy || (previousBatch?.hasSubstantiveWork ?? false),
1644
+ hasSubstantiveWork: previousBatch?.hasSubstantiveWork ?? false,
1342
1645
  invalidated: false,
1343
1646
  ownerTerminalId,
1344
1647
  herdsmanTriggered: true,
1345
1648
  };
1346
1649
  state.wakeRequested = false;
1347
1650
  state.wakeRequestedThroughEventId = 0;
1348
- // Record the presentation so a reclaim redelivery of the same id is
1349
- // never presented twice. The set is monotonic for the lifetime of
1350
- // the current orchestrator scope: ids are only removed on role
1351
- // loss, scope change, or shutdown (pruneAcknowledgedEvents keeps
1352
- // acknowledged ids on purpose).
1353
- for (const outcome of batchOutcomes) {
1354
- state.presentedEventIds.add(outcome.eventId);
1355
- }
1651
+ // The busy wait is over: whatever turn comes next starts the poll
1652
+ // ladder from its first rung again.
1653
+ state.wakeBusySpinRung = 0;
1654
+ // The ids deliberately do *not* enter `presentedEventIds` here: that set
1655
+ // means "this content is in the transcript", and only consumption
1656
+ // evidence (`pi.on("message_end")` for this hidden wake message, see
1657
+ // the handler below) knows that. Recording the injection instead would
1658
+ // let an update whose run never carried it out — a failed turn, an
1659
+ // abort, a message Pi dropped — become unrecoverable in this session,
1660
+ // which is the lock-out this path is meant to remove. The in-flight
1661
+ // state is exactly what keeps it out of the transcript twice in the
1662
+ // meantime: `alreadyPresented` reports "awaiting".
1356
1663
  } catch {
1357
1664
  state.deliveredBatch = undefined;
1358
1665
  state.wakeRequested = false;
1359
- // Nothing was queued by the refused injection, so nothing is awaiting
1360
- // consumption on its account either: the ids only enter the awaiting set
1361
- // once Pi accepted the message (see the queued branch above). The
1362
- // refused injection consumed this deferral, so clearing it keeps the
1363
- // elapsed deadline from letting a later, unrelated wake bypass the busy
1364
- // gate.
1365
- endWakeDeferral();
1666
+ // Nothing was sent, so nothing is awaiting consumption on its account
1667
+ // either: the ids only enter the awaiting set once Pi accepted the
1668
+ // message. Nothing was delivered, so the event stays pending and the
1669
+ // next wake retries it.
1366
1670
  }
1367
1671
  };
1368
1672
  void startWake();
@@ -1826,11 +2130,10 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1826
2130
  pi.on("message_end", (event: Record<string, unknown>) => {
1827
2131
  const message = record(event.message);
1828
2132
  if (message.role === "custom" && message.customType === WAKE_CONTEXT_CUSTOM_TYPE) {
1829
- // Consumption evidence: the wake content reached the transcript, so some
1830
- // run drained the queued follow-up and carried the update out. The ids the
1831
- // message presented are confirmed — no further continuation is owed for
1832
- // them, and they are now authorised for acknowledgement on their own,
1833
- // whatever conclusion the turn reached.
2133
+ // Consumption evidence: the wake content reached the transcript, so the
2134
+ // turn that carried it out is confirmed. The ids the message presented are
2135
+ // now authorised for acknowledgement on their own, whatever conclusion the
2136
+ // turn reached.
1834
2137
  const consumedEventIds = wakeConsumedEventIds(message);
1835
2138
  if (consumedEventIds.length === 0) {
1836
2139
  // Nothing may be confirmed from a message whose evidence cannot be read
@@ -1845,9 +2148,9 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1845
2148
  }
1846
2149
  for (const eventId of consumedEventIds) {
1847
2150
  state.wakeAwaitingConsumption.delete(eventId);
1848
- // Its copy reached the transcript, so the id owes no drive any more and
1849
- // its budget leaves with it.
1850
- state.wakeContinuationDrives.delete(eventId);
2151
+ // The delivery is settled (carried out, or dead-lettered and now
2152
+ // re-delivered): it owes no further drive and no further retry.
2153
+ state.wakeRetryableEventIds.delete(eventId);
1851
2154
  // The content reached the transcript, so this id counts as presented from
1852
2155
  // now on: a daemon redelivery of it (it is still unacknowledged whenever
1853
2156
  // its batch was already dropped) must not inject the same update again.
@@ -1916,12 +2219,26 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1916
2219
 
1917
2220
  pi.on("agent_settled", async (_event: unknown, ctx: PiContext) => {
1918
2221
  state.runActive = false;
1919
- // A wake delivered as a queued follow-up is drained only when a run reaches
1920
- // its stop point. If the run that received it settled without draining it,
1921
- // no further run exists to carry the update out: drive a continuation (up to
1922
- // the per-delivery bound, see `driveWakeContinuation`), which is what
1923
- // surfaces the queued update.
1924
- driveWakeContinuation(ctx);
2222
+ // The busy period this settlement ends is over, so the poll ladder starts
2223
+ // from its first rung again for whatever turn comes next (see
2224
+ // `WAKE_BUSY_BACKOFF_MS`).
2225
+ state.wakeBusySpinRung = 0;
2226
+ // The settlement is the signal a deferral has been waiting for, so the
2227
+ // pending spin is dropped: the wake decision below is taken now instead of
2228
+ // one `WAKE_BUSY_SPIN_MS` tick later. A pass that is already being injected
2229
+ // (`wakeRequested`) keeps its timer — clearing it would drop the injection.
2230
+ if (state.wakeTimer && !state.wakeRequested) {
2231
+ const pendingDeferral = state.wakeTimer;
2232
+ state.wakeTimer = undefined;
2233
+ clearTimeout(pendingDeferral);
2234
+ }
2235
+ // A wake whose content never reached the transcript gets no run of its own:
2236
+ // there is deliberately no empty-marker continuation. The settlement writes
2237
+ // those ids off locally (dead-letter, no acknowledgement) and the next
2238
+ // `scheduleWake` re-presents them with the real update content, or a later
2239
+ // settlement confirms them once their copy finally drains.
2240
+ const unconsumed = [...state.wakeAwaitingConsumption].sort((left, right) => left - right);
2241
+ if (unconsumed.length > 0) writeOffStrandedWakeDelivery(unconsumed, ctx);
1925
2242
  const failBatch = () => {
1926
2243
  ctx.ui.notify?.(
1927
2244
  "Herdsman couldn’t acknowledge agent updates · updates remain pending",
@@ -1931,8 +2248,6 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1931
2248
  const finishBatch = () => {
1932
2249
  state.ackInFlight = false;
1933
2250
  state.wakeDeferredUntilSettled = false;
1934
- state.wakeDeferredSince = undefined;
1935
- state.wakeForcedRelease = false;
1936
2251
  setHerdsmanUi(ctx);
1937
2252
  scheduleWake(ctx);
1938
2253
  };
@@ -1969,10 +2284,14 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1969
2284
 
1970
2285
  const stillOwner =
1971
2286
  state.isOrchestrator && state.currentScope?.terminalId === batch.ownerTerminalId;
1972
- const { deliveryQueue, confirmablePrefix, blockedByMissingTurn, stillAwaitingConsumption } =
1973
- confirmableDeliveryPrefix(batch.assistantFinalSucceeded || batch.abortedByUser);
2287
+ const { deliveryQueue, confirmablePrefix, blockedByMissingTurn } = confirmableDeliveryPrefix(
2288
+ batch.assistantFinalSucceeded || batch.abortedByUser,
2289
+ );
1974
2290
  const reachable = stillOwner && state.client !== undefined && state.connected;
1975
- if (!stillAwaitingConsumption) state.deliveredBatch = undefined;
2291
+ // Every id awaiting consumption was dead-lettered at the top of this
2292
+ // handler, so nothing holds the cursor open any more and the batch can be
2293
+ // dropped: a later wake is not gated by it.
2294
+ state.deliveredBatch = undefined;
1976
2295
  state.ackInFlight = true;
1977
2296
  if (!reachable) {
1978
2297
  // Unreachable (ownership gone, or a disconnect): attempting an