@dorokuma/herdsman-pi 0.11.6 → 0.13.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,25 +1,10 @@
1
1
  import { agentIdentityLabel } from "./agent-display.js";
2
- import { sanitizeText } from "./sanitize-text.js";
3
- import { appendFileSync, mkdirSync } from "node:fs";
4
- import { homedir } from "node:os";
5
- import { dirname, isAbsolute, join } from "node:path";
2
+ import { sanitizeText, textFromContent } from "./sanitize-text.js";
6
3
  import { stripVTControlCharacters } from "node:util";
4
+ import { logHerdsmanPi } from "./logger.js";
7
5
 
8
- export type HerdsmanPiLogLevel = "info" | "warn" | "error";
9
-
10
- export function logHerdsmanPi(level: HerdsmanPiLogLevel, message: string): void {
11
- try {
12
- const configuredHome = process.env.HERDSMAN_HOME?.trim();
13
- const home = configuredHome && isAbsolute(configuredHome) ? configuredHome : join(homedir(), ".herdsman");
14
- const now = new Date();
15
- const date = now.toISOString().slice(0, 10).replaceAll("-", "");
16
- const file = join(home, "logs", `herdsman-pi-${date}.log`);
17
- mkdirSync(dirname(file), { recursive: true });
18
- appendFileSync(file, `${now.toISOString()} [${level}] ${message}\n`, "utf8");
19
- } catch {
20
- // Diagnostics must never write to the terminal or interrupt the extension.
21
- }
22
- }
6
+ export { logHerdsmanPi };
7
+ export type { HerdsmanPiLogLevel } from "./logger.js";
23
8
 
24
9
  import {
25
10
  type AgentContextListItem,
@@ -41,6 +26,8 @@ import {
41
26
  formatAgentOutcomeUpdates,
42
27
  WAKE_SETTLE_MS,
43
28
  } from "./wake.js";
29
+ import { loadWakeFilterConfig } from "./wake-filter-config.js";
30
+ import type { WakeFilterConfig } from "./upstream-error.js";
44
31
  import { confirmSessionWrite } from "./turn-signal.js";
45
32
 
46
33
  type PiAgentMessage = {
@@ -126,7 +113,29 @@ type HerdsmanState = {
126
113
  roleMutationInFlight: boolean;
127
114
  sessionRef: AgentSessionRef | undefined;
128
115
  subscriberId: string | undefined;
116
+ /**
117
+ * Delivery queue: events handed to Pi whose acknowledgement is still
118
+ * outstanding. Id-keyed so a repeated id is stored exactly once (deliver
119
+ * once), and every consumer sorts by id. Entries are only removed by an
120
+ * acknowledgement (success or dead-letter) or by a role/scope reset that also
121
+ * clears `presentedEventIds`, so releasing a deferred wake can never lose an
122
+ * unconfirmed event.
123
+ */
124
+ unackedDelivered: Map<number, AgentEventWireRecord>;
129
125
  wakeDeferredUntilSettled: boolean;
126
+ /** Wall-clock start of the current bounded wake deferral, if any. */
127
+ wakeDeferredSince: number | undefined;
128
+ /**
129
+ * Set once the hard deferral budget elapsed: the next pass injects the batch
130
+ * from the current state instead of deferring again.
131
+ */
132
+ wakeForcedRelease: boolean;
133
+ /**
134
+ * Event content queued for a busy orchestrator. Injected through the
135
+ * `context` hook so the running turn sees the update without being
136
+ * interrupted.
137
+ */
138
+ wakeContext: { content: string; eventIds: number[] } | undefined;
130
139
  wakeRequested: boolean;
131
140
  wakeRequestedThroughEventId: number;
132
141
  wakeTimer: ReturnType<typeof setTimeout> | undefined;
@@ -173,6 +182,7 @@ type ExtensionOptions = {
173
182
  clientFactory?: () => HerdsmanDaemonClient;
174
183
  onTurnCompletionSignal?: (completion: Promise<void>) => void;
175
184
  onStateExposed?: (state: HerdsmanState) => void;
185
+ wakeFilter?: WakeFilterConfig;
176
186
  };
177
187
 
178
188
  const DEFAULT_HOME_NAME = ".herdsman";
@@ -182,6 +192,15 @@ const RECONNECTING_MESSAGE = "Herdsman is reconnecting · try again shortly";
182
192
  export const MAX_ACK_ATTEMPTS = 5;
183
193
  export const ACK_BACKOFF_CAP_MS = 30_000;
184
194
  const KEEPALIVE_INTERVAL_MS = 30_000;
195
+ /** Retry interval used while a wake cannot be injected (busy orchestrator). */
196
+ export const WAKE_BUSY_SPIN_MS = 100;
197
+ /**
198
+ * Hard upper bound for every deferred wake. Once it elapses the scheduler stops
199
+ * waiting for the orchestrator to become idle or for a settlement to arrive and
200
+ * forces the injection decision from the current state, so a wake can never be
201
+ * parked forever.
202
+ */
203
+ export const WAKE_DEFERRED_TIMEOUT_MS = 5_000;
185
204
 
186
205
  type AckFailureClass = "terminal" | "resync" | "transient";
187
206
 
@@ -242,6 +261,9 @@ export function defaultSocketPath() {
242
261
  export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
243
262
  return function herdsmanPiExtension(pi: PiApi): void {
244
263
  pi.registerMessageRenderer?.("herdsman-wake", renderAgentUpdateMessage);
264
+ // Read once per extension instance: config.yaml changes require a Pi restart,
265
+ // matching the daemon's startup-time config model.
266
+ const wakeFilter = options.wakeFilter ?? loadWakeFilterConfig();
245
267
 
246
268
  const state: HerdsmanState = {
247
269
  client: undefined,
@@ -261,7 +283,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
261
283
  runActive: false,
262
284
  sessionRef: undefined,
263
285
  subscriberId: undefined,
286
+ unackedDelivered: new Map(),
264
287
  wakeDeferredUntilSettled: false,
288
+ wakeDeferredSince: undefined,
289
+ wakeForcedRelease: false,
290
+ wakeContext: undefined,
265
291
  wakeRequested: false,
266
292
  wakeRequestedThroughEventId: 0,
267
293
  wakeTimer: undefined,
@@ -293,7 +319,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
293
319
  : state.isOrchestrator
294
320
  ? {
295
321
  kind: "on",
296
- updateCount: projectAgentOutcomes(state.pendingEvents).outcomes.length,
322
+ updateCount: projectAgentOutcomes(state.pendingEvents, wakeFilter).outcomes.length,
297
323
  }
298
324
  : { kind: "off" };
299
325
  ctx.ui.setStatus?.("herdsman", formatHerdsmanFooterStatus(footerState));
@@ -304,12 +330,18 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
304
330
  if (state.wakeTimer) clearTimeout(state.wakeTimer);
305
331
  state.wakeTimer = undefined;
306
332
  state.wakeDeferredUntilSettled = false;
333
+ state.wakeDeferredSince = undefined;
334
+ state.wakeForcedRelease = false;
307
335
  };
308
336
 
309
337
  const cancelWake = () => {
310
338
  cancelWakeTimer();
311
339
  state.wakeRequested = false;
312
340
  state.wakeRequestedThroughEventId = 0;
341
+ // A wake queued for a busy orchestrator belongs to the role/scope that
342
+ // queued it: dropping it here keeps a stale event body out of the context
343
+ // of whatever session takes over next.
344
+ state.wakeContext = undefined;
313
345
  };
314
346
 
315
347
  const clearAgentContext = () => {
@@ -318,6 +350,41 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
318
350
  state.runActive = false;
319
351
  };
320
352
 
353
+ const unackedDeliveredAscending = (): AgentEventWireRecord[] =>
354
+ [...state.unackedDelivered.values()].sort((left, right) => left.id - right.id);
355
+
356
+ /**
357
+ * Merges freshly injected events into the delivery queue.
358
+ *
359
+ * Invariants (Phase 1 completeness fix):
360
+ * - a new batch is merged into the queue, never substituted for it, so an
361
+ * unconfirmed batch keeps riding along with the next delivery instead of
362
+ * being stranded;
363
+ * - the result is id-ascending and id-deduped, so the same id is never
364
+ * delivered (or acknowledged) twice;
365
+ * - entries are only ever removed by `dropUnackedDelivered` (acknowledged or
366
+ * dead-lettered) or by a role/scope reset, so releasing the deferred wake
367
+ * cannot drop an unconfirmed event.
368
+ */
369
+ const mergeUnackedDelivered = (
370
+ incoming: readonly AgentEventWireRecord[],
371
+ ): AgentEventWireRecord[] => {
372
+ for (const event of incoming) {
373
+ if (!state.unackedDelivered.has(event.id)) state.unackedDelivered.set(event.id, event);
374
+ }
375
+ return unackedDeliveredAscending();
376
+ };
377
+
378
+ /**
379
+ * Drops one event from the delivery queue. The only callers are the two
380
+ * acknowledgement outcomes (accepted, or terminally refused by the daemon)
381
+ * and the role/scope reset, which clears the whole stream together with
382
+ * `presentedEventIds`.
383
+ */
384
+ const dropUnackedDelivered = (eventId: number): void => {
385
+ state.unackedDelivered.delete(eventId);
386
+ };
387
+
321
388
  const pruneAcknowledgedEvents = (ackedEventId: number | undefined) => {
322
389
  if (ackedEventId === undefined) return;
323
390
  // Acknowledged events leave the pending projection but intentionally stay
@@ -329,6 +396,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
329
396
  // set is bounded by the number of events presented per session and is
330
397
  // cleared on role loss, scope change, and shutdown.
331
398
  state.pendingEvents = state.pendingEvents.filter((event) => event.id > ackedEventId);
399
+ // The delivery queue follows the same watermark: an event covered by the
400
+ // advanced acknowledgement cursor is confirmed even when its own ack was
401
+ // superseded (the daemon acknowledges by watermark), so it leaves the
402
+ // queue and is never re-acknowledged.
403
+ for (const eventId of [...state.unackedDelivered.keys()]) {
404
+ if (eventId <= ackedEventId) state.unackedDelivered.delete(eventId);
405
+ }
332
406
  };
333
407
 
334
408
  const isWakeableEvent = (event: AgentEventWireRecord | undefined) =>
@@ -338,34 +412,290 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
338
412
  state.latestContext = isLocalOwner(response) ? response.context ?? undefined : undefined;
339
413
  };
340
414
 
415
+ const acknowledgeEventIds = async (
416
+ events: readonly AgentEventWireRecord[],
417
+ options: { notify: boolean },
418
+ ctx: PiContext,
419
+ ): Promise<void> => {
420
+ for (const event of [...events].sort((left, right) => left.id - right.id)) {
421
+ try {
422
+ // A missing client (disconnect) must never be treated as a successful
423
+ // acknowledgement: route it through the same failure path as a
424
+ // transient RPC error so the id keeps its backoff and stays pending.
425
+ if (!state.client) {
426
+ throw new Error("Herdsman Pi is not connected; cannot acknowledge notifications");
427
+ }
428
+ const ackResponse = (await state.client.request("agent.notifications.ack", {
429
+ eventId: event.id,
430
+ })) as { ackedEventId?: number; state?: { ackedEventId?: number } } | undefined;
431
+ pruneAcknowledgedEvents(ackResponse?.ackedEventId ?? ackResponse?.state?.ackedEventId);
432
+ state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
433
+ // The event is confirmed: it leaves the delivery queue for good, which
434
+ // is what keeps the acknowledgement watermark monotonic (ids are acked
435
+ // in ascending order and never re-issued).
436
+ dropUnackedDelivered(event.id);
437
+ // The id intentionally stays in presentedEventIds: the event was
438
+ // already presented this session and must not be injected again even
439
+ // if the daemon replays it (for example after a reconnect
440
+ // redelivery). The set is cleared only on role loss, scope change,
441
+ // or shutdown.
442
+ state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
443
+ setHerdsmanUi(ctx);
444
+ } catch (error) {
445
+ const failureCode = ackFailureCode(error);
446
+ const classification = classifyAckFailure(error);
447
+ // The attempt counter is read from the live projection, not from the
448
+ // delivery-queue snapshot the caller iterated over: a stale base would
449
+ // pin the counter at 1 for good and make the MAX_ACK_ATTEMPTS
450
+ // dead-letter branch unreachable for every event retried here.
451
+ const attempts =
452
+ (state.pendingEvents.find((pending) => pending.id === event.id)?.attempts ??
453
+ event.attempts ??
454
+ 0) + 1;
455
+ const attemptedAt = Date.now();
456
+ const updatedEvent = {
457
+ ...event,
458
+ attempts,
459
+ lastAttemptAt: attemptedAt,
460
+ lastFailureCode: failureCode,
461
+ };
462
+ state.pendingEvents = state.pendingEvents.map((pending) =>
463
+ pending.id === event.id ? updatedEvent : pending,
464
+ );
465
+ // Keep the delivery-queue entry in step with the live accounting: the
466
+ // same event may be retried later (another settlement, or a redelivery
467
+ // that re-attaches it to a new batch), and that attempt must resume
468
+ // from this counter instead of restarting at 1.
469
+ if (state.unackedDelivered.has(event.id)) {
470
+ state.unackedDelivered.set(event.id, updatedEvent);
471
+ }
472
+
473
+ if (classification === "terminal") {
474
+ state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
475
+ // A terminally refused event is dead-lettered by the daemon
476
+ // (failedWakeThroughEventId is the dead-letter barrier), so it can
477
+ // never be confirmed later; it leaves the delivery queue as well.
478
+ dropUnackedDelivered(event.id);
479
+ state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
480
+ if (/Only the current orchestrator can acknowledge notifications/i.test(failureCode)) {
481
+ state.isOrchestrator = false;
482
+ logHerdsmanPi(
483
+ "warn",
484
+ `[herdsman-pi] lost orchestrator ownership while acknowledging event ${event.id}`,
485
+ );
486
+ } else {
487
+ logHerdsmanPi(
488
+ "warn",
489
+ `[herdsman-pi] terminal acknowledgement failure eventId=${event.id} attempts=${attempts} code=${failureCode}`,
490
+ );
491
+ }
492
+ setHerdsmanUi(ctx);
493
+ continue;
494
+ }
495
+
496
+ if (attempts >= MAX_ACK_ATTEMPTS) {
497
+ state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
498
+ // Dead-lettered by this client: the id is behind the dead-letter
499
+ // barrier from now on, so it can never be confirmed later and leaves
500
+ // the delivery queue with the pending projection.
501
+ dropUnackedDelivered(event.id);
502
+ state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
503
+ logHerdsmanPi(
504
+ "warn",
505
+ `[herdsman-pi] acknowledgement moved to dead-letter eventId=${event.id} attempts=${attempts} code=${failureCode}`,
506
+ );
507
+ setHerdsmanUi(ctx);
508
+ continue;
509
+ }
510
+
511
+ if (classification === "resync") {
512
+ const resyncEvent = {
513
+ ...updatedEvent,
514
+ nextAttemptAt: attemptedAt + ackBackoffMs(attempts),
515
+ };
516
+ state.pendingEvents = state.pendingEvents.map((pending) =>
517
+ pending.id === event.id ? resyncEvent : pending,
518
+ );
519
+ try {
520
+ const response = (await state.client?.request(
521
+ "agent.orchestrator.get",
522
+ {},
523
+ )) as ConnectionStateResponse | undefined;
524
+ // Refresh pending data without applying the full connection response: that
525
+ // helper schedules a new wake, which would make this failed batch race
526
+ // with the current settlement and can replay an earlier event. The
527
+ // failed event remains pending and the next wake is scheduled by
528
+ // finishBatch(), so this round performs no additional acknowledgements.
529
+ if (response) addPendingEvents(response.events ?? [], ctx);
530
+ pruneAcknowledgedEvents(response?.state?.ackedEventId ?? response?.ackedEventId);
531
+ setHerdsmanUi(ctx);
532
+ } catch (resyncError) {
533
+ logHerdsmanPi(
534
+ "warn",
535
+ `[herdsman-pi] acknowledgement resync failed eventId=${event.id} attempts=${attempts} code=${ackFailureCode(resyncError)}`,
536
+ );
537
+ }
538
+ // The event stays pending with a backoff so the ack cursor can
539
+ // sweep it later, but it was already presented this session and is
540
+ // not re-presented: presentedEventIds keeps the guard until scope
541
+ // reset. Continue so one failed event does not block the rest of
542
+ // the batch.
543
+ continue;
544
+ }
545
+
546
+ state.pendingEvents = state.pendingEvents.map((pending) =>
547
+ pending.id === event.id
548
+ ? { ...pending, nextAttemptAt: attemptedAt + ackBackoffMs(attempts) }
549
+ : pending,
550
+ );
551
+ if (options.notify) {
552
+ ctx.ui.notify?.(
553
+ "Herdsman couldn’t acknowledge agent updates · updates remain pending",
554
+ "warning",
555
+ );
556
+ }
557
+ setHerdsmanUi(ctx);
558
+ // The event stays pending with a backoff so the ack cursor can sweep
559
+ // it later, but it was already presented this session and is not
560
+ // re-presented: presentedEventIds keeps the guard until scope reset.
561
+ continue;
562
+ }
563
+ }
564
+ };
565
+
566
+ // Upstream model errors wake nobody, but they still have to leave the
567
+ // daemon's pending queue or it never converges. The silent path therefore
568
+ // acknowledges them without sendMessage, without notify, and without
569
+ // touching presentedEventIds (reserved for genuinely presented outcomes).
570
+ const scheduleSilentUpstreamErrorAck = (
571
+ ctx: PiContext,
572
+ events: readonly AgentEventWireRecord[],
573
+ ) => {
574
+ if (state.wakeTimer || state.wakeRequested) return;
575
+ const scope = state.currentScope;
576
+ if (!state.isOrchestrator || !scope) return;
577
+ const generation = wakeGeneration;
578
+ const ownerHerdrSessionName = scope.herdrSessionName;
579
+ const ownerTerminalId = scope.terminalId;
580
+ const ownerWorkspaceId = scope.workspaceId;
581
+ state.wakeTimer = setTimeout(() => {
582
+ state.wakeTimer = undefined;
583
+ void (async () => {
584
+ if (
585
+ generation !== wakeGeneration ||
586
+ !state.isOrchestrator ||
587
+ state.currentScope?.herdrSessionName !== ownerHerdrSessionName ||
588
+ state.currentScope?.terminalId !== ownerTerminalId ||
589
+ state.currentScope?.workspaceId !== ownerWorkspaceId
590
+ ) {
591
+ return;
592
+ }
593
+ // A delivered batch or an in-flight ack owns the cursor; its
594
+ // settlement schedules the next sweep instead of racing this one.
595
+ if (state.deliveredBatch || state.ackInFlight) return;
596
+ if (!state.client || !state.connected) return;
597
+ state.ackInFlight = true;
598
+ setHerdsmanUi(ctx);
599
+ try {
600
+ await acknowledgeEventIds(events, { notify: false }, ctx);
601
+ } finally {
602
+ state.ackInFlight = false;
603
+ setHerdsmanUi(ctx);
604
+ }
605
+ scheduleWake(ctx);
606
+ })();
607
+ }, WAKE_SETTLE_MS);
608
+ };
609
+
610
+ /**
611
+ * Arms the bounded deferral for a wake that cannot be injected right now.
612
+ *
613
+ * The retry spins every `WAKE_BUSY_SPIN_MS` while the orchestrator is busy
614
+ * and flips `wakeForcedRelease` once `WAKE_DEFERRED_TIMEOUT_MS` has elapsed,
615
+ * so the next pass injects the batch from the current state (as a queued,
616
+ * non-triggering follow-up) instead of waiting for an idle signal or a
617
+ * settlement that may never arrive.
618
+ */
619
+ const scheduleDeferredWake = (ctx: PiContext) => {
620
+ state.wakeDeferredUntilSettled = true;
621
+ const since = state.wakeDeferredSince ?? Date.now();
622
+ state.wakeDeferredSince = since;
623
+ const remaining = WAKE_DEFERRED_TIMEOUT_MS - (Date.now() - since);
624
+ if (remaining <= 0) state.wakeForcedRelease = true;
625
+ if (state.wakeTimer) return;
626
+ state.wakeTimer = setTimeout(() => {
627
+ state.wakeTimer = undefined;
628
+ if (state.wakeForcedRelease && state.deliveredBatch && ctx.isIdle?.() !== false) {
629
+ // The hard deadline only releases the delivery — it never discards an
630
+ // unconfirmed event. The batch record is dropped so a wake turn that
631
+ // is no longer running cannot gate later wakes, but its events stay in
632
+ // the delivery queue (`unackedDelivered`) and are re-attached to the
633
+ // batch injected right below, which acknowledges them once it settles.
634
+ state.deliveredBatch = undefined;
635
+ }
636
+ scheduleWake(ctx);
637
+ }, Math.max(0, Math.min(WAKE_BUSY_SPIN_MS, remaining)));
638
+ };
639
+
341
640
  const scheduleWake = (ctx: PiContext | undefined) => {
342
641
  if (!ctx || !state.isOrchestrator || !state.currentScope || !pi.sendMessage) return;
343
642
  if (state.wakeTimer || state.wakeRequested) return;
344
- const outcomes = projectAgentOutcomes(state.pendingEvents).outcomes.filter(
643
+ const projection = projectAgentOutcomes(state.pendingEvents, wakeFilter);
644
+ const outcomes = projection.outcomes.filter(
345
645
  (outcome) =>
346
646
  outcome.eventId > state.failedWakeThroughEventId &&
347
647
  !state.presentedEventIds.has(outcome.eventId),
348
648
  );
649
+ const suppressedEvents = projection.suppressedUpstreamErrorEventIds
650
+ .filter(
651
+ (eventId) =>
652
+ eventId > state.failedWakeThroughEventId && !state.presentedEventIds.has(eventId),
653
+ )
654
+ .map((eventId) => state.pendingEvents.find((pending) => pending.id === eventId))
655
+ .filter((event): event is AgentEventWireRecord => event !== undefined)
656
+ .sort((left, right) => left.id - right.id);
349
657
 
350
658
  const wakeable = outcomes.filter((outcome) =>
351
659
  isWakeableEvent(state.pendingEvents.find((pending) => pending.id === outcome.eventId)),
352
660
  );
353
661
  if (wakeable.length === 0) {
354
- const nextAttemptAt = outcomes
355
- .map((outcome) => state.pendingEvents.find((event) => event.id === outcome.eventId)?.nextAttemptAt)
356
- .filter((value): value is number => value !== undefined)
662
+ // A suppressed upstream error that is now due must be silently
663
+ // acknowledged before any backoff timer is planted: planting the timer
664
+ // first would make scheduleSilentUpstreamErrorAck's entry guard
665
+ // (`if (state.wakeTimer || state.wakeRequested) return`) bounce the ack
666
+ // off its own timer and, once the backoff window has elapsed, the
667
+ // 0ms-timer + blocked-ack loop never converges the queue.
668
+ const dueSuppressed = suppressedEvents.filter(isWakeableEvent);
669
+ if (dueSuppressed.length > 0) {
670
+ scheduleSilentUpstreamErrorAck(ctx, dueSuppressed);
671
+ return;
672
+ }
673
+ // No suppressed event is due, so plant a backoff timer for the next
674
+ // future `nextAttemptAt`. Expired timestamps are excluded (strictly
675
+ // greater than now) so an already-past window does not produce a
676
+ // zero-delay spin.
677
+ const nextAttemptAt = [
678
+ ...outcomes.map((outcome) => outcome.eventId),
679
+ ...suppressedEvents.map((event) => event.id),
680
+ ]
681
+ .map((eventId) => state.pendingEvents.find((event) => event.id === eventId)?.nextAttemptAt)
682
+ .filter((value): value is number => value !== undefined && value > Date.now())
357
683
  .sort((left, right) => left - right)[0];
358
684
  if (nextAttemptAt !== undefined) {
359
685
  state.wakeTimer = setTimeout(() => {
360
686
  state.wakeTimer = undefined;
361
687
  scheduleWake(ctx);
362
- }, Math.max(0, nextAttemptAt - Date.now()));
688
+ }, nextAttemptAt - Date.now());
363
689
  }
364
690
  return;
365
691
  }
366
692
 
367
- if (state.deliveredBatch || state.ackInFlight || ctx.isIdle?.() === false) {
368
- state.wakeDeferredUntilSettled = true;
693
+ // An in-flight batch owns the ack cursor and a busy orchestrator must not
694
+ // be interrupted, so neither is woken immediately — but both are deferred
695
+ // on a bounded spin (never parked until an event that may never come).
696
+ const inFlight = state.deliveredBatch !== undefined || state.ackInFlight;
697
+ if (!state.wakeForcedRelease && (inFlight || ctx.isIdle?.() === false)) {
698
+ scheduleDeferredWake(ctx);
369
699
  return;
370
700
  }
371
701
  const generation = wakeGeneration;
@@ -384,9 +714,9 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
384
714
  state.wakeTimer = undefined;
385
715
  return;
386
716
  }
387
- if (ctx.isIdle?.() === false) {
717
+ if (ctx.isIdle?.() === false && !state.wakeForcedRelease) {
388
718
  state.wakeTimer = undefined;
389
- state.wakeDeferredUntilSettled = true;
719
+ scheduleDeferredWake(ctx);
390
720
  return;
391
721
  }
392
722
 
@@ -421,14 +751,16 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
421
751
  state.wakeTimer = undefined;
422
752
  return;
423
753
  }
424
- if (ctx.isIdle?.() === false) {
754
+ if (ctx.isIdle?.() === false && !state.wakeForcedRelease) {
425
755
  state.wakeTimer = undefined;
426
- state.wakeDeferredUntilSettled = true;
756
+ scheduleDeferredWake(ctx);
427
757
  return;
428
758
  }
429
759
 
430
760
  const batchEvents = [...state.pendingEvents].sort((left, right) => left.id - right.id);
431
- const batchOutcomes = projectAgentOutcomes(batchEvents).outcomes.filter(
761
+ const batchProjection = projectAgentOutcomes(batchEvents, wakeFilter);
762
+ const batchSuppressedIds = new Set(batchProjection.suppressedUpstreamErrorEventIds);
763
+ const batchOutcomes = batchProjection.outcomes.filter(
432
764
  (outcome) =>
433
765
  outcome.eventId > state.failedWakeThroughEventId &&
434
766
  !state.presentedEventIds.has(outcome.eventId) &&
@@ -439,33 +771,73 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
439
771
  return;
440
772
  }
441
773
  const current = batchOutcomes;
442
- const deliveredBatch: DeliveredBatch = {
443
- abortedByUser: false,
444
- assistantFinalSucceeded: false,
445
- events: batchEvents.filter((event) =>
446
- batchOutcomes.some((outcome) => outcome.eventId === event.id),
447
- ),
448
- hasSubstantiveWork: false,
449
- invalidated: false,
450
- ownerTerminalId,
451
- herdsmanTriggered: true,
452
- };
774
+ const incomingEvents = batchEvents.filter((event) =>
775
+ batchOutcomes.some((outcome) => outcome.eventId === event.id),
776
+ );
777
+ const previousBatch = state.deliveredBatch;
453
778
  state.wakeTimer = undefined;
454
779
  state.wakeRequested = true;
455
780
  state.wakeRequestedThroughEventId = current.at(-1)?.eventId ?? 0;
781
+ // Dual-track injection: an idle orchestrator gets a triggered
782
+ // follow-up turn (immediate delivery), while a busy one is not
783
+ // interrupted — the same content is queued as a non-triggering
784
+ // follow-up and additionally exposed through the `context` hook so the
785
+ // running turn can already see it.
786
+ const orchestratorBusy = ctx.isIdle?.() === false;
787
+ const wakeContent = formatAgentOutcomeUpdates(batchOutcomes);
456
788
  try {
457
789
  pi.sendMessage?.(
458
790
  {
459
- content: formatAgentOutcomeUpdates(batchOutcomes),
791
+ content: wakeContent,
460
792
  customType: "herdsman-wake-context",
461
- details: { eventIds: batchEvents.map((event) => event.id) },
793
+ // Suppressed upstream errors are dropped from the injected
794
+ // context, but every other pending id stays listed so the
795
+ // evidence trail for the decision still names what was pending.
796
+ details: {
797
+ eventIds: batchEvents
798
+ .filter((event) => !batchSuppressedIds.has(event.id))
799
+ .map((event) => event.id),
800
+ },
462
801
  display: false,
463
802
  },
464
- { deliverAs: "followUp", triggerTurn: true },
803
+ orchestratorBusy
804
+ ? { deliverAs: "followUp", triggerTurn: false }
805
+ : { deliverAs: "followUp", triggerTurn: true },
465
806
  );
807
+ // A queued (non-triggering) delivery keeps the content available to
808
+ // the current turn through the context hook until it is settled or
809
+ // superseded by the next injection.
810
+ state.wakeContext = orchestratorBusy
811
+ ? { content: wakeContent, eventIds: batchOutcomes.map((outcome) => outcome.eventId) }
812
+ : undefined;
813
+ state.wakeForcedRelease = false;
814
+ state.wakeDeferredSince = undefined;
466
815
  // Only expose the batch after the hidden context was accepted by pi. This
467
816
  // keeps an injection failure eligible for daemon redelivery.
468
- state.deliveredBatch = deliveredBatch;
817
+ //
818
+ // The batch is the delivery queue plus this injection: previously
819
+ // unconfirmed events are merged (never replaced) so that a forced
820
+ // release always leaves both the old and the new events deliverable
821
+ // and acknowledgeable in id order. The turn-consumption flags of a
822
+ // still-running previous batch are carried over, because they
823
+ // describe whether the content already reached the orchestrator.
824
+ //
825
+ // `hasSubstantiveWork` is the sole gate that decides whether an
826
+ // ownership/scope change may abort the in-flight turn (see loseRole
827
+ // and resetForScopeChange), and aborting is only ever allowed for a
828
+ // *pure* Herdsman wake turn. A busy orchestrator gets the batch as a
829
+ // non-triggering queued follow-up, which rides the user's own turn:
830
+ // that turn is not a Herdsman wake turn, so it must never be aborted
831
+ // on our behalf and the flag is set here.
832
+ state.deliveredBatch = {
833
+ abortedByUser: previousBatch?.abortedByUser ?? false,
834
+ assistantFinalSucceeded: previousBatch?.assistantFinalSucceeded ?? false,
835
+ events: mergeUnackedDelivered(incomingEvents),
836
+ hasSubstantiveWork: orchestratorBusy || (previousBatch?.hasSubstantiveWork ?? false),
837
+ invalidated: false,
838
+ ownerTerminalId,
839
+ herdsmanTriggered: true,
840
+ };
469
841
  state.wakeRequested = false;
470
842
  state.wakeRequestedThroughEventId = 0;
471
843
  // Record the presentation so a reclaim redelivery of the same id is
@@ -522,7 +894,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
522
894
  // A transient disconnect (reconnect) keeps the presentation guard so an
523
895
  // event already presented in this scope session is not presented again;
524
896
  // only a genuine role/scope loss or shutdown resets it.
525
- if (!options.preservePresented) state.presentedEventIds.clear();
897
+ if (!options.preservePresented) {
898
+ state.presentedEventIds.clear();
899
+ // The delivery queue dies with the presentation guard: once the guard is
900
+ // gone the daemon's pending events can be presented (and acknowledged)
901
+ // again, so keeping the old queue would only risk a stale id.
902
+ state.unackedDelivered.clear();
903
+ }
526
904
  state.reconnectingFromOn = false;
527
905
  setHerdsmanUi(ctx);
528
906
  };
@@ -553,6 +931,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
553
931
  state.failedWakeThroughEventId = 0;
554
932
  state.pendingEvents = [];
555
933
  state.presentedEventIds.clear();
934
+ state.unackedDelivered.clear();
556
935
  setHerdsmanUi(ctx);
557
936
  };
558
937
 
@@ -670,6 +1049,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
670
1049
  // already acked events are covered by the server cursor
671
1050
  // (pruneAcknowledgedEvents below).
672
1051
  state.deliveredBatch = undefined;
1052
+ state.wakeContext = undefined;
673
1053
  }
674
1054
  // Otherwise the batch's wake turn is still in flight: keep it so the
675
1055
  // settlement acknowledges it and the events are not re-presented.
@@ -911,25 +1291,16 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
911
1291
  loseRole(activeContext);
912
1292
  state.deliveredBatch = undefined;
913
1293
  state.presentedEventIds.clear();
1294
+ state.unackedDelivered.clear();
914
1295
  state.client?.close();
915
1296
  state.client = undefined;
916
1297
  activeContext = undefined;
917
1298
  });
918
1299
 
919
1300
  const assistantMessageText = (message: Record<string, unknown>): string => {
920
- const content = message.content;
921
- if (typeof content === "string") return content;
922
- if (!Array.isArray(content)) return "";
923
- const parts: string[] = [];
924
- for (const block of content) {
925
- if (typeof block === "string") {
926
- if (block.length > 0) parts.push(block);
927
- continue;
928
- }
929
- const value = record(block);
930
- if (typeof value.text === "string" && value.text.length > 0) parts.push(value.text);
931
- }
932
- return parts.join("\n");
1301
+ const text = textFromContent(message.content);
1302
+ if (text === null) return "";
1303
+ return sanitizeText(text).text;
933
1304
  };
934
1305
 
935
1306
  // Turn completion signal: after Pi's own final assistant message has been
@@ -945,13 +1316,15 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
945
1316
  const completion = (async () => {
946
1317
  const check = await confirmSessionWrite({ expectedText, path: sessionPath });
947
1318
  try {
948
- await client.request("agent.turn.completed", {
1319
+ const params: Record<string, unknown> = {
949
1320
  confirmed: check.confirmed,
950
1321
  herdrSessionName: scope.herdrSessionName,
951
1322
  paneId: scope.paneId,
952
1323
  terminalId: scope.terminalId,
953
1324
  workspaceId: scope.workspaceId,
954
- });
1325
+ };
1326
+ if (expectedText) params.expectedText = expectedText;
1327
+ await client.request("agent.turn.completed", params);
955
1328
  } catch (error) {
956
1329
  logHerdsmanPi(
957
1330
  "warn",
@@ -1002,23 +1375,43 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1002
1375
 
1003
1376
  pi.on("context", (event: { messages: PiAgentMessage[] }) => {
1004
1377
  const messages = event.messages.filter((message) => !isNormalHerdsmanContext(message));
1378
+ const additions: PiAgentMessage[] = [];
1005
1379
  const snapshot = state.pinnedContext;
1006
- if (!snapshot || snapshot.agents.length === 0) return { messages };
1007
- return {
1008
- messages: [
1009
- ...messages,
1010
- {
1011
- content: formatHiddenAgentContext({
1012
- agents: snapshot.agents,
1013
- workspaceId: snapshot.workspaceId,
1014
- }),
1015
- customType: "herdsman-agent-context",
1016
- display: false,
1017
- role: "custom",
1018
- timestamp: Date.now(),
1019
- },
1020
- ],
1021
- };
1380
+ if (snapshot && snapshot.agents.length > 0) {
1381
+ additions.push({
1382
+ content: formatHiddenAgentContext({
1383
+ agents: snapshot.agents,
1384
+ workspaceId: snapshot.workspaceId,
1385
+ }),
1386
+ customType: "herdsman-agent-context",
1387
+ display: false,
1388
+ role: "custom",
1389
+ timestamp: Date.now(),
1390
+ });
1391
+ }
1392
+ // A wake queued for a busy orchestrator is not allowed to interrupt the
1393
+ // running tool chain, so its content is additionally pinned to the current
1394
+ // context: the orchestrator sees the child-agent outcome in this turn
1395
+ // without a triggered follow-up. The entry is dropped again by
1396
+ // isNormalHerdsmanContext, so at most one copy is present per call.
1397
+ //
1398
+ // `eventIds` mirrors what this turn actually presents (the freshly injected
1399
+ // outcomes): events carried over in the delivery queue were already shown
1400
+ // to the orchestrator in the turn that presented them, so they are not
1401
+ // re-listed here. The queued follow-up message itself keeps the wider
1402
+ // `details.eventIds` set (everything still unconfirmed).
1403
+ const queuedWake = state.wakeContext;
1404
+ if (queuedWake) {
1405
+ additions.push({
1406
+ content: queuedWake.content,
1407
+ customType: "herdsman-wake-queued",
1408
+ details: { eventIds: queuedWake.eventIds },
1409
+ display: false,
1410
+ role: "custom",
1411
+ timestamp: Date.now(),
1412
+ });
1413
+ }
1414
+ return additions.length === 0 ? { messages } : { messages: [...messages, ...additions] };
1022
1415
  });
1023
1416
 
1024
1417
  pi.on("agent_settled", async (_event: unknown, ctx: PiContext) => {
@@ -1043,10 +1436,41 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1043
1436
  const finishBatch = () => {
1044
1437
  state.ackInFlight = false;
1045
1438
  state.wakeDeferredUntilSettled = false;
1439
+ state.wakeDeferredSince = undefined;
1440
+ state.wakeForcedRelease = false;
1441
+ state.wakeContext = undefined;
1046
1442
  setHerdsmanUi(ctx);
1047
1443
  scheduleWake(ctx);
1048
1444
  };
1049
1445
 
1446
+ // The delivery queue — not the injection snapshot — is what gets
1447
+ // acknowledged: it holds every event handed to Pi that is still
1448
+ // unconfirmed (a release merges batches instead of replacing them), is
1449
+ // id-ascending, and an id leaves it only when the daemon accepts it.
1450
+ //
1451
+ // An event whose own acknowledgement already failed is left out here: it is
1452
+ // never retried by this path (the daemon's cursor advance sweeps it, and a
1453
+ // retry would reset its attempt/backoff accounting), but it stays in the
1454
+ // queue until that cursor or a scope reset confirms it. A failed or
1455
+ // dead-lettered acknowledgement thus never depends on the timing of the
1456
+ // release to stay recoverable.
1457
+ //
1458
+ // Restricting to a *live* row with `attempts === 0` has two holes on
1459
+ // purpose. `?? 0` covers an event the live projection no longer holds at
1460
+ // all (the server stopped listing it, or `failedWakeThroughEventId`
1461
+ // filters it out): the projection cannot tell us it already failed, so the
1462
+ // event gets one more attempt — a deliberate self-healing opportunity that
1463
+ // then accumulates on the queue copy's counter and can reach
1464
+ // MAX_ACK_ATTEMPTS instead of restarting at 1 every round.
1465
+ const ackable = unackedDeliveredAscending().filter(
1466
+ (event) =>
1467
+ (state.pendingEvents.find((pending) => pending.id === event.id)?.attempts ?? 0) === 0,
1468
+ );
1469
+ if (ackable.length === 0) {
1470
+ finishBatch();
1471
+ return;
1472
+ }
1473
+
1050
1474
  if (
1051
1475
  (!batch.assistantFinalSucceeded && !batch.abortedByUser) ||
1052
1476
  batch.invalidated ||
@@ -1059,116 +1483,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
1059
1483
  return;
1060
1484
  }
1061
1485
 
1062
- for (const event of batch.events) {
1063
- try {
1064
- const ackResponse = (await state.client.request("agent.notifications.ack", {
1065
- eventId: event.id,
1066
- })) as { ackedEventId?: number; state?: { ackedEventId?: number } };
1067
- pruneAcknowledgedEvents(ackResponse?.ackedEventId ?? ackResponse?.state?.ackedEventId);
1068
- state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
1069
- // The id intentionally stays in presentedEventIds: the event was
1070
- // already presented this session and must not be injected again even
1071
- // if the daemon replays it (for example after a reconnect
1072
- // redelivery). The set is cleared only on role loss, scope change,
1073
- // or shutdown.
1074
- state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
1075
- setHerdsmanUi(ctx);
1076
- } catch (error) {
1077
- const failureCode = ackFailureCode(error);
1078
- const classification = classifyAckFailure(error);
1079
- const attempts = (event.attempts ?? 0) + 1;
1080
- const attemptedAt = Date.now();
1081
- const updatedEvent = {
1082
- ...event,
1083
- attempts,
1084
- lastAttemptAt: attemptedAt,
1085
- lastFailureCode: failureCode,
1086
- };
1087
- state.pendingEvents = state.pendingEvents.map((pending) =>
1088
- pending.id === event.id ? updatedEvent : pending,
1089
- );
1090
-
1091
- if (classification === "terminal") {
1092
- state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
1093
- state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
1094
- if (/Only the current orchestrator can acknowledge notifications/i.test(failureCode)) {
1095
- state.isOrchestrator = false;
1096
- logHerdsmanPi(
1097
- "warn",
1098
- `[herdsman-pi] lost orchestrator ownership while acknowledging event ${event.id}`,
1099
- );
1100
- } else {
1101
- logHerdsmanPi(
1102
- "warn",
1103
- `[herdsman-pi] terminal acknowledgement failure eventId=${event.id} attempts=${attempts} code=${failureCode}`,
1104
- );
1105
- }
1106
- setHerdsmanUi(ctx);
1107
- continue;
1108
- }
1109
-
1110
- if (attempts >= MAX_ACK_ATTEMPTS) {
1111
- state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
1112
- state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
1113
- logHerdsmanPi(
1114
- "warn",
1115
- `[herdsman-pi] acknowledgement moved to dead-letter eventId=${event.id} attempts=${attempts} code=${failureCode}`,
1116
- );
1117
- setHerdsmanUi(ctx);
1118
- continue;
1119
- }
1120
-
1121
- if (classification === "resync") {
1122
- const resyncEvent = {
1123
- ...updatedEvent,
1124
- nextAttemptAt: attemptedAt + ackBackoffMs(attempts),
1125
- };
1126
- state.pendingEvents = state.pendingEvents.map((pending) =>
1127
- pending.id === event.id ? resyncEvent : pending,
1128
- );
1129
- try {
1130
- const response = (await state.client.request(
1131
- "agent.orchestrator.get",
1132
- {},
1133
- )) as ConnectionStateResponse;
1134
- // Refresh pending data without applying the full connection response: that
1135
- // helper schedules a new wake, which would make this failed batch race
1136
- // with the current settlement and can replay an earlier event. The
1137
- // failed event remains pending and the next wake is scheduled by
1138
- // finishBatch(), so this round performs no additional acknowledgements.
1139
- addPendingEvents(response.events ?? [], ctx);
1140
- pruneAcknowledgedEvents(response.state?.ackedEventId ?? response.ackedEventId);
1141
- setHerdsmanUi(ctx);
1142
- } catch (resyncError) {
1143
- logHerdsmanPi(
1144
- "warn",
1145
- `[herdsman-pi] acknowledgement resync failed eventId=${event.id} attempts=${attempts} code=${ackFailureCode(resyncError)}`,
1146
- );
1147
- }
1148
- // The event stays pending with a backoff so the ack cursor can
1149
- // sweep it later, but it was already presented this session and is
1150
- // not re-presented: presentedEventIds keeps the guard until scope
1151
- // reset. Continue so one failed event does not block the rest of
1152
- // the batch.
1153
- continue;
1154
- }
1155
-
1156
- state.pendingEvents = state.pendingEvents.map((pending) =>
1157
- pending.id === event.id
1158
- ? { ...pending, nextAttemptAt: attemptedAt + ackBackoffMs(attempts) }
1159
- : pending,
1160
- );
1161
- ctx.ui.notify?.(
1162
- "Herdsman couldn’t acknowledge agent updates · updates remain pending",
1163
- "warning",
1164
- );
1165
- setHerdsmanUi(ctx);
1166
- // The event stays pending with a backoff so the ack cursor can sweep
1167
- // it later, but it was already presented this session and is not
1168
- // re-presented: presentedEventIds keeps the guard until scope reset.
1169
- continue;
1170
- }
1171
- }
1486
+ await acknowledgeEventIds(ackable, { notify: true }, ctx);
1172
1487
  finishBatch();
1173
1488
  });
1174
1489
 
@@ -1255,6 +1570,7 @@ export function formatHiddenAgentUpdates(events: AgentEventWireRecord[]): string
1255
1570
  function isNormalHerdsmanContext(message: PiAgentMessage): boolean {
1256
1571
  return (
1257
1572
  message.customType === "herdsman-agent-context" ||
1573
+ message.customType === "herdsman-wake-queued" ||
1258
1574
  contentIncludesMarker(message.content, "[HERDSMAN AGENT CONTEXT]")
1259
1575
  );
1260
1576
  }