@dorokuma/herdsman-pi 0.11.6 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/package.json +2 -5
- package/src/index.ts +505 -189
- package/src/logger.ts +19 -0
- package/src/sanitize-text.ts +21 -0
- package/src/upstream-error.ts +201 -0
- package/src/wake-filter-config.ts +265 -0
- package/src/wake.ts +67 -11
package/src/index.ts
CHANGED
|
@@ -1,25 +1,10 @@
|
|
|
1
1
|
import { agentIdentityLabel } from "./agent-display.js";
|
|
2
|
-
import { sanitizeText } from "./sanitize-text.js";
|
|
3
|
-
import { appendFileSync, mkdirSync } from "node:fs";
|
|
4
|
-
import { homedir } from "node:os";
|
|
5
|
-
import { dirname, isAbsolute, join } from "node:path";
|
|
2
|
+
import { sanitizeText, textFromContent } from "./sanitize-text.js";
|
|
6
3
|
import { stripVTControlCharacters } from "node:util";
|
|
4
|
+
import { logHerdsmanPi } from "./logger.js";
|
|
7
5
|
|
|
8
|
-
export
|
|
9
|
-
|
|
10
|
-
export function logHerdsmanPi(level: HerdsmanPiLogLevel, message: string): void {
|
|
11
|
-
try {
|
|
12
|
-
const configuredHome = process.env.HERDSMAN_HOME?.trim();
|
|
13
|
-
const home = configuredHome && isAbsolute(configuredHome) ? configuredHome : join(homedir(), ".herdsman");
|
|
14
|
-
const now = new Date();
|
|
15
|
-
const date = now.toISOString().slice(0, 10).replaceAll("-", "");
|
|
16
|
-
const file = join(home, "logs", `herdsman-pi-${date}.log`);
|
|
17
|
-
mkdirSync(dirname(file), { recursive: true });
|
|
18
|
-
appendFileSync(file, `${now.toISOString()} [${level}] ${message}\n`, "utf8");
|
|
19
|
-
} catch {
|
|
20
|
-
// Diagnostics must never write to the terminal or interrupt the extension.
|
|
21
|
-
}
|
|
22
|
-
}
|
|
6
|
+
export { logHerdsmanPi };
|
|
7
|
+
export type { HerdsmanPiLogLevel } from "./logger.js";
|
|
23
8
|
|
|
24
9
|
import {
|
|
25
10
|
type AgentContextListItem,
|
|
@@ -41,6 +26,8 @@ import {
|
|
|
41
26
|
formatAgentOutcomeUpdates,
|
|
42
27
|
WAKE_SETTLE_MS,
|
|
43
28
|
} from "./wake.js";
|
|
29
|
+
import { loadWakeFilterConfig } from "./wake-filter-config.js";
|
|
30
|
+
import type { WakeFilterConfig } from "./upstream-error.js";
|
|
44
31
|
import { confirmSessionWrite } from "./turn-signal.js";
|
|
45
32
|
|
|
46
33
|
type PiAgentMessage = {
|
|
@@ -126,7 +113,29 @@ type HerdsmanState = {
|
|
|
126
113
|
roleMutationInFlight: boolean;
|
|
127
114
|
sessionRef: AgentSessionRef | undefined;
|
|
128
115
|
subscriberId: string | undefined;
|
|
116
|
+
/**
|
|
117
|
+
* Delivery queue: events handed to Pi whose acknowledgement is still
|
|
118
|
+
* outstanding. Id-keyed so a repeated id is stored exactly once (deliver
|
|
119
|
+
* once), and every consumer sorts by id. Entries are only removed by an
|
|
120
|
+
* acknowledgement (success or dead-letter) or by a role/scope reset that also
|
|
121
|
+
* clears `presentedEventIds`, so releasing a deferred wake can never lose an
|
|
122
|
+
* unconfirmed event.
|
|
123
|
+
*/
|
|
124
|
+
unackedDelivered: Map<number, AgentEventWireRecord>;
|
|
129
125
|
wakeDeferredUntilSettled: boolean;
|
|
126
|
+
/** Wall-clock start of the current bounded wake deferral, if any. */
|
|
127
|
+
wakeDeferredSince: number | undefined;
|
|
128
|
+
/**
|
|
129
|
+
* Set once the hard deferral budget elapsed: the next pass injects the batch
|
|
130
|
+
* from the current state instead of deferring again.
|
|
131
|
+
*/
|
|
132
|
+
wakeForcedRelease: boolean;
|
|
133
|
+
/**
|
|
134
|
+
* Event content queued for a busy orchestrator. Injected through the
|
|
135
|
+
* `context` hook so the running turn sees the update without being
|
|
136
|
+
* interrupted.
|
|
137
|
+
*/
|
|
138
|
+
wakeContext: { content: string; eventIds: number[] } | undefined;
|
|
130
139
|
wakeRequested: boolean;
|
|
131
140
|
wakeRequestedThroughEventId: number;
|
|
132
141
|
wakeTimer: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -173,6 +182,7 @@ type ExtensionOptions = {
|
|
|
173
182
|
clientFactory?: () => HerdsmanDaemonClient;
|
|
174
183
|
onTurnCompletionSignal?: (completion: Promise<void>) => void;
|
|
175
184
|
onStateExposed?: (state: HerdsmanState) => void;
|
|
185
|
+
wakeFilter?: WakeFilterConfig;
|
|
176
186
|
};
|
|
177
187
|
|
|
178
188
|
const DEFAULT_HOME_NAME = ".herdsman";
|
|
@@ -182,6 +192,15 @@ const RECONNECTING_MESSAGE = "Herdsman is reconnecting · try again shortly";
|
|
|
182
192
|
export const MAX_ACK_ATTEMPTS = 5;
|
|
183
193
|
export const ACK_BACKOFF_CAP_MS = 30_000;
|
|
184
194
|
const KEEPALIVE_INTERVAL_MS = 30_000;
|
|
195
|
+
/** Retry interval used while a wake cannot be injected (busy orchestrator). */
|
|
196
|
+
export const WAKE_BUSY_SPIN_MS = 100;
|
|
197
|
+
/**
|
|
198
|
+
* Hard upper bound for every deferred wake. Once it elapses the scheduler stops
|
|
199
|
+
* waiting for the orchestrator to become idle or for a settlement to arrive and
|
|
200
|
+
* forces the injection decision from the current state, so a wake can never be
|
|
201
|
+
* parked forever.
|
|
202
|
+
*/
|
|
203
|
+
export const WAKE_DEFERRED_TIMEOUT_MS = 5_000;
|
|
185
204
|
|
|
186
205
|
type AckFailureClass = "terminal" | "resync" | "transient";
|
|
187
206
|
|
|
@@ -242,6 +261,9 @@ export function defaultSocketPath() {
|
|
|
242
261
|
export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
243
262
|
return function herdsmanPiExtension(pi: PiApi): void {
|
|
244
263
|
pi.registerMessageRenderer?.("herdsman-wake", renderAgentUpdateMessage);
|
|
264
|
+
// Read once per extension instance: config.yaml changes require a Pi restart,
|
|
265
|
+
// matching the daemon's startup-time config model.
|
|
266
|
+
const wakeFilter = options.wakeFilter ?? loadWakeFilterConfig();
|
|
245
267
|
|
|
246
268
|
const state: HerdsmanState = {
|
|
247
269
|
client: undefined,
|
|
@@ -261,7 +283,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
261
283
|
runActive: false,
|
|
262
284
|
sessionRef: undefined,
|
|
263
285
|
subscriberId: undefined,
|
|
286
|
+
unackedDelivered: new Map(),
|
|
264
287
|
wakeDeferredUntilSettled: false,
|
|
288
|
+
wakeDeferredSince: undefined,
|
|
289
|
+
wakeForcedRelease: false,
|
|
290
|
+
wakeContext: undefined,
|
|
265
291
|
wakeRequested: false,
|
|
266
292
|
wakeRequestedThroughEventId: 0,
|
|
267
293
|
wakeTimer: undefined,
|
|
@@ -293,7 +319,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
293
319
|
: state.isOrchestrator
|
|
294
320
|
? {
|
|
295
321
|
kind: "on",
|
|
296
|
-
updateCount: projectAgentOutcomes(state.pendingEvents).outcomes.length,
|
|
322
|
+
updateCount: projectAgentOutcomes(state.pendingEvents, wakeFilter).outcomes.length,
|
|
297
323
|
}
|
|
298
324
|
: { kind: "off" };
|
|
299
325
|
ctx.ui.setStatus?.("herdsman", formatHerdsmanFooterStatus(footerState));
|
|
@@ -304,12 +330,18 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
304
330
|
if (state.wakeTimer) clearTimeout(state.wakeTimer);
|
|
305
331
|
state.wakeTimer = undefined;
|
|
306
332
|
state.wakeDeferredUntilSettled = false;
|
|
333
|
+
state.wakeDeferredSince = undefined;
|
|
334
|
+
state.wakeForcedRelease = false;
|
|
307
335
|
};
|
|
308
336
|
|
|
309
337
|
const cancelWake = () => {
|
|
310
338
|
cancelWakeTimer();
|
|
311
339
|
state.wakeRequested = false;
|
|
312
340
|
state.wakeRequestedThroughEventId = 0;
|
|
341
|
+
// A wake queued for a busy orchestrator belongs to the role/scope that
|
|
342
|
+
// queued it: dropping it here keeps a stale event body out of the context
|
|
343
|
+
// of whatever session takes over next.
|
|
344
|
+
state.wakeContext = undefined;
|
|
313
345
|
};
|
|
314
346
|
|
|
315
347
|
const clearAgentContext = () => {
|
|
@@ -318,6 +350,41 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
318
350
|
state.runActive = false;
|
|
319
351
|
};
|
|
320
352
|
|
|
353
|
+
const unackedDeliveredAscending = (): AgentEventWireRecord[] =>
|
|
354
|
+
[...state.unackedDelivered.values()].sort((left, right) => left.id - right.id);
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Merges freshly injected events into the delivery queue.
|
|
358
|
+
*
|
|
359
|
+
* Invariants (Phase 1 completeness fix):
|
|
360
|
+
* - a new batch is merged into the queue, never substituted for it, so an
|
|
361
|
+
* unconfirmed batch keeps riding along with the next delivery instead of
|
|
362
|
+
* being stranded;
|
|
363
|
+
* - the result is id-ascending and id-deduped, so the same id is never
|
|
364
|
+
* delivered (or acknowledged) twice;
|
|
365
|
+
* - entries are only ever removed by `dropUnackedDelivered` (acknowledged or
|
|
366
|
+
* dead-lettered) or by a role/scope reset, so releasing the deferred wake
|
|
367
|
+
* cannot drop an unconfirmed event.
|
|
368
|
+
*/
|
|
369
|
+
const mergeUnackedDelivered = (
|
|
370
|
+
incoming: readonly AgentEventWireRecord[],
|
|
371
|
+
): AgentEventWireRecord[] => {
|
|
372
|
+
for (const event of incoming) {
|
|
373
|
+
if (!state.unackedDelivered.has(event.id)) state.unackedDelivered.set(event.id, event);
|
|
374
|
+
}
|
|
375
|
+
return unackedDeliveredAscending();
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Drops one event from the delivery queue. The only callers are the two
|
|
380
|
+
* acknowledgement outcomes (accepted, or terminally refused by the daemon)
|
|
381
|
+
* and the role/scope reset, which clears the whole stream together with
|
|
382
|
+
* `presentedEventIds`.
|
|
383
|
+
*/
|
|
384
|
+
const dropUnackedDelivered = (eventId: number): void => {
|
|
385
|
+
state.unackedDelivered.delete(eventId);
|
|
386
|
+
};
|
|
387
|
+
|
|
321
388
|
const pruneAcknowledgedEvents = (ackedEventId: number | undefined) => {
|
|
322
389
|
if (ackedEventId === undefined) return;
|
|
323
390
|
// Acknowledged events leave the pending projection but intentionally stay
|
|
@@ -329,6 +396,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
329
396
|
// set is bounded by the number of events presented per session and is
|
|
330
397
|
// cleared on role loss, scope change, and shutdown.
|
|
331
398
|
state.pendingEvents = state.pendingEvents.filter((event) => event.id > ackedEventId);
|
|
399
|
+
// The delivery queue follows the same watermark: an event covered by the
|
|
400
|
+
// advanced acknowledgement cursor is confirmed even when its own ack was
|
|
401
|
+
// superseded (the daemon acknowledges by watermark), so it leaves the
|
|
402
|
+
// queue and is never re-acknowledged.
|
|
403
|
+
for (const eventId of [...state.unackedDelivered.keys()]) {
|
|
404
|
+
if (eventId <= ackedEventId) state.unackedDelivered.delete(eventId);
|
|
405
|
+
}
|
|
332
406
|
};
|
|
333
407
|
|
|
334
408
|
const isWakeableEvent = (event: AgentEventWireRecord | undefined) =>
|
|
@@ -338,34 +412,290 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
338
412
|
state.latestContext = isLocalOwner(response) ? response.context ?? undefined : undefined;
|
|
339
413
|
};
|
|
340
414
|
|
|
415
|
+
const acknowledgeEventIds = async (
|
|
416
|
+
events: readonly AgentEventWireRecord[],
|
|
417
|
+
options: { notify: boolean },
|
|
418
|
+
ctx: PiContext,
|
|
419
|
+
): Promise<void> => {
|
|
420
|
+
for (const event of [...events].sort((left, right) => left.id - right.id)) {
|
|
421
|
+
try {
|
|
422
|
+
// A missing client (disconnect) must never be treated as a successful
|
|
423
|
+
// acknowledgement: route it through the same failure path as a
|
|
424
|
+
// transient RPC error so the id keeps its backoff and stays pending.
|
|
425
|
+
if (!state.client) {
|
|
426
|
+
throw new Error("Herdsman Pi is not connected; cannot acknowledge notifications");
|
|
427
|
+
}
|
|
428
|
+
const ackResponse = (await state.client.request("agent.notifications.ack", {
|
|
429
|
+
eventId: event.id,
|
|
430
|
+
})) as { ackedEventId?: number; state?: { ackedEventId?: number } } | undefined;
|
|
431
|
+
pruneAcknowledgedEvents(ackResponse?.ackedEventId ?? ackResponse?.state?.ackedEventId);
|
|
432
|
+
state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
|
|
433
|
+
// The event is confirmed: it leaves the delivery queue for good, which
|
|
434
|
+
// is what keeps the acknowledgement watermark monotonic (ids are acked
|
|
435
|
+
// in ascending order and never re-issued).
|
|
436
|
+
dropUnackedDelivered(event.id);
|
|
437
|
+
// The id intentionally stays in presentedEventIds: the event was
|
|
438
|
+
// already presented this session and must not be injected again even
|
|
439
|
+
// if the daemon replays it (for example after a reconnect
|
|
440
|
+
// redelivery). The set is cleared only on role loss, scope change,
|
|
441
|
+
// or shutdown.
|
|
442
|
+
state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
|
|
443
|
+
setHerdsmanUi(ctx);
|
|
444
|
+
} catch (error) {
|
|
445
|
+
const failureCode = ackFailureCode(error);
|
|
446
|
+
const classification = classifyAckFailure(error);
|
|
447
|
+
// The attempt counter is read from the live projection, not from the
|
|
448
|
+
// delivery-queue snapshot the caller iterated over: a stale base would
|
|
449
|
+
// pin the counter at 1 for good and make the MAX_ACK_ATTEMPTS
|
|
450
|
+
// dead-letter branch unreachable for every event retried here.
|
|
451
|
+
const attempts =
|
|
452
|
+
(state.pendingEvents.find((pending) => pending.id === event.id)?.attempts ??
|
|
453
|
+
event.attempts ??
|
|
454
|
+
0) + 1;
|
|
455
|
+
const attemptedAt = Date.now();
|
|
456
|
+
const updatedEvent = {
|
|
457
|
+
...event,
|
|
458
|
+
attempts,
|
|
459
|
+
lastAttemptAt: attemptedAt,
|
|
460
|
+
lastFailureCode: failureCode,
|
|
461
|
+
};
|
|
462
|
+
state.pendingEvents = state.pendingEvents.map((pending) =>
|
|
463
|
+
pending.id === event.id ? updatedEvent : pending,
|
|
464
|
+
);
|
|
465
|
+
// Keep the delivery-queue entry in step with the live accounting: the
|
|
466
|
+
// same event may be retried later (another settlement, or a redelivery
|
|
467
|
+
// that re-attaches it to a new batch), and that attempt must resume
|
|
468
|
+
// from this counter instead of restarting at 1.
|
|
469
|
+
if (state.unackedDelivered.has(event.id)) {
|
|
470
|
+
state.unackedDelivered.set(event.id, updatedEvent);
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
if (classification === "terminal") {
|
|
474
|
+
state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
|
|
475
|
+
// A terminally refused event is dead-lettered by the daemon
|
|
476
|
+
// (failedWakeThroughEventId is the dead-letter barrier), so it can
|
|
477
|
+
// never be confirmed later; it leaves the delivery queue as well.
|
|
478
|
+
dropUnackedDelivered(event.id);
|
|
479
|
+
state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
|
|
480
|
+
if (/Only the current orchestrator can acknowledge notifications/i.test(failureCode)) {
|
|
481
|
+
state.isOrchestrator = false;
|
|
482
|
+
logHerdsmanPi(
|
|
483
|
+
"warn",
|
|
484
|
+
`[herdsman-pi] lost orchestrator ownership while acknowledging event ${event.id}`,
|
|
485
|
+
);
|
|
486
|
+
} else {
|
|
487
|
+
logHerdsmanPi(
|
|
488
|
+
"warn",
|
|
489
|
+
`[herdsman-pi] terminal acknowledgement failure eventId=${event.id} attempts=${attempts} code=${failureCode}`,
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
setHerdsmanUi(ctx);
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
if (attempts >= MAX_ACK_ATTEMPTS) {
|
|
497
|
+
state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
|
|
498
|
+
// Dead-lettered by this client: the id is behind the dead-letter
|
|
499
|
+
// barrier from now on, so it can never be confirmed later and leaves
|
|
500
|
+
// the delivery queue with the pending projection.
|
|
501
|
+
dropUnackedDelivered(event.id);
|
|
502
|
+
state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
|
|
503
|
+
logHerdsmanPi(
|
|
504
|
+
"warn",
|
|
505
|
+
`[herdsman-pi] acknowledgement moved to dead-letter eventId=${event.id} attempts=${attempts} code=${failureCode}`,
|
|
506
|
+
);
|
|
507
|
+
setHerdsmanUi(ctx);
|
|
508
|
+
continue;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
if (classification === "resync") {
|
|
512
|
+
const resyncEvent = {
|
|
513
|
+
...updatedEvent,
|
|
514
|
+
nextAttemptAt: attemptedAt + ackBackoffMs(attempts),
|
|
515
|
+
};
|
|
516
|
+
state.pendingEvents = state.pendingEvents.map((pending) =>
|
|
517
|
+
pending.id === event.id ? resyncEvent : pending,
|
|
518
|
+
);
|
|
519
|
+
try {
|
|
520
|
+
const response = (await state.client?.request(
|
|
521
|
+
"agent.orchestrator.get",
|
|
522
|
+
{},
|
|
523
|
+
)) as ConnectionStateResponse | undefined;
|
|
524
|
+
// Refresh pending data without applying the full connection response: that
|
|
525
|
+
// helper schedules a new wake, which would make this failed batch race
|
|
526
|
+
// with the current settlement and can replay an earlier event. The
|
|
527
|
+
// failed event remains pending and the next wake is scheduled by
|
|
528
|
+
// finishBatch(), so this round performs no additional acknowledgements.
|
|
529
|
+
if (response) addPendingEvents(response.events ?? [], ctx);
|
|
530
|
+
pruneAcknowledgedEvents(response?.state?.ackedEventId ?? response?.ackedEventId);
|
|
531
|
+
setHerdsmanUi(ctx);
|
|
532
|
+
} catch (resyncError) {
|
|
533
|
+
logHerdsmanPi(
|
|
534
|
+
"warn",
|
|
535
|
+
`[herdsman-pi] acknowledgement resync failed eventId=${event.id} attempts=${attempts} code=${ackFailureCode(resyncError)}`,
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
// The event stays pending with a backoff so the ack cursor can
|
|
539
|
+
// sweep it later, but it was already presented this session and is
|
|
540
|
+
// not re-presented: presentedEventIds keeps the guard until scope
|
|
541
|
+
// reset. Continue so one failed event does not block the rest of
|
|
542
|
+
// the batch.
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
state.pendingEvents = state.pendingEvents.map((pending) =>
|
|
547
|
+
pending.id === event.id
|
|
548
|
+
? { ...pending, nextAttemptAt: attemptedAt + ackBackoffMs(attempts) }
|
|
549
|
+
: pending,
|
|
550
|
+
);
|
|
551
|
+
if (options.notify) {
|
|
552
|
+
ctx.ui.notify?.(
|
|
553
|
+
"Herdsman couldn’t acknowledge agent updates · updates remain pending",
|
|
554
|
+
"warning",
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
setHerdsmanUi(ctx);
|
|
558
|
+
// The event stays pending with a backoff so the ack cursor can sweep
|
|
559
|
+
// it later, but it was already presented this session and is not
|
|
560
|
+
// re-presented: presentedEventIds keeps the guard until scope reset.
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
};
|
|
565
|
+
|
|
566
|
+
// Upstream model errors wake nobody, but they still have to leave the
|
|
567
|
+
// daemon's pending queue or it never converges. The silent path therefore
|
|
568
|
+
// acknowledges them without sendMessage, without notify, and without
|
|
569
|
+
// touching presentedEventIds (reserved for genuinely presented outcomes).
|
|
570
|
+
const scheduleSilentUpstreamErrorAck = (
|
|
571
|
+
ctx: PiContext,
|
|
572
|
+
events: readonly AgentEventWireRecord[],
|
|
573
|
+
) => {
|
|
574
|
+
if (state.wakeTimer || state.wakeRequested) return;
|
|
575
|
+
const scope = state.currentScope;
|
|
576
|
+
if (!state.isOrchestrator || !scope) return;
|
|
577
|
+
const generation = wakeGeneration;
|
|
578
|
+
const ownerHerdrSessionName = scope.herdrSessionName;
|
|
579
|
+
const ownerTerminalId = scope.terminalId;
|
|
580
|
+
const ownerWorkspaceId = scope.workspaceId;
|
|
581
|
+
state.wakeTimer = setTimeout(() => {
|
|
582
|
+
state.wakeTimer = undefined;
|
|
583
|
+
void (async () => {
|
|
584
|
+
if (
|
|
585
|
+
generation !== wakeGeneration ||
|
|
586
|
+
!state.isOrchestrator ||
|
|
587
|
+
state.currentScope?.herdrSessionName !== ownerHerdrSessionName ||
|
|
588
|
+
state.currentScope?.terminalId !== ownerTerminalId ||
|
|
589
|
+
state.currentScope?.workspaceId !== ownerWorkspaceId
|
|
590
|
+
) {
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
// A delivered batch or an in-flight ack owns the cursor; its
|
|
594
|
+
// settlement schedules the next sweep instead of racing this one.
|
|
595
|
+
if (state.deliveredBatch || state.ackInFlight) return;
|
|
596
|
+
if (!state.client || !state.connected) return;
|
|
597
|
+
state.ackInFlight = true;
|
|
598
|
+
setHerdsmanUi(ctx);
|
|
599
|
+
try {
|
|
600
|
+
await acknowledgeEventIds(events, { notify: false }, ctx);
|
|
601
|
+
} finally {
|
|
602
|
+
state.ackInFlight = false;
|
|
603
|
+
setHerdsmanUi(ctx);
|
|
604
|
+
}
|
|
605
|
+
scheduleWake(ctx);
|
|
606
|
+
})();
|
|
607
|
+
}, WAKE_SETTLE_MS);
|
|
608
|
+
};
|
|
609
|
+
|
|
610
|
+
/**
|
|
611
|
+
* Arms the bounded deferral for a wake that cannot be injected right now.
|
|
612
|
+
*
|
|
613
|
+
* The retry spins every `WAKE_BUSY_SPIN_MS` while the orchestrator is busy
|
|
614
|
+
* and flips `wakeForcedRelease` once `WAKE_DEFERRED_TIMEOUT_MS` has elapsed,
|
|
615
|
+
* so the next pass injects the batch from the current state (as a queued,
|
|
616
|
+
* non-triggering follow-up) instead of waiting for an idle signal or a
|
|
617
|
+
* settlement that may never arrive.
|
|
618
|
+
*/
|
|
619
|
+
const scheduleDeferredWake = (ctx: PiContext) => {
|
|
620
|
+
state.wakeDeferredUntilSettled = true;
|
|
621
|
+
const since = state.wakeDeferredSince ?? Date.now();
|
|
622
|
+
state.wakeDeferredSince = since;
|
|
623
|
+
const remaining = WAKE_DEFERRED_TIMEOUT_MS - (Date.now() - since);
|
|
624
|
+
if (remaining <= 0) state.wakeForcedRelease = true;
|
|
625
|
+
if (state.wakeTimer) return;
|
|
626
|
+
state.wakeTimer = setTimeout(() => {
|
|
627
|
+
state.wakeTimer = undefined;
|
|
628
|
+
if (state.wakeForcedRelease && state.deliveredBatch && ctx.isIdle?.() !== false) {
|
|
629
|
+
// The hard deadline only releases the delivery — it never discards an
|
|
630
|
+
// unconfirmed event. The batch record is dropped so a wake turn that
|
|
631
|
+
// is no longer running cannot gate later wakes, but its events stay in
|
|
632
|
+
// the delivery queue (`unackedDelivered`) and are re-attached to the
|
|
633
|
+
// batch injected right below, which acknowledges them once it settles.
|
|
634
|
+
state.deliveredBatch = undefined;
|
|
635
|
+
}
|
|
636
|
+
scheduleWake(ctx);
|
|
637
|
+
}, Math.max(0, Math.min(WAKE_BUSY_SPIN_MS, remaining)));
|
|
638
|
+
};
|
|
639
|
+
|
|
341
640
|
const scheduleWake = (ctx: PiContext | undefined) => {
|
|
342
641
|
if (!ctx || !state.isOrchestrator || !state.currentScope || !pi.sendMessage) return;
|
|
343
642
|
if (state.wakeTimer || state.wakeRequested) return;
|
|
344
|
-
const
|
|
643
|
+
const projection = projectAgentOutcomes(state.pendingEvents, wakeFilter);
|
|
644
|
+
const outcomes = projection.outcomes.filter(
|
|
345
645
|
(outcome) =>
|
|
346
646
|
outcome.eventId > state.failedWakeThroughEventId &&
|
|
347
647
|
!state.presentedEventIds.has(outcome.eventId),
|
|
348
648
|
);
|
|
649
|
+
const suppressedEvents = projection.suppressedUpstreamErrorEventIds
|
|
650
|
+
.filter(
|
|
651
|
+
(eventId) =>
|
|
652
|
+
eventId > state.failedWakeThroughEventId && !state.presentedEventIds.has(eventId),
|
|
653
|
+
)
|
|
654
|
+
.map((eventId) => state.pendingEvents.find((pending) => pending.id === eventId))
|
|
655
|
+
.filter((event): event is AgentEventWireRecord => event !== undefined)
|
|
656
|
+
.sort((left, right) => left.id - right.id);
|
|
349
657
|
|
|
350
658
|
const wakeable = outcomes.filter((outcome) =>
|
|
351
659
|
isWakeableEvent(state.pendingEvents.find((pending) => pending.id === outcome.eventId)),
|
|
352
660
|
);
|
|
353
661
|
if (wakeable.length === 0) {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
662
|
+
// A suppressed upstream error that is now due must be silently
|
|
663
|
+
// acknowledged before any backoff timer is planted: planting the timer
|
|
664
|
+
// first would make scheduleSilentUpstreamErrorAck's entry guard
|
|
665
|
+
// (`if (state.wakeTimer || state.wakeRequested) return`) bounce the ack
|
|
666
|
+
// off its own timer and, once the backoff window has elapsed, the
|
|
667
|
+
// 0ms-timer + blocked-ack loop never converges the queue.
|
|
668
|
+
const dueSuppressed = suppressedEvents.filter(isWakeableEvent);
|
|
669
|
+
if (dueSuppressed.length > 0) {
|
|
670
|
+
scheduleSilentUpstreamErrorAck(ctx, dueSuppressed);
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
// No suppressed event is due, so plant a backoff timer for the next
|
|
674
|
+
// future `nextAttemptAt`. Expired timestamps are excluded (strictly
|
|
675
|
+
// greater than now) so an already-past window does not produce a
|
|
676
|
+
// zero-delay spin.
|
|
677
|
+
const nextAttemptAt = [
|
|
678
|
+
...outcomes.map((outcome) => outcome.eventId),
|
|
679
|
+
...suppressedEvents.map((event) => event.id),
|
|
680
|
+
]
|
|
681
|
+
.map((eventId) => state.pendingEvents.find((event) => event.id === eventId)?.nextAttemptAt)
|
|
682
|
+
.filter((value): value is number => value !== undefined && value > Date.now())
|
|
357
683
|
.sort((left, right) => left - right)[0];
|
|
358
684
|
if (nextAttemptAt !== undefined) {
|
|
359
685
|
state.wakeTimer = setTimeout(() => {
|
|
360
686
|
state.wakeTimer = undefined;
|
|
361
687
|
scheduleWake(ctx);
|
|
362
|
-
},
|
|
688
|
+
}, nextAttemptAt - Date.now());
|
|
363
689
|
}
|
|
364
690
|
return;
|
|
365
691
|
}
|
|
366
692
|
|
|
367
|
-
|
|
368
|
-
|
|
693
|
+
// An in-flight batch owns the ack cursor and a busy orchestrator must not
|
|
694
|
+
// be interrupted, so neither is woken immediately — but both are deferred
|
|
695
|
+
// on a bounded spin (never parked until an event that may never come).
|
|
696
|
+
const inFlight = state.deliveredBatch !== undefined || state.ackInFlight;
|
|
697
|
+
if (!state.wakeForcedRelease && (inFlight || ctx.isIdle?.() === false)) {
|
|
698
|
+
scheduleDeferredWake(ctx);
|
|
369
699
|
return;
|
|
370
700
|
}
|
|
371
701
|
const generation = wakeGeneration;
|
|
@@ -384,9 +714,9 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
384
714
|
state.wakeTimer = undefined;
|
|
385
715
|
return;
|
|
386
716
|
}
|
|
387
|
-
if (ctx.isIdle?.() === false) {
|
|
717
|
+
if (ctx.isIdle?.() === false && !state.wakeForcedRelease) {
|
|
388
718
|
state.wakeTimer = undefined;
|
|
389
|
-
|
|
719
|
+
scheduleDeferredWake(ctx);
|
|
390
720
|
return;
|
|
391
721
|
}
|
|
392
722
|
|
|
@@ -421,14 +751,16 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
421
751
|
state.wakeTimer = undefined;
|
|
422
752
|
return;
|
|
423
753
|
}
|
|
424
|
-
if (ctx.isIdle?.() === false) {
|
|
754
|
+
if (ctx.isIdle?.() === false && !state.wakeForcedRelease) {
|
|
425
755
|
state.wakeTimer = undefined;
|
|
426
|
-
|
|
756
|
+
scheduleDeferredWake(ctx);
|
|
427
757
|
return;
|
|
428
758
|
}
|
|
429
759
|
|
|
430
760
|
const batchEvents = [...state.pendingEvents].sort((left, right) => left.id - right.id);
|
|
431
|
-
const
|
|
761
|
+
const batchProjection = projectAgentOutcomes(batchEvents, wakeFilter);
|
|
762
|
+
const batchSuppressedIds = new Set(batchProjection.suppressedUpstreamErrorEventIds);
|
|
763
|
+
const batchOutcomes = batchProjection.outcomes.filter(
|
|
432
764
|
(outcome) =>
|
|
433
765
|
outcome.eventId > state.failedWakeThroughEventId &&
|
|
434
766
|
!state.presentedEventIds.has(outcome.eventId) &&
|
|
@@ -439,33 +771,73 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
439
771
|
return;
|
|
440
772
|
}
|
|
441
773
|
const current = batchOutcomes;
|
|
442
|
-
const
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
batchOutcomes.some((outcome) => outcome.eventId === event.id),
|
|
447
|
-
),
|
|
448
|
-
hasSubstantiveWork: false,
|
|
449
|
-
invalidated: false,
|
|
450
|
-
ownerTerminalId,
|
|
451
|
-
herdsmanTriggered: true,
|
|
452
|
-
};
|
|
774
|
+
const incomingEvents = batchEvents.filter((event) =>
|
|
775
|
+
batchOutcomes.some((outcome) => outcome.eventId === event.id),
|
|
776
|
+
);
|
|
777
|
+
const previousBatch = state.deliveredBatch;
|
|
453
778
|
state.wakeTimer = undefined;
|
|
454
779
|
state.wakeRequested = true;
|
|
455
780
|
state.wakeRequestedThroughEventId = current.at(-1)?.eventId ?? 0;
|
|
781
|
+
// Dual-track injection: an idle orchestrator gets a triggered
|
|
782
|
+
// follow-up turn (immediate delivery), while a busy one is not
|
|
783
|
+
// interrupted — the same content is queued as a non-triggering
|
|
784
|
+
// follow-up and additionally exposed through the `context` hook so the
|
|
785
|
+
// running turn can already see it.
|
|
786
|
+
const orchestratorBusy = ctx.isIdle?.() === false;
|
|
787
|
+
const wakeContent = formatAgentOutcomeUpdates(batchOutcomes);
|
|
456
788
|
try {
|
|
457
789
|
pi.sendMessage?.(
|
|
458
790
|
{
|
|
459
|
-
content:
|
|
791
|
+
content: wakeContent,
|
|
460
792
|
customType: "herdsman-wake-context",
|
|
461
|
-
|
|
793
|
+
// Suppressed upstream errors are dropped from the injected
|
|
794
|
+
// context, but every other pending id stays listed so the
|
|
795
|
+
// evidence trail for the decision still names what was pending.
|
|
796
|
+
details: {
|
|
797
|
+
eventIds: batchEvents
|
|
798
|
+
.filter((event) => !batchSuppressedIds.has(event.id))
|
|
799
|
+
.map((event) => event.id),
|
|
800
|
+
},
|
|
462
801
|
display: false,
|
|
463
802
|
},
|
|
464
|
-
|
|
803
|
+
orchestratorBusy
|
|
804
|
+
? { deliverAs: "followUp", triggerTurn: false }
|
|
805
|
+
: { deliverAs: "followUp", triggerTurn: true },
|
|
465
806
|
);
|
|
807
|
+
// A queued (non-triggering) delivery keeps the content available to
|
|
808
|
+
// the current turn through the context hook until it is settled or
|
|
809
|
+
// superseded by the next injection.
|
|
810
|
+
state.wakeContext = orchestratorBusy
|
|
811
|
+
? { content: wakeContent, eventIds: batchOutcomes.map((outcome) => outcome.eventId) }
|
|
812
|
+
: undefined;
|
|
813
|
+
state.wakeForcedRelease = false;
|
|
814
|
+
state.wakeDeferredSince = undefined;
|
|
466
815
|
// Only expose the batch after the hidden context was accepted by pi. This
|
|
467
816
|
// keeps an injection failure eligible for daemon redelivery.
|
|
468
|
-
|
|
817
|
+
//
|
|
818
|
+
// The batch is the delivery queue plus this injection: previously
|
|
819
|
+
// unconfirmed events are merged (never replaced) so that a forced
|
|
820
|
+
// release always leaves both the old and the new events deliverable
|
|
821
|
+
// and acknowledgeable in id order. The turn-consumption flags of a
|
|
822
|
+
// still-running previous batch are carried over, because they
|
|
823
|
+
// describe whether the content already reached the orchestrator.
|
|
824
|
+
//
|
|
825
|
+
// `hasSubstantiveWork` is the sole gate that decides whether an
|
|
826
|
+
// ownership/scope change may abort the in-flight turn (see loseRole
|
|
827
|
+
// and resetForScopeChange), and aborting is only ever allowed for a
|
|
828
|
+
// *pure* Herdsman wake turn. A busy orchestrator gets the batch as a
|
|
829
|
+
// non-triggering queued follow-up, which rides the user's own turn:
|
|
830
|
+
// that turn is not a Herdsman wake turn, so it must never be aborted
|
|
831
|
+
// on our behalf and the flag is set here.
|
|
832
|
+
state.deliveredBatch = {
|
|
833
|
+
abortedByUser: previousBatch?.abortedByUser ?? false,
|
|
834
|
+
assistantFinalSucceeded: previousBatch?.assistantFinalSucceeded ?? false,
|
|
835
|
+
events: mergeUnackedDelivered(incomingEvents),
|
|
836
|
+
hasSubstantiveWork: orchestratorBusy || (previousBatch?.hasSubstantiveWork ?? false),
|
|
837
|
+
invalidated: false,
|
|
838
|
+
ownerTerminalId,
|
|
839
|
+
herdsmanTriggered: true,
|
|
840
|
+
};
|
|
469
841
|
state.wakeRequested = false;
|
|
470
842
|
state.wakeRequestedThroughEventId = 0;
|
|
471
843
|
// Record the presentation so a reclaim redelivery of the same id is
|
|
@@ -522,7 +894,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
522
894
|
// A transient disconnect (reconnect) keeps the presentation guard so an
|
|
523
895
|
// event already presented in this scope session is not presented again;
|
|
524
896
|
// only a genuine role/scope loss or shutdown resets it.
|
|
525
|
-
if (!options.preservePresented)
|
|
897
|
+
if (!options.preservePresented) {
|
|
898
|
+
state.presentedEventIds.clear();
|
|
899
|
+
// The delivery queue dies with the presentation guard: once the guard is
|
|
900
|
+
// gone the daemon's pending events can be presented (and acknowledged)
|
|
901
|
+
// again, so keeping the old queue would only risk a stale id.
|
|
902
|
+
state.unackedDelivered.clear();
|
|
903
|
+
}
|
|
526
904
|
state.reconnectingFromOn = false;
|
|
527
905
|
setHerdsmanUi(ctx);
|
|
528
906
|
};
|
|
@@ -553,6 +931,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
553
931
|
state.failedWakeThroughEventId = 0;
|
|
554
932
|
state.pendingEvents = [];
|
|
555
933
|
state.presentedEventIds.clear();
|
|
934
|
+
state.unackedDelivered.clear();
|
|
556
935
|
setHerdsmanUi(ctx);
|
|
557
936
|
};
|
|
558
937
|
|
|
@@ -670,6 +1049,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
670
1049
|
// already acked events are covered by the server cursor
|
|
671
1050
|
// (pruneAcknowledgedEvents below).
|
|
672
1051
|
state.deliveredBatch = undefined;
|
|
1052
|
+
state.wakeContext = undefined;
|
|
673
1053
|
}
|
|
674
1054
|
// Otherwise the batch's wake turn is still in flight: keep it so the
|
|
675
1055
|
// settlement acknowledges it and the events are not re-presented.
|
|
@@ -911,25 +1291,16 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
911
1291
|
loseRole(activeContext);
|
|
912
1292
|
state.deliveredBatch = undefined;
|
|
913
1293
|
state.presentedEventIds.clear();
|
|
1294
|
+
state.unackedDelivered.clear();
|
|
914
1295
|
state.client?.close();
|
|
915
1296
|
state.client = undefined;
|
|
916
1297
|
activeContext = undefined;
|
|
917
1298
|
});
|
|
918
1299
|
|
|
919
1300
|
const assistantMessageText = (message: Record<string, unknown>): string => {
|
|
920
|
-
const
|
|
921
|
-
if (
|
|
922
|
-
|
|
923
|
-
const parts: string[] = [];
|
|
924
|
-
for (const block of content) {
|
|
925
|
-
if (typeof block === "string") {
|
|
926
|
-
if (block.length > 0) parts.push(block);
|
|
927
|
-
continue;
|
|
928
|
-
}
|
|
929
|
-
const value = record(block);
|
|
930
|
-
if (typeof value.text === "string" && value.text.length > 0) parts.push(value.text);
|
|
931
|
-
}
|
|
932
|
-
return parts.join("\n");
|
|
1301
|
+
const text = textFromContent(message.content);
|
|
1302
|
+
if (text === null) return "";
|
|
1303
|
+
return sanitizeText(text).text;
|
|
933
1304
|
};
|
|
934
1305
|
|
|
935
1306
|
// Turn completion signal: after Pi's own final assistant message has been
|
|
@@ -945,13 +1316,15 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
945
1316
|
const completion = (async () => {
|
|
946
1317
|
const check = await confirmSessionWrite({ expectedText, path: sessionPath });
|
|
947
1318
|
try {
|
|
948
|
-
|
|
1319
|
+
const params: Record<string, unknown> = {
|
|
949
1320
|
confirmed: check.confirmed,
|
|
950
1321
|
herdrSessionName: scope.herdrSessionName,
|
|
951
1322
|
paneId: scope.paneId,
|
|
952
1323
|
terminalId: scope.terminalId,
|
|
953
1324
|
workspaceId: scope.workspaceId,
|
|
954
|
-
}
|
|
1325
|
+
};
|
|
1326
|
+
if (expectedText) params.expectedText = expectedText;
|
|
1327
|
+
await client.request("agent.turn.completed", params);
|
|
955
1328
|
} catch (error) {
|
|
956
1329
|
logHerdsmanPi(
|
|
957
1330
|
"warn",
|
|
@@ -1002,23 +1375,43 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1002
1375
|
|
|
1003
1376
|
pi.on("context", (event: { messages: PiAgentMessage[] }) => {
|
|
1004
1377
|
const messages = event.messages.filter((message) => !isNormalHerdsmanContext(message));
|
|
1378
|
+
const additions: PiAgentMessage[] = [];
|
|
1005
1379
|
const snapshot = state.pinnedContext;
|
|
1006
|
-
if (
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1380
|
+
if (snapshot && snapshot.agents.length > 0) {
|
|
1381
|
+
additions.push({
|
|
1382
|
+
content: formatHiddenAgentContext({
|
|
1383
|
+
agents: snapshot.agents,
|
|
1384
|
+
workspaceId: snapshot.workspaceId,
|
|
1385
|
+
}),
|
|
1386
|
+
customType: "herdsman-agent-context",
|
|
1387
|
+
display: false,
|
|
1388
|
+
role: "custom",
|
|
1389
|
+
timestamp: Date.now(),
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
// A wake queued for a busy orchestrator is not allowed to interrupt the
|
|
1393
|
+
// running tool chain, so its content is additionally pinned to the current
|
|
1394
|
+
// context: the orchestrator sees the child-agent outcome in this turn
|
|
1395
|
+
// without a triggered follow-up. The entry is dropped again by
|
|
1396
|
+
// isNormalHerdsmanContext, so at most one copy is present per call.
|
|
1397
|
+
//
|
|
1398
|
+
// `eventIds` mirrors what this turn actually presents (the freshly injected
|
|
1399
|
+
// outcomes): events carried over in the delivery queue were already shown
|
|
1400
|
+
// to the orchestrator in the turn that presented them, so they are not
|
|
1401
|
+
// re-listed here. The queued follow-up message itself keeps the wider
|
|
1402
|
+
// `details.eventIds` set (everything still unconfirmed).
|
|
1403
|
+
const queuedWake = state.wakeContext;
|
|
1404
|
+
if (queuedWake) {
|
|
1405
|
+
additions.push({
|
|
1406
|
+
content: queuedWake.content,
|
|
1407
|
+
customType: "herdsman-wake-queued",
|
|
1408
|
+
details: { eventIds: queuedWake.eventIds },
|
|
1409
|
+
display: false,
|
|
1410
|
+
role: "custom",
|
|
1411
|
+
timestamp: Date.now(),
|
|
1412
|
+
});
|
|
1413
|
+
}
|
|
1414
|
+
return additions.length === 0 ? { messages } : { messages: [...messages, ...additions] };
|
|
1022
1415
|
});
|
|
1023
1416
|
|
|
1024
1417
|
pi.on("agent_settled", async (_event: unknown, ctx: PiContext) => {
|
|
@@ -1043,10 +1436,41 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1043
1436
|
const finishBatch = () => {
|
|
1044
1437
|
state.ackInFlight = false;
|
|
1045
1438
|
state.wakeDeferredUntilSettled = false;
|
|
1439
|
+
state.wakeDeferredSince = undefined;
|
|
1440
|
+
state.wakeForcedRelease = false;
|
|
1441
|
+
state.wakeContext = undefined;
|
|
1046
1442
|
setHerdsmanUi(ctx);
|
|
1047
1443
|
scheduleWake(ctx);
|
|
1048
1444
|
};
|
|
1049
1445
|
|
|
1446
|
+
// The delivery queue — not the injection snapshot — is what gets
|
|
1447
|
+
// acknowledged: it holds every event handed to Pi that is still
|
|
1448
|
+
// unconfirmed (a release merges batches instead of replacing them), is
|
|
1449
|
+
// id-ascending, and an id leaves it only when the daemon accepts it.
|
|
1450
|
+
//
|
|
1451
|
+
// An event whose own acknowledgement already failed is left out here: it is
|
|
1452
|
+
// never retried by this path (the daemon's cursor advance sweeps it, and a
|
|
1453
|
+
// retry would reset its attempt/backoff accounting), but it stays in the
|
|
1454
|
+
// queue until that cursor or a scope reset confirms it. A failed or
|
|
1455
|
+
// dead-lettered acknowledgement thus never depends on the timing of the
|
|
1456
|
+
// release to stay recoverable.
|
|
1457
|
+
//
|
|
1458
|
+
// Restricting to a *live* row with `attempts === 0` has two holes on
|
|
1459
|
+
// purpose. `?? 0` covers an event the live projection no longer holds at
|
|
1460
|
+
// all (the server stopped listing it, or `failedWakeThroughEventId`
|
|
1461
|
+
// filters it out): the projection cannot tell us it already failed, so the
|
|
1462
|
+
// event gets one more attempt — a deliberate self-healing opportunity that
|
|
1463
|
+
// then accumulates on the queue copy's counter and can reach
|
|
1464
|
+
// MAX_ACK_ATTEMPTS instead of restarting at 1 every round.
|
|
1465
|
+
const ackable = unackedDeliveredAscending().filter(
|
|
1466
|
+
(event) =>
|
|
1467
|
+
(state.pendingEvents.find((pending) => pending.id === event.id)?.attempts ?? 0) === 0,
|
|
1468
|
+
);
|
|
1469
|
+
if (ackable.length === 0) {
|
|
1470
|
+
finishBatch();
|
|
1471
|
+
return;
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1050
1474
|
if (
|
|
1051
1475
|
(!batch.assistantFinalSucceeded && !batch.abortedByUser) ||
|
|
1052
1476
|
batch.invalidated ||
|
|
@@ -1059,116 +1483,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1059
1483
|
return;
|
|
1060
1484
|
}
|
|
1061
1485
|
|
|
1062
|
-
|
|
1063
|
-
try {
|
|
1064
|
-
const ackResponse = (await state.client.request("agent.notifications.ack", {
|
|
1065
|
-
eventId: event.id,
|
|
1066
|
-
})) as { ackedEventId?: number; state?: { ackedEventId?: number } };
|
|
1067
|
-
pruneAcknowledgedEvents(ackResponse?.ackedEventId ?? ackResponse?.state?.ackedEventId);
|
|
1068
|
-
state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
|
|
1069
|
-
// The id intentionally stays in presentedEventIds: the event was
|
|
1070
|
-
// already presented this session and must not be injected again even
|
|
1071
|
-
// if the daemon replays it (for example after a reconnect
|
|
1072
|
-
// redelivery). The set is cleared only on role loss, scope change,
|
|
1073
|
-
// or shutdown.
|
|
1074
|
-
state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
|
|
1075
|
-
setHerdsmanUi(ctx);
|
|
1076
|
-
} catch (error) {
|
|
1077
|
-
const failureCode = ackFailureCode(error);
|
|
1078
|
-
const classification = classifyAckFailure(error);
|
|
1079
|
-
const attempts = (event.attempts ?? 0) + 1;
|
|
1080
|
-
const attemptedAt = Date.now();
|
|
1081
|
-
const updatedEvent = {
|
|
1082
|
-
...event,
|
|
1083
|
-
attempts,
|
|
1084
|
-
lastAttemptAt: attemptedAt,
|
|
1085
|
-
lastFailureCode: failureCode,
|
|
1086
|
-
};
|
|
1087
|
-
state.pendingEvents = state.pendingEvents.map((pending) =>
|
|
1088
|
-
pending.id === event.id ? updatedEvent : pending,
|
|
1089
|
-
);
|
|
1090
|
-
|
|
1091
|
-
if (classification === "terminal") {
|
|
1092
|
-
state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
|
|
1093
|
-
state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
|
|
1094
|
-
if (/Only the current orchestrator can acknowledge notifications/i.test(failureCode)) {
|
|
1095
|
-
state.isOrchestrator = false;
|
|
1096
|
-
logHerdsmanPi(
|
|
1097
|
-
"warn",
|
|
1098
|
-
`[herdsman-pi] lost orchestrator ownership while acknowledging event ${event.id}`,
|
|
1099
|
-
);
|
|
1100
|
-
} else {
|
|
1101
|
-
logHerdsmanPi(
|
|
1102
|
-
"warn",
|
|
1103
|
-
`[herdsman-pi] terminal acknowledgement failure eventId=${event.id} attempts=${attempts} code=${failureCode}`,
|
|
1104
|
-
);
|
|
1105
|
-
}
|
|
1106
|
-
setHerdsmanUi(ctx);
|
|
1107
|
-
continue;
|
|
1108
|
-
}
|
|
1109
|
-
|
|
1110
|
-
if (attempts >= MAX_ACK_ATTEMPTS) {
|
|
1111
|
-
state.pendingEvents = state.pendingEvents.filter((pending) => pending.id !== event.id);
|
|
1112
|
-
state.failedWakeThroughEventId = Math.max(state.failedWakeThroughEventId, event.id);
|
|
1113
|
-
logHerdsmanPi(
|
|
1114
|
-
"warn",
|
|
1115
|
-
`[herdsman-pi] acknowledgement moved to dead-letter eventId=${event.id} attempts=${attempts} code=${failureCode}`,
|
|
1116
|
-
);
|
|
1117
|
-
setHerdsmanUi(ctx);
|
|
1118
|
-
continue;
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
if (classification === "resync") {
|
|
1122
|
-
const resyncEvent = {
|
|
1123
|
-
...updatedEvent,
|
|
1124
|
-
nextAttemptAt: attemptedAt + ackBackoffMs(attempts),
|
|
1125
|
-
};
|
|
1126
|
-
state.pendingEvents = state.pendingEvents.map((pending) =>
|
|
1127
|
-
pending.id === event.id ? resyncEvent : pending,
|
|
1128
|
-
);
|
|
1129
|
-
try {
|
|
1130
|
-
const response = (await state.client.request(
|
|
1131
|
-
"agent.orchestrator.get",
|
|
1132
|
-
{},
|
|
1133
|
-
)) as ConnectionStateResponse;
|
|
1134
|
-
// Refresh pending data without applying the full connection response: that
|
|
1135
|
-
// helper schedules a new wake, which would make this failed batch race
|
|
1136
|
-
// with the current settlement and can replay an earlier event. The
|
|
1137
|
-
// failed event remains pending and the next wake is scheduled by
|
|
1138
|
-
// finishBatch(), so this round performs no additional acknowledgements.
|
|
1139
|
-
addPendingEvents(response.events ?? [], ctx);
|
|
1140
|
-
pruneAcknowledgedEvents(response.state?.ackedEventId ?? response.ackedEventId);
|
|
1141
|
-
setHerdsmanUi(ctx);
|
|
1142
|
-
} catch (resyncError) {
|
|
1143
|
-
logHerdsmanPi(
|
|
1144
|
-
"warn",
|
|
1145
|
-
`[herdsman-pi] acknowledgement resync failed eventId=${event.id} attempts=${attempts} code=${ackFailureCode(resyncError)}`,
|
|
1146
|
-
);
|
|
1147
|
-
}
|
|
1148
|
-
// The event stays pending with a backoff so the ack cursor can
|
|
1149
|
-
// sweep it later, but it was already presented this session and is
|
|
1150
|
-
// not re-presented: presentedEventIds keeps the guard until scope
|
|
1151
|
-
// reset. Continue so one failed event does not block the rest of
|
|
1152
|
-
// the batch.
|
|
1153
|
-
continue;
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
state.pendingEvents = state.pendingEvents.map((pending) =>
|
|
1157
|
-
pending.id === event.id
|
|
1158
|
-
? { ...pending, nextAttemptAt: attemptedAt + ackBackoffMs(attempts) }
|
|
1159
|
-
: pending,
|
|
1160
|
-
);
|
|
1161
|
-
ctx.ui.notify?.(
|
|
1162
|
-
"Herdsman couldn’t acknowledge agent updates · updates remain pending",
|
|
1163
|
-
"warning",
|
|
1164
|
-
);
|
|
1165
|
-
setHerdsmanUi(ctx);
|
|
1166
|
-
// The event stays pending with a backoff so the ack cursor can sweep
|
|
1167
|
-
// it later, but it was already presented this session and is not
|
|
1168
|
-
// re-presented: presentedEventIds keeps the guard until scope reset.
|
|
1169
|
-
continue;
|
|
1170
|
-
}
|
|
1171
|
-
}
|
|
1486
|
+
await acknowledgeEventIds(ackable, { notify: true }, ctx);
|
|
1172
1487
|
finishBatch();
|
|
1173
1488
|
});
|
|
1174
1489
|
|
|
@@ -1255,6 +1570,7 @@ export function formatHiddenAgentUpdates(events: AgentEventWireRecord[]): string
|
|
|
1255
1570
|
function isNormalHerdsmanContext(message: PiAgentMessage): boolean {
|
|
1256
1571
|
return (
|
|
1257
1572
|
message.customType === "herdsman-agent-context" ||
|
|
1573
|
+
message.customType === "herdsman-wake-queued" ||
|
|
1258
1574
|
contentIncludesMarker(message.content, "[HERDSMAN AGENT CONTEXT]")
|
|
1259
1575
|
);
|
|
1260
1576
|
}
|