@dorokuma/herdsman-pi 0.14.1 → 0.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.ts +655 -466
package/src/index.ts
CHANGED
|
@@ -7,7 +7,6 @@ export { logHerdsmanPi };
|
|
|
7
7
|
export type { HerdsmanPiLogLevel } from "./logger.js";
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
-
type AgentContextListItem,
|
|
11
10
|
type AgentEventWireRecord,
|
|
12
11
|
type AgentOrchestratorChanged,
|
|
13
12
|
type AgentOrchestratorWireState,
|
|
@@ -24,6 +23,7 @@ import {
|
|
|
24
23
|
import {
|
|
25
24
|
projectAgentOutcomes,
|
|
26
25
|
formatAgentOutcomeUpdates,
|
|
26
|
+
type AgentOutcome,
|
|
27
27
|
WAKE_SETTLE_MS,
|
|
28
28
|
} from "./wake.js";
|
|
29
29
|
import { loadWakeFilterConfig } from "./wake-filter-config.js";
|
|
@@ -105,7 +105,6 @@ type HerdsmanState = {
|
|
|
105
105
|
launchIdentity: LaunchIdentity | undefined;
|
|
106
106
|
latestContext: AgentWorkspaceContextSnapshot | undefined;
|
|
107
107
|
pendingEvents: AgentEventWireRecord[];
|
|
108
|
-
pinnedContext: AgentWorkspaceContextSnapshot | undefined;
|
|
109
108
|
presentedEventIds: Set<number>;
|
|
110
109
|
reconnectingFromOn: boolean;
|
|
111
110
|
registrationInFlight: Promise<void> | undefined;
|
|
@@ -123,30 +122,20 @@ type HerdsmanState = {
|
|
|
123
122
|
*/
|
|
124
123
|
unackedDelivered: Map<number, AgentEventWireRecord>;
|
|
125
124
|
wakeDeferredUntilSettled: boolean;
|
|
126
|
-
/** Wall-clock start of the current bounded wake deferral, if any. */
|
|
127
|
-
wakeDeferredSince: number | undefined;
|
|
128
125
|
/**
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*/
|
|
132
|
-
wakeForcedRelease: boolean;
|
|
133
|
-
/**
|
|
134
|
-
* Event ids handed to Pi as a *queued* (non-triggering) follow-up whose
|
|
135
|
-
* content has not been seen entering the transcript yet.
|
|
126
|
+
* Event ids handed to Pi as a follow-up whose content has not been seen
|
|
127
|
+
* entering the transcript yet.
|
|
136
128
|
*
|
|
137
|
-
* A wake
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
129
|
+
* A wake is only ever injected into an idle orchestrator, and it always
|
|
130
|
+
* triggers its own turn, so the copy Pi holds is a normal turning message
|
|
131
|
+
* rather than a queued follow-up nothing may drain. This set is what keeps such
|
|
132
|
+
* an in-flight delivery out of both the acknowledgement path and the
|
|
133
|
+
* transcript, and it carries three guarantees at once:
|
|
142
134
|
*
|
|
143
|
-
* - never lost: while it is non-empty the settlement
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
* acknowledged (`writeOffStrandedWakeDelivery`), because an update that no
|
|
148
|
-
* turn will ever carry out must not keep a run loop — and the daemon's
|
|
149
|
-
* redelivery of it — alive forever;
|
|
135
|
+
* - never lost: while it is non-empty the next settlement leaves the id
|
|
136
|
+
* pending, so the next `scheduleWake` re-presents it (with the real update
|
|
137
|
+
* content, never with an empty marker run) instead of parking it in a
|
|
138
|
+
* follow-up queue nothing drains;
|
|
150
139
|
* - never acknowledged unseen: these ids are excluded from the acknowledgement
|
|
151
140
|
* path, so the daemon keeps them pending and redelivers them when this
|
|
152
141
|
* session never consumes them (the only way a delivery Pi itself dropped —
|
|
@@ -159,7 +148,10 @@ type HerdsmanState = {
|
|
|
159
148
|
* evidence: the hidden wake message's `message_end`, which then moves them to
|
|
160
149
|
* `wakeConsumptionObserved`) or when the event leaves the delivery queue for
|
|
161
150
|
* good (acknowledged, covered by the acknowledgement watermark,
|
|
162
|
-
* dead-lettered), and with the delivery queue on a role/scope reset.
|
|
151
|
+
* dead-lettered), and with the delivery queue on a role/scope reset. A
|
|
152
|
+
* settlement that saw no evidence dead-letters them: it lifts the injection
|
|
153
|
+
* barrier (via `wakeRetryableEventIds`) but not the watermark one, because the
|
|
154
|
+
* content was never confirmed.
|
|
163
155
|
*/
|
|
164
156
|
wakeAwaitingConsumption: Set<number>;
|
|
165
157
|
/**
|
|
@@ -207,18 +199,45 @@ type HerdsmanState = {
|
|
|
207
199
|
*/
|
|
208
200
|
wakeSkipLogReasons: Map<number, string>;
|
|
209
201
|
/**
|
|
210
|
-
*
|
|
202
|
+
* Ids whose delivery was dead-lettered and which a later wake must be allowed
|
|
203
|
+
* to present again.
|
|
204
|
+
*
|
|
205
|
+
* `wakeAwaitingConsumption` deliberately keeps holding these ids: unconfirmed
|
|
206
|
+
* content must still block the acknowledgement watermark, because the daemon
|
|
207
|
+
* confirms by `where id <= ?` and letting a larger id through would swallow the
|
|
208
|
+
* dead-lettered one for good. What the write-off changes is only the *other*
|
|
209
|
+
* role of that set: an id that is still awaiting consumption is also barred
|
|
210
|
+
* from every later injection, and a dead-lettered one has to be presented
|
|
211
|
+
* again — it never reached the transcript, so a redelivery is the only remedy.
|
|
212
|
+
*
|
|
213
|
+
* An id leaves the set as soon as a wake hands it to Pi again (it is in flight
|
|
214
|
+
* once more, so the ordinary `awaiting` guard applies), when the daemon
|
|
215
|
+
* confirms it, or on a role/scope reset.
|
|
216
|
+
*
|
|
217
|
+
* Membership here does *not* by itself authorise a re-presentation: an
|
|
218
|
+
* unconditional pass turned every write-off into an immediate re-injection on
|
|
219
|
+
* the settlement that produced it, and an injection that fails the same way
|
|
220
|
+
* dead-letters the id again — an unbounded turn loop with no backoff and no
|
|
221
|
+
* limit. `representableWakeOutcomes` gates that instead (a wake carried by
|
|
222
|
+
* fresh content, or an explicit backoff with a bounded attempt budget).
|
|
223
|
+
*/
|
|
224
|
+
wakeRetryableEventIds: Set<number>;
|
|
225
|
+
/**
|
|
226
|
+
* Automatic re-presentations already spent per dead-lettered event id.
|
|
211
227
|
*
|
|
212
|
-
*
|
|
213
|
-
*
|
|
228
|
+
* Counted here rather than on the wire record because the budget is a
|
|
229
|
+
* property of *this* client's retry policy: the daemon's own `attempts`
|
|
230
|
+
* counter tracks acknowledgement failures, which are a different thing.
|
|
214
231
|
*/
|
|
215
|
-
|
|
232
|
+
wakeDeadLetterAttempts: Map<number, number>;
|
|
233
|
+
/** Earliest timestamp at which an id's next automatic retry may fire. */
|
|
234
|
+
wakeDeadLetterRetryAt: Map<number, number>;
|
|
216
235
|
/**
|
|
217
|
-
*
|
|
218
|
-
* `
|
|
219
|
-
*
|
|
236
|
+
* Consecutive ticks spent waiting for the orchestrator, i.e. the position in
|
|
237
|
+
* `WAKE_BUSY_BACKOFF_MS`. Reset when the orchestrator settles or the wake is
|
|
238
|
+
* cancelled, so a new busy period starts at the first rung again.
|
|
220
239
|
*/
|
|
221
|
-
|
|
240
|
+
wakeBusySpinRung: number;
|
|
222
241
|
wakeRequested: boolean;
|
|
223
242
|
wakeRequestedThroughEventId: number;
|
|
224
243
|
wakeTimer: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -275,47 +294,93 @@ const RECONNECTING_MESSAGE = "Herdsman is reconnecting · try again shortly";
|
|
|
275
294
|
export const MAX_ACK_ATTEMPTS = 5;
|
|
276
295
|
export const ACK_BACKOFF_CAP_MS = 30_000;
|
|
277
296
|
const KEEPALIVE_INTERVAL_MS = 30_000;
|
|
278
|
-
/**
|
|
297
|
+
/**
|
|
298
|
+
* First tick of the busy poll ladder, and the first rung of
|
|
299
|
+
* `WAKE_BUSY_BACKOFF_MS`.
|
|
300
|
+
*
|
|
301
|
+
* Kept as its own export for callers/tests that only care about the latency of
|
|
302
|
+
* the first deferral; everything else about the wait is in the ladder.
|
|
303
|
+
*/
|
|
279
304
|
export const WAKE_BUSY_SPIN_MS = 100;
|
|
280
305
|
/**
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
*
|
|
306
|
+
* Ceiling the busy poll ladder settles on for an arbitrarily long turn.
|
|
307
|
+
*
|
|
308
|
+
* The ladder does not simply stop at its last rung: `scheduleDeferredWake`
|
|
309
|
+
* clamps every tick to this value (the same shape `ackBackoffMs` uses for
|
|
310
|
+
* `ACK_BACKOFF_CAP_MS`), so raising a rung above the ceiling cannot silently
|
|
311
|
+
* raise the real cap. The ladder's top rung is this constant, which keeps the
|
|
312
|
+
* exported value, the comment, the decision note and the timed behaviour one
|
|
313
|
+
* and the same number.
|
|
285
314
|
*/
|
|
286
|
-
export const
|
|
315
|
+
export const WAKE_BUSY_BACKOFF_CAP_MS = 5_000;
|
|
287
316
|
/**
|
|
288
|
-
*
|
|
317
|
+
* Poll-frequency ladder while a wake waits for the orchestrator (milliseconds).
|
|
289
318
|
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
293
|
-
*
|
|
294
|
-
*
|
|
295
|
-
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
*
|
|
319
|
+
* Delivery does not branch on busy versus idle: every wake is queued and is
|
|
320
|
+
* injected once the orchestrator has settled (or a poll observes it idle), and
|
|
321
|
+
* a busy one cannot be interrupted anyway. `agent_settled` is therefore the
|
|
322
|
+
* primary signal that closes a deferral — it fires exactly when the turn ends
|
|
323
|
+
* and re-takes the wake decision — so the liveness bound is "until
|
|
324
|
+
* `agent_settled`, or until one of these ticks observes an idle orchestrator",
|
|
325
|
+
* never a deadline. The ladder exists to make that wait cheap, and it is a
|
|
326
|
+
* poll-frequency reduction, not a safety net: there is no deadline to miss. It
|
|
327
|
+
* used to tick at a flat 100ms forever, which made the extension re-evaluate
|
|
328
|
+
* the same deferral ten times a second for the whole turn and burn the turn's
|
|
329
|
+
* wall clock without ever being able to inject anything.
|
|
299
330
|
*
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
*
|
|
331
|
+
* Growing the gap keeps the correctness contract (nothing is injected until
|
|
332
|
+
* the orchestrator is idle, however long that takes) while making the poll
|
|
333
|
+
* cheap: the tick count per turn drops from ~10/s to at most ~2 in the first
|
|
334
|
+
* second and then ~1/s. The last rung is `WAKE_BUSY_BACKOFF_CAP_MS`, so a turn
|
|
335
|
+
* that outlives the ladder is re-evaluated at most once every 5s.
|
|
305
336
|
*/
|
|
306
|
-
export const
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
337
|
+
export const WAKE_BUSY_BACKOFF_MS = [
|
|
338
|
+
WAKE_BUSY_SPIN_MS,
|
|
339
|
+
200,
|
|
340
|
+
500,
|
|
341
|
+
1_000,
|
|
342
|
+
2_000,
|
|
343
|
+
WAKE_BUSY_BACKOFF_CAP_MS,
|
|
344
|
+
] as const;
|
|
311
345
|
/**
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
346
|
+
* How many times a dead-lettered wake event may be re-presented automatically
|
|
347
|
+
* before the client stops driving turns for it.
|
|
348
|
+
*
|
|
349
|
+
* Two, because a dead-letter means the content never reached the transcript and
|
|
350
|
+
* the usual causes (a turn that ended before its stop point, an abort, a
|
|
351
|
+
* message Pi dropped) leave no evidence that an immediate retry helps: one
|
|
352
|
+
* extra attempt covers a genuinely transient hiccup, the second covers a
|
|
353
|
+
* hiccup that also broke the first retry, and a third would only turn the
|
|
354
|
+
* retry into the unbounded turn-churn this cap exists to prevent. Each attempt
|
|
355
|
+
* is separated by `WAKE_DEAD_LETTER_RETRY_BACKOFF_MS`, so the bounded worst case
|
|
356
|
+
* is ~20s of automatic effort; after that the pending update is the human's
|
|
357
|
+
* (the write-off notify already says so), and the id stays unacked as the
|
|
358
|
+
* watermark barrier until a new external event merges it into a wake.
|
|
316
359
|
*/
|
|
317
|
-
const
|
|
318
|
-
|
|
360
|
+
export const WAKE_DEAD_LETTER_RETRY_LIMIT = 2;
|
|
361
|
+
/**
|
|
362
|
+
* Delay before the n-th automatic re-presentation of a dead-lettered event
|
|
363
|
+
* (milliseconds), i.e. `[0] === 5_000` for the first retry, `[1] === 15_000` for
|
|
364
|
+
* the second.
|
|
365
|
+
*
|
|
366
|
+
* Deliberately far from the 100ms busy tick: the retry must not read as a spin,
|
|
367
|
+
* and the settlement that produced the dead-letter has just finished, so an
|
|
368
|
+
* immediate re-injection would re-run exactly the turn that just failed.
|
|
369
|
+
*/
|
|
370
|
+
const WAKE_DEAD_LETTER_RETRY_BACKOFF_MS = [5_000, 15_000] as const;
|
|
371
|
+
/**
|
|
372
|
+
* Window after which an unchanged watermark hold-back is noted again
|
|
373
|
+
* (milliseconds).
|
|
374
|
+
*
|
|
375
|
+
* The hold-back note is deduplicated on its exact state (blocked id, reason,
|
|
376
|
+
* count), so a stable stall is logged once when it starts; this window is what
|
|
377
|
+
* keeps a stall that outlives that first line visible instead of silently
|
|
378
|
+
* freezing the queue forever, while still capping the noise at one line per
|
|
379
|
+
* minute per state.
|
|
380
|
+
*/
|
|
381
|
+
const WAKE_BARRIER_NOTE_INTERVAL_MS = 60_000;
|
|
382
|
+
/** `customType` of the hidden wake context this extension injects. */
|
|
383
|
+
const WAKE_CONTEXT_CUSTOM_TYPE = "herdsman-wake-context";
|
|
319
384
|
|
|
320
385
|
type AckFailureClass = "terminal" | "resync" | "transient";
|
|
321
386
|
|
|
@@ -390,7 +455,6 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
390
455
|
launchIdentity: undefined,
|
|
391
456
|
latestContext: undefined,
|
|
392
457
|
pendingEvents: [],
|
|
393
|
-
pinnedContext: undefined,
|
|
394
458
|
presentedEventIds: new Set(),
|
|
395
459
|
reconnectingFromOn: false,
|
|
396
460
|
registrationInFlight: undefined,
|
|
@@ -400,14 +464,14 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
400
464
|
subscriberId: undefined,
|
|
401
465
|
unackedDelivered: new Map(),
|
|
402
466
|
wakeDeferredUntilSettled: false,
|
|
403
|
-
wakeDeferredSince: undefined,
|
|
404
|
-
wakeForcedRelease: false,
|
|
405
467
|
wakeAwaitingConsumption: new Set(),
|
|
406
468
|
wakeConsumptionObserved: new Set(),
|
|
407
469
|
wakeSuppressedEventIds: new Set(),
|
|
408
470
|
wakeSkipLogReasons: new Map(),
|
|
409
|
-
|
|
410
|
-
|
|
471
|
+
wakeRetryableEventIds: new Set(),
|
|
472
|
+
wakeDeadLetterAttempts: new Map(),
|
|
473
|
+
wakeDeadLetterRetryAt: new Map(),
|
|
474
|
+
wakeBusySpinRung: 0,
|
|
411
475
|
wakeRequested: false,
|
|
412
476
|
wakeRequestedThroughEventId: 0,
|
|
413
477
|
wakeTimer: undefined,
|
|
@@ -417,6 +481,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
417
481
|
let activeContext: PiContext | undefined;
|
|
418
482
|
let keepaliveTimer: ReturnType<typeof setInterval> | undefined;
|
|
419
483
|
let wakeGeneration = 0;
|
|
484
|
+
// Throttle state for the watermark hold-back note (see
|
|
485
|
+
// `noteWakeWatermarkHoldBack`): the last noted barrier state and when it was
|
|
486
|
+
// noted, so a stable stall notes itself once and a long one re-notes itself
|
|
487
|
+
// per window instead of flooding the log.
|
|
488
|
+
let lastWakeBarrierNote: { at: number; key: string } | undefined;
|
|
420
489
|
|
|
421
490
|
const stopKeepalive = () => {
|
|
422
491
|
if (!keepaliveTimer) return;
|
|
@@ -434,15 +503,24 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
434
503
|
|
|
435
504
|
const setHerdsmanUi = (ctx: PiContext | undefined) => {
|
|
436
505
|
if (!ctx) return;
|
|
506
|
+
const pendingOutcomeCount = projectAgentOutcomes(state.pendingEvents, wakeFilter).outcomes.length;
|
|
437
507
|
const footerState: HerdsmanFooterState = state.reconnectingFromOn
|
|
438
508
|
? { kind: "reconnecting" }
|
|
439
509
|
: state.isOrchestrator
|
|
440
|
-
? {
|
|
441
|
-
kind: "on",
|
|
442
|
-
updateCount: projectAgentOutcomes(state.pendingEvents, wakeFilter).outcomes.length,
|
|
443
|
-
}
|
|
510
|
+
? { kind: "on", updateCount: pendingOutcomeCount }
|
|
444
511
|
: { kind: "off" };
|
|
445
512
|
ctx.ui.setStatus?.("herdsman", formatHerdsmanFooterStatus(footerState));
|
|
513
|
+
// A held-back wake is otherwise invisible during a long turn: updates are
|
|
514
|
+
// pending and nothing is injected, so the count alone reads as "Herdsman
|
|
515
|
+
// has work queued" instead of "the orchestrator is busy and this is what
|
|
516
|
+
// it is waiting for". The hint gets its own line (the pending count stays a
|
|
517
|
+
// count) and disappears the moment the wake is released or the queue drains.
|
|
518
|
+
ctx.ui.setStatus?.(
|
|
519
|
+
"herdsman-wake",
|
|
520
|
+
state.isOrchestrator && state.wakeDeferredUntilSettled && pendingOutcomeCount > 0
|
|
521
|
+
? "Herdsman · waiting for the current turn to end"
|
|
522
|
+
: undefined,
|
|
523
|
+
);
|
|
446
524
|
};
|
|
447
525
|
|
|
448
526
|
const cancelWakeTimer = () => {
|
|
@@ -450,23 +528,17 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
450
528
|
if (state.wakeTimer) clearTimeout(state.wakeTimer);
|
|
451
529
|
state.wakeTimer = undefined;
|
|
452
530
|
state.wakeDeferredUntilSettled = false;
|
|
453
|
-
state.
|
|
454
|
-
state.wakeForcedRelease = false;
|
|
531
|
+
state.wakeBusySpinRung = 0;
|
|
455
532
|
};
|
|
456
533
|
|
|
457
534
|
const cancelWake = () => {
|
|
458
535
|
cancelWakeTimer();
|
|
459
536
|
state.wakeRequested = false;
|
|
460
537
|
state.wakeRequestedThroughEventId = 0;
|
|
461
|
-
// A wake queued for a busy orchestrator belongs to the role/scope that
|
|
462
|
-
// queued it: dropping it here keeps a stale event body out of the context
|
|
463
|
-
// of whatever session takes over next.
|
|
464
|
-
state.wakeContext = undefined;
|
|
465
538
|
};
|
|
466
539
|
|
|
467
540
|
const clearAgentContext = () => {
|
|
468
541
|
state.latestContext = undefined;
|
|
469
|
-
state.pinnedContext = undefined;
|
|
470
542
|
state.runActive = false;
|
|
471
543
|
};
|
|
472
544
|
|
|
@@ -509,14 +581,18 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
509
581
|
* (and holding its batch open) for content that can never arrive. The other
|
|
510
582
|
* two sets describe the same ids and follow the queue just as well: a
|
|
511
583
|
* confirmed id needs no evidence flag, and an id that is gone from the queue
|
|
512
|
-
* cannot be re-injected, so it needs no suppression either.
|
|
584
|
+
* cannot be re-injected, so it needs no suppression either. The dead-letter
|
|
585
|
+
* retry bookkeeping follows for the same reason: a confirmed event needs no
|
|
586
|
+
* backoff and no attempt budget.
|
|
513
587
|
*/
|
|
514
588
|
const dropUnackedDelivered = (eventId: number): void => {
|
|
515
589
|
state.unackedDelivered.delete(eventId);
|
|
516
590
|
state.wakeAwaitingConsumption.delete(eventId);
|
|
517
591
|
state.wakeConsumptionObserved.delete(eventId);
|
|
518
592
|
state.wakeSuppressedEventIds.delete(eventId);
|
|
519
|
-
|
|
593
|
+
state.wakeRetryableEventIds.delete(eventId);
|
|
594
|
+
state.wakeDeadLetterAttempts.delete(eventId);
|
|
595
|
+
state.wakeDeadLetterRetryAt.delete(eventId);
|
|
520
596
|
};
|
|
521
597
|
|
|
522
598
|
/**
|
|
@@ -556,7 +632,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
556
632
|
state.wakeAwaitingConsumption.clear();
|
|
557
633
|
state.wakeConsumptionObserved.clear();
|
|
558
634
|
state.wakeSkipLogReasons.clear();
|
|
559
|
-
state.
|
|
635
|
+
state.wakeRetryableEventIds.clear();
|
|
636
|
+
// The automatic retry budget is a property of the id inside this scope: a
|
|
637
|
+
// scope that never held the id cannot owe it a retry, and keeping the
|
|
638
|
+
// counter would silently make the next scope's first write-off final.
|
|
639
|
+
state.wakeDeadLetterAttempts.clear();
|
|
640
|
+
state.wakeDeadLetterRetryAt.clear();
|
|
641
|
+
state.wakeBusySpinRung = 0;
|
|
560
642
|
};
|
|
561
643
|
|
|
562
644
|
const pruneAcknowledgedEvents = (ackedEventId: number | undefined) => {
|
|
@@ -586,6 +668,30 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
586
668
|
for (const eventId of [...state.wakeSkipLogReasons.keys()]) {
|
|
587
669
|
if (eventId <= ackedEventId) state.wakeSkipLogReasons.delete(eventId);
|
|
588
670
|
}
|
|
671
|
+
// A confirmed event is neither re-presentable nor owed a retry, so the three
|
|
672
|
+
// fields that describe a dead letter for it are swept together, here and in
|
|
673
|
+
// `dropUnackedDelivered`: the flag (may a wake present it again?), the spent
|
|
674
|
+
// attempt counter (does it still owe automatic retries?) and the retry
|
|
675
|
+
// deadline (when is the next one due?). Keeping any of them for an id the
|
|
676
|
+
// daemon has confirmed would let a later redelivery treat a settled event as
|
|
677
|
+
// an unfinished one — `alreadyPresented` would wave the id through and
|
|
678
|
+
// `deadLetterRetryDue` would hand it a budget it no longer needs.
|
|
679
|
+
//
|
|
680
|
+
// Ids still in the delivery queue are swept by the queue loop above (which
|
|
681
|
+
// routes through `dropUnackedDelivered`); this loop is what keeps the same
|
|
682
|
+
// three fields consistent for an id the queue no longer holds.
|
|
683
|
+
for (const eventId of [...state.wakeRetryableEventIds]) {
|
|
684
|
+
if (eventId <= ackedEventId) state.wakeRetryableEventIds.delete(eventId);
|
|
685
|
+
}
|
|
686
|
+
// A confirmed event owes no retry, so the dead-letter bookkeeping for it is
|
|
687
|
+
// garbage from here on: the counter would otherwise make the id's next
|
|
688
|
+
// write-off (after a reset) start from a spent budget.
|
|
689
|
+
for (const eventId of [...state.wakeDeadLetterAttempts.keys()]) {
|
|
690
|
+
if (eventId <= ackedEventId) {
|
|
691
|
+
state.wakeDeadLetterAttempts.delete(eventId);
|
|
692
|
+
state.wakeDeadLetterRetryAt.delete(eventId);
|
|
693
|
+
}
|
|
694
|
+
}
|
|
589
695
|
};
|
|
590
696
|
|
|
591
697
|
/**
|
|
@@ -615,9 +721,9 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
615
721
|
* Whether an event was already handed to the orchestrator in this session and
|
|
616
722
|
* therefore must not be injected again (logging why, at most once per reason).
|
|
617
723
|
*
|
|
618
|
-
* - `awaiting`: a copy of it was handed
|
|
619
|
-
*
|
|
620
|
-
*
|
|
724
|
+
* - `awaiting`: a copy of it was handed to Pi and has not been seen in the
|
|
725
|
+
* transcript yet (a dead-lettered one is re-presentable instead, see
|
|
726
|
+
* `wakeRetryableEventIds`, and is therefore checked first);
|
|
621
727
|
* - `presented`: it was injected before (or its content has since been observed
|
|
622
728
|
* in the transcript), so a redelivery would duplicate it;
|
|
623
729
|
* - `suppressed`: its copy may still sit in Pi's process-wide follow-up queue
|
|
@@ -625,6 +731,14 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
625
731
|
* only one that must not be shown (see `wakeSuppressedEventIds`).
|
|
626
732
|
*/
|
|
627
733
|
const alreadyPresented = (eventId: number): boolean => {
|
|
734
|
+
if (state.wakeRetryableEventIds.has(eventId)) {
|
|
735
|
+
// Its delivery was dead-lettered, so the content never reached the
|
|
736
|
+
// transcript: a redelivery is the only remedy and must not be skipped.
|
|
737
|
+
// *When* that redelivery may happen is a separate question, answered by
|
|
738
|
+
// `representableWakeOutcomes`; this predicate only says "no copy of this
|
|
739
|
+
// id is rightfully in the transcript".
|
|
740
|
+
return false;
|
|
741
|
+
}
|
|
628
742
|
if (state.wakeAwaitingConsumption.has(eventId)) {
|
|
629
743
|
noteSkippedWakeInjection(eventId, "awaiting");
|
|
630
744
|
return true;
|
|
@@ -640,6 +754,92 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
640
754
|
return false;
|
|
641
755
|
};
|
|
642
756
|
|
|
757
|
+
/**
|
|
758
|
+
* Backoff before the n-th automatic re-presentation of a dead-lettered id.
|
|
759
|
+
*
|
|
760
|
+
* The deadline and the budget are read together, so this maps "attempts
|
|
761
|
+
* already spent" to "how long this id must now stay quiet": 0 spent → 5s,
|
|
762
|
+
* 1 spent → 15s, anything beyond the budget is unreachable (the caller stops
|
|
763
|
+
* asking once `WAKE_DEAD_LETTER_RETRY_LIMIT` is reached).
|
|
764
|
+
*/
|
|
765
|
+
const deadLetterRetryBackoffMs = (attempts: number): number => {
|
|
766
|
+
const ladder = WAKE_DEAD_LETTER_RETRY_BACKOFF_MS;
|
|
767
|
+
const rung = Math.min(Math.max(attempts, 0), ladder.length - 1);
|
|
768
|
+
// `noUncheckedIndexedAccess` types every index access as optional; `rung` is
|
|
769
|
+
// clamped into a non-empty ladder, so the fallback is unreachable.
|
|
770
|
+
return ladder[rung] ?? ladder[0];
|
|
771
|
+
};
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Whether a dead-lettered id's automatic retry is due and still budgeted.
|
|
775
|
+
*
|
|
776
|
+
* This answers the question "may this id drive a turn of its own right
|
|
777
|
+
* now?", which is deliberately much narrower than "may it be presented
|
|
778
|
+
* again": a dead-lettered id may also ride a wake that fresh content leads
|
|
779
|
+
* (see `representableWakeOutcomes`), and that ride costs it nothing.
|
|
780
|
+
*
|
|
781
|
+
* The deadline is set when the id is written off (`writeOffStrandedWakeDelivery`)
|
|
782
|
+
* and the budget is `WAKE_DEAD_LETTER_RETRY_LIMIT`, so standing on its own
|
|
783
|
+
* means: not before the backoff has elapsed, and not after the last attempt
|
|
784
|
+
* has been spent. Once it is spent the id stops driving turns entirely — it
|
|
785
|
+
* stays in `unackedDelivered` as the watermark barrier and in the pending
|
|
786
|
+
* projection with its human-visible warning, and only fresh content can
|
|
787
|
+
* still carry it.
|
|
788
|
+
*/
|
|
789
|
+
const deadLetterRetryDue = (eventId: number): boolean => {
|
|
790
|
+
if (!state.wakeRetryableEventIds.has(eventId)) return false;
|
|
791
|
+
if ((state.wakeDeadLetterAttempts.get(eventId) ?? 0) >= WAKE_DEAD_LETTER_RETRY_LIMIT) {
|
|
792
|
+
return false;
|
|
793
|
+
}
|
|
794
|
+
return (state.wakeDeadLetterRetryAt.get(eventId) ?? 0) <= Date.now();
|
|
795
|
+
};
|
|
796
|
+
|
|
797
|
+
/** Earliest moment a still-budgeted id's automatic retry may fire. */
|
|
798
|
+
const deadLetterRetryAt = (eventId: number): number | undefined => {
|
|
799
|
+
if (!state.wakeRetryableEventIds.has(eventId)) return undefined;
|
|
800
|
+
if ((state.wakeDeadLetterAttempts.get(eventId) ?? 0) >= WAKE_DEAD_LETTER_RETRY_LIMIT) {
|
|
801
|
+
return undefined;
|
|
802
|
+
}
|
|
803
|
+
return state.wakeDeadLetterRetryAt.get(eventId);
|
|
804
|
+
};
|
|
805
|
+
|
|
806
|
+
/**
|
|
807
|
+
* The outcomes a wake may present: everything presentable that is not
|
|
808
|
+
* dead-lettered, plus the dead-lettered ids that may join it.
|
|
809
|
+
*
|
|
810
|
+
* A dead-letter used to be an unconditional pass through `alreadyPresented`,
|
|
811
|
+
* and because the settlement that produced it immediately scheduled the next
|
|
812
|
+
* wake, that turned every failed wake turn into the next wake turn: an
|
|
813
|
+
* unbounded loop with no backoff and no ceiling. The remedy is to separate
|
|
814
|
+
* the two ways an id can come back:
|
|
815
|
+
*
|
|
816
|
+
* - *rides*: the wake is led by content that is not a dead letter — an id this
|
|
817
|
+
* session has not handed over yet, or a genuinely new external event. That
|
|
818
|
+
* turn belongs to the fresh content, and merging the dead-lettered id into
|
|
819
|
+
* it is exactly the recovery it needs (the batch merge invariant), so it
|
|
820
|
+
* costs none of the id's retry budget;
|
|
821
|
+
* - *retries*: no fresh content is left, and the id waits for its explicit
|
|
822
|
+
* backoff (`deadLetterRetryDue`) and then spends one automatic attempt.
|
|
823
|
+
*
|
|
824
|
+
* Everything else leaves the id alone: the settlement that wrote it off, a
|
|
825
|
+
* daemon redelivery of the same id, an acknowledgement sweep, a reconnect,
|
|
826
|
+
* or an exhausted budget. The id is not lost in that case — it keeps blocking
|
|
827
|
+
* the watermark, keeps its pending row and its human-visible warning, and is
|
|
828
|
+
* still picked up by the next wake that fresh content leads.
|
|
829
|
+
*/
|
|
830
|
+
const representableWakeOutcomes = (
|
|
831
|
+
candidates: readonly AgentOutcome[],
|
|
832
|
+
): { lead: readonly AgentOutcome[]; outcomes: readonly AgentOutcome[] } => {
|
|
833
|
+
const lead = candidates.filter(
|
|
834
|
+
(outcome) => !state.wakeRetryableEventIds.has(outcome.eventId),
|
|
835
|
+
);
|
|
836
|
+
if (lead.length > 0) return { lead, outcomes: candidates };
|
|
837
|
+
return {
|
|
838
|
+
lead,
|
|
839
|
+
outcomes: candidates.filter((outcome) => deadLetterRetryDue(outcome.eventId)),
|
|
840
|
+
};
|
|
841
|
+
};
|
|
842
|
+
|
|
643
843
|
const isWakeableEvent = (event: AgentEventWireRecord | undefined) =>
|
|
644
844
|
!event?.nextAttemptAt || event.nextAttemptAt <= Date.now();
|
|
645
845
|
|
|
@@ -842,30 +1042,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
842
1042
|
}, WAKE_SETTLE_MS);
|
|
843
1043
|
};
|
|
844
1044
|
|
|
845
|
-
/**
|
|
846
|
-
* Ends the current wake-deferral episode without injecting.
|
|
847
|
-
*
|
|
848
|
-
* `wakeForcedRelease` and `wakeDeferredSince` describe *one* bounded
|
|
849
|
-
* deferral: the released deadline is re-derived from `wakeDeferredSince`
|
|
850
|
-
* every time `scheduleDeferredWake` runs, so a stale pair would let an
|
|
851
|
-
* unrelated later wake bypass the busy gate. Every path that ends a wake
|
|
852
|
-
* pass without injecting clears both, which gives the next deferral its own
|
|
853
|
-
* full `WAKE_DEFERRED_TIMEOUT_MS` budget. The 5s hard deadline itself is
|
|
854
|
-
* unchanged: it is measured inside a single episode, and an episode that
|
|
855
|
-
* reaches it still force-releases the batch on its next pass.
|
|
856
|
-
*/
|
|
857
|
-
const endWakeDeferral = () => {
|
|
858
|
-
state.wakeForcedRelease = false;
|
|
859
|
-
state.wakeDeferredSince = undefined;
|
|
860
|
-
};
|
|
861
|
-
|
|
862
1045
|
/**
|
|
863
1046
|
* Event ids a hidden wake message proves to have reached the transcript.
|
|
864
1047
|
*
|
|
865
1048
|
* A wake injection names the outcomes it presents in
|
|
866
1049
|
* `details.presentedEventIds`, and Pi writes those details onto the session
|
|
867
|
-
* entry it emits on `message_end
|
|
868
|
-
* finally drains the queued follow-up). That emission is the only consumption
|
|
1050
|
+
* entry it emits on `message_end`. That emission is the only consumption
|
|
869
1051
|
* evidence there is: nothing else tells us that the content — not merely the
|
|
870
1052
|
* request that carried it — reached the orchestrator.
|
|
871
1053
|
*
|
|
@@ -905,6 +1087,13 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
905
1087
|
* blocks the prefix: leaving it unacknowledged keeps it pending, which is what
|
|
906
1088
|
* lets the daemon redeliver the one copy that never arrived.
|
|
907
1089
|
*
|
|
1090
|
+
* An id awaiting consumption is the in-flight state of a delivery that has been
|
|
1091
|
+
* handed to Pi but whose `message_end` has not been seen yet. It blocks the
|
|
1092
|
+
* prefix, and the settlement that still finds it there dead-letters it
|
|
1093
|
+
* (`writeOffStrandedWakeDelivery`): the content never reached the transcript, so
|
|
1094
|
+
* it must stay pending in the daemon and be eligible for the next wake instead
|
|
1095
|
+
* of being confirmed by a heuristic.
|
|
1096
|
+
*
|
|
908
1097
|
* An event whose own acknowledgement already failed is left out (the daemon's
|
|
909
1098
|
* cursor advance sweeps it, and a retry would reset its attempt/backoff
|
|
910
1099
|
* accounting), but it must not hold the prefix back: it stays in the queue until
|
|
@@ -923,7 +1112,19 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
923
1112
|
const confirmablePrefix: AgentEventWireRecord[] = [];
|
|
924
1113
|
let blockedByMissingTurn = false;
|
|
925
1114
|
for (const event of deliveryQueue) {
|
|
926
|
-
|
|
1115
|
+
// An id that was handed to Pi and whose evidence has not arrived yet
|
|
1116
|
+
// blocks the prefix: it has no evidence at all, and the settlement still
|
|
1117
|
+
// owes it either that evidence (a run drains the copy) or a dead-letter.
|
|
1118
|
+
// A dead-lettered id keeps blocking it for the same reason — unconfirmed
|
|
1119
|
+
// content must never be swept by the watermark — while an id that is not
|
|
1120
|
+
// awaiting consumption and not observed otherwise looks like a delivered,
|
|
1121
|
+
// triggered prompt and would be confirmed by the turn heuristic.
|
|
1122
|
+
if (
|
|
1123
|
+
state.wakeAwaitingConsumption.has(event.id) ||
|
|
1124
|
+
state.wakeRetryableEventIds.has(event.id)
|
|
1125
|
+
) {
|
|
1126
|
+
break;
|
|
1127
|
+
}
|
|
927
1128
|
if (!state.wakeConsumptionObserved.has(event.id) && !turnProducedFinalResponse) {
|
|
928
1129
|
blockedByMissingTurn = true;
|
|
929
1130
|
break;
|
|
@@ -935,145 +1136,217 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
935
1136
|
}
|
|
936
1137
|
confirmablePrefix.push(event);
|
|
937
1138
|
}
|
|
938
|
-
//
|
|
939
|
-
//
|
|
940
|
-
//
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1139
|
+
// Nothing else keeps the batch in flight once the settlement has decided:
|
|
1140
|
+
// any id that is still awaiting consumption has been dead-lettered above,
|
|
1141
|
+
// which lifts only the injection barrier — it still blocks the watermark.
|
|
1142
|
+
return { deliveryQueue, confirmablePrefix, blockedByMissingTurn };
|
|
1143
|
+
};
|
|
1144
|
+
|
|
1145
|
+
/**
|
|
1146
|
+
* The smallest delivery-queue id that still blocks the acknowledgement
|
|
1147
|
+
* watermark, plus why it blocks and how many ids it holds back.
|
|
1148
|
+
*
|
|
1149
|
+
* This is the barrier `confirmableDeliveryPrefix` walks (the `break` in its
|
|
1150
|
+
* loop), read on its own because the silent upstream-error acknowledgement
|
|
1151
|
+
* must respect it without a turn authorising a prefix: the daemon confirms
|
|
1152
|
+
* by watermark (`update agent_events set status = 'acked' where id <= ?`), so
|
|
1153
|
+
* an acknowledgement at or above this id would hand the daemon the authority
|
|
1154
|
+
* to mark it confirmed without its content ever reaching the transcript —
|
|
1155
|
+
* the "busy delivery never arrives" failure in its silent form, with no
|
|
1156
|
+
* redelivery and no warning anywhere afterwards.
|
|
1157
|
+
*
|
|
1158
|
+
* An id awaiting consumption has no evidence at all yet, and a dead-lettered
|
|
1159
|
+
* one never reached the transcript: exactly the two states that must not be
|
|
1160
|
+
* confirmed by an ack meant for someone else's id. Ids the queue no longer
|
|
1161
|
+
* holds are not barriers — a confirmed id is done, and an id the daemon
|
|
1162
|
+
* itself terminally dead-lettered was its decision, made visible when it
|
|
1163
|
+
* happened.
|
|
1164
|
+
*/
|
|
1165
|
+
const unconfirmedWatermarkBarrier = ():
|
|
1166
|
+
| { blocked: number; eventId: number; reason: string }
|
|
1167
|
+
| undefined => {
|
|
1168
|
+
// How many ids the barrier holds back: awaiting consumption, or still
|
|
1169
|
+
// owed a dead-letter recovery (the two sets this walk checks).
|
|
1170
|
+
const blocked = new Set([
|
|
1171
|
+
...state.wakeAwaitingConsumption,
|
|
1172
|
+
...state.wakeRetryableEventIds,
|
|
1173
|
+
]).size;
|
|
1174
|
+
for (const queued of unackedDeliveredAscending()) {
|
|
1175
|
+
if (state.wakeRetryableEventIds.has(queued.id)) {
|
|
1176
|
+
return {
|
|
1177
|
+
blocked,
|
|
1178
|
+
eventId: queued.id,
|
|
1179
|
+
reason:
|
|
1180
|
+
(state.wakeDeadLetterAttempts.get(queued.id) ?? 0) >= WAKE_DEAD_LETTER_RETRY_LIMIT
|
|
1181
|
+
? "dead-letter-budget-exhausted"
|
|
1182
|
+
: "dead-letter-backoff",
|
|
1183
|
+
};
|
|
1184
|
+
}
|
|
1185
|
+
if (state.wakeAwaitingConsumption.has(queued.id)) {
|
|
1186
|
+
return { blocked, eventId: queued.id, reason: "awaiting-consumption" };
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
return undefined;
|
|
1190
|
+
};
|
|
1191
|
+
|
|
1192
|
+
/**
|
|
1193
|
+
* Notes — once per state, and again after `WAKE_BARRIER_NOTE_INTERVAL_MS` —
|
|
1194
|
+
* that the acknowledgement watermark is being held back on purpose.
|
|
1195
|
+
*
|
|
1196
|
+
* The hold-back is invisible otherwise: the queue simply stops advancing and
|
|
1197
|
+
* the log shows nothing, so a stalled delivery cannot be told apart from an
|
|
1198
|
+
* idle one. The line names the smallest blocked id, why it blocks
|
|
1199
|
+
* (dead-letter backoff, spent dead-letter budget, or awaiting consumption)
|
|
1200
|
+
* and how many ids it holds back, in the same field style as the write-off
|
|
1201
|
+
* warning and the injection-skip note. Deduplicating on the exact state
|
|
1202
|
+
* keeps a long stall from flooding the file; the window keeps a stall that
|
|
1203
|
+
* outlives its first line visible.
|
|
1204
|
+
*
|
|
1205
|
+
* "Long-term" cannot be decided from inside the extension — a barrier that
|
|
1206
|
+
* is about to be released looks exactly like one that never will be — so the
|
|
1207
|
+
* minimal observable contract is this: note the moment a due
|
|
1208
|
+
* acknowledgement is declined because of the barrier, not when a timer
|
|
1209
|
+
* merely fires.
|
|
1210
|
+
*/
|
|
1211
|
+
const noteWakeWatermarkHoldBack = (barrier: {
|
|
1212
|
+
blocked: number;
|
|
1213
|
+
eventId: number;
|
|
1214
|
+
reason: string;
|
|
1215
|
+
}): void => {
|
|
1216
|
+
const key = `${barrier.eventId}:${barrier.reason}:${barrier.blocked}`;
|
|
1217
|
+
const now = Date.now();
|
|
1218
|
+
if (
|
|
1219
|
+
lastWakeBarrierNote !== undefined &&
|
|
1220
|
+
lastWakeBarrierNote.key === key &&
|
|
1221
|
+
now - lastWakeBarrierNote.at < WAKE_BARRIER_NOTE_INTERVAL_MS
|
|
1222
|
+
) {
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
lastWakeBarrierNote = { at: now, key };
|
|
1226
|
+
logHerdsmanPi(
|
|
1227
|
+
"info",
|
|
1228
|
+
`[herdsman-pi] wake acknowledgement held eventId=${barrier.eventId} reason=${barrier.reason} blocked=${barrier.blocked} · the daemon confirms by watermark (id <= ?), so no larger id may be acknowledged while this one is unconfirmed; the held update stays pending in the daemon and recoverable`,
|
|
946
1229
|
);
|
|
947
|
-
return { deliveryQueue, confirmablePrefix, blockedByMissingTurn, stillAwaitingConsumption };
|
|
948
1230
|
};
|
|
949
1231
|
|
|
1232
|
+
/**
|
|
1233
|
+
* Dead-letters a wake delivery whose content never reached the transcript.
|
|
1234
|
+
*
|
|
1235
|
+
* The caller is a settlement that produced no consumption evidence for these
|
|
1236
|
+
* ids: the injected message was never carried out by any run (the turn ended
|
|
1237
|
+
* before its stop point, was aborted, or Pi dropped the message), so nothing
|
|
1238
|
+
* may be told to the daemon about them. An acknowledgement is a statement
|
|
1239
|
+
* that the orchestrator has the content, and here that statement would be a
|
|
1240
|
+
* lie: the daemon would mark the event consumed and never redeliver it.
|
|
1241
|
+
*
|
|
1242
|
+
* What the write-off does is local only: the ids stop being barred from every
|
|
1243
|
+
* later injection, so a later `scheduleWake` may re-present them with the real
|
|
1244
|
+
* update content. *May* is the whole point: `representableWakeOutcomes` decides
|
|
1245
|
+
* when, and until content that is not itself a dead letter leads a wake, the
|
|
1246
|
+
* only automatic resurface is a bounded, explicitly backed-off retry.
|
|
1247
|
+
* Re-presenting unconditionally on the settlement that had just written the id
|
|
1248
|
+
* off is what produced the turn loop this replaces. They deliberately stay in
|
|
1249
|
+
* `wakeAwaitingConsumption` (and therefore in the delivery queue): unconfirmed
|
|
1250
|
+
* content must keep blocking the acknowledgement watermark, because the daemon
|
|
1251
|
+
* confirms by `where id <= ?` and confirming a larger id would swallow them for
|
|
1252
|
+
* good. The events themselves stay pending in the daemon: nothing was
|
|
1253
|
+
* acknowledged, so a later wake is still eligible to present them, and a
|
|
1254
|
+
* replacement session can still receive them.
|
|
1255
|
+
*
|
|
1256
|
+
* The presentation guard is deliberately *not* extended here:
|
|
1257
|
+
* `presentedEventIds` is the "this content is in the transcript" set, and
|
|
1258
|
+
* these ids have no evidence of ever being there. Writing them off would
|
|
1259
|
+
* lock the update out of this session for good, which is exactly the
|
|
1260
|
+
* "busy delivery never arrives" failure this replaces — it is only the
|
|
1261
|
+
* *timing* of the next presentation that is now bounded.
|
|
1262
|
+
*/
|
|
950
1263
|
const writeOffStrandedWakeDelivery = (eventIds: number[], ctx: PiContext): void => {
|
|
951
|
-
|
|
952
|
-
// synchronously, while the acknowledgement is an RPC that may fail.
|
|
1264
|
+
if (eventIds.length === 0) return;
|
|
953
1265
|
for (const eventId of eventIds) {
|
|
954
1266
|
state.wakeAwaitingConsumption.delete(eventId);
|
|
955
|
-
//
|
|
956
|
-
//
|
|
957
|
-
//
|
|
958
|
-
//
|
|
959
|
-
|
|
1267
|
+
// Unconfirmed content must still block the acknowledgement watermark: the
|
|
1268
|
+
// daemon confirms by `where id <= ?`, so letting a later id through would
|
|
1269
|
+
// swallow this one for good. The retry set is that barrier, and it is also
|
|
1270
|
+
// what lifts the injection barrier — a delivery that never reached the
|
|
1271
|
+
// transcript has to be presented again (see `alreadyPresented`), but only
|
|
1272
|
+
// on `representableWakeOutcomes`' terms.
|
|
1273
|
+
state.wakeRetryableEventIds.add(eventId);
|
|
1274
|
+
// The write-off happens *after* the turn that failed to carry the content
|
|
1275
|
+
// out, so this is where "quiet since" is pinned: until the backoff
|
|
1276
|
+
// elapses, nothing may present the id on its own. A budget that is already
|
|
1277
|
+
// spent gets no new deadline either — the id stays a barrier, not a turn
|
|
1278
|
+
// driver, and only content that is fresh can still carry it.
|
|
1279
|
+
const attempts = state.wakeDeadLetterAttempts.get(eventId) ?? 0;
|
|
1280
|
+
if (attempts < WAKE_DEAD_LETTER_RETRY_LIMIT) {
|
|
1281
|
+
state.wakeDeadLetterRetryAt.set(
|
|
1282
|
+
eventId,
|
|
1283
|
+
Date.now() + deadLetterRetryBackoffMs(attempts),
|
|
1284
|
+
);
|
|
1285
|
+
} else {
|
|
1286
|
+
state.wakeDeadLetterRetryAt.delete(eventId);
|
|
1287
|
+
}
|
|
960
1288
|
}
|
|
961
|
-
// Fresh budget for whatever is still awaiting consumption: the ids just
|
|
962
|
-
// written off are gone, and the ones that remain have not had their own
|
|
963
|
-
// MAX_WAKE_CONTINUATION_ATTEMPTS drives spent on them yet.
|
|
964
|
-
state.wakeContinuationAttempts = 0;
|
|
965
|
-
const stranded = eventIds
|
|
966
|
-
.map((eventId) => state.unackedDelivered.get(eventId))
|
|
967
|
-
.filter((event): event is AgentEventWireRecord => event !== undefined)
|
|
968
|
-
.sort((left, right) => left.id - right.id);
|
|
969
1289
|
logHerdsmanPi(
|
|
970
1290
|
"warn",
|
|
971
|
-
`[herdsman-pi] wake
|
|
1291
|
+
`[herdsman-pi] wake delivery dead-lettered eventIds=${eventIds.join(",")} automaticRetries=${eventIds.map((eventId) => state.wakeDeadLetterAttempts.get(eventId) ?? 0).join(",")} limit=${WAKE_DEAD_LETTER_RETRY_LIMIT} · updates retained unacked in daemon: no consumption evidence, nothing acknowledged, and the ids stay eligible for a later wake`,
|
|
972
1292
|
);
|
|
973
1293
|
ctx.ui.notify?.(
|
|
974
|
-
`Herdsman · ${eventIds.length} agent update${eventIds.length === 1 ? "" : "s"} could not be delivered by a wake turn ·
|
|
1294
|
+
`Herdsman · ${eventIds.length} agent update${eventIds.length === 1 ? "" : "s"} could not be delivered by a wake turn · updates retained unacked in daemon: read the agent directly for the details, or hand this workspace to another terminal so the daemon delivers it there`,
|
|
975
1295
|
"warning",
|
|
976
1296
|
);
|
|
977
|
-
if (stranded.length === 0) return;
|
|
978
|
-
void acknowledgeEventIds(stranded, { notify: false }, ctx).catch((error: unknown) => {
|
|
979
|
-
logHerdsmanPi(
|
|
980
|
-
"warn",
|
|
981
|
-
`[herdsman-pi] wake write-off acknowledgement failed eventIds=${stranded.map((event) => event.id).join(",")} · ${String(error)}`,
|
|
982
|
-
);
|
|
983
|
-
});
|
|
984
1297
|
};
|
|
985
1298
|
|
|
986
|
-
/**
|
|
987
|
-
* Drives one continuation that carries out a wake Pi has not drained.
|
|
988
|
-
*
|
|
989
|
-
* A wake injected while the orchestrator streams is delivered as a queued
|
|
990
|
-
* follow-up (`triggerTurn: false`), and Pi only drains that queue when a run
|
|
991
|
-
* reaches its stop point (`agent-loop` "Agent would stop here. Check for
|
|
992
|
-
* follow-up messages."). When the run it rode on already passed that point,
|
|
993
|
-
* the update sits in the queue until the next user message. At settlement the
|
|
994
|
-
* orchestrator is no longer streaming, so `triggerTurn: true` starts a real
|
|
995
|
-
* run (`_runAgentPrompt`) and that run's loop drains the queued follow-up.
|
|
996
|
-
* The marker carries no wake content: the queued follow-up is what delivers
|
|
997
|
-
* the evidence, exactly once.
|
|
998
|
-
*
|
|
999
|
-
* Only consumption evidence writes a delivery off, so an intervening run that
|
|
1000
|
-
* ends before its stop point (error, user abort, a refused tool) leaves the
|
|
1001
|
-
* next settlement driving again. That is bounded by
|
|
1002
|
-
* MAX_WAKE_CONTINUATION_ATTEMPTS: starting runs cannot fix a cause that is
|
|
1003
|
-
* not about the queue, so once the budget is spent the delivery is written
|
|
1004
|
-
* off (released and acknowledged) instead of looping forever — see
|
|
1005
|
-
* `writeOffStrandedWakeDelivery` for the trade-off.
|
|
1006
|
-
*/
|
|
1007
|
-
const driveWakeContinuation = (ctx: PiContext) => {
|
|
1008
|
-
if (state.wakeAwaitingConsumption.size === 0) return;
|
|
1009
|
-
if (!pi.sendMessage) return;
|
|
1010
|
-
const eventIds = [...state.wakeAwaitingConsumption].sort((left, right) => left - right);
|
|
1011
|
-
state.wakeContinuationAttempts += 1;
|
|
1012
|
-
try {
|
|
1013
|
-
pi.sendMessage(
|
|
1014
|
-
{
|
|
1015
|
-
content: WAKE_CONTINUATION_CONTENT,
|
|
1016
|
-
customType: WAKE_CONTINUATION_CUSTOM_TYPE,
|
|
1017
|
-
display: false,
|
|
1018
|
-
},
|
|
1019
|
-
{ deliverAs: "followUp", triggerTurn: true },
|
|
1020
|
-
);
|
|
1021
|
-
logHerdsmanPi(
|
|
1022
|
-
"info",
|
|
1023
|
-
`[herdsman-pi] wake continuation driven eventIds=${eventIds[0] ?? 0}-${eventIds.at(-1) ?? 0} count=${eventIds.length} attempt=${state.wakeContinuationAttempts}`,
|
|
1024
|
-
);
|
|
1025
|
-
} catch {
|
|
1026
|
-
logHerdsmanPi("warn", "[herdsman-pi] wake continuation refused by pi");
|
|
1027
|
-
}
|
|
1028
|
-
if (state.wakeContinuationAttempts >= MAX_WAKE_CONTINUATION_ATTEMPTS) {
|
|
1029
|
-
// This was the last drive the budget allows, and the queued copy is still
|
|
1030
|
-
// undrained: the delivery is written off instead of being left pending
|
|
1031
|
-
// forever. The ids are released from `wakeAwaitingConsumption` (no further
|
|
1032
|
-
// run is started for them) and acknowledged, which is what stops the
|
|
1033
|
-
// daemon from redelivering them every freshness window — the loop that
|
|
1034
|
-
// used to flood the orchestrator session until manual sqlite surgery.
|
|
1035
|
-
// Losing one update is the accepted cost; see
|
|
1036
|
-
// `writeOffStrandedWakeDelivery`.
|
|
1037
|
-
writeOffStrandedWakeDelivery([...state.wakeAwaitingConsumption], ctx);
|
|
1038
|
-
}
|
|
1039
|
-
};
|
|
1040
1299
|
|
|
1041
1300
|
/**
|
|
1042
|
-
* Arms the
|
|
1301
|
+
* Arms the deferral for a wake that cannot be injected right now.
|
|
1302
|
+
*
|
|
1303
|
+
* Delivery does not depend on whether the orchestrator is busy or idle:
|
|
1304
|
+
* every wake is queued and injected once the orchestrator has settled (or a
|
|
1305
|
+
* poll observes it idle). `agent_settled` is therefore the primary close
|
|
1306
|
+
* signal — it fires exactly when the turn ends and re-takes the wake
|
|
1307
|
+
* decision — and the timer here is only the poll that keeps the decision
|
|
1308
|
+
* alive while the turn runs. It backs off along `WAKE_BUSY_BACKOFF_MS` up to
|
|
1309
|
+
* `WAKE_BUSY_BACKOFF_CAP_MS` (a poll-frequency reduction, not a safety net):
|
|
1310
|
+
* the ladder used to tick at a flat `WAKE_BUSY_SPIN_MS`, so a long turn was
|
|
1311
|
+
* re-evaluated ten times a second forever.
|
|
1043
1312
|
*
|
|
1044
|
-
*
|
|
1045
|
-
*
|
|
1046
|
-
*
|
|
1047
|
-
*
|
|
1048
|
-
*
|
|
1313
|
+
* There is deliberately no deadline (nothing like a `WAKE_DEFERRED_TIMEOUT_MS`
|
|
1314
|
+
* that force-releases a busy orchestrator after N seconds): a wake injected
|
|
1315
|
+
* into a busy orchestrator either lands in a follow-up queue nothing drains
|
|
1316
|
+
* — the "busy delivery never arrives" failure this replaces — or interrupts
|
|
1317
|
+
* the user's own turn, so waiting is the only outcome that loses nothing.
|
|
1318
|
+
* The liveness bound is exactly "until `agent_settled`, or until one of these
|
|
1319
|
+
* ticks observes an idle orchestrator"; backing off only makes that wait
|
|
1320
|
+
* cheaper, never shorter.
|
|
1049
1321
|
*/
|
|
1050
1322
|
const scheduleDeferredWake = (ctx: PiContext) => {
|
|
1051
1323
|
state.wakeDeferredUntilSettled = true;
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1324
|
+
// The deferral is the state the user cannot see (nothing is injected, no
|
|
1325
|
+
// turn is started, and the pending count alone reads as idle work), so it
|
|
1326
|
+
// is announced once per arming rather than left to be guessed.
|
|
1327
|
+
setHerdsmanUi(ctx);
|
|
1056
1328
|
if (state.wakeTimer) return;
|
|
1329
|
+
const rung = Math.min(state.wakeBusySpinRung, WAKE_BUSY_BACKOFF_MS.length - 1);
|
|
1330
|
+
state.wakeBusySpinRung = Math.min(rung + 1, WAKE_BUSY_BACKOFF_MS.length - 1);
|
|
1331
|
+
// Clamped to the cap rather than trusted to the ladder's last rung: a rung
|
|
1332
|
+
// edited above the ceiling would otherwise raise the real backoff while the
|
|
1333
|
+
// exported constant, the note and the tests still say 5s.
|
|
1334
|
+
const delay = Math.min(
|
|
1335
|
+
WAKE_BUSY_BACKOFF_MS[rung] ?? WAKE_BUSY_SPIN_MS,
|
|
1336
|
+
WAKE_BUSY_BACKOFF_CAP_MS,
|
|
1337
|
+
);
|
|
1057
1338
|
state.wakeTimer = setTimeout(() => {
|
|
1058
1339
|
state.wakeTimer = undefined;
|
|
1059
|
-
if (state.wakeForcedRelease && state.deliveredBatch && ctx.isIdle?.() !== false) {
|
|
1060
|
-
// The hard deadline only releases the delivery — it never discards an
|
|
1061
|
-
// unconfirmed event. The batch record is dropped so a wake turn that
|
|
1062
|
-
// is no longer running cannot gate later wakes, but its events stay in
|
|
1063
|
-
// the delivery queue (`unackedDelivered`) and are re-attached to the
|
|
1064
|
-
// batch injected right below, which acknowledges them once it settles.
|
|
1065
|
-
state.deliveredBatch = undefined;
|
|
1066
|
-
}
|
|
1067
1340
|
scheduleWake(ctx);
|
|
1068
|
-
},
|
|
1341
|
+
}, delay);
|
|
1069
1342
|
};
|
|
1070
1343
|
|
|
1071
1344
|
const scheduleWake = (ctx: PiContext | undefined) => {
|
|
1072
1345
|
if (!ctx || !state.isOrchestrator || !state.currentScope || !pi.sendMessage) return;
|
|
1073
1346
|
if (state.wakeTimer || state.wakeRequested) {
|
|
1074
1347
|
// A pending wake owns the release; this branch is also hit re-entrantly by
|
|
1075
|
-
// a running pass (its fired settle timer is still set),
|
|
1076
|
-
//
|
|
1348
|
+
// a running pass (its fired settle timer is still set), and ending the
|
|
1349
|
+
// deferral there would let a second injection ride in behind it.
|
|
1077
1350
|
return;
|
|
1078
1351
|
}
|
|
1079
1352
|
const projection = projectAgentOutcomes(state.pendingEvents, wakeFilter);
|
|
@@ -1096,11 +1369,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1096
1369
|
const wakeable = outcomes.filter((outcome) =>
|
|
1097
1370
|
isWakeableEvent(state.pendingEvents.find((pending) => pending.id === outcome.eventId)),
|
|
1098
1371
|
);
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1372
|
+
// A dead-lettered id may only join a wake that fresh content leads, or, on
|
|
1373
|
+
// its own, wait for its explicit backoff (see `representableWakeOutcomes`).
|
|
1374
|
+
// Without this the settlement that wrote it off re-injects it in the same
|
|
1375
|
+
// cycle, which is the turn loop this replaces.
|
|
1376
|
+
const { outcomes: injectableOutcomes } = representableWakeOutcomes(wakeable);
|
|
1377
|
+
if (injectableOutcomes.length === 0) {
|
|
1104
1378
|
// A suppressed upstream error that is now due must be silently
|
|
1105
1379
|
// acknowledged before any backoff timer is planted: planting the timer
|
|
1106
1380
|
// first would make scheduleSilentUpstreamErrorAck's entry guard
|
|
@@ -1108,19 +1382,48 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1108
1382
|
// off its own timer and, once the backoff window has elapsed, the
|
|
1109
1383
|
// 0ms-timer + blocked-ack loop never converges the queue.
|
|
1110
1384
|
const dueSuppressed = suppressedEvents.filter(isWakeableEvent);
|
|
1111
|
-
|
|
1112
|
-
|
|
1385
|
+
// A silent acknowledgement is a watermark statement, though: the daemon
|
|
1386
|
+
// confirms every id at or below the acked one, so it may not cross an id
|
|
1387
|
+
// that is still unconfirmed locally. The barrier is the same one
|
|
1388
|
+
// `confirmableDeliveryPrefix` walks (`unconfirmedWatermarkBarrier`), and
|
|
1389
|
+
// crossing it would hand the daemon the authority to mark the blocked id
|
|
1390
|
+
// confirmed without its content ever reaching the transcript — the
|
|
1391
|
+
// "busy delivery never arrives" failure in its silent form, with no
|
|
1392
|
+
// redelivery and no warning afterwards. The held ids stay due and are
|
|
1393
|
+
// acknowledged by a later pass, once the barrier id is confirmed (or
|
|
1394
|
+
// recovered) and the watermark has moved past it.
|
|
1395
|
+
const watermarkBarrier = unconfirmedWatermarkBarrier();
|
|
1396
|
+
const ackableSuppressed =
|
|
1397
|
+
watermarkBarrier === undefined
|
|
1398
|
+
? dueSuppressed
|
|
1399
|
+
: dueSuppressed.filter((event) => event.id < watermarkBarrier.eventId);
|
|
1400
|
+
if (ackableSuppressed.length > 0) {
|
|
1401
|
+
scheduleSilentUpstreamErrorAck(ctx, ackableSuppressed);
|
|
1113
1402
|
return;
|
|
1114
1403
|
}
|
|
1404
|
+
if (watermarkBarrier !== undefined && dueSuppressed.length > 0) {
|
|
1405
|
+
// Held back by the barrier rather than by a missing tick: say so, in
|
|
1406
|
+
// the throttled shape above, so the stall is observable.
|
|
1407
|
+
noteWakeWatermarkHoldBack(watermarkBarrier);
|
|
1408
|
+
}
|
|
1115
1409
|
// No suppressed event is due, so plant a backoff timer for the next
|
|
1116
1410
|
// future `nextAttemptAt`. Expired timestamps are excluded (strictly
|
|
1117
1411
|
// greater than now) so an already-past window does not produce a
|
|
1118
|
-
// zero-delay spin.
|
|
1412
|
+
// zero-delay spin. A dead-lettered id that still owes an automatic retry
|
|
1413
|
+
// contributes its own deadline here: without it the retry the write-off
|
|
1414
|
+
// scheduled would wait for an unrelated event to come along.
|
|
1119
1415
|
const nextAttemptAt = [
|
|
1120
1416
|
...outcomes.map((outcome) => outcome.eventId),
|
|
1121
1417
|
...suppressedEvents.map((event) => event.id),
|
|
1122
1418
|
]
|
|
1123
|
-
.
|
|
1419
|
+
.flatMap((eventId) => {
|
|
1420
|
+
const nextAttempt = state.pendingEvents.find((event) => event.id === eventId)
|
|
1421
|
+
?.nextAttemptAt;
|
|
1422
|
+
const retryAt = state.wakeRetryableEventIds.has(eventId)
|
|
1423
|
+
? deadLetterRetryAt(eventId)
|
|
1424
|
+
: undefined;
|
|
1425
|
+
return [nextAttempt, retryAt];
|
|
1426
|
+
})
|
|
1124
1427
|
.filter((value): value is number => value !== undefined && value > Date.now())
|
|
1125
1428
|
.sort((left, right) => left - right)[0];
|
|
1126
1429
|
if (nextAttemptAt !== undefined) {
|
|
@@ -1133,10 +1436,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1133
1436
|
}
|
|
1134
1437
|
|
|
1135
1438
|
// An in-flight batch owns the ack cursor and a busy orchestrator must not
|
|
1136
|
-
// be interrupted, so neither is woken immediately —
|
|
1137
|
-
// on
|
|
1439
|
+
// be interrupted, so neither is woken immediately — the wake stays
|
|
1440
|
+
// deferred on the backed-off poll until the batch settles and the
|
|
1441
|
+
// orchestrator is idle, and nothing is released past that gate.
|
|
1138
1442
|
const inFlight = state.deliveredBatch !== undefined || state.ackInFlight;
|
|
1139
|
-
if (
|
|
1443
|
+
if (inFlight || ctx.isIdle?.() === false) {
|
|
1140
1444
|
scheduleDeferredWake(ctx);
|
|
1141
1445
|
return;
|
|
1142
1446
|
}
|
|
@@ -1145,8 +1449,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1145
1449
|
const ownerTerminalId = state.currentScope.terminalId;
|
|
1146
1450
|
const ownerWorkspaceId = state.currentScope.workspaceId;
|
|
1147
1451
|
state.wakeTimer = setTimeout(() => {
|
|
1148
|
-
const startWake = async () => {
|
|
1149
|
-
if (
|
|
1452
|
+
const startWake = async () => { if (
|
|
1150
1453
|
generation !== wakeGeneration ||
|
|
1151
1454
|
!state.isOrchestrator ||
|
|
1152
1455
|
state.currentScope?.herdrSessionName !== ownerHerdrSessionName ||
|
|
@@ -1154,16 +1457,12 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1154
1457
|
state.currentScope?.workspaceId !== ownerWorkspaceId
|
|
1155
1458
|
) {
|
|
1156
1459
|
state.wakeTimer = undefined;
|
|
1157
|
-
// The pass belongs to a stale generation or scope, so its deferral
|
|
1158
|
-
// episode ends here (a scope reset usually got there first through
|
|
1159
|
-
// `cancelWakeTimer`).
|
|
1160
|
-
endWakeDeferral();
|
|
1161
1460
|
return;
|
|
1162
1461
|
}
|
|
1163
|
-
if (ctx.isIdle?.() === false
|
|
1462
|
+
if (ctx.isIdle?.() === false) {
|
|
1164
1463
|
state.wakeTimer = undefined;
|
|
1165
|
-
// Still busy: this pass re-defers
|
|
1166
|
-
//
|
|
1464
|
+
// Still busy: this pass re-defers and keeps waiting, so nothing is
|
|
1465
|
+
// injected into (or parked for) a running turn.
|
|
1167
1466
|
scheduleDeferredWake(ctx);
|
|
1168
1467
|
return;
|
|
1169
1468
|
}
|
|
@@ -1175,13 +1474,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1175
1474
|
)) as ConnectionStateResponse | undefined;
|
|
1176
1475
|
if (!response) {
|
|
1177
1476
|
state.wakeTimer = undefined;
|
|
1178
|
-
endWakeDeferral();
|
|
1179
1477
|
return;
|
|
1180
1478
|
}
|
|
1181
1479
|
applyConnectionStateResponse(response, ctx);
|
|
1182
1480
|
} catch {
|
|
1183
1481
|
state.wakeTimer = undefined;
|
|
1184
|
-
endWakeDeferral();
|
|
1185
1482
|
// A failed load is only temporary: the batch stays pending and is
|
|
1186
1483
|
// retried on the next wake instead of being permanently suppressed.
|
|
1187
1484
|
ctx.ui.notify?.(
|
|
@@ -1199,14 +1496,11 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1199
1496
|
state.currentScope?.workspaceId !== ownerWorkspaceId
|
|
1200
1497
|
) {
|
|
1201
1498
|
state.wakeTimer = undefined;
|
|
1202
|
-
// See the earlier generation re-check: the episode ends with it.
|
|
1203
|
-
endWakeDeferral();
|
|
1204
1499
|
return;
|
|
1205
1500
|
}
|
|
1206
|
-
if (ctx.isIdle?.() === false
|
|
1501
|
+
if (ctx.isIdle?.() === false) {
|
|
1207
1502
|
state.wakeTimer = undefined;
|
|
1208
|
-
// See the earlier re-check:
|
|
1209
|
-
// deadline intact.
|
|
1503
|
+
// See the earlier re-check: a busy orchestrator keeps waiting.
|
|
1210
1504
|
scheduleDeferredWake(ctx);
|
|
1211
1505
|
return;
|
|
1212
1506
|
}
|
|
@@ -1214,15 +1508,20 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1214
1508
|
const batchEvents = [...state.pendingEvents].sort((left, right) => left.id - right.id);
|
|
1215
1509
|
const batchProjection = projectAgentOutcomes(batchEvents, wakeFilter);
|
|
1216
1510
|
const batchSuppressedIds = new Set(batchProjection.suppressedUpstreamErrorEventIds);
|
|
1217
|
-
const
|
|
1511
|
+
const projectableBatchOutcomes = batchProjection.outcomes.filter(
|
|
1218
1512
|
(outcome) =>
|
|
1219
1513
|
outcome.eventId > state.failedWakeThroughEventId &&
|
|
1220
1514
|
!alreadyPresented(outcome.eventId) &&
|
|
1221
1515
|
isWakeableEvent(batchEvents.find((event) => event.id === outcome.eventId)),
|
|
1222
1516
|
);
|
|
1517
|
+
// Same split as in `scheduleWake`, re-taken here because the pass
|
|
1518
|
+
// awaited an `agent.orchestrator.get` in between: a dead-lettered id
|
|
1519
|
+
// may only ride content that is fresh, or retry on its own once its
|
|
1520
|
+
// backoff is due (see `representableWakeOutcomes`).
|
|
1521
|
+
const { lead: batchLeadOutcomes, outcomes: batchOutcomes } =
|
|
1522
|
+
representableWakeOutcomes(projectableBatchOutcomes);
|
|
1223
1523
|
if (batchOutcomes.length === 0) {
|
|
1224
1524
|
state.wakeTimer = undefined;
|
|
1225
|
-
endWakeDeferral();
|
|
1226
1525
|
return;
|
|
1227
1526
|
}
|
|
1228
1527
|
const current = batchOutcomes;
|
|
@@ -1233,19 +1532,20 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1233
1532
|
state.wakeTimer = undefined;
|
|
1234
1533
|
state.wakeRequested = true;
|
|
1235
1534
|
state.wakeRequestedThroughEventId = current.at(-1)?.eventId ?? 0;
|
|
1236
|
-
//
|
|
1237
|
-
// follow-up
|
|
1238
|
-
//
|
|
1239
|
-
// follow-up
|
|
1240
|
-
//
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
//
|
|
1245
|
-
// late can be told apart from one
|
|
1535
|
+
// Single-track injection: the wake content is handed to Pi exactly
|
|
1536
|
+
// once, as the follow-up message below, and only once the orchestrator
|
|
1537
|
+
// is idle — a busy one has been deferred (see `scheduleDeferredWake`),
|
|
1538
|
+
// because a follow-up parked in a running turn's queue may never be
|
|
1539
|
+
// drained. There is deliberately no second copy on the `context` hook
|
|
1540
|
+
// either: the update enters the transcript exactly once per event id,
|
|
1541
|
+
// through the one message this path sends.
|
|
1542
|
+
const wakeContent = formatAgentOutcomeUpdates([...batchOutcomes]);
|
|
1543
|
+
// Single line, injection path only: the state the batch was taken from,
|
|
1544
|
+
// so a wake that still arrives late can be told apart from one that was
|
|
1545
|
+
// held back by the busy gate.
|
|
1246
1546
|
logHerdsmanPi(
|
|
1247
1547
|
"info",
|
|
1248
|
-
`[herdsman-pi] wake inject deliverAs=followUp triggerTurn
|
|
1548
|
+
`[herdsman-pi] wake inject deliverAs=followUp triggerTurn=true runActive=${String(state.runActive)} isIdle=${ctx.isIdle === undefined ? "unknown" : String(ctx.isIdle())} eventIds=${batchOutcomes[0]?.eventId ?? 0}-${batchOutcomes.at(-1)?.eventId ?? 0} count=${batchOutcomes.length}`,
|
|
1249
1549
|
);
|
|
1250
1550
|
try {
|
|
1251
1551
|
pi.sendMessage?.(
|
|
@@ -1267,35 +1567,43 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1267
1567
|
},
|
|
1268
1568
|
display: false,
|
|
1269
1569
|
},
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1570
|
+
// `triggerTurn: true` is what makes this a real turn: pi hands the
|
|
1571
|
+
// message to the agent core, which produces a regular `message_end`
|
|
1572
|
+
// for the hidden entry, and only that emission is the consumption
|
|
1573
|
+
// evidence the acknowledgement waits for. A `triggerTurn: false`
|
|
1574
|
+
// variant only parks the message in a follow-up queue that a run
|
|
1575
|
+
// drains at its stop point — the copy that made "busy delivery
|
|
1576
|
+
// failed" possible.
|
|
1577
|
+
{ deliverAs: "followUp", triggerTurn: true },
|
|
1273
1578
|
);
|
|
1274
|
-
// A
|
|
1275
|
-
//
|
|
1276
|
-
//
|
|
1277
|
-
//
|
|
1278
|
-
//
|
|
1279
|
-
//
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1579
|
+
// A freshly delivered id is in flight until its content is seen in
|
|
1580
|
+
// the transcript: it can neither be acknowledged (the evidence is
|
|
1581
|
+
// what authorises that) nor injected again (the content must not
|
|
1582
|
+
// enter the transcript twice). An id merged into this batch from an
|
|
1583
|
+
// earlier delivery keeps that same state; a dead-lettered one is in
|
|
1584
|
+
// flight again and loses its retry flag.
|
|
1585
|
+
for (const outcome of batchOutcomes) {
|
|
1586
|
+
// A dead-lettered id that retried on its own spends one of its
|
|
1587
|
+
// automatic attempts; one that rode a wake fresh content led is
|
|
1588
|
+
// riding that update's turn, not its own, and rides for free.
|
|
1589
|
+
if (
|
|
1590
|
+
batchLeadOutcomes.length === 0 &&
|
|
1591
|
+
state.wakeRetryableEventIds.has(outcome.eventId)
|
|
1592
|
+
) {
|
|
1593
|
+
state.wakeDeadLetterAttempts.set(
|
|
1594
|
+
outcome.eventId,
|
|
1595
|
+
(state.wakeDeadLetterAttempts.get(outcome.eventId) ?? 0) + 1,
|
|
1596
|
+
);
|
|
1283
1597
|
}
|
|
1284
|
-
state.
|
|
1598
|
+
state.wakeAwaitingConsumption.add(outcome.eventId);
|
|
1599
|
+
state.wakeRetryableEventIds.delete(outcome.eventId);
|
|
1600
|
+
state.wakeDeadLetterRetryAt.delete(outcome.eventId);
|
|
1285
1601
|
}
|
|
1286
|
-
// A queued (non-triggering) delivery keeps the content available to
|
|
1287
|
-
// the current turn through the context hook until it is settled or
|
|
1288
|
-
// superseded by the next injection.
|
|
1289
|
-
state.wakeContext = orchestratorBusy
|
|
1290
|
-
? { content: wakeContent, eventIds: batchOutcomes.map((outcome) => outcome.eventId) }
|
|
1291
|
-
: undefined;
|
|
1292
|
-
state.wakeForcedRelease = false;
|
|
1293
|
-
state.wakeDeferredSince = undefined;
|
|
1294
1602
|
// Only expose the batch after the hidden context was accepted by pi. This
|
|
1295
1603
|
// keeps an injection failure eligible for daemon redelivery.
|
|
1296
1604
|
//
|
|
1297
1605
|
// The batch is the delivery queue plus this injection: previously
|
|
1298
|
-
// unconfirmed events are merged (never replaced) so that a
|
|
1606
|
+
// unconfirmed events are merged (never replaced) so that a later
|
|
1299
1607
|
// release always leaves both the old and the new events deliverable
|
|
1300
1608
|
// and acknowledgeable in id order. The turn-consumption flags of a
|
|
1301
1609
|
// still-running previous batch are carried over, because they
|
|
@@ -1304,39 +1612,40 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1304
1612
|
// `hasSubstantiveWork` is the sole gate that decides whether an
|
|
1305
1613
|
// ownership/scope change may abort the in-flight turn (see loseRole
|
|
1306
1614
|
// and resetForScopeChange), and aborting is only ever allowed for a
|
|
1307
|
-
// *pure* Herdsman wake turn.
|
|
1308
|
-
//
|
|
1309
|
-
//
|
|
1310
|
-
//
|
|
1615
|
+
// *pure* Herdsman wake turn. The injected message starts one of those
|
|
1616
|
+
// (it is only ever sent to an idle orchestrator, and it triggers its
|
|
1617
|
+
// own turn), and nothing here has produced substance yet, so the
|
|
1618
|
+
// flag starts false and the tool/assistant hooks raise it.
|
|
1311
1619
|
state.deliveredBatch = {
|
|
1312
1620
|
abortedByUser: previousBatch?.abortedByUser ?? false,
|
|
1313
1621
|
assistantFinalSucceeded: previousBatch?.assistantFinalSucceeded ?? false,
|
|
1314
1622
|
events: mergeUnackedDelivered(incomingEvents),
|
|
1315
|
-
hasSubstantiveWork:
|
|
1623
|
+
hasSubstantiveWork: previousBatch?.hasSubstantiveWork ?? false,
|
|
1316
1624
|
invalidated: false,
|
|
1317
1625
|
ownerTerminalId,
|
|
1318
1626
|
herdsmanTriggered: true,
|
|
1319
1627
|
};
|
|
1320
1628
|
state.wakeRequested = false;
|
|
1321
1629
|
state.wakeRequestedThroughEventId = 0;
|
|
1322
|
-
//
|
|
1323
|
-
//
|
|
1324
|
-
|
|
1325
|
-
//
|
|
1326
|
-
//
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1630
|
+
// The busy wait is over: whatever turn comes next starts the poll
|
|
1631
|
+
// ladder from its first rung again.
|
|
1632
|
+
state.wakeBusySpinRung = 0;
|
|
1633
|
+
// The ids deliberately do *not* enter `presentedEventIds` here: that set
|
|
1634
|
+
// means "this content is in the transcript", and only consumption
|
|
1635
|
+
// evidence (`pi.on("message_end")` for this hidden wake message, see
|
|
1636
|
+
// the handler below) knows that. Recording the injection instead would
|
|
1637
|
+
// let an update whose run never carried it out — a failed turn, an
|
|
1638
|
+
// abort, a message Pi dropped — become unrecoverable in this session,
|
|
1639
|
+
// which is the lock-out this path is meant to remove. The in-flight
|
|
1640
|
+
// state is exactly what keeps it out of the transcript twice in the
|
|
1641
|
+
// meantime: `alreadyPresented` reports "awaiting".
|
|
1330
1642
|
} catch {
|
|
1331
1643
|
state.deliveredBatch = undefined;
|
|
1332
1644
|
state.wakeRequested = false;
|
|
1333
|
-
// Nothing was
|
|
1334
|
-
//
|
|
1335
|
-
//
|
|
1336
|
-
//
|
|
1337
|
-
// elapsed deadline from letting a later, unrelated wake bypass the busy
|
|
1338
|
-
// gate.
|
|
1339
|
-
endWakeDeferral();
|
|
1645
|
+
// Nothing was sent, so nothing is awaiting consumption on its account
|
|
1646
|
+
// either: the ids only enter the awaiting set once Pi accepted the
|
|
1647
|
+
// message. Nothing was delivered, so the event stays pending and the
|
|
1648
|
+
// next wake retries it.
|
|
1340
1649
|
}
|
|
1341
1650
|
};
|
|
1342
1651
|
void startWake();
|
|
@@ -1528,7 +1837,6 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1528
1837
|
// already acked events are covered by the server cursor
|
|
1529
1838
|
// (pruneAcknowledgedEvents below).
|
|
1530
1839
|
state.deliveredBatch = undefined;
|
|
1531
|
-
state.wakeContext = undefined;
|
|
1532
1840
|
}
|
|
1533
1841
|
// Otherwise the batch's wake turn is still in flight: keep it so the
|
|
1534
1842
|
// settlement acknowledges it and the events are not re-presented.
|
|
@@ -1616,23 +1924,7 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1616
1924
|
state.currentScope.workspaceId === message.params.workspaceId
|
|
1617
1925
|
) {
|
|
1618
1926
|
const next = message.params.context ?? undefined;
|
|
1619
|
-
const retain = (snapshot: AgentWorkspaceContextSnapshot | undefined) =>
|
|
1620
|
-
snapshot
|
|
1621
|
-
? {
|
|
1622
|
-
...snapshot,
|
|
1623
|
-
agents: snapshot.agents.flatMap((agent) => {
|
|
1624
|
-
const nextAgents =
|
|
1625
|
-
next?.agents.filter((candidate) => candidate.paneId === agent.paneId) ?? [];
|
|
1626
|
-
if (nextAgents.length === 0) return [];
|
|
1627
|
-
const nextAgent = nextAgents.find(
|
|
1628
|
-
(candidate) => !agent.id || !candidate.id || agent.id === candidate.id,
|
|
1629
|
-
);
|
|
1630
|
-
return nextAgent ? [nextAgent] : [];
|
|
1631
|
-
}),
|
|
1632
|
-
}
|
|
1633
|
-
: undefined;
|
|
1634
1927
|
state.latestContext = next;
|
|
1635
|
-
state.pinnedContext = retain(state.pinnedContext);
|
|
1636
1928
|
}
|
|
1637
1929
|
return;
|
|
1638
1930
|
}
|
|
@@ -1817,11 +2109,10 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1817
2109
|
pi.on("message_end", (event: Record<string, unknown>) => {
|
|
1818
2110
|
const message = record(event.message);
|
|
1819
2111
|
if (message.role === "custom" && message.customType === WAKE_CONTEXT_CUSTOM_TYPE) {
|
|
1820
|
-
// Consumption evidence: the wake content reached the transcript, so
|
|
1821
|
-
//
|
|
1822
|
-
//
|
|
1823
|
-
//
|
|
1824
|
-
// whatever conclusion the turn reached.
|
|
2112
|
+
// Consumption evidence: the wake content reached the transcript, so the
|
|
2113
|
+
// turn that carried it out is confirmed. The ids the message presented are
|
|
2114
|
+
// now authorised for acknowledgement on their own, whatever conclusion the
|
|
2115
|
+
// turn reached.
|
|
1825
2116
|
const consumedEventIds = wakeConsumedEventIds(message);
|
|
1826
2117
|
if (consumedEventIds.length === 0) {
|
|
1827
2118
|
// Nothing may be confirmed from a message whose evidence cannot be read
|
|
@@ -1836,6 +2127,9 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1836
2127
|
}
|
|
1837
2128
|
for (const eventId of consumedEventIds) {
|
|
1838
2129
|
state.wakeAwaitingConsumption.delete(eventId);
|
|
2130
|
+
// The delivery is settled (carried out, or dead-lettered and now
|
|
2131
|
+
// re-delivered): it owes no further drive and no further retry.
|
|
2132
|
+
state.wakeRetryableEventIds.delete(eventId);
|
|
1839
2133
|
// The content reached the transcript, so this id counts as presented from
|
|
1840
2134
|
// now on: a daemon redelivery of it (it is still unacknowledged whenever
|
|
1841
2135
|
// its batch was already dropped) must not inject the same update again.
|
|
@@ -1847,7 +2141,6 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1847
2141
|
// Only an id the daemon can still be told about is worth remembering.
|
|
1848
2142
|
if (state.unackedDelivered.has(eventId)) state.wakeConsumptionObserved.add(eventId);
|
|
1849
2143
|
}
|
|
1850
|
-
if (state.wakeAwaitingConsumption.size === 0) state.wakeContinuationAttempts = 0;
|
|
1851
2144
|
}
|
|
1852
2145
|
if (message.role !== "assistant") return;
|
|
1853
2146
|
const stopReason = stringValue(message.stopReason);
|
|
@@ -1886,69 +2179,45 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1886
2179
|
// the next user message.
|
|
1887
2180
|
if (state.runActive) return;
|
|
1888
2181
|
state.runActive = true;
|
|
1889
|
-
state.pinnedContext =
|
|
1890
|
-
state.isOrchestrator && !state.deliveredBatch?.herdsmanTriggered
|
|
1891
|
-
? state.latestContext
|
|
1892
|
-
: undefined;
|
|
1893
2182
|
});
|
|
1894
2183
|
|
|
1895
2184
|
pi.on("context", (event: { messages: PiAgentMessage[] }) => {
|
|
2185
|
+
// The wake content rides the single follow-up track and is never pinned
|
|
2186
|
+
// into the context here, so this hook injects nothing at all. What it
|
|
2187
|
+
// still does is drop herdsman's own context/wake entries from the
|
|
2188
|
+
// incoming list: a session replayed from disk can carry an old
|
|
2189
|
+
// `[HERDSMAN AGENT CONTEXT]` preview (its truncated one-line reports
|
|
2190
|
+
// repeatedly read as cut-off reports, and live board state is queried
|
|
2191
|
+
// through herdr), so those entries are filtered out instead of being
|
|
2192
|
+
// re-served to the model. The wake message itself
|
|
2193
|
+
// (`herdsman-wake-context`, the full report text) is not filtered - it is
|
|
2194
|
+
// the evidence the turn acts on.
|
|
1896
2195
|
const messages = event.messages.filter((message) => !isNormalHerdsmanContext(message));
|
|
1897
|
-
|
|
1898
|
-
const snapshot = state.pinnedContext;
|
|
1899
|
-
if (snapshot && snapshot.agents.length > 0) {
|
|
1900
|
-
additions.push({
|
|
1901
|
-
content: formatHiddenAgentContext({
|
|
1902
|
-
agents: snapshot.agents,
|
|
1903
|
-
workspaceId: snapshot.workspaceId,
|
|
1904
|
-
}),
|
|
1905
|
-
customType: "herdsman-agent-context",
|
|
1906
|
-
display: false,
|
|
1907
|
-
role: "custom",
|
|
1908
|
-
timestamp: Date.now(),
|
|
1909
|
-
});
|
|
1910
|
-
}
|
|
1911
|
-
// A wake queued for a busy orchestrator is not allowed to interrupt the
|
|
1912
|
-
// running tool chain, so its content is additionally pinned to the current
|
|
1913
|
-
// context: the orchestrator sees the child-agent outcome in this turn
|
|
1914
|
-
// without a triggered follow-up. The entry is dropped again by
|
|
1915
|
-
// isNormalHerdsmanContext, so at most one copy is present per call.
|
|
1916
|
-
//
|
|
1917
|
-
// `eventIds` mirrors what this turn actually presents (the freshly injected
|
|
1918
|
-
// outcomes): events carried over in the delivery queue were already shown
|
|
1919
|
-
// to the orchestrator in the turn that presented them, so they are not
|
|
1920
|
-
// re-listed here. The queued follow-up message itself keeps the wider
|
|
1921
|
-
// `details.eventIds` set (everything still unconfirmed).
|
|
1922
|
-
//
|
|
1923
|
-
// The pin is not a second delivery of the update: it makes the same queued
|
|
1924
|
-
// copy visible early to the turn that is running, it is not written to the
|
|
1925
|
-
// transcript, it is never acknowledged on its own, and it is not consumption
|
|
1926
|
-
// evidence — only the hidden wake message's `message_end` is. So it may not be
|
|
1927
|
-
// dropped in the name of "one copy only" either: it is the only way this turn
|
|
1928
|
-
// ever sees the update.
|
|
1929
|
-
const queuedWake = state.wakeContext;
|
|
1930
|
-
if (queuedWake) {
|
|
1931
|
-
additions.push({
|
|
1932
|
-
content: queuedWake.content,
|
|
1933
|
-
customType: "herdsman-wake-queued",
|
|
1934
|
-
details: { eventIds: queuedWake.eventIds },
|
|
1935
|
-
display: false,
|
|
1936
|
-
role: "custom",
|
|
1937
|
-
timestamp: Date.now(),
|
|
1938
|
-
});
|
|
1939
|
-
}
|
|
1940
|
-
return additions.length === 0 ? { messages } : { messages: [...messages, ...additions] };
|
|
2196
|
+
return { messages };
|
|
1941
2197
|
});
|
|
1942
2198
|
|
|
1943
2199
|
pi.on("agent_settled", async (_event: unknown, ctx: PiContext) => {
|
|
1944
2200
|
state.runActive = false;
|
|
1945
|
-
|
|
1946
|
-
//
|
|
1947
|
-
//
|
|
1948
|
-
|
|
1949
|
-
// the
|
|
1950
|
-
//
|
|
1951
|
-
|
|
2201
|
+
// The busy period this settlement ends is over, so the poll ladder starts
|
|
2202
|
+
// from its first rung again for whatever turn comes next (see
|
|
2203
|
+
// `WAKE_BUSY_BACKOFF_MS`).
|
|
2204
|
+
state.wakeBusySpinRung = 0;
|
|
2205
|
+
// The settlement is the signal a deferral has been waiting for, so the
|
|
2206
|
+
// pending spin is dropped: the wake decision below is taken now instead of
|
|
2207
|
+
// one `WAKE_BUSY_SPIN_MS` tick later. A pass that is already being injected
|
|
2208
|
+
// (`wakeRequested`) keeps its timer — clearing it would drop the injection.
|
|
2209
|
+
if (state.wakeTimer && !state.wakeRequested) {
|
|
2210
|
+
const pendingDeferral = state.wakeTimer;
|
|
2211
|
+
state.wakeTimer = undefined;
|
|
2212
|
+
clearTimeout(pendingDeferral);
|
|
2213
|
+
}
|
|
2214
|
+
// A wake whose content never reached the transcript gets no run of its own:
|
|
2215
|
+
// there is deliberately no empty-marker continuation. The settlement writes
|
|
2216
|
+
// those ids off locally (dead-letter, no acknowledgement) and the next
|
|
2217
|
+
// `scheduleWake` re-presents them with the real update content, or a later
|
|
2218
|
+
// settlement confirms them once their copy finally drains.
|
|
2219
|
+
const unconsumed = [...state.wakeAwaitingConsumption].sort((left, right) => left - right);
|
|
2220
|
+
if (unconsumed.length > 0) writeOffStrandedWakeDelivery(unconsumed, ctx);
|
|
1952
2221
|
const failBatch = () => {
|
|
1953
2222
|
ctx.ui.notify?.(
|
|
1954
2223
|
"Herdsman couldn’t acknowledge agent updates · updates remain pending",
|
|
@@ -1958,9 +2227,6 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1958
2227
|
const finishBatch = () => {
|
|
1959
2228
|
state.ackInFlight = false;
|
|
1960
2229
|
state.wakeDeferredUntilSettled = false;
|
|
1961
|
-
state.wakeDeferredSince = undefined;
|
|
1962
|
-
state.wakeForcedRelease = false;
|
|
1963
|
-
state.wakeContext = undefined;
|
|
1964
2230
|
setHerdsmanUi(ctx);
|
|
1965
2231
|
scheduleWake(ctx);
|
|
1966
2232
|
};
|
|
@@ -1997,10 +2263,14 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
1997
2263
|
|
|
1998
2264
|
const stillOwner =
|
|
1999
2265
|
state.isOrchestrator && state.currentScope?.terminalId === batch.ownerTerminalId;
|
|
2000
|
-
const { deliveryQueue, confirmablePrefix, blockedByMissingTurn
|
|
2001
|
-
|
|
2266
|
+
const { deliveryQueue, confirmablePrefix, blockedByMissingTurn } = confirmableDeliveryPrefix(
|
|
2267
|
+
batch.assistantFinalSucceeded || batch.abortedByUser,
|
|
2268
|
+
);
|
|
2002
2269
|
const reachable = stillOwner && state.client !== undefined && state.connected;
|
|
2003
|
-
|
|
2270
|
+
// Every id awaiting consumption was dead-lettered at the top of this
|
|
2271
|
+
// handler, so nothing holds the cursor open any more and the batch can be
|
|
2272
|
+
// dropped: a later wake is not gated by it.
|
|
2273
|
+
state.deliveredBatch = undefined;
|
|
2004
2274
|
state.ackInFlight = true;
|
|
2005
2275
|
if (!reachable) {
|
|
2006
2276
|
// Unreachable (ownership gone, or a disconnect): attempting an
|
|
@@ -2028,62 +2298,6 @@ export function createHerdsmanPiExtension(options: ExtensionOptions = {}) {
|
|
|
2028
2298
|
|
|
2029
2299
|
export default createHerdsmanPiExtension();
|
|
2030
2300
|
|
|
2031
|
-
export function formatHiddenAgentContext(input: {
|
|
2032
|
-
agents: AgentContextListItem[];
|
|
2033
|
-
workspaceId: string;
|
|
2034
|
-
}): string {
|
|
2035
|
-
return [
|
|
2036
|
-
"[HERDSMAN AGENT CONTEXT]",
|
|
2037
|
-
`Current Herdr workspace: ${input.workspaceId}`,
|
|
2038
|
-
...input.agents.map((agent) => {
|
|
2039
|
-
const history = agent.history ?? {};
|
|
2040
|
-
const identity = agentIdentityLabel({
|
|
2041
|
-
agent: agent.agent ?? "unknown",
|
|
2042
|
-
name: agent.name,
|
|
2043
|
-
});
|
|
2044
|
-
const paneId = agent.paneId ?? "unknown";
|
|
2045
|
-
const status = agent.agentStatus ?? "unknown";
|
|
2046
|
-
const prefix = `- ${identity} ${paneId} ${status}`;
|
|
2047
|
-
|
|
2048
|
-
const rawAgent = record(agent);
|
|
2049
|
-
const tabTitleCandidate =
|
|
2050
|
-
stringValue(rawAgent.terminalTitle) ?? stringValue(rawAgent.label);
|
|
2051
|
-
const tabTitleCleaned =
|
|
2052
|
-
tabTitleCandidate !== null ? sanitizeAndCleanContextText(tabTitleCandidate) : "";
|
|
2053
|
-
const tabTitle =
|
|
2054
|
-
tabTitleCleaned.length > 0 ? truncateSummary(tabTitleCleaned, 60) : null;
|
|
2055
|
-
|
|
2056
|
-
const rawHistory = record(history);
|
|
2057
|
-
const lastAssistantRecord = record(history.lastAssistantMessage);
|
|
2058
|
-
const timeCandidate =
|
|
2059
|
-
lastAssistantRecord.timestamp ??
|
|
2060
|
-
history.updatedAt ??
|
|
2061
|
-
rawHistory.updatedAt ??
|
|
2062
|
-
rawAgent.updatedAt ??
|
|
2063
|
-
rawAgent.time;
|
|
2064
|
-
const formattedTime = formatTimestamp(timeCandidate);
|
|
2065
|
-
|
|
2066
|
-
const assistantRaw = history.lastAssistantMessage?.text;
|
|
2067
|
-
const assistantCleaned =
|
|
2068
|
-
assistantRaw !== undefined && assistantRaw !== null
|
|
2069
|
-
? sanitizeAndCleanContextText(assistantRaw)
|
|
2070
|
-
: "";
|
|
2071
|
-
const assistantSummary =
|
|
2072
|
-
assistantCleaned.length > 0 ? truncateSummary(assistantCleaned, 100) : null;
|
|
2073
|
-
|
|
2074
|
-
const segments = [
|
|
2075
|
-
prefix,
|
|
2076
|
-
tabTitle,
|
|
2077
|
-
formattedTime,
|
|
2078
|
-
assistantSummary,
|
|
2079
|
-
].filter((segment): segment is string => segment !== null && segment.length > 0);
|
|
2080
|
-
|
|
2081
|
-
return segments.join(" · ");
|
|
2082
|
-
}),
|
|
2083
|
-
"Use herdsman agent get/read if details are needed.",
|
|
2084
|
-
].join("\n");
|
|
2085
|
-
}
|
|
2086
|
-
|
|
2087
2301
|
export function formatHiddenAgentUpdates(events: AgentEventWireRecord[]): string {
|
|
2088
2302
|
return [
|
|
2089
2303
|
"[HERDSMAN AGENT UPDATES]",
|
|
@@ -2106,6 +2320,10 @@ export function formatHiddenAgentUpdates(events: AgentEventWireRecord[]): string
|
|
|
2106
2320
|
function isNormalHerdsmanContext(message: PiAgentMessage): boolean {
|
|
2107
2321
|
return (
|
|
2108
2322
|
message.customType === "herdsman-agent-context" ||
|
|
2323
|
+
// Defensive for session replay: older versions pinned a queued wake to the
|
|
2324
|
+
// context hook under this customType. A replayed history may still carry
|
|
2325
|
+
// those entries, and they are herdsman's own, so they are dropped from the
|
|
2326
|
+
// incoming list like the agent-context ones (see the context hook).
|
|
2109
2327
|
message.customType === "herdsman-wake-queued" ||
|
|
2110
2328
|
contentIncludesMarker(message.content, "[HERDSMAN AGENT CONTEXT]")
|
|
2111
2329
|
);
|
|
@@ -2185,32 +2403,3 @@ function sanitizeAndCleanContextText(value: string): string {
|
|
|
2185
2403
|
return cleanContextText(sanitized);
|
|
2186
2404
|
}
|
|
2187
2405
|
|
|
2188
|
-
function truncateSummary(value: string, limit = 100): string {
|
|
2189
|
-
if (value.length <= limit) return value;
|
|
2190
|
-
const maxSearch = Math.min(value.length, limit + 8);
|
|
2191
|
-
const searchSlice = value.slice(0, maxSearch);
|
|
2192
|
-
const lastSpaceIndex = searchSlice.search(/\s[^\s]*$/);
|
|
2193
|
-
if (lastSpaceIndex >= limit - 20) {
|
|
2194
|
-
return `${value.slice(0, lastSpaceIndex)}…`;
|
|
2195
|
-
}
|
|
2196
|
-
return `${value.slice(0, limit)}…`;
|
|
2197
|
-
}
|
|
2198
|
-
|
|
2199
|
-
function formatTimestamp(value: unknown): string | null {
|
|
2200
|
-
if (typeof value === "string") {
|
|
2201
|
-
const trimmed = value.trim();
|
|
2202
|
-
if (/^\d{2}:\d{2}:\d{2}$/.test(trimmed)) return trimmed;
|
|
2203
|
-
if (/^\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(trimmed)) return trimmed;
|
|
2204
|
-
}
|
|
2205
|
-
if (!value) return null;
|
|
2206
|
-
const date = value instanceof Date ? value : new Date(value as string | number);
|
|
2207
|
-
if (Number.isNaN(date.getTime())) return null;
|
|
2208
|
-
const pad = (n: number) => String(n).padStart(2, "0");
|
|
2209
|
-
const month = pad(date.getMonth() + 1);
|
|
2210
|
-
const day = pad(date.getDate());
|
|
2211
|
-
const hours = pad(date.getHours());
|
|
2212
|
-
const minutes = pad(date.getMinutes());
|
|
2213
|
-
const seconds = pad(date.getSeconds());
|
|
2214
|
-
return `${month}-${day} ${hours}:${minutes}:${seconds}`;
|
|
2215
|
-
}
|
|
2216
|
-
|