@cohortapp/agent-sdk 2.18.36 → 2.18.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/runbooks/cadence-requeue-storm-recovery.md +186 -0
- package/lib/assurance/first-reply.mjs +16 -0
- package/lib/assurance/notice-plane.mjs +218 -0
- package/lib/assurance/notice-voice.mjs +35 -2
- package/lib/budget-escalate.mjs +42 -4
- package/lib/cadence-bus.mjs +359 -46
- package/lib/comms/machine-status.mjs +377 -0
- package/package.json +1 -1
- package/scripts/ci/check-durable-write-seam.mjs +3 -3
- package/scripts/daemon/assurance.mjs +391 -15
- package/scripts/daemon/cadence-consumer.mjs +165 -9
- package/scripts/daemon/deliver.mjs +104 -0
package/lib/cadence-bus.mjs
CHANGED
|
@@ -115,6 +115,21 @@ const DEFAULT_PROCESSED_RETENTION_DAYS = 14;
|
|
|
115
115
|
const DEFAULT_TERMINAL_KEEP = 500; // newest N files kept in dlq/ & failed/
|
|
116
116
|
const DEFAULT_QUEUE_MAX_BYTES = 5 * 1024 * 1024; // queue.jsonl rotate threshold (~5MB)
|
|
117
117
|
const DEFAULT_LOG_RETENTION_DAYS = 14; // logs/cadence-bus/<date>.jsonl
|
|
118
|
+
// Candidates examined per claim before the scan gives up (and returns null —
|
|
119
|
+
// never claims an unexamined tick; see claimNextTick). Sized well above the
|
|
120
|
+
// number of distinct cadences on a seat (tens) so the cap is a safety net
|
|
121
|
+
// against a pathological inbox, not a thing the normal path meets: with the
|
|
122
|
+
// pending-dedupe holding each (cadence, lane) to one pending tick, the real
|
|
123
|
+
// scan length is the cadence count, not the queue depth.
|
|
124
|
+
const DEFAULT_CLAIM_SCAN_LIMIT = 2048;
|
|
125
|
+
|
|
126
|
+
// Hard ceiling on a deferral hold (W19 rework). The circuit gate derives its
|
|
127
|
+
// hold from its own `retry_at`, which is bounded today by the 30 min circuit
|
|
128
|
+
// window — but a retry_at taken from a provider Retry-After, a forward clock
|
|
129
|
+
// jump mid-write, or a corrupted number would park a tick for hours or days,
|
|
130
|
+
// and claimNextTick would then skip it silently forever. Clamped at the one
|
|
131
|
+
// chokepoint every hold passes through, so no call site can bypass it.
|
|
132
|
+
export const MAX_DEFER_HOLD_MS = 60 * 60_000;
|
|
118
133
|
|
|
119
134
|
// ---------------------------------------------------------------------------
|
|
120
135
|
// Path resolution
|
|
@@ -352,13 +367,20 @@ function readSchedule(paths) {
|
|
|
352
367
|
* @param {string} cadence
|
|
353
368
|
* @param {number} intervalMs
|
|
354
369
|
* @param {number} now
|
|
370
|
+
* @param {{lockHeld?: boolean}} [opts] `lockHeld` when the CALLER already holds
|
|
371
|
+
* the schedule lock and will release it. enqueueTick does that so the gate,
|
|
372
|
+
* the pending-dedupe and the inbox write happen inside one critical section:
|
|
373
|
+
* this function used to acquire and release internally, which left the dedupe
|
|
374
|
+
* that follows it unsynchronised — two concurrent enqueues could each see an
|
|
375
|
+
* empty inbox/ and both write.
|
|
355
376
|
*/
|
|
356
|
-
function scheduleGate(paths, cadence, intervalMs, now) {
|
|
377
|
+
function scheduleGate(paths, cadence, intervalMs, now, opts = {}) {
|
|
357
378
|
if (!(Number.isFinite(intervalMs) && intervalMs > 0)) {
|
|
358
379
|
return { admit: true, reason: "no-interval", kind: "passthrough" };
|
|
359
380
|
}
|
|
360
381
|
mkdirSync(paths.base, { recursive: true });
|
|
361
|
-
|
|
382
|
+
const ownLock = !opts.lockHeld;
|
|
383
|
+
if (ownLock && !acquireScheduleLock(paths, now)) {
|
|
362
384
|
return { admit: true, reason: "lock-contended-fail-open", kind: "admit" };
|
|
363
385
|
}
|
|
364
386
|
try {
|
|
@@ -399,7 +421,7 @@ function scheduleGate(paths, cadence, intervalMs, now) {
|
|
|
399
421
|
}
|
|
400
422
|
return result;
|
|
401
423
|
} finally {
|
|
402
|
-
releaseScheduleLock(paths);
|
|
424
|
+
if (ownLock) releaseScheduleLock(paths);
|
|
403
425
|
}
|
|
404
426
|
}
|
|
405
427
|
|
|
@@ -418,10 +440,75 @@ function scheduleGate(paths, cadence, intervalMs, now) {
|
|
|
418
440
|
* enables the at-most-once cron gate
|
|
419
441
|
* (grace + stale fast-forward). Absent →
|
|
420
442
|
* legacy behaviour, every call enqueues.
|
|
443
|
+
* @param {string} [input.dedupeLane] Opt this enqueue into pending-dedupe on the
|
|
444
|
+
* named lane: at most one tick carrying
|
|
445
|
+
* `${cadence}|${lane}` is pending at a
|
|
446
|
+
* time. Defaults to DEDUPE_LANE_INTERVAL
|
|
447
|
+
* when `intervalMs` is given, and to NO
|
|
448
|
+
* dedupe otherwise.
|
|
421
449
|
* @param {number} [input.now] Injectable clock (ms) for tests.
|
|
422
450
|
* @param {string} [input.agentRoot] Override AGENT_ROOT for tests.
|
|
423
451
|
* @returns {{ id: string, path: string, fallbackOnly: boolean, skipped?: string }}
|
|
424
452
|
*/
|
|
453
|
+
/**
|
|
454
|
+
* The dedupe lane of an enqueue — the second half of "the same event".
|
|
455
|
+
*
|
|
456
|
+
* A LANE, NOT JUST A CADENCE (W19 rework). The first cut keyed the dedupe on
|
|
457
|
+
* the presence of `intervalMs`, which exempted the `handoff-timeout` re-enqueue
|
|
458
|
+
* by construction. MEASURED read-only on the live seat 2026-10-01: 88 pending
|
|
459
|
+
* ticks, 87 of them `meeting-action-capture`, split 40 `source:"launchd"` and
|
|
460
|
+
* 47 `source:"handoff-timeout"`. So the interval dedupe collapsed 46% of the
|
|
461
|
+
* depth for the one cadence that caused the incident and the other 54% kept
|
|
462
|
+
* accumulating without bound — and 47 pending copies of "the handoff for this
|
|
463
|
+
* cadence timed out, run it again" cannot each do distinct work.
|
|
464
|
+
*
|
|
465
|
+
* Lanes are kept SEPARATE rather than merged into one key per cadence. A
|
|
466
|
+
* pending periodic tick and a pending handoff-timeout requeue carry different
|
|
467
|
+
* intent, and the rule that matters for safety is that a low-intent pending
|
|
468
|
+
* tick must never suppress a different lane's new one. Two pending copies per
|
|
469
|
+
* cadence is the honest bound; it is also immaterial now that the hold is
|
|
470
|
+
* per-cadence and the requeue rate no longer depends on depth at all.
|
|
471
|
+
*/
|
|
472
|
+
export const DEDUPE_LANE_INTERVAL = "interval";
|
|
473
|
+
export const DEDUPE_LANE_HANDOFF_TIMEOUT = "handoff-timeout";
|
|
474
|
+
|
|
475
|
+
export function dedupeKeyFor(cadence, lane) {
|
|
476
|
+
return `${cadence}|${lane}`;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Is an enqueue carrying `key` already pending — unclaimed in inbox/, or
|
|
481
|
+
* claimed and in flight? Returns `{ id, where }` for the first match, else null.
|
|
482
|
+
*
|
|
483
|
+
* Matches on `metadata.dedupeKey`, which `requeueTick` carries through a
|
|
484
|
+
* re-key (it spreads the prior metadata), so a tick deferred fifty times is
|
|
485
|
+
* still recognised as the pending instance of its lane.
|
|
486
|
+
*
|
|
487
|
+
* inbox/ AND claimed/, never processed/, failed/ or dlq/. A claimed tick is
|
|
488
|
+
* doing this lane's work; a processed one is done and the next boundary must
|
|
489
|
+
* run; a dead-lettered one will never run and must not block a fresh tick.
|
|
490
|
+
*
|
|
491
|
+
* Reads the candidate files; enqueues are rare (hundreds a day against ~8
|
|
492
|
+
* claims a second), so the cost is irrelevant next to what it prevents. Never
|
|
493
|
+
* throws: an unreadable bus means "nothing pending", which admits the tick —
|
|
494
|
+
* fail OPEN, because the invariant is never to drop work.
|
|
495
|
+
*/
|
|
496
|
+
function findPendingDedupe(paths, key) {
|
|
497
|
+
for (const [dir, where] of [[paths.inbox, "inbox"], [paths.claimed, "claimed"]]) {
|
|
498
|
+
let names;
|
|
499
|
+
try { names = readdirSync(dir).filter((n) => n.endsWith(".json")); } catch { continue; }
|
|
500
|
+
for (const n of names) {
|
|
501
|
+
try {
|
|
502
|
+
const ev = JSON.parse(readFileSync(join(dir, n), "utf-8"));
|
|
503
|
+
if (ev && ev.metadata && ev.metadata.dedupeKey === key) {
|
|
504
|
+
return { id: ev.id || n.slice(0, -5), where };
|
|
505
|
+
}
|
|
506
|
+
} catch { /* unreadable candidate is not a reason to suppress */ }
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return null;
|
|
510
|
+
}
|
|
511
|
+
|
|
425
512
|
export function enqueueTick(input = {}) {
|
|
426
513
|
if (!input || typeof input !== "object") {
|
|
427
514
|
throw new TypeError("enqueueTick: input must be an object");
|
|
@@ -476,58 +563,164 @@ export function enqueueTick(input = {}) {
|
|
|
476
563
|
}
|
|
477
564
|
}
|
|
478
565
|
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
//
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
566
|
+
// --- The tail of an admitted enqueue -----------------------------------
|
|
567
|
+
// Factored out so the deduped lanes can run it INSIDE the schedule lock,
|
|
568
|
+
// which is what makes "check pending, then write" one critical section.
|
|
569
|
+
function commit() {
|
|
570
|
+
// Always write the audit row first. If inbox write fails the event is
|
|
571
|
+
// still recoverable from queue.jsonl.
|
|
572
|
+
appendFallbackQueue(paths, event);
|
|
573
|
+
|
|
574
|
+
let fallbackOnly = false;
|
|
575
|
+
try {
|
|
576
|
+
writeJsonAtomic(target, event);
|
|
577
|
+
} catch (err) {
|
|
578
|
+
fallbackOnly = true;
|
|
487
579
|
logBusEvent(paths.agentRoot, {
|
|
488
|
-
level: "
|
|
489
|
-
stage: "
|
|
580
|
+
level: "error",
|
|
581
|
+
stage: "enqueue_inbox_failed",
|
|
490
582
|
cadence,
|
|
491
|
-
reason: gate.reason,
|
|
492
|
-
kind: gate.kind,
|
|
493
583
|
id: event.id,
|
|
584
|
+
error: err.message,
|
|
494
585
|
});
|
|
495
|
-
return { id: event.id, path: null, fallbackOnly: true, skipped: gate.reason };
|
|
496
586
|
}
|
|
497
|
-
// Annotate the admitted event so the catch-up case is visible downstream.
|
|
498
|
-
if (gate.kind === "catch-up") {
|
|
499
|
-
event.metadata = { ...(event.metadata || {}), schedule_kind: "catch-up", periods_missed: gate.periodsMissed };
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
587
|
|
|
503
|
-
// Always write the audit row first. If inbox write fails the event is
|
|
504
|
-
// still recoverable from queue.jsonl.
|
|
505
|
-
appendFallbackQueue(paths, event);
|
|
506
|
-
|
|
507
|
-
let fallbackOnly = false;
|
|
508
|
-
try {
|
|
509
|
-
writeJsonAtomic(target, event);
|
|
510
|
-
} catch (err) {
|
|
511
|
-
fallbackOnly = true;
|
|
512
588
|
logBusEvent(paths.agentRoot, {
|
|
513
|
-
level: "
|
|
514
|
-
stage: "
|
|
589
|
+
level: "info",
|
|
590
|
+
stage: fallbackOnly ? "enqueued_fallback" : "enqueued",
|
|
515
591
|
cadence,
|
|
592
|
+
source: event.source,
|
|
593
|
+
priority: event.priority,
|
|
516
594
|
id: event.id,
|
|
517
|
-
error: err.message,
|
|
518
595
|
});
|
|
596
|
+
|
|
597
|
+
return { id: event.id, path: fallbackOnly ? null : target, fallbackOnly };
|
|
519
598
|
}
|
|
520
599
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
600
|
+
// --- Which dedupe lane is this enqueue on? ------------------------------
|
|
601
|
+
// EXPLICIT, NOT INFERRED (W19 rework). The first cut inferred the lane from
|
|
602
|
+
// the presence of `intervalMs`, which silently exempted every other
|
|
603
|
+
// system-generated requeue — above all `handoff-timeout`, which was 47 of
|
|
604
|
+
// the live seat's 88 pending ticks. A lane is now named by the caller
|
|
605
|
+
// (`input.dedupeLane`), with the interval lane defaulted so the plist path
|
|
606
|
+
// keeps its behaviour without knowing about lanes at all.
|
|
607
|
+
//
|
|
608
|
+
// A lane that is NOT named is NOT deduped, and that is the safe default: a
|
|
609
|
+
// manual `maestro cadence run`, an escalation-driven enqueue and an upgrade
|
|
610
|
+
// tick each carry intent no pending tick can satisfy, and suppressing one
|
|
611
|
+
// would DROP REAL WORK. Opting in is a decision; opting out is the absence
|
|
612
|
+
// of one.
|
|
613
|
+
const lane = typeof input.dedupeLane === "string" && input.dedupeLane.trim()
|
|
614
|
+
? input.dedupeLane.trim()
|
|
615
|
+
: (Number.isFinite(input.intervalMs) && input.intervalMs > 0 ? DEDUPE_LANE_INTERVAL : null);
|
|
616
|
+
const dedupeKey = lane ? dedupeKeyFor(cadence, lane) : null;
|
|
617
|
+
if (dedupeKey) {
|
|
618
|
+
event.metadata = { ...(event.metadata || {}), dedupeKey };
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
if (!dedupeKey) return commit();
|
|
622
|
+
|
|
623
|
+
const nowMs = typeof input.now === "number" ? input.now : Date.now();
|
|
624
|
+
|
|
625
|
+
// ONE CRITICAL SECTION: schedule gate → pending-dedupe → inbox write.
|
|
626
|
+
// scheduleGate used to acquire and release its own lock before returning,
|
|
627
|
+
// which left the dedupe that follows it unsynchronised: two concurrent
|
|
628
|
+
// enqueues for one cadence (a launchd fire racing an org-push wake) could
|
|
629
|
+
// each see inbox/ and claimed/ empty and both write.
|
|
630
|
+
//
|
|
631
|
+
// If the lock cannot be taken we proceed anyway (the gate's own fail-open
|
|
632
|
+
// rule: never drop work, a duplicate is cheaper than a missed cadence), so
|
|
633
|
+
// the stated bound is "at most one pending tick per lane, except under lock
|
|
634
|
+
// contention, where a second copy is preferred to a lost one".
|
|
635
|
+
mkdirSync(paths.base, { recursive: true });
|
|
636
|
+
const locked = acquireScheduleLock(paths, nowMs);
|
|
637
|
+
try {
|
|
638
|
+
// At-most-once cron gate (WS4). Only engages when intervalMs is supplied by
|
|
639
|
+
// the caller (plist). Suppressed ticks are still audited so we can see what
|
|
640
|
+
// launchd fired, but nothing lands in inbox/.
|
|
641
|
+
if (Number.isFinite(input.intervalMs) && input.intervalMs > 0) {
|
|
642
|
+
const gate = scheduleGate(paths, cadence, input.intervalMs, nowMs, { lockHeld: locked });
|
|
643
|
+
if (!gate.admit) {
|
|
644
|
+
appendFallbackQueue(paths, { ...event, _suppressed: gate.reason });
|
|
645
|
+
logBusEvent(paths.agentRoot, {
|
|
646
|
+
level: "info",
|
|
647
|
+
stage: "enqueue_suppressed",
|
|
648
|
+
cadence,
|
|
649
|
+
reason: gate.reason,
|
|
650
|
+
kind: gate.kind,
|
|
651
|
+
id: event.id,
|
|
652
|
+
});
|
|
653
|
+
return { id: event.id, path: null, fallbackOnly: true, skipped: gate.reason };
|
|
654
|
+
}
|
|
655
|
+
// Annotate the admitted event so the catch-up case is visible downstream.
|
|
656
|
+
if (gate.kind === "catch-up") {
|
|
657
|
+
event.metadata = { ...(event.metadata || {}), schedule_kind: "catch-up", periods_missed: gate.periodsMissed };
|
|
658
|
+
}
|
|
659
|
+
}
|
|
529
660
|
|
|
530
|
-
|
|
661
|
+
// PENDING-DEDUPE (W19) — the amplifier.
|
|
662
|
+
//
|
|
663
|
+
// ORDER: AFTER the schedule gate, deliberately. The gate's job is "has an
|
|
664
|
+
// interval boundary passed"; it advances `nextRunAt` on its own clock and
|
|
665
|
+
// keeps doing so while this suppresses the write. That is what makes the
|
|
666
|
+
// semantic "run every N" rather than "run N times once you can" — on
|
|
667
|
+
// resume the cadence simply runs at its next real boundary, with no stale
|
|
668
|
+
// fast-forward to reason about. It also leaves the gate's own suppressions
|
|
669
|
+
// (double launchd fire inside the grace window) reported as themselves.
|
|
670
|
+
//
|
|
671
|
+
// THE FAULT. The schedule gate is a TIME gate: it admits one tick per
|
|
672
|
+
// interval boundary and nothing more. It says nothing about whether the
|
|
673
|
+
// PREVIOUS interval's tick ever ran — and the handoff-timeout sweep has no
|
|
674
|
+
// time gate at all. On 2026-10-01, 87 of this seat's 88 pending ticks were
|
|
675
|
+
// `meeting-action-capture`: 40 from launchd boundaries and 47 from handoff
|
|
676
|
+
// timeouts, accumulated over two days behind a gate (`cadence-in-flight`)
|
|
677
|
+
// that only ever lets one of them run. Backoff alone would have spread
|
|
678
|
+
// that storm over a longer window; only this stops it being created.
|
|
679
|
+
//
|
|
680
|
+
// THE IDENTITY, STATED AND DEFENDED. Two ticks are "the same event" when
|
|
681
|
+
// they carry the same `dedupeKey` — cadence + lane — and one of them is
|
|
682
|
+
// still pending in inbox/ or claimed/.
|
|
683
|
+
// - Cadence name, not event id, because different ids ARE the duplicate.
|
|
684
|
+
// A periodic tick carries no payload beyond "run this cadence now", and
|
|
685
|
+
// a handoff-timeout requeue none beyond "the session never acked it,
|
|
686
|
+
// run it again", so a second pending copy of either can do nothing the
|
|
687
|
+
// first will not do.
|
|
688
|
+
// - Lane, so one system lane can never suppress another's work. See the
|
|
689
|
+
// note on DEDUPE_LANE_INTERVAL for why the lanes are kept apart.
|
|
690
|
+
// - inbox/ AND claimed/, never processed/. A claimed tick (or one handed
|
|
691
|
+
// off to the session) is doing this lane's work. A processed tick is
|
|
692
|
+
// done, and the next boundary must run.
|
|
693
|
+
// Skipping an interval rather than queueing it is the CORRECT semantic for
|
|
694
|
+
// a periodic cadence — "run every N", not "run N times once you can" — and
|
|
695
|
+
// it is the same philosophy the gate's own stale-fast-forward already
|
|
696
|
+
// encodes: catch up ONCE, never replay the backlog.
|
|
697
|
+
//
|
|
698
|
+
// Known and accepted gap: a tick handed off to the session archives to
|
|
699
|
+
// processed/, so it is not "pending" here and the next interval enqueues.
|
|
700
|
+
// The consumer's own `cadenceInFlight` guard defers that one, and it now
|
|
701
|
+
// defers it WITH a per-cadence hold, so the cost is one held tick per
|
|
702
|
+
// handoff window rather than one requeue per poll.
|
|
703
|
+
const pending = findPendingDedupe(paths, dedupeKey);
|
|
704
|
+
if (pending) {
|
|
705
|
+
appendFallbackQueue(paths, { ...event, _suppressed: "pending-duplicate" });
|
|
706
|
+
logBusEvent(paths.agentRoot, {
|
|
707
|
+
level: "info",
|
|
708
|
+
stage: "enqueue_suppressed",
|
|
709
|
+
cadence,
|
|
710
|
+
reason: "pending-duplicate",
|
|
711
|
+
kind: "duplicate",
|
|
712
|
+
id: event.id,
|
|
713
|
+
dedupe_key: dedupeKey,
|
|
714
|
+
pending_id: pending.id,
|
|
715
|
+
pending_in: pending.where,
|
|
716
|
+
});
|
|
717
|
+
return { id: event.id, path: null, fallbackOnly: true, skipped: "pending-duplicate", pendingId: pending.id };
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
return commit();
|
|
721
|
+
} finally {
|
|
722
|
+
if (locked) releaseScheduleLock(paths);
|
|
723
|
+
}
|
|
531
724
|
}
|
|
532
725
|
|
|
533
726
|
function buildEvent(input, cadence) {
|
|
@@ -574,19 +767,106 @@ export function listClaimed(agentRoot) {
|
|
|
574
767
|
}
|
|
575
768
|
|
|
576
769
|
/**
|
|
577
|
-
*
|
|
770
|
+
* Read the two fields the claim path needs to decide DUE-ness off an inbox
|
|
771
|
+
* file, without claiming it: `{ cadence, notBefore }` (notBefore in epoch ms,
|
|
772
|
+
* or null when the tick carries no per-tick hold).
|
|
773
|
+
*
|
|
774
|
+
* Returns null when the file cannot be read or parsed at all — which the
|
|
775
|
+
* caller treats as CLAIMABLE. Fail OPEN: a corrupt file must still reach the
|
|
776
|
+
* claim path, which fails it properly into failed/. The hold check is never
|
|
777
|
+
* allowed to be the reason a tick stops being processed.
|
|
778
|
+
*
|
|
779
|
+
* `wantCadence:false` keeps the pre-W19 fast path — a substring reject before
|
|
780
|
+
* paying for a JSON parse, which matters because the overwhelming majority of
|
|
781
|
+
* inbox files carry no hold at all and the common case is "nothing is held".
|
|
782
|
+
*/
|
|
783
|
+
function peekHold(path, wantCadence) {
|
|
784
|
+
let raw;
|
|
785
|
+
try { raw = readFileSync(path, "utf-8"); } catch { return null; }
|
|
786
|
+
if (!wantCadence && !raw.includes('"notBefore"')) return { cadence: null, notBefore: null };
|
|
787
|
+
let ev;
|
|
788
|
+
try { ev = JSON.parse(raw); } catch { return null; }
|
|
789
|
+
if (!ev || typeof ev !== "object") return null;
|
|
790
|
+
const ms = Date.parse(ev?.metadata?.notBefore || "");
|
|
791
|
+
return {
|
|
792
|
+
cadence: typeof ev.cadence === "string" ? ev.cadence : null,
|
|
793
|
+
notBefore: Number.isFinite(ms) ? ms : null,
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
/**
|
|
798
|
+
* Atomically claim the oldest DUE inbox event for processing. Returns the parsed
|
|
578
799
|
* event payload (with absolute paths attached), or null if the inbox was
|
|
579
|
-
* empty / all candidates lost the race.
|
|
800
|
+
* empty / every candidate is held / all candidates lost the race.
|
|
580
801
|
*
|
|
581
802
|
* Other consumers racing for the same event will see ENOENT on rename and
|
|
582
803
|
* naturally move on.
|
|
804
|
+
*
|
|
805
|
+
* @param {string} agentRoot
|
|
806
|
+
* @param {object} [opts]
|
|
807
|
+
* @param {number} [opts.now]
|
|
808
|
+
* @param {Map<string,number>} [opts.heldCadences] cadence → epoch-ms deadline.
|
|
809
|
+
* THE DEPTH-INDEPENDENCE HALF OF THE HOLD (W19 rework). The per-tick
|
|
810
|
+
* `notBefore` stamp alone makes the requeue rate LINEAR IN QUEUE DEPTH:
|
|
811
|
+
* every pending tick carries its own deadline, so `depth` holds expire
|
|
812
|
+
* inside one hold window and the drain claims and re-defers all of them.
|
|
813
|
+
* MEASURED on this branch (15 min of 2 s polls, gate never opens, 30 s
|
|
814
|
+
* hold): at depth 1 the per-tick stamp alone gave 0.5/s → 0.03/s, but at
|
|
815
|
+
* depth 87 (the live inbox) 8.0/s → only 2.9/s, and at depth 240, 1,000
|
|
816
|
+
* and 15,674 it gave 8.0/s BEFORE AND AFTER — nothing at all, because the
|
|
817
|
+
* drain's 16-per-poll batch cap saturates. The incident's own write-up
|
|
818
|
+
* reports a queue 15,674 deep.
|
|
819
|
+
* One hold per CADENCE fixes that: a gate is shut for a cadence, not for a
|
|
820
|
+
* file, so one deferral covers every pending tick of that cadence. Measured
|
|
821
|
+
* with this map in place: 0.03/s at every depth from 1 to 15,674.
|
|
822
|
+
* @param {number} [opts.scanLimit] how many candidates to examine before
|
|
823
|
+
* giving up on this claim. Exhausting it STOPS the scan (returns null) — it
|
|
824
|
+
* never claims an unexamined tick, which is what the first cut did.
|
|
583
825
|
*/
|
|
584
|
-
export function claimNextTick(agentRoot) {
|
|
826
|
+
export function claimNextTick(agentRoot, opts = {}) {
|
|
585
827
|
const paths = ensureBusDirs(agentRoot);
|
|
586
828
|
const ids = listInbox(paths.agentRoot);
|
|
829
|
+
const now = typeof opts.now === "number" ? opts.now : Date.now();
|
|
830
|
+
const held = opts.heldCadences instanceof Map && opts.heldCadences.size > 0
|
|
831
|
+
? opts.heldCadences
|
|
832
|
+
: null;
|
|
833
|
+
// Scan budget. A held tick is SKIPPED WITHOUT BEING CLAIMED — that is the
|
|
834
|
+
// whole point: the old path claimed it, found the gate still shut and wrote
|
|
835
|
+
// it back, which is the 1 write + 1 unlink + 1 audit row per poll that
|
|
836
|
+
// produced the storm. Skipping costs one cached read and no writes.
|
|
837
|
+
//
|
|
838
|
+
// EXHAUSTION IS A STOP, NOT A BYPASS (W19 rework). The first cut guarded the
|
|
839
|
+
// hold check itself with `peeked < scanLimit`, so a candidate past the cap
|
|
840
|
+
// was claimed AS IF IT CARRIED NO HOLD — removing the protection exactly
|
|
841
|
+
// where depth made it matter, and silently. Stopping instead can only ever
|
|
842
|
+
// claim FEWER ticks, never one that is not due, and the exhaustion is
|
|
843
|
+
// logged so the degradation is visible in the bus log.
|
|
844
|
+
const scanLimit = Number.isFinite(opts.scanLimit) && opts.scanLimit > 0
|
|
845
|
+
? opts.scanLimit
|
|
846
|
+
: DEFAULT_CLAIM_SCAN_LIMIT;
|
|
847
|
+
let peeked = 0;
|
|
587
848
|
for (const id of ids) {
|
|
588
849
|
const src = join(paths.inbox, `${id}.json`);
|
|
589
850
|
const dst = join(paths.claimed, `${id}.json`);
|
|
851
|
+
// Is this tick due? Two holds, both checked before claiming:
|
|
852
|
+
// - the per-tick `metadata.notBefore` stamp (durable, survives restart);
|
|
853
|
+
// - the per-cadence hold (depth-independent, in the caller's hands).
|
|
854
|
+
if (peeked >= scanLimit) {
|
|
855
|
+
logBusEvent(paths.agentRoot, {
|
|
856
|
+
level: "warn",
|
|
857
|
+
stage: "claim_scan_exhausted",
|
|
858
|
+
scanned: peeked,
|
|
859
|
+
inbox_depth: ids.length,
|
|
860
|
+
note: "every examined tick was held; stopping rather than claiming an unexamined one",
|
|
861
|
+
});
|
|
862
|
+
return null;
|
|
863
|
+
}
|
|
864
|
+
peeked += 1;
|
|
865
|
+
const peek = peekHold(src, !!held);
|
|
866
|
+
if (peek) {
|
|
867
|
+
if (peek.notBefore !== null && peek.notBefore > now) continue;
|
|
868
|
+
if (held && peek.cadence && held.get(peek.cadence) > now) continue;
|
|
869
|
+
}
|
|
590
870
|
try {
|
|
591
871
|
renameSync(src, dst);
|
|
592
872
|
} catch (err) {
|
|
@@ -834,13 +1114,45 @@ export function requeueTick(agentRoot, event, opts = {}) {
|
|
|
834
1114
|
let out = event;
|
|
835
1115
|
if (defer) {
|
|
836
1116
|
const meta = event.metadata && typeof event.metadata === "object" ? event.metadata : {};
|
|
1117
|
+
const base = opts.now instanceof Date ? opts.now : new Date();
|
|
1118
|
+
// THE DELAY HALF OF A DEFERRAL (W19). Re-keying to the tail fixed
|
|
1119
|
+
// head-of-line blocking but left the tick immediately claimable, so a
|
|
1120
|
+
// shut gate was re-tested every poll: measured 8 requeues/s sustained,
|
|
1121
|
+
// one event at 40,150 deferrals, a 443 MB/day audit log. `notBefore`
|
|
1122
|
+
// makes the deferral a DELAY as well as a reorder.
|
|
1123
|
+
//
|
|
1124
|
+
// NOT A FAILURE, AND THE DISTINCTION IS LOAD-BEARING. `attempts` is
|
|
1125
|
+
// untouched here and is written only by failTick (audit M5), so a hold
|
|
1126
|
+
// can never advance a tick toward dlq/. The obvious implementation —
|
|
1127
|
+
// routing the delay through the retry machinery so the backoff schedule
|
|
1128
|
+
// is reused — would turn every gate closure into spent retry budget and
|
|
1129
|
+
// dead-letter healthy cadences for being busy. `metadata.deferrals` is a
|
|
1130
|
+
// counter for humans; nothing reads it as a budget.
|
|
1131
|
+
let notBefore = meta.notBefore;
|
|
1132
|
+
if (Number.isFinite(opts.holdMs) && opts.holdMs > 0) {
|
|
1133
|
+
// Clamped at the chokepoint. A hold is a DELAY, and a delay longer than
|
|
1134
|
+
// MAX_DEFER_HOLD_MS is indistinguishable from a lost tick: claimNextTick
|
|
1135
|
+
// skips a held tick silently, so an hours-long notBefore (a provider
|
|
1136
|
+
// Retry-After, a forward clock jump, a corrupted number reaching the
|
|
1137
|
+
// circuit gate's `retry_at`) would park a healthy cadence out of sight.
|
|
1138
|
+
const until = base.getTime() + Math.min(opts.holdMs, MAX_DEFER_HOLD_MS);
|
|
1139
|
+
// Monotonic: a longer hold already in force keeps its own deadline.
|
|
1140
|
+
const prior = Date.parse(notBefore || "");
|
|
1141
|
+
notBefore = new Date(Number.isFinite(prior) && prior > until ? prior : until).toISOString();
|
|
1142
|
+
} else if (Number.isFinite(opts.notBeforeMs)) {
|
|
1143
|
+
const prior = Date.parse(notBefore || "");
|
|
1144
|
+
notBefore = new Date(
|
|
1145
|
+
Number.isFinite(prior) && prior > opts.notBeforeMs ? prior : opts.notBeforeMs,
|
|
1146
|
+
).toISOString();
|
|
1147
|
+
}
|
|
837
1148
|
out = {
|
|
838
1149
|
...event,
|
|
839
|
-
id: nextEventId(
|
|
1150
|
+
id: nextEventId(base),
|
|
840
1151
|
metadata: {
|
|
841
1152
|
...meta,
|
|
842
1153
|
deferrals: (Number.isFinite(meta.deferrals) ? meta.deferrals : 0) + 1,
|
|
843
1154
|
originalId: meta.originalId || priorId,
|
|
1155
|
+
...(notBefore ? { notBefore } : {}),
|
|
844
1156
|
},
|
|
845
1157
|
};
|
|
846
1158
|
}
|
|
@@ -866,6 +1178,7 @@ export function requeueTick(agentRoot, event, opts = {}) {
|
|
|
866
1178
|
prior_id: priorId,
|
|
867
1179
|
original_id: out.metadata.originalId,
|
|
868
1180
|
deferrals: out.metadata.deferrals,
|
|
1181
|
+
not_before: out.metadata.notBefore || null,
|
|
869
1182
|
cadence: event.cadence,
|
|
870
1183
|
});
|
|
871
1184
|
return { ok: true, id: out.id };
|