@indigoai-us/hq-cli 5.108.14 → 5.108.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,44 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.108.16] — 2026-09-07
6
+
7
+ ### Fixed
8
+
9
+ - Work Mesh Live daemon no longer retries every queued work-context outbox
10
+ operation on every flush cycle. Outbox replay now applies per-operation
11
+ exponential backoff (`nextAttemptAt`, 30s base → 6h cap with jitter), a
12
+ per-cycle cap (200 due ops), an attempt ceiling (50 → quarantine as
13
+ `<code>_MAX_ATTEMPTS`), cheaper listing that skips re-reading unchanged
14
+ outbox files, and classifies HTTP 404/409/410/412 (and other 4xx) from
15
+ transport errors as non-retryable instead of retrying them forever as
16
+ `TRANSPORT_ERROR`.
17
+
18
+ ## [5.108.15] — 2026-09-07
19
+
20
+ ### Fixed
21
+
22
+ - Work Mesh Live presence daemon (`hq mesh daemon run`) no longer pins a CPU
23
+ core when the MQTT broker accepts a connection then closes it just past the
24
+ stable-connect grace window. The full-jitter reconnect backoff could collapse
25
+ to ~0 ms after the connect handler reset its counters, producing a tight
26
+ accept-then-close reconnect spin (each pass re-ran the SigV4 presign and
27
+ re-materialised the daemon state dir) with no network progress and no log
28
+ output. A hard `RECONNECT_MIN_DELAY_MS` (1s) floor now applies to every
29
+ reconnect path — normal backoff, the post-grace reset, and the refused-retry
30
+ path (which a server `Retry-After: 0` could otherwise drive to 0) — so the
31
+ delay can never shrink below 1s. Regression tests cover the reset and the
32
+ refused-retry floor. (#524)
33
+ - The Work Mesh Live daemon no longer burns CPU in a self-triggered flush loop.
34
+ Its spool directory watcher woke on files the daemon writes itself during a
35
+ flush (`held.jsonl` and the `*.claimed` files), so any box with held events
36
+ flushed every two seconds forever; the watcher now reacts only to producer
37
+ appends to `spool.jsonl`. Recovering a very large orphan claim file no longer
38
+ overflows the stack, mid-flush disposal is O(1), held events are re-attempted
39
+ at most every five minutes, and `held.jsonl` is capped at 20,000 lines with
40
+ the oldest overflow dead-lettered as `HELD_OVERFLOW` (loss-free under failure
41
+ and concurrent flushers). (#525)
42
+
5
43
  ## [5.108.14] — 2026-09-07
6
44
 
7
45
  ### Fixed
@@ -43,7 +43,12 @@ export async function meshJson(token, path, init = {}) {
43
43
  }
44
44
  if (!res.ok) {
45
45
  const err = data;
46
- throw new Error(err.error || err.message || `${res.status} ${res.statusText}`);
46
+ // Always lead with the HTTP status: callers classify retryable vs permanent
47
+ // failures from the message (see createWorkSessionDeliverer). Without it a
48
+ // 400 whose body text matched no pattern was retried forever.
49
+ const detail = err.error || err.message || res.statusText || "request failed";
50
+ const code = typeof err.code === "string" && err.code ? `${err.code}: ` : "";
51
+ throw new Error(`${res.status} ${code}${detail}`);
47
52
  }
48
53
  return data;
49
54
  }
@@ -46,17 +46,37 @@ export function createWorkSessionDeliverer(opts) {
46
46
  }
47
47
  catch (err) {
48
48
  const message = err instanceof Error ? err.message : String(err);
49
- // meshJson throws on !ok with error text; classify roughly.
50
- if (/^401\b|^403\b|unauthorized|forbidden|auth/i.test(message)) {
49
+ // meshJson throws on !ok with error text; classify by leading status when present.
50
+ const statusMatch = /^(\d{3})\b/.exec(message);
51
+ if (statusMatch) {
52
+ const status = Number(statusMatch[1]);
53
+ if (status === 401 || status === 403) {
54
+ return { ok: false, retryable: false, code: "AUTH_DENIED" };
55
+ }
56
+ if (status === 400 || status === 422) {
57
+ return { ok: false, retryable: false, code: "VALIDATION_FAILED" };
58
+ }
59
+ if (status === 429 || (status >= 500 && status <= 599)) {
60
+ return { ok: false, retryable: true, code: `HTTP_${status}` };
61
+ }
62
+ if (status === 404 ||
63
+ status === 409 ||
64
+ status === 410 ||
65
+ status === 412 ||
66
+ (status >= 400 && status < 500)) {
67
+ return { ok: false, retryable: false, code: `HTTP_${status}` };
68
+ }
69
+ }
70
+ if (/unauthorized|forbidden|auth/i.test(message)) {
51
71
  return { ok: false, retryable: false, code: "AUTH_DENIED" };
52
72
  }
53
- if (/^400\b|^422\b|invalid|validation/i.test(message)) {
73
+ if (/invalid|validation/i.test(message)) {
54
74
  return { ok: false, retryable: false, code: "VALIDATION_FAILED" };
55
75
  }
56
- if (/^5\d\d\b|ECONN|ENOTFOUND|ETIMEDOUT|network|fetch failed/i.test(message)) {
76
+ if (/ECONN|ENOTFOUND|ETIMEDOUT|network|fetch failed/i.test(message)) {
57
77
  return { ok: false, retryable: true, code: "NETWORK_OR_5XX" };
58
78
  }
59
- // Unknown: keep queued (fail open to retry, not quarantine).
79
+ // No status in the message: keep queued (fail open to retry, not quarantine).
60
80
  return { ok: false, retryable: true, code: "TRANSPORT_ERROR" };
61
81
  }
62
82
  };
@@ -73,7 +93,7 @@ function mapRegisterResponse(status, body, operationId) {
73
93
  if (status >= 500 || status === 429) {
74
94
  return { ok: false, retryable: true, code: `HTTP_${status}` };
75
95
  }
76
- if (status === 401 || status === 403 || status === 400 || status === 422) {
96
+ if (status >= 400 && status < 500) {
77
97
  return { ok: false, retryable: false, code: `HTTP_${status}` };
78
98
  }
79
99
  if (status === 0) {
@@ -20,6 +20,20 @@ export type MqttConnectionState = "idle" | "connecting" | "connected" | "reconne
20
20
  * then closes immediately after a denied publish).
21
21
  */
22
22
  export declare const CONNECT_STABLE_GRACE_MS = 5000;
23
+ /**
24
+ * Hard floor for the delay between MQTT presence reconnect attempts.
25
+ *
26
+ * The full-jitter backoff can otherwise compute ~0 ms: after a connection
27
+ * survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
28
+ * `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
29
+ * past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
30
+ * return `random() * base` — frequently a handful of ms, or 0 when `random()`
31
+ * is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
32
+ * presign and re-materialises the daemon state dir) that pins a CPU core with
33
+ * no network progress and no log output. This floor guarantees every reconnect
34
+ * waits at least this long regardless of the backoff/grace-reset state.
35
+ */
36
+ export declare const RECONNECT_MIN_DELAY_MS = 1000;
23
37
  /** Always emit the first N close info lines, then at most one per interval. */
24
38
  export declare const CLOSE_LOG_ALWAYS_COUNT = 3;
25
39
  export declare const CLOSE_LOG_INTERVAL_MS = 60000;
@@ -69,9 +83,13 @@ export interface PresenceClientOptions {
69
83
  }
70
84
  /**
71
85
  * Full-jitter capped exponential backoff (1s base → 60s cap by default),
72
- * floored at `previousMs` so consecutive failures never shrink the delay.
86
+ * floored at `previousMs` so consecutive failures never shrink the delay and at
87
+ * `minMs` so the result can never collapse to ~0 (see
88
+ * {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
89
+ * accept-then-close reconnect spin after the connect handler resets the
90
+ * backoff counters.
73
91
  */
74
- export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number): number;
92
+ export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
75
93
  export declare function buildPresencePayload(input: {
76
94
  status: "online" | "offline";
77
95
  actorUid: string;
@@ -20,17 +20,35 @@ import { presignIotWssUrl } from "./presign.js";
20
20
  * then closes immediately after a denied publish).
21
21
  */
22
22
  export const CONNECT_STABLE_GRACE_MS = 5_000;
23
+ /**
24
+ * Hard floor for the delay between MQTT presence reconnect attempts.
25
+ *
26
+ * The full-jitter backoff can otherwise compute ~0 ms: after a connection
27
+ * survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
28
+ * `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
29
+ * past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
30
+ * return `random() * base` — frequently a handful of ms, or 0 when `random()`
31
+ * is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
32
+ * presign and re-materialises the daemon state dir) that pins a CPU core with
33
+ * no network progress and no log output. This floor guarantees every reconnect
34
+ * waits at least this long regardless of the backoff/grace-reset state.
35
+ */
36
+ export const RECONNECT_MIN_DELAY_MS = 1_000;
23
37
  /** Always emit the first N close info lines, then at most one per interval. */
24
38
  export const CLOSE_LOG_ALWAYS_COUNT = 3;
25
39
  export const CLOSE_LOG_INTERVAL_MS = 60_000;
26
40
  /**
27
41
  * Full-jitter capped exponential backoff (1s base → 60s cap by default),
28
- * floored at `previousMs` so consecutive failures never shrink the delay.
42
+ * floored at `previousMs` so consecutive failures never shrink the delay and at
43
+ * `minMs` so the result can never collapse to ~0 (see
44
+ * {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
45
+ * accept-then-close reconnect spin after the connect handler resets the
46
+ * backoff counters.
29
47
  */
30
- export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0) {
48
+ export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, minMs = 0) {
31
49
  const cap = Math.min(maxMs, baseMs * 2 ** attempt);
32
50
  const raw = Math.max(0, random() * cap);
33
- return Math.min(maxMs, Math.max(previousMs, raw));
51
+ return Math.min(maxMs, Math.max(previousMs, raw, minMs));
34
52
  }
35
53
  export function buildPresencePayload(input) {
36
54
  return {
@@ -341,12 +359,14 @@ export class PresenceClient {
341
359
  else {
342
360
  delay = refusedRetryDelayMs(this.refusedRetryMs, this.random);
343
361
  }
344
- delay = clampRetryDelayMs(delay);
362
+ // Floor the refused retry too: a server `Retry-After: 0` (or a 0 env
363
+ // override) would otherwise clamp to 0 and hot-loop the refused path.
364
+ delay = Math.max(RECONNECT_MIN_DELAY_MS, clampRetryDelayMs(delay));
345
365
  const nextRetryAt = new Date(this.timers.now() + delay).toISOString();
346
366
  this.noteRefusal(refused, nextRetryAt);
347
367
  }
348
368
  else {
349
- delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs);
369
+ delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs, RECONNECT_MIN_DELAY_MS);
350
370
  this.lastBackoffMs = delay;
351
371
  this.attempt += 1;
352
372
  // Do not clear refusal here — MQTT close must retain doctor/status refusal
@@ -15,6 +15,16 @@ import { PresenceClient, type MqttConnectFn } from "./presence.js";
15
15
  import { TranscriptWatcher, type TranscriptFs } from "./transcript-watch.js";
16
16
  export declare const SPOOL_DEBOUNCE_MS = 2000;
17
17
  export declare const FLUSH_INTERVAL_MS = 10000;
18
+ /**
19
+ * Only producer appends to spool.jsonl should wake the watcher. Every other
20
+ * file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
21
+ * live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
22
+ * during a flush; reacting to those turned each flush into a self-triggered
23
+ * flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
24
+ * core on fleet boxes with a non-empty held file). A null filename (platforms
25
+ * that do not report one) keeps the conservative behaviour and flushes.
26
+ */
27
+ export declare function shouldFlushOnSpoolDirEvent(filename: string | Buffer | null | undefined): boolean;
18
28
  export interface DaemonRunDeps {
19
29
  home?: string;
20
30
  env?: NodeJS.ProcessEnv;
@@ -31,6 +41,7 @@ export interface DaemonRunDeps {
31
41
  delivered: number;
32
42
  queued: number;
33
43
  quarantined: number;
44
+ skipped?: number;
34
45
  }>;
35
46
  /** Injected board refresh (tests). */
36
47
  refreshBoards?: () => Promise<{
@@ -17,7 +17,7 @@ import { ensureCognitoToken } from "../../../../utils/cognito-session.js";
17
17
  import { replayOutbox } from "../../../work-context/outbox.js";
18
18
  import { workContextRoot } from "../../../work-context/paths.js";
19
19
  import { createSessionEventsPoster, resolveVaultApiBase, } from "../session-events-client.js";
20
- import { flushSessionEvents } from "../flush.js";
20
+ import { flushSessionEvents, HELD_RETRY_INTERVAL_MS, } from "../flush.js";
21
21
  import { workMeshRoot, workMeshSpoolPath } from "../paths.js";
22
22
  import { createVaultBoardReader, refreshBoundSessionBoards, BOARD_REFRESH_INTERVAL_MS, } from "./board-refresh.js";
23
23
  import { createContract3Fetcher, realTimerHost, } from "./credentials.js";
@@ -30,6 +30,20 @@ import { workMeshHeldPath } from "../paths.js";
30
30
  import { noteHookSessionsFromSpoolFile, TRANSCRIPT_WATCH_INTERVAL_MS, TranscriptWatcher, } from "./transcript-watch.js";
31
31
  export const SPOOL_DEBOUNCE_MS = 2_000;
32
32
  export const FLUSH_INTERVAL_MS = 10_000;
33
+ /**
34
+ * Only producer appends to spool.jsonl should wake the watcher. Every other
35
+ * file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
36
+ * live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
37
+ * during a flush; reacting to those turned each flush into a self-triggered
38
+ * flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
39
+ * core on fleet boxes with a non-empty held file). A null filename (platforms
40
+ * that do not report one) keeps the conservative behaviour and flushes.
41
+ */
42
+ export function shouldFlushOnSpoolDirEvent(filename) {
43
+ if (filename === null || filename === undefined)
44
+ return true;
45
+ return String(filename) === "spool.jsonl";
46
+ }
33
47
  function alive(pid) {
34
48
  try {
35
49
  process.kill(pid, 0);
@@ -76,6 +90,8 @@ export async function runMeshDaemon(deps = {}) {
76
90
  // daemon went dark once its IoT credentials expired.
77
91
  const defaultGetToken = async () => ensureCognitoToken({ interactive: false, tokenSource: "machine" });
78
92
  const getToken = deps.getToken ?? defaultGetToken;
93
+ /** In-memory held re-attempt throttle (survives watch-triggered flushes). */
94
+ let lastHeldRetryAtMs = 0;
79
95
  const flushFn = deps.flush ??
80
96
  (async () => {
81
97
  const t = await getToken();
@@ -87,6 +103,12 @@ export async function runMeshDaemon(deps = {}) {
87
103
  workMeshRoot: meshRoot,
88
104
  workContextRoot: ctxRoot,
89
105
  poster,
106
+ heldRetryIntervalMs: HELD_RETRY_INTERVAL_MS,
107
+ lastHeldRetryAtMs,
108
+ onHeldClaimed: (atMs) => {
109
+ lastHeldRetryAtMs = atMs;
110
+ },
111
+ log: (message) => log(dir, message),
90
112
  });
91
113
  });
92
114
  const replayFn = deps.replayOutbox ??
@@ -233,7 +255,14 @@ export async function runMeshDaemon(deps = {}) {
233
255
  noteHookSessionsFromSpoolFile(workMeshHeldPath(meshRoot), lastHookEventAt, at);
234
256
  const summary = await flushFn();
235
257
  try {
236
- await replayFn();
258
+ const replay = await replayFn();
259
+ const delivered = replay.delivered ?? 0;
260
+ const queued = replay.queued ?? 0;
261
+ const quarantined = replay.quarantined ?? 0;
262
+ const skipped = replay.skipped ?? 0;
263
+ if (delivered > 0 || queued > 0 || quarantined > 0 || skipped > 0) {
264
+ log(dir, `outbox replay: delivered=${delivered} queued=${queued} quarantined=${quarantined} skipped=${skipped}`);
265
+ }
237
266
  }
238
267
  catch (err) {
239
268
  const msg = err instanceof Error ? err.message : String(err);
@@ -243,7 +272,10 @@ export async function runMeshDaemon(deps = {}) {
243
272
  lastFlushAt: now().toISOString(),
244
273
  lastFlushResult: { ...summary, ok: true },
245
274
  }, now);
246
- if (summary.claimed > 0 || summary.posted > 0 || summary.held > 0) {
275
+ if (summary.claimed > 0 ||
276
+ summary.posted > 0 ||
277
+ summary.held > 0 ||
278
+ (summary.heldOverflow ?? 0) > 0) {
247
279
  log(dir, `flush(${reason}): claimed=${summary.claimed} posted=${summary.posted} held=${summary.held}`);
248
280
  }
249
281
  }
@@ -292,11 +324,7 @@ export async function runMeshDaemon(deps = {}) {
292
324
  watcher = fs.watch(path.dirname(spoolPath), { persistent: true }, (_event, filename) => {
293
325
  if (stopped)
294
326
  return;
295
- if (!filename ||
296
- filename === "spool.jsonl" ||
297
- filename === "held.jsonl" ||
298
- String(filename).startsWith("spool.") ||
299
- String(filename).startsWith("held.")) {
327
+ if (shouldFlushOnSpoolDirEvent(filename)) {
300
328
  scheduleDebouncedFlush();
301
329
  }
302
330
  });
@@ -11,7 +11,11 @@
11
11
  import { type RandomFn, type SleepFn } from "./backoff.js";
12
12
  import { type SessionEventsPoster } from "./session-events-client.js";
13
13
  export declare const HELD_TTL_MS: number;
14
- export type HeldReason = "STATE_ABSENT" | "NEEDS_COMPANY" | "COMPANY_CONFLICT" | "HELD_EXPIRED";
14
+ /** Re-attempt held events at most this often (spool still flushes every cycle). */
15
+ export declare const HELD_RETRY_INTERVAL_MS: number;
16
+ /** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
17
+ export declare const HELD_MAX_LINES = 20000;
18
+ export type HeldReason = "STATE_ABSENT" | "NEEDS_COMPANY" | "COMPANY_CONFLICT" | "HELD_EXPIRED" | "HELD_OVERFLOW";
15
19
  export interface HeldLineRecord {
16
20
  /** Original spool event fields. */
17
21
  event: Record<string, unknown>;
@@ -26,6 +30,19 @@ export interface FlushDeps {
26
30
  sleep?: SleepFn;
27
31
  random?: RandomFn;
28
32
  maxAttempts?: number;
33
+ /**
34
+ * Minimum gap between held re-attempts. Default HELD_RETRY_INTERVAL_MS.
35
+ * Pass 0 in tests to re-claim held every flush.
36
+ */
37
+ heldRetryIntervalMs?: number;
38
+ /** Epoch ms of the last flush that claimed held (daemon keeps this in memory). */
39
+ lastHeldRetryAtMs?: number;
40
+ /** Invoked when this flush claims held, so the daemon can advance its timestamp. */
41
+ onHeldClaimed?: (atMs: number) => void;
42
+ /** Max lines retained in held.jsonl after a flush. Default HELD_MAX_LINES. */
43
+ heldMaxLines?: number;
44
+ /** Optional logger (daemon wires this); used once per overflow trim. */
45
+ log?: (message: string) => void;
29
46
  }
30
47
  export interface FlushSummary {
31
48
  claimed: number;
@@ -35,6 +52,10 @@ export interface FlushSummary {
35
52
  deadLettered: number;
36
53
  restored: number;
37
54
  batches: number;
55
+ /** True when this flush claimed/re-read held.jsonl. */
56
+ heldClaimed?: boolean;
57
+ /** Oldest held lines dead-lettered this flush due to HELD_MAX_LINES. */
58
+ heldOverflow?: number;
38
59
  }
39
60
  /**
40
61
  * Flush spool + held. Safe to call concurrently with enqueue (claim-by-rename).
@@ -15,9 +15,13 @@ import { workContextRoot } from "../../work-context/paths.js";
15
15
  import { defaultSleep, FLUSH_MAX_ATTEMPTS, fullJitterDelayMs, } from "./backoff.js";
16
16
  import { LOCAL_ONLY_FIELDS, stripLocalOnlyFields } from "./format-spool-line.js";
17
17
  import { SESSION_EVENTS_BATCH_MAX, WRITABLE_CONTEXT_STATUSES, parseSessionEventsBatchAck, } from "./session-events-client.js";
18
- import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
19
- import { workMeshRoot } from "./paths.js";
18
+ import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, countJsonlLines, ensureSpoolFile, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
19
+ import { workMeshHeldPath, workMeshRoot } from "./paths.js";
20
20
  export const HELD_TTL_MS = 7 * 24 * 60 * 60 * 1000;
21
+ /** Re-attempt held events at most this often (spool still flushes every cycle). */
22
+ export const HELD_RETRY_INTERVAL_MS = 5 * 60 * 1000;
23
+ /** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
24
+ export const HELD_MAX_LINES = 20_000;
21
25
  function parseLine(line) {
22
26
  try {
23
27
  const v = JSON.parse(line);
@@ -101,10 +105,22 @@ export async function flushSessionEvents(deps) {
101
105
  const sleep = deps.sleep ?? defaultSleep;
102
106
  const random = deps.random ?? Math.random;
103
107
  const maxAttempts = deps.maxAttempts ?? FLUSH_MAX_ATTEMPTS;
104
- const ts = `${now().getTime()}`;
108
+ const heldRetryIntervalMs = deps.heldRetryIntervalMs ?? HELD_RETRY_INTERVAL_MS;
109
+ const heldMaxLines = deps.heldMaxLines ?? HELD_MAX_LINES;
110
+ const nowMs = now().getTime();
111
+ const ts = `${nowMs}`;
105
112
  const orphans = recoverOrphanClaims(deps.workMeshRoot);
106
113
  const spoolClaim = claimSpool(deps.workMeshRoot, ts);
107
- const heldClaim = claimHeld(deps.workMeshRoot, `${ts}.held`);
114
+ // Throttle held re-attempts; spool (fresh) events still flush every cycle.
115
+ const lastHeldRetryAtMs = deps.lastHeldRetryAtMs ?? 0;
116
+ const shouldClaimHeld = heldRetryIntervalMs <= 0 ||
117
+ nowMs - lastHeldRetryAtMs >= heldRetryIntervalMs;
118
+ const heldClaim = shouldClaimHeld
119
+ ? claimHeld(deps.workMeshRoot, `${ts}.held`)
120
+ : { claimPath: null, lines: [] };
121
+ if (shouldClaimHeld) {
122
+ deps.onHeldClaimed?.(nowMs);
123
+ }
108
124
  const claimPaths = [
109
125
  ...orphans.claimPaths,
110
126
  spoolClaim.claimPath,
@@ -118,42 +134,45 @@ export async function flushSessionEvents(deps) {
118
134
  deadLettered: 0,
119
135
  restored: 0,
120
136
  batches: 0,
137
+ heldClaimed: shouldClaimHeld,
138
+ heldOverflow: 0,
121
139
  };
122
- /** Original events still awaiting disposition (restored on throw). */
123
- const pendingRestore = [];
140
+ /** Original events still awaiting disposition (restored on throw). O(1) delete. */
141
+ const pendingRestore = new Set();
124
142
  const collected = [];
125
143
  try {
126
- for (const line of [...orphans.lines, ...spoolClaim.lines, ...heldClaim.lines]) {
127
- summary.claimed += 1;
128
- const parsed = parseLine(line);
129
- if (!parsed) {
130
- appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "SCHEMA_REJECTED", "parse_error", now().toISOString()), deps.workMeshRoot);
131
- summary.deadLettered += 1;
132
- continue;
133
- }
134
- const { event, heldAt } = unwrapHeld(parsed);
135
- if (!sessionIdOf(event)) {
136
- appendDeadLetterLine(deadLetterEnvelope(event, "SCHEMA_REJECTED", "missing_sessionId", now().toISOString()), deps.workMeshRoot);
137
- summary.deadLettered += 1;
138
- continue;
139
- }
140
- // Expire held lines older than 7 days.
141
- if (heldAt) {
142
- const age = now().getTime() - Date.parse(heldAt);
143
- if (Number.isFinite(age) && age > HELD_TTL_MS) {
144
- appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
144
+ // Iterate sources separately avoid building one giant combined array.
145
+ for (const source of [orphans.lines, spoolClaim.lines, heldClaim.lines]) {
146
+ for (const line of source) {
147
+ summary.claimed += 1;
148
+ const parsed = parseLine(line);
149
+ if (!parsed) {
150
+ appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "SCHEMA_REJECTED", "parse_error", now().toISOString()), deps.workMeshRoot);
151
+ summary.deadLettered += 1;
152
+ continue;
153
+ }
154
+ const { event, heldAt } = unwrapHeld(parsed);
155
+ if (!sessionIdOf(event)) {
156
+ appendDeadLetterLine(deadLetterEnvelope(event, "SCHEMA_REJECTED", "missing_sessionId", now().toISOString()), deps.workMeshRoot);
145
157
  summary.deadLettered += 1;
146
158
  continue;
147
159
  }
160
+ // Expire held lines older than 7 days.
161
+ if (heldAt) {
162
+ const age = now().getTime() - Date.parse(heldAt);
163
+ if (Number.isFinite(age) && age > HELD_TTL_MS) {
164
+ appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
165
+ summary.deadLettered += 1;
166
+ continue;
167
+ }
168
+ }
169
+ pendingRestore.add(event);
170
+ collected.push({ event, heldAt });
148
171
  }
149
- pendingRestore.push(event);
150
- collected.push({ event, heldAt });
151
172
  }
152
173
  const bySession = groupBySession(collected);
153
174
  const markDisposed = (event) => {
154
- const idx = pendingRestore.indexOf(event);
155
- if (idx >= 0)
156
- pendingRestore.splice(idx, 1);
175
+ pendingRestore.delete(event);
157
176
  };
158
177
  for (const [sessionId, items] of bySession) {
159
178
  const state = readSessionState(sessionId, deps.workContextRoot);
@@ -238,13 +257,100 @@ export async function flushSessionEvents(deps) {
238
257
  /* best-effort; orphan claim scan will retry next flush */
239
258
  }
240
259
  }
241
- pendingRestore.length = 0;
260
+ pendingRestore.clear();
242
261
  for (const p of claimPaths) {
243
262
  removeClaimFile(p);
244
263
  }
245
264
  }
265
+ const overflowed = trimHeldOverflow(deps.workMeshRoot, heldMaxLines, now, summary);
266
+ if (overflowed > 0) {
267
+ deps.log?.(`held overflow: dead-lettered ${overflowed} oldest events (cap=${heldMaxLines})`);
268
+ }
246
269
  return summary;
247
270
  }
271
+ /**
272
+ * After a flush, if held.jsonl exceeds maxLines, dead-letter the oldest
273
+ * overflow (by heldAt, not file position) with HELD_OVERFLOW and keep the rest.
274
+ *
275
+ * Durability rules:
276
+ * - Retained lines are APPENDED to held.jsonl, never written over it: another
277
+ * flusher (`hq mesh flush` beside the daemon) may have appended to a fresh
278
+ * held.jsonl after our claim renamed the old one away.
279
+ * - The overflow claim file is removed only after every dead letter and every
280
+ * retained line is on disk. If anything throws, the claim stays behind and
281
+ * recoverOrphanClaims re-reads it on the next flush (duplicates are possible,
282
+ * loss is not).
283
+ */
284
+ function trimHeldOverflow(meshRoot, maxLines, now, summary) {
285
+ if (maxLines <= 0)
286
+ return 0;
287
+ const heldPath = workMeshHeldPath(meshRoot);
288
+ if (countJsonlLines(heldPath) <= maxLines)
289
+ return 0;
290
+ const claim = claimHeld(meshRoot, `${now().getTime()}.overflow`);
291
+ if (!claim.claimPath)
292
+ return 0;
293
+ if (claim.lines.length === 0) {
294
+ removeClaimFile(claim.claimPath);
295
+ return 0;
296
+ }
297
+ try {
298
+ if (claim.lines.length <= maxLines) {
299
+ appendHeldLines(meshRoot, claim.lines);
300
+ removeClaimFile(claim.claimPath);
301
+ return 0;
302
+ }
303
+ const overflow = claim.lines.length - maxLines;
304
+ // Oldest by heldAt: retries regroup held by session, so file order is not
305
+ // chronological. Unparseable lines sort first (dead-lettered first).
306
+ const ordered = claim.lines
307
+ .map((line, idx) => ({ line, idx, atMs: heldAtMs(line) }))
308
+ .sort((a, b) => a.atMs - b.atMs || a.idx - b.idx);
309
+ const toDead = ordered.slice(0, overflow);
310
+ const toKeep = ordered
311
+ .slice(overflow)
312
+ .sort((a, b) => a.idx - b.idx)
313
+ .map((x) => x.line);
314
+ const at = now().toISOString();
315
+ for (const { line } of toDead) {
316
+ const parsed = parseLine(line);
317
+ if (!parsed) {
318
+ appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
319
+ }
320
+ else {
321
+ const { event } = unwrapHeld(parsed);
322
+ appendDeadLetterLine(deadLetterEnvelope(event, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
323
+ }
324
+ summary.deadLettered += 1;
325
+ }
326
+ appendHeldLines(meshRoot, toKeep);
327
+ summary.heldOverflow = overflow;
328
+ removeClaimFile(claim.claimPath);
329
+ return overflow;
330
+ }
331
+ catch {
332
+ // Leave the claim for orphan recovery; never drop held events on a failed trim.
333
+ return 0;
334
+ }
335
+ }
336
+ /** heldAt of a held line as epoch ms; 0 when missing or unparseable. */
337
+ function heldAtMs(line) {
338
+ const parsed = parseLine(line);
339
+ if (!parsed)
340
+ return 0;
341
+ const { heldAt } = unwrapHeld(parsed);
342
+ if (!heldAt)
343
+ return 0;
344
+ const ms = Date.parse(heldAt);
345
+ return Number.isFinite(ms) ? ms : 0;
346
+ }
347
+ function appendHeldLines(meshRoot, lines) {
348
+ if (lines.length === 0)
349
+ return;
350
+ const heldPath = workMeshHeldPath(meshRoot);
351
+ ensureSpoolFile(heldPath);
352
+ fs.appendFileSync(heldPath, `${lines.join("\n")}\n`, { mode: 0o600 });
353
+ }
248
354
  /**
249
355
  * Account a 2xx batch: dead-letter rejected eventIds with their codes;
250
356
  * count accepted+duplicates as posted; restore any eventId not accounted for.
@@ -13,7 +13,7 @@ export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath,
13
13
  export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
14
14
  export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
15
15
  export type { EnqueueOptions, EnqueueResult } from "./enqueue.js";
16
- export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
16
+ export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
17
17
  export type { FlushDeps, FlushSummary, HeldReason } from "./flush.js";
18
18
  export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
19
19
  export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
@@ -9,7 +9,7 @@ export { encodeCrockford, generateUlid, isUlid } from "./ulid.js";
9
9
  export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath, workMeshHeldPath, workMeshRoot, workMeshSpoolPath, } from "./paths.js";
10
10
  export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
11
11
  export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
12
- export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
12
+ export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
13
13
  export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
14
14
  export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
15
15
  export * from "./daemon/index.js";
@@ -153,7 +153,12 @@ export function recoverOrphanClaims(root) {
153
153
  const claimPaths = listOrphanClaimFiles(root);
154
154
  const lines = [];
155
155
  for (const p of claimPaths) {
156
- lines.push(...readClaimFileLines(p));
156
+ // Plain loop: Array#push(...huge) exceeds V8's argument limit (~65k–128k)
157
+ // and throws RangeError ("Maximum call stack size exceeded").
158
+ const orphanLines = readClaimFileLines(p);
159
+ for (const line of orphanLines) {
160
+ lines.push(line);
161
+ }
157
162
  }
158
163
  return { lines, claimPaths };
159
164
  }
@@ -5,7 +5,15 @@
5
5
  import type { DeliveryState } from "./contract.js";
6
6
  import { WORK_CONTEXT_CONTRACT_VERSION } from "./contract.js";
7
7
  /** Fields allowed on a durable outbox operation (privacy allowlist). */
8
- export declare const OUTBOX_ALLOWLIST: readonly ["contractVersion", "operationId", "clientOperationId", "sessionId", "companyUid", "companySlug", "projectId", "taskId", "kind", "digest", "delivery", "createdAt", "updatedAt", "attemptCount", "lastErrorCode", "receiptId", "destinationCompanyUid", "destinationCompanySlug"];
8
+ export declare const OUTBOX_ALLOWLIST: readonly ["contractVersion", "operationId", "clientOperationId", "sessionId", "companyUid", "companySlug", "projectId", "taskId", "kind", "digest", "delivery", "createdAt", "updatedAt", "attemptCount", "lastErrorCode", "receiptId", "destinationCompanyUid", "destinationCompanySlug", "nextAttemptAt"];
9
+ /** Base delay for outbox retry backoff (attempt 1 → 30s). */
10
+ export declare const OUTBOX_RETRY_BASE_MS = 30000;
11
+ /** Cap for outbox retry backoff (6 hours). */
12
+ export declare const OUTBOX_RETRY_MAX_MS: number;
13
+ /** Quarantine after this many delivery attempts on retryable failures. */
14
+ export declare const OUTBOX_MAX_ATTEMPTS = 50;
15
+ /** Max due operations processed per replayOutbox call. */
16
+ export declare const OUTBOX_REPLAY_MAX_OPS = 200;
9
17
  export type OutboxKind = "register" | "reconcile" | "migrate";
10
18
  export interface OutboxOperation {
11
19
  contractVersion: typeof WORK_CONTEXT_CONTRACT_VERSION;
@@ -27,6 +35,8 @@ export interface OutboxOperation {
27
35
  /** Destination company for kind=migrate (source is companyUid). */
28
36
  destinationCompanyUid?: string;
29
37
  destinationCompanySlug?: string;
38
+ /** ISO time when the next delivery attempt is due (absent → due immediately). */
39
+ nextAttemptAt?: string;
30
40
  }
31
41
  export interface OutboxEnqueueInput {
32
42
  clientOperationId: string;
@@ -40,6 +50,7 @@ export interface OutboxEnqueueInput {
40
50
  destinationCompanySlug?: string;
41
51
  now?: () => Date;
42
52
  }
53
+ export declare function clearOutboxListCache(): void;
43
54
  export declare function stableOperationId(clientOperationId: string, sessionId: string): string;
44
55
  export declare function digestOperation(parts: {
45
56
  sessionId: string;
@@ -52,6 +63,8 @@ export declare function digestOperation(parts: {
52
63
  destinationCompanyUid?: string;
53
64
  destinationCompanySlug?: string;
54
65
  }): string;
66
+ /** Delay before next attempt: min(BASE * 2^(attemptCount-1), MAX), then [0.5, 1.0] jitter. */
67
+ export declare function outboxRetryDelayMs(attemptCount: number, random?: () => number): number;
55
68
  /**
56
69
  * Atomically enqueue (or idempotently return) an outbox operation.
57
70
  * Disk/permission/lock failure → NotTrackingError (no network-only send).
@@ -60,9 +73,12 @@ export declare function enqueueOutbox(input: OutboxEnqueueInput, root: string):
60
73
  export declare function readOutboxOperation(operationId: string, root: string): OutboxOperation | null;
61
74
  export declare function updateOutboxOperation(op: OutboxOperation, root: string): void;
62
75
  export declare function markOutboxAcked(operationId: string, root: string, receiptId: string, now?: () => Date): OutboxOperation | null;
63
- export declare function markOutboxQueued(operationId: string, root: string, errorCode: string, now?: () => Date): OutboxOperation | null;
76
+ export declare function markOutboxQueued(operationId: string, root: string, errorCode: string, now?: () => Date, random?: () => number): OutboxOperation | null;
64
77
  export declare function markOutboxQuarantined(operationId: string, root: string, errorCode: string, now?: () => Date): OutboxOperation | null;
65
- export declare function listOutboxOperations(root: string): OutboxOperation[];
78
+ export type OutboxListReadFile = (filePath: string) => string;
79
+ export declare function listOutboxOperations(root: string, deps?: {
80
+ readFile?: OutboxListReadFile;
81
+ }): OutboxOperation[];
66
82
  export interface OutboxStats {
67
83
  depth: number;
68
84
  quarantined: number;
@@ -81,14 +97,18 @@ export type DeliverFn = (op: OutboxOperation) => Promise<{
81
97
  /**
82
98
  * Replay queued (and recover lost-receipt) operations.
83
99
  * FIFO per sessionId; bounded parallelism across sessions.
100
+ * Skips ops whose nextAttemptAt is in the future; caps work per call.
84
101
  */
85
102
  export declare function replayOutbox(root: string, deliver: DeliverFn, opts?: {
86
103
  parallel?: number;
87
104
  now?: () => Date;
105
+ maxOps?: number;
106
+ random?: () => number;
88
107
  }): Promise<{
89
108
  delivered: number;
90
109
  queued: number;
91
110
  quarantined: number;
111
+ skipped: number;
92
112
  }>;
93
113
  /** Remove acked ops older than retention (optional GC; not required by AC). */
94
114
  export declare function removeAckedOutbox(root: string, olderThanMs: number, now?: number): number;
@@ -29,8 +29,24 @@ export const OUTBOX_ALLOWLIST = [
29
29
  "receiptId",
30
30
  "destinationCompanyUid",
31
31
  "destinationCompanySlug",
32
+ "nextAttemptAt",
32
33
  ];
34
+ /** Base delay for outbox retry backoff (attempt 1 → 30s). */
35
+ export const OUTBOX_RETRY_BASE_MS = 30_000;
36
+ /** Cap for outbox retry backoff (6 hours). */
37
+ export const OUTBOX_RETRY_MAX_MS = 6 * 60 * 60 * 1000;
38
+ /** Quarantine after this many delivery attempts on retryable failures. */
39
+ export const OUTBOX_MAX_ATTEMPTS = 50;
40
+ /** Max due operations processed per replayOutbox call. */
41
+ export const OUTBOX_REPLAY_MAX_OPS = 200;
33
42
  const DEFAULT_PARALLEL = 4;
43
+ const outboxListCache = new Map();
44
+ function invalidateOutboxListCacheEntry(filePath) {
45
+ outboxListCache.delete(filePath);
46
+ }
47
+ export function clearOutboxListCache() {
48
+ outboxListCache.clear();
49
+ }
34
50
  export function stableOperationId(clientOperationId, sessionId) {
35
51
  const h = crypto
36
52
  .createHash("sha256")
@@ -85,8 +101,18 @@ function projectOutbox(op) {
85
101
  if (op.destinationCompanySlug) {
86
102
  out.destinationCompanySlug = op.destinationCompanySlug;
87
103
  }
104
+ if (op.nextAttemptAt)
105
+ out.nextAttemptAt = op.nextAttemptAt;
88
106
  return out;
89
107
  }
108
+ /** Delay before next attempt: min(BASE * 2^(attemptCount-1), MAX), then [0.5, 1.0] jitter. */
109
+ export function outboxRetryDelayMs(attemptCount, random = Math.random) {
110
+ const exp = Math.max(0, attemptCount - 1);
111
+ const delay = Math.min(OUTBOX_RETRY_BASE_MS * 2 ** exp, OUTBOX_RETRY_MAX_MS);
112
+ const unit = random();
113
+ const factor = 0.5 + 0.5 * Math.min(1, Math.max(0, unit));
114
+ return Math.floor(delay * factor);
115
+ }
90
116
  /**
91
117
  * Atomically enqueue (or idempotently return) an outbox operation.
92
118
  * Disk/permission/lock failure → NotTrackingError (no network-only send).
@@ -182,6 +208,7 @@ export function enqueueOutbox(input, root) {
182
208
  catch {
183
209
  /* ignore */
184
210
  }
211
+ invalidateOutboxListCacheEntry(dest);
185
212
  return op;
186
213
  }
187
214
  catch (err) {
@@ -204,7 +231,9 @@ export function readOutboxOperation(operationId, root) {
204
231
  }
205
232
  }
206
233
  export function updateOutboxOperation(op, root) {
207
- atomicWriteJson(workContextOutboxPath(op.operationId, root), projectOutbox(op));
234
+ const filePath = workContextOutboxPath(op.operationId, root);
235
+ atomicWriteJson(filePath, projectOutbox(op));
236
+ invalidateOutboxListCacheEntry(filePath);
208
237
  }
209
238
  export function markOutboxAcked(operationId, root, receiptId, now = () => new Date()) {
210
239
  const op = readOutboxOperation(operationId, root);
@@ -216,14 +245,17 @@ export function markOutboxAcked(operationId, root, receiptId, now = () => new Da
216
245
  updateOutboxOperation(op, root);
217
246
  return op;
218
247
  }
219
- export function markOutboxQueued(operationId, root, errorCode, now = () => new Date()) {
248
+ export function markOutboxQueued(operationId, root, errorCode, now = () => new Date(), random = Math.random) {
220
249
  const op = readOutboxOperation(operationId, root);
221
250
  if (!op)
222
251
  return null;
223
252
  op.delivery = "queued";
224
253
  op.lastErrorCode = errorCode;
225
254
  op.attemptCount += 1;
226
- op.updatedAt = now().toISOString();
255
+ const at = now();
256
+ op.updatedAt = at.toISOString();
257
+ const delayMs = outboxRetryDelayMs(op.attemptCount, random);
258
+ op.nextAttemptAt = new Date(at.getTime() + delayMs).toISOString();
227
259
  updateOutboxOperation(op, root);
228
260
  return op;
229
261
  }
@@ -238,17 +270,58 @@ export function markOutboxQuarantined(operationId, root, errorCode, now = () =>
238
270
  updateOutboxOperation(op, root);
239
271
  return op;
240
272
  }
241
- export function listOutboxOperations(root) {
273
+ export function listOutboxOperations(root, deps = {}) {
274
+ const readFile = deps.readFile ?? ((filePath) => fs.readFileSync(filePath, "utf8"));
242
275
  const dir = workContextOutboxDir(root);
243
276
  if (!fs.existsSync(dir))
244
277
  return [];
245
278
  const ops = [];
279
+ const seen = new Set();
246
280
  for (const name of fs.readdirSync(dir)) {
247
281
  if (!name.endsWith(".json") || name.startsWith("."))
248
282
  continue;
249
- const op = readOutboxOperation(name.replace(/\.json$/, ""), root);
250
- if (op)
283
+ const operationId = name.replace(/\.json$/, "");
284
+ if (!isSafeWorkContextSegment(operationId))
285
+ continue;
286
+ const filePath = workContextOutboxPath(operationId, root);
287
+ seen.add(filePath);
288
+ let st;
289
+ try {
290
+ st = fs.statSync(filePath);
291
+ }
292
+ catch {
293
+ continue;
294
+ }
295
+ const cached = outboxListCache.get(filePath);
296
+ if (cached &&
297
+ cached.ino === st.ino &&
298
+ cached.mtimeMs === st.mtimeMs &&
299
+ cached.ctimeMs === st.ctimeMs &&
300
+ cached.size === st.size) {
301
+ ops.push(cached.op);
302
+ continue;
303
+ }
304
+ try {
305
+ const op = JSON.parse(readFile(filePath));
306
+ outboxListCache.set(filePath, {
307
+ ino: st.ino,
308
+ mtimeMs: st.mtimeMs,
309
+ ctimeMs: st.ctimeMs,
310
+ size: st.size,
311
+ op,
312
+ });
251
313
  ops.push(op);
314
+ }
315
+ catch {
316
+ /* ignore corrupt */
317
+ }
318
+ }
319
+ // Drop cache entries for paths no longer present under this listing pass
320
+ // only when they sit in this outbox dir (other roots may share the module cache).
321
+ for (const key of outboxListCache.keys()) {
322
+ if (key.startsWith(dir + path.sep) && !seen.has(key)) {
323
+ outboxListCache.delete(key);
324
+ }
252
325
  }
253
326
  return ops.sort((a, b) => {
254
327
  const byTime = a.createdAt.localeCompare(b.createdAt);
@@ -266,21 +339,68 @@ export function outboxStats(root) {
266
339
  acked: ops.filter((o) => o.delivery === "acked").length,
267
340
  };
268
341
  }
342
+ function isOutboxDue(op, nowIso) {
343
+ return !op.nextAttemptAt || op.nextAttemptAt <= nowIso;
344
+ }
345
+ function outboxFifoCompare(a, b) {
346
+ const byCreated = a.createdAt.localeCompare(b.createdAt);
347
+ if (byCreated !== 0)
348
+ return byCreated;
349
+ return a.operationId.localeCompare(b.operationId);
350
+ }
351
+ /** Contiguous leading due ops; stops at the first not-yet-due op (no skip-ahead). */
352
+ function duePrefixForSession(ops, nowIso) {
353
+ const prefix = [];
354
+ for (const op of ops) {
355
+ if (!isOutboxDue(op, nowIso))
356
+ break;
357
+ prefix.push(op);
358
+ }
359
+ return prefix;
360
+ }
269
361
  /**
270
362
  * Replay queued (and recover lost-receipt) operations.
271
363
  * FIFO per sessionId; bounded parallelism across sessions.
364
+ * Skips ops whose nextAttemptAt is in the future; caps work per call.
272
365
  */
273
366
  export async function replayOutbox(root, deliver, opts = {}) {
274
367
  const parallel = opts.parallel ?? DEFAULT_PARALLEL;
275
368
  const now = opts.now ?? (() => new Date());
369
+ const random = opts.random ?? Math.random;
370
+ const maxOps = opts.maxOps ?? OUTBOX_REPLAY_MAX_OPS;
371
+ const nowIso = now().toISOString();
276
372
  const pending = listOutboxOperations(root).filter((o) => o.delivery === "queued" || (o.delivery === "acked" && !o.receiptId));
277
- // Group by session, preserve FIFO within each.
278
- const bySession = new Map();
373
+ // Group ALL pending by session first (FIFO), then take each session's due prefix.
374
+ const pendingBySession = new Map();
279
375
  for (const op of pending) {
280
- const list = bySession.get(op.sessionId) ?? [];
376
+ const list = pendingBySession.get(op.sessionId) ?? [];
281
377
  list.push(op);
282
- bySession.set(op.sessionId, list);
378
+ pendingBySession.set(op.sessionId, list);
379
+ }
380
+ for (const list of pendingBySession.values()) {
381
+ list.sort(outboxFifoCompare);
283
382
  }
383
+ const sessionPrefixes = [];
384
+ for (const [sessionId, list] of pendingBySession) {
385
+ const prefix = duePrefixForSession(list, nowIso);
386
+ if (prefix.length > 0) {
387
+ sessionPrefixes.push({ sessionId, ops: prefix });
388
+ }
389
+ }
390
+ // Cap by head age: oldest session heads first; take leading due ops only.
391
+ sessionPrefixes.sort((a, b) => outboxFifoCompare(a.ops[0], b.ops[0]));
392
+ const bySession = new Map();
393
+ let selectedCount = 0;
394
+ let eligibleDue = 0;
395
+ for (const { sessionId, ops } of sessionPrefixes) {
396
+ eligibleDue += ops.length;
397
+ if (selectedCount >= maxOps)
398
+ continue;
399
+ const take = Math.min(ops.length, maxOps - selectedCount);
400
+ bySession.set(sessionId, ops.slice(0, take));
401
+ selectedCount += take;
402
+ }
403
+ const skipped = Math.max(0, eligibleDue - selectedCount);
284
404
  let delivered = 0;
285
405
  let queued = 0;
286
406
  let quarantined = 0;
@@ -297,8 +417,14 @@ export async function replayOutbox(root, deliver, opts = {}) {
297
417
  delivered += 1;
298
418
  }
299
419
  else if (result.retryable) {
300
- markOutboxQueued(op.operationId, root, result.code, now);
301
- queued += 1;
420
+ if (op.attemptCount + 1 >= OUTBOX_MAX_ATTEMPTS) {
421
+ markOutboxQuarantined(op.operationId, root, `${result.code}_MAX_ATTEMPTS`, now);
422
+ quarantined += 1;
423
+ }
424
+ else {
425
+ markOutboxQueued(op.operationId, root, result.code, now, random);
426
+ queued += 1;
427
+ }
302
428
  // Stop this session's FIFO on transient failure.
303
429
  break;
304
430
  }
@@ -320,7 +446,7 @@ export async function replayOutbox(root, deliver, opts = {}) {
320
446
  }
321
447
  const workers = Array.from({ length: Math.min(parallel, sessions.length) }, () => worker());
322
448
  await Promise.all(workers);
323
- return { delivered, queued, quarantined };
449
+ return { delivered, queued, quarantined, skipped };
324
450
  }
325
451
  /** Remove acked ops older than retention (optional GC; not required by AC). */
326
452
  export function removeAckedOutbox(root, olderThanMs, now = Date.now()) {
@@ -330,8 +456,10 @@ export function removeAckedOutbox(root, olderThanMs, now = Date.now()) {
330
456
  continue;
331
457
  const age = now - Date.parse(op.updatedAt);
332
458
  if (Number.isFinite(age) && age > olderThanMs) {
459
+ const filePath = workContextOutboxPath(op.operationId, root);
333
460
  try {
334
- fs.unlinkSync(workContextOutboxPath(op.operationId, root));
461
+ fs.unlinkSync(filePath);
462
+ invalidateOutboxListCacheEntry(filePath);
335
463
  removed += 1;
336
464
  }
337
465
  catch {
@@ -349,6 +477,7 @@ export function quarantineCorruptOutboxFile(filePath, root) {
349
477
  const dest = path.join(workContextOutboxDir(root), `.quarantine.${base}`);
350
478
  try {
351
479
  fs.renameSync(filePath, dest);
480
+ invalidateOutboxListCacheEntry(filePath);
352
481
  }
353
482
  catch {
354
483
  /* ignore */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.108.14",
3
+ "version": "5.108.16",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {