@indigoai-us/hq-cli 5.108.14 → 5.108.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/dist/lib/mesh/api.js +6 -1
- package/dist/lib/mesh/client.js +26 -6
- package/dist/lib/mesh/live/daemon/presence.d.ts +20 -2
- package/dist/lib/mesh/live/daemon/presence.js +25 -5
- package/dist/lib/mesh/live/daemon/run.d.ts +11 -0
- package/dist/lib/mesh/live/daemon/run.js +36 -8
- package/dist/lib/mesh/live/flush.d.ts +22 -1
- package/dist/lib/mesh/live/flush.js +137 -31
- package/dist/lib/mesh/live/index.d.ts +1 -1
- package/dist/lib/mesh/live/index.js +1 -1
- package/dist/lib/mesh/live/spool.js +6 -1
- package/dist/lib/work-context/outbox.d.ts +23 -3
- package/dist/lib/work-context/outbox.js +143 -14
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,44 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.108.16] — 2026-09-07
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Work Mesh Live daemon no longer retries every queued work-context outbox
|
|
10
|
+
operation on every flush cycle. Outbox replay now applies per-operation
|
|
11
|
+
exponential backoff (`nextAttemptAt`, 30s base → 6h cap with jitter), a
|
|
12
|
+
per-cycle cap (200 due ops), an attempt ceiling (50 → quarantine as
|
|
13
|
+
`<code>_MAX_ATTEMPTS`), cheaper listing that skips re-reading unchanged
|
|
14
|
+
outbox files, and classifies HTTP 404/409/410/412 (and other 4xx) from
|
|
15
|
+
transport errors as non-retryable instead of retrying them forever as
|
|
16
|
+
`TRANSPORT_ERROR`.
|
|
17
|
+
|
|
18
|
+
## [5.108.15] — 2026-09-07
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- Work Mesh Live presence daemon (`hq mesh daemon run`) no longer pins a CPU
|
|
23
|
+
core when the MQTT broker accepts a connection then closes it just past the
|
|
24
|
+
stable-connect grace window. The full-jitter reconnect backoff could collapse
|
|
25
|
+
to ~0 ms after the connect handler reset its counters, producing a tight
|
|
26
|
+
accept-then-close reconnect spin (each pass re-ran the SigV4 presign and
|
|
27
|
+
re-materialised the daemon state dir) with no network progress and no log
|
|
28
|
+
output. A hard `RECONNECT_MIN_DELAY_MS` (1s) floor now applies to every
|
|
29
|
+
reconnect path — normal backoff, the post-grace reset, and the refused-retry
|
|
30
|
+
path (which a server `Retry-After: 0` could otherwise drive to 0) — so the
|
|
31
|
+
delay can never shrink below 1s. Regression tests cover the reset and the
|
|
32
|
+
refused-retry floor. (#524)
|
|
33
|
+
- The Work Mesh Live daemon no longer burns CPU in a self-triggered flush loop.
|
|
34
|
+
Its spool directory watcher woke on files the daemon writes itself during a
|
|
35
|
+
flush (`held.jsonl` and the `*.claimed` files), so any box with held events
|
|
36
|
+
flushed every two seconds forever; the watcher now reacts only to producer
|
|
37
|
+
appends to `spool.jsonl`. Recovering a very large orphan claim file no longer
|
|
38
|
+
overflows the stack, mid-flush disposal is O(1), held events are re-attempted
|
|
39
|
+
at most every five minutes, and `held.jsonl` is capped at 20,000 lines with
|
|
40
|
+
the oldest overflow dead-lettered as `HELD_OVERFLOW` (loss-free under failure
|
|
41
|
+
and concurrent flushers). (#525)
|
|
42
|
+
|
|
5
43
|
## [5.108.14] — 2026-09-07
|
|
6
44
|
|
|
7
45
|
### Fixed
|
package/dist/lib/mesh/api.js
CHANGED
|
@@ -43,7 +43,12 @@ export async function meshJson(token, path, init = {}) {
|
|
|
43
43
|
}
|
|
44
44
|
if (!res.ok) {
|
|
45
45
|
const err = data;
|
|
46
|
-
|
|
46
|
+
// Always lead with the HTTP status: callers classify retryable vs permanent
|
|
47
|
+
// failures from the message (see createWorkSessionDeliverer). Without it a
|
|
48
|
+
// 400 whose body text matched no pattern was retried forever.
|
|
49
|
+
const detail = err.error || err.message || res.statusText || "request failed";
|
|
50
|
+
const code = typeof err.code === "string" && err.code ? `${err.code}: ` : "";
|
|
51
|
+
throw new Error(`${res.status} ${code}${detail}`);
|
|
47
52
|
}
|
|
48
53
|
return data;
|
|
49
54
|
}
|
package/dist/lib/mesh/client.js
CHANGED
|
@@ -46,17 +46,37 @@ export function createWorkSessionDeliverer(opts) {
|
|
|
46
46
|
}
|
|
47
47
|
catch (err) {
|
|
48
48
|
const message = err instanceof Error ? err.message : String(err);
|
|
49
|
-
// meshJson throws on !ok with error text; classify
|
|
50
|
-
|
|
49
|
+
// meshJson throws on !ok with error text; classify by leading status when present.
|
|
50
|
+
const statusMatch = /^(\d{3})\b/.exec(message);
|
|
51
|
+
if (statusMatch) {
|
|
52
|
+
const status = Number(statusMatch[1]);
|
|
53
|
+
if (status === 401 || status === 403) {
|
|
54
|
+
return { ok: false, retryable: false, code: "AUTH_DENIED" };
|
|
55
|
+
}
|
|
56
|
+
if (status === 400 || status === 422) {
|
|
57
|
+
return { ok: false, retryable: false, code: "VALIDATION_FAILED" };
|
|
58
|
+
}
|
|
59
|
+
if (status === 429 || (status >= 500 && status <= 599)) {
|
|
60
|
+
return { ok: false, retryable: true, code: `HTTP_${status}` };
|
|
61
|
+
}
|
|
62
|
+
if (status === 404 ||
|
|
63
|
+
status === 409 ||
|
|
64
|
+
status === 410 ||
|
|
65
|
+
status === 412 ||
|
|
66
|
+
(status >= 400 && status < 500)) {
|
|
67
|
+
return { ok: false, retryable: false, code: `HTTP_${status}` };
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (/unauthorized|forbidden|auth/i.test(message)) {
|
|
51
71
|
return { ok: false, retryable: false, code: "AUTH_DENIED" };
|
|
52
72
|
}
|
|
53
|
-
if (
|
|
73
|
+
if (/invalid|validation/i.test(message)) {
|
|
54
74
|
return { ok: false, retryable: false, code: "VALIDATION_FAILED" };
|
|
55
75
|
}
|
|
56
|
-
if (
|
|
76
|
+
if (/ECONN|ENOTFOUND|ETIMEDOUT|network|fetch failed/i.test(message)) {
|
|
57
77
|
return { ok: false, retryable: true, code: "NETWORK_OR_5XX" };
|
|
58
78
|
}
|
|
59
|
-
//
|
|
79
|
+
// No status in the message: keep queued (fail open to retry, not quarantine).
|
|
60
80
|
return { ok: false, retryable: true, code: "TRANSPORT_ERROR" };
|
|
61
81
|
}
|
|
62
82
|
};
|
|
@@ -73,7 +93,7 @@ function mapRegisterResponse(status, body, operationId) {
|
|
|
73
93
|
if (status >= 500 || status === 429) {
|
|
74
94
|
return { ok: false, retryable: true, code: `HTTP_${status}` };
|
|
75
95
|
}
|
|
76
|
-
if (status
|
|
96
|
+
if (status >= 400 && status < 500) {
|
|
77
97
|
return { ok: false, retryable: false, code: `HTTP_${status}` };
|
|
78
98
|
}
|
|
79
99
|
if (status === 0) {
|
|
@@ -20,6 +20,20 @@ export type MqttConnectionState = "idle" | "connecting" | "connected" | "reconne
|
|
|
20
20
|
* then closes immediately after a denied publish).
|
|
21
21
|
*/
|
|
22
22
|
export declare const CONNECT_STABLE_GRACE_MS = 5000;
|
|
23
|
+
/**
|
|
24
|
+
* Hard floor for the delay between MQTT presence reconnect attempts.
|
|
25
|
+
*
|
|
26
|
+
* The full-jitter backoff can otherwise compute ~0 ms: after a connection
|
|
27
|
+
* survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
|
|
28
|
+
* `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
|
|
29
|
+
* past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
|
|
30
|
+
* return `random() * base` — frequently a handful of ms, or 0 when `random()`
|
|
31
|
+
* is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
|
|
32
|
+
* presign and re-materialises the daemon state dir) that pins a CPU core with
|
|
33
|
+
* no network progress and no log output. This floor guarantees every reconnect
|
|
34
|
+
* waits at least this long regardless of the backoff/grace-reset state.
|
|
35
|
+
*/
|
|
36
|
+
export declare const RECONNECT_MIN_DELAY_MS = 1000;
|
|
23
37
|
/** Always emit the first N close info lines, then at most one per interval. */
|
|
24
38
|
export declare const CLOSE_LOG_ALWAYS_COUNT = 3;
|
|
25
39
|
export declare const CLOSE_LOG_INTERVAL_MS = 60000;
|
|
@@ -69,9 +83,13 @@ export interface PresenceClientOptions {
|
|
|
69
83
|
}
|
|
70
84
|
/**
|
|
71
85
|
* Full-jitter capped exponential backoff (1s base → 60s cap by default),
|
|
72
|
-
* floored at `previousMs` so consecutive failures never shrink the delay
|
|
86
|
+
* floored at `previousMs` so consecutive failures never shrink the delay and at
|
|
87
|
+
* `minMs` so the result can never collapse to ~0 (see
|
|
88
|
+
* {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
|
|
89
|
+
* accept-then-close reconnect spin after the connect handler resets the
|
|
90
|
+
* backoff counters.
|
|
73
91
|
*/
|
|
74
|
-
export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number): number;
|
|
92
|
+
export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
|
|
75
93
|
export declare function buildPresencePayload(input: {
|
|
76
94
|
status: "online" | "offline";
|
|
77
95
|
actorUid: string;
|
|
@@ -20,17 +20,35 @@ import { presignIotWssUrl } from "./presign.js";
|
|
|
20
20
|
* then closes immediately after a denied publish).
|
|
21
21
|
*/
|
|
22
22
|
export const CONNECT_STABLE_GRACE_MS = 5_000;
|
|
23
|
+
/**
|
|
24
|
+
* Hard floor for the delay between MQTT presence reconnect attempts.
|
|
25
|
+
*
|
|
26
|
+
* The full-jitter backoff can otherwise compute ~0 ms: after a connection
|
|
27
|
+
* survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
|
|
28
|
+
* `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
|
|
29
|
+
* past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
|
|
30
|
+
* return `random() * base` — frequently a handful of ms, or 0 when `random()`
|
|
31
|
+
* is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
|
|
32
|
+
* presign and re-materialises the daemon state dir) that pins a CPU core with
|
|
33
|
+
* no network progress and no log output. This floor guarantees every reconnect
|
|
34
|
+
* waits at least this long regardless of the backoff/grace-reset state.
|
|
35
|
+
*/
|
|
36
|
+
export const RECONNECT_MIN_DELAY_MS = 1_000;
|
|
23
37
|
/** Always emit the first N close info lines, then at most one per interval. */
|
|
24
38
|
export const CLOSE_LOG_ALWAYS_COUNT = 3;
|
|
25
39
|
export const CLOSE_LOG_INTERVAL_MS = 60_000;
|
|
26
40
|
/**
|
|
27
41
|
* Full-jitter capped exponential backoff (1s base → 60s cap by default),
|
|
28
|
-
* floored at `previousMs` so consecutive failures never shrink the delay
|
|
42
|
+
* floored at `previousMs` so consecutive failures never shrink the delay and at
|
|
43
|
+
* `minMs` so the result can never collapse to ~0 (see
|
|
44
|
+
* {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
|
|
45
|
+
* accept-then-close reconnect spin after the connect handler resets the
|
|
46
|
+
* backoff counters.
|
|
29
47
|
*/
|
|
30
|
-
export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0) {
|
|
48
|
+
export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, minMs = 0) {
|
|
31
49
|
const cap = Math.min(maxMs, baseMs * 2 ** attempt);
|
|
32
50
|
const raw = Math.max(0, random() * cap);
|
|
33
|
-
return Math.min(maxMs, Math.max(previousMs, raw));
|
|
51
|
+
return Math.min(maxMs, Math.max(previousMs, raw, minMs));
|
|
34
52
|
}
|
|
35
53
|
export function buildPresencePayload(input) {
|
|
36
54
|
return {
|
|
@@ -341,12 +359,14 @@ export class PresenceClient {
|
|
|
341
359
|
else {
|
|
342
360
|
delay = refusedRetryDelayMs(this.refusedRetryMs, this.random);
|
|
343
361
|
}
|
|
344
|
-
|
|
362
|
+
// Floor the refused retry too: a server `Retry-After: 0` (or a 0 env
|
|
363
|
+
// override) would otherwise clamp to 0 and hot-loop the refused path.
|
|
364
|
+
delay = Math.max(RECONNECT_MIN_DELAY_MS, clampRetryDelayMs(delay));
|
|
345
365
|
const nextRetryAt = new Date(this.timers.now() + delay).toISOString();
|
|
346
366
|
this.noteRefusal(refused, nextRetryAt);
|
|
347
367
|
}
|
|
348
368
|
else {
|
|
349
|
-
delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs);
|
|
369
|
+
delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs, RECONNECT_MIN_DELAY_MS);
|
|
350
370
|
this.lastBackoffMs = delay;
|
|
351
371
|
this.attempt += 1;
|
|
352
372
|
// Do not clear refusal here — MQTT close must retain doctor/status refusal
|
|
@@ -15,6 +15,16 @@ import { PresenceClient, type MqttConnectFn } from "./presence.js";
|
|
|
15
15
|
import { TranscriptWatcher, type TranscriptFs } from "./transcript-watch.js";
|
|
16
16
|
export declare const SPOOL_DEBOUNCE_MS = 2000;
|
|
17
17
|
export declare const FLUSH_INTERVAL_MS = 10000;
|
|
18
|
+
/**
|
|
19
|
+
* Only producer appends to spool.jsonl should wake the watcher. Every other
|
|
20
|
+
* file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
|
|
21
|
+
* live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
|
|
22
|
+
* during a flush; reacting to those turned each flush into a self-triggered
|
|
23
|
+
* flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
|
|
24
|
+
* core on fleet boxes with a non-empty held file). A null filename (platforms
|
|
25
|
+
* that do not report one) keeps the conservative behaviour and flushes.
|
|
26
|
+
*/
|
|
27
|
+
export declare function shouldFlushOnSpoolDirEvent(filename: string | Buffer | null | undefined): boolean;
|
|
18
28
|
export interface DaemonRunDeps {
|
|
19
29
|
home?: string;
|
|
20
30
|
env?: NodeJS.ProcessEnv;
|
|
@@ -31,6 +41,7 @@ export interface DaemonRunDeps {
|
|
|
31
41
|
delivered: number;
|
|
32
42
|
queued: number;
|
|
33
43
|
quarantined: number;
|
|
44
|
+
skipped?: number;
|
|
34
45
|
}>;
|
|
35
46
|
/** Injected board refresh (tests). */
|
|
36
47
|
refreshBoards?: () => Promise<{
|
|
@@ -17,7 +17,7 @@ import { ensureCognitoToken } from "../../../../utils/cognito-session.js";
|
|
|
17
17
|
import { replayOutbox } from "../../../work-context/outbox.js";
|
|
18
18
|
import { workContextRoot } from "../../../work-context/paths.js";
|
|
19
19
|
import { createSessionEventsPoster, resolveVaultApiBase, } from "../session-events-client.js";
|
|
20
|
-
import { flushSessionEvents } from "../flush.js";
|
|
20
|
+
import { flushSessionEvents, HELD_RETRY_INTERVAL_MS, } from "../flush.js";
|
|
21
21
|
import { workMeshRoot, workMeshSpoolPath } from "../paths.js";
|
|
22
22
|
import { createVaultBoardReader, refreshBoundSessionBoards, BOARD_REFRESH_INTERVAL_MS, } from "./board-refresh.js";
|
|
23
23
|
import { createContract3Fetcher, realTimerHost, } from "./credentials.js";
|
|
@@ -30,6 +30,20 @@ import { workMeshHeldPath } from "../paths.js";
|
|
|
30
30
|
import { noteHookSessionsFromSpoolFile, TRANSCRIPT_WATCH_INTERVAL_MS, TranscriptWatcher, } from "./transcript-watch.js";
|
|
31
31
|
export const SPOOL_DEBOUNCE_MS = 2_000;
|
|
32
32
|
export const FLUSH_INTERVAL_MS = 10_000;
|
|
33
|
+
/**
|
|
34
|
+
* Only producer appends to spool.jsonl should wake the watcher. Every other
|
|
35
|
+
* file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
|
|
36
|
+
* live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
|
|
37
|
+
* during a flush; reacting to those turned each flush into a self-triggered
|
|
38
|
+
* flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
|
|
39
|
+
* core on fleet boxes with a non-empty held file). A null filename (platforms
|
|
40
|
+
* that do not report one) keeps the conservative behaviour and flushes.
|
|
41
|
+
*/
|
|
42
|
+
export function shouldFlushOnSpoolDirEvent(filename) {
|
|
43
|
+
if (filename === null || filename === undefined)
|
|
44
|
+
return true;
|
|
45
|
+
return String(filename) === "spool.jsonl";
|
|
46
|
+
}
|
|
33
47
|
function alive(pid) {
|
|
34
48
|
try {
|
|
35
49
|
process.kill(pid, 0);
|
|
@@ -76,6 +90,8 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
76
90
|
// daemon went dark once its IoT credentials expired.
|
|
77
91
|
const defaultGetToken = async () => ensureCognitoToken({ interactive: false, tokenSource: "machine" });
|
|
78
92
|
const getToken = deps.getToken ?? defaultGetToken;
|
|
93
|
+
/** In-memory held re-attempt throttle (survives watch-triggered flushes). */
|
|
94
|
+
let lastHeldRetryAtMs = 0;
|
|
79
95
|
const flushFn = deps.flush ??
|
|
80
96
|
(async () => {
|
|
81
97
|
const t = await getToken();
|
|
@@ -87,6 +103,12 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
87
103
|
workMeshRoot: meshRoot,
|
|
88
104
|
workContextRoot: ctxRoot,
|
|
89
105
|
poster,
|
|
106
|
+
heldRetryIntervalMs: HELD_RETRY_INTERVAL_MS,
|
|
107
|
+
lastHeldRetryAtMs,
|
|
108
|
+
onHeldClaimed: (atMs) => {
|
|
109
|
+
lastHeldRetryAtMs = atMs;
|
|
110
|
+
},
|
|
111
|
+
log: (message) => log(dir, message),
|
|
90
112
|
});
|
|
91
113
|
});
|
|
92
114
|
const replayFn = deps.replayOutbox ??
|
|
@@ -233,7 +255,14 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
233
255
|
noteHookSessionsFromSpoolFile(workMeshHeldPath(meshRoot), lastHookEventAt, at);
|
|
234
256
|
const summary = await flushFn();
|
|
235
257
|
try {
|
|
236
|
-
await replayFn();
|
|
258
|
+
const replay = await replayFn();
|
|
259
|
+
const delivered = replay.delivered ?? 0;
|
|
260
|
+
const queued = replay.queued ?? 0;
|
|
261
|
+
const quarantined = replay.quarantined ?? 0;
|
|
262
|
+
const skipped = replay.skipped ?? 0;
|
|
263
|
+
if (delivered > 0 || queued > 0 || quarantined > 0 || skipped > 0) {
|
|
264
|
+
log(dir, `outbox replay: delivered=${delivered} queued=${queued} quarantined=${quarantined} skipped=${skipped}`);
|
|
265
|
+
}
|
|
237
266
|
}
|
|
238
267
|
catch (err) {
|
|
239
268
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -243,7 +272,10 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
243
272
|
lastFlushAt: now().toISOString(),
|
|
244
273
|
lastFlushResult: { ...summary, ok: true },
|
|
245
274
|
}, now);
|
|
246
|
-
if (summary.claimed > 0 ||
|
|
275
|
+
if (summary.claimed > 0 ||
|
|
276
|
+
summary.posted > 0 ||
|
|
277
|
+
summary.held > 0 ||
|
|
278
|
+
(summary.heldOverflow ?? 0) > 0) {
|
|
247
279
|
log(dir, `flush(${reason}): claimed=${summary.claimed} posted=${summary.posted} held=${summary.held}`);
|
|
248
280
|
}
|
|
249
281
|
}
|
|
@@ -292,11 +324,7 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
292
324
|
watcher = fs.watch(path.dirname(spoolPath), { persistent: true }, (_event, filename) => {
|
|
293
325
|
if (stopped)
|
|
294
326
|
return;
|
|
295
|
-
if (
|
|
296
|
-
filename === "spool.jsonl" ||
|
|
297
|
-
filename === "held.jsonl" ||
|
|
298
|
-
String(filename).startsWith("spool.") ||
|
|
299
|
-
String(filename).startsWith("held.")) {
|
|
327
|
+
if (shouldFlushOnSpoolDirEvent(filename)) {
|
|
300
328
|
scheduleDebouncedFlush();
|
|
301
329
|
}
|
|
302
330
|
});
|
|
@@ -11,7 +11,11 @@
|
|
|
11
11
|
import { type RandomFn, type SleepFn } from "./backoff.js";
|
|
12
12
|
import { type SessionEventsPoster } from "./session-events-client.js";
|
|
13
13
|
export declare const HELD_TTL_MS: number;
|
|
14
|
-
|
|
14
|
+
/** Re-attempt held events at most this often (spool still flushes every cycle). */
|
|
15
|
+
export declare const HELD_RETRY_INTERVAL_MS: number;
|
|
16
|
+
/** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
|
|
17
|
+
export declare const HELD_MAX_LINES = 20000;
|
|
18
|
+
export type HeldReason = "STATE_ABSENT" | "NEEDS_COMPANY" | "COMPANY_CONFLICT" | "HELD_EXPIRED" | "HELD_OVERFLOW";
|
|
15
19
|
export interface HeldLineRecord {
|
|
16
20
|
/** Original spool event fields. */
|
|
17
21
|
event: Record<string, unknown>;
|
|
@@ -26,6 +30,19 @@ export interface FlushDeps {
|
|
|
26
30
|
sleep?: SleepFn;
|
|
27
31
|
random?: RandomFn;
|
|
28
32
|
maxAttempts?: number;
|
|
33
|
+
/**
|
|
34
|
+
* Minimum gap between held re-attempts. Default HELD_RETRY_INTERVAL_MS.
|
|
35
|
+
* Pass 0 in tests to re-claim held every flush.
|
|
36
|
+
*/
|
|
37
|
+
heldRetryIntervalMs?: number;
|
|
38
|
+
/** Epoch ms of the last flush that claimed held (daemon keeps this in memory). */
|
|
39
|
+
lastHeldRetryAtMs?: number;
|
|
40
|
+
/** Invoked when this flush claims held, so the daemon can advance its timestamp. */
|
|
41
|
+
onHeldClaimed?: (atMs: number) => void;
|
|
42
|
+
/** Max lines retained in held.jsonl after a flush. Default HELD_MAX_LINES. */
|
|
43
|
+
heldMaxLines?: number;
|
|
44
|
+
/** Optional logger (daemon wires this); used once per overflow trim. */
|
|
45
|
+
log?: (message: string) => void;
|
|
29
46
|
}
|
|
30
47
|
export interface FlushSummary {
|
|
31
48
|
claimed: number;
|
|
@@ -35,6 +52,10 @@ export interface FlushSummary {
|
|
|
35
52
|
deadLettered: number;
|
|
36
53
|
restored: number;
|
|
37
54
|
batches: number;
|
|
55
|
+
/** True when this flush claimed/re-read held.jsonl. */
|
|
56
|
+
heldClaimed?: boolean;
|
|
57
|
+
/** Oldest held lines dead-lettered this flush due to HELD_MAX_LINES. */
|
|
58
|
+
heldOverflow?: number;
|
|
38
59
|
}
|
|
39
60
|
/**
|
|
40
61
|
* Flush spool + held. Safe to call concurrently with enqueue (claim-by-rename).
|
|
@@ -15,9 +15,13 @@ import { workContextRoot } from "../../work-context/paths.js";
|
|
|
15
15
|
import { defaultSleep, FLUSH_MAX_ATTEMPTS, fullJitterDelayMs, } from "./backoff.js";
|
|
16
16
|
import { LOCAL_ONLY_FIELDS, stripLocalOnlyFields } from "./format-spool-line.js";
|
|
17
17
|
import { SESSION_EVENTS_BATCH_MAX, WRITABLE_CONTEXT_STATUSES, parseSessionEventsBatchAck, } from "./session-events-client.js";
|
|
18
|
-
import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
|
|
19
|
-
import { workMeshRoot } from "./paths.js";
|
|
18
|
+
import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, countJsonlLines, ensureSpoolFile, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
|
|
19
|
+
import { workMeshHeldPath, workMeshRoot } from "./paths.js";
|
|
20
20
|
export const HELD_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
21
|
+
/** Re-attempt held events at most this often (spool still flushes every cycle). */
|
|
22
|
+
export const HELD_RETRY_INTERVAL_MS = 5 * 60 * 1000;
|
|
23
|
+
/** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
|
|
24
|
+
export const HELD_MAX_LINES = 20_000;
|
|
21
25
|
function parseLine(line) {
|
|
22
26
|
try {
|
|
23
27
|
const v = JSON.parse(line);
|
|
@@ -101,10 +105,22 @@ export async function flushSessionEvents(deps) {
|
|
|
101
105
|
const sleep = deps.sleep ?? defaultSleep;
|
|
102
106
|
const random = deps.random ?? Math.random;
|
|
103
107
|
const maxAttempts = deps.maxAttempts ?? FLUSH_MAX_ATTEMPTS;
|
|
104
|
-
const
|
|
108
|
+
const heldRetryIntervalMs = deps.heldRetryIntervalMs ?? HELD_RETRY_INTERVAL_MS;
|
|
109
|
+
const heldMaxLines = deps.heldMaxLines ?? HELD_MAX_LINES;
|
|
110
|
+
const nowMs = now().getTime();
|
|
111
|
+
const ts = `${nowMs}`;
|
|
105
112
|
const orphans = recoverOrphanClaims(deps.workMeshRoot);
|
|
106
113
|
const spoolClaim = claimSpool(deps.workMeshRoot, ts);
|
|
107
|
-
|
|
114
|
+
// Throttle held re-attempts; spool (fresh) events still flush every cycle.
|
|
115
|
+
const lastHeldRetryAtMs = deps.lastHeldRetryAtMs ?? 0;
|
|
116
|
+
const shouldClaimHeld = heldRetryIntervalMs <= 0 ||
|
|
117
|
+
nowMs - lastHeldRetryAtMs >= heldRetryIntervalMs;
|
|
118
|
+
const heldClaim = shouldClaimHeld
|
|
119
|
+
? claimHeld(deps.workMeshRoot, `${ts}.held`)
|
|
120
|
+
: { claimPath: null, lines: [] };
|
|
121
|
+
if (shouldClaimHeld) {
|
|
122
|
+
deps.onHeldClaimed?.(nowMs);
|
|
123
|
+
}
|
|
108
124
|
const claimPaths = [
|
|
109
125
|
...orphans.claimPaths,
|
|
110
126
|
spoolClaim.claimPath,
|
|
@@ -118,42 +134,45 @@ export async function flushSessionEvents(deps) {
|
|
|
118
134
|
deadLettered: 0,
|
|
119
135
|
restored: 0,
|
|
120
136
|
batches: 0,
|
|
137
|
+
heldClaimed: shouldClaimHeld,
|
|
138
|
+
heldOverflow: 0,
|
|
121
139
|
};
|
|
122
|
-
/** Original events still awaiting disposition (restored on throw). */
|
|
123
|
-
const pendingRestore =
|
|
140
|
+
/** Original events still awaiting disposition (restored on throw). O(1) delete. */
|
|
141
|
+
const pendingRestore = new Set();
|
|
124
142
|
const collected = [];
|
|
125
143
|
try {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
140
|
-
// Expire held lines older than 7 days.
|
|
141
|
-
if (heldAt) {
|
|
142
|
-
const age = now().getTime() - Date.parse(heldAt);
|
|
143
|
-
if (Number.isFinite(age) && age > HELD_TTL_MS) {
|
|
144
|
-
appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
|
|
144
|
+
// Iterate sources separately — avoid building one giant combined array.
|
|
145
|
+
for (const source of [orphans.lines, spoolClaim.lines, heldClaim.lines]) {
|
|
146
|
+
for (const line of source) {
|
|
147
|
+
summary.claimed += 1;
|
|
148
|
+
const parsed = parseLine(line);
|
|
149
|
+
if (!parsed) {
|
|
150
|
+
appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "SCHEMA_REJECTED", "parse_error", now().toISOString()), deps.workMeshRoot);
|
|
151
|
+
summary.deadLettered += 1;
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
const { event, heldAt } = unwrapHeld(parsed);
|
|
155
|
+
if (!sessionIdOf(event)) {
|
|
156
|
+
appendDeadLetterLine(deadLetterEnvelope(event, "SCHEMA_REJECTED", "missing_sessionId", now().toISOString()), deps.workMeshRoot);
|
|
145
157
|
summary.deadLettered += 1;
|
|
146
158
|
continue;
|
|
147
159
|
}
|
|
160
|
+
// Expire held lines older than 7 days.
|
|
161
|
+
if (heldAt) {
|
|
162
|
+
const age = now().getTime() - Date.parse(heldAt);
|
|
163
|
+
if (Number.isFinite(age) && age > HELD_TTL_MS) {
|
|
164
|
+
appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
|
|
165
|
+
summary.deadLettered += 1;
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
pendingRestore.add(event);
|
|
170
|
+
collected.push({ event, heldAt });
|
|
148
171
|
}
|
|
149
|
-
pendingRestore.push(event);
|
|
150
|
-
collected.push({ event, heldAt });
|
|
151
172
|
}
|
|
152
173
|
const bySession = groupBySession(collected);
|
|
153
174
|
const markDisposed = (event) => {
|
|
154
|
-
|
|
155
|
-
if (idx >= 0)
|
|
156
|
-
pendingRestore.splice(idx, 1);
|
|
175
|
+
pendingRestore.delete(event);
|
|
157
176
|
};
|
|
158
177
|
for (const [sessionId, items] of bySession) {
|
|
159
178
|
const state = readSessionState(sessionId, deps.workContextRoot);
|
|
@@ -238,13 +257,100 @@ export async function flushSessionEvents(deps) {
|
|
|
238
257
|
/* best-effort; orphan claim scan will retry next flush */
|
|
239
258
|
}
|
|
240
259
|
}
|
|
241
|
-
pendingRestore.
|
|
260
|
+
pendingRestore.clear();
|
|
242
261
|
for (const p of claimPaths) {
|
|
243
262
|
removeClaimFile(p);
|
|
244
263
|
}
|
|
245
264
|
}
|
|
265
|
+
const overflowed = trimHeldOverflow(deps.workMeshRoot, heldMaxLines, now, summary);
|
|
266
|
+
if (overflowed > 0) {
|
|
267
|
+
deps.log?.(`held overflow: dead-lettered ${overflowed} oldest events (cap=${heldMaxLines})`);
|
|
268
|
+
}
|
|
246
269
|
return summary;
|
|
247
270
|
}
|
|
271
|
+
/**
|
|
272
|
+
* After a flush, if held.jsonl exceeds maxLines, dead-letter the oldest
|
|
273
|
+
* overflow (by heldAt, not file position) with HELD_OVERFLOW and keep the rest.
|
|
274
|
+
*
|
|
275
|
+
* Durability rules:
|
|
276
|
+
* - Retained lines are APPENDED to held.jsonl, never written over it: another
|
|
277
|
+
* flusher (`hq mesh flush` beside the daemon) may have appended to a fresh
|
|
278
|
+
* held.jsonl after our claim renamed the old one away.
|
|
279
|
+
* - The overflow claim file is removed only after every dead letter and every
|
|
280
|
+
* retained line is on disk. If anything throws, the claim stays behind and
|
|
281
|
+
* recoverOrphanClaims re-reads it on the next flush (duplicates are possible,
|
|
282
|
+
* loss is not).
|
|
283
|
+
*/
|
|
284
|
+
function trimHeldOverflow(meshRoot, maxLines, now, summary) {
|
|
285
|
+
if (maxLines <= 0)
|
|
286
|
+
return 0;
|
|
287
|
+
const heldPath = workMeshHeldPath(meshRoot);
|
|
288
|
+
if (countJsonlLines(heldPath) <= maxLines)
|
|
289
|
+
return 0;
|
|
290
|
+
const claim = claimHeld(meshRoot, `${now().getTime()}.overflow`);
|
|
291
|
+
if (!claim.claimPath)
|
|
292
|
+
return 0;
|
|
293
|
+
if (claim.lines.length === 0) {
|
|
294
|
+
removeClaimFile(claim.claimPath);
|
|
295
|
+
return 0;
|
|
296
|
+
}
|
|
297
|
+
try {
|
|
298
|
+
if (claim.lines.length <= maxLines) {
|
|
299
|
+
appendHeldLines(meshRoot, claim.lines);
|
|
300
|
+
removeClaimFile(claim.claimPath);
|
|
301
|
+
return 0;
|
|
302
|
+
}
|
|
303
|
+
const overflow = claim.lines.length - maxLines;
|
|
304
|
+
// Oldest by heldAt: retries regroup held by session, so file order is not
|
|
305
|
+
// chronological. Unparseable lines sort first (dead-lettered first).
|
|
306
|
+
const ordered = claim.lines
|
|
307
|
+
.map((line, idx) => ({ line, idx, atMs: heldAtMs(line) }))
|
|
308
|
+
.sort((a, b) => a.atMs - b.atMs || a.idx - b.idx);
|
|
309
|
+
const toDead = ordered.slice(0, overflow);
|
|
310
|
+
const toKeep = ordered
|
|
311
|
+
.slice(overflow)
|
|
312
|
+
.sort((a, b) => a.idx - b.idx)
|
|
313
|
+
.map((x) => x.line);
|
|
314
|
+
const at = now().toISOString();
|
|
315
|
+
for (const { line } of toDead) {
|
|
316
|
+
const parsed = parseLine(line);
|
|
317
|
+
if (!parsed) {
|
|
318
|
+
appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
|
|
319
|
+
}
|
|
320
|
+
else {
|
|
321
|
+
const { event } = unwrapHeld(parsed);
|
|
322
|
+
appendDeadLetterLine(deadLetterEnvelope(event, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
|
|
323
|
+
}
|
|
324
|
+
summary.deadLettered += 1;
|
|
325
|
+
}
|
|
326
|
+
appendHeldLines(meshRoot, toKeep);
|
|
327
|
+
summary.heldOverflow = overflow;
|
|
328
|
+
removeClaimFile(claim.claimPath);
|
|
329
|
+
return overflow;
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
// Leave the claim for orphan recovery; never drop held events on a failed trim.
|
|
333
|
+
return 0;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/** heldAt of a held line as epoch ms; 0 when missing or unparseable. */
|
|
337
|
+
function heldAtMs(line) {
|
|
338
|
+
const parsed = parseLine(line);
|
|
339
|
+
if (!parsed)
|
|
340
|
+
return 0;
|
|
341
|
+
const { heldAt } = unwrapHeld(parsed);
|
|
342
|
+
if (!heldAt)
|
|
343
|
+
return 0;
|
|
344
|
+
const ms = Date.parse(heldAt);
|
|
345
|
+
return Number.isFinite(ms) ? ms : 0;
|
|
346
|
+
}
|
|
347
|
+
function appendHeldLines(meshRoot, lines) {
|
|
348
|
+
if (lines.length === 0)
|
|
349
|
+
return;
|
|
350
|
+
const heldPath = workMeshHeldPath(meshRoot);
|
|
351
|
+
ensureSpoolFile(heldPath);
|
|
352
|
+
fs.appendFileSync(heldPath, `${lines.join("\n")}\n`, { mode: 0o600 });
|
|
353
|
+
}
|
|
248
354
|
/**
|
|
249
355
|
* Account a 2xx batch: dead-letter rejected eventIds with their codes;
|
|
250
356
|
* count accepted+duplicates as posted; restore any eventId not accounted for.
|
|
@@ -13,7 +13,7 @@ export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath,
|
|
|
13
13
|
export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
|
|
14
14
|
export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
|
|
15
15
|
export type { EnqueueOptions, EnqueueResult } from "./enqueue.js";
|
|
16
|
-
export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
16
|
+
export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
17
17
|
export type { FlushDeps, FlushSummary, HeldReason } from "./flush.js";
|
|
18
18
|
export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
|
|
19
19
|
export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
|
|
@@ -9,7 +9,7 @@ export { encodeCrockford, generateUlid, isUlid } from "./ulid.js";
|
|
|
9
9
|
export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath, workMeshHeldPath, workMeshRoot, workMeshSpoolPath, } from "./paths.js";
|
|
10
10
|
export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
|
|
11
11
|
export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
|
|
12
|
-
export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
12
|
+
export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
13
13
|
export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
|
|
14
14
|
export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
|
|
15
15
|
export * from "./daemon/index.js";
|
|
@@ -153,7 +153,12 @@ export function recoverOrphanClaims(root) {
|
|
|
153
153
|
const claimPaths = listOrphanClaimFiles(root);
|
|
154
154
|
const lines = [];
|
|
155
155
|
for (const p of claimPaths) {
|
|
156
|
-
|
|
156
|
+
// Plain loop: Array#push(...huge) exceeds V8's argument limit (~65k–128k)
|
|
157
|
+
// and throws RangeError ("Maximum call stack size exceeded").
|
|
158
|
+
const orphanLines = readClaimFileLines(p);
|
|
159
|
+
for (const line of orphanLines) {
|
|
160
|
+
lines.push(line);
|
|
161
|
+
}
|
|
157
162
|
}
|
|
158
163
|
return { lines, claimPaths };
|
|
159
164
|
}
|
|
@@ -5,7 +5,15 @@
|
|
|
5
5
|
import type { DeliveryState } from "./contract.js";
|
|
6
6
|
import { WORK_CONTEXT_CONTRACT_VERSION } from "./contract.js";
|
|
7
7
|
/** Fields allowed on a durable outbox operation (privacy allowlist). */
|
|
8
|
-
export declare const OUTBOX_ALLOWLIST: readonly ["contractVersion", "operationId", "clientOperationId", "sessionId", "companyUid", "companySlug", "projectId", "taskId", "kind", "digest", "delivery", "createdAt", "updatedAt", "attemptCount", "lastErrorCode", "receiptId", "destinationCompanyUid", "destinationCompanySlug"];
|
|
8
|
+
export declare const OUTBOX_ALLOWLIST: readonly ["contractVersion", "operationId", "clientOperationId", "sessionId", "companyUid", "companySlug", "projectId", "taskId", "kind", "digest", "delivery", "createdAt", "updatedAt", "attemptCount", "lastErrorCode", "receiptId", "destinationCompanyUid", "destinationCompanySlug", "nextAttemptAt"];
|
|
9
|
+
/** Base delay for outbox retry backoff (attempt 1 → 30s). */
|
|
10
|
+
export declare const OUTBOX_RETRY_BASE_MS = 30000;
|
|
11
|
+
/** Cap for outbox retry backoff (6 hours). */
|
|
12
|
+
export declare const OUTBOX_RETRY_MAX_MS: number;
|
|
13
|
+
/** Quarantine after this many delivery attempts on retryable failures. */
|
|
14
|
+
export declare const OUTBOX_MAX_ATTEMPTS = 50;
|
|
15
|
+
/** Max due operations processed per replayOutbox call. */
|
|
16
|
+
export declare const OUTBOX_REPLAY_MAX_OPS = 200;
|
|
9
17
|
export type OutboxKind = "register" | "reconcile" | "migrate";
|
|
10
18
|
export interface OutboxOperation {
|
|
11
19
|
contractVersion: typeof WORK_CONTEXT_CONTRACT_VERSION;
|
|
@@ -27,6 +35,8 @@ export interface OutboxOperation {
|
|
|
27
35
|
/** Destination company for kind=migrate (source is companyUid). */
|
|
28
36
|
destinationCompanyUid?: string;
|
|
29
37
|
destinationCompanySlug?: string;
|
|
38
|
+
/** ISO time when the next delivery attempt is due (absent → due immediately). */
|
|
39
|
+
nextAttemptAt?: string;
|
|
30
40
|
}
|
|
31
41
|
export interface OutboxEnqueueInput {
|
|
32
42
|
clientOperationId: string;
|
|
@@ -40,6 +50,7 @@ export interface OutboxEnqueueInput {
|
|
|
40
50
|
destinationCompanySlug?: string;
|
|
41
51
|
now?: () => Date;
|
|
42
52
|
}
|
|
53
|
+
export declare function clearOutboxListCache(): void;
|
|
43
54
|
export declare function stableOperationId(clientOperationId: string, sessionId: string): string;
|
|
44
55
|
export declare function digestOperation(parts: {
|
|
45
56
|
sessionId: string;
|
|
@@ -52,6 +63,8 @@ export declare function digestOperation(parts: {
|
|
|
52
63
|
destinationCompanyUid?: string;
|
|
53
64
|
destinationCompanySlug?: string;
|
|
54
65
|
}): string;
|
|
66
|
+
/** Delay before next attempt: min(BASE * 2^(attemptCount-1), MAX), then [0.5, 1.0] jitter. */
|
|
67
|
+
export declare function outboxRetryDelayMs(attemptCount: number, random?: () => number): number;
|
|
55
68
|
/**
|
|
56
69
|
* Atomically enqueue (or idempotently return) an outbox operation.
|
|
57
70
|
* Disk/permission/lock failure → NotTrackingError (no network-only send).
|
|
@@ -60,9 +73,12 @@ export declare function enqueueOutbox(input: OutboxEnqueueInput, root: string):
|
|
|
60
73
|
export declare function readOutboxOperation(operationId: string, root: string): OutboxOperation | null;
|
|
61
74
|
export declare function updateOutboxOperation(op: OutboxOperation, root: string): void;
|
|
62
75
|
export declare function markOutboxAcked(operationId: string, root: string, receiptId: string, now?: () => Date): OutboxOperation | null;
|
|
63
|
-
export declare function markOutboxQueued(operationId: string, root: string, errorCode: string, now?: () => Date): OutboxOperation | null;
|
|
76
|
+
export declare function markOutboxQueued(operationId: string, root: string, errorCode: string, now?: () => Date, random?: () => number): OutboxOperation | null;
|
|
64
77
|
export declare function markOutboxQuarantined(operationId: string, root: string, errorCode: string, now?: () => Date): OutboxOperation | null;
|
|
65
|
-
export
|
|
78
|
+
export type OutboxListReadFile = (filePath: string) => string;
|
|
79
|
+
export declare function listOutboxOperations(root: string, deps?: {
|
|
80
|
+
readFile?: OutboxListReadFile;
|
|
81
|
+
}): OutboxOperation[];
|
|
66
82
|
export interface OutboxStats {
|
|
67
83
|
depth: number;
|
|
68
84
|
quarantined: number;
|
|
@@ -81,14 +97,18 @@ export type DeliverFn = (op: OutboxOperation) => Promise<{
|
|
|
81
97
|
/**
|
|
82
98
|
* Replay queued (and recover lost-receipt) operations.
|
|
83
99
|
* FIFO per sessionId; bounded parallelism across sessions.
|
|
100
|
+
* Skips ops whose nextAttemptAt is in the future; caps work per call.
|
|
84
101
|
*/
|
|
85
102
|
export declare function replayOutbox(root: string, deliver: DeliverFn, opts?: {
|
|
86
103
|
parallel?: number;
|
|
87
104
|
now?: () => Date;
|
|
105
|
+
maxOps?: number;
|
|
106
|
+
random?: () => number;
|
|
88
107
|
}): Promise<{
|
|
89
108
|
delivered: number;
|
|
90
109
|
queued: number;
|
|
91
110
|
quarantined: number;
|
|
111
|
+
skipped: number;
|
|
92
112
|
}>;
|
|
93
113
|
/** Remove acked ops older than retention (optional GC; not required by AC). */
|
|
94
114
|
export declare function removeAckedOutbox(root: string, olderThanMs: number, now?: number): number;
|
|
@@ -29,8 +29,24 @@ export const OUTBOX_ALLOWLIST = [
|
|
|
29
29
|
"receiptId",
|
|
30
30
|
"destinationCompanyUid",
|
|
31
31
|
"destinationCompanySlug",
|
|
32
|
+
"nextAttemptAt",
|
|
32
33
|
];
|
|
34
|
+
/** Base delay for outbox retry backoff (attempt 1 → 30s). */
|
|
35
|
+
export const OUTBOX_RETRY_BASE_MS = 30_000;
|
|
36
|
+
/** Cap for outbox retry backoff (6 hours). */
|
|
37
|
+
export const OUTBOX_RETRY_MAX_MS = 6 * 60 * 60 * 1000;
|
|
38
|
+
/** Quarantine after this many delivery attempts on retryable failures. */
|
|
39
|
+
export const OUTBOX_MAX_ATTEMPTS = 50;
|
|
40
|
+
/** Max due operations processed per replayOutbox call. */
|
|
41
|
+
export const OUTBOX_REPLAY_MAX_OPS = 200;
|
|
33
42
|
const DEFAULT_PARALLEL = 4;
|
|
43
|
+
const outboxListCache = new Map();
|
|
44
|
+
function invalidateOutboxListCacheEntry(filePath) {
|
|
45
|
+
outboxListCache.delete(filePath);
|
|
46
|
+
}
|
|
47
|
+
export function clearOutboxListCache() {
|
|
48
|
+
outboxListCache.clear();
|
|
49
|
+
}
|
|
34
50
|
export function stableOperationId(clientOperationId, sessionId) {
|
|
35
51
|
const h = crypto
|
|
36
52
|
.createHash("sha256")
|
|
@@ -85,8 +101,18 @@ function projectOutbox(op) {
|
|
|
85
101
|
if (op.destinationCompanySlug) {
|
|
86
102
|
out.destinationCompanySlug = op.destinationCompanySlug;
|
|
87
103
|
}
|
|
104
|
+
if (op.nextAttemptAt)
|
|
105
|
+
out.nextAttemptAt = op.nextAttemptAt;
|
|
88
106
|
return out;
|
|
89
107
|
}
|
|
108
|
+
/** Delay before next attempt: min(BASE * 2^(attemptCount-1), MAX), then [0.5, 1.0] jitter. */
|
|
109
|
+
export function outboxRetryDelayMs(attemptCount, random = Math.random) {
|
|
110
|
+
const exp = Math.max(0, attemptCount - 1);
|
|
111
|
+
const delay = Math.min(OUTBOX_RETRY_BASE_MS * 2 ** exp, OUTBOX_RETRY_MAX_MS);
|
|
112
|
+
const unit = random();
|
|
113
|
+
const factor = 0.5 + 0.5 * Math.min(1, Math.max(0, unit));
|
|
114
|
+
return Math.floor(delay * factor);
|
|
115
|
+
}
|
|
90
116
|
/**
|
|
91
117
|
* Atomically enqueue (or idempotently return) an outbox operation.
|
|
92
118
|
* Disk/permission/lock failure → NotTrackingError (no network-only send).
|
|
@@ -182,6 +208,7 @@ export function enqueueOutbox(input, root) {
|
|
|
182
208
|
catch {
|
|
183
209
|
/* ignore */
|
|
184
210
|
}
|
|
211
|
+
invalidateOutboxListCacheEntry(dest);
|
|
185
212
|
return op;
|
|
186
213
|
}
|
|
187
214
|
catch (err) {
|
|
@@ -204,7 +231,9 @@ export function readOutboxOperation(operationId, root) {
|
|
|
204
231
|
}
|
|
205
232
|
}
|
|
206
233
|
export function updateOutboxOperation(op, root) {
|
|
207
|
-
|
|
234
|
+
const filePath = workContextOutboxPath(op.operationId, root);
|
|
235
|
+
atomicWriteJson(filePath, projectOutbox(op));
|
|
236
|
+
invalidateOutboxListCacheEntry(filePath);
|
|
208
237
|
}
|
|
209
238
|
export function markOutboxAcked(operationId, root, receiptId, now = () => new Date()) {
|
|
210
239
|
const op = readOutboxOperation(operationId, root);
|
|
@@ -216,14 +245,17 @@ export function markOutboxAcked(operationId, root, receiptId, now = () => new Da
|
|
|
216
245
|
updateOutboxOperation(op, root);
|
|
217
246
|
return op;
|
|
218
247
|
}
|
|
219
|
-
export function markOutboxQueued(operationId, root, errorCode, now = () => new Date()) {
|
|
248
|
+
export function markOutboxQueued(operationId, root, errorCode, now = () => new Date(), random = Math.random) {
|
|
220
249
|
const op = readOutboxOperation(operationId, root);
|
|
221
250
|
if (!op)
|
|
222
251
|
return null;
|
|
223
252
|
op.delivery = "queued";
|
|
224
253
|
op.lastErrorCode = errorCode;
|
|
225
254
|
op.attemptCount += 1;
|
|
226
|
-
|
|
255
|
+
const at = now();
|
|
256
|
+
op.updatedAt = at.toISOString();
|
|
257
|
+
const delayMs = outboxRetryDelayMs(op.attemptCount, random);
|
|
258
|
+
op.nextAttemptAt = new Date(at.getTime() + delayMs).toISOString();
|
|
227
259
|
updateOutboxOperation(op, root);
|
|
228
260
|
return op;
|
|
229
261
|
}
|
|
@@ -238,17 +270,58 @@ export function markOutboxQuarantined(operationId, root, errorCode, now = () =>
|
|
|
238
270
|
updateOutboxOperation(op, root);
|
|
239
271
|
return op;
|
|
240
272
|
}
|
|
241
|
-
export function listOutboxOperations(root) {
|
|
273
|
+
export function listOutboxOperations(root, deps = {}) {
|
|
274
|
+
const readFile = deps.readFile ?? ((filePath) => fs.readFileSync(filePath, "utf8"));
|
|
242
275
|
const dir = workContextOutboxDir(root);
|
|
243
276
|
if (!fs.existsSync(dir))
|
|
244
277
|
return [];
|
|
245
278
|
const ops = [];
|
|
279
|
+
const seen = new Set();
|
|
246
280
|
for (const name of fs.readdirSync(dir)) {
|
|
247
281
|
if (!name.endsWith(".json") || name.startsWith("."))
|
|
248
282
|
continue;
|
|
249
|
-
const
|
|
250
|
-
if (
|
|
283
|
+
const operationId = name.replace(/\.json$/, "");
|
|
284
|
+
if (!isSafeWorkContextSegment(operationId))
|
|
285
|
+
continue;
|
|
286
|
+
const filePath = workContextOutboxPath(operationId, root);
|
|
287
|
+
seen.add(filePath);
|
|
288
|
+
let st;
|
|
289
|
+
try {
|
|
290
|
+
st = fs.statSync(filePath);
|
|
291
|
+
}
|
|
292
|
+
catch {
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
const cached = outboxListCache.get(filePath);
|
|
296
|
+
if (cached &&
|
|
297
|
+
cached.ino === st.ino &&
|
|
298
|
+
cached.mtimeMs === st.mtimeMs &&
|
|
299
|
+
cached.ctimeMs === st.ctimeMs &&
|
|
300
|
+
cached.size === st.size) {
|
|
301
|
+
ops.push(cached.op);
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
try {
|
|
305
|
+
const op = JSON.parse(readFile(filePath));
|
|
306
|
+
outboxListCache.set(filePath, {
|
|
307
|
+
ino: st.ino,
|
|
308
|
+
mtimeMs: st.mtimeMs,
|
|
309
|
+
ctimeMs: st.ctimeMs,
|
|
310
|
+
size: st.size,
|
|
311
|
+
op,
|
|
312
|
+
});
|
|
251
313
|
ops.push(op);
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
/* ignore corrupt */
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
// Drop cache entries for paths no longer present under this listing pass
|
|
320
|
+
// only when they sit in this outbox dir (other roots may share the module cache).
|
|
321
|
+
for (const key of outboxListCache.keys()) {
|
|
322
|
+
if (key.startsWith(dir + path.sep) && !seen.has(key)) {
|
|
323
|
+
outboxListCache.delete(key);
|
|
324
|
+
}
|
|
252
325
|
}
|
|
253
326
|
return ops.sort((a, b) => {
|
|
254
327
|
const byTime = a.createdAt.localeCompare(b.createdAt);
|
|
@@ -266,21 +339,68 @@ export function outboxStats(root) {
|
|
|
266
339
|
acked: ops.filter((o) => o.delivery === "acked").length,
|
|
267
340
|
};
|
|
268
341
|
}
|
|
342
|
+
function isOutboxDue(op, nowIso) {
|
|
343
|
+
return !op.nextAttemptAt || op.nextAttemptAt <= nowIso;
|
|
344
|
+
}
|
|
345
|
+
function outboxFifoCompare(a, b) {
|
|
346
|
+
const byCreated = a.createdAt.localeCompare(b.createdAt);
|
|
347
|
+
if (byCreated !== 0)
|
|
348
|
+
return byCreated;
|
|
349
|
+
return a.operationId.localeCompare(b.operationId);
|
|
350
|
+
}
|
|
351
|
+
/** Contiguous leading due ops; stops at the first not-yet-due op (no skip-ahead). */
|
|
352
|
+
function duePrefixForSession(ops, nowIso) {
|
|
353
|
+
const prefix = [];
|
|
354
|
+
for (const op of ops) {
|
|
355
|
+
if (!isOutboxDue(op, nowIso))
|
|
356
|
+
break;
|
|
357
|
+
prefix.push(op);
|
|
358
|
+
}
|
|
359
|
+
return prefix;
|
|
360
|
+
}
|
|
269
361
|
/**
|
|
270
362
|
* Replay queued (and recover lost-receipt) operations.
|
|
271
363
|
* FIFO per sessionId; bounded parallelism across sessions.
|
|
364
|
+
* Skips ops whose nextAttemptAt is in the future; caps work per call.
|
|
272
365
|
*/
|
|
273
366
|
export async function replayOutbox(root, deliver, opts = {}) {
|
|
274
367
|
const parallel = opts.parallel ?? DEFAULT_PARALLEL;
|
|
275
368
|
const now = opts.now ?? (() => new Date());
|
|
369
|
+
const random = opts.random ?? Math.random;
|
|
370
|
+
const maxOps = opts.maxOps ?? OUTBOX_REPLAY_MAX_OPS;
|
|
371
|
+
const nowIso = now().toISOString();
|
|
276
372
|
const pending = listOutboxOperations(root).filter((o) => o.delivery === "queued" || (o.delivery === "acked" && !o.receiptId));
|
|
277
|
-
// Group by session
|
|
278
|
-
const
|
|
373
|
+
// Group ALL pending by session first (FIFO), then take each session's due prefix.
|
|
374
|
+
const pendingBySession = new Map();
|
|
279
375
|
for (const op of pending) {
|
|
280
|
-
const list =
|
|
376
|
+
const list = pendingBySession.get(op.sessionId) ?? [];
|
|
281
377
|
list.push(op);
|
|
282
|
-
|
|
378
|
+
pendingBySession.set(op.sessionId, list);
|
|
379
|
+
}
|
|
380
|
+
for (const list of pendingBySession.values()) {
|
|
381
|
+
list.sort(outboxFifoCompare);
|
|
283
382
|
}
|
|
383
|
+
const sessionPrefixes = [];
|
|
384
|
+
for (const [sessionId, list] of pendingBySession) {
|
|
385
|
+
const prefix = duePrefixForSession(list, nowIso);
|
|
386
|
+
if (prefix.length > 0) {
|
|
387
|
+
sessionPrefixes.push({ sessionId, ops: prefix });
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
// Cap by head age: oldest session heads first; take leading due ops only.
|
|
391
|
+
sessionPrefixes.sort((a, b) => outboxFifoCompare(a.ops[0], b.ops[0]));
|
|
392
|
+
const bySession = new Map();
|
|
393
|
+
let selectedCount = 0;
|
|
394
|
+
let eligibleDue = 0;
|
|
395
|
+
for (const { sessionId, ops } of sessionPrefixes) {
|
|
396
|
+
eligibleDue += ops.length;
|
|
397
|
+
if (selectedCount >= maxOps)
|
|
398
|
+
continue;
|
|
399
|
+
const take = Math.min(ops.length, maxOps - selectedCount);
|
|
400
|
+
bySession.set(sessionId, ops.slice(0, take));
|
|
401
|
+
selectedCount += take;
|
|
402
|
+
}
|
|
403
|
+
const skipped = Math.max(0, eligibleDue - selectedCount);
|
|
284
404
|
let delivered = 0;
|
|
285
405
|
let queued = 0;
|
|
286
406
|
let quarantined = 0;
|
|
@@ -297,8 +417,14 @@ export async function replayOutbox(root, deliver, opts = {}) {
|
|
|
297
417
|
delivered += 1;
|
|
298
418
|
}
|
|
299
419
|
else if (result.retryable) {
|
|
300
|
-
|
|
301
|
-
|
|
420
|
+
if (op.attemptCount + 1 >= OUTBOX_MAX_ATTEMPTS) {
|
|
421
|
+
markOutboxQuarantined(op.operationId, root, `${result.code}_MAX_ATTEMPTS`, now);
|
|
422
|
+
quarantined += 1;
|
|
423
|
+
}
|
|
424
|
+
else {
|
|
425
|
+
markOutboxQueued(op.operationId, root, result.code, now, random);
|
|
426
|
+
queued += 1;
|
|
427
|
+
}
|
|
302
428
|
// Stop this session's FIFO on transient failure.
|
|
303
429
|
break;
|
|
304
430
|
}
|
|
@@ -320,7 +446,7 @@ export async function replayOutbox(root, deliver, opts = {}) {
|
|
|
320
446
|
}
|
|
321
447
|
const workers = Array.from({ length: Math.min(parallel, sessions.length) }, () => worker());
|
|
322
448
|
await Promise.all(workers);
|
|
323
|
-
return { delivered, queued, quarantined };
|
|
449
|
+
return { delivered, queued, quarantined, skipped };
|
|
324
450
|
}
|
|
325
451
|
/** Remove acked ops older than retention (optional GC; not required by AC). */
|
|
326
452
|
export function removeAckedOutbox(root, olderThanMs, now = Date.now()) {
|
|
@@ -330,8 +456,10 @@ export function removeAckedOutbox(root, olderThanMs, now = Date.now()) {
|
|
|
330
456
|
continue;
|
|
331
457
|
const age = now - Date.parse(op.updatedAt);
|
|
332
458
|
if (Number.isFinite(age) && age > olderThanMs) {
|
|
459
|
+
const filePath = workContextOutboxPath(op.operationId, root);
|
|
333
460
|
try {
|
|
334
|
-
fs.unlinkSync(
|
|
461
|
+
fs.unlinkSync(filePath);
|
|
462
|
+
invalidateOutboxListCacheEntry(filePath);
|
|
335
463
|
removed += 1;
|
|
336
464
|
}
|
|
337
465
|
catch {
|
|
@@ -349,6 +477,7 @@ export function quarantineCorruptOutboxFile(filePath, root) {
|
|
|
349
477
|
const dest = path.join(workContextOutboxDir(root), `.quarantine.${base}`);
|
|
350
478
|
try {
|
|
351
479
|
fs.renameSync(filePath, dest);
|
|
480
|
+
invalidateOutboxListCacheEntry(filePath);
|
|
352
481
|
}
|
|
353
482
|
catch {
|
|
354
483
|
/* ignore */
|