@indigoai-us/hq-cli 5.108.14 → 5.108.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/lib/mesh/live/daemon/presence.d.ts +20 -2
- package/dist/lib/mesh/live/daemon/presence.js +25 -5
- package/dist/lib/mesh/live/daemon/run.d.ts +10 -0
- package/dist/lib/mesh/live/daemon/run.js +28 -7
- package/dist/lib/mesh/live/flush.d.ts +22 -1
- package/dist/lib/mesh/live/flush.js +137 -31
- package/dist/lib/mesh/live/index.d.ts +1 -1
- package/dist/lib/mesh/live/index.js +1 -1
- package/dist/lib/mesh/live/spool.js +6 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.108.15] — 2026-09-07
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Work Mesh Live presence daemon (`hq mesh daemon run`) no longer pins a CPU
|
|
10
|
+
core when the MQTT broker accepts a connection then closes it just past the
|
|
11
|
+
stable-connect grace window. The full-jitter reconnect backoff could collapse
|
|
12
|
+
to ~0 ms after the connect handler reset its counters, producing a tight
|
|
13
|
+
accept-then-close reconnect spin (each pass re-ran the SigV4 presign and
|
|
14
|
+
re-materialised the daemon state dir) with no network progress and no log
|
|
15
|
+
output. A hard `RECONNECT_MIN_DELAY_MS` (1s) floor now applies to every
|
|
16
|
+
reconnect path — normal backoff, the post-grace reset, and the refused-retry
|
|
17
|
+
path (which a server `Retry-After: 0` could otherwise drive to 0) — so the
|
|
18
|
+
delay can never shrink below 1s. Regression tests cover the reset and the
|
|
19
|
+
refused-retry floor. (#524)
|
|
20
|
+
- The Work Mesh Live daemon no longer burns CPU in a self-triggered flush loop.
|
|
21
|
+
Its spool directory watcher woke on files the daemon writes itself during a
|
|
22
|
+
flush (`held.jsonl` and the `*.claimed` files), so any box with held events
|
|
23
|
+
flushed every two seconds forever; the watcher now reacts only to producer
|
|
24
|
+
appends to `spool.jsonl`. Recovering a very large orphan claim file no longer
|
|
25
|
+
overflows the stack, mid-flush disposal is O(1), held events are re-attempted
|
|
26
|
+
at most every five minutes, and `held.jsonl` is capped at 20,000 lines with
|
|
27
|
+
the oldest overflow dead-lettered as `HELD_OVERFLOW` (loss-free under failure
|
|
28
|
+
and concurrent flushers). (#525)
|
|
29
|
+
|
|
5
30
|
## [5.108.14] — 2026-09-07
|
|
6
31
|
|
|
7
32
|
### Fixed
|
|
@@ -20,6 +20,20 @@ export type MqttConnectionState = "idle" | "connecting" | "connected" | "reconne
|
|
|
20
20
|
* then closes immediately after a denied publish).
|
|
21
21
|
*/
|
|
22
22
|
export declare const CONNECT_STABLE_GRACE_MS = 5000;
|
|
23
|
+
/**
|
|
24
|
+
* Hard floor for the delay between MQTT presence reconnect attempts.
|
|
25
|
+
*
|
|
26
|
+
* The full-jitter backoff can otherwise compute ~0 ms: after a connection
|
|
27
|
+
* survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
|
|
28
|
+
* `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
|
|
29
|
+
* past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
|
|
30
|
+
* return `random() * base` — frequently a handful of ms, or 0 when `random()`
|
|
31
|
+
* is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
|
|
32
|
+
* presign and re-materialises the daemon state dir) that pins a CPU core with
|
|
33
|
+
* no network progress and no log output. This floor guarantees every reconnect
|
|
34
|
+
* waits at least this long regardless of the backoff/grace-reset state.
|
|
35
|
+
*/
|
|
36
|
+
export declare const RECONNECT_MIN_DELAY_MS = 1000;
|
|
23
37
|
/** Always emit the first N close info lines, then at most one per interval. */
|
|
24
38
|
export declare const CLOSE_LOG_ALWAYS_COUNT = 3;
|
|
25
39
|
export declare const CLOSE_LOG_INTERVAL_MS = 60000;
|
|
@@ -69,9 +83,13 @@ export interface PresenceClientOptions {
|
|
|
69
83
|
}
|
|
70
84
|
/**
|
|
71
85
|
* Full-jitter capped exponential backoff (1s base → 60s cap by default),
|
|
72
|
-
* floored at `previousMs` so consecutive failures never shrink the delay
|
|
86
|
+
* floored at `previousMs` so consecutive failures never shrink the delay and at
|
|
87
|
+
* `minMs` so the result can never collapse to ~0 (see
|
|
88
|
+
* {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
|
|
89
|
+
* accept-then-close reconnect spin after the connect handler resets the
|
|
90
|
+
* backoff counters.
|
|
73
91
|
*/
|
|
74
|
-
export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number): number;
|
|
92
|
+
export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
|
|
75
93
|
export declare function buildPresencePayload(input: {
|
|
76
94
|
status: "online" | "offline";
|
|
77
95
|
actorUid: string;
|
|
@@ -20,17 +20,35 @@ import { presignIotWssUrl } from "./presign.js";
|
|
|
20
20
|
* then closes immediately after a denied publish).
|
|
21
21
|
*/
|
|
22
22
|
export const CONNECT_STABLE_GRACE_MS = 5_000;
|
|
23
|
+
/**
|
|
24
|
+
* Hard floor for the delay between MQTT presence reconnect attempts.
|
|
25
|
+
*
|
|
26
|
+
* The full-jitter backoff can otherwise compute ~0 ms: after a connection
|
|
27
|
+
* survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
|
|
28
|
+
* `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
|
|
29
|
+
* past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
|
|
30
|
+
* return `random() * base` — frequently a handful of ms, or 0 when `random()`
|
|
31
|
+
* is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
|
|
32
|
+
* presign and re-materialises the daemon state dir) that pins a CPU core with
|
|
33
|
+
* no network progress and no log output. This floor guarantees every reconnect
|
|
34
|
+
* waits at least this long regardless of the backoff/grace-reset state.
|
|
35
|
+
*/
|
|
36
|
+
export const RECONNECT_MIN_DELAY_MS = 1_000;
|
|
23
37
|
/** Always emit the first N close info lines, then at most one per interval. */
|
|
24
38
|
export const CLOSE_LOG_ALWAYS_COUNT = 3;
|
|
25
39
|
export const CLOSE_LOG_INTERVAL_MS = 60_000;
|
|
26
40
|
/**
|
|
27
41
|
* Full-jitter capped exponential backoff (1s base → 60s cap by default),
|
|
28
|
-
* floored at `previousMs` so consecutive failures never shrink the delay
|
|
42
|
+
* floored at `previousMs` so consecutive failures never shrink the delay and at
|
|
43
|
+
* `minMs` so the result can never collapse to ~0 (see
|
|
44
|
+
* {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
|
|
45
|
+
* accept-then-close reconnect spin after the connect handler resets the
|
|
46
|
+
* backoff counters.
|
|
29
47
|
*/
|
|
30
|
-
export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0) {
|
|
48
|
+
export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, minMs = 0) {
|
|
31
49
|
const cap = Math.min(maxMs, baseMs * 2 ** attempt);
|
|
32
50
|
const raw = Math.max(0, random() * cap);
|
|
33
|
-
return Math.min(maxMs, Math.max(previousMs, raw));
|
|
51
|
+
return Math.min(maxMs, Math.max(previousMs, raw, minMs));
|
|
34
52
|
}
|
|
35
53
|
export function buildPresencePayload(input) {
|
|
36
54
|
return {
|
|
@@ -341,12 +359,14 @@ export class PresenceClient {
|
|
|
341
359
|
else {
|
|
342
360
|
delay = refusedRetryDelayMs(this.refusedRetryMs, this.random);
|
|
343
361
|
}
|
|
344
|
-
|
|
362
|
+
// Floor the refused retry too: a server `Retry-After: 0` (or a 0 env
|
|
363
|
+
// override) would otherwise clamp to 0 and hot-loop the refused path.
|
|
364
|
+
delay = Math.max(RECONNECT_MIN_DELAY_MS, clampRetryDelayMs(delay));
|
|
345
365
|
const nextRetryAt = new Date(this.timers.now() + delay).toISOString();
|
|
346
366
|
this.noteRefusal(refused, nextRetryAt);
|
|
347
367
|
}
|
|
348
368
|
else {
|
|
349
|
-
delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs);
|
|
369
|
+
delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs, RECONNECT_MIN_DELAY_MS);
|
|
350
370
|
this.lastBackoffMs = delay;
|
|
351
371
|
this.attempt += 1;
|
|
352
372
|
// Do not clear refusal here — MQTT close must retain doctor/status refusal
|
|
@@ -15,6 +15,16 @@ import { PresenceClient, type MqttConnectFn } from "./presence.js";
|
|
|
15
15
|
import { TranscriptWatcher, type TranscriptFs } from "./transcript-watch.js";
|
|
16
16
|
export declare const SPOOL_DEBOUNCE_MS = 2000;
|
|
17
17
|
export declare const FLUSH_INTERVAL_MS = 10000;
|
|
18
|
+
/**
|
|
19
|
+
* Only producer appends to spool.jsonl should wake the watcher. Every other
|
|
20
|
+
* file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
|
|
21
|
+
* live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
|
|
22
|
+
* during a flush; reacting to those turned each flush into a self-triggered
|
|
23
|
+
* flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
|
|
24
|
+
* core on fleet boxes with a non-empty held file). A null filename (platforms
|
|
25
|
+
* that do not report one) keeps the conservative behaviour and flushes.
|
|
26
|
+
*/
|
|
27
|
+
export declare function shouldFlushOnSpoolDirEvent(filename: string | Buffer | null | undefined): boolean;
|
|
18
28
|
export interface DaemonRunDeps {
|
|
19
29
|
home?: string;
|
|
20
30
|
env?: NodeJS.ProcessEnv;
|
|
@@ -17,7 +17,7 @@ import { ensureCognitoToken } from "../../../../utils/cognito-session.js";
|
|
|
17
17
|
import { replayOutbox } from "../../../work-context/outbox.js";
|
|
18
18
|
import { workContextRoot } from "../../../work-context/paths.js";
|
|
19
19
|
import { createSessionEventsPoster, resolveVaultApiBase, } from "../session-events-client.js";
|
|
20
|
-
import { flushSessionEvents } from "../flush.js";
|
|
20
|
+
import { flushSessionEvents, HELD_RETRY_INTERVAL_MS, } from "../flush.js";
|
|
21
21
|
import { workMeshRoot, workMeshSpoolPath } from "../paths.js";
|
|
22
22
|
import { createVaultBoardReader, refreshBoundSessionBoards, BOARD_REFRESH_INTERVAL_MS, } from "./board-refresh.js";
|
|
23
23
|
import { createContract3Fetcher, realTimerHost, } from "./credentials.js";
|
|
@@ -30,6 +30,20 @@ import { workMeshHeldPath } from "../paths.js";
|
|
|
30
30
|
import { noteHookSessionsFromSpoolFile, TRANSCRIPT_WATCH_INTERVAL_MS, TranscriptWatcher, } from "./transcript-watch.js";
|
|
31
31
|
export const SPOOL_DEBOUNCE_MS = 2_000;
|
|
32
32
|
export const FLUSH_INTERVAL_MS = 10_000;
|
|
33
|
+
/**
|
|
34
|
+
* Only producer appends to spool.jsonl should wake the watcher. Every other
|
|
35
|
+
* file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
|
|
36
|
+
* live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
|
|
37
|
+
* during a flush; reacting to those turned each flush into a self-triggered
|
|
38
|
+
* flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
|
|
39
|
+
* core on fleet boxes with a non-empty held file). A null filename (platforms
|
|
40
|
+
* that do not report one) keeps the conservative behaviour and flushes.
|
|
41
|
+
*/
|
|
42
|
+
export function shouldFlushOnSpoolDirEvent(filename) {
|
|
43
|
+
if (filename === null || filename === undefined)
|
|
44
|
+
return true;
|
|
45
|
+
return String(filename) === "spool.jsonl";
|
|
46
|
+
}
|
|
33
47
|
function alive(pid) {
|
|
34
48
|
try {
|
|
35
49
|
process.kill(pid, 0);
|
|
@@ -76,6 +90,8 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
76
90
|
// daemon went dark once its IoT credentials expired.
|
|
77
91
|
const defaultGetToken = async () => ensureCognitoToken({ interactive: false, tokenSource: "machine" });
|
|
78
92
|
const getToken = deps.getToken ?? defaultGetToken;
|
|
93
|
+
/** In-memory held re-attempt throttle (survives watch-triggered flushes). */
|
|
94
|
+
let lastHeldRetryAtMs = 0;
|
|
79
95
|
const flushFn = deps.flush ??
|
|
80
96
|
(async () => {
|
|
81
97
|
const t = await getToken();
|
|
@@ -87,6 +103,12 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
87
103
|
workMeshRoot: meshRoot,
|
|
88
104
|
workContextRoot: ctxRoot,
|
|
89
105
|
poster,
|
|
106
|
+
heldRetryIntervalMs: HELD_RETRY_INTERVAL_MS,
|
|
107
|
+
lastHeldRetryAtMs,
|
|
108
|
+
onHeldClaimed: (atMs) => {
|
|
109
|
+
lastHeldRetryAtMs = atMs;
|
|
110
|
+
},
|
|
111
|
+
log: (message) => log(dir, message),
|
|
90
112
|
});
|
|
91
113
|
});
|
|
92
114
|
const replayFn = deps.replayOutbox ??
|
|
@@ -243,7 +265,10 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
243
265
|
lastFlushAt: now().toISOString(),
|
|
244
266
|
lastFlushResult: { ...summary, ok: true },
|
|
245
267
|
}, now);
|
|
246
|
-
if (summary.claimed > 0 ||
|
|
268
|
+
if (summary.claimed > 0 ||
|
|
269
|
+
summary.posted > 0 ||
|
|
270
|
+
summary.held > 0 ||
|
|
271
|
+
(summary.heldOverflow ?? 0) > 0) {
|
|
247
272
|
log(dir, `flush(${reason}): claimed=${summary.claimed} posted=${summary.posted} held=${summary.held}`);
|
|
248
273
|
}
|
|
249
274
|
}
|
|
@@ -292,11 +317,7 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
292
317
|
watcher = fs.watch(path.dirname(spoolPath), { persistent: true }, (_event, filename) => {
|
|
293
318
|
if (stopped)
|
|
294
319
|
return;
|
|
295
|
-
if (
|
|
296
|
-
filename === "spool.jsonl" ||
|
|
297
|
-
filename === "held.jsonl" ||
|
|
298
|
-
String(filename).startsWith("spool.") ||
|
|
299
|
-
String(filename).startsWith("held.")) {
|
|
320
|
+
if (shouldFlushOnSpoolDirEvent(filename)) {
|
|
300
321
|
scheduleDebouncedFlush();
|
|
301
322
|
}
|
|
302
323
|
});
|
|
@@ -11,7 +11,11 @@
|
|
|
11
11
|
import { type RandomFn, type SleepFn } from "./backoff.js";
|
|
12
12
|
import { type SessionEventsPoster } from "./session-events-client.js";
|
|
13
13
|
export declare const HELD_TTL_MS: number;
|
|
14
|
-
|
|
14
|
+
/** Re-attempt held events at most this often (spool still flushes every cycle). */
|
|
15
|
+
export declare const HELD_RETRY_INTERVAL_MS: number;
|
|
16
|
+
/** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
|
|
17
|
+
export declare const HELD_MAX_LINES = 20000;
|
|
18
|
+
export type HeldReason = "STATE_ABSENT" | "NEEDS_COMPANY" | "COMPANY_CONFLICT" | "HELD_EXPIRED" | "HELD_OVERFLOW";
|
|
15
19
|
export interface HeldLineRecord {
|
|
16
20
|
/** Original spool event fields. */
|
|
17
21
|
event: Record<string, unknown>;
|
|
@@ -26,6 +30,19 @@ export interface FlushDeps {
|
|
|
26
30
|
sleep?: SleepFn;
|
|
27
31
|
random?: RandomFn;
|
|
28
32
|
maxAttempts?: number;
|
|
33
|
+
/**
|
|
34
|
+
* Minimum gap between held re-attempts. Default HELD_RETRY_INTERVAL_MS.
|
|
35
|
+
* Pass 0 in tests to re-claim held every flush.
|
|
36
|
+
*/
|
|
37
|
+
heldRetryIntervalMs?: number;
|
|
38
|
+
/** Epoch ms of the last flush that claimed held (daemon keeps this in memory). */
|
|
39
|
+
lastHeldRetryAtMs?: number;
|
|
40
|
+
/** Invoked when this flush claims held, so the daemon can advance its timestamp. */
|
|
41
|
+
onHeldClaimed?: (atMs: number) => void;
|
|
42
|
+
/** Max lines retained in held.jsonl after a flush. Default HELD_MAX_LINES. */
|
|
43
|
+
heldMaxLines?: number;
|
|
44
|
+
/** Optional logger (daemon wires this); used once per overflow trim. */
|
|
45
|
+
log?: (message: string) => void;
|
|
29
46
|
}
|
|
30
47
|
export interface FlushSummary {
|
|
31
48
|
claimed: number;
|
|
@@ -35,6 +52,10 @@ export interface FlushSummary {
|
|
|
35
52
|
deadLettered: number;
|
|
36
53
|
restored: number;
|
|
37
54
|
batches: number;
|
|
55
|
+
/** True when this flush claimed/re-read held.jsonl. */
|
|
56
|
+
heldClaimed?: boolean;
|
|
57
|
+
/** Oldest held lines dead-lettered this flush due to HELD_MAX_LINES. */
|
|
58
|
+
heldOverflow?: number;
|
|
38
59
|
}
|
|
39
60
|
/**
|
|
40
61
|
* Flush spool + held. Safe to call concurrently with enqueue (claim-by-rename).
|
|
@@ -15,9 +15,13 @@ import { workContextRoot } from "../../work-context/paths.js";
|
|
|
15
15
|
import { defaultSleep, FLUSH_MAX_ATTEMPTS, fullJitterDelayMs, } from "./backoff.js";
|
|
16
16
|
import { LOCAL_ONLY_FIELDS, stripLocalOnlyFields } from "./format-spool-line.js";
|
|
17
17
|
import { SESSION_EVENTS_BATCH_MAX, WRITABLE_CONTEXT_STATUSES, parseSessionEventsBatchAck, } from "./session-events-client.js";
|
|
18
|
-
import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
|
|
19
|
-
import { workMeshRoot } from "./paths.js";
|
|
18
|
+
import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, countJsonlLines, ensureSpoolFile, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
|
|
19
|
+
import { workMeshHeldPath, workMeshRoot } from "./paths.js";
|
|
20
20
|
export const HELD_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
21
|
+
/** Re-attempt held events at most this often (spool still flushes every cycle). */
|
|
22
|
+
export const HELD_RETRY_INTERVAL_MS = 5 * 60 * 1000;
|
|
23
|
+
/** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
|
|
24
|
+
export const HELD_MAX_LINES = 20_000;
|
|
21
25
|
function parseLine(line) {
|
|
22
26
|
try {
|
|
23
27
|
const v = JSON.parse(line);
|
|
@@ -101,10 +105,22 @@ export async function flushSessionEvents(deps) {
|
|
|
101
105
|
const sleep = deps.sleep ?? defaultSleep;
|
|
102
106
|
const random = deps.random ?? Math.random;
|
|
103
107
|
const maxAttempts = deps.maxAttempts ?? FLUSH_MAX_ATTEMPTS;
|
|
104
|
-
const
|
|
108
|
+
const heldRetryIntervalMs = deps.heldRetryIntervalMs ?? HELD_RETRY_INTERVAL_MS;
|
|
109
|
+
const heldMaxLines = deps.heldMaxLines ?? HELD_MAX_LINES;
|
|
110
|
+
const nowMs = now().getTime();
|
|
111
|
+
const ts = `${nowMs}`;
|
|
105
112
|
const orphans = recoverOrphanClaims(deps.workMeshRoot);
|
|
106
113
|
const spoolClaim = claimSpool(deps.workMeshRoot, ts);
|
|
107
|
-
|
|
114
|
+
// Throttle held re-attempts; spool (fresh) events still flush every cycle.
|
|
115
|
+
const lastHeldRetryAtMs = deps.lastHeldRetryAtMs ?? 0;
|
|
116
|
+
const shouldClaimHeld = heldRetryIntervalMs <= 0 ||
|
|
117
|
+
nowMs - lastHeldRetryAtMs >= heldRetryIntervalMs;
|
|
118
|
+
const heldClaim = shouldClaimHeld
|
|
119
|
+
? claimHeld(deps.workMeshRoot, `${ts}.held`)
|
|
120
|
+
: { claimPath: null, lines: [] };
|
|
121
|
+
if (shouldClaimHeld) {
|
|
122
|
+
deps.onHeldClaimed?.(nowMs);
|
|
123
|
+
}
|
|
108
124
|
const claimPaths = [
|
|
109
125
|
...orphans.claimPaths,
|
|
110
126
|
spoolClaim.claimPath,
|
|
@@ -118,42 +134,45 @@ export async function flushSessionEvents(deps) {
|
|
|
118
134
|
deadLettered: 0,
|
|
119
135
|
restored: 0,
|
|
120
136
|
batches: 0,
|
|
137
|
+
heldClaimed: shouldClaimHeld,
|
|
138
|
+
heldOverflow: 0,
|
|
121
139
|
};
|
|
122
|
-
/** Original events still awaiting disposition (restored on throw). */
|
|
123
|
-
const pendingRestore =
|
|
140
|
+
/** Original events still awaiting disposition (restored on throw). O(1) delete. */
|
|
141
|
+
const pendingRestore = new Set();
|
|
124
142
|
const collected = [];
|
|
125
143
|
try {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
140
|
-
// Expire held lines older than 7 days.
|
|
141
|
-
if (heldAt) {
|
|
142
|
-
const age = now().getTime() - Date.parse(heldAt);
|
|
143
|
-
if (Number.isFinite(age) && age > HELD_TTL_MS) {
|
|
144
|
-
appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
|
|
144
|
+
// Iterate sources separately — avoid building one giant combined array.
|
|
145
|
+
for (const source of [orphans.lines, spoolClaim.lines, heldClaim.lines]) {
|
|
146
|
+
for (const line of source) {
|
|
147
|
+
summary.claimed += 1;
|
|
148
|
+
const parsed = parseLine(line);
|
|
149
|
+
if (!parsed) {
|
|
150
|
+
appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "SCHEMA_REJECTED", "parse_error", now().toISOString()), deps.workMeshRoot);
|
|
151
|
+
summary.deadLettered += 1;
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
const { event, heldAt } = unwrapHeld(parsed);
|
|
155
|
+
if (!sessionIdOf(event)) {
|
|
156
|
+
appendDeadLetterLine(deadLetterEnvelope(event, "SCHEMA_REJECTED", "missing_sessionId", now().toISOString()), deps.workMeshRoot);
|
|
145
157
|
summary.deadLettered += 1;
|
|
146
158
|
continue;
|
|
147
159
|
}
|
|
160
|
+
// Expire held lines older than 7 days.
|
|
161
|
+
if (heldAt) {
|
|
162
|
+
const age = now().getTime() - Date.parse(heldAt);
|
|
163
|
+
if (Number.isFinite(age) && age > HELD_TTL_MS) {
|
|
164
|
+
appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
|
|
165
|
+
summary.deadLettered += 1;
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
pendingRestore.add(event);
|
|
170
|
+
collected.push({ event, heldAt });
|
|
148
171
|
}
|
|
149
|
-
pendingRestore.push(event);
|
|
150
|
-
collected.push({ event, heldAt });
|
|
151
172
|
}
|
|
152
173
|
const bySession = groupBySession(collected);
|
|
153
174
|
const markDisposed = (event) => {
|
|
154
|
-
|
|
155
|
-
if (idx >= 0)
|
|
156
|
-
pendingRestore.splice(idx, 1);
|
|
175
|
+
pendingRestore.delete(event);
|
|
157
176
|
};
|
|
158
177
|
for (const [sessionId, items] of bySession) {
|
|
159
178
|
const state = readSessionState(sessionId, deps.workContextRoot);
|
|
@@ -238,13 +257,100 @@ export async function flushSessionEvents(deps) {
|
|
|
238
257
|
/* best-effort; orphan claim scan will retry next flush */
|
|
239
258
|
}
|
|
240
259
|
}
|
|
241
|
-
pendingRestore.
|
|
260
|
+
pendingRestore.clear();
|
|
242
261
|
for (const p of claimPaths) {
|
|
243
262
|
removeClaimFile(p);
|
|
244
263
|
}
|
|
245
264
|
}
|
|
265
|
+
const overflowed = trimHeldOverflow(deps.workMeshRoot, heldMaxLines, now, summary);
|
|
266
|
+
if (overflowed > 0) {
|
|
267
|
+
deps.log?.(`held overflow: dead-lettered ${overflowed} oldest events (cap=${heldMaxLines})`);
|
|
268
|
+
}
|
|
246
269
|
return summary;
|
|
247
270
|
}
|
|
271
|
+
/**
|
|
272
|
+
* After a flush, if held.jsonl exceeds maxLines, dead-letter the oldest
|
|
273
|
+
* overflow (by heldAt, not file position) with HELD_OVERFLOW and keep the rest.
|
|
274
|
+
*
|
|
275
|
+
* Durability rules:
|
|
276
|
+
* - Retained lines are APPENDED to held.jsonl, never written over it: another
|
|
277
|
+
* flusher (`hq mesh flush` beside the daemon) may have appended to a fresh
|
|
278
|
+
* held.jsonl after our claim renamed the old one away.
|
|
279
|
+
* - The overflow claim file is removed only after every dead letter and every
|
|
280
|
+
* retained line is on disk. If anything throws, the claim stays behind and
|
|
281
|
+
* recoverOrphanClaims re-reads it on the next flush (duplicates are possible,
|
|
282
|
+
* loss is not).
|
|
283
|
+
*/
|
|
284
|
+
function trimHeldOverflow(meshRoot, maxLines, now, summary) {
|
|
285
|
+
if (maxLines <= 0)
|
|
286
|
+
return 0;
|
|
287
|
+
const heldPath = workMeshHeldPath(meshRoot);
|
|
288
|
+
if (countJsonlLines(heldPath) <= maxLines)
|
|
289
|
+
return 0;
|
|
290
|
+
const claim = claimHeld(meshRoot, `${now().getTime()}.overflow`);
|
|
291
|
+
if (!claim.claimPath)
|
|
292
|
+
return 0;
|
|
293
|
+
if (claim.lines.length === 0) {
|
|
294
|
+
removeClaimFile(claim.claimPath);
|
|
295
|
+
return 0;
|
|
296
|
+
}
|
|
297
|
+
try {
|
|
298
|
+
if (claim.lines.length <= maxLines) {
|
|
299
|
+
appendHeldLines(meshRoot, claim.lines);
|
|
300
|
+
removeClaimFile(claim.claimPath);
|
|
301
|
+
return 0;
|
|
302
|
+
}
|
|
303
|
+
const overflow = claim.lines.length - maxLines;
|
|
304
|
+
// Oldest by heldAt: retries regroup held by session, so file order is not
|
|
305
|
+
// chronological. Unparseable lines sort first (dead-lettered first).
|
|
306
|
+
const ordered = claim.lines
|
|
307
|
+
.map((line, idx) => ({ line, idx, atMs: heldAtMs(line) }))
|
|
308
|
+
.sort((a, b) => a.atMs - b.atMs || a.idx - b.idx);
|
|
309
|
+
const toDead = ordered.slice(0, overflow);
|
|
310
|
+
const toKeep = ordered
|
|
311
|
+
.slice(overflow)
|
|
312
|
+
.sort((a, b) => a.idx - b.idx)
|
|
313
|
+
.map((x) => x.line);
|
|
314
|
+
const at = now().toISOString();
|
|
315
|
+
for (const { line } of toDead) {
|
|
316
|
+
const parsed = parseLine(line);
|
|
317
|
+
if (!parsed) {
|
|
318
|
+
appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
|
|
319
|
+
}
|
|
320
|
+
else {
|
|
321
|
+
const { event } = unwrapHeld(parsed);
|
|
322
|
+
appendDeadLetterLine(deadLetterEnvelope(event, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
|
|
323
|
+
}
|
|
324
|
+
summary.deadLettered += 1;
|
|
325
|
+
}
|
|
326
|
+
appendHeldLines(meshRoot, toKeep);
|
|
327
|
+
summary.heldOverflow = overflow;
|
|
328
|
+
removeClaimFile(claim.claimPath);
|
|
329
|
+
return overflow;
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
// Leave the claim for orphan recovery; never drop held events on a failed trim.
|
|
333
|
+
return 0;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/** heldAt of a held line as epoch ms; 0 when missing or unparseable. */
|
|
337
|
+
function heldAtMs(line) {
|
|
338
|
+
const parsed = parseLine(line);
|
|
339
|
+
if (!parsed)
|
|
340
|
+
return 0;
|
|
341
|
+
const { heldAt } = unwrapHeld(parsed);
|
|
342
|
+
if (!heldAt)
|
|
343
|
+
return 0;
|
|
344
|
+
const ms = Date.parse(heldAt);
|
|
345
|
+
return Number.isFinite(ms) ? ms : 0;
|
|
346
|
+
}
|
|
347
|
+
function appendHeldLines(meshRoot, lines) {
|
|
348
|
+
if (lines.length === 0)
|
|
349
|
+
return;
|
|
350
|
+
const heldPath = workMeshHeldPath(meshRoot);
|
|
351
|
+
ensureSpoolFile(heldPath);
|
|
352
|
+
fs.appendFileSync(heldPath, `${lines.join("\n")}\n`, { mode: 0o600 });
|
|
353
|
+
}
|
|
248
354
|
/**
|
|
249
355
|
* Account a 2xx batch: dead-letter rejected eventIds with their codes;
|
|
250
356
|
* count accepted+duplicates as posted; restore any eventId not accounted for.
|
|
@@ -13,7 +13,7 @@ export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath,
|
|
|
13
13
|
export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
|
|
14
14
|
export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
|
|
15
15
|
export type { EnqueueOptions, EnqueueResult } from "./enqueue.js";
|
|
16
|
-
export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
16
|
+
export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
17
17
|
export type { FlushDeps, FlushSummary, HeldReason } from "./flush.js";
|
|
18
18
|
export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
|
|
19
19
|
export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
|
|
@@ -9,7 +9,7 @@ export { encodeCrockford, generateUlid, isUlid } from "./ulid.js";
|
|
|
9
9
|
export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath, workMeshHeldPath, workMeshRoot, workMeshSpoolPath, } from "./paths.js";
|
|
10
10
|
export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
|
|
11
11
|
export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
|
|
12
|
-
export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
12
|
+
export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
|
|
13
13
|
export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
|
|
14
14
|
export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
|
|
15
15
|
export * from "./daemon/index.js";
|
|
@@ -153,7 +153,12 @@ export function recoverOrphanClaims(root) {
|
|
|
153
153
|
const claimPaths = listOrphanClaimFiles(root);
|
|
154
154
|
const lines = [];
|
|
155
155
|
for (const p of claimPaths) {
|
|
156
|
-
|
|
156
|
+
// Plain loop: Array#push(...huge) exceeds V8's argument limit (~65k–128k)
|
|
157
|
+
// and throws RangeError ("Maximum call stack size exceeded").
|
|
158
|
+
const orphanLines = readClaimFileLines(p);
|
|
159
|
+
for (const line of orphanLines) {
|
|
160
|
+
lines.push(line);
|
|
161
|
+
}
|
|
157
162
|
}
|
|
158
163
|
return { lines, claimPaths };
|
|
159
164
|
}
|