@indigoai-us/hq-cli 5.108.13 → 5.108.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,62 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.108.15] — 2026-09-07
6
+
7
+ ### Fixed
8
+
9
+ - Work Mesh Live presence daemon (`hq mesh daemon run`) no longer pins a CPU
10
+ core when the MQTT broker accepts a connection then closes it just past the
11
+ stable-connect grace window. The full-jitter reconnect backoff could collapse
12
+ to ~0 ms after the connect handler reset its counters, producing a tight
13
+ accept-then-close reconnect spin (each pass re-ran the SigV4 presign and
14
+ re-materialised the daemon state dir) with no network progress and no log
15
+ output. A hard `RECONNECT_MIN_DELAY_MS` (1s) floor now applies to every
16
+ reconnect path — normal backoff, the post-grace reset, and the refused-retry
17
+ path (which a server `Retry-After: 0` could otherwise drive to 0) — so the
18
+ delay can never shrink below 1s. Regression tests cover the reset and the
19
+ refused-retry floor. (#524)
20
+ - The Work Mesh Live daemon no longer burns CPU in a self-triggered flush loop.
21
+ Its spool directory watcher woke on files the daemon writes itself during a
22
+ flush (`held.jsonl` and the `*.claimed` files), so any box with held events
23
+ flushed every two seconds forever; the watcher now reacts only to producer
24
+ appends to `spool.jsonl`. Recovering a very large orphan claim file no longer
25
+ overflows the stack, mid-flush disposal is O(1), held events are re-attempted
26
+ at most every five minutes, and `held.jsonl` is capped at 20,000 lines with
27
+ the oldest overflow dead-lettered as `HELD_OVERFLOW` (loss-free under failure
28
+ and concurrent flushers). (#525)
29
+
30
+ ## [5.108.14] — 2026-09-07
31
+
32
+ ### Fixed
33
+
34
+ - An incomplete hq install tree no longer files an unactionable crash when hq
35
+ fails to load one of its OWN bundled modules in-process (HQ-CLI-1M, HQ-CLI-1N).
36
+ Two shapes shared one cause — hq-cli's globally installed package tree was not
37
+ intact at the moment a command loaded a module. In HQ-CLI-1N (`hq core
38
+ timeout-guard` on Windows) a partial `npm i -g` left a RELATIVE sibling
39
+ unwritten deep inside a bundled dependency, so Node threw
40
+ `MODULE_NOT_FOUND`. In HQ-CLI-1M (`hq rescue` on Linux) a concurrent global
41
+ install rewrote the running tree, so an ESM module that existed at resolve was
42
+ gone at read and Node's loader raised `ENOENT` — the loader validates
43
+ existence at resolve, so an ENOENT at LOAD proves the file vanished mid-run
44
+ rather than being merely absent. Both carried no hq-cli frames, reached the
45
+ top-level handler's final `else`, and produced a bare Sentry crash plus an
46
+ `hq:` line the operator could not act on. A new in-process classifier now
47
+ recognises both shapes — but ONLY when the failing file sits under the running
48
+ install's own `node_modules/`, the CJS shape additionally requires a relative
49
+ specifier, and the ESM shape additionally requires an esm-loader frame — and
50
+ prints an input-free reinstall remedy (run the command again first, then
51
+ `npm i -g @indigoai-us/hq-cli` / `pnpm add -g @indigoai-us/hq-cli`) while
52
+ skipping Sentry capture, the same disposition established for the qmd child in
53
+ HQ-CLI-Y. An hq-cli packaging fault stays reportable: a miss under the
54
+ package's own `dist/` or `assets/`, a bare-specifier miss (a possible
55
+ undeclared dependency), and an esm-loader ENOENT whose path did not survive
56
+ delivery are NOT suppressed — the last is captured WITH a bounded
57
+ `incomplete_install` context so the next occurrence is attributable. The drop
58
+ is wired both at the top-level boundary and in the shared `beforeSend`, so it
59
+ covers every capture route.
60
+
5
61
  ## [5.108.13] — 2026-09-06
6
62
 
7
63
  ## [5.108.12] - 2026-09-05
@@ -110,9 +110,20 @@ export interface ProfilePatch {
110
110
  description?: string;
111
111
  }
112
112
  export interface RuntimeConfigPatch {
113
+ /**
114
+ * Brain/runtime provider. When present the server routes the PATCH to a
115
+ * PROVIDER MIGRATION (handleProviderMigration): it TERMINATES and reprovisions
116
+ * the box and changes the provider/auth contract. Omit for a plain tuning
117
+ * patch (model / effort / tier).
118
+ */
119
+ provider?: string;
113
120
  codexModel?: string;
114
121
  codexReasoningEffort?: string;
115
122
  codexServiceTier?: string;
123
+ /** Provider-migration auth mode: "subscription" | "apiKey". */
124
+ codexAuthMode?: string;
125
+ /** Provider-migration apiKey-mode vault key reference (never a raw secret). */
126
+ codexApiKeyRef?: string;
116
127
  }
117
128
  /**
118
129
  * Authenticated JSON round-trip against the agents control plane. Throws
@@ -143,7 +154,12 @@ export interface ProvisionAgentInput {
143
154
  /** Server quote assertion; hq-pro re-prices and refuses a stale amount. */
144
155
  quotedNetMonthlyCents?: number;
145
156
  quoteCatalogVersion?: string;
157
+ /** Funnel attribution: which client surface made the attempt. */
158
+ surface?: AgentCreateSurface;
146
159
  }
160
+ /** Closed set shared with hq-pro's agent_create_* funnel contract. */
161
+ export declare const CLI_AGENT_CREATE_SURFACE: "cli_agents_create";
162
+ export type AgentCreateSurface = typeof CLI_AGENT_CREATE_SURFACE;
147
163
  export interface AgentCreateSizeOption {
148
164
  key: "basic" | "power" | "dev";
149
165
  productName: string;
@@ -146,6 +146,8 @@ export function slugifyAgentName(name) {
146
146
  .replace(/[^a-z0-9]+/g, "-")
147
147
  .replace(/^-+|-+$/g, "");
148
148
  }
149
+ /** Closed set shared with hq-pro's agent_create_* funnel contract. */
150
+ export const CLI_AGENT_CREATE_SURFACE = "cli_agents_create";
149
151
  /** Read hq-pro's company-specific creation prices and capacities. */
150
152
  export async function getAgentCreateOptions(token, companyUid, idempotencyKey) {
151
153
  const raw = await agentsRequest({
@@ -153,7 +155,9 @@ export async function getAgentCreateOptions(token, companyUid, idempotencyKey) {
153
155
  path: "/v1/agents/provision-options",
154
156
  query: {
155
157
  companyUid,
156
- ...(idempotencyKey ? { idempotencyKey } : {}),
158
+ ...(idempotencyKey
159
+ ? { idempotencyKey, surface: CLI_AGENT_CREATE_SURFACE }
160
+ : {}),
157
161
  },
158
162
  });
159
163
  if (!raw || typeof raw !== "object") {
@@ -842,6 +846,7 @@ export function registerAgentsCommand(program) {
842
846
  : { desiredInstanceType: quotedSize.instanceType }),
843
847
  quotedNetMonthlyCents: quotedSize.netMonthlyCents,
844
848
  quoteCatalogVersion: createOptions.catalogVersion,
849
+ surface: CLI_AGENT_CREATE_SURFACE,
845
850
  });
846
851
  const uid = typeof result.uid === "string" ? result.uid : slug;
847
852
  console.log(chalk.green(`Provisioning started for agent "${name}".`));
@@ -999,15 +1004,54 @@ export function registerAgentsCommand(program) {
999
1004
  });
1000
1005
  agents
1001
1006
  .command("config <agentUid>")
1002
- .description("Update an agent's runtime config (model / reasoning effort / service tier)")
1007
+ .description("Update an agent's runtime config (model / reasoning effort / service tier), or migrate its brain/runtime provider with --provider (DESTRUCTIVE: terminates + reprovisions the box)")
1003
1008
  .option("--company <slug>", "Company slug (resolves to companyUid)")
1004
1009
  .option("--model <model>", "Codex model id")
1005
1010
  .option("--effort <effort>", "Reasoning effort: minimal | low | medium | high | xhigh")
1006
1011
  .option("--tier <tier>", "Service tier (speed): default | priority")
1012
+ .option("--provider <provider>", "Migrate brain/runtime provider (codex | grok | claude | agents-v2). DESTRUCTIVE: terminates and reprovisions the box; requires --model and --yes.")
1013
+ .option("--auth-mode <mode>", "Provider-migration auth mode: subscription | apiKey (default: keep current)")
1014
+ .option("--api-key-ref <ref>", "Provider-migration apiKey-mode vault key reference (never a raw key)")
1015
+ .option("--yes", "Confirm a destructive provider migration (required with --provider)")
1007
1016
  .option("--json", "Emit raw JSON")
1008
1017
  .action(async function (agentUid, opts) {
1009
1018
  try {
1010
1019
  const patch = {};
1020
+ if (opts.provider !== undefined) {
1021
+ // A `provider` field routes the server to handleProviderMigration,
1022
+ // which TERMINATES and reprovisions the box. Guard it: valid provider,
1023
+ // an explicit target --model (the server requires it), and --yes.
1024
+ // Reuses the module-level VALID_PROVIDERS (shared with `provision`).
1025
+ const provider = opts.provider.trim().toLowerCase();
1026
+ if (!VALID_PROVIDERS.has(provider)) {
1027
+ console.error(chalk.red(`Invalid --provider '${opts.provider}': must be one of codex, grok, claude, agents-v2`));
1028
+ process.exit(1);
1029
+ }
1030
+ if (opts.model === undefined) {
1031
+ console.error(chalk.red("A provider migration requires --model (the target brain's model id)."));
1032
+ process.exit(1);
1033
+ }
1034
+ if (!opts.yes) {
1035
+ console.error(chalk.red(`Refusing to migrate agent ${agentUid} to provider '${provider}' without --yes.\n` +
1036
+ "This TERMINATES and reprovisions the box (irreversible) and changes the auth contract.\n" +
1037
+ "Re-run with --yes once you have confirmed the exact agent, company, and model."));
1038
+ process.exit(1);
1039
+ }
1040
+ if (provider !== "agents-v2") {
1041
+ // v1 RESIDENT runtime guard: codex|grok|claude migrate the agent OFF
1042
+ // agents-v2 onto the legacy resident runtime. Fleet boxes run
1043
+ // agents-v2 (brain derived from the model), so this is almost always
1044
+ // a mistake — warn loudly but proceed (the operator passed --yes).
1045
+ console.warn(chalk.yellow(`Warning: --provider ${provider} targets the V1 RESIDENT runtime, not agents-v2.\n` +
1046
+ "Fleet boxes run agents-v2. To keep this agent on the v2 runtime with a\n" +
1047
+ `${provider} brain, use: --provider agents-v2 --model <${provider} model id>`));
1048
+ }
1049
+ patch.provider = provider;
1050
+ if (opts.authMode !== undefined)
1051
+ patch.codexAuthMode = opts.authMode;
1052
+ if (opts.apiKeyRef !== undefined)
1053
+ patch.codexApiKeyRef = opts.apiKeyRef;
1054
+ }
1011
1055
  if (opts.model !== undefined)
1012
1056
  patch.codexModel = opts.model;
1013
1057
  if (opts.effort !== undefined) {
@@ -20,6 +20,20 @@ export type MqttConnectionState = "idle" | "connecting" | "connected" | "reconne
20
20
  * then closes immediately after a denied publish).
21
21
  */
22
22
  export declare const CONNECT_STABLE_GRACE_MS = 5000;
23
+ /**
24
+ * Hard floor for the delay between MQTT presence reconnect attempts.
25
+ *
26
+ * The full-jitter backoff can otherwise compute ~0 ms: after a connection
27
+ * survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
28
+ * `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
29
+ * past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
30
+ * return `random() * base` — frequently a handful of ms, or 0 when `random()`
31
+ * is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
32
+ * presign and re-materialises the daemon state dir) that pins a CPU core with
33
+ * no network progress and no log output. This floor guarantees every reconnect
34
+ * waits at least this long regardless of the backoff/grace-reset state.
35
+ */
36
+ export declare const RECONNECT_MIN_DELAY_MS = 1000;
23
37
  /** Always emit the first N close info lines, then at most one per interval. */
24
38
  export declare const CLOSE_LOG_ALWAYS_COUNT = 3;
25
39
  export declare const CLOSE_LOG_INTERVAL_MS = 60000;
@@ -69,9 +83,13 @@ export interface PresenceClientOptions {
69
83
  }
70
84
  /**
71
85
  * Full-jitter capped exponential backoff (1s base → 60s cap by default),
72
- * floored at `previousMs` so consecutive failures never shrink the delay.
86
+ * floored at `previousMs` so consecutive failures never shrink the delay and at
87
+ * `minMs` so the result can never collapse to ~0 (see
88
+ * {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
89
+ * accept-then-close reconnect spin after the connect handler resets the
90
+ * backoff counters.
73
91
  */
74
- export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number): number;
92
+ export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
75
93
  export declare function buildPresencePayload(input: {
76
94
  status: "online" | "offline";
77
95
  actorUid: string;
@@ -20,17 +20,35 @@ import { presignIotWssUrl } from "./presign.js";
20
20
  * then closes immediately after a denied publish).
21
21
  */
22
22
  export const CONNECT_STABLE_GRACE_MS = 5_000;
23
+ /**
24
+ * Hard floor for the delay between MQTT presence reconnect attempts.
25
+ *
26
+ * The full-jitter backoff can otherwise compute ~0 ms: after a connection
27
+ * survives {@link CONNECT_STABLE_GRACE_MS} the connect handler resets both
28
+ * `attempt` and `lastBackoffMs` to 0, so a broker that accepts-then-closes just
29
+ * past the grace window makes the next `backoffDelayMs(0, base, cap, random, 0)`
30
+ * return `random() * base` — frequently a handful of ms, or 0 when `random()`
31
+ * is near 0. That produces a tight reconnect loop (each pass re-runs the SigV4
32
+ * presign and re-materialises the daemon state dir) that pins a CPU core with
33
+ * no network progress and no log output. This floor guarantees every reconnect
34
+ * waits at least this long regardless of the backoff/grace-reset state.
35
+ */
36
+ export const RECONNECT_MIN_DELAY_MS = 1_000;
23
37
  /** Always emit the first N close info lines, then at most one per interval. */
24
38
  export const CLOSE_LOG_ALWAYS_COUNT = 3;
25
39
  export const CLOSE_LOG_INTERVAL_MS = 60_000;
26
40
  /**
27
41
  * Full-jitter capped exponential backoff (1s base → 60s cap by default),
28
- * floored at `previousMs` so consecutive failures never shrink the delay.
42
+ * floored at `previousMs` so consecutive failures never shrink the delay and at
43
+ * `minMs` so the result can never collapse to ~0 (see
44
+ * {@link RECONNECT_MIN_DELAY_MS}). The `minMs` floor is what prevents the
45
+ * accept-then-close reconnect spin after the connect handler resets the
46
+ * backoff counters.
29
47
  */
30
- export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0) {
48
+ export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, minMs = 0) {
31
49
  const cap = Math.min(maxMs, baseMs * 2 ** attempt);
32
50
  const raw = Math.max(0, random() * cap);
33
- return Math.min(maxMs, Math.max(previousMs, raw));
51
+ return Math.min(maxMs, Math.max(previousMs, raw, minMs));
34
52
  }
35
53
  export function buildPresencePayload(input) {
36
54
  return {
@@ -341,12 +359,14 @@ export class PresenceClient {
341
359
  else {
342
360
  delay = refusedRetryDelayMs(this.refusedRetryMs, this.random);
343
361
  }
344
- delay = clampRetryDelayMs(delay);
362
+ // Floor the refused retry too: a server `Retry-After: 0` (or a 0 env
363
+ // override) would otherwise clamp to 0 and hot-loop the refused path.
364
+ delay = Math.max(RECONNECT_MIN_DELAY_MS, clampRetryDelayMs(delay));
345
365
  const nextRetryAt = new Date(this.timers.now() + delay).toISOString();
346
366
  this.noteRefusal(refused, nextRetryAt);
347
367
  }
348
368
  else {
349
- delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs);
369
+ delay = backoffDelayMs(this.attempt, this.baseBackoffMs, this.maxBackoffMs, this.random, this.lastBackoffMs, RECONNECT_MIN_DELAY_MS);
350
370
  this.lastBackoffMs = delay;
351
371
  this.attempt += 1;
352
372
  // Do not clear refusal here — MQTT close must retain doctor/status refusal
@@ -15,6 +15,16 @@ import { PresenceClient, type MqttConnectFn } from "./presence.js";
15
15
  import { TranscriptWatcher, type TranscriptFs } from "./transcript-watch.js";
16
16
  export declare const SPOOL_DEBOUNCE_MS = 2000;
17
17
  export declare const FLUSH_INTERVAL_MS = 10000;
18
+ /**
19
+ * Only producer appends to spool.jsonl should wake the watcher. Every other
20
+ * file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
21
+ * live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
22
+ * during a flush; reacting to those turned each flush into a self-triggered
23
+ * flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
24
+ * core on fleet boxes with a non-empty held file). A null filename (platforms
25
+ * that do not report one) keeps the conservative behaviour and flushes.
26
+ */
27
+ export declare function shouldFlushOnSpoolDirEvent(filename: string | Buffer | null | undefined): boolean;
18
28
  export interface DaemonRunDeps {
19
29
  home?: string;
20
30
  env?: NodeJS.ProcessEnv;
@@ -17,7 +17,7 @@ import { ensureCognitoToken } from "../../../../utils/cognito-session.js";
17
17
  import { replayOutbox } from "../../../work-context/outbox.js";
18
18
  import { workContextRoot } from "../../../work-context/paths.js";
19
19
  import { createSessionEventsPoster, resolveVaultApiBase, } from "../session-events-client.js";
20
- import { flushSessionEvents } from "../flush.js";
20
+ import { flushSessionEvents, HELD_RETRY_INTERVAL_MS, } from "../flush.js";
21
21
  import { workMeshRoot, workMeshSpoolPath } from "../paths.js";
22
22
  import { createVaultBoardReader, refreshBoundSessionBoards, BOARD_REFRESH_INTERVAL_MS, } from "./board-refresh.js";
23
23
  import { createContract3Fetcher, realTimerHost, } from "./credentials.js";
@@ -30,6 +30,20 @@ import { workMeshHeldPath } from "../paths.js";
30
30
  import { noteHookSessionsFromSpoolFile, TRANSCRIPT_WATCH_INTERVAL_MS, TranscriptWatcher, } from "./transcript-watch.js";
31
31
  export const SPOOL_DEBOUNCE_MS = 2_000;
32
32
  export const FLUSH_INTERVAL_MS = 10_000;
33
+ /**
34
+ * Only producer appends to spool.jsonl should wake the watcher. Every other
35
+ * file in the work-mesh root (held.jsonl, spool.<ts>.claimed, held.<ts>.claimed,
36
+ * live-cache.json, dead-letter.jsonl, ...) is written by the daemon itself
37
+ * during a flush; reacting to those turned each flush into a self-triggered
38
+ * flush 2s later, forever (observed ~1,750 watch flushes/hour and >50% of a
39
+ * core on fleet boxes with a non-empty held file). A null filename (platforms
40
+ * that do not report one) keeps the conservative behaviour and flushes.
41
+ */
42
+ export function shouldFlushOnSpoolDirEvent(filename) {
43
+ if (filename === null || filename === undefined)
44
+ return true;
45
+ return String(filename) === "spool.jsonl";
46
+ }
33
47
  function alive(pid) {
34
48
  try {
35
49
  process.kill(pid, 0);
@@ -76,6 +90,8 @@ export async function runMeshDaemon(deps = {}) {
76
90
  // daemon went dark once its IoT credentials expired.
77
91
  const defaultGetToken = async () => ensureCognitoToken({ interactive: false, tokenSource: "machine" });
78
92
  const getToken = deps.getToken ?? defaultGetToken;
93
+ /** In-memory held re-attempt throttle (survives watch-triggered flushes). */
94
+ let lastHeldRetryAtMs = 0;
79
95
  const flushFn = deps.flush ??
80
96
  (async () => {
81
97
  const t = await getToken();
@@ -87,6 +103,12 @@ export async function runMeshDaemon(deps = {}) {
87
103
  workMeshRoot: meshRoot,
88
104
  workContextRoot: ctxRoot,
89
105
  poster,
106
+ heldRetryIntervalMs: HELD_RETRY_INTERVAL_MS,
107
+ lastHeldRetryAtMs,
108
+ onHeldClaimed: (atMs) => {
109
+ lastHeldRetryAtMs = atMs;
110
+ },
111
+ log: (message) => log(dir, message),
90
112
  });
91
113
  });
92
114
  const replayFn = deps.replayOutbox ??
@@ -243,7 +265,10 @@ export async function runMeshDaemon(deps = {}) {
243
265
  lastFlushAt: now().toISOString(),
244
266
  lastFlushResult: { ...summary, ok: true },
245
267
  }, now);
246
- if (summary.claimed > 0 || summary.posted > 0 || summary.held > 0) {
268
+ if (summary.claimed > 0 ||
269
+ summary.posted > 0 ||
270
+ summary.held > 0 ||
271
+ (summary.heldOverflow ?? 0) > 0) {
247
272
  log(dir, `flush(${reason}): claimed=${summary.claimed} posted=${summary.posted} held=${summary.held}`);
248
273
  }
249
274
  }
@@ -292,11 +317,7 @@ export async function runMeshDaemon(deps = {}) {
292
317
  watcher = fs.watch(path.dirname(spoolPath), { persistent: true }, (_event, filename) => {
293
318
  if (stopped)
294
319
  return;
295
- if (!filename ||
296
- filename === "spool.jsonl" ||
297
- filename === "held.jsonl" ||
298
- String(filename).startsWith("spool.") ||
299
- String(filename).startsWith("held.")) {
320
+ if (shouldFlushOnSpoolDirEvent(filename)) {
300
321
  scheduleDebouncedFlush();
301
322
  }
302
323
  });
@@ -11,7 +11,11 @@
11
11
  import { type RandomFn, type SleepFn } from "./backoff.js";
12
12
  import { type SessionEventsPoster } from "./session-events-client.js";
13
13
  export declare const HELD_TTL_MS: number;
14
- export type HeldReason = "STATE_ABSENT" | "NEEDS_COMPANY" | "COMPANY_CONFLICT" | "HELD_EXPIRED";
14
+ /** Re-attempt held events at most this often (spool still flushes every cycle). */
15
+ export declare const HELD_RETRY_INTERVAL_MS: number;
16
+ /** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
17
+ export declare const HELD_MAX_LINES = 20000;
18
+ export type HeldReason = "STATE_ABSENT" | "NEEDS_COMPANY" | "COMPANY_CONFLICT" | "HELD_EXPIRED" | "HELD_OVERFLOW";
15
19
  export interface HeldLineRecord {
16
20
  /** Original spool event fields. */
17
21
  event: Record<string, unknown>;
@@ -26,6 +30,19 @@ export interface FlushDeps {
26
30
  sleep?: SleepFn;
27
31
  random?: RandomFn;
28
32
  maxAttempts?: number;
33
+ /**
34
+ * Minimum gap between held re-attempts. Default HELD_RETRY_INTERVAL_MS.
35
+ * Pass 0 in tests to re-claim held every flush.
36
+ */
37
+ heldRetryIntervalMs?: number;
38
+ /** Epoch ms of the last flush that claimed held (daemon keeps this in memory). */
39
+ lastHeldRetryAtMs?: number;
40
+ /** Invoked when this flush claims held, so the daemon can advance its timestamp. */
41
+ onHeldClaimed?: (atMs: number) => void;
42
+ /** Max lines retained in held.jsonl after a flush. Default HELD_MAX_LINES. */
43
+ heldMaxLines?: number;
44
+ /** Optional logger (daemon wires this); used once per overflow trim. */
45
+ log?: (message: string) => void;
29
46
  }
30
47
  export interface FlushSummary {
31
48
  claimed: number;
@@ -35,6 +52,10 @@ export interface FlushSummary {
35
52
  deadLettered: number;
36
53
  restored: number;
37
54
  batches: number;
55
+ /** True when this flush claimed/re-read held.jsonl. */
56
+ heldClaimed?: boolean;
57
+ /** Oldest held lines dead-lettered this flush due to HELD_MAX_LINES. */
58
+ heldOverflow?: number;
38
59
  }
39
60
  /**
40
61
  * Flush spool + held. Safe to call concurrently with enqueue (claim-by-rename).
@@ -15,9 +15,13 @@ import { workContextRoot } from "../../work-context/paths.js";
15
15
  import { defaultSleep, FLUSH_MAX_ATTEMPTS, fullJitterDelayMs, } from "./backoff.js";
16
16
  import { LOCAL_ONLY_FIELDS, stripLocalOnlyFields } from "./format-spool-line.js";
17
17
  import { SESSION_EVENTS_BATCH_MAX, WRITABLE_CONTEXT_STATUSES, parseSessionEventsBatchAck, } from "./session-events-client.js";
18
- import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
19
- import { workMeshRoot } from "./paths.js";
18
+ import { appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendSpoolLine, claimHeld, claimSpool, countJsonlLines, ensureSpoolFile, recoverOrphanClaims, removeClaimFile, } from "./spool.js";
19
+ import { workMeshHeldPath, workMeshRoot } from "./paths.js";
20
20
  export const HELD_TTL_MS = 7 * 24 * 60 * 60 * 1000;
21
+ /** Re-attempt held events at most this often (spool still flushes every cycle). */
22
+ export const HELD_RETRY_INTERVAL_MS = 5 * 60 * 1000;
23
+ /** Cap held.jsonl size; oldest overflow is dead-lettered with HELD_OVERFLOW. */
24
+ export const HELD_MAX_LINES = 20_000;
21
25
  function parseLine(line) {
22
26
  try {
23
27
  const v = JSON.parse(line);
@@ -101,10 +105,22 @@ export async function flushSessionEvents(deps) {
101
105
  const sleep = deps.sleep ?? defaultSleep;
102
106
  const random = deps.random ?? Math.random;
103
107
  const maxAttempts = deps.maxAttempts ?? FLUSH_MAX_ATTEMPTS;
104
- const ts = `${now().getTime()}`;
108
+ const heldRetryIntervalMs = deps.heldRetryIntervalMs ?? HELD_RETRY_INTERVAL_MS;
109
+ const heldMaxLines = deps.heldMaxLines ?? HELD_MAX_LINES;
110
+ const nowMs = now().getTime();
111
+ const ts = `${nowMs}`;
105
112
  const orphans = recoverOrphanClaims(deps.workMeshRoot);
106
113
  const spoolClaim = claimSpool(deps.workMeshRoot, ts);
107
- const heldClaim = claimHeld(deps.workMeshRoot, `${ts}.held`);
114
+ // Throttle held re-attempts; spool (fresh) events still flush every cycle.
115
+ const lastHeldRetryAtMs = deps.lastHeldRetryAtMs ?? 0;
116
+ const shouldClaimHeld = heldRetryIntervalMs <= 0 ||
117
+ nowMs - lastHeldRetryAtMs >= heldRetryIntervalMs;
118
+ const heldClaim = shouldClaimHeld
119
+ ? claimHeld(deps.workMeshRoot, `${ts}.held`)
120
+ : { claimPath: null, lines: [] };
121
+ if (shouldClaimHeld) {
122
+ deps.onHeldClaimed?.(nowMs);
123
+ }
108
124
  const claimPaths = [
109
125
  ...orphans.claimPaths,
110
126
  spoolClaim.claimPath,
@@ -118,42 +134,45 @@ export async function flushSessionEvents(deps) {
118
134
  deadLettered: 0,
119
135
  restored: 0,
120
136
  batches: 0,
137
+ heldClaimed: shouldClaimHeld,
138
+ heldOverflow: 0,
121
139
  };
122
- /** Original events still awaiting disposition (restored on throw). */
123
- const pendingRestore = [];
140
+ /** Original events still awaiting disposition (restored on throw). O(1) delete. */
141
+ const pendingRestore = new Set();
124
142
  const collected = [];
125
143
  try {
126
- for (const line of [...orphans.lines, ...spoolClaim.lines, ...heldClaim.lines]) {
127
- summary.claimed += 1;
128
- const parsed = parseLine(line);
129
- if (!parsed) {
130
- appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "SCHEMA_REJECTED", "parse_error", now().toISOString()), deps.workMeshRoot);
131
- summary.deadLettered += 1;
132
- continue;
133
- }
134
- const { event, heldAt } = unwrapHeld(parsed);
135
- if (!sessionIdOf(event)) {
136
- appendDeadLetterLine(deadLetterEnvelope(event, "SCHEMA_REJECTED", "missing_sessionId", now().toISOString()), deps.workMeshRoot);
137
- summary.deadLettered += 1;
138
- continue;
139
- }
140
- // Expire held lines older than 7 days.
141
- if (heldAt) {
142
- const age = now().getTime() - Date.parse(heldAt);
143
- if (Number.isFinite(age) && age > HELD_TTL_MS) {
144
- appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
144
+ // Iterate sources separately avoid building one giant combined array.
145
+ for (const source of [orphans.lines, spoolClaim.lines, heldClaim.lines]) {
146
+ for (const line of source) {
147
+ summary.claimed += 1;
148
+ const parsed = parseLine(line);
149
+ if (!parsed) {
150
+ appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "SCHEMA_REJECTED", "parse_error", now().toISOString()), deps.workMeshRoot);
151
+ summary.deadLettered += 1;
152
+ continue;
153
+ }
154
+ const { event, heldAt } = unwrapHeld(parsed);
155
+ if (!sessionIdOf(event)) {
156
+ appendDeadLetterLine(deadLetterEnvelope(event, "SCHEMA_REJECTED", "missing_sessionId", now().toISOString()), deps.workMeshRoot);
145
157
  summary.deadLettered += 1;
146
158
  continue;
147
159
  }
160
+ // Expire held lines older than 7 days.
161
+ if (heldAt) {
162
+ const age = now().getTime() - Date.parse(heldAt);
163
+ if (Number.isFinite(age) && age > HELD_TTL_MS) {
164
+ appendDeadLetterLine(deadLetterEnvelope(event, "HELD_EXPIRED", "held_ttl", now().toISOString()), deps.workMeshRoot);
165
+ summary.deadLettered += 1;
166
+ continue;
167
+ }
168
+ }
169
+ pendingRestore.add(event);
170
+ collected.push({ event, heldAt });
148
171
  }
149
- pendingRestore.push(event);
150
- collected.push({ event, heldAt });
151
172
  }
152
173
  const bySession = groupBySession(collected);
153
174
  const markDisposed = (event) => {
154
- const idx = pendingRestore.indexOf(event);
155
- if (idx >= 0)
156
- pendingRestore.splice(idx, 1);
175
+ pendingRestore.delete(event);
157
176
  };
158
177
  for (const [sessionId, items] of bySession) {
159
178
  const state = readSessionState(sessionId, deps.workContextRoot);
@@ -238,13 +257,100 @@ export async function flushSessionEvents(deps) {
238
257
  /* best-effort; orphan claim scan will retry next flush */
239
258
  }
240
259
  }
241
- pendingRestore.length = 0;
260
+ pendingRestore.clear();
242
261
  for (const p of claimPaths) {
243
262
  removeClaimFile(p);
244
263
  }
245
264
  }
265
+ const overflowed = trimHeldOverflow(deps.workMeshRoot, heldMaxLines, now, summary);
266
+ if (overflowed > 0) {
267
+ deps.log?.(`held overflow: dead-lettered ${overflowed} oldest events (cap=${heldMaxLines})`);
268
+ }
246
269
  return summary;
247
270
  }
271
+ /**
272
+ * After a flush, if held.jsonl exceeds maxLines, dead-letter the oldest
273
+ * overflow (by heldAt, not file position) with HELD_OVERFLOW and keep the rest.
274
+ *
275
+ * Durability rules:
276
+ * - Retained lines are APPENDED to held.jsonl, never written over it: another
277
+ * flusher (`hq mesh flush` beside the daemon) may have appended to a fresh
278
+ * held.jsonl after our claim renamed the old one away.
279
+ * - The overflow claim file is removed only after every dead letter and every
280
+ * retained line is on disk. If anything throws, the claim stays behind and
281
+ * recoverOrphanClaims re-reads it on the next flush (duplicates are possible,
282
+ * loss is not).
283
+ */
284
+ function trimHeldOverflow(meshRoot, maxLines, now, summary) {
285
+ if (maxLines <= 0)
286
+ return 0;
287
+ const heldPath = workMeshHeldPath(meshRoot);
288
+ if (countJsonlLines(heldPath) <= maxLines)
289
+ return 0;
290
+ const claim = claimHeld(meshRoot, `${now().getTime()}.overflow`);
291
+ if (!claim.claimPath)
292
+ return 0;
293
+ if (claim.lines.length === 0) {
294
+ removeClaimFile(claim.claimPath);
295
+ return 0;
296
+ }
297
+ try {
298
+ if (claim.lines.length <= maxLines) {
299
+ appendHeldLines(meshRoot, claim.lines);
300
+ removeClaimFile(claim.claimPath);
301
+ return 0;
302
+ }
303
+ const overflow = claim.lines.length - maxLines;
304
+ // Oldest by heldAt: retries regroup held by session, so file order is not
305
+ // chronological. Unparseable lines sort first (dead-lettered first).
306
+ const ordered = claim.lines
307
+ .map((line, idx) => ({ line, idx, atMs: heldAtMs(line) }))
308
+ .sort((a, b) => a.atMs - b.atMs || a.idx - b.idx);
309
+ const toDead = ordered.slice(0, overflow);
310
+ const toKeep = ordered
311
+ .slice(overflow)
312
+ .sort((a, b) => a.idx - b.idx)
313
+ .map((x) => x.line);
314
+ const at = now().toISOString();
315
+ for (const { line } of toDead) {
316
+ const parsed = parseLine(line);
317
+ if (!parsed) {
318
+ appendDeadLetterLine(deadLetterEnvelope({ raw: line.slice(0, 200) }, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
319
+ }
320
+ else {
321
+ const { event } = unwrapHeld(parsed);
322
+ appendDeadLetterLine(deadLetterEnvelope(event, "HELD_OVERFLOW", "held_max_lines", at), meshRoot);
323
+ }
324
+ summary.deadLettered += 1;
325
+ }
326
+ appendHeldLines(meshRoot, toKeep);
327
+ summary.heldOverflow = overflow;
328
+ removeClaimFile(claim.claimPath);
329
+ return overflow;
330
+ }
331
+ catch {
332
+ // Leave the claim for orphan recovery; never drop held events on a failed trim.
333
+ return 0;
334
+ }
335
+ }
336
+ /** heldAt of a held line as epoch ms; 0 when missing or unparseable. */
337
+ function heldAtMs(line) {
338
+ const parsed = parseLine(line);
339
+ if (!parsed)
340
+ return 0;
341
+ const { heldAt } = unwrapHeld(parsed);
342
+ if (!heldAt)
343
+ return 0;
344
+ const ms = Date.parse(heldAt);
345
+ return Number.isFinite(ms) ? ms : 0;
346
+ }
347
+ function appendHeldLines(meshRoot, lines) {
348
+ if (lines.length === 0)
349
+ return;
350
+ const heldPath = workMeshHeldPath(meshRoot);
351
+ ensureSpoolFile(heldPath);
352
+ fs.appendFileSync(heldPath, `${lines.join("\n")}\n`, { mode: 0o600 });
353
+ }
248
354
  /**
249
355
  * Account a 2xx batch: dead-letter rejected eventIds with their codes;
250
356
  * count accepted+duplicates as posted; restore any eventId not accounted for.
@@ -13,7 +13,7 @@ export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath,
13
13
  export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
14
14
  export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
15
15
  export type { EnqueueOptions, EnqueueResult } from "./enqueue.js";
16
- export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
16
+ export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
17
17
  export type { FlushDeps, FlushSummary, HeldReason } from "./flush.js";
18
18
  export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
19
19
  export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
@@ -9,7 +9,7 @@ export { encodeCrockford, generateUlid, isUlid } from "./ulid.js";
9
9
  export { workMeshClaimPath, workMeshDeadLetterPath, workMeshDroppedReceiptPath, workMeshHeldPath, workMeshRoot, workMeshSpoolPath, } from "./paths.js";
10
10
  export { SPOOL_LINE_MAX_BYTES, SpoolWriteError, appendDeadLetterLine, appendDroppedReceipt, appendHeldLine, appendJsonlLine, appendSpoolLine, claimHeld, claimJsonlByRename, claimSpool, countJsonlLines, deadLetterNonEmpty, ensureSpoolFile, removeClaimFile, } from "./spool.js";
11
11
  export { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, resolveEnqueueSessionId, } from "./enqueue.js";
12
- export { HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
12
+ export { HELD_MAX_LINES, HELD_RETRY_INTERVAL_MS, HELD_TTL_MS, assertNoBindCacheAccess, bindCacheExists, defaultFlushRoots, flushSessionEvents, sessionsBindPath, } from "./flush.js";
13
13
  export { BACKOFF_BASE_MS, BACKOFF_CAP_MS, FLUSH_MAX_ATTEMPTS, defaultSleep, fullJitterDelayMs, } from "./backoff.js";
14
14
  export { SESSION_EVENTS_BATCH_MAX, SESSION_EVENTS_PATH, WRITABLE_CONTEXT_STATUSES, classifySessionEventsResponse, createSessionEventsPoster, resolveVaultApiBase, } from "./session-events-client.js";
15
15
  export * from "./daemon/index.js";
@@ -153,7 +153,12 @@ export function recoverOrphanClaims(root) {
153
153
  const claimPaths = listOrphanClaimFiles(root);
154
154
  const lines = [];
155
155
  for (const p of claimPaths) {
156
- lines.push(...readClaimFileLines(p));
156
+ // Plain loop: Array#push(...huge) exceeds V8's argument limit (~65k–128k)
157
+ // and throws RangeError ("Maximum call stack size exceeded").
158
+ const orphanLines = readClaimFileLines(p);
159
+ for (const line of orphanLines) {
160
+ lines.push(line);
161
+ }
157
162
  }
158
163
  return { lines, claimPaths };
159
164
  }
package/dist/main.js CHANGED
@@ -23,6 +23,7 @@ import { qmdQueryDocumentMessage } from "./utils/qmd-query-document-error.js";
23
23
  import { qmdModelDownloadMessage } from "./utils/qmd-model-download-error.js";
24
24
  import { qmdWorkdirMissingMessage } from "./utils/qmd-workdir-missing-error.js";
25
25
  import { hqStateWriteErrorMessage } from "./utils/hq-state-write-error.js";
26
+ import { incompleteInstallMessage, incompleteInstallCaptureContext, } from "./utils/incomplete-install-error.js";
26
27
  import { isExpectedUserError } from "./utils/expected-cli-error.js";
27
28
  import { isVarlockEnvError } from "./run/env-graph-guard.js";
28
29
  import { isEpipe } from "./utils/epipe.js";
@@ -515,7 +516,28 @@ export async function handleTopLevelError(err, deps = defaultTopLevelErrorDepend
515
516
  const stateWriteMsg = qmdMsg || collectionMsg || terminatedMsg || llmDisabledMsg || moduleMissingMsg || storeMissingMsg || storeUnopenableMsg || queryDocumentMsg || modelDownloadMsg || workdirMissingMsg
516
517
  ? null
517
518
  : hqStateWriteErrorMessage(err);
518
- const envMsg = qmdMsg || collectionMsg || terminatedMsg || llmDisabledMsg || moduleMissingMsg || storeMissingMsg || storeUnopenableMsg || queryDocumentMsg || modelDownloadMsg || workdirMissingMsg || stateWriteMsg
519
+ // An IN-PROCESS module-load failure that means hq-cli's OWN installed
520
+ // package tree is incomplete at load time — a partial/interrupted global
521
+ // install left a bundled file unwritten (HQ-CLI-1N, a CJS relative-sibling
522
+ // MODULE_NOT_FOUND), or a concurrent global install rewrote the running
523
+ // tree so an ESM module present at resolve was gone at read (HQ-CLI-1M, an
524
+ // esm-loader ENOENT). Both carry no hq-cli frames and reached the final
525
+ // else, filing a bare crash and an unactionable line. An incomplete
526
+ // install is the caller's machine, the disposition HQ-CLI-Y already
527
+ // established for the qmd CHILD — print the input-free reinstall remedy and
528
+ // skip capture. Placed with the environmental family (after the typed qmd
529
+ // carriers and the hq state-write carrier, before environmentalFsErrorMessage):
530
+ // the signatures are disjoint — ENVIRONMENTAL_FS_CODES is only
531
+ // ENOSPC/EDQUOT/EROFS (never ENOENT/MODULE_NOT_FOUND), no qmd carrier sets
532
+ // requireStack or an esm-loader frame, and the classified file must sit
533
+ // under <packageRoot>/node_modules — so ordering changes nothing that
534
+ // exists. The UNATTRIBUTABLE shape (an esm-loader ENOENT whose path did not
535
+ // survive) is deliberately NOT suppressed; it is captured WITH bounded
536
+ // context on the generic path below.
537
+ const incompleteInstallMsg = qmdMsg || collectionMsg || terminatedMsg || llmDisabledMsg || moduleMissingMsg || storeMissingMsg || storeUnopenableMsg || queryDocumentMsg || modelDownloadMsg || workdirMissingMsg || stateWriteMsg
538
+ ? null
539
+ : incompleteInstallMessage(err);
540
+ const envMsg = qmdMsg || collectionMsg || terminatedMsg || llmDisabledMsg || moduleMissingMsg || storeMissingMsg || storeUnopenableMsg || queryDocumentMsg || modelDownloadMsg || workdirMissingMsg || stateWriteMsg || incompleteInstallMsg
519
541
  ? null
520
542
  : environmentalFsErrorMessage(err);
521
543
  // A LOCAL sync-state lock failure (@indigoai-us/hq-cloud's
@@ -532,7 +554,7 @@ export async function handleTopLevelError(err, deps = defaultTopLevelErrorDepend
532
554
  // environmental-fs check, before network-transport — is pinned by tests.
533
555
  // The `in-process-async-holder` reason is deliberately NOT suppressed here
534
556
  // (see sync-state-lock-error.ts); it stays captured.
535
- const lockMsg = qmdMsg || collectionMsg || terminatedMsg || llmDisabledMsg || moduleMissingMsg || storeMissingMsg || storeUnopenableMsg || queryDocumentMsg || modelDownloadMsg || workdirMissingMsg || stateWriteMsg || envMsg
557
+ const lockMsg = qmdMsg || collectionMsg || terminatedMsg || llmDisabledMsg || moduleMissingMsg || storeMissingMsg || storeUnopenableMsg || queryDocumentMsg || modelDownloadMsg || workdirMissingMsg || stateWriteMsg || incompleteInstallMsg || envMsg
536
558
  ? null
537
559
  : syncStateLockMessage(err);
538
560
  // A raw network transport failure (undici's `TypeError: fetch failed`
@@ -556,6 +578,7 @@ export async function handleTopLevelError(err, deps = defaultTopLevelErrorDepend
556
578
  modelDownloadMsg ||
557
579
  workdirMissingMsg ||
558
580
  stateWriteMsg ||
581
+ incompleteInstallMsg ||
559
582
  envMsg ||
560
583
  lockMsg
561
584
  ? null
@@ -593,6 +616,9 @@ export async function handleTopLevelError(err, deps = defaultTopLevelErrorDepend
593
616
  else if (stateWriteMsg) {
594
617
  deps.stderr.write(`hq: ${stateWriteMsg}\n`);
595
618
  }
619
+ else if (incompleteInstallMsg) {
620
+ deps.stderr.write(`hq: ${incompleteInstallMsg}\n`);
621
+ }
596
622
  else if (envMsg) {
597
623
  deps.stderr.write(`hq: ${envMsg}\n`);
598
624
  }
@@ -603,13 +629,18 @@ export async function handleTopLevelError(err, deps = defaultTopLevelErrorDepend
603
629
  deps.stderr.write(`hq: ${transportMsg}\n`);
604
630
  }
605
631
  else {
606
- // A genuinely unclassified fault is still captured exactly once. When it
607
- // is a qmd spawn-level failure the fix could not attribute, attach the
608
- // bounded existence context so the next occurrence carries the evidence
609
- // the reported HQ-CLI-1A event lacked; every other error captures bare.
632
+ // A genuinely unclassified fault is still captured exactly once. Two
633
+ // shapes attach bounded, hq-derived context so the next occurrence
634
+ // carries the evidence this one lacked: a qmd spawn-level failure the fix
635
+ // could not attribute (HQ-CLI-1A), and an esm-loader ENOENT whose path
636
+ // did not survive delivery (HQ-CLI-1M — the unattributable incomplete-
637
+ // install shape). Every other error captures bare, exactly as before.
610
638
  const spawnContext = qmdSpawnFailureCaptureContext(err);
611
- if (spawnContext) {
612
- deps.sentry.captureException(err, spawnContext);
639
+ const installContext = incompleteInstallCaptureContext(err);
640
+ if (spawnContext || installContext) {
641
+ deps.sentry.captureException(err, {
642
+ contexts: { ...spawnContext?.contexts, ...installContext },
643
+ });
613
644
  }
614
645
  else {
615
646
  deps.sentry.captureException(err);
package/dist/sentry.js CHANGED
@@ -6,6 +6,7 @@ import { CLI_VERSION } from "./cli-version.js";
6
6
  import { getCachedSentryUser } from "./utils/sentry-identity.js";
7
7
  import { isEpipe } from "./utils/epipe.js";
8
8
  import { environmentalFsErrorMessage } from "./utils/environmental-error.js";
9
+ import { incompleteInstallMessage } from "./utils/incomplete-install-error.js";
9
10
  import { sentryFingerprintFor } from "./utils/sentry-fingerprint.js";
10
11
  /**
11
12
  * Drop broken-pipe (EPIPE) crashes before scrubbing/send. A closed downstream
@@ -31,6 +32,20 @@ export function epipeAwareBeforeSend(event, hint) {
31
32
  // route, while the CLI still exits non-zero. HQ-CLI-R (Sentry 7671416365).
32
33
  if (environmentalFsErrorMessage(hint?.originalException))
33
34
  return null;
35
+ // Path-independent belt for an IN-PROCESS incomplete-install module-load
36
+ // failure — hq-cli's own globally installed tree is not intact at load time
37
+ // (HQ-CLI-1N, a CJS relative-sibling MODULE_NOT_FOUND under its bundled
38
+ // node_modules; HQ-CLI-1M, an esm-loader ENOENT for a file present at resolve
39
+ // and gone at read). handleTopLevelError already prints the reinstall remedy
40
+ // for the top-level route; dropping the event here suppresses the fatal
41
+ // regardless of route — the unhandled-rejection boundary, the command-level
42
+ // captureException sites, and bin/hq-auth-refresh — mirroring the EPIPE and
43
+ // environmental-fs drops above. The classifier reads only structured fields
44
+ // and the failing file must sit under the running install's node_modules, so
45
+ // an hq-cli packaging fault (a dist/ miss, a bare-specifier miss) and the
46
+ // path-less unattributable shape are NOT dropped here and stay captured.
47
+ if (incompleteInstallMessage(hint?.originalException))
48
+ return null;
34
49
  // Group an event that survives to send by a BOUNDED machine discriminator so
35
50
  // unrelated gateway/HTTP failures stop colliding into one fungible issue
36
51
  // (HQ-CLI collision, Sentry 7642756130). Placed here — path-independent,
@@ -0,0 +1,50 @@
1
+ import * as fs from "fs";
2
+ /**
3
+ * The actionable remedy shown to the operator. Input-free — nothing from the
4
+ * error, the argv, or the filesystem is interpolated — so there is no injection
5
+ * surface and no way to inflate Sentry grouping, matching the bounded-remedy
6
+ * discipline of every sibling classifier. Covers BOTH sub-cases in the order a
7
+ * user should try them: re-run first (an install that finished mid-run leaves
8
+ * the next invocation healthy), then reinstall if it persists.
9
+ */
10
+ export declare const INCOMPLETE_INSTALL_REMEDY: string;
11
+ /** A resolver for the running install's root; returns null instead of throwing. */
12
+ export type PackageRootResolver = () => string | null;
13
+ /**
14
+ * If `err` is an in-process incomplete-install module-load failure — either the
15
+ * CJS relative-sibling shape (HQ-CLI-1N) or the ESM vanished-file shape
16
+ * (HQ-CLI-1M), with the failing file confirmed under `<packageRoot>/node_modules/`
17
+ * — return the actionable, input-free reinstall remedy; otherwise return null.
18
+ *
19
+ * Mirrors qmdModuleMissingMessage so the top-level handler and beforeSend branch
20
+ * the same way: a non-null result means print-the-remedy-and-skip-Sentry, null
21
+ * means "handle as usual (capture to Sentry)". Never throws — a resolver that
22
+ * fails yields null.
23
+ */
24
+ export declare function incompleteInstallMessage(err: unknown, resolvePackageRoot?: PackageRootResolver): string | null;
25
+ /** Bounded, scrubber-safe diagnostics for an unattributable esm-loader ENOENT. */
26
+ export type IncompleteInstallDiagnostics = {
27
+ packageRoot: string;
28
+ packageJsonExists: boolean;
29
+ nodeModulesExists: boolean;
30
+ esmLoaderFrame: boolean;
31
+ code: string;
32
+ };
33
+ /**
34
+ * When an esm-loader ENOENT reaches the capture path WITHOUT being suppressed —
35
+ * the exact shape the delivered HQ-CLI-1M payload arrived in, where neither the
36
+ * exception value nor node_system_error carried a `path` — return a bounded
37
+ * `contexts.incomplete_install` block so the next occurrence carries the
38
+ * evidence this one lacked; otherwise return undefined (bare capture). Built
39
+ * with the byte-capped, scrubber-safe discipline of package-root-diagnostics.ts:
40
+ * the resolved package root and whether its package.json / node_modules exist,
41
+ * the loader-frame marker, and the bounded errno code — never a caller argv,
42
+ * query, or user-minted value. Never throws.
43
+ *
44
+ * main.ts attaches this on the generic capture path exactly as
45
+ * qmdSpawnFailureCaptureContext already does.
46
+ */
47
+ export declare function incompleteInstallCaptureContext(err: unknown, resolvePackageRoot?: PackageRootResolver, fileSystem?: Pick<typeof fs, "existsSync">): {
48
+ incomplete_install: IncompleteInstallDiagnostics;
49
+ } | undefined;
50
+ //# sourceMappingURL=incomplete-install-error.d.ts.map
@@ -0,0 +1,250 @@
1
+ // src/utils/incomplete-install-error.ts
2
+ //
3
+ // Classify an IN-PROCESS module-load failure that means hq-cli's OWN globally
4
+ // installed package tree is not intact at the moment it loads a module — the
5
+ // caller's incomplete install, not an hq-cli code defect. Sibling in spirit to
6
+ // qmd-module-missing-error.ts (HQ-CLI-Y), but keyed on the ERROR OBJECT of a
7
+ // failure inside THIS process rather than a qmd child's captured stderr, which
8
+ // is the gap HQ-CLI-Y's classifier cannot cover.
9
+ //
10
+ // Two shapes, one cause — a partial/interrupted global install left a file
11
+ // unwritten, or a concurrent global install rewrote the running tree
12
+ // underneath a command:
13
+ //
14
+ // HQ-CLI-1N (Sentry 7714890525) — CJS, in-process. `hq core timeout-guard`
15
+ // loads the mesh presence client, whose `import mqtt` pulls a chain that ends
16
+ // at js-sdsl requiring a RELATIVE sibling (`./Base/TreeIterator`) that is
17
+ // absent on disk inside hq-cli's own bundled node_modules. Node throws
18
+ // `Error{ code: 'MODULE_NOT_FOUND', requireStack: [...] }`. A relative
19
+ // specifier internal to a third-party package can only be a truncated on-disk
20
+ // copy, never an hq-cli manifest defect.
21
+ //
22
+ // HQ-CLI-1M (Sentry 7714870912) — ESM load, in-process. A module that existed
23
+ // at RESOLVE was gone at READ (a concurrent writer rewrote the install tree),
24
+ // so Node's ESM loader raised `ENOENT` from getSourceSync/readFileSync/openSync
25
+ // with an `esm/…` loader frame in the stack. A merely-absent ESM module raises
26
+ // ERR_MODULE_NOT_FOUND at resolve, never ENOENT at load; an ENOENT at load
27
+ // proves the file vanished between resolve and read.
28
+ //
29
+ // Both shapes carry no hq-cli frames, reach the boundary's final `else`, and —
30
+ // before this classifier — filed a bare captureException plus an unactionable
31
+ // `hq: <fallback>` line. The disposition is the one HQ-CLI-Y already
32
+ // established: an incomplete install is the caller's machine, so the CLI prints
33
+ // an input-free reinstall remedy and skips Sentry capture.
34
+ //
35
+ // The gate is deliberately narrow so neither an hq-cli packaging fault nor
36
+ // user free-text can trip it. Only structured fields are read — `code`,
37
+ // `syscall`, `path`, `requireStack`, and the stack's loader-frame marker, plus
38
+ // the first message line for the CJS specifier. THREE independent narrowings
39
+ // keep a genuine hq-cli defect reportable:
40
+ // 1. The failing file must sit under `<packageRoot>/node_modules/` — a
41
+ // third-party file hq-cli does not author. A miss under `<packageRoot>/dist`
42
+ // or `/assets` is hq-cli's OWN shipped output and stays captured.
43
+ // 2. The CJS shape additionally requires a RELATIVE specifier — a
44
+ // bare-specifier miss (`Cannot find module 'mqtt'`) can be an undeclared
45
+ // dependency (an hq-cli manifest defect) and stays captured.
46
+ // 3. The ESM shape additionally requires an esm-loader frame — an ordinary
47
+ // `fs.readFileSync` ENOENT written by hq's own code stays captured.
48
+ import * as fs from "fs";
49
+ import * as path from "path";
50
+ import { packageRoot } from "./hq-roots.js";
51
+ import { boundedDiagnosticValue } from "./package-root-diagnostics.js";
52
+ /**
53
+ * The actionable remedy shown to the operator. Input-free — nothing from the
54
+ * error, the argv, or the filesystem is interpolated — so there is no injection
55
+ * surface and no way to inflate Sentry grouping, matching the bounded-remedy
56
+ * discipline of every sibling classifier. Covers BOTH sub-cases in the order a
57
+ * user should try them: re-run first (an install that finished mid-run leaves
58
+ * the next invocation healthy), then reinstall if it persists.
59
+ */
60
+ export const INCOMPLETE_INSTALL_REMEDY = "hq couldn't load part of its own installed files, so the hq install tree is " +
61
+ "incomplete on this machine — most often because a global install (its own " +
62
+ "self-update, the desktop background installer, another hq process, or a " +
63
+ "hand-run install) rewrote the package while this command was running, or an " +
64
+ "earlier install was interrupted before every file was written. Run the " +
65
+ "command again first: an install that finished mid-run leaves the next " +
66
+ "invocation healthy. If it keeps failing, reinstall hq — for a global install " +
67
+ "run `npm i -g @indigoai-us/hq-cli` (or the pnpm equivalent, " +
68
+ "`pnpm add -g @indigoai-us/hq-cli`).";
69
+ /** A relative module specifier — `./x`, `../x`, `.\x`, `..\x`. */
70
+ const RELATIVE_SPECIFIER = /^\.\.?[\\/]/;
71
+ /** A Node ESM loader frame — proves the ENOENT came from the module loader, not hq's own fs call. */
72
+ const ESM_LOADER_FRAME = /node:internal[\\/]modules[\\/]esm[\\/]/;
73
+ const ROOT_DIAGNOSTIC_BYTES = 256;
74
+ const CODE_DIAGNOSTIC_BYTES = 32;
75
+ /**
76
+ * packageRoot() walks up from the compiled module and THROWS
77
+ * PackageRootResolutionError when it cannot resolve. This classifier runs inside
78
+ * beforeSend on EVERY event, so it must never throw — a resolution failure
79
+ * returns null and the error stays captured.
80
+ */
81
+ function safePackageRoot() {
82
+ try {
83
+ return packageRoot();
84
+ }
85
+ catch {
86
+ return null;
87
+ }
88
+ }
89
+ /** Call a (possibly injected) resolver without letting it throw. */
90
+ function resolveRootSafely(resolve) {
91
+ try {
92
+ return resolve();
93
+ }
94
+ catch {
95
+ return null;
96
+ }
97
+ }
98
+ /** Fold `\`/`/` runs to a single `/` and drop any trailing separator. */
99
+ function foldSeparators(p) {
100
+ return p.replace(/[\\/]+/g, "/").replace(/\/+$/, "");
101
+ }
102
+ /** A Windows-shaped absolute path (drive letter or UNC), regardless of host OS. */
103
+ function looksWin32(p) {
104
+ return /^[a-zA-Z]:[\\/]/.test(p) || /^\\\\/.test(p);
105
+ }
106
+ /**
107
+ * Normalise a path for prefix comparison: separators folded, and case folded
108
+ * ONLY for a Windows-shaped path (drive-letter case and AppData\Roaming casing
109
+ * drift there, but POSIX paths are case-sensitive and must stay so). Detecting
110
+ * win32 by the path's SHAPE — not `process.platform` — lets the reported
111
+ * Windows path classify on a Linux CI runner.
112
+ */
113
+ function normalizeForCompare(p) {
114
+ const folded = foldSeparators(p);
115
+ return looksWin32(p) ? folded.toLowerCase() : folded;
116
+ }
117
+ /**
118
+ * True when `candidate` lives under `<root>/node_modules/`. Anchored at a true
119
+ * directory boundary (`<root>` + sep + `node_modules` + sep) so a sibling such
120
+ * as `<root>-old/node_modules/...` can never match.
121
+ */
122
+ function isUnderNodeModules(candidate, root) {
123
+ if (!candidate || !root)
124
+ return false;
125
+ const prefix = `${normalizeForCompare(root)}/node_modules/`;
126
+ return normalizeForCompare(candidate).startsWith(prefix);
127
+ }
128
+ /** The failing specifier from a `Cannot find module '<spec>'` message, or null. */
129
+ function parseMissingSpecifier(message) {
130
+ if (typeof message !== "string")
131
+ return null;
132
+ const match = message.match(/Cannot find module ['"]([^'"]+)['"]/);
133
+ return match ? match[1] : null;
134
+ }
135
+ /** True when `stack` carries a Node ESM loader frame. */
136
+ function hasEsmLoaderFrame(stack) {
137
+ return typeof stack === "string" && ESM_LOADER_FRAME.test(stack);
138
+ }
139
+ /**
140
+ * The ESM-loader ENOENT SIGNATURE, independent of whether a usable `path`
141
+ * survived: `code === 'ENOENT'`, `syscall === 'open'`, and an esm-loader frame
142
+ * in the stack. This is the shape the instrumentation fallback attaches context
143
+ * to; the message classifier additionally requires a `path` under node_modules.
144
+ */
145
+ function isEsmLoaderEnoent(err) {
146
+ if (err === null || typeof err !== "object")
147
+ return false;
148
+ const record = err;
149
+ return (record.code === "ENOENT" &&
150
+ record.syscall === "open" &&
151
+ hasEsmLoaderFrame(record.stack));
152
+ }
153
+ /**
154
+ * If `err` is an in-process incomplete-install module-load failure — either the
155
+ * CJS relative-sibling shape (HQ-CLI-1N) or the ESM vanished-file shape
156
+ * (HQ-CLI-1M), with the failing file confirmed under `<packageRoot>/node_modules/`
157
+ * — return the actionable, input-free reinstall remedy; otherwise return null.
158
+ *
159
+ * Mirrors qmdModuleMissingMessage so the top-level handler and beforeSend branch
160
+ * the same way: a non-null result means print-the-remedy-and-skip-Sentry, null
161
+ * means "handle as usual (capture to Sentry)". Never throws — a resolver that
162
+ * fails yields null.
163
+ */
164
+ export function incompleteInstallMessage(err, resolvePackageRoot = safePackageRoot) {
165
+ if (err === null || typeof err !== "object")
166
+ return null;
167
+ const record = err;
168
+ const code = typeof record.code === "string" ? record.code : undefined;
169
+ if (code !== "MODULE_NOT_FOUND" && code !== "ENOENT")
170
+ return null;
171
+ const root = resolveRootSafely(resolvePackageRoot);
172
+ if (!root)
173
+ return null;
174
+ if (code === "MODULE_NOT_FOUND") {
175
+ // Shape A (CJS, HQ-CLI-1N): a RELATIVE specifier internal to a package under
176
+ // the running install's node_modules can only be a truncated on-disk copy.
177
+ const requireStack = record.requireStack;
178
+ if (!Array.isArray(requireStack) || typeof requireStack[0] !== "string") {
179
+ return null;
180
+ }
181
+ const specifier = parseMissingSpecifier(record.message);
182
+ if (specifier === null || !RELATIVE_SPECIFIER.test(specifier))
183
+ return null;
184
+ return isUnderNodeModules(requireStack[0], root)
185
+ ? INCOMPLETE_INSTALL_REMEDY
186
+ : null;
187
+ }
188
+ // Shape B (ESM load, HQ-CLI-1M): an ENOENT from the module loader for a file
189
+ // that was present at resolve and gone at read.
190
+ if (record.syscall !== "open")
191
+ return null;
192
+ if (typeof record.path !== "string")
193
+ return null;
194
+ if (!hasEsmLoaderFrame(record.stack))
195
+ return null;
196
+ return isUnderNodeModules(record.path, root)
197
+ ? INCOMPLETE_INSTALL_REMEDY
198
+ : null;
199
+ }
200
+ /**
201
+ * When an esm-loader ENOENT reaches the capture path WITHOUT being suppressed —
202
+ * the exact shape the delivered HQ-CLI-1M payload arrived in, where neither the
203
+ * exception value nor node_system_error carried a `path` — return a bounded
204
+ * `contexts.incomplete_install` block so the next occurrence carries the
205
+ * evidence this one lacked; otherwise return undefined (bare capture). Built
206
+ * with the byte-capped, scrubber-safe discipline of package-root-diagnostics.ts:
207
+ * the resolved package root and whether its package.json / node_modules exist,
208
+ * the loader-frame marker, and the bounded errno code — never a caller argv,
209
+ * query, or user-minted value. Never throws.
210
+ *
211
+ * main.ts attaches this on the generic capture path exactly as
212
+ * qmdSpawnFailureCaptureContext already does.
213
+ */
214
+ export function incompleteInstallCaptureContext(err, resolvePackageRoot = safePackageRoot, fileSystem = fs) {
215
+ if (!isEsmLoaderEnoent(err))
216
+ return undefined;
217
+ // Only instrument what we did NOT already confidently suppress: a path under
218
+ // node_modules is classified and printed above, never captured.
219
+ if (incompleteInstallMessage(err, resolvePackageRoot) !== null)
220
+ return undefined;
221
+ const record = err;
222
+ const root = resolveRootSafely(resolvePackageRoot);
223
+ const code = typeof record.code === "string" ? record.code : "";
224
+ let packageJsonExists = false;
225
+ let nodeModulesExists = false;
226
+ if (root) {
227
+ try {
228
+ packageJsonExists = fileSystem.existsSync(path.join(root, "package.json"));
229
+ }
230
+ catch {
231
+ packageJsonExists = false;
232
+ }
233
+ try {
234
+ nodeModulesExists = fileSystem.existsSync(path.join(root, "node_modules"));
235
+ }
236
+ catch {
237
+ nodeModulesExists = false;
238
+ }
239
+ }
240
+ return {
241
+ incomplete_install: {
242
+ packageRoot: boundedDiagnosticValue(root ?? "<unresolved>", ROOT_DIAGNOSTIC_BYTES),
243
+ packageJsonExists,
244
+ nodeModulesExists,
245
+ esmLoaderFrame: true,
246
+ code: boundedDiagnosticValue(code, CODE_DIAGNOSTIC_BYTES),
247
+ },
248
+ };
249
+ }
250
+ //# sourceMappingURL=incomplete-install-error.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.108.13",
3
+ "version": "5.108.15",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {