@indigoai-us/hq-cloud 6.13.4 → 6.13.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Transient network-failure classification for the sync runner.
3
+ *
4
+ * The auto-sync watcher runs an unattended poll loop. At the top of every pass
5
+ * it calls `GET /membership/me` to resolve which companies to sync. When the
6
+ * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
7
+ * vault API momentarily unreachable) that call fails at the transport layer —
8
+ * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
9
+ * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
10
+ *
11
+ * This is NOT a crash: the next poll (~30s later) succeeds once the network is
12
+ * back. But the runner used to `return 1` for it, the watch loop propagated
13
+ * that non-zero exit, the process exited, and the menubar supervisor reported
14
+ * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
15
+ * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
16
+ * retry instead of surfacing a false crash, while a one-shot `hq sync` still
17
+ * exits non-zero so a human running it by hand sees the failure.
18
+ */
19
+ /**
20
+ * Distinct runner exit code for a transient network failure. The watch loop
21
+ * treats it as "stay alive, retry next tick" (never a crash); a one-shot run
22
+ * still exits non-zero so the failure is visible to a human/script. Value 75 =
23
+ * `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
24
+ * codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
25
+ */
26
+ export declare const TRANSIENT_NETWORK_EXIT = 75;
27
+ /**
28
+ * True when `err` is (or wraps) a transient network-transport failure. Walks the
29
+ * `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
30
+ * latter), matching either undici's canonical `TypeError: fetch failed` message
31
+ * or a known transient `.code`. Conservative by design: an arbitrary
32
+ * `new Error("network down")` with no code and a non-canonical message is NOT
33
+ * treated as transient, so only real transport-failure shapes qualify.
34
+ */
35
+ export declare function isTransientNetworkError(err: unknown, depth?: number): boolean;
36
+ //# sourceMappingURL=net-errors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"net-errors.d.ts","sourceRoot":"","sources":["../../src/lib/net-errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAC;AA0BzC;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,SAAI,GAAG,OAAO,CAsBxE"}
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Transient network-failure classification for the sync runner.
3
+ *
4
+ * The auto-sync watcher runs an unattended poll loop. At the top of every pass
5
+ * it calls `GET /membership/me` to resolve which companies to sync. When the
6
+ * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
7
+ * vault API momentarily unreachable) that call fails at the transport layer —
8
+ * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
9
+ * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
10
+ *
11
+ * This is NOT a crash: the next poll (~30s later) succeeds once the network is
12
+ * back. But the runner used to `return 1` for it, the watch loop propagated
13
+ * that non-zero exit, the process exited, and the menubar supervisor reported
14
+ * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
15
+ * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
16
+ * retry instead of surfacing a false crash, while a one-shot `hq sync` still
17
+ * exits non-zero so a human running it by hand sees the failure.
18
+ */
19
+ /**
20
+ * Distinct runner exit code for a transient network failure. The watch loop
21
+ * treats it as "stay alive, retry next tick" (never a crash); a one-shot run
22
+ * still exits non-zero so the failure is visible to a human/script. Value 75 =
23
+ * `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
24
+ * codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
25
+ */
26
+ export const TRANSIENT_NETWORK_EXIT = 75;
27
+ /**
28
+ * Node/undici error `code`s that mean "the request never completed at the
29
+ * transport layer" — the machine could not reach the host. All are transient:
30
+ * a later attempt from the same machine can succeed unchanged.
31
+ */
32
+ const TRANSIENT_CODES = new Set([
33
+ "ECONNREFUSED",
34
+ "ECONNRESET",
35
+ "ENOTFOUND",
36
+ "EAI_AGAIN",
37
+ "ETIMEDOUT",
38
+ "EPIPE",
39
+ "EHOSTUNREACH",
40
+ "EHOSTDOWN",
41
+ "ENETUNREACH",
42
+ "ENETDOWN",
43
+ "ECONNABORTED",
44
+ // undici's own timeout/socket failures
45
+ "UND_ERR_CONNECT_TIMEOUT",
46
+ "UND_ERR_HEADERS_TIMEOUT",
47
+ "UND_ERR_BODY_TIMEOUT",
48
+ "UND_ERR_SOCKET",
49
+ ]);
50
+ /**
51
+ * True when `err` is (or wraps) a transient network-transport failure. Walks the
52
+ * `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
53
+ * latter), matching either undici's canonical `TypeError: fetch failed` message
54
+ * or a known transient `.code`. Conservative by design: an arbitrary
55
+ * `new Error("network down")` with no code and a non-canonical message is NOT
56
+ * treated as transient, so only real transport-failure shapes qualify.
57
+ */
58
+ export function isTransientNetworkError(err, depth = 0) {
59
+ if (depth > 8 || !(err instanceof Error))
60
+ return false;
61
+ // undici's transport failure surfaces as exactly this message.
62
+ if (err.message === "fetch failed")
63
+ return true;
64
+ const code = err.code;
65
+ if (typeof code === "string" && TRANSIENT_CODES.has(code))
66
+ return true;
67
+ // AggregateError (e.g. happy-eyeballs) stashes the per-attempt errors here.
68
+ const aggregated = err.errors;
69
+ if (Array.isArray(aggregated)) {
70
+ for (const inner of aggregated) {
71
+ if (isTransientNetworkError(inner, depth + 1))
72
+ return true;
73
+ }
74
+ }
75
+ const cause = err.cause;
76
+ if (cause !== undefined && cause !== err) {
77
+ return isTransientNetworkError(cause, depth + 1);
78
+ }
79
+ return false;
80
+ }
81
+ //# sourceMappingURL=net-errors.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"net-errors.js","sourceRoot":"","sources":["../../src/lib/net-errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAEzC;;;;GAIG;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAS;IACtC,cAAc;IACd,YAAY;IACZ,WAAW;IACX,WAAW;IACX,WAAW;IACX,OAAO;IACP,cAAc;IACd,WAAW;IACX,aAAa;IACb,UAAU;IACV,cAAc;IACd,uCAAuC;IACvC,yBAAyB;IACzB,yBAAyB;IACzB,sBAAsB;IACtB,gBAAgB;CACjB,CAAC,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAY,EAAE,KAAK,GAAG,CAAC;IAC7D,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvD,+DAA+D;IAC/D,IAAI,GAAG,CAAC,OAAO,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IAEhD,MAAM,IAAI,GAAI,GAA0B,CAAC,IAAI,CAAC;IAC9C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvE,4EAA4E;IAC5E,MAAM,UAAU,GAAI,GAA4B,CAAC,MAAM,CAAC;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,IAAI,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAI,GAA2B,CAAC,KAAK,CAAC;IACjD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QACzC,OAAO,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=net-errors.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"net-errors.test.d.ts","sourceRoot":"","sources":["../../src/lib/net-errors.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,47 @@
1
+ import { describe, it, expect } from "vitest";
2
+ import { isTransientNetworkError, TRANSIENT_NETWORK_EXIT } from "./net-errors.js";
3
+ describe("isTransientNetworkError", () => {
4
+ it("treats undici's canonical `fetch failed` TypeError as transient", () => {
5
+ // This is the exact shape the runner's discovery step sees in production
6
+ // (HQ-SYNC-1W): `{"type":"error","message":"fetch failed","path":"(discovery)"}`.
7
+ expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
8
+ });
9
+ it("treats known transport `.code`s as transient", () => {
10
+ for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
11
+ const err = Object.assign(new Error("boom"), { code });
12
+ expect(isTransientNetworkError(err)).toBe(true);
13
+ }
14
+ });
15
+ it("walks the .cause chain (fetch failed wrapping a DNS error)", () => {
16
+ const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api"), {
17
+ code: "ENOTFOUND",
18
+ });
19
+ const err = Object.assign(new TypeError("fetch failed"), { cause });
20
+ expect(isTransientNetworkError(err)).toBe(true);
21
+ });
22
+ it("walks AggregateError.errors (happy-eyeballs)", () => {
23
+ const agg = Object.assign(new AggregateError([
24
+ Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),
25
+ Object.assign(new Error("v6"), { code: "EHOSTUNREACH" }),
26
+ ], "all attempts failed"), {});
27
+ expect(isTransientNetworkError(agg)).toBe(true);
28
+ });
29
+ it("is conservative: an arbitrary error is NOT transient", () => {
30
+ // A bare message like "network down" (no code, not the canonical undici
31
+ // message) must not be misclassified — only real transport shapes qualify.
32
+ expect(isTransientNetworkError(new Error("network down"))).toBe(false);
33
+ expect(isTransientNetworkError(new Error("boom"))).toBe(false);
34
+ expect(isTransientNetworkError("fetch failed")).toBe(false);
35
+ expect(isTransientNetworkError(undefined)).toBe(false);
36
+ });
37
+ it("does not infinite-loop on a self-referential cause", () => {
38
+ const err = new Error("weird");
39
+ err.cause = err;
40
+ expect(isTransientNetworkError(err)).toBe(false);
41
+ });
42
+ it("exposes a distinct exit code off the runner's other codes", () => {
43
+ expect(TRANSIENT_NETWORK_EXIT).toBe(75);
44
+ expect([0, 1, 2, 17]).not.toContain(TRANSIENT_NETWORK_EXIT);
45
+ });
46
+ });
47
+ //# sourceMappingURL=net-errors.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"net-errors.test.js","sourceRoot":"","sources":["../../src/lib/net-errors.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAElF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,yEAAyE;QACzE,kFAAkF;QAClF,MAAM,CAAC,uBAAuB,CAAC,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC;YAC3E,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,CAAC,EAAE;YAClE,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;SACzD,EAAE,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,wEAAwE;QACxE,2EAA2E;QAC3E,MAAM,CAAC,uBAAuB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,CAAC,uBAAuB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/D,MAAM,CAAC,uBAAuB,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,OAAO,CAAgC,CAAC;QAC9D,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAChB,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cloud",
3
- "version": "6.13.4",
3
+ "version": "6.13.5",
4
4
  "description": "HQ by Indigo cloud sync engine — bidirectional S3 sync for mobile access",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -8,6 +8,7 @@ import {
8
8
  OPERATION_LOCKED_EXIT,
9
9
  } from "../operation-lock.js";
10
10
  import { describeError } from "../lib/describe-error.js";
11
+ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
11
12
  import { getOrCreateMachineId } from "../lib/machine-id.js";
12
13
  import {
13
14
  TreeWatcher,
@@ -430,7 +431,16 @@ export async function runWatchLoop(
430
431
  try {
431
432
  while (!stopped) {
432
433
  const result = await runGuarded(passArgv);
433
- if (result !== 0) {
434
+ if (result === TRANSIENT_NETWORK_EXIT) {
435
+ // The machine was briefly offline (the discovery fetch failed after
436
+ // retries). That is NOT a crash — stay alive and retry on the next
437
+ // poll. Returning it here would exit the watcher and the menubar would
438
+ // report "watcher exited unexpectedly" for every network blip
439
+ // (HQ-SYNC-1W). Log to stderr so it still shows up as a breadcrumb.
440
+ process.stderr.write(
441
+ "hq-sync-runner: watch pass skipped — transient network failure; retrying next poll\n",
442
+ );
443
+ } else if (result !== 0) {
434
444
  return result;
435
445
  }
436
446
  await Promise.race([sleep(pollMs), stoppedSignal]);
@@ -35,6 +35,7 @@ import { FakeClock } from "../watcher.js";
35
35
  import { PERSONAL_VAULT_JOURNAL_SLUG } from "../journal.js";
36
36
  import { HQ_CLOUD_VERSION } from "../version.js";
37
37
  import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
38
+ import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
38
39
  import type { SyncResult, SyncOptions } from "../cli/sync.js";
39
40
  import type { ShareResult, ShareOptions } from "../cli/share.js";
40
41
  import type {
@@ -397,6 +398,49 @@ describe("memberships retry", () => {
397
398
  });
398
399
  });
399
400
 
401
+ it("a transient network failure (fetch failed) during discovery returns TRANSIENT_NETWORK_EXIT, not 1 (HQ-SYNC-1W)", async () => {
402
+ // Regression for the "auto-sync watcher exited unexpectedly (code=Some(1))"
403
+ // cluster: a machine that stays offline through all 3 membership retries
404
+ // used to `return 1`, which the watch loop propagated and the menubar
405
+ // reported as a crash for every network blip. undici surfaces the blip as
406
+ // `TypeError: fetch failed`; that must map to the retryable exit code so the
407
+ // watcher stays alive instead of crash-reporting. A one-shot run still sees
408
+ // a non-zero exit.
409
+ let listCalls = 0;
410
+ const stub = makeVaultStub();
411
+ stub.listMyMemberships = () => {
412
+ listCalls++;
413
+ return Promise.reject(new TypeError("fetch failed"));
414
+ };
415
+ const deps = makeDeps({ createVaultClient: () => stub });
416
+ const code = await runRunner(["--companies"], deps);
417
+ expect(code).toBe(TRANSIENT_NETWORK_EXIT);
418
+ expect(listCalls).toBe(3);
419
+ // Still surfaced for observability — just no longer a "code 1" crash.
420
+ const events = deps.stderr.events();
421
+ expect(events).toHaveLength(1);
422
+ expect(events[0]).toMatchObject({
423
+ type: "error",
424
+ message: "fetch failed",
425
+ path: "(discovery)",
426
+ });
427
+ });
428
+
429
+ it("a NON-transient discovery failure still returns 1 (unchanged)", async () => {
430
+ // Guard the conservative classifier: a generic error (not a transport
431
+ // shape) is still a hard exit 1, exactly as before.
432
+ let listCalls = 0;
433
+ const stub = makeVaultStub();
434
+ stub.listMyMemberships = () => {
435
+ listCalls++;
436
+ return Promise.reject(new Error("unexpected 500 from vault api"));
437
+ };
438
+ const deps = makeDeps({ createVaultClient: () => stub });
439
+ const code = await runRunner(["--companies"], deps);
440
+ expect(code).toBe(1);
441
+ expect(listCalls).toBe(3);
442
+ });
443
+
400
444
  it("VaultAuthError short-circuits retry (no retries on auth failure)", async () => {
401
445
  let listCalls = 0;
402
446
  const stub = makeVaultStub();
@@ -3307,6 +3351,39 @@ describe("runRunnerWithLoop — event-push wiring", () => {
3307
3351
  expect(watcher.disposed).toBe(true);
3308
3352
  });
3309
3353
 
3354
+ it("a TRANSIENT_NETWORK_EXIT poll pass does NOT surface as a crash — the loop retries (HQ-SYNC-1W)", async () => {
3355
+ // A transient offline blip must keep the watcher alive: the loop logs and
3356
+ // polls again instead of returning the code and letting the process exit
3357
+ // (which the menubar reports as "watcher exited unexpectedly"). A genuine
3358
+ // hard error on a later pass still terminates the loop.
3359
+ const watcher = makeWatcherStub();
3360
+ let n = 0;
3361
+ const runPass = vi.fn().mockImplementation(async () => {
3362
+ n++;
3363
+ // First pass: machine is offline (transient). Second pass: a real hard
3364
+ // error, used here only to end the otherwise-infinite loop deterministically.
3365
+ return n === 1 ? TRANSIENT_NETWORK_EXIT : 2;
3366
+ });
3367
+
3368
+ const code = await runRunnerWithLoop(
3369
+ ["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"],
3370
+ {
3371
+ runPass,
3372
+ clock: new FakeClock(),
3373
+ createWatcher: () => watcher,
3374
+ sleep: () => Promise.resolve(),
3375
+ onShutdownSignal: () => () => {},
3376
+ },
3377
+ );
3378
+
3379
+ // The transient pass did NOT end the loop — it ran again…
3380
+ expect(runPass).toHaveBeenCalledTimes(2);
3381
+ // …and the transient code was never surfaced as the watcher's exit code.
3382
+ expect(code).toBe(2);
3383
+ expect(code).not.toBe(TRANSIENT_NETWORK_EXIT);
3384
+ expect(watcher.disposed).toBe(true);
3385
+ });
3386
+
3310
3387
  it("without --event-push, no watcher is created (poll-only safety net)", async () => {
3311
3388
  let triggerShutdown = () => {};
3312
3389
  const createWatcher = vi.fn(() => makeWatcherStub());
@@ -110,6 +110,10 @@ import type { ReindexOptions, ReindexResult } from "../cli/reindex.js";
110
110
  import { pruneConflictIndex } from "../lib/conflict-index.js";
111
111
  import { materializeCodexAgents } from "../agent-codex-instructions.js";
112
112
  import { getOrCreateMachineId } from "../lib/machine-id.js";
113
+ import {
114
+ isTransientNetworkError,
115
+ TRANSIENT_NETWORK_EXIT,
116
+ } from "../lib/net-errors.js";
113
117
  import { describeError } from "../lib/describe-error.js";
114
118
  import type { Clock, TreeChangeBatch } from "../watcher.js";
115
119
  import type { PushReceiver, SyncEngineFn } from "../sync/push-receiver.js";
@@ -1160,15 +1164,19 @@ export async function runRunner(
1160
1164
  });
1161
1165
  return 0;
1162
1166
  }
1163
- // Any other failure is unrecoverable — surface as an error event and
1164
- // exit non-zero so the spawner knows the runner didn't get far enough
1165
- // to emit a useful protocol stream.
1167
+ // Surface the failure as an error event for observability either way.
1166
1168
  emit({
1167
1169
  type: "error",
1168
1170
  message: err instanceof Error ? err.message : String(err),
1169
1171
  path: "(discovery)",
1170
1172
  });
1171
- return 1;
1173
+ // A transient network failure (offline, DNS blip, vault API briefly
1174
+ // unreachable) is NOT a crash — it self-heals on the next poll. Return the
1175
+ // retryable exit code so the watch loop stays alive instead of exiting and
1176
+ // being reported as "watcher exited unexpectedly (code=Some(1))" for every
1177
+ // blip (HQ-SYNC-1W). A one-shot run still exits non-zero. Any other failure
1178
+ // is treated as a hard error (exit 1), as before.
1179
+ return isTransientNetworkError(err) ? TRANSIENT_NETWORK_EXIT : 1;
1172
1180
  }
1173
1181
 
1174
1182
  const targetPlan = await buildFanoutPlan({
@@ -0,0 +1,53 @@
1
+ import { describe, it, expect } from "vitest";
2
+ import { isTransientNetworkError, TRANSIENT_NETWORK_EXIT } from "./net-errors.js";
3
+
4
+ describe("isTransientNetworkError", () => {
5
+ it("treats undici's canonical `fetch failed` TypeError as transient", () => {
6
+ // This is the exact shape the runner's discovery step sees in production
7
+ // (HQ-SYNC-1W): `{"type":"error","message":"fetch failed","path":"(discovery)"}`.
8
+ expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
9
+ });
10
+
11
+ it("treats known transport `.code`s as transient", () => {
12
+ for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
13
+ const err = Object.assign(new Error("boom"), { code });
14
+ expect(isTransientNetworkError(err)).toBe(true);
15
+ }
16
+ });
17
+
18
+ it("walks the .cause chain (fetch failed wrapping a DNS error)", () => {
19
+ const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api"), {
20
+ code: "ENOTFOUND",
21
+ });
22
+ const err = Object.assign(new TypeError("fetch failed"), { cause });
23
+ expect(isTransientNetworkError(err)).toBe(true);
24
+ });
25
+
26
+ it("walks AggregateError.errors (happy-eyeballs)", () => {
27
+ const agg = Object.assign(new AggregateError([
28
+ Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),
29
+ Object.assign(new Error("v6"), { code: "EHOSTUNREACH" }),
30
+ ], "all attempts failed"), {});
31
+ expect(isTransientNetworkError(agg)).toBe(true);
32
+ });
33
+
34
+ it("is conservative: an arbitrary error is NOT transient", () => {
35
+ // A bare message like "network down" (no code, not the canonical undici
36
+ // message) must not be misclassified — only real transport shapes qualify.
37
+ expect(isTransientNetworkError(new Error("network down"))).toBe(false);
38
+ expect(isTransientNetworkError(new Error("boom"))).toBe(false);
39
+ expect(isTransientNetworkError("fetch failed")).toBe(false);
40
+ expect(isTransientNetworkError(undefined)).toBe(false);
41
+ });
42
+
43
+ it("does not infinite-loop on a self-referential cause", () => {
44
+ const err = new Error("weird") as Error & { cause?: unknown };
45
+ err.cause = err;
46
+ expect(isTransientNetworkError(err)).toBe(false);
47
+ });
48
+
49
+ it("exposes a distinct exit code off the runner's other codes", () => {
50
+ expect(TRANSIENT_NETWORK_EXIT).toBe(75);
51
+ expect([0, 1, 2, 17]).not.toContain(TRANSIENT_NETWORK_EXIT);
52
+ });
53
+ });
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Transient network-failure classification for the sync runner.
3
+ *
4
+ * The auto-sync watcher runs an unattended poll loop. At the top of every pass
5
+ * it calls `GET /membership/me` to resolve which companies to sync. When the
6
+ * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
7
+ * vault API momentarily unreachable) that call fails at the transport layer —
8
+ * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
9
+ * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
10
+ *
11
+ * This is NOT a crash: the next poll (~30s later) succeeds once the network is
12
+ * back. But the runner used to `return 1` for it, the watch loop propagated
13
+ * that non-zero exit, the process exited, and the menubar supervisor reported
14
+ * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
15
+ * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
16
+ * retry instead of surfacing a false crash, while a one-shot `hq sync` still
17
+ * exits non-zero so a human running it by hand sees the failure.
18
+ */
19
+
20
+ /**
21
+ * Distinct runner exit code for a transient network failure. The watch loop
22
+ * treats it as "stay alive, retry next tick" (never a crash); a one-shot run
23
+ * still exits non-zero so the failure is visible to a human/script. Value 75 =
24
+ * `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
25
+ * codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
26
+ */
27
+ export const TRANSIENT_NETWORK_EXIT = 75;
28
+
29
+ /**
30
+ * Node/undici error `code`s that mean "the request never completed at the
31
+ * transport layer" — the machine could not reach the host. All are transient:
32
+ * a later attempt from the same machine can succeed unchanged.
33
+ */
34
+ const TRANSIENT_CODES = new Set<string>([
35
+ "ECONNREFUSED",
36
+ "ECONNRESET",
37
+ "ENOTFOUND",
38
+ "EAI_AGAIN",
39
+ "ETIMEDOUT",
40
+ "EPIPE",
41
+ "EHOSTUNREACH",
42
+ "EHOSTDOWN",
43
+ "ENETUNREACH",
44
+ "ENETDOWN",
45
+ "ECONNABORTED",
46
+ // undici's own timeout/socket failures
47
+ "UND_ERR_CONNECT_TIMEOUT",
48
+ "UND_ERR_HEADERS_TIMEOUT",
49
+ "UND_ERR_BODY_TIMEOUT",
50
+ "UND_ERR_SOCKET",
51
+ ]);
52
+
53
+ /**
54
+ * True when `err` is (or wraps) a transient network-transport failure. Walks the
55
+ * `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
56
+ * latter), matching either undici's canonical `TypeError: fetch failed` message
57
+ * or a known transient `.code`. Conservative by design: an arbitrary
58
+ * `new Error("network down")` with no code and a non-canonical message is NOT
59
+ * treated as transient, so only real transport-failure shapes qualify.
60
+ */
61
+ export function isTransientNetworkError(err: unknown, depth = 0): boolean {
62
+ if (depth > 8 || !(err instanceof Error)) return false;
63
+
64
+ // undici's transport failure surfaces as exactly this message.
65
+ if (err.message === "fetch failed") return true;
66
+
67
+ const code = (err as { code?: unknown }).code;
68
+ if (typeof code === "string" && TRANSIENT_CODES.has(code)) return true;
69
+
70
+ // AggregateError (e.g. happy-eyeballs) stashes the per-attempt errors here.
71
+ const aggregated = (err as { errors?: unknown }).errors;
72
+ if (Array.isArray(aggregated)) {
73
+ for (const inner of aggregated) {
74
+ if (isTransientNetworkError(inner, depth + 1)) return true;
75
+ }
76
+ }
77
+
78
+ const cause = (err as { cause?: unknown }).cause;
79
+ if (cause !== undefined && cause !== err) {
80
+ return isTransientNetworkError(cause, depth + 1);
81
+ }
82
+ return false;
83
+ }