@indigoai-us/hq-cloud 6.13.4 → 6.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +10 -1
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +9 -4
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +69 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/lib/net-errors.d.ts +36 -0
- package/dist/lib/net-errors.d.ts.map +1 -0
- package/dist/lib/net-errors.js +81 -0
- package/dist/lib/net-errors.js.map +1 -0
- package/dist/lib/net-errors.test.d.ts +2 -0
- package/dist/lib/net-errors.test.d.ts.map +1 -0
- package/dist/lib/net-errors.test.js +47 -0
- package/dist/lib/net-errors.test.js.map +1 -0
- package/package.json +1 -1
- package/src/bin/sync-runner-watch-loop.ts +11 -1
- package/src/bin/sync-runner.test.ts +77 -0
- package/src/bin/sync-runner.ts +12 -4
- package/src/lib/net-errors.test.ts +53 -0
- package/src/lib/net-errors.ts +83 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transient network-failure classification for the sync runner.
|
|
3
|
+
*
|
|
4
|
+
* The auto-sync watcher runs an unattended poll loop. At the top of every pass
|
|
5
|
+
* it calls `GET /membership/me` to resolve which companies to sync. When the
|
|
6
|
+
* machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
|
|
7
|
+
* vault API momentarily unreachable) that call fails at the transport layer —
|
|
8
|
+
* Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
|
|
9
|
+
* `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
|
|
10
|
+
*
|
|
11
|
+
* This is NOT a crash: the next poll (~30s later) succeeds once the network is
|
|
12
|
+
* back. But the runner used to `return 1` for it, the watch loop propagated
|
|
13
|
+
* that non-zero exit, the process exited, and the menubar supervisor reported
|
|
14
|
+
* "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
|
|
15
|
+
* HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
|
|
16
|
+
* retry instead of surfacing a false crash, while a one-shot `hq sync` still
|
|
17
|
+
* exits non-zero so a human running it by hand sees the failure.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Distinct runner exit code for a transient network failure. The watch loop
|
|
21
|
+
* treats it as "stay alive, retry next tick" (never a crash); a one-shot run
|
|
22
|
+
* still exits non-zero so the failure is visible to a human/script. Value 75 =
|
|
23
|
+
* `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
|
|
24
|
+
* codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
|
|
25
|
+
*/
|
|
26
|
+
export declare const TRANSIENT_NETWORK_EXIT = 75;
|
|
27
|
+
/**
|
|
28
|
+
* True when `err` is (or wraps) a transient network-transport failure. Walks the
|
|
29
|
+
* `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
|
|
30
|
+
* latter), matching either undici's canonical `TypeError: fetch failed` message
|
|
31
|
+
* or a known transient `.code`. Conservative by design: an arbitrary
|
|
32
|
+
* `new Error("network down")` with no code and a non-canonical message is NOT
|
|
33
|
+
* treated as transient, so only real transport-failure shapes qualify.
|
|
34
|
+
*/
|
|
35
|
+
export declare function isTransientNetworkError(err: unknown, depth?: number): boolean;
|
|
36
|
+
//# sourceMappingURL=net-errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"net-errors.d.ts","sourceRoot":"","sources":["../../src/lib/net-errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAC;AA0BzC;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,SAAI,GAAG,OAAO,CAsBxE"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transient network-failure classification for the sync runner.
|
|
3
|
+
*
|
|
4
|
+
* The auto-sync watcher runs an unattended poll loop. At the top of every pass
|
|
5
|
+
* it calls `GET /membership/me` to resolve which companies to sync. When the
|
|
6
|
+
* machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
|
|
7
|
+
* vault API momentarily unreachable) that call fails at the transport layer —
|
|
8
|
+
* Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
|
|
9
|
+
* `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
|
|
10
|
+
*
|
|
11
|
+
* This is NOT a crash: the next poll (~30s later) succeeds once the network is
|
|
12
|
+
* back. But the runner used to `return 1` for it, the watch loop propagated
|
|
13
|
+
* that non-zero exit, the process exited, and the menubar supervisor reported
|
|
14
|
+
* "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
|
|
15
|
+
* HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
|
|
16
|
+
* retry instead of surfacing a false crash, while a one-shot `hq sync` still
|
|
17
|
+
* exits non-zero so a human running it by hand sees the failure.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* Distinct runner exit code for a transient network failure. The watch loop
|
|
21
|
+
* treats it as "stay alive, retry next tick" (never a crash); a one-shot run
|
|
22
|
+
* still exits non-zero so the failure is visible to a human/script. Value 75 =
|
|
23
|
+
* `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
|
|
24
|
+
* codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
|
|
25
|
+
*/
|
|
26
|
+
export const TRANSIENT_NETWORK_EXIT = 75;
|
|
27
|
+
/**
|
|
28
|
+
* Node/undici error `code`s that mean "the request never completed at the
|
|
29
|
+
* transport layer" — the machine could not reach the host. All are transient:
|
|
30
|
+
* a later attempt from the same machine can succeed unchanged.
|
|
31
|
+
*/
|
|
32
|
+
const TRANSIENT_CODES = new Set([
|
|
33
|
+
"ECONNREFUSED",
|
|
34
|
+
"ECONNRESET",
|
|
35
|
+
"ENOTFOUND",
|
|
36
|
+
"EAI_AGAIN",
|
|
37
|
+
"ETIMEDOUT",
|
|
38
|
+
"EPIPE",
|
|
39
|
+
"EHOSTUNREACH",
|
|
40
|
+
"EHOSTDOWN",
|
|
41
|
+
"ENETUNREACH",
|
|
42
|
+
"ENETDOWN",
|
|
43
|
+
"ECONNABORTED",
|
|
44
|
+
// undici's own timeout/socket failures
|
|
45
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
46
|
+
"UND_ERR_HEADERS_TIMEOUT",
|
|
47
|
+
"UND_ERR_BODY_TIMEOUT",
|
|
48
|
+
"UND_ERR_SOCKET",
|
|
49
|
+
]);
|
|
50
|
+
/**
|
|
51
|
+
* True when `err` is (or wraps) a transient network-transport failure. Walks the
|
|
52
|
+
* `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
|
|
53
|
+
* latter), matching either undici's canonical `TypeError: fetch failed` message
|
|
54
|
+
* or a known transient `.code`. Conservative by design: an arbitrary
|
|
55
|
+
* `new Error("network down")` with no code and a non-canonical message is NOT
|
|
56
|
+
* treated as transient, so only real transport-failure shapes qualify.
|
|
57
|
+
*/
|
|
58
|
+
export function isTransientNetworkError(err, depth = 0) {
|
|
59
|
+
if (depth > 8 || !(err instanceof Error))
|
|
60
|
+
return false;
|
|
61
|
+
// undici's transport failure surfaces as exactly this message.
|
|
62
|
+
if (err.message === "fetch failed")
|
|
63
|
+
return true;
|
|
64
|
+
const code = err.code;
|
|
65
|
+
if (typeof code === "string" && TRANSIENT_CODES.has(code))
|
|
66
|
+
return true;
|
|
67
|
+
// AggregateError (e.g. happy-eyeballs) stashes the per-attempt errors here.
|
|
68
|
+
const aggregated = err.errors;
|
|
69
|
+
if (Array.isArray(aggregated)) {
|
|
70
|
+
for (const inner of aggregated) {
|
|
71
|
+
if (isTransientNetworkError(inner, depth + 1))
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const cause = err.cause;
|
|
76
|
+
if (cause !== undefined && cause !== err) {
|
|
77
|
+
return isTransientNetworkError(cause, depth + 1);
|
|
78
|
+
}
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=net-errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"net-errors.js","sourceRoot":"","sources":["../../src/lib/net-errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAEzC;;;;GAIG;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAS;IACtC,cAAc;IACd,YAAY;IACZ,WAAW;IACX,WAAW;IACX,WAAW;IACX,OAAO;IACP,cAAc;IACd,WAAW;IACX,aAAa;IACb,UAAU;IACV,cAAc;IACd,uCAAuC;IACvC,yBAAyB;IACzB,yBAAyB;IACzB,sBAAsB;IACtB,gBAAgB;CACjB,CAAC,CAAC;AAEH;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAY,EAAE,KAAK,GAAG,CAAC;IAC7D,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEvD,+DAA+D;IAC/D,IAAI,GAAG,CAAC,OAAO,KAAK,cAAc;QAAE,OAAO,IAAI,CAAC;IAEhD,MAAM,IAAI,GAAI,GAA0B,CAAC,IAAI,CAAC;IAC9C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvE,4EAA4E;IAC5E,MAAM,UAAU,GAAI,GAA4B,CAAC,MAAM,CAAC;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,IAAI,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAI,GAA2B,CAAC,KAAK,CAAC;IACjD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QACzC,OAAO,uBAAuB,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"net-errors.test.d.ts","sourceRoot":"","sources":["../../src/lib/net-errors.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { isTransientNetworkError, TRANSIENT_NETWORK_EXIT } from "./net-errors.js";
|
|
3
|
+
describe("isTransientNetworkError", () => {
|
|
4
|
+
it("treats undici's canonical `fetch failed` TypeError as transient", () => {
|
|
5
|
+
// This is the exact shape the runner's discovery step sees in production
|
|
6
|
+
// (HQ-SYNC-1W): `{"type":"error","message":"fetch failed","path":"(discovery)"}`.
|
|
7
|
+
expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
|
|
8
|
+
});
|
|
9
|
+
it("treats known transport `.code`s as transient", () => {
|
|
10
|
+
for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
|
|
11
|
+
const err = Object.assign(new Error("boom"), { code });
|
|
12
|
+
expect(isTransientNetworkError(err)).toBe(true);
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
it("walks the .cause chain (fetch failed wrapping a DNS error)", () => {
|
|
16
|
+
const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api"), {
|
|
17
|
+
code: "ENOTFOUND",
|
|
18
|
+
});
|
|
19
|
+
const err = Object.assign(new TypeError("fetch failed"), { cause });
|
|
20
|
+
expect(isTransientNetworkError(err)).toBe(true);
|
|
21
|
+
});
|
|
22
|
+
it("walks AggregateError.errors (happy-eyeballs)", () => {
|
|
23
|
+
const agg = Object.assign(new AggregateError([
|
|
24
|
+
Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),
|
|
25
|
+
Object.assign(new Error("v6"), { code: "EHOSTUNREACH" }),
|
|
26
|
+
], "all attempts failed"), {});
|
|
27
|
+
expect(isTransientNetworkError(agg)).toBe(true);
|
|
28
|
+
});
|
|
29
|
+
it("is conservative: an arbitrary error is NOT transient", () => {
|
|
30
|
+
// A bare message like "network down" (no code, not the canonical undici
|
|
31
|
+
// message) must not be misclassified — only real transport shapes qualify.
|
|
32
|
+
expect(isTransientNetworkError(new Error("network down"))).toBe(false);
|
|
33
|
+
expect(isTransientNetworkError(new Error("boom"))).toBe(false);
|
|
34
|
+
expect(isTransientNetworkError("fetch failed")).toBe(false);
|
|
35
|
+
expect(isTransientNetworkError(undefined)).toBe(false);
|
|
36
|
+
});
|
|
37
|
+
it("does not infinite-loop on a self-referential cause", () => {
|
|
38
|
+
const err = new Error("weird");
|
|
39
|
+
err.cause = err;
|
|
40
|
+
expect(isTransientNetworkError(err)).toBe(false);
|
|
41
|
+
});
|
|
42
|
+
it("exposes a distinct exit code off the runner's other codes", () => {
|
|
43
|
+
expect(TRANSIENT_NETWORK_EXIT).toBe(75);
|
|
44
|
+
expect([0, 1, 2, 17]).not.toContain(TRANSIENT_NETWORK_EXIT);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
//# sourceMappingURL=net-errors.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"net-errors.test.js","sourceRoot":"","sources":["../../src/lib/net-errors.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAElF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,yEAAyE;QACzE,kFAAkF;QAClF,MAAM,CAAC,uBAAuB,CAAC,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,KAAK,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC;YAC3E,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;QACpE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,CAAC,EAAE;YAClE,IAAI,EAAE,WAAW;SAClB,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;SACzD,EAAE,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,wEAAwE;QACxE,2EAA2E;QAC3E,MAAM,CAAC,uBAAuB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,CAAC,uBAAuB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/D,MAAM,CAAC,uBAAuB,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,MAAM,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,OAAO,CAAgC,CAAC;QAC9D,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC;QAChB,MAAM,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
OPERATION_LOCKED_EXIT,
|
|
9
9
|
} from "../operation-lock.js";
|
|
10
10
|
import { describeError } from "../lib/describe-error.js";
|
|
11
|
+
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
11
12
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
12
13
|
import {
|
|
13
14
|
TreeWatcher,
|
|
@@ -430,7 +431,16 @@ export async function runWatchLoop(
|
|
|
430
431
|
try {
|
|
431
432
|
while (!stopped) {
|
|
432
433
|
const result = await runGuarded(passArgv);
|
|
433
|
-
if (result
|
|
434
|
+
if (result === TRANSIENT_NETWORK_EXIT) {
|
|
435
|
+
// The machine was briefly offline (the discovery fetch failed after
|
|
436
|
+
// retries). That is NOT a crash — stay alive and retry on the next
|
|
437
|
+
// poll. Returning it here would exit the watcher and the menubar would
|
|
438
|
+
// report "watcher exited unexpectedly" for every network blip
|
|
439
|
+
// (HQ-SYNC-1W). Log to stderr so it still shows up as a breadcrumb.
|
|
440
|
+
process.stderr.write(
|
|
441
|
+
"hq-sync-runner: watch pass skipped — transient network failure; retrying next poll\n",
|
|
442
|
+
);
|
|
443
|
+
} else if (result !== 0) {
|
|
434
444
|
return result;
|
|
435
445
|
}
|
|
436
446
|
await Promise.race([sleep(pollMs), stoppedSignal]);
|
|
@@ -35,6 +35,7 @@ import { FakeClock } from "../watcher.js";
|
|
|
35
35
|
import { PERSONAL_VAULT_JOURNAL_SLUG } from "../journal.js";
|
|
36
36
|
import { HQ_CLOUD_VERSION } from "../version.js";
|
|
37
37
|
import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
|
|
38
|
+
import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
|
|
38
39
|
import type { SyncResult, SyncOptions } from "../cli/sync.js";
|
|
39
40
|
import type { ShareResult, ShareOptions } from "../cli/share.js";
|
|
40
41
|
import type {
|
|
@@ -397,6 +398,49 @@ describe("memberships retry", () => {
|
|
|
397
398
|
});
|
|
398
399
|
});
|
|
399
400
|
|
|
401
|
+
it("a transient network failure (fetch failed) during discovery returns TRANSIENT_NETWORK_EXIT, not 1 (HQ-SYNC-1W)", async () => {
|
|
402
|
+
// Regression for the "auto-sync watcher exited unexpectedly (code=Some(1))"
|
|
403
|
+
// cluster: a machine that stays offline through all 3 membership retries
|
|
404
|
+
// used to `return 1`, which the watch loop propagated and the menubar
|
|
405
|
+
// reported as a crash for every network blip. undici surfaces the blip as
|
|
406
|
+
// `TypeError: fetch failed`; that must map to the retryable exit code so the
|
|
407
|
+
// watcher stays alive instead of crash-reporting. A one-shot run still sees
|
|
408
|
+
// a non-zero exit.
|
|
409
|
+
let listCalls = 0;
|
|
410
|
+
const stub = makeVaultStub();
|
|
411
|
+
stub.listMyMemberships = () => {
|
|
412
|
+
listCalls++;
|
|
413
|
+
return Promise.reject(new TypeError("fetch failed"));
|
|
414
|
+
};
|
|
415
|
+
const deps = makeDeps({ createVaultClient: () => stub });
|
|
416
|
+
const code = await runRunner(["--companies"], deps);
|
|
417
|
+
expect(code).toBe(TRANSIENT_NETWORK_EXIT);
|
|
418
|
+
expect(listCalls).toBe(3);
|
|
419
|
+
// Still surfaced for observability — just no longer a "code 1" crash.
|
|
420
|
+
const events = deps.stderr.events();
|
|
421
|
+
expect(events).toHaveLength(1);
|
|
422
|
+
expect(events[0]).toMatchObject({
|
|
423
|
+
type: "error",
|
|
424
|
+
message: "fetch failed",
|
|
425
|
+
path: "(discovery)",
|
|
426
|
+
});
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
it("a NON-transient discovery failure still returns 1 (unchanged)", async () => {
|
|
430
|
+
// Guard the conservative classifier: a generic error (not a transport
|
|
431
|
+
// shape) is still a hard exit 1, exactly as before.
|
|
432
|
+
let listCalls = 0;
|
|
433
|
+
const stub = makeVaultStub();
|
|
434
|
+
stub.listMyMemberships = () => {
|
|
435
|
+
listCalls++;
|
|
436
|
+
return Promise.reject(new Error("unexpected 500 from vault api"));
|
|
437
|
+
};
|
|
438
|
+
const deps = makeDeps({ createVaultClient: () => stub });
|
|
439
|
+
const code = await runRunner(["--companies"], deps);
|
|
440
|
+
expect(code).toBe(1);
|
|
441
|
+
expect(listCalls).toBe(3);
|
|
442
|
+
});
|
|
443
|
+
|
|
400
444
|
it("VaultAuthError short-circuits retry (no retries on auth failure)", async () => {
|
|
401
445
|
let listCalls = 0;
|
|
402
446
|
const stub = makeVaultStub();
|
|
@@ -3307,6 +3351,39 @@ describe("runRunnerWithLoop — event-push wiring", () => {
|
|
|
3307
3351
|
expect(watcher.disposed).toBe(true);
|
|
3308
3352
|
});
|
|
3309
3353
|
|
|
3354
|
+
it("a TRANSIENT_NETWORK_EXIT poll pass does NOT surface as a crash — the loop retries (HQ-SYNC-1W)", async () => {
|
|
3355
|
+
// A transient offline blip must keep the watcher alive: the loop logs and
|
|
3356
|
+
// polls again instead of returning the code and letting the process exit
|
|
3357
|
+
// (which the menubar reports as "watcher exited unexpectedly"). A genuine
|
|
3358
|
+
// hard error on a later pass still terminates the loop.
|
|
3359
|
+
const watcher = makeWatcherStub();
|
|
3360
|
+
let n = 0;
|
|
3361
|
+
const runPass = vi.fn().mockImplementation(async () => {
|
|
3362
|
+
n++;
|
|
3363
|
+
// First pass: machine is offline (transient). Second pass: a real hard
|
|
3364
|
+
// error, used here only to end the otherwise-infinite loop deterministically.
|
|
3365
|
+
return n === 1 ? TRANSIENT_NETWORK_EXIT : 2;
|
|
3366
|
+
});
|
|
3367
|
+
|
|
3368
|
+
const code = await runRunnerWithLoop(
|
|
3369
|
+
["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"],
|
|
3370
|
+
{
|
|
3371
|
+
runPass,
|
|
3372
|
+
clock: new FakeClock(),
|
|
3373
|
+
createWatcher: () => watcher,
|
|
3374
|
+
sleep: () => Promise.resolve(),
|
|
3375
|
+
onShutdownSignal: () => () => {},
|
|
3376
|
+
},
|
|
3377
|
+
);
|
|
3378
|
+
|
|
3379
|
+
// The transient pass did NOT end the loop — it ran again…
|
|
3380
|
+
expect(runPass).toHaveBeenCalledTimes(2);
|
|
3381
|
+
// …and the transient code was never surfaced as the watcher's exit code.
|
|
3382
|
+
expect(code).toBe(2);
|
|
3383
|
+
expect(code).not.toBe(TRANSIENT_NETWORK_EXIT);
|
|
3384
|
+
expect(watcher.disposed).toBe(true);
|
|
3385
|
+
});
|
|
3386
|
+
|
|
3310
3387
|
it("without --event-push, no watcher is created (poll-only safety net)", async () => {
|
|
3311
3388
|
let triggerShutdown = () => {};
|
|
3312
3389
|
const createWatcher = vi.fn(() => makeWatcherStub());
|
package/src/bin/sync-runner.ts
CHANGED
|
@@ -110,6 +110,10 @@ import type { ReindexOptions, ReindexResult } from "../cli/reindex.js";
|
|
|
110
110
|
import { pruneConflictIndex } from "../lib/conflict-index.js";
|
|
111
111
|
import { materializeCodexAgents } from "../agent-codex-instructions.js";
|
|
112
112
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
113
|
+
import {
|
|
114
|
+
isTransientNetworkError,
|
|
115
|
+
TRANSIENT_NETWORK_EXIT,
|
|
116
|
+
} from "../lib/net-errors.js";
|
|
113
117
|
import { describeError } from "../lib/describe-error.js";
|
|
114
118
|
import type { Clock, TreeChangeBatch } from "../watcher.js";
|
|
115
119
|
import type { PushReceiver, SyncEngineFn } from "../sync/push-receiver.js";
|
|
@@ -1160,15 +1164,19 @@ export async function runRunner(
|
|
|
1160
1164
|
});
|
|
1161
1165
|
return 0;
|
|
1162
1166
|
}
|
|
1163
|
-
//
|
|
1164
|
-
// exit non-zero so the spawner knows the runner didn't get far enough
|
|
1165
|
-
// to emit a useful protocol stream.
|
|
1167
|
+
// Surface the failure as an error event for observability either way.
|
|
1166
1168
|
emit({
|
|
1167
1169
|
type: "error",
|
|
1168
1170
|
message: err instanceof Error ? err.message : String(err),
|
|
1169
1171
|
path: "(discovery)",
|
|
1170
1172
|
});
|
|
1171
|
-
|
|
1173
|
+
// A transient network failure (offline, DNS blip, vault API briefly
|
|
1174
|
+
// unreachable) is NOT a crash — it self-heals on the next poll. Return the
|
|
1175
|
+
// retryable exit code so the watch loop stays alive instead of exiting and
|
|
1176
|
+
// being reported as "watcher exited unexpectedly (code=Some(1))" for every
|
|
1177
|
+
// blip (HQ-SYNC-1W). A one-shot run still exits non-zero. Any other failure
|
|
1178
|
+
// is treated as a hard error (exit 1), as before.
|
|
1179
|
+
return isTransientNetworkError(err) ? TRANSIENT_NETWORK_EXIT : 1;
|
|
1172
1180
|
}
|
|
1173
1181
|
|
|
1174
1182
|
const targetPlan = await buildFanoutPlan({
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { isTransientNetworkError, TRANSIENT_NETWORK_EXIT } from "./net-errors.js";
|
|
3
|
+
|
|
4
|
+
describe("isTransientNetworkError", () => {
|
|
5
|
+
it("treats undici's canonical `fetch failed` TypeError as transient", () => {
|
|
6
|
+
// This is the exact shape the runner's discovery step sees in production
|
|
7
|
+
// (HQ-SYNC-1W): `{"type":"error","message":"fetch failed","path":"(discovery)"}`.
|
|
8
|
+
expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("treats known transport `.code`s as transient", () => {
|
|
12
|
+
for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
|
|
13
|
+
const err = Object.assign(new Error("boom"), { code });
|
|
14
|
+
expect(isTransientNetworkError(err)).toBe(true);
|
|
15
|
+
}
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("walks the .cause chain (fetch failed wrapping a DNS error)", () => {
|
|
19
|
+
const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api"), {
|
|
20
|
+
code: "ENOTFOUND",
|
|
21
|
+
});
|
|
22
|
+
const err = Object.assign(new TypeError("fetch failed"), { cause });
|
|
23
|
+
expect(isTransientNetworkError(err)).toBe(true);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it("walks AggregateError.errors (happy-eyeballs)", () => {
|
|
27
|
+
const agg = Object.assign(new AggregateError([
|
|
28
|
+
Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),
|
|
29
|
+
Object.assign(new Error("v6"), { code: "EHOSTUNREACH" }),
|
|
30
|
+
], "all attempts failed"), {});
|
|
31
|
+
expect(isTransientNetworkError(agg)).toBe(true);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("is conservative: an arbitrary error is NOT transient", () => {
|
|
35
|
+
// A bare message like "network down" (no code, not the canonical undici
|
|
36
|
+
// message) must not be misclassified — only real transport shapes qualify.
|
|
37
|
+
expect(isTransientNetworkError(new Error("network down"))).toBe(false);
|
|
38
|
+
expect(isTransientNetworkError(new Error("boom"))).toBe(false);
|
|
39
|
+
expect(isTransientNetworkError("fetch failed")).toBe(false);
|
|
40
|
+
expect(isTransientNetworkError(undefined)).toBe(false);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("does not infinite-loop on a self-referential cause", () => {
|
|
44
|
+
const err = new Error("weird") as Error & { cause?: unknown };
|
|
45
|
+
err.cause = err;
|
|
46
|
+
expect(isTransientNetworkError(err)).toBe(false);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it("exposes a distinct exit code off the runner's other codes", () => {
|
|
50
|
+
expect(TRANSIENT_NETWORK_EXIT).toBe(75);
|
|
51
|
+
expect([0, 1, 2, 17]).not.toContain(TRANSIENT_NETWORK_EXIT);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transient network-failure classification for the sync runner.
|
|
3
|
+
*
|
|
4
|
+
* The auto-sync watcher runs an unattended poll loop. At the top of every pass
|
|
5
|
+
* it calls `GET /membership/me` to resolve which companies to sync. When the
|
|
6
|
+
* machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
|
|
7
|
+
* vault API momentarily unreachable) that call fails at the transport layer —
|
|
8
|
+
* Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
|
|
9
|
+
* `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
|
|
10
|
+
*
|
|
11
|
+
* This is NOT a crash: the next poll (~30s later) succeeds once the network is
|
|
12
|
+
* back. But the runner used to `return 1` for it, the watch loop propagated
|
|
13
|
+
* that non-zero exit, the process exited, and the menubar supervisor reported
|
|
14
|
+
* "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
|
|
15
|
+
* HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
|
|
16
|
+
* retry instead of surfacing a false crash, while a one-shot `hq sync` still
|
|
17
|
+
* exits non-zero so a human running it by hand sees the failure.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Distinct runner exit code for a transient network failure. The watch loop
|
|
22
|
+
* treats it as "stay alive, retry next tick" (never a crash); a one-shot run
|
|
23
|
+
* still exits non-zero so the failure is visible to a human/script. Value 75 =
|
|
24
|
+
* `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
|
|
25
|
+
* codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
|
|
26
|
+
*/
|
|
27
|
+
export const TRANSIENT_NETWORK_EXIT = 75;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Node/undici error `code`s that mean "the request never completed at the
|
|
31
|
+
* transport layer" — the machine could not reach the host. All are transient:
|
|
32
|
+
* a later attempt from the same machine can succeed unchanged.
|
|
33
|
+
*/
|
|
34
|
+
const TRANSIENT_CODES = new Set<string>([
|
|
35
|
+
"ECONNREFUSED",
|
|
36
|
+
"ECONNRESET",
|
|
37
|
+
"ENOTFOUND",
|
|
38
|
+
"EAI_AGAIN",
|
|
39
|
+
"ETIMEDOUT",
|
|
40
|
+
"EPIPE",
|
|
41
|
+
"EHOSTUNREACH",
|
|
42
|
+
"EHOSTDOWN",
|
|
43
|
+
"ENETUNREACH",
|
|
44
|
+
"ENETDOWN",
|
|
45
|
+
"ECONNABORTED",
|
|
46
|
+
// undici's own timeout/socket failures
|
|
47
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
48
|
+
"UND_ERR_HEADERS_TIMEOUT",
|
|
49
|
+
"UND_ERR_BODY_TIMEOUT",
|
|
50
|
+
"UND_ERR_SOCKET",
|
|
51
|
+
]);
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* True when `err` is (or wraps) a transient network-transport failure. Walks the
|
|
55
|
+
* `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
|
|
56
|
+
* latter), matching either undici's canonical `TypeError: fetch failed` message
|
|
57
|
+
* or a known transient `.code`. Conservative by design: an arbitrary
|
|
58
|
+
* `new Error("network down")` with no code and a non-canonical message is NOT
|
|
59
|
+
* treated as transient, so only real transport-failure shapes qualify.
|
|
60
|
+
*/
|
|
61
|
+
export function isTransientNetworkError(err: unknown, depth = 0): boolean {
|
|
62
|
+
if (depth > 8 || !(err instanceof Error)) return false;
|
|
63
|
+
|
|
64
|
+
// undici's transport failure surfaces as exactly this message.
|
|
65
|
+
if (err.message === "fetch failed") return true;
|
|
66
|
+
|
|
67
|
+
const code = (err as { code?: unknown }).code;
|
|
68
|
+
if (typeof code === "string" && TRANSIENT_CODES.has(code)) return true;
|
|
69
|
+
|
|
70
|
+
// AggregateError (e.g. happy-eyeballs) stashes the per-attempt errors here.
|
|
71
|
+
const aggregated = (err as { errors?: unknown }).errors;
|
|
72
|
+
if (Array.isArray(aggregated)) {
|
|
73
|
+
for (const inner of aggregated) {
|
|
74
|
+
if (isTransientNetworkError(inner, depth + 1)) return true;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const cause = (err as { cause?: unknown }).cause;
|
|
79
|
+
if (cause !== undefined && cause !== err) {
|
|
80
|
+
return isTransientNetworkError(cause, depth + 1);
|
|
81
|
+
}
|
|
82
|
+
return false;
|
|
83
|
+
}
|