@indigoai-us/hq-cloud 6.13.4 → 6.14.0

package/src/bin/sync-runner.ts

@@ -110,6 +110,10 @@ import type { ReindexOptions, ReindexResult } from "../cli/reindex.js";
 import { pruneConflictIndex } from "../lib/conflict-index.js";
 import { materializeCodexAgents } from "../agent-codex-instructions.js";
 import { getOrCreateMachineId } from "../lib/machine-id.js";
+import {
+  isTransientNetworkError,
+  TRANSIENT_NETWORK_EXIT,
+} from "../lib/net-errors.js";
 import { describeError } from "../lib/describe-error.js";
 import type { Clock, TreeChangeBatch } from "../watcher.js";
 import type { PushReceiver, SyncEngineFn } from "../sync/push-receiver.js";
@@ -719,6 +723,8 @@ interface ParsedArgs {
    * mode (single-company runs never visit the personal target).
    */
   skipPersonal: boolean;
+  /** Repeatable company/personal-relative path scope for push-only passes. */
+  scopePaths: string[];
   /**
    * Bounded wait (seconds) for the per-root operation lock when another op is
    * already running. `0` → refuse immediately (pre-wait behavior); omitted →
@@ -738,6 +744,7 @@ function parseArgs(argv: string[]): ParsedArgs | { error: string } {
   let watch = false;
   let pollRemoteMs: number | undefined;
   let skipPersonal = false;
+  const scopePaths: string[] = [];
   let eventPush = false;
   let lockTimeoutSec: number | undefined;
 
@@ -813,6 +820,12 @@ function parseArgs(argv: string[]): ParsedArgs | { error: string } {
         // @getindigo.ai identities for the first release.
         eventPush = true;
         break;
+      case "--scope-path": {
+        const val = argv[++i];
+        if (!val) return { error: "--scope-path requires a value" };
+        scopePaths.push(val);
+        break;
+      }
       case "--lock-timeout": {
         const val = argv[++i];
         if (!val) return { error: "--lock-timeout requires a value (seconds)" };
@@ -863,6 +876,7 @@ function parseArgs(argv: string[]): ParsedArgs | { error: string } {
     watch,
     pollRemoteMs,
     skipPersonal,
+    scopePaths,
     eventPush,
     lockTimeoutSec,
   };
@@ -1160,15 +1174,19 @@ export async function runRunner(
       });
       return 0;
     }
-    // Any other failure is unrecoverable — surface as an error event and
-    // exit non-zero so the spawner knows the runner didn't get far enough
-    // to emit a useful protocol stream.
+    // Surface the failure as an error event for observability either way.
     emit({
       type: "error",
       message: err instanceof Error ? err.message : String(err),
       path: "(discovery)",
     });
-    return 1;
+    // A transient network failure (offline, DNS blip, vault API briefly
+    // unreachable) is NOT a crash — it self-heals on the next poll. Return the
+    // retryable exit code so the watch loop stays alive instead of exiting and
+    // being reported as "watcher exited unexpectedly (code=Some(1))" for every
+    // blip (HQ-SYNC-1W). A one-shot run still exits non-zero. Any other failure
+    // is treated as a hard error (exit 1), as before.
+    return isTransientNetworkError(err) ? TRANSIENT_NETWORK_EXIT : 1;
   }
 
   const targetPlan = await buildFanoutPlan({
@@ -1211,6 +1229,7 @@ export async function runRunner(
     shareFn,
     resolveDeletePolicy,
     emit,
+    scopePaths: parsed.scopePaths,
   });
   const { errors, allConflicts } = fanout;
   const rollup = rollupAllComplete(plan, fanout.stateByCompany);

package/src/lib/net-errors.test.ts

@@ -0,0 +1,53 @@
+import { describe, it, expect } from "vitest";
+import { isTransientNetworkError, TRANSIENT_NETWORK_EXIT } from "./net-errors.js";
+
+describe("isTransientNetworkError", () => {
+  it("treats undici's canonical `fetch failed` TypeError as transient", () => {
+    // This is the exact shape the runner's discovery step sees in production
+    // (HQ-SYNC-1W): `{"type":"error","message":"fetch failed","path":"(discovery)"}`.
+    expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
+  });
+
+  it("treats known transport `.code`s as transient", () => {
+    for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
+      const err = Object.assign(new Error("boom"), { code });
+      expect(isTransientNetworkError(err)).toBe(true);
+    }
+  });
+
+  it("walks the .cause chain (fetch failed wrapping a DNS error)", () => {
+    const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api"), {
+      code: "ENOTFOUND",
+    });
+    const err = Object.assign(new TypeError("fetch failed"), { cause });
+    expect(isTransientNetworkError(err)).toBe(true);
+  });
+
+  it("walks AggregateError.errors (happy-eyeballs)", () => {
+    const agg = Object.assign(new AggregateError([
+      Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),
+      Object.assign(new Error("v6"), { code: "EHOSTUNREACH" }),
+    ], "all attempts failed"), {});
+    expect(isTransientNetworkError(agg)).toBe(true);
+  });
+
+  it("is conservative: an arbitrary error is NOT transient", () => {
+    // A bare message like "network down" (no code, not the canonical undici
+    // message) must not be misclassified — only real transport shapes qualify.
+    expect(isTransientNetworkError(new Error("network down"))).toBe(false);
+    expect(isTransientNetworkError(new Error("boom"))).toBe(false);
+    expect(isTransientNetworkError("fetch failed")).toBe(false);
+    expect(isTransientNetworkError(undefined)).toBe(false);
+  });
+
+  it("does not infinite-loop on a self-referential cause", () => {
+    const err = new Error("weird") as Error & { cause?: unknown };
+    err.cause = err;
+    expect(isTransientNetworkError(err)).toBe(false);
+  });
+
+  it("exposes a distinct exit code off the runner's other codes", () => {
+    expect(TRANSIENT_NETWORK_EXIT).toBe(75);
+    expect([0, 1, 2, 17]).not.toContain(TRANSIENT_NETWORK_EXIT);
+  });
+});

package/src/lib/net-errors.ts

@@ -0,0 +1,83 @@
+/**
+ * Transient network-failure classification for the sync runner.
+ *
+ * The auto-sync watcher runs an unattended poll loop. At the top of every pass
+ * it calls `GET /membership/me` to resolve which companies to sync. When the
+ * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
+ * vault API momentarily unreachable) that call fails at the transport layer —
+ * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
+ * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
+ *
+ * This is NOT a crash: the next poll (~30s later) succeeds once the network is
+ * back. But the runner used to `return 1` for it, the watch loop propagated
+ * that non-zero exit, the process exited, and the menubar supervisor reported
+ * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
+ * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
+ * retry instead of surfacing a false crash, while a one-shot `hq sync` still
+ * exits non-zero so a human running it by hand sees the failure.
+ */
+
+/**
+ * Distinct runner exit code for a transient network failure. The watch loop
+ * treats it as "stay alive, retry next tick" (never a crash); a one-shot run
+ * still exits non-zero so the failure is visible to a human/script. Value 75 =
+ * `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
+ * codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
+ */
+export const TRANSIENT_NETWORK_EXIT = 75;
+
+/**
+ * Node/undici error `code`s that mean "the request never completed at the
+ * transport layer" — the machine could not reach the host. All are transient:
+ * a later attempt from the same machine can succeed unchanged.
+ */
+const TRANSIENT_CODES = new Set<string>([
+  "ECONNREFUSED",
+  "ECONNRESET",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "ETIMEDOUT",
+  "EPIPE",
+  "EHOSTUNREACH",
+  "EHOSTDOWN",
+  "ENETUNREACH",
+  "ENETDOWN",
+  "ECONNABORTED",
+  // undici's own timeout/socket failures
+  "UND_ERR_CONNECT_TIMEOUT",
+  "UND_ERR_HEADERS_TIMEOUT",
+  "UND_ERR_BODY_TIMEOUT",
+  "UND_ERR_SOCKET",
+]);
+
+/**
+ * True when `err` is (or wraps) a transient network-transport failure. Walks the
+ * `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
+ * latter), matching either undici's canonical `TypeError: fetch failed` message
+ * or a known transient `.code`. Conservative by design: an arbitrary
+ * `new Error("network down")` with no code and a non-canonical message is NOT
+ * treated as transient, so only real transport-failure shapes qualify.
+ */
+export function isTransientNetworkError(err: unknown, depth = 0): boolean {
+  if (depth > 8 || !(err instanceof Error)) return false;
+
+  // undici's transport failure surfaces as exactly this message.
+  if (err.message === "fetch failed") return true;
+
+  const code = (err as { code?: unknown }).code;
+  if (typeof code === "string" && TRANSIENT_CODES.has(code)) return true;
+
+  // AggregateError (e.g. happy-eyeballs) stashes the per-attempt errors here.
+  const aggregated = (err as { errors?: unknown }).errors;
+  if (Array.isArray(aggregated)) {
+    for (const inner of aggregated) {
+      if (isTransientNetworkError(inner, depth + 1)) return true;
+    }
+  }
+
+  const cause = (err as { cause?: unknown }).cause;
+  if (cause !== undefined && cause !== err) {
+    return isTransientNetworkError(cause, depth + 1);
+  }
+  return false;
+}