@indigoai-us/hq-cloud 6.13.3 → 6.13.5

package/src/bin/sync-runner.ts

@@ -104,9 +104,16 @@ import { HQ_CLOUD_VERSION } from "../version.js";
 import { collectAndSendTelemetry } from "../telemetry.js";
 import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
 import { reindexAfterSync } from "../qmd-reindex.js";
+import type { ReindexOptions as QmdReindexOptions } from "../qmd-reindex.js";
+import { reindex as defaultReindex } from "../cli/reindex.js";
+import type { ReindexOptions, ReindexResult } from "../cli/reindex.js";
 import { pruneConflictIndex } from "../lib/conflict-index.js";
 import { materializeCodexAgents } from "../agent-codex-instructions.js";
 import { getOrCreateMachineId } from "../lib/machine-id.js";
+import {
+  isTransientNetworkError,
+  TRANSIENT_NETWORK_EXIT,
+} from "../lib/net-errors.js";
 import { describeError } from "../lib/describe-error.js";
 import type { Clock, TreeChangeBatch } from "../watcher.js";
 import type { PushReceiver, SyncEngineFn } from "../sync/push-receiver.js";
@@ -577,6 +584,13 @@ export interface RunnerDeps {
   createVaultClient?: (config: VaultServiceConfig) => VaultClientSurface;
   /** Sync function. Defaults to `cli/sync.sync`. */
   sync?: (options: SyncOptions) => Promise<SyncResult>;
+  /** Native post-fanout reindex function. Defaults to `cli/reindex.reindex`. */
+  reindex?: (options: ReindexOptions) => ReindexResult;
+  /** QMD post-sync reindex function. Defaults to `qmd-reindex.reindexAfterSync`. */
+  qmdReindex?: (
+    hqRoot: string,
+    options: QmdReindexOptions,
+  ) => ReturnType<typeof reindexAfterSync>;
   /** Internal: set when runRunner is invoked under the per-root operation lock. */
   operationLockAlreadyHeld?: boolean;
   /** Share function (push phase). Defaults to `cli/share.share`. */
@@ -622,6 +636,11 @@ function defaultGetIdTokenClaims(): IdTokenClaims | null {
   return decodeJwtClaims(tokens.idToken);
 }
 
+function isKnowledgeMarkdownPath(relPath: string): boolean {
+  const normalized = relPath.split(path.sep).join("/");
+  return /^companies\/[^/]+\/knowledge\/.+\.md$/i.test(normalized);
+}
+
 /**
  * Best-effort: claim any email-keyed pending invites that were sent before
  * this user had a person entity. Mirrors the installer's vault-handoff flow.
@@ -1145,15 +1164,19 @@ export async function runRunner(
       });
       return 0;
     }
-    // Any other failure is unrecoverable — surface as an error event and
-    // exit non-zero so the spawner knows the runner didn't get far enough
-    // to emit a useful protocol stream.
+    // Surface the failure as an error event for observability either way.
     emit({
       type: "error",
       message: err instanceof Error ? err.message : String(err),
       path: "(discovery)",
     });
-    return 1;
+    // A transient network failure (offline, DNS blip, vault API briefly
+    // unreachable) is NOT a crash — it self-heals on the next poll. Return the
+    // retryable exit code so the watch loop stays alive instead of exiting and
+    // being reported as "watcher exited unexpectedly (code=Some(1))" for every
+    // blip (HQ-SYNC-1W). A one-shot run still exits non-zero. Any other failure
+    // is treated as a hard error (exit 1), as before.
+    return isTransientNetworkError(err) ? TRANSIENT_NETWORK_EXIT : 1;
   }
 
   const targetPlan = await buildFanoutPlan({
@@ -1200,6 +1223,23 @@ export async function runRunner(
   const { errors, allConflicts } = fanout;
   const rollup = rollupAllComplete(plan, fanout.stateByCompany);
 
+  if (rollup.needsReindex) {
+    try {
+      (deps.reindex ?? defaultReindex)({
+        repoRoot: parsed.hqRoot,
+        ...(deps.operationLockAlreadyHeld === true ? { skipLock: true } : {}),
+      });
+    } catch (err) {
+      reportDiagnostic({
+        component: "native-reindex",
+        event: "runner.native_reindex.failed",
+        message: "post-fanout native reindex failed",
+        err,
+        context: { hqRoot: parsed.hqRoot },
+      });
+    }
+  }
+
   // Fire telemetry collector before the all-complete emit so the cursor at
   // `~/.hq/telemetry-cursor.json` is consistent with what the menubar sees.
   await awaitTelemetryWithTimeout(
@@ -1250,12 +1290,12 @@ export async function runRunner(
   // pulled in (nothing to reindex otherwise) and not explicitly disabled.
   // Self-contained: shells out to the global `qmd` binary, no dependency on
   // any (possibly stale) script inside the synced HQ tree. See qmd-reindex.ts.
-  if (
-    rollup.totalDownloaded > 0 &&
-    process.env.HQ_QMD_REINDEX_ON_SYNC !== "0"
-  ) {
+  const qmdChangedPaths = rollup.changedPaths.filter(isKnowledgeMarkdownPath);
+  if (qmdChangedPaths.length > 0 && process.env.HQ_QMD_REINDEX_ON_SYNC !== "0") {
     try {
-      reindexAfterSync(parsed.hqRoot, {
+      (deps.qmdReindex ?? reindexAfterSync)(parsed.hqRoot, {
+        changedPaths: qmdChangedPaths,
+        forceCollectionRefresh: true,
         log: ({ event, message, err, context }) =>
           reportDiagnostic({
             component: "qmd-reindex",

package/src/cli/share.test.ts

@@ -1264,6 +1264,52 @@ describe("share", () => {
     expect(events.some((e) => e.type === "error")).toBe(false);
   });
 
+  it("a file over the size limit is a benign skip-size-limit event, NEVER an error (HQ-SYNC-4 exit-2 flood)", async () => {
+    // Regression for the fleet-wide "auto-sync watcher exited unexpectedly
+    // (code=Some(2))" flood (HQ-SYNC-4, ~93k events across 60+ machines). An
+    // oversized file is a permanent, benign skip — it exceeds the 50 MB cap on
+    // EVERY pass, forever. Emitting it as `type: "error"` pushed it into the
+    // runner's `errors[]`, so the pass returned exit 2, which the menubar
+    // supervisor reported as an unexpected watcher crash on every single tick.
+    // A too-big file must surface for visibility but must NOT count as an error.
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    const bigFile = path.join(companyRoot, "huge.mp4");
+    // Sparse 51 MB file: statSync (isWithinSizeLimit) reports 51 MB while the
+    // file occupies ~0 bytes on disk, so the test stays fast.
+    const fd = fs.openSync(bigFile, "w");
+    fs.ftruncateSync(fd, 51 * 1024 * 1024);
+    fs.closeSync(fd);
+
+    const events: Array<{ type?: string; path?: string; bytes?: number }> = [];
+    const result = await share({
+      paths: [bigFile],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => events.push(e as { type?: string }),
+    });
+
+    // Not uploaded, counted as a skip, company did not abort.
+    expect(result.aborted).toBe(false);
+    expect(result.filesUploaded).toBe(0);
+    expect(result.filesSkipped).toBe(1);
+    expect(uploadFile).not.toHaveBeenCalled();
+
+    // The oversized file surfaces as a benign skip-size-limit event for
+    // visibility (path + byte size)…
+    const skipEv = events.find((e) => e.type === "skip-size-limit") as
+      | { path: string; bytes: number }
+      | undefined;
+    expect(skipEv).toBeDefined();
+    expect(skipEv!.path).toBe("huge.mp4");
+    expect(skipEv!.bytes).toBe(51 * 1024 * 1024);
+
+    // …and CRUCIALLY never as an error — an error event is what flipped the
+    // pass to exit 2 and drove the watcher-crash flood.
+    expect(events.some((e) => e.type === "error")).toBe(false);
+  });
+
   it("uploads (no conflict) when only the local side changed since last sync", async () => {
     // Regression for hq-cloud#<conflict-detection>: a local edit to a file
     // that exists on S3 used to trigger a push conflict because the

package/src/cli/share.ts

@@ -1135,10 +1135,22 @@ async function executeUploads(
   const uploadItems: UploadPlanItem[] = [];
   for (const item of pushPlan.items) {
     if (item.action === "skip-size-limit") {
+      // A file over the size cap is a permanent, benign skip — NOT an error.
+      // Emitting it as `type: "error"` pushed it into the runner's `errors[]`,
+      // so every watch pass returned exit 2 and the menubar reported an
+      // "auto-sync watcher exited unexpectedly (code=Some(2))" crash on every
+      // tick (the fleet-wide HQ-SYNC-4 flood). Surface it for visibility, but
+      // do not let it flip the pass to a non-zero exit.
+      let bytes = 0;
+      try {
+        bytes = fs.statSync(item.absolutePath).size;
+      } catch {
+        // Best-effort size for display only; a stat race must not break the sync.
+      }
       run.emit({
-        type: "error",
+        type: "skip-size-limit",
         path: item.relativePath,
-        message: "file exceeds size limit",
+        bytes,
       });
       counters.filesSkipped++;
       continue;

package/src/cli/sync.ts

@@ -322,6 +322,24 @@ export type SyncProgressEvent =
       type: "upload-suppressed-tombstone";
       path: string;
       deletedAt: string;
+    }
+  | {
+      /**
+       * Emitted by the PUSH leg (`share()`) once per local file skipped because
+       * it exceeds the max sync size (`isWithinSizeLimit`, 50 MB default). This
+       * is a permanent, benign, user-policy outcome — the file is too big to
+       * sync and will be over the cap on every pass — so it is NOT an error and
+       * the run exits 0. It was previously emitted as `type: "error"`, which
+       * pushed it into the runner's `errors[]` and made every watch pass return
+       * exit 2; the menubar supervisor then reported that as
+       * "auto-sync watcher exited unexpectedly (code=Some(2))" on every tick —
+       * the fleet-wide HQ-SYNC-4 flood. Surfaced here purely for visibility
+       * (`bytes` is the offending file's size) so a UI/CLI can tell the user
+       * which file was too big without treating it as a failure.
+       */
+      type: "skip-size-limit";
+      path: string;
+      bytes: number;
     };
 
 export interface SyncOptions {
@@ -503,6 +521,11 @@ export interface SyncResult {
    * true they are left on disk and tombstoned, and counted here.
    */
   scopeOrphansBlocked: number;
+  /**
+   * Remote keys whose local on-disk counterpart changed during this pull.
+   * Includes downloads, tombstone deletes, and clean scope-orphan removals.
+   */
+  changedPaths?: string[];
 }
 
 type SyncEventEmitter = (event: SyncProgressEvent) => void;
@@ -535,6 +558,7 @@ interface PullCounters {
   filesTombstoned: number;
   filesOutOfScope: number;
   conflictPaths: string[];
+  changedPaths: string[];
 }
 
 interface ScopeShrinkPlanState {
@@ -549,6 +573,7 @@ interface ScopeShrinkRun {
   shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
   shrinkResult: ReturnType<typeof applyScopeShrink>;
   scopeOrphansRemoved: number;
+  changedPaths: string[];
 }
 
 /**
@@ -831,6 +856,7 @@ function createPullCounters(): PullCounters {
     filesTombstoned: 0,
     filesOutOfScope: 0,
     conflictPaths: [],
+    changedPaths: [],
   };
 }
 
@@ -943,6 +969,10 @@ function executeScopeShrink(
     shrinkResult,
     scopeOrphansRemoved:
       shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
+    changedPaths: [
+      ...shrinkResult.quarantinedPaths,
+      ...shrinkResult.removedPaths,
+    ],
   };
 }
 
@@ -1039,6 +1069,7 @@ function executeFileTombstoneDelete(
   }
   removeEntry(run.journal, tombstoneKey);
   counters.filesTombstoned++;
+  counters.changedPaths.push(tombstoneKey);
   run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
 }
 
@@ -1198,6 +1229,7 @@ async function executeConflictItem(
       filesOutOfScope: counters.filesOutOfScope,
       scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
       scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
+      changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
     };
   }
 
@@ -1335,6 +1367,7 @@ async function downloadOne(
 
     counters.filesDownloaded++;
     counters.bytesDownloaded += size;
+    counters.changedPaths.push(remoteFile.key);
   } catch (err) {
     if (isAccessDenied(err)) {
       counters.filesSkipped++;
@@ -1474,6 +1507,7 @@ function executeJournalTombstoneDeletes(
     }
     removeEntry(run.journal, key);
     counters.filesTombstoned++;
+    counters.changedPaths.push(key);
     run.emit({
       type: "progress",
       path: key,
@@ -1532,6 +1566,7 @@ function finalizePullRun(
     filesOutOfScope: counters.filesOutOfScope,
     scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
     scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
+    changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
   };
 }
 
@@ -2462,5 +2497,10 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
     if (event.count > event.samplePaths.length) {
       console.warn(`    ... and ${event.count - event.samplePaths.length} more`);
     }
+  } else if (event.type === "skip-size-limit") {
+    const mb = (event.bytes / (1024 * 1024)).toFixed(1);
+    console.warn(
+      `  ! ${event.path} skipped — ${mb} MB exceeds the 50 MB sync size limit`,
+    );
   }
 }

package/src/lib/net-errors.test.ts

@@ -0,0 +1,53 @@
+import { describe, it, expect } from "vitest";
+import { isTransientNetworkError, TRANSIENT_NETWORK_EXIT } from "./net-errors.js";
+
+describe("isTransientNetworkError", () => {
+  it("treats undici's canonical `fetch failed` TypeError as transient", () => {
+    // This is the exact shape the runner's discovery step sees in production
+    // (HQ-SYNC-1W): `{"type":"error","message":"fetch failed","path":"(discovery)"}`.
+    expect(isTransientNetworkError(new TypeError("fetch failed"))).toBe(true);
+  });
+
+  it("treats known transport `.code`s as transient", () => {
+    for (const code of ["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "ETIMEDOUT"]) {
+      const err = Object.assign(new Error("boom"), { code });
+      expect(isTransientNetworkError(err)).toBe(true);
+    }
+  });
+
+  it("walks the .cause chain (fetch failed wrapping a DNS error)", () => {
+    const cause = Object.assign(new Error("getaddrinfo ENOTFOUND api"), {
+      code: "ENOTFOUND",
+    });
+    const err = Object.assign(new TypeError("fetch failed"), { cause });
+    expect(isTransientNetworkError(err)).toBe(true);
+  });
+
+  it("walks AggregateError.errors (happy-eyeballs)", () => {
+    const agg = Object.assign(new AggregateError([
+      Object.assign(new Error("v4"), { code: "ECONNREFUSED" }),
+      Object.assign(new Error("v6"), { code: "EHOSTUNREACH" }),
+    ], "all attempts failed"), {});
+    expect(isTransientNetworkError(agg)).toBe(true);
+  });
+
+  it("is conservative: an arbitrary error is NOT transient", () => {
+    // A bare message like "network down" (no code, not the canonical undici
+    // message) must not be misclassified — only real transport shapes qualify.
+    expect(isTransientNetworkError(new Error("network down"))).toBe(false);
+    expect(isTransientNetworkError(new Error("boom"))).toBe(false);
+    expect(isTransientNetworkError("fetch failed")).toBe(false);
+    expect(isTransientNetworkError(undefined)).toBe(false);
+  });
+
+  it("does not infinite-loop on a self-referential cause", () => {
+    const err = new Error("weird") as Error & { cause?: unknown };
+    err.cause = err;
+    expect(isTransientNetworkError(err)).toBe(false);
+  });
+
+  it("exposes a distinct exit code off the runner's other codes", () => {
+    expect(TRANSIENT_NETWORK_EXIT).toBe(75);
+    expect([0, 1, 2, 17]).not.toContain(TRANSIENT_NETWORK_EXIT);
+  });
+});

package/src/lib/net-errors.ts

@@ -0,0 +1,83 @@
+/**
+ * Transient network-failure classification for the sync runner.
+ *
+ * The auto-sync watcher runs an unattended poll loop. At the top of every pass
+ * it calls `GET /membership/me` to resolve which companies to sync. When the
+ * machine is briefly offline (wifi drop, waking from sleep, a DNS blip, the
+ * vault API momentarily unreachable) that call fails at the transport layer —
+ * Node's `fetch` throws `TypeError: fetch failed` with the real cause (an
+ * `ECONNREFUSED` / `ENOTFOUND` / `ETIMEDOUT` / … Error) on `.cause`.
+ *
+ * This is NOT a crash: the next poll (~30s later) succeeds once the network is
+ * back. But the runner used to `return 1` for it, the watch loop propagated
+ * that non-zero exit, the process exited, and the menubar supervisor reported
+ * "auto-sync watcher exited unexpectedly (code=Some(1))" for every blip — the
+ * HQ-SYNC-1W cluster. We classify these so the watch loop can stay alive and
+ * retry instead of surfacing a false crash, while a one-shot `hq sync` still
+ * exits non-zero so a human running it by hand sees the failure.
+ */
+
+/**
+ * Distinct runner exit code for a transient network failure. The watch loop
+ * treats it as "stay alive, retry next tick" (never a crash); a one-shot run
+ * still exits non-zero so the failure is visible to a human/script. Value 75 =
+ * `EX_TEMPFAIL` from sysexits.h ("temporary failure; retry"). Kept off the
+ * codes already in use by the runner (0 ok, 1 error, 2 partial, 17 op-locked).
+ */
+export const TRANSIENT_NETWORK_EXIT = 75;
+
+/**
+ * Node/undici error `code`s that mean "the request never completed at the
+ * transport layer" — the machine could not reach the host. All are transient:
+ * a later attempt from the same machine can succeed unchanged.
+ */
+const TRANSIENT_CODES = new Set<string>([
+  "ECONNREFUSED",
+  "ECONNRESET",
+  "ENOTFOUND",
+  "EAI_AGAIN",
+  "ETIMEDOUT",
+  "EPIPE",
+  "EHOSTUNREACH",
+  "EHOSTDOWN",
+  "ENETUNREACH",
+  "ENETDOWN",
+  "ECONNABORTED",
+  // undici's own timeout/socket failures
+  "UND_ERR_CONNECT_TIMEOUT",
+  "UND_ERR_HEADERS_TIMEOUT",
+  "UND_ERR_BODY_TIMEOUT",
+  "UND_ERR_SOCKET",
+]);
+
+/**
+ * True when `err` is (or wraps) a transient network-transport failure. Walks the
+ * `.cause` chain and `AggregateError.errors` (undici's happy-eyeballs raises the
+ * latter), matching either undici's canonical `TypeError: fetch failed` message
+ * or a known transient `.code`. Conservative by design: an arbitrary
+ * `new Error("network down")` with no code and a non-canonical message is NOT
+ * treated as transient, so only real transport-failure shapes qualify.
+ */
+export function isTransientNetworkError(err: unknown, depth = 0): boolean {
+  if (depth > 8 || !(err instanceof Error)) return false;
+
+  // undici's transport failure surfaces as exactly this message.
+  if (err.message === "fetch failed") return true;
+
+  const code = (err as { code?: unknown }).code;
+  if (typeof code === "string" && TRANSIENT_CODES.has(code)) return true;
+
+  // AggregateError (e.g. happy-eyeballs) stashes the per-attempt errors here.
+  const aggregated = (err as { errors?: unknown }).errors;
+  if (Array.isArray(aggregated)) {
+    for (const inner of aggregated) {
+      if (isTransientNetworkError(inner, depth + 1)) return true;
+    }
+  }
+
+  const cause = (err as { cause?: unknown }).cause;
+  if (cause !== undefined && cause !== err) {
+    return isTransientNetworkError(cause, depth + 1);
+  }
+  return false;
+}

package/src/qmd-reindex.test.ts

@@ -100,6 +100,56 @@ describe("reindexAfterSync", () => {
     expect(calls).toEqual([["collection", "list"], ["update"]]);
   });
 
+  it("skips qmd entirely when provided changed paths do not touch knowledge markdown", () => {
+    const { exec, calls } = fakeExec({ listStdout: "" });
+    const r = reindexAfterSync("/hq", {
+      exec,
+      existsSync: () => true,
+      changedPaths: ["companies/acme/docs/readme.txt"],
+    });
+    expect(r.qmdAvailable).toBe(false);
+    expect(r.updated).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it("debounces dirty updates with a pending marker and flushes the final update later", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-qmd-debounce-"));
+    try {
+      const statePath = path.join(dir, "qmd-state.json");
+      const { exec, calls } = fakeExec({ listStdout: "acme (qmd://acme/)\n" });
+
+      const first = reindexAfterSync("/hq", {
+        exec,
+        existsSync: () => true,
+        changedPaths: ["companies/acme/knowledge/note.md"],
+        debounceMs: 60_000,
+        nowMs: 100_000,
+        statePath,
+      });
+      expect(first.updated).toBe(false);
+      expect(first.pendingDirty).toBe(true);
+      expect(calls).toEqual([["collection", "list"]]);
+
+      const second = reindexAfterSync("/hq", {
+        exec,
+        existsSync: () => true,
+        changedPaths: [],
+        debounceMs: 60_000,
+        nowMs: 161_000,
+        statePath,
+      });
+      expect(second.updated).toBe(true);
+      expect(second.pendingDirty).toBe(false);
+      expect(calls).toEqual([
+        ["collection", "list"],
+        ["collection", "list"],
+        ["update"],
+      ]);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
   it("runs embed only when embed:true", () => {
     const { exec, calls } = fakeExec({ listStdout: "" });
     const r = reindexAfterSync("/hq", {