@indigoai-us/hq-cloud 6.14.26 → 6.14.28

package/src/personal-vault.ts

@@ -87,7 +87,8 @@ export interface PersonalVaultOptions {
  *      `PERSONAL_VAULT_EXCLUDED_TOP_LEVEL`.
  *   2. Every local (non-cloud) `companies/{slug}/` subdir, after excluding
  *      (a) `_template`, (b) slugs already in `opts.teamSyncedSlugs`
- *      (cloud-backed — they sync to their own team bucket), and (c) any
+ *      (cloud-backed — they sync to their own team bucket), (c) slugs with
+ *      `cloud_uid: cmp_*` in `companies/manifest.yaml`, and (d) any
  *      subdir explicitly marked `cloud: true` in its `company.yaml`.
  *      A company with no `company.yaml` (or one without a `cloud:` marker)
  *      is included — local companies sync to the personal vault by default.
@@ -363,12 +364,19 @@ export function computeSessionLogsSyncPaths(hqRoot: string): string[] {
  *      (currently just `_template`).
  *   3. The slug is NOT in `teamSyncedSlugs` (membership-backed slugs sync
  *      to their own bucket and must not be double-written).
- *   4. `companies/{slug}/company.yaml` does NOT contain an explicit
+ *   4. The slug is NOT listed with a `cloud_uid: cmp_*` in
+ *      `companies/manifest.yaml`. Full-HQ agent boxes often run
+ *      `--personal` alone (no company fanout targets), so
+ *      `teamSyncedSlugs` is empty even though indigo (etc.) is
+ *      cloud-backed; without this check the multi-GB company tree is
+ *      re-mirrored into the agent personal vault every ritual and
+ *      times out the 180s personal push.
+ *   5. `companies/{slug}/company.yaml` does NOT contain an explicit
  *      `cloud: true` line. Local companies sync to the personal vault by
  *      default, so a missing/markerless `company.yaml` is INCLUDED; only an
  *      explicit `cloud: true` marker opts the directory OUT (belt-and-braces
- *      with `teamSyncedSlugs` for a company designated cloud-backed before
- *      its membership has resolved).
+ *      with `teamSyncedSlugs` / manifest `cloud_uid` for a company designated
+ *      cloud-backed before its membership has resolved).
  *
  * Returns absolute paths.
  */
@@ -383,10 +391,16 @@ export function computePersonalCompanySubdirs(
   } catch {
     return [];
   }
+  // Manifest cloud_uid is the durable routing signal for cloud-backed
+  // companies. Prefer it over relying solely on runner membership fanout
+  // (empty for --personal-only agent ritual) or company.yaml (often missing
+  // on full-HQ agent boxes that never ran /designate-team locally).
+  const cloudManifestSlugs = readCloudBackedSlugsFromManifest(hqRoot);
   const eligible: string[] = [];
   for (const slug of slugs) {
     if (PERSONAL_VAULT_COMPANY_EXCLUDED_SLUGS.includes(slug)) continue;
     if (teamSyncedSlugs.has(slug)) continue;
+    if (cloudManifestSlugs.has(slug)) continue;
     const subdir = path.join(companiesRoot, slug);
     let isDir = false;
     try {
@@ -401,6 +415,51 @@ export function computePersonalCompanySubdirs(
   return eligible;
 }
 
+/**
+ * Slugs whose `companies/manifest.yaml` entry carries a `cloud_uid: cmp_*`.
+ * These are cloud-backed team companies and must not personal-vault mirror.
+ *
+ * Lightweight line parse (no YAML dep): fail-open on missing/unreadable
+ * manifest → empty set → companies stay eligible for personal fallback.
+ * Only `cmp_*` uids count (same rule as company-resolver / telemetry).
+ */
+export function readCloudBackedSlugsFromManifest(hqRoot: string): Set<string> {
+  const manifestPath = path.join(hqRoot, "companies", "manifest.yaml");
+  let text: string;
+  try {
+    text = fs.readFileSync(manifestPath, "utf8");
+  } catch {
+    return new Set();
+  }
+  const out = new Set<string>();
+  let currentSlug: string | null = null;
+  let inCompanies = false;
+  for (const line of text.split(/\r?\n/)) {
+    // Top-level key (no indent). Only track entries under `companies:`.
+    if (/^[A-Za-z0-9_.-]+:\s*(?:#.*)?$/.test(line)) {
+      inCompanies = /^companies:\s*(?:#.*)?$/.test(line);
+      currentSlug = null;
+      continue;
+    }
+    if (!inCompanies) continue;
+    // Two-space indent: company slug under companies:
+    const slugMatch = /^ {2}([A-Za-z0-9][A-Za-z0-9_-]*):\s*(?:#.*)?$/.exec(line);
+    if (slugMatch) {
+      currentSlug = slugMatch[1];
+      continue;
+    }
+    if (
+      currentSlug &&
+      /^\s{2,}cloud_uid:\s*cmp_[A-Za-z0-9]+\b/.test(line)
+    ) {
+      out.add(currentSlug);
+      // Keep currentSlug so a later key on the same company is fine; next
+      // sibling slug line will replace it.
+    }
+  }
+  return out;
+}
+
 /**
  * True iff `{subdir}/company.yaml` exists and contains an explicit
  * top-level `cloud: true` line — the marker `/designate-team` writes when a

package/src/watcher.test.ts

@@ -2,11 +2,14 @@ import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
+import { watch } from "chokidar";
 import {
   FakeClock,
   WatchPushDriver,
   TreeWatcher,
+  ChokidarWatchBudget,
   createWatchPathFilter,
+  toChokidarIgnored,
   PushEventEmitter,
 } from "./watcher.js";
 import { StaticFlagProvider } from "./sync/feature-flags.js";
@@ -465,6 +468,87 @@ describe("US-002: createWatchPathFilter — personal-vault exclusions", () => {
   });
 });
 
+describe("US-002: chokidar descent scope", () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "watch-scope-")));
+    // This is the real HQ allowlist shape. The synthetic heavy directories
+    // deliberately sit both under an in-scope company and under the two
+    // never-synced top-level buckets from #156.
+    fs.writeFileSync(path.join(dir, ".hqinclude"), "companies/*/knowledge/\n");
+    fs.mkdirSync(
+      path.join(dir, "companies", "acme", "knowledge", "node_modules", "heavy"),
+      { recursive: true },
+    );
+    fs.writeFileSync(
+      path.join(dir, "companies", "acme", "knowledge", "node_modules", "heavy", "index.js"),
+      "module.exports = 1;\n",
+    );
+    fs.mkdirSync(path.join(dir, "repos", "checkout", "heavy"), { recursive: true });
+    fs.mkdirSync(
+      path.join(dir, "workspace", "worktrees", "repo", "node_modules", "heavy"),
+      { recursive: true },
+    );
+  });
+
+  afterEach(() => {
+    fs.rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("uses the directory verdict, not a file-shaped false positive, for the pre-stat probe", () => {
+    const filter = createWatchPathFilter(dir, false);
+    const nodeModules = path.join(
+      dir,
+      "companies",
+      "acme",
+      "knowledge",
+      "node_modules",
+    );
+    // `node_modules/` is directory-only. Asking the generic emit filter as
+    // though it were a file returns true, which is exactly what the old
+    // `file || directory` probe used to mistake for permission to descend.
+    expect(filter(nodeModules, false)).toBe(true);
+    expect(filter(nodeModules, true)).toBe(false);
+    const ignored = toChokidarIgnored(filter, dir);
+    expect(ignored(nodeModules)).toBe(true);
+    // The allowlist's exact ancestor matcher remains the only carve-out: the
+    // watcher can still reach companies/<slug>/knowledge without opening every
+    // otherwise-file-shaped ignored directory below it.
+    expect(ignored(path.join(dir, "companies"))).toBe(false);
+    expect(ignored(path.join(dir, "companies", "acme"))).toBe(false);
+  });
+
+  it("prunes ignored directories during chokidar's pre-stat descent probe", async () => {
+    const filter = createWatchPathFilter(dir, false);
+    const chokidarWatcher = watch(dir, {
+      ignored: toChokidarIgnored(filter, dir),
+      ignoreInitial: true,
+      persistent: false,
+    });
+    try {
+      await new Promise<void>((resolve, reject) => {
+        chokidarWatcher.once("ready", resolve);
+        chokidarWatcher.once("error", reject);
+      });
+      const descended = Object.keys(chokidarWatcher.getWatched())
+        .map((watchedPath) => path.resolve(watchedPath))
+        .sort();
+
+      expect(descended).toContain(
+        path.join(dir, "companies", "acme", "knowledge"),
+      );
+      expect(descended).not.toContain(
+        path.join(dir, "companies", "acme", "knowledge", "node_modules"),
+      );
+      expect(descended).not.toContain(path.join(dir, "repos"));
+      expect(descended).not.toContain(path.join(dir, "workspace", "worktrees"));
+    } finally {
+      await chokidarWatcher.close();
+    }
+  });
+});
+
 describe("US-002: TreeWatcher — debounce coalesce (FakeClock seam)", () => {
   function makeWatcher(opts?: { debounceMs?: number; personalMode?: boolean }) {
     const clock = new FakeClock();
@@ -555,6 +639,170 @@ describe("US-002: TreeWatcher — debounce coalesce (FakeClock seam)", () => {
     expect(source).not.toContain("startPollingTreeWatch");
     expect(source).not.toContain("snapshotWatchTree");
   });
+
+  it("stops descending at the hard watch budget, reports, and degrades to polling", () => {
+    const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "watch-cap-")));
+    fs.mkdirSync(path.join(dir, "personal", "a", "deep"), { recursive: true });
+    fs.mkdirSync(path.join(dir, "personal", "b", "deep"), { recursive: true });
+    const clock = new FakeClock();
+    const degraded = vi.fn();
+    const changed = vi.fn();
+    const descended: string[] = [];
+    let backendClosed = false;
+    try {
+      const watcher = new TreeWatcher({
+        hqRoot: dir,
+        clock,
+        debounceMs: DEBOUNCE,
+        maxWatchedPaths: 3,
+        onDegraded: degraded,
+        // This faked backend drives chokidar's real ignored predicate through
+        // its pre-stat and post-stat calls, without needing an inotify slot on
+        // a host already exhausted by another process.
+        backendFactory: ({
+          hqRoot,
+          shouldEmit,
+          maxWatchedPaths,
+          onWatchBudgetExceeded,
+        }) => {
+          const budget = new ChokidarWatchBudget(
+            maxWatchedPaths,
+            onWatchBudgetExceeded,
+          );
+          const ignored = toChokidarIgnored(shouldEmit, hqRoot, budget);
+          const visit = (candidate: string): void => {
+            if (ignored(candidate)) return;
+            const stats = fs.lstatSync(candidate);
+            if (ignored(candidate, stats)) return;
+            if (!stats.isDirectory()) return;
+            descended.push(path.resolve(candidate));
+            for (const entry of fs.readdirSync(candidate)) {
+              visit(path.join(candidate, entry));
+            }
+          };
+          visit(hqRoot);
+          return {
+            close: () => {
+              backendClosed = true;
+            },
+            needsKnownKinds: false,
+            watchedPathCount: () => descended.length,
+          };
+        },
+      });
+      watcher.onChange(changed);
+      watcher.start();
+
+      expect(descended).toHaveLength(3);
+      expect(descended).not.toContain(path.join(dir, "personal", "a", "deep"));
+      expect(degraded).toHaveBeenCalledWith(
+        expect.objectContaining({
+          reason: "watch_budget_exhausted",
+          maxWatchedPaths: 3,
+          watchedPaths: 3,
+          offendingPath: expect.stringContaining(path.join("personal", "a", "deep")),
+        }),
+      );
+      expect(backendClosed).toBe(true);
+      expect(watcher.isWatching()).toBe(false);
+
+      clock.advance(DEBOUNCE);
+      expect(changed).toHaveBeenCalledWith(
+        undefined,
+        expect.objectContaining({ overflowed: true }),
+      );
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("degrades instead of dying when chokidar emits runtime ENOSPC", () => {
+    const clock = new FakeClock();
+    const degraded = vi.fn();
+    const changed = vi.fn();
+    const telemetry: TelemetryEventsBatch[] = [];
+    const telemetryClient = {
+      postTelemetryEvents: vi.fn(async (batch: TelemetryEventsBatch) => {
+        telemetry.push(batch);
+        return { ok: true, written: batch.events.length, skipped: [] };
+      }),
+    };
+    let closeCalls = 0;
+    const watcher = new TreeWatcher({
+      hqRoot: ROOT,
+      clock,
+      debounceMs: DEBOUNCE,
+      onDegraded: degraded,
+      telemetryClient,
+      backendFactory: ({ onError }) => {
+        const backend = {
+          close: () => {
+            closeCalls += 1;
+          },
+          needsKnownKinds: false,
+          watchedPathCount: () => 0,
+        };
+        queueMicrotask(() =>
+          onError(
+            Object.assign(new Error("inotify limit"), {
+              code: "ENOSPC",
+              path: path.join(ROOT, "companies", "acme", "knowledge"),
+            }),
+          ),
+        );
+        return backend;
+      },
+    });
+    watcher.onChange(changed);
+    watcher.start();
+
+    return new Promise<void>((resolve) => queueMicrotask(resolve)).then(() => {
+      expect(closeCalls).toBe(1);
+      expect(watcher.isWatching()).toBe(false);
+      expect(degraded).toHaveBeenCalledWith(
+        expect.objectContaining({
+          reason: "inotify_enospc",
+          errorCode: "ENOSPC",
+          offendingPath: path.join(ROOT, "companies", "acme", "knowledge"),
+        }),
+      );
+      expect(telemetry).toEqual([
+        expect.objectContaining({
+          events: [
+            expect.objectContaining({
+              eventName: "watcher_degraded",
+              source: "watcher",
+              properties: expect.objectContaining({
+                reason: "inotify_enospc",
+                offendingTopLevel: "companies",
+              }),
+            }),
+          ],
+        }),
+      ]);
+      clock.advance(DEBOUNCE);
+      expect(changed).toHaveBeenCalledWith(
+        undefined,
+        expect.objectContaining({ overflowed: true }),
+      );
+    });
+  });
+
+  it("does not retain one rename-kind entry for every changed file", () => {
+    const clock = new FakeClock();
+    const watcher = new TreeWatcher({
+      hqRoot: ROOT,
+      clock,
+      pathFilter: () => true,
+      maxPendingPaths: 20_000,
+      maxPendingBytes: 20_000_000,
+    });
+    for (let i = 0; i < 10_000; i++) {
+      watcher.handleEvent(path.join(ROOT, "personal", `file-${i}.md`));
+    }
+    expect(watcher.knownDirectoryCount()).toBe(0);
+    watcher.stop();
+  });
 });
 
 describe("US-002: TreeWatcher — lifecycle (real chokidar over a temp dir)", () => {