@indigoai-us/hq-cloud 6.12.0 → 6.12.2

package/src/cli/share.ts

@@ -35,7 +35,7 @@ import {
   PERSONAL_VAULT_JOURNAL_SLUG,
   migratePersonalVaultJournal,
 } from "../journal.js";
-import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
+import { createIgnoreFilter, isExpectedIgnore, isWithinSizeLimit } from "../ignore.js";
 import {
   wrapFilterWithPersonalVaultDefaults,
   type PersonalVaultExclusion,
@@ -261,6 +261,7 @@ export function isMalformedVaultKey(key: string): boolean {
 export const _testing = {
   isEphemeralPath,
   EPHEMERAL_PATH_PATTERN,
+  wrapFilterWithIgnoreVisibility,
 };
 
 /**
@@ -734,6 +735,13 @@ export interface ShareResult {
    * its in-scope subset.
    */
   filesExcludedByScope: number;
+  /**
+   * Number of distinct hq-root-relative paths skipped by the base ignore
+   * filter that look like real content rather than expected build/VCS/cache
+   * noise. Mirrors the `count` field of the `ignore-excluded` event (emitted
+   * once if this is > 0).
+   */
+  filesExcludedByIgnore: number;
   /**
    * Paths (company-relative) that were detected as push conflicts. Mirrors
    * `SyncResult.conflictPaths` so push and pull surface conflicts the same
@@ -792,6 +800,31 @@ function wrapFilterWithScope(
   };
 }
 
+/**
+ * Wrap the base ignore filter so push can observe paths it rejects before
+ * personal-vault policy or ACL scope wrappers add their own exclusions.
+ * Every base-ignore rejection increments `onAnyExcluded`; only rejections
+ * that do NOT match expected build/VCS/cache noise are tagged through
+ * `onIgnoreExcluded` for the one-shot `ignore-excluded` summary.
+ */
+export function wrapFilterWithIgnoreVisibility(
+  underlying: (absPath: string, isDir?: boolean) => boolean,
+  hqRoot: string,
+  onIgnoreExcluded: (relPath: string) => void,
+  onAnyExcluded?: () => void,
+): (absPath: string, isDir?: boolean) => boolean {
+  return (absPath: string, isDir?: boolean) => {
+    const allowed = underlying(absPath, isDir);
+    if (allowed) return true;
+
+    onAnyExcluded?.();
+    const rel = path.relative(hqRoot, absPath).split(path.sep).join("/");
+    if (rel === "" || rel.startsWith("..")) return false;
+    if (!isExpectedIgnore(rel)) onIgnoreExcluded(rel);
+    return false;
+  };
+}
+
 /**
  * Share local file(s) to the entity vault.
  */
@@ -857,6 +890,8 @@ interface PushRunContext {
   excludedSet: Set<string>;
   excludedById: Record<string, number>;
   scopeExcludedSet: Set<string>;
+  ignoreExcludedSet: Set<string>;
+  ignoreExcludedTotal: { value: number };
 }
 
 interface ShareCounters {
@@ -939,6 +974,18 @@ async function createPushRunContext(options: ShareOptions): Promise<PushRunConte
     : path.join(hqRoot, "companies", ctx.slug);
 
   const ignoreFilter = createIgnoreFilter(hqRoot);
+  const ignoreExcludedSet = new Set<string>();
+  const ignoreExcludedTotal = { value: 0 };
+  const recordedIgnoreFilter = wrapFilterWithIgnoreVisibility(
+    ignoreFilter,
+    hqRoot,
+    (rel) => {
+      ignoreExcludedSet.add(rel);
+    },
+    () => {
+      ignoreExcludedTotal.value++;
+    },
+  );
   const excludedSet = new Set<string>();
   const excludedById: Record<string, number> = {};
   const onExcluded = (rel: string, match: PersonalVaultExclusion) => {
@@ -951,8 +998,8 @@ async function createPushRunContext(options: ShareOptions): Promise<PushRunConte
     scopeExcludedSet.add(rel);
   };
   const baseFilter = options.personalMode === true
-    ? wrapFilterWithPersonalVaultDefaults(ignoreFilter, syncRoot, onExcluded)
-    : ignoreFilter;
+    ? wrapFilterWithPersonalVaultDefaults(recordedIgnoreFilter, syncRoot, onExcluded)
+    : recordedIgnoreFilter;
   const shouldSync = options.prefixSet !== undefined
     ? wrapFilterWithScope(baseFilter, syncRoot, options.prefixSet, onScopeExcluded)
     : baseFilter;
@@ -981,6 +1028,8 @@ async function createPushRunContext(options: ShareOptions): Promise<PushRunConte
     excludedSet,
     excludedById,
     scopeExcludedSet,
+    ignoreExcludedSet,
+    ignoreExcludedTotal,
   };
 }
 
@@ -1494,6 +1543,20 @@ function finalizeShareJournal(run: PushRunContext): void {
       samplePaths,
     });
   }
+
+  if (run.ignoreExcludedSet.size > 0) {
+    const samplePaths: string[] = [];
+    for (const p of run.ignoreExcludedSet) {
+      samplePaths.push(p);
+      if (samplePaths.length >= 10) break;
+    }
+    run.emit({
+      type: "ignore-excluded",
+      count: run.ignoreExcludedSet.size,
+      totalExcluded: run.ignoreExcludedTotal.value,
+      samplePaths,
+    });
+  }
 }
 
 function throwUploadWorkerErrors(workerErrors: Error[]): void {
@@ -1528,6 +1591,7 @@ function buildShareResult(
     filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
     filesExcludedByPolicy: run.excludedSet.size,
     filesExcludedByScope: run.scopeExcludedSet.size,
+    filesExcludedByIgnore: run.ignoreExcludedSet.size,
     conflictPaths,
     aborted,
   };
@@ -1589,6 +1653,20 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
       `    Sample refused paths: ${event.samplePaths.slice(0, 5).join(", ")}${event.samplePaths.length > 5 ? ", …" : ""}\n` +
       `    To proceed anyway: re-run with HQ_SYNC_DELETE_BULK_OVERRIDE=1 (or propagateDeletePolicy:"all").`,
     );
+  } else if (event.type === "ignore-excluded") {
+    // The "not silent" surface: an ignore rule dropped content that doesn't
+    // look like expected build/VCS/cache noise, so it never reached the vault.
+    // Surface it as a WARN naming the paths so an over-broad exclusion is
+    // observable rather than invisible (DEV-1791 / feedback_b9dfc7ae).
+    console.warn(
+      `  ! ${event.count} path${event.count === 1 ? "" : "s"} were EXCLUDED from sync by an ignore rule and did NOT reach the vault (review in case this is unintended):`,
+    );
+    for (const p of event.samplePaths) {
+      console.warn(`    · ${p}`);
+    }
+    if (event.count > event.samplePaths.length) {
+      console.warn(`    ... and ${event.count - event.samplePaths.length} more`);
+    }
   }
 }
 

package/src/cli/sync.ts

@@ -287,6 +287,22 @@ export type SyncProgressEvent =
       count: number;
       samplePaths: string[];
     }
+  | {
+      /**
+       * Emitted at most ONCE per `share()` push leg when the base ignore
+       * filter dropped one or more NOTEWORTHY paths: content that does not
+       * match expected build/VCS/cache noise classes. Surfaces an over-broad
+       * exclusion that would otherwise drop content silently (DEV-1791).
+       *
+       * `count` is the number of distinct noteworthy paths; `totalExcluded`
+       * is all base-ignore rejections including expected noise; `samplePaths`
+       * carries up to 10 noteworthy paths. Not emitted when `count === 0`.
+       */
+      type: "ignore-excluded";
+      count: number;
+      totalExcluded: number;
+      samplePaths: string[];
+    }
   | {
       /**
        * Emitted by the PUSH leg (`share()`) once per key whose upload was
@@ -2268,5 +2284,15 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
     if (event.count > event.samplePaths.length) {
       console.log(`    ... and ${event.count - event.samplePaths.length} more`);
     }
+  } else if (event.type === "ignore-excluded") {
+    console.warn(
+      `  ! ${event.count} path${event.count === 1 ? "" : "s"} were EXCLUDED from sync by an ignore rule and did NOT reach the vault (review in case this is unintended):`,
+    );
+    for (const p of event.samplePaths) {
+      console.warn(`    · ${p}`);
+    }
+    if (event.count > event.samplePaths.length) {
+      console.warn(`    ... and ${event.count - event.samplePaths.length} more`);
+    }
   }
 }

package/src/ignore.test.ts

@@ -11,7 +11,31 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
-import { createIgnoreFilter } from "./ignore.js";
+import { createIgnoreFilter, isExpectedIgnore } from "./ignore.js";
+import { wrapFilterWithIgnoreVisibility } from "./cli/share.js";
+
+describe("isExpectedIgnore", () => {
+  it("classifies top-level HQ-managed roots as expected", () => {
+    expect(isExpectedIgnore("repos")).toBe(true);
+    expect(isExpectedIgnore("repos/private/x.ts")).toBe(true);
+    expect(isExpectedIgnore("workspace/tmp/file.md")).toBe(true);
+  });
+
+  it("classifies build, VCS, cache, and known basename noise as expected", () => {
+    expect(isExpectedIgnore("node_modules/react/index.js")).toBe(true);
+    expect(isExpectedIgnore("apps/web/node_modules/react/index.js")).toBe(true);
+    expect(isExpectedIgnore(".git/config")).toBe(true);
+    expect(isExpectedIgnore(".DS_Store")).toBe(true);
+    expect(isExpectedIgnore("x/.env")).toBe(true);
+    expect(isExpectedIgnore("a/b/.hq-sync.pid")).toBe(true);
+    expect(isExpectedIgnore("y/foo.pyc")).toBe(true);
+  });
+
+  it("keeps nested repos/workspace and ordinary content noteworthy", () => {
+    expect(isExpectedIgnore("companies/indigo/knowledge/repos/foo/notes.md")).toBe(false);
+    expect(isExpectedIgnore("companies/indigo/knowledge/overview.md")).toBe(false);
+  });
+});
 
 describe("createIgnoreFilter", () => {
   let hqRoot: string;
@@ -69,6 +93,35 @@ describe("createIgnoreFilter", () => {
     expect(shouldSync(path.join(hqRoot, "companies/indigo/notes.md"))).toBe(true);
   });
 
+  it("visibility fires on an OVER-BROAD exclusion (real filter + recorder, DEV-1791)", () => {
+    // Simulate the regressed over-broad bug: an un-anchored `repos/` in
+    // .hqignore that matches ANY nested repos segment — exactly what
+    // silently ate /discover knowledge under companies/*/knowledge/repos/
+    // before the root-anchor fix. The visibility recorder must surface the
+    // dropped knowledge as NOTEWORTHY while staying quiet on build noise.
+    fs.writeFileSync(path.join(hqRoot, ".hqignore"), "repos/\n");
+    const ignoreFilter = createIgnoreFilter(hqRoot);
+    const noteworthy: string[] = [];
+    let total = 0;
+    const recorded = wrapFilterWithIgnoreVisibility(
+      ignoreFilter,
+      hqRoot,
+      (rel) => { noteworthy.push(rel); },
+      () => { total++; },
+    );
+
+    // Knowledge under a nested repos/ is dropped by the over-broad rule and
+    // is genuine content → captured as noteworthy.
+    expect(recorded(path.join(hqRoot, "companies/indigo/knowledge/repos/foo/notes.md"))).toBe(false);
+    // Build noise is also excluded but is EXPECTED → counted, not noteworthy.
+    expect(recorded(path.join(hqRoot, "node_modules/react/index.js"))).toBe(false);
+    // Ordinary knowledge that the filter ALLOWS fires nothing.
+    expect(recorded(path.join(hqRoot, "companies/indigo/knowledge/overview.md"))).toBe(true);
+
+    expect(noteworthy).toEqual(["companies/indigo/knowledge/repos/foo/notes.md"]);
+    expect(total).toBe(2); // both exclusions counted; only one is noteworthy
+  });
+
   it("v15+ layout (core/core.yaml present): root core.yaml is ignored, nested core/core.yaml syncs", () => {
     // On v15+ the scaffold/version manifest lives at `core/core.yaml` (a real
     // project file that DOES sync). Any root-level `core.yaml` here is a stale

package/src/ignore.ts

@@ -209,6 +209,79 @@ function segmentToRegex(seg: string): RegExp {
   return new RegExp(`^${body}$`);
 }
 
+export const EXPECTED_IGNORE_SEGMENTS = new Set([
+  ".git",
+  "node_modules",
+  ".pnpm-store",
+  "dist",
+  "build",
+  ".next",
+  ".nuxt",
+  ".svelte-kit",
+  ".turbo",
+  ".parcel-cache",
+  ".vite",
+  "coverage",
+  "target",
+  "__pycache__",
+  ".pytest_cache",
+  ".mypy_cache",
+  ".ruff_cache",
+  ".venv",
+  "venv",
+  "vendor",
+  "out",
+  ".cache",
+  "tmp",
+  ".tmp",
+  ".hq",
+  ".claude",
+]);
+
+/**
+ * Classify a base-ignore rejection as expected noise vs noteworthy content.
+ *
+ * `true` means the ignored path belongs to an expected by-design exclusion
+ * class: HQ-managed top-level roots, build/VCS/cache/machine-local segments,
+ * or known generated/noise basenames. `false` means the rejected path looks
+ * like real user content and should be surfaced by push observability.
+ *
+ * Top-level `repos/` and `workspace/` are expected, but nested segments named
+ * `repos` or `workspace` are deliberately noteworthy: that is the DEV-1791
+ * regression signal this classifier exists to preserve.
+ */
+export function isExpectedIgnore(relPath: string): boolean {
+  if (
+    relPath === "repos" ||
+    relPath.startsWith("repos/") ||
+    relPath === "workspace" ||
+    relPath.startsWith("workspace/")
+  ) {
+    return true;
+  }
+
+  const segments = relPath.split("/");
+  for (const segment of segments) {
+    if (EXPECTED_IGNORE_SEGMENTS.has(segment)) return true;
+  }
+
+  const basename = segments[segments.length - 1] ?? "";
+  return (
+    basename === ".DS_Store" ||
+    basename === "Thumbs.db" ||
+    basename === "company.yaml" ||
+    basename === "INDEX.md" ||
+    basename === "modules.lock" ||
+    basename.endsWith(".pyc") ||
+    basename.endsWith(".class") ||
+    basename.endsWith(".pid") ||
+    basename.startsWith(".hq-") ||
+    basename === ".env" ||
+    basename.startsWith(".env.") ||
+    basename === ".mcp.json"
+  );
+}
+
 export function createIgnoreFilter(
   hqRoot: string,
 ): (filePath: string, isDir?: boolean) => boolean {

package/src/object-io.test.ts

@@ -19,6 +19,7 @@ import {
 } from "./object-io.js";
 import type { EntityContext } from "./types.js";
 import type { PresignResultRow, VaultListedObject } from "./vault-client.js";
+import { isAccessDenied } from "./sync-core.js";
 
 const COMPANY = "cmp_test";
 
@@ -205,6 +206,59 @@ describe("PresignObjectIO.putObject", () => {
     expect(fetchMock).not.toHaveBeenCalled();
   });
 
+  it("a write denial (FILES_PRESIGN_FORBIDDEN) throws the access-denied shape so the push skips it like a denied GET", async () => {
+    const { vault, setPresign } = makeVault();
+    // The server refuses to MINT a PUT URL for a key the caller can't write
+    // (private-by-default: members hold read-only on a shared knowledge prefix).
+    setPresign([
+      {
+        key: "knowledge/clients/acme/brief.md",
+        op: "put",
+        error: "Caller lacks write permission on 'knowledge/clients/acme/brief.md'",
+        code: "FILES_PRESIGN_FORBIDDEN",
+      },
+    ]);
+    const io = new PresignObjectIO(vault, COMPANY);
+    let caught: unknown;
+    await io
+      .putObject({
+        key: "knowledge/clients/acme/brief.md",
+        body: Buffer.from("z"),
+        contentType: "text/markdown",
+      })
+      .catch((e) => {
+        caught = e;
+      });
+    // Same shape a denied GET/HEAD already throws (name === "Forbidden"), so the
+    // share.ts push catch routes it through `isAccessDenied` into the silent
+    // per-path skip (scopeExcludedSet) + the "N paths skipped — outside your
+    // granted access" aggregate, instead of a fatal `type:"error"` that exits
+    // the runner non-zero and floods Sentry. This is the PUT/GET symmetry that
+    // fixes the fleet-wide presign-put-denied crash-loop.
+    expect((caught as Error).name).toBe("Forbidden");
+    expect(isAccessDenied(caught)).toBe(true);
+    // Denied at mint time — the byte-moving PUT is never attempted.
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+
+  it("a non-authorization presign failure stays a generic (fatal) error — not silently skipped", async () => {
+    const { vault, setPresign } = makeVault();
+    setPresign([
+      { key: "secret/x", op: "put", error: "boom", code: "PRESIGN_INTERNAL" },
+    ]);
+    const io = new PresignObjectIO(vault, COMPANY);
+    let caught: unknown;
+    await io
+      .putObject({ key: "secret/x", body: Buffer.from("z"), contentType: "x" })
+      .catch((e) => {
+        caught = e;
+      });
+    // Only `*_FORBIDDEN` authorization denials are reclassified; a real backend
+    // failure must still surface loudly so it is not masked as an expected skip.
+    expect(isAccessDenied(caught)).toBe(false);
+    expect((caught as Error).message).toMatch(/denied for secret\/x.*PRESIGN_INTERNAL/);
+  });
+
   it("throws when the PUT itself fails", async () => {
     const { vault, setPresign } = makeVault();
     setPresign([{ key: "k", op: "put", url: "https://s3/put" }]);
@@ -516,6 +570,25 @@ describe("PresignObjectIO.headObject", () => {
     expect(await io.headObject("gone")).toBeNull();
   });
 
+  it("returns null on a server-confirmed FILES_PRESIGN_NOT_FOUND (absence, not denial — no GET issued)", async () => {
+    // The presign lambda HEAD'd S3 and confirmed absence. Unlike a read
+    // DENIAL (Forbidden, below), this is unambiguous "object missing", so
+    // headObject reports absent (null) and the push creates the new object —
+    // never reaching a 403-masked presigned GET.
+    const { vault, setPresign } = makeVault();
+    setPresign([
+      {
+        key: "new.md",
+        op: "get",
+        error: "Object not found: 'new.md'",
+        code: "FILES_PRESIGN_NOT_FOUND",
+      },
+    ]);
+    const io = new PresignObjectIO(vault, COMPANY);
+    expect(await io.headObject("new.md")).toBeNull();
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+
   it("throws Forbidden when presign denies the key — denial is NOT absence", async () => {
     // Regression: pre-fix this returned null ("absent"), which made
     // share.ts's push guard (`if (remoteMeta)`) skip every conflict check

package/src/object-io.ts

@@ -354,6 +354,24 @@ function firstRowOrThrow(
     throw new Error(`presign ${op} returned no row for ${key}`);
   }
   if (row.error || !row.url) {
+    // An authorization denial (server `*_FORBIDDEN` code — e.g.
+    // FILES_PRESIGN_FORBIDDEN when the caller lacks write on a shared prefix,
+    // or FILES_RAW_FORBIDDEN) must throw the SAME `name: "Forbidden"` shape the
+    // GET/HEAD path uses (see {@link accessDeniedError}) so it routes through
+    // the `isAccessDenied` skip in sync.ts / share.ts. Without this, a denied
+    // PUT presign fell to the generic Error below — `isAccessDenied` returned
+    // false — and the per-file ACL skip the GET path handles gracefully became
+    // a FATAL sync error: the crash-loop + Sentry flood behind the fleet-wide
+    // "presign put denied" incident (members pushing net-new keys to a
+    // company-wide-read prefix under private-by-default). The denial is
+    // expected; only the fatal handling of it was the bug.
+    if (typeof row.code === "string" && row.code.endsWith("_FORBIDDEN")) {
+      throw accessDeniedError(
+        key,
+        `no write access to '${key}' (server ${op} denied: ${row.code}) — ` +
+          `ask an admin to grant write on this shared prefix`,
+      );
+    }
     throw new Error(
       `presign ${op} denied for ${key}: ${row.error ?? "no url"}${
         row.code ? ` (${row.code})` : ""
@@ -853,6 +871,17 @@ export class PresignObjectIO implements ObjectIO {
         throw err;
       }
       const row = results[0];
+      if (row?.code === "FILES_PRESIGN_NOT_FOUND") {
+        // The presign lambda HEAD'd S3 and confirmed the object does NOT
+        // exist — an unambiguous absence, distinct from the read DENIAL
+        // handled below. Report absent (null) so the push creates the new
+        // object. Without this, a brand-new file's first upload is blocked
+        // forever: its presigned GET 403s (S3 masks absence when the URL
+        // lacks `s3:ListBucket`) and routes through the access-denied skip
+        // path. The server's explicit NOT_FOUND is the ONLY safe absence
+        // signal — a 403 stays "unknown", never "missing" (see below).
+        return null;
+      }
       if (!row || row.error || !row.url) {
         // A per-key denial means the caller can't READ the key — it says
         // nothing about whether the object EXISTS. Pre-fix this returned