@indigoai-us/hq-cloud 6.11.11 → 6.11.13

package/src/prefix-coalesce.test.ts

@@ -3,6 +3,8 @@ import {
   coalescePrefixes,
   isCoveredByAny,
   grantPathToPrefix,
+  grantPathToScopePrefix,
+  toScopePrefixEntries,
 } from "./prefix-coalesce.js";
 
 describe("coalescePrefixes", () => {
@@ -74,6 +76,12 @@ describe("coalescePrefixes", () => {
     // prefixes — the grants endpoint always does.)
     expect(coalescePrefixes(["a/", "ab/"]).sort()).toEqual(["a/", "ab/"]);
   });
+
+  it("F16: keeps exact-file scopes distinct from prefixed sibling keys", () => {
+    expect(
+      coalescePrefixes(["knowledge/README.md", "knowledge/README.md.bak"]),
+    ).toEqual(["knowledge/README.md", "knowledge/README.md.bak"]);
+  });
 });
 
 describe("isCoveredByAny", () => {
@@ -121,7 +129,11 @@ describe("grantPathToPrefix", () => {
   });
 
   it("folds a trailing bare * into its parent prefix", () => {
-    expect(grantPathToPrefix("reports*", slug)).toBe("reports");
+    const prefix = grantPathToPrefix("reports*", slug);
+    expect(toScopePrefixEntries([prefix])).toEqual([
+      { prefix: "reports", match: "prefix" },
+    ]);
+    expect(isCoveredByAny("reports-q1.md", [prefix])).toBe(true);
   });
 
   it("leaves a company-relative directory prefix unchanged", () => {
@@ -167,4 +179,62 @@ describe("grantPathToPrefix", () => {
     expect(isCoveredByAny("secrets/db.json", prefixes)).toBe(false);
     expect(isCoveredByAny("knowledge/OTHER.md", prefixes)).toBe(false);
   });
+
+  it("F16: exact-file grants do not authorize prefixed sibling keys", () => {
+    const grants = [
+      "companies/indigo/knowledge/README.md",
+      "company.yaml",
+      "data/vyg/*",
+    ];
+    const prefixes = coalescePrefixes(grants.map((g) => grantPathToPrefix(g, slug)));
+
+    expect(isCoveredByAny("knowledge/README.md", prefixes)).toBe(true);
+    expect(isCoveredByAny("company.yaml", prefixes)).toBe(true);
+    expect(isCoveredByAny("data/vyg/2026/q1.md", prefixes)).toBe(true);
+
+    expect(isCoveredByAny("knowledge/README.md.bak", prefixes)).toBe(false);
+    expect(isCoveredByAny("company.yaml.tmp", prefixes)).toBe(false);
+  });
+
+  it("R-F16: exact scope is deterministic after a prior bare-glob scope", () => {
+    expect(toScopePrefixEntries([grantPathToPrefix("companies/indigo/foo*", slug)])).toEqual([
+      { prefix: "foo", match: "prefix" },
+    ]);
+
+    const exactPrefixes = coalescePrefixes([
+      grantPathToPrefix("companies/other/foo", "other"),
+    ]);
+
+    expect(isCoveredByAny("foo", exactPrefixes)).toBe(true);
+    expect(isCoveredByAny("foo.bak", exactPrefixes)).toBe(false);
+  });
+
+  it("RF-F16DUR: exact-vs-prefix survives array copy and JSON round-trip", () => {
+    const priorBareGlob = coalescePrefixes([
+      grantPathToScopePrefix("companies/indigo/foo*", slug),
+    ]);
+    const copiedBareGlob = [...priorBareGlob];
+    const jsonBareGlob = JSON.parse(JSON.stringify(priorBareGlob)) as string[];
+
+    expect(jsonBareGlob).toEqual(priorBareGlob);
+    expect(toScopePrefixEntries(copiedBareGlob)).toEqual([
+      { prefix: "foo", match: "prefix" },
+    ]);
+    expect(toScopePrefixEntries(jsonBareGlob)).toEqual([
+      { prefix: "foo", match: "prefix" },
+    ]);
+    expect(isCoveredByAny("foo.bak", copiedBareGlob)).toBe(true);
+    expect(isCoveredByAny("foo.bak", jsonBareGlob)).toBe(true);
+
+    const exactAfterPriorGlob = coalescePrefixes([
+      grantPathToScopePrefix("companies/indigo/foo", slug),
+    ]);
+    const jsonExact = JSON.parse(JSON.stringify(exactAfterPriorGlob)) as string[];
+
+    expect(toScopePrefixEntries(jsonExact)).toEqual([
+      { prefix: "foo", match: "exact" },
+    ]);
+    expect(isCoveredByAny("foo", jsonExact)).toBe(true);
+    expect(isCoveredByAny("foo.bak", jsonExact)).toBe(false);
+  });
 });

package/src/prefix-coalesce.ts

@@ -21,18 +21,138 @@
  *   - **Determinism:** output is sorted lexicographically so the journal's
  *     `prefixSet` stays diff-stable across runs.
  *
- * Coverage rule: `a` covers `b` iff `a === b` OR `b.startsWith(a)`. This is a
- * literal string-prefix relation — it does NOT understand S3 "folders". A
- * caller that wants `a/` to NOT cover `ab/` must pass trailing-slash-bounded
- * prefixes (the grants endpoint already does — every ACL row ends in `/`).
+ * Coverage rule: exact-file entries cover only themselves. Directory prefixes
+ * (ending in `/`) and bare glob prefixes cover descendants with literal
+ * `startsWith` matching. When a string is ambiguous (e.g. `foo` from `foo*`,
+ * which must be a prefix, not an exact file), the coalesced `prefixSet` stores
+ * a small in-band marker in the string itself so the match type survives
+ * `[...prefixSet]` copies and JSON persistence.
  */
 
-export function coalescePrefixes(prefixes: readonly string[]): string[] {
-  // Dedup + drop empties.
-  const unique = new Set<string>();
+export type ScopePrefixMatch = "exact" | "prefix";
+
+export interface ScopePrefixEntry {
+  prefix: string;
+  match: ScopePrefixMatch;
+}
+
+export type ScopePrefixInput = string | ScopePrefixEntry;
+
+const SCOPE_PREFIX_MARKER = "\u0000hq-scope:";
+const SCOPE_PREFIX_MARKER_PREFIX = `${SCOPE_PREFIX_MARKER}prefix`;
+const SCOPE_PREFIX_MARKER_EXACT = `${SCOPE_PREFIX_MARKER}exact`;
+
+function isScopePrefixEntry(value: ScopePrefixInput): value is ScopePrefixEntry {
+  return typeof value !== "string";
+}
+
+function inferStringMatch(prefix: string): ScopePrefixMatch {
+  return prefix === "" || prefix.endsWith("/") ? "prefix" : "exact";
+}
+
+function parseStringPrefix(input: string): ScopePrefixEntry {
+  if (input.endsWith(SCOPE_PREFIX_MARKER_PREFIX)) {
+    return {
+      prefix: input.slice(0, -SCOPE_PREFIX_MARKER_PREFIX.length),
+      match: "prefix",
+    };
+  }
+  if (input.endsWith(SCOPE_PREFIX_MARKER_EXACT)) {
+    return {
+      prefix: input.slice(0, -SCOPE_PREFIX_MARKER_EXACT.length),
+      match: "exact",
+    };
+  }
+  return {
+    prefix: input,
+    match: inferStringMatch(input),
+  };
+}
+
+function encodeScopePrefix(entry: ScopePrefixEntry): string {
+  return entry.match === inferStringMatch(entry.prefix)
+    ? entry.prefix
+    : `${entry.prefix}${SCOPE_PREFIX_MARKER}${entry.match}`;
+}
+
+function normalizeScopePrefix(input: ScopePrefixInput): ScopePrefixEntry | null {
+  if (typeof input === "string") {
+    return parseStringPrefix(input);
+  }
+  if (typeof input.prefix !== "string") return null;
+  return {
+    prefix: input.prefix,
+    match: input.match === "prefix" ? "prefix" : "exact",
+  };
+}
+
+function materializeScopePrefixes(
+  prefixes: readonly ScopePrefixInput[],
+): ScopePrefixEntry[] {
+  const entries: ScopePrefixEntry[] = [];
   for (const p of prefixes) {
-    if (typeof p === "string" && p !== "") {
-      unique.add(p);
+    const entry = normalizeScopePrefix(p);
+    if (entry) entries.push(entry);
+  }
+  return entries;
+}
+
+function compareEntries(a: ScopePrefixEntry, b: ScopePrefixEntry): number {
+  if (a.prefix < b.prefix) return -1;
+  if (a.prefix > b.prefix) return 1;
+  if (a.match === b.match) return 0;
+  return a.match === "prefix" ? -1 : 1;
+}
+
+function entryCoversPath(entry: ScopePrefixEntry, key: string): boolean {
+  if (entry.prefix === "") return true;
+  if (key === entry.prefix) return true;
+  return entry.match === "prefix" && key.startsWith(entry.prefix);
+}
+
+function entryCoversEntry(
+  entry: ScopePrefixEntry,
+  candidate: ScopePrefixEntry,
+): boolean {
+  if (entry.prefix === "") return true;
+  if (entry.match === "prefix") {
+    return (
+      candidate.prefix === entry.prefix ||
+      candidate.prefix.startsWith(entry.prefix)
+    );
+  }
+  return candidate.match === "exact" && candidate.prefix === entry.prefix;
+}
+
+export function toScopePrefixEntries(
+  prefixSet: readonly ScopePrefixInput[],
+): ScopePrefixEntry[] {
+  return materializeScopePrefixes(prefixSet);
+}
+
+export function pathToScopePrefix(path: string): ScopePrefixEntry {
+  const p = (path ?? "").replace(/^\/+/, "");
+  if (
+    p.endsWith(SCOPE_PREFIX_MARKER_PREFIX) ||
+    p.endsWith(SCOPE_PREFIX_MARKER_EXACT)
+  ) {
+    return parseStringPrefix(p);
+  }
+  if (p === "*" || p === "") return { prefix: "", match: "prefix" };
+  if (p.endsWith("/*")) return { prefix: p.slice(0, -1), match: "prefix" };
+  if (p.endsWith("*")) return { prefix: p.slice(0, -1), match: "prefix" };
+  if (p.endsWith("/")) return { prefix: p, match: "prefix" };
+  return { prefix: p, match: "exact" };
+}
+
+export function coalescePrefixes(
+  prefixes: readonly ScopePrefixInput[],
+): string[] {
+  // Dedup + drop empties.
+  const unique = new Map<string, ScopePrefixEntry>();
+  for (const entry of materializeScopePrefixes(prefixes)) {
+    if (entry.prefix !== "") {
+      unique.set(`${entry.match}\0${entry.prefix}`, entry);
     }
   }
   if (unique.size === 0) return [];
@@ -40,33 +160,33 @@ export function coalescePrefixes(prefixes: readonly string[]): string[] {
   // Sort lexicographically so a broader prefix (`a/`) appears before its
   // narrower descendants (`a/b/`). Then a single pass keeps the broadest in
   // each cover chain.
-  const sorted = [...unique].sort();
-  const result: string[] = [];
-  let lastKept: string | null = null;
-  for (const p of sorted) {
-    if (lastKept !== null && p.startsWith(lastKept)) {
+  const sorted = [...unique.values()].sort(compareEntries);
+  const result: ScopePrefixEntry[] = [];
+  let lastKept: ScopePrefixEntry | null = null;
+  for (const entry of sorted) {
+    if (lastKept !== null && entryCoversEntry(lastKept, entry)) {
       // `lastKept` already covers `p`; skip.
       continue;
     }
-    result.push(p);
-    lastKept = p;
+    result.push(entry);
+    lastKept = entry;
   }
-  return result;
+  return result.map(encodeScopePrefix);
 }
 
 /**
  * Predicate companion: does any prefix in `prefixSet` cover `path`?
  *
  * Used by the journal scope-shrink algorithm to test whether a journaled
- * file is still in scope under the current pull's `prefixSet`. Same
- * `startsWith` semantics as `coalescePrefixes`.
+ * file is still in scope under the current pull's `prefixSet`. Exact-file
+ * entries match only themselves; directory/glob prefixes use `startsWith`.
  */
 export function isCoveredByAny(
   path: string,
-  prefixSet: readonly string[],
+  prefixSet: readonly ScopePrefixInput[],
 ): boolean {
-  for (const p of prefixSet) {
-    if (p === "" || path.startsWith(p)) return true;
+  for (const entry of materializeScopePrefixes(prefixSet)) {
+    if (entryCoversPath(entry, path)) return true;
   }
   return false;
 }
@@ -94,13 +214,14 @@ export function isCoveredByAny(
  */
 export function isDirInScope(
   relDir: string,
-  prefixSet: readonly string[],
+  prefixSet: readonly ScopePrefixInput[],
 ): boolean {
   const dir = relDir === "" || relDir.endsWith("/") ? relDir : relDir + "/";
-  for (const p of prefixSet) {
+  for (const entry of materializeScopePrefixes(prefixSet)) {
+    const p = entry.prefix;
     if (p === "") return true; // full scope
     if (dir === "") return true; // company root — descend to reach grants
-    if (dir.startsWith(p)) return true; // dir is inside a granted prefix
+    if (entry.match === "prefix" && dir.startsWith(p)) return true; // dir is inside a granted prefix
     if (p.startsWith(dir)) return true; // a granted prefix is inside dir
   }
   return false;
@@ -134,7 +255,10 @@ export function isDirInScope(
  * `syncMode: shared` would download nothing and prune everything the caller
  * actually has access to. See the live grant dump in the hq-pro vault.
  */
-export function grantPathToPrefix(grantPath: string, slug: string): string {
+export function grantPathToScopePrefix(
+  grantPath: string,
+  slug: string,
+): ScopePrefixEntry {
   let p = (grantPath ?? "").replace(/^\/+/, "");
   // 1. Strip the company anchor, if present.
   const companyAnchor = `companies/${slug}/`;
@@ -145,8 +269,9 @@ export function grantPathToPrefix(grantPath: string, slug: string): string {
     p = p.slice(slugAnchor.length);
   }
   // 2. Fold the ACL glob suffix into a startsWith prefix.
-  if (p === "*" || p === "") return ""; // company-wide / empty → everything
-  if (p.endsWith("/*")) return p.slice(0, -1); // "a/b/*" → "a/b/"
-  if (p.endsWith("*")) return p.slice(0, -1); // "a/b*"  → "a/b"
-  return p; // dir prefix ("a/b/") or exact key ("a/b.md") — already a prefix
+  return pathToScopePrefix(p);
+}
+
+export function grantPathToPrefix(grantPath: string, slug: string): string {
+  return encodeScopePrefix(grantPathToScopePrefix(grantPath, slug));
 }

package/src/remote-pull.test.ts

@@ -1,7 +1,7 @@
 /**
  * Failing-test seed for the Auto-sync (Beta) remote-pull loop.
  *
- * Background: SyncWatcher (watcher.ts) pushes local edits to S3 in seconds,
+ * Background: the watcher push path ships local edits to S3 in seconds,
  * but pulls happen only on a manual sync today. Auto-sync adds a periodic
  * (every 10 min) remote-pull pass per company. The decision of *which* keys
  * to download / delete locally / skip is pure given a remote listing, the
@@ -25,6 +25,7 @@ import {
   POST_FILTER_THRESHOLD,
   pullCompany,
   resolveCompanyScope,
+  VEND_FANOUT_CONCURRENCY,
   VEND_PATH_CAP,
 } from "./remote-pull.js";
 import type { RemoteFile } from "./s3.js";
@@ -157,6 +158,111 @@ describe("decideRemotePulls", () => {
     expect(result.download).toEqual([]);
   });
 
+  it("F07: excludes scope-shrink tombstones from remote-delete decisions", () => {
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "2026-05-01T00:00:00Z",
+      files: {
+        "docs/scope-shrunk-dirty.md": {
+          hash: "dirty-before-shrink",
+          size: 42,
+          syncedAt: "2026-05-01T00:00:00Z",
+          direction: "down",
+          remoteEtag: "old-dirty",
+          removedAt: "2026-05-02T00:00:00Z",
+          removedReason: "scope_shrink",
+        },
+        "docs/live-gone.md": {
+          hash: "h",
+          size: 100,
+          syncedAt: "2026-05-01T00:00:00Z",
+          direction: "down",
+          remoteEtag: "old-live",
+        },
+      },
+      pulls: [],
+    };
+    const result = decideRemotePulls({
+      remoteFiles: [],
+      journal,
+      conflictKeys: new Set(),
+    });
+    expect(result.deleteLocal).toEqual(["docs/live-gone.md"]);
+    expect(result.download).toEqual([]);
+  });
+
+  it("R-F07: excludes scope-shrink-protected survivors from remote-delete decisions", async () => {
+    const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rf07-"));
+    try {
+      const journal: SyncJournal = {
+        version: "2",
+        lastSync: "2026-05-01T00:00:00Z",
+        files: {
+          "companies/indigo/projects/mine.md": {
+            hash: "h-mine",
+            size: 6,
+            syncedAt: "2026-05-01T00:00:00Z",
+            direction: "down",
+            remoteEtag: "old-mine",
+            createdBySub: "owner-sub",
+          },
+          "companies/indigo/projects/legacy.md": {
+            hash: "h-legacy",
+            size: 6,
+            syncedAt: "2026-05-01T00:00:00Z",
+            direction: "down",
+            remoteEtag: "old-legacy",
+          },
+          "companies/indigo/meetings/gone.md": {
+            hash: "h-gone",
+            size: 4,
+            syncedAt: "2026-05-01T00:00:00Z",
+            direction: "down",
+            remoteEtag: "old-gone",
+            createdBySub: "peer-sub",
+          },
+        },
+        pulls: [
+          {
+            pullId: "01PREV",
+            companyUid: "cmp_indigo",
+            startedAt: "2026-05-19T00:00:00.000Z",
+            completedAt: "2026-05-19T00:00:05.000Z",
+            syncMode: "all",
+            prefixSet: ["companies/indigo/"],
+            scopeChangeDetected: false,
+            orphansRemoved: 0,
+            orphansBlocked: 0,
+          },
+        ],
+      };
+
+      const result = await pullCompany({
+        ctx: makeCtx(),
+        journal,
+        hqRoot,
+        callerSub: "owner-sub",
+        scope: {
+          companyUid: "cmp_indigo",
+          syncMode: "shared",
+          prefixSet: ["companies/indigo/meetings/"],
+          strategy: "vend-fanout",
+        },
+        listFn: async () => [],
+      });
+
+      expect(result.decision.deleteLocal).toEqual([
+        "companies/indigo/meetings/gone.md",
+      ]);
+      expect(result.decision.skip.map((f) => f.key).sort()).toEqual([
+        "companies/indigo/projects/legacy.md",
+        "companies/indigo/projects/mine.md",
+      ]);
+    } finally {
+      fs.rmSync(hqRoot, { recursive: true, force: true });
+    }
+  });
+
   it("never deletes locally for entries that have NEVER been synced down", () => {
     // Push-only entries (direction: 'up') represent files we created locally
     // and uploaded. If the user later deletes them remotely from another
@@ -474,6 +580,34 @@ describe("listRemoteForScope", () => {
     expect(calls.size).toBe(VEND_PATH_CAP + 3); // every prefix listed
   });
 
+  it("F28: bounds concurrent per-prefix listings across vend-fanout batches", async () => {
+    const prefixes = Array.from(
+      { length: POST_FILTER_THRESHOLD },
+      (_, i) => `companies/indigo/p${i}/`,
+    );
+    let active = 0;
+    let maxActive = 0;
+
+    await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: prefixes,
+        strategy: "vend-fanout",
+      },
+      listFn: async () => {
+        active += 1;
+        maxActive = Math.max(maxActive, active);
+        await new Promise((resolve) => setTimeout(resolve, 1));
+        active -= 1;
+        return [];
+      },
+    });
+
+    expect(maxActive).toBeLessThanOrEqual(VEND_FANOUT_CONCURRENCY);
+  });
+
   it("strategy=vend-fanout uses vendForBatchFn to narrow credentials per batch", async () => {
     const vendCalls: Array<{ paths: string[] }> = [];
     await listRemoteForScope({
@@ -521,6 +655,28 @@ describe("listRemoteForScope", () => {
     ]);
   });
 
+  it("R-F16: strategy=vend-fanout post-filters exact-grant siblings", async () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: [makeGrant("companies/indigo/README.md")],
+    });
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope,
+      listFn: async (_ctx, prefix) => {
+        expect(prefix).toBe("companies/indigo/README.md");
+        return [
+          remote({ key: "companies/indigo/README.md" }),
+          remote({ key: "companies/indigo/README.md.bak" }),
+        ];
+      },
+    });
+
+    expect(files.map((f) => f.key)).toEqual(["companies/indigo/README.md"]);
+  });
+
   it("strategy=vend-fanout short-circuits to [] on empty prefixSet", async () => {
     let listed = false;
     const files = await listRemoteForScope({
@@ -859,6 +1015,84 @@ describe("pullCompany (engine orchestrator)", () => {
     expect(fs.existsSync(legacyAbs)).toBe(true); // legacy file retained
   });
 
+  it("RF-F07DUR: scope-shrink-protected survivors are excluded on a second pull", async () => {
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "companies/indigo/projects/legacy.md": {
+          hash: "h-legacy",
+          size: 6,
+          syncedAt: "2026-05-01T00:00:00.000Z",
+          direction: "down",
+          remoteEtag: "old-legacy",
+        },
+        "companies/indigo/meetings/gone.md": {
+          hash: "h-gone",
+          size: 4,
+          syncedAt: "2026-05-01T00:00:00.000Z",
+          direction: "down",
+          remoteEtag: "old-gone",
+          createdBySub: "peer-sub",
+        },
+      },
+      pulls: [
+        {
+          pullId: "01PREV",
+          companyUid: "cmp_indigo",
+          startedAt: "2026-05-19T00:00:00.000Z",
+          completedAt: "2026-05-19T00:00:05.000Z",
+          syncMode: "all",
+          prefixSet: ["companies/indigo/"],
+          scopeChangeDetected: false,
+          orphansRemoved: 0,
+          orphansBlocked: 0,
+        },
+      ],
+    };
+    const narrowedScope = {
+      companyUid: "cmp_indigo",
+      syncMode: "shared" as const,
+      prefixSet: ["companies/indigo/meetings/"],
+      strategy: "vend-fanout" as const,
+    };
+
+    const first = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      callerSub: "owner-sub",
+      scope: narrowedScope,
+      listFn: async () => [],
+    });
+
+    expect(first.decision.skip.map((f) => f.key)).toContain(
+      "companies/indigo/projects/legacy.md",
+    );
+    expect(first.decision.deleteLocal).toEqual([
+      "companies/indigo/meetings/gone.md",
+    ]);
+    expect(
+      journal.files["companies/indigo/projects/legacy.md"]?.outOfScopeProtected,
+    ).toBe(true);
+
+    const second = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      callerSub: "owner-sub",
+      scope: narrowedScope,
+      listFn: async () => [],
+    });
+
+    expect(second.decision.skip.map((f) => f.key)).toContain(
+      "companies/indigo/projects/legacy.md",
+    );
+    expect(second.decision.deleteLocal).toEqual([
+      "companies/indigo/meetings/gone.md",
+    ]);
+  });
+
   it("GC's expired tombstones at the start of every leg", async () => {
     const old = new Date(
       Date.now() - 31 * 24 * 60 * 60 * 1000,