@indigoai-us/hq-cloud 6.11.11 → 6.11.13

package/src/remote-pull.ts

@@ -8,7 +8,7 @@
  * sync-runner.ts trivial: list S3 → call `decideRemotePulls` → drive S3 +
  * filesystem from the result.
  *
- * Pairs with `SyncWatcher` (push-side) — together they implement the
+ * Pairs with the TreeWatcher push path — together they implement the
  * bidirectional auto-sync the Settings toggle exposes.
  */
 import type { RemoteFile } from "./s3.js";
@@ -29,7 +29,13 @@ import type {
   ExplicitGrant,
   MembershipSyncConfig,
 } from "./vault-client.js";
-import { coalescePrefixes } from "./prefix-coalesce.js";
+import {
+  coalescePrefixes,
+  isCoveredByAny,
+  pathToScopePrefix,
+  toScopePrefixEntries,
+  type ScopePrefixInput,
+} from "./prefix-coalesce.js";
 import {
   applyScopeShrink,
   buildScopeShrinkPlan,
@@ -73,12 +79,18 @@ export interface DecideRemotePullsInput {
    * conflict resolution can't be silently overwritten.
    */
   conflictKeys: Set<string>;
+  /**
+   * Journal keys intentionally retained by scope-shrink authorship guards
+   * even though they are outside the current remote listing scope.
+   */
+  protectedMissingKeys?: Set<string>;
 }
 
 export function decideRemotePulls({
   remoteFiles,
   journal,
   conflictKeys,
+  protectedMissingKeys = new Set<string>(),
 }: DecideRemotePullsInput): RemotePullDecision {
   const download: RemoteFile[] = [];
   const skip: SkippedKey[] = [];
@@ -111,6 +123,11 @@ export function decideRemotePulls({
   // Tombstone pass: anything in the journal that's no longer remote.
   for (const relativePath of Object.keys(journal.files)) {
     if (seenRemote.has(relativePath)) continue;
+    if (journal.files[relativePath]?.removedAt) continue;
+    if (protectedMissingKeys.has(relativePath)) {
+      skip.push({ key: relativePath });
+      continue;
+    }
     if (conflictKeys.has(relativePath)) {
       // Remote tombstone for a file the user is conflict-resolving locally.
       // Skip — record so callers can log/report, but do NOT delete.
@@ -143,7 +160,7 @@ export const VEND_PATH_CAP = 10;
  */
 export const POST_FILTER_THRESHOLD = 50;
 
-/** Bounded parallelism for vend fan-out (5 concurrent STS+ListObjectsV2 calls). */
+/** Bounded parallelism for vend fan-out (5 concurrent vends/list paginators). */
 export const VEND_FANOUT_CONCURRENCY = 5;
 
 /**
@@ -204,12 +221,12 @@ export function resolveCompanyScope(
     };
   }
 
-  let raw: string[];
+  let raw: ScopePrefixInput[];
   if (syncConfig.syncMode === "custom") {
-    raw = syncConfig.customPaths ?? [];
+    raw = (syncConfig.customPaths ?? []).map(pathToScopePrefix);
   } else {
     // 'shared'
-    raw = (explicitGrants ?? []).map((g) => g.path);
+    raw = (explicitGrants ?? []).map((g) => pathToScopePrefix(g.path));
   }
   const prefixSet = coalescePrefixes(raw);
 
@@ -239,9 +256,10 @@ export function batchPrefixesForVend(
   cap: number = VEND_PATH_CAP,
 ): string[][] {
   if (cap <= 0) throw new Error(`batchPrefixesForVend: cap must be > 0`);
+  const vendPrefixes = toScopePrefixEntries(prefixes).map((entry) => entry.prefix);
   const batches: string[][] = [];
-  for (let i = 0; i < prefixes.length; i += cap) {
-    batches.push(prefixes.slice(i, i + cap));
+  for (let i = 0; i < vendPrefixes.length; i += cap) {
+    batches.push(vendPrefixes.slice(i, i + cap));
   }
   return batches;
 }
@@ -273,6 +291,27 @@ async function mapWithConcurrency<T, R>(
   return results;
 }
 
+function createConcurrencyLimiter(
+  concurrency: number,
+): <R>(fn: () => Promise<R>) => Promise<R> {
+  const limit = Math.max(1, concurrency);
+  let active = 0;
+  const waiters: Array<() => void> = [];
+
+  return async function runLimited<R>(fn: () => Promise<R>): Promise<R> {
+    if (active >= limit) {
+      await new Promise<void>((resolve) => waiters.push(resolve));
+    }
+    active += 1;
+    try {
+      return await fn();
+    } finally {
+      active -= 1;
+      waiters.shift()?.();
+    }
+  };
+}
+
 export interface ListRemoteForScopeInput {
   ctx: EntityContext;
   scope: CompanyScope;
@@ -320,14 +359,16 @@ export async function listRemoteForScope(
 
   if (scope.strategy === "broad-postfilter") {
     const all = await list(ctx);
-    return all.filter((f) =>
-      scope.prefixSet.some((p) => f.key.startsWith(p)),
-    );
+    const scopeEntries = toScopePrefixEntries(scope.prefixSet);
+    return all.filter((f) => isCoveredByAny(f.key, scopeEntries));
   }
 
   // vend-fanout
   if (scope.prefixSet.length === 0) return [];
-  const batches = batchPrefixesForVend(scope.prefixSet);
+  const scopeEntries = toScopePrefixEntries(scope.prefixSet);
+  const listPrefixes = scopeEntries.map((entry) => entry.prefix);
+  const batches = batchPrefixesForVend(listPrefixes);
+  const listWithLimit = createConcurrencyLimiter(VEND_FANOUT_CONCURRENCY);
   const perBatch = await mapWithConcurrency(
     batches,
     VEND_FANOUT_CONCURRENCY,
@@ -338,12 +379,17 @@ export async function listRemoteForScope(
       // For a coalesced batch we issue one ListObjectsV2 per prefix in the
       // batch. We can't issue one ListObjectsV2 across N prefixes (the API
       // takes a single Prefix); the per-batch grouping exists for the STS
-      // session policy ceiling, not the list call itself.
-      const lists = await Promise.all(paths.map((p) => list(batchCtx, p)));
+      // session policy ceiling, not the list call itself. The shared limiter
+      // keeps the total active prefix paginators bounded across all batches.
+      const lists = await Promise.all(
+        paths.map((p) => listWithLimit(() => list(batchCtx, p))),
+      );
       return lists.flat();
     },
   );
-  return dedupByKey(perBatch.flat());
+  return dedupByKey(
+    perBatch.flat().filter((f) => isCoveredByAny(f.key, scopeEntries)),
+  );
 }
 
 function dedupByKey(files: RemoteFile[]): RemoteFile[] {
@@ -434,11 +480,12 @@ export async function pullCompany(
         // `all` -> `shared` flip this correctly flags shared-mode orphans.
         [companyPrefixOf(input.scope, last)];
 
+  const currentPrefixSet = input.scope.prefixSet;
   const scopeShrinkPlan = buildScopeShrinkPlan({
     journal: input.journal,
     hqRoot: input.hqRoot,
     lastPrefixSet,
-    currentPrefixSet: input.scope.prefixSet,
+    currentPrefixSet,
     callerSub: input.callerSub,
     // Background runner pull: protect the caller's own work and don't make a
     // destructive guess about unknown-author (legacy) orphans. The explicit
@@ -479,6 +526,13 @@ export async function pullCompany(
     remoteFiles,
     journal: input.journal,
     conflictKeys,
+    protectedMissingKeys: collectScopeProtectedMissingKeys({
+      journal: input.journal,
+      lastPrefixSet,
+      currentPrefixSet,
+      callerSub: input.callerSub,
+      protectUnknownAuthors: true,
+    }),
   });
 
   const completedAt = now().toISOString();
@@ -509,6 +563,39 @@ export async function pullCompany(
   };
 }
 
+function collectScopeProtectedMissingKeys(input: {
+  journal: SyncJournal;
+  lastPrefixSet: readonly ScopePrefixInput[];
+  currentPrefixSet: readonly ScopePrefixInput[];
+  callerSub?: string;
+  protectUnknownAuthors: boolean;
+}): Set<string> {
+  const protectedKeys = new Set<string>();
+  for (const [relPath, entry] of Object.entries(input.journal.files)) {
+    if (entry.removedAt) continue;
+    if (entry.direction !== "down") continue;
+    if (isCoveredByAny(relPath, input.currentPrefixSet)) {
+      if (entry.outOfScopeProtected) delete entry.outOfScopeProtected;
+      continue;
+    }
+    if (entry.outOfScopeProtected) {
+      protectedKeys.add(relPath);
+      continue;
+    }
+    if (!isCoveredByAny(relPath, input.lastPrefixSet)) continue;
+    if (input.callerSub && entry.createdBySub === input.callerSub) {
+      entry.outOfScopeProtected = true;
+      protectedKeys.add(relPath);
+      continue;
+    }
+    if (input.protectUnknownAuthors && entry.createdBySub === undefined) {
+      entry.outOfScopeProtected = true;
+      protectedKeys.add(relPath);
+    }
+  }
+  return protectedKeys;
+}
+
 /**
  * Recover the "company prefix" for a v1-migrated record with no recorded
  * `prefixSet`. We derive it from the current scope's first prefix's parent
@@ -521,10 +608,11 @@ function companyPrefixOf(
   _last: PullRecord | undefined,
 ): string {
   // For `all` mode, scope.prefixSet[0] IS the company prefix.
-  if (scope.strategy === "all" && scope.prefixSet[0]) return scope.prefixSet[0];
+  const firstEntry = toScopePrefixEntries(scope.prefixSet)[0];
+  if (scope.strategy === "all" && firstEntry) return firstEntry.prefix;
   // Otherwise, derive `companies/{slug}/` from the first prefix. ACL grant
   // paths always start with `companies/{slug}/...`.
-  const first = scope.prefixSet[0] ?? "";
+  const first = firstEntry?.prefix ?? "";
   const m = first.match(/^(companies\/[^/]+\/)/);
   return m ? m[1]! : first;
 }

package/src/s3.test.ts

@@ -10,6 +10,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
+import * as crypto from "crypto";
 
 // Capture every command sent to the S3Client across the test suite. Cleared
 // in beforeEach so per-test assertions don't leak from neighbours.
@@ -96,6 +97,7 @@ import {
 import {
   setObjectIOFactory,
   presignObjectIOFactory,
+  type ObjectIO,
   type PresignTransportClient,
 } from "./object-io.js";
 import type { PresignResultRow } from "./vault-client.js";
@@ -896,6 +898,130 @@ describe("downloadFile", () => {
     expect(fs.readlinkSync(localPath)).toBe("fresh-target.md");
   });
 
+  it("F11: failed symlink downloads preserve the previous local file", async () => {
+    const localPath = path.join(tmpRoot, "preserve-on-failure.md");
+    fs.writeFileSync(localPath, "existing local copy");
+
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield new TextEncoder().encode(SYMLINK_BODY_PREFIX + "x".repeat(10_000));
+      })(),
+      Metadata: { "hq-symlink-target": SYMLINK_MARKER_META_VALUE },
+    };
+
+    await expect(
+      downloadFile(makeCtx(), "preserve-on-failure.md", localPath),
+    ).rejects.toThrow();
+
+    expect(fs.existsSync(localPath)).toBe(true);
+    expect(fs.lstatSync(localPath).isSymbolicLink()).toBe(false);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("existing local copy");
+  });
+
+  it("R-F11: downloads to a near component-limit basename via a bounded temp name", async () => {
+    nextGetObjectResponse = {
+      Body: (async function* () {
+        yield Buffer.from("near-limit bytes");
+      })(),
+      Metadata: {},
+    };
+
+    const localPath = path.join(tmpRoot, "x".repeat(240));
+    await downloadFile(makeCtx(), "near-limit.bin", localPath);
+
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("near-limit bytes");
+  });
+
+  it("F20: regular downloads do not decode the full body as UTF-8", async () => {
+    const body = Buffer.alloc(64 * 1024, 0x61);
+    const originalToString = body.toString;
+    body.toString = ((encoding?: BufferEncoding, start?: number, end?: number) => {
+      const sliceStart = start ?? 0;
+      const sliceEnd = end ?? body.length;
+      if (sliceEnd - sliceStart > SYMLINK_BODY_PREFIX.length) {
+        throw new Error("full-body UTF-8 decode attempted");
+      }
+      return originalToString.call(body, encoding, start, end);
+    }) as Buffer["toString"];
+
+    const io: ObjectIO = {
+      putObject: async () => ({ etag: '"unused"' }),
+      getObject: async () => ({ body, metadata: {} }),
+      listObjects: async () => ({ objects: [] }),
+      deleteObject: async () => undefined,
+      headObject: async () => null,
+    };
+
+    const localPath = path.join(tmpRoot, "large-regular.bin");
+    setObjectIOFactory(() => io);
+    try {
+      await downloadFile(makeCtx(), "large-regular.bin", localPath);
+    } finally {
+      setObjectIOFactory(null);
+    }
+
+    const written = fs.readFileSync(localPath);
+    expect(written.length).toBe(body.length);
+    expect(written[0]).toBe(0x61);
+    expect(written[written.length - 1]).toBe(0x61);
+  });
+
+  it("R-F20: streams regular downloads with hash metadata and still detects symlink records", async () => {
+    const regularChunks = [
+      Buffer.from("alpha-"),
+      Buffer.from("beta-"),
+      Buffer.from("gamma"),
+    ];
+    const regularBody = Buffer.concat(regularChunks);
+    const linkTarget = "../target.md";
+
+    const io: ObjectIO = {
+      putObject: async () => ({ etag: '"unused"' }),
+      getObject: async () => {
+        throw new Error("buffered getObject attempted");
+      },
+      getObjectStream: async (key: string) => {
+        if (key === "streamed-link") {
+          return {
+            body: (async function* () {
+              yield Buffer.from("hq-");
+              yield Buffer.from("symlink:");
+              yield Buffer.from(linkTarget);
+            })(),
+            metadata: {},
+          };
+        }
+        return {
+          body: (async function* () {
+            for (const chunk of regularChunks) yield chunk;
+          })(),
+          metadata: {},
+        };
+      },
+      listObjects: async () => ({ objects: [] }),
+      deleteObject: async () => undefined,
+      headObject: async () => null,
+    };
+
+    setObjectIOFactory(() => io);
+    try {
+      const localPath = path.join(tmpRoot, "streamed.bin");
+      const result = await downloadFile(makeCtx(), "streamed.bin", localPath);
+      expect(fs.readFileSync(localPath)).toEqual(regularBody);
+      expect(result.contentHash).toBe(
+        crypto.createHash("sha256").update(regularBody).digest("hex"),
+      );
+      expect(result.contentSize).toBe(regularBody.length);
+
+      const linkPath = path.join(tmpRoot, "streamed-link");
+      await downloadFile(makeCtx(), "streamed-link", linkPath);
+      expect(fs.lstatSync(linkPath).isSymbolicLink()).toBe(true);
+      expect(fs.readlinkSync(linkPath)).toBe(linkTarget);
+    } finally {
+      setObjectIOFactory(null);
+    }
+  });
+
   it("applies hq-mode metadata via chmod after byte write (Bug #5 — preserve permissions)", async () => {
     // Round-trip pair to the s3.upload test: source-side mode lives in
     // \`Metadata['hq-mode']\` as an octal string; the receiver must chmod