@indigoai-us/hq-cloud 6.11.12 → 6.11.13

package/src/daemon-worker.ts

@@ -4,11 +4,11 @@
  *
  * Day 1: not invoked by CLI surface; retained for future automatic-sync milestone.
  * When re-enabled, this worker will need to resolve an EntityContext before
- * constructing the SyncWatcher. The process argv will need to include company
- * context (slug or UID) and vault-service config.
+ * constructing the active watcher/runner path. The process argv will need to
+ * include company context (slug or UID) and vault-service config.
  */
 
-// Day 1: SyncWatcher now requires an EntityContext.
+// Day 1: this worker still requires EntityContext-aware startup wiring.
 // This file is retained for the automatic-sync milestone but is not functional
 // until the daemon startup path is updated to resolve entity context.
 

package/src/object-io.ts

@@ -540,7 +540,7 @@ export class PresignObjectIO implements ObjectIO {
   ) {}
 
   private cacheKey(op: PresignOp, key: string): string {
-    return `${op}${key}`;
+    return `${op}\u0000${key}`;
   }
 
   hasPrimedPut(key: string): boolean {

package/src/remote-pull.test.ts

@@ -1,7 +1,7 @@
 /**
  * Failing-test seed for the Auto-sync (Beta) remote-pull loop.
  *
- * Background: SyncWatcher (watcher.ts) pushes local edits to S3 in seconds,
+ * Background: the watcher push path ships local edits to S3 in seconds,
  * but pulls happen only on a manual sync today. Auto-sync adds a periodic
  * (every 10 min) remote-pull pass per company. The decision of *which* keys
  * to download / delete locally / skip is pure given a remote listing, the
@@ -25,6 +25,7 @@ import {
   POST_FILTER_THRESHOLD,
   pullCompany,
   resolveCompanyScope,
+  VEND_FANOUT_CONCURRENCY,
   VEND_PATH_CAP,
 } from "./remote-pull.js";
 import type { RemoteFile } from "./s3.js";
@@ -579,6 +580,34 @@ describe("listRemoteForScope", () => {
     expect(calls.size).toBe(VEND_PATH_CAP + 3); // every prefix listed
   });
 
+  it("F28: bounds concurrent per-prefix listings across vend-fanout batches", async () => {
+    const prefixes = Array.from(
+      { length: POST_FILTER_THRESHOLD },
+      (_, i) => `companies/indigo/p${i}/`,
+    );
+    let active = 0;
+    let maxActive = 0;
+
+    await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: prefixes,
+        strategy: "vend-fanout",
+      },
+      listFn: async () => {
+        active += 1;
+        maxActive = Math.max(maxActive, active);
+        await new Promise((resolve) => setTimeout(resolve, 1));
+        active -= 1;
+        return [];
+      },
+    });
+
+    expect(maxActive).toBeLessThanOrEqual(VEND_FANOUT_CONCURRENCY);
+  });
+
   it("strategy=vend-fanout uses vendForBatchFn to narrow credentials per batch", async () => {
     const vendCalls: Array<{ paths: string[] }> = [];
     await listRemoteForScope({

package/src/remote-pull.ts

@@ -8,7 +8,7 @@
  * sync-runner.ts trivial: list S3 → call `decideRemotePulls` → drive S3 +
  * filesystem from the result.
  *
- * Pairs with `SyncWatcher` (push-side) — together they implement the
+ * Pairs with the TreeWatcher push path — together they implement the
  * bidirectional auto-sync the Settings toggle exposes.
  */
 import type { RemoteFile } from "./s3.js";
@@ -160,7 +160,7 @@ export const VEND_PATH_CAP = 10;
  */
 export const POST_FILTER_THRESHOLD = 50;
 
-/** Bounded parallelism for vend fan-out (5 concurrent STS+ListObjectsV2 calls). */
+/** Bounded parallelism for vend fan-out (5 concurrent vends/list paginators). */
 export const VEND_FANOUT_CONCURRENCY = 5;
 
 /**
@@ -291,6 +291,27 @@ async function mapWithConcurrency<T, R>(
   return results;
 }
 
+function createConcurrencyLimiter(
+  concurrency: number,
+): <R>(fn: () => Promise<R>) => Promise<R> {
+  const limit = Math.max(1, concurrency);
+  let active = 0;
+  const waiters: Array<() => void> = [];
+
+  return async function runLimited<R>(fn: () => Promise<R>): Promise<R> {
+    if (active >= limit) {
+      await new Promise<void>((resolve) => waiters.push(resolve));
+    }
+    active += 1;
+    try {
+      return await fn();
+    } finally {
+      active -= 1;
+      waiters.shift()?.();
+    }
+  };
+}
+
 export interface ListRemoteForScopeInput {
   ctx: EntityContext;
   scope: CompanyScope;
@@ -347,6 +368,7 @@ export async function listRemoteForScope(
   const scopeEntries = toScopePrefixEntries(scope.prefixSet);
   const listPrefixes = scopeEntries.map((entry) => entry.prefix);
   const batches = batchPrefixesForVend(listPrefixes);
+  const listWithLimit = createConcurrencyLimiter(VEND_FANOUT_CONCURRENCY);
   const perBatch = await mapWithConcurrency(
     batches,
     VEND_FANOUT_CONCURRENCY,
@@ -357,8 +379,11 @@ export async function listRemoteForScope(
       // For a coalesced batch we issue one ListObjectsV2 per prefix in the
       // batch. We can't issue one ListObjectsV2 across N prefixes (the API
       // takes a single Prefix); the per-batch grouping exists for the STS
-      // session policy ceiling, not the list call itself.
-      const lists = await Promise.all(paths.map((p) => list(batchCtx, p)));
+      // session policy ceiling, not the list call itself. The shared limiter
+      // keeps the total active prefix paginators bounded across all batches.
+      const lists = await Promise.all(
+        paths.map((p) => listWithLimit(() => list(batchCtx, p))),
+      );
       return lists.flat();
     },
   );

package/src/sync/push-receiver.test.ts

@@ -396,6 +396,41 @@ describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
     expect(sqs.deleted).not.toContain("rh-first-attempt");
     expect(sqs.deleted).toContain("rh-redelivery");
   });
+
+  it("F29: bounds dedupe state by evicting least-recently-used paths", async () => {
+    const sqs = new FakeSqs();
+    const processed: string[] = [];
+    const syncFn: SyncEngineFn = async (ctx) => {
+      processed.push(ctx.event.relativePath);
+    };
+
+    sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/a.md", sequenceNumber: 1 }), "rh-a1")]);
+    sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/b.md", sequenceNumber: 1 }), "rh-b1")]);
+    sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/c.md", sequenceNumber: 1 }), "rh-c1")]);
+    sqs.enqueue([msg(makeEvent({ relativePath: "companies/indigo/a.md", sequenceNumber: 1 }), "rh-a-redelivered")]);
+
+    const receiver = new SqsPushReceiver({
+      tenantId: TENANT,
+      queueUrl: QUEUE_URL,
+      sqs,
+      syncFn,
+      enabled: true,
+      sleep: fastSleep,
+      dedupeMaxPaths: 2,
+    });
+    await receiver.start();
+    await waitFor(() => receiver.processedCount === 4);
+    await receiver.dispose();
+
+    expect(processed).toEqual([
+      "companies/indigo/a.md",
+      "companies/indigo/b.md",
+      "companies/indigo/c.md",
+      "companies/indigo/a.md",
+    ]);
+    expect(receiver.dedupedCount).toBe(0);
+    expect(sqs.deleted).toContain("rh-a-redelivered");
+  });
 });
 
 describe("US-009: SqsPushReceiver — flag gating", () => {

package/src/sync/push-receiver.ts

@@ -125,6 +125,39 @@ export const DEFAULT_MAX_MESSAGES = 10;
 export const DEFAULT_RECONNECT_INITIAL_MS = 250;
 export const DEFAULT_RECONNECT_MAX_MS = 30_000;
 
+/** Maximum distinct paths retained in receiver dedupe state. */
+export const DEFAULT_RECEIVER_DEDUPE_MAX_PATHS = 50_000;
+
+class BoundedSequenceDedupe {
+  private readonly maxPaths: number;
+  private readonly seen = new Map<string, number>();
+
+  constructor(maxPaths: number | undefined) {
+    this.maxPaths = Math.max(
+      1,
+      Math.floor(maxPaths ?? DEFAULT_RECEIVER_DEDUPE_MAX_PATHS),
+    );
+  }
+
+  get(relativePath: string): number | undefined {
+    const sequenceNumber = this.seen.get(relativePath);
+    if (sequenceNumber === undefined) return undefined;
+    this.seen.delete(relativePath);
+    this.seen.set(relativePath, sequenceNumber);
+    return sequenceNumber;
+  }
+
+  set(relativePath: string, sequenceNumber: number): void {
+    if (this.seen.has(relativePath)) this.seen.delete(relativePath);
+    this.seen.set(relativePath, sequenceNumber);
+    while (this.seen.size > this.maxPaths) {
+      const oldest = this.seen.keys().next().value;
+      if (oldest === undefined) return;
+      this.seen.delete(oldest);
+    }
+  }
+}
+
 // ─── Narrow SQS surface (the injectable transport seam) ──────────────────────
 
 /**
@@ -321,6 +354,8 @@ export interface SqsPushReceiverOptions {
   maxMessages?: number;
   /** Max time `dispose()` waits for an in-flight syncFn after abort. */
   disposeDrainMs?: number;
+  /** Maximum distinct paths retained in dedupe state. */
+  dedupeMaxPaths?: number;
   /** Reconnect backoff config. */
   reconnect?: {
     initialMs?: number;
@@ -384,7 +419,7 @@ export class SqsPushReceiver implements PushReceiver {
   private inFlightSync: Promise<void> | null = null;
 
   /** Per-path highest sequence number already PROCESSED by syncFn. */
-  private readonly seenSequencePerPath = new Map<string, number>();
+  private readonly seenSequencePerPath: BoundedSequenceDedupe;
 
   private _processedCount = 0;
   private _dedupedCount = 0;
@@ -413,6 +448,7 @@ export class SqsPushReceiver implements PushReceiver {
     this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
     this.disposeDrainMs =
       opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
+    this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
     this.reconnectInitialMs =
       opts.reconnect?.initialMs ?? DEFAULT_RECONNECT_INITIAL_MS;
     this.reconnectMaxMs = opts.reconnect?.maxMs ?? DEFAULT_RECONNECT_MAX_MS;
@@ -821,6 +857,8 @@ export interface InMemoryPushReceiverOptions {
   flagProvider?: EventDrivenPushFlagProvider;
   env?: Record<string, string | undefined>;
   disposeDrainMs?: number;
+  /** Maximum distinct paths retained in dedupe state. */
+  dedupeMaxPaths?: number;
 }
 
 /**
@@ -846,7 +884,7 @@ export class InMemoryPushReceiver implements PushReceiver {
 
   private disconnectedFlag = false;
   private readonly pendingDuringDisconnect: PushEvent[] = [];
-  private readonly seenSequencePerPath = new Map<string, number>();
+  private readonly seenSequencePerPath: BoundedSequenceDedupe;
 
   private inFlightAbort: AbortController | null = null;
   private inFlightSync: Promise<void> | null = null;
@@ -862,6 +900,7 @@ export class InMemoryPushReceiver implements PushReceiver {
     this.logger = opts.logger ?? NOOP_LOGGER;
     this.disposeDrainMs =
       opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
+    this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
     this.enabled = resolveEnabled({
       explicit: opts.enabled,
       flagProvider: opts.flagProvider,

package/src/sync-core.ts

@@ -0,0 +1,58 @@
+import * as fs from "fs";
+import * as path from "path";
+import { normalizeEtag } from "./journal.js";
+
+/**
+ * Is this error the S3/STS "access denied" class? Expected when a scoped
+ * member/guest credential touches a key outside its granted ACL prefixes
+ * (the server's `SCOPE_EXCEEDS_PARENT` surfaces as a 403 AccessDenied /
+ * Forbidden).
+ */
+export function isAccessDenied(err: unknown): boolean {
+  if (err && typeof err === "object" && "name" in err) {
+    const name = (err as { name?: unknown }).name;
+    return name === "AccessDenied" || name === "Forbidden";
+  }
+  return false;
+}
+
+export function resolveTransferConcurrency(): number {
+  const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
+  if (raw === undefined || raw === "") return 16;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
+}
+
+/**
+ * Resolve active company from .hq/config.json.
+ */
+export function resolveActiveCompany(hqRoot: string): string | undefined {
+  const configPath = path.join(hqRoot, ".hq", "config.json");
+  if (fs.existsSync(configPath)) {
+    try {
+      const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+      return config.activeCompany ?? config.companySlug;
+    } catch {
+      // Ignore parse errors
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Returns true when the remote object appears to have moved since the
+ * journal entry's last-recorded sync. Prefers ETag equality; falls back to
+ * `lastModified > syncedAt` for legacy entries written before remoteEtag
+ * was tracked. Conservative on tie (`<=` skews "remote unchanged") so an
+ * S3-side mtime that exactly equals our syncedAt is not treated as drift.
+ */
+export function hasRemoteChanged(
+  remote: { lastModified: Date; etag: string },
+  entry: { syncedAt: string; remoteEtag?: string },
+): boolean {
+  if (entry.remoteEtag) {
+    return normalizeEtag(remote.etag) !== entry.remoteEtag;
+  }
+  const syncedAt = new Date(entry.syncedAt).getTime();
+  return remote.lastModified.getTime() > syncedAt;
+}

package/src/vault-client.test.ts

@@ -440,6 +440,80 @@ describe("API surface", () => {
   });
 });
 
+describe("response validation (F26)", () => {
+  it("F26: validates a valid files/list response and returns typed data", async () => {
+    fetchSpy.mockResolvedValueOnce(
+      jsonResponse(200, {
+        objects: [
+          {
+            key: "shared/docs/a.txt",
+            size: 123,
+            lastModified: "2026-05-20T12:00:00.000Z",
+            etag: "abc123",
+            permission: "read",
+          },
+        ],
+        cursor: null,
+        truncated: false,
+      }),
+    );
+
+    const page = await client.listFiles("cmp_abc", "shared/docs/");
+
+    expect(page.objects[0]).toEqual({
+      key: "shared/docs/a.txt",
+      size: 123,
+      lastModified: "2026-05-20T12:00:00.000Z",
+      etag: "abc123",
+      permission: "read",
+    });
+    expect(page.cursor).toBeNull();
+    expect(page.truncated).toBe(false);
+  });
+
+  it("F26: defaults omitted files/list paging fields", async () => {
+    fetchSpy.mockResolvedValueOnce(jsonResponse(200, {}));
+
+    const page = await client.listFiles("cmp_abc", "shared/docs/");
+
+    expect(page).toEqual({
+      objects: [],
+      cursor: null,
+      truncated: false,
+    });
+  });
+
+  it("F26: defaults omitted telemetry opt-in fields to disabled", async () => {
+    fetchSpy.mockResolvedValueOnce(jsonResponse(200, {}));
+
+    const optIn = await client.getTelemetryOptIn();
+
+    expect(optIn).toEqual({ enabled: false, updatedAt: null });
+  });
+
+  it("F26: maps a malformed files/list response to VaultClientError", async () => {
+    fetchSpy.mockResolvedValueOnce(
+      jsonResponse(200, {
+        objects: [
+          {
+            key: "shared/docs/a.txt",
+            size: 123,
+            lastModified: "2026-05-20T12:00:00.000Z",
+            permission: "read",
+          },
+        ],
+        cursor: null,
+        truncated: false,
+      }),
+    );
+
+    await expect(client.listFiles("cmp_abc")).rejects.toMatchObject({
+      name: "VaultClientError",
+      statusCode: 502,
+    });
+  });
+});
+
 describe("VaultClient identity bootstrap", () => {
   let client: VaultClient;
   let fetchSpy: MockInstance<typeof fetch>;