@indigoai-us/hq-cloud 6.11.11 → 6.11.13

package/src/sync/push-receiver.ts

@@ -125,6 +125,39 @@ export const DEFAULT_MAX_MESSAGES = 10;
 export const DEFAULT_RECONNECT_INITIAL_MS = 250;
 export const DEFAULT_RECONNECT_MAX_MS = 30_000;
 
+/** Maximum distinct paths retained in receiver dedupe state. */
+export const DEFAULT_RECEIVER_DEDUPE_MAX_PATHS = 50_000;
+
+class BoundedSequenceDedupe {
+  private readonly maxPaths: number;
+  private readonly seen = new Map<string, number>();
+
+  constructor(maxPaths: number | undefined) {
+    this.maxPaths = Math.max(
+      1,
+      Math.floor(maxPaths ?? DEFAULT_RECEIVER_DEDUPE_MAX_PATHS),
+    );
+  }
+
+  get(relativePath: string): number | undefined {
+    const sequenceNumber = this.seen.get(relativePath);
+    if (sequenceNumber === undefined) return undefined;
+    this.seen.delete(relativePath);
+    this.seen.set(relativePath, sequenceNumber);
+    return sequenceNumber;
+  }
+
+  set(relativePath: string, sequenceNumber: number): void {
+    if (this.seen.has(relativePath)) this.seen.delete(relativePath);
+    this.seen.set(relativePath, sequenceNumber);
+    while (this.seen.size > this.maxPaths) {
+      const oldest = this.seen.keys().next().value;
+      if (oldest === undefined) return;
+      this.seen.delete(oldest);
+    }
+  }
+}
+
 // ─── Narrow SQS surface (the injectable transport seam) ──────────────────────
 
 /**
@@ -191,8 +224,8 @@ export interface PushReceiverContext {
  * company/path; in tests it's a fake recording invocations.
  *
  * Errors from `syncFn` are CAUGHT by the receiver — they log and the loop
- * continues. A misbehaving sync fn cannot crash the receiver. The poll/cadence
- * safety net is the recovery path for a thrown pull.
+ * continues. A failed sync is left unacknowledged in SQS so the queue can
+ * redeliver it after the visibility timeout.
  */
 export type SyncEngineFn = (ctx: PushReceiverContext) => Promise<void>;
 
@@ -321,6 +354,8 @@ export interface SqsPushReceiverOptions {
   maxMessages?: number;
   /** Max time `dispose()` waits for an in-flight syncFn after abort. */
   disposeDrainMs?: number;
+  /** Maximum distinct paths retained in dedupe state. */
+  dedupeMaxPaths?: number;
   /** Reconnect backoff config. */
   reconnect?: {
     initialMs?: number;
@@ -384,7 +419,7 @@ export class SqsPushReceiver implements PushReceiver {
   private inFlightSync: Promise<void> | null = null;
 
   /** Per-path highest sequence number already PROCESSED by syncFn. */
-  private readonly seenSequencePerPath = new Map<string, number>();
+  private readonly seenSequencePerPath: BoundedSequenceDedupe;
 
   private _processedCount = 0;
   private _dedupedCount = 0;
@@ -413,6 +448,7 @@ export class SqsPushReceiver implements PushReceiver {
     this.maxMessages = opts.maxMessages ?? DEFAULT_MAX_MESSAGES;
     this.disposeDrainMs =
       opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
+    this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
     this.reconnectInitialMs =
       opts.reconnect?.initialMs ?? DEFAULT_RECONNECT_INITIAL_MS;
     this.reconnectMaxMs = opts.reconnect?.maxMs ?? DEFAULT_RECONNECT_MAX_MS;
@@ -613,18 +649,18 @@ export class SqsPushReceiver implements PushReceiver {
 
     const handled = await this.dispatch(validated);
     if (handled) {
-      // Delete only after the handoff (success OR dedupe-skip — both mean we
-      // don't need this message again). A syncFn throw still counts as handled
-      // for delete purposes: the seen-counter advanced, so a redelivery would
-      // dedupe; the poll/cadence safety net is the recovery path for the throw.
+      // Delete only after a successful handoff or dedupe-skip. A syncFn throw
+      // leaves the message undeleted so SQS redelivery can retry the targeted
+      // pull.
       await this.safeDelete(msg);
     }
   }
 
   /**
-   * Dedupe + invoke `syncFn`. Returns true once the event has been accounted
-   * for (deduped, or syncFn settled) so the caller can delete the message.
-   * Stores the in-flight promise so `dispose()` can drain it.
+   * Dedupe + invoke `syncFn`. Returns true only once the event has been
+   * accounted for successfully (deduped, or syncFn completed) so the caller can
+   * delete the message. Stores the in-flight promise so `dispose()` can drain
+   * it.
    */
   private async dispatch(event: PushEvent): Promise<boolean> {
     if (this.disposing || this.disposed) return false;
@@ -645,18 +681,13 @@ export class SqsPushReceiver implements PushReceiver {
       return true;
     }
 
-    // Record BEFORE invoking syncFn — a back-to-back event for the same path
-    // must see this sequence in its dedupe check. The trade-off: a syncFn that
-    // throws still advances the counter ("latest we KNOW about"); the safety
-    // net poll is the recovery path for the throw.
-    this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
-
     const controller = new AbortController();
     this.inFlightAbort = controller;
     const ctx: PushReceiverContext = { event, signal: controller.signal };
 
     const holder: { p: Promise<void> | null } = { p: null };
     const startMs = this.now();
+    let handled = false;
     const p: Promise<void> = (async () => {
       try {
         this.logger.debug(
@@ -669,7 +700,9 @@ export class SqsPushReceiver implements PushReceiver {
           "push receiver invoking sync engine",
         );
         await this.syncFn(ctx);
+        this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
         this._processedCount += 1;
+        handled = true;
         this.logger.debug(
           {
             event: "receiver.sync.completed",
@@ -715,8 +748,8 @@ export class SqsPushReceiver implements PushReceiver {
         });
       } catch (err) {
         // Critical: catch, log, return. A misbehaving sync engine must never
-        // crash the receiver loop. The safety-net poll handles eventual
-        // consistency for failed pulls.
+        // crash the receiver loop. Leaving the SQS message undeleted lets
+        // redelivery retry the failed targeted pull.
         const e = err as NodeJS.ErrnoException;
         this.logger.error(
           {
@@ -736,7 +769,7 @@ export class SqsPushReceiver implements PushReceiver {
     holder.p = p;
     this.inFlightSync = p;
     await p;
-    return true;
+    return handled;
   }
 
   /** Delete a message, swallowing transport errors (redelivery is harmless). */
@@ -824,6 +857,8 @@ export interface InMemoryPushReceiverOptions {
   flagProvider?: EventDrivenPushFlagProvider;
   env?: Record<string, string | undefined>;
   disposeDrainMs?: number;
+  /** Maximum distinct paths retained in dedupe state. */
+  dedupeMaxPaths?: number;
 }
 
 /**
@@ -849,7 +884,7 @@ export class InMemoryPushReceiver implements PushReceiver {
 
   private disconnectedFlag = false;
   private readonly pendingDuringDisconnect: PushEvent[] = [];
-  private readonly seenSequencePerPath = new Map<string, number>();
+  private readonly seenSequencePerPath: BoundedSequenceDedupe;
 
   private inFlightAbort: AbortController | null = null;
   private inFlightSync: Promise<void> | null = null;
@@ -865,6 +900,7 @@ export class InMemoryPushReceiver implements PushReceiver {
     this.logger = opts.logger ?? NOOP_LOGGER;
     this.disposeDrainMs =
       opts.disposeDrainMs ?? DEFAULT_RECEIVER_DISPOSE_DRAIN_MS;
+    this.seenSequencePerPath = new BoundedSequenceDedupe(opts.dedupeMaxPaths);
     this.enabled = resolveEnabled({
       explicit: opts.enabled,
       flagProvider: opts.flagProvider,

package/src/sync-core.ts

@@ -0,0 +1,58 @@
+import * as fs from "fs";
+import * as path from "path";
+import { normalizeEtag } from "./journal.js";
+
+/**
+ * Is this error the S3/STS "access denied" class? Expected when a scoped
+ * member/guest credential touches a key outside its granted ACL prefixes
+ * (the server's `SCOPE_EXCEEDS_PARENT` surfaces as a 403 AccessDenied /
+ * Forbidden).
+ */
+export function isAccessDenied(err: unknown): boolean {
+  if (err && typeof err === "object" && "name" in err) {
+    const name = (err as { name?: unknown }).name;
+    return name === "AccessDenied" || name === "Forbidden";
+  }
+  return false;
+}
+
+export function resolveTransferConcurrency(): number {
+  const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
+  if (raw === undefined || raw === "") return 16;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
+}
+
+/**
+ * Resolve active company from .hq/config.json.
+ */
+export function resolveActiveCompany(hqRoot: string): string | undefined {
+  const configPath = path.join(hqRoot, ".hq", "config.json");
+  if (fs.existsSync(configPath)) {
+    try {
+      const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+      return config.activeCompany ?? config.companySlug;
+    } catch {
+      // Ignore parse errors
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Returns true when the remote object appears to have moved since the
+ * journal entry's last-recorded sync. Prefers ETag equality; falls back to
+ * `lastModified > syncedAt` for legacy entries written before remoteEtag
+ * was tracked. Conservative on tie (`<=` skews "remote unchanged") so an
+ * S3-side mtime that exactly equals our syncedAt is not treated as drift.
+ */
+export function hasRemoteChanged(
+  remote: { lastModified: Date; etag: string },
+  entry: { syncedAt: string; remoteEtag?: string },
+): boolean {
+  if (entry.remoteEtag) {
+    return normalizeEtag(remote.etag) !== entry.remoteEtag;
+  }
+  const syncedAt = new Date(entry.syncedAt).getTime();
+  return remote.lastModified.getTime() > syncedAt;
+}

package/src/telemetry.test.ts

@@ -292,6 +292,91 @@ describe("collectAndSendTelemetry", () => {
     expect(lastWire).toBeLessThan(1_000_000);
   });
 
+  it("F19: caps usage telemetry batches at 100 events", async () => {
+    const client = makeClient();
+    const lines = Array.from({ length: 101 }, (_, i) => JSON.stringify({
+      type: "user",
+      timestamp: `2026-04-25T10:${String(Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}Z`,
+      sessionId: "s-f19-row-cap",
+      uuid: `u-f19-row-${i}`,
+      cwd: "/Users/x",
+      gitBranch: "main",
+      userType: "human",
+      message: { role: "user", content: [{ type: "text", text: "hi" }], id: `m${i}` },
+    }));
+    writeJsonl(env, "proj", "f19-row-cap.jsonl", lines);
+
+    const result = await collectAndSendTelemetry(makeOpts(env, client));
+
+    expect(result.eventsSent).toBe(101);
+    expect(result.batchesSent).toBe(2);
+    expect(client.posts.map((post) => post.events.length)).toEqual([100, 1]);
+    for (const post of client.posts) {
+      expect(post.events.length).toBeLessThanOrEqual(100);
+    }
+  });
+
+  it("F19: caps usage telemetry POST bodies at 240 KiB", async () => {
+    const client = makeClient();
+    const branch = "x".repeat(9_000);
+    const lines = Array.from({ length: 30 }, (_, i) => JSON.stringify({
+      type: "user",
+      timestamp: `2026-04-25T11:${String(Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}Z`,
+      sessionId: "s-f19-byte-cap",
+      uuid: `u-f19-byte-${i}`,
+      cwd: "/Users/x",
+      gitBranch: branch,
+      userType: "human",
+      message: { role: "user", content: [{ type: "text", text: "hi" }], id: `m${i}` },
+    }));
+    writeJsonl(env, "proj", "f19-byte-cap.jsonl", lines);
+
+    const result = await collectAndSendTelemetry(makeOpts(env, client));
+    const wireSizes = client.posts.map((post) => Buffer.byteLength(JSON.stringify(post), "utf-8"));
+
+    expect(result.eventsSent).toBe(30);
+    expect(result.batchesSent).toBeGreaterThan(1);
+    for (const wireSize of wireSizes) {
+      expect(wireSize).toBeLessThanOrEqual(240 * 1024);
+    }
+  });
+
+  it("R-F19: bounds a single oversized sanitized usage row before POST", async () => {
+    const client = makeClient();
+    const logs: string[] = [];
+    const hugeBranch = "x".repeat(400_000);
+    writeJsonl(env, "proj", "r-f19-singleton.jsonl", [
+      JSON.stringify({
+        type: "user",
+        timestamp: "2026-06-19T10:00:00.000Z",
+        sessionId: "s-r-f19-singleton",
+        uuid: "u-r-f19-singleton",
+        cwd: "/Users/x",
+        gitBranch: hugeBranch,
+        userType: "human",
+        message: {
+          role: "user",
+          content: [{ type: "text", text: "hi" }],
+          id: "m-r-f19-singleton",
+        },
+      }),
+    ]);
+
+    const result = await collectAndSendTelemetry({
+      ...makeOpts(env, client),
+      log: (msg) => logs.push(msg),
+    });
+
+    expect(result.eventsSent).toBe(1);
+    expect(client.posts).toHaveLength(1);
+    const wireSize = Buffer.byteLength(JSON.stringify(client.posts[0]), "utf-8");
+    expect(wireSize).toBeLessThanOrEqual(240 * 1024);
+    expect(client.posts[0].events[0].gitBranch).not.toBe(hugeBranch);
+    expect(logs.some((line) => line.includes("oversized row truncated"))).toBe(
+      true,
+    );
+  });
+
   it("(e) POST 500 → cursor NOT advanced", async () => {
     const client = makeClient({ postResponse: new Error("server 500") });
     writeJsonl(env, "proj", "s.jsonl", [USER_ROW, ASST_ROW, USER_ROW]);

package/src/telemetry.ts

@@ -11,7 +11,7 @@
  * file against a persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`,
  * sanitizes new rows through a tight allowlist that matches the server's
  * KEEP_FIELDS set in `apps/hq-pro/src/vault-service/handlers/usage.ts`,
- * batches into ≤1 MiB POST bodies, and ships them to `/v1/usage`.
+ * batches into server-sized POST bodies, and ships them to `/v1/usage`.
  *
  * Trust model: the caller's `personUid` is resolved on the server from the
  * Cognito JWT — never from the body. `sanitizeRow` strips prompt bodies,
@@ -245,7 +245,9 @@ async function listJsonlFiles(root: string): Promise<string[]> {
 
 // ── Batching primitives ───────────────────────────────────────────────────────
 
-const MAX_BATCH_BYTES = 1_000_000;
+const MAX_BATCH_EVENTS = 100;
+const MAX_BATCH_BYTES = 240 * 1024;
+const ROW_TRUNCATION_SUFFIX = "...[truncated]";
 
 interface RowSource {
   filePath: string;
@@ -273,6 +275,49 @@ function envelopeBytes(machineId: string, installerVersion: string): number {
   );
 }
 
+function jsonBytes(value: unknown): number {
+  return Buffer.byteLength(JSON.stringify(value), "utf-8");
+}
+
+function truncateLongestStringField(row: Record<string, unknown>): boolean {
+  let longestKey: string | undefined;
+  let longestBytes = 0;
+  for (const [key, value] of Object.entries(row)) {
+    if (typeof value !== "string" || value.length === 0) continue;
+    const bytes = Buffer.byteLength(value, "utf-8");
+    if (bytes > longestBytes) {
+      longestBytes = bytes;
+      longestKey = key;
+    }
+  }
+  if (longestKey === undefined) return false;
+
+  const value = row[longestKey] as string;
+  const keepChars =
+    value.length > ROW_TRUNCATION_SUFFIX.length
+      ? Math.floor((value.length - ROW_TRUNCATION_SUFFIX.length) / 2)
+      : 0;
+  const next =
+    keepChars > 0
+      ? `${value.slice(0, keepChars)}${ROW_TRUNCATION_SUFFIX}`
+      : "";
+  if (next === value) return false;
+  row[longestKey] = next;
+  return true;
+}
+
+function boundRowForPost(
+  row: Record<string, unknown>,
+  maxRowBytes: number,
+): Record<string, unknown> | null {
+  if (maxRowBytes < 0) return null;
+  const bounded = { ...row };
+  while (jsonBytes(bounded) > maxRowBytes) {
+    if (!truncateLongestStringField(bounded)) return null;
+  }
+  return bounded;
+}
+
 // ── Main entry point ──────────────────────────────────────────────────────────
 
 /**
@@ -324,7 +369,7 @@ export async function collectAndSendTelemetry(
 
   const files = await listJsonlFiles(claudeProjectsRoot);
 
-  // 3. Walk each file, sanitize new rows, batch, flush at 1 MiB.
+  // 3. Walk each file, sanitize new rows, batch, flush at the server contract.
   //
   // Byte accounting is incremental: we track `batchBytes` as the projected
   // serialized size of the current batch (envelope + per-row JSON + commas).
@@ -440,15 +485,33 @@ export async function collectAndSendTelemetry(
       );
       const sanitized = sanitizeRow(parsed, companyUid);
       if (!sanitized) continue;
+      const maxRowBytes = MAX_BATCH_BYTES - ENVELOPE_BYTES;
+      const wasOversized = jsonBytes(sanitized) > maxRowBytes;
+      const bounded = boundRowForPost(sanitized, maxRowBytes);
+      if (!bounded) {
+        log(
+          `[telemetry] oversized row dropped before send (${filePath}:${i + 1})`,
+        );
+        continue;
+      }
+      if (wasOversized) {
+        log(
+          `[telemetry] oversized row truncated before send (${filePath}:${i + 1})`,
+        );
+      }
 
       // Cost of appending this row to the current batch: the row's JSON
       // length plus 1 byte for the leading comma when there's already at
       // least one row. (No comma when the batch is empty — the row sits
       // alone inside the events array.)
-      const rowJsonBytes = Buffer.byteLength(JSON.stringify(sanitized), "utf-8");
+      const rowJsonBytes = Buffer.byteLength(JSON.stringify(bounded), "utf-8");
       const addCost = rowJsonBytes + (batchEvents.length > 0 ? 1 : 0);
 
-      if (batchEvents.length > 0 && batchBytes + addCost > MAX_BATCH_BYTES) {
+      if (
+        batchEvents.length > 0 &&
+        (batchEvents.length >= MAX_BATCH_EVENTS ||
+          batchBytes + addCost > MAX_BATCH_BYTES)
+      ) {
         await flush();
         // After flush, batchEvents is empty → no comma needed for the first row.
         batchBytes = ENVELOPE_BYTES + rowJsonBytes;
@@ -456,7 +519,7 @@ export async function collectAndSendTelemetry(
         batchBytes += addCost;
       }
 
-      batchEvents.push(sanitized);
+      batchEvents.push(bounded);
       batchSources.push({
         filePath,
         endOffset: lineEndOffsets[i],

package/src/types.ts

@@ -72,6 +72,14 @@ export interface JournalEntry {
    */
   removedAt?: string;
   removedReason?: "scope_shrink" | "narrow_apply" | "manual";
+  /**
+   * Durable automatic-pull retention marker. Set when a scope shrink keeps an
+   * out-of-scope caller-authored or unknown-author entry on disk instead of
+   * pruning it. Subsequent pulls under the already-narrowed scope use this to
+   * exclude the survivor from remote-missing local deletes; it is cleared if
+   * the entry becomes in-scope again.
+   */
+  outOfScopeProtected?: boolean;
 }
 
 /**

package/src/vault-client.test.ts

@@ -440,6 +440,80 @@ describe("API surface", () => {
   });
 });
 
+describe("response validation (F26)", () => {
+  it("F26: validates a valid files/list response and returns typed data", async () => {
+    fetchSpy.mockResolvedValueOnce(
+      jsonResponse(200, {
+        objects: [
+          {
+            key: "shared/docs/a.txt",
+            size: 123,
+            lastModified: "2026-05-20T12:00:00.000Z",
+            etag: "abc123",
+            permission: "read",
+          },
+        ],
+        cursor: null,
+        truncated: false,
+      }),
+    );
+
+    const page = await client.listFiles("cmp_abc", "shared/docs/");
+
+    expect(page.objects[0]).toEqual({
+      key: "shared/docs/a.txt",
+      size: 123,
+      lastModified: "2026-05-20T12:00:00.000Z",
+      etag: "abc123",
+      permission: "read",
+    });
+    expect(page.cursor).toBeNull();
+    expect(page.truncated).toBe(false);
+  });
+
+  it("F26: defaults omitted files/list paging fields", async () => {
+    fetchSpy.mockResolvedValueOnce(jsonResponse(200, {}));
+
+    const page = await client.listFiles("cmp_abc", "shared/docs/");
+
+    expect(page).toEqual({
+      objects: [],
+      cursor: null,
+      truncated: false,
+    });
+  });
+
+  it("F26: defaults omitted telemetry opt-in fields to disabled", async () => {
+    fetchSpy.mockResolvedValueOnce(jsonResponse(200, {}));
+
+    const optIn = await client.getTelemetryOptIn();
+
+    expect(optIn).toEqual({ enabled: false, updatedAt: null });
+  });
+
+  it("F26: maps a malformed files/list response to VaultClientError", async () => {
+    fetchSpy.mockResolvedValueOnce(
+      jsonResponse(200, {
+        objects: [
+          {
+            key: "shared/docs/a.txt",
+            size: 123,
+            lastModified: "2026-05-20T12:00:00.000Z",
+            permission: "read",
+          },
+        ],
+        cursor: null,
+        truncated: false,
+      }),
+    );
+
+    await expect(client.listFiles("cmp_abc")).rejects.toMatchObject({
+      name: "VaultClientError",
+      statusCode: 502,
+    });
+  });
+});
+
 describe("VaultClient identity bootstrap", () => {
   let client: VaultClient;
   let fetchSpy: MockInstance<typeof fetch>;