@indigoai-us/hq-cloud 6.11.11 → 6.11.12

package/src/sync/push-receiver.ts

@@ -191,8 +191,8 @@ export interface PushReceiverContext {
  * company/path; in tests it's a fake recording invocations.
  *
  * Errors from `syncFn` are CAUGHT by the receiver — they log and the loop
- * continues. A misbehaving sync fn cannot crash the receiver. The poll/cadence
- * safety net is the recovery path for a thrown pull.
+ * continues. A failed sync is left unacknowledged in SQS so the queue can
+ * redeliver it after the visibility timeout.
  */
 export type SyncEngineFn = (ctx: PushReceiverContext) => Promise<void>;
 
@@ -613,18 +613,18 @@ export class SqsPushReceiver implements PushReceiver {
 
     const handled = await this.dispatch(validated);
     if (handled) {
-      // Delete only after the handoff (success OR dedupe-skip — both mean we
-      // don't need this message again). A syncFn throw still counts as handled
-      // for delete purposes: the seen-counter advanced, so a redelivery would
-      // dedupe; the poll/cadence safety net is the recovery path for the throw.
+      // Delete only after a successful handoff or dedupe-skip. A syncFn throw
+      // leaves the message undeleted so SQS redelivery can retry the targeted
+      // pull.
       await this.safeDelete(msg);
     }
   }
 
   /**
-   * Dedupe + invoke `syncFn`. Returns true once the event has been accounted
-   * for (deduped, or syncFn settled) so the caller can delete the message.
-   * Stores the in-flight promise so `dispose()` can drain it.
+   * Dedupe + invoke `syncFn`. Returns true only once the event has been
+   * accounted for successfully (deduped, or syncFn completed) so the caller can
+   * delete the message. Stores the in-flight promise so `dispose()` can drain
+   * it.
    */
   private async dispatch(event: PushEvent): Promise<boolean> {
     if (this.disposing || this.disposed) return false;
@@ -645,18 +645,13 @@ export class SqsPushReceiver implements PushReceiver {
       return true;
     }
 
-    // Record BEFORE invoking syncFn — a back-to-back event for the same path
-    // must see this sequence in its dedupe check. The trade-off: a syncFn that
-    // throws still advances the counter ("latest we KNOW about"); the safety
-    // net poll is the recovery path for the throw.
-    this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
-
     const controller = new AbortController();
     this.inFlightAbort = controller;
     const ctx: PushReceiverContext = { event, signal: controller.signal };
 
     const holder: { p: Promise<void> | null } = { p: null };
     const startMs = this.now();
+    let handled = false;
     const p: Promise<void> = (async () => {
       try {
         this.logger.debug(
@@ -669,7 +664,9 @@ export class SqsPushReceiver implements PushReceiver {
           "push receiver invoking sync engine",
         );
         await this.syncFn(ctx);
+        this.seenSequencePerPath.set(event.relativePath, event.sequenceNumber);
         this._processedCount += 1;
+        handled = true;
         this.logger.debug(
           {
             event: "receiver.sync.completed",
@@ -715,8 +712,8 @@ export class SqsPushReceiver implements PushReceiver {
         });
       } catch (err) {
         // Critical: catch, log, return. A misbehaving sync engine must never
-        // crash the receiver loop. The safety-net poll handles eventual
-        // consistency for failed pulls.
+        // crash the receiver loop. Leaving the SQS message undeleted lets
+        // redelivery retry the failed targeted pull.
         const e = err as NodeJS.ErrnoException;
         this.logger.error(
           {
@@ -736,7 +733,7 @@ export class SqsPushReceiver implements PushReceiver {
     holder.p = p;
     this.inFlightSync = p;
     await p;
-    return true;
+    return handled;
   }
 
   /** Delete a message, swallowing transport errors (redelivery is harmless). */

package/src/telemetry.test.ts

@@ -292,6 +292,91 @@ describe("collectAndSendTelemetry", () => {
     expect(lastWire).toBeLessThan(1_000_000);
   });
 
+  it("F19: caps usage telemetry batches at 100 events", async () => {
+    const client = makeClient();
+    const lines = Array.from({ length: 101 }, (_, i) => JSON.stringify({
+      type: "user",
+      timestamp: `2026-04-25T10:${String(Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}Z`,
+      sessionId: "s-f19-row-cap",
+      uuid: `u-f19-row-${i}`,
+      cwd: "/Users/x",
+      gitBranch: "main",
+      userType: "human",
+      message: { role: "user", content: [{ type: "text", text: "hi" }], id: `m${i}` },
+    }));
+    writeJsonl(env, "proj", "f19-row-cap.jsonl", lines);
+
+    const result = await collectAndSendTelemetry(makeOpts(env, client));
+
+    expect(result.eventsSent).toBe(101);
+    expect(result.batchesSent).toBe(2);
+    expect(client.posts.map((post) => post.events.length)).toEqual([100, 1]);
+    for (const post of client.posts) {
+      expect(post.events.length).toBeLessThanOrEqual(100);
+    }
+  });
+
+  it("F19: caps usage telemetry POST bodies at 240 KiB", async () => {
+    const client = makeClient();
+    const branch = "x".repeat(9_000);
+    const lines = Array.from({ length: 30 }, (_, i) => JSON.stringify({
+      type: "user",
+      timestamp: `2026-04-25T11:${String(Math.floor(i / 60)).padStart(2, "0")}:${String(i % 60).padStart(2, "0")}Z`,
+      sessionId: "s-f19-byte-cap",
+      uuid: `u-f19-byte-${i}`,
+      cwd: "/Users/x",
+      gitBranch: branch,
+      userType: "human",
+      message: { role: "user", content: [{ type: "text", text: "hi" }], id: `m${i}` },
+    }));
+    writeJsonl(env, "proj", "f19-byte-cap.jsonl", lines);
+
+    const result = await collectAndSendTelemetry(makeOpts(env, client));
+    const wireSizes = client.posts.map((post) => Buffer.byteLength(JSON.stringify(post), "utf-8"));
+
+    expect(result.eventsSent).toBe(30);
+    expect(result.batchesSent).toBeGreaterThan(1);
+    for (const wireSize of wireSizes) {
+      expect(wireSize).toBeLessThanOrEqual(240 * 1024);
+    }
+  });
+
+  it("R-F19: bounds a single oversized sanitized usage row before POST", async () => {
+    const client = makeClient();
+    const logs: string[] = [];
+    const hugeBranch = "x".repeat(400_000);
+    writeJsonl(env, "proj", "r-f19-singleton.jsonl", [
+      JSON.stringify({
+        type: "user",
+        timestamp: "2026-06-19T10:00:00.000Z",
+        sessionId: "s-r-f19-singleton",
+        uuid: "u-r-f19-singleton",
+        cwd: "/Users/x",
+        gitBranch: hugeBranch,
+        userType: "human",
+        message: {
+          role: "user",
+          content: [{ type: "text", text: "hi" }],
+          id: "m-r-f19-singleton",
+        },
+      }),
+    ]);
+
+    const result = await collectAndSendTelemetry({
+      ...makeOpts(env, client),
+      log: (msg) => logs.push(msg),
+    });
+
+    expect(result.eventsSent).toBe(1);
+    expect(client.posts).toHaveLength(1);
+    const wireSize = Buffer.byteLength(JSON.stringify(client.posts[0]), "utf-8");
+    expect(wireSize).toBeLessThanOrEqual(240 * 1024);
+    expect(client.posts[0].events[0].gitBranch).not.toBe(hugeBranch);
+    expect(logs.some((line) => line.includes("oversized row truncated"))).toBe(
+      true,
+    );
+  });
+
   it("(e) POST 500 → cursor NOT advanced", async () => {
     const client = makeClient({ postResponse: new Error("server 500") });
     writeJsonl(env, "proj", "s.jsonl", [USER_ROW, ASST_ROW, USER_ROW]);

package/src/telemetry.ts

@@ -11,7 +11,7 @@
  * file against a persisted byte-offset cursor at `~/.hq/telemetry-cursor.json`,
  * sanitizes new rows through a tight allowlist that matches the server's
  * KEEP_FIELDS set in `apps/hq-pro/src/vault-service/handlers/usage.ts`,
- * batches into ≤1 MiB POST bodies, and ships them to `/v1/usage`.
+ * batches into server-sized POST bodies, and ships them to `/v1/usage`.
  *
  * Trust model: the caller's `personUid` is resolved on the server from the
  * Cognito JWT — never from the body. `sanitizeRow` strips prompt bodies,
@@ -245,7 +245,9 @@ async function listJsonlFiles(root: string): Promise<string[]> {
 
 // ── Batching primitives ───────────────────────────────────────────────────────
 
-const MAX_BATCH_BYTES = 1_000_000;
+const MAX_BATCH_EVENTS = 100;
+const MAX_BATCH_BYTES = 240 * 1024;
+const ROW_TRUNCATION_SUFFIX = "...[truncated]";
 
 interface RowSource {
   filePath: string;
@@ -273,6 +275,49 @@ function envelopeBytes(machineId: string, installerVersion: string): number {
   );
 }
 
+function jsonBytes(value: unknown): number {
+  return Buffer.byteLength(JSON.stringify(value), "utf-8");
+}
+
+function truncateLongestStringField(row: Record<string, unknown>): boolean {
+  let longestKey: string | undefined;
+  let longestBytes = 0;
+  for (const [key, value] of Object.entries(row)) {
+    if (typeof value !== "string" || value.length === 0) continue;
+    const bytes = Buffer.byteLength(value, "utf-8");
+    if (bytes > longestBytes) {
+      longestBytes = bytes;
+      longestKey = key;
+    }
+  }
+  if (longestKey === undefined) return false;
+
+  const value = row[longestKey] as string;
+  const keepChars =
+    value.length > ROW_TRUNCATION_SUFFIX.length
+      ? Math.floor((value.length - ROW_TRUNCATION_SUFFIX.length) / 2)
+      : 0;
+  const next =
+    keepChars > 0
+      ? `${value.slice(0, keepChars)}${ROW_TRUNCATION_SUFFIX}`
+      : "";
+  if (next === value) return false;
+  row[longestKey] = next;
+  return true;
+}
+
+function boundRowForPost(
+  row: Record<string, unknown>,
+  maxRowBytes: number,
+): Record<string, unknown> | null {
+  if (maxRowBytes < 0) return null;
+  const bounded = { ...row };
+  while (jsonBytes(bounded) > maxRowBytes) {
+    if (!truncateLongestStringField(bounded)) return null;
+  }
+  return bounded;
+}
+
 // ── Main entry point ──────────────────────────────────────────────────────────
 
 /**
@@ -324,7 +369,7 @@ export async function collectAndSendTelemetry(
 
   const files = await listJsonlFiles(claudeProjectsRoot);
 
-  // 3. Walk each file, sanitize new rows, batch, flush at 1 MiB.
+  // 3. Walk each file, sanitize new rows, batch, flush at the server contract.
   //
   // Byte accounting is incremental: we track `batchBytes` as the projected
   // serialized size of the current batch (envelope + per-row JSON + commas).
@@ -440,15 +485,33 @@ export async function collectAndSendTelemetry(
       );
       const sanitized = sanitizeRow(parsed, companyUid);
       if (!sanitized) continue;
+      const maxRowBytes = MAX_BATCH_BYTES - ENVELOPE_BYTES;
+      const wasOversized = jsonBytes(sanitized) > maxRowBytes;
+      const bounded = boundRowForPost(sanitized, maxRowBytes);
+      if (!bounded) {
+        log(
+          `[telemetry] oversized row dropped before send (${filePath}:${i + 1})`,
+        );
+        continue;
+      }
+      if (wasOversized) {
+        log(
+          `[telemetry] oversized row truncated before send (${filePath}:${i + 1})`,
+        );
+      }
 
       // Cost of appending this row to the current batch: the row's JSON
       // length plus 1 byte for the leading comma when there's already at
       // least one row. (No comma when the batch is empty — the row sits
       // alone inside the events array.)
-      const rowJsonBytes = Buffer.byteLength(JSON.stringify(sanitized), "utf-8");
+      const rowJsonBytes = Buffer.byteLength(JSON.stringify(bounded), "utf-8");
       const addCost = rowJsonBytes + (batchEvents.length > 0 ? 1 : 0);
 
-      if (batchEvents.length > 0 && batchBytes + addCost > MAX_BATCH_BYTES) {
+      if (
+        batchEvents.length > 0 &&
+        (batchEvents.length >= MAX_BATCH_EVENTS ||
+          batchBytes + addCost > MAX_BATCH_BYTES)
+      ) {
         await flush();
         // After flush, batchEvents is empty → no comma needed for the first row.
         batchBytes = ENVELOPE_BYTES + rowJsonBytes;
@@ -456,7 +519,7 @@ export async function collectAndSendTelemetry(
         batchBytes += addCost;
       }
 
-      batchEvents.push(sanitized);
+      batchEvents.push(bounded);
       batchSources.push({
         filePath,
         endOffset: lineEndOffsets[i],

package/src/types.ts

@@ -72,6 +72,14 @@ export interface JournalEntry {
    */
   removedAt?: string;
   removedReason?: "scope_shrink" | "narrow_apply" | "manual";
+  /**
+   * Durable automatic-pull retention marker. Set when a scope shrink keeps an
+   * out-of-scope caller-authored or unknown-author entry on disk instead of
+   * pruning it. Subsequent pulls under the already-narrowed scope use this to
+   * exclude the survivor from remote-missing local deletes; it is cleared if
+   * the entry becomes in-scope again.
+   */
+  outOfScopeProtected?: boolean;
 }
 
 /**

package/src/watcher.test.ts

@@ -49,6 +49,10 @@ function makeHarness(opts?: { debounceMs?: number }) {
   };
 }
 
+async function flushImmediate(): Promise<void> {
+  await new Promise<void>((resolve) => setImmediate(resolve));
+}
+
 describe("FakeClock", () => {
   it("fires a timer exactly when its deadline is reached", () => {
     const clock = new FakeClock();
@@ -166,6 +170,34 @@ describe("US-001: WatchPushDriver — debounced push seam", () => {
     driver.dispose();
   });
 
+  it("F17: rejected watch push is caught instead of surfacing as an unhandled rejection", async () => {
+    const clock = new FakeClock();
+    const rejection = new Error("push failed");
+    const push = vi.fn(async () => {
+      throw rejection;
+    });
+    const driver = new WatchPushDriver({ debounceMs: DEBOUNCE, clock, push });
+    const unhandled: unknown[] = [];
+    const onUnhandled = (reason: unknown) => {
+      unhandled.push(reason);
+    };
+
+    process.prependListener("unhandledRejection", onUnhandled);
+    try {
+      driver.notifyChange();
+      clock.advance(DEBOUNCE);
+      await Promise.resolve();
+      await flushImmediate();
+
+      expect(push).toHaveBeenCalledTimes(1);
+      expect(driver.isPushing()).toBe(false);
+      expect(unhandled).toEqual([]);
+    } finally {
+      process.removeListener("unhandledRejection", onUnhandled);
+      driver.dispose();
+    }
+  });
+
   it("respects a custom debounce window", async () => {
     const h = makeHarness({ debounceMs: 500 });
     h.emitChange();
@@ -374,6 +406,42 @@ describe("US-002: TreeWatcher — debounce coalesce (FakeClock seam)", () => {
     clock.advance(DEBOUNCE);
     expect(changed).toHaveBeenCalledTimes(2);
   });
+
+  it("R-F13: caps watcher backlog and keeps EMFILE polling fallback removed", () => {
+    const clock = new FakeClock();
+    const changed = vi.fn();
+    const overflow = vi.fn();
+    const watcher = new TreeWatcher({
+      hqRoot: ROOT,
+      debounceMs: DEBOUNCE,
+      clock,
+      pathFilter: () => true,
+      maxPendingPaths: 3,
+      maxPendingBytes: 10_000,
+      onBacklogOverflow: overflow,
+    });
+    watcher.onChange(changed);
+
+    for (let i = 0; i < 10; i++) {
+      watcher.handleEvent(path.join(ROOT, `bulk-${i}.md`));
+    }
+
+    expect(overflow).toHaveBeenCalledTimes(1);
+    clock.advance(DEBOUNCE);
+    expect(changed).toHaveBeenCalledTimes(1);
+    const batch = changed.mock.calls[0][1] as {
+      paths: Map<string, string>;
+      overflowed?: boolean;
+      droppedPaths?: number;
+    };
+    expect(batch.paths.size).toBe(3);
+    expect(batch.overflowed).toBe(true);
+    expect(batch.droppedPaths).toBe(7);
+
+    const source = fs.readFileSync(path.join(process.cwd(), "src/watcher.ts"), "utf8");
+    expect(source).not.toContain("startPollingTreeWatch");
+    expect(source).not.toContain("snapshotWatchTree");
+  });
 });
 
 describe("US-002: TreeWatcher — lifecycle (real chokidar over a temp dir)", () => {
@@ -468,6 +536,55 @@ describe("PushEventEmitter — directory and delete tombstone handling", () => {
     });
   }
 
+  it("F13: watcher batch event publishing applies bounded backpressure", async () => {
+    const paths = new Map<string, string>();
+    for (let i = 0; i < 32; i++) {
+      const rel = `bulk-${i}.md`;
+      const abs = path.join(dir, rel);
+      fs.writeFileSync(abs, `file ${i}`);
+      paths.set(abs, rel);
+    }
+
+    let inFlight = 0;
+    let maxInFlight = 0;
+    let releaseGate!: () => void;
+    const gate = new Promise<void>((resolve) => {
+      releaseGate = resolve;
+    });
+    const transport: PushTransport = {
+      start: async () => {},
+      dispose: async () => {},
+      connected: true,
+      publish: async () => {
+        inFlight += 1;
+        maxInFlight = Math.max(maxInFlight, inFlight);
+        await gate;
+        inFlight -= 1;
+      },
+    };
+    const emitter = new PushEventEmitter({
+      originTenantId: "tenant-indigo",
+      originDeviceId: "device-a",
+      transport,
+      flagProvider: new StaticFlagProvider(["tenant-indigo"]),
+      now: () => new Date("2026-06-18T12:00:00.000Z"),
+    });
+
+    const run = emitter.emitForBatch({ paths });
+    try {
+      for (let i = 0; i < 200 && maxInFlight <= 16; i++) {
+        await flushImmediate();
+      }
+      releaseGate();
+      await run;
+
+      expect(maxInFlight).toBeLessThanOrEqual(16);
+    } finally {
+      releaseGate();
+      await run.catch(() => {});
+    }
+  });
+
   it("skips directory changes silently without publishing or reporting an error", async () => {
     const published: PushEvent[] = [];
     const onError = vi.fn();