@indigoai-us/hq-cloud 6.11.11 → 6.11.12

package/src/skill-telemetry.ts

@@ -617,7 +617,52 @@ async function readCodexSessionContext(
   }
 }
 
-const MAX_BATCH_BYTES = 1_000_000;
+const MAX_BATCH_EVENTS = 100;
+const MAX_BATCH_BYTES = 240 * 1024;
+const ROW_TRUNCATION_SUFFIX = "...[truncated]";
+
+function jsonBytes(value: unknown): number {
+  return Buffer.byteLength(JSON.stringify(value), "utf-8");
+}
+
+function truncateLongestStringField(row: Record<string, unknown>): boolean {
+  let longestKey: string | undefined;
+  let longestBytes = 0;
+  for (const [key, value] of Object.entries(row)) {
+    if (typeof value !== "string" || value.length === 0) continue;
+    const bytes = Buffer.byteLength(value, "utf-8");
+    if (bytes > longestBytes) {
+      longestBytes = bytes;
+      longestKey = key;
+    }
+  }
+  if (longestKey === undefined) return false;
+
+  const value = row[longestKey] as string;
+  const keepChars =
+    value.length > ROW_TRUNCATION_SUFFIX.length
+      ? Math.floor((value.length - ROW_TRUNCATION_SUFFIX.length) / 2)
+      : 0;
+  const next =
+    keepChars > 0
+      ? `${value.slice(0, keepChars)}${ROW_TRUNCATION_SUFFIX}`
+      : "";
+  if (next === value) return false;
+  row[longestKey] = next;
+  return true;
+}
+
+function boundRowForPost(
+  row: Record<string, unknown>,
+  maxRowBytes: number,
+): Record<string, unknown> | null {
+  if (maxRowBytes < 0) return null;
+  const bounded = { ...row };
+  while (jsonBytes(bounded) > maxRowBytes) {
+    if (!truncateLongestStringField(bounded)) return null;
+  }
+  return bounded;
+}
 
 // ── Main entry point ──────────────────────────────────────────────────────────
 
@@ -627,7 +672,7 @@ const MAX_BATCH_BYTES = 1_000_000;
  * Cursor model (per-batch commit, matching the token collector for robustness):
  * each file is scanned from its stored byte offset to EOF; extracted events
  * carry the byte offset of the line they came from. Events are flushed in
- * ≤1 MiB batches, and the cursor advances **per successful batch** — so if one
+ * server-sized batches, and the cursor advances **per successful batch** — so if one
  * batch in a large (e.g. first-run backfill) fails, the batches that already
  * succeeded stay committed and only the rest re-send next sync.
  *
@@ -707,6 +752,11 @@ export async function collectAndSendSkillTelemetry(
   const fileScans: Record<string, FileScan> = {};
   const rotationResets: Record<string, CursorEntry> = {};
   const sourced: Sourced[] = [];
+  const envelopeBytes = Buffer.byteLength(
+    JSON.stringify({ machineId: opts.machineId, installerVersion: opts.installerVersion, events: [] }),
+    "utf-8",
+  );
+  const maxRowBytes = MAX_BATCH_BYTES - envelopeBytes;
 
   for (const { filePath, kind } of files) {
     let stat;
@@ -811,13 +861,27 @@ export async function collectAndSendSkillTelemetry(
         // Resolve cwd → owning company (cmp_* uid) from the per-run map.
         // Unresolved → undefined → companyUid omitted (unattributed/personal).
         const companyUid = resolveCompanyForCwd(ev.cwd, repoCompanyMap);
-        sourced.push({ row: toWireRow(ev, companyUid), filePath, endOffset });
+        const wireRow = toWireRow(ev, companyUid);
+        const wasOversized = jsonBytes(wireRow) > maxRowBytes;
+        const bounded = boundRowForPost(wireRow, maxRowBytes);
+        if (!bounded) {
+          log(
+            `[skill-telemetry] oversized row dropped before send (${filePath}:${i + 1})`,
+          );
+          continue;
+        }
+        if (wasOversized) {
+          log(
+            `[skill-telemetry] oversized row truncated before send (${filePath}:${i + 1})`,
+          );
+        }
+        sourced.push({ row: bounded, filePath, endOffset });
         fileScans[filePath].eventCount++;
       }
     }
   }
 
-  // 4. Flush in ≤1 MiB batches, advancing per-file progress on each 2xx.
+  // 4. Flush in server-sized batches, advancing per-file progress on each 2xx.
   let eventsSent = 0;
   let batchesSent = 0;
 
@@ -825,16 +889,11 @@ export async function collectAndSendSkillTelemetry(
   const sentCount: Record<string, number> = {};
   const committedOffset: Record<string, number> = {};
 
-  const envelopeBytes = Buffer.byteLength(
-    JSON.stringify({ machineId: opts.machineId, installerVersion: opts.installerVersion, events: [] }),
-    "utf-8",
-  );
-
   let batch: Sourced[] = [];
   let batchBytes = envelopeBytes;
 
-  const flush = async (): Promise<void> => {
-    if (batch.length === 0) return;
+  const flush = async (): Promise<boolean> => {
+    if (batch.length === 0) return true;
     const toSend = batch;
     batch = [];
     batchBytes = envelopeBytes;
@@ -852,24 +911,33 @@ export async function collectAndSendSkillTelemetry(
         const prev = committedOffset[s.filePath] ?? 0;
         if (s.endOffset > prev) committedOffset[s.filePath] = s.endOffset;
       }
+      return true;
     } catch (err) {
       log(`[skill-telemetry] postSkillInvocations failed (${(err as Error).message ?? err}) — these rows re-send next sync`);
       // Cursor not advanced for this batch; eventKey dedups the eventual re-send.
+      return false;
     }
   };
 
+  let stoppedAfterFailure = false;
   for (const s of sourced) {
     const rowBytes = Buffer.byteLength(JSON.stringify(s.row), "utf-8");
     const addCost = rowBytes + (batch.length > 0 ? 1 : 0);
-    if (batch.length > 0 && batchBytes + addCost > MAX_BATCH_BYTES) {
-      await flush();
+    if (
+      batch.length > 0 &&
+      (batch.length >= MAX_BATCH_EVENTS || batchBytes + addCost > MAX_BATCH_BYTES)
+    ) {
+      if (!(await flush())) {
+        stoppedAfterFailure = true;
+        break;
+      }
       batchBytes = envelopeBytes + rowBytes;
     } else {
       batchBytes += addCost;
     }
     batch.push(s);
   }
-  await flush();
+  if (!stoppedAfterFailure) await flush();
 
   // 5. Build the new cursor: loaded < rotationResets < per-file commit.
   //    A file settles at EOF only when every event extracted from it this run

package/src/sync/event-sync.test.ts

@@ -492,6 +492,81 @@ describe("startEventSync", () => {
     await handles!.dispose();
   });
 
+  it("R-F18: disables receive metrics when subscribe omits CloudWatch credentials", async () => {
+    const oldMetricEnv = {
+      region: process.env.HQ_SYNC_CLOUDWATCH_REGION,
+      accessKeyId: process.env.HQ_SYNC_CLOUDWATCH_ACCESS_KEY_ID,
+      secretAccessKey: process.env.HQ_SYNC_CLOUDWATCH_SECRET_ACCESS_KEY,
+      sessionToken: process.env.HQ_SYNC_CLOUDWATCH_SESSION_TOKEN,
+    };
+    delete process.env.HQ_SYNC_CLOUDWATCH_REGION;
+    delete process.env.HQ_SYNC_CLOUDWATCH_ACCESS_KEY_ID;
+    delete process.env.HQ_SYNC_CLOUDWATCH_SECRET_ACCESS_KEY;
+    delete process.env.HQ_SYNC_CLOUDWATCH_SESSION_TOKEN;
+
+    const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    const syncCalls: PushEvent[] = [];
+    let deliverNext: ((body: string) => void) | null = null;
+    const queueSqs: SqsClientLike = {
+      receiveMessage: ({ signal }) =>
+        new Promise((resolve) => {
+          signal.addEventListener("abort", () => resolve({ messages: [] }), {
+            once: true,
+          });
+          deliverNext = (body: string) => {
+            deliverNext = null;
+            resolve({ messages: [{ Body: body, ReceiptHandle: "rh" }] });
+          };
+        }),
+      deleteMessage: async () => {},
+    };
+
+    let handles: Awaited<ReturnType<typeof startEventSync>> = null;
+    try {
+      handles = await startEventSync({
+        ...happyOpts(syncCalls),
+        buildSqs: () => queueSqs,
+      });
+      expect(handles).not.toBeNull();
+
+      await vi.waitFor(() => {
+        if (!deliverNext) throw new Error("receiver not polling yet");
+      });
+      deliverNext!(
+        JSON.stringify(pushEvent({ originDeviceId: "peer-device" })),
+      );
+      await vi.waitFor(() => expect(syncCalls).toHaveLength(1));
+      await new Promise((resolve) => setTimeout(resolve, 0));
+
+      expect(consoleSpy).not.toHaveBeenCalled();
+    } finally {
+      await handles?.dispose();
+      consoleSpy.mockRestore();
+      if (oldMetricEnv.region === undefined) {
+        delete process.env.HQ_SYNC_CLOUDWATCH_REGION;
+      } else {
+        process.env.HQ_SYNC_CLOUDWATCH_REGION = oldMetricEnv.region;
+      }
+      if (oldMetricEnv.accessKeyId === undefined) {
+        delete process.env.HQ_SYNC_CLOUDWATCH_ACCESS_KEY_ID;
+      } else {
+        process.env.HQ_SYNC_CLOUDWATCH_ACCESS_KEY_ID = oldMetricEnv.accessKeyId;
+      }
+      if (oldMetricEnv.secretAccessKey === undefined) {
+        delete process.env.HQ_SYNC_CLOUDWATCH_SECRET_ACCESS_KEY;
+      } else {
+        process.env.HQ_SYNC_CLOUDWATCH_SECRET_ACCESS_KEY =
+          oldMetricEnv.secretAccessKey;
+      }
+      if (oldMetricEnv.sessionToken === undefined) {
+        delete process.env.HQ_SYNC_CLOUDWATCH_SESSION_TOKEN;
+      } else {
+        process.env.HQ_SYNC_CLOUDWATCH_SESSION_TOKEN =
+          oldMetricEnv.sessionToken;
+      }
+    }
+  });
+
   it("returns null (poll-only) when subscribe fails — never throws", async () => {
     const logged: string[] = [];
     const handles = await startEventSync({

package/src/sync/event-sync.ts

@@ -42,11 +42,16 @@ import { HttpPushTransport, type AuthTokenSource } from "./push-transport.js";
 import {
   SqsPushReceiver,
   type PushReceiver,
+  type PublishMetricFn,
   type SqsClientLike,
   type SyncEngineFn,
 } from "./push-receiver.js";
 import { PushEventEmitter } from "../watcher.js";
 import type { TreeChangeBatch } from "../watcher.js";
+import {
+  createSyncLatencyMetricPublisher,
+  type SyncMetricCredentials,
+} from "./metrics.js";
 
 // ─── US-019: rollout gate ───────────────────────────────────────────────────
 
@@ -102,6 +107,11 @@ export interface SubscribeSyncResponse {
   region: string;
   /** Short-lived creds scoped to receive/delete on exactly this queue. */
   credentials: SubscribeSyncCredentials;
+  /** Optional short-lived creds scoped to publish sync latency CloudWatch metrics. */
+  cloudWatch?: {
+    region: string;
+    credentials: SyncMetricCredentials;
+  };
 }
 
 /** Minimal fetch seam (matches push-transport.ts's FetchLike posture). */
@@ -122,6 +132,42 @@ function asNonEmptyString(v: unknown, field: string): string {
   return v;
 }
 
+function parseOptionalCloudWatch(
+  obj: Record<string, unknown>,
+): SubscribeSyncResponse["cloudWatch"] {
+  if (obj.cloudWatch === undefined) return undefined;
+  if (
+    !obj.cloudWatch ||
+    typeof obj.cloudWatch !== "object" ||
+    Array.isArray(obj.cloudWatch)
+  ) {
+    throw new Error("subscribeSyncReceive: response missing cloudWatch.region");
+  }
+  const cloudWatch = obj.cloudWatch as Record<string, unknown>;
+  const creds = (cloudWatch.credentials ?? {}) as Record<string, unknown>;
+  return {
+    region: asNonEmptyString(cloudWatch.region, "cloudWatch.region"),
+    credentials: {
+      accessKeyId: asNonEmptyString(
+        creds.accessKeyId,
+        "cloudWatch.credentials.accessKeyId",
+      ),
+      secretAccessKey: asNonEmptyString(
+        creds.secretAccessKey,
+        "cloudWatch.credentials.secretAccessKey",
+      ),
+      sessionToken:
+        typeof creds.sessionToken === "string" && creds.sessionToken.length > 0
+          ? creds.sessionToken
+          : undefined,
+      expiration:
+        typeof creds.expiration === "string" && creds.expiration.length > 0
+          ? creds.expiration
+          : undefined,
+    },
+  };
+}
+
 /**
  * `POST /v1/sync/subscribe` — provision (idempotently) this device's queue
  * and vend fresh receive credentials. Auth mirrors HttpPushTransport: Bearer
@@ -170,7 +216,8 @@ export async function subscribeSyncReceive(opts: {
   const parsed: unknown = JSON.parse(await res.text());
   const obj = parsed as Record<string, unknown>;
   const creds = (obj.credentials ?? {}) as Record<string, unknown>;
-  return {
+  const cloudWatch = parseOptionalCloudWatch(obj);
+  const response: SubscribeSyncResponse = {
     queueUrl: asNonEmptyString(obj.queueUrl, "queueUrl"),
     region: asNonEmptyString(obj.region, "region"),
     credentials: {
@@ -186,6 +233,8 @@ export async function subscribeSyncReceive(opts: {
       expiration: asNonEmptyString(creds.expiration, "credentials.expiration"),
     },
   };
+  if (cloudWatch !== undefined) response.cloudWatch = cloudWatch;
+  return response;
 }
 
 // ─── SQS SDK adapter ────────────────────────────────────────────────────────
@@ -436,6 +485,9 @@ export async function startEventSync(
           deviceId: dev,
         }));
     const initial = await subscribe(deviceId);
+    const publishMetric: PublishMetricFn = initial.cloudWatch
+      ? createSyncLatencyMetricPublisher(initial.cloudWatch)
+      : async () => undefined;
     const sqs = createRefreshingSqsClient({
       initial,
       subscribe: () => subscribe(deviceId),
@@ -453,6 +505,7 @@ export async function startEventSync(
         if (ctx.event.originDeviceId === deviceId) return;
         await opts.syncFn(ctx);
       },
+      publishMetric,
       // The client rollout gate is the authority; the env-driven per-tenant
       // flag provider is a server-side convention not set on user machines.
       enabled: true,

package/src/sync/metrics.test.ts

@@ -29,6 +29,7 @@ import {
   SYNC_LATENCY_METRIC_NAME,
   SYNC_METRIC_NAMESPACE,
   _setSyncCloudWatchClient,
+  createSyncLatencyMetricPublisher,
   publishSyncLatencyMetric,
   type SyncLatencyMetric,
 } from "./metrics.js";
@@ -274,6 +275,86 @@ describe("publishSyncLatencyMetric", () => {
 
     overrideMock.restore();
   });
+
+  it("F18: rejects ambient CloudWatch clients without scoped credentials and region", async () => {
+    const constructedWith: unknown[] = [];
+    const send = vi.fn(async () => ({}));
+
+    class CapturingCloudWatchClient {
+      readonly send = send;
+
+      constructor(config: unknown) {
+        constructedWith.push(config);
+      }
+    }
+
+    class CapturingPutMetricDataCommand {
+      constructor(readonly input: unknown) {}
+    }
+
+    vi.resetModules();
+    vi.doMock("@aws-sdk/client-cloudwatch", () => ({
+      CloudWatchClient: CapturingCloudWatchClient,
+      PutMetricDataCommand: CapturingPutMetricDataCommand,
+    }));
+
+    try {
+      const metrics = await import("./metrics.js");
+
+      await metrics.publishSyncLatencyMetric(makeMetric());
+
+      expect(constructedWith).not.toContainEqual({});
+    } finally {
+      vi.doUnmock("@aws-sdk/client-cloudwatch");
+      vi.resetModules();
+    }
+  });
+
+  it("R-F18: creates a metric publisher from explicit vended CloudWatch credentials", async () => {
+    const constructedWith: unknown[] = [];
+    const sent: unknown[] = [];
+    const publisher = createSyncLatencyMetricPublisher({
+      region: "us-west-2",
+      credentials: {
+        accessKeyId: "AKIA-VENDED",
+        secretAccessKey: "secret-vended",
+        sessionToken: "token-vended",
+        expiration: "2026-06-20T12:00:00.000Z",
+      },
+      buildClient: (config) => {
+        constructedWith.push(config);
+        return {
+          send: async (command: { input: unknown }) => {
+            sent.push(command.input);
+            return {};
+          },
+        };
+      },
+    });
+
+    await publisher(makeMetric({ tenantId: "prs_vended" }));
+
+    expect(constructedWith).toHaveLength(1);
+    expect(constructedWith[0]).toMatchObject({
+      region: "us-west-2",
+      credentials: {
+        accessKeyId: "AKIA-VENDED",
+        secretAccessKey: "secret-vended",
+        sessionToken: "token-vended",
+        expiration: new Date("2026-06-20T12:00:00.000Z"),
+      },
+    });
+    expect(sent).toHaveLength(1);
+    expect(sent[0]).toMatchObject({
+      Namespace: SYNC_METRIC_NAMESPACE,
+      MetricData: [
+        {
+          MetricName: SYNC_LATENCY_METRIC_NAME,
+          Dimensions: [{ Name: "TenantId", Value: "prs_vended" }],
+        },
+      ],
+    });
+  });
 });
 
 // ── Receive-success-path metric emission (US-011 AC: "metric emission unit-

package/src/sync/metrics.ts

@@ -75,13 +75,41 @@ export interface SyncLatencyMetric {
   timestamp: Date;
 }
 
+export interface SyncMetricCredentials {
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+  expiration?: string;
+}
+
+type CloudWatchClientConfig = NonNullable<
+  ConstructorParameters<typeof CloudWatchClient>[0]
+>;
+type CloudWatchClientLike = Pick<CloudWatchClient, "send">;
+
 // ── Client ─────────────────────────────────────────────────────────────────
 
-let _cwClient: CloudWatchClient | undefined;
+let _cwClient: CloudWatchClientLike | undefined;
 
-function getCloudWatchClient(): CloudWatchClient {
+function getCloudWatchClient(): CloudWatchClientLike {
   if (!_cwClient) {
-    _cwClient = new CloudWatchClient({});
+    const region = process.env.HQ_SYNC_CLOUDWATCH_REGION;
+    const accessKeyId = process.env.HQ_SYNC_CLOUDWATCH_ACCESS_KEY_ID;
+    const secretAccessKey = process.env.HQ_SYNC_CLOUDWATCH_SECRET_ACCESS_KEY;
+    const sessionToken = process.env.HQ_SYNC_CLOUDWATCH_SESSION_TOKEN;
+    if (!region || !accessKeyId || !secretAccessKey) {
+      throw new Error(
+        "CloudWatch metric publisher requires scoped HQ_SYNC_CLOUDWATCH_REGION and credentials",
+      );
+    }
+    _cwClient = new CloudWatchClient({
+      region,
+      credentials: {
+        accessKeyId,
+        secretAccessKey,
+        ...(sessionToken ? { sessionToken } : {}),
+      },
+    });
   }
   return _cwClient;
 }
@@ -94,11 +122,38 @@ export function _setSyncCloudWatchClient(client: CloudWatchClient): void {
   _cwClient = client;
 }
 
+export interface CreateSyncLatencyMetricPublisherOptions {
+  region: string;
+  credentials: SyncMetricCredentials;
+  /** Override CloudWatch construction (tests). */
+  buildClient?: (config: CloudWatchClientConfig) => CloudWatchClientLike;
+}
+
+export function createSyncLatencyMetricPublisher(
+  opts: CreateSyncLatencyMetricPublisherOptions,
+): (metric: SyncLatencyMetric) => Promise<void> {
+  const credentials = {
+    accessKeyId: opts.credentials.accessKeyId,
+    secretAccessKey: opts.credentials.secretAccessKey,
+    ...(opts.credentials.sessionToken
+      ? { sessionToken: opts.credentials.sessionToken }
+      : {}),
+    ...(opts.credentials.expiration
+      ? { expiration: new Date(opts.credentials.expiration) }
+      : {}),
+  };
+  const client = (opts.buildClient ?? ((config) => new CloudWatchClient(config)))({
+    region: opts.region,
+    credentials,
+  });
+  return (metric) => publishSyncLatencyMetric(metric, { client });
+}
+
 // ── Publish ────────────────────────────────────────────────────────────────
 
 export interface PublishSyncLatencyMetricOptions {
   /** Override the CloudWatch client (tests). Defaults to the module singleton. */
-  client?: CloudWatchClient;
+  client?: CloudWatchClientLike;
   /** Optional pino logger for emission failures. */
   logger?: Logger;
 }

package/src/sync/pull-scope.ts

@@ -21,7 +21,11 @@
  */
 import * as fs from "fs";
 import * as path from "path";
-import { coalescePrefixes, grantPathToPrefix } from "../prefix-coalesce.js";
+import {
+  coalescePrefixes,
+  grantPathToScopePrefix,
+  pathToScopePrefix,
+} from "../prefix-coalesce.js";
 import type {
   ExplicitGrant,
   MembershipSyncConfig,
@@ -133,15 +137,20 @@ export async function resolvePullScope(
 
     if (cfg.syncMode === "custom") {
       const customPrefixes = (cfg.customPaths ?? []).map((p) =>
-        grantPathToPrefix(p, slug),
+        grantPathToScopePrefix(p, slug),
       );
       // A bare-everything entry ("" — e.g. a `*` path) collapses under
       // `coalescePrefixes` (which drops empties) to "nothing", which would
       // prune the whole tree. An everything-scope is semantically `all`.
-      if (customPrefixes.some((p) => p === "")) return { syncMode: "all" };
+      if (customPrefixes.some((p) => p.prefix === "")) {
+        return { syncMode: "all" };
+      }
       return {
         syncMode: "custom",
-        prefixSet: coalescePrefixes([...customPrefixes, ...pinPrefixes]),
+        prefixSet: coalescePrefixes([
+          ...customPrefixes,
+          ...pinPrefixes.map(pathToScopePrefix),
+        ]),
       };
     }
     // shared: scope to the caller's explicit grants. Real grant paths are
@@ -158,14 +167,21 @@ export async function resolvePullScope(
     // []) is a real "nothing shared with me" and is allowed to narrow.
     if (!client.listMyExplicitGrants) return { syncMode: "all" };
     const grants = await client.listMyExplicitGrants(companyUid);
-    const sharedPrefixes = grants.map((g) => grantPathToPrefix(g.path, slug));
+    const sharedPrefixes = grants.map((g) =>
+      grantPathToScopePrefix(g.path, slug),
+    );
     // A wildcard grant (`*`) normalizes to "" = everything. Since
     // `coalescePrefixes` drops empties (collapsing "everything" to "nothing"),
     // treat any such grant as full-access `all` rather than risk pruning.
-    if (sharedPrefixes.some((p) => p === "")) return { syncMode: "all" };
+    if (sharedPrefixes.some((p) => p.prefix === "")) {
+      return { syncMode: "all" };
+    }
     return {
       syncMode: "shared",
-      prefixSet: coalescePrefixes([...sharedPrefixes, ...pinPrefixes]),
+      prefixSet: coalescePrefixes([
+        ...sharedPrefixes,
+        ...pinPrefixes.map(pathToScopePrefix),
+      ]),
     };
   } catch {
     // Degrade to `all` — never prune on a resolution failure.

package/src/sync/push-receiver.test.ts

@@ -357,7 +357,44 @@ describe("US-009: SqsPushReceiver — reconnect / catch-up replay", () => {
     await receiver.dispose();
     // First threw (processedCount not incremented), second succeeded.
     expect(receiver.processedCount).toBe(1);
-    expect(sqs.deleted).toEqual(expect.arrayContaining(["rh-1", "rh-2"]));
+    // F10: a failed targeted pull is left undeleted for SQS redelivery; only the
+    // successful message (rh-2) is acknowledged. The thrown one (rh-1) stays on
+    // the queue so the visibility-timeout redelivery can retry it.
+    expect(sqs.deleted).toContain("rh-2");
+    expect(sqs.deleted).not.toContain("rh-1");
+  });
+
+  it("F10: leaves failed targeted pull messages undeleted so redelivery retries", async () => {
+    const sqs = new FakeSqs();
+    let calls = 0;
+    const syncFn: SyncEngineFn = async () => {
+      calls += 1;
+      if (calls === 1) throw new Error("targeted pull failed");
+    };
+    const event = makeEvent({
+      relativePath: "companies/indigo/retry-after-failure.md",
+      sequenceNumber: 7,
+    });
+    sqs.enqueue([msg(event, "rh-first-attempt")]);
+    sqs.enqueue([msg(event, "rh-redelivery")]);
+
+    const receiver = new SqsPushReceiver({
+      tenantId: TENANT,
+      queueUrl: QUEUE_URL,
+      sqs,
+      syncFn,
+      enabled: true,
+      sleep: fastSleep,
+    });
+    await receiver.start();
+    await waitFor(() => sqs.deleted.includes("rh-redelivery"));
+    await receiver.dispose();
+
+    expect(calls).toBe(2);
+    expect(receiver.processedCount).toBe(1);
+    expect(receiver.dedupedCount).toBe(0);
+    expect(sqs.deleted).not.toContain("rh-first-attempt");
+    expect(sqs.deleted).toContain("rh-redelivery");
   });
 });