@takosjp/yurucommu-core 3.4.3 → 3.4.4

package/src/backend/lib/delivery/queue.ts

@@ -9,9 +9,29 @@ import type {
   IQueueMessage,
   IQueueProducer,
 } from "../../runtime/queue.ts";
-import type { Database } from "../../../db/index.ts";
-import { and, eq, notInArray, or, sql } from "drizzle-orm";
-import { actorCache, deliveryQueue } from "../../../db/index.ts";
+import {
+  and,
+  asc,
+  eq,
+  exists,
+  inArray,
+  lt,
+  lte,
+  notInArray,
+  notExists,
+  or,
+  sql,
+} from "drizzle-orm";
+import {
+  actorCache,
+  affectedRowCount,
+  deliveryEndpointRecipients,
+  deliveryQueue,
+  insertMany,
+  runBatch,
+  type D1Statement,
+  type Database,
+} from "../../../db/index.ts";
 import { isSafeRemoteUrl } from "../../federation-helpers.ts";
 import {
   DELIVERY_QUEUE_MESSAGE_VERSION,
@@ -24,12 +44,29 @@ import {
 import { computeDeliveryJobId, safeEndpointHost } from "./transformers.ts";
 import { emitMetric } from "./metrics.ts";
 import { logger } from "../logger.ts";
-import { filterBlockedActorApIds, isActorBlocked } from "../blocklist.ts";
+import {
+  filterBlockedActorApIdsStrict,
+  isActorBlockedStrict,
+} from "../blocklist.ts";
 import {
   enqueuePendingNotificationPushJobs,
   processNotificationPushJob,
   recoverDeadLetteredNotificationPushJob,
 } from "../notification-push.ts";
+import {
+  buildResolveActorMessage,
+  enqueueDeliveryResolutionJobs,
+  enqueuePendingDeliveryResolutionJobs,
+  persistDeliveryResolutionJobs,
+} from "./resolution-outbox.ts";
+import {
+  deliveryFanoutIntentFromMessage,
+  enqueueDeliveryFanoutJob,
+  enqueuePendingDeliveryFanoutJobs,
+  failDeliveryFanoutJob,
+} from "./fanout-outbox.ts";
+
+export { buildResolveActorMessage } from "./resolution-outbox.ts";
 
 const log = logger.child({ component: "delivery.queue" });
 
@@ -165,6 +202,24 @@ export async function sendQueueMessage(
   );
 }
 
+/**
+ * DLQ recovery cannot use the ordinary best-effort producer contract: a
+ * missing binding must keep the DLQ message retryable rather than look like a
+ * successful redrive and get ACKed. Runtime send failures already throw; make
+ * binding drift do the same while preserving the shared alert/metric.
+ */
+async function sendQueueMessageRequired(
+  env: Env,
+  body: DeliveryQueueMessageV1,
+  delaySeconds?: number,
+): Promise<void> {
+  if (!queueAvailable(env)) {
+    reportProducerUnavailable("sendQueueMessageRequired");
+    throw new Error("DELIVERY_QUEUE and DELIVERY_DLQ bindings are required");
+  }
+  await sendQueueMessage(env, body, delaySeconds);
+}
+
 export async function sendDlqMessage(
   env: Env,
   payload: DeliveryDlqMessageV1,
@@ -184,6 +239,12 @@ export async function sendDlqMessage(
 // reconcile worker (processReconcileJob) share one source of truth.
 export const MAX_RECONCILE_ATTEMPTS = 5;
 
+// Cloudflare puts the raw MAIN-queue body onto the DLQ after transport/runtime
+// retries are exhausted. Redrive idempotent delivery work a few times so one
+// infrastructure incident does not become permanent data loss, while keeping
+// a poison message or persistent code failure from cycling MAIN -> DLQ forever.
+export const MAX_AUTO_DLQ_REDRIVES = 3;
+
 export function buildDeliverEndpointMessage(
   jobId: string,
   reconcileAttempt = 0,
@@ -199,21 +260,6 @@ export function buildDeliverEndpointMessage(
   };
 }
 
-export function buildResolveActorMessage(
-  activityId: string,
-  recipientActorApId: string,
-  attempts = 0,
-): DeliveryQueueMessageV1 {
-  return {
-    version: DELIVERY_QUEUE_MESSAGE_VERSION,
-    type: "resolve_actor",
-    activityId,
-    recipientActorApId,
-    ...(attempts > 0 ? { attempts } : {}),
-    scheduledAt: nowIso(),
-  };
-}
-
 export function buildReconcileJobMessage(
   jobId: string,
   reconcileAttempt: number,
@@ -236,32 +282,188 @@ export async function upsertDeliveryJob(
   jobId: string,
   activityId: string,
   endpoint: string,
-): Promise<void> {
-  await db
-    .insert(deliveryQueue)
-    .values({
-      id: jobId,
-      inboxUrl: endpoint,
-      activityApId: activityId,
-      attempts: 0,
-      nextAttemptAt: nowIso(),
-      status: "pending",
-    })
-    .onConflictDoNothing();
-
-  // Guard against overwriting in-flight or completed jobs.
-  await db
-    .update(deliveryQueue)
-    .set({
-      inboxUrl: endpoint,
-      activityApId: activityId,
-    })
-    .where(
+  recipientActorApIds: readonly string[],
+): Promise<boolean> {
+  const uniqueRecipientActorApIds = [...new Set(recipientActorApIds)];
+  if (uniqueRecipientActorApIds.length === 0) return false;
+
+  const recipientInserts = insertMany(
+    db,
+    deliveryEndpointRecipients,
+    uniqueRecipientActorApIds.map((recipientActorApId) => ({
+      deliveryJobId: jobId,
+      recipientActorApId,
+    })),
+    { conflict: "ignore" },
+  );
+  const firstRecipientInsert = recipientInserts[0];
+  if (!firstRecipientInsert) return false;
+  const hasRetainedRecipient = () =>
+    exists(
+      db
+        .select({ deliveryJobId: deliveryEndpointRecipients.deliveryJobId })
+        .from(deliveryEndpointRecipients)
+        .where(eq(deliveryEndpointRecipients.deliveryJobId, jobId)),
+    );
+  const now = nowIso();
+
+  // One deep materialization unit owns endpoint aggregation, recipient
+  // attribution, tombstone fencing, and compatibility with legacy jobs. The
+  // recipient trigger drops a deleted actor before the final cleanup runs.
+  // A newly-created job is marked attributed; a pre-0029 conflict retains its
+  // default 0 because its full historical recipient set cannot be recovered.
+  await runBatch(db, [
+    firstRecipientInsert,
+    ...recipientInserts.slice(1),
+    db
+      .insert(deliveryQueue)
+      .values({
+        id: jobId,
+        inboxUrl: endpoint,
+        activityApId: activityId,
+        attempts: 0,
+        nextAttemptAt: now,
+        status: "pending",
+        recipientAttributionComplete: 1,
+      })
+      .onConflictDoNothing() as unknown as D1Statement,
+    // Guard against overwriting in-flight or completed jobs, and do not update
+    // a new all-tombstoned candidate that retained no recipient mapping.
+    db
+      .update(deliveryQueue)
+      .set({ inboxUrl: endpoint, activityApId: activityId })
+      .where(
+        and(
+          eq(deliveryQueue.id, jobId),
+          notInArray(deliveryQueue.status, ["processing", "delivered"]),
+          hasRetainedRecipient(),
+        ),
+      ) as unknown as D1Statement,
+    // Only jobs born under the recipient-aware implementation are safe to
+    // delete when every supplied recipient was already tombstoned. Legacy jobs
+    // remain conservative because they may represent unknown co-recipients.
+    db.delete(deliveryQueue).where(
       and(
         eq(deliveryQueue.id, jobId),
-        notInArray(deliveryQueue.status, ["processing", "delivered"]),
+        eq(deliveryQueue.recipientAttributionComplete, 1),
+        notExists(
+          db
+            .select({
+              deliveryJobId: deliveryEndpointRecipients.deliveryJobId,
+            })
+            .from(deliveryEndpointRecipients)
+            .where(eq(deliveryEndpointRecipients.deliveryJobId, jobId)),
+        ),
       ),
+    ) as unknown as D1Statement,
+  ]);
+
+  return Boolean(
+    await db
+      .select({ id: deliveryQueue.id })
+      .from(deliveryQueue)
+      .where(eq(deliveryQueue.id, jobId))
+      .get(),
+  );
+}
+
+const PENDING_DELIVERY_SCAN_LIMIT = 500;
+const STALE_DELIVERY_PROCESSING_MS = 2 * 60 * 1000;
+const TERMINAL_DELIVERY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
+const TERMINAL_DELIVERY_PURGE_LIMIT = 50;
+const TERMINAL_DELIVERY_STATUSES = [
+  "delivered",
+  "dead_letter",
+  "failed",
+] as const;
+
+/**
+ * Bound terminal endpoint-ledger growth while retaining a 30-day idempotency
+ * and diagnostic window. Activity deletion still owns immediate projection
+ * cleanup; this path handles Activities that legitimately remain in history.
+ */
+export async function purgeTerminalDeliveryEndpointJobs(
+  db: Database,
+  nowDate = new Date(),
+): Promise<number> {
+  const cutoff = new Date(
+    nowDate.getTime() - TERMINAL_DELIVERY_RETENTION_MS,
+  ).toISOString();
+  const rows = await db
+    .select({ id: deliveryQueue.id })
+    .from(deliveryQueue)
+    .where(
+      and(
+        inArray(deliveryQueue.status, [...TERMINAL_DELIVERY_STATUSES]),
+        lte(deliveryQueue.createdAt, cutoff),
+      ),
+    )
+    .orderBy(asc(deliveryQueue.createdAt))
+    .limit(TERMINAL_DELIVERY_PURGE_LIMIT);
+  if (rows.length === 0) return 0;
+  const result = await db.delete(deliveryQueue).where(
+    and(
+      inArray(
+        deliveryQueue.id,
+        rows.map((row) => row.id),
+      ),
+      inArray(deliveryQueue.status, [...TERMINAL_DELIVERY_STATUSES]),
+      lte(deliveryQueue.createdAt, cutoff),
+    ),
+  );
+  return affectedRowCount(result);
+}
+
+/**
+ * Re-publish durable endpoint jobs whose first Queue RPC was lost, whose
+ * backoff has elapsed, or whose processing worker disappeared. Queue messages
+ * are wakeups; delivery_queue remains the retry authority.
+ */
+export async function enqueuePendingDeliveryEndpointJobs(
+  env: Env,
+  nowDate = new Date(),
+  options: {
+    readonly includeDeferred?: boolean;
+    readonly includeFreshPending?: boolean;
+  } = {},
+): Promise<number> {
+  await purgeTerminalDeliveryEndpointJobs(env.DB_INSTANCE, nowDate);
+  if (!env.DELIVERY_QUEUE) return 0;
+  const now = nowDate.toISOString();
+  const staleBefore = new Date(
+    nowDate.getTime() - STALE_DELIVERY_PROCESSING_MS,
+  ).toISOString();
+  const rows = await env.DB_INSTANCE.select({ id: deliveryQueue.id })
+    .from(deliveryQueue)
+    .where(
+      or(
+        options.includeFreshPending
+          ? eq(deliveryQueue.status, "pending")
+          : and(
+              eq(deliveryQueue.status, "pending"),
+              lt(deliveryQueue.createdAt, staleBefore),
+            ),
+        options.includeDeferred
+          ? eq(deliveryQueue.status, "retry_wait")
+          : and(
+              eq(deliveryQueue.status, "retry_wait"),
+              lte(deliveryQueue.nextAttemptAt, now),
+            ),
+        and(
+          eq(deliveryQueue.status, "processing"),
+          lt(deliveryQueue.processingStartedAt, staleBefore),
+        ),
+      ),
+    )
+    .limit(PENDING_DELIVERY_SCAN_LIMIT);
+  for (let offset = 0; offset < rows.length; offset += 100) {
+    await env.DELIVERY_QUEUE.sendBatch(
+      rows.slice(offset, offset + 100).map((row) => ({
+        body: buildDeliverEndpointMessage(row.id),
+      })),
     );
+  }
+  return rows.length;
 }
 
 export async function enqueueResolveForEndpointActors(
@@ -326,16 +528,35 @@ export async function enqueueResolveForEndpointActors(
       // Drop recipients the operator has defederated before re-enqueueing
       // resolve_actor jobs (outbound blocklist enforcement). Batched (2 queries)
       // rather than a serial isActorBlocked per candidate.
-      const blockedSet = await filterBlockedActorApIds(db, candidates);
+      let blockedSet: Set<string>;
+      try {
+        blockedSet = await filterBlockedActorApIdsStrict(db, candidates);
+      } catch (error) {
+        // Preserve the resolution authority before surfacing a transient local
+        // moderation outage. The resolver itself repeats the strict check, so
+        // recovery cannot deliver to a blocked actor once the DB is readable.
+        await persistDeliveryResolutionJobs(
+          db,
+          candidates.map((recipientActorApId) => ({
+            activityId,
+            recipientActorApId,
+          })),
+        );
+        throw error;
+      }
       const slice = candidates.filter((apId) => !blockedSet.has(apId));
 
       if (slice.length === 0) continue;
 
-      const requests = slice.map((recipientApId) => ({
-        body: buildResolveActorMessage(activityId, recipientApId),
-      }));
-
-      await env.DELIVERY_QUEUE.sendBatch(requests);
+      await enqueueDeliveryResolutionJobs(
+        db,
+        env.DELIVERY_QUEUE,
+        slice.map((recipientActorApId) => ({
+          activityId,
+          recipientActorApId,
+        })),
+        { reopenTerminal: true },
+      );
       enqueued += slice.length;
     }
 
@@ -368,13 +589,26 @@ export async function enqueueFanoutToFollowers(
   activityId: string,
   followeeApId: string,
 ): Promise<void> {
-  await sendQueueMessage(env, {
+  const body = {
     version: DELIVERY_QUEUE_MESSAGE_VERSION,
-    type: "fanout_followers",
+    type: "fanout_followers" as const,
     activityId,
     followeeApId,
     scheduledAt: nowIso(),
-  });
+  };
+  const db = (env as Partial<Env>).DB_INSTANCE;
+  if (db) {
+    const queue = queueAvailable(env) ? env.DELIVERY_QUEUE : undefined;
+    await enqueueDeliveryFanoutJob(
+      db,
+      queue,
+      deliveryFanoutIntentFromMessage(body),
+    );
+    if (!queue) reportProducerUnavailable("enqueueFanoutToFollowers");
+    else producerUnavailableReported = false;
+    return;
+  }
+  await sendQueueMessage(env, body);
 }
 
 export async function enqueueDeliveryToActor(
@@ -385,16 +619,35 @@ export async function enqueueDeliveryToActor(
   // Enforce the operator blocklist on the OUTBOUND single-actor path (DMs,
   // Accept/Follow responses, targeted post/story interactions). A defederated
   // domain/actor must never receive our activities, mirroring the inbound
-  // enforcement in the inbox handler. Best-effort: if the db read fails,
-  // isActorBlocked returns false (never black-hole on a transient error).
+  // enforcement in the inbox handler. A DB read failure retains the delivery
+  // intent and propagates so unavailable authority never becomes a bypass.
   const db = (env as Partial<Env>).DB_INSTANCE;
-  if (db && (await isActorBlocked(db, recipientActorApId))) {
-    log.info("Skipping outbound delivery to blocked actor", {
-      event: "delivery.blocklist.actor_skip",
-      actor: recipientActorApId,
-      activityId,
-    });
-    emitMetric("delivery.blocklist.actor_skip", 1, {});
+  if (db) {
+    try {
+      if (await isActorBlockedStrict(db, recipientActorApId)) {
+        log.info("Skipping outbound delivery to blocked actor", {
+          event: "delivery.blocklist.actor_skip",
+          actor: recipientActorApId,
+          activityId,
+        });
+        emitMetric("delivery.blocklist.actor_skip", 1, {});
+        return;
+      }
+    } catch (error) {
+      // The Activity may already be committed and some callers deliberately
+      // treat queue publication as best-effort. Persist the semantic intent so
+      // a later outbox sweep can resume after moderation authority recovers.
+      await persistDeliveryResolutionJobs(db, [
+        { activityId, recipientActorApId },
+      ]);
+      throw error;
+    }
+    const queue = queueAvailable(env) ? env.DELIVERY_QUEUE : undefined;
+    await enqueueDeliveryResolutionJobs(db, queue, [
+      { activityId, recipientActorApId },
+    ]);
+    if (!queue) reportProducerUnavailable("enqueueDeliveryToActor");
+    else producerUnavailableReported = false;
     return;
   }
 
@@ -404,6 +657,75 @@ export async function enqueueDeliveryToActor(
   );
 }
 
+/**
+ * Enqueue several distinct activities to the same actor without one Queue RPC
+ * per activity. Cloudflare Queues accepts at most 100 messages per sendBatch;
+ * keep the provider-neutral port bounded to that same size. Replaying a batch
+ * is safe: endpoint resolution later derives the durable delivery-job id from
+ * activity id + endpoint and upserts it idempotently.
+ */
+export async function enqueueDeliveriesToActor(
+  env: Env,
+  activityIds: readonly string[],
+  recipientActorApId: string,
+): Promise<void> {
+  if (activityIds.length === 0) return;
+  const db = (env as Partial<Env>).DB_INSTANCE;
+  const intents = activityIds.map((activityId) => ({
+    activityId,
+    recipientActorApId,
+  }));
+  if (db) {
+    try {
+      if (await isActorBlockedStrict(db, recipientActorApId)) {
+        log.info("Skipping outbound deliveries to blocked actor", {
+          event: "delivery.blocklist.actor_batch_skip",
+          actor: recipientActorApId,
+          activityCount: activityIds.length,
+        });
+        emitMetric(
+          "delivery.blocklist.actor_batch_skip",
+          activityIds.length,
+          {},
+        );
+        return;
+      }
+    } catch (error) {
+      await persistDeliveryResolutionJobs(db, intents);
+      throw error;
+    }
+  }
+
+  if (db) await persistDeliveryResolutionJobs(db, intents);
+  if (!queueAvailable(env)) {
+    reportProducerUnavailable("enqueueDeliveriesToActor");
+    // This batched path is used by account Move after its pending edges and
+    // outbound activities have already committed. A silent no-op would make
+    // those local users follow a destination that never learned about them,
+    // with no remaining old edges from which to reconstruct delivery. Throw so
+    // the inbound dispatch stays retryable and can resume the durable activity
+    // namespace after the operator restores the producer bindings.
+    throw new Error(
+      "DELIVERY_QUEUE and DELIVERY_DLQ bindings are required for batched delivery",
+    );
+  }
+  producerUnavailableReported = false;
+
+  if (db) {
+    await enqueueDeliveryResolutionJobs(db, env.DELIVERY_QUEUE, intents);
+    return;
+  }
+
+  const SEND_BATCH_SIZE = 100;
+  for (let offset = 0; offset < activityIds.length; offset += SEND_BATCH_SIZE) {
+    await env.DELIVERY_QUEUE.sendBatch(
+      activityIds.slice(offset, offset + SEND_BATCH_SIZE).map((activityId) => ({
+        body: buildResolveActorMessage(activityId, recipientActorApId),
+      })),
+    );
+  }
+}
+
 /**
  * Fan an activity out to a community's audience (members + community
  * followers) instead of the author's personal follower graph. Used for
@@ -415,14 +737,27 @@ export async function enqueueFanoutToCommunity(
   communityApId: string,
   announceActivityId?: string,
 ): Promise<void> {
-  await sendQueueMessage(env, {
+  const body = {
     version: DELIVERY_QUEUE_MESSAGE_VERSION,
-    type: "fanout_community",
+    type: "fanout_community" as const,
     activityId,
     communityApId,
     ...(announceActivityId ? { announceActivityId } : {}),
     scheduledAt: nowIso(),
-  });
+  };
+  const db = (env as Partial<Env>).DB_INSTANCE;
+  if (db) {
+    const queue = queueAvailable(env) ? env.DELIVERY_QUEUE : undefined;
+    await enqueueDeliveryFanoutJob(
+      db,
+      queue,
+      deliveryFanoutIntentFromMessage(body),
+    );
+    if (!queue) reportProducerUnavailable("enqueueFanoutToCommunity");
+    else producerUnavailableReported = false;
+    return;
+  }
+  await sendQueueMessage(env, body);
 }
 
 // ---------------------------------------------------------------------------
@@ -527,6 +862,16 @@ export async function handleDeliveryQueueBatch(
 
   // Community fanout can create local inbox rows inside this consumer rather
   // than an HTTP request. Flush the same DB-triggered outbox choke point here.
+  try {
+    await enqueuePendingDeliveryFanoutJobs(env);
+    await enqueuePendingDeliveryEndpointJobs(env);
+    await enqueuePendingDeliveryResolutionJobs(env);
+  } catch (error) {
+    log.error("Failed to enqueue durable federation outbox", {
+      event: "delivery.outbox.enqueue_failed",
+      error,
+    });
+  }
   try {
     await enqueuePendingNotificationPushJobs(env);
   } catch (error) {
@@ -546,7 +891,11 @@ export async function handleDeliveryDlqBatch(
   env: Env,
 ): Promise<void> {
   for (const message of batch.messages) {
-    const body = message.body;
+    // The configured DLQ receives both app-built DeliveryDlqMessageV1 bodies
+    // and Cloudflare's raw original MAIN-queue bodies. The batch generic can
+    // express only one of those at a time, so narrow the runtime union from
+    // unknown through the two wire validators below.
+    const body: unknown = message.body;
     if (!isDeliveryDlqMessageV1(body)) {
       // Not an app-built `dlq` message. Cloudflare Queues also delivers here
       // the RAW original body of any MAIN-queue message that exhausted its
@@ -554,8 +903,22 @@ export async function handleDeliveryDlqBatch(
       // "invalid": a lost fanout/resolve drops local notifications or delivery
       // planning, and a lost notification_push strands its durable outbox row.
       if (isDeliveryQueueMessageV1(body)) {
-        await handleAutoDeadLetteredMessage(env, body);
-        message.ack();
+        try {
+          await handleAutoDeadLetteredMessage(env, body);
+          message.ack();
+        } catch (error) {
+          log.warn("Failed to recover auto-dead-lettered message", {
+            event: "delivery.dlq.auto_recovery_failed",
+            messageType: body.type,
+            error,
+          });
+          emitMetric("delivery.dlq.auto_recovery_failed", 1, {
+            message_type: body.type,
+          });
+          // Keep the raw DLQ message alive until its durable repair or bounded
+          // MAIN-queue redrive succeeds.
+          message.retry({ delaySeconds: 60 });
+        }
         continue;
       }
       log.warn("Invalid DLQ message format, skipping", {
@@ -596,7 +959,7 @@ export async function handleDeliveryDlqBatch(
       continue;
     }
     try {
-      await sendQueueMessage(
+      await sendQueueMessageRequired(
         env,
         buildReconcileJobMessage(body.jobId, reconcileAttempt + 1),
         6 * 60 * 60,
@@ -607,6 +970,14 @@ export async function handleDeliveryDlqBatch(
         jobId: body.jobId,
         error: e,
       });
+      // The dead-letter row is already the durable evidence that delivery did
+      // not complete. ACKing its DLQ message here used to discard the only
+      // trigger that can revive it after a transient main-queue outage, leaving
+      // the ActivityPub delivery permanently dead_letter. Ask Cloudflare
+      // Queues to redeliver this exact DLQ message instead; the successful path
+      // below remains the sole ACK point.
+      message.retry({ delaySeconds: 60 });
+      continue;
     }
 
     message.ack();
@@ -617,7 +988,7 @@ export async function handleDeliveryDlqBatch(
  * Recover / account for a MAIN-queue message that Cloudflare auto-dead-lettered
  * (retries exhausted with the raw body). `notification_push` rows are durable,
  * so reset the job to retry through the outbox instead of stranding it;
- * everything else is logged with an alerting metric rather than swallowed.
+ * idempotent federation work is sent back to MAIN with a bounded generation.
  */
 async function handleAutoDeadLetteredMessage(
   env: Env,
@@ -642,16 +1013,51 @@ async function handleAutoDeadLetteredMessage(
         error,
       });
       emitMetric("delivery.dlq.notification_push_recover_failed", 1, {});
+      throw error;
+    }
+    return;
+  }
+
+  const autoDlqAttempt = body.autoDlqAttempt ?? 0;
+  if (autoDlqAttempt >= MAX_AUTO_DLQ_REDRIVES) {
+    if (body.type === "fanout_followers" || body.type === "fanout_community") {
+      const db = (env as Partial<Env>).DB_INSTANCE;
+      if (db) {
+        await failDeliveryFanoutJob(
+          db,
+          deliveryFanoutIntentFromMessage(body),
+          `Queue delivery exhausted after ${autoDlqAttempt} automatic redrives`,
+        );
+      }
     }
+    log.error("Delivery message exhausted automatic DLQ redrives; dropping", {
+      event: "delivery.dlq.auto_redrive_exhausted",
+      messageType: body.type,
+      autoDlqAttempt,
+    });
+    emitMetric("delivery.dlq.auto_redrive_exhausted", 1, {
+      message_type: body.type,
+    });
     return;
   }
 
-  // fanout_*/resolve_actor/deliver_endpoint/reconcile_job: no durable ledger to
-  // rewind here, but make the loss explicit (alertable) rather than a silent
-  // "invalid message" ack.
-  log.error("Delivery message auto-dead-lettered; dropping", {
-    event: "delivery.dlq.auto_dead_lettered",
+  // Fanout pages preserve their cursor/stage, resolve preserves its own fetch
+  // attempt, and deliver/reconcile preserve the durable job generation. Their
+  // handlers are idempotent (stable inbox/job keys plus CAS), so replay the
+  // exact semantic position with only the transport-redrive generation and
+  // schedule timestamp changed.
+  const redriven = {
+    ...body,
+    autoDlqAttempt: autoDlqAttempt + 1,
+    scheduledAt: nowIso(),
+  } as DeliveryQueueMessageV1;
+  await sendQueueMessageRequired(env, redriven, 60);
+  log.error("Delivery message auto-dead-lettered; bounded redrive scheduled", {
+    event: "delivery.dlq.auto_redrive_scheduled",
     messageType: body.type,
+    autoDlqAttempt: autoDlqAttempt + 1,
+  });
+  emitMetric("delivery.dlq.auto_redrive_scheduled", 1, {
+    message_type: body.type,
   });
-  emitMetric("delivery.dlq.auto_dead_lettered", 1, { message_type: body.type });
 }