@takosjp/yurucommu-core 3.4.3 → 3.4.4

package/src/backend/lib/delivery/resolution-outbox.ts

@@ -0,0 +1,456 @@
+import { and, asc, eq, inArray, lte, or, sql } from "drizzle-orm";
+
+import {
+  affectedRowCount,
+  deliveryResolutions,
+  insertMany,
+  runBatch,
+  type D1Statement,
+  type Database,
+} from "../../../db/index.ts";
+import type { Env } from "../../types.ts";
+import type { IQueueProducer } from "../../runtime/queue.ts";
+import type { DeliveryQueueMessageV1 } from "./types.ts";
+import { sha256Hex } from "./transformers.ts";
+
+const OUTBOX_SCAN_LIMIT = 50;
+const STALE_RESOLUTION_MS = 2 * 60 * 1000;
+const TERMINAL_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
+// Keep the follow-up DELETE (ids + statuses + cutoff) under D1's 100-bind cap.
+const TERMINAL_PURGE_LIMIT = 50;
+export const MAX_RESOLVE_ATTEMPTS = 8;
+
+const TERMINAL_STATUSES = ["resolved", "failed", "discarded"] as const;
+
+export type DeliveryResolutionIntent = {
+  readonly activityId: string;
+  readonly recipientActorApId: string;
+};
+
+export type DeliveryResolutionClaim =
+  | { readonly state: "missing" }
+  | { readonly state: "terminal" }
+  | { readonly state: "deferred"; readonly delaySeconds: number }
+  | { readonly state: "busy"; readonly delaySeconds: number }
+  | {
+      readonly state: "claimed";
+      readonly id: string;
+      readonly processingToken: string;
+      readonly attempts: number;
+    };
+
+export async function computeDeliveryResolutionId(
+  activityId: string,
+  recipientActorApId: string,
+): Promise<string> {
+  return sha256Hex(`resolve_actor|${activityId}|${recipientActorApId}`);
+}
+
+export function buildResolveActorMessage(
+  activityId: string,
+  recipientActorApId: string,
+  attempts = 0,
+): DeliveryQueueMessageV1 {
+  return {
+    version: 1,
+    type: "resolve_actor",
+    activityId,
+    recipientActorApId,
+    ...(attempts > 0 ? { attempts } : {}),
+    scheduledAt: new Date().toISOString(),
+  };
+}
+
+async function materializeIntents(
+  intents: readonly DeliveryResolutionIntent[],
+) {
+  const unique = new Map<string, DeliveryResolutionIntent>();
+  for (const intent of intents) {
+    unique.set(`${intent.activityId}\0${intent.recipientActorApId}`, intent);
+  }
+  return await Promise.all(
+    [...unique.values()].map(async (intent) => ({
+      id: await computeDeliveryResolutionId(
+        intent.activityId,
+        intent.recipientActorApId,
+      ),
+      ...intent,
+    })),
+  );
+}
+
+export type PreparedDeliveryResolutionJobs = {
+  readonly rows: ReadonlyArray<
+    DeliveryResolutionIntent & { readonly id: string }
+  >;
+  readonly statements: readonly D1Statement[];
+};
+
+/** Build durable first-hop intent inserts without executing them. */
+export async function prepareDeliveryResolutionJobs(
+  db: Database,
+  intents: readonly DeliveryResolutionIntent[],
+): Promise<PreparedDeliveryResolutionJobs> {
+  const rows = await materializeIntents(intents);
+  if (rows.length === 0) return { rows, statements: [] };
+  const now = new Date().toISOString();
+  const statements = insertMany(
+    db,
+    deliveryResolutions,
+    rows.map((row) => ({
+      id: row.id,
+      activityApId: row.activityId,
+      recipientActorApId: row.recipientActorApId,
+      status: "pending",
+      attempts: 0,
+      nextAttemptAt: now,
+      createdAt: now,
+      updatedAt: now,
+    })),
+    { conflict: "ignore" },
+  );
+  return { rows, statements };
+}
+
+/** Persist intent before attempting any Queue RPC. */
+export async function persistDeliveryResolutionJobs(
+  db: Database,
+  intents: readonly DeliveryResolutionIntent[],
+): Promise<ReadonlyArray<DeliveryResolutionIntent & { readonly id: string }>> {
+  const prepared = await prepareDeliveryResolutionJobs(db, intents);
+  if (prepared.statements.length > 0) {
+    await runBatch(
+      db,
+      prepared.statements as [
+        (typeof prepared.statements)[number],
+        ...(typeof prepared.statements)[number][],
+      ],
+    );
+  }
+  return prepared.rows;
+}
+
+async function sendRows(
+  db: Database,
+  queue: IQueueProducer<DeliveryQueueMessageV1>,
+  rows: readonly {
+    readonly id: string;
+    readonly activityApId: string;
+    readonly recipientActorApId: string;
+    readonly attempts: number;
+  }[],
+  now: string,
+): Promise<number> {
+  if (rows.length === 0) return 0;
+  await queue.sendBatch(
+    rows.map((row) => ({
+      body: buildResolveActorMessage(
+        row.activityApId,
+        row.recipientActorApId,
+        row.attempts,
+      ),
+    })),
+  );
+  await db
+    .update(deliveryResolutions)
+    .set({ status: "queued", processingToken: null, updatedAt: now })
+    .where(
+      and(
+        inArray(
+          deliveryResolutions.id,
+          rows.map((row) => row.id),
+        ),
+        inArray(deliveryResolutions.status, ["pending", "retry_wait"]),
+      ),
+    );
+  return rows.length;
+}
+
+/** Persist a bounded recipient set, then publish only currently-due rows. */
+export async function enqueueDeliveryResolutionJobs(
+  db: Database,
+  queue: IQueueProducer<DeliveryQueueMessageV1> | undefined,
+  intents: readonly DeliveryResolutionIntent[],
+  options: { readonly reopenTerminal?: boolean } = {},
+): Promise<number> {
+  const materialized = await persistDeliveryResolutionJobs(db, intents);
+  if (!queue || materialized.length === 0) return 0;
+  const now = new Date().toISOString();
+  let sent = 0;
+  for (
+    let offset = 0;
+    offset < materialized.length;
+    offset += OUTBOX_SCAN_LIMIT
+  ) {
+    const ids = materialized
+      .slice(offset, offset + OUTBOX_SCAN_LIMIT)
+      .map((row) => row.id);
+    if (options.reopenTerminal) {
+      // A 404/410 endpoint invalidation is a new resolution generation for the
+      // same (Activity, actor) pair. The deterministic row may already be
+      // terminal from the original successful resolution, so reactivate it
+      // before publishing the replacement wakeup.
+      await db
+        .update(deliveryResolutions)
+        .set({
+          status: "pending",
+          processingToken: null,
+          attempts: 0,
+          nextAttemptAt: now,
+          lastError: null,
+          updatedAt: now,
+          resolvedAt: null,
+        })
+        .where(
+          and(
+            inArray(deliveryResolutions.id, ids),
+            inArray(deliveryResolutions.status, [...TERMINAL_STATUSES]),
+          ),
+        );
+    }
+    const due = await db
+      .select({
+        id: deliveryResolutions.id,
+        activityApId: deliveryResolutions.activityApId,
+        recipientActorApId: deliveryResolutions.recipientActorApId,
+        attempts: deliveryResolutions.attempts,
+      })
+      .from(deliveryResolutions)
+      .where(
+        and(
+          inArray(deliveryResolutions.id, ids),
+          inArray(deliveryResolutions.status, ["pending", "retry_wait"]),
+          lte(deliveryResolutions.nextAttemptAt, now),
+        ),
+      );
+    sent += await sendRows(db, queue, due, now);
+  }
+  return sent;
+}
+
+/** Recover due or stale first-hop rows. Safe after requests, queue batches, and cron. */
+export async function enqueuePendingDeliveryResolutionJobs(
+  env: Env,
+  nowDate = new Date(),
+): Promise<number> {
+  const db = env.DB_INSTANCE;
+  const now = nowDate.toISOString();
+  await purgeTerminalDeliveryResolutionJobs(db, nowDate);
+  if (!env.DELIVERY_QUEUE || !env.DELIVERY_DLQ) return 0;
+
+  const staleBefore = new Date(
+    nowDate.getTime() - STALE_RESOLUTION_MS,
+  ).toISOString();
+  await db
+    .update(deliveryResolutions)
+    .set({ status: "pending", processingToken: null, updatedAt: now })
+    .where(
+      and(
+        inArray(deliveryResolutions.status, ["queued", "processing"]),
+        lte(deliveryResolutions.updatedAt, staleBefore),
+      ),
+    );
+
+  const rows = await db
+    .select({
+      id: deliveryResolutions.id,
+      activityApId: deliveryResolutions.activityApId,
+      recipientActorApId: deliveryResolutions.recipientActorApId,
+      attempts: deliveryResolutions.attempts,
+    })
+    .from(deliveryResolutions)
+    .where(
+      and(
+        inArray(deliveryResolutions.status, ["pending", "retry_wait"]),
+        lte(deliveryResolutions.nextAttemptAt, now),
+      ),
+    )
+    .orderBy(asc(deliveryResolutions.createdAt))
+    .limit(OUTBOX_SCAN_LIMIT);
+  return await sendRows(db, env.DELIVERY_QUEUE, rows, now);
+}
+
+export async function claimDeliveryResolutionJob(
+  db: Database,
+  activityId: string,
+  recipientActorApId: string,
+  nowDate = new Date(),
+): Promise<DeliveryResolutionClaim> {
+  const id = await computeDeliveryResolutionId(activityId, recipientActorApId);
+  const row = await db
+    .select({
+      status: deliveryResolutions.status,
+      attempts: deliveryResolutions.attempts,
+      nextAttemptAt: deliveryResolutions.nextAttemptAt,
+      updatedAt: deliveryResolutions.updatedAt,
+    })
+    .from(deliveryResolutions)
+    .where(eq(deliveryResolutions.id, id))
+    .get();
+  if (!row) return { state: "missing" };
+  if ((TERMINAL_STATUSES as readonly string[]).includes(row.status)) {
+    return { state: "terminal" };
+  }
+
+  const nowMs = nowDate.getTime();
+  const nextMs = Date.parse(row.nextAttemptAt);
+  if (
+    row.status === "retry_wait" &&
+    Number.isFinite(nextMs) &&
+    nextMs > nowMs
+  ) {
+    return {
+      state: "deferred",
+      delaySeconds: Math.max(1, Math.ceil((nextMs - nowMs) / 1000)),
+    };
+  }
+
+  const now = nowDate.toISOString();
+  const staleBefore = new Date(nowMs - STALE_RESOLUTION_MS).toISOString();
+  const processingToken = crypto.randomUUID();
+  const claim = await db
+    .update(deliveryResolutions)
+    .set({ status: "processing", processingToken, updatedAt: now })
+    .where(
+      and(
+        eq(deliveryResolutions.id, id),
+        or(
+          inArray(deliveryResolutions.status, ["pending", "queued"]),
+          and(
+            eq(deliveryResolutions.status, "retry_wait"),
+            lte(deliveryResolutions.nextAttemptAt, now),
+          ),
+          and(
+            eq(deliveryResolutions.status, "processing"),
+            lte(deliveryResolutions.updatedAt, staleBefore),
+          ),
+        ),
+      ),
+    );
+  if (affectedRowCount(claim) === 0) {
+    return { state: "busy", delaySeconds: 60 };
+  }
+  return { state: "claimed", id, processingToken, attempts: row.attempts };
+}
+
+export async function retryDeliveryResolutionJob(
+  db: Database,
+  claim: Extract<DeliveryResolutionClaim, { state: "claimed" }>,
+  error: unknown,
+  delaySeconds = 60,
+): Promise<{ readonly owned: boolean; readonly terminal: boolean }> {
+  const attempts = claim.attempts + 1;
+  const terminal = attempts >= MAX_RESOLVE_ATTEMPTS;
+  const now = new Date();
+  const result = await db
+    .update(deliveryResolutions)
+    .set({
+      status: terminal ? "failed" : "retry_wait",
+      processingToken: null,
+      attempts,
+      nextAttemptAt: new Date(
+        now.getTime() + delaySeconds * 1000,
+      ).toISOString(),
+      lastError: error instanceof Error ? error.message : String(error),
+      updatedAt: now.toISOString(),
+      resolvedAt: terminal ? now.toISOString() : null,
+    })
+    .where(
+      and(
+        eq(deliveryResolutions.id, claim.id),
+        eq(deliveryResolutions.processingToken, claim.processingToken),
+      ),
+    );
+  return { owned: affectedRowCount(result) > 0, terminal };
+}
+
+/**
+ * Release a claim after a local authority dependency fails without charging
+ * the remote actor's bounded resolution-attempt budget. The durable row stays
+ * retryable, while the processing token prevents an expired worker from
+ * overwriting a newer claim.
+ */
+export async function deferDeliveryResolutionJob(
+  db: Database,
+  claim: Extract<DeliveryResolutionClaim, { state: "claimed" }>,
+  error: unknown,
+  delaySeconds = 60,
+): Promise<boolean> {
+  const now = new Date();
+  const result = await db
+    .update(deliveryResolutions)
+    .set({
+      status: "retry_wait",
+      processingToken: null,
+      nextAttemptAt: new Date(
+        now.getTime() + delaySeconds * 1000,
+      ).toISOString(),
+      lastError: error instanceof Error ? error.message : String(error),
+      updatedAt: now.toISOString(),
+      resolvedAt: null,
+    })
+    .where(
+      and(
+        eq(deliveryResolutions.id, claim.id),
+        eq(deliveryResolutions.processingToken, claim.processingToken),
+      ),
+    );
+  return affectedRowCount(result) > 0;
+}
+
+export async function completeDeliveryResolutionJob(
+  db: Database,
+  claim: Extract<DeliveryResolutionClaim, { state: "claimed" }>,
+  status: "resolved" | "discarded",
+  reason?: string,
+): Promise<boolean> {
+  const now = new Date().toISOString();
+  const result = await db
+    .update(deliveryResolutions)
+    .set({
+      status,
+      processingToken: null,
+      lastError: reason ?? null,
+      updatedAt: now,
+      resolvedAt: now,
+    })
+    .where(
+      and(
+        eq(deliveryResolutions.id, claim.id),
+        eq(deliveryResolutions.processingToken, claim.processingToken),
+      ),
+    );
+  return affectedRowCount(result) > 0;
+}
+
+export async function purgeTerminalDeliveryResolutionJobs(
+  db: Database,
+  nowDate = new Date(),
+): Promise<number> {
+  const cutoff = new Date(
+    nowDate.getTime() - TERMINAL_RETENTION_MS,
+  ).toISOString();
+  const rows = await db
+    .select({ id: deliveryResolutions.id })
+    .from(deliveryResolutions)
+    .where(
+      and(
+        inArray(deliveryResolutions.status, [...TERMINAL_STATUSES]),
+        lte(deliveryResolutions.updatedAt, cutoff),
+      ),
+    )
+    .orderBy(asc(deliveryResolutions.updatedAt))
+    .limit(TERMINAL_PURGE_LIMIT);
+  if (rows.length === 0) return 0;
+  const result = await db.delete(deliveryResolutions).where(
+    and(
+      inArray(
+        deliveryResolutions.id,
+        rows.map((row) => row.id),
+      ),
+      inArray(deliveryResolutions.status, [...TERMINAL_STATUSES]),
+      lte(deliveryResolutions.updatedAt, cutoff),
+    ),
+  );
+  return affectedRowCount(result);
+}

package/src/backend/lib/delivery/types.ts

@@ -1,6 +1,16 @@
 export const DELIVERY_QUEUE_MESSAGE_VERSION = 1 as const;
 
-export type DeliveryFanoutFollowersMessageV1 = {
+/**
+ * Bounded generation for a MAIN-queue message that exhausted Cloudflare's
+ * retries, reached the DLQ as its raw body, and was sent back to the MAIN
+ * queue. Optional keeps existing v1 producers and in-flight messages wire-
+ * compatible; absent means no prior redrive.
+ */
+export type DeliveryAutoDlqStateV1 = {
+  autoDlqAttempt?: number;
+};
+
+export type DeliveryFanoutFollowersMessageV1 = DeliveryAutoDlqStateV1 & {
   version: typeof DELIVERY_QUEUE_MESSAGE_VERSION;
   type: "fanout_followers";
   activityId: string;
@@ -15,7 +25,7 @@ export type DeliveryFanoutFollowersMessageV1 = {
  * followers) rather than the author's personal follower graph. Used for
  * community-scoped posts so reach == community, not author-followers.
  */
-export type DeliveryFanoutCommunityMessageV1 = {
+export type DeliveryFanoutCommunityMessageV1 = DeliveryAutoDlqStateV1 & {
   version: typeof DELIVERY_QUEUE_MESSAGE_VERSION;
   type: "fanout_community";
   activityId: string;
@@ -33,7 +43,7 @@ export type DeliveryFanoutCommunityMessageV1 = {
   scheduledAt: string; // ISO8601 UTC
 };
 
-export type DeliveryResolveActorMessageV1 = {
+export type DeliveryResolveActorMessageV1 = DeliveryAutoDlqStateV1 & {
   version: typeof DELIVERY_QUEUE_MESSAGE_VERSION;
   type: "resolve_actor";
   activityId: string;
@@ -46,7 +56,7 @@ export type DeliveryResolveActorMessageV1 = {
   scheduledAt: string; // ISO8601 UTC
 };
 
-export type DeliveryDeliverEndpointMessageV1 = {
+export type DeliveryDeliverEndpointMessageV1 = DeliveryAutoDlqStateV1 & {
   version: typeof DELIVERY_QUEUE_MESSAGE_VERSION;
   type: "deliver_endpoint";
   jobId: string;
@@ -58,7 +68,7 @@ export type DeliveryDeliverEndpointMessageV1 = {
   scheduledAt: string; // ISO8601 UTC
 };
 
-export type DeliveryReconcileJobMessageV1 = {
+export type DeliveryReconcileJobMessageV1 = DeliveryAutoDlqStateV1 & {
   version: typeof DELIVERY_QUEUE_MESSAGE_VERSION;
   type: "reconcile_job";
   jobId: string;
@@ -70,7 +80,7 @@ export type DeliveryReconcileJobMessageV1 = {
  * Product notification delivery. The queue carries only the durable outbox id;
  * device pushkeys, gateway URLs, and notification content remain in the DB.
  */
-export type DeliveryNotificationPushMessageV1 = {
+export type DeliveryNotificationPushMessageV1 = DeliveryAutoDlqStateV1 & {
   version: typeof DELIVERY_QUEUE_MESSAGE_VERSION;
   type: "notification_push";
   jobId: string;
@@ -106,6 +116,7 @@ export function isDeliveryQueueMessageV1(
   const v = value as Record<string, unknown>;
   if (v.version !== DELIVERY_QUEUE_MESSAGE_VERSION) return false;
   if (typeof v.type !== "string") return false;
+  if (!isOptionalNonNegativeInteger(v.autoDlqAttempt)) return false;
 
   switch (v.type) {
     case "fanout_followers":
@@ -130,10 +141,15 @@ export function isDeliveryQueueMessageV1(
       return (
         typeof v.activityId === "string" &&
         typeof v.recipientActorApId === "string" &&
+        isOptionalNonNegativeInteger(v.attempts) &&
         typeof v.scheduledAt === "string"
       );
     case "deliver_endpoint":
-      return typeof v.jobId === "string" && typeof v.scheduledAt === "string";
+      return (
+        typeof v.jobId === "string" &&
+        isOptionalNonNegativeInteger(v.reconcileAttempt) &&
+        typeof v.scheduledAt === "string"
+      );
     case "reconcile_job":
       return (
         typeof v.jobId === "string" &&
@@ -147,6 +163,13 @@ export function isDeliveryQueueMessageV1(
   }
 }
 
+function isOptionalNonNegativeInteger(value: unknown): boolean {
+  return (
+    value === undefined ||
+    (typeof value === "number" && Number.isInteger(value) && value >= 0)
+  );
+}
+
 export function isDeliveryDlqMessageV1(
   value: unknown,
 ): value is DeliveryDlqMessageV1 {
@@ -160,6 +183,7 @@ export function isDeliveryDlqMessageV1(
     typeof v.endpoint === "string" &&
     typeof v.attempts === "number" &&
     (v.lastError === null || typeof v.lastError === "string") &&
+    isOptionalNonNegativeInteger(v.reconcileAttempt) &&
     typeof v.deadLetteredAt === "string"
   );
 }

package/src/backend/lib/feed-exclude.ts

@@ -1,48 +1,60 @@
-import { and, type AnyColumn, eq, notInArray, type SQL } from "drizzle-orm";
-import type { Database } from "../../db/index.ts";
+import { and, type AnyColumn, sql, type SQL } from "drizzle-orm";
 import { blocks, mutes, objects } from "../../db/index.ts";
+import { activityPubActorIdentitySetSql } from "./activitypub-actor-identity-sql.ts";
+import { operatorActorNotBlockedSql } from "./blocklist.ts";
 
 /**
  * Predicate excluding posts authored by anyone the viewer has blocked or muted.
  *
- * Expressed as `objects.attributed_to NOT IN (SELECT blocked) AND ... NOT IN
- * (SELECT muted)` — i.e. `NOT IN (blocked ∪ muted)` — using db.select
- * SUBQUERIES rather than materialising the id lists into an `inArray`. Each
- * subquery binds a single bound parameter (the viewer) regardless of how many
- * accounts the viewer has blocked/muted, so it never approaches Cloudflare D1's
- * 100-bound-parameter-per-query ceiling. (The previous `notInArray(attributedTo,
- * [...blocked, ...muted])` materialised up to ~2000 ids — fine on the libsql the
- * tests run on, but a "too many SQL variables" 500 on production D1 for a heavy
- * moderator.)
+ * Expressed as one uncorrelated `NOT IN` subquery over the union of raw +
+ * canonical block/mute identities. It binds only the viewer twice regardless
+ * of relation count, so it stays under Cloudflare D1's 100-parameter ceiling.
+ * The set CTE runs once per query rather than parsing actor URLs per feed row.
  *
  * Returns `undefined` for an empty viewer (anonymous) so callers can skip it.
  * Applying it unconditionally for a logged-in viewer is correct even with zero
- * blocks/mutes: `NOT IN (empty set)` is true for every row.
+ * blocks/mutes: `NOT IN (empty set)` is true.
  *
  * `column` defaults to the post author (`objects.attributedTo`) for feeds, but
  * can be any actor-id column — e.g. `activities.actorApId` so the notifications
  * list/count can suppress activity from blocked/muted accounts the same way.
  */
 export function excludeBlockedMutedAuthors(
-  db: Database,
   viewerApId: string,
   column: AnyColumn = objects.attributedTo,
 ): SQL | undefined {
   if (!viewerApId) return undefined;
-  return and(
-    notInArray(
-      column,
-      db
-        .select({ id: blocks.blockedApId })
-        .from(blocks)
-        .where(eq(blocks.blockerApId, viewerApId)),
-    ),
-    notInArray(
-      column,
-      db
-        .select({ id: mutes.mutedApId })
-        .from(mutes)
-        .where(eq(mutes.muterApId, viewerApId)),
-    ),
+  const excludedIds = activityPubActorIdentitySetSql(
+    sql`
+      SELECT ${blocks.blockedApId}
+      FROM ${blocks}
+      WHERE ${blocks.blockerApId} = ${viewerApId}
+      UNION ALL
+      SELECT ${mutes.mutedApId}
+      FROM ${mutes}
+      WHERE ${mutes.muterApId} = ${viewerApId}
+    `,
   );
+  return sql`${column} NOT IN (${excludedIds})`;
+}
+
+/**
+ * Global operator moderation plus the current viewer's personal block/mute
+ * filter for a projected actor identity. Unlike the personal-only helper this
+ * always returns a predicate: anonymous/global feeds must still hide retained
+ * content from an instance-defederated actor or domain.
+ *
+ * Apply this in SQL before cursor/offset/limit. Post-query filtering lets a run
+ * of blocked rows consume the page and can make later visible content
+ * unreachable or report incorrect pagination metadata.
+ */
+export function excludeModeratedActors(
+  viewerApId: string,
+  column: AnyColumn = objects.attributedTo,
+): SQL {
+  const operatorVisible = operatorActorNotBlockedSql(sql`${column}`);
+  const personalVisible = excludeBlockedMutedAuthors(viewerApId, column);
+  return personalVisible
+    ? and(operatorVisible, personalVisible)!
+    : operatorVisible;
 }