@takosjp/yurucommu-core 3.4.1 → 3.4.4

package/src/backend/lib/blocklist.ts

@@ -7,15 +7,21 @@
  * 4xx would cause sender instances to retry on a backoff, wasting their
  * delivery budget and ours.
  *
- * The helpers return `false` when the underlying read fails so that a
- * transient database error never causes federation traffic to be black-holed.
- * Each call site logs the failure so that the operator can investigate.
+ * The best-effort helpers return `false` when the underlying read fails so UI
+ * discovery can degrade without hiding every result. Federation ingress and
+ * delivery authority use the strict variants: an unavailable blocklist must
+ * make the message retryable, never silently route around an operator block.
  */
 
-import { eq, inArray } from "drizzle-orm";
+import { eq, inArray, sql, type SQL } from "drizzle-orm";
 
 import type { Database } from "../../db/index.ts";
-import { blockedActors, blockedDomains } from "../../db/index.ts";
+import { blockedActors, blockedDomains, runBatch } from "../../db/index.ts";
+import { normalizeActivityPubActorId } from "./activitypub-actor-identity.ts";
+import {
+  activityPubActorIdentityMatchesSql,
+  activityPubActorIdentitySetSql,
+} from "./activitypub-actor-identity-sql.ts";
 import { logger } from "./logger.ts";
 
 const log = logger.child({ component: "blocklist" });
@@ -30,6 +36,140 @@ const log = logger.child({ component: "blocklist" });
 // defederation bypass we must not allow).
 const BLOCKLIST_IN_CHUNK = 90;
 
+type RawSqlDatabase = {
+  all?: (query: SQL) => Promise<unknown[]>;
+  get?: (query: SQL) => Promise<unknown>;
+};
+
+function retainedBlockedActorIdsSql() {
+  return sql`SELECT ${blockedActors.actorApId} FROM ${blockedActors}`;
+}
+
+function retainedBlockedActorMatchesSql(actorApId: string) {
+  return activityPubActorIdentityMatchesSql(
+    retainedBlockedActorIdsSql(),
+    actorApId,
+  );
+}
+
+/**
+ * Return true when a row's actor expression is outside every blocked domain.
+ *
+ * The staged, materialized parser mirrors `normalizeDomain`: it folds scheme
+ * and authority case, removes an explicit port and one DNS root dot, rejects
+ * credentials, and leaves path/query bytes outside the host decision. Keeping
+ * each intermediate value named avoids the exponentially-expanded scalar SQL
+ * that can overflow SQLite's parser stack when embedded in a paginated query.
+ */
+function operatorActorDomainNotBlockedSql(actorApId: SQL): SQL {
+  return sql`
+    NOT EXISTS (
+      WITH
+      operator_actor_url(actor_id) AS MATERIALIZED (
+        SELECT ${actorApId}
+      ),
+      operator_actor_fragmentless(actor_id) AS MATERIALIZED (
+        SELECT CASE WHEN instr(actor_id, '#') > 0
+          THEN substr(actor_id, 1, instr(actor_id, '#') - 1)
+          ELSE actor_id
+        END
+        FROM operator_actor_url
+      ),
+      operator_actor_split(separator, scheme, tail) AS MATERIALIZED (
+        SELECT
+          instr(lower(actor_id), '://'),
+          substr(lower(actor_id), 1, instr(lower(actor_id), '://') - 1),
+          substr(lower(actor_id), instr(lower(actor_id), '://') + 3)
+        FROM operator_actor_fragmentless
+      ),
+      operator_actor_bounds(separator, scheme, tail, boundary) AS MATERIALIZED (
+        SELECT
+          separator,
+          scheme,
+          tail,
+          CASE
+            WHEN instr(tail, '/') = 0 THEN instr(tail, '?')
+            WHEN instr(tail, '?') = 0 THEN instr(tail, '/')
+            WHEN instr(tail, '/') < instr(tail, '?') THEN instr(tail, '/')
+            ELSE instr(tail, '?')
+          END
+        FROM operator_actor_split
+      ),
+      operator_actor_authority(separator, scheme, authority) AS MATERIALIZED (
+        SELECT
+          separator,
+          scheme,
+          CASE WHEN boundary = 0
+            THEN tail
+            ELSE substr(tail, 1, boundary - 1)
+          END
+        FROM operator_actor_bounds
+      ),
+      operator_actor_host(separator, scheme, authority, host) AS MATERIALIZED (
+        SELECT
+          separator,
+          scheme,
+          authority,
+          CASE
+            WHEN substr(authority, 1, 1) = '[' AND instr(authority, ']') > 1
+              THEN substr(authority, 1, instr(authority, ']'))
+            WHEN instr(authority, ':') > 0
+              THEN substr(authority, 1, instr(authority, ':') - 1)
+            ELSE authority
+          END
+        FROM operator_actor_authority
+      ),
+      operator_actor_hostname(hostname) AS MATERIALIZED (
+        SELECT CASE WHEN substr(host, -1) = '.'
+          THEN substr(host, 1, length(host) - 1)
+          ELSE host
+        END
+        FROM operator_actor_host
+        WHERE separator > 1
+          AND scheme IN ('http', 'https')
+          AND length(authority) > 0
+          AND instr(authority, '@') = 0
+      )
+      SELECT 1
+      FROM operator_actor_hostname
+      INNER JOIN ${blockedDomains}
+        ON operator_actor_hostname.hostname = ${blockedDomains.domain}
+        OR substr(
+          operator_actor_hostname.hostname,
+          -length('.' || ${blockedDomains.domain})
+        ) = '.' || ${blockedDomains.domain}
+    )
+  `;
+}
+
+/**
+ * SQL read predicate for identities that are visible under the operator
+ * blocklist. It covers exact and retained cosmetic actor spellings plus exact
+ * host/parent-domain blocks. Callers apply it before pagination; block rows and
+ * relationship edges remain stored so an unblock stays reversible.
+ */
+export function operatorActorNotBlockedSql(actorApId: SQL): SQL {
+  const retainedActorMatches = activityPubActorIdentityMatchesSql(
+    retainedBlockedActorIdsSql(),
+    actorApId,
+  );
+
+  return sql`
+    NOT EXISTS (${retainedActorMatches})
+    AND ${operatorActorDomainNotBlockedSql(actorApId)}
+  `;
+}
+
+function retainedBlockedActorKeySql(actorApId: string) {
+  const matches = retainedBlockedActorMatchesSql(actorApId);
+  return sql`
+    SELECT actor_id
+    FROM (${matches}) AS retained_blocked_actor_matches
+    ORDER BY actor_id
+    LIMIT 1
+  `;
+}
+
 /**
  * Normalise an actor AP-ID hostname for blocklist lookups: lowercase and
  * strip any trailing dot (DNS root form). Returns `null` when the input
@@ -73,25 +213,37 @@ export async function isDomainBlocked(
   db: Database,
   hostnameOrUrl: string,
 ): Promise<boolean> {
-  const domain = normalizeDomain(hostnameOrUrl);
-  if (!domain) return false;
-
   try {
-    const row = await db.query.blockedDomains.findFirst({
-      where: inArray(blockedDomains.domain, domainSuffixCandidates(domain)),
-      columns: { domain: true },
-    });
-    return !!row;
+    return await isDomainBlockedStrict(db, hostnameOrUrl);
   } catch (err) {
+    const domain = normalizeDomain(hostnameOrUrl);
     log.warn("blocklist.isDomainBlocked failed", {
       event: "blocklist.domain_lookup_failed",
-      domain,
+      domain: domain ?? hostnameOrUrl,
       error: err,
     });
     return false;
   }
 }
 
+/**
+ * Authority-bearing domain lookup. Unlike {@link isDomainBlocked}, a database
+ * read failure is propagated so the caller can retry instead of routing around
+ * an operator block whose state could not be read.
+ */
+export async function isDomainBlockedStrict(
+  db: Database,
+  hostnameOrUrl: string,
+): Promise<boolean> {
+  const domain = normalizeDomain(hostnameOrUrl);
+  if (!domain) return false;
+  const row = await db.query.blockedDomains.findFirst({
+    where: inArray(blockedDomains.domain, domainSuffixCandidates(domain)),
+    columns: { domain: true },
+  });
+  return !!row;
+}
+
 /**
  * Returns `true` when the operator has blocked the given actor AP-ID, or
  * (transitively) when the actor's hostname is blocked.
@@ -100,16 +252,8 @@ export async function isActorBlocked(
   db: Database,
   actorApId: string,
 ): Promise<boolean> {
-  if (typeof actorApId !== "string" || actorApId.length === 0) {
-    return false;
-  }
-
   try {
-    const row = await db.query.blockedActors.findFirst({
-      where: eq(blockedActors.actorApId, actorApId),
-      columns: { actorApId: true },
-    });
-    if (row) return true;
+    return await isActorBlockedStrict(db, actorApId);
   } catch (err) {
     log.warn("blocklist.isActorBlocked failed", {
       event: "blocklist.actor_lookup_failed",
@@ -118,23 +262,56 @@ export async function isActorBlocked(
     });
     return false;
   }
+}
+
+/**
+ * Authority-bearing actor/domain lookup. Database failures propagate so
+ * ingress or delivery work stays retryable and no possibly-blocked peer is
+ * admitted merely because moderation authority was temporarily unavailable.
+ */
+export async function isActorBlockedStrict(
+  db: Database,
+  actorApId: string,
+): Promise<boolean> {
+  if (typeof actorApId !== "string" || actorApId.length === 0) {
+    return false;
+  }
+
+  const row = await db.query.blockedActors.findFirst({
+    where: eq(blockedActors.actorApId, actorApId),
+    columns: { actorApId: true },
+  });
+  if (row) return true;
+
+  // Operator rows written before the verified-key-owner invariant may carry
+  // an accepted cosmetic spelling. Match the complete retained set in SQL:
+  // a fixed prefix silently turned an older block into fail-open ingress.
+  const rawDb = db as unknown as RawSqlDatabase;
+  if (typeof rawDb.get === "function") {
+    const matches = retainedBlockedActorMatchesSql(actorApId);
+    const retained = (await rawDb.get(sql`
+        SELECT (${matches}) AS actor_id
+      `)) as { actor_id?: string | null } | undefined;
+    if (retained?.actor_id) return true;
+  }
 
   const domain = normalizeDomain(actorApId);
   if (!domain) return false;
-  return await isDomainBlocked(db, domain);
+  return await isDomainBlockedStrict(db, domain);
 }
 
 /**
  * Batched blocklist filter for a list of recipient actor AP-IDs: returns the
  * SUBSET that is blocked (by actor OR transitively by hostname) using exactly
  * two queries (blocked_actors + blocked_domains) instead of two-per-recipient.
- * Replaces an O(recipients) serial `isActorBlocked` loop on the delivery
- * fan-out hot path. Fail-open like the singular helpers: a read error yields an
- * empty blocked set so a transient DB error never black-holes federation.
+ * Replaces an O(recipients) serial `isActorBlocked` loop. The default remains
+ * best-effort for UI discovery; delivery authority uses the strict wrapper
+ * below so an unreadable blocklist cannot become a defederation bypass.
  */
 export async function filterBlockedActorApIds(
   db: Database,
   actorApIds: string[],
+  options: { readonly strict?: boolean } = {},
 ): Promise<Set<string>> {
   const blocked = new Set<string>();
   const uniqueIds = [...new Set(actorApIds.filter((id) => id.length > 0))];
@@ -143,9 +320,8 @@ export async function filterBlockedActorApIds(
   try {
     // Chunk the IN(...) lookups so a very large recipient set (e.g. a big
     // community fan-out, tens of thousands of unique actors/domains) cannot
-    // exceed SQLite's bound-parameter ceiling and throw — which, under the
-    // fail-open catch below, would silently DISABLE the operator blocklist for
-    // that whole fan-out (a defederation bypass).
+    // exceed SQLite's bound-parameter ceiling. Strict delivery callers retry
+    // such failures; best-effort discovery callers degrade to an empty set.
     const blockedActorSet = new Set<string>();
     for (let i = 0; i < uniqueIds.length; i += BLOCKLIST_IN_CHUNK) {
       const rows = await db
@@ -159,6 +335,43 @@ export async function filterBlockedActorApIds(
         );
       for (const r of rows) blockedActorSet.add(r.actorApId);
     }
+    // Resolve legacy cosmetic spellings only against the requested identities.
+    // Each chunk stays below D1's parameter ceiling; the retained operator set
+    // is complete, and path/query bytes remain case-sensitive in the shared SQL
+    // identity expansion.
+    const retainedActorIdentities = new Set<string>();
+    const requestedIdentities = [
+      ...new Set(
+        uniqueIds
+          .map((id) => normalizeActivityPubActorId(id))
+          .filter((identity): identity is string => identity !== null),
+      ),
+    ];
+    const rawDb = db as unknown as RawSqlDatabase;
+    if (typeof rawDb.all === "function") {
+      for (let i = 0; i < requestedIdentities.length; i += BLOCKLIST_IN_CHUNK) {
+        const requestedChunk = requestedIdentities.slice(
+          i,
+          i + BLOCKLIST_IN_CHUNK,
+        );
+        const retainedIdentitySet = activityPubActorIdentitySetSql(
+          retainedBlockedActorIdsSql(),
+        );
+        const rows = (await rawDb.all(sql`
+          SELECT actor_id
+          FROM (${retainedIdentitySet}) AS retained_actor_identities
+          WHERE actor_id IN (${sql.join(
+            requestedChunk.map((identity) => sql`${identity}`),
+            sql`, `,
+          )})
+        `)) as Array<{ actor_id?: string }>;
+        for (const row of rows) {
+          if (typeof row.actor_id === "string") {
+            retainedActorIdentities.add(row.actor_id);
+          }
+        }
+      }
+    }
 
     // Expand each actor's hostname to its parent-domain candidates so a domain
     // block covers subdomains (see domainSuffixCandidates) — querying the union
@@ -190,8 +403,11 @@ export async function filterBlockedActorApIds(
 
     for (const id of uniqueIds) {
       const candidates = idToCandidates.get(id) ?? [];
+      const normalizedActorId = normalizeActivityPubActorId(id);
       if (
         blockedActorSet.has(id) ||
+        (normalizedActorId !== null &&
+          retainedActorIdentities.has(normalizedActorId)) ||
         candidates.some((c) => blockedDomainSet.has(c))
       ) {
         blocked.add(id);
@@ -202,11 +418,20 @@ export async function filterBlockedActorApIds(
       event: "blocklist.batch_lookup_failed",
       error: err,
     });
-    return new Set(); // fail-open
+    if (options.strict) throw err;
+    return new Set();
   }
   return blocked;
 }
 
+/** Strict batch lookup for ingress/delivery authority. */
+export async function filterBlockedActorApIdsStrict(
+  db: Database,
+  actorApIds: string[],
+): Promise<Set<string>> {
+  return await filterBlockedActorApIds(db, actorApIds, { strict: true });
+}
+
 /**
  * Insert (or update) a domain blocklist entry. Idempotent: re-blocking the
  * same domain refreshes the recorded reason but keeps the original
@@ -258,13 +483,24 @@ export async function blockActor(
   if (typeof actorApId !== "string" || actorApId.length === 0) {
     throw new Error("blocklist.blockActor: actorApId is required");
   }
-  await db
-    .insert(blockedActors)
-    .values({ actorApId, reason })
-    .onConflictDoUpdate({
-      target: blockedActors.actorApId,
-      set: { reason },
-    });
+  const retainedKey = sql`COALESCE((${retainedBlockedActorKeySql(actorApId)}), ${actorApId})`;
+  const equivalentRows = retainedBlockedActorMatchesSql(actorApId);
+  await runBatch(db, [
+    db
+      .insert(blockedActors)
+      .values({ actorApId: retainedKey, reason })
+      .onConflictDoUpdate({
+        target: blockedActors.actorApId,
+        set: { reason },
+      }),
+    db.delete(blockedActors).where(sql`
+      ${blockedActors.actorApId} IN (${equivalentRows})
+      AND ${blockedActors.actorApId} <> COALESCE(
+        (${retainedBlockedActorKeySql(actorApId)}),
+        ${actorApId}
+      )
+    `),
+  ]);
 }
 
 /**
@@ -275,5 +511,8 @@ export async function unblockActor(
   actorApId: string,
 ): Promise<void> {
   if (typeof actorApId !== "string" || actorApId.length === 0) return;
-  await db.delete(blockedActors).where(eq(blockedActors.actorApId, actorApId));
+  const equivalentRows = retainedBlockedActorMatchesSql(actorApId);
+  await db
+    .delete(blockedActors)
+    .where(sql`${blockedActors.actorApId} IN (${equivalentRows})`);
 }

package/src/backend/lib/community-visibility.ts

@@ -7,11 +7,11 @@
  * what keeps them out of the public / home / following feeds (which filter on
  * `audienceJson = "[]"`).
  *
- * For a PRIVATE community this is not enough: a single-object fetch (GET a post,
- * a reply, or an `/ap/objects/:id`) bypasses the audience filter entirely, so a
- * "public"-visibility community post would leak to any caller. This module
- * centralizes the membership gate those single-object paths must apply on top of
- * the normal visibility check.
+ * For a PRIVATE or soft-deleted community this is not enough: a single-object
+ * fetch (GET a post, a reply, or an `/ap/objects/:id`) bypasses the audience
+ * filter entirely, so a "public"-visibility community post would leak to any
+ * caller. This module centralizes the lifecycle + membership gate those
+ * single-object paths must apply on top of the normal visibility check.
  *
  * Membership model: `community_members` has no status column — the presence of a
  * row IS the acceptance (this mirrors `resolveCommunityRead` in routes/timeline.ts).
@@ -96,10 +96,11 @@ function communityApIdsFor(obj: CommunityGateObject): string[] {
 /**
  * Single-object community read-gate.
  *
- * If `obj` is addressed to a community whose `visibility` is "private", this
+ * If `obj` is addressed to a soft-deleted community, this always returns false.
+ * If it is addressed to a live community whose `visibility` is "private", this
  * returns `true` only when `viewerApId` is an accepted member (a row in
- * `community_members`) of that community; otherwise it returns `true` and leaves
- * the normal public/followers/direct visibility check to the caller.
+ * `community_members`) of that community; otherwise it returns `true` and
+ * leaves the normal public/followers/direct visibility check to the caller.
  *
  * An anonymous viewer (`viewerApId` null/undefined) against a private community
  * always returns `false`. This NEVER widens access: a non-community or
@@ -114,17 +115,23 @@ export async function canViewerReadObject(
   const communityIds = communityApIdsFor(obj);
   if (communityIds.length === 0) return true;
 
-  // Only PRIVATE communities gate single-object reads. Resolve which (if any)
-  // of the addressed communities are private in one batched query.
-  const privateRows = await db
-    .select({ apId: communities.apId })
+  // Resolve local lifecycle + visibility in one query. Unknown IDs are left to
+  // the caller's ordinary visibility rules (they may name a remote audience),
+  // while any known soft-deleted local community fails closed.
+  const communityRows = await db
+    .select({
+      apId: communities.apId,
+      visibility: communities.visibility,
+      deletedAt: communities.deletedAt,
+    })
     .from(communities)
-    .where(
-      and(
-        inArray(communities.apId, communityIds),
-        inArray(communities.visibility, MEMBERSHIP_REQUIRED_VISIBILITIES),
-      ),
-    );
+    .where(inArray(communities.apId, communityIds));
+
+  if (communityRows.some((row) => row.deletedAt !== null)) return false;
+
+  const privateRows = communityRows.filter((row) =>
+    communityRequiresMembership(row.visibility),
+  );
 
   if (privateRows.length === 0) return true;
 
@@ -154,9 +161,10 @@ export async function canViewerReadObject(
  * Returns the set of `apId`s that PASS the community read-gate, in TWO queries
  * total (private-community lookup + viewer-membership lookup) instead of the
  * 1-2 queries PER object the per-row form costs. The per-object semantics are
- * identical: an object passes unless it is addressed to a private community the
- * viewer is not a member of (an anonymous viewer never satisfies a private
- * community). Objects with no community audience always pass.
+ * identical: an object fails when it is addressed to a deleted community, and
+ * otherwise passes unless it is addressed to a private community the viewer is
+ * not a member of (an anonymous viewer never satisfies a private community).
+ * Objects with no community audience always pass.
  *
  * Both IN(...) lookups are chunked to stay under D1's 100-bound-parameter cap,
  * since a page can address up to ~90 distinct communities.
@@ -183,19 +191,24 @@ export async function communityReadableApIds<
     return readable;
   }
 
-  // Which addressed communities are private? (only private communities gate)
+  // Resolve local lifecycle + visibility. Unknown IDs may be remote audiences
+  // and remain governed by the caller's ordinary visibility rules.
   const privateSet = new Set<string>();
+  const deletedSet = new Set<string>();
   for (const chunk of chunkForInClause([...unionIds])) {
     const rows = await db
-      .select({ apId: communities.apId })
+      .select({
+        apId: communities.apId,
+        visibility: communities.visibility,
+        deletedAt: communities.deletedAt,
+      })
       .from(communities)
-      .where(
-        and(
-          inArray(communities.apId, chunk),
-          inArray(communities.visibility, MEMBERSHIP_REQUIRED_VISIBILITIES),
-        ),
-      );
-    for (const r of rows) privateSet.add(r.apId);
+      .where(inArray(communities.apId, chunk));
+    for (const row of rows) {
+      if (row.deletedAt !== null) deletedSet.add(row.apId);
+      else if (communityRequiresMembership(row.visibility))
+        privateSet.add(row.apId);
+    }
   }
 
   // Which of those private communities is the viewer a member of?
@@ -216,9 +229,10 @@ export async function communityReadableApIds<
   }
 
   for (const o of objs) {
-    const privateForObj = (perObjectCommunityIds.get(o.apId) ?? []).filter(
-      (id) => privateSet.has(id),
-    );
+    const communityIds = perObjectCommunityIds.get(o.apId) ?? [];
+    if (communityIds.some((id) => deletedSet.has(id))) continue;
+
+    const privateForObj = communityIds.filter((id) => privateSet.has(id));
     if (privateForObj.length === 0) {
       readable.add(o.apId); // not addressed to any private community
     } else if (privateForObj.some((id) => viewerMemberSet.has(id))) {