fedipod-server 0.28.1 → 0.30.0

package/lib/core/intake/activity.mjs

@@ -1,318 +0,0 @@
-// activity.mjs — what an inbound activity IS, before anything is done with
-// it: the AS2 types the fediverse actually posts, the caps that bound what a
-// stranger can make the agent hold, and the pure checks on ids, origins and
-// authorship that every other intake module shares.
-
-// A note's replies collection is rewritten WHOLE every time one is added, so
-// without a cap the bytes are quadratic in a number a stranger chooses.
-export const MAX_REPLIES_RECORDED = 500;
-// Ids of activities already forwarded to our followers (§7.1.2), so a re-drain
-// never re-broadcasts one. A ceiling on the record, not on forwarding.
-export const MAX_FORWARDED = 2000;
-// How many posts may wait for a group operator's decision. A ceiling, not a
-// window: full refuses the newest rather than dropping the oldest.
-export const MAX_PENDING_REVIEW = 500;
-// A group's membership, cached: it changes when someone joins or leaves, and
-// re-reading it on every arriving post would spend one stranger's fetch per
-// message on a list that moves in days.
-export const CO_MEMBER_TTL_MS = 24 * 60 * 60_000;
-export const CO_MEMBER_MAX = 5000;
-// An activity is a few kB. This is generous by two orders of magnitude and
-// still bounds what one Append can make us hold in memory.
-export const MAX_ITEM_BYTES = 512 * 1024;
-// The AS2 actor types. A group is as much an actor as a person is.
-export const ACTOR_TYPES = new Set(['Person', 'Group', 'Service', 'Application', 'Organization']);
-// The AS2 types the fediverse actually posts. `Note` alone is Mastodon's world
-// and not the fediverse's: an Article is a Plume or WriteFreely post, a
-// Question is a poll, a Video is PeerTube, a Page is Lemmy, an Audio is
-// Funkwhale. Insisting on Note dead-lettered every one of them as "not a
-// verifiable Note" — from people the owner had chosen to follow, silently.
-//
-// They share the shape this code reads: attributedTo, content, published,
-// inReplyTo, tag, attachment. A poll's options are dropped, which is a
-// degraded rendering rather than a lost post.
-export const CONTENT_TYPES = new Set(['Note', 'Article', 'Question', 'Page', 'Video', 'Audio', 'Image', 'Event']);
-
-// A Question is a poll: its options live in oneOf (pick one) or anyOf (pick
-// several), each carrying the tally its author's server maintains.
-// Custom emojis ride the tag list; the images live at the author's server and
-// the client fetches them from there.
-// Content, name and summary were capped; nothing else was. One remote Question
-// with thousands of options, or thousands of emoji, mention or attachment
-// entries, put megabytes into a single statuses.json row — a document rewritten
-// whole on every change. These are display lists: past a few dozen, nothing can
-// render them and nobody meant them to be rendered.
-// http(s) only. An emoji, attachment or mention URL is written straight into
-// the client's markup, so `javascript:` and `data:` have no business in one.
-// The store has guarded avatars this way all along (safeUrl, lib/store.mjs);
-// these three lists were simply never put through it.
-export const httpOnly = (u) => {
-  if (!u) return null;
-  try {
-    const p = new URL(String(u));
-    return (p.protocol === 'https:' || p.protocol === 'http:') ? String(u) : null;
-  } catch { return null; }
-};
-
-export const MAX_MODQUEUE = 200;
-export const MAX_EMOJIS = 60;
-export const MAX_POLL_OPTIONS = 50;
-export const MAX_OPTION_CHARS = 200;
-export const MAX_MENTIONS = 60;
-export const MAX_URL_CHARS = 2048;
-
-export function emojisOf(note) {
-  return [].concat(note?.tag || [])
-    .filter(t => t?.type === 'Emoji' && t.icon?.url && t.name)
-    .slice(0, MAX_EMOJIS)
-    .map(t => ({
-      shortcode: String(t.name).replace(/^:|:$/g, '').slice(0, 64),
-      url: httpOnly(String(t.icon.url).slice(0, MAX_URL_CHARS)),
-    }))
-    .filter(e => e.url);
-}
-
-export function pollOf(note) {
-  const opts = note?.oneOf || note?.anyOf;
-  if (!Array.isArray(opts) || !opts.length) return null;
-  return {
-    multiple: !!note.anyOf,
-    expiresAt: note.endTime || null,
-    closed: !!note.closed,
-    options: opts.slice(0, MAX_POLL_OPTIONS).map(o => ({
-      title: String(o?.name ?? '').slice(0, MAX_OPTION_CHARS),
-      votes: Number(o?.replies?.totalItems) || 0,
-    })),
-  };
-}
-// AS2 lets `type` be one string or a list, and implementations use both —
-// `["Person","Service"]` is an ordinary actor. Read either form.
-export const typesOf = (t) => (Array.isArray(t) ? t : [t]).filter(x => typeof x === 'string');
-export const isContentType = (t) => typesOf(t).some(x => CONTENT_TYPES.has(x));
-export const isActorType = (t) => typesOf(t).some(x => ACTOR_TYPES.has(x));
-
-// What we will carry to our followers on someone else's behalf (§7.1.2): the
-// activities a conversation is made of, and nothing else. A type this file
-// does not handle falls out of handle() with no rejection, and "no rejection"
-// is what qualifies an activity for forwarding — so without this gate a
-// stranger could have anything at all, of a type nothing here reads,
-// re-delivered to every follower over our signature.
-export const FORWARDABLE = new Set(['Create', 'Update', 'Delete', 'Like', 'Announce', 'Undo']);
-// Of those, the ones we may re-deliver to our own followers over our own
-// signature (§7.1.2). Deliberately narrower than FORWARDABLE: these three are
-// the ones whose object this drain fetched from the author's origin and checked
-// before accepting. A Like, an Announce or an Undo is taken on the envelope's
-// word alone — relaying one is signing for a claim nothing corroborated.
-export const FORWARD_TYPES = new Set(['Create', 'Update', 'Delete']);
-// How many forwards one drain may send. A reply into a busy thread of ours is a
-// handful; anything near this is a flood using us as an amplifier.
-export const MAX_FORWARDS_PER_DRAIN = 20;
-export const ACCEPT_AP = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"';
-
-// What is worth keeping of an activity we are filing rather than acting on.
-//
-// A dead letter, a moderation entry and a waiting follow request each used to
-// hold the WHOLE delivered activity — up to the 512 KB item cap — in documents
-// that are serialized and PUT whole on every change. A stranger could inflate
-// deadletter.json to ~100 MB and requests.json to ~250 MB and make us rewrite
-// them on every batch.
-//
-// Everything the consumers need survives: `acceptActivity` wraps this as the
-// Accept's object (and {id, type, actor, object} IS the standard Follow shape),
-// `applyModeration` reads only `object`, and a dead letter is read by a human
-// who wants to know what arrived, not to replay it.
-export function trimActivity(a) {
-  if (!a || typeof a !== 'object') return a ?? null;
-  const idOf = (v) => (typeof v === 'string' ? v : v?.id ?? null);
-  const out = {};
-  for (const k of ['id', 'type', 'actor', 'target']) {
-    const v = idOf(a[k]);
-    if (v) out[k] = String(v).slice(0, 2048);
-  }
-  const obj = idOf(a.object);
-  if (obj) {
-    // An Update says what to change INSIDE its object — a topic's new name, or
-    // whether it is closed — so flattening the object to its id alone threw the
-    // instruction away and left a bare topic id that meant nothing. Close,
-    // reopen and rename all arrived at the queue saying only which topic, and
-    // the moderator watched them do nothing. Kept now, bounded, and only the
-    // two fields a moderation queue acts on: everything else is still dropped.
-    const asked = a.object && typeof a.object === 'object' ? a.object : null;
-    const said = {};
-    if (asked) {
-      if (typeof asked.name === 'string') said.name = asked.name.slice(0, 300);
-      // `false` is the whole of a reopen, so presence is what counts, not truth.
-      if ('closed' in asked) {
-        // A date stays a date; a boolean stays a boolean; and the STRINGS
-        // "true"/"false" — which is what a boolean looks like after a form, a
-        // template or an RDF round trip — become the booleans they mean,
-        // because `!!"false"` is true and would turn a reopen into a close.
-        const v = asked.closed;
-        said.closed = v === 'false' ? false : v === 'true' ? true
-          : typeof v === 'string' ? v.slice(0, 64) : !!v;
-      }
-    }
-    out.object = Object.keys(said).length
-      ? { id: String(obj).slice(0, 2048), ...said }
-      : String(obj).slice(0, 2048);
-  }
-  // A typed object with no id still says what it was — a Block of nobody, an
-  // Undo of a Follow — and that is the whole of what the queue reads.
-  else if (a.object && typeof a.object === 'object' && a.object.type) {
-    out.object = { type: String(a.object.type).slice(0, 64) };
-  }
-  return out;
-}
-
-// Same origin AND, where the origin holds more than one identity, the same
-// identity within it.
-//
-// Origin alone is the right test when an origin is one person — the ordinary
-// fediverse server, and a subdomain pod. It is the WRONG test on a path-based
-// host, and above all on a multi-tenant front: every tenant of fedipod.net has
-// ids under `https://fedipod.net/u/<name>/`, so origin-equality made every
-// tenant able to vouch for every other. One tenant could publish posts
-// attributed to another, Update a neighbour's post, or Delete it — and a
-// receiving Mastodon would believe it for the same reason we did.
-//
-// So when both ids carry an identity prefix, the prefixes must match too. The
-// two shapes that exist here are the front's `/u/<name>/` and a pod's own AP
-// root (`…/<root>/ap/…`); anything else has no prefix and falls back to origin,
-// which is what a plain remote server should be judged by.
-function identityPrefix(u) {
-  const m = /^(https?:\/\/[^/]+\/u\/[^/]+\/)/u.exec(u);
-  if (m) return m[1];
-  const ap = /^(https?:\/\/[^/]+\/(?:[^/]+\/)*?)ap\//u.exec(u);
-  return ap ? ap[1] : null;
-}
-
-export function sameIdentity(a, b) {
-  if (!sameOrigin(a, b)) return false;
-  const pa = identityPrefix(String(a));
-  const pb = identityPrefix(String(b));
-  if (!pa || !pb) return true;              // no prefix to compare: origin is the answer
-  return pa === pb;
-}
-
-export function sameOrigin(a, b) {
-  try { return new URL(a).origin === new URL(b).origin; } catch { return false; }
-}
-
-// Is this socket URL the pod's own? The scheme has to be the socket form of the
-// pod's — wss for https, ws for http — so a downgrade to plaintext from an https
-// pod is somewhere else, not the same place unencrypted.
-//
-// The host may be the pod's, or a PARENT of it. Not a loosening for
-// convenience: a CSS server that gives every pod a subdomain answers
-// notifications from the server root, so jeff-zucker.teamid.live is served by
-// wss://teamid.live/.notifications/… — which is the deployment this project
-// actually runs on. Requiring an exact match dropped it to polling, and the
-// live agents are how that was found rather than the suite.
-//
-// A sibling subdomain is still refused: only a suffix of our own host passes,
-// and two labels minimum so `.live` cannot pose as everyone's parent. Not a
-// public-suffix list — that is a dependency and a data file to keep current,
-// and the party this guards against is the pod you already chose to trust.
-//
-// `localhost` is the one single-label parent allowed, because it is the one that
-// cannot be anybody else: it is reserved to the loopback interface (RFC 6761),
-// so `alice.localhost` and `localhost` are the same machine by definition and
-// there is no stranger for the rule to keep out. Without this a pod served from
-// a subdomain of localhost — which is how a Solid server with subdomain pods
-// runs on a developer's machine — refused its own socket and fell back to
-// polling, so every delivery waited up to two minutes.
-export function sameSocketOrigin(socketUrl, podBase) {
-  let s, p;
-  try { s = new URL(socketUrl); p = new URL(podBase); } catch { return false; }
-  if (s.protocol !== (p.protocol === 'https:' ? 'wss:' : 'ws:')) return false;
-  if (s.host === p.host) return true;
-  const parent = s.hostname.toLowerCase();
-  return (parent.split('.').length >= 2 || parent === 'localhost')
-    && s.port === p.port
-    && p.hostname.toLowerCase().endsWith('.' + parent);
-}
-
-export function httpUrl(u) {
-  try {
-    const p = new URL(String(u)).protocol;
-    return p === 'https:' || p === 'http:';
-  } catch { return false; }
-}
-
-// Who a note is BY. A document may only speak for an actor at its own origin.
-// `attributedTo` used to be taken at face value, so a note served anywhere
-// could name anyone: one at a host the attacker controls, claiming to be by
-// someone the owner follows, passed every check we had — the envelope's
-// sameOrigin compares the ACTIVITY to its object, never the object to its
-// author — and landed in the home timeline and in the pod as them. For a group
-// it went further still, because amplify() gates on the author's membership,
-// so the group signed an Announce of it and delivered it to every member.
-//
-// `delivered` is the actor that brought it, used only when the note names no
-// author of its own; it has to clear the same test, which is why a boost of an
-// unattributed note is refused rather than credited to the booster.
-//
-// Returns the author, or null when nothing at the note's origin vouches for one.
-export function authorOf(note, delivered = null) {
-  const claimed = [].concat(note?.attributedTo || [])
-    .map(a => (typeof a === 'string' ? a : a?.id)).find(Boolean) || null;
-  const author = claimed || delivered;
-  if (!author) return null;
-  // sameIdentity, not sameOrigin: on a multi-tenant front every tenant shares
-  // an origin, so origin-equality let any of them be credited with any other's
-  // post. See sameIdentity.
-  return sameIdentity(note?.id, author) ? author : null;
-}
-
-// A reaction: an emoji somebody put on a post. Pleroma, Akkoma, Misskey and
-// Sharkey send EmojiReact; Misskey also sends a Like whose `content` is the
-// emoji, which is what its own users see as a reaction. Either way the
-// emoji is the content — a unicode character, or `:name:` for a custom one
-// whose image rides the tag list. Null when the activity is a plain Like.
-export const MAX_REACTION_CHARS = 64;
-export function reactionOf(activity) {
-  if (!activity || (activity.type !== 'EmojiReact' && activity.type !== 'Like')) return null;
-  const content = typeof activity.content === 'string' ? activity.content.trim() : '';
-  if (!content) return null;
-  const emoji = content.slice(0, MAX_REACTION_CHARS);
-  const bare = emoji.replace(/^:|:$/g, '');
-  const custom = emojisOf(activity).find(e => e.shortcode === bare);
-  return { emoji, ...(custom ? { url: custom.url } : {}) };
-}
-
-// What a note quotes, whichever way its server says it. Mastodon writes
-// FEP-044f's `quote` alongside the two older spellings, Fedibird's quoteUri
-// and Misskey's _misskey_quote; Pleroma writes quoteUrl; FEP-e232 puts a Link
-// in the tag list with the Misskey relation. One post, one quote.
-const QUOTE_REL = 'https://misskey-hub.net/ns#_misskey_quote';
-export function quoteOf(note) {
-  const idOf = (v) => (typeof v === 'string' ? v : v?.id);
-  const named = [note?.quote, note?.quoteUri, note?._misskey_quote, note?.quoteUrl]
-    .map(v => idOf(Array.isArray(v) ? v[0] : v)).find(Boolean);
-  const linked = [].concat(note?.tag || []).find(t => t?.type === 'Link' && t.href
-    && [].concat(t.rel || []).includes(QUOTE_REL))?.href;
-  const q = named || linked;
-  return q && q !== note?.id ? httpOnly(String(q).slice(0, MAX_URL_CHARS)) : null;
-}
-
-// Who the author lets quote a post: their interaction policy's canQuote, in
-// the words a client uses. The public collection is `public`; the author's
-// own followers or following collections are `followers` and `following`.
-// Null when the note says nothing, which is the common case on servers that
-// predate the policy.
-export function quotePolicyOf(note, author) {
-  const can = note?.interactionPolicy?.canQuote;
-  if (!can || typeof can !== 'object') return null;
-  const word = (v) => {
-    const id = typeof v === 'string' ? v : v?.id;
-    if (!id) return null;
-    if (id === 'https://www.w3.org/ns/activitystreams#Public' || id === 'as:Public' || id === 'Public') return 'public';
-    if (author && sameIdentity(id, author)) {
-      if (/\/followers\/?$/u.test(id)) return 'followers';
-      if (/\/following\/?$/u.test(id)) return 'following';
-    }
-    return null;
-  };
-  const list = (v) => [...new Set([].concat(v || []).map(word).filter(Boolean))];
-  return { automatic: list(can.automaticApproval), manual: list(can.manualApproval) };
-}

package/lib/core/intake/channel.mjs

@@ -1,144 +0,0 @@
-// channel.mjs — the inbox's push channel: a WebSocketChannel2023 subscription
-// on the pod's inbox container, reconnecting with backoff, and the drain it
-// wakes. State (ws, wsState, reconnectTries, resubTimer) lives on the Intake
-// so status pages and stop() see it; this module is the behaviour.
-
-import * as podNotifications from '../../pod/notifications.mjs';
-import { USER_AGENT } from '../../shared/ua.mjs';
-import { HTTP_TIMEOUT_MS, readCapped } from '../../shared/safefetch.mjs';
-import { sameSocketOrigin } from './activity.mjs';
-
-// The channel a subscription returns outlives a dropped socket, so reconnecting
-// reuses it. Creating a new one per reconnect is what buried solidcommunity.net
-// in channel records they then had to sweep.
-const CHANNEL_DOC = 'inbox-channel.json';
-// A flapping socket used to POST a NEW WebSocketChannel2023 channel every two
-// seconds — hundreds an hour against a server that is already struggling, and
-// channel churn its operators have to sweep up. Backs off instead, and an open
-// only triggers a sweep if we have not just swept.
-const RECONNECT_MIN_MS = 2_000;
-const RECONNECT_MAX_MS = 5 * 60_000;
-// How long a socket must stay up before the backoff counts it as a success and
-// resets. Shorter than that is a flap, not a connection.
-const RECONNECT_STABLE_MS = 60_000;
-const OPEN_DRAIN_MIN_GAP_MS = 30_000;
-
-// Jittered exponential, floor to ceiling, reset by a successful open.
-export function reconnectDelay(intake) {
-  const capped = Math.min(RECONNECT_MIN_MS * 2 ** intake.reconnectTries, RECONNECT_MAX_MS);
-  intake.reconnectTries++;
-  return Math.round(capped * (0.8 + Math.random() * 0.4));
-}
-
-// --- push ---
-// Any failure in here used to end push for the life of the process: the
-// retry lived only in the "server refused the subscription" branch, so a
-// network blip left wsState at never-connected and the agent silently on
-// polling. Every path now schedules a retry on the same backoff.
-export async function subscribe(intake) {
-  try {
-    await intake._subscribeOnce();
-  } catch (e) {
-    intake.wsState = 'subscribe-error';
-    const wait = intake._reconnectDelay();
-    intake.log(`subscribe failed (${e.message}) — retrying in ${Math.round(wait / 1000)}s (polling meanwhile)`);
-    if (!intake.stopped) {
-      intake.resubTimer = setTimeout(() => intake.subscribe().catch(() => {}), wait);
-      intake.resubTimer.unref?.();
-    }
-  }
-}
-
-/**
- * Where this pod describes the services it offers. The pod says so on any
- * response about one of its resources; the well-known path is only what a
- * pod that says nothing has always used.
- */
-export async function storageDescriptionUrl(intake) {
-  return podNotifications.storageDescriptionUrl(intake.urls.base,
-    { headers: { 'user-agent': USER_AGENT }, timeoutMs: HTTP_TIMEOUT_MS });
-}
-
-export async function subscribeOnce(intake) {
-  // Reuse a channel we already have rather than asking for another one.
-  const saved = intake.store.read(CHANNEL_DOC, null);
-  if (saved?.receiveFrom && (!saved.endAt || Date.parse(saved.endAt) - Date.now() > 60_000)) {
-    intake._openSocket(saved.receiveFrom, true);
-    return;
-  }
-  const descUrl = await intake._storageDescriptionUrl();
-  const { channel, error } = await podNotifications.readWebSocketChannel(descUrl,
-    { headers: { 'user-agent': USER_AGENT }, timeoutMs: HTTP_TIMEOUT_MS });
-  if (!channel) { intake.wsState = 'unavailable'; intake.log(`${error} — polling only`); return; }
-  // The topic is a POD resource, and it travels in the BODY — so the url map
-  // RemotePod applies to the request line never reaches it. A fronted
-  // identity's inbox url names the front, which the pod cannot grant read on,
-  // and the subscription came back 403. A no-op when unfronted.
-  const topic = intake.urls.toPod ? intake.urls.toPod(intake.urls.inbox) : intake.urls.inbox;
-  const sub = await podNotifications.subscribeToInbox(intake.remote,
-    { channelUrl: channel, podTopicUrl: topic });
-  const body = await readCapped(sub).then(JSON.parse).catch(() => null);
-  if (!body?.receiveFrom) {
-    intake.wsState = `subscribe-failed-${sub.status}`;
-    const wait = intake._reconnectDelay();
-    intake.log(`subscription failed (${sub.status}) — retrying in ${Math.round(wait / 1000)}s (polling meanwhile)`);
-    if (!intake.stopped) {
-      intake.resubTimer = setTimeout(() => intake.subscribe().catch(e => intake.log(`resubscribe: ${e.message}`)), wait);
-      intake.resubTimer.unref?.();
-    }
-    return;
-  }
-  intake.store.write(CHANNEL_DOC, { receiveFrom: body.receiveFrom, endAt: body.endAt || null });
-  intake._openSocket(body.receiveFrom, false);
-}
-
-// The socket URL arrives in the pod's own subscription response, and it was
-// the one outbound address in the project that reached the network without
-// passing anything — safefetch guards every fetch, and `new WebSocket()` is
-// not a fetch. A pod that answered with somebody else's address had us open a
-// long-lived connection there and treat what came back as our inbox waking up.
-//
-// Same origin as the pod, not assertPublicUrl: a pod on this machine is a
-// documented setup and its socket is legitimately ws://localhost:3000, which
-// a public-address check would refuse.
-export function openSocket(intake, receiveFrom, reused) {
-  if (!sameSocketOrigin(receiveFrom, intake.urls.base)) {
-    intake.wsState = 'refused';
-    intake.log(`subscription named ${receiveFrom}, which is not this pod — polling only`);
-    if (reused) intake.store.write(CHANNEL_DOC, null);
-    return;
-  }
-  intake.ws = new WebSocket(receiveFrom);
-  intake.ws.onopen = () => {
-    intake.wsState = 'open';
-    if (!intake._announcedPush) { intake.log('inbox push subscription active'); intake._announcedPush = true; }
-    intake._openedAt = Date.now();
-    // Anything that arrived while the socket was down is waiting — sweep it,
-    // unless a sweep just ran: a flapping socket must not re-list the inbox
-    // on every open.
-    if (Date.now() - intake.lastDrainAtMs > OPEN_DRAIN_MIN_GAP_MS) {
-      intake.drain().catch(e => intake.log(`drain: ${e.message}`));
-    }
-  };
-  intake.ws.onmessage = () => intake.drain().catch(e => intake.log(`drain: ${e.message}`));
-  intake.ws.onclose = () => {
-    intake.wsState = 'closed';
-    // Only a connection that STAYED up counts as a success. Resetting on open
-    // alone meant the 2026-07-29 failure — a server that accepts the upgrade
-    // and then drops the socket on a crash cycle — reconnected at the 2s
-    // floor indefinitely: every cycle "succeeded", so the exponential cap was
-    // never reached, and each open also drained the inbox.
-    if (intake._openedAt && Date.now() - intake._openedAt >= RECONNECT_STABLE_MS) intake.reconnectTries = 0;
-    intake._openedAt = 0;
-    if (!intake.stopped) {
-      intake.resubTimer = setTimeout(() => intake.subscribe().catch(e => intake.log(`resubscribe: ${e.message}`)), intake._reconnectDelay());
-      intake.resubTimer.unref?.();
-    }
-  };
-  intake.ws.onerror = () => {
-    intake.wsState = 'error';
-    // A channel we reused may simply be gone: forget it so the next attempt
-    // asks for a fresh one instead of retrying a dead URL forever.
-    if (reused) intake.store.write(CHANNEL_DOC, null);
-  };
-}