fedipod 0.8.0

package/lib/deliver.mjs

@@ -0,0 +1,249 @@
+// deliver.mjs — signed outbound delivery + retry queue. Fedify's signRequest
+// does draft-cavage HTTP Signatures (what Mastodon verifies). Failures go to
+// a JSON-file queue with exponential backoff; peers that stay down are
+// dropped after MAX_ATTEMPTS (~3 days).
+
+import { signRequest } from '@fedify/fedify/sig';
+import { pinnedFor, retryAfterMs, HTTP_TIMEOUT_MS } from './safefetch.mjs';
+import { USER_AGENT } from './ua.mjs';
+
+const MAX_REDIRECTS = 3;
+
+const MAX_ATTEMPTS = 12;                       // 2^12 min ≈ 68h of backoff
+const TICK_MS = 60_000;
+const MAX_QUEUE = 2000;                        // beyond it, overflow dead-letters
+
+// Is this failure about the SERVER, or about this one recipient? A 410 Gone,
+// a 403, a 401 from an authorized-fetch instance are answers about one inbox;
+// cooling the whole host for them starves every other recipient on it — which
+// on a big instance is most of your followers. Only a refusal that is about
+// the server, or no answer at all, is grounds to leave it alone.
+const hostOf = (inbox) => { try { return new URL(inbox).host; } catch { return inbox; } };
+const aboutTheHost = (e) => !e.status || e.status >= 500 || e.status === 429;
+
+export class Deliverer {
+  // passive: signing-only (viewer-mode agents) — no queue drain timer, so a
+  // read-only agent never mutates shared delivery state. startQueue() flips
+  // it live when a viewer is promoted to active.
+  constructor({ store, keyId, rsaPrivate, log = console.log, passive = false }) {
+    this.store = store;
+    this.keyId = keyId;
+    this.rsaPrivate = rsaPrivate;
+    this.log = log;
+    if (!passive) this.startQueue();
+  }
+
+  startQueue() {
+    if (this.timer) return;
+    this.timer = setInterval(() => this.drainQueue().catch(e => this.log(`queue: ${e.message}`)), TICK_MS);
+    this.timer.unref?.();
+  }
+
+  // Signed fetch — also used for GETs so authorized-fetch instances answer
+  // us. Targets come from attacker-supplied documents, so every hop is
+  // address-checked and the connection is pinned to the checked address.
+  // Redirects are followed by hand because each hop needs its own signature
+  // (the signature covers the request target).
+  async signedFetch(url, init = {}) {
+    let current = url;
+    // Named on every hop: a remote operator seeing our deliveries or our
+    // verify-by-dereference GETs should know what we are.
+    //
+    // And bounded. This was the one remote-facing path in the project without a
+    // deadline — publisher, intake and the token grant all set one — so a peer
+    // that accepted the connection and then stalled held us for undici's 300s
+    // default. Against a 60s queue tick that is five overlapping drains
+    // re-POSTing the same activity at a server already in trouble. One signal,
+    // built once, so it covers all four redirect hops together rather than
+    // giving each hop a fresh deadline.
+    const withUa = {
+      ...init,
+      signal: init.signal || AbortSignal.timeout(HTTP_TIMEOUT_MS),
+      headers: { 'user-agent': USER_AGENT, ...(init.headers || {}) },
+    };
+    for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+      const dispatcher = await pinnedFor(current);
+      const req = new Request(current, withUa);
+      const signed = await signRequest(req, this.rsaPrivate, new URL(this.keyId));
+      const res = await fetch(signed, { ...(dispatcher ? { dispatcher } : {}), redirect: 'manual' });
+      if (res.status >= 300 && res.status < 400 && res.headers.get('location')) {
+        current = new URL(res.headers.get('location'), current).href;
+        continue;
+      }
+      return res;
+    }
+    throw new Error(`too many redirects from ${url}`);
+  }
+
+  async deliverNow(inbox, activity) {
+    const res = await this.signedFetch(inbox, {
+      method: 'POST',
+      headers: { 'content-type': 'application/activity+json' },
+      body: JSON.stringify(activity),
+    });
+    if (res.status >= 400) {
+      // Carry what the server told us instead of flattening it to a string.
+      // The Response used to be discarded here, headers and all, which is why
+      // a `Retry-After: 3600` was answered with four more POSTs inside the
+      // hour it asked for.
+      const err = new Error(`POST ${inbox} → ${res.status}`);
+      err.status = res.status;
+      // Only when the header is ACTUALLY there. Defaulting made every 503
+      // without one — which is most of them — collapse the exponential ladder
+      // to a flat 60s, the opposite of the intent. A remote's own instruction
+      // is honoured up to a day rather than the pod path's 30-minute ceiling:
+      // MAX_ATTEMPTS bounds the wait anyway.
+      const ra = retryAfterMs(res, 24 * 60 * 60_000);
+      if (ra != null) err.retryAfterMs = ra;
+      throw err;
+    }
+    return res;
+  }
+
+  // Deliver, queueing on failure.
+  async deliver(inbox, activity) {
+    // A host we already know is refusing: queue without asking again. This is
+    // the path a FRESH activity takes, so without it a fan-out to a struggling
+    // server opened one socket per follower before any of this applied.
+    const host = hostOf(inbox);
+    const until = this._cooling?.get(host);
+    if (until && until > Date.now()) {
+      this.log(`${host} is cooling — queueing ${activity.type} rather than asking again`);
+      this._enqueue({ inbox, activity, attempts: 1, nextAt: until });
+      return;
+    }
+    try {
+      await this.deliverNow(inbox, activity);
+      this.log(`delivered ${activity.type} → ${inbox}`);
+    } catch (e) {
+      this.log(`delivery failed (${e.message}) — queued`);
+      const wait = e.retryAfterMs || 60_000;
+      if (aboutTheHost(e)) {
+        this._cooling ||= new Map();
+        this._cooling.set(host, Date.now() + wait);
+      }
+      this._enqueue({ inbox, activity, attempts: 1, nextAt: Date.now() + wait });
+    }
+  }
+
+  // The queue has a ceiling: a fan-out into a sea of down servers must not grow
+  // queue.json without bound. Overflow keeps the evidence as a dead letter.
+  _enqueue(item) {
+    const q = this.store.getQueue();
+    if (q.length >= MAX_QUEUE) {
+      this.log(`delivery queue is full (${MAX_QUEUE}) — dead-lettering ${item.activity?.type} → ${item.inbox}`);
+      this.store.addDeadLetter({ inboxUrl: item.inbox, reason: 'delivery queue full', activity: item.activity });
+      return;
+    }
+    q.push(item);
+    this.store.setQueue(q);
+  }
+
+  async deliverToAll(inboxes, activity) {
+    // Shared inboxes deduplicate fan-out to the same server.
+    for (const inbox of [...new Set(inboxes)]) await this.deliver(inbox, activity);
+  }
+
+  // Serialized, for the same reason Intake.drain is: the tick is 60s and a
+  // drain over slow peers outlasts it, so a second run started on top of the
+  // first. Both read their own clone of queue.json, so both re-POSTed every due
+  // activity — duplicate deliveries at a server that is already failing — and
+  // both then wrote `keep` back from a stale snapshot, losing the attempt
+  // counts that are supposed to make the backoff converge.
+  async drainQueue() {
+    if (this._draining) { this._drainAgain = true; return this._draining; }
+    this._draining = this._drainQueueOnce().finally(() => {
+      this._draining = null;
+      if (this._drainAgain) { this._drainAgain = false; this.drainQueue().catch(e => this.log(`queue: ${e.message}`)); }
+    });
+    return this._draining;
+  }
+
+  // The queue is walked per HOST, not per item.
+  //
+  // It used to be a flat sequential loop, which had two costs against a server
+  // that was down. Every due item for that host was POSTed — the first failure
+  // told us nothing the code could act on — and because each request carries a
+  // 15s deadline, K items for one dead host held the drain for up to 15K
+  // seconds while every healthy peer's activity waited behind them. Compounding
+  // requests at a struggling server, and head-of-line blocking of everyone
+  // else: the shape of the incident this project has a postmortem for.
+  //
+  // One failure now cools the whole host for the rest of this drain, and a
+  // Retry-After it sent decides for how long. The breaker is in memory; what
+  // persists is nextAt, which the queue already carried.
+  async _drainQueueOnce() {
+    const q = this.store.getQueue();
+    if (!q.length) return;
+    const now = Date.now();
+    const keep = [];
+    // Whether anything actually moved. The only early return above is on an
+    // EMPTY queue, so one item with a far-future nextAt used to rewrite
+    // queue.json on all 1440 ticks of a day with byte-identical content —
+    // PodStore.write has no equality check, and the ordinary ladder reaches
+    // hours-long waits by attempt 6, so any peer that fails a few times got
+    // there. A length compare is not enough: getQueue() hands back a clone, so
+    // attempts/nextAt are mutated in place and the count stays the same.
+    let changed = false;
+    // Outlives the drain: a Retry-After only reached items that happened to be
+    // due in the same pass, so a sibling due a minute later ignored it entirely.
+    this._cooling ||= new Map();             // host → until
+    for (const [h, until] of this._cooling) if (until <= now) this._cooling.delete(h);
+
+    const ladderFor = (item) => {
+      const base = Math.min(2 ** item.attempts, 2 ** MAX_ATTEMPTS) * 60_000;
+      return Math.round(base * (0.85 + Math.random() * 0.3));   // jittered, not lockstep
+    };
+
+    for (const item of q) {
+      // Due first, THEN cooling: an item that is not yet due must not have its
+      // ladder advanced on every 60s tick of a 15-minute cooldown. Once
+      // deferred its nextAt IS the cooldown, so it advances exactly once per
+      // cooldown — the same rate as the item that opened the socket.
+      if (item.nextAt > now) { keep.push(item); continue; }
+      const host = hostOf(item.inbox);
+      const until = this._cooling.get(host);
+      if (until && until > now) {
+        // A sibling already failed against this host. Defer without opening a
+        // socket — but ADVANCE THE LADDER, or only the one item that actually
+        // made a request ever climbs it and a queue of K items takes K times
+        // as long to give up as the module promises.
+        item.attempts += 1;
+        changed = true;
+        if (item.attempts > MAX_ATTEMPTS) {
+          this.log(`giving up on ${item.inbox} after ${MAX_ATTEMPTS} attempts`);
+          continue;
+        }
+        item.nextAt = Math.max(item.nextAt, until);
+        keep.push(item);
+        continue;
+      }
+      try {
+        await this.deliverNow(item.inbox, item.activity);
+        changed = true;
+        this.log(`retry ok: ${item.activity.type} → ${item.inbox}`);
+      } catch (e) {
+        item.attempts += 1;
+        changed = true;
+        const wait = e.retryAfterMs || ladderFor(item);
+        // Cool the host BEFORE the give-up test: a failure is evidence about
+        // the server whether or not the item that produced it survives, and
+        // skipping this left the flat behaviour intact for exactly the backlog
+        // that has just aged out.
+        if (aboutTheHost(e)) {
+          this._cooling.set(host, now + wait);
+          if (e.retryAfterMs) this.log(`${host} asked for ${Math.round(wait / 1000)}s — honouring it`);
+        }
+        if (item.attempts > MAX_ATTEMPTS) {
+          this.log(`giving up on ${item.inbox} after ${MAX_ATTEMPTS} attempts`);
+          continue;
+        }
+        item.nextAt = now + wait;
+        keep.push(item);
+      }
+    }
+    if (changed) this.store.setQueue(keep);
+  }
+
+  stop() { clearInterval(this.timer); this.timer = null; }
+}

package/lib/directory.mjs

@@ -0,0 +1,90 @@
+// directory.mjs — the well-known door. Whichever running agent holds port
+// 8030 answers there with a redirect to its own record page, so opening
+// https://localhost:8030/ always reaches a page that lists every identity on
+// this machine, whatever ports they actually run on. Loopback only, https
+// only — like every listener this project starts. Held only by a CONFIGURED agent: an
+// unconfigured one's record page is a setup form with no identity list,
+// which strands whoever the door sends there.
+//
+// The door is held politely: it identifies itself at GET /__directory, and a
+// real owner of the port — an identity that recorded 8030 as its own — asks it
+// to step aside with POST /__directory/yield before binding. The holder frees
+// the port at once and stays off it for a minute. AP_DIRECTORY=0 disables the
+// whole mechanism (the test suite sets it, so throwaway agents do not fight
+// over a real machine's door).
+import https from 'node:https';
+import { localFetch } from './localapi.mjs';
+
+export const DIRECTORY_PORT = 8030;
+
+// Answer for a running agent. No-op when this agent already IS the door, or
+// when the mechanism is switched off. `eligible` is asked before every claim
+// attempt — the 15 s retry means an agent that configures later takes the
+// door soon after, and one that never does never holds it.
+export function claimDirectory({ port, origin, log = () => {}, eligible = () => true, tls = null }) {
+  if (process.env.AP_DIRECTORY === '0' || port === DIRECTORY_PORT) {
+    return { stop: () => {} };
+  }
+  const servers = [];
+  let held = false;
+  let pausedUntil = 0;
+  const handler = (req, res) => {
+    const p = req.url || '/';
+    if (p === '/__directory') {
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end(JSON.stringify({ directory: true, agent: origin() }) + '\n');
+      return;
+    }
+    if (req.method === 'POST' && p === '/__directory/yield') {
+      pausedUntil = Date.now() + 60_000;
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end('{"ok":true}\n');
+      release();
+      log(`directory: yielded port ${DIRECTORY_PORT}`);
+      return;
+    }
+    res.writeHead(302, { location: origin().replace(/\/$/, '') + p });
+    res.end();
+  };
+  const release = () => {
+    for (const s of servers.splice(0)) { try { s.close(); } catch { /* already down */ } }
+    held = false;
+  };
+  const tryClaim = () => {
+    if (held || !eligible() || Date.now() < pausedUntil) return;
+    // Loopback both ways, like the agent's own listener: "localhost" resolves
+    // to ::1 first on many systems. IPv4 is the one that decides `held`.
+    if (!tls) return;   // no certificate, no door: this project serves https only
+    const tlsOpts = { key: tls.key, cert: tls.cert };
+    const s4 = https.createServer(tlsOpts, handler);
+    s4.on('error', () => { /* someone else holds the door — retry later */ });
+    s4.listen(DIRECTORY_PORT, '127.0.0.1', () => {
+      held = true;
+      servers.push(s4);
+      const s6 = https.createServer(tlsOpts, handler);
+      s6.on('error', () => { /* no IPv6 loopback here — IPv4 covers it */ });
+      s6.listen(DIRECTORY_PORT, '::1', () => servers.push(s6));
+      log(`directory: https://localhost:${DIRECTORY_PORT}/ now reaches ${origin()}`);
+    });
+  };
+  tryClaim();
+  const timer = setInterval(tryClaim, 15_000);
+  timer.unref();
+  return { stop: () => { clearInterval(timer); release(); } };
+}
+
+// Free `port` if what holds it is the directory door. Returns true when the
+// port is free afterwards. Quick and quiet on every other kind of holder.
+export async function yieldDirectory(port, { portFree, home = null } = {}) {
+  const probe = await localFetch(home, port, '/__directory', { timeout: 1000 })
+    .then(r => (r.ok ? r.json() : null)).catch(() => null);
+  if (!probe?.directory) return false;
+  await localFetch(home, port, '/__directory/yield', { method: 'POST', timeout: 1000 })
+    .catch(() => {});
+  if (!portFree) return true;
+  for (let i = 0; i < 10; i++) {
+    if (await portFree(port)) return true;
+    await new Promise(r => setTimeout(r, 100));
+  }
+  return false;
+}

package/lib/embed.mjs

@@ -0,0 +1,214 @@
+// embed.mjs — the agent, running inside the pod server that holds its pod.
+//
+// Nothing here reimplements the agent: it is the same Agent class, the same
+// intake, deliverer, publisher and lease. What changes is the transport. The
+// caller hands in a session that reaches the pod through the server's own
+// store, and RemotePod takes it instead of minting a credential — so an
+// identity acts on its pod with no token, no socket, and no second process.
+//
+// The one thing a pod server can do that a laptop cannot is notice a write the
+// moment it happens, so the notification socket is replaced by the store's own
+// change events.
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { createRequire } from 'node:module';
+
+import { Agent } from '../run-agent.mjs';
+import { RemotePod } from './remote.mjs';
+import { apUrls } from './wire.mjs';
+import { writeJsonAtomic } from './home.mjs';
+import { buildAdminSurface } from './admin.mjs';
+import { FixedAuthorities } from './guard.mjs';
+
+const require = createRequire(import.meta.url);
+const { makeGate } = require('../vendor/gate.cjs');
+
+// How long to gather store events before draining, so a delivery of several
+// items costs one sweep rather than one each.
+const DRAIN_COALESCE_MS = 250;
+
+// How every secret in this project is minted (the attach flow's recipe).
+const mintSecret = () => crypto.randomBytes(32).toString('base64');
+
+/**
+ * The secret guarding one identity's owner door, kept beside its signing key.
+ * Minted when absent; `rotate` re-mints over an existing one — which is how a
+ * lost secret is recovered, by proving pod control again. Never logged.
+ */
+export function ensureDoorSecret(dataDir, handle, { rotate = false } = {}) {
+  const home = path.join(dataDir, handle);
+  fs.mkdirSync(home, { recursive: true, mode: 0o700 });
+  const file = path.join(home, 'door-secret.json');
+  if (!rotate) {
+    try {
+      const rec = JSON.parse(fs.readFileSync(file, 'utf8'));
+      if (rec?.secret) return { secret: rec.secret, path: file, rotated: false };
+    } catch { /* absent or unreadable: mint below */ }
+  }
+  const rec = { secret: mintSecret(), mintedAt: new Date().toISOString() };
+  writeJsonAtomic(file, rec, { mode: 0o600 });
+  return { secret: rec.secret, path: file, rotated: rotate };
+}
+
+/** The identity's name: a subdomain pod is its label, a path pod its last segment. */
+export function handleFor(podBase) {
+  const u = new URL(podBase);
+  const segments = u.pathname.split('/').filter(Boolean);
+  return segments.length ? segments[segments.length - 1] : u.hostname.split('.')[0];
+}
+
+/**
+ * The credential file an embedded identity runs on. It names the pod and
+ * nothing else: there is no client id and no secret, because there is nobody
+ * to authenticate to. Absent `privateRoot`, so the state tree stays on the pod.
+ */
+function ensureCredential(home, { podBase, webId }) {
+  const file = path.join(home, 'credential.json');
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
+  catch { /* first run for this identity */ }
+  const rec = {
+    webId,
+    remotePod: podBase.endsWith('/') ? podBase : podBase + '/',
+    root: 'activitypods-js/',
+  };
+  writeJsonAtomic(file, rec, { mode: 0o600 });
+  return rec;
+}
+
+/**
+ * Bring one identity up inside the server.
+ *
+ * `session` is the store-backed transport ({ fetch, warmup, stats }).
+ * `resourceStore` is optional and used only to watch for inbox writes.
+ * Returns the running agent and a stop() that leaves the pod tidy.
+ */
+export async function startEmbeddedAgent({
+  podBase,
+  dataDir,
+  session,
+  resourceStore = null,
+  webIdSuffix = 'profile/card#me',
+  log = () => {},
+  pollSeconds = null,
+  autoAcceptFollows = true,
+  gateToken = null,
+  uiPath = '/fedipod/',
+}) {
+  const base = podBase.endsWith('/') ? podBase : podBase + '/';
+  const handle = handleFor(base);
+  const home = path.join(dataDir, handle);
+  fs.mkdirSync(home, { recursive: true, mode: 0o700 });
+
+  const webId = base + webIdSuffix.replace(/^\//u, '');
+  const cred = ensureCredential(home, { podBase: base, webId });
+
+  const agent = new Agent({ home, log });
+  agent.log = log;
+  agent.store.log = log;
+  // Read by connect() when it builds Intake: in-process, the pod's own change
+  // events are the push channel.
+  agent.embedded = true;
+  if (pollSeconds) agent.pollSeconds = pollSeconds;
+  agent.remote = new RemotePod(cred, { log, session });
+  await agent.remote.warmup();
+
+  // Whether this pod already carries an identity is the question connect()
+  // assumes an answer to, so settle it first.
+  const urls = apUrls(cred.remotePod, cred.root);
+  agent.store.attach(agent.privateStorage(cred, 'state', urls));
+
+  // A state container that is NOT THERE means a pod nobody has set up yet. A
+  // state container we could not READ means a pod we could not ask, which is a
+  // different answer: treating it as empty would provision a second identity
+  // over the top of a working one.
+  const state = await agent.remote.fetch(urls.state, { headers: { accept: 'text/turtle' } });
+  const unprovisioned = state.status === 404 || state.status === 410;
+  if (!unprovisioned && state.status >= 400) {
+    throw new Error(`state at ${urls.state} → ${state.status} — cannot tell whether this pod is set up`);
+  }
+  if (!unprovisioned) {
+    await agent.store.load();
+    agent.stateLoaded = true;
+  }
+
+  if (!agent.store.getConfig()) {
+    // The pod itself must exist first. A server creates its seeded pods after
+    // it runs its initializers, so an agent that provisioned eagerly would
+    // leave a container tree exactly where a pod is about to be created.
+    const profile = await agent.remote.fetch(webId.split('#')[0], { headers: { accept: 'text/turtle' } });
+    if (profile.status === 404 || profile.status === 410) {
+      throw new Error(`no pod at ${base} yet — its owner profile is not there`);
+    }
+    log(`no identity on ${base} yet — provisioning @${handle}`);
+    await agent.bootstrap({ handle, name: handle, kind: 'person' });
+    if (autoAcceptFollows) {
+      agent.store.setConfig({ ...agent.store.getConfig(), autoAcceptFollows: true });
+      await agent.store.flush();
+    }
+    agent.stateLoaded = true;
+  }
+
+  await agent.connect();
+
+  // The client surfaces, on the pod's own origin: the Mastodon API a phone app
+  // speaks, the write API, nodeinfo, and behind the door the admin routes and
+  // the web client. Same code the standalone agent serves, minus the routes
+  // that only mean something to a process of one's own.
+  const authorities = new FixedAuthorities(base);
+  agent.authorities = authorities;
+  const surface = buildAdminSurface({
+    agent,
+    log,
+    gate: makeGate(gateToken, { secureCookie: authorities.secure }),
+    allowed: authorities,
+    embedded: true,
+    basePath: uiPath,
+    publicOrigin: base,
+    scheme: new URL(base).protocol,
+  });
+
+  // A delivery landing in the inbox — through the door, or from any other
+  // writer — wakes the drain at once. Only additions, and only in the inbox:
+  // the drain's own DELETEs must not call it back.
+  let coalesce = null;
+  let onChanged = null;
+  if (typeof resourceStore?.on === 'function') {
+    const inbox = urls.inbox;
+    onChanged = (identifier, activity) => {
+      const kind = String(activity?.value ?? activity ?? '');
+      if (!identifier?.path?.startsWith(inbox)) return;
+      if (!kind.endsWith('Create') && !kind.endsWith('Add')) return;
+      if (coalesce) return;
+      coalesce = setTimeout(() => {
+        coalesce = null;
+        if (!agent.viewer) agent.intake?.drain().catch((e) => log(`drain: ${e.message}`));
+      }, DRAIN_COALESCE_MS);
+      coalesce.unref?.();
+    };
+    resourceStore.on('changed', onChanged);
+  }
+
+  const stop = async () => {
+    if (onChanged) resourceStore.off('changed', onChanged);
+    surface.streaming?.stop?.();
+    clearTimeout(coalesce);
+    agent.intake?.stop();
+    agent.tagfeed?.stop();
+    agent.bskyfeed?.stop();
+    agent.deliverer?.stop();
+    agent.importer?.stop();
+    clearInterval(agent.schedTimer);
+    clearInterval(agent.refreshTimer);
+    // Same order the standalone agent's shutdown uses: write what is pending,
+    // then let go of the lease so the next agent need not wait out the TTL.
+    await Promise.allSettled([
+      agent.store.flush(),
+      agent.viewer ? Promise.resolve() : agent.lease?.release(),
+    ]);
+  };
+
+  return { agent, handle, home, surface, host: authorities.host, stop };
+}