fedipod-server 0.28.0 → 0.29.0

package/lib/device/cli/commands/setup.mjs

@@ -18,7 +18,10 @@ export async function setup() {
   if (process.stdin.isTTY && !has('cli') && !IDENTITY_FLAGS.some(f => args.includes('--' + f))) {
     return runBrowserSetup();
   }
-const root = flag('root') || 'fedipod/';   // new installs default to the fedipod/ container
+// Where on the pod: --root names it outright (scripts); otherwise the person
+// names the container that holds `fedipod/`, asked below once the pod is known.
+let root = flag('root') || null;
+const place = await import(new URL('../../../../lib/core/place.mjs', import.meta.url));
 const kind = has('group') ? 'group' : 'person';
 const approveJoins = has('group') && has('approve-joins');
 const summary = flag('summary');
@@ -48,10 +51,20 @@ if (!newAccount && !pod) {
   }
 }
 if (!newAccount && !pod) { console.error('no pod given'); process.exit(2); }
+const CONTAINER_Q = 'Store your data in a container named fedipod, inside this container';
+let container = flag('in');
 if (!newAccount) {
+  const podBase = pod.endsWith('/') ? pod : pod + '/';
+  if (!root) {
+    if (container == null) container = interactive ? await ask(`${CONTAINER_Q} (empty: ${new URL(podBase).pathname})`, '') : '';
+    const chosen = place.chosenRoot(podBase, container);
+    if (chosen.problem) { console.error(`Where to store it: ${chosen.problem}.`); process.exit(2); }
+    root = chosen.root;
+  }
   const { resourceExists } = await import(new URL('../../../../lib/pod/root.mjs', import.meta.url));
   const { apUrls, DEFAULT_ROOT: DR } = await import(new URL('../../../../lib/core/wire.mjs', import.meta.url));
-  if (await resourceExists(fetch, apUrls(pod, DR).actor)) {
+  // An account at the chosen place, or an older one where accounts used to go.
+  if (await resourceExists(fetch, apUrls(pod, root).actor) || await resourceExists(fetch, apUrls(pod, DR).actor)) {
     if (String(flag('address') || 'pod').toLowerCase() !== 'front') {
       console.error('The pod already hosts a FediPod account. If you want a second account, put it on a different pod.');
       console.error('To move an account whose address is at another gateway to this one, add --address front.');
@@ -115,8 +128,8 @@ if (fronted && kind === 'group') {
 // warning is their call to accept. Nobody could ever find this group, and the
 // people it would fail are not the operator reading the warning.
 if (kind === 'group' && !newAccount && !wfHost) {
-  console.error(`${pod} is a path on ${new URL(pod).host}, not the root of its own host.`);
-  console.error('WebFinger is answered only at a host root, so nobody could find this group.');
+  console.error(`${pod} is a suffixed pod, not a subdomained one.`);
+  console.error('WebFinger is answered only for a subdomained pod, so nobody could find this group.');
   console.error('Give the group a pod of its own:  fedipod setup --group --new-account');
   process.exit(2);
 }
@@ -133,6 +146,26 @@ if (fronted) {
   console.log(`pods at ${issuerHost}/${podName}/ instead cannot answer WebFinger for an address,`);
   console.log(`so setup takes an address at ${gatewayHost} for you and says so.\n`);
 }
+// Where the account lives is recorded in the pod's public type index. A new
+// pod has none; a pod you bring is asked. With none, only a yes goes on.
+let createIndex = has('create-type-index');
+if (!createIndex) {
+  const known = newAccount ? false : await place.publicIndexKnown(fetch, pod.endsWith('/') ? pod : pod + '/');
+  if (known !== true) {
+    const why = newAccount ? 'A new pod has no public type index' : 'Your pod has no public type index';
+    if (!interactive) {
+      console.error(`${why}. FediPod records where your account lives in one; add --create-type-index to make it.`);
+      process.exit(2);
+    }
+    const yes = await ask(`${why}, the list Solid apps use to find your things. FediPod needs one to record `
+      + 'where your account lives. Create it? It will be readable by anyone, like your profile. (y/n)', 'n');
+    if (!/^y/i.test(yes)) { console.log('Setup stopped. Nothing was written to your pod.'); process.exit(0); }
+    createIndex = true;
+  }
+}
+if (newAccount && !root && container == null) {
+  container = await ask(`${CONTAINER_Q} (empty: the new pod's root)`, '');
+}
 console.log('The display name can be changed later; the handle and pod cannot.');
 const go = await ask(newAccount
   ? (kind === 'group' ? 'create pod and group? (y/n)' : 'create pod and Fediverse account? (y/n)')
@@ -147,16 +180,21 @@ if (newAccount) {
   const made = await createAccountWithPod({ issuer, email, password, podName });
   pod = made.pod;
   console.log(`account + pod created: ${pod}`);
+  if (!root) {
+    const chosen = place.chosenRootInNewPod(pod.endsWith('/') ? pod : pod + '/', container);
+    if (chosen.problem) { console.error(`Where to store it: ${chosen.problem}. The pod was made; nothing else was.`); process.exit(2); }
+    root = chosen.root;
+  }
   if (!webfingerHost(pod)) {
     if (kind === 'group') {
-      console.log(`\n${issuerHost} created the pod at ${pod} — a suffix-based host.`);
-      console.log('WebFinger is answered only at a host root, so nobody could find this group.');
+      console.log(`\n${issuerHost} created the pod at ${pod} — a suffixed pod.`);
+      console.log('WebFinger is answered only for a subdomained pod, so nobody could find this group.');
       console.log('The pod exists; no actor was published.');
       process.exit(0);
     }
     // A path pod's address lives at the gateway; nothing to warn about.
     fronted = true;
-    console.log(`${issuerHost} puts pods on paths, so your address will be @${handle}@${gatewayHost}.`);
+    console.log(`${issuerHost} makes suffixed pods, so your address will be @${handle}@${gatewayHost}.`);
   }
 }
 
@@ -207,11 +245,15 @@ await agent.bootstrap({ handle, name, root, kind, approveJoins, summary, icon, g
 await agent.connect({ repair: false });   // publishProfile below is the publish
 await agent.publisher.publishProfile();
 await agent.store.flush();
+// Where the account lives, in the pod's public type index — made first on the yes.
+await place.recordPlace(agent.remote, rec.remotePod, `${rec.remotePod}${root}ap/actor`, { create: createIndex })
+  .then(() => console.log('recorded where the account lives in your public type index'))
+  .catch((e) => { console.error(`could not record where the account lives in your type index: ${e.message}`); process.exitCode = 1; });
 const finalHost = gatewayCfg ? gatewayHost : webfingerHost(rec.remotePod);
 const what = kind === 'group' ? 'group' : 'actor';
 console.log(finalHost
   ? `${what} published: @${handle}@${finalHost}`
-  : `${what} published, but not reachable as a handle \u2014 ${rec.remotePod} is not a host root`);
+  : `${what} published, but not reachable as a handle \u2014 ${rec.remotePod} is a suffixed pod`);
 
 // Straight into serving — setup ends with a working client in the browser.
 const { startAdmin } = await import(new URL('../../../../lib/device/admin/index.mjs', import.meta.url));

package/lib/device/cli/context.mjs

@@ -215,7 +215,7 @@ export async function finish(agent, code = 0) {
 // unpack-and-go line depend on that. At a terminal with none of them, setup
 // asks the two things it needs to open a browser and asks the rest there.
 export const IDENTITY_FLAGS = ['new-account', 'pod', 'issuer', 'email', 'name', 'pod-name',
-  'group', 'approve-joins', 'summary', 'icon', 'root', 'keys', 'rotate-key'];
+  'group', 'approve-joins', 'summary', 'icon', 'root', 'in', 'create-type-index', 'keys', 'rotate-key'];
 
 // Refuse before anything is asked, let alone typed: setup used to overwrite
 // credential.json in place, and a minted credential is only shown once — so

package/lib/device/migrate.mjs

@@ -65,7 +65,7 @@ export const MOVED_STATE_DOCS = new Set([
   'config.json', 'queue.json', 'blocklist.json', 'contacts.json', 'muted.json',
   'pending.json', 'requests.json', 'deadletter.json', 'statuses.json',
   'notifications.json', 'media.json', 'actors.json', 'ids.json', 'published.json',
-  'outbox.json', 'outbox-removed.json', 'parked.json', 'tagfeed.json',
+  'outbox.json', 'outbox-removed.json', 'outbox-own.json', 'liked.json', 'c2s-seen.json', 'parked.json', 'tagfeed.json',
   'intake-attempts.json', 'inbox-channel.json', 'masto-tokens.json', 'masto-markers.json',
   'lists.json', 'filters.json', 'scheduled.json', 'webpush.json',
   'import-state.json',

package/lib/device/setup.mjs

@@ -22,6 +22,7 @@ import { insecureUrlReason } from '../shared/safefetch.mjs';
 import { resourceExists } from '../pod/root.mjs';
 import { CURRENT_LAYOUT, isCurrent } from './migrate.mjs';
 import { completeGatewayMove } from './gateway-move.mjs';
+import { chosenRoot, chosenRootInNewPod, publicIndexKnown, recordPlace, needsIndex } from '../core/place.mjs';
 
 const SOLID = $rdf.Namespace('http://www.w3.org/ns/solid/terms#');
 
@@ -234,10 +235,14 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
     mode, issuer, email, password, handle, name, podName,
     kind = 'person', approveJoins = false, summary, icon, keys, uiPassword,
     gateway = null, shape = 'pod', gatewayOrigin = 'https://fedipod.net',
+    // The container that holds `fedipod/` (blank: the pod's root), and the
+    // person's yes to a new public type index when their profile names none.
+    container = '', createIndex = false,
   } = answers;
   let { pod, root } = answers;
-  if (!root) root = 'fedipod/';   // new installs default to the fedipod/ container; a
-  // resuming run overwrites this with the credential's own root below.
+  const rootGiven = !!root;
+  if (!root) root = 'fedipod/';   // until the pod is known; a resuming run
+  // overwrites this with the credential's own root below.
   let accountWebId = null;   // what createAccountWithPod reported, when it ran
   // The private half always starts here, beside the credential and the keys —
   // not on the pod. Every activity you receive would otherwise cost the pod
@@ -265,6 +270,9 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
       skip('account', 'the credential is already on disk');
     } else if (mode === 'new') {
       begin('account');
+      // A new pod has no public type index: without a yes to making one,
+      // nothing is created at all.
+      if (!createIndex) throw needsIndex('A new pod has no public type index.');
       const made = await createAccount({ issuer, email, password, podName: podName || handle });
       pod = made.pod;
       accountWebId = made.webId || null;
@@ -273,10 +281,15 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
       // nobody could ever find this group. A person was warned before we got
       // here and chose to continue; a group cannot.
       if (kind === 'group' && !webfingerHost(pod)) {
-        throw new Error(`${issuer} created the pod at ${pod} — a suffix-based host, `
-          + 'not a host root. WebFinger is only answered at a host root, so nobody '
+        throw new Error(`${issuer} created the pod at ${pod} — a suffixed pod. `
+          + 'WebFinger is only answered for a subdomained pod, so nobody '
           + 'could find this group. The pod exists; no actor was published.');
       }
+      if (!rootGiven) {
+        const place = chosenRootInNewPod(pod.endsWith('/') ? pod : pod + '/', container);
+        if (place.problem) throw new Error(`Where to store it: ${place.problem}.`);
+        root = place.root;
+      }
       done('account', pod);
     } else {
       // A pod you bring must be there and carry a usable WebID before a
@@ -284,7 +297,15 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
       // reachable, and no silent 401 later on a pod whose profile is empty.
       const usable = await checkPod(pod);
       if (!usable.ok) throw new Error(usable.error);
-      const held = await (deps.resourceExists || resourceExists)(deps.fetch || fetch, apUrls(pod, root || DEFAULT_ROOT).actor);
+      const podBase = pod.endsWith('/') ? pod : pod + '/';
+      if (!rootGiven) {
+        const place = chosenRoot(podBase, container);
+        if (place.problem) throw new Error(`Where to store it: ${place.problem}.`);
+        root = place.root;
+      }
+      // An account at the chosen place, or an older one where accounts used to go.
+      const exists = (r) => (deps.resourceExists || resourceExists)(deps.fetch || fetch, apUrls(pod, r).actor);
+      const held = await exists(root) || (root !== DEFAULT_ROOT && await exists(DEFAULT_ROOT));
       if (held && !frontedAddress({ pod, shape })) {
         throw new Error('The pod already hosts a FediPod account. If you want a second account, put it on a different pod. '
           + 'To move an account whose address is at another gateway to this one, ask for an address at this gateway.');
@@ -292,6 +313,12 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
       if (held && kind === 'group') throw new Error('The pod already hosts a FediPod account, and a group cannot move between gateways.');
       movingIn = held;
       if (movingIn) { privateRoot = null; }
+      // Asked before anything is minted: a profile with no public type index
+      // gets one only on its person's yes.
+      if (!movingIn && !createIndex
+        && await (deps.publicIndexKnown || publicIndexKnown)(deps.fetch || fetch, podBase) !== true) {
+        throw needsIndex('Your profile names no public type index.');
+      }
       skip('account', movingIn ? 'the pod already holds the account — its address moves here' : 'using the pod you already have');
     }
 
@@ -366,8 +393,8 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
     let gatewayCfg = gateway;
     if (!gatewayCfg && frontedAddress({ pod, shape })) {
       if (kind === 'group') {
-        throw new Error(`a group needs a pod at its own host — a Gateway address for a group is not supported yet`
-          + (webfingerHost(pod) ? '' : `; ${pod} is a suffix-based host`) + '.');
+        throw new Error(`a group needs a subdomained pod — a Gateway address for a group is not supported yet`
+          + (webfingerHost(pod) ? '' : `; ${pod} is suffixed`) + '.');
       }
       const urls = apUrls(pod, root || DEFAULT_ROOT);
       const cred = JSON.parse(fs.readFileSync(credPath, 'utf8'));
@@ -414,6 +441,16 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
     }
     const published = await agent.publisher.publishProfile();
     await agent.store.flush();
+    // Where the account lives, in the owner's public type index — made first
+    // if they said yes to one.
+    if (!movingIn) {
+      const urlsAt = apUrls(pod, root);
+      try {
+        await (deps.recordPlace || recordPlace)(agent.publisher?.remote || agent.remote, urlsAt.base, urlsAt.home + 'ap/actor', { create: !!createIndex });
+      } catch (e) {
+        throw new Error(`could not record where the account lives in your type index (${e.message})`);
+      }
+    }
     if (movingIn) {
       // Tell the old gateway and the followers. Not fatal: left pending, it
       // is tried again when the agent next starts acting.
@@ -448,7 +485,7 @@ export async function runSetup({ home, agent, answers, run, deps = {}, log = ()
     log(wfHost
       ? `${kind === 'group' ? 'group' : 'actor'} published: @${handle}@${wfHost}`
       : `${kind === 'group' ? 'group' : 'actor'} published, but not reachable as a handle — `
-        + `${pod} is not a host root`);
+        + `${pod} is a suffixed pod`);
   } catch (e) {
     for (const s of run.steps) if (s.state === 'running') s.state = 'error';
     run.error = scrub(e.message);

package/lib/gateway/caches.mjs

@@ -0,0 +1,36 @@
+// caches.mjs — what one warm gateway process keeps between requests: the
+// pod-token verifier, and the public keys of the servers that deliver here.
+// Built per request, each costs round trips to somebody else's server for
+// every call; built once, a process pays them once.
+
+// The Solid-OIDC verifier remembers the issuers' key sets it has seen. A
+// fresh one per call re-read the caller's WebID document, the issuer's
+// discovery document and its keys on every signed-in call.
+let verifier = null;
+export async function podTokenVerifier() {
+  if (!verifier) verifier = (await import('@solid/access-token-verifier')).createSolidTokenVerifier();
+  return verifier;
+}
+
+// The keys deliveries are signed with, by key id, in the shape Fedify's
+// verifier asks for. A key is held an hour; a key that could not be fetched
+// is held five minutes, so a server that was down is asked again soon.
+// Fedify re-fetches on its own when a held key no longer verifies, so a
+// rotated key costs one failed check, not an hour of refused mail.
+const KEY_TTL_MS = 60 * 60_000;
+const MISS_TTL_MS = 5 * 60_000;
+const KEY_CACHE_MAX = 500;
+const keys = new Map();   // key id → { key, until }
+export const senderKeys = {
+  async get(keyId) {
+    const hit = keys.get(keyId.href);
+    if (!hit) return undefined;
+    if (hit.until < Date.now()) { keys.delete(keyId.href); return undefined; }
+    return hit.key;
+  },
+  async set(keyId, key) {
+    if (keys.size >= KEY_CACHE_MAX) keys.delete(keys.keys().next().value);
+    keys.set(keyId.href, { key, until: Date.now() + (key ? KEY_TTL_MS : MISS_TTL_MS) });
+  },
+  size: () => keys.size,
+};

package/lib/gateway/front-core.mjs

@@ -23,6 +23,10 @@ import * as podPolicy from '../pod/policy.mjs';
 import { podBaseOfWebId } from '../pod/urls.mjs';
 import { routeQuietApi, noteOpened, noteReceived, closedState, accountState, closedAnswer } from './quiet.mjs';
 import { routeNoticesApi } from './notices.mjs';
+import { withSecurityHeaders } from './headers.mjs';
+import { podTokenVerifier } from './caches.mjs';
+import { claimedWebId } from './token-claims.mjs';
+import { withdrawn, overLimit, relayOne } from './relay-extras.mjs';
 
 // The one WebFinger document, spelled out here rather than imported from
 // wire.mjs: wire drags the agent's whole HTML pipeline (sanitize-html and
@@ -61,7 +65,7 @@ async function verifyPodToken(request, pathname, verifier) {
     // Solid-OIDC binds the token to a key the client proves on every request;
     // a token shown without the proof is one anyone who saw it could show.
     if (!dpop) { console.log(`front: pod token without a DPoP proof refused on ${pathname}`); return null; }
-    const v = verifier || (await import('@solid/access-token-verifier')).createSolidTokenVerifier();
+    const v = verifier || await podTokenVerifier();
     const url = request.url;
     const { webid } = await v(authz, { header: dpop, method: request.method, url });
     return webid || null;
@@ -89,9 +93,13 @@ const podOwners = (podBase, fetchImpl = fetch) =>
   podRoot.readOwnerLinks(fetchImpl, podBase, { timeoutMs: OWNER_LOOKUP_MS });
 
 // Whether a token's WebID vouches for this pod, on the evidence of where it
-// lives. Weak on purpose, and never used alone any more — see provesPod.
+// lives: the same origin, and — for a suffixed pod — inside
+// the WebID's own pod, wherever in it the account was put. Weak on purpose,
+// and never used alone for an attach — see provesPod.
 function webidUnderPod(webid, podHome) {
-  try { return new URL(webid).origin === new URL(podHome).origin; } catch { return false; }
+  try {
+    return new URL(webid).origin === new URL(podHome).origin && String(podHome).startsWith(podBaseOfWebId(webid));
+  } catch { return false; }
 }
 
 /**
@@ -142,15 +150,8 @@ function podHomeProblem(podHome, frontOrigin) {
 }
 
 const RELAY_MAX_REQUESTS = 20;
-const RELAY_MAX_BODY = 1024 * 1024;
-const RELAY_TIMEOUT_MS = 8_000;
-// The only headers a relayed request may carry to the remote server. Host
-// comes from the URL; the user agent from safeFetch.
 const SERVER_PROBE_MS = 6000;
 const SERVER_PROBE_BYTES = 64 * 1024;
-const RELAY_HEADERS = new Set(['date', 'digest', 'signature', 'content-type', 'accept']);
-
-const keyIdOf = (signature) => (/keyId="([^"]+)"/.exec(signature || '') || [])[1] || null;
 
 // The owner's APIs a browser at ANOTHER origin may call: an address moving
 // to a new gateway is driven from the new gateway's page, and it has to tell
@@ -165,49 +166,11 @@ const API_CORS = {
 const withApiCors = (out) => ({ ...out, headers: { ...(out.headers || {}), ...API_CORS } });
 const apiPreflight = () => ({ status: 204, headers: { ...API_CORS, allow: 'POST, OPTIONS' }, body: null });
 
-async function relayOne(item, rec, fetchImpl) {
-  const url = String(item?.url || '');
-  const method = String(item?.method || 'POST').toUpperCase();
-  if (method !== 'GET' && method !== 'POST') return { url, status: 0, error: 'method must be GET or POST' };
-  let u;
-  try { u = new URL(url); } catch { return { url, status: 0, error: 'not a URL' }; }
-  if (u.protocol !== 'https:' && !(u.protocol === 'http:' && process.env.AP_ALLOW_PRIVATE_TARGETS === '1')) {
-    return { url, status: 0, error: 'https only' };
-  }
-  const headers = {};
-  for (const [k, v] of Object.entries(item?.headers || {})) {
-    const name = k.toLowerCase();
-    if (RELAY_HEADERS.has(name) && typeof v === 'string') headers[name] = v;
-  }
-  const body = method === 'POST' ? String(item?.body ?? '') : undefined;
-  if (body !== undefined && Buffer.byteLength(body) > RELAY_MAX_BODY) return { url, status: 0, error: 'body too large' };
-  const keyId = keyIdOf(headers.signature);
-  if (method === 'POST' && !keyId) return { url, status: 0, error: 'a delivery must be signed' };
-  if (keyId && !keyId.startsWith(rec.actorUrl + '#')) return { url, status: 0, error: "signed with a key that is not this account's" };
-  if (headers.digest) {
-    const want = 'SHA-256=' + crypto.createHash('sha256').update(body || '').digest('base64');
-    if (headers.digest !== want) return { url, status: 0, error: 'digest does not match the body' };
-  }
-  try {
-    const res = await safeFetch(url, { method, headers, body, signal: AbortSignal.timeout(RELAY_TIMEOUT_MS) }, fetchImpl);
-    const out = { url, method, status: res.status };
-    // The far server asking to be left alone has to reach the agent that will
-    // do the asking again. Without this the browser build could not honour a
-    // Retry-After at all — every delivery it makes goes through here — and fell
-    // back to its own ladder against a server that had already said how long.
-    const retryAfter = res.headers.get('retry-after');
-    if (retryAfter) out.retryAfter = retryAfter;
-    if (method === 'GET') {
-      out.contentType = res.headers.get('content-type') || null;
-      out.body = await readCapped(res, RELAY_MAX_BODY);
-    }
-    return out;
-  } catch (e) { return { url, method, status: 0, error: e.message }; }
-}
-
 const j = (status, obj, ct = 'application/json') =>
   ({ status, headers: { 'content-type': ct, 'cache-control': 'no-store' }, body: JSON.stringify(obj) });
-const notFound = () => ({ status: 404, headers: { 'content-type': 'text/plain' }, body: 'not found\n' });
+// Held at the edge like any other missing document: a bot's scan, a dead
+// handle, a typo in a WebFinger query each cost one call, not one per asker.
+const notFound = () => ({ status: 404, headers: { 'content-type': 'text/plain', ...publicFor(MISSING_EDGE_SECONDS) }, body: 'not found\n' });
 
 // A user's public base on the front, a 1:1 mirror of their pod home.
 // The directory key is the FULL fediverse address — handle@host — never the
@@ -243,7 +206,9 @@ function parseUserPath(pathname) {
 const BROWSER_MAX_AGE = 60;
 const publicFor = (seconds, stale = seconds * 4) => ({
   'cache-control': `public, max-age=${Math.min(seconds, BROWSER_MAX_AGE)}`,
-  'netlify-cdn-cache-control': `public, s-maxage=${seconds}, stale-while-revalidate=${stale}`,
+  // `durable`: one copy for every region, and one that a deploy does not
+  // empty. Without it each region held its own and every deploy started cold.
+  'netlify-cdn-cache-control': `public, durable, s-maxage=${seconds}, stale-while-revalidate=${stale}`,
 });
 // How long the edge holds a public document, and a handle's WebFinger
 // answer. Every server that has heard of an account asks for its actor and
@@ -252,6 +217,15 @@ const publicFor = (seconds, stale = seconds * 4) => ({
 // can afford.
 const PUBLIC_EDGE_SECONDS = 600;
 const WEBFINGER_EDGE_SECONDS = 3600;
+// A closed or moved address stays gone; a picture's address stays where it is.
+const GONE_EDGE_SECONDS = 3600;
+const MEDIA_EDGE_SECONDS = 86400;
+// A public document the pod would not give (missing, or not public) is asked
+// for again and again by every server that shows the account — a pinned-posts
+// collection an old account never wrote, a forum count nobody published. Held
+// briefly, so the edge absorbs the asking; briefly, so a document published a
+// moment later is not "missing" for long.
+const MISSING_EDGE_SECONDS = 120;
 
 // Where a pod owner opts their identity in. An operator may name it, because
 // the path it takes is one their pod server can no longer serve; the dot says
@@ -344,51 +318,6 @@ export async function routeFront(request, ctx) {
   return { ...out, headers: withSecurityHeaders(out.headers, out.body) };
 }
 
-// Every response this file makes, hardened in one place rather than in each of
-// the dozen shapes below.
-//
-// `nosniff` matters most: the front serves user-supplied JSON straight from
-// somebody's pod (the proxied actor and object documents), and without it a
-// browser is free to decide for itself that a document is HTML and run what is
-// inside it. The rest is the same posture the app already has — nothing may be
-// framed, no base tag may be rewritten, no plugin content.
-//
-// A content-security-policy goes on the HTML only: it would mean nothing on a
-// JSON document, and `frame-ancestors` has to be a header rather than a meta
-// tag anyway.
-//
-// `script-src 'self'` is the one that matters, and it is only possible because
-// none of these pages carries inline script any more — each has its own file and
-// its own route above. A policy cannot tell an inline block the author wrote
-// from one an attacker injected, so as long as any inline script has to run,
-// every inline script may.
-//
-// `connect-src` allows https: because the pages sign in against the user's own
-// pod, which is a different origin by definition and not one we can name here.
-function withSecurityHeaders(headers = {}, body = null) {
-  const ct = String(headers['content-type'] || '');
-  const isHtml = ct.startsWith('text/html');
-  return {
-    ...headers,
-    'x-content-type-options': 'nosniff',
-    'referrer-policy': 'same-origin',
-    'x-frame-options': 'SAMEORIGIN',
-    ...(isHtml && body ? {
-      'content-security-policy': [
-        "default-src 'self'",
-        "script-src 'self'",             // no inline script: see above
-        "style-src 'self' 'unsafe-inline'",
-        "img-src 'self' https: data:",
-        "connect-src 'self' https:",     // sign-in goes to the user's own pod
-        "object-src 'none'",             // no plugin content, ever
-        "base-uri 'none'",               // no rewriting where relative URLs resolve
-        "frame-ancestors 'self'",        // nobody else may frame these pages
-        "form-action 'self'",            // a form here submits here
-      ].join('; '),
-    } : {}),
-  };
-}
-
 async function route(request, ctx) {
   const url = new URL(request.url);
   const { pathname } = url;
@@ -643,8 +572,8 @@ async function route(request, ctx) {
   }
 
   // The owner's say over an account that goes quiet (quiet.mjs).
-  const quiet = await routeQuietApi(request, pathname, ctx, { j, verifyPodToken, webidUnderPod });
-  if (quiet) return quiet;
+  const quiet = await routeQuietApi(request, pathname, ctx, { j, verifyPodToken, webidUnderPod, apiPreflight });
+  if (quiet) return withApiCors(quiet);
 
   // The relay: the front sends requests a browser has already signed. A page
   // may not set the Date or Host header, and both are inside an HTTP
@@ -669,7 +598,11 @@ async function route(request, ctx) {
     // Acting through the relay is being here. Hourly at most; see noteOpened.
     await noteOpened(ctx, handle, rec).catch((e) => console.log(`relay @${handle}: stamp not written: ${e?.message || e}`));
     if (items.length > RELAY_MAX_REQUESTS) return withApiCors(j(400, { error: `at most ${RELAY_MAX_REQUESTS} requests per call` }));
+    const slow = overLimit(handle, items.length) && j(429, { error: 'too many relayed requests — slow down' });
+    if (slow) return withApiCors({ ...slow, headers: { ...slow.headers, 'retry-after': '60' } });
     const results = await Promise.all(items.map((it) => relayOne(it, rec, ctx.fetchImpl || fetch)));
+    // Something of the account's withdrawn: the edge's copies go.
+    if (withdrawn(items, rec.actorUrl)) await ctx.purge?.([`u-${handle}`]).catch((e) => console.log(`purge @${handle}: ${e?.message || e}`));
     // One line per relayed request, so a lookup that fails on the far side is
     // visible here and not only as an empty result in someone's browser.
     for (const r of results) console.log(`relay @${handle}: ${r.method || ''} ${r.url} → ${r.status}${r.error ? ` (${r.error})` : ''}`);
@@ -754,7 +687,8 @@ async function route(request, ctx) {
       return j(403, { error: 'the token proves a different pod than the one you listed' });
     }
     if (action === 'opt-in') {
-      const { httpStatus, ...reply } = await ctx.agentControl.optIn({ podBase, webId: webid });
+      const { httpStatus, ...reply } = await ctx.agentControl.optIn({ podBase, webId: webid,
+        container: String(body.container || ''), createIndex: body.createIndex === true });
       if (reply.doorSecret) {
       // The secret appears here and nowhere else; the command is the
       // paste-and-run way to open the owner door once.
@@ -808,7 +742,7 @@ async function route(request, ctx) {
     if (!rec) return notFound();
     // A closed address is gone, and says so rather than pretending never to
     // have existed: the name stays taken.
-    if ((await closedState(ctx, m[1], rec)).closed) return closedAnswer({ 'access-control-allow-origin': '*' });
+    if ((await closedState(ctx, m[1], rec)).closed) return closedAnswer({ 'access-control-allow-origin': '*', ...publicFor(GONE_EDGE_SECONDS) });
     // A fronted identity's documents live on its pod; the pod's own actor id
     // is the alias, so a client signing in by the fronted address can find
     // the pod (and its login) without a lookup only the host could answer.
@@ -906,13 +840,28 @@ async function route(request, ctx) {
       if (!webid) return json(401, { error: 'a Solid-OIDC token proving this account\'s owner is required' });
       const owner = rec.webId ? webid === rec.webId : webidUnderPod(webid, rec.podHome);
       if (!owner) return json(403, { error: 'this outbox belongs to its owner alone' });
-      const { status, reason, location } = await handleOwnerPost(request, identFor(rec),
-        { podPut: (u, b, ct) => ctx.podPut(up.handle, u, b, ct), ownerWebId: webid });
+      const onPod = (u) => u.replace(rec.actorUrl.replace(/ap\/actor$/u, ''), rec.podHome);
+      const { status, reason, location, object } = await handleOwnerPost(request, identFor(rec),
+        { podPut: (u, b, ct) => ctx.podPut(up.handle, u, b, ct), ownerWebId: webid,
+          exists: async (u) => ((await (ctx.podGet || fetch)(onPod(u), { podHome: rec.podHome })).status !== 404) });
       console.log(`door @${up.handle}: owner post → ${status} (${reason})`);
-      if (status !== 202) return json(status, { error: reason });
-      return json(202, { accepted: true, ...(location ? { object: location } : {}),
+      if (status !== 201) return json(status, { error: reason });
+      return json(201, { accepted: true, ...(location ? { id: location } : {}), ...(object ? { object } : {}),
         note: 'it goes out when your FediPod agent next runs' }, location ? { location } : {});
     }
+    // Its owner, signed in, reads every message the account produced (§5.1:
+    // the outbox is filtered by who asks); that view is on the pod, under the
+    // owner's own rule. Anyone else reads the public one.
+    // The token is only read here, not verified: the redirect grants nothing,
+    // the pod checks the credential when the owner arrives there, and a
+    // verification per read would let anyone make the front fetch keys from a
+    // server they name.
+    if ((request.method === 'GET' || request.method === 'HEAD') && request.headers.get('authorization')) {
+      const webid = claimedWebId(request);
+      if (webid && (rec.webId ? webid === rec.webId : webidUnderPod(webid, rec.podHome))) {
+        return { status: 303, headers: { ...cors, location: rec.podHome + 'ap/private/outbox', 'cache-control': 'no-store' }, body: '' };
+      }
+    }
     if (rec.inboxOnly && (request.method === 'GET' || request.method === 'HEAD')) {
       return { status: 303, headers: { ...cors, location: rec.podHome + 'ap/outbox', 'cache-control': 'no-store' }, body: '' };
     }
@@ -935,14 +884,20 @@ async function route(request, ctx) {
   if (movedBase && up.rest !== 'ap/actor') {
     return { status: 301, headers: { ...open, location: movedBase + up.rest, 'cache-control': 'no-store' }, body: '' };
   }
-  if (closed) return gone(open, CLOSED);
+  if (closed) return gone({ ...open, ...publicFor(GONE_EDGE_SECONDS) }, CLOSED);
   const podTarget = rec.podHome + up.rest;
+  // The owner's own documents — the full outbox, the liked list, the pending
+  // lists — are read at the pod, which decides who may; never held here,
+  // where one reader's answer would be served to the next.
+  if (up.rest.startsWith('ap/private/')) {
+    return { status: 303, headers: { ...open, location: podTarget, 'cache-control': 'no-store' }, body: '' };
+  }
   // Media stays on the pod (lib/pod/urls.mjs keeps `media` off the front), but
   // the id rewrite below turns media links onto the front like every other
   // pod url in a document. Answer those by pointing at the pod: bytes are not
   // a document to cap and relabel, and remotes follow a redirect for a picture.
   if (up.rest.startsWith('ap/media/')) {
-    return { status: 302, headers: { location: podTarget, 'cache-control': 'no-store' }, body: '' };
+    return { status: 302, headers: { location: podTarget, ...publicFor(MEDIA_EDGE_SECONDS) }, body: '' };
   }
   // The pod this read belongs to travels with it: an adapter reading a store
   // directly (the CSS server component) has no access control of its own and
@@ -958,7 +913,11 @@ async function route(request, ctx) {
   const got = await podRoot.readPublicDocument(
     ctx.podGet || ((u) => fetch(u, { headers: { accept } })),
     podTarget, { podHome: rec.podHome, accept });
-  if (got.text === null) return { status: got.status, headers: open, body: '' };
+  if (got.text === null) {
+    const hold = [401, 403, 404, 410].includes(got.status) ? { ...publicFor(MISSING_EDGE_SECONDS), 'netlify-cache-tag': `u-${up.handle}` } : {};
+    // A stranger is not told a document exists that they may not read (§3.2).
+    return { status: got.status === 401 || got.status === 403 ? 404 : got.status, headers: { ...open, ...hold }, body: '' };
+  }
   let text = got.text;
   text = swap(text, rec.podHome, base);
   // The moved stub: the pod's actor now carries the NEW gateway's ids, and a
@@ -973,6 +932,9 @@ async function route(request, ctx) {
       doc.preferredUsername = up.handle;               // so @handle@front cross-checks
       doc.inbox = base + 'ap/inbox/';                  // deliveries come to the front to be verified
       doc.endpoints = { ...(doc.endpoints || {}), sharedInbox: base + 'ap/inbox/' };
+      // The owner's own documents are read at the pod, where their credential
+      // works; they keep the pod's address.
+      for (const k of ['liked', 'ownerOutbox']) if (typeof doc[k] === 'string') doc[k] = doc[k].replace(base, rec.podHome);
       if (movedBase) {
         doc.movedTo = rec.movedTo;
         const aka = new Set([].concat(doc.alsoKnownAs || []).filter((a) => typeof a === 'string' && a !== doc.id));
@@ -988,9 +950,16 @@ async function route(request, ctx) {
     } catch { /* leave the rewritten text as-is if it will not parse */ }
   }
   // `readPublicDocument` asked for it with no credential, so a document that
-  // came back is one anybody can read — which is what makes it cacheable.
-  return { status: 200,
-    headers: { ...open, 'content-type': got.type || (container ? 'text/turtle' : page ? 'text/html' : AP_CT), ...publicFor(PUBLIC_EDGE_SECONDS) },
+  // came back is one anybody can read — which is what makes it cacheable. A
+  // deleted post's Tombstone answers 410 Gone, and still says what it was.
+  let kind = null;
+  try { kind = JSON.parse(text)?.type; } catch { /* not JSON: a page or a container */ }
+  return { status: kind === 'Tombstone' ? 410 : 200,
+    headers: { ...open, 'content-type': got.type || (container ? 'text/turtle' : page ? 'text/html' : AP_CT), ...publicFor(PUBLIC_EDGE_SECONDS),
+      // The outbox answers its signed-in owner differently, so the edge keeps
+      // the public copy for those who show no credential.
+      ...(up.rest === 'ap/outbox' ? { 'netlify-vary': 'header=Authorization', vary: 'Authorization' } : {}),
+      'netlify-cache-tag': `u-${up.handle}` },
     body: text };
 }