fedipod 1.45.0 → 1.48.0

package/lib/core/wire.mjs

@@ -70,13 +70,13 @@ export function jrd({ handle, host, actor, aliases = [], page = null }) {
 // `kind: 'group'` publishes a Group rather than a Person, which is what makes
 // Mastodon and Lemmy treat the actor as a community you join.
 // Whether a follow will be held for the operator instead of accepted on
-// arrival. A person's follows wait unless `autoAcceptFollows` says otherwise,
-// because a delivery on its own proves nothing about who sent it; a group is
-// open unless its operator asked to see joins. The actor document has to say
+// arrival. A person's follows are accepted unless `autoAcceptFollows: false`
+// asks to see them; a group is open unless its operator asked to see joins.
+// The actor document has to say
 // the same thing, or a remote server shows "Following" for a follow that is
 // in fact still waiting, and nothing ever tells it otherwise.
 export function followsNeedApproval(config = {}) {
-  return !!config.approveJoins || (config.kind !== 'group' && !config.autoAcceptFollows);
+  return !!config.approveJoins || (config.kind !== 'group' && config.autoAcceptFollows === false);
 }
 
 export const assertionKeyId = (urls) => urls.actor + '#ed25519-key';
@@ -654,6 +654,16 @@ export function quotePolicy(visibility) {
   return { canQuote: { automaticApproval: open ? [PUBLIC] : [], manualApproval: [] } };
 }
 
+// How many liked a post and how many boosted it (§5.7, §5.8): the totals,
+// with nobody named. Both sections let a collection be filtered for whoever
+// asks, and a total with no names is how Mastodon serves them too.
+export function reactionCounts(likes = 0, shares = 0) {
+  return {
+    likes: { type: 'Collection', totalItems: Math.max(0, Number(likes) || 0) },
+    shares: { type: 'Collection', totalItems: Math.max(0, Number(shares) || 0) },
+  };
+}
+
 // `also`: actors addressed without being mentioned — the parent's author on
 // a reply. In cc but not in tag, so their server receives the post and does
 // not notify them. A direct post names its audience in the text alone.
@@ -661,7 +671,7 @@ export function quotePolicy(visibility) {
 // authorization is the quoted author's QuoteAuthorization, once granted.
 export function noteDoc({ urls, slug, content, published, inReplyTo, attachments, mentions = [],
   visibility = 'public', summary = null, sensitive = false, updated = null, container = null, also = [],
-  quote = null }) {
+  quote = null, likes = 0, shares = 0 }) {
   const id = (container || urls.notes) + slug;
   const who = mentions.map(m => m.actor);
   const extra = visibility === 'direct' ? [] : also.filter(a => a && !who.includes(a));
@@ -676,6 +686,7 @@ export function noteDoc({ urls, slug, content, published, inReplyTo, attachments
     to: addressed.to,
     cc: addressed.cc,
     replies: repliesId(id),
+    ...reactionCounts(likes, shares),
     interactionPolicy: quotePolicy(visibility),
   };
   if (quote?.id) {

package/lib/gateway/account-agent.mjs

@@ -4,9 +4,9 @@
 //
 // It works with the gateway's own pod identity and nothing else, which the
 // owner's app named in the rules on the account's folder. The account's state
-// is its copy at the gateway when there is one (copy.mjs), and the pod when
-// there is not. The signing key is read from the pod when it is needed, never
-// kept, never made.
+// is its copy at the gateway when there is one (copy.mjs), made from the pod
+// while the hold is on, and the pod when the hold is off. The signing key is
+// read from the pod when it is needed, never kept, never made.
 import crypto from 'node:crypto';
 import { PodTransport } from '../pod/transport.mjs';
 import { podBaseOfWebId } from '../pod/urls.mjs';
@@ -19,7 +19,8 @@ import { Publisher } from '../core/publisher/index.mjs';
 import { Intake } from '../core/intake/index.mjs';
 import { C2S } from '../client/c2s.mjs';
 import { keeperSession } from './keeper-session.mjs';
-import { CopyStorage, copyMeta, copyLease, keptBefore, GATEWAY_HOLDER } from './copy.mjs';
+import { CopyStorage, copyMeta, copyLease, keptBefore, holdOn, GATEWAY_HOLDER } from './copy.mjs';
+import { makeCopy } from './state-api.mjs';
 
 const RSA_ALG = { name: 'RSASSA-PKCS1-v1_5', hash: 'SHA-256' };
 
@@ -58,6 +59,21 @@ export async function reachAccount(ctx, handle, rec, { log = console.log, sessio
   return { remote, podFetch: (u, i) => remote.fetch(u, i), pod, root, urls: apUrls(pod, root), inCopy };
 }
 
+/**
+ * Where this piece of work reads and writes the account, decided by a caller
+ * holding lockCopy, so the round cannot delete the copy in between: a copy
+ * that is here is used; none, while the hold is on, is made from the pod; with
+ * the hold off, the pod. Sets `at.inCopy`; returns { ok } or { ok: false,
+ * status, why }.
+ */
+export async function settle(ctx, handle, rec, at, { log = console.log } = {}) {
+  at.inCopy = !!(ctx.copyKv && await copyMeta(ctx.copyKv, handle));
+  if (at.inCopy || !ctx.copyKv || !holdOn(ctx)) return { ok: true };
+  const made = await makeCopy(ctx, handle, rec, log);
+  at.inCopy = made.ok;
+  return made;
+}
+
 /**
  * The account's state store and lease, as `holder`: over the copy when there
  * is one, else over the pod.

package/lib/gateway/copy.mjs

@@ -3,11 +3,17 @@
 // While the gateway keeps an account running, the account's state documents
 // (everything under ap-state/) live here, and every agent works from this copy:
 // the owner's browser (through the state API, state-api.mjs), the keeper and
-// the gateway's Mastodon API (in process). The pod is written from it every
-// fifteen minutes (flushCopy), and at once when the copy is given up
-// (dropCopy). What only the pod may hold never comes here: the signing key,
-// the pod's own lease document, and the passwords and tokens of accounts on
-// other servers.
+// the gateway's Mastodon API (in process). What only the pod may hold never
+// comes here: the signing key, the pod's own lease document, and the passwords
+// and tokens of accounts on other servers.
+//
+// The hold: a person's copy is made from the pod when something needs it, and
+// every fifteen minutes, whatever is happening, it is written to the pod and
+// deleted (holdOver); the next use makes it again. So nothing private stays
+// here longer than that. A gateway's admin may turn the hold off
+// (FEDIPOD_HOLD=off): then no copy is made at all, and everything works on the
+// pod directly. A forum's copy is kept all the time, and written to its pod
+// every fifteen minutes (flushCopy), as before.
 //
 // The lease that decides which agent acts lives in the copy too, and the copy
 // is what enforces it: a state document is written only by the lease's holder.
@@ -89,6 +95,30 @@ const readJson = async (kv, key) => {
 export const keptNow = (ctx, rec) => !!(rec?.keeper && ctx.keeperWebId && (rec.keeper.webId || ctx.keeperWebId) === ctx.keeperWebId);
 export const keptBefore = (ctx, rec) => !!(rec?.keeper?.webId && ctx.keeperWebId && rec.keeper.webId !== ctx.keeperWebId);
 
+// A person's account, as against a forum or one of its categories.
+export const isPersonal = (rec) => !!rec && (rec.kind || 'person') === 'person' && !rec.forum;
+
+// The admin's setting: the hold is on unless FEDIPOD_HOLD says off.
+export const holdFrom = (v) => !/^(off|no|false|0)$/iu.test(String(v ?? '').trim());
+export const holdOn = (ctx) => ctx?.hold !== false;
+
+// While the round writes a copy to the pod and deletes it: the state API asks
+// its callers to try again in a moment, and the gateway's readers wait.
+const CLOSING_MS = 2 * 60_000;
+export async function closing(kv, h) {
+  const at = Number((await kv.get(`${account(h)}/closing`))?.text || 0);
+  return !!at && Date.now() - at < CLOSING_MS;
+}
+/** Wait, up to `waitMs`, for a copy being written and deleted to be gone. */
+export async function waitOutClosing(kv, h, { waitMs = 10_000 } = {}) {
+  const until = Date.now() + waitMs;
+  while (await closing(kv, h)) {
+    if (Date.now() > until) return false;
+    await new Promise((r) => setTimeout(r, 250));
+  }
+  return true;
+}
+
 /**
  * Forget a copy its owner has written to the pod themselves: no write-back, no
  * pod lease to free (the owner took it). Asked only by the lease's holder.
@@ -135,6 +165,12 @@ export function copyLease(kv, h, { id, log = () => {} }) {
   return new Lease({ url: `copy:${account(h)}/lease`, fetchImpl: kvDocFetch(kv, `${h}/lease`), log, id });
 }
 
+/** Whether a browser, as against the gateway, holds the copy's lease now. */
+export async function heldByBrowser(kv, h) {
+  const { doc } = await readJson(kv, `${account(h)}/lease`);
+  return !!doc && doc.holder !== GATEWAY_HOLDER && Date.now() < doc.expiresAt;
+}
+
 /** Whether `holder` holds the copy's lease now. */
 export async function holds(kv, h, holder) {
   const { doc } = await readJson(kv, `${account(h)}/lease`);
@@ -275,10 +311,14 @@ export async function fillCopy(kv, h, { pod, podFetch, stateUrl, webId = null, l
   }
   const undo = async (why) => {
     for (const b of await kv.list(`${h}/d/`).catch(() => [])) await kv.delete(b.key).catch(() => {});
+    await kv.delete(`_copies/${h}`).catch(() => {});
     await podLease.release().catch(() => {});
     return { ok: false, why };
   };
   try {
+    // Listed from the start, so a making cut off part way is found by the
+    // round and its documents deleted (clearLeftover).
+    await kv.set(`_copies/${h}`, String(Date.now()));
     // Whatever an interrupted give-up left behind is not this copy.
     for (const b of await kv.list(`${h}/d/`)) await kv.delete(b.key);
     const listing = await pod.list('');
@@ -388,3 +428,53 @@ export async function dropCopy(kv, h, { pod, podFetch, stateUrl, log = () => {}
   log(`copy @${h}: given up; the pod holds everything`);
   return { ok: true };
 }
+
+/**
+ * The end of the hold, for a person's copy: everything written to the pod, the
+ * pod's lease freed, and the copy deleted. Unlike giving the copy up
+ * (dropCopy), the copy's lease stays, so the browser holding it goes on holding
+ * it in the copy made next. Asked by the round, which holds lockCopy. Writes
+ * that reach the state API meanwhile are asked to come again (closing), and
+ * the next one finds the copy made again from the pod. Returns { ok } or
+ * { ok: false, why }; nothing is deleted unless the pod has it all.
+ */
+export async function holdOver(kv, h, { pod, podFetch, stateUrl, log = () => {} }) {
+  const meta = await copyMeta(kv, h);
+  if (!meta) return { ok: true, already: true };
+  await kv.set(`${account(h)}/closing`, String(Date.now()));
+  try {
+    // Again, twice at most, for a write that landed while the first went: the
+    // marker stops any more coming.
+    let behind = [];
+    for (let i = 0; i < 3; i++) {
+      await flushCopy(kv, h, { pod, log });
+      const { doc: flushed } = await readJson(kv, `${h}/flushed`);
+      behind = (await kv.list(`${h}/d/`)).filter((b) => flushed?.[b.key.slice(`${h}/d/`.length)]?.etag !== b.etag);
+      if (!behind.length) break;
+    }
+    if (behind.length) return { ok: false, why: `${behind.length} document(s) could not be written to the pod` };
+    const podLease = new Lease({ url: (stateUrl || meta.stateUrl) + 'lease.json', fetchImpl: podFetch, log, id: POD_HOLDER });
+    try { await podLease.write({ holder: POD_HOLDER, expiresAt: 0 }); } catch (e) {
+      return { ok: false, why: `the pod's lease could not be freed (${e.message})` };
+    }
+    // The copy is gone the moment its meta is. Its listing goes last, so a
+    // delete cut off part way is found and finished by the next round
+    // (clearLeftover).
+    await kv.delete(`${h}/meta`);
+    await kv.delete(`${h}/flushed`);
+    for (const b of await kv.list(`${h}/d/`)) await kv.delete(b.key);
+    await kv.delete(`_copies/${h}`);
+    log(`copy @${h}: written to the pod and deleted (the hold)`);
+    return { ok: true };
+  } finally { await kv.delete(`${h}/closing`).catch(() => {}); }
+}
+
+/** A copy listed with no meta is one whose delete was cut off: its documents go. */
+export async function clearLeftover(kv, h, { log = () => {} } = {}) {
+  if (await copyMeta(kv, h)) return false;
+  const left = await kv.list(`${account(h)}/d/`);
+  for (const b of left) await kv.delete(b.key);
+  await kv.delete(`_copies/${h}`);
+  if (left.length) log(`copy @${h}: ${left.length} document(s) left by a cut-off delete, deleted`);
+  return true;
+}

package/lib/gateway/front-core.mjs

@@ -23,7 +23,7 @@ import * as podRoot from '../pod/root.mjs';
 import * as podPolicy from '../pod/policy.mjs';
 import { podBaseOfWebId } from '../pod/urls.mjs';
 import { routeQuietApi, noteOpened, noteReceived, closedState, accountState, closedAnswer, provedOwner } from './quiet.mjs';
-import { holdsMail, isPresent, holdingPut, afterHeld, routeHereApi, routeKeeperApi } from './held-mail.mjs';
+import { holdsMail, mayBeAway, isPresent, holdingPut, afterHeld, routeHereApi, routeKeeperApi } from './held-mail.mjs';
 import { routeNoticesApi } from './notices.mjs';
 import { withSecurityHeaders } from './headers.mjs';
 import { podTokenVerifier } from './caches.mjs';
@@ -825,15 +825,15 @@ async function route(request, ctx) {
     if (closed) { console.log(`door @${up.handle}: delivery → 410 (closed)`); return gone(doorCors, CLOSED); }
     const policy = await policyFor(rec, ctx.fetchImpl || fetch);
     const standing = await accountState(ctx, up.handle, rec);
-    // A browser account's mail waits here while its app is closed (held-mail.mjs).
-    const held = holdsMail(ctx, rec) && !(await isPresent(ctx, up.handle));
+    // A browser account's mail waits here while its app is closed (held-mail.mjs), unless the hold is off.
+    const away = mayBeAway(ctx, rec) && !(await isPresent(ctx, up.handle)); const held = away && holdsMail(ctx, rec);
     const { status, reason, content, bytes, type, notifies, name, raw, receipt } = await handleDelivery(request, identFor(rec, policy),
       { podPut: held ? holdingPut(ctx, up.handle, rec) : (u, b, ct) => ctx.podPut(up.handle, u, b, ct),
         fetchImpl: ctx.fetchImpl, paused: standing.paused, origin: ctx.frontOrigin || null });
     // One line per delivery, so "did it arrive at the door" has an answer.
     console.log(`door @${up.handle}: delivery → ${status} (${reason}${held && status === 202 ? ', held' : ''})`);
     if (status === 202 && content) await noteReceived(ctx, up.handle, rec, bytes);
-    if (held && status === 202) await afterHeld(ctx, up.handle, rec, { type, notifies });
+    if (away && status === 202) await afterHeld(ctx, up.handle, rec, { type, notifies });
     // A forum fedipod.net keeps: the delivery is placed now, in this request,
     // and the run that carries it is started. Whatever could not be placed
     // here is still in the inbox for that run.

package/lib/gateway/held-mail.mjs

@@ -6,7 +6,9 @@
 // app is closed nobody reads it, so the door keeps it here instead
 // (ctx.holdMail), and writes it to the pod inbox as one document when the app
 // says it is open (POST /api/here) or when the timer comes round (flushAll).
-// While the app is open, mail goes straight to the pod as it always has.
+// While the app is open, mail goes straight to the pod as it always has. With
+// the hold off (copy.mjs: FEDIPOD_HOLD), nothing is held: mail goes straight
+// to the pod whether the app is open or not.
 //
 // A batch is plain JSON, `{ "batch": [ { name, body, receipt } ] }`: `name` is
 // what the item would have been called in the inbox, `body` its bytes, and
@@ -23,7 +25,7 @@
 //   presentAt(handle) -> ms            when it last said so
 //   noteNext(handle, at, {earliest})   when the account's keeper next has work
 
-import { copyMeta, keptNow } from './copy.mjs';
+import { copyMeta, keptNow, holdOn } from './copy.mjs';
 
 // The app says it is open every five minutes; a little over that counts.
 export const PRESENT_MS = 8 * 60_000;
@@ -37,7 +39,11 @@ const inboxOf = (rec) => rec.inboxUrl || rec.podHome + 'ap/inbox/';
 // Only a browser account is held: its owner signs in (so the row has
 // `openedAt`), and its app says when it is open. An account run by a DeviceAgent
 // drains whenever mail lands, and keeps its mail as it always has.
-export const holdsMail = (ctx, rec) => !!(ctx.holdMail && rec?.openedAt);
+export const holdsMail = (ctx, rec) => !!(ctx.holdMail && rec?.openedAt && holdOn(ctx));
+// A browser account whose app may be closed: its mail is held, or with the
+// hold off, sent to the pod, and either way a follow or a notification starts
+// the work that answers it (afterHeld).
+export const mayBeAway = (ctx, rec) => !!(ctx.holdMail && rec?.openedAt);
 
 // Asked on every delivery, so it is remembered a minute per running copy.
 const seen = new Map();   // handle -> { at, asked }
@@ -204,5 +210,6 @@ export async function routeHereApi(request, pathname, ctx, deps) {
   if (body.nextAt !== undefined && ctx.noteNext) await ctx.noteNext(handle, body.nextAt || null).catch(() => {});
   const flushed = await flushHeld(ctx, handle, rec)
     .catch((e) => { console.log(`here @${handle}: held mail not delivered: ${e?.message || e}`); return 0; });
-  return j(200, { ok: true, handle, flushed });
+  // `hold`: with it off, a browser working from a copy goes back to the pod.
+  return j(200, { ok: true, handle, flushed, hold: holdOn(ctx) });
 }

package/lib/gateway/keeper.mjs

@@ -18,7 +18,7 @@ import { keeperSession } from './keeper-session.mjs';
 import { flushHeld, heldEntries } from './held-mail.mjs';
 import { publishDue, nextDue } from '../core/scheduled.mjs';
 import { lockCopy } from './copy.mjs';
-import { reachAccount, stateAndLease, actingAgent } from './account-agent.mjs';
+import { reachAccount, settle, stateAndLease, actingAgent } from './account-agent.mjs';
 
 export { keeperSession };
 
@@ -27,10 +27,11 @@ export { keeperSession };
  * keeperCredential and the held-mail stores. `session` may be handed in (a
  * test does); otherwise the keeper's grant is used. Returns what happened.
  *
- * An account whose copy the gateway keeps (copy.mjs) is worked on there: its
- * held mail is read straight into the copy, and the pod is written by the
- * round, not here. Otherwise the held mail goes to the pod inbox first, so the
- * drain reads it, as before.
+ * While the hold is on, the account is worked on in its copy (copy.mjs), made
+ * from the pod if there is none: its held mail is read straight into the
+ * copy, and the pod is written by the round, not here. With the hold off, it
+ * is worked on in the pod, and any held mail goes to the pod inbox first, so
+ * the drain reads it.
  */
 export async function keepOnce(ctx, handle, rec, { log = console.log, session = null } = {}) {
   // A forum, or one of its categories, is run whole by the forum's own code
@@ -40,9 +41,11 @@ export async function keepOnce(ctx, handle, rec, { log = console.log, session =
   const at = await reachAccount(ctx, handle, rec, { log, session });
   if (at.skipped) return { skipped: at.skipped };
 
-  const unlock = at.inCopy ? await lockCopy(ctx.copyKv, handle) : async () => {};
+  const unlock = ctx.copyKv ? await lockCopy(ctx.copyKv, handle) : async () => {};
   if (!unlock) return { skipped: 'the gateway is busy with this account', retry: 'soon' };
   try {
+    const settled = await settle(ctx, handle, rec, at, { log });
+    if (!settled.ok) return { skipped: `the account's copy could not be made: ${settled.why}`, retry: 'soon' };
     // Held mail goes in first, so this drain reads it.
     const flushed = !at.inCopy && ctx.listHeld
       ? await flushHeld(ctx, handle, rec).catch((e) => { say(`held mail not delivered: ${e.message}`); return 0; }) : 0;