fedipod-bb 0.4.0 → 0.4.2

package/README.md

@@ -12,11 +12,10 @@ the forum from a moderator's machine, and the page that shows the forum.
 A moderator's console is part of that page: the queue of reports, held posts
 and asks, and the settings.
 The queue shows what needs a moderator: a report, a held post, a join
-request, and a request of a moderator's that did not take. The design and the phases are in
-`claude/plans/fedipod-bb.md` at the repository root.
+request, and a request of a moderator's that did not take.
 
 ```
-node --test packages/fedipod-bb/test/*.test.mjs
+npm test
 ```
 
 ## Hosting a forum
@@ -181,7 +180,7 @@ category's address; a reader with nothing is pointed at sign-up.
 ## Attaching to a Gateway
 
 ```
-node packages/fedipod-bb/bin/fedipod-bb.mjs attach --home DIR --front https://fedipod.net
+fedipod-bb attach --home DIR --front https://fedipod.net
 ```
 
 Takes `@<handle>@fedipod.net` for the forum and one address per category,

package/package.json

@@ -1,8 +1,15 @@
 {
   "name": "fedipod-bb",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "FediPod-BB: a forum website whose record lives on a Solid pod.",
   "license": "MIT",
+  "author": "Jeff Zucker",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jeff-zucker/fedipod-bb.git"
+  },
+  "homepage": "https://github.com/jeff-zucker/fedipod-bb#readme",
+  "bugs": "https://github.com/jeff-zucker/fedipod-bb/issues",
   "type": "module",
   "main": "src/index.mjs",
   "exports": {
@@ -23,11 +30,10 @@
     "fep-draft.md"
   ],
   "scripts": {
-    "pretest": "node scripts/link-fedipod.mjs",
     "test": "node --test test/*.test.mjs"
   },
   "dependencies": {
-    "fedipod": "^1.44.0",
+    "fedipod": "^1.50.2",
     "fediverse-session": "^0.2.1"
   }
 }

package/site/bb.js

@@ -10,7 +10,7 @@
 //   https://bb.<site>/<handle>/     the same forum at its own address
 //   #/  #/c/<slug>  #/t/<slug>/<tid>
 
-import { reader, placeOf, authorLabel, cacheKey, categoryBase } from './read.mjs';
+import { reader, placeOf, authorLabel, cacheKey, categoryBase, cleanHtml } from './read.mjs';
 import { MastoLogin, hostOfHandle, serverKind, actorOfHandle } from './masto.mjs';
 import * as pod from './pod.mjs';
 import { mine as keptByReader } from './mine.mjs';
@@ -70,7 +70,7 @@ const handleOf = (cat) => {
 // A post names the category it is for, and every reader of this page is
 // already in it: the address is how it got here, not something to read.
 function body(p, cat) {
-  const html = p?.content || '';
+  const html = cleanHtml(p?.content);
   if (!cat) return html;
   const d = document.createElement('div');
   d.innerHTML = html;
@@ -297,7 +297,7 @@ async function showReplies() {
     const href = cat && tid ? `#/t/${esc(cat.slug)}/${esc(tid)}/${esc(await cacheKey(p.id))}` : '#/';
     items.push(`<li><a class="title" href="${href}">${esc(p.topicName || 'Topic')}</a>
       <div class="meta">${who ? esc(who.handle) : esc(authorLabel(p.author || ''))} · ${esc(when(p.published))}</div>
-      <article class="post"><div class="body">${p.content || ''}</div></article></li>`);
+      <article class="post"><div class="body">${cleanHtml(p.content)}</div></article></li>`);
   }
   $('main').innerHTML = `<ul class="list">${items.join('')}</ul>`;
 }
@@ -391,7 +391,7 @@ async function showWho(handleOrId) {
     const href = c && tid ? `#/t/${esc(c.slug)}/${esc(tid)}/${esc(await cacheKey(p.id))}` : '#/';
     items.push(`<li><a class="title" href="${href}">${esc(p.topicName || 'Topic')}</a>
       <div class="meta">${c ? esc(c.name) + ' · ' : ''}${esc(when(p.published))} · ▲ ${p.likes || 0} ▼ ${p.dislikes || 0}</div>
-      <article class="post"><div class="body">${p.content || ''}</div></article></li>`);
+      <article class="post"><div class="body">${cleanHtml(p.content)}</div></article></li>`);
   }
   $('main').innerHTML = head + `<ul class="list">${items.join('')}</ul>`;
 }

package/site/read.mjs

@@ -10,6 +10,51 @@ const MAX_PAGES = 50;
 
 export const categoryBase = (actorId) => String(actorId).replace(/ap\/actor$/u, '');
 
+// A post's text, one copy: a server that sends it once per language hands
+// over a list, read as its first.
+export function textOf(v) {
+  if (typeof v === 'string') return v;
+  if (Array.isArray(v)) return v.find(s => typeof s === 'string') || '';
+  return '';
+}
+
+// A post's text as the page shows it: only the tags and attributes a post may
+// carry (the list the forum's host keeps when it stores one), built back up
+// from a parse so nothing else reaches the page. DOMParser's parse is inert:
+// it runs no script and loads no image. `parse` is for tests.
+const KEEP = new Set(['p', 'br', 'a', 'span', 'em', 'strong', 'b', 'i', 'u', 'del',
+  'code', 'pre', 'blockquote', 'ul', 'ol', 'li', 'h1', 'h2', 'h3', 'h4']);
+const ATTRS = new Set(['href', 'class', 'lang', 'title']);
+const DROP = new Set(['script', 'style', 'textarea', 'noscript', 'template', 'title', 'head']);
+const escText = (s) => String(s).replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
+const linkable = (href) => { try { return ['http:', 'https:', 'mailto:'].includes(new URL(href).protocol); } catch { return false; } };
+const inert = (html) => new DOMParser().parseFromString(html, 'text/html').body;
+
+function rebuild(node) {
+  let out = '';
+  for (const n of node.childNodes) {
+    if (n.nodeType === 3) { out += escText(n.nodeValue); continue; }
+    if (n.nodeType !== 1) continue;
+    const tag = n.nodeName.toLowerCase();
+    if (DROP.has(tag)) continue;
+    if (!KEEP.has(tag)) { out += rebuild(n); continue; }
+    let attrs = '';
+    for (const a of n.attributes) {
+      const name = a.name.toLowerCase();
+      if (!ATTRS.has(name) || (name === 'href' && (tag !== 'a' || !linkable(a.value)))) continue;
+      attrs += ` ${name}="${escText(a.value)}"`;
+    }
+    if (tag === 'a') attrs += ' rel="nofollow noopener noreferrer"';
+    out += tag === 'br' ? '<br>' : `<${tag}${attrs}>${rebuild(n)}</${tag}>`;
+  }
+  return out;
+}
+
+export function cleanHtml(html, parse = inert) {
+  const text = textOf(html);
+  return text ? rebuild(parse(text)) : '';
+}
+
 // Where a forum is read from: through a Gateway at `<origin>/u/<handle>/`, or
 // a pod home given outright.
 export const forumBase = ({ origin = null, handle = null, pod = null } = {}) => {
@@ -213,8 +258,8 @@ export function reader({ fetch: f = globalThis.fetch.bind(globalThis), session =
           id: copy.id || url,
 
           author: idOf([].concat(copy.attributedTo || [])[0]) || null,
-          name: copy.name || null,
-          content: typeof copy.content === 'string' ? copy.content : '',
+          name: textOf(copy.name) || null,
+          content: textOf(copy.content),
           published: copy.published || null,
           topic: idOf(copy.context) || null,
           inReplyTo: idOf(copy.inReplyTo) || null,
@@ -278,7 +323,7 @@ export function reader({ fetch: f = globalThis.fetch.bind(globalThis), session =
       return {
         id: copy.id || postId, type: copy.type || 'Note', gone: copy.type === 'Tombstone', page,
         author: idOf([].concat(copy.attributedTo || [])[0]) || null,
-        name: copy.name || null, content: typeof copy.content === 'string' ? copy.content : '',
+        name: textOf(copy.name) || null, content: textOf(copy.content),
         // What its author typed, when they said so: an edit reopens this
         // rather than guessing it back out of the HTML.
         source: copy.source?.mediaType === 'text/markdown' && typeof copy.source?.content === 'string' ? copy.source.content : null,

package/src/gateway.mjs

@@ -27,7 +27,9 @@ export const isForumRow = (rec) => !!rec?.inboxUrl && (rec.kind === 'group' || r
 export async function forumFor(ctx, handle, rec) {
   const forumHandle = rec.kind === 'application' ? handle : rec.forum;
   if (!forumHandle) return { skipped: 'the category names no forum; run `fedipod-bb keep` again' };
-  const forumRec = forumHandle === handle ? rec : await ctx.lookup(forumHandle);
+  // Read fresh where the front offers it: the forum's row is switched on
+  // last, and its first run starts in the same breath.
+  const forumRec = forumHandle === handle ? rec : await (ctx.lookupFresh || ctx.lookup)(forumHandle);
   if (!forumRec || !isForumRow(forumRec) || !forumRec.keeper) return { skipped: `@${forumHandle} is not a forum this gateway keeps` };
   if (!forumRec.webId || !forumRec.podHome) return { skipped: 'the forum row names no owner or pod' };
   const home = forumRec.podHome.replace(/\/?$/u, '/');
@@ -89,7 +91,7 @@ async function readConfig(ctx, f, podFetch) {
 export async function ensureForumCopies(ctx, f, config, log) {
   const rows = [[f.forumHandle, f.forumRec]];
   for (const c of config?.categories || []) {
-    const r = await ctx.lookup(c.slug);
+    const r = await (ctx.lookupFresh || ctx.lookup)(c.slug);
     if (r && isForumRow(r)) rows.push([c.slug, r]);
   }
   const have = new Set();
@@ -139,6 +141,33 @@ function nextRun(forum, asksPending) {
  * Returns what the keeper's bookkeeping expects: { drained, waiting, nextAt }
  * or { skipped, retry }.
  */
+// A queued delivery's identity: where it goes and what it carries.
+const queueKey = (item) => `${item.inbox}\u0000${typeof item.activity === 'string' ? item.activity : JSON.stringify(item.activity)}`;
+
+/**
+ * The queue after a carry that ran with nobody holding the forum: `before`
+ * is what the run took out to carry, `after` what the carry left of it
+ * (delivered items gone, failed ones with their next time), `fresh` the
+ * queue as the copy holds it now, which the door may have added to in the
+ * meantime. A door only ever appends, so: an item the run took out is kept
+ * as the carry left it or dropped if delivered; anything else is the door's
+ * and stays.
+ */
+export function mergeQueues(before, after, fresh) {
+  const taken = new Set(before.map(queueKey));
+  const left = new Map(after.map((i) => [queueKey(i), i]));
+  const out = [];
+  for (const item of fresh) {
+    const k = queueKey(item);
+    if (!taken.has(k)) { out.push(item); continue; }
+    if (left.has(k)) out.push(left.get(k));
+  }
+  return out;
+}
+
+const storesOf = (forum) => [forum.store, ...forum.categories.map((c) => c.store)];
+const deliverersOf = (forum) => [...forum.categories.map((c) => c.deliverer), forum.siteAgent.deliverer];
+
 export async function keepOnce(ctx, handle, rec, { log = console.log, session = null, remote: given = null, prepare = null } = {}) {
   const f = await forumFor(ctx, handle, rec);
   if (f.skipped) return { skipped: f.skipped };
@@ -149,10 +178,19 @@ export async function keepOnce(ctx, handle, rec, { log = console.log, session =
   const config = await readConfig(ctx, f, podFetch);
   if (!config) return { skipped: 'no forum on the pod' };
   const copies = await ensureForumCopies(ctx, f, config, say);
+  // The forum is held only while its state changes: the drain, the asks, the
+  // queue, the heartbeat, the write to the copies. The carry — one signed
+  // POST per follower server, each with its own timeout — runs with the forum
+  // free, so a post arriving at the door meanwhile is placed at once instead
+  // of waiting for the run to end. The queue the carry changes is written
+  // back under the lock, merged with whatever the door queued in between.
   const unlock = await lockCopy(ctx.copyKv, f.forumHandle);
   if (!unlock) return { skipped: 'the gateway is busy with this forum', retry: 'soon' };
+  let forum;
+  let before;
+  let asks = 0;
   try {
-    const forum = agentFor(ctx, f, { remote, podFetch, copies, log: say, mode: 'run' });
+    forum = agentFor(ctx, f, { remote, podFetch, copies, log: say, mode: 'run' });
     if (!await forum.connect({ act: false })) return { skipped: 'no forum on the pod' };
     prepare?.(forum);   // a test's origins, in place of the network
     if (!await forum.lease.acquire()) return { skipped: 'another host has the forum', retry: 'soon' };
@@ -161,22 +199,42 @@ export async function keepOnce(ctx, handle, rec, { log = console.log, session =
       await forum.intake.drain();
       await forum.applyVerifiedAsks().catch((e) => say(`asks: ${e.message}`));
       await forum.publishModQueue().catch((e) => say(`queue: ${e.message}`));
-      for (const cat of forum.categories) await cat.deliverer.drainQueue();
-      await forum.siteAgent.deliverer.drainQueue();
       await publish.publishHeartbeat(forum.siteAgent).catch((e) => say(`heartbeat: ${e.message}`));
-      await Promise.allSettled([forum.store.flush(), ...forum.categories.map((c) => c.store.flush())]);
-      const waiting = [forum.store, ...forum.categories.map((c) => c.store)].reduce((n, s) => n + (s.getQueue?.() || []).length, 0);
-      const asks = [forum.store, ...forum.categories.map((c) => c.store)].reduce((n, s) => n + s.read('modqueue.json', []).length, 0);
-      // The run wrote on the pod (what the door could not place, the queue,
-      // the heartbeat): the edge's copies of the forum's documents go.
-      if (ctx.purge) {
-        const tags = [`u-${f.forumHandle}`, ...(config.categories || []).map((c) => `u-${c.slug}`)];
-        await ctx.purge(tags).catch((e) => say(`purge: ${e.message}`));
-      }
-      say(`done: inbox drained, ${waiting} delivery(ies) waiting, ${asks} ask(s) pending`);
-      return { drained: true, waiting, nextAt: nextRun(forum, asks) };
+      await Promise.allSettled(storesOf(forum).map((s) => s.flush()));
+      asks = storesOf(forum).reduce((n, s) => n + s.read('modqueue.json', []).length, 0);
+      before = new Map(storesOf(forum).map((s) => [s, s.getQueue?.() || []]));
     } finally { await forum.lease.release().catch(() => {}); }
   } finally { await unlock(); }
+
+  // The carry, with the forum free.
+  for (const d of deliverersOf(forum)) await d.drainQueue().catch((e) => say(`carry: ${e.message}`));
+  const after = new Map(storesOf(forum).map((s) => [s, s.getQueue?.() || []]));
+
+  // What the carry left, written back under the lock beside what the door
+  // queued meanwhile. The door holds the forum for seconds, so the wait is short.
+  const relock = await lockCopy(ctx.copyKv, f.forumHandle, { waitMs: 20_000 });
+  if (!relock) { say('the queue could not be written back: the forum stayed busy'); return { drained: true, waiting: null, retry: 'soon' }; }
+  let waiting = 0;
+  try {
+    // The copy takes a write only from the lease's holder; the lease again, briefly.
+    if (!await forum.lease.acquire()) { say('the queue could not be written back: another host has the forum'); return { drained: true, waiting: null, retry: 'soon' }; }
+    for (const s of storesOf(forum)) {
+      let fresh = null;
+      try { await s.load(); fresh = s.getQueue?.() || []; } catch { fresh = null; }
+      const merged = fresh ? mergeQueues(before.get(s) || [], after.get(s) || [], fresh) : (after.get(s) || []);
+      s.setQueue?.(merged);
+      waiting += merged.length;
+    }
+    await Promise.allSettled(storesOf(forum).map((s) => s.flush()));
+  } finally { await forum.lease.release().catch(() => {}); await relock(); }
+  // The run wrote on the pod (what the door could not place, the queue, the
+  // heartbeat): the edge's copies of the forum's documents go.
+  if (ctx.purge) {
+    const tags = [`u-${f.forumHandle}`, ...(config.categories || []).map((c) => `u-${c.slug}`)];
+    await ctx.purge(tags).catch((e) => say(`purge: ${e.message}`));
+  }
+  say(`done: inbox drained, ${waiting} delivery(ies) waiting, ${asks} ask(s) pending`);
+  return { drained: true, waiting, nextAt: nextRun(forum, asks) };
 }
 
 /**

package/src/keep.mjs

@@ -12,11 +12,19 @@ import { keepRows } from './setup.mjs';
 // does, from the pod's own rules, which name it beside the owner from here
 // on. `on: false` takes the name out again.
 export async function keepForum(forum, { front, on = true }) {
-  const { keeper, handles } = await keepRows({ fetch: (u, i) => forum.remote.session.fetch(u, i), front, config: forum.config, on });
-  forum.keepers = on && keeper ? [keeper] : [];
-  forum.remote.keepers = [...forum.keepers];
-  forum.remote.aclOwner = forum.readCredential()?.webId || forum.remote.aclOwner || null;
-  await forum.restateRules();
+  // Handing over: the categories' rows, then every rule on the tree with the
+  // keeper's name, then the forum's own row — whose switch starts the
+  // Gateway's first run, which reads the pod under those rules. Taking it
+  // back: the rows, then the rules without the name.
+  const restate = async (keeper) => {
+    forum.keepers = on && keeper ? [keeper] : [];
+    forum.remote.keepers = [...forum.keepers];
+    forum.remote.aclOwner = forum.readCredential()?.webId || forum.remote.aclOwner || null;
+    await forum.restateRules();
+  };
+  const { keeper, handles } = await keepRows({ fetch: (u, i) => forum.remote.session.fetch(u, i), front, config: forum.config, on,
+    between: on ? restate : null });
+  if (!on) await restate(null);
   return { keeper: on ? keeper : null, handles, on };
 }
 

package/src/publish.mjs

@@ -175,6 +175,8 @@ export async function cachePost({ remote, urls }, note, { topic = null, replies
   // count beside every post it shows.
   if (Number.isFinite(likes)) copy.likes = { type: 'Collection', totalItems: likes };
   if (Number.isFinite(dislikes)) copy.dislikes = { type: 'Collection', totalItems: dislikes };
+  // One copy of the text: a server that sends it once per language hands over a list.
+  if (Array.isArray(copy.content)) copy.content = copy.content.find(s => typeof s === 'string') || '';
   if (typeof copy.content === 'string') copy.content = sanitizeHtml(copy.content);
   if (!copy['@context']) copy['@context'] = 'https://www.w3.org/ns/activitystreams';
   const url = urls.cached(note.id);

package/src/setup.mjs

@@ -99,15 +99,18 @@ export async function attachRows({ fetch: f, front, config, plain }, { log = ()
 /**
  * Let a Gateway keep the forum (or stop): every row, the categories first and
  * the forum's own row last, because turning the forum's row on is what starts
- * its first run, and that run wants every category kept already. Returns the
- * keeper's WebID.
+ * its first run, and that run wants every category kept already. `between`,
+ * given the keeper's WebID, runs after the categories and before that last
+ * switch: the caller names the keeper in the pod's rules there, so the first
+ * run can read the pod the moment it starts. Returns the keeper's WebID.
  */
-export async function keepRows({ fetch: f, front, config, on = true }) {
+export async function keepRows({ fetch: f, front, config, on = true, between = null }) {
   const origin = String(front).replace(/\/+$/u, '');
   const forumHandle = config.handle;
   const handles = [...(config.categories || []).map(c => c.slug), forumHandle];
   let keeper = null;
   for (const handle of handles) {
+    if (handle === forumHandle && between) await between(keeper);
     const res = await f(`${origin}/api/keeper`, {
       method: 'POST', headers: { 'content-type': 'application/json' },
       body: JSON.stringify({ handle, on, ...(handle === forumHandle ? {} : { forum: forumHandle }) }),
@@ -156,8 +159,8 @@ export async function setUpForumAtGateway({ remote, storageFor, fetch: f, front,
   await writeConfig(storage, { ...config, gateway, republish: true });
   onStep('attach', 'ok');
   onStep('keep', 'running');
-  const { keeper } = await keepRows({ fetch: f, front, config, on: true });
-  await nameKeeperInRules(remote, { config, plain, keeper, ownerWebId });
+  const { keeper } = await keepRows({ fetch: f, front, config, on: true,
+    between: (k) => nameKeeperInRules(remote, { config, plain, keeper: k, ownerWebId }) });
   onStep('keep', 'ok');
   return { handle: config.handle, front: gateway.front, handles, keeper };
 }