fedipod 0.8.0

package/lib/tagfeed.mjs

@@ -0,0 +1,158 @@
+// tagfeed.mjs — topical firehose for a single-actor instance: polls public
+// no-auth hashtag timelines on a configured instance and mirrors NEW notes
+// into the statuses index as kind 'tag'. View cache only — tag content is
+// NOT written to the pod; the pod holds followed/own content.
+//
+// Config in tagfeed.json: { instance, tags: [...], intervalMin }. Every note
+// is still verified by dereference at its origin before it is mirrored.
+
+import { isContentType, authorOf } from './intake.mjs';
+
+const DEFAULTS = {
+  instance: 'https://mastodon.social',
+  tags: ['solidproject', 'linkeddata', 'rdf'],
+  intervalMin: 15,
+};
+const PER_TAG = 20;              // statuses requested per tag per sweep
+const MAX_NEW_PER_SWEEP = 12;    // dereference budget per sweep — stay light
+const MAX_TAG_ENTRIES = 200;     // oldest tag entries pruned beyond this
+// The instance here is a stranger's server we poll on our own schedule, and it
+// has no other way to tell us to stop.
+const BACKOFF_MIN_MS = 15 * 60_000;
+const BACKOFF_MAX_MS = 6 * 60 * 60_000;
+const instanceRefusal = (status) => (status ? `the instance answered ${status}` : 'the instance did not answer');
+
+export class TagFeed {
+  constructor({ store, intake, log = console.log, fetcher = globalThis.fetch }) {
+    Object.assign(this, { store, intake, log, fetcher });
+    this.lastSweep = null;
+    this.lastAdded = 0;
+  }
+
+  config() { return { ...DEFAULTS, ...this.store.read('tagfeed.json', {}) }; }
+
+  setConfig(patch) {
+    const clean = {};
+    if (patch.instance) clean.instance = String(patch.instance).replace(/\/+$/, '');
+    if (Array.isArray(patch.tags)) clean.tags = patch.tags.map(t => String(t).replace(/^#/, '').trim()).filter(Boolean);
+    if (patch.intervalMin) clean.intervalMin = Math.max(5, Number(patch.intervalMin) || DEFAULTS.intervalMin);
+    this.store.write('tagfeed.json', { ...this.config(), ...clean });
+    this.stop();
+    this.start();
+    return this.config();
+  }
+
+  start() {
+    this.stopped = false;                     // restartable, the same way Intake.start is
+    this.sweep().catch(e => this.log(`tagfeed: ${e.message}`));
+    // Jittered and self-scheduling: every agent polling the same instance on
+    // the same 15-minute boundary is a beat nobody asked for.
+    const tick = () => {
+      this.timer = setTimeout(() => {
+        this.sweep()
+          .catch(e => this.log(`tagfeed: ${e.message}`))
+          .finally(() => { if (!this.stopped) tick(); });
+      }, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
+      this.timer.unref?.();
+    };
+    tick();
+  }
+
+  // The flag is what makes this stick. Clearing the timer only cancels a sweep
+  // that has not started: one already in flight re-arms itself in `finally`,
+  // and `this.stopped` was read there but never written — so a stop landing
+  // mid-sweep leaked that chain for the life of the process. Intake has had the
+  // flag all along; this is the same shape.
+  stop() { this.stopped = true; clearTimeout(this.timer); }
+
+  // Jittered exponential, capped, and cleared by an instance that answers.
+  // The next sweep is skipped rather than the timer stretched, so the tag
+  // config keeps meaning what it says once the far end is well again.
+  _backOff(status, retryAfter) {
+    this.failures = (this.failures || 0) + 1;
+    const ladder = Math.min(BACKOFF_MIN_MS * 2 ** (this.failures - 1), BACKOFF_MAX_MS);
+    const wait = retryAfter || Math.round(ladder * (0.85 + Math.random() * 0.3));
+    this.quietUntil = Date.now() + wait;
+    this.log(`tagfeed: ${instanceRefusal(status)} — not asking again for ${Math.round(wait / 60_000)} min`);
+  }
+
+  async sweep() {
+    const { instance, tags } = this.config();
+    if (!tags.length) return;
+    if (this.quietUntil && Date.now() < this.quietUntil) {
+      this.log(`tagfeed: still backing off for ${Math.round((this.quietUntil - Date.now()) / 60_000)} min`);
+      return;
+    }
+    this.lastSweep = new Date().toISOString();
+    const known = new Set(this.store.getStatuses().map(s => s.noteId));
+    let budget = MAX_NEW_PER_SWEEP;
+    let added = 0;
+    for (const tag of tags) {
+      let list;
+      try {
+        const { safeFetch, retryAfterMs } = await import('./safefetch.mjs');
+        const url = `${instance}/api/v1/timelines/tag/${encodeURIComponent(tag)}?limit=${PER_TAG}`;
+        const res = this.fetcher === globalThis.fetch
+          ? await safeFetch(url, { headers: { accept: 'application/json' } })
+          : await this.fetcher(url, { headers: { accept: 'application/json' } });
+        if (res.status >= 400) {
+          // This is somebody else's server, polled on OUR schedule, and it had
+          // no way to ask us to stop: a 429 or a 503 logged a line and the next
+          // sweep arrived on the same cadence regardless. One refusal now ends
+          // the sweep — the remaining tags are on the same instance, so there
+          // is nothing to be gained by asking them too.
+          this._backOff(res.status, retryAfterMs(res));
+          return;
+        }
+        list = await res.json();
+      } catch (e) {
+        // A network failure is the same signal, minus the courtesy.
+        this.log(`tagfeed #${tag}: ${e.message}`);
+        this._backOff(0, null);
+        return;
+      }
+      this.failures = 0;                      // it answered; the ladder resets
+      for (const st of Array.isArray(list) ? list : []) {
+        const noteId = st?.uri;
+        if (!noteId || known.has(noteId) || this.store.isBlocked(noteId)) continue;
+        if (budget-- <= 0) break;
+        const note = await this.intake.fetchAP(noteId).catch(() => null);
+        if (!note || note.id !== noteId || !isContentType(note.type)) continue;
+        // The author is only known once the note is dereferenced, and this
+        // path never went through ingestNote, where that check lives. So a
+        // blocked account still reached the timeline by posting under a tag —
+        // the one route round a block the owner had explicitly set.
+        // The author is only known once the note is dereferenced, and this path
+        // never went through ingestNote, where that check lives. Two things
+        // followed: a blocked account reached the timeline by posting under a
+        // tag — the one route round a block the owner had explicitly set — and
+        // a note could name an author its own origin does not vouch for, which
+        // is the same forgery ingestNote refuses. No `delivered` fallback here:
+        // a tag-feed note has no envelope, so it names its author or it is not
+        // one we can attribute.
+        const author = authorOf(note);
+        if (!author || this.store.isBlocked(author)) continue;
+        if (!this.store.getActors()[author]) {
+          await this.intake.fetchAP(author).catch(() => {});   // warm name+avatar
+        }
+        const { attachmentsOf, sanitizeHtml } = await import('./wire.mjs');
+        const attachments = attachmentsOf(note);
+        this.store.addStatus({
+          noteId, actor: author, content: sanitizeHtml(note.content),
+          published: note.published, inReplyTo: note.inReplyTo, kind: 'tag', tag,
+          ...(attachments.length ? { attachments } : {}),
+        });
+        known.add(noteId);
+        added++;
+      }
+    }
+    const all = this.store.getStatuses();
+    const tagged = all.filter(s => s.kind === 'tag');
+    if (tagged.length > MAX_TAG_ENTRIES) {
+      const drop = new Set(tagged.slice(MAX_TAG_ENTRIES).map(s => s.noteId));   // arrival order: tail = oldest
+      this.store.write('statuses.json', all.filter(s => !drop.has(s.noteId)));
+    }
+    this.lastAdded = added;
+    if (added) this.log(`tagfeed: +${added} from #${tags.join(' #')}`);
+  }
+}

package/lib/ua.mjs

@@ -0,0 +1,29 @@
+// One User-Agent for everything this agent sends. An operator reading an access
+// log should be able to tell what we are and where to raise a complaint;
+// solidcommunity.net's 2026-07-29 report had to infer us from client-credential
+// names because every request identified itself only as "node".
+//
+// vendor/idp-grant.cjs builds the same string from package.json — it is
+// CommonJS and cannot import this.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// Best-effort: a bundled deploy (the Netlify functions) rewrites this file's
+// location and the relative path with it. A User-Agent must never be the
+// reason anything fails to start.
+let version = '0';
+try {
+  ({ version } = JSON.parse(fs.readFileSync(
+    path.join(path.dirname(fileURLToPath(import.meta.url)), '../package.json'), 'utf8')));
+} catch {
+  try {
+    ({ version } = JSON.parse(fs.readFileSync(
+      path.join(process.cwd(), 'package.json'), 'utf8')));
+  } catch { /* keep '0' */ }
+}
+
+// Renamed 2026-07-31, once solidcommunity.net was back up and both actors there
+// were retired: the name in their logs no longer has to match the name in here.
+export const USER_AGENT = `fedipod/${version} (+https://github.com/jeff-zucker/FediPod)`;

package/lib/update.mjs

@@ -0,0 +1,91 @@
+// update.mjs — is a newer FediPod published, and the update itself.
+//
+// "Latest" is the version field of package.json on the repo's main branch —
+// the same file a clone reads to know what it is running. The check is one
+// GET of that file; nothing else leaves the machine, and AP_UPDATE_CHECK=0
+// turns it off. The update is what re-running the installer does: fast-forward
+// the checkout, reinstall dependencies, restart the agents.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { execFileSync, spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+export const LATEST_URL = 'https://raw.githubusercontent.com/jeff-zucker/FediPod/main/package.json';
+
+export const repoRoot = () => path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+export const localVersion = (root = repoRoot()) => {
+  try { return JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8')).version || null; }
+  catch { return null; }
+};
+
+// a newer than b, for dotted numeric versions.
+export const newerThan = (a, b) => {
+  const pa = String(a).split('.').map(Number);
+  const pb = String(b).split('.').map(Number);
+  for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+    const d = (pa[i] || 0) - (pb[i] || 0);
+    if (d) return d > 0;
+  }
+  return false;
+};
+
+export async function checkLatest({ current = localVersion(), fetchImpl = fetch } = {}) {
+  if (process.env.AP_UPDATE_CHECK === '0') return null;
+  try {
+    const res = await fetchImpl(LATEST_URL, { signal: AbortSignal.timeout(10_000) });
+    if (!res.ok) return null;
+    const latest = (await res.json()).version || null;
+    if (!latest || !current) return null;
+    return { current, latest, available: newerThan(latest, current), checkedAt: new Date().toISOString() };
+  } catch { return null; }
+}
+
+// Fast-forward only: a checkout with local changes is somebody's work, and an
+// update must refuse rather than eat it.
+export function runUpdate({ root = repoRoot(), log = () => {} } = {}) {
+  // The ordinary install comes from npm, where updating is npm's job and the
+  // files under us are not ours to move. A checkout is somebody working on it,
+  // and gets the fast-forward below.
+  if (!fs.existsSync(path.join(root, '.git'))) {
+    try {
+      log('updating from npm…');
+      execFileSync('npm', ['install', '-g', 'fedipod@latest'], { stdio: 'pipe' });
+      return { ok: true, note: 'updated from npm — restart the agent to serve it' };
+    } catch (e) {
+      return { ok: false,
+        note: `npm install -g fedipod@latest failed: ${String(e.stderr || e.message).slice(0, 300).trim()}` };
+    }
+  }
+  try {
+    execFileSync('git', ['-C', root, 'pull', '--ff-only'], { stdio: 'pipe' });
+  } catch (e) {
+    return { ok: false, note: `git pull refused: ${String(e.stderr || e.message).slice(0, 300).trim()}` };
+  }
+  try {
+    execFileSync('npm', ['install', '--omit=dev', '--no-audit', '--no-fund'], { cwd: root, stdio: 'pipe' });
+  } catch (e) {
+    return { ok: false, note: `npm install failed: ${String(e.stderr || e.message).slice(0, 300).trim()}` };
+  }
+  const version = localVersion(root);
+  log(`updated the checkout to ${version}`);
+  return { ok: true, note: `updated to ${version}`, version };
+}
+
+// New code runs only after a restart. Where systemd units exist they are all
+// restarted by name; otherwise the caller re-execs its own process.
+export function restartAgents({ log = () => {} } = {}) {
+  try {
+    const list = execFileSync('systemctl',
+      ['--user', 'list-units', '--plain', '--no-legend', 'fedipod-*'], { stdio: 'pipe' })
+      .toString().trim();
+    if (list) {
+      const child = spawn('systemctl', ['--user', 'restart', 'fedipod-*'], { detached: true, stdio: 'ignore' });
+      child.unref();
+      log('restarting the fedipod units');
+      return 'units';
+    }
+  } catch { /* no systemd here */ }
+  return 'self';
+}

package/lib/webpush.mjs

@@ -0,0 +1,98 @@
+// webpush.mjs — Mastodon-style Web Push from the agent itself. The VAPID
+// keypair is minted once into state; each client login keeps one subscription;
+// payloads go straight from this process to the browser's push service, so a
+// closed client still hears about mentions. No relay, no third party account,
+// and nothing here touches the pod.
+import crypto from 'node:crypto';
+import webpush from 'web-push';
+
+export class Push {
+  constructor({ store, subject, log = () => {} }) {
+    this.store = store;
+    this.subject = subject;             // () => the actor URL, VAPID's contact
+    this.log = log;
+  }
+
+  state() { return this.store.read('webpush.json', { subs: {} }); }
+  save(s) { this.store.write('webpush.json', s); }
+
+  vapid() {
+    const st = this.state();
+    if (!st.vapid) {
+      st.vapid = webpush.generateVAPIDKeys();
+      this.save(st);
+      this.log('webpush: VAPID keypair minted');
+    }
+    return st.vapid;
+  }
+
+  publicKey() { return this.vapid().publicKey; }
+
+  // One subscription per client login, keyed by a hash of its bearer token —
+  // the token itself is never written down twice.
+  keyOf(token) { return crypto.createHash('sha256').update(String(token)).digest('hex').slice(0, 16); }
+
+  get(token) { return this.state().subs[this.keyOf(token)] || null; }
+
+  set(token, { endpoint, keys, alerts }) {
+    if (!/^https:\/\//.test(String(endpoint || ''))) return null;
+    if (!keys?.p256dh || !keys?.auth) return null;
+    const st = this.state();
+    st.subs[this.keyOf(token)] = { endpoint, keys: { p256dh: keys.p256dh, auth: keys.auth }, alerts: alerts || {} };
+    this.save(st);
+    return st.subs[this.keyOf(token)];
+  }
+
+  setAlerts(token, alerts) {
+    const st = this.state();
+    const sub = st.subs[this.keyOf(token)];
+    if (!sub) return null;
+    sub.alerts = { ...sub.alerts, ...(alerts || {}) };
+    this.save(st);
+    return sub;
+  }
+
+  drop(token) {
+    const st = this.state();
+    delete st.subs[this.keyOf(token)];
+    this.save(st);
+  }
+
+  json(token, sub) {
+    return {
+      id: this.keyOf(token), endpoint: sub.endpoint,
+      alerts: sub.alerts || {}, policy: 'all',
+      server_key: this.publicKey(),
+    };
+  }
+
+  // Push one notification to every subscription whose alerts allow the type.
+  // A push service answering 404/410 means the browser dropped the
+  // subscription — it is removed rather than retried forever.
+  async notify(n, payload) {
+    const st = this.state();
+    const entries = Object.entries(st.subs);
+    if (!entries.length) return;
+    const { publicKey, privateKey } = this.vapid();
+    const details = { subject: this.subject(), publicKey, privateKey };
+    let dropped = false;
+    for (const [key, sub] of entries) {
+      if (sub.alerts && sub.alerts[n.type] === false) continue;
+      try {
+        await webpush.sendNotification(
+          { endpoint: sub.endpoint, keys: sub.keys },
+          JSON.stringify(payload),
+          { vapidDetails: details, contentEncoding: 'aes128gcm', TTL: 3600 },
+        );
+      } catch (e) {
+        if (e?.statusCode === 404 || e?.statusCode === 410) {
+          delete st.subs[key];
+          dropped = true;
+        } else {
+          this.log(`webpush: ${e?.statusCode || ''} ${e?.message || e}`);
+        }
+      }
+    }
+    if (dropped) this.save(st);
+  }
+}