@popcorn245/cornfield 0.11.0 → 0.12.0

package/server/agent-runs.js

@@ -28,6 +28,19 @@
  * Every attempt past step 1 is a row in `actions` (`agent-request`), refusals included. The audit never holds a brief, a token's secret or a
  * credential: the request's shape, the brief's length, and the token's id and label.
  *
+ * ## On the map (issue #82)
+ *
+ * Each run in the API carries `nodes` (the ids of the map's nodes it is changing), `nodesFrom`, `collidesWith` and a `timeline`:
+ *
+ *   - **nodes** come from the files the run has changed once it has a branch or a trusted pull request: the pull request's files, or a
+ *     comparison of the branch with the base (`forge.compare`), read once for each head sha (`run_reads`) and mapped with `nodesForFile` against the checkout's
+ *     map each time they are asked for. Before the first push: the request's own `nodes` and the `[[mentions]]` in the opening post of what the run is about
+ *   - **collidesWith** lists the other OPEN runs that share a node with this one (an open run only: a finished or quiet run marks nothing)
+ *   - `agents.refuseCollisions` (off unless set) refuses a new request that names a node an open run is already changing (409 `collision`), in the
+ *     transaction that takes the place for it, right after the cap check. Only the nodes a request names are checked: its `nodes`
+ *   - the **timeline** is the request, the runner's reports, pushes, the pull request, its checks, how it ended, a stop request, and quiet: the first
+ *     and last are worked out from the row, the rest are kept as they are seen (`run_events`, one row for each, so a second look adds none)
+ *
  * ## What counts as open
  *
  * A run is **open** unless it is finished or quiet (`statusOf`): finished is the runner's report `done`, `failed` or `stopped`, or its pull
@@ -62,10 +75,14 @@
  *   checks: string | null, last_activity: string }} RunRow
  */
 
+import fs from 'node:fs';
+import path from 'node:path';
 import { ForgeError, isMaintainer } from './forge.js';
 import { createRequest, findRunId, formatRunComment, runAuthors, runBlocksOf, runIdFromBranch, validateReport, validateRequest, MAX_NOTE_CHARS } from '../core/agent-contract.js';
 import { quietAfterMs } from '../core/config.js';
-import { listPullProposals } from './proposal-pulls.js';
+import { nodesForFile, parseMentions, resolveMentions } from '../core/mentions.js';
+import { readTopology } from '../core/validate.js';
+import { findPullProposal, listPullProposals, readPullProposal } from './proposal-pulls.js';
 import { Refused, countAttempt, throttle } from './throttle.js';
 
 /** Requests one person may make in a minute (a ceiling of its own; every other action is counted by `ATTEMPTS_PER_MINUTE` and `PER_MINUTE`) */
@@ -81,6 +98,15 @@ export const FOLLOW_FLOOR_MS = 48 * 3_600_000;
 export const RESERVATION_MS = 10 * 60_000;
 /** The runs a listing answers */
 export const LIST_LIMIT = 200;
+/** The most events a run's timeline holds */
+export const TIMELINE_LIMIT = 100;
+/** How much of a request's brief the API shows (the Runs view groups runs by it) */
+export const BRIEF_SHOWN = 2000;
+/** The most runs whose subject's opening post is read for mentions in one look, and how long a reading is kept */
+export const MAX_MENTION_READS = 10;
+/** The most runs whose changed files are read in one look */
+export const MAX_FILE_READS = 10;
+export const MENTIONS_KEEP_MS = 3_600_000;
 /** The queued requests a runner's listing answers at most, oldest first */
 export const QUEUE_LIMIT = 50;
 /** The runs that are over once the runner has said so */
@@ -147,7 +173,7 @@ const subjectWords = (refs) => [refs.issue != null && `issue #${refs.issue}`, re
  * @param {number} nowMs
  * @param {number} quietMs
  */
-export function publicRun(row, nowMs, quietMs) {
+export function publicRun(row, nowMs, quietMs, extra = {}) {
   const { status, open } = statusOf(row, nowMs, quietMs);
   let limits = null;
   try { limits = JSON.parse(row.request).limits ?? null; } catch { limits = null; }
@@ -162,9 +188,48 @@ export function publicRun(row, nowMs, quietMs) {
     stopRequested: row.stop_requested_at ? { at: row.stop_requested_at, by: row.stop_requested_by } : null,
     pull: row.pull != null ? { number: row.pull, state: row.pull_state, url: row.pull_url, by: row.pull_by, trusted: row.pull_trusted === 1, via: row.linked_via } : null,
     branch: row.branch, headSha: row.head_sha, checks: row.checks, lastActivity: row.last_activity,
+    ...extra,
   };
 }
 
+/**
+ * The run's story in order: the request first, what was seen (`events`, oldest first), and "quiet" last when the run is. Pure.
+ * @param {RunRow} row
+ * @param {{ at: string, kind: string, actor: string | null, ref: string | null, note: string | null, url: string | null }[]} events
+ * @param {{ status: string }} now `statusOf`'s answer
+ * @param {number} quietMs
+ */
+export function timelineOf(row, events, now, quietMs) {
+  const out = [{ at: row.requested_at, kind: 'requested', actor: row.requested_by, ref: row.profile, note: null, url: row.comment_url }];
+  out.push(...events);
+  if (now.status === 'quiet') out.push({ at: iso(Date.parse(row.last_activity) + quietMs), kind: 'quiet', actor: null, ref: null, note: null, url: null });
+  return out;
+}
+
+/**
+ * The ids of the nodes a proposal touches: what it adds (and their parents), changes and removes, and both ends of the edges it adds,
+ * changes and removes. Anything that is not a proposal gives none. Pure.
+ * @param {any} proposal
+ * @returns {string[]}
+ */
+export function proposalNodeIds(proposal) {
+  const out = new Set();
+  const list = (v) => (Array.isArray(v) ? v : []);
+  const id = (v) => { if (typeof v === 'string' && v !== '') out.add(v); };
+  const blocks = proposal && typeof proposal === 'object' ? proposal : {};
+  for (const op of ['add', 'change', 'remove']) {
+    for (const n of list(blocks.nodes?.[op])) { id(n?.id); if (op === 'add') id(n?.parent); }
+    for (const e of list(blocks.edges?.[op])) { id(e?.from); id(e?.to); }
+  }
+  return [...out];
+}
+
+/** The ids a request names, the ones the map knows when it is known */
+function requestNodes(request, known) {
+  const ids = Array.isArray(request?.nodes) ? request.nodes.filter((n) => typeof n === 'string') : [];
+  return known ? ids.filter((id) => known.has(id)) : ids;
+}
+
 /**
  * @typedef {object} RequestContext
  * @property {() => import('./forge.js').Forge | Promise<import('./forge.js').Forge>} forge the forge as the person (may throw a ForgeError `unauthorized`)
@@ -195,6 +260,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
   const runnerSet = new Set((Array.isArray(agents.runners) ? agents.runners : []).map(lower));
   const authorSet = new Set((Array.isArray(agents.authors) ? agents.authors : []).map(lower));
   const quietMs = quietAfterMs(config);
+  const refuseCollisions = agents.refuseCollisions === true;
   /** How requests travel from now on (`agents.transport`); a run keeps the transport it was asked for with */
   const transport = /** @type {'forge' | 'push' | 'pull'} */ (agents.transport ?? 'forge');
   const log = (what) => console.error(`[cornfield] agent runs: ${what}`);
@@ -208,6 +274,87 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
     }
   };
 
+  // -------------------------------------------------------------------------
+  //  Nodes and collisions
+  // -------------------------------------------------------------------------
+
+  /** The checkout's map, read again when the file changes (a sync): `{ nodes, ids }`, empty when there is none */
+  let mapCache = { stamp: '', nodes: /** @type {{ id: string, file?: string }[]} */ ([]), ids: /** @type {Set<string>} */ (new Set()) };
+  function theMap() {
+    let stamp = '';
+    try { stamp = String(fs.statSync(path.join(config.root, config.topology)).mtimeMs); } catch { stamp = ''; }
+    if (stamp === mapCache.stamp) return mapCache;
+    let nodes = [];
+    try { nodes = readTopology(config)?.nodes ?? []; } catch { nodes = []; }
+    nodes = Array.isArray(nodes) ? nodes.filter((n) => n && typeof n.id === 'string') : [];
+    mapCache = { stamp, nodes, ids: new Set(nodes.map((n) => n.id)) };
+    return mapCache;
+  }
+
+  const readOf = (db, runId, source) => {
+    const r = /** @type {{ sha: string, data: string } | undefined} */ (db.prepare('SELECT sha, data FROM run_reads WHERE run_id = ? AND source = ?').get(runId, source));
+    if (!r) return null;
+    try { return { sha: r.sha, data: JSON.parse(r.data) }; } catch { return null; }
+  };
+
+  /**
+   * The nodes a run is changing: from the files at its head when they were read, else (before the first push, or before the files
+   * were read) what the request names and what the opening post of its subject mentions.
+   * @returns {{ nodes: string[], from: 'files' | 'request' }}
+   */
+  function nodesOfRow(db, row) {
+    const map = theMap();
+    if (row.head_sha) {
+      const files = readOf(db, row.run_id, 'files');
+      // `null` at this sha: the forge could not list the files (see readNodes), so what the run is about stands in, below
+      if (files && files.sha === row.head_sha && Array.isArray(files.data)) {
+        const ids = new Set();
+        for (const f of files.data) {
+          for (const p of [f.path, f.previousPath]) if (typeof p === 'string') for (const n of nodesForFile(p, map.nodes)) ids.add(n.id);
+        }
+        return { nodes: [...ids], from: 'files' };
+      }
+    }
+    let request = null;
+    try { request = JSON.parse(row.request); } catch { request = null; }
+    const ids = new Set(requestNodes(request, map.ids.size ? map.ids : null));
+    const mentions = readOf(db, row.run_id, 'mentions');
+    if (mentions && Array.isArray(mentions.data)) for (const id of resolveMentions(mentions.data, map.nodes).ids) ids.add(id);
+    return { nodes: [...ids], from: 'request' };
+  }
+
+  /** What the runs add to their public form: the nodes, the open runs they share a node with, and the timeline */
+  function extrasFor(db, rows, at) {
+    const open = openRows(db, at);
+    const openIds = new Set(open.map((r) => r.run_id));
+    const nodesOf = new Map();
+    for (const r of new Map([...open, ...rows].map((x) => [x.run_id, x])).values()) nodesOf.set(r.run_id, nodesOfRow(db, r));
+    const events = new Map();
+    if (rows.length) {
+      const ask = db.prepare('SELECT at, kind, actor, ref, note, url FROM run_events WHERE run_id = ? ORDER BY at, rowid LIMIT ?');
+      for (const r of rows) events.set(r.run_id, ask.all(r.run_id, TIMELINE_LIMIT));
+    }
+    return new Map(rows.map((r) => {
+      const mine = nodesOf.get(r.run_id);
+      const collidesWith = openIds.has(r.run_id)
+        ? open.filter((o) => o.run_id !== r.run_id && nodesOf.get(o.run_id).nodes.some((id) => mine.nodes.includes(id))).map((o) => o.run_id)
+        : [];
+      let brief = '';
+      try { brief = String(JSON.parse(r.request).brief ?? '').slice(0, BRIEF_SHOWN); } catch { brief = ''; }
+      return [r.run_id, { brief, nodes: mine.nodes, nodesFrom: mine.from, collidesWith, timeline: timelineOf(r, events.get(r.run_id) ?? [], statusOf(r, at, quietMs), quietMs) }];
+    }));
+  }
+
+  /** Writes what happened to a run, once: the key says what, so a second look at the same thing adds nothing */
+  function keepEvent(db, runId, ev) {
+    // A forge's answer may lack a field (a pull request with no `createdAt`): the timeline then says when cornfield saw it, and a value
+    // that is not text is left out. One odd answer must never throw out of the look, or every run's state in it would be lost
+    const text = (v) => (typeof v === 'string' && v !== '' ? v : v == null ? null : typeof v === 'number' ? String(v) : null);
+    const at = typeof ev.at === 'string' && !Number.isNaN(Date.parse(ev.at)) ? ev.at : iso(now());
+    db.prepare('INSERT OR IGNORE INTO run_events (run_id, key, at, kind, actor, ref, note, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
+      .run(runId, String(ev.key), at, ev.kind, text(ev.actor), text(ev.ref), text(ev.note), text(ev.url));
+  }
+
   /**
    * The runs of this repo that are open as of now (the cap and the one-per-subject rule count these). A reservation whose comment never got
    * posted (a crash between the reservation and the post) holds its place for `RESERVATION_MS` only, whenever cornfield last restarted.
@@ -218,7 +365,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
   }
 
   /** Takes a place for a run: the cap and the one-per-subject rule are checked and the row inserted in one transaction. Throws a `Refused`. */
-  function reserve(db, request, { target, who, via, force, trigger = false }) {
+  function reserve(db, request, { target, who, via, force, trigger = false, derived = null }) {
     const at = now();
     const mine = subjectKeys(request.refs);
     db.exec('BEGIN IMMEDIATE');
@@ -229,6 +376,18 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
       if (open.length >= cap) {
         throw new Refused(409, `${repo} already has ${open.length} open run${open.length === 1 ? '' : 's'}, the most cornfield asks for (agents.maxOpen is ${cap}): ${open.map((r) => r.run_id).join(', ')}. Wait for one to finish or go quiet, or stop one.`, 'cap-reached');
       }
+      if (refuseCollisions) {
+        const map = theMap();
+        // What the request names; when it names nothing (the dashboard's button), what its subject is about, read before the transaction
+        const wanted = [...new Set([...requestNodes(request, map.ids.size ? map.ids : null), ...resolveMentions(derived || [], map.nodes).ids])];
+        for (const r of open) {
+          const theirs = nodesOfRow(db, r).nodes;
+          const shared = wanted.filter((id) => theirs.includes(id));
+          if (shared.length) {
+            throw new Refused(409, `${shared.length === 1 ? 'The node' : 'The nodes'} ${shared.join(', ')} ${shared.length === 1 ? 'is' : 'are'} already being changed by the open run ${r.run_id} (agents.refuseCollisions is on). Wait for it to finish or go quiet, or stop it, or ask for a request that leaves ${shared.length === 1 ? 'it' : 'them'} out.`, 'collision');
+          }
+        }
+      }
       if (!force) {
         const clash = open.find((r) => (r.profile ?? null) === (request.profile ?? null)
           && subjectKeys({ issue: r.issue ?? undefined, thread: r.thread ?? undefined, proposal: r.proposal ?? undefined }).some((k) => mine.includes(k)));
@@ -322,10 +481,11 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
 
       // Where the request goes: the thread it came from, else the open spec PR of the proposal, else the issue
       let number;
+      let entry = null;
       if (made.refs.thread != null) number = made.refs.thread;
       else if (made.refs.proposal != null) {
         const listed = await listPullProposals(forge, folder, { hooks: null });
-        const entry = listed.entries.find((e) => e.slug === made.refs.proposal);
+        entry = listed.entries.find((e) => e.slug === made.refs.proposal) ?? null;
         if (!entry) throw new Refused(404, listed.problem ? `The open spec pull requests could not be read: ${listed.problem}` : `No open spec pull request holds the proposal ${made.refs.proposal}, so there is nowhere to post the request.`, 'not-found');
         number = entry.pull.number;
         made = build(me.login, number);
@@ -359,14 +519,27 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
         const why = await ctx.stillAllowed();
         if (why) throw new Refused(403, why, 'forbidden');
       }
+      // With collisions refused, a request that names no node on the map is checked by what its subject is about (spec 0005 D9), as the
+      // map would mark it before its first push: the opening post's mentions and the proposal's nodes. Read now, outside the transaction
+      let derived = null;
+      if (refuseCollisions && !requestNodes(made, theMap().ids.size ? theMap().ids : null).length) {
+        try {
+          derived = await subjectRefs(forge, made.refs, entry);
+        } catch (err) {
+          throw new Refused(503, `What this request is about could not be read, so it cannot be checked for collisions (agents.refuseCollisions is on): ${words(err)}`, 'unavailable');
+        }
+      }
       const db = await auth.store();
-      reserve(db, made, { target: number, who, via, force: force === true, trigger: Boolean(made.trigger) });
+      reserve(db, made, { target: number, who, via, force: force === true, trigger: Boolean(made.trigger), derived });
+      // What was read is what the map marks for this run until it pushes: kept, so the overlay has it now and the poll need not read it again
+      if (derived) db.prepare('INSERT OR REPLACE INTO run_reads (run_id, source, sha, data, at) VALUES (?, ?, ?, ?, ?)').run(made.runId, 'mentions', '', JSON.stringify(derived), iso(now()));
       let posted = null;
       if (comment !== null) {
         try {
           posted = await forge.comment(number, comment);
         } catch (err) {
           db.prepare('DELETE FROM runs WHERE run_id = ?').run(made.runId);
+          db.prepare('DELETE FROM run_reads WHERE run_id = ?').run(made.runId);
           throw err;
         }
         db.prepare('UPDATE runs SET comment_id = ?, comment_url = ? WHERE run_id = ?').run(posted.id, posted.url ?? null, made.runId);
@@ -376,7 +549,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
       hooks?.emit('agent.requested', { request: made, actor: me.login });
       try { onAsk?.(); } catch { /* the poll is a convenience */ }
       const row = /** @type {RunRow} */ (db.prepare('SELECT * FROM runs WHERE run_id = ?').get(made.runId));
-      return { run: publicRun(row, now(), quietMs), request: made, comment: posted ? { id: posted.id, url: posted.url ?? null, number } : null };
+      return { run: publicRun(row, now(), quietMs, extrasFor(db, [row], now()).get(row.run_id)), request: made, comment: posted ? { id: posted.id, url: posted.url ?? null, number } : null };
     } catch (err) {
       if (status === null) {
         const out = describe(err);
@@ -447,6 +620,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
       db.exec('BEGIN IMMEDIATE');
       try {
         db.prepare('UPDATE runs SET stop_requested_at = ?, stop_requested_by = ?, last_activity = MAX(last_activity, ?) WHERE run_id = ?').run(stamp, me.login, stamp, row.run_id);
+        keepEvent(db, row.run_id, { key: posted ? `stop:${posted.id}` : `stop:cornfield:${stamp}`, at: stamp, kind: 'stop-requested', actor: me.login, note: note || null, url: posted ? posted.url ?? null : null });
         if (posted) {
           db.prepare('INSERT OR IGNORE INTO run_seen (run_id, key, at) VALUES (?, ?, ?)').run(row.run_id, `c:${posted.id}:${row.run_id}:stop-request:0`, stamp);
           db.prepare('INSERT OR IGNORE INTO run_seen (run_id, key, at) VALUES (?, ?, ?)').run(row.run_id, `ev:stop:${posted.id}:c:${posted.id}:${row.run_id}:stop-request:0`, stamp);
@@ -468,7 +642,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
         hooks?.emit('run.state', { ...runPayload(after, me.login), state: 'stopped' });
         hooks?.emit('run.finished', { ...runPayload(after, me.login), state: 'stopped' });
       }
-      return { run: publicRun(after, now(), quietMs), comment: posted ? { id: posted.id, url: posted.url ?? null } : null };
+      return { run: publicRun(after, now(), quietMs, extrasFor(db, [after], now()).get(after.run_id)), comment: posted ? { id: posted.id, url: posted.url ?? null } : null };
     } catch (err) {
       if (!done) {
         const out = describe(err);
@@ -497,6 +671,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
     const rows = /** @type {RunRow[]} */ (db.prepare(`SELECT * FROM runs WHERE repo = ? AND ${POSTED} ORDER BY requested_at DESC, run_id DESC LIMIT ?`).all(repo, limit));
     // The dashboard's notice for triggers that did nothing (server/triggers.js writes a `trigger-skipped` row each time): the last day's, by reason
     const skipped = /** @type {{ at: string, answer: string | null }[]} */ (db.prepare("SELECT at, answer FROM actions WHERE action = 'trigger-skipped' AND (target = ? OR target LIKE ?) AND at > ? ORDER BY id DESC LIMIT 200").all(repo, `${repo}#%`, iso(at - 24 * 3_600_000)));
+    const extras = extrasFor(db, rows, at);
     const reasons = {};
     for (const r of skipped) {
       let why = 'error';
@@ -504,8 +679,9 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
       reasons[why] = (reasons[why] || 0) + 1;
     }
     return {
-      runs: rows.map((r) => publicRun(r, at, quietMs)), open: openRows(db, at).length, maxOpen: cap, profiles, limits: agents.limits || {}, quietAfter: agents.quietAfter, runners: [...runnerSet].length,
+      runs: rows.map((r) => publicRun(r, at, quietMs, extras.get(r.run_id))), open: openRows(db, at).length, maxOpen: cap, profiles, limits: agents.limits || {}, quietAfter: agents.quietAfter, runners: [...runnerSet].length,
       triggerSkipped: skipped.length ? { count: skipped.length, last: skipped[0].at, reasons } : null,
+      refuseCollisions,
     };
   }
 
@@ -513,7 +689,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
   async function get(runId) {
     const db = await auth.store();
     const row = typeof runId === 'string' ? /** @type {RunRow | undefined} */ (db.prepare(`SELECT * FROM runs WHERE run_id = ? AND repo = ? AND ${POSTED}`).get(runId, repo)) : undefined;
-    return row ? publicRun(row, now(), quietMs) : null;
+    return row ? publicRun(row, now(), quietMs, extrasFor(db, [row], now()).get(row.run_id)) : null;
   }
 
   /** The runs a follow looks at: posted, not over for good (a trusted pull request merged or closed), and active within the follow window */
@@ -545,6 +721,92 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
     return following;
   }
 
+  /**
+   * What a run is about, as references the map resolves (spec 0005 D9: "before the first push, the nodes come from what the request was
+   * about: the proposal, or the thread's mentions"): the `[[mentions]]` in the opening post of its thread, issue or spec PR, and for a
+   * proposal the ids of every node it touches (what it adds, changes and removes, the parents of what it adds, and the ends of the edges it
+   * adds, changes and removes). Asks the forge; throws what it throws. `entry`: the spec PR's listing when the caller has it already.
+   * @returns {Promise<string[]>}
+   */
+  async function subjectRefs(forge, refs, entry = null) {
+    const out = [];
+    const add = (r) => { if (typeof r === 'string' && r !== '' && !out.includes(r)) out.push(r); };
+    if (refs?.proposal != null && refs.pull != null) {
+      const pull = await forge.pull(refs.pull);
+      parseMentions(pull.body ?? '').refs.forEach(add);
+      const found = entry ?? await findPullProposal(forge, folder, refs.proposal, refs.pull);
+      if (found) {
+        let proposal = null;
+        try { proposal = JSON.parse(await readPullProposal(forge, folder, found)); } catch (err) { if (!(err instanceof SyntaxError)) throw err; }
+        for (const id of proposalNodeIds(proposal)) add(id);
+      }
+    } else if (refs?.thread != null || refs?.issue != null) {
+      const n = refs.thread ?? refs.issue;
+      let item;
+      try {
+        item = await forge.issue(n);
+      } catch (err) {
+        // A thread that is a pull request is not an issue to the forge: ask for it as one
+        if (!(err instanceof ForgeError && err.code === 'not-found')) throw err;
+        item = await forge.pull(n);
+      }
+      parseMentions(item.body ?? '').refs.forEach(add);
+    }
+    return out;
+  }
+
+  /**
+   * What the map needs from the forge, kept in `run_reads` so it is asked once: for a run with a head (a trusted pull request, or a branch)
+   * the files it changes, read again only when the head moved; for a run with no head yet, the `[[mentions]]` in the opening post of what it is about.
+   * A failure is logged and tried again at the next look. At most `MAX_FILE_READS` and `MAX_MENTION_READS` a look.
+   */
+  async function readNodes(forge, db, rows, at) {
+    const save = db.prepare('INSERT OR REPLACE INTO run_reads (run_id, source, sha, data, at) VALUES (?, ?, ?, ?, ?)');
+    let files = 0;
+    let mentions = 0;
+    let base = null;
+    for (const row of rows) {
+      if (!statusOf(row, at, quietMs).open) continue;
+      if (row.head_sha) {
+        const had = readOf(db, row.run_id, 'files');
+        // Files listed at this head: they say the nodes. Listed as unknowable (`null`): what the run is about stands in, read below
+        if (had && had.sha === row.head_sha && Array.isArray(had.data)) continue;
+        if (!(had && had.sha === row.head_sha) && files < MAX_FILE_READS) {
+          files += 1;
+          try {
+            let list;
+            if (row.pull != null && row.pull_trusted === 1) list = await forge.pullFiles(row.pull);
+            else {
+              base ??= config.forge.baseBranch || await forge.defaultBranch();
+              list = await forge.compare(base, row.head_sha);
+            }
+            save.run(row.run_id, 'files', row.head_sha, JSON.stringify(list.map((f) => ({ path: f.path, previousPath: f.previousPath ?? null, status: f.status }))), iso(at));
+          } catch (err) {
+            // A forge that cannot list a comparison's files (an older Forgejo) will not at this head either: say so once, keep `null` for this
+            // sha so it is not asked again until the next push, and the run keeps the nodes of what it is about. Anything else is tried again.
+            if (err instanceof ForgeError && err.code === 'invalid') save.run(row.run_id, 'files', row.head_sha, 'null', iso(at));
+            log(`the files of ${row.run_id} could not be read: ${words(err)}`);
+          }
+        }
+        const read = readOf(db, row.run_id, 'files');
+        if (!(read && read.sha === row.head_sha && read.data === null)) continue; // only an unknowable list falls through to the subject
+      }
+      const had = readOf(db, row.run_id, 'mentions');
+      const stamp = /** @type {{ at: string } | undefined} */ (db.prepare("SELECT at FROM run_reads WHERE run_id = ? AND source = 'mentions'").get(row.run_id));
+      if (had && stamp && at - Date.parse(stamp.at) < MENTIONS_KEEP_MS) continue;
+      if (mentions >= MAX_MENTION_READS) continue;
+      let refs = null;
+      try { refs = JSON.parse(row.request).refs; } catch { refs = null; }
+      if (!(refs?.proposal != null && refs.pull != null) && refs?.thread == null && refs?.issue == null) continue;
+      mentions += 1;
+      try {
+        save.run(row.run_id, 'mentions', '', JSON.stringify(await subjectRefs(forge, refs)), iso(at));
+      } catch (err) {
+        log(`what ${row.run_id} is about could not be read for its nodes: ${words(err)}`);
+      }
+    }
+  }
+
   async function followRuns(forge) {
     const db = await auth.store();
     const at = now();
@@ -559,6 +821,9 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
     const events = [];
     /** @type {Map<string, string>} the cause of the latest change of each run, for the key of its finished event */
     const cause = new Map();
+    /** @type {{ runId: string, key: string, at: string, kind: string, actor?: string | null, ref?: string | null, note?: string | null, url?: string | null }[]} what happened, for the timelines */
+    const happened = [];
+    const happen = (row, ev) => { happened.push({ runId: row.run_id, ...ev }); };
     /** @type {{ login: string | null, request: object, answer: object, target: number }[]} audit rows to write after */
     const audits = [];
 
@@ -631,6 +896,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
         const firstLink = row.pull == null;
         const state = pr.state === 'merged' ? 'merged' : pr.state === 'closed' ? 'closed' : 'open';
         const earlier = row.pull_state;
+        const pushedBefore = row.head_sha;
         let trusted = row.pull_trusted === 1;
         if (firstLink) {
           try { trusted = await trustedAuthor(pr.author?.login ?? null); } catch (err) { log(`who opened #${pr.number} could not be asked: ${words(err)}`); continue; }
@@ -639,6 +905,8 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
         // A stranger's branch is not the run's: only a trusted pull request says where the run pushes
         if (trusted) Object.assign(row, { branch: pr.head.ref || null, head_sha: pr.head.sha || null });
         dirty.add(row.run_id);
+        if (trusted && firstLink) happen(row, { key: `pull:${pr.number}:opened`, at: pr.createdAt, kind: 'pull-opened', actor: pr.author?.login ?? null, ref: `#${pr.number}`, url: pr.url ?? null });
+        if (trusted && pr.head.sha && pr.head.sha !== pushedBefore) happen(row, { key: `push:${pr.head.sha}`, at: iso(at), kind: 'push', actor: pr.author?.login ?? null, ref: pr.head.sha, note: pr.head.ref || null });
         if (trusted) touch(row, pr.updatedAt);
         if (firstLink) {
           out.linked += 1;
@@ -647,6 +915,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
           if (trusted) queue(row, 'run.state', `ev:linked:${pr.number}`, pr.author?.login ?? null, 'working');
         }
         if (trusted && state !== 'open' && earlier !== state) {
+          happen(row, { key: `pull:${pr.number}:${state}`, at: (state === 'merged' && pr.mergedAt) || pr.updatedAt || iso(at), kind: state, actor: pr.author?.login ?? null, ref: `#${pr.number}`, url: pr.url ?? null });
           cause.set(row.run_id, `pull:${pr.number}:${state}`);
           queue(row, 'run.state', `ev:pull:${pr.number}:${state}`, pr.author?.login ?? null, state);
         }
@@ -730,11 +999,13 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
         touch(row, when);
         out.reports += 1;
         cause.set(row.run_id, key);
+        happen(row, { key: `report:${key}`, at: when, kind: r.state === 'claimed' ? 'claimed' : 'report', actor: login, ref: r.state, note: r.note ?? null, url: r.url ?? null });
         if (r.state === 'claimed') queue(row, 'run.claimed', 'ev:claimed', login, 'claimed');
         if (r.state !== previous) queue(row, 'run.state', `ev:state:${id}:${key}`, login, r.state);
       } else {
         if (!row.stop_requested_at) Object.assign(row, { stop_requested_at: when, stop_requested_by: login });
         touch(row, when);
+        happen(row, { key: `stop:${id}`, at: when, kind: 'stop-requested', actor: login, note: block.value.note ?? null });
         queue(row, 'run.stopRequested', `ev:stop:${id}:${key}`, login, 'stop-requested');
       }
     }
@@ -745,7 +1016,11 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
       try {
         const checks = await forge.checks(row.head_sha);
         const summary = !checks.length ? 'none' : checks.some((c) => c.state === 'failure') ? 'failure' : checks.some((c) => c.state === 'queued' || c.state === 'running') ? 'running' : checks.some((c) => c.state === 'success') ? 'success' : 'none';
-        if (summary !== row.checks) { row.checks = summary; dirty.add(row.run_id); }
+        if (summary !== row.checks) {
+          row.checks = summary;
+          dirty.add(row.run_id);
+          happen(row, { key: `checks:${row.head_sha}:${summary}`, at: iso(at), kind: 'checks', ref: summary });
+        }
       } catch (err) {
         log(`the checks of #${row.pull} could not be read: ${words(err)}`);
       }
@@ -760,6 +1035,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
           if (b.name !== row.branch || b.sha !== row.head_sha) {
             Object.assign(row, { branch: b.name, head_sha: b.sha });
             touch(row, iso(at)); // a push: the forge does not say when, so now
+            happen(row, { key: `push:${b.sha}`, at: iso(at), kind: 'push', ref: b.sha, note: b.name });
           }
         }
       } catch (err) {
@@ -767,6 +1043,9 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
       }
     }
 
+    // ---- the files each run has changed (once for each head sha), and what its subject mentions before its first push -----------------
+    await readNodes(forge, db, [...cur.values()], at);
+
     // ---- kept first, then announced ------------------------------------------------------------------------------------------------
     for (const row of cur.values()) {
       const now2 = statusOf(row, at, quietMs);
@@ -791,6 +1070,7 @@ export function createRuns({ config, where, auth, hooks = null, now = Date.now,
             r.pull, r.pull_state, r.pull_url, r.pull_by, r.pull_trusted, r.linked_via, r.branch, r.head_sha, r.checks, r.last_activity, id);
       }
       for (const e of events) if (mark.run(e.runId, e.key, iso(at)).changes === 1) fresh.push(e);
+      for (const ev of happened) keepEvent(db, ev.runId, ev);
       db.exec('COMMIT');
     } catch (err) {
       try { db.exec('ROLLBACK'); } catch { /* nothing was begun */ }

package/server/db.js

@@ -19,6 +19,7 @@
  *                 it and when anything last happened (server/agent-runs.js, migration 7); review_tokens also keep a scope
  *                 (review, agent, or runner: migration 8, which also gives a run its transport and the runner token that claimed it)
  *   run_seen      what has been seen of each run (a comment judged, a hook event sent), so a restart sends and judges nothing twice
+ *   run_events    what happened to each run, in order, for its timeline; run_reads what was read to work out its nodes (migration 9, issue #82)
  *                 sessions also keep a forge refresh token, the access token's expiry and when `session.expiring` was sent (migration 5)
  *
  * The schema is a list of numbered migrations (`MIGRATIONS`); the file's `PRAGMA user_version` is the number of
@@ -232,6 +233,31 @@ export const MIGRATIONS = Object.freeze([
   ALTER TABLE review_tokens_new RENAME TO review_tokens;
   CREATE INDEX review_tokens_user_id ON review_tokens (user_id);
   `,
+  // 9: runs on the map (server/agent-runs.js, issue #82): what happened to each run in order (the timeline), and what was read to
+  // work out the nodes it changes, kept by head sha so the forge is asked once per push
+  `
+  CREATE TABLE run_events (
+    run_id  TEXT NOT NULL,
+    key     TEXT NOT NULL,                        -- what happened, once: 'report:<comment>:<n>', 'push:<sha>', 'pull:<n>:opened', 'checks:<sha>:<summary>' ...
+    at      TEXT NOT NULL,                        -- when it happened (the forge's time where it says, else when cornfield saw it)
+    kind    TEXT NOT NULL CHECK (kind IN ('claimed', 'report', 'stop-requested', 'push', 'pull-opened', 'checks', 'merged', 'closed')),
+    actor   TEXT,
+    ref     TEXT,                                 -- the report's state, the sha pushed, the pull request number, the checks summary
+    note    TEXT,                                 -- the runner's words, as written: the page escapes them
+    url     TEXT,
+    PRIMARY KEY (run_id, key)
+  ) STRICT;
+  CREATE INDEX run_events_at ON run_events (run_id, at);
+
+  CREATE TABLE run_reads (
+    run_id  TEXT NOT NULL,
+    source  TEXT NOT NULL CHECK (source IN ('files', 'mentions')),   -- the files a branch changes at sha, or the [[mentions]] of what the run is about
+    sha     TEXT NOT NULL DEFAULT '',
+    data    TEXT NOT NULL CHECK (json_valid(data)),
+    at      TEXT NOT NULL,
+    PRIMARY KEY (run_id, source)
+  ) STRICT;
+  `,
 ]);
 
 /** The version a file is at: its `user_version` */

package/server/forge-forgejo.js

@@ -240,6 +240,13 @@ export function createForgejoForge(forge, { token = null, fetch, apiUrl, timeout
       return (await client.list(`repos/${repo}/pulls/${n}/files?limit=${PER_PAGE}`, { what: `the files of ${pullWhat(n)}` })).map(toFile);
     },
 
+    async compare(base, head) {
+      const { body } = await client.request('GET', `repos/${repo}/compare/${segments(base)}...${segments(head)}`, { what: `the files of ${head} against ${base} in ${forge.repo}` });
+      // Forgejo's comparison lists commits; only a version that also lists files can say which nodes a branch touches
+      if (!Array.isArray(body?.files)) throw new ForgeError('This Forgejo does not list the files of a comparison.', { code: 'invalid', forge: NAME });
+      return body.files.map(toFile);
+    },
+
     async pullCommits(n) {
       return (await client.list(`repos/${repo}/pulls/${n}/commits?limit=${PER_PAGE}`, { what: `the commits of ${pullWhat(n)}` })).map(toCommit);
     },

package/server/forge-github.js

@@ -233,6 +233,11 @@ export function createGitHubForge(forge, { token = null, fetch, apiUrl, timeoutM
       return (await client.list(`repos/${repo}/pulls/${n}/files?per_page=${PER_PAGE}`, { what: `the files of ${pullWhat(n)}` })).map(toFile);
     },
 
+    async compare(base, head) {
+      const { body } = await client.request('GET', `repos/${repo}/compare/${segments(base)}...${segments(head)}`, { what: `the files of ${head} against ${base} in ${forge.repo}` });
+      return (Array.isArray(body?.files) ? body.files : []).map(toFile);
+    },
+
     async pullCommits(n) {
       return (await client.list(`repos/${repo}/pulls/${n}/commits?per_page=${PER_PAGE}`, { what: `the commits of ${pullWhat(n)}` })).map(toCommit);
     },

package/server/forge.js

@@ -86,6 +86,7 @@
  *   createIssue(issue: { title: string, body: string, labels?: string[] }): Promise<Issue>,
  *   pullFiles(n: number): Promise<PullFile[]>,
  *   pullCommits(n: number): Promise<PullCommit[]>,
+ *   compare(base: string, head: string): Promise<PullFile[]>,   // the files a branch changes against another (optional: a forge that cannot list them throws `invalid`)
  *   comments(n: number, opts?: { since?: string }): Promise<Comment[]>,
  *   comment(n: number, body: string): Promise<Comment>,
  *   checks(ref: string): Promise<Check[]>,
@@ -280,6 +281,10 @@ export function wrap(a, forge) {
     },
     pullFiles: async (n) => a.pullFiles(number(n)),
     pullCommits: async (n) => a.pullCommits(number(n)),
+    compare: async (base, head) => {
+      if (typeof a.compare !== 'function') throw invalid('This forge does not list the files of a comparison.');
+      return a.compare(text(base, 'The base branch'), text(head, 'The branch'));
+    },
     comments: async (n, opts) => {
       if (opts !== undefined && (opts === null || typeof opts !== 'object' || Array.isArray(opts))) throw invalid('The options are an object.');
       const since = opts && opts.since;

package/server/server.js

@@ -123,6 +123,8 @@ export function clientConfig(config, signIn = false) {
     // Threads (docs/threads.md): the idea label, or false when `threads: false` turns the Threads tab off
     threads: config.threads === false ? false : { label: ideaLabel(config) },
     sources: browserSources(config),
+    // Agent runs are followed (and marked on the map) where there is a forge and a sign-in (docs/agents.md)
+    ...(config.forge && signIn ? { agents: { runs: true } } : {}),
     signIn,
   };
 }