@popcorn245/cornfield 0.4.0 → 0.5.0

package/core/config.js

@@ -23,6 +23,7 @@
  *     personas: { roles: null },              // the roles a persona may have (null: any)
  *     releaseNotes: { breakingLabel: 'breaking' },   // the pull request label that lists a change under "Breaking" in the release notes
  *     hooks: [],                              // [{ url, events }]: signed JSON events posted to outside code (docs/config.md#hooks); the secret is env only
+ *     hookPoll: { specs: '5m' },              // how often cornfield looks for ready specs on its own, for the spec.ready hook; false turns it off (docs/config.md#hook-poll)
  *     links: [],                              // [{ label, href }] shown in the explorer's header
  *     proxy: {},                              // { '/api/': 'http://127.0.0.1:3245' }: same-origin paths for a live source
  *     importMap: {},                          // bare imports a repo-local browser module needs: { name: '/repo/path.js' }
@@ -68,6 +69,33 @@ export const MERGE_METHODS = ['merge', 'squash', 'rebase'];
 export const HOOK_EVENTS = Object.freeze(['spec.ready', 'spec.approved', 'spec.sentBack', 'build.opened', 'pr.landed']);
 /** The most hooks a config lists */
 export const MAX_HOOKS = 20;
+/** The bounds of `hookPoll.specs` in milliseconds: not under a minute (the forge is not hammered), not over an hour */
+export const HOOK_POLL_MIN_MS = 60_000;
+export const HOOK_POLL_MAX_MS = 3_600_000;
+/** The keys of `hookPoll` when it is an object */
+export const HOOK_POLL_KEYS = Object.freeze(['specs']);
+
+/**
+ * A duration like '90s', '5m' or '1h' in milliseconds, or null when it is not one (whole numbers only).
+ * @param {unknown} v
+ * @returns {number | null}
+ */
+export function parseDuration(v) {
+  const m = typeof v === 'string' ? /^([1-9]\d{0,5})(s|m|h)$/.exec(v) : null;
+  return m ? Number(m[1]) * { s: 1000, m: 60_000, h: 3_600_000 }[m[2]] : null;
+}
+
+/**
+ * How often cornfield looks at the open spec PRs by itself, in milliseconds, or null when it does not: when `hookPoll.specs`
+ * is false, or no hook subscribes to `spec.ready` (by name or with '*').
+ * @param {{ hooks?: { events: string[] | '*' }[], hookPoll?: { specs?: string | false } | false }} config a loaded config
+ * @returns {number | null}
+ */
+export function specPollMs(config) {
+  const wanted = Array.isArray(config.hooks) && config.hooks.some((h) => h.events === '*' || (Array.isArray(h.events) && h.events.includes('spec.ready')));
+  const every = config.hookPoll === false ? false : config.hookPoll ? config.hookPoll.specs : DEFAULTS.hookPoll.specs;
+  return wanted && every !== false ? parseDuration(every ?? DEFAULTS.hookPoll.specs) : null;
+}
 
 export const DEFAULTS = Object.freeze({
   title: null,
@@ -86,6 +114,7 @@ export const DEFAULTS = Object.freeze({
   releaseNotes: { breakingLabel: 'breaking' },
   agents: { authors: [], labels: [], branchPrefixes: [], bots: true },
   hooks: [],
+  hookPoll: { specs: '5m' },
   links: [],
   proxy: {},
   importMap: {},
@@ -199,6 +228,19 @@ export function configProblems(c) {
       });
     }
   }
+  if (c.hookPoll !== undefined && c.hookPoll !== false) {
+    if (!isObject(c.hookPoll)) out.push('"hookPoll" is { specs: "5m" }, or false for none.');
+    else {
+      for (const k of Object.keys(c.hookPoll)) if (!HOOK_POLL_KEYS.includes(k)) out.push(`hookPoll.${k}: the key is ${HOOK_POLL_KEYS.join(', ')}.`);
+      const v = c.hookPoll.specs;
+      if (v !== undefined && v !== false) {
+        const ms = parseDuration(v);
+        if (ms === null) out.push('hookPoll.specs is a duration like "90s", "5m" or "1h" (from 60s to 1h), or false.');
+        else if (ms < HOOK_POLL_MIN_MS) out.push(`hookPoll.specs ${JSON.stringify(v)} is under the floor of 60s, so the forge is not hammered; use "60s" or more.`);
+        else if (ms > HOOK_POLL_MAX_MS) out.push(`hookPoll.specs ${JSON.stringify(v)} is over the ceiling of 1h; use "1h" or less, or false to turn the poll off.`);
+      }
+    }
+  }
   if (c.releaseNotes !== undefined) {
     if (!isObject(c.releaseNotes)) out.push('"releaseNotes" is { breakingLabel }.');
     else if (c.releaseNotes.breakingLabel !== undefined && (!isText(c.releaseNotes.breakingLabel) || c.releaseNotes.breakingLabel.includes(','))) out.push('releaseNotes.breakingLabel is a label\'s name, like "breaking".');
@@ -290,6 +332,8 @@ export async function loadConfig(root, { env = process.env, file } = {}) {
     deploys: { ...DEFAULTS.deploys, ...(raw.deploys || {}) },
     agents: { ...DEFAULTS.agents, ...(raw.agents || {}) },
   };
+  // `hookPoll: false` is `{ specs: false }`; an object is read over the default
+  config.hookPoll = raw.hookPoll === false ? { specs: false } : { ...DEFAULTS.hookPoll, ...(raw.hookPoll || {}) };
   // `proposals` is the folder to everything downstream, whichever way the file wrote it; the options are read from `approveLands`
   config.approveLands = isObject(raw.proposals) && raw.proposals.approveLands === true;
   config.proposals = (isObject(raw.proposals) ? raw.proposals.dir : raw.proposals) ?? DEFAULTS.proposals;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@popcorn245/cornfield",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "A self-hostable dev dashboard with the topology map at its centre: proposals on the map, scenarios, outages and what-if edits, for any repo.",
   "license": "MIT",
   "author": "popcorn245 (https://github.com/popcorn245)",

package/server/forge-routes.js

@@ -138,7 +138,7 @@ function routeOf(reqPath, url, config, where, hooks = null) {
       key: 'work',
       load: async (forge) => {
         const [pulls, issues, branches, specs] = await Promise.all([
-          forge.pulls({ state: 'all', limit: WORK_PULLS }), forge.issues({ state: 'all', labels: [], limit: WORK_ISSUES }), forge.branches(), listPullProposals(forge, config, { hooks }),
+          forge.pulls({ state: 'all', limit: WORK_PULLS }), forge.issues({ state: 'all', labels: [], limit: WORK_ISSUES }), forge.branches(), listPullProposals(forge, where || config, { hooks }),
         ]);
         return {
           pulls: pulls.map((p) => ({ ...p, body: p.body.slice(0, WORK_BODY_MAX) })),

package/server/hooks.js

@@ -7,8 +7,8 @@
  *
  *   spec.ready      a spec PR that is not a draft was first seen, or was seen at a new head: it is waiting for review.
  *                   A draft (a "Start a spec" stub among them) is never announced; one marked ready is, at its head.
- *                   cornfield has no webhook receiver: it notices when it next lists the PRs, so this fires when
- *                   cornfield next looks, not at the push. The first look at a repo after hooks are added announces
+ *                   cornfield has no webhook receiver: it notices when it lists the PRs (a maintainer's dashboard, or its own
+ *                   timer, server/spec-poll.js), so this fires within the poll interval of the push, not at it. The first look at a repo after hooks are added announces
  *                   nothing: it records what is open, so only what changes after it is news
  *   spec.approved   a maintainer approved a proposal          spec.sentBack   a maintainer sent one back
  *   build.opened    the build issue of an approved proposal was opened

package/server/server.js

@@ -42,7 +42,7 @@ import http from 'node:http';
 import path from 'node:path';
 import { createHash } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
-import { browserSources, BUILTIN_SOURCES } from '../core/config.js';
+import { browserSources, BUILTIN_SOURCES, specPollMs } from '../core/config.js';
 import { CLOUD_FILE } from '../core/cloud.js';
 import { decodePath, inside, notFound, sendFile } from './files.js';
 import { serveProposals } from './proposals.js';
@@ -54,6 +54,7 @@ import { assertBind, hostAllowed } from './bind.js';
 import { createAuth, signInSettings } from './auth.js';
 import { createHooks, hooksSettings } from './hooks.js';
 import { createLander } from './lander.js';
+import { createSpecPoll } from './spec-poll.js';
 
 export const CORNFIELD_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const EXPLORER_DIR = path.join(CORNFIELD_DIR, 'explorer');
@@ -126,10 +127,10 @@ export const HA_CACHE_MS = 30_000;
 /**
  * The server for one repo. It does not listen: `listen(config)` does, where server/bind.js allows.
  * @param {object} config a loaded config (core/config.js `loadConfig`)
- * @param {Parameters<typeof createAuth>[1] & { homeAssistant?: object, hookFetch?: typeof fetch, hookTimer?: { setTimeout: Function, clearTimeout: Function }, landTimer?: { setTimeout: Function, clearTimeout: Function } }} [opts] what sign-in reads (`env`, and tests' fakes;
+ * @param {Parameters<typeof createAuth>[1] & { homeAssistant?: object, hookFetch?: typeof fetch, hookTimer?: { setTimeout: Function, clearTimeout: Function }, landTimer?: { setTimeout: Function, clearTimeout: Function }, specPollTimer?: { setTimeout: Function, clearTimeout: Function } }} [opts] what sign-in reads (`env`, and tests' fakes;
  *   see server/auth.js); the Home Assistant URL and token are read from the same `env`, and `homeAssistant` is what
  *   `homeAssistantHouse` is handed besides them (the tests' fake WebSocket); `hookFetch` and `hookTimer` are the
- *   tests' fake network and clock for the event hooks (server/hooks.js); `landTimer` is the clock of landing on approval (server/lander.js)
+ *   tests' fake network and clock for the event hooks (server/hooks.js); `landTimer` is the clock of landing on approval (server/lander.js); `specPollTimer` is the clock of the `spec.ready` poll (server/spec-poll.js)
  * @returns {import('node:http').Server}
  */
 export function createCornfieldServer(config, opts = {}) {
@@ -154,6 +155,9 @@ export function createCornfieldServer(config, opts = {}) {
     // What a restart left waiting: looked at again, in the background
     lander.resume().catch((err) => console.error(`[cornfield] pending landings could not be picked up: ${err instanceof Error ? err.message : err}`));
   }
+  // Looks at the open spec PRs on a timer when a hook wants spec.ready (config `hookPoll`); started when the server listens
+  const pollMs = hooks ? specPollMs(config) : null;
+  const specPoll = pollMs ? createSpecPoll({ config, where, auth, hooks, intervalMs: pollMs, now: opts.now, timer: opts.specPollTimer }) : null;
   const proxies = Object.entries(config.proxy || {});
   const houseSource = config.sources.house && (config.sources.house === 'house' || config.sources.house.use === 'house') ? config.sources.house : null;
   const houseFile = houseSource && houseSource.options && houseSource.options.file ? inside(root, houseSource.options.file) : null;
@@ -191,7 +195,8 @@ export function createCornfieldServer(config, opts = {}) {
       else res.end();
     }
   });
-  server.on('close', () => { lander?.close(); hooks?.close(); auth.close(); });
+  server.on('listening', () => specPoll?.start());
+  server.on('close', () => { specPoll?.close(); lander?.close(); hooks?.close(); auth.close(); });
   return server;
 
   async function answer(req, res) {

package/server/spec-poll.js

@@ -0,0 +1,146 @@
+/**
+ * @module server/spec-poll
+ *
+ * `spec.ready` without anyone at the dashboard (PIPE-10, docs/config.md#hook-poll). cornfield notices a spec PR when it
+ * lists the open ones, and that listing used to run only when a maintainer opened the proposals menu or the issues and PRs
+ * panel. A team that reviews from chat never does, so this lists them on a timer.
+ *
+ * **It is the same look, not a second one.** Each tick calls `listPullProposals` (server/proposal-pulls.js) with the hooks,
+ * exactly as the dashboard does, so the draft rule, the silent first look, the once-per-(PR, head, proposal) rule and the
+ * `hook_seen` table are the ones that already exist: a poll and a dashboard look never announce the same head twice, and
+ * nothing here decides what is news. The forge answers come through the same per-head cache, so a tick with nothing new
+ * costs the open-PR listing (a call a page) plus the two calls that check the token's owner; see the docs for the count.
+ *
+ * **Whose token.** Never one from the config. The newest live sign-in of a user the forge confirms is a maintainer now
+ * (`auth.forgeForUser`, then `me()` and `permission()` with that token; the same check the review API and the lander make).
+ * A user who is not, or whose token the forge refuses, is passed over for the next newest, at most `MAX_CANDIDATES`; their
+ * token is only ever asked those two questions, never used to list. With no such session the tick does nothing and says so
+ * once for the stretch (and once more when a session is back). Whose sign-in it looks with is logged each time that changes.
+ *
+ * **Never in the way.** One tick runs at a time (the next is scheduled when one ends, so a slow forge cannot pile them up).
+ * A forge error is logged and never thrown; the next tick waits twice the interval until one succeeds. The timer is
+ * unref'd and `close()` clears it.
+ */
+
+import { isMaintainer } from './forge.js';
+import { listPullProposals } from './proposal-pulls.js';
+
+/** How long after start the first look is made: soon, so the repo's silent first look is not left to the first interval */
+export const FIRST_MS = 15_000;
+/** The most signed-in users tried per tick before giving up for this tick */
+export const MAX_CANDIDATES = 5;
+
+const iso = (ms) => new Date(ms).toISOString();
+const words = (err) => (err instanceof Error ? err.message : String(err));
+
+/**
+ * @param {{
+ *   config: any,
+ *   where: { proposals: string },
+ *   auth: { store: () => Promise<import('node:sqlite').DatabaseSync>, forgeForUser: (userId: string) => Promise<import('./forge.js').Forge> },
+ *   hooks: import('./hooks.js').Hooks,
+ *   intervalMs: number,
+ *   now?: () => number,
+ *   timer?: { setTimeout: Function, clearTimeout: Function },
+ *   firstMs?: number,
+ * }} opts `now`, `timer` and `firstMs` are the tests' clock
+ */
+export function createSpecPoll({ config, where, auth, hooks, intervalMs, now = Date.now, timer = { setTimeout, clearTimeout }, firstMs = FIRST_MS }) {
+  const repo = config.forge.repo;
+  const log = (what) => console.error(`[cornfield] spec.ready poll: ${what}`);
+  let handle = null;
+  let closed = false;
+  let running = null;
+  let failing = false;
+  /** Whether the last tick found no maintainer to look as; what makes the log once per stretch */
+  let noSession = false;
+  /** Whose token the last look used: the log names them each time that changes, so whose sign-in is read with is on record */
+  let lookingAs = null;
+
+  /** The forge as the newest live sign-in of a user who is a maintainer now, and their login, or null when there is none */
+  async function maintainerForge() {
+    const rows = /** @type {{ user_id: string }[]} */ ((await auth.store()).prepare("SELECT user_id FROM sessions WHERE token <> '' AND expires_at > ? ORDER BY created_at DESC, rowid DESC").all(iso(now())));
+    const users = [...new Set(rows.map((r) => r.user_id))].slice(0, MAX_CANDIDATES);
+    for (const userId of users) {
+      try {
+        const forge = await auth.forgeForUser(userId);
+        const me = await forge.me();
+        if (String(me.id) === userId && isMaintainer(await forge.permission(repo, me.login))) return { forge, login: me.login };
+      } catch { /* this user's token is gone or refused: the next one */ }
+    }
+    return null;
+  }
+
+  /** One look. Resolves to what happened; never rejects. */
+  async function look() {
+    let found;
+    try {
+      found = await maintainerForge();
+    } catch (err) {
+      log(`the sign-ins could not be read: ${words(err)}`);
+      return 'error';
+    }
+    // Closed while the sign-ins were checked: the open PRs are not listed with a token after close()
+    if (closed) return 'closed';
+    if (!found) {
+      lookingAs = null;
+      if (!noSession) log('no signed-in maintainer to look as, so the open spec PRs are not checked; it resumes when one signs in');
+      noSession = true;
+      return 'no-session';
+    }
+    if (noSession) log('a signed-in maintainer is back; checking the open spec PRs again');
+    noSession = false;
+    if (found.login !== lookingAs) log(`looking at the open spec PRs with the sign-in of @${found.login}, a maintainer`);
+    lookingAs = found.login;
+    try {
+      const { problem } = await listPullProposals(found.forge, where, { hooks });
+      if (problem) {
+        log(`the forge could not be asked: ${problem}`);
+        return 'error';
+      }
+      return 'ok';
+    } catch (err) {
+      log(`failed: ${words(err)}`);
+      return 'error';
+    }
+  }
+
+  function schedule(ms) {
+    if (closed) return;
+    handle = timer.setTimeout(() => { handle = null; tick(); }, ms);
+    if (handle && typeof handle.unref === 'function') handle.unref();
+  }
+
+  /**
+   * One tick: a look, unless one is still running, then the next is scheduled (twice the interval after a failure, until
+   * one succeeds).
+   * @returns {Promise<'ok' | 'error' | 'no-session' | 'busy' | 'closed'>}
+   */
+  function tick() {
+    if (closed) return Promise.resolve('closed');
+    if (running) return Promise.resolve('busy');
+    running = look().then((out) => {
+      if (out === 'closed') { running = null; return out; }
+      if (out !== 'no-session') failing = out === 'error';
+      running = null;
+      schedule(failing ? intervalMs * 2 : intervalMs);
+      return out;
+    });
+    return running;
+  }
+
+  return {
+    intervalMs,
+    tick,
+    /** Starts the timer: the first look after `firstMs`, then every interval */
+    start() {
+      if (closed || handle || running) return;
+      schedule(firstMs);
+    },
+    close() {
+      closed = true;
+      if (handle !== null) timer.clearTimeout(handle);
+      handle = null;
+    },
+  };
+}