cornfield 0.2.0 → 0.4.0

package/server/server.js

@@ -20,6 +20,9 @@
  *   POST /api/forge/pulls/<n>/merge|rerun   land a green pull request, re-run a failed check (server/forge-actions.js)
  *   POST /api/forge/issues/<n>/spec, /pulls/<n>/build   the pipeline's steps (server/pipeline.js, server/forge-actions.js)
  *   POST /api/forge/release-drafts      the release notes kept as a DRAFT release on the forge (never published, no tag made)
+ *   POST /api/reviews                   Approve or Send back with a review token (`Authorization: Bearer cf_rt_…`), no cookie: the one
+ *                                       route that answers before the sign-in guard, and takes the token alone (server/review-api.js)
+ *   *    /api/review-tokens…            make, list and revoke review tokens: the signed-in maintainer only (server/review-api.js)
  *   *    /proposals…                    server/proposals.js (with sign-in, the open spec PRs too: server/proposal-pulls.js,
  *                                       and a review is a comment on one: server/reviews.js)
  *   *    <proxy path>/…                 server/proxy.js, for each path the config's `proxy` names
@@ -44,11 +47,13 @@ import { CLOUD_FILE } from '../core/cloud.js';
 import { decodePath, inside, notFound, sendFile } from './files.js';
 import { serveProposals } from './proposals.js';
 import { serveForge } from './forge-routes.js';
+import { REVIEWS_PATH, serveReviewCall, serveTokenRoutes } from './review-api.js';
 import { proxyTo } from './proxy.js';
 import { HA_ENV, homeAssistantHouse } from '../plugins/house/home-assistant.js';
 import { assertBind, hostAllowed } from './bind.js';
 import { createAuth, signInSettings } from './auth.js';
 import { createHooks, hooksSettings } from './hooks.js';
+import { createLander } from './lander.js';
 
 export const CORNFIELD_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const EXPLORER_DIR = path.join(CORNFIELD_DIR, 'explorer');
@@ -102,7 +107,7 @@ export function clientConfig(config, signIn = false) {
     version: PKG.version,
     title: config.title,
     links: config.links || [],
-    forge: config.forge ? { type: config.forge.type, url: config.forge.url || (config.forge.type === 'github' ? 'https://github.com' : null), repo: config.forge.repo, mergeMethod: config.forge.mergeMethod || 'merge' } : null,
+    forge: config.forge ? { type: config.forge.type, url: config.forge.url || (config.forge.type === 'github' ? 'https://github.com' : null), repo: config.forge.repo, mergeMethod: config.forge.mergeMethod || 'merge', ...(config.approveLands ? { approveLands: true } : {}) } : null,
     sources: browserSources(config),
     signIn,
   };
@@ -121,10 +126,10 @@ export const HA_CACHE_MS = 30_000;
 /**
  * The server for one repo. It does not listen: `listen(config)` does, where server/bind.js allows.
  * @param {object} config a loaded config (core/config.js `loadConfig`)
- * @param {Parameters<typeof createAuth>[1] & { homeAssistant?: object, hookFetch?: typeof fetch, hookTimer?: { setTimeout: Function, clearTimeout: Function } }} [opts] what sign-in reads (`env`, and tests' fakes;
+ * @param {Parameters<typeof createAuth>[1] & { homeAssistant?: object, hookFetch?: typeof fetch, hookTimer?: { setTimeout: Function, clearTimeout: Function }, landTimer?: { setTimeout: Function, clearTimeout: Function } }} [opts] what sign-in reads (`env`, and tests' fakes;
  *   see server/auth.js); the Home Assistant URL and token are read from the same `env`, and `homeAssistant` is what
  *   `homeAssistantHouse` is handed besides them (the tests' fake WebSocket); `hookFetch` and `hookTimer` are the
- *   tests' fake network and clock for the event hooks (server/hooks.js)
+ *   tests' fake network and clock for the event hooks (server/hooks.js); `landTimer` is the clock of landing on approval (server/lander.js)
  * @returns {import('node:http').Server}
  */
 export function createCornfieldServer(config, opts = {}) {
@@ -135,6 +140,8 @@ export function createCornfieldServer(config, opts = {}) {
   const hooks = hookSettings
     ? createHooks({ hooks: config.hooks, secret: hookSettings.secret, fetch: opts.hookFetch, record: (entry) => auth.record(entry), store: auth.store, now: opts.now, publicUrl: config.publicUrl, timer: opts.hookTimer })
     : null;
+  // Landing on approval needs somebody signed in to land as, like the hooks' events: a half-set key is a startup error
+  if (config.approveLands && !auth.enabled) throw new Error('The config says proposals.approveLands, but sign-in is not set up. An approval is a signed-in maintainer\'s, and it lands as them, so without sign-in nothing would ever land: set up sign-in (docs/sign-in.md), or remove approveLands.');
   const root = config.root;
   const importMap = importMapOf(config);
   const csp = cspOf(importMap);
@@ -142,6 +149,11 @@ export function createCornfieldServer(config, opts = {}) {
   const publicHost = config.publicUrl ? new URL(config.publicUrl).hostname : null;
   const where = { root, proposals: config.proposals, worktrees: config.worktrees, topology: config.topology, dataDir: config.dataDir, publicHost };
   const fixtureCache = new Map();
+  const lander = config.approveLands ? createLander({ config, where, auth, hooks, now: opts.now, timer: opts.landTimer }) : null;
+  if (lander) {
+    // What a restart left waiting: looked at again, in the background
+    lander.resume().catch((err) => console.error(`[cornfield] pending landings could not be picked up: ${err instanceof Error ? err.message : err}`));
+  }
   const proxies = Object.entries(config.proxy || {});
   const houseSource = config.sources.house && (config.sources.house === 'house' || config.sources.house.use === 'house') ? config.sources.house : null;
   const houseFile = houseSource && houseSource.options && houseSource.options.file ? inside(root, houseSource.options.file) : null;
@@ -179,7 +191,7 @@ export function createCornfieldServer(config, opts = {}) {
       else res.end();
     }
   });
-  server.on('close', () => { hooks?.close(); auth.close(); });
+  server.on('close', () => { lander?.close(); hooks?.close(); auth.close(); });
   return server;
 
   async function answer(req, res) {
@@ -214,16 +226,20 @@ export function createCornfieldServer(config, opts = {}) {
       req.resume();
       return sendJson(res, 200, { ok: true });
     }
+    // The review API takes a token, not a session: it is the one route before the guard (and it reads no cookie)
+    if (reqPath === REVIEWS_PATH) return serveReviewCall(req, res, { config, where, auth, hooks, lander, now: opts.now });
     const gate = await auth.gate(req, res, reqPath, url);
     if (gate.done) return;
     const signedIn = gate.session ? { login: gate.session.login } : false;
 
     // Before the proxies: a repo's `/api/` proxy must not swallow the panel's own routes
-    if (await serveForge(req, res, reqPath, url, { config, where, session: gate.session, auth, hooks, now: opts.now, publicHost })) return;
+    if (await serveTokenRoutes(req, res, reqPath, { config, where, auth, session: gate.session, now: opts.now, publicHost })) return;
+    if (await serveForge(req, res, reqPath, url, { config, where, session: gate.session, auth, hooks, lander, now: opts.now, publicHost })) return;
     for (const [prefix, target] of proxies) if (reqPath.startsWith(prefix)) return proxyTo(req, res, target);
     const session = gate.session;
     const forgeFor = session ? () => auth.forgeFor(session) : null;
-    if (serveProposals(req, res, reqPath, url, where, forgeFor, session ? { login: session.login, record: (entry) => auth.record(entry), hooks } : null)) return;
+    const reviewer = session ? { login: session.login, userId: session.userId, repo: config.forge.repo, record: (entry) => auth.record(entry), limits: { auth, now: opts.now }, hooks, lander } : null;
+    if (serveProposals(req, res, reqPath, url, where, forgeFor, reviewer)) return;
 
     if (req.method !== 'GET' && req.method !== 'HEAD') {
       req.resume();

package/server/submit-review.js

@@ -0,0 +1,177 @@
+/**
+ * @module server/submit-review
+ *
+ * The one place a review is checked and posted. The Approve and Send back buttons (`PUT /proposals/<slug>/review`,
+ * server/proposals.js) and the review API (`POST /api/reviews`, server/review-api.js) both call `submitReview`, and
+ * nothing else posts a review, so the two cannot drift: a token-holder's approval counts exactly like the button's.
+ *
+ * In this order, each step refusing in words (a `Refused`, or the forge's own `ForgeError`):
+ *
+ *   1. a flood: more than `ATTEMPTS_PER_MINUTE` attempts by this person in a minute is 429, and nothing is written down
+ *      (server/throttle.js), so a flood of refused attempts cannot grow the audit table
+ *   2. a repeat: the same review of the same proposal again within a few seconds, one at a time, and a ceiling of
+ *      actions per person per minute (429)
+ *   3. the review itself (`validateReview`): a verdict, a note to send back, edits that are objects with an op (400)
+ *   4. the forge, as the person: with the button, their session's token; with the API, the token owner's newest live
+ *      sign-in (`auth.forgeForUser`), because the comment has to be the maintainer's own for `latestReview` to count it
+ *   5. a maintainer NOW: the forge is asked who that token is and what it may do on the repo, not what the session
+ *      remembered 15 minutes ago; a different person, or one who is no longer a maintainer, is 403
+ *   6. the PR is open and holds the proposal (404)
+ *   7. the proposal file's SHA-256 at the PR's head: when the caller names the version it read (`sha256`; the API
+ *      requires it) and the head holds another, 409, so a review counts only for the version that was read. An
+ *      approval records the head's hash (`proposal`) and its mockups' (`mockups`), as the server read them, never the caller's word
+ *   8. with the API, the token once more (`stillAllowed`): one revoked or expired while the forge was being asked is 403
+ *   9. the comment is posted as that person (`postReview`)
+ *
+ * Every attempt that gets past step 1 is a row in `actions` (`record`), whatever the answer: a posted review and a forge
+ * that refused it are written by `postReview`, every refusal before that here. The audit never holds a token's secret,
+ * only its id and label.
+ *
+ * `submitReview` is the one place a review has been posted, so the event hooks' `spec.approved` and `spec.sentBack`
+ * (server/hooks.js) are emitted here, after `postReview`, not in either caller. So is landing on approval
+ * (`proposals.approveLands`, server/lander.js): after an approval is posted, and never for a send-back, `lander.start` is
+ * begun in the background with the head that was approved. It is not awaited and cannot fail the review: the review is
+ * already on the PR, and what goes wrong in landing is the lander's to say on the PR.
+ */
+
+import { ForgeError, isMaintainer } from './forge.js';
+import { PROPOSAL_FILE } from './files.js';
+import { findPullProposal, forgeWords, readPullProposal } from './proposal-pulls.js';
+import { hashPullMockups } from './mockups.js';
+import { postReview, proposalHash, validateReview } from './reviews.js';
+import { Refused, countAttempt, throttle } from './throttle.js';
+import { proposalPayload } from './hooks.js';
+
+const SHA256 = /^[0-9a-f]{64}$/;
+const MAX_PULL = 999_999_999;
+
+/**
+ * @typedef {object} ReviewContext
+ * @property {() => import('./forge.js').Forge | Promise<import('./forge.js').Forge>} forge the forge as the person reviewing (may throw a ForgeError `unauthorized`)
+ * @property {{ proposals: string }} where the server's `where` (server/server.js)
+ * @property {string} repo `owner/name`, for the audit's target
+ * @property {{ login: string, userId: string }} who who is reviewing, by the forge's id
+ * @property {(entry: import('./reviews.js').AuditEntry) => Promise<void> | void} record writes a row of the audit table
+ * @property {{ auth: object, now?: () => number }} limits what `server/throttle.js` counts by (one `auth` per server)
+ * @property {import('./reviews.js').ApiVia} [via] set when the review came through the API: the token it came with
+ * @property {import('./hooks.js').Hooks | null} [hooks] the event hooks (server/hooks.js): `spec.approved` or `spec.sentBack` after a post
+ * @property {import('./lander.js').Lander | null} [lander] landing on approval (server/lander.js), set only when the config says `proposals.approveLands`
+ * @property {boolean} [requireSha256] the caller must name the version it read (the API does; the button reads the head itself)
+ * @property {() => string | null | Promise<string | null>} [stillAllowed] asked once more just before the post, after every
+ *   forge call: null to go on, or the words for why not (the API's token was revoked or expired meanwhile), refused 403
+ */
+
+/**
+ * Checks and posts a review. Throws a `Refused` or a `ForgeError` (answer with `refusal(err)`); resolves with the
+ * review as the PR will now show it.
+ * @param {ReviewContext} ctx
+ * @param {{ pull: unknown, slug: unknown, review: unknown, sha256?: unknown, problem?: { status: number, message: string } }} input
+ *   `review` is `{ verdict, note?, edits? }` as the caller sent it; `sha256` the proposal file's hash as the caller read it
+ *   (absent for the button, which sends none); `problem` is something already wrong with the request itself (it was not
+ *   JSON, it was too large): it is refused here, after the flood count and with a row, like every other refusal
+ * @returns {Promise<import('./reviews.js').FoundReview>}
+ */
+export async function submitReview(ctx, { pull, slug, review, sha256 = null, problem }) {
+  const { who, repo, via } = ctx;
+  const now = ctx.limits.now ?? Date.now;
+  const validPull = typeof pull === 'number' && Number.isInteger(pull) && pull >= 1 && pull <= MAX_PULL;
+  const target = validPull ? `${repo}#${pull}` : repo;
+  // What the audit row says was asked: only what is small, plain and of the right shape, never the whole body
+  // A slug or a hash only when it has the shape of one: anything else the caller sent is not kept (it could be a secret pasted in the wrong field)
+  const slugShown = typeof slug === 'string' && slug.length <= 200 && PROPOSAL_FILE.test(`${slug}.json`) ? slug : undefined;
+  const asked = /** @type {Record<string, unknown>} */ ({ slug: slugShown, pull: validPull ? pull : undefined, ...(isSha256(sha256) ? { sha256 } : {}), ...(via ? { via: 'api', token: via.token } : {}) });
+
+  const flood = countAttempt(ctx.limits, who.userId, now());
+  if (flood) throw flood;
+
+  let recorded = false;
+  const record = async (entry) => {
+    recorded = true;
+    return ctx.record(entry);
+  };
+  let release = () => {};
+  try {
+    if (problem) throw new Refused(problem.status, problem.message, 'invalid');
+    if (!validPull) throw new Refused(400, 'Say which pull request: its number.', 'invalid');
+    if (typeof slug !== 'string' || slug.length > 200 || !PROPOSAL_FILE.test(`${slug}.json`)) throw new Refused(400, 'Say which proposal: its name in the proposals folder.', 'invalid');
+    if ((sha256 !== null && sha256 !== undefined) || ctx.requireSha256) {
+      if (!isSha256(sha256)) throw new Refused(400, 'Say which version you read: "sha256" is the proposal file\'s SHA-256, 64 lower-case hex digits.', 'invalid');
+    }
+    release = throttle(ctx.limits, who.userId, `review|${pull}|${slug}`, now());
+
+    if (review !== null && typeof review === 'object') {
+      const verdict = /** @type {any} */ (review).verdict;
+      if (verdict === 'approved' || verdict === 'notes') asked.verdict = verdict;
+      if (Array.isArray(/** @type {any} */ (review).edits)) asked.edits = /** @type {any} */ (review).edits.length;
+    }
+    const checked = validateReview(review);
+    if (!checked.ok) throw new Refused(400, checked.error, 'invalid');
+
+    const forge = await ctx.forge();
+    const me = await forge.me();
+    if (String(me.id) !== who.userId) throw new Refused(403, 'The forge says this sign-in is someone else now; sign in again.', 'forbidden');
+    if (!isMaintainer(await forge.permission(repo, me.login))) throw new Refused(403, `Only the repo's maintainers review proposals; ${me.login} is not one now.`, 'forbidden');
+
+    const entry = await findPullProposal(forge, ctx.where, slug, pull);
+    if (!entry) throw new Refused(404, `Pull request #${pull} is not open or does not hold the proposal ${slug}.`, 'not-found');
+    // The proposal as the head holds it now: a later push that changes it is not this review's
+    const head = proposalHash(await readPullProposal(forge, ctx.where, entry));
+    if (typeof sha256 === 'string' && sha256 !== head) throw new Refused(409, `${slug} has changed since you read it (the head of pull request #${pull} holds ${head.slice(0, 12)}…, not ${sha256.slice(0, 12)}…); read it again and review that version.`, 'conflict');
+
+    let checkedReview = checked.value;
+    if (checkedReview.verdict === 'approved') {
+      checkedReview = { ...checkedReview, proposal: head };
+      const mockups = await hashPullMockups(forge, ctx.where, entry);
+      if (mockups.length) checkedReview = { ...checkedReview, mockups };
+    }
+    // The checks above took a while: a token revoked (or expired) since it was looked up does not post
+    if (ctx.stillAllowed) {
+      const why = await ctx.stillAllowed();
+      if (why) throw new Refused(403, why, 'forbidden');
+    }
+    const posted = await postReview(forge, pull, slug, checkedReview, { login: me.login, record, via });
+    // The one place a review is posted, so the button and the API fire the same event; the actor is who the forge says
+    // the poster is (with the API, the token's owner), never a name the caller sent
+    ctx.hooks?.emit(posted.verdict === 'approved' ? 'spec.approved' : 'spec.sentBack', () => proposalPayload(forge, ctx.where, entry, { actor: me.login, note: checkedReview.note, edits: checkedReview.edits }));
+    if (posted.verdict === 'approved' && ctx.lander) {
+      const { lander } = ctx;
+      Promise.resolve()
+        .then(() => lander.start({ pull, slug, sha: entry.sha, who: { userId: who.userId, login: me.login } }))
+        .catch((err) => console.error(`[cornfield] landing ${target} on approval could not begin: ${err instanceof Error ? err.message : err}`));
+    }
+    return posted;
+  } catch (err) {
+    if (!(err instanceof Refused) && !(err instanceof ForgeError)) console.error(`[cornfield] the review of ${target}: ${err instanceof Error ? err.stack || err.message : err}`);
+    // postReview writes the row of a review it tried to post; anything refused before that is written here
+    if (!recorded) {
+      const out = refusal(err);
+      try {
+        await ctx.record({ login: who.login, action: 'review', target, request: asked, status: err instanceof ForgeError ? err.status ?? (err.code === 'unavailable' ? null : out.status) : out.status, answer: { error: out.body.error, code: out.body.code } });
+      } catch (writeErr) {
+        console.error(`[cornfield] the refusal of the review of ${target} could not be written to the audit table: ${writeErr instanceof Error ? writeErr.message : writeErr}`);
+      }
+    }
+    throw err;
+  } finally {
+    release();
+  }
+}
+
+/**
+ * The answer to give for an error `submitReview` threw: the status, `{ error, code }` and the headers to send.
+ * A ForgeError keeps the words of the forge; anything else is a 500 that says nothing of what went wrong.
+ * @param {unknown} err
+ * @returns {{ status: number, body: { error: string, code: string }, headers: Record<string, string> }}
+ */
+export function refusal(err) {
+  if (err instanceof Refused) return { status: err.status, body: { error: err.message, code: err.code }, headers: err.headers };
+  if (err instanceof ForgeError) {
+    const status = { 'not-found': 404, invalid: 400, unauthorized: 401, 'rate-limited': 429 }[err.code] ?? 502;
+    const wait = err.rateLimit && err.rateLimit.retryAfter != null ? { 'Retry-After': String(Math.ceil(err.rateLimit.retryAfter)) } : {};
+    return { status, body: { error: forgeWords(err), code: err.code }, headers: wait };
+  }
+  return { status: 500, body: { error: 'The review could not be saved; the log says why.', code: 'unavailable' }, headers: {} };
+}
+
+/** The checks on a `sha256` the caller names: 64 lower-case hex digits */
+export const isSha256 = (v) => typeof v === 'string' && SHA256.test(v);

package/server/throttle.js

@@ -0,0 +1,93 @@
+/**
+ * @module server/throttle
+ *
+ * The flood and repeat limits of everything cornfield does on a forge as a person (server/forge-actions.js: land,
+ * re-run, the pipeline's steps; server/submit-review.js: Approve and Send back, from the button and from the API).
+ * They live in one place so the same person is counted once whichever door they come in by. State is kept in memory,
+ * per `auth` object (one per server), keyed by the forge's user id:
+ *
+ *   countAttempt   a ceiling on attempts of any kind per person per minute (`ATTEMPTS_PER_MINUTE`), counted before
+ *                  anything is written down, so refused attempts cannot grow the audit table without end
+ *   throttle       not twice: one at a time for the same key, `COOLDOWN_MS` before it is asked again, and a ceiling of
+ *                  `PER_MINUTE` actions per person per minute
+ *
+ * A refusal is a `Refused`: a status (429, or the action's own), words, a code and the headers to answer with.
+ */
+
+/** The same action on the same target is not asked again for this long (a double click, a retry before the forge has caught up) */
+export const COOLDOWN_MS = 5000;
+/** The most actions one person takes in a minute */
+export const PER_MINUTE = 20;
+/** The most attempts of any kind (refused ones too) one person makes in a minute, before any is written down */
+export const ATTEMPTS_PER_MINUTE = 60;
+
+/** @type {WeakMap<object, { busy: Set<string>, last: Map<string, number>, perUser: Map<string, number[]>, attempts: Map<string, number[]> }>} */
+const throttles = new WeakMap();
+
+function throttlesOf(auth) {
+  let t = throttles.get(auth);
+  if (!t) throttles.set(auth, (t = { busy: new Set(), last: new Map(), perUser: new Map(), attempts: new Map() }));
+  return t;
+}
+
+/** An action's own refusal: the status to answer and the words */
+export class Refused extends Error {
+  /**
+   * @param {number} status
+   * @param {string} message
+   * @param {string} code
+   * @param {Record<string, string>} [headers]
+   */
+  constructor(status, message, code, headers = {}) {
+    super(message);
+    this.status = status;
+    this.code = code;
+    this.headers = headers;
+  }
+}
+
+/**
+ * Counts an attempt of any kind; refuses one past the ceiling, without writing anything down (the release draft shares it).
+ * @param {{ auth: object }} ctx
+ * @param {string} userId whom to count (the forge's id for them)
+ * @param {number} at now, in ms
+ * @param {number} [max] the ceiling per minute (`ATTEMPTS_PER_MINUTE`)
+ * @returns {Refused | null}
+ */
+export function countAttempt(ctx, userId, at, max = ATTEMPTS_PER_MINUTE) {
+  const t = throttlesOf(ctx.auth);
+  for (const [k, list] of t.attempts) if (!list.length || at - list[list.length - 1] >= 60_000) t.attempts.delete(k);
+  const mine = (t.attempts.get(userId) || []).filter((when) => at - when < 60_000);
+  if (mine.length >= max) {
+    t.attempts.set(userId, mine);
+    return new Refused(429, `That is ${max} attempts in a minute; wait a little.`, 'rate-limited', { 'Retry-After': String(Math.max(1, Math.ceil((60_000 - (at - mine[0])) / 1000))) });
+  }
+  mine.push(at);
+  t.attempts.set(userId, mine);
+  return null;
+}
+
+/**
+ * Refuses a repeat and a flood; answers the function that releases `key` when the attempt is over.
+ * @param {{ auth: object }} ctx
+ * @param {string} userId
+ * @param {string} key the action and its target, like `merge|12|` or `review|23|0013-meals`
+ * @param {number} at
+ * @returns {() => void}
+ * @throws {Refused}
+ */
+export function throttle(ctx, userId, key, at) {
+  const t = throttlesOf(ctx.auth);
+  for (const [k, when] of t.last) if (at - when >= COOLDOWN_MS) t.last.delete(k);
+  const mine = (t.perUser.get(userId) || []).filter((when) => at - when < 60_000);
+  t.perUser.set(userId, mine);
+  const wait = (ms, words) => new Refused(429, words, 'rate-limited', { 'Retry-After': String(Math.max(1, Math.ceil(ms / 1000))) });
+  if (t.busy.has(key)) throw wait(COOLDOWN_MS, 'That is already being done; wait for it to finish.');
+  const before = t.last.get(key);
+  if (before !== undefined && at - before < COOLDOWN_MS) throw wait(COOLDOWN_MS - (at - before), 'That was just asked; wait a few seconds before asking again.');
+  if (mine.length >= PER_MINUTE) throw wait(60_000 - (at - mine[0]), `That is ${PER_MINUTE} actions in a minute; wait a little.`);
+  t.busy.add(key);
+  t.last.set(key, at);
+  mine.push(at);
+  return () => { t.busy.delete(key); };
+}