@bongos/core 1.21.126 → 1.21.128

package/src/module-loader/error-tally.js

@@ -0,0 +1,188 @@
+// src/module-loader/error-tally.js — count each mounted module's runtime errors
+// per day, so a store module's Reliability score has real installs behind it
+// (task 1004637, follow-up to task 1003793; ADR 0343 D1).
+//
+// WHY. The store scores Reliability from a daily runtime-error count per module
+// version, but nothing on an instance counted one: the loader console.errored a
+// poller that failed to load and attributed nothing else to a module at all. This
+// file is the instance-side tally the loader feeds; scripts/gds/module-error-report.js
+// is the sender that turns it into reports.
+//
+// WHAT COUNTS. A request a module's router took and answered 500 (a thrown error or
+// an unhandled failure — not a 4xx, which is the caller's mistake, and not a 503,
+// which modules use to say "not set up here"); a poller that failed to load, threw
+// or rejected on start; and anything a poller hands to the `reportError` the loader
+// gives it. Only a module whose manifest carries an exact X.Y.Z version is counted,
+// because the store files counts against an exact version.
+//
+// COUNTS ONLY (owner's privacy posture, 2026-10-01). The tally holds module key,
+// version, day and a whole number — never an error's text, a path or a request.
+//
+// WHO SENDS (owner, 2026-10-15). A builder's sign-in on the box: the store's report
+// routes take a builder, and no project-to-store credential exists (parked as
+// task 1004740). So this file never talks to the network. It writes the tally to the
+// instance config dir, and startDailySend() runs the sender as a CHILD process —
+// cli-lib's session plumbing must never load in the server process.
+//
+// Counting is always in memory; nothing touches disk until start() is called, and
+// only the server entrypoint calls it — so a test or tool that mounts modules never
+// writes a file.
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const FILE_NAME = 'module-error-counts.json';
+// Days kept on disk. One more than the store accepts (ERROR_DAY_WINDOW = 30), so a
+// day the sender has not reached yet is never pruned before it can go.
+const KEEP_DAYS = 31;
+const FLUSH_DELAY_MS = 5_000;
+const ROLL_CHECK_MS = 60 * 60 * 1000;
+const SEND_EVERY_MS = 6 * 60 * 60 * 1000;
+const FIRST_SEND_DELAY_MS = 10 * 60 * 1000;
+const EXACT_VERSION = /^\d+\.\d+\.\d+$/;
+// The sender, rooted at this file so what the server runs is answerable by reading
+// this line (exec-path-guard, task 1003371).
+const SENDER = path.join(__dirname, '..', '..', 'scripts', 'gds', 'module-error-report.js');
+
+function utcDay(ms = Date.now()) { return new Date(ms).toISOString().slice(0, 10); }
+
+let _now = Date.now;
+let _days = {};          // day → { "key@version": count }
+let _mounted = new Map(); // key → version, for every module seeded today and each new day
+let _file = null;         // set by start(); null = nothing written to disk
+let _flushTimer = null;
+let _timers = [];
+
+function entryOf(key, version) { return `${key}@${version}`; }
+
+// Make sure today has a 0 for every mounted module — a clean day is exactly what
+// Reliability measures, so a module that ran and threw nothing must still appear.
+function seedToday() {
+  const today = utcDay(_now());
+  const row = (_days[today] ||= {});
+  for (const [key, version] of _mounted) {
+    const e = entryOf(key, version);
+    if (!(e in row)) row[e] = 0;
+  }
+  return row;
+}
+
+// Register the modules this process runs. Call once the enabled set is known; a
+// module without an exact version is ignored. Returns how many were registered.
+function seed(modules) {
+  let n = 0;
+  for (const { key, manifest } of modules) {
+    const v = manifest && manifest.version;
+    if (typeof v !== 'string' || !EXACT_VERSION.test(v)) continue;
+    _mounted.set(key, v);
+    n += 1;
+  }
+  seedToday();
+  scheduleFlush();
+  return n;
+}
+
+// Count one runtime error against a mounted module. Unknown or unversioned keys
+// are ignored, so a stray call can never invent a module.
+function record(key) {
+  const version = _mounted.get(key);
+  if (!version) return false;
+  const row = seedToday();
+  row[entryOf(key, version)] += 1;
+  scheduleFlush();
+  return true;
+}
+
+function prune() {
+  const cutoff = utcDay(_now() - KEEP_DAYS * 86_400_000);
+  for (const d of Object.keys(_days)) if (d < cutoff) delete _days[d];
+}
+
+function scheduleFlush() {
+  if (!_file || _flushTimer) return;
+  _flushTimer = setTimeout(() => { _flushTimer = null; flush(); }, FLUSH_DELAY_MS);
+  if (_flushTimer.unref) _flushTimer.unref();
+}
+
+// Write the tally atomically (temp file, then rename), so the sender never reads
+// half a file. Best-effort: a failed write costs a count, never the server.
+function flush() {
+  if (!_file) return false;
+  prune();
+  try {
+    fs.mkdirSync(path.dirname(_file), { recursive: true });
+    const tmp = `${_file}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, JSON.stringify({ days: _days }, null, 2));
+    fs.renameSync(tmp, _file);
+    return true;
+  } catch (e) {
+    process.emitWarning(`module error tally: could not write ${_file}: ${e && e.message ? e.message : e}`);
+    return false;
+  }
+}
+
+// Read a tally file. Never throws: a missing or unreadable file is an empty tally.
+function readTally(file) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+    return parsed && typeof parsed.days === 'object' && parsed.days ? parsed.days : {};
+  } catch { return {}; }
+}
+
+function defaultFile() {
+  return require('../instance-config').configPath(FILE_NAME);
+}
+
+// Persist from now on. Merges what is already on disk (a restart mid-day keeps the
+// morning's count: the higher of the two wins per entry), seeds today, and checks
+// hourly for a new day so a server that runs for a week still files a 0 each day.
+function start({ file = defaultFile(), now } = {}) {
+  if (now) _now = now;
+  _file = file;
+  const onDisk = readTally(file);
+  for (const [day, row] of Object.entries(onDisk)) {
+    const mine = (_days[day] ||= {});
+    for (const [e, n] of Object.entries(row || {})) {
+      if (Number.isInteger(n) && n >= 0) mine[e] = Math.max(mine[e] || 0, n);
+    }
+  }
+  seedToday();
+  flush();
+  const roll = setInterval(() => { seedToday(); flush(); }, ROLL_CHECK_MS);
+  if (roll.unref) roll.unref();
+  _timers.push(roll);
+  return file;
+}
+
+// Run the sender now and then (scripts/gds/module-error-report.js) as a child
+// process. It sends nothing without a builder sign-in on this box, and only days
+// before today, so running it often is cheap and safe. Returns the timers.
+function startDailySend({ spawnImpl } = {}) {
+  const { spawn } = require('node:child_process');
+  const run = spawnImpl || (() => {
+    const child = spawn(process.execPath, [SENDER, '--quiet'], { stdio: 'ignore', detached: false });
+    child.on('error', () => {});
+    if (child.unref) child.unref();
+  });
+  if (!fs.existsSync(SENDER)) return [];
+  const go = () => { try { run(SENDER); } catch { /* best-effort */ } };
+  const first = setTimeout(go, FIRST_SEND_DELAY_MS);
+  const every = setInterval(go, SEND_EVERY_MS);
+  for (const t of [first, every]) if (t.unref) t.unref();
+  _timers.push(first, every);
+  return [first, every];
+}
+
+// The in-memory tally, for tests and a /watch tile.
+function snapshot() { return JSON.parse(JSON.stringify(_days)); }
+
+function _reset() {
+  for (const t of _timers) { clearTimeout(t); clearInterval(t); }
+  if (_flushTimer) clearTimeout(_flushTimer);
+  _timers = []; _flushTimer = null; _days = {}; _mounted = new Map(); _file = null; _now = Date.now;
+}
+
+module.exports = {
+  FILE_NAME,
+  utcDay, seed, record, readTally, start, startDailySend, snapshot, _reset,
+};

package/src/module-loader/loader.js

@@ -112,7 +112,7 @@ function discoverModules({ roots = DEFAULT_ROOTS, core = coreVersion() } = {}) {
 // Cached singleton for app code (lazy, like src/modules.js's modules()).
 let _cache = null;
 function discovered(opts) { return (_cache ||= discoverModules(opts)); }
-function _reset() { _cache = null; _startedPollers = []; _boot = { routes: null, pollers: null }; _bootShape = null; }
+function _reset() { resetErrorTally(); _cache = null; _startedPollers = []; _boot = { routes: null, pollers: null }; _bootShape = null; }
 
 // The discovered manifests as MODULE_REGISTRY-shaped entries, so src/modules.js
 // can merge them with its built-in registry. Empty until a module exists.
@@ -197,10 +197,13 @@ function mountModuleRoutes(router, { isEnabled } = {}, opts) {
     for (const rk of routeKeys) {
       const factory = require(path.join(dir, 'routes', `${rk}.js`));
       const sub = typeof factory === 'function' ? factory() : factory;
-      router.use(sub);
+      router.use(attributeErrors(key, sub));
       mounted += 1;
     }
   }
+  // Every enabled module gets a 0 for today, routes or not: a clean day is what the
+  // store's Reliability score measures (task 1004637).
+  seedErrorTally(enabled);
   return mounted;
 }
 
@@ -352,19 +355,26 @@ function startModulePollers(deps = {}, { isEnabled } = {}, opts) {
         // here. A file that EXISTS but throws is a real error (the catch below).
         if (e && e.code === 'MODULE_NOT_FOUND' && String(e.message).includes(pollerPath)) continue;
         console.error(`[module-loader] poller ${key}/${pk} failed to load: ${e && e.message ? e.message : e}`);
+        recordModuleError(key);
         continue;
       }
       try {
         if (poller && typeof poller.start === 'function') {
-          const r = poller.start(deps);
+          // reportError: how a poller counts a failure in its own runs against its
+          // module (task 1004637) — counts only, it takes no message.
+          const r = poller.start({ ...deps, reportError: () => recordModuleError(key) });
           if (r && typeof r.then === 'function') {
-            r.catch((e) => console.error(`[module-loader] poller ${key}/${pk} start error: ${e && e.message ? e.message : e}`));
+            r.catch((e) => {
+              console.error(`[module-loader] poller ${key}/${pk} start error: ${e && e.message ? e.message : e}`);
+              recordModuleError(key);
+            });
           }
           started.push({ key, poller: pk, handle: poller });
           _boot.pollers.add(key);
         }
       } catch (e) {
         console.error(`[module-loader] poller ${key}/${pk} start threw: ${e && e.message ? e.message : e}`);
+        recordModuleError(key);
       }
     }
   }
@@ -397,9 +407,47 @@ function loaderSnapshot(opts) {
     loaded: d.loaded.map((m) => m.key),
     skipped: d.skipped,
     errors: d.errors,
+    // per-day runtime-error counts per module@version, kept for the store (task 1004637)
+    errorCounts: snapshotErrorTally(),
   };
 }
 
+// Which module took this request, and whether its 500 has been hooked yet.
+const OWNER = Symbol('moduleOwner');
+const HOOKED = Symbol('moduleErrorHooked');
+
+// Wrap a module's router so a request it answers with a 500 counts against that
+// module (task 1004637). The module is the owner from the moment the request enters
+// its router until it falls through (the router calls next() with no error): a request
+// the module never matched is not its fault, and the next module or the kernel takes
+// it from there. An error the module passes on (next(err)) keeps it the owner, so the
+// kernel's error handler answering 500 for it still counts here. Only a 500 counts:
+// a 4xx is the caller's mistake and a 503 is a module saying it is not set up.
+//
+// A plain function rather than a nested express Router, so nothing about mounting
+// changes for the kernel; `stack` is passed through so tooling that reads a mounted
+// router's routes still sees the module's own.
+function attributeErrors(key, sub) {
+  const scoped = function moduleScoped(req, res, next) {
+    const prev = req[OWNER];
+    req[OWNER] = key;
+    if (!req[HOOKED] && res && typeof res.on === 'function') {
+      req[HOOKED] = true;
+      res.on('finish', () => { if (res.statusCode === 500 && req[OWNER]) recordModuleError(req[OWNER]); });
+    }
+    sub(req, res, (err) => {
+      if (!err || typeof err === 'string') req[OWNER] = prev; // fell through ('route'/'router' too)
+      next(err);
+    });
+  };
+  if (sub && sub.stack) scoped.stack = sub.stack;
+  return scoped;
+}
+
+// Per-module runtime-error counts (task 1004637). Pure fs + timers, no network and no
+// core requires at load, so it is cycle-safe here.
+const { seed: seedErrorTally, record: recordModuleError, _reset: resetErrorTally, snapshot: snapshotErrorTally } = require('./error-tally');
+
 module.exports = {
   discoverModules,
   discovered,

package/src/platform-server.js

@@ -47,6 +47,7 @@ const express = require('express');
 const { mountInternalSurfaces, isStatusHost, isBuildersHost, injectBrandingGlobal } = require('./bongos/serve-internal');
 const { isApiShapedPath, apiNotFoundBody } = require('./bongos/api-path-404');
 const moduleLoader = require('./module-loader/loader');
+const { start: startErrorTally, startDailySend: startErrorReports } = require('./module-loader/error-tally');
 const { isModuleEnabled } = require('./modules');
 // Hoisted out of an inline require inside the boot IIFE below (task 1004052 strand walk).
 // knip does not follow an inline require(), so every export of module-overrides.js read as
@@ -413,6 +414,12 @@ if (require.main === module) {
       // process starts, and /version carries the count within one query of a restart.
       schemaPending();
       startBackgroundServices();
+      // Store modules' daily runtime-error counts (task 1004637): keep the tally on
+      // disk, and now and then run the sender, which goes out only with a builder
+      // sign-in on this box. Here, not in startBackgroundServices(), so a test that
+      // starts the pollers never writes the tally or spawns the sender.
+      try { startErrorTally(); startErrorReports(); }
+      catch (e) { logger.warn({ err: e.message }, 'module error tally did not start; module error counts are not kept'); }
     });
   })().catch((err) => {
     // THE AWAIT ABOVE MUST NOT SOFTEN A FATAL BOOT (task 1002857, caught by

package/tests/howto_state.mjs

@@ -0,0 +1,178 @@
+// tests/howto_state.mjs — per-person read/unread state for the how-to steps (task 1004634)
+//
+// Two halves. The DB-free half drives howto-state.js against an in-memory pool that
+// honours the three statements it issues, so the contract runs everywhere: unread by
+// default, mark read, skip all, a read upgrades a skip (never the reverse), the first
+// read time is kept, reset some or all, and an id the guide does not name is refused.
+// The DB half applies the real migration and runs the same calls against Postgres; it
+// skips when no Postgres is reachable (DATABASE_URL), like copy_desk_flags_db.mjs.
+//
+// Run:  node tests/howto_state.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import fs from 'node:fs';
+import path from 'node:path';
+import url from 'node:url';
+
+const require = createRequire(import.meta.url);
+const hs = require('../modules/onboarding/howto-state.js');
+const guide = require('../modules/onboarding/howto-guide.js');
+const ROOT = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), '..');
+
+let passed = 0;
+let failed = 0;
+async function test(name, fn) {
+  try { await fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.log(`  FAIL ${name}`); console.log(`     ${err.stack || err.message}`); }
+}
+
+// An in-memory stand-in for the three statements howto-state.js runs.
+function fakePool() {
+  const rows = new Map(); // `${b}|${step}` → { builder_id, step_id, how, read_at }
+  let clock = 0;
+  return {
+    rows,
+    async query(sql, params) {
+      if (/^SELECT step_id/.test(sql)) {
+        return { rows: [...rows.values()].filter((r) => r.builder_id === params[0]) };
+      }
+      if (/^INSERT INTO onboarding_howto_reads/.test(sql)) {
+        const [b, ids, how] = params;
+        for (const s of ids) {
+          const k = `${b}|${s}`;
+          const cur = rows.get(k);
+          if (!cur) rows.set(k, { builder_id: b, step_id: s, how, read_at: new Date(1e12 + ++clock * 1000) });
+          else if (cur.how === 'skipped' && how === 'read') cur.how = 'read';
+        }
+        return { rows: [] };
+      }
+      if (/^DELETE FROM onboarding_howto_reads WHERE builder_id = \$1 AND/.test(sql)) {
+        for (const s of params[1]) rows.delete(`${params[0]}|${s}`);
+        return { rows: [] };
+      }
+      if (/^DELETE FROM onboarding_howto_reads WHERE builder_id = \$1$/.test(sql)) {
+        for (const [k, r] of rows) if (r.builder_id === params[0]) rows.delete(k);
+        return { rows: [] };
+      }
+      throw new Error(`unexpected SQL: ${sql}`);
+    },
+  };
+}
+
+async function contract(label, deps, A, B) {
+  const first = guide.STEPS[0].id;
+  const second = guide.STEPS[1].id;
+
+  await test(`${label}: a new person has every step unread, next is the first`, async () => {
+    const s = await hs.readState(A, deps);
+    assert.equal(s.total, guide.STEPS.length);
+    assert.equal(s.unread, s.total);
+    assert.equal(s.next_unread, first);
+    assert.ok(s.steps.every((x) => x.state === 'unread' && x.read_at === null));
+  });
+
+  await test(`${label}: mark one read → it is read, next moves on, other people untouched`, async () => {
+    const r = await hs.markRead(A, { steps: [first] }, deps);
+    assert.equal(r.ok, true);
+    assert.equal(r.state.read, 1);
+    assert.equal(r.state.next_unread, second);
+    assert.equal(r.state.steps[0].state, 'read');
+    assert.ok(r.state.steps[0].read_at);
+    const other = await hs.readState(B, deps);
+    assert.equal(other.read, 0, 'per person');
+  });
+
+  await test(`${label}: re-marking keeps the first read time (idempotent)`, async () => {
+    const before = (await hs.readState(A, deps)).steps[0].read_at;
+    await hs.markRead(A, { steps: [first] }, deps);
+    assert.equal((await hs.readState(A, deps)).steps[0].read_at, before);
+  });
+
+  await test(`${label}: an unknown step id is refused and nothing is stored`, async () => {
+    const r = await hs.markRead(A, { steps: [second, 'not_a_step'] }, deps);
+    assert.equal(r.ok, false);
+    assert.equal(r.reason, 'unknown_step');
+    assert.deepEqual(r.unknown, ['not_a_step']);
+    assert.equal((await hs.readState(A, deps)).read, 1);
+    assert.equal((await hs.markRead(A, { steps: [] }, deps)).reason, 'no_steps');
+    assert.equal((await hs.markRead(A, { steps: [second], how: 'seen' }, deps)).reason, 'bad_how');
+  });
+
+  await test(`${label}: skip all marks every other step skipped; nothing left unread`, async () => {
+    const r = await hs.markRead(A, { all: true, how: 'skipped' }, deps);
+    assert.equal(r.state.unread, 0);
+    assert.equal(r.state.next_unread, null);
+    assert.equal(r.state.read, 1, 'the step already read stays read');
+    assert.equal(r.state.skipped, guide.STEPS.length - 1);
+  });
+
+  await test(`${label}: reading a skipped step upgrades it; skipping a read one does not downgrade`, async () => {
+    await hs.markRead(A, { steps: [second] }, deps);
+    let s = await hs.readState(A, deps);
+    assert.equal(s.steps[1].state, 'read');
+    await hs.markRead(A, { steps: [second], how: 'skipped' }, deps);
+    s = await hs.readState(A, deps);
+    assert.equal(s.steps[1].state, 'read');
+  });
+
+  await test(`${label}: reset some steps, then reset everything`, async () => {
+    let r = await hs.reset(A, { steps: [first] }, deps);
+    assert.equal(r.state.steps[0].state, 'unread');
+    assert.equal(r.state.next_unread, first);
+    assert.equal((await hs.reset(A, { steps: ['nope_nope'] }, deps)).reason, 'unknown_step');
+    r = await hs.reset(A, {}, deps);
+    assert.equal(r.state.unread, guide.STEPS.length);
+  });
+}
+
+await test('shapeState ignores a stored mark for a step the guide no longer has', async () => {
+  const s = hs.shapeState([{ step_id: 'retired_step', how: 'read', read_at: new Date() }]);
+  assert.equal(s.read, 0);
+  assert.equal(s.total, guide.STEPS.length);
+});
+
+await test('the module declares its migration and the routes are mounted', async () => {
+  const mod = JSON.parse(fs.readFileSync(path.join(ROOT, 'modules/onboarding/module.json'), 'utf8'));
+  assert.equal(mod.contributes.migrations, true);
+  assert.ok(fs.existsSync(path.join(ROOT, 'modules/onboarding/migrations/onboarding_001_howto_reads.sql')));
+  const src = fs.readFileSync(path.join(ROOT, 'modules/onboarding/routes/onboarding.js'), 'utf8');
+  for (const r of ["router.get('/me/howto'", "router.post('/me/howto/read'", "router.post('/me/howto/reset'"]) assert.ok(src.includes(r), r);
+});
+
+await contract('in-memory', { pool: fakePool() }, 1, 2);
+
+// ── the real database ────────────────────────────────────────────────────────
+const { Pool } = require('pg');
+const pool = new Pool({ connectionString: process.env.DATABASE_URL || undefined });
+let dbUp = true;
+try { await pool.query('SELECT 1'); } catch (e) {
+  dbUp = false;
+  console.log(`  (db half SKIPPED — no reachable Postgres: ${e.code || e.message}. Set DATABASE_URL to run.)`);
+}
+if (dbUp) {
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY);
+    CREATE TABLE IF NOT EXISTS builders (id bigserial PRIMARY KEY, github_login text);
+  `);
+  await pool.query(fs.readFileSync(path.join(ROOT, 'modules/onboarding/migrations/onboarding_001_howto_reads.sql'), 'utf8'));
+  await pool.query(fs.readFileSync(path.join(ROOT, 'modules/onboarding/migrations/onboarding_001_howto_reads.sql'), 'utf8')); // idempotent
+  const A = 910000001;
+  const B = 910000002;
+  await pool.query('INSERT INTO builders (id, github_login) VALUES ($1, $2), ($3, $4) ON CONFLICT (id) DO NOTHING', [A, 'howto-a', B, 'howto-b']);
+  await pool.query('DELETE FROM onboarding_howto_reads WHERE builder_id IN ($1, $2)', [A, B]);
+  try {
+    await contract('postgres', { pool }, A, B);
+    await test('postgres: the CHECK refuses a malformed step id and a bad how', async () => {
+      await assert.rejects(pool.query("INSERT INTO onboarding_howto_reads (builder_id, step_id) VALUES ($1, 'Bad Id')", [A]));
+      await assert.rejects(pool.query("INSERT INTO onboarding_howto_reads (builder_id, step_id, how) VALUES ($1, 'welcome', 'seen')", [A]));
+    });
+  } finally {
+    await pool.query('DELETE FROM onboarding_howto_reads WHERE builder_id IN ($1, $2)', [A, B]).catch(() => {});
+    await pool.query('DELETE FROM builders WHERE id IN ($1, $2)', [A, B]).catch(() => {});
+  }
+}
+await pool.end().catch(() => {});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);