@bongos/core 1.19.356 → 1.19.357

package/.bongos-core.json

@@ -2,22 +2,22 @@
   "artifact": "bongos-core",
   "manifest_schema": 1,
   "generator": "scripts/gds/package-core.js",
-  "core_version": "1.19.356",
-  "core_contract": "1.19.356",
-  "source_commit": "cb63704ad7571ab5d52fe949b6f265281f754679",
+  "core_version": "1.19.357",
+  "core_contract": "1.19.357",
+  "source_commit": "3c8f80201ef2dd9992ceea78af1081b7cd74524d",
   "source_ref": "HEAD",
-  "built_at": "2026-08-25T14:41:47.258Z",
+  "built_at": "2026-08-25T16:20:35.425Z",
   "redaction": {
     "model": "docs-redacted+functional-verbatim",
     "docs_redacted": 330,
     "agent_docs_stubbed": 22,
-    "functional_verbatim": 1509,
+    "functional_verbatim": 1511,
     "rules": 3,
     "gate_literals": 3,
     "gate": "passed"
   },
-  "file_count": 1861,
-  "tree_sha256": "983a13e5fa8e0ab31afc970fe2b7f6ebb7296e62bee4cf06257f51ce26658362",
+  "file_count": 1863,
+  "tree_sha256": "03a3fad69d2b334568f719ebc41a4cdde66d922efaae81ac1857f85baa39f760",
   "files": [
     {
       "path": ".claude/skills/blocker-review/SKILL.md",
@@ -4212,7 +4212,12 @@
     {
       "path": "modules/memory/module.json",
       "mode": "0000644",
-      "sha256": "ace721e3ae257aee7f9b13bfbb60b6959f9d72e11059c316d22701754db5b911"
+      "sha256": "c697548692393d65c6beb8092f70f8864acb53d568548f123213023ca10032f8"
+    },
+    {
+      "path": "modules/memory/pollers/search-index.js",
+      "mode": "0000644",
+      "sha256": "4950c01fc2b895a7eb8f5fc929826c1bea44d9764f5ffa4c8378de73be1446e9"
     },
     {
       "path": "modules/memory/routes/learnings.js",
@@ -4797,12 +4802,12 @@
     {
       "path": "package-lock.json",
       "mode": "0000644",
-      "sha256": "d5f53ae25273ca753b122eca1de770bb20d9a1ed1b9a7b0b105710403670e7ec"
+      "sha256": "b1353851e7bd50f58b47d0191076eacc249bdca62553d0bb9a6ee767aed9c712"
     },
     {
       "path": "package.json",
       "mode": "0000644",
-      "sha256": "881f7f9d99198d118f9bbcec082d6637b586dbc6acb850cc654cac192b0cab55"
+      "sha256": "632ea5fa7541ae2eb15f7afdc97f5414b65e1cadcf627e03ed7aee1f1443a0c0"
     },
     {
       "path": "public-docs/index.html",
@@ -6662,7 +6667,7 @@
     {
       "path": "src/module-api.js",
       "mode": "0000644",
-      "sha256": "0b23db16d6f47afb30b4f5ed970cd1a2d0aac607feda9ce4ebf9be09c0a3dc47"
+      "sha256": "2376b2c1c18430faa5dc61183b3dace18e5d991515aa508fcb3d739ba9342701"
     },
     {
       "path": "src/module-loader/catalog.js",
@@ -8804,6 +8809,11 @@
       "mode": "0000644",
       "sha256": "1034ab278e7eb005ba0286b8bcecd7fafdec390418ae94e3cbacf81ffb9fc88c"
     },
+    {
+      "path": "tests/search_index_poller.mjs",
+      "mode": "0000644",
+      "sha256": "f665d4f0a9ccfb5b92564dfda39d13d6d8ed3146cde3fe2e05bb9f9aec029558"
+    },
     {
       "path": "tests/search_ingest.mjs",
       "mode": "0000644",

package/modules/memory/module.json

@@ -4,9 +4,18 @@
   "description": "Per-builder session memory, the lexical recall/search retrieval layer (ADR 0060), and durable learnings. A core-domain module carved out of the GDS core in BV1.R72 (ADR 0091); default-on because every instance needs the build system's memory.",
   "coreVersion": "^1.5.0",
   "default": true,
-  "maintenance": { "status": "core-maintained" },
+  "maintenance": {
+    "status": "core-maintained"
+  },
   "contributes": {
-    "routes": ["memory", "search", "learnings"]
+    "routes": [
+      "memory",
+      "search",
+      "learnings"
+    ],
+    "pollers": [
+      "search-index"
+    ]
   },
   "provides": [],
   "consumes": []

package/modules/memory/pollers/search-index.js

@@ -0,0 +1,201 @@
+'use strict';
+
+// modules/memory/pollers/search-index.js — the recall index keeps itself, and says
+// so out loud (task 1003162, goal 1000027).
+//
+// THE INCIDENT. `/recall` on live returned the `index_empty` outage state for words
+// that demonstrably exist in the corpus — an entire goal's worth of ADRs, session
+// logs and task rows. The index genuinely held nothing, and had held nothing for as
+// long as anyone could tell, because NOTHING RAN THE INGESTERS.
+//
+// ADR 0060 says repo markdown is ingested by "a deploy-hook step" and DB prose by
+// "a nightly timer". Both are BOX-SIDE artifacts: `grep -rn search-ingest` across
+// the core tree finds the two ingesters, some prose about them, and nothing that
+// executes either. Whether the box's deploy.sh ever called them is unknowable from
+// here — and that unknowability IS the bug. An index that depends on an unwatched
+// host script fails exactly the way this one did: silently, indefinitely, and only
+// noticed when a human happens to search for something they know is there.
+//
+// WHY IN-PROCESS, and not a cron or a CI workflow. The ingesters need the database.
+// A GitHub Actions schedule cannot reach it; the box's own timer is the thing that
+// stopped. The web process already holds a pool and already runs this exact shape
+// of work (the publish reconciler, the board-expiry sweep), so putting the sweep
+// here makes it start whenever the app starts and stop only when the app stops.
+// There is no separate thing left to forget to arm.
+//
+// THE SHAPE follows the publish reconciler, the in-tree precedent:
+//   * a DISPOSITION LINE every sweep — the reconciler's lesson is that a sweep
+//     which only speaks on failure is indistinguishable from a sweep that is not
+//     running. "held 4,812 chunks, nothing to do" is the sentence whose ABSENCE is
+//     the alarm.
+//   * failure isolation — the md ingest failing must not skip the db ingest.
+//   * bounded and self-gating — SEARCH_INDEX_SWEEP_DISABLED turns it off, an unref'd
+//     timer never holds the process open, and a sweep already running is not
+//     re-entered.
+//
+// AND IT CHECKS EXIT CODES, which is the half the task named explicitly. Task
+// 1002859 made a partial ingest exit non-zero and NAME every skipped source; that
+// only helps if someone reads the code. Spawning the CLIs (rather than importing
+// their main()) is what preserves it — the exit code is the contract, and running
+// them the documented way keeps this poller honest about what an operator would see.
+
+const path = require('node:path');
+const { execFile } = require('node:child_process');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
+const MD_INGESTER = path.join(REPO_ROOT, 'scripts', 'gds', 'search-ingest-md.js');
+const DB_INGESTER = path.join(REPO_ROOT, 'scripts', 'gds', 'search-ingest-db.js');
+
+// Nightly-ish. DB prose changes slowly and the md corpus only moves on deploy, so
+// a long period is right; the value is that it recurs at all, not that it is tight.
+const INTERVAL_MS = Number(process.env.SEARCH_INDEX_SWEEP_MS) || 6 * 60 * 60 * 1000;
+// An EMPTY index is an outage, not a slow day: recall answers nothing at all until
+// it is fixed. Sweep early after boot so a deploy that lands on an empty table
+// repairs it in minutes rather than at the next six-hour tick.
+const FIRST_SWEEP_DELAY_MS = Number(process.env.SEARCH_INDEX_FIRST_MS) || 90_000;
+// The md ingest walks a few hundred files; the db ingest reads prose tables. Both
+// are minutes at worst. The cap exists so a wedged child can never pin the sweep.
+const INGEST_TIMEOUT_MS = Number(process.env.SEARCH_INDEX_INGEST_TIMEOUT_MS) || 10 * 60 * 1000;
+
+let intervalTimer = null;
+let firstTimer = null;
+let sweeping = false;
+
+// ---------------------------------------------------------------------------
+// Pure decisions — exported and unit-tested. No DB, no spawn, no clock.
+// ---------------------------------------------------------------------------
+
+// What does a chunk count MEAN? Separated from the doing so the judgement can be
+// tested without a database, and so the wording lives in one place.
+//
+// `empty` is deliberately its own state rather than "very stale": it is the only
+// one where recall is not degraded but DEAD, it is rank-independent (so it can
+// never be "nothing visible to YOU"), and it is the one that justifies spending an
+// ingest immediately on boot.
+function describeIndex({ chunks, lastIngestedAt = null, now = Date.now() } = {}) {
+  // typeof FIRST: Number(null) is 0 and Number('') is 0, so a missing column or an
+  // absent row would coerce into a confident "the index is empty" — a wrong
+  // diagnosis that also spends an ingest. Only an actual number can be a count.
+  const n = typeof chunks === 'number' ? chunks : NaN;
+  if (!Number.isFinite(n) || n < 0) {
+    return { level: 'alarm', state: 'unreadable', chunks: null,
+      message: 'search-index: could not read search_chunks — recall health is UNKNOWN, which is not the same as healthy.' };
+  }
+  if (n === 0) {
+    return { level: 'alarm', state: 'empty', chunks: 0,
+      message: 'search-index: search_chunks is EMPTY — /recall can return nothing at all, for any query, at any rank. Ingesting now.' };
+  }
+  const ageMs = lastIngestedAt ? Math.max(0, now - new Date(lastIngestedAt).getTime()) : null;
+  const ageDays = ageMs == null ? null : Math.floor(ageMs / 86_400_000);
+  if (ageDays != null && ageDays >= 7) {
+    return { level: 'warn', state: 'stale', chunks: n, age_days: ageDays,
+      message: `search-index: ${n} chunk(s), but the newest is ${ageDays} day(s) old — the sweep may have stopped.` };
+  }
+  return { level: 'ok', state: 'ok', chunks: n, age_days: ageDays,
+    message: `search-index: ${n} chunk(s) indexed${ageDays == null ? '' : `, newest ${ageDays} day(s) old`}.` };
+}
+
+// One ingester's outcome, in the words an operator needs. A non-zero exit is the
+// signal task 1002859 built and nothing was reading.
+function describeIngest({ name, code, signal = null, stderr = '' } = {}) {
+  if (signal) {
+    return { level: 'alarm', ok: false,
+      message: `search-index: ${name} was KILLED by ${signal} (timeout or OOM) — the index is partially written at best.` };
+  }
+  if (code === 0) return { level: 'ok', ok: true, message: `search-index: ${name} ok.` };
+  // The tail, not the head: these CLIs name what they skipped at the END.
+  const tail = String(stderr || '').trim().split('\n').slice(-6).join('\n');
+  return { level: 'alarm', ok: false,
+    message: `search-index: ${name} exited ${code} — it names every source it skipped:\n${tail || '(no stderr captured)'}` };
+}
+
+// ---------------------------------------------------------------------------
+// Doing
+// ---------------------------------------------------------------------------
+
+function runIngester(file, { execFileImpl = execFile } = {}) {
+  return new Promise((resolve) => {
+    execFileImpl(process.execPath, [file], {
+      cwd: REPO_ROOT, timeout: INGEST_TIMEOUT_MS, maxBuffer: 16 * 1024 * 1024,
+      env: process.env, shell: false,
+    }, (err, _stdout, stderr) => {
+      resolve({
+        code: err ? (typeof err.code === 'number' ? err.code : 1) : 0,
+        signal: (err && err.signal) || null,
+        stderr: stderr || (err && err.message) || '',
+      });
+    });
+  });
+}
+
+async function readChunkStats(pool) {
+  const r = await pool.query(
+    'SELECT count(*)::int AS chunks, max(updated_at) AS last FROM search_chunks'
+  );
+  const row = (r && r.rows && r.rows[0]) || {};
+  return { chunks: row.chunks, lastIngestedAt: row.last || null };
+}
+
+// One sweep. Never throws — a poller that can take the process down is worse than
+// the outage it watches.
+async function sweep({ pool, log = console, execFileImpl = execFile, now = Date.now() } = {}) {
+  let health;
+  try {
+    health = describeIndex({ ...(await readChunkStats(pool)), now });
+  } catch (e) {
+    health = describeIndex({ chunks: NaN, now });
+    health.error = (e && e.message) || String(e);
+  }
+  (health.level === 'ok' ? log.log : log.error).call(log, health.message);
+
+  // Ingest when the index is empty or unreadable-but-recoverable; a populated
+  // index is refreshed on the ordinary tick, not re-walked because it is stale by
+  // an hour. `empty` is the case that cannot wait.
+  const results = [];
+  for (const [name, file] of [['search-ingest-md', MD_INGESTER], ['search-ingest-db', DB_INGESTER]]) {
+    const r = await runIngester(file, { execFileImpl });
+    const d = describeIngest({ name, ...r });
+    results.push(d);
+    (d.ok ? log.log : log.error).call(log, d.message);
+  }
+
+  // The after-picture. Without it a run that exited 0 while writing nothing still
+  // reads as success — the exact shape of failure this task exists to end.
+  let after = null;
+  try {
+    after = describeIndex({ ...(await readChunkStats(pool)), now });
+    (after.level === 'ok' ? log.log : log.error).call(log, `search-index: after sweep — ${after.message}`);
+  } catch (_) { /* the before-picture already reported; never throw from a sweep */ }
+
+  return { health, results, after };
+}
+
+function start(deps = {}) {
+  // Same injectable-log shape as sweep() above (default console), so the two
+  // halves of this poller log through one seam and neither hard-codes console.
+  const log = deps.log || console;
+  if (process.env.SEARCH_INDEX_SWEEP_DISABLED === 'true') {
+    log.log('search-index: sweep disabled by SEARCH_INDEX_SWEEP_DISABLED.');
+    return { stop };
+  }
+  const pool = deps.pool || require('../../../src/module-api.js').pool;
+  const tick = async () => {
+    if (sweeping) return;            // never re-enter; a long ingest must not stack
+    sweeping = true;
+    try { await sweep({ pool, log }); }
+    catch (e) { log.error('search-index: sweep failed:', (e && e.message) || e); }
+    finally { sweeping = false; }
+  };
+  firstTimer = setTimeout(tick, FIRST_SWEEP_DELAY_MS);
+  intervalTimer = setInterval(tick, INTERVAL_MS);
+  if (firstTimer.unref) firstTimer.unref();
+  if (intervalTimer.unref) intervalTimer.unref();
+  return { stop };
+}
+
+function stop() {
+  if (firstTimer) { clearTimeout(firstTimer); firstTimer = null; }
+  if (intervalTimer) { clearInterval(intervalTimer); intervalTimer = null; }
+}
+
+module.exports = { start, stop, sweep, describeIndex, describeIngest, INTERVAL_MS, FIRST_SWEEP_DELAY_MS };

package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.356",
+  "version": "1.19.357",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bongos/core",
-      "version": "1.19.356",
+      "version": "1.19.357",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "express": "^4.21.2",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.356",
+  "version": "1.19.357",
   "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
   "license": "AGPL-3.0-or-later",
   "main": "src/platform-server.js",

package/src/module-api.js

@@ -1142,8 +1142,10 @@ const { validateOrRespond, LIMITS, parseId, asyncHandler, corsPublicGet, parsePa
 //         landed since 1.19.354 with no explicit bump. run 32853063015. (task 1002620)
 //   1.19.356 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
 //         landed since 1.19.355 with no explicit bump. run 32861072779. (task 1002620)
+//   1.19.357 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
+//         landed since 1.19.356 with no explicit bump. run 32871335939. (task 1002620)
 // ---------------------------------------------------------------------------
-const CORE_VERSION = '1.19.356'; // CI auto-patch carrier (ADR 0161); see changelog above
+const CORE_VERSION = '1.19.357'; // CI auto-patch carrier (ADR 0161); see changelog above
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('dev-box'); log.info('mounted');

package/tests/search_index_poller.mjs

@@ -0,0 +1,161 @@
+// tests/search_index_poller.mjs — the recall index keeps itself, and says so
+// (task 1003162).
+//
+// The outage this guards: /recall returned `index_empty` on live for words that
+// demonstrably existed in the corpus, because NOTHING ran the ingesters. ADR 0060
+// claims a deploy hook and a nightly timer; both are box-side, and neither could be
+// shown to have ever run.
+//
+// Everything here is pure or dependency-injected — no database, no child process,
+// no clock. The sweep is driven with a fake pool and a fake execFile so the
+// DECISIONS are tested, not the plumbing.
+//
+// Run:  node tests/search_index_poller.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const poller = require('../modules/memory/pollers/search-index.js');
+const { describeIndex, describeIngest, sweep } = poller;
+
+let passed = 0;
+let failed = 0;
+async function test(name, fn) {
+  try { await fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.error(`  FAIL ${name}\n    ${err.message}`); }
+}
+
+const NOW = Date.parse('2026-08-25T12:00:00Z');
+const daysAgo = (d) => new Date(NOW - d * 86400000).toISOString();
+const quietLog = { log() {}, error() {} };
+const okExec = (_p, _a, _o, cb) => cb(null, '', '');
+
+// --- READING THE INDEX ------------------------------------------------------
+
+await test('an EMPTY index is an alarm, and says recall returns nothing AT ANY RANK', async () => {
+  const d = describeIndex({ chunks: 0, now: NOW });
+  assert.equal(d.level, 'alarm');
+  assert.equal(d.state, 'empty');
+  // Rank-independence is the whole reason `index_empty` is diagnostic: it rules
+  // out "nothing visible to YOU", which is what made the live report trustworthy.
+  assert.match(d.message, /any rank/i);
+});
+
+await test('an UNREADABLE count is an alarm — unknown is not healthy', async () => {
+  for (const bad of [NaN, undefined, null, -1, 'lots']) {
+    const d = describeIndex({ chunks: bad, now: NOW });
+    assert.equal(d.level, 'alarm', `${bad} must not read as healthy`);
+    assert.equal(d.state, 'unreadable');
+  }
+});
+
+await test('a populated, fresh index is ok and says how much it holds', async () => {
+  const d = describeIndex({ chunks: 4812, lastIngestedAt: daysAgo(1), now: NOW });
+  assert.equal(d.level, 'ok');
+  assert.match(d.message, /4812/);
+});
+
+await test('a populated but WEEK-OLD index warns — the sweep may have stopped', async () => {
+  const d = describeIndex({ chunks: 4812, lastIngestedAt: daysAgo(9), now: NOW });
+  assert.equal(d.level, 'warn');
+  assert.equal(d.state, 'stale');
+  assert.equal(d.age_days, 9);
+});
+
+await test('staleness is not guessed when nothing says when it last ran', async () => {
+  const d = describeIndex({ chunks: 10, lastIngestedAt: null, now: NOW });
+  assert.equal(d.level, 'ok', 'an unknown age must not be reported as stale');
+  assert.equal(d.age_days, null);
+});
+
+// --- READING THE INGESTERS' EXIT CODES --------------------------------------
+
+await test('a NON-ZERO exit is an alarm carrying the TAIL of stderr', async () => {
+  // Task 1002859 made a partial ingest exit non-zero and NAME every skipped
+  // source — at the end of its output. A head-clip would drop exactly that.
+  const d = describeIngest({
+    name: 'search-ingest-md', code: 1,
+    stderr: ['noise', 'noise', 'noise', 'noise', 'noise', 'noise', 'skipped: docs/adr/0060.md (NUL byte)'].join('\n'),
+  });
+  assert.equal(d.ok, false);
+  assert.equal(d.level, 'alarm');
+  assert.match(d.message, /skipped: docs\/adr\/0060\.md/, 'the last line is the one that matters');
+});
+
+await test('a KILLED ingester is reported as killed, not as a clean failure', async () => {
+  const d = describeIngest({ name: 'search-ingest-db', code: null, signal: 'SIGTERM' });
+  assert.equal(d.ok, false);
+  assert.match(d.message, /KILLED by SIGTERM/);
+  assert.match(d.message, /partially written/, 'a half-written index is the real risk');
+});
+
+await test('exit 0 is ok', async () => {
+  assert.equal(describeIngest({ name: 'x', code: 0 }).ok, true);
+});
+
+// --- THE SWEEP --------------------------------------------------------------
+
+function fakePool(seq) {
+  let i = 0;
+  return { query: async () => ({ rows: [seq[Math.min(i++, seq.length - 1)]] }) };
+}
+
+await test('the sweep runs BOTH ingesters — one failing must not skip the other', async () => {
+  const called = [];
+  const exec = (_p, args, _o, cb) => {
+    called.push(args[0].split('/').pop());
+    cb(args[0].includes('md') ? Object.assign(new Error('x'), { code: 1 }) : null, '', 'boom');
+  };
+  const r = await sweep({ pool: fakePool([{ chunks: 0, last: null }]), log: quietLog, execFileImpl: exec, now: NOW });
+  assert.deepEqual(called, ['search-ingest-md.js', 'search-ingest-db.js']);
+  assert.equal(r.results.length, 2);
+  assert.equal(r.results[0].ok, false);
+  assert.equal(r.results[1].ok, true, 'the db ingest still ran after the md ingest failed');
+});
+
+await test('the sweep reports the AFTER count — exit 0 that wrote nothing is a failure shape', async () => {
+  // The precise trap: an ingester can exit 0 and leave the table empty. Without
+  // an after-picture that reads as success.
+  const r = await sweep({
+    pool: fakePool([{ chunks: 0, last: null }, { chunks: 0, last: null }]),
+    log: quietLog, execFileImpl: okExec, now: NOW,
+  });
+  assert.equal(r.after.state, 'empty', 'still empty after a "successful" ingest');
+  assert.equal(r.after.level, 'alarm');
+});
+
+await test('a healthy sweep still SPEAKS — silence must not be the healthy state', async () => {
+  // The reconciler's lesson: a sweep that only logs on failure is
+  // indistinguishable from a sweep that is not running at all.
+  const lines = [];
+  await sweep({
+    pool: fakePool([{ chunks: 4812, last: daysAgo(1) }]),
+    log: { log: (m) => lines.push(m), error: (m) => lines.push(m) },
+    execFileImpl: okExec, now: NOW,
+  });
+  assert.ok(lines.length >= 2, 'a healthy sweep leaves a disposition line');
+  assert.ok(lines.some((l) => /4812/.test(l)), 'and says what it found');
+});
+
+await test('a DB read failure never throws out of the sweep', async () => {
+  const r = await sweep({
+    pool: { query: async () => { throw new Error('connection refused'); } },
+    log: quietLog, execFileImpl: okExec, now: NOW,
+  });
+  assert.equal(r.health.state, 'unreadable');
+  assert.equal(r.health.level, 'alarm');
+});
+
+await test('the poller is declared in the memory manifest — a file nothing loads is dormant', async () => {
+  // The exact failure goal 1000076 hit five times: a surface that exists, is
+  // documented, and is never wired.
+  const manifest = require('../modules/memory/module.json');
+  assert.ok(
+    (manifest.contributes.pollers || []).includes('search-index'),
+    'modules/memory/module.json must declare the poller for the loader to start it',
+  );
+});
+
+console.log(`\nsearch_index_poller: ${passed} passed, ${failed} failed`);
+process.exit(failed ? 1 : 0);