@bongos/core 1.21.175 → 1.21.177

package/scripts/bongos/seam-wiring-guard.js

@@ -0,0 +1,275 @@
+// scripts/bongos/seam-wiring-guard.js — Check 33: a declared contributes.<seam> that
+// nothing consumes (task 1003819; successor to abandoned task 1002464).
+//
+// THE GAP THIS CLOSES. A module declares what it contributes in module.json.
+// src/module-seams.js `verifyPortsSatisfied` checks the OPPOSITE direction — every
+// `consumes` port has exactly one provider among the enabled modules — and
+// src/module-loader/manifest-schema.js `validateManifest` checks the SHAPE of
+// `contributes` (a closed key set, so a typo'd KEY errors). Neither asks whether a
+// declared entry is actually WIRED. So a module can declare `contributes.pollers:
+// ["ghost"]` with no modules/<key>/pollers/ghost.js and boot stays silent: the
+// loader's startModulePollers treats a missing poller file as declarative-only and
+// skips it, hallWidgetScripts skips a uiSection with no hall-widget.js, and
+// moduleWebSurfaces skips a surface whose dir was never shipped. Every one of those
+// skips is correct behaviour for a REAL declarative-only entry, and indistinguishable
+// from a typo or a rename that left the manifest behind.
+//
+// WHAT "CONSUMED" MEANS. Each seam has one consumer in the tree, and each consumer
+// looks for one artifact at a fixed path. SEAM_CONSUMERS below records that pairing —
+// seam → the artifact its consumer resolves — so this guard asks the same question
+// the consumer asks rather than inventing a second convention that could drift from
+// it. A seam with NO consumer anywhere carries `consumer: null`, and every
+// declaration of it is reported: that is the strongest form of the defect.
+//
+// WHY A CI GATE AND NOT A BOOT REFUSAL (ADR 0289). `verifyEnabledPorts` throws at
+// boot because an unprovided port otherwise becomes a runtime throw on a request
+// path, in a process that reported healthy — refusing early is strictly safer. For
+// every seam BUT `routes`, an unconsumed contribution is not that shape: the consumer
+// skips it gracefully, nothing crashes later, and the instance runs correctly minus
+// one feature. Adding a production start-up failure for a manifest-hygiene defect
+// would trade a real outage risk for a cosmetic one.
+//
+// `routes` is the exception and it argues the SAME way: mountModuleRoutes `require`s
+// the factory with no try/catch, so a missing route file already takes the boot down
+// today. There the guard is not a new gate at all — it moves an existing crash from
+// production start-up to the build, which is strictly earlier and strictly cheaper.
+// So CI covers both cases: the seams that fail silently get a voice, and the one that
+// fails loudly gets to fail before it reaches a server.
+//
+// Exported for scripts/bongos/fitness.js (checkSeamWiring) and for the unit suite, which
+// drives scanSeamWiring() over synthetic manifests with an injected `exists` so every
+// case is hermetic — the scanScopeMapSource / scanDeterministicBody precedent, where
+// the pure scanner is the tested surface.
+
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+// The SAME enumerator + labeller validateManifest uses for the exemption cross-check,
+// imported rather than re-declared: two copies would drift, and the drift would land
+// on whether an exemption is honoured (task 1003819).
+const { declaredEntries, entryLabel } = require('../../src/module-loader/manifest-schema.js');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+// Is `child` inside `parent`? Both are repo-relative-or-absolute; normalised before
+// comparing so a '..' segment cannot pass by spelling.
+function within(parent, child) {
+  const a = path.resolve(REPO_ROOT, parent);
+  const b = path.resolve(REPO_ROOT, child);
+  const r = path.relative(a, b);
+  return r !== '' && !r.startsWith('..') && !path.isAbsolute(r);
+}
+
+// A repo-relative, forward-slashed path for an error message.
+function rel(p) {
+  if (!p) return '(no artifact path)';
+  const r = path.relative(REPO_ROOT, path.isAbsolute(p) ? p : path.join(REPO_ROOT, p));
+  return r.split(path.sep).join('/');
+}
+
+// seam → how its consumer resolves a declared entry to an on-disk artifact.
+//   consumer   the function that actually reads it, named so a reader can go check
+//   kind       'file' | 'dir' | 'registry' — what must exist
+//   artifact   (moduleDir, entry) → path, or null when the seam is per-MODULE rather
+//              than per-entry (uiSections ships ONE widget however many it names)
+//   alternates optional extra paths that also count as wired
+const SEAM_CONSUMERS = {
+  routes: {
+    consumer: 'src/module-loader/loader.js mountModuleRoutes',
+    kind: 'file',
+    artifact: (dir, entry) => path.join(dir, 'routes', `${entry}.js`),
+  },
+  pollers: {
+    consumer: 'src/module-loader/loader.js startModulePollers',
+    kind: 'file',
+    artifact: (dir, entry) => path.join(dir, 'pollers', `${entry}.js`),
+  },
+  rooms: {
+    // Allowed by the manifest schema since the doorway, but nothing in core reads
+    // contributes.rooms — the realtime seam was never wired to the loader. No module
+    // declares it today; if one ever does, this guard says so rather than letting the
+    // declaration read as though it did something.
+    consumer: null,
+    kind: 'file',
+    artifact: (dir, entry) => path.join(dir, 'rooms', `${entry}.js`),
+  },
+  uiSections: {
+    // Per-MODULE, not per-entry: hallWidgetScripts looks for one widget file per page
+    // however many sections the module names — hall-widget.js for the index, and since
+    // task 1004301 task-widget.js for a task's page. Either one wires the declaration.
+    consumer: 'src/module-loader/loader.js hallWidgetScripts',
+    kind: 'file',
+    artifact: (dir) => path.join(dir, 'public', 'hall-widget.js'),
+    alternates: (dir) => [path.join(dir, 'public', 'task-widget.js')],
+  },
+  skills: {
+    // moduleSkillSources resolves a declared name against the module's OWN tree or
+    // its vendor/ subdir. A name that exists only in the core .claude/skills/ is
+    // skipped by it (`if (!src) continue`), so it is unconsumed BY THE MANIFEST —
+    // a legitimate declarative-only case, and what ui-design's exemption records.
+    //
+    // The artifact is the SKILL.md, not the directory: moduleSkillSources tests
+    // `fs.existsSync(path.join(p, 'SKILL.md'))`, so a skills/<name>/ dir without one
+    // is NOT a skill to it. Probing the bare directory would call such an entry wired
+    // while the real consumer skipped it — this guard's own defect, on the seam with
+    // the most declared entries.
+    consumer: 'scripts/bongos/claude-materialize.js moduleSkillSources',
+    kind: 'file',
+    artifact: (dir, entry) => path.join(dir, 'skills', entry, 'SKILL.md'),
+    alternates: (dir, entry) => [path.join(dir, 'skills', 'vendor', entry, 'SKILL.md')],
+  },
+  disciplines: {
+    consumer: 'scripts/bongos/discipline-modes.json (read by src/modules.js + claim.js)',
+    kind: 'registry',
+    artifact: () => null,
+  },
+  webSurfaces: {
+    consumer: 'src/module-loader/loader.js moduleWebSurfaces',
+    kind: 'dir',
+    // `entry` is the surface OBJECT here, not a string.
+    artifact: (dir, entry) => (entry && typeof entry.dir === 'string' ? path.join(dir, entry.dir) : null),
+  },
+  migrations: {
+    consumer: 'scripts/migrate.sh (the module-migration runner)',
+    kind: 'dir',
+    artifact: (dir) => path.join(dir, 'migrations'),
+  },
+};
+
+// Default existence probe — `kind` decides file-vs-directory. A relative path is
+// resolved against the repo root so a manifest's `dir` may be either.
+function fsExists(p, kind) {
+  try {
+    const st = fs.statSync(path.isAbsolute(p) ? p : path.join(REPO_ROOT, p));
+    return kind === 'dir' ? st.isDirectory() : st.isFile();
+  } catch {
+    return false;
+  }
+}
+
+// THE SCANNER. Pure over its inputs (the only I/O is the injected `exists`), so the
+// unit suite drives every case with synthetic manifests and no tmp dirs.
+//
+//   modules       [{ key, dir, manifest }] — the loader's `discovered().loaded` shape
+//   exists        (path, kind) => boolean — defaults to the real filesystem
+//   registryKeys  Set of valid `disciplines` entries, or null to skip that seam
+//
+// Returns { ok, findings[], wired[], declarative[] }. `wired` and `declarative` are
+// populated on the SUCCESS path too, deliberately: a guard that reports nothing when
+// it passes leaves no evidence it ran, so the caller prints the counts either way.
+function scanSeamWiring(modules = [], { exists = fsExists, registryKeys = null } = {}) {
+  const findings = [];
+  const wired = [];
+  const declarative = [];
+
+  for (const { key, dir, manifest } of modules) {
+    const contributes = (manifest && manifest.contributes) || {};
+    const exempt = (manifest && manifest.declarativeSeams) || {};
+    const isExempt = (seam, label) => Array.isArray(exempt[seam]) && exempt[seam].includes(label);
+
+    for (const [seam, value] of Object.entries(contributes)) {
+      const spec = SEAM_CONSUMERS[seam];
+      if (!spec) continue; // an unknown seam key is validateManifest's error, not ours
+
+      for (const entry of declaredEntries(seam, value)) {
+        const label = entryLabel(seam, entry);
+
+        if (spec.consumer === null) {
+          findings.push(`module "${key}" declares contributes.${seam}: "${label}", but NOTHING in the tree consumes the ${seam} seam at all — the declaration does nothing`);
+          continue;
+        }
+
+        // A 'registry' seam is satisfied by membership in a roster, not by a file.
+        if (spec.kind === 'registry') {
+          if (registryKeys && !registryKeys.has(label)) {
+            findings.push(`module "${key}" declares contributes.${seam}: "${label}", but ${spec.consumer} has no such entry`);
+          } else {
+            wired.push({ key, seam, entry: label, via: spec.consumer });
+          }
+          continue;
+        }
+
+        const primary = spec.artifact(dir, entry);
+        const candidates = [primary, ...(spec.alternates ? spec.alternates(dir, entry) : [])].filter(Boolean);
+        // An entry carrying path separators or '..' would aim the probe outside
+        // modules/<key>/. It is only an existence test over repo-committed content, so
+        // the blast radius is an on/off answer — but a declaration that reaches out of
+        // its own module is a finding in its own right, not something to resolve.
+        const escapes = candidates.find((p) => !within(dir, p));
+        if (escapes) {
+          findings.push(`module "${key}" declares contributes.${seam}: "${label}", which resolves to ${rel(escapes)} — outside the module's own directory. A contributes entry names something the module ships, so it may not contain a path separator or ".."`);
+          continue;
+        }
+        const found = candidates.some((p) => exists(p, spec.kind));
+
+        if (found) {
+          // A STALE exemption is the same rot in the other direction: the artifact
+          // exists now, so the entry IS wired and the exemption reads as though a
+          // real gap were sanctioned. Fail it rather than let the list quietly lie.
+          if (isExempt(seam, label)) {
+            findings.push(`module "${key}" exempts ${seam} "${label}" as declarative-only, but ${rel(primary)} exists and ${spec.consumer} wires it — drop the stale declarativeSeams.${seam} entry`);
+          } else {
+            wired.push({ key, seam, entry: label, via: spec.consumer });
+          }
+          continue;
+        }
+
+        if (isExempt(seam, label)) {
+          declarative.push({ key, seam, entry: label });
+          continue;
+        }
+
+        findings.push(`module "${key}" declares contributes.${seam}: "${label}", but ${rel(primary)} does not exist — ${spec.consumer} silently skips it. Ship the artifact, drop the declaration, or record it in declarativeSeams.${seam} if it is declarative-only on purpose`);
+      }
+    }
+  }
+
+  return { ok: findings.length === 0, findings, wired, declarative };
+}
+
+// The roster `contributes.disciplines` is checked against. Returns null when the file
+// is unreadable, which makes the disciplines seam UNCHECKED rather than failed — a
+// missing registry is its own check's problem (fitness.js role packs), not this one's.
+function disciplineKeys() {
+  try {
+    const raw = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'scripts', 'bongos', 'discipline-modes.json'), 'utf8'));
+    return new Set(Object.keys(raw.modes || {}));
+  } catch {
+    return null;
+  }
+}
+
+// The fitness-check wrapper. Discovers every module.json through the loader — the same
+// discovery the running server uses, so the guard cannot check a different module set
+// than the one that boots.
+function checkSeamWiring() {
+  const name = 'declared module seams are consumed (task 1003819)';
+  let modules = [];
+  try {
+    modules = require('../../src/module-loader/loader.js').discoverModules().loaded;
+  } catch (e) {
+    return {
+      name,
+      ok: true,
+      hardFail: false,
+      violations: [],
+      warnings: [`module discovery failed: ${e && e.message ? e.message : e}`],
+      note: 'module discovery unavailable — skipped.',
+    };
+  }
+
+  const { ok, findings, wired, declarative } = scanSeamWiring(modules, { registryKeys: disciplineKeys() });
+  const seams = [...new Set(wired.map((w) => w.seam))].sort();
+  return {
+    name,
+    ok,
+    hardFail: !ok,
+    violations: findings,
+    warnings: [],
+    // Printed on the PASS path too — the wired set is the evidence the guard ran.
+    note: `${modules.length} module(s): ${wired.length} declared seam entr(ies) WIRED across ${seams.length} seam(s) (${seams.join(', ') || 'none'}), ${declarative.length} recorded declarative-only. A declared contributes.<seam> whose consumer's artifact is absent is a silent no-op (ADR 0289).`,
+  };
+}
+
+module.exports = { checkSeamWiring, scanSeamWiring, SEAM_CONSUMERS, declaredEntries, entryLabel, disciplineKeys };

package/scripts/bongos/search-ingest-db.js

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+// scripts/bongos/search-ingest-db.js — DB-prose ingester for the GDS recall layer
+// (task 948, V4.R10 / ADR 0060 §4). The nightly companion to the repo-md
+// ingester (#947): indexes the prose that lives in the DB, not in git — task
+// descriptions + value summaries, learnings, session-log notes, idea inbox,
+// grade notes, done-when criteria, blockers, per-builder memory — plus recent
+// git commit subjects, so "has a parallel/past session already done this?" is
+// answerable (the redundant-work case). source_kind='db'.
+//
+// Runs on a NIGHTLY systemd timer on the droplet (infra/search-index-db.timer,
+// modeled on gds-db-backup) where the GDS Postgres is on the local socket. Idempotent
+// + delta by content_hash (re-runs are cheap), prunes vanished db sources.
+//
+// SECURITY (ADR 0060 §6 / ADR 0016 / ADR 0022):
+//   * builder_memory rows are OWNER-SCOPED (owner_builder_id = the builder), so a
+//     builder's private memory only ever surfaces to that builder.
+//   * security_sensitive tasks are tagged min_rank=metic.
+//   * vulnerability_reports + security_reports are EXCLUDED entirely — raw
+//     security findings don't belong in a broadly-searchable index.
+//   * session-derived + memory text is run through ship.js scrubSecrets (shape +
+//     high-entropy redaction) before storage — the index must never hold a
+//     leaked credential.
+//
+// Usage:
+//   node scripts/bongos/search-ingest-db.js            # ingest (writes)
+//   node scripts/bongos/search-ingest-db.js --dry-run  # query + report counts, no writes (needs DB)
+
+const crypto = require('node:crypto');
+const { execFileSync } = require('node:child_process');
+const path = require('node:path');
+const { chunkMarkdown } = require('../../modules/memory/search-chunker'); // carved in BV1.R72 (ADR 0091 §2)
+const { scrubSecrets } = require('./ship');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+function requirePool() {
+  // Pool lives in the root src/db.js (see search-ingest-md.js requirePool).
+  return require('../../src/bongos/pool').pool;
+}
+
+function sha256(s) {
+  return crypto.createHash('sha256').update(s, 'utf8').digest('hex');
+}
+
+// Turn one DB record's prose into chunk rows. The title becomes an H1 so the
+// chunker's breadcrumb + FTS title-weighting work the same as for docs. `scrub`
+// runs the body through scrubSecrets first (session/memory/grade text).
+function buildRows({ sourceRef, title, body, minRank = 'xenos', ownerBuilderId = null, scrub = false }) {
+  let text = typeof body === 'string' ? body : '';
+  if (scrub) text = scrubSecrets(text);
+  if (!text || !text.trim()) return [];
+  const content = (title ? `# ${title}\n\n` : '') + text;
+  const { chunks } = chunkMarkdown({ path: sourceRef, content });
+  return chunks.map((c) => ({
+    source_ref: sourceRef,
+    chunk_index: c.chunk_index,
+    heading_path: c.heading_path,
+    title: title || sourceRef,
+    body: c.body,
+    min_rank: minRank,
+    owner_builder_id: ownerBuilderId,
+    content_hash: sha256(c.body),
+  }));
+}
+
+// The prose sources. Each maps DB rows → buildRows() args. Pure given rows, so
+// the mapping is unit-testable without a DB (the queries run in main()).
+// vulnerability_reports + security_reports are deliberately ABSENT (excluded).
+const SOURCES = [
+  {
+    name: 'tasks',
+    query: `SELECT id, title, description, value_summary, security_sensitive FROM tasks`,
+    map: (r) => buildRows({
+      sourceRef: `task:${r.id}`,
+      title: r.title,
+      body: [r.description, r.value_summary].filter(Boolean).join('\n\n'),
+      minRank: r.security_sensitive ? 'metic' : 'xenos',
+    }),
+  },
+  { name: 'learnings', query: `SELECT id, title, body_md FROM learnings`,
+    map: (r) => buildRows({ sourceRef: `learning:${r.id}`, title: r.title, body: r.body_md }) },
+  { name: 'idea_inbox', query: `SELECT id, title, body_md FROM idea_inbox`,
+    map: (r) => buildRows({ sourceRef: `idea:${r.id}`, title: r.title, body: r.body_md }) },
+  { name: 'done_when_criteria', query: `SELECT id, version_id, criterion_id, criterion_md FROM done_when_criteria`,
+    map: (r) => buildRows({ sourceRef: `criterion:${r.id}`, title: `Criterion ${r.criterion_id} · ${r.version_id}`, body: r.criterion_md }) },
+  { name: 'blockers', query: `SELECT id, title, body_md, resolution_note FROM blockers`,
+    map: (r) => buildRows({ sourceRef: `blocker:${r.id}`, title: r.title, body: [r.body_md, r.resolution_note].filter(Boolean).join('\n\n') }) },
+  // Session-derived prose → SCRUB.
+  { name: 'session_logs', query: `SELECT id, task_id, notes_md FROM session_logs WHERE notes_md IS NOT NULL`,
+    map: (r) => buildRows({ sourceRef: `session_log:${r.id}`, title: `Session log ${r.id}${r.task_id ? ` · task ${r.task_id}` : ''}`, body: r.notes_md, scrub: true }) },
+  { name: 'task_grades', query: `SELECT id, task_id, notes_md FROM task_grades WHERE notes_md IS NOT NULL`,
+    map: (r) => buildRows({ sourceRef: `grade:${r.id}`, title: `Grade · task ${r.task_id}`, body: r.notes_md, scrub: true }) },
+  // Speciality documents → OWNER-SCOPED unless the speciality is public.
+  //
+  // The knowledge an expert trained INTO a speciality (task 1003855). This source
+  // is what makes the injected contract's "search it with /recall" true rather
+  // than aspirational.
+  //
+  // SCOPING ERRS TOWARD UNDER-SHARING. This index expresses only two axes —
+  // min_rank and owner_builder_id — and a 'project' speciality is neither: it is
+  // visible to a set of builders the index cannot name. So only a PUBLIC
+  // speciality's documents are corpus-wide; private AND project both fall back to
+  // owner-scoped. A project member therefore sees the speciality and can adopt it,
+  // but does not get its documents in /recall until it is public. That is a
+  // smaller answer than the sharing rule allows, which is the right direction for
+  // an index to be wrong in — the alternative leaks one project's material to
+  // every builder on the instance.
+  { name: 'specialities_documents',
+    // A row is indexed ONLY if it is public (corpus-wide, deliberately) or has a
+    // real builder owner to scope it to. The WHERE clause is the first of two
+    // guards: a project-owned, non-public speciality never leaves Postgres.
+    query: `SELECT d.id, d.title, d.body_md, s.name AS speciality_name,
+                   s.visibility, s.owner_builder_id
+              FROM specialities_documents d JOIN specialities_catalog s ON s.id = d.speciality_id
+             WHERE s.visibility = 'public' OR s.owner_builder_id IS NOT NULL`,
+    map: (r) => {
+      // The second guard, and NOT redundant: buildRows treats a null owner as
+      // corpus-wide, so any future widening of the query above would silently
+      // publish private material. The mapper refuses to emit an unscoped chunk
+      // for anything that is not public, whatever the query hands it.
+      const isPublic = r.visibility === 'public';
+      if (!isPublic && !r.owner_builder_id) return [];
+      return buildRows({
+        sourceRef: `speciality_doc:${r.id}`,
+        title: `${r.speciality_name} · ${r.title}`,
+        body: r.body_md,
+        ownerBuilderId: isPublic ? null : r.owner_builder_id,
+      });
+    } },
+  // Per-builder memory → OWNER-SCOPED + scrubbed.
+  { name: 'builder_memory', query: `SELECT builder_id, path, content FROM builder_memory`,
+    map: (r) => buildRows({ sourceRef: `memory:${r.builder_id}:${r.path}`, title: r.path, body: r.content, ownerBuilderId: r.builder_id, scrub: true }) },
+];
+
+// Recent git commit subjects — the "did a recent session already touch this?"
+// signal. Best-effort (no git / shallow clone → skip).
+function recentCommitRows(limit = 80) {
+  let out;
+  try {
+    out = execFileSync('git', ['-C', REPO_ROOT, 'log', `-${limit}`, '--no-merges', '--format=%H%x09%s'], { encoding: 'utf8' });
+  } catch { return []; }
+  const rows = [];
+  for (const line of out.split('\n')) {
+    const tab = line.indexOf('\t');
+    if (tab < 0) continue;
+    const sha = line.slice(0, tab).trim();
+    const subject = line.slice(tab + 1).trim();
+    if (!sha || !subject) continue;
+    rows.push(...buildRows({ sourceRef: `commit:${sha.slice(0, 12)}`, title: subject, body: subject }));
+  }
+  return rows;
+}
+
+async function ingestRows(client, rows, sourceRef) {
+  const stats = { inserted: 0, updated: 0, unchanged: 0, pruned: 0 };
+  for (const r of rows) {
+    const res = await client.query(
+      `INSERT INTO search_chunks
+         (source_kind, source_ref, chunk_index, heading_path, title, body, min_rank, owner_builder_id, content_hash, updated_at)
+       VALUES ('db', $1, $2, $3, $4, $5, $6, $7, $8, now())
+       ON CONFLICT (source_kind, source_ref, chunk_index) DO UPDATE
+         SET heading_path = EXCLUDED.heading_path, title = EXCLUDED.title, body = EXCLUDED.body,
+             min_rank = EXCLUDED.min_rank, owner_builder_id = EXCLUDED.owner_builder_id,
+             content_hash = EXCLUDED.content_hash, updated_at = now()
+         WHERE search_chunks.content_hash IS DISTINCT FROM EXCLUDED.content_hash
+            OR search_chunks.min_rank IS DISTINCT FROM EXCLUDED.min_rank
+            OR search_chunks.owner_builder_id IS DISTINCT FROM EXCLUDED.owner_builder_id
+       RETURNING (xmax = 0) AS inserted`,
+      [sourceRef, r.chunk_index, r.heading_path, r.title, r.body, r.min_rank, r.owner_builder_id, r.content_hash]
+    );
+    if (res.rowCount === 0) stats.unchanged += 1;
+    else if (res.rows[0].inserted) stats.inserted += 1;
+    else stats.updated += 1;
+  }
+  const prune = await client.query(
+    `DELETE FROM search_chunks WHERE source_kind='db' AND source_ref=$1 AND chunk_index >= $2`,
+    [sourceRef, rows.length]
+  );
+  stats.pruned = prune.rowCount || 0;
+  return stats;
+}
+
+// task 1002859 (panel finding): bound the driver's error text before it reaches a
+// log. A Postgres error message is unbounded upstream text and CAN embed row
+// content (a constraint violation quotes the offending key), and these logs are
+// deploy/cron sinks that get shipped around. One line, capped, is enough to
+// identify the fault; the source_ref beside it is what actually locates it. The
+// DB corpus makes this matter more than it looks — its prose is session-log and
+// memory-derived text the file elsewhere treats as scrub-sensitive.
+const ERR_LOG_MAX = 300;
+function errLine(e) {
+  const raw = e && e.message ? String(e.message) : String(e);
+  const flat = raw.replace(/\s+/g, ' ').trim();
+  return flat.length > ERR_LOG_MAX ? `${flat.slice(0, ERR_LOG_MAX)}… (truncated)` : flat;
+}
+
+async function main() {
+  const dryRun = process.argv.includes('--dry-run');
+  const pool = requirePool();
+  const client = await pool.connect();
+  const totals = { inserted: 0, updated: 0, unchanged: 0, pruned: 0, deleted: 0, sources: 0 };
+  const failures = []; // task 1002859 — per-source casualties, named at the end
+  const presentRefs = [];
+  try {
+    // Gather every source's rows first (read-only), grouped by source_ref.
+    const byRef = new Map(); // source_ref -> rows[]
+    const add = (rows) => {
+      for (const row of rows) {
+        if (!byRef.has(row.source_ref)) byRef.set(row.source_ref, []);
+        byRef.get(row.source_ref).push(row);
+      }
+    };
+    for (const src of SOURCES) {
+      const { rows } = await client.query(src.query);
+      for (const rec of rows) add(src.map(rec));
+    }
+    add(recentCommitRows());
+    for (const ref of byRef.keys()) presentRefs.push(ref);
+    totals.sources = byRef.size;
+    const totalChunks = [...byRef.values()].reduce((n, a) => n + a.length, 0);
+    console.log(`search-ingest-db: ${byRef.size} db source(s), ${totalChunks} chunk(s)`);
+
+    if (dryRun) {
+      console.log('--dry-run: no writes.');
+      const owned = [...byRef.values()].flat().filter((r) => r.owner_builder_id != null).length;
+      const metic = [...byRef.values()].flat().filter((r) => r.min_rank === 'metic').length;
+      console.log(`  owner-scoped chunks: ${owned}  ·  metic-tagged chunks: ${metic}`);
+      client.release(); await pool.end().catch(() => {});
+      return;
+    }
+
+    // task 1002859: ONE TRANSACTION PER SOURCE, not one for the whole run — the
+    // same all-or-nothing shape that made a single bad byte a total recall
+    // outage on the markdown side. DB prose is the MORE exposed of the two: it
+    // carries session-log and transcript-derived text, which is exactly where a
+    // wrong-encoding read introduces a NUL in the first place.
+    for (const [ref, rows] of byRef) {
+      try {
+        await client.query('BEGIN');
+        const s = await ingestRows(client, rows, ref);
+        await client.query('COMMIT');
+        totals.inserted += s.inserted; totals.updated += s.updated;
+        totals.unchanged += s.unchanged; totals.pruned += s.pruned;
+      } catch (e) {
+        await client.query('ROLLBACK').catch(() => {});
+        failures.push({ sourceRef: ref, message: errLine(e) });
+        console.error(`search-ingest-db: SKIPPED ${ref} — ${errLine(e)}`);
+      }
+    }
+    // Delete db chunks whose source no longer exists (a deleted task/idea/etc.).
+    // Own transaction, and SKIPPED after any failure: presentRefs is what we
+    // meant to write, so pruning against it on a partial run could delete a
+    // still-live source's chunks.
+    if (failures.length === 0) {
+      await client.query('BEGIN');
+      const del = await client.query(
+        `DELETE FROM search_chunks WHERE source_kind='db' AND NOT (source_ref = ANY($1::text[]))`,
+        [presentRefs]
+      );
+      totals.deleted = del.rowCount || 0;
+      await client.query('COMMIT');
+    } else {
+      console.error(`search-ingest-db: vanished-source prune SKIPPED — ${failures.length} source(s) failed, so presentRefs is incomplete`);
+    }
+  } catch (e) {
+    await client.query('ROLLBACK').catch(() => {});
+    console.error('search-ingest-db: FAILED, rolled back:', e.message);
+    process.exitCode = 1;
+    client.release(); await pool.end().catch(() => {});
+    return;
+  }
+  client.release();
+  await pool.end().catch(() => {});
+  console.log(`search-ingest-db: inserted=${totals.inserted} updated=${totals.updated} unchanged=${totals.unchanged} pruned=${totals.pruned} deleted=${totals.deleted} across ${totals.sources} sources`);
+  if (failures.length > 0) {
+    // NON-SILENT: this runs best-effort from a scheduled job, so a non-zero exit
+    // is the only signal that reaches a log a human reads. Name every casualty.
+    console.error(`search-ingest-db: ${failures.length} source(s) FAILED and were skipped:`);
+    for (const f of failures) console.error(`  ${f.sourceRef}: ${f.message}`);
+    process.exitCode = 1;
+  }
+}
+
+if (require.main === module) {
+  main().catch((e) => { console.error('search-ingest-db: error', e); process.exit(1); });
+}
+
+module.exports = { buildRows, recentCommitRows, SOURCES, requirePool };