@bongos/core 1.17.0

package/tests/scrubber_corpus.mjs

@@ -0,0 +1,257 @@
+// tests/scrubber_corpus.mjs
+//
+// Transcript-grade secret-scrubber corpus (task 1003). This is the HARD gate
+// the full-transcript capture task depends on: a planted-secret corpus where
+// every secret MUST be fully redacted, plus a benign corpus that must survive
+// (recall-first, but not so blunt it nukes ordinary prose/paths/identifiers).
+//
+// Covers scrubSecrets (ship.js) end-to-end + its exported internals
+// (shannonEntropy, looksHighEntropySecret, collectEnvSecrets). Pure + DB-free +
+// network-free, so it runs in the run-unit-tests gate (ADR 0042) and standalone.
+//
+// NOTE on fixtures: every secret-SHAPED string is assembled at RUNTIME
+// (concatenation / a deterministic PRNG), so no literal secret pattern ever
+// appears in this file's source. That keeps the pre-commit guard (.husky/
+// pre-commit) and the CI TruffleHog scan clean WITHOUT allowlisting the file —
+// full scan coverage is retained; only the scrubber's behavior is exercised.
+// No Math.random / Date — the PRNG is a fixed-seed LCG so the corpus is
+// deterministic across runs.
+//
+// Run:  node tests/scrubber_corpus.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const { scrubSecrets, shannonEntropy, looksHighEntropySecret, collectEnvSecrets } =
+  require('../scripts/gds/ship.js');
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try {
+    fn();
+    passed++;
+    console.log(`  ok    ${name}`);
+  } catch (err) {
+    failed++;
+    console.log(`  FAIL  ${name}`);
+    console.log(`        ${err.message}`);
+  }
+}
+
+const REDACTED = '<redacted>';
+
+// ---- deterministic high-entropy generators (no Math.random / Date) ----
+// A tiny LCG over a chosen alphabet → reproducible, clearly-random tokens.
+function lcg(seed) {
+  let s = seed >>> 0;
+  return () => {
+    s = (Math.imul(s, 1664525) + 1013904223) >>> 0;
+    return s;
+  };
+}
+function randStr(seed, len, alphabet) {
+  const next = lcg(seed);
+  let out = '';
+  for (let i = 0; i < len; i++) out += alphabet[next() % alphabet.length];
+  return out;
+}
+const B64URL = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
+const HEX = '0123456789abcdef';
+const UPPERNUM = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
+
+// ============================================================
+// POSITIVES — planted secrets. After scrub, the secret VALUE must be gone.
+// Each entry: { name, secret, context } — context embeds the secret in
+// realistic transcript text (a command, a file dump, a header line).
+// ============================================================
+function positives() {
+  const anthropic = 'sk-' + 'ant-api03-' + randStr(11, 40, B64URL);
+  const ghpat = 'gh' + 'p_' + randStr(22, 36, UPPERNUM);
+  const ghfine = 'github' + '_pat_' + randStr(33, 50, B64URL.replace(/-/g, ''));
+  const slack = 'xo' + 'xb-' + '2488' + randStr(44, 20, UPPERNUM);
+  const stripeLive = 'sk' + '_live_' + randStr(55, 24, UPPERNUM);
+  const stripeRk = 'rk' + '_test_' + randStr(66, 24, UPPERNUM);
+  const stripeWh = 'wh' + 'sec_' + randStr(77, 32, B64URL.replace(/[-_]/g, ''));
+  const aws = 'AK' + 'IA' + randStr(88, 16, UPPERNUM);
+  const google = 'AI' + 'za' + randStr(99, 35, B64URL);
+  const gds = 'gds' + '_' + randStr(101, 40, HEX);
+  const pms = 'pms' + '_' + randStr(102, 40, HEX);
+  const apple = 'wxyz' + '-' + 'mnop' + '-' + 'qrst' + '-' + 'uvwa'; // app-specific pw shape
+  const bearer = randStr(103, 36, B64URL.replace(/-/g, '.'));
+  const jwt =
+    'eyJ' + randStr(104, 20, B64URL) + '.' + randStr(105, 30, B64URL) + '.' + randStr(106, 30, B64URL);
+  const pgPass = 'sup3r' + randStr(107, 14, B64URL.replace(/[-_]/g, ''));
+  const pgDsn = 'postgres' + 'ql://' + 'appuser:' + pgPass + '@db.internal:5432/otb';
+  const pemBody = randStr(108, 120, B64URL.replace(/-/g, '+'));
+  const pem =
+    '-'.repeat(5) + 'BEGIN OPENSSH PRIVATE KEY' + '-'.repeat(5) + '\n' +
+    pemBody.match(/.{1,64}/g).join('\n') + '\n' +
+    '-'.repeat(5) + 'END OPENSSH PRIVATE KEY' + '-'.repeat(5);
+  // gds-session.json full file contents (a `cat` of the credential file). The
+  // REAL token shape is crypto.randomBytes(32).toString('hex') — a bare 64-hex
+  // string with NO recognizable prefix (auth.js). This fixture uses that real
+  // shape on purpose, so the corpus proves the unkeyed token is still caught —
+  // by the "token": assignment + the hex entropy rule, not by any prefix match.
+  const sessTok = randStr(110, 64, HEX);
+  const sessionJson =
+    '{\n  "token": "' + sessTok + '",\n  "builder": "lars589",\n  "rank": "archon"\n}';
+  // The same real token as it appears on the wire: a bare-64-hex cookie value +
+  // an Authorization: Bearer header (the two ways it leaves a CLI session).
+  const wireTok = randStr(115, 64, HEX);
+  // env / assignment shapes
+  const envGithub = 'GITHUB_TOKEN=' + 'gh' + 'o_' + randStr(111, 36, UPPERNUM);
+  const envAccess = 'access_token=' + randStr(112, 30, B64URL);
+  const envClient = '"client_secret": "' + randStr(113, 40, HEX) + '"';
+  const envApiKey = 'MY_API_KEY = ' + randStr(114, 28, B64URL);
+  // UNKNOWN shapes — the entropy catch-all is the only thing that can catch these.
+  const unknownB64 = randStr(120, 44, B64URL); // base64url, no known prefix
+  const unknownHex64 = randStr(121, 64, HEX); // sha256-shaped
+  const oauthHex40 = randStr(122, 40, HEX); // GitHub OAuth client secret shape (== a git SHA shape)
+
+  return [
+    { name: 'Anthropic API key', secret: anthropic, context: `claude --api-key ${anthropic} run` },
+    { name: 'GitHub classic PAT', secret: ghpat, context: `git remote set-url origin https://${ghpat}@github.com/x/y` },
+    { name: 'GitHub fine-grained PAT', secret: ghfine, context: `export GH_TOKEN=${ghfine}` },
+    { name: 'Slack bot token', secret: slack, context: `curl -H "token: ${slack}" slack` },
+    { name: 'Stripe live secret key', secret: stripeLive, context: `STRIPE_KEY=${stripeLive}` },
+    { name: 'Stripe restricted key', secret: stripeRk, context: `using ${stripeRk} for tests` },
+    { name: 'Stripe webhook secret', secret: stripeWh, context: `whsec configured: ${stripeWh}` },
+    { name: 'AWS access key id', secret: aws, context: `aws_access_key_id = ${aws}` },
+    { name: 'Google API key', secret: google, context: `&key=${google}` },
+    { name: 'GDS session token (shape)', secret: gds, context: `Authorization: Bearer ${gds}` },
+    { name: 'PMS session token (shape)', secret: pms, context: `cookie pms_session=${pms}` },
+    { name: 'Apple app-specific password', secret: apple, context: `xcrun notarytool --password ${apple}` },
+    { name: 'Bearer auth header', secret: bearer, context: `Authorization: Bearer ${bearer}` },
+    { name: 'JWT', secret: jwt, context: `set-cookie: jwt=${jwt}; HttpOnly` },
+    { name: 'Postgres DSN password', secret: pgPass, context: `DATABASE_URL=${pgDsn}` },
+    { name: 'PEM private key body', secret: pemBody, context: `key:\n${pem}\n(end)` },
+    { name: 'gds-session.json token (real bare-64-hex shape)', secret: sessTok, context: `$ cat ~/.config/otb/gds-session.json\n${sessionJson}` },
+    { name: 'real GDS token on the wire (cookie)', secret: wireTok, context: `Cookie: gds_session=${wireTok}; Path=/` },
+    { name: 'real GDS token on the wire (Bearer header)', secret: wireTok, context: `curl -H "Authorization: Bearer ${wireTok}" https://amazonprimea.com/api/gds/me` },
+    { name: 'env GITHUB_TOKEN value', secret: envGithub.split('=')[1], context: envGithub },
+    { name: 'env access_token value', secret: envAccess.split('=')[1], context: envAccess },
+    { name: 'JSON client_secret value', secret: envClient.match(/"client_secret": "([^"]+)"/)[1], context: envClient },
+    { name: 'env API_KEY value', secret: envApiKey.split('= ')[1], context: envApiKey },
+    { name: 'UNKNOWN base64url token (entropy)', secret: unknownB64, context: `weird = ${unknownB64} done` },
+    { name: 'UNKNOWN 64-hex token (entropy)', secret: unknownHex64, context: `digest ${unknownHex64} computed` },
+    { name: 'UNKNOWN 40-hex (OAuth secret / SHA shape)', secret: oauthHex40, context: `value ${oauthHex40} here` },
+  ];
+}
+
+test('every planted secret is fully redacted from its context', () => {
+  for (const { name, secret, context } of positives()) {
+    const out = scrubSecrets(context, collectEnvSecrets({})); // empty env: pattern+entropy must carry it
+    assert.ok(
+      !out.includes(secret),
+      `LEAK: "${name}" survived the scrubber.\n      in : ${context}\n      out: ${out}`
+    );
+    assert.ok(out.includes(REDACTED), `"${name}" produced no redaction marker: ${out}`);
+  }
+});
+
+test('planted secrets are also redacted when handed in as literal extraSecrets', () => {
+  // The defense-in-depth path: even a value the patterns somehow miss is removed
+  // because the live token / env value is passed as a literal.
+  for (const { name, secret, context } of positives()) {
+    if (secret.length < 8) continue;
+    const out = scrubSecrets(context, [secret]);
+    assert.ok(!out.includes(secret), `LEAK via literal path: "${name}" survived: ${out}`);
+  }
+});
+
+// ============================================================
+// NEGATIVES — benign text that must survive untouched (no over-redaction of
+// ordinary prose / short identifiers / numbers). High-entropy random-looking
+// non-secrets (SHAs, UUIDs, worktree names) are intentionally NOT here: per the
+// recall-first decision they are redacted, and the BFG metrics that matter
+// (token/turn counts, tool names) are structured fields, not free-text.
+// ============================================================
+function negatives() {
+  return [
+    'claimed task 540 and shipped the fix',
+    'the cat sat on the mat today',
+    'ran 57 tasks across 4 criteria in 2 days',
+    'node scripts/gds/session-digest.js --transcript path',
+    'internationalization is a long english word',
+    'see docs/adr/0022-secrets-policy.md for the policy',
+    'merge-mode swept the confirmed queue and deployed',
+    'the vendor sells physical and digital goods in Amazonprimea',
+    'P0: kill the grader bypass and rotate the token soon', // "token" but no value after a separator
+    'git log shows acf3f3c bcb0794 282a155 fc8f4a5', // short SHAs (< 20)
+  ];
+}
+
+test('benign prose and short identifiers survive untouched', () => {
+  for (const s of negatives()) {
+    const out = scrubSecrets(s);
+    assert.equal(out, s, `over-redacted benign text:\n      in : ${s}\n      out: ${out}`);
+  }
+});
+
+test('empty / non-string / short-literal inputs are safe', () => {
+  assert.equal(scrubSecrets(''), '');
+  assert.equal(scrubSecrets(null), null);
+  assert.equal(scrubSecrets(undefined), undefined);
+  assert.equal(scrubSecrets('the cat sat', ['cat']), 'the cat sat'); // <8-char literal ignored
+});
+
+// ============================================================
+// Unit-level: the entropy primitives.
+// ============================================================
+test('shannonEntropy: zero for uniform, high for random', () => {
+  assert.equal(shannonEntropy(''), 0);
+  assert.equal(shannonEntropy('aaaaaaaa'), 0);
+  assert.ok(shannonEntropy(randStr(7, 40, B64URL)) > 4.5, 'random base64 is high-entropy');
+});
+
+test('looksHighEntropySecret: catches random tokens, spares words & short strings', () => {
+  assert.ok(looksHighEntropySecret(randStr(1, 40, B64URL)), 'random base64url → secret');
+  assert.ok(looksHighEntropySecret(randStr(2, 64, HEX)), 'random 64-hex → secret');
+  assert.ok(looksHighEntropySecret(randStr(3, 40, HEX)), '40-hex (OAuth/SHA shape) → secret');
+  assert.equal(looksHighEntropySecret('internationalization'), false, 'plain word → not secret');
+  assert.equal(looksHighEntropySecret('shortlowentropy'), false, '< 20 chars → not secret');
+  assert.equal(looksHighEntropySecret('aaaaaaaaaaaaaaaaaaaaaaaa'), false, 'repetitive → not secret');
+});
+
+test('collectEnvSecrets: gathers secret-shaped env VALUES, skips benign & short', () => {
+  const env = {
+    GDS_TOKEN: 'gds' + '_' + randStr(9, 40, B64URL),
+    MAC_INSTALLER_CERT_P12: randStr(10, 60, B64URL),
+    APPLE_API_ISSUER_ID: randStr(11, 36, B64URL),
+    GITHUB_TOKEN: 'gh' + 'o_' + randStr(12, 36, UPPERNUM),
+    ANTHROPIC_API_KEY: 'sk-' + 'ant-' + randStr(13, 30, B64URL),
+    CLIENT_SECRET: randStr(14, 40, HEX),
+    DO_TOKEN: randStr(15, 64, HEX),
+    PATH: '/usr/local/bin:/usr/bin', // benign key, must be skipped
+    HOME: '/root', // benign + short
+    SHORT_TOKEN: 'abc', // secret-ish key but < 8 chars → skipped
+  };
+  const got = collectEnvSecrets(env);
+  for (const k of ['GDS_TOKEN', 'MAC_INSTALLER_CERT_P12', 'APPLE_API_ISSUER_ID', 'GITHUB_TOKEN', 'ANTHROPIC_API_KEY', 'CLIENT_SECRET', 'DO_TOKEN']) {
+    assert.ok(got.includes(env[k]), `expected ${k} value collected`);
+  }
+  assert.ok(!got.includes(env.PATH), 'PATH value must not be collected');
+  assert.ok(!got.includes(env.HOME), 'HOME value must not be collected');
+  assert.ok(!got.includes('abc'), 'short secret-ish value must be skipped');
+
+  // And the collected literals actually scrub their values out of a transcript.
+  const text = `the build used ${env.MAC_INSTALLER_CERT_P12} and ${env.APPLE_API_ISSUER_ID}`;
+  const out = scrubSecrets(text, got);
+  assert.ok(!out.includes(env.MAC_INSTALLER_CERT_P12) && !out.includes(env.APPLE_API_ISSUER_ID), 'env literals scrubbed');
+});
+
+test('self-reference `key = key` (SQL SET col = col) survives — zero recall loss', () => {
+  // task 1688: the core packager redacts live SQL, so a `_token` COLUMN self-increment
+  // must NOT be corrupted — while a real `key = <secret>` assignment is STILL redacted.
+  const selfRef = 'SET tokens_saved_est = tokens_saved_est + $2';
+  assert.equal(scrubSecrets(selfRef), selfRef, 'a _token column self-increment must survive intact');
+  const secret = 'ghp' + '_' + randStr(101, 36, B64URL); // github-PAT shape, built at runtime
+  const out = scrubSecrets(`access_token = ${secret}`);
+  assert.ok(!out.includes(secret), 'a real key=<secret> assignment must still be redacted');
+  assert.ok(out.startsWith('access_token = '), 'only the value is redacted; the key survives');
+});
+
+console.log(`\nscrubber_corpus: ${passed} passed, ${failed} failed`);
+process.exit(failed ? 1 : 0);

package/tests/search_chunker.mjs

@@ -0,0 +1,209 @@
+// tests/search_chunker.mjs
+//
+// Hermetic unit tests for src/bongos/search-chunker.js (task 945, V4.R07 / ADR 0060
+// §3). Pure transform — literal markdown strings in, chunks out. No fs, no
+// network, no DB. Covers: front-matter strip+capture, heading-stack breadcrumbs
+// (nesting + level pop), code-fence safety (a `#` inside ``` is NOT a heading),
+// over-long section sub-splitting (breadcrumb preserved), the no-heading and
+// empty-doc edges, and the document shapes the chunker must handle (CLAUDE.md,
+// ADRs, recipes, session logs).
+//
+// Run:  node tests/search_chunker.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const { chunkMarkdown, parseFrontMatter, estTokens, stripPrivate } = require('../modules/memory/search-chunker.js');
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try { fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.log(`  FAIL ${name}`); console.log(`     ${err.message}`); }
+}
+
+test('estTokens ~ chars/4, integer, zero for empty', () => {
+  assert.equal(estTokens(''), 0);
+  assert.equal(estTokens('abcd'), 1);
+  assert.equal(estTokens('abcde'), 2);
+});
+
+test('parseFrontMatter: leading --- block is captured + stripped from body', () => {
+  const md = '---\nname: my-note\ndescription: a thing\n---\n\n# Title\n\nbody text';
+  const { frontmatter, body } = parseFrontMatter(md);
+  assert.equal(frontmatter.name, 'my-note');
+  assert.equal(frontmatter.description, 'a thing');
+  assert.ok(!body.includes('name: my-note'), 'front-matter must not leak into body');
+  assert.ok(body.startsWith('# Title'), 'body starts after the closing fence');
+});
+
+test('parseFrontMatter: a --- that is NOT first line is a horizontal rule, kept in body', () => {
+  const md = '# Title\n\nintro\n\n---\n\nmore';
+  const { frontmatter, body } = parseFrontMatter(md);
+  assert.equal(frontmatter, null);
+  assert.ok(body.includes('---'), 'a mid-document hr stays in the body');
+});
+
+test('heading breadcrumbs: nested H1>H2>H3 reflected in heading_path', () => {
+  const md = [
+    '# Alpha', 'a-body',
+    '## Beta', 'b-body',
+    '### Gamma', 'g-body',
+  ].join('\n');
+  const { chunks } = chunkMarkdown({ path: 'docs/x.md', content: md });
+  const paths = chunks.map((c) => c.heading_path);
+  assert.ok(paths.includes('x.md > Alpha'), `got: ${paths.join(' | ')}`);
+  assert.ok(paths.includes('x.md > Alpha > Beta'));
+  assert.ok(paths.includes('x.md > Alpha > Beta > Gamma'));
+});
+
+test('heading level pop: H2 after H3 returns to the H1 ancestry (no stale H3)', () => {
+  const md = [
+    '# Top', 't',
+    '## One', '1',
+    '### Deep', 'd',
+    '## Two', '2', // pops 'Deep' AND 'One' → ancestry is just Top > Two
+  ].join('\n');
+  const { chunks } = chunkMarkdown({ path: 'f.md', content: md });
+  const two = chunks.find((c) => c.body === '2');
+  assert.equal(two.heading_path, 'f.md > Top > Two', `got ${two.heading_path}`);
+});
+
+test('code fence safety: a # line inside ``` is NOT treated as a heading', () => {
+  const md = [
+    '# Real Heading',
+    'intro',
+    '```sh',
+    '# this is a shell comment, not a heading',
+    'echo hi',
+    '```',
+    'after',
+  ].join('\n');
+  const { chunks } = chunkMarkdown({ path: 'r.md', content: md });
+  // Exactly one heading section; the fenced '# comment' must not split it.
+  assert.equal(chunks.length, 1, `expected 1 chunk, got ${chunks.length}`);
+  assert.equal(chunks[0].heading_path, 'r.md > Real Heading');
+  assert.ok(chunks[0].body.includes('shell comment'), 'fenced content is retained in the body');
+});
+
+test('no headings: whole doc is one section under the file label', () => {
+  const md = 'just some prose\n\nwith two paragraphs';
+  const { chunks, title } = chunkMarkdown({ path: 'notes.md', content: md });
+  assert.equal(chunks.length, 1);
+  assert.equal(chunks[0].heading_path, 'notes.md');
+  assert.equal(title, 'notes.md', 'title falls back to the file label when no H1/front-matter');
+});
+
+test('title resolution: front-matter title wins, then first H1, then file label', () => {
+  assert.equal(chunkMarkdown({ path: 'a.md', content: '---\ntitle: FM Title\n---\n# H1\nx' }).title, 'FM Title');
+  assert.equal(chunkMarkdown({ path: 'a.md', content: '# H1 Title\nx' }).title, 'H1 Title');
+  assert.equal(chunkMarkdown({ path: 'a.md', content: 'no heading at all' }).title, 'a.md');
+});
+
+test('over-long section sub-splits, preserves the breadcrumb, stays under max', () => {
+  const para = 'word '.repeat(400); // ~2000 chars ≈ 500 tokens per paragraph
+  const md = `# Big\n\n${para}\n\n${para}\n\n${para}`;
+  const { chunks } = chunkMarkdown({ path: 'big.md', content: md, maxTokens: 200 });
+  assert.ok(chunks.length >= 3, `expected several sub-chunks, got ${chunks.length}`);
+  for (const c of chunks) {
+    assert.equal(c.heading_path, 'big.md > Big', 'every sub-chunk keeps the section breadcrumb');
+    assert.ok(c.est_tokens <= 200 + 5, `sub-chunk over cap: ${c.est_tokens}`);
+  }
+});
+
+test('chunk_index is sequential and 0-based across the document', () => {
+  const md = '# A\nx\n## B\ny\n## C\nz';
+  const { chunks } = chunkMarkdown({ path: 'i.md', content: md });
+  chunks.forEach((c, i) => assert.equal(c.chunk_index, i));
+});
+
+test('empty / front-matter-only doc → zero chunks (nothing to index)', () => {
+  assert.equal(chunkMarkdown({ path: 'e.md', content: '' }).chunks.length, 0);
+  assert.equal(chunkMarkdown({ path: 'e.md', content: '   \n\n  ' }).chunks.length, 0);
+  assert.equal(chunkMarkdown({ path: 'e.md', content: '---\nname: x\n---\n' }).chunks.length, 0);
+});
+
+test('real shapes: an ADR-style doc chunks into its sections with breadcrumbs', () => {
+  const adr = [
+    '# ADR 0060 — Retrieval layer',
+    '- **Status:** Accepted',
+    '## Context',
+    'Some context about why.',
+    '## Decision',
+    '### Schema',
+    'A table.',
+    '### Query',
+    'RRF fusion.',
+  ].join('\n');
+  const { chunks, title } = chunkMarkdown({ path: 'docs/adr/0060-x.md', content: adr });
+  assert.equal(title, 'ADR 0060 — Retrieval layer');
+  const paths = chunks.map((c) => c.heading_path);
+  assert.ok(paths.some((p) => p === '0060-x.md > ADR 0060 — Retrieval layer > Decision > Schema'), paths.join(' | '));
+  assert.ok(paths.some((p) => p === '0060-x.md > ADR 0060 — Retrieval layer > Context'));
+});
+
+test('typed DB ref label (task:431) is used verbatim as the breadcrumb root', () => {
+  const { chunks } = chunkMarkdown({ path: 'task:431', content: '# Fix the thing\nbody' });
+  assert.equal(chunks[0].heading_path, 'task:431 > Fix the thing');
+});
+
+// -------------------------------------------------------------------------
+// <private> content-exclusion marker (ADR 0095 C4, task 1661)
+// -------------------------------------------------------------------------
+
+test('stripPrivate: no marker → text returned unchanged', () => {
+  const s = '# Title\n\nplain public body';
+  assert.equal(stripPrivate(s), s);
+});
+
+test('stripPrivate: a balanced <private>…</private> region is removed', () => {
+  const out = stripPrivate('keep <private>SECRET token=abc</private> keep');
+  assert.ok(!out.includes('SECRET'), out);
+  assert.ok(!out.includes('token=abc'), out);
+  assert.ok(out.includes('keep'), out);
+});
+
+test('stripPrivate: multi-line fence spanning many lines is removed whole', () => {
+  const md = ['public line', '<private>', 'API_KEY=sk-123', 'more secret', '</private>', 'tail'].join('\n');
+  const out = stripPrivate(md);
+  assert.ok(!out.includes('API_KEY'), out);
+  assert.ok(!out.includes('more secret'), out);
+  assert.ok(out.includes('public line') && out.includes('tail'), out);
+});
+
+test('stripPrivate: multiple independent fences each strip', () => {
+  const out = stripPrivate('a <private>x</private> b <private>y</private> c');
+  assert.ok(!out.includes('x') && !out.includes('y'), out);
+  assert.ok(out.includes('a') && out.includes('b') && out.includes('c'), out);
+});
+
+test('stripPrivate: case-insensitive + whitespace-tolerant tags', () => {
+  assert.ok(!stripPrivate('p <PRIVATE>nope</PRIVATE> q').includes('nope'));
+  assert.ok(!stripPrivate('p < private >nope< / private > q').includes('nope'));
+});
+
+test('stripPrivate: fail-closed — an unbalanced open strips to end of content', () => {
+  const out = stripPrivate('public head\n<private>dangling secret with no close\nstill secret');
+  assert.ok(!out.includes('secret'), out);
+  assert.ok(out.includes('public head'), out);
+});
+
+test('chunkMarkdown: <private> content never reaches the index chunks', () => {
+  const md = [
+    '# Notes',
+    'public intro',
+    '<private>',
+    'GITHUB_TOKEN=ghp_secret',
+    '</private>',
+    'public outro',
+  ].join('\n');
+  const { chunks } = chunkMarkdown({ path: 'memory/note.md', content: md });
+  const blob = chunks.map((c) => c.body).join('\n');
+  assert.ok(!blob.includes('GITHUB_TOKEN'), blob);
+  assert.ok(!blob.includes('ghp_secret'), blob);
+  assert.ok(blob.includes('public intro') && blob.includes('public outro'), blob);
+});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+process.exit(failed === 0 ? 0 : 1);

package/tests/search_ingest.mjs

@@ -0,0 +1,96 @@
+// tests/search_ingest.mjs
+//
+// Unit tests for the PURE pieces of the repo-markdown ingester (task 947,
+// V4.R09): the rank-tagging rule (metic for read-sensitive doc surfaces, xenos
+// otherwise — incl. the CLAUDE.md read-public carve-out) and the corpus walk's
+// exclusions. The DB delta/upsert (ingestFile) is exercised against a live
+// Postgres on the droplet (the deploy hook + the initial backfill); here we pin
+// the no-DB decisions that govern WHAT gets indexed and at what rank.
+//
+// Run:  node tests/search_ingest.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const ingest = require('../scripts/gds/search-ingest-md.js');
+const { minRankForDoc, collectMarkdown, chunkFile, SENSITIVE_DOC_GLOBS } = ingest;
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try { fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.log(`  FAIL ${name}`); console.log(`     ${err.message}`); }
+}
+
+test('metic-tagging: read-sensitive doc surfaces → metic', () => {
+  assert.equal(minRankForDoc('docs/canonical-permissions.md'), 'metic');
+  assert.equal(minRankForDoc('docs/security/threat-model.md'), 'metic');
+  assert.equal(minRankForDoc('docs/security/2026-06-12-newbuilder-redteam.md'), 'metic');
+  assert.equal(minRankForDoc('docs/audits/2026-06-08-server-security-audit.md'), 'metic');
+  assert.equal(minRankForDoc('docs/adr/0016-trust-boundary-server-enforced-permissions.md'), 'metic');
+  assert.equal(minRankForDoc('docs/adr/0043-git-ssh-trust-boundary-and-rank-floor-on-permission-paths.md'), 'metic');
+});
+
+test('metic-tagging: ordinary docs → xenos (read-public)', () => {
+  assert.equal(minRankForDoc('docs/architecture.md'), 'xenos');
+  assert.equal(minRankForDoc('docs/file-map.md'), 'xenos');
+  assert.equal(minRankForDoc('docs/adr/0060-gds-retrieval-layer.md'), 'xenos');
+  assert.equal(minRankForDoc('limitations/v1-shipped.md'), 'xenos');
+  assert.equal(minRankForDoc('art/README.md'), 'xenos');
+});
+
+test('CLAUDE.md is xenos (write-protected but READ-PUBLIC — the carve-out)', () => {
+  // CLAUDE.md is in permission-path-check PROTECTED_GLOBS (write protection), but
+  // it's the universal onboarding index — a Xenos recall MUST surface it.
+  assert.equal(minRankForDoc('CLAUDE.md'), 'xenos');
+});
+
+test('SENSITIVE_DOC_GLOBS is the curated source of truth (subtrees end with /)', () => {
+  assert.ok(SENSITIVE_DOC_GLOBS.includes('docs/security/'));
+  assert.ok(SENSITIVE_DOC_GLOBS.includes('docs/audits/'));
+  assert.ok(SENSITIVE_DOC_GLOBS.includes('docs/canonical-permissions.md'));
+});
+
+test('collectMarkdown finds the repo corpus but excludes node_modules/.git/worktrees/public', () => {
+  const path = require('node:path');
+  const repoRoot = path.resolve(new URL('..', import.meta.url).pathname);
+  const real = collectMarkdown(repoRoot, repoRoot);
+  // Assert on paths RELATIVE to the repo root — the suite may run from inside a
+  // git worktree (…/.claude/worktrees/<name>/…), so the ABSOLUTE paths would all
+  // contain '.claude/worktrees'; the exclusion is about NESTED such dirs.
+  const rel = real.map((f) => path.relative(repoRoot, f).split(path.sep).join('/'));
+  assert.ok(rel.length > 50, `expected a sizable md corpus, got ${rel.length}`);
+  assert.ok(rel.includes('CLAUDE.md'), 'CLAUDE.md must be in the corpus');
+  assert.ok(!rel.some((r) => r.startsWith('node_modules/')), 'node_modules excluded');
+  assert.ok(!rel.some((r) => r.startsWith('.git/')), '.git excluded');
+  assert.ok(!rel.some((r) => r.startsWith('.claude/worktrees/')), 'nested worktrees excluded');
+  assert.ok(!rel.some((r) => r.startsWith('public/')), 'public/ web assets excluded');
+});
+
+test('chunkFile yields well-formed rows (source_kind doc, hash, rank, sequential index)', () => {
+  const path = require('node:path');
+  const repoRoot = path.resolve(new URL('..', import.meta.url).pathname);
+  const rows = chunkFile(path.join(repoRoot, 'docs', 'adr', '0016-trust-boundary-server-enforced-permissions.md'));
+  assert.ok(rows.length > 0);
+  rows.forEach((r, i) => {
+    assert.equal(r.source_kind, 'doc');
+    assert.equal(r.source_ref, 'docs/adr/0016-trust-boundary-server-enforced-permissions.md');
+    assert.equal(r.chunk_index, i);
+    assert.equal(r.min_rank, 'metic', 'a trust-boundary ADR must be metic');
+    assert.equal(r.owner_builder_id ?? null, null, 'docs are never owner-scoped');
+    assert.match(r.content_hash, /^[0-9a-f]{64}$/, 'content_hash is a sha256 hex');
+  });
+});
+
+test('requirePool() resolves the real pg Pool with a .connect (task 1156 regression guard)', () => {
+  // The bug: the ingester required src/bongos/db (no `pool` export) → the DB path
+  // crashed with "Cannot read properties of undefined (reading connect)". Pin
+  // that requirePool() yields a usable pool so a wrong path is caught here, not
+  // only when the ingest actually runs against Postgres.
+  const pool = ingest.requirePool();
+  assert.ok(pool && typeof pool.connect === 'function', 'requirePool() must return a pg Pool with .connect()');
+});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+process.exit(failed === 0 ? 0 : 1);

package/tests/search_ingest_db.mjs

@@ -0,0 +1,88 @@
+// tests/search_ingest_db.mjs
+//
+// Unit tests for the PURE pieces of the DB-prose ingester (task 948, V4.R10):
+// the row builder's scrubbing + rank/owner scoping, the excluded-sources
+// invariant, and the git-commit parser. The DB queries + upsert run nightly on
+// the droplet against live Postgres; here we pin the no-DB decisions that govern
+// what gets indexed, at what rank, for whom, and with secrets removed.
+//
+// Run:  node tests/search_ingest_db.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const ingest = require('../scripts/gds/search-ingest-db.js');
+const { buildRows, recentCommitRows, SOURCES } = ingest;
+
+let passed = 0;
+let failed = 0;
+function test(name, fn) {
+  try { fn(); passed++; console.log(`  ok  ${name}`); }
+  catch (err) { failed++; console.log(`  FAIL ${name}`); console.log(`     ${err.message}`); }
+}
+
+test('buildRows: source_kind db rows carry source_ref, sequential index, content_hash', () => {
+  const rows = buildRows({ sourceRef: 'task:431', title: 'Fix the thing', body: 'A real description with enough words to chunk.' });
+  assert.ok(rows.length >= 1);
+  rows.forEach((r, i) => {
+    assert.equal(r.source_ref, 'task:431');
+    assert.equal(r.chunk_index, i);
+    assert.match(r.content_hash, /^[0-9a-f]{64}$/);
+    assert.equal(r.min_rank, 'xenos');
+    assert.equal(r.owner_builder_id, null);
+  });
+});
+
+test('buildRows: a security_sensitive task is tagged metic (caller passes minRank)', () => {
+  const rows = buildRows({ sourceRef: 'task:9', title: 'sec', body: 'sensitive body text here', minRank: 'metic' });
+  assert.ok(rows.every((r) => r.min_rank === 'metic'));
+});
+
+test('buildRows: builder_memory is OWNER-SCOPED (owner_builder_id on every chunk)', () => {
+  const rows = buildRows({ sourceRef: 'memory:7:MEMORY.md', title: 'MEMORY.md', body: 'my private notes about the project', ownerBuilderId: 7 });
+  assert.ok(rows.length >= 1);
+  assert.ok(rows.every((r) => r.owner_builder_id === 7), 'every memory chunk must carry the owner id');
+});
+
+test('buildRows: scrub=true redacts a planted secret before storage', () => {
+  const secret = 'ghp_' + 'A'.repeat(36); // GitHub-token shape
+  const rows = buildRows({ sourceRef: 'session_log:1', title: 'sess', body: `ran a deploy, token was ${secret} oops`, scrub: true });
+  const joined = rows.map((r) => r.body).join('\n');
+  assert.ok(!joined.includes(secret), 'the secret must NOT survive into a stored chunk');
+  assert.match(joined, /<redacted>/, 'scrubSecrets replaces it with the redaction marker');
+});
+
+test('buildRows: scrub=false leaves ordinary prose intact', () => {
+  const rows = buildRows({ sourceRef: 'learning:1', title: 'L', body: 'a perfectly ordinary learning about postgres indexes' });
+  assert.match(rows.map((r) => r.body).join(' '), /ordinary learning about postgres/);
+});
+
+test('buildRows: empty / whitespace body → no rows', () => {
+  assert.equal(buildRows({ sourceRef: 'idea:1', title: 'x', body: '' }).length, 0);
+  assert.equal(buildRows({ sourceRef: 'idea:1', title: 'x', body: '   ' }).length, 0);
+  assert.equal(buildRows({ sourceRef: 'idea:1', title: 'x', body: null }).length, 0);
+});
+
+test('SECURITY: vulnerability_reports + security_reports are NOT ingested', () => {
+  const names = SOURCES.map((s) => s.name);
+  assert.ok(!names.includes('vulnerability_reports'), 'vulnerability_reports must be excluded');
+  assert.ok(!names.includes('security_reports'), 'security_reports must be excluded');
+  // The sources we DO index include the documented prose surfaces.
+  for (const expected of ['tasks', 'learnings', 'session_logs', 'idea_inbox', 'task_grades', 'done_when_criteria', 'blockers', 'builder_memory']) {
+    assert.ok(names.includes(expected), `expected source ${expected}`);
+  }
+});
+
+test('recentCommitRows: parses git log into commit: chunks (best-effort)', () => {
+  // Runs against this repo's git; in a real checkout it returns rows, in a
+  // shallow/–git env it returns [] (best-effort). Assert the SHAPE when present.
+  const rows = recentCommitRows(5);
+  for (const r of rows) {
+    assert.match(r.source_ref, /^commit:[0-9a-f]{7,12}$/);
+    assert.ok(r.body && r.body.length > 0);
+  }
+});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+process.exit(failed === 0 ? 0 : 1);