@bongos/core 1.17.0

package/modules/memory/routes/search.js

@@ -0,0 +1,188 @@
+// GDS retrieval layer — POST /api/bongos/search (task 949, V4.R11 / ADR 0060 §5–§6).
+//
+// The "one-call recall" HTTP surface: a requireBuilder-gated lexical search over
+// search_chunks. The query (FTS + pg_trgm fused with RRF) and the rank/owner
+// scoping live in src/bongos/search.js; this router is the thin auth + validation +
+// rate-limit shell over it.
+//
+// SCOPING (ADR 0016 / ADR 0060 §6): the search owner + rank are taken ONLY from
+// the authenticated session (req.builder.id / req.builder.rank), NEVER from the
+// request body — there is no parameter by which a caller could widen their own
+// visibility. searchChunks filters in SQL against those, so a chunk the caller
+// can't read is never selected. The isolation suite (#950) is the regression
+// guard. There is deliberately NO public/unauthenticated search surface.
+//
+// RATE LIMIT: the global limiter (middleware/rate-limit.js) only caps WRITE
+// methods; this is a POST but it's a READ, and an uncapped search loop could
+// hammer the FTS/trgm scan. So we add a per-builder sliding-window cap keyed on
+// the AUTHENTICATED id (never input), mirroring routes/memory.js's readRateLimit.
+
+const express = require('express');
+// Carved into modules/memory/ (BV1.R72 / ADR 0091 §2): auth via the doorway; the
+// search lib is an intra-module require.
+const api = require('../../../src/module-api');
+const auth = api;
+const search = require('../search');
+
+const READ_RATE_WINDOW_MS = 60 * 1000;
+const READ_RATE_LIMIT = 60; // searches per builder per minute
+const buckets = new Map(); // builderId -> timestamp[]
+
+const sweepTimer = setInterval(() => {
+  const cutoff = Date.now() - READ_RATE_WINDOW_MS;
+  for (const [k, arr] of buckets) {
+    while (arr.length && arr[0] < cutoff) arr.shift();
+    if (arr.length === 0) buckets.delete(k);
+  }
+}, 5 * 60 * 1000);
+if (typeof sweepTimer.unref === 'function') sweepTimer.unref();
+
+// Per-builder search cap. Runs AFTER requireBuilder so req.builder.id is set;
+// keyed on the authenticated id only — never on any body/query input.
+function searchRateLimit(req, res, next) {
+  const now = Date.now();
+  const key = String(req.builder.id);
+  let bucket = buckets.get(key);
+  if (!bucket) { bucket = []; buckets.set(key, bucket); }
+  const cutoff = now - READ_RATE_WINDOW_MS;
+  while (bucket.length && bucket[0] < cutoff) bucket.shift();
+  if (bucket.length >= READ_RATE_LIMIT) {
+    const retryAfter = Math.max(1, Math.ceil((bucket[0] + READ_RATE_WINDOW_MS - now) / 1000));
+    res.set('Retry-After', String(retryAfter));
+    res.set('X-RateLimit-Scope', 'per-builder-search');
+    res.set('X-RateLimit-Limit', String(READ_RATE_LIMIT));
+    return res.fail('rate_limited', 429, { scope: 'per-builder-search', retry_after_seconds: retryAfter });
+  }
+  bucket.push(now);
+  return next();
+}
+
+module.exports = function buildSearchRouter() {
+  const router = express.Router();
+
+  // POST /search — two modes (ADR 0095 C1 / task 1658, progressive disclosure):
+  //   1. SUMMARY (default): body { q: string, limit?: 1..50, source_kind? }.
+  //      Returns the ranked compact layer — each hit carries a snippet plus
+  //      `snippet_tokens` (what you already see) and `full_tokens` (what a full
+  //      expansion would COST), so the caller sees the price before paying it.
+  //   2. EXPAND: body { expand: <chunkId> } (with q optional). Returns that one
+  //      chunk's FULL body + `body_tokens`, scoped exactly like search (rank +
+  //      owner from the session), so expand can't widen visibility.
+  // Both shapes stay backward-compatible: the old { q } request still returns
+  // { query, count, results } — the new per-hit fields are additive.
+  // Single-line opener so the deterministic route-rank auditor classifies the
+  // chain (requireBuilder → 'any-builder') — ADR 0016 / route-rank-check.js.
+  router.post('/search', auth.requireBuilder, searchRateLimit, async (req, res) => {
+    if (api.validateOrRespond(req, res, {
+      expand: {},                        // positive chunk id (number or string) — expand mode
+      q: { type: 'string' },             // search query (required for search mode; length-checked below)
+      source_kind: { type: 'string' },   // 'doc' | 'db' (any other value is ignored below)
+      limit: {},                         // page size (clamped inside searchChunks)
+    })) return;
+    const body = req.body || {};
+
+    // EXPAND mode: fetch one chunk's full content. Takes precedence over a
+    // search when `expand` is present; q is not required to expand.
+    if (body.expand !== undefined && body.expand !== null && body.expand !== '') {
+      const expandId = Number(body.expand);
+      if (!Number.isInteger(expandId) || expandId <= 0) {
+        return res.fail('bad_request', { status: 400, message: 'expand must be a positive chunk id' });
+      }
+      try {
+        const chunk = await search.fetchChunkBody({
+          id: expandId,
+          // Scope from the SESSION only — never from the body (ADR 0016).
+          builderId: req.builder.id,
+          builderRank: req.builder.rank,
+        });
+        res.set('Cache-Control', 'no-store');
+        if (!chunk) {
+          // Not found OR not visible — indistinguishable on purpose (no oracle).
+          return res.fail('not_found', { status: 404, message: 'no such chunk visible to you' });
+        }
+        return res.json({ expand: expandId, chunk });
+      } catch (err) {
+        console.error('[gds] POST /search expand', { builderId: req.builder.id, err: err && err.message });
+        return res.fail('expand_failed', { status: 500, message: 'chunk could not be expanded' });
+      }
+    }
+
+    const q = typeof body.q === 'string' ? body.q.trim() : '';
+    if (!q) {
+      return res.fail('bad_request', { status: 400, message: 'q (a non-empty search string) is required' });
+    }
+    if (q.length > 1000) {
+      return res.fail('query_too_long', { status: 400, message: 'q must be at most 1000 characters' });
+    }
+    const sourceKind = body.source_kind === 'doc' || body.source_kind === 'db' ? body.source_kind : null;
+    try {
+      const results = await search.searchChunks({
+        q,
+        limit: body.limit,
+        sourceKind,
+        // Scope from the SESSION only — never from the body (ADR 0016).
+        builderId: req.builder.id,
+        builderRank: req.builder.rank,
+      });
+      res.set('Cache-Control', 'no-store');
+      // total_full_tokens: the cost of expanding EVERY hit — the headline an
+      // agent reads to decide whether to expand selectively (ADR 0095 C1).
+      const totalFullTokens = results.reduce((sum, r) => sum + (r.full_tokens || 0), 0);
+      return res.json({ query: q, count: results.length, total_full_tokens: totalFullTokens, results });
+    } catch (err) {
+      console.error('[gds] POST /search', { builderId: req.builder.id, err: err && err.message });
+      return res.fail('search_failed', { status: 500, message: 'search could not be completed' });
+    }
+  });
+
+  // POST /search/mark — the write half of the memory-usefulness feedback loop
+  // (ADR 0095 C3, task 1660). Body { chunk_id: int, useful?: bool }. Records that
+  // a recalled chunk did (useful=true, the default) or did NOT (useful=false)
+  // help, so the ranker lifts the useful and fades the dead-end.
+  //
+  // SCOPING (ADR 0016): the caller's id + rank come ONLY from the session, never
+  // the body — searchChunks-equivalent scope is re-applied in the UPDATE WHERE,
+  // so a chunk the caller couldn't have seen can't be voted on (→ updated:false).
+  // Single-line opener so the route-rank auditor classifies the chain.
+  router.post('/search/mark', auth.requireBuilder, searchRateLimit, async (req, res) => {
+    if (api.validateOrRespond(req, res, {
+      chunk_id: {},   // positive integer (number or string) — validated below
+      useful: {},     // optional; Boolean()-coerced, defaults true
+    })) return;
+    const body = req.body || {};
+    const chunkId = Number(body.chunk_id);
+    if (!Number.isInteger(chunkId) || chunkId <= 0) {
+      return res.fail('bad_request', { status: 400, message: 'chunk_id (a positive integer) is required' });
+    }
+    // useful defaults to true: the common case is "this result helped". Pass
+    // useful:false to record a dead end.
+    const useful = body.useful === undefined ? true : Boolean(body.useful);
+    try {
+      const result = await search.markChunkUsefulness({
+        chunkId,
+        useful,
+        // Scope from the SESSION only — never from the body (ADR 0016).
+        builderId: req.builder.id,
+        builderRank: req.builder.rank,
+      });
+      res.set('Cache-Control', 'no-store');
+      if (!result.updated) {
+        // Either the chunk doesn't exist or the caller can't see it — same
+        // 404 either way, so the endpoint never confirms a hidden chunk's id.
+        return res.fail('not_found', { status: 404, message: 'no such chunk visible to you' });
+      }
+      return res.json(result);
+    } catch (err) {
+      if (err && err.code === 'BAD_CHUNK_ID') {
+        return res.fail('bad_request', { status: 400, message: err.message });
+      }
+      console.error('[gds] POST /search/mark', { builderId: req.builder.id, err: err && err.message });
+      return res.fail('mark_failed', { status: 500, message: 'usefulness mark could not be recorded' });
+    }
+  });
+
+  return router;
+};
+
+// Exported for tests — the pure rate-limit bucket logic + limits.
+module.exports._internals = { searchRateLimit, READ_RATE_LIMIT, READ_RATE_WINDOW_MS };

package/modules/memory/search-chunker.js

@@ -0,0 +1,252 @@
+// GDS retrieval layer — heading-aware markdown chunker (task 945, V4.R07).
+//
+// ADR 0060 §3: split markdown on heading boundaries into ~300–500-token chunks,
+// each carrying a breadcrumb (file > H1 > H2 > …) so a search result is self-
+// locating, with YAML front-matter stripped from the body but kept as metadata.
+// A heading-coherent chunk beats a fixed byte window: it's a whole section, so a
+// snippet reads as one idea and the breadcrumb says exactly where it lives.
+//
+// PURE + HERMETIC (no fs, no network, no clock) — the tests/boxes.mjs posture.
+// The ingesters (#947 repo-md, #948 DB-prose) read files / rows and feed the
+// (path|label, content) in; this module only transforms text → chunks. That
+// keeps the chunking logic unit-testable against literal strings.
+//
+// Surface:
+//   chunkMarkdown({ path, content, maxTokens?, targetTokens? })
+//     → { path, title, frontmatter, chunks: [{ chunk_index, heading_path, title,
+//         body, est_tokens }] }
+//   parseFrontMatter(content) → { frontmatter, frontmatterRaw, body }
+//   estTokens(text) → integer token estimate (~chars/4)
+//
+// Tokens are ESTIMATED (chars/4) — we're sizing chunks, not billing. Good enough
+// to keep chunks in the 300–500-token band the embedding/eval work assumes.
+
+const DEFAULT_TARGET_TOKENS = 400;
+const DEFAULT_MAX_TOKENS = 500;
+
+// -------------------------------------------------------------------------
+// <private> content-exclusion marker (ADR 0095 C4, task 1661)
+// -------------------------------------------------------------------------
+//
+// An agent can wrap sensitive content in a `<private>…</private>` fence so it is
+// NEVER stored in builder memory / learnings nor indexed for recall. This is the
+// explicit, write-time opt-out that reinforces the trust boundary (ADR 0016):
+// secrets, permission-core internals, and sub-Metic-sensitive material are
+// fenced out *at the source*, complementing the BFG transcript scrub.
+//
+// stripPrivate is PURE (no fs/network/clock — the module's hermetic posture) and
+// is applied at every memory ingest/persist entry point: storeMemoryFile,
+// storeMemoryFilesBatch, bfgWriteMemoryFile (memory.js), createLearning
+// (learnings.js), and chunkMarkdown below (the recall index). Strip-at-ingest
+// means the fenced text is neither persisted nor indexed — there is no copy to
+// leak later.
+//
+// Design notes:
+//   - Case-insensitive tag match (`<private>` / `<PRIVATE>`), tolerant of
+//     whitespace inside the tag (`< private >`), so a hand-typed marker still
+//     fences.
+//   - Multi-line, non-greedy: every fenced region in the text is removed, and a
+//     fence may span many lines. Multiple independent fences each strip.
+//   - Fail-CLOSED on an unbalanced open: a `<private>` with no matching close
+//     strips to end-of-content. A half-typed marker must not silently store the
+//     secret it was meant to hide.
+//   - The surrounding whitespace is collapsed so a fenced block sitting on its
+//     own lines doesn't leave a blank gap that would change chunk boundaries.
+const OPEN_PRIVATE = /<\s*private\s*>/i;
+const PAIRED_PRIVATE = /<\s*private\s*>[\s\S]*?<\s*\/\s*private\s*>/gi;
+const DANGLING_PRIVATE = /<\s*private\s*>[\s\S]*$/i;
+
+function stripPrivate(content) {
+  let text = String(content == null ? '' : content);
+  if (!OPEN_PRIVATE.test(text)) return text; // fast path: no marker at all.
+  // Remove every balanced <private>…</private> region (non-greedy, multi-line).
+  text = text.replace(PAIRED_PRIVATE, '');
+  // Fail-closed: any leftover open marker (no matching close) strips to the end.
+  text = text.replace(DANGLING_PRIVATE, '');
+  // Collapse the blank-line gap a removed block leaves behind so chunk
+  // boundaries (paragraph splits) aren't shifted by the redaction.
+  return text.replace(/\n{3,}/g, '\n\n');
+}
+
+// ~4 chars/token is the standard rough English estimate. Integer, never < 0.
+function estTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(String(text).length / 4);
+}
+
+// Last path segment without extension — the human file label for the breadcrumb
+// root (e.g. 'docs/adr/0016-x.md' → '0016-x.md' kept WITH extension so it reads
+// as a file; a 'task:431' label is returned as-is). We keep the extension for
+// real paths because 'CLAUDE.md' reads better than 'CLAUDE' as a breadcrumb root.
+function fileLabel(path) {
+  if (!path) return 'document';
+  const p = String(path);
+  // A typed DB ref like 'task:431' / 'learning:53' has no slash+extension — keep it.
+  if (!p.includes('/') && p.includes(':')) return p;
+  const base = p.split('/').filter(Boolean).pop() || p;
+  return base;
+}
+
+// Parse leading YAML front-matter ONLY when the very first line is a `---` fence.
+// We don't pull in a YAML dependency — front-matter here is flat `key: value`
+// (memory files: name/description/metadata; the nested `metadata:` block is kept
+// raw, not deep-parsed). The body is everything after the closing `---`.
+// A `---` that is NOT the first line is a horizontal rule, left in the body.
+function parseFrontMatter(content) {
+  const text = String(content == null ? '' : content);
+  // Normalize CRLF so the fence check + line splits are stable cross-platform.
+  const norm = text.replace(/\r\n/g, '\n');
+  if (!norm.startsWith('---\n')) {
+    return { frontmatter: null, frontmatterRaw: null, body: norm };
+  }
+  // Line-based: lines[0] is the opening '---'. Find the closing '---' (or YAML
+  // doc-end '...') on its own line. If there's no closing fence it isn't real
+  // front-matter — treat the whole input as body.
+  const lines = norm.split('\n');
+  let closeIdx = -1;
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i] === '---' || lines[i] === '...') { closeIdx = i; break; }
+  }
+  if (closeIdx === -1) {
+    return { frontmatter: null, frontmatterRaw: null, body: norm };
+  }
+  const raw = lines.slice(1, closeIdx).join('\n');
+  // Body is everything after the closing fence line, with leading blank lines
+  // trimmed so a chunk starts at real content.
+  const body = lines.slice(closeIdx + 1).join('\n').replace(/^\n+/, '');
+  const frontmatter = {};
+  for (const line of raw.split('\n')) {
+    const m = /^([A-Za-z0-9_.-]+):\s*(.*)$/.exec(line);
+    if (m && m[2] !== '') frontmatter[m[1]] = m[2].trim();
+  }
+  return { frontmatter, frontmatterRaw: raw, body };
+}
+
+// True for an ATX heading line OUTSIDE a code fence. Returns {level, text} or null.
+function parseHeading(line) {
+  const m = /^(#{1,6})\s+(.*?)\s*#*\s*$/.exec(line);
+  if (!m) return null;
+  return { level: m[1].length, text: m[2].trim() };
+}
+
+// Split a too-long section body into sub-chunks on paragraph boundaries, packing
+// greedily up to maxTokens. A single paragraph over the cap is hard-split on
+// lines, then (last resort) on a character window, so a pathological wall of text
+// can't produce an unbounded chunk. Every sub-chunk keeps the section breadcrumb.
+function splitLongBody(body, maxTokens) {
+  const maxChars = maxTokens * 4;
+  const paras = body.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+  const pieces = [];
+  let buf = '';
+  const flush = () => { if (buf.trim()) pieces.push(buf.trim()); buf = ''; };
+  for (const para of paras) {
+    if (estTokens(para) > maxTokens) {
+      // Oversized single paragraph — flush what we have, then hard-split it.
+      flush();
+      let chunk = '';
+      for (const line of para.split('\n')) {
+        if (chunk && estTokens(chunk + '\n' + line) > maxTokens) { pieces.push(chunk); chunk = ''; }
+        // A single line longer than the cap: hard window it by chars.
+        if (estTokens(line) > maxTokens) {
+          if (chunk) { pieces.push(chunk); chunk = ''; }
+          for (let i = 0; i < line.length; i += maxChars) pieces.push(line.slice(i, i + maxChars));
+          continue;
+        }
+        chunk = chunk ? chunk + '\n' + line : line;
+      }
+      if (chunk.trim()) pieces.push(chunk.trim());
+      continue;
+    }
+    if (buf && estTokens(buf + '\n\n' + para) > maxTokens) flush();
+    buf = buf ? buf + '\n\n' + para : para;
+  }
+  flush();
+  return pieces.length ? pieces : [body.trim()].filter(Boolean);
+}
+
+// chunkMarkdown — the core transform. `path` is a label (a repo path for docs, or
+// a typed ref like 'task:431' for DB prose); it's only used for the breadcrumb
+// root + returned for the caller's source_ref. Returns chunks in document order
+// with 0-based chunk_index.
+function chunkMarkdown({ path, content, maxTokens = DEFAULT_MAX_TOKENS, targetTokens = DEFAULT_TARGET_TOKENS } = {}) {
+  // ADR 0095 C4: strip <private>-fenced content BEFORE chunking so it never
+  // enters the recall index. Done first so front-matter parsing + heading
+  // splitting see only the public text.
+  const { frontmatter, body } = parseFrontMatter(stripPrivate(content));
+  const root = fileLabel(path);
+
+  // Title: explicit front-matter title/name → first H1 → file label.
+  let title = (frontmatter && (frontmatter.title || frontmatter.name)) || null;
+
+  const lines = body.split('\n');
+  // Section accumulator: heading stack + the lines collected under the current
+  // heading. A section flushes when the next heading appears (or at EOF).
+  const sections = [];
+  let stack = []; // [{level, text}] current heading ancestry
+  let curLines = [];
+  let curStack = []; // snapshot of the stack for the section being accumulated
+  let inFence = false;
+  let fenceMarker = null;
+
+  const flushSection = () => {
+    const text = curLines.join('\n').trim();
+    if (text) sections.push({ stack: curStack.slice(), body: text });
+    curLines = [];
+  };
+
+  for (const line of lines) {
+    // Track fenced code blocks so a `# comment` inside ``` isn't read as a heading.
+    const fence = /^(\s*)(```+|~~~+)/.exec(line);
+    if (fence) {
+      const marker = fence[2][0]; // '`' or '~'
+      if (!inFence) { inFence = true; fenceMarker = marker; }
+      else if (fenceMarker === marker) { inFence = false; fenceMarker = null; }
+      curLines.push(line);
+      continue;
+    }
+    const heading = inFence ? null : parseHeading(line);
+    if (heading) {
+      // Close the section that was accumulating under the PREVIOUS heading.
+      flushSection();
+      // Update the ancestry stack: pop to above this level, then push.
+      stack = stack.filter((h) => h.level < heading.level);
+      stack.push(heading);
+      curStack = stack.slice();
+      if (!title && heading.level === 1) title = heading.text;
+      continue;
+    }
+    curLines.push(line);
+  }
+  flushSection();
+
+  if (!title) title = root;
+
+  // Build chunks. Each section → one chunk, or several if it exceeds maxTokens.
+  const chunks = [];
+  const pushChunk = (headingPath, text) => {
+    const body2 = text.trim();
+    if (!body2) return;
+    chunks.push({
+      chunk_index: chunks.length,
+      heading_path: headingPath,
+      title,
+      body: body2,
+      est_tokens: estTokens(body2),
+    });
+  };
+
+  for (const sec of sections) {
+    const crumb = [root, ...sec.stack.map((h) => h.text)].join(' > ');
+    if (estTokens(sec.body) <= maxTokens) {
+      pushChunk(crumb, sec.body);
+    } else {
+      for (const piece of splitLongBody(sec.body, maxTokens)) pushChunk(crumb, piece);
+    }
+  }
+
+  // A doc that is ALL front-matter / whitespace (no body) yields zero chunks —
+  // that's correct (nothing to index), and the ingester simply stores none.
+  return { path: path || null, title, frontmatter: frontmatter || null, chunks };
+}
+
+module.exports = { chunkMarkdown, parseFrontMatter, estTokens, fileLabel, stripPrivate, DEFAULT_TARGET_TOKENS, DEFAULT_MAX_TOKENS };