@backbencher/brain 0.0.0-stage → 0.1.1

package/dist/taint-analyzer-N7YLZU3N.js

@@ -0,0 +1,163 @@
+import "./chunk-MLKGABMK.js";
+
+// src/ast/taint-analyzer.ts
+var TaintAnalyzer = class {
+  static enrichSymbols(symbols, content, language) {
+    for (const symbol of symbols) {
+      if (symbol.range.startIndex !== void 0 && symbol.range.endIndex !== void 0) {
+        const snippet = content.slice(symbol.range.startIndex, symbol.range.endIndex);
+        const sources = this.extractSources(snippet, language);
+        if (sources.length > 0) {
+          symbol.taintSources = symbol.taintSources ? [...symbol.taintSources, ...sources] : sources;
+        }
+        const sinks = this.extractSinks(snippet, language);
+        if (sinks.length > 0) {
+          symbol.taintSinks = symbol.taintSinks ? [...symbol.taintSinks, ...sinks] : sinks;
+        }
+        if (symbol.taintSources) {
+          symbol.taintSources = Array.from(new Set(symbol.taintSources.map((s) => s.snippet))).map(
+            (snippet2) => symbol.taintSources.find((s) => s.snippet === snippet2)
+          );
+        }
+        if (symbol.taintSinks) {
+          symbol.taintSinks = Array.from(new Set(symbol.taintSinks.map((s) => s.snippet))).map(
+            (snippet2) => symbol.taintSinks.find((s) => s.snippet === snippet2)
+          );
+        }
+      }
+    }
+  }
+  static extractSources(snippet, language) {
+    const sources = [];
+    if (/(?:req\.body|req\.query|req\.params|req\.headers|req\.cookies|request\.json|request\.args|request\.form|request\.files|\$_GET|\$_POST|\$_REQUEST|\$_COOKIE|\$_FILES|Request\.Query|Request\.Form|\[FromBody\]|\[FromQuery\]|params\[|request\.body|request\.url|request\.uri|URLSession\.shared)/i.test(
+      snippet
+    )) {
+      sources.push({ type: "user_input", snippet: "Web Request Data" });
+    }
+    if (/(?:process\.env|os\.environ|sys\.argv|System\/getenv|getenv\(|Sys\.getenv|std::env::args|getArgs|getEnv|commandArgs|command-line-args|\$1|\$2|\$@|\$\*)/.test(
+      snippet
+    )) {
+      sources.push({ type: "user_input", snippet: "Environment / CLI Argument" });
+    }
+    if (/(?:prompt\(|readLine|Scanner\(|input\(|scanf\(|gets\(|cin\s*>>|stdin\.readLineSync|io\.read|getLine)/.test(
+      snippet
+    )) {
+      sources.push({ type: "user_input", snippet: "Interactive Console Input" });
+    }
+    if (
+      // `\.Query\s*\(`, `\.QueryRow\s*\(`, `\.Exec\s*\(` — see the sink rule's
+      // comment. Requiring the call is what stops `req.query` from being read as
+      // a database READ; the same property read was previously classified as both
+      // a `database_read` source and a `sql_injection` sink.
+      /(?:db\.query|db\.find|findOne\(|findAll\(|select\(|\.Exec\s*\(|\.QueryRow\s*\(|\.Query\s*\(|ExecuteReader|executeQuery)/i.test(
+        snippet
+      )
+    ) {
+      sources.push({ type: "database_read", snippet: "Database Query Result" });
+    }
+    if (/(?:fs\.readFileSync|fs\.readFile|fs\.createReadStream|open\(|File\.ReadAllText|File\.ReadAllBytes|ioutil\.ReadFile|os\.ReadFile|file_get_contents|fopen\(|File\.open)/.test(
+      snippet
+    )) {
+      sources.push({ type: "file_system", snippet: "File System Read" });
+    }
+    if (/(?:password|passwd|pwd|secret|token|api_key|apikey|access_token|auth_token|client_secret|session_id|jwt|private_key|ssh_key|certificate|ssn|credit_card|cc_number)/i.test(
+      snippet
+    )) {
+      sources.push({ type: "sensitive_data", snippet: "Extracted Sensitive Credentials or PII" });
+    }
+    if (language === "java" || language === "kotlin") {
+      if (/(?:HttpServletRequest|request\.getParameter|@RequestParam|@RequestBody|@PathVariable|@RequestHeader|@CookieValue)/.test(
+        snippet
+      )) {
+        sources.push({ type: "user_input", snippet: "Java/Spring Web Request Context" });
+      }
+    } else if (language === "go") {
+      if (/(?:r\.URL\.Query|r\.FormValue|r\.Body|r\.PostFormValue|c\.Bind|c\.Param|c\.Query)/.test(snippet)) {
+        sources.push({ type: "user_input", snippet: "Go HTTP Request" });
+      }
+    } else if (language === "rust") {
+      if (/(?:req\.query|req\.body|Query\(|Json\(|Path\()/.test(snippet)) {
+        sources.push({ type: "user_input", snippet: "Rust Axum/Actix Request" });
+      }
+    } else if (language === "php") {
+      if (/(?:php\:\/\/input|file_get_contents\('php\:\/\/input'\))/.test(snippet)) {
+        sources.push({ type: "user_input", snippet: "PHP Raw Input Stream" });
+      }
+    } else if (language === "elixir") {
+      if (/(?:conn\.params|conn\.query_params|conn\.body_params)/.test(snippet)) {
+        sources.push({ type: "user_input", snippet: "Elixir Plug/Phoenix Conn" });
+      }
+    } else if (language === "scala") {
+      if (/(?:request\.getQueryString|request\.body|request\.headers)/.test(snippet)) {
+        sources.push({ type: "user_input", snippet: "Scala Play Request" });
+      }
+    } else if (language === "lua") {
+      if (/(?:ngx\.req\.get_uri_args|ngx\.req\.get_post_args)/.test(snippet)) {
+        sources.push({ type: "user_input", snippet: "Lua Nginx Request" });
+      }
+    }
+    return Array.from(new Set(sources.map((s) => s.snippet))).map(
+      (snippet2) => sources.find((s) => s.snippet === snippet2)
+    );
+  }
+  static extractSinks(snippet, language) {
+    const sinks = [];
+    if (/(?:db\.query|db\.execute|\bexecute\b|executeQuery|mysql_query|execute_query|query_row|\.Exec\s*\(|\.QueryRow\s*\(|\.Query\s*\(|ExecuteReader|ExecuteScalar|ExecuteNonQuery|sp_executesql|EXECUTE\s+IMMEDIATE)/i.test(
+      snippet
+    )) {
+      sinks.push({ type: "database_write", snippet: "Raw SQL Execution", vulnerability_type: "sql_injection" });
+    }
+    if (/(?:\bexec\(|\bspawn\(|\bexecSync|\bspawnSync|\bsystem\(|os\.system|subprocess\.|exec\.Command|Runtime\.getRuntime\(\)\.exec|Process\.Start|shell_exec\(|passthru\(|popen\(|os\.execute|io\.popen|System\.cmd|System\.Process|clojure\.java\.shell\/sh|system2\()/.test(
+      snippet
+    ) || language === "ruby" && /(?:Kernel\.exec|`.*`)/.test(snippet)) {
+      sinks.push({
+        type: "shell_execution",
+        snippet: "System Command Execution",
+        vulnerability_type: "command_injection"
+      });
+    }
+    if (/(?:fs\.readFile|fs\.writeFile|\bopen\(|File\.ReadAllText|File\.WriteAllText|ioutil\.ReadFile|os\.ReadFile|file_get_contents|\bfopen\(|File\.open|fs\.unlink|fs\.rmdir)/.test(
+      snippet
+    )) {
+      sinks.push({ type: "file_system", snippet: "File System Operation", vulnerability_type: "path_traversal" });
+    }
+    if (/(?:res\.send|res\.html|res\.write|innerHTML|document\.write|\becho |\bprint\s+|<script>|dangerouslySetInnerHTML)/.test(
+      snippet
+    )) {
+      sinks.push({ type: "http_response", snippet: "Raw HTTP/HTML Output", vulnerability_type: "xss" });
+    }
+    if (/(?:\beval\(|\bexec\(|loadstring\(|Code\.eval_string|Function\()/.test(snippet)) {
+      sinks.push({ type: "code_execution", snippet: "Dynamic Code Evaluation", vulnerability_type: "code_injection" });
+    }
+    if (/(?:pickle\.loads|pickle\.load|yaml\.load|marshal\.load|ObjectInputStream|readObject|unserialize\(|JSON\.parse|JSON\.parseObject|Bson\.parse)/.test(
+      snippet
+    )) {
+      sinks.push({
+        type: "deserialization",
+        snippet: "Object Deserialization",
+        vulnerability_type: "insecure_deserialization"
+      });
+    }
+    if (/(?:fetch\(|axios\.|requests\.get|requests\.post|urllib\.request|http\.Get|http\.Post|curl_exec|HttpClient|RestTemplate|WebClient|Net::HTTP)/.test(
+      snippet
+    )) {
+      sinks.push({ type: "network_request", snippet: "Outbound Network Request", vulnerability_type: "ssrf" });
+    }
+    if (/(?:res\.redirect|res\.set|res\.header|header\(|Location:|Redirect\()/.test(snippet)) {
+      sinks.push({
+        type: "http_response",
+        snippet: "HTTP Redirect / Header Setup",
+        vulnerability_type: "open_redirect"
+      });
+    }
+    if (/(?:md5\(|sha1\(|crypto\.createHash\("md5"\)|hashlib\.md5\(|MessageDigest\.getInstance\("MD5"\)|MD5\.Create|Digest::MD5)/i.test(
+      snippet
+    )) {
+      sinks.push({ type: "cryptography", snippet: "Weak Hashing Function", vulnerability_type: "weak_crypto" });
+    }
+    return Array.from(new Set(sinks.map((s) => s.snippet))).map((snippet2) => sinks.find((s) => s.snippet === snippet2));
+  }
+};
+export {
+  TaintAnalyzer
+};

package/dist/tool-handlers-OJDC35DI.js

@@ -0,0 +1,72 @@
+import {
+  ANALYZE_GRAPH_WALK_CEILING_MS,
+  SECURITY_FINDINGS_WINDOW,
+  SECURITY_RECORD_RUN_CENSUS_CAP,
+  SECURITY_SCAN_CAP,
+  SECURITY_SUSPICIOUS_WINDOW,
+  autoRefreshFile,
+  canonicalImpactReport,
+  createToolHandlers,
+  derivedStaleness,
+  describeUnknownFilters,
+  ensureWithinProject,
+  readIndexedFile,
+  readSnippet,
+  resolveProjectPath,
+  resolveSafeReadPath,
+  textResult
+} from "./chunk-SGGCSHAX.js";
+import "./chunk-EY5ZZUUA.js";
+import "./chunk-VL52PL2T.js";
+import "./chunk-AHDWGM7Q.js";
+import "./chunk-BEIZZ73B.js";
+import "./chunk-3CTFIJJB.js";
+import "./chunk-HQSPJDHI.js";
+import "./chunk-22V37YTK.js";
+import "./chunk-DWKXZ6IF.js";
+import "./chunk-2RRBMNBQ.js";
+import "./chunk-UQNQZXHY.js";
+import "./chunk-ZETTGPAW.js";
+import "./chunk-4H22TLOT.js";
+import "./chunk-B5GEAAYF.js";
+import "./chunk-7AN7U5IZ.js";
+import "./chunk-UGR4QIGS.js";
+import "./chunk-GEFFO6O2.js";
+import "./chunk-NO3U3FJQ.js";
+import "./chunk-DYM4FQEE.js";
+import "./chunk-RM3PY6TG.js";
+import "./chunk-RCMTFSEF.js";
+import "./chunk-ENW4BTBA.js";
+import "./chunk-5BVAH26D.js";
+import "./chunk-UUNHHX4M.js";
+import "./chunk-QLK6ED4Q.js";
+import "./chunk-QKTRECYS.js";
+import "./chunk-3ZTOHHSW.js";
+import "./chunk-BQ3OVQ3P.js";
+import "./chunk-SK6YVHFZ.js";
+import "./chunk-2BSKD7O3.js";
+import "./chunk-KAET47ZL.js";
+import "./chunk-KCDECRJY.js";
+import "./chunk-D2Q4Y47A.js";
+import "./chunk-SM7RFQRO.js";
+import "./chunk-L5TDYKYK.js";
+import "./chunk-ROMM6KMC.js";
+import "./chunk-MLKGABMK.js";
+export {
+  ANALYZE_GRAPH_WALK_CEILING_MS,
+  SECURITY_FINDINGS_WINDOW,
+  SECURITY_RECORD_RUN_CENSUS_CAP,
+  SECURITY_SCAN_CAP,
+  SECURITY_SUSPICIOUS_WINDOW,
+  autoRefreshFile,
+  canonicalImpactReport,
+  createToolHandlers,
+  derivedStaleness,
+  describeUnknownFilters,
+  ensureWithinProject,
+  readIndexedFile,
+  readSnippet,
+  resolveProjectPath,
+  resolveSafeReadPath,
+  textResult
+};

package/dist/tool-registry-BPEJNSZG.js

@@ -0,0 +1,17 @@
+import {
+  PUBLIC_TOOL_DESCRIPTORS,
+  TOOL_DESCRIPTORS,
+  deriveToolContracts,
+  serializeToolMetadata,
+  toMcpToolList,
+  validateToolDescriptors
+} from "./chunk-HQSPJDHI.js";
+import "./chunk-MLKGABMK.js";
+export {
+  PUBLIC_TOOL_DESCRIPTORS,
+  TOOL_DESCRIPTORS,
+  deriveToolContracts,
+  serializeToolMetadata,
+  toMcpToolList,
+  validateToolDescriptors
+};

package/dist/tool-validation-Q57YMMBX.js

@@ -0,0 +1,32 @@
+import {
+  EDIT_OPTIONS_BY_MODE,
+  PARAMETER_ALIASES,
+  resolveParameterAliases,
+  validateToolArguments
+} from "./chunk-HJEWO5EV.js";
+import "./chunk-BEIZZ73B.js";
+import "./chunk-ZETTGPAW.js";
+import "./chunk-4H22TLOT.js";
+import "./chunk-GEFFO6O2.js";
+import "./chunk-NO3U3FJQ.js";
+import "./chunk-DYM4FQEE.js";
+import "./chunk-5BVAH26D.js";
+import "./chunk-QLK6ED4Q.js";
+import "./chunk-QKTRECYS.js";
+import "./chunk-3ZTOHHSW.js";
+import "./chunk-BQ3OVQ3P.js";
+import "./chunk-SK6YVHFZ.js";
+import "./chunk-2BSKD7O3.js";
+import "./chunk-KAET47ZL.js";
+import "./chunk-KCDECRJY.js";
+import "./chunk-D2Q4Y47A.js";
+import "./chunk-SM7RFQRO.js";
+import "./chunk-L5TDYKYK.js";
+import "./chunk-ROMM6KMC.js";
+import "./chunk-MLKGABMK.js";
+export {
+  EDIT_OPTIONS_BY_MODE,
+  PARAMETER_ALIASES,
+  resolveParameterAliases,
+  validateToolArguments
+};

package/dist/type-inference-QMT2RMLV.js

@@ -0,0 +1,13 @@
+import {
+  TypeInference
+} from "./chunk-7DRPNVIN.js";
+import "./chunk-3ZTOHHSW.js";
+import "./chunk-BQ3OVQ3P.js";
+import "./chunk-SK6YVHFZ.js";
+import "./chunk-2BSKD7O3.js";
+import "./chunk-KAET47ZL.js";
+import "./chunk-KCDECRJY.js";
+import "./chunk-MLKGABMK.js";
+export {
+  TypeInference
+};

package/dist/vector-binary-LMY4QGAN.js

@@ -0,0 +1,19 @@
+import {
+  cosineFloat32,
+  decodeVectorBlob,
+  encodeVectorBlob,
+  l2Norm,
+  readRowVector,
+  scanJsonVectorInto,
+  vectorBlobBytes
+} from "./chunk-GEFFO6O2.js";
+import "./chunk-MLKGABMK.js";
+export {
+  cosineFloat32,
+  decodeVectorBlob,
+  encodeVectorBlob,
+  l2Norm,
+  readRowVector,
+  scanJsonVectorInto,
+  vectorBlobBytes
+};

package/dist/version-ABP2TUL2.js

@@ -0,0 +1,9 @@
+import {
+  BACKBENCHER_BRAIN_VERSION,
+  PKG_NAME
+} from "./chunk-WIG36E67.js";
+import "./chunk-MLKGABMK.js";
+export {
+  BACKBENCHER_BRAIN_VERSION,
+  PKG_NAME
+};

package/dist/wasm-NMWO3WXY.js

@@ -0,0 +1,16 @@
+import {
+  ensureWasmRegistry,
+  hasWasmInRegistry,
+  isWasmRegistryLoaded,
+  loadWebTreeSitter,
+  resolveWasmAsset
+} from "./chunk-SK6YVHFZ.js";
+import "./chunk-2BSKD7O3.js";
+import "./chunk-MLKGABMK.js";
+export {
+  ensureWasmRegistry,
+  hasWasmInRegistry,
+  isWasmRegistryLoaded,
+  loadWebTreeSitter,
+  resolveWasmAsset
+};

package/docs/EMBEDDINGS.md

@@ -0,0 +1,132 @@
+# EMBEDDINGS — how semantic embeddings work in the brain
+
+> Status: verified against real evidence — M18 (Semantic Intelligence & Retrieval
+> Truth): `validation/m18/results/Q1..Q5.json`. Every claim below is either a
+> code invariant or a number measured on real corpora in M18. Nothing here is
+> aspirational.
+
+## What embeddings are
+
+The brain stores one dense vector per indexed symbol in
+`brain_semantic_embedding`:
+
+| Column | Meaning |
+| --- | --- |
+| `entity_id` | The symbol this embedding belongs to |
+| `embedding_model` | Canonical model label **that actually produced the vector** |
+| `vector_dimension` | Dimension of that model's vectors |
+| `embedding_blob` | little-endian float32, authoritative for dense vectors (`vector_dimension * 4` bytes) |
+| `embedding_vector` | empty string for dense rows; JSON numeric array for legacy rows and sparse token lists (`tfidf-v1`) |
+| `content_hash` | Hash of the snippet that produced the embedding (insert-time dedupe; **not** consulted at query time) |
+| `similarity_threshold` | Per-row threshold hint (insert-time) |
+
+Embeddings are generated in the background after indexing, batched through
+`SemanticEngine.embedMany`. They are **additive**: an index run flushes new
+batches; coverage is never a one-shot snapshot.
+
+## 2. Model labels & vector spaces (the correctness invariant)
+
+- The **query vector** is produced by `SemanticEngine.embed(query)`. The model
+  that actually produced it (`getLastEmbeddingModel()`) is authoritative —
+  the API layer may transparently fall back to local word vectors, so the
+  configured model name is never trusted.
+- Semantic vector search compares **only** stored embeddings whose
+  `embedding_model` equals the query model **and** whose `vector_dimension`
+  equals the query vector length. Cross-model comparison is **prohibited**
+  even at equal dimension, because different models occupy different vector
+  spaces (verified: Q4 inserts a fake foreign-model row; it is never compared;
+  Q5 model-mismatch case re-verifies it deterministically).
+- `tfidf-v1` rows are sparse `{token, weight}` maps — structurally
+  incompatible with dense cosine. They carry a distinct model label and are
+  excluded from the dense path (`non-dense-query-model`). Verified in Q5.
+- Any stored vector that is not a finite numeric array is skipped (corrupt
+  rows — malformed JSON, non-numeric, `Infinity` — are skipped without crash;
+  Q5 case 1). Zero/norm-0 vectors never produce NaN similarity (Q5 case 9).
+
+## 3. Coverage semantics — volatile, per-run, never hidden
+
+**Coverage is a per-run, post-flush statistic, not a fixed product number.**
+Measured on the rake corpus in M18:
+
+| Run | Embeddings | Symbols | Coverage | Phase |
+| --- | --- | --- | --- | --- |
+| Q2 rake | 445 / 779 | 445 | 57.1% | post-flush (flush 15.4s) |
+| Q4 rake | 445 / 779 | 445 | 57.1% | post-flush (flush 15.4s) |
+| Q4 pre-flush | 0 / 779 | 0 | 0.0% | during background drain |
+
+The **pre-flush `0 of N` line is distinguishable from the post-flush number**:
+evidence records which phase a number was measured in, and the health line
+labels it. Coverage is reported through `bb_health` (machine-checkable) and is
+**never hidden — even when 0** (Q4: a zero-coverage index reports `0 of N`
+explicitly, and `bb_ask` surfaces the degradation note instead of presenting
+lexical matches as semantic).
+
+Coverage is **volatile by design**: the embedding API batches and rate-limits,
+so one run may embed 62.2% of lodash and another 10.0% of the same corpus (both
+post-flush). The product therefore never claims a fixed coverage target — it
+reports what it measured, per run (see Q4 negative line).
+
+## 4. Fallback ladder (explicit, never silent)
+
+When no usable vector result exists, retrieval falls back to lexical search
+(FTS5/BM25 content search) with an explicit `fallbackReason`:
+
+`embedding-error` · `no-query-vector` · `non-dense-query-model` ·
+`no-matching-model-embeddings` · `no-vector-above-threshold` ·
+`no-resolvable-entities`
+
+Each of these is a real, reachable path — the deterministic wire‑tests in Q5
+prove the `embedding-error`, `no-query-vector`, `no-matching-model-embeddings`,
+`non-dense-query-model`, and degenerate-row paths offline.
+
+## 5. Retrieval quality (measured, bounded, honest)
+
+On the M18 benchmark (rake + lodash + element-plus, gold answers with
+provenance in `validation/m18/benchmark/benchmark.json`):
+
+| Metric | Measured | Floor asserted |
+| --- | --- | --- |
+| Vector recall@10 (median over query runs) | **71.4%** (10/14) | >= 50% |
+| Vector recall@5 | **57.1%** (8/14) | >= 0.3 |
+| `bb_ask` warm precision (answer cites gold symbol) | **71.4%** (10/14) | >= 0.5 |
+| `bb_ask` cold-start precision (no embeddings) | **42.9%** (6/14) | >= 0.3 |
+| cold/warm delta | -28.6 pts | reported, not hidden |
+
+The deltas exists for a reason: cold start has **zero** stored embeddings, so
+`bb_ask` falls back to lexical symbol/content search — symbol-name queries
+still land, natural-language paraphrases degrade (Q3 measured the full delta;
+Q3's "does not know" honesty floor: all negative queries return confidence
+< 0.5, zero fabrication).
+
+These are **candidate-discovery** numbers — they prove the correct symbol is
+*retrieved*; they do not prove behavioral semantics. Ranks are per-embedding-model
+and per-run; a gold embedded under a different model than the query is an
+out-of-domain (coverage) miss, not a retrieval defect (Q2 negative line).
+
+## 6. What embeddings do NOT do
+
+- Embeddings are **not** stored file source; content search covers the indexed
+  symbol + docstring corpus, not full file text.
+- Embeddings do **not** rank `bb_search`: `bb_search` has **no semantic mode**
+  (modes: `symbols` / `content` / `symbol` / `file`). Semantic retrieval is
+  `bb_ask`-only. No doc or UI may imply `bb_search` ranks semantically.
+- Embedding rows are **not** validated against `content_hash` at query time —
+  staleness is proven by the file-level hash (`checkResultsFreshness`), and
+  semantic results are flagged `stale` when the underlying file changed on
+  disk. Verified in Q5 case 10.
+
+## 7. Scalability
+
+The candidate scan is capped at `BB_VECTOR_SCAN_CAP` (default 20000) rows,
+ordered deterministically by `entity_id`, so truncation is stable; when the
+cap is hit, `provenance.truncated` is `true` and `metadata.scannedRows` reports
+exactly how many rows were loaded (Q5 case 11 proves this with a closed set).
+
+## 8. Verification references
+
+- `script/m18/verify-recall.ts` — recall@k / rank / latency / model labels
+- `script/m18/verify-ask.ts` — bb_ask warm/cold precision + honesty floors
+- `script/m18/verify-coverage.ts` — coverage reporting + phase labeling
+- `script/m18/verify-edge-cases.ts` — corrupt / stale / empty / mismatch
+- `script/m18/verify-scan-cap.ts` — scan-cap truncation (child process)
+- Evidence: `validation/m18/results/Q1.json .. Q5.json`

package/docs/README.md

@@ -0,0 +1,178 @@
+# Documentation index
+
+`bb`'s documentation lives in the repository. This page is the map. The pages that ship with the npm package
+are `README.md`, `EMBEDDINGS.md`, `CHANGELOG.md`, `docs/man/`, `docs/tools/`, `docs/indexing/` and
+`docs/data/public-tools.json`; everything else is repository documentation and internal evidence.
+
+## Repository map
+
+Where things live, and what each top-level directory is for (this is the folder-level index — one map, kept
+in one place, instead of a README in every directory that can drift from the code):
+
+| Path | What it is |
+| --- | --- |
+| `src/` | The product source. `src/ast/` parses languages into evidence; `src/storage/` owns the SQLite layer; `src/intelligence/` contains the search/semantic/graph/analysis/pattern/security/doc/visualization/migration/edit/health engines; `src/mcp/` (registry, descriptors, handlers) and `src/cli/` are the adapters; `src/lsp/` is the language server; `src/setup/` and `src/config/` own setup, doctor and configuration. Module contracts: [`ARCHITECTURE.md`](./ARCHITECTURE.md). |
+| `script/` | The verification machinery: top-level `script/verify-*.ts` gates for shipped surfaces, `script/mNN/` milestone gates and instruments, `script/m14/` the analyzer suite (specs, fixtures, instrument sets), `script/tool-docs/` the curation sources for the generated tool references, `script/perf/` measurement probes, `script/lib/` shared harness. What is a gate vs an instrument, and which runtime each needs: [`../script/README.md`](../script/README.md). |
+| `validation/` | Evidence: gate results (JSON) and human-readable review/verification pages, organised by milestone (`validation/m14/`, `m25/`, …). Gate-produced, not hand-edited; provenance and re-run rules: [`../validation/README.md`](../validation/README.md). |
+| `docs/` | This documentation tree (indexing, tools, analyzers, architecture and the historical milestone records). |
+| `ide/` | Editor integrations: VS Code-family extension, Zed extension, LSP4IJ template for JetBrains IDEs, Helix config. Start at [`../ide/README.md`](../ide/README.md). |
+| `grammars/` | Vendored/pinned Tree-sitter grammar sources and their `PIN.md`/`PROVENANCE.md` records; the pinned WASM binaries live in `wasm/`. |
+| `wasm/` | The pinned Tree-sitter WASM artifacts the engine loads; digest pins live in `src/ast/engine.ts`. |
+| `bin/` | The `bb` launcher; `dist/` is build output (ignored), and the compiled path is what npm/Node runs. |
+| `test/` | Harness suites and fixtures that are not part of the gate suite (CLI surface, indexing lifecycle, demo fixtures): [`../test/README.md`](../test/README.md). |
+| `plans/` | Small machine-readable plan inputs still read by milestone instruments (for example the M11 corpus plan). |
+
+`node_modules/`, `dist/`, local worktrees and other
+machine-local directories are ignored; they are not repository content.
+
+## Start here
+
+| If you want to… | Read |
+| --- | --- |
+| index a project, or understand what each flag actually does | [`indexing/README.md`](./indexing/README.md) |
+| recover an interrupted index run | [`indexing/resume.md`](./indexing/resume.md) |
+| understand why indexing is slow | [`indexing/performance.md`](./indexing/performance.md) |
+| fix "the workers don't start" | [`indexing/troubleshooting.md`](./indexing/troubleshooting.md) |
+| configure embeddings, `bb setup`, `bb doctor` | [`indexing/embeddings.md`](./indexing/embeddings.md) and [`../EMBEDDINGS.md`](../EMBEDDINGS.md) |
+| see what discovery actually scans | [`indexing/discovery-matrix.md`](./indexing/discovery-matrix.md) |
+| look up a tool | [`tools/`](./tools/) (one page per tool), or [`data/public-tools.json`](./data/public-tools.json) for machine-readable contracts |
+| understand the module layout | [`ARCHITECTURE.md`](./ARCHITECTURE.md) and [`GRAPH_ENGINE.md`](./GRAPH_ENGINE.md) |
+| read an analyzer's contract | [`analyzers/README.md`](./analyzers/README.md) — one canonical page per analyzer |
+| use the IDE extension (VS Code, Cursor, Antigravity, Windsurf, Zed, Helix, JetBrains) | [`IDE-PROGRAM-STATUS.md`](./IDE-PROGRAM-STATUS.md) for the current evidence-backed state; [`IDE-LAUNCH-TEST-PLAN.md`](./IDE-LAUNCH-TEST-PLAN.md) for the gates and the human test pass |
+
+## Indexing
+
+`bb index` is the deepest surface here, and the one most likely to be wrong in a way that costs you time rather
+than failing loudly. It has its own directory, and those pages are gated against the code
+(`script/verify-docs-indexing.ts` asserts every env var, flag, default and quoted output line exists in `src/`):
+
+- [`indexing/README.md`](./indexing/README.md) — the legal flag combinations, and how to read a run's report on itself
+- [`indexing/resume.md`](./indexing/resume.md) — what survives Ctrl-C, and the `skipped-unchanged` trap
+- [`indexing/performance.md`](./indexing/performance.md) — measured timings, each citing an evidence file, plus what has *not* been measured
+- [`indexing/troubleshooting.md`](./indexing/troubleshooting.md) — the worker pool: causes, env vars and which are test-only
+- [`indexing/embeddings.md`](./indexing/embeddings.md) — provider precedence, every `BB_EMBEDDING_*` variable, retries
+- [`indexing/environment-variables.md`](./indexing/environment-variables.md) — every `BB_*` variable and whether it is a user knob
+- [`indexing/discovery-matrix.md`](./indexing/discovery-matrix.md) — what is scanned and what is not, case by case, verified by its gate
+- [`indexing/PERFORMANCE-FINDINGS.md`](./indexing/PERFORMANCE-FINDINGS.md) — local measurements with no evidence file; read its provenance warning first
+
+## Per-tool references
+
+The sixteen public tools are documented individually and generated from the tool registry so the two cannot
+drift (`script/verify-tool-docs.ts` writes [`data/public-tools.json`](./data/public-tools.json) and
+[`tools/`](./tools/)):
+
+| Tool | Documentation | | Tool | Documentation |
+| --- | --- | --- | --- | --- |
+| `bb_analyze` | [`tools/bb_analyze.md`](./tools/bb_analyze.md) | | | `bb_annotate` | [`tools/bb_annotate.md`](./tools/bb_annotate.md) |
+| `bb_annotations` | [`tools/bb_annotations.md`](./tools/bb_annotations.md) | | | `bb_ask` | [`tools/bb_ask.md`](./tools/bb_ask.md) |
+| `bb_cross_language` | [`tools/bb_cross_language.md`](./tools/bb_cross_language.md) | | | `bb_doc` | [`tools/bb_doc.md`](./tools/bb_doc.md) |
+| `bb_edit` | [`tools/bb_edit.md`](./tools/bb_edit.md) | | | `bb_graph` | [`tools/bb_graph.md`](./tools/bb_graph.md) |
+| `bb_health` | [`tools/bb_health.md`](./tools/bb_health.md) | | | `bb_impact` | [`tools/bb_impact.md`](./tools/bb_impact.md) |
+| `bb_index` | [`tools/bb_index.md`](./tools/bb_index.md) | | | `bb_migrate` | [`tools/bb_migrate.md`](./tools/bb_migrate.md) |
+| `bb_patterns` | [`tools/bb_patterns.md`](./tools/bb_patterns.md) | | | `bb_search` | [`tools/bb_search.md`](./tools/bb_search.md) |
+| `bb_security` | [`tools/bb_security.md`](./tools/bb_security.md) | | | `bb_visualize` | [`tools/bb_visualize.md`](./tools/bb_visualize.md) |
+
+The public surface is **exactly these sixteen tools**. Test intelligence is not a separate
+tool: it is the `test_mapping`, `test_gaps`, `test_impact` and `test_quality` modes of
+`bb_analyze` (deep guides: [`BB_TEST_INTEL.md`](./BB_TEST_INTEL.md),
+[`BB_TEST_QUALITY.md`](./BB_TEST_QUALITY.md)). The per-mode names retired before 0.3.0
+(`bb_select`, `bb_write`, `bb_test_quality`, …) are not callable; they are listed only in the
+retired-name tables of [`TOOL-REGISTRY.md`](./TOOL-REGISTRY.md) and
+the tool pages under [`docs/tools/`](./tools/) so readers can migrate off them.
+
+`npm run docs:verify` regenerates the artifact set and fails on drift. `man bb` ships with the package and
+points at these directories, so an installed user has the same reference. There are no hidden tools or aliases:
+the `0.3.0` release removed them, and `bb --help` lists exactly these sixteen.
+
+Ownership and scope for future changes is tracked in the private ownership ledger; the public
+contract for each tool is its page under `docs/tools/`.
+
+### Deep guides
+
+Each generated reference page links its hand-written deep guides under **Deep guides**; they carry the
+operational detail the generated contract does not: worked runs, measured costs, refusal behaviour and the
+limits a caller must carry forward.
+
+| Tool | Deep guide(s) |
+| --- | --- |
+| `bb_analyze` | [`BB_ANALYZE.md`](./BB_ANALYZE.md), [`BB_TEST_INTEL.md`](./BB_TEST_INTEL.md), [`BB_TEST_QUALITY.md`](./BB_TEST_QUALITY.md), [`BB_ANALYSIS_HISTORY.md`](./BB_ANALYSIS_HISTORY.md) |
+| `bb_annotate` | [`BB_ANNOTATE.md`](./BB_ANNOTATE.md) |
+| `bb_annotations` | [`BB_ANNOTATIONS.md`](./BB_ANNOTATIONS.md) |
+| `bb_ask` | [`BB_ASK.md`](./BB_ASK.md) |
+| `bb_cross_language` | [`BB_CROSS_LANGUAGE.md`](./BB_CROSS_LANGUAGE.md) |
+| `bb_doc` | [`BB_DOC.md`](./BB_DOC.md), [`BB_DOC_REFERENCE.md`](./BB_DOC_REFERENCE.md), [`BB_DOC_EXAMPLES.md`](./BB_DOC_EXAMPLES.md) |
+| `bb_edit` | [`BB_EDIT.md`](./BB_EDIT.md) |
+| `bb_graph` | [`GRAPH_ENGINE.md`](./GRAPH_ENGINE.md) — the graph model rather than a per-tool guide |
+| `bb_health` | [`BB_HEALTH.md`](./BB_HEALTH.md) |
+| `bb_impact` | [`BB_IMPACT.md`](./BB_IMPACT.md) |
+| `bb_index` | [`indexing/README.md`](./indexing/README.md) — flags, resume, performance, troubleshooting |
+| `bb_migrate` | [`BB_MIGRATE.md`](./BB_MIGRATE.md) |
+| `bb_patterns` | [`BB_PATTERNS.md`](./BB_PATTERNS.md) |
+| `bb_search` | [`BB_SEARCH.md`](./BB_SEARCH.md) |
+| `bb_security` | [`BB_SECURITY.md`](./BB_SECURITY.md) |
+| `bb_visualize` | [`BB_VISUALIZE.md`](./BB_VISUALIZE.md) |
+
+The deep-guide set is itself gated: `script/m31/verify-bb-docs-truth.ts` extracts every testable claim from
+these pages (JSON invocations validated against the live registry, mode lists, parameter tables, output
+fields against the behavioural corpus, mutability claims) and records the claims it cannot verify as declared
+unverifiable in `validation/m31/results/D2.json`.
+
+### Historical documents at this level
+
+Milestone records are evidence of what was checked when, not current behaviour. That applies to `docs/m*/`
+directories and equally to the legacy milestone records kept directly under `docs/` (`M10-*`, `M13-*`,
+`M14-*`, `M26-*`, `M55-*`, `MCP_*`). They are not contracts and should not be read as current tool
+documentation; the per-tool pages and deep guides above are. Older plans and migration guides that no longer
+describe the shipped surface are kept in the private release archive rather than mixed into the
+live set.
+
+## Analyzers
+
+One canonical page per analyzer: [`analyzers/README.md`](./analyzers/README.md) indexes all fifty
+(`docs/analyzers/<language>.md`). Those pages are measured by `script/m14/verify-analyzer-docs.ts` against
+[`analyzers/TEMPLATE.md`](./analyzers/TEMPLATE.md) (eight measured sections, plus an explained
+relationship-ontology table; the measured worklist is on the analyzer index). Older analyzer journals
+that tracked code and validation evidence are kept in the private archive rather than
+mixed into the canonical set.
+
+## Architecture and programs
+
+- [`ARCHITECTURE.md`](./ARCHITECTURE.md) — module layout, evidence and mutation contracts
+- [`GRAPH_ENGINE.md`](./GRAPH_ENGINE.md) — the relationship graph: what each edge means and how it is resolved
+- [`../EMBEDDINGS.md`](../EMBEDDINGS.md) — the shipped embeddings guide (vector spaces, coverage, retrieval)
+- [`IDE-HONESTY-CONTRACT.md`](./IDE-HONESTY-CONTRACT.md) — what the IDE integration promises and does not
+- [`TOOL-REGISTRY.md`](./TOOL-REGISTRY.md) and [`TOOLS_OVERVIEW.md`](./TOOLS_OVERVIEW.md) — registry and surface overview
+
+## Records and evidence
+
+- `docs/m*/` — milestone records (M10–M63) and their audits. They are evidence of what was checked when, not
+  current behaviour; prefer the per-topic pages above.
+- `validation/` — gate results and measurement evidence (JSON and markdown). Tracked evidence is referenced by
+  gates; `script/m14/verify-self-contained.ts` checks the parts the machinery actually reads.
+- [`validation/languages/results/tool-matrix.json`](../validation/languages/results/tool-matrix.json) — the language × tool
+  matrix evidence: every analyzer language indexed from a real fixture and exercised through the public tools.
+  Contract and extension steps: [`LANGUAGE-TOOL-MATRIX.md`](./LANGUAGE-TOOL-MATRIX.md).
+
+### Superseded copies
+
+A page may keep its previous revision as `<name>.superseded-<commit>.md` next to it. That is allowed only
+while tracked evidence still cites the old content; the copy is evidence, not documentation, and it is
+removed from the live set once no gate or ledger references it. The current revision is always the unsuffixed
+file. Do not treat a superseded copy as the contract when both exist.
+
+## Verifying a doc
+
+If a doc tells you something and you have reason to doubt it, check rather than trust:
+
+```bash
+npx tsx script/verify-docs-indexing.ts          # indexing pages, against src/
+npx tsx script/verify-tool-docs.ts              # regenerates and compares the tool references
+npx tsx script/m14/verify-analyzer-docs.ts      # analyzer pages, sections and ontology cells
+npm run verify:language-matrix                  # every analyzer language × the public tool layer
+```
+
+The matrix's contract page ([LANGUAGE-TOOL-MATRIX.md](./LANGUAGE-TOOL-MATRIX.md)) also explains how to add a
+language or a new tool check without rewriting the gate.
+
+Docs that assert behaviour the code does not implement are worse than no docs: they cost time while looking
+authoritative.