neurain 0.1.0-alpha.6 → 0.1.0-alpha.7

package/CHANGELOG.md

@@ -4,6 +4,15 @@
 
 - No unreleased changes recorded.
 
+## 0.1.0-alpha.7
+
+- Hardening (recall perf, from an adversarial review): lock the "byte-identical results" claim and tighten the fast-path contracts, with no change to ranking/scores (golden-identical).
+
+  - Added `test/perf_recall_equivalence.test.mjs`: an oracle test that `countOccurrences` equals `split(term).length - 1` for every non-empty term (overlap/unicode/surrogate cases included), a proof that `scorePreparedSemantic` (prepare-once + per-doc trigram precompute) equals a naive per-doc reference scorer, a determinism check, and a guard that the shared corpus selector keeps private + secret-bearing files out of every branch.
+  - Extracted the lexical BM25 term-frequency count into an exported `countOccurrences(haystack, needle)` with an empty-needle guard (returns 0) so the index-loop form can never spin even if the term filter changes.
+  - `buildLexicalContext` now throws if a caller passes shared `markdownFiles` together with an `area` (the only safe share is whole-vault; an area context selects a strict subset, so this prevents a future caller from silently widening the corpus).
+  - `prepareSemanticQuery` now returns a frozen prepared query, and the provider fast-path contract (prepared query is immutable, no cross-call mutable state) is documented on the default provider.
+
 ## 0.1.0-alpha.6
 
 - Performance (hybrid recall): `hybrid-search` now walks the markdown corpus ONCE and shares it across its semantic and routed-lexical branches instead of each branch re-walking and re-reading the whole vault. The walk is shared only when no `--area` is set (the two branches then select the same whole-vault corpus); with an area they still walk independently. Results stay byte-identical (golden-verified) because the shared file list is exactly what each branch would have walked. Measured: `recall hybrid-search` ~970ms -> ~763ms (warm median); combined with alpha.5 that is ~1234ms -> ~763ms (-38%). npm test 153/153.

package/README.md

@@ -204,7 +204,7 @@ It exposes read/capture/scan/preview tools only. It does not silently compile, p
 
 ## Status
 
-This is `0.1.0-alpha.6`. It is not a public SaaS GA release. The alpha exists to prove installability, local-first onboarding, Codex, Claude, Gemini, and Runtime connectivity, plus safety receipts.
+This is `0.1.0-alpha.7`. It is not a public SaaS GA release. The alpha exists to prove installability, local-first onboarding, Codex, Claude, Gemini, and Runtime connectivity, plus safety receipts.
 
 Alpha publish command:
 

package/docs/development-status.en.md

@@ -2,8 +2,8 @@
 
 Version: v0.1
 Last updated: 2026-06-20 KST
-Package: `neurain@0.1.0-alpha.6`
-Latest documented commit: `908d51d perf(recall): share one corpus walk across hybrid branches, byte-identical`
+Package: `neurain@0.1.0-alpha.7`
+Latest documented commit: `18bbb9f perf(recall): lock byte-identical claim in CI + harden fast-path contracts`
 
 This document is the canonical product development snapshot for the public package. It tracks what is shipped, what has evidence, and what must not be claimed yet.
 

package/docs/development-status.kr.md

@@ -2,8 +2,8 @@
 
 Version: v0.1
 Last updated: 2026-06-20 KST
-Package: `neurain@0.1.0-alpha.6`
-Latest documented commit: `908d51d perf(recall): share one corpus walk across hybrid branches, byte-identical`
+Package: `neurain@0.1.0-alpha.7`
+Latest documented commit: `18bbb9f perf(recall): lock byte-identical claim in CI + harden fast-path contracts`
 
 이 문서는 public package 기준의 canonical 개발 상태 스냅샷입니다. 무엇이 shipped인지, 어떤 증거가 있는지, 아직 주장하면 안 되는 것이 무엇인지 함께 기록합니다.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neurain",
-  "version": "0.1.0-alpha.6",
+  "version": "0.1.0-alpha.7",
   "description": "Local-first Neurain Knowledge OS CLI and MCP connector.",
   "type": "module",
   "license": "Apache-2.0",

package/src/core/recall_lexical.mjs

@@ -30,6 +30,20 @@ import {
 import { factsFor, loadFactIntel } from './recall_facts.mjs';
 
 const sourceIdPattern = /\braw-\d{8}-(?:\d{3}|dryrun)\b/i;
+
+// Count non-overlapping occurrences of `needle` in `haystack`. For a non-empty
+// needle this is exactly `haystack.split(needle).length - 1` but without
+// allocating the split array on every doc/term pair (the hottest BM25 loop). An
+// empty needle returns 0: every term that reaches scoring is filter(Boolean)'d,
+// so this only future-proofs against an infinite loop if that contract ever
+// changes (the split form would never hit it, but the index form would).
+export function countOccurrences(haystack, needle) {
+  if (!needle) return 0;
+  let n = 0;
+  for (let i = haystack.indexOf(needle); i !== -1; i = haystack.indexOf(needle, i + needle.length)) n += 1;
+  return n;
+}
+
 // BM25 content weight relative to the additive structural boosts (vault parity).
 const BM25_WEIGHT = 4;
 const BM25_K1 = 1.5;
@@ -113,11 +127,17 @@ function slugish(value) {
 // the registry, degrading to empty when files are absent.
 export function buildLexicalContext(root, { area = '', recallCfg, intel, facts, aliasMap, markdownFiles } = {}) {
   if (!recallCfg) throw new Error('buildLexicalContext requires recallCfg');
-  const dirs = dirsFromConfig(recallCfg);
-  const classify = makeLayerClassifier(dirs);
   // `markdownFiles`, when given, must be a pre-walked listRecallMarkdownFiles()
   // result for this exact (root, recallCfg, area) selection; a caller that already
   // walked the corpus (hybrid sharing one walk) passes it to skip the redundant walk.
+  // The only safe sharing is whole-vault: an area-scoped context selects a strict
+  // subset, so accepting whole-vault files under an area would silently widen the
+  // corpus and change results. Reject that misuse loudly instead of ranking wrong.
+  if (markdownFiles && area) {
+    throw new Error('buildLexicalContext: markdownFiles can only be shared for a whole-vault context (no area)');
+  }
+  const dirs = dirsFromConfig(recallCfg);
+  const classify = makeLayerClassifier(dirs);
   const files = markdownFiles || listRecallMarkdownFiles(root, recallCfg, { area });
   const baseDocs = files.map(({ rel, text }) => ({
     text,
@@ -195,10 +215,7 @@ export function lexicalSearchWithContext(ctx, query, { top = 10, maxPerLayer = 3
 
     let bm25 = 0;
     for (const term of searchTerms) {
-      // Non-overlapping occurrence count (identical to `lower.split(term).length - 1`)
-      // without allocating the split array on every doc/term pair.
-      let tf = 0;
-      for (let i = lower.indexOf(term); i !== -1; i = lower.indexOf(term, i + term.length)) tf += 1;
+      const tf = countOccurrences(lower, term);
       if (tf === 0) continue;
       const denom = tf + BM25_K1 * (1 - BM25_B + (BM25_B * length) / avgLength);
       bm25 += (idf[term] || 0) * ((tf * (BM25_K1 + 1)) / denom);

package/src/core/semantic.mjs

@@ -135,7 +135,10 @@ function trigramJaccard(ga, gb) {
 // docs instead of re-tokenizing the query per document (the per-doc hot path).
 export function prepareSemanticQuery(query) {
   // Precompute each term's trigrams ONCE so the per-doc fuzzy loop never rebuilds them.
-  return tokenize(query).map(expandToken).map((q) => ({ ...q, trigrams: charTrigrams(q.stem) }));
+  // The prepared query is the SHARED, immutable input to scorePreparedSemantic across an
+  // entire corpus scan: freezing it makes the "no per-call mutable state" contract
+  // enforced, not just documented, so a long-lived process can reuse it safely.
+  return Object.freeze(tokenize(query).map(expandToken).map((q) => Object.freeze({ ...q, trigrams: charTrigrams(q.stem) })));
 }
 
 // Score a pre-prepared query against a document body. Behaviour is identical to
@@ -203,6 +206,11 @@ registerProvider('local-lexical', {
   expandQuery(query) {
     return tokenize(query).map(expandToken);
   },
+  // Provider fast-path contract: prepareQuery returns an IMMUTABLE prepared query
+  // (frozen) that scorePrepared treats as read-only. A provider must not mutate the
+  // prepared object nor keep cross-call mutable state keyed off it, so the same
+  // prepared query is safe to reuse across an entire corpus scan and across calls in
+  // a long-lived process. The default provider is fully stateless.
   prepareQuery(query) {
     return prepareSemanticQuery(query);
   },