neurain 0.1.0-alpha.3 → 0.1.0-alpha.5

package/CHANGELOG.md

@@ -4,6 +4,16 @@
 
 - No unreleased changes recorded.
 
+## 0.1.0-alpha.5
+
+- Performance (recall processing): cut recall/search processing time without changing results. The semantic scorer now prepares the query once and precomputes per-doc trigrams (instead of re-tokenizing the query and rebuilding `charTrigrams` per document), and the lexical BM25 counts term frequency with an index loop instead of `String.split`. Measured: `recall hybrid-search` ~1234ms -> ~970ms, `semantic-search` ~1031ms -> ~750ms (warm median), with byte-identical ranking/scores/matched_terms (golden-verified) and npm test 153/153.
+
+
+## 0.1.0-alpha.4
+
+- Performance: lazy dynamic-import CLI dispatch. Each command now imports only its own `core/*.mjs` module on demand instead of loading all ~54 command modules on every invocation. Engine subprocess latency drops ~60-75ms across the board (`tidy` 150->90ms, `structure-audit` 120->60ms, `--help`/`--version` 50ms), with no change to the command surface or behavior (npm test 153/153; reviewed).
+
+
 ## 0.1.0-alpha.3
 
 - Destination resolver (`resolve_target.mjs`): `capture` and `compile` now derive one canonical target path from area, write intent, and sensitivity instead of leaving placement to the caller. Private, multi-area, and decision-required items still gate on the `<N>건 저장 진행` confirmation and are never auto-filed.

package/README.md

@@ -204,7 +204,7 @@ It exposes read/capture/scan/preview tools only. It does not silently compile, p
 
 ## Status
 
-This is `0.1.0-alpha.3`. It is not a public SaaS GA release. The alpha exists to prove installability, local-first onboarding, Codex, Claude, Gemini, and Runtime connectivity, plus safety receipts.
+This is `0.1.0-alpha.5`. It is not a public SaaS GA release. The alpha exists to prove installability, local-first onboarding, Codex, Claude, Gemini, and Runtime connectivity, plus safety receipts.
 
 Alpha publish command:
 

package/docs/development-status.en.md

@@ -2,8 +2,8 @@
 
 Version: v0.1
 Last updated: 2026-06-19 KST
-Package: `neurain@0.1.0-alpha.3`
-Latest documented commit: `f3d3849 docs: document auto-organization wave (E27) for alpha.3`
+Package: `neurain@0.1.0-alpha.5`
+Latest documented commit: `6305d3d perf(recall): cut recall processing time, results byte-identical`
 
 This document is the canonical product development snapshot for the public package. It tracks what is shipped, what has evidence, and what must not be claimed yet.
 

package/docs/development-status.kr.md

@@ -2,8 +2,8 @@
 
 Version: v0.1
 Last updated: 2026-06-19 KST
-Package: `neurain@0.1.0-alpha.3`
-Latest documented commit: `f3d3849 docs: document auto-organization wave (E27) for alpha.3`
+Package: `neurain@0.1.0-alpha.5`
+Latest documented commit: `6305d3d perf(recall): cut recall processing time, results byte-identical`
 
 이 문서는 public package 기준의 canonical 개발 상태 스냅샷입니다. 무엇이 shipped인지, 어떤 증거가 있는지, 아직 주장하면 안 되는 것이 무엇인지 함께 기록합니다.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neurain",
-  "version": "0.1.0-alpha.3",
+  "version": "0.1.0-alpha.5",
   "description": "Local-first Neurain Knowledge OS CLI and MCP connector.",
   "type": "module",
   "license": "Apache-2.0",

package/src/cli.mjs

@@ -1,60 +1,62 @@
 import { spawnSync } from 'node:child_process';
 import path from 'node:path';
-import { adoptCommand, rollbackAdoption, scanAdoption } from './core/adopt.mjs';
-import { answerCommand } from './core/answer_eval.mjs';
-import { connectCommand } from './core/connect.mjs';
-import { capabilitiesCommand } from './core/capabilities.mjs';
-import { curatorCommand } from './core/curator.mjs';
-import { daemonCommand } from './core/daemon.mjs';
-import { doctorCommand } from './core/doctor.mjs';
-import { initCommand } from './core/init.mjs';
-import { journalCommand } from './core/journal.mjs';
-import { lessonsCommand } from './core/lessons.mjs';
-import { lifecycleCommand } from './core/lifecycle.mjs';
-import { liveCasesCommand } from './core/live_cases.mjs';
-import { onboardCommand } from './core/onboard.mjs';
-import { recapCommand } from './core/recap.mjs';
-import { recallCommand } from './core/recall.mjs';
-import { reviewCommand } from './core/review_worker.mjs';
-import { statusCommand } from './core/status.mjs';
-import { digestCommand } from './core/digest.mjs';
-import { queueCommand } from './core/queue.mjs';
-import { reviewQueueCommand } from './core/review_queue.mjs';
-import { routeCommand } from './core/route.mjs';
-import { planWritebackCommand } from './core/plan_writeback.mjs';
-import { flushCommand, sessionFlushCommand } from './core/flush.mjs';
-import { compileCommand } from './core/compile_desk.mjs';
-import { planReceiptCommand } from './core/plan_receipt.mjs';
-import { sourceDigestCommand } from './core/source_digest_gen.mjs';
-import { stageCommand } from './core/stage.mjs';
-import { queueArchiveCommand } from './core/queue_archive.mjs';
-import { captureCommand } from './core/capture_durable.mjs';
-import { completeCommand } from './core/complete.mjs';
-import { linkCheckCommand } from './core/link_check.mjs';
-import { sessionLintCommand } from './core/session_lint.mjs';
-import { labelCommand } from './core/label.mjs';
-import { orphansCommand } from './core/orphans.mjs';
-import { hubsCommand } from './core/hubs.mjs';
-import { areaIndexCommand } from './core/area_index.mjs';
-import { reindexCommand } from './core/reindex.mjs';
-import { lintCommand } from './core/lint.mjs';
-import { tidyCommand } from './core/tidy.mjs';
-import { structureAuditCommand } from './core/structure_audit.mjs';
-import { fileCommand } from './core/file_loose.mjs';
-import { organizeCommand } from './core/organize.mjs';
-import { sessionPulseCommand } from './core/session_pulse.mjs';
-import { syncCommand } from './core/sync.mjs';
-import { healthCommand } from './core/health.mjs';
-import { memoryCommand } from './core/memory.mjs';
-import { retentionCommand } from './core/retention.mjs';
-import { freezeCommand } from './core/freeze.mjs';
-import { memoryWriteCommand } from './core/memory_write_cli.mjs';
-import { backupCommand } from './core/backup.mjs';
-import { schedulerCommand } from './core/scheduler.mjs';
-import { searchCommand } from './core/search.mjs';
-import { watchCommand } from './core/watch.mjs';
-import { wrapCommand } from './core/wrap.mjs';
-import { startMcpServer } from './mcp/server.mjs';
+
+const COMMAND_HANDLERS = {
+  init: async (args) => (await import('./core/init.mjs')).initCommand(args),
+  onboard: async (args) => (await import('./core/onboard.mjs')).onboardCommand(args),
+  answer: async (args) => (await import('./core/answer_eval.mjs')).answerCommand(args),
+  doctor: async (args) => (await import('./core/doctor.mjs')).doctorCommand(args),
+  search: async (args) => (await import('./core/search.mjs')).searchCommand(args),
+  connect: async (args) => (await import('./core/connect.mjs')).connectCommand(args),
+  journal: async (args) => (await import('./core/journal.mjs')).journalCommand(args),
+  lifecycle: async (args) => (await import('./core/lifecycle.mjs')).lifecycleCommand(args),
+  'live-cases': async (args) => (await import('./core/live_cases.mjs')).liveCasesCommand(args),
+  lessons: async (args) => (await import('./core/lessons.mjs')).lessonsCommand(args),
+  curator: async (args) => (await import('./core/curator.mjs')).curatorCommand(args),
+  daemon: async (args) => (await import('./core/daemon.mjs')).daemonCommand(args),
+  recall: async (args) => (await import('./core/recall.mjs')).recallCommand(args),
+  reindex: async (args) => (await import('./core/reindex.mjs')).reindexCommand(args),
+  status: async (args) => (await import('./core/status.mjs')).statusCommand(args),
+  digest: async (args) => (await import('./core/digest.mjs')).digestCommand(args),
+  queue: async (args) => (await import('./core/queue.mjs')).queueCommand(args),
+  'review-queue': async (args) => (await import('./core/review_queue.mjs')).reviewQueueCommand(args),
+  route: async (args) => (await import('./core/route.mjs')).routeCommand(args),
+  'plan-writeback': async (args) => (await import('./core/plan_writeback.mjs')).planWritebackCommand(args),
+  flush: async (args) => (await import('./core/flush.mjs')).flushCommand(args),
+  'session-flush': async (args) => (await import('./core/flush.mjs')).sessionFlushCommand(args),
+  compile: async (args) => (await import('./core/compile_desk.mjs')).compileCommand(args),
+  'plan-receipt': async (args) => (await import('./core/plan_receipt.mjs')).planReceiptCommand(args),
+  'source-digest': async (args) => (await import('./core/source_digest_gen.mjs')).sourceDigestCommand(args),
+  stage: async (args) => (await import('./core/stage.mjs')).stageCommand(args),
+  'queue-archive': async (args) => (await import('./core/queue_archive.mjs')).queueArchiveCommand(args),
+  capture: async (args) => (await import('./core/capture_durable.mjs')).captureCommand(args),
+  complete: async (args) => (await import('./core/complete.mjs')).completeCommand(args),
+  'link-check': async (args) => (await import('./core/link_check.mjs')).linkCheckCommand(args),
+  'session-lint': async (args) => (await import('./core/session_lint.mjs')).sessionLintCommand(args),
+  label: async (args) => (await import('./core/label.mjs')).labelCommand(args),
+  orphans: async (args) => (await import('./core/orphans.mjs')).orphansCommand(args),
+  hubs: async (args) => (await import('./core/hubs.mjs')).hubsCommand(args),
+  'area-index': async (args) => (await import('./core/area_index.mjs')).areaIndexCommand(args),
+  lint: async (args) => (await import('./core/lint.mjs')).lintCommand(args),
+  tidy: async (args) => (await import('./core/tidy.mjs')).tidyCommand(args),
+  'structure-audit': async (args) => (await import('./core/structure_audit.mjs')).structureAuditCommand(args),
+  file: async (args) => (await import('./core/file_loose.mjs')).fileCommand(args),
+  organize: async (args) => (await import('./core/organize.mjs')).organizeCommand(args),
+  'session-pulse': async (args) => (await import('./core/session_pulse.mjs')).sessionPulseCommand(args),
+  sync: async (args) => (await import('./core/sync.mjs')).syncCommand(args),
+  health: async (args) => (await import('./core/health.mjs')).healthCommand(args),
+  memory: async (args) => (await import('./core/memory.mjs')).memoryCommand(args),
+  retention: async (args) => (await import('./core/retention.mjs')).retentionCommand(args),
+  freeze: async (args) => (await import('./core/freeze.mjs')).freezeCommand(args),
+  'memory-write': async (args) => (await import('./core/memory_write_cli.mjs')).memoryWriteCommand(args),
+  backup: async (args) => (await import('./core/backup.mjs')).backupCommand(args),
+  capabilities: async (args) => (await import('./core/capabilities.mjs')).capabilitiesCommand(args),
+  recap: async (args) => (await import('./core/recap.mjs')).recapCommand(args),
+  watch: async (args) => (await import('./core/watch.mjs')).watchCommand(args),
+  review: async (args) => (await import('./core/review_worker.mjs')).reviewCommand(args),
+  scheduler: async (args) => (await import('./core/scheduler.mjs')).schedulerCommand(args),
+  wrap: async (args) => (await import('./core/wrap.mjs')).wrapCommand(args),
+};
 
 export async function runCli(argv) {
   const [command, ...rest] = argv;
@@ -69,65 +71,17 @@ export async function runCli(argv) {
   }
 
   const args = parseArgs(rest);
-  if (command === 'init') return render(await initCommand(args));
-  if (command === 'onboard') return render(await onboardCommand(args));
   if (command === 'adopt') {
+    const { adoptCommand, rollbackAdoption } = await import('./core/adopt.mjs');
     if (args.rollback) return render(await rollbackAdoption(args));
     return render(await adoptCommand(args));
   }
-  if (command === 'answer') return render(await answerCommand(args));
-  if (command === 'doctor') return render(await doctorCommand(args));
-  if (command === 'search') return render(await searchCommand(args));
-  if (command === 'connect') return render(await connectCommand(args));
-  if (command === 'journal') return render(await journalCommand(args));
-  if (command === 'lifecycle') return render(await lifecycleCommand(args));
-  if (command === 'live-cases') return render(await liveCasesCommand(args));
-  if (command === 'lessons') return render(await lessonsCommand(args));
-  if (command === 'curator') return render(await curatorCommand(args));
-  if (command === 'daemon') return render(await daemonCommand(args));
-  if (command === 'recall') return render(await recallCommand(args));
-  if (command === 'reindex') return render(await reindexCommand(args));
-  if (command === 'status') return render(await statusCommand(args));
-  if (command === 'digest') return render(await digestCommand(args));
-  if (command === 'queue') return render(await queueCommand(args));
-  if (command === 'review-queue') return render(await reviewQueueCommand(args));
-  if (command === 'route') return render(await routeCommand(args));
-  if (command === 'plan-writeback') return render(await planWritebackCommand(args));
-  if (command === 'flush') return render(await flushCommand(args));
-  if (command === 'session-flush') return render(await sessionFlushCommand(args));
-  if (command === 'compile') return render(await compileCommand(args));
-  if (command === 'plan-receipt') return render(await planReceiptCommand(args));
-  if (command === 'source-digest') return render(await sourceDigestCommand(args));
-  if (command === 'stage') return render(await stageCommand(args));
-  if (command === 'queue-archive') return render(await queueArchiveCommand(args));
-  if (command === 'capture') return render(await captureCommand(args));
-  if (command === 'complete') return render(await completeCommand(args));
-  if (command === 'link-check') return render(await linkCheckCommand(args));
-  if (command === 'session-lint') return render(await sessionLintCommand(args));
-  if (command === 'label') return render(await labelCommand(args));
-  if (command === 'orphans') return render(await orphansCommand(args));
-  if (command === 'hubs') return render(await hubsCommand(args));
-  if (command === 'area-index') return render(await areaIndexCommand(args));
-  if (command === 'lint') return render(await lintCommand(args));
-  if (command === 'tidy') return render(await tidyCommand(args));
-  if (command === 'structure-audit') return render(await structureAuditCommand(args));
-  if (command === 'file') return render(await fileCommand(args));
-  if (command === 'organize') return render(await organizeCommand(args));
-  if (command === 'session-pulse') return render(await sessionPulseCommand(args));
-  if (command === 'sync') return render(await syncCommand(args));
-  if (command === 'health') return render(await healthCommand(args));
-  if (command === 'memory') return render(await memoryCommand(args));
-  if (command === 'retention') return render(await retentionCommand(args));
-  if (command === 'freeze') return render(await freezeCommand(args));
-  if (command === 'memory-write') return render(await memoryWriteCommand(args));
-  if (command === 'backup') return render(await backupCommand(args));
-  if (command === 'capabilities') return render(await capabilitiesCommand(args));
-  if (command === 'recap') return render(await recapCommand(args));
-  if (command === 'watch') return render(await watchCommand(args));
-  if (command === 'review') return render(await reviewCommand(args));
-  if (command === 'scheduler') return render(await schedulerCommand(args));
-  if (command === 'wrap') return render(await wrapCommand(args));
-  if (command === 'mcp') return startMcpServer(args);
+  const handler = COMMAND_HANDLERS[command];
+  if (handler) return render(await handler(args));
+  if (command === 'mcp') {
+    const { startMcpServer } = await import('./mcp/server.mjs');
+    return startMcpServer(args);
+  }
   if (command === 'selftest') return runSelftest();
 
   throw new Error(`Unknown command: ${command}\n\n${helpText()}`);

package/src/core/recall.mjs

@@ -195,9 +195,15 @@ export async function semanticSearchRecall(root, query, { top = 10, host = '', p
     .filter((doc) => doc.sensitivity !== 'private')
     .filter((doc) => !hostFilter || doc.host === hostFilter)
     .filter((doc) => !scopeFilter || doc.scope === scopeFilter);
+  // Prepare the query ONCE, then score every doc against it (avoids re-tokenizing the
+  // query per document). Falls back to provider.score for providers without the fast path.
+  const preparedQuery = prov.prepareQuery ? prov.prepareQuery(text) : null;
   const scored = docs
     .map((doc) => {
-      const scoredDoc = prov.score(text, `${doc.title} ${doc.body}`);
+      const docText = `${doc.title} ${doc.body}`;
+      const scoredDoc = (preparedQuery && prov.scorePrepared)
+        ? prov.scorePrepared(preparedQuery, docText)
+        : prov.score(text, docText);
       return { doc, score: Number(scoredDoc.score || 0), matched_terms: scoredDoc.matched_terms || [] };
     })
     .filter((item) => item.score >= floor)

package/src/core/recall_lexical.mjs

@@ -192,7 +192,10 @@ export function lexicalSearchWithContext(ctx, query, { top = 10, maxPerLayer = 3
 
     let bm25 = 0;
     for (const term of searchTerms) {
-      const tf = lower.split(term).length - 1;
+      // Non-overlapping occurrence count (identical to `lower.split(term).length - 1`)
+      // without allocating the split array on every doc/term pair.
+      let tf = 0;
+      for (let i = lower.indexOf(term); i !== -1; i = lower.indexOf(term, i + term.length)) tf += 1;
       if (tf === 0) continue;
       const denom = tf + BM25_K1 * (1 - BM25_B + (BM25_B * length) / avgLength);
       bm25 += (idf[term] || 0) * ((tf * (BM25_K1 + 1)) / denom);

package/src/core/semantic.mjs

@@ -117,9 +117,13 @@ function charTrigrams(token) {
 export function fuzzyOverlap(a, b) {
   if (!a || !b) return 0;
   if (a === b) return 1;
-  const ga = charTrigrams(a);
-  const gb = charTrigrams(b);
-  if (!ga.size || !gb.size) return 0;
+  return trigramJaccard(charTrigrams(a), charTrigrams(b));
+}
+
+// Jaccard over two PRE-COMPUTED trigram sets (same math as fuzzyOverlap's tail),
+// so the per-doc fuzzy loop can reuse cached trigrams instead of rebuilding them.
+function trigramJaccard(ga, gb) {
+  if (!ga || !gb || !ga.size || !gb.size) return 0;
   let inter = 0;
   for (const g of ga) if (gb.has(g)) inter += 1;
   return inter / (ga.size + gb.size - inter);
@@ -127,13 +131,25 @@ export function fuzzyOverlap(a, b) {
 
 // Deterministic lexical-semantic score of a query against a document body.
 // Returns { score: 0..1 normalized by query length, matched_terms: [...] }.
-export function lexicalSemanticScore(query, docText) {
-  const queryExpanded = tokenize(query).map(expandToken);
+// Prepare a query ONCE (tokenize + expand) so a corpus scan can reuse it across all
+// docs instead of re-tokenizing the query per document (the per-doc hot path).
+export function prepareSemanticQuery(query) {
+  // Precompute each term's trigrams ONCE so the per-doc fuzzy loop never rebuilds them.
+  return tokenize(query).map(expandToken).map((q) => ({ ...q, trigrams: charTrigrams(q.stem) }));
+}
+
+// Score a pre-prepared query against a document body. Behaviour is identical to
+// lexicalSemanticScore; only the query preparation is hoisted out.
+export function scorePreparedSemantic(queryExpanded, docText) {
   if (!queryExpanded.length) return { score: 0, matched_terms: [] };
   const docTokens = tokenize(docText).map(expandToken);
   if (!docTokens.length) return { score: 0, matched_terms: [] };
   const docStems = new Set(docTokens.map((d) => d.stem));
   const docCanons = new Set(docTokens.map((d) => d.canon).filter(Boolean));
+  // Build each unique doc-stem's trigrams ONCE per doc (was recomputed per query term
+  // inside fuzzyOverlap -> the charTrigrams hot path).
+  const docStemTrigrams = new Map();
+  for (const s of docStems) docStemTrigrams.set(s, charTrigrams(s));
   const matched = [];
   let total = 0;
   for (const q of queryExpanded) {
@@ -142,9 +158,11 @@ export function lexicalSemanticScore(query, docText) {
     if (docStems.has(q.stem)) { best = 1; how = 'exact'; }
     else if (q.canon && docCanons.has(q.canon)) { best = 0.75; how = 'synonym'; }
     else {
-      // fuzzy: best trigram overlap against any doc stem (typos / variants)
+      // fuzzy: best trigram overlap against any doc stem (typos / variants), using
+      // the precomputed query + doc-stem trigrams instead of rebuilding them.
+      const qTri = q.trigrams || charTrigrams(q.stem);
       for (const d of docStems) {
-        const ov = fuzzyOverlap(q.stem, d);
+        const ov = trigramJaccard(qTri, docStemTrigrams.get(d));
         if (ov > best) { best = ov; how = 'fuzzy'; }
       }
       if (best < 0.6) best = 0;
@@ -155,6 +173,10 @@ export function lexicalSemanticScore(query, docText) {
   return { score: Number((total / queryExpanded.length).toFixed(4)), matched_terms: matched };
 }
 
+export function lexicalSemanticScore(query, docText) {
+  return scorePreparedSemantic(prepareSemanticQuery(query), docText);
+}
+
 const PROVIDERS = new Map();
 
 export function registerProvider(name, impl) {
@@ -181,6 +203,12 @@ registerProvider('local-lexical', {
   expandQuery(query) {
     return tokenize(query).map(expandToken);
   },
+  prepareQuery(query) {
+    return prepareSemanticQuery(query);
+  },
+  scorePrepared(prepared, docText) {
+    return scorePreparedSemantic(prepared, docText);
+  },
   score(query, docText) {
     return lexicalSemanticScore(query, docText);
   },