neurain 0.1.0-alpha.0

package/src/core/search.mjs

@@ -0,0 +1,54 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { absPath, isTextFile, relPath, walkFiles } from './fs.mjs';
+
+export async function searchCommand(args) {
+  const root = absPath(args._[0]);
+  const query = args.query || args._.slice(1).join(' ');
+  if (!query.trim()) throw new Error('Missing query.');
+  const payload = searchRoot(root, query, { top: Number(args.top || 10), area: args.area || '' });
+  if (args.json) return { json: true, payload };
+  return {
+    text: [
+      '# Neurain search',
+      '',
+      `- Query: ${query}`,
+      ...payload.results.map((item) => `- ${item.path}: score ${item.score}, ${item.snippet}`),
+      payload.results.length ? '' : 'No matches found.',
+    ].filter(Boolean).join('\n'),
+  };
+}
+
+export function searchRoot(root, query, { top = 10, area = '' } = {}) {
+  const terms = String(query).toLowerCase().split(/\s+/).filter(Boolean);
+  const areaPrefix = area ? `10_areas/${String(area).replace(/^_+/, '_')}` : '';
+  const results = [];
+  for (const file of walkFiles(root, { includeRaw: false })) {
+    const rel = relPath(root, file);
+    if (areaPrefix && !rel.startsWith(areaPrefix)) continue;
+    if (!isTextFile(file)) continue;
+    const text = safeRead(file);
+    if (!text) continue;
+    const lower = text.toLowerCase();
+    let score = 0;
+    for (const term of terms) {
+      if (lower.includes(term)) score += 10;
+      if (path.basename(rel).toLowerCase().includes(term)) score += 15;
+    }
+    if (score && rel.startsWith('wiki/')) score += 12;
+    if (score && rel.startsWith('00_system/')) score -= 3;
+    if (!score) continue;
+    const line = text.split(/\r?\n/).find((candidate) => terms.some((term) => candidate.toLowerCase().includes(term))) || '';
+    results.push({ path: rel, score, snippet: line.trim().slice(0, 180) });
+  }
+  results.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+  return { query: String(query), top, results: results.slice(0, top) };
+}
+
+function safeRead(file) {
+  try {
+    return fs.readFileSync(file, 'utf8');
+  } catch {
+    return '';
+  }
+}

package/src/core/secret_scan.mjs

@@ -0,0 +1,143 @@
+// Deterministic, fail-closed secret/credential preflight (W-B). Keyword
+// classifiers only return a label; nothing inspects body text for actual secret
+// VALUES. Any durable canonical write must run this real content scan first and
+// BLOCK on a high-confidence hit, rather than trusting an instruction to "mask
+// secrets". Scope: high-confidence token SHAPES (T1) + labelled BIP39 mnemonics
+// (T2) + context-aware 64-hex keys (T3) + a Shannon-entropy check on long tokens
+// (T4); a result blocks iff it has any high-confidence hit (T5). Faithful,
+// dependency-free port of the vault lib/secret-scan.mjs.
+import fs from 'node:fs';
+
+// T1 — high-confidence token shapes. Each `re` is global.
+const PATTERNS = [
+  { type: 'openai-key', re: /\bsk-[A-Za-z0-9]{20,}\b/g, confidence: 'high' },
+  { type: 'github-token', re: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{30,}\b/g, confidence: 'high' },
+  { type: 'aws-access-key', re: /\bAKIA[0-9A-Z]{16}\b/g, confidence: 'high' },
+  { type: 'google-api-key', re: /\bAIza[0-9A-Za-z\-_]{30,}\b/g, confidence: 'high' },
+  { type: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g, confidence: 'high' },
+  { type: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{6,}\b/g, confidence: 'high' },
+  { type: 'pem-private-key', re: /-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----/g, confidence: 'high' },
+  { type: 'private-key-assignment', re: /\b(?:private[_-]?key|secret[_-]?key|api[_-]?secret)\s*[:=]\s*["']?[A-Za-z0-9+/_\-]{16,}/gi, confidence: 'high' },
+  // 64-hex blobs (ETH key vs public tx/block hash) are handled by findHexKeys
+  // below, which is context-aware so it can block a likely private key without
+  // flooding crypto notes full of tx hashes.
+];
+
+// A key-context keyword on the same line as a 64-hex blob promotes it from a
+// possible public tx/block hash (low) to a likely private key (high). Kept narrow
+// so ordinary on-chain notes full of 0x… hashes do not all become blocking.
+const KEY_CONTEXT_RE = /(?:\bprivate[\s_-]?key\b|\bsecret[\s_-]?key\b|\bpriv[\s_-]?key\b|\bprivkey\b|\bprivatekey\b|\bmnemonic\b|\bseed[\s_-]?phrase\b|\bkeystore\b|\bwallet[\s_-]?key\b|\bdeployer[\s_-]?key\b|\bsigning[\s_-]?key\b|\bkey[\s_-]?material\b)/i;
+
+function entropy(s) {
+  if (!s) return 0;
+  const freq = new Map();
+  for (const ch of s) freq.set(ch, (freq.get(ch) || 0) + 1);
+  let h = 0;
+  for (const n of freq.values()) {
+    const p = n / s.length;
+    h -= p * Math.log2(p);
+  }
+  return h;
+}
+
+function mask(token) {
+  const t = String(token);
+  if (t.length <= 8) return `${t[0]}***`;
+  return `${t.slice(0, 4)}…${t.slice(-2)} (len ${t.length})`;
+}
+
+// T2 — heuristic BIP39 mnemonic: 12 or 24 space-separated lowercase a-z words,
+// each 3-8 chars, on one line (a leading label is stripped first).
+function findMnemonics(text) {
+  const hits = [];
+  for (const rawLine of text.split(/\r?\n/)) {
+    const line = rawLine.replace(/^\s*(?:seed[\s_-]?phrase|mnemonic|recovery[\s_-]?phrase|secret[\s_-]?words?)\s*[:=\-]\s*/i, '');
+    const words = line.trim().split(/\s+/);
+    if ((words.length === 12 || words.length === 24) && words.every((w) => /^[a-z]{3,8}$/.test(w))) {
+      hits.push({ type: 'possible-bip39-mnemonic', confidence: 'high', sample: `${words.length} words: ${words[0]} … ${words[words.length - 1]}` });
+    }
+  }
+  return hits;
+}
+
+// T3 — context-aware 64-hex scan. A 0x-prefixed 64-hex is always a candidate; a
+// bare 64-hex is only flagged when a key keyword is on the line. Key-context
+// upgrades the hit to high (blocking).
+function findHexKeys(text) {
+  const hits = [];
+  const seen = new Set();
+  for (const line of text.split(/\r?\n/)) {
+    const keyCtx = KEY_CONTEXT_RE.test(line);
+    for (const m of line.matchAll(/\b0x[0-9a-fA-F]{64}\b/g)) {
+      const tok = m[0];
+      if (seen.has(tok)) continue;
+      seen.add(tok);
+      hits.push(keyCtx
+        ? { type: 'eth-private-key', confidence: 'high', sample: mask(tok) }
+        : { type: 'hex64-0x', confidence: 'low', sample: mask(tok) });
+    }
+    if (keyCtx) {
+      for (const m of line.matchAll(/\b[0-9a-fA-F]{64}\b/g)) {
+        const tok = m[0];
+        if (seen.has(tok) || seen.has(`0x${tok}`)) continue;
+        seen.add(tok);
+        hits.push({ type: 'hex64-key-context', confidence: 'high', sample: mask(tok) });
+      }
+    }
+  }
+  return hits;
+}
+
+// T4 — high-entropy long token scan (catches secrets without a known prefix).
+function findHighEntropy(text) {
+  const hits = [];
+  const seen = new Set();
+  for (const m of text.matchAll(/[A-Za-z0-9+/=_\-]{32,}/g)) {
+    const tok = m[0];
+    if (seen.has(tok)) continue;
+    seen.add(tok);
+    const h = entropy(tok);
+    const classes = (/[a-z]/.test(tok) ? 1 : 0) + (/[A-Z]/.test(tok) ? 1 : 0) + (/[0-9]/.test(tok) ? 1 : 0) + (/[+/=_\-]/.test(tok) ? 1 : 0);
+    if (h >= 4.0 && classes >= 3 && tok.length >= 32) {
+      hits.push({ type: 'high-entropy-token', confidence: 'medium', sample: mask(tok), entropy: Math.round(h * 100) / 100 });
+    }
+  }
+  return hits;
+}
+
+export function scanSecrets(text) {
+  const src = String(text || '');
+  if (!src) return [];
+  const hits = [];
+  for (const { type, re, confidence } of PATTERNS) {
+    for (const m of src.matchAll(re)) hits.push({ type, confidence, sample: mask(m[0]) });
+  }
+  hits.push(...findMnemonics(src));
+  hits.push(...findHexKeys(src));
+  hits.push(...findHighEntropy(src));
+  return hits;
+}
+
+export function scanFileForSecrets(absPath) {
+  let text = '';
+  try {
+    text = fs.readFileSync(absPath, 'utf8');
+  } catch {
+    return { ok: true, readable: false, hits: [] };
+  }
+  const hits = scanSecrets(text);
+  return { ok: hits.filter((h) => h.confidence === 'high').length === 0, readable: true, hits };
+}
+
+// T5 — a scan result is "blocking" iff it has any high-confidence hit. Low/medium
+// hits are reported for human review but never auto-block (too noisy on crypto
+// notes full of 0x hashes and base64 blobs).
+export function isBlocking(hits) {
+  return (hits || []).some((h) => h.confidence === 'high');
+}
+
+export function summarizeHits(hits) {
+  const byType = new Map();
+  for (const h of hits || []) byType.set(h.type, (byType.get(h.type) || 0) + 1);
+  return [...byType.entries()].map(([type, count]) => `${type}×${count}`).join(', ');
+}

package/src/core/semantic.mjs

@@ -0,0 +1,187 @@
+// Local, dependency-free, deterministic lexical-semantic layer for recall (E22).
+//
+// This is NOT a neural embedding model. It improves recall beyond exact-token by:
+//   1. stemming        morphological variants: fix / fixed / fixing / fixes -> fix
+//   2. a synonym map   resolved <-> fix, defect <-> bug, login <-> authentication
+//   3. fuzzy overlap   character-trigram Jaccard for typos and minor variants
+//
+// An embedding-provider interface lets a real vector model be plugged in later
+// WITHOUT changing the canonical markdown or locking Neurain to one LLM. The
+// default provider is `local-lexical`: fully deterministic, no model call, no
+// external dependency, and it needs no separate index because the canonical
+// markdown is scored directly.
+
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'for', 'with', 'at', 'by', 'from',
+  'is', 'are', 'was', 'were', 'be', 'been', 'being', 'this', 'that', 'these', 'those', 'it', 'its',
+  'as', 'into', 'after', 'before', 'should', 'must', 'will', 'can', 'not', 'no', 'than', 'then',
+]);
+
+const STEM_SUFFIXES = [
+  'ization', 'isation', 'ational', 'fulness', 'iveness', 'ousness', 'ations', 'ation',
+  'ingly', 'edly', 'ings', 'ies', 'ied', 'ment', 'ness', 'able', 'ible', 'ful', 'ous',
+  'ive', 'ize', 'ise', 'ing', 'ers', 'er', 'est', 'ed', 'es', 'ly', 's',
+];
+
+// Each group lists surface forms that mean roughly the same thing. Every form's
+// stem is mapped to the group's canonical id, so a query word and a stored word
+// in the same group match even when they share no characters.
+const SYNONYM_GROUPS = [
+  ['fix', 'fixed', 'fixes', 'fixing', 'resolve', 'resolved', 'resolving', 'repair', 'repaired', 'correct', 'corrected', 'patch', 'patched'],
+  ['bug', 'bugs', 'defect', 'defects', 'error', 'errors', 'issue', 'issues', 'fault', 'faults', 'failure', 'failures', 'problem', 'problems'],
+  ['login', 'logins', 'signin', 'authentication', 'authenticate', 'authenticated', 'credential', 'credentials'],
+  ['delete', 'deleted', 'deletes', 'remove', 'removed', 'removal', 'drop', 'dropped', 'erase', 'erased'],
+  ['add', 'added', 'create', 'created', 'creation', 'insert', 'inserted', 'new'],
+  ['update', 'updated', 'modify', 'modified', 'change', 'changed', 'edit', 'edited', 'revise', 'revised', 'revision'],
+  ['fast', 'faster', 'quick', 'quickly', 'rapid', 'rapidly', 'speed', 'speedup', 'performance', 'latency'],
+  ['doc', 'docs', 'document', 'documents', 'documentation', 'readme', 'guide', 'guides'],
+  ['config', 'configs', 'configuration', 'configure', 'configured', 'setting', 'settings', 'setup'],
+  ['test', 'tests', 'testing', 'verify', 'verified', 'verification', 'validate', 'validated', 'validation'],
+  ['rollback', 'revert', 'reverted', 'undo', 'restore', 'restored', 'restoration'],
+  ['review', 'reviewed', 'reviews', 'audit', 'audited', 'inspect', 'inspected', 'inspection'],
+  ['session', 'sessions', 'handoff', 'handoffs', 'continuity', 'resume', 'resumed'],
+  ['memory', 'memories', 'recall', 'remember', 'remembered', 'retrieval', 'retrieve', 'retrieved'],
+  ['lesson', 'lessons', 'learning', 'learned', 'insight', 'insights', 'takeaway'],
+  ['car', 'cars', 'automobile', 'automobiles', 'vehicle', 'vehicles'],
+  ['big', 'large', 'huge', 'massive', 'major'],
+  ['small', 'tiny', 'minor', 'little'],
+  ['start', 'started', 'begin', 'began', 'launch', 'launched', 'initiate', 'initiated'],
+  ['stop', 'stopped', 'halt', 'halted', 'end', 'ended', 'terminate', 'terminated'],
+  ['safe', 'safety', 'secure', 'security', 'protect', 'protected', 'protection'],
+  ['private', 'confidential', 'sensitive', 'secret'],
+  ['user', 'users', 'customer', 'customers', 'client', 'clients'],
+  ['plan', 'planned', 'planning', 'roadmap', 'schedule', 'scheduled'],
+  ['build', 'built', 'compile', 'compiled', 'assemble', 'assembled'],
+  ['note', 'notes', 'memo', 'record', 'records', 'log', 'logged'],
+];
+
+const SYNONYM_CANON = (() => {
+  const map = new Map();
+  for (const group of SYNONYM_GROUPS) {
+    const canon = stem(group[0]);
+    for (const word of group) {
+      const s = stem(word);
+      if (!map.has(s)) map.set(s, canon);
+    }
+  }
+  return map;
+})();
+
+export function stem(token) {
+  let t = String(token || '').toLowerCase().replace(/['’]/g, '');
+  if (t.length <= 3) return t;
+  for (const suf of STEM_SUFFIXES) {
+    if (t.endsWith(suf) && t.length - suf.length >= 3) {
+      t = t.slice(0, -suf.length);
+      break;
+    }
+  }
+  // collapse a doubled final consonant (running -> runn -> run) for stability
+  t = t.replace(/([bdfgklmnprt])\1$/, '$1');
+  return t;
+}
+
+export function tokenize(text) {
+  return (String(text || '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || []).filter((token) => token.length >= 2 && !STOPWORDS.has(token));
+}
+
+export function expandToken(token) {
+  const s = stem(token);
+  return { token: String(token || '').toLowerCase(), stem: s, canon: SYNONYM_CANON.get(s) || null };
+}
+
+const CANON_MEMBERS = (() => {
+  const map = new Map();
+  for (const group of SYNONYM_GROUPS) map.set(stem(group[0]), group);
+  return map;
+})();
+
+// Return a different surface form in the same synonym group (a deterministic lexical swap),
+// or null if the token has no synonym. Used to derive paraphrase queries from real content.
+export function alternativeForm(token) {
+  const s = stem(token);
+  const canon = SYNONYM_CANON.get(s);
+  if (!canon) return null;
+  const members = CANON_MEMBERS.get(canon) || [];
+  const alt = members.find((member) => stem(member) !== s && member.toLowerCase() !== String(token || '').toLowerCase());
+  return alt || null;
+}
+
+function charTrigrams(token) {
+  const padded = `#${String(token || '')}#`;
+  const grams = new Set();
+  for (let i = 0; i + 3 <= padded.length; i += 1) grams.add(padded.slice(i, i + 3));
+  return grams;
+}
+
+export function fuzzyOverlap(a, b) {
+  if (!a || !b) return 0;
+  if (a === b) return 1;
+  const ga = charTrigrams(a);
+  const gb = charTrigrams(b);
+  if (!ga.size || !gb.size) return 0;
+  let inter = 0;
+  for (const g of ga) if (gb.has(g)) inter += 1;
+  return inter / (ga.size + gb.size - inter);
+}
+
+// Deterministic lexical-semantic score of a query against a document body.
+// Returns { score: 0..1 normalized by query length, matched_terms: [...] }.
+export function lexicalSemanticScore(query, docText) {
+  const queryExpanded = tokenize(query).map(expandToken);
+  if (!queryExpanded.length) return { score: 0, matched_terms: [] };
+  const docTokens = tokenize(docText).map(expandToken);
+  if (!docTokens.length) return { score: 0, matched_terms: [] };
+  const docStems = new Set(docTokens.map((d) => d.stem));
+  const docCanons = new Set(docTokens.map((d) => d.canon).filter(Boolean));
+  const matched = [];
+  let total = 0;
+  for (const q of queryExpanded) {
+    let best = 0;
+    let how = '';
+    if (docStems.has(q.stem)) { best = 1; how = 'exact'; }
+    else if (q.canon && docCanons.has(q.canon)) { best = 0.75; how = 'synonym'; }
+    else {
+      // fuzzy: best trigram overlap against any doc stem (typos / variants)
+      for (const d of docStems) {
+        const ov = fuzzyOverlap(q.stem, d);
+        if (ov > best) { best = ov; how = 'fuzzy'; }
+      }
+      if (best < 0.6) best = 0;
+    }
+    if (best > 0) matched.push({ term: q.token, via: how, weight: Number(best.toFixed(3)) });
+    total += best;
+  }
+  return { score: Number((total / queryExpanded.length).toFixed(4)), matched_terms: matched };
+}
+
+const PROVIDERS = new Map();
+
+export function registerProvider(name, impl) {
+  PROVIDERS.set(name, impl);
+}
+
+export function getProvider(name = 'local-lexical') {
+  return PROVIDERS.get(name) || PROVIDERS.get('local-lexical');
+}
+
+export function listProviders() {
+  return [...PROVIDERS.keys()].sort();
+}
+
+// Default provider: deterministic local lexical-semantic. No model call, no
+// external dependency, and no separate generated index (markdown stays canonical).
+registerProvider('local-lexical', {
+  name: 'local-lexical',
+  kind: 'deterministic_lexical',
+  model_call: false,
+  external_call: false,
+  requires_index: false,
+  llm_locked: false,
+  expandQuery(query) {
+    return tokenize(query).map(expandToken);
+  },
+  score(query, docText) {
+    return lexicalSemanticScore(query, docText);
+  },
+});

package/src/core/source_digest.mjs

@@ -0,0 +1,56 @@
+// Read-only reader for the source-digest manifest (the capture/compile pipeline's
+// high-value candidate index, owned by W-B). The view tools only READ it to
+// surface compile and deep-compile candidates. Missing manifest -> empty result.
+import path from 'node:path';
+import { readJsonSafe } from './vault_state.mjs';
+
+const COMPILED_STATUSES = ['compiled', 'flushed', 'archived', 'superseded'];
+
+function isCompiled(entry) {
+  const status = String(entry?.status || '').toLowerCase();
+  const compiledTo = Array.isArray(entry?.compiled_to) ? entry.compiled_to : [];
+  return COMPILED_STATUSES.includes(status) || compiledTo.length > 0;
+}
+
+export function loadManifestEntries(root, vaultCfg) {
+  const manifest = readJsonSafe(path.join(root, vaultCfg.source_digest_manifest), null);
+  if (!manifest || typeof manifest.files !== 'object') return [];
+  return Object.values(manifest.files);
+}
+
+// Compile candidates for the boot/status summary: uncompiled, non-private,
+// high-value, ranked by score then recency.
+export function compileCandidateSummary(root, vaultCfg) {
+  const entries = loadManifestEntries(root, vaultCfg)
+    .filter((entry) => !isCompiled(entry))
+    .filter((entry) => ['public', 'internal'].includes(String(entry.sensitivity || 'internal')))
+    .filter((entry) => Number(entry.high_value_score || 0) >= 2)
+    .sort((a, b) => Number(b.high_value_score || 0) - Number(a.high_value_score || 0) || Number(b.mtime_ms || 0) - Number(a.mtime_ms || 0));
+  return {
+    count: entries.length,
+    examples: entries.slice(0, 3).map((entry) => ({
+      path: entry.path,
+      source_id: entry.source_id || '',
+      sensitivity: entry.sensitivity || 'internal',
+      score: Number(entry.high_value_score || 0),
+    })),
+  };
+}
+
+// Deep-compile candidates for the review desk: uncompiled with score >= 2.
+export function deepCompileCandidates(root, vaultCfg, { top = 12 } = {}) {
+  return loadManifestEntries(root, vaultCfg)
+    .filter((entry) => !isCompiled(entry) && Number(entry.high_value_score || 0) >= 2)
+    .sort((a, b) => Number(b.high_value_score || 0) - Number(a.high_value_score || 0) || Number(b.size_bytes || 0) - Number(a.size_bytes || 0))
+    .slice(0, top)
+    .map((entry) => ({
+      path: entry.path,
+      source_id: entry.source_id,
+      source_type: entry.source_type,
+      sensitivity: entry.sensitivity,
+      status: entry.status,
+      high_value_score: entry.high_value_score,
+      reasons: entry.high_value_reasons || [],
+      suggested_review: 'Select this for Deep Compile if it will affect decisions, current status, or repeated future work.',
+    }));
+}