neurain 0.1.0-alpha.0

package/src/core/capture_durable.mjs

@@ -0,0 +1,181 @@
+// `capture` command (W-B, B4, durable). Captures text into the vault as an
+// append-only raw source: renders raw markdown (routed THROUGH the stage secret
+// gate), writes the _inbox envelope, and appends one pending writeback-queue row
+// — all inside a single capture critical section (one lock on the queue) so
+// concurrent captures cannot collide on a source id or interleave the append.
+//
+// W-D BOUNDARY: the engine NEVER writes session-state.json. It returns a
+// session_state_delta the future vault shuttle applies; pending_count there is
+// ADVISORY (the shuttle must recompute at apply time). See the B4 cross-review.
+//
+// SCOPE (B4 increment): text input only. File capture (asset copy + text
+// extraction), overlap-candidate detection, and numeric-conflict flagging are NOT
+// ported yet; until they are, engine capture is for porting/shadow use and is the
+// documented FLIP GATE for `capture` (a real capture that would overlap an
+// existing summary or carry a file must keep using the vault tool).
+import fs from 'node:fs';
+import path from 'node:path';
+import { absPath, ensureDir, timestamp } from './fs.mjs';
+import { vaultConfig } from './config.mjs';
+import { firstLineTitle, inferFlushLevelFromEnvelope, inferTargetLayerFromIntent } from './classify.mjs';
+import { folderForSourceType, makeEnvelope, readArgInput, renderCaptureMarkdown, slugify } from './envelope.mjs';
+import { withFileLock, atomicWriteJson } from './durable.mjs';
+import { stageAndPromote } from './stage.mjs';
+import { loadSessionState, pendingCountForSession } from './vault_state.mjs';
+import { appendWikiLog } from './wiki_log.mjs';
+
+function handoffPathFor(vaultCfg, sessionId, session) {
+  return (session && session.handoff_path) || `${vaultCfg.session_handoff_dir}/${sessionId}.md`;
+}
+
+// max+1 over today's existing inbox envelopes (not count+1, so a deleted id is
+// never reused while a later one is live). Called inside the capture lock.
+function nextSourceId(root, vaultCfg) {
+  const day = timestamp().slice(0, 10).replace(/-/g, '');
+  const inboxAbs = path.join(root, vaultCfg.raw_inbox_dir);
+  ensureDir(inboxAbs);
+  const prefix = `raw-${day}-`;
+  const nums = fs.readdirSync(inboxAbs)
+    .filter((name) => name.startsWith(prefix) && name.endsWith('.json'))
+    .map((name) => parseInt(name.slice(prefix.length), 10))
+    .filter((n) => Number.isFinite(n));
+  return `${prefix}${String((nums.length ? Math.max(...nums) : 0) + 1).padStart(3, '0')}`;
+}
+
+function buildQueueItem(envelope, { sourceId, rawPath, title, session, sessionId, flushLevel, targetLayer, targetPath, conflictPolicy, handoffPath }) {
+  return {
+    queued_at: envelope.captured_at,
+    status: 'pending',
+    source_id: sourceId,
+    raw_path: rawPath,
+    title,
+    sensitivity: envelope.sensitivity,
+    write_intent: envelope.write_intent,
+    area_candidates: envelope.area_candidates,
+    requires_user_decision: envelope.requires_user_decision,
+    ...(session
+      ? { session_id: sessionId, scope: session.scope || '', flush_level: flushLevel, target_layer: targetLayer, target_path: targetPath, conflict_policy: conflictPolicy, handoff_path: handoffPath }
+      : {}),
+  };
+}
+
+export async function captureCommand(args) {
+  const root = absPath(args._[0] || args.root || process.cwd());
+  const vaultCfg = vaultConfig(root);
+
+  if (args['source-file'] || args.source) {
+    process.exitCode = 1;
+    return done(args, { ok: false, command: 'capture', durable_write: false, error: 'File capture is not ported to the engine yet (flip gate: asset copy + text extraction + overlap/numeric detection). Use the vault capture tool for files.' });
+  }
+
+  const sessionId = String(args['session-id'] || '');
+  let session = null;
+  if (sessionId) {
+    let state = { sessions: {} };
+    try { state = loadSessionState(root, vaultCfg); } catch (error) {
+      return done(args, { ok: false, command: 'capture', durable_write: false, error: error.message });
+    }
+    session = (state.sessions || {})[sessionId];
+    if (!session) return done(args, { ok: false, command: 'capture', durable_write: false, error: `unknown session "${sessionId}"` });
+  }
+
+  const text = readArgInput(args, root);
+  if (!text.trim()) return done(args, { ok: false, command: 'capture', durable_write: false, error: 'No input provided. Pass --text, positional text, or --file.' });
+
+  const sourceType = String(args.type || 'memo').toLowerCase();
+  const title = args.title || firstLineTitle(text, 'Untitled Capture');
+  const routeText = [title, session?.scope || '', args.notes || '', text].filter(Boolean).join('\n');
+  const folder = folderForSourceType(sourceType);
+  const [year, month] = timestamp().slice(0, 10).split('-');
+  const overrides = { area: args.area || (session && session.area), sensitivity: args.sensitivity, intent: args.intent };
+  const allowSecret = Boolean(args['allow-secret']);
+
+  const buildEnvelope = (sourceId, status) => {
+    const e = makeEnvelope(root, { sourceId, sourceType, title, text: routeText, status, overrides }, { vaultCfg });
+    e.raw_path = path.join(vaultCfg.raw_dir, folder, year, month, `${sourceId}-${slugify(title)}.md`).split(path.sep).join('/');
+    if (session) { e.session_id = sessionId; e.session_scope = session.scope || ''; e.handoff_path = handoffPathFor(vaultCfg, sessionId, session); }
+    return e;
+  };
+
+  if (args['dry-run']) {
+    const envelope = buildEnvelope(`raw-${timestamp().slice(0, 10).replace(/-/g, '')}-DRYRUN`, 'planned');
+    const flushLevel = args['flush-level'] || inferFlushLevelFromEnvelope(envelope);
+    const targetLayer = args['target-layer'] || inferTargetLayerFromIntent(envelope.write_intent);
+    const queueItem = buildQueueItem(envelope, { sourceId: envelope.source_id, rawPath: envelope.raw_path, title, session, sessionId, flushLevel, targetLayer, targetPath: args['target-path'] || '', conflictPolicy: args['conflict-policy'] || 'queue_first', handoffPath: handoffPathFor(vaultCfg, sessionId, session) });
+    return done(args, {
+      ok: true, command: 'capture', durable_write: false, dry_run: true,
+      envelope, queue_item: queueItem,
+      would_write: [envelope.raw_path, `${vaultCfg.raw_inbox_dir}/${envelope.source_id}.json`, vaultCfg.writeback_queue, `${vaultCfg.wiki_dir}/log.md`],
+    });
+  }
+
+  // Single capture critical section: id allocation + raw promote + envelope +
+  // queue append are serialized on the queue lock so two captures cannot collide.
+  const queueAbs = path.join(root, vaultCfg.writeback_queue);
+  ensureDir(path.dirname(queueAbs));
+  const outcome = withFileLock(queueAbs, () => {
+    const sourceId = nextSourceId(root, vaultCfg);
+    const envelope = buildEnvelope(sourceId, 'captured');
+    const rawPath = envelope.raw_path;
+
+    // raw markdown goes THROUGH the stage gate: a blocking secret holds it in
+    // staging and writes NOTHING durable (no envelope, no queue row).
+    const staged = stageAndPromote(root, rawPath, renderCaptureMarkdown(envelope, text), { allowSecret });
+    if (!staged.promoted) return { refused: true, staged };
+
+    atomicWriteJson(path.join(root, `${vaultCfg.raw_inbox_dir}/${sourceId}.json`), envelope);
+
+    const flushLevel = args['flush-level'] || inferFlushLevelFromEnvelope(envelope);
+    const targetLayer = args['target-layer'] || inferTargetLayerFromIntent(envelope.write_intent);
+    const queueItem = buildQueueItem(envelope, { sourceId, rawPath, title, session, sessionId, flushLevel, targetLayer, targetPath: args['target-path'] || '', conflictPolicy: args['conflict-policy'] || 'queue_first', handoffPath: handoffPathFor(vaultCfg, sessionId, session) });
+    fs.appendFileSync(queueAbs, `${JSON.stringify(queueItem)}\n`); // direct: already holding the queue lock
+    const pendingCount = pendingCountForSession(root, vaultCfg, sessionId); // after append
+    return { refused: false, sourceId, envelope, rawPath, pendingCount };
+  });
+
+  if (outcome.refused) {
+    process.exitCode = 2;
+    return done(args, { ok: false, command: 'capture', durable_write: false, refused: true, reason: `secret-like content held in staging: ${outcome.staged.summary}`, staged_path: outcome.staged.staged_path, hits: outcome.staged.hits });
+  }
+
+  const { sourceId, envelope, rawPath, pendingCount } = outcome;
+  appendWikiLog(root, vaultCfg, 'ingest', title, [
+    `Source: ${sourceId}`, `Raw path: ${rawPath}`,
+    `Areas: ${envelope.area_candidates.join(', ') || 'none'}`,
+    `Sensitivity: ${envelope.sensitivity}`, 'Status: captured',
+  ]);
+
+  return done(args, {
+    ok: true, command: 'capture', durable_write: true, captured: true,
+    source_id: sourceId, raw_path: rawPath,
+    envelope_path: `${vaultCfg.raw_inbox_dir}/${sourceId}.json`,
+    queue_path: vaultCfg.writeback_queue,
+    requires_user_decision: envelope.requires_user_decision,
+    envelope,
+    session_state_delta: session ? {
+      applied: false,
+      reason: 'reserved-for-W-D: engine never writes session-state; the vault shuttle applies this.',
+      recompute_pending_at_apply: true,
+      session_id: sessionId,
+      patch: { pending_count: pendingCount },
+      area_brief_note: null,
+      handoff_note: null,
+    } : null,
+  });
+}
+
+function done(args, payload) {
+  if (args.json) return { json: true, payload };
+  if (!payload.ok) {
+    if (payload.refused) {
+      const lines = [`⛔ BLOCKED (secret): ${payload.reason}`];
+      for (const h of payload.hits || []) lines.push(`    [${h.confidence}] ${h.type}: ${h.sample}`);
+      return { text: lines.join('\n') };
+    }
+    return { text: `# Capture\n\n- ${payload.error}` };
+  }
+  if (payload.dry_run) return { text: `# Capture Dry Run\n\n- Source ID: ${payload.envelope.source_id}\n- Raw path: ${payload.envelope.raw_path}\n- Areas: ${payload.envelope.area_candidates.join(', ') || 'none'}\n- Sensitivity: ${payload.envelope.sensitivity}\n- Intent: ${payload.envelope.write_intent}` };
+  const lines = ['# Captured', '', `- Source ID: ${payload.source_id}`, `- Raw path: ${payload.raw_path}`, `- Envelope: ${payload.envelope_path}`, `- Requires user decision: ${payload.requires_user_decision ? 'yes' : 'no'}`];
+  if (payload.session_state_delta) lines.push(`- Session-state delta (unapplied, pending_count=${payload.session_state_delta.patch.pending_count}) returned for the vault shuttle.`);
+  return { text: lines.join('\n') };
+}

package/src/core/classify.mjs

@@ -0,0 +1,237 @@
+// Registry-driven source classification (W-B). A faithful, root-injected port of
+// the vault's neurain-utils classifyText path: area routing by per-area keyword
+// profiles, sensitivity/write-intent detection, and registry-derived domain and
+// entity tagging. Pure and read-only — it reads the search-index registry and
+// per-area _area.md / index files, never writes. Kept separate from recall_intel
+// (the recall ranker's intel) because the vault's capture/route classifier is a
+// distinct code path with its own keyword-profile scoring; route and
+// plan-writeback parity tracks THIS path, so it is ported verbatim.
+import fs from 'node:fs';
+import path from 'node:path';
+import { absPath, readText } from './fs.mjs';
+import { vaultConfig } from './config.mjs';
+
+// Per-root cache of the derived intel + profiles. The vault caches per-process;
+// the engine keys by resolved root so multiple vaults (tests, MCP scoping) never
+// cross-contaminate. One-shot CLI runs read it once; long-lived MCP reuses it.
+const _intelCache = new Map();
+const _profileCache = new Map();
+
+function matchLine(text, regex) {
+  const match = String(text || '').match(regex);
+  return match ? match[1].trim() : '';
+}
+
+// Term match with word boundaries for ASCII (so "cr" does not match inside
+// "across"); CJK and multiword terms fall back to substring. Verbatim from vault.
+export function hasTerm(haystack, needle) {
+  if (!needle) return false;
+  if (!/^[a-z0-9]+$/i.test(needle)) return haystack.includes(needle);
+  return new RegExp(`(^|[^a-z0-9])${needle}([^a-z0-9]|$)`, 'i').test(haystack);
+}
+
+// Per-area routing intelligence derived from each area's OWN registered search
+// index (entities + domain routing) plus the registry sensitivity baseline, so a
+// stranger's vault routes by ITS data and nothing area-specific is hardcoded.
+export function loadAreaIntel(root, vaultCfg = vaultConfig(root)) {
+  const key = absPath(root);
+  if (_intelCache.has(key)) return _intelCache.get(key);
+  const out = { entities: [], domains: [], sensitivity: {} };
+  let reg;
+  try {
+    reg = JSON.parse(readText(path.join(root, vaultCfg.search_index_registry)));
+  } catch {
+    reg = { areas: {} };
+  }
+  for (const [area, def] of Object.entries((reg && reg.areas) || {})) {
+    if (def.sensitivity) out.sensitivity[area] = String(def.sensitivity);
+    const areaRoot = def.area_root || `${vaultCfg.areas_dir}/${area}`;
+    const idxDir = def.index_dir || 'search-index';
+    try {
+      const ents = JSON.parse(readText(path.join(root, areaRoot, idxDir, def.entities || 'entities.json')));
+      for (const e of ents || []) {
+        const aliases = [e.canonical, ...(e.aliases || [])].map((s) => String(s || '').toLowerCase()).filter((s) => s.length >= 2);
+        if (e.canonical) out.entities.push({ area, canonical: e.canonical, aliases });
+      }
+    } catch { /* area may have no entity index */ }
+    try {
+      const doms = JSON.parse(readText(path.join(root, areaRoot, idxDir, def.domain_routing || 'domain-routing.json')));
+      for (const d of doms || []) out.domains.push({ area, label: d.label || d.id || '', keywords: (d.keywords || []).map((s) => String(s || '').toLowerCase()).filter((s) => s.length >= 2) });
+    } catch { /* area may have no domain routing */ }
+  }
+  _intelCache.set(key, out);
+  return out;
+}
+
+function keywordsForArea(intel, slug, name) {
+  const ks = new Set([String(slug).toLowerCase(), String(name || '').toLowerCase()]);
+  for (const e of intel.entities) if (e.area === slug) for (const a of e.aliases) ks.add(a);
+  return [...ks].filter(Boolean);
+}
+
+// Area keyword profiles: one per 10_areas/<slug> directory, keyworded by slug +
+// _area.md Name + that area's registered entity aliases. Used for area routing.
+export function loadAreaProfiles(root, vaultCfg = vaultConfig(root), intel = loadAreaIntel(root, vaultCfg)) {
+  const key = absPath(root);
+  if (_profileCache.has(key)) return _profileCache.get(key);
+  const areasAbs = path.join(root, vaultCfg.areas_dir);
+  let profiles = [];
+  if (fs.existsSync(areasAbs)) {
+    profiles = fs
+      .readdirSync(areasAbs, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => {
+        const slug = entry.name;
+        const text = readText(path.join(areasAbs, slug, '_area.md'), '');
+        const name = matchLine(text, /- Name:\s*(.+)/i) || slug;
+        const status = matchLine(text, /- Status:\s*(.+)/i) || 'active';
+        const sensitivity = matchLine(text, /- Sensitivity:\s*(.+)/i) || 'internal';
+        return { slug, name, status, sensitivity, keywords: keywordsForArea(intel, slug, name) };
+      });
+  }
+  _profileCache.set(key, profiles);
+  return profiles;
+}
+
+export function knownAreaIds(root, vaultCfg = vaultConfig(root)) {
+  const dir = path.join(root, vaultCfg.areas_dir);
+  if (!fs.existsSync(dir)) return [];
+  return fs
+    .readdirSync(dir, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.'))
+    .map((entry) => entry.name);
+}
+
+// Map a possibly-stale bare slug ("acme") to the current area id ("_acme") when
+// the prefixed area exists, so the underscore-prefix convention never silently
+// breaks a route. An already-correct id is returned unchanged; an unknown id
+// passes through for the caller to validate.
+export function normalizeAreaId(root, area, vaultCfg = vaultConfig(root)) {
+  const value = String(area || '').trim();
+  if (!value) return value;
+  const ids = knownAreaIds(root, vaultCfg);
+  if (ids.includes(value)) return value;
+  if (ids.includes(`_${value}`)) return `_${value}`;
+  return value;
+}
+
+export function detectSensitivity(text, areas = [], intel = { sensitivity: {} }) {
+  const lower = String(text || '').toLowerCase();
+  if (
+    /\b(private|secret|password|credential|seed phrase|mnemonic|passport|ssn|bank account|api key|token)\b/.test(lower) ||
+    /개인|민감|비공개|주민|여권|계좌|부채|자산|등기부|가상자산|세금|연봉|급여|병원|의료|가족|주소/.test(lower)
+  ) {
+    return 'private';
+  }
+  const sens = intel.sensitivity || {};
+  if (areas.some((a) => /private/i.test(sens[a] || ''))) return 'private';
+  if (
+    areas.some((a) => /internal/i.test(sens[a] || '')) ||
+    /\b(internal|foundation|strategy|governance|bd|partner|deal|roadmap)\b/.test(lower) ||
+    /내부|전략|파트너|재단|거버넌스|토크노믹스/.test(lower)
+  ) {
+    return 'internal';
+  }
+  return 'public';
+}
+
+export function detectWriteIntent(text) {
+  const lower = String(text || '').toLowerCase();
+  if (/\b(deliverable|report|deck|slides|draft|export|proposal)\b|보고서|발표|초안|제안서|결과물|산출물/.test(lower)) return 'output_request';
+  if (/\b(task|todo|action item|follow up|remind|deadline)\b|할일|태스크|액션|마감|리마인드/.test(lower)) return 'create_task';
+  if (/\b(current|status|update|progress|now)\b|현재|현황|업데이트|진행상황/.test(lower)) return 'update_current';
+  if (/\b(remember|save|store|classify|summarize|capture|ingest|memo|note)\b|기억|저장|정리|분류|요약|편입|메모/.test(lower)) return 'remember';
+  return 'evidence_only';
+}
+
+export function detectDomains(text, areas = [], intel = { domains: [] }) {
+  const lower = String(text || '').toLowerCase();
+  const inScope = (a) => !areas.length || areas.includes(a);
+  const domains = new Set();
+  for (const d of intel.domains) {
+    if (inScope(d.area) && d.label && d.keywords.some((k) => hasTerm(lower, k))) domains.add(d.label);
+  }
+  return [...domains];
+}
+
+export function detectEntities(text, areas = [], intel = { entities: [] }) {
+  const lower = String(text || '').toLowerCase();
+  const inScope = (a) => !areas.length || areas.includes(a);
+  const entities = new Set();
+  for (const e of intel.entities) {
+    if (inScope(e.area) && e.aliases.some((a) => hasTerm(lower, a))) entities.add(e.canonical);
+  }
+  return [...entities];
+}
+
+// The classifier itself. Registry/profiles read from `root`; `hints` may carry an
+// area (boosts its profile), an explicit sensitivity/intent override, and a
+// requiresUserDecision flag. Returns the vault's route shape verbatim.
+export function classifyText(root, text, hints = {}, opts = {}) {
+  const vaultCfg = opts.vaultCfg || vaultConfig(root);
+  const intel = opts.intel || loadAreaIntel(root, vaultCfg);
+  const profiles = opts.profiles || loadAreaProfiles(root, vaultCfg, intel);
+  const source = String(text || '');
+  const lower = source.toLowerCase();
+
+  const scores = profiles.map((profile) => {
+    const keywordScore = profile.keywords.reduce((score, keyword) => (hasTerm(lower, keyword.toLowerCase()) ? score + 1 : score), 0);
+    const hintScore = hints.area === profile.slug ? 5 : 0;
+    return { profile, score: keywordScore + hintScore };
+  });
+  const areaCandidates = scores
+    .filter((item) => item.score > 0)
+    .sort((a, b) => b.score - a.score)
+    .map((item) => item.profile.slug);
+
+  const sensitivity = hints.sensitivity || detectSensitivity(source, areaCandidates, intel);
+  const writeIntent = hints.intent || detectWriteIntent(source);
+  const domainCandidates = detectDomains(source, areaCandidates, intel);
+  const entityCandidates = detectEntities(source, areaCandidates, intel);
+  const conflict = /\b(conflict|contradiction|contradict|supersede|disagree)\b|모순|충돌|상충|반박|정정/.test(lower);
+  const requiresUserDecision =
+    Boolean(hints.requiresUserDecision) ||
+    conflict ||
+    (sensitivity === 'private' && writeIntent !== 'evidence_only') ||
+    areaCandidates.length > 1;
+
+  return {
+    area_candidates: areaCandidates,
+    domain_candidates: domainCandidates,
+    entity_candidates: entityCandidates,
+    sensitivity,
+    write_intent: writeIntent,
+    requires_user_decision: requiresUserDecision,
+    route_score: Object.fromEntries(scores.filter((item) => item.score > 0).map((item) => [item.profile.slug, item.score])),
+  };
+}
+
+export function firstLineTitle(text, fallback = 'Untitled Capture') {
+  const first = String(text || '')
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .find(Boolean);
+  return first ? first.slice(0, 90) : fallback;
+}
+
+export function inferFlushLevelFromEnvelope(envelope = {}) {
+  if (envelope.requires_user_decision || envelope.sensitivity === 'private') return 'full';
+  if (['update_current', 'create_task'].includes(envelope.write_intent)) return 'full';
+  if (envelope.write_intent === 'evidence_only') return 'light';
+  return 'standard';
+}
+
+export function inferTargetLayerFromIntent(intent = 'evidence_only') {
+  if (intent === 'update_current') return 'current';
+  if (intent === 'create_task') return 'task_memory';
+  if (intent === 'output_request') return 'output';
+  if (intent === 'evidence_only') return 'raw';
+  return 'wiki';
+}
+
+// Test-only: clear the per-root caches (so a test can rewrite a vault's registry
+// and re-classify in the same process). Never called by command paths.
+export function _resetClassifyCache() {
+  _intelCache.clear();
+  _profileCache.clear();
+}