@mindrian_os/cli 1.13.0-beta.10 → 1.13.0-beta.43

package/lib/core/rs-pinecone-bridge.cjs

@@ -1,303 +0,0 @@
-/*
- * Copyright (c) 2026 Mindrian. BSL 1.1.
- * Phase 89.2 Plan 06 Task 1 -- Pinecone-direct bridge for 1024-dim
- * vector retrieval (CJS->Python shim).
- *
- * The bridge wraps the existing rs_cache.py::fetch_all_from_namespace
- * primitive (Phase 89-03, v1.10.16) without modifying it. This is a
- * Canon Part 7 (Reuse Before Build) consumer: zero new Python code,
- * zero new index, zero new field schema. The strict-1024-dim deferred
- * item from 89.1 (substrate carries _score not embedding on the live
- * Brain wire path) is resolved here by going Pinecone-direct against
- * the rs-external index used for external corpora.
- *
- * Architecture:
- *
- *   queryPineconeWithVectors(namespace, queryText, topK)
- *     -> spawnSync python3 -c bridgeScript
- *     -> bridgeScript reads JSON from stdin
- *     -> bridgeScript invokes rs_cache.fetch_all_from_namespace
- *     -> bridgeScript writes JSON {success, hits[]} to stdout
- *     -> CJS parses + returns
- *
- * Graceful degradation contract (Phase 89.2-01 / 02 / 03 / 04 pattern):
- *
- *   missing PINECONE_API_KEY -> {success: false, error: 'pinecone_api_key_missing'}
- *   python3 not on PATH      -> {success: false, error: 'python_unavailable'}
- *   non-zero exit            -> {success: false, error: 'python_exit_<N>'}
- *   stdout parse failure     -> {success: false, error: 'parse_error'}
- *   timeout                  -> {success: false, error: 'timeout'}
- *
- * NEVER throws. Every external boundary returns a structured envelope.
- *
- * Canon Part 8 (Graph Boundary): two defense-in-depth layers.
- *
- *   Layer 1 -- pre-egress: auditQueryString runs on the queryText
- *     parameter before spawnSync. Throws ExternalEgressViolation on
- *     FORBIDDEN_PATTERNS hit. Adversarial input never reaches Pinecone.
- *
- *   Layer 2 -- post-receive: auditQueryObject runs on the parsed hits[]
- *     payload before return. Throws ExternalEgressViolation if a hit's
- *     metadata smuggles a forbidden pattern through. The rs-external
- *     namespace is supposed to hold only public OpenAlex/arXiv metadata
- *     (Phase 89-03 Canon attestation), but we re-audit on the receive
- *     side because trust is not a security property -- enforcement is.
- *
- * Additionally exposes:
- *
- *   cosineSimilarity(a, b) -- pure JS dot(a,b)/(norm(a)*norm(b)).
- *     Both inputs MUST be Number arrays of identical length (typically
- *     1024 for Pinecone multilingual-e5-large). Returns 0 on degenerate
- *     zero-vectors. Differential scorer uses this on the two query
- *     vectors returned by twin queryPineconeWithVectors calls.
- *
- * Pure CJS, zero npm deps, node built-ins only beyond the three
- * rs-egress-* primitives shipped in Wave 1 (Plan 89.2-01).
- */
-'use strict';
-
-const path = require('node:path');
-const childProcess = require('child_process');
-const { auditQueryString, auditQueryObject } = require('./rs-egress-prompts.cjs');
-// rs-egress-violations is required transitively by rs-egress-prompts; we do
-// not throw ExternalEgressViolation directly from this module (the audit
-// helpers do). The require below is a reachability witness for the Canon
-// Part 8 audit chain at module-load time.
-require('./rs-egress-violations.cjs');
-
-// ---------- Constants ----------
-
-const REPO_ROOT = path.resolve(__dirname, '..', '..');
-const PYTHON_BIN = process.env.MINDRIAN_PYTHON || 'python3';
-const BRIDGE_TIMEOUT_MS = 30000;
-const DEFAULT_LIMIT = 1000;
-
-// ---------- Embedded Python bridge script ----------
-//
-// This script is spawned as `python3 -c <script>`. It MUST be valid Python.
-// The leading `# BSL 1.1` line is a deliberate Canon discipline marker so
-// the embedded Python source is licensed identically to the surrounding
-// CJS file (W4 fix per Plan 89.2-06).
-//
-// Protocol:
-//
-//   stdin   : JSON {namespace: string, limit?: integer}
-//   stdout  : JSON {success: boolean, hits?: array, error?: string}
-//
-// The script imports rs_cache from lib.core (REPO_ROOT must be on
-// sys.path). On any exception, returns success:false with the exception
-// message. Never raises to the parent process.
-
-const bridgeScript = [
-  '# BSL 1.1',
-  '# Copyright (c) 2026 Mindrian. Phase 89.2 Plan 06 Task 1 -- pinecone bridge embedded Python.',
-  '# This script wraps rs_cache.py::fetch_all_from_namespace to retrieve',
-  '# 1024-dim Pinecone-direct vectors. JSON over stdio: parent writes a',
-  '# request envelope to stdin; child writes a response envelope to stdout.',
-  '# Errors are reported via the response envelope (success:false), never',
-  '# raised to the parent process. The child exits 0 on graceful failure.',
-  'import sys, os, json',
-  '# Ensure REPO_ROOT is on sys.path so `from lib.core.rs_cache import ...` resolves.',
-  'sys.path.insert(0, os.environ.get("MINDRIAN_REPO_ROOT", os.getcwd()))',
-  'try:',
-  '    from lib.core.rs_cache import fetch_all_from_namespace',
-  '    payload = json.loads(sys.stdin.read())',
-  '    ns = payload["namespace"]',
-  '    limit = int(payload.get("limit", 1000))',
-  '    records = fetch_all_from_namespace(ns, limit=limit)',
-  '    # Map rs_cache.py output {id, values, metadata} to bridge hits[].',
-  '    hits = []',
-  '    for r in records:',
-  '        hits.append({',
-  '            "id": r.get("id", ""),',
-  '            "score": float(r.get("score", 0.0)) if "score" in r else 0.0,',
-  '            "values": list(r.get("values") or []),',
-  '            "metadata": dict(r.get("metadata") or {}),',
-  '        })',
-  '    sys.stdout.write(json.dumps({"success": True, "hits": hits}))',
-  'except Exception as e:',
-  '    sys.stdout.write(json.dumps({"success": False, "error": str(e)}))',
-  '',
-].join('\n');
-
-// ---------- queryPineconeWithVectors ----------
-//
-// Public entry point. Spawns python3 with the embedded bridgeScript,
-// passes the namespace + limit via stdin, parses the JSON response,
-// audits, and returns the structured envelope.
-//
-// Inputs:
-//   namespace  string   the Pinecone namespace, typically 'external:<topic-slug>'
-//   queryText  string   the original user query (audited pre-egress; not used by the
-//                       bridge call itself since fetch_all_from_namespace pulls all
-//                       cached vectors -- the BERT cosine is computed downstream).
-//                       The query text is included in the audit triple so the
-//                       caller's intent is checked at the gate.
-//   topK       integer  result cap (default 1000; bounded by rs_cache.MAX_NAMESPACE_VECTORS)
-//
-// Returns Promise resolving to:
-//   { success: true,  hits: [{id, score, values: number[], metadata}] }
-//   { success: false, error: 'pinecone_api_key_missing'|'python_unavailable'
-//                          |'python_exit_<N>'|'parse_error'|'timeout'
-//                          |'invalid_namespace'|'invalid_top_k', stderr?: string }
-//
-// Throws (the only escape route -- Canon Part 8):
-//   ExternalEgressViolation  if queryText matches FORBIDDEN_PATTERNS
-//                            OR if returned hits[] metadata smuggles a forbidden
-//                            pattern through.
-
-async function queryPineconeWithVectors(namespace, queryText, topK) {
-  // Argument validation (graceful, never throws on shape errors).
-  if (typeof namespace !== 'string' || namespace.length === 0) {
-    return { success: false, error: 'invalid_namespace' };
-  }
-  if (typeof queryText !== 'string') {
-    return { success: false, error: 'invalid_query_text' };
-  }
-  const limit = (typeof topK === 'number' && topK > 0) ? Math.floor(topK) : DEFAULT_LIMIT;
-
-  // Canon Part 8 Layer 1: pre-egress audit on user-controlled queryText.
-  // auditQueryString throws ExternalEgressViolation on hit -- this is the
-  // single intentional escape route from this module.
-  auditQueryString(queryText, 'pinecone-bridge');
-  // Audit the namespace too (defense-in-depth; namespaces are typically
-  // 'external:<topic-slug>' so the topic-slug carries the user's domain).
-  auditQueryString(namespace, 'pinecone-bridge');
-
-  // Env-var gate. Pinecone SDK requires PINECONE_API_KEY; if absent the
-  // child process raises a RuntimeError. Short-circuit here so the user
-  // sees the canonical 'pinecone_api_key_missing' tag and not a stack trace.
-  if (!process.env.PINECONE_API_KEY) {
-    return { success: false, error: 'pinecone_api_key_missing' };
-  }
-
-  const requestEnvelope = JSON.stringify({ namespace: namespace, limit: limit });
-  const childEnv = Object.assign({}, process.env, {
-    MINDRIAN_REPO_ROOT: REPO_ROOT,
-  });
-
-  let res;
-  try {
-    res = childProcess.spawnSync(PYTHON_BIN, ['-c', bridgeScript], {
-      input: requestEnvelope,
-      encoding: 'utf8',
-      timeout: BRIDGE_TIMEOUT_MS,
-      env: childEnv,
-      cwd: REPO_ROOT,
-    });
-  } catch (err) {
-    // spawnSync itself does not normally throw; this is defense-in-depth.
-    return { success: false, error: 'spawn_threw', detail: String(err && err.message) };
-  }
-
-  // Honor per-error-code graceful degradation.
-  if (res && res.error) {
-    if (res.error.code === 'ENOENT') {
-      return { success: false, error: 'python_unavailable' };
-    }
-    if (res.error.code === 'ETIMEDOUT' || res.error.code === 'ERR_CHILD_PROCESS_TIMEOUT') {
-      return { success: false, error: 'timeout' };
-    }
-    return { success: false, error: 'spawn_error', detail: String(res.error.message) };
-  }
-
-  if (res && res.signal === 'SIGTERM') {
-    // Node delivers SIGTERM on timeout; covers the case when the child was
-    // killed without the error object being populated.
-    return { success: false, error: 'timeout' };
-  }
-
-  const stdout = (res && typeof res.stdout === 'string') ? res.stdout : '';
-  const stderr = (res && typeof res.stderr === 'string') ? res.stderr : '';
-
-  if (res && res.status !== 0) {
-    return {
-      success: false,
-      error: 'python_exit_' + res.status,
-      stderr: stderr.slice(0, 500),
-    };
-  }
-
-  let parsed;
-  try {
-    parsed = JSON.parse(stdout);
-  } catch (_e) {
-    return {
-      success: false,
-      error: 'parse_error',
-      stderr: stderr.slice(0, 500),
-      stdout_sample: stdout.slice(0, 200),
-    };
-  }
-
-  if (!parsed || typeof parsed !== 'object') {
-    return { success: false, error: 'parse_error', stdout_sample: stdout.slice(0, 200) };
-  }
-
-  // Bridge-internal failure: child returned {success:false, error:'...'} envelope.
-  if (parsed.success !== true) {
-    return {
-      success: false,
-      error: typeof parsed.error === 'string' ? parsed.error : 'bridge_error',
-    };
-  }
-
-  // Canon Part 8 Layer 2: post-receive audit. The rs-external namespace is
-  // supposed to hold only public metadata, but trust is not a security
-  // property. Re-audit the hits[] payload before returning to the caller.
-  // Throws ExternalEgressViolation on hit (intentional escape route).
-  auditQueryObject(parsed, 'pinecone-bridge');
-
-  return parsed;
-}
-
-// ---------- cosineSimilarity ----------
-//
-// Pure JS pair-wise cosine similarity on equal-length numeric arrays.
-// Used by the differential scorer to compute BERT cosine between the two
-// vectors returned by twin queryPineconeWithVectors calls (one per concept).
-//
-// Inputs:
-//   a, b   number[] of equal length (typically 1024 for multilingual-e5-large)
-//
-// Returns:
-//   number in [-1, 1]  -- Pinecone vectors are typically L2-normalized,
-//                         so the result is usually in [0, 1].
-//   0                  -- if either vector is degenerate (norm == 0) OR
-//                         lengths differ. Caller treats 0 as no signal.
-//
-// Throws:
-//   never. Returns 0 on shape errors so the caller's downstream logic
-//   stays in graceful-degradation territory.
-
-function cosineSimilarity(a, b) {
-  if (!Array.isArray(a) || !Array.isArray(b)) return 0;
-  if (a.length !== b.length || a.length === 0) return 0;
-  let dot = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let i = 0; i < a.length; i += 1) {
-    const x = a[i];
-    const y = b[i];
-    if (typeof x !== 'number' || typeof y !== 'number') return 0;
-    dot += x * y;
-    normA += x * x;
-    normB += y * y;
-  }
-  if (normA === 0 || normB === 0) return 0;
-  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
-}
-
-// ---------- Exports ----------
-
-module.exports = {
-  queryPineconeWithVectors: queryPineconeWithVectors,
-  cosineSimilarity: cosineSimilarity,
-  _test: {
-    PYTHON_BIN: PYTHON_BIN,
-    BRIDGE_TIMEOUT_MS: BRIDGE_TIMEOUT_MS,
-    DEFAULT_LIMIT: DEFAULT_LIMIT,
-    REPO_ROOT: REPO_ROOT,
-    bridgeScript: bridgeScript,
-  },
-};

package/lib/core/rs-preprocessor.cjs

@@ -1,350 +0,0 @@
-/*
- * Copyright (c) 2026 Mindrian. BSL 1.1.
- * Phase 89.2 Plan 06 Task 2 -- Phase 2 deterministic preprocessor.
- *
- * Per-document deterministic processor that produces the canonical 5-field
- * preprocessed shape from raw fetcher output (papers + patents + signals
- * all flow through the same preprocessor with source-aware extraction):
- *
- *   {
- *     document_id     : string  (copy of input.id for traceability)
- *     technical_terms : string[]  (top-N from deterministic n-gram + stopword filter)
- *     concepts        : string[]  (intersection with domain_analysis.concepts)
- *     methods         : string[]  (deterministic verb-phrase patterns)
- *     relevance_score : number  in [0,1]  (concept overlap ratio)
- *     boundary_flag   : boolean  (relevance < 0.5 OR zero concept overlap)
- *   }
- *
- * Determinism contract: same documents + same opts -> byte-identical output.
- * No Math.random, no Date.now, no LLM call. The preprocessor is the
- * deterministic bridge between Wave 2 fetchers (Plans 89.2-02 / 03 / 04)
- * and Wave 3 differential scorer (Task 3 of this plan). It produces the
- * concept[] field that the differential scorer consumes pair-wise.
- *
- * Canon Part 7 (Reuse Before Build): consumes domain_analysis from
- * lib/core/rs-domain-analyzer.cjs (89.1) for concept extraction. May
- * also consume substrate metadata via lib/core/rs-brain-substrate.cjs
- * (89.1a) for boundary_flag heuristic refinement -- the relevance_score
- * derived from domain_analysis IS the boundary signal in v1.11.0.
- *
- * Canon Part 8 (Graph Boundary): two defense-in-depth layers.
- *
- *   Layer 1 -- pre-input scan: domain_analysis fields are scanned for
- *     FORBIDDEN_PATTERNS before any extraction logic runs. On hit, the
- *     preprocessor returns a structured envelope {error, reason} so the
- *     caller knows the upstream component leaked and can repair before
- *     downstream surfaces (the differential scorer + classifier + thesis
- *     generator all flow downstream from preprocessor output).
- *
- *   Layer 2 -- pre-return audit: the composite preprocessed[] output is
- *     scanned via auditQueryObject before return. Throws
- *     ExternalEgressViolation on any FORBIDDEN_PATTERNS hit. Defense-in-
- *     depth so adversarial document.abstract / document.title content
- *     that leaks through extraction is caught at the gate before the
- *     differential scorer sees it.
- *
- * NEVER throws on operational failure (empty input, missing fields, etc.).
- * ONLY throws ExternalEgressViolation on Canon Part 8 boundary breach.
- *
- * Pure CJS, zero npm deps, node built-ins only.
- */
-'use strict';
-
-const { auditQueryObject } = require('./rs-egress-prompts.cjs');
-const crossRoomAggregator = require('./cross-room-aggregator.cjs');
-// rs-egress-violations is required transitively by rs-egress-prompts; we do
-// not throw ExternalEgressViolation directly from this module (auditQueryObject
-// does). The require below is a reachability witness for the audit chain.
-require('./rs-egress-violations.cjs');
-
-const FORBIDDEN_PATTERNS = crossRoomAggregator.FORBIDDEN_PATTERNS;
-if (!Array.isArray(FORBIDDEN_PATTERNS) || FORBIDDEN_PATTERNS.length < 6) {
-  throw new Error(
-    'rs-preprocessor: FORBIDDEN_PATTERNS re-export failed; ' +
-    'Canon Part 8 drift risk (expected >= 6 patterns from cross-room-aggregator)'
-  );
-}
-
-// ---------- Constants ----------
-
-const TECHNICAL_TERMS_MAX = 20;
-const CONCEPTS_MAX = 10;
-const METHODS_MAX = 5;
-const BOUNDARY_THRESHOLD = 0.5;
-const TEXT_MAX_LEN = 100000;  // 100KB cap on per-document text scan
-
-// English stopwords (mirror rs-domain-analyzer.cjs STOPWORDS set; frozen).
-const STOPWORDS = Object.freeze(new Set([
-  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
-  'has', 'have', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that',
-  'the', 'this', 'to', 'was', 'were', 'will', 'with',
-  'i', 'you', 'we', 'they', 'he', 'she', 'our', 'their',
-  'about', 'above', 'after', 'again', 'all', 'also', 'any', 'been',
-  'before', 'between', 'both', 'but', 'can', 'did', 'do', 'does',
-  'into', 'just', 'more', 'most', 'no', 'not', 'now', 'only',
-  'over', 'own', 'same', 'so', 'some', 'such', 'than', 'then',
-  'there', 'these', 'those', 'through', 'under', 'up', 'very',
-  'what', 'when', 'where', 'which', 'who', 'why', 'how',
-  'while', 'because', 'against', 'last', 'week', 'held', 'results',
-]));
-
-// Technical-term regex: lowercase alphanumeric tokens 4-20 chars, optionally hyphenated.
-const TECHNICAL_TERM_REGEX = /\b[a-z][a-z0-9]{3,19}(?:-[a-z0-9]{1,15})?\b/gi;
-
-// Method-verb regex: deterministic verb-phrase capture.
-const METHOD_REGEX = /\b(applied|used|tested|measured|computed|calculated|analyzed|developed|implemented|proposed|designed)\s+([a-z][a-z0-9 -]{2,30})\b/gi;
-
-// ---------- preInputScan ----------
-//
-// Scans every string field of the domain_analysis input against
-// FORBIDDEN_PATTERNS. Returns true if any field hits a forbidden pattern.
-
-function preInputScan(da) {
-  if (!da || typeof da !== 'object') return false;
-  for (const key of Object.keys(da)) {
-    const v = da[key];
-    if (typeof v === 'string') {
-      for (const re of FORBIDDEN_PATTERNS) {
-        if (re.test(v)) return true;
-      }
-    } else if (Array.isArray(v)) {
-      for (const item of v) {
-        if (typeof item === 'string') {
-          for (const re of FORBIDDEN_PATTERNS) {
-            if (re.test(item)) return true;
-          }
-        }
-      }
-    }
-  }
-  return false;
-}
-
-// ---------- tokenize ----------
-//
-// Lowercase, alpha+digit only, length >= 3, stopwords stripped.
-// Deterministic.
-
-function tokenize(s) {
-  if (typeof s !== 'string') return [];
-  const truncated = s.length > TEXT_MAX_LEN ? s.slice(0, TEXT_MAX_LEN) : s;
-  return truncated.toLowerCase()
-    .replace(/[^a-z0-9\s_-]/g, ' ')
-    .split(/\s+/)
-    .filter(function (t) { return t.length >= 3 && !STOPWORDS.has(t); });
-}
-
-// ---------- extractTechnicalTerms ----------
-//
-// Extract technical terms from text via TECHNICAL_TERM_REGEX. Lowercase,
-// dedupe, filter stopwords, count by frequency, sort by count desc + alpha,
-// take top TECHNICAL_TERMS_MAX. Determinism anchor: when counts tie, alpha
-// sort breaks ties so output is byte-stable.
-
-function extractTechnicalTerms(text) {
-  if (typeof text !== 'string' || text.length === 0) return [];
-  const truncated = text.length > TEXT_MAX_LEN ? text.slice(0, TEXT_MAX_LEN) : text;
-  const matches = truncated.toLowerCase().match(TECHNICAL_TERM_REGEX) || [];
-  const counts = new Map();
-  for (const m of matches) {
-    const t = m.trim();
-    if (t.length < 4 || STOPWORDS.has(t)) continue;
-    counts.set(t, (counts.get(t) || 0) + 1);
-  }
-  const arr = [];
-  for (const [term, count] of counts.entries()) {
-    arr.push({ term: term, count: count });
-  }
-  arr.sort(function (a, b) {
-    if (b.count !== a.count) return b.count - a.count;
-    return a.term < b.term ? -1 : (a.term > b.term ? 1 : 0);
-  });
-  return arr.slice(0, TECHNICAL_TERMS_MAX).map(function (e) { return e.term; });
-}
-
-// ---------- extractConcepts ----------
-//
-// Intersection of text tokens with domain_analysis.concepts. Deterministic.
-// Capped at CONCEPTS_MAX. Order: domain_analysis.concepts iteration order.
-
-function extractConcepts(text, da) {
-  if (!da || !Array.isArray(da.concepts) || da.concepts.length === 0) return [];
-  if (typeof text !== 'string' || text.length === 0) return [];
-  const tokens = new Set(tokenize(text));
-  const out = [];
-  const seen = new Set();
-  for (const c of da.concepts) {
-    if (typeof c !== 'string') continue;
-    const cl = c.toLowerCase();
-    if (seen.has(cl)) continue;
-    if (tokens.has(cl)) {
-      seen.add(cl);
-      out.push(cl);
-      if (out.length >= CONCEPTS_MAX) break;
-    }
-  }
-  return out;
-}
-
-// ---------- extractMethods ----------
-//
-// Deterministic verb-phrase capture via METHOD_REGEX. Take match group 2,
-// trim, lowercase, dedupe, cap at METHODS_MAX.
-
-function extractMethods(text) {
-  if (typeof text !== 'string' || text.length === 0) return [];
-  const truncated = text.length > TEXT_MAX_LEN ? text.slice(0, TEXT_MAX_LEN) : text;
-  const out = [];
-  const seen = new Set();
-  let match;
-  // Reset regex stateful position for safety.
-  METHOD_REGEX.lastIndex = 0;
-  while ((match = METHOD_REGEX.exec(truncated)) !== null) {
-    const phrase = (match[2] || '').trim().toLowerCase();
-    if (phrase.length === 0) continue;
-    if (seen.has(phrase)) continue;
-    seen.add(phrase);
-    out.push(phrase);
-    if (out.length >= METHODS_MAX) break;
-  }
-  return out;
-}
-
-// ---------- computeRelevance ----------
-//
-// Concept-overlap ratio (concepts found in text / total domain_analysis.concepts).
-// Bounded [0,1]. Deterministic.
-
-function computeRelevance(text, da) {
-  if (!da || !Array.isArray(da.concepts) || da.concepts.length === 0) return 0;
-  if (typeof text !== 'string' || text.length === 0) return 0;
-  const overlap = extractConcepts(text, da).length;
-  const ratio = overlap / da.concepts.length;
-  return Math.max(0, Math.min(1, ratio));
-}
-
-// ---------- computeBoundaryFlag ----------
-//
-// True if relevance below threshold OR zero concept overlap. The boundary
-// flag tells the differential scorer (and classifier downstream) that this
-// document is on the edge of the user's substrate; cross-domain analogies
-// surface most strongly when at least one side is boundary-flagged.
-
-function computeBoundaryFlag(relevance, concepts) {
-  if (typeof relevance !== 'number' || isNaN(relevance)) return true;
-  if (!Array.isArray(concepts) || concepts.length === 0) return true;
-  return relevance < BOUNDARY_THRESHOLD;
-}
-
-// ---------- preprocess ----------
-//
-// Public entry point. Takes raw fetcher output (papers + patents + signals)
-// and produces deterministic per-document preprocessed records.
-//
-// Inputs:
-//   documents  Array<{id, abstract?, signal?, title?, source?, ...}>
-//   opts       optional object:
-//     domain_analysis  {primary_domain, concepts[], terminology[], methods[]}
-//                      from lib/core/rs-domain-analyzer.cjs
-//
-// Output (happy path):
-//   Array<{document_id, technical_terms[], concepts[], methods[],
-//          relevance_score, boundary_flag, warning?}>
-//
-// Output (Canon Part 8 input rejection):
-//   {error: 'invalid_input', reason: 'forbidden_input'}
-//
-// Throws:
-//   ExternalEgressViolation  if composite output contains FORBIDDEN_PATTERNS
-//                            (intentional Canon Part 8 escape route).
-
-function preprocess(documents, opts) {
-  if (!Array.isArray(documents)) return [];
-  if (documents.length === 0) return [];
-
-  opts = opts || {};
-  const domain_analysis = opts.domain_analysis || { concepts: [], terminology: [], methods: [] };
-
-  // Canon Part 8 Layer 1: pre-input scan on domain_analysis.
-  if (preInputScan(domain_analysis)) {
-    return { error: 'invalid_input', reason: 'forbidden_input' };
-  }
-
-  const out = [];
-  for (const doc of documents) {
-    if (!doc || typeof doc !== 'object') continue;
-    const document_id = (typeof doc.id === 'string') ? doc.id : '';
-    // Source-aware text extraction. Papers + patents have abstract; signals
-    // have signal. Title is fallback. All flow through the same preprocessing.
-    let text = '';
-    if (typeof doc.abstract === 'string' && doc.abstract.length > 0) {
-      text = doc.abstract;
-    } else if (typeof doc.signal === 'string' && doc.signal.length > 0) {
-      text = doc.signal;
-    } else if (typeof doc.title === 'string' && doc.title.length > 0) {
-      text = doc.title;
-    }
-
-    if (text.length === 0) {
-      out.push({
-        document_id: document_id,
-        technical_terms: [],
-        concepts: [],
-        methods: [],
-        relevance_score: 0,
-        boundary_flag: true,
-        warning: 'document_text_missing',
-      });
-      continue;
-    }
-
-    // Title is included in the technical-term + concept extraction surface
-    // when present, so adversarial title content (Test 9) flows through the
-    // pre-return audit and gets caught.
-    const composite = (typeof doc.title === 'string' ? doc.title + ' ' : '') + text;
-
-    const technical_terms = extractTechnicalTerms(composite);
-    const concepts = extractConcepts(composite, domain_analysis);
-    const methods = extractMethods(composite);
-    const relevance_score = computeRelevance(composite, domain_analysis);
-    const boundary_flag = computeBoundaryFlag(relevance_score, concepts);
-
-    out.push({
-      document_id: document_id,
-      technical_terms: technical_terms,
-      concepts: concepts,
-      methods: methods,
-      relevance_score: relevance_score,
-      boundary_flag: boundary_flag,
-    });
-  }
-
-  // Canon Part 8 Layer 2: pre-return audit. Throws ExternalEgressViolation
-  // on any FORBIDDEN_PATTERNS hit in the composite preprocessed[] output.
-  // This is the intentional escape route from preprocess().
-  auditQueryObject(out, 'preprocessor');
-
-  return out;
-}
-
-// ---------- Exports ----------
-
-module.exports = {
-  preprocess: preprocess,
-  _test: {
-    extractTechnicalTerms: extractTechnicalTerms,
-    extractConcepts: extractConcepts,
-    extractMethods: extractMethods,
-    computeRelevance: computeRelevance,
-    computeBoundaryFlag: computeBoundaryFlag,
-    tokenize: tokenize,
-    preInputScan: preInputScan,
-    TECHNICAL_TERM_REGEX: TECHNICAL_TERM_REGEX,
-    METHOD_REGEX: METHOD_REGEX,
-    STOPWORDS: STOPWORDS,
-    TECHNICAL_TERMS_MAX: TECHNICAL_TERMS_MAX,
-    CONCEPTS_MAX: CONCEPTS_MAX,
-    METHODS_MAX: METHODS_MAX,
-    BOUNDARY_THRESHOLD: BOUNDARY_THRESHOLD,
-  },
-};