@mindrian_os/install 1.13.0-beta.11

package/lib/core/rs-pinecone-bridge.cjs

@@ -0,0 +1,303 @@
+/*
+ * Copyright (c) 2026 Mindrian. BSL 1.1.
+ * Phase 89.2 Plan 06 Task 1 -- Pinecone-direct bridge for 1024-dim
+ * vector retrieval (CJS->Python shim).
+ *
+ * The bridge wraps the existing rs_cache.py::fetch_all_from_namespace
+ * primitive (Phase 89-03, v1.10.16) without modifying it. This is a
+ * Canon Part 7 (Reuse Before Build) consumer: zero new Python code,
+ * zero new index, zero new field schema. The strict-1024-dim deferred
+ * item from 89.1 (substrate carries _score not embedding on the live
+ * Brain wire path) is resolved here by going Pinecone-direct against
+ * the rs-external index used for external corpora.
+ *
+ * Architecture:
+ *
+ *   queryPineconeWithVectors(namespace, queryText, topK)
+ *     -> spawnSync python3 -c bridgeScript
+ *     -> bridgeScript reads JSON from stdin
+ *     -> bridgeScript invokes rs_cache.fetch_all_from_namespace
+ *     -> bridgeScript writes JSON {success, hits[]} to stdout
+ *     -> CJS parses + returns
+ *
+ * Graceful degradation contract (Phase 89.2-01 / 02 / 03 / 04 pattern):
+ *
+ *   missing PINECONE_API_KEY -> {success: false, error: 'pinecone_api_key_missing'}
+ *   python3 not on PATH      -> {success: false, error: 'python_unavailable'}
+ *   non-zero exit            -> {success: false, error: 'python_exit_<N>'}
+ *   stdout parse failure     -> {success: false, error: 'parse_error'}
+ *   timeout                  -> {success: false, error: 'timeout'}
+ *
+ * NEVER throws. Every external boundary returns a structured envelope.
+ *
+ * Canon Part 8 (Graph Boundary): two defense-in-depth layers.
+ *
+ *   Layer 1 -- pre-egress: auditQueryString runs on the queryText
+ *     parameter before spawnSync. Throws ExternalEgressViolation on
+ *     FORBIDDEN_PATTERNS hit. Adversarial input never reaches Pinecone.
+ *
+ *   Layer 2 -- post-receive: auditQueryObject runs on the parsed hits[]
+ *     payload before return. Throws ExternalEgressViolation if a hit's
+ *     metadata smuggles a forbidden pattern through. The rs-external
+ *     namespace is supposed to hold only public OpenAlex/arXiv metadata
+ *     (Phase 89-03 Canon attestation), but we re-audit on the receive
+ *     side because trust is not a security property -- enforcement is.
+ *
+ * Additionally exposes:
+ *
+ *   cosineSimilarity(a, b) -- pure JS dot(a,b)/(norm(a)*norm(b)).
+ *     Both inputs MUST be Number arrays of identical length (typically
+ *     1024 for Pinecone multilingual-e5-large). Returns 0 on degenerate
+ *     zero-vectors. Differential scorer uses this on the two query
+ *     vectors returned by twin queryPineconeWithVectors calls.
+ *
+ * Pure CJS, zero npm deps, node built-ins only beyond the three
+ * rs-egress-* primitives shipped in Wave 1 (Plan 89.2-01).
+ */
+'use strict';
+
+const path = require('node:path');
+const childProcess = require('child_process');
+const { auditQueryString, auditQueryObject } = require('./rs-egress-prompts.cjs');
+// rs-egress-violations is required transitively by rs-egress-prompts; we do
+// not throw ExternalEgressViolation directly from this module (the audit
+// helpers do). The require below is a reachability witness for the Canon
+// Part 8 audit chain at module-load time.
+require('./rs-egress-violations.cjs');
+
+// ---------- Constants ----------
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+const PYTHON_BIN = process.env.MINDRIAN_PYTHON || 'python3';
+const BRIDGE_TIMEOUT_MS = 30000;
+const DEFAULT_LIMIT = 1000;
+
+// ---------- Embedded Python bridge script ----------
+//
+// This script is spawned as `python3 -c <script>`. It MUST be valid Python.
+// The leading `# BSL 1.1` line is a deliberate Canon discipline marker so
+// the embedded Python source is licensed identically to the surrounding
+// CJS file (W4 fix per Plan 89.2-06).
+//
+// Protocol:
+//
+//   stdin   : JSON {namespace: string, limit?: integer}
+//   stdout  : JSON {success: boolean, hits?: array, error?: string}
+//
+// The script imports rs_cache from lib.core (REPO_ROOT must be on
+// sys.path). On any exception, returns success:false with the exception
+// message. Never raises to the parent process.
+
+const bridgeScript = [
+  '# BSL 1.1',
+  '# Copyright (c) 2026 Mindrian. Phase 89.2 Plan 06 Task 1 -- pinecone bridge embedded Python.',
+  '# This script wraps rs_cache.py::fetch_all_from_namespace to retrieve',
+  '# 1024-dim Pinecone-direct vectors. JSON over stdio: parent writes a',
+  '# request envelope to stdin; child writes a response envelope to stdout.',
+  '# Errors are reported via the response envelope (success:false), never',
+  '# raised to the parent process. The child exits 0 on graceful failure.',
+  'import sys, os, json',
+  '# Ensure REPO_ROOT is on sys.path so `from lib.core.rs_cache import ...` resolves.',
+  'sys.path.insert(0, os.environ.get("MINDRIAN_REPO_ROOT", os.getcwd()))',
+  'try:',
+  '    from lib.core.rs_cache import fetch_all_from_namespace',
+  '    payload = json.loads(sys.stdin.read())',
+  '    ns = payload["namespace"]',
+  '    limit = int(payload.get("limit", 1000))',
+  '    records = fetch_all_from_namespace(ns, limit=limit)',
+  '    # Map rs_cache.py output {id, values, metadata} to bridge hits[].',
+  '    hits = []',
+  '    for r in records:',
+  '        hits.append({',
+  '            "id": r.get("id", ""),',
+  '            "score": float(r.get("score", 0.0)) if "score" in r else 0.0,',
+  '            "values": list(r.get("values") or []),',
+  '            "metadata": dict(r.get("metadata") or {}),',
+  '        })',
+  '    sys.stdout.write(json.dumps({"success": True, "hits": hits}))',
+  'except Exception as e:',
+  '    sys.stdout.write(json.dumps({"success": False, "error": str(e)}))',
+  '',
+].join('\n');
+
+// ---------- queryPineconeWithVectors ----------
+//
+// Public entry point. Spawns python3 with the embedded bridgeScript,
+// passes the namespace + limit via stdin, parses the JSON response,
+// audits, and returns the structured envelope.
+//
+// Inputs:
+//   namespace  string   the Pinecone namespace, typically 'external:<topic-slug>'
+//   queryText  string   the original user query (audited pre-egress; not used by the
+//                       bridge call itself since fetch_all_from_namespace pulls all
+//                       cached vectors -- the BERT cosine is computed downstream).
+//                       The query text is included in the audit triple so the
+//                       caller's intent is checked at the gate.
+//   topK       integer  result cap (default 1000; bounded by rs_cache.MAX_NAMESPACE_VECTORS)
+//
+// Returns Promise resolving to:
+//   { success: true,  hits: [{id, score, values: number[], metadata}] }
+//   { success: false, error: 'pinecone_api_key_missing'|'python_unavailable'
+//                          |'python_exit_<N>'|'parse_error'|'timeout'
+//                          |'invalid_namespace'|'invalid_top_k', stderr?: string }
+//
+// Throws (the only escape route -- Canon Part 8):
+//   ExternalEgressViolation  if queryText matches FORBIDDEN_PATTERNS
+//                            OR if returned hits[] metadata smuggles a forbidden
+//                            pattern through.
+
+async function queryPineconeWithVectors(namespace, queryText, topK) {
+  // Argument validation (graceful, never throws on shape errors).
+  if (typeof namespace !== 'string' || namespace.length === 0) {
+    return { success: false, error: 'invalid_namespace' };
+  }
+  if (typeof queryText !== 'string') {
+    return { success: false, error: 'invalid_query_text' };
+  }
+  const limit = (typeof topK === 'number' && topK > 0) ? Math.floor(topK) : DEFAULT_LIMIT;
+
+  // Canon Part 8 Layer 1: pre-egress audit on user-controlled queryText.
+  // auditQueryString throws ExternalEgressViolation on hit -- this is the
+  // single intentional escape route from this module.
+  auditQueryString(queryText, 'pinecone-bridge');
+  // Audit the namespace too (defense-in-depth; namespaces are typically
+  // 'external:<topic-slug>' so the topic-slug carries the user's domain).
+  auditQueryString(namespace, 'pinecone-bridge');
+
+  // Env-var gate. Pinecone SDK requires PINECONE_API_KEY; if absent the
+  // child process raises a RuntimeError. Short-circuit here so the user
+  // sees the canonical 'pinecone_api_key_missing' tag and not a stack trace.
+  if (!process.env.PINECONE_API_KEY) {
+    return { success: false, error: 'pinecone_api_key_missing' };
+  }
+
+  const requestEnvelope = JSON.stringify({ namespace: namespace, limit: limit });
+  const childEnv = Object.assign({}, process.env, {
+    MINDRIAN_REPO_ROOT: REPO_ROOT,
+  });
+
+  let res;
+  try {
+    res = childProcess.spawnSync(PYTHON_BIN, ['-c', bridgeScript], {
+      input: requestEnvelope,
+      encoding: 'utf8',
+      timeout: BRIDGE_TIMEOUT_MS,
+      env: childEnv,
+      cwd: REPO_ROOT,
+    });
+  } catch (err) {
+    // spawnSync itself does not normally throw; this is defense-in-depth.
+    return { success: false, error: 'spawn_threw', detail: String(err && err.message) };
+  }
+
+  // Honor per-error-code graceful degradation.
+  if (res && res.error) {
+    if (res.error.code === 'ENOENT') {
+      return { success: false, error: 'python_unavailable' };
+    }
+    if (res.error.code === 'ETIMEDOUT' || res.error.code === 'ERR_CHILD_PROCESS_TIMEOUT') {
+      return { success: false, error: 'timeout' };
+    }
+    return { success: false, error: 'spawn_error', detail: String(res.error.message) };
+  }
+
+  if (res && res.signal === 'SIGTERM') {
+    // Node delivers SIGTERM on timeout; covers the case when the child was
+    // killed without the error object being populated.
+    return { success: false, error: 'timeout' };
+  }
+
+  const stdout = (res && typeof res.stdout === 'string') ? res.stdout : '';
+  const stderr = (res && typeof res.stderr === 'string') ? res.stderr : '';
+
+  if (res && res.status !== 0) {
+    return {
+      success: false,
+      error: 'python_exit_' + res.status,
+      stderr: stderr.slice(0, 500),
+    };
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(stdout);
+  } catch (_e) {
+    return {
+      success: false,
+      error: 'parse_error',
+      stderr: stderr.slice(0, 500),
+      stdout_sample: stdout.slice(0, 200),
+    };
+  }
+
+  if (!parsed || typeof parsed !== 'object') {
+    return { success: false, error: 'parse_error', stdout_sample: stdout.slice(0, 200) };
+  }
+
+  // Bridge-internal failure: child returned {success:false, error:'...'} envelope.
+  if (parsed.success !== true) {
+    return {
+      success: false,
+      error: typeof parsed.error === 'string' ? parsed.error : 'bridge_error',
+    };
+  }
+
+  // Canon Part 8 Layer 2: post-receive audit. The rs-external namespace is
+  // supposed to hold only public metadata, but trust is not a security
+  // property. Re-audit the hits[] payload before returning to the caller.
+  // Throws ExternalEgressViolation on hit (intentional escape route).
+  auditQueryObject(parsed, 'pinecone-bridge');
+
+  return parsed;
+}
+
+// ---------- cosineSimilarity ----------
+//
+// Pure JS pair-wise cosine similarity on equal-length numeric arrays.
+// Used by the differential scorer to compute BERT cosine between the two
+// vectors returned by twin queryPineconeWithVectors calls (one per concept).
+//
+// Inputs:
+//   a, b   number[] of equal length (typically 1024 for multilingual-e5-large)
+//
+// Returns:
+//   number in [-1, 1]  -- Pinecone vectors are typically L2-normalized,
+//                         so the result is usually in [0, 1].
+//   0                  -- if either vector is degenerate (norm == 0) OR
+//                         lengths differ. Caller treats 0 as no signal.
+//
+// Throws:
+//   never. Returns 0 on shape errors so the caller's downstream logic
+//   stays in graceful-degradation territory.
+
+function cosineSimilarity(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b)) return 0;
+  if (a.length !== b.length || a.length === 0) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    const x = a[i];
+    const y = b[i];
+    if (typeof x !== 'number' || typeof y !== 'number') return 0;
+    dot += x * y;
+    normA += x * x;
+    normB += y * y;
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+// ---------- Exports ----------
+
+module.exports = {
+  queryPineconeWithVectors: queryPineconeWithVectors,
+  cosineSimilarity: cosineSimilarity,
+  _test: {
+    PYTHON_BIN: PYTHON_BIN,
+    BRIDGE_TIMEOUT_MS: BRIDGE_TIMEOUT_MS,
+    DEFAULT_LIMIT: DEFAULT_LIMIT,
+    REPO_ROOT: REPO_ROOT,
+    bridgeScript: bridgeScript,
+  },
+};

package/lib/core/rs-preprocessor.cjs

@@ -0,0 +1,350 @@
+/*
+ * Copyright (c) 2026 Mindrian. BSL 1.1.
+ * Phase 89.2 Plan 06 Task 2 -- Phase 2 deterministic preprocessor.
+ *
+ * Per-document deterministic processor that produces the canonical 5-field
+ * preprocessed shape from raw fetcher output (papers + patents + signals
+ * all flow through the same preprocessor with source-aware extraction):
+ *
+ *   {
+ *     document_id     : string  (copy of input.id for traceability)
+ *     technical_terms : string[]  (top-N from deterministic n-gram + stopword filter)
+ *     concepts        : string[]  (intersection with domain_analysis.concepts)
+ *     methods         : string[]  (deterministic verb-phrase patterns)
+ *     relevance_score : number  in [0,1]  (concept overlap ratio)
+ *     boundary_flag   : boolean  (relevance < 0.5 OR zero concept overlap)
+ *   }
+ *
+ * Determinism contract: same documents + same opts -> byte-identical output.
+ * No Math.random, no Date.now, no LLM call. The preprocessor is the
+ * deterministic bridge between Wave 2 fetchers (Plans 89.2-02 / 03 / 04)
+ * and Wave 3 differential scorer (Task 3 of this plan). It produces the
+ * concept[] field that the differential scorer consumes pair-wise.
+ *
+ * Canon Part 7 (Reuse Before Build): consumes domain_analysis from
+ * lib/core/rs-domain-analyzer.cjs (89.1) for concept extraction. May
+ * also consume substrate metadata via lib/core/rs-brain-substrate.cjs
+ * (89.1a) for boundary_flag heuristic refinement -- the relevance_score
+ * derived from domain_analysis IS the boundary signal in v1.11.0.
+ *
+ * Canon Part 8 (Graph Boundary): two defense-in-depth layers.
+ *
+ *   Layer 1 -- pre-input scan: domain_analysis fields are scanned for
+ *     FORBIDDEN_PATTERNS before any extraction logic runs. On hit, the
+ *     preprocessor returns a structured envelope {error, reason} so the
+ *     caller knows the upstream component leaked and can repair before
+ *     downstream surfaces (the differential scorer + classifier + thesis
+ *     generator all flow downstream from preprocessor output).
+ *
+ *   Layer 2 -- pre-return audit: the composite preprocessed[] output is
+ *     scanned via auditQueryObject before return. Throws
+ *     ExternalEgressViolation on any FORBIDDEN_PATTERNS hit. Defense-in-
+ *     depth so adversarial document.abstract / document.title content
+ *     that leaks through extraction is caught at the gate before the
+ *     differential scorer sees it.
+ *
+ * NEVER throws on operational failure (empty input, missing fields, etc.).
+ * ONLY throws ExternalEgressViolation on Canon Part 8 boundary breach.
+ *
+ * Pure CJS, zero npm deps, node built-ins only.
+ */
+'use strict';
+
+const { auditQueryObject } = require('./rs-egress-prompts.cjs');
+const crossRoomAggregator = require('./cross-room-aggregator.cjs');
+// rs-egress-violations is required transitively by rs-egress-prompts; we do
+// not throw ExternalEgressViolation directly from this module (auditQueryObject
+// does). The require below is a reachability witness for the audit chain.
+require('./rs-egress-violations.cjs');
+
+const FORBIDDEN_PATTERNS = crossRoomAggregator.FORBIDDEN_PATTERNS;
+if (!Array.isArray(FORBIDDEN_PATTERNS) || FORBIDDEN_PATTERNS.length < 6) {
+  throw new Error(
+    'rs-preprocessor: FORBIDDEN_PATTERNS re-export failed; ' +
+    'Canon Part 8 drift risk (expected >= 6 patterns from cross-room-aggregator)'
+  );
+}
+
+// ---------- Constants ----------
+
+const TECHNICAL_TERMS_MAX = 20;
+const CONCEPTS_MAX = 10;
+const METHODS_MAX = 5;
+const BOUNDARY_THRESHOLD = 0.5;
+const TEXT_MAX_LEN = 100000;  // 100KB cap on per-document text scan
+
+// English stopwords (mirror rs-domain-analyzer.cjs STOPWORDS set; frozen).
+const STOPWORDS = Object.freeze(new Set([
+  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
+  'has', 'have', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that',
+  'the', 'this', 'to', 'was', 'were', 'will', 'with',
+  'i', 'you', 'we', 'they', 'he', 'she', 'our', 'their',
+  'about', 'above', 'after', 'again', 'all', 'also', 'any', 'been',
+  'before', 'between', 'both', 'but', 'can', 'did', 'do', 'does',
+  'into', 'just', 'more', 'most', 'no', 'not', 'now', 'only',
+  'over', 'own', 'same', 'so', 'some', 'such', 'than', 'then',
+  'there', 'these', 'those', 'through', 'under', 'up', 'very',
+  'what', 'when', 'where', 'which', 'who', 'why', 'how',
+  'while', 'because', 'against', 'last', 'week', 'held', 'results',
+]));
+
+// Technical-term regex: lowercase alphanumeric tokens 4-20 chars, optionally hyphenated.
+const TECHNICAL_TERM_REGEX = /\b[a-z][a-z0-9]{3,19}(?:-[a-z0-9]{1,15})?\b/gi;
+
+// Method-verb regex: deterministic verb-phrase capture.
+const METHOD_REGEX = /\b(applied|used|tested|measured|computed|calculated|analyzed|developed|implemented|proposed|designed)\s+([a-z][a-z0-9 -]{2,30})\b/gi;
+
+// ---------- preInputScan ----------
+//
+// Scans every string field of the domain_analysis input against
+// FORBIDDEN_PATTERNS. Returns true if any field hits a forbidden pattern.
+
+function preInputScan(da) {
+  if (!da || typeof da !== 'object') return false;
+  for (const key of Object.keys(da)) {
+    const v = da[key];
+    if (typeof v === 'string') {
+      for (const re of FORBIDDEN_PATTERNS) {
+        if (re.test(v)) return true;
+      }
+    } else if (Array.isArray(v)) {
+      for (const item of v) {
+        if (typeof item === 'string') {
+          for (const re of FORBIDDEN_PATTERNS) {
+            if (re.test(item)) return true;
+          }
+        }
+      }
+    }
+  }
+  return false;
+}
+
+// ---------- tokenize ----------
+//
+// Lowercase, alpha+digit only, length >= 3, stopwords stripped.
+// Deterministic.
+
+function tokenize(s) {
+  if (typeof s !== 'string') return [];
+  const truncated = s.length > TEXT_MAX_LEN ? s.slice(0, TEXT_MAX_LEN) : s;
+  return truncated.toLowerCase()
+    .replace(/[^a-z0-9\s_-]/g, ' ')
+    .split(/\s+/)
+    .filter(function (t) { return t.length >= 3 && !STOPWORDS.has(t); });
+}
+
+// ---------- extractTechnicalTerms ----------
+//
+// Extract technical terms from text via TECHNICAL_TERM_REGEX. Lowercase,
+// dedupe, filter stopwords, count by frequency, sort by count desc + alpha,
+// take top TECHNICAL_TERMS_MAX. Determinism anchor: when counts tie, alpha
+// sort breaks ties so output is byte-stable.
+
+function extractTechnicalTerms(text) {
+  if (typeof text !== 'string' || text.length === 0) return [];
+  const truncated = text.length > TEXT_MAX_LEN ? text.slice(0, TEXT_MAX_LEN) : text;
+  const matches = truncated.toLowerCase().match(TECHNICAL_TERM_REGEX) || [];
+  const counts = new Map();
+  for (const m of matches) {
+    const t = m.trim();
+    if (t.length < 4 || STOPWORDS.has(t)) continue;
+    counts.set(t, (counts.get(t) || 0) + 1);
+  }
+  const arr = [];
+  for (const [term, count] of counts.entries()) {
+    arr.push({ term: term, count: count });
+  }
+  arr.sort(function (a, b) {
+    if (b.count !== a.count) return b.count - a.count;
+    return a.term < b.term ? -1 : (a.term > b.term ? 1 : 0);
+  });
+  return arr.slice(0, TECHNICAL_TERMS_MAX).map(function (e) { return e.term; });
+}
+
+// ---------- extractConcepts ----------
+//
+// Intersection of text tokens with domain_analysis.concepts. Deterministic.
+// Capped at CONCEPTS_MAX. Order: domain_analysis.concepts iteration order.
+
+function extractConcepts(text, da) {
+  if (!da || !Array.isArray(da.concepts) || da.concepts.length === 0) return [];
+  if (typeof text !== 'string' || text.length === 0) return [];
+  const tokens = new Set(tokenize(text));
+  const out = [];
+  const seen = new Set();
+  for (const c of da.concepts) {
+    if (typeof c !== 'string') continue;
+    const cl = c.toLowerCase();
+    if (seen.has(cl)) continue;
+    if (tokens.has(cl)) {
+      seen.add(cl);
+      out.push(cl);
+      if (out.length >= CONCEPTS_MAX) break;
+    }
+  }
+  return out;
+}
+
+// ---------- extractMethods ----------
+//
+// Deterministic verb-phrase capture via METHOD_REGEX. Take match group 2,
+// trim, lowercase, dedupe, cap at METHODS_MAX.
+
+function extractMethods(text) {
+  if (typeof text !== 'string' || text.length === 0) return [];
+  const truncated = text.length > TEXT_MAX_LEN ? text.slice(0, TEXT_MAX_LEN) : text;
+  const out = [];
+  const seen = new Set();
+  let match;
+  // Reset regex stateful position for safety.
+  METHOD_REGEX.lastIndex = 0;
+  while ((match = METHOD_REGEX.exec(truncated)) !== null) {
+    const phrase = (match[2] || '').trim().toLowerCase();
+    if (phrase.length === 0) continue;
+    if (seen.has(phrase)) continue;
+    seen.add(phrase);
+    out.push(phrase);
+    if (out.length >= METHODS_MAX) break;
+  }
+  return out;
+}
+
+// ---------- computeRelevance ----------
+//
+// Concept-overlap ratio (concepts found in text / total domain_analysis.concepts).
+// Bounded [0,1]. Deterministic.
+
+function computeRelevance(text, da) {
+  if (!da || !Array.isArray(da.concepts) || da.concepts.length === 0) return 0;
+  if (typeof text !== 'string' || text.length === 0) return 0;
+  const overlap = extractConcepts(text, da).length;
+  const ratio = overlap / da.concepts.length;
+  return Math.max(0, Math.min(1, ratio));
+}
+
+// ---------- computeBoundaryFlag ----------
+//
+// True if relevance below threshold OR zero concept overlap. The boundary
+// flag tells the differential scorer (and classifier downstream) that this
+// document is on the edge of the user's substrate; cross-domain analogies
+// surface most strongly when at least one side is boundary-flagged.
+
+function computeBoundaryFlag(relevance, concepts) {
+  if (typeof relevance !== 'number' || isNaN(relevance)) return true;
+  if (!Array.isArray(concepts) || concepts.length === 0) return true;
+  return relevance < BOUNDARY_THRESHOLD;
+}
+
+// ---------- preprocess ----------
+//
+// Public entry point. Takes raw fetcher output (papers + patents + signals)
+// and produces deterministic per-document preprocessed records.
+//
+// Inputs:
+//   documents  Array<{id, abstract?, signal?, title?, source?, ...}>
+//   opts       optional object:
+//     domain_analysis  {primary_domain, concepts[], terminology[], methods[]}
+//                      from lib/core/rs-domain-analyzer.cjs
+//
+// Output (happy path):
+//   Array<{document_id, technical_terms[], concepts[], methods[],
+//          relevance_score, boundary_flag, warning?}>
+//
+// Output (Canon Part 8 input rejection):
+//   {error: 'invalid_input', reason: 'forbidden_input'}
+//
+// Throws:
+//   ExternalEgressViolation  if composite output contains FORBIDDEN_PATTERNS
+//                            (intentional Canon Part 8 escape route).
+
+function preprocess(documents, opts) {
+  if (!Array.isArray(documents)) return [];
+  if (documents.length === 0) return [];
+
+  opts = opts || {};
+  const domain_analysis = opts.domain_analysis || { concepts: [], terminology: [], methods: [] };
+
+  // Canon Part 8 Layer 1: pre-input scan on domain_analysis.
+  if (preInputScan(domain_analysis)) {
+    return { error: 'invalid_input', reason: 'forbidden_input' };
+  }
+
+  const out = [];
+  for (const doc of documents) {
+    if (!doc || typeof doc !== 'object') continue;
+    const document_id = (typeof doc.id === 'string') ? doc.id : '';
+    // Source-aware text extraction. Papers + patents have abstract; signals
+    // have signal. Title is fallback. All flow through the same preprocessing.
+    let text = '';
+    if (typeof doc.abstract === 'string' && doc.abstract.length > 0) {
+      text = doc.abstract;
+    } else if (typeof doc.signal === 'string' && doc.signal.length > 0) {
+      text = doc.signal;
+    } else if (typeof doc.title === 'string' && doc.title.length > 0) {
+      text = doc.title;
+    }
+
+    if (text.length === 0) {
+      out.push({
+        document_id: document_id,
+        technical_terms: [],
+        concepts: [],
+        methods: [],
+        relevance_score: 0,
+        boundary_flag: true,
+        warning: 'document_text_missing',
+      });
+      continue;
+    }
+
+    // Title is included in the technical-term + concept extraction surface
+    // when present, so adversarial title content (Test 9) flows through the
+    // pre-return audit and gets caught.
+    const composite = (typeof doc.title === 'string' ? doc.title + ' ' : '') + text;
+
+    const technical_terms = extractTechnicalTerms(composite);
+    const concepts = extractConcepts(composite, domain_analysis);
+    const methods = extractMethods(composite);
+    const relevance_score = computeRelevance(composite, domain_analysis);
+    const boundary_flag = computeBoundaryFlag(relevance_score, concepts);
+
+    out.push({
+      document_id: document_id,
+      technical_terms: technical_terms,
+      concepts: concepts,
+      methods: methods,
+      relevance_score: relevance_score,
+      boundary_flag: boundary_flag,
+    });
+  }
+
+  // Canon Part 8 Layer 2: pre-return audit. Throws ExternalEgressViolation
+  // on any FORBIDDEN_PATTERNS hit in the composite preprocessed[] output.
+  // This is the intentional escape route from preprocess().
+  auditQueryObject(out, 'preprocessor');
+
+  return out;
+}
+
+// ---------- Exports ----------
+
+module.exports = {
+  preprocess: preprocess,
+  _test: {
+    extractTechnicalTerms: extractTechnicalTerms,
+    extractConcepts: extractConcepts,
+    extractMethods: extractMethods,
+    computeRelevance: computeRelevance,
+    computeBoundaryFlag: computeBoundaryFlag,
+    tokenize: tokenize,
+    preInputScan: preInputScan,
+    TECHNICAL_TERM_REGEX: TECHNICAL_TERM_REGEX,
+    METHOD_REGEX: METHOD_REGEX,
+    STOPWORDS: STOPWORDS,
+    TECHNICAL_TERMS_MAX: TECHNICAL_TERMS_MAX,
+    CONCEPTS_MAX: CONCEPTS_MAX,
+    METHODS_MAX: METHODS_MAX,
+    BOUNDARY_THRESHOLD: BOUNDARY_THRESHOLD,
+  },
+};