@mindrian_os/install 1.13.0-beta.11

package/lib/memory/validators/brain-substrate-invariants.cjs

@@ -0,0 +1,285 @@
+/*
+ * Copyright (c) 2026 Mindrian. BSL 1.1.
+ * Phase 89.1a Plan 02 -- Brain methodology substrate invariants validator.
+ *
+ * Registry-compatible drop-in for the Phase 88-13 guardian. Auto-
+ * discovered by scripts/feynman-minto-guardian.cjs's validator loader
+ * (walks lib/memory/validators/*.cjs). Zero guardian.cjs edits needed.
+ *
+ * Contract (see lib/memory/validators/README.md):
+ *   module.exports = {
+ *     id:           'brain-substrate-invariants',
+ *     severity_map: { critical, error, warning, info },
+ *     scope:        'room',
+ *     validate(roomPath, ctx) -> { severity, violations[] },
+ *   }
+ *
+ * Five checks:
+ *   A) Cache missing -> zero violations (valid default)
+ *   B) Cache unreadable / malformed / schema_version drift -> warning
+ *   C) Cache substrate.length != embedding_count -> critical (partial pull, Pitfall 5)
+ *   D) Cache substrate[].metadata body contains forbidden regex hit -> critical
+ *      (Canon Part 8 attack surface; Pitfall 4)
+ *   E) Cache substrate[] entry shape -> critical (cache_entry_shape_invalid)
+ *      Phase 89.1 Plan 01 relaxation: each entry MUST carry EITHER a 1024-dim
+ *      finite-Float embedding (legacy/mocked path) OR a Pinecone score in
+ *      [0,1] (live brain_search wire path). Neither present is critical;
+ *      both present is allowed. id must match SLUG_REGEX OR the permissive
+ *      Pinecone ID regex [a-zA-Z0-9_:-]{1,512}.
+ *
+ * Fail-open: unexpected throws are caught by the guardian's validator
+ * runner. This validator does not swallow its own exceptions; the
+ * guardian owns that contract. Individual I/O paths (readFileSync) are
+ * wrapped defensively so a missing cache never produces a violation.
+ *
+ * Pure CJS, node built-ins only + re-export of prompts module for
+ * FORBIDDEN_PATTERNS + SLUG_REGEX parity. Importing the canonical
+ * vocabulary (rather than redefining) ensures any Canon Part 8
+ * amendment at lib/core/cross-room-aggregator.cjs propagates through
+ * lib/core/rs-brain-substrate-prompts.cjs to this validator with zero
+ * code change.
+ */
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const prompts = require('../../core/rs-brain-substrate-prompts.cjs');
+const { FORBIDDEN_PATTERNS, SLUG_REGEX } = prompts;
+
+// ---------- Constants ----------
+
+const SEVERITY_MAP = Object.freeze({ critical: 3, error: 2, warning: 1, info: 0 });
+const EXPECTED_EMBEDDING_DIM = 1024;
+const CACHE_SCHEMA_VERSION = '1.0';
+// Cap canon_boundary hits per validate() invocation so one poisoned
+// cache cannot spam the invariant report. Five matches is the same
+// threshold Plan 90-05 brain-md-invariants uses.
+const MAX_CANON_HITS_REPORTED = 5;
+
+// ---------- Severity aggregator ----------
+
+function aggregateSeverity(violations) {
+  if (!violations || violations.length === 0) return null;
+  let max = 0;
+  const inv = { 0: 'info', 1: 'warning', 2: 'error', 3: 'critical' };
+  for (const v of violations) {
+    const s = SEVERITY_MAP[v.severity];
+    if (typeof s === 'number' && s > max) max = s;
+  }
+  return inv[max];
+}
+
+// ---------- validate (Checks A-E) ----------
+
+function validate(roomPath, _ctx) {
+  const violations = [];
+
+  // Defensive guard: non-string roomPath -> zero violations (guardian
+  // always passes a string, but the contract demands a structured
+  // result on every path).
+  if (typeof roomPath !== 'string' || roomPath.length === 0) {
+    return { severity: null, violations: [] };
+  }
+
+  const cachePath = path.join(roomPath, '.mindrian', 'brain-substrate-cache.json');
+
+  // --- Check A: cache absent -> zero violations (healthy default) ---
+  if (!fs.existsSync(cachePath)) {
+    return { severity: null, violations: [] };
+  }
+
+  // --- Check B: unreadable / malformed / schema drift ---
+  let payload = null;
+  try {
+    const raw = fs.readFileSync(cachePath, 'utf8');
+    payload = JSON.parse(raw);
+  } catch (err) {
+    violations.push({
+      category: 'cache_unreadable',
+      severity: 'warning',
+      message: 'brain-substrate-cache.json unreadable: ' + String(err && err.message),
+      action_hint: 'invalidate_cache',
+    });
+    return { severity: aggregateSeverity(violations), violations };
+  }
+
+  if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
+    violations.push({
+      category: 'cache_malformed_root',
+      severity: 'warning',
+      message: 'brain-substrate-cache.json root is not a plain object',
+      action_hint: 'invalidate_cache',
+    });
+    return { severity: aggregateSeverity(violations), violations };
+  }
+
+  if (payload.schema_version !== CACHE_SCHEMA_VERSION) {
+    violations.push({
+      category: 'cache_schema_drift',
+      severity: 'warning',
+      message: 'schema_version expected ' + CACHE_SCHEMA_VERSION +
+               ' got ' + String(payload.schema_version),
+      action_hint: 'invalidate_cache',
+    });
+  }
+
+  if (!Array.isArray(payload.substrate)) {
+    violations.push({
+      category: 'cache_malformed_substrate',
+      severity: 'critical',
+      message: 'substrate is not an array',
+      action_hint: 'invalidate_cache',
+    });
+    return { severity: aggregateSeverity(violations), violations };
+  }
+
+  // --- Check C: completeness (embedding_count vs substrate.length) ---
+  // Pitfall 5 partial-pull completeness guard. Defense-in-depth against
+  // the reader-side check in lib/core/rs-brain-substrate.cjs readCache;
+  // either layer alone blocks a partial cache.
+  if (typeof payload.embedding_count === 'number' &&
+      payload.substrate.length !== payload.embedding_count) {
+    violations.push({
+      category: 'cache_partial_pull',
+      severity: 'critical',
+      message: 'embedding_count ' + payload.embedding_count +
+               ' != substrate.length ' + payload.substrate.length,
+      action_hint: 'invalidate_cache',
+    });
+  }
+
+  // --- Checks D + E: per-entry body scan + shape assertion ---
+  //
+  // Phase 89.1 Plan 01 relaxation: Check E now accepts entries with EITHER a
+  // 1024-dim Float embedding (legacy/mocked path; 89.1a-03 fixtures) OR a
+  // Pinecone score in [0,1] (live brain_search wire path; searchRecords does
+  // not return values today). Entry MUST carry at least one of the two; if
+  // neither is present that is a critical cache_entry_shape_invalid.
+  //
+  // Pinecone IDs (e.g. 'mindrian-knowledge_t2-trending-absurd_2cddc6ac2cf58ea0')
+  // exceed the 64-char SLUG_REGEX limit on the prompts module. The slug check
+  // here is a permissive fallback: accept [a-zA-Z0-9_:-]{1,512} per Pinecone
+  // ID rules (https://docs.pinecone.io/), which is wider than SLUG_REGEX but
+  // still bounded and forbids whitespace + arbitrary unicode.
+  const PINECONE_ID_REGEX = /^[a-zA-Z0-9_:-]{1,512}$/;
+  let canonHitCount = 0;
+  for (let i = 0; i < payload.substrate.length; i += 1) {
+    const entry = payload.substrate[i];
+    if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
+      violations.push({
+        category: 'cache_entry_shape_invalid',
+        severity: 'critical',
+        message: 'substrate[' + i + '] is not an object',
+        action_hint: 'invalidate_cache',
+      });
+      continue;
+    }
+
+    // Check E.1: id slug. Accept SLUG_REGEX OR permissive Pinecone ID regex.
+    if (typeof entry.id !== 'string' ||
+        (!SLUG_REGEX.test(entry.id) && !PINECONE_ID_REGEX.test(entry.id))) {
+      violations.push({
+        category: 'cache_entry_shape_invalid',
+        severity: 'critical',
+        message: 'substrate[' + i + '].id fails id regex: ' +
+                 String(entry.id).slice(0, 80),
+        field: 'substrate[' + i + '].id',
+        action_hint: 'invalidate_cache',
+      });
+    }
+
+    // Check E.2 + E.3 + E.4: embedding-or-score required.
+    // hasEmbedding: legacy/mocked 1024-dim path
+    // hasScore: live brain_search Pinecone score path
+    // Neither -> critical (cache_entry_shape_invalid).
+    const hasEmbedding = Array.isArray(entry.embedding);
+    const hasScore = typeof entry.score === 'number';
+    if (!hasEmbedding && !hasScore) {
+      violations.push({
+        category: 'cache_entry_shape_invalid',
+        severity: 'critical',
+        message: 'substrate[' + i + '] missing both embedding[] and score',
+        field: 'substrate[' + i + ']',
+        action_hint: 'invalidate_cache',
+      });
+    }
+    if (hasEmbedding) {
+      if (entry.embedding.length !== EXPECTED_EMBEDDING_DIM) {
+        violations.push({
+          category: 'cache_entry_shape_invalid',
+          severity: 'critical',
+          message: 'substrate[' + i + '].embedding.length expected ' +
+                   EXPECTED_EMBEDDING_DIM + ' got ' + entry.embedding.length,
+          field: 'substrate[' + i + '].embedding',
+          action_hint: 'invalidate_cache',
+        });
+      } else {
+        // Embedding must be all finite numbers.
+        for (let j = 0; j < entry.embedding.length; j += 1) {
+          const val = entry.embedding[j];
+          if (typeof val !== 'number' || !Number.isFinite(val)) {
+            violations.push({
+              category: 'cache_entry_shape_invalid',
+              severity: 'critical',
+              message: 'substrate[' + i + '].embedding[' + j + '] non-finite',
+              field: 'substrate[' + i + '].embedding[' + j + ']',
+              action_hint: 'invalidate_cache',
+            });
+            break; // one report per entry is enough
+          }
+        }
+      }
+    }
+    if (hasScore) {
+      if (!Number.isFinite(entry.score) || entry.score < 0 || entry.score > 1) {
+        violations.push({
+          category: 'cache_entry_shape_invalid',
+          severity: 'critical',
+          message: 'substrate[' + i + '].score out of [0,1]: ' + String(entry.score),
+          field: 'substrate[' + i + '].score',
+          action_hint: 'invalidate_cache',
+        });
+      }
+    }
+
+    // Check D: Canon Part 8 metadata body scan (Pitfall 4 attack surface).
+    // JSON.stringify the metadata so nested strings are visible to
+    // FORBIDDEN_PATTERNS. Cap at MAX_CANON_HITS_REPORTED overall to
+    // avoid report spam from a fully-poisoned cache.
+    if (canonHitCount < MAX_CANON_HITS_REPORTED &&
+        entry.metadata && typeof entry.metadata === 'object') {
+      const s = JSON.stringify(entry.metadata);
+      for (const re of FORBIDDEN_PATTERNS) {
+        if (re.test(s)) {
+          violations.push({
+            category: 'canon_boundary',
+            severity: 'critical',
+            message: 'substrate[' + i + '].metadata matched forbidden regex ' +
+                     re.source,
+            field: 'substrate[' + i + '].metadata',
+            action_hint: 'invalidate_cache',
+          });
+          canonHitCount += 1;
+          break; // one match per entry is enough to fail the entry
+        }
+      }
+    }
+  }
+
+  return { severity: aggregateSeverity(violations), violations };
+}
+
+// ---------- Exports ----------
+
+module.exports = {
+  id: 'brain-substrate-invariants',
+  severity_map: SEVERITY_MAP,
+  scope: 'room',
+  validate,
+  // Exported for downstream introspection + Phase 89.1a Plan 03
+  // adversarial fixture suite. Not required by the registry loader.
+  EXPECTED_EMBEDDING_DIM,
+  CACHE_SCHEMA_VERSION,
+  MAX_CANON_HITS_REPORTED,
+};

package/lib/memory/validators/external-academic-invariants.cjs

@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2026 Mindrian. BSL 1.1.
+ * Phase 89.2 Plan 02 -- External academic fetcher invariants validator.
+ *
+ * Registry-compatible drop-in for the Phase 88-13 guardian. Auto-discovered
+ * by scripts/feynman-minto-guardian.cjs's validator loader (walks
+ * lib/memory/validators/*.cjs). Zero guardian.cjs edits needed.
+ *
+ * Mirrors brain-substrate-invariants.cjs structure byte-for-byte. The one
+ * difference is scope: external-papers.json telemetry lives at the GLOBAL
+ * per-user path ~/.mindrian/telemetry/external-papers.json (NOT per-room),
+ * so this validator declares scope='global'. The guardian still invokes it
+ * once per pass; roomPath is ignored.
+ *
+ * Six checks (Checks A-F):
+ *   A) Telemetry file absent -> {severity: null}    (healthy default)
+ *   B) Per-source budget exceeded in rolling 24h    -> warning per source
+ *   C) status enum membership                       -> warning per entry
+ *      Allowed statuses: ok | rate_limited | api_error | network_error |
+ *      timeout | api_key_missing
+ *   D) Canon Part 8 audit: query_text literal       -> CRITICAL canon_boundary
+ *      The telemetry primitive guarantees query_text NEVER persists
+ *      (only query_text_hash). If a literal query_text field ever
+ *      appears, that is a Canon Part 8 breach by definition.
+ *   E) query_text_hash format                        -> warning
+ *      Must match /^[a-f0-9]{16}$/ (sha256 16-char hex prefix).
+ *   F) fetched_at parseable as ISO-8601               -> warning
+ *
+ * Fail-open: unexpected throws are caught by the guardian's validator
+ * runner. This validator does not swallow its own exceptions; the
+ * guardian owns that contract. Individual I/O paths (readFileSync) are
+ * wrapped defensively so a missing telemetry file never produces a
+ * violation.
+ *
+ * Pure CJS, node built-ins only + the rs-egress-telemetry primitive
+ * shipped in Plan 89.2-01 for TELEMETRY_FILE + DEFAULT_BUDGETS + WINDOW_MS.
+ */
+'use strict';
+
+const fs = require('node:fs');
+
+const {
+  TELEMETRY_FILE,
+  DEFAULT_BUDGETS,
+  WINDOW_MS,
+} = require('../../core/rs-egress-telemetry.cjs');
+
+// ---------- Constants ----------
+
+const SEVERITY_MAP = Object.freeze({ critical: 3, error: 2, warning: 1, info: 0 });
+
+// Mirrors lib/core/rs-egress-telemetry.cjs ALLOWED_STATUSES set. Kept in
+// sync by parity gate (any drift fails an invariant scenario downstream).
+const ALLOWED_STATUSES = Object.freeze(new Set([
+  'ok',
+  'rate_limited',
+  'api_error',
+  'network_error',
+  'timeout',
+  'api_key_missing',
+]));
+
+// 16-char lowercase hex. sha256 prefix. Must match
+// rs-egress-telemetry.cjs::hashQueryText output shape.
+const CANON_PART_8_HASH_REGEX = /^[a-f0-9]{16}$/;
+
+// Cap canon_boundary hits per validate() invocation so one poisoned
+// ledger cannot spam the invariant report. Mirrors brain-substrate
+// invariants and Plan 90-05 brain-md-invariants threshold.
+const MAX_CANON_HITS_REPORTED = 5;
+
+// Cap warning-class violations to bound report size on a fully-corrupt
+// ledger. Critical hits are not capped (Check D never exceeds
+// MAX_CANON_HITS_REPORTED on its own).
+const MAX_WARNINGS_REPORTED = 25;
+
+// ---------- Severity aggregator ----------
+
+function aggregateSeverity(violations) {
+  if (!violations || violations.length === 0) return null;
+  let max = 0;
+  const inv = { 0: 'info', 1: 'warning', 2: 'error', 3: 'critical' };
+  for (const v of violations) {
+    const s = SEVERITY_MAP[v.severity];
+    if (typeof s === 'number' && s > max) max = s;
+  }
+  return inv[max];
+}
+
+// ---------- validate (Checks A-F) ----------
+//
+// scope='global'; roomPath is ignored (telemetry is per-user, not per-room).
+// ctx is unused but accepted to match the validator contract.
+
+function validate(_roomPath, _ctx) {
+  const violations = [];
+
+  // ---- Check A: telemetry file absent -> healthy default ----
+  if (!fs.existsSync(TELEMETRY_FILE)) {
+    return { severity: null, violations: [] };
+  }
+
+  // Read + parse defensively. A malformed ledger is a warning (not
+  // critical) because the telemetry primitive treats malformed payloads
+  // as cold-start; budgets just reset rather than crashing the fetcher.
+  let payload = null;
+  try {
+    const raw = fs.readFileSync(TELEMETRY_FILE, 'utf8');
+    payload = JSON.parse(raw);
+  } catch (err) {
+    violations.push({
+      validator: 'external-academic-invariants',
+      category: 'telemetry_unreadable',
+      severity: 'warning',
+      message: 'external-papers.json unreadable: ' + String(err && err.message),
+      action_hint: 'invalidate_telemetry',
+    });
+    return { severity: aggregateSeverity(violations), violations };
+  }
+
+  if (!payload || typeof payload !== 'object' || !Array.isArray(payload.entries)) {
+    violations.push({
+      validator: 'external-academic-invariants',
+      category: 'telemetry_malformed_root',
+      severity: 'warning',
+      message: 'external-papers.json root is not a plain object with entries[]',
+      action_hint: 'invalidate_telemetry',
+    });
+    return { severity: aggregateSeverity(violations), violations };
+  }
+
+  // ---- Check B: per-source budget exceeded in rolling 24h ----
+  // Aggregate counts per source within the rolling window, then compare
+  // against DEFAULT_BUDGETS. One warning per over-budget source so the
+  // operator sees which surface is the contention point.
+  const now = Date.now();
+  const sourceCounts = {};
+  for (const entry of payload.entries) {
+    if (!entry || typeof entry !== 'object') continue;
+    if (typeof entry.source !== 'string') continue;
+    if (typeof entry.fetched_at !== 'string') continue;
+    const t = Date.parse(entry.fetched_at);
+    if (Number.isNaN(t)) continue;
+    if ((now - t) > WINDOW_MS) continue;
+    sourceCounts[entry.source] = (sourceCounts[entry.source] || 0) + 1;
+  }
+  for (const source of Object.keys(sourceCounts)) {
+    const cap = DEFAULT_BUDGETS[source];
+    if (typeof cap === 'number' && sourceCounts[source] > cap) {
+      violations.push({
+        validator: 'external-academic-invariants',
+        category: 'budget_exceeded',
+        severity: 'warning',
+        message: 'source ' + source + ' has ' + sourceCounts[source] +
+                 ' calls in last 24h; budget cap ' + cap,
+        field: 'entries[].source=' + source,
+        action_hint: 'rotate_budget',
+      });
+    }
+  }
+
+  // ---- Checks C / D / E / F: per-entry scans ----
+  let canonHitCount = 0;
+  let warningCount = 0;
+  for (let i = 0; i < payload.entries.length; i += 1) {
+    const entry = payload.entries[i];
+    if (!entry || typeof entry !== 'object') continue;
+
+    // ---- Check D: canon_boundary literal query_text presence ----
+    // The telemetry primitive guarantees query_text NEVER persists. If
+    // any entry carries the field at all, that is a Canon Part 8 breach.
+    if (Object.prototype.hasOwnProperty.call(entry, 'query_text')
+        && canonHitCount < MAX_CANON_HITS_REPORTED) {
+      violations.push({
+        validator: 'external-academic-invariants',
+        category: 'canon_boundary',
+        severity: 'critical',
+        message: 'entries[' + i + '] carries literal query_text (Canon Part 8 breach)',
+        field: 'entries[' + i + '].query_text',
+        action_hint: 'purge_telemetry',
+      });
+      canonHitCount += 1;
+    }
+
+    // ---- Check C: status enum membership ----
+    if (typeof entry.status === 'string' && !ALLOWED_STATUSES.has(entry.status)
+        && warningCount < MAX_WARNINGS_REPORTED) {
+      violations.push({
+        validator: 'external-academic-invariants',
+        category: 'status_enum_violation',
+        severity: 'warning',
+        message: 'entries[' + i + '].status is not in allowed set: ' +
+                 String(entry.status).slice(0, 40),
+        field: 'entries[' + i + '].status',
+        action_hint: 'invalidate_telemetry',
+      });
+      warningCount += 1;
+    }
+
+    // ---- Check E: query_text_hash format ----
+    if (typeof entry.query_text_hash === 'string'
+        && !CANON_PART_8_HASH_REGEX.test(entry.query_text_hash)
+        && warningCount < MAX_WARNINGS_REPORTED) {
+      violations.push({
+        validator: 'external-academic-invariants',
+        category: 'hash_format_invalid',
+        severity: 'warning',
+        message: 'entries[' + i + '].query_text_hash fails /^[a-f0-9]{16}$/: ' +
+                 String(entry.query_text_hash).slice(0, 40),
+        field: 'entries[' + i + '].query_text_hash',
+        action_hint: 'invalidate_telemetry',
+      });
+      warningCount += 1;
+    }
+
+    // ---- Check F: fetched_at parseable as ISO-8601 ----
+    if (typeof entry.fetched_at === 'string'
+        && Number.isNaN(Date.parse(entry.fetched_at))
+        && warningCount < MAX_WARNINGS_REPORTED) {
+      violations.push({
+        validator: 'external-academic-invariants',
+        category: 'fetched_at_malformed',
+        severity: 'warning',
+        message: 'entries[' + i + '].fetched_at not parseable as ISO-8601: ' +
+                 String(entry.fetched_at).slice(0, 40),
+        field: 'entries[' + i + '].fetched_at',
+        action_hint: 'invalidate_telemetry',
+      });
+      warningCount += 1;
+    }
+  }
+
+  return { severity: aggregateSeverity(violations), violations };
+}
+
+// ---------- Exports ----------
+
+module.exports = {
+  id: 'external-academic-invariants',
+  severity_map: SEVERITY_MAP,
+  scope: 'global',
+  validate: validate,
+  // Exported for downstream introspection + adversarial fixture suite.
+  // Not required by the registry loader.
+  ALLOWED_STATUSES: ALLOWED_STATUSES,
+  CANON_PART_8_HASH_REGEX: CANON_PART_8_HASH_REGEX,
+  MAX_CANON_HITS_REPORTED: MAX_CANON_HITS_REPORTED,
+  MAX_WARNINGS_REPORTED: MAX_WARNINGS_REPORTED,
+};