@dreamforgestudiollc/emet 0.0.0-stage → 0.2.6

package/src/redact.js

@@ -0,0 +1,237 @@
+/**
+ * EMET - Enduring Memory & Epistemic Tiers
+ * Copyright (c) 2026 Dreamforge Studio LLC
+ *
+ * Licensed under the MIT License - see LICENSE
+ *
+ * Secret redaction at the write path.
+ *
+ * Why this exists (2026-09-07): an OAuth passphrase was pasted into a chat and a
+ * screenshot during a connector re-add, and the session transcript that would
+ * carry it verbatim was minutes away from being written to the store. A rule
+ * that says "do not paste secrets" is an instruction; a scrubber the content
+ * passes through before it lands anywhere is a control. This module is the
+ * control. It sits inside the single write path (THE DOOR, procedural ref 280):
+ * memory saves, transcript saves, document writes, and log output all pass
+ * through it, so an instance cannot opt out by forgetting.
+ *
+ * What it masks, in order:
+ *   1. Literal values of secret-bearing environment variables (anything whose
+ *      name contains PASSPHRASE, PASSWORD, SECRET, TOKEN, API_KEY, PRIVATE_KEY,
+ *      PEPPER, or MONGODB_URI), plus the user:password pair inside any URI among them.
+ *      These are exact-string matches and cannot false-positive.
+ *   2. Credentials embedded in URIs: scheme://user:password@host.
+ *   3. Bearer tokens and JWTs.
+ *   4. key/value pairs whose key names a secret (passphrase, password, secret,
+ *      token, api_key, client_secret, access_token, refresh_token, authorization)
+ *      followed by : or = and a value. The key survives, the value does not.
+ *
+ * What it deliberately does NOT mask: hex digests (sha256 values are 64 hex
+ * chars and are part of the integrity record), ids, and ordinary prose. Rule 4
+ * can mask a word in prose that follows "token:" - that is accepted. Masking a
+ * word costs a word; missing a secret costs the secret. (An English prefix
+ * hyphenated onto a key word - "re-authorization:" - is not treated as a key;
+ * that one masked prose on 2026-09-07 and is excluded.)
+ *
+ * The mask is visible on purpose: '*****' says "something was here and was
+ * removed", which is the honest record. A silent drop would be a summary.
+ *
+ * The hit count reports CHANGES. Text that already carries the mask is matched
+ * and left as it is, uncounted - otherwise every read-modify-write of a
+ * document re-reports the masks it already held (observed 2026-09-07, meta ref 83).
+ */
+
+export const MASK = '*****';
+
+const SECRET_ENV_NAME = /(PASSPHRASE|PASSWORD|PASSWD|SECRET|TOKEN|API[_-]?KEY|PRIVATE[_-]?KEY|PEPPER|MONGODB_URI)/i;
+const MIN_LITERAL_LENGTH = 6;
+
+// Registered literal secrets. A Set so re-registering is a no-op.
+const literals = new Set();
+
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Registers one literal secret value. Anything shorter than MIN_LITERAL_LENGTH
+ * is ignored - masking "abc" everywhere would shred ordinary text.
+ * If the value is a URI with credentials, the user:password pair is registered
+ * as well, so a URI quoted with a different host or query still masks.
+ */
+export function registerSecret(value) {
+  if (typeof value !== 'string') return false;
+  const v = value.trim();
+  if (v.length < MIN_LITERAL_LENGTH) return false;
+  literals.add(v);
+  literalOrder = null;
+  const m = v.match(/^[a-z][a-z0-9+.-]*:\/\/([^/@\s]+)@/i);
+  if (m && m[1].includes(':')) {
+    const [, pass] = m[1].split(/:(.*)/s);
+    if (pass && pass.length >= MIN_LITERAL_LENGTH) literals.add(pass);
+  }
+  return true;
+}
+
+/** Registers every secret-bearing value in an env object (defaults to process.env). */
+export function registerSecretsFromEnv(env = process.env) {
+  let n = 0;
+  for (const [k, v] of Object.entries(env)) {
+    if (SECRET_ENV_NAME.test(k) && registerSecret(v)) n++;
+  }
+  return n;
+}
+
+/** Test seam. Not for production use. */
+export function _clearRegisteredSecrets() {
+  literals.clear();
+  literalOrder = null;
+}
+
+// Longest-first order, computed once per registry change rather than once per
+// write. The registry changes at startup and effectively never again; sorting
+// it on every scrub was work repeated for no reason.
+let literalOrder = null;
+function orderedLiterals() {
+  if (literalOrder === null) literalOrder = [...literals].sort((a, b) => b.length - a.length);
+  return literalOrder;
+}
+
+export function registeredSecretCount() {
+  return literals.size;
+}
+
+// A necessary-condition prefilter for the key/value rule. Every alternative in
+// KEY_NAMES contains one of these substrings, so text without any of them
+// cannot match - and skipping the full pattern then is free correctness.
+const KEY_HINT = /pass|secret|token|key|authorization|credential/i;
+
+// Pattern rules. Each replaces with a function so the hit count is exact.
+// `pre` is a cheap NECESSARY condition: a substring the full pattern cannot
+// match without. It never decides a match - it only decides whether the
+// expensive scan is worth starting.
+const KEY_NAMES = '(?:pass(?:phrase|word|wd)|secret|token|api[_-]?key|client_secret|access_token|refresh_token|authorization|credential)';
+const RULES = [
+  // scheme://user:password@host  ->  scheme://*****:*****@host
+  { name: 'uri-credentials', pre: (t) => t.includes('://'),
+    re: /([a-z][a-z0-9+.-]*:\/\/)([^/@\s:]+):([^/@\s]+)@/gi, sub: (_m, scheme) => `${scheme}${MASK}:${MASK}@` },
+  // Bearer <token>
+  { name: 'bearer', pre: (t) => t.includes('Bearer'),
+    re: /\b(Bearer)\s+[A-Za-z0-9._~+/=-]{8,}/g, sub: (_m, w) => `${w} ${MASK}` },
+  // JWT
+  { name: 'jwt', pre: (t) => t.includes('eyJ'),
+    re: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, sub: () => MASK },
+  // Delegated client sub-keys (EMET-0001). The full key, the session handle
+  // and the claim nonce. A bare key_id (32 hex, no prefix) is an index and
+  // stays visible. Lengths are exact so a shorter hex id is not masked.
+  { name: 'subkey', pre: (t) => t.includes('emk_') || t.includes('emh_') || t.includes('emn_'),
+    re: /(?<![A-Za-z0-9_])(?:emk_[0-9a-f]{32}_[A-Za-z0-9_-]{43}|emh_[A-Za-z0-9_-]{43}|emn_[A-Za-z0-9_-]{43})(?![A-Za-z0-9_-])/g,
+    sub: () => MASK },
+  // key: value / key = value / "key": "value"   (key survives, value is masked)
+  { name: 'key-value',
+    // Every alternative in KEY_NAMES contains one of these substrings, and the
+    // rule also requires a ':' or '='. Both are NECESSARY conditions, so
+    // failing this check cannot skip a match the full pattern would have found.
+    pre: (t) => KEY_HINT.test(t) && (t.includes(':') || t.includes('=')),
+    // The scheme words of an Authorization header are not the secret; the
+    // bearer rule above has already masked what followed them.
+    // G5 (2026-09-07): the prefix class lets a header name like x-api-key
+    // through, but it also let "re-authorization:" in prose count as a key and
+    // mask the word after it (thread §32). An English prefix joined by a hyphen
+    // is not a key name: re-, pre-, de-, un-, non- are excluded right before
+    // the key word.
+    // G6 (2026-09-07): the value stops at a backtick and does not swallow a
+    // trailing period or closing parenthesis - prose like `passphrase: x`. kept
+    // losing its closing backtick and full stop. A secret that itself ends in
+    // "." or ")" keeps that one character visible; accepted.
+    re: new RegExp(`(["']?\\b[\\w.-]*(?<!\\b(?:re|pre|de|un|non)-)${KEY_NAMES}["']?\\s*[:=]\\s*)(["']?)(?!(?:Bearer|Basic|Digest)\\b)(?=[^\\s"'\`,;}\\]]{4})([^\\s"'\`,;}\\]]*[^\\s"'\`,;}\\].)])([.)]*)(\\2)`, 'gi'),
+    sub: (_m, key, q, _v, trail, q2) => `${key}${q}${MASK}${trail}${q2}` }
+];
+
+/**
+ * Redacts a string. Returns { text, hits } where hits is the number of
+ * replacements made. Non-strings are returned unchanged with hits 0.
+ */
+export function redactWithReport(input) {
+  if (typeof input !== 'string' || input.length === 0) return { text: input, hits: 0 };
+  let text = input;
+  let hits = 0;
+  // Literals first: exact, longest first so a value that contains another is
+  // masked whole rather than leaving a fragment.
+  //
+  // These are literal strings, so they need no regex at all. The previous
+  // version compiled one RegExp per literal PER CALL and scanned the whole body
+  // with it whether or not the literal was present. indexOf answers "is it
+  // here" far more cheaply than a regex scan, and split/join is the same
+  // operation as a global replace on an escaped literal - identical output,
+  // identical hit count.
+  for (const lit of orderedLiterals()) {
+    if (!text.includes(lit)) continue;
+    const parts = text.split(lit);
+    hits += parts.length - 1;
+    text = parts.join(MASK);
+  }
+  // G4 (2026-09-07): a hit is a CHANGE. Text that already carries the mask
+  // ("passphrase: *****") matches the key/value rule and is replaced with
+  // itself; counting that made every read-modify-write of a document report
+  // the masks it already held as new redactions. The receipt's `redacted` is
+  // the number of spans this write changed - nothing else.
+  for (const rule of RULES) {
+    // The prefilter is a necessary condition for the pattern, so skipping on a
+    // miss cannot change the output. It exists because the scrubber was paying
+    // nearly full price on documents containing no secrets at all: measured
+    // 2026-09-08, a clean 300 KB body cost 6.40 ms against 7.25 ms for a
+    // secret-bearing one. The cost was scanning, not replacing.
+    if (rule.pre && !rule.pre(text)) continue;
+    text = text.replace(rule.re, (...args) => {
+      const out = rule.sub(...args);
+      if (out !== args[0]) hits++;
+      return out;
+    });
+  }
+  return { text, hits };
+}
+
+/** Convenience: the redacted string only. */
+export function redactText(input) {
+  return redactWithReport(input).text;
+}
+
+const SECRET_KEY = new RegExp(`^[\\w.-]*${KEY_NAMES}$`, 'i');
+
+/**
+ * Deep-redacts any value: strings are scrubbed; object keys that name a secret
+ * have their value masked outright regardless of content; arrays and plain
+ * objects are walked. Other types pass through. Returns { value, hits }.
+ */
+export function redactDeep(value, _seen = new WeakSet()) {
+  if (typeof value === 'string') {
+    const r = redactWithReport(value);
+    return { value: r.text, hits: r.hits };
+  }
+  if (Array.isArray(value)) {
+    let hits = 0;
+    const out = value.map(v => { const r = redactDeep(v, _seen); hits += r.hits; return r.value; });
+    return { value: out, hits };
+  }
+  if (value && typeof value === 'object') {
+    if (_seen.has(value)) return { value, hits: 0 };
+    _seen.add(value);
+    if (value instanceof Date || Buffer.isBuffer?.(value)) return { value, hits: 0 };
+    let hits = 0;
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (SECRET_KEY.test(k) && v !== null && v !== undefined && v !== '' && typeof v !== 'object') {
+        // G4: an already-masked value is left as it is and not counted.
+        if (v === MASK) { out[k] = v; } else { out[k] = MASK; hits++; }
+      } else {
+        const r = redactDeep(v, _seen); hits += r.hits; out[k] = r.value;
+      }
+    }
+    return { value: out, hits };
+  }
+  return { value, hits: 0 };
+}
+
+// Load-time registration: the process's own secrets are always known to it.
+registerSecretsFromEnv();

package/src/request-log.js

@@ -0,0 +1,124 @@
+/**
+ * EMET - Enduring Memory & Epistemic Tiers
+ * Copyright (c) 2026 Dreamforge Studio LLC
+ *
+ * Licensed under the MIT License - see LICENSE
+ *
+ * Request / auth logging for the HTTP transport (work list H4, 2026-09-07).
+ *
+ * Why: on 2026-09-07 a connector was added at the wrong URL, the browser
+ * authorization ran to completion, the client then POSTed to `/` and got a 404,
+ * and the user saw "an error". Railway's logs held three startup lines and
+ * nothing else. A server that cannot say what it answered cannot be diagnosed
+ * without reproducing the failure by hand - which is what happened.
+ *
+ * What is logged: one line per request on the OAuth and MCP paths, plus any
+ * request anywhere that ends 4xx/5xx (so a POST to a path that does not exist
+ * is visible). Fields: method, path, status, latency, client ip, and the
+ * identifying-but-not-secret parts of the exchange - client_id (a CIMD URL or
+ * an opaque registration id), grant_type, response_type, resource, the JSON-RPC
+ * method, tool name and the NAMES of the tool's parameters on /mcp (never their
+ * values - added 2026-09-27 so parameter use can be measured), and a reason on failures.
+ *
+ * What is never logged: request or response bodies, authorization codes,
+ * tokens, code verifiers, passphrases, or query strings as a whole. The line
+ * is still passed through the redaction scrubber before it is written -
+ * belt and braces; a control that depends on the author remembering is a rule.
+ */
+
+import { redactText } from './redact.js';
+
+// Paths always logged, whatever the status.
+export const LOGGED_PATH = /^\/(?:authorize|token|register|revoke|oauth\/approve|mcp|\.well-known\/[^\s]*)$/;
+
+const CLIP = 160;
+const clip = (v) => String(v).slice(0, CLIP);
+
+/** Whether a finished request should produce a log line. */
+export function shouldLog(path, statusCode) {
+  return LOGGED_PATH.test(path) || statusCode >= 400;
+}
+
+/**
+ * Builds the log entry for one finished request. Pure: takes the request
+ * fields it needs rather than the request object, so it is testable without
+ * express. `body` is whatever the parser left on req.body (may be undefined
+ * on a 401 that stopped before parsing), `query` is req.query.
+ */
+export function requestLogEntry({ method, path, statusCode, ms, ip, query = {}, body, wwwAuthenticate, reason }) {
+  const entry = { t: new Date().toISOString(), req: `${method} ${path}`, status: statusCode, ms: Math.round(ms) };
+  if (ip) entry.ip = ip;
+
+  const q = (query && typeof query === 'object') ? query : {};
+  const b = (body && typeof body === 'object' && !Array.isArray(body)) ? body : {};
+
+  const clientId = q.client_id ?? b.client_id;
+  if (clientId !== undefined) entry.client = clip(clientId);
+  if (b.grant_type) entry.grant = clip(b.grant_type);
+  if (q.response_type) entry.response_type = clip(q.response_type);
+  const resource = q.resource ?? b.resource;
+  if (resource !== undefined && resource !== null && resource !== '') entry.resource = clip(resource);
+  if (q.redirect_uri ?? b.redirect_uri) {
+    try { entry.redirect_host = new URL(q.redirect_uri ?? b.redirect_uri).host; } catch { entry.redirect_host = 'invalid'; }
+  }
+
+  if (path === '/mcp') {
+    if (Array.isArray(body)) entry.rpc = `batch(${body.length})`;
+    else if (b.method) {
+      entry.rpc = clip(b.method);
+      if (b.method === 'tools/call' && b.params && b.params.name) {
+        entry.tool = clip(b.params.name);
+        // Parameter NAMES only (Alex, 2026-09-27, code audit 1.6) - values never.
+        const a = b.params.arguments;
+        if (a && typeof a === 'object' && !Array.isArray(a)) entry.params = Object.keys(a).sort().map(clip);
+      }
+    }
+  }
+
+  if (reason) entry.reason = clip(reason);
+  else if (statusCode === 401 && wwwAuthenticate) {
+    const m = String(wwwAuthenticate).match(/error="([^"]+)"/);
+    if (m) entry.reason = m[1];
+  }
+
+  return entry;
+}
+
+/** The rendered line. Scrubbed even though nothing secret should be in it. */
+export function formatRequestLog(entry) {
+  return '[emet][req] ' + redactText(JSON.stringify(entry));
+}
+
+/**
+ * Express middleware. Installed before the auth router so it sees every
+ * request; logs on `finish`, when the status and any parsed body are known.
+ */
+export function requestLogger({ enabled = true, write = (line) => console.error(line) } = {}) {
+  return function emetRequestLog(req, res, next) {
+    if (!enabled) return next();
+    const t0 = process.hrtime.bigint();
+    // Capture the path NOW. The SDK's auth router mounts /authorize, /token,
+    // /register and the well-known documents as sub-routers, and express
+    // strips the mount prefix from req.path while a sub-router is handling
+    // the request - at `finish` it can read as '/' with status 200, which the
+    // filter would then skip. Found live on the first deploy (2026-09-07): the
+    // /.well-known line was the one missing from the log.
+    const path = String(req.originalUrl || req.url || '/').split('?')[0] || '/';
+    res.on('finish', () => {
+      try {
+        if (!shouldLog(path, res.statusCode)) return;
+        const ms = Number(process.hrtime.bigint() - t0) / 1e6;
+        const entry = requestLogEntry({
+          method: req.method, path, statusCode: res.statusCode, ms, ip: req.ip,
+          query: req.query, body: req.body,
+          wwwAuthenticate: res.getHeader && res.getHeader('www-authenticate'),
+          reason: res.locals && res.locals.reason
+        });
+        write(formatRequestLog(entry));
+      } catch (e) {
+        write('[emet][req] log failed: ' + (e && e.message));
+      }
+    });
+    next();
+  };
+}