@aiwayds/dsh-topics-memory 0.18.2 → 0.20.0

package/src/jev/gate.ts

@@ -0,0 +1,151 @@
+/**
+ * Outbound secret gate (design 2026-09-25 §3.3) — self-contained port of the
+ * dsh-jev-mcp `server.mjs` mechanism. The COMPLETE serialized request body
+ * (model, state, questions — byte-identical to what goes on the wire) is
+ * scanned for credential patterns BEFORE anything leaves the process; a hit
+ * aborts the call with a structured descriptor that never echoes the secret.
+ *
+ * SECURITY INVARIANT: hit descriptors carry static metadata only (pattern
+ * name, offset, ≤10 printable chars before the hit) — never the matched
+ * text, and never text at or after the hit point.
+ *
+ * @module jev/gate
+ */
+
+import { readFileSync } from 'node:fs'
+
+export interface SecretGateHit {
+  pattern: string
+  offset: number
+  context: string
+}
+
+/** Minimal logger surface (both sinks optional — bare harnesses have none);
+ *  the gate only ever warns, and a broken logger must not break the gate. */
+export interface GateLogger {
+  warn?: (m: string) => void
+}
+
+function safeWarn(log: GateLogger | undefined, message: string): void {
+  try {
+    log?.warn?.(message)
+  } catch {
+    // contained — diagnostics never take the feature down
+  }
+}
+
+// Prefix-style patterns require a 20+ char continuation so that merely
+// DISCUSSING a prefix (e.g. "set OPENROUTER_API_KEY to an sk-or-v1- key")
+// never trips the gate — only an actual long token does.
+export const BUILTIN_PATTERNS: ReadonlyArray<{ name: string; re: RegExp }> = [
+  { name: 'jwt', re: /eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/ },
+  { name: 'openrouter-key', re: /sk-or-v1-[A-Za-z0-9_-]{20,}/ },
+  { name: 'anthropic-key', re: /sk-ant-[A-Za-z0-9_-]{20,}/ },
+  { name: 'openai-project-key', re: /sk-proj-[A-Za-z0-9_-]{20,}/ },
+  { name: 'openai-sk-key', re: /sk-[A-Za-z0-9]{20,}/ },
+  { name: 'github-token', re: /ghp_[A-Za-z0-9]{20,}/ },
+  { name: 'github-pat', re: /github_pat_[A-Za-z0-9_]{20,}/ },
+  { name: 'gitlab-token', re: /glpat-[A-Za-z0-9_-]{20,}/ },
+  { name: 'slack-token', re: /xox[baprs]-[A-Za-z0-9-]{20,}/ },
+  { name: 'aws-access-key-id', re: /AKIA[0-9A-Z]{16}/ },
+  { name: 'aws-temp-key-id', re: /ASIA[0-9A-Z]{16}/ },
+  { name: 'google-api-key', re: /AIza[A-Za-z0-9_-]{20,}/ },
+  { name: 'private-key-block', re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
+]
+
+const MAX_SECRET_LINE = 512 // longer lines WARN; matching ALWAYS uses the full line
+const MIN_EXTERNAL_SECRET = 8 // shorter entries would cause runaway false hits
+
+/** Literal secrets from the external list file (JEV_SECRET_FILE semantics:
+ *  one entry per line, ≥8 chars, `#` comments skipped, matched at FULL
+ *  length — a >512-char line warns but is never truncated, or everything
+ *  past char 512 would silently escape the gate). */
+export function loadExternalSecrets(file: string, log?: GateLogger): string[] {
+  if (file === '') return []
+  let raw: string
+  try {
+    raw = readFileSync(file, 'utf8')
+  } catch (err) {
+    safeWarn(log, `topics jev: cannot read jevSecretFile (${file}): ${err instanceof Error ? err.message : String(err)}; continuing without it.`)
+    return []
+  }
+  const secrets: string[] = []
+  raw.split(/\r?\n/).forEach((line, i) => {
+    const s = line.trim()
+    if (s === '' || s.startsWith('#')) return
+    if (s.length < MIN_EXTERNAL_SECRET) {
+      safeWarn(log, `topics jev: jevSecretFile line ${i + 1} shorter than ${MIN_EXTERNAL_SECRET} chars, skipped (false-positive guard).`)
+      return
+    }
+    if (s.length > MAX_SECRET_LINE) {
+      safeWarn(log, `topics jev: jevSecretFile line ${i + 1} is ${s.length} chars (> ${MAX_SECRET_LINE}); matching uses the FULL line (no truncation).`)
+    }
+    secrets.push(s)
+  })
+  return secrets
+}
+
+// One-time-per-path load, mirroring the reference implementation's
+// load-once-at-startup while staying compatible with the volatile
+// `jevSecretFile` config: an unchanged path is served from the cache (no
+// re-read), a swapped path reloads on the next call. `reloadSecrets` forces
+// a re-read — the `/topics set jev-secret-file` wiring calls it so an
+// in-place content change is picked up too.
+let cache: { file: string; secrets: string[] } = { file: '', secrets: [] }
+
+export function currentSecrets(file: string, log?: GateLogger): string[] {
+  if (cache.file !== file) reloadSecrets(file, log)
+  return cache.secrets
+}
+
+export function reloadSecrets(file: string, log?: GateLogger): string[] {
+  cache = { file, secrets: loadExternalSecrets(file, log) }
+  return cache.secrets
+}
+
+/**
+ * Redacted context: at most 10 printable chars BEFORE the hit plus an
+ * explicit [REDACTED:<pattern>] marker. Content at or after the hit point is
+ * NEVER shown — when a prefix regex under-matches, the secret tail sits
+ * right there. Belt and suspenders: if the matched text or any known
+ * external secret still appears in the context, the whole context collapses
+ * to '[REDACTED]'.
+ */
+export function describeHit(
+  pattern: string,
+  offset: number,
+  matched: string,
+  payload: string,
+  externalSecrets: readonly string[] = [],
+): SecretGateHit {
+  const before = payload.slice(Math.max(0, offset - 10), offset).replace(/[^\x20-\x7e]/g, '.') // keep it printable
+  const context = `${before}[REDACTED:${pattern}]`
+  if (context.includes(matched) || externalSecrets.some((s) => s !== '' && context.includes(s))) {
+    return { pattern, offset, context: '[REDACTED]' }
+  }
+  return { pattern, offset, context }
+}
+
+/**
+ * Scan the serialized outbound payload for secrets.
+ * @param payload JSON.stringify({model, state, questions}) — the exact wire body
+ * @param externalSecrets literal secrets from the external list (cached via currentSecrets)
+ * @param externalSource label for hit descriptors (the list file path)
+ */
+export function scanForSecrets(
+  payload: string,
+  externalSecrets: readonly string[] = [],
+  externalSource = '',
+): SecretGateHit | null {
+  for (const { name, re } of BUILTIN_PATTERNS) {
+    const m = re.exec(payload)
+    if (m) return describeHit(name, m.index, m[0], payload, externalSecrets)
+  }
+  for (const secret of externalSecrets) {
+    const idx = payload.indexOf(secret)
+    if (idx !== -1) {
+      return describeHit(externalSource !== '' ? `external(${externalSource})` : 'external', idx, secret, payload, externalSecrets)
+    }
+  }
+  return null
+}

package/src/jev/log.ts

@@ -0,0 +1,194 @@
+/**
+ * decisions.jsonl — System One usage/optimization telemetry (design
+ * 2026-09-25 §6), always on while jevEnabled is on. Lives beside the ilog at
+ * `<bundle>/meta/decisions.jsonl` under the same discipline: 512KB
+ * auto-compaction keeping the most recent quarter, line JSON, and write
+ * failures swallowed silently — fail-open extends to the stats itself.
+ *
+ * REDACTION INVARIANT (§6.1): metadata and probabilities only — slug/pair
+ * hash/question type/score/latency/token count. NEVER the state text or
+ * conclusion bodies. Writers copy an explicit field whitelist, so extra
+ * fields on caller objects cannot leak into the file.
+ *
+ * @module jev/log
+ */
+
+import { appendFile, mkdir, readFile, rename, writeFile } from 'node:fs/promises'
+import { createHash, randomUUID } from 'node:crypto'
+import { dirname, join } from 'node:path'
+import { metaDir, resolveBundleRoot } from '../paths.ts'
+import type { JevBand } from './thresholds.ts'
+
+export type JevLane = 'slowlane-rerank' | 'consolidate-prefilter' | 'fastgate-shadow'
+
+export type JevOutcome =
+  | 'ok'
+  | 'timeout'
+  | 'network'
+  | 'http_4xx'
+  | 'http_5xx'
+  | 'bad_json'
+  | 'secret_gate_blocked'
+  | 'missing_key'
+  /** Local pre-flight refusal (e.g. questions violate the decisions/openrouter
+   *  wire contract) — nothing was sent. Reserved outcome beyond §6.2's v1 list. */
+  | 'invalid_request'
+
+export interface JevUsage {
+  input_tokens: number
+  output_tokens: number
+}
+
+/** Call layer (§6.2): one row per HTTP call, failures included — the
+ *  fallback rate is the health metric, so failed calls must be counted. */
+export interface JevCallRecord {
+  at: string
+  lane: JevLane
+  backend: string
+  model: string
+  questionCount: number
+  stateChars: number
+  latencyMs: number
+  usage: JevUsage | null
+  outcome: JevOutcome
+  fallback: boolean
+}
+
+/** Verdict layer (§6.2): one row per question, batch requests expanded.
+ *  `agree` reconciles the fast-gate shadow against the lexical disposition
+ *  (wouldBlock = lexical let it through but jev landed in the veto band —
+ *  the other half of the calibration goldmine). */
+export interface JevVerdictRecord {
+  at: string
+  lane: JevLane
+  questionId: string
+  qtype: 'noul' | 'choice' | 'score'
+  /** Candidate reference without content: `slug:xxx` | `pair:hash3`. */
+  ref: string
+  /** §6.2 digest — stable key for the repeat-decision-rate analysis. */
+  digest: string
+  probability: number
+  band: JevBand
+  agree: 'hit' | 'nearFloor' | 'gate-blocked' | 'wouldBlock' | 'n/a'
+  /** Which backend produced this verdict: 'primary' (default — the configured
+   *  backend) or 'laya' (the local pace-maker). Design §3.4. */
+  backend?: string
+  /** True when this row's source DROVE the decision because the primary
+   *  backend was unavailable (degraded mode). Comparison rows from the
+   *  pace-maker are degraded=false; laya rows are excluded from ECE either
+   *  way (uncalibrated probabilities). */
+  degraded?: boolean
+}
+
+const COMPACT_LIMIT = 512 * 1024 // ~512KB cap, same as the ilog sidecars
+
+export function decisionsFile(root: string): string {
+  return join(metaDir(root), 'decisions.jsonl')
+}
+
+function liveFile(): string {
+  return decisionsFile(resolveBundleRoot())
+}
+
+/** digest(state, questionId, instructions): `h1:` + first 16 hex of
+ *  sha256(state + NUL + questionId + NUL + instructions) — the key the
+ *  repeat-decision-rate analysis joins on. */
+export function digestFor(state: string, questionId: string, instructions: string): string {
+  const h = createHash('sha256')
+  h.update(state)
+  h.update('\u0000')
+  h.update(questionId)
+  h.update('\u0000')
+  h.update(instructions)
+  return `h1:${h.digest('hex').slice(0, 16)}`
+}
+
+// Serialized like the store's write queue: single-record appends are
+// line-atomic anyway, but the compaction rewrite must not interleave with
+// an append.
+let queue: Promise<unknown> = Promise.resolve()
+
+function enqueue<T>(op: () => Promise<T>): Promise<T> {
+  const run = queue.then(op, op)
+  queue = run.catch(() => undefined)
+  return run
+}
+
+/** All write failures die HERE — never upstream. */
+async function appendJsonl(lines: readonly string[]): Promise<void> {
+  await enqueue(async () => {
+    try {
+      const file = liveFile()
+      await mkdir(dirname(file), { recursive: true })
+      await appendFile(file, lines.map((l) => `${l}\n`).join(''), 'utf8')
+      await compactIfNeeded(file)
+    } catch {
+      // fail-open: a broken stats sink must never break the feature (§6.1)
+    }
+  })
+}
+
+// store.ts compactInjectionsIfNeeded pattern: ~512KB cap; keep the most
+// recent quarter when exceeded (headroom for the next window).
+async function compactIfNeeded(file: string): Promise<void> {
+  let raw: string
+  try {
+    raw = await readFile(file, 'utf8')
+  } catch {
+    return
+  }
+  if (raw.length <= COMPACT_LIMIT) return
+  const lines = raw.split('\n').filter((l) => l.trim() !== '')
+  const keep = lines.slice(-Math.max(1, Math.floor(lines.length / 4)))
+  const tmp = `${file}.tmp-${randomUUID()}`
+  try {
+    await writeFile(tmp, keep.map((l) => `${l}\n`).join(''), 'utf8')
+    await rename(tmp, file)
+  } catch {
+    // swallow — the next append retries the compaction
+  }
+}
+
+/** Call-layer row. Field whitelist enforced here: anything extra on the
+ *  caller's object is dropped, so no state/conclusion text can sneak in. */
+export async function logCall(rec: JevCallRecord): Promise<void> {
+  await appendJsonl([
+    JSON.stringify({
+      at: rec.at,
+      lane: rec.lane,
+      backend: rec.backend,
+      model: rec.model,
+      questionCount: rec.questionCount,
+      stateChars: rec.stateChars,
+      latencyMs: rec.latencyMs,
+      usage:
+        rec.usage === null
+          ? null
+          : { input_tokens: rec.usage.input_tokens, output_tokens: rec.usage.output_tokens },
+      outcome: rec.outcome,
+      fallback: rec.fallback,
+    }),
+  ])
+}
+
+/** Verdict-layer rows (batch expanded by the caller). */
+export async function logVerdicts(recs: readonly JevVerdictRecord[]): Promise<void> {
+  if (recs.length === 0) return
+  await appendJsonl(
+    recs.map((rec) =>
+      JSON.stringify({
+        at: rec.at,
+        lane: rec.lane,
+        questionId: rec.questionId,
+        qtype: rec.qtype,
+        ref: rec.ref,
+        digest: rec.digest,
+        probability: rec.probability,
+        band: rec.band,
+        agree: rec.agree,
+        backend: rec.backend ?? 'primary',
+        degraded: rec.degraded === true,
+      }),
+    ),
+  )
+}

package/src/jev/thresholds.ts

@@ -0,0 +1,31 @@
+/**
+ * System One decision bands (design 2026-09-25 §5).
+ *
+ * The numbers are CALIBRATED — single-case A/B plus a 383-case threshold
+ * sweep (2026-09-25) — and bound to the batch wire protocol (protocol-drift
+ * evidence: the same case scores differently single vs batch). Changing the
+ * protocol requires a re-sweep; do not tune these in place.
+ *
+ * @module jev/thresholds
+ */
+
+/** Slow-lane rerank (候选问): adopt ≥ 0.60, strong-veto < 0.10. */
+export const RERANK_ADOPT = 0.6
+export const RERANK_FALLBACK = 0.1
+
+/** Consolidation prefilter (pair 问「同一问题吗」): adopt ≥ 0.50, veto < 0.15. */
+export const MERGE_PAIR_ADOPT = 0.5
+export const MERGE_PAIR_FALLBACK = 0.15
+
+export type JevBand = 'adopt' | 'record' | 'fallback'
+export type JevBandKind = 'rerank' | 'mergePair'
+
+/** Map a noul probability to its band: adopt → act on it, record → only log
+ *  it, fallback → hard veto back to the lexical/legacy behavior. */
+export function band(probability: number, kind: JevBandKind): JevBand {
+  const floor = kind === 'rerank' ? RERANK_FALLBACK : MERGE_PAIR_FALLBACK
+  const adopt = kind === 'rerank' ? RERANK_ADOPT : MERGE_PAIR_ADOPT
+  if (probability >= adopt) return 'adopt'
+  if (probability >= floor) return 'record'
+  return 'fallback'
+}