@100xprompt/brain 0.0.0-stage → 0.1.1

package/src/memory/redact.ts

@@ -0,0 +1,72 @@
+// Secret filtering for everything the brain stores in memory or shows through a tool. Deterministic, offline, idempotent:
+// a secret becomes `[redacted]`, anything else is left alone. Deliberately conservative about what counts as a secret
+// (git hashes, UUIDs, file paths and ordinary prose must survive), and aggressive about the shapes that are always secrets
+// (provider key prefixes, PEM blocks, JWTs, bearer headers, `NAME_KEY=value` assignments, credentials inside URLs).
+
+const MARK = "[redacted]"
+
+// Values that are references to a secret, not the secret: `$API_KEY`, `${TOKEN}`, `process.env.X`, `<your-key>`, `getToken()`.
+const REFERENCE_VALUE = /^(?:\$|\{|<|%|process\.|os\.|env\b|ENV\b|config\.|settings\.|secrets?\.|args\.|opts?\.|options\.|this\.|self\.|null\b|undefined\b|true\b|false\b|none\b|None\b|\[redacted\]|\*+$|x{3,}$|\.{3})/i
+
+const PROVIDER_TOKENS: RegExp[] = [
+  /sk-(?:ant|proj|live|or-v1|svcacct)?[-_]?[A-Za-z0-9_-]{20,}/g, // OpenAI, Anthropic, OpenRouter and look-alikes
+  /\bsk_(?:live|test)_[A-Za-z0-9]{16,}\b/g, // Stripe secret keys
+  /\b[rp]k_(?:live|test)_[A-Za-z0-9]{16,}\b/g,
+  /\bAIza[0-9A-Za-z_-]{30,}\b/g, // Google API keys
+  /\bgh[pousr]_[A-Za-z0-9]{30,}\b/g, // GitHub tokens
+  /\bgithub_pat_[A-Za-z0-9_]{40,}\b/g,
+  /\bglpat-[A-Za-z0-9_-]{16,}\b/g, // GitLab
+  /\bxox[abeprs]-[A-Za-z0-9-]{10,}\b/g, // Slack
+  /\b(?:AKIA|ASIA|AGPA|AIDA|AROA)[0-9A-Z]{16}\b/g, // AWS access key ids
+  /\bnpm_[A-Za-z0-9]{30,}\b/g,
+  /\bhf_[A-Za-z0-9]{30,}\b/g,
+  /\bya29\.[0-9A-Za-z_-]{20,}/g, // Google OAuth access tokens
+  /\bSG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,}\b/g, // SendGrid
+  /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, // JWT
+]
+
+const PEM = /-----BEGIN [A-Z0-9 ]*(?:PRIVATE KEY|CERTIFICATE|PGP PRIVATE KEY BLOCK)-----[\s\S]*?(?:-----END [A-Z0-9 ]*-----|$)/g
+
+const AUTH_HEADER = /\b(Authorization|Proxy-Authorization|X-Api-Key|X-Auth-Token|Api-Key)(["']?\s*[:=]\s*["']?)(?:(?:Bearer|Basic|Token|Digest)\s+)?[^\s"'`,;]{6,}/gi
+
+const URL_CREDENTIALS = /\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@:"']+:[^\s/@"']+@/gi
+
+// NAME = value, where NAME ends in key/token/secret/password-like words. A bare `key` is not enough (prose, map keys).
+const ASSIGNMENT =
+  /((?:^|[^\w.-])[\w.-]*(?:api[_-]?key|apikey|secret[_-]?key|access[_-]?key|private[_-]?key|auth[_-]?key|client[_-]?secret|secret|token|password|passwd|pwd|credentials?|passphrase))(["']?\s*[:=]\s*["']?)(\[redacted\]|"[^"\n]*"|'[^'\n]*'|[^\s"'`,;)}\]]+)/gi
+
+// A long unbroken mixed-case+digit run (base64/hex-ish secrets). Hex-only runs are left alone (git and content hashes).
+const LONG_RUN = /(?<![\w/.+=-])[A-Za-z0-9_+-]{40,}={0,2}(?![\w/.+=-])/g
+
+function isMixedSecret(s: string): boolean {
+  if (/^[0-9a-f]+$/i.test(s)) return false // git sha, sha256, md5: public identifiers
+  if (/^[a-z]+(?:[-_][a-z]+)+$/i.test(s)) return false // kebab/snake words
+  const upper = /[A-Z]/.test(s)
+  const lower = /[a-z]/.test(s)
+  const digit = /\d/.test(s)
+  const kinds = [upper, lower, digit].filter(Boolean).length
+  if (kinds < 3) return false
+  // identifiers like VeryLongCamelCaseFunctionName2 have few digit/letter switches; random strings switch a lot
+  let switches = 0
+  for (let i = 1; i < s.length; i++) if (/\d/.test(s[i]!) !== /\d/.test(s[i - 1]!)) switches++
+  return switches >= 6
+}
+
+export function redact(text: string): string {
+  if (!text) return text
+  let out = text
+  out = out.replace(PEM, MARK)
+  out = out.replace(AUTH_HEADER, (_m, name: string, sep: string) => `${name}${sep}${MARK}`)
+  out = out.replace(URL_CREDENTIALS, (_m, scheme: string) => `${scheme}${MARK}@`)
+  for (const re of PROVIDER_TOKENS) out = out.replace(re, MARK)
+  out = out.replace(ASSIGNMENT, (m, name: string, sep: string, value: string) => {
+    const bare = value.replace(/^["']|["']$/g, "")
+    if (bare.length < 4 || REFERENCE_VALUE.test(bare) || bare.includes("(")) return m
+    return `${name}${sep}${MARK}`
+  })
+  out = out.replace(LONG_RUN, (m) => (isMixedSecret(m) ? MARK : m))
+  return out
+}
+
+/** True when `redact` would change the text. */
+export const hasSecret = (text: string): boolean => redact(text) !== text

package/src/memory/relevance.ts

@@ -0,0 +1,187 @@
+// Which part of a long stored output to show when the model asks for it without saying where to look.
+//
+// Why not "the first N characters": an agent given a truncated first chunk almost never asks for the next one
+// ("Agents Don't Paginate", arXiv 2608.26130), and the thing that matters in a log (the failure) is at its end.
+// So the default window is the head, the tail and, when the caller says what it is after, the best-matching
+// chunk(s) of the middle, ranked by BM25 over line-aligned chunks. No model, deterministic.
+//
+// Every omission is marked with the exact character range left out (offsets into the SAME text the caller sees,
+// so `offset=` / `grep=` continue from it).
+
+/** Target size of one relevance chunk, in characters. */
+export const CHUNK_CHARS = 600
+
+export interface Span {
+  start: number
+  end: number
+}
+
+/**
+ * Splits text into consecutive spans of about `target` characters that end on a line boundary (a line break is kept
+ * at the end of its span). A line longer than `target` is cut into pieces of `target` characters.
+ * The spans cover the text exactly: contiguous, no gaps, no overlap.
+ */
+export function chunkSpans(text: string, target = CHUNK_CHARS): Span[] {
+  const spans: Span[] = []
+  let start = 0
+  let i = 0
+  const n = text.length
+  while (i < n) {
+    const nl = text.indexOf("\n", i)
+    const lineEnd = nl < 0 ? n : nl + 1
+    if (lineEnd - i > target) {
+      // A long line: close the open chunk first, then cut the line itself.
+      if (i > start) spans.push({ start, end: i })
+      let p = i
+      while (lineEnd - p > target) {
+        spans.push({ start: p, end: p + target })
+        p += target
+      }
+      start = p
+      i = lineEnd
+      if (i - start >= target) {
+        spans.push({ start, end: i })
+        start = i
+      }
+      continue
+    }
+    i = lineEnd
+    if (i - start >= target) {
+      spans.push({ start, end: i })
+      start = i
+    }
+  }
+  if (start < n) spans.push({ start, end: n })
+  return spans
+}
+
+const STOP = new Set(["the", "a", "an", "and", "or", "of", "to", "in", "on", "is", "it", "for", "at", "by", "as", "be", "this", "that", "with"])
+
+/** Lower-cased word tokens (letters, digits, underscore; any script), stop words and one-letter tokens dropped. */
+export function tokenize(s: string): string[] {
+  const out: string[] = []
+  for (const m of s.toLowerCase().matchAll(/[\p{L}\p{N}_]+/gu)) {
+    const t = m[0]
+    if (t.length > 1 && !STOP.has(t)) out.push(t)
+  }
+  return out
+}
+
+/**
+ * BM25 score of each chunk for the query (k1 1.2, b 0.75, the usual idf with +1 so it is never negative).
+ * A chunk with no query term scores 0.
+ */
+export function bm25(query: string, chunks: string[]): number[] {
+  const q = [...new Set(tokenize(query))]
+  const scores = new Array<number>(chunks.length).fill(0)
+  if (!q.length || !chunks.length) return scores
+  const docs = chunks.map((c) => {
+    const tf = new Map<string, number>()
+    let len = 0
+    for (const t of tokenize(c)) {
+      tf.set(t, (tf.get(t) ?? 0) + 1)
+      len++
+    }
+    return { tf, len }
+  })
+  const avg = docs.reduce((a, d) => a + d.len, 0) / docs.length || 1
+  const N = docs.length
+  for (const term of q) {
+    let df = 0
+    for (const d of docs) if (d.tf.has(term)) df++
+    if (!df) continue
+    const idf = Math.log(1 + (N - df + 0.5) / (df + 0.5))
+    docs.forEach((d, i) => {
+      const f = d.tf.get(term)
+      if (!f) return
+      scores[i]! += idf * ((f * 2.2) / (f + 1.2 * (0.25 + 0.75 * (d.len / avg))))
+    })
+  }
+  return scores
+}
+
+const count = (s: string, ch: string) => {
+  let n = 0
+  for (let i = s.indexOf(ch); i >= 0; i = s.indexOf(ch, i + 1)) n++
+  return n
+}
+
+/** The marker for a skipped range [a, b) of `text`. */
+export function omission(text: string, a: number, b: number): string {
+  const lines = count(text.slice(a, b), "\n")
+  return `[brain: omitted characters ${a}-${b} (${b - a} chars, ${lines} line break${lines === 1 ? "" : "s"}); get them with offset=${a} limit=${b - a}, or grep=<regex>]`
+}
+
+const MARKER_RESERVE = 190
+const MAX_MIDDLE = 3
+
+export interface WindowOptions {
+  /** What the caller is after: the best-matching chunks of the middle are added. */
+  query?: string
+}
+
+/**
+ * The text itself when it fits in `budget` characters; otherwise a window of about `budget` characters (markers
+ * included) made of the head, the tail and (with a query that matches something) up to three middle chunks, in
+ * file order, each gap replaced by an omission marker. With no query the room goes to head and tail equally;
+ * with one, head 35%, tail 35%, middle 30%.
+ */
+export function windowOf(text: string, budget: number, opts: WindowOptions = {}): string {
+  budget = Math.max(600, Math.floor(budget))
+  if (text.length <= budget) return text
+  const n = text.length
+  const content = budget - MARKER_RESERVE * (MAX_MIDDLE + 2)
+  const room = Math.max(200, content)
+  const hasQuery = !!opts.query && tokenize(opts.query).length > 0
+
+  const spans = hasQuery ? chunkSpans(text) : []
+  const scores = hasQuery ? bm25(opts.query!, spans.map((s) => text.slice(s.start, s.end))) : []
+  const anyHit = scores.some((s) => s > 0)
+  const headB = Math.floor(room * (anyHit ? 0.35 : 0.5))
+  const tailB = Math.floor(room * (anyHit ? 0.35 : 0.5))
+  const midB = room - headB - tailB
+
+  // Head ends on a line boundary when one is near; the tail starts on one.
+  let headEnd = headB
+  if (text[headEnd - 1] !== "\n") {
+    const k = text.lastIndexOf("\n", headEnd - 1)
+    if (k + 1 >= headB * 0.5) headEnd = k + 1
+  }
+  let tailStart = n - tailB
+  if (text[tailStart - 1] !== "\n") {
+    const k = text.indexOf("\n", tailStart)
+    if (k >= 0 && k + 1 - tailStart <= tailB * 0.5) tailStart = k + 1
+  }
+  if (headEnd >= tailStart) return text // cannot happen when text.length > budget; stay safe
+
+  const picked: Span[] = []
+  if (anyHit) {
+    const order = spans
+      .map((s, i) => ({ s, score: scores[i]! }))
+      .filter((x) => x.score > 0 && x.s.start >= headEnd && x.s.end <= tailStart)
+      .sort((a, b) => b.score - a.score || a.s.start - b.s.start)
+    let used = 0
+    for (const x of order) {
+      if (picked.length >= MAX_MIDDLE) break
+      const size = x.s.end - x.s.start
+      if (used + size > midB) continue
+      picked.push(x.s)
+      used += size
+    }
+    picked.sort((a, b) => a.start - b.start)
+  }
+
+  const parts: string[] = [text.slice(0, headEnd)]
+  let at = headEnd
+  const gap = (to: number) => {
+    if (to > at) parts.push(`${parts[parts.length - 1]!.endsWith("\n") ? "" : "\n"}${omission(text, at, to)}\n`)
+  }
+  for (const s of picked) {
+    gap(s.start)
+    parts.push(text.slice(s.start, s.end))
+    at = s.end
+  }
+  gap(tailStart)
+  parts.push(text.slice(tailStart))
+  return parts.join("")
+}