open-memex 0.1.0 → 0.3.0-alpha

package/src/redact.ts

@@ -1,5 +1,129 @@
-export function stripPrivate(text: string): string {
-  return text.replace(/<private>[\s\S]*?<\/private>/gi, "[REDACTED]");
+/**
+ * Redaction: secrets must never land in memory files or the index in
+ * readable form.
+ *
+ * Layers, in order:
+ *   1. <private>...</private> regions are stripped first (explicit opt-out —
+ *      the author marked this span as sensitive, so it is replaced with
+ *      [REDACTED] before any detection runs).
+ *   2. Built-in provider patterns (always on, reported by id).
+ *   3. User patterns from config redactPatterns (reported verbatim).
+ *   4. High-entropy assignment heuristic (catches secrets whose provider we
+ *      don't have a pattern for, e.g. `deploy_key = "aB3d..."`).
+ *
+ * A detected secret does NOT refuse the write: the matched string is masked
+ * in place — first 4 characters kept, the rest replaced with 'x' — and the
+ * write proceeds with the masked content. hadSecret reports that masking
+ * happened so callers can surface a notice. A partially-masked memory still
+ * identifies which credential it referred to without storing the secret.
+ */
+
+export interface SecretPattern {
+  /** Stable id reported in matchedPattern. */
+  id: string;
+  /** Regex source (no slashes). */
+  source: string;
+  /** Optional RegExp flags, e.g. "i". */
+  flags?: string;
+  /**
+   * Capture-group index holding the secret value. When set, masking replaces
+   * only that group — the credential name stays readable (D14: a masked
+   * memory should still identify which key it referred to). Default: mask
+   * the whole match.
+   */
+  valueGroup?: number;
+}
+
+/**
+ * Prefixes like `sk-` also occur inside ordinary English words ("task-…",
+ * "risk-…", "disk-…"), which caused confirmed false positives. Require the
+ * token prefix NOT to be preceded by a word/hyphen char, so a real key after
+ * "=", ":", space, or a quote still matches.
+ */
+const TOKEN_BOUNDARY = "(?<![A-Za-z0-9_-])";
+
+export const BUILTIN_SECRET_PATTERNS: SecretPattern[] = [
+  { id: "openai-key", source: `${TOKEN_BOUNDARY}sk-[A-Za-z0-9_-]{20,}` },
+  {
+    id: "openai-admin-key",
+    source: `${TOKEN_BOUNDARY}sk-admin-[A-Za-z0-9_-]{20,}`,
+  },
+  {
+    id: "openai-session-key",
+    source: `${TOKEN_BOUNDARY}sm_[A-Za-z0-9_-]{20,}`,
+  },
+  { id: "github-pat", source: `${TOKEN_BOUNDARY}ghp_[A-Za-z0-9]{30,}` },
+  { id: "github-oauth-token", source: `${TOKEN_BOUNDARY}gho_[A-Za-z0-9]{30,}` },
+  { id: "github-user-token", source: `${TOKEN_BOUNDARY}ghu_[A-Za-z0-9]{30,}` },
+  {
+    id: "github-refresh-token",
+    source: `${TOKEN_BOUNDARY}ghr_[A-Za-z0-9]{30,}`,
+  },
+  {
+    id: "github-fine-grained-pat",
+    source: `${TOKEN_BOUNDARY}github_pat_[A-Za-z0-9_]{40,}`,
+  },
+  { id: "aws-access-key-id", source: `${TOKEN_BOUNDARY}AKIA[0-9A-Z]{16}` },
+  {
+    id: "aws-secret-access-key",
+    source:
+      "(aws[_-]?secret[_-]?access[_-]?key)([\"']?\\s*[:=]\\s*[\"']?)([A-Za-z0-9/+=]{40})",
+    flags: "i",
+    valueGroup: 3,
+  },
+  { id: "slack-token", source: `${TOKEN_BOUNDARY}xox[baprs]-[A-Za-z0-9-]{10,}` },
+  { id: "google-api-key", source: `${TOKEN_BOUNDARY}AIza[0-9A-Za-z_-]{30,}` },
+  { id: "npm-token", source: `${TOKEN_BOUNDARY}npm_[A-Za-z0-9]{30,}` },
+  { id: "gitlab-pat", source: `${TOKEN_BOUNDARY}glpat-[A-Za-z0-9_-]{20,}` },
+  {
+    id: "stripe-restricted-key",
+    source: `${TOKEN_BOUNDARY}rk_(live|test)_[A-Za-z0-9]{20,}`,
+  },
+  {
+    id: "stripe-webhook-secret",
+    source: `${TOKEN_BOUNDARY}whsec_[A-Za-z0-9]{20,}`,
+  },
+  {
+    id: "private-key-block",
+    // Whole block: masking only the BEGIN header would leave the base64 body
+    // readable in the memory file. Non-greedy so two blocks mask separately.
+    source:
+      "-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----",
+  },
+  {
+    id: "private-key-truncated",
+    // No END marker: mask from the header to end of text. A header without a
+    // body is still key material; over-masking is the safe direction.
+    source: "-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*$",
+  },
+  {
+    id: "jwt",
+    source: `${TOKEN_BOUNDARY}eyJ[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]{10,}`,
+  },
+  {
+    id: "generic-secret-assignment",
+    source:
+      "(api[_-]?key|secret|passwd|password|auth[_-]?token|access[_-]?token)([\"']?\\s*[:=]\\s*[\"']?)([A-Za-z0-9_\\-./+=]{16,})([\"']?)",
+    flags: "i",
+    valueGroup: 3,
+  },
+];
+
+function compile(p: SecretPattern): RegExp | null {
+  try {
+    return new RegExp(p.source, p.flags ?? "");
+  } catch {
+    return null; // ignore malformed builtin (should never happen)
+  }
+}
+
+/** Returns the matched builtin pattern id, or null. */
+export function findBuiltinSecret(text: string): string | null {
+  for (const p of BUILTIN_SECRET_PATTERNS) {
+    const re = compile(p);
+    if (re && re.test(text)) return p.id;
+  }
+  return null;
 }
 
 export function findSecret(text: string, patterns: string[]): string | null {
@@ -14,11 +138,138 @@ export function findSecret(text: string, patterns: string[]): string | null {
   return null;
 }
 
+function shannonEntropy(s: string): number {
+  const freq = new Map<string, number>();
+  for (const c of s) freq.set(c, (freq.get(c) ?? 0) + 1);
+  let h = 0;
+  for (const n of freq.values()) {
+    const p = n / s.length;
+    h -= p * Math.log2(p);
+  }
+  return h;
+}
+
+// name = value assignments with a long token-ish value.
+const ASSIGNMENT_RE =
+  /([A-Za-z_][A-Za-z0-9_]{1,63})\s*[:=]\s*["']?([A-Za-z0-9_\-./+=]{24,})["']?/g;
+
+/**
+ * Heuristic last line of defense: a long, high-entropy value assigned to a
+ * name is almost certainly a credential, even when no provider pattern
+ * matches. Tuned conservatively (length >= 24, entropy >= 4.5 bits/char):
+ * hex digests (<= 4.0) and prose (~4.0) pass through; base64-ish randomness
+ * does not. URLs are skipped.
+ */
+export function findHighEntropySecret(text: string): string | null {
+  for (const m of text.matchAll(ASSIGNMENT_RE)) {
+    const name = m[1];
+    const value = m[2];
+    if (!isSuspectAssignmentValue(value, m[0], m.index ?? 0, text)) continue;
+    return `high-entropy-secret:${name}`;
+  }
+  return null;
+}
+
+/**
+ * Shared benign-value rule for detection AND masking: a URL (e.g. the value
+ * after "https:") or a low-entropy value must never be touched. Note the
+ * "://" check: ASSIGNMENT_RE consumes the colon of "https:" as the separator,
+ * so the captured value starts with "//..." — check for "://" in the
+ * original text around the match, not just inside the value.
+ */
+function isSuspectAssignmentValue(value: string, fullMatch: string, offset: number, text: string): boolean {
+  // Bare URL: ASSIGNMENT_RE consumed the scheme colon ("https:") as the
+  // separator, so the match itself starts with "scheme://".
+  if (/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\//.test(fullMatch)) return false;
+  if (value.includes("://")) return false;
+  // Reconstruct what preceded the value inside the match (name + separator);
+  // if the text right before the value looks like scheme://, it is a URL.
+  const before = text.slice(Math.max(0, offset - 12), offset);
+  if (/^[a-zA-Z][a-zA-Z0-9+.-]*:(\/\/)?$/.test(before.trim()) || before.includes("://")) return false;
+  return shannonEntropy(value) >= 4.5;
+}
+
+export function stripPrivate(text: string): string {
+  // Closed pairs first...
+  let out = text.replace(/<private>[\s\S]*?<\/private>/gi, "[REDACTED]");
+  // ...then an unclosed <private> redacts everything after it. A dangling
+  // tag almost always means the author intended the rest to be private.
+  out = out.replace(/<private>[\s\S]*$/gi, "[REDACTED]");
+  return out;
+}
+
+/**
+ * Mask a matched secret: keep the first 4 characters, replace the rest
+ * with 'x' (length-preserving). "sk-1234567890abcdefghij" → "sk-1xxxxxxxxxxxxx".
+ */
+function maskMatch(m: string): string {
+  return m.slice(0, 4) + "x".repeat(Math.max(0, m.length - 4));
+}
+
+/** Apply the builtin patterns as masking (global, all matches). */
+function maskBuiltin(content: string): string {
+  let out = content;
+  for (const p of BUILTIN_SECRET_PATTERNS) {
+    const re = compile(p);
+    if (!re) continue;
+    const g = new RegExp(re.source, re.flags + "g");
+    if (p.valueGroup == null) {
+      out = out.replace(g, (m) => maskMatch(m));
+    } else {
+      // Mask only the secret-value group; the credential name stays readable.
+      out = out.replace(g, (...args: unknown[]) => {
+        const m = args[0] as string;
+        const val = args[p.valueGroup as number] as string | undefined;
+        if (!val) return m;
+        const idx = m.lastIndexOf(val);
+        if (idx < 0) return m;
+        return m.slice(0, idx) + maskMatch(val) + m.slice(idx + val.length);
+      });
+    }
+  }
+  return out;
+}
+
+/** Apply user patterns as masking. Invalid regexes are ignored. */
+function maskUser(content: string, patterns: string[]): string {
+  let out = content;
+  for (const p of patterns) {
+    try {
+      out = out.replace(new RegExp(p, "g"), (m) => maskMatch(m));
+    } catch {
+      // ignore malformed regex
+    }
+  }
+  return out;
+}
+
+/** Mask the value side of high-entropy assignments (name stays readable).
+ *  Uses the exact same benign-value rule as detection, so URLs and
+ *  low-entropy values are never touched even when another secret triggers. */
+function maskEntropy(content: string): string {
+  return content.replace(
+    ASSIGNMENT_RE,
+    (m, name: string, value: string, offset: number) => {
+      if (!isSuspectAssignmentValue(value, m, offset, content)) return m;
+      return m.split(value).join(maskMatch(value));
+    },
+  );
+}
+
 export function redact(
   text: string,
   patterns: string[],
 ): { content: string; hadSecret: boolean; matchedPattern: string | null } {
   const stripped = stripPrivate(text);
-  const matched = findSecret(stripped, patterns);
-  return { content: stripped, hadSecret: matched !== null, matchedPattern: matched };
+  // Detection (for reporting) runs in priority order: builtin → user → entropy.
+  const builtin = findBuiltinSecret(stripped);
+  const user = builtin ? null : findSecret(stripped, patterns);
+  const entropic =
+    builtin || user ? null : findHighEntropySecret(stripped);
+  const matched = builtin ?? user ?? entropic;
+  if (!matched) return { content: stripped, hadSecret: false, matchedPattern: null };
+  // Masking runs every family over the text (not just the reported one) so
+  // multiple credentials in one memory are all masked.
+  const masked = maskEntropy(maskUser(maskBuiltin(stripped), patterns));
+  return { content: masked, hadSecret: true, matchedPattern: matched };
 }

package/src/retrieve/cjk.ts

@@ -0,0 +1,63 @@
+// CJK retrieval helpers (pure logic, no sqlite).
+//
+// FTS5's `porter unicode61` tokenizer treats a run of CJK ideographs as one
+// token ("中文记忆" is a single token), so substring queries never match, and
+// the old query builder dropped CJK characters entirely. Per D8 the default
+// CJK strategy is bigram: at write time we pre-tokenize CJK runs into
+// space-separated unigrams + overlapping bigrams stored in the `cjk` FTS
+// column; at query time the CJK part of the query becomes an OR of bigrams
+// against that column. Unigrams are included so single-character queries
+// ("猫") still match. Works identically on bun:sqlite and better-sqlite3
+// with no native tokenizer dependency.
+
+const CJK_RE =
+  /[\u3400-\u4DBF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\u1100-\u11FF\u{20000}-\u{2A6DF}]/u;
+
+const CJK_RUN_RE =
+  /[\u3400-\u4DBF\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\u1100-\u11FF\u{20000}-\u{2A6DF}]+/gu;
+
+/** True if the string contains any CJK (Han/Hiragana/Katakana/Hangul) character. */
+export function hasCjk(s: string): boolean {
+  return CJK_RE.test(s);
+}
+
+/**
+ * Build the index text for the `cjk` FTS column: for every CJK run emit each
+ * character (unigram) then every overlapping bigram, space-separated.
+ * "中文记忆" -> "中 文 记 忆 中文 文记 记忆". Non-CJK text yields "".
+ */
+export function cjkIndexText(text: string): string {
+  const out: string[] = [];
+  for (const m of text.matchAll(CJK_RUN_RE)) {
+    const chars = [...m[0]];
+    for (const ch of chars) out.push(ch);
+    for (let i = 0; i + 1 < chars.length; i++) out.push(chars[i] + chars[i + 1]);
+  }
+  return out.join(" ");
+}
+
+/** Escape a raw token for embedding in a double-quoted FTS5 phrase. */
+function esc(t: string): string {
+  return t.replace(/"/g, '""');
+}
+
+/**
+ * Convert the CJK runs of a query into an FTS5 expression for the `cjk`
+ * column. Multi-char runs become an OR of bigrams (recall-oriented; bm25
+ * ranks docs matching more bigrams higher). Single chars stay unigrams.
+ * Returns "" when the query has no CJK.
+ */
+export function cjkQueryExpr(query: string): string {
+  const parts: string[] = [];
+  for (const m of query.matchAll(CJK_RUN_RE)) {
+    const chars = [...m[0]];
+    if (chars.length === 1) {
+      parts.push(`"${esc(chars[0])}"`);
+    } else {
+      for (let i = 0; i + 1 < chars.length; i++) {
+        parts.push(`"${esc(chars[i] + chars[i + 1])}"`);
+      }
+    }
+  }
+  return parts.join(" OR ");
+}

package/src/retrieve/inject.ts

@@ -1,6 +1,6 @@
 import { list } from "./search.ts";
 import type { Scope } from "../scope.ts";
-import { USER_SCOPE } from "../scope.ts";
+import { PERSONAL_SCOPE } from "../scope.ts";
 import type { MyOMemoryConfig } from "../config.ts";
 
 function oneLine(s: string, max = 240): string {
@@ -10,7 +10,7 @@ function oneLine(s: string, max = 240): string {
 
 export function buildContextBlock(scope: Scope, cfg: MyOMemoryConfig): string | null {
   const project = list(scope.key, { limit: cfg.maxProjectMemories });
-  const user = list(USER_SCOPE.key, { limit: cfg.maxProfileItems });
+  const user = list(PERSONAL_SCOPE.key, { limit: cfg.maxProfileItems });
 
   if (project.length === 0 && user.length === 0) return null;
 

package/src/retrieve/search.ts

@@ -1,4 +1,5 @@
 import { db } from "../store/db.ts";
+import { cjkQueryExpr, hasCjk } from "./cjk.ts";
 
 export interface SearchHit {
   id: string;
@@ -9,13 +10,105 @@ export interface SearchHit {
   snippet: string;
   score: number;
   updated_at: number;
+  status: string;
 }
 
-/** Convert free-text query into a safe FTS5 MATCH expression. */
+interface RawRow {
+  id: string;
+  scope_key: string;
+  project_name: string;
+  type: string;
+  tags: string;
+  updated_at: number;
+  snippet: string;
+  score: number;
+  status: string;
+  superseded_by: string | null;
+}
+
+function toHit(r: RawRow): SearchHit {
+  return {
+    id: r.id,
+    scope_key: r.scope_key,
+    project_name: r.project_name,
+    type: r.type,
+    tags: r.tags ? r.tags.split(",").filter(Boolean) : [],
+    snippet: r.snippet ?? "",
+    score: r.score,
+    updated_at: r.updated_at,
+    status: r.status,
+  };
+}
+
+/**
+ * Lifecycle-aware post-processing (§3.3):
+ * - retracted / archived are excluded from retrieval (kept for audit);
+ * - a superseded memory resolves to the newest of its chain (cycle-safe);
+ * - deprecated stays visible as a warning but ranks after active.
+ */
+function resolveVisible(rows: RawRow[], limit: number): SearchHit[] {
+  const byId = new Map(rows.map((r) => [r.id, r]));
+  const fullRow = (id: string): RawRow | undefined => {
+    const cached = byId.get(id);
+    if (cached) return cached;
+    const r = db()
+      .prepare(
+        `SELECT id, scope_key, project_name, type, tags, updated_at, status,
+                superseded_by, substr(content, 1, 240) AS snippet
+         FROM memories WHERE id = ?`,
+      )
+      .get(id) as
+      | (Omit<RawRow, "score" | "snippet"> & { snippet: string })
+      | undefined;
+    if (!r) return undefined;
+    const full: RawRow = { ...r, score: 0 };
+    byId.set(id, full);
+    return full;
+  };
+
+  const seen = new Set<string>();
+  const active: SearchHit[] = [];
+  const deprecated: SearchHit[] = [];
+
+  for (const r of rows) {
+    if (r.status === "retracted" || r.status === "archived") continue;
+    let target = r;
+    if (r.status === "superseded") {
+      let cur = r;
+      const chain = new Set([r.id]);
+      while (cur.status === "superseded" && cur.superseded_by) {
+        if (chain.has(cur.superseded_by)) break; // cycle guard
+        chain.add(cur.superseded_by);
+        const nxt = fullRow(cur.superseded_by);
+        if (!nxt) break;
+        cur = nxt;
+      }
+      if (cur.status === "retracted" || cur.status === "archived") continue;
+      target = cur;
+    }
+    if (seen.has(target.id)) continue;
+    seen.add(target.id);
+    const hit = toHit(target);
+    if (target.status === "deprecated") deprecated.push(hit);
+    else active.push(hit);
+  }
+  return [...active, ...deprecated].slice(0, limit);
+}
+
+/**
+ * Convert free-text query into a safe FTS5 MATCH expression.
+ * Latin tokens keep the old behavior (prefix match on content/tags/type).
+ * CJK runs become an OR of bigrams against the `cjk` column (see cjk.ts).
+ * Mixed queries OR the two parts together.
+ */
 function toFtsQuery(q: string): string {
-  const tokens = q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [];
-  if (tokens.length === 0) return "";
-  return tokens.map((t) => `"${t.replace(/"/g, '""')}"*`).join(" OR ");
+  const latin = (q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [])
+    .map((t) => `"${t.replace(/"/g, '""')}"*`)
+    .join(" OR ");
+  const cjk = hasCjk(q) ? cjkQueryExpr(q) : "";
+  if (latin && cjk) return `(${latin}) OR {cjk}:(${cjk})`;
+  if (cjk) return `{cjk}:(${cjk})`;
+  return latin;
 }
 
 export function search(
@@ -25,6 +118,9 @@ export function search(
   const q = toFtsQuery(query);
   if (!q) return [];
   const limit = Math.max(1, Math.min(opts.limit ?? 8, 50));
+  // Over-fetch: lifecycle filtering (chain resolution, exclusions) happens
+  // after the FTS query, so candidates must survive it.
+  const fetchLimit = Math.min(limit * 3 + 10, 150);
 
   const scopeFilter =
     opts.scopeKeys && opts.scopeKeys.length > 0
@@ -34,6 +130,7 @@ export function search(
 
   const sql = `
     SELECT m.id, m.scope_key, m.project_name, m.type, m.tags, m.updated_at,
+           m.status, m.superseded_by,
            snippet(memories_fts, 0, '[', ']', ' ... ', 12) AS snippet,
            bm25(memories_fts) AS score
     FROM memories_fts
@@ -45,7 +142,7 @@ export function search(
   const params: unknown[] = [q];
   if (opts.scopeKeys && opts.scopeKeys.length > 0) params.push(...opts.scopeKeys);
   if (opts.type) params.push(opts.type);
-  params.push(limit);
+  params.push(fetchLimit);
 
   const rows = db()
     .prepare(sql)
@@ -56,20 +153,15 @@ export function search(
     type: string;
     tags: string;
     updated_at: number;
+    status: string;
+    superseded_by: string | null;
     snippet: string;
     score: number;
   }>;
 
-  return rows.map((r) => ({
-    id: r.id,
-    scope_key: r.scope_key,
-    project_name: r.project_name,
-    type: r.type,
-    tags: r.tags ? r.tags.split(",").filter(Boolean) : [],
-    snippet: r.snippet ?? "",
-    score: -r.score, // FTS5 bm25: lower = better; invert for intuition
-    updated_at: r.updated_at,
-  }));
+  // FTS5 bm25: lower = better; invert for intuition.
+  const raw: RawRow[] = rows.map((r) => ({ ...r, score: -r.score }));
+  return resolveVisible(raw, limit);
 }
 
 export function list(
@@ -77,10 +169,11 @@ export function list(
   opts: { type?: string; limit?: number } = {},
 ): SearchHit[] {
   const limit = Math.max(1, Math.min(opts.limit ?? 20, 100));
+  const fetchLimit = Math.min(limit * 3 + 10, 150);
   const typeFilter = opts.type ? ` AND type = ?` : "";
   const sql = `
-    SELECT id, scope_key, project_name, type, tags, updated_at,
-           substr(content, 1, 240) AS snippet
+    SELECT id, scope_key, project_name, type, tags, updated_at, status,
+           superseded_by, substr(content, 1, 240) AS snippet
     FROM memories
     WHERE scope_key = ?${typeFilter}
     ORDER BY updated_at DESC
@@ -88,7 +181,7 @@ export function list(
   `;
   const params: unknown[] = [scopeKey];
   if (opts.type) params.push(opts.type);
-  params.push(limit);
+  params.push(fetchLimit);
 
   const rows = db()
     .prepare(sql)
@@ -99,17 +192,11 @@ export function list(
     type: string;
     tags: string;
     updated_at: number;
+    status: string;
+    superseded_by: string | null;
     snippet: string;
   }>;
 
-  return rows.map((r) => ({
-    id: r.id,
-    scope_key: r.scope_key,
-    project_name: r.project_name,
-    type: r.type,
-    tags: r.tags ? r.tags.split(",").filter(Boolean) : [],
-    snippet: r.snippet ?? "",
-    score: 1,
-    updated_at: r.updated_at,
-  }));
+  const raw: RawRow[] = rows.map((r) => ({ ...r, score: 1 }));
+  return resolveVisible(raw, limit);
 }

package/src/scope.ts

@@ -4,11 +4,16 @@ import path from "node:path";
 
 export interface Scope {
   key: string;
-  kind: "user" | "project";
+  kind: "personal" | "project" | "org";
   projectName: string;
 }
 
-export const USER_SCOPE: Scope = { key: "user", kind: "user", projectName: "user" };
+/** v2: v1 `user` scope is renamed to `personal` (§19). */
+export const PERSONAL_SCOPE: Scope = {
+  key: "personal",
+  kind: "personal",
+  projectName: "personal",
+};
 
 function normalizeRemote(url: string): string {
   return url