mlx-bun 0.5.0 → 1.0.0

package/src/memory/resolve.ts

@@ -1,307 +0,0 @@
-// mlx-bun memory — surface-variant RESOLUTION (the dedup fix for P5-T3/T4).
-//
-// canonicalize() in entity.ts is a *pure string normal form*: casefold, collapse
-// whitespace, strip articles/possessives. That is necessary but NOT sufficient —
-// the self-check turned the spacing, brand-only, and possessive surfaces of ONE
-// product name into several different stems:
-//   - "<name>" vs "<na me>"  (internal spacing the normal form keeps)
-//   - "<brand>" / "<brand> <model>"  (brand-only and compositional surfaces that
-//     no purely-lexical normalizer can fold onto one canonical)
-//
-// The resolver adds three layers on top of canonicalize():
-//   1. SQUEEZE — strip ALL non-alphanumerics so "<na me>" / "<name>" / "<nam e>"
-//      collapse to one key deterministically (fixes the spacing split).
-//   2. ALIAS SEED — a known-entity index (entities + entity_aliases in the store,
-//      seeded from goldens/dreaming-entities-gold.json) so brand-only / nickname
-//      surfaces (a "<the brand>", a "<the role>") resolve by exact alias hit.
-//   3. TOKEN-SUBSET FUZZY — a conservative compositional match: a surface folds
-//      onto an entity iff its content tokens are a SUBSET of that entity's tokens
-//      AND the overlap carries at least one DISTINCTIVE token (a token unique to
-//      that entity among all known entities). This merges "<my brand model>" → the
-//      entity via the distinctive model token while REFUSING to merge
-//      "<brand A>" into "<brand B>" (a shared brand token is not distinctive;
-//      the model-number tokens differ) — the over-merge guard.
-//
-// Resolution is conservative by construction: when in doubt it MINTS a new
-// canonical rather than fusing two distinct things. ROUTE (route.ts) escalates
-// the genuine misses to the model for binary disambiguation.
-
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-
-import { canonicalize } from "./entity";
-import type { MemoryStore } from "./db";
-
-/** Tokens that carry no disambiguating signal on their own (articles, glue, and
- *  generic gear nouns shared across many entities). Used only for the fuzzy
- *  token-subset layer — never for the exact alias index. */
-const STOP = new Set([
-  "the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with", "is",
-  "as", "my", "your", "his", "her", "its", "our", "their",
-]);
-
-/** Strip ALL non-alphanumerics and casefold: the spacing/punct-insensitive key.
- *  A name's spacing and punctuation variants all collapse here. */
-export function squeeze(surface: string): string {
-  return surface.toLowerCase().replace(/[^a-z0-9]+/g, "");
-}
-
-/** Content tokens of a surface (casefold, split on non-alphanumerics, drop
- *  stopwords and single characters). Order-insensitive set. */
-export function contentTokens(surface: string): Set<string> {
-  const out = new Set<string>();
-  for (const t of surface.toLowerCase().split(/[^a-z0-9]+/)) {
-    if (t.length > 1 && !STOP.has(t)) out.add(t);
-  }
-  return out;
-}
-
-/** How a surface resolved (for instrumentation + the deterministic gates). */
-export type MatchKind = "stem" | "squeeze" | "alias" | "fuzzy" | "new";
-
-export interface ResolveResult {
-  /** The canonical entity name the surface resolved to (or was minted as). */
-  name: string;
-  /** Which layer produced the resolution. "new" ⇒ a canonical was minted. */
-  matched: MatchKind;
-  /** True iff this call minted a NEW canonical (no existing entity matched). */
-  created: boolean;
-}
-
-interface Seed {
-  canonical: string;
-  aliases: string[];
-}
-
-/**
- * In-memory variant resolver over a KNOWN-ENTITY set. Indexes each canonical by
- * its normal-form stem, its squeeze key, and its content tokens, then resolves a
- * surface through stem → squeeze → alias → token-subset-fuzzy, minting a new
- * canonical only when every layer misses.
- *
- * Construct from a seed list, from a {@link MemoryStore} (entities +
- * entity_aliases), or from the curated gold — and compose them (store first,
- * gold to backfill notable brand-only aliases).
- */
-export class EntityResolver {
-  /** normal-form stem → canonical. */
-  private byStem = new Map<string, string>();
-  /** squeeze key → canonical. */
-  private bySqueeze = new Map<string, string>();
-  /** canonical → union of content tokens across its name + aliases. */
-  private tokensOf = new Map<string, Set<string>>();
-  /** content token → set of canonicals carrying it (distinctiveness check). */
-  private tokenOwners = new Map<string, Set<string>>();
-
-  constructor(seeds: Seed[] = []) {
-    for (const s of seeds) this.register(s.canonical, s.aliases);
-  }
-
-  /** All known canonical names (registration order is not guaranteed). */
-  canonicals(): string[] {
-    return [...this.tokensOf.keys()];
-  }
-
-  /** Is `name` an already-known canonical entity? */
-  has(name: string): boolean {
-    return this.tokensOf.has(name);
-  }
-
-  /** Register a canonical and its aliases into every index. Idempotent; adding
-   *  aliases to an existing canonical merges them. The canonical's own surface is
-   *  always indexed as an alias of itself. */
-  register(canonical: string, aliases: string[] = []): void {
-    if (!this.tokensOf.has(canonical)) this.tokensOf.set(canonical, new Set());
-    const tokset = this.tokensOf.get(canonical)!;
-    for (const surface of [canonical, ...aliases]) {
-      const stem = canonicalize(surface);
-      const sq = squeeze(surface);
-      if (stem && !this.byStem.has(stem)) this.byStem.set(stem, canonical);
-      if (sq && !this.bySqueeze.has(sq)) this.bySqueeze.set(sq, canonical);
-      for (const t of contentTokens(surface)) {
-        tokset.add(t);
-        let owners = this.tokenOwners.get(t);
-        if (!owners) this.tokenOwners.set(t, (owners = new Set()));
-        owners.add(canonical);
-      }
-    }
-  }
-
-  /** A token is DISTINCTIVE iff exactly one known canonical carries it — the
-   *  signal the fuzzy layer requires to merge a compositional surface. */
-  private distinctive(token: string): string | null {
-    const owners = this.tokenOwners.get(token);
-    if (!owners || owners.size !== 1) return null;
-    return [...owners][0]!;
-  }
-
-  /**
-   * Conservative token-subset fuzzy match: the surface's content tokens must be a
-   * non-empty SUBSET of some entity's tokens, and at least one of them must be
-   * distinctive to that same entity. Returns the canonical, or null if no
-   * unambiguous owner (zero or conflicting distinctive owners ⇒ refuse to merge).
-   */
-  private fuzzy(surface: string): string | null {
-    const toks = contentTokens(surface);
-    if (toks.size === 0) return null;
-    // The distinctive tokens present in the surface, and who owns them.
-    const owners = new Set<string>();
-    for (const t of toks) {
-      const owner = this.distinctive(t);
-      if (owner) owners.add(owner);
-    }
-    if (owners.size !== 1) return null; // none, or a tie across entities → no merge
-    const cand = [...owners][0]!;
-    // Subset guard: every surface token must belong to the candidate, else the
-    // surface names something MORE than (a variant of) the candidate.
-    const ets = this.tokensOf.get(cand)!;
-    for (const t of toks) if (!ets.has(t)) return null;
-    return cand;
-  }
-
-  /** Resolve a surface to an EXISTING canonical, or null if every layer misses.
-   *  Pure: never mutates the index. */
-  match(surface: string): { name: string; matched: MatchKind } | null {
-    const stem = canonicalize(surface);
-    if (stem) {
-      const byStem = this.byStem.get(stem);
-      if (byStem) return { name: byStem, matched: "stem" };
-    }
-    const sq = squeeze(surface);
-    if (sq) {
-      const bySq = this.bySqueeze.get(sq);
-      if (bySq) return { name: bySq, matched: "squeeze" };
-    }
-    const fz = this.fuzzy(surface);
-    if (fz) return { name: fz, matched: "fuzzy" };
-    return null;
-  }
-
-  /**
-   * Resolve a surface, MINTING a new canonical when nothing matches. The minted
-   * canonical is the cleaned surface; it is registered so later variants of the
-   * same thing fold onto it. Brand-only nicknames still need a gold/store seed —
-   * fuzzy cannot invent a "<the brand>" ≡ "<brand model>" link from lexis alone.
-   */
-  resolve(surface: string, mintName?: string): ResolveResult {
-    const hit = this.match(surface);
-    if (hit) return { name: hit.name, matched: hit.matched, created: false };
-    const canonical = (mintName ?? surface).trim();
-    this.register(canonical, surface === canonical ? [] : [surface]);
-    return { name: canonical, matched: "new", created: true };
-  }
-
-  /** Register one canonical per stem with its alias surfaces, grouped from a
-   *  normalized `alias → stem` map (the read-index's `aliasToStem`). The stem is
-   *  the canonical; its surfaces become aliases. */
-  static fromAliasMap(aliasToStem: Iterable<readonly [string, string]>): EntityResolver {
-    const byStem = new Map<string, string[]>();
-    for (const [alias, stem] of aliasToStem) {
-      let list = byStem.get(stem);
-      if (!list) byStem.set(stem, (list = []));
-      list.push(alias);
-    }
-    const r = new EntityResolver();
-    for (const [stem, aliases] of byStem) r.register(stem, aliases);
-    return r;
-  }
-
-  /** Load the known-entity index from a store's entities + entity_aliases. */
-  static fromStore(store: MemoryStore): EntityResolver {
-    const r = new EntityResolver();
-    const ents = store.db.query("SELECT name FROM entities").all() as { name: string }[];
-    for (const e of ents) r.register(e.name, []);
-    const aliases = store.db
-      .query("SELECT alias, entity_name FROM entity_aliases")
-      .all() as { alias: string; entity_name: string }[];
-    for (const a of aliases) {
-      if (!r.has(a.entity_name)) r.register(a.entity_name, [a.alias]);
-      else r.register(a.entity_name, [a.alias]);
-    }
-    return r;
-  }
-}
-
-// ---- near-name resolution (sub-concept → its home article) -----------------
-
-/** Generic "kind-of-thing" suffix words a discussed sub-concept drops when it is
- *  the SAME subject as an existing article (a query that is the article title
- *  minus a generic tail — "<subject>" for "<subject> Theory"). They carry no
- *  disambiguating signal alone, so a query made ONLY of these never redirects. */
-const GENERIC_SUFFIX = new Set([
-  "theory", "framework", "system", "model", "method", "methodology", "approach",
-  "principle", "principles", "technique", "concept", "paradigm", "effect", "law",
-  "hypothesis", "problem", "process", "pattern", "practice",
-]);
-
-/**
- * NEAR-NAME match: resolve a surface to an EXISTING article when it is clearly the
- * SAME subject — the conservative token-subset fuzzy (a query whose content tokens
- * are a distinctive subset of an article's, e.g. the title minus a generic suffix).
- * Reuses {@link EntityResolver.match} (stem → squeeze → token-subset fuzzy) but
- * REFUSES a query made only of generic suffix words (just "Theory"/"Framework"),
- * which would over-merge onto an arbitrary article. Returns the canonical (== the
- * article stem the resolver was seeded with), or null. Pure.
- */
-export function nearNameMatch(resolver: EntityResolver, surface: string): string | null {
-  const toks = contentTokens(surface);
-  if (toks.size === 0) return null;
-  let hasContentToken = false;
-  for (const t of toks) {
-    if (!GENERIC_SUFFIX.has(t)) {
-      hasContentToken = true;
-      break;
-    }
-  }
-  if (!hasContentToken) return null; // a purely-generic query must not redirect
-  const hit = resolver.match(surface);
-  return hit ? hit.name : null;
-}
-
-// ---- gold seed -------------------------------------------------------------
-
-export interface VariantGroup {
-  canonical: string;
-  kind: string;
-  domain: string;
-  variants: string[];
-}
-export interface NotableEntity {
-  name: string;
-  kind: string;
-  domain: string;
-  aliases: string[];
-  whyNotable: string;
-}
-export interface DreamingGold {
-  variantGroups: VariantGroup[];
-  notableEntities: NotableEntity[];
-}
-
-const GOLD_PATH = join(import.meta.dir, "..", "..", "goldens", "dreaming-entities-gold.json");
-
-/** Read the curated entity gold (variantGroups + notableEntities). */
-export function loadDreamingGold(path: string = GOLD_PATH): DreamingGold {
-  return JSON.parse(readFileSync(path, "utf8")) as DreamingGold;
-}
-
-/** Seeds derived from the gold: every variant group AND every notable entity
- *  becomes a (canonical, aliases) seed. Notable entries that share a canonical
- *  with a variant group merge their aliases. */
-export function goldSeeds(gold: DreamingGold = loadDreamingGold()): Seed[] {
-  const byName = new Map<string, Set<string>>();
-  const add = (canonical: string, aliases: string[]): void => {
-    let set = byName.get(canonical);
-    if (!set) byName.set(canonical, (set = new Set()));
-    for (const a of aliases) set.add(a);
-  };
-  for (const g of gold.variantGroups) add(g.canonical, g.variants);
-  for (const n of gold.notableEntities) add(n.name, n.aliases);
-  return [...byName].map(([canonical, aliases]) => ({ canonical, aliases: [...aliases] }));
-}
-
-/** A resolver pre-seeded with the curated gold — the production seed for the
- *  notable brand-only aliases the model/lexis cannot derive on its own. */
-export function goldResolver(gold?: DreamingGold): EntityResolver {
-  return new EntityResolver(goldSeeds(gold));
-}

package/src/memory/rest.ts

@@ -1,245 +0,0 @@
-// Memory REST wrappers — thin loopback JSON routes over src/memory/vault.ts
-// for the web chat's Memory panel (docs/design/web-chat-redesign.md §5.5,
-// §9 Phase 2). These are HTTP-only conveniences for the frontend: the
-// agent-tool surface (src/memory/tools.ts) stays read-only and unchanged,
-// and every route here is itself read-only except /init, which delegates to
-// the same `setupVault` the CLI's `mlx-bun memory init` calls (never
-// duplicated logic).
-//
-// Every handler returns a Response and never throws — a missing vault or a
-// bad param degrades to `{ ok: false, error }` (never a 500), matching the
-// rest of server.ts's /api/* conventions. Route dispatch (matching
-// url.pathname + method) lives in server.ts; this file is the pure handler
-// bodies so they're unit-testable without booting a model context (see
-// tests/using/memory-rest.test.ts).
-
-import { realpath } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { dirname, resolve, sep } from "node:path";
-import {
-  vaultRoot, vaultStatus, listArticles, listReferenceDocs, readArticle,
-  searchArticles, getArticleLinks, resolveArticleRelPath, articleHistory, articleDiff,
-  setupVault, type VaultStatus,
-} from "./vault";
-import { parseInfobox, parseLead, parseSeriesBanner, articleStructure } from "./article";
-
-function jsonOk<T extends object>(body: T, init?: ResponseInit): Response {
-  return Response.json({ ok: true, ...body }, init);
-}
-
-function jsonErr(error: string, status = 400): Response {
-  return Response.json({ ok: false, error }, { status });
-}
-
-/** `{ ok:false, enabled:false }` shape shared by every route when no vault
- *  exists yet — never a throw, never a 404: the frontend uses this to
- *  render the first-run consent card instead of an error toast. */
-function noVault(root: string): Response {
-  return Response.json({ ok: false, enabled: false, error: "no memory vault yet", root });
-}
-
-// ---- GET /api/memory/status -------------------------------------------
-
-export async function handleMemoryStatus(): Promise<Response> {
-  const st = await vaultStatus();
-  if (!st.exists) return noVault(st.root);
-  return jsonOk({ enabled: true, status: st satisfies VaultStatus });
-}
-
-// ---- GET /api/memory/list ----------------------------------------------
-
-export async function handleMemoryList(): Promise<Response> {
-  const root = vaultRoot();
-  const st = await vaultStatus(root);
-  if (!st.exists) return noVault(root);
-  const [articles, reference] = await Promise.all([listArticles(root), listReferenceDocs(root)]);
-  return jsonOk({ articles, reference });
-}
-
-// ---- GET /api/memory/search?q=&scope= -----------------------------------
-
-export async function handleMemorySearch(url: URL): Promise<Response> {
-  const root = vaultRoot();
-  const st = await vaultStatus(root);
-  if (!st.exists) return noVault(root);
-  const q = (url.searchParams.get("q") ?? "").trim();
-  if (!q) return jsonErr("q is required");
-  const scopeParam = url.searchParams.get("scope");
-  const scope = scopeParam === "articles" || scopeParam === "reference" ? scopeParam : "all";
-  const limitParam = Number(url.searchParams.get("limit") ?? "50");
-  const limit = Number.isFinite(limitParam) && limitParam > 0 ? Math.min(limitParam, 200) : 50;
-  try {
-    const { summaries, hits } = await searchArticles(root, q, { limit, scope });
-    return jsonOk({ summaries, hits });
-  } catch (e) {
-    return jsonErr((e as Error).message);
-  }
-}
-
-// ---- GET /api/memory/article?name= --------------------------------------
-//
-// Rendered source + metadata: the article's raw Markdown plus the
-// deterministic structure the panel needs to render it without re-parsing
-// client-side (infobox, lead, series banner, section skeleton).
-
-export async function handleMemoryArticle(url: URL): Promise<Response> {
-  const root = vaultRoot();
-  const st = await vaultStatus(root);
-  if (!st.exists) return noVault(root);
-  const name = (url.searchParams.get("name") ?? "").trim();
-  if (!name) return jsonErr("name is required");
-  try {
-    const { path, content } = await readArticle(root, name);
-    return jsonOk({
-      name,
-      path,
-      content,
-      infobox: parseInfobox(content),
-      lead: parseLead(content),
-      series: parseSeriesBanner(content),
-      structure: articleStructure(content),
-    });
-  } catch (e) {
-    return jsonErr((e as Error).message, 404);
-  }
-}
-
-// ---- GET /api/memory/links?name= -----------------------------------------
-
-export async function handleMemoryLinks(url: URL): Promise<Response> {
-  const root = vaultRoot();
-  const st = await vaultStatus(root);
-  if (!st.exists) return noVault(root);
-  const name = (url.searchParams.get("name") ?? "").trim();
-  if (!name) return jsonErr("name is required");
-  try {
-    const links = await getArticleLinks(root, name);
-    return jsonOk({ name, ...links });
-  } catch (e) {
-    return jsonErr((e as Error).message, 404);
-  }
-}
-
-// ---- GET /api/memory/history?name= ---------------------------------------
-
-export async function handleMemoryHistory(url: URL): Promise<Response> {
-  const root = vaultRoot();
-  const st = await vaultStatus(root);
-  if (!st.exists) return noVault(root);
-  const name = (url.searchParams.get("name") ?? "").trim();
-  if (!name) return jsonErr("name is required");
-  const limitParam = Number(url.searchParams.get("limit") ?? "50");
-  const limit = Number.isFinite(limitParam) && limitParam > 0 ? Math.min(limitParam, 200) : 50;
-  try {
-    const relPath = await resolveArticleRelPath(root, name); // throws if not found — validates `name`
-    const entries = await articleHistory(root, relPath, limit);
-    return jsonOk({ name, isGitRepo: st.isGitRepo, entries });
-  } catch (e) {
-    return jsonErr((e as Error).message, 404);
-  }
-}
-
-// ---- GET /api/memory/diff?name=&rev= -------------------------------------
-
-/** Full 40-hex or an abbreviated (≥4 hex) commit hash — never a ref
- *  expression (`HEAD~1`, `main^`, `refs/...`), which could otherwise be
- *  abused to walk outside the intended commit or (in principle) smuggle
- *  option-like text into the argv. `rev` is passed to Bun.spawn as its own
- *  argv element (never through a shell), but this check is the defense
- *  against a caller trying to pass anything other than a bare commit id. */
-const REV_RE = /^[0-9a-f]{4,40}$/;
-
-export async function handleMemoryDiff(url: URL): Promise<Response> {
-  const root = vaultRoot();
-  const st = await vaultStatus(root);
-  if (!st.exists) return noVault(root);
-  const name = (url.searchParams.get("name") ?? "").trim();
-  const rev = (url.searchParams.get("rev") ?? "").trim();
-  if (!name) return jsonErr("name is required");
-  if (!rev) return jsonErr("rev is required");
-  if (!REV_RE.test(rev)) return jsonErr("rev must be a commit hash (4-40 hex chars)");
-  try {
-    const relPath = await resolveArticleRelPath(root, name); // throws if not found — validates `name`
-    const diff = await articleDiff(root, rev, relPath);
-    if (diff === null) return jsonErr("commit not found or no diff for this article at that revision", 404);
-    return jsonOk({ name, rev, diff });
-  } catch (e) {
-    return jsonErr((e as Error).message, 404);
-  }
-}
-
-// ---- POST /api/memory/init { path?: string } -----------------------------
-//
-// The consent-card backend: wraps the exact same `setupVault` the CLI's
-// `mlx-bun memory init` calls (src/cli.ts, `case "memory":` → `sub === "init"`)
-// — same idempotent create-dirs/README/Meta-pages/git-init behavior, minus
-// the CLI's interactive extras (seed-from-existing-vault prompt, nightly
-// schedule prompt), which stay CLI-only/TTY-gated and are out of scope here.
-//
-// Unlike the CLI (which only ever calls setupVault(vaultRoot()) — a
-// user-supplied path there is a read-only *seed source*, never the
-// setupVault target), this route takes `path` straight from an untrusted
-// loopback caller. setupVault() mkdir -p's subdirectories into it and, if
-// it isn't already a git repo, runs `git init && git add . && git commit`
-// there — pointed at an arbitrary directory that's a destructive foot-gun
-// (force-commits whatever's already there into a fresh history). There is
-// no product reason for the web panel to redirect the vault root outside
-// the machine's own tmp/vault trees, so `path` is confined to those before
-// it ever reaches setupVault; anything else 400s instead of writing files.
-
-/** realpath() of the deepest existing ancestor of `path`, with the
- *  non-existent tail (e.g. a not-yet-created vault dir) re-appended
- *  literally. Needed because macOS resolves `/var/folders/...` (os.tmpdir())
- *  through a `/private` symlink: a brand-new mkdtemp path under it won't
- *  literal-prefix-match the realpath'd tmpdir unless we walk up to a real
- *  segment the same way. */
-async function realpathNearest(path: string): Promise<string> {
-  let cur = resolve(path);
-  const tail: string[] = [];
-  while (true) {
-    try {
-      return [await realpath(cur), ...tail.reverse()].join(sep);
-    } catch {
-      const parent = dirname(cur);
-      if (parent === cur) return resolve(path); // hit the fs root without finding anything real
-      tail.push(cur.slice(parent.length + 1));
-      cur = parent;
-    }
-  }
-}
-
-/** Is `candidate` equal to or a descendant of `ancestor`, after resolving
- *  symlinks on both sides? */
-async function isUnder(candidate: string, ancestor: string): Promise<boolean> {
-  const realAncestor = await realpathNearest(ancestor);
-  const realCandidate = await realpathNearest(candidate);
-  return realCandidate === realAncestor || realCandidate.startsWith(realAncestor + sep);
-}
-
-/** Confine an init target to the machine's own vault/tmp trees — the only
- *  legitimate destinations for this route (default vault root, or a tmp dir
- *  for tests). Rejects anything else rather than handing setupVault() an
- *  arbitrary, possibly pre-existing directory to mkdir/git-init into. */
-async function isAllowedVaultTarget(path: string): Promise<boolean> {
-  return (await isUnder(path, vaultRoot())) || (await isUnder(path, tmpdir()));
-}
-
-export async function handleMemoryInit(request: Request): Promise<Response> {
-  let body: { path?: string } = {};
-  try {
-    body = (await request.json()) as { path?: string };
-  } catch {
-    // empty body is fine — defaults to vaultRoot()
-  }
-  const root = body.path && body.path.trim() ? body.path.trim() : vaultRoot();
-  if (!(await isAllowedVaultTarget(root))) {
-    return jsonErr("path must be under the memory vault root or a temp directory", 400);
-  }
-  try {
-    const res = await setupVault(root);
-    const st = await vaultStatus(res.root);
-    return jsonOk({ result: res, status: st });
-  } catch (e) {
-    return jsonErr((e as Error).message, 500);
-  }
-}