mlx-bun 0.5.0 → 1.0.0

package/src/memory/redirect.ts

@@ -1,108 +0,0 @@
-// mlx-bun memory — SUB-CONCEPT REDIRECTS ("you already know what I'm talking about").
-//
-// A subject the user discussed should always RESOLVE, even when it never earned
-// its own article: it FOLDED into a broader article (routed/disambiguated to an
-// existing home) or it was CAPTURED (kept in its chunk, not yet articled). Either
-// way we record a redirect edge — a `(surface → home article stem)` row in the
-// `subject_redirects` ledger — so the surface resolves to its home instead of a
-// bare NOT FOUND.
-//
-// `subject_redirects` is LEDGER state (like buckets / chunk_entities): it is NOT
-// reconstructable from the vault, so reindex never deletes it. Instead reindex
-// PROJECTS each edge whose target article still exists into the derived
-// `entity_aliases` table, so `resolveName` finds the surface → its home article
-// with no extra lookup path. Registration is idempotent (INSERT OR IGNORE on the
-// (surface, target_stem) PK); a fold and a later capture of the same surface both
-// just point at the same home.
-
-import type { MemoryStore } from "./db";
-
-/** How a sub-concept reached its home article: it FOLDED into a broader existing
- *  article, or it was CAPTURED (no own article yet) and points at its most-related
- *  article. Informational only — both resolve identically. */
-export type RedirectKind = "fold" | "capture";
-
-export interface RedirectRow {
-  surface: string;
-  target_stem: string;
-  kind: RedirectKind;
-}
-
-/**
- * Register a `surface → target_stem` redirect (idempotent). Returns true when a
- * new edge was written. A blank surface, a self-pointing redirect (surface ==
- * stem title), or a blank target is ignored.
- */
-export function registerRedirect(
-  store: MemoryStore,
-  surface: string,
-  targetStem: string,
-  kind: RedirectKind,
-): boolean {
-  const s = surface.trim();
-  const t = targetStem.trim();
-  if (!s || !t) return false;
-  if (s === t || s === t.replace(/_/g, " ")) return false; // surface IS the title
-  const r = store.db
-    .query("INSERT OR IGNORE INTO subject_redirects (surface, target_stem, kind) VALUES (?, ?, ?)")
-    .run(s, t, kind);
-  return r.changes > 0;
-}
-
-/** Every redirect edge (sorted) — the reindex projection input. */
-export function listRedirects(store: MemoryStore): RedirectRow[] {
-  return store.db
-    .query("SELECT surface, target_stem, kind FROM subject_redirects ORDER BY surface, target_stem")
-    .all() as RedirectRow[];
-}
-
-/**
- * The article a CAPTURED subject most relates to: the article-homed entity that
- * co-occurs with it in the most chunks (deterministic — ties break by name). This
- * is the "surface it via its neighbourhood" signal a captured subject points at,
- * so it resolves to the article it was discussed alongside rather than NOT FOUND.
- * Returns the related entity's `article_stem`, or null when it co-occurs with no
- * homed article (the read path then falls back to memory_search over its chunk).
- * `chunkSet`, when given, restricts co-occurrence to a run's chunks.
- */
-export function relatedArticleStem(
-  store: MemoryStore,
-  entity: string,
-  chunkSet?: Set<string>,
-): string | null {
-  const rows = store.db
-    .query(
-      "SELECT e.article_stem AS stem, COUNT(*) AS c FROM chunk_entities ce1 " +
-        "JOIN chunk_entities ce2 ON ce2.chunk_id = ce1.chunk_id " +
-        "JOIN entities e ON e.name = ce2.entity_name " +
-        "WHERE ce1.entity_name = ? AND ce2.entity_name <> ? " +
-        "AND e.article_stem IS NOT NULL AND e.notable = 1 " +
-        "GROUP BY ce2.entity_name ORDER BY c DESC, ce2.entity_name ASC",
-    )
-    .all(entity, entity) as { stem: string; c: number }[];
-  if (!chunkSet) return rows[0]?.stem ?? null;
-  // Re-score within the run's chunk set when one is given.
-  const scored = new Map<string, number>();
-  const co = store.db
-    .query(
-      "SELECT ce1.chunk_id AS chunk_id, e.article_stem AS stem FROM chunk_entities ce1 " +
-        "JOIN chunk_entities ce2 ON ce2.chunk_id = ce1.chunk_id " +
-        "JOIN entities e ON e.name = ce2.entity_name " +
-        "WHERE ce1.entity_name = ? AND ce2.entity_name <> ? " +
-        "AND e.article_stem IS NOT NULL AND e.notable = 1",
-    )
-    .all(entity, entity) as { chunk_id: string; stem: string }[];
-  for (const r of co) {
-    if (!chunkSet.has(r.chunk_id)) continue;
-    scored.set(r.stem, (scored.get(r.stem) ?? 0) + 1);
-  }
-  let best: string | null = null;
-  let bestC = 0;
-  for (const [stem, c] of [...scored].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) {
-    if (c > bestC) {
-      best = stem;
-      bestC = c;
-    }
-  }
-  return best;
-}

package/src/memory/reindex.ts

@@ -1,325 +0,0 @@
-// mlx-bun memory — reindex the derived cache from the markdown vault.
-//
-// The vault is truth; the DB's derived tables are a cache. `reindex` rebuilds
-// them to MATCH the markdown — never the reverse. It drops and rebuilds exactly
-// six tables purely from the article files via the P0-T2 parser:
-//
-//   entities · entity_aliases · categories · article_categories ·
-//   infobox_facts · links
-//
-// Pipeline-state tables (messages, chunks, chunk_*, synthesized_*, watermarks,
-// buckets) are the synthesis ledger and are left UNTOUCHED. Foreign keys are
-// disabled across the rebuild so deleting+recreating `entities` does not cascade
-// into the pipeline's `chunk_entities` rows (which reference entities ON DELETE
-// CASCADE) — those are ledger state, not derivable from the vault.
-//
-// Determinism is a contract: reindexing the same vault into a fresh DB twice
-// yields byte-identical derived tables (articles are visited in sorted-stem
-// order; rows are inserted deterministically). See P2-T2.
-
-import { readFileSync, readdirSync } from "node:fs";
-import { join } from "node:path";
-
-import {
-  deriveCategories,
-  infoboxFieldValues,
-  parseInfobox,
-  parseSeriesBanner,
-  type InfoboxField,
-} from "./article";
-import type { MemoryStore } from "./db";
-import { listRedirects } from "./redirect";
-import { EntityResolver, nearNameMatch } from "./resolve";
-import { articlesDir, resolveWikilinkToStem, vaultRoot } from "./vault";
-
-// ---- normalization helpers -------------------------------------------
-
-/** Casefold + collapse whitespace/underscores so title, stem, and alias forms
- *  of the same surface text key the entity index identically. */
-function normalizeAlias(s: string): string {
-  return s.trim().toLowerCase().replace(/_/g, " ").replace(/\s+/g, " ").trim();
-}
-
-const WIKILINK_RE = /\[\[([^\]]+)\]\]/;
-const DATE_RE = /^\d{4}-\d{2}(-\d{2})?$/;
-const MAIN_BANNER_RES = [
-  /\{\{\s*Main\s*\|\s*([^}]+)\}\}/i, // {{Main|Target}}
-  /^\*?\s*Main article:\s*\[\[([^\]]+)\]\]\s*\.?\*?$/i, // *Main article: [[Target]]*
-];
-
-/** The inner target of a `[[wikilink]]` value, before any `|alias`/`#anchor`. */
-function wikilinkInner(value: string): string {
-  const m = WIKILINK_RE.exec(value);
-  if (!m) return "";
-  return m[1]!.split("|")[0]!.split("#")[0]!.trim();
-}
-
-/** Resolve a wikilink target to an existing article stem, else its underscored
- *  form (a not-yet-written "red link" still carries a stable graph key). */
-function linkStem(target: string, stems: Set<string>): string {
-  const base = target.split("|")[0]!.split("#")[0]!.trim();
-  return resolveWikilinkToStem(base, stems) ?? base.replace(/\s+/g, "_");
-}
-
-function elementKind(value: string): string {
-  if (WIKILINK_RE.test(value)) return "entity-link";
-  if (DATE_RE.test(value.trim())) return "date";
-  return "scalar";
-}
-
-// ---- result ----------------------------------------------------------
-
-export interface ReindexResult {
-  articles: number;
-  entities: number;
-  aliases: number;
-  categories: number;
-  articleCategories: number;
-  facts: number;
-  links: number;
-}
-
-// ---- the rebuild ------------------------------------------------------
-
-/** Sorted article stems under `<root>/articles/` (`*.md`, no hidden files). */
-function listArticleStems(root: string): string[] {
-  let names: string[];
-  try {
-    names = readdirSync(articlesDir(root));
-  } catch {
-    return [];
-  }
-  return names
-    .filter((n) => n.endsWith(".md") && !n.startsWith("."))
-    .map((n) => n.slice(0, -3))
-    .sort((a, b) => a.localeCompare(b));
-}
-
-/**
- * Rebuild the derived cache from the markdown vault. Idempotent and
- * deterministic: same vault → byte-identical derived tables.
- */
-export function reindex(store: MemoryStore, root: string = vaultRoot()): ReindexResult {
-  const db = store.db;
-  const stems = listArticleStems(root);
-  const stemSet = new Set(stems);
-
-  const result: ReindexResult = {
-    articles: stems.length,
-    entities: 0,
-    aliases: 0,
-    categories: 0,
-    articleCategories: 0,
-    facts: 0,
-    links: 0,
-  };
-
-  // FK off so dropping `entities` does not cascade into the pipeline ledger
-  // (`chunk_entities`). PRAGMA cannot change inside a transaction, so set it
-  // around the transactional rebuild.
-  // Sub-concept redirect edges (fold/capture) projected into entity_aliases below,
-  // so a discussed subject resolves to its home article. Read before the rebuild
-  // transaction; an older DB without the table yields none.
-  let redirects: { surface: string; target_stem: string }[] = [];
-  try {
-    redirects = listRedirects(store);
-  } catch {
-    redirects = [];
-  }
-
-  db.exec("PRAGMA foreign_keys = OFF");
-  try {
-    db.transaction(() => {
-      // Drop the six derived tables (children first for clarity).
-      db.exec("DELETE FROM links");
-      db.exec("DELETE FROM infobox_facts");
-      db.exec("DELETE FROM article_categories");
-      db.exec("DELETE FROM categories");
-      db.exec("DELETE FROM entity_aliases");
-      db.exec("DELETE FROM entities");
-
-      const insEntity = db.query(
-        "INSERT OR IGNORE INTO entities (name, article_stem, kind, notable) VALUES (?, ?, ?, 0)",
-      );
-      const insAlias = db.query("INSERT OR IGNORE INTO entity_aliases (alias, entity_name) VALUES (?, ?)");
-      const insCategory = db.query("INSERT OR IGNORE INTO categories (name) VALUES (?)");
-      const insArtCat = db.query(
-        "INSERT OR IGNORE INTO article_categories (article_stem, category, source) VALUES (?, ?, ?)",
-      );
-      const insFact = db.query(
-        "INSERT OR IGNORE INTO infobox_facts (article_stem, key, value, kind, entity_link) VALUES (?, ?, ?, ?, ?)",
-      );
-      const insLink = db.query("INSERT OR IGNORE INTO links (src_stem, dst_stem, via) VALUES (?, ?, ?)");
-
-      const addLink = (dst: string, via: string, src: string): void => {
-        if (!dst || dst === src) return;
-        const r = insLink.run(src, dst, via);
-        if (r.changes) result.links++;
-      };
-      const addAlias = (raw: string, entity: string): void => {
-        const a = normalizeAlias(raw);
-        if (!a) return;
-        const r = insAlias.run(a, entity);
-        if (r.changes) result.aliases++;
-      };
-
-      for (const stem of stems) {
-        let content: string;
-        try {
-          content = readFileSync(join(articlesDir(root), `${stem}.md`), "utf8");
-        } catch {
-          continue;
-        }
-        const box = parseInfobox(content);
-
-        // entities: one row per article; kind from the explicit infobox kind:.
-        const kind = box ? box.entityKind : "thing";
-        if (insEntity.run(stem, stem, kind).changes) result.entities++;
-
-        // entity_aliases: title (stem→spaces) + the stem itself + infobox aliases.
-        addAlias(stem, stem);
-        addAlias(stem.replace(/_/g, " "), stem);
-        if (box) {
-          const aliasField = box.fields.find((f) => f.key === "aliases");
-          if (aliasField) for (const a of infoboxFieldValues(aliasField)) addAlias(a, stem);
-        }
-
-        // categories + article_categories: from explicit [[Category:…]] declarations.
-        const derived = deriveCategories(content, stem);
-        for (const cat of derived.categories) {
-          if (insCategory.run(cat).changes) result.categories++;
-        }
-        for (const row of derived.categoryRows) {
-          if (insArtCat.run(row.article_stem, row.category, row.source).changes) result.articleCategories++;
-        }
-
-        // infobox_facts: one row per (split) field value, with element kind +
-        // resolved entity_link; infobox entity-values also feed the link graph.
-        if (box) {
-          for (const field of box.fields) emitFieldFacts(field, stem, stemSet, insFact, addLink, result);
-        }
-
-        // links: prose [[links]] (See-also tagged distinctly), series + main banners.
-        emitProseLinks(content, stem, stemSet, addLink);
-        const series = parseSeriesBanner(content);
-        if (series) addLink(linkStem(series, stemSet), "series", stem);
-      }
-
-      // Project the sub-concept redirect ledger into entity_aliases: a fold/capture
-      // surface becomes an alias of its home article (only when that article still
-      // exists), so resolveName reaches it. Articles are truth — a redirect never
-      // overrides a real alias (INSERT OR IGNORE keeps the first writer).
-      for (const r of redirects) {
-        if (stemSet.has(r.target_stem)) addAlias(r.surface, r.target_stem);
-      }
-    })();
-  } finally {
-    db.exec("PRAGMA foreign_keys = ON");
-  }
-
-  return result;
-}
-
-function emitFieldFacts(
-  field: InfoboxField,
-  stem: string,
-  stems: Set<string>,
-  insFact: ReturnType<MemoryStore["db"]["query"]>,
-  addLink: (dst: string, via: string, src: string) => void,
-  result: ReindexResult,
-): void {
-  for (const value of infoboxFieldValues(field)) {
-    const isLink = WIKILINK_RE.test(value);
-    const entityLink = isLink ? linkStem(wikilinkInner(value), stems) : null;
-    const r = insFact.run(stem, field.key, value, elementKind(value), entityLink);
-    if (r.changes) result.facts++;
-    if (isLink && entityLink) addLink(entityLink, "infobox", stem);
-  }
-}
-
-function emitProseLinks(
-  content: string,
-  stem: string,
-  stems: Set<string>,
-  addLink: (dst: string, via: string, src: string) => void,
-): void {
-  const lines = content.split(/\r?\n/);
-  const inner = /\[\[([^\]]+)\]\]/g;
-  let inFence = false;
-  let section: string | null = null;
-  for (const rawLine of lines) {
-    const line = rawLine;
-    if (/^```/.test(line)) {
-      inFence = !inFence;
-      continue;
-    }
-    if (inFence) continue;
-    const t = line.trim();
-
-    const hm = /^(#{1,6})\s+(.+)$/.exec(line);
-    if (hm) {
-      section = hm[1]!.length >= 2 ? hm[2]!.trim().toLowerCase() : section;
-      continue;
-    }
-    if (/^\*Part of a series on \[\[/.test(t)) continue; // series banner: tagged elsewhere
-    if (/^categories:/i.test(t)) continue; // category declaration: not a graph edge
-
-    // {{Main}} / *Main article: [[X]]* — a parent→child split edge.
-    let mainHit = false;
-    for (const re of MAIN_BANNER_RES) {
-      const m = re.exec(t);
-      if (m) {
-        addLink(linkStem(m[1]!.trim(), stems), "main", stem);
-        mainHit = true;
-        break;
-      }
-    }
-    if (mainHit) continue;
-
-    const via = section === "see also" ? "seealso" : "prose";
-    inner.lastIndex = 0;
-    let m: RegExpExecArray | null;
-    while ((m = inner.exec(line)) !== null) {
-      const base = m[1]!.split("|")[0]!.split("#")[0]!.trim();
-      if (!base || /^conv:/i.test(base) || /^Category:/i.test(base)) continue;
-      addLink(linkStem(base, stems), via, stem);
-    }
-  }
-}
-
-// ---- name resolution -------------------------------------------------
-
-const LEADING_ARTICLE_RE = /^(?:the|a|an)\s+/;
-
-/**
- * Resolve an entity name or alias to its article stem via `entity_aliases`.
- * Normalizes the query the same way aliases are stored (casefold, ws-collapse)
- * and also tries with a leading article word ("the"/"a"/"an") stripped, so
- * a "<the brand model>" surface and a "<model>" short form both reach the same stem.
- */
-export function resolveName(store: MemoryStore, nameOrAlias: string): string | null {
-  const q = normalizeAlias(nameOrAlias);
-  const candidates = [q, q.replace(LEADING_ARTICLE_RE, "")];
-  const stmt = store.db.query("SELECT entity_name FROM entity_aliases WHERE alias = ? LIMIT 1");
-  for (const c of candidates) {
-    if (!c) continue;
-    const row = stmt.get(c) as { entity_name: string } | null;
-    if (row) return row.entity_name;
-  }
-  // NEAR-NAME fallback: a discussed sub-concept that is plainly the same subject
-  // as an existing entity — the title minus a generic suffix ("<subject>" for a
-  // "<subject> Theory") — resolves via the conservative token-subset fuzzy. A
-  // distinct subject or a purely-generic query does not. No model, no embedding.
-  return nearNameMatch(EntityResolver.fromStore(store), q);
-}
-
-// ---- category lookup -------------------------------------------------
-
-/** Article stems that DECLARED `[[Category:<category>]]` — the only "about X"
- *  mechanism (zero substring false positives). */
-export function articlesInCategory(store: MemoryStore, category: string): string[] {
-  const rows = store.db
-    .query("SELECT article_stem FROM article_categories WHERE category = ? ORDER BY article_stem")
-    .all(category) as { article_stem: string }[];
-  return rows.map((r) => r.article_stem);
-}