mlx-bun 0.0.7 → 0.0.9

package/src/memory/history.ts

@@ -0,0 +1,152 @@
+// mlx-bun memory — the contradiction classifier + the "## History" section.
+//
+// When a later-dated note contradicts a claim already in an article, the resolve
+// must decide WHICH KIND of contradiction it is before it edits:
+//
+//   (a) EVOLUTION — the USER changed their own position / opinion / preference /
+//       relationship over time (both statements were true when made; the view
+//       simply moved). The LEAD/BODY resolve to the CURRENT state, and the
+//       trajectory is PRESERVED in a dedicated "## History" section (we KEEP the
+//       history) — both the earlier and the later source cited.
+//
+//   (b) ERROR — the earlier value was simply a WRONG world-fact / spec, or a flat
+//       "no, that is wrong, it is actually <value>" correction. The wrong value is
+//       SILENTLY overwritten with the correct one: NO History entry, the wrong
+//       value pruned entirely as if it had never been there.
+//
+// The classifier is a bounded yes/no the small local model can answer reliably:
+// "is this the user changing their own view over time, or correcting a wrong
+// fact?". Everything in this module is schematic (the prompt carries ONLY
+// instructions + the two real values under comparison, never a copyable example).
+
+import { LEAD_ANCHOR } from "./cluster";
+
+export const HISTORY_TITLE = "History";
+export const HISTORY_ANCHOR = "history";
+const HISTORY_HEADING_RE = /^##\s+History\s*$/i;
+
+/** The two outcomes of a contradiction classification. */
+export type ContradictionKind = "evolution" | "error";
+
+/**
+ * The bounded EVOLUTION-vs-ERROR classifier prompt. It carries ONLY the two real
+ * values under comparison (the article data being reconciled) plus the decision
+ * rule — no copyable illustrative example. Answered with a single word.
+ */
+export function buildContradictionClassifierPrompt(earlier: string, later: string): string {
+  return (
+    `Two statements by the SAME user conflict. Decide which KIND of conflict this is.\n\n` +
+    `EARLIER value: ${earlier}\n` +
+    `LATER value: ${later}\n\n` +
+    `Answer EVOLUTION if the user CHANGED THEIR OWN position, opinion, preference, stance, or relationship over time — ` +
+    `both were genuine at the time they were said, and the view simply moved.\n` +
+    `Answer ERROR if the earlier value was simply a WRONG fact now being corrected — ` +
+    `a mistaken world-fact or spec, or a flat "no, that is wrong, it is actually <value>" fix where the earlier value was never a real position.\n\n` +
+    `Answer with EXACTLY one word: EVOLUTION or ERROR.`
+  );
+}
+
+/**
+ * Parse a classifier reply to a {@link ContradictionKind}. Defaults to
+ * `"evolution"` — the NON-DESTRUCTIVE branch (keep the history) — unless the reply
+ * unambiguously says ERROR, so an ambiguous answer never silently destroys a
+ * genuine trajectory. Pure.
+ */
+export function classifyContradiction(raw: string): ContradictionKind {
+  const t = (raw ?? "").toLowerCase();
+  if (/\berror\b/.test(t) && !/\bevolution\b/.test(t)) return "error";
+  return "evolution";
+}
+
+/** True when the article already has a `## History` section. Pure. */
+export function hasHistorySection(article: string): boolean {
+  return article.split("\n").some((l) => HISTORY_HEADING_RE.test(l.trim()));
+}
+
+/** A `[^k]` suffix for a marker number, or `""` when none is known. */
+function markerSuffix(n: number | null | undefined): string {
+  return n != null ? `[^${n}]` : "";
+}
+
+/**
+ * Build ONE History trajectory line preserving BOTH sources' citation markers:
+ * the earlier position (past) carries `[^earlierMarker]`, the later/current one
+ * carries `[^laterMarker]`, so no provenance is lost. Both values are real article
+ * data, not copyable example values. Pure.
+ */
+export function buildHistoryEntry(opts: {
+  earlier: string;
+  earlierMarker?: number | null;
+  later: string;
+  laterMarker?: number | null;
+  laterDate?: string | null;
+}): string {
+  const when =
+    opts.laterDate && opts.laterDate !== "undated" ? `as of ${opts.laterDate}` : "now";
+  return (
+    `- Previously ${opts.earlier.trim()}${markerSuffix(opts.earlierMarker)}; ` +
+    `${when} ${opts.later.trim()}${markerSuffix(opts.laterMarker)}.`
+  );
+}
+
+/**
+ * Insert `entry` into the article's `## History` section, creating the section
+ * (just before `## See also` / `## References`, never after the references) when
+ * it is absent. Idempotent for an identical entry already present — never
+ * duplicates a line. Pure.
+ */
+export function upsertHistoryEntry(article: string, entry: string): string {
+  const trimmedEntry = entry.trim();
+  const lines = article.replace(/\s+$/, "").split("\n");
+
+  const headingIdx = lines.findIndex((l) => HISTORY_HEADING_RE.test(l.trim()));
+  if (headingIdx >= 0) {
+    // Find the end of the existing History block (next heading, or EOF).
+    let end = lines.length;
+    for (let i = headingIdx + 1; i < lines.length; i++) {
+      if (/^#{1,6}\s+/.test(lines[i]!.trim())) {
+        end = i;
+        break;
+      }
+    }
+    const block = lines.slice(headingIdx + 1, end);
+    if (block.some((l) => l.trim() === trimmedEntry)) {
+      return article; // already recorded — idempotent
+    }
+    // Append after the last non-blank line of the block.
+    let last = end - 1;
+    while (last > headingIdx && lines[last]!.trim() === "") last--;
+    const next = [...lines.slice(0, last + 1), trimmedEntry, "", ...lines.slice(end)];
+    return next.join("\n") + "\n";
+  }
+
+  // No History section yet — insert one before See also / References.
+  const tailIdx = lines.findIndex((l) => /^##\s+(see also|references)\s*$/i.test(l.trim()));
+  const section = [`## ${HISTORY_TITLE}`, "", trimmedEntry, ""];
+  if (tailIdx < 0) return [...lines, "", ...section].join("\n") + "\n";
+  return [...lines.slice(0, tailIdx), ...section, ...lines.slice(tailIdx)].join("\n") + "\n";
+}
+
+/**
+ * The `[^N]` marker number that CITES `value`, so a resolve can preserve the right
+ * source when it moves a value into History. By the citation-after-claim
+ * convention the marker follows the claim, so we take the FIRST marker at/after the
+ * value's first occurrence; failing that, the nearest marker before it. Returns
+ * null when `value` carries no nearby marker. Pure.
+ */
+export function markerNear(text: string, value: string): number | null {
+  const v = value.trim().toLowerCase();
+  if (!v) return null;
+  const idx = text.toLowerCase().indexOf(v);
+  if (idx < 0) return null;
+  // First marker AT/AFTER the value (the citation follows the claim).
+  const after = text.slice(idx).match(/\[\^(\d+)\]/);
+  if (after) return parseInt(after[1]!, 10);
+  // Fallback: the LAST marker before the value.
+  const before = [...text.slice(0, idx).matchAll(/\[\^(\d+)\]/g)];
+  const last = before[before.length - 1];
+  return last ? parseInt(last[1]!, 10) : null;
+}
+
+/** Re-export so callers can name the lead target without reaching into cluster. */
+export { LEAD_ANCHOR };

package/src/memory/ingest.ts

@@ -1,22 +1,335 @@
-// mlx-bun memory — synthesis stage 1: INGEST.  ⚠️ M1 STUB.
+// mlx-bun memory — synthesis stage 1: INGEST. (P5-T1)
 //
-// pi sessions (~/.mlx-bun/sessions + ~/.mlx-bun/pi, pi's own JSONL) →
-// memory.sqlite, normalized to {conv, source, title, transcript}, watermark-
-// gated so each run only processes what's new. No web scraping, no Playwright:
-// mlx-bun synthesizes ITS OWN sessions only. Mirrors lucien's ingest-recent.ts.
+// Two sources feed the synthesis ledger, both watermark-gated per `source`:
 //
-// See docs/design/memory-system.md → "Ingestion — mlx-bun sessions only".
+//   • pi sessions — SessionManager.listAll() walks every on-disk pi session
+//     (the same JSONL store pi-web / pi-terminal write). Each becomes a
+//     conversation; its resolved messages become `messages` rows. This is the
+//     steady-state, "synthesize my own sessions" path.
+//
+//   • the Lucien bootstrap — a one-time import of the read-only Lucien corpus
+//     (`~/Code/lucien/.lucien/lucien.db`) so the vault starts from a real body
+//     of prior conversations. Lucien's column names differ (B9 mapping below);
+//     critically we also REUSE Lucien's existing `chunks` by re-pointing each
+//     UUID span to mlx-bun `(start,end)` positions, so SEGMENT (P5-T2) only has
+//     to run on conversations Lucien never chunked.
+//
+// Everything is idempotent. Each source carries a watermark (max conversation
+// `updated_at`, as epoch-ms); a re-run only looks at conversations newer than
+// the cursor, and even a conversation that IS re-touched is rewritten in place
+// (delete-then-insert its messages/chunks), so re-ingesting changes nothing.
+//
+// Lucien → mlx-bun column map (B9):
+//   conversations.uuid→conv, name→title, source→source, updated_at→updated_at
+//   messages.conversation_uuid→conv, position→position, sender→role,
+//            uuid→uuid, text→text
+//   chunks.start_message_uuid/end_message_uuid → messages.position (per conv)
+//            → chunks.start/.end; id = `${conv}:${start}-${end}` (Lucien's
+//            INTEGER chunk id is discarded). chunked_at from
+//            chunked_conversations.chunked_at where present.
 
+import { Database } from "bun:sqlite";
+import { SessionManager } from "@earendil-works/pi-coding-agent";
+import { MemoryStore, chunkId } from "./db";
 import type { SynthesisEvent } from "./pipeline";
 
+/** Default read-only Lucien source DB for the bootstrap import. */
+export const DEFAULT_LUCIEN_DB = `${process.env.HOME}/Code/lucien/.lucien/lucien.db`;
+
 export interface IngestResult {
+  /** Conversations examined this run (across every source). */
   scanned: number;
+  /** Conversations written/refreshed this run (past the watermark). */
   ingested: number;
+  /** Message rows written this run. */
+  messages: number;
+  /** Chunk-pointer rows written this run (Lucien bootstrap only). */
+  chunks: number;
+}
+
+function emptyResult(): IngestResult {
+  return { scanned: 0, ingested: 0, messages: 0, chunks: 0 };
+}
+
+/** Parse an ISO-8601 / Date string to epoch-ms; 0 when unparseable. */
+function toEpoch(value: string | number | null | undefined): number {
+  if (value == null) return 0;
+  if (typeof value === "number") return Number.isFinite(value) ? value : 0;
+  const t = Date.parse(value);
+  return Number.isFinite(t) ? t : 0;
+}
+
+/** Read a source's watermark cursor as epoch-ms (0 when none). */
+function readWatermark(store: MemoryStore, source: string): number {
+  const row = store.db
+    .query("SELECT cursor FROM watermarks WHERE source = ?")
+    .get(source) as { cursor: string | null } | null;
+  // cursor is stored as an epoch-ms numeric string (see writeWatermark).
+  const n = row ? Number(row.cursor) : 0;
+  return Number.isFinite(n) ? n : 0;
+}
+
+/** Advance a source's watermark to `cursorEpoch` (epoch-ms, stored as text). */
+function writeWatermark(store: MemoryStore, source: string, cursorEpoch: number): void {
+  store.db.run(
+    `INSERT INTO watermarks (source, cursor, updated_at) VALUES (?, ?, ?)
+       ON CONFLICT(source) DO UPDATE SET cursor = excluded.cursor, updated_at = excluded.updated_at`,
+    [source, String(cursorEpoch), Date.now()],
+  );
+}
+
+/** Upsert one conversation header. */
+function upsertConversation(
+  store: MemoryStore,
+  conv: string,
+  source: string,
+  title: string,
+  updatedAt: number,
+  chunkedAt: number | null,
+): void {
+  store.db.run(
+    `INSERT INTO conversations (conv, source, title, updated_at, chunked_at) VALUES (?, ?, ?, ?, ?)
+       ON CONFLICT(conv) DO UPDATE SET
+         source = excluded.source, title = excluded.title,
+         updated_at = excluded.updated_at, chunked_at = excluded.chunked_at`,
+    [conv, source, title, updatedAt, chunkedAt],
+  );
+}
+
+/**
+ * Replace a conversation's messages with `rows` (delete-then-insert, so a
+ * re-ingested conversation ends up with exactly its current message set).
+ * Returns the number of message rows written.
+ */
+function replaceMessages(
+  store: MemoryStore,
+  conv: string,
+  rows: { position: number; role: string; uuid: string; text: string }[],
+): number {
+  store.db.run("DELETE FROM messages WHERE conv = ?", [conv]);
+  const ins = store.db.query(
+    "INSERT INTO messages (conv, position, role, uuid, text) VALUES (?, ?, ?, ?, ?)",
+  );
+  for (const r of rows) ins.run(conv, r.position, r.role, r.uuid, r.text);
+  return rows.length;
+}
+
+// ───────────────────────────── pi sessions ──────────────────────────────────
+
+/** First 8 lowercase hex digits of a session UUID — the conv:HASH citation id. */
+function convIdFromSessionId(id: string): string {
+  const hex = id.replace(/[^0-9a-fA-F]/g, "").toLowerCase();
+  return hex.slice(0, 8) || id.slice(0, 8);
+}
+
+/** Flatten a pi message's content to plain text (text parts only). */
+function piMessageText(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  const parts: string[] = [];
+  for (const part of content as { type?: string; text?: string }[]) {
+    if (part && part.type === "text" && typeof part.text === "string") parts.push(part.text);
+  }
+  return parts.join("\n");
+}
+
+/**
+ * Ingest pi sessions (every on-disk session SessionManager.listAll() finds)
+ * into the store under source "pi", watermark-gated by session modified time.
+ */
+export async function ingestSessions(
+  store: MemoryStore,
+  onEvent?: (e: SynthesisEvent) => void,
+): Promise<IngestResult> {
+  const result = emptyResult();
+  const source = "pi";
+  let infos: Awaited<ReturnType<typeof SessionManager.listAll>>;
+  try {
+    infos = await SessionManager.listAll();
+  } catch (err) {
+    onEvent?.({ type: "log", stage: "ingest", message: `pi sessions: none readable (${String(err)})` });
+    return result;
+  }
+
+  const cursor = readWatermark(store, source);
+  let maxEpoch = cursor;
+
+  for (const info of infos) {
+    result.scanned++;
+    const updatedAt = info.modified instanceof Date ? info.modified.getTime() : toEpoch(info.modified as never);
+    if (updatedAt <= cursor) continue;
+
+    let messages: { role: string; content: unknown }[] = [];
+    try {
+      const mgr = SessionManager.open(info.path);
+      messages = mgr.buildSessionContext().messages as { role: string; content: unknown }[];
+    } catch (err) {
+      onEvent?.({ type: "log", stage: "ingest", message: `skip session ${info.id}: ${String(err)}` });
+      continue;
+    }
+
+    const conv = convIdFromSessionId(info.id);
+    const rows: { position: number; role: string; uuid: string; text: string }[] = [];
+    let pos = 0;
+    for (const m of messages) {
+      if (m.role !== "user" && m.role !== "assistant") continue; // drop tool-result/system noise
+      const text = piMessageText(m.content);
+      if (text.trim() === "") continue;
+      rows.push({ position: pos, role: m.role, uuid: `${conv}-${pos}`, text });
+      pos++;
+    }
+    if (rows.length === 0) continue;
+
+    const title = info.name?.trim() || info.firstMessage?.trim() || conv;
+    const tx = store.db.transaction(() => {
+      upsertConversation(store, conv, source, title, updatedAt, null);
+      result.messages += replaceMessages(store, conv, rows);
+    });
+    tx();
+    result.ingested++;
+    if (updatedAt > maxEpoch) maxEpoch = updatedAt;
+  }
+
+  if (maxEpoch > cursor) writeWatermark(store, source, maxEpoch);
+  onEvent?.({
+    type: "log",
+    stage: "ingest",
+    message: `pi sessions: scanned ${result.scanned}, ingested ${result.ingested} (${result.messages} messages)`,
+  });
+  return result;
+}
+
+// ──────────────────────────── Lucien bootstrap ──────────────────────────────
+
+interface LucienConv {
+  uuid: string;
+  name: string | null;
+  source: string;
+  updated_at: string | null;
+  created_at: string | null;
+}
+interface LucienMsg {
+  uuid: string;
+  position: number;
+  sender: string;
+  text: string | null;
 }
+interface LucienChunk {
+  start_message_uuid: string;
+  end_message_uuid: string;
+  label: string | null;
+}
+
+/**
+ * One-time bootstrap import of the read-only Lucien corpus. Watermark-gated per
+ * Lucien `source` (chatgpt / claude-ai / …) so a re-run is a no-op. Reuses
+ * Lucien's existing chunks as mlx-bun chunk pointers (UUID span → positions).
+ */
+export async function ingestLucien(
+  store: MemoryStore,
+  dbPath: string = DEFAULT_LUCIEN_DB,
+  onEvent?: (e: SynthesisEvent) => void,
+): Promise<IngestResult> {
+  const result = emptyResult();
+  const src = new Database(dbPath, { readonly: true });
+  try {
+    // Map each conversation to its (optional) chunked_at epoch up front.
+    const chunkedAt = new Map<string, number>();
+    for (const row of src
+      .query("SELECT conversation_uuid AS conv, chunked_at FROM chunked_conversations")
+      .all() as { conv: string; chunked_at: string | null }[]) {
+      chunkedAt.set(row.conv, toEpoch(row.chunked_at));
+    }
+
+    const sources = (
+      src.query("SELECT DISTINCT source FROM conversations WHERE source IS NOT NULL").all() as {
+        source: string;
+      }[]
+    ).map((r) => r.source);
+
+    const convsBySource = src.query(
+      "SELECT uuid, name, source, updated_at, created_at FROM conversations WHERE source = ?",
+    );
+    const msgsByConv = src.query(
+      "SELECT uuid, position, sender, text FROM messages WHERE conversation_uuid = ? ORDER BY position, uuid",
+    );
+    const chunksByConv = src.query(
+      "SELECT start_message_uuid, end_message_uuid, label FROM chunks WHERE conversation_uuid = ?",
+    );
+
+    for (const source of sources) {
+      const cursor = readWatermark(store, source);
+      let maxEpoch = cursor;
+      const convs = convsBySource.all(source) as LucienConv[];
+
+      const pending: { conv: LucienConv; epoch: number }[] = [];
+      for (const conv of convs) {
+        result.scanned++;
+        const epoch = toEpoch(conv.updated_at) || toEpoch(conv.created_at);
+        if (epoch <= cursor) continue;
+        pending.push({ conv, epoch });
+        if (epoch > maxEpoch) maxEpoch = epoch;
+      }
+
+      const tx = store.db.transaction(() => {
+        for (const { conv, epoch } of pending) {
+          const id = conv.uuid; // B9: conversations.uuid → conv (full uuid)
+          const msgs = msgsByConv.all(id) as LucienMsg[];
+
+          // Renumber positions densely (0..n-1) in source order. Lucien keys
+          // messages by uuid, so a branched conversation can repeat a position;
+          // mlx-bun's (conv,position) PK requires uniqueness. Order is preserved,
+          // so chunk spans (resolved via this same map) stay contiguous.
+          const posByUuid = new Map<string, number>();
+          const rows = msgs.map((m, i) => {
+            posByUuid.set(m.uuid, i);
+            return { position: i, role: m.sender, uuid: m.uuid, text: m.text ?? "" };
+          });
+
+          upsertConversation(
+            store,
+            id,
+            conv.source,
+            conv.name ?? "",
+            epoch,
+            chunkedAt.get(id) ?? null,
+          );
+          result.messages += replaceMessages(store, id, rows);
+
+          // Re-point Lucien's chunks to mlx-bun positions, discarding its int id.
+          store.db.run("DELETE FROM chunks WHERE conv = ?", [id]);
+          const insChunk = store.db.query(
+            "INSERT OR IGNORE INTO chunks (id, conv, start, end, label) VALUES (?, ?, ?, ?, ?)",
+          );
+          for (const c of chunksByConv.all(id) as LucienChunk[]) {
+            const start = posByUuid.get(c.start_message_uuid);
+            const end = posByUuid.get(c.end_message_uuid);
+            if (start === undefined || end === undefined) continue; // unresolvable span
+            const lo = Math.min(start, end);
+            const hi = Math.max(start, end);
+            const info = insChunk.run(chunkId(id, lo, hi), id, lo, hi, c.label);
+            result.chunks += info.changes; // count only rows actually inserted
+          }
+          result.ingested++;
+        }
+        if (maxEpoch > cursor) writeWatermark(store, source, maxEpoch);
+      });
+      tx();
+
+      onEvent?.({
+        type: "log",
+        stage: "ingest",
+        message: `lucien[${source}]: ${pending.length} new conversations`,
+      });
+    }
+  } finally {
+    src.close();
+  }
 
-/** Read new pi sessions into memory.sqlite. STUB: returns zero, does nothing. */
-export async function ingestSessions(_onEvent?: (e: SynthesisEvent) => void): Promise<IngestResult> {
-  // TODO(M1): SessionManager.listAll() → normalize → upsert into conversations,
-  // advancing the per-source watermark. pi's session format is the contract.
-  throw new Error("memory.ingest is not implemented yet (M1)");
+  onEvent?.({
+    type: "log",
+    stage: "ingest",
+    message: `lucien bootstrap: scanned ${result.scanned}, ingested ${result.ingested} (${result.messages} messages, ${result.chunks} chunks)`,
+  });
+  return result;
 }