mlx-bun 0.0.4 → 0.0.6

package/src/memory/db.ts

@@ -0,0 +1,114 @@
+// mlx-bun memory — synthesis bookkeeping store (bun:sqlite).
+//
+// ⚠️ M1 STUB schema-wise the tables are real, but no stage writes them yet.
+// The read path (vault.ts / tools.ts) does NOT touch this — it's only used by
+// the synthesis pipeline (pipeline.ts), which is itself stubbed.
+//
+// Reuses mlx-bun's existing per-domain sqlite pattern (cf. JobStore in
+// src/jobs/db.ts, EvalDB in src/evaldb.ts): a small class over bun:sqlite with
+// WAL + schema-in-constructor + an additive migrate() hook, living under
+// ~/.cache/mlx-bun/ alongside jobs.sqlite / evals.sqlite / registry.sqlite. It's
+// derived state — rebuildable from the pi transcripts + the vault articles — so
+// it belongs in the rebuildable cache, NOT in the (precious, git-tracked) vault.
+//
+// The schema mirrors lucien's incremental design: per-source watermarks (don't
+// re-ingest), chunked_at > updated_at (re-chunk only changed convs), and a
+// synthesized-chunk ledger (never fold the same chunk into an article twice).
+
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+
+export const DEFAULT_MEMORY_DB = `${process.env.HOME}/.cache/mlx-bun/memory.sqlite`;
+
+export interface ConversationRow {
+  /** Stable conv id: first 8 hex of the session UUID (the conv:HASH citation). */
+  conv: string;
+  source: string; // "pi-web" | "pi-terminal"
+  title: string;
+  /** Full normalized transcript JSON ({role, content}[]). */
+  transcript: string;
+  updated_at: number;
+  chunked_at: number | null;
+}
+
+export interface ChunkRow {
+  id: string;
+  conv: string;
+  ordinal: number;
+  text: string;
+  bucket: string | null;
+}
+
+export interface BucketRow {
+  name: string;
+  article: string; // target article stem
+  description: string;
+}
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS conversations (
+  conv       TEXT PRIMARY KEY,
+  source     TEXT NOT NULL,
+  title      TEXT NOT NULL DEFAULT '',
+  transcript TEXT NOT NULL,
+  updated_at INTEGER NOT NULL,
+  chunked_at INTEGER
+);
+CREATE TABLE IF NOT EXISTS chunks (
+  id      TEXT PRIMARY KEY,
+  conv    TEXT NOT NULL REFERENCES conversations(conv),
+  ordinal INTEGER NOT NULL,
+  text    TEXT NOT NULL,
+  bucket  TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_chunks_conv ON chunks(conv);
+CREATE TABLE IF NOT EXISTS buckets (
+  name        TEXT PRIMARY KEY,
+  article     TEXT NOT NULL,
+  description TEXT NOT NULL DEFAULT ''
+);
+-- Per-source ingest watermark: only process what's new since last run.
+CREATE TABLE IF NOT EXISTS watermarks (
+  source     TEXT PRIMARY KEY,
+  cursor     TEXT NOT NULL,
+  updated_at INTEGER NOT NULL
+);
+-- Idempotency ledger: a (bucket, chunk) pair is folded into an article once.
+CREATE TABLE IF NOT EXISTS synthesized_bucket_chunks (
+  bucket   TEXT NOT NULL,
+  chunk_id TEXT NOT NULL,
+  PRIMARY KEY (bucket, chunk_id)
+);
+`;
+
+/**
+ * Synthesis bookkeeping DB. Same shape as JobStore/EvalDB: open in the
+ * constructor (WAL on, schema ensured, additive migrate), expose `db`, and a
+ * `close()`. Only the synthesis pipeline constructs one — opening it is what
+ * *creates* memory.sqlite, so the read path never does. Stage queries land in
+ * M1; the tables they expect already exist here.
+ */
+export class MemoryStore {
+  readonly db: Database;
+  readonly dbPath: string;
+
+  constructor(dbPath: string = DEFAULT_MEMORY_DB) {
+    if (dbPath !== ":memory:") {
+      try { mkdirSync(dbPath.slice(0, dbPath.lastIndexOf("/")), { recursive: true }); } catch {}
+    }
+    this.dbPath = dbPath;
+    this.db = new Database(dbPath);
+    this.db.exec("PRAGMA journal_mode = WAL");
+    this.db.exec(SCHEMA);
+    this.migrate();
+  }
+
+  /** Additive-only migrations (mirrors JobStore.migrate). Re-opening is a no-op. */
+  private migrate(): void {
+    // No columns to add today; the hook stays so future fields are additive.
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}

package/src/memory/ingest.ts

@@ -0,0 +1,22 @@
+// mlx-bun memory — synthesis stage 1: INGEST.  ⚠️ M1 STUB.
+//
+// pi sessions (~/.mlx-bun/sessions + ~/.mlx-bun/pi, pi's own JSONL) →
+// memory.sqlite, normalized to {conv, source, title, transcript}, watermark-
+// gated so each run only processes what's new. No web scraping, no Playwright:
+// mlx-bun synthesizes ITS OWN sessions only. Mirrors lucien's ingest-recent.ts.
+//
+// See docs/design/memory-system.md → "Ingestion — mlx-bun sessions only".
+
+import type { SynthesisEvent } from "./pipeline";
+
+export interface IngestResult {
+  scanned: number;
+  ingested: number;
+}
+
+/** Read new pi sessions into memory.sqlite. STUB: returns zero, does nothing. */
+export async function ingestSessions(_onEvent?: (e: SynthesisEvent) => void): Promise<IngestResult> {
+  // TODO(M1): SessionManager.listAll() → normalize → upsert into conversations,
+  // advancing the per-source watermark. pi's session format is the contract.
+  throw new Error("memory.ingest is not implemented yet (M1)");
+}

package/src/memory/pipeline.ts

@@ -0,0 +1,63 @@
+// mlx-bun memory — synthesis pipeline orchestrator.  ⚠️ M1 STUB.
+//
+// Ties the stages together: ingest → chunk → cluster → synthesize → wikify →
+// changelog (the lucien nightly chain, reimplemented on the local model). What
+// `mlx-bun memory synthesize` and the nightly launchd job invoke.
+//
+// CURRENTLY A NO-OP: it logs each stage as stubbed and writes nothing, so the
+// scheduled job runs cleanly tonight and every night without touching your
+// vault. When M1 lands, flip runSynthesis to actually call the stage modules
+// (each currently throws "not implemented") and move it onto the src/jobs/*
+// background runner with SSE progress, per the design doc.
+//
+// See docs/design/memory-system.md → "The nightly pipeline".
+
+export type SynthesisStage = "ingest" | "chunk" | "cluster" | "synthesize" | "wikify" | "changelog";
+
+export interface SynthesisEvent {
+  type: "stage" | "log" | "done";
+  stage?: SynthesisStage;
+  message: string;
+}
+
+export interface SynthesisOptions {
+  /** Only synthesize conversations newer than this (ISO date / conv cursor). */
+  since?: string;
+  /** Override the synthesis model (default e4b). */
+  model?: string;
+  /** Plan only — never write the vault. */
+  dryRun?: boolean;
+}
+
+export interface SynthesisSummary {
+  /** False while stubbed; flip to true when the stages are implemented. */
+  implemented: boolean;
+  stages: SynthesisStage[];
+  note: string;
+}
+
+const STAGES: SynthesisStage[] = ["ingest", "chunk", "cluster", "synthesize", "wikify", "changelog"];
+
+/**
+ * Run the synthesis pipeline. STUB: walks the stage list, emits a `stage` event
+ * for each (marked skipped), and returns implemented:false. Deterministic and
+ * side-effect-free, so it's safe for the nightly job to call today.
+ */
+export async function runSynthesis(
+  opts: SynthesisOptions = {},
+  onEvent?: (e: SynthesisEvent) => void,
+): Promise<SynthesisSummary> {
+  onEvent?.({
+    type: "log",
+    message: `memory synthesis is not implemented yet (M1)${opts.dryRun ? " · dry-run" : ""} — running as a no-op, nothing will be written.`,
+  });
+  for (const stage of STAGES) {
+    onEvent?.({ type: "stage", stage, message: `${stage}: stubbed (skipped)` });
+  }
+  onEvent?.({ type: "done", message: "no articles were written (synthesis is stubbed)." });
+  return {
+    implemented: false,
+    stages: STAGES,
+    note: "M1 stub — pipeline + scheduling wired; stage modules not implemented.",
+  };
+}

package/src/memory/prompts.ts

@@ -0,0 +1,23 @@
+// mlx-bun memory — per-stage synthesis prompts.  ⚠️ M1 STUBS.
+//
+// Each stage's prompt is the seam where a future specialized memory LoRA plugs
+// in (today: base model + prompt; later: adapterScoped per stage). Editorial
+// conventions are NOT duplicated here — they live in the vault's Meta/ pages
+// (Editorial_Guidelines.md, Article_Conventions.md, Buckets.md,
+// Topics_to_Ignore.md) and are read in at synthesis time so the system governs
+// itself in its own substrate.
+//
+// See docs/design/memory-system.md → "Synthesis runs on the local model".
+
+export const CHUNK_PROMPT = `TODO(M1): segment a conversation transcript into topic-coherent chunks.`;
+
+export const CLUSTER_PROMPT = `TODO(M1): assign a chunk to an existing bucket from Meta/Buckets.md, or propose a new bucket.`;
+
+export const SYNTHESIZE_PROMPT = `TODO(M1): integrate new chunks into the bucket's article. Preserve prose and every conv: citation; integrate, don't overwrite; respect the ≥70% word-count floor and the editorial gate.`;
+
+/** Placeholder registry so the pipeline can reference prompts by stage name. */
+export const STAGE_PROMPTS: Record<string, string> = {
+  chunk: CHUNK_PROMPT,
+  cluster: CLUSTER_PROMPT,
+  synthesize: SYNTHESIZE_PROMPT,
+};

package/src/memory/schedule.ts

@@ -0,0 +1,163 @@
+// mlx-bun memory — nightly scheduling via launchd.  ✅ REAL (the pipeline it
+// runs is the M1 stub, but install/remove/status here are fully functional).
+//
+// mlx-bun is macOS/Apple-silicon-only by identity, so launchd is the native
+// choice over crontab: it survives reboot, runs jobs missed while asleep on
+// next wake, and gives clean install/uninstall. `memory schedule` writes
+// ~/Library/LaunchAgents/com.mlx-bun.memory.plist (a StartCalendarInterval that
+// runs `mlx-bun memory synthesize`) and loads it; `unschedule` unloads + removes
+// it. Today that job is a no-op (synthesis is stubbed) — harmless; when M1 lands
+// the same schedule starts producing articles with no reconfiguration.
+//
+// See docs/design/memory-system.md → "Scheduling (launchd)".
+
+import { spawn } from "node:child_process";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export const LAUNCHD_LABEL = "com.mlx-bun.memory";
+
+export function plistPath(): string {
+  return join(homedir(), "Library", "LaunchAgents", `${LAUNCHD_LABEL}.plist`);
+}
+
+function logsDir(): string {
+  return join(homedir(), ".mlx-bun", "logs");
+}
+
+/**
+ * Resolve how to invoke mlx-bun from launchd's minimal environment. Prefer the
+ * installed binary on PATH; fall back to the current dev invocation
+ * (`<bun> <cli.ts>`). Wrapped in a login shell at call time so PATH/bun resolve.
+ */
+function mlxBunCommand(): string {
+  const onPath = Bun.which("mlx-bun");
+  if (onPath) return shQuote(onPath);
+  // Dev: process.execPath is bun, argv[1] is .../src/cli.ts.
+  return `${shQuote(process.execPath)} ${shQuote(process.argv[1] ?? "")}`.trim();
+}
+
+function shQuote(s: string): string {
+  return `'${s.replace(/'/g, "'\\''")}'`;
+}
+
+function xmlEscape(s: string): string {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+
+export interface ScheduleOptions {
+  /** Local wall-clock time, "HH:MM" 24h. Default 03:00. */
+  at?: string;
+}
+
+/** Parse "HH:MM" → {hour, minute}; defaults to 03:00 on bad/empty input. */
+export function parseAt(at?: string): { hour: number; minute: number } {
+  const m = /^(\d{1,2}):(\d{2})$/.exec((at ?? "").trim());
+  if (!m) return { hour: 3, minute: 0 };
+  const hour = Math.min(23, Math.max(0, Number(m[1])));
+  const minute = Math.min(59, Math.max(0, Number(m[2])));
+  return { hour, minute };
+}
+
+/** Pure: build the launchd plist XML for the nightly synthesis job. */
+export function buildPlist(opts: ScheduleOptions = {}): string {
+  const { hour, minute } = parseAt(opts.at);
+  // Login shell so the user's PATH (bun, mlx-bun) resolves under launchd.
+  const cmd = `exec ${mlxBunCommand()} memory synthesize`;
+  const out = join(logsDir(), "memory-synthesis.out.log");
+  const err = join(logsDir(), "memory-synthesis.err.log");
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>${LAUNCHD_LABEL}</string>
+
+    <key>ProgramArguments</key>
+    <array>
+        <string>/bin/zsh</string>
+        <string>-lc</string>
+        <string>${xmlEscape(cmd)}</string>
+    </array>
+
+    <!-- Local wall-clock; if asleep at the time, launchd runs on next wake. -->
+    <key>StartCalendarInterval</key>
+    <dict>
+        <key>Hour</key>
+        <integer>${hour}</integer>
+        <key>Minute</key>
+        <integer>${minute}</integer>
+    </dict>
+
+    <key>RunAtLoad</key>
+    <false/>
+
+    <key>StandardOutPath</key>
+    <string>${xmlEscape(out)}</string>
+    <key>StandardErrorPath</key>
+    <string>${xmlEscape(err)}</string>
+
+    <key>ProcessType</key>
+    <string>Background</string>
+</dict>
+</plist>
+`;
+}
+
+function launchctl(args: string[]): Promise<boolean> {
+  return new Promise((resolve) => {
+    const proc = spawn("launchctl", args, { stdio: "ignore" });
+    proc.on("exit", (code) => resolve(code === 0));
+    proc.on("error", () => resolve(false));
+  });
+}
+
+export interface InstallResult {
+  plistPath: string;
+  at: { hour: number; minute: number };
+  loaded: boolean;
+}
+
+/** Write the plist and (re)load it into launchd. */
+export async function installSchedule(opts: ScheduleOptions = {}): Promise<InstallResult> {
+  const path = plistPath();
+  await mkdir(join(homedir(), "Library", "LaunchAgents"), { recursive: true });
+  await mkdir(logsDir(), { recursive: true });
+  await writeFile(path, buildPlist(opts));
+  // Reload: unload first (ignore failure if not loaded), then load -w.
+  await launchctl(["unload", path]);
+  const loaded = await launchctl(["load", "-w", path]);
+  return { plistPath: path, at: parseAt(opts.at), loaded };
+}
+
+/** Unload and remove the launchd agent. Returns true if a plist was removed. */
+export async function removeSchedule(): Promise<boolean> {
+  const path = plistPath();
+  if (!existsSync(path)) return false;
+  await launchctl(["unload", "-w", path]);
+  await rm(path, { force: true });
+  return true;
+}
+
+export interface ScheduleStatus {
+  installed: boolean;
+  plistPath: string;
+  /** Whether launchctl currently lists the agent (best-effort). */
+  loaded: boolean;
+}
+
+export async function scheduleStatus(): Promise<ScheduleStatus> {
+  const path = plistPath();
+  const installed = existsSync(path);
+  let loaded = false;
+  if (installed) {
+    loaded = await new Promise<boolean>((resolve) => {
+      const proc = spawn("launchctl", ["list", LAUNCHD_LABEL], { stdio: "ignore" });
+      proc.on("exit", (code) => resolve(code === 0));
+      proc.on("error", () => resolve(false));
+    });
+  }
+  return { installed, plistPath: path, loaded };
+}

package/src/memory/synthesize.ts

@@ -0,0 +1,29 @@
+// mlx-bun memory — synthesis stage 4: SYNTHESIZE.  ⚠️ M1 STUB.
+//
+// For each bucket touched this run, integrate its new chunks into the bucket's
+// article (local-model call) — create-or-update, conservatively. The hard,
+// proven gate is inherited from lucien: preserve prose, ≥70% word-count floor,
+// every original conv: citation survives, integrate rather than overwrite. A
+// weak local pass is gated to NO-OP rather than allowed to corrupt the vault.
+// Mirrors lucien's synthesize-dispatch.ts / synthesize-update.ts.
+//
+// See docs/design/memory-system.md → "Synthesis runs on the local model".
+
+import type { SynthesisEvent } from "./pipeline";
+
+export interface SynthesizeResult {
+  articlesCreated: number;
+  articlesUpdated: number;
+  skippedByGate: number;
+}
+
+/** Bucket → article create/update + safety gate (local model). STUB. */
+export async function synthesizeBuckets(
+  _model: unknown,
+  _onEvent?: (e: SynthesisEvent) => void,
+): Promise<SynthesizeResult> {
+  // TODO(M1): per touched bucket, feed unsynthesized chunks + current article
+  // through the synthesize prompt; apply the conservative gate; write the
+  // article; record (bucket, chunk_id) in synthesized_bucket_chunks.
+  throw new Error("memory.synthesize is not implemented yet (M1)");
+}