mlx-bun 0.0.8 → 0.0.10

package/src/memory/pipeline.ts

@@ -1,18 +1,68 @@
-// mlx-bun memory — synthesis pipeline orchestrator.  ⚠️ M1 STUB.
+// mlx-bun memory — synthesis pipeline orchestrator (P6-T4).
 //
-// Ties the stages together: ingest → chunk → cluster → synthesize → wikify →
-// changelog (the lucien nightly chain, reimplemented on the local model). What
-// `mlx-bun memory synthesize` and the nightly launchd job invoke.
+// Ties the stage modules into the cold-start DAG:
+//   ingest → SEGMENT → ENTITY-EXTRACT → ROUTE → CREATE → NORMALIZE → commitVault.
 //
-// CURRENTLY A NO-OP: it logs each stage as stubbed and writes nothing, so the
-// scheduled job runs cleanly tonight and every night without touching your
-// vault. When M1 lands, flip runSynthesis to actually call the stage modules
-// (each currently throws "not implemented") and move it onto the src/jobs/*
-// background runner with SSE progress, per the design doc.
+// `ingest` + `SEGMENT` already populated the corpus (conversations / messages /
+// chunks live in the MemoryStore), so for an already-chunked conversation
+// SEGMENT is a no-op and the pipeline operates on the existing chunks. The model
+// stages reuse the existing modules verbatim:
+//   - ENTITY-EXTRACT — entity.ts `extractEntities` (one chunk per call, persists
+//     chunk_entities + a candidate entities/aliases row via the variant resolver).
+//   - ROUTE — route.ts `RouteAccumulator` over the persisted chunk_entities
+//     (the deterministic canonical join + the B8/P1-T5 CREATE gate); the optional
+//     model substantive-engagement gate promotes thin-but-owned candidates.
+//   - CREATE/NORMALIZE/commit — synthesize.ts `synthesizeCreate` (sub-cluster →
+//     OUTLINE → per-section DRAFT → INFOBOX → assemble → NORMALIZE → gate → write
+//     + ledger), with a single `commitVault` at the end of the run.
 //
-// See docs/design/memory-system.md → "The nightly pipeline".
+// `runSynthesis` keeps the M1 event/summary contract `src/cli.ts` drives; it now
+// runs the real DAG over the default store + vault. The FULL-corpus bootstrap is
+// USER-ACTION (P6-T5) — the agent never starts it; `scripts/experiments`
+// drives a bounded handful of conversations through `runPipeline` directly.
 
-export type SynthesisStage = "ingest" | "chunk" | "cluster" | "synthesize" | "wikify" | "changelog";
+import type { ChunkCall } from "./chunk";
+import type { SectionCall } from "./cluster";
+import { runLinkStage } from "./crosslink";
+import { MemoryStore } from "./db";
+import type { ExtractCall } from "./entity";
+import {
+  buildEntityMeta,
+  chronoChunkIds,
+  runExtractStage,
+  runRouteStage,
+  runSegmentStage,
+  runSynthesizeStage,
+  selectCreateTargets,
+  type EntityMeta,
+  type PipelineCreated,
+  type PipelinePatched,
+  type RouteDecision,
+} from "./stages";
+import type { SynthesisCall } from "./synthesize";
+import { vaultRoot } from "./vault";
+import { wikifyVault } from "./wikify";
+
+// Re-export the factored-out pure helpers + types so existing importers (tests,
+// scripts) keep their `from "./pipeline"` entry points.
+export { buildEntityMeta, selectCreateTargets };
+export type { EntityMeta, PipelineCreated, PipelinePatched, RouteDecision };
+
+// ---- stage / event contract (consumed by src/cli.ts) -----------------------
+
+export type SynthesisStage =
+  | "ingest"
+  | "segment"
+  | "chunk"
+  | "extract"
+  | "route"
+  | "create"
+  | "section-route"
+  | "patch"
+  | "reconcile"
+  | "link"
+  | "wikify"
+  | "commit";
 
 export interface SynthesisEvent {
   type: "stage" | "log" | "done";
@@ -23,41 +73,203 @@ export interface SynthesisEvent {
 export interface SynthesisOptions {
   /** Only synthesize conversations newer than this (ISO date / conv cursor). */
   since?: string;
-  /** Override the synthesis model (default e4b). */
+  /** Override the synthesis model (default e4b). Reserved — the stages resolve
+   *  their own per-stage adapter via model.ts. */
   model?: string;
-  /** Plan only — never write the vault. */
+  /** Plan only — never write the vault (deterministic, no model). */
   dryRun?: boolean;
 }
 
 export interface SynthesisSummary {
-  /** False while stubbed; flip to true when the stages are implemented. */
   implemented: boolean;
   stages: SynthesisStage[];
   note: string;
 }
 
-const STAGES: SynthesisStage[] = ["ingest", "chunk", "cluster", "synthesize", "wikify", "changelog"];
+const STAGES: SynthesisStage[] = [
+  "ingest",
+  "segment",
+  "extract",
+  "route",
+  "create",
+  "section-route",
+  "patch",
+  "link",
+  "wikify",
+  "commit",
+];
+
+// ---- pipeline run ----------------------------------------------------------
+
+export interface PipelineOptions {
+  /** Restrict the run to these conversation ids; default every conversation. */
+  convIds?: string[];
+  /** Vault root (honors MLX_BUN_WIKI); defaults to `vaultRoot()`. */
+  root?: string;
+  /** Max articles to CREATE this run (bounds a smoke's GPU cost). */
+  articleCap?: number;
+  /** Entities to force into the CREATE set when they earn a `create`. */
+  mustCreate?: string[];
+  /** Model-call seam for SEGMENT (the `chunk` stage — our e4b-chunk-300 /
+   *  `memory-chunk` adapter). Defaults to `callLocal("chunk", …)` inside
+   *  chunkConversations; tests inject a fake to avoid a real model load. */
+  segmentCall?: ChunkCall;
+  /** Model-call seam for the CREATE drafting AND the steady-state PATCH (both run
+   *  the `synthesis` stage). Tests inject a fake. */
+  call?: SynthesisCall;
+  /** Model-call seam for ENTITY-EXTRACT (the `entity` stage). Tests inject a fake. */
+  extractCall?: ExtractCall;
+  /** Model-call seam for SECTION-ROUTE (the `section` stage binary/name calls).
+   *  Tests inject a fake. */
+  sectionCall?: SectionCall;
+  /** Run the subject-engagement model gate to promote a thin (single-chunk)
+   *  genuine-subject candidate to a `create` stub. Off by default (a recurring
+   *  subject already creates). */
+  useSubjectGate?: boolean;
+  /** Skip the final git commit (tests). */
+  commit?: boolean;
+  now?: number;
+  onEvent?: (e: SynthesisEvent) => void;
+}
+
+export interface PipelineResult {
+  convs: number;
+  chunks: number;
+  /** Distinct resolved entities seen across the routed chunks. */
+  entities: number;
+  created: PipelineCreated[];
+  /** Sections of existing articles patched this run (the self-healing folds). */
+  patched: PipelinePatched[];
+  skippedByGate: string[];
+  /** Subjects captured (not yet articled) — still retrievable by search. */
+  captured: string[];
+}
+
+/**
+ * Run the cold-start synthesis DAG over a bounded set of conversations.
+ *
+ * This is now a thin ORCHESTRATOR: it drives the four INDEPENDENT, chronological,
+ * resumable stage workers (`stages.ts`) in sequence — SEGMENT → ENTITY-EXTRACT →
+ * ROUTE → SYNTHESIZE (CREATE + SECTION-ROUTE/PATCH) — so the full DAG behaves as
+ * before while each stage is also separately runnable (`mlx-bun memory
+ * segment|extract|route|synthesize`). Returns a structured report; SYNTHESIZE
+ * commits once at the end via `commitVault`.
+ */
+export async function runPipeline(
+  store: MemoryStore,
+  opts: PipelineOptions = {},
+): Promise<PipelineResult> {
+  const root = opts.root ?? vaultRoot();
+  const cap = opts.articleCap ?? 20;
+  const emit = opts.onEvent ?? (() => {});
+
+  emit({ type: "stage", stage: "ingest", message: `ingest: corpus already in the store` });
+
+  // SEGMENT — (re)chunk every in-scope conversation whose chunked_at is NULL or
+  // stale, oldest-first, then proceed on the fresh chunks. A re-run is a no-op
+  // (watermark). We do NOT reuse any external chunks.
+  await runSegmentStage(store, { convIds: opts.convIds, call: opts.segmentCall, onEvent: opts.onEvent });
+
+  // Chronological run scope (oldest conversation first) for the result counts.
+  const { convs, chunkIds } = chronoChunkIds(store, opts.convIds);
+  emit({ type: "log", message: `operating on ${convs} conversation(s), ${chunkIds.length} chunk(s)` });
+
+  // ENTITY-EXTRACT — every not-yet-extracted chunk, chronological + resumable.
+  await runExtractStage(store, { convIds: opts.convIds, call: opts.extractCall, onEvent: opts.onEvent });
+
+  // ROUTE — deterministic create/capture over chunk_entities + CREATE gate;
+  // persists notable + captures thin subjects into _captured (still searchable)
+  // so SYNTHESIZE reads the decision back.
+  const route = await runRouteStage(store, {
+    convIds: opts.convIds,
+    useSubjectGate: opts.useSubjectGate,
+    onEvent: opts.onEvent,
+  });
+
+  // SYNTHESIZE — chronological CREATE (oldest entity first, oldest chunk first so
+  // the latest statement dominates) + SECTION-ROUTE → PATCH self-healing fold.
+  const synth = await runSynthesizeStage(store, {
+    root,
+    limit: cap,
+    convIds: opts.convIds,
+    mustCreate: opts.mustCreate,
+    call: opts.call,
+    sectionCall: opts.sectionCall,
+    commit: opts.commit,
+    now: opts.now,
+    onEvent: opts.onEvent,
+  });
+
+  // CROSS-LINK — build the EDGES synthesis (bounded to one article) cannot:
+  // inline-link first mentions of other articles + rebuild each ## See also from
+  // mentions + co-occurrence. Deterministic (no model) and idempotent.
+  await runLinkStage(store, { root, commit: opts.commit, onEvent: (e) => emit({ type: e.type, message: e.message }) });
+
+  emit({
+    type: "done",
+    message: `synthesis done — ${synth.created.length} created, ${synth.patched.length} patched, ${synth.skippedByGate.length} gated, ${route.captured.length} captured.`,
+  });
+
+  return {
+    convs,
+    chunks: chunkIds.length,
+    entities: route.decisions.length,
+    created: synth.created,
+    patched: synth.patched,
+    skippedByGate: synth.skippedByGate,
+    captured: route.captured,
+  };
+}
+
+// ---- cli entry: the M1 contract src/cli.ts drives --------------------------
 
 /**
- * Run the synthesis pipeline. STUB: walks the stage list, emits a `stage` event
- * for each (marked skipped), and returns implemented:false. Deterministic and
- * side-effect-free, so it's safe for the nightly job to call today.
+ * Run the synthesis pipeline over the default store + vault — the `mlx-bun
+ * memory synthesize` path. Keeps the {@link SynthesisSummary} shape the CLI
+ * prints. The FULL-corpus run is a long GPU job (USER-ACTION, P6-T5); this entry
+ * exists so the wiring is complete — the agent never starts it.
  */
 export async function runSynthesis(
   opts: SynthesisOptions = {},
   onEvent?: (e: SynthesisEvent) => void,
 ): Promise<SynthesisSummary> {
-  onEvent?.({
-    type: "log",
-    message: `memory synthesis is not implemented yet (M1)${opts.dryRun ? " · dry-run" : ""} — running as a no-op, nothing will be written.`,
-  });
-  for (const stage of STAGES) {
-    onEvent?.({ type: "stage", stage, message: `${stage}: stubbed (skipped)` });
+  if (opts.dryRun) {
+    onEvent?.({ type: "log", message: "dry-run — no model calls, nothing will be written." });
+    for (const stage of STAGES) onEvent?.({ type: "stage", stage, message: `${stage}: planned (dry-run)` });
+    onEvent?.({ type: "done", message: "dry-run complete — no articles written." });
+    return { implemented: true, stages: STAGES, note: "dry-run: DAG wired; no model calls made." };
+  }
+
+  const store = new MemoryStore();
+  try {
+    // WRITE branch: ingest → segment → extract → route → create → commit.
+    const result = await runPipeline(store, { onEvent });
+
+    // WIKIFY branch: the periodic editorial sweep over EVERY article (P8). Runs
+    // after the write branch so freshly-created and patched articles are tightened
+    // + infobox-refreshed in the same nightly pass; each article commits its own
+    // change (or NO-OPs) behind the same conservative gate. Best-effort: a wikify
+    // defect must not lose the create-branch work already committed above.
+    onEvent?.({ type: "stage", stage: "wikify", message: "wikify: editorial sweep over every article" });
+    let editedCount = 0;
+    try {
+      const sweep = await wikifyVault({ root: vaultRoot() });
+      editedCount = sweep.edited.length;
+      onEvent?.({ type: "log", message: `  wikify edited ${editedCount}/${sweep.results.length} article(s)` });
+    } catch (err) {
+      onEvent?.({ type: "log", message: `  wikify sweep skipped (error: ${String(err)})` });
+    }
+    onEvent?.({
+      type: "done",
+      message: `synthesis done — ${result.created.length} created, ${result.patched.length} patched, ${editedCount} wikified.`,
+    });
+
+    return {
+      implemented: true,
+      stages: STAGES,
+      note: `synthesis wired — ${result.created.length} created, ${result.patched.length} patched, ${editedCount} wikified from ${result.convs} conversations.`,
+    };
+  } finally {
+    store.close();
   }
-  onEvent?.({ type: "done", message: "no articles were written (synthesis is stubbed)." });
-  return {
-    implemented: false,
-    stages: STAGES,
-    note: "M1 stub — pipeline + scheduling wired; stage modules not implemented.",
-  };
 }

package/src/memory/prompts.ts

@@ -9,6 +9,11 @@
 //
 // See docs/design/memory-system.md → "Synthesis runs on the local model".
 
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { vaultRoot } from "./vault";
+
 export const CHUNK_PROMPT = `TODO(M1): segment a conversation transcript into topic-coherent chunks.`;
 
 export const CLUSTER_PROMPT = `TODO(M1): assign a chunk to an existing bucket from Meta/Buckets.md, or propose a new bucket.`;
@@ -21,3 +26,40 @@ export const STAGE_PROMPTS: Record<string, string> = {
   cluster: CLUSTER_PROMPT,
   synthesize: SYNTHESIZE_PROMPT,
 };
+
+// ---- meta-policy inlining (ported from lucien scripts/meta-inline.ts) ------
+//
+// Editorial policy lives in the vault's Meta/ pages, not in code. Rather than
+// telling the model to Read those pages at synthesis time (which turns every
+// pipeline call into an agent loop whose context accumulates — a 34k-char
+// conversation was observed ballooning to a 105k-token prefill on the local
+// server), we inline the requested pages directly into the prompt. Calls stay
+// single-turn: one bounded prefill, one bounded generation. The policy is
+// edit-in-the-vault: change a Meta page on disk and the next run reflects it
+// with zero code change.
+
+/**
+ * Read the named Meta policy pages from the vault's Meta/ dir and concatenate
+ * them for inlining into a stage prompt. Each `name` addresses
+ * `<vault>/Meta/<name>.md` (a trailing `.md` is accepted and normalized).
+ *
+ * The vault root honors the MLX_BUN_WIKI override (see {@link vaultRoot}).
+ *
+ * @throws {Error} If a requested Meta page does not exist on disk, naming it.
+ */
+export function loadMetaPolicy(names: string[]): string {
+  const metaDir = join(vaultRoot(), "Meta");
+  const sections: string[] = [];
+  for (const name of names) {
+    const stem = name.trim().endsWith(".md") ? name.trim().slice(0, -3) : name.trim();
+    const path = join(metaDir, `${stem}.md`);
+    let text: string;
+    try {
+      text = readFileSync(path, "utf8");
+    } catch {
+      throw new Error(`Meta policy page not found: ${path}`);
+    }
+    sections.push(`--- Meta/${stem}.md ---\n\n${text.trim()}`);
+  }
+  return sections.join("\n\n");
+}