mulmoclaude 0.5.2 → 0.6.0

package/server/workspace/memory/llm-classifier.ts

@@ -0,0 +1,158 @@
+// LLM-backed `MemoryClassifier` for the legacy-memory migration
+// (#1029 PR-B). Asks Claude to look at a single legacy bullet plus
+// its surrounding H2 section header and produce a JSON verdict
+// `{type, description}`. The four types and their meaning live in
+// the system prompt so the LLM can pick consistently across
+// candidates.
+//
+// CLEANUP 2026-07-01: see `run.ts` — this file is part of the
+// one-shot `memory.md` → atomic migration chain and goes when the
+// chain goes.
+//
+// The function is intentionally one-call-per-candidate (not a batch
+// classify): batching saves tokens but couples failures together, so
+// a single malformed verdict could poison the whole batch. Migration
+// is one-time and small — keep the request shape simple.
+
+import type { MemoryClassification, MemoryCandidate, MemoryClassifier } from "./migrate.js";
+import { isMemoryType, type MemoryType } from "./types.js";
+
+import { ClaudeCliNotFoundError, type Summarize } from "../journal/archivist-cli.js";
+import { errorMessage } from "../../utils/errors.js";
+import { log } from "../../system/logger/index.js";
+
+const SYSTEM_PROMPT = `You are classifying a single bullet from a personal-memory file into one of four types.
+
+Types:
+- preference: a durable habit, preference, or convention that does not change over time. Examples: "uses yarn (npm not allowed)", "prefers Emacs", "writes commit messages in English".
+- interest: a topic, hobby, or domain the user follows over a long horizon. Examples: "AI research papers", "robotics", "Impressionist painting".
+- fact: a concrete personal fact that COULD become stale. Examples: "planning a trip to Egypt", "owns a toaster oven", "currently working on BootCamp project".
+- reference: a pointer to an internal/external resource (path, dashboard, recurring task id, repo URL). Examples: "main repo at ~/ss/llm/mulmoclaude4", "weekly art-exhibitions-watch task".
+
+Rules:
+- Output ONE compact JSON object. No prose, no markdown, no code fences. Output literal JSON only.
+- Schema: {"type":"<one of preference|interest|fact|reference>","description":"<short single-line description, <=100 chars>"}
+- The description is for an index file. Keep it short and informative — strip filler, don't repeat the bullet verbatim.
+- If the bullet is too vague to classify, output: null
+- Never invent fields. Output exactly the schema above or null.`;
+
+interface DepsForLlmClassifier {
+  /** Same Summarize callback used by `journal/dailyPass`. The
+   *  classifier feeds it the system prompt + a single-bullet user
+   *  prompt and parses the JSON reply. */
+  summarize: Summarize;
+}
+
+export function makeLlmMemoryClassifier(deps: DepsForLlmClassifier): MemoryClassifier {
+  return async (candidate: MemoryCandidate) => {
+    const userPrompt = buildUserPrompt(candidate);
+    let raw: string;
+    try {
+      raw = await deps.summarize(SYSTEM_PROMPT, userPrompt);
+    } catch (err) {
+      // Let `ClaudeCliNotFoundError` escape so the caller can abort
+      // the whole migration. Without this, every candidate runs
+      // through the classifier, gets `null` back, and migrateLegacyMemory
+      // produces an empty result + (pre-#1091) renames the legacy file
+      // to `.backup`. Even with #1091 the user pays N classifier
+      // failures (each spawns claude) before realising the CLI is
+      // missing (#1061 review).
+      if (err instanceof ClaudeCliNotFoundError) throw err;
+      log.warn("memory", "llm classifier: summarize threw", {
+        preview: candidate.body.slice(0, 80),
+        error: errorMessage(err),
+      });
+      return null;
+    }
+    return parseClassifierVerdict(raw);
+  };
+}
+
+function buildUserPrompt(candidate: MemoryCandidate): string {
+  const sectionLine = candidate.section ? `Section header: ${candidate.section}` : "Section header: (none)";
+  return [sectionLine, `Bullet: ${candidate.body}`, "", "Output JSON only."].join("\n");
+}
+
+// Tolerant JSON extraction — Claude occasionally wraps the verdict in
+// a code fence or adds a leading word despite the prompt ban. Strip
+// surrounding fences and pick out the first `{...}` block (or a bare
+// `null`) before parsing.
+export function parseClassifierVerdict(raw: string): MemoryClassification | null {
+  const trimmed = stripFenceAndWhitespace(raw);
+  if (trimmed === "null") return null;
+  const objectText = extractFirstObject(trimmed);
+  if (!objectText) return null;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(objectText);
+  } catch {
+    return null;
+  }
+  if (!isPlainObject(parsed)) return null;
+  const { type, description } = parsed as { type?: unknown; description?: unknown };
+  if (!isMemoryType(type)) return null;
+  const desc = typeof description === "string" ? description.trim() : "";
+  // The description is optional from the spec's perspective —
+  // migrate.ts falls back to a body-derived description when missing.
+  // But if present, normalise to a single line and cap length.
+  // Cap matches the prompt's `<=100 chars>` declaration so persisted
+  // entries never exceed the schema the classifier was told to follow
+  // (#1061 review).
+  return desc.length > 0 ? { type: type as MemoryType, description: oneLine(desc).slice(0, 100) } : { type: type as MemoryType };
+}
+
+function stripFenceAndWhitespace(raw: string): string {
+  let text = raw.trim();
+  if (text.startsWith("```")) {
+    const firstNl = text.indexOf("\n");
+    if (firstNl >= 0) text = text.slice(firstNl + 1);
+    if (text.endsWith("```")) text = text.slice(0, -3);
+  }
+  return text.trim();
+}
+
+function extractFirstObject(text: string): string | null {
+  const start = text.indexOf("{");
+  if (start < 0) return null;
+  let depth = 0;
+  let index = start;
+  while (index < text.length) {
+    const char = text[index];
+    if (char === '"') {
+      index = skipStringBody(text, index + 1);
+      continue;
+    }
+    if (char === "{") depth += 1;
+    else if (char === "}") {
+      depth -= 1;
+      if (depth === 0) return text.slice(start, index + 1);
+    }
+    index += 1;
+  }
+  return null;
+}
+
+// Returns the index immediately after the closing `"`, or `text.length`
+// if the string is unterminated. Backslash escapes the next char so
+// `\"` does not close the string.
+function skipStringBody(text: string, fromIndex: number): number {
+  let index = fromIndex;
+  while (index < text.length) {
+    const char = text[index];
+    if (char === "\\") {
+      index += 2;
+      continue;
+    }
+    if (char === '"') return index + 1;
+    index += 1;
+  }
+  return text.length;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function oneLine(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}

package/server/workspace/memory/migrate.ts

@@ -0,0 +1,263 @@
+// Migration of legacy `conversations/memory.md` into the typed
+// directory layout (#1029 PR-A). This module is a library — it does
+// NOT decide when to run. PR-B wires it into `workspace.ts`'s init
+// path so the conversion happens once on first start after the new
+// layout ships.
+//
+// CLEANUP 2026-07-01: see `run.ts` — this file is part of the
+// one-shot `memory.md` → atomic migration chain and goes when the
+// chain goes.
+//
+// Steps:
+//   1. Read existing `conversations/memory.md`. Absent → no-op.
+//   2. Split by `## ` (H2). Each H2 block becomes a candidate entry
+//      (one per top-level bullet).
+//   3. Ask the supplied classifier "preference / interest / fact /
+//      reference?" per candidate. Null verdict → skip + count.
+//   4. Write each kept candidate as `<type>_<slug>.md`.
+//   5. Rebuild `MEMORY.md`.
+//   6. Rename source to `memory.md.backup` (never deleted).
+
+import { mkdir, rename } from "node:fs/promises";
+import path from "node:path";
+
+import { writeFileAtomic } from "../../utils/files/atomic.js";
+import { readTextSafe } from "../../utils/files/safe.js";
+import { log } from "../../system/logger/index.js";
+import { errorMessage } from "../../utils/errors.js";
+import { ClaudeCliNotFoundError } from "../journal/archivist-cli.js";
+import { WORKSPACE_FILES } from "../paths.js";
+import { regenerateIndex, writeMemoryEntry } from "./io.js";
+import { isMemoryType, slugifyMemoryName, type MemoryEntry, type MemoryType } from "./types.js";
+
+export interface MemoryClassification {
+  type: MemoryType;
+  /** Optional one-line description. Falls back to a truncated body. */
+  description?: string;
+}
+
+export type MemoryClassifier = (candidate: MemoryCandidate) => Promise<MemoryClassification | null>;
+
+export interface MemoryCandidate {
+  /** H2 section header the bullet was found under. Empty string for
+   *  bullets that appeared before any H2. */
+  section: string;
+  /** Single-line text of the bullet (lead-in `- ` already stripped). */
+  body: string;
+}
+
+export interface MigrationResult {
+  /** Set when there was nothing to migrate (no `memory.md`). */
+  noop: boolean;
+  /** Count of candidates accepted, broken down by type. */
+  written: Record<MemoryType, number>;
+  /** Candidates the classifier returned `null` for. */
+  skippedByClassifier: number;
+  /** Candidates that errored on write — caller may want to retry or
+   *  surface to the user. The migration continues past write errors
+   *  so a single bad entry doesn't strand the whole batch. */
+  writeErrors: number;
+}
+
+// Number of in-flight `classify` calls. Each call spawns the Claude
+// CLI, so there's a real cost to going wide; 4 is empirically a
+// sweet spot — well above the 1-at-a-time baseline (~10× faster on
+// real legacy files) while staying polite to the user's machine and
+// to Claude's quota. The classifier callback is the only awaited
+// step, so this is the dominant knob.
+const CLASSIFY_CONCURRENCY = 4;
+
+// Progress logs emitted every N processed candidates. A long legacy
+// file (50+ bullets) takes minutes; without periodic logging it
+// looks frozen.
+const PROGRESS_LOG_INTERVAL = 5;
+
+export async function migrateLegacyMemory(workspaceRoot: string, classify: MemoryClassifier): Promise<MigrationResult> {
+  const sourcePath = path.join(workspaceRoot, WORKSPACE_FILES.memory);
+  const raw = await readTextSafe(sourcePath);
+  if (raw === null) {
+    return emptyResult(true);
+  }
+
+  const candidates = parseCandidates(raw);
+  log.info("memory", "migration: classifying candidates", {
+    candidateCount: candidates.length,
+    concurrency: CLASSIFY_CONCURRENCY,
+  });
+  const classifications = await classifyInParallel(classify, candidates);
+
+  const result = emptyResult(false);
+  const usedSlugs = new Set<string>();
+
+  for (let index = 0; index < candidates.length; index += 1) {
+    const classification = classifications[index];
+    const candidate = candidates[index];
+    if (!classification) {
+      result.skippedByClassifier += 1;
+    } else {
+      const entry = buildEntry(candidate, classification, usedSlugs);
+      try {
+        await writeMemoryEntry(workspaceRoot, entry);
+        usedSlugs.add(entry.slug);
+        result.written[entry.type] += 1;
+      } catch (err) {
+        log.warn("memory", "migration: write failed", {
+          slug: entry.slug,
+          error: errorMessage(err),
+        });
+        result.writeErrors += 1;
+      }
+    }
+    const processed = index + 1;
+    if (processed % PROGRESS_LOG_INTERVAL === 0 || processed === candidates.length) {
+      log.info("memory", "migration: progress", {
+        processed,
+        total: candidates.length,
+        written: result.written,
+        skippedByClassifier: result.skippedByClassifier,
+        writeErrors: result.writeErrors,
+      });
+    }
+  }
+
+  await regenerateIndex(workspaceRoot);
+  await renameToBackup(sourcePath);
+  return result;
+}
+
+// Classify candidates with bounded concurrency. Workers pull
+// indices off a shared cursor and write into a pre-allocated array
+// so `result[i]` matches `candidates[i]`. Each await falls inside
+// `safeClassify`, which already swallows individual classifier
+// throws (logging once); a concurrent failure can therefore not
+// poison the rest of the batch.
+async function classifyInParallel(classify: MemoryClassifier, candidates: readonly MemoryCandidate[]): Promise<(MemoryClassification | null)[]> {
+  const results: (MemoryClassification | null)[] = new Array(candidates.length).fill(null);
+  let nextIndex = 0;
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const index = nextIndex;
+      nextIndex += 1;
+      if (index >= candidates.length) return;
+      results[index] = await safeClassify(classify, candidates[index]);
+    }
+  };
+  const workers = Array.from({ length: Math.min(CLASSIFY_CONCURRENCY, candidates.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+
+function emptyResult(noop: boolean): MigrationResult {
+  return {
+    noop,
+    written: { preference: 0, interest: 0, fact: 0, reference: 0 },
+    skippedByClassifier: 0,
+    writeErrors: 0,
+  };
+}
+
+// Split the legacy file into candidate bullets. We accept either
+// `- ` or `* ` as bullet markers and ignore continuation lines (a
+// candidate is one bullet line). The H2 header — when present —
+// rides along as `section` so the classifier can use it as context.
+// Parsing is character-driven (no regex) to keep sonar happy and to
+// make the bullet-vs-header classification obvious from the code.
+function parseCandidates(raw: string): MemoryCandidate[] {
+  const lines = raw.split("\n");
+  let section = "";
+  const out: MemoryCandidate[] = [];
+  for (const lineRaw of lines) {
+    const line = stripCarriageReturn(lineRaw);
+    const header = extractHeader(line);
+    if (header !== null) {
+      section = header;
+      continue;
+    }
+    const bullet = extractBullet(line);
+    if (bullet) out.push({ section, body: bullet });
+  }
+  return out;
+}
+
+function stripCarriageReturn(line: string): string {
+  return line.endsWith("\r") ? line.slice(0, -1) : line;
+}
+
+function extractHeader(line: string): string | null {
+  if (!line.startsWith("## ")) return null;
+  return line.slice(3).trim();
+}
+
+function extractBullet(line: string): string | null {
+  const trimmedLeft = line.replace(/^[\t ]+/, "");
+  if (!trimmedLeft.startsWith("- ") && !trimmedLeft.startsWith("* ")) return null;
+  const body = trimmedLeft.slice(2).trim();
+  return body.length > 0 ? body : null;
+}
+
+async function safeClassify(classify: MemoryClassifier, candidate: MemoryCandidate): Promise<MemoryClassification | null> {
+  try {
+    const verdict = await classify(candidate);
+    if (verdict && isMemoryType(verdict.type)) return verdict;
+    return null;
+  } catch (err) {
+    // Startup-wide failures (CLI missing) MUST escape so
+    // `runMemoryMigrationOnce` can log once + leave the legacy file
+    // for retry. Without this, every candidate would silently degrade
+    // to `null` and look like normal classifier disagreement
+    // (#1061 review).
+    if (err instanceof ClaudeCliNotFoundError) throw err;
+    log.warn("memory", "migration: classifier threw", {
+      preview: candidate.body.slice(0, 80),
+      error: errorMessage(err),
+    });
+    return null;
+  }
+}
+
+function buildEntry(candidate: MemoryCandidate, classification: MemoryClassification, usedSlugs: Set<string>): MemoryEntry {
+  const name = candidate.body.length > 80 ? `${candidate.body.slice(0, 77)}…` : candidate.body;
+  const description = classification.description?.trim() || truncateForDescription(candidate.body);
+  const baseSlug = slugifyMemoryName(candidate.body, classification.type);
+  const slug = uniqueSlug(baseSlug, usedSlugs);
+  return {
+    name,
+    description,
+    type: classification.type,
+    body: candidate.body,
+    slug,
+  };
+}
+
+function truncateForDescription(text: string): string {
+  const oneLine = text.replace(/\s+/g, " ").trim();
+  return oneLine.length > 120 ? `${oneLine.slice(0, 117)}…` : oneLine;
+}
+
+function uniqueSlug(base: string, used: Set<string>): string {
+  if (!used.has(base)) return base;
+  let counter = 2;
+  while (used.has(`${base}-${counter}`)) counter += 1;
+  return `${base}-${counter}`;
+}
+
+async function renameToBackup(sourcePath: string): Promise<void> {
+  const backupPath = `${sourcePath}.backup`;
+  try {
+    await rename(sourcePath, backupPath);
+  } catch (err) {
+    log.warn("memory", "migration: backup rename failed", {
+      sourcePath,
+      error: errorMessage(err),
+    });
+  }
+}
+
+// Bare write of the source file used in tests for setup. Production
+// callers don't need this — `memory.md` is created by user activity
+// or by an older mulmoclaude version.
+export async function writeLegacyMemoryForTest(workspaceRoot: string, content: string): Promise<void> {
+  const sourcePath = path.join(workspaceRoot, WORKSPACE_FILES.memory);
+  await mkdir(path.dirname(sourcePath), { recursive: true });
+  await writeFileAtomic(sourcePath, content);
+}

package/server/workspace/memory/run.ts

@@ -0,0 +1,84 @@
+// One-shot legacy-memory migration entry point used by server
+// startup (#1029 PR-B). Idempotent: returns immediately when there
+// is nothing to do (legacy file absent or already migrated).
+//
+// Concurrency: the agent may serve requests before this finishes.
+// The brief race window is documented in
+// `plans/done/feat-memory-storage-wire.md`.
+//
+// CLEANUP 2026-07-01: this is one-shot migration code for the
+// `memory.md` → atomic transition (#1029). After every active
+// workspace has run through both `runMemoryMigrationOnce` and the
+// follow-on `runTopicMigrationOnce`, this file plus the rest of
+// the migration chain (`migrate.ts`, `llm-classifier.ts`, the
+// atomic-aware branches in `prompt.ts`, the chain in
+// `server/index.ts`) can be deleted in one sweep. Workspaces that
+// have not migrated by then will need a manual conversion path —
+// document the recovery before removing the code.
+
+import { existsSync, statSync } from "node:fs";
+import path from "node:path";
+
+import { runClaudeCli, ClaudeCliNotFoundError, type Summarize } from "../journal/archivist-cli.js";
+import { makeLlmMemoryClassifier } from "./llm-classifier.js";
+import { migrateLegacyMemory } from "./migrate.js";
+import { errorMessage } from "../../utils/errors.js";
+import { log } from "../../system/logger/index.js";
+
+export interface RunMigrationDeps {
+  /** Override the summarize callback (useful for tests). Defaults to
+   *  the production `runClaudeCli` which spawns the Claude CLI. */
+  summarize?: Summarize;
+}
+
+export async function runMemoryMigrationOnce(workspaceRoot: string, deps: RunMigrationDeps = {}): Promise<void> {
+  const legacyPath = path.join(workspaceRoot, "conversations", "memory.md");
+  if (!existsSync(legacyPath)) {
+    log.debug("memory", "migration: no legacy file, skipping");
+    return;
+  }
+  // If the file is suspiciously empty (just the placeholder we used
+  // to write at init), skip — the migration would yield zero entries
+  // and the rename to `.backup` is pointless.
+  const stat = statSync(legacyPath);
+  if (stat.size < 64) {
+    log.info("memory", "migration: legacy file is below the placeholder threshold, skipping");
+    return;
+  }
+  // The rename to `.backup` is the final step of `migrateLegacyMemory`,
+  // so its presence is the "migration completed" marker. A workspace
+  // where both `memory.md` AND `memory.md.backup` exist means the
+  // user re-introduced the legacy file after a previous successful
+  // migration (probably by mistake or by extracting the backup).
+  // Re-running here would re-classify the bullets and could clobber
+  // typed entries the user has been editing in place; skip with a
+  // clear log instead. The interrupted-migration retry case still
+  // works because no `.backup` exists in that state — the rename
+  // never ran.
+  const backupPath = `${legacyPath}.backup`;
+  if (existsSync(backupPath)) {
+    log.info("memory", "migration: legacy file present but .backup also exists — refusing to re-run", {
+      legacyPath,
+      backupPath,
+    });
+    return;
+  }
+  const summarize = deps.summarize ?? runClaudeCli;
+  const classifier = makeLlmMemoryClassifier({ summarize });
+  log.info("memory", "migration: starting", { legacyPath });
+  try {
+    const result = await migrateLegacyMemory(workspaceRoot, classifier);
+    log.info("memory", "migration: done", {
+      noop: result.noop,
+      written: result.written,
+      skippedByClassifier: result.skippedByClassifier,
+      writeErrors: result.writeErrors,
+    });
+  } catch (err) {
+    if (err instanceof ClaudeCliNotFoundError) {
+      log.warn("memory", "migration: claude CLI not on PATH; legacy memory left in place");
+      return;
+    }
+    log.error("memory", "migration: threw, legacy memory left in place", { error: errorMessage(err) });
+  }
+}