mulmoclaude 0.5.3 → 0.6.0

package/server/workspace/memory/topic-migrate.ts

@@ -0,0 +1,248 @@
+// Atomic-to-topic migration (#1070 PR-A).
+//
+// Reads the existing #1029-style atomic entries from
+// `conversations/memory/`, runs a clusterer, and writes the
+// proposed topic layout to a STAGING dir
+// `conversations/memory.next/`. Does NOT swap. The user runs
+// `topic-swap.ts` after reviewing.
+//
+// Library only — `runTopicMigrationOnce` (in PR-B) decides when
+// to call this from server startup.
+//
+// CLEANUP 2026-07-01: see `topic-run.ts` — this file is part of
+// the one-shot atomic → topic migration chain and goes when the
+// chain goes.
+
+import { mkdir, rm } from "node:fs/promises";
+import path from "node:path";
+
+import { writeFileAtomic } from "../../utils/files/atomic.js";
+import { log } from "../../system/logger/index.js";
+import { errorMessage } from "../../utils/errors.js";
+import { WORKSPACE_DIRS } from "../paths.js";
+import { loadAllMemoryEntries } from "./io.js";
+import { MEMORY_TYPES, type MemoryEntry, type MemoryType } from "./types.js";
+import type { ClusterMap, ClusterTopic, MemoryClusterer } from "./topic-cluster.js";
+import { MAX_TOPIC_SLUG_LENGTH } from "./topic-types.js";
+
+export interface TopicMigrationResult {
+  /** Whether anything was emitted to the staging dir. */
+  noop: boolean;
+  /** Atomic entries that fed the cluster call. */
+  inputCount: number;
+  /** Topic files written to the staging dir, per type. */
+  topicCounts: Record<MemoryType, number>;
+  /** Bullets the clusterer omitted (sum across types). */
+  bulletsLost: number;
+  /** Where the staging dir lives. */
+  stagingPath: string;
+}
+
+export function topicStagingPath(workspaceRoot: string): string {
+  return path.join(workspaceRoot, WORKSPACE_DIRS.memoryStaging);
+}
+
+interface WrittenTopic {
+  type: MemoryType;
+  topic: ClusterTopic;
+  /** Slug actually written. May differ from `topic.topic` when a
+   *  collision was resolved with a `-2` / `-3` suffix. */
+  writtenSlug: string;
+}
+
+export async function clusterAtomicIntoStaging(workspaceRoot: string, clusterer: MemoryClusterer): Promise<TopicMigrationResult> {
+  const entries = await loadAllMemoryEntries(workspaceRoot);
+  const stagingPath = topicStagingPath(workspaceRoot);
+  if (entries.length === 0) {
+    return emptyResult(stagingPath);
+  }
+  log.info("memory", "topic-migrate: clustering", { entryCount: entries.length });
+  // Wipe stale staging BEFORE the cluster call. If the clusterer
+  // returns null or throws, we leave the workspace with no staging
+  // dir at all — that's the correct "migration didn't complete"
+  // signal. The earlier flow only cleared on success and could
+  // leave a stale tree in place after a failed run, which a later
+  // swap would happily promote.
+  await resetStaging(stagingPath);
+  let map: ClusterMap | null = null;
+  let clustererThrew = false;
+  try {
+    map = await clusterer(entries);
+  } catch (err) {
+    clustererThrew = true;
+    log.error("memory", "topic-migrate: clusterer threw", { error: errorMessage(err) });
+  }
+  if (!map) {
+    // Only warn about a graceful null return when the clusterer
+    // didn't already log a throw. Without this, a hard failure
+    // (claude CLI missing, schema mismatch, etc.) showed up as
+    // BOTH `clusterer threw` AND `clusterer returned null`, which
+    // misleads readers into thinking there were two distinct
+    // failure modes (#1072 review).
+    if (!clustererThrew) log.warn("memory", "topic-migrate: clusterer returned null");
+    await rm(stagingPath, { recursive: true, force: true });
+    return { ...emptyResult(stagingPath), inputCount: entries.length };
+  }
+
+  const result: TopicMigrationResult = {
+    noop: false,
+    inputCount: entries.length,
+    topicCounts: { preference: 0, interest: 0, fact: 0, reference: 0 },
+    bulletsLost: countBulletsLost(entries, map),
+    stagingPath,
+  };
+  const written: WrittenTopic[] = [];
+  for (const type of MEMORY_TYPES) {
+    const usedSlugs = new Set<string>();
+    for (const topic of map[type]) {
+      const writtenSlug = pickUniqueSlug(topic.topic, usedSlugs);
+      try {
+        await writeTopicFileToStaging(stagingPath, type, topic, writtenSlug);
+        usedSlugs.add(writtenSlug);
+        written.push({ type, topic, writtenSlug });
+        result.topicCounts[type] += 1;
+      } catch (err) {
+        log.warn("memory", "topic-migrate: write failed", { type, topic: topic.topic, writtenSlug, error: errorMessage(err) });
+      }
+    }
+  }
+  await writeStagingIndex(stagingPath, written);
+  log.info("memory", "topic-migrate: staging ready", {
+    stagingPath,
+    topicCounts: result.topicCounts,
+    bulletsLost: result.bulletsLost,
+  });
+  return result;
+}
+
+// Pick a slug that hasn't been used yet within this type. The
+// clusterer may return two topics that would normalise to the same
+// slug (e.g. "Music" and "music"); without this guard the second
+// would silently overwrite the first.
+//
+// The base slug is trimmed if needed so `base + "-N"` still fits the
+// `MAX_TOPIC_SLUG_LENGTH` cap that `isSafeTopicSlug` enforces. A
+// 60-char slug colliding with a prior write would otherwise produce a
+// 62-char filename that the writer rejects (and the reader would
+// then refuse to load on the next session). After trimming we strip
+// any trailing `-` so the suffix ("-N") is the only separator at the
+// boundary.
+function pickUniqueSlug(base: string, used: Set<string>): string {
+  if (!used.has(base)) return base;
+  let counter = 2;
+  while (true) {
+    const suffix = `-${counter}`;
+    const room = MAX_TOPIC_SLUG_LENGTH - suffix.length;
+    const trimmedBase = trimTrailingDash(base.slice(0, room));
+    const candidate = trimmedBase.length > 0 ? `${trimmedBase}${suffix}` : `topic${suffix}`;
+    if (!used.has(candidate)) return candidate;
+    counter += 1;
+  }
+}
+
+function trimTrailingDash(text: string): string {
+  let end = text.length;
+  while (end > 0 && text[end - 1] === "-") end -= 1;
+  return text.slice(0, end);
+}
+
+function emptyResult(stagingPath: string): TopicMigrationResult {
+  return {
+    noop: true,
+    inputCount: 0,
+    topicCounts: { preference: 0, interest: 0, fact: 0, reference: 0 },
+    bulletsLost: 0,
+    stagingPath,
+  };
+}
+
+async function resetStaging(stagingPath: string): Promise<void> {
+  // Stale staging from a prior run is wiped — the user's review
+  // signal is the diff, so we always emit a fresh tree.
+  await rm(stagingPath, { recursive: true, force: true });
+  await mkdir(stagingPath, { recursive: true });
+}
+
+async function writeTopicFileToStaging(stagingPath: string, type: MemoryType, topic: ClusterTopic, writtenSlug: string): Promise<void> {
+  const dir = path.join(stagingPath, type);
+  await mkdir(dir, { recursive: true });
+  const absPath = path.join(dir, `${writtenSlug}.md`);
+  const body = renderTopicBody(topic, writtenSlug);
+  const content = `---\ntype: ${type}\ntopic: ${writtenSlug}\n---\n\n${body}`;
+  await writeFileAtomic(absPath, content, { uniqueTmp: true });
+}
+
+function renderTopicBody(topic: ClusterTopic, writtenSlug: string): string {
+  const heading = humaniseTopic(writtenSlug);
+  const lines: string[] = [`# ${heading}`, ""];
+  if (topic.unsectionedBullets && topic.unsectionedBullets.length > 0) {
+    for (const bullet of topic.unsectionedBullets) {
+      lines.push(`- ${bullet}`);
+    }
+    lines.push("");
+  }
+  if (topic.sections) {
+    for (const section of topic.sections) {
+      lines.push(`## ${section.heading}`, "");
+      for (const bullet of section.bullets) {
+        lines.push(`- ${bullet}`);
+      }
+      lines.push("");
+    }
+  }
+  return lines.join("\n");
+}
+
+function humaniseTopic(slug: string): string {
+  // ASCII-friendly humaniser: split on `-`, capitalise each word.
+  // Non-ASCII slugs (which fall back to a hash) render as the slug
+  // itself; the user can rename later in the file explorer.
+  return slug
+    .split("-")
+    .map((part) => (part.length > 0 ? part[0].toUpperCase() + part.slice(1) : part))
+    .join(" ");
+}
+
+// Build the staging-side `MEMORY.md` from the topics we successfully
+// wrote to disk. Using the cluster map directly would link to files
+// that failed to write (e.g. one bad slug would leave a broken index
+// entry that swap promotes); this list is the disk-truth.
+async function writeStagingIndex(stagingPath: string, written: WrittenTopic[]): Promise<void> {
+  const lines: string[] = ["# Memory Index", ""];
+  for (const type of MEMORY_TYPES) {
+    const inType = written.filter((entry) => entry.type === type);
+    if (inType.length === 0) continue;
+    lines.push(`## ${type}`, "");
+    const sorted = [...inType].sort((left, right) => left.writtenSlug.localeCompare(right.writtenSlug));
+    for (const entry of sorted) {
+      lines.push(formatStagingIndexLine(entry));
+    }
+    lines.push("");
+  }
+  if (written.length === 0) {
+    lines.push("_(no entries yet)_", "");
+  }
+  await writeFileAtomic(path.join(stagingPath, "MEMORY.md"), lines.join("\n"), { uniqueTmp: true });
+}
+
+function formatStagingIndexLine(entry: WrittenTopic): string {
+  const link = `${entry.type}/${entry.writtenSlug}.md`;
+  const headings = (entry.topic.sections ?? []).map((section) => section.heading);
+  if (headings.length === 0) return `- ${link}`;
+  return `- ${link} — ${headings.join(", ")}`;
+}
+
+function countBulletsLost(entries: readonly MemoryEntry[], map: ClusterMap): number {
+  let placed = 0;
+  for (const type of MEMORY_TYPES) {
+    for (const topic of map[type]) {
+      if (topic.unsectionedBullets) placed += topic.unsectionedBullets.length;
+      if (topic.sections) {
+        for (const section of topic.sections) {
+          placed += section.bullets.length;
+        }
+      }
+    }
+  }
+  return Math.max(0, entries.length - placed);
+}

package/server/workspace/memory/topic-run.ts

@@ -0,0 +1,172 @@
+// One-shot topic-based migration entry point used by server startup
+// (#1070 PR-B). Mirrors `runMemoryMigrationOnce` from #1029 PR-B
+// but targets the topic-format restructure instead of the legacy
+// memory.md flow.
+//
+// Idempotent: returns immediately when there is nothing to do —
+// the workspace is already topic-format, there are no atomic
+// entries to migrate, or the legacy `memory.md` is still in
+// flight. When staging is already present from a prior crash mid-
+// swap, this runner retries the swap rather than burning another
+// LLM cluster call. Failures are logged and swallowed so the
+// server can continue serving traffic.
+//
+// Concurrency: cluster runs in the background while the agent
+// continues serving requests. Atomic-format reads / writes stay in
+// effect right up until the swap completes; the next request after
+// the swap sees the new topic layout.
+//
+// CLEANUP 2026-07-01: this is one-shot migration code for the
+// atomic → topic transition (#1070). After every active workspace
+// has been swapped to the topic format, this file plus
+// `topic-migrate.ts`, `topic-cluster.ts`, `topic-swap.ts`, the CLI
+// helper at `scripts/memory-swap-topic-staging.ts`, the
+// `yarn memory:swap` script, and the migration call in
+// `server/index.ts` can be deleted in one sweep. Topic-format
+// reading / writing (`topic-types.ts`, `topic-io.ts`,
+// `topic-detect.ts` — minus the atomic-format branch) stays.
+
+import { existsSync, statSync } from "node:fs";
+import path from "node:path";
+
+import { runClaudeCli, ClaudeCliNotFoundError, type Summarize } from "../journal/archivist-cli.js";
+import { WORKSPACE_DIRS, WORKSPACE_FILES } from "../paths.js";
+import { loadAllMemoryEntries } from "./io.js";
+import { makeLlmMemoryClusterer } from "./topic-cluster.js";
+import { clusterAtomicIntoStaging, topicStagingPath } from "./topic-migrate.js";
+import { swapStagingIntoMemory } from "./topic-swap.js";
+import { MEMORY_TYPES } from "./types.js";
+import { errorMessage } from "../../utils/errors.js";
+import { log } from "../../system/logger/index.js";
+
+export interface RunTopicMigrationDeps {
+  /** Override the summarize callback (useful for tests). Defaults to
+   *  the production `runClaudeCli` which spawns the Claude CLI. */
+  summarize?: Summarize;
+}
+
+// Strict variant of `hasTopicFormat` that only looks at `memory/`,
+// not `memory.next/`. The shared `hasTopicFormat` is intentionally
+// swap-tolerant so prompt routing doesn't fall back to atomic-format
+// rules during the rename window — but the runner's idempotency
+// guard needs the OPPOSITE: when only `memory.next/` exists (a
+// swap-in-progress or a crash mid-swap), the runner must drop into
+// the "existing staging detected" retry-swap branch below, not exit.
+function memoryTreeIsTopicFormat(workspaceRoot: string): boolean {
+  const memoryRoot = path.join(workspaceRoot, WORKSPACE_DIRS.memoryDir);
+  for (const type of MEMORY_TYPES) {
+    try {
+      if (statSync(path.join(memoryRoot, type)).isDirectory()) return true;
+    } catch {
+      // ENOENT / EACCES → keep looking; only the actual presence of
+      // a `memory/<type>` dir signals migration completed.
+    }
+  }
+  return false;
+}
+
+export async function runTopicMigrationOnce(workspaceRoot: string, deps: RunTopicMigrationDeps = {}): Promise<void> {
+  if (memoryTreeIsTopicFormat(workspaceRoot)) {
+    log.debug("memory", "topic-run: workspace already uses topic format, skipping");
+    return;
+  }
+  const stagingPath = topicStagingPath(workspaceRoot);
+  // If staging is left over from a prior run that crashed between
+  // cluster and swap, just retry the swap. Re-clustering would burn
+  // another LLM call and (because clusterAtomicIntoStaging wipes
+  // staging up front) discard the prior cluster result.
+  if (existsSync(stagingPath)) {
+    log.info("memory", "topic-run: existing staging detected, retrying swap", { stagingPath });
+    await runSwap(workspaceRoot);
+    return;
+  }
+  // Don't trip over an in-progress legacy `memory.md` migration from
+  // #1029 PR-B. We mirror the conditions under which
+  // `runMemoryMigrationOnce` would actually run — legacy file
+  // present, past the placeholder threshold, AND `.backup` absent.
+  // The `.backup` check is load-bearing: when both `memory.md` and
+  // `.backup` exist, the legacy runner refuses to re-process (the
+  // backup signals "already done; user re-introduced the file"),
+  // and without this clause the topic runner would defer
+  // indefinitely waiting for a migration that's never going to
+  // happen.
+  //
+  // One guarded `statSync` (no `existsSync` first): the legacy
+  // migration runs in parallel and can rename / delete `memory.md`
+  // between an `existsSync` check and a follow-up `statSync`,
+  // turning the race into an unhandled rejection because this whole
+  // function is invoked as a floating promise on startup. ENOENT
+  // means the legacy file isn't there (or just got renamed away),
+  // so there's nothing to defer for; any other error is swallowed
+  // and the runner proceeds — a permission glitch should never block
+  // the topic restructure.
+  const legacyPath = path.join(workspaceRoot, WORKSPACE_FILES.memory);
+  let legacyStat: ReturnType<typeof statSync> | null;
+  try {
+    legacyStat = statSync(legacyPath);
+  } catch {
+    legacyStat = null;
+  }
+  if (legacyStat && legacyStat.size >= 64) {
+    const backupPath = `${legacyPath}.backup`;
+    if (!existsSync(backupPath)) {
+      log.debug("memory", "topic-run: legacy memory.md still in flight, deferring", { legacyPath });
+      return;
+    }
+  }
+  const entries = await loadAllMemoryEntries(workspaceRoot);
+  if (entries.length === 0) {
+    log.debug("memory", "topic-run: no atomic entries to migrate, skipping");
+    return;
+  }
+  const summarize = deps.summarize ?? runClaudeCli;
+  const clusterer = makeLlmMemoryClusterer({ summarize });
+  log.info("memory", "topic-run: starting", { entryCount: entries.length });
+  try {
+    const result = await clusterAtomicIntoStaging(workspaceRoot, clusterer);
+    if (result.noop) {
+      // `clusterAtomicIntoStaging` already logged the failure cause
+      // (`clusterer threw` or `clusterer returned null`) and rm'd
+      // the staging dir. Logging "staged" here would tell the user
+      // to `diff` a directory that no longer exists (#1076 review).
+      log.warn("memory", "topic-run: cluster did not produce staging — see prior log entry");
+      return;
+    }
+    log.info("memory", "topic-run: staged", {
+      stagingPath: result.stagingPath,
+      topicCounts: result.topicCounts,
+      bulletsLost: result.bulletsLost,
+    });
+    await runSwap(workspaceRoot);
+  } catch (err) {
+    // Defensive: `makeLlmMemoryClusterer` swallows summarize errors
+    // and returns null today, and `clusterAtomicIntoStaging` doesn't
+    // re-throw, so this branch is currently unreachable. Kept so a
+    // future change in the clusterer error contract surfaces a
+    // visible log instead of an unhandled rejection (the runner is
+    // invoked as a floating promise on startup).
+    if (err instanceof ClaudeCliNotFoundError) {
+      log.warn("memory", "topic-run: claude CLI not on PATH; topic restructure deferred");
+      return;
+    }
+    log.error("memory", "topic-run: cluster threw", { error: errorMessage(err) });
+  }
+}
+
+// Swap staging into the live memory dir. The atomic format is
+// parked under `memory/.atomic-backup/<ts>/` so misclassified
+// migrations can be rolled back by hand without losing data.
+// Failures leave staging in place; the next server start hits the
+// "existing staging detected" branch above and retries.
+async function runSwap(workspaceRoot: string): Promise<void> {
+  const result = await swapStagingIntoMemory(workspaceRoot);
+  if (result.swapped) {
+    log.info("memory", "topic-run: swap complete — workspace now uses topic format", {
+      backupPath: result.backupPath,
+    });
+  } else {
+    log.warn("memory", "topic-run: swap did not complete, leaving staging in place for retry", {
+      reason: result.reason ?? "unknown",
+    });
+  }
+}

package/server/workspace/memory/topic-swap.ts

@@ -0,0 +1,135 @@
+// Swap `conversations/memory/` ↔ `conversations/memory.next/`
+// after the user has approved the staging diff (#1070 PR-A).
+//
+// Library only — invoked from a CLI helper or the agent (a tool
+// surface for the agent will land later). The swap is intentionally
+// NOT auto-run: the whole point of staging is to give the user a
+// chance to inspect.
+//
+// CLEANUP 2026-07-01: see `topic-run.ts` — this file is part of
+// the one-shot atomic → topic migration chain and goes when the
+// chain goes.
+//
+// Swap mechanics:
+//   memory/             →  memory/.atomic-backup-<ts>/
+//   memory.next/        →  memory/
+//
+// The backup name carries a timestamp so re-runs (after a follow-up
+// migration on a richer workspace) don't clobber prior backups.
+
+import { mkdir, rename, stat } from "node:fs/promises";
+import path from "node:path";
+
+import { log } from "../../system/logger/index.js";
+import { errorMessage } from "../../utils/errors.js";
+import { WORKSPACE_DIRS } from "../paths.js";
+import { topicStagingPath } from "./topic-migrate.js";
+
+export interface SwapResult {
+  /** True when a swap actually happened. */
+  swapped: boolean;
+  /** Where the prior atomic layout was moved. Null if no prior data. */
+  backupPath: string | null;
+  /** Reason when `swapped: false`. */
+  reason?: string;
+}
+
+export async function swapStagingIntoMemory(workspaceRoot: string): Promise<SwapResult> {
+  const stagingPath = topicStagingPath(workspaceRoot);
+  const memoryPath = path.join(workspaceRoot, WORKSPACE_DIRS.memoryDir);
+  const stagingExists = await pathExists(stagingPath);
+  if (!stagingExists) {
+    return { swapped: false, backupPath: null, reason: "staging dir not found" };
+  }
+
+  let backupPath: string | null = null;
+  if (await pathExists(memoryPath)) {
+    backupPath = await pickBackupPath(memoryPath);
+    try {
+      await rename(memoryPath, backupPath);
+    } catch (err) {
+      log.error("memory", "topic-swap: backup rename failed", { from: memoryPath, to: backupPath, error: errorMessage(err) });
+      return { swapped: false, backupPath: null, reason: "backup rename failed" };
+    }
+  }
+
+  try {
+    await rename(stagingPath, memoryPath);
+  } catch (err) {
+    log.error("memory", "topic-swap: staging rename failed", { from: stagingPath, to: memoryPath, error: errorMessage(err) });
+    // Try to put the backup back so the workspace isn't left empty.
+    let rollbackFailed = false;
+    if (backupPath) {
+      try {
+        await rename(backupPath, memoryPath);
+      } catch (rollbackErr) {
+        rollbackFailed = true;
+        log.error("memory", "topic-swap: rollback failed; manual intervention needed", {
+          backupPath,
+          memoryPath,
+          error: errorMessage(rollbackErr),
+        });
+      }
+    }
+    // When rollback ALSO fails, the prior data still lives at
+    // `backupPath`. Returning `backupPath: null` would tell callers
+    // "no recovery point exists" and they'd give up — surface the
+    // path so a human (or a retry loop) can move it back manually
+    // (#1072 review).
+    return { swapped: false, backupPath: rollbackFailed ? backupPath : null, reason: "staging rename failed" };
+  }
+
+  // Park the backup INSIDE the new memory dir so it travels with
+  // the workspace. A flat sibling backup (`memory.atomic-backup`)
+  // is also fine but clutters `conversations/`.
+  if (backupPath) {
+    const inside = path.join(memoryPath, ".atomic-backup");
+    await mkdir(inside, { recursive: true });
+    const finalLocation = path.join(inside, path.basename(backupPath));
+    try {
+      await rename(backupPath, finalLocation);
+      backupPath = finalLocation;
+    } catch (err) {
+      log.warn("memory", "topic-swap: failed to park backup inside memory/, leaving at sibling location", {
+        backupPath,
+        error: errorMessage(err),
+      });
+    }
+  }
+
+  log.info("memory", "topic-swap: done", { backupPath, memoryPath });
+  return { swapped: true, backupPath };
+}
+
+async function pathExists(target: string): Promise<boolean> {
+  try {
+    await stat(target);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// Builds an unused backup path. We use a coarse timestamp suffix so
+// re-runs sort chronologically and don't collide.
+async function pickBackupPath(memoryPath: string): Promise<string> {
+  const parent = path.dirname(memoryPath);
+  const stamp = formatTimestamp(new Date());
+  const base = `memory.atomic-backup-${stamp}`;
+  let candidate = path.join(parent, base);
+  let counter = 2;
+  while (await pathExists(candidate)) {
+    candidate = path.join(parent, `${base}-${counter}`);
+    counter += 1;
+  }
+  return candidate;
+}
+
+function formatTimestamp(date: Date): string {
+  const year = date.getUTCFullYear();
+  const month = String(date.getUTCMonth() + 1).padStart(2, "0");
+  const day = String(date.getUTCDate()).padStart(2, "0");
+  const hour = String(date.getUTCHours()).padStart(2, "0");
+  const minute = String(date.getUTCMinutes()).padStart(2, "0");
+  return `${year}${month}${day}-${hour}${minute}`;
+}