mulmoclaude 0.1.0

package/server/workspace/chat-index/indexer.ts

@@ -0,0 +1,209 @@
+// Per-session indexing logic. `indexSession` summarizes a single
+// session jsonl and writes both a per-session file and a manifest
+// upsert to workspace/chat/index/. `readManifest` is a tiny helper
+// the sessions route uses to join entries into its /api/sessions
+// response.
+//
+// All functions take an explicit `workspaceRoot` so tests can point
+// at a `mkdtempSync` directory without touching the real
+// ~/mulmoclaude.
+
+import { readdir, readFile } from "node:fs/promises";
+import { defaultSummarize, loadJsonlInput, type SummarizeFn } from "./summarizer.js";
+import { chatDirFor, indexEntryPathFor, manifestPathFor, sessionJsonlPathFor, sessionMetaPathFor } from "./paths.js";
+import type { ChatIndexEntry, ChatIndexManifest } from "./types.js";
+import { writeJsonAtomic } from "../../utils/files/index.js";
+import { DEFAULT_ROLE_ID } from "../../../src/config/roles.js";
+import { ONE_MINUTE_MS } from "../../utils/time.js";
+import { isRecord } from "../../utils/types.js";
+
+// Freshness throttle: a session whose existing index entry is
+// newer than this is skipped. The 15-minute window is a compromise
+// — long enough that a single conversation doesn't re-summarize
+// every turn, short enough that a user who leaves for lunch and
+// comes back sees the title refresh.
+export const MIN_INDEX_INTERVAL_MS = 15 * ONE_MINUTE_MS;
+
+// Injection points for tests. Defaults are the production spawn +
+// wall-clock.
+export interface IndexerDeps {
+  summarize?: SummarizeFn;
+  now?: () => number;
+  minIntervalMs?: number;
+  // Bypass the `isFresh` freshness throttle. Used by the
+  // backfill helper and the debug trigger endpoint so a manual
+  // "rebuild everything" run doesn't silently skip entries that
+  // happen to be within the 15-minute window.
+  force?: boolean;
+}
+
+// --- manifest I/O ---------------------------------------------------
+
+export async function readManifest(workspaceRoot: string): Promise<ChatIndexManifest> {
+  try {
+    const raw = await readFile(manifestPathFor(workspaceRoot), "utf-8");
+    const parsed: unknown = JSON.parse(raw);
+    if (isManifest(parsed)) return parsed;
+    return { version: 1, entries: [] };
+  } catch {
+    return { version: 1, entries: [] };
+  }
+}
+
+function isManifest(raw: unknown): raw is ChatIndexManifest {
+  if (!isRecord(raw)) return false;
+  const o = raw as Record<string, unknown>;
+  return o.version === 1 && Array.isArray(o.entries);
+}
+
+// In-process mutex serializing the read-modify-write sequence on
+// the shared manifest file. Two concurrent `indexSession` calls
+// for different session ids would otherwise both read an empty
+// manifest, each append their own entry, and the last writer would
+// clobber the first. Chain-based mutex keeps it simple and fits
+// this module's single-process assumption.
+let manifestMutex: Promise<void> = Promise.resolve();
+
+async function withManifestLock<T>(fn: () => Promise<T>): Promise<T> {
+  const prev = manifestMutex;
+  let release: () => void = () => {};
+  manifestMutex = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  try {
+    await prev;
+    return await fn();
+  } finally {
+    release();
+  }
+}
+
+// Atomic write: stage to a per-call unique tmp file and rename.
+// The unique suffix is belt-and-suspenders — the mutex above
+// already serializes callers within this process, but a unique
+// name means the rename can't collide even if a stray .tmp file
+// is left behind by a previous crashed run.
+async function writeManifestAtomic(workspaceRoot: string, m: ChatIndexManifest): Promise<void> {
+  // `uniqueTmp` belt-and-suspenders: the in-process mutex above
+  // already serializes callers, but a unique tmp name means the
+  // rename can't collide even if a stray .tmp file is left behind
+  // by a previous crashed run.
+  await writeJsonAtomic(manifestPathFor(workspaceRoot), m, {
+    uniqueTmp: true,
+  });
+}
+
+// Read, mutate, and write the manifest under the in-process lock
+// so concurrent callers cannot lose each other's updates.
+export async function updateManifest(workspaceRoot: string, mutator: (m: ChatIndexManifest) => ChatIndexManifest): Promise<ChatIndexManifest> {
+  return withManifestLock(async () => {
+    const current = await readManifest(workspaceRoot);
+    const next = mutator(current);
+    await writeManifestAtomic(workspaceRoot, next);
+    return next;
+  });
+}
+
+// --- freshness check ------------------------------------------------
+
+// A session is "fresh" when its per-session index file exists and
+// was written less than `minIntervalMs` ago. Fresh sessions are
+// skipped so a long conversation doesn't spam the CLI on every
+// turn.
+export async function isFresh(workspaceRoot: string, sessionId: string, now: number, minIntervalMs: number): Promise<boolean> {
+  try {
+    const raw = await readFile(indexEntryPathFor(workspaceRoot, sessionId), "utf-8");
+    const entry: unknown = JSON.parse(raw);
+    if (!isRecord(entry)) return false;
+    const indexedAt = (entry as Record<string, unknown>).indexedAt;
+    if (typeof indexedAt !== "string") return false;
+    const ts = Date.parse(indexedAt);
+    if (Number.isNaN(ts)) return false;
+    return now - ts < minIntervalMs;
+  } catch {
+    return false;
+  }
+}
+
+// --- session metadata ----------------------------------------------
+
+interface SessionMeta {
+  roleId?: string;
+  startedAt?: string;
+}
+
+async function readSessionMeta(workspaceRoot: string, sessionId: string): Promise<SessionMeta> {
+  try {
+    const raw = await readFile(sessionMetaPathFor(workspaceRoot, sessionId), "utf-8");
+    const parsed: unknown = JSON.parse(raw);
+    if (!isRecord(parsed)) return {};
+    const o = parsed as Record<string, unknown>;
+    return {
+      roleId: typeof o.roleId === "string" ? o.roleId : undefined,
+      startedAt: typeof o.startedAt === "string" ? o.startedAt : undefined,
+    };
+  } catch {
+    return {};
+  }
+}
+
+// List every session id that has a .jsonl file in the workspace
+// chat dir. Used by the backfill helper.
+export async function listSessionIds(workspaceRoot: string): Promise<string[]> {
+  try {
+    const files = await readdir(chatDirFor(workspaceRoot));
+    return files.filter((f) => f.endsWith(".jsonl")).map((f) => f.slice(0, -".jsonl".length));
+  } catch {
+    return [];
+  }
+}
+
+// --- the core indexSession call ------------------------------------
+
+// Index (or re-index) a single session. Returns the entry on
+// success, or null if the session was skipped (fresh, empty,
+// missing). The only exception that escapes is
+// `ClaudeCliNotFoundError` — the caller uses it to disable the
+// module for the rest of the process lifetime.
+export async function indexSession(workspaceRoot: string, sessionId: string, deps: IndexerDeps = {}): Promise<ChatIndexEntry | null> {
+  const summarize = deps.summarize ?? defaultSummarize;
+  const now = (deps.now ?? Date.now)();
+  const minInterval = deps.minIntervalMs ?? MIN_INDEX_INTERVAL_MS;
+  const force = deps.force === true;
+
+  if (!force && (await isFresh(workspaceRoot, sessionId, now, minInterval))) {
+    return null;
+  }
+
+  const input = await loadJsonlInput(sessionJsonlPathFor(workspaceRoot, sessionId));
+  if (!input.trim()) return null;
+
+  const summary = await summarize(input);
+  const meta = await readSessionMeta(workspaceRoot, sessionId);
+
+  const entry: ChatIndexEntry = {
+    id: sessionId,
+    roleId: meta.roleId ?? DEFAULT_ROLE_ID,
+    startedAt: meta.startedAt ?? new Date(now).toISOString(),
+    indexedAt: new Date(now).toISOString(),
+    title: summary.title,
+    summary: summary.summary,
+    keywords: summary.keywords,
+  };
+
+  // Per-session file is written first so partial progress survives
+  // a crash between the two writes: the next run can still observe
+  // the fresh entry via isFresh and skip it.
+  await writeJsonAtomic(indexEntryPathFor(workspaceRoot, sessionId), entry);
+
+  // Upsert into manifest under the in-process lock: replace any
+  // prior entry with the same id, sort newest-first by startedAt.
+  await updateManifest(workspaceRoot, (current) => {
+    const filtered = current.entries.filter((e) => e.id !== sessionId);
+    filtered.push(entry);
+    filtered.sort((a, b) => Date.parse(b.startedAt) - Date.parse(a.startedAt));
+    return { version: 1, entries: filtered };
+  });
+
+  return entry;
+}

package/server/workspace/chat-index/paths.ts

@@ -0,0 +1,34 @@
+// Pure path helpers for the chat index cache. Kept in their own
+// file so tests can compute expected paths without needing the
+// summarizer / indexer modules (which transitively pull in the
+// claude CLI spawn code).
+
+import path from "node:path";
+
+export const CHAT_DIR = "chat";
+export const INDEX_DIR = "index";
+export const MANIFEST_FILE = "manifest.json";
+
+export function chatDirFor(workspaceRoot: string): string {
+  return path.join(workspaceRoot, CHAT_DIR);
+}
+
+export function indexDirFor(workspaceRoot: string): string {
+  return path.join(chatDirFor(workspaceRoot), INDEX_DIR);
+}
+
+export function sessionJsonlPathFor(workspaceRoot: string, sessionId: string): string {
+  return path.join(chatDirFor(workspaceRoot), `${sessionId}.jsonl`);
+}
+
+export function sessionMetaPathFor(workspaceRoot: string, sessionId: string): string {
+  return path.join(chatDirFor(workspaceRoot), `${sessionId}.json`);
+}
+
+export function indexEntryPathFor(workspaceRoot: string, sessionId: string): string {
+  return path.join(indexDirFor(workspaceRoot), `${sessionId}.json`);
+}
+
+export function manifestPathFor(workspaceRoot: string): string {
+  return path.join(indexDirFor(workspaceRoot), MANIFEST_FILE);
+}

package/server/workspace/chat-index/summarizer.ts

@@ -0,0 +1,247 @@
+// Summarizes a single session jsonl into a title / summary /
+// keywords triple using the Claude Code CLI. Cherry-picked and
+// trimmed from the closed PR #94.
+//
+// Splits cleanly into three layers so tests can exercise the pure
+// bits without spawning the CLI:
+//
+//   extractText / truncate         — jsonl → prompt input
+//   parseClaudeJsonResult          — CLI stdout → SummaryResult
+//   validateSummaryResult          — unknown → SummaryResult
+//
+// `defaultSummarize` composes them with the real spawn; tests
+// inject their own SummarizeFn via `IndexerDeps.summarize`.
+
+import { spawn } from "node:child_process";
+import { EVENT_TYPES } from "../../../src/types/events.js";
+import { readFile } from "node:fs/promises";
+import { formatSpawnFailure } from "../../utils/spawn.js";
+import { tmpdir } from "node:os";
+import { ClaudeCliNotFoundError } from "../journal/archivist.js";
+import { errorMessage } from "../../utils/errors.js";
+import type { SummaryResult } from "./types.js";
+import { ONE_MINUTE_MS } from "../../utils/time.js";
+import { isRecord } from "../../utils/types.js";
+
+const SYSTEM_PROMPT =
+  "You summarize a single chat session. Output strict JSON matching the provided schema. " +
+  "Rules: title <= 60 characters in the source language, summary <= 200 characters in the same language, " +
+  "5 to 10 short lowercase keywords useful for search. Respond with structured output only.";
+
+const SUMMARY_SCHEMA = {
+  type: "object",
+  properties: {
+    title: { type: "string" },
+    summary: { type: "string" },
+    keywords: { type: "array", items: { type: "string" } },
+  },
+  required: ["title", "summary", "keywords"],
+};
+
+// Prompt-building constants.
+const MAX_INPUT_CHARS = 8000;
+const HEAD_CHARS = 3000;
+const TAIL_CHARS = 5000;
+const PER_MESSAGE_MAX = 500;
+
+// Spawn / budget constants.
+const DEFAULT_TIMEOUT_MS = 2 * ONE_MINUTE_MS;
+// Budget cap per summarization call, forwarded to `claude
+// --max-budget-usd`. Previously 0.05 but that was tight enough
+// that a first-burst call — which pays a one-time cache creation
+// cost on haiku (~28k cache-creation tokens) — would trip the cap
+// and fail with `error_max_budget_usd` even for tiny 600-char
+// transcripts. 0.15 leaves comfortable headroom for cache
+// creation + a generous output allowance while still capping a
+// full 100-session backfill to well under $20.
+const MAX_BUDGET_USD = 0.15;
+
+// Any module that wants to drive the summarizer — including the
+// indexer — takes a SummarizeFn so tests can supply a deterministic
+// fake. Production path is `defaultSummarize` below.
+export type SummarizeFn = (input: string) => Promise<SummaryResult>;
+
+interface JsonlEntry {
+  source?: string;
+  type?: string;
+  message?: string;
+}
+
+function trimMessage(text: string): string {
+  if (text.length <= PER_MESSAGE_MAX) return text;
+  return `${text.slice(0, PER_MESSAGE_MAX)}…`;
+}
+
+// Walk a session jsonl and keep only the user / assistant text
+// turns, joined into a compact transcript. Tool results are
+// skipped because they are noisy and rarely contribute to a useful
+// summary title.
+export function extractText(jsonlContent: string): string {
+  const lines = jsonlContent.split("\n").filter(Boolean);
+  const parts: string[] = [];
+  for (const line of lines) {
+    let entry: JsonlEntry;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const source = entry.source;
+    if ((source === "user" || source === "assistant") && entry.type === EVENT_TYPES.text && typeof entry.message === "string") {
+      parts.push(`[${source}] ${trimMessage(entry.message)}`);
+    }
+  }
+  return parts.join("\n\n");
+}
+
+// Long sessions are truncated to first ~3000 + last ~5000 chars so
+// claude sees both the original topic and the most recent state.
+export function truncate(text: string): string {
+  if (text.length <= MAX_INPUT_CHARS) return text;
+  const head = text.slice(0, HEAD_CHARS);
+  const tail = text.slice(-TAIL_CHARS);
+  return `${head}\n\n…\n\n${tail}`;
+}
+
+interface ClaudeJsonResult {
+  type?: string;
+  is_error?: boolean;
+  structured_output?: unknown;
+  result?: string;
+}
+
+// Parse the JSON envelope that `claude --output-format json`
+// prints, raising a useful error if the envelope is malformed or
+// the CLI reported an error.
+export function parseClaudeJsonResult(stdout: string): SummaryResult {
+  let parsed: ClaudeJsonResult;
+  try {
+    parsed = JSON.parse(stdout.trim());
+  } catch (err) {
+    throw new Error(`[chat-index] failed to parse claude json output: ${errorMessage(err)}`);
+  }
+  if (parsed.is_error) {
+    throw new Error(`[chat-index] claude returned error: ${parsed.result ?? "unknown"}`);
+  }
+  return validateSummaryResult(parsed.structured_output);
+}
+
+// Build the error message for a non-zero `claude` CLI exit.
+//
+// The claude CLI writes its structured result — including error
+// envelopes like `{"is_error":true,"subtype":"error_max_budget_usd",
+// "errors":["Reached maximum budget ($0.05)"]}` — to **stdout**,
+// not stderr. Our previous handler only inspected stderr, so
+// budget-exhaustion and similar failures surfaced as
+// `claude summarize exited 1:` with no details at all, making
+// them impossible to diagnose from the log.
+//
+// Strategy: try to parse stdout as a claude JSON envelope first
+// and extract a human-readable reason from `errors[]` /
+// `subtype` / `result`; fall back to stderr, then to a raw
+// stdout slice, then to a generic "no error output".
+export function formatSpawnError(code: number | null, stdout: string, stderr: string): string {
+  return formatSpawnFailure("[chat-index]", code, stdout, stderr);
+}
+
+// Runtime-validate an arbitrary value into a SummaryResult. Missing
+// or wrong-typed fields fall back to safe defaults rather than
+// crashing the indexer — a degraded title is better than a dropped
+// session.
+export function validateSummaryResult(obj: unknown): SummaryResult {
+  if (!isRecord(obj)) {
+    throw new Error("[chat-index] summary result is not an object");
+  }
+  const o = obj as Record<string, unknown>;
+  const title = typeof o.title === "string" ? o.title : "";
+  const summary = typeof o.summary === "string" ? o.summary : "";
+  const keywords = Array.isArray(o.keywords) ? o.keywords.filter((k): k is string => typeof k === "string") : [];
+  return { title, summary, keywords };
+}
+
+// Read a jsonl file and produce the pre-truncated transcript that
+// goes into the CLI prompt. Returns the empty string for an empty
+// or unreadable file so the caller can decide whether to skip.
+export async function loadJsonlInput(jsonlPath: string): Promise<string> {
+  try {
+    const content = await readFile(jsonlPath, "utf-8");
+    return truncate(extractText(content));
+  } catch {
+    return "";
+  }
+}
+
+// --- spawn layer ----------------------------------------------------
+
+function spawnClaudeSummarize(input: string, timeoutMs: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const args = [
+      "--print",
+      "--no-session-persistence",
+      "--output-format",
+      "json",
+      "--model",
+      "haiku",
+      "--max-budget-usd",
+      String(MAX_BUDGET_USD),
+      "--json-schema",
+      JSON.stringify(SUMMARY_SCHEMA),
+      "--system-prompt",
+      SYSTEM_PROMPT,
+      "-p",
+      input,
+    ];
+    // Run from tmpdir so claude does not load the project's
+    // CLAUDE.md / plugins / memory and inflate the context.
+    const proc = spawn("claude", args, {
+      cwd: tmpdir(),
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      proc.kill("SIGKILL");
+      reject(new Error(`[chat-index] claude summarize timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    proc.stdout.on("data", (chunk: Buffer) => {
+      stdout += chunk.toString();
+    });
+    proc.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+    proc.on("error", (err: Error & { code?: string }) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (err.code === "ENOENT") {
+        reject(new ClaudeCliNotFoundError());
+      } else {
+        reject(err);
+      }
+    });
+    proc.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (code !== 0) {
+        reject(new Error(formatSpawnError(code, stdout, stderr)));
+        return;
+      }
+      resolve(stdout);
+    });
+  });
+}
+
+// Production SummarizeFn: prepare the input from a jsonl path and
+// drive the CLI. Tests inject their own SummarizeFn that bypasses
+// the CLI entirely.
+export const defaultSummarize: SummarizeFn = async (input: string) => {
+  const stdout = await spawnClaudeSummarize(input, DEFAULT_TIMEOUT_MS);
+  return parseClaudeJsonResult(stdout);
+};

package/server/workspace/chat-index/types.ts

@@ -0,0 +1,38 @@
+// On-disk shapes for the per-session chat summaries cached under
+// workspace/chat/index/. These power the title + summary shown for
+// past sessions in the sidebar history pane. The full design lives
+// in plans/done/feat-session-index-titles.md.
+
+export interface SummaryResult {
+  // <= 60 chars in the source language
+  title: string;
+  // <= 200 chars in the source language
+  summary: string;
+  // 5-10 short lowercase keywords
+  keywords: string[];
+}
+
+// One cached summary per session. Written to chat/index/<id>.json
+// and also mirrored into manifest.json for bulk-read from the
+// /api/sessions route.
+export interface ChatIndexEntry {
+  id: string;
+  roleId: string;
+  startedAt: string;
+  // ISO timestamp of when this summary was produced. Used by the
+  // freshness throttle — we skip re-summarizing a session whose
+  // existing entry is less than MIN_INDEX_INTERVAL_MS old, so a
+  // 20-turn conversation over 30 min summarizes ~twice, not 20
+  // times. See `isFresh` in indexer.ts.
+  indexedAt: string;
+  title: string;
+  summary: string;
+  keywords: string[];
+}
+
+export interface ChatIndexManifest {
+  version: 1;
+  // Sorted newest-first by startedAt so the sidebar gets them in
+  // display order without a second sort pass.
+  entries: ChatIndexEntry[];
+}