@vellumai/assistant 0.8.6 → 0.8.7

package/src/cli/commands/db/__tests__/status.test.ts

@@ -0,0 +1,253 @@
+/**
+ * Tests for `assistant db status`.
+ *
+ * Uses a real bun:sqlite database in a tmp dir rather than mocking, because
+ * the whole point of the command is the SQL it issues (page-count pragma,
+ * `memory_checkpoints` lookup, COUNT(*) fallback). Mocks would just
+ * re-state the implementation.
+ *
+ * The command's only external dependency is `getDbPath()`, which derives
+ * from `VELLUM_WORKSPACE_DIR` — we set that env var per test and let the
+ * real platform helper produce the path. No module-level mocks needed.
+ */
+
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+// ---------------------------------------------------------------------------
+// Test workspace setup
+// ---------------------------------------------------------------------------
+
+let workspaceDir: string;
+let dbPath: string;
+let originalWorkspaceEnv: string | undefined;
+
+beforeEach(() => {
+  workspaceDir = mkdtempSync(join(tmpdir(), "db-status-test-"));
+  mkdirSync(join(workspaceDir, "data", "db"), { recursive: true });
+  dbPath = join(workspaceDir, "data", "db", "assistant.db");
+  originalWorkspaceEnv = process.env.VELLUM_WORKSPACE_DIR;
+  process.env.VELLUM_WORKSPACE_DIR = workspaceDir;
+});
+
+afterEach(() => {
+  if (originalWorkspaceEnv === undefined) {
+    delete process.env.VELLUM_WORKSPACE_DIR;
+  } else {
+    process.env.VELLUM_WORKSPACE_DIR = originalWorkspaceEnv;
+  }
+  rmSync(workspaceDir, { recursive: true, force: true });
+});
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Build a minimal but realistic DB: WAL mode, a couple of tables with rows. */
+function seedDb(): void {
+  const db = new Database(dbPath);
+  try {
+    db.exec("PRAGMA journal_mode=WAL");
+    db.exec(`
+      CREATE TABLE memory_checkpoints (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        updated_at INTEGER NOT NULL
+      );
+      CREATE TABLE messages (
+        id TEXT PRIMARY KEY,
+        content TEXT
+      );
+      CREATE TABLE small (
+        id TEXT PRIMARY KEY
+      );
+    `);
+    // 100 messages, 5 small
+    const insertMsg = db.prepare(
+      "INSERT INTO messages (id, content) VALUES (?, ?)",
+    );
+    for (let i = 0; i < 100; i++) {
+      insertMsg.run(`m-${i}`, `row ${i}`);
+    }
+    const insertSmall = db.prepare("INSERT INTO small (id) VALUES (?)");
+    for (let i = 0; i < 5; i++) {
+      insertSmall.run(`s-${i}`);
+    }
+    // Two migration rows — only the later one matters.
+    const insertCk = db.prepare(
+      "INSERT INTO memory_checkpoints (key, value, updated_at) VALUES (?, ?, ?)",
+    );
+    insertCk.run("migration_older_v1", "1", 1_700_000_000_000);
+    insertCk.run("migration_newest_v1", "1", 1_700_000_001_000);
+    // A 'started' (not completed) row that must be IGNORED.
+    insertCk.run("migration_running_v1", "started", 1_700_000_002_000);
+  } finally {
+    db.close();
+  }
+}
+
+/** Run `assistant db status` with the given args, capturing stdio + exit. */
+async function runStatus(args: string[]): Promise<{
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+}> {
+  const stdoutChunks: string[] = [];
+  const stderrChunks: string[] = [];
+  let exitCode = 0;
+
+  const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+  const originalStderrWrite = process.stderr.write.bind(process.stderr);
+  const originalExit = process.exit;
+
+  process.stdout.write = ((chunk: string | Uint8Array) => {
+    stdoutChunks.push(
+      typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk),
+    );
+    return true;
+  }) as typeof process.stdout.write;
+
+  process.stderr.write = ((chunk: string | Uint8Array) => {
+    stderrChunks.push(
+      typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk),
+    );
+    return true;
+  }) as typeof process.stderr.write;
+
+  // process.exit is captured by throwing — the command finishes its work
+  // before exiting, so the throw cleanly unwinds back here.
+  process.exit = ((code?: number) => {
+    exitCode = code ?? 0;
+    throw new Error("__test_exit__");
+  }) as typeof process.exit;
+
+  try {
+    const { Command } = await import("commander");
+    const { registerDbCommand } = await import("../index.js");
+    const program = new Command();
+    program.exitOverride();
+    registerDbCommand(program);
+    try {
+      await program.parseAsync(["node", "assistant", "db", ...args]);
+    } catch (e) {
+      if ((e as Error).message !== "__test_exit__") {
+        throw e;
+      }
+    }
+  } finally {
+    process.stdout.write = originalStdoutWrite;
+    process.stderr.write = originalStderrWrite;
+    process.exit = originalExit;
+  }
+
+  return {
+    stdout: stdoutChunks.join(""),
+    stderr: stderrChunks.join(""),
+    exitCode,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("assistant db status — happy path", () => {
+  test("reports path, sizes, table count, and largest tables", async () => {
+    seedDb();
+
+    const { stdout, stderr, exitCode } = await runStatus(["status"]);
+
+    expect(exitCode).toBe(0);
+    expect(stderr).toBe("");
+    expect(stdout).toContain(dbPath);
+    expect(stdout).toContain("Tables");
+    expect(stdout).toMatch(/Tables\s+3/);
+    // Largest by row count — `messages` (100) leads, `small` (5) trails.
+    // (memory_checkpoints has 3 rows.)
+    expect(stdout).toContain("Largest tables");
+    expect(stdout.indexOf("messages")).toBeLessThan(stdout.indexOf("small"));
+  });
+
+  test("reports the latest 'completed' migration, ignoring 'started'", async () => {
+    seedDb();
+
+    const { stdout } = await runStatus(["status"]);
+
+    // The 'started' row is more recent but value != '1', so the
+    // completed row before it wins.
+    expect(stdout).toContain("newest_v1");
+    expect(stdout).not.toContain("running_v1");
+  });
+
+  test("reports WAL journal mode after seeding with WAL", async () => {
+    seedDb();
+
+    const { stdout } = await runStatus(["status"]);
+
+    expect(stdout).toMatch(/Journal\s+wal/);
+  });
+
+  test("--json emits a single-line JSON object with the expected shape", async () => {
+    seedDb();
+
+    const { stdout, exitCode } = await runStatus(["--json", "status"]);
+
+    expect(exitCode).toBe(0);
+    // One line of JSON, parses cleanly.
+    expect(stdout.trim().split("\n")).toHaveLength(1);
+    const parsed = JSON.parse(stdout);
+    expect(parsed.file.exists).toBe(true);
+    expect(parsed.file.path).toBe(dbPath);
+    expect(parsed.db.journalMode).toBe("wal");
+    expect(parsed.db.tableCount).toBe(3);
+    expect(parsed.db.latestMigration.key).toBe("migration_newest_v1");
+    expect(Array.isArray(parsed.db.largest)).toBe(true);
+    expect(parsed.db.largest.length).toBeGreaterThan(0);
+    // Either 'bytes' (if dbstat is available) or 'rows' fallback.
+    expect(["bytes", "rows"]).toContain(parsed.db.sizingMethod);
+  });
+});
+
+describe("assistant db status — DB missing", () => {
+  test("exits 1 with a loud error and recovery hint", async () => {
+    // No seedDb() — file simply does not exist.
+    const { stdout, stderr, exitCode } = await runStatus(["status"]);
+
+    expect(exitCode).toBe(1);
+    expect(stdout).toBe("");
+    expect(stderr).toContain("ERROR");
+    expect(stderr).toContain("Database not found");
+    expect(stderr).toContain(dbPath);
+    // Should point the user at recovery channels.
+    expect(stderr).toContain("assistant backup list");
+  });
+
+  test("--json missing DB emits structured payload and exits 1", async () => {
+    const { stdout, stderr, exitCode } = await runStatus(["--json", "status"]);
+
+    expect(exitCode).toBe(1);
+    expect(stderr).toBe("");
+    const parsed = JSON.parse(stdout);
+    expect(parsed.file.exists).toBe(false);
+    expect(parsed.db).toBeNull();
+  });
+});
+
+describe("assistant db status — DB without memory_checkpoints", () => {
+  test("reports schema unknown without crashing", async () => {
+    // Create a DB that has zero application tables — exercises the
+    // best-effort 'memory_checkpoints does not exist' branch.
+    const db = new Database(dbPath);
+    db.exec("PRAGMA journal_mode=WAL");
+    db.close();
+
+    const { stdout, exitCode } = await runStatus(["status"]);
+
+    expect(exitCode).toBe(0);
+    expect(stdout).toContain("Schema (last)");
+    expect(stdout).toContain("unknown");
+  });
+});

package/src/cli/commands/db/format.ts

@@ -0,0 +1,48 @@
+/**
+ * Shared formatters for `assistant db` subcommands. Inlined here (rather than
+ * importing from a sibling command) so the `db` directory is self-contained
+ * and the formatting choices stay consistent across status / repair / future
+ * subcommands.
+ */
+
+/** Format a byte count as a human-readable string. */
+export function formatBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  if (bytes < 1024 * 1024 * 1024) {
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+  }
+  return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+}
+
+/** Format a number with thousands separators (1234567 → "1,234,567"). */
+export function formatCount(n: number): string {
+  return n.toLocaleString("en-US");
+}
+
+/** Short relative age: "32s ago", "12m ago", "3h ago", "5d ago". */
+export function formatAge(ms: number): string {
+  if (ms < 0) ms = 0;
+  const s = Math.floor(ms / 1000);
+  if (s < 60) return `${s}s ago`;
+  if (s < 3600) return `${Math.floor(s / 60)}m ago`;
+  if (s < 86400) return `${Math.floor(s / 3600)}h ago`;
+  return `${Math.floor(s / 86400)}d ago`;
+}
+
+/** Format an epoch-ms timestamp as `YYYY-MM-DD HH:MM:SS UTC`. */
+export function formatTimestampUtc(ms: number): string {
+  const d = new Date(ms);
+  const y = d.getUTCFullYear().toString().padStart(4, "0");
+  const mo = (d.getUTCMonth() + 1).toString().padStart(2, "0");
+  const da = d.getUTCDate().toString().padStart(2, "0");
+  const h = d.getUTCHours().toString().padStart(2, "0");
+  const mi = d.getUTCMinutes().toString().padStart(2, "0");
+  const se = d.getUTCSeconds().toString().padStart(2, "0");
+  return `${y}-${mo}-${da} ${h}:${mi}:${se} UTC`;
+}
+
+/** Format a file mode (st_mode lower bits) as octal, e.g. "0644". */
+export function formatMode(mode: number): string {
+  return (mode & 0o7777).toString(8).padStart(4, "0");
+}

package/src/cli/commands/db/index.ts

@@ -0,0 +1,29 @@
+/**
+ * `assistant db` — inspect and repair the assistant SQLite database directly
+ * from disk.
+ *
+ * Subcommands declare `transport: "local"` so they work when the daemon is
+ * down — which is precisely the failure mode this surface is most useful in.
+ * Each subcommand opens its own bun:sqlite connection (read-only for
+ * inspection, read-write for repair steps that mutate) and never goes
+ * through IPC.
+ */
+
+import type { Command } from "commander";
+
+import { registerCommand } from "../../lib/register-command.js";
+import { registerDbRepair } from "./repair.js";
+import { registerDbStatus } from "./status.js";
+
+export function registerDbCommand(program: Command): void {
+  registerCommand(program, {
+    name: "db",
+    transport: "local",
+    description: "Inspect and repair the assistant SQLite database",
+    build: (db) => {
+      db.option("--json", "Machine-readable compact JSON output");
+      registerDbStatus(db);
+      registerDbRepair(db);
+    },
+  });
+}

package/src/cli/commands/db/repair-step-conversation-backfill.ts

@@ -0,0 +1,345 @@
+/**
+ * Repair step: conversation backfill from the on-disk view.
+ *
+ * Each conversation directory under `<workspace>/conversations/<id>/` holds
+ * a `meta.json` and `messages.jsonl` written by the runtime as the source
+ * of truth for the disk view. If the SQLite database was wiped, restored
+ * from an old backup, or otherwise lost the `conversations`/`messages`
+ * rows, this step replays the on-disk files to reconstruct them.
+ *
+ * Workspace migration 028 performs the same kind of recovery at startup,
+ * but its body is a frozen snapshot that runs against historical
+ * workspaces. This step owns its own copy so the live `db repair` surface
+ * can evolve independently — bug fixes, new edge cases, or schema changes
+ * don't risk altering migration 028's behavior on workspaces that have
+ * already run it.
+ *
+ * This step opens its own read-write bun:sqlite handle so the command
+ * works when the daemon is down — the whole point of the local transport.
+ *
+ * Idempotent: existing conversation rows are skipped without modification.
+ */
+
+import { randomUUID } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+
+import { eq } from "drizzle-orm";
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import * as schema from "../../../memory/schema.js";
+import {
+  conversations,
+  messages,
+} from "../../../memory/schema/conversations.js";
+import { getWorkspaceDir } from "../../../util/platform.js";
+import type { RepairContext, RepairStep, StepResult } from "./repair-steps.js";
+
+/**
+ * Cap on warning lines surfaced in the human-mode output. The JSON payload
+ * carries the full list (subject to `WARNING_CAP_TOTAL`) so scripted callers
+ * never lose detail.
+ */
+const MAX_REPORTED_WARNING_LINES = 20;
+
+/**
+ * Hard cap on warnings retained in memory, even for the JSON payload.
+ * Prevents a workspace with thousands of malformed entries from blowing
+ * memory on the report object.
+ */
+const WARNING_CAP_TOTAL = 500;
+
+// ---------------------------------------------------------------------------
+// On-disk record shapes
+// ---------------------------------------------------------------------------
+
+interface DiskMeta {
+  id: string;
+  title?: string;
+  type?: string;
+  channel?: string;
+  createdAt?: string;
+  updatedAt?: string;
+}
+
+interface DiskToolCall {
+  name?: string;
+  input?: unknown;
+}
+
+interface DiskToolResult {
+  content?: unknown;
+}
+
+interface DiskMessageRecord {
+  role: string;
+  ts?: string;
+  content?: string;
+  toolCalls?: DiskToolCall[];
+  toolResults?: DiskToolResult[];
+  attachments?: unknown[];
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function parseEpochMs(isoString: string | undefined): number | null {
+  if (!isoString) return null;
+  const ms = new Date(isoString).getTime();
+  return Number.isNaN(ms) ? null : ms;
+}
+
+function buildContentBlocks(record: DiskMessageRecord): unknown[] {
+  const blocks: unknown[] = [];
+
+  if (record.content) {
+    blocks.push({ type: "text", text: record.content });
+  }
+
+  if (Array.isArray(record.toolCalls)) {
+    for (const tc of record.toolCalls) {
+      blocks.push({
+        type: "tool_use",
+        id: randomUUID(),
+        name: tc.name ?? "unknown",
+        input: tc.input ?? {},
+      });
+    }
+  }
+
+  if (Array.isArray(record.toolResults)) {
+    for (const tr of record.toolResults) {
+      blocks.push({
+        type: "tool_result",
+        tool_use_id: "",
+        content:
+          typeof tr.content === "string"
+            ? tr.content
+            : JSON.stringify(tr.content),
+      });
+    }
+  }
+
+  // content column is NOT NULL — ensure at least one block
+  if (blocks.length === 0) {
+    blocks.push({ type: "text", text: "" });
+  }
+
+  return blocks;
+}
+
+// ---------------------------------------------------------------------------
+// Step body
+// ---------------------------------------------------------------------------
+
+async function runConversationBackfill(
+  ctx: RepairContext,
+): Promise<StepResult> {
+  // Open RW so we can insert recovered rows. Mirror the daemon's pragmas
+  // for consistent journal/FK behavior.
+  let sqlite: Database;
+  try {
+    sqlite = new Database(ctx.dbPath);
+    sqlite.exec("PRAGMA journal_mode=WAL");
+    sqlite.exec("PRAGMA synchronous=FULL");
+    sqlite.exec("PRAGMA busy_timeout=5000");
+    sqlite.exec("PRAGMA foreign_keys = ON");
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return {
+      status: "error",
+      summary: "could not open database for conversation backfill",
+      detailLines: [msg],
+      data: {
+        recovered: 0,
+        skipped: 0,
+        errors: 1,
+        warnings: [msg],
+        openFailed: true,
+      },
+    };
+  }
+
+  try {
+    const db = drizzle(sqlite, { schema });
+    const workspaceDir = getWorkspaceDir();
+
+    let recovered = 0;
+    let skipped = 0;
+    let errors = 0;
+    const warnings: string[] = [];
+    const pushWarning = (line: string): void => {
+      if (warnings.length < WARNING_CAP_TOTAL) warnings.push(line);
+    };
+
+    const conversationsDir = join(workspaceDir, "conversations");
+    if (existsSync(conversationsDir)) {
+      let entries: string[];
+      try {
+        entries = readdirSync(conversationsDir);
+      } catch (err) {
+        pushWarning(`failed to read conversations directory: ${String(err)}`);
+        entries = [];
+      }
+
+      for (const entry of entries) {
+        const dirPath = join(conversationsDir, entry);
+
+        try {
+          if (!statSync(dirPath).isDirectory()) continue;
+        } catch {
+          continue;
+        }
+
+        const metaPath = join(dirPath, "meta.json");
+        if (!existsSync(metaPath)) {
+          pushWarning(`${entry}: missing meta.json`);
+          skipped++;
+          continue;
+        }
+
+        let meta: DiskMeta;
+        try {
+          meta = JSON.parse(readFileSync(metaPath, "utf-8")) as DiskMeta;
+        } catch (err) {
+          pushWarning(`${entry}: malformed meta.json: ${String(err)}`);
+          skipped++;
+          continue;
+        }
+
+        if (!meta.id) {
+          pushWarning(`${entry}: meta.json missing id`);
+          skipped++;
+          continue;
+        }
+
+        const existing = db
+          .select()
+          .from(conversations)
+          .where(eq(conversations.id, meta.id))
+          .get();
+
+        if (existing) {
+          skipped++;
+          continue;
+        }
+
+        const messageRecords: DiskMessageRecord[] = [];
+        const messagesPath = join(dirPath, "messages.jsonl");
+        if (existsSync(messagesPath)) {
+          try {
+            const raw = readFileSync(messagesPath, "utf-8");
+            for (const line of raw.split("\n")) {
+              const trimmed = line.trim();
+              if (!trimmed) continue;
+              try {
+                messageRecords.push(JSON.parse(trimmed) as DiskMessageRecord);
+              } catch {
+                pushWarning(`${entry}: malformed JSONL line in messages.jsonl`);
+              }
+            }
+          } catch (err) {
+            pushWarning(
+              `${entry}: failed to read messages.jsonl: ${String(err)}`,
+            );
+          }
+        }
+
+        const createdAt = parseEpochMs(meta.createdAt) ?? Date.now();
+        const updatedAt = parseEpochMs(meta.updatedAt) ?? createdAt;
+
+        try {
+          db.transaction((tx) => {
+            tx.insert(conversations)
+              .values({
+                id: meta.id,
+                title: meta.title ?? null,
+                createdAt,
+                updatedAt,
+                conversationType: meta.type ?? "standard",
+                originChannel: meta.channel ?? null,
+                source: "user",
+                memoryScopeId: "default",
+                isAutoTitle: 1,
+                totalInputTokens: 0,
+                totalOutputTokens: 0,
+                totalEstimatedCost: 0,
+                contextSummary: null,
+                contextCompactedMessageCount: 0,
+                contextCompactedAt: null,
+                originInterface: null,
+                forkParentConversationId: null,
+                forkParentMessageId: null,
+                scheduleJobId: null,
+              })
+              .run();
+
+            for (const record of messageRecords) {
+              const contentBlocks = buildContentBlocks(record);
+              const msgCreatedAt = parseEpochMs(record.ts) ?? createdAt;
+
+              tx.insert(messages)
+                .values({
+                  id: randomUUID(),
+                  conversationId: meta.id,
+                  role: record.role,
+                  content: JSON.stringify(contentBlocks),
+                  createdAt: msgCreatedAt,
+                  metadata: null,
+                })
+                .run();
+            }
+          });
+          recovered++;
+        } catch (err) {
+          pushWarning(`${meta.id} (${entry}): insert failed: ${String(err)}`);
+          errors++;
+        }
+      }
+    }
+
+    const summary =
+      recovered === 0 && errors === 0
+        ? `nothing to backfill (${skipped} on-disk conversation${skipped === 1 ? "" : "s"} already present)`
+        : `recovered ${recovered}, skipped ${skipped}, ${errors} error${errors === 1 ? "" : "s"}`;
+
+    const truncatedWarnings = warnings.slice(0, MAX_REPORTED_WARNING_LINES);
+    const detailLines =
+      warnings.length > MAX_REPORTED_WARNING_LINES
+        ? [
+            ...truncatedWarnings,
+            `+ ${warnings.length - MAX_REPORTED_WARNING_LINES} more (use --json for full list)`,
+          ]
+        : truncatedWarnings;
+
+    // Errors during insert are surfaced as a non-halting failure so later
+    // steps still run. Warnings without errors (malformed JSONL lines,
+    // missing meta.json) are not themselves a failure — they're skips.
+    if (errors > 0) {
+      return {
+        status: "error",
+        summary,
+        detailLines,
+        data: { recovered, skipped, errors, warnings },
+      };
+    }
+
+    return {
+      status: "ok",
+      summary,
+      detailLines,
+      data: { recovered, skipped, errors, warnings },
+    };
+  } finally {
+    sqlite.close();
+  }
+}
+
+export const conversationBackfillStep: RepairStep = {
+  name: "conversation-backfill",
+  description:
+    "Replay workspace/conversations/<id>/{meta.json,messages.jsonl} into SQLite",
+  run: runConversationBackfill,
+};