botholomew 0.16.4 → 0.18.0

package/src/context/url-utils.ts

@@ -1,25 +0,0 @@
-/**
- * Attempts to parse the input as a URL and returns true if the protocol is http or https.
- */
-export function isUrl(input: string): boolean {
-  try {
-    const url = new URL(input);
-    return url.protocol === "http:" || url.protocol === "https:";
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Strips HTML tags from a string, removing script/style blocks first,
- * then all remaining tags, and collapsing whitespace.
- */
-export function stripHtmlTags(html: string): string {
-  return html
-    .replace(/<script[\s\S]*?<\/script>/gi, "") // remove script blocks
-    .replace(/<style[\s\S]*?<\/style>/gi, "") // remove style blocks
-    .replace(/<[^>]*>/g, "") // remove all remaining tags
-    .replace(/[ \t]+/g, " ") // collapse horizontal whitespace
-    .replace(/\n{3,}/g, "\n\n") // collapse excessive newlines
-    .trim();
-}

package/src/db/connection.ts

@@ -1,255 +0,0 @@
-import { DuckDBInstance } from "@duckdb/node-api";
-
-type SqlParam = string | number | boolean | null | number[];
-
-/**
- * Thin wrapper around DuckDB connection that provides a familiar
- * query interface similar to bun:sqlite. Automatically translates
- * ?N parameter placeholders to $N for DuckDB compatibility.
- */
-export class DbConnection {
-  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  private conn: any;
-  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  private readonly ownedInstance: any;
-  private readonly dbPath: string;
-  private closed = false;
-
-  constructor(
-    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-    conn: any,
-    // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-    ownedInstance: any,
-    dbPath: string,
-  ) {
-    this.conn = conn;
-    this.ownedInstance = ownedInstance;
-    this.dbPath = dbPath;
-  }
-
-  /** Execute raw SQL with no return value. */
-  async exec(sql: string): Promise<void> {
-    await this.conn.run(sql);
-  }
-
-  /** Run a query and return the first row, or null. */
-  async queryGet<T = Record<string, unknown>>(
-    sql: string,
-    ...params: SqlParam[]
-  ): Promise<T | null> {
-    const translated = translateParams(sql);
-    const result = await this.conn.runAndReadAll(
-      translated,
-      flattenParams(params),
-    );
-    const rows = await result.getRowObjectsJS();
-    return (rows[0] ? convertRow(rows[0]) : null) as T | null;
-  }
-
-  /** Run a query and return all rows. */
-  async queryAll<T = Record<string, unknown>>(
-    sql: string,
-    ...params: SqlParam[]
-  ): Promise<T[]> {
-    const translated = translateParams(sql);
-    const result = await this.conn.runAndReadAll(
-      translated,
-      flattenParams(params),
-    );
-    const rows = await result.getRowObjectsJS();
-    return rows.map(convertRow) as T[];
-  }
-
-  /** Run a mutation and return the number of changed rows. */
-  async queryRun(
-    sql: string,
-    ...params: SqlParam[]
-  ): Promise<{ changes: number }> {
-    const translated = translateParams(sql);
-    const result = await this.conn.run(translated, flattenParams(params));
-    return { changes: result.rowsChanged };
-  }
-
-  /**
-   * Disconnect and release this connection's share of the DuckDB instance.
-   * For file-backed DBs, the instance is closed (and the OS file lock
-   * released) once every overlapping connection in this process has closed.
-   * For `:memory:` DBs, the instance is owned by this connection and closed
-   * immediately.
-   */
-  close(): void {
-    if (this.closed) return;
-    this.closed = true;
-    this.conn.disconnectSync();
-    if (this.ownedInstance) {
-      this.ownedInstance.closeSync();
-    } else {
-      releaseInstance(this.dbPath);
-    }
-  }
-}
-
-/**
- * Convert DuckDB row values to JS-friendly types:
- * - BigInt → number (safe for counts and IDs)
- * - Date → ISO string (matches our TEXT column convention)
- * - Nested arrays/objects are left as-is
- */
-// biome-ignore lint/suspicious/noExplicitAny: row values are dynamic
-function convertRow(row: Record<string, any>): Record<string, unknown> {
-  const out: Record<string, unknown> = {};
-  for (const [key, val] of Object.entries(row)) {
-    if (typeof val === "bigint") {
-      out[key] = Number(val);
-    } else {
-      out[key] = val;
-    }
-  }
-  return out;
-}
-
-/** Translate ?N placeholders to $N for DuckDB. */
-function translateParams(sql: string): string {
-  return sql.replace(/\?(\d+)/g, "$$$1");
-}
-
-/** Flatten params, converting number[] to JSON strings for vector columns. */
-function flattenParams(params: SqlParam[]): SqlParam[] {
-  return params.map((p) => (Array.isArray(p) ? JSON.stringify(p) : p));
-}
-
-/**
- * Refcounted, process-local cache of open DuckDB instances keyed by dbPath.
- *
- * DuckDB's file lock is held at the instance level, so we must close the
- * instance — not just the connection — to let another process acquire the
- * writer lock. At the same time, opening two instances for the same file
- * from one process is unsafe. This cache resolves both: overlapping
- * `getConnection` calls in the same process share a single instance; once
- * every connection has closed, the instance is closed and evicted, which
- * releases the OS file lock.
- *
- * `:memory:` paths bypass the cache so each test/caller gets its own
- * isolated in-memory database.
- */
-interface CachedInstance {
-  // biome-ignore lint/suspicious/noExplicitAny: DuckDB internal types
-  instance: any;
-  refCount: number;
-}
-const instanceCache = new Map<string, CachedInstance>();
-const pendingInstance = new Map<string, Promise<CachedInstance>>();
-
-function isMemoryPath(path: string): boolean {
-  return path === ":memory:" || path.startsWith(":memory:");
-}
-
-async function acquireSharedInstance(dbPath: string): Promise<CachedInstance> {
-  const existing = instanceCache.get(dbPath);
-  if (existing) {
-    existing.refCount += 1;
-    return existing;
-  }
-  const inFlight = pendingInstance.get(dbPath);
-  if (inFlight) {
-    const cached = await inFlight;
-    cached.refCount += 1;
-    return cached;
-  }
-  const creation = (async () => {
-    const instance = await DuckDBInstance.create(dbPath);
-    const cached: CachedInstance = { instance, refCount: 1 };
-    instanceCache.set(dbPath, cached);
-    return cached;
-  })();
-  pendingInstance.set(dbPath, creation);
-  try {
-    return await creation;
-  } finally {
-    pendingInstance.delete(dbPath);
-  }
-}
-
-function releaseInstance(dbPath: string): void {
-  const cached = instanceCache.get(dbPath);
-  if (!cached) return;
-  cached.refCount -= 1;
-  if (cached.refCount <= 0) {
-    instanceCache.delete(dbPath);
-    cached.instance.closeSync();
-  }
-}
-
-export async function getConnection(dbPath?: string): Promise<DbConnection> {
-  const path = dbPath ?? ":memory:";
-
-  if (isMemoryPath(path)) {
-    const instance = await DuckDBInstance.create(path);
-    const conn = await instance.connect();
-    await conn.run("INSTALL fts; LOAD fts;");
-    return new DbConnection(conn, instance, path);
-  }
-
-  const cached = await acquireSharedInstance(path);
-  try {
-    const conn = await cached.instance.connect();
-    // INSTALL is a no-op after the first successful install (the extension
-    // is persisted to the user's DuckDB extension directory). LOAD is
-    // cheap per connection.
-    await conn.run("INSTALL fts; LOAD fts;");
-    return new DbConnection(conn, null, path);
-  } catch (err) {
-    releaseInstance(path);
-    throw err;
-  }
-}
-
-/**
- * Open a DuckDB connection for a single logical unit of work and guarantee
- * it is closed afterward. Retries on lock conflicts so two processes that
- * race on the file lock cooperate instead of failing hard.
- *
- * Prefer one `withDb` per logical operation. The file lock is only released
- * when every connection (across this process's overlapping callers) has
- * been closed, so holding the connection across non-DB work (LLM calls,
- * network I/O, filesystem walks) keeps other processes blocked.
- */
-export async function withDb<T>(
-  dbPath: string,
-  fn: (conn: DbConnection) => Promise<T>,
-): Promise<T> {
-  const conn = await withRetry(() => getConnection(dbPath));
-  try {
-    return await fn(conn);
-  } finally {
-    conn.close();
-  }
-}
-
-/**
- * Retry `fn` with exponential backoff when it fails with a DuckDB file-lock
- * conflict ("Conflicting lock is held…"). Other errors propagate immediately.
- */
-export async function withRetry<T>(
-  fn: () => Promise<T>,
-  maxRetries = 8,
-): Promise<T> {
-  let lastError: unknown;
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    try {
-      return await fn();
-    } catch (err) {
-      if (!isLockConflict(err)) throw err;
-      lastError = err;
-      if (attempt === maxRetries - 1) throw err;
-      // 100, 200, 400, 800, 1600, 3200, 6400, 12800 — up to ~25s total
-      await Bun.sleep(100 * 2 ** attempt);
-    }
-  }
-  throw lastError;
-}
-
-function isLockConflict(err: unknown): boolean {
-  const msg = err instanceof Error ? err.message : String(err);
-  return msg.includes("Conflicting lock") || msg.includes("could not be set");
-}

package/src/db/doctor.ts

@@ -1,235 +0,0 @@
-import { mkdir, rename, rm, stat } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { withDb } from "./connection.ts";
-
-/**
- * Tables we probe for primary-key index integrity. Every user table has a
- * single-column PK that we exercise with a self-update (SET pk = pk WHERE
- * pk = ...). DuckDB still walks the index for the SET, which surfaces
- * "Failed to delete all rows from index" FATAL errors when the index is
- * out of sync with the row data. `_migrations` is excluded — it is small,
- * append-only, and rebuilding it would defeat its purpose.
- */
-export const PROBE_TABLES: ReadonlyArray<{ name: string; pk: string }> = [
-  { name: "context_index", pk: "path" },
-];
-
-export type ProbeStatus = "ok" | "empty" | "missing" | "corrupt";
-
-export interface ProbeResult {
-  table: string;
-  status: ProbeStatus;
-  /** Detail message when status is corrupt or missing. */
-  message?: string;
-}
-
-/**
- * Probe a single table for index corruption by spawning a child Bun
- * process. We use a child process because a corrupt PK index in DuckDB
- * surfaces as a Bun panic (a C++ exception that unwinds past the NAPI
- * boundary), which would kill the doctor itself. The child reports its
- * verdict on stdout and exits.
- *
- * Uses absolute import path resolved against this file so the spawned
- * Bun process picks up the same `@duckdb/node-api` install.
- */
-export async function probeTable(
-  dbPath: string,
-  table: string,
-  pk: string,
-): Promise<ProbeResult> {
-  const script = `
-    const { DuckDBInstance } = await import("@duckdb/node-api");
-    const dbPath = ${JSON.stringify(dbPath)};
-    const table = ${JSON.stringify(table)};
-    const pk = ${JSON.stringify(pk)};
-    let inst;
-    try {
-      inst = await DuckDBInstance.create(dbPath);
-    } catch (e) {
-      process.stdout.write("MISSING:" + (e?.message ?? String(e)));
-      process.exit(0);
-    }
-    const c = await inst.connect();
-    try {
-      const r = await c.runAndReadAll(\`SELECT \${pk} FROM \${table} LIMIT 1\`);
-      if (r.getRows().length === 0) {
-        process.stdout.write("EMPTY");
-        process.exit(0);
-      }
-    } catch (e) {
-      const msg = String(e?.message ?? e);
-      // Table doesn't exist yet (e.g., schema older than this doctor) — not
-      // a corruption signal, just skip it.
-      if (msg.includes("does not exist") || msg.includes("Catalog Error")) {
-        process.stdout.write("MISSING:" + msg);
-        process.exit(0);
-      }
-      process.stdout.write("CORRUPT:" + msg);
-      process.exit(2);
-    }
-    try {
-      await c.run(\`UPDATE \${table} SET \${pk} = \${pk} WHERE \${pk} = (SELECT \${pk} FROM \${table} LIMIT 1)\`);
-      process.stdout.write("OK");
-      process.exit(0);
-    } catch (e) {
-      process.stdout.write("CORRUPT:" + (e?.message ?? String(e)));
-      process.exit(2);
-    }
-  `;
-
-  // Discard the child's stderr. When the probe panics, Bun writes a multi-
-  // line crash banner there which would otherwise spill into our table
-  // output via the fallback message. The exit code alone tells us what we
-  // need to know.
-  const proc = Bun.spawn(["bun", "-e", script], {
-    stdio: ["ignore", "pipe", "ignore"],
-  });
-  const [stdout, exitCode] = await Promise.all([
-    new Response(proc.stdout).text(),
-    proc.exited,
-  ]);
-
-  // Bun panic: process killed by SIGTRAP / non-zero exit with no stdout
-  // verdict. Treat any unrecognized exit as corruption — better to flag
-  // for repair than to silently miss a problem.
-  if (stdout.startsWith("OK")) return { table, status: "ok" };
-  if (stdout.startsWith("EMPTY")) return { table, status: "empty" };
-  if (stdout.startsWith("MISSING:")) {
-    return {
-      table,
-      status: "missing",
-      message: firstLine(stdout.slice("MISSING:".length)),
-    };
-  }
-  if (stdout.startsWith("CORRUPT:")) {
-    return {
-      table,
-      status: "corrupt",
-      message: firstLine(stdout.slice("CORRUPT:".length)),
-    };
-  }
-  return {
-    table,
-    status: "corrupt",
-    message: `child exited with code ${exitCode} (likely native panic)`,
-  };
-}
-
-/**
- * Run probes for every known table. Sequential rather than parallel so we
- * cooperate with DuckDB's per-process file lock and don't multiply the
- * blast radius of a panic.
- */
-export async function probeAllTables(dbPath: string): Promise<ProbeResult[]> {
-  const results: ProbeResult[] = [];
-  for (const { name, pk } of PROBE_TABLES) {
-    results.push(await probeTable(dbPath, name, pk));
-  }
-  return results;
-}
-
-export interface RepairResult {
-  backupDbPath: string;
-  exportDir: string;
-  durationMs: number;
-}
-
-/**
- * Repair `dbPath` by exporting its contents and importing into a fresh
- * file. EXPORT DATABASE reads via sequential scans, not via PK indexes,
- * so it survives the kind of index corruption that breaks UPDATE/DELETE.
- * IMPORT DATABASE rebuilds every index from the data, which restores
- * write integrity.
- *
- * Steps:
- *   1. CHECKPOINT (best-effort) to flush WAL.
- *   2. EXPORT DATABASE to `<dotDir>/.export-<timestamp>`.
- *   3. Move `data.duckdb` (and `.wal`) to `data.duckdb.bak-<timestamp>`.
- *   4. Open a fresh DB at the original path and IMPORT DATABASE.
- *   5. Leave the export dir on disk — cheap insurance if step 4 ever fails
- *      mid-way; cleanup on the next successful run.
- *
- * The caller is responsible for ensuring no other process holds the DB
- * (no running workers, no chat session, no TUI).
- */
-export async function repairDatabase(dbPath: string): Promise<RepairResult> {
-  const start = Date.now();
-  const dotDir = dirname(dbPath);
-  await mkdir(dotDir, { recursive: true });
-
-  const stamp = new Date()
-    .toISOString()
-    .replace(/[:.]/g, "-")
-    .replace(/Z$/, "");
-  const exportDir = join(dotDir, `.export-${stamp}`);
-  const backupDbPath = `${dbPath}.bak-${stamp}`;
-  const walPath = `${dbPath}.wal`;
-  const backupWalPath = `${backupDbPath}.wal`;
-
-  await withDb(dbPath, async (conn) => {
-    try {
-      await conn.exec("CHECKPOINT");
-    } catch {
-      // CHECKPOINT can fail on an already-invalidated DB; the EXPORT
-      // below is what actually matters.
-    }
-    await conn.exec(`EXPORT DATABASE '${exportDir.replace(/'/g, "''")}'`);
-  });
-
-  await rename(dbPath, backupDbPath);
-  if (await pathExists(walPath)) {
-    await rename(walPath, backupWalPath);
-  }
-
-  await withDb(dbPath, async (conn) => {
-    await conn.exec(`IMPORT DATABASE '${exportDir.replace(/'/g, "''")}'`);
-  });
-
-  // Best-effort cleanup of the export dir. Leave it on failure — the user
-  // still has data.duckdb (rebuilt) and the backup.
-  try {
-    await rm(exportDir, { recursive: true, force: true });
-  } catch {
-    // ignore
-  }
-
-  return {
-    backupDbPath,
-    exportDir,
-    durationMs: Date.now() - start,
-  };
-}
-
-async function pathExists(p: string): Promise<boolean> {
-  try {
-    await stat(p);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function firstLine(s: string): string {
-  const trimmed = s.trim();
-  const nl = trimmed.indexOf("\n");
-  return nl === -1 ? trimmed : trimmed.slice(0, nl);
-}
-
-/**
- * Send signal 0 to test whether `pid` corresponds to a live process. Returns
- * false on ESRCH (no such process) and on any other error (including EPERM,
- * which we conservatively treat as "not ours, not relevant"). Used by the
- * doctor's safety gate to distinguish workers actually running from rows
- * that say `status = 'running'` because the worker crashed before flipping
- * its row to `stopped` or `dead`.
- */
-export function isPidAlive(pid: number): boolean {
-  if (!pid || pid < 1) return false;
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    return false;
-  }
-}