botholomew 0.12.5 → 0.14.0

package/src/context/ingest.ts

@@ -1,162 +0,0 @@
-import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
-import { getContextItem, getContextItemById } from "../db/context.ts";
-import {
-  createEmbedding,
-  deleteEmbeddingsForItem,
-  rebuildSearchIndex,
-} from "../db/embeddings.ts";
-import { logger } from "../utils/logger.ts";
-import { chunk } from "./chunker.ts";
-import { type DriveTarget, formatDriveRef } from "./drives.ts";
-import { embed as defaultEmbed } from "./embedder.ts";
-
-type IngestEmbedFn = (texts: string[]) => Promise<number[][]>;
-
-export interface PreparedIngestion {
-  itemId: string;
-  title: string;
-  description: string;
-  drive: string;
-  path: string;
-  chunks: { index: number; content: string }[];
-  vectors: number[][];
-}
-
-/**
- * Prepare an item for ingestion: chunk content and compute embeddings.
- * This is the expensive (parallelizable) part — no DB writes happen here.
- */
-export async function prepareIngestion(
-  conn: DbConnection,
-  itemId: string,
-  config: Required<BotholomewConfig>,
-  embedFn?: IngestEmbedFn,
-): Promise<PreparedIngestion | null> {
-  const item = await getContextItemById(conn, itemId);
-  if (!item) {
-    logger.warn(`ingest: context item ${itemId} not found`);
-    return null;
-  }
-
-  if (!item.is_textual || !item.content) {
-    logger.debug(`ingest: skipping non-textual item ${itemId}`);
-    return null;
-  }
-
-  const doEmbed = embedFn ?? ((texts: string[]) => defaultEmbed(texts, config));
-
-  const chunks = await chunk(item.content, item.mime_type, config);
-  if (chunks.length === 0) return null;
-
-  const ref = formatDriveRef(item);
-  const textsForEmbedding = chunks.map((c) => {
-    const parts: string[] = [];
-    if (item.title) parts.push(`Title: ${item.title}`);
-    if (item.description) parts.push(`Description: ${item.description}`);
-    parts.push(`Source: ${ref}`);
-    parts.push(c.content);
-    return parts.join("\n");
-  });
-  const vectors = await doEmbed(textsForEmbedding);
-
-  return {
-    itemId,
-    title: item.title,
-    description: item.description,
-    drive: item.drive,
-    path: item.path,
-    chunks,
-    vectors,
-  };
-}
-
-export interface IngestionResult {
-  chunks: number;
-  isUpdate: boolean;
-}
-
-/**
- * Store a prepared ingestion into the database.
- * All statements in BEGIN/COMMIT/ROLLBACK must share one connection, so the
- * caller must pass a connection that lives long enough for the transaction
- * (the tool executor wraps each tool call in `withDb`, which satisfies this).
- */
-export async function storeIngestion(
-  conn: DbConnection,
-  prepared: PreparedIngestion,
-): Promise<IngestionResult> {
-  let isUpdate = false;
-  await conn.exec("BEGIN TRANSACTION");
-  try {
-    const deleted = await deleteEmbeddingsForItem(conn, prepared.itemId);
-    isUpdate = deleted > 0;
-
-    for (const [i, c] of prepared.chunks.entries()) {
-      const v = prepared.vectors[i];
-      if (!v) continue;
-      await createEmbedding(conn, {
-        contextItemId: prepared.itemId,
-        chunkIndex: c.index,
-        chunkContent: c.content,
-        title: prepared.title,
-        description: prepared.description,
-        embedding: v,
-      });
-    }
-
-    await conn.queryRun(
-      "UPDATE context_items SET indexed_at = current_timestamp::VARCHAR WHERE id = ?1",
-      prepared.itemId,
-    );
-
-    await conn.exec("COMMIT");
-  } catch (err) {
-    await conn.exec("ROLLBACK");
-    throw err;
-  }
-
-  // FTS index is a snapshot and doesn't see the writes above until rebuilt.
-  await rebuildSearchIndex(conn);
-
-  const action = isUpdate ? "updated" : "added";
-  logger.info(
-    `ingest: ${action} ${prepared.chunks.length} chunks for "${prepared.title}" (${prepared.itemId})`,
-  );
-  return { chunks: prepared.chunks.length, isUpdate };
-}
-
-/**
- * Full ingestion pipeline for a context item:
- * 1. Fetch item from DB
- * 2. Skip if non-textual or empty
- * 3. Chunk content and embed chunks (outside transaction)
- * 4. In a transaction: delete old embeddings, store new ones, update indexed_at
- */
-export async function ingestContextItem(
-  conn: DbConnection,
-  itemId: string,
-  config: Required<BotholomewConfig>,
-  embedFn?: IngestEmbedFn,
-): Promise<number> {
-  const prepared = await prepareIngestion(conn, itemId, config, embedFn);
-  if (!prepared) return 0;
-  return (await storeIngestion(conn, prepared)).chunks;
-}
-
-/**
- * Ingest a context item by its (drive, path) pair.
- */
-export async function ingestByPath(
-  conn: DbConnection,
-  target: DriveTarget,
-  config: Required<BotholomewConfig>,
-  embedFn?: IngestEmbedFn,
-): Promise<number> {
-  const item = await getContextItem(conn, target);
-  if (!item) {
-    logger.warn(`ingest: no item at ${formatDriveRef(target)}`);
-    return 0;
-  }
-  return ingestContextItem(conn, item.id, config, embedFn);
-}

package/src/context/refresh.ts

@@ -1,183 +0,0 @@
-import type { McpxClient } from "@evantahler/mcpx";
-import type { BotholomewConfig } from "../config/schemas.ts";
-import type { DbConnection } from "../db/connection.ts";
-import { type ContextItem, updateContextItem } from "../db/context.ts";
-import { formatDriveRef } from "./drives.ts";
-import { type FetchedContent, fetchUrl } from "./fetcher.ts";
-import {
-  type PreparedIngestion,
-  prepareIngestion,
-  storeIngestion,
-} from "./ingest.ts";
-
-export type RefreshItemStatus = "updated" | "unchanged" | "missing" | "error";
-
-export interface RefreshItemResult {
-  id: string;
-  drive: string;
-  path: string;
-  ref: string;
-  status: RefreshItemStatus;
-  error?: string;
-}
-
-export interface RefreshResult {
-  checked: number;
-  updated: number;
-  unchanged: number;
-  missing: number;
-  reembedded: number;
-  chunks: number;
-  embeddings_skipped: boolean;
-  items: RefreshItemResult[];
-}
-
-export interface RefreshOptions {
-  concurrency?: number;
-  onItemProgress?: (done: number, total: number) => void;
-  onEmbedProgress?: (done: number, total: number) => void;
-}
-
-type IngestEmbedFn = (texts: string[]) => Promise<number[][]>;
-
-/** Signature compatible with {@link fetchUrl}. Injectable for tests. */
-export type FetchUrlFn = (
-  url: string,
-  config: Required<BotholomewConfig>,
-  mcpxClient: McpxClient | null,
-) => Promise<FetchedContent>;
-
-/**
- * Refresh a batch of context items: re-read from origin, diff, update
- * content, and re-embed only the items that changed.
- *
- * Dispatches on `drive`:
- *   disk  → read from filesystem
- *   agent → skip (no external origin)
- *   other → re-fetch via `item.source_url` (captured at ingest time).
- *           The built-in `url` drive stores the URL as its path so it can
- *           also refresh directly from `path`. Any other drive with no
- *           `source_url` surfaces a per-item error — the user must re-add
- *           from URL. No code here knows anything about the remote
- *           service behind a drive.
- */
-export async function refreshContextItems(
-  conn: DbConnection,
-  items: ContextItem[],
-  config: Required<BotholomewConfig>,
-  mcpxClient: McpxClient | null,
-  opts: RefreshOptions = {},
-  embedFn?: IngestEmbedFn,
-  fetchFn: FetchUrlFn = fetchUrl,
-): Promise<RefreshResult> {
-  const refreshable = items.filter((i) => i.drive !== "agent");
-
-  const results: RefreshItemResult[] = [];
-  const toReembed: string[] = [];
-
-  for (const [idx, item] of refreshable.entries()) {
-    opts.onItemProgress?.(idx, refreshable.length);
-    const base = {
-      id: item.id,
-      drive: item.drive,
-      path: item.path,
-      ref: formatDriveRef(item),
-    };
-
-    try {
-      let content: string;
-
-      if (item.drive === "disk") {
-        const bunFile = Bun.file(item.path);
-        if (!(await bunFile.exists())) {
-          results.push({ ...base, status: "missing" });
-          continue;
-        }
-        content = await bunFile.text();
-      } else {
-        const url =
-          item.source_url ??
-          (item.drive === "url" ? item.path.replace(/^\//, "") : null);
-        if (!url) {
-          results.push({
-            ...base,
-            status: "error",
-            error: `Cannot refresh ${formatDriveRef(item)}: no source_url recorded. Re-add from the original URL.`,
-          });
-          continue;
-        }
-        const fetched = await fetchFn(url, config, mcpxClient);
-        content = fetched.content;
-      }
-
-      if (content === item.content) {
-        results.push({ ...base, status: "unchanged" });
-        continue;
-      }
-
-      await updateContextItem(conn, item.id, { content });
-      results.push({ ...base, status: "updated" });
-      toReembed.push(item.id);
-    } catch (err) {
-      results.push({
-        ...base,
-        status: "error",
-        error: err instanceof Error ? err.message : String(err),
-      });
-    }
-  }
-  opts.onItemProgress?.(refreshable.length, refreshable.length);
-
-  const updated = results.filter((r) => r.status === "updated").length;
-  const unchanged = results.filter((r) => r.status === "unchanged").length;
-  const missing = results.filter((r) => r.status === "missing").length;
-
-  if (toReembed.length === 0) {
-    return {
-      checked: refreshable.length,
-      updated,
-      unchanged,
-      missing,
-      reembedded: 0,
-      chunks: 0,
-      embeddings_skipped: false,
-      items: results,
-    };
-  }
-
-  const concurrency = opts.concurrency ?? 10;
-  const prepared: PreparedIngestion[] = [];
-  let completed = 0;
-
-  for (let i = 0; i < toReembed.length; i += concurrency) {
-    const batch = toReembed.slice(i, i + concurrency);
-    const batchResults = await Promise.all(
-      batch.map(async (id) => {
-        const r = await prepareIngestion(conn, id, config, embedFn);
-        completed++;
-        opts.onEmbedProgress?.(completed, toReembed.length);
-        return r;
-      }),
-    );
-    for (const r of batchResults) {
-      if (r) prepared.push(r);
-    }
-  }
-
-  let chunks = 0;
-  for (const p of prepared) {
-    const result = await storeIngestion(conn, p);
-    chunks += result.chunks;
-  }
-
-  return {
-    checked: refreshable.length,
-    updated,
-    unchanged,
-    missing,
-    reembedded: prepared.length,
-    chunks,
-    embeddings_skipped: false,
-    items: results,
-  };
-}