botholomew 0.16.4 → 0.18.0

package/src/context/markdown-converter.ts

@@ -1,186 +0,0 @@
-import type { BotholomewConfig } from "../config/schemas.ts";
-import { logger } from "../utils/logger.ts";
-import { createLlmClient } from "../worker/llm-client.ts";
-import { FetchFailureError } from "./fetcher-errors.ts";
-
-const CONVERTER_MAX_TOKENS = 16_384;
-
-const CONVERTER_SYSTEM_PROMPT = `You normalize documents to clean, well-structured Markdown.
-
-**If the input is already clean, valid Markdown, return it verbatim with no edits.** Look for ATX headings (#, ##), bullet/numbered lists, fenced code blocks, inline code, links in [text](url) form, blockquotes, GFM tables. If the structure is consistently markdown-shaped, echo it back unchanged.
-
-Otherwise, convert it. The input mime_type is a hint, not a guarantee — verify the actual content. Common non-markdown formats to recognize and convert:
-- **HTML** — strip tags, scripts, styles, navigation/footer chrome; preserve headings, paragraphs, lists, tables, links, code.
-- **JSON / XML / YAML** — render the structure as readable Markdown (headings/lists for objects, tables where appropriate, fenced code blocks for inline values).
-- **DocMD (Google Docs structured format)** — lines like \`[H1 1-31 HEADING_1 tabId=t.0 ...] Title text\` or \`[P5 884-937 PARAGRAPH ...] Body text\`. Strip the bracket annotations entirely; map H1→#, H2→##, H3→###, P→paragraph; preserve the trailing text content.
-- **RTF, plain text with mixed structure, ad-hoc formats** — extract the semantic content, drop the noise.
-
-Rules for the output:
-- Preserve all semantic content: headings, paragraphs, lists, tables, links, inline code, code blocks, blockquotes.
-- Use ATX headings (#, ##, ###), fenced code blocks (\`\`\`lang), GFM-style tables, and reference- or inline-style links — whichever is cleanest.
-- Strip metadata headers/IDs that aren't part of the document body (e.g. \`@document_id: ...\`, \`@revision_id: ...\`).
-- Output **only** the Markdown. No preamble ("Here is the converted markdown:"), no trailing commentary, no wrapping the entire output in a code fence.`;
-
-const MARKDOWN_MIME_TYPES = new Set([
-  "text/markdown",
-  "text/x-markdown",
-  "text/md",
-]);
-
-export function isMarkdownMimeType(mimeType: string): boolean {
-  const base = mimeType.split(";")[0]?.trim().toLowerCase() ?? "";
-  return MARKDOWN_MIME_TYPES.has(base);
-}
-
-/**
- * Sniff content for a non-markdown structure. Returns a mime type when the
- * content has unmistakable markers of HTML / XML / JSON / etc., otherwise
- * null. Used to verify a tool's claim of `text/markdown` — if the agent (or
- * a defaulted mime type) lies about the format, we want to convert anyway.
- *
- * Markdown is a superset of plain text, so a null return ≠ "definitely
- * markdown". It just means we found no strong contradicting signal.
- */
-export function sniffNonMarkdownMimeType(content: string): string | null {
-  const head = content.trimStart().slice(0, 4096);
-  if (!head) return null;
-
-  if (/^<!doctype\s+html/i.test(head)) return "text/html";
-  if (/^<html[\s>]/i.test(head)) return "text/html";
-  if (/^<\?xml[\s?]/i.test(head)) return "application/xml";
-
-  // JSON: parses as JSON top-to-bottom (use the full content, not the head).
-  const trimmed = content.trim();
-  if (
-    (trimmed.startsWith("{") && trimmed.endsWith("}")) ||
-    (trimmed.startsWith("[") && trimmed.endsWith("]"))
-  ) {
-    try {
-      JSON.parse(trimmed);
-      return "application/json";
-    } catch {
-      // fall through
-    }
-  }
-
-  // Heuristic HTML: dense tag markup. Markdown can contain occasional inline
-  // HTML, so we only flag it when tags dominate the sample.
-  const tagMatches = head.match(/<\/?[a-z][a-z0-9]*[\s/>]/gi) ?? [];
-  if (tagMatches.length >= 10) {
-    const charsPerTag = head.length / tagMatches.length;
-    if (charsPerTag < 80) return "text/html";
-  }
-
-  return null;
-}
-
-/**
- * Decide the effective mime type for a piece of content. If the claim is
- * markdown but the content sniffs as something else, trust the sniff so we
- * convert instead of saving mislabeled garbage.
- */
-export function resolveEffectiveMimeType(
-  claimedMimeType: string,
-  content: string,
-): { mimeType: string; sniffed: boolean } {
-  if (!isMarkdownMimeType(claimedMimeType)) {
-    return { mimeType: claimedMimeType, sniffed: false };
-  }
-  const sniffed = sniffNonMarkdownMimeType(content);
-  if (sniffed) return { mimeType: sniffed, sniffed: true };
-  return { mimeType: claimedMimeType, sniffed: false };
-}
-
-function stripLeadingMarkdownFence(text: string): string {
-  const trimmed = text.trim();
-  const fenceMatch = trimmed.match(
-    /^```(?:markdown|md)?\s*\n([\s\S]*?)\n```\s*$/,
-  );
-  if (fenceMatch?.[1]) return fenceMatch[1];
-  return text;
-}
-
-/**
- * Convert arbitrary content to Markdown via a single-shot LLM call.
- *
- * Does **not** short-circuit on `mimeType === "text/markdown"` — tools
- * frequently mislabel their output (e.g. Google Docs' "DocMD" tool returns
- * structured `[H1 ...]` annotations, not real markdown). The mime type is
- * passed in as a hint for the model; the model decides whether the content
- * is already markdown (echo unchanged) or needs converting.
- *
- * - Throws FetchFailureError when the response hits max_tokens (silently
- *   truncating the saved file would be worse than failing loudly).
- * - On transient API errors, logs a warning and returns the raw content so
- *   the import still produces *something* the user can edit.
- */
-export async function convertToMarkdown(
-  content: string,
-  mimeType: string,
-  sourceUrl: string,
-  config: Required<BotholomewConfig>,
-): Promise<string> {
-  if (!config.anthropic_api_key) return content;
-
-  const client = createLlmClient(config);
-  // Conversion is mechanical text-shaping — Haiku (the chunker model) is
-  // plenty smart for this and ~5x faster than Opus on long documents.
-  const model = config.chunker_model || config.model;
-
-  try {
-    const stream = client.messages.stream({
-      model,
-      max_tokens: CONVERTER_MAX_TOKENS,
-      system: CONVERTER_SYSTEM_PROMPT,
-      messages: [
-        {
-          role: "user",
-          content: `Convert this ${mimeType} content to Markdown. Source URL: ${sourceUrl}\n\n${content}`,
-        },
-      ],
-    });
-
-    let charsReceived = 0;
-    let lastLogged = 0;
-    const PROGRESS_INTERVAL_CHARS = 2_000;
-    for await (const event of stream) {
-      if (
-        event.type === "content_block_delta" &&
-        event.delta.type === "text_delta"
-      ) {
-        charsReceived += event.delta.text.length;
-        if (charsReceived - lastLogged >= PROGRESS_INTERVAL_CHARS) {
-          logger.dim(`  ...converted ${charsReceived} chars`);
-          lastLogged = charsReceived;
-        }
-      }
-    }
-
-    const final = await stream.finalMessage();
-
-    if (final.stop_reason === "max_tokens") {
-      throw new FetchFailureError(
-        `Markdown conversion exceeded token budget (max_tokens=${CONVERTER_MAX_TOKENS}). The source document is too large to convert in one pass — try fetching a smaller section or a tool that supports pagination.`,
-      );
-    }
-
-    const text = final.content
-      .flatMap((block) => (block.type === "text" ? [block.text] : []))
-      .join("");
-
-    if (!text.trim()) {
-      logger.warn(
-        "markdown conversion returned empty output — saving raw content",
-      );
-      return content;
-    }
-
-    return stripLeadingMarkdownFence(text);
-  } catch (err) {
-    if (err instanceof FetchFailureError) throw err;
-    logger.warn(
-      `markdown conversion failed (${err instanceof Error ? err.message : String(err)}) — saving raw content`,
-    );
-    return content;
-  }
-}

package/src/context/reindex.ts

@@ -1,198 +0,0 @@
-import { createHash } from "node:crypto";
-import { readFile, stat } from "node:fs/promises";
-import { join } from "node:path";
-import type { BotholomewConfig } from "../config/schemas.ts";
-import { CONTEXT_DIR } from "../constants.ts";
-import { withDb } from "../db/connection.ts";
-import {
-  type ChunkInput,
-  deleteIndexedPath,
-  getIndexedPath,
-  listIndexedPaths,
-  rebuildSearchIndex,
-  upsertChunksForPath,
-} from "../db/embeddings.ts";
-import { logger } from "../utils/logger.ts";
-import { chunkByTextSplit } from "./chunker.ts";
-import { embed as defaultEmbed } from "./embedder.ts";
-import { isContextPathLocked } from "./locks.ts";
-import { listContextDir } from "./store.ts";
-
-/** Embed function shape — exported for tests that want to inject a fake. */
-export type EmbedFn = (
-  texts: string[],
-  config: Required<BotholomewConfig>,
-) => Promise<number[][]>;
-
-/**
- * Walk every textual file under `<projectDir>/context/` and reconcile the
- * disk-backed search index. Adds new files, replaces stale ones whose
- * content_hash changed, and drops index rows for files that no longer exist.
- *
- * Uses the deterministic text splitter (`chunkByTextSplit`) — never the LLM
- * chunker — so a fresh project with no API key still indexes successfully.
- */
-export async function reindexContext(
-  projectDir: string,
-  config: Required<BotholomewConfig>,
-  dbPath: string,
-  opts: {
-    onProgress?: (msg: string) => void;
-    /** Override embed for tests; defaults to the real WASM embedder. */
-    embedFn?: EmbedFn;
-  } = {},
-): Promise<ReindexSummary> {
-  const onProgress = opts.onProgress ?? (() => {});
-  const embed = opts.embedFn ?? defaultEmbed;
-
-  // 1. Walk context/ for every textual file along with its current
-  //    (path, hash, mtime, size). Binary files are intentionally skipped —
-  //    embeddings on bytes are meaningless and would just consume storage.
-  onProgress("scanning files");
-  const onDisk = await collectDiskFiles(projectDir);
-
-  // 2. Read the existing index so we can decide what's add / update / skip /
-  //    remove without re-embedding files that haven't changed.
-  const indexed = await withDb(dbPath, listIndexedPaths);
-  const indexedByPath = new Map(indexed.map((r) => [r.path, r]));
-
-  let added = 0;
-  let updated = 0;
-  let unchanged = 0;
-  let removed = 0;
-  let chunksWritten = 0;
-
-  // 3. For each file on disk: skip if (path, hash) is already indexed and the
-  //    on-disk content hash matches; otherwise (re)embed.
-  for (const file of onDisk) {
-    const existing = indexedByPath.get(file.path);
-    if (existing && existing.content_hash === file.contentHash) {
-      unchanged++;
-      indexedByPath.delete(file.path);
-      continue;
-    }
-
-    onProgress(`embedding ${file.path}`);
-    const text = await readFile(
-      join(projectDir, CONTEXT_DIR, file.path),
-      "utf-8",
-    );
-    const chunks = chunkByTextSplit(text);
-    if (chunks.length === 0) {
-      // Empty/whitespace-only file. Drop any stale rows for it; otherwise
-      // there's nothing to index.
-      if (existing) {
-        await withDb(dbPath, (conn) => deleteIndexedPath(conn, file.path));
-      }
-      continue;
-    }
-    const vectors = await embed(
-      chunks.map((c) => c.content),
-      config,
-    );
-    const inputs: ChunkInput[] = chunks.map((c, i) => ({
-      chunk_index: c.index,
-      chunk_content: c.content,
-      embedding: vectors[i] ?? new Array(config.embedding_dimension).fill(0),
-    }));
-    await withDb(dbPath, (conn) =>
-      upsertChunksForPath(conn, {
-        path: file.path,
-        contentHash: file.contentHash,
-        mtimeMs: file.mtimeMs,
-        sizeBytes: file.sizeBytes,
-        chunks: inputs,
-      }),
-    );
-    if (existing) updated++;
-    else added++;
-    chunksWritten += inputs.length;
-    indexedByPath.delete(file.path);
-  }
-
-  // 4. Anything left in indexedByPath is in the index but not on disk →
-  //    delete its rows so search results don't surface ghost files. Skip
-  //    paths with an active per-path write lock: a worker may have just
-  //    written the file *after* our `collectDiskFiles` walk snapshot, and
-  //    pruning now would drop the index row for a real file. Best-effort —
-  //    the next reindex will reconcile.
-  for (const orphan of indexedByPath.keys()) {
-    if (await isContextPathLocked(projectDir, orphan)) {
-      logger.debug(`reindex: skipping orphan-prune for in-flight ${orphan}`);
-      continue;
-    }
-    await withDb(dbPath, (conn) => deleteIndexedPath(conn, orphan));
-    removed++;
-  }
-
-  if (added + updated + removed > 0) {
-    onProgress("rebuilding FTS index");
-    await withDb(dbPath, rebuildSearchIndex);
-  }
-
-  return { added, updated, unchanged, removed, chunksWritten };
-}
-
-export interface ReindexSummary {
-  added: number;
-  updated: number;
-  unchanged: number;
-  removed: number;
-  chunksWritten: number;
-}
-
-interface DiskFile {
-  path: string;
-  contentHash: string;
-  mtimeMs: number;
-  sizeBytes: number;
-}
-
-async function collectDiskFiles(projectDir: string): Promise<DiskFile[]> {
-  const entries = await listContextDir(projectDir, "", { recursive: true });
-  const out: DiskFile[] = [];
-  for (const e of entries) {
-    if (e.is_directory) continue;
-    if (!e.is_textual) continue;
-    const abs = join(projectDir, CONTEXT_DIR, e.path);
-    let st: Awaited<ReturnType<typeof stat>>;
-    try {
-      st = await stat(abs);
-    } catch (err) {
-      logger.warn(`reindex: skipping ${e.path}: ${err}`);
-      continue;
-    }
-    const buf = await readFile(abs);
-    const contentHash = createHash("sha256").update(buf).digest("hex");
-    out.push({
-      path: e.path,
-      contentHash,
-      mtimeMs: st.mtimeMs,
-      sizeBytes: st.size,
-    });
-  }
-  return out;
-}
-
-/**
- * Drop a single path from the index. Used by file/dir tool callers when
- * they delete or move a file and want the index to reflect it immediately
- * instead of waiting for the next reindex.
- */
-export async function dropIndexedPath(
-  dbPath: string,
-  path: string,
-): Promise<void> {
-  await withDb(dbPath, async (conn) => {
-    await deleteIndexedPath(conn, path);
-    await rebuildSearchIndex(conn);
-  });
-}
-
-export async function getIndexEntry(
-  dbPath: string,
-  path: string,
-): Promise<{ chunks: number } | null> {
-  const row = await withDb(dbPath, (conn) => getIndexedPath(conn, path));
-  return row ? { chunks: row.chunk_count } : null;
-}