botholomew 0.18.6 → 0.19.3

package/src/tui/markdownTables.ts

@@ -0,0 +1,288 @@
+/**
+ * GFM table extraction + width-aware ANSI rendering.
+ *
+ * `Bun.markdown.ansi` renders tables at their natural width and ignores the
+ * caller's column budget, so wide tables get hard-wrapped mid-cell by
+ * `wrap-ansi` in the detail pane. We pre-extract table blocks, render them
+ * ourselves at a width that fits, and let `Bun.markdown.ansi` handle the rest.
+ */
+
+export type Align = "left" | "center" | "right";
+
+export interface TableBlock {
+  /** First line index (inclusive) of the table in the original text. */
+  start: number;
+  /** Last line index (inclusive). */
+  end: number;
+  /** First row is the header. */
+  rows: string[][];
+  aligns: Align[];
+}
+
+const DIM_ON = "\x1b[2m";
+const BOLD_ON = "\x1b[1m";
+const RESET = "\x1b[0m";
+
+const SEPARATOR_CELL_RE = /^\s*:?-{1,}:?\s*$/;
+const FENCE_RE = /^\s{0,3}(```|~~~)/;
+
+export function extractTableBlocks(text: string): TableBlock[] {
+  const lines = text.split("\n");
+  const blocks: TableBlock[] = [];
+  let inFence = false;
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i] ?? "";
+    if (FENCE_RE.test(line)) {
+      inFence = !inFence;
+      i++;
+      continue;
+    }
+    if (inFence || !looksLikePipeRow(line)) {
+      i++;
+      continue;
+    }
+    const sep = lines[i + 1] ?? "";
+    if (!looksLikePipeRow(sep)) {
+      i++;
+      continue;
+    }
+    const sepCells = splitRow(sep);
+    if (!sepCells.every((c) => SEPARATOR_CELL_RE.test(c))) {
+      i++;
+      continue;
+    }
+    const header = splitRow(line);
+    const colCount = Math.max(header.length, sepCells.length);
+    const aligns: Align[] = sepCells.slice(0, colCount).map(parseAlignCell);
+    while (aligns.length < colCount) aligns.push("left");
+
+    const rows: string[][] = [normalizeRow(header, colCount)];
+    let j = i + 2;
+    while (j < lines.length) {
+      const body = lines[j] ?? "";
+      if (!looksLikePipeRow(body)) break;
+      // A new separator (consecutive tables) terminates this one.
+      if (splitRow(body).every((c) => SEPARATOR_CELL_RE.test(c))) break;
+      rows.push(normalizeRow(splitRow(body), colCount));
+      j++;
+    }
+
+    blocks.push({ start: i, end: j - 1, rows, aligns });
+    i = j;
+  }
+  return blocks;
+}
+
+export function renderTable(
+  rows: string[][],
+  aligns: Align[],
+  width: number,
+): string {
+  if (rows.length === 0) return "";
+  const colCount = rows[0]?.length ?? 0;
+  if (colCount === 0) return "";
+
+  const plain = rows.map((r) => r.map(stripInlineMarkdown));
+
+  // Per-column natural width (max visible width across all cells).
+  const naturalWidths: number[] = [];
+  for (let c = 0; c < colCount; c++) {
+    let w = 1;
+    for (const row of plain) {
+      const cell = row[c] ?? "";
+      if (visibleWidth(cell) > w) w = visibleWidth(cell);
+    }
+    naturalWidths.push(w);
+  }
+
+  // Overhead: leading "│ " + trailing " │" + " │ " between cols.
+  const borderOverhead = colCount * 3 + 1;
+  const naturalTotal =
+    naturalWidths.reduce((a, b) => a + b, 0) + borderOverhead;
+
+  let colWidths: number[];
+  if (naturalTotal <= width || width <= 0) {
+    colWidths = naturalWidths;
+  } else {
+    colWidths = shrinkColumns(naturalWidths, width - borderOverhead);
+  }
+
+  const lines: string[] = [];
+  lines.push(borderLine("┌", "┬", "┐", colWidths));
+  for (let r = 0; r < plain.length; r++) {
+    const cells = plain[r] ?? [];
+    const isHeader = r === 0;
+    lines.push(dataLine(cells, aligns, colWidths, isHeader));
+    if (isHeader) {
+      lines.push(borderLine("├", "┼", "┤", colWidths));
+    }
+  }
+  lines.push(borderLine("└", "┴", "┘", colWidths));
+  return lines.join("\n");
+}
+
+function looksLikePipeRow(line: string): boolean {
+  // A GFM table row contains at least one unescaped pipe and (after trimming
+  // surrounding whitespace + optional pipes) is non-empty.
+  const stripped = line.trim();
+  if (stripped === "") return false;
+  if (!stripped.includes("|")) return false;
+  return true;
+}
+
+function splitRow(line: string): string[] {
+  let s = line.trim();
+  if (s.startsWith("|")) s = s.slice(1);
+  if (s.endsWith("|") && !s.endsWith("\\|")) s = s.slice(0, -1);
+  const cells: string[] = [];
+  let buf = "";
+  for (let i = 0; i < s.length; i++) {
+    const ch = s[i];
+    if (ch === "\\" && s[i + 1] === "|") {
+      buf += "|";
+      i++;
+      continue;
+    }
+    if (ch === "|") {
+      cells.push(buf.trim());
+      buf = "";
+      continue;
+    }
+    buf += ch;
+  }
+  cells.push(buf.trim());
+  return cells;
+}
+
+function parseAlignCell(cell: string): Align {
+  const c = cell.trim();
+  const left = c.startsWith(":");
+  const right = c.endsWith(":");
+  if (left && right) return "center";
+  if (right) return "right";
+  return "left";
+}
+
+function normalizeRow(cells: string[], colCount: number): string[] {
+  const out = cells.slice(0, colCount);
+  while (out.length < colCount) out.push("");
+  return out;
+}
+
+function shrinkColumns(natural: number[], budget: number): number[] {
+  const MIN = 3;
+  const n = natural.length;
+  if (budget < n * MIN) {
+    // Not enough room even for ellipsis everywhere — give each column MIN
+    // and let the caller deal with overflow. (Detail pane minimum is much
+    // wider than this in practice.)
+    return new Array(n).fill(MIN);
+  }
+  const total = natural.reduce((a, b) => a + b, 0) || 1;
+  const raw = natural.map((w) => (w * budget) / total);
+  const floored = raw.map((v) => Math.max(MIN, Math.floor(v)));
+  let used = floored.reduce((a, b) => a + b, 0);
+  // Distribute the remainder to columns with the largest fractional part.
+  const remainders = raw
+    .map((v, i) => ({ i, frac: v - Math.floor(v) }))
+    .sort((a, b) => b.frac - a.frac);
+  let k = 0;
+  while (used < budget && k < remainders.length * 4) {
+    const idx = remainders[k % remainders.length]?.i ?? 0;
+    floored[idx] = (floored[idx] ?? MIN) + 1;
+    used++;
+    k++;
+  }
+  // If we overshot due to MIN clamping, trim from the widest column(s).
+  while (used > budget) {
+    let widest = 0;
+    for (let i = 1; i < n; i++) {
+      if ((floored[i] ?? 0) > (floored[widest] ?? 0)) widest = i;
+    }
+    if ((floored[widest] ?? 0) <= MIN) break;
+    floored[widest] = (floored[widest] ?? 0) - 1;
+    used--;
+  }
+  return floored;
+}
+
+function borderLine(
+  left: string,
+  mid: string,
+  right: string,
+  widths: number[],
+): string {
+  const segs = widths.map((w) => "─".repeat(w + 2));
+  return DIM_ON + left + segs.join(mid) + right + RESET;
+}
+
+function dataLine(
+  cells: string[],
+  aligns: Align[],
+  widths: number[],
+  bold: boolean,
+): string {
+  const parts: string[] = [];
+  parts.push(`${DIM_ON}│${RESET}`);
+  for (let i = 0; i < widths.length; i++) {
+    const w = widths[i] ?? 0;
+    const align = aligns[i] ?? "left";
+    const raw = cells[i] ?? "";
+    const fitted = padCell(raw, w, align);
+    const styled = bold ? `${BOLD_ON}${fitted}${RESET}` : fitted;
+    parts.push(` ${styled} `);
+    parts.push(`${DIM_ON}│${RESET}`);
+  }
+  return parts.join("");
+}
+
+function padCell(text: string, width: number, align: Align): string {
+  const truncated = truncateToWidth(text, width);
+  const pad = width - visibleWidth(truncated);
+  if (pad <= 0) return truncated;
+  if (align === "right") return " ".repeat(pad) + truncated;
+  if (align === "center") {
+    const l = Math.floor(pad / 2);
+    const r = pad - l;
+    return " ".repeat(l) + truncated + " ".repeat(r);
+  }
+  return truncated + " ".repeat(pad);
+}
+
+function truncateToWidth(text: string, width: number): string {
+  if (width <= 0) return "";
+  if (visibleWidth(text) <= width) return text;
+  if (width === 1) return "…";
+  const chars = Array.from(text);
+  let out = "";
+  let used = 0;
+  for (const ch of chars) {
+    if (used + 1 > width - 1) break;
+    out += ch;
+    used++;
+  }
+  return `${out}…`;
+}
+
+function visibleWidth(text: string): number {
+  // Cell text has no ANSI (we strip markdown markers before measuring), so
+  // codepoint count is sufficient. East-Asian double-width chars would be
+  // undercounted; out of scope for v1.
+  return Array.from(text).length;
+}
+
+function stripInlineMarkdown(text: string): string {
+  // Strip a small set of inline markers so cell width measurement matches what
+  // the user sees. Order matters: longer markers first.
+  let s = text;
+  s = s.replace(/`([^`]+)`/g, "$1");
+  s = s.replace(/\*\*([^*]+)\*\*/g, "$1");
+  s = s.replace(/__([^_]+)__/g, "$1");
+  s = s.replace(/~~([^~]+)~~/g, "$1");
+  s = s.replace(/(^|[^*])\*([^*\n]+)\*/g, "$1$2");
+  s = s.replace(/(^|[^_])_([^_\n]+)_/g, "$1$2");
+  // Collapse \| escapes that survived splitRow.
+  s = s.replace(/\\\|/g, "|");
+  return s;
+}

package/src/utils/title.ts

@@ -1,45 +1,44 @@
+import { generateText } from "ai";
 import type { BotholomewConfig } from "../config/schemas.ts";
+import {
+  buildProviderOptions,
+  formatLlmError,
+  getLanguageModel,
+  getMaxInputTokens,
+} from "../llm/index.ts";
 import { updateThreadTitle } from "../threads/store.ts";
-import { createLlmClient } from "../worker/llm-client.ts";
 import { logger } from "./logger.ts";
 
 /**
- * Generate a short title for a thread using the chunker model (Haiku).
- * Fire-and-forget — errors are logged and never propagated. Writes the
- * title back to the thread's CSV file by rewriting the thread_meta row.
+ * Generate a short title for a thread using the chunker model.
+ * Fire-and-forget — errors are logged and never propagated.
  */
 export async function generateThreadTitle(
-  config: Required<BotholomewConfig>,
+  config: BotholomewConfig,
   projectDir: string,
   threadId: string,
   context: string,
 ): Promise<void> {
   try {
-    const client = createLlmClient(config);
+    const model = getLanguageModel(config.chunker_llm);
+    const numCtx = await getMaxInputTokens(config.chunker_llm);
 
-    const response = await client.messages.create({
-      model: config.chunker_model,
-      max_tokens: 50,
+    const { text } = await generateText({
+      model,
+      maxOutputTokens: 50,
       system:
         "You are a title generator. The user will provide the first message from a conversation. Output a short descriptive title (5-8 words). Output ONLY the title, nothing else.",
-      messages: [
-        {
-          role: "user",
-          content: `Generate a title for this message:\n\n"${context}"`,
-        },
-      ],
+      prompt: `Generate a title for this message:\n\n"${context}"`,
+      providerOptions: buildProviderOptions(config.chunker_llm, numCtx),
     });
 
-    const title = response.content
-      .filter((b) => b.type === "text")
-      .map((b) => b.text)
-      .join("")
-      .trim();
-
+    const title = text.trim();
     if (title) {
       await updateThreadTitle(projectDir, threadId, title);
     }
   } catch (err) {
-    logger.warn(`Failed to generate thread title: ${err}`);
+    logger.warn(
+      `Failed to generate thread title: ${formatLlmError(err, config.chunker_llm)}`,
+    );
   }
 }

package/src/worker/context.ts

@@ -1,113 +1,86 @@
-import Anthropic from "@anthropic-ai/sdk";
-import type { MessageParam } from "@anthropic-ai/sdk/resources/messages";
+import type { ModelMessage } from "ai";
+import type { LlmBlock } from "../config/schemas.ts";
+import { getMaxInputTokens as llmGetMaxInputTokens } from "../llm/index.ts";
 import { logger } from "../utils/logger.ts";
 
 /** Rough estimate: ~4 characters per token for English text */
 const CHARS_PER_TOKEN = 4;
 
-/** Fallback if the models API call fails */
-const DEFAULT_MAX_INPUT_TOKENS = 200_000;
-
 /** Reserve this fraction of the context window for safety margin */
 const HEADROOM_FRACTION = 0.1;
 
 /** Maximum characters for a single tool result before truncation */
 const MAX_TOOL_RESULT_CHARS = 50_000;
 
-/** Cache model max_input_tokens to avoid repeated API calls */
-const modelTokenCache = new Map<string, number>();
-
-/**
- * Look up the model's max input tokens via the Anthropic Models API.
- * Results are cached per model ID for the lifetime of the process.
- */
-export async function getMaxInputTokens(
-  apiKey: string | undefined,
-  model: string,
-): Promise<number> {
-  const cached = modelTokenCache.get(model);
-  if (cached !== undefined) return cached;
-
-  try {
-    const client = new Anthropic({ apiKey: apiKey || undefined });
-    const info = await client.beta.models.retrieve(model);
-    const limit = info.max_input_tokens ?? DEFAULT_MAX_INPUT_TOKENS;
-    modelTokenCache.set(model, limit);
-    return limit;
-  } catch (err) {
-    logger.debug(`Failed to retrieve model info for ${model}: ${err}`);
-    modelTokenCache.set(model, DEFAULT_MAX_INPUT_TOKENS);
-    return DEFAULT_MAX_INPUT_TOKENS;
-  }
+/** Re-export so call sites have a single entry point. */
+export function getMaxInputTokens(cfg: LlmBlock): Promise<number> {
+  return llmGetMaxInputTokens(cfg);
 }
 
 function estimateTokens(text: string): number {
   return Math.ceil(text.length / CHARS_PER_TOKEN);
 }
 
-function messageChars(msg: MessageParam): number {
+function messageChars(msg: ModelMessage): number {
   if (typeof msg.content === "string") return msg.content.length;
-  if (Array.isArray(msg.content)) {
-    let total = 0;
-    for (const block of msg.content) {
-      if ("text" in block && typeof block.text === "string") {
-        total += block.text.length;
-      } else if ("content" in block && typeof block.content === "string") {
-        total += block.content.length;
-      } else {
-        // tool_use blocks with input, etc.
-        total += JSON.stringify(block).length;
-      }
+  if (!Array.isArray(msg.content)) return 0;
+  let total = 0;
+  for (const block of msg.content) {
+    const b = block as Record<string, unknown>;
+    if (typeof b.text === "string") {
+      total += b.text.length;
+    } else if (b.type === "tool-result" && typeof b.output === "object") {
+      const out = b.output as { value?: unknown };
+      total +=
+        typeof out.value === "string"
+          ? out.value.length
+          : JSON.stringify(out.value ?? "").length;
+    } else {
+      total += JSON.stringify(b).length;
     }
-    return total;
   }
-  return JSON.stringify(msg.content).length;
+  return total;
 }
 
 /**
- * Truncate individual tool results that are excessively large.
- * Mutates messages in-place.
+ * Truncate individual tool results that are excessively large. Mutates in-place.
  */
-function truncateToolResults(messages: MessageParam[]): void {
+function truncateToolResults(messages: ModelMessage[]): void {
   for (const msg of messages) {
+    if (msg.role !== "tool") continue;
     if (!Array.isArray(msg.content)) continue;
     for (const block of msg.content) {
-      if (
-        "type" in block &&
-        block.type === "tool_result" &&
-        "content" in block &&
-        typeof block.content === "string" &&
-        block.content.length > MAX_TOOL_RESULT_CHARS
-      ) {
-        const original = block.content.length;
-        (block as { content: string }).content =
-          block.content.slice(0, MAX_TOOL_RESULT_CHARS) +
-          `\n\n[truncated: ${original} chars → ${MAX_TOOL_RESULT_CHARS} chars]`;
-      }
+      const b = block as {
+        type?: string;
+        output?: { type?: string; value?: unknown };
+      };
+      if (b.type !== "tool-result" || !b.output) continue;
+      const out = b.output;
+      if (typeof out.value !== "string") continue;
+      if (out.value.length <= MAX_TOOL_RESULT_CHARS) continue;
+      const original = out.value.length;
+      out.value =
+        out.value.slice(0, MAX_TOOL_RESULT_CHARS) +
+        `\n\n[truncated: ${original} chars → ${MAX_TOOL_RESULT_CHARS} chars]`;
     }
   }
 }
 
 /**
  * Ensure the conversation fits within the context window.
- * Strategy:
- * 1. Truncate oversized tool results
- * 2. If still too large, drop oldest assistant/tool pairs from the middle
- *    (keeping the first user message and recent messages)
- *
- * Mutates messages in-place and returns the array.
+ * 1) Truncate oversized tool results in place.
+ * 2) If still too large, drop oldest messages from the middle (keeping the
+ *    first user message and recent messages).
  */
 export function fitToContextWindow(
-  messages: MessageParam[],
+  messages: ModelMessage[],
   systemPrompt: string,
   maxInputTokens: number,
-): MessageParam[] {
-  // Step 1: truncate oversized tool results
+): ModelMessage[] {
   truncateToolResults(messages);
 
-  // Step 2: estimate total tokens
   const systemTokens = estimateTokens(systemPrompt);
-  const responseBuffer = 4096; // max_tokens for the response
+  const responseBuffer = 4096;
   const headroom = Math.ceil(maxInputTokens * HEADROOM_FRACTION);
 
   const budget = maxInputTokens - systemTokens - responseBuffer - headroom;
@@ -121,16 +94,11 @@ export function fitToContextWindow(
   let totalChars = messages.reduce((sum, m) => sum + messageChars(m), 0);
   let totalTokens = Math.ceil(totalChars / CHARS_PER_TOKEN);
 
-  if (totalTokens <= budget) {
-    return messages;
-  }
+  if (totalTokens <= budget) return messages;
 
-  // Step 3: drop oldest message pairs from the middle until we fit.
-  // Keep messages[0] (initial user message) and remove from index 1 onward.
   let dropped = 0;
   while (totalTokens > budget && messages.length > 2) {
-    // Remove the oldest non-first message (index 1)
-    const removed = messages.splice(1, 1)[0] as MessageParam;
+    const removed = messages.splice(1, 1)[0] as ModelMessage;
     totalChars -= messageChars(removed);
     totalTokens = Math.ceil(totalChars / CHARS_PER_TOKEN);
     dropped++;