botholomew 0.12.3 → 0.13.0

package/src/tools/registry.ts

@@ -1,11 +1,8 @@
 // Capabilities tools
 import { capabilitiesRefreshTool } from "./capabilities/refresh.ts";
 // Context tools
-import { contextListDrivesTool } from "./context/list-drives.ts";
 import { pipeToContextTool } from "./context/pipe.ts";
 import { readLargeResultTool } from "./context/read-large-result.ts";
-import { contextRefreshTool } from "./context/refresh.ts";
-import { contextSearchTool } from "./context/search.ts";
 import { updateBeliefsTool } from "./context/update-beliefs.ts";
 import { updateGoalsTool } from "./context/update-goals.ts";
 // Context — directory operations
@@ -50,6 +47,7 @@ import { viewTaskTool } from "./task/view.ts";
 import { waitTaskTool } from "./task/wait.ts";
 // Thread tools
 import { listThreadsTool } from "./thread/list.ts";
+import { searchThreadsTool } from "./thread/search.ts";
 import { viewThreadTool } from "./thread/view.ts";
 import { registerTool } from "./tool.ts";
 // Worker tools
@@ -66,8 +64,7 @@ export function registerAllTools(): void {
   registerTool(listTasksTool);
   registerTool(viewTaskTool);
 
-  // Context
-  registerTool(contextListDrivesTool);
+  // Context (file/dir + self-reflection)
   registerTool(contextCreateDirTool);
   registerTool(contextTreeTool);
   registerTool(contextDirSizeTool);
@@ -80,8 +77,6 @@ export function registerAllTools(): void {
   registerTool(contextInfoTool);
   registerTool(contextExistsTool);
   registerTool(contextCountLinesTool);
-  registerTool(contextSearchTool);
-  registerTool(contextRefreshTool);
   registerTool(updateBeliefsTool);
   registerTool(updateGoalsTool);
   registerTool(readLargeResultTool);
@@ -108,6 +103,7 @@ export function registerAllTools(): void {
   // Thread
   registerTool(listThreadsTool);
   registerTool(viewThreadTool);
+  registerTool(searchThreadsTool);
 
   // MCP
   registerTool(mcpListToolsTool);

package/src/tools/schedule/create.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { createSchedule } from "../../db/schedules.ts";
+import { createSchedule } from "../../schedules/store.ts";
 import { logger } from "../../utils/logger.ts";
 import type { ToolDefinition } from "../tool.ts";
 
@@ -28,7 +28,7 @@ export const createScheduleTool = {
   inputSchema,
   outputSchema,
   execute: async (input, ctx) => {
-    const schedule = await createSchedule(ctx.conn, {
+    const schedule = await createSchedule(ctx.projectDir, {
       name: input.name,
       description: input.description,
       frequency: input.frequency,

package/src/tools/schedule/list.ts

@@ -1,9 +1,11 @@
 import { z } from "zod";
-import { listSchedules } from "../../db/schedules.ts";
+import { listSchedules } from "../../schedules/store.ts";
 import type { ToolDefinition } from "../tool.ts";
 
 const inputSchema = z.object({
   enabled: z.boolean().optional().describe("Filter by enabled status"),
+  limit: z.number().optional().describe("Max number of schedules to return"),
+  offset: z.number().optional().describe("Skip first N schedules"),
 });
 
 const outputSchema = z.object({
@@ -27,8 +29,10 @@ export const listSchedulesTool = {
   inputSchema,
   outputSchema,
   execute: async (input, ctx) => {
-    const schedules = await listSchedules(ctx.conn, {
+    const schedules = await listSchedules(ctx.projectDir, {
       enabled: input.enabled,
+      limit: input.limit,
+      offset: input.offset,
     });
     return {
       schedules: schedules.map((s) => ({
@@ -36,7 +40,7 @@ export const listSchedulesTool = {
         name: s.name,
         frequency: s.frequency,
         enabled: s.enabled,
-        last_run_at: s.last_run_at?.toISOString() ?? null,
+        last_run_at: s.last_run_at,
       })),
       count: schedules.length,
       is_error: false,

package/src/tools/search/fuse.ts

@@ -2,8 +2,6 @@ import type { RegexpHit } from "./regexp.ts";
 import type { SemanticHit } from "./semantic.ts";
 
 export interface FusedMatch {
-  ref: string;
-  drive: string;
   path: string;
   line: number | null;
   content: string;
@@ -16,17 +14,14 @@ export interface FusedMatch {
 const SNIPPET_MAX = 300;
 
 /**
- * Reciprocal rank fusion of regexp line hits and semantic chunk hits.
+ * Reciprocal rank fusion of regexp line hits and semantic file hits.
  *
- * Each regexp hit becomes its own row. If the file (drive + path) also has a
- * semantic hit, the regexp row picks up that semantic side's RRF contribution
- * and is tagged `match_type: "both"` — exact-line + semantic agreement is
- * the strongest signal.
+ * Each regexp hit becomes its own row. If the same file also has a semantic
+ * hit, the regexp row picks up that semantic side's RRF contribution and is
+ * tagged `match_type: "both"` — exact-line + semantic agreement is the
+ * strongest signal.
  *
- * Semantic hits are emitted as their own rows only for paths with no regexp
- * hit; otherwise the regexp row already represents that file (and is more
- * locatable). This keeps the result list focused without losing pure
- * semantic matches in files the regexp didn't touch.
+ * Semantic hits emit their own rows only for paths with no regexp hit.
  */
 export function fuseRRF(
   regexpHits: RegexpHit[],
@@ -42,26 +37,21 @@ export function fuseRRF(
   for (let i = 0; i < semanticHits.length; i++) {
     const hit = semanticHits[i];
     if (!hit) continue;
-    const key = pathKey(hit.drive, hit.path);
-    if (key == null) continue;
-    const existing = bestSemByPath.get(key);
+    const existing = bestSemByPath.get(hit.path);
     if (!existing || i < existing.rank) {
-      bestSemByPath.set(key, { rank: i, score: hit.score, hit });
+      bestSemByPath.set(hit.path, { rank: i, score: hit.score, hit });
     }
   }
 
   const regexpPaths = new Set<string>();
-  for (const hit of regexpHits) {
-    regexpPaths.add(pathKey(hit.drive, hit.path) ?? "");
-  }
+  for (const hit of regexpHits) regexpPaths.add(hit.path);
 
   const fused: FusedMatch[] = [];
 
   for (let i = 0; i < regexpHits.length; i++) {
     const rx = regexpHits[i];
     if (!rx) continue;
-    const key = pathKey(rx.drive, rx.path) ?? "";
-    const sem = bestSemByPath.get(key);
+    const sem = bestSemByPath.get(rx.path);
     let score = 1 / (k + i + 1);
     let matchType: FusedMatch["match_type"] = "regexp";
     let semanticScore: number | null = null;
@@ -71,8 +61,6 @@ export function fuseRRF(
       semanticScore = round(sem.score);
     }
     fused.push({
-      ref: rx.ref,
-      drive: rx.drive,
       path: rx.path,
       line: rx.line,
       content: rx.content,
@@ -86,14 +74,10 @@ export function fuseRRF(
   for (let i = 0; i < semanticHits.length; i++) {
     const sem = semanticHits[i];
     if (!sem) continue;
-    const key = pathKey(sem.drive, sem.path);
-    if (key == null) continue;
-    if (regexpPaths.has(key)) continue;
+    if (regexpPaths.has(sem.path)) continue;
     const score = 1 / (k + i + 1);
     fused.push({
-      ref: sem.ref,
-      drive: sem.drive ?? "",
-      path: sem.path ?? "",
+      path: sem.path,
       line: null,
       content: sem.chunk_content.slice(0, SNIPPET_MAX),
       context_lines: [],
@@ -107,11 +91,6 @@ export function fuseRRF(
   return fused.slice(0, options.limit);
 }
 
-function pathKey(drive: string | null, path: string | null): string | null {
-  if (!drive || !path) return null;
-  return `${drive}:${path}`;
-}
-
 function round(n: number): number {
   return Math.round(n * 10000) / 10000;
 }

package/src/tools/search/index.ts

@@ -1,16 +1,10 @@
 import { z } from "zod";
-import {
-  listContextItems,
-  listContextItemsByPrefix,
-} from "../../db/context.ts";
 import type { ToolDefinition } from "../tool.ts";
 import { fuseRRF } from "./fuse.ts";
 import { runRegexp } from "./regexp.ts";
 import { runSemantic } from "./semantic.ts";
 
 const MatchSchema = z.object({
-  ref: z.string(),
-  drive: z.string(),
   path: z.string(),
   line: z.number().nullable(),
   content: z.string(),
@@ -25,22 +19,20 @@ const inputSchema = z.object({
     .string()
     .optional()
     .describe(
-      "Natural-language query for semantic + keyword (BM25) hybrid search. Provide alongside `pattern` for the strongest signal — chunks matched by both methods are boosted via reciprocal rank fusion.",
+      "Natural-language query for semantic search. Provide alongside `pattern` for the strongest signal — files matched by both methods float to the top via reciprocal rank fusion.",
     ),
   pattern: z
-    .string()
-    .optional()
-    .describe("Regex pattern for exact text search across context contents."),
-  drive: z
     .string()
     .optional()
     .describe(
-      "Restrict to a single drive (applies to both `query` and `pattern`).",
+      "Regex pattern for exact text search across file contents under context/.",
     ),
-  path: z
+  scope: z
     .string()
     .optional()
-    .describe("Directory prefix within the drive. Requires `drive`."),
+    .describe(
+      "Restrict search to a sub-directory under context/ (e.g. 'notes' to only search context/notes/...).",
+    ),
   glob: z
     .string()
     .optional()
@@ -66,12 +58,13 @@ const outputSchema = z.object({
   is_error: z.boolean(),
   error_type: z.string().optional(),
   message: z.string().optional(),
+  next_action_hint: z.string().optional(),
 });
 
 export const searchTool = {
   name: "search",
   description:
-    "[[ bash equivalent command: grep -r ]] Hybrid search over indexed context. At least one of `query` (natural language → semantic + BM25) or `pattern` (regex over file contents) is required. Pass both for the strongest signal: results matched by both methods float to the top via reciprocal rank fusion. Scoping (`drive`, `path`, `glob`) applies to both sides.",
+    "[[ bash equivalent command: grep -r ]] Hybrid search over files under context/. At least one of `query` (natural language → semantic) or `pattern` (regex over file contents) is required. Pass both for the strongest signal: results matched by both methods float to the top via reciprocal rank fusion. Scoping (`scope`, `glob`) applies to both sides. Note: while a persistent index sidecar is being rebuilt, semantic search re-embeds files on every call — keep result sets small.",
   group: "search",
   inputSchema,
   outputSchema,
@@ -85,43 +78,43 @@ export const searchTool = {
           "Provide at least one of `query` (natural language) or `pattern` (regex). Pass both to fuse semantic and exact-match signals.",
       };
     }
-    if (input.path && !input.drive) {
-      return {
-        matches: [],
-        is_error: true,
-        error_type: "invalid_arguments",
-        message:
-          "`path` requires `drive` — call context_list_drives to see which drives exist, then pass `drive` alongside `path`.",
-      };
+
+    // Validate the regex up front so a malformed pattern returns a
+    // structured error instead of bubbling SyntaxError. Match the shape
+    // of search_threads' invalid_regex response so the agent can recover
+    // identically across both tools.
+    if (input.pattern) {
+      try {
+        new RegExp(input.pattern, input.ignore_case ? "i" : "");
+      } catch (err) {
+        return {
+          matches: [],
+          is_error: true,
+          error_type: "invalid_regex",
+          message: `Could not compile pattern: ${err instanceof Error ? err.message : String(err)}`,
+          next_action_hint:
+            "Double-check the regex; remember `.` is a metacharacter — escape it as `\\.` for a literal dot.",
+        };
+      }
     }
 
     const limit = input.max_results ?? 20;
 
     const regexpHits = input.pattern
-      ? runRegexp(
-          input.drive
-            ? await listContextItemsByPrefix(
-                ctx.conn,
-                input.drive,
-                input.path ?? "/",
-                { recursive: true },
-              )
-            : await listContextItems(ctx.conn),
-          {
-            pattern: input.pattern,
-            glob: input.glob,
-            ignore_case: input.ignore_case,
-            context: input.context,
-            max_results: 100,
-          },
-        )
+      ? await runRegexp(ctx.projectDir, {
+          pattern: input.pattern,
+          scope: input.scope,
+          glob: input.glob,
+          ignore_case: input.ignore_case,
+          context: input.context,
+          max_results: 100,
+        })
       : [];
 
     const semanticHits = input.query
-      ? await runSemantic(ctx, {
+      ? await runSemantic(ctx.projectDir, ctx.config, ctx.dbPath, {
           query: input.query,
-          drive: input.drive,
-          path: input.path,
+          scope: input.scope,
           glob: input.glob,
           limit: 100,
         })

package/src/tools/search/regexp.ts

@@ -1,9 +1,6 @@
-import { formatDriveRef } from "../../context/drives.ts";
-import type { ContextItem } from "../../db/context.ts";
+import { listContextDir, readContextFile } from "../../context/store.ts";
 
 export interface RegexpHit {
-  ref: string;
-  drive: string;
   path: string;
   line: number;
   content: string;
@@ -12,43 +9,58 @@ export interface RegexpHit {
 
 export interface RegexpOptions {
   pattern: string;
+  /** Optional path under context/ to scope the walk (default: whole tree). */
+  scope?: string;
   glob?: string;
   ignore_case?: boolean;
   context?: number;
   max_results?: number;
 }
 
-export function runRegexp(
-  items: ContextItem[],
+/**
+ * Walk every textual file under `context/` (or `context/<scope>/`) and run
+ * `pattern` against each line. Cheap because tools opt into reading content
+ * only for files whose names match an optional glob.
+ */
+export async function runRegexp(
+  projectDir: string,
   options: RegexpOptions,
-): RegexpHit[] {
+): Promise<RegexpHit[]> {
   const flags = options.ignore_case ? "gi" : "g";
   const regex = new RegExp(options.pattern, flags);
   const globRegex = options.glob ? globToRegex(options.glob) : null;
   const contextLines = options.context ?? 0;
   const maxResults = options.max_results ?? 100;
 
-  const hits: RegexpHit[] = [];
-
-  for (const item of items) {
-    if (item.content == null) continue;
+  const entries = await listContextDir(projectDir, options.scope ?? "", {
+    recursive: true,
+  });
 
+  const hits: RegexpHit[] = [];
+  for (const entry of entries) {
+    if (entry.is_directory) continue;
+    if (!entry.is_textual) continue;
     if (globRegex) {
-      const filename = item.path.split("/").pop() ?? "";
+      const filename = entry.path.split("/").pop() ?? "";
       if (!globRegex.test(filename)) continue;
     }
 
-    const lines = item.content.split("\n");
+    let content: string;
+    try {
+      content = await readContextFile(projectDir, entry.path);
+    } catch {
+      continue;
+    }
+    const lines = content.split("\n");
     for (let i = 0; i < lines.length; i++) {
       regex.lastIndex = 0;
       const line = lines[i];
-      if (line !== undefined && regex.test(line)) {
+      if (line === undefined) continue;
+      if (regex.test(line)) {
         const start = Math.max(0, i - contextLines);
         const end = Math.min(lines.length, i + contextLines + 1);
         hits.push({
-          ref: formatDriveRef(item),
-          drive: item.drive,
-          path: item.path,
+          path: entry.path,
           line: i + 1,
           content: line,
           context_lines: lines.slice(start, end),

package/src/tools/search/semantic.ts

@@ -1,81 +1,167 @@
-import { formatDriveRef } from "../../context/drives.ts";
-import { embedSingle } from "../../context/embedder.ts";
-import { type HybridSearchResult, hybridSearch } from "../../db/embeddings.ts";
-import type { ToolContext } from "../tool.ts";
+import type { BotholomewConfig } from "../../config/schemas.ts";
+import { embed, embedSingle } from "../../context/embedder.ts";
+import { listContextDir, readContextFile } from "../../context/store.ts";
+import { withDb } from "../../db/connection.ts";
+import { indexStats, searchSemantic } from "../../db/embeddings.ts";
 import { globToRegex } from "./regexp.ts";
 
 export interface SemanticHit {
-  ref: string;
-  drive: string | null;
-  path: string | null;
-  context_item_id: string;
+  path: string;
   chunk_index: number;
-  title: string;
   chunk_content: string;
   score: number;
 }
 
 export interface SemanticOptions {
   query: string;
-  drive?: string;
-  path?: string;
+  scope?: string;
   glob?: string;
   limit?: number;
 }
 
+// On-the-fly fallback (used when the index sidecar is empty / stale).
+// One chunk per file truncated to MAX_CHARS; the indexed path is much faster
+// and supports proper chunking via `botholomew context reindex`.
+const MAX_CHARS = 4_000;
+const MAX_FILES_TO_EMBED = 200;
+
 /**
- * Run the embedding + hybrid-search pipeline. Scoping (`drive` / `path` /
- * `glob`) is applied as a *post-filter* on results so the caller gets
- * consistent behavior whether they used the regex side, the semantic side,
- * or both.
+ * Semantic search over `context/`. Prefers the persistent index sidecar
+ * (`context_index` table, populated by `botholomew context reindex`) when
+ * it has rows. Falls back to embedding files on the fly so a fresh project
+ * still gets useful results before the user has reindexed once.
  */
 export async function runSemantic(
-  ctx: ToolContext,
+  projectDir: string,
+  config: Required<BotholomewConfig>,
+  dbPath: string | null,
   options: SemanticOptions,
 ): Promise<SemanticHit[]> {
-  const queryVec = await embedSingle(options.query, ctx.config);
-  const results = await hybridSearch(
-    ctx.conn,
-    options.query,
-    queryVec,
-    options.limit ?? 100,
+  if (dbPath) {
+    const indexed = await tryIndexedSearch(dbPath, config, options);
+    if (indexed) return indexed;
+  }
+  return runOnTheFly(projectDir, config, options);
+}
+
+async function tryIndexedSearch(
+  dbPath: string,
+  config: Required<BotholomewConfig>,
+  options: SemanticOptions,
+): Promise<SemanticHit[] | null> {
+  let stats: Awaited<ReturnType<typeof indexStats>>;
+  try {
+    stats = await withDb(dbPath, indexStats);
+  } catch {
+    return null;
+  }
+  if (stats.embedded === 0) return null;
+
+  const queryVec = await embedSingle(options.query, config);
+  const limit = options.limit ?? 100;
+  const rows = await withDb(dbPath, (conn) =>
+    searchSemantic(conn, queryVec, limit * 4),
   );
 
-  return results.filter((r) => matchesScope(r, options)).map(toHit);
+  const globRegex = options.glob ? globToRegex(options.glob) : null;
+  const scope = options.scope
+    ? options.scope.endsWith("/")
+      ? options.scope
+      : `${options.scope}/`
+    : null;
+
+  const filtered: SemanticHit[] = [];
+  for (const r of rows) {
+    if (scope && !r.path.startsWith(scope) && r.path !== options.scope) {
+      continue;
+    }
+    if (globRegex) {
+      const filename = r.path.split("/").pop() ?? "";
+      if (!globRegex.test(filename)) continue;
+    }
+    filtered.push({
+      path: r.path,
+      chunk_index: r.chunk_index,
+      chunk_content: r.chunk_content,
+      score: r.score,
+    });
+    if (filtered.length >= limit) break;
+  }
+  return filtered;
 }
 
-function matchesScope(
-  result: HybridSearchResult,
+async function runOnTheFly(
+  projectDir: string,
+  config: Required<BotholomewConfig>,
   options: SemanticOptions,
-): boolean {
-  if (options.drive && result.drive !== options.drive) return false;
-  if (options.path && result.path) {
-    const prefix = options.path.endsWith("/")
-      ? options.path
-      : `${options.path}/`;
-    if (result.path !== options.path && !result.path.startsWith(prefix)) {
-      return false;
+): Promise<SemanticHit[]> {
+  const entries = await listContextDir(projectDir, options.scope ?? "", {
+    recursive: true,
+  });
+  const globRegex = options.glob ? globToRegex(options.glob) : null;
+
+  const candidates: Array<{ path: string; content: string }> = [];
+  for (const entry of entries) {
+    if (entry.is_directory) continue;
+    if (!entry.is_textual) continue;
+    if (globRegex) {
+      const filename = entry.path.split("/").pop() ?? "";
+      if (!globRegex.test(filename)) continue;
+    }
+    let content: string;
+    try {
+      content = await readContextFile(projectDir, entry.path);
+    } catch {
+      continue;
     }
+    if (content.trim().length === 0) continue;
+    candidates.push({
+      path: entry.path,
+      content: content.slice(0, MAX_CHARS),
+    });
+    if (candidates.length >= MAX_FILES_TO_EMBED) break;
   }
-  if (options.glob && result.path) {
-    const filename = result.path.split("/").pop() ?? "";
-    if (!globToRegex(options.glob).test(filename)) return false;
+
+  if (candidates.length === 0) return [];
+
+  const [queryVec, fileVecs] = await Promise.all([
+    embedSingle(options.query, config),
+    embed(
+      candidates.map((c) => c.content),
+      config,
+    ),
+  ]);
+
+  const limit = options.limit ?? 100;
+  const scored: SemanticHit[] = [];
+  for (let i = 0; i < candidates.length; i++) {
+    const c = candidates[i];
+    const v = fileVecs[i];
+    if (!c || !v) continue;
+    const score = cosine(queryVec, v);
+    scored.push({
+      path: c.path,
+      chunk_index: 0,
+      chunk_content: c.content,
+      score,
+    });
   }
-  return true;
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, limit);
 }
 
-function toHit(r: HybridSearchResult): SemanticHit {
-  return {
-    ref:
-      r.drive && r.path
-        ? formatDriveRef({ drive: r.drive, path: r.path })
-        : r.context_item_id,
-    drive: r.drive,
-    path: r.path,
-    context_item_id: r.context_item_id,
-    chunk_index: r.chunk_index,
-    title: r.title,
-    chunk_content: r.chunk_content ?? "",
-    score: r.score,
-  };
+function cosine(a: number[], b: number[]): number {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  const len = Math.min(a.length, b.length);
+  for (let i = 0; i < len; i++) {
+    const ai = a[i] ?? 0;
+    const bi = b[i] ?? 0;
+    dot += ai * bi;
+    na += ai * ai;
+    nb += bi * bi;
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
 }

package/src/tools/skill/delete.ts

@@ -19,7 +19,7 @@ const outputSchema = z.object({
 export const skillDeleteTool = {
   name: "skill_delete",
   description:
-    "[[ bash equivalent command: rm ]] Delete a skill file (user-defined slash command) by name. The file is removed from .botholomew/skills/. Returns a not_found error with the list of available names when the skill doesn't exist.",
+    "[[ bash equivalent command: rm ]] Delete a skill file (user-defined slash command) by name. The file is removed from skills/. Returns a not_found error with the list of available names when the skill doesn't exist.",
   group: "skill",
   inputSchema,
   outputSchema,

package/src/tools/skill/list.ts

@@ -33,7 +33,7 @@ const outputSchema = z.object({
 export const skillListTool = {
   name: "skill_list",
   description:
-    "[[ bash equivalent command: ls ]] List skills (user-defined slash commands) loaded from .botholomew/skills/. Returns name, description, argument names, and file path for each.",
+    "[[ bash equivalent command: ls ]] List skills (user-defined slash commands) loaded from skills/. Returns name, description, argument names, and file path for each.",
   group: "skill",
   inputSchema,
   outputSchema,

package/src/tools/skill/write.ts

@@ -67,7 +67,7 @@ const outputSchema = z.object({
 export const skillWriteTool = {
   name: "skill_write",
   description:
-    "[[ bash equivalent command: tee ]] Create or overwrite a skill file (user-defined slash command) at .botholomew/skills/<name>.md. Fails with path_conflict when the file exists unless on_conflict='overwrite'. Reserved names (help, skills, clear, exit) are rejected. The generated file is parsed to validate before being written.",
+    "[[ bash equivalent command: tee ]] Create or overwrite a skill file (user-defined slash command) at skills/<name>.md. Fails with path_conflict when the file exists unless on_conflict='overwrite'. Reserved names (help, skills, clear, exit) are rejected. The generated file is parsed to validate before being written.",
   group: "skill",
   inputSchema,
   outputSchema,