opencode-semantic-search 0.1.0

package/src/tools/index_status.ts

@@ -0,0 +1,73 @@
+import type { ToolContext } from "@opencode-ai/plugin";
+import { tool } from "@opencode-ai/plugin";
+import type { RuntimeContext } from "../runtime";
+
+function formatBackgroundLine(runtime: RuntimeContext): string {
+  const p = runtime.indexingProgress;
+  if (p.phase === "idle") {
+    return "idle";
+  }
+  const parts: string[] = [p.phase];
+  if (p.total > 0) {
+    parts.push(`${p.current}/${p.total}`);
+  }
+  if (p.label) {
+    parts.push(`(${p.label})`);
+  }
+  if (p.source) {
+    parts.push(`[${p.source}]`);
+  }
+  return parts.join(" ");
+}
+
+export async function executeIndexStatus(
+  runtime: RuntimeContext,
+  ctx?: Pick<ToolContext, "metadata">
+): Promise<string> {
+  ctx?.metadata?.({ title: "index_status", metadata: { phase: "collecting" } });
+  const stats = runtime.store.stats();
+  const providerHealthy = await runtime.embedder.healthcheck();
+  await runtime.logger.debug("tool.index_status", {
+    message: "Index status requested",
+    extra: { filesIndexed: stats.files, chunks: stats.chunks, providerHealthy }
+  });
+  const bg = formatBackgroundLine(runtime);
+  const lastSync = stats.lastSync ?? "never";
+  const endpoint = runtime.config.embedding.api_base;
+  const model = runtime.config.embedding.model;
+  const health = providerHealthy ? "healthy" : "unreachable or unhealthy";
+  const body = [
+    "## Semantic index status",
+    "",
+    "| Field | Value |",
+    "|-------|-------|",
+    `| Files indexed | ${stats.files} |`,
+    `| Chunks | ${stats.chunks} |`,
+    `| Last sync | ${lastSync} |`,
+    `| Embedding model | ${model} |`,
+    `| Embedding endpoint | ${endpoint} |`,
+    `| Embedder | ${health} |`,
+    `| Background indexing | ${bg} |`,
+    "",
+  ].join("\n");
+  ctx?.metadata?.({
+    title: "index_status",
+    metadata: {
+      phase: "ready",
+      files: stats.files,
+      chunks: stats.chunks,
+      embedder_healthy: providerHealthy,
+    },
+  });
+  return body;
+}
+
+export function createIndexStatusTool(runtime: RuntimeContext) {
+  return tool({
+    description: "Check semantic index health and coverage.",
+    args: {},
+    async execute(_args, toolCtx) {
+      return executeIndexStatus(runtime, toolCtx);
+    },
+  });
+}

package/src/tools/reindex.ts

@@ -0,0 +1,71 @@
+import type { ToolContext } from "@opencode-ai/plugin";
+import { tool } from "@opencode-ai/plugin";
+import { deltaSync } from "../indexer/delta";
+import { applyDeltaProgress, type RuntimeContext } from "../runtime";
+import { showIndexingCompleteToast } from "../tui_toast";
+
+const metadataPollMs = 400;
+
+export async function executeReindex(
+  runtime: RuntimeContext,
+  ctx: Pick<ToolContext, "metadata">
+): Promise<string> {
+  ctx.metadata({ title: "reindex", metadata: { phase: "clearing_index_tables" } });
+  runtime.store.db.exec("DELETE FROM chunks_fts;");
+  runtime.store.db.exec("DELETE FROM chunk_vec;");
+  runtime.store.db.exec("DELETE FROM chunks;");
+  runtime.store.db.exec("DELETE FROM files;");
+  await runtime.logger.info("tool.reindex", { message: "Forced reindex requested" });
+  const reindexStartedAt = Date.now();
+
+  const tick = (): void => {
+    const p = runtime.indexingProgress;
+    ctx.metadata({
+      title: "reindex",
+      metadata: {
+        phase: p.phase,
+        file: p.label,
+        progress: p.total > 0 ? `${p.current}/${p.total}` : undefined,
+      },
+    });
+  };
+  tick();
+  const interval = setInterval(tick, metadataPollMs);
+  try {
+    await deltaSync(runtime.worktree, runtime.store, runtime.embedder, runtime.config, {
+      logger: runtime.logger,
+      onProgress: (update) => applyDeltaProgress(runtime, update, "reindex"),
+    });
+  } finally {
+    clearInterval(interval);
+  }
+  const stats = runtime.store.stats();
+  ctx.metadata({
+    title: "reindex",
+    metadata: {
+      phase: "complete",
+      files_indexed: stats.files,
+      chunks: stats.chunks,
+      last_sync: stats.lastSync,
+    },
+  });
+  void showIndexingCompleteToast(runtime, {
+    title: "Reindex complete",
+    files: stats.files,
+    chunks: stats.chunks,
+    elapsedMs: Date.now() - reindexStartedAt,
+    failedFiles: runtime.indexingProgress.failedFiles,
+    flavor: "reindex",
+  });
+  return JSON.stringify(stats);
+}
+
+export function createReindexTool(runtime: RuntimeContext) {
+  return tool({
+    description: "Force a full semantic index rebuild.",
+    args: {},
+    async execute(_args, toolCtx) {
+      return executeReindex(runtime, toolCtx);
+    },
+  });
+}

package/src/tools/semantic_search.ts

@@ -0,0 +1,91 @@
+import type { ToolContext } from "@opencode-ai/plugin";
+import { tool } from "@opencode-ai/plugin";
+import { hybridSearch } from "../search/hybrid";
+import type { RuntimeContext } from "../runtime";
+import { buildContextPreview } from "../search/context";
+
+export interface SemanticSearchToolArgs {
+  query: string;
+  top_k?: number;
+  threshold?: number;
+  path?: string;
+}
+
+export async function executeSemanticSearch(
+  runtime: RuntimeContext,
+  args: SemanticSearchToolArgs,
+  ctx?: Pick<ToolContext, "metadata">
+): Promise<string> {
+  const startedAt = Date.now();
+  const topK = args.top_k ?? runtime.config.search.top_k;
+  const originalThreshold = runtime.config.search.similarity_threshold;
+  if (typeof args.threshold === "number") runtime.config.search.similarity_threshold = args.threshold;
+  await runtime.logger.info("tool.semantic_search", {
+    message: "Semantic search requested",
+    extra: {
+      queryLength: args.query.length,
+      topK,
+      threshold: args.threshold ?? originalThreshold,
+      path: args.path
+    }
+  });
+  try {
+    ctx?.metadata?.({
+      title: "semantic_search",
+      metadata: { phase: "embedding_query" },
+    });
+    const chunkCount = runtime.store.stats().chunks;
+    ctx?.metadata?.({
+      title: "semantic_search",
+      metadata: { phase: "searching_index", indexed_chunks: chunkCount },
+    });
+    const ranked = await hybridSearch(runtime.store, runtime.embedder, args.query, topK, runtime.config);
+    const pathFilter = args.path;
+    const filtered = typeof pathFilter === "string" ? ranked.filter((row) => row.file.startsWith(pathFilter)) : ranked;
+    const enriched = filtered.map(async (row) => {
+      const fullText = await Bun.file(row.file).text();
+      return { row, fullText };
+    });
+    const resolved = await Promise.all(enriched);
+    const elapsedMs = Date.now() - startedAt;
+    await runtime.logger.info("tool.semantic_search", {
+      message: "Semantic search completed",
+      extra: {
+        results: resolved.length,
+        elapsedMs,
+      },
+    });
+    ctx?.metadata?.({
+      title: "semantic_search",
+      metadata: { phase: "done", results: resolved.length, elapsed_ms: elapsedMs },
+    });
+    return JSON.stringify(
+      resolved.map(({ row, fullText }) => ({
+        file: row.file,
+        lines: `${row.startLine}-${row.endLine}`,
+        score: Number(row.score.toFixed(4)),
+        vector_score: Number(row.vectorScore.toFixed(4)),
+        bm25_score: Number(row.bm25Score.toFixed(4)),
+        preview: buildContextPreview(fullText, row.startLine, row.endLine, runtime.config.search.context_lines).preview,
+      })),
+    );
+  } finally {
+    runtime.config.search.similarity_threshold = originalThreshold;
+  }
+}
+
+export function createSemanticSearchTool(runtime: RuntimeContext) {
+  return tool({
+    description:
+      "Search code by meaning and concept, not exact text. Use this for multi-word conceptual queries like authentication flow, retry logic, or payment handling. Prefer over grep for behavior-level search.",
+    args: {
+      query: tool.schema.string().describe("Natural language search query"),
+      top_k: tool.schema.number().optional().describe("Number of results"),
+      threshold: tool.schema.number().optional().describe("Minimum score threshold"),
+      path: tool.schema.string().optional().describe("Optional path filter prefix"),
+    },
+    async execute(args, toolCtx) {
+      return executeSemanticSearch(runtime, args, toolCtx);
+    },
+  });
+}

package/src/tools/smart_grep.ts

@@ -0,0 +1,198 @@
+import type { ToolContext } from "@opencode-ai/plugin";
+import { tool } from "@opencode-ai/plugin";
+import { appendRoutingOutcome } from "../diagnostics/routing";
+import { hybridSearch } from "../search/hybrid";
+import type { RuntimeContext } from "../runtime";
+
+function looksConceptual(query: string, minWords: number): boolean {
+  if (query.trim().split(/\s+/).length < minWords) return false;
+  if (/[.*+?^${}()|[\]\\]/.test(query)) return false;
+  if (/^[A-Za-z0-9_]+$/.test(query) && !query.includes(" ")) return false;
+  return true;
+}
+
+export interface GrepToolArgs {
+  pattern?: string;
+  query?: string;
+  path?: string;
+  glob?: string;
+  case_sensitive?: boolean;
+  word?: boolean;
+  fixed_strings?: boolean;
+  max_count?: number;
+  files_with_matches?: boolean;
+  multiline?: boolean;
+}
+
+function pickPattern(args: GrepToolArgs): string {
+  const pattern = args.pattern ?? args.query;
+  if (!pattern || pattern.trim().length === 0) return "";
+  return pattern;
+}
+
+function buildRipgrepArgs(args: GrepToolArgs): string[] {
+  const rgArgs: string[] = ["-n", "--hidden", "--glob", "!node_modules/**", "--glob", "!.git/**"];
+  if (args.case_sensitive === false) rgArgs.push("-i");
+  if (args.word) rgArgs.push("-w");
+  if (args.fixed_strings) rgArgs.push("-F");
+  if (args.files_with_matches) rgArgs.push("-l");
+  if (args.multiline) rgArgs.push("-U");
+  if (typeof args.max_count === "number") rgArgs.push("-m", `${args.max_count}`);
+  if (typeof args.glob === "string" && args.glob.length > 0) rgArgs.push("--glob", args.glob);
+  return rgArgs;
+}
+
+function toPatternPreview(pattern: string): string {
+  const normalized = pattern.replace(/\s+/g, " ").trim();
+  return normalized.length > 100 ? `${normalized.slice(0, 100)}...` : normalized;
+}
+
+async function runRipgrep(args: GrepToolArgs): Promise<string> {
+  const pattern = pickPattern(args);
+  if (!pattern) return "";
+  const pathArg = args.path && args.path.length > 0 ? args.path : ".";
+  const rgArgs = buildRipgrepArgs(args);
+  const proc = Bun.spawn({
+    cmd: ["rg", ...rgArgs, pattern, pathArg],
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const stdout = await new Response(proc.stdout).text();
+  const stderr = await new Response(proc.stderr).text();
+  const exitCode = await proc.exited;
+  if (exitCode === 0 || exitCode === 1) return stdout.trim();
+  throw new Error(stderr || `rg failed with exit code ${exitCode}`);
+}
+
+function grepMeta(ctx: Pick<ToolContext, "metadata"> | undefined, metadata: Record<string, unknown>): void {
+  ctx?.metadata?.({ title: "grep", metadata });
+}
+
+export async function executeSmartGrep(
+  runtime: RuntimeContext,
+  args: GrepToolArgs,
+  ctx?: Pick<ToolContext, "metadata">
+): Promise<string> {
+  const pattern = pickPattern(args);
+  if (!pattern) {
+    grepMeta(ctx, { route: "none", reason: "empty_pattern" });
+    return "";
+  }
+  await runtime.logger.debug("tool.smart_grep", {
+    message: "Received grep request",
+    extra: { patternLength: pattern.length, path: args.path, glob: args.glob }
+  });
+  const shouldUseSemantic = runtime.config.smart_grep.enabled && looksConceptual(pattern, runtime.config.smart_grep.min_words_for_semantic);
+  if (!shouldUseSemantic) {
+    appendRoutingOutcome(runtime.store, {
+      at: new Date().toISOString(),
+      route: "ripgrep",
+      reason: "non-conceptual-or-disabled",
+      patternPreview: toPatternPreview(pattern),
+    });
+    await runtime.logger.info("tool.smart_grep", {
+      message: "Routing to ripgrep",
+      extra: { reason: "non-conceptual-or-disabled" }
+    });
+    grepMeta(ctx, { route: "ripgrep", phase: "running" });
+    const out = await runRipgrep(args);
+    grepMeta(ctx, {
+      route: "ripgrep",
+      phase: "done",
+      output_chars: out.length,
+      lines: out ? out.split("\n").length : 0,
+    });
+    return out;
+  }
+
+  const providerHealthy = await runtime.embedder.healthcheck();
+  const stats = runtime.store.stats();
+  if (!providerHealthy || stats.chunks === 0) {
+    const reason = !providerHealthy ? "embedder-unhealthy" : "empty-index";
+    appendRoutingOutcome(runtime.store, {
+      at: new Date().toISOString(),
+      route: "ripgrep",
+      reason,
+      patternPreview: toPatternPreview(pattern),
+      providerHealthy,
+      indexedChunks: stats.chunks,
+    });
+    await runtime.logger.warn("tool.smart_grep", {
+      message: "Falling back to ripgrep",
+      extra: {
+        reason,
+        providerHealthy,
+        indexedChunks: stats.chunks
+      }
+    });
+    grepMeta(ctx, { route: "ripgrep", phase: "running", fallback_reason: reason });
+    const out = await runRipgrep(args);
+    grepMeta(ctx, {
+      route: "ripgrep",
+      phase: "done",
+      fallback_reason: reason,
+      output_chars: out.length,
+    });
+    return out;
+  }
+
+  grepMeta(ctx, { route: "semantic", phase: "embedding_and_search", indexed_chunks: stats.chunks });
+  const semanticResults = await hybridSearch(runtime.store, runtime.embedder, pattern, runtime.config.search.top_k, runtime.config);
+  const pathFilter = args.path;
+  const filtered = typeof pathFilter === "string" ? semanticResults.filter((row) => row.file.startsWith(pathFilter)) : semanticResults;
+  if (filtered.length === 0 && runtime.config.smart_grep.fallback_to_grep_on_empty) {
+    appendRoutingOutcome(runtime.store, {
+      at: new Date().toISOString(),
+      route: "ripgrep",
+      reason: "semantic-empty-results",
+      patternPreview: toPatternPreview(pattern),
+      results: 0,
+    });
+    await runtime.logger.info("tool.smart_grep", {
+      message: "Falling back to ripgrep",
+      extra: { reason: "semantic-empty-results" }
+    });
+    grepMeta(ctx, { route: "ripgrep", phase: "running", fallback_reason: "semantic-empty-results" });
+    const out = await runRipgrep(args);
+    grepMeta(ctx, { route: "ripgrep", phase: "done", output_chars: out.length });
+    return out;
+  }
+
+  appendRoutingOutcome(runtime.store, {
+    at: new Date().toISOString(),
+    route: "semantic",
+    reason: "conceptual-query",
+    patternPreview: toPatternPreview(pattern),
+    results: filtered.length,
+  });
+  await runtime.logger.info("tool.smart_grep", {
+    message: "Routing to semantic results",
+    extra: { results: filtered.length }
+  });
+  grepMeta(ctx, { route: "semantic", phase: "done", results: filtered.length });
+  return filtered
+    .map((row) => `${row.file}:${row.startLine}-${row.endLine} score=${row.score.toFixed(4)}\n${row.preview}`)
+    .join("\n\n");
+}
+
+export function createSmartGrepTool(runtime: RuntimeContext) {
+  return tool({
+    description:
+      "Intelligent grep wrapper. Routes conceptual multi-word queries to semantic search and exact-match/regex queries to ripgrep. Falls back to ripgrep if semantic search is unavailable.",
+    args: {
+      pattern: tool.schema.string().optional().describe("Pattern or conceptual search query"),
+      query: tool.schema.string().optional().describe("Alternative grep query field"),
+      path: tool.schema.string().optional().describe("Optional search path"),
+      glob: tool.schema.string().optional().describe("Optional glob filter"),
+      case_sensitive: tool.schema.boolean().optional().describe("Enable case-sensitive matching"),
+      word: tool.schema.boolean().optional().describe("Match whole words"),
+      fixed_strings: tool.schema.boolean().optional().describe("Treat pattern as literal string"),
+      max_count: tool.schema.number().optional().describe("Max matching lines per file"),
+      files_with_matches: tool.schema.boolean().optional().describe("Return matching file names only"),
+      multiline: tool.schema.boolean().optional().describe("Enable multiline regex"),
+    },
+    async execute(args, toolCtx) {
+      return executeSmartGrep(runtime, args, toolCtx);
+    },
+  });
+}

package/src/tui_toast.ts

@@ -0,0 +1,191 @@
+import path from "node:path";
+import type { IndexingProgressState, RuntimeContext } from "./runtime";
+
+const progressToastMinIntervalMs = 2200;
+
+/**
+ * OpenCode TUI: POST /tui/show-toast (`client.tui.showToast` on the SDK).
+ * Complements tool `context.metadata()` (inline tool-call cards) with transient toast pop-ups.
+ */
+export async function showTuiToast(
+  runtime: RuntimeContext,
+  input: {
+    title?: string;
+    message: string;
+    variant: "info" | "success" | "warning" | "error";
+    duration?: number;
+  }
+): Promise<void> {
+  const client = runtime.opencodeClient;
+  if (!client?.tui?.showToast) {
+    return;
+  }
+  const directory = runtime.projectDirectory ?? runtime.worktree;
+  try {
+    await client.tui.showToast({
+      body: {
+        title: input.title,
+        message: input.message,
+        variant: input.variant,
+        duration: input.duration ?? 4500,
+      },
+      query: { directory },
+    });
+  } catch {
+    // No TUI (headless), wrong directory, or API unavailable
+  }
+}
+
+function formatElapsedSuffix(startedAt: number): string {
+  if (startedAt <= 0) return "";
+  const sec = Math.max(0, Math.round((Date.now() - startedAt) / 1000));
+  return sec > 0 ? ` · ${sec}s` : "";
+}
+
+/** Maps internal progress labels to short toast text (avoid stacking file counts + batch labels). */
+function formatIndexingProgressMessage(p: IndexingProgressState): string {
+  const elapsed = formatElapsedSuffix(p.startedAt);
+
+  if (p.phase === "scanning") {
+    if (p.label === "listing files") {
+      return `Listing files…${elapsed}`;
+    }
+    return `${p.label ?? "Listing files…"}${elapsed}`;
+  }
+  if (p.phase === "gc") {
+    return `Cleaning up…${elapsed}`;
+  }
+  if (p.phase === "indexing") {
+    const label = p.label ?? "";
+    if (label.startsWith("scan:")) {
+      const file = label.slice("scan:".length).trim();
+      if (p.total > 0) {
+        return `${p.current}/${p.total} · ${file}${elapsed}`;
+      }
+      return file ? `${file}${elapsed}` : `Scanning…${elapsed}`;
+    }
+    if (/^\d+ files to scan$/.test(label)) {
+      return p.total > 0 ? `Scanning ${p.total} files…${elapsed}` : `Scanning…${elapsed}`;
+    }
+    if (label === "no files matched") {
+      return `No files to index${elapsed}`;
+    }
+    if (label === "chunking files…") {
+      return p.total > 0 ? `Preparing ${p.current}/${p.total}…${elapsed}` : `Preparing…${elapsed}`;
+    }
+    if (label === "embedding…") {
+      return p.total > 0 ? `Embedding ${p.current}/${p.total}…${elapsed}` : `Embedding…${elapsed}`;
+    }
+    if (label.startsWith("embedding batch ")) {
+      const m = /^embedding batch (\d+)\/(\d+)$/.exec(label);
+      if (m?.[1] !== undefined && m[2] !== undefined) {
+        return `Embedding ${m[1]}/${m[2]}…${elapsed}`;
+      }
+      return `Embedding…${elapsed}`;
+    }
+    if (label === "saving index…") {
+      return p.total > 0 ? `Saving ${p.current}/${p.total}…${elapsed}` : `Saving…${elapsed}`;
+    }
+    if (label.startsWith("saved ")) {
+      const head = label.split(" · ")[0] ?? label;
+      const rest = head.replace(/^saved\s+/u, "");
+      return rest ? `Saving ${rest}${elapsed}` : `Saving…${elapsed}`;
+    }
+    if (label.startsWith("writing ") && label !== "writing…") {
+      return p.total > 0 ? `Saving ${p.current}/${p.total}…${elapsed}` : `Saving…${elapsed}`;
+    }
+    if (p.total > 0 && label.length > 0) {
+      return `${p.current}/${p.total} · ${label}${elapsed}`;
+    }
+    if (p.total > 0) {
+      return `${p.current}/${p.total}${elapsed}`;
+    }
+    return `${label || "Working…"}${elapsed}`;
+  }
+  return `Working…${elapsed}`;
+}
+
+export function resetIndexingToastThrottle(runtime: RuntimeContext): void {
+  runtime.indexingToastLastMs = undefined;
+  runtime.indexingToastLastPhase = undefined;
+  runtime.indexingToastLastLabel = undefined;
+}
+
+/**
+ * Shows a throttled info toast while delta sync or incremental indexing runs.
+ * Always emits on phase transitions; otherwise at most once per {@link progressToastMinIntervalMs}.
+ */
+export function notifyIndexingProgressToast(runtime: RuntimeContext): void {
+  const p = runtime.indexingProgress;
+  if (p.phase === "idle") {
+    return;
+  }
+
+  const now = Date.now();
+  const lastAt = runtime.indexingToastLastMs ?? 0;
+  const lastPhase = runtime.indexingToastLastPhase;
+  const lastLabel = runtime.indexingToastLastLabel;
+  const phaseChanged = lastPhase === undefined || lastPhase !== p.phase;
+  const labelKey = p.label ?? "";
+  const labelChanged = lastLabel === undefined || lastLabel !== labelKey;
+
+  if (!phaseChanged && !labelChanged && now - lastAt < progressToastMinIntervalMs) {
+    return;
+  }
+
+  runtime.indexingToastLastMs = now;
+  runtime.indexingToastLastPhase = p.phase;
+  runtime.indexingToastLastLabel = labelKey;
+
+  const title =
+    p.source === "reindex" ? "Reindex" : p.source === "background" ? "Semantic index" : "Indexing";
+
+  void showTuiToast(runtime, {
+    title,
+    message: formatIndexingProgressMessage(p),
+    variant: "info",
+    duration: 3800,
+  });
+}
+
+export interface IndexingCompleteToastInput {
+  title: string;
+  files: number;
+  chunks: number;
+  elapsedMs: number;
+  /** Set when the parallel pipeline reported per-file failures. */
+  failedFiles?: string[];
+  /** `reindex` uses success styling when fully OK; background sync uses info. */
+  flavor: "background" | "reindex";
+}
+
+/**
+ * Completion popup after delta sync / reindex: warns when any files failed in the indexing pipeline.
+ */
+export function showIndexingCompleteToast(runtime: RuntimeContext, input: IndexingCompleteToastInput): void {
+  const failed = input.failedFiles?.filter((p) => p.length > 0) ?? [];
+  const n = failed.length;
+  const sec = Math.max(0, Math.round(input.elapsedMs / 1000));
+  if (n > 0) {
+    const examples =
+      n <= 3
+        ? failed.map((fp) => path.basename(fp)).join(", ")
+        : `${failed
+            .slice(0, 3)
+            .map((fp) => path.basename(fp))
+            .join(", ")} (+${n - 3} more)`;
+    void showTuiToast(runtime, {
+      title: input.title,
+      message: `${n} failed: ${examples} · ${sec}s`,
+      variant: "warning",
+      duration: 7000,
+    });
+    return;
+  }
+  void showTuiToast(runtime, {
+    title: input.title,
+    message: `${input.files} files · ${input.chunks} chunks · ${sec}s`,
+    variant: input.flavor === "reindex" ? "success" : "info",
+    duration: 4500,
+  });
+}

package/src/types.d.ts

@@ -0,0 +1 @@
+declare module "picomatch";