opencode-semantic-search 0.1.0

package/src/config.ts

@@ -0,0 +1,157 @@
+import { homedir } from "node:os";
+import path from "node:path";
+
+export type EmbeddingProvider = "ollama" | "openai";
+
+export interface PluginConfig {
+  logging: {
+    enabled: boolean;
+    level: "debug" | "info" | "warn" | "error";
+    verbose_paths?: string[];
+    /** Absolute path to a local log file for testing. Entries are appended as newline-delimited JSON. */
+    log_file?: string;
+  };
+  embedding: {
+    provider: EmbeddingProvider;
+    api_base: string;
+    model: string;
+    api_key_env?: string;
+    dimensions: number;
+    batch_size: number;
+    query_prefix: string;
+    /** Hard ceiling for a single embedding input (e.g. nomic-embed-text 8192). Splits oversized chunks. */
+    max_context_tokens: number;
+  };
+  chunking: {
+    max_tokens: number;
+    overlap_tokens: number;
+  };
+  indexing: {
+    include: string[];
+    exclude: string[];
+    respect_gitignore: boolean;
+    max_file_size_kb: number;
+    concurrency: number;
+    /** Concurrent embedding HTTP batch calls (cross-file batching in the indexing pipeline). */
+    embed_concurrency: number;
+    /** Per-file chunk/read/hash attempts before marking the file failed. */
+    retry_attempts: number;
+    /** Initial backoff in ms between per-file retries; doubles each attempt. */
+    retry_base_ms: number;
+  };
+  search: {
+    top_k: number;
+    similarity_threshold: number;
+    context_lines: number;
+    hybrid: {
+      weight_vector: number;
+      weight_bm25: number;
+      weight_rrf: number;
+      identifier_boost: number;
+      /** ANN + BM25 candidate pool size = ceil(topK * candidate_multiplier). */
+      candidate_multiplier: number;
+    };
+  };
+  smart_grep: {
+    enabled: boolean;
+    min_words_for_semantic: number;
+    fallback_to_grep_on_empty: boolean;
+  };
+  storage: {
+    global_cache_dir: string;
+  };
+}
+
+export const defaultConfig: PluginConfig = {
+  logging: {
+    enabled: true,
+    level: "info",
+    verbose_paths: [],
+    log_file: path.join(homedir(), ".cache", "opencode", "semantic-search", "plugin.log"),
+  },
+  embedding: {
+    provider: "ollama",
+    api_base: "http://localhost:11434/v1",
+    model: "nomic-embed-text",
+    dimensions: 768,
+    batch_size: 10,
+    query_prefix: "search_query: ",
+    max_context_tokens: 8192,
+  },
+  chunking: {
+    max_tokens: 512,
+    overlap_tokens: 50,
+  },
+  indexing: {
+    include: ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx", "**/*.py", "**/*.go", "**/*.rs", "**/*.java"],
+    exclude: ["node_modules/**", "dist/**", ".git/**", "**/*.min.js", "**/tree/**", "**/.worktrees/**", "**/migration-*/**"],
+    respect_gitignore: true,
+    max_file_size_kb: 500,
+    concurrency: 4,
+    embed_concurrency: 2,
+    retry_attempts: 3,
+    retry_base_ms: 500,
+  },
+  search: {
+    top_k: 10,
+    similarity_threshold: 0.30,
+    context_lines: 3,
+    hybrid: {
+      weight_vector: 0.4,
+      weight_bm25: 0.3,
+      weight_rrf: 0.3,
+      identifier_boost: 1.5,
+      candidate_multiplier: 1.2,
+    },
+  },
+  smart_grep: {
+    enabled: true,
+    min_words_for_semantic: 2,
+    fallback_to_grep_on_empty: true,
+  },
+  storage: {
+    global_cache_dir: path.join(homedir(), ".cache", "opencode", "semantic-search"),
+  },
+};
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function mergeDeep<T>(target: T, source: unknown): T {
+  if (!isObject(target) || !isObject(source)) return target;
+  const result = { ...target } as Record<string, unknown>;
+  for (const [key, value] of Object.entries(source)) {
+    const existing = result[key];
+    if (isObject(existing) && isObject(value)) {
+      result[key] = mergeDeep(existing, value);
+      continue;
+    }
+    result[key] = value;
+  }
+  return result as T;
+}
+
+const globalConfigPath = (): string =>
+  path.join(homedir(), ".config", "opencode", "semantic-search.json");
+
+async function readJsonConfig(filePath: string): Promise<unknown> {
+  try {
+    const file = Bun.file(filePath);
+    if (await file.exists()) {
+      return JSON.parse(await file.text());
+    }
+  } catch {
+    // ignore missing or invalid file
+  }
+  return {};
+}
+
+export async function loadConfig(worktree: string): Promise<PluginConfig> {
+  const globalParsed = await readJsonConfig(globalConfigPath());
+  const projectPath = path.join(worktree, ".opencode", "semantic-search.json");
+  const projectParsed = await readJsonConfig(projectPath);
+  let merged = mergeDeep(defaultConfig, globalParsed);
+  merged = mergeDeep(merged, projectParsed);
+  return merged;
+}

package/src/diagnostics/bundle.ts

@@ -0,0 +1,63 @@
+import type { RuntimeContext } from "../runtime";
+import { getRecentRoutingOutcomes } from "./routing";
+
+export async function buildDiagnosticBundle(runtime: RuntimeContext): Promise<{
+  ok: boolean;
+  worktree: string;
+  generated_at: string;
+  elapsed_ms: number;
+  provider: {
+    name: string;
+    api_base: string;
+    model: string;
+    dimensions: number;
+    healthy: boolean;
+  };
+  index: {
+    db_path: string;
+    files_indexed: number;
+    chunks_indexed: number;
+    last_sync: string | null;
+  };
+  routing: {
+    recent_count: number;
+    recent_outcomes: ReturnType<typeof getRecentRoutingOutcomes>;
+  };
+  hints: {
+    reindex_tool: string;
+    debug_logs: string;
+  };
+}> {
+  const startedAt = Date.now();
+  const stats = runtime.store.stats();
+  const providerHealthy = await runtime.embedder.healthcheck();
+  const recentRouting = getRecentRoutingOutcomes(runtime.store);
+
+  return {
+    ok: true,
+    worktree: runtime.worktree,
+    generated_at: new Date().toISOString(),
+    elapsed_ms: Date.now() - startedAt,
+    provider: {
+      name: runtime.config.embedding.provider,
+      api_base: runtime.config.embedding.api_base,
+      model: runtime.config.embedding.model,
+      dimensions: runtime.config.embedding.dimensions,
+      healthy: providerHealthy,
+    },
+    index: {
+      db_path: runtime.dbPath,
+      files_indexed: stats.files,
+      chunks_indexed: stats.chunks,
+      last_sync: stats.lastSync ?? null,
+    },
+    routing: {
+      recent_count: recentRouting.length,
+      recent_outcomes: recentRouting,
+    },
+    hints: {
+      reindex_tool: "Run reindex() in OpenCode if index is stale or empty.",
+      debug_logs: "Set logging.level to debug in .opencode/semantic-search.json.",
+    },
+  };
+}

package/src/diagnostics/routing.ts

@@ -0,0 +1,37 @@
+import type { SemanticStore } from "../store/sqlite";
+
+const ROUTING_STATE_KEY = "routing_recent";
+const MAX_ROUTING_EVENTS = 25;
+
+export interface RoutingOutcome {
+  at: string;
+  route: "semantic" | "ripgrep";
+  reason: string;
+  patternPreview: string;
+  results?: number;
+  providerHealthy?: boolean;
+  indexedChunks?: number;
+}
+
+function safeParse(value: string | null): RoutingOutcome[] {
+  if (!value) return [];
+  try {
+    const parsed = JSON.parse(value) as unknown;
+    if (!Array.isArray(parsed)) return [];
+    return parsed.filter((item) => {
+      return Boolean(item) && typeof item === "object";
+    }) as RoutingOutcome[];
+  } catch {
+    return [];
+  }
+}
+
+export function getRecentRoutingOutcomes(store: SemanticStore): RoutingOutcome[] {
+  return safeParse(store.getSyncState(ROUTING_STATE_KEY));
+}
+
+export function appendRoutingOutcome(store: SemanticStore, outcome: RoutingOutcome): void {
+  const current = safeParse(store.getSyncState(ROUTING_STATE_KEY));
+  const next = [outcome, ...current].slice(0, MAX_ROUTING_EVENTS);
+  store.setSyncState(ROUTING_STATE_KEY, JSON.stringify(next));
+}

package/src/embedder/interface.ts

@@ -0,0 +1,62 @@
+import type { PluginConfig } from "../config";
+import type { Logger } from "../logger";
+
+export interface Embedder {
+  embed(texts: string[]): Promise<number[][]>;
+  healthcheck(): Promise<boolean>;
+}
+
+export function withQueryExpansion(input: string): string {
+  const lower = input.toLowerCase();
+  if (lower.includes("auth")) return `${input} authentication authorize login session token`;
+  if (lower.includes("db") || lower.includes("database")) return `${input} database connection pool query migration`;
+  if (lower.includes("payment")) return `${input} payment checkout invoice charge transaction`;
+  return input;
+}
+
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  maxAttempts = 3,
+  onRetry?: (attempt: number, maxAttempts: number, error: unknown) => Promise<void> | void
+): Promise<T> {
+  let attempt = 0;
+  let waitMs = 1000;
+  let lastErr: unknown;
+  while (attempt < maxAttempts) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastErr = error;
+      attempt += 1;
+      if (onRetry) await onRetry(attempt, maxAttempts, error);
+      if (attempt >= maxAttempts) break;
+      await Bun.sleep(waitMs);
+      waitMs *= 4;
+    }
+  }
+  throw lastErr;
+}
+
+export function chunked<T>(items: T[], size: number): T[][] {
+  const chunkSize = Math.max(1, size);
+  const out: T[][] = [];
+  for (let i = 0; i < items.length; i += chunkSize) out.push(items.slice(i, i + chunkSize));
+  return out;
+}
+
+export async function createEmbedder(config: PluginConfig, logger: Logger): Promise<Embedder> {
+  if (config.embedding.provider === "openai") {
+    const { OpenAIEmbedder } = await import("./openai");
+    await logger.info("embedder", {
+      message: "Using OpenAI embedder",
+      extra: { provider: config.embedding.provider, model: config.embedding.model }
+    });
+    return new OpenAIEmbedder(config, logger);
+  }
+  const { OllamaEmbedder } = await import("./ollama");
+  await logger.info("embedder", {
+    message: "Using Ollama embedder",
+    extra: { provider: config.embedding.provider, model: config.embedding.model }
+  });
+  return new OllamaEmbedder(config, logger);
+}

package/src/embedder/ollama.ts

@@ -0,0 +1,60 @@
+import type { PluginConfig } from "../config";
+import type { Logger } from "../logger";
+import { chunked, withRetry, type Embedder } from "./interface";
+
+interface EmbeddingResponse {
+  data: Array<{ embedding: number[] }>;
+}
+
+export class OllamaEmbedder implements Embedder {
+  constructor(
+    private readonly config: PluginConfig,
+    private readonly logger: Logger
+  ) {}
+
+  async healthcheck(): Promise<boolean> {
+    try {
+      const endpoint = this.config.embedding.api_base.replace(/\/v1$/, "/api/tags");
+      const res = await fetch(endpoint);
+      await this.logger.debug("embedder.ollama", {
+        message: "Healthcheck completed",
+        extra: { endpoint, status: res.status, ok: res.ok }
+      });
+      return res.ok;
+    } catch (error) {
+      await this.logger.warn("embedder.ollama", { message: "Healthcheck failed" });
+      await this.logger.error("embedder.ollama", { message: "Healthcheck error", error });
+      return false;
+    }
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const results: number[][] = [];
+    for (const batch of chunked(texts, this.config.embedding.batch_size)) {
+      const response = await withRetry(async () => {
+        const res = await fetch(`${this.config.embedding.api_base}/embeddings`, {
+          method: "POST",
+          headers: { "content-type": "application/json" },
+          body: JSON.stringify({
+            model: this.config.embedding.model,
+            input: batch,
+          }),
+        });
+        if (!res.ok) throw new Error(`Embedding request failed: ${res.status}`);
+        return (await res.json()) as EmbeddingResponse;
+      }, 3, async (attempt, maxAttempts, error) => {
+        await this.logger.warn("embedder.ollama", {
+          message: "Embedding request retry",
+          extra: {
+            attempt,
+            maxAttempts,
+            endpoint: `${this.config.embedding.api_base}/embeddings`
+          }
+        });
+        await this.logger.error("embedder.ollama", { message: "Embedding request attempt failed", error });
+      });
+      for (const item of response.data) results.push(item.embedding);
+    }
+    return results;
+  }
+}

package/src/embedder/openai.ts

@@ -0,0 +1,71 @@
+import type { PluginConfig } from "../config";
+import type { Logger } from "../logger";
+import { chunked, withRetry, type Embedder } from "./interface";
+
+interface EmbeddingResponse {
+  data: Array<{ embedding: number[] }>;
+}
+
+export class OpenAIEmbedder implements Embedder {
+  constructor(
+    private readonly config: PluginConfig,
+    private readonly logger: Logger
+  ) {}
+
+  private get headers(): Record<string, string> {
+    const token = this.config.embedding.api_key_env ? process.env[this.config.embedding.api_key_env] : undefined;
+    return {
+      "content-type": "application/json",
+      ...(token ? { authorization: `Bearer ${token}` } : {}),
+    };
+  }
+
+  async healthcheck(): Promise<boolean> {
+    try {
+      const res = await fetch(`${this.config.embedding.api_base}/models`, { headers: this.headers });
+      await this.logger.debug("embedder.openai", {
+        message: "Healthcheck completed",
+        extra: { status: res.status, ok: res.ok }
+      });
+      return res.ok || res.status === 401;
+    } catch (error) {
+      await this.logger.warn("embedder.openai", {
+        message: "Healthcheck failed",
+        extra: { endpoint: `${this.config.embedding.api_base}/models` }
+      });
+      await this.logger.error("embedder.openai", { message: "Healthcheck error", error });
+      return false;
+    }
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const out: number[][] = [];
+    for (const batch of chunked(texts, this.config.embedding.batch_size)) {
+      const body = {
+        model: this.config.embedding.model,
+        input: batch,
+      };
+      const data = await withRetry(async () => {
+        const res = await fetch(`${this.config.embedding.api_base}/embeddings`, {
+          method: "POST",
+          headers: this.headers,
+          body: JSON.stringify(body),
+        });
+        if (!res.ok) throw new Error(`Embedding request failed: ${res.status}`);
+        return (await res.json()) as EmbeddingResponse;
+      }, 3, async (attempt, maxAttempts, error) => {
+        await this.logger.warn("embedder.openai", {
+          message: "Embedding request retry",
+          extra: {
+            attempt,
+            maxAttempts,
+            endpoint: `${this.config.embedding.api_base}/embeddings`
+          }
+        });
+        await this.logger.error("embedder.openai", { message: "Embedding request attempt failed", error });
+      });
+      for (const row of data.data) out.push(row.embedding);
+    }
+    return out;
+  }
+}

package/src/indexer/delta.ts

@@ -0,0 +1,165 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import ignore from "ignore";
+import picomatch from "picomatch";
+import type { PluginConfig } from "../config";
+import type { Embedder } from "../embedder/interface";
+import type { Logger } from "../logger";
+import type { DeltaProgressPayload } from "../runtime";
+import type { SemanticStore } from "../store/sqlite";
+import { garbageCollectDeletedFiles } from "./gc";
+import { mapPool } from "./pool";
+import { runIndexingPipeline, type IndexingPipelineFailure } from "./pipeline";
+import { markSyncCompleted, markSyncStarted } from "./resume";
+
+function sha256(input: string): string {
+  return crypto.createHash("sha256").update(input).digest("hex");
+}
+
+/** Paths from Bun.Glob use the platform separator; gitignore expects `/`. */
+function toPosixRelative(relativePath: string): string {
+  return path.sep === "\\" ? relativePath.replaceAll("\\", "/") : relativePath;
+}
+
+function loadRootGitignore(worktree: string): ReturnType<typeof ignore> | null {
+  try {
+    const gitignorePath = path.join(worktree, ".gitignore");
+    const content = fs.readFileSync(gitignorePath, "utf8");
+    return ignore().add(content);
+  } catch {
+    return null;
+  }
+}
+
+async function listFiles(worktree: string, config: PluginConfig): Promise<string[]> {
+  const includeMatchers = config.indexing.include.map((p) => picomatch(p, { dot: true }));
+  const excludeMatchers = config.indexing.exclude.map((p) => picomatch(p, { dot: true }));
+  const gitignoreFilter =
+    config.indexing.respect_gitignore ? loadRootGitignore(worktree) : null;
+  const files: string[] = [];
+  const glob = new Bun.Glob("**/*");
+  for await (const relativePath of glob.scan({ cwd: worktree, dot: true, onlyFiles: true })) {
+    if (gitignoreFilter !== null && gitignoreFilter.ignores(toPosixRelative(relativePath))) {
+      continue;
+    }
+    const included = includeMatchers.some((m) => m(relativePath));
+    if (!included) continue;
+    const excluded = excludeMatchers.some((m) => m(relativePath));
+    if (excluded) continue;
+    files.push(path.join(worktree, relativePath));
+  }
+  return files;
+}
+
+export interface DeltaSyncOptions {
+  logger?: Logger;
+  onProgress?: (update: DeltaProgressPayload) => void;
+}
+
+export async function deltaSync(
+  worktree: string,
+  store: SemanticStore,
+  embedder: Embedder,
+  config: PluginConfig,
+  options?: DeltaSyncOptions
+): Promise<void> {
+  const logger = options?.logger;
+  const onProgress = options?.onProgress;
+  const startedAt = Date.now();
+  let indexedCount = 0;
+  let lastFailedFiles: string[] | undefined;
+  markSyncStarted(store);
+  try {
+    onProgress?.({ phase: "scanning", current: 0, total: 0, label: "listing files" });
+    const files = await listFiles(worktree, config);
+    await logger?.info("indexer.delta", {
+      message: "Listed files for index",
+      extra: { count: files.length },
+    });
+    const previousStats = store.stats();
+    const initialChunks = previousStats.chunks;
+    const existing = new Set<string>(files);
+    const total = files.length;
+    const concurrency = Math.max(1, config.indexing.concurrency);
+    onProgress?.({
+      phase: "indexing",
+      current: 0,
+      total,
+      label: total === 0 ? "no files matched" : `${total} files to scan`,
+    });
+    let scanned = 0;
+    const dirtyFlags = await mapPool(files, concurrency, async (filePath) => {
+      const text = await Bun.file(filePath).text();
+      const contentHash = sha256(text);
+      const row = store.db.query("SELECT content_hash FROM files WHERE path = ?").get(filePath) as { content_hash: string } | null;
+      scanned += 1;
+      onProgress?.({
+        phase: "indexing",
+        current: scanned,
+        total,
+        label: `scan: ${path.basename(filePath)}`,
+      });
+      return row?.content_hash !== contentHash ? filePath : null;
+    });
+    const toIndex = dirtyFlags.filter((p): p is string => p !== null);
+    const unchanged = files.length - toIndex.length;
+    await logger?.info("indexer.delta", {
+      message: "Hash scan finished",
+      extra: {
+        totalFiles: files.length,
+        dirtyFiles: toIndex.length,
+        unchanged,
+      },
+    });
+    const pipelineResult = await runIndexingPipeline(toIndex, store, embedder, config, {
+      logger,
+      onProgress,
+    });
+    indexedCount = pipelineResult.indexed;
+    lastFailedFiles =
+      pipelineResult.failed.length > 0
+        ? pipelineResult.failed.map((f: IndexingPipelineFailure) => f.filePath)
+        : undefined;
+    onProgress?.({
+      phase: "gc",
+      current: total,
+      total,
+      label: "removing stale entries",
+    });
+    await logger?.info("indexer.delta", {
+      message: "Garbage collection started",
+      extra: { trackedFiles: existing.size },
+    });
+    store.removeMissingFiles(existing);
+    garbageCollectDeletedFiles(store);
+    const updatedStats = store.stats();
+    const chunkDelta = updatedStats.chunks - initialChunks;
+    if (logger) {
+      const failedCount = pipelineResult.failed.length;
+      if (failedCount > 0) {
+        await logger.warn("indexer.delta", {
+          message: "Some files failed during indexing pipeline",
+          extra: {
+            failedCount,
+            failedPaths: pipelineResult.failed.map((f) => f.filePath),
+          },
+        });
+      }
+      await logger.info("indexer.delta", {
+        message: "Delta sync completed",
+        extra: {
+          scannedFiles: files.length,
+          dirtyFiles: toIndex.length,
+          filesWritten: indexedCount,
+          chunkDelta,
+          elapsedMs: Date.now() - startedAt,
+          pipelineFailed: failedCount,
+        },
+      });
+    }
+  } finally {
+    onProgress?.({ phase: "idle", current: 0, total: 0, failedFiles: lastFailedFiles });
+    markSyncCompleted(store);
+  }
+}

package/src/indexer/gc.ts

@@ -0,0 +1,10 @@
+import fs from "node:fs";
+import type { SemanticStore } from "../store/sqlite";
+
+export function garbageCollectDeletedFiles(store: SemanticStore): void {
+  const rows = store.db.query("SELECT path FROM files").all() as Array<{ path: string }>;
+  for (const row of rows) {
+    if (fs.existsSync(row.path)) continue;
+    store.db.query("DELETE FROM files WHERE path = ?").run(row.path);
+  }
+}