opencode-semantic-search 0.1.0

package/src/indexer/incremental.ts

@@ -0,0 +1,105 @@
+import crypto from "node:crypto";
+import type { PluginConfig } from "../config";
+import { chunkFile } from "../chunker";
+import { withQueryExpansion, type Embedder } from "../embedder/interface";
+import type { Logger } from "../logger";
+import type { SemanticStore } from "../store/sqlite";
+
+function sha256(input: string): string {
+  return crypto.createHash("sha256").update(input).digest("hex");
+}
+
+export async function indexSingleFile(
+  filePath: string,
+  store: SemanticStore,
+  embedder: Embedder,
+  config: PluginConfig,
+  logger?: Logger
+): Promise<void> {
+  const startedAt = Date.now();
+  const file = Bun.file(filePath);
+  if (!(await file.exists())) {
+    await logger?.debug("indexer.incremental", {
+      message: "Skipped missing file",
+      extra: { filePath }
+    });
+    return;
+  }
+  const stat = await file.stat();
+  if (stat.size / 1024 > config.indexing.max_file_size_kb) {
+    await logger?.debug("indexer.incremental", {
+      message: "Skipped oversized file",
+      extra: { filePath, sizeBytes: stat.size, maxFileSizeKb: config.indexing.max_file_size_kb }
+    });
+    return;
+  }
+
+  const text = await file.text();
+  const fileHash = sha256(text);
+
+  const chunks = await chunkFile(filePath, text, config);
+  await logger?.debug("indexer.incremental", {
+    message: "Chunked file for indexing",
+    extra: { filePath, chunkCount: chunks.length }
+  });
+
+  const toEmbed: string[] = [];
+  const meta = chunks.map((chunk) => {
+    const chunkHash = sha256(chunk.text);
+    const cached = store.getChunkByHash(chunkHash);
+    if (!cached) {
+      toEmbed.push(withQueryExpansion(chunk.text));
+    }
+    return { chunk, chunkHash, cached };
+  });
+
+  const embeddings = toEmbed.length > 0 ? await embedder.embed(toEmbed) : [];
+  let cursor = 0;
+  const writes: Array<{
+    startLine: number;
+    endLine: number;
+    text: string;
+    chunkHash: string;
+    embedding: number[];
+  }> = [];
+  for (const m of meta) {
+    if (m.cached) {
+      writes.push({
+        startLine: m.chunk.startLine,
+        endLine: m.chunk.endLine,
+        text: m.chunk.text,
+        chunkHash: m.chunkHash,
+        embedding: m.cached.embedding,
+      });
+      continue;
+    }
+    const embedding = embeddings[cursor] ?? [];
+    cursor += 1;
+    writes.push({
+      startLine: m.chunk.startLine,
+      endLine: m.chunk.endLine,
+      text: m.chunk.text,
+      chunkHash: m.chunkHash,
+      embedding,
+    });
+  }
+
+  store.writeFileAndChunks(filePath, fileHash, writes);
+
+  if (toEmbed.length === 0) {
+    await logger?.debug("indexer.incremental", {
+      message: "No new embeddings needed",
+      extra: { filePath, chunkCount: chunks.length, elapsedMs: Date.now() - startedAt }
+    });
+    return;
+  }
+  await logger?.info("indexer.incremental", {
+    message: "Indexed file",
+    extra: {
+      filePath,
+      chunkCount: chunks.length,
+      embeddedChunks: toEmbed.length,
+      elapsedMs: Date.now() - startedAt
+    }
+  });
+}

package/src/indexer/pipeline.test.ts

@@ -0,0 +1,126 @@
+import fs from "node:fs";
+import path from "node:path";
+import { test, expect } from "bun:test";
+import { defaultConfig } from "../config";
+import type { Embedder } from "../embedder/interface";
+import type { DeltaProgressPayload } from "../runtime";
+import { SemanticStore } from "../store/sqlite";
+import { chunked } from "../embedder/interface";
+import { runIndexingPipeline } from "./pipeline";
+
+const dim = defaultConfig.embedding.dimensions;
+
+const mockEmbedder: Embedder = {
+  async embed(texts: string[]) {
+    return texts.map(() => new Array<number>(dim).fill(0.01));
+  },
+  async healthcheck() {
+    return true;
+  },
+};
+
+function schemaPath(): string {
+  return path.join(import.meta.dir, "../store/schema.sql");
+}
+
+test("runIndexingPipeline writes one file and chunks", async () => {
+  const dir = fs.mkdtempSync(path.join(fs.realpathSync("/tmp"), "sem-pipe-"));
+  const dbPath = path.join(dir, "idx.sqlite");
+  const store = new SemanticStore(dbPath, schemaPath());
+  const filePath = path.join(dir, "hello.ts");
+  fs.writeFileSync(filePath, "export const x = 1;\n");
+
+  const result = await runIndexingPipeline([filePath], store, mockEmbedder, defaultConfig, {});
+  expect(result.indexed).toBe(1);
+  expect(result.failed.length).toBe(0);
+
+  const stats = store.stats();
+  expect(stats.files).toBe(1);
+  expect(stats.chunks).toBeGreaterThan(0);
+  store.close();
+});
+
+test("runIndexingPipeline handles multiple files with cross-file embedding batches", async () => {
+  const dir = fs.mkdtempSync(path.join(fs.realpathSync("/tmp"), "sem-pipe-"));
+  const dbPath = path.join(dir, "idx2.sqlite");
+  const store = new SemanticStore(dbPath, schemaPath());
+  const a = path.join(dir, "a.ts");
+  const b = path.join(dir, "b.ts");
+  fs.writeFileSync(a, "export const a = 1;\n");
+  fs.writeFileSync(b, "export const b = 2;\n");
+
+  const cfg = {
+    ...defaultConfig,
+    embedding: { ...defaultConfig.embedding, batch_size: 2 },
+    indexing: { ...defaultConfig.indexing, concurrency: 2, embed_concurrency: 2 },
+  };
+
+  const result = await runIndexingPipeline([a, b], store, mockEmbedder, cfg, {});
+  expect(result.indexed).toBe(2);
+  expect(result.failed.length).toBe(0);
+  expect(store.stats().files).toBe(2);
+  store.close();
+});
+
+test("chunked batch size 0 does not hang (regression)", () => {
+  const out = chunked([1, 2, 3, 4], 0);
+  expect(out.length).toBeGreaterThan(0);
+  expect(out.flat().length).toBe(4);
+});
+
+test("runIndexingPipeline writes each file once and progress is monotonic with cache-heavy mix", async () => {
+  const dir = fs.mkdtempSync(path.join(fs.realpathSync("/tmp"), "sem-pipe-"));
+  const files: string[] = [];
+  for (let i = 0; i < 10; i += 1) {
+    const fp = path.join(dir, `f-${i}.ts`);
+    fs.writeFileSync(fp, `export const v${i} = ${i};\n`);
+    files.push(fp);
+  }
+
+  const writeCalls: string[] = [];
+  let chunkLookupCalls = 0;
+  const fakeStore = {
+    getChunkByHash(_chunkHash: string): { embedding: number[] } | null {
+      chunkLookupCalls += 1;
+      if (chunkLookupCalls <= 8) {
+        return { embedding: new Array<number>(dim).fill(0.25) };
+      }
+      return null;
+    },
+    writeFileAndChunks(filePath: string): number {
+      writeCalls.push(filePath);
+      return writeCalls.length;
+    },
+  } as unknown as SemanticStore;
+
+  const variableEmbedder: Embedder = {
+    async embed(texts: string[]) {
+      await Bun.sleep(5);
+      return texts.map(() => new Array<number>(dim).fill(0.02));
+    },
+    async healthcheck() {
+      return true;
+    },
+  };
+
+  const progress: DeltaProgressPayload[] = [];
+  const cfg = {
+    ...defaultConfig,
+    embedding: { ...defaultConfig.embedding, batch_size: 1 },
+    indexing: { ...defaultConfig.indexing, concurrency: 1, embed_concurrency: 2 },
+  };
+  const result = await runIndexingPipeline(files, fakeStore, variableEmbedder, cfg, {
+    onProgress(update) {
+      progress.push(update);
+    },
+  });
+
+  expect(result.failed.length).toBe(0);
+  expect(result.indexed).toBe(10);
+  expect(writeCalls.length).toBe(10);
+  expect(new Set(writeCalls).size).toBe(10);
+  const currentValues = progress.map((p) => p.current);
+  for (let i = 1; i < currentValues.length; i += 1) {
+    expect(currentValues[i]).toBeGreaterThanOrEqual(currentValues[i - 1] ?? 0);
+  }
+});

package/src/indexer/pipeline.ts

@@ -0,0 +1,394 @@
+import crypto from "node:crypto";
+import type { PluginConfig } from "../config";
+import { chunkFile } from "../chunker";
+import { chunked, withQueryExpansion, type Embedder } from "../embedder/interface";
+import type { Logger } from "../logger";
+import type { DeltaProgressPayload } from "../runtime";
+import type { SemanticStore } from "../store/sqlite";
+import { mapPool } from "./pool";
+
+function sha256(input: string): string {
+  return crypto.createHash("sha256").update(input).digest("hex");
+}
+
+async function withIndexingRetry<T>(
+  fn: () => Promise<T>,
+  maxAttempts: number,
+  baseMs: number
+): Promise<T> {
+  let attempt = 0;
+  let waitMs = Math.max(1, baseMs);
+  let lastErr: unknown;
+  while (attempt < maxAttempts) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastErr = error;
+      attempt += 1;
+      if (attempt >= maxAttempts) break;
+      await Bun.sleep(waitMs);
+      waitMs *= 2;
+    }
+  }
+  throw lastErr;
+}
+
+interface PendingRow {
+  startLine: number;
+  endLine: number;
+  text: string;
+  chunkHash: string;
+  embedding?: number[];
+  needsEmbed: boolean;
+  embedText?: string;
+}
+
+interface PreparedOk {
+  kind: "ok";
+  filePath: string;
+  fileHash: string;
+  rows: PendingRow[];
+}
+
+interface PreparedSkip {
+  kind: "skip";
+}
+
+interface PreparedFail {
+  kind: "fail";
+  filePath: string;
+  error: unknown;
+}
+
+type Prepared = PreparedOk | PreparedSkip | PreparedFail;
+
+export interface IndexingPipelineFailure {
+  filePath: string;
+  error: unknown;
+}
+
+export interface IndexingPipelineResult {
+  /** Files successfully written to the store. */
+  indexed: number;
+  /** Per-file failures after retries or DB errors. */
+  failed: IndexingPipelineFailure[];
+}
+
+export interface IndexingPipelineOptions {
+  logger?: Logger;
+  onProgress?: (update: DeltaProgressPayload) => void;
+}
+
+export async function runIndexingPipeline(
+  filePaths: readonly string[],
+  store: SemanticStore,
+  embedder: Embedder,
+  config: PluginConfig,
+  options?: IndexingPipelineOptions
+): Promise<IndexingPipelineResult> {
+  const logger = options?.logger;
+  const onProgress = options?.onProgress;
+  const chunkConcurrency = Math.max(1, config.indexing.concurrency);
+  const embedConcurrency = Math.max(1, config.indexing.embed_concurrency);
+  const retryAttempts = Math.max(1, config.indexing.retry_attempts);
+  const retryBaseMs = Math.max(1, config.indexing.retry_base_ms);
+
+  if (filePaths.length === 0) {
+    return { indexed: 0, failed: [] };
+  }
+
+  await logger?.info("indexer.pipeline", {
+    message: "Chunking stage started",
+    extra: {
+      totalFiles: filePaths.length,
+      chunkConcurrency,
+      retryAttempts,
+      embedConcurrency,
+      batchSize: config.embedding.batch_size,
+    },
+  });
+
+  onProgress?.({
+    phase: "indexing",
+    current: 0,
+    total: filePaths.length,
+    label: "chunking files…",
+  });
+
+  async function prepareOne(filePath: string): Promise<Prepared> {
+    try {
+      return await withIndexingRetry(async (): Promise<PreparedOk | PreparedSkip> => {
+        const file = Bun.file(filePath);
+        if (!(await file.exists())) {
+          await logger?.debug("indexer.pipeline", {
+            message: "Skipped missing file",
+            extra: { filePath }
+          });
+          return { kind: "skip" };
+        }
+        const stat = await file.stat();
+        if (stat.size / 1024 > config.indexing.max_file_size_kb) {
+          await logger?.debug("indexer.pipeline", {
+            message: "Skipped oversized file",
+            extra: { filePath, sizeBytes: stat.size, maxFileSizeKb: config.indexing.max_file_size_kb }
+          });
+          return { kind: "skip" };
+        }
+        const text = await file.text();
+        const fileHash = sha256(text);
+        const chunks = await chunkFile(filePath, text, config);
+        await logger?.debug("indexer.pipeline", {
+          message: "Chunked file for indexing",
+          extra: { filePath, chunkCount: chunks.length }
+        });
+        const rows: PendingRow[] = [];
+        for (const chunk of chunks) {
+          const chunkHash = sha256(chunk.text);
+          const cached = store.getChunkByHash(chunkHash);
+          if (cached) {
+            rows.push({
+              startLine: chunk.startLine,
+              endLine: chunk.endLine,
+              text: chunk.text,
+              chunkHash,
+              embedding: cached.embedding,
+              needsEmbed: false,
+            });
+          } else {
+            rows.push({
+              startLine: chunk.startLine,
+              endLine: chunk.endLine,
+              text: chunk.text,
+              chunkHash,
+              needsEmbed: true,
+              embedText: withQueryExpansion(chunk.text),
+            });
+          }
+        }
+        return { kind: "ok", filePath, fileHash, rows };
+      }, retryAttempts, retryBaseMs);
+    } catch (error) {
+      return { kind: "fail", filePath, error };
+    }
+  }
+
+  const preparedList = await mapPool([...filePaths], chunkConcurrency, (fp, _i) => prepareOne(fp));
+
+  const okFiles: PreparedOk[] = [];
+  const failed: IndexingPipelineFailure[] = [];
+  let skippedStage1 = 0;
+  for (const p of preparedList) {
+    if (p.kind === "ok") {
+      okFiles.push(p);
+    } else if (p.kind === "fail") {
+      failed.push({ filePath: p.filePath, error: p.error });
+    } else {
+      skippedStage1 += 1;
+    }
+  }
+
+  const failedStage1 = failed.length;
+  let done = skippedStage1 + failedStage1;
+  const totalFiles = filePaths.length;
+  let reportedCurrent = done;
+
+  const totalChunksPrepared = okFiles.reduce((n, f) => n + f.rows.length, 0);
+  await logger?.info("indexer.pipeline", {
+    message: "Chunking stage finished",
+    extra: {
+      totalFiles,
+      preparedOk: okFiles.length,
+      skipped: skippedStage1,
+      failedPrepare: failedStage1,
+      totalChunks: totalChunksPrepared,
+    },
+  });
+
+  for (const f of failed) {
+    const errMsg = f.error instanceof Error ? f.error.message : String(f.error);
+    await logger?.warn("indexer.pipeline", {
+      message: "Prepare stage failed for file (after retries)",
+      extra: { filePath: f.filePath, errorMessage: errMsg },
+    });
+  }
+
+  interface EmbedJob {
+    okIndex: number;
+    rowIdx: number;
+    text: string;
+  }
+
+  const embedJobs: EmbedJob[] = [];
+  for (let fi = 0; fi < okFiles.length; fi += 1) {
+    const f = okFiles[fi];
+    if (!f) continue;
+    for (let ri = 0; ri < f.rows.length; ri += 1) {
+      const row = f.rows[ri];
+      if (!row) continue;
+      if (row.needsEmbed && row.embedText !== undefined) {
+        embedJobs.push({ okIndex: fi, rowIdx: ri, text: row.embedText });
+      }
+    }
+  }
+
+  const postStage1Done = skippedStage1 + failedStage1;
+  function emitProgress(current: number, label: string): void {
+    const capped = Math.min(totalFiles, current);
+    reportedCurrent = Math.max(reportedCurrent, capped);
+    onProgress?.({
+      phase: "indexing",
+      current: reportedCurrent,
+      total: totalFiles,
+      label,
+    });
+  }
+  emitProgress(done, embedJobs.length > 0 ? "embedding…" : "saving index…");
+
+  const writtenFileIndices = new Set<number>();
+  const writeFailedIndices = new Set<number>();
+  let indexed = 0;
+
+  async function tryFlushReadyFiles(embedBatchLabel?: string): Promise<void> {
+    for (let fi = 0; fi < okFiles.length; fi += 1) {
+      if (writtenFileIndices.has(fi) || writeFailedIndices.has(fi)) continue;
+      const f = okFiles[fi];
+      if (!f) continue;
+      if (!f.rows.every((r) => r.embedding !== undefined)) continue;
+      const writes: Array<{
+        startLine: number;
+        endLine: number;
+        text: string;
+        chunkHash: string;
+        embedding: number[];
+      }> = [];
+      for (const row of f.rows) {
+        const emb = row.embedding;
+        if (emb === undefined) continue;
+        writes.push({
+          startLine: row.startLine,
+          endLine: row.endLine,
+          text: row.text,
+          chunkHash: row.chunkHash,
+          embedding: emb,
+        });
+      }
+      try {
+        writtenFileIndices.add(fi);
+        store.writeFileAndChunks(f.filePath, f.fileHash, writes);
+        indexed += 1;
+        await logger?.info("indexer.pipeline", {
+          message: "Indexed file",
+          extra: {
+            filePath: f.filePath,
+            chunkCount: f.rows.length,
+            embeddedChunks: f.rows.filter((r) => r.needsEmbed).length,
+          },
+        });
+        done = postStage1Done + indexed + writeFailedIndices.size;
+        const tail = embedBatchLabel ? ` · ${embedBatchLabel}` : "";
+        emitProgress(done, `saved ${writtenFileIndices.size}/${okFiles.length}${tail}`);
+      } catch (error) {
+        writtenFileIndices.delete(fi);
+        writeFailedIndices.add(fi);
+        failed.push({ filePath: f.filePath, error });
+        await logger?.error("indexer.pipeline", {
+          message: "Failed to write file to index",
+          extra: { filePath: f.filePath },
+          error,
+        });
+      }
+    }
+  }
+
+  if (embedJobs.length > 0) {
+    const embedStartedAt = Date.now();
+    const batches = chunked(embedJobs, config.embedding.batch_size);
+    await logger?.info("indexer.pipeline", {
+      message: "Embedding stage started (writes flush as each file becomes ready)",
+      extra: {
+        textsToEmbed: embedJobs.length,
+        batchCount: batches.length,
+        batchSize: config.embedding.batch_size,
+        embedConcurrency,
+      },
+    });
+    let flushQueue = Promise.resolve();
+    const flushReadyFiles = (label?: string): Promise<void> => {
+      const run = async () => {
+        await tryFlushReadyFiles(label);
+      };
+      flushQueue = flushQueue.then(run, run);
+      return flushQueue;
+    };
+    await flushReadyFiles();
+    let embedBatchesDone = 0;
+    const batchCount = batches.length;
+    await mapPool(batches, embedConcurrency, async (batch) => {
+      if (batch.length === 0) return;
+      const texts = batch.map((j) => j.text);
+      const vectors = await withIndexingRetry(
+        () => embedder.embed(texts),
+        retryAttempts,
+        retryBaseMs
+      );
+      embedBatchesDone += 1;
+      const batchLabel = `embedding batch ${embedBatchesDone}/${batchCount}`;
+      if (vectors.length !== batch.length) {
+        await logger?.warn("indexer.pipeline", {
+          message: "Embedding provider returned wrong vector count — filling zeros for batch",
+          extra: { expected: batch.length, received: vectors.length },
+        });
+        for (const job of batch) {
+          if (job) {
+            const target = okFiles[job.okIndex]?.rows[job.rowIdx];
+            if (target) {
+              target.embedding = new Array<number>(config.embedding.dimensions).fill(0);
+            }
+          }
+        }
+      } else {
+        for (let vi = 0; vi < batch.length; vi += 1) {
+          const job = batch[vi];
+          const vec = vectors[vi];
+          if (!job) continue;
+          const target = okFiles[job.okIndex]?.rows[job.rowIdx];
+          if (target) {
+            target.embedding = vec ?? [];
+          }
+        }
+      }
+      done =
+        postStage1Done +
+        (batchCount > 0 ? Math.floor((embedBatchesDone / batchCount) * okFiles.length) : okFiles.length);
+      emitProgress(done, batchLabel);
+      await flushReadyFiles(batchLabel);
+    });
+    await logger?.info("indexer.pipeline", {
+      message: "Embedding stage finished",
+      extra: { elapsedMs: Date.now() - embedStartedAt, textsEmbedded: embedJobs.length },
+    });
+  } else {
+    await logger?.info("indexer.pipeline", {
+      message: "Embedding stage skipped (all chunks served from cache)",
+      extra: { files: okFiles.length },
+    });
+    await tryFlushReadyFiles();
+  }
+
+  await logger?.info("indexer.pipeline", {
+    message: "Write sweep for any remaining files",
+    extra: { pending: okFiles.length - writtenFileIndices.size - writeFailedIndices.size },
+  });
+  await tryFlushReadyFiles();
+
+  await logger?.info("indexer.pipeline", {
+    message: "Indexing pipeline finished",
+    extra: {
+      indexed,
+      failedCount: failed.length,
+      failedPaths: failed.map((x) => x.filePath),
+    },
+  });
+
+  return { indexed, failed };
+}

package/src/indexer/pool.ts

@@ -0,0 +1,25 @@
+/**
+ * Run up to `concurrency` tasks in parallel; results match `items` order.
+ */
+export async function mapPool<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  mapper: (item: T, index: number) => Promise<R>
+): Promise<R[]> {
+  if (items.length === 0) return [];
+  const limit = Math.max(1, Math.min(concurrency, items.length));
+  const results = new Array<R>(items.length);
+  let nextIndex = 0;
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const i = nextIndex;
+      nextIndex += 1;
+      if (i >= items.length) return;
+      const item = items[i];
+      if (item === undefined) return;
+      results[i] = await mapper(item, i);
+    }
+  };
+  await Promise.all(Array.from({ length: limit }, () => worker()));
+  return results;
+}

package/src/indexer/resume.ts

@@ -0,0 +1,14 @@
+import type { SemanticStore } from "../store/sqlite";
+
+export function markSyncStarted(store: SemanticStore): void {
+  store.setSyncState("sync_in_progress", "true");
+}
+
+export function markSyncCompleted(store: SemanticStore): void {
+  store.setSyncState("sync_in_progress", "false");
+  store.setSyncState("last_sync", new Date().toISOString());
+}
+
+export function isSyncInProgress(store: SemanticStore): boolean {
+  return store.getSyncState("sync_in_progress") === "true";
+}