@remnic/core 9.3.563 → 9.3.565

package/src/search/lancedb-backend.ts

@@ -1,6 +1,6 @@
 import { log } from "../logger.js";
 import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions, SearchResult } from "./port.js";
-import type { EmbedHelper } from "./embed-helper.js";
+import type { EmbedHelper, EmbedProviderIdentity, EmbedWithProviderResult } from "./embed-helper.js";
 import { scanMemoryDir } from "./document-scanner.js";
 import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
 
@@ -27,6 +27,10 @@ export class LanceDbBackend implements SearchBackend {
   private available = false;
   private db: any = null;
   private lanceModule: any = null;
+  private readonly vectorProviderCompatibility = new WeakMap<
+    object,
+    { providerIdentity: EmbedProviderIdentity; compatible: boolean }
+  >();
 
   constructor(opts: LanceDbBackendOptions) {
     this.dbPath = opts.dbPath;
@@ -125,7 +129,7 @@ export class LanceDbBackend implements SearchBackend {
 
   async updateCollection(collection: string, execution?: SearchExecutionOptions): Promise<void> {
     if (isSearchAborted(execution)) return;
-    const table = await this.ensureTableForCollection(collection);
+    let table = await this.ensureTableForCollection(collection);
     if (isSearchAborted(execution)) return;
     if (!table) return;
 
@@ -143,32 +147,68 @@ export class LanceDbBackend implements SearchBackend {
       return;
     }
 
-    const existingVectors = new Map<string, number[]>();
-    try {
-      const existingRows = await table.query().select(["docid", "vector"]).toArray();
-      for (const row of existingRows ?? []) {
-        if (isSearchAborted(execution)) return;
-        const docid = row.docid;
-        if (typeof docid !== "string") continue;
-        const vector = row.vector;
-        if (!vector || typeof vector !== "object") continue;
-        existingVectors.set(docid, Array.from(vector as ArrayLike<number>));
+    const embeddingProviderIdentity = this.embedHelper.getProviderIdentity();
+    const existingVectors = new Map<string, {
+      vector: number[];
+      providerIdentity?: string;
+    }>();
+    const vectorProviderColumnState = await this.tableVectorProviderColumnState(table);
+    if (vectorProviderColumnState === "missing") {
+      table = await this.recreateTableForCollection(collection);
+      if (isSearchAborted(execution)) return;
+      if (!table) return;
+    } else if (vectorProviderColumnState === "present") {
+      try {
+        const existingRows = await table.query().select(["docid", "vector", "vectorProvider"]).toArray();
+        for (const row of existingRows ?? []) {
+          if (isSearchAborted(execution)) return;
+          const docid = row.docid;
+          if (typeof docid !== "string") continue;
+          const vector = row.vector;
+          if (!vector || typeof vector !== "object") continue;
+          existingVectors.set(docid, {
+            vector: Array.from(vector as ArrayLike<number>),
+            providerIdentity: typeof row.vectorProvider === "string" ? row.vectorProvider : undefined,
+          });
+        }
+      } catch {
+        log.debug("LanceDbBackend skipped refresh after vector preservation failed");
+        return;
       }
-    } catch {
-      // Vector preservation is best-effort; refresh can proceed without it.
+    } else {
+      log.debug("LanceDbBackend skipped vector preservation after vectorProvider probe failed");
+      return;
     }
 
-    const rows = docs.map((d) => ({
-      docid: d.docid,
-      path: d.path,
-      content: d.content,
-      snippet: d.snippet,
-      vector: existingVectors.get(d.docid) ?? new Array(this.embeddingDimension).fill(0),
-    }));
+    const rows = docs.map((d) => {
+      const existing = existingVectors.get(d.docid);
+      const canPreserveVector =
+        existing &&
+        this.isCompatibleStoredVector(existing.vector) &&
+        (!embeddingProviderIdentity ||
+          existing.providerIdentity === embeddingProviderIdentity);
+      return {
+        docid: d.docid,
+        path: d.path,
+        content: d.content,
+        snippet: d.snippet,
+        vector: canPreserveVector
+          ? existing.vector
+          : new Array(this.embeddingDimension).fill(0),
+        vectorProvider: canPreserveVector
+          ? existing.providerIdentity ?? ""
+          : "",
+      };
+    });
 
     try {
       if (isSearchAborted(execution)) return;
       await table.add(rows, { mode: "overwrite" });
+      this.rememberVectorProviderCompatibility(
+        table,
+        embeddingProviderIdentity,
+        rows.length > 0 && rows.every((row) => row.vectorProvider === embeddingProviderIdentity),
+      );
       if (isSearchAborted(execution)) return;
       // Create FTS index on content column
       try {
@@ -193,25 +233,66 @@ export class LanceDbBackend implements SearchBackend {
     if (!table) return;
 
     try {
-      const allRows = await table.query().select(["docid", "content", "vector"]).toArray();
+      const embeddingProviderIdentity = this.embedHelper.getProviderIdentity();
+      const allRows = await table.query().select(["docid", "content", "vector", "vectorProvider"]).toArray();
       const needsEmbed = allRows.filter((row: any) => {
-        const vec = row.vector;
-        if (!vec || (typeof vec !== "object")) return true;
-        // Support both Array and typed arrays (e.g. Float32Array from Arrow)
-        const arr = Array.from(vec as ArrayLike<number>);
-        return arr.length === 0 || arr.every((v: number) => v === 0);
+        if (embeddingProviderIdentity && row.vectorProvider !== embeddingProviderIdentity) {
+          return true;
+        }
+        return !this.isCompatibleStoredVector(row.vector);
       });
 
-      if (needsEmbed.length === 0) return;
+      if (needsEmbed.length === 0) {
+        this.rememberVectorProviderCompatibility(table, embeddingProviderIdentity, true);
+        return;
+      }
 
-      const texts = needsEmbed.map((row: any) => row.content as string);
-      const vectors = await this.embedHelper.embedBatch(texts);
+      let rowsToEmbed = needsEmbed;
+      let embedResult = await this.embedHelper.embedBatchWithProvider(
+        rowsToEmbed.map((row: any) => row.content as string),
+      );
+      if (!embedResult) return;
+      if (
+        embeddingProviderIdentity &&
+        embedResult.providerIdentity !== embeddingProviderIdentity
+      ) {
+        const effectiveProviderIdentity = embedResult.providerIdentity;
+        const originalDocids = new Set(rowsToEmbed.map((row: any) => row.docid));
+        const effectiveNeedsEmbed = allRows.filter((row: any) => (
+          row.vectorProvider !== effectiveProviderIdentity ||
+          !this.isCompatibleStoredVector(row.vector)
+        ));
+        const sameRows =
+          effectiveNeedsEmbed.length === rowsToEmbed.length &&
+          effectiveNeedsEmbed.every((row: any) => originalDocids.has(row.docid));
+        if (!sameRows) {
+          const effectiveTexts = effectiveNeedsEmbed.map((row: any) => row.content as string);
+          const effectiveEmbedResult = await this.embedHelper.embedBatchWithProvider(effectiveTexts);
+          if (effectiveEmbedResult) {
+            rowsToEmbed = effectiveNeedsEmbed;
+            embedResult = effectiveEmbedResult;
+          }
+        }
+      }
+      const { vectors, providerIdentity } = embedResult;
 
-      for (let i = 0; i < needsEmbed.length; i++) {
+      let allEmbedded = true;
+      for (let i = 0; i < rowsToEmbed.length; i++) {
         const vec = vectors[i];
-        if (!vec) continue;
-        const docid = needsEmbed[i].docid;
-        await table.update({ where: `docid = '${docid.replace(/'/g, "''")}'`, values: { vector: vec } });
+        if (!this.isExpectedDimensionVector(vec)) {
+          allEmbedded = false;
+          continue;
+        }
+        const docid = rowsToEmbed[i].docid;
+        await table.update({
+          where: `docid = '${docid.replace(/'/g, "''")}'`,
+          values: { vector: vec, vectorProvider: providerIdentity },
+        });
+      }
+      if (allEmbedded) {
+        this.rememberVectorProviderCompatibility(table, providerIdentity, true);
+      } else {
+        this.rememberVectorProviderCompatibility(table, providerIdentity, false);
       }
     } catch (err) {
       log.debug(`LanceDbBackend embed failed: ${err}`);
@@ -264,6 +345,7 @@ export class LanceDbBackend implements SearchBackend {
       content: "",
       snippet: "",
       vector: new Array(this.embeddingDimension).fill(0),
+      vectorProvider: "",
     };
     const newTable = await db.createTable(collection, [emptyRow]);
     try {
@@ -279,6 +361,30 @@ export class LanceDbBackend implements SearchBackend {
     return newTable;
   }
 
+  private async recreateTableForCollection(collection: string): Promise<any> {
+    const db = await this.ensureDb();
+    try {
+      await db.dropTable(collection).catch(() => {});
+    } catch {
+      // Best-effort legacy schema migration; table creation below may still recover.
+    }
+    if (collection === this.collection) this.table = null;
+    return this.ensureTableForCollection(collection);
+  }
+
+  private async tableVectorProviderColumnState(table: any): Promise<"present" | "missing" | "unknown"> {
+    try {
+      await table.query().select(["vectorProvider"]).toArray();
+      return "present";
+    } catch (err) {
+      if (isMissingVectorProviderColumnError(err)) {
+        return "missing";
+      }
+      log.debug(`LanceDbBackend vectorProvider column probe failed: ${err}`);
+      return "unknown";
+    }
+  }
+
   private async ensureTable(): Promise<any> {
     if (this.table) return this.table;
 
@@ -297,6 +403,7 @@ export class LanceDbBackend implements SearchBackend {
       content: "",
       snippet: "",
       vector: new Array(this.embeddingDimension).fill(0),
+      vectorProvider: "",
     };
     this.table = await db.createTable(this.collection, [emptyRow]);
     // Create FTS index on content column
@@ -330,23 +437,23 @@ export class LanceDbBackend implements SearchBackend {
       }
 
       if (mode === "vector") {
-        const vec = await this.embedHelper.embed(query, { signal: execution?.signal });
+        const embedResult = await this.resolveCompatibleQueryEmbedding(table, query, execution);
         throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
-        if (!vec) {
+        if (!embedResult) {
           // Fall back to FTS
           const results = await table.search(query, "fts").limit(limit).toArray();
           throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
           return this.mapRows(results);
         }
-        const results = await table.search(vec).limit(limit).toArray();
+        const results = await table.search(embedResult.vector).limit(limit).toArray();
         throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       }
 
       // hybrid — try FTS+vector with RRF reranking
-      const vec = await this.embedHelper.embed(query, { signal: execution?.signal });
+      const embedResult = await this.resolveCompatibleQueryEmbedding(table, query, execution);
       throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
-      if (!vec) {
+      if (!embedResult) {
         const results = await table.search(query, "fts").limit(limit).toArray();
         throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
@@ -355,14 +462,14 @@ export class LanceDbBackend implements SearchBackend {
       try {
         const results = await table
           .search(query, "hybrid")
-          .vector(vec)
+          .vector(embedResult.vector)
           .limit(limit)
           .toArray();
         throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       } catch {
         // Hybrid may not be supported in all LanceDB versions — fall back to vector
-        const results = await table.search(vec).limit(limit).toArray();
+        const results = await table.search(embedResult.vector).limit(limit).toArray();
         throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       }
@@ -372,6 +479,93 @@ export class LanceDbBackend implements SearchBackend {
     }
   }
 
+  private async resolveCompatibleQueryEmbedding(
+    table: any,
+    query: string,
+    execution?: SearchExecutionOptions,
+  ): Promise<EmbedWithProviderResult | null> {
+    const embedResult = await this.embedHelper.embedWithProvider(query, { signal: execution?.signal });
+    throwIfSearchAborted(execution, "LanceDbBackend query embedding aborted");
+    if (!embedResult || !this.isExpectedDimensionVector(embedResult.vector)) return null;
+
+    const storedProviderIdentity = await this.findCompatibleStoredVectorProvider(table, execution);
+    if (!storedProviderIdentity) {
+      this.rememberVectorProviderCompatibility(table, embedResult.providerIdentity, false);
+      return null;
+    }
+    if (storedProviderIdentity === embedResult.providerIdentity) return embedResult;
+
+    const fallbackEmbed = await this.embedQueryWithStoredFallbackProvider(query, storedProviderIdentity, execution);
+    throwIfSearchAborted(execution, "LanceDbBackend fallback query embedding aborted");
+    if (
+      fallbackEmbed &&
+      fallbackEmbed.providerIdentity === storedProviderIdentity &&
+      this.isExpectedDimensionVector(fallbackEmbed.vector)
+    ) {
+      return fallbackEmbed;
+    }
+
+    this.rememberVectorProviderCompatibility(table, embedResult.providerIdentity, false);
+    return null;
+  }
+
+  private async embedQueryWithStoredFallbackProvider(
+    query: string,
+    providerIdentity: EmbedProviderIdentity,
+    execution?: SearchExecutionOptions,
+  ): Promise<EmbedWithProviderResult | null> {
+    const embedWithIdentity = (this.embedHelper as unknown as {
+      embedWithFallbackProviderIdentity?: (
+        text: string,
+        identity: EmbedProviderIdentity,
+        options?: { signal?: AbortSignal },
+      ) => Promise<EmbedWithProviderResult | null>;
+    }).embedWithFallbackProviderIdentity;
+    if (typeof embedWithIdentity !== "function") return null;
+    return embedWithIdentity.call(this.embedHelper, query, providerIdentity, { signal: execution?.signal });
+  }
+
+  private async findCompatibleStoredVectorProvider(
+    table: any,
+    execution?: SearchExecutionOptions,
+  ): Promise<EmbedProviderIdentity | null> {
+    try {
+      const cached = this.vectorProviderCompatibility.get(table);
+      if (cached?.compatible) return cached.providerIdentity;
+      const rows = await table.query().select(["vector", "vectorProvider"]).toArray();
+      let providerIdentity: EmbedProviderIdentity | null = null;
+      let compatible = rows.length > 0;
+      for (const row of rows ?? []) {
+        throwIfSearchAborted(execution, "LanceDbBackend vector provider check aborted");
+        if (
+          typeof row.vectorProvider !== "string" ||
+          row.vectorProvider.length === 0 ||
+          !this.isCompatibleStoredVector(row.vector)
+        ) {
+          compatible = false;
+          break;
+        }
+        if (providerIdentity && row.vectorProvider !== providerIdentity) {
+          compatible = false;
+          break;
+        }
+        providerIdentity = row.vectorProvider as EmbedProviderIdentity;
+      }
+      if (compatible && providerIdentity) {
+        this.vectorProviderCompatibility.set(table, {
+          providerIdentity,
+          compatible: true,
+        });
+        return providerIdentity;
+      }
+      return null;
+    } catch (err) {
+      if (isSearchAborted(execution)) throw err;
+      log.debug(`LanceDbBackend stored vector provider check failed: ${err}`);
+      return null;
+    }
+  }
+
   private mapRows(rows: any[]): SearchResult[] {
     return (rows ?? [])
       .filter((row) => row.docid && row.docid !== "__placeholder__")
@@ -382,4 +576,66 @@ export class LanceDbBackend implements SearchBackend {
         score: row._relevance_score ?? (row._distance != null ? 1 / (1 + (row._distance ?? 0)) : 0.5),
       }));
   }
+
+  private async tableHasCompatibleVectors(
+    table: any,
+    providerIdentity: EmbedProviderIdentity,
+    execution?: SearchExecutionOptions,
+  ): Promise<boolean> {
+    try {
+      const cached = this.vectorProviderCompatibility.get(table);
+      if (cached?.providerIdentity === providerIdentity) return cached.compatible;
+      const rows = await table.query().select(["vector", "vectorProvider"]).toArray();
+      let compatible = rows.length > 0;
+      for (const row of rows ?? []) {
+        throwIfSearchAborted(execution, "LanceDbBackend vector provider check aborted");
+        if (
+          row.vectorProvider !== providerIdentity ||
+          !this.isCompatibleStoredVector(row.vector)
+        ) {
+          compatible = false;
+          break;
+        }
+      }
+      this.vectorProviderCompatibility.set(table, { providerIdentity, compatible });
+      return compatible;
+    } catch (err) {
+      if (isSearchAborted(execution)) throw err;
+      log.debug(`LanceDbBackend vector provider check failed: ${err}`);
+      return false;
+    }
+  }
+
+  private rememberVectorProviderCompatibility(
+    table: unknown,
+    providerIdentity: EmbedProviderIdentity | null,
+    compatible: boolean,
+  ): void {
+    if (!table || typeof table !== "object") return;
+    if (!providerIdentity) {
+      this.vectorProviderCompatibility.delete(table);
+      return;
+    }
+    this.vectorProviderCompatibility.set(table, { providerIdentity, compatible });
+  }
+
+  private isExpectedDimensionVector(vector: number[] | null | undefined): vector is number[] {
+    return Array.isArray(vector) && vector.length === this.embeddingDimension;
+  }
+
+  private isCompatibleStoredVector(vector: unknown): boolean {
+    if (!vector || typeof vector !== "object") return false;
+    const arr = Array.from(vector as ArrayLike<number>);
+    return (
+      arr.length === this.embeddingDimension &&
+      arr.every((value) => Number.isFinite(value)) &&
+      arr.some((value) => value !== 0)
+    );
+  }
+}
+
+function isMissingVectorProviderColumnError(err: unknown): boolean {
+  const message = err instanceof Error ? err.message : String(err);
+  return /\bvectorProvider\b/i.test(message) &&
+    /\b(column|field|schema|missing|not found|not exist|does not exist|unknown)\b/i.test(message);
 }