@remnic/core 9.3.563 → 9.3.564

package/dist/chunk-M46RYSMW.js

@@ -0,0 +1,597 @@
+import {
+  isSearchAborted,
+  throwIfSearchAborted
+} from "./chunk-CINZGPSJ.js";
+import {
+  scanMemoryDir
+} from "./chunk-Q4CAQGKQ.js";
+import {
+  log
+} from "./chunk-2ODBA7MQ.js";
+
+// src/search/orama-backend.ts
+import path from "path";
+import { mkdir, readdir, readFile, rename, rm, writeFile } from "fs/promises";
+var ORAMA_COLLECTION_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+function pathIsInside(parent, child) {
+  const relative = path.relative(parent, child);
+  return relative === "" || !relative.startsWith("..") && !path.isAbsolute(relative);
+}
+function resolveOramaCollectionDbFilePath(dbPath, collection) {
+  if (!ORAMA_COLLECTION_FILENAME_PATTERN.test(collection)) {
+    throw new Error(
+      `Invalid Orama collection name ${JSON.stringify(collection)}. Collection names must match [A-Za-z0-9][A-Za-z0-9._-]*.`
+    );
+  }
+  const resolvedDbPath = path.resolve(dbPath);
+  const filePath = path.resolve(resolvedDbPath, `${collection}.msp`);
+  if (!pathIsInside(resolvedDbPath, filePath)) {
+    throw new Error(
+      `Invalid Orama collection path for ${JSON.stringify(collection)}: resolved outside dbPath.`
+    );
+  }
+  return filePath;
+}
+var OramaBackend = class {
+  dbPath;
+  collection;
+  embedHelper;
+  memoryDir;
+  embeddingDimension;
+  available = false;
+  db = null;
+  oramaModule = null;
+  persistModule = null;
+  vectorProviderCompatibility = /* @__PURE__ */ new WeakMap();
+  constructor(opts) {
+    this.dbPath = opts.dbPath;
+    this.collection = opts.collection;
+    this.embedHelper = opts.embedHelper;
+    this.memoryDir = opts.memoryDir;
+    this.embeddingDimension = opts.embeddingDimension;
+  }
+  async probe() {
+    try {
+      await this.ensureModules();
+      await this.ensureDb();
+      this.available = true;
+      return true;
+    } catch (err) {
+      log.debug(`OramaBackend probe failed: ${err}`);
+      this.available = false;
+      return false;
+    }
+  }
+  isAvailable() {
+    return this.available;
+  }
+  debugStatus() {
+    return `backend=orama available=${this.available} dbPath=${this.dbPath}`;
+  }
+  async search(query, _collection, maxResults, _options, execution) {
+    return this.hybridSearch(query, _collection, maxResults, execution);
+  }
+  async searchGlobal(query, maxResults, execution) {
+    const limit = maxResults ?? 10;
+    if (!this.available) return [];
+    try {
+      throwIfSearchAborted(execution, "OramaBackend global search aborted");
+      const files = await this.listDbFiles();
+      const allResults = [];
+      for (const file of files) {
+        throwIfSearchAborted(execution, "OramaBackend global search aborted");
+        const db = await this.loadDbFromFile(file);
+        if (!db) continue;
+        const results = await this.searchDb(db, query, "hybrid", limit, execution);
+        allResults.push(...results);
+      }
+      allResults.sort((a, b) => b.score - a.score);
+      return allResults.slice(0, limit);
+    } catch (err) {
+      log.debug(`OramaBackend searchGlobal failed: ${err}`);
+      return [];
+    }
+  }
+  async bm25Search(query, collection, maxResults, execution) {
+    if (isSearchAborted(execution)) return [];
+    const db = await this.ensureDbForCollection(collection ?? this.collection);
+    if (isSearchAborted(execution)) return [];
+    if (!db) return [];
+    return this.searchDb(db, query, "fulltext", maxResults ?? 10, execution);
+  }
+  async vectorSearch(query, collection, maxResults, execution) {
+    if (isSearchAborted(execution)) return [];
+    const db = await this.ensureDbForCollection(collection ?? this.collection);
+    if (isSearchAborted(execution)) return [];
+    if (!db) return [];
+    return this.searchDb(db, query, "vector", maxResults ?? 10, execution);
+  }
+  async hybridSearch(query, collection, maxResults, execution) {
+    if (isSearchAborted(execution)) return [];
+    const db = await this.ensureDbForCollection(collection ?? this.collection);
+    if (isSearchAborted(execution)) return [];
+    if (!db) return [];
+    return this.searchDb(db, query, "hybrid", maxResults ?? 10, execution);
+  }
+  async update(execution) {
+    await this.updateCollection(this.collection, execution);
+  }
+  async updateCollection(collection, execution) {
+    if (isSearchAborted(execution)) return;
+    const db = await this.ensureDbForCollection(collection);
+    if (isSearchAborted(execution)) return;
+    if (!db) return;
+    const { search: oramaSearch, insert, remove, count, getByID } = this.oramaModule;
+    const docs = await scanMemoryDir(this.memoryDir);
+    if (isSearchAborted(execution)) return;
+    const docMap = new Map(docs.map((d) => [d.docid, d]));
+    const { update: oramaUpdate } = this.oramaModule;
+    const embeddingProviderIdentity = this.embedHelper.getProviderIdentity();
+    let allRowsCompatible = !!embeddingProviderIdentity && docs.length > 0;
+    const existingDocs = /* @__PURE__ */ new Map();
+    const existingCount = await count(db);
+    if (existingCount > 0) {
+      const allHits = await oramaSearch(db, {
+        term: "",
+        limit: existingCount + 100
+      });
+      for (const hit of allHits.hits) {
+        if (isSearchAborted(execution)) return;
+        const storedDocument = typeof getByID === "function" ? await getByID(db, hit.id) : hit.document;
+        const document = storedDocument ?? hit.document ?? {};
+        if (!docMap.has(document.id)) {
+          await remove(db, hit.id);
+        } else {
+          existingDocs.set(document.id, {
+            internalId: hit.id,
+            vector: this.normalizeStoredVector(document.vector) ?? void 0,
+            vectorProvider: typeof document.vectorProvider === "string" ? document.vectorProvider : void 0
+          });
+        }
+      }
+    }
+    for (const doc of docs) {
+      if (isSearchAborted(execution)) return;
+      const existing = existingDocs.get(doc.docid);
+      if (existing) {
+        const payload = {
+          id: doc.docid,
+          path: doc.path,
+          content: doc.content,
+          snippet: doc.snippet
+        };
+        const preservesCompatibleProvider = !!embeddingProviderIdentity && existing.vectorProvider === embeddingProviderIdentity;
+        if (preservesCompatibleProvider) {
+          if (this.isCompatibleStoredVector(existing.vector)) {
+            payload.vector = existing.vector;
+            payload.vectorProvider = existing.vectorProvider ?? "";
+          } else {
+            payload.vector = this.zeroVector();
+            payload.vectorProvider = "";
+            allRowsCompatible = false;
+          }
+        } else if (!embeddingProviderIdentity && this.isCompatibleStoredVector(existing.vector)) {
+          payload.vector = existing.vector;
+          payload.vectorProvider = existing.vectorProvider ?? "";
+          allRowsCompatible = false;
+        } else {
+          payload.vector = this.zeroVector();
+          payload.vectorProvider = "";
+          allRowsCompatible = false;
+        }
+        try {
+          await oramaUpdate(db, existing.internalId, payload);
+        } catch {
+          allRowsCompatible = false;
+        }
+      } else {
+        allRowsCompatible = false;
+        try {
+          await insert(db, {
+            id: doc.docid,
+            path: doc.path,
+            content: doc.content,
+            snippet: doc.snippet,
+            vector: this.zeroVector(),
+            vectorProvider: ""
+          });
+        } catch {
+          allRowsCompatible = false;
+        }
+      }
+    }
+    if (isSearchAborted(execution)) return;
+    await this.persistDbForCollection(db, collection);
+    this.rememberVectorProviderCompatibility(
+      db,
+      embeddingProviderIdentity,
+      allRowsCompatible
+    );
+  }
+  async embed() {
+    await this.embedCollection(this.collection);
+  }
+  async embedCollection(collection) {
+    if (!this.embedHelper.isAvailable()) return;
+    const db = await this.ensureDbForCollection(collection);
+    if (!db) return;
+    const { search: oramaSearch, update: oramaUpdate, count } = this.oramaModule;
+    const existingCount = await count(db);
+    if (existingCount === 0) return;
+    const embeddingProviderIdentity = this.embedHelper.getProviderIdentity();
+    const allHits = await oramaSearch(db, { term: "", limit: existingCount + 100 });
+    const needsEmbed = allHits.hits.filter((h) => {
+      const vector = this.normalizeStoredVector(h.document?.vector);
+      return embeddingProviderIdentity && h.document?.vectorProvider !== embeddingProviderIdentity || !this.isCompatibleStoredVector(vector);
+    });
+    if (needsEmbed.length === 0) {
+      this.rememberVectorProviderCompatibility(db, embeddingProviderIdentity, true);
+      return;
+    }
+    let rowsToEmbed = needsEmbed;
+    let embedResult = await this.embedHelper.embedBatchWithProvider(
+      rowsToEmbed.map((h) => h.document.content)
+    );
+    if (!embedResult) return;
+    if (embeddingProviderIdentity && embedResult.providerIdentity !== embeddingProviderIdentity) {
+      const effectiveProviderIdentity = embedResult.providerIdentity;
+      const originalIds = new Set(rowsToEmbed.map((h) => h.id));
+      const effectiveNeedsEmbed = allHits.hits.filter((h) => {
+        const vector = this.normalizeStoredVector(h.document?.vector);
+        return h.document?.vectorProvider !== effectiveProviderIdentity || !this.isCompatibleStoredVector(vector);
+      });
+      const sameRows = effectiveNeedsEmbed.length === rowsToEmbed.length && effectiveNeedsEmbed.every((h) => originalIds.has(h.id));
+      if (!sameRows) {
+        const effectiveTexts = effectiveNeedsEmbed.map((h) => h.document.content);
+        const effectiveEmbedResult = await this.embedHelper.embedBatchWithProvider(effectiveTexts);
+        if (effectiveEmbedResult) {
+          rowsToEmbed = effectiveNeedsEmbed;
+          embedResult = effectiveEmbedResult;
+        }
+      }
+    }
+    const { vectors, providerIdentity } = embedResult;
+    let allEmbedded = true;
+    for (let i = 0; i < rowsToEmbed.length; i++) {
+      const vec = vectors[i];
+      if (!this.isExpectedDimensionVector(vec)) {
+        allEmbedded = false;
+        continue;
+      }
+      const doc = rowsToEmbed[i].document;
+      await oramaUpdate(db, rowsToEmbed[i].id, {
+        id: doc.id,
+        path: doc.path,
+        content: doc.content,
+        snippet: doc.snippet,
+        vector: vec,
+        vectorProvider: providerIdentity
+      });
+    }
+    await this.persistDbForCollection(db, collection);
+    if (allEmbedded) {
+      this.rememberVectorProviderCompatibility(db, providerIdentity, true);
+    } else {
+      this.rememberVectorProviderCompatibility(db, providerIdentity, false);
+    }
+  }
+  async ensureCollection(_memoryDir, _execution) {
+    try {
+      await this.ensureModules();
+      await this.ensureDb();
+      return "present";
+    } catch {
+      return "missing";
+    }
+  }
+  async ensureModules() {
+    if (this.oramaModule && this.persistModule) return;
+    this.oramaModule = await import("@orama/orama");
+    this.persistModule = await import("@orama/plugin-data-persistence");
+  }
+  async ensureDb() {
+    if (this.db) return this.db;
+    await this.ensureModules();
+    await mkdir(this.dbPath, { recursive: true });
+    const filePath = this.dbFilePath(this.collection);
+    let raw;
+    try {
+      raw = await readFile(filePath, "utf-8");
+    } catch {
+      this.db = await this.createDb();
+      return this.db;
+    }
+    this.db = await this.migrateLegacyVectorProviderSchema(
+      await this.persistModule.restore("json", raw),
+      this.collection
+    );
+    return this.db;
+  }
+  async ensureDbForCollection(collection) {
+    if (collection === this.collection) return this.ensureDb();
+    await this.ensureModules();
+    await mkdir(this.dbPath, { recursive: true });
+    const filePath = this.dbFilePath(collection);
+    let raw;
+    try {
+      raw = await readFile(filePath, "utf-8");
+    } catch {
+      return await this.createDb();
+    }
+    return await this.migrateLegacyVectorProviderSchema(
+      await this.persistModule.restore("json", raw),
+      collection
+    );
+  }
+  async createDb() {
+    const { create } = this.oramaModule;
+    const schema = {
+      id: "string",
+      path: "string",
+      content: "string",
+      snippet: "string",
+      vectorProvider: "string",
+      vector: `vector[${this.embeddingDimension}]`
+    };
+    return await create({ schema });
+  }
+  async migrateLegacyVectorProviderSchema(db, collection) {
+    const { search: oramaSearch, count, insert } = this.oramaModule;
+    const existingCount = await count(db);
+    if (existingCount === 0) {
+      const migrated2 = await this.createDb();
+      await this.persistDbForCollection(migrated2, collection);
+      return migrated2;
+    }
+    const allHits = await oramaSearch(db, { term: "", limit: existingCount + 100 });
+    const hits = allHits.hits ?? [];
+    const needsMigration = hits.some(
+      (hit) => typeof hit.document?.vectorProvider !== "string"
+    );
+    if (!needsMigration) return db;
+    const migrated = await this.createDb();
+    for (const hit of hits) {
+      const doc = this.getStoredDocument(db, hit);
+      const vector = this.getStoredVector(db, hit, doc);
+      const payload = {
+        id: typeof doc.id === "string" && doc.id.length > 0 ? doc.id : String(hit.id),
+        path: typeof doc.path === "string" ? doc.path : "",
+        content: typeof doc.content === "string" ? doc.content : "",
+        snippet: typeof doc.snippet === "string" ? doc.snippet : typeof doc.content === "string" ? doc.content.slice(0, 200) : "",
+        vectorProvider: typeof doc.vectorProvider === "string" ? doc.vectorProvider : ""
+      };
+      if (vector) {
+        payload.vector = vector;
+      } else {
+        payload.vector = this.zeroVector();
+      }
+      await insert(migrated, payload);
+    }
+    await this.persistDbForCollection(migrated, collection);
+    return migrated;
+  }
+  async persistDbForCollection(db, collection) {
+    const data = await this.persistModule.persist(db, "json");
+    const filePath = this.dbFilePath(collection);
+    await mkdir(path.dirname(filePath), { recursive: true });
+    const tempPath = path.join(
+      path.dirname(filePath),
+      `.${path.basename(filePath)}.${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}.tmp`
+    );
+    try {
+      await writeFile(tempPath, data, "utf-8");
+      await rename(tempPath, filePath);
+    } catch (err) {
+      await rm(tempPath, { force: true }).catch(() => void 0);
+      throw err;
+    }
+  }
+  dbFilePath(collection) {
+    return resolveOramaCollectionDbFilePath(this.dbPath, collection);
+  }
+  async listDbFiles() {
+    try {
+      const entries = await readdir(this.dbPath);
+      return entries.filter((e) => e.endsWith(".msp")).map((e) => path.join(this.dbPath, e));
+    } catch {
+      return [];
+    }
+  }
+  async loadDbFromFile(filePath) {
+    try {
+      await this.ensureModules();
+      const raw = await readFile(filePath, "utf-8");
+      const collection = path.basename(filePath, ".msp");
+      return await this.migrateLegacyVectorProviderSchema(
+        await this.persistModule.restore("json", raw),
+        collection
+      );
+    } catch (err) {
+      log.debug(`OramaBackend failed to load ${filePath}: ${err}`);
+      return null;
+    }
+  }
+  async searchDb(db, query, mode, limit, execution) {
+    const { search: oramaSearch } = this.oramaModule;
+    try {
+      throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);
+      let searchParams;
+      if (mode === "fulltext") {
+        searchParams = { term: query, limit };
+      } else if (mode === "vector") {
+        const embedResult = await this.resolveCompatibleQueryEmbedding(db, query, execution);
+        throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);
+        if (!embedResult) {
+          searchParams = { term: query, limit };
+        } else {
+          searchParams = { mode: "vector", vector: { value: embedResult.vector, property: "vector" }, limit };
+        }
+      } else {
+        const embedResult = await this.resolveCompatibleQueryEmbedding(db, query, execution);
+        throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);
+        if (!embedResult) {
+          searchParams = { term: query, limit };
+        } else {
+          searchParams = { mode: "hybrid", term: query, vector: { value: embedResult.vector, property: "vector" }, limit };
+        }
+      }
+      throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);
+      const result = await oramaSearch(db, searchParams);
+      throwIfSearchAborted(execution, `OramaBackend ${mode} search aborted`);
+      return (result.hits ?? []).map((hit) => ({
+        docid: hit.document?.id ?? "",
+        path: hit.document?.path ?? "",
+        snippet: hit.document?.snippet ?? hit.document?.content?.slice(0, 200) ?? "",
+        score: hit.score ?? 0
+      }));
+    } catch (err) {
+      log.debug(`OramaBackend search (${mode}) failed: ${err}`);
+      return [];
+    }
+  }
+  async resolveCompatibleQueryEmbedding(db, query, execution) {
+    const embedResult = await this.embedHelper.embedWithProvider(query, { signal: execution?.signal });
+    throwIfSearchAborted(execution, "OramaBackend query embedding aborted");
+    if (!embedResult || !this.isExpectedDimensionVector(embedResult.vector)) return null;
+    const storedProviderIdentity = await this.findCompatibleStoredVectorProvider(db, execution);
+    if (!storedProviderIdentity) {
+      this.rememberVectorProviderCompatibility(db, embedResult.providerIdentity, false);
+      return null;
+    }
+    if (storedProviderIdentity === embedResult.providerIdentity) return embedResult;
+    const fallbackEmbed = await this.embedQueryWithStoredFallbackProvider(query, storedProviderIdentity, execution);
+    throwIfSearchAborted(execution, "OramaBackend fallback query embedding aborted");
+    if (fallbackEmbed && fallbackEmbed.providerIdentity === storedProviderIdentity && this.isExpectedDimensionVector(fallbackEmbed.vector)) {
+      return fallbackEmbed;
+    }
+    this.rememberVectorProviderCompatibility(db, embedResult.providerIdentity, false);
+    return null;
+  }
+  async embedQueryWithStoredFallbackProvider(query, providerIdentity, execution) {
+    const embedWithIdentity = this.embedHelper.embedWithFallbackProviderIdentity;
+    if (typeof embedWithIdentity !== "function") return null;
+    return embedWithIdentity.call(this.embedHelper, query, providerIdentity, { signal: execution?.signal });
+  }
+  async findCompatibleStoredVectorProvider(db, execution) {
+    const { search: oramaSearch, count } = this.oramaModule;
+    try {
+      const cached = this.vectorProviderCompatibility.get(db);
+      if (cached?.compatible) return cached.providerIdentity;
+      const existingCount = await count(db);
+      if (existingCount === 0) return null;
+      const allHits = await oramaSearch(db, {
+        term: "",
+        limit: existingCount + 100,
+        properties: ["vectorProvider"]
+      });
+      let providerIdentity = null;
+      let compatible = (allHits.hits ?? []).length > 0;
+      for (const hit of allHits.hits ?? []) {
+        throwIfSearchAborted(execution, "OramaBackend vector provider check aborted");
+        const doc = this.getStoredDocument(db, hit);
+        if (typeof doc.vectorProvider !== "string" || doc.vectorProvider.length === 0 || !this.isCompatibleStoredVector(this.getStoredVector(db, hit, doc))) {
+          compatible = false;
+          break;
+        }
+        if (providerIdentity && doc.vectorProvider !== providerIdentity) {
+          compatible = false;
+          break;
+        }
+        providerIdentity = doc.vectorProvider;
+      }
+      if (compatible && providerIdentity) {
+        this.vectorProviderCompatibility.set(db, {
+          providerIdentity,
+          compatible: true
+        });
+        return providerIdentity;
+      }
+      return null;
+    } catch (err) {
+      if (isSearchAborted(execution)) throw err;
+      log.debug(`OramaBackend stored vector provider check failed: ${err}`);
+      return null;
+    }
+  }
+  async dbHasCompatibleVectors(db, providerIdentity, execution) {
+    const { search: oramaSearch, count } = this.oramaModule;
+    try {
+      const cached = this.vectorProviderCompatibility.get(db);
+      if (cached?.providerIdentity === providerIdentity) return cached.compatible;
+      const existingCount = await count(db);
+      if (existingCount === 0) return false;
+      const allHits = await oramaSearch(db, {
+        term: "",
+        limit: existingCount + 100,
+        properties: ["vectorProvider"]
+      });
+      let compatible = (allHits.hits ?? []).length > 0;
+      for (const hit of allHits.hits ?? []) {
+        throwIfSearchAborted(execution, "OramaBackend vector provider check aborted");
+        const doc = this.getStoredDocument(db, hit);
+        if (doc.vectorProvider !== providerIdentity || !this.isCompatibleStoredVector(this.getStoredVector(db, hit, doc))) {
+          compatible = false;
+          break;
+        }
+      }
+      this.vectorProviderCompatibility.set(db, { providerIdentity, compatible });
+      return compatible;
+    } catch (err) {
+      if (isSearchAborted(execution)) throw err;
+      log.debug(`OramaBackend vector provider check failed: ${err}`);
+      return false;
+    }
+  }
+  rememberVectorProviderCompatibility(db, providerIdentity, compatible) {
+    if (!db || typeof db !== "object") return;
+    if (!providerIdentity) {
+      this.vectorProviderCompatibility.delete(db);
+      return;
+    }
+    this.vectorProviderCompatibility.set(db, { providerIdentity, compatible });
+  }
+  getStoredDocument(db, hit) {
+    const internalId = this.getInternalDocumentId(db, hit);
+    const internalDoc = internalId !== void 0 && internalId !== null ? db?.data?.docs?.docs?.[String(internalId)] : void 0;
+    if (internalDoc && typeof internalDoc === "object") {
+      return internalDoc;
+    }
+    return hit?.document && typeof hit.document === "object" ? hit.document : {};
+  }
+  getStoredVector(db, hit, doc) {
+    const documentVector = this.normalizeStoredVector(doc.vector);
+    if (documentVector) return documentVector;
+    const internalId = this.getInternalDocumentId(db, hit);
+    if (internalId === void 0 || internalId === null) return null;
+    const vectorEntry = db?.data?.index?.vectorIndexes?.vector?.node?.vectors?.get?.(internalId);
+    const vector = Array.isArray(vectorEntry) ? vectorEntry[1] : vectorEntry;
+    return this.normalizeStoredVector(vector);
+  }
+  getInternalDocumentId(db, hit) {
+    const publicId = typeof hit?.id === "string" ? hit.id : typeof hit?.document?.id === "string" ? hit.document.id : void 0;
+    return publicId && typeof db?.internalDocumentIDStore?.idToInternalId?.get === "function" ? db.internalDocumentIDStore.idToInternalId.get(publicId) : void 0;
+  }
+  isExpectedDimensionVector(vector) {
+    return Array.isArray(vector) && vector.length === this.embeddingDimension;
+  }
+  isCompatibleStoredVector(vector) {
+    if (!vector || typeof vector !== "object") return false;
+    const arr = Array.from(vector);
+    return arr.length === this.embeddingDimension && arr.every((value) => Number.isFinite(value)) && arr.some((value) => value !== 0);
+  }
+  zeroVector() {
+    return Array.from({ length: this.embeddingDimension }, () => 0);
+  }
+  normalizeStoredVector(vector) {
+    const values = Array.isArray(vector) ? vector : ArrayBuffer.isView(vector) && !(vector instanceof DataView) ? Array.from(vector) : null;
+    if (!values || values.length !== this.embeddingDimension) return null;
+    const normalized = values.map((value) => Number(value));
+    return normalized.every((value) => Number.isFinite(value)) ? normalized : null;
+  }
+};
+
+export {
+  resolveOramaCollectionDbFilePath,
+  OramaBackend
+};
+//# sourceMappingURL=chunk-M46RYSMW.js.map