@absolutejs/rag 0.0.20 → 0.0.22

package/dist/index.js

@@ -35670,6 +35670,178 @@ var createInMemoryRAGStore = (options = {}) => {
     getStatus: () => createInMemoryStatus(dimensions)
   };
 };
+// src/adapters/sync.ts
+import {
+  createSyncEngine,
+  createTextIndex,
+  createVectorIndex,
+  defineSearchCollection
+} from "@absolutejs/sync/engine";
+var createSyncRAGStore = (options = {}) => {
+  const dimensions = options.dimensions ?? RAG_VECTOR_DIMENSIONS_DEFAULT;
+  const engine = options.engine ?? createSyncEngine();
+  const retrievalCollection = options.collection ?? "ragRetrieval";
+  const table = options.table ?? "ragChunks";
+  const records = new Map;
+  const toLight = (record) => ({
+    chunkId: record.chunkId,
+    chunkText: record.text,
+    source: record.source,
+    title: record.title
+  });
+  const textIndex = createTextIndex({
+    fields: ["chunkText"],
+    key: (chunk) => chunk.chunkId
+  });
+  const vectorIndex = createVectorIndex({
+    embedding: (chunk) => records.get(chunk.chunkId)?.vector ?? [],
+    key: (chunk) => chunk.chunkId
+  });
+  engine.registerSearch(defineSearchCollection({
+    index: () => textIndex,
+    key: (chunk) => chunk.chunkId,
+    name: retrievalCollection,
+    source: () => [...records.values()].map(toLight),
+    table
+  }));
+  const embed = async (input) => {
+    if (options.embedding) {
+      return options.embedding(input);
+    }
+    if (options.mockEmbedding) {
+      return options.mockEmbedding(input.text);
+    }
+    return normalizeVector(createRAGVector(input.text, dimensions));
+  };
+  const matchesFilter = (record, filter) => matchesMetadataFilterRecord({
+    chunkId: record.chunkId,
+    metadata: record.metadata,
+    source: record.source,
+    title: record.title,
+    ...record.metadata ?? {}
+  }, filter);
+  const toResult = (record, score) => ({
+    chunkId: record.chunkId,
+    chunkText: record.text,
+    embedding: record.vector,
+    metadata: record.metadata,
+    score,
+    source: record.source,
+    title: record.title
+  });
+  const upsert = async (input) => {
+    for (const chunk of input.chunks) {
+      const vector = chunk.embedding ? normalizeVector(chunk.embedding) : normalizeVector(await embed({ text: chunk.text }));
+      const record = {
+        chunkId: chunk.chunkId,
+        metadata: chunk.metadata,
+        source: chunk.source,
+        text: chunk.text,
+        title: chunk.title,
+        vector
+      };
+      records.set(record.chunkId, record);
+      const light = toLight(record);
+      vectorIndex.add(light);
+      textIndex.add(light);
+      await engine.applyChange(table, { op: "insert", row: light });
+    }
+  };
+  const query = async (input) => {
+    const queryVector = normalizeVector(input.queryVector);
+    const results = [];
+    for (const record of records.values()) {
+      if (!matchesFilter(record, input.filter)) {
+        continue;
+      }
+      const score = querySimilarity(queryVector, normalizeVector(record.vector));
+      if (Number.isFinite(score)) {
+        results.push(toResult(record, score));
+      }
+    }
+    results.sort((first, second) => second.score - first.score);
+    return results.slice(0, input.topK);
+  };
+  const queryLexical = async (input) => {
+    const hits = textIndex.search(input.query, input.topK * 5);
+    const results = [];
+    for (const hit of hits) {
+      const record = records.get(hit.row.chunkId);
+      if (record && matchesFilter(record, input.filter)) {
+        results.push(toResult(record, hit.score));
+      }
+    }
+    return results.slice(0, input.topK);
+  };
+  const hasFilters = (chunkIds, filter) => ({
+    filtered: Boolean(filter && Object.keys(filter).length > 0),
+    ided: chunkIds.size > 0
+  });
+  const count = async (input = {}) => {
+    const chunkIds = new Set(input.chunkIds ?? []);
+    const { filtered, ided } = hasFilters(chunkIds, input.filter);
+    if (!filtered && !ided) {
+      return records.size;
+    }
+    return [...records.values()].filter((record) => ided && chunkIds.has(record.chunkId) || filtered && matchesFilter(record, input.filter)).length;
+  };
+  const remove = async (input = {}) => {
+    const chunkIds = new Set(input.chunkIds ?? []);
+    const { filtered, ided } = hasFilters(chunkIds, input.filter);
+    if (!filtered && !ided) {
+      return 0;
+    }
+    let removed = 0;
+    for (const record of [...records.values()]) {
+      const matches = ided && chunkIds.has(record.chunkId) || filtered && matchesFilter(record, input.filter);
+      if (!matches) {
+        continue;
+      }
+      records.delete(record.chunkId);
+      vectorIndex.remove(record.chunkId);
+      textIndex.remove(record.chunkId);
+      await engine.applyChange(table, {
+        op: "delete",
+        row: toLight(record)
+      });
+      removed += 1;
+    }
+    return removed;
+  };
+  const clear = async () => {
+    for (const record of [...records.values()]) {
+      records.delete(record.chunkId);
+      vectorIndex.remove(record.chunkId);
+      textIndex.remove(record.chunkId);
+      await engine.applyChange(table, { op: "delete", row: toLight(record) });
+    }
+  };
+  const status = {
+    backend: "in_memory",
+    dimensions,
+    vectorMode: "in_memory"
+  };
+  const capabilities = {
+    backend: "in_memory",
+    nativeVectorSearch: false,
+    persistence: "memory_only",
+    serverSideFiltering: false,
+    streamingIngestStatus: false
+  };
+  return {
+    clear,
+    count,
+    delete: remove,
+    embed,
+    engine,
+    getCapabilities: () => capabilities,
+    getStatus: () => status,
+    query,
+    queryLexical,
+    retrievalCollection,
+    upsert
+  };
+};
 export {
   xaiEmbeddings,
   validateRAGEmbeddingDimensions,
@@ -35775,6 +35947,7 @@ export {
   deepseekEmbeddings,
   createVoyageRAGReranker,
   createTextFileExtractor,
+  createSyncRAGStore,
   createRAGVector,
   createRAGUrlSyncSource,
   createRAGSyncScheduler,
@@ -35960,5 +36133,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=28A2EA681F0DA49464756E2164756E21
+//# debugId=7661BF2D41E88F6A64756E2164756E21
 //# sourceMappingURL=index.js.map