endee-llamaindex 1.0.0 → 1.0.1

package/README.md

@@ -112,6 +112,37 @@ const queryEngine = index.asQueryEngine();
 const response = await queryEngine.query({
   query: "Summarize the key points",
 });
+
+console.log(response.toString());
+```
+
+### Filtering in Queries
+
+`EndeeVectorStore` supports **metadata filtering** during queries via LlamaIndex's `MetadataFilters`.  
+Currently, **only the `==` (equals) and `in` operators** are supported.
+
+```typescript
+import { MetadataFilters, MetadataFilter } from "llamaindex";
+
+const vectorStore = new EndeeVectorStore({
+  indexName: "knowledge-base",
+  apiToken: process.env.ENDEE_API_TOKEN,
+});
+
+const index = await VectorStoreIndex.fromVectorStore(vectorStore);
+
+const filter : MetadataFilter[] = [{key: "type", value: "sports", operator: "=="}];
+
+const queryEngine = index.asQueryEngine({
+  preFilters: {filters: filter}
+});
+
+const response = await queryEngine.query({
+  query: "Explain the main concepts",
+  filters,
+});
+
+console.log(response.toString());
 ```
 
 ### Deleting Documents

package/dist/endeeClient.d.ts

@@ -0,0 +1,27 @@
+import { BaseNode, BaseVectorStore, Metadata, VectorStoreBaseParams, VectorStoreQuery, VectorStoreQueryResult } from "llamaindex";
+type EndeeParams = {
+    indexName: string;
+    authToken: string;
+    chunkSize?: number;
+} & VectorStoreBaseParams;
+export declare class EndeeVectorStore extends BaseVectorStore {
+    storesText: boolean;
+    private authToken;
+    private indexName;
+    private chunkSize;
+    private db;
+    constructor(params: EndeeParams);
+    private getDB;
+    client(): any;
+    index(): Promise<any>;
+    add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]>;
+    protected saveChunk(idx: any, chunk: any): Promise<boolean>;
+    nodeToRecord(node: BaseNode<Metadata>): {
+        id: string;
+        vector: number[];
+        meta: Metadata;
+    };
+    query(query: VectorStoreQuery, _options?: object): Promise<VectorStoreQueryResult>;
+    delete(refDocId: string, deleteKwargs?: object): Promise<void>;
+}
+export {};

package/dist/endeeClient.js

@@ -0,0 +1,130 @@
+import { Endee } from "endee";
+import { BaseVectorStore, metadataDictToNode, nodeToMetadata, } from "llamaindex";
+export class EndeeVectorStore extends BaseVectorStore {
+    constructor(params) {
+        super(params);
+        this.storesText = true;
+        this.authToken = params.authToken;
+        this.indexName = params.indexName;
+        this.chunkSize = params.chunkSize ?? 100;
+        this.db = new Endee(params.authToken);
+    }
+    getDB() {
+        if (!this.db) {
+            this.db = new Endee(this.authToken);
+        }
+        return this.db;
+    }
+    client() {
+        return this.getDB();
+    }
+    async index() {
+        const db = this.getDB();
+        const index = await db.getIndex(this.indexName);
+        return index;
+    }
+    async add(embeddingResults) {
+        if (embeddingResults.length == 0) {
+            return [];
+        }
+        const idx = await this.index();
+        const nodes = embeddingResults.map((node) => {
+            const nodeRecord = this.nodeToRecord(node);
+            if (nodeRecord.meta.ref_doc_id) {
+                // adding refDoc id as prefix to the chunk to find them using refDoc id
+                nodeRecord.id = `${nodeRecord.meta.ref_doc_id}_chunk_${nodeRecord.id}`;
+            }
+            return nodeRecord;
+        });
+        for (let i = 0; i < nodes.length; i += this.chunkSize) {
+            const chunk = nodes.slice(i, i + this.chunkSize);
+            const chunkData = chunk.map((node) => {
+                const nodeContent = JSON.parse(node.meta._node_content);
+                return {
+                    id: node.id,
+                    vector: node.vector,
+                    meta: { text: nodeContent.text },
+                    filter: nodeContent.relationships.SOURCE.metadata,
+                };
+            });
+            const result = await this.saveChunk(idx, chunkData);
+            if (!result) {
+                throw new Error("Failed to save chunk");
+            }
+        }
+        return [];
+    }
+    async saveChunk(idx, chunk) {
+        try {
+            await idx.upsert(chunk);
+            return true;
+        }
+        catch (err) {
+            const msg = `${err}`;
+            return false;
+        }
+    }
+    nodeToRecord(node) {
+        const id = node.id_.length ? node.id_ : null;
+        return {
+            id: id,
+            vector: node.getEmbedding(),
+            meta: nodeToMetadata(node),
+        };
+    }
+    async query(query, _options) {
+        const filter = query.filters;
+        const defaultOptions = {
+            vector: query.queryEmbedding,
+            topK: query.similarityTopK,
+            includeVectors: true,
+        };
+        if (filter) {
+            const filterArray = filter.filters.map((f) => {
+                if (f.operator == "==") {
+                    const obj = {
+                        [f.key]: { $eq: f.value },
+                    };
+                    return obj;
+                }
+                else if (f.operator == "in") {
+                    const obj = {
+                        [f.key]: { $in: f.value },
+                    };
+                    return obj;
+                }
+            });
+            defaultOptions.filter = filterArray;
+        }
+        const idx = await this.index();
+        const results = await idx.query(defaultOptions);
+        const idList = results.map((row) => row.id);
+        if (idList.length == 0) {
+            return { nodes: [], similarities: [], ids: [] };
+        }
+        const nodes = results.map((row) => {
+            const node = metadataDictToNode(row.meta ?? {}, {
+                fallback: {
+                    id: row.id,
+                    text: row._node_content,
+                    metadata: row.meta,
+                    embedding: row.vector,
+                },
+            });
+            return node;
+        });
+        return {
+            nodes: nodes,
+            similarities: results.map((row) => row.similarity || 999),
+            ids: results.map((row) => row.id),
+        };
+    }
+    async delete(refDocId, deleteKwargs) {
+        const [idx, index] = await Promise.all([
+            this.index(),
+            //to get the information about the index
+            this.db?.describeIndex(this.indexName),
+        ]);
+        await idx.deleteVector(refDocId);
+    }
+}

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "endee-llamaindex",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "LlamaIndex integration with endee vector database",
-  "main": "index.js",
+  "type": "module",
+  "main": "dist/endeeClient.js",
+  "types": "dist/endeeClient.d.ts",
   "scripts": {
+    "build": "tsc",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "keywords": [],

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "Node16",
+    "moduleResolution": "node16",
+    "declaration": true,
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*"]
+}
+
+

package/src/endeeClient.ts

@@ -1,148 +0,0 @@
-import { Endee } from "endee";
-import {
-  BaseNode,
-  BaseVectorStore,
-  Metadata,
-  metadataDictToNode,
-  nodeToMetadata,
-  VectorStoreBaseParams,
-  VectorStoreQuery,
-  VectorStoreQueryResult,
-} from "llamaindex";
-
-type EndeeParams = {
-  indexName: string;
-  apiToken: string;
-  chunkSize: number;
-} & VectorStoreBaseParams;
-
-export class EndeeVectorStore extends BaseVectorStore {
-  storesText: boolean = true;
-  private apiToken: string;
-  private indexName: string;
-  private chunkSize: number;
-  private db: any;
-
-  constructor(params: EndeeParams) {
-    super(params);
-    this.apiToken = params.apiToken;
-    this.indexName = params.indexName;
-    this.chunkSize = params.chunkSize;
-    this.db = new Endee(params.apiToken);
-  }
-
-  private getDB() {
-    if (!this.db) {
-      this.db = new Endee(this.apiToken);
-    }
-    return this.db;
-  }
-
-  client() {
-    return this.getDB();
-  }
-
-  async index() {
-    const db = this.getDB();
-    const index = await db.getIndex(this.indexName);
-    return index;
-  }
-
-  async add(embeddingResults: BaseNode<Metadata>[]): Promise<string[]> {
-    if (embeddingResults.length == 0) {
-      return [];
-    }
-
-    const idx: any = await this.index();
-    const nodes = embeddingResults.map((node) => {
-      const nodeRecord = this.nodeToRecord(node);
-
-      if (nodeRecord.meta.ref_doc_id) {
-        // adding refDoc id as prefix to the chunk to find them using refDoc id
-        nodeRecord.id = `${nodeRecord.meta.ref_doc_id}_chunk_${nodeRecord.id}`;
-      }
-      return nodeRecord;
-    });
-
-    for (let i = 0; i < nodes.length; i += this.chunkSize) {
-      const chunk = nodes.slice(i, i + this.chunkSize);
-      const result = await this.saveChunk(idx, chunk);
-      if (!result) {
-        throw new Error("Failed to save chunk");
-      }
-    }
-    return [];
-  }
-
-  protected async saveChunk(idx: any, chunk: any) {
-    try {
-      await idx.upsert(chunk);
-      return true;
-    } catch (err) {
-      const msg = `${err}`;
-      console.log(msg, err);
-      return false;
-    }
-  }
-
-  nodeToRecord(node: BaseNode<Metadata>) {
-    const id = node.id_.length ? node.id_ : null;
-    return {
-      id: id!,
-      vector: node.getEmbedding(),
-      meta: nodeToMetadata(node),
-    };
-  }
-
-  async query(
-    query: VectorStoreQuery,
-    _options?: object
-  ): Promise<VectorStoreQueryResult> {
-    const filter = "";
-    const defaultOptions: any = {
-      vector: query.queryEmbedding!,
-      topK: query.similarityTopK,
-      includeVectors: true,
-    };
-
-    if (filter) {
-      defaultOptions.filter = filter;
-    }
-
-    const idx = await this.index();
-    const results = await idx.query(defaultOptions);
-
-    const idList = results.map((row: any) => row.id);
-    if (idList.length == 0) {
-      return { nodes: [], similarities: [], ids: [] };
-    }
-
-    const nodes = results.map((row: any) => {
-      const node = metadataDictToNode(row.meta ?? {}, {
-        fallback: {
-          id: row.id,
-          text: row._node_content,
-          metadata: row.meta,
-          embedding: row.vector,
-        },
-      });
-      return node;
-    });
-
-    return {
-      nodes: nodes,
-      similarities: results.map((row: any) => row.similarity || 999),
-      ids: results.map((row: any) => row.id),
-    };
-  }
-
-  async delete(refDocId: string, deleteKwargs?: object): Promise<void> {
-    const [idx, index] = await Promise.all([
-      this.index(),
-      //to get the information about the index
-      this.db?.describeIndex(this.indexName),
-    ]);
-
-    await idx.deleteVector(refDocId);
-  }
-}