@meaningfully/core 0.1.1 → 0.1.2

package/dist/Meaningfully.d.ts

@@ -8,9 +8,10 @@ export declare class MeaningfullyAPI {
     private manager;
     private storagePath;
     private clients;
-    constructor({ storagePath, weaviateClient, metadataManager }: {
+    constructor({ storagePath, weaviateClient, postgresClient, metadataManager }: {
         storagePath: string;
         weaviateClient?: any;
+        postgresClient?: any;
         metadataManager: MetadataManager;
     });
     setClients(clients: Clients): void;
@@ -23,7 +24,7 @@ export declare class MeaningfullyAPI {
     deleteDocumentSet(documentSetId: number): Promise<{
         success: boolean;
     }>;
-    getVectorStoreType(): "simple" | "weaviate";
+    getVectorStoreType(): "simple" | "postgres" | "weaviate";
     generatePreviewData(data: DocumentSetParamsFilePath): Promise<import("./types/index.js").PreviewResult>;
     uploadCsv(data: DocumentSetParamsFilePath): Promise<{
         success: boolean;

package/dist/Meaningfully.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IAS7I,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAGpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IAa7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAuBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IAyBzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;CAY9C"}
+{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IASnL,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAGpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IAgB7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAuBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IA0BzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;CAY9C"}

package/dist/Meaningfully.js

@@ -14,12 +14,12 @@ export class MeaningfullyAPI {
     manager;
     storagePath;
     clients;
-    constructor({ storagePath, weaviateClient, metadataManager }) {
+    constructor({ storagePath, weaviateClient, postgresClient, metadataManager }) {
         this.storagePath = storagePath;
         this.manager = metadataManager;
         this.clients = {
             weaviateClient: weaviateClient,
-            postgresClient: null
+            postgresClient: postgresClient
         };
     }
     setClients(clients) {
@@ -46,8 +46,11 @@ export class MeaningfullyAPI {
         }
         return { success: true };
     }
+    // TODO: this is an awful hack.
+    // I don't really want to have it be an argument to the constructor
+    // maybe I should make clients accept EITHER, not both?
     getVectorStoreType() {
-        return this.clients.weaviateClient ? 'weaviate' : 'simple';
+        return this.clients.postgresClient ? 'postgres' : (this.clients.weaviateClient ? 'weaviate' : 'simple');
     }
     async generatePreviewData(data) {
         const vectorStoreType = this.getVectorStoreType();
@@ -152,6 +155,7 @@ export class MeaningfullyAPI {
     }
     async getDocument(documentSetId, documentNodeId) {
         const documentSet = await this.manager.getDocumentSet(documentSetId);
+        const settings = await this.manager.getSettings();
         if (!documentSet) {
             throw new Error('Document set not found');
         }
@@ -166,7 +170,7 @@ export class MeaningfullyAPI {
             storagePath: this.storagePath,
             chunkSize: 1024, // not actually used, we just re-use a config object that has this option
             chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
-        });
+        }, settings, this.clients);
         const document = await docStore.getNode(documentNodeId);
         if (!document) {
             throw new Error('Document not found');

package/dist/Meaningfully.js.map

@@ -1 +1 @@
-{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAmF;QAC3I,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,IAAI;SACrB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,kDAAkD;YAClD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,EAAE,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACtD,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEtD,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAExE,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;CAEF"}
+{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAyG;QACjL,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,kDAAkD;YAClD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,EAAE,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,mEAAmE;IACnE,uDAAuD;IACvD,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACtD,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEtD,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAExE,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;CAEF"}

package/dist/api/embedding.d.ts

@@ -1,7 +1,7 @@
 import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
 export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
 export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
-export declare function getDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
+export declare function getDocStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").BaseDocumentStore>;
 export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
 export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;
 //# sourceMappingURL=embedding.d.ts.map

package/dist/api/embedding.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAyC1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,mDAExD;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAyC1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,mDAE9F;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}

package/dist/api/embedding.js

@@ -1,4 +1,4 @@
-import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getExistingDocStore } from "../services/embeddings.js";
+import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getStorageContext } from "../services/embeddings.js";
 import { loadDocumentsFromCsv } from "../services/csvLoader.js";
 import { MetadataMode } from "llamaindex";
 import { ProgressManager } from "../services/progressManager.js";
@@ -75,8 +75,8 @@ export async function previewResults(csvPath, textColumnName, config) {
         };
     }
 }
-export async function getDocStore(config) {
-    return await getExistingDocStore(config);
+export async function getDocStore(config, settings, clients) {
+    return (await getStorageContext(config, settings, clients)).docStore;
 }
 export async function getIndex(config, settings, clients) {
     return await getExistingVectorStoreIndex(config, settings, clients);

package/dist/api/embedding.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEvL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;gBAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC;YACF,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB;IACvD,OAAO,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}
+{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAErL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;gBAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC;YACF,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7F,OAAO,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/services/embeddings.d.ts

@@ -12,7 +12,6 @@ export declare function estimateCost(nodes: TextNode[], modelName: string): {
     pricePer1M: number;
 };
 export declare function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<VectorStoreIndex>;
-export declare function getExistingDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
 export declare function transformDocumentsToNodes(documents: Document[], config: EmbeddingConfig): Promise<TextNode<import("llamaindex").Metadata>[]>;
 export declare function getEmbedModel(config: EmbeddingConfig, settings: Settings): OpenAIEmbedding | OllamaEmbedding | MistralAIEmbedding | GeminiEmbedding | MockEmbedding;
 export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;

package/dist/services/embeddings.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAKR,KAAK,cAAc,EAGpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAKpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAqD9G;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,mDAehE;AAID,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAiB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCpN;AAyCD,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAKR,KAAK,cAAc,EAMpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAKpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAK9G;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAoB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCpN;AAyED,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}

package/dist/services/embeddings.js

@@ -1,10 +1,10 @@
 import { VectorStoreIndex, 
 // OpenAIEmbedding,
-IngestionPipeline, ModalityType, storageContextFromDefaults, SimpleVectorStore, Settings as LlamaindexSettings, SimpleDocumentStore } from "llamaindex";
+IngestionPipeline, ModalityType, storageContextFromDefaults, SimpleVectorStore, Settings as LlamaindexSettings, SimpleDocumentStore, SimpleIndexStore } from "llamaindex";
 import { OllamaEmbedding } from '@llamaindex/ollama';
 import { MistralAIEmbedding, MistralAIEmbeddingModelType } from '@llamaindex/mistral';
 import { GeminiEmbedding } from '@llamaindex/google';
-import { PGVectorStore } from '@llamaindex/postgres';
+import { PGVectorStore, PostgresDocumentStore, PostgresIndexStore } from '@llamaindex/postgres';
 import { AzureOpenAIEmbedding } from "@llamaindex/azure";
 import { Sploder } from "./sploder.js";
 import { CustomSentenceSplitter } from "./sentenceSplitter.js";
@@ -68,69 +68,10 @@ export function estimateCost(nodes, modelName) {
     };
 }
 export async function getExistingVectorStoreIndex(config, settings, clients) {
-    const embedModel = getEmbedModel(config, settings);
-    switch (config.vectorStoreType) {
-        case "simple":
-            const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
-            const storageContext = await storageContextFromDefaults({
-                persistDir: persistDir,
-            });
-            let vsi = await VectorStoreIndex.init({
-                storageContext: storageContext,
-            });
-            vsi.embedModel = embedModel;
-            return vsi;
-        case "postgres":
-            if (!clients.postgresClient) {
-                throw new Error("Postgres client required but not provided");
-            }
-            const pgStore = new PGVectorStore({
-                clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
-                tableName: sanitizeProjectName(config.projectName),
-                dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
-                embeddingModel: embedModel
-            });
-            const pgStorageContext = await storageContextFromDefaults({
-                vectorStores: { [ModalityType.TEXT]: pgStore },
-            });
-            return await VectorStoreIndex.init({
-                storageContext: pgStorageContext,
-            });
-        case "weaviate":
-            if (!clients.weaviateClient) {
-                throw new Error("Weaviate client required but not provided");
-            }
-            const weaviateStore = new BatchingWeaviateVectorStore({
-                indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
-                weaviateClient: clients.weaviateClient,
-                embeddingModel: embedModel
-            });
-            // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
-            // (WeaviateVectorStore would get `score` if we were doing hybrid search)
-            // Overwrite the private getNodeSimilarity method to use 'score' from metadata
-            // @ts-ignore
-            weaviateStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
-                return entry.metadata.score;
-            };
-            return await VectorStoreIndex.fromVectorStore(weaviateStore);
-        default:
-            throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
-    }
-}
-export async function getExistingDocStore(config) {
-    // switch (config.vectorStoreType) {
-    //   case "simple":
-    const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
-    const storageContext = await storageContextFromDefaults({
-        persistDir: persistDir,
+    const storageContext = await getStorageContext(config, settings, clients);
+    return await VectorStoreIndex.init({
+        storageContext: storageContext,
     });
-    return storageContext.docStore;
-    //   case "postgres":
-    //     throw new Error(`Not yet implemented vector store type: ${config.vectorStoreType}`);
-    //     // return await createVectorStore(config);
-    //   default:
-    //     throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
-    // }
 }
 export async function transformDocumentsToNodes(documents, config) {
     console.time("transformDocumentsToNodes Run Time");
@@ -204,12 +145,15 @@ export function getEmbedModel(config, settings) {
 }
 export async function getStorageContext(config, settings, clients) {
     const vectorStore = await createVectorStore(config, settings, clients);
+    const docStore = await createDocumentStore(config, settings, clients); // new SimpleDocumentStore()
+    const indexStore = await createIndexStore(config, settings, clients);
     fs.mkdirSync(config.storagePath, { recursive: true });
     const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
     return await storageContextFromDefaults({
         persistDir: persistDir,
         vectorStores: { [ModalityType.TEXT]: vectorStore },
-        docStore: new SimpleDocumentStore()
+        docStore: docStore,
+        indexStore: indexStore
         /*
           if docStore is created with a persist path (as it is by default in storageContextFromDefaults)
           then it will write to disk after every put(), which happens 2+ times per document.
@@ -226,8 +170,14 @@ export async function persistDocuments(documents, config, settings, clients) {
     await storageContext.docStore.addDocuments(documents, true);
     // see comments in getStorageContext
     const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
-    // @ts-ignore
-    await storageContext.docStore.kvStore.persist(join(persistDir, "doc_store.json"));
+    if (storageContext.docStore instanceof SimpleDocumentStore) {
+        // @ts-ignore
+        await storageContext.docStore.kvStore.persist(join(persistDir, "doc_store.json"));
+    }
+    else if (storageContext.docStore instanceof PostgresDocumentStore) {
+        // PostgresDocumentStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
+        console.log("Pretending to persist Postgres document store, but it actually persists automatically.");
+    }
     console.timeEnd("persistDocuments Run Time");
 }
 export async function persistNodes(nodes, config, settings, clients, progressCallback) {
@@ -252,12 +202,12 @@ export async function persistNodes(nodes, config, settings, clients, progressCal
     // all the if statements are just type-checking boilerplate.
     // N.B. WeaviateVectorStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
     if (vectorStore) {
-        if (vectorStore instanceof PGVectorStore || vectorStore instanceof SimpleVectorStore) {
+        if (vectorStore instanceof SimpleVectorStore) {
             await vectorStore.persist(join(config.storagePath, sanitizeProjectName(config.projectName), "vector_store.json"));
         }
-        else if (vectorStore instanceof BatchingWeaviateVectorStore) {
+        else if (vectorStore instanceof PGVectorStore || vectorStore instanceof BatchingWeaviateVectorStore) {
             // WeaviateVectorStore does not have a persist method, it persists automatically
-            console.log("Pretending to persist Weaviate vector store, but it actually persists automatically.");
+            console.log("Pretending to persist Weaviate or Postgres vector store, but it actually persists automatically.");
         }
         else {
             throw new Error("Vector store does not support persist method");
@@ -276,8 +226,8 @@ async function createVectorStore(config, settings, clients) {
         // otherwise it defaults to Ada.
         case "postgres":
             return new PGVectorStore({
-                clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
-                tableName: sanitizeProjectName(config.projectName),
+                client: clients.postgresClient,
+                tableName: "vecs_" + sanitizeProjectName(config.projectName),
                 dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
                 embeddingModel: embeddingModel
             });
@@ -302,6 +252,35 @@ async function createVectorStore(config, settings, clients) {
             throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
     }
 }
+async function createDocumentStore(config, settings, clients) {
+    // we create the doc store without a persist path, so it doesn't write to disk after every put()
+    switch (config.documentStoreType || config.vectorStoreType) {
+        case "postgres":
+            return new PostgresDocumentStore({
+                client: clients.postgresClient,
+                tableName: "docs_" + sanitizeProjectName(config.projectName),
+            });
+        case "simple":
+        case "weaviate":
+            return new SimpleDocumentStore();
+        default:
+            throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+    }
+}
+async function createIndexStore(config, settings, clients) {
+    switch (config.documentStoreType || config.vectorStoreType) {
+        case "postgres":
+            return new PostgresIndexStore({
+                client: clients.postgresClient,
+                tableName: "idx_" + sanitizeProjectName(config.projectName),
+            });
+        case "simple":
+        case "weaviate":
+            return new SimpleIndexStore();
+        default:
+            throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+    }
+}
 export async function searchDocuments(index, query, numResults = 10, filters) {
     // const metadataFilters: MetadataFilters | undefined = filters ? {filters: filters} : undefined;
     const metadataFilters = {

package/dist/services/embeddings.js.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACnD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/B,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC;gBACtD,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;YACH,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACpC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YACH,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;YAC5B,OAAO,GAAG,CAAC;QAEb,KAAK,UAAU;YACb,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;gBAChC,YAAY,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE;gBAC1E,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAClD,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,0BAA0B,CAAC;gBACxD,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE;aAC/C,CAAC,CAAC;YACH,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACjC,cAAc,EAAE,gBAAgB;aACjC,CAAC,CAAC;QACL,KAAK,UAAU;YACb,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,2BAA2B,CAAC;gBACpD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,aAAa,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBACpE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,MAAM,gBAAgB,CAAC,eAAe,CAAC,aAAa,CAAC,CAAA;QAE9D;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB;IAC/D,oCAAoC;IACpC,mBAAmB;IACf,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC;QACtD,UAAU,EAAE,UAAU;KACvB,CAAC,CAAC;IACH,OAAO,cAAc,CAAC,QAAQ,CAAC;IAEnC,qBAAqB;IACrB,2FAA2F;IAC3F,iDAAiD;IACjD,aAAa;IACb,mFAAmF;IACnF,IAAI;AACN,CAAC;AAID,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,IAAI,mBAAmB,EAAE;QACjC;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,aAAa;IACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAE3G,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK;QACL,cAAc;QACd,WAAW,EAAE,IAAI;QACjB,gBAAgB;KACjB,CAAC,CAAC;IAEH,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YACrF,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YAC9D,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,sFAAsF,CAAC,CAAC;QACtG,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC5F,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,YAAY,EAAE,EAAC,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAC;gBACxE,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAClD,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EAGnB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC;QACjC,cAAc,EAAE,cAAc;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,4BAA4B;IACnG,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,UAAU;QACpB;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,IAAI,cAAc,CAAC,QAAQ,YAAY,mBAAmB,EAAE,CAAC;QAC3D,aAAa;QACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC7G,CAAC;SAAK,IAAI,cAAc,CAAC,QAAQ,YAAY,qBAAqB,EAAE,CAAC;QACnE,sHAAsH;QACtH,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK;QACL,cAAc;QACd,WAAW,EAAE,IAAI;QACjB,gBAAgB;KACjB,CAAC,CAAC;IAEH,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YAC7C,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YACtG,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,kGAAkG,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC5F,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC5D,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC9F,gGAAgG;IAChG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC7D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,mBAAmB,EAAE,CAAC;QACnC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC3F,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,kBAAkB,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AAEH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/types/index.d.ts

@@ -48,6 +48,8 @@ export interface EmbeddingConfig {
     modelName: string;
     modelProvider: string;
     vectorStoreType: "simple" | "postgres" | "weaviate";
+    documentStoreType?: "simple" | "postgres";
+    indexStoreType?: "simple" | "postgres";
     projectName: string;
     storagePath: string;
     splitIntoSentences: boolean;

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC/B,CAAC,CAAC;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAGD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAGD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAGD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;IACpD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAGD,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,GAAG,UAAU,GAAG,UAAU,CAAC;IACxH,KAAK,EAAE,GAAG,CAAA;CACX;AAED,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,GAAG,CAAC;IACpB,cAAc,EAAE,GAAG,CAAC;CACrB"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC/B,CAAC,CAAC;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAGD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAGD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAGD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;IACpD,iBAAiB,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IAC1C,cAAc,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAGD,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,GAAG,UAAU,GAAG,UAAU,CAAC;IACxH,KAAK,EAAE,GAAG,CAAA;CACX;AAED,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,GAAG,CAAC;IACpB,cAAc,EAAE,GAAG,CAAC;CACrB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@meaningfully/core",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Core functionality for meaningfully semantic search",
   "repo": "https://github.com/jeremybmerrill/meaningfully-core",
   "license": "MIT",
@@ -33,7 +33,8 @@
     "lodash": "^4.17.21",
     "natural": "^8.1.0",
     "openai": "^5.9.0",
-    "papaparse": "^5.5.3"
+    "papaparse": "^5.5.3",
+    "pgvector": "^0.2.0"
   },
   "devDependencies": {
     "@types/node": "^20.5.6",

package/src/Meaningfully.ts

@@ -22,12 +22,12 @@ export class MeaningfullyAPI {
   private storagePath: string;
   private clients: Clients;
 
-  constructor({ storagePath, weaviateClient, metadataManager }: { storagePath: string, weaviateClient?: any, metadataManager: MetadataManager }) {
+  constructor({ storagePath, weaviateClient, postgresClient, metadataManager }: { storagePath: string, weaviateClient?: any, postgresClient?: any, metadataManager: MetadataManager }) {
     this.storagePath = storagePath;
     this.manager = metadataManager;
     this.clients = {
       weaviateClient: weaviateClient,
-      postgresClient: null
+      postgresClient: postgresClient
     };
   }
 
@@ -58,8 +58,11 @@ export class MeaningfullyAPI {
     return { success: true };
   }
 
+  // TODO: this is an awful hack.
+  // I don't really want to have it be an argument to the constructor
+  // maybe I should make clients accept EITHER, not both?
   getVectorStoreType() {
-    return this.clients.weaviateClient ? 'weaviate' : 'simple';
+    return this.clients.postgresClient ? 'postgres' : (this.clients.weaviateClient ? 'weaviate' : 'simple');
   }
 
   async generatePreviewData(data: DocumentSetParamsFilePath) {
@@ -79,7 +82,7 @@ export class MeaningfullyAPI {
         storagePath: this.storagePath,
         chunkSize: data.chunkSize,
         chunkOverlap: data.chunkOverlap
-    });
+      });
   } catch (error) {
     throw error;
   }
@@ -171,6 +174,7 @@ export class MeaningfullyAPI {
 
   async getDocument(documentSetId: number, documentNodeId: string){
     const documentSet = await this.manager.getDocumentSet(documentSetId);
+    const settings = await this.manager.getSettings();
     if (!documentSet) {
       throw new Error('Document set not found');
     } 
@@ -185,7 +189,7 @@ export class MeaningfullyAPI {
       storagePath: this.storagePath,
       chunkSize: 1024, // not actually used, we just re-use a config object that has this option
       chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
-    });
+    }, settings, this.clients);
     const document = await docStore.getNode(documentNodeId);
     if (!document) {
       throw new Error('Document not found');

package/src/api/__tests__/embedding.test.ts

@@ -2,7 +2,7 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { createEmbeddings, previewResults, getDocStore, getIndex, search } from '../embedding.js';
 import { loadDocumentsFromCsv } from '../../services/csvLoader.js';
-import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '../../services/embeddings.js';
+import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getStorageContext } from '../../services/embeddings.js';
 import { MetadataMode } from 'llamaindex';
 
 // filepath: /Users/jeremybmerrill/code/meaningfully/src/main/api/embedding.test.ts
@@ -82,9 +82,9 @@ describe('embedding.ts', () => {
     describe('getDocStore', () => {
         it('should return existing doc store', async () => {
             const mockDocStore = 'docStore';
-            getExistingDocStore.mockResolvedValue(mockDocStore);
+            getStorageContext.mockResolvedValue({ docStore: mockDocStore });
 
-            const result = await getDocStore({});
+            const result = await getDocStore({}, {}, {});
 
             expect(result).toBe(mockDocStore);
         });
@@ -173,7 +173,7 @@ describe('embedding.ts', () => {
   describe('getDocStore', () => {
     it('should return existing doc store', async () => {
       const mockDocStore = 'docStore';
-      getExistingDocStore.mockResolvedValue(mockDocStore);
+      getStorageContext.mockResolvedValue({ docStore: mockDocStore });
 
       const result = await getDocStore({});
 

package/src/api/embedding.ts

@@ -1,4 +1,4 @@
-import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getExistingDocStore } from "../services/embeddings.js";
+import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getStorageContext } from "../services/embeddings.js";
 import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
 import { loadDocumentsFromCsv } from "../services/csvLoader.js";
 import { MetadataMode } from "llamaindex";
@@ -97,8 +97,8 @@ export async function previewResults(
   }
 } 
 
-export async function getDocStore(config: EmbeddingConfig) {
-  return await getExistingDocStore(config);
+export async function getDocStore(config: EmbeddingConfig, settings: Settings, clients: Clients) {
+  return (await getStorageContext(config, settings, clients)).docStore;
 }
 
 export async function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {

package/src/services/embeddings.d.ts

@@ -12,7 +12,6 @@ export declare function estimateCost(nodes: TextNode[], modelName: string): {
     pricePer1M: number;
 };
 export declare function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<VectorStoreIndex>;
-export declare function getExistingDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
 export declare function transformDocumentsToNodes(documents: Document[], config: EmbeddingConfig): Promise<TextNode<import("llamaindex").Metadata>[]>;
 export declare function getEmbedModel(config: EmbeddingConfig, settings: Settings): MockEmbedding | OpenAIEmbedding | OllamaEmbedding | MistralAIEmbedding | GeminiEmbedding;
 export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;

package/src/services/embeddings.ts

@@ -11,12 +11,15 @@ import {
   SimpleVectorStore,
   type StorageContext,
   Settings as LlamaindexSettings,
-  SimpleDocumentStore
+  SimpleDocumentStore,
+  BaseDocumentStore,
+  BaseIndexStore,
+  SimpleIndexStore
 } from "llamaindex";
 import { OllamaEmbedding} from '@llamaindex/ollama'
 import { MistralAIEmbedding, MistralAIEmbeddingModelType } from '@llamaindex/mistral'
 import { GeminiEmbedding } from '@llamaindex/google'
-import { PGVectorStore } from '@llamaindex/postgres';
+import { PGVectorStore, PostgresDocumentStore, PostgresIndexStore } from '@llamaindex/postgres';
 import { AzureOpenAIEmbedding } from "@llamaindex/azure";
 import { Sploder } from "./sploder.js";
 import { CustomSentenceSplitter } from "./sentenceSplitter.js";
@@ -97,79 +100,12 @@ export function estimateCost(nodes: TextNode[], modelName: string): {
 }
 
 export async function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
-  const embedModel = getEmbedModel(config, settings);
-  switch (config.vectorStoreType) {
-    case "simple":
-      const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
-      const storageContext = await storageContextFromDefaults({
-        persistDir: persistDir,
-      });
-      let vsi = await VectorStoreIndex.init({
-        storageContext: storageContext,
-      });
-      vsi.embedModel = embedModel;
-      return vsi;
-
-    case "postgres":
-      if (!clients.postgresClient) {
-        throw new Error("Postgres client required but not provided");
-      }
-      const pgStore = new PGVectorStore({
-        clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING }, 
-        tableName: sanitizeProjectName(config.projectName),
-        dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
-        embeddingModel: embedModel
-      });
-      const pgStorageContext = await storageContextFromDefaults({
-        vectorStores: { [ModalityType.TEXT]: pgStore },
-      });
-      return await VectorStoreIndex.init({
-        storageContext: pgStorageContext,
-      });
-    case "weaviate":
-      if (!clients.weaviateClient) {
-        throw new Error("Weaviate client required but not provided");
-      }
-      const weaviateStore = new BatchingWeaviateVectorStore({
-        indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
-        weaviateClient: clients.weaviateClient,
-        embeddingModel: embedModel
-      });
-
-      // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
-      // (WeaviateVectorStore would get `score` if we were doing hybrid search)
-      // Overwrite the private getNodeSimilarity method to use 'score' from metadata
-      // @ts-ignore
-      weaviateStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
-        return  entry.metadata.score;
-      }
-
-      return await VectorStoreIndex.fromVectorStore(weaviateStore)
-
-    default:
-      throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
-  }
-}
-
-export async function getExistingDocStore(config: EmbeddingConfig) {
-  // switch (config.vectorStoreType) {
-  //   case "simple":
-      const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
-      const storageContext = await storageContextFromDefaults({
-        persistDir: persistDir,
-      });
-      return storageContext.docStore;
-
-  //   case "postgres":
-  //     throw new Error(`Not yet implemented vector store type: ${config.vectorStoreType}`);
-  //     // return await createVectorStore(config);
-  //   default:
-  //     throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
-  // }
+  const storageContext = await getStorageContext(config, settings, clients);
+  return await VectorStoreIndex.init({
+    storageContext: storageContext,
+  });
 }
 
-
-
 export async function transformDocumentsToNodes(
   documents: Document[],
   config: EmbeddingConfig,
@@ -249,12 +185,15 @@ export function getEmbedModel(
 
 export async function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext> {
   const vectorStore = await createVectorStore(config, settings, clients);
+  const docStore = await createDocumentStore(config, settings, clients); // new SimpleDocumentStore()
+  const indexStore = await createIndexStore(config, settings, clients);
   fs.mkdirSync(config.storagePath, { recursive: true }); 
   const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
   return await storageContextFromDefaults({
     persistDir: persistDir,
     vectorStores: {[ModalityType.TEXT]: vectorStore},
-    docStore: new SimpleDocumentStore()
+    docStore: docStore,
+    indexStore: indexStore
       /*
         if docStore is created with a persist path (as it is by default in storageContextFromDefaults)
         then it will write to disk after every put(), which happens 2+ times per document.
@@ -273,8 +212,13 @@ export async function persistDocuments(documents: Document[], config: EmbeddingC
 
   // see comments in getStorageContext
   const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
-  // @ts-ignore
-  await (storageContext.docStore as SimpleDocumentStore).kvStore.persist(join(persistDir, "doc_store.json"));
+  if (storageContext.docStore instanceof SimpleDocumentStore) {
+    // @ts-ignore
+    await (storageContext.docStore as SimpleDocumentStore).kvStore.persist(join(persistDir, "doc_store.json"));
+  }else if (storageContext.docStore instanceof PostgresDocumentStore) {
+    // PostgresDocumentStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
+    console.log("Pretending to persist Postgres document store, but it actually persists automatically.");
+  }
 
   console.timeEnd("persistDocuments Run Time");
 }
@@ -303,11 +247,11 @@ export async function persistNodes(nodes: TextNode[], config: EmbeddingConfig, s
   // all the if statements are just type-checking boilerplate.
   // N.B. WeaviateVectorStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
   if (vectorStore) {
-    if (vectorStore instanceof PGVectorStore || vectorStore instanceof SimpleVectorStore) {
+    if (vectorStore instanceof SimpleVectorStore) {
       await vectorStore.persist(join(config.storagePath, sanitizeProjectName(config.projectName), "vector_store.json"));
-    } else if (vectorStore instanceof BatchingWeaviateVectorStore) {
+    } else if (vectorStore instanceof PGVectorStore || vectorStore instanceof BatchingWeaviateVectorStore) {
       // WeaviateVectorStore does not have a persist method, it persists automatically
-      console.log("Pretending to persist Weaviate vector store, but it actually persists automatically.");
+      console.log("Pretending to persist Weaviate or Postgres vector store, but it actually persists automatically.");
     } else {
       throw new Error("Vector store does not support persist method");
     }
@@ -326,8 +270,8 @@ async function createVectorStore(config: EmbeddingConfig, settings: Settings, cl
     // otherwise it defaults to Ada.
     case "postgres":
       return new PGVectorStore({
-        clientConfig: {connectionString: process.env.POSTGRES_CONNECTION_STRING},
-        tableName: sanitizeProjectName(config.projectName),
+        client: clients.postgresClient,
+        tableName: "vecs_" + sanitizeProjectName(config.projectName),
         dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
         embeddingModel: embeddingModel
       });
@@ -357,6 +301,38 @@ async function createVectorStore(config: EmbeddingConfig, settings: Settings, cl
   }
 }
 
+async function createDocumentStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseDocumentStore> {
+  // we create the doc store without a persist path, so it doesn't write to disk after every put()
+  switch (config.documentStoreType || config.vectorStoreType) {
+    case "postgres":
+      return new PostgresDocumentStore({
+        client: clients.postgresClient,
+        tableName: "docs_" + sanitizeProjectName(config.projectName),
+      });
+    case "simple":
+    case "weaviate": 
+      return new SimpleDocumentStore();
+    default:
+      throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+  }
+}
+
+async function createIndexStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseIndexStore> {
+  switch (config.documentStoreType || config.vectorStoreType) {
+    case "postgres":
+      return new PostgresIndexStore({
+        client: clients.postgresClient,
+        tableName: "idx_" + sanitizeProjectName(config.projectName),
+      });
+    case "simple":
+    case "weaviate": 
+      return new SimpleIndexStore();
+    default:
+      throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
+  }
+
+}
+
 export async function searchDocuments(
   index: VectorStoreIndex,
   query: string,

package/src/types/index.ts

@@ -57,6 +57,8 @@ export interface EmbeddingConfig {
   modelName: string;
   modelProvider: string
   vectorStoreType: "simple" | "postgres" | "weaviate";
+  documentStoreType?: "simple" | "postgres";
+  indexStoreType?: "simple" | "postgres";
   projectName: string;
   storagePath: string;
   splitIntoSentences: boolean;