@meaningfully/core 0.1.5 → 0.1.7

package/dist/Meaningfully.d.ts

@@ -27,8 +27,13 @@ export declare class MeaningfullyAPI {
     getVectorStoreType(): "simple" | "postgres" | "weaviate";
     generatePreviewData(data: DocumentSetParamsFilePath): Promise<import("./types/index.js").PreviewResult>;
     uploadCsv(data: DocumentSetParamsFilePath): Promise<{
+        success: boolean;
+        error: string;
+        documentSetId?: undefined;
+    } | {
         success: boolean;
         documentSetId: number;
+        error?: undefined;
     }>;
     searchDocumentSet(documentSetId: number, query: string, n_results?: number, filters?: MetadataFilter[]): Promise<import("./types/index.js").SearchResult[]>;
     getDocument(documentSetId: number, documentNodeId: string): Promise<import("llamaindex").BaseNode<import("llamaindex").Metadata>>;

package/dist/Meaningfully.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IASnL,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAKpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IA2C7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAwBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IA0BzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;IAcvC,yBAAyB,CAAC,WAAW,EAAE,MAAM;IAS7C,wBAAwB,CAAC,WAAW,EAAE,MAAM;IAS5C,sBAAsB,CAAC,WAAW,EAAE,MAAM;IAU1C,yBAAyB,CAAC,WAAW,EAAE,MAAM;IAM7C,uBAAuB,CAAC,WAAW,EAAE,MAAM;IAI3C,oBAAoB,CAAC,WAAW,EAAE,MAAM;IAIxC,sBAAsB,CAAC,WAAW,EAAE,MAAM;CAGjD"}
+{"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IASnL,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAKpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IA2C7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAgCnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;;;;;;IAsEzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IA0BzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;IAcvC,yBAAyB,CAAC,WAAW,EAAE,MAAM;IAS7C,wBAAwB,CAAC,WAAW,EAAE,MAAM;IAS5C,sBAAsB,CAAC,WAAW,EAAE,MAAM;IAU1C,yBAAyB,CAAC,WAAW,EAAE,MAAM;IAM7C,uBAAuB,CAAC,WAAW,EAAE,MAAM;IAI3C,oBAAoB,CAAC,WAAW,EAAE,MAAM;IAIxC,sBAAsB,CAAC,WAAW,EAAE,MAAM;CAGjD"}

package/dist/Meaningfully.js

@@ -89,8 +89,14 @@ export class MeaningfullyAPI {
             if (!data.textColumns[0]) {
                 throw new Error("No text column specified for preview.");
             }
-            console.log(data);
-            return await previewResults(data.filePath, data.textColumns[0], {
+            const documents = await loadDocumentsFromCsv(data.filePath, data.textColumns[0]);
+            if (documents.length === 0) {
+                return {
+                    success: false,
+                    error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+                };
+            }
+            return await previewResults(documents, {
                 modelName: data.modelName, // needed to tokenize, estimate costs
                 modelProvider: data.modelProvider,
                 splitIntoSentences: data.splitIntoSentences,
@@ -135,10 +141,17 @@ export class MeaningfullyAPI {
             // Process each text column
             for (const textColumn of data.textColumns) {
                 const documents = await loadDocumentsFromCsv(data.filePath, textColumn);
+                if (documents.length === 0) {
+                    console.timeEnd("createEmbeddings Run Time");
+                    return {
+                        success: false,
+                        error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+                    };
+                }
                 // Update total documents count
                 await this.metadataManager.updateDocumentCount(documentSetId, documents.length);
                 // Create embeddings for this column
-                let ret = await createEmbeddings(data.filePath, textColumn, {
+                let ret = await createEmbeddings(documents, {
                     modelName: data.modelName,
                     modelProvider: data.modelProvider,
                     splitIntoSentences: data.splitIntoSentences,

package/dist/Meaningfully.js.map

@@ -1 +1 @@
-{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,eAAe,CAAkB;IACjC,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAyG;QACjL,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAClE,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAExE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC5D,kDAAkD;YAClD,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,UAAU,EAAC,CAAC;gBACpD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAChC,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;wBACjD,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;wBAChD,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;oBAChD,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;oBACjG,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,UAAU,EAAC,CAAC;gBAC3D,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAChC,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACpD,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5C,CAAC;gBACD,wCAAwC;gBACxC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,mEAAmE;IACnE,uDAAuD;IACvD,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC;YAC9D,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAA;QAE9D,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAEhF,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAGD,0FAA0F;IAC1F,KAAK,CAAC,yBAAyB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,4BAA4B,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,wBAAwB,CAAC,WAAmB;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,2BAA2B,GAAG,mBAAmB,CAAC,WAAW,CAAE,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,sBAAsB,CAAC,WAAmB;QAC9C,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,4BAA4B,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IAEH,CAAC;IACD,KAAK,CAAC,yBAAyB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3F,CAAC;QACG,4HAA4H;IAClI,CAAC;IACD,KAAK,CAAC,uBAAuB,CAAC,WAAmB;QAC/C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnF,CAAC;IACD,KAAK,CAAC,oBAAoB,CAAC,WAAmB;QAC5C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnF,CAAC;IACD,KAAK,CAAC,sBAAsB,CAAC,WAAmB;QAC9C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;CACF"}
+{"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,eAAe,CAAkB;IACjC,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAyG;QACjL,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAClE,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAExE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC5D,kDAAkD;YAClD,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,UAAU,EAAC,CAAC;gBACpD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAChC,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;wBACjD,MAAM,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;wBAChD,MAAM,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;oBAChD,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;oBACjG,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,UAAU,EAAC,CAAC;gBAC3D,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;oBAChC,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACpD,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;oBACzD,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5C,CAAC;gBACD,wCAAwC;gBACxC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,mEAAmE;IACnE,uDAAuD;IACvD,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAGlD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,CAAC,CAAC;YAC3F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,yFAAyF;iBACjG,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,SAAS,EAAE;gBACrC,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC;YAC9D,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAA;QAE9D,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3B,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;oBAC7C,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,yFAAyF;qBACjG,CAAC;gBACJ,CAAC;gBAED,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAEhF,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE;oBAC1C,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YAC5D,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;IAC5C,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC1D,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpD,CAAC;IAGD,0FAA0F;IAC1F,KAAK,CAAC,yBAAyB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,4BAA4B,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,wBAAwB,CAAC,WAAmB;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,2BAA2B,GAAG,mBAAmB,CAAC,WAAW,CAAE,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,CAAC,sBAAsB,CAAC,WAAmB;QAC9C,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,4BAA4B,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC3G,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,mBAAmB,CAAC,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IAEH,CAAC;IACD,KAAK,CAAC,yBAAyB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,CAAC;QAC3F,CAAC;QACG,4HAA4H;IAClI,CAAC;IACD,KAAK,CAAC,uBAAuB,CAAC,WAAmB;QAC/C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnF,CAAC;IACD,KAAK,CAAC,oBAAoB,CAAC,WAAmB;QAC5C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAEnF,CAAC;IACD,KAAK,CAAC,sBAAsB,CAAC,WAAmB;QAC9C,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;CACF"}

package/dist/MetadataManager.d.ts

@@ -1,19 +1,7 @@
 import type { DocumentSetMetadata, Settings } from './types/index.js';
+import type { Knex } from 'knex';
 export declare abstract class MetadataManager {
-    protected queries: {
-        createDocumentSetsTable: string;
-        createSettingsTable: string;
-        insertDocumentSet: string;
-        selectDocumentSet: string;
-        selectDocumentSets: string;
-        countDocumentSets: string;
-        updateDocumentCount: string;
-        deleteDocumentSet: string;
-        selectSettings: string;
-        upsertSettings: string;
-    };
-    protected abstract runQuery<T>(query: string, params?: any[]): Promise<T[]>;
-    protected abstract runQuerySingle<T>(query: string, params?: any[]): Promise<T | null>;
+    protected abstract knex: Knex;
     protected abstract initializeDatabase(): Promise<void>;
     protected abstract close(): void;
     addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number>;

package/dist/MetadataManager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"MetadataManager.d.ts","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEtE,8BAAsB,eAAe;IACnC,SAAS,CAAC,OAAO;;;;;;;;;;;MAmDf;IAEF,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAC3E,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IACtF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IACtD,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;IAE1B,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAUrF,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAoB1E,eAAe,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAwBtH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxE,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvD,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAehC,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CAMrE"}
+{"version":3,"file":"MetadataManager.d.ts","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAEjC,8BAAsB,eAAe;IACnC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IAE9B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IACtD,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;IAE1B,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAarF,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC;IAgB1E,eAAe,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,mBAAmB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAuBtH,mBAAmB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMxE,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvD,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC;IAkBhC,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CAWrE"}

package/dist/MetadataManager.js

@@ -1,67 +1,19 @@
 export class MetadataManager {
-    queries = {
-        /*
-        Note: RETURNING on non-select/non-create statements is important for compatibility between SQLite and PostgreSQL.
-        (Without it, better-sqlite would demand to use run() instead of all() or get(), which would break the abstraction.)
-        */
-        createDocumentSetsTable: `
-      CREATE TABLE IF NOT EXISTS document_sets (
-        set_id SERIAL PRIMARY KEY,
-        name TEXT NOT NULL UNIQUE,
-        upload_date TIMESTAMP NOT NULL,
-        parameters TEXT NOT NULL,
-        total_documents INTEGER NOT NULL DEFAULT 0
-      );
-    `,
-        createSettingsTable: `
-      CREATE TABLE IF NOT EXISTS meaningfully_settings (
-        settings_id SERIAL PRIMARY KEY,
-        settings TEXT NOT NULL
-      );
-    `,
-        insertDocumentSet: `
-      INSERT INTO document_sets (name, upload_date, parameters, total_documents)
-      VALUES ($1, $2, $3, $4) RETURNING set_id
-    `,
-        selectDocumentSet: `
-      SELECT * FROM document_sets WHERE set_id = $1
-    `,
-        selectDocumentSets: `
-      SELECT * FROM document_sets ORDER BY upload_date DESC LIMIT $1 OFFSET $2
-    `,
-        countDocumentSets: `
-      SELECT COUNT(*) as count FROM document_sets
-    `,
-        updateDocumentCount: `
-      UPDATE document_sets SET total_documents = total_documents + $1 WHERE set_id = $2 RETURNING *
-    `,
-        deleteDocumentSet: `
-      DELETE FROM document_sets WHERE set_id = $1 RETURNING *
-    `,
-        selectSettings: `
-      SELECT * FROM meaningfully_settings WHERE settings_id = 1
-    `,
-        upsertSettings: `
-      INSERT INTO meaningfully_settings (settings_id, settings)
-      VALUES (1, $1)
-      ON CONFLICT (settings_id) DO UPDATE SET settings = $2
-      RETURNING *
-    `
-        // the two arguments $1 and $2 are identical, but, to work around a cross-compatibility bug in SQLite versus Postgresql,
-        // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
-        // they are specified separately.
-    };
     async addDocumentSet(metadata) {
-        const result = await this.runQuerySingle(this.queries.insertDocumentSet, [
-            metadata.name,
-            metadata.uploadDate.toISOString(),
-            JSON.stringify(metadata.parameters),
-            metadata.totalDocuments
-        ]);
-        return result?.set_id || 0;
+        const [result] = await this.knex('document_sets')
+            .insert({
+            name: metadata.name,
+            upload_date: metadata.uploadDate,
+            parameters: JSON.stringify(metadata.parameters),
+            total_documents: metadata.totalDocuments
+        })
+            .returning('set_id');
+        return typeof result === 'object' ? result.set_id : result;
     }
     async getDocumentSet(documentSetId) {
-        const row = await this.runQuerySingle(this.queries.selectDocumentSet, [documentSetId]);
+        const row = await this.knex('document_sets')
+            .where('set_id', documentSetId)
+            .first();
         if (!row)
             return null;
         return {
@@ -74,9 +26,13 @@ export class MetadataManager {
     }
     async getDocumentSets(page = 1, pageSize = 10) {
         const offset = (page - 1) * pageSize;
-        const totalCountRow = await this.runQuerySingle(this.queries.countDocumentSets);
-        const totalCount = totalCountRow?.count || 0;
-        const rows = await this.runQuery(this.queries.selectDocumentSets, [pageSize, offset]);
+        const totalCountRow = await this.knex('document_sets').count('* as count').first();
+        const totalCount = totalCountRow ? Number(totalCountRow.count) : 0;
+        const rows = await this.knex('document_sets')
+            .select('*')
+            .orderBy('upload_date', 'desc')
+            .limit(pageSize)
+            .offset(offset);
         const documents = rows.map((row) => ({
             documentSetId: row.set_id,
             name: row.name,
@@ -87,10 +43,14 @@ export class MetadataManager {
         return { documents, total: totalCount };
     }
     async updateDocumentCount(documentSetId, count) {
-        await this.runQuery(this.queries.updateDocumentCount, [count, documentSetId]);
+        await this.knex('document_sets')
+            .where('set_id', documentSetId)
+            .increment('total_documents', count);
     }
     async deleteDocumentSet(documentSetId) {
-        await this.runQuery(this.queries.deleteDocumentSet, [documentSetId]);
+        await this.knex('document_sets')
+            .where('set_id', documentSetId)
+            .delete();
     }
     async getSettings() {
         const DEFAULT_SETTINGS = {
@@ -102,13 +62,19 @@ export class MetadataManager {
             mistralApiKey: null,
             geminiApiKey: null,
         };
-        const row = await this.runQuerySingle(this.queries.selectSettings);
+        const row = await this.knex('meaningfully_settings')
+            .where('settings_id', 1)
+            .first();
         return row ? { ...DEFAULT_SETTINGS, ...JSON.parse(row.settings) } : DEFAULT_SETTINGS;
     }
     async setSettings(settings) {
-        // the JSON.stringify(settings) is repeated to work around a cross-compatibility bug in SQLite versus Postgresql
-        // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
-        await this.runQuery(this.queries.upsertSettings, [JSON.stringify(settings), JSON.stringify(settings)]);
+        await this.knex('meaningfully_settings')
+            .insert({
+            settings_id: 1,
+            settings: JSON.stringify(settings)
+        })
+            .onConflict('settings_id')
+            .merge();
         return { success: true };
     }
 }

package/dist/MetadataManager.js.map

@@ -1 +1 @@
-{"version":3,"file":"MetadataManager.js","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAEA,MAAM,OAAgB,eAAe;IACzB,OAAO,GAAG;QAClB;;;UAGE;QACF,uBAAuB,EAAE;;;;;;;;KAQxB;QACD,mBAAmB,EAAE;;;;;KAKpB;QACD,iBAAiB,EAAE;;;KAGlB;QACD,iBAAiB,EAAE;;KAElB;QACD,kBAAkB,EAAE;;KAEnB;QACD,iBAAiB,EAAE;;KAElB;QACD,mBAAmB,EAAE;;KAEpB;QACD,iBAAiB,EAAE;;KAElB;QACD,cAAc,EAAE;;KAEf;QACD,cAAc,EAAE;;;;;KAKf;QACD,wHAAwH;QACxH,8HAA8H;QAC9H,iCAAiC;KAClC,CAAC;IAOF,KAAK,CAAC,cAAc,CAAC,QAAoD;QACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAqB,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE;YAC3F,QAAQ,CAAC,IAAI;YACb,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE;YACjC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YACnC,QAAQ,CAAC,cAAc;SACxB,CAAC,CAAC;QACH,OAAO,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAMlC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;QAEpD,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC3D,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QACrC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,cAAc,CAAoB,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACnG,MAAM,UAAU,GAAG,aAAa,EAAE,KAAK,IAAI,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAM7B,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAExD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,aAAqB,EAAE,KAAa;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,gBAAgB,GAAa;YACjC,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,mBAAmB,EAAE,IAAI;YACzB,qBAAqB,EAAE,YAAY;YACnC,aAAa,EAAE,IAAI;YACnB,YAAY,EAAE,IAAI;SACnB,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAuB,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzF,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,gHAAgH;QAChH,8HAA8H;QAC9H,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}
+{"version":3,"file":"MetadataManager.js","sourceRoot":"","sources":["../src/MetadataManager.ts"],"names":[],"mappings":"AAGA,MAAM,OAAgB,eAAe;IAMnC,KAAK,CAAC,cAAc,CAAC,QAAoD;QACvE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC9C,MAAM,CAAC;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,WAAW,EAAE,QAAQ,CAAC,UAAU;YAChC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;YAC/C,eAAe,EAAE,QAAQ,CAAC,cAAc;SACzC,CAAC;aACD,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEvB,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aACzC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC;aAC9B,KAAK,EAAE,CAAC;QAEX,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC3D,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;QAErC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,EAAE,CAAC;QACnF,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC1C,MAAM,CAAC,GAAG,CAAC;aACX,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC;aAC9B,KAAK,CAAC,QAAQ,CAAC;aACf,MAAM,CAAC,MAAM,CAAC,CAAC;QAElB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;YACxC,aAAa,EAAE,GAAG,CAAC,MAAM;YACzB,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC;YACrC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC;YACtC,cAAc,EAAE,GAAG,CAAC,eAAe;SACpC,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,aAAqB,EAAE,KAAa;QAC5D,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC7B,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC;aAC9B,SAAS,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC;aAC7B,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC;aAC9B,MAAM,EAAE,CAAC;IACd,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,gBAAgB,GAAa;YACjC,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,mBAAmB,EAAE,IAAI;YACzB,qBAAqB,EAAE,YAAY;YACnC,aAAa,EAAE,IAAI;YACnB,YAAY,EAAE,IAAI;SACnB,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC;aACjD,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;aACvB,KAAK,EAAE,CAAC;QAEX,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,gBAAgB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC;aACrC,MAAM,CAAC;YACN,WAAW,EAAE,CAAC;YACd,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;SACnC,CAAC;aACD,UAAU,CAAC,aAAa,CAAC;aACzB,KAAK,EAAE,CAAC;QAEX,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}

package/dist/api/embedding.d.ts

@@ -1,6 +1,7 @@
 import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
-export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
-export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
+import { Document } from "llamaindex";
+export declare function createEmbeddings(documents: Document[], config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
+export declare function previewResults(documents: Document[], config: EmbeddingConfig): Promise<PreviewResult>;
 export declare function getDocStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").BaseDocumentStore>;
 export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
 export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;

package/dist/api/embedding.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CA0C1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,mDAE9F;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}
+{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE1I,OAAO,EAAgB,QAAQ,EAAE,MAAM,YAAY,CAAC;AAGpD,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAgC1B;AAGD,wBAAsB,cAAc,CAClC,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CA8BxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,mDAE9F;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}

package/dist/api/embedding.js

@@ -1,22 +1,12 @@
 import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getStorageContext } from "../services/embeddings.js";
-import { loadDocumentsFromCsv } from "../services/csvLoader.js";
 import { MetadataMode } from "llamaindex";
 import { ProgressManager } from "../services/progressManager.js";
-export async function createEmbeddings(csvPath, textColumnName, config, settings, clients) {
+export async function createEmbeddings(documents, config, settings, clients) {
     try {
         console.time("createEmbeddings Run Time");
         const operationId = `embed-${Date.now()}`;
         const progressManager = ProgressManager.getInstance();
         progressManager.startOperation(operationId, 100);
-        const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
-        if (documents.length === 0) {
-            progressManager.clearOperation(operationId);
-            console.timeEnd("createEmbeddings Run Time");
-            return {
-                success: false,
-                error: "That CSV does not appear to contain any documents. Please check the file and try again.",
-            };
-        }
         progressManager.updateProgress(operationId, 5);
         const nodes = await transformDocumentsToNodes(documents, config);
         const progressCallback = (progress, total) => {
@@ -42,15 +32,8 @@ export async function createEmbeddings(csvPath, textColumnName, config, settings
     }
 }
 // TODO: rename this to be parallel to createEmbeddings
-export async function previewResults(csvPath, textColumnName, config) {
+export async function previewResults(documents, config) {
     try {
-        const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
-        if (documents.length === 0) {
-            return {
-                success: false,
-                error: "That CSV does not appear to contain any documents. Please check the file and try again.",
-            };
-        }
         // Take 10 rows from the middle of the dataset for preview
         // we take a consistent 10 so that the results of the preview are consistent (i.e. with a larger chunk size, you have fewer, longer results, but more shorter ones if you adjust it)
         // and we take from the middle because the initial rows may be idiosyncratic.

package/dist/api/embedding.js.map

@@ -1 +1 @@
-{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAErL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;YACzD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;YAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAC1D,CAAC,CAAC;QACJ,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC;YAChE,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7F,OAAO,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}
+{"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAGrL,OAAO,EAAE,YAAY,EAAY,MAAM,YAAY,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAqB,EACrB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;YACzD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;YAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAC1D,CAAC,CAAC;QACJ,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC;YAChE,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAqB,EACrB,MAAuB;IAEvB,IAAI,CAAC;QACH,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7F,OAAO,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/services/csvLoader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csvLoader.d.ts","sourceRoot":"","sources":["../../src/services/csvLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgBrB"}
+{"version":3,"file":"csvLoader.d.ts","sourceRoot":"","sources":["../../src/services/csvLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAYtC,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GAErB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAiBrB"}

package/dist/services/csvLoader.js

@@ -1,7 +1,16 @@
 import { Document } from "llamaindex";
 import { readFileSync } from "fs";
 import Papa from "papaparse";
-export async function loadDocumentsFromCsv(filePath, textColumnName) {
+/*
+ I thought about only loading the user's specified metadataColumns (and omitting any unspecified ones)
+ but decided that -- on the principle that users know how to modify CSVs -- it's better to show all
+ columns in the record detail view, assuming that they left the column in the CSV for a reason.
+
+ Metadata column selection is thus about controlling which columns are shown in the results list view only.
+*/
+export async function loadDocumentsFromCsv(filePath, textColumnName
+//metadataColumns: string[]
+) {
     const fileContent = readFileSync(filePath, "utf-8");
     const { data: records } = Papa.parse(fileContent, {
         header: true,
@@ -11,7 +20,9 @@ export async function loadDocumentsFromCsv(filePath, textColumnName) {
         const { [textColumnName]: text, ...metadata } = record;
         return new Document({
             text,
-            metadata: Object.fromEntries(Object.entries(metadata).map(([k, v]) => [k, v ?? ""])),
+            metadata: Object.fromEntries(Object.entries(metadata).map(([k, v]) => [k, v ?? ""])
+            //metadataColumns.map((col) => [col, metadata[col] ?? ""])
+            ),
         });
     });
 }

package/dist/services/csvLoader.js.map

@@ -1 +1 @@
-{"version":3,"file":"csvLoader.js","sourceRoot":"","sources":["../../src/services/csvLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,cAAsB;IAEtB,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;QAChD,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE;QACjC,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;QACvD,OAAO,IAAI,QAAQ,CAAC;YAClB,IAAI;YACJ,QAAQ,EAAE,MAAM,CAAC,WAAW,CAC1B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CACvD;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"csvLoader.js","sourceRoot":"","sources":["../../src/services/csvLoader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B;;;;;;EAME;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,cAAsB;AACtB,2BAA2B;;IAE3B,MAAM,WAAW,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpD,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE;QAChD,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE;QACjC,MAAM,EAAE,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;QACvD,OAAO,IAAI,QAAQ,CAAC;YAClB,IAAI;YACJ,QAAQ,EAAE,MAAM,CAAC,WAAW,CAC1B,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACtD,0DAA0D;aAC3D;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/services/embeddings.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAIR,iBAAiB,EACjB,KAAK,cAAc,EAGnB,iBAAiB,EACjB,cAAc,EAEf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAA6C,MAAM,sBAAsB,CAAC;AAIhG,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAO9G;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAoB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAiDpN;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,aAAa,GAAG,iBAAiB,GAAG,2BAA2B,CAAC,CAqC/K;AAGD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAenI;AAGD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAe7H;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}
+{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAIR,iBAAiB,EACjB,KAAK,cAAc,EAGnB,iBAAiB,EACjB,cAAc,EAEf,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAA6C,MAAM,sBAAsB,CAAC;AAIhG,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAsB9G;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAoB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAiDpN;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,aAAa,GAAG,iBAAiB,GAAG,2BAA2B,CAAC,CAqC/K;AAGD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAenI;AAGD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAe7H;AAED,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}

package/dist/services/embeddings.js

@@ -68,12 +68,27 @@ export function estimateCost(nodes, modelName) {
     };
 }
 export async function getExistingVectorStoreIndex(config, settings, clients) {
-    const storageContext = await getStorageContext(config, settings, clients);
-    const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
-    if (!vectorStore) {
-        throw new Error("Vector store for ModalityType.TEXT is undefined");
+    let storageContext;
+    switch (config.vectorStoreType) {
+        case "simple":
+            const embedModel = getEmbedModel(config, settings);
+            const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+            storageContext = await storageContextFromDefaults({
+                persistDir: persistDir,
+            });
+            let vsi = await VectorStoreIndex.init({
+                storageContext: storageContext,
+            });
+            vsi.embedModel = embedModel;
+            return vsi;
+        default:
+            storageContext = await getStorageContext(config, settings, clients);
+            const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
+            if (!vectorStore) {
+                throw new Error("Vector store for ModalityType.TEXT is undefined");
+            }
+            return await VectorStoreIndex.fromVectorStore(vectorStore);
     }
-    return await VectorStoreIndex.fromVectorStore(vectorStore);
 }
 export async function transformDocumentsToNodes(documents, config) {
     console.time("transformDocumentsToNodes Run Time");
@@ -198,7 +213,7 @@ export async function persistNodes(nodes, config, settings, clients, progressCal
         if (progressCallback) {
             progressCallback(progress, nodes.length);
         }
-        console.log('progress total nodes.length', progress, nodes.length);
+        console.log('progress total nodes.length', progress, nodes.length); // TODO: give this `i`, so that it knows how many nodes have been processed so far outside of this chunk.
     };
     const index = await ProgressVectorStoreIndex.init({
         nodes: nodes.slice(0, NODE_CHUNK_SIZE),

package/dist/services/embeddings.js.map

@@ -1 +1 @@
-{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EAGnB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,MAAM,gBAAgB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,4BAA4B;IACnG,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,UAAU;QACpB;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,IAAI,cAAc,CAAC,QAAQ,YAAY,mBAAmB,EAAE,CAAC;QAC3D,aAAa;QACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC7G,CAAC;SAAK,IAAI,cAAc,CAAC,QAAQ,YAAY,qBAAqB,EAAE,CAAC;QACnE,sHAAsH;QACtH,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,eAAe,GAAG,KAAK,CAAA;IAC7B,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;QACnE,IAAI,gBAAgB,EAAE,CAAC;YACrB,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrE,CAAC,CAAA;IACD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC;QACtC,cAAc;QACd,gBAAgB,EAAE,wBAAwB;KAC3C,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,eAAe,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;YACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC;YAClD,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YAC7C,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YACtG,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,kGAAkG,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC5D,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACrG,gGAAgG;IAChG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC7D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,mBAAmB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACxD;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAClG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,kBAAkB,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,gBAAgB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACrD;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AAEH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EAGnB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,IAAI,cAA8B,CAAC;IACnC,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/B,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,cAAc,GAAG,MAAM,0BAA0B,CAAC;gBAChD,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;YACH,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACpC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YACH,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;YAC5B,OAAO,GAAG,CAAC;QACb;YACE,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;YACpE,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACnE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;YACrE,CAAC;YACD,OAAO,MAAM,gBAAgB,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,4BAA4B;IACnG,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,UAAU;QACpB;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,IAAI,cAAc,CAAC,QAAQ,YAAY,mBAAmB,EAAE,CAAC;QAC3D,aAAa;QACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC7G,CAAC;SAAK,IAAI,cAAc,CAAC,QAAQ,YAAY,qBAAqB,EAAE,CAAC;QACnE,sHAAsH;QACtH,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,eAAe,GAAG,KAAK,CAAA;IAC7B,MAAM,wBAAwB,GAAG,CAAC,QAAgB,EAAE,KAAa,EAAE,EAAE;QACnE,IAAI,gBAAgB,EAAE,CAAC;YACrB,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,yGAAyG;IAC/K,CAAC,CAAA;IACD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC;QACtC,cAAc;QACd,gBAAgB,EAAE,wBAAwB;KAC3C,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,eAAe,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;YACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC;YAClD,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YAC7C,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YACtG,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,kGAAkG,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC5D,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACrG,gGAAgG;IAChG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC7D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,mBAAmB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACxD;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAClG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,kBAAkB,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,gBAAgB,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QACrD;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AAEH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@meaningfully/core",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Core functionality for meaningfully semantic search",
-  "repo": "https://github.com/jeremybmerrill/meaningfully-core",
+  "repository": "https://github.com/jeremybmerrill/meaningfully-core",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.js",
@@ -29,6 +29,7 @@
     "@llamaindex/postgres": "^0.0.64",
     "@llamaindex/weaviate": "^0.0.36",
     "js-tiktoken": "^1.0.8",
+    "knex": "^3.1.0",
     "llamaindex": "^0.11.14",
     "lodash": "^4.17.21",
     "natural": "^8.1.0",

package/src/Meaningfully.ts

@@ -96,12 +96,20 @@ export class MeaningfullyAPI {
 
   async generatePreviewData(data: DocumentSetParamsFilePath) {
     const vectorStoreType = this.getVectorStoreType();
+
+
     try {
       if (!data.textColumns[0]) {
         throw new Error("No text column specified for preview.");
       }
-      console.log(data);
-      return await previewResults(data.filePath, data.textColumns[0] as string, {
+      const documents = await loadDocumentsFromCsv(data.filePath, data.textColumns[0] as string);
+      if (documents.length === 0) {
+        return {
+          success: false,
+          error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+        };
+      }      
+      return await previewResults(documents, {
         modelName: data.modelName, // needed to tokenize, estimate costs
         modelProvider: data.modelProvider,
         splitIntoSentences: data.splitIntoSentences,
@@ -149,11 +157,19 @@ export class MeaningfullyAPI {
       for (const textColumn of data.textColumns) {
         const documents = await loadDocumentsFromCsv(data.filePath, textColumn);
         
+        if (documents.length === 0) {
+          console.timeEnd("createEmbeddings Run Time");
+          return {
+            success: false,
+            error: "That CSV does not appear to contain any documents. Please check the file and try again.",
+          };
+        }
+
         // Update total documents count
         await this.metadataManager.updateDocumentCount(documentSetId, documents.length);
 
         // Create embeddings for this column
-        let ret = await createEmbeddings(data.filePath, textColumn, {
+        let ret = await createEmbeddings(documents, {
           modelName: data.modelName,
           modelProvider: data.modelProvider,
           splitIntoSentences: data.splitIntoSentences,

package/src/MetadataManager.ts

@@ -1,82 +1,29 @@
 import type { DocumentSetMetadata, Settings } from './types/index.js';
+import type { Knex } from 'knex';
 
 export abstract class MetadataManager {
-  protected queries = {
-    /* 
-    Note: RETURNING on non-select/non-create statements is important for compatibility between SQLite and PostgreSQL.
-    (Without it, better-sqlite would demand to use run() instead of all() or get(), which would break the abstraction.)
-    */
-    createDocumentSetsTable: `
-      CREATE TABLE IF NOT EXISTS document_sets (
-        set_id SERIAL PRIMARY KEY,
-        name TEXT NOT NULL UNIQUE,
-        upload_date TIMESTAMP NOT NULL,
-        parameters TEXT NOT NULL,
-        total_documents INTEGER NOT NULL DEFAULT 0
-      );
-    `,
-    createSettingsTable: `
-      CREATE TABLE IF NOT EXISTS meaningfully_settings (
-        settings_id SERIAL PRIMARY KEY,
-        settings TEXT NOT NULL
-      );
-    `,
-    insertDocumentSet: `
-      INSERT INTO document_sets (name, upload_date, parameters, total_documents)
-      VALUES ($1, $2, $3, $4) RETURNING set_id
-    `,
-    selectDocumentSet: `
-      SELECT * FROM document_sets WHERE set_id = $1
-    `,
-    selectDocumentSets: `
-      SELECT * FROM document_sets ORDER BY upload_date DESC LIMIT $1 OFFSET $2
-    `,
-    countDocumentSets: `
-      SELECT COUNT(*) as count FROM document_sets
-    `,
-    updateDocumentCount: `
-      UPDATE document_sets SET total_documents = total_documents + $1 WHERE set_id = $2 RETURNING *
-    `,
-    deleteDocumentSet: `
-      DELETE FROM document_sets WHERE set_id = $1 RETURNING *
-    `,
-    selectSettings: `
-      SELECT * FROM meaningfully_settings WHERE settings_id = 1
-    `,
-    upsertSettings: `
-      INSERT INTO meaningfully_settings (settings_id, settings)
-      VALUES (1, $1)
-      ON CONFLICT (settings_id) DO UPDATE SET settings = $2
-      RETURNING *
-    `     
-    // the two arguments $1 and $2 are identical, but, to work around a cross-compatibility bug in SQLite versus Postgresql,
-    // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
-    // they are specified separately.
-  };
+  protected abstract knex: Knex;
 
-  protected abstract runQuery<T>(query: string, params?: any[]): Promise<T[]>;
-  protected abstract runQuerySingle<T>(query: string, params?: any[]): Promise<T | null>;
   protected abstract initializeDatabase(): Promise<void>;
   protected abstract close(): void;
 
   async addDocumentSet(metadata: Omit<DocumentSetMetadata, 'documentSetId'>): Promise<number> {
-    const result = await this.runQuerySingle<{ set_id: number }>(this.queries.insertDocumentSet, [
-      metadata.name,
-      metadata.uploadDate.toISOString(),
-      JSON.stringify(metadata.parameters),
-      metadata.totalDocuments
-    ]);
-    return result?.set_id || 0;
+    const [result] = await this.knex('document_sets')
+      .insert({
+        name: metadata.name,
+        upload_date: metadata.uploadDate,
+        parameters: JSON.stringify(metadata.parameters),
+        total_documents: metadata.totalDocuments
+      })
+      .returning('set_id');
+    
+    return typeof result === 'object' ? result.set_id : result;
   }
 
   async getDocumentSet(documentSetId: number): Promise<DocumentSetMetadata | null> {
-    const row = await this.runQuerySingle<{
-      set_id: number;
-      name: string;
-      upload_date: string;
-      parameters: string;
-      total_documents: number;
-    }>(this.queries.selectDocumentSet, [documentSetId]);
+    const row = await this.knex('document_sets')
+      .where('set_id', documentSetId)
+      .first();
 
     if (!row) return null;
 
@@ -91,18 +38,17 @@ export abstract class MetadataManager {
 
   async getDocumentSets(page: number = 1, pageSize: number = 10): Promise<{ documents: DocumentSetMetadata[]; total: number }> {
     const offset = (page - 1) * pageSize;
-    const totalCountRow = await this.runQuerySingle<{ count: number }>(this.queries.countDocumentSets);
-    const totalCount = totalCountRow?.count || 0;
+    
+    const totalCountRow = await this.knex('document_sets').count('* as count').first();
+    const totalCount = totalCountRow ? Number(totalCountRow.count) : 0;
 
-    const rows = await this.runQuery<{
-      set_id: number;
-      name: string;
-      upload_date: string;
-      parameters: string;
-      total_documents: number;
-    }>(this.queries.selectDocumentSets, [pageSize, offset]);
+    const rows = await this.knex('document_sets')
+      .select('*')
+      .orderBy('upload_date', 'desc')
+      .limit(pageSize)
+      .offset(offset);
 
-    const documents = rows.map((row) => ({
+    const documents = rows.map((row: any) => ({
       documentSetId: row.set_id,
       name: row.name,
       uploadDate: new Date(row.upload_date),
@@ -114,11 +60,15 @@ export abstract class MetadataManager {
   }
 
   async updateDocumentCount(documentSetId: number, count: number): Promise<void> {
-    await this.runQuery(this.queries.updateDocumentCount, [count, documentSetId]);
+    await this.knex('document_sets')
+      .where('set_id', documentSetId)
+      .increment('total_documents', count);
   }
 
   async deleteDocumentSet(documentSetId: number): Promise<void> {
-    await this.runQuery(this.queries.deleteDocumentSet, [documentSetId]);
+    await this.knex('document_sets')
+      .where('set_id', documentSetId)
+      .delete();
   }
 
   async getSettings(): Promise<Settings> {
@@ -132,14 +82,22 @@ export abstract class MetadataManager {
       geminiApiKey: null,
     };
 
-    const row = await this.runQuerySingle<{ settings: string }>(this.queries.selectSettings);
+    const row = await this.knex('meaningfully_settings')
+      .where('settings_id', 1)
+      .first();
+    
     return row ? { ...DEFAULT_SETTINGS, ...JSON.parse(row.settings) } : DEFAULT_SETTINGS;
   }
 
   async setSettings(settings: Settings): Promise<{ success: boolean }> {
-    // the JSON.stringify(settings) is repeated to work around a cross-compatibility bug in SQLite versus Postgresql
-    // where PG can accept the same argument twice (specified as $1 in two places), but SQLITE cannot (it just has ? placeholders)
-    await this.runQuery(this.queries.upsertSettings, [JSON.stringify(settings), JSON.stringify(settings)]);
+    await this.knex('meaningfully_settings')
+      .insert({
+        settings_id: 1,
+        settings: JSON.stringify(settings)
+      })
+      .onConflict('settings_id')
+      .merge();
+    
     return { success: true };
   }
 }

package/src/__tests__/MetadataManager.test.ts

@@ -1,24 +1,47 @@
 import { MetadataManager } from '../MetadataManager.js';
 import { describe, it, expect, vi, beforeEach } from 'vitest';
+import type { Knex } from 'knex';
 
 describe('MetadataManager', () => {
   let metadataManager: MetadataManager;
+  let mockKnex: any;
 
   beforeEach(() => {
+    // Create a mock Knex instance with chainable methods
+    mockKnex = {
+      schema: {
+        hasTable: vi.fn().mockResolvedValue(true),
+        createTable: vi.fn().mockReturnThis(),
+      },
+      insert: vi.fn().mockReturnThis(),
+      returning: vi.fn().mockResolvedValue([{ set_id: 1 }]),
+      where: vi.fn().mockReturnThis(),
+      first: vi.fn().mockResolvedValue(null),
+      select: vi.fn().mockReturnThis(),
+      orderBy: vi.fn().mockReturnThis(),
+      limit: vi.fn().mockReturnThis(),
+      offset: vi.fn().mockResolvedValue([]),
+      count: vi.fn().mockReturnThis(),
+      increment: vi.fn().mockResolvedValue(undefined),
+      delete: vi.fn().mockResolvedValue(undefined),
+      onConflict: vi.fn().mockReturnThis(),
+      merge: vi.fn().mockResolvedValue(undefined),
+      destroy: vi.fn().mockResolvedValue(undefined),
+    };
+
+    // Make mockKnex callable as a function that returns itself for chaining
+    const knexFn = vi.fn().mockReturnValue(mockKnex);
+    Object.assign(knexFn, mockKnex);
+
     metadataManager = new (class extends MetadataManager {
-      protected async runQuery<T>(query: string, params?: any[]): Promise<T[]> {
-        return [] as T[];
-      }
-      protected async runQuerySingle<T>(query: string, params?: any[]): Promise<T | null> {
-        return null;
-      }
+      protected knex = knexFn as any;
       protected async initializeDatabase(): Promise<void> {}
       protected close(): void {}
     })();
   });
 
   it('should add a document set and return its ID', async () => {
-    vi.spyOn(metadataManager, 'runQuerySingle').mockResolvedValueOnce({ set_id: 1 });
+    mockKnex.returning.mockResolvedValueOnce([{ set_id: 1 }]);
 
     const documentSetId = await metadataManager.addDocumentSet({
       name: 'Test Set',
@@ -31,7 +54,7 @@ describe('MetadataManager', () => {
   });
 
   it('should retrieve a document set by ID', async () => {
-    vi.spyOn(metadataManager, 'runQuerySingle').mockResolvedValueOnce({
+    mockKnex.first.mockResolvedValueOnce({
       set_id: 1,
       name: 'Test Set',
       upload_date: new Date().toISOString(),
@@ -51,23 +74,19 @@ describe('MetadataManager', () => {
   });
 
   it('should update the document count for a document set', async () => {
-    const runQuerySpy = vi.spyOn(metadataManager, 'runQuery').mockResolvedValueOnce([]);
-
     await metadataManager.updateDocumentCount(1, 5);
 
-    expect(runQuerySpy).toHaveBeenCalledWith(metadataManager.queries.updateDocumentCount, [5, 1]);
+    expect(mockKnex.increment).toHaveBeenCalledWith('total_documents', 5);
   });
 
   it('should delete a document set by ID', async () => {
-    const runQuerySpy = vi.spyOn(metadataManager, 'runQuery').mockResolvedValueOnce([]);
-
     await metadataManager.deleteDocumentSet(1);
 
-    expect(runQuerySpy).toHaveBeenCalledWith(metadataManager.queries.deleteDocumentSet, [1]);
+    expect(mockKnex.delete).toHaveBeenCalled();
   });
 
   it('should retrieve default settings if none exist', async () => {
-    vi.spyOn(metadataManager, 'runQuerySingle').mockResolvedValueOnce(null);
+    mockKnex.first.mockResolvedValueOnce(null);
 
     const settings = await metadataManager.getSettings();
 
@@ -83,8 +102,6 @@ describe('MetadataManager', () => {
   });
 
   it('should update settings', async () => {
-    const runQuerySpy = vi.spyOn(metadataManager, 'runQuery').mockResolvedValueOnce([]);
-
     const result = await metadataManager.setSettings({
       openAIKey: 'test-key',
       oLlamaBaseURL: 'http://localhost',
@@ -96,25 +113,8 @@ describe('MetadataManager', () => {
     });
 
     expect(result).toEqual({ success: true });
-    expect(runQuerySpy).toHaveBeenCalledWith(metadataManager.queries.upsertSettings, [
-      JSON.stringify({
-        openAIKey: 'test-key',
-        oLlamaBaseURL: 'http://localhost',
-        azureOpenAIKey: 'azure-key',
-        azureOpenAIEndpoint: 'http://azure.endpoint',
-        azureOpenAIApiVersion: '2024-02-01',
-        mistralApiKey: 'mistral-key',
-        geminiApiKey: 'gemini-key',
-      }),
-      JSON.stringify({
-        openAIKey: 'test-key',
-        oLlamaBaseURL: 'http://localhost',
-        azureOpenAIKey: 'azure-key',
-        azureOpenAIEndpoint: 'http://azure.endpoint',
-        azureOpenAIApiVersion: '2024-02-01',
-        mistralApiKey: 'mistral-key',
-        geminiApiKey: 'gemini-key',
-      }),
-    ]);
+    expect(mockKnex.insert).toHaveBeenCalled();
+    expect(mockKnex.onConflict).toHaveBeenCalledWith('settings_id');
+    expect(mockKnex.merge).toHaveBeenCalled();
   });
 });

package/src/api/embedding.ts

@@ -1,12 +1,11 @@
 import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getStorageContext } from "../services/embeddings.js";
 import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
 import { loadDocumentsFromCsv } from "../services/csvLoader.js";
-import { MetadataMode } from "llamaindex";
+import { MetadataMode, Document } from "llamaindex";
 import { ProgressManager } from "../services/progressManager.js";
 
 export async function createEmbeddings(
-  csvPath: string,
-  textColumnName: string,
+  documents: Document[],
   config: EmbeddingConfig,
   settings: Settings,
   clients: Clients
@@ -17,16 +16,6 @@ export async function createEmbeddings(
     const progressManager = ProgressManager.getInstance();
     progressManager.startOperation(operationId, 100);
 
-    const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
-    if (documents.length === 0) {
-      progressManager.clearOperation(operationId);
-      console.timeEnd("createEmbeddings Run Time");
-      return {
-        success: false,
-        error: "That CSV does not appear to contain any documents. Please check the file and try again.",
-      };
-    }
-    
     progressManager.updateProgress(operationId, 5);
     
     const nodes = await transformDocumentsToNodes(documents, config);
@@ -56,18 +45,10 @@ export async function createEmbeddings(
 
 // TODO: rename this to be parallel to createEmbeddings
 export async function previewResults(
-  csvPath: string,
-  textColumnName: string,
+  documents: Document[],
   config: EmbeddingConfig
 ): Promise<PreviewResult> {
   try {
-    const documents = await loadDocumentsFromCsv(csvPath, textColumnName);
-    if (documents.length === 0) {
-      return {
-        success: false,
-        error: "That CSV does not appear to contain any documents. Please check the file and try again.",
-      };
-    }
     // Take 10 rows from the middle of the dataset for preview
     // we take a consistent 10 so that the results of the preview are consistent (i.e. with a larger chunk size, you have fewer, longer results, but more shorter ones if you adjust it)
     // and we take from the middle because the initial rows may be idiosyncratic.

package/src/services/csvLoader.ts

@@ -2,9 +2,18 @@ import { Document } from "llamaindex";
 import { readFileSync } from "fs";
 import Papa from "papaparse";
 
+/*
+ I thought about only loading the user's specified metadataColumns (and omitting any unspecified ones)
+ but decided that -- on the principle that users know how to modify CSVs -- it's better to show all 
+ columns in the record detail view, assuming that they left the column in the CSV for a reason.
+
+ Metadata column selection is thus about controlling which columns are shown in the results list view only.
+*/
+
 export async function loadDocumentsFromCsv(
   filePath: string,
   textColumnName: string
+  //metadataColumns: string[]
 ): Promise<Document[]> {
   const fileContent = readFileSync(filePath, "utf-8");
   const { data: records } = Papa.parse(fileContent, {
@@ -18,6 +27,7 @@ export async function loadDocumentsFromCsv(
       text,
       metadata: Object.fromEntries(
         Object.entries(metadata).map(([k, v]) => [k, v ?? ""])
+        //metadataColumns.map((col) => [col, metadata[col] ?? ""])
       ),
     });
   });

package/src/services/embeddings.ts

@@ -100,12 +100,27 @@ export function estimateCost(nodes: TextNode[], modelName: string): {
 }
 
 export async function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
-  const storageContext = await getStorageContext(config, settings, clients);
-  const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
-  if (!vectorStore) {
-    throw new Error("Vector store for ModalityType.TEXT is undefined");
+  let storageContext: StorageContext;
+  switch (config.vectorStoreType) {
+    case "simple":
+      const embedModel = getEmbedModel(config, settings);      
+      const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
+      storageContext = await storageContextFromDefaults({
+        persistDir: persistDir,
+      });
+      let vsi = await VectorStoreIndex.init({
+        storageContext: storageContext,
+      });
+      vsi.embedModel = embedModel;
+      return vsi;
+    default:
+      storageContext = await getStorageContext(config, settings, clients);
+      const vectorStore = storageContext.vectorStores[ModalityType.TEXT];
+      if (!vectorStore) {
+        throw new Error("Vector store for ModalityType.TEXT is undefined");
+      }
+      return await VectorStoreIndex.fromVectorStore(vectorStore);
   }
-  return await VectorStoreIndex.fromVectorStore(vectorStore);
 }
 
 export async function transformDocumentsToNodes(
@@ -242,7 +257,7 @@ export async function persistNodes(nodes: TextNode[], config: EmbeddingConfig, s
     if (progressCallback) {
       progressCallback(progress, nodes.length)
     }
-    console.log('progress total nodes.length', progress, nodes.length);
+    console.log('progress total nodes.length', progress, nodes.length); // TODO: give this `i`, so that it knows how many nodes have been processed so far outside of this chunk.
   }
   const index = await ProgressVectorStoreIndex.init({
     nodes: nodes.slice(0, NODE_CHUNK_SIZE),