@meaningfully/core 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/dist/Meaningfully.d.ts +3 -2
  2. package/dist/Meaningfully.d.ts.map +1 -1
  3. package/dist/Meaningfully.js +8 -4
  4. package/dist/Meaningfully.js.map +1 -1
  5. package/dist/api/embedding.d.ts +1 -1
  6. package/dist/api/embedding.d.ts.map +1 -1
  7. package/dist/api/embedding.js +3 -3
  8. package/dist/api/embedding.js.map +1 -1
  9. package/dist/services/embeddings.d.ts +0 -1
  10. package/dist/services/embeddings.d.ts.map +1 -1
  11. package/dist/services/embeddings.js +51 -72
  12. package/dist/services/embeddings.js.map +1 -1
  13. package/dist/types/index.d.ts +2 -0
  14. package/dist/types/index.d.ts.map +1 -1
  15. package/package.json +3 -2
  16. package/src/Meaningfully.ts +9 -5
  17. package/src/api/__tests__/embedding.test.ts +4 -4
  18. package/src/api/embedding.ts +3 -3
  19. package/src/services/embeddings.d.ts +0 -1
  20. package/src/services/embeddings.ts +57 -81
  21. package/src/types/index.ts +2 -0
package/dist/Meaningfully.d.ts CHANGED
@@ -8,9 +8,10 @@ export declare class MeaningfullyAPI {
8
8
  private manager;
9
9
  private storagePath;
10
10
  private clients;
11
- constructor({ storagePath, weaviateClient, metadataManager }: {
11
+ constructor({ storagePath, weaviateClient, postgresClient, metadataManager }: {
12
12
  storagePath: string;
13
13
  weaviateClient?: any;
14
+ postgresClient?: any;
14
15
  metadataManager: MetadataManager;
15
16
  });
16
17
  setClients(clients: Clients): void;
@@ -23,7 +24,7 @@ export declare class MeaningfullyAPI {
23
24
  deleteDocumentSet(documentSetId: number): Promise<{
24
25
  success: boolean;
25
26
  }>;
26
- getVectorStoreType(): "simple" | "weaviate";
27
+ getVectorStoreType(): "simple" | "postgres" | "weaviate";
27
28
  generatePreviewData(data: DocumentSetParamsFilePath): Promise<import("./types/index.js").PreviewResult>;
28
29
  uploadCsv(data: DocumentSetParamsFilePath): Promise<{
29
30
  success: boolean;
package/dist/Meaningfully.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IAS7I,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAGpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IAa7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAuBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IAyBzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;CAY9C"}
1
+ {"version":3,"file":"Meaningfully.d.ts","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAKvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG7F,KAAK,WAAW,GAAG;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC;AACtC,KAAK,yBAAyB,GAAG,iBAAiB,GAAG,WAAW,CAAC;AAUjE,qBAAa,eAAe;IAC1B,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAU;gBAEb,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,cAAc,CAAC,EAAE,GAAG,CAAC;QAAC,eAAe,EAAE,eAAe,CAAA;KAAE;IASnL,UAAU,CAAC,OAAO,EAAE,OAAO;IAG3B,UAAU;IAIJ,gBAAgB,CAAC,IAAI,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAW;;;;IAIxD,cAAc,CAAC,aAAa,EAAE,MAAM;IAGpC,iBAAiB,CAAC,aAAa,EAAE,MAAM;;;IAgB7C,kBAAkB;IAIZ,mBAAmB,CAAC,IAAI,EAAE,yBAAyB;IAuBnD,SAAS,CAAC,IAAI,EAAE,yBAAyB;;;;IA8DzC,iBAAiB,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAW,EAAI,OAAO,CAAC,EAAE,cAAc,EAAE;IAsB5G,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM;IA0BzD,WAAW;IAGX,WAAW,CAAC,QAAQ,EAAE,QAAQ;;;IAI9B,iBAAiB;;;;;;;;;IAYjB,iBAAiB,CAAC,WAAW,EAAE,QAAQ;;;CAY9C"}
package/dist/Meaningfully.js CHANGED
@@ -14,12 +14,12 @@ export class MeaningfullyAPI {
14
14
  manager;
15
15
  storagePath;
16
16
  clients;
17
- constructor({ storagePath, weaviateClient, metadataManager }) {
17
+ constructor({ storagePath, weaviateClient, postgresClient, metadataManager }) {
18
18
  this.storagePath = storagePath;
19
19
  this.manager = metadataManager;
20
20
  this.clients = {
21
21
  weaviateClient: weaviateClient,
22
- postgresClient: null
22
+ postgresClient: postgresClient
23
23
  };
24
24
  }
25
25
  setClients(clients) {
@@ -46,8 +46,11 @@ export class MeaningfullyAPI {
46
46
  }
47
47
  return { success: true };
48
48
  }
49
+ // TODO: this is an awful hack.
50
+ // I don't really want to have it be an argument to the constructor
51
+ // maybe I should make clients accept EITHER, not both?
49
52
  getVectorStoreType() {
50
- return this.clients.weaviateClient ? 'weaviate' : 'simple';
53
+ return this.clients.postgresClient ? 'postgres' : (this.clients.weaviateClient ? 'weaviate' : 'simple');
51
54
  }
52
55
  async generatePreviewData(data) {
53
56
  const vectorStoreType = this.getVectorStoreType();
@@ -152,6 +155,7 @@ export class MeaningfullyAPI {
152
155
  }
153
156
  async getDocument(documentSetId, documentNodeId) {
154
157
  const documentSet = await this.manager.getDocumentSet(documentSetId);
158
+ const settings = await this.manager.getSettings();
155
159
  if (!documentSet) {
156
160
  throw new Error('Document set not found');
157
161
  }
@@ -166,7 +170,7 @@ export class MeaningfullyAPI {
166
170
  storagePath: this.storagePath,
167
171
  chunkSize: 1024, // not actually used, we just re-use a config object that has this option
168
172
  chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
169
- });
173
+ }, settings, this.clients);
170
174
  const document = await docStore.getNode(documentNodeId);
171
175
  if (!document) {
172
176
  throw new Error('Document not found');
package/dist/Meaningfully.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAmF;QAC3I,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,IAAI;SACrB,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,kDAAkD;YAClD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,EAAE,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACtD,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEtD,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAExE,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;CAEF"}
1
+ {"version":3,"file":"Meaningfully.js","sourceRoot":"","sources":["../src/Meaningfully.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACrG,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,MAAM,IAAI,CAAC;AAKpB,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC,6FAA6F;AAC5F,gHAAgH;AAClJ,MAAM,OAAO,GAAG,CAAC,GAAkB,EAAE,IAAY,qBAAqB,EAAiB,EAAE;IACvF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9F,CAAC,CAAC;AAGF,MAAM,OAAO,eAAe;IAClB,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,OAAO,CAAU;IAEzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAyG;QACjL,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,cAAc,EAAE,cAAc;YAC9B,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,OAAgB;QACzB,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;IACjD,CAAC;IACD,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAe,CAAC,EAAE,WAAmB,EAAE;QAC5D,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,aAAqB;QACxC,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,aAAqB;QAC3C,4CAA4C;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,MAAM,EAAC,CAAC;YACV,4CAA4C;YAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,kDAAkD;YAClD,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,EAAE,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3H,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,+BAA+B;IAC/B,mEAAmE;IACnE,uDAAuD;IACvD,kBAAkB;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1G,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,IAA+B;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAW,EAAE;gBACxE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,qCAAqC;gBAChE,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,GAAG;gBACnB,eAAe,EAAE,eAAe;gBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEC,KAAK,CAAC,SAAS,CAAC,IAA+B;QAC7C,sCAAsC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClD,uCAAuC;QACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;YACtD,IAAI,EAAE,IAAI,CAAC,WAAW;YACtB,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,UAAU,EAAE;gBACV,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;gBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;gBAC3D,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,eAAe,EAAE,eAAe;aACjC;YACD,cAAc,EAAE,CAAC,CAAC,qCAAqC;SACxD,CAAC,CAAC;QAEH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAA;QAEtD,iCAAiC;QACjC,IAAI,CAAC;YACH,2BAA2B;YAC3B,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC1C,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAExE,+BAA+B;gBAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAExE,oCAAoC;gBACpC,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE;oBAC1D,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,kBAAkB,EAAE,IAAI,CAAC,kBAAkB;oBAC3C,0BAA0B,EAAE,IAAI,CAAC,0BAA0B;oBAC3D,cAAc,EAAE,GAAG,EAAE,0BAA0B;oBAC/C,eAAe,EAAE,eAAe;oBAChC,WAAW,EAAE,IAAI,CAAC,WAAW;oBACf,uFAAuF;oBACrG,WAAW,EAAG,IAAI,CAAC,WAAW;oBAC9B,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;iBAChC,EAAE,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;oBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iEAAiE;YACjE,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YAC7E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,KAAK,CAAC,iBAAiB,CAAC,aAAqB,EAAE,KAAa,EAAE,YAAoB,EAAE,EAAI,OAA0B;QAChH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,aAAqB,EAAE,cAAsB;QAC7D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;YACjC,SAAS,EAAE,WAAW,CAAC,UAAU,CAAC,SAAmB;YACrD,aAAa,EAAE,WAAW,CAAC,UAAU,CAAC,aAAuB;YAC7D,kBAAkB,EAAE,WAAW,CAAC,UAAU,CAAC,kBAA6B;YACxE,0BAA0B,EAAE,WAAW,CAAC,UAAU,CAAC,0BAAqC;YACxF,cAAc,EAAE,GAAG;YACnB,eAAe,EAAE,WAAW,CAAC,UAAU,CAAC,eAAwC;YAChF,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,SAAS,EAAE,IAAI,EAAE,yEAAyE;YAC1F,YAAY,EAAE,EAAE,EAAE,yEAAyE;SAC5F,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACxD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,WAAW,CAAC,QAAkB;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,iBAAiB;QACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,OAAO;YACL,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;YACtC,aAAa,EAAE,QAAQ,CAAC,aAAa;YACrC,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChD,mBAAmB,EAAE,QAAQ,CAAC,mBAAmB;YACjD,qBAAqB,EAAE,QAAQ,CAAC,qBAAqB;YACrD,aAAa,EAAE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;YAC9C,YAAY,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;SAC7C,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,iBAAiB,CAAC,WAAqB;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG;YACf,GAAG,WAAW;YACd,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS;YAClH,cAAc,EAAE,WAAW,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,cAAc;YAC3I,aAAa,EAAE,WAAW,CAAC,aAAa,IAAI,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa;YACtI,YAAY,EAAE,WAAW,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,YAAY;SAClI,CAAC;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC5C,CAAC;CAEF"}
package/dist/api/embedding.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
2
2
  export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
3
3
  export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
4
- export declare function getDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
4
+ export declare function getDocStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").BaseDocumentStore>;
5
5
  export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
6
6
  export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;
7
7
  //# sourceMappingURL=embedding.d.ts.map
package/dist/api/embedding.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAyC1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,mDAExD;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}
1
+ {"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAK1I,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,OAAO,GACf,OAAO,CAAC,eAAe,CAAC,CAyC1B;AAGD,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC,aAAa,CAAC,CAqCxB;AAED,wBAAsB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,mDAE9F;AAED,wBAAsB,QAAQ,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,kDAE3F;AAED,wBAAsB,MAAM,CAC1B,KAAK,EAAE,GAAG,EACV,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC,CASzB"}
package/dist/api/embedding.js CHANGED
@@ -1,4 +1,4 @@
1
- import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getExistingDocStore } from "../services/embeddings.js";
1
+ import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getStorageContext } from "../services/embeddings.js";
2
2
  import { loadDocumentsFromCsv } from "../services/csvLoader.js";
3
3
  import { MetadataMode } from "llamaindex";
4
4
  import { ProgressManager } from "../services/progressManager.js";
@@ -75,8 +75,8 @@ export async function previewResults(csvPath, textColumnName, config) {
75
75
  };
76
76
  }
77
77
  }
78
- export async function getDocStore(config) {
79
- return await getExistingDocStore(config);
78
+ export async function getDocStore(config, settings, clients) {
79
+ return (await getStorageContext(config, settings, clients)).docStore;
80
80
  }
81
81
  export async function getIndex(config, settings, clients) {
82
82
  return await getExistingVectorStoreIndex(config, settings, clients);
package/dist/api/embedding.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEvL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;gBAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC;YACF,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB;IACvD,OAAO,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}
1
+ {"version":3,"file":"embedding.js","sourceRoot":"","sources":["../../src/api/embedding.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,eAAe,EAAE,2BAA2B,EAAE,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAErL,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,cAAsB,EACtB,MAAuB,EACvB,QAAkB,EAClB,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,SAAS,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1C,MAAM,eAAe,GAAG,eAAe,CAAC,WAAW,EAAE,CAAC;QACtD,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEjD,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,eAAe,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;YAC7C,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QAED,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/C,MAAM,KAAK,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAEjE,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAChC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;gBACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,iCAAiC;gBAC7F,eAAe,CAAC,cAAc,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;YAC1D,CAAC,CAAC;YACF,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC;SACvD,CAAC,CAAC;QAEH,eAAe,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC7C,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;SACN,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,cAAsB,EACtB,MAAuB;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,yFAAyF;aACjG,CAAC;QACJ,CAAC;QACD,0DAA0D;QAC1D,oLAAoL;QACpL,6EAA6E;QAC7E,MAAM,sBAAsB,GAAG,SAAS,CAAC,KAAK,CAC5C,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAChC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CACtC,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,kBAAkB,GAAG,MAAM,yBAAyB,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC;QAC3F,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAEhG,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACxB,CAAC,CAAC;YACH,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7F,OAAO,CAAC,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC1F,OAAO,MAAM,2BAA2B,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAU,EACV,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACzE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;QAC9B,cAAc;QACd,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM;KACxD,CAAC,CAAC,CAAC;AACN,CAAC"}
package/dist/services/embeddings.d.ts CHANGED
@@ -12,7 +12,6 @@ export declare function estimateCost(nodes: TextNode[], modelName: string): {
12
12
  pricePer1M: number;
13
13
  };
14
14
  export declare function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<VectorStoreIndex>;
15
- export declare function getExistingDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
16
15
  export declare function transformDocumentsToNodes(documents: Document[], config: EmbeddingConfig): Promise<TextNode<import("llamaindex").Metadata>[]>;
17
16
  export declare function getEmbedModel(config: EmbeddingConfig, settings: Settings): OpenAIEmbedding | OllamaEmbedding | MistralAIEmbedding | GeminiEmbedding | MockEmbedding;
18
17
  export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;
package/dist/services/embeddings.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAKR,KAAK,cAAc,EAGpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAKpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAqD9G;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,eAAe,mDAehE;AAID,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAiB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAW1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCpN;AAyCD,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}
1
+ {"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,gBAAgB,EAIhB,QAAQ,EAKR,KAAK,cAAc,EAMpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAA+B,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAKpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAG,MAAM,mBAAmB,CAAC;AAG7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAuCzE,wBAAgB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAuBA;AAED,wBAAsB,2BAA2B,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,6BAK9G;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,QAAQ,EAAE,EACrB,MAAM,EAAE,eAAe,sDA0BxB;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,QAAQ,4FA2CnB;AAED,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,cAAc,CAAC,CAoB9H;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1I;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAqCpN;AAyED,wBAAsB,eAAe,CACnC,KAAK,EAAE,gBAAgB,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,GAAE,MAAW,EACvB,OAAO,CAAC,EAAE,cAAc,EAAE,gFAU3B"}
package/dist/services/embeddings.js CHANGED
@@ -1,10 +1,10 @@
1
1
  import { VectorStoreIndex,
2
2
  // OpenAIEmbedding,
3
- IngestionPipeline, ModalityType, storageContextFromDefaults, SimpleVectorStore, Settings as LlamaindexSettings, SimpleDocumentStore } from "llamaindex";
3
+ IngestionPipeline, ModalityType, storageContextFromDefaults, SimpleVectorStore, Settings as LlamaindexSettings, SimpleDocumentStore, SimpleIndexStore } from "llamaindex";
4
4
  import { OllamaEmbedding } from '@llamaindex/ollama';
5
5
  import { MistralAIEmbedding, MistralAIEmbeddingModelType } from '@llamaindex/mistral';
6
6
  import { GeminiEmbedding } from '@llamaindex/google';
7
- import { PGVectorStore } from '@llamaindex/postgres';
7
+ import { PGVectorStore, PostgresDocumentStore, PostgresIndexStore } from '@llamaindex/postgres';
8
8
  import { AzureOpenAIEmbedding } from "@llamaindex/azure";
9
9
  import { Sploder } from "./sploder.js";
10
10
  import { CustomSentenceSplitter } from "./sentenceSplitter.js";
@@ -68,69 +68,10 @@ export function estimateCost(nodes, modelName) {
68
68
  };
69
69
  }
70
70
  export async function getExistingVectorStoreIndex(config, settings, clients) {
71
- const embedModel = getEmbedModel(config, settings);
72
- switch (config.vectorStoreType) {
73
- case "simple":
74
- const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
75
- const storageContext = await storageContextFromDefaults({
76
- persistDir: persistDir,
77
- });
78
- let vsi = await VectorStoreIndex.init({
79
- storageContext: storageContext,
80
- });
81
- vsi.embedModel = embedModel;
82
- return vsi;
83
- case "postgres":
84
- if (!clients.postgresClient) {
85
- throw new Error("Postgres client required but not provided");
86
- }
87
- const pgStore = new PGVectorStore({
88
- clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
89
- tableName: sanitizeProjectName(config.projectName),
90
- dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
91
- embeddingModel: embedModel
92
- });
93
- const pgStorageContext = await storageContextFromDefaults({
94
- vectorStores: { [ModalityType.TEXT]: pgStore },
95
- });
96
- return await VectorStoreIndex.init({
97
- storageContext: pgStorageContext,
98
- });
99
- case "weaviate":
100
- if (!clients.weaviateClient) {
101
- throw new Error("Weaviate client required but not provided");
102
- }
103
- const weaviateStore = new BatchingWeaviateVectorStore({
104
- indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
105
- weaviateClient: clients.weaviateClient,
106
- embeddingModel: embedModel
107
- });
108
- // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
109
- // (WeaviateVectorStore would get `score` if we were doing hybrid search)
110
- // Overwrite the private getNodeSimilarity method to use 'score' from metadata
111
- // @ts-ignore
112
- weaviateStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
113
- return entry.metadata.score;
114
- };
115
- return await VectorStoreIndex.fromVectorStore(weaviateStore);
116
- default:
117
- throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
118
- }
119
- }
120
- export async function getExistingDocStore(config) {
121
- // switch (config.vectorStoreType) {
122
- // case "simple":
123
- const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
124
- const storageContext = await storageContextFromDefaults({
125
- persistDir: persistDir,
71
+ const storageContext = await getStorageContext(config, settings, clients);
72
+ return await VectorStoreIndex.init({
73
+ storageContext: storageContext,
126
74
  });
127
- return storageContext.docStore;
128
- // case "postgres":
129
- // throw new Error(`Not yet implemented vector store type: ${config.vectorStoreType}`);
130
- // // return await createVectorStore(config);
131
- // default:
132
- // throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
133
- // }
134
75
  }
135
76
  export async function transformDocumentsToNodes(documents, config) {
136
77
  console.time("transformDocumentsToNodes Run Time");
@@ -204,12 +145,15 @@ export function getEmbedModel(config, settings) {
204
145
  }
205
146
  export async function getStorageContext(config, settings, clients) {
206
147
  const vectorStore = await createVectorStore(config, settings, clients);
148
+ const docStore = await createDocumentStore(config, settings, clients); // new SimpleDocumentStore()
149
+ const indexStore = await createIndexStore(config, settings, clients);
207
150
  fs.mkdirSync(config.storagePath, { recursive: true });
208
151
  const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
209
152
  return await storageContextFromDefaults({
210
153
  persistDir: persistDir,
211
154
  vectorStores: { [ModalityType.TEXT]: vectorStore },
212
- docStore: new SimpleDocumentStore()
155
+ docStore: docStore,
156
+ indexStore: indexStore
213
157
  /*
214
158
  if docStore is created with a persist path (as it is by default in storageContextFromDefaults)
215
159
  then it will write to disk after every put(), which happens 2+ times per document.
@@ -226,8 +170,14 @@ export async function persistDocuments(documents, config, settings, clients) {
226
170
  await storageContext.docStore.addDocuments(documents, true);
227
171
  // see comments in getStorageContext
228
172
  const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
229
- // @ts-ignore
230
- await storageContext.docStore.kvStore.persist(join(persistDir, "doc_store.json"));
173
+ if (storageContext.docStore instanceof SimpleDocumentStore) {
174
+ // @ts-ignore
175
+ await storageContext.docStore.kvStore.persist(join(persistDir, "doc_store.json"));
176
+ }
177
+ else if (storageContext.docStore instanceof PostgresDocumentStore) {
178
+ // PostgresDocumentStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
179
+ console.log("Pretending to persist Postgres document store, but it actually persists automatically.");
180
+ }
231
181
  console.timeEnd("persistDocuments Run Time");
232
182
  }
233
183
  export async function persistNodes(nodes, config, settings, clients, progressCallback) {
@@ -252,12 +202,12 @@ export async function persistNodes(nodes, config, settings, clients, progressCal
252
202
  // all the if statements are just type-checking boilerplate.
253
203
  // N.B. WeaviateVectorStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
254
204
  if (vectorStore) {
255
- if (vectorStore instanceof PGVectorStore || vectorStore instanceof SimpleVectorStore) {
205
+ if (vectorStore instanceof SimpleVectorStore) {
256
206
  await vectorStore.persist(join(config.storagePath, sanitizeProjectName(config.projectName), "vector_store.json"));
257
207
  }
258
- else if (vectorStore instanceof BatchingWeaviateVectorStore) {
208
+ else if (vectorStore instanceof PGVectorStore || vectorStore instanceof BatchingWeaviateVectorStore) {
259
209
  // WeaviateVectorStore does not have a persist method, it persists automatically
260
- console.log("Pretending to persist Weaviate vector store, but it actually persists automatically.");
210
+ console.log("Pretending to persist Weaviate or Postgres vector store, but it actually persists automatically.");
261
211
  }
262
212
  else {
263
213
  throw new Error("Vector store does not support persist method");
@@ -276,8 +226,8 @@ async function createVectorStore(config, settings, clients) {
276
226
  // otherwise it defaults to Ada.
277
227
  case "postgres":
278
228
  return new PGVectorStore({
279
- clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
280
- tableName: sanitizeProjectName(config.projectName),
229
+ client: clients.postgresClient,
230
+ tableName: "vecs_" + sanitizeProjectName(config.projectName),
281
231
  dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
282
232
  embeddingModel: embeddingModel
283
233
  });
@@ -302,6 +252,35 @@ async function createVectorStore(config, settings, clients) {
302
252
  throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
303
253
  }
304
254
  }
255
+ async function createDocumentStore(config, settings, clients) {
256
+ // we create the doc store without a persist path, so it doesn't write to disk after every put()
257
+ switch (config.documentStoreType || config.vectorStoreType) {
258
+ case "postgres":
259
+ return new PostgresDocumentStore({
260
+ client: clients.postgresClient,
261
+ tableName: "docs_" + sanitizeProjectName(config.projectName),
262
+ });
263
+ case "simple":
264
+ case "weaviate":
265
+ return new SimpleDocumentStore();
266
+ default:
267
+ throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
268
+ }
269
+ }
270
+ async function createIndexStore(config, settings, clients) {
271
+ switch (config.documentStoreType || config.vectorStoreType) {
272
+ case "postgres":
273
+ return new PostgresIndexStore({
274
+ client: clients.postgresClient,
275
+ tableName: "idx_" + sanitizeProjectName(config.projectName),
276
+ });
277
+ case "simple":
278
+ case "weaviate":
279
+ return new SimpleIndexStore();
280
+ default:
281
+ throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
282
+ }
283
+ }
305
284
  export async function searchDocuments(index, query, numResults = 10, filters) {
306
285
  // const metadataFilters: MetadataFilters | undefined = filters ? {filters: filters} : undefined;
307
286
  const metadataFilters = {
package/dist/services/embeddings.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EACpB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACnD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAC/B,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC;gBACtD,UAAU,EAAE,UAAU;aACvB,CAAC,CAAC;YACH,IAAI,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACpC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YACH,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC;YAC5B,OAAO,GAAG,CAAC;QAEb,KAAK,UAAU;YACb,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;gBAChC,YAAY,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE;gBAC1E,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAClD,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,0BAA0B,CAAC;gBACxD,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE;aAC/C,CAAC,CAAC;YACH,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC;gBACjC,cAAc,EAAE,gBAAgB;aACjC,CAAC,CAAC;QACL,KAAK,UAAU;YACb,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,2BAA2B,CAAC;gBACpD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,aAAa,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBACpE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,MAAM,gBAAgB,CAAC,eAAe,CAAC,aAAa,CAAC,CAAA;QAE9D;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,MAAuB;IAC/D,oCAAoC;IACpC,mBAAmB;IACf,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,MAAM,cAAc,GAAG,MAAM,0BAA0B,CAAC;QACtD,UAAU,EAAE,UAAU;KACvB,CAAC,CAAC;IACH,OAAO,cAAc,CAAC,QAAQ,CAAC;IAEnC,qBAAqB;IACrB,2FAA2F;IAC3F,iDAAiD;IACjD,aAAa;IACb,mFAAmF;IACnF,IAAI;AACN,CAAC;AAID,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,IAAI,mBAAmB,EAAE;QACjC;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,aAAa;IACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAE3G,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK;QACL,cAAc;QACd,WAAW,EAAE,IAAI;QACjB,gBAAgB;KACjB,CAAC,CAAC;IAEH,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YACrF,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YAC9D,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,sFAAsF,CAAC,CAAC;QACtG,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC5F,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,YAAY,EAAE,EAAC,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAC;gBACxE,SAAS,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAClD,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}
1
+ {"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/services/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,gBAAgB;AAChB,mBAAmB;AACnB,iBAAiB,EAGjB,YAAY,EAEZ,0BAA0B,EAC1B,iBAAiB,EAEjB,QAAQ,IAAI,kBAAkB,EAC9B,mBAAmB,EAGnB,gBAAgB,EACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,qBAAqB,CAAA;AACrF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAChG,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,gDAAgD;AAChD,4FAA4F;AAC5F,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,GAAG,EAAE,yBAAyB;CACvD,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,eAAe,EAAE,GAAG;IACpB,mBAAmB,EAAE,CAAC,EAAE,oBAAoB;IAC5C,kBAAkB,EAAE,CAAC,EAAE,oBAAoB;IAC3C,sBAAsB,EAAE,GAAG,EAAE,gHAAgH;CAC9I,CAAC;AAGF,+FAA+F;AAC/F,SAAS,sBAAsB,CAAC,MAAuB;IACrD,MAAM,eAAe,GAAyB;QAC5C,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC;KAC/F,CAAC;IAEF,IAAI,MAAM,CAAC,0BAA0B,EAAE,CAAC;QACtC,eAAe,CAAC,IAAI,CAClB,IAAI,OAAO,CAAC;YACV,mBAAmB,EAAE,MAAM,CAAC,cAAc;SAC3C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,SAAiB;IAK/D,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,0CAA0C;IAE3F,IAAI,SAAS,CAAC;IACd,IAAG,CAAC;QACF,SAAS,GAAG,gBAAgB,CAAC,SAA0B,CAAC,CAAC,CAAC,+BAA+B;IAC3F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wFAAwF;QACxF,+EAA+E;QAC/E,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,gCAAgC;QACxF,OAAO,CAAC,IAAI,CAAC,uBAAuB,SAAS,uCAAuC,CAAC,CAAC;IACxF,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QAC5C,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC,EAAE,CAAC,CAAC,CAAC;IAEN,MAAM,cAAc,GAAG,UAAU,GAAG,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAE7D,OAAO;QACL,cAAc;QACd,UAAU;QACV,UAAU;KACX,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC7G,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,MAAM,gBAAgB,CAAC,IAAI,CAAC;QACjC,cAAc,EAAE,cAAc;KAC/B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,SAAqB,EACrB,MAAuB;IAEvB,OAAO,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,qEAAqE;IACrE,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,QAAQ,CAAC,yBAAyB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,6FAA6F;IAC7F,oDAAoD;IACpD,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEzF,4DAA4D;IAC5D,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;QACrC,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAe,CAAC;IAEzE,OAAO,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACzE,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,QAAkB;IAElB,IAAI,UAAU,CAAC;IACf,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAAC,CAAE,CAAC;QAC3H,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC,wGAAwG;IAC1I,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,UAAU,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE;gBAClE,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAClE,GAAG,CAAC,CAAC;IACR,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,UAAU,GAAG,IAAI,oBAAoB,CAAC;YACpC,KAAK,EAAE,MAAM,CAAC,SAAS;YACvB,MAAM,EAAE,QAAQ,CAAC,cAAc;YAC/B,QAAQ,EAAE,QAAQ,CAAC,mBAAmB;YACtC,UAAU,EAAE,QAAQ,CAAC,qBAAqB,IAAI,SAAS;SACxD,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,UAAU,GAAG,IAAI,kBAAkB,CAAC;YAClC,KAAK,EAAE,2BAA2B,CAAC,aAAa,EAAE,mBAAmB;YACrE,MAAM,EAAE,QAAQ,CAAC,aAAa;SAC/B,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,UAAU,GAAG,IAAI,eAAe,CAAC;YAC/B,MAAM,EAAE,QAAQ,CAAC,YAAY;SAC9B,CAAC,CAAC;QACH,UAAU,CAAC,cAAc,GAAG,EAAE,CAAC;IACjC,CAAC;SAAM,IAAI,MAAM,CAAC,aAAa,KAAK,MAAM,EAAE,CAAC;QAC3C,UAAU,GAAG,IAAI,aAAa,EAAE,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,kBAAkB,CAAC,UAAU,GAAG,UAAU,CAAC;IAC3C,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACnG,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,4BAA4B;IACnG,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,OAAO,MAAM,0BAA0B,CAAC;QACtC,UAAU,EAAE,UAAU;QACtB,YAAY,EAAE,EAAC,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,WAAW,EAAC;QAChD,QAAQ,EAAE,QAAQ;QAClB,UAAU,EAAE,UAAU;QACpB;;;;;;;UAOE;KACL,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAqB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IACzH,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,cAAc,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAE5D,oCAAoC;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAE,CAAC;IACtF,IAAI,cAAc,CAAC,QAAQ,YAAY,mBAAmB,EAAE,CAAC;QAC3D,aAAa;QACb,MAAO,cAAc,CAAC,QAAgC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC7G,CAAC;SAAK,IAAI,cAAc,CAAC,QAAQ,YAAY,qBAAqB,EAAE,CAAC;QACnE,sHAAsH;QACtH,OAAO,CAAC,GAAG,CAAC,wFAAwF,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAuB,EAAE,QAAkB,EAAE,OAAgB,EAAE,gBAA4D;IAC/K,kDAAkD;IAClD,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAEtC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,mCAAmC;IACnC,yCAAyC;IACzC,mDAAmD;IACnD,MAAM,KAAK,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC;QAChD,KAAK;QACL,cAAc;QACd,WAAW,EAAE,IAAI;QACjB,gBAAgB;KACjB,CAAC,CAAC;IAEH,gEAAgE;IAChE,qDAAqD;IACrD,4DAA4D;IAC5D,yHAAyH;IACzH,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,WAAW,YAAY,iBAAiB,EAAE,CAAC;YAC7C,MAAM,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACpH,CAAC;aAAM,IAAI,WAAW,YAAY,aAAa,IAAI,WAAW,YAAY,2BAA2B,EAAE,CAAC;YACtG,gFAAgF;YAChF,OAAO,CAAC,GAAG,CAAC,kGAAkG,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC5F,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACvD,QAAQ,MAAM,CAAC,eAAe,EAAE,CAAC;QAE/B,mEAAmE;QACnE,gCAAgC;QAChC,KAAK,UAAU;YACb,OAAO,IAAI,aAAa,CAAC;gBACvB,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC5D,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,IAAI,EAAE,qCAAqC;gBAC7F,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;YACrF,OAAO,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAEtE,KAAK,UAAU;YACb,MAAM,WAAW,GAAG,IAAI,2BAA2B,CAAC;gBAClD,SAAS,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACzE,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,cAAc;aAC/B,CAAC,CAAC;YAEH,yGAAyG;YACzG,yEAAyE;YACzE,8EAA8E;YAC9E,aAAa;YACb,WAAW,CAAC,iBAAiB,GAAG,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,EAAE,EAAE;gBAClE,OAAQ,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC/B,CAAC,CAAA;YAED,OAAO,WAAW,CAAC;QACrB;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC9F,gGAAgG;IAChG,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,qBAAqB,CAAC;gBAC/B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC7D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,mBAAmB,EAAE,CAAC;QACnC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,MAAuB,EAAE,QAAkB,EAAE,OAAgB;IAC3F,QAAQ,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QAC3D,KAAK,UAAU;YACb,OAAO,IAAI,kBAAkB,CAAC;gBAC5B,MAAM,EAAE,OAAO,CAAC,cAAc;gBAC9B,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC;aAC5D,CAAC,CAAC;QACL,KAAK,QAAQ,CAAC;QACd,KAAK,UAAU;YACb,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC;YACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAChF,CAAC;AAEH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAuB,EACvB,KAAa,EACb,aAAqB,EAAE,EACvB,OAA0B;IAE1B,iGAAiG;IACjG,MAAM,eAAe,GAAoB;QACvC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KAChC,CAAC;IACF,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAE9F,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAE,CAAC;IACjD,OAAO,OAAO,CAAC;AACjB,CAAC"}
package/dist/types/index.d.ts CHANGED
@@ -48,6 +48,8 @@ export interface EmbeddingConfig {
48
48
  modelName: string;
49
49
  modelProvider: string;
50
50
  vectorStoreType: "simple" | "postgres" | "weaviate";
51
+ documentStoreType?: "simple" | "postgres";
52
+ indexStoreType?: "simple" | "postgres";
51
53
  projectName: string;
52
54
  storagePath: string;
53
55
  splitIntoSentences: boolean;
package/dist/types/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC/B,CAAC,CAAC;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAGD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAGD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAGD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;IACpD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAGD,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,GAAG,UAAU,GAAG,UAAU,CAAC;IACxH,KAAK,EAAE,GAAG,CAAA;CACX;AAED,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,GAAG,CAAC;IACpB,cAAc,EAAE,GAAG,CAAC;CACrB"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,GAAG,CAAC;CACb;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAC/B,CAAC,CAAC;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAGD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAGD,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,IAAI,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;CACtB;AAGD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,QAAQ,GAAG,UAAU,GAAG,UAAU,CAAC;IACpD,iBAAiB,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IAC1C,cAAc,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,OAAO,CAAC;IAC5B,0BAA0B,EAAE,OAAO,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAGD,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,YAAY,GAAG,UAAU,GAAG,UAAU,CAAC;IACxH,KAAK,EAAE,GAAG,CAAA;CACX;AAED,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,GAAG,CAAC;IACpB,cAAc,EAAE,GAAG,CAAC;CACrB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@meaningfully/core",
3
- "version": "0.1.1",
3
+ "version": "0.1.2",
4
4
  "description": "Core functionality for meaningfully semantic search",
5
5
  "repo": "https://github.com/jeremybmerrill/meaningfully-core",
6
6
  "license": "MIT",
@@ -33,7 +33,8 @@
33
33
  "lodash": "^4.17.21",
34
34
  "natural": "^8.1.0",
35
35
  "openai": "^5.9.0",
36
- "papaparse": "^5.5.3"
36
+ "papaparse": "^5.5.3",
37
+ "pgvector": "^0.2.0"
37
38
  },
38
39
  "devDependencies": {
39
40
  "@types/node": "^20.5.6",
package/src/Meaningfully.ts CHANGED
@@ -22,12 +22,12 @@ export class MeaningfullyAPI {
22
22
  private storagePath: string;
23
23
  private clients: Clients;
24
24
 
25
- constructor({ storagePath, weaviateClient, metadataManager }: { storagePath: string, weaviateClient?: any, metadataManager: MetadataManager }) {
25
+ constructor({ storagePath, weaviateClient, postgresClient, metadataManager }: { storagePath: string, weaviateClient?: any, postgresClient?: any, metadataManager: MetadataManager }) {
26
26
  this.storagePath = storagePath;
27
27
  this.manager = metadataManager;
28
28
  this.clients = {
29
29
  weaviateClient: weaviateClient,
30
- postgresClient: null
30
+ postgresClient: postgresClient
31
31
  };
32
32
  }
33
33
 
@@ -58,8 +58,11 @@ export class MeaningfullyAPI {
58
58
  return { success: true };
59
59
  }
60
60
 
61
+ // TODO: this is an awful hack.
62
+ // I don't really want to have it be an argument to the constructor
63
+ // maybe I should make clients accept EITHER, not both?
61
64
  getVectorStoreType() {
62
- return this.clients.weaviateClient ? 'weaviate' : 'simple';
65
+ return this.clients.postgresClient ? 'postgres' : (this.clients.weaviateClient ? 'weaviate' : 'simple');
63
66
  }
64
67
 
65
68
  async generatePreviewData(data: DocumentSetParamsFilePath) {
@@ -79,7 +82,7 @@ export class MeaningfullyAPI {
79
82
  storagePath: this.storagePath,
80
83
  chunkSize: data.chunkSize,
81
84
  chunkOverlap: data.chunkOverlap
82
- });
85
+ });
83
86
  } catch (error) {
84
87
  throw error;
85
88
  }
@@ -171,6 +174,7 @@ export class MeaningfullyAPI {
171
174
 
172
175
  async getDocument(documentSetId: number, documentNodeId: string){
173
176
  const documentSet = await this.manager.getDocumentSet(documentSetId);
177
+ const settings = await this.manager.getSettings();
174
178
  if (!documentSet) {
175
179
  throw new Error('Document set not found');
176
180
  }
@@ -185,7 +189,7 @@ export class MeaningfullyAPI {
185
189
  storagePath: this.storagePath,
186
190
  chunkSize: 1024, // not actually used, we just re-use a config object that has this option
187
191
  chunkOverlap: 20, // not actually used, we just re-use a config object that has this option
188
- });
192
+ }, settings, this.clients);
189
193
  const document = await docStore.getNode(documentNodeId);
190
194
  if (!document) {
191
195
  throw new Error('Document not found');
package/src/api/__tests__/embedding.test.ts CHANGED
@@ -2,7 +2,7 @@
2
2
  import { describe, it, expect, vi, beforeEach } from 'vitest';
3
3
  import { createEmbeddings, previewResults, getDocStore, getIndex, search } from '../embedding.js';
4
4
  import { loadDocumentsFromCsv } from '../../services/csvLoader.js';
5
- import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getExistingDocStore } from '../../services/embeddings.js';
5
+ import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, getStorageContext } from '../../services/embeddings.js';
6
6
  import { MetadataMode } from 'llamaindex';
7
7
 
8
8
  // filepath: /Users/jeremybmerrill/code/meaningfully/src/main/api/embedding.test.ts
@@ -82,9 +82,9 @@ describe('embedding.ts', () => {
82
82
  describe('getDocStore', () => {
83
83
  it('should return existing doc store', async () => {
84
84
  const mockDocStore = 'docStore';
85
- getExistingDocStore.mockResolvedValue(mockDocStore);
85
+ getStorageContext.mockResolvedValue({ docStore: mockDocStore });
86
86
 
87
- const result = await getDocStore({});
87
+ const result = await getDocStore({}, {}, {});
88
88
 
89
89
  expect(result).toBe(mockDocStore);
90
90
  });
@@ -173,7 +173,7 @@ describe('embedding.ts', () => {
173
173
  describe('getDocStore', () => {
174
174
  it('should return existing doc store', async () => {
175
175
  const mockDocStore = 'docStore';
176
- getExistingDocStore.mockResolvedValue(mockDocStore);
176
+ getStorageContext.mockResolvedValue({ docStore: mockDocStore });
177
177
 
178
178
  const result = await getDocStore({});
179
179
 
package/src/api/embedding.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getExistingDocStore } from "../services/embeddings.js";
1
+ import { transformDocumentsToNodes, estimateCost, searchDocuments, getExistingVectorStoreIndex, persistNodes, persistDocuments, getStorageContext } from "../services/embeddings.js";
2
2
  import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types/index.js";
3
3
  import { loadDocumentsFromCsv } from "../services/csvLoader.js";
4
4
  import { MetadataMode } from "llamaindex";
@@ -97,8 +97,8 @@ export async function previewResults(
97
97
  }
98
98
  }
99
99
 
100
- export async function getDocStore(config: EmbeddingConfig) {
101
- return await getExistingDocStore(config);
100
+ export async function getDocStore(config: EmbeddingConfig, settings: Settings, clients: Clients) {
101
+ return (await getStorageContext(config, settings, clients)).docStore;
102
102
  }
103
103
 
104
104
  export async function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
package/src/services/embeddings.d.ts CHANGED
@@ -12,7 +12,6 @@ export declare function estimateCost(nodes: TextNode[], modelName: string): {
12
12
  pricePer1M: number;
13
13
  };
14
14
  export declare function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<VectorStoreIndex>;
15
- export declare function getExistingDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
16
15
  export declare function transformDocumentsToNodes(documents: Document[], config: EmbeddingConfig): Promise<TextNode<import("llamaindex").Metadata>[]>;
17
16
  export declare function getEmbedModel(config: EmbeddingConfig, settings: Settings): MockEmbedding | OpenAIEmbedding | OllamaEmbedding | MistralAIEmbedding | GeminiEmbedding;
18
17
  export declare function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext>;
package/src/services/embeddings.ts CHANGED
@@ -11,12 +11,15 @@ import {
11
11
  SimpleVectorStore,
12
12
  type StorageContext,
13
13
  Settings as LlamaindexSettings,
14
- SimpleDocumentStore
14
+ SimpleDocumentStore,
15
+ BaseDocumentStore,
16
+ BaseIndexStore,
17
+ SimpleIndexStore
15
18
  } from "llamaindex";
16
19
  import { OllamaEmbedding} from '@llamaindex/ollama'
17
20
  import { MistralAIEmbedding, MistralAIEmbeddingModelType } from '@llamaindex/mistral'
18
21
  import { GeminiEmbedding } from '@llamaindex/google'
19
- import { PGVectorStore } from '@llamaindex/postgres';
22
+ import { PGVectorStore, PostgresDocumentStore, PostgresIndexStore } from '@llamaindex/postgres';
20
23
  import { AzureOpenAIEmbedding } from "@llamaindex/azure";
21
24
  import { Sploder } from "./sploder.js";
22
25
  import { CustomSentenceSplitter } from "./sentenceSplitter.js";
@@ -97,79 +100,12 @@ export function estimateCost(nodes: TextNode[], modelName: string): {
97
100
  }
98
101
 
99
102
  export async function getExistingVectorStoreIndex(config: EmbeddingConfig, settings: Settings, clients: Clients) {
100
- const embedModel = getEmbedModel(config, settings);
101
- switch (config.vectorStoreType) {
102
- case "simple":
103
- const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName));
104
- const storageContext = await storageContextFromDefaults({
105
- persistDir: persistDir,
106
- });
107
- let vsi = await VectorStoreIndex.init({
108
- storageContext: storageContext,
109
- });
110
- vsi.embedModel = embedModel;
111
- return vsi;
112
-
113
- case "postgres":
114
- if (!clients.postgresClient) {
115
- throw new Error("Postgres client required but not provided");
116
- }
117
- const pgStore = new PGVectorStore({
118
- clientConfig: { connectionString: process.env.POSTGRES_CONNECTION_STRING },
119
- tableName: sanitizeProjectName(config.projectName),
120
- dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
121
- embeddingModel: embedModel
122
- });
123
- const pgStorageContext = await storageContextFromDefaults({
124
- vectorStores: { [ModalityType.TEXT]: pgStore },
125
- });
126
- return await VectorStoreIndex.init({
127
- storageContext: pgStorageContext,
128
- });
129
- case "weaviate":
130
- if (!clients.weaviateClient) {
131
- throw new Error("Weaviate client required but not provided");
132
- }
133
- const weaviateStore = new BatchingWeaviateVectorStore({
134
- indexName: capitalizeFirstLetter(sanitizeProjectName(config.projectName)),
135
- weaviateClient: clients.weaviateClient,
136
- embeddingModel: embedModel
137
- });
138
-
139
- // WeaviateVectorStore's getNodeSimilarity method looks for distance, but current weaviate provides score
140
- // (WeaviateVectorStore would get `score` if we were doing hybrid search)
141
- // Overwrite the private getNodeSimilarity method to use 'score' from metadata
142
- // @ts-ignore
143
- weaviateStore.getNodeSimilarity = (entry, _similarityKey = "score") => {
144
- return entry.metadata.score;
145
- }
146
-
147
- return await VectorStoreIndex.fromVectorStore(weaviateStore)
148
-
149
- default:
150
- throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
151
- }
152
- }
153
-
154
- export async function getExistingDocStore(config: EmbeddingConfig) {
155
- // switch (config.vectorStoreType) {
156
- // case "simple":
157
- const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
158
- const storageContext = await storageContextFromDefaults({
159
- persistDir: persistDir,
160
- });
161
- return storageContext.docStore;
162
-
163
- // case "postgres":
164
- // throw new Error(`Not yet implemented vector store type: ${config.vectorStoreType}`);
165
- // // return await createVectorStore(config);
166
- // default:
167
- // throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
168
- // }
103
+ const storageContext = await getStorageContext(config, settings, clients);
104
+ return await VectorStoreIndex.init({
105
+ storageContext: storageContext,
106
+ });
169
107
  }
170
108
 
171
-
172
-
173
109
  export async function transformDocumentsToNodes(
174
110
  documents: Document[],
175
111
  config: EmbeddingConfig,
@@ -249,12 +185,15 @@ export function getEmbedModel(
249
185
 
250
186
  export async function getStorageContext(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<StorageContext> {
251
187
  const vectorStore = await createVectorStore(config, settings, clients);
188
+ const docStore = await createDocumentStore(config, settings, clients); // new SimpleDocumentStore()
189
+ const indexStore = await createIndexStore(config, settings, clients);
252
190
  fs.mkdirSync(config.storagePath, { recursive: true });
253
191
  const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
254
192
  return await storageContextFromDefaults({
255
193
  persistDir: persistDir,
256
194
  vectorStores: {[ModalityType.TEXT]: vectorStore},
257
- docStore: new SimpleDocumentStore()
195
+ docStore: docStore,
196
+ indexStore: indexStore
258
197
  /*
259
198
  if docStore is created with a persist path (as it is by default in storageContextFromDefaults)
260
199
  then it will write to disk after every put(), which happens 2+ times per document.
@@ -273,8 +212,13 @@ export async function persistDocuments(documents: Document[], config: EmbeddingC
273
212
 
274
213
  // see comments in getStorageContext
275
214
  const persistDir = join(config.storagePath, sanitizeProjectName(config.projectName) );
276
- // @ts-ignore
277
- await (storageContext.docStore as SimpleDocumentStore).kvStore.persist(join(persistDir, "doc_store.json"));
215
+ if (storageContext.docStore instanceof SimpleDocumentStore) {
216
+ // @ts-ignore
217
+ await (storageContext.docStore as SimpleDocumentStore).kvStore.persist(join(persistDir, "doc_store.json"));
218
+ }else if (storageContext.docStore instanceof PostgresDocumentStore) {
219
+ // PostgresDocumentStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
220
+ console.log("Pretending to persist Postgres document store, but it actually persists automatically.");
221
+ }
278
222
 
279
223
  console.timeEnd("persistDocuments Run Time");
280
224
  }
@@ -303,11 +247,11 @@ export async function persistNodes(nodes: TextNode[], config: EmbeddingConfig, s
303
247
  // all the if statements are just type-checking boilerplate.
304
248
  // N.B. WeaviateVectorStore does not need to be explicitly persisted, so we don't include it in the OR conditional here..
305
249
  if (vectorStore) {
306
- if (vectorStore instanceof PGVectorStore || vectorStore instanceof SimpleVectorStore) {
250
+ if (vectorStore instanceof SimpleVectorStore) {
307
251
  await vectorStore.persist(join(config.storagePath, sanitizeProjectName(config.projectName), "vector_store.json"));
308
- } else if (vectorStore instanceof BatchingWeaviateVectorStore) {
252
+ } else if (vectorStore instanceof PGVectorStore || vectorStore instanceof BatchingWeaviateVectorStore) {
309
253
  // WeaviateVectorStore does not have a persist method, it persists automatically
310
- console.log("Pretending to persist Weaviate vector store, but it actually persists automatically.");
254
+ console.log("Pretending to persist Weaviate or Postgres vector store, but it actually persists automatically.");
311
255
  } else {
312
256
  throw new Error("Vector store does not support persist method");
313
257
  }
@@ -326,8 +270,8 @@ async function createVectorStore(config: EmbeddingConfig, settings: Settings, cl
326
270
  // otherwise it defaults to Ada.
327
271
  case "postgres":
328
272
  return new PGVectorStore({
329
- clientConfig: {connectionString: process.env.POSTGRES_CONNECTION_STRING},
330
- tableName: sanitizeProjectName(config.projectName),
273
+ client: clients.postgresClient,
274
+ tableName: "vecs_" + sanitizeProjectName(config.projectName),
331
275
  dimensions: MODEL_DIMENSIONS[config.modelName] || 1536, // default to 1536 if model not found
332
276
  embeddingModel: embeddingModel
333
277
  });
@@ -357,6 +301,38 @@ async function createVectorStore(config: EmbeddingConfig, settings: Settings, cl
357
301
  }
358
302
  }
359
303
 
304
+ async function createDocumentStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseDocumentStore> {
305
+ // we create the doc store without a persist path, so it doesn't write to disk after every put()
306
+ switch (config.documentStoreType || config.vectorStoreType) {
307
+ case "postgres":
308
+ return new PostgresDocumentStore({
309
+ client: clients.postgresClient,
310
+ tableName: "docs_" + sanitizeProjectName(config.projectName),
311
+ });
312
+ case "simple":
313
+ case "weaviate":
314
+ return new SimpleDocumentStore();
315
+ default:
316
+ throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
317
+ }
318
+ }
319
+
320
+ async function createIndexStore(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<BaseIndexStore> {
321
+ switch (config.documentStoreType || config.vectorStoreType) {
322
+ case "postgres":
323
+ return new PostgresIndexStore({
324
+ client: clients.postgresClient,
325
+ tableName: "idx_" + sanitizeProjectName(config.projectName),
326
+ });
327
+ case "simple":
328
+ case "weaviate":
329
+ return new SimpleIndexStore();
330
+ default:
331
+ throw new Error(`Unsupported vector store type: ${config.vectorStoreType}`);
332
+ }
333
+
334
+ }
335
+
360
336
  export async function searchDocuments(
361
337
  index: VectorStoreIndex,
362
338
  query: string,
package/src/types/index.ts CHANGED
@@ -57,6 +57,8 @@ export interface EmbeddingConfig {
57
57
  modelName: string;
58
58
  modelProvider: string
59
59
  vectorStoreType: "simple" | "postgres" | "weaviate";
60
+ documentStoreType?: "simple" | "postgres";
61
+ indexStoreType?: "simple" | "postgres";
60
62
  projectName: string;
61
63
  storagePath: string;
62
64
  splitIntoSentences: boolean;