@n8n/ai-utilities 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.d.ts +6 -0
- package/dist/cjs/index.js +18 -2
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/typecheck.tsbuildinfo +1 -1
- package/dist/cjs/utils/fromai-tool-factory.d.ts +14 -0
- package/dist/cjs/utils/fromai-tool-factory.js +45 -0
- package/dist/cjs/utils/fromai-tool-factory.js.map +1 -0
- package/dist/cjs/utils/shared-fields.d.ts +8 -0
- package/dist/cjs/utils/shared-fields.js +159 -0
- package/dist/cjs/utils/shared-fields.js.map +1 -0
- package/dist/cjs/utils/vector-store/MemoryManager/MemoryCalculator.d.ts +7 -0
- package/dist/cjs/utils/vector-store/MemoryManager/MemoryCalculator.js +60 -0
- package/dist/cjs/utils/vector-store/MemoryManager/MemoryCalculator.js.map +1 -0
- package/dist/cjs/utils/vector-store/MemoryManager/MemoryVectorStoreManager.d.ts +34 -0
- package/dist/cjs/utils/vector-store/MemoryManager/MemoryVectorStoreManager.js +186 -0
- package/dist/cjs/utils/vector-store/MemoryManager/MemoryVectorStoreManager.js.map +1 -0
- package/dist/cjs/utils/vector-store/MemoryManager/StoreCleanupService.d.ts +16 -0
- package/dist/cjs/utils/vector-store/MemoryManager/StoreCleanupService.js +105 -0
- package/dist/cjs/utils/vector-store/MemoryManager/StoreCleanupService.js.map +1 -0
- package/dist/cjs/utils/vector-store/MemoryManager/config.d.ts +4 -0
- package/dist/cjs/utils/vector-store/MemoryManager/config.js +48 -0
- package/dist/cjs/utils/vector-store/MemoryManager/config.js.map +1 -0
- package/dist/cjs/utils/vector-store/MemoryManager/types.d.ts +39 -0
- package/dist/cjs/utils/vector-store/MemoryManager/types.js +13 -0
- package/dist/cjs/utils/vector-store/MemoryManager/types.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/constants.d.ts +4 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/constants.js +55 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/constants.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/createVectorStoreNode.d.ts +18 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/createVectorStoreNode.js +294 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/createVectorStoreNode.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/index.d.ts +6 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/index.js +33 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/index.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/insertOperation.d.ts +5 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/insertOperation.js +48 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/insertOperation.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/loadOperation.d.ts +5 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/loadOperation.js +55 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/loadOperation.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolExecuteOperation.d.ts +5 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolExecuteOperation.js +74 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolExecuteOperation.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolOperation.d.ts +5 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolOperation.js +70 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolOperation.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveOperation.d.ts +5 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveOperation.js +39 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/retrieveOperation.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/updateOperation.d.ts +5 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/updateOperation.js +49 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/operations/updateOperation.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/types.d.ts +39 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/types.js +13 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/types.js.map +1 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/utils.d.ts +6 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/utils.js +30 -0
- package/dist/cjs/utils/vector-store/createVectorStoreNode/utils.js.map +1 -0
- package/dist/cjs/utils/vector-store/processDocuments.d.ts +25 -0
- package/dist/cjs/utils/vector-store/processDocuments.js +52 -0
- package/dist/cjs/utils/vector-store/processDocuments.js.map +1 -0
- package/dist/esm/index.d.ts +6 -0
- package/dist/esm/index.js +17 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/typecheck.tsbuildinfo +1 -1
- package/dist/esm/utils/fromai-tool-factory.d.ts +14 -0
- package/dist/esm/utils/fromai-tool-factory.js +35 -0
- package/dist/esm/utils/fromai-tool-factory.js.map +1 -0
- package/dist/esm/utils/shared-fields.d.ts +8 -0
- package/dist/esm/utils/shared-fields.js +149 -0
- package/dist/esm/utils/shared-fields.js.map +1 -0
- package/dist/esm/utils/vector-store/MemoryManager/MemoryCalculator.d.ts +7 -0
- package/dist/esm/utils/vector-store/MemoryManager/MemoryCalculator.js +50 -0
- package/dist/esm/utils/vector-store/MemoryManager/MemoryCalculator.js.map +1 -0
- package/dist/esm/utils/vector-store/MemoryManager/MemoryVectorStoreManager.d.ts +34 -0
- package/dist/esm/utils/vector-store/MemoryManager/MemoryVectorStoreManager.js +176 -0
- package/dist/esm/utils/vector-store/MemoryManager/MemoryVectorStoreManager.js.map +1 -0
- package/dist/esm/utils/vector-store/MemoryManager/StoreCleanupService.d.ts +16 -0
- package/dist/esm/utils/vector-store/MemoryManager/StoreCleanupService.js +95 -0
- package/dist/esm/utils/vector-store/MemoryManager/StoreCleanupService.js.map +1 -0
- package/dist/esm/utils/vector-store/MemoryManager/config.d.ts +4 -0
- package/dist/esm/utils/vector-store/MemoryManager/config.js +38 -0
- package/dist/esm/utils/vector-store/MemoryManager/config.js.map +1 -0
- package/dist/esm/utils/vector-store/MemoryManager/types.d.ts +39 -0
- package/dist/esm/utils/vector-store/MemoryManager/types.js +3 -0
- package/dist/esm/utils/vector-store/MemoryManager/types.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/constants.d.ts +4 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/constants.js +45 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/constants.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/createVectorStoreNode.d.ts +18 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/createVectorStoreNode.js +284 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/createVectorStoreNode.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/index.d.ts +6 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/index.js +23 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/index.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/insertOperation.d.ts +5 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/insertOperation.js +38 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/insertOperation.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/loadOperation.d.ts +5 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/loadOperation.js +45 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/loadOperation.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolExecuteOperation.d.ts +5 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolExecuteOperation.js +64 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolExecuteOperation.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolOperation.d.ts +5 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolOperation.js +60 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveAsToolOperation.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveOperation.d.ts +5 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveOperation.js +29 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/retrieveOperation.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/updateOperation.d.ts +5 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/updateOperation.js +39 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/operations/updateOperation.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/types.d.ts +39 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/types.js +3 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/types.js.map +1 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/utils.d.ts +6 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/utils.js +20 -0
- package/dist/esm/utils/vector-store/createVectorStoreNode/utils.js.map +1 -0
- package/dist/esm/utils/vector-store/processDocuments.d.ts +25 -0
- package/dist/esm/utils/vector-store/processDocuments.js +42 -0
- package/dist/esm/utils/vector-store/processDocuments.js.map +1 -0
- package/package.json +8 -5
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
(function (factory) {
|
|
2
|
+
if (typeof module === "object" && typeof module.exports === "object") {
|
|
3
|
+
var v = factory(require, exports);
|
|
4
|
+
if (v !== undefined) module.exports = v;
|
|
5
|
+
}
|
|
6
|
+
else if (typeof define === "function" && define.amd) {
|
|
7
|
+
define(["require", "exports"], factory);
|
|
8
|
+
}
|
|
9
|
+
})(function (require, exports) {
|
|
10
|
+
"use strict";
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.StoreCleanupService = void 0;
|
|
13
|
+
class StoreCleanupService {
|
|
14
|
+
constructor(maxMemorySizeBytes, inactiveTtlMs, vectorStores, storeMetadata, onCleanup) {
|
|
15
|
+
this.maxMemorySizeBytes = maxMemorySizeBytes;
|
|
16
|
+
this.inactiveTtlMs = inactiveTtlMs;
|
|
17
|
+
this.vectorStores = vectorStores;
|
|
18
|
+
this.storeMetadata = storeMetadata;
|
|
19
|
+
this.onCleanup = onCleanup;
|
|
20
|
+
this.oldestStoreKeys = [];
|
|
21
|
+
this.lastSortTime = 0;
|
|
22
|
+
this.CACHE_TTL_MS = 5000;
|
|
23
|
+
}
|
|
24
|
+
isStoreInactive(metadata) {
|
|
25
|
+
if (this.inactiveTtlMs <= 0) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const now = Date.now();
|
|
29
|
+
const lastAccessedTime = metadata.lastAccessed.getTime();
|
|
30
|
+
return now - lastAccessedTime > this.inactiveTtlMs;
|
|
31
|
+
}
|
|
32
|
+
cleanupInactiveStores() {
|
|
33
|
+
if (this.inactiveTtlMs <= 0) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
let freedBytes = 0;
|
|
37
|
+
const removedStores = [];
|
|
38
|
+
for (const [key, metadata] of this.storeMetadata.entries()) {
|
|
39
|
+
if (this.isStoreInactive(metadata)) {
|
|
40
|
+
this.vectorStores.delete(key);
|
|
41
|
+
freedBytes += metadata.size;
|
|
42
|
+
removedStores.push(key);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
for (const key of removedStores) {
|
|
46
|
+
this.storeMetadata.delete(key);
|
|
47
|
+
}
|
|
48
|
+
if (removedStores.length > 0) {
|
|
49
|
+
this.oldestStoreKeys = [];
|
|
50
|
+
this.onCleanup(removedStores, freedBytes, 'ttl');
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
cleanupOldestStores(requiredBytes) {
|
|
54
|
+
if (this.maxMemorySizeBytes <= 0) {
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
let currentMemoryUsage = 0;
|
|
58
|
+
for (const metadata of this.storeMetadata.values()) {
|
|
59
|
+
currentMemoryUsage += metadata.size;
|
|
60
|
+
}
|
|
61
|
+
this.cleanupInactiveStores();
|
|
62
|
+
currentMemoryUsage = 0;
|
|
63
|
+
for (const metadata of this.storeMetadata.values()) {
|
|
64
|
+
currentMemoryUsage += metadata.size;
|
|
65
|
+
}
|
|
66
|
+
if (currentMemoryUsage + requiredBytes <= this.maxMemorySizeBytes) {
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const now = Date.now();
|
|
70
|
+
if (this.oldestStoreKeys.length === 0 || now - this.lastSortTime > this.CACHE_TTL_MS) {
|
|
71
|
+
const stores = [];
|
|
72
|
+
for (const [key, metadata] of this.storeMetadata.entries()) {
|
|
73
|
+
stores.push([key, metadata.createdAt.getTime()]);
|
|
74
|
+
}
|
|
75
|
+
stores.sort((a, b) => a[1] - b[1]);
|
|
76
|
+
this.oldestStoreKeys = stores.map(([key]) => key);
|
|
77
|
+
this.lastSortTime = now;
|
|
78
|
+
}
|
|
79
|
+
let freedBytes = 0;
|
|
80
|
+
const removedStores = [];
|
|
81
|
+
for (const key of this.oldestStoreKeys) {
|
|
82
|
+
if (!this.storeMetadata.has(key))
|
|
83
|
+
continue;
|
|
84
|
+
if (currentMemoryUsage - freedBytes + requiredBytes <= this.maxMemorySizeBytes) {
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
const metadata = this.storeMetadata.get(key);
|
|
88
|
+
if (metadata) {
|
|
89
|
+
this.vectorStores.delete(key);
|
|
90
|
+
freedBytes += metadata.size;
|
|
91
|
+
removedStores.push(key);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
for (const key of removedStores) {
|
|
95
|
+
this.storeMetadata.delete(key);
|
|
96
|
+
}
|
|
97
|
+
if (removedStores.length > 0) {
|
|
98
|
+
this.oldestStoreKeys = this.oldestStoreKeys.filter((key) => !removedStores.includes(key));
|
|
99
|
+
this.onCleanup(removedStores, freedBytes, 'memory');
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
exports.StoreCleanupService = StoreCleanupService;
|
|
104
|
+
});
|
|
105
|
+
//# sourceMappingURL=StoreCleanupService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"StoreCleanupService.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/MemoryManager/StoreCleanupService.ts"],"names":[],"mappings":";;;;;;;;;;;;IAOA,MAAa,mBAAmB;QAQ/B,YACkB,kBAA0B,EAC1B,aAAqB,EACrB,YAA4C,EAC5C,aAA+C,EAC/C,SAIR;YARQ,uBAAkB,GAAlB,kBAAkB,CAAQ;YAC1B,kBAAa,GAAb,aAAa,CAAQ;YACrB,iBAAY,GAAZ,YAAY,CAAgC;YAC5C,kBAAa,GAAb,aAAa,CAAkC;YAC/C,cAAS,GAAT,SAAS,CAIjB;YAfF,oBAAe,GAAa,EAAE,CAAC;YAE/B,iBAAY,GAAG,CAAC,CAAC;YAER,iBAAY,GAAG,IAAI,CAAC;QAYlC,CAAC;QAKJ,eAAe,CAAC,QAA6B;YAE5C,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;gBAC7B,OAAO,KAAK,CAAC;YACd,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,gBAAgB,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YACzD,OAAO,GAAG,GAAG,gBAAgB,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,CAAC;QAKD,qBAAqB;YAEpB,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;gBAC7B,OAAO;YACR,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,MAAM,aAAa,GAAa,EAAE,CAAC;YAGnC,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5D,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAEpC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC9B,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC;oBAC5B,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACF,CAAC;YAGD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBACjC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;YAGD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;gBAC1B,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YAClD,CAAC;QACF,CAAC;QAKD,mBAAmB,CAAC,aAAqB;YAExC,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,EAAE,CAAC;gBAClC,OAAO;YACR,CAAC;YAGD,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpD,kBAAkB,IAAI,QAAQ,CAAC,IAAI,CAAC;YACrC,CAAC;YAGD,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAG7B,kBAAkB,GAAG,CAAC,CAAC;YACvB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpD,kBAAkB,IAAI,QAAQ,CAAC,IAAI,CAAC;YACrC,CAAC;YAGD,IAAI,kBAAkB,GAAG,aAAa,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACnE,OAAO;YACR,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAGvB,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBAEtF,MAAM,MAAM,GAA4B,EAAE,CAAC;gBAE3C,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC5D,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAClD,CAAC;gBAGD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAGnC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;gBAClD,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;YACzB,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,MAAM,aAAa,GAAa,EAAE,CAAC;YAGnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAExC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAG3C,IAAI,kBAAkB,GAAG,UAAU,GAAG,aAAa,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;oBAChF,MAAM;gBACP,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC7C,IAAI,QAAQ,EAAE,CAAC;oBACd,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC9B,UAAU,IAAI,QAAQ,CAAC,IAAI,CAAC;oBAC5B,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACF,CAAC;YAGD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBACjC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;YAGD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAE9B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1F,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;YACrD,CAAC;QACF,CAAC;KACD;IArJD,kDAqJC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
(function (factory) {
|
|
2
|
+
if (typeof module === "object" && typeof module.exports === "object") {
|
|
3
|
+
var v = factory(require, exports);
|
|
4
|
+
if (v !== undefined) module.exports = v;
|
|
5
|
+
}
|
|
6
|
+
else if (typeof define === "function" && define.amd) {
|
|
7
|
+
define(["require", "exports"], factory);
|
|
8
|
+
}
|
|
9
|
+
})(function (require, exports) {
|
|
10
|
+
"use strict";
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.getConfig = getConfig;
|
|
13
|
+
exports.mbToBytes = mbToBytes;
|
|
14
|
+
exports.hoursToMs = hoursToMs;
|
|
15
|
+
const DEFAULT_MAX_MEMORY_MB = -1;
|
|
16
|
+
const DEFAULT_INACTIVE_TTL_HOURS = -1;
|
|
17
|
+
function getConfig() {
|
|
18
|
+
let maxMemoryMB = DEFAULT_MAX_MEMORY_MB;
|
|
19
|
+
if (process.env.N8N_VECTOR_STORE_MAX_MEMORY) {
|
|
20
|
+
const parsed = parseInt(process.env.N8N_VECTOR_STORE_MAX_MEMORY, 10);
|
|
21
|
+
if (!isNaN(parsed)) {
|
|
22
|
+
maxMemoryMB = parsed;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
let ttlHours = DEFAULT_INACTIVE_TTL_HOURS;
|
|
26
|
+
if (process.env.N8N_VECTOR_STORE_TTL_HOURS) {
|
|
27
|
+
const parsed = parseInt(process.env.N8N_VECTOR_STORE_TTL_HOURS, 10);
|
|
28
|
+
if (!isNaN(parsed)) {
|
|
29
|
+
ttlHours = parsed;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
maxMemoryMB,
|
|
34
|
+
ttlHours,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
function mbToBytes(mb) {
|
|
38
|
+
if (mb <= 0)
|
|
39
|
+
return -1;
|
|
40
|
+
return mb * 1024 * 1024;
|
|
41
|
+
}
|
|
42
|
+
function hoursToMs(hours) {
|
|
43
|
+
if (hours <= 0)
|
|
44
|
+
return -1;
|
|
45
|
+
return hours * 60 * 60 * 1000;
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/MemoryManager/config.ts"],"names":[],"mappings":";;;;;;;;;;;IASA,8BAuBC;IAKD,8BAIC;IAKD,8BAIC;IA/CD,MAAM,qBAAqB,GAAG,CAAC,CAAC,CAAC;IACjC,MAAM,0BAA0B,GAAG,CAAC,CAAC,CAAC;IAKtC,SAAgB,SAAS;QAExB,IAAI,WAAW,GAAG,qBAAqB,CAAC;QACxC,IAAI,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;YACrE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpB,WAAW,GAAG,MAAM,CAAC;YACtB,CAAC;QACF,CAAC;QAGD,IAAI,QAAQ,GAAG,0BAA0B,CAAC;QAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpB,QAAQ,GAAG,MAAM,CAAC;YACnB,CAAC;QACF,CAAC;QAED,OAAO;YACN,WAAW;YACX,QAAQ;SACR,CAAC;IACH,CAAC;IAKD,SAAgB,SAAS,CAAC,EAAU;QAEnC,IAAI,EAAE,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC;QACvB,OAAO,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IAKD,SAAgB,SAAS,CAAC,KAAa;QAEtC,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC/B,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { MemoryVectorStore } from '@langchain/classic/vectorstores/memory';
|
|
2
|
+
import type { Document } from '@langchain/core/documents';
|
|
3
|
+
export interface MemoryVectorStoreConfig {
|
|
4
|
+
maxMemoryMB: number;
|
|
5
|
+
ttlHours: number;
|
|
6
|
+
}
|
|
7
|
+
export interface VectorStoreMetadata {
|
|
8
|
+
size: number;
|
|
9
|
+
createdAt: Date;
|
|
10
|
+
lastAccessed: Date;
|
|
11
|
+
}
|
|
12
|
+
export interface StoreStats {
|
|
13
|
+
sizeBytes: number;
|
|
14
|
+
sizeMB: number;
|
|
15
|
+
percentOfTotal: number;
|
|
16
|
+
vectors: number;
|
|
17
|
+
createdAt: string;
|
|
18
|
+
lastAccessed: string;
|
|
19
|
+
inactive?: boolean;
|
|
20
|
+
inactiveForHours?: number;
|
|
21
|
+
}
|
|
22
|
+
export interface VectorStoreStats {
|
|
23
|
+
totalSizeBytes: number;
|
|
24
|
+
totalSizeMB: number;
|
|
25
|
+
percentOfLimit: number;
|
|
26
|
+
maxMemoryMB: number;
|
|
27
|
+
storeCount: number;
|
|
28
|
+
inactiveStoreCount: number;
|
|
29
|
+
ttlHours: number;
|
|
30
|
+
stores: Record<string, StoreStats>;
|
|
31
|
+
}
|
|
32
|
+
export interface IMemoryCalculator {
|
|
33
|
+
estimateBatchSize(documents: Document[]): number;
|
|
34
|
+
calculateVectorStoreSize(vectorStore: MemoryVectorStore): number;
|
|
35
|
+
}
|
|
36
|
+
export interface IStoreCleanupService {
|
|
37
|
+
cleanupInactiveStores(): void;
|
|
38
|
+
cleanupOldestStores(requiredBytes: number): void;
|
|
39
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
(function (factory) {
|
|
2
|
+
if (typeof module === "object" && typeof module.exports === "object") {
|
|
3
|
+
var v = factory(require, exports);
|
|
4
|
+
if (v !== undefined) module.exports = v;
|
|
5
|
+
}
|
|
6
|
+
else if (typeof define === "function" && define.amd) {
|
|
7
|
+
define(["require", "exports"], factory);
|
|
8
|
+
}
|
|
9
|
+
})(function (require, exports) {
|
|
10
|
+
"use strict";
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
});
|
|
13
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/MemoryManager/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
(function (factory) {
|
|
2
|
+
if (typeof module === "object" && typeof module.exports === "object") {
|
|
3
|
+
var v = factory(require, exports);
|
|
4
|
+
if (v !== undefined) module.exports = v;
|
|
5
|
+
}
|
|
6
|
+
else if (typeof define === "function" && define.amd) {
|
|
7
|
+
define(["require", "exports", "n8n-workflow"], factory);
|
|
8
|
+
}
|
|
9
|
+
})(function (require, exports) {
|
|
10
|
+
"use strict";
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.OPERATION_MODE_DESCRIPTIONS = exports.DEFAULT_OPERATION_MODES = void 0;
|
|
13
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
14
|
+
exports.DEFAULT_OPERATION_MODES = [
|
|
15
|
+
'load',
|
|
16
|
+
'insert',
|
|
17
|
+
'retrieve',
|
|
18
|
+
'retrieve-as-tool',
|
|
19
|
+
];
|
|
20
|
+
exports.OPERATION_MODE_DESCRIPTIONS = [
|
|
21
|
+
{
|
|
22
|
+
name: 'Get Many',
|
|
23
|
+
value: 'load',
|
|
24
|
+
description: 'Get many ranked documents from vector store for query',
|
|
25
|
+
action: 'Get ranked documents from vector store',
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: 'Insert Documents',
|
|
29
|
+
value: 'insert',
|
|
30
|
+
description: 'Insert documents into vector store',
|
|
31
|
+
action: 'Add documents to vector store',
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
name: 'Retrieve Documents (As Vector Store for Chain/Tool)',
|
|
35
|
+
value: 'retrieve',
|
|
36
|
+
description: 'Retrieve documents from vector store to be used as vector store with AI nodes',
|
|
37
|
+
action: 'Retrieve documents for Chain/Tool as Vector Store',
|
|
38
|
+
outputConnectionType: n8n_workflow_1.NodeConnectionTypes.AiVectorStore,
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
name: 'Retrieve Documents (As Tool for AI Agent)',
|
|
42
|
+
value: 'retrieve-as-tool',
|
|
43
|
+
description: 'Retrieve documents from vector store to be used as tool with AI nodes',
|
|
44
|
+
action: 'Retrieve documents for AI Agent as Tool',
|
|
45
|
+
outputConnectionType: n8n_workflow_1.NodeConnectionTypes.AiTool,
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
name: 'Update Documents',
|
|
49
|
+
value: 'update',
|
|
50
|
+
description: 'Update documents in vector store by ID',
|
|
51
|
+
action: 'Update vector store documents',
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
});
|
|
55
|
+
//# sourceMappingURL=constants.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/createVectorStoreNode/constants.ts"],"names":[],"mappings":";;;;;;;;;;;;IAAA,+CAAmD;IAKtC,QAAA,uBAAuB,GAAwB;QAC3D,MAAM;QACN,QAAQ;QACR,UAAU;QACV,kBAAkB;KAClB,CAAC;IAEW,QAAA,2BAA2B,GAA2B;QAClE;YACC,IAAI,EAAE,UAAU;YAChB,KAAK,EAAE,MAAM;YACb,WAAW,EAAE,uDAAuD;YACpE,MAAM,EAAE,wCAAwC;SAChD;QACD;YACC,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,oCAAoC;YACjD,MAAM,EAAE,+BAA+B;SACvC;QACD;YACC,IAAI,EAAE,qDAAqD;YAC3D,KAAK,EAAE,UAAU;YACjB,WAAW,EAAE,+EAA+E;YAC5F,MAAM,EAAE,mDAAmD;YAC3D,oBAAoB,EAAE,kCAAmB,CAAC,aAAa;SACvD;QACD;YACC,IAAI,EAAE,2CAA2C;YACjD,KAAK,EAAE,kBAAkB;YACzB,WAAW,EAAE,uEAAuE;YACpF,MAAM,EAAE,yCAAyC;YACjD,oBAAoB,EAAE,kCAAmB,CAAC,MAAM;SAChD;QACD;YACC,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,wCAAwC;YACrD,MAAM,EAAE,+BAA+B;SACvC;KACD,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { VectorStore } from '@langchain/core/vectorstores';
|
|
2
|
+
import type { IExecuteFunctions, INodeExecutionData, INodeTypeDescription, SupplyData, ISupplyDataFunctions } from 'n8n-workflow';
|
|
3
|
+
import type { VectorStoreNodeConstructorArgs } from './types';
|
|
4
|
+
export declare const createVectorStoreNode: <T extends VectorStore = VectorStore>(args: VectorStoreNodeConstructorArgs<T>) => {
|
|
5
|
+
new (): {
|
|
6
|
+
description: INodeTypeDescription;
|
|
7
|
+
methods: {
|
|
8
|
+
listSearch?: {
|
|
9
|
+
[key: string]: (this: import("n8n-workflow").ILoadOptionsFunctions, filter?: string, paginationToken?: string) => Promise<import("n8n-workflow").INodeListSearchResult>;
|
|
10
|
+
};
|
|
11
|
+
actionHandler?: {
|
|
12
|
+
[functionName: string]: (this: import("n8n-workflow").ILoadOptionsFunctions, payload: import("n8n-workflow").IDataObject | string | undefined) => Promise<import("n8n-workflow").NodeParameterValueType>;
|
|
13
|
+
};
|
|
14
|
+
} | undefined;
|
|
15
|
+
execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]>;
|
|
16
|
+
supplyData(this: ISupplyDataFunctions, itemIndex: number): Promise<SupplyData>;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
(function (factory) {
|
|
2
|
+
if (typeof module === "object" && typeof module.exports === "object") {
|
|
3
|
+
var v = factory(require, exports);
|
|
4
|
+
if (v !== undefined) module.exports = v;
|
|
5
|
+
}
|
|
6
|
+
else if (typeof define === "function" && define.amd) {
|
|
7
|
+
define(["require", "exports", "n8n-workflow", "./operations", "./utils", "../../shared-fields"], factory);
|
|
8
|
+
}
|
|
9
|
+
})(function (require, exports) {
|
|
10
|
+
"use strict";
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.createVectorStoreNode = void 0;
|
|
13
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
14
|
+
const operations_1 = require("./operations");
|
|
15
|
+
const utils_1 = require("./utils");
|
|
16
|
+
const shared_fields_1 = require("../../shared-fields");
|
|
17
|
+
const ragStarterCallout = {
|
|
18
|
+
displayName: 'Tip: Get a feel for vector stores in n8n with our',
|
|
19
|
+
name: 'ragStarterCallout',
|
|
20
|
+
type: 'callout',
|
|
21
|
+
typeOptions: {
|
|
22
|
+
calloutAction: {
|
|
23
|
+
label: 'RAG starter template',
|
|
24
|
+
type: 'openSampleWorkflowTemplate',
|
|
25
|
+
templateId: 'rag-starter-template',
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
default: '',
|
|
29
|
+
};
|
|
30
|
+
const createVectorStoreNode = (args) => class VectorStoreNodeType {
|
|
31
|
+
constructor() {
|
|
32
|
+
this.description = {
|
|
33
|
+
displayName: args.meta.displayName,
|
|
34
|
+
name: args.meta.name,
|
|
35
|
+
description: args.meta.description,
|
|
36
|
+
icon: args.meta.icon,
|
|
37
|
+
iconColor: args.meta.iconColor,
|
|
38
|
+
group: ['transform'],
|
|
39
|
+
version: [1, 1.1, 1.2, 1.3],
|
|
40
|
+
defaults: {
|
|
41
|
+
name: args.meta.displayName,
|
|
42
|
+
},
|
|
43
|
+
codex: {
|
|
44
|
+
categories: args.meta.categories ?? ['AI'],
|
|
45
|
+
subcategories: args.meta.subcategories ?? {
|
|
46
|
+
AI: ['Vector Stores', 'Tools', 'Root Nodes'],
|
|
47
|
+
'Vector Stores': ['Other Vector Stores'],
|
|
48
|
+
Tools: ['Other Tools'],
|
|
49
|
+
},
|
|
50
|
+
resources: {
|
|
51
|
+
primaryDocumentation: [
|
|
52
|
+
{
|
|
53
|
+
url: args.meta.docsUrl,
|
|
54
|
+
},
|
|
55
|
+
],
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
builderHint: {
|
|
59
|
+
...args.meta.builderHint,
|
|
60
|
+
inputs: {
|
|
61
|
+
ai_embedding: { required: true },
|
|
62
|
+
ai_document: {
|
|
63
|
+
required: true,
|
|
64
|
+
displayOptions: { show: { mode: ['insert'] } },
|
|
65
|
+
},
|
|
66
|
+
ai_reranker: {
|
|
67
|
+
required: true,
|
|
68
|
+
displayOptions: {
|
|
69
|
+
show: { mode: ['load', 'retrieve', 'retrieve-as-tool'], useReranker: [true] },
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
},
|
|
74
|
+
credentials: args.meta.credentials,
|
|
75
|
+
inputs: `={{
|
|
76
|
+
((parameters) => {
|
|
77
|
+
const mode = parameters?.mode;
|
|
78
|
+
const useReranker = parameters?.useReranker;
|
|
79
|
+
const inputs = [{ displayName: "Embedding", type: "${n8n_workflow_1.NodeConnectionTypes.AiEmbedding}", required: true, maxConnections: 1}]
|
|
80
|
+
|
|
81
|
+
if (['load', 'retrieve', 'retrieve-as-tool'].includes(mode) && useReranker) {
|
|
82
|
+
inputs.push({ displayName: "Reranker", type: "${n8n_workflow_1.NodeConnectionTypes.AiReranker}", required: true, maxConnections: 1})
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (mode === 'retrieve-as-tool') {
|
|
86
|
+
return inputs;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (['insert', 'load', 'update'].includes(mode)) {
|
|
90
|
+
inputs.push({ displayName: "", type: "${n8n_workflow_1.NodeConnectionTypes.Main}"})
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (['insert'].includes(mode)) {
|
|
94
|
+
inputs.push({ displayName: "Document", type: "${n8n_workflow_1.NodeConnectionTypes.AiDocument}", required: true, maxConnections: 1})
|
|
95
|
+
}
|
|
96
|
+
return inputs
|
|
97
|
+
})($parameter)
|
|
98
|
+
}}`,
|
|
99
|
+
outputs: `={{
|
|
100
|
+
((parameters) => {
|
|
101
|
+
const mode = parameters?.mode ?? 'retrieve';
|
|
102
|
+
|
|
103
|
+
if (mode === 'retrieve-as-tool') {
|
|
104
|
+
return [{ displayName: "Tool", type: "${n8n_workflow_1.NodeConnectionTypes.AiTool}"}]
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (mode === 'retrieve') {
|
|
108
|
+
return [{ displayName: "Vector Store", type: "${n8n_workflow_1.NodeConnectionTypes.AiVectorStore}"}]
|
|
109
|
+
}
|
|
110
|
+
return [{ displayName: "", type: "${n8n_workflow_1.NodeConnectionTypes.Main}"}]
|
|
111
|
+
})($parameter)
|
|
112
|
+
}}`,
|
|
113
|
+
properties: [
|
|
114
|
+
ragStarterCallout,
|
|
115
|
+
{
|
|
116
|
+
displayName: 'Operation Mode',
|
|
117
|
+
name: 'mode',
|
|
118
|
+
type: 'options',
|
|
119
|
+
noDataExpression: true,
|
|
120
|
+
default: 'retrieve',
|
|
121
|
+
options: (0, utils_1.getOperationModeOptions)(args),
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
...(0, shared_fields_1.getConnectionHintNoticeField)([n8n_workflow_1.NodeConnectionTypes.AiRetriever]),
|
|
125
|
+
displayOptions: {
|
|
126
|
+
show: {
|
|
127
|
+
mode: ['retrieve'],
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
displayName: 'Name',
|
|
133
|
+
name: 'toolName',
|
|
134
|
+
type: 'string',
|
|
135
|
+
default: '',
|
|
136
|
+
required: true,
|
|
137
|
+
description: 'Name of the vector store',
|
|
138
|
+
placeholder: 'e.g. company_knowledge_base',
|
|
139
|
+
validateType: 'string-alphanumeric',
|
|
140
|
+
displayOptions: {
|
|
141
|
+
show: {
|
|
142
|
+
'@version': [{ _cnd: { lte: 1.2 } }],
|
|
143
|
+
mode: ['retrieve-as-tool'],
|
|
144
|
+
},
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
displayName: 'Description',
|
|
149
|
+
name: 'toolDescription',
|
|
150
|
+
type: 'string',
|
|
151
|
+
default: '',
|
|
152
|
+
required: true,
|
|
153
|
+
typeOptions: { rows: 2 },
|
|
154
|
+
description: 'Explain to the LLM what this tool does, a good, specific description would allow LLMs to produce expected results much more often',
|
|
155
|
+
placeholder: `e.g. ${args.meta.description}`,
|
|
156
|
+
displayOptions: {
|
|
157
|
+
show: {
|
|
158
|
+
mode: ['retrieve-as-tool'],
|
|
159
|
+
},
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
...args.sharedFields,
|
|
163
|
+
{
|
|
164
|
+
displayName: 'Embedding Batch Size',
|
|
165
|
+
name: 'embeddingBatchSize',
|
|
166
|
+
type: 'number',
|
|
167
|
+
default: 200,
|
|
168
|
+
description: 'Number of documents to embed in a single batch',
|
|
169
|
+
displayOptions: {
|
|
170
|
+
show: {
|
|
171
|
+
mode: ['insert'],
|
|
172
|
+
'@version': [{ _cnd: { gte: 1.1 } }],
|
|
173
|
+
},
|
|
174
|
+
},
|
|
175
|
+
},
|
|
176
|
+
...(0, utils_1.transformDescriptionForOperationMode)(args.insertFields ?? [], 'insert'),
|
|
177
|
+
{
|
|
178
|
+
displayName: 'Prompt',
|
|
179
|
+
name: 'prompt',
|
|
180
|
+
type: 'string',
|
|
181
|
+
default: '',
|
|
182
|
+
required: true,
|
|
183
|
+
description: 'Search prompt to retrieve matching documents from the vector store using similarity-based ranking',
|
|
184
|
+
displayOptions: {
|
|
185
|
+
show: {
|
|
186
|
+
mode: ['load'],
|
|
187
|
+
},
|
|
188
|
+
},
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
displayName: 'Limit',
|
|
192
|
+
name: 'topK',
|
|
193
|
+
type: 'number',
|
|
194
|
+
default: 4,
|
|
195
|
+
description: 'Number of top results to fetch from vector store',
|
|
196
|
+
displayOptions: {
|
|
197
|
+
show: {
|
|
198
|
+
mode: ['load', 'retrieve-as-tool'],
|
|
199
|
+
},
|
|
200
|
+
},
|
|
201
|
+
},
|
|
202
|
+
{
|
|
203
|
+
displayName: 'Include Metadata',
|
|
204
|
+
name: 'includeDocumentMetadata',
|
|
205
|
+
type: 'boolean',
|
|
206
|
+
default: true,
|
|
207
|
+
description: 'Whether or not to include document metadata',
|
|
208
|
+
displayOptions: {
|
|
209
|
+
show: {
|
|
210
|
+
mode: ['load', 'retrieve-as-tool'],
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
displayName: 'Rerank Results',
|
|
216
|
+
name: 'useReranker',
|
|
217
|
+
type: 'boolean',
|
|
218
|
+
default: false,
|
|
219
|
+
description: 'Whether or not to rerank results',
|
|
220
|
+
displayOptions: {
|
|
221
|
+
show: {
|
|
222
|
+
mode: ['load', 'retrieve', 'retrieve-as-tool'],
|
|
223
|
+
},
|
|
224
|
+
},
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
displayName: 'ID',
|
|
228
|
+
name: 'id',
|
|
229
|
+
type: 'string',
|
|
230
|
+
default: '',
|
|
231
|
+
required: true,
|
|
232
|
+
description: 'ID of an embedding entry',
|
|
233
|
+
displayOptions: {
|
|
234
|
+
show: {
|
|
235
|
+
mode: ['update'],
|
|
236
|
+
},
|
|
237
|
+
},
|
|
238
|
+
},
|
|
239
|
+
...(0, utils_1.transformDescriptionForOperationMode)(args.loadFields ?? [], [
|
|
240
|
+
'load',
|
|
241
|
+
'retrieve-as-tool',
|
|
242
|
+
]),
|
|
243
|
+
...(0, utils_1.transformDescriptionForOperationMode)(args.retrieveFields ?? [], 'retrieve'),
|
|
244
|
+
...(0, utils_1.transformDescriptionForOperationMode)(args.updateFields ?? [], 'update'),
|
|
245
|
+
],
|
|
246
|
+
};
|
|
247
|
+
this.methods = args.methods;
|
|
248
|
+
}
|
|
249
|
+
async execute() {
|
|
250
|
+
const mode = this.getNodeParameter('mode', 0);
|
|
251
|
+
const embeddings = (await this.getInputConnectionData(n8n_workflow_1.NodeConnectionTypes.AiEmbedding, 0));
|
|
252
|
+
if (mode === 'load') {
|
|
253
|
+
const items = this.getInputData(0);
|
|
254
|
+
const resultData = [];
|
|
255
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
256
|
+
const docs = await (0, operations_1.handleLoadOperation)(this, args, embeddings, itemIndex);
|
|
257
|
+
resultData.push(...docs);
|
|
258
|
+
}
|
|
259
|
+
return [resultData];
|
|
260
|
+
}
|
|
261
|
+
if (mode === 'insert') {
|
|
262
|
+
const resultData = await (0, operations_1.handleInsertOperation)(this, args, embeddings);
|
|
263
|
+
return [resultData];
|
|
264
|
+
}
|
|
265
|
+
if (mode === 'update') {
|
|
266
|
+
const resultData = await (0, operations_1.handleUpdateOperation)(this, args, embeddings);
|
|
267
|
+
return [resultData];
|
|
268
|
+
}
|
|
269
|
+
if (mode === 'retrieve-as-tool') {
|
|
270
|
+
const items = this.getInputData(0);
|
|
271
|
+
const resultData = [];
|
|
272
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
273
|
+
const docs = await (0, operations_1.handleRetrieveAsToolExecuteOperation)(this, args, embeddings, itemIndex);
|
|
274
|
+
resultData.push(...docs);
|
|
275
|
+
}
|
|
276
|
+
return [resultData];
|
|
277
|
+
}
|
|
278
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Only the "load", "update", "insert", and "retrieve-as-tool" operation modes are supported with execute');
|
|
279
|
+
}
|
|
280
|
+
async supplyData(itemIndex) {
|
|
281
|
+
const mode = this.getNodeParameter('mode', 0);
|
|
282
|
+
const embeddings = (await this.getInputConnectionData(n8n_workflow_1.NodeConnectionTypes.AiEmbedding, 0));
|
|
283
|
+
if (mode === 'retrieve') {
|
|
284
|
+
return await (0, operations_1.handleRetrieveOperation)(this, args, embeddings, itemIndex);
|
|
285
|
+
}
|
|
286
|
+
if (mode === 'retrieve-as-tool') {
|
|
287
|
+
return await (0, operations_1.handleRetrieveAsToolOperation)(this, args, embeddings, itemIndex);
|
|
288
|
+
}
|
|
289
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Only the "retrieve" and "retrieve-as-tool" operation mode is supported to supply data');
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
exports.createVectorStoreNode = createVectorStoreNode;
|
|
293
|
+
});
|
|
294
|
+
//# sourceMappingURL=createVectorStoreNode.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"createVectorStoreNode.js","sourceRoot":"","sources":["../../../../../src/utils/vector-store/createVectorStoreNode/createVectorStoreNode.ts"],"names":[],"mappings":";;;;;;;;;;;;IAEA,+CAAuE;IAYvE,6CAOsB;IAGtB,mCAAwF;IACxF,uDAAmE;IAEnE,MAAM,iBAAiB,GAAoB;QAC1C,WAAW,EAAE,mDAAmD;QAChE,IAAI,EAAE,mBAAmB;QACzB,IAAI,EAAE,SAAS;QACf,WAAW,EAAE;YACZ,aAAa,EAAE;gBACd,KAAK,EAAE,sBAAsB;gBAC7B,IAAI,EAAE,4BAA4B;gBAClC,UAAU,EAAE,sBAAsB;aAClC;SACD;QACD,OAAO,EAAE,EAAE;KACX,CAAC;IAMK,MAAM,qBAAqB,GAAG,CACpC,IAAuC,EACtC,EAAE,CACH,MAAM,mBAAmB;QAAzB;YACC,gBAAW,GAAyB;gBACnC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;gBACpB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;gBAClC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;gBACpB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS;gBAC9B,KAAK,EAAE,CAAC,WAAW,CAAC;gBAGpB,OAAO,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;gBAC3B,QAAQ,EAAE;oBACT,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;iBAC3B;gBACD,KAAK,EAAE;oBACN,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC;oBAC1C,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI;wBACzC,EAAE,EAAE,CAAC,eAAe,EAAE,OAAO,EAAE,YAAY,CAAC;wBAC5C,eAAe,EAAE,CAAC,qBAAqB,CAAC;wBACxC,KAAK,EAAE,CAAC,aAAa,CAAC;qBACtB;oBACD,SAAS,EAAE;wBACV,oBAAoB,EAAE;4BACrB;gCACC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO;6BACtB;yBACD;qBACD;iBACD;gBACD,WAAW,EAAE;oBACZ,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW;oBACxB,MAAM,EAAE;wBACP,YAAY,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;wBAChC,WAAW,EAAE;4BACZ,QAAQ,EAAE,IAAI;4BACd,cAAc,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE;yBAC9C;wBACD,WAAW,EAAE;4BACZ,QAAQ,EAAE,IAAI;4BACd,cAAc,EAAE;gCACf,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,kBAAkB,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE;6BAC7E;yBACD;qBACD;iBACD;gBACD,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW;gBAElC,MAAM,EAAE;;;;yDAI8C,kCAAmB,CAAC,WAAW;;;qDAGnC,kCAAmB,CAAC,UAAU;;;;;;;;6CAQtC,kCAAmB,CAAC,IAAI;;;;qDAIhB,kCAAmB,CAAC,UAAU;;;;KAI9E;gBACF,OAAO,EAAE;;;;;6CAKiC,kCAAmB,CAAC,MAAM;;;;qDAIlB,kCAAmB,CAAC,aAAa;;wCAE9C,kCAAmB,CAAC,IAAI;;KAE3D;gBACF,UAAU,EAAE;oBACX,iBAAiB;oBACjB;wBACC,WAAW,EAAE,gBAAgB;wBAC7B,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,SAAS;wBACf,gBAAgB,EAAE,IAAI;wBACtB,OAAO,EAAE,UAAU;wBACnB,OAAO,EAAE,IAAA,+BAAuB,EAAC,IAAI,CAAC;qBACtC;oBACD;wBACC,GAAG,IAAA,4CAA4B,EAAC,CAAC,kCAAmB,CAAC,WAAW,CAAC,CAAC;wBAClE,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,UAAU,CAAC;6BAClB;yBACD;qBACD;oBACD;wBACC,WAAW,EAAE,MAAM;wBACnB,IAAI,EAAE,UAAU;wBAChB,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,IAAI;wBACd,WAAW,EAAE,0BAA0B;wBACvC,WAAW,EAAE,6BAA6B;wBAC1C,YAAY,EAAE,qBAAqB;wBACnC,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gCACpC,IAAI,EAAE,CAAC,kBAAkB,CAAC;6BAC1B;yBACD;qBACD;oBACD;wBACC,WAAW,EAAE,aAAa;wBAC1B,IAAI,EAAE,iBAAiB;wBACvB,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,IAAI;wBACd,WAAW,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBACxB,WAAW,EACV,mIAAmI;wBACpI,WAAW,EAAE,QAAQ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;wBAC5C,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,kBAAkB,CAAC;6BAC1B;yBACD;qBACD;oBACD,GAAG,IAAI,CAAC,YAAY;oBACpB;wBACC,WAAW,EAAE,sBAAsB;wBACnC,IAAI,EAAE,oBAAoB;wBAC1B,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,GAAG;wBACZ,WAAW,EAAE,gDAAgD;wBAC7D,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,QAAQ,CAAC;gCAChB,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;6BACpC;yBACD;qBACD;oBACD,GAAG,IAAA,4CAAoC,EAAC,IAAI,CAAC,YAAY,IAAI,EAAE,EAAE,QAAQ,CAAC;oBAE1E;wBACC,WAAW,EAAE,QAAQ;wBACrB,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,IAAI;wBACd,WAAW,EACV,mGAAmG;wBACpG,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,MAAM,CAAC;6BACd;yBACD;qBACD;oBACD;wBACC,WAAW,EAAE,OAAO;wBACpB,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,CAAC;wBACV,WAAW,EAAE,kDAAkD;wBAC/D,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,MAAM,EAAE,kBAAkB,CAAC;6BAClC;yBACD;qBACD;oBACD;wBACC,WAAW,EAAE,kBAAkB;wBAC/B,IAAI,EAAE,yBAAyB;wBAC/B,IAAI,EAAE,SAAS;wBACf,OAAO,EAAE,IAAI;wBACb,WAAW,EAAE,6CAA6C;wBAC1D,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,MAAM,EAAE,kBAAkB,CAAC;6BAClC;yBACD;qBACD;oBACD;wBACC,WAAW,EAAE,gBAAgB;wBAC7B,IAAI,EAAE,aAAa;wBACnB,IAAI,EAAE,SAAS;wBACf,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,kCAAkC;wBAC/C,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,kBAAkB,CAAC;6BAC9C;yBACD;qBACD;oBAED;wBACC,WAAW,EAAE,IAAI;wBACjB,IAAI,EAAE,IAAI;wBACV,IAAI,EAAE,QAAQ;wBACd,OAAO,EAAE,EAAE;wBACX,QAAQ,EAAE,IAAI;wBACd,WAAW,EAAE,0BAA0B;wBACvC,cAAc,EAAE;4BACf,IAAI,EAAE;gCACL,IAAI,EAAE,CAAC,QAAQ,CAAC;6BAChB;yBACD;qBACD;oBACD,GAAG,IAAA,4CAAoC,EAAC,IAAI,CAAC,UAAU,IAAI,EAAE,EAAE;wBAC9D,MAAM;wBACN,kBAAkB;qBAClB,CAAC;oBACF,GAAG,IAAA,4CAAoC,EAAC,IAAI,CAAC,cAAc,IAAI,EAAE,EAAE,UAAU,CAAC;oBAC9E,GAAG,IAAA,4CAAoC,EAAC,IAAI,CAAC,YAAY,IAAI,EAAE,EAAE,QAAQ,CAAC;iBAC1E;aACD,CAAC;YAEF,YAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAuFxB,CAAC;QAjFA,KAAK,CAAC,OAAO;YACZ,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAsB,CAAC;YAEnE,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,sBAAsB,CACpD,kCAAmB,CAAC,WAAW,EAC/B,CAAC,CACD,CAAe,CAAC;YAGjB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,UAAU,GAAG,EAAE,CAAC;gBAEtB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;oBAC/D,MAAM,IAAI,GAAG,MAAM,IAAA,gCAAmB,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC1E,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gBAC1B,CAAC;gBAED,OAAO,CAAC,UAAU,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;gBACvE,OAAO,CAAC,UAAU,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,IAAA,kCAAqB,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;gBACvE,OAAO,CAAC,UAAU,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,UAAU,GAAG,EAAE,CAAC;gBAEtB,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;oBAC/D,MAAM,IAAI,GAAG,MAAM,IAAA,iDAAoC,EACtD,IAAI,EACJ,IAAI,EACJ,UAAU,EACV,SAAS,CACT,CAAC;oBACF,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gBAC1B,CAAC;gBAED,OAAO,CAAC,UAAU,CAAC,CAAC;YACrB,CAAC;YAED,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,EAAE,EACd,wGAAwG,CACxG,CAAC;QACH,CAAC;QAMD,KAAK,CAAC,UAAU,CAA6B,SAAiB;YAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAsB,CAAC;YAGnE,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,sBAAsB,CACpD,kCAAmB,CAAC,WAAW,EAC/B,CAAC,CACD,CAAe,CAAC;YAGjB,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;gBACzB,OAAO,MAAM,IAAA,oCAAuB,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YACzE,CAAC;YAED,IAAI,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACjC,OAAO,MAAM,IAAA,0CAA6B,EAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;YAC/E,CAAC;YAED,MAAM,IAAI,iCAAkB,CAC3B,IAAI,CAAC,OAAO,EAAE,EACd,uFAAuF,CACvF,CAAC;QACH,CAAC;KACD,CAAC;IA1TU,QAAA,qBAAqB,yBA0T/B"}
|