@cmdoss/memwal-sdk 0.6.2 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. package/ARCHITECTURE.md +547 -547
  2. package/BENCHMARKS.md +238 -238
  3. package/README.md +310 -181
  4. package/dist/ai-sdk/tools.d.ts +2 -2
  5. package/dist/ai-sdk/tools.js +2 -2
  6. package/dist/client/ClientMemoryManager.js +2 -2
  7. package/dist/client/ClientMemoryManager.js.map +1 -1
  8. package/dist/client/PersonalDataWallet.d.ts.map +1 -1
  9. package/dist/client/SimplePDWClient.d.ts +29 -1
  10. package/dist/client/SimplePDWClient.d.ts.map +1 -1
  11. package/dist/client/SimplePDWClient.js +45 -13
  12. package/dist/client/SimplePDWClient.js.map +1 -1
  13. package/dist/client/namespaces/EmbeddingsNamespace.d.ts +1 -1
  14. package/dist/client/namespaces/EmbeddingsNamespace.js +1 -1
  15. package/dist/client/namespaces/MemoryNamespace.d.ts +31 -0
  16. package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
  17. package/dist/client/namespaces/MemoryNamespace.js +272 -39
  18. package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
  19. package/dist/client/namespaces/consolidated/AINamespace.d.ts +2 -2
  20. package/dist/client/namespaces/consolidated/AINamespace.js +2 -2
  21. package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts +12 -2
  22. package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
  23. package/dist/client/namespaces/consolidated/BlockchainNamespace.js +62 -4
  24. package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
  25. package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +67 -2
  26. package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
  27. package/dist/client/namespaces/consolidated/StorageNamespace.js +549 -16
  28. package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
  29. package/dist/config/ConfigurationHelper.js +61 -61
  30. package/dist/config/defaults.js +2 -2
  31. package/dist/config/defaults.js.map +1 -1
  32. package/dist/graph/GraphService.js +21 -21
  33. package/dist/graph/GraphService.js.map +1 -1
  34. package/dist/index.d.ts +3 -1
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/index.js +3 -1
  37. package/dist/index.js.map +1 -1
  38. package/dist/infrastructure/seal/EncryptionService.d.ts +9 -5
  39. package/dist/infrastructure/seal/EncryptionService.d.ts.map +1 -1
  40. package/dist/infrastructure/seal/EncryptionService.js +37 -15
  41. package/dist/infrastructure/seal/EncryptionService.js.map +1 -1
  42. package/dist/infrastructure/seal/SealService.d.ts +13 -5
  43. package/dist/infrastructure/seal/SealService.d.ts.map +1 -1
  44. package/dist/infrastructure/seal/SealService.js +36 -34
  45. package/dist/infrastructure/seal/SealService.js.map +1 -1
  46. package/dist/langchain/createPDWRAG.js +30 -30
  47. package/dist/retrieval/MemoryDecryptionPipeline.d.ts.map +1 -1
  48. package/dist/retrieval/MemoryDecryptionPipeline.js +2 -1
  49. package/dist/retrieval/MemoryDecryptionPipeline.js.map +1 -1
  50. package/dist/retrieval/MemoryRetrievalService.d.ts +31 -0
  51. package/dist/retrieval/MemoryRetrievalService.d.ts.map +1 -1
  52. package/dist/retrieval/MemoryRetrievalService.js +44 -4
  53. package/dist/retrieval/MemoryRetrievalService.js.map +1 -1
  54. package/dist/services/CapabilityService.d.ts.map +1 -1
  55. package/dist/services/CapabilityService.js +30 -14
  56. package/dist/services/CapabilityService.js.map +1 -1
  57. package/dist/services/CrossContextPermissionService.d.ts.map +1 -1
  58. package/dist/services/CrossContextPermissionService.js +9 -7
  59. package/dist/services/CrossContextPermissionService.js.map +1 -1
  60. package/dist/services/EmbeddingService.d.ts +28 -1
  61. package/dist/services/EmbeddingService.d.ts.map +1 -1
  62. package/dist/services/EmbeddingService.js +54 -0
  63. package/dist/services/EmbeddingService.js.map +1 -1
  64. package/dist/services/EncryptionService.d.ts.map +1 -1
  65. package/dist/services/EncryptionService.js +6 -5
  66. package/dist/services/EncryptionService.js.map +1 -1
  67. package/dist/services/GeminiAIService.js +309 -309
  68. package/dist/services/IndexManager.d.ts +5 -1
  69. package/dist/services/IndexManager.d.ts.map +1 -1
  70. package/dist/services/IndexManager.js +17 -40
  71. package/dist/services/IndexManager.js.map +1 -1
  72. package/dist/services/QueryService.js +1 -1
  73. package/dist/services/QueryService.js.map +1 -1
  74. package/dist/services/StorageService.d.ts +11 -0
  75. package/dist/services/StorageService.d.ts.map +1 -1
  76. package/dist/services/StorageService.js +73 -10
  77. package/dist/services/StorageService.js.map +1 -1
  78. package/dist/services/TransactionService.d.ts +20 -0
  79. package/dist/services/TransactionService.d.ts.map +1 -1
  80. package/dist/services/TransactionService.js +43 -0
  81. package/dist/services/TransactionService.js.map +1 -1
  82. package/dist/services/ViewService.js +2 -2
  83. package/dist/services/ViewService.js.map +1 -1
  84. package/dist/services/storage/QuiltBatchManager.d.ts +101 -1
  85. package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
  86. package/dist/services/storage/QuiltBatchManager.js +410 -20
  87. package/dist/services/storage/QuiltBatchManager.js.map +1 -1
  88. package/dist/services/storage/index.d.ts +1 -1
  89. package/dist/services/storage/index.d.ts.map +1 -1
  90. package/dist/services/storage/index.js.map +1 -1
  91. package/dist/utils/LRUCache.d.ts +106 -0
  92. package/dist/utils/LRUCache.d.ts.map +1 -0
  93. package/dist/utils/LRUCache.js +281 -0
  94. package/dist/utils/LRUCache.js.map +1 -0
  95. package/dist/utils/index.d.ts +1 -0
  96. package/dist/utils/index.d.ts.map +1 -1
  97. package/dist/utils/index.js +2 -0
  98. package/dist/utils/index.js.map +1 -1
  99. package/dist/utils/memoryIndexOnChain.d.ts +212 -0
  100. package/dist/utils/memoryIndexOnChain.d.ts.map +1 -0
  101. package/dist/utils/memoryIndexOnChain.js +312 -0
  102. package/dist/utils/memoryIndexOnChain.js.map +1 -0
  103. package/dist/utils/rebuildIndexNode.d.ts +29 -0
  104. package/dist/utils/rebuildIndexNode.d.ts.map +1 -1
  105. package/dist/utils/rebuildIndexNode.js +366 -98
  106. package/dist/utils/rebuildIndexNode.js.map +1 -1
  107. package/dist/vector/HnswWasmService.d.ts +20 -5
  108. package/dist/vector/HnswWasmService.d.ts.map +1 -1
  109. package/dist/vector/HnswWasmService.js +73 -40
  110. package/dist/vector/HnswWasmService.js.map +1 -1
  111. package/dist/vector/IHnswService.d.ts +10 -1
  112. package/dist/vector/IHnswService.d.ts.map +1 -1
  113. package/dist/vector/IHnswService.js.map +1 -1
  114. package/dist/vector/NodeHnswService.d.ts +16 -0
  115. package/dist/vector/NodeHnswService.d.ts.map +1 -1
  116. package/dist/vector/NodeHnswService.js +84 -5
  117. package/dist/vector/NodeHnswService.js.map +1 -1
  118. package/dist/vector/createHnswService.d.ts +1 -1
  119. package/dist/vector/createHnswService.js +1 -1
  120. package/dist/vector/index.d.ts +1 -1
  121. package/dist/vector/index.js +1 -1
  122. package/package.json +157 -157
  123. package/src/access/PermissionService.ts +635 -635
  124. package/src/aggregation/AggregationService.ts +389 -389
  125. package/src/ai-sdk/PDWVectorStore.ts +715 -715
  126. package/src/ai-sdk/index.ts +65 -65
  127. package/src/ai-sdk/tools.ts +460 -460
  128. package/src/ai-sdk/types.ts +404 -404
  129. package/src/batch/BatchManager.ts +597 -597
  130. package/src/batch/BatchingService.ts +429 -429
  131. package/src/batch/MemoryProcessingCache.ts +492 -492
  132. package/src/batch/index.ts +30 -30
  133. package/src/browser.ts +200 -200
  134. package/src/client/ClientMemoryManager.ts +987 -987
  135. package/src/client/PersonalDataWallet.ts +345 -345
  136. package/src/client/SimplePDWClient.ts +1289 -1222
  137. package/src/client/factory.ts +154 -154
  138. package/src/client/namespaces/AnalyticsNamespace.ts +377 -377
  139. package/src/client/namespaces/BatchNamespace.ts +356 -356
  140. package/src/client/namespaces/CacheNamespace.ts +123 -123
  141. package/src/client/namespaces/CapabilityNamespace.ts +217 -217
  142. package/src/client/namespaces/ClassifyNamespace.ts +169 -169
  143. package/src/client/namespaces/ContextNamespace.ts +297 -297
  144. package/src/client/namespaces/EmbeddingsNamespace.ts +99 -99
  145. package/src/client/namespaces/EncryptionNamespace.ts +221 -221
  146. package/src/client/namespaces/GraphNamespace.ts +468 -468
  147. package/src/client/namespaces/IndexNamespace.ts +361 -361
  148. package/src/client/namespaces/MemoryNamespace.ts +1422 -1135
  149. package/src/client/namespaces/PermissionsNamespace.ts +254 -254
  150. package/src/client/namespaces/PipelineNamespace.ts +220 -220
  151. package/src/client/namespaces/SearchNamespace.ts +1049 -1049
  152. package/src/client/namespaces/StorageNamespace.ts +458 -458
  153. package/src/client/namespaces/TxNamespace.ts +260 -260
  154. package/src/client/namespaces/WalletNamespace.ts +243 -243
  155. package/src/client/namespaces/consolidated/AINamespace.ts +449 -449
  156. package/src/client/namespaces/consolidated/BlockchainNamespace.ts +607 -546
  157. package/src/client/namespaces/consolidated/SecurityNamespace.ts +648 -648
  158. package/src/client/namespaces/consolidated/StorageNamespace.ts +1141 -497
  159. package/src/client/namespaces/consolidated/index.ts +39 -39
  160. package/src/client/signers/KeypairSigner.ts +108 -108
  161. package/src/client/signers/UnifiedSigner.ts +110 -110
  162. package/src/client/signers/WalletAdapterSigner.ts +159 -159
  163. package/src/client/signers/index.ts +26 -26
  164. package/src/config/ConfigurationHelper.ts +412 -412
  165. package/src/config/defaults.ts +51 -51
  166. package/src/config/index.ts +8 -8
  167. package/src/config/validation.ts +70 -70
  168. package/src/core/index.ts +14 -14
  169. package/src/core/interfaces/IService.ts +307 -307
  170. package/src/core/interfaces/index.ts +8 -8
  171. package/src/core/types/capability.ts +297 -297
  172. package/src/core/types/index.ts +870 -870
  173. package/src/core/types/wallet.ts +270 -270
  174. package/src/core/types.ts +9 -9
  175. package/src/core/wallet.ts +222 -222
  176. package/src/embedding/index.ts +19 -19
  177. package/src/embedding/types.ts +357 -357
  178. package/src/errors/index.ts +602 -602
  179. package/src/errors/recovery.ts +461 -461
  180. package/src/errors/validation.ts +567 -567
  181. package/src/generated/pdw/capability.ts +319 -319
  182. package/src/graph/GraphService.ts +887 -887
  183. package/src/graph/KnowledgeGraphManager.ts +728 -728
  184. package/src/graph/index.ts +25 -25
  185. package/src/index.ts +498 -474
  186. package/src/infrastructure/index.ts +22 -22
  187. package/src/infrastructure/seal/EncryptionService.ts +628 -603
  188. package/src/infrastructure/seal/SealService.ts +613 -615
  189. package/src/infrastructure/seal/index.ts +9 -9
  190. package/src/infrastructure/sui/BlockchainManager.ts +627 -627
  191. package/src/infrastructure/sui/SuiService.ts +888 -888
  192. package/src/infrastructure/sui/index.ts +9 -9
  193. package/src/infrastructure/walrus/StorageManager.ts +604 -604
  194. package/src/infrastructure/walrus/WalrusStorageService.ts +612 -612
  195. package/src/infrastructure/walrus/index.ts +9 -9
  196. package/src/langchain/PDWEmbeddings.ts +145 -145
  197. package/src/langchain/PDWVectorStore.ts +456 -456
  198. package/src/langchain/createPDWRAG.ts +303 -303
  199. package/src/langchain/index.ts +47 -47
  200. package/src/permissions/ConsentRepository.browser.ts +249 -249
  201. package/src/permissions/ConsentRepository.ts +364 -364
  202. package/src/pipeline/MemoryPipeline.ts +862 -862
  203. package/src/pipeline/PipelineManager.ts +683 -683
  204. package/src/pipeline/index.ts +26 -26
  205. package/src/retrieval/AdvancedSearchService.ts +629 -629
  206. package/src/retrieval/MemoryAnalyticsService.ts +711 -711
  207. package/src/retrieval/MemoryDecryptionPipeline.ts +825 -824
  208. package/src/retrieval/MemoryRetrievalService.ts +904 -830
  209. package/src/retrieval/index.ts +42 -42
  210. package/src/services/BatchService.ts +352 -352
  211. package/src/services/CapabilityService.ts +464 -448
  212. package/src/services/ClassifierService.ts +465 -465
  213. package/src/services/CrossContextPermissionService.ts +486 -484
  214. package/src/services/EmbeddingService.ts +771 -706
  215. package/src/services/EncryptionService.ts +712 -711
  216. package/src/services/GeminiAIService.ts +753 -753
  217. package/src/services/IndexManager.ts +977 -1004
  218. package/src/services/MemoryIndexService.ts +1003 -1003
  219. package/src/services/MemoryService.ts +369 -369
  220. package/src/services/QueryService.ts +890 -890
  221. package/src/services/StorageService.ts +1182 -1111
  222. package/src/services/TransactionService.ts +838 -790
  223. package/src/services/VectorService.ts +462 -462
  224. package/src/services/ViewService.ts +484 -484
  225. package/src/services/index.ts +25 -25
  226. package/src/services/storage/BlobAttributesManager.ts +333 -333
  227. package/src/services/storage/KnowledgeGraphManager.ts +425 -425
  228. package/src/services/storage/MemorySearchManager.ts +387 -387
  229. package/src/services/storage/QuiltBatchManager.ts +1130 -660
  230. package/src/services/storage/WalrusMetadataManager.ts +268 -268
  231. package/src/services/storage/WalrusStorageManager.ts +287 -287
  232. package/src/services/storage/index.ts +57 -52
  233. package/src/types/index.ts +13 -13
  234. package/src/utils/LRUCache.ts +378 -0
  235. package/src/utils/index.ts +76 -68
  236. package/src/utils/memoryIndexOnChain.ts +507 -0
  237. package/src/utils/rebuildIndex.ts +290 -290
  238. package/src/utils/rebuildIndexNode.ts +771 -424
  239. package/src/vector/BrowserHnswIndexService.ts +758 -758
  240. package/src/vector/HnswWasmService.ts +731 -679
  241. package/src/vector/IHnswService.ts +233 -224
  242. package/src/vector/NodeHnswService.ts +833 -735
  243. package/src/vector/VectorManager.ts +478 -478
  244. package/src/vector/createHnswService.ts +135 -135
  245. package/src/vector/index.ts +56 -56
  246. package/src/wallet/ContextWalletService.ts +656 -656
  247. package/src/wallet/MainWalletService.ts +317 -317
@@ -1,430 +1,430 @@
1
- /**
2
- * BatchingService - Intelligent Batch Processing & Caching
3
- *
4
- * Provides advanced batching capabilities with intelligent scheduling,
5
- * cache management, and performance optimization for the PDW SDK.
6
- */
7
-
8
- import { BatchConfig, BatchJob, BatchStats } from '../embedding/types';
9
-
10
- export interface BatchItem<T = any> {
11
- id: string;
12
- data: T;
13
- timestamp: Date;
14
- priority?: number;
15
- metadata?: any;
16
- }
17
-
18
- export interface BatchProcessor<T = any> {
19
- process(items: BatchItem<T>[]): Promise<void>;
20
- }
21
-
22
- export interface CacheConfig {
23
- maxSize?: number;
24
- ttlMs?: number;
25
- cleanupIntervalMs?: number;
26
- enableMetrics?: boolean;
27
- }
28
-
29
- export interface CacheItem<T = any> {
30
- key: string;
31
- value: T;
32
- createdAt: Date;
33
- accessedAt: Date;
34
- accessCount: number;
35
- size?: number;
36
- }
37
-
38
- export interface CacheMetrics {
39
- totalItems: number;
40
- totalSize: number;
41
- hitCount: number;
42
- missCount: number;
43
- hitRate: number;
44
- averageAccessCount: number;
45
- oldestItem?: Date;
46
- newestItem?: Date;
47
- }
48
-
49
- /**
50
- * Advanced batching service with intelligent scheduling and caching
51
- */
52
- export class BatchingService<T = any> {
53
- private batches = new Map<string, BatchItem<T>[]>();
54
- private processors = new Map<string, BatchProcessor<T>>();
55
- private timers = new Map<string, NodeJS.Timeout>();
56
- private cache = new Map<string, CacheItem<T>>();
57
- private metrics = {
58
- hitCount: 0,
59
- missCount: 0,
60
- totalProcessed: 0,
61
- averageProcessingTime: 0
62
- };
63
-
64
- private readonly batchConfig: Required<BatchConfig>;
65
- private readonly cacheConfig: Required<CacheConfig>;
66
- private cleanupTimer?: NodeJS.Timeout;
67
-
68
- constructor(
69
- batchConfig: Partial<BatchConfig> = {},
70
- cacheConfig: Partial<CacheConfig> = {}
71
- ) {
72
- this.batchConfig = {
73
- maxBatchSize: batchConfig.maxBatchSize || 50,
74
- batchDelayMs: batchConfig.batchDelayMs || 5000,
75
- maxCacheSize: batchConfig.maxCacheSize || 1000,
76
- cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
77
- };
78
-
79
- this.cacheConfig = {
80
- maxSize: cacheConfig.maxSize || this.batchConfig.maxCacheSize,
81
- ttlMs: cacheConfig.ttlMs || this.batchConfig.cacheTtlMs,
82
- cleanupIntervalMs: cacheConfig.cleanupIntervalMs || 5 * 60 * 1000, // 5 minutes
83
- enableMetrics: cacheConfig.enableMetrics !== false
84
- };
85
-
86
- this.startCacheCleanup();
87
- }
88
-
89
- /**
90
- * Register a batch processor for a specific batch type
91
- */
92
- registerProcessor(batchType: string, processor: BatchProcessor<T>): void {
93
- this.processors.set(batchType, processor);
94
- }
95
-
96
- /**
97
- * Add item to batch for processing
98
- */
99
- addToBatch(
100
- batchType: string,
101
- item: BatchItem<T>,
102
- options: {
103
- forceImmediate?: boolean;
104
- highPriority?: boolean;
105
- } = {}
106
- ): void {
107
- // Get or create batch
108
- let batch = this.batches.get(batchType);
109
- if (!batch) {
110
- batch = [];
111
- this.batches.set(batchType, batch);
112
- }
113
-
114
- // Add item to batch
115
- batch.push({
116
- ...item,
117
- priority: options.highPriority ? 1 : (item.priority || 0)
118
- });
119
-
120
- // Sort by priority (higher priority first)
121
- batch.sort((a, b) => (b.priority || 0) - (a.priority || 0));
122
-
123
- console.debug(`Added item to batch ${batchType}. Size: ${batch.length}`);
124
-
125
- // Schedule processing
126
- this.scheduleBatchProcessing(batchType, options.forceImmediate);
127
- }
128
-
129
- /**
130
- * Process batch immediately
131
- */
132
- async processBatchNow(batchType: string): Promise<void> {
133
- const batch = this.batches.get(batchType);
134
- if (!batch || batch.length === 0) {
135
- return;
136
- }
137
-
138
- const processor = this.processors.get(batchType);
139
- if (!processor) {
140
- console.warn(`No processor registered for batch type: ${batchType}`);
141
- return;
142
- }
143
-
144
- const startTime = Date.now();
145
-
146
- try {
147
- console.log(`Processing batch ${batchType} with ${batch.length} items`);
148
-
149
- // Process the batch
150
- await processor.process([...batch]);
151
-
152
- // Clear batch and timer
153
- this.batches.delete(batchType);
154
- const timer = this.timers.get(batchType);
155
- if (timer) {
156
- clearTimeout(timer);
157
- this.timers.delete(batchType);
158
- }
159
-
160
- // Update metrics
161
- this.metrics.totalProcessed += batch.length;
162
- const processingTime = Date.now() - startTime;
163
- this.metrics.averageProcessingTime =
164
- (this.metrics.averageProcessingTime + processingTime) / 2;
165
-
166
- console.log(`Successfully processed batch ${batchType} in ${processingTime}ms`);
167
- } catch (error) {
168
- console.error(`Error processing batch ${batchType}:`, error);
169
- throw error;
170
- }
171
- }
172
-
173
- /**
174
- * Get cached value
175
- */
176
- getFromCache<R = T>(key: string): R | undefined {
177
- const item = this.cache.get(key) as CacheItem<R> | undefined;
178
-
179
- if (!item) {
180
- this.metrics.missCount++;
181
- return undefined;
182
- }
183
-
184
- // Check TTL
185
- const now = Date.now();
186
- if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
187
- this.cache.delete(key);
188
- this.metrics.missCount++;
189
- return undefined;
190
- }
191
-
192
- // Update access info
193
- item.accessedAt = new Date();
194
- item.accessCount++;
195
- this.metrics.hitCount++;
196
-
197
- return item.value;
198
- }
199
-
200
- /**
201
- * Set cached value
202
- */
203
- setInCache<R = T>(key: string, value: R, metadata?: any): void {
204
- // Check cache size limit
205
- if (this.cache.size >= this.cacheConfig.maxSize) {
206
- this.evictOldestItems(Math.floor(this.cacheConfig.maxSize * 0.1)); // Evict 10%
207
- }
208
-
209
- const now = new Date();
210
- const item: CacheItem<R> = {
211
- key,
212
- value,
213
- createdAt: now,
214
- accessedAt: now,
215
- accessCount: 1,
216
- size: this.calculateSize(value)
217
- };
218
-
219
- this.cache.set(key, item as any);
220
- }
221
-
222
- /**
223
- * Remove from cache
224
- */
225
- removeFromCache(key: string): boolean {
226
- return this.cache.delete(key);
227
- }
228
-
229
- /**
230
- * Clear entire cache
231
- */
232
- clearCache(): void {
233
- this.cache.clear();
234
- }
235
-
236
- /**
237
- * Get batch statistics
238
- */
239
- getBatchStats(): BatchStats & {
240
- cache: CacheMetrics;
241
- processing: {
242
- totalProcessed: number;
243
- averageProcessingTime: number;
244
- };
245
- } {
246
- const activeBatches = Array.from(this.batches.entries());
247
- const totalPendingItems = activeBatches.reduce((sum, [, batch]) => sum + batch.length, 0);
248
-
249
- return {
250
- totalUsers: activeBatches.length,
251
- totalPendingVectors: totalPendingItems, // Generic pending items
252
- activeBatchJobs: this.timers.size,
253
- cacheHitRate: this.getCacheHitRate(),
254
- averageBatchSize: activeBatches.length > 0
255
- ? totalPendingItems / activeBatches.length
256
- : 0,
257
- averageProcessingTime: this.metrics.averageProcessingTime,
258
- cache: this.getCacheMetrics(),
259
- processing: {
260
- totalProcessed: this.metrics.totalProcessed,
261
- averageProcessingTime: this.metrics.averageProcessingTime
262
- }
263
- };
264
- }
265
-
266
- /**
267
- * Get pending batch info
268
- */
269
- getPendingBatches(): Array<{
270
- batchType: string;
271
- itemCount: number;
272
- oldestItem: Date;
273
- scheduledProcessing: Date | null;
274
- }> {
275
- return Array.from(this.batches.entries()).map(([batchType, batch]) => {
276
- const oldestItem = batch.reduce((oldest, item) =>
277
- item.timestamp < oldest ? item.timestamp : oldest,
278
- batch[0]?.timestamp || new Date()
279
- );
280
-
281
- return {
282
- batchType,
283
- itemCount: batch.length,
284
- oldestItem,
285
- scheduledProcessing: null // TODO: Track scheduled processing time
286
- };
287
- });
288
- }
289
-
290
- /**
291
- * Force process all pending batches
292
- */
293
- async processAllBatches(): Promise<void> {
294
- const batchTypes = Array.from(this.batches.keys());
295
-
296
- for (const batchType of batchTypes) {
297
- try {
298
- await this.processBatchNow(batchType);
299
- } catch (error) {
300
- console.error(`Failed to process batch ${batchType}:`, error);
301
- }
302
- }
303
- }
304
-
305
- /**
306
- * Cleanup and destroy service
307
- */
308
- destroy(): void {
309
- // Clear all timers
310
- for (const timer of this.timers.values()) {
311
- clearTimeout(timer);
312
- }
313
- this.timers.clear();
314
-
315
- // Clear cleanup timer
316
- if (this.cleanupTimer) {
317
- clearInterval(this.cleanupTimer);
318
- }
319
-
320
- // Clear data
321
- this.batches.clear();
322
- this.processors.clear();
323
- this.cache.clear();
324
-
325
- console.log('BatchingService destroyed');
326
- }
327
-
328
- // ==================== PRIVATE METHODS ====================
329
-
330
- private scheduleBatchProcessing(batchType: string, forceImmediate = false): void {
331
- const batch = this.batches.get(batchType);
332
- if (!batch) return;
333
-
334
- // Clear existing timer
335
- const existingTimer = this.timers.get(batchType);
336
- if (existingTimer) {
337
- clearTimeout(existingTimer);
338
- }
339
-
340
- // Process immediately if conditions met
341
- if (forceImmediate || batch.length >= this.batchConfig.maxBatchSize) {
342
- setImmediate(() => this.processBatchNow(batchType));
343
- return;
344
- }
345
-
346
- // Schedule delayed processing
347
- const timer = setTimeout(
348
- () => this.processBatchNow(batchType),
349
- this.batchConfig.batchDelayMs
350
- );
351
-
352
- this.timers.set(batchType, timer);
353
- }
354
-
355
- private startCacheCleanup(): void {
356
- this.cleanupTimer = setInterval(() => {
357
- this.cleanupExpiredItems();
358
- }, this.cacheConfig.cleanupIntervalMs);
359
- }
360
-
361
- private cleanupExpiredItems(): void {
362
- const now = Date.now();
363
- let cleaned = 0;
364
-
365
- for (const [key, item] of this.cache.entries()) {
366
- if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
367
- this.cache.delete(key);
368
- cleaned++;
369
- }
370
- }
371
-
372
- if (cleaned > 0) {
373
- console.debug(`Cleaned up ${cleaned} expired cache items`);
374
- }
375
- }
376
-
377
- private evictOldestItems(count: number): void {
378
- const items = Array.from(this.cache.entries())
379
- .sort(([, a], [, b]) => a.accessedAt.getTime() - b.accessedAt.getTime())
380
- .slice(0, count);
381
-
382
- for (const [key] of items) {
383
- this.cache.delete(key);
384
- }
385
-
386
- console.debug(`Evicted ${items.length} oldest cache items`);
387
- }
388
-
389
- private getCacheHitRate(): number {
390
- const totalRequests = this.metrics.hitCount + this.metrics.missCount;
391
- return totalRequests > 0 ? this.metrics.hitCount / totalRequests : 0;
392
- }
393
-
394
- private getCacheMetrics(): CacheMetrics {
395
- const items = Array.from(this.cache.values());
396
-
397
- return {
398
- totalItems: items.length,
399
- totalSize: items.reduce((sum, item) => sum + (item.size || 0), 0),
400
- hitCount: this.metrics.hitCount,
401
- missCount: this.metrics.missCount,
402
- hitRate: this.getCacheHitRate(),
403
- averageAccessCount: items.length > 0
404
- ? items.reduce((sum, item) => sum + item.accessCount, 0) / items.length
405
- : 0,
406
- oldestItem: items.length > 0
407
- ? items.reduce((oldest, item) =>
408
- item.createdAt < oldest ? item.createdAt : oldest,
409
- items[0].createdAt
410
- )
411
- : undefined,
412
- newestItem: items.length > 0
413
- ? items.reduce((newest, item) =>
414
- item.createdAt > newest ? item.createdAt : newest,
415
- items[0].createdAt
416
- )
417
- : undefined
418
- };
419
- }
420
-
421
- private calculateSize(value: any): number {
422
- try {
423
- return JSON.stringify(value).length;
424
- } catch {
425
- return 0;
426
- }
427
- }
428
- }
429
-
1
+ /**
2
+ * BatchingService - Intelligent Batch Processing & Caching
3
+ *
4
+ * Provides advanced batching capabilities with intelligent scheduling,
5
+ * cache management, and performance optimization for the PDW SDK.
6
+ */
7
+
8
+ import { BatchConfig, BatchJob, BatchStats } from '../embedding/types';
9
+
10
+ export interface BatchItem<T = any> {
11
+ id: string;
12
+ data: T;
13
+ timestamp: Date;
14
+ priority?: number;
15
+ metadata?: any;
16
+ }
17
+
18
+ export interface BatchProcessor<T = any> {
19
+ process(items: BatchItem<T>[]): Promise<void>;
20
+ }
21
+
22
+ export interface CacheConfig {
23
+ maxSize?: number;
24
+ ttlMs?: number;
25
+ cleanupIntervalMs?: number;
26
+ enableMetrics?: boolean;
27
+ }
28
+
29
+ export interface CacheItem<T = any> {
30
+ key: string;
31
+ value: T;
32
+ createdAt: Date;
33
+ accessedAt: Date;
34
+ accessCount: number;
35
+ size?: number;
36
+ }
37
+
38
+ export interface CacheMetrics {
39
+ totalItems: number;
40
+ totalSize: number;
41
+ hitCount: number;
42
+ missCount: number;
43
+ hitRate: number;
44
+ averageAccessCount: number;
45
+ oldestItem?: Date;
46
+ newestItem?: Date;
47
+ }
48
+
49
+ /**
50
+ * Advanced batching service with intelligent scheduling and caching
51
+ */
52
+ export class BatchingService<T = any> {
53
+ private batches = new Map<string, BatchItem<T>[]>();
54
+ private processors = new Map<string, BatchProcessor<T>>();
55
+ private timers = new Map<string, NodeJS.Timeout>();
56
+ private cache = new Map<string, CacheItem<T>>();
57
+ private metrics = {
58
+ hitCount: 0,
59
+ missCount: 0,
60
+ totalProcessed: 0,
61
+ averageProcessingTime: 0
62
+ };
63
+
64
+ private readonly batchConfig: Required<BatchConfig>;
65
+ private readonly cacheConfig: Required<CacheConfig>;
66
+ private cleanupTimer?: NodeJS.Timeout;
67
+
68
+ constructor(
69
+ batchConfig: Partial<BatchConfig> = {},
70
+ cacheConfig: Partial<CacheConfig> = {}
71
+ ) {
72
+ this.batchConfig = {
73
+ maxBatchSize: batchConfig.maxBatchSize || 50,
74
+ batchDelayMs: batchConfig.batchDelayMs || 5000,
75
+ maxCacheSize: batchConfig.maxCacheSize || 1000,
76
+ cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
77
+ };
78
+
79
+ this.cacheConfig = {
80
+ maxSize: cacheConfig.maxSize || this.batchConfig.maxCacheSize,
81
+ ttlMs: cacheConfig.ttlMs || this.batchConfig.cacheTtlMs,
82
+ cleanupIntervalMs: cacheConfig.cleanupIntervalMs || 5 * 60 * 1000, // 5 minutes
83
+ enableMetrics: cacheConfig.enableMetrics !== false
84
+ };
85
+
86
+ this.startCacheCleanup();
87
+ }
88
+
89
+ /**
90
+ * Register a batch processor for a specific batch type
91
+ */
92
+ registerProcessor(batchType: string, processor: BatchProcessor<T>): void {
93
+ this.processors.set(batchType, processor);
94
+ }
95
+
96
+ /**
97
+ * Add item to batch for processing
98
+ */
99
+ addToBatch(
100
+ batchType: string,
101
+ item: BatchItem<T>,
102
+ options: {
103
+ forceImmediate?: boolean;
104
+ highPriority?: boolean;
105
+ } = {}
106
+ ): void {
107
+ // Get or create batch
108
+ let batch = this.batches.get(batchType);
109
+ if (!batch) {
110
+ batch = [];
111
+ this.batches.set(batchType, batch);
112
+ }
113
+
114
+ // Add item to batch
115
+ batch.push({
116
+ ...item,
117
+ priority: options.highPriority ? 1 : (item.priority || 0)
118
+ });
119
+
120
+ // Sort by priority (higher priority first)
121
+ batch.sort((a, b) => (b.priority || 0) - (a.priority || 0));
122
+
123
+ console.debug(`Added item to batch ${batchType}. Size: ${batch.length}`);
124
+
125
+ // Schedule processing
126
+ this.scheduleBatchProcessing(batchType, options.forceImmediate);
127
+ }
128
+
129
+ /**
130
+ * Process batch immediately
131
+ */
132
+ async processBatchNow(batchType: string): Promise<void> {
133
+ const batch = this.batches.get(batchType);
134
+ if (!batch || batch.length === 0) {
135
+ return;
136
+ }
137
+
138
+ const processor = this.processors.get(batchType);
139
+ if (!processor) {
140
+ console.warn(`No processor registered for batch type: ${batchType}`);
141
+ return;
142
+ }
143
+
144
+ const startTime = Date.now();
145
+
146
+ try {
147
+ console.log(`Processing batch ${batchType} with ${batch.length} items`);
148
+
149
+ // Process the batch
150
+ await processor.process([...batch]);
151
+
152
+ // Clear batch and timer
153
+ this.batches.delete(batchType);
154
+ const timer = this.timers.get(batchType);
155
+ if (timer) {
156
+ clearTimeout(timer);
157
+ this.timers.delete(batchType);
158
+ }
159
+
160
+ // Update metrics
161
+ this.metrics.totalProcessed += batch.length;
162
+ const processingTime = Date.now() - startTime;
163
+ this.metrics.averageProcessingTime =
164
+ (this.metrics.averageProcessingTime + processingTime) / 2;
165
+
166
+ console.log(`Successfully processed batch ${batchType} in ${processingTime}ms`);
167
+ } catch (error) {
168
+ console.error(`Error processing batch ${batchType}:`, error);
169
+ throw error;
170
+ }
171
+ }
172
+
173
+ /**
174
+ * Get cached value
175
+ */
176
+ getFromCache<R = T>(key: string): R | undefined {
177
+ const item = this.cache.get(key) as CacheItem<R> | undefined;
178
+
179
+ if (!item) {
180
+ this.metrics.missCount++;
181
+ return undefined;
182
+ }
183
+
184
+ // Check TTL
185
+ const now = Date.now();
186
+ if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
187
+ this.cache.delete(key);
188
+ this.metrics.missCount++;
189
+ return undefined;
190
+ }
191
+
192
+ // Update access info
193
+ item.accessedAt = new Date();
194
+ item.accessCount++;
195
+ this.metrics.hitCount++;
196
+
197
+ return item.value;
198
+ }
199
+
200
+ /**
201
+ * Set cached value
202
+ */
203
+ setInCache<R = T>(key: string, value: R, metadata?: any): void {
204
+ // Check cache size limit
205
+ if (this.cache.size >= this.cacheConfig.maxSize) {
206
+ this.evictOldestItems(Math.floor(this.cacheConfig.maxSize * 0.1)); // Evict 10%
207
+ }
208
+
209
+ const now = new Date();
210
+ const item: CacheItem<R> = {
211
+ key,
212
+ value,
213
+ createdAt: now,
214
+ accessedAt: now,
215
+ accessCount: 1,
216
+ size: this.calculateSize(value)
217
+ };
218
+
219
+ this.cache.set(key, item as any);
220
+ }
221
+
222
+ /**
223
+ * Remove from cache
224
+ */
225
+ removeFromCache(key: string): boolean {
226
+ return this.cache.delete(key);
227
+ }
228
+
229
+ /**
230
+ * Clear entire cache
231
+ */
232
+ clearCache(): void {
233
+ this.cache.clear();
234
+ }
235
+
236
+ /**
237
+ * Get batch statistics
238
+ */
239
+ getBatchStats(): BatchStats & {
240
+ cache: CacheMetrics;
241
+ processing: {
242
+ totalProcessed: number;
243
+ averageProcessingTime: number;
244
+ };
245
+ } {
246
+ const activeBatches = Array.from(this.batches.entries());
247
+ const totalPendingItems = activeBatches.reduce((sum, [, batch]) => sum + batch.length, 0);
248
+
249
+ return {
250
+ totalUsers: activeBatches.length,
251
+ totalPendingVectors: totalPendingItems, // Generic pending items
252
+ activeBatchJobs: this.timers.size,
253
+ cacheHitRate: this.getCacheHitRate(),
254
+ averageBatchSize: activeBatches.length > 0
255
+ ? totalPendingItems / activeBatches.length
256
+ : 0,
257
+ averageProcessingTime: this.metrics.averageProcessingTime,
258
+ cache: this.getCacheMetrics(),
259
+ processing: {
260
+ totalProcessed: this.metrics.totalProcessed,
261
+ averageProcessingTime: this.metrics.averageProcessingTime
262
+ }
263
+ };
264
+ }
265
+
266
+ /**
267
+ * Get pending batch info
268
+ */
269
+ getPendingBatches(): Array<{
270
+ batchType: string;
271
+ itemCount: number;
272
+ oldestItem: Date;
273
+ scheduledProcessing: Date | null;
274
+ }> {
275
+ return Array.from(this.batches.entries()).map(([batchType, batch]) => {
276
+ const oldestItem = batch.reduce((oldest, item) =>
277
+ item.timestamp < oldest ? item.timestamp : oldest,
278
+ batch[0]?.timestamp || new Date()
279
+ );
280
+
281
+ return {
282
+ batchType,
283
+ itemCount: batch.length,
284
+ oldestItem,
285
+ scheduledProcessing: null // TODO: Track scheduled processing time
286
+ };
287
+ });
288
+ }
289
+
290
+ /**
291
+ * Force process all pending batches
292
+ */
293
+ async processAllBatches(): Promise<void> {
294
+ const batchTypes = Array.from(this.batches.keys());
295
+
296
+ for (const batchType of batchTypes) {
297
+ try {
298
+ await this.processBatchNow(batchType);
299
+ } catch (error) {
300
+ console.error(`Failed to process batch ${batchType}:`, error);
301
+ }
302
+ }
303
+ }
304
+
305
+ /**
306
+ * Cleanup and destroy service
307
+ */
308
+ destroy(): void {
309
+ // Clear all timers
310
+ for (const timer of this.timers.values()) {
311
+ clearTimeout(timer);
312
+ }
313
+ this.timers.clear();
314
+
315
+ // Clear cleanup timer
316
+ if (this.cleanupTimer) {
317
+ clearInterval(this.cleanupTimer);
318
+ }
319
+
320
+ // Clear data
321
+ this.batches.clear();
322
+ this.processors.clear();
323
+ this.cache.clear();
324
+
325
+ console.log('BatchingService destroyed');
326
+ }
327
+
328
+ // ==================== PRIVATE METHODS ====================
329
+
330
+ private scheduleBatchProcessing(batchType: string, forceImmediate = false): void {
331
+ const batch = this.batches.get(batchType);
332
+ if (!batch) return;
333
+
334
+ // Clear existing timer
335
+ const existingTimer = this.timers.get(batchType);
336
+ if (existingTimer) {
337
+ clearTimeout(existingTimer);
338
+ }
339
+
340
+ // Process immediately if conditions met
341
+ if (forceImmediate || batch.length >= this.batchConfig.maxBatchSize) {
342
+ setImmediate(() => this.processBatchNow(batchType));
343
+ return;
344
+ }
345
+
346
+ // Schedule delayed processing
347
+ const timer = setTimeout(
348
+ () => this.processBatchNow(batchType),
349
+ this.batchConfig.batchDelayMs
350
+ );
351
+
352
+ this.timers.set(batchType, timer);
353
+ }
354
+
355
+ private startCacheCleanup(): void {
356
+ this.cleanupTimer = setInterval(() => {
357
+ this.cleanupExpiredItems();
358
+ }, this.cacheConfig.cleanupIntervalMs);
359
+ }
360
+
361
+ private cleanupExpiredItems(): void {
362
+ const now = Date.now();
363
+ let cleaned = 0;
364
+
365
+ for (const [key, item] of this.cache.entries()) {
366
+ if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
367
+ this.cache.delete(key);
368
+ cleaned++;
369
+ }
370
+ }
371
+
372
+ if (cleaned > 0) {
373
+ console.debug(`Cleaned up ${cleaned} expired cache items`);
374
+ }
375
+ }
376
+
377
+ private evictOldestItems(count: number): void {
378
+ const items = Array.from(this.cache.entries())
379
+ .sort(([, a], [, b]) => a.accessedAt.getTime() - b.accessedAt.getTime())
380
+ .slice(0, count);
381
+
382
+ for (const [key] of items) {
383
+ this.cache.delete(key);
384
+ }
385
+
386
+ console.debug(`Evicted ${items.length} oldest cache items`);
387
+ }
388
+
389
+ private getCacheHitRate(): number {
390
+ const totalRequests = this.metrics.hitCount + this.metrics.missCount;
391
+ return totalRequests > 0 ? this.metrics.hitCount / totalRequests : 0;
392
+ }
393
+
394
+ private getCacheMetrics(): CacheMetrics {
395
+ const items = Array.from(this.cache.values());
396
+
397
+ return {
398
+ totalItems: items.length,
399
+ totalSize: items.reduce((sum, item) => sum + (item.size || 0), 0),
400
+ hitCount: this.metrics.hitCount,
401
+ missCount: this.metrics.missCount,
402
+ hitRate: this.getCacheHitRate(),
403
+ averageAccessCount: items.length > 0
404
+ ? items.reduce((sum, item) => sum + item.accessCount, 0) / items.length
405
+ : 0,
406
+ oldestItem: items.length > 0
407
+ ? items.reduce((oldest, item) =>
408
+ item.createdAt < oldest ? item.createdAt : oldest,
409
+ items[0].createdAt
410
+ )
411
+ : undefined,
412
+ newestItem: items.length > 0
413
+ ? items.reduce((newest, item) =>
414
+ item.createdAt > newest ? item.createdAt : newest,
415
+ items[0].createdAt
416
+ )
417
+ : undefined
418
+ };
419
+ }
420
+
421
+ private calculateSize(value: any): number {
422
+ try {
423
+ return JSON.stringify(value).length;
424
+ } catch {
425
+ return 0;
426
+ }
427
+ }
428
+ }
429
+
430
430
  export default BatchingService;