@cmdoss/memwal-sdk 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/README.md +129 -0
  2. package/dist/client/ClientMemoryManager.js +2 -2
  3. package/dist/client/ClientMemoryManager.js.map +1 -1
  4. package/dist/client/PersonalDataWallet.d.ts.map +1 -1
  5. package/dist/client/SimplePDWClient.d.ts +28 -0
  6. package/dist/client/SimplePDWClient.d.ts.map +1 -1
  7. package/dist/client/SimplePDWClient.js +29 -6
  8. package/dist/client/SimplePDWClient.js.map +1 -1
  9. package/dist/client/namespaces/MemoryNamespace.d.ts +4 -0
  10. package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
  11. package/dist/client/namespaces/MemoryNamespace.js +168 -39
  12. package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
  13. package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts +12 -2
  14. package/dist/client/namespaces/consolidated/BlockchainNamespace.d.ts.map +1 -1
  15. package/dist/client/namespaces/consolidated/BlockchainNamespace.js +40 -2
  16. package/dist/client/namespaces/consolidated/BlockchainNamespace.js.map +1 -1
  17. package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +67 -2
  18. package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
  19. package/dist/client/namespaces/consolidated/StorageNamespace.js +549 -16
  20. package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
  21. package/dist/config/ConfigurationHelper.js +61 -61
  22. package/dist/config/defaults.js +2 -2
  23. package/dist/config/defaults.js.map +1 -1
  24. package/dist/graph/GraphService.js +20 -20
  25. package/dist/infrastructure/seal/EncryptionService.d.ts +9 -5
  26. package/dist/infrastructure/seal/EncryptionService.d.ts.map +1 -1
  27. package/dist/infrastructure/seal/EncryptionService.js +37 -15
  28. package/dist/infrastructure/seal/EncryptionService.js.map +1 -1
  29. package/dist/infrastructure/seal/SealService.d.ts +13 -5
  30. package/dist/infrastructure/seal/SealService.d.ts.map +1 -1
  31. package/dist/infrastructure/seal/SealService.js +36 -34
  32. package/dist/infrastructure/seal/SealService.js.map +1 -1
  33. package/dist/langchain/createPDWRAG.js +30 -30
  34. package/dist/retrieval/MemoryDecryptionPipeline.d.ts.map +1 -1
  35. package/dist/retrieval/MemoryDecryptionPipeline.js +2 -1
  36. package/dist/retrieval/MemoryDecryptionPipeline.js.map +1 -1
  37. package/dist/services/CapabilityService.d.ts.map +1 -1
  38. package/dist/services/CapabilityService.js +30 -14
  39. package/dist/services/CapabilityService.js.map +1 -1
  40. package/dist/services/CrossContextPermissionService.d.ts.map +1 -1
  41. package/dist/services/CrossContextPermissionService.js +9 -7
  42. package/dist/services/CrossContextPermissionService.js.map +1 -1
  43. package/dist/services/EncryptionService.d.ts.map +1 -1
  44. package/dist/services/EncryptionService.js +6 -5
  45. package/dist/services/EncryptionService.js.map +1 -1
  46. package/dist/services/GeminiAIService.js +309 -309
  47. package/dist/services/StorageService.d.ts +1 -0
  48. package/dist/services/StorageService.d.ts.map +1 -1
  49. package/dist/services/StorageService.js +60 -10
  50. package/dist/services/StorageService.js.map +1 -1
  51. package/dist/services/TransactionService.d.ts +20 -0
  52. package/dist/services/TransactionService.d.ts.map +1 -1
  53. package/dist/services/TransactionService.js +43 -0
  54. package/dist/services/TransactionService.js.map +1 -1
  55. package/dist/services/ViewService.js +2 -2
  56. package/dist/services/ViewService.js.map +1 -1
  57. package/package.json +1 -1
  58. package/src/access/PermissionService.ts +635 -635
  59. package/src/access/index.ts +8 -8
  60. package/src/aggregation/AggregationService.ts +389 -389
  61. package/src/aggregation/index.ts +8 -8
  62. package/src/ai-sdk/PDWVectorStore.ts +715 -715
  63. package/src/ai-sdk/index.ts +65 -65
  64. package/src/ai-sdk/tools.ts +460 -460
  65. package/src/ai-sdk/types.ts +404 -404
  66. package/src/batch/BatchManager.ts +597 -597
  67. package/src/batch/BatchingService.ts +429 -429
  68. package/src/batch/MemoryProcessingCache.ts +492 -492
  69. package/src/batch/index.ts +30 -30
  70. package/src/browser.ts +200 -200
  71. package/src/client/ClientMemoryManager.ts +987 -987
  72. package/src/client/PersonalDataWallet.ts +345 -345
  73. package/src/client/SimplePDWClient.ts +1289 -1237
  74. package/src/client/factory.ts +154 -154
  75. package/src/client/namespaces/AnalyticsNamespace.ts +377 -377
  76. package/src/client/namespaces/BatchNamespace.ts +356 -356
  77. package/src/client/namespaces/CacheNamespace.ts +123 -123
  78. package/src/client/namespaces/CapabilityNamespace.ts +217 -217
  79. package/src/client/namespaces/ClassifyNamespace.ts +169 -169
  80. package/src/client/namespaces/ContextNamespace.ts +297 -297
  81. package/src/client/namespaces/EmbeddingsNamespace.ts +99 -99
  82. package/src/client/namespaces/EncryptionNamespace.ts +221 -221
  83. package/src/client/namespaces/GraphNamespace.ts +468 -468
  84. package/src/client/namespaces/IndexNamespace.ts +361 -361
  85. package/src/client/namespaces/MemoryNamespace.ts +1422 -1272
  86. package/src/client/namespaces/PermissionsNamespace.ts +254 -254
  87. package/src/client/namespaces/PipelineNamespace.ts +220 -220
  88. package/src/client/namespaces/SearchNamespace.ts +1049 -1049
  89. package/src/client/namespaces/StorageNamespace.ts +458 -458
  90. package/src/client/namespaces/TxNamespace.ts +260 -260
  91. package/src/client/namespaces/WalletNamespace.ts +243 -243
  92. package/src/client/namespaces/consolidated/AINamespace.ts +449 -449
  93. package/src/client/namespaces/consolidated/BlockchainNamespace.ts +607 -564
  94. package/src/client/namespaces/consolidated/SecurityNamespace.ts +648 -648
  95. package/src/client/namespaces/consolidated/StorageNamespace.ts +1141 -497
  96. package/src/client/namespaces/consolidated/index.ts +39 -39
  97. package/src/client/signers/DappKitSigner.ts +207 -207
  98. package/src/client/signers/KeypairSigner.ts +108 -108
  99. package/src/client/signers/UnifiedSigner.ts +110 -110
  100. package/src/client/signers/WalletAdapterSigner.ts +159 -159
  101. package/src/client/signers/index.ts +26 -26
  102. package/src/config/ConfigurationHelper.ts +412 -412
  103. package/src/config/defaults.ts +51 -51
  104. package/src/config/index.ts +8 -8
  105. package/src/config/validation.ts +70 -70
  106. package/src/core/index.ts +14 -14
  107. package/src/core/interfaces/IService.ts +307 -307
  108. package/src/core/interfaces/index.ts +8 -8
  109. package/src/core/types/capability.ts +297 -297
  110. package/src/core/types/index.ts +870 -870
  111. package/src/core/types/wallet.ts +270 -270
  112. package/src/core/types.ts +9 -9
  113. package/src/core/wallet.ts +222 -222
  114. package/src/embedding/index.ts +19 -19
  115. package/src/embedding/types.ts +357 -357
  116. package/src/errors/index.ts +602 -602
  117. package/src/errors/recovery.ts +461 -461
  118. package/src/errors/validation.ts +567 -567
  119. package/src/generated/pdw/capability.ts +319 -319
  120. package/src/generated/pdw/deps/sui/object.ts +12 -12
  121. package/src/generated/pdw/deps/sui/vec_map.ts +32 -32
  122. package/src/generated/pdw/memory.ts +1087 -1087
  123. package/src/generated/pdw/wallet.ts +123 -123
  124. package/src/generated/utils/index.ts +159 -159
  125. package/src/graph/GraphService.ts +887 -887
  126. package/src/graph/KnowledgeGraphManager.ts +728 -728
  127. package/src/graph/index.ts +25 -25
  128. package/src/index.ts +498 -498
  129. package/src/infrastructure/index.ts +22 -22
  130. package/src/infrastructure/seal/EncryptionService.ts +628 -603
  131. package/src/infrastructure/seal/SealService.ts +613 -615
  132. package/src/infrastructure/seal/index.ts +9 -9
  133. package/src/infrastructure/sui/BlockchainManager.ts +627 -627
  134. package/src/infrastructure/sui/SuiService.ts +888 -888
  135. package/src/infrastructure/sui/index.ts +9 -9
  136. package/src/infrastructure/walrus/StorageManager.ts +604 -604
  137. package/src/infrastructure/walrus/WalrusStorageService.ts +612 -612
  138. package/src/infrastructure/walrus/index.ts +9 -9
  139. package/src/langchain/PDWEmbeddings.ts +145 -145
  140. package/src/langchain/PDWVectorStore.ts +456 -456
  141. package/src/langchain/createPDWRAG.ts +303 -303
  142. package/src/langchain/index.ts +47 -47
  143. package/src/permissions/ConsentRepository.browser.ts +249 -249
  144. package/src/permissions/ConsentRepository.ts +364 -364
  145. package/src/permissions/index.ts +9 -9
  146. package/src/pipeline/MemoryPipeline.ts +862 -862
  147. package/src/pipeline/PipelineManager.ts +683 -683
  148. package/src/pipeline/index.ts +26 -26
  149. package/src/retrieval/AdvancedSearchService.ts +629 -629
  150. package/src/retrieval/MemoryAnalyticsService.ts +711 -711
  151. package/src/retrieval/MemoryDecryptionPipeline.ts +825 -824
  152. package/src/retrieval/MemoryRetrievalService.ts +904 -904
  153. package/src/retrieval/index.ts +42 -42
  154. package/src/services/BatchService.ts +352 -352
  155. package/src/services/CapabilityService.ts +464 -448
  156. package/src/services/ClassifierService.ts +465 -465
  157. package/src/services/CrossContextPermissionService.ts +486 -484
  158. package/src/services/EmbeddingService.ts +771 -771
  159. package/src/services/EncryptionService.ts +712 -711
  160. package/src/services/GeminiAIService.ts +753 -753
  161. package/src/services/IndexManager.ts +977 -977
  162. package/src/services/MemoryIndexService.ts +1003 -1003
  163. package/src/services/MemoryService.ts +369 -369
  164. package/src/services/QueryService.ts +890 -890
  165. package/src/services/StorageService.ts +1182 -1126
  166. package/src/services/TransactionService.ts +838 -790
  167. package/src/services/VectorService.ts +462 -462
  168. package/src/services/ViewService.ts +484 -484
  169. package/src/services/index.ts +25 -25
  170. package/src/services/storage/BlobAttributesManager.ts +333 -333
  171. package/src/services/storage/KnowledgeGraphManager.ts +425 -425
  172. package/src/services/storage/MemorySearchManager.ts +387 -387
  173. package/src/services/storage/QuiltBatchManager.ts +1130 -1130
  174. package/src/services/storage/WalrusMetadataManager.ts +268 -268
  175. package/src/services/storage/WalrusStorageManager.ts +287 -287
  176. package/src/services/storage/index.ts +57 -57
  177. package/src/types/index.ts +13 -13
  178. package/src/utils/LRUCache.ts +378 -378
  179. package/src/utils/index.ts +76 -76
  180. package/src/utils/memoryIndexOnChain.ts +507 -507
  181. package/src/utils/rebuildIndex.ts +290 -290
  182. package/src/utils/rebuildIndexNode.ts +771 -771
  183. package/src/vector/BrowserHnswIndexService.ts +758 -758
  184. package/src/vector/HnswWasmService.ts +731 -731
  185. package/src/vector/IHnswService.ts +233 -233
  186. package/src/vector/NodeHnswService.ts +833 -833
  187. package/src/vector/VectorManager.ts +478 -478
  188. package/src/vector/createHnswService.ts +135 -135
  189. package/src/vector/index.ts +56 -56
  190. package/src/wallet/ContextWalletService.ts +656 -656
  191. package/src/wallet/MainWalletService.ts +317 -317
  192. package/src/wallet/index.ts +17 -17
@@ -1,430 +1,430 @@
1
- /**
2
- * BatchingService - Intelligent Batch Processing & Caching
3
- *
4
- * Provides advanced batching capabilities with intelligent scheduling,
5
- * cache management, and performance optimization for the PDW SDK.
6
- */
7
-
8
- import { BatchConfig, BatchJob, BatchStats } from '../embedding/types';
9
-
10
- export interface BatchItem<T = any> {
11
- id: string;
12
- data: T;
13
- timestamp: Date;
14
- priority?: number;
15
- metadata?: any;
16
- }
17
-
18
- export interface BatchProcessor<T = any> {
19
- process(items: BatchItem<T>[]): Promise<void>;
20
- }
21
-
22
- export interface CacheConfig {
23
- maxSize?: number;
24
- ttlMs?: number;
25
- cleanupIntervalMs?: number;
26
- enableMetrics?: boolean;
27
- }
28
-
29
- export interface CacheItem<T = any> {
30
- key: string;
31
- value: T;
32
- createdAt: Date;
33
- accessedAt: Date;
34
- accessCount: number;
35
- size?: number;
36
- }
37
-
38
- export interface CacheMetrics {
39
- totalItems: number;
40
- totalSize: number;
41
- hitCount: number;
42
- missCount: number;
43
- hitRate: number;
44
- averageAccessCount: number;
45
- oldestItem?: Date;
46
- newestItem?: Date;
47
- }
48
-
49
- /**
50
- * Advanced batching service with intelligent scheduling and caching
51
- */
52
- export class BatchingService<T = any> {
53
- private batches = new Map<string, BatchItem<T>[]>();
54
- private processors = new Map<string, BatchProcessor<T>>();
55
- private timers = new Map<string, NodeJS.Timeout>();
56
- private cache = new Map<string, CacheItem<T>>();
57
- private metrics = {
58
- hitCount: 0,
59
- missCount: 0,
60
- totalProcessed: 0,
61
- averageProcessingTime: 0
62
- };
63
-
64
- private readonly batchConfig: Required<BatchConfig>;
65
- private readonly cacheConfig: Required<CacheConfig>;
66
- private cleanupTimer?: NodeJS.Timeout;
67
-
68
- constructor(
69
- batchConfig: Partial<BatchConfig> = {},
70
- cacheConfig: Partial<CacheConfig> = {}
71
- ) {
72
- this.batchConfig = {
73
- maxBatchSize: batchConfig.maxBatchSize || 50,
74
- batchDelayMs: batchConfig.batchDelayMs || 5000,
75
- maxCacheSize: batchConfig.maxCacheSize || 1000,
76
- cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
77
- };
78
-
79
- this.cacheConfig = {
80
- maxSize: cacheConfig.maxSize || this.batchConfig.maxCacheSize,
81
- ttlMs: cacheConfig.ttlMs || this.batchConfig.cacheTtlMs,
82
- cleanupIntervalMs: cacheConfig.cleanupIntervalMs || 5 * 60 * 1000, // 5 minutes
83
- enableMetrics: cacheConfig.enableMetrics !== false
84
- };
85
-
86
- this.startCacheCleanup();
87
- }
88
-
89
- /**
90
- * Register a batch processor for a specific batch type
91
- */
92
- registerProcessor(batchType: string, processor: BatchProcessor<T>): void {
93
- this.processors.set(batchType, processor);
94
- }
95
-
96
- /**
97
- * Add item to batch for processing
98
- */
99
- addToBatch(
100
- batchType: string,
101
- item: BatchItem<T>,
102
- options: {
103
- forceImmediate?: boolean;
104
- highPriority?: boolean;
105
- } = {}
106
- ): void {
107
- // Get or create batch
108
- let batch = this.batches.get(batchType);
109
- if (!batch) {
110
- batch = [];
111
- this.batches.set(batchType, batch);
112
- }
113
-
114
- // Add item to batch
115
- batch.push({
116
- ...item,
117
- priority: options.highPriority ? 1 : (item.priority || 0)
118
- });
119
-
120
- // Sort by priority (higher priority first)
121
- batch.sort((a, b) => (b.priority || 0) - (a.priority || 0));
122
-
123
- console.debug(`Added item to batch ${batchType}. Size: ${batch.length}`);
124
-
125
- // Schedule processing
126
- this.scheduleBatchProcessing(batchType, options.forceImmediate);
127
- }
128
-
129
- /**
130
- * Process batch immediately
131
- */
132
- async processBatchNow(batchType: string): Promise<void> {
133
- const batch = this.batches.get(batchType);
134
- if (!batch || batch.length === 0) {
135
- return;
136
- }
137
-
138
- const processor = this.processors.get(batchType);
139
- if (!processor) {
140
- console.warn(`No processor registered for batch type: ${batchType}`);
141
- return;
142
- }
143
-
144
- const startTime = Date.now();
145
-
146
- try {
147
- console.log(`Processing batch ${batchType} with ${batch.length} items`);
148
-
149
- // Process the batch
150
- await processor.process([...batch]);
151
-
152
- // Clear batch and timer
153
- this.batches.delete(batchType);
154
- const timer = this.timers.get(batchType);
155
- if (timer) {
156
- clearTimeout(timer);
157
- this.timers.delete(batchType);
158
- }
159
-
160
- // Update metrics
161
- this.metrics.totalProcessed += batch.length;
162
- const processingTime = Date.now() - startTime;
163
- this.metrics.averageProcessingTime =
164
- (this.metrics.averageProcessingTime + processingTime) / 2;
165
-
166
- console.log(`Successfully processed batch ${batchType} in ${processingTime}ms`);
167
- } catch (error) {
168
- console.error(`Error processing batch ${batchType}:`, error);
169
- throw error;
170
- }
171
- }
172
-
173
- /**
174
- * Get cached value
175
- */
176
- getFromCache<R = T>(key: string): R | undefined {
177
- const item = this.cache.get(key) as CacheItem<R> | undefined;
178
-
179
- if (!item) {
180
- this.metrics.missCount++;
181
- return undefined;
182
- }
183
-
184
- // Check TTL
185
- const now = Date.now();
186
- if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
187
- this.cache.delete(key);
188
- this.metrics.missCount++;
189
- return undefined;
190
- }
191
-
192
- // Update access info
193
- item.accessedAt = new Date();
194
- item.accessCount++;
195
- this.metrics.hitCount++;
196
-
197
- return item.value;
198
- }
199
-
200
- /**
201
- * Set cached value
202
- */
203
- setInCache<R = T>(key: string, value: R, metadata?: any): void {
204
- // Check cache size limit
205
- if (this.cache.size >= this.cacheConfig.maxSize) {
206
- this.evictOldestItems(Math.floor(this.cacheConfig.maxSize * 0.1)); // Evict 10%
207
- }
208
-
209
- const now = new Date();
210
- const item: CacheItem<R> = {
211
- key,
212
- value,
213
- createdAt: now,
214
- accessedAt: now,
215
- accessCount: 1,
216
- size: this.calculateSize(value)
217
- };
218
-
219
- this.cache.set(key, item as any);
220
- }
221
-
222
- /**
223
- * Remove from cache
224
- */
225
- removeFromCache(key: string): boolean {
226
- return this.cache.delete(key);
227
- }
228
-
229
- /**
230
- * Clear entire cache
231
- */
232
- clearCache(): void {
233
- this.cache.clear();
234
- }
235
-
236
- /**
237
- * Get batch statistics
238
- */
239
- getBatchStats(): BatchStats & {
240
- cache: CacheMetrics;
241
- processing: {
242
- totalProcessed: number;
243
- averageProcessingTime: number;
244
- };
245
- } {
246
- const activeBatches = Array.from(this.batches.entries());
247
- const totalPendingItems = activeBatches.reduce((sum, [, batch]) => sum + batch.length, 0);
248
-
249
- return {
250
- totalUsers: activeBatches.length,
251
- totalPendingVectors: totalPendingItems, // Generic pending items
252
- activeBatchJobs: this.timers.size,
253
- cacheHitRate: this.getCacheHitRate(),
254
- averageBatchSize: activeBatches.length > 0
255
- ? totalPendingItems / activeBatches.length
256
- : 0,
257
- averageProcessingTime: this.metrics.averageProcessingTime,
258
- cache: this.getCacheMetrics(),
259
- processing: {
260
- totalProcessed: this.metrics.totalProcessed,
261
- averageProcessingTime: this.metrics.averageProcessingTime
262
- }
263
- };
264
- }
265
-
266
- /**
267
- * Get pending batch info
268
- */
269
- getPendingBatches(): Array<{
270
- batchType: string;
271
- itemCount: number;
272
- oldestItem: Date;
273
- scheduledProcessing: Date | null;
274
- }> {
275
- return Array.from(this.batches.entries()).map(([batchType, batch]) => {
276
- const oldestItem = batch.reduce((oldest, item) =>
277
- item.timestamp < oldest ? item.timestamp : oldest,
278
- batch[0]?.timestamp || new Date()
279
- );
280
-
281
- return {
282
- batchType,
283
- itemCount: batch.length,
284
- oldestItem,
285
- scheduledProcessing: null // TODO: Track scheduled processing time
286
- };
287
- });
288
- }
289
-
290
- /**
291
- * Force process all pending batches
292
- */
293
- async processAllBatches(): Promise<void> {
294
- const batchTypes = Array.from(this.batches.keys());
295
-
296
- for (const batchType of batchTypes) {
297
- try {
298
- await this.processBatchNow(batchType);
299
- } catch (error) {
300
- console.error(`Failed to process batch ${batchType}:`, error);
301
- }
302
- }
303
- }
304
-
305
- /**
306
- * Cleanup and destroy service
307
- */
308
- destroy(): void {
309
- // Clear all timers
310
- for (const timer of this.timers.values()) {
311
- clearTimeout(timer);
312
- }
313
- this.timers.clear();
314
-
315
- // Clear cleanup timer
316
- if (this.cleanupTimer) {
317
- clearInterval(this.cleanupTimer);
318
- }
319
-
320
- // Clear data
321
- this.batches.clear();
322
- this.processors.clear();
323
- this.cache.clear();
324
-
325
- console.log('BatchingService destroyed');
326
- }
327
-
328
- // ==================== PRIVATE METHODS ====================
329
-
330
- private scheduleBatchProcessing(batchType: string, forceImmediate = false): void {
331
- const batch = this.batches.get(batchType);
332
- if (!batch) return;
333
-
334
- // Clear existing timer
335
- const existingTimer = this.timers.get(batchType);
336
- if (existingTimer) {
337
- clearTimeout(existingTimer);
338
- }
339
-
340
- // Process immediately if conditions met
341
- if (forceImmediate || batch.length >= this.batchConfig.maxBatchSize) {
342
- setImmediate(() => this.processBatchNow(batchType));
343
- return;
344
- }
345
-
346
- // Schedule delayed processing
347
- const timer = setTimeout(
348
- () => this.processBatchNow(batchType),
349
- this.batchConfig.batchDelayMs
350
- );
351
-
352
- this.timers.set(batchType, timer);
353
- }
354
-
355
- private startCacheCleanup(): void {
356
- this.cleanupTimer = setInterval(() => {
357
- this.cleanupExpiredItems();
358
- }, this.cacheConfig.cleanupIntervalMs);
359
- }
360
-
361
- private cleanupExpiredItems(): void {
362
- const now = Date.now();
363
- let cleaned = 0;
364
-
365
- for (const [key, item] of this.cache.entries()) {
366
- if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
367
- this.cache.delete(key);
368
- cleaned++;
369
- }
370
- }
371
-
372
- if (cleaned > 0) {
373
- console.debug(`Cleaned up ${cleaned} expired cache items`);
374
- }
375
- }
376
-
377
- private evictOldestItems(count: number): void {
378
- const items = Array.from(this.cache.entries())
379
- .sort(([, a], [, b]) => a.accessedAt.getTime() - b.accessedAt.getTime())
380
- .slice(0, count);
381
-
382
- for (const [key] of items) {
383
- this.cache.delete(key);
384
- }
385
-
386
- console.debug(`Evicted ${items.length} oldest cache items`);
387
- }
388
-
389
- private getCacheHitRate(): number {
390
- const totalRequests = this.metrics.hitCount + this.metrics.missCount;
391
- return totalRequests > 0 ? this.metrics.hitCount / totalRequests : 0;
392
- }
393
-
394
- private getCacheMetrics(): CacheMetrics {
395
- const items = Array.from(this.cache.values());
396
-
397
- return {
398
- totalItems: items.length,
399
- totalSize: items.reduce((sum, item) => sum + (item.size || 0), 0),
400
- hitCount: this.metrics.hitCount,
401
- missCount: this.metrics.missCount,
402
- hitRate: this.getCacheHitRate(),
403
- averageAccessCount: items.length > 0
404
- ? items.reduce((sum, item) => sum + item.accessCount, 0) / items.length
405
- : 0,
406
- oldestItem: items.length > 0
407
- ? items.reduce((oldest, item) =>
408
- item.createdAt < oldest ? item.createdAt : oldest,
409
- items[0].createdAt
410
- )
411
- : undefined,
412
- newestItem: items.length > 0
413
- ? items.reduce((newest, item) =>
414
- item.createdAt > newest ? item.createdAt : newest,
415
- items[0].createdAt
416
- )
417
- : undefined
418
- };
419
- }
420
-
421
- private calculateSize(value: any): number {
422
- try {
423
- return JSON.stringify(value).length;
424
- } catch {
425
- return 0;
426
- }
427
- }
428
- }
429
-
1
+ /**
2
+ * BatchingService - Intelligent Batch Processing & Caching
3
+ *
4
+ * Provides advanced batching capabilities with intelligent scheduling,
5
+ * cache management, and performance optimization for the PDW SDK.
6
+ */
7
+
8
+ import { BatchConfig, BatchJob, BatchStats } from '../embedding/types';
9
+
10
+ export interface BatchItem<T = any> {
11
+ id: string;
12
+ data: T;
13
+ timestamp: Date;
14
+ priority?: number;
15
+ metadata?: any;
16
+ }
17
+
18
+ export interface BatchProcessor<T = any> {
19
+ process(items: BatchItem<T>[]): Promise<void>;
20
+ }
21
+
22
+ export interface CacheConfig {
23
+ maxSize?: number;
24
+ ttlMs?: number;
25
+ cleanupIntervalMs?: number;
26
+ enableMetrics?: boolean;
27
+ }
28
+
29
+ export interface CacheItem<T = any> {
30
+ key: string;
31
+ value: T;
32
+ createdAt: Date;
33
+ accessedAt: Date;
34
+ accessCount: number;
35
+ size?: number;
36
+ }
37
+
38
+ export interface CacheMetrics {
39
+ totalItems: number;
40
+ totalSize: number;
41
+ hitCount: number;
42
+ missCount: number;
43
+ hitRate: number;
44
+ averageAccessCount: number;
45
+ oldestItem?: Date;
46
+ newestItem?: Date;
47
+ }
48
+
49
+ /**
50
+ * Advanced batching service with intelligent scheduling and caching
51
+ */
52
+ export class BatchingService<T = any> {
53
+ private batches = new Map<string, BatchItem<T>[]>();
54
+ private processors = new Map<string, BatchProcessor<T>>();
55
+ private timers = new Map<string, NodeJS.Timeout>();
56
+ private cache = new Map<string, CacheItem<T>>();
57
+ private metrics = {
58
+ hitCount: 0,
59
+ missCount: 0,
60
+ totalProcessed: 0,
61
+ averageProcessingTime: 0
62
+ };
63
+
64
+ private readonly batchConfig: Required<BatchConfig>;
65
+ private readonly cacheConfig: Required<CacheConfig>;
66
+ private cleanupTimer?: NodeJS.Timeout;
67
+
68
+ constructor(
69
+ batchConfig: Partial<BatchConfig> = {},
70
+ cacheConfig: Partial<CacheConfig> = {}
71
+ ) {
72
+ this.batchConfig = {
73
+ maxBatchSize: batchConfig.maxBatchSize || 50,
74
+ batchDelayMs: batchConfig.batchDelayMs || 5000,
75
+ maxCacheSize: batchConfig.maxCacheSize || 1000,
76
+ cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
77
+ };
78
+
79
+ this.cacheConfig = {
80
+ maxSize: cacheConfig.maxSize || this.batchConfig.maxCacheSize,
81
+ ttlMs: cacheConfig.ttlMs || this.batchConfig.cacheTtlMs,
82
+ cleanupIntervalMs: cacheConfig.cleanupIntervalMs || 5 * 60 * 1000, // 5 minutes
83
+ enableMetrics: cacheConfig.enableMetrics !== false
84
+ };
85
+
86
+ this.startCacheCleanup();
87
+ }
88
+
89
+ /**
90
+ * Register a batch processor for a specific batch type
91
+ */
92
+ registerProcessor(batchType: string, processor: BatchProcessor<T>): void {
93
+ this.processors.set(batchType, processor);
94
+ }
95
+
96
+ /**
97
+ * Add item to batch for processing
98
+ */
99
+ addToBatch(
100
+ batchType: string,
101
+ item: BatchItem<T>,
102
+ options: {
103
+ forceImmediate?: boolean;
104
+ highPriority?: boolean;
105
+ } = {}
106
+ ): void {
107
+ // Get or create batch
108
+ let batch = this.batches.get(batchType);
109
+ if (!batch) {
110
+ batch = [];
111
+ this.batches.set(batchType, batch);
112
+ }
113
+
114
+ // Add item to batch
115
+ batch.push({
116
+ ...item,
117
+ priority: options.highPriority ? 1 : (item.priority || 0)
118
+ });
119
+
120
+ // Sort by priority (higher priority first)
121
+ batch.sort((a, b) => (b.priority || 0) - (a.priority || 0));
122
+
123
+ console.debug(`Added item to batch ${batchType}. Size: ${batch.length}`);
124
+
125
+ // Schedule processing
126
+ this.scheduleBatchProcessing(batchType, options.forceImmediate);
127
+ }
128
+
129
+ /**
130
+ * Process batch immediately
131
+ */
132
+ async processBatchNow(batchType: string): Promise<void> {
133
+ const batch = this.batches.get(batchType);
134
+ if (!batch || batch.length === 0) {
135
+ return;
136
+ }
137
+
138
+ const processor = this.processors.get(batchType);
139
+ if (!processor) {
140
+ console.warn(`No processor registered for batch type: ${batchType}`);
141
+ return;
142
+ }
143
+
144
+ const startTime = Date.now();
145
+
146
+ try {
147
+ console.log(`Processing batch ${batchType} with ${batch.length} items`);
148
+
149
+ // Process the batch
150
+ await processor.process([...batch]);
151
+
152
+ // Clear batch and timer
153
+ this.batches.delete(batchType);
154
+ const timer = this.timers.get(batchType);
155
+ if (timer) {
156
+ clearTimeout(timer);
157
+ this.timers.delete(batchType);
158
+ }
159
+
160
+ // Update metrics
161
+ this.metrics.totalProcessed += batch.length;
162
+ const processingTime = Date.now() - startTime;
163
+ this.metrics.averageProcessingTime =
164
+ (this.metrics.averageProcessingTime + processingTime) / 2;
165
+
166
+ console.log(`Successfully processed batch ${batchType} in ${processingTime}ms`);
167
+ } catch (error) {
168
+ console.error(`Error processing batch ${batchType}:`, error);
169
+ throw error;
170
+ }
171
+ }
172
+
173
+ /**
174
+ * Get cached value
175
+ */
176
+ getFromCache<R = T>(key: string): R | undefined {
177
+ const item = this.cache.get(key) as CacheItem<R> | undefined;
178
+
179
+ if (!item) {
180
+ this.metrics.missCount++;
181
+ return undefined;
182
+ }
183
+
184
+ // Check TTL
185
+ const now = Date.now();
186
+ if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
187
+ this.cache.delete(key);
188
+ this.metrics.missCount++;
189
+ return undefined;
190
+ }
191
+
192
+ // Update access info
193
+ item.accessedAt = new Date();
194
+ item.accessCount++;
195
+ this.metrics.hitCount++;
196
+
197
+ return item.value;
198
+ }
199
+
200
+ /**
201
+ * Set cached value
202
+ */
203
+ setInCache<R = T>(key: string, value: R, metadata?: any): void {
204
+ // Check cache size limit
205
+ if (this.cache.size >= this.cacheConfig.maxSize) {
206
+ this.evictOldestItems(Math.floor(this.cacheConfig.maxSize * 0.1)); // Evict 10%
207
+ }
208
+
209
+ const now = new Date();
210
+ const item: CacheItem<R> = {
211
+ key,
212
+ value,
213
+ createdAt: now,
214
+ accessedAt: now,
215
+ accessCount: 1,
216
+ size: this.calculateSize(value)
217
+ };
218
+
219
+ this.cache.set(key, item as any);
220
+ }
221
+
222
+ /**
223
+ * Remove from cache
224
+ */
225
+ removeFromCache(key: string): boolean {
226
+ return this.cache.delete(key);
227
+ }
228
+
229
+ /**
230
+ * Clear entire cache
231
+ */
232
+ clearCache(): void {
233
+ this.cache.clear();
234
+ }
235
+
236
+ /**
237
+ * Get batch statistics
238
+ */
239
+ getBatchStats(): BatchStats & {
240
+ cache: CacheMetrics;
241
+ processing: {
242
+ totalProcessed: number;
243
+ averageProcessingTime: number;
244
+ };
245
+ } {
246
+ const activeBatches = Array.from(this.batches.entries());
247
+ const totalPendingItems = activeBatches.reduce((sum, [, batch]) => sum + batch.length, 0);
248
+
249
+ return {
250
+ totalUsers: activeBatches.length,
251
+ totalPendingVectors: totalPendingItems, // Generic pending items
252
+ activeBatchJobs: this.timers.size,
253
+ cacheHitRate: this.getCacheHitRate(),
254
+ averageBatchSize: activeBatches.length > 0
255
+ ? totalPendingItems / activeBatches.length
256
+ : 0,
257
+ averageProcessingTime: this.metrics.averageProcessingTime,
258
+ cache: this.getCacheMetrics(),
259
+ processing: {
260
+ totalProcessed: this.metrics.totalProcessed,
261
+ averageProcessingTime: this.metrics.averageProcessingTime
262
+ }
263
+ };
264
+ }
265
+
266
+ /**
267
+ * Get pending batch info
268
+ */
269
+ getPendingBatches(): Array<{
270
+ batchType: string;
271
+ itemCount: number;
272
+ oldestItem: Date;
273
+ scheduledProcessing: Date | null;
274
+ }> {
275
+ return Array.from(this.batches.entries()).map(([batchType, batch]) => {
276
+ const oldestItem = batch.reduce((oldest, item) =>
277
+ item.timestamp < oldest ? item.timestamp : oldest,
278
+ batch[0]?.timestamp || new Date()
279
+ );
280
+
281
+ return {
282
+ batchType,
283
+ itemCount: batch.length,
284
+ oldestItem,
285
+ scheduledProcessing: null // TODO: Track scheduled processing time
286
+ };
287
+ });
288
+ }
289
+
290
+ /**
291
+ * Force process all pending batches
292
+ */
293
+ async processAllBatches(): Promise<void> {
294
+ const batchTypes = Array.from(this.batches.keys());
295
+
296
+ for (const batchType of batchTypes) {
297
+ try {
298
+ await this.processBatchNow(batchType);
299
+ } catch (error) {
300
+ console.error(`Failed to process batch ${batchType}:`, error);
301
+ }
302
+ }
303
+ }
304
+
305
+ /**
306
+ * Cleanup and destroy service
307
+ */
308
+ destroy(): void {
309
+ // Clear all timers
310
+ for (const timer of this.timers.values()) {
311
+ clearTimeout(timer);
312
+ }
313
+ this.timers.clear();
314
+
315
+ // Clear cleanup timer
316
+ if (this.cleanupTimer) {
317
+ clearInterval(this.cleanupTimer);
318
+ }
319
+
320
+ // Clear data
321
+ this.batches.clear();
322
+ this.processors.clear();
323
+ this.cache.clear();
324
+
325
+ console.log('BatchingService destroyed');
326
+ }
327
+
328
+ // ==================== PRIVATE METHODS ====================
329
+
330
+ private scheduleBatchProcessing(batchType: string, forceImmediate = false): void {
331
+ const batch = this.batches.get(batchType);
332
+ if (!batch) return;
333
+
334
+ // Clear existing timer
335
+ const existingTimer = this.timers.get(batchType);
336
+ if (existingTimer) {
337
+ clearTimeout(existingTimer);
338
+ }
339
+
340
+ // Process immediately if conditions met
341
+ if (forceImmediate || batch.length >= this.batchConfig.maxBatchSize) {
342
+ setImmediate(() => this.processBatchNow(batchType));
343
+ return;
344
+ }
345
+
346
+ // Schedule delayed processing
347
+ const timer = setTimeout(
348
+ () => this.processBatchNow(batchType),
349
+ this.batchConfig.batchDelayMs
350
+ );
351
+
352
+ this.timers.set(batchType, timer);
353
+ }
354
+
355
+ private startCacheCleanup(): void {
356
+ this.cleanupTimer = setInterval(() => {
357
+ this.cleanupExpiredItems();
358
+ }, this.cacheConfig.cleanupIntervalMs);
359
+ }
360
+
361
+ private cleanupExpiredItems(): void {
362
+ const now = Date.now();
363
+ let cleaned = 0;
364
+
365
+ for (const [key, item] of this.cache.entries()) {
366
+ if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
367
+ this.cache.delete(key);
368
+ cleaned++;
369
+ }
370
+ }
371
+
372
+ if (cleaned > 0) {
373
+ console.debug(`Cleaned up ${cleaned} expired cache items`);
374
+ }
375
+ }
376
+
377
+ private evictOldestItems(count: number): void {
378
+ const items = Array.from(this.cache.entries())
379
+ .sort(([, a], [, b]) => a.accessedAt.getTime() - b.accessedAt.getTime())
380
+ .slice(0, count);
381
+
382
+ for (const [key] of items) {
383
+ this.cache.delete(key);
384
+ }
385
+
386
+ console.debug(`Evicted ${items.length} oldest cache items`);
387
+ }
388
+
389
+ private getCacheHitRate(): number {
390
+ const totalRequests = this.metrics.hitCount + this.metrics.missCount;
391
+ return totalRequests > 0 ? this.metrics.hitCount / totalRequests : 0;
392
+ }
393
+
394
+ private getCacheMetrics(): CacheMetrics {
395
+ const items = Array.from(this.cache.values());
396
+
397
+ return {
398
+ totalItems: items.length,
399
+ totalSize: items.reduce((sum, item) => sum + (item.size || 0), 0),
400
+ hitCount: this.metrics.hitCount,
401
+ missCount: this.metrics.missCount,
402
+ hitRate: this.getCacheHitRate(),
403
+ averageAccessCount: items.length > 0
404
+ ? items.reduce((sum, item) => sum + item.accessCount, 0) / items.length
405
+ : 0,
406
+ oldestItem: items.length > 0
407
+ ? items.reduce((oldest, item) =>
408
+ item.createdAt < oldest ? item.createdAt : oldest,
409
+ items[0].createdAt
410
+ )
411
+ : undefined,
412
+ newestItem: items.length > 0
413
+ ? items.reduce((newest, item) =>
414
+ item.createdAt > newest ? item.createdAt : newest,
415
+ items[0].createdAt
416
+ )
417
+ : undefined
418
+ };
419
+ }
420
+
421
+ private calculateSize(value: any): number {
422
+ try {
423
+ return JSON.stringify(value).length;
424
+ } catch {
425
+ return 0;
426
+ }
427
+ }
428
+ }
429
+
430
430
  export default BatchingService;