@cmdoss/memwal-sdk 0.9.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/README.md +157 -52
  2. package/dist/client/ClientMemoryManager.d.ts.map +1 -1
  3. package/dist/client/ClientMemoryManager.js +25 -8
  4. package/dist/client/ClientMemoryManager.js.map +1 -1
  5. package/dist/client/PersonalDataWallet.d.ts.map +1 -1
  6. package/dist/client/SimplePDWClient.d.ts +2 -1
  7. package/dist/client/SimplePDWClient.d.ts.map +1 -1
  8. package/dist/client/SimplePDWClient.js +23 -6
  9. package/dist/client/SimplePDWClient.js.map +1 -1
  10. package/dist/client/namespaces/MemoryNamespace.d.ts +6 -0
  11. package/dist/client/namespaces/MemoryNamespace.d.ts.map +1 -1
  12. package/dist/client/namespaces/MemoryNamespace.js +131 -18
  13. package/dist/client/namespaces/MemoryNamespace.js.map +1 -1
  14. package/dist/client/namespaces/consolidated/StorageNamespace.d.ts +3 -1
  15. package/dist/client/namespaces/consolidated/StorageNamespace.d.ts.map +1 -1
  16. package/dist/client/namespaces/consolidated/StorageNamespace.js.map +1 -1
  17. package/dist/config/ConfigurationHelper.js +61 -61
  18. package/dist/config/index.d.ts +1 -0
  19. package/dist/config/index.d.ts.map +1 -1
  20. package/dist/config/index.js +2 -0
  21. package/dist/config/index.js.map +1 -1
  22. package/dist/config/modelDefaults.d.ts +67 -0
  23. package/dist/config/modelDefaults.d.ts.map +1 -0
  24. package/dist/config/modelDefaults.js +91 -0
  25. package/dist/config/modelDefaults.js.map +1 -0
  26. package/dist/graph/GraphService.d.ts.map +1 -1
  27. package/dist/graph/GraphService.js +22 -21
  28. package/dist/graph/GraphService.js.map +1 -1
  29. package/dist/index.d.ts +1 -1
  30. package/dist/index.d.ts.map +1 -1
  31. package/dist/index.js +1 -1
  32. package/dist/index.js.map +1 -1
  33. package/dist/langchain/createPDWRAG.js +30 -30
  34. package/dist/pipeline/MemoryPipeline.d.ts.map +1 -1
  35. package/dist/pipeline/MemoryPipeline.js +2 -1
  36. package/dist/pipeline/MemoryPipeline.js.map +1 -1
  37. package/dist/services/GeminiAIService.d.ts.map +1 -1
  38. package/dist/services/GeminiAIService.js +311 -310
  39. package/dist/services/GeminiAIService.js.map +1 -1
  40. package/dist/services/StorageService.d.ts +4 -1
  41. package/dist/services/StorageService.d.ts.map +1 -1
  42. package/dist/services/StorageService.js.map +1 -1
  43. package/dist/services/storage/QuiltBatchManager.d.ts +7 -0
  44. package/dist/services/storage/QuiltBatchManager.d.ts.map +1 -1
  45. package/dist/services/storage/QuiltBatchManager.js +24 -5
  46. package/dist/services/storage/QuiltBatchManager.js.map +1 -1
  47. package/dist/services/storage/WalrusStorageManager.d.ts +10 -1
  48. package/dist/services/storage/WalrusStorageManager.d.ts.map +1 -1
  49. package/dist/services/storage/WalrusStorageManager.js +53 -12
  50. package/dist/services/storage/WalrusStorageManager.js.map +1 -1
  51. package/dist/vector/BrowserHnswIndexService.js +2 -2
  52. package/dist/vector/BrowserHnswIndexService.js.map +1 -1
  53. package/dist/vector/NodeHnswService.js +4 -4
  54. package/dist/vector/NodeHnswService.js.map +1 -1
  55. package/dist/vector/createHnswService.d.ts +4 -0
  56. package/dist/vector/createHnswService.d.ts.map +1 -1
  57. package/dist/vector/createHnswService.js +15 -3
  58. package/dist/vector/createHnswService.js.map +1 -1
  59. package/package.json +1 -1
  60. package/src/access/PermissionService.ts +635 -635
  61. package/src/aggregation/AggregationService.ts +389 -389
  62. package/src/ai-sdk/PDWVectorStore.ts +715 -715
  63. package/src/ai-sdk/index.ts +65 -65
  64. package/src/ai-sdk/tools.ts +460 -460
  65. package/src/ai-sdk/types.ts +404 -404
  66. package/src/batch/BatchManager.ts +597 -597
  67. package/src/batch/BatchingService.ts +429 -429
  68. package/src/batch/MemoryProcessingCache.ts +492 -492
  69. package/src/batch/index.ts +30 -30
  70. package/src/browser.ts +200 -200
  71. package/src/client/ClientMemoryManager.ts +1004 -987
  72. package/src/client/PersonalDataWallet.ts +345 -345
  73. package/src/client/SimplePDWClient.ts +1387 -1369
  74. package/src/client/factory.ts +154 -154
  75. package/src/client/namespaces/AnalyticsNamespace.ts +377 -377
  76. package/src/client/namespaces/BatchNamespace.ts +356 -356
  77. package/src/client/namespaces/CacheNamespace.ts +123 -123
  78. package/src/client/namespaces/CapabilityNamespace.ts +217 -217
  79. package/src/client/namespaces/ClassifyNamespace.ts +169 -169
  80. package/src/client/namespaces/ContextNamespace.ts +297 -297
  81. package/src/client/namespaces/EncryptionNamespace.ts +221 -221
  82. package/src/client/namespaces/GraphNamespace.ts +468 -468
  83. package/src/client/namespaces/IndexNamespace.ts +364 -364
  84. package/src/client/namespaces/MemoryNamespace.ts +1704 -1569
  85. package/src/client/namespaces/PermissionsNamespace.ts +254 -254
  86. package/src/client/namespaces/PipelineNamespace.ts +220 -220
  87. package/src/client/namespaces/StorageNamespace.ts +458 -458
  88. package/src/client/namespaces/TxNamespace.ts +260 -260
  89. package/src/client/namespaces/WalletNamespace.ts +243 -243
  90. package/src/client/namespaces/consolidated/BlockchainNamespace.ts +607 -607
  91. package/src/client/namespaces/consolidated/SecurityNamespace.ts +648 -648
  92. package/src/client/namespaces/consolidated/StorageNamespace.ts +1143 -1141
  93. package/src/client/namespaces/consolidated/index.ts +41 -41
  94. package/src/client/signers/KeypairSigner.ts +108 -108
  95. package/src/client/signers/UnifiedSigner.ts +110 -110
  96. package/src/client/signers/WalletAdapterSigner.ts +159 -159
  97. package/src/client/signers/index.ts +26 -26
  98. package/src/config/ConfigurationHelper.ts +412 -412
  99. package/src/config/defaults.ts +56 -56
  100. package/src/config/index.ts +16 -9
  101. package/src/config/modelDefaults.ts +103 -0
  102. package/src/config/validation.ts +70 -70
  103. package/src/core/index.ts +14 -14
  104. package/src/core/interfaces/IService.ts +307 -307
  105. package/src/core/interfaces/index.ts +8 -8
  106. package/src/core/types/capability.ts +297 -297
  107. package/src/core/types/index.ts +874 -874
  108. package/src/core/types/wallet.ts +270 -270
  109. package/src/core/types.ts +9 -9
  110. package/src/core/wallet.ts +222 -222
  111. package/src/embedding/index.ts +19 -19
  112. package/src/embedding/types.ts +357 -357
  113. package/src/errors/index.ts +602 -602
  114. package/src/errors/recovery.ts +461 -461
  115. package/src/errors/validation.ts +567 -567
  116. package/src/generated/pdw/capability.ts +319 -319
  117. package/src/graph/GraphService.ts +888 -887
  118. package/src/graph/KnowledgeGraphManager.ts +728 -728
  119. package/src/graph/index.ts +25 -25
  120. package/src/index.ts +498 -498
  121. package/src/infrastructure/index.ts +22 -22
  122. package/src/infrastructure/seal/EncryptionService.ts +628 -628
  123. package/src/infrastructure/seal/SealService.ts +613 -613
  124. package/src/infrastructure/seal/index.ts +9 -9
  125. package/src/infrastructure/sui/BlockchainManager.ts +627 -627
  126. package/src/infrastructure/sui/SuiService.ts +888 -888
  127. package/src/infrastructure/sui/index.ts +9 -9
  128. package/src/infrastructure/walrus/StorageManager.ts +604 -604
  129. package/src/infrastructure/walrus/WalrusStorageService.ts +637 -637
  130. package/src/infrastructure/walrus/index.ts +9 -9
  131. package/src/langchain/createPDWRAG.ts +303 -303
  132. package/src/langchain/index.ts +47 -47
  133. package/src/permissions/ConsentRepository.browser.ts +249 -249
  134. package/src/permissions/ConsentRepository.ts +364 -364
  135. package/src/pipeline/MemoryPipeline.ts +863 -862
  136. package/src/pipeline/PipelineManager.ts +683 -683
  137. package/src/pipeline/index.ts +26 -26
  138. package/src/retrieval/AdvancedSearchService.ts +629 -629
  139. package/src/retrieval/MemoryAnalyticsService.ts +711 -711
  140. package/src/retrieval/MemoryDecryptionPipeline.ts +825 -825
  141. package/src/retrieval/index.ts +42 -42
  142. package/src/services/BatchService.ts +352 -352
  143. package/src/services/CapabilityService.ts +464 -464
  144. package/src/services/ClassifierService.ts +465 -465
  145. package/src/services/CrossContextPermissionService.ts +486 -486
  146. package/src/services/EmbeddingService.ts +796 -796
  147. package/src/services/EncryptionService.ts +712 -712
  148. package/src/services/GeminiAIService.ts +754 -753
  149. package/src/services/MemoryIndexService.ts +1009 -1009
  150. package/src/services/MemoryService.ts +369 -369
  151. package/src/services/QueryService.ts +890 -890
  152. package/src/services/StorageService.ts +1185 -1182
  153. package/src/services/TransactionService.ts +838 -838
  154. package/src/services/VectorService.ts +462 -462
  155. package/src/services/ViewService.ts +484 -484
  156. package/src/services/index.ts +25 -25
  157. package/src/services/storage/BlobAttributesManager.ts +333 -333
  158. package/src/services/storage/KnowledgeGraphManager.ts +425 -425
  159. package/src/services/storage/MemorySearchManager.ts +387 -387
  160. package/src/services/storage/QuiltBatchManager.ts +1157 -1130
  161. package/src/services/storage/WalrusMetadataManager.ts +268 -268
  162. package/src/services/storage/WalrusStorageManager.ts +333 -287
  163. package/src/services/storage/index.ts +57 -57
  164. package/src/types/index.ts +13 -13
  165. package/src/utils/index.ts +76 -76
  166. package/src/utils/memoryIndexOnChain.ts +507 -507
  167. package/src/vector/BrowserHnswIndexService.ts +758 -758
  168. package/src/vector/HnswWasmService.ts +731 -731
  169. package/src/vector/IHnswService.ts +233 -233
  170. package/src/vector/NodeHnswService.ts +833 -833
  171. package/src/vector/createHnswService.ts +147 -135
  172. package/src/vector/index.ts +56 -56
  173. package/src/wallet/ContextWalletService.ts +656 -656
  174. package/src/wallet/MainWalletService.ts +317 -317
@@ -1,430 +1,430 @@
1
- /**
2
- * BatchingService - Intelligent Batch Processing & Caching
3
- *
4
- * Provides advanced batching capabilities with intelligent scheduling,
5
- * cache management, and performance optimization for the PDW SDK.
6
- */
7
-
8
- import { BatchConfig, BatchJob, BatchStats } from '../embedding/types';
9
-
10
- export interface BatchItem<T = any> {
11
- id: string;
12
- data: T;
13
- timestamp: Date;
14
- priority?: number;
15
- metadata?: any;
16
- }
17
-
18
- export interface BatchProcessor<T = any> {
19
- process(items: BatchItem<T>[]): Promise<void>;
20
- }
21
-
22
- export interface CacheConfig {
23
- maxSize?: number;
24
- ttlMs?: number;
25
- cleanupIntervalMs?: number;
26
- enableMetrics?: boolean;
27
- }
28
-
29
- export interface CacheItem<T = any> {
30
- key: string;
31
- value: T;
32
- createdAt: Date;
33
- accessedAt: Date;
34
- accessCount: number;
35
- size?: number;
36
- }
37
-
38
- export interface CacheMetrics {
39
- totalItems: number;
40
- totalSize: number;
41
- hitCount: number;
42
- missCount: number;
43
- hitRate: number;
44
- averageAccessCount: number;
45
- oldestItem?: Date;
46
- newestItem?: Date;
47
- }
48
-
49
- /**
50
- * Advanced batching service with intelligent scheduling and caching
51
- */
52
- export class BatchingService<T = any> {
53
- private batches = new Map<string, BatchItem<T>[]>();
54
- private processors = new Map<string, BatchProcessor<T>>();
55
- private timers = new Map<string, NodeJS.Timeout>();
56
- private cache = new Map<string, CacheItem<T>>();
57
- private metrics = {
58
- hitCount: 0,
59
- missCount: 0,
60
- totalProcessed: 0,
61
- averageProcessingTime: 0
62
- };
63
-
64
- private readonly batchConfig: Required<BatchConfig>;
65
- private readonly cacheConfig: Required<CacheConfig>;
66
- private cleanupTimer?: NodeJS.Timeout;
67
-
68
- constructor(
69
- batchConfig: Partial<BatchConfig> = {},
70
- cacheConfig: Partial<CacheConfig> = {}
71
- ) {
72
- this.batchConfig = {
73
- maxBatchSize: batchConfig.maxBatchSize || 50,
74
- batchDelayMs: batchConfig.batchDelayMs || 5000,
75
- maxCacheSize: batchConfig.maxCacheSize || 1000,
76
- cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
77
- };
78
-
79
- this.cacheConfig = {
80
- maxSize: cacheConfig.maxSize || this.batchConfig.maxCacheSize,
81
- ttlMs: cacheConfig.ttlMs || this.batchConfig.cacheTtlMs,
82
- cleanupIntervalMs: cacheConfig.cleanupIntervalMs || 5 * 60 * 1000, // 5 minutes
83
- enableMetrics: cacheConfig.enableMetrics !== false
84
- };
85
-
86
- this.startCacheCleanup();
87
- }
88
-
89
- /**
90
- * Register a batch processor for a specific batch type
91
- */
92
- registerProcessor(batchType: string, processor: BatchProcessor<T>): void {
93
- this.processors.set(batchType, processor);
94
- }
95
-
96
- /**
97
- * Add item to batch for processing
98
- */
99
- addToBatch(
100
- batchType: string,
101
- item: BatchItem<T>,
102
- options: {
103
- forceImmediate?: boolean;
104
- highPriority?: boolean;
105
- } = {}
106
- ): void {
107
- // Get or create batch
108
- let batch = this.batches.get(batchType);
109
- if (!batch) {
110
- batch = [];
111
- this.batches.set(batchType, batch);
112
- }
113
-
114
- // Add item to batch
115
- batch.push({
116
- ...item,
117
- priority: options.highPriority ? 1 : (item.priority || 0)
118
- });
119
-
120
- // Sort by priority (higher priority first)
121
- batch.sort((a, b) => (b.priority || 0) - (a.priority || 0));
122
-
123
- console.debug(`Added item to batch ${batchType}. Size: ${batch.length}`);
124
-
125
- // Schedule processing
126
- this.scheduleBatchProcessing(batchType, options.forceImmediate);
127
- }
128
-
129
- /**
130
- * Process batch immediately
131
- */
132
- async processBatchNow(batchType: string): Promise<void> {
133
- const batch = this.batches.get(batchType);
134
- if (!batch || batch.length === 0) {
135
- return;
136
- }
137
-
138
- const processor = this.processors.get(batchType);
139
- if (!processor) {
140
- console.warn(`No processor registered for batch type: ${batchType}`);
141
- return;
142
- }
143
-
144
- const startTime = Date.now();
145
-
146
- try {
147
- console.log(`Processing batch ${batchType} with ${batch.length} items`);
148
-
149
- // Process the batch
150
- await processor.process([...batch]);
151
-
152
- // Clear batch and timer
153
- this.batches.delete(batchType);
154
- const timer = this.timers.get(batchType);
155
- if (timer) {
156
- clearTimeout(timer);
157
- this.timers.delete(batchType);
158
- }
159
-
160
- // Update metrics
161
- this.metrics.totalProcessed += batch.length;
162
- const processingTime = Date.now() - startTime;
163
- this.metrics.averageProcessingTime =
164
- (this.metrics.averageProcessingTime + processingTime) / 2;
165
-
166
- console.log(`Successfully processed batch ${batchType} in ${processingTime}ms`);
167
- } catch (error) {
168
- console.error(`Error processing batch ${batchType}:`, error);
169
- throw error;
170
- }
171
- }
172
-
173
- /**
174
- * Get cached value
175
- */
176
- getFromCache<R = T>(key: string): R | undefined {
177
- const item = this.cache.get(key) as CacheItem<R> | undefined;
178
-
179
- if (!item) {
180
- this.metrics.missCount++;
181
- return undefined;
182
- }
183
-
184
- // Check TTL
185
- const now = Date.now();
186
- if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
187
- this.cache.delete(key);
188
- this.metrics.missCount++;
189
- return undefined;
190
- }
191
-
192
- // Update access info
193
- item.accessedAt = new Date();
194
- item.accessCount++;
195
- this.metrics.hitCount++;
196
-
197
- return item.value;
198
- }
199
-
200
- /**
201
- * Set cached value
202
- */
203
- setInCache<R = T>(key: string, value: R, metadata?: any): void {
204
- // Check cache size limit
205
- if (this.cache.size >= this.cacheConfig.maxSize) {
206
- this.evictOldestItems(Math.floor(this.cacheConfig.maxSize * 0.1)); // Evict 10%
207
- }
208
-
209
- const now = new Date();
210
- const item: CacheItem<R> = {
211
- key,
212
- value,
213
- createdAt: now,
214
- accessedAt: now,
215
- accessCount: 1,
216
- size: this.calculateSize(value)
217
- };
218
-
219
- this.cache.set(key, item as any);
220
- }
221
-
222
- /**
223
- * Remove from cache
224
- */
225
- removeFromCache(key: string): boolean {
226
- return this.cache.delete(key);
227
- }
228
-
229
- /**
230
- * Clear entire cache
231
- */
232
- clearCache(): void {
233
- this.cache.clear();
234
- }
235
-
236
- /**
237
- * Get batch statistics
238
- */
239
- getBatchStats(): BatchStats & {
240
- cache: CacheMetrics;
241
- processing: {
242
- totalProcessed: number;
243
- averageProcessingTime: number;
244
- };
245
- } {
246
- const activeBatches = Array.from(this.batches.entries());
247
- const totalPendingItems = activeBatches.reduce((sum, [, batch]) => sum + batch.length, 0);
248
-
249
- return {
250
- totalUsers: activeBatches.length,
251
- totalPendingVectors: totalPendingItems, // Generic pending items
252
- activeBatchJobs: this.timers.size,
253
- cacheHitRate: this.getCacheHitRate(),
254
- averageBatchSize: activeBatches.length > 0
255
- ? totalPendingItems / activeBatches.length
256
- : 0,
257
- averageProcessingTime: this.metrics.averageProcessingTime,
258
- cache: this.getCacheMetrics(),
259
- processing: {
260
- totalProcessed: this.metrics.totalProcessed,
261
- averageProcessingTime: this.metrics.averageProcessingTime
262
- }
263
- };
264
- }
265
-
266
- /**
267
- * Get pending batch info
268
- */
269
- getPendingBatches(): Array<{
270
- batchType: string;
271
- itemCount: number;
272
- oldestItem: Date;
273
- scheduledProcessing: Date | null;
274
- }> {
275
- return Array.from(this.batches.entries()).map(([batchType, batch]) => {
276
- const oldestItem = batch.reduce((oldest, item) =>
277
- item.timestamp < oldest ? item.timestamp : oldest,
278
- batch[0]?.timestamp || new Date()
279
- );
280
-
281
- return {
282
- batchType,
283
- itemCount: batch.length,
284
- oldestItem,
285
- scheduledProcessing: null // TODO: Track scheduled processing time
286
- };
287
- });
288
- }
289
-
290
- /**
291
- * Force process all pending batches
292
- */
293
- async processAllBatches(): Promise<void> {
294
- const batchTypes = Array.from(this.batches.keys());
295
-
296
- for (const batchType of batchTypes) {
297
- try {
298
- await this.processBatchNow(batchType);
299
- } catch (error) {
300
- console.error(`Failed to process batch ${batchType}:`, error);
301
- }
302
- }
303
- }
304
-
305
- /**
306
- * Cleanup and destroy service
307
- */
308
- destroy(): void {
309
- // Clear all timers
310
- for (const timer of this.timers.values()) {
311
- clearTimeout(timer);
312
- }
313
- this.timers.clear();
314
-
315
- // Clear cleanup timer
316
- if (this.cleanupTimer) {
317
- clearInterval(this.cleanupTimer);
318
- }
319
-
320
- // Clear data
321
- this.batches.clear();
322
- this.processors.clear();
323
- this.cache.clear();
324
-
325
- console.log('BatchingService destroyed');
326
- }
327
-
328
- // ==================== PRIVATE METHODS ====================
329
-
330
- private scheduleBatchProcessing(batchType: string, forceImmediate = false): void {
331
- const batch = this.batches.get(batchType);
332
- if (!batch) return;
333
-
334
- // Clear existing timer
335
- const existingTimer = this.timers.get(batchType);
336
- if (existingTimer) {
337
- clearTimeout(existingTimer);
338
- }
339
-
340
- // Process immediately if conditions met
341
- if (forceImmediate || batch.length >= this.batchConfig.maxBatchSize) {
342
- setImmediate(() => this.processBatchNow(batchType));
343
- return;
344
- }
345
-
346
- // Schedule delayed processing
347
- const timer = setTimeout(
348
- () => this.processBatchNow(batchType),
349
- this.batchConfig.batchDelayMs
350
- );
351
-
352
- this.timers.set(batchType, timer);
353
- }
354
-
355
- private startCacheCleanup(): void {
356
- this.cleanupTimer = setInterval(() => {
357
- this.cleanupExpiredItems();
358
- }, this.cacheConfig.cleanupIntervalMs);
359
- }
360
-
361
- private cleanupExpiredItems(): void {
362
- const now = Date.now();
363
- let cleaned = 0;
364
-
365
- for (const [key, item] of this.cache.entries()) {
366
- if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
367
- this.cache.delete(key);
368
- cleaned++;
369
- }
370
- }
371
-
372
- if (cleaned > 0) {
373
- console.debug(`Cleaned up ${cleaned} expired cache items`);
374
- }
375
- }
376
-
377
- private evictOldestItems(count: number): void {
378
- const items = Array.from(this.cache.entries())
379
- .sort(([, a], [, b]) => a.accessedAt.getTime() - b.accessedAt.getTime())
380
- .slice(0, count);
381
-
382
- for (const [key] of items) {
383
- this.cache.delete(key);
384
- }
385
-
386
- console.debug(`Evicted ${items.length} oldest cache items`);
387
- }
388
-
389
- private getCacheHitRate(): number {
390
- const totalRequests = this.metrics.hitCount + this.metrics.missCount;
391
- return totalRequests > 0 ? this.metrics.hitCount / totalRequests : 0;
392
- }
393
-
394
- private getCacheMetrics(): CacheMetrics {
395
- const items = Array.from(this.cache.values());
396
-
397
- return {
398
- totalItems: items.length,
399
- totalSize: items.reduce((sum, item) => sum + (item.size || 0), 0),
400
- hitCount: this.metrics.hitCount,
401
- missCount: this.metrics.missCount,
402
- hitRate: this.getCacheHitRate(),
403
- averageAccessCount: items.length > 0
404
- ? items.reduce((sum, item) => sum + item.accessCount, 0) / items.length
405
- : 0,
406
- oldestItem: items.length > 0
407
- ? items.reduce((oldest, item) =>
408
- item.createdAt < oldest ? item.createdAt : oldest,
409
- items[0].createdAt
410
- )
411
- : undefined,
412
- newestItem: items.length > 0
413
- ? items.reduce((newest, item) =>
414
- item.createdAt > newest ? item.createdAt : newest,
415
- items[0].createdAt
416
- )
417
- : undefined
418
- };
419
- }
420
-
421
- private calculateSize(value: any): number {
422
- try {
423
- return JSON.stringify(value).length;
424
- } catch {
425
- return 0;
426
- }
427
- }
428
- }
429
-
1
+ /**
2
+ * BatchingService - Intelligent Batch Processing & Caching
3
+ *
4
+ * Provides advanced batching capabilities with intelligent scheduling,
5
+ * cache management, and performance optimization for the PDW SDK.
6
+ */
7
+
8
+ import { BatchConfig, BatchJob, BatchStats } from '../embedding/types';
9
+
10
+ export interface BatchItem<T = any> {
11
+ id: string;
12
+ data: T;
13
+ timestamp: Date;
14
+ priority?: number;
15
+ metadata?: any;
16
+ }
17
+
18
+ export interface BatchProcessor<T = any> {
19
+ process(items: BatchItem<T>[]): Promise<void>;
20
+ }
21
+
22
+ export interface CacheConfig {
23
+ maxSize?: number;
24
+ ttlMs?: number;
25
+ cleanupIntervalMs?: number;
26
+ enableMetrics?: boolean;
27
+ }
28
+
29
+ export interface CacheItem<T = any> {
30
+ key: string;
31
+ value: T;
32
+ createdAt: Date;
33
+ accessedAt: Date;
34
+ accessCount: number;
35
+ size?: number;
36
+ }
37
+
38
+ export interface CacheMetrics {
39
+ totalItems: number;
40
+ totalSize: number;
41
+ hitCount: number;
42
+ missCount: number;
43
+ hitRate: number;
44
+ averageAccessCount: number;
45
+ oldestItem?: Date;
46
+ newestItem?: Date;
47
+ }
48
+
49
+ /**
50
+ * Advanced batching service with intelligent scheduling and caching
51
+ */
52
+ export class BatchingService<T = any> {
53
+ private batches = new Map<string, BatchItem<T>[]>();
54
+ private processors = new Map<string, BatchProcessor<T>>();
55
+ private timers = new Map<string, NodeJS.Timeout>();
56
+ private cache = new Map<string, CacheItem<T>>();
57
+ private metrics = {
58
+ hitCount: 0,
59
+ missCount: 0,
60
+ totalProcessed: 0,
61
+ averageProcessingTime: 0
62
+ };
63
+
64
+ private readonly batchConfig: Required<BatchConfig>;
65
+ private readonly cacheConfig: Required<CacheConfig>;
66
+ private cleanupTimer?: NodeJS.Timeout;
67
+
68
+ constructor(
69
+ batchConfig: Partial<BatchConfig> = {},
70
+ cacheConfig: Partial<CacheConfig> = {}
71
+ ) {
72
+ this.batchConfig = {
73
+ maxBatchSize: batchConfig.maxBatchSize || 50,
74
+ batchDelayMs: batchConfig.batchDelayMs || 5000,
75
+ maxCacheSize: batchConfig.maxCacheSize || 1000,
76
+ cacheTtlMs: batchConfig.cacheTtlMs || 30 * 60 * 1000 // 30 minutes
77
+ };
78
+
79
+ this.cacheConfig = {
80
+ maxSize: cacheConfig.maxSize || this.batchConfig.maxCacheSize,
81
+ ttlMs: cacheConfig.ttlMs || this.batchConfig.cacheTtlMs,
82
+ cleanupIntervalMs: cacheConfig.cleanupIntervalMs || 5 * 60 * 1000, // 5 minutes
83
+ enableMetrics: cacheConfig.enableMetrics !== false
84
+ };
85
+
86
+ this.startCacheCleanup();
87
+ }
88
+
89
+ /**
90
+ * Register a batch processor for a specific batch type
91
+ */
92
+ registerProcessor(batchType: string, processor: BatchProcessor<T>): void {
93
+ this.processors.set(batchType, processor);
94
+ }
95
+
96
+ /**
97
+ * Add item to batch for processing
98
+ */
99
+ addToBatch(
100
+ batchType: string,
101
+ item: BatchItem<T>,
102
+ options: {
103
+ forceImmediate?: boolean;
104
+ highPriority?: boolean;
105
+ } = {}
106
+ ): void {
107
+ // Get or create batch
108
+ let batch = this.batches.get(batchType);
109
+ if (!batch) {
110
+ batch = [];
111
+ this.batches.set(batchType, batch);
112
+ }
113
+
114
+ // Add item to batch
115
+ batch.push({
116
+ ...item,
117
+ priority: options.highPriority ? 1 : (item.priority || 0)
118
+ });
119
+
120
+ // Sort by priority (higher priority first)
121
+ batch.sort((a, b) => (b.priority || 0) - (a.priority || 0));
122
+
123
+ console.debug(`Added item to batch ${batchType}. Size: ${batch.length}`);
124
+
125
+ // Schedule processing
126
+ this.scheduleBatchProcessing(batchType, options.forceImmediate);
127
+ }
128
+
129
+ /**
130
+ * Process batch immediately
131
+ */
132
+ async processBatchNow(batchType: string): Promise<void> {
133
+ const batch = this.batches.get(batchType);
134
+ if (!batch || batch.length === 0) {
135
+ return;
136
+ }
137
+
138
+ const processor = this.processors.get(batchType);
139
+ if (!processor) {
140
+ console.warn(`No processor registered for batch type: ${batchType}`);
141
+ return;
142
+ }
143
+
144
+ const startTime = Date.now();
145
+
146
+ try {
147
+ console.log(`Processing batch ${batchType} with ${batch.length} items`);
148
+
149
+ // Process the batch
150
+ await processor.process([...batch]);
151
+
152
+ // Clear batch and timer
153
+ this.batches.delete(batchType);
154
+ const timer = this.timers.get(batchType);
155
+ if (timer) {
156
+ clearTimeout(timer);
157
+ this.timers.delete(batchType);
158
+ }
159
+
160
+ // Update metrics
161
+ this.metrics.totalProcessed += batch.length;
162
+ const processingTime = Date.now() - startTime;
163
+ this.metrics.averageProcessingTime =
164
+ (this.metrics.averageProcessingTime + processingTime) / 2;
165
+
166
+ console.log(`Successfully processed batch ${batchType} in ${processingTime}ms`);
167
+ } catch (error) {
168
+ console.error(`Error processing batch ${batchType}:`, error);
169
+ throw error;
170
+ }
171
+ }
172
+
173
+ /**
174
+ * Get cached value
175
+ */
176
+ getFromCache<R = T>(key: string): R | undefined {
177
+ const item = this.cache.get(key) as CacheItem<R> | undefined;
178
+
179
+ if (!item) {
180
+ this.metrics.missCount++;
181
+ return undefined;
182
+ }
183
+
184
+ // Check TTL
185
+ const now = Date.now();
186
+ if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
187
+ this.cache.delete(key);
188
+ this.metrics.missCount++;
189
+ return undefined;
190
+ }
191
+
192
+ // Update access info
193
+ item.accessedAt = new Date();
194
+ item.accessCount++;
195
+ this.metrics.hitCount++;
196
+
197
+ return item.value;
198
+ }
199
+
200
+ /**
201
+ * Set cached value
202
+ */
203
+ setInCache<R = T>(key: string, value: R, metadata?: any): void {
204
+ // Check cache size limit
205
+ if (this.cache.size >= this.cacheConfig.maxSize) {
206
+ this.evictOldestItems(Math.floor(this.cacheConfig.maxSize * 0.1)); // Evict 10%
207
+ }
208
+
209
+ const now = new Date();
210
+ const item: CacheItem<R> = {
211
+ key,
212
+ value,
213
+ createdAt: now,
214
+ accessedAt: now,
215
+ accessCount: 1,
216
+ size: this.calculateSize(value)
217
+ };
218
+
219
+ this.cache.set(key, item as any);
220
+ }
221
+
222
+ /**
223
+ * Remove from cache
224
+ */
225
+ removeFromCache(key: string): boolean {
226
+ return this.cache.delete(key);
227
+ }
228
+
229
+ /**
230
+ * Clear entire cache
231
+ */
232
+ clearCache(): void {
233
+ this.cache.clear();
234
+ }
235
+
236
+ /**
237
+ * Get batch statistics
238
+ */
239
+ getBatchStats(): BatchStats & {
240
+ cache: CacheMetrics;
241
+ processing: {
242
+ totalProcessed: number;
243
+ averageProcessingTime: number;
244
+ };
245
+ } {
246
+ const activeBatches = Array.from(this.batches.entries());
247
+ const totalPendingItems = activeBatches.reduce((sum, [, batch]) => sum + batch.length, 0);
248
+
249
+ return {
250
+ totalUsers: activeBatches.length,
251
+ totalPendingVectors: totalPendingItems, // Generic pending items
252
+ activeBatchJobs: this.timers.size,
253
+ cacheHitRate: this.getCacheHitRate(),
254
+ averageBatchSize: activeBatches.length > 0
255
+ ? totalPendingItems / activeBatches.length
256
+ : 0,
257
+ averageProcessingTime: this.metrics.averageProcessingTime,
258
+ cache: this.getCacheMetrics(),
259
+ processing: {
260
+ totalProcessed: this.metrics.totalProcessed,
261
+ averageProcessingTime: this.metrics.averageProcessingTime
262
+ }
263
+ };
264
+ }
265
+
266
+ /**
267
+ * Get pending batch info
268
+ */
269
+ getPendingBatches(): Array<{
270
+ batchType: string;
271
+ itemCount: number;
272
+ oldestItem: Date;
273
+ scheduledProcessing: Date | null;
274
+ }> {
275
+ return Array.from(this.batches.entries()).map(([batchType, batch]) => {
276
+ const oldestItem = batch.reduce((oldest, item) =>
277
+ item.timestamp < oldest ? item.timestamp : oldest,
278
+ batch[0]?.timestamp || new Date()
279
+ );
280
+
281
+ return {
282
+ batchType,
283
+ itemCount: batch.length,
284
+ oldestItem,
285
+ scheduledProcessing: null // TODO: Track scheduled processing time
286
+ };
287
+ });
288
+ }
289
+
290
+ /**
291
+ * Force process all pending batches
292
+ */
293
+ async processAllBatches(): Promise<void> {
294
+ const batchTypes = Array.from(this.batches.keys());
295
+
296
+ for (const batchType of batchTypes) {
297
+ try {
298
+ await this.processBatchNow(batchType);
299
+ } catch (error) {
300
+ console.error(`Failed to process batch ${batchType}:`, error);
301
+ }
302
+ }
303
+ }
304
+
305
+ /**
306
+ * Cleanup and destroy service
307
+ */
308
+ destroy(): void {
309
+ // Clear all timers
310
+ for (const timer of this.timers.values()) {
311
+ clearTimeout(timer);
312
+ }
313
+ this.timers.clear();
314
+
315
+ // Clear cleanup timer
316
+ if (this.cleanupTimer) {
317
+ clearInterval(this.cleanupTimer);
318
+ }
319
+
320
+ // Clear data
321
+ this.batches.clear();
322
+ this.processors.clear();
323
+ this.cache.clear();
324
+
325
+ console.log('BatchingService destroyed');
326
+ }
327
+
328
+ // ==================== PRIVATE METHODS ====================
329
+
330
+ private scheduleBatchProcessing(batchType: string, forceImmediate = false): void {
331
+ const batch = this.batches.get(batchType);
332
+ if (!batch) return;
333
+
334
+ // Clear existing timer
335
+ const existingTimer = this.timers.get(batchType);
336
+ if (existingTimer) {
337
+ clearTimeout(existingTimer);
338
+ }
339
+
340
+ // Process immediately if conditions met
341
+ if (forceImmediate || batch.length >= this.batchConfig.maxBatchSize) {
342
+ setImmediate(() => this.processBatchNow(batchType));
343
+ return;
344
+ }
345
+
346
+ // Schedule delayed processing
347
+ const timer = setTimeout(
348
+ () => this.processBatchNow(batchType),
349
+ this.batchConfig.batchDelayMs
350
+ );
351
+
352
+ this.timers.set(batchType, timer);
353
+ }
354
+
355
+ private startCacheCleanup(): void {
356
+ this.cleanupTimer = setInterval(() => {
357
+ this.cleanupExpiredItems();
358
+ }, this.cacheConfig.cleanupIntervalMs);
359
+ }
360
+
361
+ private cleanupExpiredItems(): void {
362
+ const now = Date.now();
363
+ let cleaned = 0;
364
+
365
+ for (const [key, item] of this.cache.entries()) {
366
+ if (now - item.createdAt.getTime() > this.cacheConfig.ttlMs) {
367
+ this.cache.delete(key);
368
+ cleaned++;
369
+ }
370
+ }
371
+
372
+ if (cleaned > 0) {
373
+ console.debug(`Cleaned up ${cleaned} expired cache items`);
374
+ }
375
+ }
376
+
377
+ private evictOldestItems(count: number): void {
378
+ const items = Array.from(this.cache.entries())
379
+ .sort(([, a], [, b]) => a.accessedAt.getTime() - b.accessedAt.getTime())
380
+ .slice(0, count);
381
+
382
+ for (const [key] of items) {
383
+ this.cache.delete(key);
384
+ }
385
+
386
+ console.debug(`Evicted ${items.length} oldest cache items`);
387
+ }
388
+
389
+ private getCacheHitRate(): number {
390
+ const totalRequests = this.metrics.hitCount + this.metrics.missCount;
391
+ return totalRequests > 0 ? this.metrics.hitCount / totalRequests : 0;
392
+ }
393
+
394
+ private getCacheMetrics(): CacheMetrics {
395
+ const items = Array.from(this.cache.values());
396
+
397
+ return {
398
+ totalItems: items.length,
399
+ totalSize: items.reduce((sum, item) => sum + (item.size || 0), 0),
400
+ hitCount: this.metrics.hitCount,
401
+ missCount: this.metrics.missCount,
402
+ hitRate: this.getCacheHitRate(),
403
+ averageAccessCount: items.length > 0
404
+ ? items.reduce((sum, item) => sum + item.accessCount, 0) / items.length
405
+ : 0,
406
+ oldestItem: items.length > 0
407
+ ? items.reduce((oldest, item) =>
408
+ item.createdAt < oldest ? item.createdAt : oldest,
409
+ items[0].createdAt
410
+ )
411
+ : undefined,
412
+ newestItem: items.length > 0
413
+ ? items.reduce((newest, item) =>
414
+ item.createdAt > newest ? item.createdAt : newest,
415
+ items[0].createdAt
416
+ )
417
+ : undefined
418
+ };
419
+ }
420
+
421
+ private calculateSize(value: any): number {
422
+ try {
423
+ return JSON.stringify(value).length;
424
+ } catch {
425
+ return 0;
426
+ }
427
+ }
428
+ }
429
+
430
430
  export default BatchingService;