@soulcraft/brainy 6.2.5 β 6.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/storage/adapters/azureBlobStorage.d.ts +2 -8
- package/dist/storage/adapters/azureBlobStorage.js +17 -48
- package/dist/storage/adapters/gcsStorage.d.ts +2 -8
- package/dist/storage/adapters/gcsStorage.js +17 -48
- package/dist/storage/adapters/r2Storage.d.ts +5 -8
- package/dist/storage/adapters/r2Storage.js +19 -40
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +1 -8
- package/dist/storage/adapters/s3CompatibleStorage.js +10 -86
- package/package.json +1 -1
|
@@ -52,10 +52,6 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
52
52
|
private nounWriteBuffer;
|
|
53
53
|
private verbWriteBuffer;
|
|
54
54
|
private requestCoalescer;
|
|
55
|
-
private highVolumeMode;
|
|
56
|
-
private lastVolumeCheck;
|
|
57
|
-
private volumeCheckInterval;
|
|
58
|
-
private forceHighVolumeMode;
|
|
59
55
|
private nounCacheManager;
|
|
60
56
|
private verbCacheManager;
|
|
61
57
|
private logger;
|
|
@@ -155,10 +151,6 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
155
151
|
* @param requestId Request ID from applyBackpressure()
|
|
156
152
|
*/
|
|
157
153
|
private releaseBackpressure;
|
|
158
|
-
/**
|
|
159
|
-
* Check if high-volume mode should be enabled
|
|
160
|
-
*/
|
|
161
|
-
private checkVolumeMode;
|
|
162
154
|
/**
|
|
163
155
|
* Flush noun buffer to Azure
|
|
164
156
|
*/
|
|
@@ -169,6 +161,7 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
169
161
|
private flushVerbBuffer;
|
|
170
162
|
/**
|
|
171
163
|
* Save a node to storage
|
|
164
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
172
165
|
*/
|
|
173
166
|
protected saveNode(node: HNSWNode): Promise<void>;
|
|
174
167
|
/**
|
|
@@ -231,6 +224,7 @@ export declare class AzureBlobStorage extends BaseStorage {
|
|
|
231
224
|
private streamToBuffer;
|
|
232
225
|
/**
|
|
233
226
|
* Save an edge to storage
|
|
227
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
234
228
|
*/
|
|
235
229
|
protected saveEdge(edge: Edge): Promise<void>;
|
|
236
230
|
/**
|
|
@@ -59,11 +59,6 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
59
59
|
this.verbWriteBuffer = null;
|
|
60
60
|
// Request coalescer for deduplication
|
|
61
61
|
this.requestCoalescer = null;
|
|
62
|
-
// High-volume mode detection
|
|
63
|
-
this.highVolumeMode = false;
|
|
64
|
-
this.lastVolumeCheck = 0;
|
|
65
|
-
this.volumeCheckInterval = 1000; // Check every second
|
|
66
|
-
this.forceHighVolumeMode = false; // Environment variable override
|
|
67
62
|
// Module logger
|
|
68
63
|
this.logger = createModuleLogger('AzureBlobStorage');
|
|
69
64
|
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
|
@@ -83,12 +78,7 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
83
78
|
// Initialize cache managers
|
|
84
79
|
this.nounCacheManager = new CacheManager(options.cacheConfig);
|
|
85
80
|
this.verbCacheManager = new CacheManager(options.cacheConfig);
|
|
86
|
-
//
|
|
87
|
-
if (typeof process !== 'undefined' && process.env?.BRAINY_FORCE_HIGH_VOLUME === 'true') {
|
|
88
|
-
this.forceHighVolumeMode = true;
|
|
89
|
-
this.highVolumeMode = true;
|
|
90
|
-
prodLog.info('π High-volume mode FORCED via BRAINY_FORCE_HIGH_VOLUME environment variable');
|
|
91
|
-
}
|
|
81
|
+
// v6.2.7: Write buffering always enabled - no env var check needed
|
|
92
82
|
}
|
|
93
83
|
/**
|
|
94
84
|
* Get Azure Blob-optimized batch configuration with native batch API support
|
|
@@ -326,29 +316,7 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
326
316
|
this.backpressure.releasePermission(requestId, success);
|
|
327
317
|
}
|
|
328
318
|
}
|
|
329
|
-
|
|
330
|
-
* Check if high-volume mode should be enabled
|
|
331
|
-
*/
|
|
332
|
-
checkVolumeMode() {
|
|
333
|
-
if (this.forceHighVolumeMode) {
|
|
334
|
-
return; // Already forced on
|
|
335
|
-
}
|
|
336
|
-
const now = Date.now();
|
|
337
|
-
if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
|
|
338
|
-
return;
|
|
339
|
-
}
|
|
340
|
-
this.lastVolumeCheck = now;
|
|
341
|
-
// Enable high-volume mode if we have many pending operations
|
|
342
|
-
const shouldEnable = this.pendingOperations > 20;
|
|
343
|
-
if (shouldEnable && !this.highVolumeMode) {
|
|
344
|
-
this.highVolumeMode = true;
|
|
345
|
-
prodLog.info('π High-volume mode ENABLED (pending operations:', this.pendingOperations, ')');
|
|
346
|
-
}
|
|
347
|
-
else if (!shouldEnable && this.highVolumeMode && !this.forceHighVolumeMode) {
|
|
348
|
-
this.highVolumeMode = false;
|
|
349
|
-
prodLog.info('π High-volume mode DISABLED (pending operations:', this.pendingOperations, ')');
|
|
350
|
-
}
|
|
351
|
-
}
|
|
319
|
+
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
|
352
320
|
/**
|
|
353
321
|
* Flush noun buffer to Azure
|
|
354
322
|
*/
|
|
@@ -380,21 +348,21 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
380
348
|
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
|
|
381
349
|
/**
|
|
382
350
|
* Save a node to storage
|
|
351
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
383
352
|
*/
|
|
384
353
|
async saveNode(node) {
|
|
385
354
|
await this.ensureInitialized();
|
|
386
|
-
//
|
|
387
|
-
this.
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
355
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
356
|
+
if (this.nounWriteBuffer) {
|
|
357
|
+
this.logger.trace(`π BUFFERING: Adding noun ${node.id} to write buffer`);
|
|
358
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
359
|
+
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
|
360
|
+
this.nounCacheManager.set(node.id, node);
|
|
361
|
+
}
|
|
391
362
|
await this.nounWriteBuffer.add(node.id, node);
|
|
392
363
|
return;
|
|
393
364
|
}
|
|
394
|
-
|
|
395
|
-
this.logger.trace(`π DIRECT WRITE: Saving noun ${node.id} directly (high-volume mode inactive)`);
|
|
396
|
-
}
|
|
397
|
-
// Direct write in normal mode
|
|
365
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
398
366
|
await this.saveNodeDirect(node);
|
|
399
367
|
}
|
|
400
368
|
/**
|
|
@@ -771,18 +739,19 @@ export class AzureBlobStorage extends BaseStorage {
|
|
|
771
739
|
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
|
|
772
740
|
/**
|
|
773
741
|
* Save an edge to storage
|
|
742
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
774
743
|
*/
|
|
775
744
|
async saveEdge(edge) {
|
|
776
745
|
await this.ensureInitialized();
|
|
777
|
-
//
|
|
778
|
-
this.
|
|
779
|
-
// Use write buffer in high-volume mode
|
|
780
|
-
if (this.highVolumeMode && this.verbWriteBuffer) {
|
|
746
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
747
|
+
if (this.verbWriteBuffer) {
|
|
781
748
|
this.logger.trace(`π BUFFERING: Adding verb ${edge.id} to write buffer`);
|
|
749
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
750
|
+
this.verbCacheManager.set(edge.id, edge);
|
|
782
751
|
await this.verbWriteBuffer.add(edge.id, edge);
|
|
783
752
|
return;
|
|
784
753
|
}
|
|
785
|
-
//
|
|
754
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
786
755
|
await this.saveEdgeDirect(edge);
|
|
787
756
|
}
|
|
788
757
|
/**
|
|
@@ -54,10 +54,6 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
54
54
|
private nounWriteBuffer;
|
|
55
55
|
private verbWriteBuffer;
|
|
56
56
|
private requestCoalescer;
|
|
57
|
-
private highVolumeMode;
|
|
58
|
-
private lastVolumeCheck;
|
|
59
|
-
private volumeCheckInterval;
|
|
60
|
-
private forceHighVolumeMode;
|
|
61
57
|
private nounCacheManager;
|
|
62
58
|
private verbCacheManager;
|
|
63
59
|
private logger;
|
|
@@ -133,10 +129,6 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
133
129
|
* @param requestId Request ID from applyBackpressure()
|
|
134
130
|
*/
|
|
135
131
|
private releaseBackpressure;
|
|
136
|
-
/**
|
|
137
|
-
* Check if high-volume mode should be enabled
|
|
138
|
-
*/
|
|
139
|
-
private checkVolumeMode;
|
|
140
132
|
/**
|
|
141
133
|
* Flush noun buffer to GCS
|
|
142
134
|
*/
|
|
@@ -147,6 +139,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
147
139
|
private flushVerbBuffer;
|
|
148
140
|
/**
|
|
149
141
|
* Save a node to storage
|
|
142
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
150
143
|
*/
|
|
151
144
|
protected saveNode(node: HNSWNode): Promise<void>;
|
|
152
145
|
/**
|
|
@@ -212,6 +205,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
212
205
|
protected listObjectsUnderPath(prefix: string): Promise<string[]>;
|
|
213
206
|
/**
|
|
214
207
|
* Save an edge to storage
|
|
208
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
215
209
|
*/
|
|
216
210
|
protected saveEdge(edge: Edge): Promise<void>;
|
|
217
211
|
/**
|
|
@@ -63,11 +63,6 @@ export class GcsStorage extends BaseStorage {
|
|
|
63
63
|
this.verbWriteBuffer = null;
|
|
64
64
|
// Request coalescer for deduplication
|
|
65
65
|
this.requestCoalescer = null;
|
|
66
|
-
// High-volume mode detection - MUCH more aggressive
|
|
67
|
-
this.highVolumeMode = false;
|
|
68
|
-
this.lastVolumeCheck = 0;
|
|
69
|
-
this.volumeCheckInterval = 1000; // Check every second, not 5
|
|
70
|
-
this.forceHighVolumeMode = false; // Environment variable override
|
|
71
66
|
// Module logger
|
|
72
67
|
this.logger = createModuleLogger('GcsStorage');
|
|
73
68
|
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
|
@@ -92,12 +87,7 @@ export class GcsStorage extends BaseStorage {
|
|
|
92
87
|
// Initialize cache managers
|
|
93
88
|
this.nounCacheManager = new CacheManager(options.cacheConfig);
|
|
94
89
|
this.verbCacheManager = new CacheManager(options.cacheConfig);
|
|
95
|
-
//
|
|
96
|
-
if (typeof process !== 'undefined' && process.env?.BRAINY_FORCE_HIGH_VOLUME === 'true') {
|
|
97
|
-
this.forceHighVolumeMode = true;
|
|
98
|
-
this.highVolumeMode = true;
|
|
99
|
-
prodLog.info('π High-volume mode FORCED via BRAINY_FORCE_HIGH_VOLUME environment variable');
|
|
100
|
-
}
|
|
90
|
+
// v6.2.7: Write buffering always enabled - no env var check needed
|
|
101
91
|
}
|
|
102
92
|
/**
|
|
103
93
|
* Initialize the storage adapter
|
|
@@ -251,29 +241,7 @@ export class GcsStorage extends BaseStorage {
|
|
|
251
241
|
this.backpressure.releasePermission(requestId, success);
|
|
252
242
|
}
|
|
253
243
|
}
|
|
254
|
-
|
|
255
|
-
* Check if high-volume mode should be enabled
|
|
256
|
-
*/
|
|
257
|
-
checkVolumeMode() {
|
|
258
|
-
if (this.forceHighVolumeMode) {
|
|
259
|
-
return; // Already forced on
|
|
260
|
-
}
|
|
261
|
-
const now = Date.now();
|
|
262
|
-
if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
|
|
263
|
-
return;
|
|
264
|
-
}
|
|
265
|
-
this.lastVolumeCheck = now;
|
|
266
|
-
// Enable high-volume mode if we have many pending operations
|
|
267
|
-
const shouldEnable = this.pendingOperations > 20;
|
|
268
|
-
if (shouldEnable && !this.highVolumeMode) {
|
|
269
|
-
this.highVolumeMode = true;
|
|
270
|
-
prodLog.info('π High-volume mode ENABLED (pending operations:', this.pendingOperations, ')');
|
|
271
|
-
}
|
|
272
|
-
else if (!shouldEnable && this.highVolumeMode && !this.forceHighVolumeMode) {
|
|
273
|
-
this.highVolumeMode = false;
|
|
274
|
-
prodLog.info('π High-volume mode DISABLED (pending operations:', this.pendingOperations, ')');
|
|
275
|
-
}
|
|
276
|
-
}
|
|
244
|
+
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
|
277
245
|
/**
|
|
278
246
|
* Flush noun buffer to GCS
|
|
279
247
|
*/
|
|
@@ -305,21 +273,21 @@ export class GcsStorage extends BaseStorage {
|
|
|
305
273
|
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
|
|
306
274
|
/**
|
|
307
275
|
* Save a node to storage
|
|
276
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
308
277
|
*/
|
|
309
278
|
async saveNode(node) {
|
|
310
279
|
await this.ensureInitialized();
|
|
311
|
-
//
|
|
312
|
-
this.
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
280
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
281
|
+
if (this.nounWriteBuffer) {
|
|
282
|
+
this.logger.trace(`π BUFFERING: Adding noun ${node.id} to write buffer`);
|
|
283
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
284
|
+
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
|
285
|
+
this.nounCacheManager.set(node.id, node);
|
|
286
|
+
}
|
|
316
287
|
await this.nounWriteBuffer.add(node.id, node);
|
|
317
288
|
return;
|
|
318
289
|
}
|
|
319
|
-
|
|
320
|
-
this.logger.trace(`π DIRECT WRITE: Saving noun ${node.id} directly (high-volume mode inactive)`);
|
|
321
|
-
}
|
|
322
|
-
// Direct write in normal mode
|
|
290
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
323
291
|
await this.saveNodeDirect(node);
|
|
324
292
|
}
|
|
325
293
|
/**
|
|
@@ -640,18 +608,19 @@ export class GcsStorage extends BaseStorage {
|
|
|
640
608
|
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
|
|
641
609
|
/**
|
|
642
610
|
* Save an edge to storage
|
|
611
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
643
612
|
*/
|
|
644
613
|
async saveEdge(edge) {
|
|
645
614
|
await this.ensureInitialized();
|
|
646
|
-
//
|
|
647
|
-
this.
|
|
648
|
-
// Use write buffer in high-volume mode
|
|
649
|
-
if (this.highVolumeMode && this.verbWriteBuffer) {
|
|
615
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
616
|
+
if (this.verbWriteBuffer) {
|
|
650
617
|
this.logger.trace(`π BUFFERING: Adding verb ${edge.id} to write buffer`);
|
|
618
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
619
|
+
this.verbCacheManager.set(edge.id, edge);
|
|
651
620
|
await this.verbWriteBuffer.add(edge.id, edge);
|
|
652
621
|
return;
|
|
653
622
|
}
|
|
654
|
-
//
|
|
623
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
655
624
|
await this.saveEdgeDirect(edge);
|
|
656
625
|
}
|
|
657
626
|
/**
|
|
@@ -58,10 +58,6 @@ export declare class R2Storage extends BaseStorage {
|
|
|
58
58
|
private nounWriteBuffer;
|
|
59
59
|
private verbWriteBuffer;
|
|
60
60
|
private requestCoalescer;
|
|
61
|
-
private highVolumeMode;
|
|
62
|
-
private lastVolumeCheck;
|
|
63
|
-
private volumeCheckInterval;
|
|
64
|
-
private forceHighVolumeMode;
|
|
65
61
|
private nounCacheManager;
|
|
66
62
|
private verbCacheManager;
|
|
67
63
|
private logger;
|
|
@@ -139,10 +135,6 @@ export declare class R2Storage extends BaseStorage {
|
|
|
139
135
|
* Release backpressure after completing an operation
|
|
140
136
|
*/
|
|
141
137
|
private releaseBackpressure;
|
|
142
|
-
/**
|
|
143
|
-
* Check if high-volume mode should be enabled
|
|
144
|
-
*/
|
|
145
|
-
private checkVolumeMode;
|
|
146
138
|
/**
|
|
147
139
|
* Flush noun buffer to R2
|
|
148
140
|
*/
|
|
@@ -153,6 +145,7 @@ export declare class R2Storage extends BaseStorage {
|
|
|
153
145
|
private flushVerbBuffer;
|
|
154
146
|
/**
|
|
155
147
|
* Save a node to storage
|
|
148
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
156
149
|
*/
|
|
157
150
|
protected saveNode(node: HNSWNode): Promise<void>;
|
|
158
151
|
/**
|
|
@@ -179,6 +172,10 @@ export declare class R2Storage extends BaseStorage {
|
|
|
179
172
|
* List all objects under a specific prefix in R2
|
|
180
173
|
*/
|
|
181
174
|
protected listObjectsUnderPath(prefix: string): Promise<string[]>;
|
|
175
|
+
/**
|
|
176
|
+
* Save an edge to storage
|
|
177
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
178
|
+
*/
|
|
182
179
|
protected saveEdge(edge: Edge): Promise<void>;
|
|
183
180
|
private saveEdgeDirect;
|
|
184
181
|
protected getEdge(id: string): Promise<Edge | null>;
|
|
@@ -65,11 +65,6 @@ export class R2Storage extends BaseStorage {
|
|
|
65
65
|
this.verbWriteBuffer = null;
|
|
66
66
|
// Request coalescer for deduplication
|
|
67
67
|
this.requestCoalescer = null;
|
|
68
|
-
// High-volume mode detection (R2-specific thresholds)
|
|
69
|
-
this.highVolumeMode = false;
|
|
70
|
-
this.lastVolumeCheck = 0;
|
|
71
|
-
this.volumeCheckInterval = 800; // Check more frequently on R2
|
|
72
|
-
this.forceHighVolumeMode = false;
|
|
73
68
|
// Module logger
|
|
74
69
|
this.logger = createModuleLogger('R2Storage');
|
|
75
70
|
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
|
@@ -94,12 +89,7 @@ export class R2Storage extends BaseStorage {
|
|
|
94
89
|
warmCacheTTL: options.cacheConfig?.warmCacheTTL || 3600000 // 1 hour
|
|
95
90
|
});
|
|
96
91
|
this.verbCacheManager = new CacheManager(options.cacheConfig);
|
|
97
|
-
//
|
|
98
|
-
if (typeof process !== 'undefined' && process.env?.BRAINY_FORCE_HIGH_VOLUME === 'true') {
|
|
99
|
-
this.forceHighVolumeMode = true;
|
|
100
|
-
this.highVolumeMode = true;
|
|
101
|
-
prodLog.info('π R2: High-volume mode FORCED via environment variable');
|
|
102
|
-
}
|
|
92
|
+
// v6.2.7: Write buffering always enabled - no env var check needed
|
|
103
93
|
}
|
|
104
94
|
/**
|
|
105
95
|
* Get R2-optimized batch configuration with native batch API support
|
|
@@ -295,29 +285,7 @@ export class R2Storage extends BaseStorage {
|
|
|
295
285
|
this.backpressure.releasePermission(requestId, success);
|
|
296
286
|
}
|
|
297
287
|
}
|
|
298
|
-
|
|
299
|
-
* Check if high-volume mode should be enabled
|
|
300
|
-
*/
|
|
301
|
-
checkVolumeMode() {
|
|
302
|
-
if (this.forceHighVolumeMode) {
|
|
303
|
-
return;
|
|
304
|
-
}
|
|
305
|
-
const now = Date.now();
|
|
306
|
-
if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
|
|
307
|
-
return;
|
|
308
|
-
}
|
|
309
|
-
this.lastVolumeCheck = now;
|
|
310
|
-
// R2 threshold: enable at 15 pending operations (lower than S3/GCS)
|
|
311
|
-
const shouldEnable = this.pendingOperations > 15;
|
|
312
|
-
if (shouldEnable && !this.highVolumeMode) {
|
|
313
|
-
this.highVolumeMode = true;
|
|
314
|
-
prodLog.info('π R2: High-volume mode ENABLED (pending:', this.pendingOperations, ')');
|
|
315
|
-
}
|
|
316
|
-
else if (!shouldEnable && this.highVolumeMode && !this.forceHighVolumeMode) {
|
|
317
|
-
this.highVolumeMode = false;
|
|
318
|
-
prodLog.info('π R2: High-volume mode DISABLED (pending:', this.pendingOperations, ')');
|
|
319
|
-
}
|
|
320
|
-
}
|
|
288
|
+
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
|
321
289
|
/**
|
|
322
290
|
* Flush noun buffer to R2
|
|
323
291
|
*/
|
|
@@ -348,17 +316,21 @@ export class R2Storage extends BaseStorage {
|
|
|
348
316
|
}
|
|
349
317
|
/**
|
|
350
318
|
* Save a node to storage
|
|
319
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
351
320
|
*/
|
|
352
321
|
async saveNode(node) {
|
|
353
322
|
await this.ensureInitialized();
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
if (this.highVolumeMode && this.nounWriteBuffer) {
|
|
323
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
324
|
+
if (this.nounWriteBuffer) {
|
|
357
325
|
this.logger.trace(`π BUFFERING: Adding noun ${node.id} to write buffer`);
|
|
326
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
327
|
+
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
|
328
|
+
this.nounCacheManager.set(node.id, node);
|
|
329
|
+
}
|
|
358
330
|
await this.nounWriteBuffer.add(node.id, node);
|
|
359
331
|
return;
|
|
360
332
|
}
|
|
361
|
-
//
|
|
333
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
362
334
|
await this.saveNodeDirect(node);
|
|
363
335
|
}
|
|
364
336
|
/**
|
|
@@ -567,13 +539,20 @@ export class R2Storage extends BaseStorage {
|
|
|
567
539
|
}
|
|
568
540
|
}
|
|
569
541
|
// Verb storage methods (similar to noun methods - implementing key methods for space)
|
|
542
|
+
/**
|
|
543
|
+
* Save an edge to storage
|
|
544
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
545
|
+
*/
|
|
570
546
|
async saveEdge(edge) {
|
|
571
547
|
await this.ensureInitialized();
|
|
572
|
-
|
|
573
|
-
if (this.
|
|
548
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
549
|
+
if (this.verbWriteBuffer) {
|
|
550
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
551
|
+
this.verbCacheManager.set(edge.id, edge);
|
|
574
552
|
await this.verbWriteBuffer.add(edge.id, edge);
|
|
575
553
|
return;
|
|
576
554
|
}
|
|
555
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
577
556
|
await this.saveEdgeDirect(edge);
|
|
578
557
|
}
|
|
579
558
|
async saveEdgeDirect(edge) {
|
|
@@ -73,10 +73,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
73
73
|
private cacheSync?;
|
|
74
74
|
private readWriteSeparation?;
|
|
75
75
|
private requestCoalescer;
|
|
76
|
-
private highVolumeMode;
|
|
77
|
-
private lastVolumeCheck;
|
|
78
|
-
private volumeCheckInterval;
|
|
79
|
-
private forceHighVolumeMode;
|
|
80
76
|
private operationExecutors;
|
|
81
77
|
private nounCacheManager;
|
|
82
78
|
private verbCacheManager;
|
|
@@ -193,10 +189,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
193
189
|
* Initialize request coalescer
|
|
194
190
|
*/
|
|
195
191
|
private initializeCoalescer;
|
|
196
|
-
/**
|
|
197
|
-
* Check if we should enable high-volume mode
|
|
198
|
-
*/
|
|
199
|
-
private checkVolumeMode;
|
|
200
192
|
/**
|
|
201
193
|
* Bulk write nouns to S3
|
|
202
194
|
*/
|
|
@@ -239,6 +231,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
239
231
|
private getBatchSize;
|
|
240
232
|
/**
|
|
241
233
|
* Save a node to storage
|
|
234
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
242
235
|
*/
|
|
243
236
|
protected saveNode(node: HNSWNode): Promise<void>;
|
|
244
237
|
/**
|
|
@@ -78,11 +78,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
78
78
|
// ShardManager is no longer used - sharding is deterministic
|
|
79
79
|
// Request coalescer for deduplication
|
|
80
80
|
this.requestCoalescer = null;
|
|
81
|
-
// High-volume mode detection - MUCH more aggressive
|
|
82
|
-
this.highVolumeMode = false;
|
|
83
|
-
this.lastVolumeCheck = 0;
|
|
84
|
-
this.volumeCheckInterval = 1000; // Check every second, not 5
|
|
85
|
-
this.forceHighVolumeMode = false; // Environment variable override
|
|
86
81
|
// Module logger
|
|
87
82
|
this.logger = createModuleLogger('S3Storage');
|
|
88
83
|
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
|
@@ -526,79 +521,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
526
521
|
await this.processCoalescedBatch(batch);
|
|
527
522
|
});
|
|
528
523
|
}
|
|
529
|
-
|
|
530
|
-
* Check if we should enable high-volume mode
|
|
531
|
-
*/
|
|
532
|
-
checkVolumeMode() {
|
|
533
|
-
const now = Date.now();
|
|
534
|
-
if (now - this.lastVolumeCheck < this.volumeCheckInterval) {
|
|
535
|
-
return;
|
|
536
|
-
}
|
|
537
|
-
this.lastVolumeCheck = now;
|
|
538
|
-
// Check environment variable override
|
|
539
|
-
const envThreshold = process.env.BRAINY_BUFFER_THRESHOLD;
|
|
540
|
-
const threshold = envThreshold ? parseInt(envThreshold) : 0; // Default to 0 for immediate activation!
|
|
541
|
-
// Force enable from environment
|
|
542
|
-
if (process.env.BRAINY_FORCE_BUFFERING === 'true') {
|
|
543
|
-
this.forceHighVolumeMode = true;
|
|
544
|
-
}
|
|
545
|
-
// Get metrics
|
|
546
|
-
const backpressureStatus = this.backpressure.getStatus();
|
|
547
|
-
const socketMetrics = this.socketManager.getMetrics();
|
|
548
|
-
// Reasonable high-volume detection - only activate under real load
|
|
549
|
-
const isTestEnvironment = process.env.NODE_ENV === 'test';
|
|
550
|
-
const explicitlyDisabled = process.env.BRAINY_FORCE_BUFFERING === 'false';
|
|
551
|
-
// Use reasonable thresholds instead of emergency aggressive ones
|
|
552
|
-
const reasonableThreshold = Math.max(threshold, 10); // At least 10 pending operations
|
|
553
|
-
const highSocketUtilization = 0.8; // 80% socket utilization
|
|
554
|
-
const highRequestRate = 50; // 50 requests per second
|
|
555
|
-
const significantErrors = 5; // 5 consecutive errors
|
|
556
|
-
const shouldEnableHighVolume = !isTestEnvironment && // Disable in test environment
|
|
557
|
-
!explicitlyDisabled && // Allow explicit disabling
|
|
558
|
-
(this.forceHighVolumeMode || // Environment override
|
|
559
|
-
backpressureStatus.queueLength >= reasonableThreshold || // High queue backlog
|
|
560
|
-
socketMetrics.pendingRequests >= reasonableThreshold || // Many pending requests
|
|
561
|
-
this.pendingOperations >= reasonableThreshold || // Many pending ops
|
|
562
|
-
socketMetrics.socketUtilization >= highSocketUtilization || // High socket pressure
|
|
563
|
-
(socketMetrics.requestsPerSecond >= highRequestRate) || // High request rate
|
|
564
|
-
(this.consecutiveErrors >= significantErrors)); // Significant error pattern
|
|
565
|
-
if (shouldEnableHighVolume && !this.highVolumeMode) {
|
|
566
|
-
this.highVolumeMode = true;
|
|
567
|
-
this.logger.warn(`π¨ HIGH-VOLUME MODE ACTIVATED π¨`);
|
|
568
|
-
this.logger.warn(` Queue Length: ${backpressureStatus.queueLength}`);
|
|
569
|
-
this.logger.warn(` Pending Requests: ${socketMetrics.pendingRequests}`);
|
|
570
|
-
this.logger.warn(` Pending Operations: ${this.pendingOperations}`);
|
|
571
|
-
this.logger.warn(` Socket Utilization: ${(socketMetrics.socketUtilization * 100).toFixed(1)}%`);
|
|
572
|
-
this.logger.warn(` Requests/sec: ${socketMetrics.requestsPerSecond}`);
|
|
573
|
-
this.logger.warn(` Consecutive Errors: ${this.consecutiveErrors}`);
|
|
574
|
-
this.logger.warn(` Threshold: ${threshold}`);
|
|
575
|
-
// Adjust buffer parameters for high volume
|
|
576
|
-
const queueLength = Math.max(backpressureStatus.queueLength, socketMetrics.pendingRequests, 100);
|
|
577
|
-
if (this.nounWriteBuffer) {
|
|
578
|
-
this.nounWriteBuffer.adjustForLoad(queueLength);
|
|
579
|
-
const stats = this.nounWriteBuffer.getStats();
|
|
580
|
-
this.logger.warn(` Noun Buffer: ${stats.bufferSize} items, ${stats.totalWrites} total writes`);
|
|
581
|
-
}
|
|
582
|
-
if (this.verbWriteBuffer) {
|
|
583
|
-
this.verbWriteBuffer.adjustForLoad(queueLength);
|
|
584
|
-
const stats = this.verbWriteBuffer.getStats();
|
|
585
|
-
this.logger.warn(` Verb Buffer: ${stats.bufferSize} items, ${stats.totalWrites} total writes`);
|
|
586
|
-
}
|
|
587
|
-
if (this.requestCoalescer) {
|
|
588
|
-
this.requestCoalescer.adjustParameters(queueLength);
|
|
589
|
-
const sizes = this.requestCoalescer.getQueueSizes();
|
|
590
|
-
this.logger.warn(` Coalescer: ${sizes.total} queued operations`);
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
else if (!shouldEnableHighVolume && this.highVolumeMode && !this.forceHighVolumeMode) {
|
|
594
|
-
this.highVolumeMode = false;
|
|
595
|
-
this.logger.info('β
High-volume mode deactivated - load normalized');
|
|
596
|
-
}
|
|
597
|
-
// Log current status every 10 checks when in high-volume mode
|
|
598
|
-
if (this.highVolumeMode && (now % 10000) < this.volumeCheckInterval) {
|
|
599
|
-
this.logger.info(`π High-volume mode status: Queue=${backpressureStatus.queueLength}, Pending=${socketMetrics.pendingRequests}, Sockets=${(socketMetrics.socketUtilization * 100).toFixed(1)}%`);
|
|
600
|
-
}
|
|
601
|
-
}
|
|
524
|
+
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
|
|
602
525
|
/**
|
|
603
526
|
* Bulk write nouns to S3
|
|
604
527
|
*/
|
|
@@ -802,20 +725,21 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
802
725
|
// v5.4.0: Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
|
|
803
726
|
/**
|
|
804
727
|
* Save a node to storage
|
|
728
|
+
* v6.2.7: Always uses write buffer for consistent performance
|
|
805
729
|
*/
|
|
806
730
|
async saveNode(node) {
|
|
807
731
|
await this.ensureInitialized();
|
|
808
|
-
//
|
|
809
|
-
this.
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
732
|
+
// v6.2.7: Always use write buffer - cloud storage benefits from batching
|
|
733
|
+
if (this.nounWriteBuffer) {
|
|
734
|
+
this.logger.trace(`π BUFFERING: Adding noun ${node.id} to write buffer`);
|
|
735
|
+
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
|
|
736
|
+
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
|
|
737
|
+
this.nounCacheManager.set(node.id, node);
|
|
738
|
+
}
|
|
813
739
|
await this.nounWriteBuffer.add(node.id, node);
|
|
814
740
|
return;
|
|
815
741
|
}
|
|
816
|
-
|
|
817
|
-
this.logger.trace(`π DIRECT WRITE: Saving noun ${node.id} directly (high-volume mode inactive)`);
|
|
818
|
-
}
|
|
742
|
+
// Fallback to direct write if buffer not initialized (shouldn't happen after init)
|
|
819
743
|
// Apply backpressure before starting operation
|
|
820
744
|
const requestId = await this.applyBackpressure();
|
|
821
745
|
try {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soulcraft/brainy",
|
|
3
|
-
"version": "6.2.
|
|
3
|
+
"version": "6.2.7",
|
|
4
4
|
"description": "Universal Knowledge Protocolβ’ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns Γ 127 verbs covering 96-97% of all human knowledge.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.js",
|