@soulcraft/brainy 5.10.4 → 5.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brainy.d.ts +89 -0
- package/dist/brainy.js +154 -1
- package/dist/storage/adapters/azureBlobStorage.d.ts +2 -5
- package/dist/storage/adapters/azureBlobStorage.js +4 -40
- package/dist/storage/adapters/fileSystemStorage.d.ts +2 -5
- package/dist/storage/adapters/fileSystemStorage.js +4 -42
- package/dist/storage/adapters/gcsStorage.d.ts +2 -5
- package/dist/storage/adapters/gcsStorage.js +10 -47
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +2 -5
- package/dist/storage/adapters/historicalStorageAdapter.js +2 -9
- package/dist/storage/adapters/memoryStorage.d.ts +2 -5
- package/dist/storage/adapters/memoryStorage.js +2 -10
- package/dist/storage/adapters/opfsStorage.d.ts +2 -5
- package/dist/storage/adapters/opfsStorage.js +12 -54
- package/dist/storage/adapters/r2Storage.d.ts +0 -13
- package/dist/storage/adapters/r2Storage.js +18 -53
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +2 -5
- package/dist/storage/adapters/s3CompatibleStorage.js +18 -70
- package/dist/storage/baseStorage.d.ts +8 -14
- package/dist/storage/baseStorage.js +16 -40
- package/dist/storage/cow/CommitLog.d.ts +24 -0
- package/dist/storage/cow/CommitLog.js +37 -0
- package/dist/types/brainy.types.d.ts +2 -0
- package/dist/utils/paramValidation.d.ts +43 -0
- package/dist/utils/paramValidation.js +135 -22
- package/package.json +1 -1
|
@@ -42,6 +42,16 @@ export class OPFSStorage extends BaseStorage {
|
|
|
42
42
|
this.statistics = null;
|
|
43
43
|
this.activeLocks = new Set();
|
|
44
44
|
this.lockPrefix = 'opfs-lock-';
|
|
45
|
+
/**
|
|
46
|
+
* Check if COW has been explicitly disabled via clear()
|
|
47
|
+
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
|
48
|
+
* @returns true if marker file exists, false otherwise
|
|
49
|
+
* @protected
|
|
50
|
+
*/
|
|
51
|
+
/**
|
|
52
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
53
|
+
* COW is now always enabled - marker files are no longer used
|
|
54
|
+
*/
|
|
45
55
|
// Quota monitoring configuration (v4.0.0)
|
|
46
56
|
this.quotaWarningThreshold = 0.8; // Warn at 80% usage
|
|
47
57
|
this.quotaCriticalThreshold = 0.95; // Critical at 95% usage
|
|
@@ -393,18 +403,11 @@ export class OPFSStorage extends BaseStorage {
|
|
|
393
403
|
try {
|
|
394
404
|
// Delete the entire _cow/ directory (not just contents)
|
|
395
405
|
await this.rootDir.removeEntry('_cow', { recursive: true });
|
|
396
|
-
//
|
|
397
|
-
//
|
|
398
|
-
// Otherwise initializeCOW() will auto-recreate initial commit on next operation
|
|
406
|
+
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
|
407
|
+
// COW will re-initialize automatically on next use
|
|
399
408
|
this.refManager = undefined;
|
|
400
409
|
this.blobStorage = undefined;
|
|
401
410
|
this.commitLog = undefined;
|
|
402
|
-
this.cowEnabled = false;
|
|
403
|
-
// v5.10.4: Create persistent marker file (CRITICAL FIX)
|
|
404
|
-
// Bug: cowEnabled = false only affects current instance, not future instances
|
|
405
|
-
// Fix: Create marker file that persists across instance restarts
|
|
406
|
-
// When new instance calls initializeCOW(), it checks for this marker
|
|
407
|
-
await this.createClearMarker();
|
|
408
411
|
}
|
|
409
412
|
catch (error) {
|
|
410
413
|
// Ignore if _cow directory doesn't exist (not all instances use COW)
|
|
@@ -423,51 +426,6 @@ export class OPFSStorage extends BaseStorage {
|
|
|
423
426
|
throw error;
|
|
424
427
|
}
|
|
425
428
|
}
|
|
426
|
-
/**
|
|
427
|
-
* Check if COW has been explicitly disabled via clear()
|
|
428
|
-
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
|
429
|
-
* @returns true if marker file exists, false otherwise
|
|
430
|
-
* @protected
|
|
431
|
-
*/
|
|
432
|
-
async checkClearMarker() {
|
|
433
|
-
await this.ensureInitialized();
|
|
434
|
-
try {
|
|
435
|
-
// Get system directory (may not exist yet)
|
|
436
|
-
const systemDir = await this.rootDir.getDirectoryHandle('system', { create: false });
|
|
437
|
-
// Try to get the marker file
|
|
438
|
-
await systemDir.getFileHandle('cow-disabled', { create: false });
|
|
439
|
-
return true; // Marker exists
|
|
440
|
-
}
|
|
441
|
-
catch (error) {
|
|
442
|
-
if (error.name === 'NotFoundError') {
|
|
443
|
-
return false; // Marker doesn't exist (or system dir doesn't exist)
|
|
444
|
-
}
|
|
445
|
-
// Other errors (permissions, etc.) - treat as marker not existing
|
|
446
|
-
console.warn('OPFSStorage.checkClearMarker: Error checking marker', error);
|
|
447
|
-
return false;
|
|
448
|
-
}
|
|
449
|
-
}
|
|
450
|
-
/**
|
|
451
|
-
* Create marker indicating COW has been explicitly disabled
|
|
452
|
-
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
|
453
|
-
* @protected
|
|
454
|
-
*/
|
|
455
|
-
async createClearMarker() {
|
|
456
|
-
await this.ensureInitialized();
|
|
457
|
-
try {
|
|
458
|
-
// Get or create system directory
|
|
459
|
-
const systemDir = await this.rootDir.getDirectoryHandle('system', { create: true });
|
|
460
|
-
// Create empty marker file
|
|
461
|
-
const fileHandle = await systemDir.getFileHandle('cow-disabled', { create: true });
|
|
462
|
-
const writable = await fileHandle.createWritable();
|
|
463
|
-
await writable.write(new Uint8Array(0)); // Empty file
|
|
464
|
-
await writable.close();
|
|
465
|
-
}
|
|
466
|
-
catch (error) {
|
|
467
|
-
console.error('OPFSStorage.createClearMarker: Failed to create marker file', error);
|
|
468
|
-
// Don't throw - marker creation failure shouldn't break clear()
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
429
|
/**
|
|
472
430
|
* Get information about storage usage and capacity
|
|
473
431
|
*/
|
|
@@ -200,18 +200,5 @@ export declare class R2Storage extends BaseStorage {
|
|
|
200
200
|
quota: number | null;
|
|
201
201
|
details?: Record<string, any>;
|
|
202
202
|
}>;
|
|
203
|
-
/**
|
|
204
|
-
* Check if COW has been explicitly disabled via clear()
|
|
205
|
-
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
|
206
|
-
* @returns true if marker object exists, false otherwise
|
|
207
|
-
* @protected
|
|
208
|
-
*/
|
|
209
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
210
|
-
/**
|
|
211
|
-
* Create marker indicating COW has been explicitly disabled
|
|
212
|
-
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
|
213
|
-
* @protected
|
|
214
|
-
*/
|
|
215
|
-
protected createClearMarker(): Promise<void>;
|
|
216
203
|
}
|
|
217
204
|
export {};
|
|
@@ -771,27 +771,27 @@ export class R2Storage extends BaseStorage {
|
|
|
771
771
|
async clear() {
|
|
772
772
|
await this.ensureInitialized();
|
|
773
773
|
prodLog.info('🧹 R2: Clearing all data from bucket...');
|
|
774
|
-
//
|
|
775
|
-
//
|
|
776
|
-
|
|
777
|
-
for (const
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
//
|
|
774
|
+
// v5.11.0: Clear ALL data using correct paths
|
|
775
|
+
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
|
776
|
+
const branchObjects = await this.listObjectsUnderPath('branches/');
|
|
777
|
+
for (const key of branchObjects) {
|
|
778
|
+
await this.deleteObjectFromPath(key);
|
|
779
|
+
}
|
|
780
|
+
// Delete COW version control data
|
|
781
|
+
const cowObjects = await this.listObjectsUnderPath('_cow/');
|
|
782
|
+
for (const key of cowObjects) {
|
|
783
|
+
await this.deleteObjectFromPath(key);
|
|
784
|
+
}
|
|
785
|
+
// Delete system metadata
|
|
786
|
+
const systemObjects = await this.listObjectsUnderPath('_system/');
|
|
787
|
+
for (const key of systemObjects) {
|
|
788
|
+
await this.deleteObjectFromPath(key);
|
|
789
|
+
}
|
|
790
|
+
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
|
791
|
+
// COW will re-initialize automatically on next use
|
|
786
792
|
this.refManager = undefined;
|
|
787
793
|
this.blobStorage = undefined;
|
|
788
794
|
this.commitLog = undefined;
|
|
789
|
-
this.cowEnabled = false;
|
|
790
|
-
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
|
791
|
-
// Bug: cowEnabled = false only affects current instance, not future instances
|
|
792
|
-
// Fix: Create marker object that persists across instance restarts
|
|
793
|
-
// When new instance calls initializeCOW(), it checks for this marker
|
|
794
|
-
await this.createClearMarker();
|
|
795
795
|
this.nounCacheManager.clear();
|
|
796
796
|
this.verbCacheManager.clear();
|
|
797
797
|
this.totalNounCount = 0;
|
|
@@ -818,40 +818,5 @@ export class R2Storage extends BaseStorage {
|
|
|
818
818
|
}
|
|
819
819
|
};
|
|
820
820
|
}
|
|
821
|
-
/**
|
|
822
|
-
* Check if COW has been explicitly disabled via clear()
|
|
823
|
-
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
|
824
|
-
* @returns true if marker object exists, false otherwise
|
|
825
|
-
* @protected
|
|
826
|
-
*/
|
|
827
|
-
async checkClearMarker() {
|
|
828
|
-
await this.ensureInitialized();
|
|
829
|
-
try {
|
|
830
|
-
const markerPath = `${this.systemPrefix}cow-disabled`;
|
|
831
|
-
const data = await this.readObjectFromPath(markerPath);
|
|
832
|
-
return data !== null; // Marker exists if we got any data
|
|
833
|
-
}
|
|
834
|
-
catch (error) {
|
|
835
|
-
prodLog.warn('R2Storage.checkClearMarker: Error checking marker', error);
|
|
836
|
-
return false;
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
/**
|
|
840
|
-
* Create marker indicating COW has been explicitly disabled
|
|
841
|
-
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
|
842
|
-
* @protected
|
|
843
|
-
*/
|
|
844
|
-
async createClearMarker() {
|
|
845
|
-
await this.ensureInitialized();
|
|
846
|
-
try {
|
|
847
|
-
const markerPath = `${this.systemPrefix}cow-disabled`;
|
|
848
|
-
// Create empty marker object
|
|
849
|
-
await this.writeObjectToPath(markerPath, '');
|
|
850
|
-
}
|
|
851
|
-
catch (error) {
|
|
852
|
-
prodLog.error('R2Storage.createClearMarker: Failed to create marker object', error);
|
|
853
|
-
// Don't throw - marker creation failure shouldn't break clear()
|
|
854
|
-
}
|
|
855
|
-
}
|
|
856
821
|
}
|
|
857
822
|
//# sourceMappingURL=r2Storage.js.map
|
|
@@ -379,13 +379,10 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
379
379
|
* @returns true if marker object exists, false otherwise
|
|
380
380
|
* @protected
|
|
381
381
|
*/
|
|
382
|
-
protected checkClearMarker(): Promise<boolean>;
|
|
383
382
|
/**
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
* @protected
|
|
383
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
384
|
+
* COW is now always enabled - marker files are no longer used
|
|
387
385
|
*/
|
|
388
|
-
protected createClearMarker(): Promise<void>;
|
|
389
386
|
protected statisticsBatchUpdateTimerId: NodeJS.Timeout | null;
|
|
390
387
|
protected statisticsModified: boolean;
|
|
391
388
|
protected lastStatisticsFlushTime: number;
|
|
@@ -89,6 +89,16 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
89
89
|
this.hnswLocks = new Map();
|
|
90
90
|
// Node cache to avoid redundant API calls
|
|
91
91
|
this.nodeCache = new Map();
|
|
92
|
+
/**
|
|
93
|
+
* Check if COW has been explicitly disabled via clear()
|
|
94
|
+
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
|
95
|
+
* @returns true if marker object exists, false otherwise
|
|
96
|
+
* @protected
|
|
97
|
+
*/
|
|
98
|
+
/**
|
|
99
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
|
|
100
|
+
* COW is now always enabled - marker files are no longer used
|
|
101
|
+
*/
|
|
92
102
|
// Batch update timer ID
|
|
93
103
|
this.statisticsBatchUpdateTimerId = null;
|
|
94
104
|
// Flag to indicate if statistics have been modified since last save
|
|
@@ -1611,32 +1621,18 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1611
1621
|
}
|
|
1612
1622
|
}
|
|
1613
1623
|
};
|
|
1614
|
-
//
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
// Delete all objects in the noun metadata directory
|
|
1619
|
-
await deleteObjectsWithPrefix(this.metadataPrefix);
|
|
1620
|
-
// Delete all objects in the verb metadata directory
|
|
1621
|
-
await deleteObjectsWithPrefix(this.verbMetadataPrefix);
|
|
1622
|
-
// Delete all objects in the index directory
|
|
1623
|
-
await deleteObjectsWithPrefix(this.indexPrefix);
|
|
1624
|
-
// v5.6.1: Delete COW (copy-on-write) version control data
|
|
1625
|
-
// This includes all git-like versioning data (commits, trees, blobs, refs)
|
|
1626
|
-
// Must be deleted to fully clear all data including version history
|
|
1624
|
+
// v5.11.0: Clear ALL data using correct paths
|
|
1625
|
+
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
|
|
1626
|
+
await deleteObjectsWithPrefix('branches/');
|
|
1627
|
+
// Delete COW version control data
|
|
1627
1628
|
await deleteObjectsWithPrefix('_cow/');
|
|
1628
|
-
//
|
|
1629
|
-
|
|
1630
|
-
//
|
|
1629
|
+
// Delete system metadata
|
|
1630
|
+
await deleteObjectsWithPrefix('_system/');
|
|
1631
|
+
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
|
|
1632
|
+
// COW will re-initialize automatically on next use
|
|
1631
1633
|
this.refManager = undefined;
|
|
1632
1634
|
this.blobStorage = undefined;
|
|
1633
1635
|
this.commitLog = undefined;
|
|
1634
|
-
this.cowEnabled = false;
|
|
1635
|
-
// v5.10.4: Create persistent marker object (CRITICAL FIX)
|
|
1636
|
-
// Bug: cowEnabled = false only affects current instance, not future instances
|
|
1637
|
-
// Fix: Create marker object that persists across instance restarts
|
|
1638
|
-
// When new instance calls initializeCOW(), it checks for this marker
|
|
1639
|
-
await this.createClearMarker();
|
|
1640
1636
|
// Clear the statistics cache
|
|
1641
1637
|
this.statisticsCache = null;
|
|
1642
1638
|
this.statisticsModified = false;
|
|
@@ -1736,54 +1732,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1736
1732
|
};
|
|
1737
1733
|
}
|
|
1738
1734
|
}
|
|
1739
|
-
/**
|
|
1740
|
-
* Check if COW has been explicitly disabled via clear()
|
|
1741
|
-
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
|
|
1742
|
-
* @returns true if marker object exists, false otherwise
|
|
1743
|
-
* @protected
|
|
1744
|
-
*/
|
|
1745
|
-
async checkClearMarker() {
|
|
1746
|
-
await this.ensureInitialized();
|
|
1747
|
-
try {
|
|
1748
|
-
const { HeadObjectCommand } = await import('@aws-sdk/client-s3');
|
|
1749
|
-
const markerKey = `${this.systemPrefix}cow-disabled`;
|
|
1750
|
-
await this.s3Client.send(new HeadObjectCommand({
|
|
1751
|
-
Bucket: this.bucketName,
|
|
1752
|
-
Key: markerKey
|
|
1753
|
-
}));
|
|
1754
|
-
return true; // Marker exists
|
|
1755
|
-
}
|
|
1756
|
-
catch (error) {
|
|
1757
|
-
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
|
1758
|
-
return false; // Marker doesn't exist
|
|
1759
|
-
}
|
|
1760
|
-
prodLog.warn('S3CompatibleStorage.checkClearMarker: Error checking marker', error);
|
|
1761
|
-
return false;
|
|
1762
|
-
}
|
|
1763
|
-
}
|
|
1764
|
-
/**
|
|
1765
|
-
* Create marker indicating COW has been explicitly disabled
|
|
1766
|
-
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
|
1767
|
-
* @protected
|
|
1768
|
-
*/
|
|
1769
|
-
async createClearMarker() {
|
|
1770
|
-
await this.ensureInitialized();
|
|
1771
|
-
try {
|
|
1772
|
-
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
|
1773
|
-
const markerKey = `${this.systemPrefix}cow-disabled`;
|
|
1774
|
-
// Create empty marker object
|
|
1775
|
-
await this.s3Client.send(new PutObjectCommand({
|
|
1776
|
-
Bucket: this.bucketName,
|
|
1777
|
-
Key: markerKey,
|
|
1778
|
-
Body: Buffer.from(''),
|
|
1779
|
-
ContentType: 'text/plain'
|
|
1780
|
-
}));
|
|
1781
|
-
}
|
|
1782
|
-
catch (error) {
|
|
1783
|
-
prodLog.error('S3CompatibleStorage.createClearMarker: Failed to create marker object', error);
|
|
1784
|
-
// Don't throw - marker creation failure shouldn't break clear()
|
|
1785
|
-
}
|
|
1786
|
-
}
|
|
1787
1735
|
/**
|
|
1788
1736
|
* Get the statistics key for a specific date
|
|
1789
1737
|
* @param date The date to get the key for
|
|
@@ -58,7 +58,6 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
58
58
|
blobStorage?: BlobStorage;
|
|
59
59
|
commitLog?: CommitLog;
|
|
60
60
|
currentBranch: string;
|
|
61
|
-
protected cowEnabled: boolean;
|
|
62
61
|
protected nounCountsByType: Uint32Array<ArrayBuffer>;
|
|
63
62
|
protected verbCountsByType: Uint32Array<ArrayBuffer>;
|
|
64
63
|
protected nounTypeCache: Map<string, NounType>;
|
|
@@ -88,6 +87,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
88
87
|
* Called during init() to ensure all data is stored with branch prefixes from the start
|
|
89
88
|
* RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
|
|
90
89
|
* @param branch - Branch name to use (default: 'main')
|
|
90
|
+
*
|
|
91
|
+
* v5.11.0: COW is always enabled - this method now just sets the branch name (idempotent)
|
|
91
92
|
*/
|
|
92
93
|
enableCOWLightweight(branch?: string): void;
|
|
93
94
|
/**
|
|
@@ -119,6 +120,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
119
120
|
* Read object with inheritance from parent branches (COW layer)
|
|
120
121
|
* Tries current branch first, then walks commit history
|
|
121
122
|
* @protected - Available to subclasses for COW implementation
|
|
123
|
+
*
|
|
124
|
+
* v5.11.0: COW is always enabled - always use branch-scoped paths with inheritance
|
|
122
125
|
*/
|
|
123
126
|
protected readWithInheritance(path: string, branch?: string): Promise<any | null>;
|
|
124
127
|
/**
|
|
@@ -137,6 +140,8 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
137
140
|
* This enables fork to see parent's data in pagination operations
|
|
138
141
|
*
|
|
139
142
|
* Simplified approach: All branches inherit from main
|
|
143
|
+
*
|
|
144
|
+
* v5.11.0: COW is always enabled - always use inheritance
|
|
140
145
|
*/
|
|
141
146
|
protected listObjectsWithInheritance(prefix: string, branch?: string): Promise<string[]>;
|
|
142
147
|
/**
|
|
@@ -328,20 +333,9 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
328
333
|
*/
|
|
329
334
|
abstract clear(): Promise<void>;
|
|
330
335
|
/**
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
* Each adapter checks for a marker file/object (e.g., "_system/cow-disabled")
|
|
334
|
-
* @returns true if COW was disabled by clear(), false otherwise
|
|
335
|
-
* @protected
|
|
336
|
-
*/
|
|
337
|
-
protected abstract checkClearMarker(): Promise<boolean>;
|
|
338
|
-
/**
|
|
339
|
-
* Create marker indicating COW has been explicitly disabled
|
|
340
|
-
* v5.10.4: Called by clear() to prevent COW reinitialization on new instances
|
|
341
|
-
* Each adapter creates a marker file/object (e.g., "_system/cow-disabled")
|
|
342
|
-
* @protected
|
|
336
|
+
* v5.11.0: Removed checkClearMarker() and createClearMarker() abstract methods
|
|
337
|
+
* COW is now always enabled - marker files are no longer used
|
|
343
338
|
*/
|
|
344
|
-
protected abstract createClearMarker(): Promise<void>;
|
|
345
339
|
/**
|
|
346
340
|
* Get information about storage usage and capacity
|
|
347
341
|
* This method should be implemented by each specific adapter
|
|
@@ -83,7 +83,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
83
83
|
// Memory footprint: Bounded by batch size (typically <1000 items during imports)
|
|
84
84
|
this.writeCache = new Map();
|
|
85
85
|
this.currentBranch = 'main';
|
|
86
|
-
|
|
86
|
+
// v5.11.0: Removed cowEnabled flag - COW is ALWAYS enabled (mandatory, cannot be disabled)
|
|
87
87
|
// Type-first indexing support (v5.4.0)
|
|
88
88
|
// Built into all storage adapters for billion-scale efficiency
|
|
89
89
|
this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
|
|
@@ -191,13 +191,11 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
191
191
|
* Called during init() to ensure all data is stored with branch prefixes from the start
|
|
192
192
|
* RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
|
|
193
193
|
* @param branch - Branch name to use (default: 'main')
|
|
194
|
+
*
|
|
195
|
+
* v5.11.0: COW is always enabled - this method now just sets the branch name (idempotent)
|
|
194
196
|
*/
|
|
195
197
|
enableCOWLightweight(branch = 'main') {
|
|
196
|
-
if (this.cowEnabled) {
|
|
197
|
-
return;
|
|
198
|
-
}
|
|
199
198
|
this.currentBranch = branch;
|
|
200
|
-
this.cowEnabled = true;
|
|
201
199
|
// RefManager/BlobStorage/CommitLog remain undefined until first fork()
|
|
202
200
|
}
|
|
203
201
|
/**
|
|
@@ -212,27 +210,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
212
210
|
* @returns Promise that resolves when COW is initialized
|
|
213
211
|
*/
|
|
214
212
|
async initializeCOW(options) {
|
|
215
|
-
// v5.
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
|
|
219
|
-
const markerExists = await this.checkClearMarker();
|
|
220
|
-
if (markerExists) {
|
|
221
|
-
return; // COW was disabled by clear() - don't recreate _cow/ directory
|
|
222
|
-
}
|
|
223
|
-
// v5.6.1: If COW was explicitly disabled (e.g., via clear()), don't reinitialize
|
|
224
|
-
// This prevents automatic recreation of COW data after clear() operations
|
|
225
|
-
if (this.cowEnabled === false) {
|
|
213
|
+
// v5.11.0: COW is ALWAYS enabled - idempotent initialization only
|
|
214
|
+
// Removed marker file check (cowEnabled flag removed, COW is mandatory)
|
|
215
|
+
// Check if RefManager already initialized (idempotent)
|
|
216
|
+
if (this.refManager && this.blobStorage && this.commitLog) {
|
|
226
217
|
return;
|
|
227
218
|
}
|
|
228
|
-
//
|
|
229
|
-
if (
|
|
230
|
-
|
|
231
|
-
}
|
|
232
|
-
// Enable lightweight COW if not already enabled
|
|
233
|
-
if (!this.cowEnabled) {
|
|
234
|
-
this.currentBranch = options?.branch || 'main';
|
|
235
|
-
this.cowEnabled = true;
|
|
219
|
+
// Set current branch if provided
|
|
220
|
+
if (options?.branch) {
|
|
221
|
+
this.currentBranch = options.branch;
|
|
236
222
|
}
|
|
237
223
|
// Create COWStorageAdapter bridge
|
|
238
224
|
// This adapts BaseStorage's methods to the simple key-value interface
|
|
@@ -334,7 +320,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
334
320
|
await this.refManager.setHead(this.currentBranch);
|
|
335
321
|
}
|
|
336
322
|
}
|
|
337
|
-
|
|
323
|
+
// v5.11.0: COW is always enabled - no flag to set
|
|
338
324
|
}
|
|
339
325
|
/**
|
|
340
326
|
* Resolve branch-scoped path for COW isolation
|
|
@@ -348,9 +334,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
348
334
|
if (basePath.startsWith('_cow/')) {
|
|
349
335
|
return basePath; // COW metadata is global across all branches
|
|
350
336
|
}
|
|
351
|
-
|
|
352
|
-
return basePath; // COW disabled, use direct path
|
|
353
|
-
}
|
|
337
|
+
// v5.11.0: COW is always enabled - always use branch-scoped paths
|
|
354
338
|
const targetBranch = branch || this.currentBranch || 'main';
|
|
355
339
|
// Branch-scoped path: branches/<branch>/<basePath>
|
|
356
340
|
return `branches/${targetBranch}/${basePath}`;
|
|
@@ -374,17 +358,10 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
374
358
|
* Read object with inheritance from parent branches (COW layer)
|
|
375
359
|
* Tries current branch first, then walks commit history
|
|
376
360
|
* @protected - Available to subclasses for COW implementation
|
|
361
|
+
*
|
|
362
|
+
* v5.11.0: COW is always enabled - always use branch-scoped paths with inheritance
|
|
377
363
|
*/
|
|
378
364
|
async readWithInheritance(path, branch) {
|
|
379
|
-
if (!this.cowEnabled) {
|
|
380
|
-
// COW disabled: check write cache, then direct read
|
|
381
|
-
// v5.7.2: Check cache first for read-after-write consistency
|
|
382
|
-
const cachedData = this.writeCache.get(path);
|
|
383
|
-
if (cachedData !== undefined) {
|
|
384
|
-
return cachedData;
|
|
385
|
-
}
|
|
386
|
-
return this.readObjectFromPath(path);
|
|
387
|
-
}
|
|
388
365
|
const targetBranch = branch || this.currentBranch || 'main';
|
|
389
366
|
const branchPath = this.resolveBranchPath(path, targetBranch);
|
|
390
367
|
// v5.7.2: Check write cache FIRST (synchronous, instant)
|
|
@@ -461,11 +438,10 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
461
438
|
* This enables fork to see parent's data in pagination operations
|
|
462
439
|
*
|
|
463
440
|
* Simplified approach: All branches inherit from main
|
|
441
|
+
*
|
|
442
|
+
* v5.11.0: COW is always enabled - always use inheritance
|
|
464
443
|
*/
|
|
465
444
|
async listObjectsWithInheritance(prefix, branch) {
|
|
466
|
-
if (!this.cowEnabled) {
|
|
467
|
-
return this.listObjectsInBranch(prefix, branch);
|
|
468
|
-
}
|
|
469
445
|
const targetBranch = branch || this.currentBranch || 'main';
|
|
470
446
|
// Collect paths from current branch
|
|
471
447
|
const pathsSet = new Set();
|
|
@@ -115,6 +115,30 @@ export declare class CommitLog {
|
|
|
115
115
|
since?: number;
|
|
116
116
|
until?: number;
|
|
117
117
|
}): Promise<CommitObject[]>;
|
|
118
|
+
/**
|
|
119
|
+
* Stream commit history (memory-efficient for large histories)
|
|
120
|
+
*
|
|
121
|
+
* Yields commits one at a time without accumulating in memory.
|
|
122
|
+
* Use this for large commit histories (1000s of commits) where
|
|
123
|
+
* memory efficiency is important.
|
|
124
|
+
*
|
|
125
|
+
* @param ref - Starting ref
|
|
126
|
+
* @param options - Walk options
|
|
127
|
+
* @yields Commits in reverse chronological order (newest first)
|
|
128
|
+
*
|
|
129
|
+
* @example
|
|
130
|
+
* ```typescript
|
|
131
|
+
* // Stream all commits without memory accumulation
|
|
132
|
+
* for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) {
|
|
133
|
+
* console.log(commit.message)
|
|
134
|
+
* }
|
|
135
|
+
* ```
|
|
136
|
+
*/
|
|
137
|
+
streamHistory(ref: string, options?: {
|
|
138
|
+
maxCount?: number;
|
|
139
|
+
since?: number;
|
|
140
|
+
until?: number;
|
|
141
|
+
}): AsyncIterableIterator<CommitObject>;
|
|
118
142
|
/**
|
|
119
143
|
* Count commits between two commits
|
|
120
144
|
*
|
|
@@ -183,6 +183,43 @@ export class CommitLog {
|
|
|
183
183
|
}
|
|
184
184
|
return commits;
|
|
185
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Stream commit history (memory-efficient for large histories)
|
|
188
|
+
*
|
|
189
|
+
* Yields commits one at a time without accumulating in memory.
|
|
190
|
+
* Use this for large commit histories (1000s of commits) where
|
|
191
|
+
* memory efficiency is important.
|
|
192
|
+
*
|
|
193
|
+
* @param ref - Starting ref
|
|
194
|
+
* @param options - Walk options
|
|
195
|
+
* @yields Commits in reverse chronological order (newest first)
|
|
196
|
+
*
|
|
197
|
+
* @example
|
|
198
|
+
* ```typescript
|
|
199
|
+
* // Stream all commits without memory accumulation
|
|
200
|
+
* for await (const commit of commitLog.streamHistory('main', { maxCount: 10000 })) {
|
|
201
|
+
* console.log(commit.message)
|
|
202
|
+
* }
|
|
203
|
+
* ```
|
|
204
|
+
*/
|
|
205
|
+
async *streamHistory(ref, options) {
|
|
206
|
+
let count = 0;
|
|
207
|
+
for await (const commit of this.walk(ref, {
|
|
208
|
+
maxDepth: options?.maxCount,
|
|
209
|
+
until: options?.until
|
|
210
|
+
})) {
|
|
211
|
+
// Filter by since timestamp if provided
|
|
212
|
+
if (options?.since && commit.timestamp < options.since) {
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
yield commit;
|
|
216
|
+
count++;
|
|
217
|
+
// Stop after maxCount commits
|
|
218
|
+
if (options?.maxCount && count >= options.maxCount) {
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
186
223
|
/**
|
|
187
224
|
* Count commits between two commits
|
|
188
225
|
*
|
|
@@ -5,6 +5,49 @@
|
|
|
5
5
|
* Only enforces universal truths, learns everything else
|
|
6
6
|
*/
|
|
7
7
|
import { FindParams, AddParams, UpdateParams, RelateParams } from '../types/brainy.types.js';
|
|
8
|
+
/**
|
|
9
|
+
* Configuration options for ValidationConfig
|
|
10
|
+
*/
|
|
11
|
+
export interface ValidationConfigOptions {
|
|
12
|
+
/**
|
|
13
|
+
* Explicit maximum query limit override
|
|
14
|
+
* Bypasses all auto-detection
|
|
15
|
+
*/
|
|
16
|
+
maxQueryLimit?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Memory reserved for query operations (in bytes)
|
|
19
|
+
* Bypasses auto-detection but still applies safety limits
|
|
20
|
+
*/
|
|
21
|
+
reservedQueryMemory?: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Auto-configured limits based on system resources
|
|
25
|
+
* These adapt to available memory and observed performance
|
|
26
|
+
*/
|
|
27
|
+
export declare class ValidationConfig {
|
|
28
|
+
private static instance;
|
|
29
|
+
maxLimit: number;
|
|
30
|
+
maxQueryLength: number;
|
|
31
|
+
maxVectorDimensions: number;
|
|
32
|
+
limitBasis: 'override' | 'reservedMemory' | 'containerMemory' | 'freeMemory';
|
|
33
|
+
detectedContainerLimit: number | null;
|
|
34
|
+
private avgQueryTime;
|
|
35
|
+
private queryCount;
|
|
36
|
+
private constructor();
|
|
37
|
+
static getInstance(options?: ValidationConfigOptions): ValidationConfig;
|
|
38
|
+
/**
|
|
39
|
+
* Reset singleton (for testing or reconfiguration)
|
|
40
|
+
*/
|
|
41
|
+
static reset(): void;
|
|
42
|
+
/**
|
|
43
|
+
* Reconfigure with new options
|
|
44
|
+
*/
|
|
45
|
+
static reconfigure(options: ValidationConfigOptions): ValidationConfig;
|
|
46
|
+
/**
|
|
47
|
+
* Learn from actual usage to adjust limits
|
|
48
|
+
*/
|
|
49
|
+
recordQuery(duration: number, resultCount: number): void;
|
|
50
|
+
}
|
|
8
51
|
/**
|
|
9
52
|
* Universal validations - things that are always invalid
|
|
10
53
|
* These are mathematical/logical truths, not configuration
|