@soulcraft/brainy 4.11.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +271 -0
- package/README.md +38 -1
- package/dist/augmentations/brainyAugmentation.d.ts +76 -0
- package/dist/augmentations/brainyAugmentation.js +126 -0
- package/dist/augmentations/cacheAugmentation.js +9 -4
- package/dist/brainy.d.ts +248 -15
- package/dist/brainy.js +707 -17
- package/dist/cli/commands/cow.d.ts +60 -0
- package/dist/cli/commands/cow.js +444 -0
- package/dist/cli/commands/import.js +1 -1
- package/dist/cli/commands/vfs.js +24 -40
- package/dist/cli/index.js +50 -0
- package/dist/hnsw/hnswIndex.d.ts +41 -0
- package/dist/hnsw/hnswIndex.js +96 -1
- package/dist/hnsw/typeAwareHNSWIndex.d.ts +9 -0
- package/dist/hnsw/typeAwareHNSWIndex.js +22 -0
- package/dist/import/ImportHistory.js +3 -3
- package/dist/importers/VFSStructureGenerator.d.ts +1 -1
- package/dist/importers/VFSStructureGenerator.js +3 -3
- package/dist/index.d.ts +6 -0
- package/dist/index.js +10 -0
- package/dist/storage/adapters/memoryStorage.d.ts +6 -0
- package/dist/storage/adapters/memoryStorage.js +39 -14
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +31 -1
- package/dist/storage/adapters/typeAwareStorageAdapter.js +272 -43
- package/dist/storage/baseStorage.d.ts +64 -0
- package/dist/storage/baseStorage.js +252 -12
- package/dist/storage/cow/BlobStorage.d.ts +232 -0
- package/dist/storage/cow/BlobStorage.js +437 -0
- package/dist/storage/cow/CommitLog.d.ts +199 -0
- package/dist/storage/cow/CommitLog.js +363 -0
- package/dist/storage/cow/CommitObject.d.ts +276 -0
- package/dist/storage/cow/CommitObject.js +431 -0
- package/dist/storage/cow/RefManager.d.ts +213 -0
- package/dist/storage/cow/RefManager.js +409 -0
- package/dist/storage/cow/TreeObject.d.ts +177 -0
- package/dist/storage/cow/TreeObject.js +293 -0
- package/dist/storage/storageFactory.d.ts +6 -0
- package/dist/storage/storageFactory.js +92 -74
- package/dist/types/brainy.types.d.ts +1 -0
- package/dist/vfs/FSCompat.d.ts +1 -1
- package/dist/vfs/FSCompat.js +1 -1
- package/dist/vfs/VirtualFileSystem.js +5 -6
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -15,6 +15,7 @@ import { storageCommands } from './commands/storage.js';
|
|
|
15
15
|
import { nlpCommands } from './commands/nlp.js';
|
|
16
16
|
import { insightsCommands } from './commands/insights.js';
|
|
17
17
|
import { importCommands } from './commands/import.js';
|
|
18
|
+
import { cowCommands } from './commands/cow.js';
|
|
18
19
|
import { readFileSync } from 'fs';
|
|
19
20
|
import { fileURLToPath } from 'url';
|
|
20
21
|
import { dirname, join } from 'path';
|
|
@@ -521,6 +522,55 @@ program
|
|
|
521
522
|
.option('--operations <ops>', 'Operations to benchmark', 'all')
|
|
522
523
|
.option('--iterations <n>', 'Number of iterations', '100')
|
|
523
524
|
.action(utilityCommands.benchmark);
|
|
525
|
+
// ===== COW Commands (v5.0.0) - Instant Fork & Branching =====
|
|
526
|
+
program
|
|
527
|
+
.command('fork [name]')
|
|
528
|
+
.description('🚀 Fork the brain (instant clone in 1-2 seconds)')
|
|
529
|
+
.option('--message <msg>', 'Commit message')
|
|
530
|
+
.option('--author <name>', 'Author name')
|
|
531
|
+
.action(cowCommands.fork);
|
|
532
|
+
program
|
|
533
|
+
.command('branch')
|
|
534
|
+
.description('🌿 Branch management')
|
|
535
|
+
.addCommand(new Command('list')
|
|
536
|
+
.alias('ls')
|
|
537
|
+
.description('List all branches/forks')
|
|
538
|
+
.action((options) => {
|
|
539
|
+
cowCommands.branchList(options);
|
|
540
|
+
}))
|
|
541
|
+
.addCommand(new Command('delete')
|
|
542
|
+
.alias('rm')
|
|
543
|
+
.argument('[name]', 'Branch name to delete')
|
|
544
|
+
.description('Delete a branch/fork')
|
|
545
|
+
.option('-f, --force', 'Skip confirmation')
|
|
546
|
+
.action((name, options) => {
|
|
547
|
+
cowCommands.branchDelete(name, options);
|
|
548
|
+
}));
|
|
549
|
+
program
|
|
550
|
+
.command('checkout [branch]')
|
|
551
|
+
.alias('co')
|
|
552
|
+
.description('Switch to a different branch')
|
|
553
|
+
.action(cowCommands.checkout);
|
|
554
|
+
program
|
|
555
|
+
.command('merge [source] [target]')
|
|
556
|
+
.description('Merge a fork/branch into another branch')
|
|
557
|
+
.option('--strategy <type>', 'Merge strategy (last-write-wins|custom)', 'last-write-wins')
|
|
558
|
+
.option('-f, --force', 'Force merge on conflicts')
|
|
559
|
+
.action(cowCommands.merge);
|
|
560
|
+
program
|
|
561
|
+
.command('history')
|
|
562
|
+
.alias('log')
|
|
563
|
+
.description('Show commit history')
|
|
564
|
+
.option('-l, --limit <number>', 'Number of commits to show', '10')
|
|
565
|
+
.action(cowCommands.history);
|
|
566
|
+
program
|
|
567
|
+
.command('migrate')
|
|
568
|
+
.description('🔄 Migrate from v4.x to v5.0.0 (one-time)')
|
|
569
|
+
.option('--from <path>', 'Old Brainy data path (v4.x)')
|
|
570
|
+
.option('--to <path>', 'New Brainy data path (v5.0.0)')
|
|
571
|
+
.option('--backup', 'Create backup before migration')
|
|
572
|
+
.option('--dry-run', 'Show migration plan without executing')
|
|
573
|
+
.action(cowCommands.migrate);
|
|
524
574
|
// ===== Interactive Mode =====
|
|
525
575
|
program
|
|
526
576
|
.command('interactive')
|
package/dist/hnsw/hnswIndex.d.ts
CHANGED
|
@@ -16,6 +16,9 @@ export declare class HNSWIndex {
|
|
|
16
16
|
private useParallelization;
|
|
17
17
|
private storage;
|
|
18
18
|
private unifiedCache;
|
|
19
|
+
private cowEnabled;
|
|
20
|
+
private cowModifiedNodes;
|
|
21
|
+
private cowParent;
|
|
19
22
|
constructor(config?: Partial<HNSWConfig>, distanceFunction?: DistanceFunction, options?: {
|
|
20
23
|
useParallelization?: boolean;
|
|
21
24
|
storage?: BaseStorage;
|
|
@@ -28,6 +31,44 @@ export declare class HNSWIndex {
|
|
|
28
31
|
* Get whether parallelization is enabled
|
|
29
32
|
*/
|
|
30
33
|
getUseParallelization(): boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
|
|
36
|
+
*
|
|
37
|
+
* Snowflake-style instant fork: O(1) shallow copy of Maps, lazy deep copy on write.
|
|
38
|
+
*
|
|
39
|
+
* @param parent - Parent HNSW index to copy from
|
|
40
|
+
*
|
|
41
|
+
* Performance:
|
|
42
|
+
* - Fork time: <10ms for 1M+ nodes (just copies Map references)
|
|
43
|
+
* - Memory: Shared reads, only modified nodes duplicated (~10-20% overhead)
|
|
44
|
+
* - Reads: Same speed as parent (shared data structures)
|
|
45
|
+
*
|
|
46
|
+
* @example
|
|
47
|
+
* ```typescript
|
|
48
|
+
* const parent = new HNSWIndex(config)
|
|
49
|
+
* // ... parent has 1M nodes ...
|
|
50
|
+
*
|
|
51
|
+
* const fork = new HNSWIndex(config)
|
|
52
|
+
* fork.enableCOW(parent) // <10ms - instant!
|
|
53
|
+
*
|
|
54
|
+
* // Reads share data
|
|
55
|
+
* await fork.search(query) // Fast, uses parent's data
|
|
56
|
+
*
|
|
57
|
+
* // Writes trigger COW
|
|
58
|
+
* await fork.addItem(newItem) // Deep copies only modified nodes
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
enableCOW(parent: HNSWIndex): void;
|
|
62
|
+
/**
|
|
63
|
+
* Ensure node is copied before modification (lazy COW)
|
|
64
|
+
*
|
|
65
|
+
* Deep copies a node only when first modified. Subsequent modifications
|
|
66
|
+
* use the already-copied node.
|
|
67
|
+
*
|
|
68
|
+
* @param nodeId - Node ID to ensure is copied
|
|
69
|
+
* @private
|
|
70
|
+
*/
|
|
71
|
+
private ensureCOW;
|
|
31
72
|
/**
|
|
32
73
|
* Calculate distances between a query vector and multiple vectors in parallel
|
|
33
74
|
* This is used to optimize performance for search operations
|
package/dist/hnsw/hnswIndex.js
CHANGED
|
@@ -13,7 +13,6 @@ const DEFAULT_CONFIG = {
|
|
|
13
13
|
ml: 16 // Max level
|
|
14
14
|
};
|
|
15
15
|
export class HNSWIndex {
|
|
16
|
-
// Always-adaptive caching (v3.36.0+) - no "mode" concept, system adapts automatically
|
|
17
16
|
constructor(config = {}, distanceFunction = euclideanDistance, options = {}) {
|
|
18
17
|
this.nouns = new Map();
|
|
19
18
|
this.entryPointId = null;
|
|
@@ -24,6 +23,11 @@ export class HNSWIndex {
|
|
|
24
23
|
this.dimension = null;
|
|
25
24
|
this.useParallelization = true; // Whether to use parallelization for performance-critical operations
|
|
26
25
|
this.storage = null; // Storage adapter for HNSW persistence (v3.35.0+)
|
|
26
|
+
// Always-adaptive caching (v3.36.0+) - no "mode" concept, system adapts automatically
|
|
27
|
+
// COW (Copy-on-Write) support - v5.0.0
|
|
28
|
+
this.cowEnabled = false;
|
|
29
|
+
this.cowModifiedNodes = new Set();
|
|
30
|
+
this.cowParent = null;
|
|
27
31
|
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
28
32
|
this.distanceFunction = distanceFunction;
|
|
29
33
|
this.useParallelization =
|
|
@@ -46,6 +50,87 @@ export class HNSWIndex {
|
|
|
46
50
|
getUseParallelization() {
|
|
47
51
|
return this.useParallelization;
|
|
48
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
|
|
55
|
+
*
|
|
56
|
+
* Snowflake-style instant fork: O(1) shallow copy of Maps, lazy deep copy on write.
|
|
57
|
+
*
|
|
58
|
+
* @param parent - Parent HNSW index to copy from
|
|
59
|
+
*
|
|
60
|
+
* Performance:
|
|
61
|
+
* - Fork time: <10ms for 1M+ nodes (just copies Map references)
|
|
62
|
+
* - Memory: Shared reads, only modified nodes duplicated (~10-20% overhead)
|
|
63
|
+
* - Reads: Same speed as parent (shared data structures)
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* ```typescript
|
|
67
|
+
* const parent = new HNSWIndex(config)
|
|
68
|
+
* // ... parent has 1M nodes ...
|
|
69
|
+
*
|
|
70
|
+
* const fork = new HNSWIndex(config)
|
|
71
|
+
* fork.enableCOW(parent) // <10ms - instant!
|
|
72
|
+
*
|
|
73
|
+
* // Reads share data
|
|
74
|
+
* await fork.search(query) // Fast, uses parent's data
|
|
75
|
+
*
|
|
76
|
+
* // Writes trigger COW
|
|
77
|
+
* await fork.addItem(newItem) // Deep copies only modified nodes
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
enableCOW(parent) {
|
|
81
|
+
this.cowEnabled = true;
|
|
82
|
+
this.cowParent = parent;
|
|
83
|
+
// Shallow copy Maps - O(1) per Map, just copies references
|
|
84
|
+
// All nodes/connections are shared until first write
|
|
85
|
+
this.nouns = new Map(parent.nouns);
|
|
86
|
+
this.highLevelNodes = new Map();
|
|
87
|
+
for (const [level, nodeSet] of parent.highLevelNodes.entries()) {
|
|
88
|
+
this.highLevelNodes.set(level, new Set(nodeSet));
|
|
89
|
+
}
|
|
90
|
+
// Copy scalar values
|
|
91
|
+
this.entryPointId = parent.entryPointId;
|
|
92
|
+
this.maxLevel = parent.maxLevel;
|
|
93
|
+
this.dimension = parent.dimension;
|
|
94
|
+
// Share cache (COW at cache level)
|
|
95
|
+
this.unifiedCache = parent.unifiedCache;
|
|
96
|
+
// Share config and distance function
|
|
97
|
+
this.config = parent.config;
|
|
98
|
+
this.distanceFunction = parent.distanceFunction;
|
|
99
|
+
this.useParallelization = parent.useParallelization;
|
|
100
|
+
prodLog.info(`HNSW COW enabled: ${parent.nouns.size} nodes shallow copied`);
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Ensure node is copied before modification (lazy COW)
|
|
104
|
+
*
|
|
105
|
+
* Deep copies a node only when first modified. Subsequent modifications
|
|
106
|
+
* use the already-copied node.
|
|
107
|
+
*
|
|
108
|
+
* @param nodeId - Node ID to ensure is copied
|
|
109
|
+
* @private
|
|
110
|
+
*/
|
|
111
|
+
ensureCOW(nodeId) {
|
|
112
|
+
if (!this.cowEnabled)
|
|
113
|
+
return;
|
|
114
|
+
if (this.cowModifiedNodes.has(nodeId))
|
|
115
|
+
return; // Already copied
|
|
116
|
+
const original = this.nouns.get(nodeId);
|
|
117
|
+
if (!original)
|
|
118
|
+
return;
|
|
119
|
+
// Deep copy connections Map (separate Map + Sets for each level)
|
|
120
|
+
const connectionsCopy = new Map();
|
|
121
|
+
for (const [level, ids] of original.connections.entries()) {
|
|
122
|
+
connectionsCopy.set(level, new Set(ids));
|
|
123
|
+
}
|
|
124
|
+
// Deep copy node
|
|
125
|
+
const nodeCopy = {
|
|
126
|
+
id: original.id,
|
|
127
|
+
vector: [...original.vector], // Deep copy vector array
|
|
128
|
+
connections: connectionsCopy,
|
|
129
|
+
level: original.level
|
|
130
|
+
};
|
|
131
|
+
this.nouns.set(nodeId, nodeCopy);
|
|
132
|
+
this.cowModifiedNodes.add(nodeId);
|
|
133
|
+
}
|
|
49
134
|
/**
|
|
50
135
|
* Calculate distances between a query vector and multiple vectors in parallel
|
|
51
136
|
* This is used to optimize performance for search operations
|
|
@@ -186,6 +271,8 @@ export class HNSWIndex {
|
|
|
186
271
|
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
|
187
272
|
continue;
|
|
188
273
|
}
|
|
274
|
+
// COW: Ensure neighbor is copied before modification
|
|
275
|
+
this.ensureCOW(neighborId);
|
|
189
276
|
noun.connections.get(level).add(neighborId);
|
|
190
277
|
// Add reverse connection
|
|
191
278
|
if (!neighbor.connections.has(level)) {
|
|
@@ -392,10 +479,14 @@ export class HNSWIndex {
|
|
|
392
479
|
if (!this.nouns.has(id)) {
|
|
393
480
|
return false;
|
|
394
481
|
}
|
|
482
|
+
// COW: Ensure node is copied before modification
|
|
483
|
+
this.ensureCOW(id);
|
|
395
484
|
const noun = this.nouns.get(id);
|
|
396
485
|
// Remove connections to this noun from all neighbors
|
|
397
486
|
for (const [level, connections] of noun.connections.entries()) {
|
|
398
487
|
for (const neighborId of connections) {
|
|
488
|
+
// COW: Ensure neighbor is copied before modification
|
|
489
|
+
this.ensureCOW(neighborId);
|
|
399
490
|
const neighbor = this.nouns.get(neighborId);
|
|
400
491
|
if (!neighbor) {
|
|
401
492
|
// Skip neighbors that don't exist (expected during rapid additions/deletions)
|
|
@@ -412,6 +503,8 @@ export class HNSWIndex {
|
|
|
412
503
|
for (const [nounId, otherNoun] of this.nouns.entries()) {
|
|
413
504
|
if (nounId === id)
|
|
414
505
|
continue; // Skip the noun being removed
|
|
506
|
+
// COW: Ensure noun is copied before modification
|
|
507
|
+
this.ensureCOW(nounId);
|
|
415
508
|
for (const [level, connections] of otherNoun.connections.entries()) {
|
|
416
509
|
if (connections.has(id)) {
|
|
417
510
|
connections.delete(id);
|
|
@@ -1109,6 +1202,8 @@ export class HNSWIndex {
|
|
|
1109
1202
|
* Ensure a noun doesn't have too many connections at a given level
|
|
1110
1203
|
*/
|
|
1111
1204
|
async pruneConnections(noun, level) {
|
|
1205
|
+
// COW: Ensure noun is copied before modification
|
|
1206
|
+
this.ensureCOW(noun.id);
|
|
1112
1207
|
const connections = noun.connections.get(level);
|
|
1113
1208
|
if (connections.size <= this.config.M) {
|
|
1114
1209
|
return;
|
|
@@ -54,6 +54,15 @@ export declare class TypeAwareHNSWIndex {
|
|
|
54
54
|
useParallelization?: boolean;
|
|
55
55
|
storage?: BaseStorage;
|
|
56
56
|
});
|
|
57
|
+
/**
|
|
58
|
+
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
|
|
59
|
+
*
|
|
60
|
+
* Propagates enableCOW() to all underlying type-specific HNSW indexes.
|
|
61
|
+
* Each index performs O(1) shallow copy of its own data structures.
|
|
62
|
+
*
|
|
63
|
+
* @param parent - Parent TypeAwareHNSWIndex to copy from
|
|
64
|
+
*/
|
|
65
|
+
enableCOW(parent: TypeAwareHNSWIndex): void;
|
|
57
66
|
/**
|
|
58
67
|
* Get or create HNSW index for a specific type (lazy initialization)
|
|
59
68
|
*
|
|
@@ -49,6 +49,28 @@ export class TypeAwareHNSWIndex {
|
|
|
49
49
|
: true;
|
|
50
50
|
prodLog.info('TypeAwareHNSWIndex initialized (Phase 2: Type-Aware HNSW)');
|
|
51
51
|
}
|
|
52
|
+
/**
|
|
53
|
+
* Enable COW (Copy-on-Write) mode - Instant fork via shallow copy
|
|
54
|
+
*
|
|
55
|
+
* Propagates enableCOW() to all underlying type-specific HNSW indexes.
|
|
56
|
+
* Each index performs O(1) shallow copy of its own data structures.
|
|
57
|
+
*
|
|
58
|
+
* @param parent - Parent TypeAwareHNSWIndex to copy from
|
|
59
|
+
*/
|
|
60
|
+
enableCOW(parent) {
|
|
61
|
+
// Shallow copy indexes Map
|
|
62
|
+
this.indexes = new Map(parent.indexes);
|
|
63
|
+
// Enable COW on each underlying type-specific index
|
|
64
|
+
for (const [type, parentIndex] of parent.indexes.entries()) {
|
|
65
|
+
const childIndex = new HNSWIndex(this.config, this.distanceFunction, {
|
|
66
|
+
useParallelization: this.useParallelization,
|
|
67
|
+
storage: this.storage || undefined
|
|
68
|
+
});
|
|
69
|
+
childIndex.enableCOW(parentIndex);
|
|
70
|
+
this.indexes.set(type, childIndex);
|
|
71
|
+
}
|
|
72
|
+
prodLog.info(`TypeAwareHNSWIndex COW enabled: ${parent.indexes.size} type-specific indexes shallow copied`);
|
|
73
|
+
}
|
|
52
74
|
/**
|
|
53
75
|
* Get or create HNSW index for a specific type (lazy initialization)
|
|
54
76
|
*
|
|
@@ -23,7 +23,7 @@ export class ImportHistory {
|
|
|
23
23
|
*/
|
|
24
24
|
async init() {
|
|
25
25
|
try {
|
|
26
|
-
const vfs = this.brain.vfs
|
|
26
|
+
const vfs = this.brain.vfs;
|
|
27
27
|
await vfs.init();
|
|
28
28
|
// Try to load existing history
|
|
29
29
|
const content = await vfs.readFile(this.historyFile);
|
|
@@ -102,7 +102,7 @@ export class ImportHistory {
|
|
|
102
102
|
}
|
|
103
103
|
// Delete VFS files
|
|
104
104
|
try {
|
|
105
|
-
const vfs = this.brain.vfs
|
|
105
|
+
const vfs = this.brain.vfs;
|
|
106
106
|
await vfs.init();
|
|
107
107
|
for (const vfsPath of entry.vfsPaths) {
|
|
108
108
|
try {
|
|
@@ -160,7 +160,7 @@ export class ImportHistory {
|
|
|
160
160
|
*/
|
|
161
161
|
async persist() {
|
|
162
162
|
try {
|
|
163
|
-
const vfs = this.brain.vfs
|
|
163
|
+
const vfs = this.brain.vfs;
|
|
164
164
|
await vfs.init();
|
|
165
165
|
// Ensure directory exists
|
|
166
166
|
const dir = this.historyFile.substring(0, this.historyFile.lastIndexOf('/'));
|
|
@@ -65,7 +65,7 @@ export declare class VFSStructureGenerator {
|
|
|
65
65
|
* Initialize the generator
|
|
66
66
|
*
|
|
67
67
|
* CRITICAL: Gets brain's VFS instance and initializes it if needed.
|
|
68
|
-
* This ensures that after import, brain.vfs
|
|
68
|
+
* This ensures that after import, brain.vfs returns an initialized instance.
|
|
69
69
|
*/
|
|
70
70
|
init(): Promise<void>;
|
|
71
71
|
/**
|
|
@@ -15,7 +15,7 @@ import { NounType } from '../types/graphTypes.js';
|
|
|
15
15
|
export class VFSStructureGenerator {
|
|
16
16
|
constructor(brain) {
|
|
17
17
|
this.brain = brain;
|
|
18
|
-
// CRITICAL FIX: Use brain.vfs
|
|
18
|
+
// CRITICAL FIX: Use brain.vfs instead of creating separate instance
|
|
19
19
|
// This ensures VFSStructureGenerator and user code share the same VFS instance
|
|
20
20
|
// Before: Created separate instance that wasn't accessible to users
|
|
21
21
|
// After: Uses brain's cached instance, making VFS queryable after import
|
|
@@ -24,11 +24,11 @@ export class VFSStructureGenerator {
|
|
|
24
24
|
* Initialize the generator
|
|
25
25
|
*
|
|
26
26
|
* CRITICAL: Gets brain's VFS instance and initializes it if needed.
|
|
27
|
-
* This ensures that after import, brain.vfs
|
|
27
|
+
* This ensures that after import, brain.vfs returns an initialized instance.
|
|
28
28
|
*/
|
|
29
29
|
async init() {
|
|
30
30
|
// Get brain's cached VFS instance (creates if doesn't exist)
|
|
31
|
-
this.vfs = this.brain.vfs
|
|
31
|
+
this.vfs = this.brain.vfs;
|
|
32
32
|
// CRITICAL FIX (v4.10.2): Always call vfs.init() explicitly
|
|
33
33
|
// The previous code tried to check if initialized via stat('/') but this was unreliable
|
|
34
34
|
// vfs.init() is idempotent, so calling it multiple times is safe
|
package/dist/index.d.ts
CHANGED
|
@@ -29,6 +29,12 @@ export { UniversalSentenceEncoder, TransformerEmbedding, createEmbeddingFunction
|
|
|
29
29
|
import { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage } from './storage/storageFactory.js';
|
|
30
30
|
export { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage };
|
|
31
31
|
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js';
|
|
32
|
+
import { CommitLog } from './storage/cow/CommitLog.js';
|
|
33
|
+
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js';
|
|
34
|
+
import { BlobStorage } from './storage/cow/BlobStorage.js';
|
|
35
|
+
import { RefManager } from './storage/cow/RefManager.js';
|
|
36
|
+
import { TreeObject } from './storage/cow/TreeObject.js';
|
|
37
|
+
export { CommitLog, CommitObject, CommitBuilder, BlobStorage, RefManager, TreeObject };
|
|
32
38
|
import { Pipeline, pipeline, augmentationPipeline, ExecutionMode, PipelineOptions, PipelineResult, createPipeline, createStreamingPipeline, StreamlinedExecutionMode, StreamlinedPipelineOptions, StreamlinedPipelineResult } from './pipeline.js';
|
|
33
39
|
export { Pipeline, pipeline, augmentationPipeline, ExecutionMode, createPipeline, createStreamingPipeline, StreamlinedExecutionMode, };
|
|
34
40
|
export type { PipelineOptions, PipelineResult, StreamlinedPipelineOptions, StreamlinedPipelineResult };
|
package/dist/index.js
CHANGED
|
@@ -67,6 +67,16 @@ import { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStora
|
|
|
67
67
|
export { OPFSStorage, MemoryStorage, R2Storage, S3CompatibleStorage, createStorage };
|
|
68
68
|
// FileSystemStorage is exported separately to avoid browser build issues
|
|
69
69
|
export { FileSystemStorage } from './storage/adapters/fileSystemStorage.js';
|
|
70
|
+
// Export COW (Copy-on-Write) infrastructure for v5.0.0
|
|
71
|
+
// Enables premium augmentations to implement temporal features
|
|
72
|
+
import { CommitLog } from './storage/cow/CommitLog.js';
|
|
73
|
+
import { CommitObject, CommitBuilder } from './storage/cow/CommitObject.js';
|
|
74
|
+
import { BlobStorage } from './storage/cow/BlobStorage.js';
|
|
75
|
+
import { RefManager } from './storage/cow/RefManager.js';
|
|
76
|
+
import { TreeObject } from './storage/cow/TreeObject.js';
|
|
77
|
+
export {
|
|
78
|
+
// COW infrastructure
|
|
79
|
+
CommitLog, CommitObject, CommitBuilder, BlobStorage, RefManager, TreeObject };
|
|
70
80
|
// Export unified pipeline
|
|
71
81
|
import { Pipeline, pipeline, augmentationPipeline, ExecutionMode, createPipeline, createStreamingPipeline, StreamlinedExecutionMode } from './pipeline.js';
|
|
72
82
|
// Sequential pipeline removed - use unified pipeline instead
|
|
@@ -37,11 +37,13 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
37
37
|
init(): Promise<void>;
|
|
38
38
|
/**
|
|
39
39
|
* Save a noun to storage (v4.0.0: pure vector only, no metadata)
|
|
40
|
+
* v5.0.1: COW-aware - uses branch-prefixed paths for fork isolation
|
|
40
41
|
*/
|
|
41
42
|
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
42
43
|
/**
|
|
43
44
|
* Get a noun from storage (v4.0.0: returns pure vector only)
|
|
44
45
|
* Base class handles combining with metadata
|
|
46
|
+
* v5.0.1: COW-aware - reads from branch-prefixed paths with inheritance
|
|
45
47
|
*/
|
|
46
48
|
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
47
49
|
/**
|
|
@@ -90,15 +92,18 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
90
92
|
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
91
93
|
/**
|
|
92
94
|
* Delete a noun from storage (v4.0.0)
|
|
95
|
+
* v5.0.1: COW-aware - deletes from branch-prefixed paths
|
|
93
96
|
*/
|
|
94
97
|
protected deleteNoun_internal(id: string): Promise<void>;
|
|
95
98
|
/**
|
|
96
99
|
* Save a verb to storage (v4.0.0: pure vector + core fields, no metadata)
|
|
100
|
+
* v5.0.1: COW-aware - uses branch-prefixed paths for fork isolation
|
|
97
101
|
*/
|
|
98
102
|
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
99
103
|
/**
|
|
100
104
|
* Get a verb from storage (v4.0.0: returns pure vector + core fields)
|
|
101
105
|
* Base class handles combining with metadata
|
|
106
|
+
* v5.0.1: COW-aware - reads from branch-prefixed paths with inheritance
|
|
102
107
|
*/
|
|
103
108
|
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
104
109
|
/**
|
|
@@ -143,6 +148,7 @@ export declare class MemoryStorage extends BaseStorage {
|
|
|
143
148
|
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
144
149
|
/**
|
|
145
150
|
* Delete a verb from storage
|
|
151
|
+
* v5.0.1: COW-aware - deletes from branch-prefixed paths
|
|
146
152
|
*/
|
|
147
153
|
protected deleteVerb_internal(id: string): Promise<void>;
|
|
148
154
|
/**
|
|
@@ -65,6 +65,7 @@ export class MemoryStorage extends BaseStorage {
|
|
|
65
65
|
}
|
|
66
66
|
/**
|
|
67
67
|
* Save a noun to storage (v4.0.0: pure vector only, no metadata)
|
|
68
|
+
* v5.0.1: COW-aware - uses branch-prefixed paths for fork isolation
|
|
68
69
|
*/
|
|
69
70
|
async saveNoun_internal(noun) {
|
|
70
71
|
const isNew = !this.nouns.has(noun.id);
|
|
@@ -82,7 +83,12 @@ export class MemoryStorage extends BaseStorage {
|
|
|
82
83
|
for (const [level, connections] of noun.connections.entries()) {
|
|
83
84
|
nounCopy.connections.set(level, new Set(connections));
|
|
84
85
|
}
|
|
85
|
-
//
|
|
86
|
+
// v5.0.1: COW-aware write using branch-prefixed path
|
|
87
|
+
// Use synthetic path for vector storage (nouns don't have types in standalone mode)
|
|
88
|
+
const path = `hnsw/nouns/${noun.id}.json`;
|
|
89
|
+
await this.writeObjectToBranch(path, nounCopy);
|
|
90
|
+
// ALSO store in nouns Map for fast iteration (getNouns, initializeCounts)
|
|
91
|
+
// This is redundant but maintains backward compatibility
|
|
86
92
|
this.nouns.set(noun.id, nounCopy);
|
|
87
93
|
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
|
|
88
94
|
// This fixes the race condition where metadata didn't exist yet
|
|
@@ -90,10 +96,12 @@ export class MemoryStorage extends BaseStorage {
|
|
|
90
96
|
/**
|
|
91
97
|
* Get a noun from storage (v4.0.0: returns pure vector only)
|
|
92
98
|
* Base class handles combining with metadata
|
|
99
|
+
* v5.0.1: COW-aware - reads from branch-prefixed paths with inheritance
|
|
93
100
|
*/
|
|
94
101
|
async getNoun_internal(id) {
|
|
95
|
-
//
|
|
96
|
-
const
|
|
102
|
+
// v5.0.1: COW-aware read using branch-prefixed path with inheritance
|
|
103
|
+
const path = `hnsw/nouns/${id}.json`;
|
|
104
|
+
const noun = await this.readWithInheritance(path);
|
|
97
105
|
// If not found, return null
|
|
98
106
|
if (!noun) {
|
|
99
107
|
return null;
|
|
@@ -107,9 +115,10 @@ export class MemoryStorage extends BaseStorage {
|
|
|
107
115
|
level: noun.level || 0
|
|
108
116
|
// ✅ NO metadata field in v4.0.0
|
|
109
117
|
};
|
|
110
|
-
// Copy connections
|
|
111
|
-
|
|
112
|
-
|
|
118
|
+
// Copy connections (handle both Map and plain object from JSON)
|
|
119
|
+
const connections = noun.connections instanceof Map ? noun.connections : new Map(Object.entries(noun.connections || {}));
|
|
120
|
+
for (const [level, conns] of connections.entries()) {
|
|
121
|
+
nounCopy.connections.set(Number(level), new Set(conns));
|
|
113
122
|
}
|
|
114
123
|
return nounCopy;
|
|
115
124
|
}
|
|
@@ -252,6 +261,7 @@ export class MemoryStorage extends BaseStorage {
|
|
|
252
261
|
}
|
|
253
262
|
/**
|
|
254
263
|
* Delete a noun from storage (v4.0.0)
|
|
264
|
+
* v5.0.1: COW-aware - deletes from branch-prefixed paths
|
|
255
265
|
*/
|
|
256
266
|
async deleteNoun_internal(id) {
|
|
257
267
|
// v4.0.0: Get type from separate metadata storage
|
|
@@ -260,10 +270,15 @@ export class MemoryStorage extends BaseStorage {
|
|
|
260
270
|
const type = metadata.noun || 'default';
|
|
261
271
|
this.decrementEntityCount(type);
|
|
262
272
|
}
|
|
273
|
+
// v5.0.1: COW-aware delete using branch-prefixed path
|
|
274
|
+
const path = `hnsw/nouns/${id}.json`;
|
|
275
|
+
await this.deleteObjectFromBranch(path);
|
|
276
|
+
// Also remove from nouns Map for fast iteration
|
|
263
277
|
this.nouns.delete(id);
|
|
264
278
|
}
|
|
265
279
|
/**
|
|
266
280
|
* Save a verb to storage (v4.0.0: pure vector + core fields, no metadata)
|
|
281
|
+
* v5.0.1: COW-aware - uses branch-prefixed paths for fork isolation
|
|
267
282
|
*/
|
|
268
283
|
async saveVerb_internal(verb) {
|
|
269
284
|
const isNew = !this.verbs.has(verb.id);
|
|
@@ -283,17 +298,22 @@ export class MemoryStorage extends BaseStorage {
|
|
|
283
298
|
for (const [level, connections] of verb.connections.entries()) {
|
|
284
299
|
verbCopy.connections.set(level, new Set(connections));
|
|
285
300
|
}
|
|
286
|
-
//
|
|
301
|
+
// v5.0.1: COW-aware write using branch-prefixed path
|
|
302
|
+
const path = `hnsw/verbs/${verb.id}.json`;
|
|
303
|
+
await this.writeObjectToBranch(path, verbCopy);
|
|
304
|
+
// ALSO store in verbs Map for fast iteration (getVerbs, initializeCounts)
|
|
287
305
|
this.verbs.set(verb.id, verbCopy);
|
|
288
306
|
// Note: Count tracking happens in saveVerbMetadata since metadata is separate
|
|
289
307
|
}
|
|
290
308
|
/**
|
|
291
309
|
* Get a verb from storage (v4.0.0: returns pure vector + core fields)
|
|
292
310
|
* Base class handles combining with metadata
|
|
311
|
+
* v5.0.1: COW-aware - reads from branch-prefixed paths with inheritance
|
|
293
312
|
*/
|
|
294
313
|
async getVerb_internal(id) {
|
|
295
|
-
//
|
|
296
|
-
const
|
|
314
|
+
// v5.0.1: COW-aware read using branch-prefixed path with inheritance
|
|
315
|
+
const path = `hnsw/verbs/${id}.json`;
|
|
316
|
+
const verb = await this.readWithInheritance(path);
|
|
297
317
|
// If not found, return null
|
|
298
318
|
if (!verb) {
|
|
299
319
|
return null;
|
|
@@ -310,9 +330,10 @@ export class MemoryStorage extends BaseStorage {
|
|
|
310
330
|
targetId: verb.targetId
|
|
311
331
|
// ✅ NO metadata field in v4.0.0
|
|
312
332
|
};
|
|
313
|
-
// Copy connections
|
|
314
|
-
|
|
315
|
-
|
|
333
|
+
// Copy connections (handle both Map and plain object from JSON)
|
|
334
|
+
const connections = verb.connections instanceof Map ? verb.connections : new Map(Object.entries(verb.connections || {}));
|
|
335
|
+
for (const [level, conns] of connections.entries()) {
|
|
336
|
+
verbCopy.connections.set(Number(level), new Set(conns));
|
|
316
337
|
}
|
|
317
338
|
return verbCopy;
|
|
318
339
|
}
|
|
@@ -474,10 +495,9 @@ export class MemoryStorage extends BaseStorage {
|
|
|
474
495
|
}
|
|
475
496
|
/**
|
|
476
497
|
* Delete a verb from storage
|
|
498
|
+
* v5.0.1: COW-aware - deletes from branch-prefixed paths
|
|
477
499
|
*/
|
|
478
500
|
async deleteVerb_internal(id) {
|
|
479
|
-
// Delete the HNSWVerb from the verbs map
|
|
480
|
-
this.verbs.delete(id);
|
|
481
501
|
// CRITICAL: Also delete verb metadata - this is what getVerbs() uses to find verbs
|
|
482
502
|
// Without this, getVerbsBySource() will still find "deleted" verbs via their metadata
|
|
483
503
|
const metadata = await this.getVerbMetadata(id);
|
|
@@ -487,6 +507,11 @@ export class MemoryStorage extends BaseStorage {
|
|
|
487
507
|
// Delete the metadata using the base storage method
|
|
488
508
|
await this.deleteVerbMetadata(id);
|
|
489
509
|
}
|
|
510
|
+
// v5.0.1: COW-aware delete using branch-prefixed path
|
|
511
|
+
const path = `hnsw/verbs/${id}.json`;
|
|
512
|
+
await this.deleteObjectFromBranch(path);
|
|
513
|
+
// Also remove from verbs Map for fast iteration
|
|
514
|
+
this.verbs.delete(id);
|
|
490
515
|
}
|
|
491
516
|
/**
|
|
492
517
|
* Primitive operation: Write object to path
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
* @since Phase 1 - Type-First Implementation
|
|
37
37
|
*/
|
|
38
38
|
import { BaseStorage } from '../baseStorage.js';
|
|
39
|
-
import { HNSWNoun, HNSWVerb, HNSWVerbWithMetadata, NounMetadata, VerbMetadata, StatisticsData } from '../../coreTypes.js';
|
|
39
|
+
import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, NounMetadata, VerbMetadata, StatisticsData } from '../../coreTypes.js';
|
|
40
40
|
import { NounType, VerbType } from '../../types/graphTypes.js';
|
|
41
41
|
/**
|
|
42
42
|
* Options for TypeAwareStorageAdapter
|
|
@@ -238,6 +238,36 @@ export declare class TypeAwareStorageAdapter extends BaseStorage {
|
|
|
238
238
|
level: number;
|
|
239
239
|
connections: Record<string, string[]>;
|
|
240
240
|
} | null>;
|
|
241
|
+
/**
|
|
242
|
+
* Get nouns with pagination (v5.0.1: COW-aware)
|
|
243
|
+
* Required for find() to work with TypeAwareStorage
|
|
244
|
+
*/
|
|
245
|
+
getNounsWithPagination(options: {
|
|
246
|
+
limit?: number;
|
|
247
|
+
offset?: number;
|
|
248
|
+
cursor?: string;
|
|
249
|
+
filter?: any;
|
|
250
|
+
}): Promise<{
|
|
251
|
+
items: HNSWNounWithMetadata[];
|
|
252
|
+
totalCount: number;
|
|
253
|
+
hasMore: boolean;
|
|
254
|
+
nextCursor?: string;
|
|
255
|
+
}>;
|
|
256
|
+
/**
|
|
257
|
+
* Get verbs with pagination (v5.0.1: COW-aware)
|
|
258
|
+
* Required for GraphAdjacencyIndex rebuild and find() to work
|
|
259
|
+
*/
|
|
260
|
+
getVerbsWithPagination(options: {
|
|
261
|
+
limit?: number;
|
|
262
|
+
offset?: number;
|
|
263
|
+
cursor?: string;
|
|
264
|
+
filter?: any;
|
|
265
|
+
}): Promise<{
|
|
266
|
+
items: HNSWVerbWithMetadata[];
|
|
267
|
+
totalCount: number;
|
|
268
|
+
hasMore: boolean;
|
|
269
|
+
nextCursor?: string;
|
|
270
|
+
}>;
|
|
241
271
|
/**
|
|
242
272
|
* Save HNSW system data (entry point, max level)
|
|
243
273
|
*/
|