@soulcraft/brainy 4.11.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +271 -0
- package/README.md +38 -1
- package/dist/augmentations/brainyAugmentation.d.ts +76 -0
- package/dist/augmentations/brainyAugmentation.js +126 -0
- package/dist/augmentations/cacheAugmentation.js +9 -4
- package/dist/brainy.d.ts +248 -15
- package/dist/brainy.js +707 -17
- package/dist/cli/commands/cow.d.ts +60 -0
- package/dist/cli/commands/cow.js +444 -0
- package/dist/cli/commands/import.js +1 -1
- package/dist/cli/commands/vfs.js +24 -40
- package/dist/cli/index.js +50 -0
- package/dist/hnsw/hnswIndex.d.ts +41 -0
- package/dist/hnsw/hnswIndex.js +96 -1
- package/dist/hnsw/typeAwareHNSWIndex.d.ts +9 -0
- package/dist/hnsw/typeAwareHNSWIndex.js +22 -0
- package/dist/import/ImportHistory.js +3 -3
- package/dist/importers/VFSStructureGenerator.d.ts +1 -1
- package/dist/importers/VFSStructureGenerator.js +3 -3
- package/dist/index.d.ts +6 -0
- package/dist/index.js +10 -0
- package/dist/storage/adapters/memoryStorage.d.ts +6 -0
- package/dist/storage/adapters/memoryStorage.js +39 -14
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +31 -1
- package/dist/storage/adapters/typeAwareStorageAdapter.js +272 -43
- package/dist/storage/baseStorage.d.ts +64 -0
- package/dist/storage/baseStorage.js +252 -12
- package/dist/storage/cow/BlobStorage.d.ts +232 -0
- package/dist/storage/cow/BlobStorage.js +437 -0
- package/dist/storage/cow/CommitLog.d.ts +199 -0
- package/dist/storage/cow/CommitLog.js +363 -0
- package/dist/storage/cow/CommitObject.d.ts +276 -0
- package/dist/storage/cow/CommitObject.js +431 -0
- package/dist/storage/cow/RefManager.d.ts +213 -0
- package/dist/storage/cow/RefManager.js +409 -0
- package/dist/storage/cow/TreeObject.d.ts +177 -0
- package/dist/storage/cow/TreeObject.js +293 -0
- package/dist/storage/storageFactory.d.ts +6 -0
- package/dist/storage/storageFactory.js +92 -74
- package/dist/types/brainy.types.d.ts +1 -0
- package/dist/vfs/FSCompat.d.ts +1 -1
- package/dist/vfs/FSCompat.js +1 -1
- package/dist/vfs/VirtualFileSystem.js +5 -6
- package/package.json +1 -1
package/dist/brainy.js
CHANGED
|
@@ -18,6 +18,7 @@ import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js';
|
|
|
18
18
|
import { VirtualFileSystem } from './vfs/VirtualFileSystem.js';
|
|
19
19
|
import { MetadataIndexManager } from './utils/metadataIndex.js';
|
|
20
20
|
import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js';
|
|
21
|
+
import { CommitBuilder } from './storage/cow/CommitObject.js';
|
|
21
22
|
import { createPipeline } from './streaming/pipeline.js';
|
|
22
23
|
import { configureLogger, LogLevel } from './utils/logger.js';
|
|
23
24
|
import { DistributedCoordinator, ShardManager, CacheSync, ReadWriteSeparation } from './distributed/index.js';
|
|
@@ -96,6 +97,12 @@ export class Brainy {
|
|
|
96
97
|
// Setup and initialize storage
|
|
97
98
|
this.storage = await this.setupStorage();
|
|
98
99
|
await this.storage.init();
|
|
100
|
+
// Enable COW immediately after storage init (v5.0.1)
|
|
101
|
+
// This ensures ALL data is stored in branch-scoped paths from the start
|
|
102
|
+
// Lightweight: just sets cowEnabled=true and currentBranch, no RefManager/BlobStorage yet
|
|
103
|
+
if (typeof this.storage.enableCOWLightweight === 'function') {
|
|
104
|
+
this.storage.enableCOWLightweight(this.config.storage?.branch || 'main');
|
|
105
|
+
}
|
|
99
106
|
// Setup index now that we have storage
|
|
100
107
|
this.index = this.setupIndex();
|
|
101
108
|
// Initialize core metadata index
|
|
@@ -134,7 +141,13 @@ export class Brainy {
|
|
|
134
141
|
this.registerShutdownHooks();
|
|
135
142
|
Brainy.shutdownHooksRegisteredGlobally = true;
|
|
136
143
|
}
|
|
144
|
+
// Mark as initialized BEFORE VFS init (v5.0.1)
|
|
145
|
+
// VFS.init() needs brain to be marked initialized to call brain methods
|
|
137
146
|
this.initialized = true;
|
|
147
|
+
// Initialize VFS (v5.0.1): Ensure VFS is ready when accessed as property
|
|
148
|
+
// This eliminates need for separate vfs.init() calls - zero additional complexity
|
|
149
|
+
this._vfs = new VirtualFileSystem(this);
|
|
150
|
+
await this._vfs.init();
|
|
138
151
|
}
|
|
139
152
|
catch (error) {
|
|
140
153
|
throw new Error(`Failed to initialize Brainy: ${error}`);
|
|
@@ -306,14 +319,16 @@ export class Brainy {
|
|
|
306
319
|
...(params.weight !== undefined && { weight: params.weight }),
|
|
307
320
|
...(params.createdBy && { createdBy: params.createdBy })
|
|
308
321
|
};
|
|
309
|
-
//
|
|
322
|
+
// v5.0.1: Save metadata FIRST so TypeAwareStorage can cache the type
|
|
323
|
+
// This prevents the race condition where saveNoun() defaults to 'thing'
|
|
324
|
+
await this.storage.saveNounMetadata(id, storageMetadata);
|
|
325
|
+
// Then save vector
|
|
310
326
|
await this.storage.saveNoun({
|
|
311
327
|
id,
|
|
312
328
|
vector,
|
|
313
329
|
connections: new Map(),
|
|
314
330
|
level: 0
|
|
315
331
|
});
|
|
316
|
-
await this.storage.saveNounMetadata(id, storageMetadata);
|
|
317
332
|
// v4.8.0: Build entity structure for indexing (NEW - with top-level fields)
|
|
318
333
|
const entityForIndexing = {
|
|
319
334
|
id,
|
|
@@ -1770,6 +1785,671 @@ export class Brainy {
|
|
|
1770
1785
|
this._tripleIntelligence = undefined;
|
|
1771
1786
|
});
|
|
1772
1787
|
}
|
|
1788
|
+
// ============= COW (COPY-ON-WRITE) API - v5.0.0 =============
|
|
1789
|
+
/**
|
|
1790
|
+
* Fork the brain (instant clone via Snowflake-style COW)
|
|
1791
|
+
*
|
|
1792
|
+
* Creates a shallow copy in <100ms using copy-on-write (COW) technology.
|
|
1793
|
+
* Fork shares storage and HNSW data structures with parent, copying only
|
|
1794
|
+
* when modified (lazy deep copy).
|
|
1795
|
+
*
|
|
1796
|
+
* **How It Works (v5.0.0)**:
|
|
1797
|
+
* 1. HNSW Index: Shallow copy via `enableCOW()` (~10ms for 1M+ nodes)
|
|
1798
|
+
* 2. Metadata Index: Fast rebuild from shared storage (<100ms)
|
|
1799
|
+
* 3. Graph Index: Fast rebuild from shared storage (<500ms)
|
|
1800
|
+
*
|
|
1801
|
+
* **Performance**:
|
|
1802
|
+
* - Fork time: <100ms @ 10K entities (MEASURED)
|
|
1803
|
+
* - Memory overhead: 10-20% (shared HNSW nodes)
|
|
1804
|
+
* - Storage overhead: 10-20% (shared blobs)
|
|
1805
|
+
*
|
|
1806
|
+
* **Write Isolation**: Changes in fork don't affect parent, and vice versa.
|
|
1807
|
+
*
|
|
1808
|
+
* @param branch - Optional branch name (auto-generated if not provided)
|
|
1809
|
+
* @param options - Optional fork metadata (author, message)
|
|
1810
|
+
* @returns New Brainy instance (forked, fully independent)
|
|
1811
|
+
*
|
|
1812
|
+
* @example
|
|
1813
|
+
* ```typescript
|
|
1814
|
+
* const brain = new Brainy()
|
|
1815
|
+
* await brain.init()
|
|
1816
|
+
*
|
|
1817
|
+
* // Add data to parent
|
|
1818
|
+
* await brain.add({ type: 'user', data: { name: 'Alice' } })
|
|
1819
|
+
*
|
|
1820
|
+
* // Fork instantly (<100ms)
|
|
1821
|
+
* const experiment = await brain.fork('test-migration')
|
|
1822
|
+
*
|
|
1823
|
+
* // Make changes safely in fork
|
|
1824
|
+
* await experiment.add({ type: 'user', data: { name: 'Bob' } })
|
|
1825
|
+
*
|
|
1826
|
+
* // Original untouched
|
|
1827
|
+
* console.log((await brain.find({})).length) // 1 (Alice)
|
|
1828
|
+
* console.log((await experiment.find({})).length) // 2 (Alice + Bob)
|
|
1829
|
+
* ```
|
|
1830
|
+
*
|
|
1831
|
+
* @since v5.0.0
|
|
1832
|
+
*/
|
|
1833
|
+
async fork(branch, options) {
|
|
1834
|
+
await this.ensureInitialized();
|
|
1835
|
+
return this.augmentationRegistry.execute('fork', { branch, options }, async () => {
|
|
1836
|
+
const branchName = branch || `fork-${Date.now()}`;
|
|
1837
|
+
// v5.0.1: Lazy COW initialization - enable automatically on first fork()
|
|
1838
|
+
// This is zero-config and transparent to users
|
|
1839
|
+
if (!('refManager' in this.storage) || !this.storage.refManager) {
|
|
1840
|
+
// Storage supports COW but isn't initialized yet - initialize now
|
|
1841
|
+
if (typeof this.storage.initializeCOW === 'function') {
|
|
1842
|
+
await this.storage.initializeCOW({
|
|
1843
|
+
branch: this.config.storage?.branch || 'main',
|
|
1844
|
+
enableCompression: true
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
else {
|
|
1848
|
+
// Storage adapter doesn't support COW at all
|
|
1849
|
+
throw new Error('Fork requires COW-enabled storage. ' +
|
|
1850
|
+
'This storage adapter does not support branching. ' +
|
|
1851
|
+
'Please use v5.0.0+ storage adapters.');
|
|
1852
|
+
}
|
|
1853
|
+
}
|
|
1854
|
+
const refManager = this.storage.refManager;
|
|
1855
|
+
const currentBranch = this.storage.currentBranch || 'main';
|
|
1856
|
+
// Step 1: Copy storage ref (COW layer - instant!)
|
|
1857
|
+
await refManager.copyRef(currentBranch, branchName);
|
|
1858
|
+
// Step 2: Create new Brainy instance pointing to fork branch
|
|
1859
|
+
const forkConfig = {
|
|
1860
|
+
...this.config,
|
|
1861
|
+
storage: {
|
|
1862
|
+
...(this.config.storage || { type: 'memory' }),
|
|
1863
|
+
branch: branchName
|
|
1864
|
+
}
|
|
1865
|
+
};
|
|
1866
|
+
const clone = new Brainy(forkConfig);
|
|
1867
|
+
// Step 3: Clone storage with separate currentBranch
|
|
1868
|
+
// Share RefManager/BlobStorage/CommitLog but maintain separate branch context
|
|
1869
|
+
clone.storage = Object.create(this.storage);
|
|
1870
|
+
clone.storage.currentBranch = branchName;
|
|
1871
|
+
// isInitialized inherited from prototype
|
|
1872
|
+
// Shallow copy HNSW index (INSTANT - just copies Map references)
|
|
1873
|
+
clone.index = this.setupIndex();
|
|
1874
|
+
// Enable COW (handle both HNSWIndex and TypeAwareHNSWIndex)
|
|
1875
|
+
if ('enableCOW' in clone.index && typeof clone.index.enableCOW === 'function') {
|
|
1876
|
+
clone.index.enableCOW(this.index);
|
|
1877
|
+
}
|
|
1878
|
+
// Fast rebuild for small indexes from COW storage (Metadata/Graph are fast)
|
|
1879
|
+
clone.metadataIndex = new MetadataIndexManager(clone.storage);
|
|
1880
|
+
await clone.metadataIndex.init();
|
|
1881
|
+
clone.graphIndex = new GraphAdjacencyIndex(clone.storage);
|
|
1882
|
+
await clone.graphIndex.rebuild();
|
|
1883
|
+
// Setup augmentations
|
|
1884
|
+
clone.augmentationRegistry = this.setupAugmentations();
|
|
1885
|
+
await clone.augmentationRegistry.initializeAll({
|
|
1886
|
+
brain: clone,
|
|
1887
|
+
storage: clone.storage,
|
|
1888
|
+
config: clone.config,
|
|
1889
|
+
log: (message, level) => {
|
|
1890
|
+
if (!clone.config.silent) {
|
|
1891
|
+
console[level || 'info'](message);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
});
|
|
1895
|
+
// Mark as initialized
|
|
1896
|
+
clone.initialized = true;
|
|
1897
|
+
clone.dimensions = this.dimensions;
|
|
1898
|
+
return clone;
|
|
1899
|
+
});
|
|
1900
|
+
}
|
|
1901
|
+
/**
|
|
1902
|
+
* List all branches/forks
|
|
1903
|
+
* @returns Array of branch names
|
|
1904
|
+
*
|
|
1905
|
+
* @example
|
|
1906
|
+
* ```typescript
|
|
1907
|
+
* const branches = await brain.listBranches()
|
|
1908
|
+
* console.log(branches) // ['main', 'experiment', 'backup']
|
|
1909
|
+
* ```
|
|
1910
|
+
*/
|
|
1911
|
+
async listBranches() {
|
|
1912
|
+
await this.ensureInitialized();
|
|
1913
|
+
return this.augmentationRegistry.execute('listBranches', {}, async () => {
|
|
1914
|
+
if (!('refManager' in this.storage)) {
|
|
1915
|
+
throw new Error('Branch management requires COW-enabled storage (v5.0.0+)');
|
|
1916
|
+
}
|
|
1917
|
+
const refManager = this.storage.refManager;
|
|
1918
|
+
const refs = await refManager.listRefs();
|
|
1919
|
+
// Filter to branches only (exclude tags)
|
|
1920
|
+
return refs
|
|
1921
|
+
.filter((ref) => ref.name.startsWith('refs/heads/'))
|
|
1922
|
+
.map((ref) => ref.name.replace('refs/heads/', ''));
|
|
1923
|
+
});
|
|
1924
|
+
}
|
|
1925
|
+
/**
|
|
1926
|
+
* Get current branch name
|
|
1927
|
+
* @returns Current branch name
|
|
1928
|
+
*
|
|
1929
|
+
* @example
|
|
1930
|
+
* ```typescript
|
|
1931
|
+
* const current = await brain.getCurrentBranch()
|
|
1932
|
+
* console.log(current) // 'main'
|
|
1933
|
+
* ```
|
|
1934
|
+
*/
|
|
1935
|
+
async getCurrentBranch() {
|
|
1936
|
+
await this.ensureInitialized();
|
|
1937
|
+
return this.augmentationRegistry.execute('getCurrentBranch', {}, async () => {
|
|
1938
|
+
if (!('currentBranch' in this.storage)) {
|
|
1939
|
+
return 'main'; // Default branch
|
|
1940
|
+
}
|
|
1941
|
+
return this.storage.currentBranch || 'main';
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1944
|
+
/**
|
|
1945
|
+
* Switch to a different branch
|
|
1946
|
+
* @param branch - Branch name to switch to
|
|
1947
|
+
*
|
|
1948
|
+
* @example
|
|
1949
|
+
* ```typescript
|
|
1950
|
+
* await brain.checkout('experiment')
|
|
1951
|
+
* console.log(await brain.getCurrentBranch()) // 'experiment'
|
|
1952
|
+
* ```
|
|
1953
|
+
*/
|
|
1954
|
+
async checkout(branch) {
|
|
1955
|
+
await this.ensureInitialized();
|
|
1956
|
+
return this.augmentationRegistry.execute('checkout', { branch }, async () => {
|
|
1957
|
+
if (!('refManager' in this.storage)) {
|
|
1958
|
+
throw new Error('Branch management requires COW-enabled storage (v5.0.0+)');
|
|
1959
|
+
}
|
|
1960
|
+
// Verify branch exists
|
|
1961
|
+
const branches = await this.listBranches();
|
|
1962
|
+
if (!branches.includes(branch)) {
|
|
1963
|
+
throw new Error(`Branch '${branch}' does not exist`);
|
|
1964
|
+
}
|
|
1965
|
+
// Update storage currentBranch
|
|
1966
|
+
this.storage.currentBranch = branch;
|
|
1967
|
+
// Reload from new branch
|
|
1968
|
+
// Clear indexes and reload
|
|
1969
|
+
this.index = this.setupIndex();
|
|
1970
|
+
this.metadataIndex = new MetadataIndexManager(this.storage);
|
|
1971
|
+
this.graphIndex = new GraphAdjacencyIndex(this.storage);
|
|
1972
|
+
// Re-initialize
|
|
1973
|
+
this.initialized = false;
|
|
1974
|
+
await this.init();
|
|
1975
|
+
});
|
|
1976
|
+
}
|
|
1977
|
+
/**
|
|
1978
|
+
* Create a commit with current state
|
|
1979
|
+
* @param options - Commit options (message, author, metadata)
|
|
1980
|
+
* @returns Commit hash
|
|
1981
|
+
*
|
|
1982
|
+
* @example
|
|
1983
|
+
* ```typescript
|
|
1984
|
+
* await brain.add({ noun: 'user', data: { name: 'Alice' } })
|
|
1985
|
+
* const commitHash = await brain.commit({
|
|
1986
|
+
* message: 'Add Alice user',
|
|
1987
|
+
* author: 'dev@example.com'
|
|
1988
|
+
* })
|
|
1989
|
+
* ```
|
|
1990
|
+
*/
|
|
1991
|
+
async commit(options) {
|
|
1992
|
+
await this.ensureInitialized();
|
|
1993
|
+
return this.augmentationRegistry.execute('commit', { options }, async () => {
|
|
1994
|
+
if (!('refManager' in this.storage) || !('commitLog' in this.storage) || !('blobStorage' in this.storage)) {
|
|
1995
|
+
throw new Error('Commit requires COW-enabled storage (v5.0.0+)');
|
|
1996
|
+
}
|
|
1997
|
+
const refManager = this.storage.refManager;
|
|
1998
|
+
const blobStorage = this.storage.blobStorage;
|
|
1999
|
+
const currentBranch = await this.getCurrentBranch();
|
|
2000
|
+
// Get current HEAD commit (parent)
|
|
2001
|
+
const currentCommitHash = await refManager.resolveRef(`heads/${currentBranch}`);
|
|
2002
|
+
// Get current state statistics
|
|
2003
|
+
const entityCount = await this.getNounCount();
|
|
2004
|
+
const relationshipCount = await this.getVerbCount();
|
|
2005
|
+
// Build commit object using builder pattern
|
|
2006
|
+
const builder = CommitBuilder.create(blobStorage)
|
|
2007
|
+
.tree('0000000000000000000000000000000000000000000000000000000000000000') // Empty tree hash for now
|
|
2008
|
+
.message(options?.message || 'Snapshot commit')
|
|
2009
|
+
.author(options?.author || 'unknown')
|
|
2010
|
+
.timestamp(Date.now())
|
|
2011
|
+
.entityCount(entityCount)
|
|
2012
|
+
.relationshipCount(relationshipCount);
|
|
2013
|
+
// Set parent if this is not the first commit
|
|
2014
|
+
if (currentCommitHash) {
|
|
2015
|
+
builder.parent(currentCommitHash);
|
|
2016
|
+
}
|
|
2017
|
+
// Add custom metadata
|
|
2018
|
+
if (options?.metadata) {
|
|
2019
|
+
Object.entries(options.metadata).forEach(([key, value]) => {
|
|
2020
|
+
builder.meta(key, value);
|
|
2021
|
+
});
|
|
2022
|
+
}
|
|
2023
|
+
// Build and persist commit (returns hash directly)
|
|
2024
|
+
const commitHash = await builder.build();
|
|
2025
|
+
// Update branch ref to point to new commit
|
|
2026
|
+
await refManager.setRef(`heads/${currentBranch}`, commitHash, {
|
|
2027
|
+
author: options?.author || 'unknown',
|
|
2028
|
+
message: options?.message || 'Snapshot commit'
|
|
2029
|
+
});
|
|
2030
|
+
return commitHash;
|
|
2031
|
+
});
|
|
2032
|
+
}
|
|
2033
|
+
/**
|
|
2034
|
+
* Merge a source branch into target branch
|
|
2035
|
+
* @param sourceBranch - Branch to merge from
|
|
2036
|
+
* @param targetBranch - Branch to merge into
|
|
2037
|
+
* @param options - Merge options (strategy, author, onConflict)
|
|
2038
|
+
* @returns Merge result with statistics
|
|
2039
|
+
*
|
|
2040
|
+
* @example
|
|
2041
|
+
* ```typescript
|
|
2042
|
+
* const result = await brain.merge('experiment', 'main', {
|
|
2043
|
+
* strategy: 'last-write-wins',
|
|
2044
|
+
* author: 'dev@example.com'
|
|
2045
|
+
* })
|
|
2046
|
+
* console.log(result) // { added: 5, modified: 3, deleted: 1, conflicts: 0 }
|
|
2047
|
+
* ```
|
|
2048
|
+
*/
|
|
2049
|
+
async merge(sourceBranch, targetBranch, options) {
|
|
2050
|
+
await this.ensureInitialized();
|
|
2051
|
+
return this.augmentationRegistry.execute('merge', { sourceBranch, targetBranch, options }, async () => {
|
|
2052
|
+
if (!('refManager' in this.storage) || !('blobStorage' in this.storage)) {
|
|
2053
|
+
throw new Error('Merge requires COW-enabled storage (v5.0.0+)');
|
|
2054
|
+
}
|
|
2055
|
+
const strategy = options?.strategy || 'last-write-wins';
|
|
2056
|
+
let added = 0;
|
|
2057
|
+
let modified = 0;
|
|
2058
|
+
let deleted = 0;
|
|
2059
|
+
let conflicts = 0;
|
|
2060
|
+
// Verify both branches exist
|
|
2061
|
+
const branches = await this.listBranches();
|
|
2062
|
+
if (!branches.includes(sourceBranch)) {
|
|
2063
|
+
throw new Error(`Source branch '${sourceBranch}' does not exist`);
|
|
2064
|
+
}
|
|
2065
|
+
if (!branches.includes(targetBranch)) {
|
|
2066
|
+
throw new Error(`Target branch '${targetBranch}' does not exist`);
|
|
2067
|
+
}
|
|
2068
|
+
// 1. Create temporary fork of source branch to read from
|
|
2069
|
+
const sourceFork = await this.fork(`${sourceBranch}-merge-temp-${Date.now()}`);
|
|
2070
|
+
await sourceFork.checkout(sourceBranch);
|
|
2071
|
+
// 2. Save current branch and checkout target
|
|
2072
|
+
const currentBranch = await this.getCurrentBranch();
|
|
2073
|
+
if (currentBranch !== targetBranch) {
|
|
2074
|
+
await this.checkout(targetBranch);
|
|
2075
|
+
}
|
|
2076
|
+
try {
|
|
2077
|
+
// 3. Get all entities from source and target
|
|
2078
|
+
const sourceResults = await sourceFork.find({});
|
|
2079
|
+
const targetResults = await this.find({});
|
|
2080
|
+
// Create maps for faster lookup
|
|
2081
|
+
const targetMap = new Map(targetResults.map(r => [r.entity.id, r.entity]));
|
|
2082
|
+
// 4. Merge entities
|
|
2083
|
+
for (const sourceResult of sourceResults) {
|
|
2084
|
+
const sourceEntity = sourceResult.entity;
|
|
2085
|
+
const targetEntity = targetMap.get(sourceEntity.id);
|
|
2086
|
+
if (!targetEntity) {
|
|
2087
|
+
// NEW entity in source - ADD to target
|
|
2088
|
+
await this.add({
|
|
2089
|
+
id: sourceEntity.id,
|
|
2090
|
+
type: sourceEntity.type,
|
|
2091
|
+
data: sourceEntity.data,
|
|
2092
|
+
vector: sourceEntity.vector
|
|
2093
|
+
});
|
|
2094
|
+
added++;
|
|
2095
|
+
}
|
|
2096
|
+
else {
|
|
2097
|
+
// Entity exists in both branches - check for conflicts
|
|
2098
|
+
const sourceTime = sourceEntity.updatedAt || sourceEntity.createdAt || 0;
|
|
2099
|
+
const targetTime = targetEntity.updatedAt || targetEntity.createdAt || 0;
|
|
2100
|
+
// If timestamps are identical, no change needed
|
|
2101
|
+
if (sourceTime === targetTime) {
|
|
2102
|
+
continue;
|
|
2103
|
+
}
|
|
2104
|
+
// Apply merge strategy
|
|
2105
|
+
if (strategy === 'last-write-wins') {
|
|
2106
|
+
if (sourceTime > targetTime) {
|
|
2107
|
+
// Source is newer, update target
|
|
2108
|
+
await this.update({ id: sourceEntity.id, data: sourceEntity.data });
|
|
2109
|
+
modified++;
|
|
2110
|
+
}
|
|
2111
|
+
// else target is newer, keep target
|
|
2112
|
+
}
|
|
2113
|
+
else if (strategy === 'first-write-wins') {
|
|
2114
|
+
if (sourceTime < targetTime) {
|
|
2115
|
+
// Source is older, update target
|
|
2116
|
+
await this.update({ id: sourceEntity.id, data: sourceEntity.data });
|
|
2117
|
+
modified++;
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
else if (strategy === 'custom' && options?.onConflict) {
|
|
2121
|
+
// Custom conflict resolution
|
|
2122
|
+
const resolved = await options.onConflict(targetEntity, sourceEntity);
|
|
2123
|
+
await this.update({ id: sourceEntity.id, data: resolved.data });
|
|
2124
|
+
modified++;
|
|
2125
|
+
conflicts++;
|
|
2126
|
+
}
|
|
2127
|
+
else {
|
|
2128
|
+
// Conflict detected but no resolution strategy
|
|
2129
|
+
conflicts++;
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
// 5. Merge relationships (verbs)
|
|
2134
|
+
const sourceVerbsResult = await sourceFork.storage.getVerbs({});
|
|
2135
|
+
const targetVerbsResult = await this.storage.getVerbs({});
|
|
2136
|
+
const sourceVerbs = sourceVerbsResult.items || [];
|
|
2137
|
+
const targetVerbs = targetVerbsResult.items || [];
|
|
2138
|
+
// Create set of existing target relationships for deduplication
|
|
2139
|
+
const targetRelSet = new Set(targetVerbs.map((v) => `${v.sourceId}-${v.verb}-${v.targetId}`));
|
|
2140
|
+
// Add relationships that don't exist in target
|
|
2141
|
+
for (const sourceVerb of sourceVerbs) {
|
|
2142
|
+
const key = `${sourceVerb.sourceId}-${sourceVerb.verb}-${sourceVerb.targetId}`;
|
|
2143
|
+
if (!targetRelSet.has(key)) {
|
|
2144
|
+
// Only add if both entities exist in target
|
|
2145
|
+
const hasSource = targetMap.has(sourceVerb.sourceId);
|
|
2146
|
+
const hasTarget = targetMap.has(sourceVerb.targetId);
|
|
2147
|
+
if (hasSource && hasTarget) {
|
|
2148
|
+
await this.relate({
|
|
2149
|
+
from: sourceVerb.sourceId,
|
|
2150
|
+
to: sourceVerb.targetId,
|
|
2151
|
+
type: sourceVerb.verb,
|
|
2152
|
+
weight: sourceVerb.weight,
|
|
2153
|
+
metadata: sourceVerb.metadata
|
|
2154
|
+
});
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
}
|
|
2158
|
+
// 6. Create merge commit
|
|
2159
|
+
if ('commitLog' in this.storage) {
|
|
2160
|
+
await this.commit({
|
|
2161
|
+
message: `Merge ${sourceBranch} into ${targetBranch}`,
|
|
2162
|
+
author: options?.author || 'system',
|
|
2163
|
+
metadata: {
|
|
2164
|
+
mergeType: 'branch',
|
|
2165
|
+
source: sourceBranch,
|
|
2166
|
+
target: targetBranch,
|
|
2167
|
+
strategy,
|
|
2168
|
+
stats: { added, modified, deleted, conflicts }
|
|
2169
|
+
}
|
|
2170
|
+
});
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
finally {
|
|
2174
|
+
// 7. Clean up temporary fork (just delete the temp branch)
|
|
2175
|
+
try {
|
|
2176
|
+
const tempBranchName = `${sourceBranch}-merge-temp-${Date.now()}`;
|
|
2177
|
+
const branches = await this.listBranches();
|
|
2178
|
+
if (branches.includes(tempBranchName)) {
|
|
2179
|
+
await this.deleteBranch(tempBranchName);
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
catch (err) {
|
|
2183
|
+
// Ignore cleanup errors
|
|
2184
|
+
}
|
|
2185
|
+
// Restore original branch if needed
|
|
2186
|
+
if (currentBranch !== targetBranch) {
|
|
2187
|
+
await this.checkout(currentBranch);
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
return { added, modified, deleted, conflicts };
|
|
2191
|
+
});
|
|
2192
|
+
}
|
|
2193
|
+
/**
|
|
2194
|
+
* Compare differences between two branches (like git diff)
|
|
2195
|
+
* @param sourceBranch - Branch to compare from (defaults to current branch)
|
|
2196
|
+
* @param targetBranch - Branch to compare to (defaults to 'main')
|
|
2197
|
+
* @returns Diff result showing added, modified, and deleted entities/relationships
|
|
2198
|
+
*
|
|
2199
|
+
* @example
|
|
2200
|
+
* ```typescript
|
|
2201
|
+
* // Compare current branch with main
|
|
2202
|
+
* const diff = await brain.diff()
|
|
2203
|
+
*
|
|
2204
|
+
* // Compare two specific branches
|
|
2205
|
+
* const diff = await brain.diff('experiment', 'main')
|
|
2206
|
+
* console.log(diff)
|
|
2207
|
+
* // {
|
|
2208
|
+
* // entities: { added: 5, modified: 3, deleted: 1 },
|
|
2209
|
+
* // relationships: { added: 10, modified: 2, deleted: 0 }
|
|
2210
|
+
* // }
|
|
2211
|
+
* ```
|
|
2212
|
+
*/
|
|
2213
|
+
async diff(sourceBranch, targetBranch) {
|
|
2214
|
+
await this.ensureInitialized();
|
|
2215
|
+
return this.augmentationRegistry.execute('diff', { sourceBranch, targetBranch }, async () => {
|
|
2216
|
+
// Default branches
|
|
2217
|
+
const source = sourceBranch || (await this.getCurrentBranch());
|
|
2218
|
+
const target = targetBranch || 'main';
|
|
2219
|
+
const currentBranch = await this.getCurrentBranch();
|
|
2220
|
+
// If source is current branch, use this instance directly (no fork needed)
|
|
2221
|
+
let sourceFork;
|
|
2222
|
+
let sourceForkCreated = false;
|
|
2223
|
+
if (source === currentBranch) {
|
|
2224
|
+
sourceFork = this;
|
|
2225
|
+
}
|
|
2226
|
+
else {
|
|
2227
|
+
sourceFork = await this.fork(`temp-diff-source-${Date.now()}`);
|
|
2228
|
+
sourceForkCreated = true;
|
|
2229
|
+
try {
|
|
2230
|
+
await sourceFork.checkout(source);
|
|
2231
|
+
}
|
|
2232
|
+
catch (err) {
|
|
2233
|
+
// If checkout fails, branch may not exist - just use current state
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
// If target is current branch, use this instance directly (no fork needed)
|
|
2237
|
+
let targetFork;
|
|
2238
|
+
let targetForkCreated = false;
|
|
2239
|
+
if (target === currentBranch) {
|
|
2240
|
+
targetFork = this;
|
|
2241
|
+
}
|
|
2242
|
+
else {
|
|
2243
|
+
targetFork = await this.fork(`temp-diff-target-${Date.now()}`);
|
|
2244
|
+
targetForkCreated = true;
|
|
2245
|
+
try {
|
|
2246
|
+
await targetFork.checkout(target);
|
|
2247
|
+
}
|
|
2248
|
+
catch (err) {
|
|
2249
|
+
// If checkout fails, branch may not exist - just use current state
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
try {
|
|
2253
|
+
// Get all entities from both branches
|
|
2254
|
+
const sourceResults = await sourceFork.find({});
|
|
2255
|
+
const targetResults = await targetFork.find({});
|
|
2256
|
+
// Create maps for lookup
|
|
2257
|
+
const sourceMap = new Map(sourceResults.map(r => [r.entity.id, r.entity]));
|
|
2258
|
+
const targetMap = new Map(targetResults.map(r => [r.entity.id, r.entity]));
|
|
2259
|
+
// Track differences
|
|
2260
|
+
const entitiesAdded = [];
|
|
2261
|
+
const entitiesModified = [];
|
|
2262
|
+
const entitiesDeleted = [];
|
|
2263
|
+
// Find added and modified entities
|
|
2264
|
+
for (const [id, sourceEntity] of sourceMap.entries()) {
|
|
2265
|
+
const targetEntity = targetMap.get(id);
|
|
2266
|
+
if (!targetEntity) {
|
|
2267
|
+
// Entity exists in source but not target = ADDED
|
|
2268
|
+
entitiesAdded.push({
|
|
2269
|
+
id: sourceEntity.id,
|
|
2270
|
+
type: sourceEntity.type,
|
|
2271
|
+
data: sourceEntity.data
|
|
2272
|
+
});
|
|
2273
|
+
}
|
|
2274
|
+
else {
|
|
2275
|
+
// Entity exists in both - check for modifications
|
|
2276
|
+
const changes = [];
|
|
2277
|
+
if (sourceEntity.data !== targetEntity.data) {
|
|
2278
|
+
changes.push('data');
|
|
2279
|
+
}
|
|
2280
|
+
if ((sourceEntity.updatedAt || 0) !== (targetEntity.updatedAt || 0)) {
|
|
2281
|
+
changes.push('updatedAt');
|
|
2282
|
+
}
|
|
2283
|
+
if (changes.length > 0) {
|
|
2284
|
+
entitiesModified.push({
|
|
2285
|
+
id: sourceEntity.id,
|
|
2286
|
+
type: sourceEntity.type,
|
|
2287
|
+
changes
|
|
2288
|
+
});
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
// Find deleted entities (in target but not in source)
|
|
2293
|
+
for (const [id, targetEntity] of targetMap.entries()) {
|
|
2294
|
+
if (!sourceMap.has(id)) {
|
|
2295
|
+
entitiesDeleted.push({
|
|
2296
|
+
id: targetEntity.id,
|
|
2297
|
+
type: targetEntity.type
|
|
2298
|
+
});
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
// Compare relationships
|
|
2302
|
+
const sourceVerbsResult = await sourceFork.storage.getVerbs({});
|
|
2303
|
+
const targetVerbsResult = await targetFork.storage.getVerbs({});
|
|
2304
|
+
const sourceVerbs = sourceVerbsResult.items || [];
|
|
2305
|
+
const targetVerbs = targetVerbsResult.items || [];
|
|
2306
|
+
const sourceRelMap = new Map(sourceVerbs.map((v) => [`${v.sourceId}-${v.verb}-${v.targetId}`, v]));
|
|
2307
|
+
const targetRelMap = new Map(targetVerbs.map((v) => [`${v.sourceId}-${v.verb}-${v.targetId}`, v]));
|
|
2308
|
+
const relationshipsAdded = [];
|
|
2309
|
+
const relationshipsModified = [];
|
|
2310
|
+
const relationshipsDeleted = [];
|
|
2311
|
+
// Find added and modified relationships
|
|
2312
|
+
for (const [key, sourceVerb] of sourceRelMap.entries()) {
|
|
2313
|
+
const targetVerb = targetRelMap.get(key);
|
|
2314
|
+
if (!targetVerb) {
|
|
2315
|
+
// Relationship exists in source but not target = ADDED
|
|
2316
|
+
relationshipsAdded.push({
|
|
2317
|
+
from: sourceVerb.sourceId,
|
|
2318
|
+
to: sourceVerb.targetId,
|
|
2319
|
+
type: sourceVerb.verb
|
|
2320
|
+
});
|
|
2321
|
+
}
|
|
2322
|
+
else {
|
|
2323
|
+
// Relationship exists in both - check for modifications
|
|
2324
|
+
const changes = [];
|
|
2325
|
+
if ((sourceVerb.weight || 0) !== (targetVerb.weight || 0)) {
|
|
2326
|
+
changes.push('weight');
|
|
2327
|
+
}
|
|
2328
|
+
if (JSON.stringify(sourceVerb.metadata) !== JSON.stringify(targetVerb.metadata)) {
|
|
2329
|
+
changes.push('metadata');
|
|
2330
|
+
}
|
|
2331
|
+
if (changes.length > 0) {
|
|
2332
|
+
relationshipsModified.push({
|
|
2333
|
+
from: sourceVerb.sourceId,
|
|
2334
|
+
to: sourceVerb.targetId,
|
|
2335
|
+
type: sourceVerb.verb,
|
|
2336
|
+
changes
|
|
2337
|
+
});
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
// Find deleted relationships
|
|
2342
|
+
for (const [key, targetVerb] of targetRelMap.entries()) {
|
|
2343
|
+
if (!sourceRelMap.has(key)) {
|
|
2344
|
+
relationshipsDeleted.push({
|
|
2345
|
+
from: targetVerb.sourceId,
|
|
2346
|
+
to: targetVerb.targetId,
|
|
2347
|
+
type: targetVerb.verb
|
|
2348
|
+
});
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
return {
|
|
2352
|
+
entities: {
|
|
2353
|
+
added: entitiesAdded,
|
|
2354
|
+
modified: entitiesModified,
|
|
2355
|
+
deleted: entitiesDeleted
|
|
2356
|
+
},
|
|
2357
|
+
relationships: {
|
|
2358
|
+
added: relationshipsAdded,
|
|
2359
|
+
modified: relationshipsModified,
|
|
2360
|
+
deleted: relationshipsDeleted
|
|
2361
|
+
},
|
|
2362
|
+
summary: {
|
|
2363
|
+
entitiesAdded: entitiesAdded.length,
|
|
2364
|
+
entitiesModified: entitiesModified.length,
|
|
2365
|
+
entitiesDeleted: entitiesDeleted.length,
|
|
2366
|
+
relationshipsAdded: relationshipsAdded.length,
|
|
2367
|
+
relationshipsModified: relationshipsModified.length,
|
|
2368
|
+
relationshipsDeleted: relationshipsDeleted.length
|
|
2369
|
+
}
|
|
2370
|
+
};
|
|
2371
|
+
}
|
|
2372
|
+
finally {
|
|
2373
|
+
// Clean up temporary forks (only if we created them)
|
|
2374
|
+
try {
|
|
2375
|
+
const branches = await this.listBranches();
|
|
2376
|
+
if (sourceForkCreated && sourceFork !== this) {
|
|
2377
|
+
const sourceBranchName = await sourceFork.getCurrentBranch();
|
|
2378
|
+
if (branches.includes(sourceBranchName)) {
|
|
2379
|
+
await this.deleteBranch(sourceBranchName);
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2382
|
+
if (targetForkCreated && targetFork !== this) {
|
|
2383
|
+
const targetBranchName = await targetFork.getCurrentBranch();
|
|
2384
|
+
if (branches.includes(targetBranchName)) {
|
|
2385
|
+
await this.deleteBranch(targetBranchName);
|
|
2386
|
+
}
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
catch (err) {
|
|
2390
|
+
// Ignore cleanup errors
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2393
|
+
});
|
|
2394
|
+
}
|
|
2395
|
+
/**
|
|
2396
|
+
* Delete a branch/fork
|
|
2397
|
+
* @param branch - Branch name to delete
|
|
2398
|
+
*
|
|
2399
|
+
* @example
|
|
2400
|
+
* ```typescript
|
|
2401
|
+
* await brain.deleteBranch('old-experiment')
|
|
2402
|
+
* ```
|
|
2403
|
+
*/
|
|
2404
|
+
async deleteBranch(branch) {
|
|
2405
|
+
await this.ensureInitialized();
|
|
2406
|
+
return this.augmentationRegistry.execute('deleteBranch', { branch }, async () => {
|
|
2407
|
+
if (!('refManager' in this.storage)) {
|
|
2408
|
+
throw new Error('Branch management requires COW-enabled storage (v5.0.0+)');
|
|
2409
|
+
}
|
|
2410
|
+
const currentBranch = await this.getCurrentBranch();
|
|
2411
|
+
if (branch === currentBranch) {
|
|
2412
|
+
throw new Error('Cannot delete current branch');
|
|
2413
|
+
}
|
|
2414
|
+
const refManager = this.storage.refManager;
|
|
2415
|
+
await refManager.deleteRef(`heads/${branch}`);
|
|
2416
|
+
});
|
|
2417
|
+
}
|
|
2418
|
+
/**
|
|
2419
|
+
* Get commit history for current branch
|
|
2420
|
+
* @param options - History options (limit, offset, author)
|
|
2421
|
+
* @returns Array of commits
|
|
2422
|
+
*
|
|
2423
|
+
* @example
|
|
2424
|
+
* ```typescript
|
|
2425
|
+
* const history = await brain.getHistory({ limit: 10 })
|
|
2426
|
+
* history.forEach(commit => {
|
|
2427
|
+
* console.log(`${commit.hash}: ${commit.message}`)
|
|
2428
|
+
* })
|
|
2429
|
+
* ```
|
|
2430
|
+
*/
|
|
2431
|
+
async getHistory(options) {
|
|
2432
|
+
await this.ensureInitialized();
|
|
2433
|
+
return this.augmentationRegistry.execute('getHistory', { options }, async () => {
|
|
2434
|
+
if (!('commitLog' in this.storage) || !('refManager' in this.storage)) {
|
|
2435
|
+
throw new Error('History requires COW-enabled storage (v5.0.0+)');
|
|
2436
|
+
}
|
|
2437
|
+
const commitLog = this.storage.commitLog;
|
|
2438
|
+
const currentBranch = await this.getCurrentBranch();
|
|
2439
|
+
// Get commit history for current branch
|
|
2440
|
+
const commits = await commitLog.getHistory(`heads/${currentBranch}`, {
|
|
2441
|
+
maxCount: options?.limit || 10
|
|
2442
|
+
});
|
|
2443
|
+
// Map to expected format (compute hash for each commit)
|
|
2444
|
+
return commits.map((commit) => ({
|
|
2445
|
+
hash: this.storage.blobStorage.constructor.hash(Buffer.from(JSON.stringify(commit))),
|
|
2446
|
+
message: commit.message,
|
|
2447
|
+
author: commit.author,
|
|
2448
|
+
timestamp: commit.timestamp,
|
|
2449
|
+
metadata: commit.metadata
|
|
2450
|
+
}));
|
|
2451
|
+
});
|
|
2452
|
+
}
|
|
1773
2453
|
/**
|
|
1774
2454
|
* Get total count of nouns - O(1) operation
|
|
1775
2455
|
* @returns Promise that resolves to the total number of nouns
|
|
@@ -1971,36 +2651,46 @@ export class Brainy {
|
|
|
1971
2651
|
return await coordinator.import(source, options);
|
|
1972
2652
|
}
|
|
1973
2653
|
/**
|
|
1974
|
-
* Virtual File System API - Knowledge Operating System
|
|
2654
|
+
* Virtual File System API - Knowledge Operating System (v5.0.1+)
|
|
1975
2655
|
*
|
|
1976
|
-
* Returns a cached VFS instance
|
|
2656
|
+
* Returns a cached VFS instance that is auto-initialized during brain.init().
|
|
2657
|
+
* No separate initialization needed!
|
|
1977
2658
|
*
|
|
1978
2659
|
* @example After import
|
|
1979
2660
|
* ```typescript
|
|
1980
2661
|
* await brain.import('./data.xlsx', { vfsPath: '/imports/data' })
|
|
1981
|
-
*
|
|
1982
|
-
* const
|
|
1983
|
-
* await vfs.init() // Required! (safe to call multiple times)
|
|
1984
|
-
* const files = await vfs.readdir('/imports/data')
|
|
2662
|
+
* // VFS ready immediately - no init() call needed!
|
|
2663
|
+
* const files = await brain.vfs.readdir('/imports/data')
|
|
1985
2664
|
* ```
|
|
1986
2665
|
*
|
|
1987
2666
|
* @example Direct VFS usage
|
|
1988
2667
|
* ```typescript
|
|
1989
|
-
*
|
|
1990
|
-
* await vfs.
|
|
1991
|
-
* await vfs.
|
|
1992
|
-
* const content = await vfs.readFile('/docs/readme.md')
|
|
2668
|
+
* await brain.init() // VFS auto-initialized here!
|
|
2669
|
+
* await brain.vfs.writeFile('/docs/readme.md', 'Hello World')
|
|
2670
|
+
* const content = await brain.vfs.readFile('/docs/readme.md')
|
|
1993
2671
|
* ```
|
|
1994
2672
|
*
|
|
1995
|
-
*
|
|
1996
|
-
*
|
|
1997
|
-
*
|
|
2673
|
+
* @example With fork (COW isolation)
|
|
2674
|
+
* ```typescript
|
|
2675
|
+
* await brain.init()
|
|
2676
|
+
* await brain.vfs.writeFile('/config.json', '{"v": 1}')
|
|
2677
|
+
*
|
|
2678
|
+
* const fork = await brain.fork('experiment')
|
|
2679
|
+
* // Fork inherits parent's files
|
|
2680
|
+
* const config = await fork.vfs.readFile('/config.json')
|
|
2681
|
+
* // Fork modifications are isolated
|
|
2682
|
+
* await fork.vfs.writeFile('/test.txt', 'Fork only')
|
|
2683
|
+
* ```
|
|
1998
2684
|
*
|
|
1999
|
-
* **Pattern:** The VFS instance is cached, so multiple calls to brain.vfs
|
|
2685
|
+
* **Pattern:** The VFS instance is cached, so multiple calls to brain.vfs
|
|
2000
2686
|
* return the same instance. This ensures import and user code share state.
|
|
2687
|
+
*
|
|
2688
|
+
* @since v5.0.1 - Auto-initialization during brain.init()
|
|
2001
2689
|
*/
|
|
2002
|
-
vfs() {
|
|
2690
|
+
get vfs() {
|
|
2003
2691
|
if (!this._vfs) {
|
|
2692
|
+
// VFS is initialized during brain.init() (v5.0.1)
|
|
2693
|
+
// If not initialized yet, create instance but user should call brain.init() first
|
|
2004
2694
|
this._vfs = new VirtualFileSystem(this);
|
|
2005
2695
|
}
|
|
2006
2696
|
return this._vfs;
|