@soulcraft/brainy 6.1.0 → 6.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +271 -0
- package/dist/augmentations/KnowledgeAugmentation.d.ts +40 -0
- package/dist/augmentations/KnowledgeAugmentation.js +251 -0
- package/dist/brainy.d.ts +17 -13
- package/dist/brainy.js +172 -41
- package/dist/coreTypes.d.ts +12 -0
- package/dist/graph/graphAdjacencyIndex.d.ts +23 -0
- package/dist/graph/graphAdjacencyIndex.js +49 -0
- package/dist/importManager.d.ts +78 -0
- package/dist/importManager.js +267 -0
- package/dist/query/typeInference.d.ts +158 -0
- package/dist/query/typeInference.js +760 -0
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +252 -0
- package/dist/storage/adapters/typeAwareStorageAdapter.js +814 -0
- package/dist/storage/baseStorage.d.ts +36 -0
- package/dist/storage/baseStorage.js +159 -4
- package/dist/storage/cow/binaryDataCodec.d.ts +13 -2
- package/dist/storage/cow/binaryDataCodec.js +15 -2
- package/dist/types/brainy.types.d.ts +1 -0
- package/dist/types/brainyDataInterface.d.ts +52 -0
- package/dist/types/brainyDataInterface.js +10 -0
- package/dist/utils/metadataIndex.d.ts +17 -0
- package/dist/utils/metadataIndex.js +63 -0
- package/dist/vfs/ConceptSystem.d.ts +203 -0
- package/dist/vfs/ConceptSystem.js +545 -0
- package/dist/vfs/EntityManager.d.ts +75 -0
- package/dist/vfs/EntityManager.js +216 -0
- package/dist/vfs/EventRecorder.d.ts +84 -0
- package/dist/vfs/EventRecorder.js +269 -0
- package/dist/vfs/GitBridge.d.ts +167 -0
- package/dist/vfs/GitBridge.js +537 -0
- package/dist/vfs/KnowledgeLayer.d.ts +35 -0
- package/dist/vfs/KnowledgeLayer.js +443 -0
- package/dist/vfs/PersistentEntitySystem.d.ts +165 -0
- package/dist/vfs/PersistentEntitySystem.js +503 -0
- package/dist/vfs/SemanticVersioning.d.ts +105 -0
- package/dist/vfs/SemanticVersioning.js +309 -0
- package/dist/vfs/VirtualFileSystem.d.ts +37 -2
- package/dist/vfs/VirtualFileSystem.js +105 -68
- package/package.json +1 -1
|
@@ -181,6 +181,24 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
181
181
|
* @returns Combined verb + metadata or null
|
|
182
182
|
*/
|
|
183
183
|
getVerb(id: string): Promise<HNSWVerbWithMetadata | null>;
|
|
184
|
+
/**
|
|
185
|
+
* Batch get multiple verbs (v6.2.0 - N+1 fix)
|
|
186
|
+
*
|
|
187
|
+
* **Performance**: Eliminates N+1 pattern for verb loading
|
|
188
|
+
* - Current: N × getVerb() = N × 50ms on GCS = 250ms for 5 verbs
|
|
189
|
+
* - Batched: 1 × getVerbsBatch() = 1 × 50ms on GCS = 50ms (**5x faster**)
|
|
190
|
+
*
|
|
191
|
+
* **Use cases:**
|
|
192
|
+
* - graphIndex.getVerbsBatchCached() for relate() duplicate checking
|
|
193
|
+
* - Loading relationships in batch operations
|
|
194
|
+
* - Pre-loading verbs for graph traversal
|
|
195
|
+
*
|
|
196
|
+
* @param ids Array of verb IDs to fetch
|
|
197
|
+
* @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
|
|
198
|
+
*
|
|
199
|
+
* @since v6.2.0
|
|
200
|
+
*/
|
|
201
|
+
getVerbsBatch(ids: string[]): Promise<Map<string, HNSWVerbWithMetadata>>;
|
|
184
202
|
/**
|
|
185
203
|
* Convert HNSWVerb to GraphVerb by combining with metadata
|
|
186
204
|
* DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata.
|
|
@@ -494,6 +512,24 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
494
512
|
* @since v5.12.0
|
|
495
513
|
*/
|
|
496
514
|
getNounMetadataBatch(ids: string[]): Promise<Map<string, NounMetadata>>;
|
|
515
|
+
/**
|
|
516
|
+
* Batch get multiple nouns with vectors (v6.2.0 - N+1 fix)
|
|
517
|
+
*
|
|
518
|
+
* **Performance**: Eliminates N+1 pattern for vector loading
|
|
519
|
+
* - Current: N × getNoun() = N × 50ms on GCS = 500ms for 10 entities
|
|
520
|
+
* - Batched: 1 × getNounBatch() = 1 × 50ms on GCS = 50ms (**10x faster**)
|
|
521
|
+
*
|
|
522
|
+
* **Use cases:**
|
|
523
|
+
* - batchGet() with includeVectors: true
|
|
524
|
+
* - Loading entities for similarity computation
|
|
525
|
+
* - Pre-loading vectors for batch processing
|
|
526
|
+
*
|
|
527
|
+
* @param ids Array of entity IDs to fetch (with vectors)
|
|
528
|
+
* @returns Map of id → HNSWNounWithMetadata (only successful reads included)
|
|
529
|
+
*
|
|
530
|
+
* @since v6.2.0
|
|
531
|
+
*/
|
|
532
|
+
getNounBatch(ids: string[]): Promise<Map<string, HNSWNounWithMetadata>>;
|
|
497
533
|
/**
|
|
498
534
|
* Batch read multiple storage paths with COW inheritance support (v5.12.0)
|
|
499
535
|
*
|
|
@@ -10,7 +10,7 @@ import { getShardIdFromUuid } from './sharding.js';
|
|
|
10
10
|
import { RefManager } from './cow/RefManager.js';
|
|
11
11
|
import { BlobStorage } from './cow/BlobStorage.js';
|
|
12
12
|
import { CommitLog } from './cow/CommitLog.js';
|
|
13
|
-
import { unwrapBinaryData
|
|
13
|
+
import { unwrapBinaryData } from './cow/binaryDataCodec.js';
|
|
14
14
|
import { prodLog } from '../utils/logger.js';
|
|
15
15
|
// Clean directory structure (v4.7.2+)
|
|
16
16
|
// All storage adapters use this consistent structure
|
|
@@ -278,9 +278,25 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
278
278
|
}
|
|
279
279
|
},
|
|
280
280
|
put: async (key, data) => {
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
|
|
281
|
+
// v6.2.0 PERMANENT FIX: Use key naming convention (explicit type contract)
|
|
282
|
+
// NO GUESSING - key format explicitly declares data type:
|
|
283
|
+
//
|
|
284
|
+
// JSON keys (metadata and refs):
|
|
285
|
+
// - 'ref:*' → JSON (RefManager: refs, HEAD, branches)
|
|
286
|
+
// - 'blob-meta:hash' → JSON (BlobStorage: blob metadata)
|
|
287
|
+
// - 'commit-meta:hash'→ JSON (BlobStorage: commit metadata)
|
|
288
|
+
// - 'tree-meta:hash' → JSON (BlobStorage: tree metadata)
|
|
289
|
+
//
|
|
290
|
+
// Binary keys (blob data):
|
|
291
|
+
// - 'blob:hash' → Binary (BlobStorage: compressed/raw blob data)
|
|
292
|
+
// - 'commit:hash' → Binary (BlobStorage: commit object data)
|
|
293
|
+
// - 'tree:hash' → Binary (BlobStorage: tree object data)
|
|
294
|
+
//
|
|
295
|
+
// This eliminates the fragile JSON.parse() guessing that caused blob integrity
|
|
296
|
+
// failures when compressed data accidentally parsed as valid JSON.
|
|
297
|
+
const obj = key.includes('-meta:') || key.startsWith('ref:')
|
|
298
|
+
? JSON.parse(data.toString()) // Metadata/refs: ALWAYS JSON.stringify'd
|
|
299
|
+
: { _binary: true, data: data.toString('base64') }; // Blobs: ALWAYS binary (possibly compressed)
|
|
284
300
|
await this.writeObjectToPath(`_cow/${key}`, obj);
|
|
285
301
|
},
|
|
286
302
|
delete: async (key) => {
|
|
@@ -642,6 +658,76 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
642
658
|
metadata: customMetadata
|
|
643
659
|
};
|
|
644
660
|
}
|
|
661
|
+
/**
|
|
662
|
+
* Batch get multiple verbs (v6.2.0 - N+1 fix)
|
|
663
|
+
*
|
|
664
|
+
* **Performance**: Eliminates N+1 pattern for verb loading
|
|
665
|
+
* - Current: N × getVerb() = N × 50ms on GCS = 250ms for 5 verbs
|
|
666
|
+
* - Batched: 1 × getVerbsBatch() = 1 × 50ms on GCS = 50ms (**5x faster**)
|
|
667
|
+
*
|
|
668
|
+
* **Use cases:**
|
|
669
|
+
* - graphIndex.getVerbsBatchCached() for relate() duplicate checking
|
|
670
|
+
* - Loading relationships in batch operations
|
|
671
|
+
* - Pre-loading verbs for graph traversal
|
|
672
|
+
*
|
|
673
|
+
* @param ids Array of verb IDs to fetch
|
|
674
|
+
* @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
|
|
675
|
+
*
|
|
676
|
+
* @since v6.2.0
|
|
677
|
+
*/
|
|
678
|
+
async getVerbsBatch(ids) {
|
|
679
|
+
await this.ensureInitialized();
|
|
680
|
+
const results = new Map();
|
|
681
|
+
if (ids.length === 0)
|
|
682
|
+
return results;
|
|
683
|
+
// v6.2.0: Batch-fetch vectors and metadata in parallel
|
|
684
|
+
// Build paths for vectors
|
|
685
|
+
const vectorPaths = ids.map(id => ({
|
|
686
|
+
path: getVerbVectorPath(id),
|
|
687
|
+
id
|
|
688
|
+
}));
|
|
689
|
+
// Build paths for metadata
|
|
690
|
+
const metadataPaths = ids.map(id => ({
|
|
691
|
+
path: getVerbMetadataPath(id),
|
|
692
|
+
id
|
|
693
|
+
}));
|
|
694
|
+
// Batch read vectors and metadata in parallel
|
|
695
|
+
const [vectorResults, metadataResults] = await Promise.all([
|
|
696
|
+
this.readBatchWithInheritance(vectorPaths.map(p => p.path)),
|
|
697
|
+
this.readBatchWithInheritance(metadataPaths.map(p => p.path))
|
|
698
|
+
]);
|
|
699
|
+
// Combine vectors + metadata into HNSWVerbWithMetadata
|
|
700
|
+
for (const { path: vectorPath, id } of vectorPaths) {
|
|
701
|
+
const vectorData = vectorResults.get(vectorPath);
|
|
702
|
+
const metadataPath = getVerbMetadataPath(id);
|
|
703
|
+
const metadataData = metadataResults.get(metadataPath);
|
|
704
|
+
if (vectorData && metadataData) {
|
|
705
|
+
// Deserialize verb
|
|
706
|
+
const verb = this.deserializeVerb(vectorData);
|
|
707
|
+
// Extract standard fields to top-level (v4.8.0 pattern)
|
|
708
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataData;
|
|
709
|
+
results.set(id, {
|
|
710
|
+
id: verb.id,
|
|
711
|
+
vector: verb.vector,
|
|
712
|
+
connections: verb.connections,
|
|
713
|
+
verb: verb.verb,
|
|
714
|
+
sourceId: verb.sourceId,
|
|
715
|
+
targetId: verb.targetId,
|
|
716
|
+
// v4.8.0: Standard fields at top-level
|
|
717
|
+
createdAt: createdAt || Date.now(),
|
|
718
|
+
updatedAt: updatedAt || Date.now(),
|
|
719
|
+
confidence: confidence,
|
|
720
|
+
weight: weight,
|
|
721
|
+
service: service,
|
|
722
|
+
data: data,
|
|
723
|
+
createdBy,
|
|
724
|
+
// Only custom user fields remain in metadata
|
|
725
|
+
metadata: customMetadata
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
return results;
|
|
730
|
+
}
|
|
645
731
|
/**
|
|
646
732
|
* Convert HNSWVerb to GraphVerb by combining with metadata
|
|
647
733
|
* DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata.
|
|
@@ -1553,6 +1639,75 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1553
1639
|
}
|
|
1554
1640
|
return results;
|
|
1555
1641
|
}
|
|
1642
|
+
/**
|
|
1643
|
+
* Batch get multiple nouns with vectors (v6.2.0 - N+1 fix)
|
|
1644
|
+
*
|
|
1645
|
+
* **Performance**: Eliminates N+1 pattern for vector loading
|
|
1646
|
+
* - Current: N × getNoun() = N × 50ms on GCS = 500ms for 10 entities
|
|
1647
|
+
* - Batched: 1 × getNounBatch() = 1 × 50ms on GCS = 50ms (**10x faster**)
|
|
1648
|
+
*
|
|
1649
|
+
* **Use cases:**
|
|
1650
|
+
* - batchGet() with includeVectors: true
|
|
1651
|
+
* - Loading entities for similarity computation
|
|
1652
|
+
* - Pre-loading vectors for batch processing
|
|
1653
|
+
*
|
|
1654
|
+
* @param ids Array of entity IDs to fetch (with vectors)
|
|
1655
|
+
* @returns Map of id → HNSWNounWithMetadata (only successful reads included)
|
|
1656
|
+
*
|
|
1657
|
+
* @since v6.2.0
|
|
1658
|
+
*/
|
|
1659
|
+
async getNounBatch(ids) {
|
|
1660
|
+
await this.ensureInitialized();
|
|
1661
|
+
const results = new Map();
|
|
1662
|
+
if (ids.length === 0)
|
|
1663
|
+
return results;
|
|
1664
|
+
// v6.2.0: Batch-fetch vectors and metadata in parallel
|
|
1665
|
+
// Build paths for vectors
|
|
1666
|
+
const vectorPaths = ids.map(id => ({
|
|
1667
|
+
path: getNounVectorPath(id),
|
|
1668
|
+
id
|
|
1669
|
+
}));
|
|
1670
|
+
// Build paths for metadata
|
|
1671
|
+
const metadataPaths = ids.map(id => ({
|
|
1672
|
+
path: getNounMetadataPath(id),
|
|
1673
|
+
id
|
|
1674
|
+
}));
|
|
1675
|
+
// Batch read vectors and metadata in parallel
|
|
1676
|
+
const [vectorResults, metadataResults] = await Promise.all([
|
|
1677
|
+
this.readBatchWithInheritance(vectorPaths.map(p => p.path)),
|
|
1678
|
+
this.readBatchWithInheritance(metadataPaths.map(p => p.path))
|
|
1679
|
+
]);
|
|
1680
|
+
// Combine vectors + metadata into HNSWNounWithMetadata
|
|
1681
|
+
for (const { path: vectorPath, id } of vectorPaths) {
|
|
1682
|
+
const vectorData = vectorResults.get(vectorPath);
|
|
1683
|
+
const metadataPath = getNounMetadataPath(id);
|
|
1684
|
+
const metadataData = metadataResults.get(metadataPath);
|
|
1685
|
+
if (vectorData && metadataData) {
|
|
1686
|
+
// Deserialize noun
|
|
1687
|
+
const noun = this.deserializeNoun(vectorData);
|
|
1688
|
+
// Extract standard fields to top-level (v4.8.0 pattern)
|
|
1689
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataData;
|
|
1690
|
+
results.set(id, {
|
|
1691
|
+
id: noun.id,
|
|
1692
|
+
vector: noun.vector,
|
|
1693
|
+
connections: noun.connections,
|
|
1694
|
+
level: noun.level,
|
|
1695
|
+
// v4.8.0: Standard fields at top-level
|
|
1696
|
+
type: nounType || NounType.Thing,
|
|
1697
|
+
createdAt: createdAt || Date.now(),
|
|
1698
|
+
updatedAt: updatedAt || Date.now(),
|
|
1699
|
+
confidence: confidence,
|
|
1700
|
+
weight: weight,
|
|
1701
|
+
service: service,
|
|
1702
|
+
data: data,
|
|
1703
|
+
createdBy,
|
|
1704
|
+
// Only custom user fields remain in metadata
|
|
1705
|
+
metadata: customMetadata
|
|
1706
|
+
});
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
return results;
|
|
1710
|
+
}
|
|
1556
1711
|
/**
|
|
1557
1712
|
* Batch read multiple storage paths with COW inheritance support (v5.12.0)
|
|
1558
1713
|
*
|
|
@@ -49,11 +49,22 @@ export declare function unwrapBinaryData(data: any): Buffer;
|
|
|
49
49
|
/**
|
|
50
50
|
* Wrap binary data for JSON storage
|
|
51
51
|
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
52
|
+
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
|
|
53
|
+
* ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
|
|
54
|
+
* ⚠️ This function exists for legacy/compatibility only.
|
|
55
|
+
*
|
|
56
|
+
* DEPRECATED APPROACH: Tries to guess if data is JSON by parsing.
|
|
57
|
+
* This is FRAGILE because compressed binary can accidentally parse as valid JSON,
|
|
58
|
+
* causing blob integrity failures.
|
|
59
|
+
*
|
|
60
|
+
* v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
|
|
61
|
+
* - Keys with '-meta:' or 'ref:' prefix → Always JSON
|
|
62
|
+
* - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
|
|
63
|
+
* No guessing needed!
|
|
54
64
|
*
|
|
55
65
|
* @param data - Buffer to wrap
|
|
56
66
|
* @returns Wrapped object or parsed JSON object
|
|
67
|
+
* @deprecated Use key-based dispatch in baseStorage.ts instead
|
|
57
68
|
*/
|
|
58
69
|
export declare function wrapBinaryData(data: Buffer): any;
|
|
59
70
|
/**
|
|
@@ -66,14 +66,27 @@ export function unwrapBinaryData(data) {
|
|
|
66
66
|
/**
|
|
67
67
|
* Wrap binary data for JSON storage
|
|
68
68
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
69
|
+
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
|
|
70
|
+
* ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
|
|
71
|
+
* ⚠️ This function exists for legacy/compatibility only.
|
|
72
|
+
*
|
|
73
|
+
* DEPRECATED APPROACH: Tries to guess if data is JSON by parsing.
|
|
74
|
+
* This is FRAGILE because compressed binary can accidentally parse as valid JSON,
|
|
75
|
+
* causing blob integrity failures.
|
|
76
|
+
*
|
|
77
|
+
* v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
|
|
78
|
+
* - Keys with '-meta:' or 'ref:' prefix → Always JSON
|
|
79
|
+
* - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
|
|
80
|
+
* No guessing needed!
|
|
71
81
|
*
|
|
72
82
|
* @param data - Buffer to wrap
|
|
73
83
|
* @returns Wrapped object or parsed JSON object
|
|
84
|
+
* @deprecated Use key-based dispatch in baseStorage.ts instead
|
|
74
85
|
*/
|
|
75
86
|
export function wrapBinaryData(data) {
|
|
76
87
|
// Try to parse as JSON first (for metadata, trees, commits)
|
|
88
|
+
// NOTE: This is the OLD approach - fragile because compressed data
|
|
89
|
+
// can accidentally parse as valid JSON!
|
|
77
90
|
try {
|
|
78
91
|
return JSON.parse(data.toString());
|
|
79
92
|
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BrainyInterface - Modern API Only
|
|
3
|
+
*
|
|
4
|
+
* This interface defines the MODERN methods from Brainy 3.0.
|
|
5
|
+
* Used to break circular dependencies while enforcing modern API usage.
|
|
6
|
+
*
|
|
7
|
+
* NO DEPRECATED METHODS - Only clean, modern API patterns.
|
|
8
|
+
*/
|
|
9
|
+
import { Vector } from '../coreTypes.js';
|
|
10
|
+
import { AddParams, RelateParams, Result, Entity, FindParams, SimilarParams } from './brainy.types.js';
|
|
11
|
+
export interface BrainyInterface<T = unknown> {
|
|
12
|
+
/**
|
|
13
|
+
* Initialize the database
|
|
14
|
+
*/
|
|
15
|
+
init(): Promise<void>;
|
|
16
|
+
/**
|
|
17
|
+
* Modern add method - unified entity creation
|
|
18
|
+
* @param params Parameters for adding entities
|
|
19
|
+
* @returns The ID of the created entity
|
|
20
|
+
*/
|
|
21
|
+
add(params: AddParams<T>): Promise<string>;
|
|
22
|
+
/**
|
|
23
|
+
* Modern relate method - unified relationship creation
|
|
24
|
+
* @param params Parameters for creating relationships
|
|
25
|
+
* @returns The ID of the created relationship
|
|
26
|
+
*/
|
|
27
|
+
relate(params: RelateParams<T>): Promise<string>;
|
|
28
|
+
/**
|
|
29
|
+
* Modern find method - unified search and discovery
|
|
30
|
+
* @param query Search query or parameters object
|
|
31
|
+
* @returns Array of search results
|
|
32
|
+
*/
|
|
33
|
+
find(query: string | FindParams<T>): Promise<Result<T>[]>;
|
|
34
|
+
/**
|
|
35
|
+
* Modern get method - retrieve entities by ID
|
|
36
|
+
* @param id The entity ID to retrieve
|
|
37
|
+
* @returns Entity or null if not found
|
|
38
|
+
*/
|
|
39
|
+
get(id: string): Promise<Entity<T> | null>;
|
|
40
|
+
/**
|
|
41
|
+
* Modern similar method - find similar entities
|
|
42
|
+
* @param params Parameters for similarity search
|
|
43
|
+
* @returns Array of similar entities with scores
|
|
44
|
+
*/
|
|
45
|
+
similar(params: SimilarParams<T>): Promise<Result<T>[]>;
|
|
46
|
+
/**
|
|
47
|
+
* Generate embedding vector from text
|
|
48
|
+
* @param text The text to embed
|
|
49
|
+
* @returns Vector representation of the text
|
|
50
|
+
*/
|
|
51
|
+
embed(text: string): Promise<Vector>;
|
|
52
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BrainyInterface - Modern API Only
|
|
3
|
+
*
|
|
4
|
+
* This interface defines the MODERN methods from Brainy 3.0.
|
|
5
|
+
* Used to break circular dependencies while enforcing modern API usage.
|
|
6
|
+
*
|
|
7
|
+
* NO DEPRECATED METHODS - Only clean, modern API patterns.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=brainyDataInterface.js.map
|
|
@@ -436,6 +436,23 @@ export declare class MetadataIndexManager {
|
|
|
436
436
|
* Get all entity types and their counts - O(1) operation
|
|
437
437
|
*/
|
|
438
438
|
getAllEntityCounts(): Map<string, number>;
|
|
439
|
+
/**
|
|
440
|
+
* Get VFS entity count for a specific type using Roaring bitmap intersection
|
|
441
|
+
* Uses hardware-accelerated SIMD operations (AVX2/SSE4.2)
|
|
442
|
+
* @param type The noun type to query
|
|
443
|
+
* @returns Count of VFS entities of this type
|
|
444
|
+
*/
|
|
445
|
+
getVFSEntityCountByType(type: string): Promise<number>;
|
|
446
|
+
/**
|
|
447
|
+
* Get all VFS entity counts by type using Roaring bitmap operations
|
|
448
|
+
* @returns Map of type -> VFS entity count
|
|
449
|
+
*/
|
|
450
|
+
getAllVFSEntityCounts(): Promise<Map<string, number>>;
|
|
451
|
+
/**
|
|
452
|
+
* Get total count of VFS entities - O(1) using Roaring bitmap cardinality
|
|
453
|
+
* @returns Total VFS entity count
|
|
454
|
+
*/
|
|
455
|
+
getTotalVFSEntityCount(): Promise<number>;
|
|
439
456
|
/**
|
|
440
457
|
* Get entity count for a noun type using type enum (O(1) array access)
|
|
441
458
|
* More efficient than Map-based getEntityCountByType
|
|
@@ -1230,6 +1230,21 @@ export class MetadataIndexManager {
|
|
|
1230
1230
|
const subIds = await this.getIdsForFilter(subFilter);
|
|
1231
1231
|
subIds.forEach(id => unionIds.add(id));
|
|
1232
1232
|
}
|
|
1233
|
+
// v6.2.1: Fix - Check for outer-level field conditions that need AND application
|
|
1234
|
+
// This handles cases like { anyOf: [...], vfsType: { exists: false } }
|
|
1235
|
+
// where the anyOf results must be intersected with other field conditions
|
|
1236
|
+
const outerFields = Object.keys(filter).filter((k) => k !== 'anyOf' && k !== 'allOf' && k !== 'not');
|
|
1237
|
+
if (outerFields.length > 0) {
|
|
1238
|
+
// Build filter with just outer fields and get matching IDs
|
|
1239
|
+
const outerFilter = {};
|
|
1240
|
+
for (const field of outerFields) {
|
|
1241
|
+
outerFilter[field] = filter[field];
|
|
1242
|
+
}
|
|
1243
|
+
const outerIds = await this.getIdsForFilter(outerFilter);
|
|
1244
|
+
const outerIdSet = new Set(outerIds);
|
|
1245
|
+
// Intersect: anyOf union AND outer field conditions
|
|
1246
|
+
return Array.from(unionIds).filter((id) => outerIdSet.has(id));
|
|
1247
|
+
}
|
|
1233
1248
|
return Array.from(unionIds);
|
|
1234
1249
|
}
|
|
1235
1250
|
// Process field filters with range support
|
|
@@ -1865,6 +1880,54 @@ export class MetadataIndexManager {
|
|
|
1865
1880
|
return new Map(this.totalEntitiesByType);
|
|
1866
1881
|
}
|
|
1867
1882
|
// ============================================================================
|
|
1883
|
+
// v6.2.1: VFS Statistics Methods (uses existing Roaring bitmap infrastructure)
|
|
1884
|
+
// ============================================================================
|
|
1885
|
+
/**
|
|
1886
|
+
* Get VFS entity count for a specific type using Roaring bitmap intersection
|
|
1887
|
+
* Uses hardware-accelerated SIMD operations (AVX2/SSE4.2)
|
|
1888
|
+
* @param type The noun type to query
|
|
1889
|
+
* @returns Count of VFS entities of this type
|
|
1890
|
+
*/
|
|
1891
|
+
async getVFSEntityCountByType(type) {
|
|
1892
|
+
const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
|
|
1893
|
+
const typeBitmap = await this.getBitmapFromChunks('noun', type);
|
|
1894
|
+
if (!vfsBitmap || !typeBitmap)
|
|
1895
|
+
return 0;
|
|
1896
|
+
// Hardware-accelerated intersection + O(1) cardinality
|
|
1897
|
+
const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap);
|
|
1898
|
+
return intersection.size;
|
|
1899
|
+
}
|
|
1900
|
+
/**
|
|
1901
|
+
* Get all VFS entity counts by type using Roaring bitmap operations
|
|
1902
|
+
* @returns Map of type -> VFS entity count
|
|
1903
|
+
*/
|
|
1904
|
+
async getAllVFSEntityCounts() {
|
|
1905
|
+
const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
|
|
1906
|
+
if (!vfsBitmap || vfsBitmap.size === 0) {
|
|
1907
|
+
return new Map();
|
|
1908
|
+
}
|
|
1909
|
+
const result = new Map();
|
|
1910
|
+
// Iterate through all known types and compute VFS count via intersection
|
|
1911
|
+
for (const type of this.totalEntitiesByType.keys()) {
|
|
1912
|
+
const typeBitmap = await this.getBitmapFromChunks('noun', type);
|
|
1913
|
+
if (typeBitmap) {
|
|
1914
|
+
const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap);
|
|
1915
|
+
if (intersection.size > 0) {
|
|
1916
|
+
result.set(type, intersection.size);
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
return result;
|
|
1921
|
+
}
|
|
1922
|
+
/**
|
|
1923
|
+
* Get total count of VFS entities - O(1) using Roaring bitmap cardinality
|
|
1924
|
+
* @returns Total VFS entity count
|
|
1925
|
+
*/
|
|
1926
|
+
async getTotalVFSEntityCount() {
|
|
1927
|
+
const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
|
|
1928
|
+
return vfsBitmap?.size ?? 0;
|
|
1929
|
+
}
|
|
1930
|
+
// ============================================================================
|
|
1868
1931
|
// Phase 1b: Type Enum Methods (O(1) access via Uint32Arrays)
|
|
1869
1932
|
// ============================================================================
|
|
1870
1933
|
/**
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Universal Concept System for VFS
|
|
3
|
+
*
|
|
4
|
+
* Manages concepts that transcend files and exist independently
|
|
5
|
+
* Ideas that can be linked to multiple manifestations across domains
|
|
6
|
+
* PRODUCTION-READY: Real implementation using Brainy
|
|
7
|
+
*/
|
|
8
|
+
import { Brainy } from '../brainy.js';
|
|
9
|
+
import { EntityManager, ManagedEntity } from './EntityManager.js';
|
|
10
|
+
/**
|
|
11
|
+
* Universal concept that exists independently of files
|
|
12
|
+
*/
|
|
13
|
+
export interface UniversalConcept extends ManagedEntity {
|
|
14
|
+
id: string;
|
|
15
|
+
name: string;
|
|
16
|
+
description?: string;
|
|
17
|
+
domain: string;
|
|
18
|
+
category: string;
|
|
19
|
+
keywords: string[];
|
|
20
|
+
links: ConceptLink[];
|
|
21
|
+
manifestations: ConceptManifestation[];
|
|
22
|
+
strength: number;
|
|
23
|
+
created: number;
|
|
24
|
+
lastUpdated: number;
|
|
25
|
+
version: number;
|
|
26
|
+
metadata: Record<string, any>;
|
|
27
|
+
conceptType?: string;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* A link between concepts
|
|
31
|
+
*/
|
|
32
|
+
export interface ConceptLink {
|
|
33
|
+
id: string;
|
|
34
|
+
targetConceptId: string;
|
|
35
|
+
relationship: 'extends' | 'implements' | 'uses' | 'opposite' | 'related' | 'contains' | 'part-of';
|
|
36
|
+
strength: number;
|
|
37
|
+
context?: string;
|
|
38
|
+
bidirectional: boolean;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* A manifestation of a concept in a specific location
|
|
42
|
+
*/
|
|
43
|
+
export interface ConceptManifestation extends ManagedEntity {
|
|
44
|
+
id: string;
|
|
45
|
+
conceptId: string;
|
|
46
|
+
filePath: string;
|
|
47
|
+
context: string;
|
|
48
|
+
form: 'definition' | 'usage' | 'example' | 'discussion' | 'implementation';
|
|
49
|
+
position?: {
|
|
50
|
+
line?: number;
|
|
51
|
+
column?: number;
|
|
52
|
+
offset?: number;
|
|
53
|
+
};
|
|
54
|
+
confidence: number;
|
|
55
|
+
timestamp: number;
|
|
56
|
+
extractedBy: 'manual' | 'auto' | 'ai';
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Configuration for concept system
|
|
60
|
+
*/
|
|
61
|
+
export interface ConceptSystemConfig {
|
|
62
|
+
autoLink?: boolean;
|
|
63
|
+
similarityThreshold?: number;
|
|
64
|
+
maxManifestations?: number;
|
|
65
|
+
strengthDecay?: number;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Concept graph structure for visualization
|
|
69
|
+
*/
|
|
70
|
+
export interface ConceptGraph {
|
|
71
|
+
concepts: Array<{
|
|
72
|
+
id: string;
|
|
73
|
+
name: string;
|
|
74
|
+
domain: string;
|
|
75
|
+
strength: number;
|
|
76
|
+
manifestationCount: number;
|
|
77
|
+
}>;
|
|
78
|
+
links: Array<{
|
|
79
|
+
source: string;
|
|
80
|
+
target: string;
|
|
81
|
+
relationship: string;
|
|
82
|
+
strength: number;
|
|
83
|
+
}>;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Universal Concept System
|
|
87
|
+
*
|
|
88
|
+
* Manages concepts that exist independently of any specific file or context
|
|
89
|
+
* Examples:
|
|
90
|
+
* - "Authentication" concept appearing in docs, code, tests
|
|
91
|
+
* - "Customer Journey" concept in marketing, UX, analytics
|
|
92
|
+
* - "Dependency Injection" pattern across multiple codebases
|
|
93
|
+
* - "Sustainability" theme in various research papers
|
|
94
|
+
*/
|
|
95
|
+
export declare class ConceptSystem extends EntityManager {
|
|
96
|
+
private config;
|
|
97
|
+
private conceptCache;
|
|
98
|
+
constructor(brain: Brainy, config?: ConceptSystemConfig);
|
|
99
|
+
/**
|
|
100
|
+
* Create a new universal concept
|
|
101
|
+
*/
|
|
102
|
+
createConcept(concept: Omit<UniversalConcept, 'id' | 'created' | 'lastUpdated' | 'version' | 'links' | 'manifestations'>): Promise<string>;
|
|
103
|
+
/**
|
|
104
|
+
* Find concepts by various criteria
|
|
105
|
+
*/
|
|
106
|
+
findConcepts(query: {
|
|
107
|
+
name?: string;
|
|
108
|
+
domain?: string;
|
|
109
|
+
category?: string;
|
|
110
|
+
keywords?: string[];
|
|
111
|
+
similar?: string;
|
|
112
|
+
manifestedIn?: string;
|
|
113
|
+
}): Promise<UniversalConcept[]>;
|
|
114
|
+
/**
|
|
115
|
+
* Link two concepts together
|
|
116
|
+
*/
|
|
117
|
+
linkConcept(fromConceptId: string, toConceptId: string, relationship: ConceptLink['relationship'], options?: {
|
|
118
|
+
strength?: number;
|
|
119
|
+
context?: string;
|
|
120
|
+
bidirectional?: boolean;
|
|
121
|
+
}): Promise<string>;
|
|
122
|
+
/**
|
|
123
|
+
* Record a manifestation of a concept in a file
|
|
124
|
+
*/
|
|
125
|
+
recordManifestation(conceptId: string, filePath: string, context: string, form: ConceptManifestation['form'], options?: {
|
|
126
|
+
position?: ConceptManifestation['position'];
|
|
127
|
+
confidence?: number;
|
|
128
|
+
extractedBy?: ConceptManifestation['extractedBy'];
|
|
129
|
+
}): Promise<string>;
|
|
130
|
+
/**
|
|
131
|
+
* Extract and link concepts from content
|
|
132
|
+
*/
|
|
133
|
+
extractAndLinkConcepts(filePath: string, content: Buffer): Promise<string[]>;
|
|
134
|
+
/**
|
|
135
|
+
* Get concept graph for visualization
|
|
136
|
+
*/
|
|
137
|
+
getConceptGraph(options?: {
|
|
138
|
+
domain?: string;
|
|
139
|
+
minStrength?: number;
|
|
140
|
+
maxConcepts?: number;
|
|
141
|
+
}): Promise<ConceptGraph>;
|
|
142
|
+
/**
|
|
143
|
+
* Find appearances of a concept
|
|
144
|
+
*/
|
|
145
|
+
findAppearances(conceptId: string, options?: {
|
|
146
|
+
filePath?: string;
|
|
147
|
+
form?: ConceptManifestation['form'];
|
|
148
|
+
minConfidence?: number;
|
|
149
|
+
limit?: number;
|
|
150
|
+
}): Promise<ConceptManifestation[]>;
|
|
151
|
+
/**
|
|
152
|
+
* Auto-link concept to similar concepts
|
|
153
|
+
*/
|
|
154
|
+
private autoLinkConcept;
|
|
155
|
+
/**
|
|
156
|
+
* Get concept by ID
|
|
157
|
+
*/
|
|
158
|
+
private getConcept;
|
|
159
|
+
/**
|
|
160
|
+
* Update stored concept
|
|
161
|
+
*/
|
|
162
|
+
private updateConcept;
|
|
163
|
+
/**
|
|
164
|
+
* Calculate similarity between two concepts
|
|
165
|
+
*/
|
|
166
|
+
private calculateConceptSimilarity;
|
|
167
|
+
/**
|
|
168
|
+
* Generate embedding for concept
|
|
169
|
+
*/
|
|
170
|
+
private generateConceptEmbedding;
|
|
171
|
+
/**
|
|
172
|
+
* Generate embedding for text
|
|
173
|
+
*/
|
|
174
|
+
private generateTextEmbedding;
|
|
175
|
+
/**
|
|
176
|
+
* Get reverse relationship type
|
|
177
|
+
*/
|
|
178
|
+
private getReverseRelationship;
|
|
179
|
+
/**
|
|
180
|
+
* Map concept relationship to VerbType
|
|
181
|
+
*/
|
|
182
|
+
private getVerbType;
|
|
183
|
+
/**
|
|
184
|
+
* Detect concept domain from context
|
|
185
|
+
*/
|
|
186
|
+
private detectDomain;
|
|
187
|
+
/**
|
|
188
|
+
* Detect concept category
|
|
189
|
+
*/
|
|
190
|
+
private detectCategory;
|
|
191
|
+
/**
|
|
192
|
+
* Detect manifestation form from context
|
|
193
|
+
*/
|
|
194
|
+
private detectManifestationForm;
|
|
195
|
+
/**
|
|
196
|
+
* Extract context around a position
|
|
197
|
+
*/
|
|
198
|
+
private extractContext;
|
|
199
|
+
/**
|
|
200
|
+
* Clear concept cache
|
|
201
|
+
*/
|
|
202
|
+
clearCache(conceptId?: string): void;
|
|
203
|
+
}
|