@soulcraft/brainy 6.1.0 → 6.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +271 -0
  2. package/dist/augmentations/KnowledgeAugmentation.d.ts +40 -0
  3. package/dist/augmentations/KnowledgeAugmentation.js +251 -0
  4. package/dist/brainy.d.ts +17 -13
  5. package/dist/brainy.js +172 -41
  6. package/dist/coreTypes.d.ts +12 -0
  7. package/dist/graph/graphAdjacencyIndex.d.ts +23 -0
  8. package/dist/graph/graphAdjacencyIndex.js +49 -0
  9. package/dist/importManager.d.ts +78 -0
  10. package/dist/importManager.js +267 -0
  11. package/dist/query/typeInference.d.ts +158 -0
  12. package/dist/query/typeInference.js +760 -0
  13. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +252 -0
  14. package/dist/storage/adapters/typeAwareStorageAdapter.js +814 -0
  15. package/dist/storage/baseStorage.d.ts +36 -0
  16. package/dist/storage/baseStorage.js +159 -4
  17. package/dist/storage/cow/binaryDataCodec.d.ts +13 -2
  18. package/dist/storage/cow/binaryDataCodec.js +15 -2
  19. package/dist/types/brainy.types.d.ts +1 -0
  20. package/dist/types/brainyDataInterface.d.ts +52 -0
  21. package/dist/types/brainyDataInterface.js +10 -0
  22. package/dist/utils/metadataIndex.d.ts +17 -0
  23. package/dist/utils/metadataIndex.js +63 -0
  24. package/dist/vfs/ConceptSystem.d.ts +203 -0
  25. package/dist/vfs/ConceptSystem.js +545 -0
  26. package/dist/vfs/EntityManager.d.ts +75 -0
  27. package/dist/vfs/EntityManager.js +216 -0
  28. package/dist/vfs/EventRecorder.d.ts +84 -0
  29. package/dist/vfs/EventRecorder.js +269 -0
  30. package/dist/vfs/GitBridge.d.ts +167 -0
  31. package/dist/vfs/GitBridge.js +537 -0
  32. package/dist/vfs/KnowledgeLayer.d.ts +35 -0
  33. package/dist/vfs/KnowledgeLayer.js +443 -0
  34. package/dist/vfs/PersistentEntitySystem.d.ts +165 -0
  35. package/dist/vfs/PersistentEntitySystem.js +503 -0
  36. package/dist/vfs/SemanticVersioning.d.ts +105 -0
  37. package/dist/vfs/SemanticVersioning.js +309 -0
  38. package/dist/vfs/VirtualFileSystem.d.ts +37 -2
  39. package/dist/vfs/VirtualFileSystem.js +105 -68
  40. package/package.json +1 -1
@@ -181,6 +181,24 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
181
181
  * @returns Combined verb + metadata or null
182
182
  */
183
183
  getVerb(id: string): Promise<HNSWVerbWithMetadata | null>;
184
+ /**
185
+ * Batch get multiple verbs (v6.2.0 - N+1 fix)
186
+ *
187
+ * **Performance**: Eliminates N+1 pattern for verb loading
188
+ * - Current: N × getVerb() = N × 50ms on GCS = 250ms for 5 verbs
189
+ * - Batched: 1 × getVerbsBatch() = 1 × 50ms on GCS = 50ms (**5x faster**)
190
+ *
191
+ * **Use cases:**
192
+ * - graphIndex.getVerbsBatchCached() for relate() duplicate checking
193
+ * - Loading relationships in batch operations
194
+ * - Pre-loading verbs for graph traversal
195
+ *
196
+ * @param ids Array of verb IDs to fetch
197
+ * @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
198
+ *
199
+ * @since v6.2.0
200
+ */
201
+ getVerbsBatch(ids: string[]): Promise<Map<string, HNSWVerbWithMetadata>>;
184
202
  /**
185
203
  * Convert HNSWVerb to GraphVerb by combining with metadata
186
204
  * DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata.
@@ -494,6 +512,24 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
494
512
  * @since v5.12.0
495
513
  */
496
514
  getNounMetadataBatch(ids: string[]): Promise<Map<string, NounMetadata>>;
515
+ /**
516
+ * Batch get multiple nouns with vectors (v6.2.0 - N+1 fix)
517
+ *
518
+ * **Performance**: Eliminates N+1 pattern for vector loading
519
+ * - Current: N × getNoun() = N × 50ms on GCS = 500ms for 10 entities
520
+ * - Batched: 1 × getNounBatch() = 1 × 50ms on GCS = 50ms (**10x faster**)
521
+ *
522
+ * **Use cases:**
523
+ * - batchGet() with includeVectors: true
524
+ * - Loading entities for similarity computation
525
+ * - Pre-loading vectors for batch processing
526
+ *
527
+ * @param ids Array of entity IDs to fetch (with vectors)
528
+ * @returns Map of id → HNSWNounWithMetadata (only successful reads included)
529
+ *
530
+ * @since v6.2.0
531
+ */
532
+ getNounBatch(ids: string[]): Promise<Map<string, HNSWNounWithMetadata>>;
497
533
  /**
498
534
  * Batch read multiple storage paths with COW inheritance support (v5.12.0)
499
535
  *
@@ -10,7 +10,7 @@ import { getShardIdFromUuid } from './sharding.js';
10
10
  import { RefManager } from './cow/RefManager.js';
11
11
  import { BlobStorage } from './cow/BlobStorage.js';
12
12
  import { CommitLog } from './cow/CommitLog.js';
13
- import { unwrapBinaryData, wrapBinaryData } from './cow/binaryDataCodec.js';
13
+ import { unwrapBinaryData } from './cow/binaryDataCodec.js';
14
14
  import { prodLog } from '../utils/logger.js';
15
15
  // Clean directory structure (v4.7.2+)
16
16
  // All storage adapters use this consistent structure
@@ -278,9 +278,25 @@ export class BaseStorage extends BaseStorageAdapter {
278
278
  }
279
279
  },
280
280
  put: async (key, data) => {
281
- // v5.10.1: Use shared binaryDataCodec utility (single source of truth)
282
- // Wraps binary data or parses JSON for storage
283
- const obj = wrapBinaryData(data);
281
+ // v6.2.0 PERMANENT FIX: Use key naming convention (explicit type contract)
282
+ // NO GUESSING - key format explicitly declares data type:
283
+ //
284
+ // JSON keys (metadata and refs):
285
+ // - 'ref:*' → JSON (RefManager: refs, HEAD, branches)
286
+ // - 'blob-meta:hash' → JSON (BlobStorage: blob metadata)
287
+ // - 'commit-meta:hash'→ JSON (BlobStorage: commit metadata)
288
+ // - 'tree-meta:hash' → JSON (BlobStorage: tree metadata)
289
+ //
290
+ // Binary keys (blob data):
291
+ // - 'blob:hash' → Binary (BlobStorage: compressed/raw blob data)
292
+ // - 'commit:hash' → Binary (BlobStorage: commit object data)
293
+ // - 'tree:hash' → Binary (BlobStorage: tree object data)
294
+ //
295
+ // This eliminates the fragile JSON.parse() guessing that caused blob integrity
296
+ // failures when compressed data accidentally parsed as valid JSON.
297
+ const obj = key.includes('-meta:') || key.startsWith('ref:')
298
+ ? JSON.parse(data.toString()) // Metadata/refs: ALWAYS JSON.stringify'd
299
+ : { _binary: true, data: data.toString('base64') }; // Blobs: ALWAYS binary (possibly compressed)
284
300
  await this.writeObjectToPath(`_cow/${key}`, obj);
285
301
  },
286
302
  delete: async (key) => {
@@ -642,6 +658,76 @@ export class BaseStorage extends BaseStorageAdapter {
642
658
  metadata: customMetadata
643
659
  };
644
660
  }
661
+ /**
662
+ * Batch get multiple verbs (v6.2.0 - N+1 fix)
663
+ *
664
+ * **Performance**: Eliminates N+1 pattern for verb loading
665
+ * - Current: N × getVerb() = N × 50ms on GCS = 250ms for 5 verbs
666
+ * - Batched: 1 × getVerbsBatch() = 1 × 50ms on GCS = 50ms (**5x faster**)
667
+ *
668
+ * **Use cases:**
669
+ * - graphIndex.getVerbsBatchCached() for relate() duplicate checking
670
+ * - Loading relationships in batch operations
671
+ * - Pre-loading verbs for graph traversal
672
+ *
673
+ * @param ids Array of verb IDs to fetch
674
+ * @returns Map of id → HNSWVerbWithMetadata (only successful reads included)
675
+ *
676
+ * @since v6.2.0
677
+ */
678
+ async getVerbsBatch(ids) {
679
+ await this.ensureInitialized();
680
+ const results = new Map();
681
+ if (ids.length === 0)
682
+ return results;
683
+ // v6.2.0: Batch-fetch vectors and metadata in parallel
684
+ // Build paths for vectors
685
+ const vectorPaths = ids.map(id => ({
686
+ path: getVerbVectorPath(id),
687
+ id
688
+ }));
689
+ // Build paths for metadata
690
+ const metadataPaths = ids.map(id => ({
691
+ path: getVerbMetadataPath(id),
692
+ id
693
+ }));
694
+ // Batch read vectors and metadata in parallel
695
+ const [vectorResults, metadataResults] = await Promise.all([
696
+ this.readBatchWithInheritance(vectorPaths.map(p => p.path)),
697
+ this.readBatchWithInheritance(metadataPaths.map(p => p.path))
698
+ ]);
699
+ // Combine vectors + metadata into HNSWVerbWithMetadata
700
+ for (const { path: vectorPath, id } of vectorPaths) {
701
+ const vectorData = vectorResults.get(vectorPath);
702
+ const metadataPath = getVerbMetadataPath(id);
703
+ const metadataData = metadataResults.get(metadataPath);
704
+ if (vectorData && metadataData) {
705
+ // Deserialize verb
706
+ const verb = this.deserializeVerb(vectorData);
707
+ // Extract standard fields to top-level (v4.8.0 pattern)
708
+ const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataData;
709
+ results.set(id, {
710
+ id: verb.id,
711
+ vector: verb.vector,
712
+ connections: verb.connections,
713
+ verb: verb.verb,
714
+ sourceId: verb.sourceId,
715
+ targetId: verb.targetId,
716
+ // v4.8.0: Standard fields at top-level
717
+ createdAt: createdAt || Date.now(),
718
+ updatedAt: updatedAt || Date.now(),
719
+ confidence: confidence,
720
+ weight: weight,
721
+ service: service,
722
+ data: data,
723
+ createdBy,
724
+ // Only custom user fields remain in metadata
725
+ metadata: customMetadata
726
+ });
727
+ }
728
+ }
729
+ return results;
730
+ }
645
731
  /**
646
732
  * Convert HNSWVerb to GraphVerb by combining with metadata
647
733
  * DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata.
@@ -1553,6 +1639,75 @@ export class BaseStorage extends BaseStorageAdapter {
1553
1639
  }
1554
1640
  return results;
1555
1641
  }
1642
+ /**
1643
+ * Batch get multiple nouns with vectors (v6.2.0 - N+1 fix)
1644
+ *
1645
+ * **Performance**: Eliminates N+1 pattern for vector loading
1646
+ * - Current: N × getNoun() = N × 50ms on GCS = 500ms for 10 entities
1647
+ * - Batched: 1 × getNounBatch() = 1 × 50ms on GCS = 50ms (**10x faster**)
1648
+ *
1649
+ * **Use cases:**
1650
+ * - batchGet() with includeVectors: true
1651
+ * - Loading entities for similarity computation
1652
+ * - Pre-loading vectors for batch processing
1653
+ *
1654
+ * @param ids Array of entity IDs to fetch (with vectors)
1655
+ * @returns Map of id → HNSWNounWithMetadata (only successful reads included)
1656
+ *
1657
+ * @since v6.2.0
1658
+ */
1659
+ async getNounBatch(ids) {
1660
+ await this.ensureInitialized();
1661
+ const results = new Map();
1662
+ if (ids.length === 0)
1663
+ return results;
1664
+ // v6.2.0: Batch-fetch vectors and metadata in parallel
1665
+ // Build paths for vectors
1666
+ const vectorPaths = ids.map(id => ({
1667
+ path: getNounVectorPath(id),
1668
+ id
1669
+ }));
1670
+ // Build paths for metadata
1671
+ const metadataPaths = ids.map(id => ({
1672
+ path: getNounMetadataPath(id),
1673
+ id
1674
+ }));
1675
+ // Batch read vectors and metadata in parallel
1676
+ const [vectorResults, metadataResults] = await Promise.all([
1677
+ this.readBatchWithInheritance(vectorPaths.map(p => p.path)),
1678
+ this.readBatchWithInheritance(metadataPaths.map(p => p.path))
1679
+ ]);
1680
+ // Combine vectors + metadata into HNSWNounWithMetadata
1681
+ for (const { path: vectorPath, id } of vectorPaths) {
1682
+ const vectorData = vectorResults.get(vectorPath);
1683
+ const metadataPath = getNounMetadataPath(id);
1684
+ const metadataData = metadataResults.get(metadataPath);
1685
+ if (vectorData && metadataData) {
1686
+ // Deserialize noun
1687
+ const noun = this.deserializeNoun(vectorData);
1688
+ // Extract standard fields to top-level (v4.8.0 pattern)
1689
+ const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataData;
1690
+ results.set(id, {
1691
+ id: noun.id,
1692
+ vector: noun.vector,
1693
+ connections: noun.connections,
1694
+ level: noun.level,
1695
+ // v4.8.0: Standard fields at top-level
1696
+ type: nounType || NounType.Thing,
1697
+ createdAt: createdAt || Date.now(),
1698
+ updatedAt: updatedAt || Date.now(),
1699
+ confidence: confidence,
1700
+ weight: weight,
1701
+ service: service,
1702
+ data: data,
1703
+ createdBy,
1704
+ // Only custom user fields remain in metadata
1705
+ metadata: customMetadata
1706
+ });
1707
+ }
1708
+ }
1709
+ return results;
1710
+ }
1556
1711
  /**
1557
1712
  * Batch read multiple storage paths with COW inheritance support (v5.12.0)
1558
1713
  *
@@ -49,11 +49,22 @@ export declare function unwrapBinaryData(data: any): Buffer;
49
49
  /**
50
50
  * Wrap binary data for JSON storage
51
51
  *
52
- * This is the SINGLE SOURCE OF TRUTH for wrapping binary data.
53
- * All storage operations MUST use this function.
52
+ * ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
53
+ * ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
54
+ * ⚠️ This function exists for legacy/compatibility only.
55
+ *
56
+ * DEPRECATED APPROACH: Tries to guess if data is JSON by parsing.
57
+ * This is FRAGILE because compressed binary can accidentally parse as valid JSON,
58
+ * causing blob integrity failures.
59
+ *
60
+ * v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
61
+ * - Keys with '-meta:' or 'ref:' prefix → Always JSON
62
+ * - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
63
+ * No guessing needed!
54
64
  *
55
65
  * @param data - Buffer to wrap
56
66
  * @returns Wrapped object or parsed JSON object
67
+ * @deprecated Use key-based dispatch in baseStorage.ts instead
57
68
  */
58
69
  export declare function wrapBinaryData(data: Buffer): any;
59
70
  /**
@@ -66,14 +66,27 @@ export function unwrapBinaryData(data) {
66
66
  /**
67
67
  * Wrap binary data for JSON storage
68
68
  *
69
- * This is the SINGLE SOURCE OF TRUTH for wrapping binary data.
70
- * All storage operations MUST use this function.
69
+ * ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
70
+ * ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
71
+ * ⚠️ This function exists for legacy/compatibility only.
72
+ *
73
+ * DEPRECATED APPROACH: Tries to guess if data is JSON by parsing.
74
+ * This is FRAGILE because compressed binary can accidentally parse as valid JSON,
75
+ * causing blob integrity failures.
76
+ *
77
+ * v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
78
+ * - Keys with '-meta:' or 'ref:' prefix → Always JSON
79
+ * - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
80
+ * No guessing needed!
71
81
  *
72
82
  * @param data - Buffer to wrap
73
83
  * @returns Wrapped object or parsed JSON object
84
+ * @deprecated Use key-based dispatch in baseStorage.ts instead
74
85
  */
75
86
  export function wrapBinaryData(data) {
76
87
  // Try to parse as JSON first (for metadata, trees, commits)
88
+ // NOTE: This is the OLD approach - fragile because compressed data
89
+ // can accidentally parse as valid JSON!
77
90
  try {
78
91
  return JSON.parse(data.toString());
79
92
  }
@@ -302,6 +302,7 @@ export interface DeleteManyParams {
302
302
  where?: any;
303
303
  limit?: number;
304
304
  onProgress?: (done: number, total: number) => void;
305
+ continueOnError?: boolean;
305
306
  }
306
307
  /**
307
308
  * Batch relate parameters
@@ -0,0 +1,52 @@
1
+ /**
2
+ * BrainyInterface - Modern API Only
3
+ *
4
+ * This interface defines the MODERN methods from Brainy 3.0.
5
+ * Used to break circular dependencies while enforcing modern API usage.
6
+ *
7
+ * NO DEPRECATED METHODS - Only clean, modern API patterns.
8
+ */
9
+ import { Vector } from '../coreTypes.js';
10
+ import { AddParams, RelateParams, Result, Entity, FindParams, SimilarParams } from './brainy.types.js';
11
+ export interface BrainyInterface<T = unknown> {
12
+ /**
13
+ * Initialize the database
14
+ */
15
+ init(): Promise<void>;
16
+ /**
17
+ * Modern add method - unified entity creation
18
+ * @param params Parameters for adding entities
19
+ * @returns The ID of the created entity
20
+ */
21
+ add(params: AddParams<T>): Promise<string>;
22
+ /**
23
+ * Modern relate method - unified relationship creation
24
+ * @param params Parameters for creating relationships
25
+ * @returns The ID of the created relationship
26
+ */
27
+ relate(params: RelateParams<T>): Promise<string>;
28
+ /**
29
+ * Modern find method - unified search and discovery
30
+ * @param query Search query or parameters object
31
+ * @returns Array of search results
32
+ */
33
+ find(query: string | FindParams<T>): Promise<Result<T>[]>;
34
+ /**
35
+ * Modern get method - retrieve entities by ID
36
+ * @param id The entity ID to retrieve
37
+ * @returns Entity or null if not found
38
+ */
39
+ get(id: string): Promise<Entity<T> | null>;
40
+ /**
41
+ * Modern similar method - find similar entities
42
+ * @param params Parameters for similarity search
43
+ * @returns Array of similar entities with scores
44
+ */
45
+ similar(params: SimilarParams<T>): Promise<Result<T>[]>;
46
+ /**
47
+ * Generate embedding vector from text
48
+ * @param text The text to embed
49
+ * @returns Vector representation of the text
50
+ */
51
+ embed(text: string): Promise<Vector>;
52
+ }
@@ -0,0 +1,10 @@
1
+ /**
2
+ * BrainyInterface - Modern API Only
3
+ *
4
+ * This interface defines the MODERN methods from Brainy 3.0.
5
+ * Used to break circular dependencies while enforcing modern API usage.
6
+ *
7
+ * NO DEPRECATED METHODS - Only clean, modern API patterns.
8
+ */
9
+ export {};
10
+ //# sourceMappingURL=brainyDataInterface.js.map
@@ -436,6 +436,23 @@ export declare class MetadataIndexManager {
436
436
  * Get all entity types and their counts - O(1) operation
437
437
  */
438
438
  getAllEntityCounts(): Map<string, number>;
439
+ /**
440
+ * Get VFS entity count for a specific type using Roaring bitmap intersection
441
+ * Uses hardware-accelerated SIMD operations (AVX2/SSE4.2)
442
+ * @param type The noun type to query
443
+ * @returns Count of VFS entities of this type
444
+ */
445
+ getVFSEntityCountByType(type: string): Promise<number>;
446
+ /**
447
+ * Get all VFS entity counts by type using Roaring bitmap operations
448
+ * @returns Map of type -> VFS entity count
449
+ */
450
+ getAllVFSEntityCounts(): Promise<Map<string, number>>;
451
+ /**
452
+ * Get total count of VFS entities - O(1) using Roaring bitmap cardinality
453
+ * @returns Total VFS entity count
454
+ */
455
+ getTotalVFSEntityCount(): Promise<number>;
439
456
  /**
440
457
  * Get entity count for a noun type using type enum (O(1) array access)
441
458
  * More efficient than Map-based getEntityCountByType
@@ -1230,6 +1230,21 @@ export class MetadataIndexManager {
1230
1230
  const subIds = await this.getIdsForFilter(subFilter);
1231
1231
  subIds.forEach(id => unionIds.add(id));
1232
1232
  }
1233
+ // v6.2.1: Fix - Check for outer-level field conditions that need AND application
1234
+ // This handles cases like { anyOf: [...], vfsType: { exists: false } }
1235
+ // where the anyOf results must be intersected with other field conditions
1236
+ const outerFields = Object.keys(filter).filter((k) => k !== 'anyOf' && k !== 'allOf' && k !== 'not');
1237
+ if (outerFields.length > 0) {
1238
+ // Build filter with just outer fields and get matching IDs
1239
+ const outerFilter = {};
1240
+ for (const field of outerFields) {
1241
+ outerFilter[field] = filter[field];
1242
+ }
1243
+ const outerIds = await this.getIdsForFilter(outerFilter);
1244
+ const outerIdSet = new Set(outerIds);
1245
+ // Intersect: anyOf union AND outer field conditions
1246
+ return Array.from(unionIds).filter((id) => outerIdSet.has(id));
1247
+ }
1233
1248
  return Array.from(unionIds);
1234
1249
  }
1235
1250
  // Process field filters with range support
@@ -1865,6 +1880,54 @@ export class MetadataIndexManager {
1865
1880
  return new Map(this.totalEntitiesByType);
1866
1881
  }
1867
1882
  // ============================================================================
1883
+ // v6.2.1: VFS Statistics Methods (uses existing Roaring bitmap infrastructure)
1884
+ // ============================================================================
1885
+ /**
1886
+ * Get VFS entity count for a specific type using Roaring bitmap intersection
1887
+ * Uses hardware-accelerated SIMD operations (AVX2/SSE4.2)
1888
+ * @param type The noun type to query
1889
+ * @returns Count of VFS entities of this type
1890
+ */
1891
+ async getVFSEntityCountByType(type) {
1892
+ const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
1893
+ const typeBitmap = await this.getBitmapFromChunks('noun', type);
1894
+ if (!vfsBitmap || !typeBitmap)
1895
+ return 0;
1896
+ // Hardware-accelerated intersection + O(1) cardinality
1897
+ const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap);
1898
+ return intersection.size;
1899
+ }
1900
+ /**
1901
+ * Get all VFS entity counts by type using Roaring bitmap operations
1902
+ * @returns Map of type -> VFS entity count
1903
+ */
1904
+ async getAllVFSEntityCounts() {
1905
+ const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
1906
+ if (!vfsBitmap || vfsBitmap.size === 0) {
1907
+ return new Map();
1908
+ }
1909
+ const result = new Map();
1910
+ // Iterate through all known types and compute VFS count via intersection
1911
+ for (const type of this.totalEntitiesByType.keys()) {
1912
+ const typeBitmap = await this.getBitmapFromChunks('noun', type);
1913
+ if (typeBitmap) {
1914
+ const intersection = RoaringBitmap32.and(vfsBitmap, typeBitmap);
1915
+ if (intersection.size > 0) {
1916
+ result.set(type, intersection.size);
1917
+ }
1918
+ }
1919
+ }
1920
+ return result;
1921
+ }
1922
+ /**
1923
+ * Get total count of VFS entities - O(1) using Roaring bitmap cardinality
1924
+ * @returns Total VFS entity count
1925
+ */
1926
+ async getTotalVFSEntityCount() {
1927
+ const vfsBitmap = await this.getBitmapFromChunks('isVFSEntity', true);
1928
+ return vfsBitmap?.size ?? 0;
1929
+ }
1930
+ // ============================================================================
1868
1931
  // Phase 1b: Type Enum Methods (O(1) access via Uint32Arrays)
1869
1932
  // ============================================================================
1870
1933
  /**
@@ -0,0 +1,203 @@
1
+ /**
2
+ * Universal Concept System for VFS
3
+ *
4
+ * Manages concepts that transcend files and exist independently
5
+ * Ideas that can be linked to multiple manifestations across domains
6
+ * PRODUCTION-READY: Real implementation using Brainy
7
+ */
8
+ import { Brainy } from '../brainy.js';
9
+ import { EntityManager, ManagedEntity } from './EntityManager.js';
10
+ /**
11
+ * Universal concept that exists independently of files
12
+ */
13
+ export interface UniversalConcept extends ManagedEntity {
14
+ id: string;
15
+ name: string;
16
+ description?: string;
17
+ domain: string;
18
+ category: string;
19
+ keywords: string[];
20
+ links: ConceptLink[];
21
+ manifestations: ConceptManifestation[];
22
+ strength: number;
23
+ created: number;
24
+ lastUpdated: number;
25
+ version: number;
26
+ metadata: Record<string, any>;
27
+ conceptType?: string;
28
+ }
29
+ /**
30
+ * A link between concepts
31
+ */
32
+ export interface ConceptLink {
33
+ id: string;
34
+ targetConceptId: string;
35
+ relationship: 'extends' | 'implements' | 'uses' | 'opposite' | 'related' | 'contains' | 'part-of';
36
+ strength: number;
37
+ context?: string;
38
+ bidirectional: boolean;
39
+ }
40
+ /**
41
+ * A manifestation of a concept in a specific location
42
+ */
43
+ export interface ConceptManifestation extends ManagedEntity {
44
+ id: string;
45
+ conceptId: string;
46
+ filePath: string;
47
+ context: string;
48
+ form: 'definition' | 'usage' | 'example' | 'discussion' | 'implementation';
49
+ position?: {
50
+ line?: number;
51
+ column?: number;
52
+ offset?: number;
53
+ };
54
+ confidence: number;
55
+ timestamp: number;
56
+ extractedBy: 'manual' | 'auto' | 'ai';
57
+ }
58
+ /**
59
+ * Configuration for concept system
60
+ */
61
+ export interface ConceptSystemConfig {
62
+ autoLink?: boolean;
63
+ similarityThreshold?: number;
64
+ maxManifestations?: number;
65
+ strengthDecay?: number;
66
+ }
67
+ /**
68
+ * Concept graph structure for visualization
69
+ */
70
+ export interface ConceptGraph {
71
+ concepts: Array<{
72
+ id: string;
73
+ name: string;
74
+ domain: string;
75
+ strength: number;
76
+ manifestationCount: number;
77
+ }>;
78
+ links: Array<{
79
+ source: string;
80
+ target: string;
81
+ relationship: string;
82
+ strength: number;
83
+ }>;
84
+ }
85
+ /**
86
+ * Universal Concept System
87
+ *
88
+ * Manages concepts that exist independently of any specific file or context
89
+ * Examples:
90
+ * - "Authentication" concept appearing in docs, code, tests
91
+ * - "Customer Journey" concept in marketing, UX, analytics
92
+ * - "Dependency Injection" pattern across multiple codebases
93
+ * - "Sustainability" theme in various research papers
94
+ */
95
+ export declare class ConceptSystem extends EntityManager {
96
+ private config;
97
+ private conceptCache;
98
+ constructor(brain: Brainy, config?: ConceptSystemConfig);
99
+ /**
100
+ * Create a new universal concept
101
+ */
102
+ createConcept(concept: Omit<UniversalConcept, 'id' | 'created' | 'lastUpdated' | 'version' | 'links' | 'manifestations'>): Promise<string>;
103
+ /**
104
+ * Find concepts by various criteria
105
+ */
106
+ findConcepts(query: {
107
+ name?: string;
108
+ domain?: string;
109
+ category?: string;
110
+ keywords?: string[];
111
+ similar?: string;
112
+ manifestedIn?: string;
113
+ }): Promise<UniversalConcept[]>;
114
+ /**
115
+ * Link two concepts together
116
+ */
117
+ linkConcept(fromConceptId: string, toConceptId: string, relationship: ConceptLink['relationship'], options?: {
118
+ strength?: number;
119
+ context?: string;
120
+ bidirectional?: boolean;
121
+ }): Promise<string>;
122
+ /**
123
+ * Record a manifestation of a concept in a file
124
+ */
125
+ recordManifestation(conceptId: string, filePath: string, context: string, form: ConceptManifestation['form'], options?: {
126
+ position?: ConceptManifestation['position'];
127
+ confidence?: number;
128
+ extractedBy?: ConceptManifestation['extractedBy'];
129
+ }): Promise<string>;
130
+ /**
131
+ * Extract and link concepts from content
132
+ */
133
+ extractAndLinkConcepts(filePath: string, content: Buffer): Promise<string[]>;
134
+ /**
135
+ * Get concept graph for visualization
136
+ */
137
+ getConceptGraph(options?: {
138
+ domain?: string;
139
+ minStrength?: number;
140
+ maxConcepts?: number;
141
+ }): Promise<ConceptGraph>;
142
+ /**
143
+ * Find appearances of a concept
144
+ */
145
+ findAppearances(conceptId: string, options?: {
146
+ filePath?: string;
147
+ form?: ConceptManifestation['form'];
148
+ minConfidence?: number;
149
+ limit?: number;
150
+ }): Promise<ConceptManifestation[]>;
151
+ /**
152
+ * Auto-link concept to similar concepts
153
+ */
154
+ private autoLinkConcept;
155
+ /**
156
+ * Get concept by ID
157
+ */
158
+ private getConcept;
159
+ /**
160
+ * Update stored concept
161
+ */
162
+ private updateConcept;
163
+ /**
164
+ * Calculate similarity between two concepts
165
+ */
166
+ private calculateConceptSimilarity;
167
+ /**
168
+ * Generate embedding for concept
169
+ */
170
+ private generateConceptEmbedding;
171
+ /**
172
+ * Generate embedding for text
173
+ */
174
+ private generateTextEmbedding;
175
+ /**
176
+ * Get reverse relationship type
177
+ */
178
+ private getReverseRelationship;
179
+ /**
180
+ * Map concept relationship to VerbType
181
+ */
182
+ private getVerbType;
183
+ /**
184
+ * Detect concept domain from context
185
+ */
186
+ private detectDomain;
187
+ /**
188
+ * Detect concept category
189
+ */
190
+ private detectCategory;
191
+ /**
192
+ * Detect manifestation form from context
193
+ */
194
+ private detectManifestationForm;
195
+ /**
196
+ * Extract context around a position
197
+ */
198
+ private extractContext;
199
+ /**
200
+ * Clear concept cache
201
+ */
202
+ clearCache(conceptId?: string): void;
203
+ }