@soulcraft/brainy 5.12.0 → 6.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,32 +39,36 @@ export function getDirectoryPath(entityType, dataType) {
39
39
  * Built-in type-aware organization for all storage adapters
40
40
  */
41
41
  /**
42
- * Get type-first path for noun vectors
42
+ * Get ID-first path for noun vectors (v6.0.0)
43
+ * No type parameter needed - direct O(1) lookup by ID
43
44
  */
44
- function getNounVectorPath(type, id) {
45
+ function getNounVectorPath(id) {
45
46
  const shard = getShardIdFromUuid(id);
46
- return `entities/nouns/${type}/vectors/${shard}/${id}.json`;
47
+ return `entities/nouns/${shard}/${id}/vectors.json`;
47
48
  }
48
49
  /**
49
- * Get type-first path for noun metadata
50
+ * Get ID-first path for noun metadata (v6.0.0)
51
+ * No type parameter needed - direct O(1) lookup by ID
50
52
  */
51
- function getNounMetadataPath(type, id) {
53
+ function getNounMetadataPath(id) {
52
54
  const shard = getShardIdFromUuid(id);
53
- return `entities/nouns/${type}/metadata/${shard}/${id}.json`;
55
+ return `entities/nouns/${shard}/${id}/metadata.json`;
54
56
  }
55
57
  /**
56
- * Get type-first path for verb vectors
58
+ * Get ID-first path for verb vectors (v6.0.0)
59
+ * No type parameter needed - direct O(1) lookup by ID
57
60
  */
58
- function getVerbVectorPath(type, id) {
61
+ function getVerbVectorPath(id) {
59
62
  const shard = getShardIdFromUuid(id);
60
- return `entities/verbs/${type}/vectors/${shard}/${id}.json`;
63
+ return `entities/verbs/${shard}/${id}/vectors.json`;
61
64
  }
62
65
  /**
63
- * Get type-first path for verb metadata
66
+ * Get ID-first path for verb metadata (v6.0.0)
67
+ * No type parameter needed - direct O(1) lookup by ID
64
68
  */
65
- function getVerbMetadataPath(type, id) {
69
+ function getVerbMetadataPath(id) {
66
70
  const shard = getShardIdFromUuid(id);
67
- return `entities/verbs/${type}/metadata/${shard}/${id}.json`;
71
+ return `entities/verbs/${shard}/${id}/metadata.json`;
68
72
  }
69
73
  /**
70
74
  * Base storage adapter that implements common functionality
@@ -89,9 +93,9 @@ export class BaseStorage extends BaseStorageAdapter {
89
93
  this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
90
94
  this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 508 bytes (Stage 3: 127 types)
91
95
  // Total: 676 bytes (99.2% reduction vs Map-based tracking)
92
- // Type cache for O(1) lookups after first access
93
- this.nounTypeCache = new Map();
94
- this.verbTypeCache = new Map();
96
+ // v6.0.0: Type caches REMOVED - ID-first paths eliminate need for type lookups!
97
+ // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json
98
+ // Type is just a field in the metadata, indexed by MetadataIndexManager for queries
95
99
  // v5.5.0: Track if type counts have been rebuilt (prevent repeated rebuilds)
96
100
  this.typeCountsRebuilt = false;
97
101
  }
@@ -174,9 +178,44 @@ export class BaseStorage extends BaseStorageAdapter {
174
178
  * IMPORTANT: If your adapter overrides init(), call await super.init() first!
175
179
  */
176
180
  async init() {
177
- // Load type statistics from storage (if they exist)
178
- await this.loadTypeStatistics();
181
+ // v6.0.1: CRITICAL FIX - Set flag FIRST to prevent infinite recursion
182
+ // If any code path during initialization calls ensureInitialized(), it would
183
+ // trigger init() again. Setting the flag immediately breaks the recursion cycle.
179
184
  this.isInitialized = true;
185
+ try {
186
+ // Load type statistics from storage (if they exist)
187
+ await this.loadTypeStatistics();
188
+ // v6.0.0: Create GraphAdjacencyIndex (lazy-loaded, no rebuild)
189
+ // LSM-trees are initialized on first use via ensureInitialized()
190
+ // Index is populated incrementally as verbs are added via addVerb()
191
+ try {
192
+ prodLog.debug('[BaseStorage] Creating GraphAdjacencyIndex...');
193
+ this.graphIndex = new GraphAdjacencyIndex(this);
194
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex instantiated (lazy-loaded), graphIndex=${!!this.graphIndex}`);
195
+ }
196
+ catch (error) {
197
+ prodLog.error('[BaseStorage] Failed to create GraphAdjacencyIndex:', error);
198
+ throw error;
199
+ }
200
+ }
201
+ catch (error) {
202
+ // Reset flag on failure to allow retry
203
+ this.isInitialized = false;
204
+ throw error;
205
+ }
206
+ }
207
+ /**
208
+ * Rebuild GraphAdjacencyIndex from existing verbs (v6.0.0)
209
+ * Call this manually if you have existing verb data that needs to be indexed
210
+ * @public
211
+ */
212
+ async rebuildGraphIndex() {
213
+ if (!this.graphIndex) {
214
+ throw new Error('GraphAdjacencyIndex not initialized');
215
+ }
216
+ prodLog.info('[BaseStorage] Rebuilding graph index from existing data...');
217
+ await this.graphIndex.rebuild();
218
+ prodLog.info('[BaseStorage] Graph index rebuild complete');
180
219
  }
181
220
  /**
182
221
  * Ensure the storage adapter is initialized
@@ -851,60 +890,44 @@ export class BaseStorage extends BaseStorageAdapter {
851
890
  */
852
891
  async getNounsWithPagination(options) {
853
892
  await this.ensureInitialized();
854
- const { limit, offset = 0, filter } = options; // cursor intentionally not extracted (not yet implemented)
893
+ const { limit, offset = 0, filter } = options;
855
894
  const collectedNouns = [];
856
- const targetCount = offset + limit; // Early termination target
857
- // v5.5.0 BUG FIX: Only use optimization if counts are reliable
858
- const totalNounCountFromArray = this.nounCountsByType.reduce((sum, c) => sum + c, 0);
859
- const useOptimization = totalNounCountFromArray > 0;
860
- // v5.5.0: Iterate through noun types with billion-scale optimizations
861
- for (let i = 0; i < NOUN_TYPE_COUNT && collectedNouns.length < targetCount; i++) {
862
- // OPTIMIZATION 1: Skip empty types (only if counts are reliable)
863
- if (useOptimization && this.nounCountsByType[i] === 0) {
864
- continue;
865
- }
866
- const type = TypeUtils.getNounFromIndex(i);
867
- // If filtering by type, skip other types
868
- if (filter?.nounType) {
869
- const filterTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
870
- if (!filterTypes.includes(type)) {
871
- continue;
872
- }
873
- }
874
- const typeDir = `entities/nouns/${type}/vectors`;
895
+ const targetCount = offset + limit;
896
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types
897
+ for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) {
898
+ const shardHex = shard.toString(16).padStart(2, '0');
899
+ const shardDir = `entities/nouns/${shardHex}`;
875
900
  try {
876
- // List all noun files for this type
877
- const nounFiles = await this.listObjectsInBranch(typeDir);
901
+ const nounFiles = await this.listObjectsInBranch(shardDir);
878
902
  for (const nounPath of nounFiles) {
879
- // OPTIMIZATION 2: Early termination (stop when we have enough)
880
- if (collectedNouns.length >= targetCount) {
903
+ if (collectedNouns.length >= targetCount)
881
904
  break;
882
- }
883
- // Skip if not a .json file
884
- if (!nounPath.endsWith('.json'))
905
+ if (!nounPath.includes('/vectors.json'))
885
906
  continue;
886
907
  try {
887
- const rawNoun = await this.readWithInheritance(nounPath);
888
- if (rawNoun) {
889
- // v5.7.10: Deserialize connections Map from JSON storage format
890
- // Replaces v5.7.8 manual deserialization (removed 13 lines at 1156-1168)
891
- const noun = this.deserializeNoun(rawNoun);
892
- // Load metadata
893
- const metadataPath = getNounMetadataPath(type, noun.id);
894
- const metadata = await this.readWithInheritance(metadataPath);
908
+ const noun = await this.readWithInheritance(nounPath);
909
+ if (noun) {
910
+ const deserialized = this.deserializeNoun(noun);
911
+ const metadata = await this.getNounMetadata(deserialized.id);
895
912
  if (metadata) {
896
- // Apply service filter if specified
913
+ // Apply type filter
914
+ if (filter?.nounType && metadata.noun) {
915
+ const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
916
+ if (!types.includes(metadata.noun)) {
917
+ continue;
918
+ }
919
+ }
920
+ // Apply service filter
897
921
  if (filter?.service) {
898
922
  const services = Array.isArray(filter.service) ? filter.service : [filter.service];
899
923
  if (metadata.service && !services.includes(metadata.service)) {
900
924
  continue;
901
925
  }
902
926
  }
903
- // Combine noun + metadata (v5.4.0: Extract standard fields to top-level)
927
+ // Combine noun + metadata
904
928
  collectedNouns.push({
905
- ...noun,
906
- // v5.7.10: connections already deserialized by deserializeNoun()
907
- type: metadata.noun || type, // Required: Extract type from metadata
929
+ ...deserialized,
930
+ type: (metadata.noun || 'thing'),
908
931
  confidence: metadata.confidence,
909
932
  weight: metadata.weight,
910
933
  createdAt: metadata.createdAt
@@ -927,15 +950,15 @@ export class BaseStorage extends BaseStorageAdapter {
927
950
  }
928
951
  }
929
952
  catch (error) {
930
- // Skip types that have no data
953
+ // Skip shards that have no data
931
954
  }
932
955
  }
933
- // Apply pagination (v5.5.0: Efficient slicing after early termination)
956
+ // Apply pagination
934
957
  const paginatedNouns = collectedNouns.slice(offset, offset + limit);
935
- const hasMore = collectedNouns.length > targetCount; // v5.7.11: Fixed >= to > (was causing infinite loop)
958
+ const hasMore = collectedNouns.length > targetCount;
936
959
  return {
937
960
  items: paginatedNouns,
938
- totalCount: collectedNouns.length, // Accurate count of collected results
961
+ totalCount: collectedNouns.length,
939
962
  hasMore,
940
963
  nextCursor: hasMore && paginatedNouns.length > 0
941
964
  ? paginatedNouns[paginatedNouns.length - 1].id
@@ -962,58 +985,70 @@ export class BaseStorage extends BaseStorageAdapter {
962
985
  const { limit, offset = 0, filter } = options; // cursor intentionally not extracted (not yet implemented)
963
986
  const collectedVerbs = [];
964
987
  const targetCount = offset + limit; // Early termination target
965
- // v5.5.0 BUG FIX: Only use optimization if counts are reliable
966
- const totalVerbCountFromArray = this.verbCountsByType.reduce((sum, c) => sum + c, 0);
967
- const useOptimization = totalVerbCountFromArray > 0;
968
- // v5.5.0: Iterate through verb types with billion-scale optimizations
969
- for (let i = 0; i < VERB_TYPE_COUNT && collectedVerbs.length < targetCount; i++) {
970
- // OPTIMIZATION 1: Skip empty types (only if counts are reliable)
971
- if (useOptimization && this.verbCountsByType[i] === 0) {
972
- continue;
973
- }
974
- const type = TypeUtils.getVerbFromIndex(i);
975
- // If filtering by verbType, skip other types
976
- if (filter?.verbType) {
977
- const filterTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
978
- if (!filterTypes.includes(type)) {
979
- continue;
980
- }
981
- }
988
+ // Prepare filter sets for efficient lookup
989
+ const filterVerbTypes = filter?.verbType
990
+ ? new Set(Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType])
991
+ : null;
992
+ const filterSourceIds = filter?.sourceId
993
+ ? new Set(Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId])
994
+ : null;
995
+ const filterTargetIds = filter?.targetId
996
+ ? new Set(Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId])
997
+ : null;
998
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types - single pass!
999
+ for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
1000
+ const shardHex = shard.toString(16).padStart(2, '0');
1001
+ const shardDir = `entities/verbs/${shardHex}`;
982
1002
  try {
983
- const verbsOfType = await this.getVerbsByType_internal(type);
984
- // Apply filtering inline (memory efficient)
985
- for (const verb of verbsOfType) {
986
- // OPTIMIZATION 2: Early termination (stop when we have enough)
987
- if (collectedVerbs.length >= targetCount) {
1003
+ const verbFiles = await this.listObjectsInBranch(shardDir);
1004
+ for (const verbPath of verbFiles) {
1005
+ if (collectedVerbs.length >= targetCount)
988
1006
  break;
989
- }
990
- // Apply filters if specified
991
- if (filter) {
992
- // Filter by sourceId
993
- if (filter.sourceId) {
994
- const sourceIds = Array.isArray(filter.sourceId)
995
- ? filter.sourceId
996
- : [filter.sourceId];
997
- if (!sourceIds.includes(verb.sourceId)) {
998
- continue;
999
- }
1007
+ if (!verbPath.includes('/vectors.json'))
1008
+ continue;
1009
+ try {
1010
+ const rawVerb = await this.readWithInheritance(verbPath);
1011
+ if (!rawVerb)
1012
+ continue;
1013
+ // v6.0.0: Deserialize connections Map from JSON storage format
1014
+ const verb = this.deserializeVerb(rawVerb);
1015
+ // Apply type filter
1016
+ if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
1017
+ continue;
1000
1018
  }
1001
- // Filter by targetId
1002
- if (filter.targetId) {
1003
- const targetIds = Array.isArray(filter.targetId)
1004
- ? filter.targetId
1005
- : [filter.targetId];
1006
- if (!targetIds.includes(verb.targetId)) {
1007
- continue;
1008
- }
1019
+ // Apply sourceId filter
1020
+ if (filterSourceIds && !filterSourceIds.has(verb.sourceId)) {
1021
+ continue;
1022
+ }
1023
+ // Apply targetId filter
1024
+ if (filterTargetIds && !filterTargetIds.has(verb.targetId)) {
1025
+ continue;
1009
1026
  }
1027
+ // Load metadata
1028
+ const metadata = await this.getVerbMetadata(verb.id);
1029
+ // Combine verb + metadata
1030
+ collectedVerbs.push({
1031
+ ...verb,
1032
+ weight: metadata?.weight,
1033
+ confidence: metadata?.confidence,
1034
+ createdAt: metadata?.createdAt
1035
+ ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
1036
+ : Date.now(),
1037
+ updatedAt: metadata?.updatedAt
1038
+ ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
1039
+ : Date.now(),
1040
+ service: metadata?.service,
1041
+ createdBy: metadata?.createdBy,
1042
+ metadata: metadata || {}
1043
+ });
1044
+ }
1045
+ catch (error) {
1046
+ // Skip verbs that fail to load
1010
1047
  }
1011
- // Verb passed all filters - add to collection
1012
- collectedVerbs.push(verb);
1013
1048
  }
1014
1049
  }
1015
1050
  catch (error) {
1016
- // Skip types that have no data (directory may not exist)
1051
+ // Skip shards that have no data
1017
1052
  }
1018
1053
  }
1019
1054
  // Apply pagination (v5.5.0: Efficient slicing after early termination)
@@ -1386,11 +1421,8 @@ export class BaseStorage extends BaseStorageAdapter {
1386
1421
  */
1387
1422
  async saveNounMetadata_internal(id, metadata) {
1388
1423
  await this.ensureInitialized();
1389
- // v5.4.0: Extract and cache type for type-first routing
1390
- const type = (metadata.noun || 'thing');
1391
- this.nounTypeCache.set(id, type);
1392
- // v5.4.0: Use type-first path
1393
- const path = getNounMetadataPath(type, id);
1424
+ // v6.0.0: ID-first path - no type needed!
1425
+ const path = getNounMetadataPath(id);
1394
1426
  // Determine if this is a new entity by checking if metadata already exists
1395
1427
  const existingMetadata = await this.readWithInheritance(path);
1396
1428
  const isNew = !existingMetadata;
@@ -1411,6 +1443,17 @@ export class BaseStorage extends BaseStorageAdapter {
1411
1443
  /**
1412
1444
  * Get noun metadata from storage (METADATA-ONLY, NO VECTORS)
1413
1445
  *
1446
+ * **Performance (v6.0.0)**: Direct O(1) ID-first lookup - NO type search needed!
1447
+ * - **All lookups**: 1 read, ~500ms on cloud (consistent performance)
1448
+ * - **No cache needed**: Type is in the metadata, not the path
1449
+ * - **No type search**: ID-first paths eliminate 42-type search entirely
1450
+ *
1451
+ * **Clean architecture (v6.0.0)**:
1452
+ * - Path: `entities/nouns/{SHARD}/{ID}/metadata.json`
1453
+ * - Type is just a field in metadata (`noun: "document"`)
1454
+ * - MetadataIndex handles type queries (no path scanning needed)
1455
+ * - Scales to billions without any overhead
1456
+ *
1414
1457
  * **Performance (v5.11.1)**: Fast path for metadata-only reads
1415
1458
  * - **Speed**: 10ms vs 43ms (76-81% faster than getNoun)
1416
1459
  * - **Bandwidth**: 300 bytes vs 6KB (95% less)
@@ -1440,39 +1483,21 @@ export class BaseStorage extends BaseStorageAdapter {
1440
1483
  * @returns Metadata or null if not found
1441
1484
  *
1442
1485
  * @performance
1443
- * - Type cache O(1) lookup for cached entities
1444
- * - Type scan O(N_types) for cache misses (typically <100ms)
1445
- * - Uses readWithInheritance() for COW branch support
1486
+ * - O(1) direct ID lookup - always 1 read (~500ms on cloud, ~10ms local)
1487
+ * - No caching complexity
1488
+ * - No type search fallbacks
1489
+ * - Works in distributed systems without sync issues
1446
1490
  *
1447
1491
  * @since v4.0.0
1448
- * @since v5.4.0 - Type-first paths
1492
+ * @since v5.4.0 - Type-first paths (removed in v6.0.0)
1449
1493
  * @since v5.11.1 - Promoted to fast path for brain.get() optimization
1494
+ * @since v6.0.0 - CLEAN FIX: ID-first paths eliminate all type-search complexity
1450
1495
  */
1451
1496
  async getNounMetadata(id) {
1452
1497
  await this.ensureInitialized();
1453
- // v5.4.0: Check type cache first (populated during save)
1454
- const cachedType = this.nounTypeCache.get(id);
1455
- if (cachedType) {
1456
- const path = getNounMetadataPath(cachedType, id);
1457
- return this.readWithInheritance(path);
1458
- }
1459
- // Fallback: search across all types (expensive but necessary if cache miss)
1460
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1461
- const type = TypeUtils.getNounFromIndex(i);
1462
- const path = getNounMetadataPath(type, id);
1463
- try {
1464
- const metadata = await this.readWithInheritance(path);
1465
- if (metadata) {
1466
- // Cache the type for next time
1467
- this.nounTypeCache.set(id, type);
1468
- return metadata;
1469
- }
1470
- }
1471
- catch (error) {
1472
- // Not in this type, continue searching
1473
- }
1474
- }
1475
- return null;
1498
+ // v6.0.0: Clean, simple, O(1) lookup - no type needed!
1499
+ const path = getNounMetadataPath(id);
1500
+ return this.readWithInheritance(path);
1476
1501
  }
1477
1502
  /**
1478
1503
  * Batch fetch noun metadata from storage (v5.12.0 - Cloud Storage Optimization)
@@ -1511,74 +1536,19 @@ export class BaseStorage extends BaseStorageAdapter {
1511
1536
  const results = new Map();
1512
1537
  if (ids.length === 0)
1513
1538
  return results;
1514
- // Group IDs by cached type for efficient path construction
1515
- const idsByType = new Map();
1516
- const uncachedIds = [];
1517
- for (const id of ids) {
1518
- const cachedType = this.nounTypeCache.get(id);
1519
- if (cachedType) {
1520
- const idsForType = idsByType.get(cachedType) || [];
1521
- idsForType.push(id);
1522
- idsByType.set(cachedType, idsForType);
1523
- }
1524
- else {
1525
- uncachedIds.push(id);
1526
- }
1527
- }
1528
- // Build paths for known types
1529
- const pathsToFetch = [];
1530
- for (const [type, typeIds] of idsByType.entries()) {
1531
- for (const id of typeIds) {
1532
- pathsToFetch.push({
1533
- path: getNounMetadataPath(type, id),
1534
- id
1535
- });
1536
- }
1537
- }
1538
- // For uncached IDs, we need to search across types (expensive but unavoidable)
1539
- // Strategy: Try most common types first (Document, Thing, Person), then others
1540
- const commonTypes = [NounType.Document, NounType.Thing, NounType.Person, NounType.File];
1541
- const commonTypeSet = new Set(commonTypes);
1542
- const otherTypes = [];
1543
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1544
- const type = TypeUtils.getNounFromIndex(i);
1545
- if (!commonTypeSet.has(type)) {
1546
- otherTypes.push(type);
1547
- }
1548
- }
1549
- const searchOrder = [...commonTypes, ...otherTypes];
1550
- for (const id of uncachedIds) {
1551
- for (const type of searchOrder) {
1552
- // Build path manually to avoid type issues
1553
- const shard = getShardIdFromUuid(id);
1554
- const path = `entities/nouns/${type}/metadata/${shard}/${id}.json`;
1555
- pathsToFetch.push({ path, id });
1556
- }
1557
- }
1558
- // Batch read all paths
1539
+ // v6.0.0: ID-first paths - no type grouping or search needed!
1540
+ // Build direct paths for all IDs
1541
+ const pathsToFetch = ids.map(id => ({
1542
+ path: getNounMetadataPath(id),
1543
+ id
1544
+ }));
1545
+ // Batch read all paths (uses adapter's native batch API or parallel fallback)
1559
1546
  const batchResults = await this.readBatchWithInheritance(pathsToFetch.map(p => p.path));
1560
- // Process results and update cache
1561
- const foundUncached = new Set();
1562
- for (let i = 0; i < pathsToFetch.length; i++) {
1563
- const { path, id } = pathsToFetch[i];
1547
+ // Map results back to IDs
1548
+ for (const { path, id } of pathsToFetch) {
1564
1549
  const metadata = batchResults.get(path);
1565
1550
  if (metadata) {
1566
1551
  results.set(id, metadata);
1567
- // Cache the type for uncached IDs (only on first find)
1568
- if (uncachedIds.includes(id) && !foundUncached.has(id)) {
1569
- // Extract type from path: "entities/nouns/metadata/{type}/{shard}/{id}.json"
1570
- const parts = path.split('/');
1571
- const typeStr = parts[3]; // "document", "thing", etc.
1572
- // Find matching type by string comparison
1573
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1574
- const type = TypeUtils.getNounFromIndex(i);
1575
- if (type === typeStr) {
1576
- this.nounTypeCache.set(id, type);
1577
- break;
1578
- }
1579
- }
1580
- foundUncached.add(id);
1581
- }
1582
1552
  }
1583
1553
  }
1584
1554
  return results;
@@ -1736,36 +1706,13 @@ export class BaseStorage extends BaseStorageAdapter {
1736
1706
  };
1737
1707
  }
1738
1708
  /**
1739
- * Delete noun metadata from storage
1740
- * v5.4.0: Uses type-first paths (must match saveNounMetadata_internal)
1709
+ * Delete noun metadata from storage (v6.0.0: ID-first, O(1) delete)
1741
1710
  */
1742
1711
  async deleteNounMetadata(id) {
1743
1712
  await this.ensureInitialized();
1744
- // v5.4.0: Use cached type for path
1745
- const cachedType = this.nounTypeCache.get(id);
1746
- if (cachedType) {
1747
- const path = getNounMetadataPath(cachedType, id);
1748
- await this.deleteObjectFromBranch(path);
1749
- // Remove from cache after deletion
1750
- this.nounTypeCache.delete(id);
1751
- return;
1752
- }
1753
- // If not in cache, search all types to find and delete
1754
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1755
- const type = TypeUtils.getNounFromIndex(i);
1756
- const path = getNounMetadataPath(type, id);
1757
- try {
1758
- // Check if exists before deleting
1759
- const exists = await this.readWithInheritance(path);
1760
- if (exists) {
1761
- await this.deleteObjectFromBranch(path);
1762
- return;
1763
- }
1764
- }
1765
- catch (error) {
1766
- // Not in this type, continue searching
1767
- }
1768
- }
1713
+ // v6.0.0: Direct O(1) delete with ID-first path
1714
+ const path = getNounMetadataPath(id);
1715
+ await this.deleteObjectFromBranch(path);
1769
1716
  }
1770
1717
  /**
1771
1718
  * Save verb metadata to storage (v4.0.0: now typed)
@@ -1777,7 +1724,7 @@ export class BaseStorage extends BaseStorageAdapter {
1777
1724
  }
1778
1725
  /**
1779
1726
  * Internal method for saving verb metadata (v4.0.0: now typed)
1780
- * v5.4.0: Uses type-first paths (must match getVerbMetadata)
1727
+ * v5.4.0: Uses ID-first paths (must match getVerbMetadata)
1781
1728
  *
1782
1729
  * CRITICAL (v4.1.2): Count synchronization happens here
1783
1730
  * This ensures verb counts are updated AFTER metadata exists, fixing the race condition
@@ -1789,7 +1736,7 @@ export class BaseStorage extends BaseStorageAdapter {
1789
1736
  */
1790
1737
  async saveVerbMetadata_internal(id, metadata) {
1791
1738
  await this.ensureInitialized();
1792
- // v5.4.0: Extract verb type from metadata for type-first path
1739
+ // v5.4.0: Extract verb type from metadata for ID-first path
1793
1740
  const verbType = metadata.verb;
1794
1741
  if (!verbType) {
1795
1742
  // Backward compatibility: fallback to old path if no verb type
@@ -1797,15 +1744,14 @@ export class BaseStorage extends BaseStorageAdapter {
1797
1744
  await this.writeObjectToBranch(keyInfo.fullPath, metadata);
1798
1745
  return;
1799
1746
  }
1800
- // v5.4.0: Use type-first path
1801
- const path = getVerbMetadataPath(verbType, id);
1747
+ // v5.4.0: Use ID-first path
1748
+ const path = getVerbMetadataPath(id);
1802
1749
  // Determine if this is a new verb by checking if metadata already exists
1803
1750
  const existingMetadata = await this.readWithInheritance(path);
1804
1751
  const isNew = !existingMetadata;
1805
1752
  // Save the metadata (COW-aware - writes to branch-specific path)
1806
1753
  await this.writeObjectToBranch(path, metadata);
1807
1754
  // v5.4.0: Cache verb type for faster lookups
1808
- this.verbTypeCache.set(id, verbType);
1809
1755
  // CRITICAL FIX (v4.1.2): Increment verb count for new relationships
1810
1756
  // This runs AFTER metadata is saved
1811
1757
  // Uses synchronous increment since storage operations are already serialized
@@ -1820,69 +1766,34 @@ export class BaseStorage extends BaseStorageAdapter {
1820
1766
  }
1821
1767
  /**
1822
1768
  * Get verb metadata from storage (v4.0.0: now typed)
1823
- * v5.4.0: Uses type-first paths (must match saveVerbMetadata_internal)
1769
+ * v5.4.0: Uses ID-first paths (must match saveVerbMetadata_internal)
1824
1770
  */
1825
1771
  async getVerbMetadata(id) {
1826
1772
  await this.ensureInitialized();
1827
- // v5.4.0: Check type cache first (populated during save)
1828
- const cachedType = this.verbTypeCache.get(id);
1829
- if (cachedType) {
1830
- const path = getVerbMetadataPath(cachedType, id);
1831
- return this.readWithInheritance(path);
1832
- }
1833
- // Fallback: search across all types (expensive but necessary if cache miss)
1834
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
1835
- const type = TypeUtils.getVerbFromIndex(i);
1836
- const path = getVerbMetadataPath(type, id);
1837
- try {
1838
- const metadata = await this.readWithInheritance(path);
1839
- if (metadata) {
1840
- // Cache the type for next time
1841
- this.verbTypeCache.set(id, type);
1842
- return metadata;
1843
- }
1844
- }
1845
- catch (error) {
1846
- // Not in this type, continue searching
1847
- }
1773
+ // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
1774
+ const path = getVerbMetadataPath(id);
1775
+ try {
1776
+ const metadata = await this.readWithInheritance(path);
1777
+ return metadata || null;
1778
+ }
1779
+ catch (error) {
1780
+ // Entity not found
1781
+ return null;
1848
1782
  }
1849
- return null;
1850
1783
  }
1851
1784
  /**
1852
- * Delete verb metadata from storage
1853
- * v5.4.0: Uses type-first paths (must match saveVerbMetadata_internal)
1785
+ * Delete verb metadata from storage (v6.0.0: ID-first, O(1) delete)
1854
1786
  */
1855
1787
  async deleteVerbMetadata(id) {
1856
1788
  await this.ensureInitialized();
1857
- // v5.4.0: Use cached type for path
1858
- const cachedType = this.verbTypeCache.get(id);
1859
- if (cachedType) {
1860
- const path = getVerbMetadataPath(cachedType, id);
1861
- await this.deleteObjectFromBranch(path);
1862
- // Remove from cache after deletion
1863
- this.verbTypeCache.delete(id);
1864
- return;
1865
- }
1866
- // If not in cache, search all types to find and delete
1867
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
1868
- const type = TypeUtils.getVerbFromIndex(i);
1869
- const path = getVerbMetadataPath(type, id);
1870
- try {
1871
- // Check if exists before deleting
1872
- const exists = await this.readWithInheritance(path);
1873
- if (exists) {
1874
- await this.deleteObjectFromBranch(path);
1875
- return;
1876
- }
1877
- }
1878
- catch (error) {
1879
- // Not in this type, continue searching
1880
- }
1881
- }
1789
+ // v6.0.0: Direct O(1) delete with ID-first path
1790
+ const path = getVerbMetadataPath(id);
1791
+ await this.deleteObjectFromBranch(path);
1882
1792
  }
1883
1793
  // ============================================================================
1884
- // TYPE-FIRST HELPER METHODS (v5.4.0)
1885
- // Built-in type-aware support for all storage adapters
1794
+ // ID-FIRST HELPER METHODS (v6.0.0)
1795
+ // Direct O(1) ID lookups - no type needed!
1796
+ // Clean, simple architecture for billion-scale performance
1886
1797
  // ============================================================================
1887
1798
  /**
1888
1799
  * Load type statistics from storage
@@ -1924,30 +1835,61 @@ export class BaseStorage extends BaseStorageAdapter {
1924
1835
  */
1925
1836
  async rebuildTypeCounts() {
1926
1837
  prodLog.info('[BaseStorage] Rebuilding type counts from storage...');
1927
- // Rebuild verb counts by checking each type directory
1928
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
1929
- const type = TypeUtils.getVerbFromIndex(i);
1930
- const prefix = `entities/verbs/${type}/vectors/`;
1838
+ // v6.0.0: Rebuild by scanning shards (0x00-0xFF) and reading metadata
1839
+ this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT);
1840
+ this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT);
1841
+ // Scan noun shards
1842
+ for (let shard = 0; shard < 256; shard++) {
1843
+ const shardHex = shard.toString(16).padStart(2, '0');
1844
+ const shardDir = `entities/nouns/${shardHex}`;
1931
1845
  try {
1932
- const paths = await this.listObjectsInBranch(prefix);
1933
- this.verbCountsByType[i] = paths.length;
1846
+ const paths = await this.listObjectsInBranch(shardDir);
1847
+ for (const path of paths) {
1848
+ if (!path.includes('/metadata.json'))
1849
+ continue;
1850
+ try {
1851
+ const metadata = await this.readWithInheritance(path);
1852
+ if (metadata && metadata.noun) {
1853
+ const typeIndex = TypeUtils.getNounIndex(metadata.noun);
1854
+ if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
1855
+ this.nounCountsByType[typeIndex]++;
1856
+ }
1857
+ }
1858
+ }
1859
+ catch (error) {
1860
+ // Skip entities that fail to load
1861
+ }
1862
+ }
1934
1863
  }
1935
1864
  catch (error) {
1936
- // Type directory doesn't exist - count is 0
1937
- this.verbCountsByType[i] = 0;
1865
+ // Skip shards that don't exist
1938
1866
  }
1939
1867
  }
1940
- // Rebuild noun counts similarly
1941
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1942
- const type = TypeUtils.getNounFromIndex(i);
1943
- const prefix = `entities/nouns/${type}/vectors/`;
1868
+ // Scan verb shards
1869
+ for (let shard = 0; shard < 256; shard++) {
1870
+ const shardHex = shard.toString(16).padStart(2, '0');
1871
+ const shardDir = `entities/verbs/${shardHex}`;
1944
1872
  try {
1945
- const paths = await this.listObjectsInBranch(prefix);
1946
- this.nounCountsByType[i] = paths.length;
1873
+ const paths = await this.listObjectsInBranch(shardDir);
1874
+ for (const path of paths) {
1875
+ if (!path.includes('/metadata.json'))
1876
+ continue;
1877
+ try {
1878
+ const metadata = await this.readWithInheritance(path);
1879
+ if (metadata && metadata.verb) {
1880
+ const typeIndex = TypeUtils.getVerbIndex(metadata.verb);
1881
+ if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
1882
+ this.verbCountsByType[typeIndex]++;
1883
+ }
1884
+ }
1885
+ }
1886
+ catch (error) {
1887
+ // Skip entities that fail to load
1888
+ }
1889
+ }
1947
1890
  }
1948
1891
  catch (error) {
1949
- // Type directory doesn't exist - count is 0
1950
- this.nounCountsByType[i] = 0;
1892
+ // Skip shards that don't exist
1951
1893
  }
1952
1894
  }
1953
1895
  // Save rebuilt counts to storage
@@ -1957,18 +1899,13 @@ export class BaseStorage extends BaseStorageAdapter {
1957
1899
  prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
1958
1900
  }
1959
1901
  /**
1960
- * Get noun type from cache or metadata
1961
- * Relies on nounTypeCache populated during metadata saves
1902
+ * Get noun type (v6.0.0: type no longer needed for paths!)
1903
+ * With ID-first paths, this is only used for internal statistics tracking.
1904
+ * The actual type is stored in metadata and indexed by MetadataIndexManager.
1962
1905
  */
1963
1906
  getNounType(noun) {
1964
- // Check cache (populated when metadata is saved)
1965
- const cached = this.nounTypeCache.get(noun.id);
1966
- if (cached) {
1967
- return cached;
1968
- }
1969
- // Default to 'thing' if unknown
1970
- // This should only happen if saveNoun_internal is called before saveNounMetadata
1971
- prodLog.warn(`[BaseStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
1907
+ // v6.0.0: Type cache removed - default to 'thing' for statistics
1908
+ // The real type is in metadata, accessible via getNounMetadata(id)
1972
1909
  return 'thing';
1973
1910
  }
1974
1911
  /**
@@ -2051,15 +1988,14 @@ export class BaseStorage extends BaseStorageAdapter {
2051
1988
  // Converted from abstract to concrete - all adapters now have built-in type-aware
2052
1989
  // ============================================================================
2053
1990
  /**
2054
- * Save a noun to storage (type-first path)
1991
+ * Save a noun to storage (ID-first path)
2055
1992
  */
2056
1993
  async saveNoun_internal(noun) {
2057
1994
  const type = this.getNounType(noun);
2058
- const path = getNounVectorPath(type, noun.id);
1995
+ const path = getNounVectorPath(noun.id);
2059
1996
  // Update type tracking
2060
1997
  const typeIndex = TypeUtils.getNounIndex(type);
2061
1998
  this.nounCountsByType[typeIndex]++;
2062
- this.nounTypeCache.set(noun.id, type);
2063
1999
  // COW-aware write (v5.0.1): Use COW helper for branch isolation
2064
2000
  await this.writeObjectToBranch(path, noun);
2065
2001
  // Periodically save statistics (every 100 saves)
@@ -2068,120 +2004,89 @@ export class BaseStorage extends BaseStorageAdapter {
2068
2004
  }
2069
2005
  }
2070
2006
  /**
2071
- * Get a noun from storage (type-first path)
2007
+ * Get a noun from storage (ID-first path)
2072
2008
  */
2073
2009
  async getNoun_internal(id) {
2074
- // Try cache first
2075
- const cachedType = this.nounTypeCache.get(id);
2076
- if (cachedType) {
2077
- const path = getNounVectorPath(cachedType, id);
2010
+ // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
2011
+ const path = getNounVectorPath(id);
2012
+ try {
2078
2013
  // COW-aware read (v5.0.1): Use COW helper for branch isolation
2079
- const data = await this.readWithInheritance(path);
2080
- // v5.7.10: Deserialize connections Map from JSON storage format
2081
- return data ? this.deserializeNoun(data) : null;
2082
- }
2083
- // Need to search across all types (expensive, but cached after first access)
2084
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
2085
- const type = TypeUtils.getNounFromIndex(i);
2086
- const path = getNounVectorPath(type, id);
2087
- try {
2088
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2089
- const noun = await this.readWithInheritance(path);
2090
- if (noun) {
2091
- // Cache the type for next time
2092
- this.nounTypeCache.set(id, type);
2093
- // v5.7.10: Deserialize connections Map from JSON storage format
2094
- return this.deserializeNoun(noun);
2095
- }
2096
- }
2097
- catch (error) {
2098
- // Not in this type, continue searching
2014
+ const noun = await this.readWithInheritance(path);
2015
+ if (noun) {
2016
+ // v5.7.10: Deserialize connections Map from JSON storage format
2017
+ return this.deserializeNoun(noun);
2099
2018
  }
2100
2019
  }
2020
+ catch (error) {
2021
+ // Entity not found
2022
+ return null;
2023
+ }
2101
2024
  return null;
2102
2025
  }
2103
2026
  /**
2104
- * Get nouns by noun type (O(1) with type-first paths!)
2027
+ * Get nouns by noun type (v6.0.0: Shard-based iteration!)
2105
2028
  */
2106
2029
  async getNounsByNounType_internal(nounType) {
2107
- const type = nounType;
2108
- const prefix = `entities/nouns/${type}/vectors/`;
2109
- // COW-aware list (v5.0.1): Use COW helper for branch isolation
2110
- const paths = await this.listObjectsInBranch(prefix);
2111
- // Load all nouns of this type
2030
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types
2031
+ // Type is stored in metadata.noun field, we filter as we load
2112
2032
  const nouns = [];
2113
- for (const path of paths) {
2033
+ for (let shard = 0; shard < 256; shard++) {
2034
+ const shardHex = shard.toString(16).padStart(2, '0');
2035
+ const shardDir = `entities/nouns/${shardHex}`;
2114
2036
  try {
2115
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2116
- const noun = await this.readWithInheritance(path);
2117
- if (noun) {
2118
- // v5.7.10: Deserialize connections Map from JSON storage format
2119
- nouns.push(this.deserializeNoun(noun));
2120
- // Cache the type
2121
- this.nounTypeCache.set(noun.id, type);
2037
+ const nounFiles = await this.listObjectsInBranch(shardDir);
2038
+ for (const nounPath of nounFiles) {
2039
+ if (!nounPath.includes('/vectors.json'))
2040
+ continue;
2041
+ try {
2042
+ const noun = await this.readWithInheritance(nounPath);
2043
+ if (noun) {
2044
+ const deserialized = this.deserializeNoun(noun);
2045
+ // Check type from metadata
2046
+ const metadata = await this.getNounMetadata(deserialized.id);
2047
+ if (metadata && metadata.noun === nounType) {
2048
+ nouns.push(deserialized);
2049
+ }
2050
+ }
2051
+ }
2052
+ catch (error) {
2053
+ // Skip nouns that fail to load
2054
+ }
2122
2055
  }
2123
2056
  }
2124
2057
  catch (error) {
2125
- prodLog.warn(`[BaseStorage] Failed to load noun from ${path}:`, error);
2058
+ // Skip shards that have no data
2126
2059
  }
2127
2060
  }
2128
2061
  return nouns;
2129
2062
  }
2130
2063
  /**
2131
- * Delete a noun from storage (type-first path)
2064
+ * Delete a noun from storage (v6.0.0: ID-first, O(1) delete)
2132
2065
  */
2133
2066
  async deleteNoun_internal(id) {
2134
- // Try cache first
2135
- const cachedType = this.nounTypeCache.get(id);
2136
- if (cachedType) {
2137
- const path = getNounVectorPath(cachedType, id);
2138
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2139
- await this.deleteObjectFromBranch(path);
2140
- // Update counts
2141
- const typeIndex = TypeUtils.getNounIndex(cachedType);
2142
- if (this.nounCountsByType[typeIndex] > 0) {
2143
- this.nounCountsByType[typeIndex]--;
2144
- }
2145
- this.nounTypeCache.delete(id);
2146
- return;
2147
- }
2148
- // Search across all types
2149
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
2150
- const type = TypeUtils.getNounFromIndex(i);
2151
- const path = getNounVectorPath(type, id);
2152
- try {
2153
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2154
- await this.deleteObjectFromBranch(path);
2155
- // Update counts
2156
- if (this.nounCountsByType[i] > 0) {
2157
- this.nounCountsByType[i]--;
2158
- }
2159
- this.nounTypeCache.delete(id);
2160
- return;
2161
- }
2162
- catch (error) {
2163
- // Not in this type, continue
2164
- }
2165
- }
2067
+ // v6.0.0: Direct O(1) delete with ID-first path
2068
+ const path = getNounVectorPath(id);
2069
+ await this.deleteObjectFromBranch(path);
2070
+ // Note: Type-specific counts will be decremented via metadata tracking
2071
+ // The real type is in metadata, accessible if needed via getNounMetadata(id)
2166
2072
  }
2167
2073
  /**
2168
- * Save a verb to storage (type-first path)
2074
+ * Save a verb to storage (ID-first path)
2169
2075
  */
2170
2076
  async saveVerb_internal(verb) {
2171
2077
  // Type is now a first-class field in HNSWVerb - no caching needed!
2172
2078
  const type = verb.verb;
2173
- const path = getVerbVectorPath(type, verb.id);
2079
+ const path = getVerbVectorPath(verb.id);
2080
+ prodLog.debug(`[BaseStorage] saveVerb_internal: id=${verb.id}, sourceId=${verb.sourceId}, targetId=${verb.targetId}, type=${type}`);
2174
2081
  // Update type tracking
2175
2082
  const typeIndex = TypeUtils.getVerbIndex(type);
2176
2083
  this.verbCountsByType[typeIndex]++;
2177
- this.verbTypeCache.set(verb.id, type);
2178
2084
  // COW-aware write (v5.0.1): Use COW helper for branch isolation
2179
2085
  await this.writeObjectToBranch(path, verb);
2180
- // v5.7.0: Update GraphAdjacencyIndex incrementally for billion-scale optimization
2181
- // CRITICAL: Only update if index already initialized to avoid circular dependency
2182
- // Index is lazy-loaded on first query, then maintained incrementally
2183
- if (this.graphIndex && this.graphIndex.isInitialized) {
2184
- // Fast incremental update - no rebuild needed
2086
+ // v6.0.0: Update GraphAdjacencyIndex incrementally (always available after init())
2087
+ // GraphAdjacencyIndex.addVerb() calls ensureInitialized() automatically
2088
+ if (this.graphIndex) {
2089
+ prodLog.debug(`[BaseStorage] Updating GraphAdjacencyIndex with verb ${verb.id}`);
2185
2090
  await this.graphIndex.addVerb({
2186
2091
  id: verb.id,
2187
2092
  sourceId: verb.sourceId,
@@ -2193,8 +2098,12 @@ export class BaseStorage extends BaseStorageAdapter {
2193
2098
  type: verb.verb,
2194
2099
  createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
2195
2100
  updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
2196
- createdBy: { augmentation: 'storage', version: '5.7.0' }
2101
+ createdBy: { augmentation: 'storage', version: '6.0.0' }
2197
2102
  });
2103
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex updated successfully`);
2104
+ }
2105
+ else {
2106
+ prodLog.warn(`[BaseStorage] graphIndex is null, cannot update index for verb ${verb.id}`);
2198
2107
  }
2199
2108
  // Periodically save statistics
2200
2109
  if (this.verbCountsByType[typeIndex] % 100 === 0) {
@@ -2202,69 +2111,89 @@ export class BaseStorage extends BaseStorageAdapter {
2202
2111
  }
2203
2112
  }
2204
2113
  /**
2205
- * Get a verb from storage (type-first path)
2114
+ * Get a verb from storage (ID-first path)
2206
2115
  */
2207
2116
  async getVerb_internal(id) {
2208
- // Try cache first for O(1) retrieval
2209
- const cachedType = this.verbTypeCache.get(id);
2210
- if (cachedType) {
2211
- const path = getVerbVectorPath(cachedType, id);
2117
+ // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
2118
+ const path = getVerbVectorPath(id);
2119
+ try {
2212
2120
  // COW-aware read (v5.0.1): Use COW helper for branch isolation
2213
2121
  const verb = await this.readWithInheritance(path);
2214
- // v5.7.10: Deserialize connections Map from JSON storage format
2215
- return verb ? this.deserializeVerb(verb) : null;
2216
- }
2217
- // Search across all types (only on first access)
2218
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2219
- const type = TypeUtils.getVerbFromIndex(i);
2220
- const path = getVerbVectorPath(type, id);
2221
- try {
2222
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2223
- const verb = await this.readWithInheritance(path);
2224
- if (verb) {
2225
- // Cache the type for next time (read from verb.verb field)
2226
- this.verbTypeCache.set(id, verb.verb);
2227
- // v5.7.10: Deserialize connections Map from JSON storage format
2228
- return this.deserializeVerb(verb);
2229
- }
2230
- }
2231
- catch (error) {
2232
- // Not in this type, continue
2122
+ if (verb) {
2123
+ // v5.7.10: Deserialize connections Map from JSON storage format
2124
+ return this.deserializeVerb(verb);
2233
2125
  }
2234
2126
  }
2127
+ catch (error) {
2128
+ // Entity not found
2129
+ return null;
2130
+ }
2235
2131
  return null;
2236
2132
  }
2237
2133
  /**
2238
- * Get verbs by source (COW-aware implementation)
2239
- * v5.4.0: Fixed to directly list verb files instead of directories
2134
+ * Get verbs by source (v6.0.0: Uses GraphAdjacencyIndex when available)
2135
+ * Falls back to shard iteration during initialization to avoid circular dependency
2240
2136
  */
2241
2137
  async getVerbsBySource_internal(sourceId) {
2242
- // v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
2243
- // v5.7.0 called getGraphIndex() here, creating deadlock during initialization:
2244
- // GraphAdjacencyIndex.rebuild() → storage.getVerbs() → getVerbsBySource_internal() → getGraphIndex() → [deadlock]
2245
- // v5.4.0: Type-first implementation - scan across all verb types
2246
- // COW-aware: uses readWithInheritance for each verb
2247
2138
  await this.ensureInitialized();
2139
+ prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`);
2140
+ // v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
2141
+ if (this.graphIndex && this.graphIndex.isInitialized) {
2142
+ try {
2143
+ const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId);
2144
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`);
2145
+ const results = [];
2146
+ for (const verbId of verbIds) {
2147
+ const verb = await this.getVerb_internal(verbId);
2148
+ const metadata = await this.getVerbMetadata(verbId);
2149
+ if (verb && metadata) {
2150
+ results.push({
2151
+ ...verb,
2152
+ weight: metadata.weight,
2153
+ confidence: metadata.confidence,
2154
+ createdAt: metadata.createdAt
2155
+ ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
2156
+ : Date.now(),
2157
+ updatedAt: metadata.updatedAt
2158
+ ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
2159
+ : Date.now(),
2160
+ service: metadata.service,
2161
+ createdBy: metadata.createdBy,
2162
+ metadata: metadata || {}
2163
+ });
2164
+ }
2165
+ }
2166
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex path returned ${results.length} verbs`);
2167
+ return results;
2168
+ }
2169
+ catch (error) {
2170
+ prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
2171
+ }
2172
+ }
2173
+ // v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
2174
+ prodLog.debug(`[BaseStorage] Using shard iteration fallback for sourceId=${sourceId}`);
2248
2175
  const results = [];
2249
- // Iterate through all verb types
2250
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2251
- const type = TypeUtils.getVerbFromIndex(i);
2252
- const typeDir = `entities/verbs/${type}/vectors`;
2176
+ let shardsScanned = 0;
2177
+ let verbsFound = 0;
2178
+ for (let shard = 0; shard < 256; shard++) {
2179
+ const shardHex = shard.toString(16).padStart(2, '0');
2180
+ const shardDir = `entities/verbs/${shardHex}`;
2253
2181
  try {
2254
- // v5.4.0 FIX: List all verb files directly (not shard directories)
2255
- // listObjectsInBranch returns full paths to .json files, not directories
2256
- const verbFiles = await this.listObjectsInBranch(typeDir);
2182
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2183
+ shardsScanned++;
2257
2184
  for (const verbPath of verbFiles) {
2258
- // Skip if not a .json file
2259
- if (!verbPath.endsWith('.json'))
2185
+ if (!verbPath.includes('/vectors.json'))
2260
2186
  continue;
2261
2187
  try {
2262
- const verb = await this.readWithInheritance(verbPath);
2263
- if (verb && verb.sourceId === sourceId) {
2264
- // v5.4.0: Use proper path helper instead of string replacement
2265
- const metadataPath = getVerbMetadataPath(type, verb.id);
2188
+ const rawVerb = await this.readWithInheritance(verbPath);
2189
+ if (!rawVerb)
2190
+ continue;
2191
+ verbsFound++;
2192
+ // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
2193
+ const verb = this.deserializeVerb(rawVerb);
2194
+ if (verb.sourceId === sourceId) {
2195
+ const metadataPath = getVerbMetadataPath(verb.id);
2266
2196
  const metadata = await this.readWithInheritance(metadataPath);
2267
- // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
2268
2197
  results.push({
2269
2198
  ...verb,
2270
2199
  weight: metadata?.weight,
@@ -2283,13 +2212,15 @@ export class BaseStorage extends BaseStorageAdapter {
2283
2212
  }
2284
2213
  catch (error) {
2285
2214
  // Skip verbs that fail to load
2215
+ prodLog.debug(`[BaseStorage] Failed to load verb from ${verbPath}:`, error);
2286
2216
  }
2287
2217
  }
2288
2218
  }
2289
2219
  catch (error) {
2290
- // Skip types that have no data
2220
+ // Skip shards that have no data
2291
2221
  }
2292
2222
  }
2223
+ prodLog.debug(`[BaseStorage] Shard iteration: scanned ${shardsScanned} shards, found ${verbsFound} total verbs, matched ${results.length} for sourceId=${sourceId}`);
2293
2224
  return results;
2294
2225
  }
2295
2226
  /**
@@ -2335,52 +2266,46 @@ export class BaseStorage extends BaseStorageAdapter {
2335
2266
  }
2336
2267
  // Convert sourceIds to Set for O(1) lookup
2337
2268
  const sourceIdSet = new Set(sourceIds);
2338
- // Determine which verb types to scan
2339
- const typesToScan = [];
2340
- if (verbType) {
2341
- typesToScan.push(verbType);
2342
- }
2343
- else {
2344
- // Scan all verb types
2345
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2346
- typesToScan.push(TypeUtils.getVerbFromIndex(i));
2347
- }
2348
- }
2349
- // Scan verb types and collect matching verbs
2350
- for (const type of typesToScan) {
2351
- const typeDir = `entities/verbs/${type}/vectors`;
2269
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types
2270
+ for (let shard = 0; shard < 256; shard++) {
2271
+ const shardHex = shard.toString(16).padStart(2, '0');
2272
+ const shardDir = `entities/verbs/${shardHex}`;
2352
2273
  try {
2353
- // List all verb files of this type
2354
- const verbFiles = await this.listObjectsInBranch(typeDir);
2274
+ // List all verb files in this shard
2275
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2355
2276
  // Build paths for batch read
2356
2277
  const verbPaths = [];
2357
2278
  const metadataPaths = [];
2358
2279
  const pathToId = new Map();
2359
2280
  for (const verbPath of verbFiles) {
2360
- if (!verbPath.endsWith('.json'))
2281
+ if (!verbPath.includes('/vectors.json'))
2361
2282
  continue;
2362
2283
  verbPaths.push(verbPath);
2363
- // Extract ID from path: "entities/verbs/{type}/vectors/{shard}/{id}.json"
2284
+ // Extract ID from path: "entities/verbs/{shard}/{id}/vector.json"
2364
2285
  const parts = verbPath.split('/');
2365
- const filename = parts[parts.length - 1];
2366
- const verbId = filename.replace('.json', '');
2286
+ const verbId = parts[parts.length - 2]; // ID is second-to-last segment
2367
2287
  pathToId.set(verbPath, verbId);
2368
2288
  // Prepare metadata path
2369
- metadataPaths.push(getVerbMetadataPath(type, verbId));
2289
+ metadataPaths.push(getVerbMetadataPath(verbId));
2370
2290
  }
2371
- // Batch read all verb files for this type
2291
+ // Batch read all verb files for this shard
2372
2292
  const verbDataMap = await this.readBatchWithInheritance(verbPaths);
2373
2293
  const metadataMap = await this.readBatchWithInheritance(metadataPaths);
2374
2294
  // Process results
2375
- for (const [verbPath, verbData] of verbDataMap.entries()) {
2376
- if (!verbData || !verbData.sourceId)
2295
+ for (const [verbPath, rawVerbData] of verbDataMap.entries()) {
2296
+ if (!rawVerbData || !rawVerbData.sourceId)
2377
2297
  continue;
2298
+ // v6.0.0: Deserialize connections Map from JSON storage format
2299
+ const verbData = this.deserializeVerb(rawVerbData);
2378
2300
  // Check if this verb's source is in our requested set
2379
2301
  if (!sourceIdSet.has(verbData.sourceId))
2380
2302
  continue;
2303
+ // If verbType specified, filter by type
2304
+ if (verbType && verbData.verb !== verbType)
2305
+ continue;
2381
2306
  // Found matching verb - hydrate with metadata
2382
2307
  const verbId = pathToId.get(verbPath);
2383
- const metadataPath = getVerbMetadataPath(type, verbId);
2308
+ const metadataPath = getVerbMetadataPath(verbId);
2384
2309
  const metadata = metadataMap.get(metadataPath) || {};
2385
2310
  const hydratedVerb = {
2386
2311
  ...verbData,
@@ -2402,7 +2327,7 @@ export class BaseStorage extends BaseStorageAdapter {
2402
2327
  }
2403
2328
  }
2404
2329
  catch (error) {
2405
- // Skip types that have no data
2330
+ // Skip shards that have no data
2406
2331
  }
2407
2332
  }
2408
2333
  return results;
@@ -2413,31 +2338,57 @@ export class BaseStorage extends BaseStorageAdapter {
2413
2338
  * v5.4.0: Fixed to directly list verb files instead of directories
2414
2339
  */
2415
2340
  async getVerbsByTarget_internal(targetId) {
2416
- // v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
2417
- // v5.7.0 called getGraphIndex() here, creating deadlock during initialization
2418
- // v5.4.0: Type-first implementation - scan across all verb types
2419
- // COW-aware: uses readWithInheritance for each verb
2420
2341
  await this.ensureInitialized();
2342
+ // v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
2343
+ if (this.graphIndex && this.graphIndex.isInitialized) {
2344
+ try {
2345
+ const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId);
2346
+ const results = [];
2347
+ for (const verbId of verbIds) {
2348
+ const verb = await this.getVerb_internal(verbId);
2349
+ const metadata = await this.getVerbMetadata(verbId);
2350
+ if (verb && metadata) {
2351
+ results.push({
2352
+ ...verb,
2353
+ weight: metadata.weight,
2354
+ confidence: metadata.confidence,
2355
+ createdAt: metadata.createdAt
2356
+ ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
2357
+ : Date.now(),
2358
+ updatedAt: metadata.updatedAt
2359
+ ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
2360
+ : Date.now(),
2361
+ service: metadata.service,
2362
+ createdBy: metadata.createdBy,
2363
+ metadata: metadata || {}
2364
+ });
2365
+ }
2366
+ }
2367
+ return results;
2368
+ }
2369
+ catch (error) {
2370
+ prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
2371
+ }
2372
+ }
2373
+ // v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
2421
2374
  const results = [];
2422
- // Iterate through all verb types
2423
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2424
- const type = TypeUtils.getVerbFromIndex(i);
2425
- const typeDir = `entities/verbs/${type}/vectors`;
2375
+ for (let shard = 0; shard < 256; shard++) {
2376
+ const shardHex = shard.toString(16).padStart(2, '0');
2377
+ const shardDir = `entities/verbs/${shardHex}`;
2426
2378
  try {
2427
- // v5.4.0 FIX: List all verb files directly (not shard directories)
2428
- // listObjectsInBranch returns full paths to .json files, not directories
2429
- const verbFiles = await this.listObjectsInBranch(typeDir);
2379
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2430
2380
  for (const verbPath of verbFiles) {
2431
- // Skip if not a .json file
2432
- if (!verbPath.endsWith('.json'))
2381
+ if (!verbPath.includes('/vectors.json'))
2433
2382
  continue;
2434
2383
  try {
2435
- const verb = await this.readWithInheritance(verbPath);
2436
- if (verb && verb.targetId === targetId) {
2437
- // v5.4.0: Use proper path helper instead of string replacement
2438
- const metadataPath = getVerbMetadataPath(type, verb.id);
2384
+ const rawVerb = await this.readWithInheritance(verbPath);
2385
+ if (!rawVerb)
2386
+ continue;
2387
+ // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
2388
+ const verb = this.deserializeVerb(rawVerb);
2389
+ if (verb.targetId === targetId) {
2390
+ const metadataPath = getVerbMetadataPath(verb.id);
2439
2391
  const metadata = await this.readWithInheritance(metadataPath);
2440
- // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
2441
2392
  results.push({
2442
2393
  ...verb,
2443
2394
  weight: metadata?.weight,
@@ -2460,95 +2411,77 @@ export class BaseStorage extends BaseStorageAdapter {
2460
2411
  }
2461
2412
  }
2462
2413
  catch (error) {
2463
- // Skip types that have no data
2414
+ // Skip shards that have no data
2464
2415
  }
2465
2416
  }
2466
2417
  return results;
2467
2418
  }
2468
2419
  /**
2469
- * Get verbs by type (O(1) with type-first paths!)
2420
+ * Get verbs by type (v6.0.0: Shard iteration with type filtering)
2470
2421
  */
2471
2422
  async getVerbsByType_internal(verbType) {
2472
- const type = verbType;
2473
- const prefix = `entities/verbs/${type}/vectors/`;
2474
- // COW-aware list (v5.0.1): Use COW helper for branch isolation
2475
- const paths = await this.listObjectsInBranch(prefix);
2423
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of type-first paths
2476
2424
  const verbs = [];
2477
- for (const path of paths) {
2425
+ for (let shard = 0; shard < 256; shard++) {
2426
+ const shardHex = shard.toString(16).padStart(2, '0');
2427
+ const shardDir = `entities/verbs/${shardHex}`;
2478
2428
  try {
2479
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2480
- const rawVerb = await this.readWithInheritance(path);
2481
- if (!rawVerb)
2482
- continue;
2483
- // v5.7.10: Deserialize connections Map from JSON storage format
2484
- // Replaces v5.7.8 manual deserialization (lines 2599-2605)
2485
- const hnswVerb = this.deserializeVerb(rawVerb);
2486
- // Cache type from HNSWVerb for future O(1) retrievals
2487
- this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb);
2488
- // Load metadata separately (optional in v4.0.0!)
2489
- // FIX: Don't skip verbs without metadata - metadata is optional!
2490
- const metadata = await this.getVerbMetadata(hnswVerb.id);
2491
- // v4.8.0: Extract standard fields from metadata to top-level
2492
- const metadataObj = (metadata || {});
2493
- const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
2494
- const verbWithMetadata = {
2495
- id: hnswVerb.id,
2496
- vector: [...hnswVerb.vector],
2497
- connections: hnswVerb.connections, // v5.7.10: Already deserialized
2498
- verb: hnswVerb.verb,
2499
- sourceId: hnswVerb.sourceId,
2500
- targetId: hnswVerb.targetId,
2501
- createdAt: createdAt || Date.now(),
2502
- updatedAt: updatedAt || Date.now(),
2503
- confidence: confidence,
2504
- weight: weight,
2505
- service: service,
2506
- data: data,
2507
- createdBy,
2508
- metadata: customMetadata
2509
- };
2510
- verbs.push(verbWithMetadata);
2429
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2430
+ for (const verbPath of verbFiles) {
2431
+ if (!verbPath.includes('/vectors.json'))
2432
+ continue;
2433
+ try {
2434
+ const rawVerb = await this.readWithInheritance(verbPath);
2435
+ if (!rawVerb)
2436
+ continue;
2437
+ // v5.7.10: Deserialize connections Map from JSON storage format
2438
+ const hnswVerb = this.deserializeVerb(rawVerb);
2439
+ // Filter by verb type
2440
+ if (hnswVerb.verb !== verbType)
2441
+ continue;
2442
+ // Load metadata separately (optional in v4.0.0!)
2443
+ const metadata = await this.getVerbMetadata(hnswVerb.id);
2444
+ // v4.8.0: Extract standard fields from metadata to top-level
2445
+ const metadataObj = (metadata || {});
2446
+ const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
2447
+ const verbWithMetadata = {
2448
+ id: hnswVerb.id,
2449
+ vector: [...hnswVerb.vector],
2450
+ connections: hnswVerb.connections, // v5.7.10: Already deserialized
2451
+ verb: hnswVerb.verb,
2452
+ sourceId: hnswVerb.sourceId,
2453
+ targetId: hnswVerb.targetId,
2454
+ createdAt: createdAt || Date.now(),
2455
+ updatedAt: updatedAt || Date.now(),
2456
+ confidence: confidence,
2457
+ weight: weight,
2458
+ service: service,
2459
+ data: data,
2460
+ createdBy,
2461
+ metadata: customMetadata
2462
+ };
2463
+ verbs.push(verbWithMetadata);
2464
+ }
2465
+ catch (error) {
2466
+ // Skip verbs that fail to load
2467
+ }
2468
+ }
2511
2469
  }
2512
2470
  catch (error) {
2513
- prodLog.warn(`[BaseStorage] Failed to load verb from ${path}:`, error);
2471
+ // Skip shards that have no data
2514
2472
  }
2515
2473
  }
2516
2474
  return verbs;
2517
2475
  }
2518
2476
  /**
2519
- * Delete a verb from storage (type-first path)
2477
+ * Delete a verb from storage (v6.0.0: ID-first, O(1) delete)
2520
2478
  */
2521
2479
  async deleteVerb_internal(id) {
2522
- // Try cache first
2523
- const cachedType = this.verbTypeCache.get(id);
2524
- if (cachedType) {
2525
- const path = getVerbVectorPath(cachedType, id);
2526
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2527
- await this.deleteObjectFromBranch(path);
2528
- const typeIndex = TypeUtils.getVerbIndex(cachedType);
2529
- if (this.verbCountsByType[typeIndex] > 0) {
2530
- this.verbCountsByType[typeIndex]--;
2531
- }
2532
- this.verbTypeCache.delete(id);
2533
- return;
2534
- }
2535
- // Search across all types
2536
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2537
- const type = TypeUtils.getVerbFromIndex(i);
2538
- const path = getVerbVectorPath(type, id);
2539
- try {
2540
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2541
- await this.deleteObjectFromBranch(path);
2542
- if (this.verbCountsByType[i] > 0) {
2543
- this.verbCountsByType[i]--;
2544
- }
2545
- this.verbTypeCache.delete(id);
2546
- return;
2547
- }
2548
- catch (error) {
2549
- // Continue
2550
- }
2551
- }
2480
+ // v6.0.0: Direct O(1) delete with ID-first path
2481
+ const path = getVerbVectorPath(id);
2482
+ await this.deleteObjectFromBranch(path);
2483
+ // Note: Type-specific counts will be decremented via metadata tracking
2484
+ // The real type is in metadata, accessible if needed via getVerbMetadata(id)
2552
2485
  }
2553
2486
  /**
2554
2487
  * Helper method to convert a Map to a plain object for serialization