@soulcraft/brainy 5.12.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,32 +39,36 @@ export function getDirectoryPath(entityType, dataType) {
39
39
  * Built-in type-aware organization for all storage adapters
40
40
  */
41
41
  /**
42
- * Get type-first path for noun vectors
42
+ * Get ID-first path for noun vectors (v6.0.0)
43
+ * No type parameter needed - direct O(1) lookup by ID
43
44
  */
44
- function getNounVectorPath(type, id) {
45
+ function getNounVectorPath(id) {
45
46
  const shard = getShardIdFromUuid(id);
46
- return `entities/nouns/${type}/vectors/${shard}/${id}.json`;
47
+ return `entities/nouns/${shard}/${id}/vectors.json`;
47
48
  }
48
49
  /**
49
- * Get type-first path for noun metadata
50
+ * Get ID-first path for noun metadata (v6.0.0)
51
+ * No type parameter needed - direct O(1) lookup by ID
50
52
  */
51
- function getNounMetadataPath(type, id) {
53
+ function getNounMetadataPath(id) {
52
54
  const shard = getShardIdFromUuid(id);
53
- return `entities/nouns/${type}/metadata/${shard}/${id}.json`;
55
+ return `entities/nouns/${shard}/${id}/metadata.json`;
54
56
  }
55
57
  /**
56
- * Get type-first path for verb vectors
58
+ * Get ID-first path for verb vectors (v6.0.0)
59
+ * No type parameter needed - direct O(1) lookup by ID
57
60
  */
58
- function getVerbVectorPath(type, id) {
61
+ function getVerbVectorPath(id) {
59
62
  const shard = getShardIdFromUuid(id);
60
- return `entities/verbs/${type}/vectors/${shard}/${id}.json`;
63
+ return `entities/verbs/${shard}/${id}/vectors.json`;
61
64
  }
62
65
  /**
63
- * Get type-first path for verb metadata
66
+ * Get ID-first path for verb metadata (v6.0.0)
67
+ * No type parameter needed - direct O(1) lookup by ID
64
68
  */
65
- function getVerbMetadataPath(type, id) {
69
+ function getVerbMetadataPath(id) {
66
70
  const shard = getShardIdFromUuid(id);
67
- return `entities/verbs/${type}/metadata/${shard}/${id}.json`;
71
+ return `entities/verbs/${shard}/${id}/metadata.json`;
68
72
  }
69
73
  /**
70
74
  * Base storage adapter that implements common functionality
@@ -89,9 +93,9 @@ export class BaseStorage extends BaseStorageAdapter {
89
93
  this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
90
94
  this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 508 bytes (Stage 3: 127 types)
91
95
  // Total: 676 bytes (99.2% reduction vs Map-based tracking)
92
- // Type cache for O(1) lookups after first access
93
- this.nounTypeCache = new Map();
94
- this.verbTypeCache = new Map();
96
+ // v6.0.0: Type caches REMOVED - ID-first paths eliminate need for type lookups!
97
+ // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json
98
+ // Type is just a field in the metadata, indexed by MetadataIndexManager for queries
95
99
  // v5.5.0: Track if type counts have been rebuilt (prevent repeated rebuilds)
96
100
  this.typeCountsRebuilt = false;
97
101
  }
@@ -176,8 +180,33 @@ export class BaseStorage extends BaseStorageAdapter {
176
180
  async init() {
177
181
  // Load type statistics from storage (if they exist)
178
182
  await this.loadTypeStatistics();
183
+ // v6.0.0: Create GraphAdjacencyIndex (lazy-loaded, no rebuild)
184
+ // LSM-trees are initialized on first use via ensureInitialized()
185
+ // Index is populated incrementally as verbs are added via addVerb()
186
+ try {
187
+ prodLog.debug('[BaseStorage] Creating GraphAdjacencyIndex...');
188
+ this.graphIndex = new GraphAdjacencyIndex(this);
189
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex instantiated (lazy-loaded), graphIndex=${!!this.graphIndex}`);
190
+ }
191
+ catch (error) {
192
+ prodLog.error('[BaseStorage] Failed to create GraphAdjacencyIndex:', error);
193
+ throw error;
194
+ }
179
195
  this.isInitialized = true;
180
196
  }
197
+ /**
198
+ * Rebuild GraphAdjacencyIndex from existing verbs (v6.0.0)
199
+ * Call this manually if you have existing verb data that needs to be indexed
200
+ * @public
201
+ */
202
+ async rebuildGraphIndex() {
203
+ if (!this.graphIndex) {
204
+ throw new Error('GraphAdjacencyIndex not initialized');
205
+ }
206
+ prodLog.info('[BaseStorage] Rebuilding graph index from existing data...');
207
+ await this.graphIndex.rebuild();
208
+ prodLog.info('[BaseStorage] Graph index rebuild complete');
209
+ }
181
210
  /**
182
211
  * Ensure the storage adapter is initialized
183
212
  */
@@ -851,60 +880,44 @@ export class BaseStorage extends BaseStorageAdapter {
851
880
  */
852
881
  async getNounsWithPagination(options) {
853
882
  await this.ensureInitialized();
854
- const { limit, offset = 0, filter } = options; // cursor intentionally not extracted (not yet implemented)
883
+ const { limit, offset = 0, filter } = options;
855
884
  const collectedNouns = [];
856
- const targetCount = offset + limit; // Early termination target
857
- // v5.5.0 BUG FIX: Only use optimization if counts are reliable
858
- const totalNounCountFromArray = this.nounCountsByType.reduce((sum, c) => sum + c, 0);
859
- const useOptimization = totalNounCountFromArray > 0;
860
- // v5.5.0: Iterate through noun types with billion-scale optimizations
861
- for (let i = 0; i < NOUN_TYPE_COUNT && collectedNouns.length < targetCount; i++) {
862
- // OPTIMIZATION 1: Skip empty types (only if counts are reliable)
863
- if (useOptimization && this.nounCountsByType[i] === 0) {
864
- continue;
865
- }
866
- const type = TypeUtils.getNounFromIndex(i);
867
- // If filtering by type, skip other types
868
- if (filter?.nounType) {
869
- const filterTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
870
- if (!filterTypes.includes(type)) {
871
- continue;
872
- }
873
- }
874
- const typeDir = `entities/nouns/${type}/vectors`;
885
+ const targetCount = offset + limit;
886
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types
887
+ for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) {
888
+ const shardHex = shard.toString(16).padStart(2, '0');
889
+ const shardDir = `entities/nouns/${shardHex}`;
875
890
  try {
876
- // List all noun files for this type
877
- const nounFiles = await this.listObjectsInBranch(typeDir);
891
+ const nounFiles = await this.listObjectsInBranch(shardDir);
878
892
  for (const nounPath of nounFiles) {
879
- // OPTIMIZATION 2: Early termination (stop when we have enough)
880
- if (collectedNouns.length >= targetCount) {
893
+ if (collectedNouns.length >= targetCount)
881
894
  break;
882
- }
883
- // Skip if not a .json file
884
- if (!nounPath.endsWith('.json'))
895
+ if (!nounPath.includes('/vectors.json'))
885
896
  continue;
886
897
  try {
887
- const rawNoun = await this.readWithInheritance(nounPath);
888
- if (rawNoun) {
889
- // v5.7.10: Deserialize connections Map from JSON storage format
890
- // Replaces v5.7.8 manual deserialization (removed 13 lines at 1156-1168)
891
- const noun = this.deserializeNoun(rawNoun);
892
- // Load metadata
893
- const metadataPath = getNounMetadataPath(type, noun.id);
894
- const metadata = await this.readWithInheritance(metadataPath);
898
+ const noun = await this.readWithInheritance(nounPath);
899
+ if (noun) {
900
+ const deserialized = this.deserializeNoun(noun);
901
+ const metadata = await this.getNounMetadata(deserialized.id);
895
902
  if (metadata) {
896
- // Apply service filter if specified
903
+ // Apply type filter
904
+ if (filter?.nounType && metadata.noun) {
905
+ const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
906
+ if (!types.includes(metadata.noun)) {
907
+ continue;
908
+ }
909
+ }
910
+ // Apply service filter
897
911
  if (filter?.service) {
898
912
  const services = Array.isArray(filter.service) ? filter.service : [filter.service];
899
913
  if (metadata.service && !services.includes(metadata.service)) {
900
914
  continue;
901
915
  }
902
916
  }
903
- // Combine noun + metadata (v5.4.0: Extract standard fields to top-level)
917
+ // Combine noun + metadata
904
918
  collectedNouns.push({
905
- ...noun,
906
- // v5.7.10: connections already deserialized by deserializeNoun()
907
- type: metadata.noun || type, // Required: Extract type from metadata
919
+ ...deserialized,
920
+ type: (metadata.noun || 'thing'),
908
921
  confidence: metadata.confidence,
909
922
  weight: metadata.weight,
910
923
  createdAt: metadata.createdAt
@@ -927,15 +940,15 @@ export class BaseStorage extends BaseStorageAdapter {
927
940
  }
928
941
  }
929
942
  catch (error) {
930
- // Skip types that have no data
943
+ // Skip shards that have no data
931
944
  }
932
945
  }
933
- // Apply pagination (v5.5.0: Efficient slicing after early termination)
946
+ // Apply pagination
934
947
  const paginatedNouns = collectedNouns.slice(offset, offset + limit);
935
- const hasMore = collectedNouns.length > targetCount; // v5.7.11: Fixed >= to > (was causing infinite loop)
948
+ const hasMore = collectedNouns.length > targetCount;
936
949
  return {
937
950
  items: paginatedNouns,
938
- totalCount: collectedNouns.length, // Accurate count of collected results
951
+ totalCount: collectedNouns.length,
939
952
  hasMore,
940
953
  nextCursor: hasMore && paginatedNouns.length > 0
941
954
  ? paginatedNouns[paginatedNouns.length - 1].id
@@ -962,58 +975,70 @@ export class BaseStorage extends BaseStorageAdapter {
962
975
  const { limit, offset = 0, filter } = options; // cursor intentionally not extracted (not yet implemented)
963
976
  const collectedVerbs = [];
964
977
  const targetCount = offset + limit; // Early termination target
965
- // v5.5.0 BUG FIX: Only use optimization if counts are reliable
966
- const totalVerbCountFromArray = this.verbCountsByType.reduce((sum, c) => sum + c, 0);
967
- const useOptimization = totalVerbCountFromArray > 0;
968
- // v5.5.0: Iterate through verb types with billion-scale optimizations
969
- for (let i = 0; i < VERB_TYPE_COUNT && collectedVerbs.length < targetCount; i++) {
970
- // OPTIMIZATION 1: Skip empty types (only if counts are reliable)
971
- if (useOptimization && this.verbCountsByType[i] === 0) {
972
- continue;
973
- }
974
- const type = TypeUtils.getVerbFromIndex(i);
975
- // If filtering by verbType, skip other types
976
- if (filter?.verbType) {
977
- const filterTypes = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
978
- if (!filterTypes.includes(type)) {
979
- continue;
980
- }
981
- }
978
+ // Prepare filter sets for efficient lookup
979
+ const filterVerbTypes = filter?.verbType
980
+ ? new Set(Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType])
981
+ : null;
982
+ const filterSourceIds = filter?.sourceId
983
+ ? new Set(Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId])
984
+ : null;
985
+ const filterTargetIds = filter?.targetId
986
+ ? new Set(Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId])
987
+ : null;
988
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types - single pass!
989
+ for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
990
+ const shardHex = shard.toString(16).padStart(2, '0');
991
+ const shardDir = `entities/verbs/${shardHex}`;
982
992
  try {
983
- const verbsOfType = await this.getVerbsByType_internal(type);
984
- // Apply filtering inline (memory efficient)
985
- for (const verb of verbsOfType) {
986
- // OPTIMIZATION 2: Early termination (stop when we have enough)
987
- if (collectedVerbs.length >= targetCount) {
993
+ const verbFiles = await this.listObjectsInBranch(shardDir);
994
+ for (const verbPath of verbFiles) {
995
+ if (collectedVerbs.length >= targetCount)
988
996
  break;
989
- }
990
- // Apply filters if specified
991
- if (filter) {
992
- // Filter by sourceId
993
- if (filter.sourceId) {
994
- const sourceIds = Array.isArray(filter.sourceId)
995
- ? filter.sourceId
996
- : [filter.sourceId];
997
- if (!sourceIds.includes(verb.sourceId)) {
998
- continue;
999
- }
997
+ if (!verbPath.includes('/vectors.json'))
998
+ continue;
999
+ try {
1000
+ const rawVerb = await this.readWithInheritance(verbPath);
1001
+ if (!rawVerb)
1002
+ continue;
1003
+ // v6.0.0: Deserialize connections Map from JSON storage format
1004
+ const verb = this.deserializeVerb(rawVerb);
1005
+ // Apply type filter
1006
+ if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
1007
+ continue;
1000
1008
  }
1001
- // Filter by targetId
1002
- if (filter.targetId) {
1003
- const targetIds = Array.isArray(filter.targetId)
1004
- ? filter.targetId
1005
- : [filter.targetId];
1006
- if (!targetIds.includes(verb.targetId)) {
1007
- continue;
1008
- }
1009
+ // Apply sourceId filter
1010
+ if (filterSourceIds && !filterSourceIds.has(verb.sourceId)) {
1011
+ continue;
1012
+ }
1013
+ // Apply targetId filter
1014
+ if (filterTargetIds && !filterTargetIds.has(verb.targetId)) {
1015
+ continue;
1009
1016
  }
1017
+ // Load metadata
1018
+ const metadata = await this.getVerbMetadata(verb.id);
1019
+ // Combine verb + metadata
1020
+ collectedVerbs.push({
1021
+ ...verb,
1022
+ weight: metadata?.weight,
1023
+ confidence: metadata?.confidence,
1024
+ createdAt: metadata?.createdAt
1025
+ ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
1026
+ : Date.now(),
1027
+ updatedAt: metadata?.updatedAt
1028
+ ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
1029
+ : Date.now(),
1030
+ service: metadata?.service,
1031
+ createdBy: metadata?.createdBy,
1032
+ metadata: metadata || {}
1033
+ });
1034
+ }
1035
+ catch (error) {
1036
+ // Skip verbs that fail to load
1010
1037
  }
1011
- // Verb passed all filters - add to collection
1012
- collectedVerbs.push(verb);
1013
1038
  }
1014
1039
  }
1015
1040
  catch (error) {
1016
- // Skip types that have no data (directory may not exist)
1041
+ // Skip shards that have no data
1017
1042
  }
1018
1043
  }
1019
1044
  // Apply pagination (v5.5.0: Efficient slicing after early termination)
@@ -1386,11 +1411,8 @@ export class BaseStorage extends BaseStorageAdapter {
1386
1411
  */
1387
1412
  async saveNounMetadata_internal(id, metadata) {
1388
1413
  await this.ensureInitialized();
1389
- // v5.4.0: Extract and cache type for type-first routing
1390
- const type = (metadata.noun || 'thing');
1391
- this.nounTypeCache.set(id, type);
1392
- // v5.4.0: Use type-first path
1393
- const path = getNounMetadataPath(type, id);
1414
+ // v6.0.0: ID-first path - no type needed!
1415
+ const path = getNounMetadataPath(id);
1394
1416
  // Determine if this is a new entity by checking if metadata already exists
1395
1417
  const existingMetadata = await this.readWithInheritance(path);
1396
1418
  const isNew = !existingMetadata;
@@ -1411,6 +1433,17 @@ export class BaseStorage extends BaseStorageAdapter {
1411
1433
  /**
1412
1434
  * Get noun metadata from storage (METADATA-ONLY, NO VECTORS)
1413
1435
  *
1436
+ * **Performance (v6.0.0)**: Direct O(1) ID-first lookup - NO type search needed!
1437
+ * - **All lookups**: 1 read, ~500ms on cloud (consistent performance)
1438
+ * - **No cache needed**: Type is in the metadata, not the path
1439
+ * - **No type search**: ID-first paths eliminate 42-type search entirely
1440
+ *
1441
+ * **Clean architecture (v6.0.0)**:
1442
+ * - Path: `entities/nouns/{SHARD}/{ID}/metadata.json`
1443
+ * - Type is just a field in metadata (`noun: "document"`)
1444
+ * - MetadataIndex handles type queries (no path scanning needed)
1445
+ * - Scales to billions without any overhead
1446
+ *
1414
1447
  * **Performance (v5.11.1)**: Fast path for metadata-only reads
1415
1448
  * - **Speed**: 10ms vs 43ms (76-81% faster than getNoun)
1416
1449
  * - **Bandwidth**: 300 bytes vs 6KB (95% less)
@@ -1440,39 +1473,21 @@ export class BaseStorage extends BaseStorageAdapter {
1440
1473
  * @returns Metadata or null if not found
1441
1474
  *
1442
1475
  * @performance
1443
- * - Type cache O(1) lookup for cached entities
1444
- * - Type scan O(N_types) for cache misses (typically <100ms)
1445
- * - Uses readWithInheritance() for COW branch support
1476
+ * - O(1) direct ID lookup - always 1 read (~500ms on cloud, ~10ms local)
1477
+ * - No caching complexity
1478
+ * - No type search fallbacks
1479
+ * - Works in distributed systems without sync issues
1446
1480
  *
1447
1481
  * @since v4.0.0
1448
- * @since v5.4.0 - Type-first paths
1482
+ * @since v5.4.0 - Type-first paths (removed in v6.0.0)
1449
1483
  * @since v5.11.1 - Promoted to fast path for brain.get() optimization
1484
+ * @since v6.0.0 - CLEAN FIX: ID-first paths eliminate all type-search complexity
1450
1485
  */
1451
1486
  async getNounMetadata(id) {
1452
1487
  await this.ensureInitialized();
1453
- // v5.4.0: Check type cache first (populated during save)
1454
- const cachedType = this.nounTypeCache.get(id);
1455
- if (cachedType) {
1456
- const path = getNounMetadataPath(cachedType, id);
1457
- return this.readWithInheritance(path);
1458
- }
1459
- // Fallback: search across all types (expensive but necessary if cache miss)
1460
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1461
- const type = TypeUtils.getNounFromIndex(i);
1462
- const path = getNounMetadataPath(type, id);
1463
- try {
1464
- const metadata = await this.readWithInheritance(path);
1465
- if (metadata) {
1466
- // Cache the type for next time
1467
- this.nounTypeCache.set(id, type);
1468
- return metadata;
1469
- }
1470
- }
1471
- catch (error) {
1472
- // Not in this type, continue searching
1473
- }
1474
- }
1475
- return null;
1488
+ // v6.0.0: Clean, simple, O(1) lookup - no type needed!
1489
+ const path = getNounMetadataPath(id);
1490
+ return this.readWithInheritance(path);
1476
1491
  }
1477
1492
  /**
1478
1493
  * Batch fetch noun metadata from storage (v5.12.0 - Cloud Storage Optimization)
@@ -1511,74 +1526,19 @@ export class BaseStorage extends BaseStorageAdapter {
1511
1526
  const results = new Map();
1512
1527
  if (ids.length === 0)
1513
1528
  return results;
1514
- // Group IDs by cached type for efficient path construction
1515
- const idsByType = new Map();
1516
- const uncachedIds = [];
1517
- for (const id of ids) {
1518
- const cachedType = this.nounTypeCache.get(id);
1519
- if (cachedType) {
1520
- const idsForType = idsByType.get(cachedType) || [];
1521
- idsForType.push(id);
1522
- idsByType.set(cachedType, idsForType);
1523
- }
1524
- else {
1525
- uncachedIds.push(id);
1526
- }
1527
- }
1528
- // Build paths for known types
1529
- const pathsToFetch = [];
1530
- for (const [type, typeIds] of idsByType.entries()) {
1531
- for (const id of typeIds) {
1532
- pathsToFetch.push({
1533
- path: getNounMetadataPath(type, id),
1534
- id
1535
- });
1536
- }
1537
- }
1538
- // For uncached IDs, we need to search across types (expensive but unavoidable)
1539
- // Strategy: Try most common types first (Document, Thing, Person), then others
1540
- const commonTypes = [NounType.Document, NounType.Thing, NounType.Person, NounType.File];
1541
- const commonTypeSet = new Set(commonTypes);
1542
- const otherTypes = [];
1543
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1544
- const type = TypeUtils.getNounFromIndex(i);
1545
- if (!commonTypeSet.has(type)) {
1546
- otherTypes.push(type);
1547
- }
1548
- }
1549
- const searchOrder = [...commonTypes, ...otherTypes];
1550
- for (const id of uncachedIds) {
1551
- for (const type of searchOrder) {
1552
- // Build path manually to avoid type issues
1553
- const shard = getShardIdFromUuid(id);
1554
- const path = `entities/nouns/${type}/metadata/${shard}/${id}.json`;
1555
- pathsToFetch.push({ path, id });
1556
- }
1557
- }
1558
- // Batch read all paths
1529
+ // v6.0.0: ID-first paths - no type grouping or search needed!
1530
+ // Build direct paths for all IDs
1531
+ const pathsToFetch = ids.map(id => ({
1532
+ path: getNounMetadataPath(id),
1533
+ id
1534
+ }));
1535
+ // Batch read all paths (uses adapter's native batch API or parallel fallback)
1559
1536
  const batchResults = await this.readBatchWithInheritance(pathsToFetch.map(p => p.path));
1560
- // Process results and update cache
1561
- const foundUncached = new Set();
1562
- for (let i = 0; i < pathsToFetch.length; i++) {
1563
- const { path, id } = pathsToFetch[i];
1537
+ // Map results back to IDs
1538
+ for (const { path, id } of pathsToFetch) {
1564
1539
  const metadata = batchResults.get(path);
1565
1540
  if (metadata) {
1566
1541
  results.set(id, metadata);
1567
- // Cache the type for uncached IDs (only on first find)
1568
- if (uncachedIds.includes(id) && !foundUncached.has(id)) {
1569
- // Extract type from path: "entities/nouns/metadata/{type}/{shard}/{id}.json"
1570
- const parts = path.split('/');
1571
- const typeStr = parts[3]; // "document", "thing", etc.
1572
- // Find matching type by string comparison
1573
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1574
- const type = TypeUtils.getNounFromIndex(i);
1575
- if (type === typeStr) {
1576
- this.nounTypeCache.set(id, type);
1577
- break;
1578
- }
1579
- }
1580
- foundUncached.add(id);
1581
- }
1582
1542
  }
1583
1543
  }
1584
1544
  return results;
@@ -1736,36 +1696,13 @@ export class BaseStorage extends BaseStorageAdapter {
1736
1696
  };
1737
1697
  }
1738
1698
  /**
1739
- * Delete noun metadata from storage
1740
- * v5.4.0: Uses type-first paths (must match saveNounMetadata_internal)
1699
+ * Delete noun metadata from storage (v6.0.0: ID-first, O(1) delete)
1741
1700
  */
1742
1701
  async deleteNounMetadata(id) {
1743
1702
  await this.ensureInitialized();
1744
- // v5.4.0: Use cached type for path
1745
- const cachedType = this.nounTypeCache.get(id);
1746
- if (cachedType) {
1747
- const path = getNounMetadataPath(cachedType, id);
1748
- await this.deleteObjectFromBranch(path);
1749
- // Remove from cache after deletion
1750
- this.nounTypeCache.delete(id);
1751
- return;
1752
- }
1753
- // If not in cache, search all types to find and delete
1754
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1755
- const type = TypeUtils.getNounFromIndex(i);
1756
- const path = getNounMetadataPath(type, id);
1757
- try {
1758
- // Check if exists before deleting
1759
- const exists = await this.readWithInheritance(path);
1760
- if (exists) {
1761
- await this.deleteObjectFromBranch(path);
1762
- return;
1763
- }
1764
- }
1765
- catch (error) {
1766
- // Not in this type, continue searching
1767
- }
1768
- }
1703
+ // v6.0.0: Direct O(1) delete with ID-first path
1704
+ const path = getNounMetadataPath(id);
1705
+ await this.deleteObjectFromBranch(path);
1769
1706
  }
1770
1707
  /**
1771
1708
  * Save verb metadata to storage (v4.0.0: now typed)
@@ -1777,7 +1714,7 @@ export class BaseStorage extends BaseStorageAdapter {
1777
1714
  }
1778
1715
  /**
1779
1716
  * Internal method for saving verb metadata (v4.0.0: now typed)
1780
- * v5.4.0: Uses type-first paths (must match getVerbMetadata)
1717
+ * v5.4.0: Uses ID-first paths (must match getVerbMetadata)
1781
1718
  *
1782
1719
  * CRITICAL (v4.1.2): Count synchronization happens here
1783
1720
  * This ensures verb counts are updated AFTER metadata exists, fixing the race condition
@@ -1789,7 +1726,7 @@ export class BaseStorage extends BaseStorageAdapter {
1789
1726
  */
1790
1727
  async saveVerbMetadata_internal(id, metadata) {
1791
1728
  await this.ensureInitialized();
1792
- // v5.4.0: Extract verb type from metadata for type-first path
1729
+ // v5.4.0: Extract verb type from metadata for ID-first path
1793
1730
  const verbType = metadata.verb;
1794
1731
  if (!verbType) {
1795
1732
  // Backward compatibility: fallback to old path if no verb type
@@ -1797,15 +1734,14 @@ export class BaseStorage extends BaseStorageAdapter {
1797
1734
  await this.writeObjectToBranch(keyInfo.fullPath, metadata);
1798
1735
  return;
1799
1736
  }
1800
- // v5.4.0: Use type-first path
1801
- const path = getVerbMetadataPath(verbType, id);
1737
+ // v5.4.0: Use ID-first path
1738
+ const path = getVerbMetadataPath(id);
1802
1739
  // Determine if this is a new verb by checking if metadata already exists
1803
1740
  const existingMetadata = await this.readWithInheritance(path);
1804
1741
  const isNew = !existingMetadata;
1805
1742
  // Save the metadata (COW-aware - writes to branch-specific path)
1806
1743
  await this.writeObjectToBranch(path, metadata);
1807
1744
  // v5.4.0: Cache verb type for faster lookups
1808
- this.verbTypeCache.set(id, verbType);
1809
1745
  // CRITICAL FIX (v4.1.2): Increment verb count for new relationships
1810
1746
  // This runs AFTER metadata is saved
1811
1747
  // Uses synchronous increment since storage operations are already serialized
@@ -1820,69 +1756,34 @@ export class BaseStorage extends BaseStorageAdapter {
1820
1756
  }
1821
1757
  /**
1822
1758
  * Get verb metadata from storage (v4.0.0: now typed)
1823
- * v5.4.0: Uses type-first paths (must match saveVerbMetadata_internal)
1759
+ * v5.4.0: Uses ID-first paths (must match saveVerbMetadata_internal)
1824
1760
  */
1825
1761
  async getVerbMetadata(id) {
1826
1762
  await this.ensureInitialized();
1827
- // v5.4.0: Check type cache first (populated during save)
1828
- const cachedType = this.verbTypeCache.get(id);
1829
- if (cachedType) {
1830
- const path = getVerbMetadataPath(cachedType, id);
1831
- return this.readWithInheritance(path);
1832
- }
1833
- // Fallback: search across all types (expensive but necessary if cache miss)
1834
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
1835
- const type = TypeUtils.getVerbFromIndex(i);
1836
- const path = getVerbMetadataPath(type, id);
1837
- try {
1838
- const metadata = await this.readWithInheritance(path);
1839
- if (metadata) {
1840
- // Cache the type for next time
1841
- this.verbTypeCache.set(id, type);
1842
- return metadata;
1843
- }
1844
- }
1845
- catch (error) {
1846
- // Not in this type, continue searching
1847
- }
1763
+ // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
1764
+ const path = getVerbMetadataPath(id);
1765
+ try {
1766
+ const metadata = await this.readWithInheritance(path);
1767
+ return metadata || null;
1768
+ }
1769
+ catch (error) {
1770
+ // Entity not found
1771
+ return null;
1848
1772
  }
1849
- return null;
1850
1773
  }
1851
1774
  /**
1852
- * Delete verb metadata from storage
1853
- * v5.4.0: Uses type-first paths (must match saveVerbMetadata_internal)
1775
+ * Delete verb metadata from storage (v6.0.0: ID-first, O(1) delete)
1854
1776
  */
1855
1777
  async deleteVerbMetadata(id) {
1856
1778
  await this.ensureInitialized();
1857
- // v5.4.0: Use cached type for path
1858
- const cachedType = this.verbTypeCache.get(id);
1859
- if (cachedType) {
1860
- const path = getVerbMetadataPath(cachedType, id);
1861
- await this.deleteObjectFromBranch(path);
1862
- // Remove from cache after deletion
1863
- this.verbTypeCache.delete(id);
1864
- return;
1865
- }
1866
- // If not in cache, search all types to find and delete
1867
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
1868
- const type = TypeUtils.getVerbFromIndex(i);
1869
- const path = getVerbMetadataPath(type, id);
1870
- try {
1871
- // Check if exists before deleting
1872
- const exists = await this.readWithInheritance(path);
1873
- if (exists) {
1874
- await this.deleteObjectFromBranch(path);
1875
- return;
1876
- }
1877
- }
1878
- catch (error) {
1879
- // Not in this type, continue searching
1880
- }
1881
- }
1779
+ // v6.0.0: Direct O(1) delete with ID-first path
1780
+ const path = getVerbMetadataPath(id);
1781
+ await this.deleteObjectFromBranch(path);
1882
1782
  }
1883
1783
  // ============================================================================
1884
- // TYPE-FIRST HELPER METHODS (v5.4.0)
1885
- // Built-in type-aware support for all storage adapters
1784
+ // ID-FIRST HELPER METHODS (v6.0.0)
1785
+ // Direct O(1) ID lookups - no type needed!
1786
+ // Clean, simple architecture for billion-scale performance
1886
1787
  // ============================================================================
1887
1788
  /**
1888
1789
  * Load type statistics from storage
@@ -1924,30 +1825,61 @@ export class BaseStorage extends BaseStorageAdapter {
1924
1825
  */
1925
1826
  async rebuildTypeCounts() {
1926
1827
  prodLog.info('[BaseStorage] Rebuilding type counts from storage...');
1927
- // Rebuild verb counts by checking each type directory
1928
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
1929
- const type = TypeUtils.getVerbFromIndex(i);
1930
- const prefix = `entities/verbs/${type}/vectors/`;
1828
+ // v6.0.0: Rebuild by scanning shards (0x00-0xFF) and reading metadata
1829
+ this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT);
1830
+ this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT);
1831
+ // Scan noun shards
1832
+ for (let shard = 0; shard < 256; shard++) {
1833
+ const shardHex = shard.toString(16).padStart(2, '0');
1834
+ const shardDir = `entities/nouns/${shardHex}`;
1931
1835
  try {
1932
- const paths = await this.listObjectsInBranch(prefix);
1933
- this.verbCountsByType[i] = paths.length;
1836
+ const paths = await this.listObjectsInBranch(shardDir);
1837
+ for (const path of paths) {
1838
+ if (!path.includes('/metadata.json'))
1839
+ continue;
1840
+ try {
1841
+ const metadata = await this.readWithInheritance(path);
1842
+ if (metadata && metadata.noun) {
1843
+ const typeIndex = TypeUtils.getNounIndex(metadata.noun);
1844
+ if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
1845
+ this.nounCountsByType[typeIndex]++;
1846
+ }
1847
+ }
1848
+ }
1849
+ catch (error) {
1850
+ // Skip entities that fail to load
1851
+ }
1852
+ }
1934
1853
  }
1935
1854
  catch (error) {
1936
- // Type directory doesn't exist - count is 0
1937
- this.verbCountsByType[i] = 0;
1855
+ // Skip shards that don't exist
1938
1856
  }
1939
1857
  }
1940
- // Rebuild noun counts similarly
1941
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
1942
- const type = TypeUtils.getNounFromIndex(i);
1943
- const prefix = `entities/nouns/${type}/vectors/`;
1858
+ // Scan verb shards
1859
+ for (let shard = 0; shard < 256; shard++) {
1860
+ const shardHex = shard.toString(16).padStart(2, '0');
1861
+ const shardDir = `entities/verbs/${shardHex}`;
1944
1862
  try {
1945
- const paths = await this.listObjectsInBranch(prefix);
1946
- this.nounCountsByType[i] = paths.length;
1863
+ const paths = await this.listObjectsInBranch(shardDir);
1864
+ for (const path of paths) {
1865
+ if (!path.includes('/metadata.json'))
1866
+ continue;
1867
+ try {
1868
+ const metadata = await this.readWithInheritance(path);
1869
+ if (metadata && metadata.verb) {
1870
+ const typeIndex = TypeUtils.getVerbIndex(metadata.verb);
1871
+ if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
1872
+ this.verbCountsByType[typeIndex]++;
1873
+ }
1874
+ }
1875
+ }
1876
+ catch (error) {
1877
+ // Skip entities that fail to load
1878
+ }
1879
+ }
1947
1880
  }
1948
1881
  catch (error) {
1949
- // Type directory doesn't exist - count is 0
1950
- this.nounCountsByType[i] = 0;
1882
+ // Skip shards that don't exist
1951
1883
  }
1952
1884
  }
1953
1885
  // Save rebuilt counts to storage
@@ -1957,18 +1889,13 @@ export class BaseStorage extends BaseStorageAdapter {
1957
1889
  prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
1958
1890
  }
1959
1891
  /**
1960
- * Get noun type from cache or metadata
1961
- * Relies on nounTypeCache populated during metadata saves
1892
+ * Get noun type (v6.0.0: type no longer needed for paths!)
1893
+ * With ID-first paths, this is only used for internal statistics tracking.
1894
+ * The actual type is stored in metadata and indexed by MetadataIndexManager.
1962
1895
  */
1963
1896
  getNounType(noun) {
1964
- // Check cache (populated when metadata is saved)
1965
- const cached = this.nounTypeCache.get(noun.id);
1966
- if (cached) {
1967
- return cached;
1968
- }
1969
- // Default to 'thing' if unknown
1970
- // This should only happen if saveNoun_internal is called before saveNounMetadata
1971
- prodLog.warn(`[BaseStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
1897
+ // v6.0.0: Type cache removed - default to 'thing' for statistics
1898
+ // The real type is in metadata, accessible via getNounMetadata(id)
1972
1899
  return 'thing';
1973
1900
  }
1974
1901
  /**
@@ -2051,15 +1978,14 @@ export class BaseStorage extends BaseStorageAdapter {
2051
1978
  // Converted from abstract to concrete - all adapters now have built-in type-aware
2052
1979
  // ============================================================================
2053
1980
  /**
2054
- * Save a noun to storage (type-first path)
1981
+ * Save a noun to storage (ID-first path)
2055
1982
  */
2056
1983
  async saveNoun_internal(noun) {
2057
1984
  const type = this.getNounType(noun);
2058
- const path = getNounVectorPath(type, noun.id);
1985
+ const path = getNounVectorPath(noun.id);
2059
1986
  // Update type tracking
2060
1987
  const typeIndex = TypeUtils.getNounIndex(type);
2061
1988
  this.nounCountsByType[typeIndex]++;
2062
- this.nounTypeCache.set(noun.id, type);
2063
1989
  // COW-aware write (v5.0.1): Use COW helper for branch isolation
2064
1990
  await this.writeObjectToBranch(path, noun);
2065
1991
  // Periodically save statistics (every 100 saves)
@@ -2068,120 +1994,89 @@ export class BaseStorage extends BaseStorageAdapter {
2068
1994
  }
2069
1995
  }
2070
1996
  /**
2071
- * Get a noun from storage (type-first path)
1997
+ * Get a noun from storage (ID-first path)
2072
1998
  */
2073
1999
  async getNoun_internal(id) {
2074
- // Try cache first
2075
- const cachedType = this.nounTypeCache.get(id);
2076
- if (cachedType) {
2077
- const path = getNounVectorPath(cachedType, id);
2000
+ // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
2001
+ const path = getNounVectorPath(id);
2002
+ try {
2078
2003
  // COW-aware read (v5.0.1): Use COW helper for branch isolation
2079
- const data = await this.readWithInheritance(path);
2080
- // v5.7.10: Deserialize connections Map from JSON storage format
2081
- return data ? this.deserializeNoun(data) : null;
2082
- }
2083
- // Need to search across all types (expensive, but cached after first access)
2084
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
2085
- const type = TypeUtils.getNounFromIndex(i);
2086
- const path = getNounVectorPath(type, id);
2087
- try {
2088
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2089
- const noun = await this.readWithInheritance(path);
2090
- if (noun) {
2091
- // Cache the type for next time
2092
- this.nounTypeCache.set(id, type);
2093
- // v5.7.10: Deserialize connections Map from JSON storage format
2094
- return this.deserializeNoun(noun);
2095
- }
2096
- }
2097
- catch (error) {
2098
- // Not in this type, continue searching
2004
+ const noun = await this.readWithInheritance(path);
2005
+ if (noun) {
2006
+ // v5.7.10: Deserialize connections Map from JSON storage format
2007
+ return this.deserializeNoun(noun);
2099
2008
  }
2100
2009
  }
2010
+ catch (error) {
2011
+ // Entity not found
2012
+ return null;
2013
+ }
2101
2014
  return null;
2102
2015
  }
2103
2016
  /**
2104
- * Get nouns by noun type (O(1) with type-first paths!)
2017
+ * Get nouns by noun type (v6.0.0: Shard-based iteration!)
2105
2018
  */
2106
2019
  async getNounsByNounType_internal(nounType) {
2107
- const type = nounType;
2108
- const prefix = `entities/nouns/${type}/vectors/`;
2109
- // COW-aware list (v5.0.1): Use COW helper for branch isolation
2110
- const paths = await this.listObjectsInBranch(prefix);
2111
- // Load all nouns of this type
2020
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types
2021
+ // Type is stored in metadata.noun field, we filter as we load
2112
2022
  const nouns = [];
2113
- for (const path of paths) {
2023
+ for (let shard = 0; shard < 256; shard++) {
2024
+ const shardHex = shard.toString(16).padStart(2, '0');
2025
+ const shardDir = `entities/nouns/${shardHex}`;
2114
2026
  try {
2115
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2116
- const noun = await this.readWithInheritance(path);
2117
- if (noun) {
2118
- // v5.7.10: Deserialize connections Map from JSON storage format
2119
- nouns.push(this.deserializeNoun(noun));
2120
- // Cache the type
2121
- this.nounTypeCache.set(noun.id, type);
2027
+ const nounFiles = await this.listObjectsInBranch(shardDir);
2028
+ for (const nounPath of nounFiles) {
2029
+ if (!nounPath.includes('/vectors.json'))
2030
+ continue;
2031
+ try {
2032
+ const noun = await this.readWithInheritance(nounPath);
2033
+ if (noun) {
2034
+ const deserialized = this.deserializeNoun(noun);
2035
+ // Check type from metadata
2036
+ const metadata = await this.getNounMetadata(deserialized.id);
2037
+ if (metadata && metadata.noun === nounType) {
2038
+ nouns.push(deserialized);
2039
+ }
2040
+ }
2041
+ }
2042
+ catch (error) {
2043
+ // Skip nouns that fail to load
2044
+ }
2122
2045
  }
2123
2046
  }
2124
2047
  catch (error) {
2125
- prodLog.warn(`[BaseStorage] Failed to load noun from ${path}:`, error);
2048
+ // Skip shards that have no data
2126
2049
  }
2127
2050
  }
2128
2051
  return nouns;
2129
2052
  }
2130
2053
  /**
2131
- * Delete a noun from storage (type-first path)
2054
+ * Delete a noun from storage (v6.0.0: ID-first, O(1) delete)
2132
2055
  */
2133
2056
  async deleteNoun_internal(id) {
2134
- // Try cache first
2135
- const cachedType = this.nounTypeCache.get(id);
2136
- if (cachedType) {
2137
- const path = getNounVectorPath(cachedType, id);
2138
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2139
- await this.deleteObjectFromBranch(path);
2140
- // Update counts
2141
- const typeIndex = TypeUtils.getNounIndex(cachedType);
2142
- if (this.nounCountsByType[typeIndex] > 0) {
2143
- this.nounCountsByType[typeIndex]--;
2144
- }
2145
- this.nounTypeCache.delete(id);
2146
- return;
2147
- }
2148
- // Search across all types
2149
- for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
2150
- const type = TypeUtils.getNounFromIndex(i);
2151
- const path = getNounVectorPath(type, id);
2152
- try {
2153
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2154
- await this.deleteObjectFromBranch(path);
2155
- // Update counts
2156
- if (this.nounCountsByType[i] > 0) {
2157
- this.nounCountsByType[i]--;
2158
- }
2159
- this.nounTypeCache.delete(id);
2160
- return;
2161
- }
2162
- catch (error) {
2163
- // Not in this type, continue
2164
- }
2165
- }
2057
+ // v6.0.0: Direct O(1) delete with ID-first path
2058
+ const path = getNounVectorPath(id);
2059
+ await this.deleteObjectFromBranch(path);
2060
+ // Note: Type-specific counts will be decremented via metadata tracking
2061
+ // The real type is in metadata, accessible if needed via getNounMetadata(id)
2166
2062
  }
2167
2063
  /**
2168
- * Save a verb to storage (type-first path)
2064
+ * Save a verb to storage (ID-first path)
2169
2065
  */
2170
2066
  async saveVerb_internal(verb) {
2171
2067
  // Type is now a first-class field in HNSWVerb - no caching needed!
2172
2068
  const type = verb.verb;
2173
- const path = getVerbVectorPath(type, verb.id);
2069
+ const path = getVerbVectorPath(verb.id);
2070
+ prodLog.debug(`[BaseStorage] saveVerb_internal: id=${verb.id}, sourceId=${verb.sourceId}, targetId=${verb.targetId}, type=${type}`);
2174
2071
  // Update type tracking
2175
2072
  const typeIndex = TypeUtils.getVerbIndex(type);
2176
2073
  this.verbCountsByType[typeIndex]++;
2177
- this.verbTypeCache.set(verb.id, type);
2178
2074
  // COW-aware write (v5.0.1): Use COW helper for branch isolation
2179
2075
  await this.writeObjectToBranch(path, verb);
2180
- // v5.7.0: Update GraphAdjacencyIndex incrementally for billion-scale optimization
2181
- // CRITICAL: Only update if index already initialized to avoid circular dependency
2182
- // Index is lazy-loaded on first query, then maintained incrementally
2183
- if (this.graphIndex && this.graphIndex.isInitialized) {
2184
- // Fast incremental update - no rebuild needed
2076
+ // v6.0.0: Update GraphAdjacencyIndex incrementally (always available after init())
2077
+ // GraphAdjacencyIndex.addVerb() calls ensureInitialized() automatically
2078
+ if (this.graphIndex) {
2079
+ prodLog.debug(`[BaseStorage] Updating GraphAdjacencyIndex with verb ${verb.id}`);
2185
2080
  await this.graphIndex.addVerb({
2186
2081
  id: verb.id,
2187
2082
  sourceId: verb.sourceId,
@@ -2193,8 +2088,12 @@ export class BaseStorage extends BaseStorageAdapter {
2193
2088
  type: verb.verb,
2194
2089
  createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
2195
2090
  updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
2196
- createdBy: { augmentation: 'storage', version: '5.7.0' }
2091
+ createdBy: { augmentation: 'storage', version: '6.0.0' }
2197
2092
  });
2093
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex updated successfully`);
2094
+ }
2095
+ else {
2096
+ prodLog.warn(`[BaseStorage] graphIndex is null, cannot update index for verb ${verb.id}`);
2198
2097
  }
2199
2098
  // Periodically save statistics
2200
2099
  if (this.verbCountsByType[typeIndex] % 100 === 0) {
@@ -2202,69 +2101,89 @@ export class BaseStorage extends BaseStorageAdapter {
2202
2101
  }
2203
2102
  }
2204
2103
  /**
2205
- * Get a verb from storage (type-first path)
2104
+ * Get a verb from storage (ID-first path)
2206
2105
  */
2207
2106
  async getVerb_internal(id) {
2208
- // Try cache first for O(1) retrieval
2209
- const cachedType = this.verbTypeCache.get(id);
2210
- if (cachedType) {
2211
- const path = getVerbVectorPath(cachedType, id);
2107
+ // v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
2108
+ const path = getVerbVectorPath(id);
2109
+ try {
2212
2110
  // COW-aware read (v5.0.1): Use COW helper for branch isolation
2213
2111
  const verb = await this.readWithInheritance(path);
2214
- // v5.7.10: Deserialize connections Map from JSON storage format
2215
- return verb ? this.deserializeVerb(verb) : null;
2216
- }
2217
- // Search across all types (only on first access)
2218
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2219
- const type = TypeUtils.getVerbFromIndex(i);
2220
- const path = getVerbVectorPath(type, id);
2221
- try {
2222
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2223
- const verb = await this.readWithInheritance(path);
2224
- if (verb) {
2225
- // Cache the type for next time (read from verb.verb field)
2226
- this.verbTypeCache.set(id, verb.verb);
2227
- // v5.7.10: Deserialize connections Map from JSON storage format
2228
- return this.deserializeVerb(verb);
2229
- }
2230
- }
2231
- catch (error) {
2232
- // Not in this type, continue
2112
+ if (verb) {
2113
+ // v5.7.10: Deserialize connections Map from JSON storage format
2114
+ return this.deserializeVerb(verb);
2233
2115
  }
2234
2116
  }
2117
+ catch (error) {
2118
+ // Entity not found
2119
+ return null;
2120
+ }
2235
2121
  return null;
2236
2122
  }
2237
2123
  /**
2238
- * Get verbs by source (COW-aware implementation)
2239
- * v5.4.0: Fixed to directly list verb files instead of directories
2124
+ * Get verbs by source (v6.0.0: Uses GraphAdjacencyIndex when available)
2125
+ * Falls back to shard iteration during initialization to avoid circular dependency
2240
2126
  */
2241
2127
  async getVerbsBySource_internal(sourceId) {
2242
- // v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
2243
- // v5.7.0 called getGraphIndex() here, creating deadlock during initialization:
2244
- // GraphAdjacencyIndex.rebuild() → storage.getVerbs() → getVerbsBySource_internal() → getGraphIndex() → [deadlock]
2245
- // v5.4.0: Type-first implementation - scan across all verb types
2246
- // COW-aware: uses readWithInheritance for each verb
2247
2128
  await this.ensureInitialized();
2129
+ prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`);
2130
+ // v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
2131
+ if (this.graphIndex && this.graphIndex.isInitialized) {
2132
+ try {
2133
+ const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId);
2134
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`);
2135
+ const results = [];
2136
+ for (const verbId of verbIds) {
2137
+ const verb = await this.getVerb_internal(verbId);
2138
+ const metadata = await this.getVerbMetadata(verbId);
2139
+ if (verb && metadata) {
2140
+ results.push({
2141
+ ...verb,
2142
+ weight: metadata.weight,
2143
+ confidence: metadata.confidence,
2144
+ createdAt: metadata.createdAt
2145
+ ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
2146
+ : Date.now(),
2147
+ updatedAt: metadata.updatedAt
2148
+ ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
2149
+ : Date.now(),
2150
+ service: metadata.service,
2151
+ createdBy: metadata.createdBy,
2152
+ metadata: metadata || {}
2153
+ });
2154
+ }
2155
+ }
2156
+ prodLog.debug(`[BaseStorage] GraphAdjacencyIndex path returned ${results.length} verbs`);
2157
+ return results;
2158
+ }
2159
+ catch (error) {
2160
+ prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
2161
+ }
2162
+ }
2163
+ // v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
2164
+ prodLog.debug(`[BaseStorage] Using shard iteration fallback for sourceId=${sourceId}`);
2248
2165
  const results = [];
2249
- // Iterate through all verb types
2250
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2251
- const type = TypeUtils.getVerbFromIndex(i);
2252
- const typeDir = `entities/verbs/${type}/vectors`;
2166
+ let shardsScanned = 0;
2167
+ let verbsFound = 0;
2168
+ for (let shard = 0; shard < 256; shard++) {
2169
+ const shardHex = shard.toString(16).padStart(2, '0');
2170
+ const shardDir = `entities/verbs/${shardHex}`;
2253
2171
  try {
2254
- // v5.4.0 FIX: List all verb files directly (not shard directories)
2255
- // listObjectsInBranch returns full paths to .json files, not directories
2256
- const verbFiles = await this.listObjectsInBranch(typeDir);
2172
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2173
+ shardsScanned++;
2257
2174
  for (const verbPath of verbFiles) {
2258
- // Skip if not a .json file
2259
- if (!verbPath.endsWith('.json'))
2175
+ if (!verbPath.includes('/vectors.json'))
2260
2176
  continue;
2261
2177
  try {
2262
- const verb = await this.readWithInheritance(verbPath);
2263
- if (verb && verb.sourceId === sourceId) {
2264
- // v5.4.0: Use proper path helper instead of string replacement
2265
- const metadataPath = getVerbMetadataPath(type, verb.id);
2178
+ const rawVerb = await this.readWithInheritance(verbPath);
2179
+ if (!rawVerb)
2180
+ continue;
2181
+ verbsFound++;
2182
+ // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
2183
+ const verb = this.deserializeVerb(rawVerb);
2184
+ if (verb.sourceId === sourceId) {
2185
+ const metadataPath = getVerbMetadataPath(verb.id);
2266
2186
  const metadata = await this.readWithInheritance(metadataPath);
2267
- // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
2268
2187
  results.push({
2269
2188
  ...verb,
2270
2189
  weight: metadata?.weight,
@@ -2283,13 +2202,15 @@ export class BaseStorage extends BaseStorageAdapter {
2283
2202
  }
2284
2203
  catch (error) {
2285
2204
  // Skip verbs that fail to load
2205
+ prodLog.debug(`[BaseStorage] Failed to load verb from ${verbPath}:`, error);
2286
2206
  }
2287
2207
  }
2288
2208
  }
2289
2209
  catch (error) {
2290
- // Skip types that have no data
2210
+ // Skip shards that have no data
2291
2211
  }
2292
2212
  }
2213
+ prodLog.debug(`[BaseStorage] Shard iteration: scanned ${shardsScanned} shards, found ${verbsFound} total verbs, matched ${results.length} for sourceId=${sourceId}`);
2293
2214
  return results;
2294
2215
  }
2295
2216
  /**
@@ -2335,52 +2256,46 @@ export class BaseStorage extends BaseStorageAdapter {
2335
2256
  }
2336
2257
  // Convert sourceIds to Set for O(1) lookup
2337
2258
  const sourceIdSet = new Set(sourceIds);
2338
- // Determine which verb types to scan
2339
- const typesToScan = [];
2340
- if (verbType) {
2341
- typesToScan.push(verbType);
2342
- }
2343
- else {
2344
- // Scan all verb types
2345
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2346
- typesToScan.push(TypeUtils.getVerbFromIndex(i));
2347
- }
2348
- }
2349
- // Scan verb types and collect matching verbs
2350
- for (const type of typesToScan) {
2351
- const typeDir = `entities/verbs/${type}/vectors`;
2259
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of types
2260
+ for (let shard = 0; shard < 256; shard++) {
2261
+ const shardHex = shard.toString(16).padStart(2, '0');
2262
+ const shardDir = `entities/verbs/${shardHex}`;
2352
2263
  try {
2353
- // List all verb files of this type
2354
- const verbFiles = await this.listObjectsInBranch(typeDir);
2264
+ // List all verb files in this shard
2265
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2355
2266
  // Build paths for batch read
2356
2267
  const verbPaths = [];
2357
2268
  const metadataPaths = [];
2358
2269
  const pathToId = new Map();
2359
2270
  for (const verbPath of verbFiles) {
2360
- if (!verbPath.endsWith('.json'))
2271
+ if (!verbPath.includes('/vectors.json'))
2361
2272
  continue;
2362
2273
  verbPaths.push(verbPath);
2363
- // Extract ID from path: "entities/verbs/{type}/vectors/{shard}/{id}.json"
2274
+ // Extract ID from path: "entities/verbs/{shard}/{id}/vector.json"
2364
2275
  const parts = verbPath.split('/');
2365
- const filename = parts[parts.length - 1];
2366
- const verbId = filename.replace('.json', '');
2276
+ const verbId = parts[parts.length - 2]; // ID is second-to-last segment
2367
2277
  pathToId.set(verbPath, verbId);
2368
2278
  // Prepare metadata path
2369
- metadataPaths.push(getVerbMetadataPath(type, verbId));
2279
+ metadataPaths.push(getVerbMetadataPath(verbId));
2370
2280
  }
2371
- // Batch read all verb files for this type
2281
+ // Batch read all verb files for this shard
2372
2282
  const verbDataMap = await this.readBatchWithInheritance(verbPaths);
2373
2283
  const metadataMap = await this.readBatchWithInheritance(metadataPaths);
2374
2284
  // Process results
2375
- for (const [verbPath, verbData] of verbDataMap.entries()) {
2376
- if (!verbData || !verbData.sourceId)
2285
+ for (const [verbPath, rawVerbData] of verbDataMap.entries()) {
2286
+ if (!rawVerbData || !rawVerbData.sourceId)
2377
2287
  continue;
2288
+ // v6.0.0: Deserialize connections Map from JSON storage format
2289
+ const verbData = this.deserializeVerb(rawVerbData);
2378
2290
  // Check if this verb's source is in our requested set
2379
2291
  if (!sourceIdSet.has(verbData.sourceId))
2380
2292
  continue;
2293
+ // If verbType specified, filter by type
2294
+ if (verbType && verbData.verb !== verbType)
2295
+ continue;
2381
2296
  // Found matching verb - hydrate with metadata
2382
2297
  const verbId = pathToId.get(verbPath);
2383
- const metadataPath = getVerbMetadataPath(type, verbId);
2298
+ const metadataPath = getVerbMetadataPath(verbId);
2384
2299
  const metadata = metadataMap.get(metadataPath) || {};
2385
2300
  const hydratedVerb = {
2386
2301
  ...verbData,
@@ -2402,7 +2317,7 @@ export class BaseStorage extends BaseStorageAdapter {
2402
2317
  }
2403
2318
  }
2404
2319
  catch (error) {
2405
- // Skip types that have no data
2320
+ // Skip shards that have no data
2406
2321
  }
2407
2322
  }
2408
2323
  return results;
@@ -2413,31 +2328,57 @@ export class BaseStorage extends BaseStorageAdapter {
2413
2328
  * v5.4.0: Fixed to directly list verb files instead of directories
2414
2329
  */
2415
2330
  async getVerbsByTarget_internal(targetId) {
2416
- // v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
2417
- // v5.7.0 called getGraphIndex() here, creating deadlock during initialization
2418
- // v5.4.0: Type-first implementation - scan across all verb types
2419
- // COW-aware: uses readWithInheritance for each verb
2420
2331
  await this.ensureInitialized();
2332
+ // v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
2333
+ if (this.graphIndex && this.graphIndex.isInitialized) {
2334
+ try {
2335
+ const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId);
2336
+ const results = [];
2337
+ for (const verbId of verbIds) {
2338
+ const verb = await this.getVerb_internal(verbId);
2339
+ const metadata = await this.getVerbMetadata(verbId);
2340
+ if (verb && metadata) {
2341
+ results.push({
2342
+ ...verb,
2343
+ weight: metadata.weight,
2344
+ confidence: metadata.confidence,
2345
+ createdAt: metadata.createdAt
2346
+ ? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
2347
+ : Date.now(),
2348
+ updatedAt: metadata.updatedAt
2349
+ ? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
2350
+ : Date.now(),
2351
+ service: metadata.service,
2352
+ createdBy: metadata.createdBy,
2353
+ metadata: metadata || {}
2354
+ });
2355
+ }
2356
+ }
2357
+ return results;
2358
+ }
2359
+ catch (error) {
2360
+ prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
2361
+ }
2362
+ }
2363
+ // v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
2421
2364
  const results = [];
2422
- // Iterate through all verb types
2423
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2424
- const type = TypeUtils.getVerbFromIndex(i);
2425
- const typeDir = `entities/verbs/${type}/vectors`;
2365
+ for (let shard = 0; shard < 256; shard++) {
2366
+ const shardHex = shard.toString(16).padStart(2, '0');
2367
+ const shardDir = `entities/verbs/${shardHex}`;
2426
2368
  try {
2427
- // v5.4.0 FIX: List all verb files directly (not shard directories)
2428
- // listObjectsInBranch returns full paths to .json files, not directories
2429
- const verbFiles = await this.listObjectsInBranch(typeDir);
2369
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2430
2370
  for (const verbPath of verbFiles) {
2431
- // Skip if not a .json file
2432
- if (!verbPath.endsWith('.json'))
2371
+ if (!verbPath.includes('/vectors.json'))
2433
2372
  continue;
2434
2373
  try {
2435
- const verb = await this.readWithInheritance(verbPath);
2436
- if (verb && verb.targetId === targetId) {
2437
- // v5.4.0: Use proper path helper instead of string replacement
2438
- const metadataPath = getVerbMetadataPath(type, verb.id);
2374
+ const rawVerb = await this.readWithInheritance(verbPath);
2375
+ if (!rawVerb)
2376
+ continue;
2377
+ // v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
2378
+ const verb = this.deserializeVerb(rawVerb);
2379
+ if (verb.targetId === targetId) {
2380
+ const metadataPath = getVerbMetadataPath(verb.id);
2439
2381
  const metadata = await this.readWithInheritance(metadataPath);
2440
- // v5.4.0: Extract standard fields from metadata to top-level (like nouns)
2441
2382
  results.push({
2442
2383
  ...verb,
2443
2384
  weight: metadata?.weight,
@@ -2460,95 +2401,77 @@ export class BaseStorage extends BaseStorageAdapter {
2460
2401
  }
2461
2402
  }
2462
2403
  catch (error) {
2463
- // Skip types that have no data
2404
+ // Skip shards that have no data
2464
2405
  }
2465
2406
  }
2466
2407
  return results;
2467
2408
  }
2468
2409
  /**
2469
- * Get verbs by type (O(1) with type-first paths!)
2410
+ * Get verbs by type (v6.0.0: Shard iteration with type filtering)
2470
2411
  */
2471
2412
  async getVerbsByType_internal(verbType) {
2472
- const type = verbType;
2473
- const prefix = `entities/verbs/${type}/vectors/`;
2474
- // COW-aware list (v5.0.1): Use COW helper for branch isolation
2475
- const paths = await this.listObjectsInBranch(prefix);
2413
+ // v6.0.0: Iterate by shards (0x00-0xFF) instead of type-first paths
2476
2414
  const verbs = [];
2477
- for (const path of paths) {
2415
+ for (let shard = 0; shard < 256; shard++) {
2416
+ const shardHex = shard.toString(16).padStart(2, '0');
2417
+ const shardDir = `entities/verbs/${shardHex}`;
2478
2418
  try {
2479
- // COW-aware read (v5.0.1): Use COW helper for branch isolation
2480
- const rawVerb = await this.readWithInheritance(path);
2481
- if (!rawVerb)
2482
- continue;
2483
- // v5.7.10: Deserialize connections Map from JSON storage format
2484
- // Replaces v5.7.8 manual deserialization (lines 2599-2605)
2485
- const hnswVerb = this.deserializeVerb(rawVerb);
2486
- // Cache type from HNSWVerb for future O(1) retrievals
2487
- this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb);
2488
- // Load metadata separately (optional in v4.0.0!)
2489
- // FIX: Don't skip verbs without metadata - metadata is optional!
2490
- const metadata = await this.getVerbMetadata(hnswVerb.id);
2491
- // v4.8.0: Extract standard fields from metadata to top-level
2492
- const metadataObj = (metadata || {});
2493
- const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
2494
- const verbWithMetadata = {
2495
- id: hnswVerb.id,
2496
- vector: [...hnswVerb.vector],
2497
- connections: hnswVerb.connections, // v5.7.10: Already deserialized
2498
- verb: hnswVerb.verb,
2499
- sourceId: hnswVerb.sourceId,
2500
- targetId: hnswVerb.targetId,
2501
- createdAt: createdAt || Date.now(),
2502
- updatedAt: updatedAt || Date.now(),
2503
- confidence: confidence,
2504
- weight: weight,
2505
- service: service,
2506
- data: data,
2507
- createdBy,
2508
- metadata: customMetadata
2509
- };
2510
- verbs.push(verbWithMetadata);
2419
+ const verbFiles = await this.listObjectsInBranch(shardDir);
2420
+ for (const verbPath of verbFiles) {
2421
+ if (!verbPath.includes('/vectors.json'))
2422
+ continue;
2423
+ try {
2424
+ const rawVerb = await this.readWithInheritance(verbPath);
2425
+ if (!rawVerb)
2426
+ continue;
2427
+ // v5.7.10: Deserialize connections Map from JSON storage format
2428
+ const hnswVerb = this.deserializeVerb(rawVerb);
2429
+ // Filter by verb type
2430
+ if (hnswVerb.verb !== verbType)
2431
+ continue;
2432
+ // Load metadata separately (optional in v4.0.0!)
2433
+ const metadata = await this.getVerbMetadata(hnswVerb.id);
2434
+ // v4.8.0: Extract standard fields from metadata to top-level
2435
+ const metadataObj = (metadata || {});
2436
+ const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
2437
+ const verbWithMetadata = {
2438
+ id: hnswVerb.id,
2439
+ vector: [...hnswVerb.vector],
2440
+ connections: hnswVerb.connections, // v5.7.10: Already deserialized
2441
+ verb: hnswVerb.verb,
2442
+ sourceId: hnswVerb.sourceId,
2443
+ targetId: hnswVerb.targetId,
2444
+ createdAt: createdAt || Date.now(),
2445
+ updatedAt: updatedAt || Date.now(),
2446
+ confidence: confidence,
2447
+ weight: weight,
2448
+ service: service,
2449
+ data: data,
2450
+ createdBy,
2451
+ metadata: customMetadata
2452
+ };
2453
+ verbs.push(verbWithMetadata);
2454
+ }
2455
+ catch (error) {
2456
+ // Skip verbs that fail to load
2457
+ }
2458
+ }
2511
2459
  }
2512
2460
  catch (error) {
2513
- prodLog.warn(`[BaseStorage] Failed to load verb from ${path}:`, error);
2461
+ // Skip shards that have no data
2514
2462
  }
2515
2463
  }
2516
2464
  return verbs;
2517
2465
  }
2518
2466
  /**
2519
- * Delete a verb from storage (type-first path)
2467
+ * Delete a verb from storage (v6.0.0: ID-first, O(1) delete)
2520
2468
  */
2521
2469
  async deleteVerb_internal(id) {
2522
- // Try cache first
2523
- const cachedType = this.verbTypeCache.get(id);
2524
- if (cachedType) {
2525
- const path = getVerbVectorPath(cachedType, id);
2526
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2527
- await this.deleteObjectFromBranch(path);
2528
- const typeIndex = TypeUtils.getVerbIndex(cachedType);
2529
- if (this.verbCountsByType[typeIndex] > 0) {
2530
- this.verbCountsByType[typeIndex]--;
2531
- }
2532
- this.verbTypeCache.delete(id);
2533
- return;
2534
- }
2535
- // Search across all types
2536
- for (let i = 0; i < VERB_TYPE_COUNT; i++) {
2537
- const type = TypeUtils.getVerbFromIndex(i);
2538
- const path = getVerbVectorPath(type, id);
2539
- try {
2540
- // COW-aware delete (v5.0.1): Use COW helper for branch isolation
2541
- await this.deleteObjectFromBranch(path);
2542
- if (this.verbCountsByType[i] > 0) {
2543
- this.verbCountsByType[i]--;
2544
- }
2545
- this.verbTypeCache.delete(id);
2546
- return;
2547
- }
2548
- catch (error) {
2549
- // Continue
2550
- }
2551
- }
2470
+ // v6.0.0: Direct O(1) delete with ID-first path
2471
+ const path = getVerbVectorPath(id);
2472
+ await this.deleteObjectFromBranch(path);
2473
+ // Note: Type-specific counts will be decremented via metadata tracking
2474
+ // The real type is in metadata, accessible if needed via getVerbMetadata(id)
2552
2475
  }
2553
2476
  /**
2554
2477
  * Helper method to convert a Map to a plain object for serialization