@soulcraft/brainy 5.12.0 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +180 -5
- package/README.md +2 -6
- package/dist/api/DataAPI.d.ts +0 -40
- package/dist/api/DataAPI.js +0 -235
- package/dist/brainy.d.ts +0 -106
- package/dist/brainy.js +0 -370
- package/dist/cli/commands/cow.d.ts +1 -9
- package/dist/cli/commands/cow.js +1 -61
- package/dist/cli/commands/data.d.ts +1 -13
- package/dist/cli/commands/data.js +1 -74
- package/dist/cli/index.js +1 -16
- package/dist/neural/embeddedTypeEmbeddings.d.ts +1 -1
- package/dist/neural/embeddedTypeEmbeddings.js +2 -2
- package/dist/storage/adapters/azureBlobStorage.js +2 -1
- package/dist/storage/adapters/fileSystemStorage.js +2 -1
- package/dist/storage/adapters/gcsStorage.js +2 -1
- package/dist/storage/adapters/historicalStorageAdapter.js +2 -2
- package/dist/storage/adapters/memoryStorage.d.ts +1 -1
- package/dist/storage/adapters/memoryStorage.js +9 -11
- package/dist/storage/adapters/opfsStorage.js +2 -1
- package/dist/storage/adapters/r2Storage.js +2 -1
- package/dist/storage/adapters/s3CompatibleStorage.js +2 -1
- package/dist/storage/baseStorage.d.ts +40 -24
- package/dist/storage/baseStorage.js +490 -557
- package/dist/vfs/VirtualFileSystem.d.ts +46 -24
- package/dist/vfs/VirtualFileSystem.js +153 -146
- package/package.json +1 -1
|
@@ -39,32 +39,36 @@ export function getDirectoryPath(entityType, dataType) {
|
|
|
39
39
|
* Built-in type-aware organization for all storage adapters
|
|
40
40
|
*/
|
|
41
41
|
/**
|
|
42
|
-
* Get
|
|
42
|
+
* Get ID-first path for noun vectors (v6.0.0)
|
|
43
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
43
44
|
*/
|
|
44
|
-
function getNounVectorPath(
|
|
45
|
+
function getNounVectorPath(id) {
|
|
45
46
|
const shard = getShardIdFromUuid(id);
|
|
46
|
-
return `entities/nouns/${
|
|
47
|
+
return `entities/nouns/${shard}/${id}/vectors.json`;
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
49
|
-
* Get
|
|
50
|
+
* Get ID-first path for noun metadata (v6.0.0)
|
|
51
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
50
52
|
*/
|
|
51
|
-
function getNounMetadataPath(
|
|
53
|
+
function getNounMetadataPath(id) {
|
|
52
54
|
const shard = getShardIdFromUuid(id);
|
|
53
|
-
return `entities/nouns/${
|
|
55
|
+
return `entities/nouns/${shard}/${id}/metadata.json`;
|
|
54
56
|
}
|
|
55
57
|
/**
|
|
56
|
-
* Get
|
|
58
|
+
* Get ID-first path for verb vectors (v6.0.0)
|
|
59
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
57
60
|
*/
|
|
58
|
-
function getVerbVectorPath(
|
|
61
|
+
function getVerbVectorPath(id) {
|
|
59
62
|
const shard = getShardIdFromUuid(id);
|
|
60
|
-
return `entities/verbs/${
|
|
63
|
+
return `entities/verbs/${shard}/${id}/vectors.json`;
|
|
61
64
|
}
|
|
62
65
|
/**
|
|
63
|
-
* Get
|
|
66
|
+
* Get ID-first path for verb metadata (v6.0.0)
|
|
67
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
64
68
|
*/
|
|
65
|
-
function getVerbMetadataPath(
|
|
69
|
+
function getVerbMetadataPath(id) {
|
|
66
70
|
const shard = getShardIdFromUuid(id);
|
|
67
|
-
return `entities/verbs/${
|
|
71
|
+
return `entities/verbs/${shard}/${id}/metadata.json`;
|
|
68
72
|
}
|
|
69
73
|
/**
|
|
70
74
|
* Base storage adapter that implements common functionality
|
|
@@ -89,9 +93,9 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
89
93
|
this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
|
|
90
94
|
this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 508 bytes (Stage 3: 127 types)
|
|
91
95
|
// Total: 676 bytes (99.2% reduction vs Map-based tracking)
|
|
92
|
-
// Type
|
|
93
|
-
|
|
94
|
-
|
|
96
|
+
// v6.0.0: Type caches REMOVED - ID-first paths eliminate need for type lookups!
|
|
97
|
+
// With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json
|
|
98
|
+
// Type is just a field in the metadata, indexed by MetadataIndexManager for queries
|
|
95
99
|
// v5.5.0: Track if type counts have been rebuilt (prevent repeated rebuilds)
|
|
96
100
|
this.typeCountsRebuilt = false;
|
|
97
101
|
}
|
|
@@ -174,9 +178,44 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
174
178
|
* IMPORTANT: If your adapter overrides init(), call await super.init() first!
|
|
175
179
|
*/
|
|
176
180
|
async init() {
|
|
177
|
-
//
|
|
178
|
-
|
|
181
|
+
// v6.0.1: CRITICAL FIX - Set flag FIRST to prevent infinite recursion
|
|
182
|
+
// If any code path during initialization calls ensureInitialized(), it would
|
|
183
|
+
// trigger init() again. Setting the flag immediately breaks the recursion cycle.
|
|
179
184
|
this.isInitialized = true;
|
|
185
|
+
try {
|
|
186
|
+
// Load type statistics from storage (if they exist)
|
|
187
|
+
await this.loadTypeStatistics();
|
|
188
|
+
// v6.0.0: Create GraphAdjacencyIndex (lazy-loaded, no rebuild)
|
|
189
|
+
// LSM-trees are initialized on first use via ensureInitialized()
|
|
190
|
+
// Index is populated incrementally as verbs are added via addVerb()
|
|
191
|
+
try {
|
|
192
|
+
prodLog.debug('[BaseStorage] Creating GraphAdjacencyIndex...');
|
|
193
|
+
this.graphIndex = new GraphAdjacencyIndex(this);
|
|
194
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex instantiated (lazy-loaded), graphIndex=${!!this.graphIndex}`);
|
|
195
|
+
}
|
|
196
|
+
catch (error) {
|
|
197
|
+
prodLog.error('[BaseStorage] Failed to create GraphAdjacencyIndex:', error);
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
// Reset flag on failure to allow retry
|
|
203
|
+
this.isInitialized = false;
|
|
204
|
+
throw error;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Rebuild GraphAdjacencyIndex from existing verbs (v6.0.0)
|
|
209
|
+
* Call this manually if you have existing verb data that needs to be indexed
|
|
210
|
+
* @public
|
|
211
|
+
*/
|
|
212
|
+
async rebuildGraphIndex() {
|
|
213
|
+
if (!this.graphIndex) {
|
|
214
|
+
throw new Error('GraphAdjacencyIndex not initialized');
|
|
215
|
+
}
|
|
216
|
+
prodLog.info('[BaseStorage] Rebuilding graph index from existing data...');
|
|
217
|
+
await this.graphIndex.rebuild();
|
|
218
|
+
prodLog.info('[BaseStorage] Graph index rebuild complete');
|
|
180
219
|
}
|
|
181
220
|
/**
|
|
182
221
|
* Ensure the storage adapter is initialized
|
|
@@ -851,60 +890,44 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
851
890
|
*/
|
|
852
891
|
async getNounsWithPagination(options) {
|
|
853
892
|
await this.ensureInitialized();
|
|
854
|
-
const { limit, offset = 0, filter } = options;
|
|
893
|
+
const { limit, offset = 0, filter } = options;
|
|
855
894
|
const collectedNouns = [];
|
|
856
|
-
const targetCount = offset + limit;
|
|
857
|
-
//
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
for (let i = 0; i < NOUN_TYPE_COUNT && collectedNouns.length < targetCount; i++) {
|
|
862
|
-
// OPTIMIZATION 1: Skip empty types (only if counts are reliable)
|
|
863
|
-
if (useOptimization && this.nounCountsByType[i] === 0) {
|
|
864
|
-
continue;
|
|
865
|
-
}
|
|
866
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
867
|
-
// If filtering by type, skip other types
|
|
868
|
-
if (filter?.nounType) {
|
|
869
|
-
const filterTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
|
870
|
-
if (!filterTypes.includes(type)) {
|
|
871
|
-
continue;
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
const typeDir = `entities/nouns/${type}/vectors`;
|
|
895
|
+
const targetCount = offset + limit;
|
|
896
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types
|
|
897
|
+
for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) {
|
|
898
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
899
|
+
const shardDir = `entities/nouns/${shardHex}`;
|
|
875
900
|
try {
|
|
876
|
-
|
|
877
|
-
const nounFiles = await this.listObjectsInBranch(typeDir);
|
|
901
|
+
const nounFiles = await this.listObjectsInBranch(shardDir);
|
|
878
902
|
for (const nounPath of nounFiles) {
|
|
879
|
-
|
|
880
|
-
if (collectedNouns.length >= targetCount) {
|
|
903
|
+
if (collectedNouns.length >= targetCount)
|
|
881
904
|
break;
|
|
882
|
-
|
|
883
|
-
// Skip if not a .json file
|
|
884
|
-
if (!nounPath.endsWith('.json'))
|
|
905
|
+
if (!nounPath.includes('/vectors.json'))
|
|
885
906
|
continue;
|
|
886
907
|
try {
|
|
887
|
-
const
|
|
888
|
-
if (
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
const noun = this.deserializeNoun(rawNoun);
|
|
892
|
-
// Load metadata
|
|
893
|
-
const metadataPath = getNounMetadataPath(type, noun.id);
|
|
894
|
-
const metadata = await this.readWithInheritance(metadataPath);
|
|
908
|
+
const noun = await this.readWithInheritance(nounPath);
|
|
909
|
+
if (noun) {
|
|
910
|
+
const deserialized = this.deserializeNoun(noun);
|
|
911
|
+
const metadata = await this.getNounMetadata(deserialized.id);
|
|
895
912
|
if (metadata) {
|
|
896
|
-
// Apply
|
|
913
|
+
// Apply type filter
|
|
914
|
+
if (filter?.nounType && metadata.noun) {
|
|
915
|
+
const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
|
916
|
+
if (!types.includes(metadata.noun)) {
|
|
917
|
+
continue;
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
// Apply service filter
|
|
897
921
|
if (filter?.service) {
|
|
898
922
|
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
|
899
923
|
if (metadata.service && !services.includes(metadata.service)) {
|
|
900
924
|
continue;
|
|
901
925
|
}
|
|
902
926
|
}
|
|
903
|
-
// Combine noun + metadata
|
|
927
|
+
// Combine noun + metadata
|
|
904
928
|
collectedNouns.push({
|
|
905
|
-
...
|
|
906
|
-
|
|
907
|
-
type: metadata.noun || type, // Required: Extract type from metadata
|
|
929
|
+
...deserialized,
|
|
930
|
+
type: (metadata.noun || 'thing'),
|
|
908
931
|
confidence: metadata.confidence,
|
|
909
932
|
weight: metadata.weight,
|
|
910
933
|
createdAt: metadata.createdAt
|
|
@@ -927,15 +950,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
927
950
|
}
|
|
928
951
|
}
|
|
929
952
|
catch (error) {
|
|
930
|
-
// Skip
|
|
953
|
+
// Skip shards that have no data
|
|
931
954
|
}
|
|
932
955
|
}
|
|
933
|
-
// Apply pagination
|
|
956
|
+
// Apply pagination
|
|
934
957
|
const paginatedNouns = collectedNouns.slice(offset, offset + limit);
|
|
935
|
-
const hasMore = collectedNouns.length > targetCount;
|
|
958
|
+
const hasMore = collectedNouns.length > targetCount;
|
|
936
959
|
return {
|
|
937
960
|
items: paginatedNouns,
|
|
938
|
-
totalCount: collectedNouns.length,
|
|
961
|
+
totalCount: collectedNouns.length,
|
|
939
962
|
hasMore,
|
|
940
963
|
nextCursor: hasMore && paginatedNouns.length > 0
|
|
941
964
|
? paginatedNouns[paginatedNouns.length - 1].id
|
|
@@ -962,58 +985,70 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
962
985
|
const { limit, offset = 0, filter } = options; // cursor intentionally not extracted (not yet implemented)
|
|
963
986
|
const collectedVerbs = [];
|
|
964
987
|
const targetCount = offset + limit; // Early termination target
|
|
965
|
-
//
|
|
966
|
-
const
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
continue;
|
|
980
|
-
}
|
|
981
|
-
}
|
|
988
|
+
// Prepare filter sets for efficient lookup
|
|
989
|
+
const filterVerbTypes = filter?.verbType
|
|
990
|
+
? new Set(Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType])
|
|
991
|
+
: null;
|
|
992
|
+
const filterSourceIds = filter?.sourceId
|
|
993
|
+
? new Set(Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId])
|
|
994
|
+
: null;
|
|
995
|
+
const filterTargetIds = filter?.targetId
|
|
996
|
+
? new Set(Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId])
|
|
997
|
+
: null;
|
|
998
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types - single pass!
|
|
999
|
+
for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
|
|
1000
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
1001
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
982
1002
|
try {
|
|
983
|
-
const
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
// OPTIMIZATION 2: Early termination (stop when we have enough)
|
|
987
|
-
if (collectedVerbs.length >= targetCount) {
|
|
1003
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
1004
|
+
for (const verbPath of verbFiles) {
|
|
1005
|
+
if (collectedVerbs.length >= targetCount)
|
|
988
1006
|
break;
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
if (
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1007
|
+
if (!verbPath.includes('/vectors.json'))
|
|
1008
|
+
continue;
|
|
1009
|
+
try {
|
|
1010
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
1011
|
+
if (!rawVerb)
|
|
1012
|
+
continue;
|
|
1013
|
+
// v6.0.0: Deserialize connections Map from JSON storage format
|
|
1014
|
+
const verb = this.deserializeVerb(rawVerb);
|
|
1015
|
+
// Apply type filter
|
|
1016
|
+
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
|
|
1017
|
+
continue;
|
|
1000
1018
|
}
|
|
1001
|
-
//
|
|
1002
|
-
if (
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
}
|
|
1019
|
+
// Apply sourceId filter
|
|
1020
|
+
if (filterSourceIds && !filterSourceIds.has(verb.sourceId)) {
|
|
1021
|
+
continue;
|
|
1022
|
+
}
|
|
1023
|
+
// Apply targetId filter
|
|
1024
|
+
if (filterTargetIds && !filterTargetIds.has(verb.targetId)) {
|
|
1025
|
+
continue;
|
|
1009
1026
|
}
|
|
1027
|
+
// Load metadata
|
|
1028
|
+
const metadata = await this.getVerbMetadata(verb.id);
|
|
1029
|
+
// Combine verb + metadata
|
|
1030
|
+
collectedVerbs.push({
|
|
1031
|
+
...verb,
|
|
1032
|
+
weight: metadata?.weight,
|
|
1033
|
+
confidence: metadata?.confidence,
|
|
1034
|
+
createdAt: metadata?.createdAt
|
|
1035
|
+
? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
|
|
1036
|
+
: Date.now(),
|
|
1037
|
+
updatedAt: metadata?.updatedAt
|
|
1038
|
+
? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
|
|
1039
|
+
: Date.now(),
|
|
1040
|
+
service: metadata?.service,
|
|
1041
|
+
createdBy: metadata?.createdBy,
|
|
1042
|
+
metadata: metadata || {}
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
catch (error) {
|
|
1046
|
+
// Skip verbs that fail to load
|
|
1010
1047
|
}
|
|
1011
|
-
// Verb passed all filters - add to collection
|
|
1012
|
-
collectedVerbs.push(verb);
|
|
1013
1048
|
}
|
|
1014
1049
|
}
|
|
1015
1050
|
catch (error) {
|
|
1016
|
-
// Skip
|
|
1051
|
+
// Skip shards that have no data
|
|
1017
1052
|
}
|
|
1018
1053
|
}
|
|
1019
1054
|
// Apply pagination (v5.5.0: Efficient slicing after early termination)
|
|
@@ -1386,11 +1421,8 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1386
1421
|
*/
|
|
1387
1422
|
async saveNounMetadata_internal(id, metadata) {
|
|
1388
1423
|
await this.ensureInitialized();
|
|
1389
|
-
//
|
|
1390
|
-
const
|
|
1391
|
-
this.nounTypeCache.set(id, type);
|
|
1392
|
-
// v5.4.0: Use type-first path
|
|
1393
|
-
const path = getNounMetadataPath(type, id);
|
|
1424
|
+
// v6.0.0: ID-first path - no type needed!
|
|
1425
|
+
const path = getNounMetadataPath(id);
|
|
1394
1426
|
// Determine if this is a new entity by checking if metadata already exists
|
|
1395
1427
|
const existingMetadata = await this.readWithInheritance(path);
|
|
1396
1428
|
const isNew = !existingMetadata;
|
|
@@ -1411,6 +1443,17 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1411
1443
|
/**
|
|
1412
1444
|
* Get noun metadata from storage (METADATA-ONLY, NO VECTORS)
|
|
1413
1445
|
*
|
|
1446
|
+
* **Performance (v6.0.0)**: Direct O(1) ID-first lookup - NO type search needed!
|
|
1447
|
+
* - **All lookups**: 1 read, ~500ms on cloud (consistent performance)
|
|
1448
|
+
* - **No cache needed**: Type is in the metadata, not the path
|
|
1449
|
+
* - **No type search**: ID-first paths eliminate 42-type search entirely
|
|
1450
|
+
*
|
|
1451
|
+
* **Clean architecture (v6.0.0)**:
|
|
1452
|
+
* - Path: `entities/nouns/{SHARD}/{ID}/metadata.json`
|
|
1453
|
+
* - Type is just a field in metadata (`noun: "document"`)
|
|
1454
|
+
* - MetadataIndex handles type queries (no path scanning needed)
|
|
1455
|
+
* - Scales to billions without any overhead
|
|
1456
|
+
*
|
|
1414
1457
|
* **Performance (v5.11.1)**: Fast path for metadata-only reads
|
|
1415
1458
|
* - **Speed**: 10ms vs 43ms (76-81% faster than getNoun)
|
|
1416
1459
|
* - **Bandwidth**: 300 bytes vs 6KB (95% less)
|
|
@@ -1440,39 +1483,21 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1440
1483
|
* @returns Metadata or null if not found
|
|
1441
1484
|
*
|
|
1442
1485
|
* @performance
|
|
1443
|
-
* -
|
|
1444
|
-
* -
|
|
1445
|
-
* -
|
|
1486
|
+
* - O(1) direct ID lookup - always 1 read (~500ms on cloud, ~10ms local)
|
|
1487
|
+
* - No caching complexity
|
|
1488
|
+
* - No type search fallbacks
|
|
1489
|
+
* - Works in distributed systems without sync issues
|
|
1446
1490
|
*
|
|
1447
1491
|
* @since v4.0.0
|
|
1448
|
-
* @since v5.4.0 - Type-first paths
|
|
1492
|
+
* @since v5.4.0 - Type-first paths (removed in v6.0.0)
|
|
1449
1493
|
* @since v5.11.1 - Promoted to fast path for brain.get() optimization
|
|
1494
|
+
* @since v6.0.0 - CLEAN FIX: ID-first paths eliminate all type-search complexity
|
|
1450
1495
|
*/
|
|
1451
1496
|
async getNounMetadata(id) {
|
|
1452
1497
|
await this.ensureInitialized();
|
|
1453
|
-
//
|
|
1454
|
-
const
|
|
1455
|
-
|
|
1456
|
-
const path = getNounMetadataPath(cachedType, id);
|
|
1457
|
-
return this.readWithInheritance(path);
|
|
1458
|
-
}
|
|
1459
|
-
// Fallback: search across all types (expensive but necessary if cache miss)
|
|
1460
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1461
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1462
|
-
const path = getNounMetadataPath(type, id);
|
|
1463
|
-
try {
|
|
1464
|
-
const metadata = await this.readWithInheritance(path);
|
|
1465
|
-
if (metadata) {
|
|
1466
|
-
// Cache the type for next time
|
|
1467
|
-
this.nounTypeCache.set(id, type);
|
|
1468
|
-
return metadata;
|
|
1469
|
-
}
|
|
1470
|
-
}
|
|
1471
|
-
catch (error) {
|
|
1472
|
-
// Not in this type, continue searching
|
|
1473
|
-
}
|
|
1474
|
-
}
|
|
1475
|
-
return null;
|
|
1498
|
+
// v6.0.0: Clean, simple, O(1) lookup - no type needed!
|
|
1499
|
+
const path = getNounMetadataPath(id);
|
|
1500
|
+
return this.readWithInheritance(path);
|
|
1476
1501
|
}
|
|
1477
1502
|
/**
|
|
1478
1503
|
* Batch fetch noun metadata from storage (v5.12.0 - Cloud Storage Optimization)
|
|
@@ -1511,74 +1536,19 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1511
1536
|
const results = new Map();
|
|
1512
1537
|
if (ids.length === 0)
|
|
1513
1538
|
return results;
|
|
1514
|
-
//
|
|
1515
|
-
|
|
1516
|
-
const
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
idsForType.push(id);
|
|
1522
|
-
idsByType.set(cachedType, idsForType);
|
|
1523
|
-
}
|
|
1524
|
-
else {
|
|
1525
|
-
uncachedIds.push(id);
|
|
1526
|
-
}
|
|
1527
|
-
}
|
|
1528
|
-
// Build paths for known types
|
|
1529
|
-
const pathsToFetch = [];
|
|
1530
|
-
for (const [type, typeIds] of idsByType.entries()) {
|
|
1531
|
-
for (const id of typeIds) {
|
|
1532
|
-
pathsToFetch.push({
|
|
1533
|
-
path: getNounMetadataPath(type, id),
|
|
1534
|
-
id
|
|
1535
|
-
});
|
|
1536
|
-
}
|
|
1537
|
-
}
|
|
1538
|
-
// For uncached IDs, we need to search across types (expensive but unavoidable)
|
|
1539
|
-
// Strategy: Try most common types first (Document, Thing, Person), then others
|
|
1540
|
-
const commonTypes = [NounType.Document, NounType.Thing, NounType.Person, NounType.File];
|
|
1541
|
-
const commonTypeSet = new Set(commonTypes);
|
|
1542
|
-
const otherTypes = [];
|
|
1543
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1544
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1545
|
-
if (!commonTypeSet.has(type)) {
|
|
1546
|
-
otherTypes.push(type);
|
|
1547
|
-
}
|
|
1548
|
-
}
|
|
1549
|
-
const searchOrder = [...commonTypes, ...otherTypes];
|
|
1550
|
-
for (const id of uncachedIds) {
|
|
1551
|
-
for (const type of searchOrder) {
|
|
1552
|
-
// Build path manually to avoid type issues
|
|
1553
|
-
const shard = getShardIdFromUuid(id);
|
|
1554
|
-
const path = `entities/nouns/${type}/metadata/${shard}/${id}.json`;
|
|
1555
|
-
pathsToFetch.push({ path, id });
|
|
1556
|
-
}
|
|
1557
|
-
}
|
|
1558
|
-
// Batch read all paths
|
|
1539
|
+
// v6.0.0: ID-first paths - no type grouping or search needed!
|
|
1540
|
+
// Build direct paths for all IDs
|
|
1541
|
+
const pathsToFetch = ids.map(id => ({
|
|
1542
|
+
path: getNounMetadataPath(id),
|
|
1543
|
+
id
|
|
1544
|
+
}));
|
|
1545
|
+
// Batch read all paths (uses adapter's native batch API or parallel fallback)
|
|
1559
1546
|
const batchResults = await this.readBatchWithInheritance(pathsToFetch.map(p => p.path));
|
|
1560
|
-
//
|
|
1561
|
-
const
|
|
1562
|
-
for (let i = 0; i < pathsToFetch.length; i++) {
|
|
1563
|
-
const { path, id } = pathsToFetch[i];
|
|
1547
|
+
// Map results back to IDs
|
|
1548
|
+
for (const { path, id } of pathsToFetch) {
|
|
1564
1549
|
const metadata = batchResults.get(path);
|
|
1565
1550
|
if (metadata) {
|
|
1566
1551
|
results.set(id, metadata);
|
|
1567
|
-
// Cache the type for uncached IDs (only on first find)
|
|
1568
|
-
if (uncachedIds.includes(id) && !foundUncached.has(id)) {
|
|
1569
|
-
// Extract type from path: "entities/nouns/metadata/{type}/{shard}/{id}.json"
|
|
1570
|
-
const parts = path.split('/');
|
|
1571
|
-
const typeStr = parts[3]; // "document", "thing", etc.
|
|
1572
|
-
// Find matching type by string comparison
|
|
1573
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1574
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1575
|
-
if (type === typeStr) {
|
|
1576
|
-
this.nounTypeCache.set(id, type);
|
|
1577
|
-
break;
|
|
1578
|
-
}
|
|
1579
|
-
}
|
|
1580
|
-
foundUncached.add(id);
|
|
1581
|
-
}
|
|
1582
1552
|
}
|
|
1583
1553
|
}
|
|
1584
1554
|
return results;
|
|
@@ -1736,36 +1706,13 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1736
1706
|
};
|
|
1737
1707
|
}
|
|
1738
1708
|
/**
|
|
1739
|
-
* Delete noun metadata from storage
|
|
1740
|
-
* v5.4.0: Uses type-first paths (must match saveNounMetadata_internal)
|
|
1709
|
+
* Delete noun metadata from storage (v6.0.0: ID-first, O(1) delete)
|
|
1741
1710
|
*/
|
|
1742
1711
|
async deleteNounMetadata(id) {
|
|
1743
1712
|
await this.ensureInitialized();
|
|
1744
|
-
//
|
|
1745
|
-
const
|
|
1746
|
-
|
|
1747
|
-
const path = getNounMetadataPath(cachedType, id);
|
|
1748
|
-
await this.deleteObjectFromBranch(path);
|
|
1749
|
-
// Remove from cache after deletion
|
|
1750
|
-
this.nounTypeCache.delete(id);
|
|
1751
|
-
return;
|
|
1752
|
-
}
|
|
1753
|
-
// If not in cache, search all types to find and delete
|
|
1754
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1755
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1756
|
-
const path = getNounMetadataPath(type, id);
|
|
1757
|
-
try {
|
|
1758
|
-
// Check if exists before deleting
|
|
1759
|
-
const exists = await this.readWithInheritance(path);
|
|
1760
|
-
if (exists) {
|
|
1761
|
-
await this.deleteObjectFromBranch(path);
|
|
1762
|
-
return;
|
|
1763
|
-
}
|
|
1764
|
-
}
|
|
1765
|
-
catch (error) {
|
|
1766
|
-
// Not in this type, continue searching
|
|
1767
|
-
}
|
|
1768
|
-
}
|
|
1713
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
1714
|
+
const path = getNounMetadataPath(id);
|
|
1715
|
+
await this.deleteObjectFromBranch(path);
|
|
1769
1716
|
}
|
|
1770
1717
|
/**
|
|
1771
1718
|
* Save verb metadata to storage (v4.0.0: now typed)
|
|
@@ -1777,7 +1724,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1777
1724
|
}
|
|
1778
1725
|
/**
|
|
1779
1726
|
* Internal method for saving verb metadata (v4.0.0: now typed)
|
|
1780
|
-
* v5.4.0: Uses
|
|
1727
|
+
* v5.4.0: Uses ID-first paths (must match getVerbMetadata)
|
|
1781
1728
|
*
|
|
1782
1729
|
* CRITICAL (v4.1.2): Count synchronization happens here
|
|
1783
1730
|
* This ensures verb counts are updated AFTER metadata exists, fixing the race condition
|
|
@@ -1789,7 +1736,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1789
1736
|
*/
|
|
1790
1737
|
async saveVerbMetadata_internal(id, metadata) {
|
|
1791
1738
|
await this.ensureInitialized();
|
|
1792
|
-
// v5.4.0: Extract verb type from metadata for
|
|
1739
|
+
// v5.4.0: Extract verb type from metadata for ID-first path
|
|
1793
1740
|
const verbType = metadata.verb;
|
|
1794
1741
|
if (!verbType) {
|
|
1795
1742
|
// Backward compatibility: fallback to old path if no verb type
|
|
@@ -1797,15 +1744,14 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1797
1744
|
await this.writeObjectToBranch(keyInfo.fullPath, metadata);
|
|
1798
1745
|
return;
|
|
1799
1746
|
}
|
|
1800
|
-
// v5.4.0: Use
|
|
1801
|
-
const path = getVerbMetadataPath(
|
|
1747
|
+
// v5.4.0: Use ID-first path
|
|
1748
|
+
const path = getVerbMetadataPath(id);
|
|
1802
1749
|
// Determine if this is a new verb by checking if metadata already exists
|
|
1803
1750
|
const existingMetadata = await this.readWithInheritance(path);
|
|
1804
1751
|
const isNew = !existingMetadata;
|
|
1805
1752
|
// Save the metadata (COW-aware - writes to branch-specific path)
|
|
1806
1753
|
await this.writeObjectToBranch(path, metadata);
|
|
1807
1754
|
// v5.4.0: Cache verb type for faster lookups
|
|
1808
|
-
this.verbTypeCache.set(id, verbType);
|
|
1809
1755
|
// CRITICAL FIX (v4.1.2): Increment verb count for new relationships
|
|
1810
1756
|
// This runs AFTER metadata is saved
|
|
1811
1757
|
// Uses synchronous increment since storage operations are already serialized
|
|
@@ -1820,69 +1766,34 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1820
1766
|
}
|
|
1821
1767
|
/**
|
|
1822
1768
|
* Get verb metadata from storage (v4.0.0: now typed)
|
|
1823
|
-
* v5.4.0: Uses
|
|
1769
|
+
* v5.4.0: Uses ID-first paths (must match saveVerbMetadata_internal)
|
|
1824
1770
|
*/
|
|
1825
1771
|
async getVerbMetadata(id) {
|
|
1826
1772
|
await this.ensureInitialized();
|
|
1827
|
-
//
|
|
1828
|
-
const
|
|
1829
|
-
|
|
1830
|
-
const
|
|
1831
|
-
return
|
|
1832
|
-
}
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
const path = getVerbMetadataPath(type, id);
|
|
1837
|
-
try {
|
|
1838
|
-
const metadata = await this.readWithInheritance(path);
|
|
1839
|
-
if (metadata) {
|
|
1840
|
-
// Cache the type for next time
|
|
1841
|
-
this.verbTypeCache.set(id, type);
|
|
1842
|
-
return metadata;
|
|
1843
|
-
}
|
|
1844
|
-
}
|
|
1845
|
-
catch (error) {
|
|
1846
|
-
// Not in this type, continue searching
|
|
1847
|
-
}
|
|
1773
|
+
// v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
|
|
1774
|
+
const path = getVerbMetadataPath(id);
|
|
1775
|
+
try {
|
|
1776
|
+
const metadata = await this.readWithInheritance(path);
|
|
1777
|
+
return metadata || null;
|
|
1778
|
+
}
|
|
1779
|
+
catch (error) {
|
|
1780
|
+
// Entity not found
|
|
1781
|
+
return null;
|
|
1848
1782
|
}
|
|
1849
|
-
return null;
|
|
1850
1783
|
}
|
|
1851
1784
|
/**
|
|
1852
|
-
* Delete verb metadata from storage
|
|
1853
|
-
* v5.4.0: Uses type-first paths (must match saveVerbMetadata_internal)
|
|
1785
|
+
* Delete verb metadata from storage (v6.0.0: ID-first, O(1) delete)
|
|
1854
1786
|
*/
|
|
1855
1787
|
async deleteVerbMetadata(id) {
|
|
1856
1788
|
await this.ensureInitialized();
|
|
1857
|
-
//
|
|
1858
|
-
const
|
|
1859
|
-
|
|
1860
|
-
const path = getVerbMetadataPath(cachedType, id);
|
|
1861
|
-
await this.deleteObjectFromBranch(path);
|
|
1862
|
-
// Remove from cache after deletion
|
|
1863
|
-
this.verbTypeCache.delete(id);
|
|
1864
|
-
return;
|
|
1865
|
-
}
|
|
1866
|
-
// If not in cache, search all types to find and delete
|
|
1867
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
1868
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
1869
|
-
const path = getVerbMetadataPath(type, id);
|
|
1870
|
-
try {
|
|
1871
|
-
// Check if exists before deleting
|
|
1872
|
-
const exists = await this.readWithInheritance(path);
|
|
1873
|
-
if (exists) {
|
|
1874
|
-
await this.deleteObjectFromBranch(path);
|
|
1875
|
-
return;
|
|
1876
|
-
}
|
|
1877
|
-
}
|
|
1878
|
-
catch (error) {
|
|
1879
|
-
// Not in this type, continue searching
|
|
1880
|
-
}
|
|
1881
|
-
}
|
|
1789
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
1790
|
+
const path = getVerbMetadataPath(id);
|
|
1791
|
+
await this.deleteObjectFromBranch(path);
|
|
1882
1792
|
}
|
|
1883
1793
|
// ============================================================================
|
|
1884
|
-
//
|
|
1885
|
-
//
|
|
1794
|
+
// ID-FIRST HELPER METHODS (v6.0.0)
|
|
1795
|
+
// Direct O(1) ID lookups - no type needed!
|
|
1796
|
+
// Clean, simple architecture for billion-scale performance
|
|
1886
1797
|
// ============================================================================
|
|
1887
1798
|
/**
|
|
1888
1799
|
* Load type statistics from storage
|
|
@@ -1924,30 +1835,61 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1924
1835
|
*/
|
|
1925
1836
|
async rebuildTypeCounts() {
|
|
1926
1837
|
prodLog.info('[BaseStorage] Rebuilding type counts from storage...');
|
|
1927
|
-
// Rebuild
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1838
|
+
// v6.0.0: Rebuild by scanning shards (0x00-0xFF) and reading metadata
|
|
1839
|
+
this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT);
|
|
1840
|
+
this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT);
|
|
1841
|
+
// Scan noun shards
|
|
1842
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
1843
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
1844
|
+
const shardDir = `entities/nouns/${shardHex}`;
|
|
1931
1845
|
try {
|
|
1932
|
-
const paths = await this.listObjectsInBranch(
|
|
1933
|
-
|
|
1846
|
+
const paths = await this.listObjectsInBranch(shardDir);
|
|
1847
|
+
for (const path of paths) {
|
|
1848
|
+
if (!path.includes('/metadata.json'))
|
|
1849
|
+
continue;
|
|
1850
|
+
try {
|
|
1851
|
+
const metadata = await this.readWithInheritance(path);
|
|
1852
|
+
if (metadata && metadata.noun) {
|
|
1853
|
+
const typeIndex = TypeUtils.getNounIndex(metadata.noun);
|
|
1854
|
+
if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
|
|
1855
|
+
this.nounCountsByType[typeIndex]++;
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
catch (error) {
|
|
1860
|
+
// Skip entities that fail to load
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1934
1863
|
}
|
|
1935
1864
|
catch (error) {
|
|
1936
|
-
//
|
|
1937
|
-
this.verbCountsByType[i] = 0;
|
|
1865
|
+
// Skip shards that don't exist
|
|
1938
1866
|
}
|
|
1939
1867
|
}
|
|
1940
|
-
//
|
|
1941
|
-
for (let
|
|
1942
|
-
const
|
|
1943
|
-
const
|
|
1868
|
+
// Scan verb shards
|
|
1869
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
1870
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
1871
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
1944
1872
|
try {
|
|
1945
|
-
const paths = await this.listObjectsInBranch(
|
|
1946
|
-
|
|
1873
|
+
const paths = await this.listObjectsInBranch(shardDir);
|
|
1874
|
+
for (const path of paths) {
|
|
1875
|
+
if (!path.includes('/metadata.json'))
|
|
1876
|
+
continue;
|
|
1877
|
+
try {
|
|
1878
|
+
const metadata = await this.readWithInheritance(path);
|
|
1879
|
+
if (metadata && metadata.verb) {
|
|
1880
|
+
const typeIndex = TypeUtils.getVerbIndex(metadata.verb);
|
|
1881
|
+
if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
|
|
1882
|
+
this.verbCountsByType[typeIndex]++;
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
catch (error) {
|
|
1887
|
+
// Skip entities that fail to load
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1947
1890
|
}
|
|
1948
1891
|
catch (error) {
|
|
1949
|
-
//
|
|
1950
|
-
this.nounCountsByType[i] = 0;
|
|
1892
|
+
// Skip shards that don't exist
|
|
1951
1893
|
}
|
|
1952
1894
|
}
|
|
1953
1895
|
// Save rebuilt counts to storage
|
|
@@ -1957,18 +1899,13 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1957
1899
|
prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
|
|
1958
1900
|
}
|
|
1959
1901
|
/**
|
|
1960
|
-
* Get noun type
|
|
1961
|
-
*
|
|
1902
|
+
* Get noun type (v6.0.0: type no longer needed for paths!)
|
|
1903
|
+
* With ID-first paths, this is only used for internal statistics tracking.
|
|
1904
|
+
* The actual type is stored in metadata and indexed by MetadataIndexManager.
|
|
1962
1905
|
*/
|
|
1963
1906
|
getNounType(noun) {
|
|
1964
|
-
//
|
|
1965
|
-
|
|
1966
|
-
if (cached) {
|
|
1967
|
-
return cached;
|
|
1968
|
-
}
|
|
1969
|
-
// Default to 'thing' if unknown
|
|
1970
|
-
// This should only happen if saveNoun_internal is called before saveNounMetadata
|
|
1971
|
-
prodLog.warn(`[BaseStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
|
|
1907
|
+
// v6.0.0: Type cache removed - default to 'thing' for statistics
|
|
1908
|
+
// The real type is in metadata, accessible via getNounMetadata(id)
|
|
1972
1909
|
return 'thing';
|
|
1973
1910
|
}
|
|
1974
1911
|
/**
|
|
@@ -2051,15 +1988,14 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2051
1988
|
// Converted from abstract to concrete - all adapters now have built-in type-aware
|
|
2052
1989
|
// ============================================================================
|
|
2053
1990
|
/**
|
|
2054
|
-
* Save a noun to storage (
|
|
1991
|
+
* Save a noun to storage (ID-first path)
|
|
2055
1992
|
*/
|
|
2056
1993
|
async saveNoun_internal(noun) {
|
|
2057
1994
|
const type = this.getNounType(noun);
|
|
2058
|
-
const path = getNounVectorPath(
|
|
1995
|
+
const path = getNounVectorPath(noun.id);
|
|
2059
1996
|
// Update type tracking
|
|
2060
1997
|
const typeIndex = TypeUtils.getNounIndex(type);
|
|
2061
1998
|
this.nounCountsByType[typeIndex]++;
|
|
2062
|
-
this.nounTypeCache.set(noun.id, type);
|
|
2063
1999
|
// COW-aware write (v5.0.1): Use COW helper for branch isolation
|
|
2064
2000
|
await this.writeObjectToBranch(path, noun);
|
|
2065
2001
|
// Periodically save statistics (every 100 saves)
|
|
@@ -2068,120 +2004,89 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2068
2004
|
}
|
|
2069
2005
|
}
|
|
2070
2006
|
/**
|
|
2071
|
-
* Get a noun from storage (
|
|
2007
|
+
* Get a noun from storage (ID-first path)
|
|
2072
2008
|
*/
|
|
2073
2009
|
async getNoun_internal(id) {
|
|
2074
|
-
//
|
|
2075
|
-
const
|
|
2076
|
-
|
|
2077
|
-
const path = getNounVectorPath(cachedType, id);
|
|
2010
|
+
// v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
|
|
2011
|
+
const path = getNounVectorPath(id);
|
|
2012
|
+
try {
|
|
2078
2013
|
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2079
|
-
const
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
// Need to search across all types (expensive, but cached after first access)
|
|
2084
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
2085
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
2086
|
-
const path = getNounVectorPath(type, id);
|
|
2087
|
-
try {
|
|
2088
|
-
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2089
|
-
const noun = await this.readWithInheritance(path);
|
|
2090
|
-
if (noun) {
|
|
2091
|
-
// Cache the type for next time
|
|
2092
|
-
this.nounTypeCache.set(id, type);
|
|
2093
|
-
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2094
|
-
return this.deserializeNoun(noun);
|
|
2095
|
-
}
|
|
2096
|
-
}
|
|
2097
|
-
catch (error) {
|
|
2098
|
-
// Not in this type, continue searching
|
|
2014
|
+
const noun = await this.readWithInheritance(path);
|
|
2015
|
+
if (noun) {
|
|
2016
|
+
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2017
|
+
return this.deserializeNoun(noun);
|
|
2099
2018
|
}
|
|
2100
2019
|
}
|
|
2020
|
+
catch (error) {
|
|
2021
|
+
// Entity not found
|
|
2022
|
+
return null;
|
|
2023
|
+
}
|
|
2101
2024
|
return null;
|
|
2102
2025
|
}
|
|
2103
2026
|
/**
|
|
2104
|
-
* Get nouns by noun type (
|
|
2027
|
+
* Get nouns by noun type (v6.0.0: Shard-based iteration!)
|
|
2105
2028
|
*/
|
|
2106
2029
|
async getNounsByNounType_internal(nounType) {
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
// COW-aware list (v5.0.1): Use COW helper for branch isolation
|
|
2110
|
-
const paths = await this.listObjectsInBranch(prefix);
|
|
2111
|
-
// Load all nouns of this type
|
|
2030
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types
|
|
2031
|
+
// Type is stored in metadata.noun field, we filter as we load
|
|
2112
2032
|
const nouns = [];
|
|
2113
|
-
for (
|
|
2033
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2034
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2035
|
+
const shardDir = `entities/nouns/${shardHex}`;
|
|
2114
2036
|
try {
|
|
2115
|
-
|
|
2116
|
-
const
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2037
|
+
const nounFiles = await this.listObjectsInBranch(shardDir);
|
|
2038
|
+
for (const nounPath of nounFiles) {
|
|
2039
|
+
if (!nounPath.includes('/vectors.json'))
|
|
2040
|
+
continue;
|
|
2041
|
+
try {
|
|
2042
|
+
const noun = await this.readWithInheritance(nounPath);
|
|
2043
|
+
if (noun) {
|
|
2044
|
+
const deserialized = this.deserializeNoun(noun);
|
|
2045
|
+
// Check type from metadata
|
|
2046
|
+
const metadata = await this.getNounMetadata(deserialized.id);
|
|
2047
|
+
if (metadata && metadata.noun === nounType) {
|
|
2048
|
+
nouns.push(deserialized);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
catch (error) {
|
|
2053
|
+
// Skip nouns that fail to load
|
|
2054
|
+
}
|
|
2122
2055
|
}
|
|
2123
2056
|
}
|
|
2124
2057
|
catch (error) {
|
|
2125
|
-
|
|
2058
|
+
// Skip shards that have no data
|
|
2126
2059
|
}
|
|
2127
2060
|
}
|
|
2128
2061
|
return nouns;
|
|
2129
2062
|
}
|
|
2130
2063
|
/**
|
|
2131
|
-
* Delete a noun from storage (
|
|
2064
|
+
* Delete a noun from storage (v6.0.0: ID-first, O(1) delete)
|
|
2132
2065
|
*/
|
|
2133
2066
|
async deleteNoun_internal(id) {
|
|
2134
|
-
//
|
|
2135
|
-
const
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
await this.deleteObjectFromBranch(path);
|
|
2140
|
-
// Update counts
|
|
2141
|
-
const typeIndex = TypeUtils.getNounIndex(cachedType);
|
|
2142
|
-
if (this.nounCountsByType[typeIndex] > 0) {
|
|
2143
|
-
this.nounCountsByType[typeIndex]--;
|
|
2144
|
-
}
|
|
2145
|
-
this.nounTypeCache.delete(id);
|
|
2146
|
-
return;
|
|
2147
|
-
}
|
|
2148
|
-
// Search across all types
|
|
2149
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
2150
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
2151
|
-
const path = getNounVectorPath(type, id);
|
|
2152
|
-
try {
|
|
2153
|
-
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
|
|
2154
|
-
await this.deleteObjectFromBranch(path);
|
|
2155
|
-
// Update counts
|
|
2156
|
-
if (this.nounCountsByType[i] > 0) {
|
|
2157
|
-
this.nounCountsByType[i]--;
|
|
2158
|
-
}
|
|
2159
|
-
this.nounTypeCache.delete(id);
|
|
2160
|
-
return;
|
|
2161
|
-
}
|
|
2162
|
-
catch (error) {
|
|
2163
|
-
// Not in this type, continue
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2067
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
2068
|
+
const path = getNounVectorPath(id);
|
|
2069
|
+
await this.deleteObjectFromBranch(path);
|
|
2070
|
+
// Note: Type-specific counts will be decremented via metadata tracking
|
|
2071
|
+
// The real type is in metadata, accessible if needed via getNounMetadata(id)
|
|
2166
2072
|
}
|
|
2167
2073
|
/**
|
|
2168
|
-
* Save a verb to storage (
|
|
2074
|
+
* Save a verb to storage (ID-first path)
|
|
2169
2075
|
*/
|
|
2170
2076
|
async saveVerb_internal(verb) {
|
|
2171
2077
|
// Type is now a first-class field in HNSWVerb - no caching needed!
|
|
2172
2078
|
const type = verb.verb;
|
|
2173
|
-
const path = getVerbVectorPath(
|
|
2079
|
+
const path = getVerbVectorPath(verb.id);
|
|
2080
|
+
prodLog.debug(`[BaseStorage] saveVerb_internal: id=${verb.id}, sourceId=${verb.sourceId}, targetId=${verb.targetId}, type=${type}`);
|
|
2174
2081
|
// Update type tracking
|
|
2175
2082
|
const typeIndex = TypeUtils.getVerbIndex(type);
|
|
2176
2083
|
this.verbCountsByType[typeIndex]++;
|
|
2177
|
-
this.verbTypeCache.set(verb.id, type);
|
|
2178
2084
|
// COW-aware write (v5.0.1): Use COW helper for branch isolation
|
|
2179
2085
|
await this.writeObjectToBranch(path, verb);
|
|
2180
|
-
//
|
|
2181
|
-
//
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
// Fast incremental update - no rebuild needed
|
|
2086
|
+
// v6.0.0: Update GraphAdjacencyIndex incrementally (always available after init())
|
|
2087
|
+
// GraphAdjacencyIndex.addVerb() calls ensureInitialized() automatically
|
|
2088
|
+
if (this.graphIndex) {
|
|
2089
|
+
prodLog.debug(`[BaseStorage] Updating GraphAdjacencyIndex with verb ${verb.id}`);
|
|
2185
2090
|
await this.graphIndex.addVerb({
|
|
2186
2091
|
id: verb.id,
|
|
2187
2092
|
sourceId: verb.sourceId,
|
|
@@ -2193,8 +2098,12 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2193
2098
|
type: verb.verb,
|
|
2194
2099
|
createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
|
|
2195
2100
|
updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
|
|
2196
|
-
createdBy: { augmentation: 'storage', version: '
|
|
2101
|
+
createdBy: { augmentation: 'storage', version: '6.0.0' }
|
|
2197
2102
|
});
|
|
2103
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex updated successfully`);
|
|
2104
|
+
}
|
|
2105
|
+
else {
|
|
2106
|
+
prodLog.warn(`[BaseStorage] graphIndex is null, cannot update index for verb ${verb.id}`);
|
|
2198
2107
|
}
|
|
2199
2108
|
// Periodically save statistics
|
|
2200
2109
|
if (this.verbCountsByType[typeIndex] % 100 === 0) {
|
|
@@ -2202,69 +2111,89 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2202
2111
|
}
|
|
2203
2112
|
}
|
|
2204
2113
|
/**
|
|
2205
|
-
* Get a verb from storage (
|
|
2114
|
+
* Get a verb from storage (ID-first path)
|
|
2206
2115
|
*/
|
|
2207
2116
|
async getVerb_internal(id) {
|
|
2208
|
-
//
|
|
2209
|
-
const
|
|
2210
|
-
|
|
2211
|
-
const path = getVerbVectorPath(cachedType, id);
|
|
2117
|
+
// v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
|
|
2118
|
+
const path = getVerbVectorPath(id);
|
|
2119
|
+
try {
|
|
2212
2120
|
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2213
2121
|
const verb = await this.readWithInheritance(path);
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
// Search across all types (only on first access)
|
|
2218
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
2219
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
2220
|
-
const path = getVerbVectorPath(type, id);
|
|
2221
|
-
try {
|
|
2222
|
-
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2223
|
-
const verb = await this.readWithInheritance(path);
|
|
2224
|
-
if (verb) {
|
|
2225
|
-
// Cache the type for next time (read from verb.verb field)
|
|
2226
|
-
this.verbTypeCache.set(id, verb.verb);
|
|
2227
|
-
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2228
|
-
return this.deserializeVerb(verb);
|
|
2229
|
-
}
|
|
2230
|
-
}
|
|
2231
|
-
catch (error) {
|
|
2232
|
-
// Not in this type, continue
|
|
2122
|
+
if (verb) {
|
|
2123
|
+
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2124
|
+
return this.deserializeVerb(verb);
|
|
2233
2125
|
}
|
|
2234
2126
|
}
|
|
2127
|
+
catch (error) {
|
|
2128
|
+
// Entity not found
|
|
2129
|
+
return null;
|
|
2130
|
+
}
|
|
2235
2131
|
return null;
|
|
2236
2132
|
}
|
|
2237
2133
|
/**
|
|
2238
|
-
* Get verbs by source (
|
|
2239
|
-
*
|
|
2134
|
+
* Get verbs by source (v6.0.0: Uses GraphAdjacencyIndex when available)
|
|
2135
|
+
* Falls back to shard iteration during initialization to avoid circular dependency
|
|
2240
2136
|
*/
|
|
2241
2137
|
async getVerbsBySource_internal(sourceId) {
|
|
2242
|
-
// v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
|
|
2243
|
-
// v5.7.0 called getGraphIndex() here, creating deadlock during initialization:
|
|
2244
|
-
// GraphAdjacencyIndex.rebuild() → storage.getVerbs() → getVerbsBySource_internal() → getGraphIndex() → [deadlock]
|
|
2245
|
-
// v5.4.0: Type-first implementation - scan across all verb types
|
|
2246
|
-
// COW-aware: uses readWithInheritance for each verb
|
|
2247
2138
|
await this.ensureInitialized();
|
|
2139
|
+
prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`);
|
|
2140
|
+
// v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
|
|
2141
|
+
if (this.graphIndex && this.graphIndex.isInitialized) {
|
|
2142
|
+
try {
|
|
2143
|
+
const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId);
|
|
2144
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`);
|
|
2145
|
+
const results = [];
|
|
2146
|
+
for (const verbId of verbIds) {
|
|
2147
|
+
const verb = await this.getVerb_internal(verbId);
|
|
2148
|
+
const metadata = await this.getVerbMetadata(verbId);
|
|
2149
|
+
if (verb && metadata) {
|
|
2150
|
+
results.push({
|
|
2151
|
+
...verb,
|
|
2152
|
+
weight: metadata.weight,
|
|
2153
|
+
confidence: metadata.confidence,
|
|
2154
|
+
createdAt: metadata.createdAt
|
|
2155
|
+
? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
|
|
2156
|
+
: Date.now(),
|
|
2157
|
+
updatedAt: metadata.updatedAt
|
|
2158
|
+
? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
|
|
2159
|
+
: Date.now(),
|
|
2160
|
+
service: metadata.service,
|
|
2161
|
+
createdBy: metadata.createdBy,
|
|
2162
|
+
metadata: metadata || {}
|
|
2163
|
+
});
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex path returned ${results.length} verbs`);
|
|
2167
|
+
return results;
|
|
2168
|
+
}
|
|
2169
|
+
catch (error) {
|
|
2170
|
+
prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
// v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
|
|
2174
|
+
prodLog.debug(`[BaseStorage] Using shard iteration fallback for sourceId=${sourceId}`);
|
|
2248
2175
|
const results = [];
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
const
|
|
2176
|
+
let shardsScanned = 0;
|
|
2177
|
+
let verbsFound = 0;
|
|
2178
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2179
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2180
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2253
2181
|
try {
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
const verbFiles = await this.listObjectsInBranch(typeDir);
|
|
2182
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2183
|
+
shardsScanned++;
|
|
2257
2184
|
for (const verbPath of verbFiles) {
|
|
2258
|
-
|
|
2259
|
-
if (!verbPath.endsWith('.json'))
|
|
2185
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2260
2186
|
continue;
|
|
2261
2187
|
try {
|
|
2262
|
-
const
|
|
2263
|
-
if (
|
|
2264
|
-
|
|
2265
|
-
|
|
2188
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
2189
|
+
if (!rawVerb)
|
|
2190
|
+
continue;
|
|
2191
|
+
verbsFound++;
|
|
2192
|
+
// v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
|
|
2193
|
+
const verb = this.deserializeVerb(rawVerb);
|
|
2194
|
+
if (verb.sourceId === sourceId) {
|
|
2195
|
+
const metadataPath = getVerbMetadataPath(verb.id);
|
|
2266
2196
|
const metadata = await this.readWithInheritance(metadataPath);
|
|
2267
|
-
// v5.4.0: Extract standard fields from metadata to top-level (like nouns)
|
|
2268
2197
|
results.push({
|
|
2269
2198
|
...verb,
|
|
2270
2199
|
weight: metadata?.weight,
|
|
@@ -2283,13 +2212,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2283
2212
|
}
|
|
2284
2213
|
catch (error) {
|
|
2285
2214
|
// Skip verbs that fail to load
|
|
2215
|
+
prodLog.debug(`[BaseStorage] Failed to load verb from ${verbPath}:`, error);
|
|
2286
2216
|
}
|
|
2287
2217
|
}
|
|
2288
2218
|
}
|
|
2289
2219
|
catch (error) {
|
|
2290
|
-
// Skip
|
|
2220
|
+
// Skip shards that have no data
|
|
2291
2221
|
}
|
|
2292
2222
|
}
|
|
2223
|
+
prodLog.debug(`[BaseStorage] Shard iteration: scanned ${shardsScanned} shards, found ${verbsFound} total verbs, matched ${results.length} for sourceId=${sourceId}`);
|
|
2293
2224
|
return results;
|
|
2294
2225
|
}
|
|
2295
2226
|
/**
|
|
@@ -2335,52 +2266,46 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2335
2266
|
}
|
|
2336
2267
|
// Convert sourceIds to Set for O(1) lookup
|
|
2337
2268
|
const sourceIdSet = new Set(sourceIds);
|
|
2338
|
-
//
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
}
|
|
2343
|
-
else {
|
|
2344
|
-
// Scan all verb types
|
|
2345
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
2346
|
-
typesToScan.push(TypeUtils.getVerbFromIndex(i));
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
// Scan verb types and collect matching verbs
|
|
2350
|
-
for (const type of typesToScan) {
|
|
2351
|
-
const typeDir = `entities/verbs/${type}/vectors`;
|
|
2269
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types
|
|
2270
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2271
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2272
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2352
2273
|
try {
|
|
2353
|
-
// List all verb files
|
|
2354
|
-
const verbFiles = await this.listObjectsInBranch(
|
|
2274
|
+
// List all verb files in this shard
|
|
2275
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2355
2276
|
// Build paths for batch read
|
|
2356
2277
|
const verbPaths = [];
|
|
2357
2278
|
const metadataPaths = [];
|
|
2358
2279
|
const pathToId = new Map();
|
|
2359
2280
|
for (const verbPath of verbFiles) {
|
|
2360
|
-
if (!verbPath.
|
|
2281
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2361
2282
|
continue;
|
|
2362
2283
|
verbPaths.push(verbPath);
|
|
2363
|
-
// Extract ID from path: "entities/verbs/{
|
|
2284
|
+
// Extract ID from path: "entities/verbs/{shard}/{id}/vector.json"
|
|
2364
2285
|
const parts = verbPath.split('/');
|
|
2365
|
-
const
|
|
2366
|
-
const verbId = filename.replace('.json', '');
|
|
2286
|
+
const verbId = parts[parts.length - 2]; // ID is second-to-last segment
|
|
2367
2287
|
pathToId.set(verbPath, verbId);
|
|
2368
2288
|
// Prepare metadata path
|
|
2369
|
-
metadataPaths.push(getVerbMetadataPath(
|
|
2289
|
+
metadataPaths.push(getVerbMetadataPath(verbId));
|
|
2370
2290
|
}
|
|
2371
|
-
// Batch read all verb files for this
|
|
2291
|
+
// Batch read all verb files for this shard
|
|
2372
2292
|
const verbDataMap = await this.readBatchWithInheritance(verbPaths);
|
|
2373
2293
|
const metadataMap = await this.readBatchWithInheritance(metadataPaths);
|
|
2374
2294
|
// Process results
|
|
2375
|
-
for (const [verbPath,
|
|
2376
|
-
if (!
|
|
2295
|
+
for (const [verbPath, rawVerbData] of verbDataMap.entries()) {
|
|
2296
|
+
if (!rawVerbData || !rawVerbData.sourceId)
|
|
2377
2297
|
continue;
|
|
2298
|
+
// v6.0.0: Deserialize connections Map from JSON storage format
|
|
2299
|
+
const verbData = this.deserializeVerb(rawVerbData);
|
|
2378
2300
|
// Check if this verb's source is in our requested set
|
|
2379
2301
|
if (!sourceIdSet.has(verbData.sourceId))
|
|
2380
2302
|
continue;
|
|
2303
|
+
// If verbType specified, filter by type
|
|
2304
|
+
if (verbType && verbData.verb !== verbType)
|
|
2305
|
+
continue;
|
|
2381
2306
|
// Found matching verb - hydrate with metadata
|
|
2382
2307
|
const verbId = pathToId.get(verbPath);
|
|
2383
|
-
const metadataPath = getVerbMetadataPath(
|
|
2308
|
+
const metadataPath = getVerbMetadataPath(verbId);
|
|
2384
2309
|
const metadata = metadataMap.get(metadataPath) || {};
|
|
2385
2310
|
const hydratedVerb = {
|
|
2386
2311
|
...verbData,
|
|
@@ -2402,7 +2327,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2402
2327
|
}
|
|
2403
2328
|
}
|
|
2404
2329
|
catch (error) {
|
|
2405
|
-
// Skip
|
|
2330
|
+
// Skip shards that have no data
|
|
2406
2331
|
}
|
|
2407
2332
|
}
|
|
2408
2333
|
return results;
|
|
@@ -2413,31 +2338,57 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2413
2338
|
* v5.4.0: Fixed to directly list verb files instead of directories
|
|
2414
2339
|
*/
|
|
2415
2340
|
async getVerbsByTarget_internal(targetId) {
|
|
2416
|
-
// v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
|
|
2417
|
-
// v5.7.0 called getGraphIndex() here, creating deadlock during initialization
|
|
2418
|
-
// v5.4.0: Type-first implementation - scan across all verb types
|
|
2419
|
-
// COW-aware: uses readWithInheritance for each verb
|
|
2420
2341
|
await this.ensureInitialized();
|
|
2342
|
+
// v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
|
|
2343
|
+
if (this.graphIndex && this.graphIndex.isInitialized) {
|
|
2344
|
+
try {
|
|
2345
|
+
const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId);
|
|
2346
|
+
const results = [];
|
|
2347
|
+
for (const verbId of verbIds) {
|
|
2348
|
+
const verb = await this.getVerb_internal(verbId);
|
|
2349
|
+
const metadata = await this.getVerbMetadata(verbId);
|
|
2350
|
+
if (verb && metadata) {
|
|
2351
|
+
results.push({
|
|
2352
|
+
...verb,
|
|
2353
|
+
weight: metadata.weight,
|
|
2354
|
+
confidence: metadata.confidence,
|
|
2355
|
+
createdAt: metadata.createdAt
|
|
2356
|
+
? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
|
|
2357
|
+
: Date.now(),
|
|
2358
|
+
updatedAt: metadata.updatedAt
|
|
2359
|
+
? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
|
|
2360
|
+
: Date.now(),
|
|
2361
|
+
service: metadata.service,
|
|
2362
|
+
createdBy: metadata.createdBy,
|
|
2363
|
+
metadata: metadata || {}
|
|
2364
|
+
});
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
2367
|
+
return results;
|
|
2368
|
+
}
|
|
2369
|
+
catch (error) {
|
|
2370
|
+
prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
// v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
|
|
2421
2374
|
const results = [];
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
const
|
|
2425
|
-
const typeDir = `entities/verbs/${type}/vectors`;
|
|
2375
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2376
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2377
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2426
2378
|
try {
|
|
2427
|
-
|
|
2428
|
-
// listObjectsInBranch returns full paths to .json files, not directories
|
|
2429
|
-
const verbFiles = await this.listObjectsInBranch(typeDir);
|
|
2379
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2430
2380
|
for (const verbPath of verbFiles) {
|
|
2431
|
-
|
|
2432
|
-
if (!verbPath.endsWith('.json'))
|
|
2381
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2433
2382
|
continue;
|
|
2434
2383
|
try {
|
|
2435
|
-
const
|
|
2436
|
-
if (
|
|
2437
|
-
|
|
2438
|
-
|
|
2384
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
2385
|
+
if (!rawVerb)
|
|
2386
|
+
continue;
|
|
2387
|
+
// v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
|
|
2388
|
+
const verb = this.deserializeVerb(rawVerb);
|
|
2389
|
+
if (verb.targetId === targetId) {
|
|
2390
|
+
const metadataPath = getVerbMetadataPath(verb.id);
|
|
2439
2391
|
const metadata = await this.readWithInheritance(metadataPath);
|
|
2440
|
-
// v5.4.0: Extract standard fields from metadata to top-level (like nouns)
|
|
2441
2392
|
results.push({
|
|
2442
2393
|
...verb,
|
|
2443
2394
|
weight: metadata?.weight,
|
|
@@ -2460,95 +2411,77 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2460
2411
|
}
|
|
2461
2412
|
}
|
|
2462
2413
|
catch (error) {
|
|
2463
|
-
// Skip
|
|
2414
|
+
// Skip shards that have no data
|
|
2464
2415
|
}
|
|
2465
2416
|
}
|
|
2466
2417
|
return results;
|
|
2467
2418
|
}
|
|
2468
2419
|
/**
|
|
2469
|
-
* Get verbs by type (
|
|
2420
|
+
* Get verbs by type (v6.0.0: Shard iteration with type filtering)
|
|
2470
2421
|
*/
|
|
2471
2422
|
async getVerbsByType_internal(verbType) {
|
|
2472
|
-
|
|
2473
|
-
const prefix = `entities/verbs/${type}/vectors/`;
|
|
2474
|
-
// COW-aware list (v5.0.1): Use COW helper for branch isolation
|
|
2475
|
-
const paths = await this.listObjectsInBranch(prefix);
|
|
2423
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of type-first paths
|
|
2476
2424
|
const verbs = [];
|
|
2477
|
-
for (
|
|
2425
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2426
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2427
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2478
2428
|
try {
|
|
2479
|
-
|
|
2480
|
-
const
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2429
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2430
|
+
for (const verbPath of verbFiles) {
|
|
2431
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2432
|
+
continue;
|
|
2433
|
+
try {
|
|
2434
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
2435
|
+
if (!rawVerb)
|
|
2436
|
+
continue;
|
|
2437
|
+
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2438
|
+
const hnswVerb = this.deserializeVerb(rawVerb);
|
|
2439
|
+
// Filter by verb type
|
|
2440
|
+
if (hnswVerb.verb !== verbType)
|
|
2441
|
+
continue;
|
|
2442
|
+
// Load metadata separately (optional in v4.0.0!)
|
|
2443
|
+
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
2444
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
2445
|
+
const metadataObj = (metadata || {});
|
|
2446
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
2447
|
+
const verbWithMetadata = {
|
|
2448
|
+
id: hnswVerb.id,
|
|
2449
|
+
vector: [...hnswVerb.vector],
|
|
2450
|
+
connections: hnswVerb.connections, // v5.7.10: Already deserialized
|
|
2451
|
+
verb: hnswVerb.verb,
|
|
2452
|
+
sourceId: hnswVerb.sourceId,
|
|
2453
|
+
targetId: hnswVerb.targetId,
|
|
2454
|
+
createdAt: createdAt || Date.now(),
|
|
2455
|
+
updatedAt: updatedAt || Date.now(),
|
|
2456
|
+
confidence: confidence,
|
|
2457
|
+
weight: weight,
|
|
2458
|
+
service: service,
|
|
2459
|
+
data: data,
|
|
2460
|
+
createdBy,
|
|
2461
|
+
metadata: customMetadata
|
|
2462
|
+
};
|
|
2463
|
+
verbs.push(verbWithMetadata);
|
|
2464
|
+
}
|
|
2465
|
+
catch (error) {
|
|
2466
|
+
// Skip verbs that fail to load
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2511
2469
|
}
|
|
2512
2470
|
catch (error) {
|
|
2513
|
-
|
|
2471
|
+
// Skip shards that have no data
|
|
2514
2472
|
}
|
|
2515
2473
|
}
|
|
2516
2474
|
return verbs;
|
|
2517
2475
|
}
|
|
2518
2476
|
/**
|
|
2519
|
-
* Delete a verb from storage (
|
|
2477
|
+
* Delete a verb from storage (v6.0.0: ID-first, O(1) delete)
|
|
2520
2478
|
*/
|
|
2521
2479
|
async deleteVerb_internal(id) {
|
|
2522
|
-
//
|
|
2523
|
-
const
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
await this.deleteObjectFromBranch(path);
|
|
2528
|
-
const typeIndex = TypeUtils.getVerbIndex(cachedType);
|
|
2529
|
-
if (this.verbCountsByType[typeIndex] > 0) {
|
|
2530
|
-
this.verbCountsByType[typeIndex]--;
|
|
2531
|
-
}
|
|
2532
|
-
this.verbTypeCache.delete(id);
|
|
2533
|
-
return;
|
|
2534
|
-
}
|
|
2535
|
-
// Search across all types
|
|
2536
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
2537
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
2538
|
-
const path = getVerbVectorPath(type, id);
|
|
2539
|
-
try {
|
|
2540
|
-
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
|
|
2541
|
-
await this.deleteObjectFromBranch(path);
|
|
2542
|
-
if (this.verbCountsByType[i] > 0) {
|
|
2543
|
-
this.verbCountsByType[i]--;
|
|
2544
|
-
}
|
|
2545
|
-
this.verbTypeCache.delete(id);
|
|
2546
|
-
return;
|
|
2547
|
-
}
|
|
2548
|
-
catch (error) {
|
|
2549
|
-
// Continue
|
|
2550
|
-
}
|
|
2551
|
-
}
|
|
2480
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
2481
|
+
const path = getVerbVectorPath(id);
|
|
2482
|
+
await this.deleteObjectFromBranch(path);
|
|
2483
|
+
// Note: Type-specific counts will be decremented via metadata tracking
|
|
2484
|
+
// The real type is in metadata, accessible if needed via getVerbMetadata(id)
|
|
2552
2485
|
}
|
|
2553
2486
|
/**
|
|
2554
2487
|
* Helper method to convert a Map to a plain object for serialization
|