@soulcraft/brainy 5.12.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +155 -5
- package/README.md +2 -6
- package/dist/api/DataAPI.d.ts +0 -40
- package/dist/api/DataAPI.js +0 -235
- package/dist/brainy.d.ts +0 -106
- package/dist/brainy.js +0 -370
- package/dist/cli/commands/cow.d.ts +1 -9
- package/dist/cli/commands/cow.js +1 -61
- package/dist/cli/commands/data.d.ts +1 -13
- package/dist/cli/commands/data.js +1 -74
- package/dist/cli/index.js +1 -16
- package/dist/neural/embeddedTypeEmbeddings.d.ts +1 -1
- package/dist/neural/embeddedTypeEmbeddings.js +2 -2
- package/dist/storage/adapters/azureBlobStorage.js +2 -1
- package/dist/storage/adapters/fileSystemStorage.js +2 -1
- package/dist/storage/adapters/gcsStorage.js +2 -1
- package/dist/storage/adapters/historicalStorageAdapter.js +2 -2
- package/dist/storage/adapters/memoryStorage.d.ts +1 -1
- package/dist/storage/adapters/memoryStorage.js +9 -11
- package/dist/storage/adapters/opfsStorage.js +2 -1
- package/dist/storage/adapters/r2Storage.js +2 -1
- package/dist/storage/adapters/s3CompatibleStorage.js +2 -1
- package/dist/storage/baseStorage.d.ts +40 -24
- package/dist/storage/baseStorage.js +478 -555
- package/dist/vfs/VirtualFileSystem.d.ts +46 -24
- package/dist/vfs/VirtualFileSystem.js +153 -146
- package/package.json +1 -1
|
@@ -39,32 +39,36 @@ export function getDirectoryPath(entityType, dataType) {
|
|
|
39
39
|
* Built-in type-aware organization for all storage adapters
|
|
40
40
|
*/
|
|
41
41
|
/**
|
|
42
|
-
* Get
|
|
42
|
+
* Get ID-first path for noun vectors (v6.0.0)
|
|
43
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
43
44
|
*/
|
|
44
|
-
function getNounVectorPath(
|
|
45
|
+
function getNounVectorPath(id) {
|
|
45
46
|
const shard = getShardIdFromUuid(id);
|
|
46
|
-
return `entities/nouns/${
|
|
47
|
+
return `entities/nouns/${shard}/${id}/vectors.json`;
|
|
47
48
|
}
|
|
48
49
|
/**
|
|
49
|
-
* Get
|
|
50
|
+
* Get ID-first path for noun metadata (v6.0.0)
|
|
51
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
50
52
|
*/
|
|
51
|
-
function getNounMetadataPath(
|
|
53
|
+
function getNounMetadataPath(id) {
|
|
52
54
|
const shard = getShardIdFromUuid(id);
|
|
53
|
-
return `entities/nouns/${
|
|
55
|
+
return `entities/nouns/${shard}/${id}/metadata.json`;
|
|
54
56
|
}
|
|
55
57
|
/**
|
|
56
|
-
* Get
|
|
58
|
+
* Get ID-first path for verb vectors (v6.0.0)
|
|
59
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
57
60
|
*/
|
|
58
|
-
function getVerbVectorPath(
|
|
61
|
+
function getVerbVectorPath(id) {
|
|
59
62
|
const shard = getShardIdFromUuid(id);
|
|
60
|
-
return `entities/verbs/${
|
|
63
|
+
return `entities/verbs/${shard}/${id}/vectors.json`;
|
|
61
64
|
}
|
|
62
65
|
/**
|
|
63
|
-
* Get
|
|
66
|
+
* Get ID-first path for verb metadata (v6.0.0)
|
|
67
|
+
* No type parameter needed - direct O(1) lookup by ID
|
|
64
68
|
*/
|
|
65
|
-
function getVerbMetadataPath(
|
|
69
|
+
function getVerbMetadataPath(id) {
|
|
66
70
|
const shard = getShardIdFromUuid(id);
|
|
67
|
-
return `entities/verbs/${
|
|
71
|
+
return `entities/verbs/${shard}/${id}/metadata.json`;
|
|
68
72
|
}
|
|
69
73
|
/**
|
|
70
74
|
* Base storage adapter that implements common functionality
|
|
@@ -89,9 +93,9 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
89
93
|
this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT); // 168 bytes (Stage 3: 42 types)
|
|
90
94
|
this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT); // 508 bytes (Stage 3: 127 types)
|
|
91
95
|
// Total: 676 bytes (99.2% reduction vs Map-based tracking)
|
|
92
|
-
// Type
|
|
93
|
-
|
|
94
|
-
|
|
96
|
+
// v6.0.0: Type caches REMOVED - ID-first paths eliminate need for type lookups!
|
|
97
|
+
// With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json
|
|
98
|
+
// Type is just a field in the metadata, indexed by MetadataIndexManager for queries
|
|
95
99
|
// v5.5.0: Track if type counts have been rebuilt (prevent repeated rebuilds)
|
|
96
100
|
this.typeCountsRebuilt = false;
|
|
97
101
|
}
|
|
@@ -176,8 +180,33 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
176
180
|
async init() {
|
|
177
181
|
// Load type statistics from storage (if they exist)
|
|
178
182
|
await this.loadTypeStatistics();
|
|
183
|
+
// v6.0.0: Create GraphAdjacencyIndex (lazy-loaded, no rebuild)
|
|
184
|
+
// LSM-trees are initialized on first use via ensureInitialized()
|
|
185
|
+
// Index is populated incrementally as verbs are added via addVerb()
|
|
186
|
+
try {
|
|
187
|
+
prodLog.debug('[BaseStorage] Creating GraphAdjacencyIndex...');
|
|
188
|
+
this.graphIndex = new GraphAdjacencyIndex(this);
|
|
189
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex instantiated (lazy-loaded), graphIndex=${!!this.graphIndex}`);
|
|
190
|
+
}
|
|
191
|
+
catch (error) {
|
|
192
|
+
prodLog.error('[BaseStorage] Failed to create GraphAdjacencyIndex:', error);
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
179
195
|
this.isInitialized = true;
|
|
180
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* Rebuild GraphAdjacencyIndex from existing verbs (v6.0.0)
|
|
199
|
+
* Call this manually if you have existing verb data that needs to be indexed
|
|
200
|
+
* @public
|
|
201
|
+
*/
|
|
202
|
+
async rebuildGraphIndex() {
|
|
203
|
+
if (!this.graphIndex) {
|
|
204
|
+
throw new Error('GraphAdjacencyIndex not initialized');
|
|
205
|
+
}
|
|
206
|
+
prodLog.info('[BaseStorage] Rebuilding graph index from existing data...');
|
|
207
|
+
await this.graphIndex.rebuild();
|
|
208
|
+
prodLog.info('[BaseStorage] Graph index rebuild complete');
|
|
209
|
+
}
|
|
181
210
|
/**
|
|
182
211
|
* Ensure the storage adapter is initialized
|
|
183
212
|
*/
|
|
@@ -851,60 +880,44 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
851
880
|
*/
|
|
852
881
|
async getNounsWithPagination(options) {
|
|
853
882
|
await this.ensureInitialized();
|
|
854
|
-
const { limit, offset = 0, filter } = options;
|
|
883
|
+
const { limit, offset = 0, filter } = options;
|
|
855
884
|
const collectedNouns = [];
|
|
856
|
-
const targetCount = offset + limit;
|
|
857
|
-
//
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
for (let i = 0; i < NOUN_TYPE_COUNT && collectedNouns.length < targetCount; i++) {
|
|
862
|
-
// OPTIMIZATION 1: Skip empty types (only if counts are reliable)
|
|
863
|
-
if (useOptimization && this.nounCountsByType[i] === 0) {
|
|
864
|
-
continue;
|
|
865
|
-
}
|
|
866
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
867
|
-
// If filtering by type, skip other types
|
|
868
|
-
if (filter?.nounType) {
|
|
869
|
-
const filterTypes = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
|
870
|
-
if (!filterTypes.includes(type)) {
|
|
871
|
-
continue;
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
const typeDir = `entities/nouns/${type}/vectors`;
|
|
885
|
+
const targetCount = offset + limit;
|
|
886
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types
|
|
887
|
+
for (let shard = 0; shard < 256 && collectedNouns.length < targetCount; shard++) {
|
|
888
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
889
|
+
const shardDir = `entities/nouns/${shardHex}`;
|
|
875
890
|
try {
|
|
876
|
-
|
|
877
|
-
const nounFiles = await this.listObjectsInBranch(typeDir);
|
|
891
|
+
const nounFiles = await this.listObjectsInBranch(shardDir);
|
|
878
892
|
for (const nounPath of nounFiles) {
|
|
879
|
-
|
|
880
|
-
if (collectedNouns.length >= targetCount) {
|
|
893
|
+
if (collectedNouns.length >= targetCount)
|
|
881
894
|
break;
|
|
882
|
-
|
|
883
|
-
// Skip if not a .json file
|
|
884
|
-
if (!nounPath.endsWith('.json'))
|
|
895
|
+
if (!nounPath.includes('/vectors.json'))
|
|
885
896
|
continue;
|
|
886
897
|
try {
|
|
887
|
-
const
|
|
888
|
-
if (
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
const noun = this.deserializeNoun(rawNoun);
|
|
892
|
-
// Load metadata
|
|
893
|
-
const metadataPath = getNounMetadataPath(type, noun.id);
|
|
894
|
-
const metadata = await this.readWithInheritance(metadataPath);
|
|
898
|
+
const noun = await this.readWithInheritance(nounPath);
|
|
899
|
+
if (noun) {
|
|
900
|
+
const deserialized = this.deserializeNoun(noun);
|
|
901
|
+
const metadata = await this.getNounMetadata(deserialized.id);
|
|
895
902
|
if (metadata) {
|
|
896
|
-
// Apply
|
|
903
|
+
// Apply type filter
|
|
904
|
+
if (filter?.nounType && metadata.noun) {
|
|
905
|
+
const types = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType];
|
|
906
|
+
if (!types.includes(metadata.noun)) {
|
|
907
|
+
continue;
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
// Apply service filter
|
|
897
911
|
if (filter?.service) {
|
|
898
912
|
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
|
899
913
|
if (metadata.service && !services.includes(metadata.service)) {
|
|
900
914
|
continue;
|
|
901
915
|
}
|
|
902
916
|
}
|
|
903
|
-
// Combine noun + metadata
|
|
917
|
+
// Combine noun + metadata
|
|
904
918
|
collectedNouns.push({
|
|
905
|
-
...
|
|
906
|
-
|
|
907
|
-
type: metadata.noun || type, // Required: Extract type from metadata
|
|
919
|
+
...deserialized,
|
|
920
|
+
type: (metadata.noun || 'thing'),
|
|
908
921
|
confidence: metadata.confidence,
|
|
909
922
|
weight: metadata.weight,
|
|
910
923
|
createdAt: metadata.createdAt
|
|
@@ -927,15 +940,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
927
940
|
}
|
|
928
941
|
}
|
|
929
942
|
catch (error) {
|
|
930
|
-
// Skip
|
|
943
|
+
// Skip shards that have no data
|
|
931
944
|
}
|
|
932
945
|
}
|
|
933
|
-
// Apply pagination
|
|
946
|
+
// Apply pagination
|
|
934
947
|
const paginatedNouns = collectedNouns.slice(offset, offset + limit);
|
|
935
|
-
const hasMore = collectedNouns.length > targetCount;
|
|
948
|
+
const hasMore = collectedNouns.length > targetCount;
|
|
936
949
|
return {
|
|
937
950
|
items: paginatedNouns,
|
|
938
|
-
totalCount: collectedNouns.length,
|
|
951
|
+
totalCount: collectedNouns.length,
|
|
939
952
|
hasMore,
|
|
940
953
|
nextCursor: hasMore && paginatedNouns.length > 0
|
|
941
954
|
? paginatedNouns[paginatedNouns.length - 1].id
|
|
@@ -962,58 +975,70 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
962
975
|
const { limit, offset = 0, filter } = options; // cursor intentionally not extracted (not yet implemented)
|
|
963
976
|
const collectedVerbs = [];
|
|
964
977
|
const targetCount = offset + limit; // Early termination target
|
|
965
|
-
//
|
|
966
|
-
const
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
continue;
|
|
980
|
-
}
|
|
981
|
-
}
|
|
978
|
+
// Prepare filter sets for efficient lookup
|
|
979
|
+
const filterVerbTypes = filter?.verbType
|
|
980
|
+
? new Set(Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType])
|
|
981
|
+
: null;
|
|
982
|
+
const filterSourceIds = filter?.sourceId
|
|
983
|
+
? new Set(Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId])
|
|
984
|
+
: null;
|
|
985
|
+
const filterTargetIds = filter?.targetId
|
|
986
|
+
? new Set(Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId])
|
|
987
|
+
: null;
|
|
988
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types - single pass!
|
|
989
|
+
for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
|
|
990
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
991
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
982
992
|
try {
|
|
983
|
-
const
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
// OPTIMIZATION 2: Early termination (stop when we have enough)
|
|
987
|
-
if (collectedVerbs.length >= targetCount) {
|
|
993
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
994
|
+
for (const verbPath of verbFiles) {
|
|
995
|
+
if (collectedVerbs.length >= targetCount)
|
|
988
996
|
break;
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
if (
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
997
|
+
if (!verbPath.includes('/vectors.json'))
|
|
998
|
+
continue;
|
|
999
|
+
try {
|
|
1000
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
1001
|
+
if (!rawVerb)
|
|
1002
|
+
continue;
|
|
1003
|
+
// v6.0.0: Deserialize connections Map from JSON storage format
|
|
1004
|
+
const verb = this.deserializeVerb(rawVerb);
|
|
1005
|
+
// Apply type filter
|
|
1006
|
+
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
|
|
1007
|
+
continue;
|
|
1000
1008
|
}
|
|
1001
|
-
//
|
|
1002
|
-
if (
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
}
|
|
1009
|
+
// Apply sourceId filter
|
|
1010
|
+
if (filterSourceIds && !filterSourceIds.has(verb.sourceId)) {
|
|
1011
|
+
continue;
|
|
1012
|
+
}
|
|
1013
|
+
// Apply targetId filter
|
|
1014
|
+
if (filterTargetIds && !filterTargetIds.has(verb.targetId)) {
|
|
1015
|
+
continue;
|
|
1009
1016
|
}
|
|
1017
|
+
// Load metadata
|
|
1018
|
+
const metadata = await this.getVerbMetadata(verb.id);
|
|
1019
|
+
// Combine verb + metadata
|
|
1020
|
+
collectedVerbs.push({
|
|
1021
|
+
...verb,
|
|
1022
|
+
weight: metadata?.weight,
|
|
1023
|
+
confidence: metadata?.confidence,
|
|
1024
|
+
createdAt: metadata?.createdAt
|
|
1025
|
+
? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
|
|
1026
|
+
: Date.now(),
|
|
1027
|
+
updatedAt: metadata?.updatedAt
|
|
1028
|
+
? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
|
|
1029
|
+
: Date.now(),
|
|
1030
|
+
service: metadata?.service,
|
|
1031
|
+
createdBy: metadata?.createdBy,
|
|
1032
|
+
metadata: metadata || {}
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
catch (error) {
|
|
1036
|
+
// Skip verbs that fail to load
|
|
1010
1037
|
}
|
|
1011
|
-
// Verb passed all filters - add to collection
|
|
1012
|
-
collectedVerbs.push(verb);
|
|
1013
1038
|
}
|
|
1014
1039
|
}
|
|
1015
1040
|
catch (error) {
|
|
1016
|
-
// Skip
|
|
1041
|
+
// Skip shards that have no data
|
|
1017
1042
|
}
|
|
1018
1043
|
}
|
|
1019
1044
|
// Apply pagination (v5.5.0: Efficient slicing after early termination)
|
|
@@ -1386,11 +1411,8 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1386
1411
|
*/
|
|
1387
1412
|
async saveNounMetadata_internal(id, metadata) {
|
|
1388
1413
|
await this.ensureInitialized();
|
|
1389
|
-
//
|
|
1390
|
-
const
|
|
1391
|
-
this.nounTypeCache.set(id, type);
|
|
1392
|
-
// v5.4.0: Use type-first path
|
|
1393
|
-
const path = getNounMetadataPath(type, id);
|
|
1414
|
+
// v6.0.0: ID-first path - no type needed!
|
|
1415
|
+
const path = getNounMetadataPath(id);
|
|
1394
1416
|
// Determine if this is a new entity by checking if metadata already exists
|
|
1395
1417
|
const existingMetadata = await this.readWithInheritance(path);
|
|
1396
1418
|
const isNew = !existingMetadata;
|
|
@@ -1411,6 +1433,17 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1411
1433
|
/**
|
|
1412
1434
|
* Get noun metadata from storage (METADATA-ONLY, NO VECTORS)
|
|
1413
1435
|
*
|
|
1436
|
+
* **Performance (v6.0.0)**: Direct O(1) ID-first lookup - NO type search needed!
|
|
1437
|
+
* - **All lookups**: 1 read, ~500ms on cloud (consistent performance)
|
|
1438
|
+
* - **No cache needed**: Type is in the metadata, not the path
|
|
1439
|
+
* - **No type search**: ID-first paths eliminate 42-type search entirely
|
|
1440
|
+
*
|
|
1441
|
+
* **Clean architecture (v6.0.0)**:
|
|
1442
|
+
* - Path: `entities/nouns/{SHARD}/{ID}/metadata.json`
|
|
1443
|
+
* - Type is just a field in metadata (`noun: "document"`)
|
|
1444
|
+
* - MetadataIndex handles type queries (no path scanning needed)
|
|
1445
|
+
* - Scales to billions without any overhead
|
|
1446
|
+
*
|
|
1414
1447
|
* **Performance (v5.11.1)**: Fast path for metadata-only reads
|
|
1415
1448
|
* - **Speed**: 10ms vs 43ms (76-81% faster than getNoun)
|
|
1416
1449
|
* - **Bandwidth**: 300 bytes vs 6KB (95% less)
|
|
@@ -1440,39 +1473,21 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1440
1473
|
* @returns Metadata or null if not found
|
|
1441
1474
|
*
|
|
1442
1475
|
* @performance
|
|
1443
|
-
* -
|
|
1444
|
-
* -
|
|
1445
|
-
* -
|
|
1476
|
+
* - O(1) direct ID lookup - always 1 read (~500ms on cloud, ~10ms local)
|
|
1477
|
+
* - No caching complexity
|
|
1478
|
+
* - No type search fallbacks
|
|
1479
|
+
* - Works in distributed systems without sync issues
|
|
1446
1480
|
*
|
|
1447
1481
|
* @since v4.0.0
|
|
1448
|
-
* @since v5.4.0 - Type-first paths
|
|
1482
|
+
* @since v5.4.0 - Type-first paths (removed in v6.0.0)
|
|
1449
1483
|
* @since v5.11.1 - Promoted to fast path for brain.get() optimization
|
|
1484
|
+
* @since v6.0.0 - CLEAN FIX: ID-first paths eliminate all type-search complexity
|
|
1450
1485
|
*/
|
|
1451
1486
|
async getNounMetadata(id) {
|
|
1452
1487
|
await this.ensureInitialized();
|
|
1453
|
-
//
|
|
1454
|
-
const
|
|
1455
|
-
|
|
1456
|
-
const path = getNounMetadataPath(cachedType, id);
|
|
1457
|
-
return this.readWithInheritance(path);
|
|
1458
|
-
}
|
|
1459
|
-
// Fallback: search across all types (expensive but necessary if cache miss)
|
|
1460
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1461
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1462
|
-
const path = getNounMetadataPath(type, id);
|
|
1463
|
-
try {
|
|
1464
|
-
const metadata = await this.readWithInheritance(path);
|
|
1465
|
-
if (metadata) {
|
|
1466
|
-
// Cache the type for next time
|
|
1467
|
-
this.nounTypeCache.set(id, type);
|
|
1468
|
-
return metadata;
|
|
1469
|
-
}
|
|
1470
|
-
}
|
|
1471
|
-
catch (error) {
|
|
1472
|
-
// Not in this type, continue searching
|
|
1473
|
-
}
|
|
1474
|
-
}
|
|
1475
|
-
return null;
|
|
1488
|
+
// v6.0.0: Clean, simple, O(1) lookup - no type needed!
|
|
1489
|
+
const path = getNounMetadataPath(id);
|
|
1490
|
+
return this.readWithInheritance(path);
|
|
1476
1491
|
}
|
|
1477
1492
|
/**
|
|
1478
1493
|
* Batch fetch noun metadata from storage (v5.12.0 - Cloud Storage Optimization)
|
|
@@ -1511,74 +1526,19 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1511
1526
|
const results = new Map();
|
|
1512
1527
|
if (ids.length === 0)
|
|
1513
1528
|
return results;
|
|
1514
|
-
//
|
|
1515
|
-
|
|
1516
|
-
const
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
idsForType.push(id);
|
|
1522
|
-
idsByType.set(cachedType, idsForType);
|
|
1523
|
-
}
|
|
1524
|
-
else {
|
|
1525
|
-
uncachedIds.push(id);
|
|
1526
|
-
}
|
|
1527
|
-
}
|
|
1528
|
-
// Build paths for known types
|
|
1529
|
-
const pathsToFetch = [];
|
|
1530
|
-
for (const [type, typeIds] of idsByType.entries()) {
|
|
1531
|
-
for (const id of typeIds) {
|
|
1532
|
-
pathsToFetch.push({
|
|
1533
|
-
path: getNounMetadataPath(type, id),
|
|
1534
|
-
id
|
|
1535
|
-
});
|
|
1536
|
-
}
|
|
1537
|
-
}
|
|
1538
|
-
// For uncached IDs, we need to search across types (expensive but unavoidable)
|
|
1539
|
-
// Strategy: Try most common types first (Document, Thing, Person), then others
|
|
1540
|
-
const commonTypes = [NounType.Document, NounType.Thing, NounType.Person, NounType.File];
|
|
1541
|
-
const commonTypeSet = new Set(commonTypes);
|
|
1542
|
-
const otherTypes = [];
|
|
1543
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1544
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1545
|
-
if (!commonTypeSet.has(type)) {
|
|
1546
|
-
otherTypes.push(type);
|
|
1547
|
-
}
|
|
1548
|
-
}
|
|
1549
|
-
const searchOrder = [...commonTypes, ...otherTypes];
|
|
1550
|
-
for (const id of uncachedIds) {
|
|
1551
|
-
for (const type of searchOrder) {
|
|
1552
|
-
// Build path manually to avoid type issues
|
|
1553
|
-
const shard = getShardIdFromUuid(id);
|
|
1554
|
-
const path = `entities/nouns/${type}/metadata/${shard}/${id}.json`;
|
|
1555
|
-
pathsToFetch.push({ path, id });
|
|
1556
|
-
}
|
|
1557
|
-
}
|
|
1558
|
-
// Batch read all paths
|
|
1529
|
+
// v6.0.0: ID-first paths - no type grouping or search needed!
|
|
1530
|
+
// Build direct paths for all IDs
|
|
1531
|
+
const pathsToFetch = ids.map(id => ({
|
|
1532
|
+
path: getNounMetadataPath(id),
|
|
1533
|
+
id
|
|
1534
|
+
}));
|
|
1535
|
+
// Batch read all paths (uses adapter's native batch API or parallel fallback)
|
|
1559
1536
|
const batchResults = await this.readBatchWithInheritance(pathsToFetch.map(p => p.path));
|
|
1560
|
-
//
|
|
1561
|
-
const
|
|
1562
|
-
for (let i = 0; i < pathsToFetch.length; i++) {
|
|
1563
|
-
const { path, id } = pathsToFetch[i];
|
|
1537
|
+
// Map results back to IDs
|
|
1538
|
+
for (const { path, id } of pathsToFetch) {
|
|
1564
1539
|
const metadata = batchResults.get(path);
|
|
1565
1540
|
if (metadata) {
|
|
1566
1541
|
results.set(id, metadata);
|
|
1567
|
-
// Cache the type for uncached IDs (only on first find)
|
|
1568
|
-
if (uncachedIds.includes(id) && !foundUncached.has(id)) {
|
|
1569
|
-
// Extract type from path: "entities/nouns/metadata/{type}/{shard}/{id}.json"
|
|
1570
|
-
const parts = path.split('/');
|
|
1571
|
-
const typeStr = parts[3]; // "document", "thing", etc.
|
|
1572
|
-
// Find matching type by string comparison
|
|
1573
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1574
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1575
|
-
if (type === typeStr) {
|
|
1576
|
-
this.nounTypeCache.set(id, type);
|
|
1577
|
-
break;
|
|
1578
|
-
}
|
|
1579
|
-
}
|
|
1580
|
-
foundUncached.add(id);
|
|
1581
|
-
}
|
|
1582
1542
|
}
|
|
1583
1543
|
}
|
|
1584
1544
|
return results;
|
|
@@ -1736,36 +1696,13 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1736
1696
|
};
|
|
1737
1697
|
}
|
|
1738
1698
|
/**
|
|
1739
|
-
* Delete noun metadata from storage
|
|
1740
|
-
* v5.4.0: Uses type-first paths (must match saveNounMetadata_internal)
|
|
1699
|
+
* Delete noun metadata from storage (v6.0.0: ID-first, O(1) delete)
|
|
1741
1700
|
*/
|
|
1742
1701
|
async deleteNounMetadata(id) {
|
|
1743
1702
|
await this.ensureInitialized();
|
|
1744
|
-
//
|
|
1745
|
-
const
|
|
1746
|
-
|
|
1747
|
-
const path = getNounMetadataPath(cachedType, id);
|
|
1748
|
-
await this.deleteObjectFromBranch(path);
|
|
1749
|
-
// Remove from cache after deletion
|
|
1750
|
-
this.nounTypeCache.delete(id);
|
|
1751
|
-
return;
|
|
1752
|
-
}
|
|
1753
|
-
// If not in cache, search all types to find and delete
|
|
1754
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
1755
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
1756
|
-
const path = getNounMetadataPath(type, id);
|
|
1757
|
-
try {
|
|
1758
|
-
// Check if exists before deleting
|
|
1759
|
-
const exists = await this.readWithInheritance(path);
|
|
1760
|
-
if (exists) {
|
|
1761
|
-
await this.deleteObjectFromBranch(path);
|
|
1762
|
-
return;
|
|
1763
|
-
}
|
|
1764
|
-
}
|
|
1765
|
-
catch (error) {
|
|
1766
|
-
// Not in this type, continue searching
|
|
1767
|
-
}
|
|
1768
|
-
}
|
|
1703
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
1704
|
+
const path = getNounMetadataPath(id);
|
|
1705
|
+
await this.deleteObjectFromBranch(path);
|
|
1769
1706
|
}
|
|
1770
1707
|
/**
|
|
1771
1708
|
* Save verb metadata to storage (v4.0.0: now typed)
|
|
@@ -1777,7 +1714,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1777
1714
|
}
|
|
1778
1715
|
/**
|
|
1779
1716
|
* Internal method for saving verb metadata (v4.0.0: now typed)
|
|
1780
|
-
* v5.4.0: Uses
|
|
1717
|
+
* v5.4.0: Uses ID-first paths (must match getVerbMetadata)
|
|
1781
1718
|
*
|
|
1782
1719
|
* CRITICAL (v4.1.2): Count synchronization happens here
|
|
1783
1720
|
* This ensures verb counts are updated AFTER metadata exists, fixing the race condition
|
|
@@ -1789,7 +1726,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1789
1726
|
*/
|
|
1790
1727
|
async saveVerbMetadata_internal(id, metadata) {
|
|
1791
1728
|
await this.ensureInitialized();
|
|
1792
|
-
// v5.4.0: Extract verb type from metadata for
|
|
1729
|
+
// v5.4.0: Extract verb type from metadata for ID-first path
|
|
1793
1730
|
const verbType = metadata.verb;
|
|
1794
1731
|
if (!verbType) {
|
|
1795
1732
|
// Backward compatibility: fallback to old path if no verb type
|
|
@@ -1797,15 +1734,14 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1797
1734
|
await this.writeObjectToBranch(keyInfo.fullPath, metadata);
|
|
1798
1735
|
return;
|
|
1799
1736
|
}
|
|
1800
|
-
// v5.4.0: Use
|
|
1801
|
-
const path = getVerbMetadataPath(
|
|
1737
|
+
// v5.4.0: Use ID-first path
|
|
1738
|
+
const path = getVerbMetadataPath(id);
|
|
1802
1739
|
// Determine if this is a new verb by checking if metadata already exists
|
|
1803
1740
|
const existingMetadata = await this.readWithInheritance(path);
|
|
1804
1741
|
const isNew = !existingMetadata;
|
|
1805
1742
|
// Save the metadata (COW-aware - writes to branch-specific path)
|
|
1806
1743
|
await this.writeObjectToBranch(path, metadata);
|
|
1807
1744
|
// v5.4.0: Cache verb type for faster lookups
|
|
1808
|
-
this.verbTypeCache.set(id, verbType);
|
|
1809
1745
|
// CRITICAL FIX (v4.1.2): Increment verb count for new relationships
|
|
1810
1746
|
// This runs AFTER metadata is saved
|
|
1811
1747
|
// Uses synchronous increment since storage operations are already serialized
|
|
@@ -1820,69 +1756,34 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1820
1756
|
}
|
|
1821
1757
|
/**
|
|
1822
1758
|
* Get verb metadata from storage (v4.0.0: now typed)
|
|
1823
|
-
* v5.4.0: Uses
|
|
1759
|
+
* v5.4.0: Uses ID-first paths (must match saveVerbMetadata_internal)
|
|
1824
1760
|
*/
|
|
1825
1761
|
async getVerbMetadata(id) {
|
|
1826
1762
|
await this.ensureInitialized();
|
|
1827
|
-
//
|
|
1828
|
-
const
|
|
1829
|
-
|
|
1830
|
-
const
|
|
1831
|
-
return
|
|
1832
|
-
}
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
const path = getVerbMetadataPath(type, id);
|
|
1837
|
-
try {
|
|
1838
|
-
const metadata = await this.readWithInheritance(path);
|
|
1839
|
-
if (metadata) {
|
|
1840
|
-
// Cache the type for next time
|
|
1841
|
-
this.verbTypeCache.set(id, type);
|
|
1842
|
-
return metadata;
|
|
1843
|
-
}
|
|
1844
|
-
}
|
|
1845
|
-
catch (error) {
|
|
1846
|
-
// Not in this type, continue searching
|
|
1847
|
-
}
|
|
1763
|
+
// v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
|
|
1764
|
+
const path = getVerbMetadataPath(id);
|
|
1765
|
+
try {
|
|
1766
|
+
const metadata = await this.readWithInheritance(path);
|
|
1767
|
+
return metadata || null;
|
|
1768
|
+
}
|
|
1769
|
+
catch (error) {
|
|
1770
|
+
// Entity not found
|
|
1771
|
+
return null;
|
|
1848
1772
|
}
|
|
1849
|
-
return null;
|
|
1850
1773
|
}
|
|
1851
1774
|
/**
|
|
1852
|
-
* Delete verb metadata from storage
|
|
1853
|
-
* v5.4.0: Uses type-first paths (must match saveVerbMetadata_internal)
|
|
1775
|
+
* Delete verb metadata from storage (v6.0.0: ID-first, O(1) delete)
|
|
1854
1776
|
*/
|
|
1855
1777
|
async deleteVerbMetadata(id) {
|
|
1856
1778
|
await this.ensureInitialized();
|
|
1857
|
-
//
|
|
1858
|
-
const
|
|
1859
|
-
|
|
1860
|
-
const path = getVerbMetadataPath(cachedType, id);
|
|
1861
|
-
await this.deleteObjectFromBranch(path);
|
|
1862
|
-
// Remove from cache after deletion
|
|
1863
|
-
this.verbTypeCache.delete(id);
|
|
1864
|
-
return;
|
|
1865
|
-
}
|
|
1866
|
-
// If not in cache, search all types to find and delete
|
|
1867
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
1868
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
1869
|
-
const path = getVerbMetadataPath(type, id);
|
|
1870
|
-
try {
|
|
1871
|
-
// Check if exists before deleting
|
|
1872
|
-
const exists = await this.readWithInheritance(path);
|
|
1873
|
-
if (exists) {
|
|
1874
|
-
await this.deleteObjectFromBranch(path);
|
|
1875
|
-
return;
|
|
1876
|
-
}
|
|
1877
|
-
}
|
|
1878
|
-
catch (error) {
|
|
1879
|
-
// Not in this type, continue searching
|
|
1880
|
-
}
|
|
1881
|
-
}
|
|
1779
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
1780
|
+
const path = getVerbMetadataPath(id);
|
|
1781
|
+
await this.deleteObjectFromBranch(path);
|
|
1882
1782
|
}
|
|
1883
1783
|
// ============================================================================
|
|
1884
|
-
//
|
|
1885
|
-
//
|
|
1784
|
+
// ID-FIRST HELPER METHODS (v6.0.0)
|
|
1785
|
+
// Direct O(1) ID lookups - no type needed!
|
|
1786
|
+
// Clean, simple architecture for billion-scale performance
|
|
1886
1787
|
// ============================================================================
|
|
1887
1788
|
/**
|
|
1888
1789
|
* Load type statistics from storage
|
|
@@ -1924,30 +1825,61 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1924
1825
|
*/
|
|
1925
1826
|
async rebuildTypeCounts() {
|
|
1926
1827
|
prodLog.info('[BaseStorage] Rebuilding type counts from storage...');
|
|
1927
|
-
// Rebuild
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1828
|
+
// v6.0.0: Rebuild by scanning shards (0x00-0xFF) and reading metadata
|
|
1829
|
+
this.nounCountsByType = new Uint32Array(NOUN_TYPE_COUNT);
|
|
1830
|
+
this.verbCountsByType = new Uint32Array(VERB_TYPE_COUNT);
|
|
1831
|
+
// Scan noun shards
|
|
1832
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
1833
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
1834
|
+
const shardDir = `entities/nouns/${shardHex}`;
|
|
1931
1835
|
try {
|
|
1932
|
-
const paths = await this.listObjectsInBranch(
|
|
1933
|
-
|
|
1836
|
+
const paths = await this.listObjectsInBranch(shardDir);
|
|
1837
|
+
for (const path of paths) {
|
|
1838
|
+
if (!path.includes('/metadata.json'))
|
|
1839
|
+
continue;
|
|
1840
|
+
try {
|
|
1841
|
+
const metadata = await this.readWithInheritance(path);
|
|
1842
|
+
if (metadata && metadata.noun) {
|
|
1843
|
+
const typeIndex = TypeUtils.getNounIndex(metadata.noun);
|
|
1844
|
+
if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
|
|
1845
|
+
this.nounCountsByType[typeIndex]++;
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
catch (error) {
|
|
1850
|
+
// Skip entities that fail to load
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1934
1853
|
}
|
|
1935
1854
|
catch (error) {
|
|
1936
|
-
//
|
|
1937
|
-
this.verbCountsByType[i] = 0;
|
|
1855
|
+
// Skip shards that don't exist
|
|
1938
1856
|
}
|
|
1939
1857
|
}
|
|
1940
|
-
//
|
|
1941
|
-
for (let
|
|
1942
|
-
const
|
|
1943
|
-
const
|
|
1858
|
+
// Scan verb shards
|
|
1859
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
1860
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
1861
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
1944
1862
|
try {
|
|
1945
|
-
const paths = await this.listObjectsInBranch(
|
|
1946
|
-
|
|
1863
|
+
const paths = await this.listObjectsInBranch(shardDir);
|
|
1864
|
+
for (const path of paths) {
|
|
1865
|
+
if (!path.includes('/metadata.json'))
|
|
1866
|
+
continue;
|
|
1867
|
+
try {
|
|
1868
|
+
const metadata = await this.readWithInheritance(path);
|
|
1869
|
+
if (metadata && metadata.verb) {
|
|
1870
|
+
const typeIndex = TypeUtils.getVerbIndex(metadata.verb);
|
|
1871
|
+
if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
|
|
1872
|
+
this.verbCountsByType[typeIndex]++;
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
catch (error) {
|
|
1877
|
+
// Skip entities that fail to load
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1947
1880
|
}
|
|
1948
1881
|
catch (error) {
|
|
1949
|
-
//
|
|
1950
|
-
this.nounCountsByType[i] = 0;
|
|
1882
|
+
// Skip shards that don't exist
|
|
1951
1883
|
}
|
|
1952
1884
|
}
|
|
1953
1885
|
// Save rebuilt counts to storage
|
|
@@ -1957,18 +1889,13 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
1957
1889
|
prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs`);
|
|
1958
1890
|
}
|
|
1959
1891
|
/**
|
|
1960
|
-
* Get noun type
|
|
1961
|
-
*
|
|
1892
|
+
* Get noun type (v6.0.0: type no longer needed for paths!)
|
|
1893
|
+
* With ID-first paths, this is only used for internal statistics tracking.
|
|
1894
|
+
* The actual type is stored in metadata and indexed by MetadataIndexManager.
|
|
1962
1895
|
*/
|
|
1963
1896
|
getNounType(noun) {
|
|
1964
|
-
//
|
|
1965
|
-
|
|
1966
|
-
if (cached) {
|
|
1967
|
-
return cached;
|
|
1968
|
-
}
|
|
1969
|
-
// Default to 'thing' if unknown
|
|
1970
|
-
// This should only happen if saveNoun_internal is called before saveNounMetadata
|
|
1971
|
-
prodLog.warn(`[BaseStorage] Unknown noun type for ${noun.id}, defaulting to 'thing'`);
|
|
1897
|
+
// v6.0.0: Type cache removed - default to 'thing' for statistics
|
|
1898
|
+
// The real type is in metadata, accessible via getNounMetadata(id)
|
|
1972
1899
|
return 'thing';
|
|
1973
1900
|
}
|
|
1974
1901
|
/**
|
|
@@ -2051,15 +1978,14 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2051
1978
|
// Converted from abstract to concrete - all adapters now have built-in type-aware
|
|
2052
1979
|
// ============================================================================
|
|
2053
1980
|
/**
|
|
2054
|
-
* Save a noun to storage (
|
|
1981
|
+
* Save a noun to storage (ID-first path)
|
|
2055
1982
|
*/
|
|
2056
1983
|
async saveNoun_internal(noun) {
|
|
2057
1984
|
const type = this.getNounType(noun);
|
|
2058
|
-
const path = getNounVectorPath(
|
|
1985
|
+
const path = getNounVectorPath(noun.id);
|
|
2059
1986
|
// Update type tracking
|
|
2060
1987
|
const typeIndex = TypeUtils.getNounIndex(type);
|
|
2061
1988
|
this.nounCountsByType[typeIndex]++;
|
|
2062
|
-
this.nounTypeCache.set(noun.id, type);
|
|
2063
1989
|
// COW-aware write (v5.0.1): Use COW helper for branch isolation
|
|
2064
1990
|
await this.writeObjectToBranch(path, noun);
|
|
2065
1991
|
// Periodically save statistics (every 100 saves)
|
|
@@ -2068,120 +1994,89 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2068
1994
|
}
|
|
2069
1995
|
}
|
|
2070
1996
|
/**
|
|
2071
|
-
* Get a noun from storage (
|
|
1997
|
+
* Get a noun from storage (ID-first path)
|
|
2072
1998
|
*/
|
|
2073
1999
|
async getNoun_internal(id) {
|
|
2074
|
-
//
|
|
2075
|
-
const
|
|
2076
|
-
|
|
2077
|
-
const path = getNounVectorPath(cachedType, id);
|
|
2000
|
+
// v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
|
|
2001
|
+
const path = getNounVectorPath(id);
|
|
2002
|
+
try {
|
|
2078
2003
|
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2079
|
-
const
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
// Need to search across all types (expensive, but cached after first access)
|
|
2084
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
2085
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
2086
|
-
const path = getNounVectorPath(type, id);
|
|
2087
|
-
try {
|
|
2088
|
-
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2089
|
-
const noun = await this.readWithInheritance(path);
|
|
2090
|
-
if (noun) {
|
|
2091
|
-
// Cache the type for next time
|
|
2092
|
-
this.nounTypeCache.set(id, type);
|
|
2093
|
-
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2094
|
-
return this.deserializeNoun(noun);
|
|
2095
|
-
}
|
|
2096
|
-
}
|
|
2097
|
-
catch (error) {
|
|
2098
|
-
// Not in this type, continue searching
|
|
2004
|
+
const noun = await this.readWithInheritance(path);
|
|
2005
|
+
if (noun) {
|
|
2006
|
+
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2007
|
+
return this.deserializeNoun(noun);
|
|
2099
2008
|
}
|
|
2100
2009
|
}
|
|
2010
|
+
catch (error) {
|
|
2011
|
+
// Entity not found
|
|
2012
|
+
return null;
|
|
2013
|
+
}
|
|
2101
2014
|
return null;
|
|
2102
2015
|
}
|
|
2103
2016
|
/**
|
|
2104
|
-
* Get nouns by noun type (
|
|
2017
|
+
* Get nouns by noun type (v6.0.0: Shard-based iteration!)
|
|
2105
2018
|
*/
|
|
2106
2019
|
async getNounsByNounType_internal(nounType) {
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
// COW-aware list (v5.0.1): Use COW helper for branch isolation
|
|
2110
|
-
const paths = await this.listObjectsInBranch(prefix);
|
|
2111
|
-
// Load all nouns of this type
|
|
2020
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types
|
|
2021
|
+
// Type is stored in metadata.noun field, we filter as we load
|
|
2112
2022
|
const nouns = [];
|
|
2113
|
-
for (
|
|
2023
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2024
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2025
|
+
const shardDir = `entities/nouns/${shardHex}`;
|
|
2114
2026
|
try {
|
|
2115
|
-
|
|
2116
|
-
const
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2027
|
+
const nounFiles = await this.listObjectsInBranch(shardDir);
|
|
2028
|
+
for (const nounPath of nounFiles) {
|
|
2029
|
+
if (!nounPath.includes('/vectors.json'))
|
|
2030
|
+
continue;
|
|
2031
|
+
try {
|
|
2032
|
+
const noun = await this.readWithInheritance(nounPath);
|
|
2033
|
+
if (noun) {
|
|
2034
|
+
const deserialized = this.deserializeNoun(noun);
|
|
2035
|
+
// Check type from metadata
|
|
2036
|
+
const metadata = await this.getNounMetadata(deserialized.id);
|
|
2037
|
+
if (metadata && metadata.noun === nounType) {
|
|
2038
|
+
nouns.push(deserialized);
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
catch (error) {
|
|
2043
|
+
// Skip nouns that fail to load
|
|
2044
|
+
}
|
|
2122
2045
|
}
|
|
2123
2046
|
}
|
|
2124
2047
|
catch (error) {
|
|
2125
|
-
|
|
2048
|
+
// Skip shards that have no data
|
|
2126
2049
|
}
|
|
2127
2050
|
}
|
|
2128
2051
|
return nouns;
|
|
2129
2052
|
}
|
|
2130
2053
|
/**
|
|
2131
|
-
* Delete a noun from storage (
|
|
2054
|
+
* Delete a noun from storage (v6.0.0: ID-first, O(1) delete)
|
|
2132
2055
|
*/
|
|
2133
2056
|
async deleteNoun_internal(id) {
|
|
2134
|
-
//
|
|
2135
|
-
const
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
await this.deleteObjectFromBranch(path);
|
|
2140
|
-
// Update counts
|
|
2141
|
-
const typeIndex = TypeUtils.getNounIndex(cachedType);
|
|
2142
|
-
if (this.nounCountsByType[typeIndex] > 0) {
|
|
2143
|
-
this.nounCountsByType[typeIndex]--;
|
|
2144
|
-
}
|
|
2145
|
-
this.nounTypeCache.delete(id);
|
|
2146
|
-
return;
|
|
2147
|
-
}
|
|
2148
|
-
// Search across all types
|
|
2149
|
-
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
|
|
2150
|
-
const type = TypeUtils.getNounFromIndex(i);
|
|
2151
|
-
const path = getNounVectorPath(type, id);
|
|
2152
|
-
try {
|
|
2153
|
-
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
|
|
2154
|
-
await this.deleteObjectFromBranch(path);
|
|
2155
|
-
// Update counts
|
|
2156
|
-
if (this.nounCountsByType[i] > 0) {
|
|
2157
|
-
this.nounCountsByType[i]--;
|
|
2158
|
-
}
|
|
2159
|
-
this.nounTypeCache.delete(id);
|
|
2160
|
-
return;
|
|
2161
|
-
}
|
|
2162
|
-
catch (error) {
|
|
2163
|
-
// Not in this type, continue
|
|
2164
|
-
}
|
|
2165
|
-
}
|
|
2057
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
2058
|
+
const path = getNounVectorPath(id);
|
|
2059
|
+
await this.deleteObjectFromBranch(path);
|
|
2060
|
+
// Note: Type-specific counts will be decremented via metadata tracking
|
|
2061
|
+
// The real type is in metadata, accessible if needed via getNounMetadata(id)
|
|
2166
2062
|
}
|
|
2167
2063
|
/**
|
|
2168
|
-
* Save a verb to storage (
|
|
2064
|
+
* Save a verb to storage (ID-first path)
|
|
2169
2065
|
*/
|
|
2170
2066
|
async saveVerb_internal(verb) {
|
|
2171
2067
|
// Type is now a first-class field in HNSWVerb - no caching needed!
|
|
2172
2068
|
const type = verb.verb;
|
|
2173
|
-
const path = getVerbVectorPath(
|
|
2069
|
+
const path = getVerbVectorPath(verb.id);
|
|
2070
|
+
prodLog.debug(`[BaseStorage] saveVerb_internal: id=${verb.id}, sourceId=${verb.sourceId}, targetId=${verb.targetId}, type=${type}`);
|
|
2174
2071
|
// Update type tracking
|
|
2175
2072
|
const typeIndex = TypeUtils.getVerbIndex(type);
|
|
2176
2073
|
this.verbCountsByType[typeIndex]++;
|
|
2177
|
-
this.verbTypeCache.set(verb.id, type);
|
|
2178
2074
|
// COW-aware write (v5.0.1): Use COW helper for branch isolation
|
|
2179
2075
|
await this.writeObjectToBranch(path, verb);
|
|
2180
|
-
//
|
|
2181
|
-
//
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
// Fast incremental update - no rebuild needed
|
|
2076
|
+
// v6.0.0: Update GraphAdjacencyIndex incrementally (always available after init())
|
|
2077
|
+
// GraphAdjacencyIndex.addVerb() calls ensureInitialized() automatically
|
|
2078
|
+
if (this.graphIndex) {
|
|
2079
|
+
prodLog.debug(`[BaseStorage] Updating GraphAdjacencyIndex with verb ${verb.id}`);
|
|
2185
2080
|
await this.graphIndex.addVerb({
|
|
2186
2081
|
id: verb.id,
|
|
2187
2082
|
sourceId: verb.sourceId,
|
|
@@ -2193,8 +2088,12 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2193
2088
|
type: verb.verb,
|
|
2194
2089
|
createdAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
|
|
2195
2090
|
updatedAt: { seconds: Math.floor(Date.now() / 1000), nanoseconds: 0 },
|
|
2196
|
-
createdBy: { augmentation: 'storage', version: '
|
|
2091
|
+
createdBy: { augmentation: 'storage', version: '6.0.0' }
|
|
2197
2092
|
});
|
|
2093
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex updated successfully`);
|
|
2094
|
+
}
|
|
2095
|
+
else {
|
|
2096
|
+
prodLog.warn(`[BaseStorage] graphIndex is null, cannot update index for verb ${verb.id}`);
|
|
2198
2097
|
}
|
|
2199
2098
|
// Periodically save statistics
|
|
2200
2099
|
if (this.verbCountsByType[typeIndex] % 100 === 0) {
|
|
@@ -2202,69 +2101,89 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2202
2101
|
}
|
|
2203
2102
|
}
|
|
2204
2103
|
/**
|
|
2205
|
-
* Get a verb from storage (
|
|
2104
|
+
* Get a verb from storage (ID-first path)
|
|
2206
2105
|
*/
|
|
2207
2106
|
async getVerb_internal(id) {
|
|
2208
|
-
//
|
|
2209
|
-
const
|
|
2210
|
-
|
|
2211
|
-
const path = getVerbVectorPath(cachedType, id);
|
|
2107
|
+
// v6.0.0: Direct O(1) lookup with ID-first paths - no type search needed!
|
|
2108
|
+
const path = getVerbVectorPath(id);
|
|
2109
|
+
try {
|
|
2212
2110
|
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2213
2111
|
const verb = await this.readWithInheritance(path);
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
// Search across all types (only on first access)
|
|
2218
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
2219
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
2220
|
-
const path = getVerbVectorPath(type, id);
|
|
2221
|
-
try {
|
|
2222
|
-
// COW-aware read (v5.0.1): Use COW helper for branch isolation
|
|
2223
|
-
const verb = await this.readWithInheritance(path);
|
|
2224
|
-
if (verb) {
|
|
2225
|
-
// Cache the type for next time (read from verb.verb field)
|
|
2226
|
-
this.verbTypeCache.set(id, verb.verb);
|
|
2227
|
-
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2228
|
-
return this.deserializeVerb(verb);
|
|
2229
|
-
}
|
|
2230
|
-
}
|
|
2231
|
-
catch (error) {
|
|
2232
|
-
// Not in this type, continue
|
|
2112
|
+
if (verb) {
|
|
2113
|
+
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2114
|
+
return this.deserializeVerb(verb);
|
|
2233
2115
|
}
|
|
2234
2116
|
}
|
|
2117
|
+
catch (error) {
|
|
2118
|
+
// Entity not found
|
|
2119
|
+
return null;
|
|
2120
|
+
}
|
|
2235
2121
|
return null;
|
|
2236
2122
|
}
|
|
2237
2123
|
/**
|
|
2238
|
-
* Get verbs by source (
|
|
2239
|
-
*
|
|
2124
|
+
* Get verbs by source (v6.0.0: Uses GraphAdjacencyIndex when available)
|
|
2125
|
+
* Falls back to shard iteration during initialization to avoid circular dependency
|
|
2240
2126
|
*/
|
|
2241
2127
|
async getVerbsBySource_internal(sourceId) {
|
|
2242
|
-
// v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
|
|
2243
|
-
// v5.7.0 called getGraphIndex() here, creating deadlock during initialization:
|
|
2244
|
-
// GraphAdjacencyIndex.rebuild() → storage.getVerbs() → getVerbsBySource_internal() → getGraphIndex() → [deadlock]
|
|
2245
|
-
// v5.4.0: Type-first implementation - scan across all verb types
|
|
2246
|
-
// COW-aware: uses readWithInheritance for each verb
|
|
2247
2128
|
await this.ensureInitialized();
|
|
2129
|
+
prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`);
|
|
2130
|
+
// v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
|
|
2131
|
+
if (this.graphIndex && this.graphIndex.isInitialized) {
|
|
2132
|
+
try {
|
|
2133
|
+
const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId);
|
|
2134
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`);
|
|
2135
|
+
const results = [];
|
|
2136
|
+
for (const verbId of verbIds) {
|
|
2137
|
+
const verb = await this.getVerb_internal(verbId);
|
|
2138
|
+
const metadata = await this.getVerbMetadata(verbId);
|
|
2139
|
+
if (verb && metadata) {
|
|
2140
|
+
results.push({
|
|
2141
|
+
...verb,
|
|
2142
|
+
weight: metadata.weight,
|
|
2143
|
+
confidence: metadata.confidence,
|
|
2144
|
+
createdAt: metadata.createdAt
|
|
2145
|
+
? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
|
|
2146
|
+
: Date.now(),
|
|
2147
|
+
updatedAt: metadata.updatedAt
|
|
2148
|
+
? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
|
|
2149
|
+
: Date.now(),
|
|
2150
|
+
service: metadata.service,
|
|
2151
|
+
createdBy: metadata.createdBy,
|
|
2152
|
+
metadata: metadata || {}
|
|
2153
|
+
});
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex path returned ${results.length} verbs`);
|
|
2157
|
+
return results;
|
|
2158
|
+
}
|
|
2159
|
+
catch (error) {
|
|
2160
|
+
prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
// v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
|
|
2164
|
+
prodLog.debug(`[BaseStorage] Using shard iteration fallback for sourceId=${sourceId}`);
|
|
2248
2165
|
const results = [];
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
const
|
|
2166
|
+
let shardsScanned = 0;
|
|
2167
|
+
let verbsFound = 0;
|
|
2168
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2169
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2170
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2253
2171
|
try {
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
const verbFiles = await this.listObjectsInBranch(typeDir);
|
|
2172
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2173
|
+
shardsScanned++;
|
|
2257
2174
|
for (const verbPath of verbFiles) {
|
|
2258
|
-
|
|
2259
|
-
if (!verbPath.endsWith('.json'))
|
|
2175
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2260
2176
|
continue;
|
|
2261
2177
|
try {
|
|
2262
|
-
const
|
|
2263
|
-
if (
|
|
2264
|
-
|
|
2265
|
-
|
|
2178
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
2179
|
+
if (!rawVerb)
|
|
2180
|
+
continue;
|
|
2181
|
+
verbsFound++;
|
|
2182
|
+
// v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
|
|
2183
|
+
const verb = this.deserializeVerb(rawVerb);
|
|
2184
|
+
if (verb.sourceId === sourceId) {
|
|
2185
|
+
const metadataPath = getVerbMetadataPath(verb.id);
|
|
2266
2186
|
const metadata = await this.readWithInheritance(metadataPath);
|
|
2267
|
-
// v5.4.0: Extract standard fields from metadata to top-level (like nouns)
|
|
2268
2187
|
results.push({
|
|
2269
2188
|
...verb,
|
|
2270
2189
|
weight: metadata?.weight,
|
|
@@ -2283,13 +2202,15 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2283
2202
|
}
|
|
2284
2203
|
catch (error) {
|
|
2285
2204
|
// Skip verbs that fail to load
|
|
2205
|
+
prodLog.debug(`[BaseStorage] Failed to load verb from ${verbPath}:`, error);
|
|
2286
2206
|
}
|
|
2287
2207
|
}
|
|
2288
2208
|
}
|
|
2289
2209
|
catch (error) {
|
|
2290
|
-
// Skip
|
|
2210
|
+
// Skip shards that have no data
|
|
2291
2211
|
}
|
|
2292
2212
|
}
|
|
2213
|
+
prodLog.debug(`[BaseStorage] Shard iteration: scanned ${shardsScanned} shards, found ${verbsFound} total verbs, matched ${results.length} for sourceId=${sourceId}`);
|
|
2293
2214
|
return results;
|
|
2294
2215
|
}
|
|
2295
2216
|
/**
|
|
@@ -2335,52 +2256,46 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2335
2256
|
}
|
|
2336
2257
|
// Convert sourceIds to Set for O(1) lookup
|
|
2337
2258
|
const sourceIdSet = new Set(sourceIds);
|
|
2338
|
-
//
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
}
|
|
2343
|
-
else {
|
|
2344
|
-
// Scan all verb types
|
|
2345
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
2346
|
-
typesToScan.push(TypeUtils.getVerbFromIndex(i));
|
|
2347
|
-
}
|
|
2348
|
-
}
|
|
2349
|
-
// Scan verb types and collect matching verbs
|
|
2350
|
-
for (const type of typesToScan) {
|
|
2351
|
-
const typeDir = `entities/verbs/${type}/vectors`;
|
|
2259
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of types
|
|
2260
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2261
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2262
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2352
2263
|
try {
|
|
2353
|
-
// List all verb files
|
|
2354
|
-
const verbFiles = await this.listObjectsInBranch(
|
|
2264
|
+
// List all verb files in this shard
|
|
2265
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2355
2266
|
// Build paths for batch read
|
|
2356
2267
|
const verbPaths = [];
|
|
2357
2268
|
const metadataPaths = [];
|
|
2358
2269
|
const pathToId = new Map();
|
|
2359
2270
|
for (const verbPath of verbFiles) {
|
|
2360
|
-
if (!verbPath.
|
|
2271
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2361
2272
|
continue;
|
|
2362
2273
|
verbPaths.push(verbPath);
|
|
2363
|
-
// Extract ID from path: "entities/verbs/{
|
|
2274
|
+
// Extract ID from path: "entities/verbs/{shard}/{id}/vector.json"
|
|
2364
2275
|
const parts = verbPath.split('/');
|
|
2365
|
-
const
|
|
2366
|
-
const verbId = filename.replace('.json', '');
|
|
2276
|
+
const verbId = parts[parts.length - 2]; // ID is second-to-last segment
|
|
2367
2277
|
pathToId.set(verbPath, verbId);
|
|
2368
2278
|
// Prepare metadata path
|
|
2369
|
-
metadataPaths.push(getVerbMetadataPath(
|
|
2279
|
+
metadataPaths.push(getVerbMetadataPath(verbId));
|
|
2370
2280
|
}
|
|
2371
|
-
// Batch read all verb files for this
|
|
2281
|
+
// Batch read all verb files for this shard
|
|
2372
2282
|
const verbDataMap = await this.readBatchWithInheritance(verbPaths);
|
|
2373
2283
|
const metadataMap = await this.readBatchWithInheritance(metadataPaths);
|
|
2374
2284
|
// Process results
|
|
2375
|
-
for (const [verbPath,
|
|
2376
|
-
if (!
|
|
2285
|
+
for (const [verbPath, rawVerbData] of verbDataMap.entries()) {
|
|
2286
|
+
if (!rawVerbData || !rawVerbData.sourceId)
|
|
2377
2287
|
continue;
|
|
2288
|
+
// v6.0.0: Deserialize connections Map from JSON storage format
|
|
2289
|
+
const verbData = this.deserializeVerb(rawVerbData);
|
|
2378
2290
|
// Check if this verb's source is in our requested set
|
|
2379
2291
|
if (!sourceIdSet.has(verbData.sourceId))
|
|
2380
2292
|
continue;
|
|
2293
|
+
// If verbType specified, filter by type
|
|
2294
|
+
if (verbType && verbData.verb !== verbType)
|
|
2295
|
+
continue;
|
|
2381
2296
|
// Found matching verb - hydrate with metadata
|
|
2382
2297
|
const verbId = pathToId.get(verbPath);
|
|
2383
|
-
const metadataPath = getVerbMetadataPath(
|
|
2298
|
+
const metadataPath = getVerbMetadataPath(verbId);
|
|
2384
2299
|
const metadata = metadataMap.get(metadataPath) || {};
|
|
2385
2300
|
const hydratedVerb = {
|
|
2386
2301
|
...verbData,
|
|
@@ -2402,7 +2317,7 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2402
2317
|
}
|
|
2403
2318
|
}
|
|
2404
2319
|
catch (error) {
|
|
2405
|
-
// Skip
|
|
2320
|
+
// Skip shards that have no data
|
|
2406
2321
|
}
|
|
2407
2322
|
}
|
|
2408
2323
|
return results;
|
|
@@ -2413,31 +2328,57 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2413
2328
|
* v5.4.0: Fixed to directly list verb files instead of directories
|
|
2414
2329
|
*/
|
|
2415
2330
|
async getVerbsByTarget_internal(targetId) {
|
|
2416
|
-
// v5.7.1: Reverted to v5.6.3 implementation to fix circular dependency deadlock
|
|
2417
|
-
// v5.7.0 called getGraphIndex() here, creating deadlock during initialization
|
|
2418
|
-
// v5.4.0: Type-first implementation - scan across all verb types
|
|
2419
|
-
// COW-aware: uses readWithInheritance for each verb
|
|
2420
2331
|
await this.ensureInitialized();
|
|
2332
|
+
// v6.0.0: Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
|
|
2333
|
+
if (this.graphIndex && this.graphIndex.isInitialized) {
|
|
2334
|
+
try {
|
|
2335
|
+
const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId);
|
|
2336
|
+
const results = [];
|
|
2337
|
+
for (const verbId of verbIds) {
|
|
2338
|
+
const verb = await this.getVerb_internal(verbId);
|
|
2339
|
+
const metadata = await this.getVerbMetadata(verbId);
|
|
2340
|
+
if (verb && metadata) {
|
|
2341
|
+
results.push({
|
|
2342
|
+
...verb,
|
|
2343
|
+
weight: metadata.weight,
|
|
2344
|
+
confidence: metadata.confidence,
|
|
2345
|
+
createdAt: metadata.createdAt
|
|
2346
|
+
? (typeof metadata.createdAt === 'number' ? metadata.createdAt : metadata.createdAt.seconds * 1000)
|
|
2347
|
+
: Date.now(),
|
|
2348
|
+
updatedAt: metadata.updatedAt
|
|
2349
|
+
? (typeof metadata.updatedAt === 'number' ? metadata.updatedAt : metadata.updatedAt.seconds * 1000)
|
|
2350
|
+
: Date.now(),
|
|
2351
|
+
service: metadata.service,
|
|
2352
|
+
createdBy: metadata.createdBy,
|
|
2353
|
+
metadata: metadata || {}
|
|
2354
|
+
});
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
return results;
|
|
2358
|
+
}
|
|
2359
|
+
catch (error) {
|
|
2360
|
+
prodLog.warn('[BaseStorage] GraphAdjacencyIndex lookup failed, falling back to shard iteration:', error);
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
// v6.0.0: Fallback - iterate by shards (WITH deserialization fix!)
|
|
2421
2364
|
const results = [];
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
const
|
|
2425
|
-
const typeDir = `entities/verbs/${type}/vectors`;
|
|
2365
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2366
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2367
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2426
2368
|
try {
|
|
2427
|
-
|
|
2428
|
-
// listObjectsInBranch returns full paths to .json files, not directories
|
|
2429
|
-
const verbFiles = await this.listObjectsInBranch(typeDir);
|
|
2369
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2430
2370
|
for (const verbPath of verbFiles) {
|
|
2431
|
-
|
|
2432
|
-
if (!verbPath.endsWith('.json'))
|
|
2371
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2433
2372
|
continue;
|
|
2434
2373
|
try {
|
|
2435
|
-
const
|
|
2436
|
-
if (
|
|
2437
|
-
|
|
2438
|
-
|
|
2374
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
2375
|
+
if (!rawVerb)
|
|
2376
|
+
continue;
|
|
2377
|
+
// v6.0.0: CRITICAL - Deserialize connections Map from JSON storage format
|
|
2378
|
+
const verb = this.deserializeVerb(rawVerb);
|
|
2379
|
+
if (verb.targetId === targetId) {
|
|
2380
|
+
const metadataPath = getVerbMetadataPath(verb.id);
|
|
2439
2381
|
const metadata = await this.readWithInheritance(metadataPath);
|
|
2440
|
-
// v5.4.0: Extract standard fields from metadata to top-level (like nouns)
|
|
2441
2382
|
results.push({
|
|
2442
2383
|
...verb,
|
|
2443
2384
|
weight: metadata?.weight,
|
|
@@ -2460,95 +2401,77 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
2460
2401
|
}
|
|
2461
2402
|
}
|
|
2462
2403
|
catch (error) {
|
|
2463
|
-
// Skip
|
|
2404
|
+
// Skip shards that have no data
|
|
2464
2405
|
}
|
|
2465
2406
|
}
|
|
2466
2407
|
return results;
|
|
2467
2408
|
}
|
|
2468
2409
|
/**
|
|
2469
|
-
* Get verbs by type (
|
|
2410
|
+
* Get verbs by type (v6.0.0: Shard iteration with type filtering)
|
|
2470
2411
|
*/
|
|
2471
2412
|
async getVerbsByType_internal(verbType) {
|
|
2472
|
-
|
|
2473
|
-
const prefix = `entities/verbs/${type}/vectors/`;
|
|
2474
|
-
// COW-aware list (v5.0.1): Use COW helper for branch isolation
|
|
2475
|
-
const paths = await this.listObjectsInBranch(prefix);
|
|
2413
|
+
// v6.0.0: Iterate by shards (0x00-0xFF) instead of type-first paths
|
|
2476
2414
|
const verbs = [];
|
|
2477
|
-
for (
|
|
2415
|
+
for (let shard = 0; shard < 256; shard++) {
|
|
2416
|
+
const shardHex = shard.toString(16).padStart(2, '0');
|
|
2417
|
+
const shardDir = `entities/verbs/${shardHex}`;
|
|
2478
2418
|
try {
|
|
2479
|
-
|
|
2480
|
-
const
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2419
|
+
const verbFiles = await this.listObjectsInBranch(shardDir);
|
|
2420
|
+
for (const verbPath of verbFiles) {
|
|
2421
|
+
if (!verbPath.includes('/vectors.json'))
|
|
2422
|
+
continue;
|
|
2423
|
+
try {
|
|
2424
|
+
const rawVerb = await this.readWithInheritance(verbPath);
|
|
2425
|
+
if (!rawVerb)
|
|
2426
|
+
continue;
|
|
2427
|
+
// v5.7.10: Deserialize connections Map from JSON storage format
|
|
2428
|
+
const hnswVerb = this.deserializeVerb(rawVerb);
|
|
2429
|
+
// Filter by verb type
|
|
2430
|
+
if (hnswVerb.verb !== verbType)
|
|
2431
|
+
continue;
|
|
2432
|
+
// Load metadata separately (optional in v4.0.0!)
|
|
2433
|
+
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
2434
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
2435
|
+
const metadataObj = (metadata || {});
|
|
2436
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
2437
|
+
const verbWithMetadata = {
|
|
2438
|
+
id: hnswVerb.id,
|
|
2439
|
+
vector: [...hnswVerb.vector],
|
|
2440
|
+
connections: hnswVerb.connections, // v5.7.10: Already deserialized
|
|
2441
|
+
verb: hnswVerb.verb,
|
|
2442
|
+
sourceId: hnswVerb.sourceId,
|
|
2443
|
+
targetId: hnswVerb.targetId,
|
|
2444
|
+
createdAt: createdAt || Date.now(),
|
|
2445
|
+
updatedAt: updatedAt || Date.now(),
|
|
2446
|
+
confidence: confidence,
|
|
2447
|
+
weight: weight,
|
|
2448
|
+
service: service,
|
|
2449
|
+
data: data,
|
|
2450
|
+
createdBy,
|
|
2451
|
+
metadata: customMetadata
|
|
2452
|
+
};
|
|
2453
|
+
verbs.push(verbWithMetadata);
|
|
2454
|
+
}
|
|
2455
|
+
catch (error) {
|
|
2456
|
+
// Skip verbs that fail to load
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2511
2459
|
}
|
|
2512
2460
|
catch (error) {
|
|
2513
|
-
|
|
2461
|
+
// Skip shards that have no data
|
|
2514
2462
|
}
|
|
2515
2463
|
}
|
|
2516
2464
|
return verbs;
|
|
2517
2465
|
}
|
|
2518
2466
|
/**
|
|
2519
|
-
* Delete a verb from storage (
|
|
2467
|
+
* Delete a verb from storage (v6.0.0: ID-first, O(1) delete)
|
|
2520
2468
|
*/
|
|
2521
2469
|
async deleteVerb_internal(id) {
|
|
2522
|
-
//
|
|
2523
|
-
const
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
await this.deleteObjectFromBranch(path);
|
|
2528
|
-
const typeIndex = TypeUtils.getVerbIndex(cachedType);
|
|
2529
|
-
if (this.verbCountsByType[typeIndex] > 0) {
|
|
2530
|
-
this.verbCountsByType[typeIndex]--;
|
|
2531
|
-
}
|
|
2532
|
-
this.verbTypeCache.delete(id);
|
|
2533
|
-
return;
|
|
2534
|
-
}
|
|
2535
|
-
// Search across all types
|
|
2536
|
-
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
|
|
2537
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
2538
|
-
const path = getVerbVectorPath(type, id);
|
|
2539
|
-
try {
|
|
2540
|
-
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
|
|
2541
|
-
await this.deleteObjectFromBranch(path);
|
|
2542
|
-
if (this.verbCountsByType[i] > 0) {
|
|
2543
|
-
this.verbCountsByType[i]--;
|
|
2544
|
-
}
|
|
2545
|
-
this.verbTypeCache.delete(id);
|
|
2546
|
-
return;
|
|
2547
|
-
}
|
|
2548
|
-
catch (error) {
|
|
2549
|
-
// Continue
|
|
2550
|
-
}
|
|
2551
|
-
}
|
|
2470
|
+
// v6.0.0: Direct O(1) delete with ID-first path
|
|
2471
|
+
const path = getVerbVectorPath(id);
|
|
2472
|
+
await this.deleteObjectFromBranch(path);
|
|
2473
|
+
// Note: Type-specific counts will be decremented via metadata tracking
|
|
2474
|
+
// The real type is in metadata, accessible if needed via getVerbMetadata(id)
|
|
2552
2475
|
}
|
|
2553
2476
|
/**
|
|
2554
2477
|
* Helper method to convert a Map to a plain object for serialization
|