@soulcraft/brainy 4.7.4 → 4.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/DataAPI.js +3 -3
- package/dist/brainy.d.ts +7 -2
- package/dist/brainy.js +69 -39
- package/dist/coreTypes.d.ts +64 -14
- package/dist/coreTypes.js +3 -1
- package/dist/graph/graphAdjacencyIndex.js +38 -2
- package/dist/neural/embeddedTypeEmbeddings.d.ts +1 -1
- package/dist/neural/embeddedTypeEmbeddings.js +2 -2
- package/dist/storage/adapters/azureBlobStorage.js +68 -9
- package/dist/storage/adapters/fileSystemStorage.js +41 -16
- package/dist/storage/adapters/gcsStorage.js +24 -4
- package/dist/storage/adapters/memoryStorage.js +30 -5
- package/dist/storage/adapters/opfsStorage.js +24 -4
- package/dist/storage/adapters/r2Storage.js +13 -2
- package/dist/storage/adapters/s3CompatibleStorage.js +24 -3
- package/dist/storage/adapters/typeAwareStorageAdapter.js +27 -95
- package/dist/storage/baseStorage.js +43 -6
- package/dist/types/brainy.types.d.ts +4 -0
- package/dist/types/graphTypes.d.ts +1 -0
- package/dist/utils/entityIdMapper.js +3 -2
- package/dist/utils/metadataIndex.d.ts +23 -2
- package/dist/utils/metadataIndex.js +43 -12
- package/package.json +1 -1
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* File System Storage Adapter
|
|
3
3
|
* File system storage adapter for Node.js environments
|
|
4
4
|
*/
|
|
5
|
+
import { NounType } from '../../coreTypes.js';
|
|
5
6
|
import { BaseStorage, SYSTEM_DIR, STATISTICS_KEY } from '../baseStorage.js';
|
|
6
7
|
// Node.js modules - dynamically imported to avoid issues in browser environments
|
|
7
8
|
let fs;
|
|
@@ -768,13 +769,23 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
768
769
|
}
|
|
769
770
|
connections = connectionsMap;
|
|
770
771
|
}
|
|
771
|
-
// v4.
|
|
772
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
773
|
+
const metadataObj = (metadata || {});
|
|
774
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj;
|
|
772
775
|
const nounWithMetadata = {
|
|
773
776
|
id: parsedNoun.id,
|
|
774
777
|
vector: parsedNoun.vector,
|
|
775
778
|
connections: connections,
|
|
776
779
|
level: parsedNoun.level || 0,
|
|
777
|
-
|
|
780
|
+
type: nounType || NounType.Thing,
|
|
781
|
+
createdAt: createdAt || Date.now(),
|
|
782
|
+
updatedAt: updatedAt || Date.now(),
|
|
783
|
+
confidence: confidence,
|
|
784
|
+
weight: weight,
|
|
785
|
+
service: service,
|
|
786
|
+
data: dataField,
|
|
787
|
+
createdBy,
|
|
788
|
+
metadata: customMetadata
|
|
778
789
|
};
|
|
779
790
|
items.push(nounWithMetadata);
|
|
780
791
|
successfullyLoaded++;
|
|
@@ -1124,11 +1135,9 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
1124
1135
|
const edge = JSON.parse(data);
|
|
1125
1136
|
// Get metadata which contains the actual verb information
|
|
1126
1137
|
const metadata = await this.getVerbMetadata(id);
|
|
1127
|
-
// v4.
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
continue;
|
|
1131
|
-
}
|
|
1138
|
+
// v4.8.1: Don't skip verbs without metadata - metadata is optional
|
|
1139
|
+
// FIX: This was the root cause of the VFS bug (11 versions)
|
|
1140
|
+
// Verbs can exist without metadata files (e.g., from imports/migrations)
|
|
1132
1141
|
// Convert connections Map to proper format if needed
|
|
1133
1142
|
let connections = edge.connections;
|
|
1134
1143
|
if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
|
|
@@ -1138,7 +1147,9 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
1138
1147
|
}
|
|
1139
1148
|
connections = connectionsMap;
|
|
1140
1149
|
}
|
|
1141
|
-
// v4.
|
|
1150
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1151
|
+
const metadataObj = (metadata || {});
|
|
1152
|
+
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj;
|
|
1142
1153
|
const verbWithMetadata = {
|
|
1143
1154
|
id: edge.id,
|
|
1144
1155
|
vector: edge.vector,
|
|
@@ -1146,7 +1157,14 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
1146
1157
|
verb: edge.verb,
|
|
1147
1158
|
sourceId: edge.sourceId,
|
|
1148
1159
|
targetId: edge.targetId,
|
|
1149
|
-
|
|
1160
|
+
createdAt: createdAt || Date.now(),
|
|
1161
|
+
updatedAt: updatedAt || Date.now(),
|
|
1162
|
+
confidence: confidence,
|
|
1163
|
+
weight: weight,
|
|
1164
|
+
service: service,
|
|
1165
|
+
data: dataField,
|
|
1166
|
+
createdBy,
|
|
1167
|
+
metadata: customMetadata
|
|
1150
1168
|
};
|
|
1151
1169
|
// Apply filters if provided
|
|
1152
1170
|
if (options.filter) {
|
|
@@ -2005,11 +2023,9 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
2005
2023
|
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
2006
2024
|
const edge = JSON.parse(data);
|
|
2007
2025
|
const metadata = await this.getVerbMetadata(id);
|
|
2008
|
-
// v4.
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
return true; // continue, skip this verb
|
|
2012
|
-
}
|
|
2026
|
+
// v4.8.1: Don't skip verbs without metadata - metadata is optional
|
|
2027
|
+
// FIX: This was the root cause of the VFS bug (11 versions)
|
|
2028
|
+
// Verbs can exist without metadata files (e.g., from imports/migrations)
|
|
2013
2029
|
// Convert connections if needed
|
|
2014
2030
|
let connections = edge.connections;
|
|
2015
2031
|
if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
|
|
@@ -2019,7 +2035,9 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
2019
2035
|
}
|
|
2020
2036
|
connections = connectionsMap;
|
|
2021
2037
|
}
|
|
2022
|
-
// v4.
|
|
2038
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
2039
|
+
const metadataObj = (metadata || {});
|
|
2040
|
+
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj;
|
|
2023
2041
|
const verbWithMetadata = {
|
|
2024
2042
|
id: edge.id,
|
|
2025
2043
|
vector: edge.vector,
|
|
@@ -2027,7 +2045,14 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
2027
2045
|
verb: edge.verb,
|
|
2028
2046
|
sourceId: edge.sourceId,
|
|
2029
2047
|
targetId: edge.targetId,
|
|
2030
|
-
|
|
2048
|
+
createdAt: createdAt || Date.now(),
|
|
2049
|
+
updatedAt: updatedAt || Date.now(),
|
|
2050
|
+
confidence: confidence,
|
|
2051
|
+
weight: weight,
|
|
2052
|
+
service: service,
|
|
2053
|
+
data: dataField,
|
|
2054
|
+
createdBy,
|
|
2055
|
+
metadata: customMetadata
|
|
2031
2056
|
};
|
|
2032
2057
|
// Apply filters
|
|
2033
2058
|
if (options.filter) {
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* 3. Service Account Credentials Object
|
|
9
9
|
* 4. HMAC Keys (fallback for backward compatibility)
|
|
10
10
|
*/
|
|
11
|
+
import { NounType } from '../../coreTypes.js';
|
|
11
12
|
import { BaseStorage, SYSTEM_DIR, STATISTICS_KEY, getDirectoryPath } from '../baseStorage.js';
|
|
12
13
|
import { BrainyError } from '../../errors/brainyError.js';
|
|
13
14
|
import { CacheManager } from '../cacheManager.js';
|
|
@@ -832,13 +833,23 @@ export class GcsStorage extends BaseStorage {
|
|
|
832
833
|
continue;
|
|
833
834
|
}
|
|
834
835
|
}
|
|
835
|
-
//
|
|
836
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
837
|
+
const metadataObj = (metadata || {});
|
|
838
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
836
839
|
const nounWithMetadata = {
|
|
837
840
|
id: node.id,
|
|
838
841
|
vector: [...node.vector],
|
|
839
842
|
connections: new Map(node.connections),
|
|
840
843
|
level: node.level || 0,
|
|
841
|
-
|
|
844
|
+
type: nounType || NounType.Thing,
|
|
845
|
+
createdAt: createdAt || Date.now(),
|
|
846
|
+
updatedAt: updatedAt || Date.now(),
|
|
847
|
+
confidence: confidence,
|
|
848
|
+
weight: weight,
|
|
849
|
+
service: service,
|
|
850
|
+
data: data,
|
|
851
|
+
createdBy,
|
|
852
|
+
metadata: customMetadata
|
|
842
853
|
};
|
|
843
854
|
items.push(nounWithMetadata);
|
|
844
855
|
}
|
|
@@ -1076,7 +1087,9 @@ export class GcsStorage extends BaseStorage {
|
|
|
1076
1087
|
continue;
|
|
1077
1088
|
}
|
|
1078
1089
|
}
|
|
1079
|
-
//
|
|
1090
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1091
|
+
const metadataObj = (metadata || {});
|
|
1092
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
1080
1093
|
const verbWithMetadata = {
|
|
1081
1094
|
id: hnswVerb.id,
|
|
1082
1095
|
vector: [...hnswVerb.vector],
|
|
@@ -1084,7 +1097,14 @@ export class GcsStorage extends BaseStorage {
|
|
|
1084
1097
|
verb: hnswVerb.verb,
|
|
1085
1098
|
sourceId: hnswVerb.sourceId,
|
|
1086
1099
|
targetId: hnswVerb.targetId,
|
|
1087
|
-
|
|
1100
|
+
createdAt: createdAt || Date.now(),
|
|
1101
|
+
updatedAt: updatedAt || Date.now(),
|
|
1102
|
+
confidence: confidence,
|
|
1103
|
+
weight: weight,
|
|
1104
|
+
service: service,
|
|
1105
|
+
data: data,
|
|
1106
|
+
createdBy,
|
|
1107
|
+
metadata: customMetadata
|
|
1088
1108
|
};
|
|
1089
1109
|
items.push(verbWithMetadata);
|
|
1090
1110
|
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Memory Storage Adapter
|
|
3
3
|
* In-memory storage adapter for environments where persistent storage is not available or needed
|
|
4
4
|
*/
|
|
5
|
+
import { NounType } from '../../coreTypes.js';
|
|
5
6
|
import { BaseStorage } from '../baseStorage.js';
|
|
6
7
|
// No type aliases needed - using the original types directly
|
|
7
8
|
/**
|
|
@@ -154,13 +155,26 @@ export class MemoryStorage extends BaseStorage {
|
|
|
154
155
|
// Get metadata from separate storage
|
|
155
156
|
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
|
|
156
157
|
const metadata = await this.getNounMetadata(id);
|
|
157
|
-
// v4.
|
|
158
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
159
|
+
const metadataObj = (metadata || {});
|
|
160
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
161
|
+
// v4.8.0: Create HNSWNounWithMetadata with standard fields at top-level
|
|
158
162
|
const nounWithMetadata = {
|
|
159
163
|
id: noun.id,
|
|
160
164
|
vector: [...noun.vector],
|
|
161
165
|
connections: new Map(),
|
|
162
166
|
level: noun.level || 0,
|
|
163
|
-
|
|
167
|
+
// v4.8.0: Standard fields at top-level
|
|
168
|
+
type: nounType || NounType.Thing,
|
|
169
|
+
createdAt: createdAt || Date.now(),
|
|
170
|
+
updatedAt: updatedAt || Date.now(),
|
|
171
|
+
confidence: confidence,
|
|
172
|
+
weight: weight,
|
|
173
|
+
service: service,
|
|
174
|
+
data: data,
|
|
175
|
+
createdBy,
|
|
176
|
+
// Only custom user fields in metadata
|
|
177
|
+
metadata: customMetadata
|
|
164
178
|
};
|
|
165
179
|
// Copy connections
|
|
166
180
|
for (const [level, connections] of noun.connections.entries()) {
|
|
@@ -359,7 +373,10 @@ export class MemoryStorage extends BaseStorage {
|
|
|
359
373
|
// FIX v4.7.4: Don't skip verbs without metadata - metadata is optional in v4.0.0
|
|
360
374
|
// Core fields (verb, sourceId, targetId) are in HNSWVerb itself
|
|
361
375
|
const metadata = await this.getVerbMetadata(id);
|
|
362
|
-
// v4.
|
|
376
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
377
|
+
const metadataObj = metadata || {};
|
|
378
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
379
|
+
// v4.8.0: Create HNSWVerbWithMetadata with standard fields at top-level
|
|
363
380
|
const verbWithMetadata = {
|
|
364
381
|
id: hnswVerb.id,
|
|
365
382
|
vector: [...hnswVerb.vector],
|
|
@@ -368,8 +385,16 @@ export class MemoryStorage extends BaseStorage {
|
|
|
368
385
|
verb: hnswVerb.verb,
|
|
369
386
|
sourceId: hnswVerb.sourceId,
|
|
370
387
|
targetId: hnswVerb.targetId,
|
|
371
|
-
//
|
|
372
|
-
|
|
388
|
+
// v4.8.0: Standard fields at top-level
|
|
389
|
+
createdAt: createdAt || Date.now(),
|
|
390
|
+
updatedAt: updatedAt || Date.now(),
|
|
391
|
+
confidence: confidence,
|
|
392
|
+
weight: weight,
|
|
393
|
+
service: service,
|
|
394
|
+
data: data,
|
|
395
|
+
createdBy,
|
|
396
|
+
// Only custom user fields in metadata
|
|
397
|
+
metadata: customMetadata
|
|
373
398
|
};
|
|
374
399
|
// Copy connections
|
|
375
400
|
for (const [level, connections] of hnswVerb.connections.entries()) {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* OPFS (Origin Private File System) Storage Adapter
|
|
3
3
|
* Provides persistent storage for the vector database using the Origin Private File System API
|
|
4
4
|
*/
|
|
5
|
+
import { NounType } from '../../coreTypes.js';
|
|
5
6
|
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, NOUN_METADATA_DIR, VERB_METADATA_DIR, INDEX_DIR } from '../baseStorage.js';
|
|
6
7
|
import { getShardIdFromUuid } from '../sharding.js';
|
|
7
8
|
import '../../types/fileSystemTypes.js';
|
|
@@ -1446,13 +1447,23 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1446
1447
|
continue;
|
|
1447
1448
|
}
|
|
1448
1449
|
}
|
|
1449
|
-
// v4.
|
|
1450
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1451
|
+
const metadataObj = (metadata || {});
|
|
1452
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
1450
1453
|
const nounWithMetadata = {
|
|
1451
1454
|
id: noun.id,
|
|
1452
1455
|
vector: [...noun.vector],
|
|
1453
1456
|
connections: new Map(noun.connections),
|
|
1454
1457
|
level: noun.level || 0,
|
|
1455
|
-
|
|
1458
|
+
type: nounType || NounType.Thing,
|
|
1459
|
+
createdAt: createdAt || Date.now(),
|
|
1460
|
+
updatedAt: updatedAt || Date.now(),
|
|
1461
|
+
confidence: confidence,
|
|
1462
|
+
weight: weight,
|
|
1463
|
+
service: service,
|
|
1464
|
+
data: data,
|
|
1465
|
+
createdBy,
|
|
1466
|
+
metadata: customMetadata
|
|
1456
1467
|
};
|
|
1457
1468
|
items.push(nounWithMetadata);
|
|
1458
1469
|
}
|
|
@@ -1572,7 +1583,9 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1572
1583
|
continue;
|
|
1573
1584
|
}
|
|
1574
1585
|
}
|
|
1575
|
-
// v4.
|
|
1586
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1587
|
+
const metadataObj = (metadata || {});
|
|
1588
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
1576
1589
|
const verbWithMetadata = {
|
|
1577
1590
|
id: hnswVerb.id,
|
|
1578
1591
|
vector: [...hnswVerb.vector],
|
|
@@ -1580,7 +1593,14 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1580
1593
|
verb: hnswVerb.verb,
|
|
1581
1594
|
sourceId: hnswVerb.sourceId,
|
|
1582
1595
|
targetId: hnswVerb.targetId,
|
|
1583
|
-
|
|
1596
|
+
createdAt: createdAt || Date.now(),
|
|
1597
|
+
updatedAt: updatedAt || Date.now(),
|
|
1598
|
+
confidence: confidence,
|
|
1599
|
+
weight: weight,
|
|
1600
|
+
service: service,
|
|
1601
|
+
data: data,
|
|
1602
|
+
createdBy,
|
|
1603
|
+
metadata: customMetadata
|
|
1584
1604
|
};
|
|
1585
1605
|
items.push(verbWithMetadata);
|
|
1586
1606
|
}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Based on latest GCS and S3 implementations with R2-specific enhancements
|
|
13
13
|
*/
|
|
14
|
+
import { NounType } from '../../coreTypes.js';
|
|
14
15
|
import { BaseStorage, SYSTEM_DIR, STATISTICS_KEY, getDirectoryPath } from '../baseStorage.js';
|
|
15
16
|
import { BrainyError } from '../../errors/brainyError.js';
|
|
16
17
|
import { CacheManager } from '../cacheManager.js';
|
|
@@ -918,13 +919,23 @@ export class R2Storage extends BaseStorage {
|
|
|
918
919
|
continue;
|
|
919
920
|
}
|
|
920
921
|
}
|
|
921
|
-
// v4.
|
|
922
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
923
|
+
const metadataObj = (metadata || {});
|
|
924
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
922
925
|
const nounWithMetadata = {
|
|
923
926
|
id: noun.id,
|
|
924
927
|
vector: [...noun.vector],
|
|
925
928
|
connections: new Map(noun.connections),
|
|
926
929
|
level: noun.level || 0,
|
|
927
|
-
|
|
930
|
+
type: nounType || NounType.Thing,
|
|
931
|
+
createdAt: createdAt || Date.now(),
|
|
932
|
+
updatedAt: updatedAt || Date.now(),
|
|
933
|
+
confidence: confidence,
|
|
934
|
+
weight: weight,
|
|
935
|
+
service: service,
|
|
936
|
+
data: data,
|
|
937
|
+
createdBy,
|
|
938
|
+
metadata: customMetadata
|
|
928
939
|
};
|
|
929
940
|
items.push(nounWithMetadata);
|
|
930
941
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Uses the AWS S3 client to interact with S3-compatible storage services
|
|
4
4
|
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
|
5
5
|
*/
|
|
6
|
+
import { NounType } from '../../coreTypes.js';
|
|
6
7
|
import { BaseStorage, INDEX_DIR, SYSTEM_DIR, STATISTICS_KEY, getDirectoryPath } from '../baseStorage.js';
|
|
7
8
|
import { StorageCompatibilityLayer } from '../backwardCompatibility.js';
|
|
8
9
|
import { StorageOperationExecutors } from '../../utils/operationUtils.js';
|
|
@@ -1479,6 +1480,9 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1479
1480
|
const verbsWithMetadata = [];
|
|
1480
1481
|
for (const hnswVerb of result.edges) {
|
|
1481
1482
|
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
1483
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1484
|
+
const metadataObj = (metadata || {});
|
|
1485
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
1482
1486
|
const verbWithMetadata = {
|
|
1483
1487
|
id: hnswVerb.id,
|
|
1484
1488
|
vector: [...hnswVerb.vector],
|
|
@@ -1486,7 +1490,14 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1486
1490
|
verb: hnswVerb.verb,
|
|
1487
1491
|
sourceId: hnswVerb.sourceId,
|
|
1488
1492
|
targetId: hnswVerb.targetId,
|
|
1489
|
-
|
|
1493
|
+
createdAt: createdAt || Date.now(),
|
|
1494
|
+
updatedAt: updatedAt || Date.now(),
|
|
1495
|
+
confidence: confidence,
|
|
1496
|
+
weight: weight,
|
|
1497
|
+
service: service,
|
|
1498
|
+
data: data,
|
|
1499
|
+
createdBy,
|
|
1500
|
+
metadata: customMetadata
|
|
1490
1501
|
};
|
|
1491
1502
|
verbsWithMetadata.push(verbWithMetadata);
|
|
1492
1503
|
}
|
|
@@ -2899,13 +2910,23 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
2899
2910
|
}
|
|
2900
2911
|
}
|
|
2901
2912
|
}
|
|
2902
|
-
//
|
|
2913
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
2914
|
+
const metadataObj = (metadata || {});
|
|
2915
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
2903
2916
|
const nounWithMetadata = {
|
|
2904
2917
|
id: node.id,
|
|
2905
2918
|
vector: [...node.vector],
|
|
2906
2919
|
connections: new Map(node.connections),
|
|
2907
2920
|
level: node.level || 0,
|
|
2908
|
-
|
|
2921
|
+
type: nounType || NounType.Thing,
|
|
2922
|
+
createdAt: createdAt || Date.now(),
|
|
2923
|
+
updatedAt: updatedAt || Date.now(),
|
|
2924
|
+
confidence: confidence,
|
|
2925
|
+
weight: weight,
|
|
2926
|
+
service: service,
|
|
2927
|
+
data: data,
|
|
2928
|
+
createdBy,
|
|
2929
|
+
metadata: customMetadata
|
|
2909
2930
|
};
|
|
2910
2931
|
nounsWithMetadata.push(nounWithMetadata);
|
|
2911
2932
|
}
|
|
@@ -335,105 +335,27 @@ export class TypeAwareStorageAdapter extends BaseStorage {
|
|
|
335
335
|
* Get verbs by source
|
|
336
336
|
*/
|
|
337
337
|
async getVerbsBySource_internal(sourceId) {
|
|
338
|
-
//
|
|
339
|
-
//
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
const hnswVerb = await this.u.readObjectFromPath(path);
|
|
352
|
-
if (!hnswVerb)
|
|
353
|
-
continue;
|
|
354
|
-
// Check sourceId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
|
|
355
|
-
if (hnswVerb.sourceId !== sourceId)
|
|
356
|
-
continue;
|
|
357
|
-
// Load metadata separately (optional in v4.0.0!)
|
|
358
|
-
// FIX: Don't skip verbs without metadata - metadata is optional!
|
|
359
|
-
// VFS relationships often have NO metadata (just verb/source/target)
|
|
360
|
-
const metadata = await this.getVerbMetadata(id);
|
|
361
|
-
// Create HNSWVerbWithMetadata (verbs don't have level field)
|
|
362
|
-
// Convert connections from plain object to Map<number, Set<string>>
|
|
363
|
-
const connectionsMap = new Map();
|
|
364
|
-
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
|
|
365
|
-
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
|
|
366
|
-
connectionsMap.set(Number(level), new Set(ids));
|
|
367
|
-
}
|
|
368
|
-
}
|
|
369
|
-
const verbWithMetadata = {
|
|
370
|
-
id: hnswVerb.id,
|
|
371
|
-
vector: [...hnswVerb.vector],
|
|
372
|
-
connections: connectionsMap,
|
|
373
|
-
verb: hnswVerb.verb,
|
|
374
|
-
sourceId: hnswVerb.sourceId,
|
|
375
|
-
targetId: hnswVerb.targetId,
|
|
376
|
-
metadata: metadata || {} // Empty metadata if none exists
|
|
377
|
-
};
|
|
378
|
-
verbs.push(verbWithMetadata);
|
|
379
|
-
}
|
|
380
|
-
catch (error) {
|
|
381
|
-
// Continue searching
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
return verbs;
|
|
338
|
+
// v4.8.1 PERFORMANCE FIX: Delegate to underlying storage instead of scanning all files
|
|
339
|
+
// Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
|
|
340
|
+
// This was the root cause of the 11-version VFS bug (timeouts/zero results)
|
|
341
|
+
//
|
|
342
|
+
// Underlying storage adapters have optimized implementations:
|
|
343
|
+
// - FileSystemStorage: Uses getVerbsWithPagination with sourceId filter
|
|
344
|
+
// - GcsStorage: Uses batch queries with prefix filtering
|
|
345
|
+
// - S3Storage: Uses listObjects with sourceId-based filtering
|
|
346
|
+
//
|
|
347
|
+
// Phase 1b TODO: Add graph adjacency index query for O(1) lookups:
|
|
348
|
+
// const verbIds = await this.graphIndex?.getOutgoingEdges(sourceId) || []
|
|
349
|
+
// return Promise.all(verbIds.map(id => this.getVerb(id)))
|
|
350
|
+
return this.underlying.getVerbsBySource(sourceId);
|
|
386
351
|
}
|
|
387
352
|
/**
|
|
388
353
|
* Get verbs by target
|
|
389
354
|
*/
|
|
390
355
|
async getVerbsByTarget_internal(targetId) {
|
|
391
|
-
//
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
const type = TypeUtils.getVerbFromIndex(i);
|
|
395
|
-
const prefix = `entities/verbs/${type}/vectors/`;
|
|
396
|
-
const paths = await this.u.listObjectsUnderPath(prefix);
|
|
397
|
-
for (const path of paths) {
|
|
398
|
-
try {
|
|
399
|
-
const id = path.split('/').pop()?.replace('.json', '');
|
|
400
|
-
if (!id)
|
|
401
|
-
continue;
|
|
402
|
-
// Load the HNSWVerb
|
|
403
|
-
const hnswVerb = await this.u.readObjectFromPath(path);
|
|
404
|
-
if (!hnswVerb)
|
|
405
|
-
continue;
|
|
406
|
-
// Check targetId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
|
|
407
|
-
if (hnswVerb.targetId !== targetId)
|
|
408
|
-
continue;
|
|
409
|
-
// Load metadata separately (optional in v4.0.0!)
|
|
410
|
-
// FIX: Don't skip verbs without metadata - metadata is optional!
|
|
411
|
-
const metadata = await this.getVerbMetadata(id);
|
|
412
|
-
// Create HNSWVerbWithMetadata (verbs don't have level field)
|
|
413
|
-
// Convert connections from plain object to Map<number, Set<string>>
|
|
414
|
-
const connectionsMap = new Map();
|
|
415
|
-
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
|
|
416
|
-
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
|
|
417
|
-
connectionsMap.set(Number(level), new Set(ids));
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
const verbWithMetadata = {
|
|
421
|
-
id: hnswVerb.id,
|
|
422
|
-
vector: [...hnswVerb.vector],
|
|
423
|
-
connections: connectionsMap,
|
|
424
|
-
verb: hnswVerb.verb,
|
|
425
|
-
sourceId: hnswVerb.sourceId,
|
|
426
|
-
targetId: hnswVerb.targetId,
|
|
427
|
-
metadata: metadata || {} // Empty metadata if none exists
|
|
428
|
-
};
|
|
429
|
-
verbs.push(verbWithMetadata);
|
|
430
|
-
}
|
|
431
|
-
catch (error) {
|
|
432
|
-
// Continue
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
return verbs;
|
|
356
|
+
// v4.8.1 PERFORMANCE FIX: Delegate to underlying storage (same as getVerbsBySource fix)
|
|
357
|
+
// Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
|
|
358
|
+
return this.underlying.getVerbsByTarget(targetId);
|
|
437
359
|
}
|
|
438
360
|
/**
|
|
439
361
|
* Get verbs by type (O(1) with type-first paths!)
|
|
@@ -463,6 +385,9 @@ export class TypeAwareStorageAdapter extends BaseStorage {
|
|
|
463
385
|
connectionsMap.set(Number(level), new Set(ids));
|
|
464
386
|
}
|
|
465
387
|
}
|
|
388
|
+
// v4.8.0: Extract standard fields from metadata to top-level
|
|
389
|
+
const metadataObj = (metadata || {});
|
|
390
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
466
391
|
const verbWithMetadata = {
|
|
467
392
|
id: hnswVerb.id,
|
|
468
393
|
vector: [...hnswVerb.vector],
|
|
@@ -470,7 +395,14 @@ export class TypeAwareStorageAdapter extends BaseStorage {
|
|
|
470
395
|
verb: hnswVerb.verb,
|
|
471
396
|
sourceId: hnswVerb.sourceId,
|
|
472
397
|
targetId: hnswVerb.targetId,
|
|
473
|
-
|
|
398
|
+
createdAt: createdAt || Date.now(),
|
|
399
|
+
updatedAt: updatedAt || Date.now(),
|
|
400
|
+
confidence: confidence,
|
|
401
|
+
weight: weight,
|
|
402
|
+
service: service,
|
|
403
|
+
data: data,
|
|
404
|
+
createdBy,
|
|
405
|
+
metadata: customMetadata
|
|
474
406
|
};
|
|
475
407
|
verbs.push(verbWithMetadata);
|
|
476
408
|
}
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js';
|
|
6
6
|
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
|
|
7
7
|
import { validateNounType, validateVerbType } from '../utils/typeValidation.js';
|
|
8
|
+
import { NounType } from '../types/graphTypes.js';
|
|
8
9
|
import { getShardIdFromUuid } from './sharding.js';
|
|
9
10
|
// Clean directory structure (v4.7.2+)
|
|
10
11
|
// All storage adapters use this consistent structure
|
|
@@ -46,6 +47,10 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
46
47
|
* @private
|
|
47
48
|
*/
|
|
48
49
|
analyzeKey(id, context) {
|
|
50
|
+
// v4.8.0: Guard against undefined/null IDs
|
|
51
|
+
if (!id || typeof id !== 'string') {
|
|
52
|
+
throw new Error(`Invalid storage key: ${id} (must be a non-empty string)`);
|
|
53
|
+
}
|
|
49
54
|
// System resource detection
|
|
50
55
|
const isSystemKey = id.startsWith('__metadata_') ||
|
|
51
56
|
id.startsWith('__index_') ||
|
|
@@ -142,13 +147,24 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
142
147
|
console.warn(`[Storage] Noun ${id} has vector but no metadata - this should not happen in v4.0.0`);
|
|
143
148
|
return null;
|
|
144
149
|
}
|
|
145
|
-
// Combine into HNSWNounWithMetadata
|
|
150
|
+
// Combine into HNSWNounWithMetadata - v4.8.0: Extract standard fields to top-level
|
|
151
|
+
const { noun, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
|
|
146
152
|
return {
|
|
147
153
|
id: vector.id,
|
|
148
154
|
vector: vector.vector,
|
|
149
155
|
connections: vector.connections,
|
|
150
156
|
level: vector.level,
|
|
151
|
-
|
|
157
|
+
// v4.8.0: Standard fields at top-level
|
|
158
|
+
type: noun || NounType.Thing,
|
|
159
|
+
createdAt: createdAt || Date.now(),
|
|
160
|
+
updatedAt: updatedAt || Date.now(),
|
|
161
|
+
confidence: confidence,
|
|
162
|
+
weight: weight,
|
|
163
|
+
service: service,
|
|
164
|
+
data: data,
|
|
165
|
+
createdBy,
|
|
166
|
+
// Only custom user fields remain in metadata
|
|
167
|
+
metadata: customMetadata
|
|
152
168
|
};
|
|
153
169
|
}
|
|
154
170
|
/**
|
|
@@ -160,14 +176,25 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
160
176
|
await this.ensureInitialized();
|
|
161
177
|
// Internal method returns HNSWNoun[], need to combine with metadata
|
|
162
178
|
const nouns = await this.getNounsByNounType_internal(nounType);
|
|
163
|
-
// Combine each noun with its metadata
|
|
179
|
+
// Combine each noun with its metadata - v4.8.0: Extract standard fields to top-level
|
|
164
180
|
const nounsWithMetadata = [];
|
|
165
181
|
for (const noun of nouns) {
|
|
166
182
|
const metadata = await this.getNounMetadata(noun.id);
|
|
167
183
|
if (metadata) {
|
|
184
|
+
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
|
|
168
185
|
nounsWithMetadata.push({
|
|
169
186
|
...noun,
|
|
170
|
-
|
|
187
|
+
// v4.8.0: Standard fields at top-level
|
|
188
|
+
type: nounType || NounType.Thing,
|
|
189
|
+
createdAt: createdAt || Date.now(),
|
|
190
|
+
updatedAt: updatedAt || Date.now(),
|
|
191
|
+
confidence: confidence,
|
|
192
|
+
weight: weight,
|
|
193
|
+
service: service,
|
|
194
|
+
data: data,
|
|
195
|
+
createdBy,
|
|
196
|
+
// Only custom user fields in metadata
|
|
197
|
+
metadata: customMetadata
|
|
171
198
|
});
|
|
172
199
|
}
|
|
173
200
|
}
|
|
@@ -220,7 +247,8 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
220
247
|
console.warn(`[Storage] Verb ${id} has vector but no metadata - this should not happen in v4.0.0`);
|
|
221
248
|
return null;
|
|
222
249
|
}
|
|
223
|
-
// Combine into HNSWVerbWithMetadata
|
|
250
|
+
// Combine into HNSWVerbWithMetadata - v4.8.0: Extract standard fields to top-level
|
|
251
|
+
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata;
|
|
224
252
|
return {
|
|
225
253
|
id: verb.id,
|
|
226
254
|
vector: verb.vector,
|
|
@@ -228,7 +256,16 @@ export class BaseStorage extends BaseStorageAdapter {
|
|
|
228
256
|
verb: verb.verb,
|
|
229
257
|
sourceId: verb.sourceId,
|
|
230
258
|
targetId: verb.targetId,
|
|
231
|
-
|
|
259
|
+
// v4.8.0: Standard fields at top-level
|
|
260
|
+
createdAt: createdAt || Date.now(),
|
|
261
|
+
updatedAt: updatedAt || Date.now(),
|
|
262
|
+
confidence: confidence,
|
|
263
|
+
weight: weight,
|
|
264
|
+
service: service,
|
|
265
|
+
data: data,
|
|
266
|
+
createdBy,
|
|
267
|
+
// Only custom user fields remain in metadata
|
|
268
|
+
metadata: customMetadata
|
|
232
269
|
};
|
|
233
270
|
}
|
|
234
271
|
/**
|