@soulcraft/brainy 5.3.6 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/dist/brainy.d.ts +61 -0
- package/dist/brainy.js +179 -23
- package/dist/storage/adapters/azureBlobStorage.d.ts +13 -64
- package/dist/storage/adapters/azureBlobStorage.js +78 -388
- package/dist/storage/adapters/fileSystemStorage.d.ts +12 -78
- package/dist/storage/adapters/fileSystemStorage.js +49 -395
- package/dist/storage/adapters/gcsStorage.d.ts +13 -134
- package/dist/storage/adapters/gcsStorage.js +79 -557
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +181 -0
- package/dist/storage/adapters/historicalStorageAdapter.js +332 -0
- package/dist/storage/adapters/memoryStorage.d.ts +4 -113
- package/dist/storage/adapters/memoryStorage.js +34 -471
- package/dist/storage/adapters/opfsStorage.d.ts +14 -127
- package/dist/storage/adapters/opfsStorage.js +44 -693
- package/dist/storage/adapters/r2Storage.d.ts +8 -41
- package/dist/storage/adapters/r2Storage.js +49 -237
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +13 -111
- package/dist/storage/adapters/s3CompatibleStorage.js +77 -596
- package/dist/storage/baseStorage.d.ts +78 -38
- package/dist/storage/baseStorage.js +692 -23
- package/dist/storage/cow/BlobStorage.d.ts +2 -2
- package/dist/storage/cow/BlobStorage.js +4 -4
- package/dist/storage/storageFactory.d.ts +2 -3
- package/dist/storage/storageFactory.js +114 -66
- package/dist/vfs/types.d.ts +6 -2
- package/package.json +1 -1
|
@@ -2,13 +2,19 @@
|
|
|
2
2
|
* File System Storage Adapter
|
|
3
3
|
* File system storage adapter for Node.js environments
|
|
4
4
|
*/
|
|
5
|
-
import { HNSWNoun, HNSWVerb,
|
|
5
|
+
import { HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
|
6
6
|
import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
|
|
7
7
|
type HNSWNode = HNSWNoun;
|
|
8
8
|
type Edge = HNSWVerb;
|
|
9
9
|
/**
|
|
10
10
|
* File system storage adapter for Node.js environments
|
|
11
11
|
* Uses the file system to store data in the specified directory structure
|
|
12
|
+
*
|
|
13
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
14
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
15
|
+
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
|
16
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
17
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
12
18
|
*/
|
|
13
19
|
export declare class FileSystemStorage extends BaseStorage {
|
|
14
20
|
private countsFilePath?;
|
|
@@ -156,16 +162,6 @@ export declare class FileSystemStorage extends BaseStorage {
|
|
|
156
162
|
* Get nouns with pagination support
|
|
157
163
|
* @param options Pagination options
|
|
158
164
|
*/
|
|
159
|
-
getNounsWithPagination(options?: {
|
|
160
|
-
limit?: number;
|
|
161
|
-
cursor?: string;
|
|
162
|
-
filter?: any;
|
|
163
|
-
}): Promise<{
|
|
164
|
-
items: HNSWNounWithMetadata[];
|
|
165
|
-
totalCount: number;
|
|
166
|
-
hasMore: boolean;
|
|
167
|
-
nextCursor?: string;
|
|
168
|
-
}>;
|
|
169
165
|
/**
|
|
170
166
|
* Clear all data from storage
|
|
171
167
|
*/
|
|
@@ -184,73 +180,6 @@ export declare class FileSystemStorage extends BaseStorage {
|
|
|
184
180
|
quota: number | null;
|
|
185
181
|
details?: Record<string, any>;
|
|
186
182
|
}>;
|
|
187
|
-
/**
|
|
188
|
-
* Implementation of abstract methods from BaseStorage
|
|
189
|
-
*/
|
|
190
|
-
/**
|
|
191
|
-
* Save a noun to storage
|
|
192
|
-
*/
|
|
193
|
-
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
194
|
-
/**
|
|
195
|
-
* Get a noun from storage (internal implementation)
|
|
196
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
197
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
198
|
-
*/
|
|
199
|
-
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
200
|
-
/**
|
|
201
|
-
* Get nouns by noun type
|
|
202
|
-
*/
|
|
203
|
-
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
204
|
-
/**
|
|
205
|
-
* Delete a noun from storage
|
|
206
|
-
*/
|
|
207
|
-
protected deleteNoun_internal(id: string): Promise<void>;
|
|
208
|
-
/**
|
|
209
|
-
* Save a verb to storage
|
|
210
|
-
*/
|
|
211
|
-
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
212
|
-
/**
|
|
213
|
-
* Get a verb from storage (internal implementation)
|
|
214
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
215
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
216
|
-
*/
|
|
217
|
-
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
218
|
-
/**
|
|
219
|
-
* Get verbs by source
|
|
220
|
-
*/
|
|
221
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
222
|
-
/**
|
|
223
|
-
* Get verbs by target
|
|
224
|
-
*/
|
|
225
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
226
|
-
/**
|
|
227
|
-
* Get verbs by type
|
|
228
|
-
*/
|
|
229
|
-
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
230
|
-
/**
|
|
231
|
-
* Get verbs with pagination
|
|
232
|
-
* This method reads verb files from the filesystem and returns them with pagination
|
|
233
|
-
*/
|
|
234
|
-
getVerbsWithPagination(options?: {
|
|
235
|
-
limit?: number;
|
|
236
|
-
cursor?: string;
|
|
237
|
-
filter?: {
|
|
238
|
-
verbType?: string | string[];
|
|
239
|
-
sourceId?: string | string[];
|
|
240
|
-
targetId?: string | string[];
|
|
241
|
-
service?: string | string[];
|
|
242
|
-
metadata?: Record<string, any>;
|
|
243
|
-
};
|
|
244
|
-
}): Promise<{
|
|
245
|
-
items: HNSWVerbWithMetadata[];
|
|
246
|
-
totalCount?: number;
|
|
247
|
-
hasMore: boolean;
|
|
248
|
-
nextCursor?: string;
|
|
249
|
-
}>;
|
|
250
|
-
/**
|
|
251
|
-
* Delete a verb from storage
|
|
252
|
-
*/
|
|
253
|
-
protected deleteVerb_internal(id: string): Promise<void>;
|
|
254
183
|
/**
|
|
255
184
|
* Acquire a file-based lock for coordinating operations across multiple processes
|
|
256
185
|
* @param lockKey The key to lock on
|
|
@@ -390,10 +319,14 @@ export declare class FileSystemStorage extends BaseStorage {
|
|
|
390
319
|
private fileExists;
|
|
391
320
|
/**
|
|
392
321
|
* Get vector for a noun
|
|
322
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
393
323
|
*/
|
|
394
324
|
getNounVector(id: string): Promise<number[] | null>;
|
|
395
325
|
/**
|
|
396
326
|
* Save HNSW graph data for a noun
|
|
327
|
+
*
|
|
328
|
+
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
|
329
|
+
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
|
397
330
|
*/
|
|
398
331
|
saveHNSWData(nounId: string, hnswData: {
|
|
399
332
|
level: number;
|
|
@@ -401,6 +334,7 @@ export declare class FileSystemStorage extends BaseStorage {
|
|
|
401
334
|
}): Promise<void>;
|
|
402
335
|
/**
|
|
403
336
|
* Get HNSW graph data for a noun
|
|
337
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
404
338
|
*/
|
|
405
339
|
getHNSWData(nounId: string): Promise<{
|
|
406
340
|
level: number;
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
* File System Storage Adapter
|
|
3
3
|
* File system storage adapter for Node.js environments
|
|
4
4
|
*/
|
|
5
|
-
import { NounType } from '../../coreTypes.js';
|
|
6
5
|
import { BaseStorage, SYSTEM_DIR, STATISTICS_KEY } from '../baseStorage.js';
|
|
7
6
|
// Node.js modules - dynamically imported to avoid issues in browser environments
|
|
8
7
|
let fs;
|
|
@@ -32,6 +31,12 @@ catch (error) {
|
|
|
32
31
|
/**
|
|
33
32
|
* File system storage adapter for Node.js environments
|
|
34
33
|
* Uses the file system to store data in the specified directory structure
|
|
34
|
+
*
|
|
35
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
36
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
37
|
+
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
|
38
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
39
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
35
40
|
*/
|
|
36
41
|
export class FileSystemStorage extends BaseStorage {
|
|
37
42
|
/**
|
|
@@ -820,115 +825,7 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
820
825
|
* Get nouns with pagination support
|
|
821
826
|
* @param options Pagination options
|
|
822
827
|
*/
|
|
823
|
-
|
|
824
|
-
await this.ensureInitialized();
|
|
825
|
-
const limit = options.limit || 100;
|
|
826
|
-
const cursor = options.cursor;
|
|
827
|
-
try {
|
|
828
|
-
// Get all noun files (handles sharding properly)
|
|
829
|
-
const nounFiles = await this.getAllShardedFiles(this.nounsDir);
|
|
830
|
-
// Sort for consistent pagination
|
|
831
|
-
nounFiles.sort();
|
|
832
|
-
// Find starting position - prioritize offset for O(1) operation
|
|
833
|
-
let startIndex = 0;
|
|
834
|
-
const offset = options.offset; // Cast to any since offset might not be in type
|
|
835
|
-
if (offset !== undefined) {
|
|
836
|
-
// Direct offset - O(1) operation
|
|
837
|
-
startIndex = offset;
|
|
838
|
-
}
|
|
839
|
-
else if (cursor) {
|
|
840
|
-
// Cursor-based pagination
|
|
841
|
-
startIndex = nounFiles.findIndex((f) => f.replace('.json', '') > cursor);
|
|
842
|
-
if (startIndex === -1)
|
|
843
|
-
startIndex = nounFiles.length;
|
|
844
|
-
}
|
|
845
|
-
// Get page of files
|
|
846
|
-
const pageFiles = nounFiles.slice(startIndex, startIndex + limit);
|
|
847
|
-
// v4.0.0: Load nouns and combine with metadata
|
|
848
|
-
const items = [];
|
|
849
|
-
let successfullyLoaded = 0;
|
|
850
|
-
let totalValidFiles = 0;
|
|
851
|
-
// Use persisted counts - O(1) operation!
|
|
852
|
-
totalValidFiles = this.totalNounCount;
|
|
853
|
-
// No need to count files anymore - we maintain accurate counts
|
|
854
|
-
// This eliminates the O(n) operation completely
|
|
855
|
-
// Second pass: load the current page with metadata
|
|
856
|
-
for (const file of pageFiles) {
|
|
857
|
-
try {
|
|
858
|
-
const id = file.replace('.json', '');
|
|
859
|
-
const data = await fs.promises.readFile(this.getNodePath(id), 'utf-8');
|
|
860
|
-
const parsedNoun = JSON.parse(data);
|
|
861
|
-
// v4.0.0: Load metadata from separate storage
|
|
862
|
-
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
|
|
863
|
-
const metadata = await this.getNounMetadata(id);
|
|
864
|
-
// Apply filter if provided
|
|
865
|
-
if (options.filter && metadata) {
|
|
866
|
-
let matches = true;
|
|
867
|
-
for (const [key, value] of Object.entries(options.filter)) {
|
|
868
|
-
if (metadata[key] !== value) {
|
|
869
|
-
matches = false;
|
|
870
|
-
break;
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
if (!matches)
|
|
874
|
-
continue;
|
|
875
|
-
}
|
|
876
|
-
// Convert connections if needed
|
|
877
|
-
let connections = parsedNoun.connections;
|
|
878
|
-
if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
|
|
879
|
-
const connectionsMap = new Map();
|
|
880
|
-
for (const [level, nodeIds] of Object.entries(connections)) {
|
|
881
|
-
connectionsMap.set(Number(level), new Set(nodeIds));
|
|
882
|
-
}
|
|
883
|
-
connections = connectionsMap;
|
|
884
|
-
}
|
|
885
|
-
// v4.8.0: Extract standard fields from metadata to top-level
|
|
886
|
-
const metadataObj = (metadata || {});
|
|
887
|
-
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj;
|
|
888
|
-
const nounWithMetadata = {
|
|
889
|
-
id: parsedNoun.id,
|
|
890
|
-
vector: parsedNoun.vector,
|
|
891
|
-
connections: connections,
|
|
892
|
-
level: parsedNoun.level || 0,
|
|
893
|
-
type: nounType || NounType.Thing,
|
|
894
|
-
createdAt: createdAt || Date.now(),
|
|
895
|
-
updatedAt: updatedAt || Date.now(),
|
|
896
|
-
confidence: confidence,
|
|
897
|
-
weight: weight,
|
|
898
|
-
service: service,
|
|
899
|
-
data: dataField,
|
|
900
|
-
createdBy,
|
|
901
|
-
metadata: customMetadata
|
|
902
|
-
};
|
|
903
|
-
items.push(nounWithMetadata);
|
|
904
|
-
successfullyLoaded++;
|
|
905
|
-
}
|
|
906
|
-
catch (error) {
|
|
907
|
-
console.warn(`Failed to read noun file ${file}:`, error);
|
|
908
|
-
}
|
|
909
|
-
}
|
|
910
|
-
// CRITICAL FIX: hasMore should be based on actual valid files, not just file count
|
|
911
|
-
// Also check if we actually loaded any items from this page
|
|
912
|
-
const hasMore = (startIndex + limit < totalValidFiles) && (successfullyLoaded > 0 || startIndex === 0);
|
|
913
|
-
const nextCursor = hasMore && pageFiles.length > 0
|
|
914
|
-
? pageFiles[pageFiles.length - 1].replace('.json', '')
|
|
915
|
-
: undefined;
|
|
916
|
-
return {
|
|
917
|
-
items,
|
|
918
|
-
totalCount: totalValidFiles, // Use actual valid file count, not all files
|
|
919
|
-
hasMore,
|
|
920
|
-
nextCursor
|
|
921
|
-
};
|
|
922
|
-
}
|
|
923
|
-
catch (error) {
|
|
924
|
-
console.error('Error getting nouns with pagination:', error);
|
|
925
|
-
return {
|
|
926
|
-
items: [],
|
|
927
|
-
totalCount: 0,
|
|
928
|
-
hasMore: false
|
|
929
|
-
};
|
|
930
|
-
}
|
|
931
|
-
}
|
|
828
|
+
// v5.4.0: Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
|
|
932
829
|
/**
|
|
933
830
|
* Clear all data from storage
|
|
934
831
|
*/
|
|
@@ -1111,231 +1008,8 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
1111
1008
|
};
|
|
1112
1009
|
}
|
|
1113
1010
|
}
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
*/
|
|
1117
|
-
/**
|
|
1118
|
-
* Save a noun to storage
|
|
1119
|
-
*/
|
|
1120
|
-
async saveNoun_internal(noun) {
|
|
1121
|
-
return this.saveNode(noun);
|
|
1122
|
-
}
|
|
1123
|
-
/**
|
|
1124
|
-
* Get a noun from storage (internal implementation)
|
|
1125
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
1126
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
1127
|
-
*/
|
|
1128
|
-
async getNoun_internal(id) {
|
|
1129
|
-
// v4.0.0: Return ONLY vector data (no metadata field)
|
|
1130
|
-
const node = await this.getNode(id);
|
|
1131
|
-
if (!node) {
|
|
1132
|
-
return null;
|
|
1133
|
-
}
|
|
1134
|
-
// Return pure vector structure
|
|
1135
|
-
return node;
|
|
1136
|
-
}
|
|
1137
|
-
/**
|
|
1138
|
-
* Get nouns by noun type
|
|
1139
|
-
*/
|
|
1140
|
-
async getNounsByNounType_internal(nounType) {
|
|
1141
|
-
return this.getNodesByNounType(nounType);
|
|
1142
|
-
}
|
|
1143
|
-
/**
|
|
1144
|
-
* Delete a noun from storage
|
|
1145
|
-
*/
|
|
1146
|
-
async deleteNoun_internal(id) {
|
|
1147
|
-
return this.deleteNode(id);
|
|
1148
|
-
}
|
|
1149
|
-
/**
|
|
1150
|
-
* Save a verb to storage
|
|
1151
|
-
*/
|
|
1152
|
-
async saveVerb_internal(verb) {
|
|
1153
|
-
return this.saveEdge(verb);
|
|
1154
|
-
}
|
|
1155
|
-
/**
|
|
1156
|
-
* Get a verb from storage (internal implementation)
|
|
1157
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
1158
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
1159
|
-
*/
|
|
1160
|
-
async getVerb_internal(id) {
|
|
1161
|
-
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
|
1162
|
-
const edge = await this.getEdge(id);
|
|
1163
|
-
if (!edge) {
|
|
1164
|
-
return null;
|
|
1165
|
-
}
|
|
1166
|
-
// Return pure vector + core fields structure
|
|
1167
|
-
return edge;
|
|
1168
|
-
}
|
|
1169
|
-
/**
|
|
1170
|
-
* Get verbs by source
|
|
1171
|
-
*/
|
|
1172
|
-
async getVerbsBySource_internal(sourceId) {
|
|
1173
|
-
// Use the working pagination method with source filter
|
|
1174
|
-
const result = await this.getVerbsWithPagination({
|
|
1175
|
-
limit: 10000,
|
|
1176
|
-
filter: { sourceId: [sourceId] }
|
|
1177
|
-
});
|
|
1178
|
-
return result.items;
|
|
1179
|
-
}
|
|
1180
|
-
/**
|
|
1181
|
-
* Get verbs by target
|
|
1182
|
-
*/
|
|
1183
|
-
async getVerbsByTarget_internal(targetId) {
|
|
1184
|
-
// Use the working pagination method with target filter
|
|
1185
|
-
const result = await this.getVerbsWithPagination({
|
|
1186
|
-
limit: 10000,
|
|
1187
|
-
filter: { targetId: [targetId] }
|
|
1188
|
-
});
|
|
1189
|
-
return result.items;
|
|
1190
|
-
}
|
|
1191
|
-
/**
|
|
1192
|
-
* Get verbs by type
|
|
1193
|
-
*/
|
|
1194
|
-
async getVerbsByType_internal(type) {
|
|
1195
|
-
// Use the working pagination method with type filter
|
|
1196
|
-
const result = await this.getVerbsWithPagination({
|
|
1197
|
-
limit: 10000,
|
|
1198
|
-
filter: { verbType: [type] }
|
|
1199
|
-
});
|
|
1200
|
-
return result.items;
|
|
1201
|
-
}
|
|
1202
|
-
/**
|
|
1203
|
-
* Get verbs with pagination
|
|
1204
|
-
* This method reads verb files from the filesystem and returns them with pagination
|
|
1205
|
-
*/
|
|
1206
|
-
async getVerbsWithPagination(options = {}) {
|
|
1207
|
-
await this.ensureInitialized();
|
|
1208
|
-
const limit = options.limit || 100;
|
|
1209
|
-
const startIndex = options.cursor ? parseInt(options.cursor, 10) : 0;
|
|
1210
|
-
try {
|
|
1211
|
-
// Get actual verb files first (critical for accuracy)
|
|
1212
|
-
const verbFiles = await this.getAllShardedFiles(this.verbsDir);
|
|
1213
|
-
verbFiles.sort(); // Consistent ordering for pagination
|
|
1214
|
-
// Use actual file count - don't trust cached totalVerbCount
|
|
1215
|
-
// This prevents accessing undefined array elements
|
|
1216
|
-
const actualFileCount = verbFiles.length;
|
|
1217
|
-
// For large datasets, warn about performance
|
|
1218
|
-
if (actualFileCount > 1000000) {
|
|
1219
|
-
console.warn(`Very large verb dataset detected (${actualFileCount} verbs). Performance may be degraded. Consider database storage for optimal performance.`);
|
|
1220
|
-
}
|
|
1221
|
-
// For production-scale datasets, use streaming approach
|
|
1222
|
-
if (actualFileCount > 50000) {
|
|
1223
|
-
return await this.getVerbsWithPaginationStreaming(options, startIndex, limit);
|
|
1224
|
-
}
|
|
1225
|
-
// Calculate pagination bounds using ACTUAL file count
|
|
1226
|
-
const endIndex = Math.min(startIndex + limit, actualFileCount);
|
|
1227
|
-
// Load the requested page of verbs
|
|
1228
|
-
const verbs = [];
|
|
1229
|
-
let successfullyLoaded = 0;
|
|
1230
|
-
for (let i = startIndex; i < endIndex; i++) {
|
|
1231
|
-
const file = verbFiles[i];
|
|
1232
|
-
// CRITICAL: Null-safety check for undefined array elements
|
|
1233
|
-
if (!file) {
|
|
1234
|
-
console.warn(`Unexpected undefined file at index ${i}, skipping`);
|
|
1235
|
-
continue;
|
|
1236
|
-
}
|
|
1237
|
-
const id = file.replace('.json', '');
|
|
1238
|
-
try {
|
|
1239
|
-
// Read the verb data (HNSWVerb stored as edge) - use sharded path
|
|
1240
|
-
const filePath = this.getVerbPath(id);
|
|
1241
|
-
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
1242
|
-
const edge = JSON.parse(data);
|
|
1243
|
-
// Get metadata which contains the actual verb information
|
|
1244
|
-
const metadata = await this.getVerbMetadata(id);
|
|
1245
|
-
// v4.8.1: Don't skip verbs without metadata - metadata is optional
|
|
1246
|
-
// FIX: This was the root cause of the VFS bug (11 versions)
|
|
1247
|
-
// Verbs can exist without metadata files (e.g., from imports/migrations)
|
|
1248
|
-
// Convert connections Map to proper format if needed
|
|
1249
|
-
let connections = edge.connections;
|
|
1250
|
-
if (connections && typeof connections === 'object' && !(connections instanceof Map)) {
|
|
1251
|
-
const connectionsMap = new Map();
|
|
1252
|
-
for (const [level, nodeIds] of Object.entries(connections)) {
|
|
1253
|
-
connectionsMap.set(Number(level), new Set(nodeIds));
|
|
1254
|
-
}
|
|
1255
|
-
connections = connectionsMap;
|
|
1256
|
-
}
|
|
1257
|
-
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1258
|
-
const metadataObj = (metadata || {});
|
|
1259
|
-
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj;
|
|
1260
|
-
const verbWithMetadata = {
|
|
1261
|
-
id: edge.id,
|
|
1262
|
-
vector: edge.vector,
|
|
1263
|
-
connections: connections,
|
|
1264
|
-
verb: edge.verb,
|
|
1265
|
-
sourceId: edge.sourceId,
|
|
1266
|
-
targetId: edge.targetId,
|
|
1267
|
-
createdAt: createdAt || Date.now(),
|
|
1268
|
-
updatedAt: updatedAt || Date.now(),
|
|
1269
|
-
confidence: confidence,
|
|
1270
|
-
weight: weight,
|
|
1271
|
-
service: service,
|
|
1272
|
-
data: dataField,
|
|
1273
|
-
createdBy,
|
|
1274
|
-
metadata: customMetadata
|
|
1275
|
-
};
|
|
1276
|
-
// Apply filters if provided
|
|
1277
|
-
if (options.filter) {
|
|
1278
|
-
const filter = options.filter;
|
|
1279
|
-
// Check verbType filter
|
|
1280
|
-
if (filter.verbType) {
|
|
1281
|
-
const types = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType];
|
|
1282
|
-
if (!types.includes(verbWithMetadata.verb))
|
|
1283
|
-
continue;
|
|
1284
|
-
}
|
|
1285
|
-
// Check sourceId filter
|
|
1286
|
-
if (filter.sourceId) {
|
|
1287
|
-
const sources = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId];
|
|
1288
|
-
if (!sources.includes(verbWithMetadata.sourceId))
|
|
1289
|
-
continue;
|
|
1290
|
-
}
|
|
1291
|
-
// Check targetId filter
|
|
1292
|
-
if (filter.targetId) {
|
|
1293
|
-
const targets = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId];
|
|
1294
|
-
if (!targets.includes(verbWithMetadata.targetId))
|
|
1295
|
-
continue;
|
|
1296
|
-
}
|
|
1297
|
-
// Check service filter
|
|
1298
|
-
if (filter.service && metadata?.service) {
|
|
1299
|
-
const services = Array.isArray(filter.service) ? filter.service : [filter.service];
|
|
1300
|
-
if (!services.includes(metadata.service))
|
|
1301
|
-
continue;
|
|
1302
|
-
}
|
|
1303
|
-
}
|
|
1304
|
-
verbs.push(verbWithMetadata);
|
|
1305
|
-
successfullyLoaded++;
|
|
1306
|
-
}
|
|
1307
|
-
catch (error) {
|
|
1308
|
-
console.warn(`Failed to read verb ${id}:`, error);
|
|
1309
|
-
}
|
|
1310
|
-
}
|
|
1311
|
-
// CRITICAL FIX: hasMore based on actual file count, not cached totalVerbCount
|
|
1312
|
-
// Also verify we successfully loaded items (prevents infinite loops on corrupted storage)
|
|
1313
|
-
const hasMore = (endIndex < actualFileCount) && (successfullyLoaded > 0 || startIndex === 0);
|
|
1314
|
-
return {
|
|
1315
|
-
items: verbs,
|
|
1316
|
-
totalCount: actualFileCount, // Return actual count, not cached value
|
|
1317
|
-
hasMore,
|
|
1318
|
-
nextCursor: hasMore ? String(endIndex) : undefined
|
|
1319
|
-
};
|
|
1320
|
-
}
|
|
1321
|
-
catch (error) {
|
|
1322
|
-
if (error.code === 'ENOENT') {
|
|
1323
|
-
// Verbs directory doesn't exist yet
|
|
1324
|
-
return {
|
|
1325
|
-
items: [],
|
|
1326
|
-
totalCount: 0,
|
|
1327
|
-
hasMore: false
|
|
1328
|
-
};
|
|
1329
|
-
}
|
|
1330
|
-
throw error;
|
|
1331
|
-
}
|
|
1332
|
-
}
|
|
1333
|
-
/**
|
|
1334
|
-
* Delete a verb from storage
|
|
1335
|
-
*/
|
|
1336
|
-
async deleteVerb_internal(id) {
|
|
1337
|
-
return this.deleteEdge(id);
|
|
1338
|
-
}
|
|
1011
|
+
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
|
1012
|
+
// v5.4.0: Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
|
|
1339
1013
|
/**
|
|
1340
1014
|
* Acquire a file-based lock for coordinating operations across multiple processes
|
|
1341
1015
|
* @param lockKey The key to lock on
|
|
@@ -2276,23 +1950,24 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
2276
1950
|
// =============================================
|
|
2277
1951
|
/**
|
|
2278
1952
|
* Get vector for a noun
|
|
1953
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
2279
1954
|
*/
|
|
2280
1955
|
async getNounVector(id) {
|
|
2281
|
-
await this.
|
|
2282
|
-
const noun = await this.getNode(id);
|
|
1956
|
+
const noun = await this.getNoun(id);
|
|
2283
1957
|
return noun ? noun.vector : null;
|
|
2284
1958
|
}
|
|
2285
1959
|
/**
|
|
2286
1960
|
* Save HNSW graph data for a noun
|
|
1961
|
+
*
|
|
1962
|
+
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
|
1963
|
+
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
|
2287
1964
|
*/
|
|
2288
1965
|
async saveHNSWData(nounId, hnswData) {
|
|
2289
|
-
await this.ensureInitialized();
|
|
2290
|
-
const filePath = this.getNodePath(nounId);
|
|
2291
1966
|
const lockKey = `hnsw/${nounId}`;
|
|
2292
1967
|
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
|
|
2293
1968
|
// Problem: Without mutex, concurrent operations can:
|
|
2294
|
-
// 1. Thread A reads
|
|
2295
|
-
// 2. Thread B reads
|
|
1969
|
+
// 1. Thread A reads noun (connections: [1,2,3])
|
|
1970
|
+
// 2. Thread B reads noun (connections: [1,2,3])
|
|
2296
1971
|
// 3. Thread A adds connection 4, writes [1,2,3,4]
|
|
2297
1972
|
// 4. Thread B adds connection 5, writes [1,2,3,5] ← Connection 4 LOST!
|
|
2298
1973
|
// Solution: Mutex serializes operations per entity (like Memory/OPFS adapters)
|
|
@@ -2306,50 +1981,26 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
2306
1981
|
const lockPromise = new Promise(resolve => { releaseLock = resolve; });
|
|
2307
1982
|
this.hnswLocks.set(lockKey, lockPromise);
|
|
2308
1983
|
try {
|
|
2309
|
-
//
|
|
2310
|
-
//
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
// Preserve id and vector, update only HNSW graph metadata
|
|
2330
|
-
const updatedNode = {
|
|
2331
|
-
...existingNode, // Preserve all existing fields (id, vector, etc.)
|
|
2332
|
-
level: hnswData.level,
|
|
2333
|
-
connections: hnswData.connections
|
|
2334
|
-
};
|
|
2335
|
-
// ATOMIC WRITE SEQUENCE:
|
|
2336
|
-
// 1. Write to temp file
|
|
2337
|
-
await this.ensureDirectoryExists(path.dirname(tempPath));
|
|
2338
|
-
await fs.promises.writeFile(tempPath, JSON.stringify(updatedNode, null, 2));
|
|
2339
|
-
// 2. Atomic rename temp → final (POSIX atomicity guarantee)
|
|
2340
|
-
// This operation is guaranteed atomic by POSIX - either succeeds completely or fails
|
|
2341
|
-
await fs.promises.rename(tempPath, filePath);
|
|
2342
|
-
}
|
|
2343
|
-
catch (error) {
|
|
2344
|
-
// Clean up temp file on any error
|
|
2345
|
-
try {
|
|
2346
|
-
await fs.promises.unlink(tempPath);
|
|
2347
|
-
}
|
|
2348
|
-
catch (cleanupError) {
|
|
2349
|
-
// Ignore cleanup errors - temp file may not exist
|
|
2350
|
-
}
|
|
2351
|
-
throw error;
|
|
2352
|
-
}
|
|
1984
|
+
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
|
1985
|
+
// Read existing noun data (if exists)
|
|
1986
|
+
const existingNoun = await this.getNoun(nounId);
|
|
1987
|
+
if (!existingNoun) {
|
|
1988
|
+
// Noun doesn't exist - cannot update HNSW data for non-existent noun
|
|
1989
|
+
throw new Error(`Cannot save HNSW data: noun ${nounId} not found`);
|
|
1990
|
+
}
|
|
1991
|
+
// Convert connections from Record to Map format for storage
|
|
1992
|
+
const connectionsMap = new Map();
|
|
1993
|
+
for (const [level, nodeIds] of Object.entries(hnswData.connections)) {
|
|
1994
|
+
connectionsMap.set(Number(level), new Set(nodeIds));
|
|
1995
|
+
}
|
|
1996
|
+
// Preserve id and vector, update only HNSW graph metadata
|
|
1997
|
+
const updatedNoun = {
|
|
1998
|
+
...existingNoun,
|
|
1999
|
+
level: hnswData.level,
|
|
2000
|
+
connections: connectionsMap
|
|
2001
|
+
};
|
|
2002
|
+
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write)
|
|
2003
|
+
await this.saveNoun(updatedNoun);
|
|
2353
2004
|
}
|
|
2354
2005
|
finally {
|
|
2355
2006
|
// Release lock (ALWAYS runs, even if error thrown)
|
|
@@ -2359,21 +2010,24 @@ export class FileSystemStorage extends BaseStorage {
|
|
|
2359
2010
|
}
|
|
2360
2011
|
/**
|
|
2361
2012
|
* Get HNSW graph data for a noun
|
|
2013
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
2362
2014
|
*/
|
|
2363
2015
|
async getHNSWData(nounId) {
|
|
2364
|
-
await this.
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
try {
|
|
2368
|
-
const data = await fs.promises.readFile(filePath, 'utf-8');
|
|
2369
|
-
return JSON.parse(data);
|
|
2016
|
+
const noun = await this.getNoun(nounId);
|
|
2017
|
+
if (!noun) {
|
|
2018
|
+
return null;
|
|
2370
2019
|
}
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2020
|
+
// Convert connections from Map to Record format
|
|
2021
|
+
const connectionsRecord = {};
|
|
2022
|
+
if (noun.connections) {
|
|
2023
|
+
for (const [level, nodeIds] of noun.connections.entries()) {
|
|
2024
|
+
connectionsRecord[String(level)] = Array.from(nodeIds);
|
|
2374
2025
|
}
|
|
2375
|
-
return null;
|
|
2376
2026
|
}
|
|
2027
|
+
return {
|
|
2028
|
+
level: noun.level || 0,
|
|
2029
|
+
connections: connectionsRecord
|
|
2030
|
+
};
|
|
2377
2031
|
}
|
|
2378
2032
|
/**
|
|
2379
2033
|
* Save HNSW system data (entry point, max level)
|