@soulcraft/brainy 5.3.6 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -0
- package/README.md +4 -3
- package/dist/augmentations/display/fieldPatterns.js +3 -3
- package/dist/augmentations/display/intelligentComputation.js +0 -2
- package/dist/augmentations/typeMatching/brainyTypes.js +6 -8
- package/dist/brainy.d.ts +61 -0
- package/dist/brainy.js +180 -24
- package/dist/cortex/neuralImport.js +0 -1
- package/dist/importers/SmartExcelImporter.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/neural/embeddedKeywordEmbeddings.d.ts +1 -1
- package/dist/neural/embeddedKeywordEmbeddings.js +56 -56
- package/dist/neural/embeddedTypeEmbeddings.d.ts +3 -3
- package/dist/neural/embeddedTypeEmbeddings.js +14 -14
- package/dist/neural/entityExtractor.js +2 -2
- package/dist/neural/relationshipConfidence.js +1 -1
- package/dist/neural/signals/VerbContextSignal.js +6 -6
- package/dist/neural/signals/VerbExactMatchSignal.js +9 -9
- package/dist/neural/signals/VerbPatternSignal.js +5 -5
- package/dist/query/typeAwareQueryPlanner.js +2 -3
- package/dist/storage/adapters/azureBlobStorage.d.ts +13 -64
- package/dist/storage/adapters/azureBlobStorage.js +78 -388
- package/dist/storage/adapters/fileSystemStorage.d.ts +12 -78
- package/dist/storage/adapters/fileSystemStorage.js +49 -395
- package/dist/storage/adapters/gcsStorage.d.ts +13 -134
- package/dist/storage/adapters/gcsStorage.js +79 -557
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +181 -0
- package/dist/storage/adapters/historicalStorageAdapter.js +332 -0
- package/dist/storage/adapters/memoryStorage.d.ts +4 -113
- package/dist/storage/adapters/memoryStorage.js +34 -471
- package/dist/storage/adapters/opfsStorage.d.ts +14 -127
- package/dist/storage/adapters/opfsStorage.js +44 -693
- package/dist/storage/adapters/r2Storage.d.ts +8 -41
- package/dist/storage/adapters/r2Storage.js +49 -237
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +13 -111
- package/dist/storage/adapters/s3CompatibleStorage.js +77 -596
- package/dist/storage/baseStorage.d.ts +78 -38
- package/dist/storage/baseStorage.js +692 -23
- package/dist/storage/cow/BlobStorage.d.ts +2 -2
- package/dist/storage/cow/BlobStorage.js +4 -4
- package/dist/storage/storageFactory.d.ts +2 -3
- package/dist/storage/storageFactory.js +114 -66
- package/dist/types/graphTypes.d.ts +588 -230
- package/dist/types/graphTypes.js +683 -248
- package/dist/types/typeMigration.d.ts +95 -0
- package/dist/types/typeMigration.js +141 -0
- package/dist/utils/intelligentTypeMapper.js +2 -2
- package/dist/utils/metadataIndex.js +6 -6
- package/dist/vfs/types.d.ts +6 -2
- package/package.json +2 -2
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
* OPFS (Origin Private File System) Storage Adapter
|
|
3
3
|
* Provides persistent storage for the vector database using the Origin Private File System API
|
|
4
4
|
*/
|
|
5
|
-
import { NounType } from '../../coreTypes.js';
|
|
6
5
|
import { BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, NOUN_METADATA_DIR, VERB_METADATA_DIR, INDEX_DIR } from '../baseStorage.js';
|
|
7
6
|
import { getShardIdFromUuid } from '../sharding.js';
|
|
8
7
|
import '../../types/fileSystemTypes.js';
|
|
@@ -20,6 +19,12 @@ const ROOT_DIR = 'opfs-vector-db';
|
|
|
20
19
|
/**
|
|
21
20
|
* OPFS storage adapter for browser environments
|
|
22
21
|
* Uses the Origin Private File System API to store data persistently
|
|
22
|
+
*
|
|
23
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
24
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
25
|
+
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
|
26
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
27
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
23
28
|
*/
|
|
24
29
|
export class OPFSStorage extends BaseStorage {
|
|
25
30
|
constructor() {
|
|
@@ -170,398 +175,7 @@ export class OPFSStorage extends BaseStorage {
|
|
|
170
175
|
return false;
|
|
171
176
|
}
|
|
172
177
|
}
|
|
173
|
-
|
|
174
|
-
* Save a noun to storage
|
|
175
|
-
*/
|
|
176
|
-
async saveNoun_internal(noun) {
|
|
177
|
-
await this.ensureInitialized();
|
|
178
|
-
try {
|
|
179
|
-
// CRITICAL: Only save lightweight vector data (no metadata)
|
|
180
|
-
// Metadata is saved separately via saveNounMetadata() (2-file system)
|
|
181
|
-
const serializableNoun = {
|
|
182
|
-
id: noun.id,
|
|
183
|
-
vector: noun.vector,
|
|
184
|
-
connections: this.mapToObject(noun.connections, (set) => Array.from(set)),
|
|
185
|
-
level: noun.level || 0
|
|
186
|
-
// NO metadata field - saved separately for scalability
|
|
187
|
-
};
|
|
188
|
-
// Use UUID-based sharding for nouns
|
|
189
|
-
const shardId = getShardIdFromUuid(noun.id);
|
|
190
|
-
// Get or create the shard directory
|
|
191
|
-
const shardDir = await this.nounsDir.getDirectoryHandle(shardId, {
|
|
192
|
-
create: true
|
|
193
|
-
});
|
|
194
|
-
// Create or get the file in the shard directory
|
|
195
|
-
const fileHandle = await shardDir.getFileHandle(`${noun.id}.json`, {
|
|
196
|
-
create: true
|
|
197
|
-
});
|
|
198
|
-
// Write the noun data to the file
|
|
199
|
-
const writable = await fileHandle.createWritable();
|
|
200
|
-
await writable.write(JSON.stringify(serializableNoun));
|
|
201
|
-
await writable.close();
|
|
202
|
-
}
|
|
203
|
-
catch (error) {
|
|
204
|
-
console.error(`Failed to save noun ${noun.id}:`, error);
|
|
205
|
-
throw new Error(`Failed to save noun ${noun.id}: ${error}`);
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
/**
|
|
209
|
-
* Get a noun from storage (internal implementation)
|
|
210
|
-
* Combines vector data from file with metadata from getNounMetadata()
|
|
211
|
-
*/
|
|
212
|
-
async getNoun_internal(id) {
|
|
213
|
-
await this.ensureInitialized();
|
|
214
|
-
try {
|
|
215
|
-
// Use UUID-based sharding for nouns
|
|
216
|
-
const shardId = getShardIdFromUuid(id);
|
|
217
|
-
// Get the shard directory
|
|
218
|
-
const shardDir = await this.nounsDir.getDirectoryHandle(shardId);
|
|
219
|
-
// Get the file handle from the shard directory
|
|
220
|
-
const fileHandle = await shardDir.getFileHandle(`${id}.json`);
|
|
221
|
-
// Read the noun data from the file
|
|
222
|
-
const file = await fileHandle.getFile();
|
|
223
|
-
const text = await file.text();
|
|
224
|
-
const data = JSON.parse(text);
|
|
225
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
226
|
-
const connections = new Map();
|
|
227
|
-
for (const [level, nounIds] of Object.entries(data.connections)) {
|
|
228
|
-
connections.set(Number(level), new Set(nounIds));
|
|
229
|
-
}
|
|
230
|
-
// v4.0.0: Return ONLY vector data (no metadata field)
|
|
231
|
-
const node = {
|
|
232
|
-
id: data.id,
|
|
233
|
-
vector: data.vector,
|
|
234
|
-
connections,
|
|
235
|
-
level: data.level || 0
|
|
236
|
-
};
|
|
237
|
-
// Return pure vector structure
|
|
238
|
-
return node;
|
|
239
|
-
}
|
|
240
|
-
catch (error) {
|
|
241
|
-
// Noun not found or other error
|
|
242
|
-
return null;
|
|
243
|
-
}
|
|
244
|
-
}
|
|
245
|
-
/**
|
|
246
|
-
* Get nouns by noun type (internal implementation)
|
|
247
|
-
* @param nounType The noun type to filter by
|
|
248
|
-
* @returns Promise that resolves to an array of nouns of the specified noun type
|
|
249
|
-
*/
|
|
250
|
-
async getNounsByNounType_internal(nounType) {
|
|
251
|
-
return this.getNodesByNounType(nounType);
|
|
252
|
-
}
|
|
253
|
-
/**
|
|
254
|
-
* Get nodes by noun type
|
|
255
|
-
* @param nounType The noun type to filter by
|
|
256
|
-
* @returns Promise that resolves to an array of nodes of the specified noun type
|
|
257
|
-
*/
|
|
258
|
-
async getNodesByNounType(nounType) {
|
|
259
|
-
await this.ensureInitialized();
|
|
260
|
-
const nodes = [];
|
|
261
|
-
try {
|
|
262
|
-
// Iterate through all shard directories
|
|
263
|
-
for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
|
|
264
|
-
if (shardHandle.kind === 'directory') {
|
|
265
|
-
const shardDir = shardHandle;
|
|
266
|
-
// Iterate through all files in this shard
|
|
267
|
-
for await (const [fileName, fileHandle] of shardDir.entries()) {
|
|
268
|
-
if (fileHandle.kind === 'file') {
|
|
269
|
-
try {
|
|
270
|
-
// Read the node data from the file
|
|
271
|
-
const file = await safeGetFile(fileHandle);
|
|
272
|
-
const text = await file.text();
|
|
273
|
-
const data = JSON.parse(text);
|
|
274
|
-
// Get the metadata to check the noun type
|
|
275
|
-
const metadata = await this.getMetadata(data.id);
|
|
276
|
-
// Include the node if its noun type matches the requested type
|
|
277
|
-
if (metadata && metadata.noun === nounType) {
|
|
278
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
279
|
-
const connections = new Map();
|
|
280
|
-
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
|
281
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
282
|
-
}
|
|
283
|
-
nodes.push({
|
|
284
|
-
id: data.id,
|
|
285
|
-
vector: data.vector,
|
|
286
|
-
connections,
|
|
287
|
-
level: data.level || 0
|
|
288
|
-
});
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
catch (error) {
|
|
292
|
-
console.error(`Error reading node file ${shardName}/${fileName}:`, error);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
catch (error) {
|
|
300
|
-
console.error('Error reading nouns directory:', error);
|
|
301
|
-
}
|
|
302
|
-
return nodes;
|
|
303
|
-
}
|
|
304
|
-
/**
|
|
305
|
-
* Delete a noun from storage (internal implementation)
|
|
306
|
-
*/
|
|
307
|
-
async deleteNoun_internal(id) {
|
|
308
|
-
return this.deleteNode(id);
|
|
309
|
-
}
|
|
310
|
-
/**
|
|
311
|
-
* Delete a node from storage
|
|
312
|
-
*/
|
|
313
|
-
async deleteNode(id) {
|
|
314
|
-
await this.ensureInitialized();
|
|
315
|
-
try {
|
|
316
|
-
// Use UUID-based sharding for nouns
|
|
317
|
-
const shardId = getShardIdFromUuid(id);
|
|
318
|
-
// Get the shard directory
|
|
319
|
-
const shardDir = await this.nounsDir.getDirectoryHandle(shardId);
|
|
320
|
-
// Delete the file from the shard directory
|
|
321
|
-
await shardDir.removeEntry(`${id}.json`);
|
|
322
|
-
}
|
|
323
|
-
catch (error) {
|
|
324
|
-
// Ignore NotFoundError, which means the file doesn't exist
|
|
325
|
-
if (error.name !== 'NotFoundError') {
|
|
326
|
-
console.error(`Error deleting node ${id}:`, error);
|
|
327
|
-
throw error;
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
/**
|
|
332
|
-
* Save a verb to storage (internal implementation)
|
|
333
|
-
*/
|
|
334
|
-
async saveVerb_internal(verb) {
|
|
335
|
-
return this.saveEdge(verb);
|
|
336
|
-
}
|
|
337
|
-
/**
|
|
338
|
-
* Save an edge to storage
|
|
339
|
-
*/
|
|
340
|
-
async saveEdge(edge) {
|
|
341
|
-
await this.ensureInitialized();
|
|
342
|
-
try {
|
|
343
|
-
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
|
|
344
|
-
// These fields are essential for 90% of operations - no metadata lookup needed
|
|
345
|
-
const serializableEdge = {
|
|
346
|
-
id: edge.id,
|
|
347
|
-
vector: edge.vector,
|
|
348
|
-
connections: this.mapToObject(edge.connections, (set) => Array.from(set)),
|
|
349
|
-
// CORE RELATIONAL DATA (v3.50.1+)
|
|
350
|
-
verb: edge.verb,
|
|
351
|
-
sourceId: edge.sourceId,
|
|
352
|
-
targetId: edge.targetId,
|
|
353
|
-
// User metadata (if any) - saved separately for scalability
|
|
354
|
-
// metadata field is saved separately via saveVerbMetadata()
|
|
355
|
-
};
|
|
356
|
-
// Use UUID-based sharding for verbs
|
|
357
|
-
const shardId = getShardIdFromUuid(edge.id);
|
|
358
|
-
// Get or create the shard directory
|
|
359
|
-
const shardDir = await this.verbsDir.getDirectoryHandle(shardId, {
|
|
360
|
-
create: true
|
|
361
|
-
});
|
|
362
|
-
// Create or get the file in the shard directory
|
|
363
|
-
const fileHandle = await shardDir.getFileHandle(`${edge.id}.json`, {
|
|
364
|
-
create: true
|
|
365
|
-
});
|
|
366
|
-
// Write the verb data to the file
|
|
367
|
-
const writable = await fileHandle.createWritable();
|
|
368
|
-
await writable.write(JSON.stringify(serializableEdge));
|
|
369
|
-
await writable.close();
|
|
370
|
-
}
|
|
371
|
-
catch (error) {
|
|
372
|
-
console.error(`Failed to save edge ${edge.id}:`, error);
|
|
373
|
-
throw new Error(`Failed to save edge ${edge.id}: ${error}`);
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
/**
|
|
377
|
-
* Get a verb from storage (internal implementation)
|
|
378
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
379
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
380
|
-
*/
|
|
381
|
-
async getVerb_internal(id) {
|
|
382
|
-
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
|
383
|
-
const edge = await this.getEdge(id);
|
|
384
|
-
if (!edge) {
|
|
385
|
-
return null;
|
|
386
|
-
}
|
|
387
|
-
// Return pure vector + core fields structure
|
|
388
|
-
return edge;
|
|
389
|
-
}
|
|
390
|
-
/**
|
|
391
|
-
* Get an edge from storage
|
|
392
|
-
*/
|
|
393
|
-
async getEdge(id) {
|
|
394
|
-
await this.ensureInitialized();
|
|
395
|
-
try {
|
|
396
|
-
// Use UUID-based sharding for verbs
|
|
397
|
-
const shardId = getShardIdFromUuid(id);
|
|
398
|
-
// Get the shard directory
|
|
399
|
-
const shardDir = await this.verbsDir.getDirectoryHandle(shardId);
|
|
400
|
-
// Get the file handle from the shard directory
|
|
401
|
-
const fileHandle = await shardDir.getFileHandle(`${id}.json`);
|
|
402
|
-
// Read the edge data from the file
|
|
403
|
-
const file = await fileHandle.getFile();
|
|
404
|
-
const text = await file.text();
|
|
405
|
-
const data = JSON.parse(text);
|
|
406
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
407
|
-
const connections = new Map();
|
|
408
|
-
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
|
409
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
410
|
-
}
|
|
411
|
-
// Create default timestamp if not present
|
|
412
|
-
const defaultTimestamp = {
|
|
413
|
-
seconds: Math.floor(Date.now() / 1000),
|
|
414
|
-
nanoseconds: (Date.now() % 1000) * 1000000
|
|
415
|
-
};
|
|
416
|
-
// Create default createdBy if not present
|
|
417
|
-
const defaultCreatedBy = {
|
|
418
|
-
augmentation: 'unknown',
|
|
419
|
-
version: '1.0'
|
|
420
|
-
};
|
|
421
|
-
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
|
422
|
-
return {
|
|
423
|
-
id: data.id,
|
|
424
|
-
vector: data.vector,
|
|
425
|
-
connections,
|
|
426
|
-
// CORE RELATIONAL DATA (read from vector file)
|
|
427
|
-
verb: data.verb,
|
|
428
|
-
sourceId: data.sourceId,
|
|
429
|
-
targetId: data.targetId
|
|
430
|
-
// ✅ NO metadata field in v4.0.0
|
|
431
|
-
// User metadata retrieved separately via getVerbMetadata()
|
|
432
|
-
};
|
|
433
|
-
}
|
|
434
|
-
catch (error) {
|
|
435
|
-
// Edge not found or other error
|
|
436
|
-
return null;
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
/**
|
|
440
|
-
* Get all edges from storage
|
|
441
|
-
*/
|
|
442
|
-
async getAllEdges() {
|
|
443
|
-
await this.ensureInitialized();
|
|
444
|
-
const allEdges = [];
|
|
445
|
-
try {
|
|
446
|
-
// Iterate through all shard directories
|
|
447
|
-
for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
|
|
448
|
-
if (shardHandle.kind === 'directory') {
|
|
449
|
-
const shardDir = shardHandle;
|
|
450
|
-
// Iterate through all files in this shard
|
|
451
|
-
for await (const [fileName, fileHandle] of shardDir.entries()) {
|
|
452
|
-
if (fileHandle.kind === 'file') {
|
|
453
|
-
try {
|
|
454
|
-
// Read the edge data from the file
|
|
455
|
-
const file = await safeGetFile(fileHandle);
|
|
456
|
-
const text = await file.text();
|
|
457
|
-
const data = JSON.parse(text);
|
|
458
|
-
// Convert serialized connections back to Map<number, Set<string>>
|
|
459
|
-
const connections = new Map();
|
|
460
|
-
for (const [level, nodeIds] of Object.entries(data.connections)) {
|
|
461
|
-
connections.set(Number(level), new Set(nodeIds));
|
|
462
|
-
}
|
|
463
|
-
// Create default timestamp if not present
|
|
464
|
-
const defaultTimestamp = {
|
|
465
|
-
seconds: Math.floor(Date.now() / 1000),
|
|
466
|
-
nanoseconds: (Date.now() % 1000) * 1000000
|
|
467
|
-
};
|
|
468
|
-
// Create default createdBy if not present
|
|
469
|
-
const defaultCreatedBy = {
|
|
470
|
-
augmentation: 'unknown',
|
|
471
|
-
version: '1.0'
|
|
472
|
-
};
|
|
473
|
-
// v4.0.0: Include core relational fields (NO metadata field)
|
|
474
|
-
allEdges.push({
|
|
475
|
-
id: data.id,
|
|
476
|
-
vector: data.vector,
|
|
477
|
-
connections,
|
|
478
|
-
// CORE RELATIONAL DATA
|
|
479
|
-
verb: data.verb,
|
|
480
|
-
sourceId: data.sourceId,
|
|
481
|
-
targetId: data.targetId
|
|
482
|
-
// ✅ NO metadata field in v4.0.0
|
|
483
|
-
// User metadata retrieved separately via getVerbMetadata()
|
|
484
|
-
});
|
|
485
|
-
}
|
|
486
|
-
catch (error) {
|
|
487
|
-
console.error(`Error reading edge file ${shardName}/${fileName}:`, error);
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
catch (error) {
|
|
495
|
-
console.error('Error reading verbs directory:', error);
|
|
496
|
-
}
|
|
497
|
-
return allEdges;
|
|
498
|
-
}
|
|
499
|
-
/**
|
|
500
|
-
* Get verbs by source (internal implementation)
|
|
501
|
-
*/
|
|
502
|
-
async getVerbsBySource_internal(sourceId) {
|
|
503
|
-
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
|
504
|
-
const result = await this.getVerbsWithPagination({
|
|
505
|
-
filter: { sourceId: [sourceId] },
|
|
506
|
-
limit: Number.MAX_SAFE_INTEGER // Get all matching results
|
|
507
|
-
});
|
|
508
|
-
return result.items;
|
|
509
|
-
}
|
|
510
|
-
/**
|
|
511
|
-
* Get edges by source
|
|
512
|
-
*/
|
|
513
|
-
async getEdgesBySource(sourceId) {
|
|
514
|
-
// This method is deprecated and would require loading metadata for each edge
|
|
515
|
-
// For now, return empty array since this is not efficiently implementable with new storage pattern
|
|
516
|
-
console.warn('getEdgesBySource is deprecated and not efficiently supported in new storage pattern');
|
|
517
|
-
return [];
|
|
518
|
-
}
|
|
519
|
-
/**
|
|
520
|
-
* Get verbs by target (internal implementation)
|
|
521
|
-
*/
|
|
522
|
-
async getVerbsByTarget_internal(targetId) {
|
|
523
|
-
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
|
524
|
-
const result = await this.getVerbsWithPagination({
|
|
525
|
-
filter: { targetId: [targetId] },
|
|
526
|
-
limit: Number.MAX_SAFE_INTEGER // Get all matching results
|
|
527
|
-
});
|
|
528
|
-
return result.items;
|
|
529
|
-
}
|
|
530
|
-
/**
|
|
531
|
-
* Get edges by target
|
|
532
|
-
*/
|
|
533
|
-
async getEdgesByTarget(targetId) {
|
|
534
|
-
// This method is deprecated and would require loading metadata for each edge
|
|
535
|
-
// For now, return empty array since this is not efficiently implementable with new storage pattern
|
|
536
|
-
console.warn('getEdgesByTarget is deprecated and not efficiently supported in new storage pattern');
|
|
537
|
-
return [];
|
|
538
|
-
}
|
|
539
|
-
/**
|
|
540
|
-
* Get verbs by type (internal implementation)
|
|
541
|
-
*/
|
|
542
|
-
async getVerbsByType_internal(type) {
|
|
543
|
-
// Use the paginated approach to properly handle HNSWVerb to GraphVerb conversion
|
|
544
|
-
const result = await this.getVerbsWithPagination({
|
|
545
|
-
filter: { verbType: [type] },
|
|
546
|
-
limit: Number.MAX_SAFE_INTEGER // Get all matching results
|
|
547
|
-
});
|
|
548
|
-
return result.items;
|
|
549
|
-
}
|
|
550
|
-
/**
|
|
551
|
-
* Get edges by type
|
|
552
|
-
*/
|
|
553
|
-
async getEdgesByType(type) {
|
|
554
|
-
// This method is deprecated and would require loading metadata for each edge
|
|
555
|
-
// For now, return empty array since this is not efficiently implementable with new storage pattern
|
|
556
|
-
console.warn('getEdgesByType is deprecated and not efficiently supported in new storage pattern');
|
|
557
|
-
return [];
|
|
558
|
-
}
|
|
559
|
-
/**
|
|
560
|
-
* Delete a verb from storage (internal implementation)
|
|
561
|
-
*/
|
|
562
|
-
async deleteVerb_internal(id) {
|
|
563
|
-
return this.deleteEdge(id);
|
|
564
|
-
}
|
|
178
|
+
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
|
|
565
179
|
/**
|
|
566
180
|
* Delete an edge from storage
|
|
567
181
|
*/
|
|
@@ -1399,252 +1013,7 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1399
1013
|
* @param options Pagination and filter options
|
|
1400
1014
|
* @returns Promise that resolves to a paginated result of nouns
|
|
1401
1015
|
*/
|
|
1402
|
-
|
|
1403
|
-
await this.ensureInitialized();
|
|
1404
|
-
const limit = options.limit || 100;
|
|
1405
|
-
const cursor = options.cursor;
|
|
1406
|
-
// Get all noun files from all shards
|
|
1407
|
-
const nounFiles = [];
|
|
1408
|
-
if (this.nounsDir) {
|
|
1409
|
-
// Iterate through all shard directories
|
|
1410
|
-
for await (const [shardName, shardHandle] of this.nounsDir.entries()) {
|
|
1411
|
-
if (shardHandle.kind === 'directory') {
|
|
1412
|
-
// Iterate through files in this shard
|
|
1413
|
-
const shardDir = shardHandle;
|
|
1414
|
-
for await (const [fileName, fileHandle] of shardDir.entries()) {
|
|
1415
|
-
if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
|
|
1416
|
-
nounFiles.push(`${shardName}/${fileName}`);
|
|
1417
|
-
}
|
|
1418
|
-
}
|
|
1419
|
-
}
|
|
1420
|
-
}
|
|
1421
|
-
}
|
|
1422
|
-
// Sort files for consistent ordering
|
|
1423
|
-
nounFiles.sort();
|
|
1424
|
-
// Apply cursor-based pagination
|
|
1425
|
-
let startIndex = 0;
|
|
1426
|
-
if (cursor) {
|
|
1427
|
-
const cursorIndex = nounFiles.findIndex(file => file > cursor);
|
|
1428
|
-
if (cursorIndex >= 0) {
|
|
1429
|
-
startIndex = cursorIndex;
|
|
1430
|
-
}
|
|
1431
|
-
}
|
|
1432
|
-
// Get the subset of files for this page
|
|
1433
|
-
const pageFiles = nounFiles.slice(startIndex, startIndex + limit);
|
|
1434
|
-
// v4.0.0: Load nouns from files and combine with metadata
|
|
1435
|
-
const items = [];
|
|
1436
|
-
for (const fileName of pageFiles) {
|
|
1437
|
-
// fileName is in format "shard/uuid.json", extract just the UUID
|
|
1438
|
-
const id = fileName.split('/')[1].replace('.json', '');
|
|
1439
|
-
const noun = await this.getNoun_internal(id);
|
|
1440
|
-
if (noun) {
|
|
1441
|
-
// Load metadata for filtering and combining
|
|
1442
|
-
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
|
|
1443
|
-
const metadata = await this.getNounMetadata(id);
|
|
1444
|
-
// Apply filters if provided
|
|
1445
|
-
if (options.filter && metadata) {
|
|
1446
|
-
// Filter by noun type
|
|
1447
|
-
if (options.filter.nounType) {
|
|
1448
|
-
const nounTypes = Array.isArray(options.filter.nounType)
|
|
1449
|
-
? options.filter.nounType
|
|
1450
|
-
: [options.filter.nounType];
|
|
1451
|
-
if (!nounTypes.includes((metadata.type || metadata.noun))) {
|
|
1452
|
-
continue;
|
|
1453
|
-
}
|
|
1454
|
-
}
|
|
1455
|
-
// Filter by service
|
|
1456
|
-
if (options.filter.service) {
|
|
1457
|
-
const services = Array.isArray(options.filter.service)
|
|
1458
|
-
? options.filter.service
|
|
1459
|
-
: [options.filter.service];
|
|
1460
|
-
if (!metadata.createdBy?.augmentation || !services.includes(metadata.createdBy.augmentation)) {
|
|
1461
|
-
continue;
|
|
1462
|
-
}
|
|
1463
|
-
}
|
|
1464
|
-
// Filter by metadata
|
|
1465
|
-
if (options.filter.metadata) {
|
|
1466
|
-
let matches = true;
|
|
1467
|
-
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
|
1468
|
-
if (metadata[key] !== value) {
|
|
1469
|
-
matches = false;
|
|
1470
|
-
break;
|
|
1471
|
-
}
|
|
1472
|
-
}
|
|
1473
|
-
if (!matches)
|
|
1474
|
-
continue;
|
|
1475
|
-
}
|
|
1476
|
-
}
|
|
1477
|
-
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1478
|
-
const metadataObj = (metadata || {});
|
|
1479
|
-
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
1480
|
-
const nounWithMetadata = {
|
|
1481
|
-
id: noun.id,
|
|
1482
|
-
vector: [...noun.vector],
|
|
1483
|
-
connections: new Map(noun.connections),
|
|
1484
|
-
level: noun.level || 0,
|
|
1485
|
-
type: nounType || NounType.Thing,
|
|
1486
|
-
createdAt: createdAt || Date.now(),
|
|
1487
|
-
updatedAt: updatedAt || Date.now(),
|
|
1488
|
-
confidence: confidence,
|
|
1489
|
-
weight: weight,
|
|
1490
|
-
service: service,
|
|
1491
|
-
data: data,
|
|
1492
|
-
createdBy,
|
|
1493
|
-
metadata: customMetadata
|
|
1494
|
-
};
|
|
1495
|
-
items.push(nounWithMetadata);
|
|
1496
|
-
}
|
|
1497
|
-
}
|
|
1498
|
-
// Determine if there are more items
|
|
1499
|
-
const hasMore = startIndex + limit < nounFiles.length;
|
|
1500
|
-
// Generate next cursor if there are more items
|
|
1501
|
-
const nextCursor = hasMore && pageFiles.length > 0
|
|
1502
|
-
? pageFiles[pageFiles.length - 1]
|
|
1503
|
-
: undefined;
|
|
1504
|
-
return {
|
|
1505
|
-
items,
|
|
1506
|
-
totalCount: nounFiles.length,
|
|
1507
|
-
hasMore,
|
|
1508
|
-
nextCursor
|
|
1509
|
-
};
|
|
1510
|
-
}
|
|
1511
|
-
/**
|
|
1512
|
-
* Get verbs with pagination support
|
|
1513
|
-
* @param options Pagination and filter options
|
|
1514
|
-
* @returns Promise that resolves to a paginated result of verbs
|
|
1515
|
-
*/
|
|
1516
|
-
async getVerbsWithPagination(options = {}) {
|
|
1517
|
-
await this.ensureInitialized();
|
|
1518
|
-
const limit = options.limit || 100;
|
|
1519
|
-
const cursor = options.cursor;
|
|
1520
|
-
// Get all verb files from all shards
|
|
1521
|
-
const verbFiles = [];
|
|
1522
|
-
if (this.verbsDir) {
|
|
1523
|
-
// Iterate through all shard directories
|
|
1524
|
-
for await (const [shardName, shardHandle] of this.verbsDir.entries()) {
|
|
1525
|
-
if (shardHandle.kind === 'directory') {
|
|
1526
|
-
// Iterate through files in this shard
|
|
1527
|
-
const shardDir = shardHandle;
|
|
1528
|
-
for await (const [fileName, fileHandle] of shardDir.entries()) {
|
|
1529
|
-
if (fileHandle.kind === 'file' && fileName.endsWith('.json')) {
|
|
1530
|
-
verbFiles.push(`${shardName}/${fileName}`);
|
|
1531
|
-
}
|
|
1532
|
-
}
|
|
1533
|
-
}
|
|
1534
|
-
}
|
|
1535
|
-
}
|
|
1536
|
-
// Sort files for consistent ordering
|
|
1537
|
-
verbFiles.sort();
|
|
1538
|
-
// Apply cursor-based pagination
|
|
1539
|
-
let startIndex = 0;
|
|
1540
|
-
if (cursor) {
|
|
1541
|
-
const cursorIndex = verbFiles.findIndex(file => file > cursor);
|
|
1542
|
-
if (cursorIndex >= 0) {
|
|
1543
|
-
startIndex = cursorIndex;
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1546
|
-
// Get the subset of files for this page
|
|
1547
|
-
const pageFiles = verbFiles.slice(startIndex, startIndex + limit);
|
|
1548
|
-
// v4.0.0: Load verbs from files and combine with metadata
|
|
1549
|
-
const items = [];
|
|
1550
|
-
for (const fileName of pageFiles) {
|
|
1551
|
-
// fileName is in format "shard/uuid.json", extract just the UUID
|
|
1552
|
-
const id = fileName.split('/')[1].replace('.json', '');
|
|
1553
|
-
const hnswVerb = await this.getVerb_internal(id);
|
|
1554
|
-
if (hnswVerb) {
|
|
1555
|
-
// Load metadata for filtering and combining
|
|
1556
|
-
// FIX v4.7.4: Don't skip verbs without metadata - metadata is optional in v4.0.0
|
|
1557
|
-
// Core fields (verb, sourceId, targetId) are in HNSWVerb itself
|
|
1558
|
-
const metadata = await this.getVerbMetadata(id);
|
|
1559
|
-
// Apply filters if provided
|
|
1560
|
-
if (options.filter && metadata) {
|
|
1561
|
-
// Filter by verb type
|
|
1562
|
-
// v4.0.0: verb field is in HNSWVerb structure (NOT in metadata)
|
|
1563
|
-
if (options.filter.verbType) {
|
|
1564
|
-
const verbTypes = Array.isArray(options.filter.verbType)
|
|
1565
|
-
? options.filter.verbType
|
|
1566
|
-
: [options.filter.verbType];
|
|
1567
|
-
if (!hnswVerb.verb || !verbTypes.includes(hnswVerb.verb)) {
|
|
1568
|
-
continue;
|
|
1569
|
-
}
|
|
1570
|
-
}
|
|
1571
|
-
// Filter by source ID
|
|
1572
|
-
// v4.0.0: sourceId field is in HNSWVerb structure (NOT in metadata)
|
|
1573
|
-
if (options.filter.sourceId) {
|
|
1574
|
-
const sourceIds = Array.isArray(options.filter.sourceId)
|
|
1575
|
-
? options.filter.sourceId
|
|
1576
|
-
: [options.filter.sourceId];
|
|
1577
|
-
if (!hnswVerb.sourceId || !sourceIds.includes(hnswVerb.sourceId)) {
|
|
1578
|
-
continue;
|
|
1579
|
-
}
|
|
1580
|
-
}
|
|
1581
|
-
// Filter by target ID
|
|
1582
|
-
// v4.0.0: targetId field is in HNSWVerb structure (NOT in metadata)
|
|
1583
|
-
if (options.filter.targetId) {
|
|
1584
|
-
const targetIds = Array.isArray(options.filter.targetId)
|
|
1585
|
-
? options.filter.targetId
|
|
1586
|
-
: [options.filter.targetId];
|
|
1587
|
-
if (!hnswVerb.targetId || !targetIds.includes(hnswVerb.targetId)) {
|
|
1588
|
-
continue;
|
|
1589
|
-
}
|
|
1590
|
-
}
|
|
1591
|
-
// Filter by service
|
|
1592
|
-
if (options.filter.service) {
|
|
1593
|
-
const services = Array.isArray(options.filter.service)
|
|
1594
|
-
? options.filter.service
|
|
1595
|
-
: [options.filter.service];
|
|
1596
|
-
if (!metadata.createdBy?.augmentation || !services.includes(metadata.createdBy.augmentation)) {
|
|
1597
|
-
continue;
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
// Filter by metadata
|
|
1601
|
-
if (options.filter.metadata) {
|
|
1602
|
-
let matches = true;
|
|
1603
|
-
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
|
1604
|
-
if (metadata[key] !== value) {
|
|
1605
|
-
matches = false;
|
|
1606
|
-
break;
|
|
1607
|
-
}
|
|
1608
|
-
}
|
|
1609
|
-
if (!matches)
|
|
1610
|
-
continue;
|
|
1611
|
-
}
|
|
1612
|
-
}
|
|
1613
|
-
// v4.8.0: Extract standard fields from metadata to top-level
|
|
1614
|
-
const metadataObj = (metadata || {});
|
|
1615
|
-
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
1616
|
-
const verbWithMetadata = {
|
|
1617
|
-
id: hnswVerb.id,
|
|
1618
|
-
vector: [...hnswVerb.vector],
|
|
1619
|
-
connections: new Map(hnswVerb.connections),
|
|
1620
|
-
verb: hnswVerb.verb,
|
|
1621
|
-
sourceId: hnswVerb.sourceId,
|
|
1622
|
-
targetId: hnswVerb.targetId,
|
|
1623
|
-
createdAt: createdAt || Date.now(),
|
|
1624
|
-
updatedAt: updatedAt || Date.now(),
|
|
1625
|
-
confidence: confidence,
|
|
1626
|
-
weight: weight,
|
|
1627
|
-
service: service,
|
|
1628
|
-
data: data,
|
|
1629
|
-
createdBy,
|
|
1630
|
-
metadata: customMetadata
|
|
1631
|
-
};
|
|
1632
|
-
items.push(verbWithMetadata);
|
|
1633
|
-
}
|
|
1634
|
-
}
|
|
1635
|
-
// Determine if there are more items
|
|
1636
|
-
const hasMore = startIndex + limit < verbFiles.length;
|
|
1637
|
-
// Generate next cursor if there are more items
|
|
1638
|
-
const nextCursor = hasMore && pageFiles.length > 0
|
|
1639
|
-
? pageFiles[pageFiles.length - 1]
|
|
1640
|
-
: undefined;
|
|
1641
|
-
return {
|
|
1642
|
-
items,
|
|
1643
|
-
totalCount: verbFiles.length,
|
|
1644
|
-
hasMore,
|
|
1645
|
-
nextCursor
|
|
1646
|
-
};
|
|
1647
|
-
}
|
|
1016
|
+
// v5.4.0: Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
|
|
1648
1017
|
/**
|
|
1649
1018
|
* Initialize counts from OPFS storage
|
|
1650
1019
|
*/
|
|
@@ -1727,21 +1096,21 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1727
1096
|
/**
|
|
1728
1097
|
* Get a noun's vector for HNSW rebuild
|
|
1729
1098
|
*/
|
|
1099
|
+
/**
|
|
1100
|
+
* Get vector for a noun
|
|
1101
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
1102
|
+
*/
|
|
1730
1103
|
async getNounVector(id) {
|
|
1731
|
-
await this.
|
|
1732
|
-
const noun = await this.getNoun_internal(id);
|
|
1104
|
+
const noun = await this.getNoun(id);
|
|
1733
1105
|
return noun ? noun.vector : null;
|
|
1734
1106
|
}
|
|
1735
1107
|
/**
|
|
1736
1108
|
* Save HNSW graph data for a noun
|
|
1737
|
-
* Storage path: nouns/hnsw/{shard}/{id}.json
|
|
1738
1109
|
*
|
|
1739
|
-
*
|
|
1740
|
-
*
|
|
1741
|
-
* Prevents data corruption when multiple entities connect to same neighbor simultaneously
|
|
1110
|
+
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
|
1111
|
+
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
|
|
1742
1112
|
*/
|
|
1743
1113
|
async saveHNSWData(nounId, hnswData) {
|
|
1744
|
-
await this.ensureInitialized();
|
|
1745
1114
|
const lockKey = `hnsw/${nounId}`;
|
|
1746
1115
|
// MUTEX LOCK: Wait for any pending operations on this entity
|
|
1747
1116
|
while (this.hnswLocks.has(lockKey)) {
|
|
@@ -1752,36 +1121,24 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1752
1121
|
const lockPromise = new Promise(resolve => { releaseLock = resolve; });
|
|
1753
1122
|
this.hnswLocks.set(lockKey, lockPromise);
|
|
1754
1123
|
try {
|
|
1755
|
-
//
|
|
1756
|
-
const
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
const fileHandle = await shardDir.getFileHandle(`${nounId}.json`, { create: true });
|
|
1760
|
-
try {
|
|
1761
|
-
// Read existing node data
|
|
1762
|
-
const file = await fileHandle.getFile();
|
|
1763
|
-
const existingData = await file.text();
|
|
1764
|
-
const existingNode = JSON.parse(existingData);
|
|
1765
|
-
// Preserve id and vector, update only HNSW graph metadata
|
|
1766
|
-
const updatedNode = {
|
|
1767
|
-
...existingNode,
|
|
1768
|
-
level: hnswData.level,
|
|
1769
|
-
connections: hnswData.connections
|
|
1770
|
-
};
|
|
1771
|
-
const writable = await fileHandle.createWritable();
|
|
1772
|
-
await writable.write(JSON.stringify(updatedNode, null, 2));
|
|
1773
|
-
await writable.close();
|
|
1124
|
+
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
|
|
1125
|
+
const existingNoun = await this.getNoun(nounId);
|
|
1126
|
+
if (!existingNoun) {
|
|
1127
|
+
throw new Error(`Cannot save HNSW data: noun ${nounId} not found`);
|
|
1774
1128
|
}
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
await writable.close();
|
|
1129
|
+
// Convert connections from Record to Map format
|
|
1130
|
+
const connectionsMap = new Map();
|
|
1131
|
+
for (const [level, nodeIds] of Object.entries(hnswData.connections)) {
|
|
1132
|
+
connectionsMap.set(Number(level), new Set(nodeIds));
|
|
1780
1133
|
}
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1134
|
+
// Preserve id and vector, update only HNSW graph metadata
|
|
1135
|
+
const updatedNoun = {
|
|
1136
|
+
...existingNoun,
|
|
1137
|
+
level: hnswData.level,
|
|
1138
|
+
connections: connectionsMap
|
|
1139
|
+
};
|
|
1140
|
+
// v5.4.0: Use BaseStorage's saveNoun (type-first paths)
|
|
1141
|
+
await this.saveNoun(updatedNoun);
|
|
1785
1142
|
}
|
|
1786
1143
|
finally {
|
|
1787
1144
|
// Release lock
|
|
@@ -1791,30 +1148,24 @@ export class OPFSStorage extends BaseStorage {
|
|
|
1791
1148
|
}
|
|
1792
1149
|
/**
|
|
1793
1150
|
* Get HNSW graph data for a noun
|
|
1794
|
-
*
|
|
1151
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
1795
1152
|
*/
|
|
1796
1153
|
async getHNSWData(nounId) {
|
|
1797
|
-
await this.
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
const hnswDir = await this.nounsDir.getDirectoryHandle('hnsw');
|
|
1801
|
-
// Use sharded path for HNSW data
|
|
1802
|
-
const shard = getShardIdFromUuid(nounId);
|
|
1803
|
-
const shardDir = await hnswDir.getDirectoryHandle(shard);
|
|
1804
|
-
// Get the file handle from the shard directory
|
|
1805
|
-
const fileHandle = await shardDir.getFileHandle(`${nounId}.json`);
|
|
1806
|
-
// Read the HNSW data from the file
|
|
1807
|
-
const file = await fileHandle.getFile();
|
|
1808
|
-
const text = await file.text();
|
|
1809
|
-
return JSON.parse(text);
|
|
1154
|
+
const noun = await this.getNoun(nounId);
|
|
1155
|
+
if (!noun) {
|
|
1156
|
+
return null;
|
|
1810
1157
|
}
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1158
|
+
// Convert connections from Map to Record format
|
|
1159
|
+
const connectionsRecord = {};
|
|
1160
|
+
if (noun.connections) {
|
|
1161
|
+
for (const [level, nodeIds] of noun.connections.entries()) {
|
|
1162
|
+
connectionsRecord[String(level)] = Array.from(nodeIds);
|
|
1814
1163
|
}
|
|
1815
|
-
console.error(`Failed to get HNSW data for ${nounId}:`, error);
|
|
1816
|
-
throw new Error(`Failed to get HNSW data for ${nounId}: ${error}`);
|
|
1817
1164
|
}
|
|
1165
|
+
return {
|
|
1166
|
+
level: noun.level || 0,
|
|
1167
|
+
connections: connectionsRecord
|
|
1168
|
+
};
|
|
1818
1169
|
}
|
|
1819
1170
|
/**
|
|
1820
1171
|
* Save HNSW system data (entry point, max level)
|