@soulcraft/brainy 5.3.6 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/dist/brainy.d.ts +61 -0
- package/dist/brainy.js +179 -23
- package/dist/storage/adapters/azureBlobStorage.d.ts +13 -64
- package/dist/storage/adapters/azureBlobStorage.js +78 -388
- package/dist/storage/adapters/fileSystemStorage.d.ts +12 -78
- package/dist/storage/adapters/fileSystemStorage.js +49 -395
- package/dist/storage/adapters/gcsStorage.d.ts +13 -134
- package/dist/storage/adapters/gcsStorage.js +79 -557
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +181 -0
- package/dist/storage/adapters/historicalStorageAdapter.js +332 -0
- package/dist/storage/adapters/memoryStorage.d.ts +4 -113
- package/dist/storage/adapters/memoryStorage.js +34 -471
- package/dist/storage/adapters/opfsStorage.d.ts +14 -127
- package/dist/storage/adapters/opfsStorage.js +44 -693
- package/dist/storage/adapters/r2Storage.d.ts +8 -41
- package/dist/storage/adapters/r2Storage.js +49 -237
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +13 -111
- package/dist/storage/adapters/s3CompatibleStorage.js +77 -596
- package/dist/storage/baseStorage.d.ts +78 -38
- package/dist/storage/baseStorage.js +692 -23
- package/dist/storage/cow/BlobStorage.d.ts +2 -2
- package/dist/storage/cow/BlobStorage.js +4 -4
- package/dist/storage/storageFactory.d.ts +2 -3
- package/dist/storage/storageFactory.js +114 -66
- package/dist/vfs/types.d.ts +6 -2
- package/package.json +1 -1
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Based on latest GCS and S3 implementations with R2-specific enhancements
|
|
13
13
|
*/
|
|
14
|
-
import { HNSWNoun, HNSWVerb,
|
|
14
|
+
import { HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
|
15
15
|
import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
|
|
16
16
|
type HNSWNode = HNSWNoun;
|
|
17
17
|
type Edge = HNSWVerb;
|
|
@@ -19,6 +19,12 @@ type Edge = HNSWVerb;
|
|
|
19
19
|
* Dedicated Cloudflare R2 storage adapter
|
|
20
20
|
* Optimized for R2's unique characteristics and global edge network
|
|
21
21
|
*
|
|
22
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
23
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
24
|
+
* - Removed getNounsWithPagination override
|
|
25
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
26
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
27
|
+
*
|
|
22
28
|
* Configuration:
|
|
23
29
|
* ```typescript
|
|
24
30
|
* const r2Storage = new R2Storage({
|
|
@@ -59,6 +65,7 @@ export declare class R2Storage extends BaseStorage {
|
|
|
59
65
|
private nounCacheManager;
|
|
60
66
|
private verbCacheManager;
|
|
61
67
|
private logger;
|
|
68
|
+
private hnswLocks;
|
|
62
69
|
/**
|
|
63
70
|
* Initialize the R2 storage adapter
|
|
64
71
|
* @param options Configuration options for Cloudflare R2
|
|
@@ -133,10 +140,6 @@ export declare class R2Storage extends BaseStorage {
|
|
|
133
140
|
* Flush verb buffer to R2
|
|
134
141
|
*/
|
|
135
142
|
private flushVerbBuffer;
|
|
136
|
-
/**
|
|
137
|
-
* Save a noun to storage (internal implementation)
|
|
138
|
-
*/
|
|
139
|
-
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
140
143
|
/**
|
|
141
144
|
* Save a node to storage
|
|
142
145
|
*/
|
|
@@ -145,20 +148,10 @@ export declare class R2Storage extends BaseStorage {
|
|
|
145
148
|
* Save a node directly to R2 (bypass buffer)
|
|
146
149
|
*/
|
|
147
150
|
private saveNodeDirect;
|
|
148
|
-
/**
|
|
149
|
-
* Get a noun from storage (internal implementation)
|
|
150
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
151
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
152
|
-
*/
|
|
153
|
-
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
154
151
|
/**
|
|
155
152
|
* Get a node from storage
|
|
156
153
|
*/
|
|
157
154
|
protected getNode(id: string): Promise<HNSWNode | null>;
|
|
158
|
-
/**
|
|
159
|
-
* Delete a noun from storage (internal implementation)
|
|
160
|
-
*/
|
|
161
|
-
protected deleteNoun_internal(id: string): Promise<void>;
|
|
162
155
|
/**
|
|
163
156
|
* Write an object to a specific path in R2
|
|
164
157
|
*/
|
|
@@ -175,17 +168,9 @@ export declare class R2Storage extends BaseStorage {
|
|
|
175
168
|
* List all objects under a specific prefix in R2
|
|
176
169
|
*/
|
|
177
170
|
protected listObjectsUnderPath(prefix: string): Promise<string[]>;
|
|
178
|
-
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
179
171
|
protected saveEdge(edge: Edge): Promise<void>;
|
|
180
172
|
private saveEdgeDirect;
|
|
181
|
-
/**
|
|
182
|
-
* Get a verb from storage (internal implementation)
|
|
183
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
184
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
185
|
-
*/
|
|
186
|
-
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
187
173
|
protected getEdge(id: string): Promise<Edge | null>;
|
|
188
|
-
protected deleteVerb_internal(id: string): Promise<void>;
|
|
189
174
|
protected initializeCounts(): Promise<void>;
|
|
190
175
|
private initializeCountsFromScan;
|
|
191
176
|
protected persistCounts(): Promise<void>;
|
|
@@ -215,23 +200,5 @@ export declare class R2Storage extends BaseStorage {
|
|
|
215
200
|
quota: number | null;
|
|
216
201
|
details?: Record<string, any>;
|
|
217
202
|
}>;
|
|
218
|
-
getNounsWithPagination(options?: {
|
|
219
|
-
limit?: number;
|
|
220
|
-
cursor?: string;
|
|
221
|
-
filter?: {
|
|
222
|
-
nounType?: string | string[];
|
|
223
|
-
service?: string | string[];
|
|
224
|
-
metadata?: Record<string, any>;
|
|
225
|
-
};
|
|
226
|
-
}): Promise<{
|
|
227
|
-
items: HNSWNounWithMetadata[];
|
|
228
|
-
totalCount?: number;
|
|
229
|
-
hasMore: boolean;
|
|
230
|
-
nextCursor?: string;
|
|
231
|
-
}>;
|
|
232
|
-
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
233
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
234
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
235
|
-
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
236
203
|
}
|
|
237
204
|
export {};
|
|
@@ -11,7 +11,6 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Based on latest GCS and S3 implementations with R2-specific enhancements
|
|
13
13
|
*/
|
|
14
|
-
import { NounType } from '../../coreTypes.js';
|
|
15
14
|
import { BaseStorage, SYSTEM_DIR, STATISTICS_KEY, getDirectoryPath } from '../baseStorage.js';
|
|
16
15
|
import { BrainyError } from '../../errors/brainyError.js';
|
|
17
16
|
import { CacheManager } from '../cacheManager.js';
|
|
@@ -26,6 +25,12 @@ const MAX_R2_PAGE_SIZE = 1000;
|
|
|
26
25
|
* Dedicated Cloudflare R2 storage adapter
|
|
27
26
|
* Optimized for R2's unique characteristics and global edge network
|
|
28
27
|
*
|
|
28
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
29
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
30
|
+
* - Removed getNounsWithPagination override
|
|
31
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
32
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
33
|
+
*
|
|
29
34
|
* Configuration:
|
|
30
35
|
* ```typescript
|
|
31
36
|
* const r2Storage = new R2Storage({
|
|
@@ -67,6 +72,8 @@ export class R2Storage extends BaseStorage {
|
|
|
67
72
|
this.forceHighVolumeMode = false;
|
|
68
73
|
// Module logger
|
|
69
74
|
this.logger = createModuleLogger('R2Storage');
|
|
75
|
+
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
|
|
76
|
+
this.hnswLocks = new Map();
|
|
70
77
|
this.bucketName = options.bucketName;
|
|
71
78
|
this.accountId = options.accountId;
|
|
72
79
|
this.accessKeyId = options.accessKeyId;
|
|
@@ -283,12 +290,6 @@ export class R2Storage extends BaseStorage {
|
|
|
283
290
|
});
|
|
284
291
|
await Promise.all(writes);
|
|
285
292
|
}
|
|
286
|
-
/**
|
|
287
|
-
* Save a noun to storage (internal implementation)
|
|
288
|
-
*/
|
|
289
|
-
async saveNoun_internal(noun) {
|
|
290
|
-
return this.saveNode(noun);
|
|
291
|
-
}
|
|
292
293
|
/**
|
|
293
294
|
* Save a node to storage
|
|
294
295
|
*/
|
|
@@ -353,20 +354,6 @@ export class R2Storage extends BaseStorage {
|
|
|
353
354
|
throw new Error(`Failed to save node ${node.id}: ${error}`);
|
|
354
355
|
}
|
|
355
356
|
}
|
|
356
|
-
/**
|
|
357
|
-
* Get a noun from storage (internal implementation)
|
|
358
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
359
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
360
|
-
*/
|
|
361
|
-
async getNoun_internal(id) {
|
|
362
|
-
// v4.0.0: Return ONLY vector data (no metadata field)
|
|
363
|
-
const node = await this.getNode(id);
|
|
364
|
-
if (!node) {
|
|
365
|
-
return null;
|
|
366
|
-
}
|
|
367
|
-
// Return pure vector structure
|
|
368
|
-
return node;
|
|
369
|
-
}
|
|
370
357
|
/**
|
|
371
358
|
* Get a node from storage
|
|
372
359
|
*/
|
|
@@ -429,45 +416,6 @@ export class R2Storage extends BaseStorage {
|
|
|
429
416
|
throw BrainyError.fromError(error, `getNoun(${id})`);
|
|
430
417
|
}
|
|
431
418
|
}
|
|
432
|
-
/**
|
|
433
|
-
* Delete a noun from storage (internal implementation)
|
|
434
|
-
*/
|
|
435
|
-
async deleteNoun_internal(id) {
|
|
436
|
-
await this.ensureInitialized();
|
|
437
|
-
const requestId = await this.applyBackpressure();
|
|
438
|
-
try {
|
|
439
|
-
this.logger.trace(`Deleting noun ${id}`);
|
|
440
|
-
const key = this.getNounKey(id);
|
|
441
|
-
// Delete from R2 using S3 DeleteObject
|
|
442
|
-
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
|
443
|
-
await this.s3Client.send(new DeleteObjectCommand({
|
|
444
|
-
Bucket: this.bucketName,
|
|
445
|
-
Key: key
|
|
446
|
-
}));
|
|
447
|
-
// Remove from cache
|
|
448
|
-
this.nounCacheManager.delete(id);
|
|
449
|
-
// Decrement noun count
|
|
450
|
-
const metadata = await this.getNounMetadata(id);
|
|
451
|
-
if (metadata && metadata.type) {
|
|
452
|
-
await this.decrementEntityCountSafe(metadata.type);
|
|
453
|
-
}
|
|
454
|
-
this.logger.trace(`Noun ${id} deleted successfully`);
|
|
455
|
-
this.releaseBackpressure(true, requestId);
|
|
456
|
-
}
|
|
457
|
-
catch (error) {
|
|
458
|
-
this.releaseBackpressure(false, requestId);
|
|
459
|
-
if (error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
|
|
460
|
-
this.logger.trace(`Noun ${id} not found (already deleted)`);
|
|
461
|
-
return;
|
|
462
|
-
}
|
|
463
|
-
if (this.isThrottlingError(error)) {
|
|
464
|
-
await this.handleThrottling(error);
|
|
465
|
-
throw error;
|
|
466
|
-
}
|
|
467
|
-
this.logger.error(`Failed to delete noun ${id}:`, error);
|
|
468
|
-
throw new Error(`Failed to delete noun ${id}: ${error}`);
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
419
|
/**
|
|
472
420
|
* Write an object to a specific path in R2
|
|
473
421
|
*/
|
|
@@ -563,9 +511,6 @@ export class R2Storage extends BaseStorage {
|
|
|
563
511
|
}
|
|
564
512
|
}
|
|
565
513
|
// Verb storage methods (similar to noun methods - implementing key methods for space)
|
|
566
|
-
async saveVerb_internal(verb) {
|
|
567
|
-
return this.saveEdge(verb);
|
|
568
|
-
}
|
|
569
514
|
async saveEdge(edge) {
|
|
570
515
|
await this.ensureInitialized();
|
|
571
516
|
this.checkVolumeMode();
|
|
@@ -616,20 +561,6 @@ export class R2Storage extends BaseStorage {
|
|
|
616
561
|
throw new Error(`Failed to save edge ${edge.id}: ${error}`);
|
|
617
562
|
}
|
|
618
563
|
}
|
|
619
|
-
/**
|
|
620
|
-
* Get a verb from storage (internal implementation)
|
|
621
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
622
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
623
|
-
*/
|
|
624
|
-
async getVerb_internal(id) {
|
|
625
|
-
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
|
626
|
-
const edge = await this.getEdge(id);
|
|
627
|
-
if (!edge) {
|
|
628
|
-
return null;
|
|
629
|
-
}
|
|
630
|
-
// Return pure vector + core fields structure
|
|
631
|
-
return edge;
|
|
632
|
-
}
|
|
633
564
|
async getEdge(id) {
|
|
634
565
|
await this.ensureInitialized();
|
|
635
566
|
const cached = this.verbCacheManager.get(id);
|
|
@@ -678,35 +609,6 @@ export class R2Storage extends BaseStorage {
|
|
|
678
609
|
throw BrainyError.fromError(error, `getVerb(${id})`);
|
|
679
610
|
}
|
|
680
611
|
}
|
|
681
|
-
async deleteVerb_internal(id) {
|
|
682
|
-
await this.ensureInitialized();
|
|
683
|
-
const requestId = await this.applyBackpressure();
|
|
684
|
-
try {
|
|
685
|
-
const key = this.getVerbKey(id);
|
|
686
|
-
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
|
687
|
-
await this.s3Client.send(new DeleteObjectCommand({
|
|
688
|
-
Bucket: this.bucketName,
|
|
689
|
-
Key: key
|
|
690
|
-
}));
|
|
691
|
-
this.verbCacheManager.delete(id);
|
|
692
|
-
const metadata = await this.getVerbMetadata(id);
|
|
693
|
-
if (metadata && metadata.type) {
|
|
694
|
-
await this.decrementVerbCount(metadata.type);
|
|
695
|
-
}
|
|
696
|
-
this.releaseBackpressure(true, requestId);
|
|
697
|
-
}
|
|
698
|
-
catch (error) {
|
|
699
|
-
this.releaseBackpressure(false, requestId);
|
|
700
|
-
if (error.name === 'NoSuchKey' || error.$metadata?.httpStatusCode === 404) {
|
|
701
|
-
return;
|
|
702
|
-
}
|
|
703
|
-
if (this.isThrottlingError(error)) {
|
|
704
|
-
await this.handleThrottling(error);
|
|
705
|
-
throw error;
|
|
706
|
-
}
|
|
707
|
-
throw new Error(`Failed to delete verb ${id}: ${error}`);
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
612
|
// Pagination and count management (simplified for space - full implementation similar to GCS)
|
|
711
613
|
async initializeCounts() {
|
|
712
614
|
const key = `${this.systemPrefix}counts.json`;
|
|
@@ -776,42 +678,55 @@ export class R2Storage extends BaseStorage {
|
|
|
776
678
|
}
|
|
777
679
|
// HNSW Index Persistence (Phase 2 support)
|
|
778
680
|
async getNounVector(id) {
|
|
779
|
-
await this.
|
|
780
|
-
const noun = await this.getNode(id);
|
|
681
|
+
const noun = await this.getNoun(id);
|
|
781
682
|
return noun ? noun.vector : null;
|
|
782
683
|
}
|
|
783
684
|
async saveHNSWData(nounId, hnswData) {
|
|
784
|
-
|
|
785
|
-
//
|
|
786
|
-
|
|
787
|
-
|
|
685
|
+
const lockKey = `hnsw/${nounId}`;
|
|
686
|
+
// Wait for pending operations
|
|
687
|
+
while (this.hnswLocks.has(lockKey)) {
|
|
688
|
+
await this.hnswLocks.get(lockKey);
|
|
689
|
+
}
|
|
690
|
+
// Acquire lock
|
|
691
|
+
let releaseLock;
|
|
692
|
+
const lockPromise = new Promise(resolve => { releaseLock = resolve; });
|
|
693
|
+
this.hnswLocks.set(lockKey, lockPromise);
|
|
788
694
|
try {
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
}
|
|
695
|
+
const existingNoun = await this.getNoun(nounId);
|
|
696
|
+
if (!existingNoun) {
|
|
697
|
+
throw new Error(`Cannot save HNSW data: noun ${nounId} not found`);
|
|
698
|
+
}
|
|
699
|
+
const connectionsMap = new Map();
|
|
700
|
+
for (const [level, nodeIds] of Object.entries(hnswData.connections)) {
|
|
701
|
+
connectionsMap.set(Number(level), new Set(nodeIds));
|
|
702
|
+
}
|
|
703
|
+
const updatedNoun = {
|
|
704
|
+
...existingNoun,
|
|
705
|
+
level: hnswData.level,
|
|
706
|
+
connections: connectionsMap
|
|
707
|
+
};
|
|
708
|
+
await this.saveNoun(updatedNoun);
|
|
804
709
|
}
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
710
|
+
finally {
|
|
711
|
+
this.hnswLocks.delete(lockKey);
|
|
712
|
+
releaseLock();
|
|
808
713
|
}
|
|
809
714
|
}
|
|
810
715
|
async getHNSWData(nounId) {
|
|
811
|
-
await this.
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
716
|
+
const noun = await this.getNoun(nounId);
|
|
717
|
+
if (!noun) {
|
|
718
|
+
return null;
|
|
719
|
+
}
|
|
720
|
+
const connectionsRecord = {};
|
|
721
|
+
if (noun.connections) {
|
|
722
|
+
for (const [level, nodeIds] of noun.connections.entries()) {
|
|
723
|
+
connectionsRecord[String(level)] = Array.from(nodeIds);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
return {
|
|
727
|
+
level: noun.level || 0,
|
|
728
|
+
connections: connectionsRecord
|
|
729
|
+
};
|
|
815
730
|
}
|
|
816
731
|
async saveHNSWSystem(systemData) {
|
|
817
732
|
await this.ensureInitialized();
|
|
@@ -889,108 +804,5 @@ export class R2Storage extends BaseStorage {
|
|
|
889
804
|
}
|
|
890
805
|
};
|
|
891
806
|
}
|
|
892
|
-
// Pagination support (simplified - full implementation would match GCS pattern)
|
|
893
|
-
async getNounsWithPagination(options = {}) {
|
|
894
|
-
await this.ensureInitialized();
|
|
895
|
-
// Simplified pagination - full implementation would be similar to GCS
|
|
896
|
-
const limit = options.limit || 100;
|
|
897
|
-
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
|
898
|
-
const response = await this.s3Client.send(new ListObjectsV2Command({
|
|
899
|
-
Bucket: this.bucketName,
|
|
900
|
-
Prefix: this.nounPrefix,
|
|
901
|
-
MaxKeys: limit,
|
|
902
|
-
ContinuationToken: options.cursor
|
|
903
|
-
}));
|
|
904
|
-
const items = [];
|
|
905
|
-
const contents = response.Contents || [];
|
|
906
|
-
for (const obj of contents) {
|
|
907
|
-
if (!obj.Key?.endsWith('.json'))
|
|
908
|
-
continue;
|
|
909
|
-
const id = obj.Key.split('/').pop()?.replace('.json', '');
|
|
910
|
-
if (!id)
|
|
911
|
-
continue;
|
|
912
|
-
const noun = await this.getNoun_internal(id);
|
|
913
|
-
if (noun) {
|
|
914
|
-
// v4.0.0: Load metadata and combine with noun to create HNSWNounWithMetadata
|
|
915
|
-
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
|
|
916
|
-
const metadata = await this.getNounMetadata(id);
|
|
917
|
-
// Apply filters if provided
|
|
918
|
-
if (options.filter && metadata) {
|
|
919
|
-
// Filter by noun type
|
|
920
|
-
if (options.filter.nounType) {
|
|
921
|
-
const nounTypes = Array.isArray(options.filter.nounType)
|
|
922
|
-
? options.filter.nounType
|
|
923
|
-
: [options.filter.nounType];
|
|
924
|
-
if (!nounTypes.includes((metadata.type || metadata.noun))) {
|
|
925
|
-
continue;
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
// Filter by service
|
|
929
|
-
if (options.filter.service) {
|
|
930
|
-
const services = Array.isArray(options.filter.service)
|
|
931
|
-
? options.filter.service
|
|
932
|
-
: [options.filter.service];
|
|
933
|
-
if (!metadata.createdBy?.augmentation || !services.includes(metadata.createdBy.augmentation)) {
|
|
934
|
-
continue;
|
|
935
|
-
}
|
|
936
|
-
}
|
|
937
|
-
// Filter by metadata
|
|
938
|
-
if (options.filter.metadata) {
|
|
939
|
-
let matches = true;
|
|
940
|
-
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
|
941
|
-
if (metadata[key] !== value) {
|
|
942
|
-
matches = false;
|
|
943
|
-
break;
|
|
944
|
-
}
|
|
945
|
-
}
|
|
946
|
-
if (!matches)
|
|
947
|
-
continue;
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
// v4.8.0: Extract standard fields from metadata to top-level
|
|
951
|
-
const metadataObj = (metadata || {});
|
|
952
|
-
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj;
|
|
953
|
-
const nounWithMetadata = {
|
|
954
|
-
id: noun.id,
|
|
955
|
-
vector: [...noun.vector],
|
|
956
|
-
connections: new Map(noun.connections),
|
|
957
|
-
level: noun.level || 0,
|
|
958
|
-
type: nounType || NounType.Thing,
|
|
959
|
-
createdAt: createdAt || Date.now(),
|
|
960
|
-
updatedAt: updatedAt || Date.now(),
|
|
961
|
-
confidence: confidence,
|
|
962
|
-
weight: weight,
|
|
963
|
-
service: service,
|
|
964
|
-
data: data,
|
|
965
|
-
createdBy,
|
|
966
|
-
metadata: customMetadata
|
|
967
|
-
};
|
|
968
|
-
items.push(nounWithMetadata);
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
return {
|
|
972
|
-
items,
|
|
973
|
-
totalCount: this.totalNounCount,
|
|
974
|
-
hasMore: !!response.IsTruncated,
|
|
975
|
-
nextCursor: response.NextContinuationToken
|
|
976
|
-
};
|
|
977
|
-
}
|
|
978
|
-
async getNounsByNounType_internal(nounType) {
|
|
979
|
-
const result = await this.getNounsWithPagination({
|
|
980
|
-
limit: 10000,
|
|
981
|
-
filter: { nounType }
|
|
982
|
-
});
|
|
983
|
-
return result.items;
|
|
984
|
-
}
|
|
985
|
-
async getVerbsBySource_internal(sourceId) {
|
|
986
|
-
// Simplified - full implementation would include proper filtering
|
|
987
|
-
return [];
|
|
988
|
-
}
|
|
989
|
-
async getVerbsByTarget_internal(targetId) {
|
|
990
|
-
return [];
|
|
991
|
-
}
|
|
992
|
-
async getVerbsByType_internal(type) {
|
|
993
|
-
return [];
|
|
994
|
-
}
|
|
995
807
|
}
|
|
996
808
|
//# sourceMappingURL=r2Storage.js.map
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Uses the AWS S3 client to interact with S3-compatible storage services
|
|
4
4
|
* including Amazon S3, Cloudflare R2, and Google Cloud Storage
|
|
5
5
|
*/
|
|
6
|
-
import { Change, HNSWNoun, HNSWVerb,
|
|
6
|
+
import { Change, HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
|
7
7
|
import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
|
|
8
8
|
import { OperationConfig } from '../../utils/operationUtils.js';
|
|
9
9
|
type HNSWNode = HNSWNoun;
|
|
@@ -29,6 +29,12 @@ type Edge = HNSWVerb;
|
|
|
29
29
|
* - credentials: GCS credentials (accessKeyId and secretAccessKey)
|
|
30
30
|
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
|
|
31
31
|
* - bucketName: GCS bucket name
|
|
32
|
+
*
|
|
33
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
34
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
35
|
+
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
|
36
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
37
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
32
38
|
*/
|
|
33
39
|
export declare class S3CompatibleStorage extends BaseStorage {
|
|
34
40
|
private s3Client;
|
|
@@ -75,6 +81,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
75
81
|
private nounCacheManager;
|
|
76
82
|
private verbCacheManager;
|
|
77
83
|
private logger;
|
|
84
|
+
private hnswLocks;
|
|
78
85
|
/**
|
|
79
86
|
* Initialize the storage adapter
|
|
80
87
|
* @param options Configuration options for the S3-compatible storage
|
|
@@ -217,20 +224,10 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
217
224
|
* Get current batch size for operations
|
|
218
225
|
*/
|
|
219
226
|
private getBatchSize;
|
|
220
|
-
/**
|
|
221
|
-
* Save a noun to storage (internal implementation)
|
|
222
|
-
*/
|
|
223
|
-
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
224
227
|
/**
|
|
225
228
|
* Save a node to storage
|
|
226
229
|
*/
|
|
227
230
|
protected saveNode(node: HNSWNode): Promise<void>;
|
|
228
|
-
/**
|
|
229
|
-
* Get a noun from storage (internal implementation)
|
|
230
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
231
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
232
|
-
*/
|
|
233
|
-
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
234
231
|
/**
|
|
235
232
|
* Get a node from storage
|
|
236
233
|
*/
|
|
@@ -274,44 +271,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
274
271
|
* Load nodes by IDs efficiently using cache or direct fetch
|
|
275
272
|
*/
|
|
276
273
|
private loadNodesByIds;
|
|
277
|
-
/**
|
|
278
|
-
* Get nouns by noun type (internal implementation)
|
|
279
|
-
* @param nounType The noun type to filter by
|
|
280
|
-
* @returns Promise that resolves to an array of nouns of the specified noun type
|
|
281
|
-
*/
|
|
282
|
-
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
283
|
-
/**
|
|
284
|
-
* Get nodes by noun type
|
|
285
|
-
* @param nounType The noun type to filter by
|
|
286
|
-
* @returns Promise that resolves to an array of nodes of the specified noun type
|
|
287
|
-
*/
|
|
288
|
-
protected getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
|
|
289
|
-
/**
|
|
290
|
-
* Delete a noun from storage (internal implementation)
|
|
291
|
-
*/
|
|
292
|
-
protected deleteNoun_internal(id: string): Promise<void>;
|
|
293
|
-
/**
|
|
294
|
-
* Delete a node from storage
|
|
295
|
-
*/
|
|
296
|
-
protected deleteNode(id: string): Promise<void>;
|
|
297
|
-
/**
|
|
298
|
-
* Save a verb to storage (internal implementation)
|
|
299
|
-
*/
|
|
300
|
-
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
301
|
-
/**
|
|
302
|
-
* Save an edge to storage
|
|
303
|
-
*/
|
|
304
|
-
protected saveEdge(edge: Edge): Promise<void>;
|
|
305
|
-
/**
|
|
306
|
-
* Get a verb from storage (internal implementation)
|
|
307
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
308
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
309
|
-
*/
|
|
310
|
-
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
311
|
-
/**
|
|
312
|
-
* Get an edge from storage
|
|
313
|
-
*/
|
|
314
|
-
protected getEdge(id: string): Promise<Edge | null>;
|
|
315
274
|
/**
|
|
316
275
|
* Get all edges from storage
|
|
317
276
|
* @deprecated This method is deprecated and will be removed in a future version.
|
|
@@ -344,47 +303,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
344
303
|
* @returns True if the edge matches the filter, false otherwise
|
|
345
304
|
*/
|
|
346
305
|
private filterEdge;
|
|
347
|
-
/**
|
|
348
|
-
* Get verbs with pagination
|
|
349
|
-
* @param options Pagination options
|
|
350
|
-
* @returns Promise that resolves to a paginated result of verbs
|
|
351
|
-
*/
|
|
352
|
-
getVerbsWithPagination(options?: {
|
|
353
|
-
limit?: number;
|
|
354
|
-
cursor?: string;
|
|
355
|
-
filter?: {
|
|
356
|
-
verbType?: string | string[];
|
|
357
|
-
sourceId?: string | string[];
|
|
358
|
-
targetId?: string | string[];
|
|
359
|
-
service?: string | string[];
|
|
360
|
-
metadata?: Record<string, any>;
|
|
361
|
-
};
|
|
362
|
-
}): Promise<{
|
|
363
|
-
items: HNSWVerbWithMetadata[];
|
|
364
|
-
totalCount?: number;
|
|
365
|
-
hasMore: boolean;
|
|
366
|
-
nextCursor?: string;
|
|
367
|
-
}>;
|
|
368
|
-
/**
|
|
369
|
-
* Get verbs by source (internal implementation)
|
|
370
|
-
*/
|
|
371
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
372
|
-
/**
|
|
373
|
-
* Get verbs by target (internal implementation)
|
|
374
|
-
*/
|
|
375
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
376
|
-
/**
|
|
377
|
-
* Get verbs by type (internal implementation)
|
|
378
|
-
*/
|
|
379
|
-
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
380
|
-
/**
|
|
381
|
-
* Delete a verb from storage (internal implementation)
|
|
382
|
-
*/
|
|
383
|
-
protected deleteVerb_internal(id: string): Promise<void>;
|
|
384
|
-
/**
|
|
385
|
-
* Delete an edge from storage
|
|
386
|
-
*/
|
|
387
|
-
protected deleteEdge(id: string): Promise<void>;
|
|
388
306
|
/**
|
|
389
307
|
* Primitive operation: Write object to path
|
|
390
308
|
* All metadata operations use this internally via base class routing
|
|
@@ -558,25 +476,6 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
558
476
|
* This method should be called periodically
|
|
559
477
|
*/
|
|
560
478
|
private cleanupExpiredLocks;
|
|
561
|
-
/**
|
|
562
|
-
* Get nouns with pagination support
|
|
563
|
-
* @param options Pagination options
|
|
564
|
-
* @returns Promise that resolves to a paginated result of nouns
|
|
565
|
-
*/
|
|
566
|
-
getNounsWithPagination(options?: {
|
|
567
|
-
limit?: number;
|
|
568
|
-
cursor?: string;
|
|
569
|
-
filter?: {
|
|
570
|
-
nounType?: string | string[];
|
|
571
|
-
service?: string | string[];
|
|
572
|
-
metadata?: Record<string, any>;
|
|
573
|
-
};
|
|
574
|
-
}): Promise<{
|
|
575
|
-
items: HNSWNounWithMetadata[];
|
|
576
|
-
totalCount?: number;
|
|
577
|
-
hasMore: boolean;
|
|
578
|
-
nextCursor?: string;
|
|
579
|
-
}>;
|
|
580
479
|
/**
|
|
581
480
|
* Estimate total noun count by listing objects across all shards
|
|
582
481
|
* This is more efficient than loading all nouns
|
|
@@ -605,11 +504,14 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
605
504
|
protected isCloudStorage(): boolean;
|
|
606
505
|
/**
|
|
607
506
|
* Get a noun's vector for HNSW rebuild
|
|
507
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
608
508
|
*/
|
|
609
509
|
getNounVector(id: string): Promise<number[] | null>;
|
|
610
510
|
/**
|
|
611
511
|
* Save HNSW graph data for a noun
|
|
612
|
-
*
|
|
512
|
+
*
|
|
513
|
+
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
|
514
|
+
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
|
613
515
|
*/
|
|
614
516
|
saveHNSWData(nounId: string, hnswData: {
|
|
615
517
|
level: number;
|
|
@@ -617,7 +519,7 @@ export declare class S3CompatibleStorage extends BaseStorage {
|
|
|
617
519
|
}): Promise<void>;
|
|
618
520
|
/**
|
|
619
521
|
* Get HNSW graph data for a noun
|
|
620
|
-
*
|
|
522
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
621
523
|
*/
|
|
622
524
|
getHNSWData(nounId: string): Promise<{
|
|
623
525
|
level: number;
|