@soulcraft/brainy 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +516 -402
- package/dist/brainyData.d.ts +136 -0
- package/dist/coreTypes.d.ts +26 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts.map +1 -1
- package/dist/storage/adapters/fileSystemStorage.d.ts.map +1 -1
- package/dist/storage/cacheManager.d.ts +94 -13
- package/dist/storage/cacheManager.d.ts.map +1 -1
- package/dist/storage/storageFactory.d.ts +47 -3
- package/dist/storage/storageFactory.d.ts.map +1 -1
- package/dist/unified.js +61974 -29728
- package/dist/unified.min.js +4624 -982
- package/dist/utils/embedding.d.ts +11 -17
- package/dist/utils/embedding.d.ts.map +1 -1
- package/dist/utils/fieldNameTracking.d.ts +21 -0
- package/dist/utils/fieldNameTracking.d.ts.map +1 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/jsonProcessing.d.ts +43 -0
- package/dist/utils/jsonProcessing.d.ts.map +1 -0
- package/dist/utils/robustModelLoader.d.ts +94 -0
- package/dist/utils/robustModelLoader.d.ts.map +1 -0
- package/package.json +21 -21
package/dist/brainyData.d.ts
CHANGED
|
@@ -14,6 +14,12 @@ export interface BrainyDataConfig {
|
|
|
14
14
|
* through product quantization and disk-based storage
|
|
15
15
|
*/
|
|
16
16
|
hnsw?: Partial<HNSWOptimizedConfig>;
|
|
17
|
+
/**
|
|
18
|
+
* Default service name to use for all operations
|
|
19
|
+
* When specified, this service name will be used for all operations
|
|
20
|
+
* that don't explicitly provide a service name
|
|
21
|
+
*/
|
|
22
|
+
defaultService?: string;
|
|
17
23
|
/**
|
|
18
24
|
* Distance function to use for similarity calculations
|
|
19
25
|
*/
|
|
@@ -55,6 +61,15 @@ export interface BrainyDataConfig {
|
|
|
55
61
|
};
|
|
56
62
|
forceFileSystemStorage?: boolean;
|
|
57
63
|
forceMemoryStorage?: boolean;
|
|
64
|
+
cacheConfig?: {
|
|
65
|
+
hotCacheMaxSize?: number;
|
|
66
|
+
hotCacheEvictionThreshold?: number;
|
|
67
|
+
warmCacheTTL?: number;
|
|
68
|
+
batchSize?: number;
|
|
69
|
+
autoTune?: boolean;
|
|
70
|
+
autoTuneInterval?: number;
|
|
71
|
+
readOnly?: boolean;
|
|
72
|
+
};
|
|
58
73
|
};
|
|
59
74
|
/**
|
|
60
75
|
* Embedding function to convert data to vectors
|
|
@@ -180,6 +195,73 @@ export interface BrainyDataConfig {
|
|
|
180
195
|
*/
|
|
181
196
|
updateIndex?: boolean;
|
|
182
197
|
};
|
|
198
|
+
/**
|
|
199
|
+
* Cache configuration for optimizing search performance
|
|
200
|
+
* Controls how the system caches data for faster access
|
|
201
|
+
* Particularly important for large datasets in S3 or other remote storage
|
|
202
|
+
*/
|
|
203
|
+
cache?: {
|
|
204
|
+
/**
|
|
205
|
+
* Whether to enable auto-tuning of cache parameters
|
|
206
|
+
* When true, the system will automatically adjust cache sizes based on usage patterns
|
|
207
|
+
* Default: true
|
|
208
|
+
*/
|
|
209
|
+
autoTune?: boolean;
|
|
210
|
+
/**
|
|
211
|
+
* The interval (in milliseconds) at which to auto-tune cache parameters
|
|
212
|
+
* Only applies when autoTune is true
|
|
213
|
+
* Default: 60000 (60 seconds)
|
|
214
|
+
*/
|
|
215
|
+
autoTuneInterval?: number;
|
|
216
|
+
/**
|
|
217
|
+
* Maximum size of the hot cache (most frequently accessed items)
|
|
218
|
+
* If provided, overrides the automatically detected optimal size
|
|
219
|
+
* For large datasets, consider values between 5000-50000 depending on available memory
|
|
220
|
+
*/
|
|
221
|
+
hotCacheMaxSize?: number;
|
|
222
|
+
/**
|
|
223
|
+
* Threshold at which to start evicting items from the hot cache
|
|
224
|
+
* Expressed as a fraction of hotCacheMaxSize (0.0 to 1.0)
|
|
225
|
+
* Default: 0.8 (start evicting when cache is 80% full)
|
|
226
|
+
*/
|
|
227
|
+
hotCacheEvictionThreshold?: number;
|
|
228
|
+
/**
|
|
229
|
+
* Time-to-live for items in the warm cache in milliseconds
|
|
230
|
+
* Default: 3600000 (1 hour)
|
|
231
|
+
*/
|
|
232
|
+
warmCacheTTL?: number;
|
|
233
|
+
/**
|
|
234
|
+
* Batch size for operations like prefetching
|
|
235
|
+
* Larger values improve throughput but use more memory
|
|
236
|
+
* For S3 or remote storage with large datasets, consider values between 50-200
|
|
237
|
+
*/
|
|
238
|
+
batchSize?: number;
|
|
239
|
+
/**
|
|
240
|
+
* Read-only mode specific optimizations
|
|
241
|
+
* These settings are only applied when readOnly is true
|
|
242
|
+
*/
|
|
243
|
+
readOnlyMode?: {
|
|
244
|
+
/**
|
|
245
|
+
* Maximum size of the hot cache in read-only mode
|
|
246
|
+
* In read-only mode, larger cache sizes can be used since there are no write operations
|
|
247
|
+
* For large datasets, consider values between 10000-100000 depending on available memory
|
|
248
|
+
*/
|
|
249
|
+
hotCacheMaxSize?: number;
|
|
250
|
+
/**
|
|
251
|
+
* Batch size for operations in read-only mode
|
|
252
|
+
* Larger values improve throughput in read-only mode
|
|
253
|
+
* For S3 or remote storage with large datasets, consider values between 100-300
|
|
254
|
+
*/
|
|
255
|
+
batchSize?: number;
|
|
256
|
+
/**
|
|
257
|
+
* Prefetch strategy for read-only mode
|
|
258
|
+
* Controls how aggressively the system prefetches data
|
|
259
|
+
* Options: 'conservative', 'moderate', 'aggressive'
|
|
260
|
+
* Default: 'moderate'
|
|
261
|
+
*/
|
|
262
|
+
prefetchStrategy?: 'conservative' | 'moderate' | 'aggressive';
|
|
263
|
+
};
|
|
264
|
+
};
|
|
183
265
|
}
|
|
184
266
|
export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
185
267
|
private index;
|
|
@@ -196,8 +278,10 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
196
278
|
private useOptimizedIndex;
|
|
197
279
|
private _dimensions;
|
|
198
280
|
private loggingConfig;
|
|
281
|
+
private defaultService;
|
|
199
282
|
private timeoutConfig;
|
|
200
283
|
private retryConfig;
|
|
284
|
+
private cacheConfig;
|
|
201
285
|
private realtimeUpdateConfig;
|
|
202
286
|
private updateTimerId;
|
|
203
287
|
private lastUpdateTime;
|
|
@@ -279,6 +363,13 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
279
363
|
* @returns The name of the current augmentation or 'default' if none is detected
|
|
280
364
|
*/
|
|
281
365
|
private getCurrentAugmentation;
|
|
366
|
+
/**
|
|
367
|
+
* Get the service name from options or fallback to default service
|
|
368
|
+
* This provides a consistent way to handle service names across all methods
|
|
369
|
+
* @param options Options object that may contain a service property
|
|
370
|
+
* @returns The service name to use for operations
|
|
371
|
+
*/
|
|
372
|
+
private getServiceName;
|
|
282
373
|
/**
|
|
283
374
|
* Initialize the database
|
|
284
375
|
* Loads existing data from storage if available
|
|
@@ -401,6 +492,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
401
492
|
searchConnectedNouns?: boolean;
|
|
402
493
|
verbDirection?: 'outgoing' | 'incoming' | 'both';
|
|
403
494
|
service?: string;
|
|
495
|
+
searchField?: string;
|
|
404
496
|
}): Promise<SearchResult<T>[]>;
|
|
405
497
|
/**
|
|
406
498
|
* Search the local database for similar vectors
|
|
@@ -414,6 +506,8 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
414
506
|
nounTypes?: string[];
|
|
415
507
|
includeVerbs?: boolean;
|
|
416
508
|
service?: string;
|
|
509
|
+
searchField?: string;
|
|
510
|
+
priorityFields?: string[];
|
|
417
511
|
}): Promise<SearchResult<T>[]>;
|
|
418
512
|
/**
|
|
419
513
|
* Find entities similar to a given entity ID
|
|
@@ -664,6 +758,20 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
664
758
|
* @returns A promise that resolves to the embedded vector
|
|
665
759
|
*/
|
|
666
760
|
embed(data: string | string[]): Promise<Vector>;
|
|
761
|
+
/**
|
|
762
|
+
* Calculate similarity between two vectors or between two pieces of text/data
|
|
763
|
+
* This method allows clients to directly calculate similarity scores between items
|
|
764
|
+
* without needing to add them to the database
|
|
765
|
+
*
|
|
766
|
+
* @param a First vector or text/data to compare
|
|
767
|
+
* @param b Second vector or text/data to compare
|
|
768
|
+
* @param options Additional options
|
|
769
|
+
* @returns A promise that resolves to the similarity score (higher means more similar)
|
|
770
|
+
*/
|
|
771
|
+
calculateSimilarity(a: Vector | string | string[], b: Vector | string | string[], options?: {
|
|
772
|
+
forceEmbed?: boolean;
|
|
773
|
+
distanceFunction?: DistanceFunction;
|
|
774
|
+
}): Promise<number>;
|
|
667
775
|
/**
|
|
668
776
|
* Search for verbs by type and/or vector similarity
|
|
669
777
|
* @param queryVectorOrData Query vector or data to search for
|
|
@@ -717,6 +825,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
717
825
|
includeVerbs?: boolean;
|
|
718
826
|
storeResults?: boolean;
|
|
719
827
|
service?: string;
|
|
828
|
+
searchField?: string;
|
|
720
829
|
}): Promise<SearchResult<T>[]>;
|
|
721
830
|
/**
|
|
722
831
|
* Search both local and remote Brainy instances, combining the results
|
|
@@ -731,6 +840,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
731
840
|
includeVerbs?: boolean;
|
|
732
841
|
localFirst?: boolean;
|
|
733
842
|
service?: string;
|
|
843
|
+
searchField?: string;
|
|
734
844
|
}): Promise<SearchResult<T>[]>;
|
|
735
845
|
/**
|
|
736
846
|
* Check if the instance is connected to a remote server
|
|
@@ -855,5 +965,31 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
855
965
|
nounIds: string[];
|
|
856
966
|
verbIds: string[];
|
|
857
967
|
}>;
|
|
968
|
+
/**
|
|
969
|
+
* Get available field names by service
|
|
970
|
+
* This helps users understand what fields are available for searching from different data sources
|
|
971
|
+
* @returns Record of field names by service
|
|
972
|
+
*/
|
|
973
|
+
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
|
974
|
+
/**
|
|
975
|
+
* Get standard field mappings
|
|
976
|
+
* This helps users understand how fields from different services map to standard field names
|
|
977
|
+
* @returns Record of standard field mappings
|
|
978
|
+
*/
|
|
979
|
+
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
980
|
+
/**
|
|
981
|
+
* Search using a standard field name
|
|
982
|
+
* This allows searching across multiple services using a standardized field name
|
|
983
|
+
* @param standardField The standard field name to search in
|
|
984
|
+
* @param searchTerm The term to search for
|
|
985
|
+
* @param k Number of results to return
|
|
986
|
+
* @param options Additional search options
|
|
987
|
+
* @returns Array of search results
|
|
988
|
+
*/
|
|
989
|
+
searchByStandardField(standardField: string, searchTerm: string, k?: number, options?: {
|
|
990
|
+
services?: string[];
|
|
991
|
+
includeVerbs?: boolean;
|
|
992
|
+
searchMode?: 'local' | 'remote' | 'combined';
|
|
993
|
+
}): Promise<SearchResult<T>[]>;
|
|
858
994
|
}
|
|
859
995
|
export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';
|
package/dist/coreTypes.d.ts
CHANGED
|
@@ -140,6 +140,16 @@ export interface StatisticsData {
|
|
|
140
140
|
relate: number;
|
|
141
141
|
total: number;
|
|
142
142
|
};
|
|
143
|
+
/**
|
|
144
|
+
* Field names available for searching, organized by service
|
|
145
|
+
* This helps users understand what fields are available from different data sources
|
|
146
|
+
*/
|
|
147
|
+
fieldNames?: Record<string, string[]>;
|
|
148
|
+
/**
|
|
149
|
+
* Standard field mappings for common field names across services
|
|
150
|
+
* Maps standard field names to the actual field names used by each service
|
|
151
|
+
*/
|
|
152
|
+
standardFieldMappings?: Record<string, Record<string, string[]>>;
|
|
143
153
|
/**
|
|
144
154
|
* Last updated timestamp
|
|
145
155
|
*/
|
|
@@ -298,6 +308,22 @@ export interface StorageAdapter {
|
|
|
298
308
|
* This ensures that any pending statistics updates are written to persistent storage
|
|
299
309
|
*/
|
|
300
310
|
flushStatisticsToStorage(): Promise<void>;
|
|
311
|
+
/**
|
|
312
|
+
* Track field names from a JSON document
|
|
313
|
+
* @param jsonDocument The JSON document to extract field names from
|
|
314
|
+
* @param service The service that inserted the data
|
|
315
|
+
*/
|
|
316
|
+
trackFieldNames(jsonDocument: any, service: string): Promise<void>;
|
|
317
|
+
/**
|
|
318
|
+
* Get available field names by service
|
|
319
|
+
* @returns Record of field names by service
|
|
320
|
+
*/
|
|
321
|
+
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
|
322
|
+
/**
|
|
323
|
+
* Get standard field mappings
|
|
324
|
+
* @returns Record of standard field mappings
|
|
325
|
+
*/
|
|
326
|
+
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
301
327
|
/**
|
|
302
328
|
* Get changes since a specific timestamp
|
|
303
329
|
* @param timestamp The timestamp to get changes since
|
|
@@ -125,6 +125,22 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
|
|
|
125
125
|
* This ensures that any pending statistics updates are written to persistent storage
|
|
126
126
|
*/
|
|
127
127
|
flushStatisticsToStorage(): Promise<void>;
|
|
128
|
+
/**
|
|
129
|
+
* Track field names from a JSON document
|
|
130
|
+
* @param jsonDocument The JSON document to extract field names from
|
|
131
|
+
* @param service The service that inserted the data
|
|
132
|
+
*/
|
|
133
|
+
trackFieldNames(jsonDocument: any, service: string): Promise<void>;
|
|
134
|
+
/**
|
|
135
|
+
* Get available field names by service
|
|
136
|
+
* @returns Record of field names by service
|
|
137
|
+
*/
|
|
138
|
+
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
|
139
|
+
/**
|
|
140
|
+
* Get standard field mappings
|
|
141
|
+
* @returns Record of standard field mappings
|
|
142
|
+
*/
|
|
143
|
+
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
128
144
|
/**
|
|
129
145
|
* Create default statistics data
|
|
130
146
|
* @returns Default statistics data
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseStorageAdapter.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/baseStorageAdapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;
|
|
1
|
+
{"version":3,"file":"baseStorageAdapter.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/baseStorageAdapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAGnE;;GAEG;AACH,8BAAsB,kBAAmB,YAAW,cAAc;IAEhE,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3C,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAEjD,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE7D,QAAQ,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAE9C,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3C,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAEjD,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE3D,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE3D,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAErD,QAAQ,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAE9C,QAAQ,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/D,QAAQ,CAAC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAErD,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAE/B,QAAQ,CAAC,gBAAgB,IAAI,OAAO,CAAC;QACnC,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC;IAEF;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,EAAE;YACX,MAAM,CAAC,EAAE,MAAM,CAAA;YACf,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;QACD,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACF,GAAG,OAAO,CAAC;QACV,KAAK,EAAE,GAAG,EAAE,CAAA;QACZ,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAEF;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,EAAE;YACX,MAAM,CAAC,EAAE,MAAM,CAAA;YACf,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;QACD,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACF,GAAG,OAAO,CAAC;QACV,KAAK,EAAE,GAAG,EAAE,CAAA;QACZ,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAGF,SAAS,CAAC,eAAe,EAAE,cAAc,GAAG,IAAI,CAAO;IAGvD,SAAS,CAAC,4BAA4B,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAO;IAGpE,SAAS,CAAC,kBAAkB,UAAQ;IAGpC,SAAS,CAAC,uBAAuB,SAAI;IAGrC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,QAAO;IAG/C,SAAS,CAAC,QAAQ,CAAC,kBAAkB,SAAQ;IAG7C,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CACnC,UAAU,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,iBAAiB,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAEtE;;;OAGG;IACG,cAAc,CAAC,UAAU,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAc/D;;;OAGG;IACG,aAAa,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IA8BrD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,IAAI;IAyBrC;;OAEG;cACa,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BhD;;;;;OAKG;IACG,kBAAkB,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAClC,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IAoChB;;;;;OAKG;IACG,kBAAkB,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAClC,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IAoChB;;;OAGG;IACG,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BtD;;;OAGG;IACG,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/C;;;;OAIG;IACG,eAAe,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6ExE;;;OAGG;IACG,sBAAsB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAcjE;;;OAGG;IACG,wBAAwB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAcnF;;;OAGG;IACH,SAAS,CAAC,uBAAuB,IAAI,cAAc;CAWpD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fileSystemStorage.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/fileSystemStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxE,OAAO,EACL,WAAW,EAMZ,MAAM,mBAAmB,CAAA;AAG1B,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,SAAS,CAAA;
|
|
1
|
+
{"version":3,"file":"fileSystemStorage.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/fileSystemStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxE,OAAO,EACL,WAAW,EAMZ,MAAM,mBAAmB,CAAA;AAG1B,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,SAAS,CAAA;AA6BrB;;;GAGG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,OAAO,CAAC,OAAO,CAAQ;IACvB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAyB;IAE5C;;;OAGG;gBACS,aAAa,EAAE,MAAM;IAMjC;;OAEG;IACU,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAwDlC;;OAEG;YACW,qBAAqB;IAWnC;;OAEG;cACa,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBvD;;OAEG;cACa,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IA2B7D;;OAEG;cACa,WAAW,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAmClD;;;;OAIG;cACa,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAyCzE;;OAEG;cACa,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrD;;OAEG;cACa,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBnD;;OAEG;cACa,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAgCzD;;OAEG;cACa,WAAW,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;IAwC9C;;OAEG;cACa,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAKnE;;OAEG;cACa,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAKnE;;OAEG;cACa,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAK7D;;OAEG;cACa,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcrD;;OAEG;IACU,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAOnE;;OAEG;IACU,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAezD;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA4CnC;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC;QACvC,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC;IAsHF;;OAEG;IAEH;;OAEG;cACa,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhE;;OAEG;cACa,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAItE;;OAEG;cACa,oBAAoB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAI3D;;OAEG;cACa,2BAA2B,CACzC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,QAAQ,EAAE,CAAC;IAItB;;OAEG;cACa,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D;;OAEG;cACa,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIjE;;OAEG;cACa,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAIvE;;OAEG;cACa,oBAAoB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAI5D;;OAEG;cACa,yBAAyB,CACvC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,EAAE,CAAC;IAIvB;;OAEG;cACa,yBAAyB,CACvC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,EAAE,CAAC;IAIvB;;OAEG;cACa,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAI3E;;OAEG;cACa,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D;;;;;OAKG;YACW,WAAW;IAsDzB;;;;;OAKG;YACW,WAAW;IAwCzB;;OAEG;YACW,mBAAmB;IAqCjC;;OAEG;cACa,kBAAkB,CAChC,UAAU,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC;IAsDhB;;OAEG;cACa,iBAAiB,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;CAGpE"}
|
|
@@ -26,6 +26,12 @@ interface CacheStats {
|
|
|
26
26
|
evictions: number;
|
|
27
27
|
size: number;
|
|
28
28
|
maxSize: number;
|
|
29
|
+
hotCacheSize: number;
|
|
30
|
+
warmCacheSize: number;
|
|
31
|
+
hotCacheHits: number;
|
|
32
|
+
hotCacheMisses: number;
|
|
33
|
+
warmCacheHits: number;
|
|
34
|
+
warmCacheMisses: number;
|
|
29
35
|
}
|
|
30
36
|
/**
|
|
31
37
|
* Multi-level cache manager for efficient data access
|
|
@@ -46,6 +52,7 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
46
52
|
private storageStatistics;
|
|
47
53
|
private warmStorage;
|
|
48
54
|
private coldStorage;
|
|
55
|
+
private options;
|
|
49
56
|
/**
|
|
50
57
|
* Initialize the cache manager
|
|
51
58
|
* @param options Configuration options
|
|
@@ -58,15 +65,67 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
58
65
|
autoTune?: boolean;
|
|
59
66
|
warmStorage?: any;
|
|
60
67
|
coldStorage?: any;
|
|
68
|
+
readOnly?: boolean;
|
|
69
|
+
environmentConfig?: {
|
|
70
|
+
node?: {
|
|
71
|
+
hotCacheMaxSize?: number;
|
|
72
|
+
hotCacheEvictionThreshold?: number;
|
|
73
|
+
warmCacheTTL?: number;
|
|
74
|
+
batchSize?: number;
|
|
75
|
+
};
|
|
76
|
+
browser?: {
|
|
77
|
+
hotCacheMaxSize?: number;
|
|
78
|
+
hotCacheEvictionThreshold?: number;
|
|
79
|
+
warmCacheTTL?: number;
|
|
80
|
+
batchSize?: number;
|
|
81
|
+
};
|
|
82
|
+
worker?: {
|
|
83
|
+
hotCacheMaxSize?: number;
|
|
84
|
+
hotCacheEvictionThreshold?: number;
|
|
85
|
+
warmCacheTTL?: number;
|
|
86
|
+
batchSize?: number;
|
|
87
|
+
};
|
|
88
|
+
[key: string]: {
|
|
89
|
+
hotCacheMaxSize?: number;
|
|
90
|
+
hotCacheEvictionThreshold?: number;
|
|
91
|
+
warmCacheTTL?: number;
|
|
92
|
+
batchSize?: number;
|
|
93
|
+
} | undefined;
|
|
94
|
+
};
|
|
61
95
|
});
|
|
62
96
|
/**
|
|
63
97
|
* Detect the current environment
|
|
64
98
|
*/
|
|
65
99
|
private detectEnvironment;
|
|
66
100
|
/**
|
|
67
|
-
* Detect the optimal cache size based on available memory
|
|
101
|
+
* Detect the optimal cache size based on available memory and operating mode
|
|
102
|
+
*
|
|
103
|
+
* Enhanced to better handle large datasets in S3 or other storage:
|
|
104
|
+
* - Increases cache size for read-only mode
|
|
105
|
+
* - Adjusts based on total dataset size when available
|
|
106
|
+
* - Provides more aggressive caching for large datasets
|
|
107
|
+
* - Optimizes memory usage based on environment
|
|
68
108
|
*/
|
|
69
109
|
private detectOptimalCacheSize;
|
|
110
|
+
/**
|
|
111
|
+
* Async version of detectOptimalCacheSize that uses dynamic imports
|
|
112
|
+
* to access system information in Node.js environments
|
|
113
|
+
*
|
|
114
|
+
* This method provides more accurate memory detection by using
|
|
115
|
+
* the OS module's dynamic import in Node.js environments
|
|
116
|
+
*/
|
|
117
|
+
private detectOptimalCacheSizeAsync;
|
|
118
|
+
/**
|
|
119
|
+
* Detects available memory across different environments
|
|
120
|
+
*
|
|
121
|
+
* This method uses different techniques to detect memory in:
|
|
122
|
+
* - Node.js: Uses the OS module with dynamic import
|
|
123
|
+
* - Browser: Uses performance.memory or navigator.deviceMemory
|
|
124
|
+
* - Worker: Uses performance.memory if available
|
|
125
|
+
*
|
|
126
|
+
* @returns An object with totalMemory and freeMemory in bytes, or null if detection fails
|
|
127
|
+
*/
|
|
128
|
+
private detectAvailableMemory;
|
|
70
129
|
/**
|
|
71
130
|
* Tune cache parameters based on statistics and environment
|
|
72
131
|
* This method is called periodically if auto-tuning is enabled
|
|
@@ -84,18 +143,21 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
84
143
|
*/
|
|
85
144
|
private tuneParameters;
|
|
86
145
|
/**
|
|
87
|
-
* Tune hot cache size based on statistics and
|
|
146
|
+
* Tune hot cache size based on statistics, environment, and operating mode
|
|
88
147
|
*
|
|
89
148
|
* The hot cache size is tuned based on:
|
|
90
149
|
* 1. Available memory in the current environment
|
|
91
150
|
* 2. Total number of nodes and edges in the system
|
|
92
151
|
* 3. Cache hit/miss ratio
|
|
152
|
+
* 4. Operating mode (read-only vs. read-write)
|
|
153
|
+
* 5. Storage type (S3, filesystem, memory)
|
|
93
154
|
*
|
|
94
|
-
*
|
|
95
|
-
* - Start with a size based on available memory
|
|
96
|
-
* -
|
|
97
|
-
* -
|
|
98
|
-
* -
|
|
155
|
+
* Enhanced algorithm:
|
|
156
|
+
* - Start with a size based on available memory and operating mode
|
|
157
|
+
* - For large datasets in S3 or other remote storage, use more aggressive caching
|
|
158
|
+
* - Adjust based on access patterns (read-heavy vs. write-heavy)
|
|
159
|
+
* - For read-only mode, prioritize cache size over eviction speed
|
|
160
|
+
* - Dynamically adjust based on hit/miss ratio and query patterns
|
|
99
161
|
*/
|
|
100
162
|
private tuneHotCacheSize;
|
|
101
163
|
/**
|
|
@@ -105,6 +167,7 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
105
167
|
* It is tuned based on:
|
|
106
168
|
* 1. Cache hit/miss ratio
|
|
107
169
|
* 2. Operation patterns (read-heavy vs. write-heavy workloads)
|
|
170
|
+
* 3. Memory pressure and available resources
|
|
108
171
|
*
|
|
109
172
|
* Algorithm:
|
|
110
173
|
* - Start with a default threshold of 0.8 (80% of max size)
|
|
@@ -112,6 +175,9 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
112
175
|
* - For low hit ratios, decrease the threshold to evict items more aggressively
|
|
113
176
|
* - For read-heavy workloads, use a higher threshold
|
|
114
177
|
* - For write-heavy workloads, use a lower threshold
|
|
178
|
+
* - Under memory pressure, use a lower threshold to conserve resources
|
|
179
|
+
*
|
|
180
|
+
* @param cacheStats Optional cache statistics for more adaptive tuning
|
|
115
181
|
*/
|
|
116
182
|
private tuneEvictionThreshold;
|
|
117
183
|
/**
|
|
@@ -120,15 +186,23 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
120
186
|
* The warm cache TTL determines how long items remain in the warm cache.
|
|
121
187
|
* It is tuned based on:
|
|
122
188
|
* 1. Update frequency from operation statistics
|
|
189
|
+
* 2. Warm cache hit/miss ratio
|
|
190
|
+
* 3. Access patterns and frequency
|
|
191
|
+
* 4. Available storage resources
|
|
123
192
|
*
|
|
124
193
|
* Algorithm:
|
|
125
194
|
* - Start with a default TTL of 24 hours
|
|
126
195
|
* - For frequently updated data, use a shorter TTL
|
|
127
196
|
* - For rarely updated data, use a longer TTL
|
|
197
|
+
* - For frequently accessed data, use a longer TTL
|
|
198
|
+
* - For rarely accessed data, use a shorter TTL
|
|
199
|
+
* - Under storage pressure, use a shorter TTL
|
|
200
|
+
*
|
|
201
|
+
* @param cacheStats Optional cache statistics for more adaptive tuning
|
|
128
202
|
*/
|
|
129
203
|
private tuneWarmCacheTTL;
|
|
130
204
|
/**
|
|
131
|
-
* Tune batch size based on statistics and
|
|
205
|
+
* Tune batch size based on environment, statistics, and operating mode
|
|
132
206
|
*
|
|
133
207
|
* The batch size determines how many items are processed in a single batch
|
|
134
208
|
* for operations like prefetching. It is tuned based on:
|
|
@@ -136,13 +210,20 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
136
210
|
* 2. Available memory
|
|
137
211
|
* 3. Operation patterns
|
|
138
212
|
* 4. Cache hit/miss ratio
|
|
213
|
+
* 5. Operating mode (read-only vs. read-write)
|
|
214
|
+
* 6. Storage type (S3, filesystem, memory)
|
|
215
|
+
* 7. Dataset size
|
|
216
|
+
* 8. Cache efficiency and access patterns
|
|
139
217
|
*
|
|
140
|
-
*
|
|
218
|
+
* Enhanced algorithm:
|
|
141
219
|
* - Start with a default based on the environment
|
|
142
|
-
* -
|
|
143
|
-
* - For
|
|
144
|
-
* -
|
|
145
|
-
* -
|
|
220
|
+
* - For large datasets in S3 or other remote storage, use larger batches
|
|
221
|
+
* - For read-only mode, use larger batches to improve throughput
|
|
222
|
+
* - Dynamically adjust based on network latency and throughput
|
|
223
|
+
* - Balance between memory usage and performance
|
|
224
|
+
* - Adapt to cache hit/miss patterns
|
|
225
|
+
*
|
|
226
|
+
* @param cacheStats Optional cache statistics for more adaptive tuning
|
|
146
227
|
*/
|
|
147
228
|
private tuneBatchSize;
|
|
148
229
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cacheManager.d.ts","sourceRoot":"","sources":["../../src/storage/cacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAKrD,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,SAAS;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB;IAED,UAAU,iBAAiB;QACzB,OAAO,CAAC,EAAE;YACR,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;SACpB,CAAC;KACH;CACF;AAGD,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,SAAS,CAAA;AAWrB,UAAU,UAAU;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"cacheManager.d.ts","sourceRoot":"","sources":["../../src/storage/cacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAKrD,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,SAAS;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB;IAED,UAAU,iBAAiB;QACzB,OAAO,CAAC,EAAE;YACR,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;SACpB,CAAC;KACH;CACF;AAGD,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,SAAS,CAAA;AAWrB,UAAU,UAAU;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,eAAe,EAAE,MAAM,CAAA;CACxB;AAkBD;;GAEG;AACH,qBAAa,YAAY,CAAC,CAAC,SAAS,QAAQ,GAAG,IAAI;IAEjD,OAAO,CAAC,QAAQ,CAAmC;IAGnD,OAAO,CAAC,KAAK,CAYZ;IAGD,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,eAAe,CAAa;IAGpC,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,yBAAyB,CAAQ;IACzC,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,SAAS,CAAQ;IAGzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,iBAAiB,CAAY;IAGrC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,WAAW,CAAK;IAGxB,OAAO,CAAC,OAAO,CAmCd;IAED;;;OAGG;gBACS,OAAO,GAAE;QACnB,eAAe,CAAC,EAAE,MAAM,CAAA;QACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;QAClC,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;QAClB,WAAW,CAAC,EAAE,GAAG,CAAA;QACjB,WAAW,CAAC,EAAE,GAAG,CAAA;QACjB,QAAQ,CAAC,EAAE,OAAO,CAAA;QAClB,iBAAiB,CAAC,EAAE;YAClB,IAAI,CAAC,EAAE;gBACL,eAAe,CAAC,EAAE,MAAM,CAAA;gBACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;gBAClC,YAAY,CAAC,EAAE,MAAM,CAAA;gBACrB,SAAS,CAAC,EAAE,MAAM,CAAA;aACnB,CAAC;YACF,OAAO,CAAC,EAAE;gBACR,eAAe,CAAC,EAAE,MAAM,CAAA;gBACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;gBAClC,YAAY,CAAC,EAAE,MAAM,CAAA;gBACrB,SAAS,CAAC,EAAE,MAAM,CAAA;aACnB,CAAC;YACF,MAAM,CAAC,EAAE;gBACP,eAAe,CAAC,EAAE,MAAM,CAAA;gBACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;gBAClC,YAAY,CAAC,EAAE,MAAM,CAAA;gBACrB,SAAS,CAAC,EAAE,MAAM,CAAA;aACnB,CAAC;YACF,CAAC,GAAG,EAAE,MAAM,GAAG;gBACb,eAAe,CAAC,EAAE,MAAM,CAAA;gBACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;gBAClC,YAAY,CAAC,EAAE,MAAM,CAAA;gBACrB,SAAS,CAAC,EAAE,MAAM,CAAA;aACnB,GAAG,SAAS,CAAA;SACd,CAAA;KACG;IA+CN;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;;;;;;OAQG;IACH,OAAO,CAAC,sBAAsB;IAmH9B;;;;;;OAMG;YACW,2BAA2B;IAmEzC;;;;;;;;;OASG;YACW,qBAAqB;IA6EnC;;;;;;;;;;;;;;OAcG;YACW,cAAc;IAsD5B;;;;;;;;;;;;;;;;OAgBG;YACW,gBAAgB;IA0H9B;;;;;;;;;;;;;;;;;;OAkBG;IACH,OAAO,CAAC,qBAAqB;IAsE7B;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,gBAAgB;IAiExB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,OAAO,CAAC,aAAa;IAmMrB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAmB7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAmB7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;;;OAIG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAuD/C;;;;OAIG;YACW,gBAAgB;IAW9B;;;;OAIG;YACW,kBAAkB;IAWhC;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAkBrB;;;;OAIG;YACW,cAAc;IAa5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;;;OAIG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBpD;;;OAGG;IACU,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B9C;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsCnC;;;OAGG;IACI,QAAQ,IAAI,UAAU;IAI7B;;;OAGG;IACU,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCnD;;;;;;;;;;;;OAYG;YACW,sBAAsB;IAWpC;;;;OAIG;IACU,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAqF5D;;;;OAIG;IACI,kBAAkB,CAAC,WAAW,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,GAAG,IAAI;CAIpE"}
|
|
@@ -6,7 +6,6 @@ import { StorageAdapter } from '../coreTypes.js';
|
|
|
6
6
|
import { MemoryStorage } from './adapters/memoryStorage.js';
|
|
7
7
|
import { OPFSStorage } from './adapters/opfsStorage.js';
|
|
8
8
|
import { S3CompatibleStorage, R2Storage } from './adapters/s3CompatibleStorage.js';
|
|
9
|
-
import { FileSystemStorage } from './adapters/fileSystemStorage.js';
|
|
10
9
|
import { OperationConfig } from '../utils/operationUtils.js';
|
|
11
10
|
/**
|
|
12
11
|
* Options for creating a storage adapter
|
|
@@ -143,6 +142,50 @@ export interface StorageOptions {
|
|
|
143
142
|
* Operation configuration for timeout and retry behavior
|
|
144
143
|
*/
|
|
145
144
|
operationConfig?: OperationConfig;
|
|
145
|
+
/**
|
|
146
|
+
* Cache configuration for optimizing data access
|
|
147
|
+
* Particularly important for S3 and other remote storage
|
|
148
|
+
*/
|
|
149
|
+
cacheConfig?: {
|
|
150
|
+
/**
|
|
151
|
+
* Maximum size of the hot cache (most frequently accessed items)
|
|
152
|
+
* For large datasets, consider values between 5000-50000 depending on available memory
|
|
153
|
+
*/
|
|
154
|
+
hotCacheMaxSize?: number;
|
|
155
|
+
/**
|
|
156
|
+
* Threshold at which to start evicting items from the hot cache
|
|
157
|
+
* Expressed as a fraction of hotCacheMaxSize (0.0 to 1.0)
|
|
158
|
+
* Default: 0.8 (start evicting when cache is 80% full)
|
|
159
|
+
*/
|
|
160
|
+
hotCacheEvictionThreshold?: number;
|
|
161
|
+
/**
|
|
162
|
+
* Time-to-live for items in the warm cache in milliseconds
|
|
163
|
+
* Default: 3600000 (1 hour)
|
|
164
|
+
*/
|
|
165
|
+
warmCacheTTL?: number;
|
|
166
|
+
/**
|
|
167
|
+
* Batch size for operations like prefetching
|
|
168
|
+
* Larger values improve throughput but use more memory
|
|
169
|
+
*/
|
|
170
|
+
batchSize?: number;
|
|
171
|
+
/**
|
|
172
|
+
* Whether to enable auto-tuning of cache parameters
|
|
173
|
+
* When true, the system will automatically adjust cache sizes based on usage patterns
|
|
174
|
+
* Default: true
|
|
175
|
+
*/
|
|
176
|
+
autoTune?: boolean;
|
|
177
|
+
/**
|
|
178
|
+
* The interval (in milliseconds) at which to auto-tune cache parameters
|
|
179
|
+
* Only applies when autoTune is true
|
|
180
|
+
* Default: 60000 (1 minute)
|
|
181
|
+
*/
|
|
182
|
+
autoTuneInterval?: number;
|
|
183
|
+
/**
|
|
184
|
+
* Whether the storage is in read-only mode
|
|
185
|
+
* This affects cache sizing and prefetching strategies
|
|
186
|
+
*/
|
|
187
|
+
readOnly?: boolean;
|
|
188
|
+
};
|
|
146
189
|
}
|
|
147
190
|
/**
|
|
148
191
|
* Create a storage adapter based on the environment and configuration
|
|
@@ -151,6 +194,7 @@ export interface StorageOptions {
|
|
|
151
194
|
*/
|
|
152
195
|
export declare function createStorage(options?: StorageOptions): Promise<StorageAdapter>;
|
|
153
196
|
/**
|
|
154
|
-
* Export
|
|
197
|
+
* Export storage adapters
|
|
155
198
|
*/
|
|
156
|
-
export { MemoryStorage, OPFSStorage,
|
|
199
|
+
export { MemoryStorage, OPFSStorage, S3CompatibleStorage, R2Storage };
|
|
200
|
+
export { FileSystemStorage } from './adapters/fileSystemStorage.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageFactory.d.ts","sourceRoot":"","sources":["../../src/storage/storageFactory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"storageFactory.d.ts","sourceRoot":"","sources":["../../src/storage/storageFactory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EACL,mBAAmB,EACnB,SAAS,EACV,MAAM,mCAAmC,CAAA;AAG1C,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAE5D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,CAAA;IAEtE;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAE5B;;OAEG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAEhC;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAA;IAElC;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE;QACV;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAA;KACtB,CAAA;IAED;;OAEG;IACH,SAAS,CAAC,EAAE;QACV;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAA;QAEjB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;KACxB,CAAA;IAED;;OAEG;IACH,UAAU,CAAC,EAAE;QACX;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAA;KAClB,CAAA;IAED;;OAEG;IACH,eAAe,CAAC,EAAE;QAChB;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAA;QAEhB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAA;KACrB,CAAA;IAED;;OAEG;IACH,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC;;;OAGG;IACH,WAAW,CAAC,EAAE;QACZ;;;WAGG;QACH,eAAe,CAAC,EAAE,MAAM,CAAA;QAExB;;;;WAIG;QACH,yBAAyB,CAAC,EAAE,MAAM,CAAA;QAElC;;;WAGG;QACH,YAAY,CAAC,EAAE,MAAM,CAAA;QAErB;;;WAGG;QACH,SAAS,CAAC,EAAE,MAAM,CAAA;QAElB;;;;WAIG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAA;QAElB;;;;WAIG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;QAEzB;;;WAGG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAA;KACnB,CAAA;CACF;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,cAAc,CAAC,CA2PzB;AAED;;GAEG;AACH,OAAO,EACL,aAAa,EACb,WAAW,EACX,mBAAmB,EACnB,SAAS,EACV,CAAA;AAGD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAA"}
|