@soulcraft/brainy 0.32.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +495 -400
- package/dist/brainyData.d.ts +115 -0
- package/dist/coreTypes.d.ts +26 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts.map +1 -1
- package/dist/storage/cacheManager.d.ts +27 -13
- package/dist/storage/cacheManager.d.ts.map +1 -1
- package/dist/storage/storageFactory.d.ts +44 -0
- package/dist/storage/storageFactory.d.ts.map +1 -1
- package/dist/unified.js +823 -56
- package/dist/unified.min.js +747 -747
- package/dist/utils/fieldNameTracking.d.ts +21 -0
- package/dist/utils/fieldNameTracking.d.ts.map +1 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/jsonProcessing.d.ts +43 -0
- package/dist/utils/jsonProcessing.d.ts.map +1 -0
- package/package.json +1 -1
package/dist/brainyData.d.ts
CHANGED
|
@@ -55,6 +55,15 @@ export interface BrainyDataConfig {
|
|
|
55
55
|
};
|
|
56
56
|
forceFileSystemStorage?: boolean;
|
|
57
57
|
forceMemoryStorage?: boolean;
|
|
58
|
+
cacheConfig?: {
|
|
59
|
+
hotCacheMaxSize?: number;
|
|
60
|
+
hotCacheEvictionThreshold?: number;
|
|
61
|
+
warmCacheTTL?: number;
|
|
62
|
+
batchSize?: number;
|
|
63
|
+
autoTune?: boolean;
|
|
64
|
+
autoTuneInterval?: number;
|
|
65
|
+
readOnly?: boolean;
|
|
66
|
+
};
|
|
58
67
|
};
|
|
59
68
|
/**
|
|
60
69
|
* Embedding function to convert data to vectors
|
|
@@ -180,6 +189,73 @@ export interface BrainyDataConfig {
|
|
|
180
189
|
*/
|
|
181
190
|
updateIndex?: boolean;
|
|
182
191
|
};
|
|
192
|
+
/**
|
|
193
|
+
* Cache configuration for optimizing search performance
|
|
194
|
+
* Controls how the system caches data for faster access
|
|
195
|
+
* Particularly important for large datasets in S3 or other remote storage
|
|
196
|
+
*/
|
|
197
|
+
cache?: {
|
|
198
|
+
/**
|
|
199
|
+
* Whether to enable auto-tuning of cache parameters
|
|
200
|
+
* When true, the system will automatically adjust cache sizes based on usage patterns
|
|
201
|
+
* Default: true
|
|
202
|
+
*/
|
|
203
|
+
autoTune?: boolean;
|
|
204
|
+
/**
|
|
205
|
+
* The interval (in milliseconds) at which to auto-tune cache parameters
|
|
206
|
+
* Only applies when autoTune is true
|
|
207
|
+
* Default: 60000 (60 seconds)
|
|
208
|
+
*/
|
|
209
|
+
autoTuneInterval?: number;
|
|
210
|
+
/**
|
|
211
|
+
* Maximum size of the hot cache (most frequently accessed items)
|
|
212
|
+
* If provided, overrides the automatically detected optimal size
|
|
213
|
+
* For large datasets, consider values between 5000-50000 depending on available memory
|
|
214
|
+
*/
|
|
215
|
+
hotCacheMaxSize?: number;
|
|
216
|
+
/**
|
|
217
|
+
* Threshold at which to start evicting items from the hot cache
|
|
218
|
+
* Expressed as a fraction of hotCacheMaxSize (0.0 to 1.0)
|
|
219
|
+
* Default: 0.8 (start evicting when cache is 80% full)
|
|
220
|
+
*/
|
|
221
|
+
hotCacheEvictionThreshold?: number;
|
|
222
|
+
/**
|
|
223
|
+
* Time-to-live for items in the warm cache in milliseconds
|
|
224
|
+
* Default: 3600000 (1 hour)
|
|
225
|
+
*/
|
|
226
|
+
warmCacheTTL?: number;
|
|
227
|
+
/**
|
|
228
|
+
* Batch size for operations like prefetching
|
|
229
|
+
* Larger values improve throughput but use more memory
|
|
230
|
+
* For S3 or remote storage with large datasets, consider values between 50-200
|
|
231
|
+
*/
|
|
232
|
+
batchSize?: number;
|
|
233
|
+
/**
|
|
234
|
+
* Read-only mode specific optimizations
|
|
235
|
+
* These settings are only applied when readOnly is true
|
|
236
|
+
*/
|
|
237
|
+
readOnlyMode?: {
|
|
238
|
+
/**
|
|
239
|
+
* Maximum size of the hot cache in read-only mode
|
|
240
|
+
* In read-only mode, larger cache sizes can be used since there are no write operations
|
|
241
|
+
* For large datasets, consider values between 10000-100000 depending on available memory
|
|
242
|
+
*/
|
|
243
|
+
hotCacheMaxSize?: number;
|
|
244
|
+
/**
|
|
245
|
+
* Batch size for operations in read-only mode
|
|
246
|
+
* Larger values improve throughput in read-only mode
|
|
247
|
+
* For S3 or remote storage with large datasets, consider values between 100-300
|
|
248
|
+
*/
|
|
249
|
+
batchSize?: number;
|
|
250
|
+
/**
|
|
251
|
+
* Prefetch strategy for read-only mode
|
|
252
|
+
* Controls how aggressively the system prefetches data
|
|
253
|
+
* Options: 'conservative', 'moderate', 'aggressive'
|
|
254
|
+
* Default: 'moderate'
|
|
255
|
+
*/
|
|
256
|
+
prefetchStrategy?: 'conservative' | 'moderate' | 'aggressive';
|
|
257
|
+
};
|
|
258
|
+
};
|
|
183
259
|
}
|
|
184
260
|
export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
185
261
|
private index;
|
|
@@ -198,6 +274,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
198
274
|
private loggingConfig;
|
|
199
275
|
private timeoutConfig;
|
|
200
276
|
private retryConfig;
|
|
277
|
+
private cacheConfig;
|
|
201
278
|
private realtimeUpdateConfig;
|
|
202
279
|
private updateTimerId;
|
|
203
280
|
private lastUpdateTime;
|
|
@@ -279,6 +356,13 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
279
356
|
* @returns The name of the current augmentation or 'default' if none is detected
|
|
280
357
|
*/
|
|
281
358
|
private getCurrentAugmentation;
|
|
359
|
+
/**
|
|
360
|
+
* Get the service name from options or fallback to current augmentation
|
|
361
|
+
* This provides a consistent way to handle service names across all methods
|
|
362
|
+
* @param options Options object that may contain a service property
|
|
363
|
+
* @returns The service name to use for operations
|
|
364
|
+
*/
|
|
365
|
+
private getServiceName;
|
|
282
366
|
/**
|
|
283
367
|
* Initialize the database
|
|
284
368
|
* Loads existing data from storage if available
|
|
@@ -401,6 +485,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
401
485
|
searchConnectedNouns?: boolean;
|
|
402
486
|
verbDirection?: 'outgoing' | 'incoming' | 'both';
|
|
403
487
|
service?: string;
|
|
488
|
+
searchField?: string;
|
|
404
489
|
}): Promise<SearchResult<T>[]>;
|
|
405
490
|
/**
|
|
406
491
|
* Search the local database for similar vectors
|
|
@@ -414,6 +499,8 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
414
499
|
nounTypes?: string[];
|
|
415
500
|
includeVerbs?: boolean;
|
|
416
501
|
service?: string;
|
|
502
|
+
searchField?: string;
|
|
503
|
+
priorityFields?: string[];
|
|
417
504
|
}): Promise<SearchResult<T>[]>;
|
|
418
505
|
/**
|
|
419
506
|
* Find entities similar to a given entity ID
|
|
@@ -717,6 +804,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
717
804
|
includeVerbs?: boolean;
|
|
718
805
|
storeResults?: boolean;
|
|
719
806
|
service?: string;
|
|
807
|
+
searchField?: string;
|
|
720
808
|
}): Promise<SearchResult<T>[]>;
|
|
721
809
|
/**
|
|
722
810
|
* Search both local and remote Brainy instances, combining the results
|
|
@@ -731,6 +819,7 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
731
819
|
includeVerbs?: boolean;
|
|
732
820
|
localFirst?: boolean;
|
|
733
821
|
service?: string;
|
|
822
|
+
searchField?: string;
|
|
734
823
|
}): Promise<SearchResult<T>[]>;
|
|
735
824
|
/**
|
|
736
825
|
* Check if the instance is connected to a remote server
|
|
@@ -855,5 +944,31 @@ export declare class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|
|
855
944
|
nounIds: string[];
|
|
856
945
|
verbIds: string[];
|
|
857
946
|
}>;
|
|
947
|
+
/**
|
|
948
|
+
* Get available field names by service
|
|
949
|
+
* This helps users understand what fields are available for searching from different data sources
|
|
950
|
+
* @returns Record of field names by service
|
|
951
|
+
*/
|
|
952
|
+
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
|
953
|
+
/**
|
|
954
|
+
* Get standard field mappings
|
|
955
|
+
* This helps users understand how fields from different services map to standard field names
|
|
956
|
+
* @returns Record of standard field mappings
|
|
957
|
+
*/
|
|
958
|
+
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
959
|
+
/**
|
|
960
|
+
* Search using a standard field name
|
|
961
|
+
* This allows searching across multiple services using a standardized field name
|
|
962
|
+
* @param standardField The standard field name to search in
|
|
963
|
+
* @param searchTerm The term to search for
|
|
964
|
+
* @param k Number of results to return
|
|
965
|
+
* @param options Additional search options
|
|
966
|
+
* @returns Array of search results
|
|
967
|
+
*/
|
|
968
|
+
searchByStandardField(standardField: string, searchTerm: string, k?: number, options?: {
|
|
969
|
+
services?: string[];
|
|
970
|
+
includeVerbs?: boolean;
|
|
971
|
+
searchMode?: 'local' | 'remote' | 'combined';
|
|
972
|
+
}): Promise<SearchResult<T>[]>;
|
|
858
973
|
}
|
|
859
974
|
export { euclideanDistance, cosineDistance, manhattanDistance, dotProductDistance } from './utils/index.js';
|
package/dist/coreTypes.d.ts
CHANGED
|
@@ -140,6 +140,16 @@ export interface StatisticsData {
|
|
|
140
140
|
relate: number;
|
|
141
141
|
total: number;
|
|
142
142
|
};
|
|
143
|
+
/**
|
|
144
|
+
* Field names available for searching, organized by service
|
|
145
|
+
* This helps users understand what fields are available from different data sources
|
|
146
|
+
*/
|
|
147
|
+
fieldNames?: Record<string, string[]>;
|
|
148
|
+
/**
|
|
149
|
+
* Standard field mappings for common field names across services
|
|
150
|
+
* Maps standard field names to the actual field names used by each service
|
|
151
|
+
*/
|
|
152
|
+
standardFieldMappings?: Record<string, Record<string, string[]>>;
|
|
143
153
|
/**
|
|
144
154
|
* Last updated timestamp
|
|
145
155
|
*/
|
|
@@ -298,6 +308,22 @@ export interface StorageAdapter {
|
|
|
298
308
|
* This ensures that any pending statistics updates are written to persistent storage
|
|
299
309
|
*/
|
|
300
310
|
flushStatisticsToStorage(): Promise<void>;
|
|
311
|
+
/**
|
|
312
|
+
* Track field names from a JSON document
|
|
313
|
+
* @param jsonDocument The JSON document to extract field names from
|
|
314
|
+
* @param service The service that inserted the data
|
|
315
|
+
*/
|
|
316
|
+
trackFieldNames(jsonDocument: any, service: string): Promise<void>;
|
|
317
|
+
/**
|
|
318
|
+
* Get available field names by service
|
|
319
|
+
* @returns Record of field names by service
|
|
320
|
+
*/
|
|
321
|
+
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
|
322
|
+
/**
|
|
323
|
+
* Get standard field mappings
|
|
324
|
+
* @returns Record of standard field mappings
|
|
325
|
+
*/
|
|
326
|
+
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
301
327
|
/**
|
|
302
328
|
* Get changes since a specific timestamp
|
|
303
329
|
* @param timestamp The timestamp to get changes since
|
|
@@ -125,6 +125,22 @@ export declare abstract class BaseStorageAdapter implements StorageAdapter {
|
|
|
125
125
|
* This ensures that any pending statistics updates are written to persistent storage
|
|
126
126
|
*/
|
|
127
127
|
flushStatisticsToStorage(): Promise<void>;
|
|
128
|
+
/**
|
|
129
|
+
* Track field names from a JSON document
|
|
130
|
+
* @param jsonDocument The JSON document to extract field names from
|
|
131
|
+
* @param service The service that inserted the data
|
|
132
|
+
*/
|
|
133
|
+
trackFieldNames(jsonDocument: any, service: string): Promise<void>;
|
|
134
|
+
/**
|
|
135
|
+
* Get available field names by service
|
|
136
|
+
* @returns Record of field names by service
|
|
137
|
+
*/
|
|
138
|
+
getAvailableFieldNames(): Promise<Record<string, string[]>>;
|
|
139
|
+
/**
|
|
140
|
+
* Get standard field mappings
|
|
141
|
+
* @returns Record of standard field mappings
|
|
142
|
+
*/
|
|
143
|
+
getStandardFieldMappings(): Promise<Record<string, Record<string, string[]>>>;
|
|
128
144
|
/**
|
|
129
145
|
* Create default statistics data
|
|
130
146
|
* @returns Default statistics data
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseStorageAdapter.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/baseStorageAdapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;
|
|
1
|
+
{"version":3,"file":"baseStorageAdapter.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/baseStorageAdapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAGnE;;GAEG;AACH,8BAAsB,kBAAmB,YAAW,cAAc;IAEhE,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAE9B,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3C,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAEjD,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE7D,QAAQ,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAE9C,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE3C,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAEjD,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE3D,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAE3D,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAErD,QAAQ,CAAC,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAE9C,QAAQ,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/D,QAAQ,CAAC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAErD,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAE/B,QAAQ,CAAC,gBAAgB,IAAI,OAAO,CAAC;QACnC,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC;IAEF;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,EAAE;YACX,MAAM,CAAC,EAAE,MAAM,CAAA;YACf,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;QACD,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACF,GAAG,OAAO,CAAC;QACV,KAAK,EAAE,GAAG,EAAE,CAAA;QACZ,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAEF;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;QAC1B,UAAU,CAAC,EAAE;YACX,MAAM,CAAC,EAAE,MAAM,CAAA;YACf,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;QACD,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACF,GAAG,OAAO,CAAC;QACV,KAAK,EAAE,GAAG,EAAE,CAAA;QACZ,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAGF,SAAS,CAAC,eAAe,EAAE,cAAc,GAAG,IAAI,CAAO;IAGvD,SAAS,CAAC,4BAA4B,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAO;IAGpE,SAAS,CAAC,kBAAkB,UAAQ;IAGpC,SAAS,CAAC,uBAAuB,SAAI;IAGrC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,QAAO;IAG/C,SAAS,CAAC,QAAQ,CAAC,kBAAkB,SAAQ;IAG7C,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CACnC,UAAU,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,iBAAiB,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAEtE;;;OAGG;IACG,cAAc,CAAC,UAAU,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAc/D;;;OAGG;IACG,aAAa,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IA8BrD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,IAAI;IAyBrC;;OAEG;cACa,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BhD;;;;;OAKG;IACG,kBAAkB,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAClC,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IAoChB;;;;;OAKG;IACG,kBAAkB,CACtB,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAClC,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAU,GACjB,OAAO,CAAC,IAAI,CAAC;IAoChB;;;OAGG;IACG,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BtD;;;OAGG;IACG,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;IAU/C;;;;OAIG;IACG,eAAe,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA6ExE;;;OAGG;IACG,sBAAsB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAcjE;;;OAGG;IACG,wBAAwB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAcnF;;;OAGG;IACH,SAAS,CAAC,uBAAuB,IAAI,cAAc;CAWpD"}
|
|
@@ -46,6 +46,7 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
46
46
|
private storageStatistics;
|
|
47
47
|
private warmStorage;
|
|
48
48
|
private coldStorage;
|
|
49
|
+
private options;
|
|
49
50
|
/**
|
|
50
51
|
* Initialize the cache manager
|
|
51
52
|
* @param options Configuration options
|
|
@@ -58,13 +59,20 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
58
59
|
autoTune?: boolean;
|
|
59
60
|
warmStorage?: any;
|
|
60
61
|
coldStorage?: any;
|
|
62
|
+
readOnly?: boolean;
|
|
61
63
|
});
|
|
62
64
|
/**
|
|
63
65
|
* Detect the current environment
|
|
64
66
|
*/
|
|
65
67
|
private detectEnvironment;
|
|
66
68
|
/**
|
|
67
|
-
* Detect the optimal cache size based on available memory
|
|
69
|
+
* Detect the optimal cache size based on available memory and operating mode
|
|
70
|
+
*
|
|
71
|
+
* Enhanced to better handle large datasets in S3 or other storage:
|
|
72
|
+
* - Increases cache size for read-only mode
|
|
73
|
+
* - Adjusts based on total dataset size when available
|
|
74
|
+
* - Provides more aggressive caching for large datasets
|
|
75
|
+
* - Optimizes memory usage based on environment
|
|
68
76
|
*/
|
|
69
77
|
private detectOptimalCacheSize;
|
|
70
78
|
/**
|
|
@@ -84,18 +92,21 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
84
92
|
*/
|
|
85
93
|
private tuneParameters;
|
|
86
94
|
/**
|
|
87
|
-
* Tune hot cache size based on statistics and
|
|
95
|
+
* Tune hot cache size based on statistics, environment, and operating mode
|
|
88
96
|
*
|
|
89
97
|
* The hot cache size is tuned based on:
|
|
90
98
|
* 1. Available memory in the current environment
|
|
91
99
|
* 2. Total number of nodes and edges in the system
|
|
92
100
|
* 3. Cache hit/miss ratio
|
|
101
|
+
* 4. Operating mode (read-only vs. read-write)
|
|
102
|
+
* 5. Storage type (S3, filesystem, memory)
|
|
93
103
|
*
|
|
94
|
-
*
|
|
95
|
-
* - Start with a size based on available memory
|
|
96
|
-
* -
|
|
97
|
-
* -
|
|
98
|
-
* -
|
|
104
|
+
* Enhanced algorithm:
|
|
105
|
+
* - Start with a size based on available memory and operating mode
|
|
106
|
+
* - For large datasets in S3 or other remote storage, use more aggressive caching
|
|
107
|
+
* - Adjust based on access patterns (read-heavy vs. write-heavy)
|
|
108
|
+
* - For read-only mode, prioritize cache size over eviction speed
|
|
109
|
+
* - Dynamically adjust based on hit/miss ratio and query patterns
|
|
99
110
|
*/
|
|
100
111
|
private tuneHotCacheSize;
|
|
101
112
|
/**
|
|
@@ -128,7 +139,7 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
128
139
|
*/
|
|
129
140
|
private tuneWarmCacheTTL;
|
|
130
141
|
/**
|
|
131
|
-
* Tune batch size based on statistics and
|
|
142
|
+
* Tune batch size based on environment, statistics, and operating mode
|
|
132
143
|
*
|
|
133
144
|
* The batch size determines how many items are processed in a single batch
|
|
134
145
|
* for operations like prefetching. It is tuned based on:
|
|
@@ -136,13 +147,16 @@ export declare class CacheManager<T extends HNSWNode | Edge> {
|
|
|
136
147
|
* 2. Available memory
|
|
137
148
|
* 3. Operation patterns
|
|
138
149
|
* 4. Cache hit/miss ratio
|
|
150
|
+
* 5. Operating mode (read-only vs. read-write)
|
|
151
|
+
* 6. Storage type (S3, filesystem, memory)
|
|
152
|
+
* 7. Dataset size
|
|
139
153
|
*
|
|
140
|
-
*
|
|
154
|
+
* Enhanced algorithm:
|
|
141
155
|
* - Start with a default based on the environment
|
|
142
|
-
* -
|
|
143
|
-
* - For
|
|
144
|
-
* -
|
|
145
|
-
* -
|
|
156
|
+
* - For large datasets in S3 or other remote storage, use larger batches
|
|
157
|
+
* - For read-only mode, use larger batches to improve throughput
|
|
158
|
+
* - Dynamically adjust based on network latency and throughput
|
|
159
|
+
* - Balance between memory usage and performance
|
|
146
160
|
*/
|
|
147
161
|
private tuneBatchSize;
|
|
148
162
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cacheManager.d.ts","sourceRoot":"","sources":["../../src/storage/cacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAKrD,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,SAAS;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB;IAED,UAAU,iBAAiB;QACzB,OAAO,CAAC,EAAE;YACR,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;SACpB,CAAC;KACH;CACF;AAGD,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,SAAS,CAAA;AAWrB,UAAU,UAAU;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;CAChB;
|
|
1
|
+
{"version":3,"file":"cacheManager.d.ts","sourceRoot":"","sources":["../../src/storage/cacheManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAKrD,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,SAAS;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB;IAED,UAAU,iBAAiB;QACzB,OAAO,CAAC,EAAE;YACR,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;SACpB,CAAC;KACH;CACF;AAGD,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,SAAS,CAAA;AAWrB,UAAU,UAAU;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;CAChB;AAkBD;;GAEG;AACH,qBAAa,YAAY,CAAC,CAAC,SAAS,QAAQ,GAAG,IAAI;IAEjD,OAAO,CAAC,QAAQ,CAAmC;IAGnD,OAAO,CAAC,KAAK,CAMZ;IAGD,OAAO,CAAC,WAAW,CAAa;IAChC,OAAO,CAAC,eAAe,CAAa;IACpC,OAAO,CAAC,eAAe,CAAa;IAGpC,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,yBAAyB,CAAQ;IACzC,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,SAAS,CAAQ;IAGzB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,gBAAgB,CAAY;IACpC,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,iBAAiB,CAAY;IAGrC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,WAAW,CAAK;IAGxB,OAAO,CAAC,OAAO,CASd;IAED;;;OAGG;gBACS,OAAO,GAAE;QACnB,eAAe,CAAC,EAAE,MAAM,CAAA;QACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;QAClC,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;QAClB,WAAW,CAAC,EAAE,GAAG,CAAA;QACjB,WAAW,CAAC,EAAE,GAAG,CAAA;QACjB,QAAQ,CAAC,EAAE,OAAO,CAAA;KACd;IA4CN;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;;;;;;OAQG;IACH,OAAO,CAAC,sBAAsB;IAoH9B;;;;;;;;;;;;;;OAcG;YACW,cAAc;IA2C5B;;;;;;;;;;;;;;;;OAgBG;IACH,OAAO,CAAC,gBAAgB;IA0HxB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,qBAAqB;IAiD7B;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,gBAAgB;IA6BxB;;;;;;;;;;;;;;;;;;;OAmBG;IACH,OAAO,CAAC,aAAa;IAqJrB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAmB7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAmB7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;;;OAIG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAuD/C;;;;OAIG;YACW,gBAAgB;IAW9B;;;;OAIG;YACW,kBAAkB;IAWhC;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAkBrB;;;;OAIG;YACW,cAAc;IAa5B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;;;OAIG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBpD;;;OAGG;IACU,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B9C;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCnC;;;OAGG;IACI,QAAQ,IAAI,UAAU;IAI7B;;;OAGG;IACU,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCnD;;;;;;;;;;;;OAYG;YACW,sBAAsB;IAWpC;;;;OAIG;IACU,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAqF5D;;;;OAIG;IACI,kBAAkB,CAAC,WAAW,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,GAAG,IAAI;CAIpE"}
|
|
@@ -143,6 +143,50 @@ export interface StorageOptions {
|
|
|
143
143
|
* Operation configuration for timeout and retry behavior
|
|
144
144
|
*/
|
|
145
145
|
operationConfig?: OperationConfig;
|
|
146
|
+
/**
|
|
147
|
+
* Cache configuration for optimizing data access
|
|
148
|
+
* Particularly important for S3 and other remote storage
|
|
149
|
+
*/
|
|
150
|
+
cacheConfig?: {
|
|
151
|
+
/**
|
|
152
|
+
* Maximum size of the hot cache (most frequently accessed items)
|
|
153
|
+
* For large datasets, consider values between 5000-50000 depending on available memory
|
|
154
|
+
*/
|
|
155
|
+
hotCacheMaxSize?: number;
|
|
156
|
+
/**
|
|
157
|
+
* Threshold at which to start evicting items from the hot cache
|
|
158
|
+
* Expressed as a fraction of hotCacheMaxSize (0.0 to 1.0)
|
|
159
|
+
* Default: 0.8 (start evicting when cache is 80% full)
|
|
160
|
+
*/
|
|
161
|
+
hotCacheEvictionThreshold?: number;
|
|
162
|
+
/**
|
|
163
|
+
* Time-to-live for items in the warm cache in milliseconds
|
|
164
|
+
* Default: 3600000 (1 hour)
|
|
165
|
+
*/
|
|
166
|
+
warmCacheTTL?: number;
|
|
167
|
+
/**
|
|
168
|
+
* Batch size for operations like prefetching
|
|
169
|
+
* Larger values improve throughput but use more memory
|
|
170
|
+
*/
|
|
171
|
+
batchSize?: number;
|
|
172
|
+
/**
|
|
173
|
+
* Whether to enable auto-tuning of cache parameters
|
|
174
|
+
* When true, the system will automatically adjust cache sizes based on usage patterns
|
|
175
|
+
* Default: true
|
|
176
|
+
*/
|
|
177
|
+
autoTune?: boolean;
|
|
178
|
+
/**
|
|
179
|
+
* The interval (in milliseconds) at which to auto-tune cache parameters
|
|
180
|
+
* Only applies when autoTune is true
|
|
181
|
+
* Default: 60000 (1 minute)
|
|
182
|
+
*/
|
|
183
|
+
autoTuneInterval?: number;
|
|
184
|
+
/**
|
|
185
|
+
* Whether the storage is in read-only mode
|
|
186
|
+
* This affects cache sizing and prefetching strategies
|
|
187
|
+
*/
|
|
188
|
+
readOnly?: boolean;
|
|
189
|
+
};
|
|
146
190
|
}
|
|
147
191
|
/**
|
|
148
192
|
* Create a storage adapter based on the environment and configuration
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storageFactory.d.ts","sourceRoot":"","sources":["../../src/storage/storageFactory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAC,aAAa,EAAC,MAAM,6BAA6B,CAAA;AACzD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAA;AACrD,OAAO,EAAC,mBAAmB,EAAE,SAAS,EAAC,MAAM,mCAAmC,CAAA;AAChF,OAAO,EAAC,iBAAiB,EAAC,MAAM,iCAAiC,CAAA;AAEjE,OAAO,EAAC,eAAe,EAAC,MAAM,4BAA4B,CAAA;AAE1D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,CAAA;IAEtE;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAE5B;;OAEG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAEhC;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAA;IAElC;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE;QACR;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAA;KACxB,CAAA;IAED;;OAEG;IACH,SAAS,CAAC,EAAE;QACR;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAA;QAEjB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;KAC1B,CAAA;IAED;;OAEG;IACH,UAAU,CAAC,EAAE;QACT;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;IAED;;OAEG;IACH,eAAe,CAAC,EAAE;QACd;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAA;QAEhB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAA;KACvB,CAAA;IAED;;OAEG;IACH,eAAe,CAAC,EAAE,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"storageFactory.d.ts","sourceRoot":"","sources":["../../src/storage/storageFactory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAC,cAAc,EAAC,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAAC,aAAa,EAAC,MAAM,6BAA6B,CAAA;AACzD,OAAO,EAAC,WAAW,EAAC,MAAM,2BAA2B,CAAA;AACrD,OAAO,EAAC,mBAAmB,EAAE,SAAS,EAAC,MAAM,mCAAmC,CAAA;AAChF,OAAO,EAAC,iBAAiB,EAAC,MAAM,iCAAiC,CAAA;AAEjE,OAAO,EAAC,eAAe,EAAC,MAAM,4BAA4B,CAAA;AAE1D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;;;;;;;;OASG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,CAAA;IAEtE;;OAEG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAE5B;;OAEG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAEhC;;OAEG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAA;IAElC;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE;QACR;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,YAAY,CAAC,EAAE,MAAM,CAAA;KACxB,CAAA;IAED;;OAEG;IACH,SAAS,CAAC,EAAE;QACR;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAA;QAEjB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;KAC1B,CAAA;IAED;;OAEG;IACH,UAAU,CAAC,EAAE;QACT;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,QAAQ,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;IAED;;OAEG;IACH,eAAe,CAAC,EAAE;QACd;;WAEG;QACH,UAAU,EAAE,MAAM,CAAA;QAElB;;WAEG;QACH,MAAM,CAAC,EAAE,MAAM,CAAA;QAEf;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAA;QAEhB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAA;QAEnB;;WAEG;QACH,eAAe,EAAE,MAAM,CAAA;QAEvB;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,CAAA;KACvB,CAAA;IAED;;OAEG;IACH,eAAe,CAAC,EAAE,eAAe,CAAA;IAEjC;;;OAGG;IACH,WAAW,CAAC,EAAE;QACV;;;WAGG;QACH,eAAe,CAAC,EAAE,MAAM,CAAA;QAExB;;;;WAIG;QACH,yBAAyB,CAAC,EAAE,MAAM,CAAA;QAElC;;;WAGG;QACH,YAAY,CAAC,EAAE,MAAM,CAAA;QAErB;;;WAGG;QACH,SAAS,CAAC,EAAE,MAAM,CAAA;QAElB;;;;WAIG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAA;QAElB;;;;WAIG;QACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;QAEzB;;;WAGG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAA;KACrB,CAAA;CACJ;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAC/B,OAAO,GAAE,cAAmB,GAC7B,OAAO,CAAC,cAAc,CAAC,CAqMzB;AAED;;GAEG;AACH,OAAO,EACH,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,mBAAmB,EACnB,SAAS,EACZ,CAAA"}
|