@soulcraft/brainy 0.35.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +190 -20
- package/dist/coreTypes.d.ts +27 -2
- package/dist/hnsw/distributedSearch.d.ts +118 -0
- package/dist/hnsw/distributedSearch.d.ts.map +1 -0
- package/dist/hnsw/optimizedHNSWIndex.d.ts +97 -0
- package/dist/hnsw/optimizedHNSWIndex.d.ts.map +1 -0
- package/dist/hnsw/partitionedHNSWIndex.d.ts +101 -0
- package/dist/hnsw/partitionedHNSWIndex.d.ts.map +1 -0
- package/dist/hnsw/scaledHNSWSystem.d.ts +142 -0
- package/dist/hnsw/scaledHNSWSystem.d.ts.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/storage/adapters/baseStorageAdapter.d.ts +2 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts.map +1 -1
- package/dist/storage/adapters/batchS3Operations.d.ts +71 -0
- package/dist/storage/adapters/batchS3Operations.d.ts.map +1 -0
- package/dist/storage/adapters/fileSystemStorage.d.ts +23 -5
- package/dist/storage/adapters/fileSystemStorage.d.ts.map +1 -1
- package/dist/storage/adapters/memoryStorage.d.ts +22 -4
- package/dist/storage/adapters/memoryStorage.d.ts.map +1 -1
- package/dist/storage/adapters/opfsStorage.d.ts +27 -9
- package/dist/storage/adapters/opfsStorage.d.ts.map +1 -1
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +24 -8
- package/dist/storage/adapters/s3CompatibleStorage.d.ts.map +1 -1
- package/dist/storage/baseStorage.d.ts +30 -4
- package/dist/storage/baseStorage.d.ts.map +1 -1
- package/dist/storage/enhancedCacheManager.d.ts +141 -0
- package/dist/storage/enhancedCacheManager.d.ts.map +1 -0
- package/dist/storage/readOnlyOptimizations.d.ts +133 -0
- package/dist/storage/readOnlyOptimizations.d.ts.map +1 -0
- package/dist/unified.js +635 -215
- package/dist/unified.min.js +991 -991
- package/dist/utils/autoConfiguration.d.ts +125 -0
- package/dist/utils/autoConfiguration.d.ts.map +1 -0
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3CompatibleStorage.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/s3CompatibleStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;
|
|
1
|
+
{"version":3,"file":"s3CompatibleStorage.d.ts","sourceRoot":"","sources":["../../../src/storage/adapters/s3CompatibleStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAClF,OAAO,EACL,WAAW,EAMZ,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAEL,eAAe,EAChB,MAAM,+BAA+B,CAAA;AAKtC,KAAK,QAAQ,GAAG,QAAQ,CAAA;AACxB,KAAK,IAAI,GAAG,QAAQ,CAAA;AAGpB,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACtC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,CAAA;IACxC,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,CAAC,EAAE,GAAG,CAAA;IACV,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAGD,OAAO,EAAE,mBAAmB,IAAI,SAAS,EAAE,CAAA;AAM3C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,mBAAoB,SAAQ,WAAW;IAClD,OAAO,CAAC,QAAQ,CAAwB;IACxC,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAQ;IACzB,OAAO,CAAC,SAAS,CAAC,CAAQ;IAC1B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,YAAY,CAAC,CAAQ;IAG7B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,cAAc,CAAQ;IAC9B,OAAO,CAAC,WAAW,CAAQ;IAG3B,SAAS,CAAC,eAAe,EAAE,cAAc,GAAG,IAAI,CAAO;IAGvD,OAAO,CAAC,UAAU,CAAmB;IACrC,OAAO,CAAC,WAAW,CAAyB;IAG5C,OAAO,CAAC,eAAe,CAAwB;IAG/C,OAAO,CAAC,kBAAkB,CAA2B;IAGrD,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,gBAAgB,CAAoB;IAE5C;;;OAGG;gBACS,OAAO,EAAE;QACnB,UAAU,EAAE,MAAM,CAAA;QAClB,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,QAAQ,CAAC,EAAE,MAAM,CAAA;QACjB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,eAAe,EAAE,MAAM,CAAA;QACvB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,eAAe,CAAC,EAAE,eAAe,CAAA;QACjC,WAAW,CAAC,EAAE;YACZ,eAAe,CAAC,EAAE,MAAM,CAAA;YACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;YAClC,YAAY,CAAC,EAAE,MAAM,CAAA;SACtB,CAAA;KACF;IA2BD;;OAEG;IACU,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA4IlC;;OAEG;cACa,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhE;;OAEG;cACa,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA8EvD;;OAEG;cACa,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAItE;;OAEG;cACa,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAwE7D;;OAEG;cACa,oBAAoB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAK3D,OAAO,CAAC,SAAS,CAA8B;IAE/C;;;;OAIG;cACa,WAAW,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAwBlD;;;;OAIG;cACa,sBAAsB,CAAC,OAAO,GAAE;QAC9C,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,QAAQ,CAAC,EAAE,OAAO,CAAA;KACd,GAAG,OAAO,CAAC;QACf,KAAK,EAAE,QAAQ,EAAE,CAAA;QACjB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAwGF;;;;OAIG;cACa,2BAA2B,CACzC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,QAAQ,EAAE,CAAC;IAItB;;;;OAIG;cACa,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA2CzE;;OAEG;cACa,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D;;OAEG;cACa,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BrD;;OAEG;cACa,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhE;;OAEG;cACa,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCnD;;OAEG;cACa,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAItE;;OAEG;cACa,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAwEzD;;;;OAIG;cACa,oBAAoB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAK3D;;;;OAIG;cACa,WAAW,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;IAwB9C;;;;OAIG;cACa,sBAAsB,CAAC,OAAO,GAAE;QAC9C,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,QAAQ,CAAC,EAAE,OAAO,CAAA;QAClB,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,QAAQ,CAAC,EAAE,MAAM,CAAA;YACjB,IAAI,CAAC,EAAE,MAAM,CAAA;SACd,CAAA;KACG,GAAG,OAAO,CAAC;QACf,KAAK,EAAE,IAAI,EAAE,CAAA;QACb,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAiHF;;;;;OAKG;IACH,OAAO,CAAC,UAAU;IAWlB;;;;OAIG;IACU,sBAAsB,CAAC,OAAO,GAAE;QAC3C,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACG,GAAG,OAAO,CAAC;QACf,KAAK,EAAE,SAAS,EAAE,CAAA;QAClB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAyDF;;OAEG;cACa,yBAAyB,CACvC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,EAAE,CAAC;IAIvB;;OAEG;cACa,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAOxE;;OAEG;cACa,yBAAyB,CACvC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,EAAE,CAAC;IAIvB;;OAEG;cACa,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAOxE;;OAEG;cACa,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAI3E;;OAEG;cACa,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAOlE;;OAEG;cACa,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI9D;;OAEG;cACa,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BrD;;OAEG;IACU,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAoEnE;;OAEG;IACU,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCvE;;OAEG;IACU,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IA2D7D;;OAEG;IACU,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCvE;;OAEG;IACU,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IA2D7D;;OAEG;IACU,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IA+DzD;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA0DnC;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC;QACvC,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC;IA+KF,SAAS,CAAC,4BAA4B,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAO;IAEpE,SAAS,CAAC,kBAAkB,UAAQ;IAEpC,SAAS,CAAC,uBAAuB,SAAI;IAErC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,QAAO;IAE/C,SAAS,CAAC,QAAQ,CAAC,kBAAkB,SAAQ;IAE7C;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAO/B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAI/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAI9B;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,IAAI;IAyBrC;;OAEG;cACa,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IA4GhD;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAgDvB;;;OAGG;cACa,kBAAkB,CAChC,UAAU,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC;IAqBhB;;;OAGG;cACa,iBAAiB,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAuDnE;;;;OAIG;YACW,uBAAuB;IA0CrC;;;OAGG;YACW,iBAAiB;IAmC/B;;;;;OAKG;IACU,eAAe,CAC1B,cAAc,EAAE,MAAM,EACtB,UAAU,GAAE,MAAa,GACxB,OAAO,CAAC,cAAc,EAAE,CAAC;IA8D5B;;;OAGG;IACU,oBAAoB,CAAC,kBAAkB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiE5E;;;;;OAKG;YACW,WAAW;IA0EzB;;;;;OAKG;YACW,WAAW;IA0DzB;;;OAGG;YACW,mBAAmB;CAmElC"}
|
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
* Base Storage Adapter
|
|
3
3
|
* Provides common functionality for all storage adapters
|
|
4
4
|
*/
|
|
5
|
-
import { GraphVerb, HNSWNoun, StatisticsData } from '../coreTypes.js';
|
|
5
|
+
import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js';
|
|
6
6
|
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js';
|
|
7
7
|
export declare const NOUNS_DIR = "nouns";
|
|
8
8
|
export declare const VERBS_DIR = "verbs";
|
|
9
9
|
export declare const METADATA_DIR = "metadata";
|
|
10
|
+
export declare const NOUN_METADATA_DIR = "noun-metadata";
|
|
11
|
+
export declare const VERB_METADATA_DIR = "verb-metadata";
|
|
10
12
|
export declare const INDEX_DIR = "index";
|
|
11
13
|
export declare const STATISTICS_KEY = "statistics";
|
|
12
14
|
/**
|
|
@@ -56,6 +58,10 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
56
58
|
* Get a verb from storage
|
|
57
59
|
*/
|
|
58
60
|
getVerb(id: string): Promise<GraphVerb | null>;
|
|
61
|
+
/**
|
|
62
|
+
* Convert HNSWVerb to GraphVerb by combining with metadata
|
|
63
|
+
*/
|
|
64
|
+
protected convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise<GraphVerb | null>;
|
|
59
65
|
/**
|
|
60
66
|
* Get all verbs from storage
|
|
61
67
|
* @deprecated This method is deprecated and will be removed in a future version.
|
|
@@ -149,6 +155,26 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
149
155
|
* This method should be implemented by each specific adapter
|
|
150
156
|
*/
|
|
151
157
|
abstract getMetadata(id: string): Promise<any | null>;
|
|
158
|
+
/**
|
|
159
|
+
* Save noun metadata to storage
|
|
160
|
+
* This method should be implemented by each specific adapter
|
|
161
|
+
*/
|
|
162
|
+
abstract saveNounMetadata(id: string, metadata: any): Promise<void>;
|
|
163
|
+
/**
|
|
164
|
+
* Get noun metadata from storage
|
|
165
|
+
* This method should be implemented by each specific adapter
|
|
166
|
+
*/
|
|
167
|
+
abstract getNounMetadata(id: string): Promise<any | null>;
|
|
168
|
+
/**
|
|
169
|
+
* Save verb metadata to storage
|
|
170
|
+
* This method should be implemented by each specific adapter
|
|
171
|
+
*/
|
|
172
|
+
abstract saveVerbMetadata(id: string, metadata: any): Promise<void>;
|
|
173
|
+
/**
|
|
174
|
+
* Get verb metadata from storage
|
|
175
|
+
* This method should be implemented by each specific adapter
|
|
176
|
+
*/
|
|
177
|
+
abstract getVerbMetadata(id: string): Promise<any | null>;
|
|
152
178
|
/**
|
|
153
179
|
* Save a noun to storage
|
|
154
180
|
* This method should be implemented by each specific adapter
|
|
@@ -178,17 +204,17 @@ export declare abstract class BaseStorage extends BaseStorageAdapter {
|
|
|
178
204
|
* Save a verb to storage
|
|
179
205
|
* This method should be implemented by each specific adapter
|
|
180
206
|
*/
|
|
181
|
-
protected abstract saveVerb_internal(verb:
|
|
207
|
+
protected abstract saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
182
208
|
/**
|
|
183
209
|
* Get a verb from storage
|
|
184
210
|
* This method should be implemented by each specific adapter
|
|
185
211
|
*/
|
|
186
|
-
protected abstract getVerb_internal(id: string): Promise<
|
|
212
|
+
protected abstract getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
187
213
|
/**
|
|
188
214
|
* Get all verbs from storage
|
|
189
215
|
* This method should be implemented by each specific adapter
|
|
190
216
|
*/
|
|
191
|
-
protected abstract getAllVerbs_internal(): Promise<
|
|
217
|
+
protected abstract getAllVerbs_internal(): Promise<HNSWVerb[]>;
|
|
192
218
|
/**
|
|
193
219
|
* Get verbs by source
|
|
194
220
|
* This method should be implemented by each specific adapter
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseStorage.d.ts","sourceRoot":"","sources":["../../src/storage/baseStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"baseStorage.d.ts","sourceRoot":"","sources":["../../src/storage/baseStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAGrE,eAAO,MAAM,SAAS,UAAU,CAAA;AAChC,eAAO,MAAM,SAAS,UAAU,CAAA;AAChC,eAAO,MAAM,YAAY,aAAa,CAAA;AACtC,eAAO,MAAM,iBAAiB,kBAAkB,CAAA;AAChD,eAAO,MAAM,iBAAiB,kBAAkB,CAAA;AAChD,eAAO,MAAM,SAAS,UAAU,CAAA;AAChC,eAAO,MAAM,cAAc,eAAe,CAAA;AAE1C;;;GAGG;AACH,8BAAsB,WAAY,SAAQ,kBAAkB;IAC1D,SAAS,CAAC,aAAa,UAAQ;IAE/B;;;OAGG;aACa,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAErC;;OAEG;cACa,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IAMlD;;OAEG;IACU,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpD;;OAEG;IACU,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAK1D;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAM/C;;;;OAIG;IACU,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAKtE;;OAEG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlD;;OAEG;IACU,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCrD;;OAEG;IACU,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAS3D;;OAEG;cACa,0BAA0B,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IA0CzF;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAiBhD;;OAEG;IACU,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAUrE;;OAEG;IACU,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAUrE;;OAEG;IACU,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAU/D;;;;OAIG;IACU,QAAQ,CAAC,OAAO,CAAC,EAAE;QAC9B,UAAU,CAAC,EAAE;YACX,MAAM,CAAC,EAAE,MAAM,CAAA;YACf,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;QACD,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACF,GAAG,OAAO,CAAC;QACV,KAAK,EAAE,QAAQ,EAAE,CAAA;QACjB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAwLF;;;;OAIG;IACU,QAAQ,CAAC,OAAO,CAAC,EAAE;QAC9B,UAAU,CAAC,EAAE;YACX,MAAM,CAAC,EAAE,MAAM,CAAA;YACf,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,MAAM,CAAC,EAAE,MAAM,CAAA;SAChB,CAAA;QACD,MAAM,CAAC,EAAE;YACP,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;YAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAC/B,CAAA;KACF,GAAG,OAAO,CAAC;QACV,KAAK,EAAE,SAAS,EAAE,CAAA;QAClB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,OAAO,CAAA;QAChB,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAC;IAoRF;;OAEG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlD;;;OAGG;aACa,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAEtC;;;OAGG;aACa,gBAAgB,IAAI,OAAO,CAAC;QAC1C,IAAI,EAAE,MAAM,CAAA;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;KAC9B,CAAC;IAEF;;;OAGG;aACa,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAEtE;;;OAGG;aACa,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAE5D;;;OAGG;aACa,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1E;;;OAGG;aACa,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAEhE;;;OAGG;aACa,gBAAgB,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAE1E;;;OAGG;aACa,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAEhE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAEzE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAE9D;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,2BAA2B,CAC5C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,QAAQ,EAAE,CAAC;IAEtB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEjE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAEzE;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,oBAAoB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;IAE9D;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,yBAAyB,CAC1C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,EAAE,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,yBAAyB,CAC1C,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,SAAS,EAAE,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAE9E;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAEjE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,CAAC,EAChD,GAAG,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EACd,gBAAgB,GAAE,CAAC,KAAK,EAAE,CAAC,KAAK,GAAc,GAC7C,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAQtB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CACnC,UAAU,EAAE,cAAc,GACzB,OAAO,CAAC,IAAI,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,iBAAiB,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;CACvE"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enhanced Multi-Level Cache Manager with Predictive Prefetching
|
|
3
|
+
* Optimized for HNSW search patterns and large-scale vector operations
|
|
4
|
+
*/
|
|
5
|
+
import { HNSWNoun, HNSWVerb } from '../coreTypes.js';
|
|
6
|
+
import { BatchS3Operations } from './adapters/batchS3Operations.js';
|
|
7
|
+
declare enum PrefetchStrategy {
|
|
8
|
+
GRAPH_CONNECTIVITY = "connectivity",
|
|
9
|
+
VECTOR_SIMILARITY = "similarity",
|
|
10
|
+
ACCESS_PATTERN = "pattern",
|
|
11
|
+
HYBRID = "hybrid"
|
|
12
|
+
}
|
|
13
|
+
interface EnhancedCacheConfig {
|
|
14
|
+
hotCacheMaxSize?: number;
|
|
15
|
+
hotCacheEvictionThreshold?: number;
|
|
16
|
+
warmCacheMaxSize?: number;
|
|
17
|
+
warmCacheTTL?: number;
|
|
18
|
+
prefetchEnabled?: boolean;
|
|
19
|
+
prefetchStrategy?: PrefetchStrategy;
|
|
20
|
+
prefetchBatchSize?: number;
|
|
21
|
+
predictionLookahead?: number;
|
|
22
|
+
similarityThreshold?: number;
|
|
23
|
+
maxSimilarityDistance?: number;
|
|
24
|
+
backgroundOptimization?: boolean;
|
|
25
|
+
statisticsCollection?: boolean;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Enhanced cache manager with intelligent prefetching for HNSW operations
|
|
29
|
+
* Provides multi-level caching optimized for vector search workloads
|
|
30
|
+
*/
|
|
31
|
+
export declare class EnhancedCacheManager<T extends HNSWNoun | HNSWVerb> {
|
|
32
|
+
private hotCache;
|
|
33
|
+
private warmCache;
|
|
34
|
+
private prefetchQueue;
|
|
35
|
+
private accessPatterns;
|
|
36
|
+
private vectorIndex;
|
|
37
|
+
private config;
|
|
38
|
+
private batchOperations?;
|
|
39
|
+
private storageAdapter?;
|
|
40
|
+
private prefetchInProgress;
|
|
41
|
+
private stats;
|
|
42
|
+
constructor(config?: EnhancedCacheConfig);
|
|
43
|
+
/**
|
|
44
|
+
* Set storage adapters for warm/cold storage operations
|
|
45
|
+
*/
|
|
46
|
+
setStorageAdapters(storageAdapter: any, batchOperations?: BatchS3Operations): void;
|
|
47
|
+
/**
|
|
48
|
+
* Get item with intelligent prefetching
|
|
49
|
+
*/
|
|
50
|
+
get(id: string): Promise<T | null>;
|
|
51
|
+
/**
|
|
52
|
+
* Get multiple items efficiently with batch operations
|
|
53
|
+
*/
|
|
54
|
+
getMany(ids: string[]): Promise<Map<string, T>>;
|
|
55
|
+
/**
|
|
56
|
+
* Set item in cache with metadata
|
|
57
|
+
*/
|
|
58
|
+
set(id: string, item: T): Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* Intelligent prefetch based on access patterns and graph structure
|
|
61
|
+
*/
|
|
62
|
+
private schedulePrefetch;
|
|
63
|
+
/**
|
|
64
|
+
* Predict next nodes based on graph connectivity
|
|
65
|
+
*/
|
|
66
|
+
private predictByConnectivity;
|
|
67
|
+
/**
|
|
68
|
+
* Predict next nodes based on vector similarity
|
|
69
|
+
*/
|
|
70
|
+
private predictBySimilarity;
|
|
71
|
+
/**
|
|
72
|
+
* Predict based on historical access patterns
|
|
73
|
+
*/
|
|
74
|
+
private predictByAccessPattern;
|
|
75
|
+
/**
|
|
76
|
+
* Hybrid prediction combining multiple strategies
|
|
77
|
+
*/
|
|
78
|
+
private hybridPrediction;
|
|
79
|
+
/**
|
|
80
|
+
* Execute prefetch operation in background
|
|
81
|
+
*/
|
|
82
|
+
private executePrefetch;
|
|
83
|
+
/**
|
|
84
|
+
* Load item from storage adapter
|
|
85
|
+
*/
|
|
86
|
+
private loadFromStorage;
|
|
87
|
+
/**
|
|
88
|
+
* Promote frequently accessed item to hot cache
|
|
89
|
+
*/
|
|
90
|
+
private promoteToHotCache;
|
|
91
|
+
/**
|
|
92
|
+
* Evict least recently used items from hot cache
|
|
93
|
+
*/
|
|
94
|
+
private evictFromHotCache;
|
|
95
|
+
/**
|
|
96
|
+
* Evict expired items from warm cache
|
|
97
|
+
*/
|
|
98
|
+
private evictFromWarmCache;
|
|
99
|
+
/**
|
|
100
|
+
* Record access pattern for prediction
|
|
101
|
+
*/
|
|
102
|
+
private recordAccess;
|
|
103
|
+
/**
|
|
104
|
+
* Extract connected node IDs from HNSW item
|
|
105
|
+
*/
|
|
106
|
+
private extractConnectedNodes;
|
|
107
|
+
/**
|
|
108
|
+
* Check if cache entry is expired
|
|
109
|
+
*/
|
|
110
|
+
private isExpired;
|
|
111
|
+
/**
|
|
112
|
+
* Calculate cosine similarity between vectors
|
|
113
|
+
*/
|
|
114
|
+
private cosineSimilarity;
|
|
115
|
+
/**
|
|
116
|
+
* Calculate pattern similarity between access patterns
|
|
117
|
+
*/
|
|
118
|
+
private patternSimilarity;
|
|
119
|
+
/**
|
|
120
|
+
* Start background optimization process
|
|
121
|
+
*/
|
|
122
|
+
private startBackgroundOptimization;
|
|
123
|
+
/**
|
|
124
|
+
* Run background optimization tasks
|
|
125
|
+
*/
|
|
126
|
+
private runBackgroundOptimization;
|
|
127
|
+
/**
|
|
128
|
+
* Get cache statistics
|
|
129
|
+
*/
|
|
130
|
+
getStats(): typeof this.stats & {
|
|
131
|
+
hotCacheSize: number;
|
|
132
|
+
warmCacheSize: number;
|
|
133
|
+
prefetchQueueSize: number;
|
|
134
|
+
accessPatternsTracked: number;
|
|
135
|
+
};
|
|
136
|
+
/**
|
|
137
|
+
* Clear all caches
|
|
138
|
+
*/
|
|
139
|
+
clear(): void;
|
|
140
|
+
}
|
|
141
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enhancedCacheManager.d.ts","sourceRoot":"","sources":["../../src/storage/enhancedCacheManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAU,MAAM,iBAAiB,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAe,MAAM,iCAAiC,CAAA;AAchF,aAAK,gBAAgB;IACnB,kBAAkB,iBAAiB;IACnC,iBAAiB,eAAe;IAChC,cAAc,YAAY;IAC1B,MAAM,WAAW;CAClB;AAGD,UAAU,mBAAmB;IAE3B,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,yBAAyB,CAAC,EAAE,MAAM,CAAA;IAGlC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,YAAY,CAAC,EAAE,MAAM,CAAA;IAGrB,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAG5B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,qBAAqB,CAAC,EAAE,MAAM,CAAA;IAG9B,sBAAsB,CAAC,EAAE,OAAO,CAAA;IAChC,oBAAoB,CAAC,EAAE,OAAO,CAAA;CAC/B;AAED;;;GAGG;AACH,qBAAa,oBAAoB,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAC7D,OAAO,CAAC,QAAQ,CAA2C;IAC3D,OAAO,CAAC,SAAS,CAA2C;IAC5D,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,WAAW,CAA4B;IAE/C,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,eAAe,CAAC,CAAmB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAK;IAC5B,OAAO,CAAC,kBAAkB,CAAQ;IAGlC,OAAO,CAAC,KAAK,CAUZ;gBAEW,MAAM,GAAE,mBAAwB;IAuB5C;;OAEG;IACI,kBAAkB,CACvB,cAAc,EAAE,GAAG,EACnB,eAAe,CAAC,EAAE,iBAAiB,GAClC,IAAI;IAKP;;OAEG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAqD/C;;OAEG;IACU,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IA4B5D;;OAEG;IACU,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BpD;;OAEG;YACW,gBAAgB;IAoC9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwB7B;;OAEG;YACW,mBAAmB;IAuBjC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAsB9B;;OAEG;YACW,gBAAgB;IAkC9B;;OAEG;YACW,eAAe;IAiC7B;;OAEG;YACW,eAAe;IAa7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAazB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsBzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA8B1B;;OAEG;IACH,OAAO,CAAC,YAAY;IAmBpB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAa7B;;OAEG;IACH,OAAO,CAAC,SAAS;IAIjB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAqBzB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAMnC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAmBjC;;OAEG;IACI,QAAQ,IAAI,OAAO,IAAI,CAAC,KAAK,GAAG;QACrC,YAAY,EAAE,MAAM,CAAA;QACpB,aAAa,EAAE,MAAM,CAAA;QACrB,iBAAiB,EAAE,MAAM,CAAA;QACzB,qBAAqB,EAAE,MAAM,CAAA;KAC9B;IAUD;;OAEG;IACI,KAAK,IAAI,IAAI;CAOrB"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-Only Storage Optimizations for Production Deployments
|
|
3
|
+
* Implements compression, memory-mapping, and pre-built index segments
|
|
4
|
+
*/
|
|
5
|
+
import { HNSWNoun, Vector } from '../coreTypes.js';
|
|
6
|
+
declare enum CompressionType {
|
|
7
|
+
NONE = "none",
|
|
8
|
+
GZIP = "gzip",
|
|
9
|
+
BROTLI = "brotli",
|
|
10
|
+
QUANTIZATION = "quantization",
|
|
11
|
+
HYBRID = "hybrid"
|
|
12
|
+
}
|
|
13
|
+
declare enum QuantizationType {
|
|
14
|
+
SCALAR = "scalar",// 8-bit scalar quantization
|
|
15
|
+
PRODUCT = "product",// Product quantization
|
|
16
|
+
BINARY = "binary"
|
|
17
|
+
}
|
|
18
|
+
interface CompressionConfig {
|
|
19
|
+
vectorCompression: CompressionType;
|
|
20
|
+
metadataCompression: CompressionType;
|
|
21
|
+
quantizationType?: QuantizationType;
|
|
22
|
+
quantizationBits?: number;
|
|
23
|
+
compressionLevel?: number;
|
|
24
|
+
}
|
|
25
|
+
interface ReadOnlyConfig {
|
|
26
|
+
prebuiltIndexPath?: string;
|
|
27
|
+
memoryMapped?: boolean;
|
|
28
|
+
compression: CompressionConfig;
|
|
29
|
+
segmentSize?: number;
|
|
30
|
+
prefetchSegments?: number;
|
|
31
|
+
cacheIndexInMemory?: boolean;
|
|
32
|
+
}
|
|
33
|
+
interface IndexSegment {
|
|
34
|
+
id: string;
|
|
35
|
+
nodeCount: number;
|
|
36
|
+
vectorDimension: number;
|
|
37
|
+
compression: CompressionType;
|
|
38
|
+
s3Key?: string;
|
|
39
|
+
localPath?: string;
|
|
40
|
+
loadedInMemory: boolean;
|
|
41
|
+
lastAccessed: number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Read-only storage optimizations for high-performance production deployments
|
|
45
|
+
*/
|
|
46
|
+
export declare class ReadOnlyOptimizations {
|
|
47
|
+
private config;
|
|
48
|
+
private segments;
|
|
49
|
+
private compressionStats;
|
|
50
|
+
private quantizationCodebooks;
|
|
51
|
+
private memoryMappedBuffers;
|
|
52
|
+
constructor(config?: Partial<ReadOnlyConfig>);
|
|
53
|
+
/**
|
|
54
|
+
* Compress vector data using specified compression method
|
|
55
|
+
*/
|
|
56
|
+
compressVector(vector: Vector, segmentId: string): Promise<ArrayBuffer>;
|
|
57
|
+
/**
|
|
58
|
+
* Decompress vector data
|
|
59
|
+
*/
|
|
60
|
+
decompressVector(compressedData: ArrayBuffer, segmentId: string, originalDimension: number): Promise<Vector>;
|
|
61
|
+
/**
|
|
62
|
+
* Scalar quantization of vectors to 8-bit integers
|
|
63
|
+
*/
|
|
64
|
+
private quantizeVector;
|
|
65
|
+
/**
|
|
66
|
+
* Dequantize 8-bit vectors back to float32
|
|
67
|
+
*/
|
|
68
|
+
private dequantizeVector;
|
|
69
|
+
/**
|
|
70
|
+
* GZIP compression using browser/Node.js APIs
|
|
71
|
+
*/
|
|
72
|
+
private gzipCompress;
|
|
73
|
+
/**
|
|
74
|
+
* GZIP decompression
|
|
75
|
+
*/
|
|
76
|
+
private gzipDecompress;
|
|
77
|
+
/**
|
|
78
|
+
* Brotli compression (placeholder - similar to GZIP)
|
|
79
|
+
*/
|
|
80
|
+
private brotliCompress;
|
|
81
|
+
/**
|
|
82
|
+
* Brotli decompression (placeholder)
|
|
83
|
+
*/
|
|
84
|
+
private brotliDecompress;
|
|
85
|
+
/**
|
|
86
|
+
* Create prebuilt index segments for faster loading
|
|
87
|
+
*/
|
|
88
|
+
createPrebuiltSegments(nodes: HNSWNoun[], outputPath: string): Promise<IndexSegment[]>;
|
|
89
|
+
/**
|
|
90
|
+
* Compress an entire segment of nodes
|
|
91
|
+
*/
|
|
92
|
+
private compressSegment;
|
|
93
|
+
/**
|
|
94
|
+
* Load a segment from storage with caching
|
|
95
|
+
*/
|
|
96
|
+
loadSegment(segmentId: string): Promise<HNSWNoun[]>;
|
|
97
|
+
/**
|
|
98
|
+
* Load segment data from storage
|
|
99
|
+
*/
|
|
100
|
+
private loadSegmentFromStorage;
|
|
101
|
+
/**
|
|
102
|
+
* Deserialize and decompress segment data
|
|
103
|
+
*/
|
|
104
|
+
private deserializeSegment;
|
|
105
|
+
/**
|
|
106
|
+
* Serialize connections Map for storage
|
|
107
|
+
*/
|
|
108
|
+
private serializeConnections;
|
|
109
|
+
/**
|
|
110
|
+
* Deserialize connections from storage format
|
|
111
|
+
*/
|
|
112
|
+
private deserializeConnections;
|
|
113
|
+
/**
|
|
114
|
+
* Prefetch segments based on access patterns
|
|
115
|
+
*/
|
|
116
|
+
prefetchSegments(currentSegmentId: string): Promise<void>;
|
|
117
|
+
/**
|
|
118
|
+
* Update compression statistics
|
|
119
|
+
*/
|
|
120
|
+
private updateCompressionRatio;
|
|
121
|
+
/**
|
|
122
|
+
* Get compression statistics
|
|
123
|
+
*/
|
|
124
|
+
getCompressionStats(): typeof this.compressionStats & {
|
|
125
|
+
segmentCount: number;
|
|
126
|
+
memoryUsage: number;
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Cleanup memory-mapped buffers
|
|
130
|
+
*/
|
|
131
|
+
cleanup(): void;
|
|
132
|
+
}
|
|
133
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"readOnlyOptimizations.d.ts","sourceRoot":"","sources":["../../src/storage/readOnlyOptimizations.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAY,MAAM,EAAE,MAAM,iBAAiB,CAAA;AAG5D,aAAK,eAAe;IAClB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,MAAM,WAAW;IACjB,YAAY,iBAAiB;IAC7B,MAAM,WAAW;CAClB;AAGD,aAAK,gBAAgB;IACnB,MAAM,WAAW,CAAO,4BAA4B;IACpD,OAAO,YAAY,CAAK,uBAAuB;IAC/C,MAAM,WAAW;CAClB;AAED,UAAU,iBAAiB;IACzB,iBAAiB,EAAE,eAAe,CAAA;IAClC,mBAAmB,EAAE,eAAe,CAAA;IACpC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,UAAU,cAAc;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,WAAW,EAAE,iBAAiB,CAAA;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAC7B;AAED,UAAU,YAAY;IACpB,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,eAAe,CAAA;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,OAAO,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,gBAAgB,CAKvB;IAGD,OAAO,CAAC,qBAAqB,CAAuC;IAGpE,OAAO,CAAC,mBAAmB,CAAsC;gBAErD,MAAM,GAAE,OAAO,CAAC,cAAc,CAAM;IAsBhD;;OAEG;IACU,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IA0CpF;;OAEG;IACU,gBAAgB,CAC3B,cAAc,EAAE,WAAW,EAC3B,SAAS,EAAE,MAAM,EACjB,iBAAiB,EAAE,MAAM,GACxB,OAAO,CAAC,MAAM,CAAC;IAsBlB;;OAEG;YACW,cAAc;IA+B5B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAwBxB;;OAEG;YACW,YAAY;IAoC1B;;OAEG;YACW,cAAc;IAmC5B;;OAEG;YACW,cAAc;IAM5B;;OAEG;YACW,gBAAgB;IAK9B;;OAEG;IACU,sBAAsB,CACjC,KAAK,EAAE,QAAQ,EAAE,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAiC1B;;OAEG;YACW,eAAe;IAqB7B;;OAEG;IACU,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAyBhE;;OAEG;YACW,sBAAsB;IAOpC;;OAEG;YACW,kBAAkB;IA6BhC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAQ9B;;OAEG;IACU,gBAAgB,CAAC,gBAAgB,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtE;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAO9B;;OAEG;IACI,mBAAmB,IAAI,OAAO,IAAI,CAAC,gBAAgB,GAAG;QAC3D,YAAY,EAAE,MAAM,CAAA;QACpB,WAAW,EAAE,MAAM,CAAA;KACpB;IAWD;;OAEG;IACI,OAAO,IAAI,IAAI;CASvB"}
|