@soulcraft/brainy 5.3.6 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -0
- package/README.md +4 -3
- package/dist/augmentations/display/fieldPatterns.js +3 -3
- package/dist/augmentations/display/intelligentComputation.js +0 -2
- package/dist/augmentations/typeMatching/brainyTypes.js +6 -8
- package/dist/brainy.d.ts +61 -0
- package/dist/brainy.js +180 -24
- package/dist/cortex/neuralImport.js +0 -1
- package/dist/importers/SmartExcelImporter.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/neural/embeddedKeywordEmbeddings.d.ts +1 -1
- package/dist/neural/embeddedKeywordEmbeddings.js +56 -56
- package/dist/neural/embeddedTypeEmbeddings.d.ts +3 -3
- package/dist/neural/embeddedTypeEmbeddings.js +14 -14
- package/dist/neural/entityExtractor.js +2 -2
- package/dist/neural/relationshipConfidence.js +1 -1
- package/dist/neural/signals/VerbContextSignal.js +6 -6
- package/dist/neural/signals/VerbExactMatchSignal.js +9 -9
- package/dist/neural/signals/VerbPatternSignal.js +5 -5
- package/dist/query/typeAwareQueryPlanner.js +2 -3
- package/dist/storage/adapters/azureBlobStorage.d.ts +13 -64
- package/dist/storage/adapters/azureBlobStorage.js +78 -388
- package/dist/storage/adapters/fileSystemStorage.d.ts +12 -78
- package/dist/storage/adapters/fileSystemStorage.js +49 -395
- package/dist/storage/adapters/gcsStorage.d.ts +13 -134
- package/dist/storage/adapters/gcsStorage.js +79 -557
- package/dist/storage/adapters/historicalStorageAdapter.d.ts +181 -0
- package/dist/storage/adapters/historicalStorageAdapter.js +332 -0
- package/dist/storage/adapters/memoryStorage.d.ts +4 -113
- package/dist/storage/adapters/memoryStorage.js +34 -471
- package/dist/storage/adapters/opfsStorage.d.ts +14 -127
- package/dist/storage/adapters/opfsStorage.js +44 -693
- package/dist/storage/adapters/r2Storage.d.ts +8 -41
- package/dist/storage/adapters/r2Storage.js +49 -237
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +13 -111
- package/dist/storage/adapters/s3CompatibleStorage.js +77 -596
- package/dist/storage/baseStorage.d.ts +78 -38
- package/dist/storage/baseStorage.js +692 -23
- package/dist/storage/cow/BlobStorage.d.ts +2 -2
- package/dist/storage/cow/BlobStorage.js +4 -4
- package/dist/storage/storageFactory.d.ts +2 -3
- package/dist/storage/storageFactory.js +114 -66
- package/dist/types/graphTypes.d.ts +588 -230
- package/dist/types/graphTypes.js +683 -248
- package/dist/types/typeMigration.d.ts +95 -0
- package/dist/types/typeMigration.js +141 -0
- package/dist/utils/intelligentTypeMapper.js +2 -2
- package/dist/utils/metadataIndex.js +6 -6
- package/dist/vfs/types.d.ts +6 -2
- package/package.json +2 -2
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* 3. Service Account Credentials Object
|
|
9
9
|
* 4. HMAC Keys (fallback for backward compatibility)
|
|
10
10
|
*/
|
|
11
|
-
import { HNSWNoun, HNSWVerb,
|
|
11
|
+
import { HNSWNoun, HNSWVerb, StatisticsData } from '../../coreTypes.js';
|
|
12
12
|
import { BaseStorage, StorageBatchConfig } from '../baseStorage.js';
|
|
13
13
|
type HNSWNode = HNSWNoun;
|
|
14
14
|
type Edge = HNSWVerb;
|
|
@@ -21,6 +21,12 @@ type Edge = HNSWVerb;
|
|
|
21
21
|
* 2. Service Account Key File (if keyFilename provided)
|
|
22
22
|
* 3. Service Account Credentials Object (if credentials provided)
|
|
23
23
|
* 4. HMAC Keys (if accessKeyId/secretAccessKey provided)
|
|
24
|
+
*
|
|
25
|
+
* v5.4.0: Type-aware storage now built into BaseStorage
|
|
26
|
+
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
|
|
27
|
+
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
|
|
28
|
+
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
|
|
29
|
+
* - All operations now use type-first paths: entities/nouns/{type}/vectors/{shard}/{id}.json
|
|
24
30
|
*/
|
|
25
31
|
export declare class GcsStorage extends BaseStorage {
|
|
26
32
|
private storage;
|
|
@@ -55,6 +61,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
55
61
|
private nounCacheManager;
|
|
56
62
|
private verbCacheManager;
|
|
57
63
|
private logger;
|
|
64
|
+
private hnswLocks;
|
|
58
65
|
private skipInitialScan;
|
|
59
66
|
private skipCountsFile;
|
|
60
67
|
/**
|
|
@@ -153,10 +160,6 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
153
160
|
* Flush verb buffer to GCS
|
|
154
161
|
*/
|
|
155
162
|
private flushVerbBuffer;
|
|
156
|
-
/**
|
|
157
|
-
* Save a noun to storage (internal implementation)
|
|
158
|
-
*/
|
|
159
|
-
protected saveNoun_internal(noun: HNSWNoun): Promise<void>;
|
|
160
163
|
/**
|
|
161
164
|
* Save a node to storage
|
|
162
165
|
*/
|
|
@@ -165,20 +168,10 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
165
168
|
* Save a node directly to GCS (bypass buffer)
|
|
166
169
|
*/
|
|
167
170
|
private saveNodeDirect;
|
|
168
|
-
/**
|
|
169
|
-
* Get a noun from storage (internal implementation)
|
|
170
|
-
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
171
|
-
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
172
|
-
*/
|
|
173
|
-
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
174
171
|
/**
|
|
175
172
|
* Get a node from storage
|
|
176
173
|
*/
|
|
177
174
|
protected getNode(id: string): Promise<HNSWNode | null>;
|
|
178
|
-
/**
|
|
179
|
-
* Delete a noun from storage (internal implementation)
|
|
180
|
-
*/
|
|
181
|
-
protected deleteNoun_internal(id: string): Promise<void>;
|
|
182
175
|
/**
|
|
183
176
|
* Write an object to a specific path in GCS
|
|
184
177
|
* Primitive operation required by base class
|
|
@@ -203,10 +196,6 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
203
196
|
* @protected
|
|
204
197
|
*/
|
|
205
198
|
protected listObjectsUnderPath(prefix: string): Promise<string[]>;
|
|
206
|
-
/**
|
|
207
|
-
* Save a verb to storage (internal implementation)
|
|
208
|
-
*/
|
|
209
|
-
protected saveVerb_internal(verb: HNSWVerb): Promise<void>;
|
|
210
199
|
/**
|
|
211
200
|
* Save an edge to storage
|
|
212
201
|
*/
|
|
@@ -215,123 +204,10 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
215
204
|
* Save an edge directly to GCS (bypass buffer)
|
|
216
205
|
*/
|
|
217
206
|
private saveEdgeDirect;
|
|
218
|
-
/**
|
|
219
|
-
* Get a verb from storage (internal implementation)
|
|
220
|
-
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
221
|
-
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
222
|
-
*/
|
|
223
|
-
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
224
207
|
/**
|
|
225
208
|
* Get an edge from storage
|
|
226
209
|
*/
|
|
227
210
|
protected getEdge(id: string): Promise<Edge | null>;
|
|
228
|
-
/**
|
|
229
|
-
* Delete a verb from storage (internal implementation)
|
|
230
|
-
*/
|
|
231
|
-
protected deleteVerb_internal(id: string): Promise<void>;
|
|
232
|
-
/**
|
|
233
|
-
* Get nouns with pagination
|
|
234
|
-
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
235
|
-
* Iterates through all UUID-based shards (00-ff) for consistent pagination
|
|
236
|
-
*/
|
|
237
|
-
getNounsWithPagination(options?: {
|
|
238
|
-
limit?: number;
|
|
239
|
-
cursor?: string;
|
|
240
|
-
filter?: {
|
|
241
|
-
nounType?: string | string[];
|
|
242
|
-
service?: string | string[];
|
|
243
|
-
metadata?: Record<string, any>;
|
|
244
|
-
};
|
|
245
|
-
}): Promise<{
|
|
246
|
-
items: HNSWNounWithMetadata[];
|
|
247
|
-
totalCount?: number;
|
|
248
|
-
hasMore: boolean;
|
|
249
|
-
nextCursor?: string;
|
|
250
|
-
}>;
|
|
251
|
-
/**
|
|
252
|
-
* Get nodes with pagination (internal implementation)
|
|
253
|
-
* Iterates through UUID-based shards for consistent pagination
|
|
254
|
-
*/
|
|
255
|
-
private getNodesWithPagination;
|
|
256
|
-
/**
|
|
257
|
-
* Get nouns by noun type (internal implementation)
|
|
258
|
-
*/
|
|
259
|
-
protected getNounsByNounType_internal(nounType: string): Promise<HNSWNoun[]>;
|
|
260
|
-
/**
|
|
261
|
-
* Get verbs by source ID (internal implementation)
|
|
262
|
-
*/
|
|
263
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
264
|
-
/**
|
|
265
|
-
* Get verbs by target ID (internal implementation)
|
|
266
|
-
*/
|
|
267
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
268
|
-
/**
|
|
269
|
-
* Get verbs by type (internal implementation)
|
|
270
|
-
*/
|
|
271
|
-
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
272
|
-
/**
|
|
273
|
-
* Get verbs with pagination
|
|
274
|
-
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
275
|
-
*/
|
|
276
|
-
getVerbsWithPagination(options?: {
|
|
277
|
-
limit?: number;
|
|
278
|
-
cursor?: string;
|
|
279
|
-
filter?: {
|
|
280
|
-
verbType?: string | string[];
|
|
281
|
-
sourceId?: string | string[];
|
|
282
|
-
targetId?: string | string[];
|
|
283
|
-
service?: string | string[];
|
|
284
|
-
metadata?: Record<string, any>;
|
|
285
|
-
};
|
|
286
|
-
}): Promise<{
|
|
287
|
-
items: HNSWVerbWithMetadata[];
|
|
288
|
-
totalCount?: number;
|
|
289
|
-
hasMore: boolean;
|
|
290
|
-
nextCursor?: string;
|
|
291
|
-
}>;
|
|
292
|
-
/**
|
|
293
|
-
* Get nouns with filtering and pagination (public API)
|
|
294
|
-
*/
|
|
295
|
-
getNouns(options?: {
|
|
296
|
-
pagination?: {
|
|
297
|
-
offset?: number;
|
|
298
|
-
limit?: number;
|
|
299
|
-
cursor?: string;
|
|
300
|
-
};
|
|
301
|
-
filter?: {
|
|
302
|
-
nounType?: string | string[];
|
|
303
|
-
service?: string | string[];
|
|
304
|
-
metadata?: Record<string, any>;
|
|
305
|
-
};
|
|
306
|
-
}): Promise<{
|
|
307
|
-
items: any[];
|
|
308
|
-
totalCount?: number;
|
|
309
|
-
hasMore: boolean;
|
|
310
|
-
nextCursor?: string;
|
|
311
|
-
}>;
|
|
312
|
-
/**
|
|
313
|
-
* Get verbs with filtering and pagination (public API)
|
|
314
|
-
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
315
|
-
*/
|
|
316
|
-
getVerbs(options?: {
|
|
317
|
-
pagination?: {
|
|
318
|
-
offset?: number;
|
|
319
|
-
limit?: number;
|
|
320
|
-
cursor?: string;
|
|
321
|
-
};
|
|
322
|
-
filter?: {
|
|
323
|
-
verbType?: string | string[];
|
|
324
|
-
sourceId?: string | string[];
|
|
325
|
-
targetId?: string | string[];
|
|
326
|
-
service?: string | string[];
|
|
327
|
-
metadata?: Record<string, any>;
|
|
328
|
-
};
|
|
329
|
-
}): Promise<{
|
|
330
|
-
items: HNSWVerbWithMetadata[];
|
|
331
|
-
totalCount?: number;
|
|
332
|
-
hasMore: boolean;
|
|
333
|
-
nextCursor?: string;
|
|
334
|
-
}>;
|
|
335
211
|
/**
|
|
336
212
|
* Batch fetch metadata for multiple noun IDs (efficient for large queries)
|
|
337
213
|
* Uses smaller batches to prevent GCS socket exhaustion
|
|
@@ -375,11 +251,14 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
375
251
|
protected persistCounts(): Promise<void>;
|
|
376
252
|
/**
|
|
377
253
|
* Get a noun's vector for HNSW rebuild
|
|
254
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
378
255
|
*/
|
|
379
256
|
getNounVector(id: string): Promise<number[] | null>;
|
|
380
257
|
/**
|
|
381
258
|
* Save HNSW graph data for a noun
|
|
382
|
-
*
|
|
259
|
+
*
|
|
260
|
+
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
|
|
261
|
+
* CRITICAL: Uses mutex locking to prevent read-modify-write races
|
|
383
262
|
*/
|
|
384
263
|
saveHNSWData(nounId: string, hnswData: {
|
|
385
264
|
level: number;
|
|
@@ -387,7 +266,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
387
266
|
}): Promise<void>;
|
|
388
267
|
/**
|
|
389
268
|
* Get HNSW graph data for a noun
|
|
390
|
-
*
|
|
269
|
+
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
|
|
391
270
|
*/
|
|
392
271
|
getHNSWData(nounId: string): Promise<{
|
|
393
272
|
level: number;
|