@soulcraft/brainy 3.50.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.js +2 -0
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* 3. Service Account Credentials Object
|
|
9
9
|
* 4. HMAC Keys (fallback for backward compatibility)
|
|
10
10
|
*/
|
|
11
|
-
import {
|
|
11
|
+
import { HNSWNoun, HNSWVerb, HNSWNounWithMetadata, HNSWVerbWithMetadata, StatisticsData } from '../../coreTypes.js';
|
|
12
12
|
import { BaseStorage } from '../baseStorage.js';
|
|
13
13
|
type HNSWNode = HNSWNoun;
|
|
14
14
|
type Edge = HNSWVerb;
|
|
@@ -148,7 +148,8 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
148
148
|
private saveNodeDirect;
|
|
149
149
|
/**
|
|
150
150
|
* Get a noun from storage (internal implementation)
|
|
151
|
-
*
|
|
151
|
+
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
152
|
+
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
152
153
|
*/
|
|
153
154
|
protected getNoun_internal(id: string): Promise<HNSWNoun | null>;
|
|
154
155
|
/**
|
|
@@ -197,7 +198,8 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
197
198
|
private saveEdgeDirect;
|
|
198
199
|
/**
|
|
199
200
|
* Get a verb from storage (internal implementation)
|
|
200
|
-
*
|
|
201
|
+
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
202
|
+
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
201
203
|
*/
|
|
202
204
|
protected getVerb_internal(id: string): Promise<HNSWVerb | null>;
|
|
203
205
|
/**
|
|
@@ -210,6 +212,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
210
212
|
protected deleteVerb_internal(id: string): Promise<void>;
|
|
211
213
|
/**
|
|
212
214
|
* Get nouns with pagination
|
|
215
|
+
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
213
216
|
* Iterates through all UUID-based shards (00-ff) for consistent pagination
|
|
214
217
|
*/
|
|
215
218
|
getNounsWithPagination(options?: {
|
|
@@ -221,7 +224,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
221
224
|
metadata?: Record<string, any>;
|
|
222
225
|
};
|
|
223
226
|
}): Promise<{
|
|
224
|
-
items:
|
|
227
|
+
items: HNSWNounWithMetadata[];
|
|
225
228
|
totalCount?: number;
|
|
226
229
|
hasMore: boolean;
|
|
227
230
|
nextCursor?: string;
|
|
@@ -238,17 +241,18 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
238
241
|
/**
|
|
239
242
|
* Get verbs by source ID (internal implementation)
|
|
240
243
|
*/
|
|
241
|
-
protected getVerbsBySource_internal(sourceId: string): Promise<
|
|
244
|
+
protected getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
242
245
|
/**
|
|
243
246
|
* Get verbs by target ID (internal implementation)
|
|
244
247
|
*/
|
|
245
|
-
protected getVerbsByTarget_internal(targetId: string): Promise<
|
|
248
|
+
protected getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]>;
|
|
246
249
|
/**
|
|
247
250
|
* Get verbs by type (internal implementation)
|
|
248
251
|
*/
|
|
249
|
-
protected getVerbsByType_internal(type: string): Promise<
|
|
252
|
+
protected getVerbsByType_internal(type: string): Promise<HNSWVerbWithMetadata[]>;
|
|
250
253
|
/**
|
|
251
254
|
* Get verbs with pagination
|
|
255
|
+
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
252
256
|
*/
|
|
253
257
|
getVerbsWithPagination(options?: {
|
|
254
258
|
limit?: number;
|
|
@@ -261,7 +265,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
261
265
|
metadata?: Record<string, any>;
|
|
262
266
|
};
|
|
263
267
|
}): Promise<{
|
|
264
|
-
items:
|
|
268
|
+
items: HNSWVerbWithMetadata[];
|
|
265
269
|
totalCount?: number;
|
|
266
270
|
hasMore: boolean;
|
|
267
271
|
nextCursor?: string;
|
|
@@ -288,6 +292,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
288
292
|
}>;
|
|
289
293
|
/**
|
|
290
294
|
* Get verbs with filtering and pagination (public API)
|
|
295
|
+
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
291
296
|
*/
|
|
292
297
|
getVerbs(options?: {
|
|
293
298
|
pagination?: {
|
|
@@ -303,7 +308,7 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
303
308
|
metadata?: Record<string, any>;
|
|
304
309
|
};
|
|
305
310
|
}): Promise<{
|
|
306
|
-
items:
|
|
311
|
+
items: HNSWVerbWithMetadata[];
|
|
307
312
|
totalCount?: number;
|
|
308
313
|
hasMore: boolean;
|
|
309
314
|
nextCursor?: string;
|
|
@@ -384,5 +389,110 @@ export declare class GcsStorage extends BaseStorage {
|
|
|
384
389
|
entryPointId: string | null;
|
|
385
390
|
maxLevel: number;
|
|
386
391
|
} | null>;
|
|
392
|
+
/**
|
|
393
|
+
* Set lifecycle policy for automatic tier transitions and deletions
|
|
394
|
+
*
|
|
395
|
+
* GCS Storage Classes:
|
|
396
|
+
* - STANDARD: Hot data, most expensive (~$0.020/GB/month)
|
|
397
|
+
* - NEARLINE: <1 access/month (~$0.010/GB/month, 50% cheaper)
|
|
398
|
+
* - COLDLINE: <1 access/quarter (~$0.004/GB/month, 80% cheaper)
|
|
399
|
+
* - ARCHIVE: <1 access/year (~$0.0012/GB/month, 94% cheaper!)
|
|
400
|
+
*
|
|
401
|
+
* Example usage:
|
|
402
|
+
* ```typescript
|
|
403
|
+
* await storage.setLifecyclePolicy({
|
|
404
|
+
* rules: [
|
|
405
|
+
* {
|
|
406
|
+
* action: { type: 'SetStorageClass', storageClass: 'NEARLINE' },
|
|
407
|
+
* condition: { age: 30 }
|
|
408
|
+
* },
|
|
409
|
+
* {
|
|
410
|
+
* action: { type: 'SetStorageClass', storageClass: 'COLDLINE' },
|
|
411
|
+
* condition: { age: 90 }
|
|
412
|
+
* },
|
|
413
|
+
* {
|
|
414
|
+
* action: { type: 'Delete' },
|
|
415
|
+
* condition: { age: 365 }
|
|
416
|
+
* }
|
|
417
|
+
* ]
|
|
418
|
+
* })
|
|
419
|
+
* ```
|
|
420
|
+
*
|
|
421
|
+
* @param options Lifecycle configuration with rules for transitions and deletions
|
|
422
|
+
*/
|
|
423
|
+
setLifecyclePolicy(options: {
|
|
424
|
+
rules: Array<{
|
|
425
|
+
action: {
|
|
426
|
+
type: 'Delete' | 'SetStorageClass';
|
|
427
|
+
storageClass?: 'STANDARD' | 'NEARLINE' | 'COLDLINE' | 'ARCHIVE';
|
|
428
|
+
};
|
|
429
|
+
condition: {
|
|
430
|
+
age?: number;
|
|
431
|
+
createdBefore?: string;
|
|
432
|
+
matchesPrefix?: string[];
|
|
433
|
+
matchesSuffix?: string[];
|
|
434
|
+
};
|
|
435
|
+
}>;
|
|
436
|
+
}): Promise<void>;
|
|
437
|
+
/**
|
|
438
|
+
* Get current lifecycle policy configuration
|
|
439
|
+
*
|
|
440
|
+
* @returns Lifecycle configuration with all rules, or null if no policy is set
|
|
441
|
+
*/
|
|
442
|
+
getLifecyclePolicy(): Promise<{
|
|
443
|
+
rules: Array<{
|
|
444
|
+
action: {
|
|
445
|
+
type: string;
|
|
446
|
+
storageClass?: string;
|
|
447
|
+
};
|
|
448
|
+
condition: {
|
|
449
|
+
age?: number;
|
|
450
|
+
createdBefore?: string;
|
|
451
|
+
matchesPrefix?: string[];
|
|
452
|
+
matchesSuffix?: string[];
|
|
453
|
+
};
|
|
454
|
+
}>;
|
|
455
|
+
} | null>;
|
|
456
|
+
/**
|
|
457
|
+
* Remove lifecycle policy from bucket
|
|
458
|
+
*/
|
|
459
|
+
removeLifecyclePolicy(): Promise<void>;
|
|
460
|
+
/**
|
|
461
|
+
* Enable Autoclass for automatic storage class optimization
|
|
462
|
+
*
|
|
463
|
+
* GCS Autoclass automatically moves objects between storage classes based on access patterns:
|
|
464
|
+
* - Frequent Access → STANDARD
|
|
465
|
+
* - Infrequent Access (30 days) → NEARLINE
|
|
466
|
+
* - Rarely Accessed (90 days) → COLDLINE
|
|
467
|
+
* - Archive Access (365 days) → ARCHIVE
|
|
468
|
+
*
|
|
469
|
+
* Benefits:
|
|
470
|
+
* - Automatic optimization based on access patterns (no manual rules needed)
|
|
471
|
+
* - No early deletion fees
|
|
472
|
+
* - No retrieval fees for NEARLINE/COLDLINE (only ARCHIVE has retrieval fees)
|
|
473
|
+
* - Up to 94% cost savings automatically
|
|
474
|
+
*
|
|
475
|
+
* Note: Autoclass is a bucket-level feature that requires bucket.update permission.
|
|
476
|
+
* It cannot be enabled per-object or per-prefix.
|
|
477
|
+
*
|
|
478
|
+
* @param options Autoclass configuration
|
|
479
|
+
*/
|
|
480
|
+
enableAutoclass(options?: {
|
|
481
|
+
terminalStorageClass?: 'NEARLINE' | 'ARCHIVE';
|
|
482
|
+
}): Promise<void>;
|
|
483
|
+
/**
|
|
484
|
+
* Get Autoclass configuration and status
|
|
485
|
+
*
|
|
486
|
+
* @returns Autoclass status, or null if not configured
|
|
487
|
+
*/
|
|
488
|
+
getAutoclassStatus(): Promise<{
|
|
489
|
+
enabled: boolean;
|
|
490
|
+
terminalStorageClass?: string;
|
|
491
|
+
toggleTime?: string;
|
|
492
|
+
} | null>;
|
|
493
|
+
/**
|
|
494
|
+
* Disable Autoclass for the bucket
|
|
495
|
+
*/
|
|
496
|
+
disableAutoclass(): Promise<void>;
|
|
387
497
|
}
|
|
388
498
|
export {};
|
|
@@ -369,21 +369,17 @@ export class GcsStorage extends BaseStorage {
|
|
|
369
369
|
}
|
|
370
370
|
/**
|
|
371
371
|
* Get a noun from storage (internal implementation)
|
|
372
|
-
*
|
|
372
|
+
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
373
|
+
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
373
374
|
*/
|
|
374
375
|
async getNoun_internal(id) {
|
|
375
|
-
//
|
|
376
|
+
// v4.0.0: Return ONLY vector data (no metadata field)
|
|
376
377
|
const node = await this.getNode(id);
|
|
377
378
|
if (!node) {
|
|
378
379
|
return null;
|
|
379
380
|
}
|
|
380
|
-
//
|
|
381
|
-
|
|
382
|
-
// Combine into complete noun object
|
|
383
|
-
return {
|
|
384
|
-
...node,
|
|
385
|
-
metadata: metadata || {}
|
|
386
|
-
};
|
|
381
|
+
// Return pure vector structure
|
|
382
|
+
return node;
|
|
387
383
|
}
|
|
388
384
|
/**
|
|
389
385
|
* Get a node from storage
|
|
@@ -682,21 +678,17 @@ export class GcsStorage extends BaseStorage {
|
|
|
682
678
|
}
|
|
683
679
|
/**
|
|
684
680
|
* Get a verb from storage (internal implementation)
|
|
685
|
-
*
|
|
681
|
+
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
682
|
+
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
686
683
|
*/
|
|
687
684
|
async getVerb_internal(id) {
|
|
688
|
-
//
|
|
685
|
+
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
|
689
686
|
const edge = await this.getEdge(id);
|
|
690
687
|
if (!edge) {
|
|
691
688
|
return null;
|
|
692
689
|
}
|
|
693
|
-
//
|
|
694
|
-
|
|
695
|
-
// Combine into complete verb object
|
|
696
|
-
return {
|
|
697
|
-
...edge,
|
|
698
|
-
metadata: metadata || {}
|
|
699
|
-
};
|
|
690
|
+
// Return pure vector + core fields structure
|
|
691
|
+
return edge;
|
|
700
692
|
}
|
|
701
693
|
/**
|
|
702
694
|
* Get an edge from storage
|
|
@@ -724,7 +716,7 @@ export class GcsStorage extends BaseStorage {
|
|
|
724
716
|
for (const [level, verbIds] of Object.entries(data.connections || {})) {
|
|
725
717
|
connections.set(Number(level), new Set(verbIds));
|
|
726
718
|
}
|
|
727
|
-
//
|
|
719
|
+
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
|
728
720
|
const edge = {
|
|
729
721
|
id: data.id,
|
|
730
722
|
vector: data.vector,
|
|
@@ -732,9 +724,9 @@ export class GcsStorage extends BaseStorage {
|
|
|
732
724
|
// CORE RELATIONAL DATA (read from vector file)
|
|
733
725
|
verb: data.verb,
|
|
734
726
|
sourceId: data.sourceId,
|
|
735
|
-
targetId: data.targetId
|
|
736
|
-
//
|
|
737
|
-
metadata
|
|
727
|
+
targetId: data.targetId
|
|
728
|
+
// ✅ NO metadata field in v4.0.0
|
|
729
|
+
// User metadata retrieved separately via getVerbMetadata()
|
|
738
730
|
};
|
|
739
731
|
// Update cache
|
|
740
732
|
this.verbCacheManager.set(id, edge);
|
|
@@ -797,6 +789,7 @@ export class GcsStorage extends BaseStorage {
|
|
|
797
789
|
}
|
|
798
790
|
/**
|
|
799
791
|
* Get nouns with pagination
|
|
792
|
+
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
|
|
800
793
|
* Iterates through all UUID-based shards (00-ff) for consistent pagination
|
|
801
794
|
*/
|
|
802
795
|
async getNounsWithPagination(options = {}) {
|
|
@@ -809,27 +802,50 @@ export class GcsStorage extends BaseStorage {
|
|
|
809
802
|
cursor,
|
|
810
803
|
useCache: true
|
|
811
804
|
});
|
|
812
|
-
//
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
if (
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
const
|
|
823
|
-
|
|
824
|
-
|
|
805
|
+
// v4.0.0: Combine nodes with metadata to create HNSWNounWithMetadata[]
|
|
806
|
+
const items = [];
|
|
807
|
+
for (const node of result.nodes) {
|
|
808
|
+
const metadata = await this.getNounMetadata(node.id);
|
|
809
|
+
if (!metadata)
|
|
810
|
+
continue;
|
|
811
|
+
// Apply filters if provided
|
|
812
|
+
if (options.filter) {
|
|
813
|
+
// Filter by noun type
|
|
814
|
+
if (options.filter.nounType) {
|
|
815
|
+
const nounTypes = Array.isArray(options.filter.nounType)
|
|
816
|
+
? options.filter.nounType
|
|
817
|
+
: [options.filter.nounType];
|
|
818
|
+
const nounType = metadata.type || metadata.noun;
|
|
819
|
+
if (!nounType || !nounTypes.includes(nounType)) {
|
|
820
|
+
continue;
|
|
825
821
|
}
|
|
826
822
|
}
|
|
827
|
-
|
|
823
|
+
// Filter by metadata fields if specified
|
|
824
|
+
if (options.filter.metadata) {
|
|
825
|
+
let metadataMatch = true;
|
|
826
|
+
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
|
827
|
+
const metadataValue = metadata[key];
|
|
828
|
+
if (metadataValue !== value) {
|
|
829
|
+
metadataMatch = false;
|
|
830
|
+
break;
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
if (!metadataMatch)
|
|
834
|
+
continue;
|
|
835
|
+
}
|
|
828
836
|
}
|
|
829
|
-
//
|
|
837
|
+
// Combine node with metadata
|
|
838
|
+
const nounWithMetadata = {
|
|
839
|
+
id: node.id,
|
|
840
|
+
vector: [...node.vector],
|
|
841
|
+
connections: new Map(node.connections),
|
|
842
|
+
level: node.level || 0,
|
|
843
|
+
metadata: metadata
|
|
844
|
+
};
|
|
845
|
+
items.push(nounWithMetadata);
|
|
830
846
|
}
|
|
831
847
|
return {
|
|
832
|
-
items
|
|
848
|
+
items,
|
|
833
849
|
totalCount: result.totalCount,
|
|
834
850
|
hasMore: result.hasMore,
|
|
835
851
|
nextCursor: result.nextCursor
|
|
@@ -976,6 +992,7 @@ export class GcsStorage extends BaseStorage {
|
|
|
976
992
|
}
|
|
977
993
|
/**
|
|
978
994
|
* Get verbs with pagination
|
|
995
|
+
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
979
996
|
*/
|
|
980
997
|
async getVerbsWithPagination(options = {}) {
|
|
981
998
|
await this.ensureInitialized();
|
|
@@ -1016,51 +1033,65 @@ export class GcsStorage extends BaseStorage {
|
|
|
1016
1033
|
hnswVerbs.push(verb);
|
|
1017
1034
|
}
|
|
1018
1035
|
}
|
|
1019
|
-
//
|
|
1020
|
-
const
|
|
1036
|
+
// v4.0.0: Combine HNSWVerbs with metadata to create HNSWVerbWithMetadata[]
|
|
1037
|
+
const items = [];
|
|
1021
1038
|
for (const hnswVerb of hnswVerbs) {
|
|
1022
|
-
const
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
}
|
|
1027
|
-
// Apply filters
|
|
1028
|
-
let filteredVerbs = graphVerbs;
|
|
1029
|
-
if (options.filter) {
|
|
1030
|
-
filteredVerbs = graphVerbs.filter((graphVerb) => {
|
|
1031
|
-
// Filter by sourceId
|
|
1039
|
+
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
1040
|
+
// Apply filters
|
|
1041
|
+
if (options.filter) {
|
|
1042
|
+
// v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb structure
|
|
1032
1043
|
if (options.filter.sourceId) {
|
|
1033
1044
|
const sourceIds = Array.isArray(options.filter.sourceId)
|
|
1034
1045
|
? options.filter.sourceId
|
|
1035
1046
|
: [options.filter.sourceId];
|
|
1036
|
-
if (!sourceIds.includes(
|
|
1037
|
-
|
|
1047
|
+
if (!hnswVerb.sourceId || !sourceIds.includes(hnswVerb.sourceId)) {
|
|
1048
|
+
continue;
|
|
1038
1049
|
}
|
|
1039
1050
|
}
|
|
1040
|
-
// Filter by targetId
|
|
1041
1051
|
if (options.filter.targetId) {
|
|
1042
1052
|
const targetIds = Array.isArray(options.filter.targetId)
|
|
1043
1053
|
? options.filter.targetId
|
|
1044
1054
|
: [options.filter.targetId];
|
|
1045
|
-
if (!targetIds.includes(
|
|
1046
|
-
|
|
1055
|
+
if (!hnswVerb.targetId || !targetIds.includes(hnswVerb.targetId)) {
|
|
1056
|
+
continue;
|
|
1047
1057
|
}
|
|
1048
1058
|
}
|
|
1049
|
-
// Filter by verbType
|
|
1050
1059
|
if (options.filter.verbType) {
|
|
1051
1060
|
const verbTypes = Array.isArray(options.filter.verbType)
|
|
1052
1061
|
? options.filter.verbType
|
|
1053
1062
|
: [options.filter.verbType];
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
return false;
|
|
1063
|
+
if (!hnswVerb.verb || !verbTypes.includes(hnswVerb.verb)) {
|
|
1064
|
+
continue;
|
|
1057
1065
|
}
|
|
1058
1066
|
}
|
|
1059
|
-
|
|
1060
|
-
|
|
1067
|
+
// Filter by metadata fields if specified
|
|
1068
|
+
if (options.filter.metadata && metadata) {
|
|
1069
|
+
let metadataMatch = true;
|
|
1070
|
+
for (const [key, value] of Object.entries(options.filter.metadata)) {
|
|
1071
|
+
const metadataValue = metadata[key];
|
|
1072
|
+
if (metadataValue !== value) {
|
|
1073
|
+
metadataMatch = false;
|
|
1074
|
+
break;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
if (!metadataMatch)
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
// Combine verb with metadata
|
|
1082
|
+
const verbWithMetadata = {
|
|
1083
|
+
id: hnswVerb.id,
|
|
1084
|
+
vector: [...hnswVerb.vector],
|
|
1085
|
+
connections: new Map(hnswVerb.connections),
|
|
1086
|
+
verb: hnswVerb.verb,
|
|
1087
|
+
sourceId: hnswVerb.sourceId,
|
|
1088
|
+
targetId: hnswVerb.targetId,
|
|
1089
|
+
metadata: metadata || {}
|
|
1090
|
+
};
|
|
1091
|
+
items.push(verbWithMetadata);
|
|
1061
1092
|
}
|
|
1062
1093
|
return {
|
|
1063
|
-
items
|
|
1094
|
+
items,
|
|
1064
1095
|
totalCount: this.totalVerbCount,
|
|
1065
1096
|
hasMore: !!response?.nextPageToken,
|
|
1066
1097
|
nextCursor: response?.nextPageToken
|
|
@@ -1085,6 +1116,7 @@ export class GcsStorage extends BaseStorage {
|
|
|
1085
1116
|
}
|
|
1086
1117
|
/**
|
|
1087
1118
|
* Get verbs with filtering and pagination (public API)
|
|
1119
|
+
* v4.0.0: Returns HNSWVerbWithMetadata[] (includes metadata field)
|
|
1088
1120
|
*/
|
|
1089
1121
|
async getVerbs(options) {
|
|
1090
1122
|
const limit = options?.pagination?.limit || 100;
|
|
@@ -1448,5 +1480,228 @@ export class GcsStorage extends BaseStorage {
|
|
|
1448
1480
|
throw new Error(`Failed to get HNSW system data: ${error}`);
|
|
1449
1481
|
}
|
|
1450
1482
|
}
|
|
1483
|
+
// ============================================================================
|
|
1484
|
+
// GCS Lifecycle Management & Autoclass (v4.0.0)
|
|
1485
|
+
// Cost optimization through automatic tier transitions and Autoclass
|
|
1486
|
+
// ============================================================================
|
|
1487
|
+
/**
|
|
1488
|
+
* Set lifecycle policy for automatic tier transitions and deletions
|
|
1489
|
+
*
|
|
1490
|
+
* GCS Storage Classes:
|
|
1491
|
+
* - STANDARD: Hot data, most expensive (~$0.020/GB/month)
|
|
1492
|
+
* - NEARLINE: <1 access/month (~$0.010/GB/month, 50% cheaper)
|
|
1493
|
+
* - COLDLINE: <1 access/quarter (~$0.004/GB/month, 80% cheaper)
|
|
1494
|
+
* - ARCHIVE: <1 access/year (~$0.0012/GB/month, 94% cheaper!)
|
|
1495
|
+
*
|
|
1496
|
+
* Example usage:
|
|
1497
|
+
* ```typescript
|
|
1498
|
+
* await storage.setLifecyclePolicy({
|
|
1499
|
+
* rules: [
|
|
1500
|
+
* {
|
|
1501
|
+
* action: { type: 'SetStorageClass', storageClass: 'NEARLINE' },
|
|
1502
|
+
* condition: { age: 30 }
|
|
1503
|
+
* },
|
|
1504
|
+
* {
|
|
1505
|
+
* action: { type: 'SetStorageClass', storageClass: 'COLDLINE' },
|
|
1506
|
+
* condition: { age: 90 }
|
|
1507
|
+
* },
|
|
1508
|
+
* {
|
|
1509
|
+
* action: { type: 'Delete' },
|
|
1510
|
+
* condition: { age: 365 }
|
|
1511
|
+
* }
|
|
1512
|
+
* ]
|
|
1513
|
+
* })
|
|
1514
|
+
* ```
|
|
1515
|
+
*
|
|
1516
|
+
* @param options Lifecycle configuration with rules for transitions and deletions
|
|
1517
|
+
*/
|
|
1518
|
+
async setLifecyclePolicy(options) {
|
|
1519
|
+
await this.ensureInitialized();
|
|
1520
|
+
try {
|
|
1521
|
+
this.logger.info(`Setting GCS lifecycle policy with ${options.rules.length} rules`);
|
|
1522
|
+
// GCS lifecycle rules format
|
|
1523
|
+
const lifecycleRules = options.rules.map(rule => {
|
|
1524
|
+
const gcsRule = {
|
|
1525
|
+
action: {
|
|
1526
|
+
type: rule.action.type
|
|
1527
|
+
},
|
|
1528
|
+
condition: {}
|
|
1529
|
+
};
|
|
1530
|
+
// Add storage class for SetStorageClass action
|
|
1531
|
+
if (rule.action.type === 'SetStorageClass' && rule.action.storageClass) {
|
|
1532
|
+
gcsRule.action.storageClass = rule.action.storageClass;
|
|
1533
|
+
}
|
|
1534
|
+
// Add conditions
|
|
1535
|
+
if (rule.condition.age !== undefined) {
|
|
1536
|
+
gcsRule.condition.age = rule.condition.age;
|
|
1537
|
+
}
|
|
1538
|
+
if (rule.condition.createdBefore) {
|
|
1539
|
+
gcsRule.condition.createdBefore = rule.condition.createdBefore;
|
|
1540
|
+
}
|
|
1541
|
+
if (rule.condition.matchesPrefix) {
|
|
1542
|
+
gcsRule.condition.matchesPrefix = rule.condition.matchesPrefix;
|
|
1543
|
+
}
|
|
1544
|
+
if (rule.condition.matchesSuffix) {
|
|
1545
|
+
gcsRule.condition.matchesSuffix = rule.condition.matchesSuffix;
|
|
1546
|
+
}
|
|
1547
|
+
return gcsRule;
|
|
1548
|
+
});
|
|
1549
|
+
// Update bucket lifecycle configuration
|
|
1550
|
+
await this.bucket.setMetadata({
|
|
1551
|
+
lifecycle: {
|
|
1552
|
+
rule: lifecycleRules
|
|
1553
|
+
}
|
|
1554
|
+
});
|
|
1555
|
+
this.logger.info(`Successfully set lifecycle policy with ${options.rules.length} rules`);
|
|
1556
|
+
}
|
|
1557
|
+
catch (error) {
|
|
1558
|
+
this.logger.error('Failed to set lifecycle policy:', error);
|
|
1559
|
+
throw new Error(`Failed to set GCS lifecycle policy: ${error.message || error}`);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
/**
|
|
1563
|
+
* Get current lifecycle policy configuration
|
|
1564
|
+
*
|
|
1565
|
+
* @returns Lifecycle configuration with all rules, or null if no policy is set
|
|
1566
|
+
*/
|
|
1567
|
+
async getLifecyclePolicy() {
|
|
1568
|
+
await this.ensureInitialized();
|
|
1569
|
+
try {
|
|
1570
|
+
this.logger.info('Getting GCS lifecycle policy');
|
|
1571
|
+
const [metadata] = await this.bucket.getMetadata();
|
|
1572
|
+
if (!metadata.lifecycle || !metadata.lifecycle.rule || metadata.lifecycle.rule.length === 0) {
|
|
1573
|
+
this.logger.info('No lifecycle policy configured');
|
|
1574
|
+
return null;
|
|
1575
|
+
}
|
|
1576
|
+
// Convert GCS format to our format
|
|
1577
|
+
const rules = metadata.lifecycle.rule.map((rule) => ({
|
|
1578
|
+
action: {
|
|
1579
|
+
type: rule.action.type,
|
|
1580
|
+
...(rule.action.storageClass && { storageClass: rule.action.storageClass })
|
|
1581
|
+
},
|
|
1582
|
+
condition: {
|
|
1583
|
+
...(rule.condition.age !== undefined && { age: rule.condition.age }),
|
|
1584
|
+
...(rule.condition.createdBefore && { createdBefore: rule.condition.createdBefore }),
|
|
1585
|
+
...(rule.condition.matchesPrefix && { matchesPrefix: rule.condition.matchesPrefix }),
|
|
1586
|
+
...(rule.condition.matchesSuffix && { matchesSuffix: rule.condition.matchesSuffix })
|
|
1587
|
+
}
|
|
1588
|
+
}));
|
|
1589
|
+
this.logger.info(`Found lifecycle policy with ${rules.length} rules`);
|
|
1590
|
+
return { rules };
|
|
1591
|
+
}
|
|
1592
|
+
catch (error) {
|
|
1593
|
+
this.logger.error('Failed to get lifecycle policy:', error);
|
|
1594
|
+
throw new Error(`Failed to get GCS lifecycle policy: ${error.message || error}`);
|
|
1595
|
+
}
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Remove lifecycle policy from bucket
|
|
1599
|
+
*/
|
|
1600
|
+
async removeLifecyclePolicy() {
|
|
1601
|
+
await this.ensureInitialized();
|
|
1602
|
+
try {
|
|
1603
|
+
this.logger.info('Removing GCS lifecycle policy');
|
|
1604
|
+
// Remove lifecycle configuration
|
|
1605
|
+
await this.bucket.setMetadata({
|
|
1606
|
+
lifecycle: null
|
|
1607
|
+
});
|
|
1608
|
+
this.logger.info('Successfully removed lifecycle policy');
|
|
1609
|
+
}
|
|
1610
|
+
catch (error) {
|
|
1611
|
+
this.logger.error('Failed to remove lifecycle policy:', error);
|
|
1612
|
+
throw new Error(`Failed to remove GCS lifecycle policy: ${error.message || error}`);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
/**
|
|
1616
|
+
* Enable Autoclass for automatic storage class optimization
|
|
1617
|
+
*
|
|
1618
|
+
* GCS Autoclass automatically moves objects between storage classes based on access patterns:
|
|
1619
|
+
* - Frequent Access → STANDARD
|
|
1620
|
+
* - Infrequent Access (30 days) → NEARLINE
|
|
1621
|
+
* - Rarely Accessed (90 days) → COLDLINE
|
|
1622
|
+
* - Archive Access (365 days) → ARCHIVE
|
|
1623
|
+
*
|
|
1624
|
+
* Benefits:
|
|
1625
|
+
* - Automatic optimization based on access patterns (no manual rules needed)
|
|
1626
|
+
* - No early deletion fees
|
|
1627
|
+
* - No retrieval fees for NEARLINE/COLDLINE (only ARCHIVE has retrieval fees)
|
|
1628
|
+
* - Up to 94% cost savings automatically
|
|
1629
|
+
*
|
|
1630
|
+
* Note: Autoclass is a bucket-level feature that requires bucket.update permission.
|
|
1631
|
+
* It cannot be enabled per-object or per-prefix.
|
|
1632
|
+
*
|
|
1633
|
+
* @param options Autoclass configuration
|
|
1634
|
+
*/
|
|
1635
|
+
async enableAutoclass(options = {}) {
|
|
1636
|
+
await this.ensureInitialized();
|
|
1637
|
+
try {
|
|
1638
|
+
this.logger.info('Enabling GCS Autoclass');
|
|
1639
|
+
const autoclassConfig = {
|
|
1640
|
+
enabled: true
|
|
1641
|
+
};
|
|
1642
|
+
// Set terminal storage class if specified
|
|
1643
|
+
if (options.terminalStorageClass) {
|
|
1644
|
+
autoclassConfig.terminalStorageClass = options.terminalStorageClass;
|
|
1645
|
+
}
|
|
1646
|
+
await this.bucket.setMetadata({
|
|
1647
|
+
autoclass: autoclassConfig
|
|
1648
|
+
});
|
|
1649
|
+
this.logger.info(`Successfully enabled Autoclass${options.terminalStorageClass ? ` with terminal class ${options.terminalStorageClass}` : ''}`);
|
|
1650
|
+
}
|
|
1651
|
+
catch (error) {
|
|
1652
|
+
this.logger.error('Failed to enable Autoclass:', error);
|
|
1653
|
+
throw new Error(`Failed to enable GCS Autoclass: ${error.message || error}`);
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Get Autoclass configuration and status
|
|
1658
|
+
*
|
|
1659
|
+
* @returns Autoclass status, or null if not configured
|
|
1660
|
+
*/
|
|
1661
|
+
async getAutoclassStatus() {
|
|
1662
|
+
await this.ensureInitialized();
|
|
1663
|
+
try {
|
|
1664
|
+
this.logger.info('Getting GCS Autoclass status');
|
|
1665
|
+
const [metadata] = await this.bucket.getMetadata();
|
|
1666
|
+
if (!metadata.autoclass) {
|
|
1667
|
+
this.logger.info('Autoclass not configured');
|
|
1668
|
+
return null;
|
|
1669
|
+
}
|
|
1670
|
+
const status = {
|
|
1671
|
+
enabled: metadata.autoclass.enabled || false,
|
|
1672
|
+
...(metadata.autoclass.terminalStorageClass && {
|
|
1673
|
+
terminalStorageClass: metadata.autoclass.terminalStorageClass
|
|
1674
|
+
}),
|
|
1675
|
+
...(metadata.autoclass.toggleTime && {
|
|
1676
|
+
toggleTime: metadata.autoclass.toggleTime
|
|
1677
|
+
})
|
|
1678
|
+
};
|
|
1679
|
+
this.logger.info(`Autoclass status: ${status.enabled ? 'enabled' : 'disabled'}`);
|
|
1680
|
+
return status;
|
|
1681
|
+
}
|
|
1682
|
+
catch (error) {
|
|
1683
|
+
this.logger.error('Failed to get Autoclass status:', error);
|
|
1684
|
+
throw new Error(`Failed to get GCS Autoclass status: ${error.message || error}`);
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
/**
|
|
1688
|
+
* Disable Autoclass for the bucket
|
|
1689
|
+
*/
|
|
1690
|
+
async disableAutoclass() {
|
|
1691
|
+
await this.ensureInitialized();
|
|
1692
|
+
try {
|
|
1693
|
+
this.logger.info('Disabling GCS Autoclass');
|
|
1694
|
+
await this.bucket.setMetadata({
|
|
1695
|
+
autoclass: {
|
|
1696
|
+
enabled: false
|
|
1697
|
+
}
|
|
1698
|
+
});
|
|
1699
|
+
this.logger.info('Successfully disabled Autoclass');
|
|
1700
|
+
}
|
|
1701
|
+
catch (error) {
|
|
1702
|
+
this.logger.error('Failed to disable Autoclass:', error);
|
|
1703
|
+
throw new Error(`Failed to disable GCS Autoclass: ${error.message || error}`);
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1451
1706
|
}
|
|
1452
1707
|
//# sourceMappingURL=gcsStorage.js.map
|