@soulcraft/brainy 3.50.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +201 -0
- package/README.md +358 -658
- package/dist/api/ConfigAPI.js +56 -19
- package/dist/api/DataAPI.js +24 -18
- package/dist/augmentations/storageAugmentations.d.ts +24 -0
- package/dist/augmentations/storageAugmentations.js +22 -0
- package/dist/brainy.js +32 -9
- package/dist/cli/commands/core.d.ts +20 -10
- package/dist/cli/commands/core.js +384 -82
- package/dist/cli/commands/import.d.ts +41 -0
- package/dist/cli/commands/import.js +456 -0
- package/dist/cli/commands/insights.d.ts +34 -0
- package/dist/cli/commands/insights.js +300 -0
- package/dist/cli/commands/neural.d.ts +6 -12
- package/dist/cli/commands/neural.js +113 -10
- package/dist/cli/commands/nlp.d.ts +28 -0
- package/dist/cli/commands/nlp.js +246 -0
- package/dist/cli/commands/storage.d.ts +64 -0
- package/dist/cli/commands/storage.js +730 -0
- package/dist/cli/index.js +210 -24
- package/dist/coreTypes.d.ts +206 -34
- package/dist/distributed/configManager.js +8 -6
- package/dist/distributed/shardMigration.js +2 -0
- package/dist/distributed/storageDiscovery.js +6 -4
- package/dist/embeddings/EmbeddingManager.d.ts +2 -2
- package/dist/embeddings/EmbeddingManager.js +5 -1
- package/dist/graph/lsm/LSMTree.js +32 -20
- package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
- package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
- package/dist/storage/adapters/azureBlobStorage.js +1809 -0
- package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
- package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
- package/dist/storage/adapters/fileSystemStorage.js +204 -127
- package/dist/storage/adapters/gcsStorage.d.ts +119 -9
- package/dist/storage/adapters/gcsStorage.js +317 -62
- package/dist/storage/adapters/memoryStorage.d.ts +30 -18
- package/dist/storage/adapters/memoryStorage.js +99 -94
- package/dist/storage/adapters/opfsStorage.d.ts +48 -10
- package/dist/storage/adapters/opfsStorage.js +201 -80
- package/dist/storage/adapters/r2Storage.d.ts +12 -5
- package/dist/storage/adapters/r2Storage.js +63 -15
- package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
- package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
- package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
- package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
- package/dist/storage/baseStorage.d.ts +41 -38
- package/dist/storage/baseStorage.js +110 -134
- package/dist/storage/storageFactory.d.ts +29 -2
- package/dist/storage/storageFactory.js +30 -1
- package/dist/utils/entityIdMapper.js +5 -2
- package/dist/utils/fieldTypeInference.js +8 -1
- package/dist/utils/metadataFilter.d.ts +3 -2
- package/dist/utils/metadataFilter.js +1 -0
- package/dist/utils/metadataIndex.js +2 -0
- package/dist/utils/metadataIndexChunking.js +9 -4
- package/dist/utils/periodicCleanup.js +1 -0
- package/package.json +3 -1
|
@@ -748,15 +748,15 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
748
748
|
ContentType: 'application/json'
|
|
749
749
|
}));
|
|
750
750
|
this.logger.debug(`Node ${node.id} saved successfully`);
|
|
751
|
-
// Log the change for efficient synchronization
|
|
751
|
+
// Log the change for efficient synchronization (v4.0.0: no metadata on node)
|
|
752
752
|
await this.appendToChangeLog({
|
|
753
753
|
timestamp: Date.now(),
|
|
754
754
|
operation: 'add', // Could be 'update' if we track existing nodes
|
|
755
755
|
entityType: 'noun',
|
|
756
756
|
entityId: node.id,
|
|
757
757
|
data: {
|
|
758
|
-
vector: node.vector
|
|
759
|
-
metadata
|
|
758
|
+
vector: node.vector
|
|
759
|
+
// ✅ NO metadata field in v4.0.0 - stored separately
|
|
760
760
|
}
|
|
761
761
|
});
|
|
762
762
|
// Verify the node was saved by trying to retrieve it
|
|
@@ -795,21 +795,17 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
795
795
|
}
|
|
796
796
|
/**
|
|
797
797
|
* Get a noun from storage (internal implementation)
|
|
798
|
-
*
|
|
798
|
+
* v4.0.0: Returns ONLY vector data (no metadata field)
|
|
799
|
+
* Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
|
|
799
800
|
*/
|
|
800
801
|
async getNoun_internal(id) {
|
|
801
|
-
//
|
|
802
|
+
// v4.0.0: Return ONLY vector data (no metadata field)
|
|
802
803
|
const node = await this.getNode(id);
|
|
803
804
|
if (!node) {
|
|
804
805
|
return null;
|
|
805
806
|
}
|
|
806
|
-
//
|
|
807
|
-
|
|
808
|
-
// Combine into complete noun object
|
|
809
|
-
return {
|
|
810
|
-
...node,
|
|
811
|
-
metadata: metadata || {}
|
|
812
|
-
};
|
|
807
|
+
// Return pure vector structure
|
|
808
|
+
return node;
|
|
813
809
|
}
|
|
814
810
|
/**
|
|
815
811
|
* Get a node from storage
|
|
@@ -1228,21 +1224,17 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1228
1224
|
}
|
|
1229
1225
|
/**
|
|
1230
1226
|
* Get a verb from storage (internal implementation)
|
|
1231
|
-
*
|
|
1227
|
+
* v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
|
|
1228
|
+
* Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
|
|
1232
1229
|
*/
|
|
1233
1230
|
async getVerb_internal(id) {
|
|
1234
|
-
//
|
|
1231
|
+
// v4.0.0: Return ONLY vector + core relational data (no metadata field)
|
|
1235
1232
|
const edge = await this.getEdge(id);
|
|
1236
1233
|
if (!edge) {
|
|
1237
1234
|
return null;
|
|
1238
1235
|
}
|
|
1239
|
-
//
|
|
1240
|
-
|
|
1241
|
-
// Combine into complete verb object
|
|
1242
|
-
return {
|
|
1243
|
-
...edge,
|
|
1244
|
-
metadata: metadata || {}
|
|
1245
|
-
};
|
|
1236
|
+
// Return pure vector + core fields structure
|
|
1237
|
+
return edge;
|
|
1246
1238
|
}
|
|
1247
1239
|
/**
|
|
1248
1240
|
* Get an edge from storage
|
|
@@ -1284,7 +1276,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1284
1276
|
for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
|
|
1285
1277
|
connections.set(Number(level), new Set(nodeIds));
|
|
1286
1278
|
}
|
|
1287
|
-
//
|
|
1279
|
+
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
|
|
1288
1280
|
const edge = {
|
|
1289
1281
|
id: parsedEdge.id,
|
|
1290
1282
|
vector: parsedEdge.vector,
|
|
@@ -1292,9 +1284,9 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1292
1284
|
// CORE RELATIONAL DATA (read from vector file)
|
|
1293
1285
|
verb: parsedEdge.verb,
|
|
1294
1286
|
sourceId: parsedEdge.sourceId,
|
|
1295
|
-
targetId: parsedEdge.targetId
|
|
1296
|
-
//
|
|
1297
|
-
metadata
|
|
1287
|
+
targetId: parsedEdge.targetId
|
|
1288
|
+
// ✅ NO metadata field in v4.0.0
|
|
1289
|
+
// User metadata retrieved separately via getVerbMetadata()
|
|
1298
1290
|
};
|
|
1299
1291
|
this.logger.trace(`Successfully retrieved edge ${id}`);
|
|
1300
1292
|
return edge;
|
|
@@ -1483,24 +1475,32 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1483
1475
|
useCache: true,
|
|
1484
1476
|
filter: edgeFilter
|
|
1485
1477
|
});
|
|
1486
|
-
// Convert HNSWVerbs to
|
|
1487
|
-
const
|
|
1478
|
+
// v4.0.0: Convert HNSWVerbs to HNSWVerbWithMetadata by combining with metadata
|
|
1479
|
+
const verbsWithMetadata = [];
|
|
1488
1480
|
for (const hnswVerb of result.edges) {
|
|
1489
|
-
const
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1481
|
+
const metadata = await this.getVerbMetadata(hnswVerb.id);
|
|
1482
|
+
const verbWithMetadata = {
|
|
1483
|
+
id: hnswVerb.id,
|
|
1484
|
+
vector: [...hnswVerb.vector],
|
|
1485
|
+
connections: new Map(hnswVerb.connections),
|
|
1486
|
+
verb: hnswVerb.verb,
|
|
1487
|
+
sourceId: hnswVerb.sourceId,
|
|
1488
|
+
targetId: hnswVerb.targetId,
|
|
1489
|
+
metadata: metadata || {}
|
|
1490
|
+
};
|
|
1491
|
+
verbsWithMetadata.push(verbWithMetadata);
|
|
1493
1492
|
}
|
|
1494
|
-
// Apply filtering at
|
|
1495
|
-
|
|
1493
|
+
// Apply filtering at HNSWVerbWithMetadata level
|
|
1494
|
+
// v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb, not metadata
|
|
1495
|
+
let filteredVerbs = verbsWithMetadata;
|
|
1496
1496
|
if (options.filter) {
|
|
1497
|
-
|
|
1497
|
+
filteredVerbs = verbsWithMetadata.filter((verbWithMetadata) => {
|
|
1498
1498
|
// Filter by sourceId
|
|
1499
1499
|
if (options.filter.sourceId) {
|
|
1500
1500
|
const sourceIds = Array.isArray(options.filter.sourceId)
|
|
1501
1501
|
? options.filter.sourceId
|
|
1502
1502
|
: [options.filter.sourceId];
|
|
1503
|
-
if (!sourceIds.includes(
|
|
1503
|
+
if (!verbWithMetadata.sourceId || !sourceIds.includes(verbWithMetadata.sourceId)) {
|
|
1504
1504
|
return false;
|
|
1505
1505
|
}
|
|
1506
1506
|
}
|
|
@@ -1509,16 +1509,16 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1509
1509
|
const targetIds = Array.isArray(options.filter.targetId)
|
|
1510
1510
|
? options.filter.targetId
|
|
1511
1511
|
: [options.filter.targetId];
|
|
1512
|
-
if (!targetIds.includes(
|
|
1512
|
+
if (!verbWithMetadata.targetId || !targetIds.includes(verbWithMetadata.targetId)) {
|
|
1513
1513
|
return false;
|
|
1514
1514
|
}
|
|
1515
1515
|
}
|
|
1516
|
-
// Filter by verbType
|
|
1516
|
+
// Filter by verbType
|
|
1517
1517
|
if (options.filter.verbType) {
|
|
1518
1518
|
const verbTypes = Array.isArray(options.filter.verbType)
|
|
1519
1519
|
? options.filter.verbType
|
|
1520
1520
|
: [options.filter.verbType];
|
|
1521
|
-
if (
|
|
1521
|
+
if (!verbWithMetadata.verb || !verbTypes.includes(verbWithMetadata.verb)) {
|
|
1522
1522
|
return false;
|
|
1523
1523
|
}
|
|
1524
1524
|
}
|
|
@@ -1526,7 +1526,7 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1526
1526
|
});
|
|
1527
1527
|
}
|
|
1528
1528
|
return {
|
|
1529
|
-
items:
|
|
1529
|
+
items: filteredVerbs,
|
|
1530
1530
|
totalCount: this.totalVerbCount, // Use pre-calculated count from init()
|
|
1531
1531
|
hasMore: result.hasMore,
|
|
1532
1532
|
nextCursor: result.nextCursor
|
|
@@ -1700,6 +1700,127 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
1700
1700
|
throw new Error(`Failed to delete object from ${path}: ${error}`);
|
|
1701
1701
|
}
|
|
1702
1702
|
}
|
|
1703
|
+
/**
|
|
1704
|
+
* Batch delete multiple objects from S3-compatible storage
|
|
1705
|
+
* Deletes up to 1000 objects per batch (S3 limit)
|
|
1706
|
+
* Handles throttling, retries, and partial failures
|
|
1707
|
+
*
|
|
1708
|
+
* @param keys - Array of object keys (paths) to delete
|
|
1709
|
+
* @param options - Configuration options for batch deletion
|
|
1710
|
+
* @returns Statistics about successful and failed deletions
|
|
1711
|
+
*/
|
|
1712
|
+
async batchDelete(keys, options = {}) {
|
|
1713
|
+
await this.ensureInitialized();
|
|
1714
|
+
const { maxRetries = 3, retryDelayMs = 1000, continueOnError = true } = options;
|
|
1715
|
+
if (!keys || keys.length === 0) {
|
|
1716
|
+
return {
|
|
1717
|
+
totalRequested: 0,
|
|
1718
|
+
successfulDeletes: 0,
|
|
1719
|
+
failedDeletes: 0,
|
|
1720
|
+
errors: []
|
|
1721
|
+
};
|
|
1722
|
+
}
|
|
1723
|
+
this.logger.info(`Starting batch delete of ${keys.length} objects`);
|
|
1724
|
+
const stats = {
|
|
1725
|
+
totalRequested: keys.length,
|
|
1726
|
+
successfulDeletes: 0,
|
|
1727
|
+
failedDeletes: 0,
|
|
1728
|
+
errors: []
|
|
1729
|
+
};
|
|
1730
|
+
// Chunk keys into batches of max 1000 (S3 limit)
|
|
1731
|
+
const MAX_BATCH_SIZE = 1000;
|
|
1732
|
+
const batches = [];
|
|
1733
|
+
for (let i = 0; i < keys.length; i += MAX_BATCH_SIZE) {
|
|
1734
|
+
batches.push(keys.slice(i, i + MAX_BATCH_SIZE));
|
|
1735
|
+
}
|
|
1736
|
+
this.logger.debug(`Split ${keys.length} keys into ${batches.length} batches`);
|
|
1737
|
+
// Process each batch
|
|
1738
|
+
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
|
1739
|
+
const batch = batches[batchIndex];
|
|
1740
|
+
let retryCount = 0;
|
|
1741
|
+
let batchSuccess = false;
|
|
1742
|
+
while (retryCount <= maxRetries && !batchSuccess) {
|
|
1743
|
+
try {
|
|
1744
|
+
const { DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
|
|
1745
|
+
this.logger.debug(`Processing batch ${batchIndex + 1}/${batches.length} with ${batch.length} keys (attempt ${retryCount + 1}/${maxRetries + 1})`);
|
|
1746
|
+
// Execute batch delete
|
|
1747
|
+
const response = await this.s3Client.send(new DeleteObjectsCommand({
|
|
1748
|
+
Bucket: this.bucketName,
|
|
1749
|
+
Delete: {
|
|
1750
|
+
Objects: batch.map((key) => ({ Key: key })),
|
|
1751
|
+
Quiet: false // Get detailed response about each deletion
|
|
1752
|
+
}
|
|
1753
|
+
}));
|
|
1754
|
+
// Count successful deletions
|
|
1755
|
+
const deleted = response.Deleted || [];
|
|
1756
|
+
stats.successfulDeletes += deleted.length;
|
|
1757
|
+
this.logger.debug(`Batch ${batchIndex + 1} completed: ${deleted.length} deleted`);
|
|
1758
|
+
// Handle errors from S3 (partial failures)
|
|
1759
|
+
if (response.Errors && response.Errors.length > 0) {
|
|
1760
|
+
this.logger.warn(`Batch ${batchIndex + 1} had ${response.Errors.length} partial failures`);
|
|
1761
|
+
for (const error of response.Errors) {
|
|
1762
|
+
const errorKey = error.Key || 'unknown';
|
|
1763
|
+
const errorCode = error.Code || 'UnknownError';
|
|
1764
|
+
const errorMessage = error.Message || 'No error message';
|
|
1765
|
+
// Skip NoSuchKey errors (already deleted)
|
|
1766
|
+
if (errorCode === 'NoSuchKey') {
|
|
1767
|
+
this.logger.trace(`Object ${errorKey} already deleted (NoSuchKey)`);
|
|
1768
|
+
stats.successfulDeletes++;
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
stats.failedDeletes++;
|
|
1772
|
+
stats.errors.push({
|
|
1773
|
+
key: errorKey,
|
|
1774
|
+
error: `${errorCode}: ${errorMessage}`
|
|
1775
|
+
});
|
|
1776
|
+
this.logger.error(`Failed to delete ${errorKey}: ${errorCode} - ${errorMessage}`);
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
batchSuccess = true;
|
|
1780
|
+
}
|
|
1781
|
+
catch (error) {
|
|
1782
|
+
// Handle throttling
|
|
1783
|
+
if (this.isThrottlingError(error)) {
|
|
1784
|
+
this.logger.warn(`Batch ${batchIndex + 1} throttled, waiting before retry...`);
|
|
1785
|
+
await this.handleThrottling(error);
|
|
1786
|
+
retryCount++;
|
|
1787
|
+
if (retryCount <= maxRetries) {
|
|
1788
|
+
const delay = retryDelayMs * Math.pow(2, retryCount - 1); // Exponential backoff
|
|
1789
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1790
|
+
}
|
|
1791
|
+
continue;
|
|
1792
|
+
}
|
|
1793
|
+
// Handle other errors
|
|
1794
|
+
this.logger.error(`Batch ${batchIndex + 1} failed (attempt ${retryCount + 1}/${maxRetries + 1}):`, error);
|
|
1795
|
+
if (retryCount < maxRetries) {
|
|
1796
|
+
retryCount++;
|
|
1797
|
+
const delay = retryDelayMs * Math.pow(2, retryCount - 1);
|
|
1798
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1799
|
+
continue;
|
|
1800
|
+
}
|
|
1801
|
+
// Max retries exceeded
|
|
1802
|
+
if (continueOnError) {
|
|
1803
|
+
// Mark all keys in this batch as failed and continue to next batch
|
|
1804
|
+
for (const key of batch) {
|
|
1805
|
+
stats.failedDeletes++;
|
|
1806
|
+
stats.errors.push({
|
|
1807
|
+
key,
|
|
1808
|
+
error: error.message || String(error)
|
|
1809
|
+
});
|
|
1810
|
+
}
|
|
1811
|
+
this.logger.error(`Batch ${batchIndex + 1} failed after ${maxRetries} retries, continuing to next batch`);
|
|
1812
|
+
batchSuccess = true; // Mark as "handled" to move to next batch
|
|
1813
|
+
}
|
|
1814
|
+
else {
|
|
1815
|
+
// Stop processing and throw error
|
|
1816
|
+
throw BrainyError.storage(`Batch delete failed at batch ${batchIndex + 1}/${batches.length} after ${maxRetries} retries. Total: ${stats.successfulDeletes} deleted, ${stats.failedDeletes} failed`, error instanceof Error ? error : undefined);
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
this.logger.info(`Batch delete completed: ${stats.successfulDeletes}/${stats.totalRequested} successful, ${stats.failedDeletes} failed`);
|
|
1822
|
+
return stats;
|
|
1823
|
+
}
|
|
1703
1824
|
/**
|
|
1704
1825
|
* Primitive operation: List objects under path prefix
|
|
1705
1826
|
* All metadata operations use this internally via base class routing
|
|
@@ -2417,7 +2538,15 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
2417
2538
|
const entry = JSON.parse(entryData);
|
|
2418
2539
|
// Only include entries newer than the specified timestamp
|
|
2419
2540
|
if (entry.timestamp > sinceTimestamp) {
|
|
2420
|
-
|
|
2541
|
+
// Convert ChangeLogEntry to Change
|
|
2542
|
+
const change = {
|
|
2543
|
+
id: entry.entityId,
|
|
2544
|
+
type: entry.entityType === 'metadata' ? 'noun' : entry.entityType,
|
|
2545
|
+
operation: entry.operation === 'add' ? 'create' : entry.operation,
|
|
2546
|
+
timestamp: entry.timestamp,
|
|
2547
|
+
data: entry.data
|
|
2548
|
+
};
|
|
2549
|
+
changes.push(change);
|
|
2421
2550
|
}
|
|
2422
2551
|
}
|
|
2423
2552
|
}
|
|
@@ -2735,55 +2864,54 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
2735
2864
|
cursor,
|
|
2736
2865
|
useCache: true
|
|
2737
2866
|
});
|
|
2738
|
-
//
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
if (
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
const
|
|
2749
|
-
|
|
2750
|
-
|
|
2867
|
+
// v4.0.0: Combine nodes with metadata to create HNSWNounWithMetadata[]
|
|
2868
|
+
const nounsWithMetadata = [];
|
|
2869
|
+
for (const node of result.nodes) {
|
|
2870
|
+
const metadata = await this.getNounMetadata(node.id);
|
|
2871
|
+
if (!metadata)
|
|
2872
|
+
continue;
|
|
2873
|
+
// Apply filters if provided
|
|
2874
|
+
if (options.filter) {
|
|
2875
|
+
// Filter by noun type
|
|
2876
|
+
if (options.filter.nounType) {
|
|
2877
|
+
const nounTypes = Array.isArray(options.filter.nounType)
|
|
2878
|
+
? options.filter.nounType
|
|
2879
|
+
: [options.filter.nounType];
|
|
2880
|
+
const nounType = (metadata.type || metadata.noun);
|
|
2881
|
+
if (!nounType || !nounTypes.includes(nounType)) {
|
|
2882
|
+
continue;
|
|
2751
2883
|
}
|
|
2752
2884
|
}
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
const filteredByService = [];
|
|
2761
|
-
for (const node of filteredNodes) {
|
|
2762
|
-
const metadata = await this.getNounMetadata(node.id);
|
|
2763
|
-
if (metadata && services.includes(metadata.service)) {
|
|
2764
|
-
filteredByService.push(node);
|
|
2885
|
+
// Filter by service
|
|
2886
|
+
if (options.filter.service) {
|
|
2887
|
+
const services = Array.isArray(options.filter.service)
|
|
2888
|
+
? options.filter.service
|
|
2889
|
+
: [options.filter.service];
|
|
2890
|
+
if (!metadata.service || !services.includes(metadata.service)) {
|
|
2891
|
+
continue;
|
|
2765
2892
|
}
|
|
2766
2893
|
}
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
|
|
2773
|
-
for (const node of filteredNodes) {
|
|
2774
|
-
const metadata = await this.getNounMetadata(node.id);
|
|
2775
|
-
if (metadata) {
|
|
2776
|
-
const matches = Object.entries(metadataFilter).every(([key, value]) => metadata[key] === value);
|
|
2777
|
-
if (matches) {
|
|
2778
|
-
filteredByMetadata.push(node);
|
|
2779
|
-
}
|
|
2894
|
+
// Filter by metadata fields
|
|
2895
|
+
if (options.filter.metadata) {
|
|
2896
|
+
const metadataFilter = options.filter.metadata;
|
|
2897
|
+
const matches = Object.entries(metadataFilter).every(([key, value]) => metadata[key] === value);
|
|
2898
|
+
if (!matches) {
|
|
2899
|
+
continue;
|
|
2780
2900
|
}
|
|
2781
2901
|
}
|
|
2782
|
-
filteredNodes = filteredByMetadata;
|
|
2783
2902
|
}
|
|
2903
|
+
// Create HNSWNounWithMetadata
|
|
2904
|
+
const nounWithMetadata = {
|
|
2905
|
+
id: node.id,
|
|
2906
|
+
vector: [...node.vector],
|
|
2907
|
+
connections: new Map(node.connections),
|
|
2908
|
+
level: node.level || 0,
|
|
2909
|
+
metadata: metadata
|
|
2910
|
+
};
|
|
2911
|
+
nounsWithMetadata.push(nounWithMetadata);
|
|
2784
2912
|
}
|
|
2785
2913
|
return {
|
|
2786
|
-
items:
|
|
2914
|
+
items: nounsWithMetadata,
|
|
2787
2915
|
totalCount: this.totalNounCount, // Use pre-calculated count from init()
|
|
2788
2916
|
hasMore: result.hasMore,
|
|
2789
2917
|
nextCursor: result.nextCursor
|
|
@@ -3023,5 +3151,269 @@ export class S3CompatibleStorage extends BaseStorage {
|
|
|
3023
3151
|
throw new Error(`Failed to get HNSW system data: ${error}`);
|
|
3024
3152
|
}
|
|
3025
3153
|
}
|
|
3154
|
+
/**
|
|
3155
|
+
* Set S3 lifecycle policy for automatic tier transitions and deletions (v4.0.0)
|
|
3156
|
+
* Automates cost optimization by moving old data to cheaper storage classes
|
|
3157
|
+
*
|
|
3158
|
+
* S3 Storage Classes:
|
|
3159
|
+
* - Standard: $0.023/GB/month - Frequent access
|
|
3160
|
+
* - Standard-IA: $0.0125/GB/month - Infrequent access (46% cheaper)
|
|
3161
|
+
* - Glacier Instant: $0.004/GB/month - Archive with instant retrieval (83% cheaper)
|
|
3162
|
+
* - Glacier Flexible: $0.0036/GB/month - Archive with 1-5 min retrieval (84% cheaper)
|
|
3163
|
+
* - Glacier Deep Archive: $0.00099/GB/month - Long-term archive (96% cheaper!)
|
|
3164
|
+
*
|
|
3165
|
+
* @param options - Lifecycle policy configuration
|
|
3166
|
+
* @returns Promise that resolves when policy is set
|
|
3167
|
+
*
|
|
3168
|
+
* @example
|
|
3169
|
+
* // Auto-archive old vectors for 96% cost savings
|
|
3170
|
+
* await storage.setLifecyclePolicy({
|
|
3171
|
+
* rules: [
|
|
3172
|
+
* {
|
|
3173
|
+
* id: 'archive-old-vectors',
|
|
3174
|
+
* prefix: 'entities/nouns/vectors/',
|
|
3175
|
+
* status: 'Enabled',
|
|
3176
|
+
* transitions: [
|
|
3177
|
+
* { days: 30, storageClass: 'STANDARD_IA' },
|
|
3178
|
+
* { days: 90, storageClass: 'GLACIER' },
|
|
3179
|
+
* { days: 365, storageClass: 'DEEP_ARCHIVE' }
|
|
3180
|
+
* ],
|
|
3181
|
+
* expiration: { days: 730 }
|
|
3182
|
+
* }
|
|
3183
|
+
* ]
|
|
3184
|
+
* })
|
|
3185
|
+
*/
|
|
3186
|
+
async setLifecyclePolicy(options) {
|
|
3187
|
+
await this.ensureInitialized();
|
|
3188
|
+
try {
|
|
3189
|
+
this.logger.info(`Setting S3 lifecycle policy with ${options.rules.length} rules`);
|
|
3190
|
+
const { PutBucketLifecycleConfigurationCommand } = await import('@aws-sdk/client-s3');
|
|
3191
|
+
// Format rules according to S3's expected structure
|
|
3192
|
+
const lifecycleRules = options.rules.map(rule => ({
|
|
3193
|
+
ID: rule.id,
|
|
3194
|
+
Status: rule.status,
|
|
3195
|
+
Filter: {
|
|
3196
|
+
Prefix: rule.prefix
|
|
3197
|
+
},
|
|
3198
|
+
...(rule.transitions && rule.transitions.length > 0 && {
|
|
3199
|
+
Transitions: rule.transitions.map(t => ({
|
|
3200
|
+
Days: t.days,
|
|
3201
|
+
StorageClass: t.storageClass
|
|
3202
|
+
}))
|
|
3203
|
+
}),
|
|
3204
|
+
...(rule.expiration && {
|
|
3205
|
+
Expiration: {
|
|
3206
|
+
Days: rule.expiration.days
|
|
3207
|
+
}
|
|
3208
|
+
})
|
|
3209
|
+
}));
|
|
3210
|
+
await this.s3Client.send(new PutBucketLifecycleConfigurationCommand({
|
|
3211
|
+
Bucket: this.bucketName,
|
|
3212
|
+
LifecycleConfiguration: {
|
|
3213
|
+
Rules: lifecycleRules
|
|
3214
|
+
}
|
|
3215
|
+
}));
|
|
3216
|
+
this.logger.info(`Successfully set lifecycle policy with ${options.rules.length} rules`);
|
|
3217
|
+
}
|
|
3218
|
+
catch (error) {
|
|
3219
|
+
this.logger.error('Failed to set lifecycle policy:', error);
|
|
3220
|
+
throw new Error(`Failed to set S3 lifecycle policy: ${error.message || error}`);
|
|
3221
|
+
}
|
|
3222
|
+
}
|
|
3223
|
+
/**
|
|
3224
|
+
* Get the current S3 lifecycle policy
|
|
3225
|
+
*
|
|
3226
|
+
* @returns Promise that resolves to the current policy or null if not set
|
|
3227
|
+
*
|
|
3228
|
+
* @example
|
|
3229
|
+
* const policy = await storage.getLifecyclePolicy()
|
|
3230
|
+
* if (policy) {
|
|
3231
|
+
* console.log(`Found ${policy.rules.length} lifecycle rules`)
|
|
3232
|
+
* }
|
|
3233
|
+
*/
|
|
3234
|
+
async getLifecyclePolicy() {
|
|
3235
|
+
await this.ensureInitialized();
|
|
3236
|
+
try {
|
|
3237
|
+
this.logger.info('Getting S3 lifecycle policy');
|
|
3238
|
+
const { GetBucketLifecycleConfigurationCommand } = await import('@aws-sdk/client-s3');
|
|
3239
|
+
const response = await this.s3Client.send(new GetBucketLifecycleConfigurationCommand({
|
|
3240
|
+
Bucket: this.bucketName
|
|
3241
|
+
}));
|
|
3242
|
+
if (!response.Rules || response.Rules.length === 0) {
|
|
3243
|
+
this.logger.info('No lifecycle policy configured');
|
|
3244
|
+
return null;
|
|
3245
|
+
}
|
|
3246
|
+
const rules = response.Rules.map((rule) => ({
|
|
3247
|
+
id: rule.ID || 'unnamed',
|
|
3248
|
+
prefix: rule.Filter?.Prefix || '',
|
|
3249
|
+
status: rule.Status || 'Disabled',
|
|
3250
|
+
...(rule.Transitions && rule.Transitions.length > 0 && {
|
|
3251
|
+
transitions: rule.Transitions.map((t) => ({
|
|
3252
|
+
days: t.Days || 0,
|
|
3253
|
+
storageClass: t.StorageClass || 'STANDARD'
|
|
3254
|
+
}))
|
|
3255
|
+
}),
|
|
3256
|
+
...(rule.Expiration && rule.Expiration.Days && {
|
|
3257
|
+
expiration: {
|
|
3258
|
+
days: rule.Expiration.Days
|
|
3259
|
+
}
|
|
3260
|
+
})
|
|
3261
|
+
}));
|
|
3262
|
+
this.logger.info(`Found lifecycle policy with ${rules.length} rules`);
|
|
3263
|
+
return { rules };
|
|
3264
|
+
}
|
|
3265
|
+
catch (error) {
|
|
3266
|
+
// NoSuchLifecycleConfiguration means no policy is set
|
|
3267
|
+
if (error.name === 'NoSuchLifecycleConfiguration' || error.message?.includes('NoSuchLifecycleConfiguration')) {
|
|
3268
|
+
this.logger.info('No lifecycle policy configured');
|
|
3269
|
+
return null;
|
|
3270
|
+
}
|
|
3271
|
+
this.logger.error('Failed to get lifecycle policy:', error);
|
|
3272
|
+
throw new Error(`Failed to get S3 lifecycle policy: ${error.message || error}`);
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
3275
|
+
/**
|
|
3276
|
+
* Remove the S3 lifecycle policy
|
|
3277
|
+
* All automatic tier transitions and deletions will stop
|
|
3278
|
+
*
|
|
3279
|
+
* @returns Promise that resolves when policy is removed
|
|
3280
|
+
*
|
|
3281
|
+
* @example
|
|
3282
|
+
* await storage.removeLifecyclePolicy()
|
|
3283
|
+
* console.log('Lifecycle policy removed - auto-archival disabled')
|
|
3284
|
+
*/
|
|
3285
|
+
async removeLifecyclePolicy() {
|
|
3286
|
+
await this.ensureInitialized();
|
|
3287
|
+
try {
|
|
3288
|
+
this.logger.info('Removing S3 lifecycle policy');
|
|
3289
|
+
const { DeleteBucketLifecycleCommand } = await import('@aws-sdk/client-s3');
|
|
3290
|
+
await this.s3Client.send(new DeleteBucketLifecycleCommand({
|
|
3291
|
+
Bucket: this.bucketName
|
|
3292
|
+
}));
|
|
3293
|
+
this.logger.info('Successfully removed lifecycle policy');
|
|
3294
|
+
}
|
|
3295
|
+
catch (error) {
|
|
3296
|
+
this.logger.error('Failed to remove lifecycle policy:', error);
|
|
3297
|
+
throw new Error(`Failed to remove S3 lifecycle policy: ${error.message || error}`);
|
|
3298
|
+
}
|
|
3299
|
+
}
|
|
3300
|
+
/**
|
|
3301
|
+
* Enable S3 Intelligent-Tiering for automatic cost optimization (v4.0.0)
|
|
3302
|
+
* Automatically moves objects between access tiers based on usage patterns
|
|
3303
|
+
*
|
|
3304
|
+
* Intelligent-Tiering automatically saves up to 95% on storage costs:
|
|
3305
|
+
* - Frequent Access: $0.023/GB (same as Standard)
|
|
3306
|
+
* - Infrequent Access: $0.0125/GB (after 30 days no access)
|
|
3307
|
+
* - Archive Instant Access: $0.004/GB (after 90 days no access)
|
|
3308
|
+
* - Archive Access: $0.0036/GB (after 180 days no access, optional)
|
|
3309
|
+
* - Deep Archive Access: $0.00099/GB (after 180 days no access, optional)
|
|
3310
|
+
*
|
|
3311
|
+
* No retrieval fees, no operational overhead, automatic optimization!
|
|
3312
|
+
*
|
|
3313
|
+
* @param prefix - Object prefix to apply Intelligent-Tiering (e.g., 'entities/nouns/vectors/')
|
|
3314
|
+
* @param configId - Configuration ID (default: 'brainy-intelligent-tiering')
|
|
3315
|
+
* @returns Promise that resolves when configuration is set
|
|
3316
|
+
*
|
|
3317
|
+
* @example
|
|
3318
|
+
* // Enable Intelligent-Tiering for all vectors
|
|
3319
|
+
* await storage.enableIntelligentTiering('entities/')
|
|
3320
|
+
*/
|
|
3321
|
+
async enableIntelligentTiering(prefix = '', configId = 'brainy-intelligent-tiering') {
|
|
3322
|
+
await this.ensureInitialized();
|
|
3323
|
+
try {
|
|
3324
|
+
this.logger.info(`Enabling S3 Intelligent-Tiering for prefix: ${prefix}`);
|
|
3325
|
+
const { PutBucketIntelligentTieringConfigurationCommand } = await import('@aws-sdk/client-s3');
|
|
3326
|
+
await this.s3Client.send(new PutBucketIntelligentTieringConfigurationCommand({
|
|
3327
|
+
Bucket: this.bucketName,
|
|
3328
|
+
Id: configId,
|
|
3329
|
+
IntelligentTieringConfiguration: {
|
|
3330
|
+
Id: configId,
|
|
3331
|
+
Status: 'Enabled',
|
|
3332
|
+
Filter: prefix ? {
|
|
3333
|
+
Prefix: prefix
|
|
3334
|
+
} : undefined,
|
|
3335
|
+
Tierings: [
|
|
3336
|
+
// Move to Archive Instant Access tier after 90 days
|
|
3337
|
+
{
|
|
3338
|
+
Days: 90,
|
|
3339
|
+
AccessTier: 'ARCHIVE_ACCESS'
|
|
3340
|
+
},
|
|
3341
|
+
// Move to Deep Archive Access tier after 180 days (optional, 96% savings!)
|
|
3342
|
+
{
|
|
3343
|
+
Days: 180,
|
|
3344
|
+
AccessTier: 'DEEP_ARCHIVE_ACCESS'
|
|
3345
|
+
}
|
|
3346
|
+
]
|
|
3347
|
+
}
|
|
3348
|
+
}));
|
|
3349
|
+
this.logger.info(`Successfully enabled Intelligent-Tiering for prefix: ${prefix}`);
|
|
3350
|
+
}
|
|
3351
|
+
catch (error) {
|
|
3352
|
+
this.logger.error('Failed to enable Intelligent-Tiering:', error);
|
|
3353
|
+
throw new Error(`Failed to enable S3 Intelligent-Tiering: ${error.message || error}`);
|
|
3354
|
+
}
|
|
3355
|
+
}
|
|
3356
|
+
/**
|
|
3357
|
+
* Get S3 Intelligent-Tiering configurations
|
|
3358
|
+
*
|
|
3359
|
+
* @returns Promise that resolves to array of configurations
|
|
3360
|
+
*
|
|
3361
|
+
* @example
|
|
3362
|
+
* const configs = await storage.getIntelligentTieringConfigs()
|
|
3363
|
+
* for (const config of configs) {
|
|
3364
|
+
* console.log(`Config: ${config.id}, Status: ${config.status}`)
|
|
3365
|
+
* }
|
|
3366
|
+
*/
|
|
3367
|
+
async getIntelligentTieringConfigs() {
|
|
3368
|
+
await this.ensureInitialized();
|
|
3369
|
+
try {
|
|
3370
|
+
this.logger.info('Getting S3 Intelligent-Tiering configurations');
|
|
3371
|
+
const { ListBucketIntelligentTieringConfigurationsCommand } = await import('@aws-sdk/client-s3');
|
|
3372
|
+
const response = await this.s3Client.send(new ListBucketIntelligentTieringConfigurationsCommand({
|
|
3373
|
+
Bucket: this.bucketName
|
|
3374
|
+
}));
|
|
3375
|
+
if (!response.IntelligentTieringConfigurationList || response.IntelligentTieringConfigurationList.length === 0) {
|
|
3376
|
+
this.logger.info('No Intelligent-Tiering configurations found');
|
|
3377
|
+
return [];
|
|
3378
|
+
}
|
|
3379
|
+
const configs = response.IntelligentTieringConfigurationList.map((config) => ({
|
|
3380
|
+
id: config.Id || 'unnamed',
|
|
3381
|
+
status: config.Status || 'Disabled',
|
|
3382
|
+
...(config.Filter?.Prefix && { prefix: config.Filter.Prefix })
|
|
3383
|
+
}));
|
|
3384
|
+
this.logger.info(`Found ${configs.length} Intelligent-Tiering configurations`);
|
|
3385
|
+
return configs;
|
|
3386
|
+
}
|
|
3387
|
+
catch (error) {
|
|
3388
|
+
this.logger.error('Failed to get Intelligent-Tiering configurations:', error);
|
|
3389
|
+
throw new Error(`Failed to get S3 Intelligent-Tiering configurations: ${error.message || error}`);
|
|
3390
|
+
}
|
|
3391
|
+
}
|
|
3392
|
+
/**
|
|
3393
|
+
* Disable S3 Intelligent-Tiering
|
|
3394
|
+
*
|
|
3395
|
+
* @param configId - Configuration ID to remove (default: 'brainy-intelligent-tiering')
|
|
3396
|
+
* @returns Promise that resolves when configuration is removed
|
|
3397
|
+
*
|
|
3398
|
+
* @example
|
|
3399
|
+
* await storage.disableIntelligentTiering()
|
|
3400
|
+
* console.log('Intelligent-Tiering disabled')
|
|
3401
|
+
*/
|
|
3402
|
+
async disableIntelligentTiering(configId = 'brainy-intelligent-tiering') {
|
|
3403
|
+
await this.ensureInitialized();
|
|
3404
|
+
try {
|
|
3405
|
+
this.logger.info(`Disabling S3 Intelligent-Tiering config: ${configId}`);
|
|
3406
|
+
const { DeleteBucketIntelligentTieringConfigurationCommand } = await import('@aws-sdk/client-s3');
|
|
3407
|
+
await this.s3Client.send(new DeleteBucketIntelligentTieringConfigurationCommand({
|
|
3408
|
+
Bucket: this.bucketName,
|
|
3409
|
+
Id: configId
|
|
3410
|
+
}));
|
|
3411
|
+
this.logger.info(`Successfully disabled Intelligent-Tiering config: ${configId}`);
|
|
3412
|
+
}
|
|
3413
|
+
catch (error) {
|
|
3414
|
+
this.logger.error('Failed to disable Intelligent-Tiering:', error);
|
|
3415
|
+
throw new Error(`Failed to disable S3 Intelligent-Tiering: ${error.message || error}`);
|
|
3416
|
+
}
|
|
3417
|
+
}
|
|
3026
3418
|
}
|
|
3027
3419
|
//# sourceMappingURL=s3CompatibleStorage.js.map
|