@soulcraft/brainy 3.50.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +201 -0
  2. package/README.md +358 -658
  3. package/dist/api/ConfigAPI.js +56 -19
  4. package/dist/api/DataAPI.js +24 -18
  5. package/dist/augmentations/storageAugmentations.d.ts +24 -0
  6. package/dist/augmentations/storageAugmentations.js +22 -0
  7. package/dist/brainy.js +32 -9
  8. package/dist/cli/commands/core.d.ts +20 -10
  9. package/dist/cli/commands/core.js +384 -82
  10. package/dist/cli/commands/import.d.ts +41 -0
  11. package/dist/cli/commands/import.js +456 -0
  12. package/dist/cli/commands/insights.d.ts +34 -0
  13. package/dist/cli/commands/insights.js +300 -0
  14. package/dist/cli/commands/neural.d.ts +6 -12
  15. package/dist/cli/commands/neural.js +113 -10
  16. package/dist/cli/commands/nlp.d.ts +28 -0
  17. package/dist/cli/commands/nlp.js +246 -0
  18. package/dist/cli/commands/storage.d.ts +64 -0
  19. package/dist/cli/commands/storage.js +730 -0
  20. package/dist/cli/index.js +210 -24
  21. package/dist/coreTypes.d.ts +206 -34
  22. package/dist/distributed/configManager.js +8 -6
  23. package/dist/distributed/shardMigration.js +2 -0
  24. package/dist/distributed/storageDiscovery.js +6 -4
  25. package/dist/embeddings/EmbeddingManager.d.ts +2 -2
  26. package/dist/embeddings/EmbeddingManager.js +5 -1
  27. package/dist/graph/lsm/LSMTree.js +32 -20
  28. package/dist/hnsw/typeAwareHNSWIndex.js +6 -2
  29. package/dist/storage/adapters/azureBlobStorage.d.ts +545 -0
  30. package/dist/storage/adapters/azureBlobStorage.js +1809 -0
  31. package/dist/storage/adapters/baseStorageAdapter.d.ts +16 -13
  32. package/dist/storage/adapters/fileSystemStorage.d.ts +21 -9
  33. package/dist/storage/adapters/fileSystemStorage.js +204 -127
  34. package/dist/storage/adapters/gcsStorage.d.ts +119 -9
  35. package/dist/storage/adapters/gcsStorage.js +317 -62
  36. package/dist/storage/adapters/memoryStorage.d.ts +30 -18
  37. package/dist/storage/adapters/memoryStorage.js +99 -94
  38. package/dist/storage/adapters/opfsStorage.d.ts +48 -10
  39. package/dist/storage/adapters/opfsStorage.js +201 -80
  40. package/dist/storage/adapters/r2Storage.d.ts +12 -5
  41. package/dist/storage/adapters/r2Storage.js +63 -15
  42. package/dist/storage/adapters/s3CompatibleStorage.d.ts +164 -17
  43. package/dist/storage/adapters/s3CompatibleStorage.js +472 -80
  44. package/dist/storage/adapters/typeAwareStorageAdapter.d.ts +38 -6
  45. package/dist/storage/adapters/typeAwareStorageAdapter.js +218 -39
  46. package/dist/storage/baseStorage.d.ts +41 -38
  47. package/dist/storage/baseStorage.js +110 -134
  48. package/dist/storage/storageFactory.d.ts +29 -2
  49. package/dist/storage/storageFactory.js +30 -1
  50. package/dist/utils/entityIdMapper.js +5 -2
  51. package/dist/utils/fieldTypeInference.js +8 -1
  52. package/dist/utils/metadataFilter.d.ts +3 -2
  53. package/dist/utils/metadataFilter.js +1 -0
  54. package/dist/utils/metadataIndex.js +2 -0
  55. package/dist/utils/metadataIndexChunking.js +9 -4
  56. package/dist/utils/periodicCleanup.js +1 -0
  57. package/package.json +3 -1
@@ -748,15 +748,15 @@ export class S3CompatibleStorage extends BaseStorage {
748
748
  ContentType: 'application/json'
749
749
  }));
750
750
  this.logger.debug(`Node ${node.id} saved successfully`);
751
- // Log the change for efficient synchronization
751
+ // Log the change for efficient synchronization (v4.0.0: no metadata on node)
752
752
  await this.appendToChangeLog({
753
753
  timestamp: Date.now(),
754
754
  operation: 'add', // Could be 'update' if we track existing nodes
755
755
  entityType: 'noun',
756
756
  entityId: node.id,
757
757
  data: {
758
- vector: node.vector,
759
- metadata: node.metadata
758
+ vector: node.vector
759
+ // ✅ NO metadata field in v4.0.0 - stored separately
760
760
  }
761
761
  });
762
762
  // Verify the node was saved by trying to retrieve it
@@ -795,21 +795,17 @@ export class S3CompatibleStorage extends BaseStorage {
795
795
  }
796
796
  /**
797
797
  * Get a noun from storage (internal implementation)
798
- * Combines vector data from getNode() with metadata from getNounMetadata()
798
+ * v4.0.0: Returns ONLY vector data (no metadata field)
799
+ * Base class combines with metadata via getNoun() -> HNSWNounWithMetadata
799
800
  */
800
801
  async getNoun_internal(id) {
801
- // Get vector data (lightweight)
802
+ // v4.0.0: Return ONLY vector data (no metadata field)
802
803
  const node = await this.getNode(id);
803
804
  if (!node) {
804
805
  return null;
805
806
  }
806
- // Get metadata (entity data in 2-file system)
807
- const metadata = await this.getNounMetadata(id);
808
- // Combine into complete noun object
809
- return {
810
- ...node,
811
- metadata: metadata || {}
812
- };
807
+ // Return pure vector structure
808
+ return node;
813
809
  }
814
810
  /**
815
811
  * Get a node from storage
@@ -1228,21 +1224,17 @@ export class S3CompatibleStorage extends BaseStorage {
1228
1224
  }
1229
1225
  /**
1230
1226
  * Get a verb from storage (internal implementation)
1231
- * Combines vector data from getEdge() with metadata from getVerbMetadata()
1227
+ * v4.0.0: Returns ONLY vector + core relational fields (no metadata field)
1228
+ * Base class combines with metadata via getVerb() -> HNSWVerbWithMetadata
1232
1229
  */
1233
1230
  async getVerb_internal(id) {
1234
- // Get vector data (lightweight)
1231
+ // v4.0.0: Return ONLY vector + core relational data (no metadata field)
1235
1232
  const edge = await this.getEdge(id);
1236
1233
  if (!edge) {
1237
1234
  return null;
1238
1235
  }
1239
- // Get metadata (relationship data in 2-file system)
1240
- const metadata = await this.getVerbMetadata(id);
1241
- // Combine into complete verb object
1242
- return {
1243
- ...edge,
1244
- metadata: metadata || {}
1245
- };
1236
+ // Return pure vector + core fields structure
1237
+ return edge;
1246
1238
  }
1247
1239
  /**
1248
1240
  * Get an edge from storage
@@ -1284,7 +1276,7 @@ export class S3CompatibleStorage extends BaseStorage {
1284
1276
  for (const [level, nodeIds] of Object.entries(parsedEdge.connections)) {
1285
1277
  connections.set(Number(level), new Set(nodeIds));
1286
1278
  }
1287
- // ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
1279
+ // v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
1288
1280
  const edge = {
1289
1281
  id: parsedEdge.id,
1290
1282
  vector: parsedEdge.vector,
@@ -1292,9 +1284,9 @@ export class S3CompatibleStorage extends BaseStorage {
1292
1284
  // CORE RELATIONAL DATA (read from vector file)
1293
1285
  verb: parsedEdge.verb,
1294
1286
  sourceId: parsedEdge.sourceId,
1295
- targetId: parsedEdge.targetId,
1296
- // User metadata (retrieved separately via getVerbMetadata())
1297
- metadata: parsedEdge.metadata
1287
+ targetId: parsedEdge.targetId
1288
+ // NO metadata field in v4.0.0
1289
+ // User metadata retrieved separately via getVerbMetadata()
1298
1290
  };
1299
1291
  this.logger.trace(`Successfully retrieved edge ${id}`);
1300
1292
  return edge;
@@ -1483,24 +1475,32 @@ export class S3CompatibleStorage extends BaseStorage {
1483
1475
  useCache: true,
1484
1476
  filter: edgeFilter
1485
1477
  });
1486
- // Convert HNSWVerbs to GraphVerbs by combining with metadata
1487
- const graphVerbs = [];
1478
+ // v4.0.0: Convert HNSWVerbs to HNSWVerbWithMetadata by combining with metadata
1479
+ const verbsWithMetadata = [];
1488
1480
  for (const hnswVerb of result.edges) {
1489
- const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb);
1490
- if (graphVerb) {
1491
- graphVerbs.push(graphVerb);
1492
- }
1481
+ const metadata = await this.getVerbMetadata(hnswVerb.id);
1482
+ const verbWithMetadata = {
1483
+ id: hnswVerb.id,
1484
+ vector: [...hnswVerb.vector],
1485
+ connections: new Map(hnswVerb.connections),
1486
+ verb: hnswVerb.verb,
1487
+ sourceId: hnswVerb.sourceId,
1488
+ targetId: hnswVerb.targetId,
1489
+ metadata: metadata || {}
1490
+ };
1491
+ verbsWithMetadata.push(verbWithMetadata);
1493
1492
  }
1494
- // Apply filtering at GraphVerb level since HNSWVerb filtering is not supported
1495
- let filteredGraphVerbs = graphVerbs;
1493
+ // Apply filtering at HNSWVerbWithMetadata level
1494
+ // v4.0.0: Core fields (verb, sourceId, targetId) are in HNSWVerb, not metadata
1495
+ let filteredVerbs = verbsWithMetadata;
1496
1496
  if (options.filter) {
1497
- filteredGraphVerbs = graphVerbs.filter((graphVerb) => {
1497
+ filteredVerbs = verbsWithMetadata.filter((verbWithMetadata) => {
1498
1498
  // Filter by sourceId
1499
1499
  if (options.filter.sourceId) {
1500
1500
  const sourceIds = Array.isArray(options.filter.sourceId)
1501
1501
  ? options.filter.sourceId
1502
1502
  : [options.filter.sourceId];
1503
- if (!sourceIds.includes(graphVerb.sourceId)) {
1503
+ if (!verbWithMetadata.sourceId || !sourceIds.includes(verbWithMetadata.sourceId)) {
1504
1504
  return false;
1505
1505
  }
1506
1506
  }
@@ -1509,16 +1509,16 @@ export class S3CompatibleStorage extends BaseStorage {
1509
1509
  const targetIds = Array.isArray(options.filter.targetId)
1510
1510
  ? options.filter.targetId
1511
1511
  : [options.filter.targetId];
1512
- if (!targetIds.includes(graphVerb.targetId)) {
1512
+ if (!verbWithMetadata.targetId || !targetIds.includes(verbWithMetadata.targetId)) {
1513
1513
  return false;
1514
1514
  }
1515
1515
  }
1516
- // Filter by verbType (maps to type field)
1516
+ // Filter by verbType
1517
1517
  if (options.filter.verbType) {
1518
1518
  const verbTypes = Array.isArray(options.filter.verbType)
1519
1519
  ? options.filter.verbType
1520
1520
  : [options.filter.verbType];
1521
- if (graphVerb.type && !verbTypes.includes(graphVerb.type)) {
1521
+ if (!verbWithMetadata.verb || !verbTypes.includes(verbWithMetadata.verb)) {
1522
1522
  return false;
1523
1523
  }
1524
1524
  }
@@ -1526,7 +1526,7 @@ export class S3CompatibleStorage extends BaseStorage {
1526
1526
  });
1527
1527
  }
1528
1528
  return {
1529
- items: filteredGraphVerbs,
1529
+ items: filteredVerbs,
1530
1530
  totalCount: this.totalVerbCount, // Use pre-calculated count from init()
1531
1531
  hasMore: result.hasMore,
1532
1532
  nextCursor: result.nextCursor
@@ -1700,6 +1700,127 @@ export class S3CompatibleStorage extends BaseStorage {
1700
1700
  throw new Error(`Failed to delete object from ${path}: ${error}`);
1701
1701
  }
1702
1702
  }
1703
+ /**
1704
+ * Batch delete multiple objects from S3-compatible storage
1705
+ * Deletes up to 1000 objects per batch (S3 limit)
1706
+ * Handles throttling, retries, and partial failures
1707
+ *
1708
+ * @param keys - Array of object keys (paths) to delete
1709
+ * @param options - Configuration options for batch deletion
1710
+ * @returns Statistics about successful and failed deletions
1711
+ */
1712
+ async batchDelete(keys, options = {}) {
1713
+ await this.ensureInitialized();
1714
+ const { maxRetries = 3, retryDelayMs = 1000, continueOnError = true } = options;
1715
+ if (!keys || keys.length === 0) {
1716
+ return {
1717
+ totalRequested: 0,
1718
+ successfulDeletes: 0,
1719
+ failedDeletes: 0,
1720
+ errors: []
1721
+ };
1722
+ }
1723
+ this.logger.info(`Starting batch delete of ${keys.length} objects`);
1724
+ const stats = {
1725
+ totalRequested: keys.length,
1726
+ successfulDeletes: 0,
1727
+ failedDeletes: 0,
1728
+ errors: []
1729
+ };
1730
+ // Chunk keys into batches of max 1000 (S3 limit)
1731
+ const MAX_BATCH_SIZE = 1000;
1732
+ const batches = [];
1733
+ for (let i = 0; i < keys.length; i += MAX_BATCH_SIZE) {
1734
+ batches.push(keys.slice(i, i + MAX_BATCH_SIZE));
1735
+ }
1736
+ this.logger.debug(`Split ${keys.length} keys into ${batches.length} batches`);
1737
+ // Process each batch
1738
+ for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
1739
+ const batch = batches[batchIndex];
1740
+ let retryCount = 0;
1741
+ let batchSuccess = false;
1742
+ while (retryCount <= maxRetries && !batchSuccess) {
1743
+ try {
1744
+ const { DeleteObjectsCommand } = await import('@aws-sdk/client-s3');
1745
+ this.logger.debug(`Processing batch ${batchIndex + 1}/${batches.length} with ${batch.length} keys (attempt ${retryCount + 1}/${maxRetries + 1})`);
1746
+ // Execute batch delete
1747
+ const response = await this.s3Client.send(new DeleteObjectsCommand({
1748
+ Bucket: this.bucketName,
1749
+ Delete: {
1750
+ Objects: batch.map((key) => ({ Key: key })),
1751
+ Quiet: false // Get detailed response about each deletion
1752
+ }
1753
+ }));
1754
+ // Count successful deletions
1755
+ const deleted = response.Deleted || [];
1756
+ stats.successfulDeletes += deleted.length;
1757
+ this.logger.debug(`Batch ${batchIndex + 1} completed: ${deleted.length} deleted`);
1758
+ // Handle errors from S3 (partial failures)
1759
+ if (response.Errors && response.Errors.length > 0) {
1760
+ this.logger.warn(`Batch ${batchIndex + 1} had ${response.Errors.length} partial failures`);
1761
+ for (const error of response.Errors) {
1762
+ const errorKey = error.Key || 'unknown';
1763
+ const errorCode = error.Code || 'UnknownError';
1764
+ const errorMessage = error.Message || 'No error message';
1765
+ // Skip NoSuchKey errors (already deleted)
1766
+ if (errorCode === 'NoSuchKey') {
1767
+ this.logger.trace(`Object ${errorKey} already deleted (NoSuchKey)`);
1768
+ stats.successfulDeletes++;
1769
+ continue;
1770
+ }
1771
+ stats.failedDeletes++;
1772
+ stats.errors.push({
1773
+ key: errorKey,
1774
+ error: `${errorCode}: ${errorMessage}`
1775
+ });
1776
+ this.logger.error(`Failed to delete ${errorKey}: ${errorCode} - ${errorMessage}`);
1777
+ }
1778
+ }
1779
+ batchSuccess = true;
1780
+ }
1781
+ catch (error) {
1782
+ // Handle throttling
1783
+ if (this.isThrottlingError(error)) {
1784
+ this.logger.warn(`Batch ${batchIndex + 1} throttled, waiting before retry...`);
1785
+ await this.handleThrottling(error);
1786
+ retryCount++;
1787
+ if (retryCount <= maxRetries) {
1788
+ const delay = retryDelayMs * Math.pow(2, retryCount - 1); // Exponential backoff
1789
+ await new Promise((resolve) => setTimeout(resolve, delay));
1790
+ }
1791
+ continue;
1792
+ }
1793
+ // Handle other errors
1794
+ this.logger.error(`Batch ${batchIndex + 1} failed (attempt ${retryCount + 1}/${maxRetries + 1}):`, error);
1795
+ if (retryCount < maxRetries) {
1796
+ retryCount++;
1797
+ const delay = retryDelayMs * Math.pow(2, retryCount - 1);
1798
+ await new Promise((resolve) => setTimeout(resolve, delay));
1799
+ continue;
1800
+ }
1801
+ // Max retries exceeded
1802
+ if (continueOnError) {
1803
+ // Mark all keys in this batch as failed and continue to next batch
1804
+ for (const key of batch) {
1805
+ stats.failedDeletes++;
1806
+ stats.errors.push({
1807
+ key,
1808
+ error: error.message || String(error)
1809
+ });
1810
+ }
1811
+ this.logger.error(`Batch ${batchIndex + 1} failed after ${maxRetries} retries, continuing to next batch`);
1812
+ batchSuccess = true; // Mark as "handled" to move to next batch
1813
+ }
1814
+ else {
1815
+ // Stop processing and throw error
1816
+ throw BrainyError.storage(`Batch delete failed at batch ${batchIndex + 1}/${batches.length} after ${maxRetries} retries. Total: ${stats.successfulDeletes} deleted, ${stats.failedDeletes} failed`, error instanceof Error ? error : undefined);
1817
+ }
1818
+ }
1819
+ }
1820
+ }
1821
+ this.logger.info(`Batch delete completed: ${stats.successfulDeletes}/${stats.totalRequested} successful, ${stats.failedDeletes} failed`);
1822
+ return stats;
1823
+ }
1703
1824
  /**
1704
1825
  * Primitive operation: List objects under path prefix
1705
1826
  * All metadata operations use this internally via base class routing
@@ -2417,7 +2538,15 @@ export class S3CompatibleStorage extends BaseStorage {
2417
2538
  const entry = JSON.parse(entryData);
2418
2539
  // Only include entries newer than the specified timestamp
2419
2540
  if (entry.timestamp > sinceTimestamp) {
2420
- changes.push(entry);
2541
+ // Convert ChangeLogEntry to Change
2542
+ const change = {
2543
+ id: entry.entityId,
2544
+ type: entry.entityType === 'metadata' ? 'noun' : entry.entityType,
2545
+ operation: entry.operation === 'add' ? 'create' : entry.operation,
2546
+ timestamp: entry.timestamp,
2547
+ data: entry.data
2548
+ };
2549
+ changes.push(change);
2421
2550
  }
2422
2551
  }
2423
2552
  }
@@ -2735,55 +2864,54 @@ export class S3CompatibleStorage extends BaseStorage {
2735
2864
  cursor,
2736
2865
  useCache: true
2737
2866
  });
2738
- // Apply filters if provided
2739
- let filteredNodes = result.nodes;
2740
- if (options.filter) {
2741
- // Filter by noun type
2742
- if (options.filter.nounType) {
2743
- const nounTypes = Array.isArray(options.filter.nounType)
2744
- ? options.filter.nounType
2745
- : [options.filter.nounType];
2746
- const filteredByType = [];
2747
- for (const node of filteredNodes) {
2748
- const metadata = await this.getNounMetadata(node.id);
2749
- if (metadata && nounTypes.includes(metadata.type || metadata.noun)) {
2750
- filteredByType.push(node);
2867
+ // v4.0.0: Combine nodes with metadata to create HNSWNounWithMetadata[]
2868
+ const nounsWithMetadata = [];
2869
+ for (const node of result.nodes) {
2870
+ const metadata = await this.getNounMetadata(node.id);
2871
+ if (!metadata)
2872
+ continue;
2873
+ // Apply filters if provided
2874
+ if (options.filter) {
2875
+ // Filter by noun type
2876
+ if (options.filter.nounType) {
2877
+ const nounTypes = Array.isArray(options.filter.nounType)
2878
+ ? options.filter.nounType
2879
+ : [options.filter.nounType];
2880
+ const nounType = (metadata.type || metadata.noun);
2881
+ if (!nounType || !nounTypes.includes(nounType)) {
2882
+ continue;
2751
2883
  }
2752
2884
  }
2753
- filteredNodes = filteredByType;
2754
- }
2755
- // Filter by service
2756
- if (options.filter.service) {
2757
- const services = Array.isArray(options.filter.service)
2758
- ? options.filter.service
2759
- : [options.filter.service];
2760
- const filteredByService = [];
2761
- for (const node of filteredNodes) {
2762
- const metadata = await this.getNounMetadata(node.id);
2763
- if (metadata && services.includes(metadata.service)) {
2764
- filteredByService.push(node);
2885
+ // Filter by service
2886
+ if (options.filter.service) {
2887
+ const services = Array.isArray(options.filter.service)
2888
+ ? options.filter.service
2889
+ : [options.filter.service];
2890
+ if (!metadata.service || !services.includes(metadata.service)) {
2891
+ continue;
2765
2892
  }
2766
2893
  }
2767
- filteredNodes = filteredByService;
2768
- }
2769
- // Filter by metadata
2770
- if (options.filter.metadata) {
2771
- const metadataFilter = options.filter.metadata;
2772
- const filteredByMetadata = [];
2773
- for (const node of filteredNodes) {
2774
- const metadata = await this.getNounMetadata(node.id);
2775
- if (metadata) {
2776
- const matches = Object.entries(metadataFilter).every(([key, value]) => metadata[key] === value);
2777
- if (matches) {
2778
- filteredByMetadata.push(node);
2779
- }
2894
+ // Filter by metadata fields
2895
+ if (options.filter.metadata) {
2896
+ const metadataFilter = options.filter.metadata;
2897
+ const matches = Object.entries(metadataFilter).every(([key, value]) => metadata[key] === value);
2898
+ if (!matches) {
2899
+ continue;
2780
2900
  }
2781
2901
  }
2782
- filteredNodes = filteredByMetadata;
2783
2902
  }
2903
+ // Create HNSWNounWithMetadata
2904
+ const nounWithMetadata = {
2905
+ id: node.id,
2906
+ vector: [...node.vector],
2907
+ connections: new Map(node.connections),
2908
+ level: node.level || 0,
2909
+ metadata: metadata
2910
+ };
2911
+ nounsWithMetadata.push(nounWithMetadata);
2784
2912
  }
2785
2913
  return {
2786
- items: filteredNodes,
2914
+ items: nounsWithMetadata,
2787
2915
  totalCount: this.totalNounCount, // Use pre-calculated count from init()
2788
2916
  hasMore: result.hasMore,
2789
2917
  nextCursor: result.nextCursor
@@ -3023,5 +3151,269 @@ export class S3CompatibleStorage extends BaseStorage {
3023
3151
  throw new Error(`Failed to get HNSW system data: ${error}`);
3024
3152
  }
3025
3153
  }
3154
+ /**
3155
+ * Set S3 lifecycle policy for automatic tier transitions and deletions (v4.0.0)
3156
+ * Automates cost optimization by moving old data to cheaper storage classes
3157
+ *
3158
+ * S3 Storage Classes:
3159
+ * - Standard: $0.023/GB/month - Frequent access
3160
+ * - Standard-IA: $0.0125/GB/month - Infrequent access (46% cheaper)
3161
+ * - Glacier Instant: $0.004/GB/month - Archive with instant retrieval (83% cheaper)
3162
+ * - Glacier Flexible: $0.0036/GB/month - Archive with 1-5 min retrieval (84% cheaper)
3163
+ * - Glacier Deep Archive: $0.00099/GB/month - Long-term archive (96% cheaper!)
3164
+ *
3165
+ * @param options - Lifecycle policy configuration
3166
+ * @returns Promise that resolves when policy is set
3167
+ *
3168
+ * @example
3169
+ * // Auto-archive old vectors for 96% cost savings
3170
+ * await storage.setLifecyclePolicy({
3171
+ * rules: [
3172
+ * {
3173
+ * id: 'archive-old-vectors',
3174
+ * prefix: 'entities/nouns/vectors/',
3175
+ * status: 'Enabled',
3176
+ * transitions: [
3177
+ * { days: 30, storageClass: 'STANDARD_IA' },
3178
+ * { days: 90, storageClass: 'GLACIER' },
3179
+ * { days: 365, storageClass: 'DEEP_ARCHIVE' }
3180
+ * ],
3181
+ * expiration: { days: 730 }
3182
+ * }
3183
+ * ]
3184
+ * })
3185
+ */
3186
+ async setLifecyclePolicy(options) {
3187
+ await this.ensureInitialized();
3188
+ try {
3189
+ this.logger.info(`Setting S3 lifecycle policy with ${options.rules.length} rules`);
3190
+ const { PutBucketLifecycleConfigurationCommand } = await import('@aws-sdk/client-s3');
3191
+ // Format rules according to S3's expected structure
3192
+ const lifecycleRules = options.rules.map(rule => ({
3193
+ ID: rule.id,
3194
+ Status: rule.status,
3195
+ Filter: {
3196
+ Prefix: rule.prefix
3197
+ },
3198
+ ...(rule.transitions && rule.transitions.length > 0 && {
3199
+ Transitions: rule.transitions.map(t => ({
3200
+ Days: t.days,
3201
+ StorageClass: t.storageClass
3202
+ }))
3203
+ }),
3204
+ ...(rule.expiration && {
3205
+ Expiration: {
3206
+ Days: rule.expiration.days
3207
+ }
3208
+ })
3209
+ }));
3210
+ await this.s3Client.send(new PutBucketLifecycleConfigurationCommand({
3211
+ Bucket: this.bucketName,
3212
+ LifecycleConfiguration: {
3213
+ Rules: lifecycleRules
3214
+ }
3215
+ }));
3216
+ this.logger.info(`Successfully set lifecycle policy with ${options.rules.length} rules`);
3217
+ }
3218
+ catch (error) {
3219
+ this.logger.error('Failed to set lifecycle policy:', error);
3220
+ throw new Error(`Failed to set S3 lifecycle policy: ${error.message || error}`);
3221
+ }
3222
+ }
3223
+ /**
3224
+ * Get the current S3 lifecycle policy
3225
+ *
3226
+ * @returns Promise that resolves to the current policy or null if not set
3227
+ *
3228
+ * @example
3229
+ * const policy = await storage.getLifecyclePolicy()
3230
+ * if (policy) {
3231
+ * console.log(`Found ${policy.rules.length} lifecycle rules`)
3232
+ * }
3233
+ */
3234
+ async getLifecyclePolicy() {
3235
+ await this.ensureInitialized();
3236
+ try {
3237
+ this.logger.info('Getting S3 lifecycle policy');
3238
+ const { GetBucketLifecycleConfigurationCommand } = await import('@aws-sdk/client-s3');
3239
+ const response = await this.s3Client.send(new GetBucketLifecycleConfigurationCommand({
3240
+ Bucket: this.bucketName
3241
+ }));
3242
+ if (!response.Rules || response.Rules.length === 0) {
3243
+ this.logger.info('No lifecycle policy configured');
3244
+ return null;
3245
+ }
3246
+ const rules = response.Rules.map((rule) => ({
3247
+ id: rule.ID || 'unnamed',
3248
+ prefix: rule.Filter?.Prefix || '',
3249
+ status: rule.Status || 'Disabled',
3250
+ ...(rule.Transitions && rule.Transitions.length > 0 && {
3251
+ transitions: rule.Transitions.map((t) => ({
3252
+ days: t.Days || 0,
3253
+ storageClass: t.StorageClass || 'STANDARD'
3254
+ }))
3255
+ }),
3256
+ ...(rule.Expiration && rule.Expiration.Days && {
3257
+ expiration: {
3258
+ days: rule.Expiration.Days
3259
+ }
3260
+ })
3261
+ }));
3262
+ this.logger.info(`Found lifecycle policy with ${rules.length} rules`);
3263
+ return { rules };
3264
+ }
3265
+ catch (error) {
3266
+ // NoSuchLifecycleConfiguration means no policy is set
3267
+ if (error.name === 'NoSuchLifecycleConfiguration' || error.message?.includes('NoSuchLifecycleConfiguration')) {
3268
+ this.logger.info('No lifecycle policy configured');
3269
+ return null;
3270
+ }
3271
+ this.logger.error('Failed to get lifecycle policy:', error);
3272
+ throw new Error(`Failed to get S3 lifecycle policy: ${error.message || error}`);
3273
+ }
3274
+ }
3275
+ /**
3276
+ * Remove the S3 lifecycle policy
3277
+ * All automatic tier transitions and deletions will stop
3278
+ *
3279
+ * @returns Promise that resolves when policy is removed
3280
+ *
3281
+ * @example
3282
+ * await storage.removeLifecyclePolicy()
3283
+ * console.log('Lifecycle policy removed - auto-archival disabled')
3284
+ */
3285
+ async removeLifecyclePolicy() {
3286
+ await this.ensureInitialized();
3287
+ try {
3288
+ this.logger.info('Removing S3 lifecycle policy');
3289
+ const { DeleteBucketLifecycleCommand } = await import('@aws-sdk/client-s3');
3290
+ await this.s3Client.send(new DeleteBucketLifecycleCommand({
3291
+ Bucket: this.bucketName
3292
+ }));
3293
+ this.logger.info('Successfully removed lifecycle policy');
3294
+ }
3295
+ catch (error) {
3296
+ this.logger.error('Failed to remove lifecycle policy:', error);
3297
+ throw new Error(`Failed to remove S3 lifecycle policy: ${error.message || error}`);
3298
+ }
3299
+ }
3300
+ /**
3301
+ * Enable S3 Intelligent-Tiering for automatic cost optimization (v4.0.0)
3302
+ * Automatically moves objects between access tiers based on usage patterns
3303
+ *
3304
+ * Intelligent-Tiering automatically saves up to 95% on storage costs:
3305
+ * - Frequent Access: $0.023/GB (same as Standard)
3306
+ * - Infrequent Access: $0.0125/GB (after 30 days no access)
3307
+ * - Archive Instant Access: $0.004/GB (after 90 days no access)
3308
+ * - Archive Access: $0.0036/GB (after 180 days no access, optional)
3309
+ * - Deep Archive Access: $0.00099/GB (after 180 days no access, optional)
3310
+ *
3311
+ * No retrieval fees, no operational overhead, automatic optimization!
3312
+ *
3313
+ * @param prefix - Object prefix to apply Intelligent-Tiering (e.g., 'entities/nouns/vectors/')
3314
+ * @param configId - Configuration ID (default: 'brainy-intelligent-tiering')
3315
+ * @returns Promise that resolves when configuration is set
3316
+ *
3317
+ * @example
3318
+ * // Enable Intelligent-Tiering for all vectors
3319
+ * await storage.enableIntelligentTiering('entities/')
3320
+ */
3321
+ async enableIntelligentTiering(prefix = '', configId = 'brainy-intelligent-tiering') {
3322
+ await this.ensureInitialized();
3323
+ try {
3324
+ this.logger.info(`Enabling S3 Intelligent-Tiering for prefix: ${prefix}`);
3325
+ const { PutBucketIntelligentTieringConfigurationCommand } = await import('@aws-sdk/client-s3');
3326
+ await this.s3Client.send(new PutBucketIntelligentTieringConfigurationCommand({
3327
+ Bucket: this.bucketName,
3328
+ Id: configId,
3329
+ IntelligentTieringConfiguration: {
3330
+ Id: configId,
3331
+ Status: 'Enabled',
3332
+ Filter: prefix ? {
3333
+ Prefix: prefix
3334
+ } : undefined,
3335
+ Tierings: [
3336
+ // Move to Archive Instant Access tier after 90 days
3337
+ {
3338
+ Days: 90,
3339
+ AccessTier: 'ARCHIVE_ACCESS'
3340
+ },
3341
+ // Move to Deep Archive Access tier after 180 days (optional, 96% savings!)
3342
+ {
3343
+ Days: 180,
3344
+ AccessTier: 'DEEP_ARCHIVE_ACCESS'
3345
+ }
3346
+ ]
3347
+ }
3348
+ }));
3349
+ this.logger.info(`Successfully enabled Intelligent-Tiering for prefix: ${prefix}`);
3350
+ }
3351
+ catch (error) {
3352
+ this.logger.error('Failed to enable Intelligent-Tiering:', error);
3353
+ throw new Error(`Failed to enable S3 Intelligent-Tiering: ${error.message || error}`);
3354
+ }
3355
+ }
3356
+ /**
3357
+ * Get S3 Intelligent-Tiering configurations
3358
+ *
3359
+ * @returns Promise that resolves to array of configurations
3360
+ *
3361
+ * @example
3362
+ * const configs = await storage.getIntelligentTieringConfigs()
3363
+ * for (const config of configs) {
3364
+ * console.log(`Config: ${config.id}, Status: ${config.status}`)
3365
+ * }
3366
+ */
3367
+ async getIntelligentTieringConfigs() {
3368
+ await this.ensureInitialized();
3369
+ try {
3370
+ this.logger.info('Getting S3 Intelligent-Tiering configurations');
3371
+ const { ListBucketIntelligentTieringConfigurationsCommand } = await import('@aws-sdk/client-s3');
3372
+ const response = await this.s3Client.send(new ListBucketIntelligentTieringConfigurationsCommand({
3373
+ Bucket: this.bucketName
3374
+ }));
3375
+ if (!response.IntelligentTieringConfigurationList || response.IntelligentTieringConfigurationList.length === 0) {
3376
+ this.logger.info('No Intelligent-Tiering configurations found');
3377
+ return [];
3378
+ }
3379
+ const configs = response.IntelligentTieringConfigurationList.map((config) => ({
3380
+ id: config.Id || 'unnamed',
3381
+ status: config.Status || 'Disabled',
3382
+ ...(config.Filter?.Prefix && { prefix: config.Filter.Prefix })
3383
+ }));
3384
+ this.logger.info(`Found ${configs.length} Intelligent-Tiering configurations`);
3385
+ return configs;
3386
+ }
3387
+ catch (error) {
3388
+ this.logger.error('Failed to get Intelligent-Tiering configurations:', error);
3389
+ throw new Error(`Failed to get S3 Intelligent-Tiering configurations: ${error.message || error}`);
3390
+ }
3391
+ }
3392
+ /**
3393
+ * Disable S3 Intelligent-Tiering
3394
+ *
3395
+ * @param configId - Configuration ID to remove (default: 'brainy-intelligent-tiering')
3396
+ * @returns Promise that resolves when configuration is removed
3397
+ *
3398
+ * @example
3399
+ * await storage.disableIntelligentTiering()
3400
+ * console.log('Intelligent-Tiering disabled')
3401
+ */
3402
+ async disableIntelligentTiering(configId = 'brainy-intelligent-tiering') {
3403
+ await this.ensureInitialized();
3404
+ try {
3405
+ this.logger.info(`Disabling S3 Intelligent-Tiering config: ${configId}`);
3406
+ const { DeleteBucketIntelligentTieringConfigurationCommand } = await import('@aws-sdk/client-s3');
3407
+ await this.s3Client.send(new DeleteBucketIntelligentTieringConfigurationCommand({
3408
+ Bucket: this.bucketName,
3409
+ Id: configId
3410
+ }));
3411
+ this.logger.info(`Successfully disabled Intelligent-Tiering config: ${configId}`);
3412
+ }
3413
+ catch (error) {
3414
+ this.logger.error('Failed to disable Intelligent-Tiering:', error);
3415
+ throw new Error(`Failed to disable S3 Intelligent-Tiering: ${error.message || error}`);
3416
+ }
3417
+ }
3026
3418
  }
3027
3419
  //# sourceMappingURL=s3CompatibleStorage.js.map