@mapbox/cloudfriend 9.2.1 → 9.3.2

package/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-custom.json

@@ -0,0 +1,75 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "OptimizerRole": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "AssumeRolePolicyDocument": {}
+      }
+    },
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    },
+    "MyTableRetentionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "retention",
+        "TableOptimizerConfiguration": {
+          "RoleArn": {
+            "Fn::GetAtt": [
+              "OptimizerRole",
+              "Arn"
+            ]
+          },
+          "Enabled": true,
+          "RetentionConfiguration": {
+            "IcebergConfiguration": {
+              "SnapshotRetentionPeriodInDays": 7,
+              "NumberOfSnapshotsToRetain": 3,
+              "CleanExpiredFiles": false
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}

package/test/fixtures/shortcuts/glue-iceberg-table-with-optimizer-defaults.json

@@ -0,0 +1,64 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    },
+    "MyTableRetentionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "retention",
+        "TableOptimizerConfiguration": {
+          "RoleArn": "arn:aws:iam::123456789012:role/OptimizerRole",
+          "Enabled": true,
+          "RetentionConfiguration": {
+            "IcebergConfiguration": {
+              "SnapshotRetentionPeriodInDays": 5,
+              "NumberOfSnapshotsToRetain": 1,
+              "CleanExpiredFiles": true
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}

package/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-custom.json

@@ -0,0 +1,74 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "OrphanFileDeletionRole": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "AssumeRolePolicyDocument": {}
+      }
+    },
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    },
+    "MyTableOrphanFileDeletionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "orphan_file_deletion",
+        "TableOptimizerConfiguration": {
+          "RoleArn": {
+            "Fn::GetAtt": [
+              "OrphanFileDeletionRole",
+              "Arn"
+            ]
+          },
+          "Enabled": true,
+          "OrphanFileDeletionConfiguration": {
+            "IcebergConfiguration": {
+              "OrphanFileRetentionPeriodInDays": 7,
+              "Location": "s3://fake/location/subdir"
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}

package/test/fixtures/shortcuts/glue-iceberg-table-with-orphan-deletion-defaults.json

@@ -0,0 +1,62 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    },
+    "MyTableOrphanFileDeletionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "orphan_file_deletion",
+        "TableOptimizerConfiguration": {
+          "RoleArn": "arn:aws:iam::123456789012:role/OrphanFileDeletionRole",
+          "Enabled": true,
+          "OrphanFileDeletionConfiguration": {
+            "IcebergConfiguration": {
+              "OrphanFileRetentionPeriodInDays": 3
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}

package/test/shortcuts.test.js

@@ -15,8 +15,13 @@ const noUndefined = (template) => JSON.parse(JSON.stringify(template));
 describe('[shortcuts] fixture validation', () => {
   // Runs cfn-lint, ignoring "warnings". Install via pip or Homebrew to run these
   // tests locally.
-  const cfnLint = (filepath) => new Promise((resolve, reject) => {
-    cp.exec(`cfn-lint ${filepath} --ignore-checks W`, (err, stdout) => {
+  const cfnLint = (filepath, filename) => new Promise((resolve, reject) => {
+    // Ignore E3003 (missing TableInput) and E3002 (unexpected properties) for Iceberg tables only
+    // cfn-lint doesn't yet support OpenTableFormatInput (Iceberg table format)
+    const isIcebergTable = filename.includes('glue-iceberg-table');
+    const ignoreChecks = isIcebergTable ? 'W,E3003,E3002' : 'W';
+
+    cp.execFile('cfn-lint', [filepath, '--ignore-checks', ignoreChecks], (err, stdout) => {
       if (err) return reject(new Error(stdout));
       return resolve();
     });
@@ -28,7 +33,7 @@ describe('[shortcuts] fixture validation', () => {
 
   test.each(toValidate)('%s fixture passes validation', async (filename) => {
     await Promise.all([
-      cfnLint(path.join(__dirname, 'fixtures', 'shortcuts', filename)),
+      cfnLint(path.join(__dirname, 'fixtures', 'shortcuts', filename), filename),
       sleep(1000)
     ]);
   });
@@ -1246,6 +1251,292 @@ describe('[shortcuts] glue parquet table', () => {
   });
 });
 
+describe('[shortcuts] glue iceberg table', () => {
+  test('throws without options', () => {
+    expect(() => new cf.shortcuts.GlueIcebergTable()).toThrow('Options required');
+  });
+
+  test('throws without required parameters', () => {
+    expect(() => new cf.shortcuts.GlueIcebergTable({})).toThrow(/You must provide a LogicalName, Name, DatabaseName, Location, and Schema/);
+  });
+
+  test('expected resources generated with defaults', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-defaults', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-defaults'));
+  });
+
+  test('expected resources generated without defaults', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      CatalogId: '1234',
+      Location: 's3://fake/location',
+      IcebergVersion: '2'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-no-defaults', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-no-defaults'));
+  });
+
+  test('throws when EnableOptimizer is true but OptimizerRoleArn is missing', () => {
+    expect(() => new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOptimizer: true
+    })).toThrow(/You must provide an OptimizerRoleArn when EnableOptimizer is true/);
+  });
+
+  test('expected resources generated with optimizer using default retention settings', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOptimizer: true,
+      OptimizerRoleArn: 'arn:aws:iam::123456789012:role/OptimizerRole'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-with-optimizer-defaults', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-optimizer-defaults'));
+  });
+
+  test('expected resources generated with optimizer using custom retention settings', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOptimizer: true,
+      OptimizerRoleArn: cf.getAtt('OptimizerRole', 'Arn'),
+      SnapshotRetentionPeriodInDays: 7,
+      NumberOfSnapshotsToRetain: 3,
+      CleanExpiredFiles: false
+    });
+
+    const template = cf.merge(
+      { Resources: { OptimizerRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } },
+      db
+    );
+    if (update) fixtures.update('glue-iceberg-table-with-optimizer-custom', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-optimizer-custom'));
+  });
+
+  test('throws when EnableCompaction is true but CompactionRoleArn is missing', () => {
+    expect(() => new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableCompaction: true
+    })).toThrow(/You must provide a CompactionRoleArn when EnableCompaction is true/);
+  });
+
+  test('expected resources generated with compaction using default settings', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableCompaction: true,
+      CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-with-compaction-defaults', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-compaction-defaults'));
+  });
+
+  test('expected resources generated with compaction using custom settings', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableCompaction: true,
+      CompactionRoleArn: cf.getAtt('CompactionRole', 'Arn')
+    });
+
+    const template = cf.merge(
+      { Resources: { CompactionRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } },
+      db
+    );
+    if (update) fixtures.update('glue-iceberg-table-with-compaction-custom', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-compaction-custom'));
+  });
+
+  test('expected resources generated with both retention and compaction optimizers', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOptimizer: true,
+      OptimizerRoleArn: 'arn:aws:iam::123456789012:role/RetentionRole',
+      EnableCompaction: true,
+      CompactionRoleArn: 'arn:aws:iam::123456789012:role/CompactionRole'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-with-both-optimizers', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-both-optimizers'));
+  });
+
+  test('throws when EnableOrphanFileDeletion is true but OrphanFileDeletionRoleArn is missing', () => {
+    expect(() => new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOrphanFileDeletion: true
+    })).toThrow(/You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true/);
+  });
+
+  test('expected resources generated with orphan file deletion using default settings', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOrphanFileDeletion: true,
+      OrphanFileDeletionRoleArn: 'arn:aws:iam::123456789012:role/OrphanFileDeletionRole'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-with-orphan-deletion-defaults', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-orphan-deletion-defaults'));
+  });
+
+  test('expected resources generated with orphan file deletion using custom settings', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOrphanFileDeletion: true,
+      OrphanFileDeletionRoleArn: cf.getAtt('OrphanFileDeletionRole', 'Arn'),
+      OrphanFileRetentionPeriodInDays: 7,
+      OrphanFileDeletionLocation: 's3://fake/location/subdir'
+    });
+
+    const template = cf.merge(
+      { Resources: { OrphanFileDeletionRole: { Type: 'AWS::IAM::Role', Properties: { AssumeRolePolicyDocument: {} } } } },
+      db
+    );
+    if (update) fixtures.update('glue-iceberg-table-with-orphan-deletion-custom', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-orphan-deletion-custom'));
+  });
+
+  test('expected resources generated with all three optimizers using same role', () => {
+    const db = new cf.shortcuts.GlueIcebergTable({
+      LogicalName: 'MyTable',
+      DatabaseName: 'my_database',
+      Name: 'my_table',
+      Schema: {
+        Type: 'struct',
+        Fields: [
+          { Name: 'column', Type: 'string', Id: 1, Required: true }
+        ]
+      },
+      Location: 's3://fake/location',
+      EnableOptimizer: true,
+      OptimizerRoleArn: 'arn:aws:iam::123456789012:role/SharedRole',
+      EnableCompaction: true,
+      CompactionRoleArn: 'arn:aws:iam::123456789012:role/SharedRole',
+      EnableOrphanFileDeletion: true,
+      OrphanFileDeletionRoleArn: 'arn:aws:iam::123456789012:role/SharedRole'
+    });
+
+    const template = cf.merge(db);
+    if (update) fixtures.update('glue-iceberg-table-with-all-optimizers', template);
+    expect(noUndefined(template)).toEqual(fixtures.get('glue-iceberg-table-with-all-optimizers'));
+  });
+});
+
 describe('[shortcuts] glue view', () => {
   test('throws without options', () => {
     expect(() => new cf.shortcuts.GluePrestoView()).toThrow('Options required');