@mapbox/cloudfriend 9.2.0 → 9.3.1

package/jest.config.js

@@ -0,0 +1,14 @@
+'use strict';
+
+module.exports = {
+  testEnvironment: 'node',
+  testMatch: ['**/test/*.test.js'],
+  collectCoverageFrom: [
+    'index.js',
+    'lib/**/*.js',
+    'bin/**/*.js',
+    '!**/node_modules/**'
+  ],
+  coverageDirectory: 'coverage',
+  verbose: true
+};

package/lib/shortcuts/api.md

@@ -12,6 +12,9 @@ a Lambda permission.</p>
 <dt><a href="#GlueDatabase">GlueDatabase</a></dt>
 <dd><p>Create a Glue Database.</p>
 </dd>
+<dt><a href="#GlueIcebergTable">GlueIcebergTable</a></dt>
+<dd><p>Create a Glue table backed by Apache Iceberg format on S3.</p>
+</dd>
 <dt><a href="#GlueJsonTable">GlueJsonTable</a></dt>
 <dd><p>Create a Glue Table backed by line-delimited JSON files on S3.</p>
 </dd>
@@ -202,6 +205,40 @@ const db = new cf.shortcuts.GlueDatabase({
 
 module.exports = cf.merge(myTemplate, db);
 ```
+<a name="GlueIcebergTable"></a>
+
+## GlueIcebergTable
+Create a Glue table backed by Apache Iceberg format on S3.
+
+**Kind**: global class  
+<a name="new_GlueIcebergTable_new"></a>
+
+### new GlueIcebergTable(options)
+
+| Param | Type | Default | Description |
+| --- | --- | --- | --- |
+| options | <code>Object</code> |  | Options for creating an Iceberg table. |
+| options.LogicalName | <code>String</code> |  | The logical name of the Glue Table within the CloudFormation template. |
+| options.Name | <code>String</code> |  | The name of the table. |
+| options.DatabaseName | <code>String</code> |  | The name of the database the table resides in. |
+| options.Location | <code>String</code> |  | The physical location of the table (S3 URI). Required. |
+| options.Schema | <code>Object</code> |  | Full Iceberg schema definition with Type: "struct" and Fields array. Each field must have Id (integer), Name (string), Type (string or object for complex types), and Required (boolean). See [AWS documentation](https://docs.aws.amazon.com/glue/latest/webapi/API_IcebergSchema.html). |
+| [options.PartitionSpec] | <code>Object</code> |  | Iceberg partition specification. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/webapi/API_IcebergPartitionSpec.html). |
+| [options.WriteOrder] | <code>Object</code> |  | Iceberg write order specification. See [AWS documentation](https://docs.aws.amazon.com/glue/latest/webapi/API_IcebergSortOrder.html). |
+| [options.CatalogId] | <code>String</code> | <code>AccountId</code> | The AWS account ID for the account in which to create the table. |
+| [options.IcebergVersion] | <code>String</code> | <code>&#x27;2&#x27;</code> | The table version for the Iceberg table. |
+| [options.EnableOptimizer] | <code>Boolean</code> | <code>false</code> | Whether to enable the snapshot retention optimizer. |
+| [options.OptimizerRoleArn] | <code>String</code> |  | The ARN of the IAM role for the retention optimizer. Required if EnableOptimizer is true. |
+| [options.SnapshotRetentionPeriodInDays] | <code>Number</code> | <code>5</code> | The number of days to retain snapshots. |
+| [options.NumberOfSnapshotsToRetain] | <code>Number</code> | <code>1</code> | The minimum number of snapshots to retain. |
+| [options.CleanExpiredFiles] | <code>Boolean</code> | <code>true</code> | Whether to delete expired data files after expiring snapshots. |
+| [options.EnableCompaction] | <code>Boolean</code> | <code>false</code> | Whether to enable the compaction optimizer. |
+| [options.CompactionRoleArn] | <code>String</code> |  | The ARN of the IAM role for the compaction optimizer. Required if EnableCompaction is true. |
+| [options.EnableOrphanFileDeletion] | <code>Boolean</code> | <code>false</code> | Whether to enable the orphan file deletion optimizer. |
+| [options.OrphanFileDeletionRoleArn] | <code>String</code> |  | The ARN of the IAM role for the orphan file deletion optimizer. Required if EnableOrphanFileDeletion is true. |
+| [options.OrphanFileRetentionPeriodInDays] | <code>Number</code> | <code>3</code> | The number of days to retain orphan files before deleting them. |
+| [options.OrphanFileDeletionLocation] | <code>String</code> |  | The S3 location to scan for orphan files. |
+
 <a name="GlueJsonTable"></a>
 
 ## GlueJsonTable

package/lib/shortcuts/glue-iceberg-table.js

@@ -0,0 +1,187 @@
+'use strict';
+
+/**
+ * Create a Glue table backed by Apache Iceberg format on S3.
+ *
+ * @param {Object} options - Options for creating an Iceberg table.
+ * @param {String} options.LogicalName - The logical name of the Glue Table within the CloudFormation template.
+ * @param {String} options.Name - The name of the table.
+ * @param {String} options.DatabaseName - The name of the database the table resides in.
+ * @param {String} options.Location - The physical location of the table (S3 URI). Required.
+ * @param {Object} options.Schema - Full Iceberg schema definition with Type: "struct" and Fields array.
+ * Each field must have Id (integer), Name (string), Type (string or object for complex types), and Required (boolean).
+ * See [AWS
+ * documentation](https://docs.aws.amazon.com/glue/latest/webapi/API_IcebergSchema.html).
+ * @param {Object} [options.PartitionSpec] - Iceberg partition specification. See [AWS
+ * documentation](https://docs.aws.amazon.com/glue/latest/webapi/API_IcebergPartitionSpec.html).
+ * @param {Object} [options.WriteOrder] - Iceberg write order specification. See [AWS
+ * documentation](https://docs.aws.amazon.com/glue/latest/webapi/API_IcebergSortOrder.html).
+ * @param {String} [options.CatalogId=AccountId] - The AWS account ID for the account in which to create the table.
+ * @param {String} [options.IcebergVersion='2'] - The table version for the Iceberg table.
+ * @param {Boolean} [options.EnableOptimizer=false] - Whether to enable the snapshot retention optimizer.
+ * @param {String} [options.OptimizerRoleArn=undefined] - The ARN of the IAM role for the retention optimizer. Required if EnableOptimizer is true.
+ * @param {Number} [options.SnapshotRetentionPeriodInDays=5] - The number of days to retain snapshots.
+ * @param {Number} [options.NumberOfSnapshotsToRetain=1] - The minimum number of snapshots to retain.
+ * @param {Boolean} [options.CleanExpiredFiles=true] - Whether to delete expired data files after expiring snapshots.
+ * @param {Boolean} [options.EnableCompaction=false] - Whether to enable the compaction optimizer.
+ * @param {String} [options.CompactionRoleArn=undefined] - The ARN of the IAM role for the compaction optimizer. Required if EnableCompaction is true.
+ * @param {Boolean} [options.EnableOrphanFileDeletion=false] - Whether to enable the orphan file deletion optimizer.
+ * @param {String} [options.OrphanFileDeletionRoleArn=undefined] - The ARN of the IAM role for the orphan file deletion optimizer. Required if EnableOrphanFileDeletion is true.
+ * @param {Number} [options.OrphanFileRetentionPeriodInDays=3] - The number of days to retain orphan files before deleting them.
+ * @param {String} [options.OrphanFileDeletionLocation=undefined] - The S3 location to scan for orphan files.
+ */
+class GlueIcebergTable {
+  constructor(options) {
+    if (!options) throw new Error('Options required');
+    const {
+      LogicalName,
+      Name,
+      DatabaseName,
+      Location,
+      Schema,
+      PartitionSpec,
+      WriteOrder,
+      CatalogId = { Ref: 'AWS::AccountId' },
+      IcebergVersion = '2',
+      EnableOptimizer = false,
+      OptimizerRoleArn,
+      SnapshotRetentionPeriodInDays = 5,
+      NumberOfSnapshotsToRetain = 1,
+      CleanExpiredFiles = true,
+      EnableCompaction = false,
+      CompactionRoleArn,
+      EnableOrphanFileDeletion = false,
+      OrphanFileDeletionRoleArn,
+      OrphanFileRetentionPeriodInDays = 3,
+      OrphanFileDeletionLocation
+    } = options;
+
+    // Validate required fields
+    const required = [LogicalName, Name, DatabaseName, Location, Schema];
+    if (required.some((variable) => !variable))
+      throw new Error('You must provide a LogicalName, Name, DatabaseName, Location, and Schema');
+
+    if (EnableOptimizer && !OptimizerRoleArn)
+      throw new Error('You must provide an OptimizerRoleArn when EnableOptimizer is true');
+
+    if (EnableCompaction && !CompactionRoleArn)
+      throw new Error('You must provide a CompactionRoleArn when EnableCompaction is true');
+
+    if (EnableOrphanFileDeletion && !OrphanFileDeletionRoleArn)
+      throw new Error('You must provide an OrphanFileDeletionRoleArn when EnableOrphanFileDeletion is true');
+
+    // Build the Iceberg table resource (no TableInput!)
+    this.Resources = {
+      [LogicalName]: {
+        Type: 'AWS::Glue::Table',
+        Properties: {
+          CatalogId,
+          DatabaseName,
+          Name,
+          OpenTableFormatInput: {
+            IcebergInput: {
+              MetadataOperation: 'CREATE', // NOTE: this is _always_ CREATE because it's about creating a new metadata version, not creating a new table
+              Version: IcebergVersion,
+              IcebergTableInput: {
+                Location,
+                Schema
+              }
+            }
+          }
+        }
+      }
+    };
+
+    // Add optional PartitionSpec if provided
+    if (PartitionSpec) {
+      this.Resources[LogicalName].Properties.OpenTableFormatInput.IcebergInput.IcebergTableInput.PartitionSpec = PartitionSpec;
+    }
+
+    // Add optional WriteOrder if provided
+    if (WriteOrder) {
+      this.Resources[LogicalName].Properties.OpenTableFormatInput.IcebergInput.IcebergTableInput.WriteOrder = WriteOrder;
+    }
+
+    // Optionally add TableOptimizer for configuring snapshot retention
+    if (EnableOptimizer) {
+      const optimizerLogicalName = `${LogicalName}RetentionOptimizer`;
+      this.Resources[optimizerLogicalName] = {
+        Type: 'AWS::Glue::TableOptimizer',
+        DependsOn: LogicalName,
+        Properties: {
+          CatalogId,
+          DatabaseName,
+          TableName: Name,
+          Type: 'retention',
+          TableOptimizerConfiguration: {
+            RoleArn: OptimizerRoleArn,
+            Enabled: true,
+            RetentionConfiguration: {
+              IcebergConfiguration: {
+                SnapshotRetentionPeriodInDays,
+                NumberOfSnapshotsToRetain,
+                CleanExpiredFiles
+              }
+            }
+          }
+        }
+      };
+    }
+
+    // Optionally add TableOptimizer for compaction
+    // NOTE: CloudFormation does not support CompactionConfiguration properties
+    // (strategy, minInputFiles, deleteFileThreshold). These must be configured
+    // via AWS CLI/API after stack creation, or will use AWS defaults.
+    // See: https://github.com/aws-cloudformation/cloudformation-coverage-roadmap/issues/2257
+    if (EnableCompaction) {
+      const compactionLogicalName = `${LogicalName}CompactionOptimizer`;
+      this.Resources[compactionLogicalName] = {
+        Type: 'AWS::Glue::TableOptimizer',
+        DependsOn: LogicalName,
+        Properties: {
+          CatalogId,
+          DatabaseName,
+          TableName: Name,
+          Type: 'compaction',
+          TableOptimizerConfiguration: {
+            RoleArn: CompactionRoleArn,
+            Enabled: true
+          }
+        }
+      };
+    }
+
+    // Optionally add TableOptimizer for orphan file deletion
+    if (EnableOrphanFileDeletion) {
+      const orphanLogicalName = `${LogicalName}OrphanFileDeletionOptimizer`;
+      const icebergConfiguration = {
+        OrphanFileRetentionPeriodInDays
+      };
+
+      // Only add Location if specified, otherwise it defaults to table location
+      if (OrphanFileDeletionLocation) {
+        icebergConfiguration.Location = OrphanFileDeletionLocation;
+      }
+
+      this.Resources[orphanLogicalName] = {
+        Type: 'AWS::Glue::TableOptimizer',
+        DependsOn: LogicalName,
+        Properties: {
+          CatalogId,
+          DatabaseName,
+          TableName: Name,
+          Type: 'orphan_file_deletion',
+          TableOptimizerConfiguration: {
+            RoleArn: OrphanFileDeletionRoleArn,
+            Enabled: true,
+            OrphanFileDeletionConfiguration: {
+              IcebergConfiguration: icebergConfiguration
+            }
+          }
+        }
+      };
+    }
+  }
+}
+
+module.exports = GlueIcebergTable;

package/lib/shortcuts/index.js

@@ -16,6 +16,7 @@ module.exports = {
   GlueJsonTable: require('./glue-json-table'),
   GlueOrcTable: require('./glue-orc-table'),
   GlueParquetTable: require('./glue-parquet-table'),
+  GlueIcebergTable: require('./glue-iceberg-table'),
   GluePrestoView: require('./glue-presto-view'),
   GlueSparkView: require('./glue-spark-view'),
   hookshot: require('./hookshot'),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mapbox/cloudfriend",
-  "version": "9.2.0",
+  "version": "9.3.1",
   "description": "Helper functions for assembling CloudFormation templates in JavaScript",
   "main": "index.js",
   "engines": {
@@ -8,11 +8,10 @@
   },
   "scripts": {
     "pretest": "npm run lint && npm run shortcuts-api-doc",
-    "lint": "eslint index.js test lib bin cloudformation",
+    "lint": "eslint index.js test lib bin",
     "lint:fix": "npm run lint -- --fix",
-    "test": "nyc tape test/*.test.js | tap-spec",
-    "test:update": "UPDATE=1 npm run test",
-    "coverage": "nyc --reporter html tape test/*.test.js && opener coverage/index.html",
+    "test": "jest --coverage",
+    "test:update": "jest --updateSnapshot",
     "build-ci-template": "bin/build-template.js cloudformation/ci.template.js > cloudformation/ci.template.json",
     "shortcuts-api-doc": "jsdoc2md lib/shortcuts/*.js > lib/shortcuts/api.md"
   },
@@ -35,14 +34,11 @@
   },
   "homepage": "https://github.com/mapbox/cloudfriend#readme",
   "devDependencies": {
-    "@mapbox/eslint-config-mapbox": "^1.2.1",
-    "eslint": "^4.18.2",
+    "@mapbox/eslint-config-mapbox": "^5.0.1",
+    "eslint": "^8.57.0",
     "eslint-plugin-node": "^6.0.1",
-    "jsdoc-to-markdown": "^7.1.1",
-    "nyc": "^14.1.0",
-    "opener": "^1.4.1",
-    "tap-spec": "^5.0.0",
-    "tape": "^4.6.0"
+    "jest": "^29.7.0",
+    "jsdoc-to-markdown": "^9.1.3"
   },
   "dependencies": {
     "@aws-sdk/client-cloudformation": "^3.848.0",
@@ -50,6 +46,12 @@
     "redent": "^2.0.0"
   },
   "eslintConfig": {
-    "extends": "@mapbox/eslint-config-mapbox"
+    "extends": "@mapbox/eslint-config-mapbox",
+    "env": {
+      "jest": true
+    },
+    "ignorePatterns": [
+      "test/fixtures/malformed.json"
+    ]
   }
 }

package/readme.md

@@ -1,6 +1,6 @@
 # cloudfriend
 
-[![Build Status](https://travis-ci.com/mapbox/cloudfriend.svg?branch=master)](https://travis-ci.com/mapbox/cloudfriend)
+[![mapbox/cloudfriend](https://github.com/mapbox/cloudfriend/actions/workflows/test.yml/badge.svg)](https://github.com/mapbox/cloudfriend/actions/workflows/test.yml)
 
 Helper functions for assembling CloudFormation templates in JavaScript.
 

package/requirements.dev.txt

@@ -1,2 +1,2 @@
-aws-sam-cli==1.142.1
-cfn-lint==1.36.1
+aws-sam-cli==1.149.0
+cfn-lint==1.41.0

package/test/bin.test.js

@@ -1,22 +1,26 @@
 'use strict';
 
 const path = require('path');
-const exec = require('child_process').exec;
-const test = require('tape');
+const { execFile } = require('child_process');
+const { promisify } = require('util');
 
-test('bin/build-template', (t) => {
+const execFileAsync = promisify(execFile);
+
+describe('bin/build-template', () => {
   const script = path.normalize(__dirname + '/../bin/build-template.js');
   const template = path.normalize(__dirname + '/fixtures/sync-args.js');
-  t.test('outputs expected', (q) => {
-    exec([script, template, '--this', 'that', '-r', 'cn-north-1'].join(' '), (err, stdout, stderr) => {
-      q.error(err);
-      q.error(stderr);
-      const result = JSON.parse(stdout);
-      q.equal('that', result.this);
-      q.equal('cn-north-1', result.region);
-      q.end();
-    });
+
+  test('outputs expected', async () => {
+    const { stdout, stderr } = await execFileAsync(script, [
+      template,
+      '--this',
+      'that',
+      '-r',
+      'cn-north-1'
+    ]);
+    expect(stderr).toBeFalsy();
+    const result = JSON.parse(stdout);
+    expect(result.this).toBe('that');
+    expect(result.region).toBe('cn-north-1');
   });
-  t.end();
 });
-

package/test/fixtures/shortcuts/cross-account-role-defaults.json

@@ -30,4 +30,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/cross-account-role-no-defaults.json

@@ -63,4 +63,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/event-lambda-defaults.json

@@ -199,4 +199,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/event-lambda-full.json

@@ -199,4 +199,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/firehose-defaults.json

@@ -197,4 +197,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/firehose-with-stream.json

@@ -231,4 +231,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/glue-database-defaults.json

@@ -22,4 +22,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/glue-database-no-defaults.json

@@ -34,4 +34,4 @@
     }
   },
   "Outputs": {}
-}
+}

package/test/fixtures/shortcuts/glue-iceberg-table-defaults.json

@@ -0,0 +1,41 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}

package/test/fixtures/shortcuts/glue-iceberg-table-no-defaults.json

@@ -0,0 +1,39 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": "1234",
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}

package/test/fixtures/shortcuts/glue-iceberg-table-with-all-optimizers.json

@@ -0,0 +1,101 @@
+{
+  "AWSTemplateFormatVersion": "2010-09-09",
+  "Metadata": {},
+  "Parameters": {},
+  "Rules": {},
+  "Mappings": {},
+  "Conditions": {},
+  "Resources": {
+    "MyTable": {
+      "Type": "AWS::Glue::Table",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "Name": "my_table",
+        "OpenTableFormatInput": {
+          "IcebergInput": {
+            "MetadataOperation": "CREATE",
+            "Version": "2",
+            "IcebergTableInput": {
+              "Location": "s3://fake/location",
+              "Schema": {
+                "Type": "struct",
+                "Fields": [
+                  {
+                    "Name": "column",
+                    "Type": "string",
+                    "Id": 1,
+                    "Required": true
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+    },
+    "MyTableRetentionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "retention",
+        "TableOptimizerConfiguration": {
+          "RoleArn": "arn:aws:iam::123456789012:role/SharedRole",
+          "Enabled": true,
+          "RetentionConfiguration": {
+            "IcebergConfiguration": {
+              "SnapshotRetentionPeriodInDays": 5,
+              "NumberOfSnapshotsToRetain": 1,
+              "CleanExpiredFiles": true
+            }
+          }
+        }
+      }
+    },
+    "MyTableCompactionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "compaction",
+        "TableOptimizerConfiguration": {
+          "RoleArn": "arn:aws:iam::123456789012:role/SharedRole",
+          "Enabled": true
+        }
+      }
+    },
+    "MyTableOrphanFileDeletionOptimizer": {
+      "Type": "AWS::Glue::TableOptimizer",
+      "DependsOn": "MyTable",
+      "Properties": {
+        "CatalogId": {
+          "Ref": "AWS::AccountId"
+        },
+        "DatabaseName": "my_database",
+        "TableName": "my_table",
+        "Type": "orphan_file_deletion",
+        "TableOptimizerConfiguration": {
+          "RoleArn": "arn:aws:iam::123456789012:role/SharedRole",
+          "Enabled": true,
+          "OrphanFileDeletionConfiguration": {
+            "IcebergConfiguration": {
+              "OrphanFileRetentionPeriodInDays": 3
+            }
+          }
+        }
+      }
+    }
+  },
+  "Outputs": {}
+}