@aws-cdk/aws-glue-alpha 2.257.0-alpha.0 → 2.258.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +121 -82
- package/.jsii.tabl.json.gz +0 -0
- package/.warnings.jsii.js +64 -0
- package/README.md +5 -47
- package/lib/code.js +40 -7
- package/lib/connection.js +37 -4
- package/lib/constants.d.ts +11 -0
- package/lib/constants.js +12 -1
- package/lib/data-format.js +5 -5
- package/lib/data-quality-ruleset.js +37 -4
- package/lib/database.js +1 -1
- package/lib/external-table.js +1 -1
- package/lib/jobs/job.js +40 -7
- package/lib/jobs/pyspark-etl-job.d.ts +7 -0
- package/lib/jobs/pyspark-etl-job.js +3 -2
- package/lib/jobs/pyspark-flex-etl-job.js +1 -1
- package/lib/jobs/pyspark-streaming-job.js +1 -1
- package/lib/jobs/python-shell-job.js +1 -1
- package/lib/jobs/ray-job.d.ts +8 -0
- package/lib/jobs/ray-job.js +25 -2
- package/lib/jobs/scala-spark-etl-job.d.ts +7 -0
- package/lib/jobs/scala-spark-etl-job.js +3 -2
- package/lib/jobs/scala-spark-flex-etl-job.js +1 -1
- package/lib/jobs/scala-spark-streaming-job.js +1 -1
- package/lib/jobs/spark-job.js +36 -3
- package/lib/partition-projection.js +1 -1
- package/lib/s3-table.js +37 -4
- package/lib/schema.js +1 -1
- package/lib/security-configuration.js +37 -4
- package/lib/storage-parameter.js +1 -1
- package/lib/table-base.js +37 -4
- package/lib/table-deprecated.js +1 -1
- package/lib/triggers/trigger-options.js +36 -3
- package/lib/triggers/workflow.js +37 -4
- package/package.json +7 -7
package/.jsii
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"url": "https://aws.amazon.com"
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"aws-cdk-lib": "^2.
|
|
11
|
+
"aws-cdk-lib": "^2.258.1",
|
|
12
12
|
"constructs": "^10.5.0"
|
|
13
13
|
},
|
|
14
14
|
"dependencyClosure": {
|
|
@@ -8966,7 +8966,7 @@
|
|
|
8966
8966
|
"stability": "experimental"
|
|
8967
8967
|
},
|
|
8968
8968
|
"homepage": "https://github.com/aws/aws-cdk",
|
|
8969
|
-
"jsiiVersion": "5.9.
|
|
8969
|
+
"jsiiVersion": "5.9.44 (build 150b837)",
|
|
8970
8970
|
"keywords": [
|
|
8971
8971
|
"aws",
|
|
8972
8972
|
"cdk",
|
|
@@ -8987,7 +8987,7 @@
|
|
|
8987
8987
|
},
|
|
8988
8988
|
"name": "@aws-cdk/aws-glue-alpha",
|
|
8989
8989
|
"readme": {
|
|
8990
|
-
"markdown": "# AWS Glue Construct Library\n<!--BEGIN STABILITY BANNER-->\n\n---\n\n\n\n> The APIs of higher level constructs in this module are experimental and under active development.\n> They are subject to non-backward compatible changes or removal in any future version. These are\n> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be\n> announced in the release notes. This means that while you may use them, you may need to update\n> your source code when upgrading to a newer version of this package.\n\n---\n\n<!--END STABILITY BANNER-->\n\nThis module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project.\n\n## README\n\n[AWS Glue](https://aws.amazon.com/glue/) is a serverless data integration\nservice that makes it easier to discover, prepare, move, and integrate data\nfrom multiple sources for analytics, machine learning (ML), and application\ndevelopment.\n\nThe Glue L2 construct has convenience methods working backwards from common\nuse cases and sets required parameters to defaults that align with recommended\nbest practices for each job type. It also provides customers with a balance\nbetween flexibility via optional parameter overrides, and opinionated\ninterfaces that discouraging anti-patterns, resulting in reduced time to develop\nand deploy new resources.\n\n### References\n\n* [Glue Launch Announcement](https://aws.amazon.com/blogs/aws/launch-aws-glue-now-generally-available/)\n* [Glue Documentation](https://docs.aws.amazon.com/glue/index.html)\n* [Glue L1 (CloudFormation) Constructs](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/AWS_Glue.html)\n* Prior version of the [@aws-cdk/aws-glue-alpha module](https://github.com/aws/aws-cdk/blob/v2.51.1/packages/%40aws-cdk/aws-glue/README.md)\n\n## Create a Glue Job\n\nA Job encapsulates a script that connects to data sources, processes\nthem, and then writes output to a data target. There are four types of Glue\nJobs: Spark (ETL and Streaming), Python Shell, Ray, and Flex Jobs. Most\nof the required parameters for these jobs are common across all types,\nbut there are a few differences depending on the languages supported\nand features provided by each type. For all job types, the L2 defaults\nto AWS best practice recommendations, such as:\n\n* Use of Secrets Manager for Connection JDBC strings\n* Glue job autoscaling\n* Default parameter values for Glue job creation\n\nThis iteration of the L2 construct introduces breaking changes to\nthe existing glue-alpha-module, but these changes streamline the developer\nexperience, introduce new constants for defaults, and replacing synth-time\nvalidations with interface contracts for enforcement of the parameter combinations\nthat Glue supports. As an opinionated construct, the Glue L2 construct does\nnot allow developers to create resources that use non-current versions\nof Glue or deprecated language dependencies (e.g. deprecated versions of Python).\nAs always, L1s allow you to specify a wider range of parameters if you need\nor want to use alternative configurations.\n\nOptional and required parameters for each job are enforced via interface\nrather than validation; see [Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api.html)\nfor more granular details.\n\n### Spark Jobs\n\n#### ETL Jobs\n\nETL jobs support pySpark and Scala languages, for which there are separate but\nsimilar constructors. ETL jobs default to the G2 worker type, but you can\noverride this default with other supported worker type values (G1, G2, G4\nand G8). ETL jobs defaults to Glue version 4.0, which you can override to 3.0.\nThe following ETL features are enabled by default:\n`—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.`\nYou can find more details about version, worker type and other features in\n[Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html).\n\nReference the pyspark-etl-jobs.test.ts and scalaspark-etl-jobs.test.ts unit tests\nfor examples of required-only and optional job parameters when creating these\ntypes of jobs.\n\nFor the sake of brevity, examples are shown using the pySpark job variety.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n role,\n script,\n jobName: 'PySparkETLJob',\n});\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n jobName: 'PySparkETLJobCustomName',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n#### Streaming Jobs\n\nStreaming jobs are similar to ETL jobs, except that they perform ETL on data\nstreams using the Apache Spark Structured Streaming framework. Some Spark\njob features are not available to Streaming ETL jobs. They support Scala\nand pySpark languages. PySpark streaming jobs default Python 3.9,\nwhich you can override with any non-deprecated version of Python. It\ndefaults to the G2 worker type and Glue 4.0, both of which you can override.\nThe following best practice features are enabled by default:\n`—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log`.\n\nReference the pyspark-streaming-jobs.test.ts and scalaspark-streaming-jobs.test.ts\nunit tests for examples of required-only and optional job parameters when creating\nthese types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkStreamingJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {\n jobName: 'PySparkStreamingJobCustomName',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n#### Flex Jobs\n\nThe flexible execution class is appropriate for non-urgent jobs such as\npre-production jobs, testing, and one-time data loads. Flexible jobs default\nto Glue version 3.0 and worker type `G_2X`. The following best practice\nfeatures are enabled by default:\n`—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log`\n\nReference the pyspark-flex-etl-jobs.test.ts and scalaspark-flex-etl-jobs.test.ts\nunit tests for examples of required-only and optional job parameters when creating\nthese types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkFlexEtlJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'pySparkEtlJob', {\n jobName: 'pySparkEtlJob',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n### Python Shell Jobs\n\nPython shell jobs support a Python version that depends on the AWS Glue\nversion you use. These can be used to schedule and run tasks that don't\nrequire an Apache Spark environment. Python shell jobs default to\nPython 3.9 and a MaxCapacity of `0.0625`. Python 3.9 supports pre-loaded\nanalytics libraries using the `library-set=analytics` flag, which is\nenabled by default.\n\nReference the pyspark-shell-job.test.ts unit tests for examples of\nrequired-only and optional job parameters when creating these types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PythonShellJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\ndeclare const extraPythonFile: glue.Code;\nnew glue.PythonShellJob(stack, 'PythonShellJob', {\n jobName: 'PythonShellJobCustomName',\n description: 'This is a description',\n pythonVersion: glue.PythonVersion.TWO,\n maxCapacity: glue.MaxCapacity.DPU_1,\n role,\n script,\n extraPythonFiles: [extraPythonFile],\n glueVersion: glue.GlueVersion.V2_0,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n### Ray Jobs\n\nGlue Ray jobs use worker type Z.2X and Glue version 4.0. These are not\noverrideable since these are the only configuration that Glue Ray jobs\ncurrently support. The runtime defaults to Ray2.4 and min workers defaults to 3.\n\nReference the ray-job.test.ts unit tests for examples of required-only and\noptional job parameters when creating these types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.RayJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.RayJob(stack, 'ImportedJob', {\n role,\n script,\n jobName: 'RayCustomJobName',\n description: 'This is a description',\n workerType: glue.WorkerType.Z_2X,\n numberOfWorkers: 5,\n runtime: glue.Runtime.RAY_TWO_FOUR,\n maxRetries: 3,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n});\n```\n\n### Metrics Control\n\nBy default, Glue jobs enable CloudWatch metrics (`--enable-metrics`) and observability metrics (`--enable-observability-metrics`) for monitoring and debugging. You can disable these metrics to reduce CloudWatch costs:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\n\n// Disable both metrics for cost optimization\nnew glue.PySparkEtlJob(stack, 'CostOptimizedJob', {\n role,\n script,\n enableMetrics: false,\n enableObservabilityMetrics: false,\n});\n\n// Selective control - keep observability, disable profiling\nnew glue.PySparkEtlJob(stack, 'SelectiveJob', {\n role,\n script,\n enableMetrics: false,\n // enableObservabilityMetrics defaults to true\n});\n```\n\nThis feature is available for all Spark job types (ETL, Streaming, Flex) and Ray jobs.\n\n### Enable Job Run Queuing\n\nAWS Glue job queuing monitors your account level quotas and limits. If quotas or limits are insufficient to start a Glue job run, AWS Glue will automatically queue the job and wait for limits to free up. Once limits become available, AWS Glue will retry the job run. Glue jobs will queue for limits like max concurrent job runs per account, max concurrent Data Processing Units (DPU), and resource unavailable due to IP address exhaustion in Amazon Virtual Private Cloud (Amazon VPC).\n\nEnable job run queuing by setting the `jobRunQueuingEnabled` property to `true`.\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n role,\n script,\n jobName: 'PySparkETLJob',\n jobRunQueuingEnabled: true\n});\n```\n\n### Uploading scripts from the CDK app repository to S3\n\nSimilar to other L2 constructs, the Glue L2 automates uploading / updating\nscripts to S3 via an optional fromAsset parameter pointing to a script\nin the local file structure. You provide the existing S3 bucket and\npath to which you'd like the script to be uploaded.\n\nReference the unit tests for examples of repo and S3 code target examples.\n\n### Workflow Triggers\n\nYou can use Glue workflows to create and visualize complex\nextract, transform, and load (ETL) activities involving multiple crawlers,\njobs, and triggers. Standalone triggers are an anti-pattern, so you must\ncreate triggers from within a workflow using the L2 construct.\n\nWithin a workflow object, there are functions to create different\ntypes of triggers with actions and predicates. You then add those triggers\nto jobs.\n\nStartOnCreation defaults to true for all trigger types, but you can\noverride it if you prefer for your trigger not to start on creation.\n\nReference the workflow-triggers.test.ts unit tests for examples of creating\nworkflows and triggers.\n\n#### **1. On-Demand Triggers**\n\nOn-demand triggers can start glue jobs or crawlers. This construct provides\nconvenience functions to create on-demand crawler or job triggers. The constructor\ntakes an optional description parameter, but abstracts the requirement of an\nactions list using the job or crawler objects using conditional types.\n\n#### **2. Scheduled Triggers**\n\nYou can create scheduled triggers using cron expressions. This construct\nprovides daily, weekly, and monthly convenience functions,\nas well as a custom function that allows you to create your own\ncustom timing using the [existing event Schedule class](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_events.Schedule.html)\nwithout having to build your own cron expressions. The L2 extracts\nthe expression that Glue requires from the Schedule object. The constructor\ntakes an optional description and a list of jobs or crawlers as actions.\n\n#### **3. Notify Event Triggers**\n\nThere are two types of notify event triggers: batching and non-batching.\nFor batching triggers, you must specify `BatchSize`. For non-batching\ntriggers, `BatchSize` defaults to 1. For both triggers, `BatchWindow`\ndefaults to 900 seconds, but you can override the window to align with\nyour workload's requirements.\n\n#### **4. Conditional Triggers**\n\nConditional triggers have a predicate and actions associated with them.\nThe trigger actions are executed when the predicateCondition is true.\n\n### Connection Properties\n\nA `Connection` allows Glue jobs, crawlers and development endpoints to access\ncertain types of data stores.\n\n* **Secrets Management**\n You must specify JDBC connection credentials in Secrets Manager and\n provide the Secrets Manager Key name as a property to the job connection.\n\n* **Networking - the CDK determines the best fit subnet for Glue connection\nconfiguration**\n The prior version of the glue-alpha-module requires the developer to\n specify the subnet of the Connection when it’s defined. Now, you can still\n specify the specific subnet you want to use, but are no longer required\n to. You are only required to provide a VPC and either a public or private\n subnet selection. Without a specific subnet provided, the L2 leverages the\n existing [EC2 Subnet Selection](https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/SubnetSelection.html)\n library to make the best choice selection for the subnet.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\nnew glue.Connection(this, 'MyConnection', {\n type: glue.ConnectionType.NETWORK,\n // The security groups granting AWS Glue inbound access to the data source within the VPC\n securityGroups: [securityGroup],\n // The VPC subnet which contains the data source\n subnet,\n});\n```\n\nFor RDS `Connection` by JDBC, it is recommended to manage credentials using AWS Secrets Manager. To use Secret, specify `SECRET_ID` in `properties` like the following code. Note that in this case, the subnet must have a route to the AWS Secrets Manager VPC endpoint or to the AWS Secrets Manager endpoint through a NAT gateway.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\ndeclare const db: rds.DatabaseCluster;\nnew glue.Connection(this, \"RdsConnection\", {\n type: glue.ConnectionType.JDBC,\n securityGroups: [securityGroup],\n subnet,\n properties: {\n JDBC_CONNECTION_URL: `jdbc:mysql://${db.clusterEndpoint.socketAddress}/databasename`,\n JDBC_ENFORCE_SSL: \"false\",\n SECRET_ID: db.secret!.secretName,\n },\n});\n```\n\nIf you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.\n\nSee [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.\n\n## SecurityConfiguration\n\nA `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.\n\n```ts\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n },\n jobBookmarksEncryption: {\n mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,\n },\n s3Encryption: {\n mode: glue.S3EncryptionMode.KMS,\n },\n});\n```\n\nBy default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:\n\n```ts\ndeclare const key: kms.Key;\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n kmsKey: key,\n },\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.\n\n## Database\n\nA `Database` is a logical grouping of `Tables` in the Glue Catalog.\n\n```ts\nnew glue.Database(this, 'MyDatabase', {\n databaseName: 'my_database',\n description: 'my_database_description',\n});\n```\n\n## Table\n\nA Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }, {\n name: 'col2',\n type: glue.Schema.array(glue.Schema.STRING),\n comment: 'col2 is an array of strings' // comment is optional\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:\n\n```ts\ndeclare const myBucket: s3.Bucket;\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n bucket: myBucket,\n s3Prefix: 'my-table/',\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nGlue tables can be configured to contain user-defined properties, to describe the physical storage of table data, through the `storageParameters` property:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n storageParameters: [\n glue.StorageParameter.skipHeaderLineCount(1),\n glue.StorageParameter.compressionType(glue.CompressionType.GZIP),\n glue.StorageParameter.custom('separatorChar', ',')\n ],\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nGlue tables can also be configured to contain user-defined table properties through the [`parameters`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-tableinput.html#cfn-glue-table-tableinput-parameters) property:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n parameters: {\n key1: 'val1',\n key2: 'val2',\n },\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Keys\n\nTo improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Indexes\n\nAnother way to improve query performance is to specify partition indexes. If no partition indexes are\npresent on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using\nthe query expression. The query takes more time to run as the number of partitions increase. With an\nindex, the query will try to fetch a subset of the partitions instead of loading all partitions of the\ntable.\n\nThe keys of a partition index must be a subset of the partition keys of the table. You can have a\nmaximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`\nproperty:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n partitionIndexes: [{\n indexName: 'my-index', // optional\n keyNames: ['year'],\n }], // supply up to 3 indexes\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nAlternatively, you can call the `addPartitionIndex()` function on a table:\n\n```ts\ndeclare const myTable: glue.Table;\nmyTable.addPartitionIndex({\n indexName: 'my-index',\n keyNames: ['year'],\n});\n```\n\n### Partition Filtering\n\nIf you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n enablePartitionFiltering: true,\n});\n```\n\n### Partition Projection\n\nPartition projection allows Athena to automatically add new partitions as new data arrives, without requiring `ALTER TABLE ADD PARTITION` statements. This improves query performance and reduces management overhead by eliminating the need to manually manage partition metadata.\n\nFor more information, see the [AWS documentation on partition projection](https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html).\n\n#### INTEGER Projection\n\nFor partition keys with sequential numeric values:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.INTEGER,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n year: glue.PartitionProjectionConfiguration.integer({\n min: 2020,\n max: 2023,\n interval: 1, // optional, defaults to 1\n digits: 4, // optional, pads with leading zeros\n }),\n },\n});\n```\n\n#### DATE Projection\n\nFor partition keys with date or timestamp values. Supports both fixed dates and relative dates using `NOW`:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'date',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n date: glue.PartitionProjectionConfiguration.date({\n min: '2020-01-01',\n max: '2023-12-31',\n format: 'yyyy-MM-dd',\n interval: 1, // optional, defaults to 1\n intervalUnit: glue.DateIntervalUnit.DAYS, // optional: YEARS, MONTHS, WEEKS, DAYS, HOURS, MINUTES, SECONDS\n }),\n },\n});\n```\n\nYou can also use relative dates with `NOW`:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'date',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n date: glue.PartitionProjectionConfiguration.date({\n min: 'NOW-3YEARS',\n max: 'NOW',\n format: 'yyyy-MM-dd',\n }),\n },\n});\n```\n\n#### ENUM Projection\n\nFor partition keys with a known set of values:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'region',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n region: glue.PartitionProjectionConfiguration.enum({\n values: ['us-east-1', 'us-west-2', 'eu-west-1'],\n }),\n },\n});\n```\n\n#### INJECTED Projection\n\nFor custom partition values injected at query time:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'custom',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n custom: glue.PartitionProjectionConfiguration.injected(),\n },\n});\n```\n\n#### Multiple Partition Projections\n\nYou can configure partition projection for multiple partition keys:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [\n {\n name: 'year',\n type: glue.Schema.INTEGER,\n },\n {\n name: 'month',\n type: glue.Schema.INTEGER,\n },\n {\n name: 'region',\n type: glue.Schema.STRING,\n },\n ],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n year: glue.PartitionProjectionConfiguration.integer({\n min: 2020,\n max: 2023,\n }),\n month: glue.PartitionProjectionConfiguration.integer({\n min: 1,\n max: 12,\n digits: 2,\n }),\n region: glue.PartitionProjectionConfiguration.enum({\n values: ['us-east-1', 'us-west-2'],\n }),\n },\n});\n```\n\n### Glue Connections\n\nGlue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog.\n\nWhereas `S3Table` will point to (and if needed, create) a bucket to store the tables' data, `ExternalTable` will point to an existing table in a data source. For example, to create a table in Glue that points to a table in Redshift:\n\n```ts\ndeclare const myConnection: glue.Connection;\ndeclare const myDatabase: glue.Database;\nnew glue.ExternalTable(this, 'MyTable', {\n connection: myConnection,\n externalDataLocation: 'default_db_public_example', // A table in Redshift\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)\n\nYou can enable encryption on a Table's data:\n\n* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - (default) Server side encryption (`SSE-S3`) with an Amazon S3-managed key.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.S3_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n*Note: you cannot provide a `Bucket` when creating the `S3Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.\n\n## Types\n\nA table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n columns: [{\n name: 'primitive_column',\n type: glue.Schema.STRING,\n }, {\n name: 'array_column',\n type: glue.Schema.array(glue.Schema.INTEGER),\n comment: 'array<integer>',\n }, {\n name: 'map_column',\n type: glue.Schema.map(\n glue.Schema.STRING,\n glue.Schema.TIMESTAMP),\n comment: 'map<string,string>',\n }, {\n name: 'struct_column',\n type: glue.Schema.struct([{\n name: 'nested_column',\n type: glue.Schema.DATE,\n comment: 'nested comment',\n }]),\n comment: \"struct<nested_column:date COMMENT 'nested comment'>\",\n }],\n // ...\n database: myDatabase,\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n## Public FAQ\n\n### What are we launching today?\n\nWe’re launching new features to an AWS CDK Glue L2 Construct to provide\nbest-practice defaults and convenience methods to create Glue Jobs, Connections,\nTriggers, Workflows, and the underlying permissions and configuration.\n\n### Why should I use this Construct?\n\nDevelopers should use this Construct to reduce the amount of boilerplate\ncode and complexity each individual has to navigate, and make it easier to\ncreate best-practice Glue resources.\n\n### What’s not in scope?\n\nGlue Crawlers and other resources that are now managed by the AWS LakeFormation\nteam are not in scope for this effort. Developers should use existing methods\nto create these resources, and the new Glue L2 construct assumes they already\nexist as inputs. While best practice is for application and infrastructure code\nto be as close as possible for teams using fully-implemented DevOps mechanisms,\nin practice these ETL scripts are likely managed by a data science team who\nknow Python or Scala and don’t necessarily own or manage their own\ninfrastructure deployments. We want to meet developers where they are, and not\nassume that all of the code resides in the same repository, Developers can\nautomate this themselves via the CDK, however, if they do own both.\n\nValidating Glue version and feature use per AWS region at synth time is also\nnot in scope. AWS’ intention is for all features to eventually be propagated to\nall Global regions, so the complexity involved in creating and updating region-\nspecific configuration to match shifting feature sets does not out-weigh the\nlikelihood that a developer will use this construct to deploy resources to a\nregion without a particular new feature to a region that doesn’t yet support\nit without researching or manually attempting to use that feature before\ndeveloping it via IaC. The developer will, of course, still get feedback from\nthe underlying Glue APIs as CloudFormation deploys the resources similar to the\ncurrent CDK L1 Glue experience.\n"
|
|
8990
|
+
"markdown": "# AWS Glue Construct Library\n<!--BEGIN STABILITY BANNER-->\n\n---\n\n\n\n> The APIs of higher level constructs in this module are experimental and under active development.\n> They are subject to non-backward compatible changes or removal in any future version. These are\n> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be\n> announced in the release notes. This means that while you may use them, you may need to update\n> your source code when upgrading to a newer version of this package.\n\n---\n\n<!--END STABILITY BANNER-->\n\nThis module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project.\n\n## README\n\n[AWS Glue](https://aws.amazon.com/glue/) is a serverless data integration\nservice that makes it easier to discover, prepare, move, and integrate data\nfrom multiple sources for analytics, machine learning (ML), and application\ndevelopment.\n\nThe Glue L2 construct has convenience methods working backwards from common\nuse cases and sets required parameters to defaults that align with recommended\nbest practices for each job type. It also provides customers with a balance\nbetween flexibility via optional parameter overrides, and opinionated\ninterfaces that discouraging anti-patterns, resulting in reduced time to develop\nand deploy new resources.\n\n### References\n\n* [Glue Launch Announcement](https://aws.amazon.com/blogs/aws/launch-aws-glue-now-generally-available/)\n* [Glue Documentation](https://docs.aws.amazon.com/glue/index.html)\n* [Glue L1 (CloudFormation) Constructs](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/AWS_Glue.html)\n* Prior version of the [@aws-cdk/aws-glue-alpha module](https://github.com/aws/aws-cdk/blob/v2.51.1/packages/%40aws-cdk/aws-glue/README.md)\n\n## Create a Glue Job\n\nA Job encapsulates a script that connects to data sources, processes\nthem, and then writes output to a data target. There are four types of Glue\nJobs: Spark (ETL and Streaming), Python Shell, and Flex Jobs. Most\nof the required parameters for these jobs are common across all types,\nbut there are a few differences depending on the languages supported\nand features provided by each type. For all job types, the L2 defaults\nto AWS best practice recommendations, such as:\n\n* Use of Secrets Manager for Connection JDBC strings\n* Glue job autoscaling\n* Default parameter values for Glue job creation\n\nThis iteration of the L2 construct introduces breaking changes to\nthe existing glue-alpha-module, but these changes streamline the developer\nexperience, introduce new constants for defaults, and replacing synth-time\nvalidations with interface contracts for enforcement of the parameter combinations\nthat Glue supports. As an opinionated construct, the Glue L2 construct does\nnot allow developers to create resources that use non-current versions\nof Glue or deprecated language dependencies (e.g. deprecated versions of Python).\nAs always, L1s allow you to specify a wider range of parameters if you need\nor want to use alternative configurations.\n\nOptional and required parameters for each job are enforced via interface\nrather than validation; see [Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api.html)\nfor more granular details.\n\n### Spark Jobs\n\n#### ETL Jobs\n\nETL jobs support pySpark and Scala languages, for which there are separate but\nsimilar constructors. ETL jobs default to the G2 worker type, but you can\noverride this default with other supported worker type values (G1, G2, G4\nand G8). ETL jobs defaults to Glue version 4.0, which you can override to 3.0.\nThe following ETL features are enabled by default:\n`—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.`\nYou can find more details about version, worker type and other features in\n[Glue's public documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html).\n\nReference the pyspark-etl-jobs.test.ts and scalaspark-etl-jobs.test.ts unit tests\nfor examples of required-only and optional job parameters when creating these\ntypes of jobs.\n\nFor the sake of brevity, examples are shown using the pySpark job variety.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n role,\n script,\n jobName: 'PySparkETLJob',\n});\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n jobName: 'PySparkETLJobCustomName',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n#### Streaming Jobs\n\nStreaming jobs are similar to ETL jobs, except that they perform ETL on data\nstreams using the Apache Spark Structured Streaming framework. Some Spark\njob features are not available to Streaming ETL jobs. They support Scala\nand pySpark languages. PySpark streaming jobs default Python 3.9,\nwhich you can override with any non-deprecated version of Python. It\ndefaults to the G2 worker type and Glue 4.0, both of which you can override.\nThe following best practice features are enabled by default:\n`—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log`.\n\nReference the pyspark-streaming-jobs.test.ts and scalaspark-streaming-jobs.test.ts\nunit tests for examples of required-only and optional job parameters when creating\nthese types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkStreamingJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkStreamingJob(stack, 'PySparkStreamingJob', {\n jobName: 'PySparkStreamingJobCustomName',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n#### Flex Jobs\n\nThe flexible execution class is appropriate for non-urgent jobs such as\npre-production jobs, testing, and one-time data loads. Flexible jobs default\nto Glue version 3.0 and worker type `G_2X`. The following best practice\nfeatures are enabled by default:\n`—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log`\n\nReference the pyspark-flex-etl-jobs.test.ts and scalaspark-flex-etl-jobs.test.ts\nunit tests for examples of required-only and optional job parameters when creating\nthese types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkFlexEtlJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'pySparkEtlJob', {\n jobName: 'pySparkEtlJob',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n### Python Shell Jobs\n\nPython shell jobs support a Python version that depends on the AWS Glue\nversion you use. These can be used to schedule and run tasks that don't\nrequire an Apache Spark environment. Python shell jobs default to\nPython 3.9 and a MaxCapacity of `0.0625`. Python 3.9 supports pre-loaded\nanalytics libraries using the `library-set=analytics` flag, which is\nenabled by default.\n\nReference the pyspark-shell-job.test.ts unit tests for examples of\nrequired-only and optional job parameters when creating these types of jobs.\n\nExample with only required parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PythonShellJob(stack, 'ImportedJob', { role, script });\n```\n\nExample with optional override parameters:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\ndeclare const extraPythonFile: glue.Code;\nnew glue.PythonShellJob(stack, 'PythonShellJob', {\n jobName: 'PythonShellJobCustomName',\n description: 'This is a description',\n pythonVersion: glue.PythonVersion.TWO,\n maxCapacity: glue.MaxCapacity.DPU_1,\n role,\n script,\n extraPythonFiles: [extraPythonFile],\n glueVersion: glue.GlueVersion.V2_0,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});\n```\n\n### Ray Jobs\n\n> **⚠️ DEPRECATED:** AWS Glue for Ray is closed to new customers as of April 30, 2026 and is in maintenance mode.\n> Migrate to [Amazon EKS with KubeRay Operator](https://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html).\n\nThe `RayJob` construct, `Runtime.RAY_TWO_FOUR`, and `JobType.RAY` are deprecated and will be removed in a future release.\n\n### Metrics Control\n\nBy default, Glue jobs enable CloudWatch metrics (`--enable-metrics`) and observability metrics (`--enable-observability-metrics`) for monitoring and debugging. You can disable these metrics to reduce CloudWatch costs:\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\n\n// Disable both metrics for cost optimization\nnew glue.PySparkEtlJob(stack, 'CostOptimizedJob', {\n role,\n script,\n enableMetrics: false,\n enableObservabilityMetrics: false,\n});\n\n// Selective control - keep observability, disable profiling\nnew glue.PySparkEtlJob(stack, 'SelectiveJob', {\n role,\n script,\n enableMetrics: false,\n // enableObservabilityMetrics defaults to true\n});\n```\n\nThis feature is available for all Spark job types (ETL, Streaming, Flex).\n\n### Enable Job Run Queuing\n\nAWS Glue job queuing monitors your account level quotas and limits. If quotas or limits are insufficient to start a Glue job run, AWS Glue will automatically queue the job and wait for limits to free up. Once limits become available, AWS Glue will retry the job run. Glue jobs will queue for limits like max concurrent job runs per account, max concurrent Data Processing Units (DPU), and resource unavailable due to IP address exhaustion in Amazon Virtual Private Cloud (Amazon VPC).\n\nEnable job run queuing by setting the `jobRunQueuingEnabled` property to `true`.\n\n```ts\nimport * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n role,\n script,\n jobName: 'PySparkETLJob',\n jobRunQueuingEnabled: true\n});\n```\n\n### Uploading scripts from the CDK app repository to S3\n\nSimilar to other L2 constructs, the Glue L2 automates uploading / updating\nscripts to S3 via an optional fromAsset parameter pointing to a script\nin the local file structure. You provide the existing S3 bucket and\npath to which you'd like the script to be uploaded.\n\nReference the unit tests for examples of repo and S3 code target examples.\n\n### Workflow Triggers\n\nYou can use Glue workflows to create and visualize complex\nextract, transform, and load (ETL) activities involving multiple crawlers,\njobs, and triggers. Standalone triggers are an anti-pattern, so you must\ncreate triggers from within a workflow using the L2 construct.\n\nWithin a workflow object, there are functions to create different\ntypes of triggers with actions and predicates. You then add those triggers\nto jobs.\n\nStartOnCreation defaults to true for all trigger types, but you can\noverride it if you prefer for your trigger not to start on creation.\n\nReference the workflow-triggers.test.ts unit tests for examples of creating\nworkflows and triggers.\n\n#### **1. On-Demand Triggers**\n\nOn-demand triggers can start glue jobs or crawlers. This construct provides\nconvenience functions to create on-demand crawler or job triggers. The constructor\ntakes an optional description parameter, but abstracts the requirement of an\nactions list using the job or crawler objects using conditional types.\n\n#### **2. Scheduled Triggers**\n\nYou can create scheduled triggers using cron expressions. This construct\nprovides daily, weekly, and monthly convenience functions,\nas well as a custom function that allows you to create your own\ncustom timing using the [existing event Schedule class](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_events.Schedule.html)\nwithout having to build your own cron expressions. The L2 extracts\nthe expression that Glue requires from the Schedule object. The constructor\ntakes an optional description and a list of jobs or crawlers as actions.\n\n#### **3. Notify Event Triggers**\n\nThere are two types of notify event triggers: batching and non-batching.\nFor batching triggers, you must specify `BatchSize`. For non-batching\ntriggers, `BatchSize` defaults to 1. For both triggers, `BatchWindow`\ndefaults to 900 seconds, but you can override the window to align with\nyour workload's requirements.\n\n#### **4. Conditional Triggers**\n\nConditional triggers have a predicate and actions associated with them.\nThe trigger actions are executed when the predicateCondition is true.\n\n### Connection Properties\n\nA `Connection` allows Glue jobs, crawlers and development endpoints to access\ncertain types of data stores.\n\n* **Secrets Management**\n You must specify JDBC connection credentials in Secrets Manager and\n provide the Secrets Manager Key name as a property to the job connection.\n\n* **Networking - the CDK determines the best fit subnet for Glue connection\nconfiguration**\n The prior version of the glue-alpha-module requires the developer to\n specify the subnet of the Connection when it’s defined. Now, you can still\n specify the specific subnet you want to use, but are no longer required\n to. You are only required to provide a VPC and either a public or private\n subnet selection. Without a specific subnet provided, the L2 leverages the\n existing [EC2 Subnet Selection](https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/SubnetSelection.html)\n library to make the best choice selection for the subnet.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\nnew glue.Connection(this, 'MyConnection', {\n type: glue.ConnectionType.NETWORK,\n // The security groups granting AWS Glue inbound access to the data source within the VPC\n securityGroups: [securityGroup],\n // The VPC subnet which contains the data source\n subnet,\n});\n```\n\nFor RDS `Connection` by JDBC, it is recommended to manage credentials using AWS Secrets Manager. To use Secret, specify `SECRET_ID` in `properties` like the following code. Note that in this case, the subnet must have a route to the AWS Secrets Manager VPC endpoint or to the AWS Secrets Manager endpoint through a NAT gateway.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\ndeclare const db: rds.DatabaseCluster;\nnew glue.Connection(this, \"RdsConnection\", {\n type: glue.ConnectionType.JDBC,\n securityGroups: [securityGroup],\n subnet,\n properties: {\n JDBC_CONNECTION_URL: `jdbc:mysql://${db.clusterEndpoint.socketAddress}/databasename`,\n JDBC_ENFORCE_SSL: \"false\",\n SECRET_ID: db.secret!.secretName,\n },\n});\n```\n\nIf you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.\n\nSee [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.\n\n## SecurityConfiguration\n\nA `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.\n\n```ts\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n },\n jobBookmarksEncryption: {\n mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,\n },\n s3Encryption: {\n mode: glue.S3EncryptionMode.KMS,\n },\n});\n```\n\nBy default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:\n\n```ts\ndeclare const key: kms.Key;\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n kmsKey: key,\n },\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.\n\n## Database\n\nA `Database` is a logical grouping of `Tables` in the Glue Catalog.\n\n```ts\nnew glue.Database(this, 'MyDatabase', {\n databaseName: 'my_database',\n description: 'my_database_description',\n});\n```\n\n## Table\n\nA Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }, {\n name: 'col2',\n type: glue.Schema.array(glue.Schema.STRING),\n comment: 'col2 is an array of strings' // comment is optional\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:\n\n```ts\ndeclare const myBucket: s3.Bucket;\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n bucket: myBucket,\n s3Prefix: 'my-table/',\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nGlue tables can be configured to contain user-defined properties, to describe the physical storage of table data, through the `storageParameters` property:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n storageParameters: [\n glue.StorageParameter.skipHeaderLineCount(1),\n glue.StorageParameter.compressionType(glue.CompressionType.GZIP),\n glue.StorageParameter.custom('separatorChar', ',')\n ],\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nGlue tables can also be configured to contain user-defined table properties through the [`parameters`](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-glue-table-tableinput.html#cfn-glue-table-tableinput-parameters) property:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n parameters: {\n key1: 'val1',\n key2: 'val2',\n },\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Keys\n\nTo improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Indexes\n\nAnother way to improve query performance is to specify partition indexes. If no partition indexes are\npresent on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using\nthe query expression. The query takes more time to run as the number of partitions increase. With an\nindex, the query will try to fetch a subset of the partitions instead of loading all partitions of the\ntable.\n\nThe keys of a partition index must be a subset of the partition keys of the table. You can have a\nmaximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`\nproperty:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n partitionIndexes: [{\n indexName: 'my-index', // optional\n keyNames: ['year'],\n }], // supply up to 3 indexes\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nAlternatively, you can call the `addPartitionIndex()` function on a table:\n\n```ts\ndeclare const myTable: glue.Table;\nmyTable.addPartitionIndex({\n indexName: 'my-index',\n keyNames: ['year'],\n});\n```\n\n### Partition Filtering\n\nIf you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n enablePartitionFiltering: true,\n});\n```\n\n### Partition Projection\n\nPartition projection allows Athena to automatically add new partitions as new data arrives, without requiring `ALTER TABLE ADD PARTITION` statements. This improves query performance and reduces management overhead by eliminating the need to manually manage partition metadata.\n\nFor more information, see the [AWS documentation on partition projection](https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html).\n\n#### INTEGER Projection\n\nFor partition keys with sequential numeric values:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.INTEGER,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n year: glue.PartitionProjectionConfiguration.integer({\n min: 2020,\n max: 2023,\n interval: 1, // optional, defaults to 1\n digits: 4, // optional, pads with leading zeros\n }),\n },\n});\n```\n\n#### DATE Projection\n\nFor partition keys with date or timestamp values. Supports both fixed dates and relative dates using `NOW`:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'date',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n date: glue.PartitionProjectionConfiguration.date({\n min: '2020-01-01',\n max: '2023-12-31',\n format: 'yyyy-MM-dd',\n interval: 1, // optional, defaults to 1\n intervalUnit: glue.DateIntervalUnit.DAYS, // optional: YEARS, MONTHS, WEEKS, DAYS, HOURS, MINUTES, SECONDS\n }),\n },\n});\n```\n\nYou can also use relative dates with `NOW`:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'date',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n date: glue.PartitionProjectionConfiguration.date({\n min: 'NOW-3YEARS',\n max: 'NOW',\n format: 'yyyy-MM-dd',\n }),\n },\n});\n```\n\n#### ENUM Projection\n\nFor partition keys with a known set of values:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'region',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n region: glue.PartitionProjectionConfiguration.enum({\n values: ['us-east-1', 'us-west-2', 'eu-west-1'],\n }),\n },\n});\n```\n\n#### INJECTED Projection\n\nFor custom partition values injected at query time:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'custom',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n custom: glue.PartitionProjectionConfiguration.injected(),\n },\n});\n```\n\n#### Multiple Partition Projections\n\nYou can configure partition projection for multiple partition keys:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'data',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [\n {\n name: 'year',\n type: glue.Schema.INTEGER,\n },\n {\n name: 'month',\n type: glue.Schema.INTEGER,\n },\n {\n name: 'region',\n type: glue.Schema.STRING,\n },\n ],\n dataFormat: glue.DataFormat.JSON,\n partitionProjection: {\n year: glue.PartitionProjectionConfiguration.integer({\n min: 2020,\n max: 2023,\n }),\n month: glue.PartitionProjectionConfiguration.integer({\n min: 1,\n max: 12,\n digits: 2,\n }),\n region: glue.PartitionProjectionConfiguration.enum({\n values: ['us-east-1', 'us-west-2'],\n }),\n },\n});\n```\n\n### Glue Connections\n\nGlue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog.\n\nWhereas `S3Table` will point to (and if needed, create) a bucket to store the tables' data, `ExternalTable` will point to an existing table in a data source. For example, to create a table in Glue that points to a table in Redshift:\n\n```ts\ndeclare const myConnection: glue.Connection;\ndeclare const myDatabase: glue.Database;\nnew glue.ExternalTable(this, 'MyTable', {\n connection: myConnection,\n externalDataLocation: 'default_db_public_example', // A table in Redshift\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)\n\nYou can enable encryption on a Table's data:\n\n* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - (default) Server side encryption (`SSE-S3`) with an Amazon S3-managed key.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.S3_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.S3Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n*Note: you cannot provide a `Bucket` when creating the `S3Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.\n\n## Types\n\nA table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.S3Table(this, 'MyTable', {\n columns: [{\n name: 'primitive_column',\n type: glue.Schema.STRING,\n }, {\n name: 'array_column',\n type: glue.Schema.array(glue.Schema.INTEGER),\n comment: 'array<integer>',\n }, {\n name: 'map_column',\n type: glue.Schema.map(\n glue.Schema.STRING,\n glue.Schema.TIMESTAMP),\n comment: 'map<string,string>',\n }, {\n name: 'struct_column',\n type: glue.Schema.struct([{\n name: 'nested_column',\n type: glue.Schema.DATE,\n comment: 'nested comment',\n }]),\n comment: \"struct<nested_column:date COMMENT 'nested comment'>\",\n }],\n // ...\n database: myDatabase,\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n## Public FAQ\n\n### What are we launching today?\n\nWe’re launching new features to an AWS CDK Glue L2 Construct to provide\nbest-practice defaults and convenience methods to create Glue Jobs, Connections,\nTriggers, Workflows, and the underlying permissions and configuration.\n\n### Why should I use this Construct?\n\nDevelopers should use this Construct to reduce the amount of boilerplate\ncode and complexity each individual has to navigate, and make it easier to\ncreate best-practice Glue resources.\n\n### What’s not in scope?\n\nGlue Crawlers and other resources that are now managed by the AWS LakeFormation\nteam are not in scope for this effort. Developers should use existing methods\nto create these resources, and the new Glue L2 construct assumes they already\nexist as inputs. While best practice is for application and infrastructure code\nto be as close as possible for teams using fully-implemented DevOps mechanisms,\nin practice these ETL scripts are likely managed by a data science team who\nknow Python or Scala and don’t necessarily own or manage their own\ninfrastructure deployments. We want to meet developers where they are, and not\nassume that all of the code resides in the same repository, Developers can\nautomate this themselves via the CDK, however, if they do own both.\n\nValidating Glue version and feature use per AWS region at synth time is also\nnot in scope. AWS’ intention is for all features to eventually be propagated to\nall Global regions, so the complexity involved in creating and updating region-\nspecific configuration to match shifting feature sets does not out-weigh the\nlikelihood that a developer will use this construct to deploy resources to a\nregion without a particular new feature to a region that doesn’t yet support\nit without researching or manually attempting to use that feature before\ndeveloping it via IaC. The developer will, of course, still get feedback from\nthe underlying Glue APIs as CloudFormation deploys the resources similar to the\ncurrent CDK L1 Glue experience.\n"
|
|
8991
8991
|
},
|
|
8992
8992
|
"repository": {
|
|
8993
8993
|
"directory": "packages/@aws-cdk/aws-glue-alpha",
|
|
@@ -9923,7 +9923,7 @@
|
|
|
9923
9923
|
"kind": "enum",
|
|
9924
9924
|
"locationInModule": {
|
|
9925
9925
|
"filename": "lib/constants.ts",
|
|
9926
|
-
"line":
|
|
9926
|
+
"line": 341
|
|
9927
9927
|
},
|
|
9928
9928
|
"members": [
|
|
9929
9929
|
{
|
|
@@ -9984,7 +9984,7 @@
|
|
|
9984
9984
|
"docs": {
|
|
9985
9985
|
"stability": "experimental",
|
|
9986
9986
|
"summary": "An AWS Glue connection to a data source.",
|
|
9987
|
-
"example": "import * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.
|
|
9987
|
+
"example": "import * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n jobName: 'PySparkETLJobCustomName',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});",
|
|
9988
9988
|
"custom": {
|
|
9989
9989
|
"exampleMetadata": "infused"
|
|
9990
9990
|
}
|
|
@@ -11270,7 +11270,7 @@
|
|
|
11270
11270
|
"kind": "enum",
|
|
11271
11271
|
"locationInModule": {
|
|
11272
11272
|
"filename": "lib/constants.ts",
|
|
11273
|
-
"line":
|
|
11273
|
+
"line": 349
|
|
11274
11274
|
},
|
|
11275
11275
|
"members": [
|
|
11276
11276
|
{
|
|
@@ -12706,7 +12706,7 @@
|
|
|
12706
12706
|
"kind": "enum",
|
|
12707
12707
|
"locationInModule": {
|
|
12708
12708
|
"filename": "lib/constants.ts",
|
|
12709
|
-
"line":
|
|
12709
|
+
"line": 172
|
|
12710
12710
|
},
|
|
12711
12711
|
"members": [
|
|
12712
12712
|
{
|
|
@@ -13041,7 +13041,7 @@
|
|
|
13041
13041
|
"kind": "enum",
|
|
13042
13042
|
"locationInModule": {
|
|
13043
13043
|
"filename": "lib/constants.ts",
|
|
13044
|
-
"line":
|
|
13044
|
+
"line": 191
|
|
13045
13045
|
},
|
|
13046
13046
|
"members": [
|
|
13047
13047
|
{
|
|
@@ -14206,7 +14206,7 @@
|
|
|
14206
14206
|
},
|
|
14207
14207
|
"locationInModule": {
|
|
14208
14208
|
"filename": "core/lib/resource.ts",
|
|
14209
|
-
"line":
|
|
14209
|
+
"line": 124
|
|
14210
14210
|
},
|
|
14211
14211
|
"parameters": [
|
|
14212
14212
|
{
|
|
@@ -14478,7 +14478,7 @@
|
|
|
14478
14478
|
},
|
|
14479
14479
|
"locationInModule": {
|
|
14480
14480
|
"filename": "core/lib/resource.ts",
|
|
14481
|
-
"line":
|
|
14481
|
+
"line": 124
|
|
14482
14482
|
},
|
|
14483
14483
|
"parameters": [
|
|
14484
14484
|
{
|
|
@@ -15028,7 +15028,7 @@
|
|
|
15028
15028
|
"kind": "enum",
|
|
15029
15029
|
"locationInModule": {
|
|
15030
15030
|
"filename": "lib/constants.ts",
|
|
15031
|
-
"line":
|
|
15031
|
+
"line": 231
|
|
15032
15032
|
},
|
|
15033
15033
|
"members": [
|
|
15034
15034
|
{
|
|
@@ -15389,7 +15389,7 @@
|
|
|
15389
15389
|
"kind": "enum",
|
|
15390
15390
|
"locationInModule": {
|
|
15391
15391
|
"filename": "lib/constants.ts",
|
|
15392
|
-
"line":
|
|
15392
|
+
"line": 112
|
|
15393
15393
|
},
|
|
15394
15394
|
"members": [
|
|
15395
15395
|
{
|
|
@@ -15455,7 +15455,7 @@
|
|
|
15455
15455
|
"kind": "enum",
|
|
15456
15456
|
"locationInModule": {
|
|
15457
15457
|
"filename": "lib/constants.ts",
|
|
15458
|
-
"line":
|
|
15458
|
+
"line": 281
|
|
15459
15459
|
},
|
|
15460
15460
|
"members": [
|
|
15461
15461
|
{
|
|
@@ -15481,7 +15481,8 @@
|
|
|
15481
15481
|
},
|
|
15482
15482
|
{
|
|
15483
15483
|
"docs": {
|
|
15484
|
-
"
|
|
15484
|
+
"deprecated": "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html",
|
|
15485
|
+
"stability": "deprecated",
|
|
15485
15486
|
"summary": "Command for running a Glue Ray job."
|
|
15486
15487
|
},
|
|
15487
15488
|
"name": "RAY"
|
|
@@ -15505,7 +15506,7 @@
|
|
|
15505
15506
|
"kind": "enum",
|
|
15506
15507
|
"locationInModule": {
|
|
15507
15508
|
"filename": "lib/constants.ts",
|
|
15508
|
-
"line":
|
|
15509
|
+
"line": 311
|
|
15509
15510
|
},
|
|
15510
15511
|
"members": [
|
|
15511
15512
|
{
|
|
@@ -15537,7 +15538,7 @@
|
|
|
15537
15538
|
"kind": "enum",
|
|
15538
15539
|
"locationInModule": {
|
|
15539
15540
|
"filename": "lib/constants.ts",
|
|
15540
|
-
"line":
|
|
15541
|
+
"line": 154
|
|
15541
15542
|
},
|
|
15542
15543
|
"members": [
|
|
15543
15544
|
{
|
|
@@ -16268,7 +16269,7 @@
|
|
|
16268
16269
|
"kind": "enum",
|
|
16269
16270
|
"locationInModule": {
|
|
16270
16271
|
"filename": "lib/constants.ts",
|
|
16271
|
-
"line":
|
|
16272
|
+
"line": 326
|
|
16272
16273
|
},
|
|
16273
16274
|
"members": [
|
|
16274
16275
|
{
|
|
@@ -16309,7 +16310,7 @@
|
|
|
16309
16310
|
},
|
|
16310
16311
|
"locationInModule": {
|
|
16311
16312
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16312
|
-
"line":
|
|
16313
|
+
"line": 90
|
|
16313
16314
|
},
|
|
16314
16315
|
"parameters": [
|
|
16315
16316
|
{
|
|
@@ -16335,7 +16336,7 @@
|
|
|
16335
16336
|
"kind": "class",
|
|
16336
16337
|
"locationInModule": {
|
|
16337
16338
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16338
|
-
"line":
|
|
16339
|
+
"line": 81
|
|
16339
16340
|
},
|
|
16340
16341
|
"name": "PySparkEtlJob",
|
|
16341
16342
|
"properties": [
|
|
@@ -16348,7 +16349,7 @@
|
|
|
16348
16349
|
"immutable": true,
|
|
16349
16350
|
"locationInModule": {
|
|
16350
16351
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16351
|
-
"line":
|
|
16352
|
+
"line": 84
|
|
16352
16353
|
},
|
|
16353
16354
|
"name": "PROPERTY_INJECTION_ID",
|
|
16354
16355
|
"static": true,
|
|
@@ -16364,7 +16365,7 @@
|
|
|
16364
16365
|
"immutable": true,
|
|
16365
16366
|
"locationInModule": {
|
|
16366
16367
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16367
|
-
"line":
|
|
16368
|
+
"line": 125
|
|
16368
16369
|
},
|
|
16369
16370
|
"name": "jobArn",
|
|
16370
16371
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -16380,7 +16381,7 @@
|
|
|
16380
16381
|
"immutable": true,
|
|
16381
16382
|
"locationInModule": {
|
|
16382
16383
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16383
|
-
"line":
|
|
16384
|
+
"line": 130
|
|
16384
16385
|
},
|
|
16385
16386
|
"name": "jobName",
|
|
16386
16387
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -16409,7 +16410,7 @@
|
|
|
16409
16410
|
"kind": "interface",
|
|
16410
16411
|
"locationInModule": {
|
|
16411
16412
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16412
|
-
"line":
|
|
16413
|
+
"line": 15
|
|
16413
16414
|
},
|
|
16414
16415
|
"name": "PySparkEtlJobProps",
|
|
16415
16416
|
"properties": [
|
|
@@ -16424,7 +16425,7 @@
|
|
|
16424
16425
|
"immutable": true,
|
|
16425
16426
|
"locationInModule": {
|
|
16426
16427
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16427
|
-
"line":
|
|
16428
|
+
"line": 38
|
|
16428
16429
|
},
|
|
16429
16430
|
"name": "extraFiles",
|
|
16430
16431
|
"optional": true,
|
|
@@ -16447,7 +16448,7 @@
|
|
|
16447
16448
|
"immutable": true,
|
|
16448
16449
|
"locationInModule": {
|
|
16449
16450
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16450
|
-
"line":
|
|
16451
|
+
"line": 45
|
|
16451
16452
|
},
|
|
16452
16453
|
"name": "extraJars",
|
|
16453
16454
|
"optional": true,
|
|
@@ -16471,7 +16472,7 @@
|
|
|
16471
16472
|
"immutable": true,
|
|
16472
16473
|
"locationInModule": {
|
|
16473
16474
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16474
|
-
"line":
|
|
16475
|
+
"line": 54
|
|
16475
16476
|
},
|
|
16476
16477
|
"name": "extraJarsFirst",
|
|
16477
16478
|
"optional": true,
|
|
@@ -16489,7 +16490,7 @@
|
|
|
16489
16490
|
"immutable": true,
|
|
16490
16491
|
"locationInModule": {
|
|
16491
16492
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16492
|
-
"line":
|
|
16493
|
+
"line": 29
|
|
16493
16494
|
},
|
|
16494
16495
|
"name": "extraPythonFiles",
|
|
16495
16496
|
"optional": true,
|
|
@@ -16513,13 +16514,32 @@
|
|
|
16513
16514
|
"immutable": true,
|
|
16514
16515
|
"locationInModule": {
|
|
16515
16516
|
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16516
|
-
"line":
|
|
16517
|
+
"line": 66
|
|
16517
16518
|
},
|
|
16518
16519
|
"name": "jobRunQueuingEnabled",
|
|
16519
16520
|
"optional": true,
|
|
16520
16521
|
"type": {
|
|
16521
16522
|
"primitive": "boolean"
|
|
16522
16523
|
}
|
|
16524
|
+
},
|
|
16525
|
+
{
|
|
16526
|
+
"abstract": true,
|
|
16527
|
+
"docs": {
|
|
16528
|
+
"default": "- undefined",
|
|
16529
|
+
"remarks": "After a job run starts, the number of minutes to wait before sending a job run delay notification.",
|
|
16530
|
+
"stability": "experimental",
|
|
16531
|
+
"summary": "Specifies configuration properties of a notification (optional)."
|
|
16532
|
+
},
|
|
16533
|
+
"immutable": true,
|
|
16534
|
+
"locationInModule": {
|
|
16535
|
+
"filename": "lib/jobs/pyspark-etl-job.ts",
|
|
16536
|
+
"line": 21
|
|
16537
|
+
},
|
|
16538
|
+
"name": "notifyDelayAfter",
|
|
16539
|
+
"optional": true,
|
|
16540
|
+
"type": {
|
|
16541
|
+
"fqn": "aws-cdk-lib.Duration"
|
|
16542
|
+
}
|
|
16523
16543
|
}
|
|
16524
16544
|
],
|
|
16525
16545
|
"symbolId": "lib/jobs/pyspark-etl-job:PySparkEtlJobProps"
|
|
@@ -17247,7 +17267,7 @@
|
|
|
17247
17267
|
"kind": "enum",
|
|
17248
17268
|
"locationInModule": {
|
|
17249
17269
|
"filename": "lib/constants.ts",
|
|
17250
|
-
"line":
|
|
17270
|
+
"line": 246
|
|
17251
17271
|
},
|
|
17252
17272
|
"members": [
|
|
17253
17273
|
{
|
|
@@ -17279,23 +17299,24 @@
|
|
|
17279
17299
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
17280
17300
|
"base": "@aws-cdk/aws-glue-alpha.Job",
|
|
17281
17301
|
"docs": {
|
|
17302
|
+
"deprecated": "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html",
|
|
17282
17303
|
"remarks": "Glue Ray jobs use worker type Z.2X and Glue version 4.0.\nThese are not overrideable since these are the only configuration that\nGlue Ray jobs currently support. The runtime defaults to Ray2.4 and min\nworkers defaults to 3.",
|
|
17283
|
-
"stability": "
|
|
17304
|
+
"stability": "deprecated",
|
|
17284
17305
|
"summary": "Ray Jobs class.",
|
|
17285
|
-
"example": "
|
|
17306
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\n\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst rayJob = new glue_alpha.RayJob(this, 'MyRayJob', {\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableMetrics: false,\n enableObservabilityMetrics: false,\n enableProfilingMetrics: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n runtime: glue_alpha.Runtime.RAY_TWO_FOUR,\n securityConfiguration: securityConfiguration,\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n});",
|
|
17286
17307
|
"custom": {
|
|
17287
|
-
"exampleMetadata": "
|
|
17308
|
+
"exampleMetadata": "fixture=_generated"
|
|
17288
17309
|
}
|
|
17289
17310
|
},
|
|
17290
17311
|
"fqn": "@aws-cdk/aws-glue-alpha.RayJob",
|
|
17291
17312
|
"initializer": {
|
|
17292
17313
|
"docs": {
|
|
17293
|
-
"stability": "
|
|
17314
|
+
"stability": "deprecated",
|
|
17294
17315
|
"summary": "RayJob constructor."
|
|
17295
17316
|
},
|
|
17296
17317
|
"locationInModule": {
|
|
17297
17318
|
"filename": "lib/jobs/ray-job.ts",
|
|
17298
|
-
"line":
|
|
17319
|
+
"line": 81
|
|
17299
17320
|
},
|
|
17300
17321
|
"parameters": [
|
|
17301
17322
|
{
|
|
@@ -17321,20 +17342,20 @@
|
|
|
17321
17342
|
"kind": "class",
|
|
17322
17343
|
"locationInModule": {
|
|
17323
17344
|
"filename": "lib/jobs/ray-job.ts",
|
|
17324
|
-
"line":
|
|
17345
|
+
"line": 70
|
|
17325
17346
|
},
|
|
17326
17347
|
"name": "RayJob",
|
|
17327
17348
|
"properties": [
|
|
17328
17349
|
{
|
|
17329
17350
|
"const": true,
|
|
17330
17351
|
"docs": {
|
|
17331
|
-
"stability": "
|
|
17352
|
+
"stability": "deprecated",
|
|
17332
17353
|
"summary": "Uniquely identifies this class."
|
|
17333
17354
|
},
|
|
17334
17355
|
"immutable": true,
|
|
17335
17356
|
"locationInModule": {
|
|
17336
17357
|
"filename": "lib/jobs/ray-job.ts",
|
|
17337
|
-
"line":
|
|
17358
|
+
"line": 73
|
|
17338
17359
|
},
|
|
17339
17360
|
"name": "PROPERTY_INJECTION_ID",
|
|
17340
17361
|
"static": true,
|
|
@@ -17344,13 +17365,13 @@
|
|
|
17344
17365
|
},
|
|
17345
17366
|
{
|
|
17346
17367
|
"docs": {
|
|
17347
|
-
"stability": "
|
|
17368
|
+
"stability": "deprecated",
|
|
17348
17369
|
"summary": "The principal to grant permissions to."
|
|
17349
17370
|
},
|
|
17350
17371
|
"immutable": true,
|
|
17351
17372
|
"locationInModule": {
|
|
17352
17373
|
"filename": "lib/jobs/ray-job.ts",
|
|
17353
|
-
"line":
|
|
17374
|
+
"line": 75
|
|
17354
17375
|
},
|
|
17355
17376
|
"name": "grantPrincipal",
|
|
17356
17377
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -17360,13 +17381,13 @@
|
|
|
17360
17381
|
},
|
|
17361
17382
|
{
|
|
17362
17383
|
"docs": {
|
|
17363
|
-
"stability": "
|
|
17384
|
+
"stability": "deprecated",
|
|
17364
17385
|
"summary": "The ARN of the job."
|
|
17365
17386
|
},
|
|
17366
17387
|
"immutable": true,
|
|
17367
17388
|
"locationInModule": {
|
|
17368
17389
|
"filename": "lib/jobs/ray-job.ts",
|
|
17369
|
-
"line":
|
|
17390
|
+
"line": 134
|
|
17370
17391
|
},
|
|
17371
17392
|
"name": "jobArn",
|
|
17372
17393
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -17376,13 +17397,13 @@
|
|
|
17376
17397
|
},
|
|
17377
17398
|
{
|
|
17378
17399
|
"docs": {
|
|
17379
|
-
"stability": "
|
|
17400
|
+
"stability": "deprecated",
|
|
17380
17401
|
"summary": "The name of the job."
|
|
17381
17402
|
},
|
|
17382
17403
|
"immutable": true,
|
|
17383
17404
|
"locationInModule": {
|
|
17384
17405
|
"filename": "lib/jobs/ray-job.ts",
|
|
17385
|
-
"line":
|
|
17406
|
+
"line": 139
|
|
17386
17407
|
},
|
|
17387
17408
|
"name": "jobName",
|
|
17388
17409
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -17392,13 +17413,13 @@
|
|
|
17392
17413
|
},
|
|
17393
17414
|
{
|
|
17394
17415
|
"docs": {
|
|
17395
|
-
"stability": "
|
|
17416
|
+
"stability": "deprecated",
|
|
17396
17417
|
"summary": "The IAM role Glue assumes to run this job."
|
|
17397
17418
|
},
|
|
17398
17419
|
"immutable": true,
|
|
17399
17420
|
"locationInModule": {
|
|
17400
17421
|
"filename": "lib/jobs/ray-job.ts",
|
|
17401
|
-
"line":
|
|
17422
|
+
"line": 74
|
|
17402
17423
|
},
|
|
17403
17424
|
"name": "role",
|
|
17404
17425
|
"overrides": "@aws-cdk/aws-glue-alpha.Job",
|
|
@@ -17413,11 +17434,12 @@
|
|
|
17413
17434
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
17414
17435
|
"datatype": true,
|
|
17415
17436
|
"docs": {
|
|
17416
|
-
"
|
|
17437
|
+
"deprecated": "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html",
|
|
17438
|
+
"stability": "deprecated",
|
|
17417
17439
|
"summary": "Properties for creating a Ray Glue job.",
|
|
17418
|
-
"example": "
|
|
17440
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\n\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst rayJobProps: glue_alpha.RayJobProps = {\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableMetrics: false,\n enableObservabilityMetrics: false,\n enableProfilingMetrics: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n runtime: glue_alpha.Runtime.RAY_TWO_FOUR,\n securityConfiguration: securityConfiguration,\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
17419
17441
|
"custom": {
|
|
17420
|
-
"exampleMetadata": "
|
|
17442
|
+
"exampleMetadata": "fixture=_generated"
|
|
17421
17443
|
}
|
|
17422
17444
|
},
|
|
17423
17445
|
"fqn": "@aws-cdk/aws-glue-alpha.RayJobProps",
|
|
@@ -17427,7 +17449,7 @@
|
|
|
17427
17449
|
"kind": "interface",
|
|
17428
17450
|
"locationInModule": {
|
|
17429
17451
|
"filename": "lib/jobs/ray-job.ts",
|
|
17430
|
-
"line":
|
|
17452
|
+
"line": 19
|
|
17431
17453
|
},
|
|
17432
17454
|
"name": "RayJobProps",
|
|
17433
17455
|
"properties": [
|
|
@@ -17436,13 +17458,13 @@
|
|
|
17436
17458
|
"docs": {
|
|
17437
17459
|
"default": "true",
|
|
17438
17460
|
"remarks": "When enabled, adds '--enable-metrics' to job arguments.",
|
|
17439
|
-
"stability": "
|
|
17461
|
+
"stability": "deprecated",
|
|
17440
17462
|
"summary": "Enable profiling metrics for the Glue job."
|
|
17441
17463
|
},
|
|
17442
17464
|
"immutable": true,
|
|
17443
17465
|
"locationInModule": {
|
|
17444
17466
|
"filename": "lib/jobs/ray-job.ts",
|
|
17445
|
-
"line":
|
|
17467
|
+
"line": 46
|
|
17446
17468
|
},
|
|
17447
17469
|
"name": "enableMetrics",
|
|
17448
17470
|
"optional": true,
|
|
@@ -17455,13 +17477,13 @@
|
|
|
17455
17477
|
"docs": {
|
|
17456
17478
|
"default": "true",
|
|
17457
17479
|
"remarks": "When enabled, adds '--enable-observability-metrics': 'true' to job arguments.",
|
|
17458
|
-
"stability": "
|
|
17480
|
+
"stability": "deprecated",
|
|
17459
17481
|
"summary": "Enable observability metrics for the Glue job."
|
|
17460
17482
|
},
|
|
17461
17483
|
"immutable": true,
|
|
17462
17484
|
"locationInModule": {
|
|
17463
17485
|
"filename": "lib/jobs/ray-job.ts",
|
|
17464
|
-
"line":
|
|
17486
|
+
"line": 55
|
|
17465
17487
|
},
|
|
17466
17488
|
"name": "enableObservabilityMetrics",
|
|
17467
17489
|
"optional": true,
|
|
@@ -17474,13 +17496,13 @@
|
|
|
17474
17496
|
"docs": {
|
|
17475
17497
|
"default": "- no job run queuing",
|
|
17476
17498
|
"remarks": "A value of true means job run queuing is enabled for the job runs.\nIf false or not populated, the job runs will not be considered for queueing.\nIf this field does not match the value set in the job run, then the value from\nthe job run field will be used. This property must be set to false for flex jobs.\nIf this property is enabled, maxRetries must be set to zero.",
|
|
17477
|
-
"stability": "
|
|
17499
|
+
"stability": "deprecated",
|
|
17478
17500
|
"summary": "Specifies whether job run queuing is enabled for the job runs for this job."
|
|
17479
17501
|
},
|
|
17480
17502
|
"immutable": true,
|
|
17481
17503
|
"locationInModule": {
|
|
17482
17504
|
"filename": "lib/jobs/ray-job.ts",
|
|
17483
|
-
"line":
|
|
17505
|
+
"line": 37
|
|
17484
17506
|
},
|
|
17485
17507
|
"name": "jobRunQueuingEnabled",
|
|
17486
17508
|
"optional": true,
|
|
@@ -17492,13 +17514,13 @@
|
|
|
17492
17514
|
"abstract": true,
|
|
17493
17515
|
"docs": {
|
|
17494
17516
|
"default": "- Runtime version will default to Ray2.4",
|
|
17495
|
-
"stability": "
|
|
17517
|
+
"stability": "deprecated",
|
|
17496
17518
|
"summary": "Sets the Ray runtime environment version."
|
|
17497
17519
|
},
|
|
17498
17520
|
"immutable": true,
|
|
17499
17521
|
"locationInModule": {
|
|
17500
17522
|
"filename": "lib/jobs/ray-job.ts",
|
|
17501
|
-
"line":
|
|
17523
|
+
"line": 25
|
|
17502
17524
|
},
|
|
17503
17525
|
"name": "runtime",
|
|
17504
17526
|
"optional": true,
|
|
@@ -17512,23 +17534,20 @@
|
|
|
17512
17534
|
"@aws-cdk/aws-glue-alpha.Runtime": {
|
|
17513
17535
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
17514
17536
|
"docs": {
|
|
17515
|
-
"
|
|
17516
|
-
"
|
|
17517
|
-
"
|
|
17518
|
-
"custom": {
|
|
17519
|
-
"exampleMetadata": "infused"
|
|
17520
|
-
}
|
|
17537
|
+
"deprecated": "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nMigrate to Amazon EKS with KubeRay Operator. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html",
|
|
17538
|
+
"stability": "deprecated",
|
|
17539
|
+
"summary": "AWS Glue runtime determines the runtime engine of the job."
|
|
17521
17540
|
},
|
|
17522
17541
|
"fqn": "@aws-cdk/aws-glue-alpha.Runtime",
|
|
17523
17542
|
"kind": "enum",
|
|
17524
17543
|
"locationInModule": {
|
|
17525
17544
|
"filename": "lib/constants.ts",
|
|
17526
|
-
"line":
|
|
17545
|
+
"line": 271
|
|
17527
17546
|
},
|
|
17528
17547
|
"members": [
|
|
17529
17548
|
{
|
|
17530
17549
|
"docs": {
|
|
17531
|
-
"stability": "
|
|
17550
|
+
"stability": "deprecated",
|
|
17532
17551
|
"summary": "Runtime for a Glue for Ray 2.4."
|
|
17533
17552
|
},
|
|
17534
17553
|
"name": "RAY_TWO_FOUR"
|
|
@@ -18118,7 +18137,7 @@
|
|
|
18118
18137
|
"remarks": "ETL jobs support pySpark and Scala languages, for which there are separate\nbut similar constructors. ETL jobs default to the G2 worker type, but you\ncan override this default with other supported worker type values\n(G1, G2, G4 and G8). ETL jobs defaults to Glue version 4.0, which you can\noverride to 3.0. The following ETL features are enabled by default:\n—enable-metrics, —enable-spark-ui, —enable-continuous-cloudwatch-log.\nYou can find more details about version, worker type and other features\nin Glue's public documentation.",
|
|
18119
18138
|
"stability": "experimental",
|
|
18120
18139
|
"summary": "Spark ETL Jobs class.",
|
|
18121
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJob = new glue_alpha.ScalaSparkEtlJob(this, 'MyScalaSparkEtlJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableMetrics: false,\n enableObservabilityMetrics: false,\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n});",
|
|
18140
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJob = new glue_alpha.ScalaSparkEtlJob(this, 'MyScalaSparkEtlJob', {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableMetrics: false,\n enableObservabilityMetrics: false,\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n notifyDelayAfter: cdk.Duration.minutes(30),\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n});",
|
|
18122
18141
|
"custom": {
|
|
18123
18142
|
"exampleMetadata": "fixture=_generated"
|
|
18124
18143
|
}
|
|
@@ -18131,7 +18150,7 @@
|
|
|
18131
18150
|
},
|
|
18132
18151
|
"locationInModule": {
|
|
18133
18152
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18134
|
-
"line":
|
|
18153
|
+
"line": 90
|
|
18135
18154
|
},
|
|
18136
18155
|
"parameters": [
|
|
18137
18156
|
{
|
|
@@ -18157,7 +18176,7 @@
|
|
|
18157
18176
|
"kind": "class",
|
|
18158
18177
|
"locationInModule": {
|
|
18159
18178
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18160
|
-
"line":
|
|
18179
|
+
"line": 81
|
|
18161
18180
|
},
|
|
18162
18181
|
"name": "ScalaSparkEtlJob",
|
|
18163
18182
|
"properties": [
|
|
@@ -18170,7 +18189,7 @@
|
|
|
18170
18189
|
"immutable": true,
|
|
18171
18190
|
"locationInModule": {
|
|
18172
18191
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18173
|
-
"line":
|
|
18192
|
+
"line": 84
|
|
18174
18193
|
},
|
|
18175
18194
|
"name": "PROPERTY_INJECTION_ID",
|
|
18176
18195
|
"static": true,
|
|
@@ -18186,7 +18205,7 @@
|
|
|
18186
18205
|
"immutable": true,
|
|
18187
18206
|
"locationInModule": {
|
|
18188
18207
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18189
|
-
"line":
|
|
18208
|
+
"line": 128
|
|
18190
18209
|
},
|
|
18191
18210
|
"name": "jobArn",
|
|
18192
18211
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -18202,7 +18221,7 @@
|
|
|
18202
18221
|
"immutable": true,
|
|
18203
18222
|
"locationInModule": {
|
|
18204
18223
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18205
|
-
"line":
|
|
18224
|
+
"line": 133
|
|
18206
18225
|
},
|
|
18207
18226
|
"name": "jobName",
|
|
18208
18227
|
"overrides": "@aws-cdk/aws-glue-alpha.JobBase",
|
|
@@ -18219,7 +18238,7 @@
|
|
|
18219
18238
|
"docs": {
|
|
18220
18239
|
"stability": "experimental",
|
|
18221
18240
|
"summary": "Properties for creating a Scala Spark ETL job.",
|
|
18222
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJobProps: glue_alpha.ScalaSparkEtlJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableMetrics: false,\n enableObservabilityMetrics: false,\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
18241
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\nimport { aws_logs as logs } from 'aws-cdk-lib';\nimport { aws_s3 as s3 } from 'aws-cdk-lib';\n\ndeclare const bucket: s3.Bucket;\ndeclare const code: glue_alpha.Code;\ndeclare const connection: glue_alpha.Connection;\ndeclare const logGroup: logs.LogGroup;\ndeclare const role: iam.Role;\ndeclare const securityConfiguration: glue_alpha.SecurityConfiguration;\nconst scalaSparkEtlJobProps: glue_alpha.ScalaSparkEtlJobProps = {\n className: 'className',\n role: role,\n script: code,\n\n // the properties below are optional\n connections: [connection],\n continuousLogging: {\n enabled: false,\n\n // the properties below are optional\n conversionPattern: 'conversionPattern',\n logGroup: logGroup,\n logStreamPrefix: 'logStreamPrefix',\n quiet: false,\n },\n defaultArguments: {\n defaultArgumentsKey: 'defaultArguments',\n },\n description: 'description',\n enableMetrics: false,\n enableObservabilityMetrics: false,\n enableProfilingMetrics: false,\n extraFiles: [code],\n extraJars: [code],\n extraJarsFirst: false,\n glueVersion: glue_alpha.GlueVersion.V0_9,\n jobName: 'jobName',\n jobRunQueuingEnabled: false,\n maxConcurrentRuns: 123,\n maxRetries: 123,\n notifyDelayAfter: cdk.Duration.minutes(30),\n numberOfWorkers: 123,\n securityConfiguration: securityConfiguration,\n sparkUI: {\n bucket: bucket,\n prefix: 'prefix',\n },\n tags: {\n tagsKey: 'tags',\n },\n timeout: cdk.Duration.minutes(30),\n workerType: glue_alpha.WorkerType.STANDARD,\n};",
|
|
18223
18242
|
"custom": {
|
|
18224
18243
|
"exampleMetadata": "fixture=_generated"
|
|
18225
18244
|
}
|
|
@@ -18231,7 +18250,7 @@
|
|
|
18231
18250
|
"kind": "interface",
|
|
18232
18251
|
"locationInModule": {
|
|
18233
18252
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18234
|
-
"line":
|
|
18253
|
+
"line": 16
|
|
18235
18254
|
},
|
|
18236
18255
|
"name": "ScalaSparkEtlJobProps",
|
|
18237
18256
|
"properties": [
|
|
@@ -18244,7 +18263,7 @@
|
|
|
18244
18263
|
"immutable": true,
|
|
18245
18264
|
"locationInModule": {
|
|
18246
18265
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18247
|
-
"line":
|
|
18266
|
+
"line": 22
|
|
18248
18267
|
},
|
|
18249
18268
|
"name": "className",
|
|
18250
18269
|
"type": {
|
|
@@ -18262,7 +18281,7 @@
|
|
|
18262
18281
|
"immutable": true,
|
|
18263
18282
|
"locationInModule": {
|
|
18264
18283
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18265
|
-
"line":
|
|
18284
|
+
"line": 38
|
|
18266
18285
|
},
|
|
18267
18286
|
"name": "extraFiles",
|
|
18268
18287
|
"optional": true,
|
|
@@ -18285,7 +18304,7 @@
|
|
|
18285
18304
|
"immutable": true,
|
|
18286
18305
|
"locationInModule": {
|
|
18287
18306
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18288
|
-
"line":
|
|
18307
|
+
"line": 45
|
|
18289
18308
|
},
|
|
18290
18309
|
"name": "extraJars",
|
|
18291
18310
|
"optional": true,
|
|
@@ -18309,7 +18328,7 @@
|
|
|
18309
18328
|
"immutable": true,
|
|
18310
18329
|
"locationInModule": {
|
|
18311
18330
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18312
|
-
"line":
|
|
18331
|
+
"line": 54
|
|
18313
18332
|
},
|
|
18314
18333
|
"name": "extraJarsFirst",
|
|
18315
18334
|
"optional": true,
|
|
@@ -18328,13 +18347,32 @@
|
|
|
18328
18347
|
"immutable": true,
|
|
18329
18348
|
"locationInModule": {
|
|
18330
18349
|
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18331
|
-
"line":
|
|
18350
|
+
"line": 66
|
|
18332
18351
|
},
|
|
18333
18352
|
"name": "jobRunQueuingEnabled",
|
|
18334
18353
|
"optional": true,
|
|
18335
18354
|
"type": {
|
|
18336
18355
|
"primitive": "boolean"
|
|
18337
18356
|
}
|
|
18357
|
+
},
|
|
18358
|
+
{
|
|
18359
|
+
"abstract": true,
|
|
18360
|
+
"docs": {
|
|
18361
|
+
"default": "- undefined",
|
|
18362
|
+
"remarks": "After a job run starts, the number of minutes to wait before sending a job run delay notification.",
|
|
18363
|
+
"stability": "experimental",
|
|
18364
|
+
"summary": "Specifies configuration properties of a notification (optional)."
|
|
18365
|
+
},
|
|
18366
|
+
"immutable": true,
|
|
18367
|
+
"locationInModule": {
|
|
18368
|
+
"filename": "lib/jobs/scala-spark-etl-job.ts",
|
|
18369
|
+
"line": 29
|
|
18370
|
+
},
|
|
18371
|
+
"name": "notifyDelayAfter",
|
|
18372
|
+
"optional": true,
|
|
18373
|
+
"type": {
|
|
18374
|
+
"fqn": "aws-cdk-lib.Duration"
|
|
18375
|
+
}
|
|
18338
18376
|
}
|
|
18339
18377
|
],
|
|
18340
18378
|
"symbolId": "lib/jobs/scala-spark-etl-job:ScalaSparkEtlJobProps"
|
|
@@ -19208,7 +19246,7 @@
|
|
|
19208
19246
|
"remarks": "The following scenarios show some of the ways that you can use a security configuration.\n- Attach a security configuration to an AWS Glue crawler to write encrypted Amazon CloudWatch Logs.\n- Attach a security configuration to an extract, transform, and load (ETL) job to write encrypted Amazon Simple Storage Service (Amazon S3) targets and encrypted CloudWatch Logs.\n- Attach a security configuration to an ETL job to write its jobs bookmarks as encrypted Amazon S3 data.\n- Attach a security configuration to a development endpoint to write encrypted Amazon S3 targets.",
|
|
19209
19247
|
"stability": "experimental",
|
|
19210
19248
|
"summary": "A security configuration is a set of security properties that can be used by AWS Glue to encrypt data at rest.",
|
|
19211
|
-
"example": "import * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.
|
|
19249
|
+
"example": "import * as cdk from 'aws-cdk-lib';\nimport * as iam from 'aws-cdk-lib/aws-iam';\ndeclare const stack: cdk.Stack;\ndeclare const role: iam.IRole;\ndeclare const script: glue.Code;\nnew glue.PySparkEtlJob(stack, 'PySparkETLJob', {\n jobName: 'PySparkETLJobCustomName',\n description: 'This is a description',\n role,\n script,\n glueVersion: glue.GlueVersion.V5_1,\n continuousLogging: { enabled: false },\n workerType: glue.WorkerType.G_2X,\n maxConcurrentRuns: 100,\n timeout: cdk.Duration.hours(2),\n connections: [glue.Connection.fromConnectionName(stack, 'Connection', 'connectionName')],\n securityConfiguration: glue.SecurityConfiguration.fromSecurityConfigurationName(stack, 'SecurityConfig', 'securityConfigName'),\n tags: {\n FirstTagName: 'FirstTagValue',\n SecondTagName: 'SecondTagValue',\n XTagName: 'XTagValue',\n },\n numberOfWorkers: 2,\n maxRetries: 2,\n});",
|
|
19212
19250
|
"custom": {
|
|
19213
19251
|
"exampleMetadata": "infused"
|
|
19214
19252
|
}
|
|
@@ -22251,7 +22289,8 @@
|
|
|
22251
22289
|
},
|
|
22252
22290
|
{
|
|
22253
22291
|
"docs": {
|
|
22254
|
-
"
|
|
22292
|
+
"deprecated": "AWS Glue for Ray is closed to new customers as of April 30, 2026.\nThis worker type was only used for Ray jobs. See\nhttps://docs.aws.amazon.com/glue/latest/dg/awsglue-ray-jobs-availability-change.html",
|
|
22293
|
+
"stability": "deprecated",
|
|
22255
22294
|
"summary": "Z.2X Worker Type."
|
|
22256
22295
|
},
|
|
22257
22296
|
"name": "Z_2X"
|
|
@@ -22576,7 +22615,7 @@
|
|
|
22576
22615
|
},
|
|
22577
22616
|
"locationInModule": {
|
|
22578
22617
|
"filename": "core/lib/resource.ts",
|
|
22579
|
-
"line":
|
|
22618
|
+
"line": 124
|
|
22580
22619
|
},
|
|
22581
22620
|
"parameters": [
|
|
22582
22621
|
{
|
|
@@ -23088,6 +23127,6 @@
|
|
|
23088
23127
|
"symbolId": "lib/storage-parameter:WriteParallel"
|
|
23089
23128
|
}
|
|
23090
23129
|
},
|
|
23091
|
-
"version": "2.
|
|
23130
|
+
"version": "2.258.1-alpha.0",
|
|
23092
23131
|
"fingerprint": "**********"
|
|
23093
23132
|
}
|