@aws-cdk/aws-glue-alpha 2.80.0-alpha.0 → 2.82.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +125 -76
- package/.jsii.tabl.json.gz +0 -0
- package/README.md +1 -0
- package/lib/code.js +3 -3
- package/lib/connection.js +2 -2
- package/lib/data-format.js +5 -5
- package/lib/database.js +1 -1
- package/lib/job-executable.js +3 -3
- package/lib/job.d.ts +8 -0
- package/lib/job.js +11 -3
- package/lib/schema.js +1 -1
- package/lib/security-configuration.js +1 -1
- package/lib/table.js +1 -1
- package/package.json +7 -7
package/.jsii
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"url": "https://aws.amazon.com"
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"aws-cdk-lib": "2.
|
|
11
|
+
"aws-cdk-lib": "2.82.0",
|
|
12
12
|
"constructs": "^10.0.0"
|
|
13
13
|
},
|
|
14
14
|
"dependencyClosure": {
|
|
@@ -2953,6 +2953,19 @@
|
|
|
2953
2953
|
}
|
|
2954
2954
|
}
|
|
2955
2955
|
},
|
|
2956
|
+
"aws-cdk-lib.aws_shield": {
|
|
2957
|
+
"targets": {
|
|
2958
|
+
"dotnet": {
|
|
2959
|
+
"package": "Amazon.CDK.AWS.Shield"
|
|
2960
|
+
},
|
|
2961
|
+
"java": {
|
|
2962
|
+
"package": "services.shield"
|
|
2963
|
+
},
|
|
2964
|
+
"python": {
|
|
2965
|
+
"module": "aws_cdk.aws_shield"
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2968
|
+
},
|
|
2956
2969
|
"aws-cdk-lib.aws_signer": {
|
|
2957
2970
|
"targets": {
|
|
2958
2971
|
"dotnet": {
|
|
@@ -3474,7 +3487,7 @@
|
|
|
3474
3487
|
},
|
|
3475
3488
|
"name": "@aws-cdk/aws-glue-alpha",
|
|
3476
3489
|
"readme": {
|
|
3477
|
-
"markdown": "# AWS Glue Construct Library\n<!--BEGIN STABILITY BANNER-->\n\n---\n\n\n\n> The APIs of higher level constructs in this module are experimental and under active development.\n> They are subject to non-backward compatible changes or removal in any future version. These are\n> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be\n> announced in the release notes. This means that while you may use them, you may need to update\n> your source code when upgrading to a newer version of this package.\n\n---\n\n<!--END STABILITY BANNER-->\n\nThis module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project.\n\n## Job\n\nA `Job` encapsulates a script that connects to data sources, processes them, and then writes output to a data target.\n\nThere are 3 types of jobs supported by AWS Glue: Spark ETL, Spark Streaming, and Python Shell jobs.\n\nThe `glue.JobExecutable` allows you to specify the type of job, the language to use and the code assets required by the job.\n\n`glue.Code` allows you to refer to the different code assets required by the job, either from an existing S3 location or from a local file path.\n\n### Spark Jobs\n\nThese jobs run in an Apache Spark environment managed by AWS Glue.\n\n#### ETL Jobs\n\nAn ETL job processes data in batches using Apache Spark.\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n description: 'an example Scala ETL job',\n});\n```\n\n#### Streaming Jobs\n\nA Streaming job is similar to an ETL job, except that it performs ETL on data streams. It uses the Apache Spark Structured Streaming framework. Some Spark job features are not available to streaming ETL jobs.\n\n```ts\nnew glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});\n```\n\n### Python Shell Jobs\n\nA Python shell job runs Python scripts as a shell and supports a Python version that depends on the AWS Glue version you are using.\nThis can be used to schedule and run tasks that don't require an Apache Spark environment. Currently, three flavors are supported:\n\n* PythonVersion.TWO (2.7; EOL)\n* PythonVersion.THREE (3.6)\n* PythonVersion.THREE_NINE (3.9)\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'PythonShellJob', {\n executable: glue.JobExecutable.pythonShell({\n glueVersion: glue.GlueVersion.V1_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromBucket(bucket, 'script.py'),\n }),\n description: 'an example Python Shell job',\n});\n```\n\n### Ray Jobs\n\nThese jobs run in a Ray environment managed by AWS Glue.\n\n```ts\nnew glue.Job(this, 'RayJob', {\n executable: glue.JobExecutable.pythonRay({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE_NINE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n workerType: glue.WorkerType.Z_2X,\n workerCount: 2,\n description: 'an example Ray job'\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/add-job.html) for more information on adding jobs in Glue.\n\n## Connection\n\nA `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. For example, to create a network connection to connect to a data source within a VPC:\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\nnew glue.Connection(this, 'MyConnection', {\n type: glue.ConnectionType.NETWORK,\n // The security groups granting AWS Glue inbound access to the data source within the VPC\n securityGroups: [securityGroup],\n // The VPC subnet which contains the data source\n subnet,\n});\n```\n\nFor RDS `Connection` by JDBC, it is recommended to manage credentials using AWS Secrets Manager. To use Secret, specify `SECRET_ID` in `properties` like the following code. Note that in this case, the subnet must have a route to the AWS Secrets Manager VPC endpoint or to the AWS Secrets Manager endpoint through a NAT gateway.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\ndeclare const db: rds.DatabaseCluster;\nnew glue.Connection(this, \"RdsConnection\", {\n type: glue.ConnectionType.JDBC,\n securityGroups: [securityGroup],\n subnet,\n properties: {\n JDBC_CONNECTION_URL: `jdbc:mysql://${db.clusterEndpoint.socketAddress}/databasename`,\n JDBC_ENFORCE_SSL: \"false\",\n SECRET_ID: db.secret!.secretName,\n },\n});\n```\n\nIf you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.\n\nSee [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.\n\n## SecurityConfiguration\n\nA `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.\n\n```ts\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n },\n jobBookmarksEncryption: {\n mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,\n },\n s3Encryption: {\n mode: glue.S3EncryptionMode.KMS,\n },\n});\n```\n\nBy default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:\n\n```ts\ndeclare const key: kms.Key;\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n kmsKey: key,\n },\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.\n\n## Database\n\nA `Database` is a logical grouping of `Tables` in the Glue Catalog.\n\n```ts\nnew glue.Database(this, 'MyDatabase');\n```\n\n## Table\n\nA Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }, {\n name: 'col2',\n type: glue.Schema.array(glue.Schema.STRING),\n comment: 'col2 is an array of strings' // comment is optional\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:\n\n```ts\ndeclare const myBucket: s3.Bucket;\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n bucket: myBucket,\n s3Prefix: 'my-table/',\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`:\n\n### Partition Keys\n\nTo improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Indexes\n\nAnother way to improve query performance is to specify partition indexes. If no partition indexes are\npresent on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using\nthe query expression. The query takes more time to run as the number of partitions increase. With an\nindex, the query will try to fetch a subset of the partitions instead of loading all partitions of the\ntable.\n\nThe keys of a partition index must be a subset of the partition keys of the table. You can have a\nmaximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`\nproperty:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n partitionIndexes: [{\n indexName: 'my-index', // optional\n keyNames: ['year'],\n }], // supply up to 3 indexes\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nAlternatively, you can call the `addPartitionIndex()` function on a table:\n\n```ts\ndeclare const myTable: glue.Table;\nmyTable.addPartitionIndex({\n indexName: 'my-index',\n keyNames: ['year'],\n});\n```\n\n### Partition Filtering\n\nIf you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n enablePartitionFiltering: true,\n});\n```\n\n## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)\n\nYou can enable encryption on a Table's data:\n\n* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - (default) Server side encryption (`SSE-S3`) with an Amazon S3-managed key.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.S3_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n*Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.\n\n## Types\n\nA table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n columns: [{\n name: 'primitive_column',\n type: glue.Schema.STRING,\n }, {\n name: 'array_column',\n type: glue.Schema.array(glue.Schema.INTEGER),\n comment: 'array<integer>',\n }, {\n name: 'map_column',\n type: glue.Schema.map(\n glue.Schema.STRING,\n glue.Schema.TIMESTAMP),\n comment: 'map<string,string>',\n }, {\n name: 'struct_column',\n type: glue.Schema.struct([{\n name: 'nested_column',\n type: glue.Schema.DATE,\n comment: 'nested comment',\n }]),\n comment: \"struct<nested_column:date COMMENT 'nested comment'>\",\n }],\n // ...\n database: myDatabase,\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Primitives\n\n#### Numeric\n\n| Name \t| Type \t| Comments |\n|-----------\t|----------\t|------------------------------------------------------------------------------------------------------------------\t|\n| FLOAT \t| Constant \t| A 32-bit single-precision floating point number |\n| INTEGER \t| Constant \t| A 32-bit signed value in two's complement format, with a minimum value of -2^31 and a maximum value of 2^31-1 \t|\n| DOUBLE \t| Constant \t| A 64-bit double-precision floating point number |\n| BIG_INT \t| Constant \t| A 64-bit signed INTEGER in two’s complement format, with a minimum value of -2^63 and a maximum value of 2^63 -1 |\n| SMALL_INT \t| Constant \t| A 16-bit signed INTEGER in two’s complement format, with a minimum value of -2^15 and a maximum value of 2^15-1 |\n| TINY_INT \t| Constant \t| A 8-bit signed INTEGER in two’s complement format, with a minimum value of -2^7 and a maximum value of 2^7-1 |\n\n#### Date and time\n\n| Name \t| Type \t| Comments \t|\n|-----------\t|----------\t|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| DATE \t| Constant \t| A date in UNIX format, such as YYYY-MM-DD. \t|\n| TIMESTAMP \t| Constant \t| Date and time instant in the UNiX format, such as yyyy-mm-dd hh:mm:ss[.f...]. For example, TIMESTAMP '2008-09-15 03:04:05.324'. This format uses the session time zone. \t|\n\n#### String\n\n| Name \t| Type \t| Comments \t|\n|--------------------------------------------\t|----------\t|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| STRING \t| Constant \t| A string literal enclosed in single or double quotes \t|\n| decimal(precision: number, scale?: number) \t| Function \t| `precision` is the total number of digits. `scale` (optional) is the number of digits in fractional part with a default of 0. For example, use these type definitions: decimal(11,5), decimal(15) \t|\n| char(length: number) \t| Function \t| Fixed length character data, with a specified length between 1 and 255, such as char(10) \t|\n| varchar(length: number) \t| Function \t| Variable length character data, with a specified length between 1 and 65535, such as varchar(10) \t|\n\n#### Miscellaneous\n\n| Name \t| Type \t| Comments \t|\n|---------\t|----------\t|-------------------------------\t|\n| BOOLEAN \t| Constant \t| Values are `true` and `false` \t|\n| BINARY \t| Constant \t| Value is in binary \t|\n\n### Complex\n\n| Name \t| Type \t| Comments \t|\n|-------------------------------------\t|----------\t|-------------------------------------------------------------------\t|\n| array(itemType: Type) \t| Function \t| An array of some other type \t|\n| map(keyType: Type, valueType: Type) \t| Function \t| A map of some primitive key type to any value type \t|\n| struct(collumns: Column[]) \t| Function \t| Nested structure containing individually named and typed collumns \t|\n"
|
|
3490
|
+
"markdown": "# AWS Glue Construct Library\n<!--BEGIN STABILITY BANNER-->\n\n---\n\n\n\n> The APIs of higher level constructs in this module are experimental and under active development.\n> They are subject to non-backward compatible changes or removal in any future version. These are\n> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be\n> announced in the release notes. This means that while you may use them, you may need to update\n> your source code when upgrading to a newer version of this package.\n\n---\n\n<!--END STABILITY BANNER-->\n\nThis module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project.\n\n## Job\n\nA `Job` encapsulates a script that connects to data sources, processes them, and then writes output to a data target.\n\nThere are 3 types of jobs supported by AWS Glue: Spark ETL, Spark Streaming, and Python Shell jobs.\n\nThe `glue.JobExecutable` allows you to specify the type of job, the language to use and the code assets required by the job.\n\n`glue.Code` allows you to refer to the different code assets required by the job, either from an existing S3 location or from a local file path.\n\n### Spark Jobs\n\nThese jobs run in an Apache Spark environment managed by AWS Glue.\n\n#### ETL Jobs\n\nAn ETL job processes data in batches using Apache Spark.\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n workerType: glue.WorkerType.G_8X,\n description: 'an example Scala ETL job',\n});\n```\n\n#### Streaming Jobs\n\nA Streaming job is similar to an ETL job, except that it performs ETL on data streams. It uses the Apache Spark Structured Streaming framework. Some Spark job features are not available to streaming ETL jobs.\n\n```ts\nnew glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});\n```\n\n### Python Shell Jobs\n\nA Python shell job runs Python scripts as a shell and supports a Python version that depends on the AWS Glue version you are using.\nThis can be used to schedule and run tasks that don't require an Apache Spark environment. Currently, three flavors are supported:\n\n* PythonVersion.TWO (2.7; EOL)\n* PythonVersion.THREE (3.6)\n* PythonVersion.THREE_NINE (3.9)\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'PythonShellJob', {\n executable: glue.JobExecutable.pythonShell({\n glueVersion: glue.GlueVersion.V1_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromBucket(bucket, 'script.py'),\n }),\n description: 'an example Python Shell job',\n});\n```\n\n### Ray Jobs\n\nThese jobs run in a Ray environment managed by AWS Glue.\n\n```ts\nnew glue.Job(this, 'RayJob', {\n executable: glue.JobExecutable.pythonRay({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE_NINE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n workerType: glue.WorkerType.Z_2X,\n workerCount: 2,\n description: 'an example Ray job'\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/add-job.html) for more information on adding jobs in Glue.\n\n## Connection\n\nA `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. For example, to create a network connection to connect to a data source within a VPC:\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\nnew glue.Connection(this, 'MyConnection', {\n type: glue.ConnectionType.NETWORK,\n // The security groups granting AWS Glue inbound access to the data source within the VPC\n securityGroups: [securityGroup],\n // The VPC subnet which contains the data source\n subnet,\n});\n```\n\nFor RDS `Connection` by JDBC, it is recommended to manage credentials using AWS Secrets Manager. To use Secret, specify `SECRET_ID` in `properties` like the following code. Note that in this case, the subnet must have a route to the AWS Secrets Manager VPC endpoint or to the AWS Secrets Manager endpoint through a NAT gateway.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\ndeclare const db: rds.DatabaseCluster;\nnew glue.Connection(this, \"RdsConnection\", {\n type: glue.ConnectionType.JDBC,\n securityGroups: [securityGroup],\n subnet,\n properties: {\n JDBC_CONNECTION_URL: `jdbc:mysql://${db.clusterEndpoint.socketAddress}/databasename`,\n JDBC_ENFORCE_SSL: \"false\",\n SECRET_ID: db.secret!.secretName,\n },\n});\n```\n\nIf you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.\n\nSee [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.\n\n## SecurityConfiguration\n\nA `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.\n\n```ts\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n },\n jobBookmarksEncryption: {\n mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,\n },\n s3Encryption: {\n mode: glue.S3EncryptionMode.KMS,\n },\n});\n```\n\nBy default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:\n\n```ts\ndeclare const key: kms.Key;\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n kmsKey: key,\n },\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.\n\n## Database\n\nA `Database` is a logical grouping of `Tables` in the Glue Catalog.\n\n```ts\nnew glue.Database(this, 'MyDatabase');\n```\n\n## Table\n\nA Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }, {\n name: 'col2',\n type: glue.Schema.array(glue.Schema.STRING),\n comment: 'col2 is an array of strings' // comment is optional\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:\n\n```ts\ndeclare const myBucket: s3.Bucket;\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n bucket: myBucket,\n s3Prefix: 'my-table/',\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`:\n\n### Partition Keys\n\nTo improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Indexes\n\nAnother way to improve query performance is to specify partition indexes. If no partition indexes are\npresent on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using\nthe query expression. The query takes more time to run as the number of partitions increase. With an\nindex, the query will try to fetch a subset of the partitions instead of loading all partitions of the\ntable.\n\nThe keys of a partition index must be a subset of the partition keys of the table. You can have a\nmaximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`\nproperty:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n partitionIndexes: [{\n indexName: 'my-index', // optional\n keyNames: ['year'],\n }], // supply up to 3 indexes\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nAlternatively, you can call the `addPartitionIndex()` function on a table:\n\n```ts\ndeclare const myTable: glue.Table;\nmyTable.addPartitionIndex({\n indexName: 'my-index',\n keyNames: ['year'],\n});\n```\n\n### Partition Filtering\n\nIf you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n enablePartitionFiltering: true,\n});\n```\n\n## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)\n\nYou can enable encryption on a Table's data:\n\n* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - (default) Server side encryption (`SSE-S3`) with an Amazon S3-managed key.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.S3_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n*Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.\n\n## Types\n\nA table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n columns: [{\n name: 'primitive_column',\n type: glue.Schema.STRING,\n }, {\n name: 'array_column',\n type: glue.Schema.array(glue.Schema.INTEGER),\n comment: 'array<integer>',\n }, {\n name: 'map_column',\n type: glue.Schema.map(\n glue.Schema.STRING,\n glue.Schema.TIMESTAMP),\n comment: 'map<string,string>',\n }, {\n name: 'struct_column',\n type: glue.Schema.struct([{\n name: 'nested_column',\n type: glue.Schema.DATE,\n comment: 'nested comment',\n }]),\n comment: \"struct<nested_column:date COMMENT 'nested comment'>\",\n }],\n // ...\n database: myDatabase,\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Primitives\n\n#### Numeric\n\n| Name \t| Type \t| Comments |\n|-----------\t|----------\t|------------------------------------------------------------------------------------------------------------------\t|\n| FLOAT \t| Constant \t| A 32-bit single-precision floating point number |\n| INTEGER \t| Constant \t| A 32-bit signed value in two's complement format, with a minimum value of -2^31 and a maximum value of 2^31-1 \t|\n| DOUBLE \t| Constant \t| A 64-bit double-precision floating point number |\n| BIG_INT \t| Constant \t| A 64-bit signed INTEGER in two’s complement format, with a minimum value of -2^63 and a maximum value of 2^63 -1 |\n| SMALL_INT \t| Constant \t| A 16-bit signed INTEGER in two’s complement format, with a minimum value of -2^15 and a maximum value of 2^15-1 |\n| TINY_INT \t| Constant \t| A 8-bit signed INTEGER in two’s complement format, with a minimum value of -2^7 and a maximum value of 2^7-1 |\n\n#### Date and time\n\n| Name \t| Type \t| Comments \t|\n|-----------\t|----------\t|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| DATE \t| Constant \t| A date in UNIX format, such as YYYY-MM-DD. \t|\n| TIMESTAMP \t| Constant \t| Date and time instant in the UNiX format, such as yyyy-mm-dd hh:mm:ss[.f...]. For example, TIMESTAMP '2008-09-15 03:04:05.324'. This format uses the session time zone. \t|\n\n#### String\n\n| Name \t| Type \t| Comments \t|\n|--------------------------------------------\t|----------\t|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| STRING \t| Constant \t| A string literal enclosed in single or double quotes \t|\n| decimal(precision: number, scale?: number) \t| Function \t| `precision` is the total number of digits. `scale` (optional) is the number of digits in fractional part with a default of 0. For example, use these type definitions: decimal(11,5), decimal(15) \t|\n| char(length: number) \t| Function \t| Fixed length character data, with a specified length between 1 and 255, such as char(10) \t|\n| varchar(length: number) \t| Function \t| Variable length character data, with a specified length between 1 and 65535, such as varchar(10) \t|\n\n#### Miscellaneous\n\n| Name \t| Type \t| Comments \t|\n|---------\t|----------\t|-------------------------------\t|\n| BOOLEAN \t| Constant \t| Values are `true` and `false` \t|\n| BINARY \t| Constant \t| Value is in binary \t|\n\n### Complex\n\n| Name \t| Type \t| Comments \t|\n|-------------------------------------\t|----------\t|-------------------------------------------------------------------\t|\n| array(itemType: Type) \t| Function \t| An array of some other type \t|\n| map(keyType: Type, valueType: Type) \t| Function \t| A map of some primitive key type to any value type \t|\n| struct(collumns: Column[]) \t| Function \t| Nested structure containing individually named and typed collumns \t|\n"
|
|
3478
3491
|
},
|
|
3479
3492
|
"repository": {
|
|
3480
3493
|
"directory": "packages/@aws-cdk/aws-glue-alpha",
|
|
@@ -3518,7 +3531,7 @@
|
|
|
3518
3531
|
"docs": {
|
|
3519
3532
|
"stability": "experimental",
|
|
3520
3533
|
"summary": "Job Code from a local file.",
|
|
3521
|
-
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\n\ndeclare const dockerImage: cdk.DockerImage;\ndeclare const grantable: iam.IGrantable;\ndeclare const localBundling: cdk.ILocalBundling;\nconst assetCode = new glue_alpha.AssetCode('path', /* all optional props */ {\n assetHash: 'assetHash',\n assetHashType: cdk.AssetHashType.SOURCE,\n bundling: {\n image: dockerImage,\n\n // the properties below are optional\n bundlingFileAccess: cdk.BundlingFileAccess.VOLUME_COPY,\n command: ['command'],\n entrypoint: ['entrypoint'],\n environment: {\n environmentKey: 'environment',\n },\n local: localBundling,\n network: 'network',\n outputType: cdk.BundlingOutput.ARCHIVED,\n securityOpt: 'securityOpt',\n user: 'user',\n volumes: [{\n containerPath: 'containerPath',\n hostPath: 'hostPath',\n\n // the properties below are optional\n consistency: cdk.DockerVolumeConsistency.CONSISTENT,\n }],\n volumesFrom: ['volumesFrom'],\n workingDirectory: 'workingDirectory',\n },\n exclude: ['exclude'],\n followSymlinks: cdk.SymlinkFollowMode.NEVER,\n ignoreMode: cdk.IgnoreMode.GLOB,\n readers: [grantable],\n});",
|
|
3534
|
+
"example": "// The code below shows an example of how to instantiate this type.\n// The values are placeholders you should change.\nimport * as glue_alpha from '@aws-cdk/aws-glue-alpha';\nimport * as cdk from 'aws-cdk-lib';\nimport { aws_iam as iam } from 'aws-cdk-lib';\n\ndeclare const dockerImage: cdk.DockerImage;\ndeclare const grantable: iam.IGrantable;\ndeclare const localBundling: cdk.ILocalBundling;\nconst assetCode = new glue_alpha.AssetCode('path', /* all optional props */ {\n assetHash: 'assetHash',\n assetHashType: cdk.AssetHashType.SOURCE,\n bundling: {\n image: dockerImage,\n\n // the properties below are optional\n bundlingFileAccess: cdk.BundlingFileAccess.VOLUME_COPY,\n command: ['command'],\n entrypoint: ['entrypoint'],\n environment: {\n environmentKey: 'environment',\n },\n local: localBundling,\n network: 'network',\n outputType: cdk.BundlingOutput.ARCHIVED,\n securityOpt: 'securityOpt',\n user: 'user',\n volumes: [{\n containerPath: 'containerPath',\n hostPath: 'hostPath',\n\n // the properties below are optional\n consistency: cdk.DockerVolumeConsistency.CONSISTENT,\n }],\n volumesFrom: ['volumesFrom'],\n workingDirectory: 'workingDirectory',\n },\n deployTime: false,\n exclude: ['exclude'],\n followSymlinks: cdk.SymlinkFollowMode.NEVER,\n ignoreMode: cdk.IgnoreMode.GLOB,\n readers: [grantable],\n});",
|
|
3522
3535
|
"custom": {
|
|
3523
3536
|
"exampleMetadata": "fixture=_generated"
|
|
3524
3537
|
}
|
|
@@ -4629,7 +4642,7 @@
|
|
|
4629
4642
|
"kind": "interface",
|
|
4630
4643
|
"locationInModule": {
|
|
4631
4644
|
"filename": "lib/job.ts",
|
|
4632
|
-
"line":
|
|
4645
|
+
"line": 404
|
|
4633
4646
|
},
|
|
4634
4647
|
"name": "ContinuousLoggingProps",
|
|
4635
4648
|
"properties": [
|
|
@@ -4642,7 +4655,7 @@
|
|
|
4642
4655
|
"immutable": true,
|
|
4643
4656
|
"locationInModule": {
|
|
4644
4657
|
"filename": "lib/job.ts",
|
|
4645
|
-
"line":
|
|
4658
|
+
"line": 408
|
|
4646
4659
|
},
|
|
4647
4660
|
"name": "enabled",
|
|
4648
4661
|
"type": {
|
|
@@ -4660,7 +4673,7 @@
|
|
|
4660
4673
|
"immutable": true,
|
|
4661
4674
|
"locationInModule": {
|
|
4662
4675
|
"filename": "lib/job.ts",
|
|
4663
|
-
"line":
|
|
4676
|
+
"line": 438
|
|
4664
4677
|
},
|
|
4665
4678
|
"name": "conversionPattern",
|
|
4666
4679
|
"optional": true,
|
|
@@ -4678,7 +4691,7 @@
|
|
|
4678
4691
|
"immutable": true,
|
|
4679
4692
|
"locationInModule": {
|
|
4680
4693
|
"filename": "lib/job.ts",
|
|
4681
|
-
"line":
|
|
4694
|
+
"line": 415
|
|
4682
4695
|
},
|
|
4683
4696
|
"name": "logGroup",
|
|
4684
4697
|
"optional": true,
|
|
@@ -4696,7 +4709,7 @@
|
|
|
4696
4709
|
"immutable": true,
|
|
4697
4710
|
"locationInModule": {
|
|
4698
4711
|
"filename": "lib/job.ts",
|
|
4699
|
-
"line":
|
|
4712
|
+
"line": 422
|
|
4700
4713
|
},
|
|
4701
4714
|
"name": "logStreamPrefix",
|
|
4702
4715
|
"optional": true,
|
|
@@ -4714,7 +4727,7 @@
|
|
|
4714
4727
|
"immutable": true,
|
|
4715
4728
|
"locationInModule": {
|
|
4716
4729
|
"filename": "lib/job.ts",
|
|
4717
|
-
"line":
|
|
4730
|
+
"line": 429
|
|
4718
4731
|
},
|
|
4719
4732
|
"name": "quiet",
|
|
4720
4733
|
"optional": true,
|
|
@@ -5620,7 +5633,7 @@
|
|
|
5620
5633
|
"kind": "interface",
|
|
5621
5634
|
"locationInModule": {
|
|
5622
5635
|
"filename": "lib/job.ts",
|
|
5623
|
-
"line":
|
|
5636
|
+
"line": 135
|
|
5624
5637
|
},
|
|
5625
5638
|
"methods": [
|
|
5626
5639
|
{
|
|
@@ -5632,7 +5645,7 @@
|
|
|
5632
5645
|
},
|
|
5633
5646
|
"locationInModule": {
|
|
5634
5647
|
"filename": "lib/job.ts",
|
|
5635
|
-
"line":
|
|
5648
|
+
"line": 192
|
|
5636
5649
|
},
|
|
5637
5650
|
"name": "metric",
|
|
5638
5651
|
"parameters": [
|
|
@@ -5679,7 +5692,7 @@
|
|
|
5679
5692
|
},
|
|
5680
5693
|
"locationInModule": {
|
|
5681
5694
|
"filename": "lib/job.ts",
|
|
5682
|
-
"line":
|
|
5695
|
+
"line": 202
|
|
5683
5696
|
},
|
|
5684
5697
|
"name": "metricFailure",
|
|
5685
5698
|
"parameters": [
|
|
@@ -5705,7 +5718,7 @@
|
|
|
5705
5718
|
},
|
|
5706
5719
|
"locationInModule": {
|
|
5707
5720
|
"filename": "lib/job.ts",
|
|
5708
|
-
"line":
|
|
5721
|
+
"line": 197
|
|
5709
5722
|
},
|
|
5710
5723
|
"name": "metricSuccess",
|
|
5711
5724
|
"parameters": [
|
|
@@ -5731,7 +5744,7 @@
|
|
|
5731
5744
|
},
|
|
5732
5745
|
"locationInModule": {
|
|
5733
5746
|
"filename": "lib/job.ts",
|
|
5734
|
-
"line":
|
|
5747
|
+
"line": 207
|
|
5735
5748
|
},
|
|
5736
5749
|
"name": "metricTimeout",
|
|
5737
5750
|
"parameters": [
|
|
@@ -5758,7 +5771,7 @@
|
|
|
5758
5771
|
},
|
|
5759
5772
|
"locationInModule": {
|
|
5760
5773
|
"filename": "lib/job.ts",
|
|
5761
|
-
"line":
|
|
5774
|
+
"line": 153
|
|
5762
5775
|
},
|
|
5763
5776
|
"name": "onEvent",
|
|
5764
5777
|
"parameters": [
|
|
@@ -5791,7 +5804,7 @@
|
|
|
5791
5804
|
},
|
|
5792
5805
|
"locationInModule": {
|
|
5793
5806
|
"filename": "lib/job.ts",
|
|
5794
|
-
"line":
|
|
5807
|
+
"line": 174
|
|
5795
5808
|
},
|
|
5796
5809
|
"name": "onFailure",
|
|
5797
5810
|
"parameters": [
|
|
@@ -5824,7 +5837,7 @@
|
|
|
5824
5837
|
},
|
|
5825
5838
|
"locationInModule": {
|
|
5826
5839
|
"filename": "lib/job.ts",
|
|
5827
|
-
"line":
|
|
5840
|
+
"line": 160
|
|
5828
5841
|
},
|
|
5829
5842
|
"name": "onStateChange",
|
|
5830
5843
|
"parameters": [
|
|
@@ -5863,7 +5876,7 @@
|
|
|
5863
5876
|
},
|
|
5864
5877
|
"locationInModule": {
|
|
5865
5878
|
"filename": "lib/job.ts",
|
|
5866
|
-
"line":
|
|
5879
|
+
"line": 167
|
|
5867
5880
|
},
|
|
5868
5881
|
"name": "onSuccess",
|
|
5869
5882
|
"parameters": [
|
|
@@ -5896,7 +5909,7 @@
|
|
|
5896
5909
|
},
|
|
5897
5910
|
"locationInModule": {
|
|
5898
5911
|
"filename": "lib/job.ts",
|
|
5899
|
-
"line":
|
|
5912
|
+
"line": 181
|
|
5900
5913
|
},
|
|
5901
5914
|
"name": "onTimeout",
|
|
5902
5915
|
"parameters": [
|
|
@@ -5935,7 +5948,7 @@
|
|
|
5935
5948
|
"immutable": true,
|
|
5936
5949
|
"locationInModule": {
|
|
5937
5950
|
"filename": "lib/job.ts",
|
|
5938
|
-
"line":
|
|
5951
|
+
"line": 146
|
|
5939
5952
|
},
|
|
5940
5953
|
"name": "jobArn",
|
|
5941
5954
|
"type": {
|
|
@@ -5954,7 +5967,7 @@
|
|
|
5954
5967
|
"immutable": true,
|
|
5955
5968
|
"locationInModule": {
|
|
5956
5969
|
"filename": "lib/job.ts",
|
|
5957
|
-
"line":
|
|
5970
|
+
"line": 140
|
|
5958
5971
|
},
|
|
5959
5972
|
"name": "jobName",
|
|
5960
5973
|
"type": {
|
|
@@ -6219,7 +6232,7 @@
|
|
|
6219
6232
|
},
|
|
6220
6233
|
"locationInModule": {
|
|
6221
6234
|
"filename": "lib/job.ts",
|
|
6222
|
-
"line":
|
|
6235
|
+
"line": 654
|
|
6223
6236
|
},
|
|
6224
6237
|
"parameters": [
|
|
6225
6238
|
{
|
|
@@ -6248,7 +6261,7 @@
|
|
|
6248
6261
|
"kind": "class",
|
|
6249
6262
|
"locationInModule": {
|
|
6250
6263
|
"filename": "lib/job.ts",
|
|
6251
|
-
"line":
|
|
6264
|
+
"line": 608
|
|
6252
6265
|
},
|
|
6253
6266
|
"methods": [
|
|
6254
6267
|
{
|
|
@@ -6258,7 +6271,7 @@
|
|
|
6258
6271
|
},
|
|
6259
6272
|
"locationInModule": {
|
|
6260
6273
|
"filename": "lib/job.ts",
|
|
6261
|
-
"line":
|
|
6274
|
+
"line": 616
|
|
6262
6275
|
},
|
|
6263
6276
|
"name": "fromJobAttributes",
|
|
6264
6277
|
"parameters": [
|
|
@@ -6305,7 +6318,7 @@
|
|
|
6305
6318
|
},
|
|
6306
6319
|
"locationInModule": {
|
|
6307
6320
|
"filename": "lib/job.ts",
|
|
6308
|
-
"line":
|
|
6321
|
+
"line": 299
|
|
6309
6322
|
},
|
|
6310
6323
|
"name": "metric",
|
|
6311
6324
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6353,7 +6366,7 @@
|
|
|
6353
6366
|
},
|
|
6354
6367
|
"locationInModule": {
|
|
6355
6368
|
"filename": "lib/job.ts",
|
|
6356
|
-
"line":
|
|
6369
|
+
"line": 326
|
|
6357
6370
|
},
|
|
6358
6371
|
"name": "metricFailure",
|
|
6359
6372
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6380,7 +6393,7 @@
|
|
|
6380
6393
|
},
|
|
6381
6394
|
"locationInModule": {
|
|
6382
6395
|
"filename": "lib/job.ts",
|
|
6383
|
-
"line":
|
|
6396
|
+
"line": 317
|
|
6384
6397
|
},
|
|
6385
6398
|
"name": "metricSuccess",
|
|
6386
6399
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6407,7 +6420,7 @@
|
|
|
6407
6420
|
},
|
|
6408
6421
|
"locationInModule": {
|
|
6409
6422
|
"filename": "lib/job.ts",
|
|
6410
|
-
"line":
|
|
6423
|
+
"line": 335
|
|
6411
6424
|
},
|
|
6412
6425
|
"name": "metricTimeout",
|
|
6413
6426
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6434,7 +6447,7 @@
|
|
|
6434
6447
|
},
|
|
6435
6448
|
"locationInModule": {
|
|
6436
6449
|
"filename": "lib/job.ts",
|
|
6437
|
-
"line":
|
|
6450
|
+
"line": 227
|
|
6438
6451
|
},
|
|
6439
6452
|
"name": "onEvent",
|
|
6440
6453
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6473,7 +6486,7 @@
|
|
|
6473
6486
|
},
|
|
6474
6487
|
"locationInModule": {
|
|
6475
6488
|
"filename": "lib/job.ts",
|
|
6476
|
-
"line":
|
|
6489
|
+
"line": 276
|
|
6477
6490
|
},
|
|
6478
6491
|
"name": "onFailure",
|
|
6479
6492
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6512,7 +6525,7 @@
|
|
|
6512
6525
|
},
|
|
6513
6526
|
"locationInModule": {
|
|
6514
6527
|
"filename": "lib/job.ts",
|
|
6515
|
-
"line":
|
|
6528
|
+
"line": 247
|
|
6516
6529
|
},
|
|
6517
6530
|
"name": "onStateChange",
|
|
6518
6531
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6559,7 +6572,7 @@
|
|
|
6559
6572
|
},
|
|
6560
6573
|
"locationInModule": {
|
|
6561
6574
|
"filename": "lib/job.ts",
|
|
6562
|
-
"line":
|
|
6575
|
+
"line": 266
|
|
6563
6576
|
},
|
|
6564
6577
|
"name": "onSuccess",
|
|
6565
6578
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6598,7 +6611,7 @@
|
|
|
6598
6611
|
},
|
|
6599
6612
|
"locationInModule": {
|
|
6600
6613
|
"filename": "lib/job.ts",
|
|
6601
|
-
"line":
|
|
6614
|
+
"line": 286
|
|
6602
6615
|
},
|
|
6603
6616
|
"name": "onTimeout",
|
|
6604
6617
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6641,7 +6654,7 @@
|
|
|
6641
6654
|
"immutable": true,
|
|
6642
6655
|
"locationInModule": {
|
|
6643
6656
|
"filename": "lib/job.ts",
|
|
6644
|
-
"line":
|
|
6657
|
+
"line": 644
|
|
6645
6658
|
},
|
|
6646
6659
|
"name": "grantPrincipal",
|
|
6647
6660
|
"overrides": "aws-cdk-lib.aws_iam.IGrantable",
|
|
@@ -6657,7 +6670,7 @@
|
|
|
6657
6670
|
"immutable": true,
|
|
6658
6671
|
"locationInModule": {
|
|
6659
6672
|
"filename": "lib/job.ts",
|
|
6660
|
-
"line":
|
|
6673
|
+
"line": 629
|
|
6661
6674
|
},
|
|
6662
6675
|
"name": "jobArn",
|
|
6663
6676
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6673,7 +6686,7 @@
|
|
|
6673
6686
|
"immutable": true,
|
|
6674
6687
|
"locationInModule": {
|
|
6675
6688
|
"filename": "lib/job.ts",
|
|
6676
|
-
"line":
|
|
6689
|
+
"line": 634
|
|
6677
6690
|
},
|
|
6678
6691
|
"name": "jobName",
|
|
6679
6692
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6689,7 +6702,7 @@
|
|
|
6689
6702
|
"immutable": true,
|
|
6690
6703
|
"locationInModule": {
|
|
6691
6704
|
"filename": "lib/job.ts",
|
|
6692
|
-
"line":
|
|
6705
|
+
"line": 639
|
|
6693
6706
|
},
|
|
6694
6707
|
"name": "role",
|
|
6695
6708
|
"type": {
|
|
@@ -6705,7 +6718,7 @@
|
|
|
6705
6718
|
"immutable": true,
|
|
6706
6719
|
"locationInModule": {
|
|
6707
6720
|
"filename": "lib/job.ts",
|
|
6708
|
-
"line":
|
|
6721
|
+
"line": 652
|
|
6709
6722
|
},
|
|
6710
6723
|
"name": "sparkUILoggingLocation",
|
|
6711
6724
|
"optional": true,
|
|
@@ -6731,7 +6744,7 @@
|
|
|
6731
6744
|
"kind": "interface",
|
|
6732
6745
|
"locationInModule": {
|
|
6733
6746
|
"filename": "lib/job.ts",
|
|
6734
|
-
"line":
|
|
6747
|
+
"line": 444
|
|
6735
6748
|
},
|
|
6736
6749
|
"name": "JobAttributes",
|
|
6737
6750
|
"properties": [
|
|
@@ -6744,7 +6757,7 @@
|
|
|
6744
6757
|
"immutable": true,
|
|
6745
6758
|
"locationInModule": {
|
|
6746
6759
|
"filename": "lib/job.ts",
|
|
6747
|
-
"line":
|
|
6760
|
+
"line": 448
|
|
6748
6761
|
},
|
|
6749
6762
|
"name": "jobName",
|
|
6750
6763
|
"type": {
|
|
@@ -6761,7 +6774,7 @@
|
|
|
6761
6774
|
"immutable": true,
|
|
6762
6775
|
"locationInModule": {
|
|
6763
6776
|
"filename": "lib/job.ts",
|
|
6764
|
-
"line":
|
|
6777
|
+
"line": 455
|
|
6765
6778
|
},
|
|
6766
6779
|
"name": "role",
|
|
6767
6780
|
"optional": true,
|
|
@@ -7354,7 +7367,7 @@
|
|
|
7354
7367
|
"kind": "interface",
|
|
7355
7368
|
"locationInModule": {
|
|
7356
7369
|
"filename": "lib/job.ts",
|
|
7357
|
-
"line":
|
|
7370
|
+
"line": 461
|
|
7358
7371
|
},
|
|
7359
7372
|
"name": "JobProps",
|
|
7360
7373
|
"properties": [
|
|
@@ -7367,7 +7380,7 @@
|
|
|
7367
7380
|
"immutable": true,
|
|
7368
7381
|
"locationInModule": {
|
|
7369
7382
|
"filename": "lib/job.ts",
|
|
7370
|
-
"line":
|
|
7383
|
+
"line": 465
|
|
7371
7384
|
},
|
|
7372
7385
|
"name": "executable",
|
|
7373
7386
|
"type": {
|
|
@@ -7385,7 +7398,7 @@
|
|
|
7385
7398
|
"immutable": true,
|
|
7386
7399
|
"locationInModule": {
|
|
7387
7400
|
"filename": "lib/job.ts",
|
|
7388
|
-
"line":
|
|
7401
|
+
"line": 540
|
|
7389
7402
|
},
|
|
7390
7403
|
"name": "connections",
|
|
7391
7404
|
"optional": true,
|
|
@@ -7409,7 +7422,7 @@
|
|
|
7409
7422
|
"immutable": true,
|
|
7410
7423
|
"locationInModule": {
|
|
7411
7424
|
"filename": "lib/job.ts",
|
|
7412
|
-
"line":
|
|
7425
|
+
"line": 602
|
|
7413
7426
|
},
|
|
7414
7427
|
"name": "continuousLogging",
|
|
7415
7428
|
"optional": true,
|
|
@@ -7428,7 +7441,7 @@
|
|
|
7428
7441
|
"immutable": true,
|
|
7429
7442
|
"locationInModule": {
|
|
7430
7443
|
"filename": "lib/job.ts",
|
|
7431
|
-
"line":
|
|
7444
|
+
"line": 555
|
|
7432
7445
|
},
|
|
7433
7446
|
"name": "defaultArguments",
|
|
7434
7447
|
"optional": true,
|
|
@@ -7451,7 +7464,7 @@
|
|
|
7451
7464
|
"immutable": true,
|
|
7452
7465
|
"locationInModule": {
|
|
7453
7466
|
"filename": "lib/job.ts",
|
|
7454
|
-
"line":
|
|
7467
|
+
"line": 479
|
|
7455
7468
|
},
|
|
7456
7469
|
"name": "description",
|
|
7457
7470
|
"optional": true,
|
|
@@ -7470,7 +7483,7 @@
|
|
|
7470
7483
|
"immutable": true,
|
|
7471
7484
|
"locationInModule": {
|
|
7472
7485
|
"filename": "lib/job.ts",
|
|
7473
|
-
"line":
|
|
7486
|
+
"line": 582
|
|
7474
7487
|
},
|
|
7475
7488
|
"name": "enableProfilingMetrics",
|
|
7476
7489
|
"optional": true,
|
|
@@ -7488,7 +7501,7 @@
|
|
|
7488
7501
|
"immutable": true,
|
|
7489
7502
|
"locationInModule": {
|
|
7490
7503
|
"filename": "lib/job.ts",
|
|
7491
|
-
"line":
|
|
7504
|
+
"line": 472
|
|
7492
7505
|
},
|
|
7493
7506
|
"name": "jobName",
|
|
7494
7507
|
"optional": true,
|
|
@@ -7507,7 +7520,7 @@
|
|
|
7507
7520
|
"immutable": true,
|
|
7508
7521
|
"locationInModule": {
|
|
7509
7522
|
"filename": "lib/job.ts",
|
|
7510
|
-
"line":
|
|
7523
|
+
"line": 487
|
|
7511
7524
|
},
|
|
7512
7525
|
"name": "maxCapacity",
|
|
7513
7526
|
"optional": true,
|
|
@@ -7526,7 +7539,7 @@
|
|
|
7526
7539
|
"immutable": true,
|
|
7527
7540
|
"locationInModule": {
|
|
7528
7541
|
"filename": "lib/job.ts",
|
|
7529
|
-
"line":
|
|
7542
|
+
"line": 503
|
|
7530
7543
|
},
|
|
7531
7544
|
"name": "maxConcurrentRuns",
|
|
7532
7545
|
"optional": true,
|
|
@@ -7544,7 +7557,7 @@
|
|
|
7544
7557
|
"immutable": true,
|
|
7545
7558
|
"locationInModule": {
|
|
7546
7559
|
"filename": "lib/job.ts",
|
|
7547
|
-
"line":
|
|
7560
|
+
"line": 494
|
|
7548
7561
|
},
|
|
7549
7562
|
"name": "maxRetries",
|
|
7550
7563
|
"optional": true,
|
|
@@ -7562,7 +7575,7 @@
|
|
|
7562
7575
|
"immutable": true,
|
|
7563
7576
|
"locationInModule": {
|
|
7564
7577
|
"filename": "lib/job.ts",
|
|
7565
|
-
"line":
|
|
7578
|
+
"line": 510
|
|
7566
7579
|
},
|
|
7567
7580
|
"name": "notifyDelayAfter",
|
|
7568
7581
|
"optional": true,
|
|
@@ -7582,7 +7595,7 @@
|
|
|
7582
7595
|
"immutable": true,
|
|
7583
7596
|
"locationInModule": {
|
|
7584
7597
|
"filename": "lib/job.ts",
|
|
7585
|
-
"line":
|
|
7598
|
+
"line": 573
|
|
7586
7599
|
},
|
|
7587
7600
|
"name": "role",
|
|
7588
7601
|
"optional": true,
|
|
@@ -7600,7 +7613,7 @@
|
|
|
7600
7613
|
"immutable": true,
|
|
7601
7614
|
"locationInModule": {
|
|
7602
7615
|
"filename": "lib/job.ts",
|
|
7603
|
-
"line":
|
|
7616
|
+
"line": 547
|
|
7604
7617
|
},
|
|
7605
7618
|
"name": "securityConfiguration",
|
|
7606
7619
|
"optional": true,
|
|
@@ -7619,7 +7632,7 @@
|
|
|
7619
7632
|
"immutable": true,
|
|
7620
7633
|
"locationInModule": {
|
|
7621
7634
|
"filename": "lib/job.ts",
|
|
7622
|
-
"line":
|
|
7635
|
+
"line": 592
|
|
7623
7636
|
},
|
|
7624
7637
|
"name": "sparkUI",
|
|
7625
7638
|
"optional": true,
|
|
@@ -7637,7 +7650,7 @@
|
|
|
7637
7650
|
"immutable": true,
|
|
7638
7651
|
"locationInModule": {
|
|
7639
7652
|
"filename": "lib/job.ts",
|
|
7640
|
-
"line":
|
|
7653
|
+
"line": 562
|
|
7641
7654
|
},
|
|
7642
7655
|
"name": "tags",
|
|
7643
7656
|
"optional": true,
|
|
@@ -7660,7 +7673,7 @@
|
|
|
7660
7673
|
"immutable": true,
|
|
7661
7674
|
"locationInModule": {
|
|
7662
7675
|
"filename": "lib/job.ts",
|
|
7663
|
-
"line":
|
|
7676
|
+
"line": 517
|
|
7664
7677
|
},
|
|
7665
7678
|
"name": "timeout",
|
|
7666
7679
|
"optional": true,
|
|
@@ -7678,7 +7691,7 @@
|
|
|
7678
7691
|
"immutable": true,
|
|
7679
7692
|
"locationInModule": {
|
|
7680
7693
|
"filename": "lib/job.ts",
|
|
7681
|
-
"line":
|
|
7694
|
+
"line": 531
|
|
7682
7695
|
},
|
|
7683
7696
|
"name": "workerCount",
|
|
7684
7697
|
"optional": true,
|
|
@@ -7696,7 +7709,7 @@
|
|
|
7696
7709
|
"immutable": true,
|
|
7697
7710
|
"locationInModule": {
|
|
7698
7711
|
"filename": "lib/job.ts",
|
|
7699
|
-
"line":
|
|
7712
|
+
"line": 524
|
|
7700
7713
|
},
|
|
7701
7714
|
"name": "workerType",
|
|
7702
7715
|
"optional": true,
|
|
@@ -7718,7 +7731,7 @@
|
|
|
7718
7731
|
"kind": "enum",
|
|
7719
7732
|
"locationInModule": {
|
|
7720
7733
|
"filename": "lib/job.ts",
|
|
7721
|
-
"line":
|
|
7734
|
+
"line": 78
|
|
7722
7735
|
},
|
|
7723
7736
|
"members": [
|
|
7724
7737
|
{
|
|
@@ -7920,7 +7933,7 @@
|
|
|
7920
7933
|
"kind": "enum",
|
|
7921
7934
|
"locationInModule": {
|
|
7922
7935
|
"filename": "lib/job.ts",
|
|
7923
|
-
"line":
|
|
7936
|
+
"line": 120
|
|
7924
7937
|
},
|
|
7925
7938
|
"members": [
|
|
7926
7939
|
{
|
|
@@ -8751,7 +8764,7 @@
|
|
|
8751
8764
|
"docs": {
|
|
8752
8765
|
"stability": "experimental",
|
|
8753
8766
|
"summary": "Props for creating a Scala Spark (ETL or Streaming) job executable.",
|
|
8754
|
-
"example": "declare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n description: 'an example Scala ETL job',\n});",
|
|
8767
|
+
"example": "declare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n workerType: glue.WorkerType.G_8X,\n description: 'an example Scala ETL job',\n});",
|
|
8755
8768
|
"custom": {
|
|
8756
8769
|
"exampleMetadata": "infused"
|
|
8757
8770
|
}
|
|
@@ -9793,7 +9806,7 @@
|
|
|
9793
9806
|
"kind": "interface",
|
|
9794
9807
|
"locationInModule": {
|
|
9795
9808
|
"filename": "lib/job.ts",
|
|
9796
|
-
"line":
|
|
9809
|
+
"line": 384
|
|
9797
9810
|
},
|
|
9798
9811
|
"name": "SparkUILoggingLocation",
|
|
9799
9812
|
"properties": [
|
|
@@ -9806,7 +9819,7 @@
|
|
|
9806
9819
|
"immutable": true,
|
|
9807
9820
|
"locationInModule": {
|
|
9808
9821
|
"filename": "lib/job.ts",
|
|
9809
|
-
"line":
|
|
9822
|
+
"line": 388
|
|
9810
9823
|
},
|
|
9811
9824
|
"name": "bucket",
|
|
9812
9825
|
"type": {
|
|
@@ -9823,7 +9836,7 @@
|
|
|
9823
9836
|
"immutable": true,
|
|
9824
9837
|
"locationInModule": {
|
|
9825
9838
|
"filename": "lib/job.ts",
|
|
9826
|
-
"line":
|
|
9839
|
+
"line": 395
|
|
9827
9840
|
},
|
|
9828
9841
|
"name": "prefix",
|
|
9829
9842
|
"optional": true,
|
|
@@ -9850,7 +9863,7 @@
|
|
|
9850
9863
|
"kind": "interface",
|
|
9851
9864
|
"locationInModule": {
|
|
9852
9865
|
"filename": "lib/job.ts",
|
|
9853
|
-
"line":
|
|
9866
|
+
"line": 357
|
|
9854
9867
|
},
|
|
9855
9868
|
"name": "SparkUIProps",
|
|
9856
9869
|
"properties": [
|
|
@@ -9863,7 +9876,7 @@
|
|
|
9863
9876
|
"immutable": true,
|
|
9864
9877
|
"locationInModule": {
|
|
9865
9878
|
"filename": "lib/job.ts",
|
|
9866
|
-
"line":
|
|
9879
|
+
"line": 361
|
|
9867
9880
|
},
|
|
9868
9881
|
"name": "enabled",
|
|
9869
9882
|
"type": {
|
|
@@ -9880,7 +9893,7 @@
|
|
|
9880
9893
|
"immutable": true,
|
|
9881
9894
|
"locationInModule": {
|
|
9882
9895
|
"filename": "lib/job.ts",
|
|
9883
|
-
"line":
|
|
9896
|
+
"line": 368
|
|
9884
9897
|
},
|
|
9885
9898
|
"name": "bucket",
|
|
9886
9899
|
"optional": true,
|
|
@@ -9898,7 +9911,7 @@
|
|
|
9898
9911
|
"immutable": true,
|
|
9899
9912
|
"locationInModule": {
|
|
9900
9913
|
"filename": "lib/job.ts",
|
|
9901
|
-
"line":
|
|
9914
|
+
"line": 375
|
|
9902
9915
|
},
|
|
9903
9916
|
"name": "prefix",
|
|
9904
9917
|
"optional": true,
|
|
@@ -10871,7 +10884,7 @@
|
|
|
10871
10884
|
"remarks": "If you need to use a WorkerType that doesn't exist as a static member, you\ncan instantiate a `WorkerType` object, e.g: `WorkerType.of('other type')`.",
|
|
10872
10885
|
"stability": "experimental",
|
|
10873
10886
|
"summary": "The type of predefined worker that is allocated when a job runs.",
|
|
10874
|
-
"example": "
|
|
10887
|
+
"example": "declare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n workerType: glue.WorkerType.G_8X,\n description: 'an example Scala ETL job',\n});",
|
|
10875
10888
|
"custom": {
|
|
10876
10889
|
"exampleMetadata": "infused"
|
|
10877
10890
|
}
|
|
@@ -10890,7 +10903,7 @@
|
|
|
10890
10903
|
},
|
|
10891
10904
|
"locationInModule": {
|
|
10892
10905
|
"filename": "lib/job.ts",
|
|
10893
|
-
"line":
|
|
10906
|
+
"line": 59
|
|
10894
10907
|
},
|
|
10895
10908
|
"name": "of",
|
|
10896
10909
|
"parameters": [
|
|
@@ -10923,7 +10936,7 @@
|
|
|
10923
10936
|
"immutable": true,
|
|
10924
10937
|
"locationInModule": {
|
|
10925
10938
|
"filename": "lib/job.ts",
|
|
10926
|
-
"line":
|
|
10939
|
+
"line": 48
|
|
10927
10940
|
},
|
|
10928
10941
|
"name": "G_025X",
|
|
10929
10942
|
"static": true,
|
|
@@ -10967,6 +10980,42 @@
|
|
|
10967
10980
|
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType"
|
|
10968
10981
|
}
|
|
10969
10982
|
},
|
|
10983
|
+
{
|
|
10984
|
+
"const": true,
|
|
10985
|
+
"docs": {
|
|
10986
|
+
"remarks": "We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for AWS Glue version 3.0 or later jobs.",
|
|
10987
|
+
"stability": "experimental",
|
|
10988
|
+
"summary": "Each worker maps to 4 DPU (16 vCPU, 64 GB of memory, 256 GB disk), and provides 1 executor per worker."
|
|
10989
|
+
},
|
|
10990
|
+
"immutable": true,
|
|
10991
|
+
"locationInModule": {
|
|
10992
|
+
"filename": "lib/job.ts",
|
|
10993
|
+
"line": 38
|
|
10994
|
+
},
|
|
10995
|
+
"name": "G_4X",
|
|
10996
|
+
"static": true,
|
|
10997
|
+
"type": {
|
|
10998
|
+
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType"
|
|
10999
|
+
}
|
|
11000
|
+
},
|
|
11001
|
+
{
|
|
11002
|
+
"const": true,
|
|
11003
|
+
"docs": {
|
|
11004
|
+
"remarks": "We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for AWS Glue version 3.0 or later jobs.",
|
|
11005
|
+
"stability": "experimental",
|
|
11006
|
+
"summary": "Each worker maps to 8 DPU (32 vCPU, 128 GB of memory, 512 GB disk), and provides 1 executor per worker."
|
|
11007
|
+
},
|
|
11008
|
+
"immutable": true,
|
|
11009
|
+
"locationInModule": {
|
|
11010
|
+
"filename": "lib/job.ts",
|
|
11011
|
+
"line": 43
|
|
11012
|
+
},
|
|
11013
|
+
"name": "G_8X",
|
|
11014
|
+
"static": true,
|
|
11015
|
+
"type": {
|
|
11016
|
+
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType"
|
|
11017
|
+
}
|
|
11018
|
+
},
|
|
10970
11019
|
{
|
|
10971
11020
|
"const": true,
|
|
10972
11021
|
"docs": {
|
|
@@ -10994,7 +11043,7 @@
|
|
|
10994
11043
|
"immutable": true,
|
|
10995
11044
|
"locationInModule": {
|
|
10996
11045
|
"filename": "lib/job.ts",
|
|
10997
|
-
"line":
|
|
11046
|
+
"line": 53
|
|
10998
11047
|
},
|
|
10999
11048
|
"name": "Z_2X",
|
|
11000
11049
|
"static": true,
|
|
@@ -11010,7 +11059,7 @@
|
|
|
11010
11059
|
"immutable": true,
|
|
11011
11060
|
"locationInModule": {
|
|
11012
11061
|
"filename": "lib/job.ts",
|
|
11013
|
-
"line":
|
|
11062
|
+
"line": 66
|
|
11014
11063
|
},
|
|
11015
11064
|
"name": "name",
|
|
11016
11065
|
"type": {
|
|
@@ -11021,6 +11070,6 @@
|
|
|
11021
11070
|
"symbolId": "lib/job:WorkerType"
|
|
11022
11071
|
}
|
|
11023
11072
|
},
|
|
11024
|
-
"version": "2.
|
|
11073
|
+
"version": "2.82.0-alpha.0",
|
|
11025
11074
|
"fingerprint": "**********"
|
|
11026
11075
|
}
|