@aws-cdk/aws-glue-alpha 2.64.0-alpha.0 → 2.65.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +323 -122
- package/.jsii.tabl.json.gz +0 -0
- package/.warnings.jsii.js +25 -1
- package/README.md +17 -0
- package/lib/code.js +3 -3
- package/lib/connection.js +2 -2
- package/lib/data-format.js +5 -5
- package/lib/database.js +1 -1
- package/lib/job-executable.d.ts +17 -2
- package/lib/job-executable.js +45 -9
- package/lib/job.d.ts +8 -0
- package/lib/job.js +14 -3
- package/lib/schema.js +1 -1
- package/lib/security-configuration.js +1 -1
- package/lib/table.js +1 -1
- package/package.json +7 -7
package/.jsii
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"url": "https://aws.amazon.com"
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"aws-cdk-lib": "^2.
|
|
11
|
+
"aws-cdk-lib": "^2.65.0",
|
|
12
12
|
"constructs": "^10.0.0"
|
|
13
13
|
},
|
|
14
14
|
"dependencyClosure": {
|
|
@@ -3370,7 +3370,7 @@
|
|
|
3370
3370
|
},
|
|
3371
3371
|
"name": "@aws-cdk/aws-glue-alpha",
|
|
3372
3372
|
"readme": {
|
|
3373
|
-
"markdown": "# AWS Glue Construct Library\n<!--BEGIN STABILITY BANNER-->\n\n---\n\n\n\n> The APIs of higher level constructs in this module are experimental and under active development.\n> They are subject to non-backward compatible changes or removal in any future version. These are\n> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be\n> announced in the release notes. This means that while you may use them, you may need to update\n> your source code when upgrading to a newer version of this package.\n\n---\n\n<!--END STABILITY BANNER-->\n\nThis module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project.\n\n## Job\n\nA `Job` encapsulates a script that connects to data sources, processes them, and then writes output to a data target.\n\nThere are 3 types of jobs supported by AWS Glue: Spark ETL, Spark Streaming, and Python Shell jobs.\n\nThe `glue.JobExecutable` allows you to specify the type of job, the language to use and the code assets required by the job.\n\n`glue.Code` allows you to refer to the different code assets required by the job, either from an existing S3 location or from a local file path.\n\n### Spark Jobs\n\nThese jobs run in an Apache Spark environment managed by AWS Glue.\n\n#### ETL Jobs\n\nAn ETL job processes data in batches using Apache Spark.\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n description: 'an example Scala ETL job',\n});\n```\n\n#### Streaming Jobs\n\nA Streaming job is similar to an ETL job, except that it performs ETL on data streams. It uses the Apache Spark Structured Streaming framework. Some Spark job features are not available to streaming ETL jobs.\n\n```ts\nnew glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});\n```\n\n### Python Shell Jobs\n\nA Python shell job runs Python scripts as a shell and supports a Python version that depends on the AWS Glue version you are using.\nThis can be used to schedule and run tasks that don't require an Apache Spark environment. Currently, three flavors are supported:\n\n* PythonVersion.TWO (2.7; EOL)\n* PythonVersion.THREE (3.6)\n* PythonVersion.THREE_NINE (3.9)\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'PythonShellJob', {\n executable: glue.JobExecutable.pythonShell({\n glueVersion: glue.GlueVersion.V1_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromBucket(bucket, 'script.py'),\n }),\n description: 'an example Python Shell job',\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/add-job.html) for more information on adding jobs in Glue.\n\n## Connection\n\nA `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. For example, to create a network connection to connect to a data source within a VPC:\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\nnew glue.Connection(this, 'MyConnection', {\n type: glue.ConnectionType.NETWORK,\n // The security groups granting AWS Glue inbound access to the data source within the VPC\n securityGroups: [securityGroup],\n // The VPC subnet which contains the data source\n subnet,\n});\n```\n\nFor RDS `Connection` by JDBC, it is recommended to manage credentials using AWS Secrets Manager. To use Secret, specify `SECRET_ID` in `properties` like the following code. Note that in this case, the subnet must have a route to the AWS Secrets Manager VPC endpoint or to the AWS Secrets Manager endpoint through a NAT gateway.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\ndeclare const db: rds.DatabaseCluster;\nnew glue.Connection(this, \"RdsConnection\", {\n type: glue.ConnectionType.JDBC,\n securityGroups: [securityGroup],\n subnet,\n properties: {\n JDBC_CONNECTION_URL: `jdbc:mysql://${db.clusterEndpoint.socketAddress}/databasename`,\n JDBC_ENFORCE_SSL: \"false\",\n SECRET_ID: db.secret!.secretName,\n },\n});\n```\n\nIf you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.\n\nSee [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.\n\n## SecurityConfiguration\n\nA `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.\n\n```ts\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n },\n jobBookmarksEncryption: {\n mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,\n },\n s3Encryption: {\n mode: glue.S3EncryptionMode.KMS,\n },\n});\n```\n\nBy default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:\n\n```ts\ndeclare const key: kms.Key;\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n kmsKey: key,\n },\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.\n\n## Database\n\nA `Database` is a logical grouping of `Tables` in the Glue Catalog.\n\n```ts\nnew glue.Database(this, 'MyDatabase');\n```\n\n## Table\n\nA Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }, {\n name: 'col2',\n type: glue.Schema.array(glue.Schema.STRING),\n comment: 'col2 is an array of strings' // comment is optional\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:\n\n```ts\ndeclare const myBucket: s3.Bucket;\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n bucket: myBucket,\n s3Prefix: 'my-table/',\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`:\n\n### Partition Keys\n\nTo improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Indexes\n\nAnother way to improve query performance is to specify partition indexes. If no partition indexes are\npresent on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using\nthe query expression. The query takes more time to run as the number of partitions increase. With an\nindex, the query will try to fetch a subset of the partitions instead of loading all partitions of the\ntable.\n\nThe keys of a partition index must be a subset of the partition keys of the table. You can have a\nmaximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`\nproperty:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n partitionIndexes: [{\n indexName: 'my-index', // optional\n keyNames: ['year'],\n }], // supply up to 3 indexes\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nAlternatively, you can call the `addPartitionIndex()` function on a table:\n\n```ts\ndeclare const myTable: glue.Table;\nmyTable.addPartitionIndex({\n indexName: 'my-index',\n keyNames: ['year'],\n});\n```\n\n### Partition Filtering\n\nIf you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n enablePartitionFiltering: true,\n});\n```\n\n## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)\n\nYou can enable encryption on a Table's data:\n\n* `Unencrypted` - files are not encrypted. The default encryption setting.\n* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - Server side encryption (`SSE-S3`) with an Amazon S3-managed key.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.S3_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n*Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.\n\n## Types\n\nA table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n columns: [{\n name: 'primitive_column',\n type: glue.Schema.STRING,\n }, {\n name: 'array_column',\n type: glue.Schema.array(glue.Schema.INTEGER),\n comment: 'array<integer>',\n }, {\n name: 'map_column',\n type: glue.Schema.map(\n glue.Schema.STRING,\n glue.Schema.TIMESTAMP),\n comment: 'map<string,string>',\n }, {\n name: 'struct_column',\n type: glue.Schema.struct([{\n name: 'nested_column',\n type: glue.Schema.DATE,\n comment: 'nested comment',\n }]),\n comment: \"struct<nested_column:date COMMENT 'nested comment'>\",\n }],\n // ...\n database: myDatabase,\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Primitives\n\n#### Numeric\n\n| Name \t| Type \t| Comments |\n|-----------\t|----------\t|------------------------------------------------------------------------------------------------------------------\t|\n| FLOAT \t| Constant \t| A 32-bit single-precision floating point number |\n| INTEGER \t| Constant \t| A 32-bit signed value in two's complement format, with a minimum value of -2^31 and a maximum value of 2^31-1 \t|\n| DOUBLE \t| Constant \t| A 64-bit double-precision floating point number |\n| BIG_INT \t| Constant \t| A 64-bit signed INTEGER in two’s complement format, with a minimum value of -2^63 and a maximum value of 2^63 -1 |\n| SMALL_INT \t| Constant \t| A 16-bit signed INTEGER in two’s complement format, with a minimum value of -2^15 and a maximum value of 2^15-1 |\n| TINY_INT \t| Constant \t| A 8-bit signed INTEGER in two’s complement format, with a minimum value of -2^7 and a maximum value of 2^7-1 |\n\n#### Date and time\n\n| Name \t| Type \t| Comments \t|\n|-----------\t|----------\t|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| DATE \t| Constant \t| A date in UNIX format, such as YYYY-MM-DD. \t|\n| TIMESTAMP \t| Constant \t| Date and time instant in the UNiX format, such as yyyy-mm-dd hh:mm:ss[.f...]. For example, TIMESTAMP '2008-09-15 03:04:05.324'. This format uses the session time zone. \t|\n\n#### String\n\n| Name \t| Type \t| Comments \t|\n|--------------------------------------------\t|----------\t|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| STRING \t| Constant \t| A string literal enclosed in single or double quotes \t|\n| decimal(precision: number, scale?: number) \t| Function \t| `precision` is the total number of digits. `scale` (optional) is the number of digits in fractional part with a default of 0. For example, use these type definitions: decimal(11,5), decimal(15) \t|\n| char(length: number) \t| Function \t| Fixed length character data, with a specified length between 1 and 255, such as char(10) \t|\n| varchar(length: number) \t| Function \t| Variable length character data, with a specified length between 1 and 65535, such as varchar(10) \t|\n\n#### Miscellaneous\n\n| Name \t| Type \t| Comments \t|\n|---------\t|----------\t|-------------------------------\t|\n| BOOLEAN \t| Constant \t| Values are `true` and `false` \t|\n| BINARY \t| Constant \t| Value is in binary \t|\n\n### Complex\n\n| Name \t| Type \t| Comments \t|\n|-------------------------------------\t|----------\t|-------------------------------------------------------------------\t|\n| array(itemType: Type) \t| Function \t| An array of some other type \t|\n| map(keyType: Type, valueType: Type) \t| Function \t| A map of some primitive key type to any value type \t|\n| struct(collumns: Column[]) \t| Function \t| Nested structure containing individually named and typed collumns \t|\n"
|
|
3373
|
+
"markdown": "# AWS Glue Construct Library\n<!--BEGIN STABILITY BANNER-->\n\n---\n\n\n\n> The APIs of higher level constructs in this module are experimental and under active development.\n> They are subject to non-backward compatible changes or removal in any future version. These are\n> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be\n> announced in the release notes. This means that while you may use them, you may need to update\n> your source code when upgrading to a newer version of this package.\n\n---\n\n<!--END STABILITY BANNER-->\n\nThis module is part of the [AWS Cloud Development Kit](https://github.com/aws/aws-cdk) project.\n\n## Job\n\nA `Job` encapsulates a script that connects to data sources, processes them, and then writes output to a data target.\n\nThere are 3 types of jobs supported by AWS Glue: Spark ETL, Spark Streaming, and Python Shell jobs.\n\nThe `glue.JobExecutable` allows you to specify the type of job, the language to use and the code assets required by the job.\n\n`glue.Code` allows you to refer to the different code assets required by the job, either from an existing S3 location or from a local file path.\n\n### Spark Jobs\n\nThese jobs run in an Apache Spark environment managed by AWS Glue.\n\n#### ETL Jobs\n\nAn ETL job processes data in batches using Apache Spark.\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'ScalaSparkEtlJob', {\n executable: glue.JobExecutable.scalaEtl({\n glueVersion: glue.GlueVersion.V4_0,\n script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'),\n className: 'com.example.HelloWorld',\n extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')],\n }),\n description: 'an example Scala ETL job',\n});\n```\n\n#### Streaming Jobs\n\nA Streaming job is similar to an ETL job, except that it performs ETL on data streams. It uses the Apache Spark Structured Streaming framework. Some Spark job features are not available to streaming ETL jobs.\n\n```ts\nnew glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});\n```\n\n### Python Shell Jobs\n\nA Python shell job runs Python scripts as a shell and supports a Python version that depends on the AWS Glue version you are using.\nThis can be used to schedule and run tasks that don't require an Apache Spark environment. Currently, three flavors are supported:\n\n* PythonVersion.TWO (2.7; EOL)\n* PythonVersion.THREE (3.6)\n* PythonVersion.THREE_NINE (3.9)\n\n```ts\ndeclare const bucket: s3.Bucket;\nnew glue.Job(this, 'PythonShellJob', {\n executable: glue.JobExecutable.pythonShell({\n glueVersion: glue.GlueVersion.V1_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromBucket(bucket, 'script.py'),\n }),\n description: 'an example Python Shell job',\n});\n```\n\n### Ray Jobs\n\nThese jobs run in a Ray environment managed by AWS Glue.\n\n```ts\nnew glue.Job(this, 'RayJob', {\n executable: glue.JobExecutable.pythonRay({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE_NINE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n workerType: glue.WorkerType.Z_2X,\n workerCount: 2,\n description: 'an example Ray job'\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/add-job.html) for more information on adding jobs in Glue.\n\n## Connection\n\nA `Connection` allows Glue jobs, crawlers and development endpoints to access certain types of data stores. For example, to create a network connection to connect to a data source within a VPC:\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\nnew glue.Connection(this, 'MyConnection', {\n type: glue.ConnectionType.NETWORK,\n // The security groups granting AWS Glue inbound access to the data source within the VPC\n securityGroups: [securityGroup],\n // The VPC subnet which contains the data source\n subnet,\n});\n```\n\nFor RDS `Connection` by JDBC, it is recommended to manage credentials using AWS Secrets Manager. To use Secret, specify `SECRET_ID` in `properties` like the following code. Note that in this case, the subnet must have a route to the AWS Secrets Manager VPC endpoint or to the AWS Secrets Manager endpoint through a NAT gateway.\n\n```ts\ndeclare const securityGroup: ec2.SecurityGroup;\ndeclare const subnet: ec2.Subnet;\ndeclare const db: rds.DatabaseCluster;\nnew glue.Connection(this, \"RdsConnection\", {\n type: glue.ConnectionType.JDBC,\n securityGroups: [securityGroup],\n subnet,\n properties: {\n JDBC_CONNECTION_URL: `jdbc:mysql://${db.clusterEndpoint.socketAddress}/databasename`,\n JDBC_ENFORCE_SSL: \"false\",\n SECRET_ID: db.secret!.secretName,\n },\n});\n```\n\nIf you need to use a connection type that doesn't exist as a static member on `ConnectionType`, you can instantiate a `ConnectionType` object, e.g: `new glue.ConnectionType('NEW_TYPE')`.\n\nSee [Adding a Connection to Your Data Store](https://docs.aws.amazon.com/glue/latest/dg/populate-add-connection.html) and [Connection Structure](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-connections.html#aws-glue-api-catalog-connections-Connection) documentation for more information on the supported data stores and their configurations.\n\n## SecurityConfiguration\n\nA `SecurityConfiguration` is a set of security properties that can be used by AWS Glue to encrypt data at rest.\n\n```ts\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n },\n jobBookmarksEncryption: {\n mode: glue.JobBookmarksEncryptionMode.CLIENT_SIDE_KMS,\n },\n s3Encryption: {\n mode: glue.S3EncryptionMode.KMS,\n },\n});\n```\n\nBy default, a shared KMS key is created for use with the encryption configurations that require one. You can also supply your own key for each encryption config, for example, for CloudWatch encryption:\n\n```ts\ndeclare const key: kms.Key;\nnew glue.SecurityConfiguration(this, 'MySecurityConfiguration', {\n cloudWatchEncryption: {\n mode: glue.CloudWatchEncryptionMode.KMS,\n kmsKey: key,\n },\n});\n```\n\nSee [documentation](https://docs.aws.amazon.com/glue/latest/dg/encryption-security-configuration.html) for more info for Glue encrypting data written by Crawlers, Jobs, and Development Endpoints.\n\n## Database\n\nA `Database` is a logical grouping of `Tables` in the Glue Catalog.\n\n```ts\nnew glue.Database(this, 'MyDatabase');\n```\n\n## Table\n\nA Glue table describes a table of data in S3: its structure (column names and types), location of data (S3 objects with a common prefix in a S3 bucket), and format for the files (Json, Avro, Parquet, etc.):\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }, {\n name: 'col2',\n type: glue.Schema.array(glue.Schema.STRING),\n comment: 'col2 is an array of strings' // comment is optional\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, a S3 bucket will be created to store the table's data but you can manually pass the `bucket` and `s3Prefix`:\n\n```ts\ndeclare const myBucket: s3.Bucket;\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n bucket: myBucket,\n s3Prefix: 'my-table/',\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nBy default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`:\n\n### Partition Keys\n\nTo improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Partition Indexes\n\nAnother way to improve query performance is to specify partition indexes. If no partition indexes are\npresent on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using\nthe query expression. The query takes more time to run as the number of partitions increase. With an\nindex, the query will try to fetch a subset of the partitions instead of loading all partitions of the\ntable.\n\nThe keys of a partition index must be a subset of the partition keys of the table. You can have a\nmaximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes`\nproperty:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n partitionIndexes: [{\n indexName: 'my-index', // optional\n keyNames: ['year'],\n }], // supply up to 3 indexes\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\nAlternatively, you can call the `addPartitionIndex()` function on a table:\n\n```ts\ndeclare const myTable: glue.Table;\nmyTable.addPartitionIndex({\n indexName: 'my-index',\n keyNames: ['year'],\n});\n```\n\n### Partition Filtering\n\nIf you have a table with a large number of partitions that grows over time, consider using AWS Glue partition indexing and filtering.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n partitionKeys: [{\n name: 'year',\n type: glue.Schema.SMALL_INT,\n }, {\n name: 'month',\n type: glue.Schema.SMALL_INT,\n }],\n dataFormat: glue.DataFormat.JSON,\n enablePartitionFiltering: true,\n});\n```\n\n## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html)\n\nYou can enable encryption on a Table's data:\n\n* `Unencrypted` - files are not encrypted. The default encryption setting.\n* [S3Managed](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) - Server side encryption (`SSE-S3`) with an Amazon S3-managed key.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.S3_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [Kms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [KmsManaged](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) - Server-side encryption (`SSE-KMS`), like `Kms`, except with an AWS KMS Key managed by the AWS Key Management Service.\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.KMS_MANAGED,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n* [ClientSideKms](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html#client-side-encryption-kms-managed-master-key-intro) - Client-side encryption (`CSE-KMS`) with an AWS KMS Key managed by the account owner.\n\n```ts\ndeclare const myDatabase: glue.Database;\n// KMS key is created automatically\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n\n// with an explicit KMS key\nnew glue.Table(this, 'MyTable', {\n encryption: glue.TableEncryption.CLIENT_SIDE_KMS,\n encryptionKey: new kms.Key(this, 'MyKey'),\n // ...\n database: myDatabase,\n columns: [{\n name: 'col1',\n type: glue.Schema.STRING,\n }],\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n*Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.\n\n## Types\n\nA table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:\n\n```ts\ndeclare const myDatabase: glue.Database;\nnew glue.Table(this, 'MyTable', {\n columns: [{\n name: 'primitive_column',\n type: glue.Schema.STRING,\n }, {\n name: 'array_column',\n type: glue.Schema.array(glue.Schema.INTEGER),\n comment: 'array<integer>',\n }, {\n name: 'map_column',\n type: glue.Schema.map(\n glue.Schema.STRING,\n glue.Schema.TIMESTAMP),\n comment: 'map<string,string>',\n }, {\n name: 'struct_column',\n type: glue.Schema.struct([{\n name: 'nested_column',\n type: glue.Schema.DATE,\n comment: 'nested comment',\n }]),\n comment: \"struct<nested_column:date COMMENT 'nested comment'>\",\n }],\n // ...\n database: myDatabase,\n dataFormat: glue.DataFormat.JSON,\n});\n```\n\n### Primitives\n\n#### Numeric\n\n| Name \t| Type \t| Comments |\n|-----------\t|----------\t|------------------------------------------------------------------------------------------------------------------\t|\n| FLOAT \t| Constant \t| A 32-bit single-precision floating point number |\n| INTEGER \t| Constant \t| A 32-bit signed value in two's complement format, with a minimum value of -2^31 and a maximum value of 2^31-1 \t|\n| DOUBLE \t| Constant \t| A 64-bit double-precision floating point number |\n| BIG_INT \t| Constant \t| A 64-bit signed INTEGER in two’s complement format, with a minimum value of -2^63 and a maximum value of 2^63 -1 |\n| SMALL_INT \t| Constant \t| A 16-bit signed INTEGER in two’s complement format, with a minimum value of -2^15 and a maximum value of 2^15-1 |\n| TINY_INT \t| Constant \t| A 8-bit signed INTEGER in two’s complement format, with a minimum value of -2^7 and a maximum value of 2^7-1 |\n\n#### Date and time\n\n| Name \t| Type \t| Comments \t|\n|-----------\t|----------\t|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| DATE \t| Constant \t| A date in UNIX format, such as YYYY-MM-DD. \t|\n| TIMESTAMP \t| Constant \t| Date and time instant in the UNiX format, such as yyyy-mm-dd hh:mm:ss[.f...]. For example, TIMESTAMP '2008-09-15 03:04:05.324'. This format uses the session time zone. \t|\n\n#### String\n\n| Name \t| Type \t| Comments \t|\n|--------------------------------------------\t|----------\t|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\t|\n| STRING \t| Constant \t| A string literal enclosed in single or double quotes \t|\n| decimal(precision: number, scale?: number) \t| Function \t| `precision` is the total number of digits. `scale` (optional) is the number of digits in fractional part with a default of 0. For example, use these type definitions: decimal(11,5), decimal(15) \t|\n| char(length: number) \t| Function \t| Fixed length character data, with a specified length between 1 and 255, such as char(10) \t|\n| varchar(length: number) \t| Function \t| Variable length character data, with a specified length between 1 and 65535, such as varchar(10) \t|\n\n#### Miscellaneous\n\n| Name \t| Type \t| Comments \t|\n|---------\t|----------\t|-------------------------------\t|\n| BOOLEAN \t| Constant \t| Values are `true` and `false` \t|\n| BINARY \t| Constant \t| Value is in binary \t|\n\n### Complex\n\n| Name \t| Type \t| Comments \t|\n|-------------------------------------\t|----------\t|-------------------------------------------------------------------\t|\n| array(itemType: Type) \t| Function \t| An array of some other type \t|\n| map(keyType: Type, valueType: Type) \t| Function \t| A map of some primitive key type to any value type \t|\n| struct(collumns: Column[]) \t| Function \t| Nested structure containing individually named and typed collumns \t|\n"
|
|
3374
3374
|
},
|
|
3375
3375
|
"repository": {
|
|
3376
3376
|
"directory": "packages/individual-packages/aws-glue",
|
|
@@ -3735,7 +3735,7 @@
|
|
|
3735
3735
|
"docs": {
|
|
3736
3736
|
"stability": "experimental",
|
|
3737
3737
|
"summary": "Represents a Glue Job's Code assets (an asset can be a scripts, a jar, a python file or any other file).",
|
|
3738
|
-
"example": "
|
|
3738
|
+
"example": "new glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});",
|
|
3739
3739
|
"custom": {
|
|
3740
3740
|
"exampleMetadata": "infused"
|
|
3741
3741
|
}
|
|
@@ -4525,7 +4525,7 @@
|
|
|
4525
4525
|
"kind": "interface",
|
|
4526
4526
|
"locationInModule": {
|
|
4527
4527
|
"filename": "lib/job.ts",
|
|
4528
|
-
"line":
|
|
4528
|
+
"line": 394
|
|
4529
4529
|
},
|
|
4530
4530
|
"name": "ContinuousLoggingProps",
|
|
4531
4531
|
"properties": [
|
|
@@ -4538,7 +4538,7 @@
|
|
|
4538
4538
|
"immutable": true,
|
|
4539
4539
|
"locationInModule": {
|
|
4540
4540
|
"filename": "lib/job.ts",
|
|
4541
|
-
"line":
|
|
4541
|
+
"line": 398
|
|
4542
4542
|
},
|
|
4543
4543
|
"name": "enabled",
|
|
4544
4544
|
"type": {
|
|
@@ -4556,7 +4556,7 @@
|
|
|
4556
4556
|
"immutable": true,
|
|
4557
4557
|
"locationInModule": {
|
|
4558
4558
|
"filename": "lib/job.ts",
|
|
4559
|
-
"line":
|
|
4559
|
+
"line": 428
|
|
4560
4560
|
},
|
|
4561
4561
|
"name": "conversionPattern",
|
|
4562
4562
|
"optional": true,
|
|
@@ -4574,7 +4574,7 @@
|
|
|
4574
4574
|
"immutable": true,
|
|
4575
4575
|
"locationInModule": {
|
|
4576
4576
|
"filename": "lib/job.ts",
|
|
4577
|
-
"line":
|
|
4577
|
+
"line": 405
|
|
4578
4578
|
},
|
|
4579
4579
|
"name": "logGroup",
|
|
4580
4580
|
"optional": true,
|
|
@@ -4592,7 +4592,7 @@
|
|
|
4592
4592
|
"immutable": true,
|
|
4593
4593
|
"locationInModule": {
|
|
4594
4594
|
"filename": "lib/job.ts",
|
|
4595
|
-
"line":
|
|
4595
|
+
"line": 412
|
|
4596
4596
|
},
|
|
4597
4597
|
"name": "logStreamPrefix",
|
|
4598
4598
|
"optional": true,
|
|
@@ -4610,7 +4610,7 @@
|
|
|
4610
4610
|
"immutable": true,
|
|
4611
4611
|
"locationInModule": {
|
|
4612
4612
|
"filename": "lib/job.ts",
|
|
4613
|
-
"line":
|
|
4613
|
+
"line": 419
|
|
4614
4614
|
},
|
|
4615
4615
|
"name": "quiet",
|
|
4616
4616
|
"optional": true,
|
|
@@ -5209,7 +5209,7 @@
|
|
|
5209
5209
|
"see": "https://docs.aws.amazon.com/glue/latest/dg/add-job.html.\n\nIf you need to use a GlueVersion that doesn't exist as a static member, you\ncan instantiate a `GlueVersion` object, e.g: `GlueVersion.of('1.5')`.",
|
|
5210
5210
|
"stability": "experimental",
|
|
5211
5211
|
"summary": "AWS Glue version determines the versions of Apache Spark and Python that are available to the job.",
|
|
5212
|
-
"example": "
|
|
5212
|
+
"example": "new glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});",
|
|
5213
5213
|
"custom": {
|
|
5214
5214
|
"exampleMetadata": "infused"
|
|
5215
5215
|
}
|
|
@@ -5516,7 +5516,7 @@
|
|
|
5516
5516
|
"kind": "interface",
|
|
5517
5517
|
"locationInModule": {
|
|
5518
5518
|
"filename": "lib/job.ts",
|
|
5519
|
-
"line":
|
|
5519
|
+
"line": 125
|
|
5520
5520
|
},
|
|
5521
5521
|
"methods": [
|
|
5522
5522
|
{
|
|
@@ -5528,7 +5528,7 @@
|
|
|
5528
5528
|
},
|
|
5529
5529
|
"locationInModule": {
|
|
5530
5530
|
"filename": "lib/job.ts",
|
|
5531
|
-
"line":
|
|
5531
|
+
"line": 182
|
|
5532
5532
|
},
|
|
5533
5533
|
"name": "metric",
|
|
5534
5534
|
"parameters": [
|
|
@@ -5575,7 +5575,7 @@
|
|
|
5575
5575
|
},
|
|
5576
5576
|
"locationInModule": {
|
|
5577
5577
|
"filename": "lib/job.ts",
|
|
5578
|
-
"line":
|
|
5578
|
+
"line": 192
|
|
5579
5579
|
},
|
|
5580
5580
|
"name": "metricFailure",
|
|
5581
5581
|
"parameters": [
|
|
@@ -5601,7 +5601,7 @@
|
|
|
5601
5601
|
},
|
|
5602
5602
|
"locationInModule": {
|
|
5603
5603
|
"filename": "lib/job.ts",
|
|
5604
|
-
"line":
|
|
5604
|
+
"line": 187
|
|
5605
5605
|
},
|
|
5606
5606
|
"name": "metricSuccess",
|
|
5607
5607
|
"parameters": [
|
|
@@ -5627,7 +5627,7 @@
|
|
|
5627
5627
|
},
|
|
5628
5628
|
"locationInModule": {
|
|
5629
5629
|
"filename": "lib/job.ts",
|
|
5630
|
-
"line":
|
|
5630
|
+
"line": 197
|
|
5631
5631
|
},
|
|
5632
5632
|
"name": "metricTimeout",
|
|
5633
5633
|
"parameters": [
|
|
@@ -5654,7 +5654,7 @@
|
|
|
5654
5654
|
},
|
|
5655
5655
|
"locationInModule": {
|
|
5656
5656
|
"filename": "lib/job.ts",
|
|
5657
|
-
"line":
|
|
5657
|
+
"line": 143
|
|
5658
5658
|
},
|
|
5659
5659
|
"name": "onEvent",
|
|
5660
5660
|
"parameters": [
|
|
@@ -5687,7 +5687,7 @@
|
|
|
5687
5687
|
},
|
|
5688
5688
|
"locationInModule": {
|
|
5689
5689
|
"filename": "lib/job.ts",
|
|
5690
|
-
"line":
|
|
5690
|
+
"line": 164
|
|
5691
5691
|
},
|
|
5692
5692
|
"name": "onFailure",
|
|
5693
5693
|
"parameters": [
|
|
@@ -5720,7 +5720,7 @@
|
|
|
5720
5720
|
},
|
|
5721
5721
|
"locationInModule": {
|
|
5722
5722
|
"filename": "lib/job.ts",
|
|
5723
|
-
"line":
|
|
5723
|
+
"line": 150
|
|
5724
5724
|
},
|
|
5725
5725
|
"name": "onStateChange",
|
|
5726
5726
|
"parameters": [
|
|
@@ -5759,7 +5759,7 @@
|
|
|
5759
5759
|
},
|
|
5760
5760
|
"locationInModule": {
|
|
5761
5761
|
"filename": "lib/job.ts",
|
|
5762
|
-
"line":
|
|
5762
|
+
"line": 157
|
|
5763
5763
|
},
|
|
5764
5764
|
"name": "onSuccess",
|
|
5765
5765
|
"parameters": [
|
|
@@ -5792,7 +5792,7 @@
|
|
|
5792
5792
|
},
|
|
5793
5793
|
"locationInModule": {
|
|
5794
5794
|
"filename": "lib/job.ts",
|
|
5795
|
-
"line":
|
|
5795
|
+
"line": 171
|
|
5796
5796
|
},
|
|
5797
5797
|
"name": "onTimeout",
|
|
5798
5798
|
"parameters": [
|
|
@@ -5831,7 +5831,7 @@
|
|
|
5831
5831
|
"immutable": true,
|
|
5832
5832
|
"locationInModule": {
|
|
5833
5833
|
"filename": "lib/job.ts",
|
|
5834
|
-
"line":
|
|
5834
|
+
"line": 136
|
|
5835
5835
|
},
|
|
5836
5836
|
"name": "jobArn",
|
|
5837
5837
|
"type": {
|
|
@@ -5850,7 +5850,7 @@
|
|
|
5850
5850
|
"immutable": true,
|
|
5851
5851
|
"locationInModule": {
|
|
5852
5852
|
"filename": "lib/job.ts",
|
|
5853
|
-
"line":
|
|
5853
|
+
"line": 130
|
|
5854
5854
|
},
|
|
5855
5855
|
"name": "jobName",
|
|
5856
5856
|
"type": {
|
|
@@ -6103,7 +6103,7 @@
|
|
|
6103
6103
|
"docs": {
|
|
6104
6104
|
"stability": "experimental",
|
|
6105
6105
|
"summary": "A Glue Job.",
|
|
6106
|
-
"example": "
|
|
6106
|
+
"example": "new glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});",
|
|
6107
6107
|
"custom": {
|
|
6108
6108
|
"exampleMetadata": "infused"
|
|
6109
6109
|
}
|
|
@@ -6115,7 +6115,7 @@
|
|
|
6115
6115
|
},
|
|
6116
6116
|
"locationInModule": {
|
|
6117
6117
|
"filename": "lib/job.ts",
|
|
6118
|
-
"line":
|
|
6118
|
+
"line": 644
|
|
6119
6119
|
},
|
|
6120
6120
|
"parameters": [
|
|
6121
6121
|
{
|
|
@@ -6144,7 +6144,7 @@
|
|
|
6144
6144
|
"kind": "class",
|
|
6145
6145
|
"locationInModule": {
|
|
6146
6146
|
"filename": "lib/job.ts",
|
|
6147
|
-
"line":
|
|
6147
|
+
"line": 598
|
|
6148
6148
|
},
|
|
6149
6149
|
"methods": [
|
|
6150
6150
|
{
|
|
@@ -6154,7 +6154,7 @@
|
|
|
6154
6154
|
},
|
|
6155
6155
|
"locationInModule": {
|
|
6156
6156
|
"filename": "lib/job.ts",
|
|
6157
|
-
"line":
|
|
6157
|
+
"line": 606
|
|
6158
6158
|
},
|
|
6159
6159
|
"name": "fromJobAttributes",
|
|
6160
6160
|
"parameters": [
|
|
@@ -6201,7 +6201,7 @@
|
|
|
6201
6201
|
},
|
|
6202
6202
|
"locationInModule": {
|
|
6203
6203
|
"filename": "lib/job.ts",
|
|
6204
|
-
"line":
|
|
6204
|
+
"line": 289
|
|
6205
6205
|
},
|
|
6206
6206
|
"name": "metric",
|
|
6207
6207
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6249,7 +6249,7 @@
|
|
|
6249
6249
|
},
|
|
6250
6250
|
"locationInModule": {
|
|
6251
6251
|
"filename": "lib/job.ts",
|
|
6252
|
-
"line":
|
|
6252
|
+
"line": 316
|
|
6253
6253
|
},
|
|
6254
6254
|
"name": "metricFailure",
|
|
6255
6255
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6276,7 +6276,7 @@
|
|
|
6276
6276
|
},
|
|
6277
6277
|
"locationInModule": {
|
|
6278
6278
|
"filename": "lib/job.ts",
|
|
6279
|
-
"line":
|
|
6279
|
+
"line": 307
|
|
6280
6280
|
},
|
|
6281
6281
|
"name": "metricSuccess",
|
|
6282
6282
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6303,7 +6303,7 @@
|
|
|
6303
6303
|
},
|
|
6304
6304
|
"locationInModule": {
|
|
6305
6305
|
"filename": "lib/job.ts",
|
|
6306
|
-
"line":
|
|
6306
|
+
"line": 325
|
|
6307
6307
|
},
|
|
6308
6308
|
"name": "metricTimeout",
|
|
6309
6309
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6330,7 +6330,7 @@
|
|
|
6330
6330
|
},
|
|
6331
6331
|
"locationInModule": {
|
|
6332
6332
|
"filename": "lib/job.ts",
|
|
6333
|
-
"line":
|
|
6333
|
+
"line": 217
|
|
6334
6334
|
},
|
|
6335
6335
|
"name": "onEvent",
|
|
6336
6336
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6369,7 +6369,7 @@
|
|
|
6369
6369
|
},
|
|
6370
6370
|
"locationInModule": {
|
|
6371
6371
|
"filename": "lib/job.ts",
|
|
6372
|
-
"line":
|
|
6372
|
+
"line": 266
|
|
6373
6373
|
},
|
|
6374
6374
|
"name": "onFailure",
|
|
6375
6375
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6408,7 +6408,7 @@
|
|
|
6408
6408
|
},
|
|
6409
6409
|
"locationInModule": {
|
|
6410
6410
|
"filename": "lib/job.ts",
|
|
6411
|
-
"line":
|
|
6411
|
+
"line": 237
|
|
6412
6412
|
},
|
|
6413
6413
|
"name": "onStateChange",
|
|
6414
6414
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6455,7 +6455,7 @@
|
|
|
6455
6455
|
},
|
|
6456
6456
|
"locationInModule": {
|
|
6457
6457
|
"filename": "lib/job.ts",
|
|
6458
|
-
"line":
|
|
6458
|
+
"line": 256
|
|
6459
6459
|
},
|
|
6460
6460
|
"name": "onSuccess",
|
|
6461
6461
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6494,7 +6494,7 @@
|
|
|
6494
6494
|
},
|
|
6495
6495
|
"locationInModule": {
|
|
6496
6496
|
"filename": "lib/job.ts",
|
|
6497
|
-
"line":
|
|
6497
|
+
"line": 276
|
|
6498
6498
|
},
|
|
6499
6499
|
"name": "onTimeout",
|
|
6500
6500
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6537,7 +6537,7 @@
|
|
|
6537
6537
|
"immutable": true,
|
|
6538
6538
|
"locationInModule": {
|
|
6539
6539
|
"filename": "lib/job.ts",
|
|
6540
|
-
"line":
|
|
6540
|
+
"line": 634
|
|
6541
6541
|
},
|
|
6542
6542
|
"name": "grantPrincipal",
|
|
6543
6543
|
"overrides": "aws-cdk-lib.aws_iam.IGrantable",
|
|
@@ -6553,7 +6553,7 @@
|
|
|
6553
6553
|
"immutable": true,
|
|
6554
6554
|
"locationInModule": {
|
|
6555
6555
|
"filename": "lib/job.ts",
|
|
6556
|
-
"line":
|
|
6556
|
+
"line": 619
|
|
6557
6557
|
},
|
|
6558
6558
|
"name": "jobArn",
|
|
6559
6559
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6569,7 +6569,7 @@
|
|
|
6569
6569
|
"immutable": true,
|
|
6570
6570
|
"locationInModule": {
|
|
6571
6571
|
"filename": "lib/job.ts",
|
|
6572
|
-
"line":
|
|
6572
|
+
"line": 624
|
|
6573
6573
|
},
|
|
6574
6574
|
"name": "jobName",
|
|
6575
6575
|
"overrides": "@aws-cdk/aws-glue-alpha.IJob",
|
|
@@ -6585,7 +6585,7 @@
|
|
|
6585
6585
|
"immutable": true,
|
|
6586
6586
|
"locationInModule": {
|
|
6587
6587
|
"filename": "lib/job.ts",
|
|
6588
|
-
"line":
|
|
6588
|
+
"line": 629
|
|
6589
6589
|
},
|
|
6590
6590
|
"name": "role",
|
|
6591
6591
|
"type": {
|
|
@@ -6601,7 +6601,7 @@
|
|
|
6601
6601
|
"immutable": true,
|
|
6602
6602
|
"locationInModule": {
|
|
6603
6603
|
"filename": "lib/job.ts",
|
|
6604
|
-
"line":
|
|
6604
|
+
"line": 642
|
|
6605
6605
|
},
|
|
6606
6606
|
"name": "sparkUILoggingLocation",
|
|
6607
6607
|
"optional": true,
|
|
@@ -6627,7 +6627,7 @@
|
|
|
6627
6627
|
"kind": "interface",
|
|
6628
6628
|
"locationInModule": {
|
|
6629
6629
|
"filename": "lib/job.ts",
|
|
6630
|
-
"line":
|
|
6630
|
+
"line": 434
|
|
6631
6631
|
},
|
|
6632
6632
|
"name": "JobAttributes",
|
|
6633
6633
|
"properties": [
|
|
@@ -6640,7 +6640,7 @@
|
|
|
6640
6640
|
"immutable": true,
|
|
6641
6641
|
"locationInModule": {
|
|
6642
6642
|
"filename": "lib/job.ts",
|
|
6643
|
-
"line":
|
|
6643
|
+
"line": 438
|
|
6644
6644
|
},
|
|
6645
6645
|
"name": "jobName",
|
|
6646
6646
|
"type": {
|
|
@@ -6657,7 +6657,7 @@
|
|
|
6657
6657
|
"immutable": true,
|
|
6658
6658
|
"locationInModule": {
|
|
6659
6659
|
"filename": "lib/job.ts",
|
|
6660
|
-
"line":
|
|
6660
|
+
"line": 445
|
|
6661
6661
|
},
|
|
6662
6662
|
"name": "role",
|
|
6663
6663
|
"optional": true,
|
|
@@ -6759,7 +6759,7 @@
|
|
|
6759
6759
|
"docs": {
|
|
6760
6760
|
"stability": "experimental",
|
|
6761
6761
|
"summary": "The executable properties related to the Glue job's GlueVersion, JobType and code.",
|
|
6762
|
-
"example": "
|
|
6762
|
+
"example": "new glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});",
|
|
6763
6763
|
"custom": {
|
|
6764
6764
|
"exampleMetadata": "infused"
|
|
6765
6765
|
}
|
|
@@ -6768,7 +6768,7 @@
|
|
|
6768
6768
|
"kind": "class",
|
|
6769
6769
|
"locationInModule": {
|
|
6770
6770
|
"filename": "lib/job-executable.ts",
|
|
6771
|
-
"line":
|
|
6771
|
+
"line": 227
|
|
6772
6772
|
},
|
|
6773
6773
|
"methods": [
|
|
6774
6774
|
{
|
|
@@ -6778,7 +6778,7 @@
|
|
|
6778
6778
|
},
|
|
6779
6779
|
"locationInModule": {
|
|
6780
6780
|
"filename": "lib/job-executable.ts",
|
|
6781
|
-
"line":
|
|
6781
|
+
"line": 312
|
|
6782
6782
|
},
|
|
6783
6783
|
"name": "of",
|
|
6784
6784
|
"parameters": [
|
|
@@ -6806,7 +6806,7 @@
|
|
|
6806
6806
|
},
|
|
6807
6807
|
"locationInModule": {
|
|
6808
6808
|
"filename": "lib/job-executable.ts",
|
|
6809
|
-
"line":
|
|
6809
|
+
"line": 260
|
|
6810
6810
|
},
|
|
6811
6811
|
"name": "pythonEtl",
|
|
6812
6812
|
"parameters": [
|
|
@@ -6827,6 +6827,34 @@
|
|
|
6827
6827
|
},
|
|
6828
6828
|
"static": true
|
|
6829
6829
|
},
|
|
6830
|
+
{
|
|
6831
|
+
"docs": {
|
|
6832
|
+
"stability": "experimental",
|
|
6833
|
+
"summary": "Create Python executable props for Ray jobs."
|
|
6834
|
+
},
|
|
6835
|
+
"locationInModule": {
|
|
6836
|
+
"filename": "lib/job-executable.ts",
|
|
6837
|
+
"line": 299
|
|
6838
|
+
},
|
|
6839
|
+
"name": "pythonRay",
|
|
6840
|
+
"parameters": [
|
|
6841
|
+
{
|
|
6842
|
+
"docs": {
|
|
6843
|
+
"summary": "Ray Job props."
|
|
6844
|
+
},
|
|
6845
|
+
"name": "props",
|
|
6846
|
+
"type": {
|
|
6847
|
+
"fqn": "@aws-cdk/aws-glue-alpha.PythonRayExecutableProps"
|
|
6848
|
+
}
|
|
6849
|
+
}
|
|
6850
|
+
],
|
|
6851
|
+
"returns": {
|
|
6852
|
+
"type": {
|
|
6853
|
+
"fqn": "@aws-cdk/aws-glue-alpha.JobExecutable"
|
|
6854
|
+
}
|
|
6855
|
+
},
|
|
6856
|
+
"static": true
|
|
6857
|
+
},
|
|
6830
6858
|
{
|
|
6831
6859
|
"docs": {
|
|
6832
6860
|
"stability": "experimental",
|
|
@@ -6834,7 +6862,7 @@
|
|
|
6834
6862
|
},
|
|
6835
6863
|
"locationInModule": {
|
|
6836
6864
|
"filename": "lib/job-executable.ts",
|
|
6837
|
-
"line":
|
|
6865
|
+
"line": 286
|
|
6838
6866
|
},
|
|
6839
6867
|
"name": "pythonShell",
|
|
6840
6868
|
"parameters": [
|
|
@@ -6862,7 +6890,7 @@
|
|
|
6862
6890
|
},
|
|
6863
6891
|
"locationInModule": {
|
|
6864
6892
|
"filename": "lib/job-executable.ts",
|
|
6865
|
-
"line":
|
|
6893
|
+
"line": 273
|
|
6866
6894
|
},
|
|
6867
6895
|
"name": "pythonStreaming",
|
|
6868
6896
|
"parameters": [
|
|
@@ -6890,7 +6918,7 @@
|
|
|
6890
6918
|
},
|
|
6891
6919
|
"locationInModule": {
|
|
6892
6920
|
"filename": "lib/job-executable.ts",
|
|
6893
|
-
"line":
|
|
6921
|
+
"line": 234
|
|
6894
6922
|
},
|
|
6895
6923
|
"name": "scalaEtl",
|
|
6896
6924
|
"parameters": [
|
|
@@ -6918,7 +6946,7 @@
|
|
|
6918
6946
|
},
|
|
6919
6947
|
"locationInModule": {
|
|
6920
6948
|
"filename": "lib/job-executable.ts",
|
|
6921
|
-
"line":
|
|
6949
|
+
"line": 247
|
|
6922
6950
|
},
|
|
6923
6951
|
"name": "scalaStreaming",
|
|
6924
6952
|
"parameters": [
|
|
@@ -6946,7 +6974,7 @@
|
|
|
6946
6974
|
},
|
|
6947
6975
|
"locationInModule": {
|
|
6948
6976
|
"filename": "lib/job-executable.ts",
|
|
6949
|
-
"line":
|
|
6977
|
+
"line": 356
|
|
6950
6978
|
},
|
|
6951
6979
|
"name": "bind",
|
|
6952
6980
|
"returns": {
|
|
@@ -6974,7 +7002,7 @@
|
|
|
6974
7002
|
"kind": "interface",
|
|
6975
7003
|
"locationInModule": {
|
|
6976
7004
|
"filename": "lib/job-executable.ts",
|
|
6977
|
-
"line":
|
|
7005
|
+
"line": 364
|
|
6978
7006
|
},
|
|
6979
7007
|
"name": "JobExecutableConfig",
|
|
6980
7008
|
"properties": [
|
|
@@ -6988,7 +7016,7 @@
|
|
|
6988
7016
|
"immutable": true,
|
|
6989
7017
|
"locationInModule": {
|
|
6990
7018
|
"filename": "lib/job-executable.ts",
|
|
6991
|
-
"line":
|
|
7019
|
+
"line": 370
|
|
6992
7020
|
},
|
|
6993
7021
|
"name": "glueVersion",
|
|
6994
7022
|
"type": {
|
|
@@ -7005,7 +7033,7 @@
|
|
|
7005
7033
|
"immutable": true,
|
|
7006
7034
|
"locationInModule": {
|
|
7007
7035
|
"filename": "lib/job-executable.ts",
|
|
7008
|
-
"line":
|
|
7036
|
+
"line": 377
|
|
7009
7037
|
},
|
|
7010
7038
|
"name": "language",
|
|
7011
7039
|
"type": {
|
|
@@ -7021,7 +7049,7 @@
|
|
|
7021
7049
|
"immutable": true,
|
|
7022
7050
|
"locationInModule": {
|
|
7023
7051
|
"filename": "lib/job-executable.ts",
|
|
7024
|
-
"line":
|
|
7052
|
+
"line": 394
|
|
7025
7053
|
},
|
|
7026
7054
|
"name": "script",
|
|
7027
7055
|
"type": {
|
|
@@ -7037,7 +7065,7 @@
|
|
|
7037
7065
|
"immutable": true,
|
|
7038
7066
|
"locationInModule": {
|
|
7039
7067
|
"filename": "lib/job-executable.ts",
|
|
7040
|
-
"line":
|
|
7068
|
+
"line": 382
|
|
7041
7069
|
},
|
|
7042
7070
|
"name": "type",
|
|
7043
7071
|
"type": {
|
|
@@ -7056,7 +7084,7 @@
|
|
|
7056
7084
|
"immutable": true,
|
|
7057
7085
|
"locationInModule": {
|
|
7058
7086
|
"filename": "lib/job-executable.ts",
|
|
7059
|
-
"line":
|
|
7087
|
+
"line": 403
|
|
7060
7088
|
},
|
|
7061
7089
|
"name": "className",
|
|
7062
7090
|
"optional": true,
|
|
@@ -7075,7 +7103,7 @@
|
|
|
7075
7103
|
"immutable": true,
|
|
7076
7104
|
"locationInModule": {
|
|
7077
7105
|
"filename": "lib/job-executable.ts",
|
|
7078
|
-
"line":
|
|
7106
|
+
"line": 430
|
|
7079
7107
|
},
|
|
7080
7108
|
"name": "extraFiles",
|
|
7081
7109
|
"optional": true,
|
|
@@ -7099,7 +7127,7 @@
|
|
|
7099
7127
|
"immutable": true,
|
|
7100
7128
|
"locationInModule": {
|
|
7101
7129
|
"filename": "lib/job-executable.ts",
|
|
7102
|
-
"line":
|
|
7130
|
+
"line": 412
|
|
7103
7131
|
},
|
|
7104
7132
|
"name": "extraJars",
|
|
7105
7133
|
"optional": true,
|
|
@@ -7123,7 +7151,7 @@
|
|
|
7123
7151
|
"immutable": true,
|
|
7124
7152
|
"locationInModule": {
|
|
7125
7153
|
"filename": "lib/job-executable.ts",
|
|
7126
|
-
"line":
|
|
7154
|
+
"line": 439
|
|
7127
7155
|
},
|
|
7128
7156
|
"name": "extraJarsFirst",
|
|
7129
7157
|
"optional": true,
|
|
@@ -7142,7 +7170,7 @@
|
|
|
7142
7170
|
"immutable": true,
|
|
7143
7171
|
"locationInModule": {
|
|
7144
7172
|
"filename": "lib/job-executable.ts",
|
|
7145
|
-
"line":
|
|
7173
|
+
"line": 421
|
|
7146
7174
|
},
|
|
7147
7175
|
"name": "extraPythonFiles",
|
|
7148
7176
|
"optional": true,
|
|
@@ -7165,7 +7193,7 @@
|
|
|
7165
7193
|
"immutable": true,
|
|
7166
7194
|
"locationInModule": {
|
|
7167
7195
|
"filename": "lib/job-executable.ts",
|
|
7168
|
-
"line":
|
|
7196
|
+
"line": 389
|
|
7169
7197
|
},
|
|
7170
7198
|
"name": "pythonVersion",
|
|
7171
7199
|
"optional": true,
|
|
@@ -7213,7 +7241,7 @@
|
|
|
7213
7241
|
"docs": {
|
|
7214
7242
|
"stability": "experimental",
|
|
7215
7243
|
"summary": "Construction properties for `Job`.",
|
|
7216
|
-
"example": "
|
|
7244
|
+
"example": "new glue.Job(this, 'PythonSparkStreamingJob', {\n executable: glue.JobExecutable.pythonStreaming({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n description: 'an example Python Streaming job',\n});",
|
|
7217
7245
|
"custom": {
|
|
7218
7246
|
"exampleMetadata": "infused"
|
|
7219
7247
|
}
|
|
@@ -7222,7 +7250,7 @@
|
|
|
7222
7250
|
"kind": "interface",
|
|
7223
7251
|
"locationInModule": {
|
|
7224
7252
|
"filename": "lib/job.ts",
|
|
7225
|
-
"line":
|
|
7253
|
+
"line": 451
|
|
7226
7254
|
},
|
|
7227
7255
|
"name": "JobProps",
|
|
7228
7256
|
"properties": [
|
|
@@ -7235,7 +7263,7 @@
|
|
|
7235
7263
|
"immutable": true,
|
|
7236
7264
|
"locationInModule": {
|
|
7237
7265
|
"filename": "lib/job.ts",
|
|
7238
|
-
"line":
|
|
7266
|
+
"line": 455
|
|
7239
7267
|
},
|
|
7240
7268
|
"name": "executable",
|
|
7241
7269
|
"type": {
|
|
@@ -7253,7 +7281,7 @@
|
|
|
7253
7281
|
"immutable": true,
|
|
7254
7282
|
"locationInModule": {
|
|
7255
7283
|
"filename": "lib/job.ts",
|
|
7256
|
-
"line":
|
|
7284
|
+
"line": 530
|
|
7257
7285
|
},
|
|
7258
7286
|
"name": "connections",
|
|
7259
7287
|
"optional": true,
|
|
@@ -7277,7 +7305,7 @@
|
|
|
7277
7305
|
"immutable": true,
|
|
7278
7306
|
"locationInModule": {
|
|
7279
7307
|
"filename": "lib/job.ts",
|
|
7280
|
-
"line":
|
|
7308
|
+
"line": 592
|
|
7281
7309
|
},
|
|
7282
7310
|
"name": "continuousLogging",
|
|
7283
7311
|
"optional": true,
|
|
@@ -7296,7 +7324,7 @@
|
|
|
7296
7324
|
"immutable": true,
|
|
7297
7325
|
"locationInModule": {
|
|
7298
7326
|
"filename": "lib/job.ts",
|
|
7299
|
-
"line":
|
|
7327
|
+
"line": 545
|
|
7300
7328
|
},
|
|
7301
7329
|
"name": "defaultArguments",
|
|
7302
7330
|
"optional": true,
|
|
@@ -7319,7 +7347,7 @@
|
|
|
7319
7347
|
"immutable": true,
|
|
7320
7348
|
"locationInModule": {
|
|
7321
7349
|
"filename": "lib/job.ts",
|
|
7322
|
-
"line":
|
|
7350
|
+
"line": 469
|
|
7323
7351
|
},
|
|
7324
7352
|
"name": "description",
|
|
7325
7353
|
"optional": true,
|
|
@@ -7338,7 +7366,7 @@
|
|
|
7338
7366
|
"immutable": true,
|
|
7339
7367
|
"locationInModule": {
|
|
7340
7368
|
"filename": "lib/job.ts",
|
|
7341
|
-
"line":
|
|
7369
|
+
"line": 572
|
|
7342
7370
|
},
|
|
7343
7371
|
"name": "enableProfilingMetrics",
|
|
7344
7372
|
"optional": true,
|
|
@@ -7356,7 +7384,7 @@
|
|
|
7356
7384
|
"immutable": true,
|
|
7357
7385
|
"locationInModule": {
|
|
7358
7386
|
"filename": "lib/job.ts",
|
|
7359
|
-
"line":
|
|
7387
|
+
"line": 462
|
|
7360
7388
|
},
|
|
7361
7389
|
"name": "jobName",
|
|
7362
7390
|
"optional": true,
|
|
@@ -7375,7 +7403,7 @@
|
|
|
7375
7403
|
"immutable": true,
|
|
7376
7404
|
"locationInModule": {
|
|
7377
7405
|
"filename": "lib/job.ts",
|
|
7378
|
-
"line":
|
|
7406
|
+
"line": 477
|
|
7379
7407
|
},
|
|
7380
7408
|
"name": "maxCapacity",
|
|
7381
7409
|
"optional": true,
|
|
@@ -7394,7 +7422,7 @@
|
|
|
7394
7422
|
"immutable": true,
|
|
7395
7423
|
"locationInModule": {
|
|
7396
7424
|
"filename": "lib/job.ts",
|
|
7397
|
-
"line":
|
|
7425
|
+
"line": 493
|
|
7398
7426
|
},
|
|
7399
7427
|
"name": "maxConcurrentRuns",
|
|
7400
7428
|
"optional": true,
|
|
@@ -7412,7 +7440,7 @@
|
|
|
7412
7440
|
"immutable": true,
|
|
7413
7441
|
"locationInModule": {
|
|
7414
7442
|
"filename": "lib/job.ts",
|
|
7415
|
-
"line":
|
|
7443
|
+
"line": 484
|
|
7416
7444
|
},
|
|
7417
7445
|
"name": "maxRetries",
|
|
7418
7446
|
"optional": true,
|
|
@@ -7430,7 +7458,7 @@
|
|
|
7430
7458
|
"immutable": true,
|
|
7431
7459
|
"locationInModule": {
|
|
7432
7460
|
"filename": "lib/job.ts",
|
|
7433
|
-
"line":
|
|
7461
|
+
"line": 500
|
|
7434
7462
|
},
|
|
7435
7463
|
"name": "notifyDelayAfter",
|
|
7436
7464
|
"optional": true,
|
|
@@ -7450,7 +7478,7 @@
|
|
|
7450
7478
|
"immutable": true,
|
|
7451
7479
|
"locationInModule": {
|
|
7452
7480
|
"filename": "lib/job.ts",
|
|
7453
|
-
"line":
|
|
7481
|
+
"line": 563
|
|
7454
7482
|
},
|
|
7455
7483
|
"name": "role",
|
|
7456
7484
|
"optional": true,
|
|
@@ -7468,7 +7496,7 @@
|
|
|
7468
7496
|
"immutable": true,
|
|
7469
7497
|
"locationInModule": {
|
|
7470
7498
|
"filename": "lib/job.ts",
|
|
7471
|
-
"line":
|
|
7499
|
+
"line": 537
|
|
7472
7500
|
},
|
|
7473
7501
|
"name": "securityConfiguration",
|
|
7474
7502
|
"optional": true,
|
|
@@ -7487,7 +7515,7 @@
|
|
|
7487
7515
|
"immutable": true,
|
|
7488
7516
|
"locationInModule": {
|
|
7489
7517
|
"filename": "lib/job.ts",
|
|
7490
|
-
"line":
|
|
7518
|
+
"line": 582
|
|
7491
7519
|
},
|
|
7492
7520
|
"name": "sparkUI",
|
|
7493
7521
|
"optional": true,
|
|
@@ -7505,7 +7533,7 @@
|
|
|
7505
7533
|
"immutable": true,
|
|
7506
7534
|
"locationInModule": {
|
|
7507
7535
|
"filename": "lib/job.ts",
|
|
7508
|
-
"line":
|
|
7536
|
+
"line": 552
|
|
7509
7537
|
},
|
|
7510
7538
|
"name": "tags",
|
|
7511
7539
|
"optional": true,
|
|
@@ -7528,7 +7556,7 @@
|
|
|
7528
7556
|
"immutable": true,
|
|
7529
7557
|
"locationInModule": {
|
|
7530
7558
|
"filename": "lib/job.ts",
|
|
7531
|
-
"line":
|
|
7559
|
+
"line": 507
|
|
7532
7560
|
},
|
|
7533
7561
|
"name": "timeout",
|
|
7534
7562
|
"optional": true,
|
|
@@ -7546,7 +7574,7 @@
|
|
|
7546
7574
|
"immutable": true,
|
|
7547
7575
|
"locationInModule": {
|
|
7548
7576
|
"filename": "lib/job.ts",
|
|
7549
|
-
"line":
|
|
7577
|
+
"line": 521
|
|
7550
7578
|
},
|
|
7551
7579
|
"name": "workerCount",
|
|
7552
7580
|
"optional": true,
|
|
@@ -7564,7 +7592,7 @@
|
|
|
7564
7592
|
"immutable": true,
|
|
7565
7593
|
"locationInModule": {
|
|
7566
7594
|
"filename": "lib/job.ts",
|
|
7567
|
-
"line":
|
|
7595
|
+
"line": 514
|
|
7568
7596
|
},
|
|
7569
7597
|
"name": "workerType",
|
|
7570
7598
|
"optional": true,
|
|
@@ -7586,7 +7614,7 @@
|
|
|
7586
7614
|
"kind": "enum",
|
|
7587
7615
|
"locationInModule": {
|
|
7588
7616
|
"filename": "lib/job.ts",
|
|
7589
|
-
"line":
|
|
7617
|
+
"line": 68
|
|
7590
7618
|
},
|
|
7591
7619
|
"members": [
|
|
7592
7620
|
{
|
|
@@ -7667,7 +7695,7 @@
|
|
|
7667
7695
|
},
|
|
7668
7696
|
"locationInModule": {
|
|
7669
7697
|
"filename": "lib/job-executable.ts",
|
|
7670
|
-
"line":
|
|
7698
|
+
"line": 121
|
|
7671
7699
|
},
|
|
7672
7700
|
"name": "of",
|
|
7673
7701
|
"parameters": [
|
|
@@ -7695,7 +7723,7 @@
|
|
|
7695
7723
|
"const": true,
|
|
7696
7724
|
"docs": {
|
|
7697
7725
|
"stability": "experimental",
|
|
7698
|
-
"summary": "Command for running a Glue
|
|
7726
|
+
"summary": "Command for running a Glue Spark job."
|
|
7699
7727
|
},
|
|
7700
7728
|
"immutable": true,
|
|
7701
7729
|
"locationInModule": {
|
|
@@ -7729,7 +7757,24 @@
|
|
|
7729
7757
|
"const": true,
|
|
7730
7758
|
"docs": {
|
|
7731
7759
|
"stability": "experimental",
|
|
7732
|
-
"summary": "Command for running a Glue
|
|
7760
|
+
"summary": "Command for running a Glue Ray job."
|
|
7761
|
+
},
|
|
7762
|
+
"immutable": true,
|
|
7763
|
+
"locationInModule": {
|
|
7764
|
+
"filename": "lib/job-executable.ts",
|
|
7765
|
+
"line": 115
|
|
7766
|
+
},
|
|
7767
|
+
"name": "RAY",
|
|
7768
|
+
"static": true,
|
|
7769
|
+
"type": {
|
|
7770
|
+
"fqn": "@aws-cdk/aws-glue-alpha.JobType"
|
|
7771
|
+
}
|
|
7772
|
+
},
|
|
7773
|
+
{
|
|
7774
|
+
"const": true,
|
|
7775
|
+
"docs": {
|
|
7776
|
+
"stability": "experimental",
|
|
7777
|
+
"summary": "Command for running a Glue Spark streaming job."
|
|
7733
7778
|
},
|
|
7734
7779
|
"immutable": true,
|
|
7735
7780
|
"locationInModule": {
|
|
@@ -7750,7 +7795,7 @@
|
|
|
7750
7795
|
"immutable": true,
|
|
7751
7796
|
"locationInModule": {
|
|
7752
7797
|
"filename": "lib/job-executable.ts",
|
|
7753
|
-
"line":
|
|
7798
|
+
"line": 128
|
|
7754
7799
|
},
|
|
7755
7800
|
"name": "name",
|
|
7756
7801
|
"type": {
|
|
@@ -7771,7 +7816,7 @@
|
|
|
7771
7816
|
"kind": "enum",
|
|
7772
7817
|
"locationInModule": {
|
|
7773
7818
|
"filename": "lib/job.ts",
|
|
7774
|
-
"line":
|
|
7819
|
+
"line": 110
|
|
7775
7820
|
},
|
|
7776
7821
|
"members": [
|
|
7777
7822
|
{
|
|
@@ -7978,6 +8023,127 @@
|
|
|
7978
8023
|
],
|
|
7979
8024
|
"symbolId": "lib/table:PartitionIndex"
|
|
7980
8025
|
},
|
|
8026
|
+
"@aws-cdk/aws-glue-alpha.PythonRayExecutableProps": {
|
|
8027
|
+
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
8028
|
+
"datatype": true,
|
|
8029
|
+
"docs": {
|
|
8030
|
+
"stability": "experimental",
|
|
8031
|
+
"summary": "Props for creating a Python Ray job executable.",
|
|
8032
|
+
"example": "new glue.Job(this, 'RayJob', {\n executable: glue.JobExecutable.pythonRay({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE_NINE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n workerType: glue.WorkerType.Z_2X,\n workerCount: 2,\n description: 'an example Ray job'\n});",
|
|
8033
|
+
"custom": {
|
|
8034
|
+
"exampleMetadata": "infused"
|
|
8035
|
+
}
|
|
8036
|
+
},
|
|
8037
|
+
"fqn": "@aws-cdk/aws-glue-alpha.PythonRayExecutableProps",
|
|
8038
|
+
"kind": "interface",
|
|
8039
|
+
"locationInModule": {
|
|
8040
|
+
"filename": "lib/job-executable.ts",
|
|
8041
|
+
"line": 222
|
|
8042
|
+
},
|
|
8043
|
+
"name": "PythonRayExecutableProps",
|
|
8044
|
+
"properties": [
|
|
8045
|
+
{
|
|
8046
|
+
"abstract": true,
|
|
8047
|
+
"docs": {
|
|
8048
|
+
"see": "https://docs.aws.amazon.com/glue/latest/dg/release-notes.html",
|
|
8049
|
+
"stability": "experimental",
|
|
8050
|
+
"summary": "Glue version."
|
|
8051
|
+
},
|
|
8052
|
+
"immutable": true,
|
|
8053
|
+
"locationInModule": {
|
|
8054
|
+
"filename": "lib/job-executable.ts",
|
|
8055
|
+
"line": 158
|
|
8056
|
+
},
|
|
8057
|
+
"name": "glueVersion",
|
|
8058
|
+
"type": {
|
|
8059
|
+
"fqn": "@aws-cdk/aws-glue-alpha.GlueVersion"
|
|
8060
|
+
}
|
|
8061
|
+
},
|
|
8062
|
+
{
|
|
8063
|
+
"abstract": true,
|
|
8064
|
+
"docs": {
|
|
8065
|
+
"stability": "experimental",
|
|
8066
|
+
"summary": "The Python version to use."
|
|
8067
|
+
},
|
|
8068
|
+
"immutable": true,
|
|
8069
|
+
"locationInModule": {
|
|
8070
|
+
"filename": "lib/job-executable.ts",
|
|
8071
|
+
"line": 139
|
|
8072
|
+
},
|
|
8073
|
+
"name": "pythonVersion",
|
|
8074
|
+
"type": {
|
|
8075
|
+
"fqn": "@aws-cdk/aws-glue-alpha.PythonVersion"
|
|
8076
|
+
}
|
|
8077
|
+
},
|
|
8078
|
+
{
|
|
8079
|
+
"abstract": true,
|
|
8080
|
+
"docs": {
|
|
8081
|
+
"stability": "experimental",
|
|
8082
|
+
"summary": "The script that executes a job."
|
|
8083
|
+
},
|
|
8084
|
+
"immutable": true,
|
|
8085
|
+
"locationInModule": {
|
|
8086
|
+
"filename": "lib/job-executable.ts",
|
|
8087
|
+
"line": 163
|
|
8088
|
+
},
|
|
8089
|
+
"name": "script",
|
|
8090
|
+
"type": {
|
|
8091
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
8092
|
+
}
|
|
8093
|
+
},
|
|
8094
|
+
{
|
|
8095
|
+
"abstract": true,
|
|
8096
|
+
"docs": {
|
|
8097
|
+
"default": "[] - no extra files are copied to the working directory",
|
|
8098
|
+
"remarks": "Only individual files are supported, directories are not supported.",
|
|
8099
|
+
"see": "`--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
8100
|
+
"stability": "experimental",
|
|
8101
|
+
"summary": "Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it."
|
|
8102
|
+
},
|
|
8103
|
+
"immutable": true,
|
|
8104
|
+
"locationInModule": {
|
|
8105
|
+
"filename": "lib/job-executable.ts",
|
|
8106
|
+
"line": 173
|
|
8107
|
+
},
|
|
8108
|
+
"name": "extraFiles",
|
|
8109
|
+
"optional": true,
|
|
8110
|
+
"type": {
|
|
8111
|
+
"collection": {
|
|
8112
|
+
"elementtype": {
|
|
8113
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
8114
|
+
},
|
|
8115
|
+
"kind": "array"
|
|
8116
|
+
}
|
|
8117
|
+
}
|
|
8118
|
+
},
|
|
8119
|
+
{
|
|
8120
|
+
"abstract": true,
|
|
8121
|
+
"docs": {
|
|
8122
|
+
"default": "- no extra python files and argument is not set",
|
|
8123
|
+
"remarks": "Only individual files are supported, directories are not supported.",
|
|
8124
|
+
"see": "`--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html",
|
|
8125
|
+
"stability": "experimental",
|
|
8126
|
+
"summary": "Additional Python files that AWS Glue adds to the Python path before executing your script."
|
|
8127
|
+
},
|
|
8128
|
+
"immutable": true,
|
|
8129
|
+
"locationInModule": {
|
|
8130
|
+
"filename": "lib/job-executable.ts",
|
|
8131
|
+
"line": 149
|
|
8132
|
+
},
|
|
8133
|
+
"name": "extraPythonFiles",
|
|
8134
|
+
"optional": true,
|
|
8135
|
+
"type": {
|
|
8136
|
+
"collection": {
|
|
8137
|
+
"elementtype": {
|
|
8138
|
+
"fqn": "@aws-cdk/aws-glue-alpha.Code"
|
|
8139
|
+
},
|
|
8140
|
+
"kind": "array"
|
|
8141
|
+
}
|
|
8142
|
+
}
|
|
8143
|
+
}
|
|
8144
|
+
],
|
|
8145
|
+
"symbolId": "lib/job-executable:PythonRayExecutableProps"
|
|
8146
|
+
},
|
|
7981
8147
|
"@aws-cdk/aws-glue-alpha.PythonShellExecutableProps": {
|
|
7982
8148
|
"assembly": "@aws-cdk/aws-glue-alpha",
|
|
7983
8149
|
"datatype": true,
|
|
@@ -7993,7 +8159,7 @@
|
|
|
7993
8159
|
"kind": "interface",
|
|
7994
8160
|
"locationInModule": {
|
|
7995
8161
|
"filename": "lib/job-executable.ts",
|
|
7996
|
-
"line":
|
|
8162
|
+
"line": 217
|
|
7997
8163
|
},
|
|
7998
8164
|
"name": "PythonShellExecutableProps",
|
|
7999
8165
|
"properties": [
|
|
@@ -8007,7 +8173,7 @@
|
|
|
8007
8173
|
"immutable": true,
|
|
8008
8174
|
"locationInModule": {
|
|
8009
8175
|
"filename": "lib/job-executable.ts",
|
|
8010
|
-
"line":
|
|
8176
|
+
"line": 158
|
|
8011
8177
|
},
|
|
8012
8178
|
"name": "glueVersion",
|
|
8013
8179
|
"type": {
|
|
@@ -8023,7 +8189,7 @@
|
|
|
8023
8189
|
"immutable": true,
|
|
8024
8190
|
"locationInModule": {
|
|
8025
8191
|
"filename": "lib/job-executable.ts",
|
|
8026
|
-
"line":
|
|
8192
|
+
"line": 139
|
|
8027
8193
|
},
|
|
8028
8194
|
"name": "pythonVersion",
|
|
8029
8195
|
"type": {
|
|
@@ -8039,7 +8205,7 @@
|
|
|
8039
8205
|
"immutable": true,
|
|
8040
8206
|
"locationInModule": {
|
|
8041
8207
|
"filename": "lib/job-executable.ts",
|
|
8042
|
-
"line":
|
|
8208
|
+
"line": 163
|
|
8043
8209
|
},
|
|
8044
8210
|
"name": "script",
|
|
8045
8211
|
"type": {
|
|
@@ -8058,7 +8224,7 @@
|
|
|
8058
8224
|
"immutable": true,
|
|
8059
8225
|
"locationInModule": {
|
|
8060
8226
|
"filename": "lib/job-executable.ts",
|
|
8061
|
-
"line":
|
|
8227
|
+
"line": 173
|
|
8062
8228
|
},
|
|
8063
8229
|
"name": "extraFiles",
|
|
8064
8230
|
"optional": true,
|
|
@@ -8083,7 +8249,7 @@
|
|
|
8083
8249
|
"immutable": true,
|
|
8084
8250
|
"locationInModule": {
|
|
8085
8251
|
"filename": "lib/job-executable.ts",
|
|
8086
|
-
"line":
|
|
8252
|
+
"line": 149
|
|
8087
8253
|
},
|
|
8088
8254
|
"name": "extraPythonFiles",
|
|
8089
8255
|
"optional": true,
|
|
@@ -8114,7 +8280,7 @@
|
|
|
8114
8280
|
"kind": "interface",
|
|
8115
8281
|
"locationInModule": {
|
|
8116
8282
|
"filename": "lib/job-executable.ts",
|
|
8117
|
-
"line":
|
|
8283
|
+
"line": 212
|
|
8118
8284
|
},
|
|
8119
8285
|
"name": "PythonSparkJobExecutableProps",
|
|
8120
8286
|
"properties": [
|
|
@@ -8128,7 +8294,7 @@
|
|
|
8128
8294
|
"immutable": true,
|
|
8129
8295
|
"locationInModule": {
|
|
8130
8296
|
"filename": "lib/job-executable.ts",
|
|
8131
|
-
"line":
|
|
8297
|
+
"line": 158
|
|
8132
8298
|
},
|
|
8133
8299
|
"name": "glueVersion",
|
|
8134
8300
|
"type": {
|
|
@@ -8144,7 +8310,7 @@
|
|
|
8144
8310
|
"immutable": true,
|
|
8145
8311
|
"locationInModule": {
|
|
8146
8312
|
"filename": "lib/job-executable.ts",
|
|
8147
|
-
"line":
|
|
8313
|
+
"line": 139
|
|
8148
8314
|
},
|
|
8149
8315
|
"name": "pythonVersion",
|
|
8150
8316
|
"type": {
|
|
@@ -8160,7 +8326,7 @@
|
|
|
8160
8326
|
"immutable": true,
|
|
8161
8327
|
"locationInModule": {
|
|
8162
8328
|
"filename": "lib/job-executable.ts",
|
|
8163
|
-
"line":
|
|
8329
|
+
"line": 163
|
|
8164
8330
|
},
|
|
8165
8331
|
"name": "script",
|
|
8166
8332
|
"type": {
|
|
@@ -8179,7 +8345,7 @@
|
|
|
8179
8345
|
"immutable": true,
|
|
8180
8346
|
"locationInModule": {
|
|
8181
8347
|
"filename": "lib/job-executable.ts",
|
|
8182
|
-
"line":
|
|
8348
|
+
"line": 173
|
|
8183
8349
|
},
|
|
8184
8350
|
"name": "extraFiles",
|
|
8185
8351
|
"optional": true,
|
|
@@ -8203,7 +8369,7 @@
|
|
|
8203
8369
|
"immutable": true,
|
|
8204
8370
|
"locationInModule": {
|
|
8205
8371
|
"filename": "lib/job-executable.ts",
|
|
8206
|
-
"line":
|
|
8372
|
+
"line": 185
|
|
8207
8373
|
},
|
|
8208
8374
|
"name": "extraJars",
|
|
8209
8375
|
"optional": true,
|
|
@@ -8227,7 +8393,7 @@
|
|
|
8227
8393
|
"immutable": true,
|
|
8228
8394
|
"locationInModule": {
|
|
8229
8395
|
"filename": "lib/job-executable.ts",
|
|
8230
|
-
"line":
|
|
8396
|
+
"line": 194
|
|
8231
8397
|
},
|
|
8232
8398
|
"name": "extraJarsFirst",
|
|
8233
8399
|
"optional": true,
|
|
@@ -8247,7 +8413,7 @@
|
|
|
8247
8413
|
"immutable": true,
|
|
8248
8414
|
"locationInModule": {
|
|
8249
8415
|
"filename": "lib/job-executable.ts",
|
|
8250
|
-
"line":
|
|
8416
|
+
"line": 149
|
|
8251
8417
|
},
|
|
8252
8418
|
"name": "extraPythonFiles",
|
|
8253
8419
|
"optional": true,
|
|
@@ -8490,7 +8656,7 @@
|
|
|
8490
8656
|
"kind": "interface",
|
|
8491
8657
|
"locationInModule": {
|
|
8492
8658
|
"filename": "lib/job-executable.ts",
|
|
8493
|
-
"line":
|
|
8659
|
+
"line": 200
|
|
8494
8660
|
},
|
|
8495
8661
|
"name": "ScalaJobExecutableProps",
|
|
8496
8662
|
"properties": [
|
|
@@ -8504,7 +8670,7 @@
|
|
|
8504
8670
|
"immutable": true,
|
|
8505
8671
|
"locationInModule": {
|
|
8506
8672
|
"filename": "lib/job-executable.ts",
|
|
8507
|
-
"line":
|
|
8673
|
+
"line": 206
|
|
8508
8674
|
},
|
|
8509
8675
|
"name": "className",
|
|
8510
8676
|
"type": {
|
|
@@ -8521,7 +8687,7 @@
|
|
|
8521
8687
|
"immutable": true,
|
|
8522
8688
|
"locationInModule": {
|
|
8523
8689
|
"filename": "lib/job-executable.ts",
|
|
8524
|
-
"line":
|
|
8690
|
+
"line": 158
|
|
8525
8691
|
},
|
|
8526
8692
|
"name": "glueVersion",
|
|
8527
8693
|
"type": {
|
|
@@ -8537,7 +8703,7 @@
|
|
|
8537
8703
|
"immutable": true,
|
|
8538
8704
|
"locationInModule": {
|
|
8539
8705
|
"filename": "lib/job-executable.ts",
|
|
8540
|
-
"line":
|
|
8706
|
+
"line": 163
|
|
8541
8707
|
},
|
|
8542
8708
|
"name": "script",
|
|
8543
8709
|
"type": {
|
|
@@ -8556,7 +8722,7 @@
|
|
|
8556
8722
|
"immutable": true,
|
|
8557
8723
|
"locationInModule": {
|
|
8558
8724
|
"filename": "lib/job-executable.ts",
|
|
8559
|
-
"line":
|
|
8725
|
+
"line": 173
|
|
8560
8726
|
},
|
|
8561
8727
|
"name": "extraFiles",
|
|
8562
8728
|
"optional": true,
|
|
@@ -8580,7 +8746,7 @@
|
|
|
8580
8746
|
"immutable": true,
|
|
8581
8747
|
"locationInModule": {
|
|
8582
8748
|
"filename": "lib/job-executable.ts",
|
|
8583
|
-
"line":
|
|
8749
|
+
"line": 185
|
|
8584
8750
|
},
|
|
8585
8751
|
"name": "extraJars",
|
|
8586
8752
|
"optional": true,
|
|
@@ -8604,7 +8770,7 @@
|
|
|
8604
8770
|
"immutable": true,
|
|
8605
8771
|
"locationInModule": {
|
|
8606
8772
|
"filename": "lib/job-executable.ts",
|
|
8607
|
-
"line":
|
|
8773
|
+
"line": 194
|
|
8608
8774
|
},
|
|
8609
8775
|
"name": "extraJarsFirst",
|
|
8610
8776
|
"optional": true,
|
|
@@ -9523,7 +9689,7 @@
|
|
|
9523
9689
|
"kind": "interface",
|
|
9524
9690
|
"locationInModule": {
|
|
9525
9691
|
"filename": "lib/job.ts",
|
|
9526
|
-
"line":
|
|
9692
|
+
"line": 374
|
|
9527
9693
|
},
|
|
9528
9694
|
"name": "SparkUILoggingLocation",
|
|
9529
9695
|
"properties": [
|
|
@@ -9536,7 +9702,7 @@
|
|
|
9536
9702
|
"immutable": true,
|
|
9537
9703
|
"locationInModule": {
|
|
9538
9704
|
"filename": "lib/job.ts",
|
|
9539
|
-
"line":
|
|
9705
|
+
"line": 378
|
|
9540
9706
|
},
|
|
9541
9707
|
"name": "bucket",
|
|
9542
9708
|
"type": {
|
|
@@ -9553,7 +9719,7 @@
|
|
|
9553
9719
|
"immutable": true,
|
|
9554
9720
|
"locationInModule": {
|
|
9555
9721
|
"filename": "lib/job.ts",
|
|
9556
|
-
"line":
|
|
9722
|
+
"line": 385
|
|
9557
9723
|
},
|
|
9558
9724
|
"name": "prefix",
|
|
9559
9725
|
"optional": true,
|
|
@@ -9580,7 +9746,7 @@
|
|
|
9580
9746
|
"kind": "interface",
|
|
9581
9747
|
"locationInModule": {
|
|
9582
9748
|
"filename": "lib/job.ts",
|
|
9583
|
-
"line":
|
|
9749
|
+
"line": 347
|
|
9584
9750
|
},
|
|
9585
9751
|
"name": "SparkUIProps",
|
|
9586
9752
|
"properties": [
|
|
@@ -9593,7 +9759,7 @@
|
|
|
9593
9759
|
"immutable": true,
|
|
9594
9760
|
"locationInModule": {
|
|
9595
9761
|
"filename": "lib/job.ts",
|
|
9596
|
-
"line":
|
|
9762
|
+
"line": 351
|
|
9597
9763
|
},
|
|
9598
9764
|
"name": "enabled",
|
|
9599
9765
|
"type": {
|
|
@@ -9610,7 +9776,7 @@
|
|
|
9610
9776
|
"immutable": true,
|
|
9611
9777
|
"locationInModule": {
|
|
9612
9778
|
"filename": "lib/job.ts",
|
|
9613
|
-
"line":
|
|
9779
|
+
"line": 358
|
|
9614
9780
|
},
|
|
9615
9781
|
"name": "bucket",
|
|
9616
9782
|
"optional": true,
|
|
@@ -9628,7 +9794,7 @@
|
|
|
9628
9794
|
"immutable": true,
|
|
9629
9795
|
"locationInModule": {
|
|
9630
9796
|
"filename": "lib/job.ts",
|
|
9631
|
-
"line":
|
|
9797
|
+
"line": 365
|
|
9632
9798
|
},
|
|
9633
9799
|
"name": "prefix",
|
|
9634
9800
|
"optional": true,
|
|
@@ -10607,9 +10773,9 @@
|
|
|
10607
10773
|
"remarks": "If you need to use a WorkerType that doesn't exist as a static member, you\ncan instantiate a `WorkerType` object, e.g: `WorkerType.of('other type')`.",
|
|
10608
10774
|
"stability": "experimental",
|
|
10609
10775
|
"summary": "The type of predefined worker that is allocated when a job runs.",
|
|
10610
|
-
"example": "
|
|
10776
|
+
"example": "new glue.Job(this, 'RayJob', {\n executable: glue.JobExecutable.pythonRay({\n glueVersion: glue.GlueVersion.V4_0,\n pythonVersion: glue.PythonVersion.THREE_NINE,\n script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')),\n }),\n workerType: glue.WorkerType.Z_2X,\n workerCount: 2,\n description: 'an example Ray job'\n});",
|
|
10611
10777
|
"custom": {
|
|
10612
|
-
"exampleMetadata": "
|
|
10778
|
+
"exampleMetadata": "infused"
|
|
10613
10779
|
}
|
|
10614
10780
|
},
|
|
10615
10781
|
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType",
|
|
@@ -10626,7 +10792,7 @@
|
|
|
10626
10792
|
},
|
|
10627
10793
|
"locationInModule": {
|
|
10628
10794
|
"filename": "lib/job.ts",
|
|
10629
|
-
"line":
|
|
10795
|
+
"line": 49
|
|
10630
10796
|
},
|
|
10631
10797
|
"name": "of",
|
|
10632
10798
|
"parameters": [
|
|
@@ -10650,6 +10816,23 @@
|
|
|
10650
10816
|
],
|
|
10651
10817
|
"name": "WorkerType",
|
|
10652
10818
|
"properties": [
|
|
10819
|
+
{
|
|
10820
|
+
"const": true,
|
|
10821
|
+
"docs": {
|
|
10822
|
+
"stability": "experimental",
|
|
10823
|
+
"summary": "Each worker maps to 0.25 DPU (2 vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per worker. Suitable for low volume streaming jobs."
|
|
10824
|
+
},
|
|
10825
|
+
"immutable": true,
|
|
10826
|
+
"locationInModule": {
|
|
10827
|
+
"filename": "lib/job.ts",
|
|
10828
|
+
"line": 38
|
|
10829
|
+
},
|
|
10830
|
+
"name": "G_025X",
|
|
10831
|
+
"static": true,
|
|
10832
|
+
"type": {
|
|
10833
|
+
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType"
|
|
10834
|
+
}
|
|
10835
|
+
},
|
|
10653
10836
|
{
|
|
10654
10837
|
"const": true,
|
|
10655
10838
|
"docs": {
|
|
@@ -10703,6 +10886,24 @@
|
|
|
10703
10886
|
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType"
|
|
10704
10887
|
}
|
|
10705
10888
|
},
|
|
10889
|
+
{
|
|
10890
|
+
"const": true,
|
|
10891
|
+
"docs": {
|
|
10892
|
+
"remarks": "Supported in Ray jobs.",
|
|
10893
|
+
"stability": "experimental",
|
|
10894
|
+
"summary": "Each worker maps to 2 high-memory DPU [M-DPU] (8 vCPU, 64 GB of memory, 128 GB disk)."
|
|
10895
|
+
},
|
|
10896
|
+
"immutable": true,
|
|
10897
|
+
"locationInModule": {
|
|
10898
|
+
"filename": "lib/job.ts",
|
|
10899
|
+
"line": 43
|
|
10900
|
+
},
|
|
10901
|
+
"name": "Z_2X",
|
|
10902
|
+
"static": true,
|
|
10903
|
+
"type": {
|
|
10904
|
+
"fqn": "@aws-cdk/aws-glue-alpha.WorkerType"
|
|
10905
|
+
}
|
|
10906
|
+
},
|
|
10706
10907
|
{
|
|
10707
10908
|
"docs": {
|
|
10708
10909
|
"stability": "experimental",
|
|
@@ -10711,7 +10912,7 @@
|
|
|
10711
10912
|
"immutable": true,
|
|
10712
10913
|
"locationInModule": {
|
|
10713
10914
|
"filename": "lib/job.ts",
|
|
10714
|
-
"line":
|
|
10915
|
+
"line": 56
|
|
10715
10916
|
},
|
|
10716
10917
|
"name": "name",
|
|
10717
10918
|
"type": {
|
|
@@ -10722,6 +10923,6 @@
|
|
|
10722
10923
|
"symbolId": "lib/job:WorkerType"
|
|
10723
10924
|
}
|
|
10724
10925
|
},
|
|
10725
|
-
"version": "2.
|
|
10926
|
+
"version": "2.65.0-alpha.0",
|
|
10726
10927
|
"fingerprint": "**********"
|
|
10727
10928
|
}
|