aws-sdk-glue 1.149.0 → 1.150.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-glue/client.rb +158 -59
- data/lib/aws-sdk-glue/client_api.rb +11 -0
- data/lib/aws-sdk-glue/types.rb +285 -135
- data/lib/aws-sdk-glue.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bafae6a5db6eec5869b8fe9e36a225e5a5cde8d9bb2ba4671ef836e6c1cb6cbc
|
4
|
+
data.tar.gz: f2f541472d9510817d5e30ec63ba46dbf52d8ada085448d82b94bbc5731d7576
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 61347736e9ff2a97e6bba1a7c18e6c570cd78653b63fe48f7a335233ae0250c3668369743762132baee695bceffab8cc621b9d368f8d150ef4e1d08d0abe7de9
|
7
|
+
data.tar.gz: 1ad639c12a24b788e71574d2f13a525cfa4550d97403dbf14a1210ca76825916ea1217a56812f6df5ef39201bd660b75ba910b9f073d94fb0daa1b0eda157910
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
Unreleased Changes
|
2
2
|
------------------
|
3
3
|
|
4
|
+
1.150.0 (2023-07-21)
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Feature - This release adds support for AWS Glue Crawler with Apache Hudi Tables, allowing Crawlers to discover Hudi Tables in S3 and register them in Glue Data Catalog for query engines to query against.
|
8
|
+
|
4
9
|
1.149.0 (2023-07-17)
|
5
10
|
------------------
|
6
11
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.150.0
|
data/lib/aws-sdk-glue/client.rb
CHANGED
@@ -826,6 +826,13 @@ module Aws::Glue
|
|
826
826
|
# resp.crawlers[0].targets.iceberg_targets[0].exclusions #=> Array
|
827
827
|
# resp.crawlers[0].targets.iceberg_targets[0].exclusions[0] #=> String
|
828
828
|
# resp.crawlers[0].targets.iceberg_targets[0].maximum_traversal_depth #=> Integer
|
829
|
+
# resp.crawlers[0].targets.hudi_targets #=> Array
|
830
|
+
# resp.crawlers[0].targets.hudi_targets[0].paths #=> Array
|
831
|
+
# resp.crawlers[0].targets.hudi_targets[0].paths[0] #=> String
|
832
|
+
# resp.crawlers[0].targets.hudi_targets[0].connection_name #=> String
|
833
|
+
# resp.crawlers[0].targets.hudi_targets[0].exclusions #=> Array
|
834
|
+
# resp.crawlers[0].targets.hudi_targets[0].exclusions[0] #=> String
|
835
|
+
# resp.crawlers[0].targets.hudi_targets[0].maximum_traversal_depth #=> Integer
|
829
836
|
# resp.crawlers[0].database_name #=> String
|
830
837
|
# resp.crawlers[0].description #=> String
|
831
838
|
# resp.crawlers[0].classifiers #=> Array
|
@@ -2795,6 +2802,14 @@ module Aws::Glue
|
|
2795
2802
|
# maximum_traversal_depth: 1,
|
2796
2803
|
# },
|
2797
2804
|
# ],
|
2805
|
+
# hudi_targets: [
|
2806
|
+
# {
|
2807
|
+
# paths: ["Path"],
|
2808
|
+
# connection_name: "ConnectionName",
|
2809
|
+
# exclusions: ["Path"],
|
2810
|
+
# maximum_traversal_depth: 1,
|
2811
|
+
# },
|
2812
|
+
# ],
|
2798
2813
|
# },
|
2799
2814
|
# schedule: "CronExpression",
|
2800
2815
|
# classifiers: ["NameString"],
|
@@ -3354,28 +3369,49 @@ module Aws::Glue
|
|
3354
3369
|
#
|
3355
3370
|
# @option params [String] :worker_type
|
3356
3371
|
# The type of predefined worker that is allocated when a job runs.
|
3357
|
-
# Accepts a value of
|
3372
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
3358
3373
|
# Accepts the value Z.2X for Ray jobs.
|
3359
3374
|
#
|
3360
|
-
# * For the `
|
3361
|
-
# of memory
|
3362
|
-
#
|
3363
|
-
#
|
3364
|
-
#
|
3365
|
-
#
|
3366
|
-
#
|
3367
|
-
#
|
3368
|
-
#
|
3369
|
-
#
|
3370
|
-
#
|
3371
|
-
#
|
3372
|
-
#
|
3373
|
-
#
|
3374
|
-
#
|
3375
|
-
#
|
3376
|
-
#
|
3377
|
-
#
|
3378
|
-
#
|
3375
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
3376
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and provides
|
3377
|
+
# 1 executor per worker. We recommend this worker type for workloads
|
3378
|
+
# such as data transforms, joins, and queries, to offers a scalable
|
3379
|
+
# and cost effective way to run most jobs.
|
3380
|
+
#
|
3381
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
3382
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
3383
|
+
# provides 1 executor per worker. We recommend this worker type for
|
3384
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
3385
|
+
# scalable and cost effective way to run most jobs.
|
3386
|
+
#
|
3387
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs, 64
|
3388
|
+
# GB of memory) with 256GB disk (approximately 235GB free), and
|
3389
|
+
# provides 1 executor per worker. We recommend this worker type for
|
3390
|
+
# jobs whose workloads contain your most demanding transforms,
|
3391
|
+
# aggregations, joins, and queries. This worker type is available only
|
3392
|
+
# for Glue version 3.0 or later Spark ETL jobs in the following Amazon
|
3393
|
+
# Web Services Regions: US East (Ohio), US East (N. Virginia), US West
|
3394
|
+
# (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
|
3395
|
+
# Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
|
3396
|
+
# (Ireland), and Europe (Stockholm).
|
3397
|
+
#
|
3398
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs, 128
|
3399
|
+
# GB of memory) with 512GB disk (approximately 487GB free), and
|
3400
|
+
# provides 1 executor per worker. We recommend this worker type for
|
3401
|
+
# jobs whose workloads contain your most demanding transforms,
|
3402
|
+
# aggregations, joins, and queries. This worker type is available only
|
3403
|
+
# for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web
|
3404
|
+
# Services Regions as supported for the `G.4X` worker type.
|
3405
|
+
#
|
3406
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPUs,
|
3407
|
+
# 4 GB of memory) with 84GB disk (approximately 34GB free), and
|
3408
|
+
# provides 1 executor per worker. We recommend this worker type for
|
3409
|
+
# low volume streaming jobs. This worker type is only available for
|
3410
|
+
# Glue version 3.0 streaming jobs.
|
3411
|
+
#
|
3412
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs, 64
|
3413
|
+
# GB of memory) with 128 GB disk (approximately 120GB free), and
|
3414
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
3379
3415
|
#
|
3380
3416
|
# @option params [Hash<String,Types::CodeGenConfigurationNode>] :code_gen_configuration_nodes
|
3381
3417
|
# The representation of a directed acyclic graph on which both the Glue
|
@@ -4114,24 +4150,44 @@ module Aws::Glue
|
|
4114
4150
|
# session.
|
4115
4151
|
#
|
4116
4152
|
# @option params [String] :worker_type
|
4117
|
-
# The type of predefined worker that is allocated
|
4118
|
-
#
|
4119
|
-
#
|
4120
|
-
#
|
4121
|
-
#
|
4122
|
-
#
|
4123
|
-
#
|
4124
|
-
#
|
4125
|
-
#
|
4126
|
-
#
|
4127
|
-
# * For the `G.2X` worker type, each worker maps to 2 DPU (8
|
4128
|
-
# of memory
|
4129
|
-
# recommend this worker type for
|
4130
|
-
#
|
4131
|
-
#
|
4132
|
-
#
|
4133
|
-
#
|
4134
|
-
#
|
4153
|
+
# The type of predefined worker that is allocated when a job runs.
|
4154
|
+
# Accepts a value of G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts
|
4155
|
+
# the value Z.2X for Ray notebooks.
|
4156
|
+
#
|
4157
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
4158
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and provides
|
4159
|
+
# 1 executor per worker. We recommend this worker type for workloads
|
4160
|
+
# such as data transforms, joins, and queries, to offers a scalable
|
4161
|
+
# and cost effective way to run most jobs.
|
4162
|
+
#
|
4163
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
4164
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
4165
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4166
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
4167
|
+
# scalable and cost effective way to run most jobs.
|
4168
|
+
#
|
4169
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs, 64
|
4170
|
+
# GB of memory) with 256GB disk (approximately 235GB free), and
|
4171
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4172
|
+
# jobs whose workloads contain your most demanding transforms,
|
4173
|
+
# aggregations, joins, and queries. This worker type is available only
|
4174
|
+
# for Glue version 3.0 or later Spark ETL jobs in the following Amazon
|
4175
|
+
# Web Services Regions: US East (Ohio), US East (N. Virginia), US West
|
4176
|
+
# (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
|
4177
|
+
# Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
|
4178
|
+
# (Ireland), and Europe (Stockholm).
|
4179
|
+
#
|
4180
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs, 128
|
4181
|
+
# GB of memory) with 512GB disk (approximately 487GB free), and
|
4182
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4183
|
+
# jobs whose workloads contain your most demanding transforms,
|
4184
|
+
# aggregations, joins, and queries. This worker type is available only
|
4185
|
+
# for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web
|
4186
|
+
# Services Regions as supported for the `G.4X` worker type.
|
4187
|
+
#
|
4188
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs, 64
|
4189
|
+
# GB of memory) with 128 GB disk (approximately 120GB free), and
|
4190
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
4135
4191
|
#
|
4136
4192
|
# @option params [String] :security_configuration
|
4137
4193
|
# The name of the SecurityConfiguration structure to be used with the
|
@@ -6042,6 +6098,13 @@ module Aws::Glue
|
|
6042
6098
|
# resp.crawler.targets.iceberg_targets[0].exclusions #=> Array
|
6043
6099
|
# resp.crawler.targets.iceberg_targets[0].exclusions[0] #=> String
|
6044
6100
|
# resp.crawler.targets.iceberg_targets[0].maximum_traversal_depth #=> Integer
|
6101
|
+
# resp.crawler.targets.hudi_targets #=> Array
|
6102
|
+
# resp.crawler.targets.hudi_targets[0].paths #=> Array
|
6103
|
+
# resp.crawler.targets.hudi_targets[0].paths[0] #=> String
|
6104
|
+
# resp.crawler.targets.hudi_targets[0].connection_name #=> String
|
6105
|
+
# resp.crawler.targets.hudi_targets[0].exclusions #=> Array
|
6106
|
+
# resp.crawler.targets.hudi_targets[0].exclusions[0] #=> String
|
6107
|
+
# resp.crawler.targets.hudi_targets[0].maximum_traversal_depth #=> Integer
|
6045
6108
|
# resp.crawler.database_name #=> String
|
6046
6109
|
# resp.crawler.description #=> String
|
6047
6110
|
# resp.crawler.classifiers #=> Array
|
@@ -6196,6 +6259,13 @@ module Aws::Glue
|
|
6196
6259
|
# resp.crawlers[0].targets.iceberg_targets[0].exclusions #=> Array
|
6197
6260
|
# resp.crawlers[0].targets.iceberg_targets[0].exclusions[0] #=> String
|
6198
6261
|
# resp.crawlers[0].targets.iceberg_targets[0].maximum_traversal_depth #=> Integer
|
6262
|
+
# resp.crawlers[0].targets.hudi_targets #=> Array
|
6263
|
+
# resp.crawlers[0].targets.hudi_targets[0].paths #=> Array
|
6264
|
+
# resp.crawlers[0].targets.hudi_targets[0].paths[0] #=> String
|
6265
|
+
# resp.crawlers[0].targets.hudi_targets[0].connection_name #=> String
|
6266
|
+
# resp.crawlers[0].targets.hudi_targets[0].exclusions #=> Array
|
6267
|
+
# resp.crawlers[0].targets.hudi_targets[0].exclusions[0] #=> String
|
6268
|
+
# resp.crawlers[0].targets.hudi_targets[0].maximum_traversal_depth #=> Integer
|
6199
6269
|
# resp.crawlers[0].database_name #=> String
|
6200
6270
|
# resp.crawlers[0].description #=> String
|
6201
6271
|
# resp.crawlers[0].classifiers #=> Array
|
@@ -13931,28 +14001,49 @@ module Aws::Glue
|
|
13931
14001
|
#
|
13932
14002
|
# @option params [String] :worker_type
|
13933
14003
|
# The type of predefined worker that is allocated when a job runs.
|
13934
|
-
# Accepts a value of
|
14004
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
13935
14005
|
# Accepts the value Z.2X for Ray jobs.
|
13936
14006
|
#
|
13937
|
-
# * For the `
|
13938
|
-
# of memory
|
13939
|
-
#
|
13940
|
-
#
|
13941
|
-
#
|
13942
|
-
#
|
13943
|
-
#
|
13944
|
-
#
|
13945
|
-
#
|
13946
|
-
#
|
13947
|
-
#
|
13948
|
-
#
|
13949
|
-
#
|
13950
|
-
#
|
13951
|
-
#
|
13952
|
-
#
|
13953
|
-
#
|
13954
|
-
#
|
13955
|
-
#
|
14007
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
14008
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and provides
|
14009
|
+
# 1 executor per worker. We recommend this worker type for workloads
|
14010
|
+
# such as data transforms, joins, and queries, to offers a scalable
|
14011
|
+
# and cost effective way to run most jobs.
|
14012
|
+
#
|
14013
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
14014
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
14015
|
+
# provides 1 executor per worker. We recommend this worker type for
|
14016
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
14017
|
+
# scalable and cost effective way to run most jobs.
|
14018
|
+
#
|
14019
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs, 64
|
14020
|
+
# GB of memory) with 256GB disk (approximately 235GB free), and
|
14021
|
+
# provides 1 executor per worker. We recommend this worker type for
|
14022
|
+
# jobs whose workloads contain your most demanding transforms,
|
14023
|
+
# aggregations, joins, and queries. This worker type is available only
|
14024
|
+
# for Glue version 3.0 or later Spark ETL jobs in the following Amazon
|
14025
|
+
# Web Services Regions: US East (Ohio), US East (N. Virginia), US West
|
14026
|
+
# (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
|
14027
|
+
# Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
|
14028
|
+
# (Ireland), and Europe (Stockholm).
|
14029
|
+
#
|
14030
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs, 128
|
14031
|
+
# GB of memory) with 512GB disk (approximately 487GB free), and
|
14032
|
+
# provides 1 executor per worker. We recommend this worker type for
|
14033
|
+
# jobs whose workloads contain your most demanding transforms,
|
14034
|
+
# aggregations, joins, and queries. This worker type is available only
|
14035
|
+
# for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web
|
14036
|
+
# Services Regions as supported for the `G.4X` worker type.
|
14037
|
+
#
|
14038
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPUs,
|
14039
|
+
# 4 GB of memory) with 84GB disk (approximately 34GB free), and
|
14040
|
+
# provides 1 executor per worker. We recommend this worker type for
|
14041
|
+
# low volume streaming jobs. This worker type is only available for
|
14042
|
+
# Glue version 3.0 streaming jobs.
|
14043
|
+
#
|
14044
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs, 64
|
14045
|
+
# GB of memory) with 128 GB disk (approximately 120GB free), and
|
14046
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
13956
14047
|
#
|
13957
14048
|
# @option params [Integer] :number_of_workers
|
13958
14049
|
# The number of workers of a defined `workerType` that are allocated
|
@@ -14898,6 +14989,14 @@ module Aws::Glue
|
|
14898
14989
|
# maximum_traversal_depth: 1,
|
14899
14990
|
# },
|
14900
14991
|
# ],
|
14992
|
+
# hudi_targets: [
|
14993
|
+
# {
|
14994
|
+
# paths: ["Path"],
|
14995
|
+
# connection_name: "ConnectionName",
|
14996
|
+
# exclusions: ["Path"],
|
14997
|
+
# maximum_traversal_depth: 1,
|
14998
|
+
# },
|
14999
|
+
# ],
|
14901
15000
|
# },
|
14902
15001
|
# schedule: "CronExpression",
|
14903
15002
|
# classifiers: ["NameString"],
|
@@ -15954,7 +16053,7 @@ module Aws::Glue
|
|
15954
16053
|
params: params,
|
15955
16054
|
config: config)
|
15956
16055
|
context[:gem_name] = 'aws-sdk-glue'
|
15957
|
-
context[:gem_version] = '1.
|
16056
|
+
context[:gem_version] = '1.150.0'
|
15958
16057
|
Seahorse::Client::Request.new(handlers, context)
|
15959
16058
|
end
|
15960
16059
|
|
@@ -609,7 +609,9 @@ module Aws::Glue
|
|
609
609
|
GrokClassifier = Shapes::StructureShape.new(name: 'GrokClassifier')
|
610
610
|
GrokPattern = Shapes::StringShape.new(name: 'GrokPattern')
|
611
611
|
HashString = Shapes::StringShape.new(name: 'HashString')
|
612
|
+
HudiTarget = Shapes::StructureShape.new(name: 'HudiTarget')
|
612
613
|
HudiTargetCompressionType = Shapes::StringShape.new(name: 'HudiTargetCompressionType')
|
614
|
+
HudiTargetList = Shapes::ListShape.new(name: 'HudiTargetList')
|
613
615
|
IcebergInput = Shapes::StructureShape.new(name: 'IcebergInput')
|
614
616
|
IcebergTarget = Shapes::StructureShape.new(name: 'IcebergTarget')
|
615
617
|
IcebergTargetList = Shapes::ListShape.new(name: 'IcebergTargetList')
|
@@ -1872,6 +1874,7 @@ module Aws::Glue
|
|
1872
1874
|
CrawlerTargets.add_member(:catalog_targets, Shapes::ShapeRef.new(shape: CatalogTargetList, location_name: "CatalogTargets"))
|
1873
1875
|
CrawlerTargets.add_member(:delta_targets, Shapes::ShapeRef.new(shape: DeltaTargetList, location_name: "DeltaTargets"))
|
1874
1876
|
CrawlerTargets.add_member(:iceberg_targets, Shapes::ShapeRef.new(shape: IcebergTargetList, location_name: "IcebergTargets"))
|
1877
|
+
CrawlerTargets.add_member(:hudi_targets, Shapes::ShapeRef.new(shape: HudiTargetList, location_name: "HudiTargets"))
|
1875
1878
|
CrawlerTargets.struct_class = Types::CrawlerTargets
|
1876
1879
|
|
1877
1880
|
CrawlsFilter.add_member(:field_name, Shapes::ShapeRef.new(shape: FieldName, location_name: "FieldName"))
|
@@ -3536,6 +3539,14 @@ module Aws::Glue
|
|
3536
3539
|
GrokClassifier.add_member(:custom_patterns, Shapes::ShapeRef.new(shape: CustomPatterns, location_name: "CustomPatterns"))
|
3537
3540
|
GrokClassifier.struct_class = Types::GrokClassifier
|
3538
3541
|
|
3542
|
+
HudiTarget.add_member(:paths, Shapes::ShapeRef.new(shape: PathList, location_name: "Paths"))
|
3543
|
+
HudiTarget.add_member(:connection_name, Shapes::ShapeRef.new(shape: ConnectionName, location_name: "ConnectionName"))
|
3544
|
+
HudiTarget.add_member(:exclusions, Shapes::ShapeRef.new(shape: PathList, location_name: "Exclusions"))
|
3545
|
+
HudiTarget.add_member(:maximum_traversal_depth, Shapes::ShapeRef.new(shape: NullableInteger, location_name: "MaximumTraversalDepth"))
|
3546
|
+
HudiTarget.struct_class = Types::HudiTarget
|
3547
|
+
|
3548
|
+
HudiTargetList.member = Shapes::ShapeRef.new(shape: HudiTarget)
|
3549
|
+
|
3539
3550
|
IcebergInput.add_member(:metadata_operation, Shapes::ShapeRef.new(shape: MetadataOperation, required: true, location_name: "MetadataOperation"))
|
3540
3551
|
IcebergInput.add_member(:version, Shapes::ShapeRef.new(shape: VersionString, location_name: "Version"))
|
3541
3552
|
IcebergInput.struct_class = Types::IcebergInput
|
data/lib/aws-sdk-glue/types.rb
CHANGED
@@ -3458,6 +3458,10 @@ module Aws::Glue
|
|
3458
3458
|
# Specifies Apache Iceberg data store targets.
|
3459
3459
|
# @return [Array<Types::IcebergTarget>]
|
3460
3460
|
#
|
3461
|
+
# @!attribute [rw] hudi_targets
|
3462
|
+
# Specifies Apache Hudi data store targets.
|
3463
|
+
# @return [Array<Types::HudiTarget>]
|
3464
|
+
#
|
3461
3465
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CrawlerTargets AWS API Documentation
|
3462
3466
|
#
|
3463
3467
|
class CrawlerTargets < Struct.new(
|
@@ -3467,7 +3471,8 @@ module Aws::Glue
|
|
3467
3471
|
:dynamo_db_targets,
|
3468
3472
|
:catalog_targets,
|
3469
3473
|
:delta_targets,
|
3470
|
-
:iceberg_targets
|
3474
|
+
:iceberg_targets,
|
3475
|
+
:hudi_targets)
|
3471
3476
|
SENSITIVE = []
|
3472
3477
|
include Aws::Structure
|
3473
3478
|
end
|
@@ -4400,29 +4405,50 @@ module Aws::Glue
|
|
4400
4405
|
#
|
4401
4406
|
# @!attribute [rw] worker_type
|
4402
4407
|
# The type of predefined worker that is allocated when a job runs.
|
4403
|
-
# Accepts a value of
|
4408
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
4404
4409
|
# Accepts the value Z.2X for Ray jobs.
|
4405
4410
|
#
|
4406
|
-
# * For the `
|
4407
|
-
# of memory
|
4408
|
-
#
|
4409
|
-
#
|
4410
|
-
#
|
4411
|
-
#
|
4412
|
-
#
|
4413
|
-
#
|
4414
|
-
#
|
4415
|
-
#
|
4411
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
4412
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and
|
4413
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4414
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
4415
|
+
# scalable and cost effective way to run most jobs.
|
4416
|
+
#
|
4417
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
4418
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
4419
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4420
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
4421
|
+
# scalable and cost effective way to run most jobs.
|
4422
|
+
#
|
4423
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
|
4424
|
+
# 64 GB of memory) with 256GB disk (approximately 235GB free), and
|
4425
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4426
|
+
# jobs whose workloads contain your most demanding transforms,
|
4427
|
+
# aggregations, joins, and queries. This worker type is available
|
4428
|
+
# only for Glue version 3.0 or later Spark ETL jobs in the following
|
4429
|
+
# Amazon Web Services Regions: US East (Ohio), US East (N.
|
4430
|
+
# Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
|
4431
|
+
# Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
|
4432
|
+
# (Frankfurt), Europe (Ireland), and Europe (Stockholm).
|
4433
|
+
#
|
4434
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
4435
|
+
# 128 GB of memory) with 512GB disk (approximately 487GB free), and
|
4436
|
+
# provides 1 executor per worker. We recommend this worker type for
|
4437
|
+
# jobs whose workloads contain your most demanding transforms,
|
4438
|
+
# aggregations, joins, and queries. This worker type is available
|
4439
|
+
# only for Glue version 3.0 or later Spark ETL jobs, in the same
|
4440
|
+
# Amazon Web Services Regions as supported for the `G.4X` worker
|
4441
|
+
# type.
|
4416
4442
|
#
|
4417
4443
|
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
4418
|
-
#
|
4419
|
-
# worker. We recommend this worker type
|
4420
|
-
# jobs. This worker type is only available
|
4421
|
-
# streaming jobs.
|
4444
|
+
# vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
|
4445
|
+
# and provides 1 executor per worker. We recommend this worker type
|
4446
|
+
# for low volume streaming jobs. This worker type is only available
|
4447
|
+
# for Glue version 3.0 streaming jobs.
|
4422
4448
|
#
|
4423
|
-
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (
|
4424
|
-
# GB of
|
4425
|
-
# based on the autoscaler.
|
4449
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
4450
|
+
# 64 GB of memory) with 128 GB disk (approximately 120GB free), and
|
4451
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
4426
4452
|
# @return [String]
|
4427
4453
|
#
|
4428
4454
|
# @!attribute [rw] code_gen_configuration_nodes
|
@@ -5117,25 +5143,45 @@ module Aws::Glue
|
|
5117
5143
|
# @return [Integer]
|
5118
5144
|
#
|
5119
5145
|
# @!attribute [rw] worker_type
|
5120
|
-
# The type of predefined worker that is allocated
|
5121
|
-
#
|
5122
|
-
#
|
5123
|
-
#
|
5124
|
-
#
|
5125
|
-
#
|
5126
|
-
#
|
5127
|
-
#
|
5128
|
-
#
|
5129
|
-
#
|
5130
|
-
# * For the `G.2X` worker type, each worker maps to 2 DPU (8
|
5131
|
-
# GB of memory
|
5132
|
-
# recommend this worker type for
|
5133
|
-
#
|
5134
|
-
#
|
5135
|
-
#
|
5136
|
-
#
|
5137
|
-
#
|
5138
|
-
#
|
5146
|
+
# The type of predefined worker that is allocated when a job runs.
|
5147
|
+
# Accepts a value of G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts
|
5148
|
+
# the value Z.2X for Ray notebooks.
|
5149
|
+
#
|
5150
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
5151
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and
|
5152
|
+
# provides 1 executor per worker. We recommend this worker type for
|
5153
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
5154
|
+
# scalable and cost effective way to run most jobs.
|
5155
|
+
#
|
5156
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
5157
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
5158
|
+
# provides 1 executor per worker. We recommend this worker type for
|
5159
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
5160
|
+
# scalable and cost effective way to run most jobs.
|
5161
|
+
#
|
5162
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
|
5163
|
+
# 64 GB of memory) with 256GB disk (approximately 235GB free), and
|
5164
|
+
# provides 1 executor per worker. We recommend this worker type for
|
5165
|
+
# jobs whose workloads contain your most demanding transforms,
|
5166
|
+
# aggregations, joins, and queries. This worker type is available
|
5167
|
+
# only for Glue version 3.0 or later Spark ETL jobs in the following
|
5168
|
+
# Amazon Web Services Regions: US East (Ohio), US East (N.
|
5169
|
+
# Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
|
5170
|
+
# Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
|
5171
|
+
# (Frankfurt), Europe (Ireland), and Europe (Stockholm).
|
5172
|
+
#
|
5173
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
5174
|
+
# 128 GB of memory) with 512GB disk (approximately 487GB free), and
|
5175
|
+
# provides 1 executor per worker. We recommend this worker type for
|
5176
|
+
# jobs whose workloads contain your most demanding transforms,
|
5177
|
+
# aggregations, joins, and queries. This worker type is available
|
5178
|
+
# only for Glue version 3.0 or later Spark ETL jobs, in the same
|
5179
|
+
# Amazon Web Services Regions as supported for the `G.4X` worker
|
5180
|
+
# type.
|
5181
|
+
#
|
5182
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
5183
|
+
# 64 GB of memory) with 128 GB disk (approximately 120GB free), and
|
5184
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
5139
5185
|
# @return [String]
|
5140
5186
|
#
|
5141
5187
|
# @!attribute [rw] security_configuration
|
@@ -11808,6 +11854,49 @@ module Aws::Glue
|
|
11808
11854
|
include Aws::Structure
|
11809
11855
|
end
|
11810
11856
|
|
11857
|
+
# Specifies an Apache Hudi data source.
|
11858
|
+
#
|
11859
|
+
# @!attribute [rw] paths
|
11860
|
+
# An array of Amazon S3 location strings for Hudi, each indicating the
|
11861
|
+
# root folder with which the metadata files for a Hudi table resides.
|
11862
|
+
# The Hudi folder may be located in a child folder of the root folder.
|
11863
|
+
#
|
11864
|
+
# The crawler will scan all folders underneath a path for a Hudi
|
11865
|
+
# folder.
|
11866
|
+
# @return [Array<String>]
|
11867
|
+
#
|
11868
|
+
# @!attribute [rw] connection_name
|
11869
|
+
# The name of the connection to use to connect to the Hudi target. If
|
11870
|
+
# your Hudi files are stored in buckets that require VPC
|
11871
|
+
# authorization, you can set their connection properties here.
|
11872
|
+
# @return [String]
|
11873
|
+
#
|
11874
|
+
# @!attribute [rw] exclusions
|
11875
|
+
# A list of glob patterns used to exclude from the crawl. For more
|
11876
|
+
# information, see [Catalog Tables with a Crawler][1].
|
11877
|
+
#
|
11878
|
+
#
|
11879
|
+
#
|
11880
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-crawler.html
|
11881
|
+
# @return [Array<String>]
|
11882
|
+
#
|
11883
|
+
# @!attribute [rw] maximum_traversal_depth
|
11884
|
+
# The maximum depth of Amazon S3 paths that the crawler can traverse
|
11885
|
+
# to discover the Hudi metadata folder in your Amazon S3 path. Used to
|
11886
|
+
# limit the crawler run time.
|
11887
|
+
# @return [Integer]
|
11888
|
+
#
|
11889
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/HudiTarget AWS API Documentation
|
11890
|
+
#
|
11891
|
+
class HudiTarget < Struct.new(
|
11892
|
+
:paths,
|
11893
|
+
:connection_name,
|
11894
|
+
:exclusions,
|
11895
|
+
:maximum_traversal_depth)
|
11896
|
+
SENSITIVE = []
|
11897
|
+
include Aws::Structure
|
11898
|
+
end
|
11899
|
+
|
11811
11900
|
# A structure that defines an Apache Iceberg metadata table to create in
|
11812
11901
|
# the catalog.
|
11813
11902
|
#
|
@@ -12356,52 +12445,50 @@ module Aws::Glue
|
|
12356
12445
|
#
|
12357
12446
|
# @!attribute [rw] worker_type
|
12358
12447
|
# The type of predefined worker that is allocated when a job runs.
|
12359
|
-
# Accepts a value of
|
12360
|
-
#
|
12361
|
-
#
|
12362
|
-
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
12363
|
-
# of memory and a 50GB disk, and 2 executors per worker.
|
12364
|
-
#
|
12365
|
-
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
|
12366
|
-
# GB of memory, 64 GB disk), and provides 1 executor per worker. We
|
12367
|
-
# recommend this worker type for workloads such as data transforms,
|
12368
|
-
# joins, and queries, to offers a scalable and cost effective way to
|
12369
|
-
# run most jobs.
|
12448
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
12449
|
+
# Accepts the value Z.2X for Ray jobs.
|
12370
12450
|
#
|
12371
|
-
# * For the `G.
|
12372
|
-
# GB of memory
|
12373
|
-
# recommend this worker type for
|
12374
|
-
# joins, and queries, to offers a
|
12375
|
-
# run most jobs.
|
12376
|
-
#
|
12377
|
-
# * For the `G.
|
12378
|
-
# GB of memory
|
12379
|
-
# recommend this worker type for
|
12380
|
-
#
|
12381
|
-
#
|
12382
|
-
#
|
12383
|
-
#
|
12384
|
-
#
|
12385
|
-
#
|
12386
|
-
#
|
12387
|
-
#
|
12388
|
-
#
|
12389
|
-
#
|
12390
|
-
#
|
12391
|
-
#
|
12392
|
-
#
|
12393
|
-
#
|
12394
|
-
#
|
12451
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
12452
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and
|
12453
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12454
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
12455
|
+
# scalable and cost effective way to run most jobs.
|
12456
|
+
#
|
12457
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
12458
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
12459
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12460
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
12461
|
+
# scalable and cost effective way to run most jobs.
|
12462
|
+
#
|
12463
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
|
12464
|
+
# 64 GB of memory) with 256GB disk (approximately 235GB free), and
|
12465
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12466
|
+
# jobs whose workloads contain your most demanding transforms,
|
12467
|
+
# aggregations, joins, and queries. This worker type is available
|
12468
|
+
# only for Glue version 3.0 or later Spark ETL jobs in the following
|
12469
|
+
# Amazon Web Services Regions: US East (Ohio), US East (N.
|
12470
|
+
# Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
|
12471
|
+
# Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
|
12472
|
+
# (Frankfurt), Europe (Ireland), and Europe (Stockholm).
|
12473
|
+
#
|
12474
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
12475
|
+
# 128 GB of memory) with 512GB disk (approximately 487GB free), and
|
12476
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12477
|
+
# jobs whose workloads contain your most demanding transforms,
|
12478
|
+
# aggregations, joins, and queries. This worker type is available
|
12479
|
+
# only for Glue version 3.0 or later Spark ETL jobs, in the same
|
12480
|
+
# Amazon Web Services Regions as supported for the `G.4X` worker
|
12481
|
+
# type.
|
12395
12482
|
#
|
12396
12483
|
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
12397
|
-
#
|
12398
|
-
# worker. We recommend this worker type
|
12399
|
-
# jobs. This worker type is only available
|
12400
|
-
# streaming jobs.
|
12484
|
+
# vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
|
12485
|
+
# and provides 1 executor per worker. We recommend this worker type
|
12486
|
+
# for low volume streaming jobs. This worker type is only available
|
12487
|
+
# for Glue version 3.0 streaming jobs.
|
12401
12488
|
#
|
12402
|
-
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (
|
12403
|
-
# GB of
|
12404
|
-
# workers
|
12489
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
12490
|
+
# 64 GB of memory) with 128 GB disk (approximately 120GB free), and
|
12491
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
12405
12492
|
# @return [String]
|
12406
12493
|
#
|
12407
12494
|
# @!attribute [rw] number_of_workers
|
@@ -12761,29 +12848,50 @@ module Aws::Glue
|
|
12761
12848
|
#
|
12762
12849
|
# @!attribute [rw] worker_type
|
12763
12850
|
# The type of predefined worker that is allocated when a job runs.
|
12764
|
-
# Accepts a value of
|
12851
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
12765
12852
|
# Accepts the value Z.2X for Ray jobs.
|
12766
12853
|
#
|
12767
|
-
# * For the `
|
12768
|
-
# of memory
|
12769
|
-
#
|
12770
|
-
#
|
12771
|
-
#
|
12772
|
-
#
|
12773
|
-
#
|
12774
|
-
#
|
12775
|
-
#
|
12776
|
-
#
|
12854
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
12855
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and
|
12856
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12857
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
12858
|
+
# scalable and cost effective way to run most jobs.
|
12859
|
+
#
|
12860
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
12861
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
12862
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12863
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
12864
|
+
# scalable and cost effective way to run most jobs.
|
12865
|
+
#
|
12866
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
|
12867
|
+
# 64 GB of memory) with 256GB disk (approximately 235GB free), and
|
12868
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12869
|
+
# jobs whose workloads contain your most demanding transforms,
|
12870
|
+
# aggregations, joins, and queries. This worker type is available
|
12871
|
+
# only for Glue version 3.0 or later Spark ETL jobs in the following
|
12872
|
+
# Amazon Web Services Regions: US East (Ohio), US East (N.
|
12873
|
+
# Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
|
12874
|
+
# Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
|
12875
|
+
# (Frankfurt), Europe (Ireland), and Europe (Stockholm).
|
12876
|
+
#
|
12877
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
12878
|
+
# 128 GB of memory) with 512GB disk (approximately 487GB free), and
|
12879
|
+
# provides 1 executor per worker. We recommend this worker type for
|
12880
|
+
# jobs whose workloads contain your most demanding transforms,
|
12881
|
+
# aggregations, joins, and queries. This worker type is available
|
12882
|
+
# only for Glue version 3.0 or later Spark ETL jobs, in the same
|
12883
|
+
# Amazon Web Services Regions as supported for the `G.4X` worker
|
12884
|
+
# type.
|
12777
12885
|
#
|
12778
12886
|
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
12779
|
-
#
|
12780
|
-
# worker. We recommend this worker type
|
12781
|
-
# jobs. This worker type is only available
|
12782
|
-
# streaming jobs.
|
12887
|
+
# vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
|
12888
|
+
# and provides 1 executor per worker. We recommend this worker type
|
12889
|
+
# for low volume streaming jobs. This worker type is only available
|
12890
|
+
# for Glue version 3.0 streaming jobs.
|
12783
12891
|
#
|
12784
|
-
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (
|
12785
|
-
# GB of
|
12786
|
-
#
|
12892
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
12893
|
+
# 64 GB of memory) with 128 GB disk (approximately 120GB free), and
|
12894
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
12787
12895
|
# @return [String]
|
12788
12896
|
#
|
12789
12897
|
# @!attribute [rw] number_of_workers
|
@@ -13013,29 +13121,50 @@ module Aws::Glue
|
|
13013
13121
|
#
|
13014
13122
|
# @!attribute [rw] worker_type
|
13015
13123
|
# The type of predefined worker that is allocated when a job runs.
|
13016
|
-
# Accepts a value of
|
13124
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
13017
13125
|
# Accepts the value Z.2X for Ray jobs.
|
13018
13126
|
#
|
13019
|
-
# * For the `
|
13020
|
-
# of memory
|
13021
|
-
#
|
13022
|
-
#
|
13023
|
-
#
|
13024
|
-
#
|
13025
|
-
#
|
13026
|
-
#
|
13027
|
-
#
|
13028
|
-
#
|
13127
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
13128
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and
|
13129
|
+
# provides 1 executor per worker. We recommend this worker type for
|
13130
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
13131
|
+
# scalable and cost effective way to run most jobs.
|
13132
|
+
#
|
13133
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
13134
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
13135
|
+
# provides 1 executor per worker. We recommend this worker type for
|
13136
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
13137
|
+
# scalable and cost effective way to run most jobs.
|
13138
|
+
#
|
13139
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
|
13140
|
+
# 64 GB of memory) with 256GB disk (approximately 235GB free), and
|
13141
|
+
# provides 1 executor per worker. We recommend this worker type for
|
13142
|
+
# jobs whose workloads contain your most demanding transforms,
|
13143
|
+
# aggregations, joins, and queries. This worker type is available
|
13144
|
+
# only for Glue version 3.0 or later Spark ETL jobs in the following
|
13145
|
+
# Amazon Web Services Regions: US East (Ohio), US East (N.
|
13146
|
+
# Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
|
13147
|
+
# Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
|
13148
|
+
# (Frankfurt), Europe (Ireland), and Europe (Stockholm).
|
13149
|
+
#
|
13150
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
13151
|
+
# 128 GB of memory) with 512GB disk (approximately 487GB free), and
|
13152
|
+
# provides 1 executor per worker. We recommend this worker type for
|
13153
|
+
# jobs whose workloads contain your most demanding transforms,
|
13154
|
+
# aggregations, joins, and queries. This worker type is available
|
13155
|
+
# only for Glue version 3.0 or later Spark ETL jobs, in the same
|
13156
|
+
# Amazon Web Services Regions as supported for the `G.4X` worker
|
13157
|
+
# type.
|
13029
13158
|
#
|
13030
13159
|
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
13031
|
-
#
|
13032
|
-
# worker. We recommend this worker type
|
13033
|
-
# jobs. This worker type is only available
|
13034
|
-
# streaming jobs.
|
13160
|
+
# vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
|
13161
|
+
# and provides 1 executor per worker. We recommend this worker type
|
13162
|
+
# for low volume streaming jobs. This worker type is only available
|
13163
|
+
# for Glue version 3.0 streaming jobs.
|
13035
13164
|
#
|
13036
|
-
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (
|
13037
|
-
# GB of
|
13038
|
-
# based on the autoscaler.
|
13165
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
13166
|
+
# 64 GB of memory) with 128 GB disk (approximately 120GB free), and
|
13167
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
13039
13168
|
# @return [String]
|
13040
13169
|
#
|
13041
13170
|
# @!attribute [rw] number_of_workers
|
@@ -18665,29 +18794,50 @@ module Aws::Glue
|
|
18665
18794
|
#
|
18666
18795
|
# @!attribute [rw] worker_type
|
18667
18796
|
# The type of predefined worker that is allocated when a job runs.
|
18668
|
-
# Accepts a value of
|
18797
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
|
18669
18798
|
# Accepts the value Z.2X for Ray jobs.
|
18670
18799
|
#
|
18671
|
-
# * For the `
|
18672
|
-
# of memory
|
18673
|
-
#
|
18674
|
-
#
|
18675
|
-
#
|
18676
|
-
#
|
18677
|
-
#
|
18678
|
-
#
|
18679
|
-
#
|
18680
|
-
#
|
18800
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
18801
|
+
# GB of memory) with 84GB disk (approximately 34GB free), and
|
18802
|
+
# provides 1 executor per worker. We recommend this worker type for
|
18803
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
18804
|
+
# scalable and cost effective way to run most jobs.
|
18805
|
+
#
|
18806
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
|
18807
|
+
# GB of memory) with 128GB disk (approximately 77GB free), and
|
18808
|
+
# provides 1 executor per worker. We recommend this worker type for
|
18809
|
+
# workloads such as data transforms, joins, and queries, to offers a
|
18810
|
+
# scalable and cost effective way to run most jobs.
|
18811
|
+
#
|
18812
|
+
# * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
|
18813
|
+
# 64 GB of memory) with 256GB disk (approximately 235GB free), and
|
18814
|
+
# provides 1 executor per worker. We recommend this worker type for
|
18815
|
+
# jobs whose workloads contain your most demanding transforms,
|
18816
|
+
# aggregations, joins, and queries. This worker type is available
|
18817
|
+
# only for Glue version 3.0 or later Spark ETL jobs in the following
|
18818
|
+
# Amazon Web Services Regions: US East (Ohio), US East (N.
|
18819
|
+
# Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
|
18820
|
+
# Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
|
18821
|
+
# (Frankfurt), Europe (Ireland), and Europe (Stockholm).
|
18822
|
+
#
|
18823
|
+
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
18824
|
+
# 128 GB of memory) with 512GB disk (approximately 487GB free), and
|
18825
|
+
# provides 1 executor per worker. We recommend this worker type for
|
18826
|
+
# jobs whose workloads contain your most demanding transforms,
|
18827
|
+
# aggregations, joins, and queries. This worker type is available
|
18828
|
+
# only for Glue version 3.0 or later Spark ETL jobs, in the same
|
18829
|
+
# Amazon Web Services Regions as supported for the `G.4X` worker
|
18830
|
+
# type.
|
18681
18831
|
#
|
18682
18832
|
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
18683
|
-
#
|
18684
|
-
# worker. We recommend this worker type
|
18685
|
-
# jobs. This worker type is only available
|
18686
|
-
# streaming jobs.
|
18687
|
-
#
|
18688
|
-
# * For the `Z.2X` worker type, each worker maps to 2 DPU (
|
18689
|
-
# GB of
|
18690
|
-
#
|
18833
|
+
# vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
|
18834
|
+
# and provides 1 executor per worker. We recommend this worker type
|
18835
|
+
# for low volume streaming jobs. This worker type is only available
|
18836
|
+
# for Glue version 3.0 streaming jobs.
|
18837
|
+
#
|
18838
|
+
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
18839
|
+
# 64 GB of memory) with 128 GB disk (approximately 120GB free), and
|
18840
|
+
# provides up to 8 Ray workers based on the autoscaler.
|
18691
18841
|
# @return [String]
|
18692
18842
|
#
|
18693
18843
|
# @!attribute [rw] number_of_workers
|
data/lib/aws-sdk-glue.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aws-sdk-glue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.150.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon Web Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-07-
|
11
|
+
date: 2023-07-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-core
|