aws-sdk-glue 1.149.0 → 1.150.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b02021283a27d804a028be43362e1422178e5a9fbf4e0983a97b6b875eb71ce7
4
- data.tar.gz: be8a9d4ab32b3f9e9ef909cfa553e2e69057794917d1b5802c4cc7bc7cd3b0f6
3
+ metadata.gz: bafae6a5db6eec5869b8fe9e36a225e5a5cde8d9bb2ba4671ef836e6c1cb6cbc
4
+ data.tar.gz: f2f541472d9510817d5e30ec63ba46dbf52d8ada085448d82b94bbc5731d7576
5
5
  SHA512:
6
- metadata.gz: f1af20d4d5bd7da7cd8513ab954ed9d3e40621101a54c86e3f596385be8ad1b29c2bb89b8eb73a39b7f5338e4249bb289107dde5bb9317c3eb7a3d8e6d0f9c42
7
- data.tar.gz: 4a3bcbd479e4c13b2a28e95a94bf1e675ca6749b64a1081347de57dd429c3c970260a7a0f7f89e80dc5dc53d5076b046e0570e2b72b1b884d342715f13d18b4f
6
+ metadata.gz: 61347736e9ff2a97e6bba1a7c18e6c570cd78653b63fe48f7a335233ae0250c3668369743762132baee695bceffab8cc621b9d368f8d150ef4e1d08d0abe7de9
7
+ data.tar.gz: 1ad639c12a24b788e71574d2f13a525cfa4550d97403dbf14a1210ca76825916ea1217a56812f6df5ef39201bd660b75ba910b9f073d94fb0daa1b0eda157910
data/CHANGELOG.md CHANGED
@@ -1,6 +1,11 @@
1
1
  Unreleased Changes
2
2
  ------------------
3
3
 
4
+ 1.150.0 (2023-07-21)
5
+ ------------------
6
+
7
+ * Feature - This release adds support for AWS Glue Crawler with Apache Hudi Tables, allowing Crawlers to discover Hudi Tables in S3 and register them in Glue Data Catalog for query engines to query against.
8
+
4
9
  1.149.0 (2023-07-17)
5
10
  ------------------
6
11
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.149.0
1
+ 1.150.0
@@ -826,6 +826,13 @@ module Aws::Glue
826
826
  # resp.crawlers[0].targets.iceberg_targets[0].exclusions #=> Array
827
827
  # resp.crawlers[0].targets.iceberg_targets[0].exclusions[0] #=> String
828
828
  # resp.crawlers[0].targets.iceberg_targets[0].maximum_traversal_depth #=> Integer
829
+ # resp.crawlers[0].targets.hudi_targets #=> Array
830
+ # resp.crawlers[0].targets.hudi_targets[0].paths #=> Array
831
+ # resp.crawlers[0].targets.hudi_targets[0].paths[0] #=> String
832
+ # resp.crawlers[0].targets.hudi_targets[0].connection_name #=> String
833
+ # resp.crawlers[0].targets.hudi_targets[0].exclusions #=> Array
834
+ # resp.crawlers[0].targets.hudi_targets[0].exclusions[0] #=> String
835
+ # resp.crawlers[0].targets.hudi_targets[0].maximum_traversal_depth #=> Integer
829
836
  # resp.crawlers[0].database_name #=> String
830
837
  # resp.crawlers[0].description #=> String
831
838
  # resp.crawlers[0].classifiers #=> Array
@@ -2795,6 +2802,14 @@ module Aws::Glue
2795
2802
  # maximum_traversal_depth: 1,
2796
2803
  # },
2797
2804
  # ],
2805
+ # hudi_targets: [
2806
+ # {
2807
+ # paths: ["Path"],
2808
+ # connection_name: "ConnectionName",
2809
+ # exclusions: ["Path"],
2810
+ # maximum_traversal_depth: 1,
2811
+ # },
2812
+ # ],
2798
2813
  # },
2799
2814
  # schedule: "CronExpression",
2800
2815
  # classifiers: ["NameString"],
@@ -3354,28 +3369,49 @@ module Aws::Glue
3354
3369
  #
3355
3370
  # @option params [String] :worker_type
3356
3371
  # The type of predefined worker that is allocated when a job runs.
3357
- # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
3372
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
3358
3373
  # Accepts the value Z.2X for Ray jobs.
3359
3374
  #
3360
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
3361
- # of memory and a 50GB disk, and 2 executors per worker.
3362
- #
3363
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16 GB
3364
- # of memory, 64 GB disk), and provides 1 executor per worker. We
3365
- # recommend this worker type for memory-intensive jobs.
3366
- #
3367
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32 GB
3368
- # of memory, 128 GB disk), and provides 1 executor per worker. We
3369
- # recommend this worker type for memory-intensive jobs.
3370
- #
3371
- # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPU,
3372
- # 4 GB of memory, 64 GB disk), and provides 1 executor per worker. We
3373
- # recommend this worker type for low volume streaming jobs. This
3374
- # worker type is only available for Glue version 3.0 streaming jobs.
3375
- #
3376
- # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPU, 64
3377
- # GB of m emory, 128 GB disk), and provides up to 8 Ray workers based
3378
- # on the autoscaler.
3375
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
3376
+ # GB of memory) with 84GB disk (approximately 34GB free), and provides
3377
+ # 1 executor per worker. We recommend this worker type for workloads
3378
+ # such as data transforms, joins, and queries, to offers a scalable
3379
+ # and cost effective way to run most jobs.
3380
+ #
3381
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
3382
+ # GB of memory) with 128GB disk (approximately 77GB free), and
3383
+ # provides 1 executor per worker. We recommend this worker type for
3384
+ # workloads such as data transforms, joins, and queries, to offers a
3385
+ # scalable and cost effective way to run most jobs.
3386
+ #
3387
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs, 64
3388
+ # GB of memory) with 256GB disk (approximately 235GB free), and
3389
+ # provides 1 executor per worker. We recommend this worker type for
3390
+ # jobs whose workloads contain your most demanding transforms,
3391
+ # aggregations, joins, and queries. This worker type is available only
3392
+ # for Glue version 3.0 or later Spark ETL jobs in the following Amazon
3393
+ # Web Services Regions: US East (Ohio), US East (N. Virginia), US West
3394
+ # (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
3395
+ # Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
3396
+ # (Ireland), and Europe (Stockholm).
3397
+ #
3398
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs, 128
3399
+ # GB of memory) with 512GB disk (approximately 487GB free), and
3400
+ # provides 1 executor per worker. We recommend this worker type for
3401
+ # jobs whose workloads contain your most demanding transforms,
3402
+ # aggregations, joins, and queries. This worker type is available only
3403
+ # for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web
3404
+ # Services Regions as supported for the `G.4X` worker type.
3405
+ #
3406
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPUs,
3407
+ # 4 GB of memory) with 84GB disk (approximately 34GB free), and
3408
+ # provides 1 executor per worker. We recommend this worker type for
3409
+ # low volume streaming jobs. This worker type is only available for
3410
+ # Glue version 3.0 streaming jobs.
3411
+ #
3412
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs, 64
3413
+ # GB of memory) with 128 GB disk (approximately 120GB free), and
3414
+ # provides up to 8 Ray workers based on the autoscaler.
3379
3415
  #
3380
3416
  # @option params [Hash<String,Types::CodeGenConfigurationNode>] :code_gen_configuration_nodes
3381
3417
  # The representation of a directed acyclic graph on which both the Glue
@@ -4114,24 +4150,44 @@ module Aws::Glue
4114
4150
  # session.
4115
4151
  #
4116
4152
  # @option params [String] :worker_type
4117
- # The type of predefined worker that is allocated to use for the
4118
- # session. Accepts a value of Standard, G.1X, G.2X, or G.025X.
4119
- #
4120
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
4121
- # of memory and a 50GB disk, and 2 executors per worker.
4122
- #
4123
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16 GB
4124
- # of memory, 64 GB disk), and provides 1 executor per worker. We
4125
- # recommend this worker type for memory-intensive jobs.
4126
- #
4127
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32 GB
4128
- # of memory, 128 GB disk), and provides 1 executor per worker. We
4129
- # recommend this worker type for memory-intensive jobs.
4130
- #
4131
- # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPU,
4132
- # 4 GB of memory, 64 GB disk), and provides 1 executor per worker. We
4133
- # recommend this worker type for low volume streaming jobs. This
4134
- # worker type is only available for Glue version 3.0 streaming jobs.
4153
+ # The type of predefined worker that is allocated when a job runs.
4154
+ # Accepts a value of G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts
4155
+ # the value Z.2X for Ray notebooks.
4156
+ #
4157
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
4158
+ # GB of memory) with 84GB disk (approximately 34GB free), and provides
4159
+ # 1 executor per worker. We recommend this worker type for workloads
4160
+ # such as data transforms, joins, and queries, to offers a scalable
4161
+ # and cost effective way to run most jobs.
4162
+ #
4163
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
4164
+ # GB of memory) with 128GB disk (approximately 77GB free), and
4165
+ # provides 1 executor per worker. We recommend this worker type for
4166
+ # workloads such as data transforms, joins, and queries, to offers a
4167
+ # scalable and cost effective way to run most jobs.
4168
+ #
4169
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs, 64
4170
+ # GB of memory) with 256GB disk (approximately 235GB free), and
4171
+ # provides 1 executor per worker. We recommend this worker type for
4172
+ # jobs whose workloads contain your most demanding transforms,
4173
+ # aggregations, joins, and queries. This worker type is available only
4174
+ # for Glue version 3.0 or later Spark ETL jobs in the following Amazon
4175
+ # Web Services Regions: US East (Ohio), US East (N. Virginia), US West
4176
+ # (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
4177
+ # Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
4178
+ # (Ireland), and Europe (Stockholm).
4179
+ #
4180
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs, 128
4181
+ # GB of memory) with 512GB disk (approximately 487GB free), and
4182
+ # provides 1 executor per worker. We recommend this worker type for
4183
+ # jobs whose workloads contain your most demanding transforms,
4184
+ # aggregations, joins, and queries. This worker type is available only
4185
+ # for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web
4186
+ # Services Regions as supported for the `G.4X` worker type.
4187
+ #
4188
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs, 64
4189
+ # GB of memory) with 128 GB disk (approximately 120GB free), and
4190
+ # provides up to 8 Ray workers based on the autoscaler.
4135
4191
  #
4136
4192
  # @option params [String] :security_configuration
4137
4193
  # The name of the SecurityConfiguration structure to be used with the
@@ -6042,6 +6098,13 @@ module Aws::Glue
6042
6098
  # resp.crawler.targets.iceberg_targets[0].exclusions #=> Array
6043
6099
  # resp.crawler.targets.iceberg_targets[0].exclusions[0] #=> String
6044
6100
  # resp.crawler.targets.iceberg_targets[0].maximum_traversal_depth #=> Integer
6101
+ # resp.crawler.targets.hudi_targets #=> Array
6102
+ # resp.crawler.targets.hudi_targets[0].paths #=> Array
6103
+ # resp.crawler.targets.hudi_targets[0].paths[0] #=> String
6104
+ # resp.crawler.targets.hudi_targets[0].connection_name #=> String
6105
+ # resp.crawler.targets.hudi_targets[0].exclusions #=> Array
6106
+ # resp.crawler.targets.hudi_targets[0].exclusions[0] #=> String
6107
+ # resp.crawler.targets.hudi_targets[0].maximum_traversal_depth #=> Integer
6045
6108
  # resp.crawler.database_name #=> String
6046
6109
  # resp.crawler.description #=> String
6047
6110
  # resp.crawler.classifiers #=> Array
@@ -6196,6 +6259,13 @@ module Aws::Glue
6196
6259
  # resp.crawlers[0].targets.iceberg_targets[0].exclusions #=> Array
6197
6260
  # resp.crawlers[0].targets.iceberg_targets[0].exclusions[0] #=> String
6198
6261
  # resp.crawlers[0].targets.iceberg_targets[0].maximum_traversal_depth #=> Integer
6262
+ # resp.crawlers[0].targets.hudi_targets #=> Array
6263
+ # resp.crawlers[0].targets.hudi_targets[0].paths #=> Array
6264
+ # resp.crawlers[0].targets.hudi_targets[0].paths[0] #=> String
6265
+ # resp.crawlers[0].targets.hudi_targets[0].connection_name #=> String
6266
+ # resp.crawlers[0].targets.hudi_targets[0].exclusions #=> Array
6267
+ # resp.crawlers[0].targets.hudi_targets[0].exclusions[0] #=> String
6268
+ # resp.crawlers[0].targets.hudi_targets[0].maximum_traversal_depth #=> Integer
6199
6269
  # resp.crawlers[0].database_name #=> String
6200
6270
  # resp.crawlers[0].description #=> String
6201
6271
  # resp.crawlers[0].classifiers #=> Array
@@ -13931,28 +14001,49 @@ module Aws::Glue
13931
14001
  #
13932
14002
  # @option params [String] :worker_type
13933
14003
  # The type of predefined worker that is allocated when a job runs.
13934
- # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
14004
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
13935
14005
  # Accepts the value Z.2X for Ray jobs.
13936
14006
  #
13937
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
13938
- # of memory and a 50GB disk, and 2 executors per worker.
13939
- #
13940
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16 GB
13941
- # of memory, 64 GB disk), and provides 1 executor per worker. We
13942
- # recommend this worker type for memory-intensive jobs.
13943
- #
13944
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32 GB
13945
- # of memory, 128 GB disk), and provides 1 executor per worker. We
13946
- # recommend this worker type for memory-intensive jobs.
13947
- #
13948
- # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPU,
13949
- # 4 GB of memory, 64 GB disk), and provides 1 executor per worker. We
13950
- # recommend this worker type for low volume streaming jobs. This
13951
- # worker type is only available for Glue version 3.0 streaming jobs.
13952
- #
13953
- # * For the `Z.2X` worker type, each worker maps to 2 DPU (8vCPU, 64 GB
13954
- # of m emory, 128 GB disk), and provides up to 8 Ray workers (one per
13955
- # vCPU) based on the autoscaler.
14007
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
14008
+ # GB of memory) with 84GB disk (approximately 34GB free), and provides
14009
+ # 1 executor per worker. We recommend this worker type for workloads
14010
+ # such as data transforms, joins, and queries, to offers a scalable
14011
+ # and cost effective way to run most jobs.
14012
+ #
14013
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
14014
+ # GB of memory) with 128GB disk (approximately 77GB free), and
14015
+ # provides 1 executor per worker. We recommend this worker type for
14016
+ # workloads such as data transforms, joins, and queries, to offers a
14017
+ # scalable and cost effective way to run most jobs.
14018
+ #
14019
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs, 64
14020
+ # GB of memory) with 256GB disk (approximately 235GB free), and
14021
+ # provides 1 executor per worker. We recommend this worker type for
14022
+ # jobs whose workloads contain your most demanding transforms,
14023
+ # aggregations, joins, and queries. This worker type is available only
14024
+ # for Glue version 3.0 or later Spark ETL jobs in the following Amazon
14025
+ # Web Services Regions: US East (Ohio), US East (N. Virginia), US West
14026
+ # (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
14027
+ # Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
14028
+ # (Ireland), and Europe (Stockholm).
14029
+ #
14030
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs, 128
14031
+ # GB of memory) with 512GB disk (approximately 487GB free), and
14032
+ # provides 1 executor per worker. We recommend this worker type for
14033
+ # jobs whose workloads contain your most demanding transforms,
14034
+ # aggregations, joins, and queries. This worker type is available only
14035
+ # for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web
14036
+ # Services Regions as supported for the `G.4X` worker type.
14037
+ #
14038
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPUs,
14039
+ # 4 GB of memory) with 84GB disk (approximately 34GB free), and
14040
+ # provides 1 executor per worker. We recommend this worker type for
14041
+ # low volume streaming jobs. This worker type is only available for
14042
+ # Glue version 3.0 streaming jobs.
14043
+ #
14044
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs, 64
14045
+ # GB of memory) with 128 GB disk (approximately 120GB free), and
14046
+ # provides up to 8 Ray workers based on the autoscaler.
13956
14047
  #
13957
14048
  # @option params [Integer] :number_of_workers
13958
14049
  # The number of workers of a defined `workerType` that are allocated
@@ -14898,6 +14989,14 @@ module Aws::Glue
14898
14989
  # maximum_traversal_depth: 1,
14899
14990
  # },
14900
14991
  # ],
14992
+ # hudi_targets: [
14993
+ # {
14994
+ # paths: ["Path"],
14995
+ # connection_name: "ConnectionName",
14996
+ # exclusions: ["Path"],
14997
+ # maximum_traversal_depth: 1,
14998
+ # },
14999
+ # ],
14901
15000
  # },
14902
15001
  # schedule: "CronExpression",
14903
15002
  # classifiers: ["NameString"],
@@ -15954,7 +16053,7 @@ module Aws::Glue
15954
16053
  params: params,
15955
16054
  config: config)
15956
16055
  context[:gem_name] = 'aws-sdk-glue'
15957
- context[:gem_version] = '1.149.0'
16056
+ context[:gem_version] = '1.150.0'
15958
16057
  Seahorse::Client::Request.new(handlers, context)
15959
16058
  end
15960
16059
 
@@ -609,7 +609,9 @@ module Aws::Glue
609
609
  GrokClassifier = Shapes::StructureShape.new(name: 'GrokClassifier')
610
610
  GrokPattern = Shapes::StringShape.new(name: 'GrokPattern')
611
611
  HashString = Shapes::StringShape.new(name: 'HashString')
612
+ HudiTarget = Shapes::StructureShape.new(name: 'HudiTarget')
612
613
  HudiTargetCompressionType = Shapes::StringShape.new(name: 'HudiTargetCompressionType')
614
+ HudiTargetList = Shapes::ListShape.new(name: 'HudiTargetList')
613
615
  IcebergInput = Shapes::StructureShape.new(name: 'IcebergInput')
614
616
  IcebergTarget = Shapes::StructureShape.new(name: 'IcebergTarget')
615
617
  IcebergTargetList = Shapes::ListShape.new(name: 'IcebergTargetList')
@@ -1872,6 +1874,7 @@ module Aws::Glue
1872
1874
  CrawlerTargets.add_member(:catalog_targets, Shapes::ShapeRef.new(shape: CatalogTargetList, location_name: "CatalogTargets"))
1873
1875
  CrawlerTargets.add_member(:delta_targets, Shapes::ShapeRef.new(shape: DeltaTargetList, location_name: "DeltaTargets"))
1874
1876
  CrawlerTargets.add_member(:iceberg_targets, Shapes::ShapeRef.new(shape: IcebergTargetList, location_name: "IcebergTargets"))
1877
+ CrawlerTargets.add_member(:hudi_targets, Shapes::ShapeRef.new(shape: HudiTargetList, location_name: "HudiTargets"))
1875
1878
  CrawlerTargets.struct_class = Types::CrawlerTargets
1876
1879
 
1877
1880
  CrawlsFilter.add_member(:field_name, Shapes::ShapeRef.new(shape: FieldName, location_name: "FieldName"))
@@ -3536,6 +3539,14 @@ module Aws::Glue
3536
3539
  GrokClassifier.add_member(:custom_patterns, Shapes::ShapeRef.new(shape: CustomPatterns, location_name: "CustomPatterns"))
3537
3540
  GrokClassifier.struct_class = Types::GrokClassifier
3538
3541
 
3542
+ HudiTarget.add_member(:paths, Shapes::ShapeRef.new(shape: PathList, location_name: "Paths"))
3543
+ HudiTarget.add_member(:connection_name, Shapes::ShapeRef.new(shape: ConnectionName, location_name: "ConnectionName"))
3544
+ HudiTarget.add_member(:exclusions, Shapes::ShapeRef.new(shape: PathList, location_name: "Exclusions"))
3545
+ HudiTarget.add_member(:maximum_traversal_depth, Shapes::ShapeRef.new(shape: NullableInteger, location_name: "MaximumTraversalDepth"))
3546
+ HudiTarget.struct_class = Types::HudiTarget
3547
+
3548
+ HudiTargetList.member = Shapes::ShapeRef.new(shape: HudiTarget)
3549
+
3539
3550
  IcebergInput.add_member(:metadata_operation, Shapes::ShapeRef.new(shape: MetadataOperation, required: true, location_name: "MetadataOperation"))
3540
3551
  IcebergInput.add_member(:version, Shapes::ShapeRef.new(shape: VersionString, location_name: "Version"))
3541
3552
  IcebergInput.struct_class = Types::IcebergInput
@@ -3458,6 +3458,10 @@ module Aws::Glue
3458
3458
  # Specifies Apache Iceberg data store targets.
3459
3459
  # @return [Array<Types::IcebergTarget>]
3460
3460
  #
3461
+ # @!attribute [rw] hudi_targets
3462
+ # Specifies Apache Hudi data store targets.
3463
+ # @return [Array<Types::HudiTarget>]
3464
+ #
3461
3465
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CrawlerTargets AWS API Documentation
3462
3466
  #
3463
3467
  class CrawlerTargets < Struct.new(
@@ -3467,7 +3471,8 @@ module Aws::Glue
3467
3471
  :dynamo_db_targets,
3468
3472
  :catalog_targets,
3469
3473
  :delta_targets,
3470
- :iceberg_targets)
3474
+ :iceberg_targets,
3475
+ :hudi_targets)
3471
3476
  SENSITIVE = []
3472
3477
  include Aws::Structure
3473
3478
  end
@@ -4400,29 +4405,50 @@ module Aws::Glue
4400
4405
  #
4401
4406
  # @!attribute [rw] worker_type
4402
4407
  # The type of predefined worker that is allocated when a job runs.
4403
- # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
4408
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
4404
4409
  # Accepts the value Z.2X for Ray jobs.
4405
4410
  #
4406
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
4407
- # of memory and a 50GB disk, and 2 executors per worker.
4408
- #
4409
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
4410
- # GB of memory, 64 GB disk), and provides 1 executor per worker. We
4411
- # recommend this worker type for memory-intensive jobs.
4412
- #
4413
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
4414
- # GB of memory, 128 GB disk), and provides 1 executor per worker. We
4415
- # recommend this worker type for memory-intensive jobs.
4411
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
4412
+ # GB of memory) with 84GB disk (approximately 34GB free), and
4413
+ # provides 1 executor per worker. We recommend this worker type for
4414
+ # workloads such as data transforms, joins, and queries, to offers a
4415
+ # scalable and cost effective way to run most jobs.
4416
+ #
4417
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
4418
+ # GB of memory) with 128GB disk (approximately 77GB free), and
4419
+ # provides 1 executor per worker. We recommend this worker type for
4420
+ # workloads such as data transforms, joins, and queries, to offers a
4421
+ # scalable and cost effective way to run most jobs.
4422
+ #
4423
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
4424
+ # 64 GB of memory) with 256GB disk (approximately 235GB free), and
4425
+ # provides 1 executor per worker. We recommend this worker type for
4426
+ # jobs whose workloads contain your most demanding transforms,
4427
+ # aggregations, joins, and queries. This worker type is available
4428
+ # only for Glue version 3.0 or later Spark ETL jobs in the following
4429
+ # Amazon Web Services Regions: US East (Ohio), US East (N.
4430
+ # Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
4431
+ # Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
4432
+ # (Frankfurt), Europe (Ireland), and Europe (Stockholm).
4433
+ #
4434
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
4435
+ # 128 GB of memory) with 512GB disk (approximately 487GB free), and
4436
+ # provides 1 executor per worker. We recommend this worker type for
4437
+ # jobs whose workloads contain your most demanding transforms,
4438
+ # aggregations, joins, and queries. This worker type is available
4439
+ # only for Glue version 3.0 or later Spark ETL jobs, in the same
4440
+ # Amazon Web Services Regions as supported for the `G.4X` worker
4441
+ # type.
4416
4442
  #
4417
4443
  # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
4418
- # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
4419
- # worker. We recommend this worker type for low volume streaming
4420
- # jobs. This worker type is only available for Glue version 3.0
4421
- # streaming jobs.
4444
+ # vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
4445
+ # and provides 1 executor per worker. We recommend this worker type
4446
+ # for low volume streaming jobs. This worker type is only available
4447
+ # for Glue version 3.0 streaming jobs.
4422
4448
  #
4423
- # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPU, 64
4424
- # GB of m emory, 128 GB disk), and provides up to 8 Ray workers
4425
- # based on the autoscaler.
4449
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
4450
+ # 64 GB of memory) with 128 GB disk (approximately 120GB free), and
4451
+ # provides up to 8 Ray workers based on the autoscaler.
4426
4452
  # @return [String]
4427
4453
  #
4428
4454
  # @!attribute [rw] code_gen_configuration_nodes
@@ -5117,25 +5143,45 @@ module Aws::Glue
5117
5143
  # @return [Integer]
5118
5144
  #
5119
5145
  # @!attribute [rw] worker_type
5120
- # The type of predefined worker that is allocated to use for the
5121
- # session. Accepts a value of Standard, G.1X, G.2X, or G.025X.
5122
- #
5123
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
5124
- # of memory and a 50GB disk, and 2 executors per worker.
5125
- #
5126
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
5127
- # GB of memory, 64 GB disk), and provides 1 executor per worker. We
5128
- # recommend this worker type for memory-intensive jobs.
5129
- #
5130
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
5131
- # GB of memory, 128 GB disk), and provides 1 executor per worker. We
5132
- # recommend this worker type for memory-intensive jobs.
5133
- #
5134
- # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
5135
- # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
5136
- # worker. We recommend this worker type for low volume streaming
5137
- # jobs. This worker type is only available for Glue version 3.0
5138
- # streaming jobs.
5146
+ # The type of predefined worker that is allocated when a job runs.
5147
+ # Accepts a value of G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts
5148
+ # the value Z.2X for Ray notebooks.
5149
+ #
5150
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
5151
+ # GB of memory) with 84GB disk (approximately 34GB free), and
5152
+ # provides 1 executor per worker. We recommend this worker type for
5153
+ # workloads such as data transforms, joins, and queries, to offers a
5154
+ # scalable and cost effective way to run most jobs.
5155
+ #
5156
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
5157
+ # GB of memory) with 128GB disk (approximately 77GB free), and
5158
+ # provides 1 executor per worker. We recommend this worker type for
5159
+ # workloads such as data transforms, joins, and queries, to offers a
5160
+ # scalable and cost effective way to run most jobs.
5161
+ #
5162
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
5163
+ # 64 GB of memory) with 256GB disk (approximately 235GB free), and
5164
+ # provides 1 executor per worker. We recommend this worker type for
5165
+ # jobs whose workloads contain your most demanding transforms,
5166
+ # aggregations, joins, and queries. This worker type is available
5167
+ # only for Glue version 3.0 or later Spark ETL jobs in the following
5168
+ # Amazon Web Services Regions: US East (Ohio), US East (N.
5169
+ # Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
5170
+ # Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
5171
+ # (Frankfurt), Europe (Ireland), and Europe (Stockholm).
5172
+ #
5173
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
5174
+ # 128 GB of memory) with 512GB disk (approximately 487GB free), and
5175
+ # provides 1 executor per worker. We recommend this worker type for
5176
+ # jobs whose workloads contain your most demanding transforms,
5177
+ # aggregations, joins, and queries. This worker type is available
5178
+ # only for Glue version 3.0 or later Spark ETL jobs, in the same
5179
+ # Amazon Web Services Regions as supported for the `G.4X` worker
5180
+ # type.
5181
+ #
5182
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
5183
+ # 64 GB of memory) with 128 GB disk (approximately 120GB free), and
5184
+ # provides up to 8 Ray workers based on the autoscaler.
5139
5185
  # @return [String]
5140
5186
  #
5141
5187
  # @!attribute [rw] security_configuration
@@ -11808,6 +11854,49 @@ module Aws::Glue
11808
11854
  include Aws::Structure
11809
11855
  end
11810
11856
 
11857
+ # Specifies an Apache Hudi data source.
11858
+ #
11859
+ # @!attribute [rw] paths
11860
+ # An array of Amazon S3 location strings for Hudi, each indicating the
11861
+ # root folder with which the metadata files for a Hudi table resides.
11862
+ # The Hudi folder may be located in a child folder of the root folder.
11863
+ #
11864
+ # The crawler will scan all folders underneath a path for a Hudi
11865
+ # folder.
11866
+ # @return [Array<String>]
11867
+ #
11868
+ # @!attribute [rw] connection_name
11869
+ # The name of the connection to use to connect to the Hudi target. If
11870
+ # your Hudi files are stored in buckets that require VPC
11871
+ # authorization, you can set their connection properties here.
11872
+ # @return [String]
11873
+ #
11874
+ # @!attribute [rw] exclusions
11875
+ # A list of glob patterns used to exclude from the crawl. For more
11876
+ # information, see [Catalog Tables with a Crawler][1].
11877
+ #
11878
+ #
11879
+ #
11880
+ # [1]: https://docs.aws.amazon.com/glue/latest/dg/add-crawler.html
11881
+ # @return [Array<String>]
11882
+ #
11883
+ # @!attribute [rw] maximum_traversal_depth
11884
+ # The maximum depth of Amazon S3 paths that the crawler can traverse
11885
+ # to discover the Hudi metadata folder in your Amazon S3 path. Used to
11886
+ # limit the crawler run time.
11887
+ # @return [Integer]
11888
+ #
11889
+ # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/HudiTarget AWS API Documentation
11890
+ #
11891
+ class HudiTarget < Struct.new(
11892
+ :paths,
11893
+ :connection_name,
11894
+ :exclusions,
11895
+ :maximum_traversal_depth)
11896
+ SENSITIVE = []
11897
+ include Aws::Structure
11898
+ end
11899
+
11811
11900
  # A structure that defines an Apache Iceberg metadata table to create in
11812
11901
  # the catalog.
11813
11902
  #
@@ -12356,52 +12445,50 @@ module Aws::Glue
12356
12445
  #
12357
12446
  # @!attribute [rw] worker_type
12358
12447
  # The type of predefined worker that is allocated when a job runs.
12359
- # Accepts a value of Standard, G.1X, G.2X, G.4X, G.8X, or G.025X for
12360
- # Spark jobs. Accepts the value Z.2X for Ray jobs.
12361
- #
12362
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
12363
- # of memory and a 50GB disk, and 2 executors per worker.
12364
- #
12365
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
12366
- # GB of memory, 64 GB disk), and provides 1 executor per worker. We
12367
- # recommend this worker type for workloads such as data transforms,
12368
- # joins, and queries, to offers a scalable and cost effective way to
12369
- # run most jobs.
12448
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
12449
+ # Accepts the value Z.2X for Ray jobs.
12370
12450
  #
12371
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
12372
- # GB of memory, 128 GB disk), and provides 1 executor per worker. We
12373
- # recommend this worker type for workloads such as data transforms,
12374
- # joins, and queries, to offers a scalable and cost effective way to
12375
- # run most jobs.
12376
- #
12377
- # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPU, 64
12378
- # GB of memory, 256 GB disk), and provides 1 executor per worker. We
12379
- # recommend this worker type for jobs whose workloads contain your
12380
- # most demanding transforms, aggregations, joins, and queries. This
12381
- # worker type is available only for Glue version 3.0 or later Spark
12382
- # ETL jobs in the following Amazon Web Services Regions: US East
12383
- # (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific
12384
- # (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada
12385
- # (Central), Europe (Frankfurt), Europe (Ireland), and Europe
12386
- # (Stockholm).
12387
- #
12388
- # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPU,
12389
- # 128 GB of memory, 512 GB disk), and provides 1 executor per
12390
- # worker. We recommend this worker type for jobs whose workloads
12391
- # contain your most demanding transforms, aggregations, joins, and
12392
- # queries. This worker type is available only for Glue version 3.0
12393
- # or later Spark ETL jobs, in the same Amazon Web Services Regions
12394
- # as supported for the `G.4X` worker type.
12451
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
12452
+ # GB of memory) with 84GB disk (approximately 34GB free), and
12453
+ # provides 1 executor per worker. We recommend this worker type for
12454
+ # workloads such as data transforms, joins, and queries, to offers a
12455
+ # scalable and cost effective way to run most jobs.
12456
+ #
12457
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
12458
+ # GB of memory) with 128GB disk (approximately 77GB free), and
12459
+ # provides 1 executor per worker. We recommend this worker type for
12460
+ # workloads such as data transforms, joins, and queries, to offers a
12461
+ # scalable and cost effective way to run most jobs.
12462
+ #
12463
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
12464
+ # 64 GB of memory) with 256GB disk (approximately 235GB free), and
12465
+ # provides 1 executor per worker. We recommend this worker type for
12466
+ # jobs whose workloads contain your most demanding transforms,
12467
+ # aggregations, joins, and queries. This worker type is available
12468
+ # only for Glue version 3.0 or later Spark ETL jobs in the following
12469
+ # Amazon Web Services Regions: US East (Ohio), US East (N.
12470
+ # Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
12471
+ # Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
12472
+ # (Frankfurt), Europe (Ireland), and Europe (Stockholm).
12473
+ #
12474
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
12475
+ # 128 GB of memory) with 512GB disk (approximately 487GB free), and
12476
+ # provides 1 executor per worker. We recommend this worker type for
12477
+ # jobs whose workloads contain your most demanding transforms,
12478
+ # aggregations, joins, and queries. This worker type is available
12479
+ # only for Glue version 3.0 or later Spark ETL jobs, in the same
12480
+ # Amazon Web Services Regions as supported for the `G.4X` worker
12481
+ # type.
12395
12482
  #
12396
12483
  # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
12397
- # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
12398
- # worker. We recommend this worker type for low volume streaming
12399
- # jobs. This worker type is only available for Glue version 3.0
12400
- # streaming jobs.
12484
+ # vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
12485
+ # and provides 1 executor per worker. We recommend this worker type
12486
+ # for low volume streaming jobs. This worker type is only available
12487
+ # for Glue version 3.0 streaming jobs.
12401
12488
  #
12402
- # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPU, 64
12403
- # GB of m emory, 128 GB disk), and provides a default of 8 Ray
12404
- # workers (1 per vCPU).
12489
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
12490
+ # 64 GB of memory) with 128 GB disk (approximately 120GB free), and
12491
+ # provides up to 8 Ray workers based on the autoscaler.
12405
12492
  # @return [String]
12406
12493
  #
12407
12494
  # @!attribute [rw] number_of_workers
@@ -12761,29 +12848,50 @@ module Aws::Glue
12761
12848
  #
12762
12849
  # @!attribute [rw] worker_type
12763
12850
  # The type of predefined worker that is allocated when a job runs.
12764
- # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
12851
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
12765
12852
  # Accepts the value Z.2X for Ray jobs.
12766
12853
  #
12767
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
12768
- # of memory and a 50GB disk, and 2 executors per worker.
12769
- #
12770
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
12771
- # GB of memory, 64 GB disk), and provides 1 executor per worker. We
12772
- # recommend this worker type for memory-intensive jobs.
12773
- #
12774
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
12775
- # GB of memory, 128 GB disk), and provides 1 executor per worker. We
12776
- # recommend this worker type for memory-intensive jobs.
12854
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
12855
+ # GB of memory) with 84GB disk (approximately 34GB free), and
12856
+ # provides 1 executor per worker. We recommend this worker type for
12857
+ # workloads such as data transforms, joins, and queries, to offers a
12858
+ # scalable and cost effective way to run most jobs.
12859
+ #
12860
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
12861
+ # GB of memory) with 128GB disk (approximately 77GB free), and
12862
+ # provides 1 executor per worker. We recommend this worker type for
12863
+ # workloads such as data transforms, joins, and queries, to offers a
12864
+ # scalable and cost effective way to run most jobs.
12865
+ #
12866
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
12867
+ # 64 GB of memory) with 256GB disk (approximately 235GB free), and
12868
+ # provides 1 executor per worker. We recommend this worker type for
12869
+ # jobs whose workloads contain your most demanding transforms,
12870
+ # aggregations, joins, and queries. This worker type is available
12871
+ # only for Glue version 3.0 or later Spark ETL jobs in the following
12872
+ # Amazon Web Services Regions: US East (Ohio), US East (N.
12873
+ # Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
12874
+ # Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
12875
+ # (Frankfurt), Europe (Ireland), and Europe (Stockholm).
12876
+ #
12877
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
12878
+ # 128 GB of memory) with 512GB disk (approximately 487GB free), and
12879
+ # provides 1 executor per worker. We recommend this worker type for
12880
+ # jobs whose workloads contain your most demanding transforms,
12881
+ # aggregations, joins, and queries. This worker type is available
12882
+ # only for Glue version 3.0 or later Spark ETL jobs, in the same
12883
+ # Amazon Web Services Regions as supported for the `G.4X` worker
12884
+ # type.
12777
12885
  #
12778
12886
  # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
12779
- # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
12780
- # worker. We recommend this worker type for low volume streaming
12781
- # jobs. This worker type is only available for Glue version 3.0
12782
- # streaming jobs.
12887
+ # vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
12888
+ # and provides 1 executor per worker. We recommend this worker type
12889
+ # for low volume streaming jobs. This worker type is only available
12890
+ # for Glue version 3.0 streaming jobs.
12783
12891
  #
12784
- # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPU, 64
12785
- # GB of m emory, 128 GB disk), and provides up to 8 Ray workers (one
12786
- # per vCPU) based on the autoscaler.
12892
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
12893
+ # 64 GB of memory) with 128 GB disk (approximately 120GB free), and
12894
+ # provides up to 8 Ray workers based on the autoscaler.
12787
12895
  # @return [String]
12788
12896
  #
12789
12897
  # @!attribute [rw] number_of_workers
@@ -13013,29 +13121,50 @@ module Aws::Glue
13013
13121
  #
13014
13122
  # @!attribute [rw] worker_type
13015
13123
  # The type of predefined worker that is allocated when a job runs.
13016
- # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
13124
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
13017
13125
  # Accepts the value Z.2X for Ray jobs.
13018
13126
  #
13019
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
13020
- # of memory and a 50GB disk, and 2 executors per worker.
13021
- #
13022
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
13023
- # GB of memory, 64 GB disk), and provides 1 executor per worker. We
13024
- # recommend this worker type for memory-intensive jobs.
13025
- #
13026
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
13027
- # GB of memory, 128 GB disk), and provides 1 executor per worker. We
13028
- # recommend this worker type for memory-intensive jobs.
13127
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
13128
+ # GB of memory) with 84GB disk (approximately 34GB free), and
13129
+ # provides 1 executor per worker. We recommend this worker type for
13130
+ # workloads such as data transforms, joins, and queries, to offers a
13131
+ # scalable and cost effective way to run most jobs.
13132
+ #
13133
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
13134
+ # GB of memory) with 128GB disk (approximately 77GB free), and
13135
+ # provides 1 executor per worker. We recommend this worker type for
13136
+ # workloads such as data transforms, joins, and queries, to offers a
13137
+ # scalable and cost effective way to run most jobs.
13138
+ #
13139
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
13140
+ # 64 GB of memory) with 256GB disk (approximately 235GB free), and
13141
+ # provides 1 executor per worker. We recommend this worker type for
13142
+ # jobs whose workloads contain your most demanding transforms,
13143
+ # aggregations, joins, and queries. This worker type is available
13144
+ # only for Glue version 3.0 or later Spark ETL jobs in the following
13145
+ # Amazon Web Services Regions: US East (Ohio), US East (N.
13146
+ # Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
13147
+ # Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
13148
+ # (Frankfurt), Europe (Ireland), and Europe (Stockholm).
13149
+ #
13150
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
13151
+ # 128 GB of memory) with 512GB disk (approximately 487GB free), and
13152
+ # provides 1 executor per worker. We recommend this worker type for
13153
+ # jobs whose workloads contain your most demanding transforms,
13154
+ # aggregations, joins, and queries. This worker type is available
13155
+ # only for Glue version 3.0 or later Spark ETL jobs, in the same
13156
+ # Amazon Web Services Regions as supported for the `G.4X` worker
13157
+ # type.
13029
13158
  #
13030
13159
  # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
13031
- # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
13032
- # worker. We recommend this worker type for low volume streaming
13033
- # jobs. This worker type is only available for Glue version 3.0
13034
- # streaming jobs.
13160
+ # vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
13161
+ # and provides 1 executor per worker. We recommend this worker type
13162
+ # for low volume streaming jobs. This worker type is only available
13163
+ # for Glue version 3.0 streaming jobs.
13035
13164
  #
13036
- # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPU, 64
13037
- # GB of m emory, 128 GB disk), and provides up to 8 Ray workers
13038
- # based on the autoscaler.
13165
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
13166
+ # 64 GB of memory) with 128 GB disk (approximately 120GB free), and
13167
+ # provides up to 8 Ray workers based on the autoscaler.
13039
13168
  # @return [String]
13040
13169
  #
13041
13170
  # @!attribute [rw] number_of_workers
@@ -18665,29 +18794,50 @@ module Aws::Glue
18665
18794
  #
18666
18795
  # @!attribute [rw] worker_type
18667
18796
  # The type of predefined worker that is allocated when a job runs.
18668
- # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
18797
+ # Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs.
18669
18798
  # Accepts the value Z.2X for Ray jobs.
18670
18799
  #
18671
- # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
18672
- # of memory and a 50GB disk, and 2 executors per worker.
18673
- #
18674
- # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
18675
- # GB of memory, 64 GB disk), and provides 1 executor per worker. We
18676
- # recommend this worker type for memory-intensive jobs.
18677
- #
18678
- # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
18679
- # GB of memory, 128 GB disk), and provides 1 executor per worker. We
18680
- # recommend this worker type for memory-intensive jobs.
18800
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
18801
+ # GB of memory) with 84GB disk (approximately 34GB free), and
18802
+ # provides 1 executor per worker. We recommend this worker type for
18803
+ # workloads such as data transforms, joins, and queries, to offers a
18804
+ # scalable and cost effective way to run most jobs.
18805
+ #
18806
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPUs, 32
18807
+ # GB of memory) with 128GB disk (approximately 77GB free), and
18808
+ # provides 1 executor per worker. We recommend this worker type for
18809
+ # workloads such as data transforms, joins, and queries, to offers a
18810
+ # scalable and cost effective way to run most jobs.
18811
+ #
18812
+ # * For the `G.4X` worker type, each worker maps to 4 DPU (16 vCPUs,
18813
+ # 64 GB of memory) with 256GB disk (approximately 235GB free), and
18814
+ # provides 1 executor per worker. We recommend this worker type for
18815
+ # jobs whose workloads contain your most demanding transforms,
18816
+ # aggregations, joins, and queries. This worker type is available
18817
+ # only for Glue version 3.0 or later Spark ETL jobs in the following
18818
+ # Amazon Web Services Regions: US East (Ohio), US East (N.
18819
+ # Virginia), US West (Oregon), Asia Pacific (Singapore), Asia
18820
+ # Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe
18821
+ # (Frankfurt), Europe (Ireland), and Europe (Stockholm).
18822
+ #
18823
+ # * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
18824
+ # 128 GB of memory) with 512GB disk (approximately 487GB free), and
18825
+ # provides 1 executor per worker. We recommend this worker type for
18826
+ # jobs whose workloads contain your most demanding transforms,
18827
+ # aggregations, joins, and queries. This worker type is available
18828
+ # only for Glue version 3.0 or later Spark ETL jobs, in the same
18829
+ # Amazon Web Services Regions as supported for the `G.4X` worker
18830
+ # type.
18681
18831
  #
18682
18832
  # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
18683
- # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
18684
- # worker. We recommend this worker type for low volume streaming
18685
- # jobs. This worker type is only available for Glue version 3.0
18686
- # streaming jobs.
18687
- #
18688
- # * For the `Z.2X` worker type, each worker maps to 2 DPU (8vCPU, 64
18689
- # GB of m emory, 128 GB disk), and provides up to 8 Ray workers (one
18690
- # per vCPU) based on the autoscaler.
18833
+ # vCPUs, 4 GB of memory) with 84GB disk (approximately 34GB free),
18834
+ # and provides 1 executor per worker. We recommend this worker type
18835
+ # for low volume streaming jobs. This worker type is only available
18836
+ # for Glue version 3.0 streaming jobs.
18837
+ #
18838
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
18839
+ # 64 GB of memory) with 128 GB disk (approximately 120GB free), and
18840
+ # provides up to 8 Ray workers based on the autoscaler.
18691
18841
  # @return [String]
18692
18842
  #
18693
18843
  # @!attribute [rw] number_of_workers
data/lib/aws-sdk-glue.rb CHANGED
@@ -52,6 +52,6 @@ require_relative 'aws-sdk-glue/customizations'
52
52
  # @!group service
53
53
  module Aws::Glue
54
54
 
55
- GEM_VERSION = '1.149.0'
55
+ GEM_VERSION = '1.150.0'
56
56
 
57
57
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aws-sdk-glue
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.149.0
4
+ version: 1.150.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-07-17 00:00:00.000000000 Z
11
+ date: 2023-07-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-core