aws-sdk-glue 1.114.0 → 1.117.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-glue/client.rb +169 -86
- data/lib/aws-sdk-glue/client_api.rb +8 -0
- data/lib/aws-sdk-glue/types.rb +221 -74
- data/lib/aws-sdk-glue.rb +1 -1
- metadata +2 -2
data/lib/aws-sdk-glue/types.rb
CHANGED
@@ -3558,9 +3558,9 @@ module Aws::Glue
|
|
3558
3558
|
# {
|
3559
3559
|
# logical_operator: "EQUALS", # accepts EQUALS
|
3560
3560
|
# job_name: "NameString",
|
3561
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
3561
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
3562
3562
|
# crawler_name: "NameString",
|
3563
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
3563
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
3564
3564
|
# }
|
3565
3565
|
#
|
3566
3566
|
# @!attribute [rw] logical_operator
|
@@ -5042,7 +5042,7 @@ module Aws::Glue
|
|
5042
5042
|
# public_key: "GenericString",
|
5043
5043
|
# public_keys: ["GenericString"],
|
5044
5044
|
# number_of_nodes: 1,
|
5045
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
5045
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
5046
5046
|
# glue_version: "GlueVersionString",
|
5047
5047
|
# number_of_workers: 1,
|
5048
5048
|
# extra_python_libs_s3_path: "GenericString",
|
@@ -5420,7 +5420,7 @@ module Aws::Glue
|
|
5420
5420
|
# },
|
5421
5421
|
# glue_version: "GlueVersionString",
|
5422
5422
|
# number_of_workers: 1,
|
5423
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
5423
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
5424
5424
|
# code_gen_configuration_nodes: {
|
5425
5425
|
# "NodeId" => {
|
5426
5426
|
# athena_connector_source: {
|
@@ -6089,6 +6089,7 @@ module Aws::Glue
|
|
6089
6089
|
# },
|
6090
6090
|
# },
|
6091
6091
|
# },
|
6092
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
6092
6093
|
# }
|
6093
6094
|
#
|
6094
6095
|
# @!attribute [rw] name
|
@@ -6124,6 +6125,11 @@ module Aws::Glue
|
|
6124
6125
|
# You can specify arguments here that your own job-execution script
|
6125
6126
|
# consumes, as well as arguments that Glue itself consumes.
|
6126
6127
|
#
|
6128
|
+
# Job arguments may be logged. Do not pass plaintext secrets as
|
6129
|
+
# arguments. Retrieve secrets from a Glue Connection, Secrets Manager
|
6130
|
+
# or other secret management mechanism if you intend to keep them
|
6131
|
+
# within the Job.
|
6132
|
+
#
|
6127
6133
|
# For information about how to specify and consume your own Job
|
6128
6134
|
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
6129
6135
|
# developer guide.
|
@@ -6155,7 +6161,7 @@ module Aws::Glue
|
|
6155
6161
|
# This parameter is deprecated. Use `MaxCapacity` instead.
|
6156
6162
|
#
|
6157
6163
|
# The number of Glue data processing units (DPUs) to allocate to this
|
6158
|
-
# Job. You can allocate
|
6164
|
+
# Job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU
|
6159
6165
|
# is a relative measure of processing power that consists of 4 vCPUs
|
6160
6166
|
# of compute capacity and 16 GB of memory. For more information, see
|
6161
6167
|
# the [Glue pricing page][1].
|
@@ -6190,8 +6196,8 @@ module Aws::Glue
|
|
6190
6196
|
#
|
6191
6197
|
# * When you specify an Apache Spark ETL job
|
6192
6198
|
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
|
6193
|
-
# (`JobCommand.Name`="gluestreaming"), you can allocate
|
6194
|
-
#
|
6199
|
+
# (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
|
6200
|
+
# of 2 DPUs. The default is 10 DPUs. This job type cannot have a
|
6195
6201
|
# fractional DPU allocation.
|
6196
6202
|
#
|
6197
6203
|
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
@@ -6242,14 +6248,11 @@ module Aws::Glue
|
|
6242
6248
|
# @!attribute [rw] number_of_workers
|
6243
6249
|
# The number of workers of a defined `workerType` that are allocated
|
6244
6250
|
# when a job runs.
|
6245
|
-
#
|
6246
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
6247
|
-
# 149 for `G.2X`.
|
6248
6251
|
# @return [Integer]
|
6249
6252
|
#
|
6250
6253
|
# @!attribute [rw] worker_type
|
6251
6254
|
# The type of predefined worker that is allocated when a job runs.
|
6252
|
-
# Accepts a value of Standard, G.1X, or G.
|
6255
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
6253
6256
|
#
|
6254
6257
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
6255
6258
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -6261,6 +6264,12 @@ module Aws::Glue
|
|
6261
6264
|
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
6262
6265
|
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
6263
6266
|
# recommend this worker type for memory-intensive jobs.
|
6267
|
+
#
|
6268
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
6269
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
6270
|
+
# worker. We recommend this worker type for low volume streaming
|
6271
|
+
# jobs. This worker type is only available for Glue version 3.0
|
6272
|
+
# streaming jobs.
|
6264
6273
|
# @return [String]
|
6265
6274
|
#
|
6266
6275
|
# @!attribute [rw] code_gen_configuration_nodes
|
@@ -6269,6 +6278,20 @@ module Aws::Glue
|
|
6269
6278
|
# based.
|
6270
6279
|
# @return [Hash<String,Types::CodeGenConfigurationNode>]
|
6271
6280
|
#
|
6281
|
+
# @!attribute [rw] execution_class
|
6282
|
+
# Indicates whether the job is run with a standard or flexible
|
6283
|
+
# execution class. The standard execution-class is ideal for
|
6284
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
6285
|
+
# resources.
|
6286
|
+
#
|
6287
|
+
# The flexible execution class is appropriate for time-insensitive
|
6288
|
+
# jobs whose start and completion times may vary.
|
6289
|
+
#
|
6290
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
6291
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
6292
|
+
# execution class is available for Spark jobs.
|
6293
|
+
# @return [String]
|
6294
|
+
#
|
6272
6295
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateJobRequest AWS API Documentation
|
6273
6296
|
#
|
6274
6297
|
class CreateJobRequest < Struct.new(
|
@@ -6291,7 +6314,8 @@ module Aws::Glue
|
|
6291
6314
|
:glue_version,
|
6292
6315
|
:number_of_workers,
|
6293
6316
|
:worker_type,
|
6294
|
-
:code_gen_configuration_nodes
|
6317
|
+
:code_gen_configuration_nodes,
|
6318
|
+
:execution_class)
|
6295
6319
|
SENSITIVE = [:code_gen_configuration_nodes]
|
6296
6320
|
include Aws::Structure
|
6297
6321
|
end
|
@@ -6367,7 +6391,7 @@ module Aws::Glue
|
|
6367
6391
|
# role: "RoleString", # required
|
6368
6392
|
# glue_version: "GlueVersionString",
|
6369
6393
|
# max_capacity: 1.0,
|
6370
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
6394
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
6371
6395
|
# number_of_workers: 1,
|
6372
6396
|
# timeout: 1,
|
6373
6397
|
# max_retries: 1,
|
@@ -7118,7 +7142,7 @@ module Aws::Glue
|
|
7118
7142
|
# },
|
7119
7143
|
# max_capacity: 1.0,
|
7120
7144
|
# number_of_workers: 1,
|
7121
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
7145
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
7122
7146
|
# security_configuration: "NameString",
|
7123
7147
|
# glue_version: "GlueVersionString",
|
7124
7148
|
# tags: {
|
@@ -7160,18 +7184,37 @@ module Aws::Glue
|
|
7160
7184
|
# @return [Types::ConnectionsList]
|
7161
7185
|
#
|
7162
7186
|
# @!attribute [rw] max_capacity
|
7163
|
-
# The number of
|
7187
|
+
# The number of Glue data processing units (DPUs) that can be
|
7164
7188
|
# allocated when the job runs. A DPU is a relative measure of
|
7165
7189
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
7166
7190
|
# GB memory.
|
7167
7191
|
# @return [Float]
|
7168
7192
|
#
|
7169
7193
|
# @!attribute [rw] number_of_workers
|
7170
|
-
# The number of workers to use for the
|
7194
|
+
# The number of workers of a defined `WorkerType` to use for the
|
7195
|
+
# session.
|
7171
7196
|
# @return [Integer]
|
7172
7197
|
#
|
7173
7198
|
# @!attribute [rw] worker_type
|
7174
|
-
# The
|
7199
|
+
# The type of predefined worker that is allocated to use for the
|
7200
|
+
# session. Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
7201
|
+
#
|
7202
|
+
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
7203
|
+
# of memory and a 50GB disk, and 2 executors per worker.
|
7204
|
+
#
|
7205
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
|
7206
|
+
# GB of memory, 64 GB disk), and provides 1 executor per worker. We
|
7207
|
+
# recommend this worker type for memory-intensive jobs.
|
7208
|
+
#
|
7209
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
7210
|
+
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
7211
|
+
# recommend this worker type for memory-intensive jobs.
|
7212
|
+
#
|
7213
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
7214
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
7215
|
+
# worker. We recommend this worker type for low volume streaming
|
7216
|
+
# jobs. This worker type is only available for Glue version 3.0
|
7217
|
+
# streaming jobs.
|
7175
7218
|
# @return [String]
|
7176
7219
|
#
|
7177
7220
|
# @!attribute [rw] security_configuration
|
@@ -7181,7 +7224,7 @@ module Aws::Glue
|
|
7181
7224
|
#
|
7182
7225
|
# @!attribute [rw] glue_version
|
7183
7226
|
# The Glue version determines the versions of Apache Spark and Python
|
7184
|
-
# that
|
7227
|
+
# that Glue supports. The GlueVersion must be greater than 2.0.
|
7185
7228
|
# @return [String]
|
7186
7229
|
#
|
7187
7230
|
# @!attribute [rw] tags
|
@@ -7376,9 +7419,9 @@ module Aws::Glue
|
|
7376
7419
|
# {
|
7377
7420
|
# logical_operator: "EQUALS", # accepts EQUALS
|
7378
7421
|
# job_name: "NameString",
|
7379
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
7422
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
7380
7423
|
# crawler_name: "NameString",
|
7381
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
7424
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
7382
7425
|
# },
|
7383
7426
|
# ],
|
7384
7427
|
# },
|
@@ -9724,8 +9767,8 @@ module Aws::Glue
|
|
9724
9767
|
include Aws::Structure
|
9725
9768
|
end
|
9726
9769
|
|
9727
|
-
# An edge represents a directed connection between two components
|
9728
|
-
# workflow
|
9770
|
+
# An edge represents a directed connection between two Glue components
|
9771
|
+
# that are part of the workflow the edge belongs to.
|
9729
9772
|
#
|
9730
9773
|
# @!attribute [rw] source_id
|
9731
9774
|
# The unique of the node within the workflow where the edge starts.
|
@@ -14128,7 +14171,10 @@ module Aws::Glue
|
|
14128
14171
|
include Aws::Structure
|
14129
14172
|
end
|
14130
14173
|
|
14174
|
+
# The blueprint is in an invalid state to perform a requested operation.
|
14175
|
+
#
|
14131
14176
|
# @!attribute [rw] message
|
14177
|
+
# A message describing the problem.
|
14132
14178
|
# @return [String]
|
14133
14179
|
#
|
14134
14180
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/IllegalBlueprintStateException AWS API Documentation
|
@@ -14612,7 +14658,7 @@ module Aws::Glue
|
|
14612
14658
|
# This field is deprecated. Use `MaxCapacity` instead.
|
14613
14659
|
#
|
14614
14660
|
# The number of Glue data processing units (DPUs) allocated to runs of
|
14615
|
-
# this job. You can allocate
|
14661
|
+
# this job. You can allocate a minimum of 2 DPUs; the default is 10. A
|
14616
14662
|
# DPU is a relative measure of processing power that consists of 4
|
14617
14663
|
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
14618
14664
|
# see the [Glue pricing page][1].
|
@@ -14650,8 +14696,8 @@ module Aws::Glue
|
|
14650
14696
|
#
|
14651
14697
|
# * When you specify an Apache Spark ETL job
|
14652
14698
|
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
|
14653
|
-
# (`JobCommand.Name`="gluestreaming"), you can allocate
|
14654
|
-
#
|
14699
|
+
# (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
|
14700
|
+
# of 2 DPUs. The default is 10 DPUs. This job type cannot have a
|
14655
14701
|
# fractional DPU allocation.
|
14656
14702
|
#
|
14657
14703
|
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
@@ -14665,7 +14711,7 @@ module Aws::Glue
|
|
14665
14711
|
#
|
14666
14712
|
# @!attribute [rw] worker_type
|
14667
14713
|
# The type of predefined worker that is allocated when a job runs.
|
14668
|
-
# Accepts a value of Standard, G.1X, or G.
|
14714
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
14669
14715
|
#
|
14670
14716
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
14671
14717
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -14677,14 +14723,17 @@ module Aws::Glue
|
|
14677
14723
|
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
14678
14724
|
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
14679
14725
|
# recommend this worker type for memory-intensive jobs.
|
14726
|
+
#
|
14727
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
14728
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
14729
|
+
# worker. We recommend this worker type for low volume streaming
|
14730
|
+
# jobs. This worker type is only available for Glue version 3.0
|
14731
|
+
# streaming jobs.
|
14680
14732
|
# @return [String]
|
14681
14733
|
#
|
14682
14734
|
# @!attribute [rw] number_of_workers
|
14683
14735
|
# The number of workers of a defined `workerType` that are allocated
|
14684
14736
|
# when a job runs.
|
14685
|
-
#
|
14686
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
14687
|
-
# 149 for `G.2X`.
|
14688
14737
|
# @return [Integer]
|
14689
14738
|
#
|
14690
14739
|
# @!attribute [rw] security_configuration
|
@@ -14719,6 +14768,20 @@ module Aws::Glue
|
|
14719
14768
|
# based.
|
14720
14769
|
# @return [Hash<String,Types::CodeGenConfigurationNode>]
|
14721
14770
|
#
|
14771
|
+
# @!attribute [rw] execution_class
|
14772
|
+
# Indicates whether the job is run with a standard or flexible
|
14773
|
+
# execution class. The standard execution class is ideal for
|
14774
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
14775
|
+
# resources.
|
14776
|
+
#
|
14777
|
+
# The flexible execution class is appropriate for time-insensitive
|
14778
|
+
# jobs whose start and completion times may vary.
|
14779
|
+
#
|
14780
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
14781
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
14782
|
+
# execution class is available for Spark jobs.
|
14783
|
+
# @return [String]
|
14784
|
+
#
|
14722
14785
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Job AWS API Documentation
|
14723
14786
|
#
|
14724
14787
|
class Job < Struct.new(
|
@@ -14742,7 +14805,8 @@ module Aws::Glue
|
|
14742
14805
|
:security_configuration,
|
14743
14806
|
:notification_property,
|
14744
14807
|
:glue_version,
|
14745
|
-
:code_gen_configuration_nodes
|
14808
|
+
:code_gen_configuration_nodes,
|
14809
|
+
:execution_class)
|
14746
14810
|
SENSITIVE = [:code_gen_configuration_nodes]
|
14747
14811
|
include Aws::Structure
|
14748
14812
|
end
|
@@ -14965,8 +15029,11 @@ module Aws::Glue
|
|
14965
15029
|
# @!attribute [rw] timeout
|
14966
15030
|
# The `JobRun` timeout in minutes. This is the maximum time that a job
|
14967
15031
|
# run can consume resources before it is terminated and enters
|
14968
|
-
# `TIMEOUT` status.
|
14969
|
-
#
|
15032
|
+
# `TIMEOUT` status. This value overrides the timeout value set in the
|
15033
|
+
# parent job.
|
15034
|
+
#
|
15035
|
+
# Streaming jobs do not have a timeout. The default for non-streaming
|
15036
|
+
# jobs is 2,880 minutes (48 hours).
|
14970
15037
|
# @return [Integer]
|
14971
15038
|
#
|
14972
15039
|
# @!attribute [rw] max_capacity
|
@@ -14986,7 +15053,7 @@ module Aws::Glue
|
|
14986
15053
|
# 0.0625 or 1 DPU. The default is 0.0625 DPU.
|
14987
15054
|
#
|
14988
15055
|
# * When you specify an Apache Spark ETL job
|
14989
|
-
# (`JobCommand.Name`="glueetl"), you can allocate
|
15056
|
+
# (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
|
14990
15057
|
# DPUs. The default is 10 DPUs. This job type cannot have a
|
14991
15058
|
# fractional DPU allocation.
|
14992
15059
|
#
|
@@ -14997,7 +15064,7 @@ module Aws::Glue
|
|
14997
15064
|
#
|
14998
15065
|
# @!attribute [rw] worker_type
|
14999
15066
|
# The type of predefined worker that is allocated when a job runs.
|
15000
|
-
# Accepts a value of Standard, G.1X, or G.
|
15067
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
15001
15068
|
#
|
15002
15069
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
15003
15070
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -15007,14 +15074,17 @@ module Aws::Glue
|
|
15007
15074
|
#
|
15008
15075
|
# * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
|
15009
15076
|
# memory and a 128GB disk, and 1 executor per worker.
|
15077
|
+
#
|
15078
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
15079
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
15080
|
+
# worker. We recommend this worker type for low volume streaming
|
15081
|
+
# jobs. This worker type is only available for Glue version 3.0
|
15082
|
+
# streaming jobs.
|
15010
15083
|
# @return [String]
|
15011
15084
|
#
|
15012
15085
|
# @!attribute [rw] number_of_workers
|
15013
15086
|
# The number of workers of a defined `workerType` that are allocated
|
15014
15087
|
# when a job runs.
|
15015
|
-
#
|
15016
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
15017
|
-
# 149 for `G.2X`.
|
15018
15088
|
# @return [Integer]
|
15019
15089
|
#
|
15020
15090
|
# @!attribute [rw] security_configuration
|
@@ -15054,10 +15124,10 @@ module Aws::Glue
|
|
15054
15124
|
# @return [String]
|
15055
15125
|
#
|
15056
15126
|
# @!attribute [rw] dpu_seconds
|
15057
|
-
# This field populates only
|
15058
|
-
#
|
15059
|
-
#
|
15060
|
-
#
|
15127
|
+
# This field populates only for Auto Scaling job runs, and represents
|
15128
|
+
# the total time each executor ran during the lifecycle of a job run
|
15129
|
+
# in seconds, multiplied by a DPU factor (1 for `G.1X`, 2 for `G.2X`,
|
15130
|
+
# or 0.25 for `G.025X` workers). This value may be different than the
|
15061
15131
|
# `executionEngineRuntime` * `MaxCapacity` as in the case of Auto
|
15062
15132
|
# Scaling jobs, as the number of executors running at a given time may
|
15063
15133
|
# be less than the `MaxCapacity`. Therefore, it is possible that the
|
@@ -15065,6 +15135,20 @@ module Aws::Glue
|
|
15065
15135
|
# `MaxCapacity`.
|
15066
15136
|
# @return [Float]
|
15067
15137
|
#
|
15138
|
+
# @!attribute [rw] execution_class
|
15139
|
+
# Indicates whether the job is run with a standard or flexible
|
15140
|
+
# execution class. The standard execution-class is ideal for
|
15141
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
15142
|
+
# resources.
|
15143
|
+
#
|
15144
|
+
# The flexible execution class is appropriate for time-insensitive
|
15145
|
+
# jobs whose start and completion times may vary.
|
15146
|
+
#
|
15147
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
15148
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
15149
|
+
# execution class is available for Spark jobs.
|
15150
|
+
# @return [String]
|
15151
|
+
#
|
15068
15152
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JobRun AWS API Documentation
|
15069
15153
|
#
|
15070
15154
|
class JobRun < Struct.new(
|
@@ -15090,7 +15174,8 @@ module Aws::Glue
|
|
15090
15174
|
:log_group_name,
|
15091
15175
|
:notification_property,
|
15092
15176
|
:glue_version,
|
15093
|
-
:dpu_seconds
|
15177
|
+
:dpu_seconds,
|
15178
|
+
:execution_class)
|
15094
15179
|
SENSITIVE = []
|
15095
15180
|
include Aws::Structure
|
15096
15181
|
end
|
@@ -15126,7 +15211,7 @@ module Aws::Glue
|
|
15126
15211
|
# allocated_capacity: 1,
|
15127
15212
|
# timeout: 1,
|
15128
15213
|
# max_capacity: 1.0,
|
15129
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
15214
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
15130
15215
|
# number_of_workers: 1,
|
15131
15216
|
# security_configuration: "NameString",
|
15132
15217
|
# notification_property: {
|
@@ -15801,6 +15886,7 @@ module Aws::Glue
|
|
15801
15886
|
# },
|
15802
15887
|
# },
|
15803
15888
|
# },
|
15889
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
15804
15890
|
# }
|
15805
15891
|
#
|
15806
15892
|
# @!attribute [rw] description
|
@@ -15862,7 +15948,7 @@ module Aws::Glue
|
|
15862
15948
|
# This field is deprecated. Use `MaxCapacity` instead.
|
15863
15949
|
#
|
15864
15950
|
# The number of Glue data processing units (DPUs) to allocate to this
|
15865
|
-
# job. You can allocate
|
15951
|
+
# job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU
|
15866
15952
|
# is a relative measure of processing power that consists of 4 vCPUs
|
15867
15953
|
# of compute capacity and 16 GB of memory. For more information, see
|
15868
15954
|
# the [Glue pricing page][1].
|
@@ -15897,8 +15983,8 @@ module Aws::Glue
|
|
15897
15983
|
#
|
15898
15984
|
# * When you specify an Apache Spark ETL job
|
15899
15985
|
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
|
15900
|
-
# (`JobCommand.Name`="gluestreaming"), you can allocate
|
15901
|
-
#
|
15986
|
+
# (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
|
15987
|
+
# of 2 DPUs. The default is 10 DPUs. This job type cannot have a
|
15902
15988
|
# fractional DPU allocation.
|
15903
15989
|
#
|
15904
15990
|
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
@@ -15912,7 +15998,7 @@ module Aws::Glue
|
|
15912
15998
|
#
|
15913
15999
|
# @!attribute [rw] worker_type
|
15914
16000
|
# The type of predefined worker that is allocated when a job runs.
|
15915
|
-
# Accepts a value of Standard, G.1X, or G.
|
16001
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
15916
16002
|
#
|
15917
16003
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
15918
16004
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -15924,14 +16010,17 @@ module Aws::Glue
|
|
15924
16010
|
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
15925
16011
|
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
15926
16012
|
# recommend this worker type for memory-intensive jobs.
|
16013
|
+
#
|
16014
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
16015
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
16016
|
+
# worker. We recommend this worker type for low volume streaming
|
16017
|
+
# jobs. This worker type is only available for Glue version 3.0
|
16018
|
+
# streaming jobs.
|
15927
16019
|
# @return [String]
|
15928
16020
|
#
|
15929
16021
|
# @!attribute [rw] number_of_workers
|
15930
16022
|
# The number of workers of a defined `workerType` that are allocated
|
15931
16023
|
# when a job runs.
|
15932
|
-
#
|
15933
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
15934
|
-
# 149 for `G.2X`.
|
15935
16024
|
# @return [Integer]
|
15936
16025
|
#
|
15937
16026
|
# @!attribute [rw] security_configuration
|
@@ -15963,6 +16052,20 @@ module Aws::Glue
|
|
15963
16052
|
# based.
|
15964
16053
|
# @return [Hash<String,Types::CodeGenConfigurationNode>]
|
15965
16054
|
#
|
16055
|
+
# @!attribute [rw] execution_class
|
16056
|
+
# Indicates whether the job is run with a standard or flexible
|
16057
|
+
# execution class. The standard execution-class is ideal for
|
16058
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
16059
|
+
# resources.
|
16060
|
+
#
|
16061
|
+
# The flexible execution class is appropriate for time-insensitive
|
16062
|
+
# jobs whose start and completion times may vary.
|
16063
|
+
#
|
16064
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
16065
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
16066
|
+
# execution class is available for Spark jobs.
|
16067
|
+
# @return [String]
|
16068
|
+
#
|
15966
16069
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JobUpdate AWS API Documentation
|
15967
16070
|
#
|
15968
16071
|
class JobUpdate < Struct.new(
|
@@ -15983,7 +16086,8 @@ module Aws::Glue
|
|
15983
16086
|
:security_configuration,
|
15984
16087
|
:notification_property,
|
15985
16088
|
:glue_version,
|
15986
|
-
:code_gen_configuration_nodes
|
16089
|
+
:code_gen_configuration_nodes,
|
16090
|
+
:execution_class)
|
15987
16091
|
SENSITIVE = [:code_gen_configuration_nodes]
|
15988
16092
|
include Aws::Structure
|
15989
16093
|
end
|
@@ -17144,7 +17248,7 @@ module Aws::Glue
|
|
17144
17248
|
end
|
17145
17249
|
|
17146
17250
|
# @!attribute [rw] ids
|
17147
|
-
# Returns the
|
17251
|
+
# Returns the ID of the session.
|
17148
17252
|
# @return [Array<String>]
|
17149
17253
|
#
|
17150
17254
|
# @!attribute [rw] sessions
|
@@ -17184,6 +17288,7 @@ module Aws::Glue
|
|
17184
17288
|
# @return [String]
|
17185
17289
|
#
|
17186
17290
|
# @!attribute [rw] next_token
|
17291
|
+
# A continuation token, if this is a continuation call.
|
17187
17292
|
# @return [String]
|
17188
17293
|
#
|
17189
17294
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ListStatementsRequest AWS API Documentation
|
@@ -17201,6 +17306,7 @@ module Aws::Glue
|
|
17201
17306
|
# @return [Array<Types::Statement>]
|
17202
17307
|
#
|
17203
17308
|
# @!attribute [rw] next_token
|
17309
|
+
# A continuation token, if not all statements have yet been returned.
|
17204
17310
|
# @return [String]
|
17205
17311
|
#
|
17206
17312
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ListStatementsResponse AWS API Documentation
|
@@ -18832,9 +18938,9 @@ module Aws::Glue
|
|
18832
18938
|
# {
|
18833
18939
|
# logical_operator: "EQUALS", # accepts EQUALS
|
18834
18940
|
# job_name: "NameString",
|
18835
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
18941
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
18836
18942
|
# crawler_name: "NameString",
|
18837
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
18943
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
18838
18944
|
# },
|
18839
18945
|
# ],
|
18840
18946
|
# }
|
@@ -21332,7 +21438,7 @@ module Aws::Glue
|
|
21332
21438
|
# @return [Float]
|
21333
21439
|
#
|
21334
21440
|
# @!attribute [rw] max_capacity
|
21335
|
-
# The number of
|
21441
|
+
# The number of Glue data processing units (DPUs) that can be
|
21336
21442
|
# allocated when the job runs. A DPU is a relative measure of
|
21337
21443
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
21338
21444
|
# GB memory.
|
@@ -21345,7 +21451,7 @@ module Aws::Glue
|
|
21345
21451
|
#
|
21346
21452
|
# @!attribute [rw] glue_version
|
21347
21453
|
# The Glue version determines the versions of Apache Spark and Python
|
21348
|
-
# that
|
21454
|
+
# that Glue supports. The GlueVersion must be greater than 2.0.
|
21349
21455
|
# @return [String]
|
21350
21456
|
#
|
21351
21457
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Session AWS API Documentation
|
@@ -21379,7 +21485,7 @@ module Aws::Glue
|
|
21379
21485
|
# }
|
21380
21486
|
#
|
21381
21487
|
# @!attribute [rw] name
|
21382
|
-
# Specifies the name of the SessionCommand.Can be 'glueetl' or
|
21488
|
+
# Specifies the name of the SessionCommand. Can be 'glueetl' or
|
21383
21489
|
# 'gluestreaming'.
|
21384
21490
|
# @return [String]
|
21385
21491
|
#
|
@@ -21965,8 +22071,9 @@ module Aws::Glue
|
|
21965
22071
|
# notification_property: {
|
21966
22072
|
# notify_delay_after: 1,
|
21967
22073
|
# },
|
21968
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
22074
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
21969
22075
|
# number_of_workers: 1,
|
22076
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
21970
22077
|
# }
|
21971
22078
|
#
|
21972
22079
|
# @!attribute [rw] job_name
|
@@ -21984,6 +22091,11 @@ module Aws::Glue
|
|
21984
22091
|
# You can specify arguments here that your own job-execution script
|
21985
22092
|
# consumes, as well as arguments that Glue itself consumes.
|
21986
22093
|
#
|
22094
|
+
# Job arguments may be logged. Do not pass plaintext secrets as
|
22095
|
+
# arguments. Retrieve secrets from a Glue Connection, Secrets Manager
|
22096
|
+
# or other secret management mechanism if you intend to keep them
|
22097
|
+
# within the Job.
|
22098
|
+
#
|
21987
22099
|
# For information about how to specify and consume your own Job
|
21988
22100
|
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
21989
22101
|
# developer guide.
|
@@ -22002,7 +22114,7 @@ module Aws::Glue
|
|
22002
22114
|
# This field is deprecated. Use `MaxCapacity` instead.
|
22003
22115
|
#
|
22004
22116
|
# The number of Glue data processing units (DPUs) to allocate to this
|
22005
|
-
# JobRun.
|
22117
|
+
# JobRun. You can allocate a minimum of 2 DPUs; the default is 10. A
|
22006
22118
|
# DPU is a relative measure of processing power that consists of 4
|
22007
22119
|
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
22008
22120
|
# see the [Glue pricing page][1].
|
@@ -22015,8 +22127,11 @@ module Aws::Glue
|
|
22015
22127
|
# @!attribute [rw] timeout
|
22016
22128
|
# The `JobRun` timeout in minutes. This is the maximum time that a job
|
22017
22129
|
# run can consume resources before it is terminated and enters
|
22018
|
-
# `TIMEOUT` status.
|
22019
|
-
#
|
22130
|
+
# `TIMEOUT` status. This value overrides the timeout value set in the
|
22131
|
+
# parent job.
|
22132
|
+
#
|
22133
|
+
# Streaming jobs do not have a timeout. The default for non-streaming
|
22134
|
+
# jobs is 2,880 minutes (48 hours).
|
22020
22135
|
# @return [Integer]
|
22021
22136
|
#
|
22022
22137
|
# @!attribute [rw] max_capacity
|
@@ -22036,7 +22151,7 @@ module Aws::Glue
|
|
22036
22151
|
# 0.0625 or 1 DPU. The default is 0.0625 DPU.
|
22037
22152
|
#
|
22038
22153
|
# * When you specify an Apache Spark ETL job
|
22039
|
-
# (`JobCommand.Name`="glueetl"), you can allocate
|
22154
|
+
# (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
|
22040
22155
|
# DPUs. The default is 10 DPUs. This job type cannot have a
|
22041
22156
|
# fractional DPU allocation.
|
22042
22157
|
#
|
@@ -22056,7 +22171,7 @@ module Aws::Glue
|
|
22056
22171
|
#
|
22057
22172
|
# @!attribute [rw] worker_type
|
22058
22173
|
# The type of predefined worker that is allocated when a job runs.
|
22059
|
-
# Accepts a value of Standard, G.1X, or G.
|
22174
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
22060
22175
|
#
|
22061
22176
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
22062
22177
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -22066,16 +22181,33 @@ module Aws::Glue
|
|
22066
22181
|
#
|
22067
22182
|
# * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
|
22068
22183
|
# memory and a 128GB disk, and 1 executor per worker.
|
22184
|
+
#
|
22185
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
22186
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
22187
|
+
# worker. We recommend this worker type for low volume streaming
|
22188
|
+
# jobs. This worker type is only available for Glue version 3.0
|
22189
|
+
# streaming jobs.
|
22069
22190
|
# @return [String]
|
22070
22191
|
#
|
22071
22192
|
# @!attribute [rw] number_of_workers
|
22072
22193
|
# The number of workers of a defined `workerType` that are allocated
|
22073
22194
|
# when a job runs.
|
22074
|
-
#
|
22075
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
22076
|
-
# 149 for `G.2X`.
|
22077
22195
|
# @return [Integer]
|
22078
22196
|
#
|
22197
|
+
# @!attribute [rw] execution_class
|
22198
|
+
# Indicates whether the job is run with a standard or flexible
|
22199
|
+
# execution class. The standard execution-class is ideal for
|
22200
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
22201
|
+
# resources.
|
22202
|
+
#
|
22203
|
+
# The flexible execution class is appropriate for time-insensitive
|
22204
|
+
# jobs whose start and completion times may vary.
|
22205
|
+
#
|
22206
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
22207
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
22208
|
+
# execution class is available for Spark jobs.
|
22209
|
+
# @return [String]
|
22210
|
+
#
|
22079
22211
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StartJobRunRequest AWS API Documentation
|
22080
22212
|
#
|
22081
22213
|
class StartJobRunRequest < Struct.new(
|
@@ -22088,7 +22220,8 @@ module Aws::Glue
|
|
22088
22220
|
:security_configuration,
|
22089
22221
|
:notification_property,
|
22090
22222
|
:worker_type,
|
22091
|
-
:number_of_workers
|
22223
|
+
:number_of_workers,
|
22224
|
+
:execution_class)
|
22092
22225
|
SENSITIVE = []
|
22093
22226
|
include Aws::Structure
|
22094
22227
|
end
|
@@ -23626,9 +23759,9 @@ module Aws::Glue
|
|
23626
23759
|
# {
|
23627
23760
|
# logical_operator: "EQUALS", # accepts EQUALS
|
23628
23761
|
# job_name: "NameString",
|
23629
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
23762
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
23630
23763
|
# crawler_name: "NameString",
|
23631
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
23764
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
23632
23765
|
# },
|
23633
23766
|
# ],
|
23634
23767
|
# },
|
@@ -24630,7 +24763,7 @@ module Aws::Glue
|
|
24630
24763
|
# allocated_capacity: 1,
|
24631
24764
|
# timeout: 1,
|
24632
24765
|
# max_capacity: 1.0,
|
24633
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
24766
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
24634
24767
|
# number_of_workers: 1,
|
24635
24768
|
# security_configuration: "NameString",
|
24636
24769
|
# notification_property: {
|
@@ -25305,6 +25438,7 @@ module Aws::Glue
|
|
25305
25438
|
# },
|
25306
25439
|
# },
|
25307
25440
|
# },
|
25441
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
25308
25442
|
# },
|
25309
25443
|
# }
|
25310
25444
|
#
|
@@ -25314,6 +25448,7 @@ module Aws::Glue
|
|
25314
25448
|
#
|
25315
25449
|
# @!attribute [rw] job_update
|
25316
25450
|
# Specifies the values with which to update the job definition.
|
25451
|
+
# Unspecified configuration is removed or reset to default values.
|
25317
25452
|
# @return [Types::JobUpdate]
|
25318
25453
|
#
|
25319
25454
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateJobRequest AWS API Documentation
|
@@ -25389,7 +25524,7 @@ module Aws::Glue
|
|
25389
25524
|
# role: "RoleString",
|
25390
25525
|
# glue_version: "GlueVersionString",
|
25391
25526
|
# max_capacity: 1.0,
|
25392
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
25527
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
25393
25528
|
# number_of_workers: 1,
|
25394
25529
|
# timeout: 1,
|
25395
25530
|
# max_retries: 1,
|
@@ -25908,9 +26043,9 @@ module Aws::Glue
|
|
25908
26043
|
# {
|
25909
26044
|
# logical_operator: "EQUALS", # accepts EQUALS
|
25910
26045
|
# job_name: "NameString",
|
25911
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
26046
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
25912
26047
|
# crawler_name: "NameString",
|
25913
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
26048
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
25914
26049
|
# },
|
25915
26050
|
# ],
|
25916
26051
|
# },
|
@@ -26444,6 +26579,16 @@ module Aws::Glue
|
|
26444
26579
|
# Total number Actions in running state.
|
26445
26580
|
# @return [Integer]
|
26446
26581
|
#
|
26582
|
+
# @!attribute [rw] errored_actions
|
26583
|
+
# Indicates the count of job runs in the ERROR state in the workflow
|
26584
|
+
# run.
|
26585
|
+
# @return [Integer]
|
26586
|
+
#
|
26587
|
+
# @!attribute [rw] waiting_actions
|
26588
|
+
# Indicates the count of job runs in WAITING state in the workflow
|
26589
|
+
# run.
|
26590
|
+
# @return [Integer]
|
26591
|
+
#
|
26447
26592
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/WorkflowRunStatistics AWS API Documentation
|
26448
26593
|
#
|
26449
26594
|
class WorkflowRunStatistics < Struct.new(
|
@@ -26452,7 +26597,9 @@ module Aws::Glue
|
|
26452
26597
|
:failed_actions,
|
26453
26598
|
:stopped_actions,
|
26454
26599
|
:succeeded_actions,
|
26455
|
-
:running_actions
|
26600
|
+
:running_actions,
|
26601
|
+
:errored_actions,
|
26602
|
+
:waiting_actions)
|
26456
26603
|
SENSITIVE = []
|
26457
26604
|
include Aws::Structure
|
26458
26605
|
end
|