aws-sdk-glue 1.114.0 → 1.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-glue/client.rb +169 -86
- data/lib/aws-sdk-glue/client_api.rb +8 -0
- data/lib/aws-sdk-glue/types.rb +221 -74
- data/lib/aws-sdk-glue.rb +1 -1
- metadata +2 -2
data/lib/aws-sdk-glue/types.rb
CHANGED
@@ -3558,9 +3558,9 @@ module Aws::Glue
|
|
3558
3558
|
# {
|
3559
3559
|
# logical_operator: "EQUALS", # accepts EQUALS
|
3560
3560
|
# job_name: "NameString",
|
3561
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
3561
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
3562
3562
|
# crawler_name: "NameString",
|
3563
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
3563
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
3564
3564
|
# }
|
3565
3565
|
#
|
3566
3566
|
# @!attribute [rw] logical_operator
|
@@ -5042,7 +5042,7 @@ module Aws::Glue
|
|
5042
5042
|
# public_key: "GenericString",
|
5043
5043
|
# public_keys: ["GenericString"],
|
5044
5044
|
# number_of_nodes: 1,
|
5045
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
5045
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
5046
5046
|
# glue_version: "GlueVersionString",
|
5047
5047
|
# number_of_workers: 1,
|
5048
5048
|
# extra_python_libs_s3_path: "GenericString",
|
@@ -5420,7 +5420,7 @@ module Aws::Glue
|
|
5420
5420
|
# },
|
5421
5421
|
# glue_version: "GlueVersionString",
|
5422
5422
|
# number_of_workers: 1,
|
5423
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
5423
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
5424
5424
|
# code_gen_configuration_nodes: {
|
5425
5425
|
# "NodeId" => {
|
5426
5426
|
# athena_connector_source: {
|
@@ -6089,6 +6089,7 @@ module Aws::Glue
|
|
6089
6089
|
# },
|
6090
6090
|
# },
|
6091
6091
|
# },
|
6092
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
6092
6093
|
# }
|
6093
6094
|
#
|
6094
6095
|
# @!attribute [rw] name
|
@@ -6124,6 +6125,11 @@ module Aws::Glue
|
|
6124
6125
|
# You can specify arguments here that your own job-execution script
|
6125
6126
|
# consumes, as well as arguments that Glue itself consumes.
|
6126
6127
|
#
|
6128
|
+
# Job arguments may be logged. Do not pass plaintext secrets as
|
6129
|
+
# arguments. Retrieve secrets from a Glue Connection, Secrets Manager
|
6130
|
+
# or other secret management mechanism if you intend to keep them
|
6131
|
+
# within the Job.
|
6132
|
+
#
|
6127
6133
|
# For information about how to specify and consume your own Job
|
6128
6134
|
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
6129
6135
|
# developer guide.
|
@@ -6155,7 +6161,7 @@ module Aws::Glue
|
|
6155
6161
|
# This parameter is deprecated. Use `MaxCapacity` instead.
|
6156
6162
|
#
|
6157
6163
|
# The number of Glue data processing units (DPUs) to allocate to this
|
6158
|
-
# Job. You can allocate
|
6164
|
+
# Job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU
|
6159
6165
|
# is a relative measure of processing power that consists of 4 vCPUs
|
6160
6166
|
# of compute capacity and 16 GB of memory. For more information, see
|
6161
6167
|
# the [Glue pricing page][1].
|
@@ -6190,8 +6196,8 @@ module Aws::Glue
|
|
6190
6196
|
#
|
6191
6197
|
# * When you specify an Apache Spark ETL job
|
6192
6198
|
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
|
6193
|
-
# (`JobCommand.Name`="gluestreaming"), you can allocate
|
6194
|
-
#
|
6199
|
+
# (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
|
6200
|
+
# of 2 DPUs. The default is 10 DPUs. This job type cannot have a
|
6195
6201
|
# fractional DPU allocation.
|
6196
6202
|
#
|
6197
6203
|
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
@@ -6242,14 +6248,11 @@ module Aws::Glue
|
|
6242
6248
|
# @!attribute [rw] number_of_workers
|
6243
6249
|
# The number of workers of a defined `workerType` that are allocated
|
6244
6250
|
# when a job runs.
|
6245
|
-
#
|
6246
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
6247
|
-
# 149 for `G.2X`.
|
6248
6251
|
# @return [Integer]
|
6249
6252
|
#
|
6250
6253
|
# @!attribute [rw] worker_type
|
6251
6254
|
# The type of predefined worker that is allocated when a job runs.
|
6252
|
-
# Accepts a value of Standard, G.1X, or G.
|
6255
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
6253
6256
|
#
|
6254
6257
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
6255
6258
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -6261,6 +6264,12 @@ module Aws::Glue
|
|
6261
6264
|
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
6262
6265
|
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
6263
6266
|
# recommend this worker type for memory-intensive jobs.
|
6267
|
+
#
|
6268
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
6269
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
6270
|
+
# worker. We recommend this worker type for low volume streaming
|
6271
|
+
# jobs. This worker type is only available for Glue version 3.0
|
6272
|
+
# streaming jobs.
|
6264
6273
|
# @return [String]
|
6265
6274
|
#
|
6266
6275
|
# @!attribute [rw] code_gen_configuration_nodes
|
@@ -6269,6 +6278,20 @@ module Aws::Glue
|
|
6269
6278
|
# based.
|
6270
6279
|
# @return [Hash<String,Types::CodeGenConfigurationNode>]
|
6271
6280
|
#
|
6281
|
+
# @!attribute [rw] execution_class
|
6282
|
+
# Indicates whether the job is run with a standard or flexible
|
6283
|
+
# execution class. The standard execution-class is ideal for
|
6284
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
6285
|
+
# resources.
|
6286
|
+
#
|
6287
|
+
# The flexible execution class is appropriate for time-insensitive
|
6288
|
+
# jobs whose start and completion times may vary.
|
6289
|
+
#
|
6290
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
6291
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
6292
|
+
# execution class is available for Spark jobs.
|
6293
|
+
# @return [String]
|
6294
|
+
#
|
6272
6295
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateJobRequest AWS API Documentation
|
6273
6296
|
#
|
6274
6297
|
class CreateJobRequest < Struct.new(
|
@@ -6291,7 +6314,8 @@ module Aws::Glue
|
|
6291
6314
|
:glue_version,
|
6292
6315
|
:number_of_workers,
|
6293
6316
|
:worker_type,
|
6294
|
-
:code_gen_configuration_nodes
|
6317
|
+
:code_gen_configuration_nodes,
|
6318
|
+
:execution_class)
|
6295
6319
|
SENSITIVE = [:code_gen_configuration_nodes]
|
6296
6320
|
include Aws::Structure
|
6297
6321
|
end
|
@@ -6367,7 +6391,7 @@ module Aws::Glue
|
|
6367
6391
|
# role: "RoleString", # required
|
6368
6392
|
# glue_version: "GlueVersionString",
|
6369
6393
|
# max_capacity: 1.0,
|
6370
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
6394
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
6371
6395
|
# number_of_workers: 1,
|
6372
6396
|
# timeout: 1,
|
6373
6397
|
# max_retries: 1,
|
@@ -7118,7 +7142,7 @@ module Aws::Glue
|
|
7118
7142
|
# },
|
7119
7143
|
# max_capacity: 1.0,
|
7120
7144
|
# number_of_workers: 1,
|
7121
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
7145
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
7122
7146
|
# security_configuration: "NameString",
|
7123
7147
|
# glue_version: "GlueVersionString",
|
7124
7148
|
# tags: {
|
@@ -7160,18 +7184,37 @@ module Aws::Glue
|
|
7160
7184
|
# @return [Types::ConnectionsList]
|
7161
7185
|
#
|
7162
7186
|
# @!attribute [rw] max_capacity
|
7163
|
-
# The number of
|
7187
|
+
# The number of Glue data processing units (DPUs) that can be
|
7164
7188
|
# allocated when the job runs. A DPU is a relative measure of
|
7165
7189
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
7166
7190
|
# GB memory.
|
7167
7191
|
# @return [Float]
|
7168
7192
|
#
|
7169
7193
|
# @!attribute [rw] number_of_workers
|
7170
|
-
# The number of workers to use for the
|
7194
|
+
# The number of workers of a defined `WorkerType` to use for the
|
7195
|
+
# session.
|
7171
7196
|
# @return [Integer]
|
7172
7197
|
#
|
7173
7198
|
# @!attribute [rw] worker_type
|
7174
|
-
# The
|
7199
|
+
# The type of predefined worker that is allocated to use for the
|
7200
|
+
# session. Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
7201
|
+
#
|
7202
|
+
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
7203
|
+
# of memory and a 50GB disk, and 2 executors per worker.
|
7204
|
+
#
|
7205
|
+
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
|
7206
|
+
# GB of memory, 64 GB disk), and provides 1 executor per worker. We
|
7207
|
+
# recommend this worker type for memory-intensive jobs.
|
7208
|
+
#
|
7209
|
+
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
7210
|
+
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
7211
|
+
# recommend this worker type for memory-intensive jobs.
|
7212
|
+
#
|
7213
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
7214
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
7215
|
+
# worker. We recommend this worker type for low volume streaming
|
7216
|
+
# jobs. This worker type is only available for Glue version 3.0
|
7217
|
+
# streaming jobs.
|
7175
7218
|
# @return [String]
|
7176
7219
|
#
|
7177
7220
|
# @!attribute [rw] security_configuration
|
@@ -7181,7 +7224,7 @@ module Aws::Glue
|
|
7181
7224
|
#
|
7182
7225
|
# @!attribute [rw] glue_version
|
7183
7226
|
# The Glue version determines the versions of Apache Spark and Python
|
7184
|
-
# that
|
7227
|
+
# that Glue supports. The GlueVersion must be greater than 2.0.
|
7185
7228
|
# @return [String]
|
7186
7229
|
#
|
7187
7230
|
# @!attribute [rw] tags
|
@@ -7376,9 +7419,9 @@ module Aws::Glue
|
|
7376
7419
|
# {
|
7377
7420
|
# logical_operator: "EQUALS", # accepts EQUALS
|
7378
7421
|
# job_name: "NameString",
|
7379
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
7422
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
7380
7423
|
# crawler_name: "NameString",
|
7381
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
7424
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
7382
7425
|
# },
|
7383
7426
|
# ],
|
7384
7427
|
# },
|
@@ -9724,8 +9767,8 @@ module Aws::Glue
|
|
9724
9767
|
include Aws::Structure
|
9725
9768
|
end
|
9726
9769
|
|
9727
|
-
# An edge represents a directed connection between two components
|
9728
|
-
# workflow
|
9770
|
+
# An edge represents a directed connection between two Glue components
|
9771
|
+
# that are part of the workflow the edge belongs to.
|
9729
9772
|
#
|
9730
9773
|
# @!attribute [rw] source_id
|
9731
9774
|
# The unique of the node within the workflow where the edge starts.
|
@@ -14128,7 +14171,10 @@ module Aws::Glue
|
|
14128
14171
|
include Aws::Structure
|
14129
14172
|
end
|
14130
14173
|
|
14174
|
+
# The blueprint is in an invalid state to perform a requested operation.
|
14175
|
+
#
|
14131
14176
|
# @!attribute [rw] message
|
14177
|
+
# A message describing the problem.
|
14132
14178
|
# @return [String]
|
14133
14179
|
#
|
14134
14180
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/IllegalBlueprintStateException AWS API Documentation
|
@@ -14612,7 +14658,7 @@ module Aws::Glue
|
|
14612
14658
|
# This field is deprecated. Use `MaxCapacity` instead.
|
14613
14659
|
#
|
14614
14660
|
# The number of Glue data processing units (DPUs) allocated to runs of
|
14615
|
-
# this job. You can allocate
|
14661
|
+
# this job. You can allocate a minimum of 2 DPUs; the default is 10. A
|
14616
14662
|
# DPU is a relative measure of processing power that consists of 4
|
14617
14663
|
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
14618
14664
|
# see the [Glue pricing page][1].
|
@@ -14650,8 +14696,8 @@ module Aws::Glue
|
|
14650
14696
|
#
|
14651
14697
|
# * When you specify an Apache Spark ETL job
|
14652
14698
|
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
|
14653
|
-
# (`JobCommand.Name`="gluestreaming"), you can allocate
|
14654
|
-
#
|
14699
|
+
# (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
|
14700
|
+
# of 2 DPUs. The default is 10 DPUs. This job type cannot have a
|
14655
14701
|
# fractional DPU allocation.
|
14656
14702
|
#
|
14657
14703
|
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
@@ -14665,7 +14711,7 @@ module Aws::Glue
|
|
14665
14711
|
#
|
14666
14712
|
# @!attribute [rw] worker_type
|
14667
14713
|
# The type of predefined worker that is allocated when a job runs.
|
14668
|
-
# Accepts a value of Standard, G.1X, or G.
|
14714
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
14669
14715
|
#
|
14670
14716
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
14671
14717
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -14677,14 +14723,17 @@ module Aws::Glue
|
|
14677
14723
|
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
14678
14724
|
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
14679
14725
|
# recommend this worker type for memory-intensive jobs.
|
14726
|
+
#
|
14727
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
14728
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
14729
|
+
# worker. We recommend this worker type for low volume streaming
|
14730
|
+
# jobs. This worker type is only available for Glue version 3.0
|
14731
|
+
# streaming jobs.
|
14680
14732
|
# @return [String]
|
14681
14733
|
#
|
14682
14734
|
# @!attribute [rw] number_of_workers
|
14683
14735
|
# The number of workers of a defined `workerType` that are allocated
|
14684
14736
|
# when a job runs.
|
14685
|
-
#
|
14686
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
14687
|
-
# 149 for `G.2X`.
|
14688
14737
|
# @return [Integer]
|
14689
14738
|
#
|
14690
14739
|
# @!attribute [rw] security_configuration
|
@@ -14719,6 +14768,20 @@ module Aws::Glue
|
|
14719
14768
|
# based.
|
14720
14769
|
# @return [Hash<String,Types::CodeGenConfigurationNode>]
|
14721
14770
|
#
|
14771
|
+
# @!attribute [rw] execution_class
|
14772
|
+
# Indicates whether the job is run with a standard or flexible
|
14773
|
+
# execution class. The standard execution class is ideal for
|
14774
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
14775
|
+
# resources.
|
14776
|
+
#
|
14777
|
+
# The flexible execution class is appropriate for time-insensitive
|
14778
|
+
# jobs whose start and completion times may vary.
|
14779
|
+
#
|
14780
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
14781
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
14782
|
+
# execution class is available for Spark jobs.
|
14783
|
+
# @return [String]
|
14784
|
+
#
|
14722
14785
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Job AWS API Documentation
|
14723
14786
|
#
|
14724
14787
|
class Job < Struct.new(
|
@@ -14742,7 +14805,8 @@ module Aws::Glue
|
|
14742
14805
|
:security_configuration,
|
14743
14806
|
:notification_property,
|
14744
14807
|
:glue_version,
|
14745
|
-
:code_gen_configuration_nodes
|
14808
|
+
:code_gen_configuration_nodes,
|
14809
|
+
:execution_class)
|
14746
14810
|
SENSITIVE = [:code_gen_configuration_nodes]
|
14747
14811
|
include Aws::Structure
|
14748
14812
|
end
|
@@ -14965,8 +15029,11 @@ module Aws::Glue
|
|
14965
15029
|
# @!attribute [rw] timeout
|
14966
15030
|
# The `JobRun` timeout in minutes. This is the maximum time that a job
|
14967
15031
|
# run can consume resources before it is terminated and enters
|
14968
|
-
# `TIMEOUT` status.
|
14969
|
-
#
|
15032
|
+
# `TIMEOUT` status. This value overrides the timeout value set in the
|
15033
|
+
# parent job.
|
15034
|
+
#
|
15035
|
+
# Streaming jobs do not have a timeout. The default for non-streaming
|
15036
|
+
# jobs is 2,880 minutes (48 hours).
|
14970
15037
|
# @return [Integer]
|
14971
15038
|
#
|
14972
15039
|
# @!attribute [rw] max_capacity
|
@@ -14986,7 +15053,7 @@ module Aws::Glue
|
|
14986
15053
|
# 0.0625 or 1 DPU. The default is 0.0625 DPU.
|
14987
15054
|
#
|
14988
15055
|
# * When you specify an Apache Spark ETL job
|
14989
|
-
# (`JobCommand.Name`="glueetl"), you can allocate
|
15056
|
+
# (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
|
14990
15057
|
# DPUs. The default is 10 DPUs. This job type cannot have a
|
14991
15058
|
# fractional DPU allocation.
|
14992
15059
|
#
|
@@ -14997,7 +15064,7 @@ module Aws::Glue
|
|
14997
15064
|
#
|
14998
15065
|
# @!attribute [rw] worker_type
|
14999
15066
|
# The type of predefined worker that is allocated when a job runs.
|
15000
|
-
# Accepts a value of Standard, G.1X, or G.
|
15067
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
15001
15068
|
#
|
15002
15069
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
15003
15070
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -15007,14 +15074,17 @@ module Aws::Glue
|
|
15007
15074
|
#
|
15008
15075
|
# * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
|
15009
15076
|
# memory and a 128GB disk, and 1 executor per worker.
|
15077
|
+
#
|
15078
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
15079
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
15080
|
+
# worker. We recommend this worker type for low volume streaming
|
15081
|
+
# jobs. This worker type is only available for Glue version 3.0
|
15082
|
+
# streaming jobs.
|
15010
15083
|
# @return [String]
|
15011
15084
|
#
|
15012
15085
|
# @!attribute [rw] number_of_workers
|
15013
15086
|
# The number of workers of a defined `workerType` that are allocated
|
15014
15087
|
# when a job runs.
|
15015
|
-
#
|
15016
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
15017
|
-
# 149 for `G.2X`.
|
15018
15088
|
# @return [Integer]
|
15019
15089
|
#
|
15020
15090
|
# @!attribute [rw] security_configuration
|
@@ -15054,10 +15124,10 @@ module Aws::Glue
|
|
15054
15124
|
# @return [String]
|
15055
15125
|
#
|
15056
15126
|
# @!attribute [rw] dpu_seconds
|
15057
|
-
# This field populates only
|
15058
|
-
#
|
15059
|
-
#
|
15060
|
-
#
|
15127
|
+
# This field populates only for Auto Scaling job runs, and represents
|
15128
|
+
# the total time each executor ran during the lifecycle of a job run
|
15129
|
+
# in seconds, multiplied by a DPU factor (1 for `G.1X`, 2 for `G.2X`,
|
15130
|
+
# or 0.25 for `G.025X` workers). This value may be different than the
|
15061
15131
|
# `executionEngineRuntime` * `MaxCapacity` as in the case of Auto
|
15062
15132
|
# Scaling jobs, as the number of executors running at a given time may
|
15063
15133
|
# be less than the `MaxCapacity`. Therefore, it is possible that the
|
@@ -15065,6 +15135,20 @@ module Aws::Glue
|
|
15065
15135
|
# `MaxCapacity`.
|
15066
15136
|
# @return [Float]
|
15067
15137
|
#
|
15138
|
+
# @!attribute [rw] execution_class
|
15139
|
+
# Indicates whether the job is run with a standard or flexible
|
15140
|
+
# execution class. The standard execution-class is ideal for
|
15141
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
15142
|
+
# resources.
|
15143
|
+
#
|
15144
|
+
# The flexible execution class is appropriate for time-insensitive
|
15145
|
+
# jobs whose start and completion times may vary.
|
15146
|
+
#
|
15147
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
15148
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
15149
|
+
# execution class is available for Spark jobs.
|
15150
|
+
# @return [String]
|
15151
|
+
#
|
15068
15152
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JobRun AWS API Documentation
|
15069
15153
|
#
|
15070
15154
|
class JobRun < Struct.new(
|
@@ -15090,7 +15174,8 @@ module Aws::Glue
|
|
15090
15174
|
:log_group_name,
|
15091
15175
|
:notification_property,
|
15092
15176
|
:glue_version,
|
15093
|
-
:dpu_seconds
|
15177
|
+
:dpu_seconds,
|
15178
|
+
:execution_class)
|
15094
15179
|
SENSITIVE = []
|
15095
15180
|
include Aws::Structure
|
15096
15181
|
end
|
@@ -15126,7 +15211,7 @@ module Aws::Glue
|
|
15126
15211
|
# allocated_capacity: 1,
|
15127
15212
|
# timeout: 1,
|
15128
15213
|
# max_capacity: 1.0,
|
15129
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
15214
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
15130
15215
|
# number_of_workers: 1,
|
15131
15216
|
# security_configuration: "NameString",
|
15132
15217
|
# notification_property: {
|
@@ -15801,6 +15886,7 @@ module Aws::Glue
|
|
15801
15886
|
# },
|
15802
15887
|
# },
|
15803
15888
|
# },
|
15889
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
15804
15890
|
# }
|
15805
15891
|
#
|
15806
15892
|
# @!attribute [rw] description
|
@@ -15862,7 +15948,7 @@ module Aws::Glue
|
|
15862
15948
|
# This field is deprecated. Use `MaxCapacity` instead.
|
15863
15949
|
#
|
15864
15950
|
# The number of Glue data processing units (DPUs) to allocate to this
|
15865
|
-
# job. You can allocate
|
15951
|
+
# job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU
|
15866
15952
|
# is a relative measure of processing power that consists of 4 vCPUs
|
15867
15953
|
# of compute capacity and 16 GB of memory. For more information, see
|
15868
15954
|
# the [Glue pricing page][1].
|
@@ -15897,8 +15983,8 @@ module Aws::Glue
|
|
15897
15983
|
#
|
15898
15984
|
# * When you specify an Apache Spark ETL job
|
15899
15985
|
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
|
15900
|
-
# (`JobCommand.Name`="gluestreaming"), you can allocate
|
15901
|
-
#
|
15986
|
+
# (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
|
15987
|
+
# of 2 DPUs. The default is 10 DPUs. This job type cannot have a
|
15902
15988
|
# fractional DPU allocation.
|
15903
15989
|
#
|
15904
15990
|
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
@@ -15912,7 +15998,7 @@ module Aws::Glue
|
|
15912
15998
|
#
|
15913
15999
|
# @!attribute [rw] worker_type
|
15914
16000
|
# The type of predefined worker that is allocated when a job runs.
|
15915
|
-
# Accepts a value of Standard, G.1X, or G.
|
16001
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
15916
16002
|
#
|
15917
16003
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
15918
16004
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -15924,14 +16010,17 @@ module Aws::Glue
|
|
15924
16010
|
# * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
|
15925
16011
|
# GB of memory, 128 GB disk), and provides 1 executor per worker. We
|
15926
16012
|
# recommend this worker type for memory-intensive jobs.
|
16013
|
+
#
|
16014
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
16015
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
16016
|
+
# worker. We recommend this worker type for low volume streaming
|
16017
|
+
# jobs. This worker type is only available for Glue version 3.0
|
16018
|
+
# streaming jobs.
|
15927
16019
|
# @return [String]
|
15928
16020
|
#
|
15929
16021
|
# @!attribute [rw] number_of_workers
|
15930
16022
|
# The number of workers of a defined `workerType` that are allocated
|
15931
16023
|
# when a job runs.
|
15932
|
-
#
|
15933
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
15934
|
-
# 149 for `G.2X`.
|
15935
16024
|
# @return [Integer]
|
15936
16025
|
#
|
15937
16026
|
# @!attribute [rw] security_configuration
|
@@ -15963,6 +16052,20 @@ module Aws::Glue
|
|
15963
16052
|
# based.
|
15964
16053
|
# @return [Hash<String,Types::CodeGenConfigurationNode>]
|
15965
16054
|
#
|
16055
|
+
# @!attribute [rw] execution_class
|
16056
|
+
# Indicates whether the job is run with a standard or flexible
|
16057
|
+
# execution class. The standard execution-class is ideal for
|
16058
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
16059
|
+
# resources.
|
16060
|
+
#
|
16061
|
+
# The flexible execution class is appropriate for time-insensitive
|
16062
|
+
# jobs whose start and completion times may vary.
|
16063
|
+
#
|
16064
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
16065
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
16066
|
+
# execution class is available for Spark jobs.
|
16067
|
+
# @return [String]
|
16068
|
+
#
|
15966
16069
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JobUpdate AWS API Documentation
|
15967
16070
|
#
|
15968
16071
|
class JobUpdate < Struct.new(
|
@@ -15983,7 +16086,8 @@ module Aws::Glue
|
|
15983
16086
|
:security_configuration,
|
15984
16087
|
:notification_property,
|
15985
16088
|
:glue_version,
|
15986
|
-
:code_gen_configuration_nodes
|
16089
|
+
:code_gen_configuration_nodes,
|
16090
|
+
:execution_class)
|
15987
16091
|
SENSITIVE = [:code_gen_configuration_nodes]
|
15988
16092
|
include Aws::Structure
|
15989
16093
|
end
|
@@ -17144,7 +17248,7 @@ module Aws::Glue
|
|
17144
17248
|
end
|
17145
17249
|
|
17146
17250
|
# @!attribute [rw] ids
|
17147
|
-
# Returns the
|
17251
|
+
# Returns the ID of the session.
|
17148
17252
|
# @return [Array<String>]
|
17149
17253
|
#
|
17150
17254
|
# @!attribute [rw] sessions
|
@@ -17184,6 +17288,7 @@ module Aws::Glue
|
|
17184
17288
|
# @return [String]
|
17185
17289
|
#
|
17186
17290
|
# @!attribute [rw] next_token
|
17291
|
+
# A continuation token, if this is a continuation call.
|
17187
17292
|
# @return [String]
|
17188
17293
|
#
|
17189
17294
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ListStatementsRequest AWS API Documentation
|
@@ -17201,6 +17306,7 @@ module Aws::Glue
|
|
17201
17306
|
# @return [Array<Types::Statement>]
|
17202
17307
|
#
|
17203
17308
|
# @!attribute [rw] next_token
|
17309
|
+
# A continuation token, if not all statements have yet been returned.
|
17204
17310
|
# @return [String]
|
17205
17311
|
#
|
17206
17312
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ListStatementsResponse AWS API Documentation
|
@@ -18832,9 +18938,9 @@ module Aws::Glue
|
|
18832
18938
|
# {
|
18833
18939
|
# logical_operator: "EQUALS", # accepts EQUALS
|
18834
18940
|
# job_name: "NameString",
|
18835
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
18941
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
18836
18942
|
# crawler_name: "NameString",
|
18837
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
18943
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
18838
18944
|
# },
|
18839
18945
|
# ],
|
18840
18946
|
# }
|
@@ -21332,7 +21438,7 @@ module Aws::Glue
|
|
21332
21438
|
# @return [Float]
|
21333
21439
|
#
|
21334
21440
|
# @!attribute [rw] max_capacity
|
21335
|
-
# The number of
|
21441
|
+
# The number of Glue data processing units (DPUs) that can be
|
21336
21442
|
# allocated when the job runs. A DPU is a relative measure of
|
21337
21443
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
21338
21444
|
# GB memory.
|
@@ -21345,7 +21451,7 @@ module Aws::Glue
|
|
21345
21451
|
#
|
21346
21452
|
# @!attribute [rw] glue_version
|
21347
21453
|
# The Glue version determines the versions of Apache Spark and Python
|
21348
|
-
# that
|
21454
|
+
# that Glue supports. The GlueVersion must be greater than 2.0.
|
21349
21455
|
# @return [String]
|
21350
21456
|
#
|
21351
21457
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Session AWS API Documentation
|
@@ -21379,7 +21485,7 @@ module Aws::Glue
|
|
21379
21485
|
# }
|
21380
21486
|
#
|
21381
21487
|
# @!attribute [rw] name
|
21382
|
-
# Specifies the name of the SessionCommand.Can be 'glueetl' or
|
21488
|
+
# Specifies the name of the SessionCommand. Can be 'glueetl' or
|
21383
21489
|
# 'gluestreaming'.
|
21384
21490
|
# @return [String]
|
21385
21491
|
#
|
@@ -21965,8 +22071,9 @@ module Aws::Glue
|
|
21965
22071
|
# notification_property: {
|
21966
22072
|
# notify_delay_after: 1,
|
21967
22073
|
# },
|
21968
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
22074
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
21969
22075
|
# number_of_workers: 1,
|
22076
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
21970
22077
|
# }
|
21971
22078
|
#
|
21972
22079
|
# @!attribute [rw] job_name
|
@@ -21984,6 +22091,11 @@ module Aws::Glue
|
|
21984
22091
|
# You can specify arguments here that your own job-execution script
|
21985
22092
|
# consumes, as well as arguments that Glue itself consumes.
|
21986
22093
|
#
|
22094
|
+
# Job arguments may be logged. Do not pass plaintext secrets as
|
22095
|
+
# arguments. Retrieve secrets from a Glue Connection, Secrets Manager
|
22096
|
+
# or other secret management mechanism if you intend to keep them
|
22097
|
+
# within the Job.
|
22098
|
+
#
|
21987
22099
|
# For information about how to specify and consume your own Job
|
21988
22100
|
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
21989
22101
|
# developer guide.
|
@@ -22002,7 +22114,7 @@ module Aws::Glue
|
|
22002
22114
|
# This field is deprecated. Use `MaxCapacity` instead.
|
22003
22115
|
#
|
22004
22116
|
# The number of Glue data processing units (DPUs) to allocate to this
|
22005
|
-
# JobRun.
|
22117
|
+
# JobRun. You can allocate a minimum of 2 DPUs; the default is 10. A
|
22006
22118
|
# DPU is a relative measure of processing power that consists of 4
|
22007
22119
|
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
22008
22120
|
# see the [Glue pricing page][1].
|
@@ -22015,8 +22127,11 @@ module Aws::Glue
|
|
22015
22127
|
# @!attribute [rw] timeout
|
22016
22128
|
# The `JobRun` timeout in minutes. This is the maximum time that a job
|
22017
22129
|
# run can consume resources before it is terminated and enters
|
22018
|
-
# `TIMEOUT` status.
|
22019
|
-
#
|
22130
|
+
# `TIMEOUT` status. This value overrides the timeout value set in the
|
22131
|
+
# parent job.
|
22132
|
+
#
|
22133
|
+
# Streaming jobs do not have a timeout. The default for non-streaming
|
22134
|
+
# jobs is 2,880 minutes (48 hours).
|
22020
22135
|
# @return [Integer]
|
22021
22136
|
#
|
22022
22137
|
# @!attribute [rw] max_capacity
|
@@ -22036,7 +22151,7 @@ module Aws::Glue
|
|
22036
22151
|
# 0.0625 or 1 DPU. The default is 0.0625 DPU.
|
22037
22152
|
#
|
22038
22153
|
# * When you specify an Apache Spark ETL job
|
22039
|
-
# (`JobCommand.Name`="glueetl"), you can allocate
|
22154
|
+
# (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
|
22040
22155
|
# DPUs. The default is 10 DPUs. This job type cannot have a
|
22041
22156
|
# fractional DPU allocation.
|
22042
22157
|
#
|
@@ -22056,7 +22171,7 @@ module Aws::Glue
|
|
22056
22171
|
#
|
22057
22172
|
# @!attribute [rw] worker_type
|
22058
22173
|
# The type of predefined worker that is allocated when a job runs.
|
22059
|
-
# Accepts a value of Standard, G.1X, or G.
|
22174
|
+
# Accepts a value of Standard, G.1X, G.2X, or G.025X.
|
22060
22175
|
#
|
22061
22176
|
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
|
22062
22177
|
# of memory and a 50GB disk, and 2 executors per worker.
|
@@ -22066,16 +22181,33 @@ module Aws::Glue
|
|
22066
22181
|
#
|
22067
22182
|
# * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
|
22068
22183
|
# memory and a 128GB disk, and 1 executor per worker.
|
22184
|
+
#
|
22185
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
22186
|
+
# vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
|
22187
|
+
# worker. We recommend this worker type for low volume streaming
|
22188
|
+
# jobs. This worker type is only available for Glue version 3.0
|
22189
|
+
# streaming jobs.
|
22069
22190
|
# @return [String]
|
22070
22191
|
#
|
22071
22192
|
# @!attribute [rw] number_of_workers
|
22072
22193
|
# The number of workers of a defined `workerType` that are allocated
|
22073
22194
|
# when a job runs.
|
22074
|
-
#
|
22075
|
-
# The maximum number of workers you can define are 299 for `G.1X`, and
|
22076
|
-
# 149 for `G.2X`.
|
22077
22195
|
# @return [Integer]
|
22078
22196
|
#
|
22197
|
+
# @!attribute [rw] execution_class
|
22198
|
+
# Indicates whether the job is run with a standard or flexible
|
22199
|
+
# execution class. The standard execution-class is ideal for
|
22200
|
+
# time-sensitive workloads that require fast job startup and dedicated
|
22201
|
+
# resources.
|
22202
|
+
#
|
22203
|
+
# The flexible execution class is appropriate for time-insensitive
|
22204
|
+
# jobs whose start and completion times may vary.
|
22205
|
+
#
|
22206
|
+
# Only jobs with Glue version 3.0 and above and command type `glueetl`
|
22207
|
+
# will be allowed to set `ExecutionClass` to `FLEX`. The flexible
|
22208
|
+
# execution class is available for Spark jobs.
|
22209
|
+
# @return [String]
|
22210
|
+
#
|
22079
22211
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StartJobRunRequest AWS API Documentation
|
22080
22212
|
#
|
22081
22213
|
class StartJobRunRequest < Struct.new(
|
@@ -22088,7 +22220,8 @@ module Aws::Glue
|
|
22088
22220
|
:security_configuration,
|
22089
22221
|
:notification_property,
|
22090
22222
|
:worker_type,
|
22091
|
-
:number_of_workers
|
22223
|
+
:number_of_workers,
|
22224
|
+
:execution_class)
|
22092
22225
|
SENSITIVE = []
|
22093
22226
|
include Aws::Structure
|
22094
22227
|
end
|
@@ -23626,9 +23759,9 @@ module Aws::Glue
|
|
23626
23759
|
# {
|
23627
23760
|
# logical_operator: "EQUALS", # accepts EQUALS
|
23628
23761
|
# job_name: "NameString",
|
23629
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
23762
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
23630
23763
|
# crawler_name: "NameString",
|
23631
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
23764
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
23632
23765
|
# },
|
23633
23766
|
# ],
|
23634
23767
|
# },
|
@@ -24630,7 +24763,7 @@ module Aws::Glue
|
|
24630
24763
|
# allocated_capacity: 1,
|
24631
24764
|
# timeout: 1,
|
24632
24765
|
# max_capacity: 1.0,
|
24633
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
24766
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
24634
24767
|
# number_of_workers: 1,
|
24635
24768
|
# security_configuration: "NameString",
|
24636
24769
|
# notification_property: {
|
@@ -25305,6 +25438,7 @@ module Aws::Glue
|
|
25305
25438
|
# },
|
25306
25439
|
# },
|
25307
25440
|
# },
|
25441
|
+
# execution_class: "FLEX", # accepts FLEX, STANDARD
|
25308
25442
|
# },
|
25309
25443
|
# }
|
25310
25444
|
#
|
@@ -25314,6 +25448,7 @@ module Aws::Glue
|
|
25314
25448
|
#
|
25315
25449
|
# @!attribute [rw] job_update
|
25316
25450
|
# Specifies the values with which to update the job definition.
|
25451
|
+
# Unspecified configuration is removed or reset to default values.
|
25317
25452
|
# @return [Types::JobUpdate]
|
25318
25453
|
#
|
25319
25454
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateJobRequest AWS API Documentation
|
@@ -25389,7 +25524,7 @@ module Aws::Glue
|
|
25389
25524
|
# role: "RoleString",
|
25390
25525
|
# glue_version: "GlueVersionString",
|
25391
25526
|
# max_capacity: 1.0,
|
25392
|
-
# worker_type: "Standard", # accepts Standard, G.1X, G.2X
|
25527
|
+
# worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
|
25393
25528
|
# number_of_workers: 1,
|
25394
25529
|
# timeout: 1,
|
25395
25530
|
# max_retries: 1,
|
@@ -25908,9 +26043,9 @@ module Aws::Glue
|
|
25908
26043
|
# {
|
25909
26044
|
# logical_operator: "EQUALS", # accepts EQUALS
|
25910
26045
|
# job_name: "NameString",
|
25911
|
-
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
|
26046
|
+
# state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
|
25912
26047
|
# crawler_name: "NameString",
|
25913
|
-
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
|
26048
|
+
# crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
|
25914
26049
|
# },
|
25915
26050
|
# ],
|
25916
26051
|
# },
|
@@ -26444,6 +26579,16 @@ module Aws::Glue
|
|
26444
26579
|
# Total number Actions in running state.
|
26445
26580
|
# @return [Integer]
|
26446
26581
|
#
|
26582
|
+
# @!attribute [rw] errored_actions
|
26583
|
+
# Indicates the count of job runs in the ERROR state in the workflow
|
26584
|
+
# run.
|
26585
|
+
# @return [Integer]
|
26586
|
+
#
|
26587
|
+
# @!attribute [rw] waiting_actions
|
26588
|
+
# Indicates the count of job runs in WAITING state in the workflow
|
26589
|
+
# run.
|
26590
|
+
# @return [Integer]
|
26591
|
+
#
|
26447
26592
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/WorkflowRunStatistics AWS API Documentation
|
26448
26593
|
#
|
26449
26594
|
class WorkflowRunStatistics < Struct.new(
|
@@ -26452,7 +26597,9 @@ module Aws::Glue
|
|
26452
26597
|
:failed_actions,
|
26453
26598
|
:stopped_actions,
|
26454
26599
|
:succeeded_actions,
|
26455
|
-
:running_actions
|
26600
|
+
:running_actions,
|
26601
|
+
:errored_actions,
|
26602
|
+
:waiting_actions)
|
26456
26603
|
SENSITIVE = []
|
26457
26604
|
include Aws::Structure
|
26458
26605
|
end
|