aws-sdk-glue 1.114.0 → 1.117.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -3558,9 +3558,9 @@ module Aws::Glue
3558
3558
  # {
3559
3559
  # logical_operator: "EQUALS", # accepts EQUALS
3560
3560
  # job_name: "NameString",
3561
- # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
3561
+ # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
3562
3562
  # crawler_name: "NameString",
3563
- # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
3563
+ # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
3564
3564
  # }
3565
3565
  #
3566
3566
  # @!attribute [rw] logical_operator
@@ -5042,7 +5042,7 @@ module Aws::Glue
5042
5042
  # public_key: "GenericString",
5043
5043
  # public_keys: ["GenericString"],
5044
5044
  # number_of_nodes: 1,
5045
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
5045
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
5046
5046
  # glue_version: "GlueVersionString",
5047
5047
  # number_of_workers: 1,
5048
5048
  # extra_python_libs_s3_path: "GenericString",
@@ -5420,7 +5420,7 @@ module Aws::Glue
5420
5420
  # },
5421
5421
  # glue_version: "GlueVersionString",
5422
5422
  # number_of_workers: 1,
5423
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
5423
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
5424
5424
  # code_gen_configuration_nodes: {
5425
5425
  # "NodeId" => {
5426
5426
  # athena_connector_source: {
@@ -6089,6 +6089,7 @@ module Aws::Glue
6089
6089
  # },
6090
6090
  # },
6091
6091
  # },
6092
+ # execution_class: "FLEX", # accepts FLEX, STANDARD
6092
6093
  # }
6093
6094
  #
6094
6095
  # @!attribute [rw] name
@@ -6124,6 +6125,11 @@ module Aws::Glue
6124
6125
  # You can specify arguments here that your own job-execution script
6125
6126
  # consumes, as well as arguments that Glue itself consumes.
6126
6127
  #
6128
+ # Job arguments may be logged. Do not pass plaintext secrets as
6129
+ # arguments. Retrieve secrets from a Glue Connection, Secrets Manager
6130
+ # or other secret management mechanism if you intend to keep them
6131
+ # within the Job.
6132
+ #
6127
6133
  # For information about how to specify and consume your own Job
6128
6134
  # arguments, see the [Calling Glue APIs in Python][1] topic in the
6129
6135
  # developer guide.
@@ -6155,7 +6161,7 @@ module Aws::Glue
6155
6161
  # This parameter is deprecated. Use `MaxCapacity` instead.
6156
6162
  #
6157
6163
  # The number of Glue data processing units (DPUs) to allocate to this
6158
- # Job. You can allocate from 2 to 100 DPUs; the default is 10. A DPU
6164
+ # Job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU
6159
6165
  # is a relative measure of processing power that consists of 4 vCPUs
6160
6166
  # of compute capacity and 16 GB of memory. For more information, see
6161
6167
  # the [Glue pricing page][1].
@@ -6190,8 +6196,8 @@ module Aws::Glue
6190
6196
  #
6191
6197
  # * When you specify an Apache Spark ETL job
6192
6198
  # (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
6193
- # (`JobCommand.Name`="gluestreaming"), you can allocate from 2 to
6194
- # 100 DPUs. The default is 10 DPUs. This job type cannot have a
6199
+ # (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
6200
+ # of 2 DPUs. The default is 10 DPUs. This job type cannot have a
6195
6201
  # fractional DPU allocation.
6196
6202
  #
6197
6203
  # For Glue version 2.0 jobs, you cannot instead specify a `Maximum
@@ -6242,14 +6248,11 @@ module Aws::Glue
6242
6248
  # @!attribute [rw] number_of_workers
6243
6249
  # The number of workers of a defined `workerType` that are allocated
6244
6250
  # when a job runs.
6245
- #
6246
- # The maximum number of workers you can define are 299 for `G.1X`, and
6247
- # 149 for `G.2X`.
6248
6251
  # @return [Integer]
6249
6252
  #
6250
6253
  # @!attribute [rw] worker_type
6251
6254
  # The type of predefined worker that is allocated when a job runs.
6252
- # Accepts a value of Standard, G.1X, or G.2X.
6255
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X.
6253
6256
  #
6254
6257
  # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
6255
6258
  # of memory and a 50GB disk, and 2 executors per worker.
@@ -6261,6 +6264,12 @@ module Aws::Glue
6261
6264
  # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
6262
6265
  # GB of memory, 128 GB disk), and provides 1 executor per worker. We
6263
6266
  # recommend this worker type for memory-intensive jobs.
6267
+ #
6268
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
6269
+ # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
6270
+ # worker. We recommend this worker type for low volume streaming
6271
+ # jobs. This worker type is only available for Glue version 3.0
6272
+ # streaming jobs.
6264
6273
  # @return [String]
6265
6274
  #
6266
6275
  # @!attribute [rw] code_gen_configuration_nodes
@@ -6269,6 +6278,20 @@ module Aws::Glue
6269
6278
  # based.
6270
6279
  # @return [Hash<String,Types::CodeGenConfigurationNode>]
6271
6280
  #
6281
+ # @!attribute [rw] execution_class
6282
+ # Indicates whether the job is run with a standard or flexible
6283
+ # execution class. The standard execution-class is ideal for
6284
+ # time-sensitive workloads that require fast job startup and dedicated
6285
+ # resources.
6286
+ #
6287
+ # The flexible execution class is appropriate for time-insensitive
6288
+ # jobs whose start and completion times may vary.
6289
+ #
6290
+ # Only jobs with Glue version 3.0 and above and command type `glueetl`
6291
+ # will be allowed to set `ExecutionClass` to `FLEX`. The flexible
6292
+ # execution class is available for Spark jobs.
6293
+ # @return [String]
6294
+ #
6272
6295
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateJobRequest AWS API Documentation
6273
6296
  #
6274
6297
  class CreateJobRequest < Struct.new(
@@ -6291,7 +6314,8 @@ module Aws::Glue
6291
6314
  :glue_version,
6292
6315
  :number_of_workers,
6293
6316
  :worker_type,
6294
- :code_gen_configuration_nodes)
6317
+ :code_gen_configuration_nodes,
6318
+ :execution_class)
6295
6319
  SENSITIVE = [:code_gen_configuration_nodes]
6296
6320
  include Aws::Structure
6297
6321
  end
@@ -6367,7 +6391,7 @@ module Aws::Glue
6367
6391
  # role: "RoleString", # required
6368
6392
  # glue_version: "GlueVersionString",
6369
6393
  # max_capacity: 1.0,
6370
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
6394
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
6371
6395
  # number_of_workers: 1,
6372
6396
  # timeout: 1,
6373
6397
  # max_retries: 1,
@@ -7118,7 +7142,7 @@ module Aws::Glue
7118
7142
  # },
7119
7143
  # max_capacity: 1.0,
7120
7144
  # number_of_workers: 1,
7121
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
7145
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
7122
7146
  # security_configuration: "NameString",
7123
7147
  # glue_version: "GlueVersionString",
7124
7148
  # tags: {
@@ -7160,18 +7184,37 @@ module Aws::Glue
7160
7184
  # @return [Types::ConnectionsList]
7161
7185
  #
7162
7186
  # @!attribute [rw] max_capacity
7163
- # The number of AWS Glue data processing units (DPUs) that can be
7187
+ # The number of Glue data processing units (DPUs) that can be
7164
7188
  # allocated when the job runs. A DPU is a relative measure of
7165
7189
  # processing power that consists of 4 vCPUs of compute capacity and 16
7166
7190
  # GB memory.
7167
7191
  # @return [Float]
7168
7192
  #
7169
7193
  # @!attribute [rw] number_of_workers
7170
- # The number of workers to use for the session.
7194
+ # The number of workers of a defined `WorkerType` to use for the
7195
+ # session.
7171
7196
  # @return [Integer]
7172
7197
  #
7173
7198
  # @!attribute [rw] worker_type
7174
- # The Worker Type. Can be one of G.1X, G.2X, Standard
7199
+ # The type of predefined worker that is allocated to use for the
7200
+ # session. Accepts a value of Standard, G.1X, G.2X, or G.025X.
7201
+ #
7202
+ # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
7203
+ # of memory and a 50GB disk, and 2 executors per worker.
7204
+ #
7205
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16
7206
+ # GB of memory, 64 GB disk), and provides 1 executor per worker. We
7207
+ # recommend this worker type for memory-intensive jobs.
7208
+ #
7209
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
7210
+ # GB of memory, 128 GB disk), and provides 1 executor per worker. We
7211
+ # recommend this worker type for memory-intensive jobs.
7212
+ #
7213
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
7214
+ # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
7215
+ # worker. We recommend this worker type for low volume streaming
7216
+ # jobs. This worker type is only available for Glue version 3.0
7217
+ # streaming jobs.
7175
7218
  # @return [String]
7176
7219
  #
7177
7220
  # @!attribute [rw] security_configuration
@@ -7181,7 +7224,7 @@ module Aws::Glue
7181
7224
  #
7182
7225
  # @!attribute [rw] glue_version
7183
7226
  # The Glue version determines the versions of Apache Spark and Python
7184
- # that AWS Glue supports. The GlueVersion must be greater than 2.0.
7227
+ # that Glue supports. The GlueVersion must be greater than 2.0.
7185
7228
  # @return [String]
7186
7229
  #
7187
7230
  # @!attribute [rw] tags
@@ -7376,9 +7419,9 @@ module Aws::Glue
7376
7419
  # {
7377
7420
  # logical_operator: "EQUALS", # accepts EQUALS
7378
7421
  # job_name: "NameString",
7379
- # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
7422
+ # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
7380
7423
  # crawler_name: "NameString",
7381
- # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
7424
+ # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
7382
7425
  # },
7383
7426
  # ],
7384
7427
  # },
@@ -9724,8 +9767,8 @@ module Aws::Glue
9724
9767
  include Aws::Structure
9725
9768
  end
9726
9769
 
9727
- # An edge represents a directed connection between two components on a
9728
- # workflow graph.
9770
+ # An edge represents a directed connection between two Glue components
9771
+ # that are part of the workflow the edge belongs to.
9729
9772
  #
9730
9773
  # @!attribute [rw] source_id
9731
9774
  # The unique of the node within the workflow where the edge starts.
@@ -14128,7 +14171,10 @@ module Aws::Glue
14128
14171
  include Aws::Structure
14129
14172
  end
14130
14173
 
14174
+ # The blueprint is in an invalid state to perform a requested operation.
14175
+ #
14131
14176
  # @!attribute [rw] message
14177
+ # A message describing the problem.
14132
14178
  # @return [String]
14133
14179
  #
14134
14180
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/IllegalBlueprintStateException AWS API Documentation
@@ -14612,7 +14658,7 @@ module Aws::Glue
14612
14658
  # This field is deprecated. Use `MaxCapacity` instead.
14613
14659
  #
14614
14660
  # The number of Glue data processing units (DPUs) allocated to runs of
14615
- # this job. You can allocate from 2 to 100 DPUs; the default is 10. A
14661
+ # this job. You can allocate a minimum of 2 DPUs; the default is 10. A
14616
14662
  # DPU is a relative measure of processing power that consists of 4
14617
14663
  # vCPUs of compute capacity and 16 GB of memory. For more information,
14618
14664
  # see the [Glue pricing page][1].
@@ -14650,8 +14696,8 @@ module Aws::Glue
14650
14696
  #
14651
14697
  # * When you specify an Apache Spark ETL job
14652
14698
  # (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
14653
- # (`JobCommand.Name`="gluestreaming"), you can allocate from 2 to
14654
- # 100 DPUs. The default is 10 DPUs. This job type cannot have a
14699
+ # (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
14700
+ # of 2 DPUs. The default is 10 DPUs. This job type cannot have a
14655
14701
  # fractional DPU allocation.
14656
14702
  #
14657
14703
  # For Glue version 2.0 jobs, you cannot instead specify a `Maximum
@@ -14665,7 +14711,7 @@ module Aws::Glue
14665
14711
  #
14666
14712
  # @!attribute [rw] worker_type
14667
14713
  # The type of predefined worker that is allocated when a job runs.
14668
- # Accepts a value of Standard, G.1X, or G.2X.
14714
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X.
14669
14715
  #
14670
14716
  # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
14671
14717
  # of memory and a 50GB disk, and 2 executors per worker.
@@ -14677,14 +14723,17 @@ module Aws::Glue
14677
14723
  # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
14678
14724
  # GB of memory, 128 GB disk), and provides 1 executor per worker. We
14679
14725
  # recommend this worker type for memory-intensive jobs.
14726
+ #
14727
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
14728
+ # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
14729
+ # worker. We recommend this worker type for low volume streaming
14730
+ # jobs. This worker type is only available for Glue version 3.0
14731
+ # streaming jobs.
14680
14732
  # @return [String]
14681
14733
  #
14682
14734
  # @!attribute [rw] number_of_workers
14683
14735
  # The number of workers of a defined `workerType` that are allocated
14684
14736
  # when a job runs.
14685
- #
14686
- # The maximum number of workers you can define are 299 for `G.1X`, and
14687
- # 149 for `G.2X`.
14688
14737
  # @return [Integer]
14689
14738
  #
14690
14739
  # @!attribute [rw] security_configuration
@@ -14719,6 +14768,20 @@ module Aws::Glue
14719
14768
  # based.
14720
14769
  # @return [Hash<String,Types::CodeGenConfigurationNode>]
14721
14770
  #
14771
+ # @!attribute [rw] execution_class
14772
+ # Indicates whether the job is run with a standard or flexible
14773
+ # execution class. The standard execution class is ideal for
14774
+ # time-sensitive workloads that require fast job startup and dedicated
14775
+ # resources.
14776
+ #
14777
+ # The flexible execution class is appropriate for time-insensitive
14778
+ # jobs whose start and completion times may vary.
14779
+ #
14780
+ # Only jobs with Glue version 3.0 and above and command type `glueetl`
14781
+ # will be allowed to set `ExecutionClass` to `FLEX`. The flexible
14782
+ # execution class is available for Spark jobs.
14783
+ # @return [String]
14784
+ #
14722
14785
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Job AWS API Documentation
14723
14786
  #
14724
14787
  class Job < Struct.new(
@@ -14742,7 +14805,8 @@ module Aws::Glue
14742
14805
  :security_configuration,
14743
14806
  :notification_property,
14744
14807
  :glue_version,
14745
- :code_gen_configuration_nodes)
14808
+ :code_gen_configuration_nodes,
14809
+ :execution_class)
14746
14810
  SENSITIVE = [:code_gen_configuration_nodes]
14747
14811
  include Aws::Structure
14748
14812
  end
@@ -14965,8 +15029,11 @@ module Aws::Glue
14965
15029
  # @!attribute [rw] timeout
14966
15030
  # The `JobRun` timeout in minutes. This is the maximum time that a job
14967
15031
  # run can consume resources before it is terminated and enters
14968
- # `TIMEOUT` status. The default is 2,880 minutes (48 hours). This
14969
- # overrides the timeout value set in the parent job.
15032
+ # `TIMEOUT` status. This value overrides the timeout value set in the
15033
+ # parent job.
15034
+ #
15035
+ # Streaming jobs do not have a timeout. The default for non-streaming
15036
+ # jobs is 2,880 minutes (48 hours).
14970
15037
  # @return [Integer]
14971
15038
  #
14972
15039
  # @!attribute [rw] max_capacity
@@ -14986,7 +15053,7 @@ module Aws::Glue
14986
15053
  # 0.0625 or 1 DPU. The default is 0.0625 DPU.
14987
15054
  #
14988
15055
  # * When you specify an Apache Spark ETL job
14989
- # (`JobCommand.Name`="glueetl"), you can allocate from 2 to 100
15056
+ # (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
14990
15057
  # DPUs. The default is 10 DPUs. This job type cannot have a
14991
15058
  # fractional DPU allocation.
14992
15059
  #
@@ -14997,7 +15064,7 @@ module Aws::Glue
14997
15064
  #
14998
15065
  # @!attribute [rw] worker_type
14999
15066
  # The type of predefined worker that is allocated when a job runs.
15000
- # Accepts a value of Standard, G.1X, or G.2X.
15067
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X.
15001
15068
  #
15002
15069
  # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
15003
15070
  # of memory and a 50GB disk, and 2 executors per worker.
@@ -15007,14 +15074,17 @@ module Aws::Glue
15007
15074
  #
15008
15075
  # * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
15009
15076
  # memory and a 128GB disk, and 1 executor per worker.
15077
+ #
15078
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
15079
+ # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
15080
+ # worker. We recommend this worker type for low volume streaming
15081
+ # jobs. This worker type is only available for Glue version 3.0
15082
+ # streaming jobs.
15010
15083
  # @return [String]
15011
15084
  #
15012
15085
  # @!attribute [rw] number_of_workers
15013
15086
  # The number of workers of a defined `workerType` that are allocated
15014
15087
  # when a job runs.
15015
- #
15016
- # The maximum number of workers you can define are 299 for `G.1X`, and
15017
- # 149 for `G.2X`.
15018
15088
  # @return [Integer]
15019
15089
  #
15020
15090
  # @!attribute [rw] security_configuration
@@ -15054,10 +15124,10 @@ module Aws::Glue
15054
15124
  # @return [String]
15055
15125
  #
15056
15126
  # @!attribute [rw] dpu_seconds
15057
- # This field populates only when an Auto Scaling job run completes,
15058
- # and represents the total time each executor ran during the lifecycle
15059
- # of a job run in seconds, multiplied by a DPU factor (1 for `G.1X`
15060
- # and 2 for `G.2X` workers). This value may be different than the
15127
+ # This field populates only for Auto Scaling job runs, and represents
15128
+ # the total time each executor ran during the lifecycle of a job run
15129
+ # in seconds, multiplied by a DPU factor (1 for `G.1X`, 2 for `G.2X`,
15130
+ # or 0.25 for `G.025X` workers). This value may be different than the
15061
15131
  # `executionEngineRuntime` * `MaxCapacity` as in the case of Auto
15062
15132
  # Scaling jobs, as the number of executors running at a given time may
15063
15133
  # be less than the `MaxCapacity`. Therefore, it is possible that the
@@ -15065,6 +15135,20 @@ module Aws::Glue
15065
15135
  # `MaxCapacity`.
15066
15136
  # @return [Float]
15067
15137
  #
15138
+ # @!attribute [rw] execution_class
15139
+ # Indicates whether the job is run with a standard or flexible
15140
+ # execution class. The standard execution-class is ideal for
15141
+ # time-sensitive workloads that require fast job startup and dedicated
15142
+ # resources.
15143
+ #
15144
+ # The flexible execution class is appropriate for time-insensitive
15145
+ # jobs whose start and completion times may vary.
15146
+ #
15147
+ # Only jobs with Glue version 3.0 and above and command type `glueetl`
15148
+ # will be allowed to set `ExecutionClass` to `FLEX`. The flexible
15149
+ # execution class is available for Spark jobs.
15150
+ # @return [String]
15151
+ #
15068
15152
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JobRun AWS API Documentation
15069
15153
  #
15070
15154
  class JobRun < Struct.new(
@@ -15090,7 +15174,8 @@ module Aws::Glue
15090
15174
  :log_group_name,
15091
15175
  :notification_property,
15092
15176
  :glue_version,
15093
- :dpu_seconds)
15177
+ :dpu_seconds,
15178
+ :execution_class)
15094
15179
  SENSITIVE = []
15095
15180
  include Aws::Structure
15096
15181
  end
@@ -15126,7 +15211,7 @@ module Aws::Glue
15126
15211
  # allocated_capacity: 1,
15127
15212
  # timeout: 1,
15128
15213
  # max_capacity: 1.0,
15129
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
15214
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
15130
15215
  # number_of_workers: 1,
15131
15216
  # security_configuration: "NameString",
15132
15217
  # notification_property: {
@@ -15801,6 +15886,7 @@ module Aws::Glue
15801
15886
  # },
15802
15887
  # },
15803
15888
  # },
15889
+ # execution_class: "FLEX", # accepts FLEX, STANDARD
15804
15890
  # }
15805
15891
  #
15806
15892
  # @!attribute [rw] description
@@ -15862,7 +15948,7 @@ module Aws::Glue
15862
15948
  # This field is deprecated. Use `MaxCapacity` instead.
15863
15949
  #
15864
15950
  # The number of Glue data processing units (DPUs) to allocate to this
15865
- # job. You can allocate from 2 to 100 DPUs; the default is 10. A DPU
15951
+ # job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU
15866
15952
  # is a relative measure of processing power that consists of 4 vCPUs
15867
15953
  # of compute capacity and 16 GB of memory. For more information, see
15868
15954
  # the [Glue pricing page][1].
@@ -15897,8 +15983,8 @@ module Aws::Glue
15897
15983
  #
15898
15984
  # * When you specify an Apache Spark ETL job
15899
15985
  # (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
15900
- # (`JobCommand.Name`="gluestreaming"), you can allocate from 2 to
15901
- # 100 DPUs. The default is 10 DPUs. This job type cannot have a
15986
+ # (`JobCommand.Name`="gluestreaming"), you can allocate a minimum
15987
+ # of 2 DPUs. The default is 10 DPUs. This job type cannot have a
15902
15988
  # fractional DPU allocation.
15903
15989
  #
15904
15990
  # For Glue version 2.0 jobs, you cannot instead specify a `Maximum
@@ -15912,7 +15998,7 @@ module Aws::Glue
15912
15998
  #
15913
15999
  # @!attribute [rw] worker_type
15914
16000
  # The type of predefined worker that is allocated when a job runs.
15915
- # Accepts a value of Standard, G.1X, or G.2X.
16001
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X.
15916
16002
  #
15917
16003
  # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
15918
16004
  # of memory and a 50GB disk, and 2 executors per worker.
@@ -15924,14 +16010,17 @@ module Aws::Glue
15924
16010
  # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32
15925
16011
  # GB of memory, 128 GB disk), and provides 1 executor per worker. We
15926
16012
  # recommend this worker type for memory-intensive jobs.
16013
+ #
16014
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
16015
+ # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
16016
+ # worker. We recommend this worker type for low volume streaming
16017
+ # jobs. This worker type is only available for Glue version 3.0
16018
+ # streaming jobs.
15927
16019
  # @return [String]
15928
16020
  #
15929
16021
  # @!attribute [rw] number_of_workers
15930
16022
  # The number of workers of a defined `workerType` that are allocated
15931
16023
  # when a job runs.
15932
- #
15933
- # The maximum number of workers you can define are 299 for `G.1X`, and
15934
- # 149 for `G.2X`.
15935
16024
  # @return [Integer]
15936
16025
  #
15937
16026
  # @!attribute [rw] security_configuration
@@ -15963,6 +16052,20 @@ module Aws::Glue
15963
16052
  # based.
15964
16053
  # @return [Hash<String,Types::CodeGenConfigurationNode>]
15965
16054
  #
16055
+ # @!attribute [rw] execution_class
16056
+ # Indicates whether the job is run with a standard or flexible
16057
+ # execution class. The standard execution-class is ideal for
16058
+ # time-sensitive workloads that require fast job startup and dedicated
16059
+ # resources.
16060
+ #
16061
+ # The flexible execution class is appropriate for time-insensitive
16062
+ # jobs whose start and completion times may vary.
16063
+ #
16064
+ # Only jobs with Glue version 3.0 and above and command type `glueetl`
16065
+ # will be allowed to set `ExecutionClass` to `FLEX`. The flexible
16066
+ # execution class is available for Spark jobs.
16067
+ # @return [String]
16068
+ #
15966
16069
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JobUpdate AWS API Documentation
15967
16070
  #
15968
16071
  class JobUpdate < Struct.new(
@@ -15983,7 +16086,8 @@ module Aws::Glue
15983
16086
  :security_configuration,
15984
16087
  :notification_property,
15985
16088
  :glue_version,
15986
- :code_gen_configuration_nodes)
16089
+ :code_gen_configuration_nodes,
16090
+ :execution_class)
15987
16091
  SENSITIVE = [:code_gen_configuration_nodes]
15988
16092
  include Aws::Structure
15989
16093
  end
@@ -17144,7 +17248,7 @@ module Aws::Glue
17144
17248
  end
17145
17249
 
17146
17250
  # @!attribute [rw] ids
17147
- # Returns the Id of the session.
17251
+ # Returns the ID of the session.
17148
17252
  # @return [Array<String>]
17149
17253
  #
17150
17254
  # @!attribute [rw] sessions
@@ -17184,6 +17288,7 @@ module Aws::Glue
17184
17288
  # @return [String]
17185
17289
  #
17186
17290
  # @!attribute [rw] next_token
17291
+ # A continuation token, if this is a continuation call.
17187
17292
  # @return [String]
17188
17293
  #
17189
17294
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ListStatementsRequest AWS API Documentation
@@ -17201,6 +17306,7 @@ module Aws::Glue
17201
17306
  # @return [Array<Types::Statement>]
17202
17307
  #
17203
17308
  # @!attribute [rw] next_token
17309
+ # A continuation token, if not all statements have yet been returned.
17204
17310
  # @return [String]
17205
17311
  #
17206
17312
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ListStatementsResponse AWS API Documentation
@@ -18832,9 +18938,9 @@ module Aws::Glue
18832
18938
  # {
18833
18939
  # logical_operator: "EQUALS", # accepts EQUALS
18834
18940
  # job_name: "NameString",
18835
- # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
18941
+ # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
18836
18942
  # crawler_name: "NameString",
18837
- # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
18943
+ # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
18838
18944
  # },
18839
18945
  # ],
18840
18946
  # }
@@ -21332,7 +21438,7 @@ module Aws::Glue
21332
21438
  # @return [Float]
21333
21439
  #
21334
21440
  # @!attribute [rw] max_capacity
21335
- # The number of AWS Glue data processing units (DPUs) that can be
21441
+ # The number of Glue data processing units (DPUs) that can be
21336
21442
  # allocated when the job runs. A DPU is a relative measure of
21337
21443
  # processing power that consists of 4 vCPUs of compute capacity and 16
21338
21444
  # GB memory.
@@ -21345,7 +21451,7 @@ module Aws::Glue
21345
21451
  #
21346
21452
  # @!attribute [rw] glue_version
21347
21453
  # The Glue version determines the versions of Apache Spark and Python
21348
- # that AWS Glue supports. The GlueVersion must be greater than 2.0.
21454
+ # that Glue supports. The GlueVersion must be greater than 2.0.
21349
21455
  # @return [String]
21350
21456
  #
21351
21457
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Session AWS API Documentation
@@ -21379,7 +21485,7 @@ module Aws::Glue
21379
21485
  # }
21380
21486
  #
21381
21487
  # @!attribute [rw] name
21382
- # Specifies the name of the SessionCommand.Can be 'glueetl' or
21488
+ # Specifies the name of the SessionCommand. Can be 'glueetl' or
21383
21489
  # 'gluestreaming'.
21384
21490
  # @return [String]
21385
21491
  #
@@ -21965,8 +22071,9 @@ module Aws::Glue
21965
22071
  # notification_property: {
21966
22072
  # notify_delay_after: 1,
21967
22073
  # },
21968
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
22074
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
21969
22075
  # number_of_workers: 1,
22076
+ # execution_class: "FLEX", # accepts FLEX, STANDARD
21970
22077
  # }
21971
22078
  #
21972
22079
  # @!attribute [rw] job_name
@@ -21984,6 +22091,11 @@ module Aws::Glue
21984
22091
  # You can specify arguments here that your own job-execution script
21985
22092
  # consumes, as well as arguments that Glue itself consumes.
21986
22093
  #
22094
+ # Job arguments may be logged. Do not pass plaintext secrets as
22095
+ # arguments. Retrieve secrets from a Glue Connection, Secrets Manager
22096
+ # or other secret management mechanism if you intend to keep them
22097
+ # within the Job.
22098
+ #
21987
22099
  # For information about how to specify and consume your own Job
21988
22100
  # arguments, see the [Calling Glue APIs in Python][1] topic in the
21989
22101
  # developer guide.
@@ -22002,7 +22114,7 @@ module Aws::Glue
22002
22114
  # This field is deprecated. Use `MaxCapacity` instead.
22003
22115
  #
22004
22116
  # The number of Glue data processing units (DPUs) to allocate to this
22005
- # JobRun. From 2 to 100 DPUs can be allocated; the default is 10. A
22117
+ # JobRun. You can allocate a minimum of 2 DPUs; the default is 10. A
22006
22118
  # DPU is a relative measure of processing power that consists of 4
22007
22119
  # vCPUs of compute capacity and 16 GB of memory. For more information,
22008
22120
  # see the [Glue pricing page][1].
@@ -22015,8 +22127,11 @@ module Aws::Glue
22015
22127
  # @!attribute [rw] timeout
22016
22128
  # The `JobRun` timeout in minutes. This is the maximum time that a job
22017
22129
  # run can consume resources before it is terminated and enters
22018
- # `TIMEOUT` status. The default is 2,880 minutes (48 hours). This
22019
- # overrides the timeout value set in the parent job.
22130
+ # `TIMEOUT` status. This value overrides the timeout value set in the
22131
+ # parent job.
22132
+ #
22133
+ # Streaming jobs do not have a timeout. The default for non-streaming
22134
+ # jobs is 2,880 minutes (48 hours).
22020
22135
  # @return [Integer]
22021
22136
  #
22022
22137
  # @!attribute [rw] max_capacity
@@ -22036,7 +22151,7 @@ module Aws::Glue
22036
22151
  # 0.0625 or 1 DPU. The default is 0.0625 DPU.
22037
22152
  #
22038
22153
  # * When you specify an Apache Spark ETL job
22039
- # (`JobCommand.Name`="glueetl"), you can allocate from 2 to 100
22154
+ # (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
22040
22155
  # DPUs. The default is 10 DPUs. This job type cannot have a
22041
22156
  # fractional DPU allocation.
22042
22157
  #
@@ -22056,7 +22171,7 @@ module Aws::Glue
22056
22171
  #
22057
22172
  # @!attribute [rw] worker_type
22058
22173
  # The type of predefined worker that is allocated when a job runs.
22059
- # Accepts a value of Standard, G.1X, or G.2X.
22174
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X.
22060
22175
  #
22061
22176
  # * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
22062
22177
  # of memory and a 50GB disk, and 2 executors per worker.
@@ -22066,16 +22181,33 @@ module Aws::Glue
22066
22181
  #
22067
22182
  # * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
22068
22183
  # memory and a 128GB disk, and 1 executor per worker.
22184
+ #
22185
+ # * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
22186
+ # vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per
22187
+ # worker. We recommend this worker type for low volume streaming
22188
+ # jobs. This worker type is only available for Glue version 3.0
22189
+ # streaming jobs.
22069
22190
  # @return [String]
22070
22191
  #
22071
22192
  # @!attribute [rw] number_of_workers
22072
22193
  # The number of workers of a defined `workerType` that are allocated
22073
22194
  # when a job runs.
22074
- #
22075
- # The maximum number of workers you can define are 299 for `G.1X`, and
22076
- # 149 for `G.2X`.
22077
22195
  # @return [Integer]
22078
22196
  #
22197
+ # @!attribute [rw] execution_class
22198
+ # Indicates whether the job is run with a standard or flexible
22199
+ # execution class. The standard execution-class is ideal for
22200
+ # time-sensitive workloads that require fast job startup and dedicated
22201
+ # resources.
22202
+ #
22203
+ # The flexible execution class is appropriate for time-insensitive
22204
+ # jobs whose start and completion times may vary.
22205
+ #
22206
+ # Only jobs with Glue version 3.0 and above and command type `glueetl`
22207
+ # will be allowed to set `ExecutionClass` to `FLEX`. The flexible
22208
+ # execution class is available for Spark jobs.
22209
+ # @return [String]
22210
+ #
22079
22211
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StartJobRunRequest AWS API Documentation
22080
22212
  #
22081
22213
  class StartJobRunRequest < Struct.new(
@@ -22088,7 +22220,8 @@ module Aws::Glue
22088
22220
  :security_configuration,
22089
22221
  :notification_property,
22090
22222
  :worker_type,
22091
- :number_of_workers)
22223
+ :number_of_workers,
22224
+ :execution_class)
22092
22225
  SENSITIVE = []
22093
22226
  include Aws::Structure
22094
22227
  end
@@ -23626,9 +23759,9 @@ module Aws::Glue
23626
23759
  # {
23627
23760
  # logical_operator: "EQUALS", # accepts EQUALS
23628
23761
  # job_name: "NameString",
23629
- # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
23762
+ # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
23630
23763
  # crawler_name: "NameString",
23631
- # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
23764
+ # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
23632
23765
  # },
23633
23766
  # ],
23634
23767
  # },
@@ -24630,7 +24763,7 @@ module Aws::Glue
24630
24763
  # allocated_capacity: 1,
24631
24764
  # timeout: 1,
24632
24765
  # max_capacity: 1.0,
24633
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
24766
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
24634
24767
  # number_of_workers: 1,
24635
24768
  # security_configuration: "NameString",
24636
24769
  # notification_property: {
@@ -25305,6 +25438,7 @@ module Aws::Glue
25305
25438
  # },
25306
25439
  # },
25307
25440
  # },
25441
+ # execution_class: "FLEX", # accepts FLEX, STANDARD
25308
25442
  # },
25309
25443
  # }
25310
25444
  #
@@ -25314,6 +25448,7 @@ module Aws::Glue
25314
25448
  #
25315
25449
  # @!attribute [rw] job_update
25316
25450
  # Specifies the values with which to update the job definition.
25451
+ # Unspecified configuration is removed or reset to default values.
25317
25452
  # @return [Types::JobUpdate]
25318
25453
  #
25319
25454
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateJobRequest AWS API Documentation
@@ -25389,7 +25524,7 @@ module Aws::Glue
25389
25524
  # role: "RoleString",
25390
25525
  # glue_version: "GlueVersionString",
25391
25526
  # max_capacity: 1.0,
25392
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X
25527
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X
25393
25528
  # number_of_workers: 1,
25394
25529
  # timeout: 1,
25395
25530
  # max_retries: 1,
@@ -25908,9 +26043,9 @@ module Aws::Glue
25908
26043
  # {
25909
26044
  # logical_operator: "EQUALS", # accepts EQUALS
25910
26045
  # job_name: "NameString",
25911
- # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT
26046
+ # state: "STARTING", # accepts STARTING, RUNNING, STOPPING, STOPPED, SUCCEEDED, FAILED, TIMEOUT, ERROR, WAITING
25912
26047
  # crawler_name: "NameString",
25913
- # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED
26048
+ # crawl_state: "RUNNING", # accepts RUNNING, CANCELLING, CANCELLED, SUCCEEDED, FAILED, ERROR
25914
26049
  # },
25915
26050
  # ],
25916
26051
  # },
@@ -26444,6 +26579,16 @@ module Aws::Glue
26444
26579
  # Total number Actions in running state.
26445
26580
  # @return [Integer]
26446
26581
  #
26582
+ # @!attribute [rw] errored_actions
26583
+ # Indicates the count of job runs in the ERROR state in the workflow
26584
+ # run.
26585
+ # @return [Integer]
26586
+ #
26587
+ # @!attribute [rw] waiting_actions
26588
+ # Indicates the count of job runs in WAITING state in the workflow
26589
+ # run.
26590
+ # @return [Integer]
26591
+ #
26447
26592
  # @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/WorkflowRunStatistics AWS API Documentation
26448
26593
  #
26449
26594
  class WorkflowRunStatistics < Struct.new(
@@ -26452,7 +26597,9 @@ module Aws::Glue
26452
26597
  :failed_actions,
26453
26598
  :stopped_actions,
26454
26599
  :succeeded_actions,
26455
- :running_actions)
26600
+ :running_actions,
26601
+ :errored_actions,
26602
+ :waiting_actions)
26456
26603
  SENSITIVE = []
26457
26604
  include Aws::Structure
26458
26605
  end