aws-sdk-glue 1.286.0 → 1.288.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-glue/client.rb +216 -61
- data/lib/aws-sdk-glue/types.rb +300 -65
- data/lib/aws-sdk-glue.rb +1 -1
- data/sig/client.rbs +9 -9
- data/sig/types.rbs +14 -14
- metadata +3 -3
data/lib/aws-sdk-glue/types.rb
CHANGED
|
@@ -7057,8 +7057,15 @@ module Aws::Glue
|
|
|
7057
7057
|
#
|
|
7058
7058
|
# @!attribute [rw] worker_type
|
|
7059
7059
|
# The type of predefined worker that is allocated when a job runs.
|
|
7060
|
-
# Accepts a value of G.1X, G.2X, G.4X, G.8X
|
|
7061
|
-
# Accepts the value Z.2X for
|
|
7060
|
+
# Accepts a value of G.025X, G.1X, G.2X, G.4X, G.8X, G.12X, G.16X,
|
|
7061
|
+
# R.1X, R.2X, R.4X, or R.8X for Spark jobs. Accepts the value Z.2X for
|
|
7062
|
+
# Ray jobs.
|
|
7063
|
+
#
|
|
7064
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
|
7065
|
+
# vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per
|
|
7066
|
+
# worker. We recommend this worker type for low volume streaming
|
|
7067
|
+
# jobs. This worker type is only available for Glue version 3.0 or
|
|
7068
|
+
# later streaming jobs.
|
|
7062
7069
|
#
|
|
7063
7070
|
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
|
7064
7071
|
# GB of memory) with 94GB disk, and provides 1 executor per worker.
|
|
@@ -7077,31 +7084,74 @@ module Aws::Glue
|
|
|
7077
7084
|
# worker. We recommend this worker type for jobs whose workloads
|
|
7078
7085
|
# contain your most demanding transforms, aggregations, joins, and
|
|
7079
7086
|
# queries. This worker type is available only for Glue version 3.0
|
|
7080
|
-
# or later Spark ETL jobs in
|
|
7081
|
-
#
|
|
7082
|
-
#
|
|
7083
|
-
# (Seoul), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia
|
|
7084
|
-
# Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe
|
|
7085
|
-
# (Ireland), Europe (London), Europe (Spain), Europe (Stockholm),
|
|
7086
|
-
# and South America (São Paulo).
|
|
7087
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7088
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7089
|
+
# jobs][1] in the developer guide.
|
|
7087
7090
|
#
|
|
7088
7091
|
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
|
7089
7092
|
# 128 GB of memory) with 512GB disk, and provides 1 executor per
|
|
7090
7093
|
# worker. We recommend this worker type for jobs whose workloads
|
|
7091
7094
|
# contain your most demanding transforms, aggregations, joins, and
|
|
7092
7095
|
# queries. This worker type is available only for Glue version 3.0
|
|
7093
|
-
# or later Spark ETL jobs
|
|
7094
|
-
#
|
|
7095
|
-
#
|
|
7096
|
-
#
|
|
7097
|
-
#
|
|
7098
|
-
#
|
|
7099
|
-
#
|
|
7100
|
-
#
|
|
7096
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7097
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7098
|
+
# jobs][1] in the developer guide.
|
|
7099
|
+
#
|
|
7100
|
+
# * For the `G.12X` worker type, each worker maps to 12 DPU (48 vCPUs,
|
|
7101
|
+
# 192 GB of memory) with 768GB disk, and provides 1 executor per
|
|
7102
|
+
# worker. We recommend this worker type for jobs with very large
|
|
7103
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7104
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7105
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7106
|
+
# jobs][1] in the developer guide.
|
|
7107
|
+
#
|
|
7108
|
+
# * For the `G.16X` worker type, each worker maps to 16 DPU (64 vCPUs,
|
|
7109
|
+
# 256 GB of memory) with 1024GB disk, and provides 1 executor per
|
|
7110
|
+
# worker. We recommend this worker type for jobs with very large
|
|
7111
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7112
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7113
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7114
|
+
# jobs][1] in the developer guide.
|
|
7115
|
+
#
|
|
7116
|
+
# * For the `R.1X` worker type, each worker maps to 1 M-DPU (4 vCPUs,
|
|
7117
|
+
# 32 GB of memory) with 94GB disk, and provides 1 executor per
|
|
7118
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7119
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7120
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7121
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7122
|
+
# jobs][1] in the developer guide.
|
|
7123
|
+
#
|
|
7124
|
+
# * For the `R.2X` worker type, each worker maps to 2 M-DPU (8 vCPUs,
|
|
7125
|
+
# 64 GB of memory) with 138GB disk, and provides 1 executor per
|
|
7126
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7127
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7128
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7129
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7130
|
+
# jobs][1] in the developer guide.
|
|
7131
|
+
#
|
|
7132
|
+
# * For the `R.4X` worker type, each worker maps to 4 M-DPU (16 vCPUs,
|
|
7133
|
+
# 128 GB of memory) with 256GB disk, and provides 1 executor per
|
|
7134
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7135
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7136
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7137
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7138
|
+
# jobs][1] in the developer guide.
|
|
7139
|
+
#
|
|
7140
|
+
# * For the `R.8X` worker type, each worker maps to 8 M-DPU (32 vCPUs,
|
|
7141
|
+
# 256 GB of memory) with 512GB disk, and provides 1 executor per
|
|
7142
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7143
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7144
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7145
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7146
|
+
# jobs][1] in the developer guide.
|
|
7101
7147
|
#
|
|
7102
7148
|
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
|
7103
7149
|
# 64 GB of memory) with 128 GB disk, and provides up to 8 Ray
|
|
7104
7150
|
# workers based on the autoscaler.
|
|
7151
|
+
#
|
|
7152
|
+
#
|
|
7153
|
+
#
|
|
7154
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
7105
7155
|
# @return [String]
|
|
7106
7156
|
#
|
|
7107
7157
|
# @!attribute [rw] code_gen_configuration_nodes
|
|
@@ -7812,8 +7862,9 @@ module Aws::Glue
|
|
|
7812
7862
|
#
|
|
7813
7863
|
# @!attribute [rw] worker_type
|
|
7814
7864
|
# The type of predefined worker that is allocated when a job runs.
|
|
7815
|
-
# Accepts a value of G.1X, G.2X, G.4X,
|
|
7816
|
-
# the value Z.2X for Ray
|
|
7865
|
+
# Accepts a value of G.1X, G.2X, G.4X, G.8X, G.12X, G.16X, R.1X, R.2X,
|
|
7866
|
+
# R.4X, or R.8X for Spark jobs. Accepts the value Z.2X for Ray
|
|
7867
|
+
# notebooks.
|
|
7817
7868
|
#
|
|
7818
7869
|
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
|
7819
7870
|
# GB of memory) with 94GB disk, and provides 1 executor per worker.
|
|
@@ -7832,23 +7883,74 @@ module Aws::Glue
|
|
|
7832
7883
|
# worker. We recommend this worker type for jobs whose workloads
|
|
7833
7884
|
# contain your most demanding transforms, aggregations, joins, and
|
|
7834
7885
|
# queries. This worker type is available only for Glue version 3.0
|
|
7835
|
-
# or later Spark ETL jobs in
|
|
7836
|
-
#
|
|
7837
|
-
#
|
|
7838
|
-
# (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland),
|
|
7839
|
-
# and Europe (Stockholm).
|
|
7886
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7887
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7888
|
+
# jobs][1] in the developer guide.
|
|
7840
7889
|
#
|
|
7841
7890
|
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
|
7842
7891
|
# 128 GB of memory) with 512GB disk, and provides 1 executor per
|
|
7843
7892
|
# worker. We recommend this worker type for jobs whose workloads
|
|
7844
7893
|
# contain your most demanding transforms, aggregations, joins, and
|
|
7845
7894
|
# queries. This worker type is available only for Glue version 3.0
|
|
7846
|
-
# or later Spark ETL jobs
|
|
7847
|
-
#
|
|
7895
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7896
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7897
|
+
# jobs][1] in the developer guide.
|
|
7898
|
+
#
|
|
7899
|
+
# * For the `G.12X` worker type, each worker maps to 12 DPU (48 vCPUs,
|
|
7900
|
+
# 192 GB of memory) with 768GB disk, and provides 1 executor per
|
|
7901
|
+
# worker. We recommend this worker type for jobs with very large
|
|
7902
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7903
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7904
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7905
|
+
# jobs][1] in the developer guide.
|
|
7906
|
+
#
|
|
7907
|
+
# * For the `G.16X` worker type, each worker maps to 16 DPU (64 vCPUs,
|
|
7908
|
+
# 256 GB of memory) with 1024GB disk, and provides 1 executor per
|
|
7909
|
+
# worker. We recommend this worker type for jobs with very large
|
|
7910
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7911
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7912
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7913
|
+
# jobs][1] in the developer guide.
|
|
7914
|
+
#
|
|
7915
|
+
# * For the `R.1X` worker type, each worker maps to 1 M-DPU (4 vCPUs,
|
|
7916
|
+
# 32 GB of memory) with 94GB disk, and provides 1 executor per
|
|
7917
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7918
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7919
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7920
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7921
|
+
# jobs][1] in the developer guide.
|
|
7922
|
+
#
|
|
7923
|
+
# * For the `R.2X` worker type, each worker maps to 2 M-DPU (8 vCPUs,
|
|
7924
|
+
# 64 GB of memory) with 138GB disk, and provides 1 executor per
|
|
7925
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7926
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7927
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7928
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7929
|
+
# jobs][1] in the developer guide.
|
|
7930
|
+
#
|
|
7931
|
+
# * For the `R.4X` worker type, each worker maps to 4 M-DPU (16 vCPUs,
|
|
7932
|
+
# 128 GB of memory) with 256GB disk, and provides 1 executor per
|
|
7933
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7934
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7935
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7936
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7937
|
+
# jobs][1] in the developer guide.
|
|
7938
|
+
#
|
|
7939
|
+
# * For the `R.8X` worker type, each worker maps to 8 M-DPU (32 vCPUs,
|
|
7940
|
+
# 256 GB of memory) with 512GB disk, and provides 1 executor per
|
|
7941
|
+
# worker. We recommend this worker type for memory-intensive
|
|
7942
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
7943
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
7944
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
7945
|
+
# jobs][1] in the developer guide.
|
|
7848
7946
|
#
|
|
7849
7947
|
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
|
7850
7948
|
# 64 GB of memory) with 128 GB disk, and provides up to 8 Ray
|
|
7851
7949
|
# workers based on the autoscaler.
|
|
7950
|
+
#
|
|
7951
|
+
#
|
|
7952
|
+
#
|
|
7953
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
7852
7954
|
# @return [String]
|
|
7853
7955
|
#
|
|
7854
7956
|
# @!attribute [rw] security_configuration
|
|
@@ -20373,13 +20475,23 @@ module Aws::Glue
|
|
|
20373
20475
|
#
|
|
20374
20476
|
# R Worker Types (Memory-optimized workers):
|
|
20375
20477
|
#
|
|
20376
|
-
# * R.1X: 1 M-DPU (4 vCPUs, 32 GB memory)
|
|
20478
|
+
# * R.1X: 1 M-DPU (4 vCPUs, 32 GB memory, 94GB disk)
|
|
20479
|
+
#
|
|
20480
|
+
# * R.2X: 2 M-DPU (8 vCPUs, 64 GB memory, 138GB disk)
|
|
20481
|
+
#
|
|
20482
|
+
# * R.4X: 4 M-DPU (16 vCPUs, 128 GB memory, 256GB disk)
|
|
20483
|
+
#
|
|
20484
|
+
# * R.8X: 8 M-DPU (32 vCPUs, 256 GB memory, 512GB disk)
|
|
20485
|
+
#
|
|
20486
|
+
# The `G.12X`, `G.16X`, and `R.1X` through `R.8X` worker types are
|
|
20487
|
+
# available only for Glue version 4.0 or later Spark ETL jobs in
|
|
20488
|
+
# select Amazon Web Services Regions. For the supported Regions, see
|
|
20489
|
+
# [Configuring job properties for Spark jobs][1] in the developer
|
|
20490
|
+
# guide.
|
|
20377
20491
|
#
|
|
20378
|
-
# * R.2X: 2 M-DPU (8 vCPUs, 64 GB memory)
|
|
20379
20492
|
#
|
|
20380
|
-
# * R.4X: 4 M-DPU (16 vCPUs, 128 GB memory)
|
|
20381
20493
|
#
|
|
20382
|
-
#
|
|
20494
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
20383
20495
|
# @return [String]
|
|
20384
20496
|
#
|
|
20385
20497
|
# @!attribute [rw] number_of_workers
|
|
@@ -20796,8 +20908,15 @@ module Aws::Glue
|
|
|
20796
20908
|
#
|
|
20797
20909
|
# @!attribute [rw] worker_type
|
|
20798
20910
|
# The type of predefined worker that is allocated when a job runs.
|
|
20799
|
-
# Accepts a value of G.1X, G.2X, G.4X, G.8X
|
|
20800
|
-
# Accepts the value Z.2X for
|
|
20911
|
+
# Accepts a value of G.025X, G.1X, G.2X, G.4X, G.8X, G.12X, G.16X,
|
|
20912
|
+
# R.1X, R.2X, R.4X, or R.8X for Spark jobs. Accepts the value Z.2X for
|
|
20913
|
+
# Ray jobs.
|
|
20914
|
+
#
|
|
20915
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
|
20916
|
+
# vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per
|
|
20917
|
+
# worker. We recommend this worker type for low volume streaming
|
|
20918
|
+
# jobs. This worker type is only available for Glue version 3.0 or
|
|
20919
|
+
# later streaming jobs.
|
|
20801
20920
|
#
|
|
20802
20921
|
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
|
20803
20922
|
# GB of memory) with 94GB disk, and provides 1 executor per worker.
|
|
@@ -20816,29 +20935,74 @@ module Aws::Glue
|
|
|
20816
20935
|
# worker. We recommend this worker type for jobs whose workloads
|
|
20817
20936
|
# contain your most demanding transforms, aggregations, joins, and
|
|
20818
20937
|
# queries. This worker type is available only for Glue version 3.0
|
|
20819
|
-
# or later Spark ETL jobs in
|
|
20820
|
-
#
|
|
20821
|
-
#
|
|
20822
|
-
# (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland),
|
|
20823
|
-
# and Europe (Stockholm).
|
|
20938
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20939
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20940
|
+
# jobs][1] in the developer guide.
|
|
20824
20941
|
#
|
|
20825
20942
|
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
|
20826
20943
|
# 128 GB of memory) with 512GB disk, and provides 1 executor per
|
|
20827
20944
|
# worker. We recommend this worker type for jobs whose workloads
|
|
20828
20945
|
# contain your most demanding transforms, aggregations, joins, and
|
|
20829
20946
|
# queries. This worker type is available only for Glue version 3.0
|
|
20830
|
-
# or later Spark ETL jobs
|
|
20831
|
-
#
|
|
20832
|
-
#
|
|
20833
|
-
#
|
|
20834
|
-
#
|
|
20835
|
-
#
|
|
20836
|
-
#
|
|
20837
|
-
#
|
|
20947
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20948
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20949
|
+
# jobs][1] in the developer guide.
|
|
20950
|
+
#
|
|
20951
|
+
# * For the `G.12X` worker type, each worker maps to 12 DPU (48 vCPUs,
|
|
20952
|
+
# 192 GB of memory) with 768GB disk, and provides 1 executor per
|
|
20953
|
+
# worker. We recommend this worker type for jobs with very large
|
|
20954
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
20955
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20956
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20957
|
+
# jobs][1] in the developer guide.
|
|
20958
|
+
#
|
|
20959
|
+
# * For the `G.16X` worker type, each worker maps to 16 DPU (64 vCPUs,
|
|
20960
|
+
# 256 GB of memory) with 1024GB disk, and provides 1 executor per
|
|
20961
|
+
# worker. We recommend this worker type for jobs with very large
|
|
20962
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
20963
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20964
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20965
|
+
# jobs][1] in the developer guide.
|
|
20966
|
+
#
|
|
20967
|
+
# * For the `R.1X` worker type, each worker maps to 1 M-DPU (4 vCPUs,
|
|
20968
|
+
# 32 GB of memory) with 94GB disk, and provides 1 executor per
|
|
20969
|
+
# worker. We recommend this worker type for memory-intensive
|
|
20970
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
20971
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20972
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20973
|
+
# jobs][1] in the developer guide.
|
|
20974
|
+
#
|
|
20975
|
+
# * For the `R.2X` worker type, each worker maps to 2 M-DPU (8 vCPUs,
|
|
20976
|
+
# 64 GB of memory) with 138GB disk, and provides 1 executor per
|
|
20977
|
+
# worker. We recommend this worker type for memory-intensive
|
|
20978
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
20979
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20980
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20981
|
+
# jobs][1] in the developer guide.
|
|
20982
|
+
#
|
|
20983
|
+
# * For the `R.4X` worker type, each worker maps to 4 M-DPU (16 vCPUs,
|
|
20984
|
+
# 128 GB of memory) with 256GB disk, and provides 1 executor per
|
|
20985
|
+
# worker. We recommend this worker type for memory-intensive
|
|
20986
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
20987
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20988
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20989
|
+
# jobs][1] in the developer guide.
|
|
20990
|
+
#
|
|
20991
|
+
# * For the `R.8X` worker type, each worker maps to 8 M-DPU (32 vCPUs,
|
|
20992
|
+
# 256 GB of memory) with 512GB disk, and provides 1 executor per
|
|
20993
|
+
# worker. We recommend this worker type for memory-intensive
|
|
20994
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
20995
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
20996
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
20997
|
+
# jobs][1] in the developer guide.
|
|
20838
20998
|
#
|
|
20839
20999
|
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
|
20840
21000
|
# 64 GB of memory) with 128 GB disk, and provides up to 8 Ray
|
|
20841
21001
|
# workers based on the autoscaler.
|
|
21002
|
+
#
|
|
21003
|
+
#
|
|
21004
|
+
#
|
|
21005
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
20842
21006
|
# @return [String]
|
|
20843
21007
|
#
|
|
20844
21008
|
# @!attribute [rw] number_of_workers
|
|
@@ -21150,13 +21314,21 @@ module Aws::Glue
|
|
|
21150
21314
|
#
|
|
21151
21315
|
# @!attribute [rw] worker_type
|
|
21152
21316
|
# The type of predefined worker that is allocated when a job runs.
|
|
21153
|
-
# Accepts a value of G.1X, G.2X, G.4X, G.8X
|
|
21154
|
-
#
|
|
21155
|
-
# [Defining job properties for
|
|
21317
|
+
# Accepts a value of G.025X, G.1X, G.2X, G.4X, G.8X, G.12X, G.16X,
|
|
21318
|
+
# R.1X, R.2X, R.4X, or R.8X for Spark jobs. Accepts the value Z.2X for
|
|
21319
|
+
# Ray jobs. For more information, see [Defining job properties for
|
|
21320
|
+
# Spark jobs ][1]
|
|
21321
|
+
#
|
|
21322
|
+
# The `G.12X`, `G.16X`, and `R.1X` through `R.8X` worker types are
|
|
21323
|
+
# available only for Glue version 4.0 or later Spark ETL jobs in
|
|
21324
|
+
# select Amazon Web Services Regions. For the supported Regions, see
|
|
21325
|
+
# [Configuring job properties for Spark jobs][2] in the developer
|
|
21326
|
+
# guide.
|
|
21156
21327
|
#
|
|
21157
21328
|
#
|
|
21158
21329
|
#
|
|
21159
21330
|
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html#create-job
|
|
21331
|
+
# [2]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
21160
21332
|
# @return [String]
|
|
21161
21333
|
#
|
|
21162
21334
|
# @!attribute [rw] number_of_workers
|
|
@@ -28998,8 +29170,19 @@ module Aws::Glue
|
|
|
28998
29170
|
#
|
|
28999
29171
|
# @!attribute [rw] worker_type
|
|
29000
29172
|
# The type of predefined worker that is allocated when a session runs.
|
|
29001
|
-
# Accepts a value of `G.1X`, `G.2X`, `G.4X`,
|
|
29002
|
-
#
|
|
29173
|
+
# Accepts a value of `G.1X`, `G.2X`, `G.4X`, `G.8X`, `G.12X`, `G.16X`,
|
|
29174
|
+
# `R.1X`, `R.2X`, `R.4X`, or `R.8X` for Spark sessions. Accepts the
|
|
29175
|
+
# value `Z.2X` for Ray sessions.
|
|
29176
|
+
#
|
|
29177
|
+
# The `G.12X`, `G.16X`, and `R.1X` through `R.8X` worker types are
|
|
29178
|
+
# available only for Glue version 4.0 or later Spark ETL jobs in
|
|
29179
|
+
# select Amazon Web Services Regions. For the supported Regions, see
|
|
29180
|
+
# [Configuring job properties for Spark jobs][1] in the developer
|
|
29181
|
+
# guide.
|
|
29182
|
+
#
|
|
29183
|
+
#
|
|
29184
|
+
#
|
|
29185
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
29003
29186
|
# @return [String]
|
|
29004
29187
|
#
|
|
29005
29188
|
# @!attribute [rw] completed_on
|
|
@@ -30198,8 +30381,15 @@ module Aws::Glue
|
|
|
30198
30381
|
#
|
|
30199
30382
|
# @!attribute [rw] worker_type
|
|
30200
30383
|
# The type of predefined worker that is allocated when a job runs.
|
|
30201
|
-
# Accepts a value of G.1X, G.2X, G.4X, G.8X
|
|
30202
|
-
# Accepts the value Z.2X for
|
|
30384
|
+
# Accepts a value of G.025X, G.1X, G.2X, G.4X, G.8X, G.12X, G.16X,
|
|
30385
|
+
# R.1X, R.2X, R.4X, or R.8X for Spark jobs. Accepts the value Z.2X for
|
|
30386
|
+
# Ray jobs.
|
|
30387
|
+
#
|
|
30388
|
+
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2
|
|
30389
|
+
# vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per
|
|
30390
|
+
# worker. We recommend this worker type for low volume streaming
|
|
30391
|
+
# jobs. This worker type is only available for Glue version 3.0 or
|
|
30392
|
+
# later streaming jobs.
|
|
30203
30393
|
#
|
|
30204
30394
|
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPUs, 16
|
|
30205
30395
|
# GB of memory) with 94GB disk, and provides 1 executor per worker.
|
|
@@ -30218,29 +30408,74 @@ module Aws::Glue
|
|
|
30218
30408
|
# worker. We recommend this worker type for jobs whose workloads
|
|
30219
30409
|
# contain your most demanding transforms, aggregations, joins, and
|
|
30220
30410
|
# queries. This worker type is available only for Glue version 3.0
|
|
30221
|
-
# or later Spark ETL jobs in
|
|
30222
|
-
#
|
|
30223
|
-
#
|
|
30224
|
-
# (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland),
|
|
30225
|
-
# and Europe (Stockholm).
|
|
30411
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30412
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30413
|
+
# jobs][1] in the developer guide.
|
|
30226
30414
|
#
|
|
30227
30415
|
# * For the `G.8X` worker type, each worker maps to 8 DPU (32 vCPUs,
|
|
30228
30416
|
# 128 GB of memory) with 512GB disk, and provides 1 executor per
|
|
30229
30417
|
# worker. We recommend this worker type for jobs whose workloads
|
|
30230
30418
|
# contain your most demanding transforms, aggregations, joins, and
|
|
30231
30419
|
# queries. This worker type is available only for Glue version 3.0
|
|
30232
|
-
# or later Spark ETL jobs
|
|
30233
|
-
#
|
|
30234
|
-
#
|
|
30235
|
-
#
|
|
30236
|
-
#
|
|
30237
|
-
#
|
|
30238
|
-
#
|
|
30239
|
-
#
|
|
30420
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30421
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30422
|
+
# jobs][1] in the developer guide.
|
|
30423
|
+
#
|
|
30424
|
+
# * For the `G.12X` worker type, each worker maps to 12 DPU (48 vCPUs,
|
|
30425
|
+
# 192 GB of memory) with 768GB disk, and provides 1 executor per
|
|
30426
|
+
# worker. We recommend this worker type for jobs with very large
|
|
30427
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
30428
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30429
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30430
|
+
# jobs][1] in the developer guide.
|
|
30431
|
+
#
|
|
30432
|
+
# * For the `G.16X` worker type, each worker maps to 16 DPU (64 vCPUs,
|
|
30433
|
+
# 256 GB of memory) with 1024GB disk, and provides 1 executor per
|
|
30434
|
+
# worker. We recommend this worker type for jobs with very large
|
|
30435
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
30436
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30437
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30438
|
+
# jobs][1] in the developer guide.
|
|
30439
|
+
#
|
|
30440
|
+
# * For the `R.1X` worker type, each worker maps to 1 M-DPU (4 vCPUs,
|
|
30441
|
+
# 32 GB of memory) with 94GB disk, and provides 1 executor per
|
|
30442
|
+
# worker. We recommend this worker type for memory-intensive
|
|
30443
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
30444
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30445
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30446
|
+
# jobs][1] in the developer guide.
|
|
30447
|
+
#
|
|
30448
|
+
# * For the `R.2X` worker type, each worker maps to 2 M-DPU (8 vCPUs,
|
|
30449
|
+
# 64 GB of memory) with 138GB disk, and provides 1 executor per
|
|
30450
|
+
# worker. We recommend this worker type for memory-intensive
|
|
30451
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
30452
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30453
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30454
|
+
# jobs][1] in the developer guide.
|
|
30455
|
+
#
|
|
30456
|
+
# * For the `R.4X` worker type, each worker maps to 4 M-DPU (16 vCPUs,
|
|
30457
|
+
# 128 GB of memory) with 256GB disk, and provides 1 executor per
|
|
30458
|
+
# worker. We recommend this worker type for memory-intensive
|
|
30459
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
30460
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30461
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30462
|
+
# jobs][1] in the developer guide.
|
|
30463
|
+
#
|
|
30464
|
+
# * For the `R.8X` worker type, each worker maps to 8 M-DPU (32 vCPUs,
|
|
30465
|
+
# 256 GB of memory) with 512GB disk, and provides 1 executor per
|
|
30466
|
+
# worker. We recommend this worker type for memory-intensive
|
|
30467
|
+
# workloads. This worker type is available only for Glue version 4.0
|
|
30468
|
+
# or later Spark ETL jobs in select Amazon Web Services Regions. For
|
|
30469
|
+
# the supported Regions, see [Configuring job properties for Spark
|
|
30470
|
+
# jobs][1] in the developer guide.
|
|
30240
30471
|
#
|
|
30241
30472
|
# * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPUs,
|
|
30242
30473
|
# 64 GB of memory) with 128 GB disk, and provides up to 8 Ray
|
|
30243
30474
|
# workers based on the autoscaler.
|
|
30475
|
+
#
|
|
30476
|
+
#
|
|
30477
|
+
#
|
|
30478
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
|
30244
30479
|
# @return [String]
|
|
30245
30480
|
#
|
|
30246
30481
|
# @!attribute [rw] number_of_workers
|
data/lib/aws-sdk-glue.rb
CHANGED
data/sig/client.rbs
CHANGED
|
@@ -617,7 +617,7 @@ module Aws
|
|
|
617
617
|
def yarn_endpoint_address: () -> ::String
|
|
618
618
|
def zeppelin_remote_spark_interpreter_port: () -> ::Integer
|
|
619
619
|
def number_of_nodes: () -> ::Integer
|
|
620
|
-
def worker_type: () -> ("Standard" | "G.1X" | "G.2X" | "G.
|
|
620
|
+
def worker_type: () -> ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X")
|
|
621
621
|
def glue_version: () -> ::String
|
|
622
622
|
def number_of_workers: () -> ::Integer
|
|
623
623
|
def availability_zone: () -> ::String
|
|
@@ -638,7 +638,7 @@ module Aws
|
|
|
638
638
|
?public_key: ::String,
|
|
639
639
|
?public_keys: Array[::String],
|
|
640
640
|
?number_of_nodes: ::Integer,
|
|
641
|
-
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
641
|
+
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X"),
|
|
642
642
|
?glue_version: ::String,
|
|
643
643
|
?number_of_workers: ::Integer,
|
|
644
644
|
?extra_python_libs_s3_path: ::String,
|
|
@@ -815,7 +815,7 @@ module Aws
|
|
|
815
815
|
},
|
|
816
816
|
?glue_version: ::String,
|
|
817
817
|
?number_of_workers: ::Integer,
|
|
818
|
-
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
818
|
+
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X"),
|
|
819
819
|
?code_gen_configuration_nodes: Hash[::String, Params::code_gen_configuration_node],
|
|
820
820
|
?execution_class: ("FLEX" | "STANDARD"),
|
|
821
821
|
?source_control_details: Params::source_control_details,
|
|
@@ -844,7 +844,7 @@ module Aws
|
|
|
844
844
|
role: ::String,
|
|
845
845
|
?glue_version: ::String,
|
|
846
846
|
?max_capacity: ::Float,
|
|
847
|
-
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
847
|
+
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X"),
|
|
848
848
|
?number_of_workers: ::Integer,
|
|
849
849
|
?timeout: ::Integer,
|
|
850
850
|
?max_retries: ::Integer,
|
|
@@ -1017,7 +1017,7 @@ module Aws
|
|
|
1017
1017
|
},
|
|
1018
1018
|
?max_capacity: ::Float,
|
|
1019
1019
|
?number_of_workers: ::Integer,
|
|
1020
|
-
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
1020
|
+
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X"),
|
|
1021
1021
|
?security_configuration: ::String,
|
|
1022
1022
|
?glue_version: ::String,
|
|
1023
1023
|
?tags: Hash[::String, ::String],
|
|
@@ -2373,7 +2373,7 @@ module Aws
|
|
|
2373
2373
|
def role: () -> ::String
|
|
2374
2374
|
def glue_version: () -> ::String
|
|
2375
2375
|
def max_capacity: () -> ::Float
|
|
2376
|
-
def worker_type: () -> ("Standard" | "G.1X" | "G.2X" | "G.
|
|
2376
|
+
def worker_type: () -> ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X")
|
|
2377
2377
|
def number_of_workers: () -> ::Integer
|
|
2378
2378
|
def timeout: () -> ::Integer
|
|
2379
2379
|
def max_retries: () -> ::Integer
|
|
@@ -4349,7 +4349,7 @@ module Aws
|
|
|
4349
4349
|
?notification_property: {
|
|
4350
4350
|
notify_delay_after: ::Integer?
|
|
4351
4351
|
},
|
|
4352
|
-
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
4352
|
+
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X"),
|
|
4353
4353
|
?number_of_workers: ::Integer,
|
|
4354
4354
|
?execution_class: ("FLEX" | "STANDARD"),
|
|
4355
4355
|
?execution_role_session_policy: ::String
|
|
@@ -4859,7 +4859,7 @@ module Aws
|
|
|
4859
4859
|
allocated_capacity: ::Integer?,
|
|
4860
4860
|
timeout: ::Integer?,
|
|
4861
4861
|
max_capacity: ::Float?,
|
|
4862
|
-
worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
4862
|
+
worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X")?,
|
|
4863
4863
|
number_of_workers: ::Integer?,
|
|
4864
4864
|
security_configuration: ::String?,
|
|
4865
4865
|
notification_property: {
|
|
@@ -4905,7 +4905,7 @@ module Aws
|
|
|
4905
4905
|
?role: ::String,
|
|
4906
4906
|
?glue_version: ::String,
|
|
4907
4907
|
?max_capacity: ::Float,
|
|
4908
|
-
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.
|
|
4908
|
+
?worker_type: ("Standard" | "G.1X" | "G.2X" | "G.4X" | "G.8X" | "G.025X" | "Z.2X" | "G.12X" | "G.16X" | "R.1X" | "R.2X" | "R.4X" | "R.8X"),
|
|
4909
4909
|
?number_of_workers: ::Integer,
|
|
4910
4910
|
?timeout: ::Integer,
|
|
4911
4911
|
?max_retries: ::Integer
|