aws-sdk-glue 1.84.0 → 1.89.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +469 -0
- data/LICENSE.txt +202 -0
- data/VERSION +1 -0
- data/lib/aws-sdk-glue.rb +2 -2
- data/lib/aws-sdk-glue/client.rb +317 -291
- data/lib/aws-sdk-glue/client_api.rb +11 -1
- data/lib/aws-sdk-glue/errors.rb +1 -1
- data/lib/aws-sdk-glue/resource.rb +1 -1
- data/lib/aws-sdk-glue/types.rb +449 -373
- metadata +9 -7
@@ -3,7 +3,7 @@
|
|
3
3
|
# WARNING ABOUT GENERATED CODE
|
4
4
|
#
|
5
5
|
# This file is generated. See the contributing guide for more information:
|
6
|
-
# https://github.com/aws/aws-sdk-ruby/blob/
|
6
|
+
# https://github.com/aws/aws-sdk-ruby/blob/version-3/CONTRIBUTING.md
|
7
7
|
#
|
8
8
|
# WARNING ABOUT GENERATED CODE
|
9
9
|
|
@@ -516,6 +516,8 @@ module Aws::Glue
|
|
516
516
|
OrchestrationStringList = Shapes::ListShape.new(name: 'OrchestrationStringList')
|
517
517
|
Order = Shapes::StructureShape.new(name: 'Order')
|
518
518
|
OrderList = Shapes::ListShape.new(name: 'OrderList')
|
519
|
+
OtherMetadataValueList = Shapes::ListShape.new(name: 'OtherMetadataValueList')
|
520
|
+
OtherMetadataValueListItem = Shapes::StructureShape.new(name: 'OtherMetadataValueListItem')
|
519
521
|
PageSize = Shapes::IntegerShape.new(name: 'PageSize')
|
520
522
|
PaginationToken = Shapes::StringShape.new(name: 'PaginationToken')
|
521
523
|
ParametersMap = Shapes::MapShape.new(name: 'ParametersMap')
|
@@ -2576,6 +2578,7 @@ module Aws::Glue
|
|
2576
2578
|
|
2577
2579
|
MetadataInfo.add_member(:metadata_value, Shapes::ShapeRef.new(shape: MetadataValueString, location_name: "MetadataValue"))
|
2578
2580
|
MetadataInfo.add_member(:created_time, Shapes::ShapeRef.new(shape: CreatedTimestamp, location_name: "CreatedTime"))
|
2581
|
+
MetadataInfo.add_member(:other_metadata_value_list, Shapes::ShapeRef.new(shape: OtherMetadataValueList, location_name: "OtherMetadataValueList"))
|
2579
2582
|
MetadataInfo.struct_class = Types::MetadataInfo
|
2580
2583
|
|
2581
2584
|
MetadataInfoMap.key = Shapes::ShapeRef.new(shape: MetadataKeyString)
|
@@ -2625,6 +2628,12 @@ module Aws::Glue
|
|
2625
2628
|
|
2626
2629
|
OrderList.member = Shapes::ShapeRef.new(shape: Order)
|
2627
2630
|
|
2631
|
+
OtherMetadataValueList.member = Shapes::ShapeRef.new(shape: OtherMetadataValueListItem)
|
2632
|
+
|
2633
|
+
OtherMetadataValueListItem.add_member(:metadata_value, Shapes::ShapeRef.new(shape: MetadataValueString, location_name: "MetadataValue"))
|
2634
|
+
OtherMetadataValueListItem.add_member(:created_time, Shapes::ShapeRef.new(shape: CreatedTimestamp, location_name: "CreatedTime"))
|
2635
|
+
OtherMetadataValueListItem.struct_class = Types::OtherMetadataValueListItem
|
2636
|
+
|
2628
2637
|
ParametersMap.key = Shapes::ShapeRef.new(shape: KeyString)
|
2629
2638
|
ParametersMap.value = Shapes::ShapeRef.new(shape: ParametersMapValue)
|
2630
2639
|
|
@@ -2833,6 +2842,7 @@ module Aws::Glue
|
|
2833
2842
|
S3Target.add_member(:path, Shapes::ShapeRef.new(shape: Path, location_name: "Path"))
|
2834
2843
|
S3Target.add_member(:exclusions, Shapes::ShapeRef.new(shape: PathList, location_name: "Exclusions"))
|
2835
2844
|
S3Target.add_member(:connection_name, Shapes::ShapeRef.new(shape: ConnectionName, location_name: "ConnectionName"))
|
2845
|
+
S3Target.add_member(:sample_size, Shapes::ShapeRef.new(shape: NullableInteger, location_name: "SampleSize"))
|
2836
2846
|
S3Target.struct_class = Types::S3Target
|
2837
2847
|
|
2838
2848
|
S3TargetList.member = Shapes::ShapeRef.new(shape: S3Target)
|
data/lib/aws-sdk-glue/errors.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# WARNING ABOUT GENERATED CODE
|
4
4
|
#
|
5
5
|
# This file is generated. See the contributing guide for more information:
|
6
|
-
# https://github.com/aws/aws-sdk-ruby/blob/
|
6
|
+
# https://github.com/aws/aws-sdk-ruby/blob/version-3/CONTRIBUTING.md
|
7
7
|
#
|
8
8
|
# WARNING ABOUT GENERATED CODE
|
9
9
|
|
@@ -3,7 +3,7 @@
|
|
3
3
|
# WARNING ABOUT GENERATED CODE
|
4
4
|
#
|
5
5
|
# This file is generated. See the contributing guide for more information:
|
6
|
-
# https://github.com/aws/aws-sdk-ruby/blob/
|
6
|
+
# https://github.com/aws/aws-sdk-ruby/blob/version-3/CONTRIBUTING.md
|
7
7
|
#
|
8
8
|
# WARNING ABOUT GENERATED CODE
|
9
9
|
|
data/lib/aws-sdk-glue/types.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# WARNING ABOUT GENERATED CODE
|
4
4
|
#
|
5
5
|
# This file is generated. See the contributing guide for more information:
|
6
|
-
# https://github.com/aws/aws-sdk-ruby/blob/
|
6
|
+
# https://github.com/aws/aws-sdk-ruby/blob/version-3/CONTRIBUTING.md
|
7
7
|
#
|
8
8
|
# WARNING ABOUT GENERATED CODE
|
9
9
|
|
@@ -43,7 +43,7 @@ module Aws::Glue
|
|
43
43
|
# }
|
44
44
|
#
|
45
45
|
# @!attribute [rw] job_name
|
46
|
-
# The name of a job to be
|
46
|
+
# The name of a job to be run.
|
47
47
|
# @return [String]
|
48
48
|
#
|
49
49
|
# @!attribute [rw] arguments
|
@@ -51,15 +51,15 @@ module Aws::Glue
|
|
51
51
|
# they replace the default arguments set in the job definition itself.
|
52
52
|
#
|
53
53
|
# You can specify arguments here that your own job-execution script
|
54
|
-
# consumes, as well as arguments that
|
54
|
+
# consumes, as well as arguments that Glue itself consumes.
|
55
55
|
#
|
56
56
|
# For information about how to specify and consume your own Job
|
57
|
-
# arguments, see the [Calling
|
57
|
+
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
58
58
|
# developer guide.
|
59
59
|
#
|
60
|
-
# For information about the key-value pairs that
|
61
|
-
#
|
62
|
-
#
|
60
|
+
# For information about the key-value pairs that Glue consumes to set
|
61
|
+
# up your job, see the [Special Parameters Used by Glue][2] topic in
|
62
|
+
# the developer guide.
|
63
63
|
#
|
64
64
|
#
|
65
65
|
#
|
@@ -225,7 +225,7 @@ module Aws::Glue
|
|
225
225
|
#
|
226
226
|
# @!attribute [rw] catalog_id
|
227
227
|
# The ID of the catalog in which the partition is to be created.
|
228
|
-
# Currently, this should be the
|
228
|
+
# Currently, this should be the Amazon Web Services account ID.
|
229
229
|
# @return [String]
|
230
230
|
#
|
231
231
|
# @!attribute [rw] database_name
|
@@ -277,7 +277,7 @@ module Aws::Glue
|
|
277
277
|
#
|
278
278
|
# @!attribute [rw] catalog_id
|
279
279
|
# The ID of the Data Catalog in which the connections reside. If none
|
280
|
-
# is provided, the
|
280
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
281
281
|
# @return [String]
|
282
282
|
#
|
283
283
|
# @!attribute [rw] connection_name_list
|
@@ -328,7 +328,8 @@ module Aws::Glue
|
|
328
328
|
#
|
329
329
|
# @!attribute [rw] catalog_id
|
330
330
|
# The ID of the Data Catalog where the partition to be deleted
|
331
|
-
# resides. If none is provided, the
|
331
|
+
# resides. If none is provided, the Amazon Web Services account ID is
|
332
|
+
# used by default.
|
332
333
|
# @return [String]
|
333
334
|
#
|
334
335
|
# @!attribute [rw] database_name
|
@@ -380,7 +381,7 @@ module Aws::Glue
|
|
380
381
|
#
|
381
382
|
# @!attribute [rw] catalog_id
|
382
383
|
# The ID of the Data Catalog where the table resides. If none is
|
383
|
-
# provided, the
|
384
|
+
# provided, the Amazon Web Services account ID is used by default.
|
384
385
|
# @return [String]
|
385
386
|
#
|
386
387
|
# @!attribute [rw] database_name
|
@@ -427,7 +428,7 @@ module Aws::Glue
|
|
427
428
|
#
|
428
429
|
# @!attribute [rw] catalog_id
|
429
430
|
# The ID of the Data Catalog where the tables reside. If none is
|
430
|
-
# provided, the
|
431
|
+
# provided, the Amazon Web Services account ID is used by default.
|
431
432
|
# @return [String]
|
432
433
|
#
|
433
434
|
# @!attribute [rw] database_name
|
@@ -597,7 +598,8 @@ module Aws::Glue
|
|
597
598
|
#
|
598
599
|
# @!attribute [rw] catalog_id
|
599
600
|
# The ID of the Data Catalog where the partitions in question reside.
|
600
|
-
# If none is supplied, the
|
601
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
602
|
+
# default.
|
601
603
|
# @return [String]
|
602
604
|
#
|
603
605
|
# @!attribute [rw] database_name
|
@@ -906,7 +908,7 @@ module Aws::Glue
|
|
906
908
|
#
|
907
909
|
# @!attribute [rw] catalog_id
|
908
910
|
# The ID of the catalog in which the partition is to be updated.
|
909
|
-
# Currently, this should be the
|
911
|
+
# Currently, this should be the Amazon Web Services account ID.
|
910
912
|
# @return [String]
|
911
913
|
#
|
912
914
|
# @!attribute [rw] database_name
|
@@ -1147,7 +1149,7 @@ module Aws::Glue
|
|
1147
1149
|
include Aws::Structure
|
1148
1150
|
end
|
1149
1151
|
|
1150
|
-
# Specifies a table definition in the
|
1152
|
+
# Specifies a table definition in the Glue Data Catalog.
|
1151
1153
|
#
|
1152
1154
|
# @note When making an API call, you may pass CatalogEntry
|
1153
1155
|
# data as a hash:
|
@@ -1198,7 +1200,7 @@ module Aws::Glue
|
|
1198
1200
|
include Aws::Structure
|
1199
1201
|
end
|
1200
1202
|
|
1201
|
-
# Specifies an
|
1203
|
+
# Specifies an Glue Data Catalog target.
|
1202
1204
|
#
|
1203
1205
|
# @note When making an API call, you may pass CatalogTarget
|
1204
1206
|
# data as a hash:
|
@@ -1229,13 +1231,13 @@ module Aws::Glue
|
|
1229
1231
|
# data as a hash:
|
1230
1232
|
#
|
1231
1233
|
# {
|
1232
|
-
# data_format: "AVRO", # required, accepts AVRO
|
1234
|
+
# data_format: "AVRO", # required, accepts AVRO, JSON
|
1233
1235
|
# schema_definition: "SchemaDefinitionString", # required
|
1234
1236
|
# }
|
1235
1237
|
#
|
1236
1238
|
# @!attribute [rw] data_format
|
1237
|
-
# The data format of the schema definition. Currently
|
1238
|
-
# supported.
|
1239
|
+
# The data format of the schema definition. Currently `AVRO` and
|
1240
|
+
# `JSON` are supported.
|
1239
1241
|
# @return [String]
|
1240
1242
|
#
|
1241
1243
|
# @!attribute [rw] schema_definition
|
@@ -1273,10 +1275,10 @@ module Aws::Glue
|
|
1273
1275
|
# classifier creates a schema in the form of a `StructType` object that
|
1274
1276
|
# matches that data format.
|
1275
1277
|
#
|
1276
|
-
# You can use the standard classifiers that
|
1277
|
-
#
|
1278
|
-
#
|
1279
|
-
#
|
1278
|
+
# You can use the standard classifiers that Glue provides, or you can
|
1279
|
+
# write your own classifiers to best categorize your data sources and
|
1280
|
+
# specify the appropriate schemas to use for them. A classifier can be a
|
1281
|
+
# `grok` classifier, an `XML` classifier, a `JSON` classifier, or a
|
1280
1282
|
# custom `CSV` classifier, as specified in one of the fields in the
|
1281
1283
|
# `Classifier` object.
|
1282
1284
|
#
|
@@ -1948,20 +1950,20 @@ module Aws::Glue
|
|
1948
1950
|
# false.
|
1949
1951
|
#
|
1950
1952
|
# * `CUSTOM_JDBC_CERT` - An Amazon S3 location specifying the
|
1951
|
-
# customer's root certificate.
|
1952
|
-
#
|
1953
|
-
# customer database.
|
1953
|
+
# customer's root certificate. Glue uses this root certificate to
|
1954
|
+
# validate the customer’s certificate when connecting to the
|
1955
|
+
# customer database. Glue only handles X.509 certificates. The
|
1954
1956
|
# certificate provided must be DER-encoded and supplied in Base64
|
1955
1957
|
# encoding PEM format.
|
1956
1958
|
#
|
1957
1959
|
# * `SKIP_CUSTOM_JDBC_CERT_VALIDATION` - By default, this is `false`.
|
1958
|
-
#
|
1960
|
+
# Glue validates the Signature algorithm and Subject Public Key
|
1959
1961
|
# Algorithm for the customer certificate. The only permitted
|
1960
1962
|
# algorithms for the Signature algorithm are SHA256withRSA,
|
1961
1963
|
# SHA384withRSA or SHA512withRSA. For the Subject Public Key
|
1962
1964
|
# Algorithm, the key length must be at least 2048. You can set the
|
1963
|
-
# value of this property to `true` to skip
|
1964
|
-
#
|
1965
|
+
# value of this property to `true` to skip Glue’s validation of the
|
1966
|
+
# customer certificate.
|
1965
1967
|
#
|
1966
1968
|
# * `CUSTOM_JDBC_CERT_STRING` - A custom JDBC certificate string which
|
1967
1969
|
# is used for domain match or distinguished name match to prevent a
|
@@ -1984,8 +1986,8 @@ module Aws::Glue
|
|
1984
1986
|
# file (.pem format). The default is an empty string.
|
1985
1987
|
#
|
1986
1988
|
# * `KAFKA_SKIP_CUSTOM_CERT_VALIDATION` - Whether to skip the
|
1987
|
-
# validation of the CA cert file or not.
|
1988
|
-
#
|
1989
|
+
# validation of the CA cert file or not. Glue validates for three
|
1990
|
+
# algorithms: SHA256withRSA, SHA384withRSA and SHA512withRSA.
|
1989
1991
|
# Default value is "false".
|
1990
1992
|
#
|
1991
1993
|
# * `SECRET_ID` - The secret ID used for the secret manager of
|
@@ -1999,6 +2001,24 @@ module Aws::Glue
|
|
1999
2001
|
#
|
2000
2002
|
# * `CONNECTOR_CLASS_NAME` - The connector class name for a
|
2001
2003
|
# MARKETPLACE or CUSTOM connection.
|
2004
|
+
#
|
2005
|
+
# * `KAFKA_CLIENT_KEYSTORE` - The Amazon S3 location of the client
|
2006
|
+
# keystore file for Kafka client side authentication (Optional).
|
2007
|
+
#
|
2008
|
+
# * `KAFKA_CLIENT_KEYSTORE_PASSWORD` - The password to access the
|
2009
|
+
# provided keystore (Optional).
|
2010
|
+
#
|
2011
|
+
# * `KAFKA_CLIENT_KEY_PASSWORD` - A keystore can consist of multiple
|
2012
|
+
# keys, so this is the password to access the client key to be used
|
2013
|
+
# with the Kafka server side key (Optional).
|
2014
|
+
#
|
2015
|
+
# * `ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD` - The encrypted version
|
2016
|
+
# of the Kafka client keystore password (if the user has the Glue
|
2017
|
+
# encrypt passwords setting selected).
|
2018
|
+
#
|
2019
|
+
# * `ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD` - The encrypted version of
|
2020
|
+
# the Kafka client key password (if the user has the Glue encrypt
|
2021
|
+
# passwords setting selected).
|
2002
2022
|
# @return [Hash<String,String>]
|
2003
2023
|
#
|
2004
2024
|
# @!attribute [rw] physical_connection_requirements
|
@@ -2080,12 +2100,12 @@ module Aws::Glue
|
|
2080
2100
|
# within an Amazon Virtual Private Cloud environment (Amazon VPC).
|
2081
2101
|
#
|
2082
2102
|
# * `MARKETPLACE` - Uses configuration settings contained in a
|
2083
|
-
# connector purchased from
|
2084
|
-
# data stores that are not natively supported by
|
2103
|
+
# connector purchased from Marketplace to read from and write to
|
2104
|
+
# data stores that are not natively supported by Glue.
|
2085
2105
|
#
|
2086
2106
|
# * `CUSTOM` - Uses configuration settings contained in a custom
|
2087
2107
|
# connector to read from and write to data stores that are not
|
2088
|
-
# natively supported by
|
2108
|
+
# natively supported by Glue.
|
2089
2109
|
#
|
2090
2110
|
# SFTP is not supported.
|
2091
2111
|
# @return [String]
|
@@ -2123,14 +2143,14 @@ module Aws::Glue
|
|
2123
2143
|
# enable catalog encryption or only password encryption.
|
2124
2144
|
#
|
2125
2145
|
# When a `CreationConnection` request arrives containing a password, the
|
2126
|
-
# Data Catalog first encrypts the password using your
|
2127
|
-
#
|
2128
|
-
#
|
2146
|
+
# Data Catalog first encrypts the password using your KMS key. It then
|
2147
|
+
# encrypts the whole connection object again if catalog encryption is
|
2148
|
+
# also enabled.
|
2129
2149
|
#
|
2130
|
-
# This encryption requires that you set
|
2131
|
-
#
|
2132
|
-
#
|
2133
|
-
#
|
2150
|
+
# This encryption requires that you set KMS key permissions to enable or
|
2151
|
+
# restrict access on the password key according to your security
|
2152
|
+
# requirements. For example, you might want only administrators to have
|
2153
|
+
# decrypt permission on the password key.
|
2134
2154
|
#
|
2135
2155
|
# @note When making an API call, you may pass ConnectionPasswordEncryption
|
2136
2156
|
# data as a hash:
|
@@ -2148,11 +2168,11 @@ module Aws::Glue
|
|
2148
2168
|
# @return [Boolean]
|
2149
2169
|
#
|
2150
2170
|
# @!attribute [rw] aws_kms_key_id
|
2151
|
-
# An
|
2171
|
+
# An KMS key that is used to encrypt the connection password.
|
2152
2172
|
#
|
2153
2173
|
# If connection password protection is enabled, the caller of
|
2154
2174
|
# `CreateConnection` and `UpdateConnection` needs at least
|
2155
|
-
# `kms:Encrypt` permission on the specified
|
2175
|
+
# `kms:Encrypt` permission on the specified KMS key, to encrypt
|
2156
2176
|
# passwords before storing them in the Data Catalog.
|
2157
2177
|
#
|
2158
2178
|
# You can set the decrypt permission to enable or restrict access on
|
@@ -2230,8 +2250,7 @@ module Aws::Glue
|
|
2230
2250
|
|
2231
2251
|
# Specifies a crawler program that examines a data source and uses
|
2232
2252
|
# classifiers to try to determine its schema. If successful, the crawler
|
2233
|
-
# records metadata concerning the data source in the
|
2234
|
-
# Catalog.
|
2253
|
+
# records metadata concerning the data source in the Glue Data Catalog.
|
2235
2254
|
#
|
2236
2255
|
# @!attribute [rw] name
|
2237
2256
|
# The name of the crawler.
|
@@ -2313,11 +2332,11 @@ module Aws::Glue
|
|
2313
2332
|
# @!attribute [rw] configuration
|
2314
2333
|
# Crawler configuration information. This versioned JSON string allows
|
2315
2334
|
# users to specify aspects of a crawler's behavior. For more
|
2316
|
-
# information, see [
|
2335
|
+
# information, see [Include and Exclude Patterns][1].
|
2317
2336
|
#
|
2318
2337
|
#
|
2319
2338
|
#
|
2320
|
-
# [1]: https://docs.aws.amazon.com/glue/latest/dg/crawler
|
2339
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/define-crawler.html#crawler-data-stores-exclude
|
2321
2340
|
# @return [String]
|
2322
2341
|
#
|
2323
2342
|
# @!attribute [rw] crawler_security_configuration
|
@@ -2469,6 +2488,7 @@ module Aws::Glue
|
|
2469
2488
|
# path: "Path",
|
2470
2489
|
# exclusions: ["Path"],
|
2471
2490
|
# connection_name: "ConnectionName",
|
2491
|
+
# sample_size: 1,
|
2472
2492
|
# },
|
2473
2493
|
# ],
|
2474
2494
|
# jdbc_targets: [
|
@@ -2517,7 +2537,7 @@ module Aws::Glue
|
|
2517
2537
|
# @return [Array<Types::DynamoDBTarget>]
|
2518
2538
|
#
|
2519
2539
|
# @!attribute [rw] catalog_targets
|
2520
|
-
# Specifies
|
2540
|
+
# Specifies Glue Data Catalog targets.
|
2521
2541
|
# @return [Array<Types::CatalogTarget>]
|
2522
2542
|
#
|
2523
2543
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CrawlerTargets AWS API Documentation
|
@@ -2616,7 +2636,8 @@ module Aws::Glue
|
|
2616
2636
|
#
|
2617
2637
|
# @!attribute [rw] catalog_id
|
2618
2638
|
# The ID of the Data Catalog in which to create the connection. If
|
2619
|
-
# none is provided, the
|
2639
|
+
# none is provided, the Amazon Web Services account ID is used by
|
2640
|
+
# default.
|
2620
2641
|
# @return [String]
|
2621
2642
|
#
|
2622
2643
|
# @!attribute [rw] connection_input
|
@@ -2650,6 +2671,7 @@ module Aws::Glue
|
|
2650
2671
|
# path: "Path",
|
2651
2672
|
# exclusions: ["Path"],
|
2652
2673
|
# connection_name: "ConnectionName",
|
2674
|
+
# sample_size: 1,
|
2653
2675
|
# },
|
2654
2676
|
# ],
|
2655
2677
|
# jdbc_targets: [
|
@@ -2710,7 +2732,7 @@ module Aws::Glue
|
|
2710
2732
|
# @return [String]
|
2711
2733
|
#
|
2712
2734
|
# @!attribute [rw] database_name
|
2713
|
-
# The
|
2735
|
+
# The Glue database where results are written, such as:
|
2714
2736
|
# `arn:aws:daylight:us-east-1::database/sometable/*`.
|
2715
2737
|
# @return [String]
|
2716
2738
|
#
|
@@ -2773,8 +2795,8 @@ module Aws::Glue
|
|
2773
2795
|
#
|
2774
2796
|
# @!attribute [rw] tags
|
2775
2797
|
# The tags to use with this crawler request. You may use tags to limit
|
2776
|
-
# access to the crawler. For more information about tags in
|
2777
|
-
#
|
2798
|
+
# access to the crawler. For more information about tags in Glue, see
|
2799
|
+
# [Amazon Web Services Tags in Glue][1] in the developer guide.
|
2778
2800
|
#
|
2779
2801
|
#
|
2780
2802
|
#
|
@@ -2895,7 +2917,7 @@ module Aws::Glue
|
|
2895
2917
|
#
|
2896
2918
|
# @!attribute [rw] catalog_id
|
2897
2919
|
# The ID of the Data Catalog in which to create the database. If none
|
2898
|
-
# is provided, the
|
2920
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
2899
2921
|
# @return [String]
|
2900
2922
|
#
|
2901
2923
|
# @!attribute [rw] database_input
|
@@ -2979,8 +3001,8 @@ module Aws::Glue
|
|
2979
3001
|
# @return [Array<String>]
|
2980
3002
|
#
|
2981
3003
|
# @!attribute [rw] number_of_nodes
|
2982
|
-
# The number of
|
2983
|
-
#
|
3004
|
+
# The number of Glue Data Processing Units (DPUs) to allocate to this
|
3005
|
+
# `DevEndpoint`.
|
2984
3006
|
# @return [Integer]
|
2985
3007
|
#
|
2986
3008
|
# @!attribute [rw] worker_type
|
@@ -3005,10 +3027,10 @@ module Aws::Glue
|
|
3005
3027
|
#
|
3006
3028
|
# @!attribute [rw] glue_version
|
3007
3029
|
# Glue version determines the versions of Apache Spark and Python that
|
3008
|
-
#
|
3009
|
-
#
|
3030
|
+
# Glue supports. The Python version indicates the version supported
|
3031
|
+
# for running your ETL scripts on development endpoints.
|
3010
3032
|
#
|
3011
|
-
# For more information about the available
|
3033
|
+
# For more information about the available Glue versions and
|
3012
3034
|
# corresponding Spark and Python versions, see [Glue version][1] in
|
3013
3035
|
# the developer guide.
|
3014
3036
|
#
|
@@ -3061,8 +3083,8 @@ module Aws::Glue
|
|
3061
3083
|
#
|
3062
3084
|
# @!attribute [rw] tags
|
3063
3085
|
# The tags to use with this DevEndpoint. You may use tags to limit
|
3064
|
-
# access to the DevEndpoint. For more information about tags in
|
3065
|
-
#
|
3086
|
+
# access to the DevEndpoint. For more information about tags in Glue,
|
3087
|
+
# see [Amazon Web Services Tags in Glue][1] in the developer guide.
|
3066
3088
|
#
|
3067
3089
|
#
|
3068
3090
|
#
|
@@ -3125,8 +3147,8 @@ module Aws::Glue
|
|
3125
3147
|
# @return [Integer]
|
3126
3148
|
#
|
3127
3149
|
# @!attribute [rw] number_of_nodes
|
3128
|
-
# The number of
|
3129
|
-
#
|
3150
|
+
# The number of Glue Data Processing Units (DPUs) allocated to this
|
3151
|
+
# DevEndpoint.
|
3130
3152
|
# @return [Integer]
|
3131
3153
|
#
|
3132
3154
|
# @!attribute [rw] worker_type
|
@@ -3136,8 +3158,16 @@ module Aws::Glue
|
|
3136
3158
|
#
|
3137
3159
|
# @!attribute [rw] glue_version
|
3138
3160
|
# Glue version determines the versions of Apache Spark and Python that
|
3139
|
-
#
|
3140
|
-
#
|
3161
|
+
# Glue supports. The Python version indicates the version supported
|
3162
|
+
# for running your ETL scripts on development endpoints.
|
3163
|
+
#
|
3164
|
+
# For more information about the available Glue versions and
|
3165
|
+
# corresponding Spark and Python versions, see [Glue version][1] in
|
3166
|
+
# the developer guide.
|
3167
|
+
#
|
3168
|
+
#
|
3169
|
+
#
|
3170
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/add-job.html
|
3141
3171
|
# @return [String]
|
3142
3172
|
#
|
3143
3173
|
# @!attribute [rw] number_of_workers
|
@@ -3184,9 +3214,7 @@ module Aws::Glue
|
|
3184
3214
|
#
|
3185
3215
|
# * `"--enable-glue-datacatalog": ""`
|
3186
3216
|
#
|
3187
|
-
#
|
3188
|
-
#
|
3189
|
-
# * `"GLUE_PYTHON_VERSION": "2"`
|
3217
|
+
# ^
|
3190
3218
|
#
|
3191
3219
|
# You can specify a version of Python support for development
|
3192
3220
|
# endpoints by using the `Arguments` parameter in the
|
@@ -3325,22 +3353,22 @@ module Aws::Glue
|
|
3325
3353
|
# @return [Types::ExecutionProperty]
|
3326
3354
|
#
|
3327
3355
|
# @!attribute [rw] command
|
3328
|
-
# The `JobCommand` that
|
3356
|
+
# The `JobCommand` that runs this job.
|
3329
3357
|
# @return [Types::JobCommand]
|
3330
3358
|
#
|
3331
3359
|
# @!attribute [rw] default_arguments
|
3332
3360
|
# The default arguments for this job.
|
3333
3361
|
#
|
3334
3362
|
# You can specify arguments here that your own job-execution script
|
3335
|
-
# consumes, as well as arguments that
|
3363
|
+
# consumes, as well as arguments that Glue itself consumes.
|
3336
3364
|
#
|
3337
3365
|
# For information about how to specify and consume your own Job
|
3338
|
-
# arguments, see the [Calling
|
3366
|
+
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
3339
3367
|
# developer guide.
|
3340
3368
|
#
|
3341
|
-
# For information about the key-value pairs that
|
3342
|
-
#
|
3343
|
-
#
|
3369
|
+
# For information about the key-value pairs that Glue consumes to set
|
3370
|
+
# up your job, see the [Special Parameters Used by Glue][2] topic in
|
3371
|
+
# the developer guide.
|
3344
3372
|
#
|
3345
3373
|
#
|
3346
3374
|
#
|
@@ -3364,11 +3392,11 @@ module Aws::Glue
|
|
3364
3392
|
# @!attribute [rw] allocated_capacity
|
3365
3393
|
# This parameter is deprecated. Use `MaxCapacity` instead.
|
3366
3394
|
#
|
3367
|
-
# The number of
|
3368
|
-
#
|
3369
|
-
#
|
3370
|
-
#
|
3371
|
-
#
|
3395
|
+
# The number of Glue data processing units (DPUs) to allocate to this
|
3396
|
+
# Job. You can allocate from 2 to 100 DPUs; the default is 10. A DPU
|
3397
|
+
# is a relative measure of processing power that consists of 4 vCPUs
|
3398
|
+
# of compute capacity and 16 GB of memory. For more information, see
|
3399
|
+
# the [Glue pricing page][1].
|
3372
3400
|
#
|
3373
3401
|
#
|
3374
3402
|
#
|
@@ -3382,11 +3410,11 @@ module Aws::Glue
|
|
3382
3410
|
# @return [Integer]
|
3383
3411
|
#
|
3384
3412
|
# @!attribute [rw] max_capacity
|
3385
|
-
#
|
3413
|
+
# For Glue version 1.0 or earlier jobs, using the standard worker
|
3414
|
+
# type, the number of Glue data processing units (DPUs) that can be
|
3386
3415
|
# allocated when this job runs. A DPU is a relative measure of
|
3387
3416
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
3388
|
-
# GB of memory. For more information, see the [
|
3389
|
-
# page][1].
|
3417
|
+
# GB of memory. For more information, see the [Glue pricing page][1].
|
3390
3418
|
#
|
3391
3419
|
# Do not set `Max Capacity` if using `WorkerType` and
|
3392
3420
|
# `NumberOfWorkers`.
|
@@ -3404,6 +3432,10 @@ module Aws::Glue
|
|
3404
3432
|
# 100 DPUs. The default is 10 DPUs. This job type cannot have a
|
3405
3433
|
# fractional DPU allocation.
|
3406
3434
|
#
|
3435
|
+
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
3436
|
+
# capacity`. Instead, you should specify a `Worker type` and the
|
3437
|
+
# `Number of workers`.
|
3438
|
+
#
|
3407
3439
|
#
|
3408
3440
|
#
|
3409
3441
|
# [1]: https://aws.amazon.com/glue/pricing/
|
@@ -3416,8 +3448,8 @@ module Aws::Glue
|
|
3416
3448
|
#
|
3417
3449
|
# @!attribute [rw] tags
|
3418
3450
|
# The tags to use with this job. You may use tags to limit access to
|
3419
|
-
# the job. For more information about tags in
|
3420
|
-
# in
|
3451
|
+
# the job. For more information about tags in Glue, see [Amazon Web
|
3452
|
+
# Services Tags in Glue][1] in the developer guide.
|
3421
3453
|
#
|
3422
3454
|
#
|
3423
3455
|
#
|
@@ -3430,10 +3462,10 @@ module Aws::Glue
|
|
3430
3462
|
#
|
3431
3463
|
# @!attribute [rw] glue_version
|
3432
3464
|
# Glue version determines the versions of Apache Spark and Python that
|
3433
|
-
#
|
3434
|
-
#
|
3465
|
+
# Glue supports. The Python version indicates the version supported
|
3466
|
+
# for jobs of type Spark.
|
3435
3467
|
#
|
3436
|
-
# For more information about the available
|
3468
|
+
# For more information about the available Glue versions and
|
3437
3469
|
# corresponding Spark and Python versions, see [Glue version][1] in
|
3438
3470
|
# the developer guide.
|
3439
3471
|
#
|
@@ -3523,7 +3555,7 @@ module Aws::Glue
|
|
3523
3555
|
#
|
3524
3556
|
# @!attribute [rw] json_path
|
3525
3557
|
# A `JsonPath` string defining the JSON data for the classifier to
|
3526
|
-
# classify.
|
3558
|
+
# classify. Glue supports a subset of JsonPath, as described in
|
3527
3559
|
# [Writing JsonPath Custom Classifiers][1].
|
3528
3560
|
#
|
3529
3561
|
#
|
@@ -3592,7 +3624,7 @@ module Aws::Glue
|
|
3592
3624
|
# @return [String]
|
3593
3625
|
#
|
3594
3626
|
# @!attribute [rw] input_record_tables
|
3595
|
-
# A list of
|
3627
|
+
# A list of Glue table definitions used by the transform.
|
3596
3628
|
# @return [Array<Types::GlueTable>]
|
3597
3629
|
#
|
3598
3630
|
# @!attribute [rw] parameters
|
@@ -3602,13 +3634,13 @@ module Aws::Glue
|
|
3602
3634
|
#
|
3603
3635
|
# @!attribute [rw] role
|
3604
3636
|
# The name or Amazon Resource Name (ARN) of the IAM role with the
|
3605
|
-
# required permissions. The required permissions include both
|
3606
|
-
# service role permissions to
|
3637
|
+
# required permissions. The required permissions include both Glue
|
3638
|
+
# service role permissions to Glue resources, and Amazon S3
|
3607
3639
|
# permissions required by the transform.
|
3608
3640
|
#
|
3609
|
-
# * This role needs
|
3610
|
-
#
|
3611
|
-
#
|
3641
|
+
# * This role needs Glue service role permissions to allow access to
|
3642
|
+
# resources in Glue. See [Attach a Policy to IAM Users That Access
|
3643
|
+
# Glue][1].
|
3612
3644
|
#
|
3613
3645
|
# * This role needs permission to your Amazon Simple Storage Service
|
3614
3646
|
# (Amazon S3) sources, targets, temporary directory, scripts, and
|
@@ -3620,11 +3652,11 @@ module Aws::Glue
|
|
3620
3652
|
# @return [String]
|
3621
3653
|
#
|
3622
3654
|
# @!attribute [rw] glue_version
|
3623
|
-
# This value determines which version of
|
3624
|
-
#
|
3625
|
-
#
|
3626
|
-
#
|
3627
|
-
#
|
3655
|
+
# This value determines which version of Glue this machine learning
|
3656
|
+
# transform is compatible with. Glue 1.0 is recommended for most
|
3657
|
+
# customers. If the value is not set, the Glue compatibility defaults
|
3658
|
+
# to Glue 0.9. For more information, see [Glue Versions][1] in the
|
3659
|
+
# developer guide.
|
3628
3660
|
#
|
3629
3661
|
#
|
3630
3662
|
#
|
@@ -3632,12 +3664,11 @@ module Aws::Glue
|
|
3632
3664
|
# @return [String]
|
3633
3665
|
#
|
3634
3666
|
# @!attribute [rw] max_capacity
|
3635
|
-
# The number of
|
3636
|
-
#
|
3637
|
-
#
|
3638
|
-
#
|
3639
|
-
#
|
3640
|
-
# page][1].
|
3667
|
+
# The number of Glue data processing units (DPUs) that are allocated
|
3668
|
+
# to task runs for this transform. You can allocate from 2 to 100
|
3669
|
+
# DPUs; the default is 10. A DPU is a relative measure of processing
|
3670
|
+
# power that consists of 4 vCPUs of compute capacity and 16 GB of
|
3671
|
+
# memory. For more information, see the [Glue pricing page][1].
|
3641
3672
|
#
|
3642
3673
|
# `MaxCapacity` is a mutually exclusive option with `NumberOfWorkers`
|
3643
3674
|
# and `WorkerType`.
|
@@ -3715,8 +3746,8 @@ module Aws::Glue
|
|
3715
3746
|
# @!attribute [rw] tags
|
3716
3747
|
# The tags to use with this machine learning transform. You may use
|
3717
3748
|
# tags to limit access to the machine learning transform. For more
|
3718
|
-
# information about tags in
|
3719
|
-
# the developer guide.
|
3749
|
+
# information about tags in Glue, see [Amazon Web Services Tags in
|
3750
|
+
# Glue][1] in the developer guide.
|
3720
3751
|
#
|
3721
3752
|
#
|
3722
3753
|
#
|
@@ -3877,8 +3908,8 @@ module Aws::Glue
|
|
3877
3908
|
# }
|
3878
3909
|
#
|
3879
3910
|
# @!attribute [rw] catalog_id
|
3880
|
-
# The
|
3881
|
-
# created.
|
3911
|
+
# The Amazon Web Services account ID of the catalog in which the
|
3912
|
+
# partition is to be created.
|
3882
3913
|
# @return [String]
|
3883
3914
|
#
|
3884
3915
|
# @!attribute [rw] database_name
|
@@ -3933,8 +3964,8 @@ module Aws::Glue
|
|
3933
3964
|
# @return [String]
|
3934
3965
|
#
|
3935
3966
|
# @!attribute [rw] tags
|
3936
|
-
#
|
3937
|
-
# console, command line, or API.
|
3967
|
+
# Amazon Web Services tags that contain a key value pair and may be
|
3968
|
+
# searched by console, command line, or API.
|
3938
3969
|
# @return [Hash<String,String>]
|
3939
3970
|
#
|
3940
3971
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateRegistryInput AWS API Documentation
|
@@ -3983,7 +4014,7 @@ module Aws::Glue
|
|
3983
4014
|
# registry_arn: "GlueResourceArn",
|
3984
4015
|
# },
|
3985
4016
|
# schema_name: "SchemaRegistryNameString", # required
|
3986
|
-
# data_format: "AVRO", # required, accepts AVRO
|
4017
|
+
# data_format: "AVRO", # required, accepts AVRO, JSON
|
3987
4018
|
# compatibility: "NONE", # accepts NONE, DISABLED, BACKWARD, BACKWARD_ALL, FORWARD, FORWARD_ALL, FULL, FULL_ALL
|
3988
4019
|
# description: "DescriptionString",
|
3989
4020
|
# tags: {
|
@@ -4006,8 +4037,8 @@ module Aws::Glue
|
|
4006
4037
|
# @return [String]
|
4007
4038
|
#
|
4008
4039
|
# @!attribute [rw] data_format
|
4009
|
-
# The data format of the schema definition. Currently
|
4010
|
-
# supported.
|
4040
|
+
# The data format of the schema definition. Currently `AVRO` and
|
4041
|
+
# `JSON` are supported.
|
4011
4042
|
# @return [String]
|
4012
4043
|
#
|
4013
4044
|
# @!attribute [rw] compatibility
|
@@ -4065,9 +4096,9 @@ module Aws::Glue
|
|
4065
4096
|
# @return [String]
|
4066
4097
|
#
|
4067
4098
|
# @!attribute [rw] tags
|
4068
|
-
#
|
4069
|
-
# console, command line, or API. If specified, follows the
|
4070
|
-
# tags-on-create pattern.
|
4099
|
+
# Amazon Web Services tags that contain a key value pair and may be
|
4100
|
+
# searched by console, command line, or API. If specified, follows the
|
4101
|
+
# Amazon Web Services tags-on-create pattern.
|
4071
4102
|
# @return [Hash<String,String>]
|
4072
4103
|
#
|
4073
4104
|
# @!attribute [rw] schema_definition
|
@@ -4110,8 +4141,8 @@ module Aws::Glue
|
|
4110
4141
|
# @return [String]
|
4111
4142
|
#
|
4112
4143
|
# @!attribute [rw] data_format
|
4113
|
-
# The data format of the schema definition. Currently
|
4114
|
-
# supported.
|
4144
|
+
# The data format of the schema definition. Currently `AVRO` and
|
4145
|
+
# `JSON` are supported.
|
4115
4146
|
# @return [String]
|
4116
4147
|
#
|
4117
4148
|
# @!attribute [rw] compatibility
|
@@ -4390,7 +4421,7 @@ module Aws::Glue
|
|
4390
4421
|
#
|
4391
4422
|
# @!attribute [rw] catalog_id
|
4392
4423
|
# The ID of the Data Catalog in which to create the `Table`. If none
|
4393
|
-
# is supplied, the
|
4424
|
+
# is supplied, the Amazon Web Services account ID is used by default.
|
4394
4425
|
# @return [String]
|
4395
4426
|
#
|
4396
4427
|
# @!attribute [rw] database_name
|
@@ -4509,8 +4540,8 @@ module Aws::Glue
|
|
4509
4540
|
#
|
4510
4541
|
# @!attribute [rw] tags
|
4511
4542
|
# The tags to use with this trigger. You may use tags to limit access
|
4512
|
-
# to the trigger. For more information about tags in
|
4513
|
-
#
|
4543
|
+
# to the trigger. For more information about tags in Glue, see [Amazon
|
4544
|
+
# Web Services Tags in Glue][1] in the developer guide.
|
4514
4545
|
#
|
4515
4546
|
#
|
4516
4547
|
#
|
@@ -4567,7 +4598,7 @@ module Aws::Glue
|
|
4567
4598
|
#
|
4568
4599
|
# @!attribute [rw] catalog_id
|
4569
4600
|
# The ID of the Data Catalog in which to create the function. If none
|
4570
|
-
# is provided, the
|
4601
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
4571
4602
|
# @return [String]
|
4572
4603
|
#
|
4573
4604
|
# @!attribute [rw] database_name
|
@@ -4796,7 +4827,7 @@ module Aws::Glue
|
|
4796
4827
|
include Aws::Structure
|
4797
4828
|
end
|
4798
4829
|
|
4799
|
-
# The
|
4830
|
+
# The Lake Formation principal.
|
4800
4831
|
#
|
4801
4832
|
# @note When making an API call, you may pass DataLakePrincipal
|
4802
4833
|
# data as a hash:
|
@@ -4806,7 +4837,7 @@ module Aws::Glue
|
|
4806
4837
|
# }
|
4807
4838
|
#
|
4808
4839
|
# @!attribute [rw] data_lake_principal_identifier
|
4809
|
-
# An identifier for the
|
4840
|
+
# An identifier for the Lake Formation principal.
|
4810
4841
|
# @return [String]
|
4811
4842
|
#
|
4812
4843
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/DataLakePrincipal AWS API Documentation
|
@@ -5115,7 +5146,8 @@ module Aws::Glue
|
|
5115
5146
|
#
|
5116
5147
|
# @!attribute [rw] catalog_id
|
5117
5148
|
# The ID of the Data Catalog where the partitions in question reside.
|
5118
|
-
# If none is supplied, the
|
5149
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
5150
|
+
# default.
|
5119
5151
|
# @return [String]
|
5120
5152
|
#
|
5121
5153
|
# @!attribute [rw] database_name
|
@@ -5162,7 +5194,8 @@ module Aws::Glue
|
|
5162
5194
|
#
|
5163
5195
|
# @!attribute [rw] catalog_id
|
5164
5196
|
# The ID of the Data Catalog where the partitions in question reside.
|
5165
|
-
# If none is supplied, the
|
5197
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
5198
|
+
# default.
|
5166
5199
|
# @return [String]
|
5167
5200
|
#
|
5168
5201
|
# @!attribute [rw] database_name
|
@@ -5202,7 +5235,7 @@ module Aws::Glue
|
|
5202
5235
|
#
|
5203
5236
|
# @!attribute [rw] catalog_id
|
5204
5237
|
# The ID of the Data Catalog in which the connection resides. If none
|
5205
|
-
# is provided, the
|
5238
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
5206
5239
|
# @return [String]
|
5207
5240
|
#
|
5208
5241
|
# @!attribute [rw] connection_name
|
@@ -5255,7 +5288,7 @@ module Aws::Glue
|
|
5255
5288
|
#
|
5256
5289
|
# @!attribute [rw] catalog_id
|
5257
5290
|
# The ID of the Data Catalog in which the database resides. If none is
|
5258
|
-
# provided, the
|
5291
|
+
# provided, the Amazon Web Services account ID is used by default.
|
5259
5292
|
# @return [String]
|
5260
5293
|
#
|
5261
5294
|
# @!attribute [rw] name
|
@@ -5416,7 +5449,8 @@ module Aws::Glue
|
|
5416
5449
|
#
|
5417
5450
|
# @!attribute [rw] catalog_id
|
5418
5451
|
# The ID of the Data Catalog where the partition to be deleted
|
5419
|
-
# resides. If none is provided, the
|
5452
|
+
# resides. If none is provided, the Amazon Web Services account ID is
|
5453
|
+
# used by default.
|
5420
5454
|
# @return [String]
|
5421
5455
|
#
|
5422
5456
|
# @!attribute [rw] database_name
|
@@ -5506,8 +5540,7 @@ module Aws::Glue
|
|
5506
5540
|
# @return [String]
|
5507
5541
|
#
|
5508
5542
|
# @!attribute [rw] resource_arn
|
5509
|
-
# The ARN of the
|
5510
|
-
# deleted.
|
5543
|
+
# The ARN of the Glue resource for the resource policy to be deleted.
|
5511
5544
|
# @return [String]
|
5512
5545
|
#
|
5513
5546
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/DeleteResourcePolicyRequest AWS API Documentation
|
@@ -5650,7 +5683,7 @@ module Aws::Glue
|
|
5650
5683
|
#
|
5651
5684
|
# @!attribute [rw] catalog_id
|
5652
5685
|
# The ID of the Data Catalog where the table resides. If none is
|
5653
|
-
# provided, the
|
5686
|
+
# provided, the Amazon Web Services account ID is used by default.
|
5654
5687
|
# @return [String]
|
5655
5688
|
#
|
5656
5689
|
# @!attribute [rw] database_name
|
@@ -5689,7 +5722,7 @@ module Aws::Glue
|
|
5689
5722
|
#
|
5690
5723
|
# @!attribute [rw] catalog_id
|
5691
5724
|
# The ID of the Data Catalog where the tables reside. If none is
|
5692
|
-
# provided, the
|
5725
|
+
# provided, the Amazon Web Services account ID is used by default.
|
5693
5726
|
# @return [String]
|
5694
5727
|
#
|
5695
5728
|
# @!attribute [rw] database_name
|
@@ -5764,7 +5797,8 @@ module Aws::Glue
|
|
5764
5797
|
#
|
5765
5798
|
# @!attribute [rw] catalog_id
|
5766
5799
|
# The ID of the Data Catalog where the function to be deleted is
|
5767
|
-
# located. If none is supplied, the
|
5800
|
+
# located. If none is supplied, the Amazon Web Services account ID is
|
5801
|
+
# used by default.
|
5768
5802
|
# @return [String]
|
5769
5803
|
#
|
5770
5804
|
# @!attribute [rw] database_name
|
@@ -5886,10 +5920,10 @@ module Aws::Glue
|
|
5886
5920
|
#
|
5887
5921
|
# @!attribute [rw] glue_version
|
5888
5922
|
# Glue version determines the versions of Apache Spark and Python that
|
5889
|
-
#
|
5890
|
-
#
|
5923
|
+
# Glue supports. The Python version indicates the version supported
|
5924
|
+
# for running your ETL scripts on development endpoints.
|
5891
5925
|
#
|
5892
|
-
# For more information about the available
|
5926
|
+
# For more information about the available Glue versions and
|
5893
5927
|
# corresponding Spark and Python versions, see [Glue version][1] in
|
5894
5928
|
# the developer guide.
|
5895
5929
|
#
|
@@ -5915,8 +5949,8 @@ module Aws::Glue
|
|
5915
5949
|
# @return [Integer]
|
5916
5950
|
#
|
5917
5951
|
# @!attribute [rw] number_of_nodes
|
5918
|
-
# The number of
|
5919
|
-
#
|
5952
|
+
# The number of Glue Data Processing Units (DPUs) allocated to this
|
5953
|
+
# `DevEndpoint`.
|
5920
5954
|
# @return [Integer]
|
5921
5955
|
#
|
5922
5956
|
# @!attribute [rw] availability_zone
|
@@ -6002,9 +6036,7 @@ module Aws::Glue
|
|
6002
6036
|
#
|
6003
6037
|
# * `"--enable-glue-datacatalog": ""`
|
6004
6038
|
#
|
6005
|
-
#
|
6006
|
-
#
|
6007
|
-
# * `"GLUE_PYTHON_VERSION": "2"`
|
6039
|
+
# ^
|
6008
6040
|
#
|
6009
6041
|
# You can specify a version of Python support for development
|
6010
6042
|
# endpoints by using the `Arguments` parameter in the
|
@@ -6156,9 +6188,9 @@ module Aws::Glue
|
|
6156
6188
|
#
|
6157
6189
|
# @!attribute [rw] scan_rate
|
6158
6190
|
# The percentage of the configured read capacity units to use by the
|
6159
|
-
#
|
6160
|
-
#
|
6161
|
-
#
|
6191
|
+
# Glue crawler. Read capacity units is a term defined by DynamoDB, and
|
6192
|
+
# is a numeric value that acts as rate limiter for the number of reads
|
6193
|
+
# that can be performed on that table per second.
|
6162
6194
|
#
|
6163
6195
|
# The valid values are null or a value between 0.1 to 1.5. A null
|
6164
6196
|
# value is used when user does not provide a value, and defaults to
|
@@ -6177,8 +6209,8 @@ module Aws::Glue
|
|
6177
6209
|
include Aws::Structure
|
6178
6210
|
end
|
6179
6211
|
|
6180
|
-
# An edge represents a directed connection between two
|
6181
|
-
#
|
6212
|
+
# An edge represents a directed connection between two Glue components
|
6213
|
+
# that are part of the workflow the edge belongs to.
|
6182
6214
|
#
|
6183
6215
|
# @!attribute [rw] source_id
|
6184
6216
|
# The unique of the node within the workflow where the edge starts.
|
@@ -6212,7 +6244,7 @@ module Aws::Glue
|
|
6212
6244
|
# @return [String]
|
6213
6245
|
#
|
6214
6246
|
# @!attribute [rw] sse_aws_kms_key_id
|
6215
|
-
# The ID of the
|
6247
|
+
# The ID of the KMS key to use for encryption at rest.
|
6216
6248
|
# @return [String]
|
6217
6249
|
#
|
6218
6250
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/EncryptionAtRest AWS API Documentation
|
@@ -6568,8 +6600,8 @@ module Aws::Glue
|
|
6568
6600
|
# }
|
6569
6601
|
#
|
6570
6602
|
# @!attribute [rw] catalog_id
|
6571
|
-
# The ID of the catalog to migrate. Currently, this should be the
|
6572
|
-
# account ID.
|
6603
|
+
# The ID of the catalog to migrate. Currently, this should be the
|
6604
|
+
# Amazon Web Services account ID.
|
6573
6605
|
# @return [String]
|
6574
6606
|
#
|
6575
6607
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetCatalogImportStatusRequest AWS API Documentation
|
@@ -6678,7 +6710,8 @@ module Aws::Glue
|
|
6678
6710
|
#
|
6679
6711
|
# @!attribute [rw] catalog_id
|
6680
6712
|
# The ID of the Data Catalog where the partitions in question reside.
|
6681
|
-
# If none is supplied, the
|
6713
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
6714
|
+
# default.
|
6682
6715
|
# @return [String]
|
6683
6716
|
#
|
6684
6717
|
# @!attribute [rw] database_name
|
@@ -6738,7 +6771,8 @@ module Aws::Glue
|
|
6738
6771
|
#
|
6739
6772
|
# @!attribute [rw] catalog_id
|
6740
6773
|
# The ID of the Data Catalog where the partitions in question reside.
|
6741
|
-
# If none is supplied, the
|
6774
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
6775
|
+
# default.
|
6742
6776
|
# @return [String]
|
6743
6777
|
#
|
6744
6778
|
# @!attribute [rw] database_name
|
@@ -6792,7 +6826,7 @@ module Aws::Glue
|
|
6792
6826
|
#
|
6793
6827
|
# @!attribute [rw] catalog_id
|
6794
6828
|
# The ID of the Data Catalog in which the connection resides. If none
|
6795
|
-
# is provided, the
|
6829
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
6796
6830
|
# @return [String]
|
6797
6831
|
#
|
6798
6832
|
# @!attribute [rw] name
|
@@ -6803,9 +6837,9 @@ module Aws::Glue
|
|
6803
6837
|
# Allows you to retrieve the connection metadata without returning the
|
6804
6838
|
# password. For instance, the AWS Glue console uses this flag to
|
6805
6839
|
# retrieve the connection, and does not display the password. Set this
|
6806
|
-
# parameter when the caller might not have permission to use the
|
6807
|
-
#
|
6808
|
-
#
|
6840
|
+
# parameter when the caller might not have permission to use the KMS
|
6841
|
+
# key to decrypt the password, but it does have permission to access
|
6842
|
+
# the rest of the connection properties.
|
6809
6843
|
# @return [Boolean]
|
6810
6844
|
#
|
6811
6845
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetConnectionRequest AWS API Documentation
|
@@ -6875,7 +6909,7 @@ module Aws::Glue
|
|
6875
6909
|
#
|
6876
6910
|
# @!attribute [rw] catalog_id
|
6877
6911
|
# The ID of the Data Catalog in which the connections reside. If none
|
6878
|
-
# is provided, the
|
6912
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
6879
6913
|
# @return [String]
|
6880
6914
|
#
|
6881
6915
|
# @!attribute [rw] filter
|
@@ -6886,9 +6920,9 @@ module Aws::Glue
|
|
6886
6920
|
# Allows you to retrieve the connection metadata without returning the
|
6887
6921
|
# password. For instance, the AWS Glue console uses this flag to
|
6888
6922
|
# retrieve the connection, and does not display the password. Set this
|
6889
|
-
# parameter when the caller might not have permission to use the
|
6890
|
-
#
|
6891
|
-
#
|
6923
|
+
# parameter when the caller might not have permission to use the KMS
|
6924
|
+
# key to decrypt the password, but it does have permission to access
|
6925
|
+
# the rest of the connection properties.
|
6892
6926
|
# @return [Boolean]
|
6893
6927
|
#
|
6894
6928
|
# @!attribute [rw] next_token
|
@@ -7061,7 +7095,8 @@ module Aws::Glue
|
|
7061
7095
|
#
|
7062
7096
|
# @!attribute [rw] catalog_id
|
7063
7097
|
# The ID of the Data Catalog to retrieve the security configuration
|
7064
|
-
# for. If none is provided, the
|
7098
|
+
# for. If none is provided, the Amazon Web Services account ID is used
|
7099
|
+
# by default.
|
7065
7100
|
# @return [String]
|
7066
7101
|
#
|
7067
7102
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetDataCatalogEncryptionSettingsRequest AWS API Documentation
|
@@ -7094,7 +7129,7 @@ module Aws::Glue
|
|
7094
7129
|
#
|
7095
7130
|
# @!attribute [rw] catalog_id
|
7096
7131
|
# The ID of the Data Catalog in which the database resides. If none is
|
7097
|
-
# provided, the
|
7132
|
+
# provided, the Amazon Web Services account ID is used by default.
|
7098
7133
|
# @return [String]
|
7099
7134
|
#
|
7100
7135
|
# @!attribute [rw] name
|
@@ -7135,7 +7170,8 @@ module Aws::Glue
|
|
7135
7170
|
#
|
7136
7171
|
# @!attribute [rw] catalog_id
|
7137
7172
|
# The ID of the Data Catalog from which to retrieve `Databases`. If
|
7138
|
-
# none is provided, the
|
7173
|
+
# none is provided, the Amazon Web Services account ID is used by
|
7174
|
+
# default.
|
7139
7175
|
# @return [String]
|
7140
7176
|
#
|
7141
7177
|
# @!attribute [rw] next_token
|
@@ -7699,7 +7735,7 @@ module Aws::Glue
|
|
7699
7735
|
# @return [Time]
|
7700
7736
|
#
|
7701
7737
|
# @!attribute [rw] input_record_tables
|
7702
|
-
# A list of
|
7738
|
+
# A list of Glue table definitions used by the transform.
|
7703
7739
|
# @return [Array<Types::GlueTable>]
|
7704
7740
|
#
|
7705
7741
|
# @!attribute [rw] parameters
|
@@ -7726,11 +7762,11 @@ module Aws::Glue
|
|
7726
7762
|
# @return [String]
|
7727
7763
|
#
|
7728
7764
|
# @!attribute [rw] glue_version
|
7729
|
-
# This value determines which version of
|
7730
|
-
#
|
7731
|
-
#
|
7732
|
-
#
|
7733
|
-
#
|
7765
|
+
# This value determines which version of Glue this machine learning
|
7766
|
+
# transform is compatible with. Glue 1.0 is recommended for most
|
7767
|
+
# customers. If the value is not set, the Glue compatibility defaults
|
7768
|
+
# to Glue 0.9. For more information, see [Glue Versions][1] in the
|
7769
|
+
# developer guide.
|
7734
7770
|
#
|
7735
7771
|
#
|
7736
7772
|
#
|
@@ -7738,12 +7774,11 @@ module Aws::Glue
|
|
7738
7774
|
# @return [String]
|
7739
7775
|
#
|
7740
7776
|
# @!attribute [rw] max_capacity
|
7741
|
-
# The number of
|
7742
|
-
#
|
7743
|
-
#
|
7744
|
-
#
|
7745
|
-
#
|
7746
|
-
# page][1].
|
7777
|
+
# The number of Glue data processing units (DPUs) that are allocated
|
7778
|
+
# to task runs for this transform. You can allocate from 2 to 100
|
7779
|
+
# DPUs; the default is 10. A DPU is a relative measure of processing
|
7780
|
+
# power that consists of 4 vCPUs of compute capacity and 16 GB of
|
7781
|
+
# memory. For more information, see the [Glue pricing page][1].
|
7747
7782
|
#
|
7748
7783
|
# When the `WorkerType` field is set to a value other than `Standard`,
|
7749
7784
|
# the `MaxCapacity` field is set automatically and becomes read-only.
|
@@ -8030,7 +8065,8 @@ module Aws::Glue
|
|
8030
8065
|
#
|
8031
8066
|
# @!attribute [rw] catalog_id
|
8032
8067
|
# The ID of the Data Catalog where the partition in question resides.
|
8033
|
-
# If none is provided, the
|
8068
|
+
# If none is provided, the Amazon Web Services account ID is used by
|
8069
|
+
# default.
|
8034
8070
|
# @return [String]
|
8035
8071
|
#
|
8036
8072
|
# @!attribute [rw] database_name
|
@@ -8087,7 +8123,8 @@ module Aws::Glue
|
|
8087
8123
|
#
|
8088
8124
|
# @!attribute [rw] catalog_id
|
8089
8125
|
# The ID of the Data Catalog where the partitions in question reside.
|
8090
|
-
# If none is provided, the
|
8126
|
+
# If none is provided, the Amazon Web Services account ID is used by
|
8127
|
+
# default.
|
8091
8128
|
# @return [String]
|
8092
8129
|
#
|
8093
8130
|
# @!attribute [rw] database_name
|
@@ -8182,7 +8219,7 @@ module Aws::Glue
|
|
8182
8219
|
#
|
8183
8220
|
# * `decimal`
|
8184
8221
|
#
|
8185
|
-
# If an
|
8222
|
+
# If an type is encountered that is not valid, an exception is thrown.
|
8186
8223
|
#
|
8187
8224
|
# The following list shows the valid operators on each type. When you
|
8188
8225
|
# define a crawler, the `partitionKey` type is created as a `STRING`,
|
@@ -8323,7 +8360,7 @@ module Aws::Glue
|
|
8323
8360
|
# Currently, these key-value pairs are supported:
|
8324
8361
|
#
|
8325
8362
|
# * `inferSchema` — Specifies whether to set `inferSchema` to true
|
8326
|
-
# or false for the default script generated by an
|
8363
|
+
# or false for the default script generated by an Glue job. For
|
8327
8364
|
# example, to set `inferSchema` to true, pass the following key
|
8328
8365
|
# value pair:
|
8329
8366
|
#
|
@@ -8472,13 +8509,15 @@ module Aws::Glue
|
|
8472
8509
|
# }
|
8473
8510
|
#
|
8474
8511
|
# @!attribute [rw] resource_arn
|
8475
|
-
# The ARN of the
|
8476
|
-
#
|
8477
|
-
#
|
8512
|
+
# The ARN of the Glue resource for which to retrieve the resource
|
8513
|
+
# policy. If not supplied, the Data Catalog resource policy is
|
8514
|
+
# returned. Use `GetResourcePolicies` to view all existing resource
|
8515
|
+
# policies. For more information see [Specifying Glue Resource
|
8516
|
+
# ARNs][1].
|
8478
8517
|
#
|
8479
8518
|
#
|
8480
8519
|
#
|
8481
|
-
# [1]: https://docs.aws.amazon.com/glue/latest/dg/
|
8520
|
+
# [1]: https://docs.aws.amazon.com/glue/latest/dg/glue-specifying-resource-arns.html
|
8482
8521
|
# @return [String]
|
8483
8522
|
#
|
8484
8523
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetResourcePolicyRequest AWS API Documentation
|
@@ -8561,8 +8600,8 @@ module Aws::Glue
|
|
8561
8600
|
# @return [String]
|
8562
8601
|
#
|
8563
8602
|
# @!attribute [rw] data_format
|
8564
|
-
# The data format of the schema definition. Currently only `AVRO`
|
8565
|
-
# supported.
|
8603
|
+
# The data format of the schema definition. Currently only `AVRO` and
|
8604
|
+
# `JSON` are supported.
|
8566
8605
|
# @return [String]
|
8567
8606
|
#
|
8568
8607
|
# @!attribute [rw] status
|
@@ -8637,8 +8676,8 @@ module Aws::Glue
|
|
8637
8676
|
# @return [String]
|
8638
8677
|
#
|
8639
8678
|
# @!attribute [rw] data_format
|
8640
|
-
# The data format of the schema definition. Currently
|
8641
|
-
# supported.
|
8679
|
+
# The data format of the schema definition. Currently `AVRO` and
|
8680
|
+
# `JSON` are supported.
|
8642
8681
|
# @return [String]
|
8643
8682
|
#
|
8644
8683
|
# @!attribute [rw] compatibility
|
@@ -8749,8 +8788,8 @@ module Aws::Glue
|
|
8749
8788
|
# @return [String]
|
8750
8789
|
#
|
8751
8790
|
# @!attribute [rw] data_format
|
8752
|
-
# The data format of the schema definition. Currently
|
8753
|
-
# supported.
|
8791
|
+
# The data format of the schema definition. Currently `AVRO` and
|
8792
|
+
# `JSON` are supported.
|
8754
8793
|
# @return [String]
|
8755
8794
|
#
|
8756
8795
|
# @!attribute [rw] schema_arn
|
@@ -8934,7 +8973,7 @@ module Aws::Glue
|
|
8934
8973
|
#
|
8935
8974
|
# @!attribute [rw] catalog_id
|
8936
8975
|
# The ID of the Data Catalog where the table resides. If none is
|
8937
|
-
# provided, the
|
8976
|
+
# provided, the Amazon Web Services account ID is used by default.
|
8938
8977
|
# @return [String]
|
8939
8978
|
#
|
8940
8979
|
# @!attribute [rw] database_name
|
@@ -8981,7 +9020,7 @@ module Aws::Glue
|
|
8981
9020
|
#
|
8982
9021
|
# @!attribute [rw] catalog_id
|
8983
9022
|
# The ID of the Data Catalog where the tables reside. If none is
|
8984
|
-
# provided, the
|
9023
|
+
# provided, the Amazon Web Services account ID is used by default.
|
8985
9024
|
# @return [String]
|
8986
9025
|
#
|
8987
9026
|
# @!attribute [rw] database_name
|
@@ -9036,7 +9075,7 @@ module Aws::Glue
|
|
9036
9075
|
#
|
9037
9076
|
# @!attribute [rw] catalog_id
|
9038
9077
|
# The ID of the Data Catalog where the tables reside. If none is
|
9039
|
-
# provided, the
|
9078
|
+
# provided, the Amazon Web Services account ID is used by default.
|
9040
9079
|
# @return [String]
|
9041
9080
|
#
|
9042
9081
|
# @!attribute [rw] database_name
|
@@ -9101,7 +9140,7 @@ module Aws::Glue
|
|
9101
9140
|
#
|
9102
9141
|
# @!attribute [rw] catalog_id
|
9103
9142
|
# The ID of the Data Catalog where the tables reside. If none is
|
9104
|
-
# provided, the
|
9143
|
+
# provided, the Amazon Web Services account ID is used by default.
|
9105
9144
|
# @return [String]
|
9106
9145
|
#
|
9107
9146
|
# @!attribute [rw] database_name
|
@@ -9277,7 +9316,8 @@ module Aws::Glue
|
|
9277
9316
|
#
|
9278
9317
|
# @!attribute [rw] catalog_id
|
9279
9318
|
# The ID of the Data Catalog where the function to be retrieved is
|
9280
|
-
# located. If none is provided, the
|
9319
|
+
# located. If none is provided, the Amazon Web Services account ID is
|
9320
|
+
# used by default.
|
9281
9321
|
# @return [String]
|
9282
9322
|
#
|
9283
9323
|
# @!attribute [rw] database_name
|
@@ -9323,7 +9363,8 @@ module Aws::Glue
|
|
9323
9363
|
#
|
9324
9364
|
# @!attribute [rw] catalog_id
|
9325
9365
|
# The ID of the Data Catalog where the functions to be retrieved are
|
9326
|
-
# located. If none is provided, the
|
9366
|
+
# located. If none is provided, the Amazon Web Services account ID is
|
9367
|
+
# used by default.
|
9327
9368
|
# @return [String]
|
9328
9369
|
#
|
9329
9370
|
# @!attribute [rw] database_name
|
@@ -9591,8 +9632,8 @@ module Aws::Glue
|
|
9591
9632
|
include Aws::Structure
|
9592
9633
|
end
|
9593
9634
|
|
9594
|
-
# The database and table in the
|
9595
|
-
#
|
9635
|
+
# The database and table in the Glue Data Catalog that is used for input
|
9636
|
+
# or output data.
|
9596
9637
|
#
|
9597
9638
|
# @note When making an API call, you may pass GlueTable
|
9598
9639
|
# data as a hash:
|
@@ -9605,19 +9646,19 @@ module Aws::Glue
|
|
9605
9646
|
# }
|
9606
9647
|
#
|
9607
9648
|
# @!attribute [rw] database_name
|
9608
|
-
# A database name in the
|
9649
|
+
# A database name in the Glue Data Catalog.
|
9609
9650
|
# @return [String]
|
9610
9651
|
#
|
9611
9652
|
# @!attribute [rw] table_name
|
9612
|
-
# A table name in the
|
9653
|
+
# A table name in the Glue Data Catalog.
|
9613
9654
|
# @return [String]
|
9614
9655
|
#
|
9615
9656
|
# @!attribute [rw] catalog_id
|
9616
|
-
# A unique identifier for the
|
9657
|
+
# A unique identifier for the Glue Data Catalog.
|
9617
9658
|
# @return [String]
|
9618
9659
|
#
|
9619
9660
|
# @!attribute [rw] connection_name
|
9620
|
-
# The name of the connection to the
|
9661
|
+
# The name of the connection to the Glue Data Catalog.
|
9621
9662
|
# @return [String]
|
9622
9663
|
#
|
9623
9664
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GlueTable AWS API Documentation
|
@@ -9723,8 +9764,8 @@ module Aws::Glue
|
|
9723
9764
|
# }
|
9724
9765
|
#
|
9725
9766
|
# @!attribute [rw] catalog_id
|
9726
|
-
# The ID of the catalog to import. Currently, this should be the
|
9727
|
-
# account ID.
|
9767
|
+
# The ID of the catalog to import. Currently, this should be the
|
9768
|
+
# Amazon Web Services account ID.
|
9728
9769
|
# @return [String]
|
9729
9770
|
#
|
9730
9771
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/ImportCatalogToGlueRequest AWS API Documentation
|
@@ -9858,22 +9899,22 @@ module Aws::Glue
|
|
9858
9899
|
# @return [Types::ExecutionProperty]
|
9859
9900
|
#
|
9860
9901
|
# @!attribute [rw] command
|
9861
|
-
# The `JobCommand` that
|
9902
|
+
# The `JobCommand` that runs this job.
|
9862
9903
|
# @return [Types::JobCommand]
|
9863
9904
|
#
|
9864
9905
|
# @!attribute [rw] default_arguments
|
9865
9906
|
# The default arguments for this job, specified as name-value pairs.
|
9866
9907
|
#
|
9867
9908
|
# You can specify arguments here that your own job-execution script
|
9868
|
-
# consumes, as well as arguments that
|
9909
|
+
# consumes, as well as arguments that Glue itself consumes.
|
9869
9910
|
#
|
9870
9911
|
# For information about how to specify and consume your own Job
|
9871
|
-
# arguments, see the [Calling
|
9912
|
+
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
9872
9913
|
# developer guide.
|
9873
9914
|
#
|
9874
|
-
# For information about the key-value pairs that
|
9875
|
-
#
|
9876
|
-
#
|
9915
|
+
# For information about the key-value pairs that Glue consumes to set
|
9916
|
+
# up your job, see the [Special Parameters Used by Glue][2] topic in
|
9917
|
+
# the developer guide.
|
9877
9918
|
#
|
9878
9919
|
#
|
9879
9920
|
#
|
@@ -9897,11 +9938,11 @@ module Aws::Glue
|
|
9897
9938
|
# @!attribute [rw] allocated_capacity
|
9898
9939
|
# This field is deprecated. Use `MaxCapacity` instead.
|
9899
9940
|
#
|
9900
|
-
# The number of
|
9901
|
-
#
|
9902
|
-
#
|
9903
|
-
#
|
9904
|
-
#
|
9941
|
+
# The number of Glue data processing units (DPUs) allocated to runs of
|
9942
|
+
# this job. You can allocate from 2 to 100 DPUs; the default is 10. A
|
9943
|
+
# DPU is a relative measure of processing power that consists of 4
|
9944
|
+
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
9945
|
+
# see the [Glue pricing page][1].
|
9905
9946
|
#
|
9906
9947
|
#
|
9907
9948
|
#
|
@@ -9917,11 +9958,11 @@ module Aws::Glue
|
|
9917
9958
|
# @return [Integer]
|
9918
9959
|
#
|
9919
9960
|
# @!attribute [rw] max_capacity
|
9920
|
-
#
|
9961
|
+
# For Glue version 1.0 or earlier jobs, using the standard worker
|
9962
|
+
# type, the number of Glue data processing units (DPUs) that can be
|
9921
9963
|
# allocated when this job runs. A DPU is a relative measure of
|
9922
9964
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
9923
|
-
# GB of memory. For more information, see the [
|
9924
|
-
# page][1].
|
9965
|
+
# GB of memory. For more information, see the [Glue pricing page][1].
|
9925
9966
|
#
|
9926
9967
|
# Do not set `Max Capacity` if using `WorkerType` and
|
9927
9968
|
# `NumberOfWorkers`.
|
@@ -9940,6 +9981,10 @@ module Aws::Glue
|
|
9940
9981
|
# 100 DPUs. The default is 10 DPUs. This job type cannot have a
|
9941
9982
|
# fractional DPU allocation.
|
9942
9983
|
#
|
9984
|
+
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
9985
|
+
# capacity`. Instead, you should specify a `Worker type` and the
|
9986
|
+
# `Number of workers`.
|
9987
|
+
#
|
9943
9988
|
#
|
9944
9989
|
#
|
9945
9990
|
# [1]: https://aws.amazon.com/glue/pricing/
|
@@ -9980,10 +10025,10 @@ module Aws::Glue
|
|
9980
10025
|
#
|
9981
10026
|
# @!attribute [rw] glue_version
|
9982
10027
|
# Glue version determines the versions of Apache Spark and Python that
|
9983
|
-
#
|
9984
|
-
#
|
10028
|
+
# Glue supports. The Python version indicates the version supported
|
10029
|
+
# for jobs of type Spark.
|
9985
10030
|
#
|
9986
|
-
# For more information about the available
|
10031
|
+
# For more information about the available Glue versions and
|
9987
10032
|
# corresponding Spark and Python versions, see [Glue version][1] in
|
9988
10033
|
# the developer guide.
|
9989
10034
|
#
|
@@ -10094,7 +10139,7 @@ module Aws::Glue
|
|
10094
10139
|
include Aws::Structure
|
10095
10140
|
end
|
10096
10141
|
|
10097
|
-
# Specifies code
|
10142
|
+
# Specifies code that runs when a job is run.
|
10098
10143
|
#
|
10099
10144
|
# @note When making an API call, you may pass JobCommand
|
10100
10145
|
# data as a hash:
|
@@ -10113,11 +10158,11 @@ module Aws::Glue
|
|
10113
10158
|
#
|
10114
10159
|
# @!attribute [rw] script_location
|
10115
10160
|
# Specifies the Amazon Simple Storage Service (Amazon S3) path to a
|
10116
|
-
# script that
|
10161
|
+
# script that runs a job.
|
10117
10162
|
# @return [String]
|
10118
10163
|
#
|
10119
10164
|
# @!attribute [rw] python_version
|
10120
|
-
# The Python version being used to
|
10165
|
+
# The Python version being used to run a Python shell job. Allowed
|
10121
10166
|
# values are 2 or 3.
|
10122
10167
|
# @return [String]
|
10123
10168
|
#
|
@@ -10182,8 +10227,8 @@ module Aws::Glue
|
|
10182
10227
|
#
|
10183
10228
|
# @!attribute [rw] job_run_state
|
10184
10229
|
# The current state of the job run. For more information about the
|
10185
|
-
# statuses of jobs that have terminated abnormally, see [
|
10186
|
-
#
|
10230
|
+
# statuses of jobs that have terminated abnormally, see [Glue Job Run
|
10231
|
+
# Statuses][1].
|
10187
10232
|
#
|
10188
10233
|
#
|
10189
10234
|
#
|
@@ -10195,15 +10240,15 @@ module Aws::Glue
|
|
10195
10240
|
# replace the default arguments set in the job definition itself.
|
10196
10241
|
#
|
10197
10242
|
# You can specify arguments here that your own job-execution script
|
10198
|
-
# consumes, as well as arguments that
|
10243
|
+
# consumes, as well as arguments that Glue itself consumes.
|
10199
10244
|
#
|
10200
10245
|
# For information about how to specify and consume your own job
|
10201
|
-
# arguments, see the [Calling
|
10246
|
+
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
10202
10247
|
# developer guide.
|
10203
10248
|
#
|
10204
|
-
# For information about the key-value pairs that
|
10205
|
-
#
|
10206
|
-
#
|
10249
|
+
# For information about the key-value pairs that Glue consumes to set
|
10250
|
+
# up your job, see the [Special Parameters Used by Glue][2] topic in
|
10251
|
+
# the developer guide.
|
10207
10252
|
#
|
10208
10253
|
#
|
10209
10254
|
#
|
@@ -10222,11 +10267,11 @@ module Aws::Glue
|
|
10222
10267
|
# @!attribute [rw] allocated_capacity
|
10223
10268
|
# This field is deprecated. Use `MaxCapacity` instead.
|
10224
10269
|
#
|
10225
|
-
# The number of
|
10226
|
-
#
|
10227
|
-
#
|
10270
|
+
# The number of Glue data processing units (DPUs) allocated to this
|
10271
|
+
# JobRun. From 2 to 100 DPUs can be allocated; the default is 10. A
|
10272
|
+
# DPU is a relative measure of processing power that consists of 4
|
10228
10273
|
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
10229
|
-
# see the [
|
10274
|
+
# see the [Glue pricing page][1].
|
10230
10275
|
#
|
10231
10276
|
#
|
10232
10277
|
#
|
@@ -10245,11 +10290,10 @@ module Aws::Glue
|
|
10245
10290
|
# @return [Integer]
|
10246
10291
|
#
|
10247
10292
|
# @!attribute [rw] max_capacity
|
10248
|
-
# The number of
|
10293
|
+
# The number of Glue data processing units (DPUs) that can be
|
10249
10294
|
# allocated when this job runs. A DPU is a relative measure of
|
10250
10295
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
10251
|
-
# GB of memory. For more information, see the [
|
10252
|
-
# page][1].
|
10296
|
+
# GB of memory. For more information, see the [Glue pricing page][1].
|
10253
10297
|
#
|
10254
10298
|
# Do not set `Max Capacity` if using `WorkerType` and
|
10255
10299
|
# `NumberOfWorkers`.
|
@@ -10268,7 +10312,7 @@ module Aws::Glue
|
|
10268
10312
|
#
|
10269
10313
|
#
|
10270
10314
|
#
|
10271
|
-
# [1]: https://
|
10315
|
+
# [1]: https://aws.amazon.com/glue/pricing/
|
10272
10316
|
# @return [Float]
|
10273
10317
|
#
|
10274
10318
|
# @!attribute [rw] worker_type
|
@@ -10300,7 +10344,7 @@ module Aws::Glue
|
|
10300
10344
|
#
|
10301
10345
|
# @!attribute [rw] log_group_name
|
10302
10346
|
# The name of the log group for secure logging that can be server-side
|
10303
|
-
# encrypted in Amazon CloudWatch using
|
10347
|
+
# encrypted in Amazon CloudWatch using KMS. This name can be
|
10304
10348
|
# `/aws-glue/jobs/`, in which case the default encryption is `NONE`.
|
10305
10349
|
# If you add a role name and `SecurityConfiguration` name (in other
|
10306
10350
|
# words,
|
@@ -10314,10 +10358,10 @@ module Aws::Glue
|
|
10314
10358
|
#
|
10315
10359
|
# @!attribute [rw] glue_version
|
10316
10360
|
# Glue version determines the versions of Apache Spark and Python that
|
10317
|
-
#
|
10318
|
-
#
|
10361
|
+
# Glue supports. The Python version indicates the version supported
|
10362
|
+
# for jobs of type Spark.
|
10319
10363
|
#
|
10320
|
-
# For more information about the available
|
10364
|
+
# For more information about the available Glue versions and
|
10321
10365
|
# corresponding Spark and Python versions, see [Glue version][1] in
|
10322
10366
|
# the developer guide.
|
10323
10367
|
#
|
@@ -10417,22 +10461,22 @@ module Aws::Glue
|
|
10417
10461
|
# @return [Types::ExecutionProperty]
|
10418
10462
|
#
|
10419
10463
|
# @!attribute [rw] command
|
10420
|
-
# The `JobCommand` that
|
10464
|
+
# The `JobCommand` that runs this job (required).
|
10421
10465
|
# @return [Types::JobCommand]
|
10422
10466
|
#
|
10423
10467
|
# @!attribute [rw] default_arguments
|
10424
10468
|
# The default arguments for this job.
|
10425
10469
|
#
|
10426
10470
|
# You can specify arguments here that your own job-execution script
|
10427
|
-
# consumes, as well as arguments that
|
10471
|
+
# consumes, as well as arguments that Glue itself consumes.
|
10428
10472
|
#
|
10429
10473
|
# For information about how to specify and consume your own Job
|
10430
|
-
# arguments, see the [Calling
|
10474
|
+
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
10431
10475
|
# developer guide.
|
10432
10476
|
#
|
10433
|
-
# For information about the key-value pairs that
|
10434
|
-
#
|
10435
|
-
#
|
10477
|
+
# For information about the key-value pairs that Glue consumes to set
|
10478
|
+
# up your job, see the [Special Parameters Used by Glue][2] topic in
|
10479
|
+
# the developer guide.
|
10436
10480
|
#
|
10437
10481
|
#
|
10438
10482
|
#
|
@@ -10456,11 +10500,11 @@ module Aws::Glue
|
|
10456
10500
|
# @!attribute [rw] allocated_capacity
|
10457
10501
|
# This field is deprecated. Use `MaxCapacity` instead.
|
10458
10502
|
#
|
10459
|
-
# The number of
|
10460
|
-
#
|
10461
|
-
#
|
10462
|
-
#
|
10463
|
-
#
|
10503
|
+
# The number of Glue data processing units (DPUs) to allocate to this
|
10504
|
+
# job. You can allocate from 2 to 100 DPUs; the default is 10. A DPU
|
10505
|
+
# is a relative measure of processing power that consists of 4 vCPUs
|
10506
|
+
# of compute capacity and 16 GB of memory. For more information, see
|
10507
|
+
# the [Glue pricing page][1].
|
10464
10508
|
#
|
10465
10509
|
#
|
10466
10510
|
#
|
@@ -10474,11 +10518,11 @@ module Aws::Glue
|
|
10474
10518
|
# @return [Integer]
|
10475
10519
|
#
|
10476
10520
|
# @!attribute [rw] max_capacity
|
10477
|
-
#
|
10521
|
+
# For Glue version 1.0 or earlier jobs, using the standard worker
|
10522
|
+
# type, the number of Glue data processing units (DPUs) that can be
|
10478
10523
|
# allocated when this job runs. A DPU is a relative measure of
|
10479
10524
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
10480
|
-
# GB of memory. For more information, see the [
|
10481
|
-
# page][1].
|
10525
|
+
# GB of memory. For more information, see the [Glue pricing page][1].
|
10482
10526
|
#
|
10483
10527
|
# Do not set `Max Capacity` if using `WorkerType` and
|
10484
10528
|
# `NumberOfWorkers`.
|
@@ -10496,6 +10540,10 @@ module Aws::Glue
|
|
10496
10540
|
# 100 DPUs. The default is 10 DPUs. This job type cannot have a
|
10497
10541
|
# fractional DPU allocation.
|
10498
10542
|
#
|
10543
|
+
# For Glue version 2.0 jobs, you cannot instead specify a `Maximum
|
10544
|
+
# capacity`. Instead, you should specify a `Worker type` and the
|
10545
|
+
# `Number of workers`.
|
10546
|
+
#
|
10499
10547
|
#
|
10500
10548
|
#
|
10501
10549
|
# [1]: https://aws.amazon.com/glue/pricing/
|
@@ -10536,10 +10584,10 @@ module Aws::Glue
|
|
10536
10584
|
#
|
10537
10585
|
# @!attribute [rw] glue_version
|
10538
10586
|
# Glue version determines the versions of Apache Spark and Python that
|
10539
|
-
#
|
10540
|
-
#
|
10587
|
+
# Glue supports. The Python version indicates the version supported
|
10588
|
+
# for jobs of type Spark.
|
10541
10589
|
#
|
10542
|
-
# For more information about the available
|
10590
|
+
# For more information about the available Glue versions and
|
10543
10591
|
# corresponding Spark and Python versions, see [Glue version][1] in
|
10544
10592
|
# the developer guide.
|
10545
10593
|
#
|
@@ -10592,7 +10640,7 @@ module Aws::Glue
|
|
10592
10640
|
#
|
10593
10641
|
# @!attribute [rw] json_path
|
10594
10642
|
# A `JsonPath` string defining the JSON data for the classifier to
|
10595
|
-
# classify.
|
10643
|
+
# classify. Glue supports a subset of JsonPath, as described in
|
10596
10644
|
# [Writing JsonPath Custom Classifiers][1].
|
10597
10645
|
#
|
10598
10646
|
#
|
@@ -11340,7 +11388,7 @@ module Aws::Glue
|
|
11340
11388
|
# @return [Time]
|
11341
11389
|
#
|
11342
11390
|
# @!attribute [rw] input_record_tables
|
11343
|
-
# A list of
|
11391
|
+
# A list of Glue table definitions used by the transform.
|
11344
11392
|
# @return [Array<Types::GlueTable>]
|
11345
11393
|
#
|
11346
11394
|
# @!attribute [rw] parameters
|
@@ -11356,9 +11404,9 @@ module Aws::Glue
|
|
11356
11404
|
# @return [Types::EvaluationMetrics]
|
11357
11405
|
#
|
11358
11406
|
# @!attribute [rw] label_count
|
11359
|
-
# A count identifier for the labeling files generated by
|
11360
|
-
#
|
11361
|
-
#
|
11407
|
+
# A count identifier for the labeling files generated by Glue for this
|
11408
|
+
# transform. As you create a better transform, you can iteratively
|
11409
|
+
# download, label, and upload the labeling file.
|
11362
11410
|
# @return [Integer]
|
11363
11411
|
#
|
11364
11412
|
# @!attribute [rw] schema
|
@@ -11369,13 +11417,13 @@ module Aws::Glue
|
|
11369
11417
|
#
|
11370
11418
|
# @!attribute [rw] role
|
11371
11419
|
# The name or Amazon Resource Name (ARN) of the IAM role with the
|
11372
|
-
# required permissions. The required permissions include both
|
11373
|
-
# service role permissions to
|
11420
|
+
# required permissions. The required permissions include both Glue
|
11421
|
+
# service role permissions to Glue resources, and Amazon S3
|
11374
11422
|
# permissions required by the transform.
|
11375
11423
|
#
|
11376
|
-
# * This role needs
|
11377
|
-
#
|
11378
|
-
#
|
11424
|
+
# * This role needs Glue service role permissions to allow access to
|
11425
|
+
# resources in Glue. See [Attach a Policy to IAM Users That Access
|
11426
|
+
# Glue][1].
|
11379
11427
|
#
|
11380
11428
|
# * This role needs permission to your Amazon Simple Storage Service
|
11381
11429
|
# (Amazon S3) sources, targets, temporary directory, scripts, and
|
@@ -11387,11 +11435,11 @@ module Aws::Glue
|
|
11387
11435
|
# @return [String]
|
11388
11436
|
#
|
11389
11437
|
# @!attribute [rw] glue_version
|
11390
|
-
# This value determines which version of
|
11391
|
-
#
|
11392
|
-
#
|
11393
|
-
#
|
11394
|
-
#
|
11438
|
+
# This value determines which version of Glue this machine learning
|
11439
|
+
# transform is compatible with. Glue 1.0 is recommended for most
|
11440
|
+
# customers. If the value is not set, the Glue compatibility defaults
|
11441
|
+
# to Glue 0.9. For more information, see [Glue Versions][1] in the
|
11442
|
+
# developer guide.
|
11395
11443
|
#
|
11396
11444
|
#
|
11397
11445
|
#
|
@@ -11399,12 +11447,11 @@ module Aws::Glue
|
|
11399
11447
|
# @return [String]
|
11400
11448
|
#
|
11401
11449
|
# @!attribute [rw] max_capacity
|
11402
|
-
# The number of
|
11403
|
-
#
|
11404
|
-
#
|
11405
|
-
#
|
11406
|
-
#
|
11407
|
-
# page][1].
|
11450
|
+
# The number of Glue data processing units (DPUs) that are allocated
|
11451
|
+
# to task runs for this transform. You can allocate from 2 to 100
|
11452
|
+
# DPUs; the default is 10. A DPU is a relative measure of processing
|
11453
|
+
# power that consists of 4 vCPUs of compute capacity and 16 GB of
|
11454
|
+
# memory. For more information, see the [Glue pricing page][1].
|
11408
11455
|
#
|
11409
11456
|
# `MaxCapacity` is a mutually exclusive option with `NumberOfWorkers`
|
11410
11457
|
# and `WorkerType`.
|
@@ -11535,8 +11582,8 @@ module Aws::Glue
|
|
11535
11582
|
#
|
11536
11583
|
# * DISABLED: encryption is disabled
|
11537
11584
|
#
|
11538
|
-
# * SSEKMS: use of server-side encryption with
|
11539
|
-
#
|
11585
|
+
# * SSEKMS: use of server-side encryption with Key Management Service
|
11586
|
+
# (SSE-KMS) for user data stored in Amazon S3.
|
11540
11587
|
# @return [String]
|
11541
11588
|
#
|
11542
11589
|
# @!attribute [rw] kms_key_id
|
@@ -11613,11 +11660,16 @@ module Aws::Glue
|
|
11613
11660
|
# The time at which the entry was created.
|
11614
11661
|
# @return [String]
|
11615
11662
|
#
|
11663
|
+
# @!attribute [rw] other_metadata_value_list
|
11664
|
+
# Other metadata belonging to the same metadata key.
|
11665
|
+
# @return [Array<Types::OtherMetadataValueListItem>]
|
11666
|
+
#
|
11616
11667
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/MetadataInfo AWS API Documentation
|
11617
11668
|
#
|
11618
11669
|
class MetadataInfo < Struct.new(
|
11619
11670
|
:metadata_value,
|
11620
|
-
:created_time
|
11671
|
+
:created_time,
|
11672
|
+
:other_metadata_value_list)
|
11621
11673
|
SENSITIVE = []
|
11622
11674
|
include Aws::Structure
|
11623
11675
|
end
|
@@ -11704,15 +11756,15 @@ module Aws::Glue
|
|
11704
11756
|
include Aws::Structure
|
11705
11757
|
end
|
11706
11758
|
|
11707
|
-
# A node represents an
|
11708
|
-
#
|
11759
|
+
# A node represents an Glue component such as a trigger, or job, etc.,
|
11760
|
+
# that is part of a workflow.
|
11709
11761
|
#
|
11710
11762
|
# @!attribute [rw] type
|
11711
|
-
# The type of
|
11763
|
+
# The type of Glue component represented by the node.
|
11712
11764
|
# @return [String]
|
11713
11765
|
#
|
11714
11766
|
# @!attribute [rw] name
|
11715
|
-
# The name of the
|
11767
|
+
# The name of the Glue component represented by the node.
|
11716
11768
|
# @return [String]
|
11717
11769
|
#
|
11718
11770
|
# @!attribute [rw] unique_id
|
@@ -11808,6 +11860,27 @@ module Aws::Glue
|
|
11808
11860
|
include Aws::Structure
|
11809
11861
|
end
|
11810
11862
|
|
11863
|
+
# A structure containing other metadata for a schema version belonging
|
11864
|
+
# to the same metadata key.
|
11865
|
+
#
|
11866
|
+
# @!attribute [rw] metadata_value
|
11867
|
+
# The metadata key’s corresponding value for the other metadata
|
11868
|
+
# belonging to the same metadata key.
|
11869
|
+
# @return [String]
|
11870
|
+
#
|
11871
|
+
# @!attribute [rw] created_time
|
11872
|
+
# The time at which the entry was created.
|
11873
|
+
# @return [String]
|
11874
|
+
#
|
11875
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/OtherMetadataValueListItem AWS API Documentation
|
11876
|
+
#
|
11877
|
+
class OtherMetadataValueListItem < Struct.new(
|
11878
|
+
:metadata_value,
|
11879
|
+
:created_time)
|
11880
|
+
SENSITIVE = []
|
11881
|
+
include Aws::Structure
|
11882
|
+
end
|
11883
|
+
|
11811
11884
|
# Represents a slice of table data.
|
11812
11885
|
#
|
11813
11886
|
# @!attribute [rw] values
|
@@ -12023,8 +12096,8 @@ module Aws::Glue
|
|
12023
12096
|
#
|
12024
12097
|
# The values for the keys for the new partition must be passed as an
|
12025
12098
|
# array of String objects that must be ordered in the same order as
|
12026
|
-
# the partition keys appearing in the Amazon S3 prefix. Otherwise
|
12027
|
-
#
|
12099
|
+
# the partition keys appearing in the Amazon S3 prefix. Otherwise Glue
|
12100
|
+
# will add the values to the wrong keys.
|
12028
12101
|
# @return [Array<String>]
|
12029
12102
|
#
|
12030
12103
|
# @!attribute [rw] last_access_time
|
@@ -12251,7 +12324,8 @@ module Aws::Glue
|
|
12251
12324
|
#
|
12252
12325
|
# @!attribute [rw] catalog_id
|
12253
12326
|
# The ID of the Data Catalog to set the security configuration for. If
|
12254
|
-
# none is provided, the
|
12327
|
+
# none is provided, the Amazon Web Services account ID is used by
|
12328
|
+
# default.
|
12255
12329
|
# @return [String]
|
12256
12330
|
#
|
12257
12331
|
# @!attribute [rw] data_catalog_encryption_settings
|
@@ -12287,13 +12361,7 @@ module Aws::Glue
|
|
12287
12361
|
# @return [String]
|
12288
12362
|
#
|
12289
12363
|
# @!attribute [rw] resource_arn
|
12290
|
-
#
|
12291
|
-
# For more information about AWS Glue resource ARNs, see the [AWS Glue
|
12292
|
-
# ARN string pattern][1]
|
12293
|
-
#
|
12294
|
-
#
|
12295
|
-
#
|
12296
|
-
# [1]: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-common.html#aws-glue-api-regex-aws-glue-arn-id
|
12364
|
+
# Do not use. For internal use only.
|
12297
12365
|
# @return [String]
|
12298
12366
|
#
|
12299
12367
|
# @!attribute [rw] policy_hash_condition
|
@@ -12306,20 +12374,22 @@ module Aws::Glue
|
|
12306
12374
|
# @!attribute [rw] policy_exists_condition
|
12307
12375
|
# A value of `MUST_EXIST` is used to update a policy. A value of
|
12308
12376
|
# `NOT_EXIST` is used to create a new policy. If a value of `NONE` or
|
12309
|
-
# a null value is used, the call
|
12377
|
+
# a null value is used, the call does not depend on the existence of a
|
12310
12378
|
# policy.
|
12311
12379
|
# @return [String]
|
12312
12380
|
#
|
12313
12381
|
# @!attribute [rw] enable_hybrid
|
12314
|
-
#
|
12315
|
-
# account
|
12316
|
-
#
|
12317
|
-
#
|
12382
|
+
# If `'TRUE'`, indicates that you are using both methods to grant
|
12383
|
+
# cross-account access to Data Catalog resources:
|
12384
|
+
#
|
12385
|
+
# * By directly updating the resource policy with `PutResourePolicy`
|
12386
|
+
#
|
12387
|
+
# * By using the **Grant permissions** command on the Management
|
12388
|
+
# Console.
|
12318
12389
|
#
|
12319
|
-
#
|
12320
|
-
#
|
12321
|
-
#
|
12322
|
-
# policies is allowed.
|
12390
|
+
# Must be set to `'TRUE'` if you have already used the Management
|
12391
|
+
# Console to grant cross-account access, otherwise the call fails.
|
12392
|
+
# Default is 'FALSE'.
|
12323
12393
|
# @return [String]
|
12324
12394
|
#
|
12325
12395
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/PutResourcePolicyRequest AWS API Documentation
|
@@ -12569,8 +12639,8 @@ module Aws::Glue
|
|
12569
12639
|
# When crawling an Amazon S3 data source after the first crawl is
|
12570
12640
|
# complete, specifies whether to crawl the entire dataset again or to
|
12571
12641
|
# crawl only folders that were added since the last crawler run. For
|
12572
|
-
# more information, see [Incremental Crawls in
|
12573
|
-
#
|
12642
|
+
# more information, see [Incremental Crawls in Glue][1] in the developer
|
12643
|
+
# guide.
|
12574
12644
|
#
|
12575
12645
|
#
|
12576
12646
|
#
|
@@ -12994,6 +13064,7 @@ module Aws::Glue
|
|
12994
13064
|
# path: "Path",
|
12995
13065
|
# exclusions: ["Path"],
|
12996
13066
|
# connection_name: "ConnectionName",
|
13067
|
+
# sample_size: 1,
|
12997
13068
|
# }
|
12998
13069
|
#
|
12999
13070
|
# @!attribute [rw] path
|
@@ -13015,12 +13086,19 @@ module Aws::Glue
|
|
13015
13086
|
# (Amazon VPC).
|
13016
13087
|
# @return [String]
|
13017
13088
|
#
|
13089
|
+
# @!attribute [rw] sample_size
|
13090
|
+
# Sets the number of files in each leaf folder to be crawled when
|
13091
|
+
# crawling sample files in a dataset. If not set, all the files are
|
13092
|
+
# crawled. A valid value is an integer between 1 and 249.
|
13093
|
+
# @return [Integer]
|
13094
|
+
#
|
13018
13095
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3Target AWS API Documentation
|
13019
13096
|
#
|
13020
13097
|
class S3Target < Struct.new(
|
13021
13098
|
:path,
|
13022
13099
|
:exclusions,
|
13023
|
-
:connection_name
|
13100
|
+
:connection_name,
|
13101
|
+
:sample_size)
|
13024
13102
|
SENSITIVE = []
|
13025
13103
|
include Aws::Structure
|
13026
13104
|
end
|
@@ -13148,7 +13226,7 @@ module Aws::Glue
|
|
13148
13226
|
include Aws::Structure
|
13149
13227
|
end
|
13150
13228
|
|
13151
|
-
# The unique ID of the schema in the
|
13229
|
+
# The unique ID of the schema in the Glue schema registry.
|
13152
13230
|
#
|
13153
13231
|
# @note When making an API call, you may pass SchemaId
|
13154
13232
|
# data as a hash:
|
@@ -13227,8 +13305,7 @@ module Aws::Glue
|
|
13227
13305
|
include Aws::Structure
|
13228
13306
|
end
|
13229
13307
|
|
13230
|
-
# An object that references a schema stored in the
|
13231
|
-
# Registry.
|
13308
|
+
# An object that references a schema stored in the Glue Schema Registry.
|
13232
13309
|
#
|
13233
13310
|
# @note When making an API call, you may pass SchemaReference
|
13234
13311
|
# data as a hash:
|
@@ -13482,7 +13559,7 @@ module Aws::Glue
|
|
13482
13559
|
end
|
13483
13560
|
|
13484
13561
|
# Defines a non-overlapping region of a table's partitions, allowing
|
13485
|
-
# multiple requests to be
|
13562
|
+
# multiple requests to be run in parallel.
|
13486
13563
|
#
|
13487
13564
|
# @note When making an API call, you may pass Segment
|
13488
13565
|
# data as a hash:
|
@@ -13773,15 +13850,15 @@ module Aws::Glue
|
|
13773
13850
|
# replace the default arguments set in the job definition itself.
|
13774
13851
|
#
|
13775
13852
|
# You can specify arguments here that your own job-execution script
|
13776
|
-
# consumes, as well as arguments that
|
13853
|
+
# consumes, as well as arguments that Glue itself consumes.
|
13777
13854
|
#
|
13778
13855
|
# For information about how to specify and consume your own Job
|
13779
|
-
# arguments, see the [Calling
|
13856
|
+
# arguments, see the [Calling Glue APIs in Python][1] topic in the
|
13780
13857
|
# developer guide.
|
13781
13858
|
#
|
13782
|
-
# For information about the key-value pairs that
|
13783
|
-
#
|
13784
|
-
#
|
13859
|
+
# For information about the key-value pairs that Glue consumes to set
|
13860
|
+
# up your job, see the [Special Parameters Used by Glue][2] topic in
|
13861
|
+
# the developer guide.
|
13785
13862
|
#
|
13786
13863
|
#
|
13787
13864
|
#
|
@@ -13792,15 +13869,15 @@ module Aws::Glue
|
|
13792
13869
|
# @!attribute [rw] allocated_capacity
|
13793
13870
|
# This field is deprecated. Use `MaxCapacity` instead.
|
13794
13871
|
#
|
13795
|
-
# The number of
|
13796
|
-
#
|
13797
|
-
#
|
13872
|
+
# The number of Glue data processing units (DPUs) to allocate to this
|
13873
|
+
# JobRun. From 2 to 100 DPUs can be allocated; the default is 10. A
|
13874
|
+
# DPU is a relative measure of processing power that consists of 4
|
13798
13875
|
# vCPUs of compute capacity and 16 GB of memory. For more information,
|
13799
|
-
# see the [
|
13876
|
+
# see the [Glue pricing page][1].
|
13800
13877
|
#
|
13801
13878
|
#
|
13802
13879
|
#
|
13803
|
-
# [1]: https://
|
13880
|
+
# [1]: https://aws.amazon.com/glue/pricing/
|
13804
13881
|
# @return [Integer]
|
13805
13882
|
#
|
13806
13883
|
# @!attribute [rw] timeout
|
@@ -13811,11 +13888,10 @@ module Aws::Glue
|
|
13811
13888
|
# @return [Integer]
|
13812
13889
|
#
|
13813
13890
|
# @!attribute [rw] max_capacity
|
13814
|
-
# The number of
|
13891
|
+
# The number of Glue data processing units (DPUs) that can be
|
13815
13892
|
# allocated when this job runs. A DPU is a relative measure of
|
13816
13893
|
# processing power that consists of 4 vCPUs of compute capacity and 16
|
13817
|
-
# GB of memory. For more information, see the [
|
13818
|
-
# page][1].
|
13894
|
+
# GB of memory. For more information, see the [Glue pricing page][1].
|
13819
13895
|
#
|
13820
13896
|
# Do not set `Max Capacity` if using `WorkerType` and
|
13821
13897
|
# `NumberOfWorkers`.
|
@@ -13834,7 +13910,7 @@ module Aws::Glue
|
|
13834
13910
|
#
|
13835
13911
|
#
|
13836
13912
|
#
|
13837
|
-
# [1]: https://
|
13913
|
+
# [1]: https://aws.amazon.com/glue/pricing/
|
13838
13914
|
# @return [Float]
|
13839
13915
|
#
|
13840
13916
|
# @!attribute [rw] security_configuration
|
@@ -14247,7 +14323,7 @@ module Aws::Glue
|
|
14247
14323
|
# @return [Boolean]
|
14248
14324
|
#
|
14249
14325
|
# @!attribute [rw] schema_reference
|
14250
|
-
# An object that references a schema stored in the
|
14326
|
+
# An object that references a schema stored in the Glue Schema
|
14251
14327
|
# Registry.
|
14252
14328
|
#
|
14253
14329
|
# When creating a table, you can pass an empty list of columns for the
|
@@ -14394,8 +14470,7 @@ module Aws::Glue
|
|
14394
14470
|
# @return [String]
|
14395
14471
|
#
|
14396
14472
|
# @!attribute [rw] is_registered_with_lake_formation
|
14397
|
-
# Indicates whether the table has been registered with
|
14398
|
-
# Formation.
|
14473
|
+
# Indicates whether the table has been registered with Lake Formation.
|
14399
14474
|
# @return [Boolean]
|
14400
14475
|
#
|
14401
14476
|
# @!attribute [rw] target_table
|
@@ -14713,9 +14788,9 @@ module Aws::Glue
|
|
14713
14788
|
# }
|
14714
14789
|
#
|
14715
14790
|
# @!attribute [rw] resource_arn
|
14716
|
-
# The ARN of the
|
14717
|
-
# information about
|
14718
|
-
#
|
14791
|
+
# The ARN of the Glue resource to which to add the tags. For more
|
14792
|
+
# information about Glue resource ARNs, see the [Glue ARN string
|
14793
|
+
# pattern][1].
|
14719
14794
|
#
|
14720
14795
|
#
|
14721
14796
|
#
|
@@ -14980,11 +15055,11 @@ module Aws::Glue
|
|
14980
15055
|
# @return [String]
|
14981
15056
|
#
|
14982
15057
|
# @!attribute [rw] glue_version
|
14983
|
-
# This value determines which version of
|
14984
|
-
#
|
14985
|
-
#
|
14986
|
-
#
|
14987
|
-
#
|
15058
|
+
# This value determines which version of Glue this machine learning
|
15059
|
+
# transform is compatible with. Glue 1.0 is recommended for most
|
15060
|
+
# customers. If the value is not set, the Glue compatibility defaults
|
15061
|
+
# to Glue 0.9. For more information, see [Glue Versions][1] in the
|
15062
|
+
# developer guide.
|
14988
15063
|
#
|
14989
15064
|
#
|
14990
15065
|
#
|
@@ -15411,7 +15486,8 @@ module Aws::Glue
|
|
15411
15486
|
#
|
15412
15487
|
# @!attribute [rw] catalog_id
|
15413
15488
|
# The ID of the Data Catalog where the partitions in question reside.
|
15414
|
-
# If none is supplied, the
|
15489
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
15490
|
+
# default.
|
15415
15491
|
# @return [String]
|
15416
15492
|
#
|
15417
15493
|
# @!attribute [rw] database_name
|
@@ -15521,7 +15597,8 @@ module Aws::Glue
|
|
15521
15597
|
#
|
15522
15598
|
# @!attribute [rw] catalog_id
|
15523
15599
|
# The ID of the Data Catalog where the partitions in question reside.
|
15524
|
-
# If none is supplied, the
|
15600
|
+
# If none is supplied, the Amazon Web Services account ID is used by
|
15601
|
+
# default.
|
15525
15602
|
# @return [String]
|
15526
15603
|
#
|
15527
15604
|
# @!attribute [rw] database_name
|
@@ -15583,7 +15660,7 @@ module Aws::Glue
|
|
15583
15660
|
#
|
15584
15661
|
# @!attribute [rw] catalog_id
|
15585
15662
|
# The ID of the Data Catalog in which the connection resides. If none
|
15586
|
-
# is provided, the
|
15663
|
+
# is provided, the Amazon Web Services account ID is used by default.
|
15587
15664
|
# @return [String]
|
15588
15665
|
#
|
15589
15666
|
# @!attribute [rw] name
|
@@ -15623,6 +15700,7 @@ module Aws::Glue
|
|
15623
15700
|
# path: "Path",
|
15624
15701
|
# exclusions: ["Path"],
|
15625
15702
|
# connection_name: "ConnectionName",
|
15703
|
+
# sample_size: 1,
|
15626
15704
|
# },
|
15627
15705
|
# ],
|
15628
15706
|
# jdbc_targets: [
|
@@ -15680,7 +15758,7 @@ module Aws::Glue
|
|
15680
15758
|
# @return [String]
|
15681
15759
|
#
|
15682
15760
|
# @!attribute [rw] database_name
|
15683
|
-
# The
|
15761
|
+
# The Glue database where results are stored, such as:
|
15684
15762
|
# `arn:aws:daylight:us-east-1::database/sometable/*`.
|
15685
15763
|
# @return [String]
|
15686
15764
|
#
|
@@ -15891,7 +15969,8 @@ module Aws::Glue
|
|
15891
15969
|
#
|
15892
15970
|
# @!attribute [rw] catalog_id
|
15893
15971
|
# The ID of the Data Catalog in which the metadata database resides.
|
15894
|
-
# If none is provided, the
|
15972
|
+
# If none is provided, the Amazon Web Services account ID is used by
|
15973
|
+
# default.
|
15895
15974
|
# @return [String]
|
15896
15975
|
#
|
15897
15976
|
# @!attribute [rw] name
|
@@ -15975,9 +16054,7 @@ module Aws::Glue
|
|
15975
16054
|
#
|
15976
16055
|
# * `"--enable-glue-datacatalog": ""`
|
15977
16056
|
#
|
15978
|
-
#
|
15979
|
-
#
|
15980
|
-
# * `"GLUE_PYTHON_VERSION": "2"`
|
16057
|
+
# ^
|
15981
16058
|
#
|
15982
16059
|
# You can specify a version of Python support for development
|
15983
16060
|
# endpoints by using the `Arguments` parameter in the
|
@@ -16130,7 +16207,7 @@ module Aws::Glue
|
|
16130
16207
|
#
|
16131
16208
|
# @!attribute [rw] json_path
|
16132
16209
|
# A `JsonPath` string defining the JSON data for the classifier to
|
16133
|
-
# classify.
|
16210
|
+
# classify. Glue supports a subset of JsonPath, as described in
|
16134
16211
|
# [Writing JsonPath Custom Classifiers][1].
|
16135
16212
|
#
|
16136
16213
|
#
|
@@ -16196,11 +16273,11 @@ module Aws::Glue
|
|
16196
16273
|
# @return [String]
|
16197
16274
|
#
|
16198
16275
|
# @!attribute [rw] glue_version
|
16199
|
-
# This value determines which version of
|
16200
|
-
#
|
16201
|
-
#
|
16202
|
-
#
|
16203
|
-
#
|
16276
|
+
# This value determines which version of Glue this machine learning
|
16277
|
+
# transform is compatible with. Glue 1.0 is recommended for most
|
16278
|
+
# customers. If the value is not set, the Glue compatibility defaults
|
16279
|
+
# to Glue 0.9. For more information, see [Glue Versions][1] in the
|
16280
|
+
# developer guide.
|
16204
16281
|
#
|
16205
16282
|
#
|
16206
16283
|
#
|
@@ -16208,12 +16285,11 @@ module Aws::Glue
|
|
16208
16285
|
# @return [String]
|
16209
16286
|
#
|
16210
16287
|
# @!attribute [rw] max_capacity
|
16211
|
-
# The number of
|
16212
|
-
#
|
16213
|
-
#
|
16214
|
-
#
|
16215
|
-
#
|
16216
|
-
# page][1].
|
16288
|
+
# The number of Glue data processing units (DPUs) that are allocated
|
16289
|
+
# to task runs for this transform. You can allocate from 2 to 100
|
16290
|
+
# DPUs; the default is 10. A DPU is a relative measure of processing
|
16291
|
+
# power that consists of 4 vCPUs of compute capacity and 16 GB of
|
16292
|
+
# memory. For more information, see the [Glue pricing page][1].
|
16217
16293
|
#
|
16218
16294
|
# When the `WorkerType` field is set to a value other than `Standard`,
|
16219
16295
|
# the `MaxCapacity` field is set automatically and becomes read-only.
|
@@ -16355,7 +16431,8 @@ module Aws::Glue
|
|
16355
16431
|
#
|
16356
16432
|
# @!attribute [rw] catalog_id
|
16357
16433
|
# The ID of the Data Catalog where the partition to be updated
|
16358
|
-
# resides. If none is provided, the
|
16434
|
+
# resides. If none is provided, the Amazon Web Services account ID is
|
16435
|
+
# used by default.
|
16359
16436
|
# @return [String]
|
16360
16437
|
#
|
16361
16438
|
# @!attribute [rw] database_name
|
@@ -16608,7 +16685,7 @@ module Aws::Glue
|
|
16608
16685
|
#
|
16609
16686
|
# @!attribute [rw] catalog_id
|
16610
16687
|
# The ID of the Data Catalog where the table resides. If none is
|
16611
|
-
# provided, the
|
16688
|
+
# provided, the Amazon Web Services account ID is used by default.
|
16612
16689
|
# @return [String]
|
16613
16690
|
#
|
16614
16691
|
# @!attribute [rw] database_name
|
@@ -16732,7 +16809,8 @@ module Aws::Glue
|
|
16732
16809
|
#
|
16733
16810
|
# @!attribute [rw] catalog_id
|
16734
16811
|
# The ID of the Data Catalog where the function to be updated is
|
16735
|
-
# located. If none is provided, the
|
16812
|
+
# located. If none is provided, the Amazon Web Services account ID is
|
16813
|
+
# used by default.
|
16736
16814
|
# @return [String]
|
16737
16815
|
#
|
16738
16816
|
# @!attribute [rw] database_name
|
@@ -16986,8 +17064,8 @@ module Aws::Glue
|
|
16986
17064
|
include Aws::Structure
|
16987
17065
|
end
|
16988
17066
|
|
16989
|
-
# A workflow represents a flow in which
|
16990
|
-
#
|
17067
|
+
# A workflow represents a flow in which Glue components should be run to
|
17068
|
+
# complete a logical task.
|
16991
17069
|
#
|
16992
17070
|
# @!attribute [rw] name
|
16993
17071
|
# The name of the workflow representing the flow.
|
@@ -17015,9 +17093,8 @@ module Aws::Glue
|
|
17015
17093
|
# @return [Types::WorkflowRun]
|
17016
17094
|
#
|
17017
17095
|
# @!attribute [rw] graph
|
17018
|
-
# The graph representing all the
|
17019
|
-
#
|
17020
|
-
# edges.
|
17096
|
+
# The graph representing all the Glue components that belong to the
|
17097
|
+
# workflow as nodes and directed connections between them as edges.
|
17021
17098
|
# @return [Types::WorkflowGraph]
|
17022
17099
|
#
|
17023
17100
|
# @!attribute [rw] max_concurrent_runs
|
@@ -17044,12 +17121,12 @@ module Aws::Glue
|
|
17044
17121
|
end
|
17045
17122
|
|
17046
17123
|
# A workflow graph represents the complete workflow containing all the
|
17047
|
-
#
|
17124
|
+
# Glue components present in the workflow and all the directed
|
17048
17125
|
# connections between them.
|
17049
17126
|
#
|
17050
17127
|
# @!attribute [rw] nodes
|
17051
|
-
# A list of the the
|
17052
|
-
#
|
17128
|
+
# A list of the the Glue components belong to the workflow represented
|
17129
|
+
# as nodes.
|
17053
17130
|
# @return [Array<Types::Node>]
|
17054
17131
|
#
|
17055
17132
|
# @!attribute [rw] edges
|
@@ -17070,7 +17147,7 @@ module Aws::Glue
|
|
17070
17147
|
# information.
|
17071
17148
|
#
|
17072
17149
|
# @!attribute [rw] name
|
17073
|
-
# Name of the workflow that was
|
17150
|
+
# Name of the workflow that was run.
|
17074
17151
|
# @return [String]
|
17075
17152
|
#
|
17076
17153
|
# @!attribute [rw] workflow_run_id
|
@@ -17108,9 +17185,8 @@ module Aws::Glue
|
|
17108
17185
|
# @return [Types::WorkflowRunStatistics]
|
17109
17186
|
#
|
17110
17187
|
# @!attribute [rw] graph
|
17111
|
-
# The graph representing all the
|
17112
|
-
#
|
17113
|
-
# edges.
|
17188
|
+
# The graph representing all the Glue components that belong to the
|
17189
|
+
# workflow as nodes and directed connections between them as edges.
|
17114
17190
|
# @return [Types::WorkflowGraph]
|
17115
17191
|
#
|
17116
17192
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/WorkflowRun AWS API Documentation
|