aws-sdk-gluedatabrew 1.6.0 → 1.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-gluedatabrew.rb +1 -1
- data/lib/aws-sdk-gluedatabrew/client.rb +145 -16
- data/lib/aws-sdk-gluedatabrew/client_api.rb +63 -0
- data/lib/aws-sdk-gluedatabrew/types.rb +479 -82
- metadata +4 -4
@@ -70,7 +70,7 @@ module Aws::GlueDataBrew
|
|
70
70
|
|
71
71
|
# Represents an individual condition that evaluates to true or false.
|
72
72
|
#
|
73
|
-
# Conditions are used with recipe actions
|
73
|
+
# Conditions are used with recipe actions. The action is only performed
|
74
74
|
# for column values where the condition evaluates to true.
|
75
75
|
#
|
76
76
|
# If a recipe requires more than one condition, then the recipe must
|
@@ -163,6 +163,45 @@ module Aws::GlueDataBrew
|
|
163
163
|
# key: "Key",
|
164
164
|
# },
|
165
165
|
# },
|
166
|
+
# database_input_definition: {
|
167
|
+
# glue_connection_name: "GlueConnectionName", # required
|
168
|
+
# database_table_name: "DatabaseTableName", # required
|
169
|
+
# temp_directory: {
|
170
|
+
# bucket: "Bucket", # required
|
171
|
+
# key: "Key",
|
172
|
+
# },
|
173
|
+
# },
|
174
|
+
# },
|
175
|
+
# path_options: {
|
176
|
+
# last_modified_date_condition: {
|
177
|
+
# expression: "Expression", # required
|
178
|
+
# values_map: { # required
|
179
|
+
# "ValueReference" => "ConditionValue",
|
180
|
+
# },
|
181
|
+
# },
|
182
|
+
# files_limit: {
|
183
|
+
# max_files: 1, # required
|
184
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
185
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
186
|
+
# },
|
187
|
+
# parameters: {
|
188
|
+
# "PathParameterName" => {
|
189
|
+
# name: "PathParameterName", # required
|
190
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
191
|
+
# datetime_options: {
|
192
|
+
# format: "DatetimeFormat", # required
|
193
|
+
# timezone_offset: "TimezoneOffset",
|
194
|
+
# locale_code: "LocaleCode",
|
195
|
+
# },
|
196
|
+
# create_column: false,
|
197
|
+
# filter: {
|
198
|
+
# expression: "Expression", # required
|
199
|
+
# values_map: { # required
|
200
|
+
# "ValueReference" => "ConditionValue",
|
201
|
+
# },
|
202
|
+
# },
|
203
|
+
# },
|
204
|
+
# },
|
166
205
|
# },
|
167
206
|
# tags: {
|
168
207
|
# "TagKey" => "TagValue",
|
@@ -175,20 +214,25 @@ module Aws::GlueDataBrew
|
|
175
214
|
# @return [String]
|
176
215
|
#
|
177
216
|
# @!attribute [rw] format
|
178
|
-
#
|
217
|
+
# The file format of a dataset that is created from an S3 file or
|
179
218
|
# folder.
|
180
219
|
# @return [String]
|
181
220
|
#
|
182
221
|
# @!attribute [rw] format_options
|
183
|
-
#
|
184
|
-
# input.
|
222
|
+
# Represents a set of options that define the structure of either
|
223
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
185
224
|
# @return [Types::FormatOptions]
|
186
225
|
#
|
187
226
|
# @!attribute [rw] input
|
188
|
-
#
|
189
|
-
# Data Catalog or Amazon S3.
|
227
|
+
# Represents information on how DataBrew can find data, in either the
|
228
|
+
# AWS Glue Data Catalog or Amazon S3.
|
190
229
|
# @return [Types::Input]
|
191
230
|
#
|
231
|
+
# @!attribute [rw] path_options
|
232
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
233
|
+
# the dataset.
|
234
|
+
# @return [Types::PathOptions]
|
235
|
+
#
|
192
236
|
# @!attribute [rw] tags
|
193
237
|
# Metadata tags to apply to this dataset.
|
194
238
|
# @return [Hash<String,String>]
|
@@ -200,6 +244,7 @@ module Aws::GlueDataBrew
|
|
200
244
|
:format,
|
201
245
|
:format_options,
|
202
246
|
:input,
|
247
|
+
:path_options,
|
203
248
|
:tags)
|
204
249
|
SENSITIVE = []
|
205
250
|
include Aws::Structure
|
@@ -282,8 +327,8 @@ module Aws::GlueDataBrew
|
|
282
327
|
# @return [Integer]
|
283
328
|
#
|
284
329
|
# @!attribute [rw] output_location
|
285
|
-
#
|
286
|
-
# read input data, or write output from a job.
|
330
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
331
|
+
# DataBrew can read input data, or write output from a job.
|
287
332
|
# @return [Types::S3Location]
|
288
333
|
#
|
289
334
|
# @!attribute [rw] role_arn
|
@@ -673,8 +718,9 @@ module Aws::GlueDataBrew
|
|
673
718
|
include Aws::Structure
|
674
719
|
end
|
675
720
|
|
676
|
-
#
|
677
|
-
# dataset from that
|
721
|
+
# Represents a set of options that define how DataBrew will read a
|
722
|
+
# comma-separated value (CSV) file when creating a dataset from that
|
723
|
+
# file.
|
678
724
|
#
|
679
725
|
# @note When making an API call, you may pass CsvOptions
|
680
726
|
# data as a hash:
|
@@ -686,12 +732,13 @@ module Aws::GlueDataBrew
|
|
686
732
|
#
|
687
733
|
# @!attribute [rw] delimiter
|
688
734
|
# A single character that specifies the delimiter being used in the
|
689
|
-
#
|
735
|
+
# CSV file.
|
690
736
|
# @return [String]
|
691
737
|
#
|
692
738
|
# @!attribute [rw] header_row
|
693
|
-
# A variable that specifies whether the first row in the file
|
694
|
-
# parsed as the header. If false, column names
|
739
|
+
# A variable that specifies whether the first row in the file is
|
740
|
+
# parsed as the header. If this value is false, column names are
|
741
|
+
# auto-generated.
|
695
742
|
# @return [Boolean]
|
696
743
|
#
|
697
744
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/CsvOptions AWS API Documentation
|
@@ -703,7 +750,8 @@ module Aws::GlueDataBrew
|
|
703
750
|
include Aws::Structure
|
704
751
|
end
|
705
752
|
|
706
|
-
#
|
753
|
+
# Represents a set of options that define how DataBrew will write a
|
754
|
+
# comma-separated value (CSV) file.
|
707
755
|
#
|
708
756
|
# @note When making an API call, you may pass CsvOutputOptions
|
709
757
|
# data as a hash:
|
@@ -713,7 +761,7 @@ module Aws::GlueDataBrew
|
|
713
761
|
# }
|
714
762
|
#
|
715
763
|
# @!attribute [rw] delimiter
|
716
|
-
# A single character that specifies the delimiter used to create
|
764
|
+
# A single character that specifies the delimiter used to create CSV
|
717
765
|
# job output.
|
718
766
|
# @return [String]
|
719
767
|
#
|
@@ -771,6 +819,44 @@ module Aws::GlueDataBrew
|
|
771
819
|
include Aws::Structure
|
772
820
|
end
|
773
821
|
|
822
|
+
# Connection information for dataset input files stored in a database.
|
823
|
+
#
|
824
|
+
# @note When making an API call, you may pass DatabaseInputDefinition
|
825
|
+
# data as a hash:
|
826
|
+
#
|
827
|
+
# {
|
828
|
+
# glue_connection_name: "GlueConnectionName", # required
|
829
|
+
# database_table_name: "DatabaseTableName", # required
|
830
|
+
# temp_directory: {
|
831
|
+
# bucket: "Bucket", # required
|
832
|
+
# key: "Key",
|
833
|
+
# },
|
834
|
+
# }
|
835
|
+
#
|
836
|
+
# @!attribute [rw] glue_connection_name
|
837
|
+
# The AWS Glue Connection that stores the connection information for
|
838
|
+
# the target database.
|
839
|
+
# @return [String]
|
840
|
+
#
|
841
|
+
# @!attribute [rw] database_table_name
|
842
|
+
# The table within the target database.
|
843
|
+
# @return [String]
|
844
|
+
#
|
845
|
+
# @!attribute [rw] temp_directory
|
846
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
847
|
+
# DataBrew can read input data, or write output from a job.
|
848
|
+
# @return [Types::S3Location]
|
849
|
+
#
|
850
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/DatabaseInputDefinition AWS API Documentation
|
851
|
+
#
|
852
|
+
class DatabaseInputDefinition < Struct.new(
|
853
|
+
:glue_connection_name,
|
854
|
+
:database_table_name,
|
855
|
+
:temp_directory)
|
856
|
+
SENSITIVE = []
|
857
|
+
include Aws::Structure
|
858
|
+
end
|
859
|
+
|
774
860
|
# Represents a dataset that can be processed by DataBrew.
|
775
861
|
#
|
776
862
|
# @!attribute [rw] account_id
|
@@ -790,12 +876,13 @@ module Aws::GlueDataBrew
|
|
790
876
|
# @return [String]
|
791
877
|
#
|
792
878
|
# @!attribute [rw] format
|
793
|
-
#
|
879
|
+
# The file format of a dataset that is created from an S3 file or
|
794
880
|
# folder.
|
795
881
|
# @return [String]
|
796
882
|
#
|
797
883
|
# @!attribute [rw] format_options
|
798
|
-
#
|
884
|
+
# A set of options that define how DataBrew interprets the data in the
|
885
|
+
# dataset.
|
799
886
|
# @return [Types::FormatOptions]
|
800
887
|
#
|
801
888
|
# @!attribute [rw] input
|
@@ -817,6 +904,11 @@ module Aws::GlueDataBrew
|
|
817
904
|
# AWS Glue Data Catalog.
|
818
905
|
# @return [String]
|
819
906
|
#
|
907
|
+
# @!attribute [rw] path_options
|
908
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
909
|
+
# the dataset.
|
910
|
+
# @return [Types::PathOptions]
|
911
|
+
#
|
820
912
|
# @!attribute [rw] tags
|
821
913
|
# Metadata tags that have been applied to the dataset.
|
822
914
|
# @return [Hash<String,String>]
|
@@ -838,12 +930,114 @@ module Aws::GlueDataBrew
|
|
838
930
|
:last_modified_date,
|
839
931
|
:last_modified_by,
|
840
932
|
:source,
|
933
|
+
:path_options,
|
841
934
|
:tags,
|
842
935
|
:resource_arn)
|
843
936
|
SENSITIVE = []
|
844
937
|
include Aws::Structure
|
845
938
|
end
|
846
939
|
|
940
|
+
# Represents a dataset paramater that defines type and conditions for a
|
941
|
+
# parameter in the S3 path of the dataset.
|
942
|
+
#
|
943
|
+
# @note When making an API call, you may pass DatasetParameter
|
944
|
+
# data as a hash:
|
945
|
+
#
|
946
|
+
# {
|
947
|
+
# name: "PathParameterName", # required
|
948
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
949
|
+
# datetime_options: {
|
950
|
+
# format: "DatetimeFormat", # required
|
951
|
+
# timezone_offset: "TimezoneOffset",
|
952
|
+
# locale_code: "LocaleCode",
|
953
|
+
# },
|
954
|
+
# create_column: false,
|
955
|
+
# filter: {
|
956
|
+
# expression: "Expression", # required
|
957
|
+
# values_map: { # required
|
958
|
+
# "ValueReference" => "ConditionValue",
|
959
|
+
# },
|
960
|
+
# },
|
961
|
+
# }
|
962
|
+
#
|
963
|
+
# @!attribute [rw] name
|
964
|
+
# The name of the parameter that is used in the dataset's S3 path.
|
965
|
+
# @return [String]
|
966
|
+
#
|
967
|
+
# @!attribute [rw] type
|
968
|
+
# The type of the dataset parameter, can be one of a 'String',
|
969
|
+
# 'Number' or 'Datetime'.
|
970
|
+
# @return [String]
|
971
|
+
#
|
972
|
+
# @!attribute [rw] datetime_options
|
973
|
+
# Additional parameter options such as a format and a timezone.
|
974
|
+
# Required for datetime parameters.
|
975
|
+
# @return [Types::DatetimeOptions]
|
976
|
+
#
|
977
|
+
# @!attribute [rw] create_column
|
978
|
+
# Optional boolean value that defines whether the captured value of
|
979
|
+
# this parameter should be loaded as an additional column in the
|
980
|
+
# dataset.
|
981
|
+
# @return [Boolean]
|
982
|
+
#
|
983
|
+
# @!attribute [rw] filter
|
984
|
+
# The optional filter expression structure to apply additional
|
985
|
+
# matching criteria to the parameter.
|
986
|
+
# @return [Types::FilterExpression]
|
987
|
+
#
|
988
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/DatasetParameter AWS API Documentation
|
989
|
+
#
|
990
|
+
class DatasetParameter < Struct.new(
|
991
|
+
:name,
|
992
|
+
:type,
|
993
|
+
:datetime_options,
|
994
|
+
:create_column,
|
995
|
+
:filter)
|
996
|
+
SENSITIVE = []
|
997
|
+
include Aws::Structure
|
998
|
+
end
|
999
|
+
|
1000
|
+
# Represents additional options for correct interpretation of datetime
|
1001
|
+
# parameters used in the S3 path of a dataset.
|
1002
|
+
#
|
1003
|
+
# @note When making an API call, you may pass DatetimeOptions
|
1004
|
+
# data as a hash:
|
1005
|
+
#
|
1006
|
+
# {
|
1007
|
+
# format: "DatetimeFormat", # required
|
1008
|
+
# timezone_offset: "TimezoneOffset",
|
1009
|
+
# locale_code: "LocaleCode",
|
1010
|
+
# }
|
1011
|
+
#
|
1012
|
+
# @!attribute [rw] format
|
1013
|
+
# Required option, that defines the datetime format used for a date
|
1014
|
+
# parameter in the S3 path. Should use only supported datetime
|
1015
|
+
# specifiers and separation characters, all litera a-z or A-Z
|
1016
|
+
# character should be escaped with single quotes. E.g.
|
1017
|
+
# "MM.dd.yyyy-'at'-HH:mm".
|
1018
|
+
# @return [String]
|
1019
|
+
#
|
1020
|
+
# @!attribute [rw] timezone_offset
|
1021
|
+
# Optional value for a timezone offset of the datetime parameter value
|
1022
|
+
# in the S3 path. Shouldn't be used if Format for this parameter
|
1023
|
+
# includes timezone fields. If no offset specified, UTC is assumed.
|
1024
|
+
# @return [String]
|
1025
|
+
#
|
1026
|
+
# @!attribute [rw] locale_code
|
1027
|
+
# Optional value for a non-US locale code, needed for correct
|
1028
|
+
# interpretation of some date formats.
|
1029
|
+
# @return [String]
|
1030
|
+
#
|
1031
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/DatetimeOptions AWS API Documentation
|
1032
|
+
#
|
1033
|
+
class DatetimeOptions < Struct.new(
|
1034
|
+
:format,
|
1035
|
+
:timezone_offset,
|
1036
|
+
:locale_code)
|
1037
|
+
SENSITIVE = []
|
1038
|
+
include Aws::Structure
|
1039
|
+
end
|
1040
|
+
|
847
1041
|
# @note When making an API call, you may pass DeleteDatasetRequest
|
848
1042
|
# data as a hash:
|
849
1043
|
#
|
@@ -1044,18 +1238,18 @@ module Aws::GlueDataBrew
|
|
1044
1238
|
# @return [String]
|
1045
1239
|
#
|
1046
1240
|
# @!attribute [rw] format
|
1047
|
-
#
|
1241
|
+
# The file format of a dataset that is created from an S3 file or
|
1048
1242
|
# folder.
|
1049
1243
|
# @return [String]
|
1050
1244
|
#
|
1051
1245
|
# @!attribute [rw] format_options
|
1052
|
-
#
|
1053
|
-
# input.
|
1246
|
+
# Represents a set of options that define the structure of either
|
1247
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
1054
1248
|
# @return [Types::FormatOptions]
|
1055
1249
|
#
|
1056
1250
|
# @!attribute [rw] input
|
1057
|
-
#
|
1058
|
-
# Data Catalog or Amazon S3.
|
1251
|
+
# Represents information on how DataBrew can find data, in either the
|
1252
|
+
# AWS Glue Data Catalog or Amazon S3.
|
1059
1253
|
# @return [Types::Input]
|
1060
1254
|
#
|
1061
1255
|
# @!attribute [rw] last_modified_date
|
@@ -1072,6 +1266,11 @@ module Aws::GlueDataBrew
|
|
1072
1266
|
# Data Catalog.
|
1073
1267
|
# @return [String]
|
1074
1268
|
#
|
1269
|
+
# @!attribute [rw] path_options
|
1270
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
1271
|
+
# the dataset.
|
1272
|
+
# @return [Types::PathOptions]
|
1273
|
+
#
|
1075
1274
|
# @!attribute [rw] tags
|
1076
1275
|
# Metadata tags associated with this dataset.
|
1077
1276
|
# @return [Hash<String,String>]
|
@@ -1092,6 +1291,7 @@ module Aws::GlueDataBrew
|
|
1092
1291
|
:last_modified_date,
|
1093
1292
|
:last_modified_by,
|
1094
1293
|
:source,
|
1294
|
+
:path_options,
|
1095
1295
|
:tags,
|
1096
1296
|
:resource_arn)
|
1097
1297
|
SENSITIVE = []
|
@@ -1641,8 +1841,8 @@ module Aws::GlueDataBrew
|
|
1641
1841
|
include Aws::Structure
|
1642
1842
|
end
|
1643
1843
|
|
1644
|
-
#
|
1645
|
-
# file
|
1844
|
+
# Represents a set of options that define how DataBrew will interpret a
|
1845
|
+
# Microsoft Excel file when creating a dataset from that file.
|
1646
1846
|
#
|
1647
1847
|
# @note When making an API call, you may pass ExcelOptions
|
1648
1848
|
# data as a hash:
|
@@ -1654,18 +1854,19 @@ module Aws::GlueDataBrew
|
|
1654
1854
|
# }
|
1655
1855
|
#
|
1656
1856
|
# @!attribute [rw] sheet_names
|
1657
|
-
#
|
1658
|
-
#
|
1857
|
+
# One or more named sheets in the Excel file that will be included in
|
1858
|
+
# the dataset.
|
1659
1859
|
# @return [Array<String>]
|
1660
1860
|
#
|
1661
1861
|
# @!attribute [rw] sheet_indexes
|
1662
|
-
#
|
1663
|
-
#
|
1862
|
+
# One or more sheet numbers in the Excel file that will be included in
|
1863
|
+
# the dataset.
|
1664
1864
|
# @return [Array<Integer>]
|
1665
1865
|
#
|
1666
1866
|
# @!attribute [rw] header_row
|
1667
|
-
# A variable that specifies whether the first row in the file
|
1668
|
-
# parsed as the header. If false, column names
|
1867
|
+
# A variable that specifies whether the first row in the file is
|
1868
|
+
# parsed as the header. If this value is false, column names are
|
1869
|
+
# auto-generated.
|
1669
1870
|
# @return [Boolean]
|
1670
1871
|
#
|
1671
1872
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/ExcelOptions AWS API Documentation
|
@@ -1678,7 +1879,80 @@ module Aws::GlueDataBrew
|
|
1678
1879
|
include Aws::Structure
|
1679
1880
|
end
|
1680
1881
|
|
1681
|
-
#
|
1882
|
+
# Represents a limit imposed on number of S3 files that should be
|
1883
|
+
# selected for a dataset from a connected S3 path.
|
1884
|
+
#
|
1885
|
+
# @note When making an API call, you may pass FilesLimit
|
1886
|
+
# data as a hash:
|
1887
|
+
#
|
1888
|
+
# {
|
1889
|
+
# max_files: 1, # required
|
1890
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
1891
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
1892
|
+
# }
|
1893
|
+
#
|
1894
|
+
# @!attribute [rw] max_files
|
1895
|
+
# The number of S3 files to select.
|
1896
|
+
# @return [Integer]
|
1897
|
+
#
|
1898
|
+
# @!attribute [rw] ordered_by
|
1899
|
+
# A criteria to use for S3 files sorting before their selection. By
|
1900
|
+
# default uses LAST\_MODIFIED\_DATE as a sorting criteria. Currently
|
1901
|
+
# it's the only allowed value.
|
1902
|
+
# @return [String]
|
1903
|
+
#
|
1904
|
+
# @!attribute [rw] order
|
1905
|
+
# A criteria to use for S3 files sorting before their selection. By
|
1906
|
+
# default uses DESCENDING order, i.e. most recent files are selected
|
1907
|
+
# first. Anotherpossible value is ASCENDING.
|
1908
|
+
# @return [String]
|
1909
|
+
#
|
1910
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/FilesLimit AWS API Documentation
|
1911
|
+
#
|
1912
|
+
class FilesLimit < Struct.new(
|
1913
|
+
:max_files,
|
1914
|
+
:ordered_by,
|
1915
|
+
:order)
|
1916
|
+
SENSITIVE = []
|
1917
|
+
include Aws::Structure
|
1918
|
+
end
|
1919
|
+
|
1920
|
+
# Represents a structure for defining parameter conditions.
|
1921
|
+
#
|
1922
|
+
# @note When making an API call, you may pass FilterExpression
|
1923
|
+
# data as a hash:
|
1924
|
+
#
|
1925
|
+
# {
|
1926
|
+
# expression: "Expression", # required
|
1927
|
+
# values_map: { # required
|
1928
|
+
# "ValueReference" => "ConditionValue",
|
1929
|
+
# },
|
1930
|
+
# }
|
1931
|
+
#
|
1932
|
+
# @!attribute [rw] expression
|
1933
|
+
# The expression which includes condition names followed by
|
1934
|
+
# substitution variables, possibly grouped and combined with other
|
1935
|
+
# conditions. For example, "(starts\_with :prefix1 or starts\_with
|
1936
|
+
# :prefix2) and (ends\_with :suffix1 or ends\_with :suffix2)".
|
1937
|
+
# Substitution variables should start with ':' symbol.
|
1938
|
+
# @return [String]
|
1939
|
+
#
|
1940
|
+
# @!attribute [rw] values_map
|
1941
|
+
# The map of substitution variable names to their values used in this
|
1942
|
+
# filter expression.
|
1943
|
+
# @return [Hash<String,String>]
|
1944
|
+
#
|
1945
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/FilterExpression AWS API Documentation
|
1946
|
+
#
|
1947
|
+
class FilterExpression < Struct.new(
|
1948
|
+
:expression,
|
1949
|
+
:values_map)
|
1950
|
+
SENSITIVE = []
|
1951
|
+
include Aws::Structure
|
1952
|
+
end
|
1953
|
+
|
1954
|
+
# Represents a set of options that define the structure of either
|
1955
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
1682
1956
|
#
|
1683
1957
|
# @note When making an API call, you may pass FormatOptions
|
1684
1958
|
# data as a hash:
|
@@ -1708,7 +1982,7 @@ module Aws::GlueDataBrew
|
|
1708
1982
|
# @return [Types::ExcelOptions]
|
1709
1983
|
#
|
1710
1984
|
# @!attribute [rw] csv
|
1711
|
-
# Options that define how
|
1985
|
+
# Options that define how CSV input is to be interpreted by DataBrew.
|
1712
1986
|
# @return [Types::CsvOptions]
|
1713
1987
|
#
|
1714
1988
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/FormatOptions AWS API Documentation
|
@@ -1721,8 +1995,8 @@ module Aws::GlueDataBrew
|
|
1721
1995
|
include Aws::Structure
|
1722
1996
|
end
|
1723
1997
|
|
1724
|
-
#
|
1725
|
-
# Catalog or Amazon S3.
|
1998
|
+
# Represents information on how DataBrew can find data, in either the
|
1999
|
+
# AWS Glue Data Catalog or Amazon S3.
|
1726
2000
|
#
|
1727
2001
|
# @note When making an API call, you may pass Input
|
1728
2002
|
# data as a hash:
|
@@ -1741,6 +2015,14 @@ module Aws::GlueDataBrew
|
|
1741
2015
|
# key: "Key",
|
1742
2016
|
# },
|
1743
2017
|
# },
|
2018
|
+
# database_input_definition: {
|
2019
|
+
# glue_connection_name: "GlueConnectionName", # required
|
2020
|
+
# database_table_name: "DatabaseTableName", # required
|
2021
|
+
# temp_directory: {
|
2022
|
+
# bucket: "Bucket", # required
|
2023
|
+
# key: "Key",
|
2024
|
+
# },
|
2025
|
+
# },
|
1744
2026
|
# }
|
1745
2027
|
#
|
1746
2028
|
# @!attribute [rw] s3_input_definition
|
@@ -1751,11 +2033,16 @@ module Aws::GlueDataBrew
|
|
1751
2033
|
# The AWS Glue Data Catalog parameters for the data.
|
1752
2034
|
# @return [Types::DataCatalogInputDefinition]
|
1753
2035
|
#
|
2036
|
+
# @!attribute [rw] database_input_definition
|
2037
|
+
# Connection information for dataset input files stored in a database.
|
2038
|
+
# @return [Types::DatabaseInputDefinition]
|
2039
|
+
#
|
1754
2040
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/Input AWS API Documentation
|
1755
2041
|
#
|
1756
2042
|
class Input < Struct.new(
|
1757
2043
|
:s3_input_definition,
|
1758
|
-
:data_catalog_input_definition
|
2044
|
+
:data_catalog_input_definition,
|
2045
|
+
:database_input_definition)
|
1759
2046
|
SENSITIVE = []
|
1760
2047
|
include Aws::Structure
|
1761
2048
|
end
|
@@ -1804,7 +2091,7 @@ module Aws::GlueDataBrew
|
|
1804
2091
|
# @!attribute [rw] encryption_mode
|
1805
2092
|
# The encryption mode for the job, which can be one of the following:
|
1806
2093
|
#
|
1807
|
-
# * `SSE-KMS` - Server-side encryption with AWS KMS
|
2094
|
+
# * `SSE-KMS` - Server-side encryption with keys managed by AWS KMS.
|
1808
2095
|
#
|
1809
2096
|
# * `SSE-S3` - Server-side encryption with keys managed by Amazon S3.
|
1810
2097
|
# @return [String]
|
@@ -1862,8 +2149,8 @@ module Aws::GlueDataBrew
|
|
1862
2149
|
# @return [String]
|
1863
2150
|
#
|
1864
2151
|
# @!attribute [rw] role_arn
|
1865
|
-
# The Amazon Resource Name (ARN) of the role
|
1866
|
-
#
|
2152
|
+
# The Amazon Resource Name (ARN) of the role to be assumed for this
|
2153
|
+
# job.
|
1867
2154
|
# @return [String]
|
1868
2155
|
#
|
1869
2156
|
# @!attribute [rw] timeout
|
@@ -1876,10 +2163,10 @@ module Aws::GlueDataBrew
|
|
1876
2163
|
# @return [Hash<String,String>]
|
1877
2164
|
#
|
1878
2165
|
# @!attribute [rw] job_sample
|
1879
|
-
#
|
1880
|
-
# rows on which the profile job
|
1881
|
-
#
|
1882
|
-
# is CUSTOM\_ROWS for the mode parameter and
|
2166
|
+
# A sample configuration for profile jobs only, which determines the
|
2167
|
+
# number of rows on which the profile job is run. If a `JobSample`
|
2168
|
+
# value isn't provided, the default value is used. The default value
|
2169
|
+
# is CUSTOM\_ROWS for the mode parameter and 20,000 for the size
|
1883
2170
|
# parameter.
|
1884
2171
|
# @return [Types::JobSample]
|
1885
2172
|
#
|
@@ -1974,10 +2261,10 @@ module Aws::GlueDataBrew
|
|
1974
2261
|
# @return [Time]
|
1975
2262
|
#
|
1976
2263
|
# @!attribute [rw] job_sample
|
1977
|
-
#
|
1978
|
-
# rows on which the profile job
|
1979
|
-
#
|
1980
|
-
#
|
2264
|
+
# A sample configuration for profile jobs only, which determines the
|
2265
|
+
# number of rows on which the profile job is run. If a `JobSample`
|
2266
|
+
# value isn't provided, the default is used. The default value is
|
2267
|
+
# CUSTOM\_ROWS for the mode parameter and 20,000 for the size
|
1981
2268
|
# parameter.
|
1982
2269
|
# @return [Types::JobSample]
|
1983
2270
|
#
|
@@ -2003,11 +2290,10 @@ module Aws::GlueDataBrew
|
|
2003
2290
|
include Aws::Structure
|
2004
2291
|
end
|
2005
2292
|
|
2006
|
-
#
|
2007
|
-
# rows on which the
|
2008
|
-
#
|
2009
|
-
#
|
2010
|
-
# size parameter.
|
2293
|
+
# A sample configuration for profile jobs only, which determines the
|
2294
|
+
# number of rows on which the profile job is run. If a `JobSample` value
|
2295
|
+
# isn't provided, the default is used. The default value is
|
2296
|
+
# CUSTOM\_ROWS for the mode parameter and 20,000 for the size parameter.
|
2011
2297
|
#
|
2012
2298
|
# @note When making an API call, you may pass JobSample
|
2013
2299
|
# data as a hash:
|
@@ -2018,20 +2304,20 @@ module Aws::GlueDataBrew
|
|
2018
2304
|
# }
|
2019
2305
|
#
|
2020
2306
|
# @!attribute [rw] mode
|
2021
|
-
#
|
2022
|
-
# dataset or
|
2307
|
+
# A value that determines whether the profile job is run on the entire
|
2308
|
+
# dataset or a specified number of rows. This value must be one of the
|
2023
2309
|
# following:
|
2024
2310
|
#
|
2025
|
-
# * FULL\_DATASET
|
2311
|
+
# * FULL\_DATASET - The profile job is run on the entire dataset.
|
2026
2312
|
#
|
2027
|
-
# * CUSTOM\_ROWS
|
2028
|
-
# specified in the Size parameter.
|
2313
|
+
# * CUSTOM\_ROWS - The profile job is run on the number of rows
|
2314
|
+
# specified in the `Size` parameter.
|
2029
2315
|
# @return [String]
|
2030
2316
|
#
|
2031
2317
|
# @!attribute [rw] size
|
2032
|
-
# Size parameter is only required when the mode is CUSTOM\_ROWS.
|
2033
|
-
#
|
2034
|
-
#
|
2318
|
+
# The `Size` parameter is only required when the mode is CUSTOM\_ROWS.
|
2319
|
+
# The profile job is run on the specified number of rows. The maximum
|
2320
|
+
# value for size is Long.MAX\_VALUE.
|
2035
2321
|
#
|
2036
2322
|
# Long.MAX\_VALUE = 9223372036854775807
|
2037
2323
|
# @return [Integer]
|
@@ -2453,8 +2739,8 @@ module Aws::GlueDataBrew
|
|
2453
2739
|
include Aws::Structure
|
2454
2740
|
end
|
2455
2741
|
|
2456
|
-
#
|
2457
|
-
# generated by recipe jobs or profile jobs.
|
2742
|
+
# Represents options that specify how and where DataBrew writes the
|
2743
|
+
# output generated by recipe jobs or profile jobs.
|
2458
2744
|
#
|
2459
2745
|
# @note When making an API call, you may pass Output
|
2460
2746
|
# data as a hash:
|
@@ -2499,7 +2785,8 @@ module Aws::GlueDataBrew
|
|
2499
2785
|
# @return [Boolean]
|
2500
2786
|
#
|
2501
2787
|
# @!attribute [rw] format_options
|
2502
|
-
#
|
2788
|
+
# Represents options that define how DataBrew formats job output
|
2789
|
+
# files.
|
2503
2790
|
# @return [Types::OutputFormatOptions]
|
2504
2791
|
#
|
2505
2792
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/Output AWS API Documentation
|
@@ -2515,7 +2802,8 @@ module Aws::GlueDataBrew
|
|
2515
2802
|
include Aws::Structure
|
2516
2803
|
end
|
2517
2804
|
|
2518
|
-
#
|
2805
|
+
# Represents a set of options that define the structure of
|
2806
|
+
# comma-separated (CSV) job output.
|
2519
2807
|
#
|
2520
2808
|
# @note When making an API call, you may pass OutputFormatOptions
|
2521
2809
|
# data as a hash:
|
@@ -2527,7 +2815,8 @@ module Aws::GlueDataBrew
|
|
2527
2815
|
# }
|
2528
2816
|
#
|
2529
2817
|
# @!attribute [rw] csv
|
2530
|
-
#
|
2818
|
+
# Represents a set of options that define the structure of
|
2819
|
+
# comma-separated value (CSV) job output.
|
2531
2820
|
# @return [Types::CsvOutputOptions]
|
2532
2821
|
#
|
2533
2822
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/OutputFormatOptions AWS API Documentation
|
@@ -2538,6 +2827,69 @@ module Aws::GlueDataBrew
|
|
2538
2827
|
include Aws::Structure
|
2539
2828
|
end
|
2540
2829
|
|
2830
|
+
# Represents a set of options that define how DataBrew selects files for
|
2831
|
+
# a given S3 path in a dataset.
|
2832
|
+
#
|
2833
|
+
# @note When making an API call, you may pass PathOptions
|
2834
|
+
# data as a hash:
|
2835
|
+
#
|
2836
|
+
# {
|
2837
|
+
# last_modified_date_condition: {
|
2838
|
+
# expression: "Expression", # required
|
2839
|
+
# values_map: { # required
|
2840
|
+
# "ValueReference" => "ConditionValue",
|
2841
|
+
# },
|
2842
|
+
# },
|
2843
|
+
# files_limit: {
|
2844
|
+
# max_files: 1, # required
|
2845
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
2846
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
2847
|
+
# },
|
2848
|
+
# parameters: {
|
2849
|
+
# "PathParameterName" => {
|
2850
|
+
# name: "PathParameterName", # required
|
2851
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
2852
|
+
# datetime_options: {
|
2853
|
+
# format: "DatetimeFormat", # required
|
2854
|
+
# timezone_offset: "TimezoneOffset",
|
2855
|
+
# locale_code: "LocaleCode",
|
2856
|
+
# },
|
2857
|
+
# create_column: false,
|
2858
|
+
# filter: {
|
2859
|
+
# expression: "Expression", # required
|
2860
|
+
# values_map: { # required
|
2861
|
+
# "ValueReference" => "ConditionValue",
|
2862
|
+
# },
|
2863
|
+
# },
|
2864
|
+
# },
|
2865
|
+
# },
|
2866
|
+
# }
|
2867
|
+
#
|
2868
|
+
# @!attribute [rw] last_modified_date_condition
|
2869
|
+
# If provided, this structure defines a date range for matching S3
|
2870
|
+
# objects based on their LastModifiedDate attribute in S3.
|
2871
|
+
# @return [Types::FilterExpression]
|
2872
|
+
#
|
2873
|
+
# @!attribute [rw] files_limit
|
2874
|
+
# If provided, this structure imposes a limit on a number of files
|
2875
|
+
# that should be selected.
|
2876
|
+
# @return [Types::FilesLimit]
|
2877
|
+
#
|
2878
|
+
# @!attribute [rw] parameters
|
2879
|
+
# A structure that maps names of parameters used in the S3 path of a
|
2880
|
+
# dataset to their definitions.
|
2881
|
+
# @return [Hash<String,Types::DatasetParameter>]
|
2882
|
+
#
|
2883
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/PathOptions AWS API Documentation
|
2884
|
+
#
|
2885
|
+
class PathOptions < Struct.new(
|
2886
|
+
:last_modified_date_condition,
|
2887
|
+
:files_limit,
|
2888
|
+
:parameters)
|
2889
|
+
SENSITIVE = []
|
2890
|
+
include Aws::Structure
|
2891
|
+
end
|
2892
|
+
|
2541
2893
|
# Represents all of the attributes of a DataBrew project.
|
2542
2894
|
#
|
2543
2895
|
# @!attribute [rw] account_id
|
@@ -2580,8 +2932,8 @@ module Aws::GlueDataBrew
|
|
2580
2932
|
#
|
2581
2933
|
# @!attribute [rw] sample
|
2582
2934
|
# The sample size and sampling type to apply to the data. If this
|
2583
|
-
# parameter isn't specified, then the sample
|
2584
|
-
#
|
2935
|
+
# parameter isn't specified, then the sample consists of the first
|
2936
|
+
# 500 rows from the dataset.
|
2585
2937
|
# @return [Types::Sample]
|
2586
2938
|
#
|
2587
2939
|
# @!attribute [rw] tags
|
@@ -2719,7 +3071,7 @@ module Aws::GlueDataBrew
|
|
2719
3071
|
# * Numeric version (`X.Y`) - `X` and `Y` stand for major and minor
|
2720
3072
|
# version numbers. The maximum length of each is 6 digits, and
|
2721
3073
|
# neither can be negative values. Both `X` and `Y` are required, and
|
2722
|
-
# "0.0"
|
3074
|
+
# "0.0" isn't a valid version.
|
2723
3075
|
#
|
2724
3076
|
# * `LATEST_WORKING` - the most recent valid version being developed
|
2725
3077
|
# in a DataBrew project.
|
@@ -2837,8 +3189,8 @@ module Aws::GlueDataBrew
|
|
2837
3189
|
# @return [Types::RecipeAction]
|
2838
3190
|
#
|
2839
3191
|
# @!attribute [rw] condition_expressions
|
2840
|
-
# One or more conditions that must be met
|
2841
|
-
#
|
3192
|
+
# One or more conditions that must be met for the recipe step to
|
3193
|
+
# succeed.
|
2842
3194
|
#
|
2843
3195
|
# <note markdown="1"> All of the conditions in the array must be met. In other words, all
|
2844
3196
|
# of the conditions must be combined using a logical AND operation.
|
@@ -2893,8 +3245,8 @@ module Aws::GlueDataBrew
|
|
2893
3245
|
include Aws::Structure
|
2894
3246
|
end
|
2895
3247
|
|
2896
|
-
#
|
2897
|
-
# read input data, or write output from a job.
|
3248
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
3249
|
+
# DataBrew can read input data, or write output from a job.
|
2898
3250
|
#
|
2899
3251
|
# @note When making an API call, you may pass S3Location
|
2900
3252
|
# data as a hash:
|
@@ -2981,7 +3333,7 @@ module Aws::GlueDataBrew
|
|
2981
3333
|
# @return [String]
|
2982
3334
|
#
|
2983
3335
|
# @!attribute [rw] cron_expression
|
2984
|
-
# The
|
3336
|
+
# The dates and times when the job is to run. For more information,
|
2985
3337
|
# see [Cron expressions][1] in the *AWS Glue DataBrew Developer
|
2986
3338
|
# Guide*.
|
2987
3339
|
#
|
@@ -3071,7 +3423,7 @@ module Aws::GlueDataBrew
|
|
3071
3423
|
# @return [String]
|
3072
3424
|
#
|
3073
3425
|
# @!attribute [rw] view_frame
|
3074
|
-
# Represents the data being
|
3426
|
+
# Represents the data being transformed during an action.
|
3075
3427
|
# @return [Types::ViewFrame]
|
3076
3428
|
#
|
3077
3429
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/SendProjectSessionActionRequest AWS API Documentation
|
@@ -3330,6 +3682,45 @@ module Aws::GlueDataBrew
|
|
3330
3682
|
# key: "Key",
|
3331
3683
|
# },
|
3332
3684
|
# },
|
3685
|
+
# database_input_definition: {
|
3686
|
+
# glue_connection_name: "GlueConnectionName", # required
|
3687
|
+
# database_table_name: "DatabaseTableName", # required
|
3688
|
+
# temp_directory: {
|
3689
|
+
# bucket: "Bucket", # required
|
3690
|
+
# key: "Key",
|
3691
|
+
# },
|
3692
|
+
# },
|
3693
|
+
# },
|
3694
|
+
# path_options: {
|
3695
|
+
# last_modified_date_condition: {
|
3696
|
+
# expression: "Expression", # required
|
3697
|
+
# values_map: { # required
|
3698
|
+
# "ValueReference" => "ConditionValue",
|
3699
|
+
# },
|
3700
|
+
# },
|
3701
|
+
# files_limit: {
|
3702
|
+
# max_files: 1, # required
|
3703
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
3704
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
3705
|
+
# },
|
3706
|
+
# parameters: {
|
3707
|
+
# "PathParameterName" => {
|
3708
|
+
# name: "PathParameterName", # required
|
3709
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
3710
|
+
# datetime_options: {
|
3711
|
+
# format: "DatetimeFormat", # required
|
3712
|
+
# timezone_offset: "TimezoneOffset",
|
3713
|
+
# locale_code: "LocaleCode",
|
3714
|
+
# },
|
3715
|
+
# create_column: false,
|
3716
|
+
# filter: {
|
3717
|
+
# expression: "Expression", # required
|
3718
|
+
# values_map: { # required
|
3719
|
+
# "ValueReference" => "ConditionValue",
|
3720
|
+
# },
|
3721
|
+
# },
|
3722
|
+
# },
|
3723
|
+
# },
|
3333
3724
|
# },
|
3334
3725
|
# }
|
3335
3726
|
#
|
@@ -3338,27 +3729,33 @@ module Aws::GlueDataBrew
|
|
3338
3729
|
# @return [String]
|
3339
3730
|
#
|
3340
3731
|
# @!attribute [rw] format
|
3341
|
-
#
|
3732
|
+
# The file format of a dataset that is created from an S3 file or
|
3342
3733
|
# folder.
|
3343
3734
|
# @return [String]
|
3344
3735
|
#
|
3345
3736
|
# @!attribute [rw] format_options
|
3346
|
-
#
|
3347
|
-
# input.
|
3737
|
+
# Represents a set of options that define the structure of either
|
3738
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
3348
3739
|
# @return [Types::FormatOptions]
|
3349
3740
|
#
|
3350
3741
|
# @!attribute [rw] input
|
3351
|
-
#
|
3352
|
-
# Data Catalog or Amazon S3.
|
3742
|
+
# Represents information on how DataBrew can find data, in either the
|
3743
|
+
# AWS Glue Data Catalog or Amazon S3.
|
3353
3744
|
# @return [Types::Input]
|
3354
3745
|
#
|
3746
|
+
# @!attribute [rw] path_options
|
3747
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
3748
|
+
# the dataset.
|
3749
|
+
# @return [Types::PathOptions]
|
3750
|
+
#
|
3355
3751
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/UpdateDatasetRequest AWS API Documentation
|
3356
3752
|
#
|
3357
3753
|
class UpdateDatasetRequest < Struct.new(
|
3358
3754
|
:name,
|
3359
3755
|
:format,
|
3360
3756
|
:format_options,
|
3361
|
-
:input
|
3757
|
+
:input,
|
3758
|
+
:path_options)
|
3362
3759
|
SENSITIVE = []
|
3363
3760
|
include Aws::Structure
|
3364
3761
|
end
|
@@ -3430,8 +3827,8 @@ module Aws::GlueDataBrew
|
|
3430
3827
|
# @return [Integer]
|
3431
3828
|
#
|
3432
3829
|
# @!attribute [rw] output_location
|
3433
|
-
#
|
3434
|
-
# read input data, or write output from a job.
|
3830
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
3831
|
+
# DataBrew can read input data, or write output from a job.
|
3435
3832
|
# @return [Types::S3Location]
|
3436
3833
|
#
|
3437
3834
|
# @!attribute [rw] role_arn
|
@@ -3763,7 +4160,7 @@ module Aws::GlueDataBrew
|
|
3763
4160
|
include Aws::Structure
|
3764
4161
|
end
|
3765
4162
|
|
3766
|
-
# Represents the data being
|
4163
|
+
# Represents the data being transformed during an action.
|
3767
4164
|
#
|
3768
4165
|
# @note When making an API call, you may pass ViewFrame
|
3769
4166
|
# data as a hash:
|