aws-sdk-gluedatabrew 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-gluedatabrew.rb +1 -1
- data/lib/aws-sdk-gluedatabrew/client.rb +145 -16
- data/lib/aws-sdk-gluedatabrew/client_api.rb +63 -0
- data/lib/aws-sdk-gluedatabrew/types.rb +479 -82
- metadata +4 -4
@@ -70,7 +70,7 @@ module Aws::GlueDataBrew
|
|
70
70
|
|
71
71
|
# Represents an individual condition that evaluates to true or false.
|
72
72
|
#
|
73
|
-
# Conditions are used with recipe actions
|
73
|
+
# Conditions are used with recipe actions. The action is only performed
|
74
74
|
# for column values where the condition evaluates to true.
|
75
75
|
#
|
76
76
|
# If a recipe requires more than one condition, then the recipe must
|
@@ -163,6 +163,45 @@ module Aws::GlueDataBrew
|
|
163
163
|
# key: "Key",
|
164
164
|
# },
|
165
165
|
# },
|
166
|
+
# database_input_definition: {
|
167
|
+
# glue_connection_name: "GlueConnectionName", # required
|
168
|
+
# database_table_name: "DatabaseTableName", # required
|
169
|
+
# temp_directory: {
|
170
|
+
# bucket: "Bucket", # required
|
171
|
+
# key: "Key",
|
172
|
+
# },
|
173
|
+
# },
|
174
|
+
# },
|
175
|
+
# path_options: {
|
176
|
+
# last_modified_date_condition: {
|
177
|
+
# expression: "Expression", # required
|
178
|
+
# values_map: { # required
|
179
|
+
# "ValueReference" => "ConditionValue",
|
180
|
+
# },
|
181
|
+
# },
|
182
|
+
# files_limit: {
|
183
|
+
# max_files: 1, # required
|
184
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
185
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
186
|
+
# },
|
187
|
+
# parameters: {
|
188
|
+
# "PathParameterName" => {
|
189
|
+
# name: "PathParameterName", # required
|
190
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
191
|
+
# datetime_options: {
|
192
|
+
# format: "DatetimeFormat", # required
|
193
|
+
# timezone_offset: "TimezoneOffset",
|
194
|
+
# locale_code: "LocaleCode",
|
195
|
+
# },
|
196
|
+
# create_column: false,
|
197
|
+
# filter: {
|
198
|
+
# expression: "Expression", # required
|
199
|
+
# values_map: { # required
|
200
|
+
# "ValueReference" => "ConditionValue",
|
201
|
+
# },
|
202
|
+
# },
|
203
|
+
# },
|
204
|
+
# },
|
166
205
|
# },
|
167
206
|
# tags: {
|
168
207
|
# "TagKey" => "TagValue",
|
@@ -175,20 +214,25 @@ module Aws::GlueDataBrew
|
|
175
214
|
# @return [String]
|
176
215
|
#
|
177
216
|
# @!attribute [rw] format
|
178
|
-
#
|
217
|
+
# The file format of a dataset that is created from an S3 file or
|
179
218
|
# folder.
|
180
219
|
# @return [String]
|
181
220
|
#
|
182
221
|
# @!attribute [rw] format_options
|
183
|
-
#
|
184
|
-
# input.
|
222
|
+
# Represents a set of options that define the structure of either
|
223
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
185
224
|
# @return [Types::FormatOptions]
|
186
225
|
#
|
187
226
|
# @!attribute [rw] input
|
188
|
-
#
|
189
|
-
# Data Catalog or Amazon S3.
|
227
|
+
# Represents information on how DataBrew can find data, in either the
|
228
|
+
# AWS Glue Data Catalog or Amazon S3.
|
190
229
|
# @return [Types::Input]
|
191
230
|
#
|
231
|
+
# @!attribute [rw] path_options
|
232
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
233
|
+
# the dataset.
|
234
|
+
# @return [Types::PathOptions]
|
235
|
+
#
|
192
236
|
# @!attribute [rw] tags
|
193
237
|
# Metadata tags to apply to this dataset.
|
194
238
|
# @return [Hash<String,String>]
|
@@ -200,6 +244,7 @@ module Aws::GlueDataBrew
|
|
200
244
|
:format,
|
201
245
|
:format_options,
|
202
246
|
:input,
|
247
|
+
:path_options,
|
203
248
|
:tags)
|
204
249
|
SENSITIVE = []
|
205
250
|
include Aws::Structure
|
@@ -282,8 +327,8 @@ module Aws::GlueDataBrew
|
|
282
327
|
# @return [Integer]
|
283
328
|
#
|
284
329
|
# @!attribute [rw] output_location
|
285
|
-
#
|
286
|
-
# read input data, or write output from a job.
|
330
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
331
|
+
# DataBrew can read input data, or write output from a job.
|
287
332
|
# @return [Types::S3Location]
|
288
333
|
#
|
289
334
|
# @!attribute [rw] role_arn
|
@@ -673,8 +718,9 @@ module Aws::GlueDataBrew
|
|
673
718
|
include Aws::Structure
|
674
719
|
end
|
675
720
|
|
676
|
-
#
|
677
|
-
# dataset from that
|
721
|
+
# Represents a set of options that define how DataBrew will read a
|
722
|
+
# comma-separated value (CSV) file when creating a dataset from that
|
723
|
+
# file.
|
678
724
|
#
|
679
725
|
# @note When making an API call, you may pass CsvOptions
|
680
726
|
# data as a hash:
|
@@ -686,12 +732,13 @@ module Aws::GlueDataBrew
|
|
686
732
|
#
|
687
733
|
# @!attribute [rw] delimiter
|
688
734
|
# A single character that specifies the delimiter being used in the
|
689
|
-
#
|
735
|
+
# CSV file.
|
690
736
|
# @return [String]
|
691
737
|
#
|
692
738
|
# @!attribute [rw] header_row
|
693
|
-
# A variable that specifies whether the first row in the file
|
694
|
-
# parsed as the header. If false, column names
|
739
|
+
# A variable that specifies whether the first row in the file is
|
740
|
+
# parsed as the header. If this value is false, column names are
|
741
|
+
# auto-generated.
|
695
742
|
# @return [Boolean]
|
696
743
|
#
|
697
744
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/CsvOptions AWS API Documentation
|
@@ -703,7 +750,8 @@ module Aws::GlueDataBrew
|
|
703
750
|
include Aws::Structure
|
704
751
|
end
|
705
752
|
|
706
|
-
#
|
753
|
+
# Represents a set of options that define how DataBrew will write a
|
754
|
+
# comma-separated value (CSV) file.
|
707
755
|
#
|
708
756
|
# @note When making an API call, you may pass CsvOutputOptions
|
709
757
|
# data as a hash:
|
@@ -713,7 +761,7 @@ module Aws::GlueDataBrew
|
|
713
761
|
# }
|
714
762
|
#
|
715
763
|
# @!attribute [rw] delimiter
|
716
|
-
# A single character that specifies the delimiter used to create
|
764
|
+
# A single character that specifies the delimiter used to create CSV
|
717
765
|
# job output.
|
718
766
|
# @return [String]
|
719
767
|
#
|
@@ -771,6 +819,44 @@ module Aws::GlueDataBrew
|
|
771
819
|
include Aws::Structure
|
772
820
|
end
|
773
821
|
|
822
|
+
# Connection information for dataset input files stored in a database.
|
823
|
+
#
|
824
|
+
# @note When making an API call, you may pass DatabaseInputDefinition
|
825
|
+
# data as a hash:
|
826
|
+
#
|
827
|
+
# {
|
828
|
+
# glue_connection_name: "GlueConnectionName", # required
|
829
|
+
# database_table_name: "DatabaseTableName", # required
|
830
|
+
# temp_directory: {
|
831
|
+
# bucket: "Bucket", # required
|
832
|
+
# key: "Key",
|
833
|
+
# },
|
834
|
+
# }
|
835
|
+
#
|
836
|
+
# @!attribute [rw] glue_connection_name
|
837
|
+
# The AWS Glue Connection that stores the connection information for
|
838
|
+
# the target database.
|
839
|
+
# @return [String]
|
840
|
+
#
|
841
|
+
# @!attribute [rw] database_table_name
|
842
|
+
# The table within the target database.
|
843
|
+
# @return [String]
|
844
|
+
#
|
845
|
+
# @!attribute [rw] temp_directory
|
846
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
847
|
+
# DataBrew can read input data, or write output from a job.
|
848
|
+
# @return [Types::S3Location]
|
849
|
+
#
|
850
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/DatabaseInputDefinition AWS API Documentation
|
851
|
+
#
|
852
|
+
class DatabaseInputDefinition < Struct.new(
|
853
|
+
:glue_connection_name,
|
854
|
+
:database_table_name,
|
855
|
+
:temp_directory)
|
856
|
+
SENSITIVE = []
|
857
|
+
include Aws::Structure
|
858
|
+
end
|
859
|
+
|
774
860
|
# Represents a dataset that can be processed by DataBrew.
|
775
861
|
#
|
776
862
|
# @!attribute [rw] account_id
|
@@ -790,12 +876,13 @@ module Aws::GlueDataBrew
|
|
790
876
|
# @return [String]
|
791
877
|
#
|
792
878
|
# @!attribute [rw] format
|
793
|
-
#
|
879
|
+
# The file format of a dataset that is created from an S3 file or
|
794
880
|
# folder.
|
795
881
|
# @return [String]
|
796
882
|
#
|
797
883
|
# @!attribute [rw] format_options
|
798
|
-
#
|
884
|
+
# A set of options that define how DataBrew interprets the data in the
|
885
|
+
# dataset.
|
799
886
|
# @return [Types::FormatOptions]
|
800
887
|
#
|
801
888
|
# @!attribute [rw] input
|
@@ -817,6 +904,11 @@ module Aws::GlueDataBrew
|
|
817
904
|
# AWS Glue Data Catalog.
|
818
905
|
# @return [String]
|
819
906
|
#
|
907
|
+
# @!attribute [rw] path_options
|
908
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
909
|
+
# the dataset.
|
910
|
+
# @return [Types::PathOptions]
|
911
|
+
#
|
820
912
|
# @!attribute [rw] tags
|
821
913
|
# Metadata tags that have been applied to the dataset.
|
822
914
|
# @return [Hash<String,String>]
|
@@ -838,12 +930,114 @@ module Aws::GlueDataBrew
|
|
838
930
|
:last_modified_date,
|
839
931
|
:last_modified_by,
|
840
932
|
:source,
|
933
|
+
:path_options,
|
841
934
|
:tags,
|
842
935
|
:resource_arn)
|
843
936
|
SENSITIVE = []
|
844
937
|
include Aws::Structure
|
845
938
|
end
|
846
939
|
|
940
|
+
# Represents a dataset paramater that defines type and conditions for a
|
941
|
+
# parameter in the S3 path of the dataset.
|
942
|
+
#
|
943
|
+
# @note When making an API call, you may pass DatasetParameter
|
944
|
+
# data as a hash:
|
945
|
+
#
|
946
|
+
# {
|
947
|
+
# name: "PathParameterName", # required
|
948
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
949
|
+
# datetime_options: {
|
950
|
+
# format: "DatetimeFormat", # required
|
951
|
+
# timezone_offset: "TimezoneOffset",
|
952
|
+
# locale_code: "LocaleCode",
|
953
|
+
# },
|
954
|
+
# create_column: false,
|
955
|
+
# filter: {
|
956
|
+
# expression: "Expression", # required
|
957
|
+
# values_map: { # required
|
958
|
+
# "ValueReference" => "ConditionValue",
|
959
|
+
# },
|
960
|
+
# },
|
961
|
+
# }
|
962
|
+
#
|
963
|
+
# @!attribute [rw] name
|
964
|
+
# The name of the parameter that is used in the dataset's S3 path.
|
965
|
+
# @return [String]
|
966
|
+
#
|
967
|
+
# @!attribute [rw] type
|
968
|
+
# The type of the dataset parameter, can be one of a 'String',
|
969
|
+
# 'Number' or 'Datetime'.
|
970
|
+
# @return [String]
|
971
|
+
#
|
972
|
+
# @!attribute [rw] datetime_options
|
973
|
+
# Additional parameter options such as a format and a timezone.
|
974
|
+
# Required for datetime parameters.
|
975
|
+
# @return [Types::DatetimeOptions]
|
976
|
+
#
|
977
|
+
# @!attribute [rw] create_column
|
978
|
+
# Optional boolean value that defines whether the captured value of
|
979
|
+
# this parameter should be loaded as an additional column in the
|
980
|
+
# dataset.
|
981
|
+
# @return [Boolean]
|
982
|
+
#
|
983
|
+
# @!attribute [rw] filter
|
984
|
+
# The optional filter expression structure to apply additional
|
985
|
+
# matching criteria to the parameter.
|
986
|
+
# @return [Types::FilterExpression]
|
987
|
+
#
|
988
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/DatasetParameter AWS API Documentation
|
989
|
+
#
|
990
|
+
class DatasetParameter < Struct.new(
|
991
|
+
:name,
|
992
|
+
:type,
|
993
|
+
:datetime_options,
|
994
|
+
:create_column,
|
995
|
+
:filter)
|
996
|
+
SENSITIVE = []
|
997
|
+
include Aws::Structure
|
998
|
+
end
|
999
|
+
|
1000
|
+
# Represents additional options for correct interpretation of datetime
|
1001
|
+
# parameters used in the S3 path of a dataset.
|
1002
|
+
#
|
1003
|
+
# @note When making an API call, you may pass DatetimeOptions
|
1004
|
+
# data as a hash:
|
1005
|
+
#
|
1006
|
+
# {
|
1007
|
+
# format: "DatetimeFormat", # required
|
1008
|
+
# timezone_offset: "TimezoneOffset",
|
1009
|
+
# locale_code: "LocaleCode",
|
1010
|
+
# }
|
1011
|
+
#
|
1012
|
+
# @!attribute [rw] format
|
1013
|
+
# Required option, that defines the datetime format used for a date
|
1014
|
+
# parameter in the S3 path. Should use only supported datetime
|
1015
|
+
# specifiers and separation characters, all litera a-z or A-Z
|
1016
|
+
# character should be escaped with single quotes. E.g.
|
1017
|
+
# "MM.dd.yyyy-'at'-HH:mm".
|
1018
|
+
# @return [String]
|
1019
|
+
#
|
1020
|
+
# @!attribute [rw] timezone_offset
|
1021
|
+
# Optional value for a timezone offset of the datetime parameter value
|
1022
|
+
# in the S3 path. Shouldn't be used if Format for this parameter
|
1023
|
+
# includes timezone fields. If no offset specified, UTC is assumed.
|
1024
|
+
# @return [String]
|
1025
|
+
#
|
1026
|
+
# @!attribute [rw] locale_code
|
1027
|
+
# Optional value for a non-US locale code, needed for correct
|
1028
|
+
# interpretation of some date formats.
|
1029
|
+
# @return [String]
|
1030
|
+
#
|
1031
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/DatetimeOptions AWS API Documentation
|
1032
|
+
#
|
1033
|
+
class DatetimeOptions < Struct.new(
|
1034
|
+
:format,
|
1035
|
+
:timezone_offset,
|
1036
|
+
:locale_code)
|
1037
|
+
SENSITIVE = []
|
1038
|
+
include Aws::Structure
|
1039
|
+
end
|
1040
|
+
|
847
1041
|
# @note When making an API call, you may pass DeleteDatasetRequest
|
848
1042
|
# data as a hash:
|
849
1043
|
#
|
@@ -1044,18 +1238,18 @@ module Aws::GlueDataBrew
|
|
1044
1238
|
# @return [String]
|
1045
1239
|
#
|
1046
1240
|
# @!attribute [rw] format
|
1047
|
-
#
|
1241
|
+
# The file format of a dataset that is created from an S3 file or
|
1048
1242
|
# folder.
|
1049
1243
|
# @return [String]
|
1050
1244
|
#
|
1051
1245
|
# @!attribute [rw] format_options
|
1052
|
-
#
|
1053
|
-
# input.
|
1246
|
+
# Represents a set of options that define the structure of either
|
1247
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
1054
1248
|
# @return [Types::FormatOptions]
|
1055
1249
|
#
|
1056
1250
|
# @!attribute [rw] input
|
1057
|
-
#
|
1058
|
-
# Data Catalog or Amazon S3.
|
1251
|
+
# Represents information on how DataBrew can find data, in either the
|
1252
|
+
# AWS Glue Data Catalog or Amazon S3.
|
1059
1253
|
# @return [Types::Input]
|
1060
1254
|
#
|
1061
1255
|
# @!attribute [rw] last_modified_date
|
@@ -1072,6 +1266,11 @@ module Aws::GlueDataBrew
|
|
1072
1266
|
# Data Catalog.
|
1073
1267
|
# @return [String]
|
1074
1268
|
#
|
1269
|
+
# @!attribute [rw] path_options
|
1270
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
1271
|
+
# the dataset.
|
1272
|
+
# @return [Types::PathOptions]
|
1273
|
+
#
|
1075
1274
|
# @!attribute [rw] tags
|
1076
1275
|
# Metadata tags associated with this dataset.
|
1077
1276
|
# @return [Hash<String,String>]
|
@@ -1092,6 +1291,7 @@ module Aws::GlueDataBrew
|
|
1092
1291
|
:last_modified_date,
|
1093
1292
|
:last_modified_by,
|
1094
1293
|
:source,
|
1294
|
+
:path_options,
|
1095
1295
|
:tags,
|
1096
1296
|
:resource_arn)
|
1097
1297
|
SENSITIVE = []
|
@@ -1641,8 +1841,8 @@ module Aws::GlueDataBrew
|
|
1641
1841
|
include Aws::Structure
|
1642
1842
|
end
|
1643
1843
|
|
1644
|
-
#
|
1645
|
-
# file
|
1844
|
+
# Represents a set of options that define how DataBrew will interpret a
|
1845
|
+
# Microsoft Excel file when creating a dataset from that file.
|
1646
1846
|
#
|
1647
1847
|
# @note When making an API call, you may pass ExcelOptions
|
1648
1848
|
# data as a hash:
|
@@ -1654,18 +1854,19 @@ module Aws::GlueDataBrew
|
|
1654
1854
|
# }
|
1655
1855
|
#
|
1656
1856
|
# @!attribute [rw] sheet_names
|
1657
|
-
#
|
1658
|
-
#
|
1857
|
+
# One or more named sheets in the Excel file that will be included in
|
1858
|
+
# the dataset.
|
1659
1859
|
# @return [Array<String>]
|
1660
1860
|
#
|
1661
1861
|
# @!attribute [rw] sheet_indexes
|
1662
|
-
#
|
1663
|
-
#
|
1862
|
+
# One or more sheet numbers in the Excel file that will be included in
|
1863
|
+
# the dataset.
|
1664
1864
|
# @return [Array<Integer>]
|
1665
1865
|
#
|
1666
1866
|
# @!attribute [rw] header_row
|
1667
|
-
# A variable that specifies whether the first row in the file
|
1668
|
-
# parsed as the header. If false, column names
|
1867
|
+
# A variable that specifies whether the first row in the file is
|
1868
|
+
# parsed as the header. If this value is false, column names are
|
1869
|
+
# auto-generated.
|
1669
1870
|
# @return [Boolean]
|
1670
1871
|
#
|
1671
1872
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/ExcelOptions AWS API Documentation
|
@@ -1678,7 +1879,80 @@ module Aws::GlueDataBrew
|
|
1678
1879
|
include Aws::Structure
|
1679
1880
|
end
|
1680
1881
|
|
1681
|
-
#
|
1882
|
+
# Represents a limit imposed on number of S3 files that should be
|
1883
|
+
# selected for a dataset from a connected S3 path.
|
1884
|
+
#
|
1885
|
+
# @note When making an API call, you may pass FilesLimit
|
1886
|
+
# data as a hash:
|
1887
|
+
#
|
1888
|
+
# {
|
1889
|
+
# max_files: 1, # required
|
1890
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
1891
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
1892
|
+
# }
|
1893
|
+
#
|
1894
|
+
# @!attribute [rw] max_files
|
1895
|
+
# The number of S3 files to select.
|
1896
|
+
# @return [Integer]
|
1897
|
+
#
|
1898
|
+
# @!attribute [rw] ordered_by
|
1899
|
+
# A criteria to use for S3 files sorting before their selection. By
|
1900
|
+
# default uses LAST\_MODIFIED\_DATE as a sorting criteria. Currently
|
1901
|
+
# it's the only allowed value.
|
1902
|
+
# @return [String]
|
1903
|
+
#
|
1904
|
+
# @!attribute [rw] order
|
1905
|
+
# A criteria to use for S3 files sorting before their selection. By
|
1906
|
+
# default uses DESCENDING order, i.e. most recent files are selected
|
1907
|
+
# first. Anotherpossible value is ASCENDING.
|
1908
|
+
# @return [String]
|
1909
|
+
#
|
1910
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/FilesLimit AWS API Documentation
|
1911
|
+
#
|
1912
|
+
class FilesLimit < Struct.new(
|
1913
|
+
:max_files,
|
1914
|
+
:ordered_by,
|
1915
|
+
:order)
|
1916
|
+
SENSITIVE = []
|
1917
|
+
include Aws::Structure
|
1918
|
+
end
|
1919
|
+
|
1920
|
+
# Represents a structure for defining parameter conditions.
|
1921
|
+
#
|
1922
|
+
# @note When making an API call, you may pass FilterExpression
|
1923
|
+
# data as a hash:
|
1924
|
+
#
|
1925
|
+
# {
|
1926
|
+
# expression: "Expression", # required
|
1927
|
+
# values_map: { # required
|
1928
|
+
# "ValueReference" => "ConditionValue",
|
1929
|
+
# },
|
1930
|
+
# }
|
1931
|
+
#
|
1932
|
+
# @!attribute [rw] expression
|
1933
|
+
# The expression which includes condition names followed by
|
1934
|
+
# substitution variables, possibly grouped and combined with other
|
1935
|
+
# conditions. For example, "(starts\_with :prefix1 or starts\_with
|
1936
|
+
# :prefix2) and (ends\_with :suffix1 or ends\_with :suffix2)".
|
1937
|
+
# Substitution variables should start with ':' symbol.
|
1938
|
+
# @return [String]
|
1939
|
+
#
|
1940
|
+
# @!attribute [rw] values_map
|
1941
|
+
# The map of substitution variable names to their values used in this
|
1942
|
+
# filter expression.
|
1943
|
+
# @return [Hash<String,String>]
|
1944
|
+
#
|
1945
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/FilterExpression AWS API Documentation
|
1946
|
+
#
|
1947
|
+
class FilterExpression < Struct.new(
|
1948
|
+
:expression,
|
1949
|
+
:values_map)
|
1950
|
+
SENSITIVE = []
|
1951
|
+
include Aws::Structure
|
1952
|
+
end
|
1953
|
+
|
1954
|
+
# Represents a set of options that define the structure of either
|
1955
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
1682
1956
|
#
|
1683
1957
|
# @note When making an API call, you may pass FormatOptions
|
1684
1958
|
# data as a hash:
|
@@ -1708,7 +1982,7 @@ module Aws::GlueDataBrew
|
|
1708
1982
|
# @return [Types::ExcelOptions]
|
1709
1983
|
#
|
1710
1984
|
# @!attribute [rw] csv
|
1711
|
-
# Options that define how
|
1985
|
+
# Options that define how CSV input is to be interpreted by DataBrew.
|
1712
1986
|
# @return [Types::CsvOptions]
|
1713
1987
|
#
|
1714
1988
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/FormatOptions AWS API Documentation
|
@@ -1721,8 +1995,8 @@ module Aws::GlueDataBrew
|
|
1721
1995
|
include Aws::Structure
|
1722
1996
|
end
|
1723
1997
|
|
1724
|
-
#
|
1725
|
-
# Catalog or Amazon S3.
|
1998
|
+
# Represents information on how DataBrew can find data, in either the
|
1999
|
+
# AWS Glue Data Catalog or Amazon S3.
|
1726
2000
|
#
|
1727
2001
|
# @note When making an API call, you may pass Input
|
1728
2002
|
# data as a hash:
|
@@ -1741,6 +2015,14 @@ module Aws::GlueDataBrew
|
|
1741
2015
|
# key: "Key",
|
1742
2016
|
# },
|
1743
2017
|
# },
|
2018
|
+
# database_input_definition: {
|
2019
|
+
# glue_connection_name: "GlueConnectionName", # required
|
2020
|
+
# database_table_name: "DatabaseTableName", # required
|
2021
|
+
# temp_directory: {
|
2022
|
+
# bucket: "Bucket", # required
|
2023
|
+
# key: "Key",
|
2024
|
+
# },
|
2025
|
+
# },
|
1744
2026
|
# }
|
1745
2027
|
#
|
1746
2028
|
# @!attribute [rw] s3_input_definition
|
@@ -1751,11 +2033,16 @@ module Aws::GlueDataBrew
|
|
1751
2033
|
# The AWS Glue Data Catalog parameters for the data.
|
1752
2034
|
# @return [Types::DataCatalogInputDefinition]
|
1753
2035
|
#
|
2036
|
+
# @!attribute [rw] database_input_definition
|
2037
|
+
# Connection information for dataset input files stored in a database.
|
2038
|
+
# @return [Types::DatabaseInputDefinition]
|
2039
|
+
#
|
1754
2040
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/Input AWS API Documentation
|
1755
2041
|
#
|
1756
2042
|
class Input < Struct.new(
|
1757
2043
|
:s3_input_definition,
|
1758
|
-
:data_catalog_input_definition
|
2044
|
+
:data_catalog_input_definition,
|
2045
|
+
:database_input_definition)
|
1759
2046
|
SENSITIVE = []
|
1760
2047
|
include Aws::Structure
|
1761
2048
|
end
|
@@ -1804,7 +2091,7 @@ module Aws::GlueDataBrew
|
|
1804
2091
|
# @!attribute [rw] encryption_mode
|
1805
2092
|
# The encryption mode for the job, which can be one of the following:
|
1806
2093
|
#
|
1807
|
-
# * `SSE-KMS` - Server-side encryption with AWS KMS
|
2094
|
+
# * `SSE-KMS` - Server-side encryption with keys managed by AWS KMS.
|
1808
2095
|
#
|
1809
2096
|
# * `SSE-S3` - Server-side encryption with keys managed by Amazon S3.
|
1810
2097
|
# @return [String]
|
@@ -1862,8 +2149,8 @@ module Aws::GlueDataBrew
|
|
1862
2149
|
# @return [String]
|
1863
2150
|
#
|
1864
2151
|
# @!attribute [rw] role_arn
|
1865
|
-
# The Amazon Resource Name (ARN) of the role
|
1866
|
-
#
|
2152
|
+
# The Amazon Resource Name (ARN) of the role to be assumed for this
|
2153
|
+
# job.
|
1867
2154
|
# @return [String]
|
1868
2155
|
#
|
1869
2156
|
# @!attribute [rw] timeout
|
@@ -1876,10 +2163,10 @@ module Aws::GlueDataBrew
|
|
1876
2163
|
# @return [Hash<String,String>]
|
1877
2164
|
#
|
1878
2165
|
# @!attribute [rw] job_sample
|
1879
|
-
#
|
1880
|
-
# rows on which the profile job
|
1881
|
-
#
|
1882
|
-
# is CUSTOM\_ROWS for the mode parameter and
|
2166
|
+
# A sample configuration for profile jobs only, which determines the
|
2167
|
+
# number of rows on which the profile job is run. If a `JobSample`
|
2168
|
+
# value isn't provided, the default value is used. The default value
|
2169
|
+
# is CUSTOM\_ROWS for the mode parameter and 20,000 for the size
|
1883
2170
|
# parameter.
|
1884
2171
|
# @return [Types::JobSample]
|
1885
2172
|
#
|
@@ -1974,10 +2261,10 @@ module Aws::GlueDataBrew
|
|
1974
2261
|
# @return [Time]
|
1975
2262
|
#
|
1976
2263
|
# @!attribute [rw] job_sample
|
1977
|
-
#
|
1978
|
-
# rows on which the profile job
|
1979
|
-
#
|
1980
|
-
#
|
2264
|
+
# A sample configuration for profile jobs only, which determines the
|
2265
|
+
# number of rows on which the profile job is run. If a `JobSample`
|
2266
|
+
# value isn't provided, the default is used. The default value is
|
2267
|
+
# CUSTOM\_ROWS for the mode parameter and 20,000 for the size
|
1981
2268
|
# parameter.
|
1982
2269
|
# @return [Types::JobSample]
|
1983
2270
|
#
|
@@ -2003,11 +2290,10 @@ module Aws::GlueDataBrew
|
|
2003
2290
|
include Aws::Structure
|
2004
2291
|
end
|
2005
2292
|
|
2006
|
-
#
|
2007
|
-
# rows on which the
|
2008
|
-
#
|
2009
|
-
#
|
2010
|
-
# size parameter.
|
2293
|
+
# A sample configuration for profile jobs only, which determines the
|
2294
|
+
# number of rows on which the profile job is run. If a `JobSample` value
|
2295
|
+
# isn't provided, the default is used. The default value is
|
2296
|
+
# CUSTOM\_ROWS for the mode parameter and 20,000 for the size parameter.
|
2011
2297
|
#
|
2012
2298
|
# @note When making an API call, you may pass JobSample
|
2013
2299
|
# data as a hash:
|
@@ -2018,20 +2304,20 @@ module Aws::GlueDataBrew
|
|
2018
2304
|
# }
|
2019
2305
|
#
|
2020
2306
|
# @!attribute [rw] mode
|
2021
|
-
#
|
2022
|
-
# dataset or
|
2307
|
+
# A value that determines whether the profile job is run on the entire
|
2308
|
+
# dataset or a specified number of rows. This value must be one of the
|
2023
2309
|
# following:
|
2024
2310
|
#
|
2025
|
-
# * FULL\_DATASET
|
2311
|
+
# * FULL\_DATASET - The profile job is run on the entire dataset.
|
2026
2312
|
#
|
2027
|
-
# * CUSTOM\_ROWS
|
2028
|
-
# specified in the Size parameter.
|
2313
|
+
# * CUSTOM\_ROWS - The profile job is run on the number of rows
|
2314
|
+
# specified in the `Size` parameter.
|
2029
2315
|
# @return [String]
|
2030
2316
|
#
|
2031
2317
|
# @!attribute [rw] size
|
2032
|
-
# Size parameter is only required when the mode is CUSTOM\_ROWS.
|
2033
|
-
#
|
2034
|
-
#
|
2318
|
+
# The `Size` parameter is only required when the mode is CUSTOM\_ROWS.
|
2319
|
+
# The profile job is run on the specified number of rows. The maximum
|
2320
|
+
# value for size is Long.MAX\_VALUE.
|
2035
2321
|
#
|
2036
2322
|
# Long.MAX\_VALUE = 9223372036854775807
|
2037
2323
|
# @return [Integer]
|
@@ -2453,8 +2739,8 @@ module Aws::GlueDataBrew
|
|
2453
2739
|
include Aws::Structure
|
2454
2740
|
end
|
2455
2741
|
|
2456
|
-
#
|
2457
|
-
# generated by recipe jobs or profile jobs.
|
2742
|
+
# Represents options that specify how and where DataBrew writes the
|
2743
|
+
# output generated by recipe jobs or profile jobs.
|
2458
2744
|
#
|
2459
2745
|
# @note When making an API call, you may pass Output
|
2460
2746
|
# data as a hash:
|
@@ -2499,7 +2785,8 @@ module Aws::GlueDataBrew
|
|
2499
2785
|
# @return [Boolean]
|
2500
2786
|
#
|
2501
2787
|
# @!attribute [rw] format_options
|
2502
|
-
#
|
2788
|
+
# Represents options that define how DataBrew formats job output
|
2789
|
+
# files.
|
2503
2790
|
# @return [Types::OutputFormatOptions]
|
2504
2791
|
#
|
2505
2792
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/Output AWS API Documentation
|
@@ -2515,7 +2802,8 @@ module Aws::GlueDataBrew
|
|
2515
2802
|
include Aws::Structure
|
2516
2803
|
end
|
2517
2804
|
|
2518
|
-
#
|
2805
|
+
# Represents a set of options that define the structure of
|
2806
|
+
# comma-separated (CSV) job output.
|
2519
2807
|
#
|
2520
2808
|
# @note When making an API call, you may pass OutputFormatOptions
|
2521
2809
|
# data as a hash:
|
@@ -2527,7 +2815,8 @@ module Aws::GlueDataBrew
|
|
2527
2815
|
# }
|
2528
2816
|
#
|
2529
2817
|
# @!attribute [rw] csv
|
2530
|
-
#
|
2818
|
+
# Represents a set of options that define the structure of
|
2819
|
+
# comma-separated value (CSV) job output.
|
2531
2820
|
# @return [Types::CsvOutputOptions]
|
2532
2821
|
#
|
2533
2822
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/OutputFormatOptions AWS API Documentation
|
@@ -2538,6 +2827,69 @@ module Aws::GlueDataBrew
|
|
2538
2827
|
include Aws::Structure
|
2539
2828
|
end
|
2540
2829
|
|
2830
|
+
# Represents a set of options that define how DataBrew selects files for
|
2831
|
+
# a given S3 path in a dataset.
|
2832
|
+
#
|
2833
|
+
# @note When making an API call, you may pass PathOptions
|
2834
|
+
# data as a hash:
|
2835
|
+
#
|
2836
|
+
# {
|
2837
|
+
# last_modified_date_condition: {
|
2838
|
+
# expression: "Expression", # required
|
2839
|
+
# values_map: { # required
|
2840
|
+
# "ValueReference" => "ConditionValue",
|
2841
|
+
# },
|
2842
|
+
# },
|
2843
|
+
# files_limit: {
|
2844
|
+
# max_files: 1, # required
|
2845
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
2846
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
2847
|
+
# },
|
2848
|
+
# parameters: {
|
2849
|
+
# "PathParameterName" => {
|
2850
|
+
# name: "PathParameterName", # required
|
2851
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
2852
|
+
# datetime_options: {
|
2853
|
+
# format: "DatetimeFormat", # required
|
2854
|
+
# timezone_offset: "TimezoneOffset",
|
2855
|
+
# locale_code: "LocaleCode",
|
2856
|
+
# },
|
2857
|
+
# create_column: false,
|
2858
|
+
# filter: {
|
2859
|
+
# expression: "Expression", # required
|
2860
|
+
# values_map: { # required
|
2861
|
+
# "ValueReference" => "ConditionValue",
|
2862
|
+
# },
|
2863
|
+
# },
|
2864
|
+
# },
|
2865
|
+
# },
|
2866
|
+
# }
|
2867
|
+
#
|
2868
|
+
# @!attribute [rw] last_modified_date_condition
|
2869
|
+
# If provided, this structure defines a date range for matching S3
|
2870
|
+
# objects based on their LastModifiedDate attribute in S3.
|
2871
|
+
# @return [Types::FilterExpression]
|
2872
|
+
#
|
2873
|
+
# @!attribute [rw] files_limit
|
2874
|
+
# If provided, this structure imposes a limit on a number of files
|
2875
|
+
# that should be selected.
|
2876
|
+
# @return [Types::FilesLimit]
|
2877
|
+
#
|
2878
|
+
# @!attribute [rw] parameters
|
2879
|
+
# A structure that maps names of parameters used in the S3 path of a
|
2880
|
+
# dataset to their definitions.
|
2881
|
+
# @return [Hash<String,Types::DatasetParameter>]
|
2882
|
+
#
|
2883
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/PathOptions AWS API Documentation
|
2884
|
+
#
|
2885
|
+
class PathOptions < Struct.new(
|
2886
|
+
:last_modified_date_condition,
|
2887
|
+
:files_limit,
|
2888
|
+
:parameters)
|
2889
|
+
SENSITIVE = []
|
2890
|
+
include Aws::Structure
|
2891
|
+
end
|
2892
|
+
|
2541
2893
|
# Represents all of the attributes of a DataBrew project.
|
2542
2894
|
#
|
2543
2895
|
# @!attribute [rw] account_id
|
@@ -2580,8 +2932,8 @@ module Aws::GlueDataBrew
|
|
2580
2932
|
#
|
2581
2933
|
# @!attribute [rw] sample
|
2582
2934
|
# The sample size and sampling type to apply to the data. If this
|
2583
|
-
# parameter isn't specified, then the sample
|
2584
|
-
#
|
2935
|
+
# parameter isn't specified, then the sample consists of the first
|
2936
|
+
# 500 rows from the dataset.
|
2585
2937
|
# @return [Types::Sample]
|
2586
2938
|
#
|
2587
2939
|
# @!attribute [rw] tags
|
@@ -2719,7 +3071,7 @@ module Aws::GlueDataBrew
|
|
2719
3071
|
# * Numeric version (`X.Y`) - `X` and `Y` stand for major and minor
|
2720
3072
|
# version numbers. The maximum length of each is 6 digits, and
|
2721
3073
|
# neither can be negative values. Both `X` and `Y` are required, and
|
2722
|
-
# "0.0"
|
3074
|
+
# "0.0" isn't a valid version.
|
2723
3075
|
#
|
2724
3076
|
# * `LATEST_WORKING` - the most recent valid version being developed
|
2725
3077
|
# in a DataBrew project.
|
@@ -2837,8 +3189,8 @@ module Aws::GlueDataBrew
|
|
2837
3189
|
# @return [Types::RecipeAction]
|
2838
3190
|
#
|
2839
3191
|
# @!attribute [rw] condition_expressions
|
2840
|
-
# One or more conditions that must be met
|
2841
|
-
#
|
3192
|
+
# One or more conditions that must be met for the recipe step to
|
3193
|
+
# succeed.
|
2842
3194
|
#
|
2843
3195
|
# <note markdown="1"> All of the conditions in the array must be met. In other words, all
|
2844
3196
|
# of the conditions must be combined using a logical AND operation.
|
@@ -2893,8 +3245,8 @@ module Aws::GlueDataBrew
|
|
2893
3245
|
include Aws::Structure
|
2894
3246
|
end
|
2895
3247
|
|
2896
|
-
#
|
2897
|
-
# read input data, or write output from a job.
|
3248
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
3249
|
+
# DataBrew can read input data, or write output from a job.
|
2898
3250
|
#
|
2899
3251
|
# @note When making an API call, you may pass S3Location
|
2900
3252
|
# data as a hash:
|
@@ -2981,7 +3333,7 @@ module Aws::GlueDataBrew
|
|
2981
3333
|
# @return [String]
|
2982
3334
|
#
|
2983
3335
|
# @!attribute [rw] cron_expression
|
2984
|
-
# The
|
3336
|
+
# The dates and times when the job is to run. For more information,
|
2985
3337
|
# see [Cron expressions][1] in the *AWS Glue DataBrew Developer
|
2986
3338
|
# Guide*.
|
2987
3339
|
#
|
@@ -3071,7 +3423,7 @@ module Aws::GlueDataBrew
|
|
3071
3423
|
# @return [String]
|
3072
3424
|
#
|
3073
3425
|
# @!attribute [rw] view_frame
|
3074
|
-
# Represents the data being
|
3426
|
+
# Represents the data being transformed during an action.
|
3075
3427
|
# @return [Types::ViewFrame]
|
3076
3428
|
#
|
3077
3429
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/SendProjectSessionActionRequest AWS API Documentation
|
@@ -3330,6 +3682,45 @@ module Aws::GlueDataBrew
|
|
3330
3682
|
# key: "Key",
|
3331
3683
|
# },
|
3332
3684
|
# },
|
3685
|
+
# database_input_definition: {
|
3686
|
+
# glue_connection_name: "GlueConnectionName", # required
|
3687
|
+
# database_table_name: "DatabaseTableName", # required
|
3688
|
+
# temp_directory: {
|
3689
|
+
# bucket: "Bucket", # required
|
3690
|
+
# key: "Key",
|
3691
|
+
# },
|
3692
|
+
# },
|
3693
|
+
# },
|
3694
|
+
# path_options: {
|
3695
|
+
# last_modified_date_condition: {
|
3696
|
+
# expression: "Expression", # required
|
3697
|
+
# values_map: { # required
|
3698
|
+
# "ValueReference" => "ConditionValue",
|
3699
|
+
# },
|
3700
|
+
# },
|
3701
|
+
# files_limit: {
|
3702
|
+
# max_files: 1, # required
|
3703
|
+
# ordered_by: "LAST_MODIFIED_DATE", # accepts LAST_MODIFIED_DATE
|
3704
|
+
# order: "DESCENDING", # accepts DESCENDING, ASCENDING
|
3705
|
+
# },
|
3706
|
+
# parameters: {
|
3707
|
+
# "PathParameterName" => {
|
3708
|
+
# name: "PathParameterName", # required
|
3709
|
+
# type: "Datetime", # required, accepts Datetime, Number, String
|
3710
|
+
# datetime_options: {
|
3711
|
+
# format: "DatetimeFormat", # required
|
3712
|
+
# timezone_offset: "TimezoneOffset",
|
3713
|
+
# locale_code: "LocaleCode",
|
3714
|
+
# },
|
3715
|
+
# create_column: false,
|
3716
|
+
# filter: {
|
3717
|
+
# expression: "Expression", # required
|
3718
|
+
# values_map: { # required
|
3719
|
+
# "ValueReference" => "ConditionValue",
|
3720
|
+
# },
|
3721
|
+
# },
|
3722
|
+
# },
|
3723
|
+
# },
|
3333
3724
|
# },
|
3334
3725
|
# }
|
3335
3726
|
#
|
@@ -3338,27 +3729,33 @@ module Aws::GlueDataBrew
|
|
3338
3729
|
# @return [String]
|
3339
3730
|
#
|
3340
3731
|
# @!attribute [rw] format
|
3341
|
-
#
|
3732
|
+
# The file format of a dataset that is created from an S3 file or
|
3342
3733
|
# folder.
|
3343
3734
|
# @return [String]
|
3344
3735
|
#
|
3345
3736
|
# @!attribute [rw] format_options
|
3346
|
-
#
|
3347
|
-
# input.
|
3737
|
+
# Represents a set of options that define the structure of either
|
3738
|
+
# comma-separated value (CSV), Excel, or JSON input.
|
3348
3739
|
# @return [Types::FormatOptions]
|
3349
3740
|
#
|
3350
3741
|
# @!attribute [rw] input
|
3351
|
-
#
|
3352
|
-
# Data Catalog or Amazon S3.
|
3742
|
+
# Represents information on how DataBrew can find data, in either the
|
3743
|
+
# AWS Glue Data Catalog or Amazon S3.
|
3353
3744
|
# @return [Types::Input]
|
3354
3745
|
#
|
3746
|
+
# @!attribute [rw] path_options
|
3747
|
+
# A set of options that defines how DataBrew interprets an S3 path of
|
3748
|
+
# the dataset.
|
3749
|
+
# @return [Types::PathOptions]
|
3750
|
+
#
|
3355
3751
|
# @see http://docs.aws.amazon.com/goto/WebAPI/databrew-2017-07-25/UpdateDatasetRequest AWS API Documentation
|
3356
3752
|
#
|
3357
3753
|
class UpdateDatasetRequest < Struct.new(
|
3358
3754
|
:name,
|
3359
3755
|
:format,
|
3360
3756
|
:format_options,
|
3361
|
-
:input
|
3757
|
+
:input,
|
3758
|
+
:path_options)
|
3362
3759
|
SENSITIVE = []
|
3363
3760
|
include Aws::Structure
|
3364
3761
|
end
|
@@ -3430,8 +3827,8 @@ module Aws::GlueDataBrew
|
|
3430
3827
|
# @return [Integer]
|
3431
3828
|
#
|
3432
3829
|
# @!attribute [rw] output_location
|
3433
|
-
#
|
3434
|
-
# read input data, or write output from a job.
|
3830
|
+
# Represents an Amazon S3 location (bucket name and object key) where
|
3831
|
+
# DataBrew can read input data, or write output from a job.
|
3435
3832
|
# @return [Types::S3Location]
|
3436
3833
|
#
|
3437
3834
|
# @!attribute [rw] role_arn
|
@@ -3763,7 +4160,7 @@ module Aws::GlueDataBrew
|
|
3763
4160
|
include Aws::Structure
|
3764
4161
|
end
|
3765
4162
|
|
3766
|
-
# Represents the data being
|
4163
|
+
# Represents the data being transformed during an action.
|
3767
4164
|
#
|
3768
4165
|
# @note When making an API call, you may pass ViewFrame
|
3769
4166
|
# data as a hash:
|