aws-sdk-glue 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/aws-sdk-glue.rb +1 -1
- data/lib/aws-sdk-glue/client.rb +98 -46
- data/lib/aws-sdk-glue/client_api.rb +34 -3
- data/lib/aws-sdk-glue/types.rb +288 -88
- metadata +2 -2
data/lib/aws-sdk-glue/types.rb
CHANGED
@@ -466,21 +466,29 @@ module Aws::Glue
|
|
466
466
|
include Aws::Structure
|
467
467
|
end
|
468
468
|
|
469
|
-
# Classifiers are written in Python and triggered during a
|
470
|
-
# You can write your own
|
469
|
+
# Classifiers are written in Python and triggered during a crawl task.
|
470
|
+
# You can write your own classifiers to best categorize your data
|
471
471
|
# sources and specify the appropriate schemas to use for them. A
|
472
|
-
#
|
473
|
-
#
|
472
|
+
# classifier checks whether a given file is in a format it can handle,
|
473
|
+
# and if it is, the classifier creates a schema in the form of a
|
474
474
|
# `StructType` object that matches that data format.
|
475
475
|
#
|
476
|
+
# A classifier can be either a `grok` classifier or an XML classifier,
|
477
|
+
# specified in one or the other field of the `Classifier` object.
|
478
|
+
#
|
476
479
|
# @!attribute [rw] grok_classifier
|
477
|
-
# A GrokClassifier object.
|
480
|
+
# A `GrokClassifier` object.
|
478
481
|
# @return [Types::GrokClassifier]
|
479
482
|
#
|
483
|
+
# @!attribute [rw] xml_classifier
|
484
|
+
# An `XMLClassifier` object.
|
485
|
+
# @return [Types::XMLClassifier]
|
486
|
+
#
|
480
487
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Classifier AWS API Documentation
|
481
488
|
#
|
482
489
|
class Classifier < Struct.new(
|
483
|
-
:grok_classifier
|
490
|
+
:grok_classifier,
|
491
|
+
:xml_classifier)
|
484
492
|
include Aws::Structure
|
485
493
|
end
|
486
494
|
|
@@ -792,15 +800,16 @@ module Aws::Glue
|
|
792
800
|
|
793
801
|
# Specifies a crawler program that examines a data source and uses
|
794
802
|
# classifiers to try to determine its schema. If successful, the crawler
|
795
|
-
# records
|
803
|
+
# records metadata concerning the data source in the AWS Glue Data
|
804
|
+
# Catalog.
|
796
805
|
#
|
797
806
|
# @!attribute [rw] name
|
798
|
-
# The
|
807
|
+
# The crawler name.
|
799
808
|
# @return [String]
|
800
809
|
#
|
801
810
|
# @!attribute [rw] role
|
802
811
|
# The IAM role (or ARN of an IAM role) used to access customer
|
803
|
-
# resources such as data in S3.
|
812
|
+
# resources, such as data in Amazon S3.
|
804
813
|
# @return [String]
|
805
814
|
#
|
806
815
|
# @!attribute [rw] targets
|
@@ -808,46 +817,46 @@ module Aws::Glue
|
|
808
817
|
# @return [Types::CrawlerTargets]
|
809
818
|
#
|
810
819
|
# @!attribute [rw] database_name
|
811
|
-
# The
|
820
|
+
# The database where metadata is written by this crawler.
|
812
821
|
# @return [String]
|
813
822
|
#
|
814
823
|
# @!attribute [rw] description
|
815
|
-
# A description of
|
824
|
+
# A description of the crawler.
|
816
825
|
# @return [String]
|
817
826
|
#
|
818
827
|
# @!attribute [rw] classifiers
|
819
|
-
# A list of custom
|
828
|
+
# A list of custom classifiers associated with the crawler.
|
820
829
|
# @return [Array<String>]
|
821
830
|
#
|
822
831
|
# @!attribute [rw] schema_change_policy
|
823
|
-
# Sets
|
832
|
+
# Sets the behavior when the crawler finds a changed or deleted
|
833
|
+
# object.
|
824
834
|
# @return [Types::SchemaChangePolicy]
|
825
835
|
#
|
826
836
|
# @!attribute [rw] state
|
827
|
-
# Indicates whether
|
837
|
+
# Indicates whether the crawler is running, or whether a run is
|
828
838
|
# pending.
|
829
839
|
# @return [String]
|
830
840
|
#
|
831
841
|
# @!attribute [rw] table_prefix
|
832
|
-
# The
|
842
|
+
# The prefix added to the names of tables that are created.
|
833
843
|
# @return [String]
|
834
844
|
#
|
835
845
|
# @!attribute [rw] schedule
|
836
|
-
#
|
837
|
-
# Crawler is to be run.
|
846
|
+
# For scheduled crawlers, the schedule when the crawler runs.
|
838
847
|
# @return [Types::Schedule]
|
839
848
|
#
|
840
849
|
# @!attribute [rw] crawl_elapsed_time
|
841
|
-
# If
|
842
|
-
#
|
850
|
+
# If the crawler is running, contains the total time elapsed since the
|
851
|
+
# last crawl began.
|
843
852
|
# @return [Integer]
|
844
853
|
#
|
845
854
|
# @!attribute [rw] creation_time
|
846
|
-
# The time when the
|
855
|
+
# The time when the crawler was created.
|
847
856
|
# @return [Time]
|
848
857
|
#
|
849
858
|
# @!attribute [rw] last_updated
|
850
|
-
# The time the
|
859
|
+
# The time the crawler was last updated.
|
851
860
|
# @return [Time]
|
852
861
|
#
|
853
862
|
# @!attribute [rw] last_crawl
|
@@ -856,9 +865,20 @@ module Aws::Glue
|
|
856
865
|
# @return [Types::LastCrawlInfo]
|
857
866
|
#
|
858
867
|
# @!attribute [rw] version
|
859
|
-
# The version of the
|
868
|
+
# The version of the crawler.
|
860
869
|
# @return [Integer]
|
861
870
|
#
|
871
|
+
# @!attribute [rw] configuration
|
872
|
+
# Crawler configuration information. This versioned JSON string allows
|
873
|
+
# users to specify aspects of a Crawler's behavior.
|
874
|
+
#
|
875
|
+
# You can use this field to force partitions to inherit metadata such
|
876
|
+
# as classification, input format, output format, serde information,
|
877
|
+
# and schema from their parent table, rather than detect this
|
878
|
+
# information separately for each partition. Use the following JSON
|
879
|
+
# string to specify that behavior:
|
880
|
+
# @return [String]
|
881
|
+
#
|
862
882
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Crawler AWS API Documentation
|
863
883
|
#
|
864
884
|
class Crawler < Struct.new(
|
@@ -876,7 +896,8 @@ module Aws::Glue
|
|
876
896
|
:creation_time,
|
877
897
|
:last_updated,
|
878
898
|
:last_crawl,
|
879
|
-
:version
|
899
|
+
:version,
|
900
|
+
:configuration)
|
880
901
|
include Aws::Structure
|
881
902
|
end
|
882
903
|
|
@@ -891,7 +912,8 @@ module Aws::Glue
|
|
891
912
|
# @return [Float]
|
892
913
|
#
|
893
914
|
# @!attribute [rw] still_estimating
|
894
|
-
# True if the crawler is estimating
|
915
|
+
# True if the crawler is still estimating how long it will take to
|
916
|
+
# complete this run.
|
895
917
|
# @return [Boolean]
|
896
918
|
#
|
897
919
|
# @!attribute [rw] last_runtime_seconds
|
@@ -903,15 +925,15 @@ module Aws::Glue
|
|
903
925
|
# @return [Float]
|
904
926
|
#
|
905
927
|
# @!attribute [rw] tables_created
|
906
|
-
#
|
928
|
+
# The number of tables created by this crawler.
|
907
929
|
# @return [Integer]
|
908
930
|
#
|
909
931
|
# @!attribute [rw] tables_updated
|
910
|
-
#
|
932
|
+
# The number of tables updated by this crawler.
|
911
933
|
# @return [Integer]
|
912
934
|
#
|
913
935
|
# @!attribute [rw] tables_deleted
|
914
|
-
#
|
936
|
+
# The number of tables deleted by this crawler.
|
915
937
|
# @return [Integer]
|
916
938
|
#
|
917
939
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CrawlerMetrics AWS API Documentation
|
@@ -928,7 +950,7 @@ module Aws::Glue
|
|
928
950
|
include Aws::Structure
|
929
951
|
end
|
930
952
|
|
931
|
-
# Specifies
|
953
|
+
# Specifies data stores to crawl.
|
932
954
|
#
|
933
955
|
# @note When making an API call, you may pass CrawlerTargets
|
934
956
|
# data as a hash:
|
@@ -950,7 +972,7 @@ module Aws::Glue
|
|
950
972
|
# }
|
951
973
|
#
|
952
974
|
# @!attribute [rw] s3_targets
|
953
|
-
# Specifies
|
975
|
+
# Specifies Amazon S3 targets.
|
954
976
|
# @return [Array<Types::S3Target>]
|
955
977
|
#
|
956
978
|
# @!attribute [rw] jdbc_targets
|
@@ -975,16 +997,26 @@ module Aws::Glue
|
|
975
997
|
# grok_pattern: "GrokPattern", # required
|
976
998
|
# custom_patterns: "CustomPatterns",
|
977
999
|
# },
|
1000
|
+
# xml_classifier: {
|
1001
|
+
# classification: "Classification", # required
|
1002
|
+
# name: "NameString", # required
|
1003
|
+
# row_tag: "RowTag",
|
1004
|
+
# },
|
978
1005
|
# }
|
979
1006
|
#
|
980
1007
|
# @!attribute [rw] grok_classifier
|
981
|
-
# A
|
1008
|
+
# A `GrokClassifier` object specifying the classifier to create.
|
982
1009
|
# @return [Types::CreateGrokClassifierRequest]
|
983
1010
|
#
|
1011
|
+
# @!attribute [rw] xml_classifier
|
1012
|
+
# An `XMLClassifier` object specifying the classifier to create.
|
1013
|
+
# @return [Types::CreateXMLClassifierRequest]
|
1014
|
+
#
|
984
1015
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateClassifierRequest AWS API Documentation
|
985
1016
|
#
|
986
1017
|
class CreateClassifierRequest < Struct.new(
|
987
|
-
:grok_classifier
|
1018
|
+
:grok_classifier,
|
1019
|
+
:xml_classifier)
|
988
1020
|
include Aws::Structure
|
989
1021
|
end
|
990
1022
|
|
@@ -1039,7 +1071,7 @@ module Aws::Glue
|
|
1039
1071
|
#
|
1040
1072
|
# {
|
1041
1073
|
# name: "NameString", # required
|
1042
|
-
# role: "
|
1074
|
+
# role: "Role", # required
|
1043
1075
|
# database_name: "DatabaseName", # required
|
1044
1076
|
# description: "DescriptionString",
|
1045
1077
|
# targets: { # required
|
@@ -1064,24 +1096,25 @@ module Aws::Glue
|
|
1064
1096
|
# update_behavior: "LOG", # accepts LOG, UPDATE_IN_DATABASE
|
1065
1097
|
# delete_behavior: "LOG", # accepts LOG, DELETE_FROM_DATABASE, DEPRECATE_IN_DATABASE
|
1066
1098
|
# },
|
1099
|
+
# configuration: "CrawlerConfiguration",
|
1067
1100
|
# }
|
1068
1101
|
#
|
1069
1102
|
# @!attribute [rw] name
|
1070
|
-
# Name of the new
|
1103
|
+
# Name of the new crawler.
|
1071
1104
|
# @return [String]
|
1072
1105
|
#
|
1073
1106
|
# @!attribute [rw] role
|
1074
|
-
# The IAM role (or ARN of an IAM role) used by the new
|
1107
|
+
# The IAM role (or ARN of an IAM role) used by the new crawler to
|
1075
1108
|
# access customer resources.
|
1076
1109
|
# @return [String]
|
1077
1110
|
#
|
1078
1111
|
# @!attribute [rw] database_name
|
1079
|
-
# The Glue
|
1112
|
+
# The AWS Glue database where results are written, such as:
|
1080
1113
|
# `arn:aws:daylight:us-east-1::database/sometable/*`.
|
1081
1114
|
# @return [String]
|
1082
1115
|
#
|
1083
1116
|
# @!attribute [rw] description
|
1084
|
-
# A description of the new
|
1117
|
+
# A description of the new crawler.
|
1085
1118
|
# @return [String]
|
1086
1119
|
#
|
1087
1120
|
# @!attribute [rw] targets
|
@@ -1099,20 +1132,30 @@ module Aws::Glue
|
|
1099
1132
|
# @return [String]
|
1100
1133
|
#
|
1101
1134
|
# @!attribute [rw] classifiers
|
1102
|
-
# A list of custom
|
1135
|
+
# A list of custom classifiers that the user has registered. By
|
1103
1136
|
# default, all AWS classifiers are included in a crawl, but these
|
1104
1137
|
# custom classifiers always override the default classifiers for a
|
1105
1138
|
# given classification.
|
1106
1139
|
# @return [Array<String>]
|
1107
1140
|
#
|
1108
1141
|
# @!attribute [rw] table_prefix
|
1109
|
-
# The table prefix used for catalog tables created.
|
1142
|
+
# The table prefix used for catalog tables that are created.
|
1110
1143
|
# @return [String]
|
1111
1144
|
#
|
1112
1145
|
# @!attribute [rw] schema_change_policy
|
1113
1146
|
# Policy for the crawler's update and deletion behavior.
|
1114
1147
|
# @return [Types::SchemaChangePolicy]
|
1115
1148
|
#
|
1149
|
+
# @!attribute [rw] configuration
|
1150
|
+
# Crawler configuration information. This versioned JSON string allows
|
1151
|
+
# users to specify aspects of a Crawler's behavior.
|
1152
|
+
#
|
1153
|
+
# You can use this field to force partitions to inherit metadata such
|
1154
|
+
# as classification, input format, output format, serde information,
|
1155
|
+
# and schema from their parent table, rather than detect this
|
1156
|
+
# information separately for each partition.
|
1157
|
+
# @return [String]
|
1158
|
+
#
|
1116
1159
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateCrawlerRequest AWS API Documentation
|
1117
1160
|
#
|
1118
1161
|
class CreateCrawlerRequest < Struct.new(
|
@@ -1124,7 +1167,8 @@ module Aws::Glue
|
|
1124
1167
|
:schedule,
|
1125
1168
|
:classifiers,
|
1126
1169
|
:table_prefix,
|
1127
|
-
:schema_change_policy
|
1170
|
+
:schema_change_policy,
|
1171
|
+
:configuration)
|
1128
1172
|
include Aws::Structure
|
1129
1173
|
end
|
1130
1174
|
|
@@ -1321,7 +1365,7 @@ module Aws::Glue
|
|
1321
1365
|
include Aws::Structure
|
1322
1366
|
end
|
1323
1367
|
|
1324
|
-
# Specifies a
|
1368
|
+
# Specifies a `grok` classifier for `CreateClassifier` to create.
|
1325
1369
|
#
|
1326
1370
|
# @note When making an API call, you may pass CreateGrokClassifierRequest
|
1327
1371
|
# data as a hash:
|
@@ -1334,12 +1378,12 @@ module Aws::Glue
|
|
1334
1378
|
# }
|
1335
1379
|
#
|
1336
1380
|
# @!attribute [rw] classification
|
1337
|
-
#
|
1338
|
-
#
|
1381
|
+
# An identifier of the data format that the classifier matches, such
|
1382
|
+
# as Twitter, JSON, Omniture logs, Amazon CloudWatch Logs, and so on.
|
1339
1383
|
# @return [String]
|
1340
1384
|
#
|
1341
1385
|
# @!attribute [rw] name
|
1342
|
-
# The name of the new
|
1386
|
+
# The name of the new classifier.
|
1343
1387
|
# @return [String]
|
1344
1388
|
#
|
1345
1389
|
# @!attribute [rw] grok_pattern
|
@@ -1347,7 +1391,7 @@ module Aws::Glue
|
|
1347
1391
|
# @return [String]
|
1348
1392
|
#
|
1349
1393
|
# @!attribute [rw] custom_patterns
|
1350
|
-
#
|
1394
|
+
# Optional custom grok patterns used by this classifier.
|
1351
1395
|
# @return [String]
|
1352
1396
|
#
|
1353
1397
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateGrokClassifierRequest AWS API Documentation
|
@@ -1819,6 +1863,41 @@ module Aws::Glue
|
|
1819
1863
|
#
|
1820
1864
|
class CreateUserDefinedFunctionResponse < Aws::EmptyStructure; end
|
1821
1865
|
|
1866
|
+
# Specifies an XML classifier for `CreateClassifier` to create.
|
1867
|
+
#
|
1868
|
+
# @note When making an API call, you may pass CreateXMLClassifierRequest
|
1869
|
+
# data as a hash:
|
1870
|
+
#
|
1871
|
+
# {
|
1872
|
+
# classification: "Classification", # required
|
1873
|
+
# name: "NameString", # required
|
1874
|
+
# row_tag: "RowTag",
|
1875
|
+
# }
|
1876
|
+
#
|
1877
|
+
# @!attribute [rw] classification
|
1878
|
+
# An identifier of the data format that the classifier matches.
|
1879
|
+
# @return [String]
|
1880
|
+
#
|
1881
|
+
# @!attribute [rw] name
|
1882
|
+
# The name of the classifier.
|
1883
|
+
# @return [String]
|
1884
|
+
#
|
1885
|
+
# @!attribute [rw] row_tag
|
1886
|
+
# The XML tag designating the element that contains each record in an
|
1887
|
+
# XML document being parsed. Note that this cannot be an empty
|
1888
|
+
# element. It must contain child elements representing fields in the
|
1889
|
+
# record.
|
1890
|
+
# @return [String]
|
1891
|
+
#
|
1892
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateXMLClassifierRequest AWS API Documentation
|
1893
|
+
#
|
1894
|
+
class CreateXMLClassifierRequest < Struct.new(
|
1895
|
+
:classification,
|
1896
|
+
:name,
|
1897
|
+
:row_tag)
|
1898
|
+
include Aws::Structure
|
1899
|
+
end
|
1900
|
+
|
1822
1901
|
# The `Database` object represents a logical grouping of tables that may
|
1823
1902
|
# reside in a Hive metastore or an RDBMS.
|
1824
1903
|
#
|
@@ -1903,7 +1982,7 @@ module Aws::Glue
|
|
1903
1982
|
# }
|
1904
1983
|
#
|
1905
1984
|
# @!attribute [rw] name
|
1906
|
-
# Name of the
|
1985
|
+
# Name of the classifier to remove.
|
1907
1986
|
# @return [String]
|
1908
1987
|
#
|
1909
1988
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/DeleteClassifierRequest AWS API Documentation
|
@@ -1954,7 +2033,7 @@ module Aws::Glue
|
|
1954
2033
|
# }
|
1955
2034
|
#
|
1956
2035
|
# @!attribute [rw] name
|
1957
|
-
# Name of the
|
2036
|
+
# Name of the crawler to remove.
|
1958
2037
|
# @return [String]
|
1959
2038
|
#
|
1960
2039
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/DeleteCrawlerRequest AWS API Documentation
|
@@ -2419,7 +2498,7 @@ module Aws::Glue
|
|
2419
2498
|
# }
|
2420
2499
|
#
|
2421
2500
|
# @!attribute [rw] name
|
2422
|
-
# Name of the
|
2501
|
+
# Name of the classifier to retrieve.
|
2423
2502
|
# @return [String]
|
2424
2503
|
#
|
2425
2504
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetClassifierRequest AWS API Documentation
|
@@ -2430,7 +2509,7 @@ module Aws::Glue
|
|
2430
2509
|
end
|
2431
2510
|
|
2432
2511
|
# @!attribute [rw] classifier
|
2433
|
-
# The requested
|
2512
|
+
# The requested classifier.
|
2434
2513
|
# @return [Types::Classifier]
|
2435
2514
|
#
|
2436
2515
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetClassifierResponse AWS API Documentation
|
@@ -2465,7 +2544,7 @@ module Aws::Glue
|
|
2465
2544
|
end
|
2466
2545
|
|
2467
2546
|
# @!attribute [rw] classifiers
|
2468
|
-
# The requested list of
|
2547
|
+
# The requested list of classifier objects.
|
2469
2548
|
# @return [Array<Types::Classifier>]
|
2470
2549
|
#
|
2471
2550
|
# @!attribute [rw] next_token
|
@@ -2657,7 +2736,7 @@ module Aws::Glue
|
|
2657
2736
|
# }
|
2658
2737
|
#
|
2659
2738
|
# @!attribute [rw] name
|
2660
|
-
# Name of the
|
2739
|
+
# Name of the crawler to retrieve metadata for.
|
2661
2740
|
# @return [String]
|
2662
2741
|
#
|
2663
2742
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetCrawlerRequest AWS API Documentation
|
@@ -2668,7 +2747,7 @@ module Aws::Glue
|
|
2668
2747
|
end
|
2669
2748
|
|
2670
2749
|
# @!attribute [rw] crawler
|
2671
|
-
# The metadata for the specified
|
2750
|
+
# The metadata for the specified crawler.
|
2672
2751
|
# @return [Types::Crawler]
|
2673
2752
|
#
|
2674
2753
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetCrawlerResponse AWS API Documentation
|
@@ -2687,7 +2766,7 @@ module Aws::Glue
|
|
2687
2766
|
# }
|
2688
2767
|
#
|
2689
2768
|
# @!attribute [rw] max_results
|
2690
|
-
# The number of
|
2769
|
+
# The number of crawlers to return on each call.
|
2691
2770
|
# @return [Integer]
|
2692
2771
|
#
|
2693
2772
|
# @!attribute [rw] next_token
|
@@ -2703,7 +2782,7 @@ module Aws::Glue
|
|
2703
2782
|
end
|
2704
2783
|
|
2705
2784
|
# @!attribute [rw] crawlers
|
2706
|
-
# A list of
|
2785
|
+
# A list of crawler metadata.
|
2707
2786
|
# @return [Array<Types::Crawler>]
|
2708
2787
|
#
|
2709
2788
|
# @!attribute [rw] next_token
|
@@ -3321,7 +3400,7 @@ module Aws::Glue
|
|
3321
3400
|
end
|
3322
3401
|
|
3323
3402
|
# @!attribute [rw] python_script
|
3324
|
-
# A
|
3403
|
+
# A Python script to perform the mapping.
|
3325
3404
|
# @return [String]
|
3326
3405
|
#
|
3327
3406
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetPlanResponse AWS API Documentation
|
@@ -3674,15 +3753,15 @@ module Aws::Glue
|
|
3674
3753
|
include Aws::Structure
|
3675
3754
|
end
|
3676
3755
|
|
3677
|
-
# A classifier that uses `grok
|
3756
|
+
# A classifier that uses `grok` patterns.
|
3678
3757
|
#
|
3679
3758
|
# @!attribute [rw] name
|
3680
3759
|
# The name of the classifier.
|
3681
3760
|
# @return [String]
|
3682
3761
|
#
|
3683
3762
|
# @!attribute [rw] classification
|
3684
|
-
#
|
3685
|
-
# Omniture
|
3763
|
+
# An identifier of the data format that the classifier matches, such
|
3764
|
+
# as Twitter, JSON, Omniture logs, and so on.
|
3686
3765
|
# @return [String]
|
3687
3766
|
#
|
3688
3767
|
# @!attribute [rw] creation_time
|
@@ -3698,11 +3777,22 @@ module Aws::Glue
|
|
3698
3777
|
# @return [Integer]
|
3699
3778
|
#
|
3700
3779
|
# @!attribute [rw] grok_pattern
|
3701
|
-
# The grok pattern
|
3780
|
+
# The grok pattern applied to a data store by this classifier. For
|
3781
|
+
# more information, see built-in patterns in [Writing Custom
|
3782
|
+
# Classifers][1].
|
3783
|
+
#
|
3784
|
+
#
|
3785
|
+
#
|
3786
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/custom-classifier.html
|
3702
3787
|
# @return [String]
|
3703
3788
|
#
|
3704
3789
|
# @!attribute [rw] custom_patterns
|
3705
|
-
#
|
3790
|
+
# Optional custom grok patterns defined by this classifier. For more
|
3791
|
+
# information, see custom patterns in [Writing Custom Classifers][1].
|
3792
|
+
#
|
3793
|
+
#
|
3794
|
+
#
|
3795
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/custom-classifier.html
|
3706
3796
|
# @return [String]
|
3707
3797
|
#
|
3708
3798
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GrokClassifier AWS API Documentation
|
@@ -3741,7 +3831,7 @@ module Aws::Glue
|
|
3741
3831
|
#
|
3742
3832
|
class ImportCatalogToGlueResponse < Aws::EmptyStructure; end
|
3743
3833
|
|
3744
|
-
# Specifies a JDBC
|
3834
|
+
# Specifies a JDBC data store to crawl.
|
3745
3835
|
#
|
3746
3836
|
# @note When making an API call, you may pass JdbcTarget
|
3747
3837
|
# data as a hash:
|
@@ -3753,7 +3843,7 @@ module Aws::Glue
|
|
3753
3843
|
# }
|
3754
3844
|
#
|
3755
3845
|
# @!attribute [rw] connection_name
|
3756
|
-
# The name of the connection to use
|
3846
|
+
# The name of the connection to use to connect to the JDBC target.
|
3757
3847
|
# @return [String]
|
3758
3848
|
#
|
3759
3849
|
# @!attribute [rw] path
|
@@ -3761,7 +3851,12 @@ module Aws::Glue
|
|
3761
3851
|
# @return [String]
|
3762
3852
|
#
|
3763
3853
|
# @!attribute [rw] exclusions
|
3764
|
-
# A list of
|
3854
|
+
# A list of glob patterns used to exclude from the crawl. For more
|
3855
|
+
# information, see [Catalog Tables with a Crawler][1].
|
3856
|
+
#
|
3857
|
+
#
|
3858
|
+
#
|
3859
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/add-crawler.html
|
3765
3860
|
# @return [Array<String>]
|
3766
3861
|
#
|
3767
3862
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JdbcTarget AWS API Documentation
|
@@ -4059,7 +4154,7 @@ module Aws::Glue
|
|
4059
4154
|
# @return [String]
|
4060
4155
|
#
|
4061
4156
|
# @!attribute [rw] error_message
|
4062
|
-
#
|
4157
|
+
# If an error occurred, the error information about the last crawl.
|
4063
4158
|
# @return [String]
|
4064
4159
|
#
|
4065
4160
|
# @!attribute [rw] log_group
|
@@ -4117,7 +4212,7 @@ module Aws::Glue
|
|
4117
4212
|
# @return [Array<Types::CodeGenNodeArg>]
|
4118
4213
|
#
|
4119
4214
|
# @!attribute [rw] s3
|
4120
|
-
# An
|
4215
|
+
# An Amazon S3 location.
|
4121
4216
|
# @return [Array<Types::CodeGenNodeArg>]
|
4122
4217
|
#
|
4123
4218
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Location AWS API Documentation
|
@@ -4517,7 +4612,7 @@ module Aws::Glue
|
|
4517
4612
|
include Aws::Structure
|
4518
4613
|
end
|
4519
4614
|
|
4520
|
-
# Specifies a
|
4615
|
+
# Specifies a data store in Amazon S3.
|
4521
4616
|
#
|
4522
4617
|
# @note When making an API call, you may pass S3Target
|
4523
4618
|
# data as a hash:
|
@@ -4528,11 +4623,16 @@ module Aws::Glue
|
|
4528
4623
|
# }
|
4529
4624
|
#
|
4530
4625
|
# @!attribute [rw] path
|
4531
|
-
# The path to the S3 target.
|
4626
|
+
# The path to the Amazon S3 target.
|
4532
4627
|
# @return [String]
|
4533
4628
|
#
|
4534
4629
|
# @!attribute [rw] exclusions
|
4535
|
-
# A list of
|
4630
|
+
# A list of glob patterns used to exclude from the crawl. For more
|
4631
|
+
# information, see [Catalog Tables with a Crawler][1].
|
4632
|
+
#
|
4633
|
+
#
|
4634
|
+
#
|
4635
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/add-crawler.html
|
4536
4636
|
# @return [Array<String>]
|
4537
4637
|
#
|
4538
4638
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3Target AWS API Documentation
|
@@ -4578,11 +4678,11 @@ module Aws::Glue
|
|
4578
4678
|
# }
|
4579
4679
|
#
|
4580
4680
|
# @!attribute [rw] update_behavior
|
4581
|
-
# The update behavior.
|
4681
|
+
# The update behavior when the crawler finds a changed schema.
|
4582
4682
|
# @return [String]
|
4583
4683
|
#
|
4584
4684
|
# @!attribute [rw] delete_behavior
|
4585
|
-
# The deletion behavior.
|
4685
|
+
# The deletion behavior when the crawler finds a deleted object.
|
4586
4686
|
# @return [String]
|
4587
4687
|
#
|
4588
4688
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/SchemaChangePolicy AWS API Documentation
|
@@ -4703,7 +4803,7 @@ module Aws::Glue
|
|
4703
4803
|
# }
|
4704
4804
|
#
|
4705
4805
|
# @!attribute [rw] name
|
4706
|
-
# Name of the
|
4806
|
+
# Name of the crawler to start.
|
4707
4807
|
# @return [String]
|
4708
4808
|
#
|
4709
4809
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StartCrawlerRequest AWS API Documentation
|
@@ -4825,7 +4925,7 @@ module Aws::Glue
|
|
4825
4925
|
# }
|
4826
4926
|
#
|
4827
4927
|
# @!attribute [rw] name
|
4828
|
-
# Name of the
|
4928
|
+
# Name of the crawler to stop.
|
4829
4929
|
# @return [String]
|
4830
4930
|
#
|
4831
4931
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StopCrawlerRequest AWS API Documentation
|
@@ -5403,16 +5503,26 @@ module Aws::Glue
|
|
5403
5503
|
# grok_pattern: "GrokPattern",
|
5404
5504
|
# custom_patterns: "CustomPatterns",
|
5405
5505
|
# },
|
5506
|
+
# xml_classifier: {
|
5507
|
+
# name: "NameString", # required
|
5508
|
+
# classification: "Classification",
|
5509
|
+
# row_tag: "RowTag",
|
5510
|
+
# },
|
5406
5511
|
# }
|
5407
5512
|
#
|
5408
5513
|
# @!attribute [rw] grok_classifier
|
5409
5514
|
# A `GrokClassifier` object with updated fields.
|
5410
5515
|
# @return [Types::UpdateGrokClassifierRequest]
|
5411
5516
|
#
|
5517
|
+
# @!attribute [rw] xml_classifier
|
5518
|
+
# An `XMLClassifier` object with updated fields.
|
5519
|
+
# @return [Types::UpdateXMLClassifierRequest]
|
5520
|
+
#
|
5412
5521
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateClassifierRequest AWS API Documentation
|
5413
5522
|
#
|
5414
5523
|
class UpdateClassifierRequest < Struct.new(
|
5415
|
-
:grok_classifier
|
5524
|
+
:grok_classifier,
|
5525
|
+
:xml_classifier)
|
5416
5526
|
include Aws::Structure
|
5417
5527
|
end
|
5418
5528
|
|
@@ -5474,7 +5584,7 @@ module Aws::Glue
|
|
5474
5584
|
#
|
5475
5585
|
# {
|
5476
5586
|
# name: "NameString", # required
|
5477
|
-
# role: "
|
5587
|
+
# role: "Role",
|
5478
5588
|
# database_name: "DatabaseName",
|
5479
5589
|
# description: "DescriptionStringRemovable",
|
5480
5590
|
# targets: {
|
@@ -5499,28 +5609,29 @@ module Aws::Glue
|
|
5499
5609
|
# update_behavior: "LOG", # accepts LOG, UPDATE_IN_DATABASE
|
5500
5610
|
# delete_behavior: "LOG", # accepts LOG, DELETE_FROM_DATABASE, DEPRECATE_IN_DATABASE
|
5501
5611
|
# },
|
5612
|
+
# configuration: "CrawlerConfiguration",
|
5502
5613
|
# }
|
5503
5614
|
#
|
5504
5615
|
# @!attribute [rw] name
|
5505
|
-
# Name of the new
|
5616
|
+
# Name of the new crawler.
|
5506
5617
|
# @return [String]
|
5507
5618
|
#
|
5508
5619
|
# @!attribute [rw] role
|
5509
|
-
# The IAM role (or ARN of an IAM role) used by the new
|
5620
|
+
# The IAM role (or ARN of an IAM role) used by the new crawler to
|
5510
5621
|
# access customer resources.
|
5511
5622
|
# @return [String]
|
5512
5623
|
#
|
5513
5624
|
# @!attribute [rw] database_name
|
5514
|
-
# The Glue
|
5625
|
+
# The AWS Glue database where results are stored, such as:
|
5515
5626
|
# `arn:aws:daylight:us-east-1::database/sometable/*`.
|
5516
5627
|
# @return [String]
|
5517
5628
|
#
|
5518
5629
|
# @!attribute [rw] description
|
5519
|
-
# A description of the new
|
5630
|
+
# A description of the new crawler.
|
5520
5631
|
# @return [String]
|
5521
5632
|
#
|
5522
5633
|
# @!attribute [rw] targets
|
5523
|
-
# A list of
|
5634
|
+
# A list of targets to crawl.
|
5524
5635
|
# @return [Types::CrawlerTargets]
|
5525
5636
|
#
|
5526
5637
|
# @!attribute [rw] schedule
|
@@ -5534,20 +5645,31 @@ module Aws::Glue
|
|
5534
5645
|
# @return [String]
|
5535
5646
|
#
|
5536
5647
|
# @!attribute [rw] classifiers
|
5537
|
-
# A list of custom
|
5538
|
-
# default, all
|
5539
|
-
#
|
5540
|
-
#
|
5648
|
+
# A list of custom classifiers that the user has registered. By
|
5649
|
+
# default, all classifiers are included in a crawl, but these custom
|
5650
|
+
# classifiers always override the default classifiers for a given
|
5651
|
+
# classification.
|
5541
5652
|
# @return [Array<String>]
|
5542
5653
|
#
|
5543
5654
|
# @!attribute [rw] table_prefix
|
5544
|
-
# The table prefix used for catalog tables created.
|
5655
|
+
# The table prefix used for catalog tables that are created.
|
5545
5656
|
# @return [String]
|
5546
5657
|
#
|
5547
5658
|
# @!attribute [rw] schema_change_policy
|
5548
5659
|
# Policy for the crawler's update and deletion behavior.
|
5549
5660
|
# @return [Types::SchemaChangePolicy]
|
5550
5661
|
#
|
5662
|
+
# @!attribute [rw] configuration
|
5663
|
+
# Crawler configuration information. This versioned JSON string allows
|
5664
|
+
# users to specify aspects of a Crawler's behavior.
|
5665
|
+
#
|
5666
|
+
# You can use this field to force partitions to inherit metadata such
|
5667
|
+
# as classification, input format, output format, serde information,
|
5668
|
+
# and schema from their parent table, rather than detect this
|
5669
|
+
# information separately for each partition. Use the following JSON
|
5670
|
+
# string to specify that behavior:
|
5671
|
+
# @return [String]
|
5672
|
+
#
|
5551
5673
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateCrawlerRequest AWS API Documentation
|
5552
5674
|
#
|
5553
5675
|
class UpdateCrawlerRequest < Struct.new(
|
@@ -5559,7 +5681,8 @@ module Aws::Glue
|
|
5559
5681
|
:schedule,
|
5560
5682
|
:classifiers,
|
5561
5683
|
:table_prefix,
|
5562
|
-
:schema_change_policy
|
5684
|
+
:schema_change_policy,
|
5685
|
+
:configuration)
|
5563
5686
|
include Aws::Structure
|
5564
5687
|
end
|
5565
5688
|
|
@@ -5689,7 +5812,8 @@ module Aws::Glue
|
|
5689
5812
|
#
|
5690
5813
|
class UpdateDevEndpointResponse < Aws::EmptyStructure; end
|
5691
5814
|
|
5692
|
-
# Specifies a
|
5815
|
+
# Specifies a grok classifier to update when passed to
|
5816
|
+
# `UpdateClassifier`.
|
5693
5817
|
#
|
5694
5818
|
# @note When making an API call, you may pass UpdateGrokClassifierRequest
|
5695
5819
|
# data as a hash:
|
@@ -5706,8 +5830,8 @@ module Aws::Glue
|
|
5706
5830
|
# @return [String]
|
5707
5831
|
#
|
5708
5832
|
# @!attribute [rw] classification
|
5709
|
-
#
|
5710
|
-
#
|
5833
|
+
# An identifier of the data format that the classifier matches, such
|
5834
|
+
# as Twitter, JSON, Omniture logs, Amazon CloudWatch Logs, and so on.
|
5711
5835
|
# @return [String]
|
5712
5836
|
#
|
5713
5837
|
# @!attribute [rw] grok_pattern
|
@@ -5715,7 +5839,7 @@ module Aws::Glue
|
|
5715
5839
|
# @return [String]
|
5716
5840
|
#
|
5717
5841
|
# @!attribute [rw] custom_patterns
|
5718
|
-
#
|
5842
|
+
# Optional custom grok patterns used by this classifier.
|
5719
5843
|
# @return [String]
|
5720
5844
|
#
|
5721
5845
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateGrokClassifierRequest AWS API Documentation
|
@@ -6082,6 +6206,41 @@ module Aws::Glue
|
|
6082
6206
|
#
|
6083
6207
|
class UpdateUserDefinedFunctionResponse < Aws::EmptyStructure; end
|
6084
6208
|
|
6209
|
+
# Specifies an XML classifier to be updated.
|
6210
|
+
#
|
6211
|
+
# @note When making an API call, you may pass UpdateXMLClassifierRequest
|
6212
|
+
# data as a hash:
|
6213
|
+
#
|
6214
|
+
# {
|
6215
|
+
# name: "NameString", # required
|
6216
|
+
# classification: "Classification",
|
6217
|
+
# row_tag: "RowTag",
|
6218
|
+
# }
|
6219
|
+
#
|
6220
|
+
# @!attribute [rw] name
|
6221
|
+
# The name of the classifier.
|
6222
|
+
# @return [String]
|
6223
|
+
#
|
6224
|
+
# @!attribute [rw] classification
|
6225
|
+
# An identifier of the data format that the classifier matches.
|
6226
|
+
# @return [String]
|
6227
|
+
#
|
6228
|
+
# @!attribute [rw] row_tag
|
6229
|
+
# The XML tag designating the element that contains each record in an
|
6230
|
+
# XML document being parsed. Note that this cannot be an empty
|
6231
|
+
# element. It must contain child elements representing fields in the
|
6232
|
+
# record.
|
6233
|
+
# @return [String]
|
6234
|
+
#
|
6235
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateXMLClassifierRequest AWS API Documentation
|
6236
|
+
#
|
6237
|
+
class UpdateXMLClassifierRequest < Struct.new(
|
6238
|
+
:name,
|
6239
|
+
:classification,
|
6240
|
+
:row_tag)
|
6241
|
+
include Aws::Structure
|
6242
|
+
end
|
6243
|
+
|
6085
6244
|
# Represents the equivalent of a Hive user-defined function (`UDF`)
|
6086
6245
|
# definition.
|
6087
6246
|
#
|
@@ -6170,5 +6329,46 @@ module Aws::Glue
|
|
6170
6329
|
include Aws::Structure
|
6171
6330
|
end
|
6172
6331
|
|
6332
|
+
# A classifier for `XML` content.
|
6333
|
+
#
|
6334
|
+
# @!attribute [rw] name
|
6335
|
+
# The name of the classifier.
|
6336
|
+
# @return [String]
|
6337
|
+
#
|
6338
|
+
# @!attribute [rw] classification
|
6339
|
+
# An identifier of the data format that the classifier matches.
|
6340
|
+
# @return [String]
|
6341
|
+
#
|
6342
|
+
# @!attribute [rw] creation_time
|
6343
|
+
# The time this classifier was registered.
|
6344
|
+
# @return [Time]
|
6345
|
+
#
|
6346
|
+
# @!attribute [rw] last_updated
|
6347
|
+
# The time this classifier was last updated.
|
6348
|
+
# @return [Time]
|
6349
|
+
#
|
6350
|
+
# @!attribute [rw] version
|
6351
|
+
# The version of this classifier.
|
6352
|
+
# @return [Integer]
|
6353
|
+
#
|
6354
|
+
# @!attribute [rw] row_tag
|
6355
|
+
# The XML tag designating the element that contains each record in an
|
6356
|
+
# XML document being parsed. Note that this cannot be an empty
|
6357
|
+
# element. It must contain child elements representing fields in the
|
6358
|
+
# record.
|
6359
|
+
# @return [String]
|
6360
|
+
#
|
6361
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/XMLClassifier AWS API Documentation
|
6362
|
+
#
|
6363
|
+
class XMLClassifier < Struct.new(
|
6364
|
+
:name,
|
6365
|
+
:classification,
|
6366
|
+
:creation_time,
|
6367
|
+
:last_updated,
|
6368
|
+
:version,
|
6369
|
+
:row_tag)
|
6370
|
+
include Aws::Structure
|
6371
|
+
end
|
6372
|
+
|
6173
6373
|
end
|
6174
6374
|
end
|