aws-sdk-glue 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/aws-sdk-glue.rb +1 -1
- data/lib/aws-sdk-glue/client.rb +98 -46
- data/lib/aws-sdk-glue/client_api.rb +34 -3
- data/lib/aws-sdk-glue/types.rb +288 -88
- metadata +2 -2
data/lib/aws-sdk-glue/types.rb
CHANGED
@@ -466,21 +466,29 @@ module Aws::Glue
|
|
466
466
|
include Aws::Structure
|
467
467
|
end
|
468
468
|
|
469
|
-
# Classifiers are written in Python and triggered during a
|
470
|
-
# You can write your own
|
469
|
+
# Classifiers are written in Python and triggered during a crawl task.
|
470
|
+
# You can write your own classifiers to best categorize your data
|
471
471
|
# sources and specify the appropriate schemas to use for them. A
|
472
|
-
#
|
473
|
-
#
|
472
|
+
# classifier checks whether a given file is in a format it can handle,
|
473
|
+
# and if it is, the classifier creates a schema in the form of a
|
474
474
|
# `StructType` object that matches that data format.
|
475
475
|
#
|
476
|
+
# A classifier can be either a `grok` classifier or an XML classifier,
|
477
|
+
# specified in one or the other field of the `Classifier` object.
|
478
|
+
#
|
476
479
|
# @!attribute [rw] grok_classifier
|
477
|
-
# A GrokClassifier object.
|
480
|
+
# A `GrokClassifier` object.
|
478
481
|
# @return [Types::GrokClassifier]
|
479
482
|
#
|
483
|
+
# @!attribute [rw] xml_classifier
|
484
|
+
# An `XMLClassifier` object.
|
485
|
+
# @return [Types::XMLClassifier]
|
486
|
+
#
|
480
487
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Classifier AWS API Documentation
|
481
488
|
#
|
482
489
|
class Classifier < Struct.new(
|
483
|
-
:grok_classifier
|
490
|
+
:grok_classifier,
|
491
|
+
:xml_classifier)
|
484
492
|
include Aws::Structure
|
485
493
|
end
|
486
494
|
|
@@ -792,15 +800,16 @@ module Aws::Glue
|
|
792
800
|
|
793
801
|
# Specifies a crawler program that examines a data source and uses
|
794
802
|
# classifiers to try to determine its schema. If successful, the crawler
|
795
|
-
# records
|
803
|
+
# records metadata concerning the data source in the AWS Glue Data
|
804
|
+
# Catalog.
|
796
805
|
#
|
797
806
|
# @!attribute [rw] name
|
798
|
-
# The
|
807
|
+
# The crawler name.
|
799
808
|
# @return [String]
|
800
809
|
#
|
801
810
|
# @!attribute [rw] role
|
802
811
|
# The IAM role (or ARN of an IAM role) used to access customer
|
803
|
-
# resources such as data in S3.
|
812
|
+
# resources, such as data in Amazon S3.
|
804
813
|
# @return [String]
|
805
814
|
#
|
806
815
|
# @!attribute [rw] targets
|
@@ -808,46 +817,46 @@ module Aws::Glue
|
|
808
817
|
# @return [Types::CrawlerTargets]
|
809
818
|
#
|
810
819
|
# @!attribute [rw] database_name
|
811
|
-
# The
|
820
|
+
# The database where metadata is written by this crawler.
|
812
821
|
# @return [String]
|
813
822
|
#
|
814
823
|
# @!attribute [rw] description
|
815
|
-
# A description of
|
824
|
+
# A description of the crawler.
|
816
825
|
# @return [String]
|
817
826
|
#
|
818
827
|
# @!attribute [rw] classifiers
|
819
|
-
# A list of custom
|
828
|
+
# A list of custom classifiers associated with the crawler.
|
820
829
|
# @return [Array<String>]
|
821
830
|
#
|
822
831
|
# @!attribute [rw] schema_change_policy
|
823
|
-
# Sets
|
832
|
+
# Sets the behavior when the crawler finds a changed or deleted
|
833
|
+
# object.
|
824
834
|
# @return [Types::SchemaChangePolicy]
|
825
835
|
#
|
826
836
|
# @!attribute [rw] state
|
827
|
-
# Indicates whether
|
837
|
+
# Indicates whether the crawler is running, or whether a run is
|
828
838
|
# pending.
|
829
839
|
# @return [String]
|
830
840
|
#
|
831
841
|
# @!attribute [rw] table_prefix
|
832
|
-
# The
|
842
|
+
# The prefix added to the names of tables that are created.
|
833
843
|
# @return [String]
|
834
844
|
#
|
835
845
|
# @!attribute [rw] schedule
|
836
|
-
#
|
837
|
-
# Crawler is to be run.
|
846
|
+
# For scheduled crawlers, the schedule when the crawler runs.
|
838
847
|
# @return [Types::Schedule]
|
839
848
|
#
|
840
849
|
# @!attribute [rw] crawl_elapsed_time
|
841
|
-
# If
|
842
|
-
#
|
850
|
+
# If the crawler is running, contains the total time elapsed since the
|
851
|
+
# last crawl began.
|
843
852
|
# @return [Integer]
|
844
853
|
#
|
845
854
|
# @!attribute [rw] creation_time
|
846
|
-
# The time when the
|
855
|
+
# The time when the crawler was created.
|
847
856
|
# @return [Time]
|
848
857
|
#
|
849
858
|
# @!attribute [rw] last_updated
|
850
|
-
# The time the
|
859
|
+
# The time the crawler was last updated.
|
851
860
|
# @return [Time]
|
852
861
|
#
|
853
862
|
# @!attribute [rw] last_crawl
|
@@ -856,9 +865,20 @@ module Aws::Glue
|
|
856
865
|
# @return [Types::LastCrawlInfo]
|
857
866
|
#
|
858
867
|
# @!attribute [rw] version
|
859
|
-
# The version of the
|
868
|
+
# The version of the crawler.
|
860
869
|
# @return [Integer]
|
861
870
|
#
|
871
|
+
# @!attribute [rw] configuration
|
872
|
+
# Crawler configuration information. This versioned JSON string allows
|
873
|
+
# users to specify aspects of a Crawler's behavior.
|
874
|
+
#
|
875
|
+
# You can use this field to force partitions to inherit metadata such
|
876
|
+
# as classification, input format, output format, serde information,
|
877
|
+
# and schema from their parent table, rather than detect this
|
878
|
+
# information separately for each partition. Use the following JSON
|
879
|
+
# string to specify that behavior:
|
880
|
+
# @return [String]
|
881
|
+
#
|
862
882
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Crawler AWS API Documentation
|
863
883
|
#
|
864
884
|
class Crawler < Struct.new(
|
@@ -876,7 +896,8 @@ module Aws::Glue
|
|
876
896
|
:creation_time,
|
877
897
|
:last_updated,
|
878
898
|
:last_crawl,
|
879
|
-
:version
|
899
|
+
:version,
|
900
|
+
:configuration)
|
880
901
|
include Aws::Structure
|
881
902
|
end
|
882
903
|
|
@@ -891,7 +912,8 @@ module Aws::Glue
|
|
891
912
|
# @return [Float]
|
892
913
|
#
|
893
914
|
# @!attribute [rw] still_estimating
|
894
|
-
# True if the crawler is estimating
|
915
|
+
# True if the crawler is still estimating how long it will take to
|
916
|
+
# complete this run.
|
895
917
|
# @return [Boolean]
|
896
918
|
#
|
897
919
|
# @!attribute [rw] last_runtime_seconds
|
@@ -903,15 +925,15 @@ module Aws::Glue
|
|
903
925
|
# @return [Float]
|
904
926
|
#
|
905
927
|
# @!attribute [rw] tables_created
|
906
|
-
#
|
928
|
+
# The number of tables created by this crawler.
|
907
929
|
# @return [Integer]
|
908
930
|
#
|
909
931
|
# @!attribute [rw] tables_updated
|
910
|
-
#
|
932
|
+
# The number of tables updated by this crawler.
|
911
933
|
# @return [Integer]
|
912
934
|
#
|
913
935
|
# @!attribute [rw] tables_deleted
|
914
|
-
#
|
936
|
+
# The number of tables deleted by this crawler.
|
915
937
|
# @return [Integer]
|
916
938
|
#
|
917
939
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CrawlerMetrics AWS API Documentation
|
@@ -928,7 +950,7 @@ module Aws::Glue
|
|
928
950
|
include Aws::Structure
|
929
951
|
end
|
930
952
|
|
931
|
-
# Specifies
|
953
|
+
# Specifies data stores to crawl.
|
932
954
|
#
|
933
955
|
# @note When making an API call, you may pass CrawlerTargets
|
934
956
|
# data as a hash:
|
@@ -950,7 +972,7 @@ module Aws::Glue
|
|
950
972
|
# }
|
951
973
|
#
|
952
974
|
# @!attribute [rw] s3_targets
|
953
|
-
# Specifies
|
975
|
+
# Specifies Amazon S3 targets.
|
954
976
|
# @return [Array<Types::S3Target>]
|
955
977
|
#
|
956
978
|
# @!attribute [rw] jdbc_targets
|
@@ -975,16 +997,26 @@ module Aws::Glue
|
|
975
997
|
# grok_pattern: "GrokPattern", # required
|
976
998
|
# custom_patterns: "CustomPatterns",
|
977
999
|
# },
|
1000
|
+
# xml_classifier: {
|
1001
|
+
# classification: "Classification", # required
|
1002
|
+
# name: "NameString", # required
|
1003
|
+
# row_tag: "RowTag",
|
1004
|
+
# },
|
978
1005
|
# }
|
979
1006
|
#
|
980
1007
|
# @!attribute [rw] grok_classifier
|
981
|
-
# A
|
1008
|
+
# A `GrokClassifier` object specifying the classifier to create.
|
982
1009
|
# @return [Types::CreateGrokClassifierRequest]
|
983
1010
|
#
|
1011
|
+
# @!attribute [rw] xml_classifier
|
1012
|
+
# An `XMLClassifier` object specifying the classifier to create.
|
1013
|
+
# @return [Types::CreateXMLClassifierRequest]
|
1014
|
+
#
|
984
1015
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateClassifierRequest AWS API Documentation
|
985
1016
|
#
|
986
1017
|
class CreateClassifierRequest < Struct.new(
|
987
|
-
:grok_classifier
|
1018
|
+
:grok_classifier,
|
1019
|
+
:xml_classifier)
|
988
1020
|
include Aws::Structure
|
989
1021
|
end
|
990
1022
|
|
@@ -1039,7 +1071,7 @@ module Aws::Glue
|
|
1039
1071
|
#
|
1040
1072
|
# {
|
1041
1073
|
# name: "NameString", # required
|
1042
|
-
# role: "
|
1074
|
+
# role: "Role", # required
|
1043
1075
|
# database_name: "DatabaseName", # required
|
1044
1076
|
# description: "DescriptionString",
|
1045
1077
|
# targets: { # required
|
@@ -1064,24 +1096,25 @@ module Aws::Glue
|
|
1064
1096
|
# update_behavior: "LOG", # accepts LOG, UPDATE_IN_DATABASE
|
1065
1097
|
# delete_behavior: "LOG", # accepts LOG, DELETE_FROM_DATABASE, DEPRECATE_IN_DATABASE
|
1066
1098
|
# },
|
1099
|
+
# configuration: "CrawlerConfiguration",
|
1067
1100
|
# }
|
1068
1101
|
#
|
1069
1102
|
# @!attribute [rw] name
|
1070
|
-
# Name of the new
|
1103
|
+
# Name of the new crawler.
|
1071
1104
|
# @return [String]
|
1072
1105
|
#
|
1073
1106
|
# @!attribute [rw] role
|
1074
|
-
# The IAM role (or ARN of an IAM role) used by the new
|
1107
|
+
# The IAM role (or ARN of an IAM role) used by the new crawler to
|
1075
1108
|
# access customer resources.
|
1076
1109
|
# @return [String]
|
1077
1110
|
#
|
1078
1111
|
# @!attribute [rw] database_name
|
1079
|
-
# The Glue
|
1112
|
+
# The AWS Glue database where results are written, such as:
|
1080
1113
|
# `arn:aws:daylight:us-east-1::database/sometable/*`.
|
1081
1114
|
# @return [String]
|
1082
1115
|
#
|
1083
1116
|
# @!attribute [rw] description
|
1084
|
-
# A description of the new
|
1117
|
+
# A description of the new crawler.
|
1085
1118
|
# @return [String]
|
1086
1119
|
#
|
1087
1120
|
# @!attribute [rw] targets
|
@@ -1099,20 +1132,30 @@ module Aws::Glue
|
|
1099
1132
|
# @return [String]
|
1100
1133
|
#
|
1101
1134
|
# @!attribute [rw] classifiers
|
1102
|
-
# A list of custom
|
1135
|
+
# A list of custom classifiers that the user has registered. By
|
1103
1136
|
# default, all AWS classifiers are included in a crawl, but these
|
1104
1137
|
# custom classifiers always override the default classifiers for a
|
1105
1138
|
# given classification.
|
1106
1139
|
# @return [Array<String>]
|
1107
1140
|
#
|
1108
1141
|
# @!attribute [rw] table_prefix
|
1109
|
-
# The table prefix used for catalog tables created.
|
1142
|
+
# The table prefix used for catalog tables that are created.
|
1110
1143
|
# @return [String]
|
1111
1144
|
#
|
1112
1145
|
# @!attribute [rw] schema_change_policy
|
1113
1146
|
# Policy for the crawler's update and deletion behavior.
|
1114
1147
|
# @return [Types::SchemaChangePolicy]
|
1115
1148
|
#
|
1149
|
+
# @!attribute [rw] configuration
|
1150
|
+
# Crawler configuration information. This versioned JSON string allows
|
1151
|
+
# users to specify aspects of a Crawler's behavior.
|
1152
|
+
#
|
1153
|
+
# You can use this field to force partitions to inherit metadata such
|
1154
|
+
# as classification, input format, output format, serde information,
|
1155
|
+
# and schema from their parent table, rather than detect this
|
1156
|
+
# information separately for each partition.
|
1157
|
+
# @return [String]
|
1158
|
+
#
|
1116
1159
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateCrawlerRequest AWS API Documentation
|
1117
1160
|
#
|
1118
1161
|
class CreateCrawlerRequest < Struct.new(
|
@@ -1124,7 +1167,8 @@ module Aws::Glue
|
|
1124
1167
|
:schedule,
|
1125
1168
|
:classifiers,
|
1126
1169
|
:table_prefix,
|
1127
|
-
:schema_change_policy
|
1170
|
+
:schema_change_policy,
|
1171
|
+
:configuration)
|
1128
1172
|
include Aws::Structure
|
1129
1173
|
end
|
1130
1174
|
|
@@ -1321,7 +1365,7 @@ module Aws::Glue
|
|
1321
1365
|
include Aws::Structure
|
1322
1366
|
end
|
1323
1367
|
|
1324
|
-
# Specifies a
|
1368
|
+
# Specifies a `grok` classifier for `CreateClassifier` to create.
|
1325
1369
|
#
|
1326
1370
|
# @note When making an API call, you may pass CreateGrokClassifierRequest
|
1327
1371
|
# data as a hash:
|
@@ -1334,12 +1378,12 @@ module Aws::Glue
|
|
1334
1378
|
# }
|
1335
1379
|
#
|
1336
1380
|
# @!attribute [rw] classification
|
1337
|
-
#
|
1338
|
-
#
|
1381
|
+
# An identifier of the data format that the classifier matches, such
|
1382
|
+
# as Twitter, JSON, Omniture logs, Amazon CloudWatch Logs, and so on.
|
1339
1383
|
# @return [String]
|
1340
1384
|
#
|
1341
1385
|
# @!attribute [rw] name
|
1342
|
-
# The name of the new
|
1386
|
+
# The name of the new classifier.
|
1343
1387
|
# @return [String]
|
1344
1388
|
#
|
1345
1389
|
# @!attribute [rw] grok_pattern
|
@@ -1347,7 +1391,7 @@ module Aws::Glue
|
|
1347
1391
|
# @return [String]
|
1348
1392
|
#
|
1349
1393
|
# @!attribute [rw] custom_patterns
|
1350
|
-
#
|
1394
|
+
# Optional custom grok patterns used by this classifier.
|
1351
1395
|
# @return [String]
|
1352
1396
|
#
|
1353
1397
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateGrokClassifierRequest AWS API Documentation
|
@@ -1819,6 +1863,41 @@ module Aws::Glue
|
|
1819
1863
|
#
|
1820
1864
|
class CreateUserDefinedFunctionResponse < Aws::EmptyStructure; end
|
1821
1865
|
|
1866
|
+
# Specifies an XML classifier for `CreateClassifier` to create.
|
1867
|
+
#
|
1868
|
+
# @note When making an API call, you may pass CreateXMLClassifierRequest
|
1869
|
+
# data as a hash:
|
1870
|
+
#
|
1871
|
+
# {
|
1872
|
+
# classification: "Classification", # required
|
1873
|
+
# name: "NameString", # required
|
1874
|
+
# row_tag: "RowTag",
|
1875
|
+
# }
|
1876
|
+
#
|
1877
|
+
# @!attribute [rw] classification
|
1878
|
+
# An identifier of the data format that the classifier matches.
|
1879
|
+
# @return [String]
|
1880
|
+
#
|
1881
|
+
# @!attribute [rw] name
|
1882
|
+
# The name of the classifier.
|
1883
|
+
# @return [String]
|
1884
|
+
#
|
1885
|
+
# @!attribute [rw] row_tag
|
1886
|
+
# The XML tag designating the element that contains each record in an
|
1887
|
+
# XML document being parsed. Note that this cannot be an empty
|
1888
|
+
# element. It must contain child elements representing fields in the
|
1889
|
+
# record.
|
1890
|
+
# @return [String]
|
1891
|
+
#
|
1892
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/CreateXMLClassifierRequest AWS API Documentation
|
1893
|
+
#
|
1894
|
+
class CreateXMLClassifierRequest < Struct.new(
|
1895
|
+
:classification,
|
1896
|
+
:name,
|
1897
|
+
:row_tag)
|
1898
|
+
include Aws::Structure
|
1899
|
+
end
|
1900
|
+
|
1822
1901
|
# The `Database` object represents a logical grouping of tables that may
|
1823
1902
|
# reside in a Hive metastore or an RDBMS.
|
1824
1903
|
#
|
@@ -1903,7 +1982,7 @@ module Aws::Glue
|
|
1903
1982
|
# }
|
1904
1983
|
#
|
1905
1984
|
# @!attribute [rw] name
|
1906
|
-
# Name of the
|
1985
|
+
# Name of the classifier to remove.
|
1907
1986
|
# @return [String]
|
1908
1987
|
#
|
1909
1988
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/DeleteClassifierRequest AWS API Documentation
|
@@ -1954,7 +2033,7 @@ module Aws::Glue
|
|
1954
2033
|
# }
|
1955
2034
|
#
|
1956
2035
|
# @!attribute [rw] name
|
1957
|
-
# Name of the
|
2036
|
+
# Name of the crawler to remove.
|
1958
2037
|
# @return [String]
|
1959
2038
|
#
|
1960
2039
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/DeleteCrawlerRequest AWS API Documentation
|
@@ -2419,7 +2498,7 @@ module Aws::Glue
|
|
2419
2498
|
# }
|
2420
2499
|
#
|
2421
2500
|
# @!attribute [rw] name
|
2422
|
-
# Name of the
|
2501
|
+
# Name of the classifier to retrieve.
|
2423
2502
|
# @return [String]
|
2424
2503
|
#
|
2425
2504
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetClassifierRequest AWS API Documentation
|
@@ -2430,7 +2509,7 @@ module Aws::Glue
|
|
2430
2509
|
end
|
2431
2510
|
|
2432
2511
|
# @!attribute [rw] classifier
|
2433
|
-
# The requested
|
2512
|
+
# The requested classifier.
|
2434
2513
|
# @return [Types::Classifier]
|
2435
2514
|
#
|
2436
2515
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetClassifierResponse AWS API Documentation
|
@@ -2465,7 +2544,7 @@ module Aws::Glue
|
|
2465
2544
|
end
|
2466
2545
|
|
2467
2546
|
# @!attribute [rw] classifiers
|
2468
|
-
# The requested list of
|
2547
|
+
# The requested list of classifier objects.
|
2469
2548
|
# @return [Array<Types::Classifier>]
|
2470
2549
|
#
|
2471
2550
|
# @!attribute [rw] next_token
|
@@ -2657,7 +2736,7 @@ module Aws::Glue
|
|
2657
2736
|
# }
|
2658
2737
|
#
|
2659
2738
|
# @!attribute [rw] name
|
2660
|
-
# Name of the
|
2739
|
+
# Name of the crawler to retrieve metadata for.
|
2661
2740
|
# @return [String]
|
2662
2741
|
#
|
2663
2742
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetCrawlerRequest AWS API Documentation
|
@@ -2668,7 +2747,7 @@ module Aws::Glue
|
|
2668
2747
|
end
|
2669
2748
|
|
2670
2749
|
# @!attribute [rw] crawler
|
2671
|
-
# The metadata for the specified
|
2750
|
+
# The metadata for the specified crawler.
|
2672
2751
|
# @return [Types::Crawler]
|
2673
2752
|
#
|
2674
2753
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetCrawlerResponse AWS API Documentation
|
@@ -2687,7 +2766,7 @@ module Aws::Glue
|
|
2687
2766
|
# }
|
2688
2767
|
#
|
2689
2768
|
# @!attribute [rw] max_results
|
2690
|
-
# The number of
|
2769
|
+
# The number of crawlers to return on each call.
|
2691
2770
|
# @return [Integer]
|
2692
2771
|
#
|
2693
2772
|
# @!attribute [rw] next_token
|
@@ -2703,7 +2782,7 @@ module Aws::Glue
|
|
2703
2782
|
end
|
2704
2783
|
|
2705
2784
|
# @!attribute [rw] crawlers
|
2706
|
-
# A list of
|
2785
|
+
# A list of crawler metadata.
|
2707
2786
|
# @return [Array<Types::Crawler>]
|
2708
2787
|
#
|
2709
2788
|
# @!attribute [rw] next_token
|
@@ -3321,7 +3400,7 @@ module Aws::Glue
|
|
3321
3400
|
end
|
3322
3401
|
|
3323
3402
|
# @!attribute [rw] python_script
|
3324
|
-
# A
|
3403
|
+
# A Python script to perform the mapping.
|
3325
3404
|
# @return [String]
|
3326
3405
|
#
|
3327
3406
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GetPlanResponse AWS API Documentation
|
@@ -3674,15 +3753,15 @@ module Aws::Glue
|
|
3674
3753
|
include Aws::Structure
|
3675
3754
|
end
|
3676
3755
|
|
3677
|
-
# A classifier that uses `grok
|
3756
|
+
# A classifier that uses `grok` patterns.
|
3678
3757
|
#
|
3679
3758
|
# @!attribute [rw] name
|
3680
3759
|
# The name of the classifier.
|
3681
3760
|
# @return [String]
|
3682
3761
|
#
|
3683
3762
|
# @!attribute [rw] classification
|
3684
|
-
#
|
3685
|
-
# Omniture
|
3763
|
+
# An identifier of the data format that the classifier matches, such
|
3764
|
+
# as Twitter, JSON, Omniture logs, and so on.
|
3686
3765
|
# @return [String]
|
3687
3766
|
#
|
3688
3767
|
# @!attribute [rw] creation_time
|
@@ -3698,11 +3777,22 @@ module Aws::Glue
|
|
3698
3777
|
# @return [Integer]
|
3699
3778
|
#
|
3700
3779
|
# @!attribute [rw] grok_pattern
|
3701
|
-
# The grok pattern
|
3780
|
+
# The grok pattern applied to a data store by this classifier. For
|
3781
|
+
# more information, see built-in patterns in [Writing Custom
|
3782
|
+
# Classifers][1].
|
3783
|
+
#
|
3784
|
+
#
|
3785
|
+
#
|
3786
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/custom-classifier.html
|
3702
3787
|
# @return [String]
|
3703
3788
|
#
|
3704
3789
|
# @!attribute [rw] custom_patterns
|
3705
|
-
#
|
3790
|
+
# Optional custom grok patterns defined by this classifier. For more
|
3791
|
+
# information, see custom patterns in [Writing Custom Classifers][1].
|
3792
|
+
#
|
3793
|
+
#
|
3794
|
+
#
|
3795
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/custom-classifier.html
|
3706
3796
|
# @return [String]
|
3707
3797
|
#
|
3708
3798
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/GrokClassifier AWS API Documentation
|
@@ -3741,7 +3831,7 @@ module Aws::Glue
|
|
3741
3831
|
#
|
3742
3832
|
class ImportCatalogToGlueResponse < Aws::EmptyStructure; end
|
3743
3833
|
|
3744
|
-
# Specifies a JDBC
|
3834
|
+
# Specifies a JDBC data store to crawl.
|
3745
3835
|
#
|
3746
3836
|
# @note When making an API call, you may pass JdbcTarget
|
3747
3837
|
# data as a hash:
|
@@ -3753,7 +3843,7 @@ module Aws::Glue
|
|
3753
3843
|
# }
|
3754
3844
|
#
|
3755
3845
|
# @!attribute [rw] connection_name
|
3756
|
-
# The name of the connection to use
|
3846
|
+
# The name of the connection to use to connect to the JDBC target.
|
3757
3847
|
# @return [String]
|
3758
3848
|
#
|
3759
3849
|
# @!attribute [rw] path
|
@@ -3761,7 +3851,12 @@ module Aws::Glue
|
|
3761
3851
|
# @return [String]
|
3762
3852
|
#
|
3763
3853
|
# @!attribute [rw] exclusions
|
3764
|
-
# A list of
|
3854
|
+
# A list of glob patterns used to exclude from the crawl. For more
|
3855
|
+
# information, see [Catalog Tables with a Crawler][1].
|
3856
|
+
#
|
3857
|
+
#
|
3858
|
+
#
|
3859
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/add-crawler.html
|
3765
3860
|
# @return [Array<String>]
|
3766
3861
|
#
|
3767
3862
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/JdbcTarget AWS API Documentation
|
@@ -4059,7 +4154,7 @@ module Aws::Glue
|
|
4059
4154
|
# @return [String]
|
4060
4155
|
#
|
4061
4156
|
# @!attribute [rw] error_message
|
4062
|
-
#
|
4157
|
+
# If an error occurred, the error information about the last crawl.
|
4063
4158
|
# @return [String]
|
4064
4159
|
#
|
4065
4160
|
# @!attribute [rw] log_group
|
@@ -4117,7 +4212,7 @@ module Aws::Glue
|
|
4117
4212
|
# @return [Array<Types::CodeGenNodeArg>]
|
4118
4213
|
#
|
4119
4214
|
# @!attribute [rw] s3
|
4120
|
-
# An
|
4215
|
+
# An Amazon S3 location.
|
4121
4216
|
# @return [Array<Types::CodeGenNodeArg>]
|
4122
4217
|
#
|
4123
4218
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/Location AWS API Documentation
|
@@ -4517,7 +4612,7 @@ module Aws::Glue
|
|
4517
4612
|
include Aws::Structure
|
4518
4613
|
end
|
4519
4614
|
|
4520
|
-
# Specifies a
|
4615
|
+
# Specifies a data store in Amazon S3.
|
4521
4616
|
#
|
4522
4617
|
# @note When making an API call, you may pass S3Target
|
4523
4618
|
# data as a hash:
|
@@ -4528,11 +4623,16 @@ module Aws::Glue
|
|
4528
4623
|
# }
|
4529
4624
|
#
|
4530
4625
|
# @!attribute [rw] path
|
4531
|
-
# The path to the S3 target.
|
4626
|
+
# The path to the Amazon S3 target.
|
4532
4627
|
# @return [String]
|
4533
4628
|
#
|
4534
4629
|
# @!attribute [rw] exclusions
|
4535
|
-
# A list of
|
4630
|
+
# A list of glob patterns used to exclude from the crawl. For more
|
4631
|
+
# information, see [Catalog Tables with a Crawler][1].
|
4632
|
+
#
|
4633
|
+
#
|
4634
|
+
#
|
4635
|
+
# [1]: http://docs.aws.amazon.com/glue/latest/dg/add-crawler.html
|
4536
4636
|
# @return [Array<String>]
|
4537
4637
|
#
|
4538
4638
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/S3Target AWS API Documentation
|
@@ -4578,11 +4678,11 @@ module Aws::Glue
|
|
4578
4678
|
# }
|
4579
4679
|
#
|
4580
4680
|
# @!attribute [rw] update_behavior
|
4581
|
-
# The update behavior.
|
4681
|
+
# The update behavior when the crawler finds a changed schema.
|
4582
4682
|
# @return [String]
|
4583
4683
|
#
|
4584
4684
|
# @!attribute [rw] delete_behavior
|
4585
|
-
# The deletion behavior.
|
4685
|
+
# The deletion behavior when the crawler finds a deleted object.
|
4586
4686
|
# @return [String]
|
4587
4687
|
#
|
4588
4688
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/SchemaChangePolicy AWS API Documentation
|
@@ -4703,7 +4803,7 @@ module Aws::Glue
|
|
4703
4803
|
# }
|
4704
4804
|
#
|
4705
4805
|
# @!attribute [rw] name
|
4706
|
-
# Name of the
|
4806
|
+
# Name of the crawler to start.
|
4707
4807
|
# @return [String]
|
4708
4808
|
#
|
4709
4809
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StartCrawlerRequest AWS API Documentation
|
@@ -4825,7 +4925,7 @@ module Aws::Glue
|
|
4825
4925
|
# }
|
4826
4926
|
#
|
4827
4927
|
# @!attribute [rw] name
|
4828
|
-
# Name of the
|
4928
|
+
# Name of the crawler to stop.
|
4829
4929
|
# @return [String]
|
4830
4930
|
#
|
4831
4931
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/StopCrawlerRequest AWS API Documentation
|
@@ -5403,16 +5503,26 @@ module Aws::Glue
|
|
5403
5503
|
# grok_pattern: "GrokPattern",
|
5404
5504
|
# custom_patterns: "CustomPatterns",
|
5405
5505
|
# },
|
5506
|
+
# xml_classifier: {
|
5507
|
+
# name: "NameString", # required
|
5508
|
+
# classification: "Classification",
|
5509
|
+
# row_tag: "RowTag",
|
5510
|
+
# },
|
5406
5511
|
# }
|
5407
5512
|
#
|
5408
5513
|
# @!attribute [rw] grok_classifier
|
5409
5514
|
# A `GrokClassifier` object with updated fields.
|
5410
5515
|
# @return [Types::UpdateGrokClassifierRequest]
|
5411
5516
|
#
|
5517
|
+
# @!attribute [rw] xml_classifier
|
5518
|
+
# An `XMLClassifier` object with updated fields.
|
5519
|
+
# @return [Types::UpdateXMLClassifierRequest]
|
5520
|
+
#
|
5412
5521
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateClassifierRequest AWS API Documentation
|
5413
5522
|
#
|
5414
5523
|
class UpdateClassifierRequest < Struct.new(
|
5415
|
-
:grok_classifier
|
5524
|
+
:grok_classifier,
|
5525
|
+
:xml_classifier)
|
5416
5526
|
include Aws::Structure
|
5417
5527
|
end
|
5418
5528
|
|
@@ -5474,7 +5584,7 @@ module Aws::Glue
|
|
5474
5584
|
#
|
5475
5585
|
# {
|
5476
5586
|
# name: "NameString", # required
|
5477
|
-
# role: "
|
5587
|
+
# role: "Role",
|
5478
5588
|
# database_name: "DatabaseName",
|
5479
5589
|
# description: "DescriptionStringRemovable",
|
5480
5590
|
# targets: {
|
@@ -5499,28 +5609,29 @@ module Aws::Glue
|
|
5499
5609
|
# update_behavior: "LOG", # accepts LOG, UPDATE_IN_DATABASE
|
5500
5610
|
# delete_behavior: "LOG", # accepts LOG, DELETE_FROM_DATABASE, DEPRECATE_IN_DATABASE
|
5501
5611
|
# },
|
5612
|
+
# configuration: "CrawlerConfiguration",
|
5502
5613
|
# }
|
5503
5614
|
#
|
5504
5615
|
# @!attribute [rw] name
|
5505
|
-
# Name of the new
|
5616
|
+
# Name of the new crawler.
|
5506
5617
|
# @return [String]
|
5507
5618
|
#
|
5508
5619
|
# @!attribute [rw] role
|
5509
|
-
# The IAM role (or ARN of an IAM role) used by the new
|
5620
|
+
# The IAM role (or ARN of an IAM role) used by the new crawler to
|
5510
5621
|
# access customer resources.
|
5511
5622
|
# @return [String]
|
5512
5623
|
#
|
5513
5624
|
# @!attribute [rw] database_name
|
5514
|
-
# The Glue
|
5625
|
+
# The AWS Glue database where results are stored, such as:
|
5515
5626
|
# `arn:aws:daylight:us-east-1::database/sometable/*`.
|
5516
5627
|
# @return [String]
|
5517
5628
|
#
|
5518
5629
|
# @!attribute [rw] description
|
5519
|
-
# A description of the new
|
5630
|
+
# A description of the new crawler.
|
5520
5631
|
# @return [String]
|
5521
5632
|
#
|
5522
5633
|
# @!attribute [rw] targets
|
5523
|
-
# A list of
|
5634
|
+
# A list of targets to crawl.
|
5524
5635
|
# @return [Types::CrawlerTargets]
|
5525
5636
|
#
|
5526
5637
|
# @!attribute [rw] schedule
|
@@ -5534,20 +5645,31 @@ module Aws::Glue
|
|
5534
5645
|
# @return [String]
|
5535
5646
|
#
|
5536
5647
|
# @!attribute [rw] classifiers
|
5537
|
-
# A list of custom
|
5538
|
-
# default, all
|
5539
|
-
#
|
5540
|
-
#
|
5648
|
+
# A list of custom classifiers that the user has registered. By
|
5649
|
+
# default, all classifiers are included in a crawl, but these custom
|
5650
|
+
# classifiers always override the default classifiers for a given
|
5651
|
+
# classification.
|
5541
5652
|
# @return [Array<String>]
|
5542
5653
|
#
|
5543
5654
|
# @!attribute [rw] table_prefix
|
5544
|
-
# The table prefix used for catalog tables created.
|
5655
|
+
# The table prefix used for catalog tables that are created.
|
5545
5656
|
# @return [String]
|
5546
5657
|
#
|
5547
5658
|
# @!attribute [rw] schema_change_policy
|
5548
5659
|
# Policy for the crawler's update and deletion behavior.
|
5549
5660
|
# @return [Types::SchemaChangePolicy]
|
5550
5661
|
#
|
5662
|
+
# @!attribute [rw] configuration
|
5663
|
+
# Crawler configuration information. This versioned JSON string allows
|
5664
|
+
# users to specify aspects of a Crawler's behavior.
|
5665
|
+
#
|
5666
|
+
# You can use this field to force partitions to inherit metadata such
|
5667
|
+
# as classification, input format, output format, serde information,
|
5668
|
+
# and schema from their parent table, rather than detect this
|
5669
|
+
# information separately for each partition. Use the following JSON
|
5670
|
+
# string to specify that behavior:
|
5671
|
+
# @return [String]
|
5672
|
+
#
|
5551
5673
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateCrawlerRequest AWS API Documentation
|
5552
5674
|
#
|
5553
5675
|
class UpdateCrawlerRequest < Struct.new(
|
@@ -5559,7 +5681,8 @@ module Aws::Glue
|
|
5559
5681
|
:schedule,
|
5560
5682
|
:classifiers,
|
5561
5683
|
:table_prefix,
|
5562
|
-
:schema_change_policy
|
5684
|
+
:schema_change_policy,
|
5685
|
+
:configuration)
|
5563
5686
|
include Aws::Structure
|
5564
5687
|
end
|
5565
5688
|
|
@@ -5689,7 +5812,8 @@ module Aws::Glue
|
|
5689
5812
|
#
|
5690
5813
|
class UpdateDevEndpointResponse < Aws::EmptyStructure; end
|
5691
5814
|
|
5692
|
-
# Specifies a
|
5815
|
+
# Specifies a grok classifier to update when passed to
|
5816
|
+
# `UpdateClassifier`.
|
5693
5817
|
#
|
5694
5818
|
# @note When making an API call, you may pass UpdateGrokClassifierRequest
|
5695
5819
|
# data as a hash:
|
@@ -5706,8 +5830,8 @@ module Aws::Glue
|
|
5706
5830
|
# @return [String]
|
5707
5831
|
#
|
5708
5832
|
# @!attribute [rw] classification
|
5709
|
-
#
|
5710
|
-
#
|
5833
|
+
# An identifier of the data format that the classifier matches, such
|
5834
|
+
# as Twitter, JSON, Omniture logs, Amazon CloudWatch Logs, and so on.
|
5711
5835
|
# @return [String]
|
5712
5836
|
#
|
5713
5837
|
# @!attribute [rw] grok_pattern
|
@@ -5715,7 +5839,7 @@ module Aws::Glue
|
|
5715
5839
|
# @return [String]
|
5716
5840
|
#
|
5717
5841
|
# @!attribute [rw] custom_patterns
|
5718
|
-
#
|
5842
|
+
# Optional custom grok patterns used by this classifier.
|
5719
5843
|
# @return [String]
|
5720
5844
|
#
|
5721
5845
|
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateGrokClassifierRequest AWS API Documentation
|
@@ -6082,6 +6206,41 @@ module Aws::Glue
|
|
6082
6206
|
#
|
6083
6207
|
class UpdateUserDefinedFunctionResponse < Aws::EmptyStructure; end
|
6084
6208
|
|
6209
|
+
# Specifies an XML classifier to be updated.
|
6210
|
+
#
|
6211
|
+
# @note When making an API call, you may pass UpdateXMLClassifierRequest
|
6212
|
+
# data as a hash:
|
6213
|
+
#
|
6214
|
+
# {
|
6215
|
+
# name: "NameString", # required
|
6216
|
+
# classification: "Classification",
|
6217
|
+
# row_tag: "RowTag",
|
6218
|
+
# }
|
6219
|
+
#
|
6220
|
+
# @!attribute [rw] name
|
6221
|
+
# The name of the classifier.
|
6222
|
+
# @return [String]
|
6223
|
+
#
|
6224
|
+
# @!attribute [rw] classification
|
6225
|
+
# An identifier of the data format that the classifier matches.
|
6226
|
+
# @return [String]
|
6227
|
+
#
|
6228
|
+
# @!attribute [rw] row_tag
|
6229
|
+
# The XML tag designating the element that contains each record in an
|
6230
|
+
# XML document being parsed. Note that this cannot be an empty
|
6231
|
+
# element. It must contain child elements representing fields in the
|
6232
|
+
# record.
|
6233
|
+
# @return [String]
|
6234
|
+
#
|
6235
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/UpdateXMLClassifierRequest AWS API Documentation
|
6236
|
+
#
|
6237
|
+
class UpdateXMLClassifierRequest < Struct.new(
|
6238
|
+
:name,
|
6239
|
+
:classification,
|
6240
|
+
:row_tag)
|
6241
|
+
include Aws::Structure
|
6242
|
+
end
|
6243
|
+
|
6085
6244
|
# Represents the equivalent of a Hive user-defined function (`UDF`)
|
6086
6245
|
# definition.
|
6087
6246
|
#
|
@@ -6170,5 +6329,46 @@ module Aws::Glue
|
|
6170
6329
|
include Aws::Structure
|
6171
6330
|
end
|
6172
6331
|
|
6332
|
+
# A classifier for `XML` content.
|
6333
|
+
#
|
6334
|
+
# @!attribute [rw] name
|
6335
|
+
# The name of the classifier.
|
6336
|
+
# @return [String]
|
6337
|
+
#
|
6338
|
+
# @!attribute [rw] classification
|
6339
|
+
# An identifier of the data format that the classifier matches.
|
6340
|
+
# @return [String]
|
6341
|
+
#
|
6342
|
+
# @!attribute [rw] creation_time
|
6343
|
+
# The time this classifier was registered.
|
6344
|
+
# @return [Time]
|
6345
|
+
#
|
6346
|
+
# @!attribute [rw] last_updated
|
6347
|
+
# The time this classifier was last updated.
|
6348
|
+
# @return [Time]
|
6349
|
+
#
|
6350
|
+
# @!attribute [rw] version
|
6351
|
+
# The version of this classifier.
|
6352
|
+
# @return [Integer]
|
6353
|
+
#
|
6354
|
+
# @!attribute [rw] row_tag
|
6355
|
+
# The XML tag designating the element that contains each record in an
|
6356
|
+
# XML document being parsed. Note that this cannot be an empty
|
6357
|
+
# element. It must contain child elements representing fields in the
|
6358
|
+
# record.
|
6359
|
+
# @return [String]
|
6360
|
+
#
|
6361
|
+
# @see http://docs.aws.amazon.com/goto/WebAPI/glue-2017-03-31/XMLClassifier AWS API Documentation
|
6362
|
+
#
|
6363
|
+
class XMLClassifier < Struct.new(
|
6364
|
+
:name,
|
6365
|
+
:classification,
|
6366
|
+
:creation_time,
|
6367
|
+
:last_updated,
|
6368
|
+
:version,
|
6369
|
+
:row_tag)
|
6370
|
+
include Aws::Structure
|
6371
|
+
end
|
6372
|
+
|
6173
6373
|
end
|
6174
6374
|
end
|