aws-sdk-kendra 1.26.0 → 1.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db6db0d7989bf395d9d66113616dd832292db250ceae884e79f1ff68fc72c2da
-  data.tar.gz: 5d5c1a752c7eb99a07c0c60f67b913244b2731f0301eb4918171af9bdbe95ba6
+  metadata.gz: 18f9a0673af8278a3066d865646525130986d0d6fce53f4a84fc8b10efb26e21
+  data.tar.gz: 1b782bf81e35136fc3b2fd540bcc97821f38707b6159977a0e6243bc51b33253
 SHA512:
-  metadata.gz: 3c11d993010e54b7ba55c7503fa00b331b0b34da815eac768955ec402bc436885327483dceac3b85a46da932002ad9be3049b3db2bac82eae20ed84900e5be53
-  data.tar.gz: 13d14d88704eb78936b534371ef132863b4db06403b68648b7e3a03a53da263a1a2e5641e6cc5d5678241270af4b94960de35301e5cfb0c1fc1f3062400f121a
+  metadata.gz: e91319bbb2301cbf8c9fb6900922854688faf4fac89b62c2ce9c5e1f5f36265e33cc3e5faa5dbfb581128c06ea406acc49be1b6c168c9920707b87aa28dffd72
+  data.tar.gz: ab38e61328b8e4b91f9354a2f23e74b98040d4477742ad86219cfc5de2c79e3ca36423574a13dbe2c6de1b5d45e1a36de594a36d8d9e29cd464a08a760986408

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 Unreleased Changes
 ------------------
 
+1.27.0 (2021-06-17)
+------------------
+
+* Feature - Amazon Kendra now supports the indexing of web documents for search through the web crawler.
+
 1.26.0 (2021-06-09)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-1.26.0
+1.27.0

data/lib/aws-sdk-kendra.rb

@@ -48,6 +48,6 @@ require_relative 'aws-sdk-kendra/customizations'
 # @!group service
 module Aws::Kendra
 
-  GEM_VERSION = '1.26.0'
+  GEM_VERSION = '1.27.0'
 
 end

data/lib/aws-sdk-kendra/client.rb

@@ -679,7 +679,7 @@ module Aws::Kendra
     #   resp = client.create_data_source({
     #     name: "DataSourceName", # required
     #     index_id: "IndexId", # required
-    #     type: "S3", # required, accepts S3, SHAREPOINT, DATABASE, SALESFORCE, ONEDRIVE, SERVICENOW, CUSTOM, CONFLUENCE, GOOGLEDRIVE
+    #     type: "S3", # required, accepts S3, SHAREPOINT, DATABASE, SALESFORCE, ONEDRIVE, SERVICENOW, CUSTOM, CONFLUENCE, GOOGLEDRIVE, WEBCRAWLER
     #     configuration: {
     #       s3_configuration: {
     #         bucket_name: "S3BucketName", # required
@@ -942,6 +942,37 @@ module Aws::Kendra
     #         exclude_user_accounts: ["UserAccount"],
     #         exclude_shared_drives: ["SharedDriveId"],
     #       },
+    #       web_crawler_configuration: {
+    #         urls: { # required
+    #           seed_url_configuration: {
+    #             seed_urls: ["SeedUrl"], # required
+    #             web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #           },
+    #           site_maps_configuration: {
+    #             site_maps: ["SiteMap"], # required
+    #           },
+    #         },
+    #         crawl_depth: 1,
+    #         max_links_per_page: 1,
+    #         max_content_size_per_page_in_mega_bytes: 1.0,
+    #         max_urls_per_minute_crawl_rate: 1,
+    #         url_inclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #         url_exclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #         proxy_configuration: {
+    #           host: "Host", # required
+    #           port: 1, # required
+    #           credentials: "SecretArn",
+    #         },
+    #         authentication_configuration: {
+    #           basic_authentication: [
+    #             {
+    #               host: "Host", # required
+    #               port: 1, # required
+    #               credentials: "SecretArn", # required
+    #             },
+    #           ],
+    #         },
+    #       },
     #     },
     #     description: "Description",
     #     schedule: "ScanSchedule",
@@ -1525,7 +1556,7 @@ module Aws::Kendra
     #   resp.id #=> String
     #   resp.index_id #=> String
     #   resp.name #=> String
-    #   resp.type #=> String, one of "S3", "SHAREPOINT", "DATABASE", "SALESFORCE", "ONEDRIVE", "SERVICENOW", "CUSTOM", "CONFLUENCE", "GOOGLEDRIVE"
+    #   resp.type #=> String, one of "S3", "SHAREPOINT", "DATABASE", "SALESFORCE", "ONEDRIVE", "SERVICENOW", "CUSTOM", "CONFLUENCE", "GOOGLEDRIVE", "WEBCRAWLER"
     #   resp.configuration.s3_configuration.bucket_name #=> String
     #   resp.configuration.s3_configuration.inclusion_prefixes #=> Array
     #   resp.configuration.s3_configuration.inclusion_prefixes[0] #=> String
@@ -1711,6 +1742,26 @@ module Aws::Kendra
     #   resp.configuration.google_drive_configuration.exclude_user_accounts[0] #=> String
     #   resp.configuration.google_drive_configuration.exclude_shared_drives #=> Array
     #   resp.configuration.google_drive_configuration.exclude_shared_drives[0] #=> String
+    #   resp.configuration.web_crawler_configuration.urls.seed_url_configuration.seed_urls #=> Array
+    #   resp.configuration.web_crawler_configuration.urls.seed_url_configuration.seed_urls[0] #=> String
+    #   resp.configuration.web_crawler_configuration.urls.seed_url_configuration.web_crawler_mode #=> String, one of "HOST_ONLY", "SUBDOMAINS", "EVERYTHING"
+    #   resp.configuration.web_crawler_configuration.urls.site_maps_configuration.site_maps #=> Array
+    #   resp.configuration.web_crawler_configuration.urls.site_maps_configuration.site_maps[0] #=> String
+    #   resp.configuration.web_crawler_configuration.crawl_depth #=> Integer
+    #   resp.configuration.web_crawler_configuration.max_links_per_page #=> Integer
+    #   resp.configuration.web_crawler_configuration.max_content_size_per_page_in_mega_bytes #=> Float
+    #   resp.configuration.web_crawler_configuration.max_urls_per_minute_crawl_rate #=> Integer
+    #   resp.configuration.web_crawler_configuration.url_inclusion_patterns #=> Array
+    #   resp.configuration.web_crawler_configuration.url_inclusion_patterns[0] #=> String
+    #   resp.configuration.web_crawler_configuration.url_exclusion_patterns #=> Array
+    #   resp.configuration.web_crawler_configuration.url_exclusion_patterns[0] #=> String
+    #   resp.configuration.web_crawler_configuration.proxy_configuration.host #=> String
+    #   resp.configuration.web_crawler_configuration.proxy_configuration.port #=> Integer
+    #   resp.configuration.web_crawler_configuration.proxy_configuration.credentials #=> String
+    #   resp.configuration.web_crawler_configuration.authentication_configuration.basic_authentication #=> Array
+    #   resp.configuration.web_crawler_configuration.authentication_configuration.basic_authentication[0].host #=> String
+    #   resp.configuration.web_crawler_configuration.authentication_configuration.basic_authentication[0].port #=> Integer
+    #   resp.configuration.web_crawler_configuration.authentication_configuration.basic_authentication[0].credentials #=> String
     #   resp.created_at #=> Time
     #   resp.updated_at #=> Time
     #   resp.description #=> String
@@ -2182,7 +2233,7 @@ module Aws::Kendra
     #   resp.summary_items #=> Array
     #   resp.summary_items[0].name #=> String
     #   resp.summary_items[0].id #=> String
-    #   resp.summary_items[0].type #=> String, one of "S3", "SHAREPOINT", "DATABASE", "SALESFORCE", "ONEDRIVE", "SERVICENOW", "CUSTOM", "CONFLUENCE", "GOOGLEDRIVE"
+    #   resp.summary_items[0].type #=> String, one of "S3", "SHAREPOINT", "DATABASE", "SALESFORCE", "ONEDRIVE", "SERVICENOW", "CUSTOM", "CONFLUENCE", "GOOGLEDRIVE", "WEBCRAWLER"
     #   resp.summary_items[0].created_at #=> Time
     #   resp.summary_items[0].updated_at #=> Time
     #   resp.summary_items[0].status #=> String, one of "CREATING", "DELETING", "FAILED", "UPDATING", "ACTIVE"
@@ -3180,6 +3231,37 @@ module Aws::Kendra
     #         exclude_user_accounts: ["UserAccount"],
     #         exclude_shared_drives: ["SharedDriveId"],
     #       },
+    #       web_crawler_configuration: {
+    #         urls: { # required
+    #           seed_url_configuration: {
+    #             seed_urls: ["SeedUrl"], # required
+    #             web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #           },
+    #           site_maps_configuration: {
+    #             site_maps: ["SiteMap"], # required
+    #           },
+    #         },
+    #         crawl_depth: 1,
+    #         max_links_per_page: 1,
+    #         max_content_size_per_page_in_mega_bytes: 1.0,
+    #         max_urls_per_minute_crawl_rate: 1,
+    #         url_inclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #         url_exclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #         proxy_configuration: {
+    #           host: "Host", # required
+    #           port: 1, # required
+    #           credentials: "SecretArn",
+    #         },
+    #         authentication_configuration: {
+    #           basic_authentication: [
+    #             {
+    #               host: "Host", # required
+    #               port: 1, # required
+    #               credentials: "SecretArn", # required
+    #             },
+    #           ],
+    #         },
+    #       },
     #     },
     #     description: "Description",
     #     schedule: "ScanSchedule",
@@ -3509,7 +3591,7 @@ module Aws::Kendra
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-kendra'
-      context[:gem_version] = '1.26.0'
+      context[:gem_version] = '1.27.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-kendra/client_api.rb

@@ -23,6 +23,9 @@ module Aws::Kendra
     AmazonResourceName = Shapes::StringShape.new(name: 'AmazonResourceName')
     AttributeFilter = Shapes::StructureShape.new(name: 'AttributeFilter')
     AttributeFilterList = Shapes::ListShape.new(name: 'AttributeFilterList')
+    AuthenticationConfiguration = Shapes::StructureShape.new(name: 'AuthenticationConfiguration')
+    BasicAuthenticationConfiguration = Shapes::StructureShape.new(name: 'BasicAuthenticationConfiguration')
+    BasicAuthenticationConfigurationList = Shapes::ListShape.new(name: 'BasicAuthenticationConfigurationList')
     BatchDeleteDocumentRequest = Shapes::StructureShape.new(name: 'BatchDeleteDocumentRequest')
     BatchDeleteDocumentResponse = Shapes::StructureShape.new(name: 'BatchDeleteDocumentResponse')
     BatchDeleteDocumentResponseFailedDocument = Shapes::StructureShape.new(name: 'BatchDeleteDocumentResponseFailedDocument')
@@ -69,6 +72,7 @@ module Aws::Kendra
     ConfluenceVersion = Shapes::StringShape.new(name: 'ConfluenceVersion')
     ConnectionConfiguration = Shapes::StructureShape.new(name: 'ConnectionConfiguration')
     ContentType = Shapes::StringShape.new(name: 'ContentType')
+    CrawlDepth = Shapes::IntegerShape.new(name: 'CrawlDepth')
     CreateDataSourceRequest = Shapes::StructureShape.new(name: 'CreateDataSourceRequest')
     CreateDataSourceResponse = Shapes::StructureShape.new(name: 'CreateDataSourceResponse')
     CreateFaqRequest = Shapes::StructureShape.new(name: 'CreateFaqRequest')
@@ -172,6 +176,7 @@ module Aws::Kendra
     Highlight = Shapes::StructureShape.new(name: 'Highlight')
     HighlightList = Shapes::ListShape.new(name: 'HighlightList')
     HighlightType = Shapes::StringShape.new(name: 'HighlightType')
+    Host = Shapes::StringShape.new(name: 'Host')
     Importance = Shapes::IntegerShape.new(name: 'Importance')
     IndexConfigurationSummary = Shapes::StructureShape.new(name: 'IndexConfigurationSummary')
     IndexConfigurationSummaryList = Shapes::ListShape.new(name: 'IndexConfigurationSummaryList')
@@ -206,12 +211,15 @@ module Aws::Kendra
     ListThesauriRequest = Shapes::StructureShape.new(name: 'ListThesauriRequest')
     ListThesauriResponse = Shapes::StructureShape.new(name: 'ListThesauriResponse')
     Long = Shapes::IntegerShape.new(name: 'Long')
+    MaxContentSizePerPageInMegaBytes = Shapes::FloatShape.new(name: 'MaxContentSizePerPageInMegaBytes')
+    MaxLinksPerPage = Shapes::IntegerShape.new(name: 'MaxLinksPerPage')
     MaxResultsIntegerForListDataSourceSyncJobsRequest = Shapes::IntegerShape.new(name: 'MaxResultsIntegerForListDataSourceSyncJobsRequest')
     MaxResultsIntegerForListDataSourcesRequest = Shapes::IntegerShape.new(name: 'MaxResultsIntegerForListDataSourcesRequest')
     MaxResultsIntegerForListFaqsRequest = Shapes::IntegerShape.new(name: 'MaxResultsIntegerForListFaqsRequest')
     MaxResultsIntegerForListIndicesRequest = Shapes::IntegerShape.new(name: 'MaxResultsIntegerForListIndicesRequest')
     MaxResultsIntegerForListQuerySuggestionsBlockLists = Shapes::IntegerShape.new(name: 'MaxResultsIntegerForListQuerySuggestionsBlockLists')
     MaxResultsIntegerForListThesauriRequest = Shapes::IntegerShape.new(name: 'MaxResultsIntegerForListThesauriRequest')
+    MaxUrlsPerMinuteCrawlRate = Shapes::IntegerShape.new(name: 'MaxUrlsPerMinuteCrawlRate')
     MetricValue = Shapes::StringShape.new(name: 'MetricValue')
     MimeType = Shapes::StringShape.new(name: 'MimeType')
     MinimumNumberOfQueryingUsers = Shapes::IntegerShape.new(name: 'MinimumNumberOfQueryingUsers')
@@ -224,10 +232,12 @@ module Aws::Kendra
     OneDriveUserList = Shapes::ListShape.new(name: 'OneDriveUserList')
     OneDriveUsers = Shapes::StructureShape.new(name: 'OneDriveUsers')
     Order = Shapes::StringShape.new(name: 'Order')
+    Port = Shapes::IntegerShape.new(name: 'Port')
     Principal = Shapes::StructureShape.new(name: 'Principal')
     PrincipalList = Shapes::ListShape.new(name: 'PrincipalList')
     PrincipalName = Shapes::StringShape.new(name: 'PrincipalName')
     PrincipalType = Shapes::StringShape.new(name: 'PrincipalType')
+    ProxyConfiguration = Shapes::StructureShape.new(name: 'ProxyConfiguration')
     QueryCapacityUnit = Shapes::IntegerShape.new(name: 'QueryCapacityUnit')
     QueryId = Shapes::StringShape.new(name: 'QueryId')
     QueryIdentifiersEnclosingOption = Shapes::StringShape.new(name: 'QueryIdentifiersEnclosingOption')
@@ -280,6 +290,9 @@ module Aws::Kendra
     Search = Shapes::StructureShape.new(name: 'Search')
     SecretArn = Shapes::StringShape.new(name: 'SecretArn')
     SecurityGroupIdList = Shapes::ListShape.new(name: 'SecurityGroupIdList')
+    SeedUrl = Shapes::StringShape.new(name: 'SeedUrl')
+    SeedUrlConfiguration = Shapes::StructureShape.new(name: 'SeedUrlConfiguration')
+    SeedUrlList = Shapes::ListShape.new(name: 'SeedUrlList')
     ServerSideEncryptionConfiguration = Shapes::StructureShape.new(name: 'ServerSideEncryptionConfiguration')
     ServiceNowAuthenticationType = Shapes::StringShape.new(name: 'ServiceNowAuthenticationType')
     ServiceNowBuildVersionType = Shapes::StringShape.new(name: 'ServiceNowBuildVersionType')
@@ -293,6 +306,9 @@ module Aws::Kendra
     SharePointUrlList = Shapes::ListShape.new(name: 'SharePointUrlList')
     SharePointVersion = Shapes::StringShape.new(name: 'SharePointVersion')
     SharedDriveId = Shapes::StringShape.new(name: 'SharedDriveId')
+    SiteMap = Shapes::StringShape.new(name: 'SiteMap')
+    SiteMapsConfiguration = Shapes::StructureShape.new(name: 'SiteMapsConfiguration')
+    SiteMapsList = Shapes::ListShape.new(name: 'SiteMapsList')
     SortOrder = Shapes::StringShape.new(name: 'SortOrder')
     SortingConfiguration = Shapes::StructureShape.new(name: 'SortingConfiguration')
     SqlConfiguration = Shapes::StructureShape.new(name: 'SqlConfiguration')
@@ -341,6 +357,7 @@ module Aws::Kendra
     UpdateQuerySuggestionsConfigRequest = Shapes::StructureShape.new(name: 'UpdateQuerySuggestionsConfigRequest')
     UpdateThesaurusRequest = Shapes::StructureShape.new(name: 'UpdateThesaurusRequest')
     Url = Shapes::StringShape.new(name: 'Url')
+    Urls = Shapes::StructureShape.new(name: 'Urls')
     UserAccount = Shapes::StringShape.new(name: 'UserAccount')
     UserContext = Shapes::StructureShape.new(name: 'UserContext')
     UserContextPolicy = Shapes::StringShape.new(name: 'UserContextPolicy')
@@ -352,6 +369,8 @@ module Aws::Kendra
     ValueImportanceMapKey = Shapes::StringShape.new(name: 'ValueImportanceMapKey')
     VisitorId = Shapes::StringShape.new(name: 'VisitorId')
     VpcSecurityGroupId = Shapes::StringShape.new(name: 'VpcSecurityGroupId')
+    WebCrawlerConfiguration = Shapes::StructureShape.new(name: 'WebCrawlerConfiguration')
+    WebCrawlerMode = Shapes::StringShape.new(name: 'WebCrawlerMode')
 
     AccessControlListConfiguration.add_member(:key_path, Shapes::ShapeRef.new(shape: S3ObjectKey, location_name: "KeyPath"))
     AccessControlListConfiguration.struct_class = Types::AccessControlListConfiguration
@@ -386,6 +405,16 @@ module Aws::Kendra
 
     AttributeFilterList.member = Shapes::ShapeRef.new(shape: AttributeFilter)
 
+    AuthenticationConfiguration.add_member(:basic_authentication, Shapes::ShapeRef.new(shape: BasicAuthenticationConfigurationList, location_name: "BasicAuthentication"))
+    AuthenticationConfiguration.struct_class = Types::AuthenticationConfiguration
+
+    BasicAuthenticationConfiguration.add_member(:host, Shapes::ShapeRef.new(shape: Host, required: true, location_name: "Host"))
+    BasicAuthenticationConfiguration.add_member(:port, Shapes::ShapeRef.new(shape: Port, required: true, location_name: "Port"))
+    BasicAuthenticationConfiguration.add_member(:credentials, Shapes::ShapeRef.new(shape: SecretArn, required: true, location_name: "Credentials"))
+    BasicAuthenticationConfiguration.struct_class = Types::BasicAuthenticationConfiguration
+
+    BasicAuthenticationConfigurationList.member = Shapes::ShapeRef.new(shape: BasicAuthenticationConfiguration)
+
     BatchDeleteDocumentRequest.add_member(:index_id, Shapes::ShapeRef.new(shape: IndexId, required: true, location_name: "IndexId"))
     BatchDeleteDocumentRequest.add_member(:document_id_list, Shapes::ShapeRef.new(shape: DocumentIdList, required: true, location_name: "DocumentIdList"))
     BatchDeleteDocumentRequest.add_member(:data_source_sync_job_metric_target, Shapes::ShapeRef.new(shape: DataSourceSyncJobMetricTarget, location_name: "DataSourceSyncJobMetricTarget"))
@@ -595,6 +624,7 @@ module Aws::Kendra
     DataSourceConfiguration.add_member(:service_now_configuration, Shapes::ShapeRef.new(shape: ServiceNowConfiguration, location_name: "ServiceNowConfiguration"))
     DataSourceConfiguration.add_member(:confluence_configuration, Shapes::ShapeRef.new(shape: ConfluenceConfiguration, location_name: "ConfluenceConfiguration"))
     DataSourceConfiguration.add_member(:google_drive_configuration, Shapes::ShapeRef.new(shape: GoogleDriveConfiguration, location_name: "GoogleDriveConfiguration"))
+    DataSourceConfiguration.add_member(:web_crawler_configuration, Shapes::ShapeRef.new(shape: WebCrawlerConfiguration, location_name: "WebCrawlerConfiguration"))
     DataSourceConfiguration.struct_class = Types::DataSourceConfiguration
 
     DataSourceInclusionsExclusionsStrings.member = Shapes::ShapeRef.new(shape: DataSourceInclusionsExclusionsStringsMember)
@@ -1007,6 +1037,11 @@ module Aws::Kendra
 
     PrincipalList.member = Shapes::ShapeRef.new(shape: Principal)
 
+    ProxyConfiguration.add_member(:host, Shapes::ShapeRef.new(shape: Host, required: true, location_name: "Host"))
+    ProxyConfiguration.add_member(:port, Shapes::ShapeRef.new(shape: Port, required: true, location_name: "Port"))
+    ProxyConfiguration.add_member(:credentials, Shapes::ShapeRef.new(shape: SecretArn, location_name: "Credentials"))
+    ProxyConfiguration.struct_class = Types::ProxyConfiguration
+
     QueryRequest.add_member(:index_id, Shapes::ShapeRef.new(shape: IndexId, required: true, location_name: "IndexId"))
     QueryRequest.add_member(:query_text, Shapes::ShapeRef.new(shape: QueryText, required: true, location_name: "QueryText"))
     QueryRequest.add_member(:attribute_filter, Shapes::ShapeRef.new(shape: AttributeFilter, location_name: "AttributeFilter"))
@@ -1150,6 +1185,12 @@ module Aws::Kendra
 
     SecurityGroupIdList.member = Shapes::ShapeRef.new(shape: VpcSecurityGroupId)
 
+    SeedUrlConfiguration.add_member(:seed_urls, Shapes::ShapeRef.new(shape: SeedUrlList, required: true, location_name: "SeedUrls"))
+    SeedUrlConfiguration.add_member(:web_crawler_mode, Shapes::ShapeRef.new(shape: WebCrawlerMode, location_name: "WebCrawlerMode"))
+    SeedUrlConfiguration.struct_class = Types::SeedUrlConfiguration
+
+    SeedUrlList.member = Shapes::ShapeRef.new(shape: SeedUrl)
+
     ServerSideEncryptionConfiguration.add_member(:kms_key_id, Shapes::ShapeRef.new(shape: KmsKeyId, location_name: "KmsKeyId"))
     ServerSideEncryptionConfiguration.struct_class = Types::ServerSideEncryptionConfiguration
 
@@ -1196,6 +1237,11 @@ module Aws::Kendra
 
     SharePointUrlList.member = Shapes::ShapeRef.new(shape: Url)
 
+    SiteMapsConfiguration.add_member(:site_maps, Shapes::ShapeRef.new(shape: SiteMapsList, required: true, location_name: "SiteMaps"))
+    SiteMapsConfiguration.struct_class = Types::SiteMapsConfiguration
+
+    SiteMapsList.member = Shapes::ShapeRef.new(shape: SiteMap)
+
     SortingConfiguration.add_member(:document_attribute_key, Shapes::ShapeRef.new(shape: DocumentAttributeKey, required: true, location_name: "DocumentAttributeKey"))
     SortingConfiguration.add_member(:sort_order, Shapes::ShapeRef.new(shape: SortOrder, required: true, location_name: "SortOrder"))
     SortingConfiguration.struct_class = Types::SortingConfiguration
@@ -1334,6 +1380,10 @@ module Aws::Kendra
     UpdateThesaurusRequest.add_member(:source_s3_path, Shapes::ShapeRef.new(shape: S3Path, location_name: "SourceS3Path"))
     UpdateThesaurusRequest.struct_class = Types::UpdateThesaurusRequest
 
+    Urls.add_member(:seed_url_configuration, Shapes::ShapeRef.new(shape: SeedUrlConfiguration, location_name: "SeedUrlConfiguration"))
+    Urls.add_member(:site_maps_configuration, Shapes::ShapeRef.new(shape: SiteMapsConfiguration, location_name: "SiteMapsConfiguration"))
+    Urls.struct_class = Types::Urls
+
     UserContext.add_member(:token, Shapes::ShapeRef.new(shape: Token, location_name: "Token"))
     UserContext.struct_class = Types::UserContext
 
@@ -1349,6 +1399,17 @@ module Aws::Kendra
     ValueImportanceMap.key = Shapes::ShapeRef.new(shape: ValueImportanceMapKey)
     ValueImportanceMap.value = Shapes::ShapeRef.new(shape: Importance)
 
+    WebCrawlerConfiguration.add_member(:urls, Shapes::ShapeRef.new(shape: Urls, required: true, location_name: "Urls"))
+    WebCrawlerConfiguration.add_member(:crawl_depth, Shapes::ShapeRef.new(shape: CrawlDepth, location_name: "CrawlDepth"))
+    WebCrawlerConfiguration.add_member(:max_links_per_page, Shapes::ShapeRef.new(shape: MaxLinksPerPage, location_name: "MaxLinksPerPage"))
+    WebCrawlerConfiguration.add_member(:max_content_size_per_page_in_mega_bytes, Shapes::ShapeRef.new(shape: MaxContentSizePerPageInMegaBytes, location_name: "MaxContentSizePerPageInMegaBytes"))
+    WebCrawlerConfiguration.add_member(:max_urls_per_minute_crawl_rate, Shapes::ShapeRef.new(shape: MaxUrlsPerMinuteCrawlRate, location_name: "MaxUrlsPerMinuteCrawlRate"))
+    WebCrawlerConfiguration.add_member(:url_inclusion_patterns, Shapes::ShapeRef.new(shape: DataSourceInclusionsExclusionsStrings, location_name: "UrlInclusionPatterns"))
+    WebCrawlerConfiguration.add_member(:url_exclusion_patterns, Shapes::ShapeRef.new(shape: DataSourceInclusionsExclusionsStrings, location_name: "UrlExclusionPatterns"))
+    WebCrawlerConfiguration.add_member(:proxy_configuration, Shapes::ShapeRef.new(shape: ProxyConfiguration, location_name: "ProxyConfiguration"))
+    WebCrawlerConfiguration.add_member(:authentication_configuration, Shapes::ShapeRef.new(shape: AuthenticationConfiguration, location_name: "AuthenticationConfiguration"))
+    WebCrawlerConfiguration.struct_class = Types::WebCrawlerConfiguration
+
 
     # @api private
     API = Seahorse::Model::Api.new.tap do |api|

data/lib/aws-sdk-kendra/types.rb

@@ -492,6 +492,87 @@ module Aws::Kendra
       include Aws::Structure
     end
 
+    # Provides the configuration information to connect to websites that
+    # require user authentication.
+    #
+    # @note When making an API call, you may pass AuthenticationConfiguration
+    #   data as a hash:
+    #
+    #       {
+    #         basic_authentication: [
+    #           {
+    #             host: "Host", # required
+    #             port: 1, # required
+    #             credentials: "SecretArn", # required
+    #           },
+    #         ],
+    #       }
+    #
+    # @!attribute [rw] basic_authentication
+    #   The list of configuration information that's required to connect to
+    #   and crawl a website host using basic authentication credentials.
+    #
+    #   The list includes the name and port number of the website host.
+    #   @return [Array<Types::BasicAuthenticationConfiguration>]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/AuthenticationConfiguration AWS API Documentation
+    #
+    class AuthenticationConfiguration < Struct.new(
+      :basic_authentication)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
+    # Provides the configuration information to connect to websites that
+    # require basic user authentication.
+    #
+    # @note When making an API call, you may pass BasicAuthenticationConfiguration
+    #   data as a hash:
+    #
+    #       {
+    #         host: "Host", # required
+    #         port: 1, # required
+    #         credentials: "SecretArn", # required
+    #       }
+    #
+    # @!attribute [rw] host
+    #   The name of the website host you want to connect to using
+    #   authentication credentials.
+    #
+    #   For example, the host name of https://a.example.com/page1.html is
+    #   "a.example.com".
+    #   @return [String]
+    #
+    # @!attribute [rw] port
+    #   The port number of the website host you want to connect to using
+    #   authentication credentials.
+    #
+    #   For example, the port for https://a.example.com/page1.html is 443,
+    #   the standard port for HTTPS.
+    #   @return [Integer]
+    #
+    # @!attribute [rw] credentials
+    #   Your secret ARN, which you can create in [AWS Secrets Manager][1]
+    #
+    #   You use a secret if basic authentication credentials are required to
+    #   connect to a website. The secret stores your credentials of user
+    #   name and password.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html
+    #   @return [String]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/BasicAuthenticationConfiguration AWS API Documentation
+    #
+    class BasicAuthenticationConfiguration < Struct.new(
+      :host,
+      :port,
+      :credentials)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
     # @note When making an API call, you may pass BatchDeleteDocumentRequest
     #   data as a hash:
     #
@@ -801,8 +882,9 @@ module Aws::Kendra
       include Aws::Structure
     end
 
-    # Specifies capacity units configured for your index. You can add and
-    # remove capacity units to tune an index to your requirements.
+    # Specifies capacity units configured for your enterprise edition index.
+    # You can add and remove capacity units to tune an index to your
+    # requirements.
     #
     # @note When making an API call, you may pass CapacityUnitsConfiguration
     #   data as a hash:
@@ -813,14 +895,28 @@ module Aws::Kendra
     #       }
     #
     # @!attribute [rw] storage_capacity_units
-    #   The amount of extra storage capacity for an index. Each capacity
-    #   unit provides 150 Gb of storage space or 500,000 documents,
-    #   whichever is reached first.
+    #   The amount of extra storage capacity for an index. A single capacity
+    #   unit for an index provides 150 GB of storage space or 500,000
+    #   documents, whichever is reached first.
     #   @return [Integer]
     #
     # @!attribute [rw] query_capacity_units
-    #   The amount of extra query capacity for an index. Each capacity unit
-    #   provides 0.5 queries per second and 40,000 queries per day.
+    #   The amount of extra query capacity for an index and
+    #   [GetQuerySuggestions][1] capacity.
+    #
+    #   A single extra capacity unit for an index provides 0.5 queries per
+    #   second or approximately 40,000 queries per day.
+    #
+    #   `GetQuerySuggestions` capacity is 5 times the provisioned query
+    #   capacity for an index. For example, the base capacity for an index
+    #   is 0.5 queries per second, so GetQuerySuggestions capacity is 2.5
+    #   calls per second. If adding another 0.5 queries per second to total
+    #   1 queries per second for an index, the `GetQuerySuggestions`
+    #   capacity is 5 calls per second.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/kendra/latest/dg/API_GetQuerySuggestions.html
     #   @return [Integer]
     #
     # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/CapacityUnitsConfiguration AWS API Documentation
@@ -1499,7 +1595,7 @@ module Aws::Kendra
     #       {
     #         name: "DataSourceName", # required
     #         index_id: "IndexId", # required
-    #         type: "S3", # required, accepts S3, SHAREPOINT, DATABASE, SALESFORCE, ONEDRIVE, SERVICENOW, CUSTOM, CONFLUENCE, GOOGLEDRIVE
+    #         type: "S3", # required, accepts S3, SHAREPOINT, DATABASE, SALESFORCE, ONEDRIVE, SERVICENOW, CUSTOM, CONFLUENCE, GOOGLEDRIVE, WEBCRAWLER
     #         configuration: {
     #           s3_configuration: {
     #             bucket_name: "S3BucketName", # required
@@ -1762,6 +1858,37 @@ module Aws::Kendra
     #             exclude_user_accounts: ["UserAccount"],
     #             exclude_shared_drives: ["SharedDriveId"],
     #           },
+    #           web_crawler_configuration: {
+    #             urls: { # required
+    #               seed_url_configuration: {
+    #                 seed_urls: ["SeedUrl"], # required
+    #                 web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #               },
+    #               site_maps_configuration: {
+    #                 site_maps: ["SiteMap"], # required
+    #               },
+    #             },
+    #             crawl_depth: 1,
+    #             max_links_per_page: 1,
+    #             max_content_size_per_page_in_mega_bytes: 1.0,
+    #             max_urls_per_minute_crawl_rate: 1,
+    #             url_inclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #             url_exclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #             proxy_configuration: {
+    #               host: "Host", # required
+    #               port: 1, # required
+    #               credentials: "SecretArn",
+    #             },
+    #             authentication_configuration: {
+    #               basic_authentication: [
+    #                 {
+    #                   host: "Host", # required
+    #                   port: 1, # required
+    #                   credentials: "SecretArn", # required
+    #                 },
+    #               ],
+    #             },
+    #           },
     #         },
     #         description: "Description",
     #         schedule: "ScanSchedule",
@@ -2570,6 +2697,37 @@ module Aws::Kendra
     #           exclude_user_accounts: ["UserAccount"],
     #           exclude_shared_drives: ["SharedDriveId"],
     #         },
+    #         web_crawler_configuration: {
+    #           urls: { # required
+    #             seed_url_configuration: {
+    #               seed_urls: ["SeedUrl"], # required
+    #               web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #             },
+    #             site_maps_configuration: {
+    #               site_maps: ["SiteMap"], # required
+    #             },
+    #           },
+    #           crawl_depth: 1,
+    #           max_links_per_page: 1,
+    #           max_content_size_per_page_in_mega_bytes: 1.0,
+    #           max_urls_per_minute_crawl_rate: 1,
+    #           url_inclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #           url_exclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #           proxy_configuration: {
+    #             host: "Host", # required
+    #             port: 1, # required
+    #             credentials: "SecretArn",
+    #           },
+    #           authentication_configuration: {
+    #             basic_authentication: [
+    #               {
+    #                 host: "Host", # required
+    #                 port: 1, # required
+    #                 credentials: "SecretArn", # required
+    #               },
+    #             ],
+    #           },
+    #         },
     #       }
     #
     # @!attribute [rw] s3_configuration
@@ -2612,6 +2770,11 @@ module Aws::Kendra
     #   Drive.
     #   @return [Types::GoogleDriveConfiguration]
     #
+    # @!attribute [rw] web_crawler_configuration
+    #   Provides the configuration information required for Amazon Kendra
+    #   web crawler.
+    #   @return [Types::WebCrawlerConfiguration]
+    #
     # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/DataSourceConfiguration AWS API Documentation
     #
     class DataSourceConfiguration < Struct.new(
@@ -2622,7 +2785,8 @@ module Aws::Kendra
       :one_drive_configuration,
       :service_now_configuration,
       :confluence_configuration,
-      :google_drive_configuration)
+      :google_drive_configuration,
+      :web_crawler_configuration)
       SENSITIVE = []
       include Aws::Structure
     end
@@ -5048,6 +5212,57 @@ module Aws::Kendra
       include Aws::Structure
     end
 
+    # Provides the configuration information for a web proxy to connect to
+    # website hosts.
+    #
+    # @note When making an API call, you may pass ProxyConfiguration
+    #   data as a hash:
+    #
+    #       {
+    #         host: "Host", # required
+    #         port: 1, # required
+    #         credentials: "SecretArn",
+    #       }
+    #
+    # @!attribute [rw] host
+    #   The name of the website host you want to connect to via a web proxy
+    #   server.
+    #
+    #   For example, the host name of https://a.example.com/page1.html is
+    #   "a.example.com".
+    #   @return [String]
+    #
+    # @!attribute [rw] port
+    #   The port number of the website host you want to connect to via a web
+    #   proxy server.
+    #
+    #   For example, the port for https://a.example.com/page1.html is 443,
+    #   the standard port for HTTPS.
+    #   @return [Integer]
+    #
+    # @!attribute [rw] credentials
+    #   Your secret ARN, which you can create in [AWS Secrets Manager][1]
+    #
+    #   The credentials are optional. You use a secret if web proxy
+    #   credentials are required to connect to a website host. Amazon Kendra
+    #   currently support basic authentication to connect to a web proxy
+    #   server. The secret stores your credentials.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html
+    #   @return [String]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/ProxyConfiguration AWS API Documentation
+    #
+    class ProxyConfiguration < Struct.new(
+      :host,
+      :port,
+      :credentials)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
     # @note When making an API call, you may pass QueryRequest
     #   data as a hash:
     #
@@ -6253,6 +6468,59 @@ module Aws::Kendra
       include Aws::Structure
     end
 
+    # Provides the configuration information of the seed or starting point
+    # URLs to crawl.
+    #
+    # *When selecting websites to index, you must adhere to the [Amazon
+    # Acceptable Use Policy][1] and all other Amazon terms. Remember that
+    # you must only use the Amazon Kendra web crawler to index your own
+    # webpages, or webpages that you have authorization to index.*
+    #
+    #
+    #
+    # [1]: https://aws.amazon.com/aup/
+    #
+    # @note When making an API call, you may pass SeedUrlConfiguration
+    #   data as a hash:
+    #
+    #       {
+    #         seed_urls: ["SeedUrl"], # required
+    #         web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #       }
+    #
+    # @!attribute [rw] seed_urls
+    #   The list of seed or starting point URLs of the websites you want to
+    #   crawl.
+    #
+    #   The list can include a maximum of 100 seed URLs.
+    #   @return [Array<String>]
+    #
+    # @!attribute [rw] web_crawler_mode
+    #   You can choose one of the following modes:
+    #
+    #   * `HOST_ONLY` – crawl only the website host names. For example, if
+    #     the seed URL is "abc.example.com", then only URLs with host name
+    #     "abc.example.com" are crawled.
+    #
+    #   * `SUBDOMAINS` – crawl the website host names with subdomains. For
+    #     example, if the seed URL is "abc.example.com", then
+    #     "a.abc.example.com" and "b.abc.example.com" are also crawled.
+    #
+    #   * `EVERYTHING` – crawl the website host names with subdomains and
+    #     other domains that the webpages link to.
+    #
+    #   The default mode is set to `HOST_ONLY`.
+    #   @return [String]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/SeedUrlConfiguration AWS API Documentation
+    #
+    class SeedUrlConfiguration < Struct.new(
+      :seed_urls,
+      :web_crawler_mode)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
     # Provides the identifier of the AWS KMS customer master key (CMK) used
     # to encrypt data indexed by Amazon Kendra. Amazon Kendra doesn't
     # support asymmetric CMKs.
@@ -6675,6 +6943,38 @@ module Aws::Kendra
       include Aws::Structure
     end
 
+    # Provides the configuration information of the sitemap URLs to crawl.
+    #
+    # *When selecting websites to index, you must adhere to the [Amazon
+    # Acceptable Use Policy][1] and all other Amazon terms. Remember that
+    # you must only use the Amazon Kendra web crawler to index your own
+    # webpages, or webpages that you have authorization to index.*
+    #
+    #
+    #
+    # [1]: https://aws.amazon.com/aup/
+    #
+    # @note When making an API call, you may pass SiteMapsConfiguration
+    #   data as a hash:
+    #
+    #       {
+    #         site_maps: ["SiteMap"], # required
+    #       }
+    #
+    # @!attribute [rw] site_maps
+    #   The list of sitemap URLs of the websites you want to crawl.
+    #
+    #   The list can include a maximum of three sitemap URLs.
+    #   @return [Array<String>]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/SiteMapsConfiguration AWS API Documentation
+    #
+    class SiteMapsConfiguration < Struct.new(
+      :site_maps)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
     # Specifies the document attribute to use to sort the response to a
     # Amazon Kendra query. You can specify a single attribute for sorting.
     # The attribute must have the `Sortable` flag set to `true`, otherwise
@@ -7476,6 +7776,37 @@ module Aws::Kendra
     #             exclude_user_accounts: ["UserAccount"],
     #             exclude_shared_drives: ["SharedDriveId"],
     #           },
+    #           web_crawler_configuration: {
+    #             urls: { # required
+    #               seed_url_configuration: {
+    #                 seed_urls: ["SeedUrl"], # required
+    #                 web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #               },
+    #               site_maps_configuration: {
+    #                 site_maps: ["SiteMap"], # required
+    #               },
+    #             },
+    #             crawl_depth: 1,
+    #             max_links_per_page: 1,
+    #             max_content_size_per_page_in_mega_bytes: 1.0,
+    #             max_urls_per_minute_crawl_rate: 1,
+    #             url_inclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #             url_exclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #             proxy_configuration: {
+    #               host: "Host", # required
+    #               port: 1, # required
+    #               credentials: "SecretArn",
+    #             },
+    #             authentication_configuration: {
+    #               basic_authentication: [
+    #                 {
+    #                   host: "Host", # required
+    #                   port: 1, # required
+    #                   credentials: "SecretArn", # required
+    #                 },
+    #               ],
+    #             },
+    #           },
     #         },
     #         description: "Description",
     #         schedule: "ScanSchedule",
@@ -7838,6 +8169,58 @@ module Aws::Kendra
       include Aws::Structure
     end
 
+    # Provides the configuration information of the URLs to crawl.
+    #
+    # *When selecting websites to index, you must adhere to the [Amazon
+    # Acceptable Use Policy][1] and all other Amazon terms. Remember that
+    # you must only use the Amazon Kendra web crawler to index your own
+    # webpages, or webpages that you have authorization to index.*
+    #
+    #
+    #
+    # [1]: https://aws.amazon.com/aup/
+    #
+    # @note When making an API call, you may pass Urls
+    #   data as a hash:
+    #
+    #       {
+    #         seed_url_configuration: {
+    #           seed_urls: ["SeedUrl"], # required
+    #           web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #         },
+    #         site_maps_configuration: {
+    #           site_maps: ["SiteMap"], # required
+    #         },
+    #       }
+    #
+    # @!attribute [rw] seed_url_configuration
+    #   Provides the configuration of the seed or starting point URLs of the
+    #   websites you want to crawl.
+    #
+    #   You can choose to crawl only the website host names, or the website
+    #   host names with subdomains, or the website host names with
+    #   subdomains and other domains that the webpages link to.
+    #
+    #   You can list up to 100 seed URLs.
+    #   @return [Types::SeedUrlConfiguration]
+    #
+    # @!attribute [rw] site_maps_configuration
+    #   Provides the configuration of the sitemap URLs of the websites you
+    #   want to crawl.
+    #
+    #   Only URLs belonging to the same website host names are crawled. You
+    #   can list up to three sitemap URLs.
+    #   @return [Types::SiteMapsConfiguration]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/Urls AWS API Documentation
+    #
+    class Urls < Struct.new(
+      :seed_url_configuration,
+      :site_maps_configuration)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
     # Provides information about the user context for a Amazon Kendra index.
     #
     # @note When making an API call, you may pass UserContext
@@ -7908,5 +8291,167 @@ module Aws::Kendra
       include Aws::Structure
     end
 
+    # Provides the configuration information required for Amazon Kendra web
+    # crawler.
+    #
+    # @note When making an API call, you may pass WebCrawlerConfiguration
+    #   data as a hash:
+    #
+    #       {
+    #         urls: { # required
+    #           seed_url_configuration: {
+    #             seed_urls: ["SeedUrl"], # required
+    #             web_crawler_mode: "HOST_ONLY", # accepts HOST_ONLY, SUBDOMAINS, EVERYTHING
+    #           },
+    #           site_maps_configuration: {
+    #             site_maps: ["SiteMap"], # required
+    #           },
+    #         },
+    #         crawl_depth: 1,
+    #         max_links_per_page: 1,
+    #         max_content_size_per_page_in_mega_bytes: 1.0,
+    #         max_urls_per_minute_crawl_rate: 1,
+    #         url_inclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #         url_exclusion_patterns: ["DataSourceInclusionsExclusionsStringsMember"],
+    #         proxy_configuration: {
+    #           host: "Host", # required
+    #           port: 1, # required
+    #           credentials: "SecretArn",
+    #         },
+    #         authentication_configuration: {
+    #           basic_authentication: [
+    #             {
+    #               host: "Host", # required
+    #               port: 1, # required
+    #               credentials: "SecretArn", # required
+    #             },
+    #           ],
+    #         },
+    #       }
+    #
+    # @!attribute [rw] urls
+    #   Specifies the seed or starting point URLs of the websites or the
+    #   sitemap URLs of the websites you want to crawl.
+    #
+    #   You can include website subdomains. You can list up to 100 seed URLs
+    #   and up to three sitemap URLs.
+    #
+    #   *When selecting websites to index, you must adhere to the [Amazon
+    #   Acceptable Use Policy][1] and all other Amazon terms. Remember that
+    #   you must only use the Amazon Kendra web crawler to index your own
+    #   webpages, or webpages that you have authorization to index.*
+    #
+    #
+    #
+    #   [1]: https://aws.amazon.com/aup/
+    #   @return [Types::Urls]
+    #
+    # @!attribute [rw] crawl_depth
+    #   Specifies the number of levels in a website that you want to crawl.
+    #
+    #   The first level begins from the website seed or starting point URL.
+    #   For example, if a website has 3 levels – index level (i.e. seed in
+    #   this example), sections level, and subsections level – and you are
+    #   only interested in crawling information up to the sections level
+    #   (i.e. levels 0-1), you can set your depth to 1.
+    #
+    #   The default crawl depth is set to 2.
+    #   @return [Integer]
+    #
+    # @!attribute [rw] max_links_per_page
+    #   The maximum number of URLs on a webpage to include when crawling a
+    #   website. This number is per webpage.
+    #
+    #   As a website’s webpages are crawled, any URLs the webpages link to
+    #   are also crawled. URLs on a webpage are crawled in order of
+    #   appearance.
+    #
+    #   The default maximum links per page is 100.
+    #   @return [Integer]
+    #
+    # @!attribute [rw] max_content_size_per_page_in_mega_bytes
+    #   The maximum size (in MB) of a webpage or attachment to crawl.
+    #
+    #   Files larger than this size (in MB) are skipped/not crawled.
+    #
+    #   The default maximum size of a webpage or attachment is set to 50 MB.
+    #   @return [Float]
+    #
+    # @!attribute [rw] max_urls_per_minute_crawl_rate
+    #   The maximum number of URLs crawled per website host per minute.
+    #
+    #   A minimum of one URL is required.
+    #
+    #   The default maximum number of URLs crawled per website host per
+    #   minute is 300.
+    #   @return [Integer]
+    #
+    # @!attribute [rw] url_inclusion_patterns
+    #   The regular expression pattern to include certain URLs to crawl.
+    #
+    #   If there is a regular expression pattern to exclude certain URLs
+    #   that conflicts with the include pattern, the exclude pattern takes
+    #   precedence.
+    #   @return [Array<String>]
+    #
+    # @!attribute [rw] url_exclusion_patterns
+    #   The regular expression pattern to exclude certain URLs to crawl.
+    #
+    #   If there is a regular expression pattern to include certain URLs
+    #   that conflicts with the exclude pattern, the exclude pattern takes
+    #   precedence.
+    #   @return [Array<String>]
+    #
+    # @!attribute [rw] proxy_configuration
+    #   Provides configuration information required to connect to your
+    #   internal websites via a web proxy.
+    #
+    #   You must provide the website host name and port number. For example,
+    #   the host name of https://a.example.com/page1.html is
+    #   "a.example.com" and the port is 443, the standard port for HTTPS.
+    #
+    #   Web proxy credentials are optional and you can use them to connect
+    #   to a web proxy server that requires basic authentication. To store
+    #   web proxy credentials, you use a secret in [AWS Secrets Manager][1].
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html
+    #   @return [Types::ProxyConfiguration]
+    #
+    # @!attribute [rw] authentication_configuration
+    #   Provides configuration information required to connect to websites
+    #   using authentication.
+    #
+    #   You can connect to websites using basic authentication of user name
+    #   and password.
+    #
+    #   You must provide the website host name and port number. For example,
+    #   the host name of https://a.example.com/page1.html is
+    #   "a.example.com" and the port is 443, the standard port for HTTPS.
+    #   You use a secret in [AWS Secrets Manager][1] to store your
+    #   authentication credentials.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html
+    #   @return [Types::AuthenticationConfiguration]
+    #
+    # @see http://docs.aws.amazon.com/goto/WebAPI/kendra-2019-02-03/WebCrawlerConfiguration AWS API Documentation
+    #
+    class WebCrawlerConfiguration < Struct.new(
+      :urls,
+      :crawl_depth,
+      :max_links_per_page,
+      :max_content_size_per_page_in_mega_bytes,
+      :max_urls_per_minute_crawl_rate,
+      :url_inclusion_patterns,
+      :url_exclusion_patterns,
+      :proxy_configuration,
+      :authentication_configuration)
+      SENSITIVE = []
+      include Aws::Structure
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-kendra
 version: !ruby/object:Gem::Version
-  version: 1.26.0
+  version: 1.27.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-09 00:00:00.000000000 Z
+date: 2021-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core