elastic-enterprise-search 7.14.1 → 7.15.0

data/spec/integration/app-search/crawler_entry_point_spec.rb

@@ -0,0 +1,89 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler entry point' do
+    let(:engine_name) { 'crawler-entry-point' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      attempts = 0
+      loop do
+        attempts += 1
+        client.create_engine(name: engine_name)
+        break if attempts > 9
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e
+        if e.message.match?(/Name is already taken/)
+          sleep 1
+          next
+        end
+      else
+        break
+      end
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'creates an entry point' do
+      value = '/enterprise-search'
+      body = { value: value }
+      response = client.create_crawler_entry_point(engine_name, domain_id: @domain['id'], body: body)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['id', 'value', 'created_at'])
+      expect(response.body['value']).to eq value
+    end
+
+    it 'updates an entry point' do
+      value = '/elastic-stack'
+      body = { value: value }
+      @entry_point = client.create_crawler_entry_point(engine_name, domain_id: @domain['id'], body: body).body
+
+      response = client.put_crawler_entry_point(
+        engine_name,
+        domain_id: @domain['id'],
+        entry_point_id: @entry_point['id'],
+        body: body
+      )
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['id', 'value', 'created_at'])
+      expect(response.body['value']).to eq value
+    end
+
+    it 'deletes an entry point' do
+      value = '/security'
+      body = { value: value }
+      @entry_point = client.create_crawler_entry_point(engine_name, domain_id: @domain['id'], body: body).body
+
+      response = client.delete_crawler_entry_point(
+        engine_name,
+        domain_id: @domain['id'],
+        entry_point_id: @entry_point['id']
+      )
+      expect(response.status).to eq 200
+      expect(response.body).to eq({ 'deleted' => true })
+    end
+  end
+end

data/spec/integration/app-search/crawler_metrics_spec.rb

@@ -0,0 +1,46 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler metrics' do
+    let(:engine_name) { 'crawler-metrics' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'retrieves crawler metrics' do
+      response = client.crawler_metrics
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['global', 'node'])
+      expect(response.body.dig('global', 'crawl_requests'))
+      expect(response.body.dig('node', 'workers'))
+    end
+  end
+end

data/spec/integration/app-search/crawler_overview_spec.rb

@@ -0,0 +1,45 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler overview' do
+    let(:engine_name) { 'crawler-overview' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'retrieves crawler configuration overview' do
+      response = client.crawler_overview(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body['domains'].count).to eq 1
+      expect(response.body['domains'].first['name']).to eq name
+    end
+  end
+end

data/spec/integration/app-search/crawler_process_crawl_denied_urls_spec.rb

@@ -0,0 +1,50 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler process crawl denied URLs' do
+    let(:engine_name) { 'crawler-process-crawl-denied-urls' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'retrieves denied urls for process crawl' do
+      response = client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+      id = response.body['id']
+
+      response = client.crawler_process_crawl_denied_urls(engine_name, process_crawl_id: id)
+      expect(response.status).to eq 200
+      expect(
+        response.body.keys &
+        ['total_url_count', 'denied_url_count', 'sample_size', 'denied_urls_sample']
+      ).to eq response.body.keys
+    end
+  end
+end

data/spec/integration/app-search/crawler_process_crawl_spec.rb

@@ -0,0 +1,66 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler Process Crawl' do
+    let(:engine_name) { 'crawler-urls' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+      sleep 1
+    end
+
+    it 'creates, retrieves and shows denied for a process crawl' do
+      response = client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+      expect(response.status).to eq 200
+
+      id = response.body['id']
+      expect(response.body['dry_run']).to eq true
+      expect(response.body['domains']).to eq(['https://www.elastic.co'])
+
+      response = client.crawler_process_crawl(engine_name, process_crawl_id: id)
+      expect(response.status).to eq 200
+      expect(response.body['dry_run']).to eq true
+      expect(response.body['domains']).to eq(['https://www.elastic.co'])
+
+      response = client.denied_urls(engine_name, process_crawl_id: id)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['total_url_count', 'denied_url_count', 'sample_size', 'denied_urls_sample'])
+    end
+
+    it 'lists process crawls' do
+      client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+      client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+
+      response = client.list_crawler_process_crawls(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body.count).to be 2
+    end
+  end
+end

data/spec/integration/app-search/crawler_scheduling_spec.rb

@@ -0,0 +1,81 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler scheduling' do
+    let(:engine_name) { 'crawler-scheduling' }
+    let(:body) { { frequency: 1, unit: 'day' } }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      attempts = 0
+      loop do
+        attempts += 1
+        client.create_engine(name: engine_name)
+        break if attempts > 9
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e
+        if e.message.match?(/Name is already taken/)
+          sleep 1
+          next
+        end
+      else
+        break
+      end
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'sets a crawl schedule' do
+      response = client.put_crawler_crawl_schedule(engine_name, body: body)
+      expect(response.status).to eq 200
+      expect(response.body).to eq({
+                                    'engine' => engine_name,
+                                    'frequency' => 1,
+                                    'unit' => 'day'
+                                  })
+    end
+
+    it 'gets the crawl schedule' do
+      client.put_crawler_crawl_schedule(engine_name, body: body)
+
+      response = client.crawler_crawl_schedule(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body).to eq({
+                                    'engine' => engine_name,
+                                    'frequency' => 1,
+                                    'unit' => 'day'
+                                  })
+    end
+
+    it 'deletes a crawler crawl schedule' do
+      client.put_crawler_crawl_schedule(engine_name, body: body)
+
+      response = client.delete_crawler_crawl_schedule(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body).to eq({ 'deleted' => true })
+    end
+  end
+end

data/spec/integration/app-search/crawler_sitemap_spec.rb

@@ -0,0 +1,72 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler Sitemap' do
+    let(:engine_name) { 'crawler-sitemap' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'creates, updates and deletes a crawler sitemap configuration' do
+      # Create a sitemap
+      sitemap_url = 'https://www.elastic.co/sitemap.xml'
+      body = { url: sitemap_url }
+      response = client.create_crawler_sitemap(engine_name, domain_id: @domain['id'], body: body)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['id', 'url', 'created_at'])
+      expect(response.body['url']).to eq(sitemap_url)
+
+      # Update a sitemap
+      sitemap_id = response.body['id']
+      sitemap_url = 'https://www.elastic.co/sitemap2.xml'
+      body = { url: sitemap_url }
+      response = client.put_crawler_sitemap(
+        engine_name,
+        domain_id: @domain['id'],
+        sitemap_id: sitemap_id,
+        body: body
+      )
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['id', 'url', 'created_at'])
+      expect(response.body['url']).to eq(sitemap_url)
+
+      # Delete sitemap
+      response = client.delete_crawler_sitemap(
+        engine_name,
+        domain_id: @domain['id'],
+        sitemap_id: sitemap_id
+      )
+
+      expect(response.status).to eq 200
+      expect(response.body).to eq({ 'deleted' => true })
+    end
+  end
+end

data/spec/integration/app-search/crawler_urls_spec.rb

@@ -0,0 +1,60 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler URLs' do
+    let(:engine_name) { 'crawler-urls' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+      sleep 1
+    end
+
+    it 'validates a URL' do
+      response = client.crawler_url_validation_result(engine_name, url: name)
+      expect(response.status).to eq 200
+      expect(response.body['url']).to eq name
+      expect(response.body['valid']).to eq true
+    end
+
+    it 'extracts content from a URL' do
+      response = client.crawler_url_extraction_result(engine_name, url: name)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['url', 'normalized_url', 'results'])
+    end
+
+    it 'traces history for a crawler URL' do
+      response = client.crawler_url_tracing_result(engine_name, url: name)
+      expect(response.status).to eq 200
+      expect(response.body['url']).to eq name
+      expect(response.body['normalized_url']).to eq "#{name}/"
+      expect(response.body['crawl_requests'])
+    end
+  end
+end

data/spec/{app-search/api_count_analytics_spec.rb → integration/app-search/crawler_user_agent_spec.rb}

@@ -17,18 +17,15 @@
 
 # frozen_string_literal: true
 
-require 'spec_helper'
-require_relative './api_spec_helper'
+require_relative "#{__dir__}/app_search_helper.rb"
 
 describe Elastic::EnterpriseSearch::AppSearch::Client do
-  context 'api_logs' do
-    it 'returns api logs' do
-      VCR.use_cassette('app_search/count_analytics') do
-        response = @client.count_analytics('videogames')
+  context 'Crawler User Agent' do
+    it 'retrieves the user-agent header value for the crawler' do
+      response = client.crawler_user_agent
 
-        expect(response.status).to eq 200
-        expect(response.body['results'].count).to be > 1
-      end
+      expect(response.status).to eq 200
+      expect(response.body['user_agent']).to match(/Elastic-Crawler\ \([0-9.]+\)/)
     end
   end
 end

data/spec/integration/app-search/curations_spec.rb

@@ -0,0 +1,118 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Curations' do
+    let(:engine_name) { 'books' }
+
+    before do
+      client.create_engine(name: engine_name)
+      documents = [
+        { title: 'Jungle Tales', author: 'Horacio Quiroga' },
+        { title: 'The Jungle Book', author: 'Rudyard Kipling' }
+      ]
+      response = client.index_documents(engine_name, documents: documents)
+
+      @promoted = response.body.first
+      @hidden = response.body.last
+    end
+    after do
+      client.delete_engine(engine_name)
+      sleep 1
+    end
+
+    it 'creates a new curation' do
+      response = create_curation(engine_name, 'jungle', @promoted['id'], @hidden['id'])
+      expect(response.status).to eq 200
+      expect(response.body['id']).to match(/cur-[0-9a-f]+/)
+
+      response = client.search(engine_name, query: 'jungle')
+      expect(response.status).to eq 200
+      expect(response.body['results'].count).to eq 1
+      expect(response.body['results'].first['title']['raw']).to eq 'Jungle Tales'
+    end
+
+    it 'retrieves a curation by id' do
+      id = create_curation(engine_name, 'book', @promoted['id'], @hidden['id']).body['id']
+
+      response = client.curation(engine_name, curation_id: id)
+
+      expect(response.status).to eq 200
+      expect(response.body['id']).to match(/cur-[0-9a-f]+/)
+      expect(response.body['queries']).to eq ['book']
+      expect(response.body['promoted']).to eq [@promoted['id']]
+      expect(response.body['hidden']).to eq [@hidden['id']]
+    end
+
+    def create_curation(name, query, promoted_id, hidden_id)
+      response = client.create_curation(
+        name,
+        queries: [query],
+        promoted: [promoted_id],
+        hidden: [hidden_id]
+      )
+      # Give the server a second
+      sleep 1
+      response
+    end
+
+    it 'updates an existing curation' do
+      id = create_curation(engine_name, 'jungle', @promoted['id'], @hidden['id']).body['id']
+
+      response = client.put_curation(
+        engine_name,
+        curation_id: id,
+        queries: ['jungle'],
+        promoted: [@hidden['id']],
+        hidden: [@promoted['id']]
+      )
+
+      expect(response.status).to eq 200
+      expect(response.body['id']).to match(/cur-[0-9a-f]+/)
+
+      response = @client.search(engine_name, query: 'jungle')
+      expect(response.status).to eq 200
+      expect(response.body['results'].count).to eq 1
+      expect(response.body['results'].first['title']['raw']).to eq 'The Jungle Book'
+    end
+
+    it 'lists curations' do
+      create_curation(engine_name, 'jungle', @promoted['id'], @hidden['id']).body['id']
+      create_curation(engine_name, 'book', @promoted['id'], @hidden['id']).body['id']
+      create_curation(engine_name, 'tales', @promoted['id'], @hidden['id']).body['id']
+
+      response = client.list_curations(engine_name)
+
+      expect(response.status).to eq 200
+      expect(response.body['results'].count).to eq 3
+      curation = response.body['results'].first
+      expect(curation['id']).to match(/cur-[0-9a-f]+/)
+    end
+
+    it 'deletes a  curation' do
+      id = create_curation(engine_name, 'jungle', @promoted['id'], @hidden['id']).body['id']
+      response = client.delete_curation(engine_name, curation_id: id)
+
+      expect(response.status).to eq 200
+      expect(response.body).to eq({ 'deleted' => true })
+    end
+  end
+end