elastic-enterprise-search 7.12.1 → 7.15.0

data/spec/integration/app-search/app_search_helper.rb

@@ -0,0 +1,39 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+# App Search Integration tests Client Configuration
+def client
+  @client ||= Elastic::EnterpriseSearch::AppSearch::Client.new(
+    host: ENV['ELASTIC_ENTERPRISE_HOST'] || 'http://localhost:3002',
+    http_auth: {
+      user: ENV['ELASTIC_ENTERPRISE_USER'] || 'enterprise_search',
+      password: ENV['ELASTIC_ENTERPRISE_PASSWORD'] || 'changeme'
+    }
+  )
+end
+
+def delete_engines
+  engines = client.list_engines.body['results']
+  engines.each do |engine|
+    client.delete_engine(engine['name'])
+  end
+  sleep 1
+end

data/spec/integration/app-search/count_analytics_spec.rb

@@ -0,0 +1,47 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Count Analytics' do
+    let(:engine_name) { 'count-analytics' }
+
+    before do
+      client.create_engine(name: engine_name)
+      sleep 1
+    end
+
+    after do
+      delete_engines
+    end
+
+    it 'returns count analytics' do
+      response = client.count_analytics(engine_name)
+
+      expect(response.status).to eq 200
+      expect(response.body['results'].count).to be > 1
+    end
+
+    it 'returns count analytics with filters' do
+      response = client.count_analytics(engine_name, body: { filters: { tag: 'web' } })
+      expect(response.status).to eq 200
+      expect(response.body['results'].count).to be > 1
+    end
+  end
+end

data/spec/integration/app-search/crawl_requests_spec.rb

@@ -0,0 +1,86 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler crawl requests' do
+    let(:engine_name) { 'crawler-requests' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_crawler_active_crawl_request(engine_name)
+      delete_engines
+    end
+
+    it 'creates, gets, lists and deletes a crawl request' do
+      response = client.create_crawler_crawl_request(engine_name)
+
+      expect(response.status).to eq 200
+      expect(response.body.keys).to include('status')
+      expect(response.body.keys).to include('id')
+      request_id = response.body['id']
+
+      # get crawl request
+      response = client.crawler_crawl_request(engine_name, crawl_request_id: request_id)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to include('status')
+      expect(response.body.keys).to include('id')
+
+      # get active crawl request details
+      response = client.crawler_active_crawl_request(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to include('status')
+      expect(response.body.keys).to include('id')
+
+      attempts = 0
+      while response.body['status'] != 'running' && attempts < 20
+        sleep 1
+        attempts += 1
+        response = client.crawler_active_crawl_request(engine_name)
+      end
+      expect(response.body['status']).to eq 'running'
+
+      # list crawl requests
+      response = client.list_crawler_crawl_requests(engine_name)
+      expect(response.status).to eq 200
+
+      # delete active crawl request
+      response = client.delete_active_crawl_request(engine_name)
+      expect(response.status).to eq 200
+    end
+
+    it 'creates, starts and stops a crawl request' do
+      response = client.create_crawler_crawl_request(engine_name)
+      request_id = response.body['id']
+
+      response = client.delete_crawler_active_crawl_request(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body['id']).to eq request_id
+      expect(response.body['status']).to eq('canceling')
+    end
+  end
+end

data/spec/integration/app-search/crawler_crawl_rule_spec.rb

@@ -0,0 +1,73 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler crawl rules' do
+    let(:engine_name) { 'crawler-crawl-rule' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'creates, updates and deletes a rule' do
+      # Create a crawler crawl rule
+      body = { order: 1, policy: 'allow', rule: 'contains', pattern: '/stack' }
+      response = client.create_crawler_crawl_rule(engine_name, domain_id: @domain['id'], body: body)
+      expect(response.status).to eq 200
+      rule_id = response.body['id']
+      expect(response.body['order']).to eq 1
+      expect(response.body['policy']).to eq 'allow'
+      expect(response.body['rule']).to eq 'contains'
+      expect(response.body['pattern']).to eq '/stack'
+
+      # Update a crawler crawl rule
+      body = { order: 2, policy: 'allow', rule: 'begins', pattern: '/stack' }
+      response = client.put_crawler_crawl_rule(
+        engine_name,
+        domain_id: @domain['id'],
+        crawl_rule_id: rule_id,
+        body: body
+      )
+      expect(response.status).to eq 200
+      expect(response.body['order']).to eq 2
+      expect(response.body['policy']).to eq 'allow'
+      expect(response.body['rule']).to eq 'begins'
+      expect(response.body['pattern']).to eq '/stack'
+
+      # Delete a crawler crawl rule
+      response = client.delete_crawler_crawl_rule(
+        engine_name,
+        domain_id: @domain['id'],
+        crawl_rule_id: rule_id
+      )
+      expect(response.status).to eq 200
+      expect(response.body).to eq({ 'deleted' => true })
+    end
+  end
+end

data/spec/integration/app-search/crawler_domain_spec.rb

@@ -0,0 +1,82 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler Domain' do
+    let(:engine_name) { 'crawler' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+    end
+
+    after do
+      client.delete_crawler_domain(engine_name, domain_id: @domain['id'])
+      client.delete_engine(engine_name)
+      sleep 1
+    end
+
+    it 'creates and gets a crawler domain' do
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+
+      expect(response.status).to eq 200
+      expect(response.body['id']).to eq @domain['id']
+      expect(response.body).to include('name' => name)
+
+      # client.crawler_domain
+      response = client.crawler_domain(engine_name, domain_id: @domain['id'])
+      expect(response.status).to eq 200
+      expect(response.body['id']).to eq @domain['id']
+      expect(response.body).to include('name' => name)
+    end
+
+    it 'creates and updates a crawler domain' do
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+
+      expect(response.status).to eq 200
+      expect(response.body['id']).to eq @domain['id']
+      expect(response.body).to include('name' => name)
+
+      new_name = 'https://www.wikipedia.org'
+      body = { name: new_name }
+      response = client.put_crawler_domain(engine_name, domain_id: @domain['id'], domain: body)
+
+      expect(response.status).to eq 200
+      expect(response.body['id']).to eq @domain['id']
+      expect(response.body).to include('name' => new_name)
+    end
+
+    it 'validates a domain' do
+      body = { name: name }
+      @domain = client.create_crawler_domain(engine_name, body: body).body
+      body = { url: name }
+      response = client.crawler_domain_validation_result(body: body)
+
+      expect(response.status).to eq 200
+      expect(response.body['url']).to eq name
+      expect(response.body['valid']).to eq true
+    end
+  end
+end

data/spec/integration/app-search/crawler_entry_point_spec.rb

@@ -0,0 +1,89 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler entry point' do
+    let(:engine_name) { 'crawler-entry-point' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      attempts = 0
+      loop do
+        attempts += 1
+        client.create_engine(name: engine_name)
+        break if attempts > 9
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest => e
+        if e.message.match?(/Name is already taken/)
+          sleep 1
+          next
+        end
+      else
+        break
+      end
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'creates an entry point' do
+      value = '/enterprise-search'
+      body = { value: value }
+      response = client.create_crawler_entry_point(engine_name, domain_id: @domain['id'], body: body)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['id', 'value', 'created_at'])
+      expect(response.body['value']).to eq value
+    end
+
+    it 'updates an entry point' do
+      value = '/elastic-stack'
+      body = { value: value }
+      @entry_point = client.create_crawler_entry_point(engine_name, domain_id: @domain['id'], body: body).body
+
+      response = client.put_crawler_entry_point(
+        engine_name,
+        domain_id: @domain['id'],
+        entry_point_id: @entry_point['id'],
+        body: body
+      )
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['id', 'value', 'created_at'])
+      expect(response.body['value']).to eq value
+    end
+
+    it 'deletes an entry point' do
+      value = '/security'
+      body = { value: value }
+      @entry_point = client.create_crawler_entry_point(engine_name, domain_id: @domain['id'], body: body).body
+
+      response = client.delete_crawler_entry_point(
+        engine_name,
+        domain_id: @domain['id'],
+        entry_point_id: @entry_point['id']
+      )
+      expect(response.status).to eq 200
+      expect(response.body).to eq({ 'deleted' => true })
+    end
+  end
+end

data/spec/integration/app-search/crawler_metrics_spec.rb

@@ -0,0 +1,46 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler metrics' do
+    let(:engine_name) { 'crawler-metrics' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'retrieves crawler metrics' do
+      response = client.crawler_metrics
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['global', 'node'])
+      expect(response.body.dig('global', 'crawl_requests'))
+      expect(response.body.dig('node', 'workers'))
+    end
+  end
+end

data/spec/integration/app-search/crawler_overview_spec.rb

@@ -0,0 +1,45 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler overview' do
+    let(:engine_name) { 'crawler-overview' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'retrieves crawler configuration overview' do
+      response = client.crawler_overview(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body['domains'].count).to eq 1
+      expect(response.body['domains'].first['name']).to eq name
+    end
+  end
+end

data/spec/integration/app-search/crawler_process_crawl_denied_urls_spec.rb

@@ -0,0 +1,50 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler process crawl denied URLs' do
+    let(:engine_name) { 'crawler-process-crawl-denied-urls' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+    end
+
+    it 'retrieves denied urls for process crawl' do
+      response = client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+      id = response.body['id']
+
+      response = client.crawler_process_crawl_denied_urls(engine_name, process_crawl_id: id)
+      expect(response.status).to eq 200
+      expect(
+        response.body.keys &
+        ['total_url_count', 'denied_url_count', 'sample_size', 'denied_urls_sample']
+      ).to eq response.body.keys
+    end
+  end
+end

data/spec/integration/app-search/crawler_process_crawl_spec.rb

@@ -0,0 +1,66 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require_relative "#{__dir__}/app_search_helper.rb"
+
+describe Elastic::EnterpriseSearch::AppSearch::Client do
+  context 'Crawler Process Crawl' do
+    let(:engine_name) { 'crawler-urls' }
+    let(:name) { 'https://www.elastic.co' }
+
+    before do
+      client.create_engine(name: engine_name)
+      body = { name: name }
+      response = client.create_crawler_domain(engine_name, body: body)
+      @domain = response.body
+    end
+
+    after do
+      client.delete_engine(engine_name)
+      sleep 1
+    end
+
+    it 'creates, retrieves and shows denied for a process crawl' do
+      response = client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+      expect(response.status).to eq 200
+
+      id = response.body['id']
+      expect(response.body['dry_run']).to eq true
+      expect(response.body['domains']).to eq(['https://www.elastic.co'])
+
+      response = client.crawler_process_crawl(engine_name, process_crawl_id: id)
+      expect(response.status).to eq 200
+      expect(response.body['dry_run']).to eq true
+      expect(response.body['domains']).to eq(['https://www.elastic.co'])
+
+      response = client.denied_urls(engine_name, process_crawl_id: id)
+      expect(response.status).to eq 200
+      expect(response.body.keys).to eq(['total_url_count', 'denied_url_count', 'sample_size', 'denied_urls_sample'])
+    end
+
+    it 'lists process crawls' do
+      client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+      client.create_crawler_process_crawl(engine_name, body: { dry_run: true })
+
+      response = client.list_crawler_process_crawls(engine_name)
+      expect(response.status).to eq 200
+      expect(response.body.count).to be 2
+    end
+  end
+end