elasticsearch-api 7.2.0 → 8.0.0

data/spec/elasticsearch/api/rest_api_yaml_spec.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -14,144 +14,134 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
+
 require 'spec_helper'
-require 'rest_yaml_tests_helper'
+require 'rest_api_tests_helper'
 
 describe 'Rest API YAML tests' do
+  LOGGER = Logger.new($stdout)
+  if REST_API_YAML_FILES.empty?
+    LOGGER.error 'No test files found!'
+    LOGGER.info 'Use rake rake elasticsearch:download_artifacts in the root directory of the project to download the test artifacts.'
+    exit 1
+  end
 
+  # Traverse YAML files and create TestFile object:
   REST_API_YAML_FILES.each do |file|
+    begin
+      test_file = Elasticsearch::RestAPIYAMLTests::TestFile.new(file, ADMIN_CLIENT, REST_API_YAML_SKIP_FEATURES)
+    rescue SkipTestsException => _e
+      # If the test file has a `skip` at the top level that applies to this
+      # version of Elasticsearch, continue with the next text.
+      LOGGER.info "Skipping #{file} due to 'skip all'."
+      next
+    end
 
-    test_file = Elasticsearch::RestAPIYAMLTests::TestFile.new(file, REST_API_YAML_SKIP_FEATURES)
-
-    context "#{file.gsub("#{YAML_FILES_DIRECTORY}/", '')}" do
+    ctx = file.gsub("#{YAML_FILES_DIRECTORY}/", '')
+    context ctx do
+      let(:client) { DEFAULT_CLIENT }
 
       test_file.tests.each do |test|
-
-        context "#{test.description}" do
-
+        context test.description do
           if test.skip_test?(ADMIN_CLIENT)
-            skip 'Test contains feature(s) not yet support or version is not satisfied'
-
-          else
-
-            let(:client) do
-              DEFAULT_CLIENT
-            end
+            skip 'Test contains feature(s) not yet supported or version is not satisfied'
+            next
+          end
 
-            # Runs once before each test in a test file
-            before(:all) do
-              # Runs once before each test in a test file
-              Elasticsearch::RestAPIYAMLTests::TestFile.clear_data(ADMIN_CLIENT)
-              test_file.setup(ADMIN_CLIENT)
-            end
+          before(:all) { test_file.setup }
+          after(:all) do
+            test_file.teardown
+            Elasticsearch::RestAPIYAMLTests::WipeCluster.run(ADMIN_CLIENT)
+          end
 
-            after(:all) do
-              test_file.teardown(ADMIN_CLIENT)
-              Elasticsearch::RestAPIYAMLTests::TestFile.clear_data(ADMIN_CLIENT)
+          test.task_groups.each do |task_group|
+            before do
+              task_group.run(client)
             end
 
-            test.task_groups.each do |task_group|
-
-              before do
-                task_group.run(client)
+            # 'catch' is in the task group definition
+            if task_group.catch_exception?
+              it 'sends the request and throws the expected error' do
+                expect(task_group.exception).to match_error(task_group.expected_exception_message)
               end
 
-              # 'catch' is in the task group definition
-              if task_group.catch_exception?
-
-                it 'sends the request and throws the expected error' do
-                  expect(task_group.exception).to match_error(task_group.expected_exception_message)
-                end
-
-                # 'match' on error description is in the task group definition
-                if task_group.has_match_clauses?
-
-                  task_group.match_clauses.each do |match|
-                    it 'contains the expected error in the request response' do
-                      expect(task_group.exception.message).to match(Regexp.new(Regexp.escape(match['match'].values.first.to_s)))
-                    end
+              # 'match' on error description is in the task group definition
+              if task_group.has_match_clauses?
+                task_group.match_clauses.each do |match|
+                  it 'contains the expected error in the request response' do
+                    expect(task_group.exception.message).to match(Regexp.new(Regexp.escape(match['match'].values.first.to_s)))
                   end
                 end
-              else
-
-                # 'match' is in the task group definition
-                if task_group.has_match_clauses?
-
-                  task_group.match_clauses.each do |match|
-                    it "has the expected value (#{match['match'].values.join(',')}) in the response field (#{match['match'].keys.join(',')})" do
-                      expect(task_group.response).to match_response(match['match'], test)
-                    end
+              end
+            else
+              # 'match' is in the task group definition
+              if task_group.has_match_clauses?
+                task_group.match_clauses.each do |match|
+                  it "has the expected value (#{match['match'].values.join(',')}) in the response field (#{match['match'].keys.join(',')})" do
+                    expect(task_group.response).to match_response(match['match'], test)
                   end
                 end
+              end
 
-                # 'length' is in the task group definition
-                if task_group.has_length_match_clauses?
-
-                  task_group.length_match_clauses.each do |match|
-                    it "the '#{match['length'].keys.join(',')}' field have the expected length" do
-                      expect(task_group.response).to match_response_field_length(match['length'], test)
-                    end
+              # 'length' is in the task group definition
+              if task_group.has_length_match_clauses?
+                task_group.length_match_clauses.each do |match|
+                  it "the '#{match['length'].keys.join(',')}' field have the expected length" do
+                    expect(task_group.response).to match_response_field_length(match['length'], test)
                   end
                 end
+              end
 
-                # 'is_true' is in the task group definition
-                if task_group.has_true_clauses?
-
-                  task_group.true_clauses.each do |match|
-                    it "sends the request and the '#{match['is_true']}' field is set to true" do
-                      expect(task_group.response).to match_true_field(match['is_true'], test)
-                    end
+              # 'is_true' is in the task group definition
+              if task_group.has_true_clauses?
+                task_group.true_clauses.each do |match|
+                  it "sends the request and the '#{match['is_true']}' field is set to true" do
+                    expect(task_group.response).to match_true_field(match['is_true'], test)
                   end
                 end
+              end
 
-                # 'is_false' is in the task group definition
-                if task_group.has_false_clauses?
-
-                  task_group.false_clauses.each do |match|
-                    it "sends the request and the '#{match['is_false']}' field is set to true" do
-                      expect(task_group.response).to match_false_field(match['is_false'], test)
-                    end
+              # 'is_false' is in the task group definition
+              if task_group.has_false_clauses?
+                task_group.false_clauses.each do |match|
+                  it "sends the request and the '#{match['is_false']}' field is set to true" do
+                    expect(task_group.response).to match_false_field(match['is_false'], test)
                   end
                 end
+              end
 
-                # 'gte' is in the task group definition
-                if task_group.has_gte_clauses?
-
-                  task_group.gte_clauses.each do |match|
-                    it "sends the request and the '#{match['gte']}' field is greater than or equal to the expected value" do
-                      expect(task_group.response).to match_gte_field(match['gte'], test)
-                    end
+              # 'gte' is in the task group definition
+              if task_group.has_gte_clauses?
+                task_group.gte_clauses.each do |match|
+                  it "sends the request and the '#{match['gte']}' field is greater than or equal to the expected value" do
+                    expect(task_group.response).to match_gte_field(match['gte'], test)
                   end
                 end
+              end
 
-                # 'gt' is in the task group definition
-                if task_group.has_gt_clauses?
-
-                  task_group.gt_clauses.each do |match|
-                    it "sends the request and the '#{match['gt']}' field is greater than the expected value" do
-                      expect(task_group.response).to match_gt_field(match['gt'], test)
-                    end
+              # 'gt' is in the task group definition
+              if task_group.has_gt_clauses?
+                task_group.gt_clauses.each do |match|
+                  it "sends the request and the '#{match['gt']}' field is greater than the expected value" do
+                    expect(task_group.response).to match_gt_field(match['gt'], test)
                   end
                 end
+              end
 
-                # 'lte' is in the task group definition
-                if task_group.has_lte_clauses?
-
-                  task_group.lte_clauses.each do |match|
-                    it "sends the request and the '#{match['lte']}' field is less than or equal to the expected value" do
-                      expect(task_group.response).to match_lte_field(match['lte'], test)
-                    end
+              # 'lte' is in the task group definition
+              if task_group.has_lte_clauses?
+                task_group.lte_clauses.each do |match|
+                  it "sends the request and the '#{match['lte']}' field is less than or equal to the expected value" do
+                    expect(task_group.response).to match_lte_field(match['lte'], test)
                   end
                 end
+              end
 
-                # 'lt' is in the task group definition
-                if task_group.has_lt_clauses?
-
-                  task_group.lt_clauses.each do |match|
-                    it "sends the request and the '#{match['lt']}' field is less than the expected value" do
-                      expect(task_group.response).to match_lt_field(match['lt'], test)
-                    end
+              # 'lt' is in the task group definition
+              if task_group.has_lt_clauses?
+                task_group.lt_clauses.each do |match|
+                  it "sends the request and the '#{match['lt']}' field is less than the expected value" do
+                    expect(task_group.response).to match_lt_field(match['lt'], test)
                   end
                 end
               end

data/spec/elasticsearch/api/utils_spec.rb

@@ -6,7 +6,7 @@
 # not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#	http://www.apache.org/licenses/LICENSE-2.0
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
@@ -18,19 +18,17 @@
 require 'spec_helper'
 
 describe Elasticsearch::API::Utils do
-
   let(:utils) do
     Class.new { include Elasticsearch::API::Utils }.new
   end
 
   describe '#__escape' do
-
     it 'encodes Unicode characters' do
       expect(utils.__escape('中文')).to eq('%E4%B8%AD%E6%96%87')
     end
 
     it 'encodes special characters' do
-      expect(utils.__escape('foo bar')).to eq('foo+bar')
+      expect(utils.__escape('foo bar')).to eq('foo%20bar')
       expect(utils.__escape('foo/bar')).to eq('foo%2Fbar')
       expect(utils.__escape('foo^bar')).to eq('foo%5Ebar')
     end
@@ -38,22 +36,9 @@ describe Elasticsearch::API::Utils do
     it 'does not encode asterisks' do
       expect(utils.__escape('*')).to eq('*')
     end
-
-    it 'users CGI.escape by default' do
-      expect(CGI).to receive(:escape).and_call_original
-      expect(utils.__escape('foo bar')).to eq('foo+bar')
-    end
-
-    it 'uses the escape_utils gem when available', unless: defined?(JRUBY_VERSION) do
-      require 'escape_utils'
-      expect(CGI).not_to receive(:escape)
-      expect(EscapeUtils).to receive(:escape_url).and_call_original
-      expect(utils.__escape('foo bar')).to eq('foo+bar')
-    end
   end
 
   describe '#__listify' do
-
     it 'creates a list from a single value' do
       expect(utils.__listify('foo')).to eq('foo')
     end
@@ -75,15 +60,13 @@ describe Elasticsearch::API::Utils do
     end
 
     context 'when the escape option is set to false' do
-
       it 'does not escape the characters' do
-        expect(utils.__listify(['foo', 'bar^bam'], :escape => false)).to eq('foo,bar^bam')
+        expect(utils.__listify(['foo', 'bar^bam'], escape: false)).to eq('foo,bar^bam')
       end
     end
   end
 
   describe '#__pathify' do
-
     it 'creates a path from a single value' do
       expect(utils.__pathify('foo')).to eq('foo')
     end
@@ -102,24 +85,22 @@ describe Elasticsearch::API::Utils do
   end
 
   describe '#__bulkify' do
-
     context 'when the input is an array of hashes' do
-
       let(:result) do
         utils.__bulkify [
-          { :index =>  { :_index => 'myindexA', :_type => 'mytype', :_id => '1', :data => { :title => 'Test' } } },
-          { :update => { :_index => 'myindexB', :_type => 'mytype', :_id => '2', :data => { :doc => { :title => 'Update' } } } },
-          { :delete => { :_index => 'myindexC', :_type => 'mytypeC', :_id => '3' } }
+          { index: { _index: 'myindexA', _id: '1', data: { title: 'Test' } } },
+          { update: { _index: 'myindexB', _id: '2', data: { doc: { title: 'Update' } } } },
+          { delete: { _index: 'myindexC', _id: '3' } }
         ]
       end
 
       let(:expected_string) do
         <<-PAYLOAD.gsub(/^\s+/, '')
-                {"index":{"_index":"myindexA","_type":"mytype","_id":"1"}}
+                {"index":{"_index":"myindexA","_id":"1"}}
                 {"title":"Test"}
-                {"update":{"_index":"myindexB","_type":"mytype","_id":"2"}}
+                {"update":{"_index":"myindexB","_id":"2"}}
                 {"doc":{"title":"Update"}}
-                {"delete":{"_index":"myindexC","_type":"mytypeC","_id":"3"}}
+                {"delete":{"_index":"myindexC","_id":"3"}}
         PAYLOAD
       end
 
@@ -192,7 +173,6 @@ describe Elasticsearch::API::Utils do
     end
 
     context 'when the payload has nested :data options' do
-
       let(:data) do
         { data: { a: 'b', data: { c: 'd' } } }
       end
@@ -226,83 +206,28 @@ describe Elasticsearch::API::Utils do
   end
 
   context '#__validate_and_extract_params' do
-
     it 'listify Arrays' do
-      expect(utils.__validate_and_extract_params({ foo: ['a', 'b'] }, [:foo] )).to eq(foo: 'a,b')
+      expect(utils.process_params({ foo: ['a', 'b'] })).to eq(foo: 'a,b')
     end
 
     it 'does not escape the parameters' do
-      expect(utils.__validate_and_extract_params({ foo: ['a.*', 'b.*'] }, [:foo] )).to eq(foo: 'a.*,b.*')
+      expect(utils.process_params({ foo: ['a.*', 'b.*'] })).to eq(foo: 'a.*,b.*')
     end
 
     context 'when the params are valid' do
-
       it 'extracts the valid params from the hash' do
-        expect(utils.__validate_and_extract_params({ foo: 'qux' }, [:foo, :bar]) ).to eq(foo: 'qux')
-      end
-    end
-
-    context 'when the params are invalid' do
-
-      it 'raises an ArgumentError' do
-        expect {
-          utils.__validate_and_extract_params({ foo: 'qux', bam: 'mux' }, [:foo, :bar])
-        }.to raise_exception(ArgumentError)
-      end
-    end
-
-    context 'when COMMON_PARAMS are provided' do
-
-      it 'extracts the params' do
-        expect(utils.__validate_and_extract_params({ index: 'foo'}, [:foo])).to eq({})
-      end
-    end
-
-    context 'when COMMON_QUERY_PARAMS are provided' do
-
-      it 'extracts the params' do
-        expect(utils.__validate_and_extract_params(format: 'yaml')).to eq(format: 'yaml')
-      end
-    end
-
-    context 'when the :skip_paramter_validation option is set' do
-
-      let(:result) do
-        utils.__validate_and_extract_params( { foo: 'q', bam: 'm' }, [:foo, :bar], { skip_parameter_validation: true } )
-      end
-
-      it 'skips parameter validation' do
-        expect(result).to eq(foo: 'q', bam: 'm')
-      end
-    end
-
-    context 'when the module has the setting to skip parameter validation' do
-
-      around do |example|
-        original_value = Elasticsearch::API.settings[:skip_parameter_validation]
-        Elasticsearch::API.settings[:skip_parameter_validation] = true
-        example.run
-        Elasticsearch::API.settings[:skip_parameter_validation] = original_value
-      end
-
-      let(:result) do
-        utils.__validate_and_extract_params( { foo: 'q', bam: 'm' }, [:foo, :bar])
-      end
-
-      it 'applies the module setting' do
-        expect(result).to eq(foo: 'q', bam: 'm')
+        expect(utils.process_params({ foo: 'qux' })).to eq(foo: 'qux')
       end
     end
   end
 
   describe '#__extract_parts' do
-
     it 'extracts parts with true value from a Hash' do
-      expect(utils.__extract_parts({ foo: true, moo: 'blah' }, [:foo, :bar])).to eq(['foo'])
+      expect(utils.__extract_parts({ foo: true, moo: 'blah' })).to eq(['foo', 'blah'])
     end
 
     it 'extracts parts with string value from a Hash' do
-      expect(utils.__extract_parts({ foo: 'qux', moo: 'blah' }, [:foo, :bar])).to eq(['qux'])
+      expect(utils.__extract_parts({ foo: 'qux', moo: 'blah' })).to eq(['qux', 'blah'])
     end
   end
 
@@ -379,7 +304,7 @@ describe Elasticsearch::API::Utils do
       end
 
       let(:unsupported_params) do
-        [ { :foo => { :explanation => 'NOT_SUPPORTED'} }, :moo ]
+        [ { foo: { explanation: 'NOT_SUPPORTED'} }, :moo ]
       end
 
 

data/spec/rest_api_tests_helper.rb

@@ -0,0 +1,91 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "#{File.expand_path(File.dirname('..'))}/api-spec-testing/test_file"
+require "#{File.expand_path(File.dirname('..'))}/api-spec-testing/rspec_matchers"
+require "#{File.expand_path(File.dirname('..'))}/api-spec-testing/wipe_cluster"
+include Elasticsearch::RestAPIYAMLTests
+
+PROJECT_PATH = File.join(File.dirname(__FILE__), '..')
+host = ENV['TEST_ES_SERVER'] || 'http://localhost:9200'
+raise URI::InvalidURIError unless host =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
+
+test_suite = ENV['TEST_SUITE'] || 'free'
+password = ENV['ELASTIC_PASSWORD'] || 'changeme'
+uri = URI.parse(host)
+
+if test_suite == 'platinum'
+  raw_certificate = File.read(File.join(PROJECT_PATH, '../.ci/certs/testnode.crt'))
+  certificate = OpenSSL::X509::Certificate.new(raw_certificate)
+  raw_key = File.read(File.join(PROJECT_PATH, '../.ci/certs/testnode.key'))
+  key = OpenSSL::PKey::RSA.new(raw_key)
+  ca_file = File.expand_path(File.join(PROJECT_PATH, '/.ci/certs/ca.crt'))
+  host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
+  transport_options = { ssl: { verify: false, client_cert: certificate, client_key: key, ca_file: ca_file } }
+else
+  host = "http://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
+  transport_options = {}
+end
+
+ADMIN_CLIENT = Elasticsearch::Client.new(host: host, transport_options: transport_options)
+
+DEFAULT_CLIENT = if ENV['QUIET'] == 'true'
+                   Elasticsearch::Client.new(host: host, transport_options: transport_options)
+                 else
+                   Elasticsearch::Client.new(
+                     host: host,
+                     tracer: Logger.new($stdout),
+                     transport_options: transport_options
+                   )
+                 end
+
+unless ADMIN_CLIENT.ping
+  raise('Couldn\'t connect to Elasticsearch cluster')
+  exit
+end
+
+YAML_FILES_DIRECTORY = "#{PROJECT_PATH}/../tmp/rest-api-spec/test/#{test_suite}".freeze
+
+SINGLE_TEST = if ENV['SINGLE_TEST'] && !ENV['SINGLE_TEST'].empty?
+                test_target = ENV['SINGLE_TEST']
+
+                if test_target.match?(/\.yml$/)
+                  ["#{PROJECT_PATH}/../tmp/rest-api-spec/test/#{test_suite}/#{test_target}"]
+                else
+                  Dir.glob(
+                    ["#{PROJECT_PATH}/../tmp/rest-api-spec/test/#{test_suite}/#{test_target}/*.yml"]
+                  )
+                end
+              end
+
+# Skipped tests
+file = File.expand_path("#{__dir__}/skipped_tests_#{test_suite}.yml")
+skipped_tests = YAML.load_file(file)
+
+# The directory of rest api YAML files.
+REST_API_YAML_FILES = if ENV['RUN_SKIPPED_TESTS'] # only run the skipped tests if true
+                        SKIPPED_TESTS = []
+                        skipped_tests.map { |test| "#{YAML_FILES_DIRECTORY}/#{test[:file]}" }
+                      else
+                        # If not, define the skipped tests constant and try the single test or all
+                        # the tests
+                        SKIPPED_TESTS = skipped_tests
+                        SINGLE_TEST || Dir.glob("#{YAML_FILES_DIRECTORY}/**/*.yml")
+                      end
+
+# The features to skip
+REST_API_YAML_SKIP_FEATURES = ['warnings', 'node_selector'].freeze

data/spec/skipped_tests_free.yml

@@ -0,0 +1,32 @@
+# Response from Elasticsearch is just a String, so it's not possible to compare using headers.
+-
+  :file:        'cat.aliases/20_headers.yml'
+  :description: 'Simple alias with yaml body through Accept header'
+
+# Check version skip logic
+-
+  :file:        'create/15_without_id.yml'
+  :description: 'Create without ID'
+
+# Error message doesn't match
+-
+  :file:        'tasks.get/10_basic.yml'
+  :description: 'get task test'
+
+# Figure out how to match response when there is an error
+-
+  :file:        'cat.templates/10_basic.yml'
+  :description: '*'
+
+# https://github.com/elastic/elasticsearch-ruby/issues/1469
+-
+  :file:        'indices.get_alias/10_basic.yml'
+  :description: 'Get alias against closed indices'
+
+# https://github.com/elastic/elasticsearch-ruby/issues/1563
+-
+  :file:        'tsdb/80_index_resize.yml'
+  :description: 'shrink'
+-
+  :file:        'tsdb/80_index_resize.yml'
+  :description: 'clone'

data/spec/skipped_tests_platinum.yml

@@ -0,0 +1,59 @@
+# Response from Elasticsearch includes the ca.crt, so length doesn't match.
+-
+  :file:        'ssl/10_basic.yml'
+  :description: 'Test get SSL certificates'
+
+# TODO: Failing due to processing of regexp in test
+-
+  :file: 'ml/explain_data_frame_analytics.yml'
+  :description: 'Test non-empty data frame given body'
+
+# TODO https://github.com/elastic/elasticsearch-ruby/issues/852
+-
+  :file: 'analytics/usage.yml'
+  :description: 'Usage stats on analytics indices'
+
+# TODO https://github.com/elastic/elasticsearch-ruby/issues/1269
+-
+  :file: 'searchable_snapshots/10_usage.yml'
+  :description: 'Tests searchable snapshots usage stats with full_copy and shared_cache indices'
+
+# TODO https://github.com/elastic/elasticsearch-ruby/issues/1291
+-
+  :file: 'service_accounts/10_basic.yml'
+  :description: 'Test service account tokens'
+
+# TODO https://github.com/elastic/elasticsearch-ruby/issues/1340
+-
+  :file: 'transform/transforms_start_stop.yml'
+  :description: '*'
+-
+  :file: 'transform/transforms_cat_apis.yml'
+  :description: '*'
+-
+  :file: 'transform/transforms_crud.yml'
+  :description: '*'
+-
+  :file: 'transform/transforms_force_delete.yml'
+  :description: '*'
+-
+  :file: 'transform/transforms_stats.yml'
+  :description: '*'
+-
+  :file: 'transform/transforms_stats_continuous.yml'
+  :description: '*'
+-
+  :file: 'transform/transforms_update.yml'
+  :description: '*'
+-
+  :file: 'transform/preview_transforms.yml'
+  :description: 'Test preview transform by id'
+
+-
+  :file: 'transform/transforms_upgrade.yml'
+  :description: 'Test upgrade transform'
+
+# TODO: https://github.com/elastic/elasticsearch-ruby/issues/1539
+-
+  :file: 'ml/categorization_agg.yml'
+  :description: 'Test categorization aggregation with poor settings'