elasticsearch-api 8.14.0 → 8.16.0

data/spec/elasticsearch/api/perform_request_spec.rb

@@ -23,7 +23,7 @@ require_relative '../../../utils/thor/generator/files_helper'
 describe 'Perform request args' do
   Elasticsearch::API::FilesHelper.files.each do |filepath|
     spec = Elasticsearch::API::EndpointSpec.new(filepath)
-    next if spec.module_namespace.flatten.first == '_internal'
+    next if spec.module_namespace.flatten.first == '_internal' || spec.visibility != 'public'
 
     # These are the path parts defined by the user in the method argument
     defined_path_parts = spec.path_params.inject({}) do |params, part|
@@ -58,7 +58,7 @@ describe 'Perform request args' do
         if defined_path_parts.empty?
           { endpoint: spec.endpoint_name }
         else
-          { endpoint: spec.endpoint_name, defined_params:  defined_path_parts}
+          { endpoint: spec.endpoint_name, defined_params: defined_path_parts }
         end
       end
 

data/spec/free/integration/capabilities_spec.rb

@@ -0,0 +1,26 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# TODO: These tests won't be necessary once we migrate to the new YAML tests
+require_relative './spec_helper'
+
+describe 'Capabilities' do
+  it 'performs the request' do
+    response = CLIENT.capabilities(method: 'GET', path: '/_capabilities', parameters: 'unknown')
+    expect(response.status).to eq 200
+    expect(response['cluster_name']).not_to be_nil
+  end
+end

data/spec/rest_api/rest_api_tests_helper.rb

@@ -65,7 +65,10 @@ unless ADMIN_CLIENT.ping
   exit
 end
 
-Elasticsearch::RestAPIYAMLTests::WipeCluster.create_xpack_rest_user(ADMIN_CLIENT) if test_suite == 'platinum'
+if test_suite == 'platinum'
+  Elasticsearch::RestAPIYAMLTests::WipeCluster.create_xpack_rest_user(ADMIN_CLIENT)
+  Elasticsearch::RestAPIYAMLTests::WipeCluster.create_enterprise_search_users(ADMIN_CLIENT)
+end
 
 YAML_FILES_DIRECTORY = "#{PROJECT_PATH}/../../tmp/rest-api-spec/test/#{test_suite}".freeze
 

data/spec/rest_api/rest_api_yaml_spec.rb

@@ -20,18 +20,27 @@ require_relative 'rest_api_tests_helper'
 require_relative './run_rspec_matchers'
 
 LOGGER = Logger.new($stdout)
+CLUSTER_FEATURES = ADMIN_CLIENT.features.get_features['features'].map { |f| f['name'] }
+
+def skip_test?(test)
+  # To support new features skipping in YAML tests. This will go away with new YAML tests:
+  if (feature_to_skip = test.skip&.first&.[]('skip')&.[]('cluster_features'))
+    return true unless CLUSTER_FEATURES.include?(feature_to_skip)
+  end
+  # Support skipping 'awaits_fix'
+  !!test.test_file.instance_variable_get('@skip')&.first&.[]('skip')&.[]('awaits_fix') ||
+    !!test.skip&.first&.[]('skip')&.[]('awaits_fix')
+end
 
 describe 'Rest API YAML tests' do
   LOGGER.info "Elastic Transport version: #{Elastic::Transport::VERSION}"
 
   if REST_API_YAML_FILES.empty?
     LOGGER.error 'No test files found!'
-    LOGGER.info 'Use rake rake elasticsearch:download_artifacts in the root directory of the project to download the test artifacts.'
+    LOGGER.info 'Use rake rake es:download_artifacts in the root directory of the project to download the test artifacts.'
     exit 1
   end
 
-  CLUSTER_FEATURES = ADMIN_CLIENT.features.get_features['features'].map { |f| f['name'] }
-
   # Traverse YAML files and create TestFile object:
   REST_API_YAML_FILES.each do |file|
     begin
@@ -48,10 +57,8 @@ describe 'Rest API YAML tests' do
       let(:client) { DEFAULT_CLIENT }
 
       test_file.tests.each do |test|
-        # To support new features skipping in YAML tests. This will go away with new YAML tests:
-        if (feature_to_skip = test.skip&.first&.[]('skip')&.[]('cluster_features'))
-          next unless CLUSTER_FEATURES.include? feature_to_skip
-        end
+        next if skip_test?(test)
+
         context test.description do
           if test.skip_test?(ADMIN_CLIENT)
             skip 'Test contains feature(s) not yet supported or version is not satisfied'
@@ -75,6 +82,9 @@ describe 'Rest API YAML tests' do
             # ./run_task_groups.rb
             run_rspec_matchers_on_task_group(task_group, test)
           end
+        rescue StandardError => e
+          LOGGER.error e
+          raise e
         end
       end
     end

data/spec/rest_api/skipped_tests_free.yml

@@ -40,6 +40,9 @@
 -
   :file: 'search.vectors/40_knn_search.yml'
   :description: 'Direct kNN queries are disallowed'
+-
+  :file: 'search.vectors/40_knn_search.yml'
+  :description: 'Knn search with mip'
 -
   :file: 'search.vectors/45_knn_search_byte.yml'
   :description: 'Direct kNN queries are disallowed'
@@ -82,3 +85,80 @@
 -
   :file: 'cluster.desired_nodes/11_old_format.yml'
   :description: 'Test node version must have content'
+-
+  :file: 'aggregations/stats_metric_fail_formatting.yml'
+  :description: 'fail formatting'
+# TODO: Test failing due to implementation
+-
+  :file: 'capabilities/10_basic.yml'
+  :description: '*'
+-
+  :file: 'indices.simulate_index_template/10_basic.yml'
+  :description: 'Simulate index matches overlapping legacy and composable templates'
+-
+  :file: 'indices.simulate_template/10_basic.yml'
+  :description: 'Simulate template matches overlapping legacy and composable templates'
+-
+  :file: 'search.vectors/180_update_dense_vector_type.yml'
+  :description: 'Test create and update dense vector mapping with per-doc indexing and flush'
+-
+  :file: 'search.vectors/180_update_dense_vector_type.yml'
+  :description: 'Test create and update dense vector mapping with bulk indexing'
+-
+  :file: 'search.vectors/180_update_dense_vector_type.yml'
+  :description: 'Index, update and merge'
+-
+  :file: 'search.vectors/90_sparse_vector.yml'
+  :description: 'Indexing and searching multi-value sparse vectors in >=8.15'
+-
+  :file: 'search/330_disallow_scripted_metrics.yml'
+  :description: 'all scripts allowed by default'
+-
+  :file: 'search/330_disallow_scripted_metrics.yml'
+  :description: 'disallow all for empty allow lists'
+-
+  :file: 'search/330_disallow_scripted_metrics.yml'
+  :description: 'allowed stored scripts do not affect allowed inline scripts'
+-
+  :file: 'aggregations/scripted_metric_allow_list.yml'
+  :description: 'allowed stored scripts do not affect allowed inline scripts'
+-
+  :file: 'logsdb/10_settings.yml'
+  :description: '*'
+-
+  :file: 'search.vectors/180_update_dense_vector_type.yml'
+  :description: 'Test create and update dense vector mapping to int4 with per-doc indexing and flush'
+-
+  :file: 'simulate.ingest/10_basic.yml'
+  :description: 'Test mapping validation'
+-
+  :file: 'nodes.stats/11_indices_metrics.yml'
+  :description: 'Lucene segment level fields stats'
+-
+  :file: 'search.suggest/20_phrase.yml'
+  :description: 'breaks ties by sorting terms'
+# Regexp issues with catching an error
+-
+  :file: 'analysis-common/40_token_filters.yml'
+  :description: 'stemmer_override file access'
+- # Migrated to YAML tests:
+  :file: 'ingest_geoip/30_geoip_stats.yml'
+  :description: '*'
+- # Testing server:
+  :file: 'ingest_geoip/20_geoip_processor.yml'
+  :description: '*'
+-
+  :file: 'simulate.ingest/10_basic.yml'
+  :description: 'Test mapping validation from templates'
+-
+  :file: 'indices.split/40_routing_partition_size.yml'
+  :description: '*'
+-
+  :file: 'ingest_geoip/40_geoip_databases.yml'
+  :description: 'Test adding, getting, and removing geoip databases'
+-
+  :file: 'ingest_geoip/50_ip_lookup_processor.yml'
+  :description: '*'
+-
+  :file: 'ingest_geoip/60_ip_location_databases.yml'
+  :description: '*'

data/spec/rest_api/skipped_tests_platinum.yml

@@ -49,7 +49,9 @@
 -
   :file: 'snapshot/10_basic.yml'
   :description: 'Create a source only snapshot and then restore it'
-
+-
+  :file: 'snapshot/10_basic.yml'
+  :description: 'Failed to snapshot indices with synthetic source'
 -
   :file: 'token/10_basic.yml'
   :description: "Test invalidate user's tokens"
@@ -118,3 +120,100 @@
 -
   :file: 'api_key/21_query_with_aggs.yml'
   :description: 'Test composite aggs api key'
+
+# Issue: key_as_string should not be parsed to date:
+-
+  :file: 'analytics/empty_field_metric.yml'
+  :description: 'Rate test'
+# Migrated to yaml tests in elastic/elasticsearch-clients-tests
+-
+  :file: 'roles/10_basic.yml'
+  :description: '*'
+-
+  :file: 'roles/50_remote_only.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/secret/10_connector_secret_post.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/secret/30_connector_secret_get.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/secret/40_connector_secret_delete.yml'
+  :description: '*'
+-
+  :file: 'entsearch/search/20_search_application_put.yml'
+  :description: '*'
+-
+  :file: 'entsearch/search/30_search_application_get.yml'
+  :description: '*'
+-
+  :file: 'entsearch/search/40_search_application_delete.yml'
+  :description: '*'
+-
+  :file: 'entsearch/search/50_search_application_list.yml'
+  :description: '*'
+-
+  :file: 'entsearch/search/56_search_application_search_with_apikey.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/secret/20_connector_secret_put.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/sync_job/50_connector_sync_job_get.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/sync_job/70_connector_sync_job_update_stats.yml'
+  :description: '*'
+-
+  :file: 'entsearch/analytics/40_behavioral_analytics_event_post.yml'
+  :description: '*'
+-
+  :file: 'entsearch/connector/20_connector_list.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/5_query_rulesets_before_setup.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/10_query_ruleset_put.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/20_query_ruleset_list.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/50_query_rule_put.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/10_query_ruleset_put.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/40_rule_query_search.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/60_query_rule_delete.yml'
+  :description: '*'
+-
+  :file: 'entsearch/rules/30_query_ruleset_delete.yml'
+  :description: '*'
+-
+  :file: 'roles/60_bulk_roles.yml'
+  :description: '*'
+-
+  :file: 'entsearch/analytics/30_behavioral_analytics_delete.yml'
+  :description: '*'
+-
+  :file: 'entsearch/analytics/40_behavioral_analytics_event_post.yml'
+  :description: '*'
+-
+  :file: 'entsearch/20_usage.yml'
+  :description: '*'
+# TODO: Update test runnner. Error is ok but Regexp shenanigans
+-
+  :file: 'eql/60_no_exact.yml'
+  :description: 'Case insensitive match on text field'
+-
+  :file: 'esql/180_match_operator.yml'
+  :description: '*'
+-
+  :file: 'wildcard/30_ignore_above_synthetic_source.yml'
+  :description: 'wildcard field type ignore_above'

data/spec/spec_helper.rb

@@ -59,6 +59,7 @@ end
 RSpec.configure do |config|
   config.include(HelperModule)
   config.add_formatter('documentation')
+  config.filter_run_excluding skip: true
   if defined?(JRUBY_VERSION)
     config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-jruby-#{JRUBY_VERSION}.html")
   else

data/spec/yaml-test-runner/run.rb

@@ -0,0 +1,67 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'logger'
+require 'openssl'
+require 'elasticsearch'
+require 'elasticsearch/tests/test_runner'
+require 'elasticsearch/tests/downloader'
+
+PROJECT_PATH = File.join(File.dirname(__FILE__), '../..')
+CERTS_PATH = "#{PROJECT_PATH}/../.buildkite/certs/".freeze
+host = ENV['TEST_ES_SERVER'] || 'https://localhost:9200'
+raise URI::InvalidURIError unless host =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
+
+password = ENV['ELASTIC_PASSWORD'] || 'changeme'
+uri = URI.parse(host)
+
+if uri.is_a?(URI::HTTPS)
+  raw_certificate = File.read("#{CERTS_PATH}/testnode.crt")
+  certificate = OpenSSL::X509::Certificate.new(raw_certificate)
+  raw_key = File.read("#{CERTS_PATH}/testnode.key")
+  key = OpenSSL::PKey::RSA.new(raw_key)
+  ca_file = File.expand_path("#{CERTS_PATH}/ca.crt")
+  host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
+  transport_options = {
+    ssl: {
+      client_cert: certificate,
+      client_key: key,
+      ca_file: ca_file,
+      verify: false
+    }
+  }
+elsif uri.is_a?(URI::HTTP)
+  host = "http://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
+  transport_options = {}
+end
+
+CLIENT = if ENV['ES_API_KEY']
+           Elasticsearch::Client.new(host: host, api_key: ENV['ES_API_KEY'], transport_options: transport_options)
+         else
+           Elasticsearch::Client.new(host: host, transport_options: transport_options)
+         end
+
+tests_path = File.expand_path('./tmp', __dir__)
+
+logger = Logger.new($stdout)
+logger.level = Logger::WARN unless ENV['DEBUG']
+
+# If we're running in a release branch, download the corresponding branch for tests
+current_branch = `git rev-parse --abbrev-ref HEAD`.strip
+branch = current_branch.match(/[0-9]\.[0-9]+/)&.[](0) || ENV['ES_YAML_TESTS_BRANCH'] || nil
+Elasticsearch::Tests::Downloader::run(tests_path, branch)
+Elasticsearch::Tests::TestRunner.new(CLIENT, tests_path, logger).run

data/utils/Gemfile

@@ -18,9 +18,11 @@
 source 'https://rubygems.org'
 
 gem 'coderay'
-gem 'debug' unless defined?(JRUBY_VERSION)
+
 gem 'json'
 gem 'multi_json'
 gem 'pry'
-gem 'rubocop'
 gem 'thor'
+
+gem 'rubocop', '>= 1.51' unless defined?(JRUBY_VERSION) && Gem::Version.new(JRUBY_VERSION) <= Gem::Version.new('9.4')
+gem 'debug' unless defined?(JRUBY_VERSION)

data/utils/README.md

@@ -8,7 +8,7 @@ This directory hosts The Generator, a tool that generates the classes for each A
 
 To generate the code, you need to have the Elasticsearch REST API spec files in `tmp/rest-api-spec` in the root of the project. You can run a rake task from the root of the project to download the specs corresponding to the current running cluster:
 ```bash
-$ rake elasticsearch:download_artifacts
+$ rake es:download_artifacts
 ```
 
 Once the JSON files have been downloaded, you need to run (from this folder):
@@ -24,7 +24,7 @@ $ thor code:generate
 $ IGNORE_VERSION=true thor code:generate
 ```
 
-- You can use the environment variable `BUILD_HASH` to update the build hash for the generated code from the `tmp/rest-api-spec/build_hash` file. This file is updated every time you use the `elasticsearch:download_artifacts` Rake task is used in the root of the project to download the latest Elasticsearch specs and tests:
+- You can use the environment variable `BUILD_HASH` to update the build hash for the generated code from the `tmp/rest-api-spec/build_hash` file. This file is updated every time you use the `es:download_artifacts` Rake task is used in the root of the project to download the latest Elasticsearch specs and tests:
 ```bash
 $ BUILD_HASH=true thor code:generate
 ```

data/utils/thor/.rubocop.yml

@@ -1,2 +1,4 @@
 Layout/EndAlignment:
   AutoCorrect: true
+Layout/EmptyComment:
+  Enabled: false

data/utils/thor/endpoint_spec.rb

@@ -24,6 +24,23 @@ module Elasticsearch
     class EndpointSpec
       include EndpointSpecifics
 
+      # These APIs are private, but were added in 8.x since the generator didn't consider
+      # visibility. They will be removed in 9.x since we're using a different generator that
+      # considers visibility. But new private APIs code won't be generated for the client.
+      EXCLUDED_8X = [
+        'autoscaling.delete_autoscaling_policy', 'autoscaling.get_autoscaling_capacity',
+        'autoscaling.get_autoscaling_policy', 'autoscaling.put_autoscaling_policy', 'capabilities',
+        'connector.secret_delete', 'connector.secret_get', 'connector.secret_post',
+        'connector.secret_put', 'fleet.delete_secret', 'fleet.get_secret', 'fleet.post_secret',
+        'ml.validate', 'ml.validate_detector', 'monitoring.bulk', 'profiling.flamegraph',
+        'profiling.stacktraces', 'profiling.status', 'profiling.topn_functions',
+        'security.activate_user_profile', 'security.disable_user_profile',
+        'security.enable_user_profile', 'security.get_user_profile',
+        'security.has_privileges_user_profile', 'security.suggest_user_profiles',
+        'security.update_user_profile_data', 'shutdown.delete_node', 'shutdown.get_node',
+        'shutdown.put_node'
+      ].freeze
+
       def initialize(filepath)
         @path = Pathname(filepath)
         json = MultiJson.load(File.read(@path))
@@ -31,14 +48,14 @@ module Elasticsearch
         @endpoint_name = json.keys.first
 
         full_namespace = parse_full_namespace
-        @namespace_depth = full_namespace.size > 0 ? full_namespace.size - 1 : 0
+        @namespace_depth = full_namespace.size.positive? ? full_namespace.size - 1 : 0
         @module_namespace = full_namespace[0, @namespace_depth]
         @method_name = full_namespace.last
 
         @path_parts = parse_endpoint_parts(@spec)
         @params = @spec['params'] || {}
         @paths = @spec['url']['paths'].map { |b| b['path'] } if @spec['url']
-        @path_params   = path_variables.flatten.uniq.collect(&:to_sym)
+        @path_params = path_variables.flatten.uniq.collect(&:to_sym)
         @http_method = parse_http_method(@spec)
         @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
         @http_path        = parse_http_path(@paths)
@@ -71,6 +88,16 @@ module Elasticsearch
         @spec['stability']
       end
 
+      def visibility
+        @spec['visibility']
+      end
+
+      def skippable?
+        return true if module_namespace.flatten.first == '_internal'
+
+        visibility != 'public' && !EXCLUDED_8X.include?(endpoint_name)
+      end
+
       # Function that adds the listified h param code
       def specific_params
         super(@module_namespace.first, @method_name)

data/utils/thor/endpoint_specifics.rb

@@ -39,6 +39,7 @@ module Elasticsearch
         indices.delete
         indices.delete_template
         indices.flush_synced
+        query_rules.delete_ruleset
         security.get_role
         security.get_user
         snapshot.delete
@@ -99,16 +100,14 @@ module Elasticsearch
         <<~SRC
           case
           when body.is_a?(Array) && body.any? { |d| d.has_key? :search }
-            payload = body.
-              inject([]) do |sum, item|
+            payload = body.inject([]) do |sum, item|
                 meta = item
                 data = meta.delete(:search)
 
                 sum << meta
                 sum << data
                 sum
-              end.
-              map { |item| Elasticsearch::API.serializer.dump(item) }
+              end.map { |item| Elasticsearch::API.serializer.dump(item) }
             payload << "" unless payload.empty?
             payload = payload.join("\\n")
           when body.is_a?(Array)

data/utils/thor/generate_source.rb

@@ -68,7 +68,7 @@ module Elasticsearch
           @spec = EndpointSpec.new(filepath)
           say_status 'json', @spec.path, :yellow
           # Don't generate code for internal APIs:
-          next if @spec.module_namespace.flatten.first == '_internal'
+          next if @spec.skippable?
 
           path_to_file = output.join(@spec.module_namespace.join('/')).join("#{@spec.method_name}.rb")
           dir = output.join(@spec.module_namespace.join('/'))