elasticsearch-api 8.12.2 → 8.14.0

data/spec/elasticsearch/api/perform_request_spec.rb

@@ -17,7 +17,7 @@
 
 require 'spec_helper'
 require 'elastic-transport'
-require_relative '../../../utils/endpoint_spec'
+require_relative '../../../utils/thor/endpoint_spec'
 require_relative '../../../utils/thor/generator/files_helper'
 
 describe 'Perform request args' do

data/spec/free/integration/health_spec.rb

@@ -0,0 +1,28 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# TODO: These tests won't be necessary once we migrate to the new YAML tests
+require_relative './spec_helper'
+
+describe "Health basic test" do
+  it 'performs the request' do
+    response = CLIENT.health_report
+    expect(response.status).to eq 200
+    expect(response['cluster_name']).not_to be_nil
+    expect(response.dig('indicators', 'master_is_stable', 'symptom')).to eq 'The cluster has a stable master node'
+    expect(response.dig('indicators', 'master_is_stable', 'status')).to eq 'green'
+  end
+end

data/spec/free/integration/spec_helper.rb

@@ -0,0 +1,28 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+require 'elasticsearch'
+
+host = ENV['TEST_ES_SERVER'] || 'http://localhost:9200'
+raise URI::InvalidURIError unless host =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
+
+test_suite = ENV['TEST_SUITE'] || 'free'
+password = ENV['ELASTIC_PASSWORD'] || 'changeme'
+uri = URI.parse(host)
+transport_options = {}
+host = "http://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
+
+CLIENT = Elasticsearch::Client.new(host: host, transport_options: transport_options)

data/spec/platinum/integration/api_key/api_key_cross_cluster_spec.rb

@@ -30,7 +30,7 @@ describe 'API keys API' do
       username: user,
       body: {
         password: password,
-        roles: [ 'admin_role' ],
+        roles: ['admin_role'],
         full_name: 'Admin user'
       }
     )
@@ -45,31 +45,22 @@ describe 'API keys API' do
     it 'updates api key' do
       response = ADMIN_CLIENT.security.create_cross_cluster_api_key(
         body: {
-          name: "my-cc-api-key",
-          expiration: "1d",
+          name: 'my-cc-api-key',
+          expiration: '1d',
           access: {
             search: [
-              {
-                names: ["logs*"],
-                query: {
-                  term: { "category": "shared" }
-                },
-                field_security: {
-                  grant: ["*"],
-                  except: ["private"]
-                }
-              }
+              { names: ['logs*'] }
             ],
             replication: [
               {
-                names: ["archive"],
+                names: ['archive'],
                 allow_restricted_indices: false
               }
             ]
           },
           metadata: {
             answer: 42,
-            tag: "dev"
+            tag: 'dev'
           }
         }
       )

data/spec/platinum/integration/api_key/api_key_query_with_aggs_spec.rb

@@ -0,0 +1,120 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../platinum_helper'
+
+describe 'API keys' do
+  before do
+    ADMIN_CLIENT.security.put_role(
+      name: 'admin_role',
+      body: { cluster: ['manage_api_key'] }
+    )
+    ADMIN_CLIENT.security.put_role(
+      name: 'user_role',
+      body: {
+        cluster: ['manage_own_api_key'],
+      }
+    )
+    ADMIN_CLIENT.security.put_user(
+      username: 'api_key_manager',
+      body: {
+        password: 'x-pack-test-password',
+        roles: [ 'admin_role' ],
+        full_name: 'API key manager'
+      }
+    )
+
+    ADMIN_CLIENT.security.put_user(
+      username: 'api_key_user',
+      body: {
+        password: 'x-pack-test-password',
+        roles: [ 'user_role' ],
+        full_name: 'API key user'
+      }
+    )
+  end
+  after do
+    ADMIN_CLIENT.security.delete_role(name: "admin_role", ignore: 404)
+    ADMIN_CLIENT.security.delete_role(name: "user_role", ignore: 404)
+    ADMIN_CLIENT.security.delete_user(username: "api_key_user", ignore: 404)
+    ADMIN_CLIENT.security.delete_user(username: "api_key_manager", ignore: 404)
+  end
+
+  def client(headers)
+    Elasticsearch::Client.new(
+      host: "https://#{HOST_URI.host}:#{HOST_URI.port}",
+      transport_options: TRANSPORT_OPTIONS.merge(headers: headers)
+    )
+  end
+
+  let(:manager_auth) do
+    { Authorization: "Basic YXBpX2tleV9tYW5hZ2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" }
+  end
+  let(:user_auth) do
+    { Authorization: "Basic YXBpX2tleV9tYW5hZ2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" }
+  end
+
+  it 'queries api key with args' do
+    # api_key_manager authorization:
+    client = client(manager_auth)
+    response = client.security.create_api_key(
+      body: {
+        name: 'manager-api-key',
+        expiration: '10d',
+        metadata: {
+          letter: 'a',
+          number: 42
+        }
+      }
+    )
+    expect(response.status).to eq 200
+    id = response['id']
+    api_key = response['api_key']
+    expect(response['encoded']).to eq Base64.strict_encode64("#{id}:#{api_key}")
+
+    client = client(user_auth)
+    response = client.security.create_api_key(
+      body: {
+        name: 'user-api-key',
+        expiration: '1d',
+        metadata: {
+          letter: 'a',
+          number: 1
+        }
+      }
+    )
+    expect(response.status).to eq 200
+    id = response['id']
+    api_key = response['api_key']
+    expect(response['encoded']).to eq Base64.strict_encode64("#{id}:#{api_key}")
+
+    response = client.security.query_api_keys(
+      body: {
+        size: 0,
+        aggs: {
+          my_buckets: {
+            composite: {
+              sources: [{ key_name: { terms: { field: "name" } } } ]
+            }
+          }
+        }
+      }
+    )
+    expect(response.dig('aggregations', 'my_buckets', 'buckets').find { |a| a.dig('key', 'key_name') == 'manager-api-key'})
+    expect(response.dig('aggregations', 'my_buckets', 'buckets').find { |a| a.dig('key', 'key_name') == 'user-api-key'})
+  end
+end

data/spec/platinum/integration/platinum_helper.rb

@@ -37,6 +37,6 @@ ADMIN_CLIENT = Elasticsearch::Client.new(host: HOST, transport_options: TRANSPOR
 
 RSpec.configure do |config|
   config.add_formatter('documentation')
-  config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/platinum-integration.html")
+  config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/platinum-#{ENV['RUBY_VERSION']}-manual-integration.html")
   config.color_mode = :on
 end

data/spec/platinum/integration/security/users_query_spec.rb

@@ -0,0 +1,49 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../platinum_helper'
+
+describe 'security/Query User' do
+  before do
+    ADMIN_CLIENT.security.put_user(
+      username: "query_users_test",
+      body: {
+        password: "x-pack-test-password",
+        roles: [ "manage_users_role" ],
+        full_name: "Read User"
+      }
+    )
+  end
+
+  it 'queries users' do
+    response = ADMIN_CLIENT.security.query_user(body: {})
+    expect(response.status).to eq 200
+    expect(response['count']).to be > 0
+    expect(response['total']).to be > 0
+  end
+
+  it 'queries user and finds created one' do
+    body = {
+      query: { wildcard: { username: 'query_users_*' } }
+    }
+    response = ADMIN_CLIENT.security.query_user(body: body)
+    expect(response.status).to eq 200
+    expect(response['count']).to be 1
+    expect(response['total']).to be 1
+    expect(response['users'].first['username']).to eq 'query_users_test'
+  end
+end

data/spec/rest_api/rest_api_yaml_spec.rb

@@ -30,6 +30,8 @@ describe 'Rest API YAML tests' do
     exit 1
   end
 
+  CLUSTER_FEATURES = ADMIN_CLIENT.features.get_features['features'].map { |f| f['name'] }
+
   # Traverse YAML files and create TestFile object:
   REST_API_YAML_FILES.each do |file|
     begin
@@ -46,6 +48,10 @@ describe 'Rest API YAML tests' do
       let(:client) { DEFAULT_CLIENT }
 
       test_file.tests.each do |test|
+        # To support new features skipping in YAML tests. This will go away with new YAML tests:
+        if (feature_to_skip = test.skip&.first&.[]('skip')&.[]('cluster_features'))
+          next unless CLUSTER_FEATURES.include? feature_to_skip
+        end
         context test.description do
           if test.skip_test?(ADMIN_CLIENT)
             skip 'Test contains feature(s) not yet supported or version is not satisfied'

data/spec/rest_api/skipped_tests_free.yml

@@ -31,9 +31,54 @@
 -
   :file: 'cluster.desired_nodes/10_basic.yml'
   :description: 'Test update desired nodes is idempotent with different order'
+-
+  :file: 'cluster.desired_nodes/10_basic.yml'
+  :description: 'Test node version must be at least the current master version'
+-
+  :file: 'cluster.desired_nodes/10_basic.yml'
+  :description: 'Test node version must have content'
 -
   :file: 'search.vectors/40_knn_search.yml'
   :description: 'Direct kNN queries are disallowed'
 -
   :file: 'search.vectors/45_knn_search_byte.yml'
   :description: 'Direct kNN queries are disallowed'
+-
+  :file: 'tsdb/25_id_generation.yml'
+  :description: 'generates a consistent id'
+-
+  :file: 'indices.resolve_cluster/10_basic_resolve_cluster.yml'
+  :description: 'Resolve cluster using alias and index that does not exist'
+-
+  :file: 'health/10_basic.yml' # Migrated to written manually
+  :description: 'cluster health basic test'
+-
+  :file: 'cat.allocation/10_basic.yml'
+  :description: 'One index'
+-
+  :file: 'cat.allocation/10_basic.yml'
+  :description: 'Node ID'
+-
+  :file: 'cat.allocation/10_basic.yml'
+  :description: 'All Nodes'
+-
+  :file: 'cat.allocation/10_basic.yml'
+  :description: 'Column headers'
+-
+  :file: 'cat.allocation/10_basic.yml'
+  :description: 'Bytes'
+-
+  :file: 'search.vectors/120_knn_query_multiple_shards.yml'
+  :description: 'Search for 2 knn queries combines scores from them'
+-
+  :file: 'search.vectors/140_knn_query_with_other_queries.yml'
+  :description: 'Function score query with knn query'
+-
+  :file: 'search.vectors/60_knn_search_filter_alias.yml'
+  :description: 'kNN filter alias'
+-
+  :file: 'cluster.desired_nodes/11_old_format.yml'
+  :description: 'Test node version must be at least the current master version'
+-
+  :file: 'cluster.desired_nodes/11_old_format.yml'
+  :description: 'Test node version must have content'

data/spec/rest_api/skipped_tests_platinum.yml

@@ -112,3 +112,9 @@
 -
   :file: 'esql/60_usage.yml'
   :description: 'Basic ESQL usage output (telemetry)'
+-
+  :file: 'users/40_query.yml'
+  :description: 'Test query user'
+-
+  :file: 'api_key/21_query_with_aggs.yml'
+  :description: 'Test composite aggs api key'

data/{lib/elasticsearch/api/namespace/connector_sync_job.rb → spec/rspec_formatter.rb}

@@ -15,22 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
-module Elasticsearch
-  module API
-    module ConnectorSyncJob
-      module Actions; end
+class RSpecCustomFormatter < RSpec::Core::Formatters::JsonFormatter
+  RSpec::Core::Formatters.register self
 
-      # Client for the "connector_sync_job" namespace (includes the {ConnectorSyncJob::Actions} methods)
-      #
-      class ConnectorSyncJobClient
-        include Common::Client, Common::Client::Base, ConnectorSyncJob::Actions
-      end
+  LOGFILE = File.new("tmp/rspec-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}.log", 'w')
 
-      # Proxy method for {ConnectorSyncJobClient}, available in the receiving object
-      #
-      def connector_sync_job
-        @connector_sync_job ||= ConnectorSyncJobClient.new(self)
-      end
+  def close(_notification)
+    @output_hash[:examples].map do |example|
+      regexp = /\S+\/\S+/
+      filename = example[:full_description].match(/\S+\/\S+/)[0]
+      log ="#{filename} | #{example[:status]} | #{example[:run_time]} | #{example[:full_description]}\n"
+      File.write(LOGFILE, log, mode: 'a')
     end
   end
 end

data/spec/spec_helper.rb

@@ -22,7 +22,7 @@ end
 if defined?(JRUBY_VERSION)
   require 'pry-nav'
 else
-  require 'pry-byebug'
+  require 'debug'
 end
 require 'yaml'
 require 'active_support/isolated_execution_state' unless RUBY_VERSION < '2.7.0'
@@ -59,8 +59,15 @@ end
 RSpec.configure do |config|
   config.include(HelperModule)
   config.add_formatter('documentation')
-  config.add_formatter('RspecJunitFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}-junit.xml")
-  config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}.html")
+  if defined?(JRUBY_VERSION)
+    config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-jruby-#{JRUBY_VERSION}.html")
+  else
+    config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}.html")
+  end
+  if ENV['BUILDKITE']
+    require_relative "./rspec_formatter.rb"
+    config.add_formatter('RSpecCustomFormatter')
+  end
   config.color_mode = :on
 end
 

data/utils/Gemfile

@@ -17,8 +17,8 @@
 
 source 'https://rubygems.org'
 
-gem 'byebug' unless defined?(JRUBY_VERSION)
 gem 'coderay'
+gem 'debug' unless defined?(JRUBY_VERSION)
 gem 'json'
 gem 'multi_json'
 gem 'pry'

data/utils/{endpoint_spec.rb → thor/endpoint_spec.rb}

@@ -32,7 +32,7 @@ module Elasticsearch
 
         full_namespace = parse_full_namespace
         @namespace_depth = full_namespace.size > 0 ? full_namespace.size - 1 : 0
-        @module_namespace = full_namespace[0, namespace_depth]
+        @module_namespace = full_namespace[0, @namespace_depth]
         @method_name = full_namespace.last
 
         @path_parts = parse_endpoint_parts(@spec)
@@ -56,7 +56,6 @@ module Elasticsearch
                   :http_path,
                   :required_parts,
                   :http_method,
-                  :namespace_depth,
                   :path_params,
                   :perform_request_opts
 

data/utils/{endpoint_specifics.rb → thor/endpoint_specifics.rb}

@@ -136,11 +136,11 @@ module Elasticsearch
 
       def bulk_body_helper
         <<~SRC
-          if body.is_a? Array
-            payload = Elasticsearch::API::Utils.__bulkify(body)
-          else
-            payload = body
-          end
+          payload = if body.is_a? Array
+                      Elasticsearch::API::Utils.__bulkify(body)
+                    else
+                      body
+                    end
         SRC
       end
 

data/utils/thor/generate_source.rb

@@ -22,8 +22,10 @@ require 'pathname'
 require 'multi_json'
 require 'coderay'
 require 'pry'
+require_relative 'generator/build_hash_helper'
+require_relative 'generator/docs_helper'
 require_relative 'generator/files_helper'
-require_relative '../endpoint_spec'
+require_relative './endpoint_spec'
 
 module Elasticsearch
   module API
@@ -41,20 +43,17 @@ module Elasticsearch
       namespace 'code'
       include Thor::Actions
       include EndpointSpecifics
+      include DocsHelper
 
       desc 'generate', 'Generate source code and tests from the REST API JSON specification'
       method_option :verbose, type: :boolean, default: false,  desc: 'Output more information'
       method_option :tests,   type: :boolean, default: false,  desc: 'Generate test files'
 
       def generate
-        @build_hash = if ENV['BUILD_HASH']
-                        File.read(File.expand_path('../../../tmp/rest-api-spec/build_hash',__dir__))
-                      else
-                        original_build_hash
-                      end
-
+        @build_hash = BuildHashHelper.build_hash
         self.class.source_root File.expand_path(__dir__)
         generate_source
+
         # -- Tree output
         print_tree if options[:verbose]
       end
@@ -68,101 +67,22 @@ module Elasticsearch
         FilesHelper.files.each do |filepath|
           @spec = EndpointSpec.new(filepath)
           say_status 'json', @spec.path, :yellow
-
           # Don't generate code for internal APIs:
           next if @spec.module_namespace.flatten.first == '_internal'
 
           path_to_file = output.join(@spec.module_namespace.join('/')).join("#{@spec.method_name}.rb")
           dir = output.join(@spec.module_namespace.join('/'))
-
           empty_directory(dir, verbose: false)
 
           # Write the file with the ERB template:
           template('templates/method.erb', path_to_file, force: true)
 
+          # Optionals:
           print_source_code(path_to_file) if options[:verbose]
-
           generate_tests if options[:tests]
-
-          puts
         end
-
         run_rubocop
-        add_hash
-      end
-
-      def add_hash
-        Dir.glob("#{FilesHelper.output_dir}/**/*.rb").each do |file|
-          content = File.read(file)
-          new_content = content.gsub(/(^#\sunder\sthe\sLicense.\n#)/) do |_|
-            match = Regexp.last_match
-            "#{match[1]}\n#{build_hash_comment}"
-          end
-          File.open(file, 'w') { |f| f.puts new_content }
-        end
-      end
-
-      def build_hash_comment
-        [
-          "Auto generated from build hash #{@build_hash}",
-          '@see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec',
-          ''
-        ].map { |b| "# #{b}" }.join("\n").strip
-      end
-
-      def original_build_hash
-        content = File.read("#{FilesHelper.output_dir}/info.rb")
-        return unless (match = content.match(/Auto generated from build hash ([a-f0-9]+)/))
-
-        match[1]
-      end
-
-      # Create the hierarchy of directories based on API namespaces
-      #
-      def create_directories(key, value)
-        return if value['documentation']
-
-        empty_directory @output.join(key)
-        create_directory_hierarchy * value.to_a.first
-      end
-
-      def docs_helper(name, info)
-        info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings'
-        info['type'] = 'Integer' if info['type'] == 'int' # Rename 'int' to 'Integer'
-        tipo = info['type'] ? info['type'].capitalize : 'String'
-        description = info['description'] ? info['description'].strip : '[TODO]'
-        options = info['options'] ? "(options: #{info['options'].join(', ').strip})" : nil
-        required = info['required'] ? '(*Required*)' : ''
-        deprecated = info['deprecated'] ? '*Deprecated*' : ''
-        optionals = [required, deprecated, options].join(' ').strip
-
-        "# @option arguments [#{tipo}] :#{name} #{description} #{optionals}\n"
-      end
-
-      def stability_doc_helper(stability)
-        return if stability == 'stable'
-
-        if stability == 'experimental'
-          <<~MSG
-            # This functionality is Experimental and may be changed or removed
-            # completely in a future release. Elastic will take a best effort approach
-            # to fix any issues, but experimental features are not subject to the
-            # support SLA of official GA features.
-          MSG
-        elsif stability == 'beta'
-          <<~MSG
-            # This functionality is in Beta and is subject to change. The design and
-            # code is less mature than official GA features and is being provided
-            # as-is with no warranties. Beta features are not subject to the support
-            # SLA of official GA features.
-          MSG
-        else
-          <<~MSG
-            # This functionality is subject to potential breaking changes within a
-            # minor version, meaning that your referencing code may break when this
-            # library is upgraded.
-          MSG
-        end
+        BuildHashHelper.add_hash(@build_hash)
       end
 
       def generate_tests
@@ -200,7 +120,7 @@ module Elasticsearch
       end
 
       def run_rubocop
-        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -x #{FilesHelper::output_dir}")
+        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -a #{FilesHelper::output_dir}")
       end
     end
   end