elasticsearch-api 8.9.0 → 8.11.0

data/spec/platinum/integration/eql/eql_async_spec.rb

@@ -0,0 +1,72 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../platinum_helper'
+
+describe 'eql/Executes async EQL with missing events' do
+  let(:index) { 'eql_test_m' }
+  before do
+    ADMIN_CLIENT.indices.create(
+      index: index,
+      body: {
+        mappings: {
+          properties: {
+            '@timestamp' => { type: 'date' },
+            'event.category' => { type: 'keyword' },
+            user: { type: 'keyword' }
+          }
+        }
+      }
+    )
+    ADMIN_CLIENT.bulk(
+      refresh: true,
+      body: [
+        { index: { _index: 'eql_test_m', _id: '1' } },
+        { event: [{ category: 'process' }], '@timestamp' => '2023-07-11T11:09:05.529Z', user: 'foo' },
+        { index: { _index: 'eql_test_m', _id: '2'} },
+        { event: [{ category: 'process' }], '@timestamp' => '2023-07-11T11:09:06.529Z', user: 'bar' }
+      ]
+    )
+  end
+
+  after do
+    ADMIN_CLIENT.indices.delete(index: index)
+  end
+
+  it 'Executes async EQL with missing events' do
+    response = ADMIN_CLIENT.eql.search(
+      index: 'eql_test_m',
+      wait_for_completion_timeout: '0ms',
+      keep_on_completion: true,
+      body: { query: 'sequence with maxspan=24h [ process where true ] ![ process where true ]' }
+    )
+    expect(response.status).to eq 200
+    expect(response['id']).not_to be_nil
+    id = response['id']
+
+    response = ADMIN_CLIENT.eql.get(id: id, wait_for_completion_timeout: '10s')
+    expect(response.status).to eq 200
+    expect(response['is_running']).to be false
+    expect(response['is_partial']).to be false
+    expect(response['timed_out']).to be false
+    expect(response['hits']['total']['value']).to eq 1
+    expect(response['hits']['total']['relation']).to eq 'eq'
+    expect(response['hits']['total']['relation']).to eq 'eq'
+    expect(response['hits']['sequences'].first['events'].first['_source']['user']).to eq 'bar'
+    expect(response['hits']['sequences'].first['events'].last['missing']).to be true
+  end
+end

data/spec/platinum/integration/ml/get_memory_stats_spec.rb

@@ -18,30 +18,30 @@
 require_relative '../platinum_helper'
 
 describe 'ML/Get Memory Stats API' do
-    it 'gets memory for all nodes' do
-      response = ADMIN_CLIENT.ml.get_memory_stats
-      expect(response.status).to eq 200
-      expect(response['cluster_name']).not_to be_nil
-      expect(response['_nodes']['total']).to be >= 1
-      expect(response['nodes'].first[1]['mem']['total_in_bytes']).to be > 1
-    end
+  it 'gets memory for all nodes' do
+    response = ADMIN_CLIENT.ml.get_memory_stats
+    expect(response.status).to eq 200
+    expect(response['cluster_name']).not_to be_nil
+    expect(response['_nodes']['total']).to be >= 1
+    expect(response['nodes'].first[1]['mem']['total_in_bytes']).to be > 1
+  end
 
-    it 'gets memory for ML nodes' do
-      response = ADMIN_CLIENT.ml.get_memory_stats
-      expect(response.status).to eq 200
-      expect(response['cluster_name']).not_to be_nil
-      expect(response['nodes'].first[1]['mem']['total_in_bytes']).to be > 1
-    end
+  it 'gets memory for ML nodes' do
+    response = ADMIN_CLIENT.ml.get_memory_stats
+    expect(response.status).to eq 200
+    expect(response['cluster_name']).not_to be_nil
+    expect(response['nodes'].first[1]['mem']['total_in_bytes']).to be > 1
+  end
 
-    it 'gets memory for specific node' do
-      response = ADMIN_CLIENT.ml.get_memory_stats
-      expect(response.status).to eq 200
-      node_id = response['nodes'].keys.first
+  it 'gets memory for specific node' do
+    response = ADMIN_CLIENT.ml.get_memory_stats
+    expect(response.status).to eq 200
+    node_id = response['nodes'].keys.first
 
-      response = ADMIN_CLIENT.ml.get_memory_stats(node_id: node_id, timeout: '29s')
-      expect(response.status).to eq 200
-      expect(response['cluster_name']).not_to be_nil
-      expect(response['nodes'].first[1]['mem']['total_in_bytes']).to be > 1
-      expect(response['_nodes']['total']).to be 1
-    end
+    response = ADMIN_CLIENT.ml.get_memory_stats(node_id: node_id, timeout: '29s')
+    expect(response.status).to eq 200
+    expect(response['cluster_name']).not_to be_nil
+    expect(response['nodes'].first[1]['mem']['total_in_bytes']).to be > 1
+    expect(response['_nodes']['total']).to be 1
   end
+end

data/spec/platinum/integration/platinum_helper.rb

@@ -34,3 +34,9 @@ TRANSPORT_OPTIONS = { ssl: { verify: false, client_cert: certificate, client_key
 HOST = "https://elastic:#{password}@#{HOST_URI.host}:#{HOST_URI.port}".freeze
 
 ADMIN_CLIENT = Elasticsearch::Client.new(host: HOST, transport_options: TRANSPORT_OPTIONS)
+
+RSpec.configure do |config|
+  config.add_formatter('documentation')
+  config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/platinum-integration.html")
+  config.color_mode = :on
+end

data/spec/platinum/integration/spatial/script_doc_values_spec.rb

@@ -0,0 +1,147 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../platinum_helper'
+
+describe 'Spatial API' do
+  context 'script doc values tests' do
+    index = 'geospatial_test'
+    before(:all) do
+      ADMIN_CLIENT.indices.create(
+        index: index,
+        body: {
+          settings: { number_of_shards: 1},
+          mappings: { properties: { geo_shape: { type: 'geo_shape' } } }
+        }
+      )
+      ADMIN_CLIENT.index(
+        index: index,
+        id: '1',
+        body: {
+          geo_shape: 'POLYGON((24.04725 59.942,24.04825 59.94125,24.04875 59.94125,24.04875 59.94175,24.048 59.9425,24.0475 59.94275,24.0465 59.94225,24.046 59.94225,24.04575 59.9425,24.04525 59.94225,24.04725 59.942))'
+        },
+        refresh: true
+      )
+      ADMIN_CLIENT.indices.refresh
+    end
+
+    after(:all) do
+      ADMIN_CLIENT.indices.delete(index: index)
+    end
+
+    it 'centroid' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { centroid: { script: { source: "doc['geo_shape'].getCentroid()" } } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['centroid'].first['lat']).to eq 59.942043484188616
+      expect(response['hits']['hits'].first['fields']['centroid'].first['lon']).to eq 24.047588920220733
+    end
+
+    it 'bounding box' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { bbox: { script: { source: "doc['geo_shape'].getBoundingBox()" } } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['bbox'].first['top_left']['lat']).to eq 59.942749994806945
+      expect(response['hits']['hits'].first['fields']['bbox'].first['top_left']['lon']).to eq 24.045249950140715
+      expect(response['hits']['hits'].first['fields']['bbox'].first['bottom_right']['lat']).to eq 59.94124996941537
+      expect(response['hits']['hits'].first['fields']['bbox'].first['bottom_right']['lon']).to eq 24.048749981448054
+    end
+
+    it 'label position' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { label_position: { script: { source: "doc['geo_shape'].getLabelPosition()" } } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['label_position'].first['lat']).to eq 59.942043484188616
+      expect(response['hits']['hits'].first['fields']['label_position'].first['lon']).to eq 24.047588920220733
+    end
+
+    it 'bounding box points' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: {
+            topLeft: { script: { source: "doc['geo_shape'].getBoundingBox().topLeft()" } },
+            bottomRight: { script: { source: "doc['geo_shape'].getBoundingBox().bottomRight()" } }
+          }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['topLeft'].first['lat']).to eq 59.942749994806945
+      expect(response['hits']['hits'].first['fields']['topLeft'].first['lon']).to eq 24.045249950140715
+      expect(response['hits']['hits'].first['fields']['bottomRight'].first['lat']).to eq 59.94124996941537
+      expect(response['hits']['hits'].first['fields']['bottomRight'].first['lon']).to eq 24.048749981448054
+    end
+
+    it 'dimensional type' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { type: { script: { source:  "doc['geo_shape'].getDimensionalType()" } } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['type'].first).to eq 2
+    end
+
+    it 'geoshape value' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { type: { script: { source:  "doc['geo_shape'].get(0)" } } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['type'].first).to eq '<unserializable>'
+
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { type: { script: { source:  "field('geo_shape').get(null)"} } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['type'].first).to eq '<unserializable>'
+
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: { type: { script: { source:  "$('geo_shape', null)"} } }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['type'].first).to eq '<unserializable>'
+    end
+
+    it 'diagonal length' do
+      response = ADMIN_CLIENT.search(
+        rest_total_hits_as_int: true,
+        body: {
+          script_fields: {
+            width: { script: { source: "doc['geo_shape'].getMercatorWidth()" } },
+            height: { script: { source: "doc['geo_shape'].getMercatorHeight()" } }
+          }
+        }
+      )
+      expect(response['hits']['hits'].first['fields']['width'].first).to eq 389.62170283915475
+      expect(response['hits']['hits'].first['fields']['height'].first).to eq 333.37976841442287
+    end
+  end
+end

data/spec/rest_api/rest_api_tests_helper.rb

@@ -97,4 +97,4 @@ REST_API_YAML_FILES = if ENV['RUN_SKIPPED_TESTS'] # only run the skipped tests i
                       end
 
 # The features to skip
-REST_API_YAML_SKIP_FEATURES = ['warnings', 'node_selector'].freeze
+REST_API_YAML_SKIP_FEATURES = ['warnings', 'node_selector', 'close_to'].freeze

data/spec/rest_api/rest_api_yaml_spec.rb

@@ -22,6 +22,8 @@ require_relative './run_rspec_matchers'
 LOGGER = Logger.new($stdout)
 
 describe 'Rest API YAML tests' do
+  LOGGER.info "Elastic Transport version: #{Elastic::Transport::VERSION}"
+
   if REST_API_YAML_FILES.empty?
     LOGGER.error 'No test files found!'
     LOGGER.info 'Use rake rake elasticsearch:download_artifacts in the root directory of the project to download the test artifacts.'

data/spec/rest_api/skipped_tests_platinum.yml

@@ -8,7 +8,6 @@
   :file: 'ml/explain_data_frame_analytics.yml'
   :description: 'Test non-empty data frame given body'
 
-# TODO https://github.com/elastic/elasticsearch-ruby/issues/852
 -
   :file: 'analytics/usage.yml'
   :description: 'Usage stats on analytics indices'
@@ -74,6 +73,9 @@
 -
   :file: 'ml/3rd_party_deployment.yml'
   :description: 'Test clear deployment cache'
+- # Wrong error matching
+  :file: 'ml/3rd_party_deployment.yml'
+  :description: 'Test getting and putting Fill Mask with two mask tokens, as well as exceptions caused by requests with the wrong token'
 
 -
   :file: 'searchable_snapshots/20_synthetic_source.yml'
@@ -93,3 +95,17 @@
 -
   :file: 'ml/get_memory_stats.yml'
   :description: '*'
+
+-
+  :file: 'eql/30_async_missing_events.yml'
+  :description: 'Execute async EQL with missing events'
+-
+  :file: 'api_key/60_admin_user.yml'
+  :description: 'Test get api key (with role descriptors + metadata)'
+-
+  :file: 'api_key/60_admin_user.yml'
+  :description: 'Test query api keys'
+
+-
+  :file: 'spatial/70_script_doc_values.yml'
+  :description: '*'

data/spec/spec_helper.rb

@@ -61,10 +61,6 @@ RSpec.configure do |config|
   config.add_formatter('documentation')
   config.add_formatter('RspecJunitFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}-junit.xml")
   config.add_formatter('RSpec::Core::Formatters::HtmlFormatter', "tmp/elasticsearch-#{ENV['TEST_SUITE']}-#{RUBY_VERSION}.html")
-  if ENV['BUILDKITE']
-    require_relative "./rspec_formatter.rb"
-    config.add_formatter('RSpecCustomFormatter')
-  end
   config.color_mode = :on
 end
 

data/utils/endpoint_spec.rb

@@ -0,0 +1,194 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# encoding: UTF-8
+
+require_relative 'endpoint_specifics'
+
+module Elasticsearch
+  module API
+    class EndpointSpec
+      include EndpointSpecifics
+
+      def initialize(filepath)
+        @path = Pathname(filepath)
+        json = MultiJson.load(File.read(@path))
+        @spec = json.values.first
+        @endpoint_name = json.keys.first
+
+        full_namespace = parse_full_namespace
+        @namespace_depth = full_namespace.size > 0 ? full_namespace.size - 1 : 0
+        @module_namespace = full_namespace[0, namespace_depth]
+        @method_name = full_namespace.last
+
+        @path_parts = parse_endpoint_parts(@spec)
+        @params = @spec['params'] || {}
+        @paths = @spec['url']['paths'].map { |b| b['path'] } if @spec['url']
+        @path_params   = path_variables.flatten.uniq.collect(&:to_sym)
+        @http_method = parse_http_method(@spec)
+        @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
+        @http_path        = parse_http_path(@paths)
+        @required_parts   = parse_required_parts(@spec)
+      end
+
+      attr_reader :module_namespace,
+                  :method_name,
+                  :endpoint_name,
+                  :path,
+                  :path_parts,
+                  :params,
+                  :deprecation_note,
+                  :namespace_depth,
+                  :http_path,
+                  :required_parts,
+                  :http_method,
+                  :namespace_depth,
+                  :path_params,
+                  :perform_request_opts
+
+      def body
+        @spec['body']
+      end
+
+      def documentation
+        @spec['documentation']
+      end
+
+      def stability
+        @spec['stability']
+      end
+
+      # Function that adds the listified h param code
+      def specific_params
+        super(@module_namespace.first, @method_name)
+      end
+
+      private
+
+      def parse_full_namespace
+        names = @endpoint_name.split('.')
+        # Return an array to expand 'ccr', 'ilm', 'ml' and 'slm'
+        names.map do |name|
+          name
+            .gsub(/^ml$/, 'machine_learning')
+            .gsub(/^ilm$/, 'index_lifecycle_management')
+            .gsub(/^ccr/, 'cross_cluster_replication')
+            .gsub(/^slm/, 'snapshot_lifecycle_management')
+        end
+      end
+
+      def parse_endpoint_parts(spec)
+        parts = spec['url']['paths'].select do |a|
+          a.keys.include?('parts')
+        end.map do |path|
+          path&.[]('parts')
+        end
+        (parts.inject(&:merge) || [])
+      end
+
+      def parse_http_method(spec)
+        return '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @endpoint_name == 'index'
+        return '_name ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @method_name == 'create_service_token'
+        return post_and_get if @endpoint_name == 'count'
+
+        default_method = spec['url']['paths'].map { |a| a['methods'] }.flatten.first
+        if spec['body'] && default_method == 'GET'
+          # When default method is GET and body is required, we should always use POST
+          if spec['body']['required']
+            'Elasticsearch::API::HTTP_POST'
+          else
+            post_and_get
+          end
+        else
+          "Elasticsearch::API::HTTP_#{default_method}"
+        end
+      end
+
+      def parse_http_path(paths)
+        return "\"#{parse_path(paths.first)}\"" if paths.size == 1
+
+        result = ''
+        anchor_string = []
+        paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
+          var_string = extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
+          next if anchor_string.include? var_string
+
+          anchor_string << var_string
+          result += if i.zero?
+                      "if #{var_string}\n"
+                    elsif (i == paths.size - 1) || var_string.empty?
+                      "else\n"
+                    else
+                      "elsif #{var_string}\n"
+                    end
+          result += "\"#{parse_path(path)}\"\n"
+        end
+        result += 'end'
+        result
+      end
+
+      def parse_path(path)
+        path.gsub(/^\//, '')
+            .gsub(/\/$/, '')
+            .gsub('{', "\#{Utils.__listify(_")
+            .gsub('}', ')}')
+      end
+
+      def path_variables
+        @paths.map do |path|
+          extract_path_variables(path)
+        end
+      end
+
+      def parse_path_variables
+        @paths.map do |path|
+          extract_path_variables(path)
+        end
+      end
+
+      # extract values that are in the {var} format:
+      def extract_path_variables(path)
+        path.scan(/{(\w+)}/).flatten
+      end
+
+      # Find parts that are definitely required and should raise an error if
+      # they're not present
+      #
+      def parse_required_parts(spec)
+        required = []
+        return required if @endpoint_name == 'tasks.get'
+
+        required << 'body' if (spec['body'] && spec['body']['required'])
+        # Get required variables from paths:
+        req_variables = parse_path_variables.inject(:&) # find intersection
+        required << req_variables unless req_variables.empty?
+        required.flatten
+      end
+
+      def post_and_get
+        # the METHOD is defined after doing arguments.delete(:body), so we need to check for `body`
+        <<~SRC
+          if body
+            Elasticsearch::API::HTTP_POST
+          else
+            Elasticsearch::API::HTTP_GET
+          end
+        SRC
+      end
+    end
+  end
+end

data/utils/{thor/generator/endpoint_specifics.rb → endpoint_specifics.rb}

@@ -55,10 +55,10 @@ module Elasticsearch
                     recovery shards thread_pool].freeze
 
       # Function that adds the listified h param code
-      def specific_params(namespace)
+      def specific_params(namespace, method_name)
         params = []
-        if H_PARAMS.include?(@method_name) && namespace == 'cat'
-          if @method_name == 'nodes'
+        if H_PARAMS.include?(method_name) && namespace == 'cat'
+          if method_name == 'nodes'
             params << 'params[:h] = Utils.__listify(params[:h], escape: false) if params[:h]'
           else
             params << 'params[:h] = Utils.__listify(params[:h]) if params[:h]'
@@ -84,7 +84,7 @@ module Elasticsearch
       def ping_perform_request
         <<~SRC
           begin
-            perform_request(method, path, params, body, headers).status == 200 ? true : false
+            perform_request(method, path, params, body, headers, request_opts).status == 200 ? true : false
           rescue Exception => e
             if e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i
               false