elasticsearch-api 8.17.2 → 9.0.0

data/spec/yaml-test-runner/run.rb

@@ -1,67 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-require 'logger'
-require 'openssl'
-require 'elasticsearch'
-require 'elasticsearch/tests/test_runner'
-require 'elasticsearch/tests/downloader'
-
-PROJECT_PATH = File.join(File.dirname(__FILE__), '../..')
-CERTS_PATH = "#{PROJECT_PATH}/../.buildkite/certs/".freeze
-host = ENV['TEST_ES_SERVER'] || 'https://localhost:9200'
-raise URI::InvalidURIError unless host =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
-
-password = ENV['ELASTIC_PASSWORD'] || 'changeme'
-uri = URI.parse(host)
-
-if uri.is_a?(URI::HTTPS)
-  raw_certificate = File.read("#{CERTS_PATH}/testnode.crt")
-  certificate = OpenSSL::X509::Certificate.new(raw_certificate)
-  raw_key = File.read("#{CERTS_PATH}/testnode.key")
-  key = OpenSSL::PKey::RSA.new(raw_key)
-  ca_file = File.expand_path("#{CERTS_PATH}/ca.crt")
-  host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
-  transport_options = {
-    ssl: {
-      client_cert: certificate,
-      client_key: key,
-      ca_file: ca_file,
-      verify: false
-    }
-  }
-elsif uri.is_a?(URI::HTTP)
-  host = "http://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
-  transport_options = {}
-end
-
-CLIENT = if ENV['ES_API_KEY']
-           Elasticsearch::Client.new(host: host, api_key: ENV['ES_API_KEY'], transport_options: transport_options)
-         else
-           Elasticsearch::Client.new(host: host, transport_options: transport_options)
-         end
-
-tests_path = File.expand_path('./tmp', __dir__)
-
-logger = Logger.new($stdout)
-logger.level = Logger::WARN unless ENV['DEBUG']
-
-# If we're running in a release branch, download the corresponding branch for tests
-current_branch = `git rev-parse --abbrev-ref HEAD`.strip
-branch = current_branch.match(/[0-9]\.[0-9]+/)&.[](0) || ENV['ES_YAML_TESTS_BRANCH'] || nil
-Elasticsearch::Tests::Downloader::run(tests_path, branch)
-Elasticsearch::Tests::TestRunner.new(CLIENT, tests_path, logger).run

data/utils/Gemfile

@@ -1,28 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-source 'https://rubygems.org'
-
-gem 'coderay'
-
-gem 'json'
-gem 'multi_json'
-gem 'pry'
-gem 'thor'
-
-gem 'rubocop', '>= 1.51' unless defined?(JRUBY_VERSION)
-gem 'debug' unless defined?(JRUBY_VERSION)

data/utils/README.md

@@ -1,48 +0,0 @@
-# Utils
-
-## The Generator
-
-This directory hosts The Generator, a tool that generates the classes for each API endpoint from the [Elasticsearch REST API JSON Specification](https://github.com/elastic/elasticsearch/tree/main/rest-api-spec).
-
-### Generate
-
-To generate the code, you need to have the Elasticsearch REST API spec files in `tmp/rest-api-spec` in the root of the project. You can run a rake task from the root of the project to download the specs corresponding to the current running cluster:
-```bash
-$ rake es:download_artifacts
-```
-
-Once the JSON files have been downloaded, you need to run (from this folder):
-```bash
-$ thor code:generate
-```
-
-- The Ruby code will be generated in `elasticsearch-api/lib/elasticsearch/api/actions`.
-- The generator runs Rubocop to autolint and clean up the generated files.
-- You can use the environment variable `IGNORE_VERSION` to ignore the current version of the client when generating the source code documentation urls. This is currently used when generating code from `main`:
-
-```bash
-$ IGNORE_VERSION=true thor code:generate
-```
-
-- You can use the environment variable `BUILD_HASH` to update the build hash for the generated code from the `tmp/rest-api-spec/build_hash` file. This file is updated every time you use the `es:download_artifacts` Rake task is used in the root of the project to download the latest Elasticsearch specs and tests:
-```bash
-$ BUILD_HASH=true thor code:generate
-```
-
-### Development
-
-The main entry point is `generate_source.rb`, which contains a class that implements a Thor task: `generate`:
-
-```
-$ thor api:code:generate
-```
-
-It uses [Thor::Actions](https://github.com/erikhuda/thor/wiki/Actions)' `template` method and `templates/method.erb` to generate the final code. The `generator` directory contains some helpers used to generate the code. The ERB template is split into partials and you can find all ERB files in the `templates` directory.
-
-There's also a lister task:
-
-```
-$ thor api:list
-```
-
-It's implemented in `lister.rb` and it lists all the REST API endpoints from the JSON specification.

data/utils/Thorfile

@@ -1,6 +0,0 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
-
-require File.expand_path('./thor/generate_source')
-require File.expand_path('./thor/lister')

data/utils/thor/.rubocop.yml

@@ -1,4 +0,0 @@
-Layout/EndAlignment:
-  AutoCorrect: true
-Layout/EmptyComment:
-  Enabled: false

data/utils/thor/endpoint_spec.rb

@@ -1,220 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# encoding: UTF-8
-
-require_relative 'endpoint_specifics'
-
-module Elasticsearch
-  module API
-    class EndpointSpec
-      include EndpointSpecifics
-
-      # These APIs are private, but were added in 8.x since the generator didn't consider
-      # visibility. They will be removed in 9.x since we're using a different generator that
-      # considers visibility. But new private APIs code won't be generated for the client.
-      EXCLUDED_8X = [
-        'autoscaling.delete_autoscaling_policy', 'autoscaling.get_autoscaling_capacity',
-        'autoscaling.get_autoscaling_policy', 'autoscaling.put_autoscaling_policy', 'capabilities',
-        'connector.secret_delete', 'connector.secret_get', 'connector.secret_post',
-        'connector.secret_put', 'fleet.delete_secret', 'fleet.get_secret', 'fleet.post_secret',
-        'ml.validate', 'ml.validate_detector', 'monitoring.bulk', 'profiling.flamegraph',
-        'profiling.stacktraces', 'profiling.status', 'profiling.topn_functions',
-        'security.activate_user_profile', 'security.disable_user_profile',
-        'security.enable_user_profile', 'security.get_user_profile',
-        'security.has_privileges_user_profile', 'security.suggest_user_profiles',
-        'security.update_user_profile_data', 'shutdown.delete_node', 'shutdown.get_node',
-        'shutdown.put_node'
-      ].freeze
-
-      def initialize(filepath)
-        @path = Pathname(filepath)
-        json = MultiJson.load(File.read(@path))
-        @spec = json.values.first
-        @endpoint_name = json.keys.first
-
-        full_namespace = parse_full_namespace
-        @namespace_depth = full_namespace.size.positive? ? full_namespace.size - 1 : 0
-        @module_namespace = full_namespace[0, @namespace_depth]
-        @method_name = full_namespace.last
-
-        @path_parts = parse_endpoint_parts(@spec)
-        @params = @spec['params'] || {}
-        @paths = @spec['url']['paths'].map { |b| b['path'] } if @spec['url']
-        @path_params = path_variables.flatten.uniq.collect(&:to_sym)
-        @http_method = parse_http_method(@spec)
-        @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
-        @http_path        = parse_http_path(@paths)
-        @required_parts   = parse_required_parts(@spec)
-      end
-
-      attr_reader :module_namespace,
-                  :method_name,
-                  :endpoint_name,
-                  :path,
-                  :path_parts,
-                  :params,
-                  :deprecation_note,
-                  :namespace_depth,
-                  :http_path,
-                  :required_parts,
-                  :http_method,
-                  :path_params,
-                  :perform_request_opts
-
-      def body
-        @spec['body']
-      end
-
-      def documentation
-        @spec['documentation']
-      end
-
-      def stability
-        @spec['stability']
-      end
-
-      def visibility
-        @spec['visibility']
-      end
-
-      def skippable?
-        return true if module_namespace.flatten.first == '_internal'
-
-        visibility != 'public' && !EXCLUDED_8X.include?(endpoint_name)
-      end
-
-      # Function that adds the listified h param code
-      def specific_params
-        super(@module_namespace.first, @method_name)
-      end
-
-      private
-
-      def parse_full_namespace
-        names = @endpoint_name.split('.')
-        # Return an array to expand 'ccr', 'ilm', 'ml' and 'slm'
-        names.map do |name|
-          name
-            .gsub(/^ml$/, 'machine_learning')
-            .gsub(/^ilm$/, 'index_lifecycle_management')
-            .gsub(/^ccr/, 'cross_cluster_replication')
-            .gsub(/^slm/, 'snapshot_lifecycle_management')
-        end
-      end
-
-      def parse_endpoint_parts(spec)
-        parts = spec['url']['paths'].select do |a|
-          a.keys.include?('parts')
-        end.map do |path|
-          path&.[]('parts')
-        end
-        (parts.inject(&:merge) || [])
-      end
-
-      def parse_http_method(spec)
-        return '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @endpoint_name == 'index'
-        return '_name ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @method_name == 'create_service_token'
-        return post_and_get if @endpoint_name == 'count'
-
-        default_method = spec['url']['paths'].map { |a| a['methods'] }.flatten.first
-        if spec['body'] && default_method == 'GET'
-          # When default method is GET and body is required, we should always use POST
-          if spec['body']['required']
-            'Elasticsearch::API::HTTP_POST'
-          else
-            post_and_get
-          end
-        else
-          "Elasticsearch::API::HTTP_#{default_method}"
-        end
-      end
-
-      def parse_http_path(paths)
-        return "\"#{parse_path(paths.first)}\"" if paths.size == 1
-
-        result = ''
-        anchor_string = []
-        paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
-          var_string = extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
-          next if anchor_string.include? var_string
-
-          anchor_string << var_string
-          result += if i.zero?
-                      "if #{var_string}\n"
-                    elsif (i == paths.size - 1) || var_string.empty?
-                      "else\n"
-                    else
-                      "elsif #{var_string}\n"
-                    end
-          result += "\"#{parse_path(path)}\"\n"
-        end
-        result += 'end'
-        result
-      end
-
-      def parse_path(path)
-        path.gsub(/^\//, '')
-            .gsub(/\/$/, '')
-            .gsub('{', "\#{Utils.__listify(_")
-            .gsub('}', ')}')
-      end
-
-      def path_variables
-        @paths.map do |path|
-          extract_path_variables(path)
-        end
-      end
-
-      def parse_path_variables
-        @paths.map do |path|
-          extract_path_variables(path)
-        end
-      end
-
-      # extract values that are in the {var} format:
-      def extract_path_variables(path)
-        path.scan(/{(\w+)}/).flatten
-      end
-
-      # Find parts that are definitely required and should raise an error if
-      # they're not present
-      #
-      def parse_required_parts(spec)
-        required = []
-        return required if @endpoint_name == 'tasks.get'
-
-        required << 'body' if (spec['body'] && spec['body']['required'])
-        # Get required variables from paths:
-        req_variables = parse_path_variables.inject(:&) # find intersection
-        required << req_variables unless req_variables.empty?
-        required.flatten
-      end
-
-      def post_and_get
-        # the METHOD is defined after doing arguments.delete(:body), so we need to check for `body`
-        <<~SRC
-          if body
-            Elasticsearch::API::HTTP_POST
-          else
-            Elasticsearch::API::HTTP_GET
-          end
-        SRC
-      end
-    end
-  end
-end

data/utils/thor/endpoint_specifics.rb

@@ -1,158 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-module Elasticsearch
-  module API
-    # Handles specific exceptional parameters and code snippets that need to be
-    # included in the generated code. This module is included in SourceGenerator
-    # so its methods can be used from the ERB template (method.erb). This will
-    # potentially be refactored into different templates.
-    module EndpointSpecifics
-      # Endpoints that need Utils.__rescue_from_not_found
-      IGNORE_404 = %w[
-        exists
-        indices.exists
-        indices.exists_alias
-        indices.exists_template
-      ].freeze
-
-      # Endpoints that need Utils.__rescue_from_not_found if the ignore
-      # parameter is included
-      COMPLEX_IGNORE_404 = %w[
-        clear_scroll
-        delete
-        get
-        indices.delete
-        indices.delete_template
-        indices.flush_synced
-        query_rules.delete_ruleset
-        security.get_role
-        security.get_user
-        snapshot.delete
-        snapshot.delete_repository
-        snapshot.get
-        snapshot.get_repository
-        snapshot.status
-        update
-        watcher.delete_watch
-      ].freeze
-
-      # Endpoints that need params[:h] listified
-      H_PARAMS = %w[aliases allocation count health indices nodes pending_tasks
-                    recovery shards thread_pool].freeze
-
-      # Function that adds the listified h param code
-      def specific_params(namespace, method_name)
-        params = []
-        if H_PARAMS.include?(method_name) && namespace == 'cat'
-          if method_name == 'nodes'
-            params << 'params[:h] = Utils.__listify(params[:h], escape: false) if params[:h]'
-          else
-            params << 'params[:h] = Utils.__listify(params[:h]) if params[:h]'
-          end
-        end
-        params
-      end
-
-      def needs_ignore_404?(endpoint)
-        IGNORE_404.include? endpoint
-      end
-
-      def needs_complex_ignore_404?(endpoint)
-        COMPLEX_IGNORE_404.include? endpoint
-      end
-
-      def module_name_helper(name)
-        return name.upcase if %w[sql ssl].include? name
-
-        name.split('_').map(&:capitalize).map{ |n| n == 'Xpack' ? 'XPack' : n }.join
-      end
-
-      def ping_perform_request
-        <<~SRC
-          begin
-            perform_request(method, path, params, body, headers, request_opts).status == 200 ? true : false
-          rescue Exception => e
-            if e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i
-              false
-            else
-              raise e
-            end
-          end
-        SRC
-      end
-
-      def msearch_body_helper
-        <<~SRC
-          case
-          when body.is_a?(Array) && body.any? { |d| d.has_key? :search }
-            payload = body.inject([]) do |sum, item|
-                meta = item
-                data = meta.delete(:search)
-
-                sum << meta
-                sum << data
-                sum
-              end.map { |item| Elasticsearch::API.serializer.dump(item) }
-            payload << "" unless payload.empty?
-            payload = payload.join("\\n")
-          when body.is_a?(Array)
-            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
-            payload << "" unless payload.empty?
-            payload = payload.join("\\n")
-          else
-            payload = body
-          end
-        SRC
-      end
-
-      def msearch_template_body_helper
-        <<~SRC
-          case
-          when body.is_a?(Array)
-            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
-            payload << "" unless payload.empty?
-            payload = payload.join("\n")
-          else
-            payload = body
-          end
-        SRC
-      end
-
-      def bulk_body_helper
-        <<~SRC
-          payload = if body.is_a? Array
-                      Elasticsearch::API::Utils.__bulkify(body)
-                    else
-                      body
-                    end
-        SRC
-      end
-
-      def find_structure_body_helper
-        bulk_body_helper
-      end
-
-      def bulk_doc_helper(info)
-        <<~SRC
-          # @option arguments [String|Array] :body #{info}. Array of Strings, Header/Data pairs,
-          # or the conveniency "combined" format can be passed, refer to Elasticsearch::API::Utils.__bulkify documentation.
-        SRC
-      end
-    end
-  end
-end

data/utils/thor/generate_source.rb

@@ -1,127 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# encoding: UTF-8
-
-require 'thor'
-require 'pathname'
-require 'multi_json'
-require 'coderay'
-require 'pry'
-require_relative 'generator/build_hash_helper'
-require_relative 'generator/docs_helper'
-require_relative 'generator/files_helper'
-require_relative './endpoint_spec'
-
-module Elasticsearch
-  module API
-    # A command line application based on [Thor](https://github.com/wycats/thor),
-    # which will read the JSON API spec file(s), and generate
-    # the Ruby source code (one file per API endpoint) with correct
-    # module namespace, method names, and RDoc documentation,
-    # as well as test files for each endpoint.
-    #
-    # Specific exceptions and code snippets that need to be included are written
-    # in EndpointSpecifics (generator/endpoint_specifics) and the module is included
-    # here.
-    #
-    class SourceGenerator < Thor
-      namespace 'code'
-      include Thor::Actions
-      include EndpointSpecifics
-      include DocsHelper
-
-      desc 'generate', 'Generate source code and tests from the REST API JSON specification'
-      method_option :verbose, type: :boolean, default: false,  desc: 'Output more information'
-      method_option :tests,   type: :boolean, default: false,  desc: 'Generate test files'
-
-      def generate
-        @build_hash = BuildHashHelper.build_hash
-        self.class.source_root File.expand_path(__dir__)
-        generate_source
-
-        # -- Tree output
-        print_tree if options[:verbose]
-      end
-
-      private
-
-      def generate_source
-        output = FilesHelper.output_dir
-        cleanup_directory!(output)
-
-        FilesHelper.files.each do |filepath|
-          @spec = EndpointSpec.new(filepath)
-          say_status 'json', @spec.path, :yellow
-          # Don't generate code for internal APIs:
-          next if @spec.skippable?
-
-          path_to_file = output.join(@spec.module_namespace.join('/')).join("#{@spec.method_name}.rb")
-          dir = output.join(@spec.module_namespace.join('/'))
-          empty_directory(dir, verbose: false)
-
-          # Write the file with the ERB template:
-          template('templates/method.erb', path_to_file, force: true)
-
-          # Optionals:
-          print_source_code(path_to_file) if options[:verbose]
-          generate_tests if options[:tests]
-        end
-        run_rubocop
-        BuildHashHelper.add_hash(@build_hash)
-      end
-
-      def generate_tests
-        copy_file 'templates/test_helper.rb', @output.join('test').join('test_helper.rb')
-
-        @test_directory = @output.join('test/api').join(@module_namespace.join('/'))
-        @test_file      = @test_directory.join("#{@method_name}_test.rb")
-
-        empty_directory @test_directory
-        template 'templates/test.erb', @test_file
-
-        print_source_code(@test_file) if options[:verbose]
-      end
-
-      def print_source_code(path_to_file)
-        colorized_output = CodeRay.scan_file(path_to_file, :ruby).terminal
-        lines            = colorized_output.split("\n")
-        formatted        = lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n")
-
-        say_status('ruby', formatted, :yellow)
-      end
-
-      def print_tree
-        return unless `which tree > /dev/null 2>&1; echo $?`.to_i < 1
-
-        lines = `tree #{@output}`.split("\n")
-        say_status('tree', lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n"))
-      end
-
-      def cleanup_directory!(output)
-        Dir["#{output}/**/*.rb"].each do |file|
-          # file = File.join(@output, f)
-          File.delete(file) unless (['.', '..'].include? file) || Pathname(file).directory?
-        end
-      end
-
-      def run_rubocop
-        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -a #{FilesHelper::output_dir}")
-      end
-    end
-  end
-end