elasticsearch-api 7.17.11 → 9.0.2

data/utils/README.md

@@ -1,46 +0,0 @@
-# Utils
-
-## The Generator
-
-This directory hosts The Generator, a tool that generates the classes for each API endpoint from the [Elasticsearch REST API JSON Specification](https://github.com/elastic/elasticsearch/tree/master/rest-api-spec).
-
-### Generate
-
-To generate the code, you need to have the Elasticsearch REST API spec files in `tmp/rest-api-spec` in the root of the project. You can run a rake task from the root of the project to download the specs corresponding to the current running cluster:
-```bash
-$ rake elasticsearch:download_artifacts
-```
-
-Once the JSON files have been downloaded, you need to run (from this folder):
-```bash
-$ thor api:code:generate
-```
-
-- The oss Ruby code will be generated in `elasticsearch-api/lib/elasticsearch/api/actions`.  
-- The xpack Ruby code will be generated in `elasticsearch-xpack/lib/elasticsearch/xpack/api/actions`.
-- The generator runs Rubocop to autolint and clean up the generated files.
-
-Alternatively, you can pass in `oss` or `xpack` as parameters to generate only one of the 2 sets of endpoints:
-
-```bash
-$ thor api:code:generate --api=xpack
-$ thor api:code:generate --api=oss
-```
-
-### Development
-
-The main entry point is `generate_source.rb`, which contains a class that implements a Thor task: `generate`:
-
-```
-$ thor api:code:generate
-```
-
-It uses [Thor::Actions](https://github.com/erikhuda/thor/wiki/Actions)' `template` method and `templates/method.erb` to generate the final code. The `generator` directory contains some helpers used to generate the code. The ERB template is split into partials and you can find all ERB files in the `templates` directory.
-
-There's also a lister task:
-
-```
-$ thor api:list
-```
-
-It's implemented in `lister.rb` and it lists all the REST API endpoints from the JSON specification.

data/utils/Thorfile

@@ -1,6 +0,0 @@
-# Licensed to Elasticsearch B.V under one or more agreements.
-# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
-# See the LICENSE file in the project root for more information
-
-require File.expand_path('./thor/generate_source')
-require File.expand_path('./thor/lister')

data/utils/thor/.rubocop.yml

@@ -1,4 +0,0 @@
-Layout/EndAlignment:
-  AutoCorrect: true
-Metrics/LineLength:
-  Max: 160

data/utils/thor/generate_source.rb

@@ -1,307 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# encoding: UTF-8
-
-require 'thor'
-require 'pathname'
-require 'active_support/core_ext/hash/deep_merge'
-require 'active_support/inflector'
-require 'multi_json'
-require 'coderay'
-require 'pry'
-require_relative 'generator/files_helper'
-require_relative 'generator/endpoint_specifics'
-
-module Elasticsearch
-  module API
-    # A command line application based on [Thor](https://github.com/wycats/thor),
-    # which will read the JSON API spec file(s), and generate
-    # the Ruby source code (one file per API endpoint) with correct
-    # module namespace, method names, and RDoc documentation,
-    # as well as test files for each endpoint.
-    #
-    # Specific exceptions and code snippets that need to be included are written
-    # in EndpointSpecifics (generator/endpoint_specifics) and the module is included
-    # here.
-    #
-    class SourceGenerator < Thor
-      namespace 'api:code'
-      include Thor::Actions
-      include EndpointSpecifics
-
-      __root = Pathname(File.expand_path('../../..', __FILE__))
-
-      desc 'generate', 'Generate source code and tests from the REST API JSON specification'
-      method_option :verbose, type: :boolean, default: false,  desc: 'Output more information'
-      method_option :tests,   type: :boolean, default: false,  desc: 'Generate test files'
-      method_option :api,     type: :array,   default: %w[oss xpack], desc: 'APIs to generate (oss, x-pack)'
-
-      def generate
-        self.class.source_root File.expand_path(__dir__)
-        @xpack = options[:api].include? 'xpack'
-        @oss   = options[:api].include? 'oss'
-
-        __generate_source(:xpack) if @xpack
-        __generate_source(:oss) if @oss
-        # -- Tree output
-        print_tree if options[:verbose]
-      end
-
-      private
-
-      def __generate_source(api)
-        @current_api = api
-        @output = FilesHelper.output_dir(api)
-
-        FilesHelper.files(api).each do |filepath|
-          @path = Pathname(filepath)
-          @json = MultiJson.load(File.read(@path))
-          @spec = @json.values.first
-          say_status 'json', @path, :yellow
-
-          @spec['url'] ||= {}
-
-          @endpoint_name    = @json.keys.first
-          @full_namespace   = __full_namespace
-          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size - 1 : 0
-          @module_namespace = @full_namespace[0, @namespace_depth]
-          @method_name      = @full_namespace.last
-          @parts            = __endpoint_parts
-          @params           = @spec['params'] || {}
-          @specific_params  = specific_params(@module_namespace.first) # See EndpointSpecifics
-          @http_method      = __http_method
-          @paths            = @spec['url']['paths'].map { |b| b['path'] }
-          # Using Ruby's safe operator on array:
-          @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
-          @http_path        = __http_path
-          @required_parts   = __required_parts
-
-          @module_namespace.shift if @module_namespace.first == 'xpack'
-          @path_to_file = @output.join(@module_namespace.join('/')).join("#{@method_name}.rb")
-          dir = @output.join(@module_namespace.join('/'))
-          empty_directory(dir, verbose: false)
-
-          # Write the file with the ERB template:
-          template('templates/method.erb', @path_to_file, force: true)
-
-          print_source_code(@path_to_file) if options[:verbose]
-
-          generate_tests if options[:tests]
-
-          puts
-        end
-
-        run_rubocop(api)
-      end
-
-      def __full_namespace
-        names = @endpoint_name.split('.')
-        if @current_api == :xpack
-          names = (names.first == 'xpack' ? names : ['xpack', names].flatten)
-          # Return an array to expand 'ccr', 'ilm', 'ml' and 'slm'
-          names.map do |name|
-            name
-              .gsub(/^ml$/, 'machine_learning')
-              .gsub(/^ilm$/, 'index_lifecycle_management')
-              .gsub(/^ccr/, 'cross_cluster_replication')
-              .gsub(/^slm/, 'snapshot_lifecycle_management')
-          end
-        else
-          names
-        end
-      end
-
-      # Create the hierarchy of directories based on API namespaces
-      #
-      def __create_directories(key, value)
-        return if value['documentation']
-
-        empty_directory @output.join(key)
-        create_directory_hierarchy * value.to_a.first
-      end
-
-      # Extract parts from each path
-      #
-      def __endpoint_parts
-        parts = @spec['url']['paths'].select do |a|
-          a.keys.include?('parts')
-        end.map do |path|
-          path&.[]('parts')
-        end
-        (parts.inject(&:merge) || [])
-      end
-
-      def __http_method
-        return '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @endpoint_name == 'index'
-        return post_and_get if @endpoint_name == 'count'
-
-        default_method = @spec['url']['paths'].map { |a| a['methods'] }.flatten.first
-        if @spec['body'] && default_method == 'GET'
-          # When default method is GET and body is required, we should always use POST
-          if @spec['body']['required']
-            'Elasticsearch::API::HTTP_POST'
-          else
-            post_and_get
-          end
-        else
-          "Elasticsearch::API::HTTP_#{default_method}"
-        end
-      end
-
-      def post_and_get
-        <<~SRC
-          if arguments[:body]
-            Elasticsearch::API::HTTP_POST
-          else
-            Elasticsearch::API::HTTP_GET
-          end
-        SRC
-      end
-
-      def __http_path
-        return "\"#{__parse_path(@paths.first)}\"" if @paths.size == 1
-        return termvectors_path if @method_name == 'termvectors'
-
-        result = ''
-        anchor_string = []
-        @paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
-          var_string = __extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
-          next if anchor_string.include? var_string
-
-          anchor_string << var_string
-          result += if i.zero?
-                      "if #{var_string}\n"
-                    elsif (i == @paths.size - 1) || var_string.empty?
-                      "else\n"
-                    else
-                      "elsif #{var_string}\n"
-                    end
-          result += "\"#{__parse_path(path)}\"\n"
-        end
-        result += 'end'
-        result
-      end
-
-      def __parse_path(path)
-        path.gsub(/^\//, '')
-          .gsub(/\/$/, '')
-          .gsub('{', "\#{#{__utils}.__listify(_")
-          .gsub('}', ')}')
-      end
-
-      def __path_variables
-        @paths.map do |path|
-          __extract_path_variables(path)
-        end
-      end
-
-      # extract values that are in the {var} format:
-      def __extract_path_variables(path)
-        path.scan(/{(\w+)}/).flatten
-      end
-
-      # Find parts that are definitely required and should raise an error if
-      # they're not present
-      #
-      def __required_parts
-        required = []
-        return required if @endpoint_name == 'tasks.get'
-
-        required << 'body' if (@spec['body'] && @spec['body']['required'])
-        # Get required variables from paths:
-        req_variables = __path_variables.inject(:&) # find intersection
-        required << req_variables unless req_variables.empty?
-        required.flatten
-      end
-
-      def docs_helper(name, info)
-        info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings'
-        info['type'] = 'Integer' if info['type'] == 'int' # Rename 'int' to 'Integer'
-        tipo = info['type'] ? info['type'].capitalize : 'String'
-        description = info['description'] ? info['description'].strip : '[TODO]'
-        options = info['options'] ? "(options: #{info['options'].join(', ').strip})" : nil
-        required = info['required'] ? '(*Required*)' : ''
-        deprecated = info['deprecated'] ? '*Deprecated*' : ''
-        optionals = [required, deprecated, options].join(' ').strip
-
-        "# @option arguments [#{tipo}] :#{name} #{description} #{optionals}\n"
-      end
-
-      def stability_doc_helper(stability)
-        return if stability == 'stable'
-
-        if stability == 'experimental'
-          <<~MSG
-            # This functionality is Experimental and may be changed or removed
-            # completely in a future release. Elastic will take a best effort approach
-            # to fix any issues, but experimental features are not subject to the
-            # support SLA of official GA features.
-          MSG
-        elsif stability == 'beta'
-          <<~MSG
-            # This functionality is in Beta and is subject to change. The design and
-            # code is less mature than official GA features and is being provided
-            # as-is with no warranties. Beta features are not subject to the support
-            # SLA of official GA features.
-          MSG
-        else
-          <<~MSG
-            # This functionality is subject to potential breaking changes within a
-            # minor version, meaning that your referencing code may break when this
-            # library is upgraded.
-          MSG
-        end
-      end
-
-      def generate_tests
-        copy_file 'templates/test_helper.rb', @output.join('test').join('test_helper.rb')
-
-        @test_directory = @output.join('test/api').join(@module_namespace.join('/'))
-        @test_file      = @test_directory.join("#{@method_name}_test.rb")
-
-        empty_directory @test_directory
-        template 'templates/test.erb', @test_file
-
-        print_source_code(@test_file) if options[:verbose]
-      end
-
-      def print_source_code(path_to_file)
-        colorized_output = CodeRay.scan_file(path_to_file, :ruby).terminal
-        lines            = colorized_output.split("\n")
-        formatted        = lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n")
-
-        say_status('ruby', formatted, :yellow)
-      end
-
-      def print_tree
-        return unless `which tree > /dev/null 2>&1; echo $?`.to_i < 1
-
-        lines = `tree #{@output}`.split("\n")
-        say_status('tree', lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n"))
-      end
-
-      def __utils
-        @current_api == :xpack ? 'Elasticsearch::API::Utils' : 'Utils'
-      end
-
-      def run_rubocop(api)
-        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -x #{FilesHelper::output_dir(api)}")
-      end
-    end
-  end
-end

data/utils/thor/generator/endpoint_specifics.rb

@@ -1,187 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-module Elasticsearch
-  module API
-    # Handles specific exceptional parameters and code snippets that need to be
-    # included in the generated code. This module is included in SourceGenerator
-    # so its methods can be used from the ERB template (method.erb). This will
-    # potentially be refactored into different templates.
-    module EndpointSpecifics
-      # Endpoints that need Utils.__rescue_from_not_found
-      IGNORE_404 = %w[
-        exists
-        indices.exists
-        indices.exists_alias
-        indices.exists_index_template
-        indices.exists_template
-        indices.exists_type
-      ].freeze
-
-      # Endpoints that need Utils.__rescue_from_not_found if the ignore
-      # parameter is included
-      COMPLEX_IGNORE_404 = %w[
-        delete
-        get
-        indices.flush_synced
-        indices.delete
-        indices.delete_index_template
-        indices.delete_template
-        security.get_role
-        security.get_user
-        snapshot.status
-        snapshot.get
-        snapshot.get_repository
-        snapshot.delete_repository
-        snapshot.delete
-        update
-        watcher.delete_watch
-      ].freeze
-
-      # Endpoints that need params[:h] listified
-      H_PARAMS = %w[aliases allocation count health indices nodes pending_tasks
-                    recovery shards thread_pool].freeze
-
-      # Function that adds the listified h param code
-      def specific_params(namespace)
-        params = []
-        if H_PARAMS.include?(@method_name) && namespace == 'cat'
-          if @method_name == 'nodes'
-            params << 'params[:h] = Utils.__listify(params[:h], escape: false) if params[:h]'
-          else
-            params << 'params[:h] = Utils.__listify(params[:h]) if params[:h]'
-          end
-        end
-        params
-      end
-
-      def needs_ignore_404?(endpoint)
-        IGNORE_404.include? endpoint
-      end
-
-      def needs_complex_ignore_404?(endpoint)
-        COMPLEX_IGNORE_404.include? endpoint
-      end
-
-      def module_name_helper(name)
-        return name.upcase if %w[sql ssl].include? name
-
-        name.split('_').map(&:capitalize).join
-      end
-
-      def termvectors_path
-        <<~SRC
-          if _index && _type && _id
-            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{Utils.__listify(_id)}/\#{endpoint}"
-          elsif _index && _type
-            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{endpoint}"
-          elsif _index && _id
-            "\#{Utils.__listify(_index)}/\#{endpoint}/\#{Utils.__listify(_id)}"
-          else
-            "\#{Utils.__listify(_index)}/\#{endpoint}"
-          end
-        SRC
-      end
-
-      def ping_perform_request
-        <<~SRC
-          begin
-            perform_request(method, path, params, body, headers).status == 200 ? true : false
-          rescue Exception => e
-            if e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i
-              false
-            else
-              raise e
-            end
-          end
-        SRC
-      end
-
-      def indices_stats_params_registry
-        <<~SRC
-          ParamsRegistry.register(:stats_params, [
-            #{@spec['params'].keys.map { |k| ":#{k}" }.join(",\n")}
-          ].freeze)
-
-          ParamsRegistry.register(:stats_parts, [
-            #{@parts['metric']['options'].push('metric').map { |k| ":#{k}" }.join(",\n")}
-          ].freeze)
-        SRC
-      end
-
-      def msearch_body_helper
-        <<~SRC
-          case
-          when body.is_a?(Array) && body.any? { |d| d.has_key? :search }
-            payload = body.
-              inject([]) do |sum, item|
-                meta = item
-                data = meta.delete(:search)
-
-                sum << meta
-                sum << data
-                sum
-              end.
-              map { |item| Elasticsearch::API.serializer.dump(item) }
-            payload << "" unless payload.empty?
-            payload = payload.join("\\n")
-          when body.is_a?(Array)
-            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
-            payload << "" unless payload.empty?
-            payload = payload.join("\\n")
-          else
-            payload = body
-          end
-        SRC
-      end
-
-      def msearch_template_body_helper
-        <<~SRC
-          case
-          when body.is_a?(Array)
-            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
-            payload << "" unless payload.empty?
-            payload = payload.join("\n")
-          else
-            payload = body
-          end
-        SRC
-      end
-
-      def bulk_body_helper
-        <<~SRC
-          if body.is_a? Array
-            payload = Elasticsearch::API::Utils.__bulkify(body)
-          else
-            payload = body
-          end
-        SRC
-      end
-
-      def find_structure_body_helper
-        bulk_body_helper
-      end
-
-      def bulk_doc_helper(info)
-        <<~SRC
-          # @option arguments [String|Array] :body #{info}. Array of Strings, Header/Data pairs,
-          # or the conveniency "combined" format can be passed, refer to Elasticsearch::API::Utils.__bulkify documentation.
-        SRC
-      end
-    end
-  end
-end

data/utils/thor/generator/files_helper.rb

@@ -1,78 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-require 'pathname'
-require_relative '../../../lib/elasticsearch/api/version.rb'
-
-module Elasticsearch
-  module API
-    module FilesHelper
-      PROJECT_PATH = File.join(File.dirname(__FILE__), '..')
-      SRC_PATH   = File.join(PROJECT_PATH, '..', '..', '..', 'tmp/rest-api-spec/api/')
-      OSS_OUTPUT_DIR = '../../elasticsearch-api/lib/elasticsearch/api/actions'.freeze
-      XPACK_OUTPUT_DIR = '../../elasticsearch-xpack/lib/elasticsearch/xpack/api/actions'.freeze
-
-      TESTS_DIRECTORIES = {
-        oss: "#{PROJECT_PATH}/../../../tmp/rest-api-spec/test/free",
-        xpack: "#{PROJECT_PATH}/../../../tmp/rest-api-spec/test/platinum"
-      }.freeze
-
-      # Only get JSON files and remove hidden files
-      def self.files(api)
-        json_files = if api == :xpack
-                       xpack_files
-                     else
-                       Dir.entries(SRC_PATH) - xpack_files
-                     end
-        json_files.reject do |file|
-          File.extname(file) != '.json' ||
-            File.basename(file) == '_common.json'
-        end.map { |file| "#{SRC_PATH}#{file}" }
-      end
-
-      XPACK_ENDPOINTS = [
-        'autoscaling', 'cross_cluster_replication', 'ccr', 'data_frame_transform_deprecated',
-        'enrich', 'eql', 'fleet', 'ilm', 'logstash', 'migration', 'watcher', 'slm'
-      ]
-      XPACK_ENDPOINTS_REGEXP = /data_stream|ml_|reload_search_analyzers|transform|freeze|unfreeze/
-
-      def self.xpack_files
-        xpack_tests = Dir.entries(TESTS_DIRECTORIES[:xpack])
-        Dir.entries(SRC_PATH).map do |entry|
-          filename = entry.split('.').first
-          next if filename == 'snapshot'
-
-          if xpack_tests.include?(filename) ||
-                                  XPACK_ENDPOINTS.include?(filename) ||
-                                  entry.match?(XPACK_ENDPOINTS_REGEXP)
-            entry
-          end
-        end.compact
-      end
-
-      # Path to directory to copy generated files
-      def self.output_dir(api)
-        api == :xpack ? Pathname(XPACK_OUTPUT_DIR) : Pathname(OSS_OUTPUT_DIR)
-      end
-
-      def self.gem_version
-        regex = /([0-9]{1,2}\.[0-9x]{1,2})/
-        Elasticsearch::API::VERSION.match(regex)[0]
-      end
-    end
-  end
-end

data/utils/thor/lister.rb

@@ -1,57 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# encoding: UTF-8
-
-require 'thor'
-
-require 'pathname'
-
-module Elasticsearch
-  module API
-    class Lister < Thor
-      namespace 'api'
-
-      DEFAULT_PATH = '../../tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/'.freeze
-
-      desc "list <PATH DIRECTORY WITH JSON SPEC FILES>", "List all the REST API endpoints from the JSON specification"
-      method_option :verbose,  type: :boolean, default: false, desc: 'Output more information'
-      method_option :format,   default: 'text', desc: 'Output format (text, json)'
-      def list(directory = DEFAULT_PATH)
-        input = Pathname(directory).join('*.json')
-        apis = Dir[input.to_s].map do |f|
-          File.basename(f, '.json')
-        end.sort
-
-        if options[:verbose]
-          say_status 'Count', apis.size
-          say '▬'*terminal_width
-        end
-
-        case options[:format]
-          when 'text'
-            apis.each { |a| puts "* #{a}" }
-          when 'json'
-            puts apis.inspect
-          else
-            puts "[!] ERROR: Unknown output format '#{options[:format]}'"
-            exit(1)
-        end
-      end
-    end
-  end
-end