elasticsearch-api 7.3.0 → 7.5.0

data/spec/elasticsearch/api/actions/update_document_spec.rb

@@ -53,16 +53,12 @@ describe 'client#update' do
       'foo/bar/1/_update'
     end
 
-    let(:params) do
-      { version: 100 }
-    end
-
     let(:body) do
       {}
     end
 
     it 'performs the request' do
-      expect(client_double.update(index: 'foo', type: 'bar', id: '1', version: 100, body: {}))
+      expect(client_double.update(index: 'foo', type: 'bar', id: '1', body: {}))
     end
   end
 
@@ -98,14 +94,14 @@ describe 'client#update' do
 
     it 'raises it to the user' do
       expect {
-        client.update(index: 'foo', type: 'bar', id: 'XXX')
+        client.update(index: 'foo', type: 'bar', id: 'XXX', body: {})
       }.to raise_exception(NotFound)
     end
 
     context 'when the :ignore parameter is specified' do
 
       it 'does not raise the error to the user' do
-        expect(client.update(index: 'foo', type: 'bar', id: 'XXX', ignore: 404)).to eq(false)
+        expect(client.update(index: 'foo', type: 'bar', id: 'XXX', body: {}, ignore: 404)).to eq(false)
       end
     end
   end

data/spec/elasticsearch/api/rest_api_yaml_spec.rb

@@ -18,7 +18,7 @@ describe 'Rest API YAML tests' do
         context "#{test.description}" do
 
           if test.skip_test?(ADMIN_CLIENT)
-            skip 'Test contains feature(s) not yet support or version is not satisfied'
+            skip 'Test contains feature(s) not yet supported or version is not satisfied'
 
           else
 

data/spec/rest_yaml_tests_helper.rb

@@ -72,6 +72,18 @@ skipped_tests << { file:        'delete/70_mix_typeless_typeful.yml',
 skipped_tests << { file:        'cat.templates/10_basic.yml',
                    description: '*' }
 
+# node_selector is not supported yet
+skipped_tests << { file: 'cat.aliases/10_basic.yml',
+                   description: '*' }
+
+# Responses are there but not equal (eg.: yellow status)
+skipped_tests << { file: 'cluster.health/10_basic.yml',
+                   description: 'cluster health with closed index (pre 7.2.0)' }
+
+# Regular expression not catching exact match:
+skipped_tests << { file: 'cat.indices/10_basic.yml',
+                   description: 'Test cat indices output for closed index (pre 7.2.0)' }
+
 SKIPPED_TESTS = skipped_tests
 
 # The directory of rest api YAML files.

data/utils/Gemfile

@@ -11,3 +11,4 @@ gem 'multi_json'
 gem 'thor'
 gem 'json'
 gem 'pry'
+gem 'rubocop'

data/utils/Thorfile

@@ -2,6 +2,5 @@
 # Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 # See the LICENSE file in the project root for more information
 
-require File.expand_path('./thor/generate_api')
 require File.expand_path('./thor/generate_source')
 require File.expand_path('./thor/lister')

data/utils/thor/generate_source.rb

@@ -5,138 +5,245 @@
 # encoding: UTF-8
 
 require 'thor'
-
 require 'pathname'
 require 'active_support/core_ext/hash/deep_merge'
 require 'active_support/inflector'
 require 'multi_json'
 require 'coderay'
 require 'pry'
+require_relative 'generator/files_helper'
+require_relative 'generator/endpoint_specifics'
 
 module Elasticsearch
-
   module API
-
     # A command line application based on [Thor](https://github.com/wycats/thor),
     # which will read the JSON API spec file(s), and generate
     # the Ruby source code (one file per API endpoint) with correct
     # module namespace, method names, and RDoc documentation,
     # as well as test files for each endpoint.
     #
-    # Currently it only generates Ruby source, but can easily be
-    # extended and adapted to generate source code for other
-    # programming languages.
+    # Specific exceptions and code snippets that need to be included are written
+    # in EndpointSpecifics (generator/endpoint_specifics) and the module is included
+    # here.
     #
     class SourceGenerator < Thor
       namespace 'api:code'
-
       include Thor::Actions
+      include EndpointSpecifics
 
-      __root = Pathname( File.expand_path('../../..', __FILE__) )
-
-      desc "generate", "Generate source code and tests from the REST API JSON specification"
-      method_option :language,  default: 'ruby',                                          desc: 'Programming language'
-      method_option :force,     type: :boolean, default: false,                           desc: 'Overwrite the output'
-      method_option :verbose,   type: :boolean, default: false,                           desc: 'Output more information'
-      method_option :input,     default: File.expand_path('../../../../tmp/elasticsearch/rest-api-spec/api/**/*.json', __FILE__), desc: 'Path to directory with JSON API specs'
-      method_option :output,    default: File.expand_path('../../../tmp/out', __FILE__),        desc: 'Path to output directory'
-
-      def generate(*files)
-        self.class.source_root File.expand_path('../', __FILE__)
-
-        @input  = Pathname(options[:input])
-        @output = Pathname(options[:output])
+      __root = Pathname(File.expand_path('../../..', __FILE__))
 
-        # -- Test helper
-        copy_file "templates/ruby/test_helper.rb", @output.join('test').join('test_helper.rb') if options[:language] == 'ruby'
+      desc 'generate', 'Generate source code and tests from the REST API JSON specification'
+      method_option :verbose, type: :boolean, default: false, desc: 'Output more information'
+      method_option :tests,   type: :boolean, default: false, desc: 'Generate test files'
+      method_option :xpack,   type: :boolean, default: false, desc: 'Generate X-Pack'
 
-        files = Dir.entries(@input.to_s).reject { |f| f.start_with?('.') || f.start_with?('_') }
+      def generate
+        self.class.source_root File.expand_path(__dir__)
+        @xpack = options[:xpack]
 
-        files.each do |filepath|
-          file    = @input.join(filepath)
+        @input = FilesHelper.input_dir(@xpack)
+        @output = FilesHelper.output_dir(@xpack)
 
-          @path   = Pathname(file)
-          @json   = MultiJson.load( File.read(@path) )
-          @spec   = @json.values.first
+        FilesHelper.files(@xpack).each do |filepath|
+          @path = Pathname(@input.join(filepath))
+          @json = MultiJson.load(File.read(@path))
+          @spec = @json.values.first
           say_status 'json', @path, :yellow
 
           @spec['url'] ||= {}
-          @spec['url']['parts']  ||= []
-          @spec['url']['params'] ||= {}
 
-          # say_status 'JSON', @spec.inspect, options[:verbose]
-
-          @full_namespace   = @json.keys.first.split('.')
-          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size-1 : 0
+          @endpoint_name    = @json.keys.first
+          @full_namespace   = __full_namespace
+          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size - 1 : 0
           @module_namespace = @full_namespace[0, @namespace_depth]
           @method_name      = @full_namespace.last
-          @http_method      = "Elasticsearch::API::HTTP_#{@spec['methods'].first}"
-          @http_path        = unless @spec['url']['parts'].empty?
-                                @spec['url']['path']
-                                  .split('/')
-                                  .compact
-                                  .reject { |p| p =~ /^\s*$/ }
-                                  .map { |p| p =~ /\{/ ? "\#\{arguments[:#{p.tr('{}', '')}]\}" : p }
-                                  .join('/')
-                                  .gsub(/^\//, '')
-                              else
-                                @spec['url']['path'].gsub(/^\//, '')
-                              end
-
-          # -- Ruby files
-
-          @path_to_file = @output.join('api').join( @module_namespace.join('/') ).join("#{@method_name}.rb")
-
-          empty_directory @output.join('api').join( @module_namespace.join('/') )
-
-          template "templates/#{options[:language]}/method.erb", @path_to_file
-
-          if options[:verbose]
-            colorized_output   = CodeRay.scan_file(@path_to_file, :ruby).terminal
-            lines              =  colorized_output.split("\n")
-            say_status options[:language].downcase,
-                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"),
-                       :yellow
-          end
+          @parts            = __endpoint_parts
+          @params           = @spec['params'] || {}
+          @specific_params  = specific_params(@module_namespace.first) # See EndpointSpecifics
+          @http_method      = __http_method
+          @paths            = @spec['url']['paths'].map { |b| b['path'] }
+          # Using Ruby's safe operator on array:
+          @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
+          @http_path        = __http_path
+          @required_parts   = __required_parts
 
-          # --- Test files
+          @module_namespace.shift if @module_namespace.first == 'xpack'
+          @path_to_file = @output.join(@module_namespace.join('/')).join("#{@method_name}.rb")
+          dir = @output.join(@module_namespace.join('/'))
+          empty_directory(dir, verbose: false)
 
-          @test_directory = @output.join('test/api').join( @module_namespace.join('/') )
-          @test_file      = @test_directory.join("#{@method_name}_test.rb")
+          # Write the file with the ERB template:
+          template('templates/method.erb', @path_to_file, force: true)
 
-          empty_directory @test_directory
-          template "templates/#{options[:language]}/test.erb", @test_file
+          print_source_code(@path_to_file) if options[:verbose]
 
-          if options[:verbose]
-            colorized_output   = colorized_output   = CodeRay.scan_file(@test_file, :ruby).terminal
-            lines              =  colorized_output.split("\n")
-            say_status options[:language].downcase,
-                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"),
-                       :yellow
-            say '▬'*terminal_width
-          end
+          generate_tests if options[:tests]
+
+          puts
         end
 
-        # -- Tree output
+        run_rubocop
 
-        if options[:verbose] && `which tree > /dev/null 2>&1; echo $?`.to_i < 1
-          lines = `tree #{@output}`.split("\n")
-          say_status 'tree',
-                     lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n")
-        end
+        # -- Tree output
+        print_tree if options[:verbose]
       end
 
       private
 
+      def __full_namespace
+        names = @endpoint_name.split('.')
+        if @xpack
+          names = (names.first == 'xpack' ? names : ['xpack', names].flatten)
+          # Return an array with 'ml' renamed to 'machine_learning' and 'ilm' to
+          # 'index_lifecycle_management'
+          names.map do |name|
+            name
+              .gsub(/^ml$/, 'machine_learning')
+              .gsub(/^ilm$/, 'index_lifecycle_management')
+          end
+        else
+          names
+        end
+      end
+
       # Create the hierarchy of directories based on API namespaces
       #
       def __create_directories(key, value)
-        unless value['documentation']
-          empty_directory @output.join(key)
-          create_directory_hierarchy *value.to_a.first
+        return if value['documentation']
+
+        empty_directory @output.join(key)
+        create_directory_hierarchy * value.to_a.first
+      end
+
+      # Extract parts from each path
+      #
+      def __endpoint_parts
+        parts = @spec['url']['paths'].select do |a|
+          a.keys.include?('parts')
+        end.map do |path|
+          path&.[]('parts')
+        end
+        (parts.inject(&:merge) || [])
+      end
+
+      def __http_method
+        case @endpoint_name
+        when 'index'
+          '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST'
+        when 'count'
+          <<~SRC
+            if arguments[:body]
+              Elasticsearch::API::HTTP_POST
+            else
+              Elasticsearch::API::HTTP_GET
+            end
+          SRC
+        else
+          "Elasticsearch::API::HTTP_#{@spec['url']['paths'].map { |a| a['methods'] }.flatten.first}"
+        end
+      end
+
+      def __http_path
+        return "\"#{__parse_path(@paths.first)}\"" if @paths.size == 1
+        return termvectors_path if @method_name == 'termvectors'
+
+        result = ''
+        anchor_string = ''
+        @paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
+          var_string = __extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
+          next if anchor_string == var_string
+
+          anchor_string = var_string
+          result += if i.zero?
+                      "if #{var_string}\n"
+                    elsif (i == @paths.size - 1) || var_string.empty?
+                      "else\n"
+                    else
+                      "elsif #{var_string}\n"
+                    end
+          result += "\"#{__parse_path(path)}\"\n"
         end
+        result += 'end'
+        result
+      end
+
+      def __parse_path(path)
+        path.gsub(/^\//, '')
+            .gsub(/\/$/, '')
+            .gsub('{', "\#{#{__utils}.__listify(_")
+            .gsub('}', ')}')
+      end
+
+      def __path_variables
+        @paths.map do |path|
+          __extract_path_variables(path)
+        end
+      end
+
+      # extract values that are in the {var} format:
+      def __extract_path_variables(path)
+        path.scan(/{(\w+)}/).flatten
+      end
+
+      # Find parts that are definitely required and should raise an error if
+      # they're not present
+      #
+      def __required_parts
+        required = []
+        return required if @endpoint_name == 'tasks.get'
+
+        required << 'body' if (@spec['body'] && @spec['body']['required'])
+        # Get required variables from paths:
+        req_variables = __path_variables.inject(:&) # find intersection
+        required << req_variables unless req_variables.empty?
+        required.flatten
       end
 
+      def docs_helper(name, info)
+        info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings'
+        tipo = info['type'] ? info['type'].capitalize : 'String'
+        description = info['description'] ? info['description'].strip : '[TODO]'
+        options = info['options'] ? "\n    #   (options: #{info['options'].join(', '.strip)})\n" : ''
+        required = info['required'] ? ' (*Required*)' : ''
+        deprecated = info['deprecated'] ? ' *Deprecated*' : ''
+        "# @option arguments [#{tipo}] :#{name} #{description} #{required} #{deprecated} #{options}\n"
+      end
+
+      def generate_tests
+        copy_file 'templates/test_helper.rb', @output.join('test').join('test_helper.rb')
+
+        @test_directory = @output.join('test/api').join(@module_namespace.join('/'))
+        @test_file      = @test_directory.join("#{@method_name}_test.rb")
+
+        empty_directory @test_directory
+        template 'templates/test.erb', @test_file
+
+        print_source_code(@test_file) if options[:verbose]
+      end
+
+      def print_source_code(path_to_file)
+        colorized_output = CodeRay.scan_file(path_to_file, :ruby).terminal
+        lines            = colorized_output.split("\n")
+        formatted        = lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n")
+
+        say_status('ruby', formatted, :yellow)
+      end
+
+      def print_tree
+        return unless `which tree > /dev/null 2>&1; echo $?`.to_i < 1
+
+        lines = `tree #{@output}`.split("\n")
+        say_status('tree', lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n"))
+      end
+
+      def __utils
+        @xpack ? 'Elasticsearch::API::Utils' : 'Utils'
+      end
+
+      def run_rubocop
+        system("rubocop --format autogenconf -x #{FilesHelper::output_dir(@xpack)}")
+      end
     end
   end
 end

data/utils/thor/generator/endpoint_specifics.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+# Licensed to Elasticsearch B.V under one or more agreements.
+# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+
+module Elasticsearch
+  module API
+    # Handles specific exceptional parameters and code snippets that need to be
+    # included in the generated code. This module is included in SourceGenerator
+    # so its methods can be used from the ERB template (method.erb). This will
+    # potentially be refactored into different templates.
+    module EndpointSpecifics
+      # Endpoints that need Utils.__rescue_from_not_found
+      IGNORE_404 = %w[
+        exists
+        indices.exists
+        indices.exists_alias
+        indices.exists_template
+        indices.exists_type
+      ].freeze
+
+      # Endpoints that need Utils.__rescue_from_not_found if the ignore
+      # parameter is included
+      COMPLEX_IGNORE_404 = %w[
+        delete
+        get
+        indices.flush_synced
+        indices.delete_template
+        indices.delete
+        security.get_role
+        security.get_user
+        snapshot.status
+        snapshot.get
+        snapshot.get_repository
+        snapshot.delete_repository
+        snapshot.delete
+        update
+        watcher.delete_watch
+      ].freeze
+
+      # Endpoints that need params[:h] listified
+      H_PARAMS = %w[aliases allocation count health indices nodes pending_tasks
+                    recovery shards thread_pool].freeze
+
+      # Function that adds the listified h param code
+      def specific_params(namespace)
+        params = []
+        if H_PARAMS.include?(@method_name) && namespace == 'cat'
+          if @method_name == 'nodes'
+            params << 'params[:h] = Utils.__listify(params[:h], escape: false) if params[:h]'
+          else
+            params << 'params[:h] = Utils.__listify(params[:h]) if params[:h]'
+          end
+        end
+        params
+      end
+
+      def needs_ignore_404?(endpoint)
+        IGNORE_404.include? endpoint
+      end
+
+      def needs_complex_ignore_404?(endpoint)
+        COMPLEX_IGNORE_404.include? endpoint
+      end
+
+      def termvectors_path
+        <<~SRC
+          if _index && _type && _id
+            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{Utils.__listify(_id)}/\#{endpoint}"
+          elsif _index && _type
+            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{endpoint}"
+          elsif _index && _id
+            "\#{Utils.__listify(_index)}/\#{endpoint}/\#{Utils.__listify(_id)}"
+          else
+            "\#{Utils.__listify(_index)}/\#{endpoint}"
+          end
+        SRC
+      end
+
+      def ping_perform_request
+        <<~SRC
+          begin
+            perform_request(method, path, params, body).status == 200 ? true : false
+          rescue Exception => e
+            if e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i
+              false
+            else
+              raise e
+            end
+          end
+        SRC
+      end
+
+      def indices_stats_params_registry
+        <<~SRC
+          ParamsRegistry.register(:stats_params, [
+            #{@spec['params'].keys.map { |k| ":#{k}" }.join(",\n")}
+          ].freeze)
+
+          ParamsRegistry.register(:stats_parts, [
+            #{@parts['metric']['options'].push('metric').map { |k| ":#{k}" }.join(",\n")}
+          ].freeze)
+        SRC
+      end
+
+      def msearch_body_helper
+        <<~SRC
+          case
+          when body.is_a?(Array) && body.any? { |d| d.has_key? :search }
+            payload = body.
+              inject([]) do |sum, item|
+                meta = item
+                data = meta.delete(:search)
+
+                sum << meta
+                sum << data
+                sum
+              end.
+              map { |item| Elasticsearch::API.serializer.dump(item) }
+            payload << "" unless payload.empty?
+            payload = payload.join("\n")
+          when body.is_a?(Array)
+            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
+            payload << "" unless payload.empty?
+            payload = payload.join("\n")
+          else
+            payload = body
+          end
+        SRC
+      end
+
+      def msearch_template_body_helper
+        <<~SRC
+          case
+          when body.is_a?(Array)
+            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
+            payload << "" unless payload.empty?
+            payload = payload.join("\n")
+          else
+            payload = body
+          end
+        SRC
+      end
+
+      def bulk_body_helper
+        <<~SRC
+          if body.is_a? Array
+            payload = Utils.__bulkify(body)
+          else
+            payload = body
+          end
+        SRC
+      end
+    end
+  end
+end

data/utils/thor/generator/files_helper.rb

@@ -0,0 +1,37 @@
+require 'pathname'
+
+module Elasticsearch
+  module API
+    module FilesHelper
+      OSS_SRC_PATH   = '../../../../../tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/'.freeze
+      OSS_OUTPUT_DIR = '../../elasticsearch-api/lib/elasticsearch/api/actions'.freeze
+
+      XPACK_SRC_PATH   = '../../../../../tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/api'.freeze
+      XPACK_OUTPUT_DIR = '../../elasticsearch-xpack/lib/elasticsearch/xpack/api/actions'.freeze
+
+      # Path to directory with JSON API specs
+      def self.input_dir(xpack = false)
+        input_dir = if xpack
+                      File.expand_path(XPACK_SRC_PATH, __FILE__)
+                    else
+                      File.expand_path(OSS_SRC_PATH, __FILE__)
+                    end
+        Pathname(input_dir)
+      end
+
+      # Path to directory to copy generated files
+      def self.output_dir(xpack = false)
+        xpack ? Pathname(XPACK_OUTPUT_DIR) : Pathname(OSS_OUTPUT_DIR)
+      end
+
+      # Only get JSON files and remove hidden files
+      def self.files(xpack = false)
+        Dir.entries(input_dir(xpack).to_s).reject do |f|
+          f.start_with?('.') ||
+            f.start_with?('_') ||
+            File.extname(f) != '.json'
+        end
+      end
+    end
+  end
+end

data/utils/thor/lister.rb

@@ -9,16 +9,16 @@ require 'thor'
 require 'pathname'
 
 module Elasticsearch
-
   module API
-
     class Lister < Thor
       namespace 'api'
 
+      DEFAULT_PATH = '../../tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/'.freeze
+
       desc "list <PATH DIRECTORY WITH JSON SPEC FILES>", "List all the REST API endpoints from the JSON specification"
       method_option :verbose,  type: :boolean, default: false, desc: 'Output more information'
       method_option :format,   default: 'text', desc: 'Output format (text, json)'
-      def list(directory)
+      def list(directory = DEFAULT_PATH)
         input = Pathname(directory).join('*.json')
         apis = Dir[input.to_s].map do |f|
           File.basename(f, '.json')
@@ -40,6 +40,5 @@ module Elasticsearch
         end
       end
     end
-
   end
 end

data/utils/thor/templates/_documentation_top.erb

@@ -0,0 +1,27 @@
+<%- if @spec['documentation']['description'] -%>
+  <%= '  '*(@namespace_depth+3) %># <%= @spec['documentation']['description'].gsub("\n", "\n#{'  '*(@namespace_depth+3)}# ") -%>
+<%- else %>
+  <%= '  '*(@namespace_depth+3) %># TODO: Description
+<%- end %>
+<%= '  '*(@namespace_depth+3) %>#
+<%- unless @parts.nil? || @parts.empty? %><%# URL parts -%>
+  <%- @parts.each do |name, info| -%>
+    <%= docs_helper(name, info) -%>
+  <%- end -%>
+<%- end -%><%# Body -%>
+<%# URL parameters -%>
+<%- @params.each do |name, info| -%>
+  <%= docs_helper(name, info) -%>
+<%- end %>
+<%= '  '*(@namespace_depth+3) + '# @option arguments [Hash] :body ' + (@spec['body']['description'] ? @spec['body']['description'].strip : 'TODO: Description') + (@spec['body']['required'] ? ' (*Required*)' : '') + "\n" if @spec['body'] -%>
+<% if @deprecation_note -%>
+  #
+  # *Deprecation notice*:
+  # <%=  @deprecation_note['description'] %>
+  # Deprecated since version <%=  @deprecation_note['version'] %>
+  #
+<% end -%>
+<%= '  '*(@namespace_depth+3) -%>#
+<%# Documentation link -%>
+<%=  '  '*(@namespace_depth+3) %># @see <%= @spec['documentation']['url'] ? @spec['documentation']['url'].gsub!(/\/(current|master)\//, '/7.5/') : "[TODO]" %>
+<%= '  '*(@namespace_depth+3) %>#