elasticsearch-api 6.8.0 → 7.10.0

data/utils/thor/generate_source.rb

@@ -1,135 +1,302 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: UTF-8
 
 require 'thor'
-
 require 'pathname'
 require 'active_support/core_ext/hash/deep_merge'
 require 'active_support/inflector'
 require 'multi_json'
 require 'coderay'
 require 'pry'
+require_relative 'generator/files_helper'
+require_relative 'generator/endpoint_specifics'
 
 module Elasticsearch
-
   module API
-
     # A command line application based on [Thor](https://github.com/wycats/thor),
     # which will read the JSON API spec file(s), and generate
     # the Ruby source code (one file per API endpoint) with correct
     # module namespace, method names, and RDoc documentation,
     # as well as test files for each endpoint.
     #
-    # Currently it only generates Ruby source, but can easily be
-    # extended and adapted to generate source code for other
-    # programming languages.
+    # Specific exceptions and code snippets that need to be included are written
+    # in EndpointSpecifics (generator/endpoint_specifics) and the module is included
+    # here.
     #
     class SourceGenerator < Thor
       namespace 'api:code'
-
       include Thor::Actions
+      include EndpointSpecifics
+
+      __root = Pathname(File.expand_path('../../..', __FILE__))
 
-      __root = Pathname( File.expand_path('../../..', __FILE__) )
+      desc 'generate', 'Generate source code and tests from the REST API JSON specification'
+      method_option :verbose, type: :boolean, default: false,  desc: 'Output more information'
+      method_option :tests,   type: :boolean, default: false,  desc: 'Generate test files'
+      method_option :api,     type: :array,   default: %w[oss xpack], desc: 'APIs to generate (oss, x-pack)'
 
-      desc "generate <PATH TO JSON SPEC FILES>", "Generate source code and tests from the REST API JSON specification"
-      method_option :language,  default: 'ruby',                                          desc: 'Programming language'
-      method_option :force,     type: :boolean, default: false,                           desc: 'Overwrite the output'
-      method_option :verbose,   type: :boolean, default: false,                           desc: 'Output more information'
-      method_option :input,     default: File.expand_path('../../../../tmp/elasticsearch/rest-api-spec/api/**/*.json', __FILE__), desc: 'Path to directory with JSON API specs'
-      method_option :output,    default: File.expand_path('../../../tmp/out', __FILE__),        desc: 'Path to output directory'
+      def generate
+        self.class.source_root File.expand_path(__dir__)
+        @xpack = options[:api].include? 'xpack'
+        @oss   = options[:api].include? 'oss'
 
-      def generate(*files)
-        self.class.source_root File.expand_path('../', __FILE__)
+        __generate_source(:xpack) if @xpack
+        __generate_source(:oss) if @oss
+        # -- Tree output
+        print_tree if options[:verbose]
+      end
 
-        @input  = Pathname(options[:input])
-        @output = Pathname(options[:output])
+      private
 
-        # -- Test helper
-        copy_file "templates/ruby/test_helper.rb", @output.join('test').join('test_helper.rb') if options[:language] == 'ruby'
+      def __generate_source(api)
+        @current_api = api
+        @input = FilesHelper.input_dir(api)
+        @output = FilesHelper.output_dir(api)
 
-        Dir[@input].each do |file|
-          next if File.basename(file)[0] == '_'
-          @path   = Pathname(file)
-          @json   = MultiJson.load( File.read(@path) )
-          @spec   = @json.values.first
+        FilesHelper.files(api).each do |filepath|
+          @path = Pathname(@input.join(filepath))
+          @json = MultiJson.load(File.read(@path))
+          @spec = @json.values.first
           say_status 'json', @path, :yellow
 
           @spec['url'] ||= {}
-          @spec['url']['parts']  ||= []
-          @spec['url']['params'] ||= {}
-
-          # say_status 'JSON', @spec.inspect, options[:verbose]
 
-          @full_namespace   = @json.keys.first.split('.')
-          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size-1 : 0
+          @endpoint_name    = @json.keys.first
+          @full_namespace   = __full_namespace
+          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size - 1 : 0
           @module_namespace = @full_namespace[0, @namespace_depth]
           @method_name      = @full_namespace.last
-          @http_method      = "Elasticsearch::API::HTTP_#{@spec['methods'].first}"
-          @http_path        = unless @spec['url']['parts'].empty?
-                                @spec['url']['path']
-                                  .split('/')
-                                  .compact
-                                  .reject { |p| p =~ /^\s*$/ }
-                                  .map { |p| p =~ /\{/ ? "\#\{arguments[:#{p.tr('{}', '')}]\}" : p }
-                                  .join('/')
-                                  .gsub(/^\//, '')
-                              else
-                                @spec['url']['path'].gsub(/^\//, '')
-                              end
-
-          # -- Ruby files
-
-          @path_to_file = @output.join('api').join( @module_namespace.join('/') ).join("#{@method_name}.rb")
-
-          empty_directory @output.join('api').join( @module_namespace.join('/') )
-
-          template "templates/#{options[:language]}/method.erb", @path_to_file
-
-          if options[:verbose]
-            colorized_output   = CodeRay.scan_file(@path_to_file, :ruby).terminal
-            lines              =  colorized_output.split("\n")
-            say_status options[:language].downcase,
-                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"),
-                       :yellow
+          @parts            = __endpoint_parts
+          @params           = @spec['params'] || {}
+          @specific_params  = specific_params(@module_namespace.first) # See EndpointSpecifics
+          @http_method      = __http_method
+          @paths            = @spec['url']['paths'].map { |b| b['path'] }
+          # Using Ruby's safe operator on array:
+          @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
+          @http_path        = __http_path
+          @required_parts   = __required_parts
+
+          @module_namespace.shift if @module_namespace.first == 'xpack'
+          @path_to_file = @output.join(@module_namespace.join('/')).join("#{@method_name}.rb")
+          dir = @output.join(@module_namespace.join('/'))
+          empty_directory(dir, verbose: false)
+
+          # Write the file with the ERB template:
+          template('templates/method.erb', @path_to_file, force: true)
+
+          print_source_code(@path_to_file) if options[:verbose]
+
+          generate_tests if options[:tests]
+
+          puts
+        end
+
+        run_rubocop(api)
+      end
+
+      def __full_namespace
+        names = @endpoint_name.split('.')
+        if @current_api == :xpack
+          names = (names.first == 'xpack' ? names : ['xpack', names].flatten)
+          # Return an array to expand 'ccr', 'ilm', 'ml' and 'slm'
+          names.map do |name|
+            name
+              .gsub(/^ml$/, 'machine_learning')
+              .gsub(/^ilm$/, 'index_lifecycle_management')
+              .gsub(/^ccr/, 'cross_cluster_replication')
+              .gsub(/^slm/, 'snapshot_lifecycle_management')
           end
+        else
+          names
+        end
+      end
 
-          # --- Test files
+      # Create the hierarchy of directories based on API namespaces
+      #
+      def __create_directories(key, value)
+        return if value['documentation']
 
-          @test_directory = @output.join('test/api').join( @module_namespace.join('/') )
-          @test_file      = @test_directory.join("#{@method_name}_test.rb")
+        empty_directory @output.join(key)
+        create_directory_hierarchy * value.to_a.first
+      end
 
-          empty_directory @test_directory
-          template "templates/#{options[:language]}/test.erb", @test_file
+      # Extract parts from each path
+      #
+      def __endpoint_parts
+        parts = @spec['url']['paths'].select do |a|
+          a.keys.include?('parts')
+        end.map do |path|
+          path&.[]('parts')
+        end
+        (parts.inject(&:merge) || [])
+      end
 
-          if options[:verbose]
-            colorized_output   = colorized_output   = CodeRay.scan_file(@test_file, :ruby).terminal
-            lines              =  colorized_output.split("\n")
-            say_status options[:language].downcase,
-                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"),
-                       :yellow
-            say '▬'*terminal_width
+      def __http_method
+        return '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @endpoint_name == 'index'
+
+        default_method = @spec['url']['paths'].map { |a| a['methods'] }.flatten.first
+        if @spec['body'] && default_method == 'GET'
+          # When default method is GET and body is required, we should always use POST
+          if @spec['body']['required']
+            'Elasticsearch::API::HTTP_POST'
+          else
+            <<~SRC
+              if arguments[:body]
+                Elasticsearch::API::HTTP_POST
+              else
+                Elasticsearch::API::HTTP_GET
+              end
+            SRC
           end
+        else
+          "Elasticsearch::API::HTTP_#{default_method}"
         end
+      end
 
-        # -- Tree output
+      def __http_path
+        return "\"#{__parse_path(@paths.first)}\"" if @paths.size == 1
+        return termvectors_path if @method_name == 'termvectors'
 
-        if options[:verbose] && `which tree > /dev/null 2>&1; echo $?`.to_i < 1
-          lines = `tree #{@output}`.split("\n")
-          say_status 'tree',
-                     lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n")
+        result = ''
+        anchor_string = []
+        @paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
+          var_string = __extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
+          next if anchor_string.include? var_string
+
+          anchor_string << var_string
+          result += if i.zero?
+                      "if #{var_string}\n"
+                    elsif (i == @paths.size - 1) || var_string.empty?
+                      "else\n"
+                    else
+                      "elsif #{var_string}\n"
+                    end
+          result += "\"#{__parse_path(path)}\"\n"
         end
+        result += 'end'
+        result
       end
 
-      private
+      def __parse_path(path)
+        path.gsub(/^\//, '')
+            .gsub(/\/$/, '')
+            .gsub('{', "\#{#{__utils}.__listify(_")
+            .gsub('}', ')}')
+      end
 
-      # Create the hierarchy of directories based on API namespaces
+      def __path_variables
+        @paths.map do |path|
+          __extract_path_variables(path)
+        end
+      end
+
+      # extract values that are in the {var} format:
+      def __extract_path_variables(path)
+        path.scan(/{(\w+)}/).flatten
+      end
+
+      # Find parts that are definitely required and should raise an error if
+      # they're not present
       #
-      def __create_directories(key, value)
-        unless value['documentation']
-          empty_directory @output.join(key)
-          create_directory_hierarchy *value.to_a.first
+      def __required_parts
+        required = []
+        return required if @endpoint_name == 'tasks.get'
+
+        required << 'body' if (@spec['body'] && @spec['body']['required'])
+        # Get required variables from paths:
+        req_variables = __path_variables.inject(:&) # find intersection
+        required << req_variables unless req_variables.empty?
+        required.flatten
+      end
+
+      def docs_helper(name, info)
+        info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings'
+        tipo = info['type'] ? info['type'].capitalize : 'String'
+        description = info['description'] ? info['description'].strip : '[TODO]'
+        options = info['options'] ? "(options: #{info['options'].join(', ').strip})" : nil
+        required = info['required'] ? '(*Required*)' : ''
+        deprecated = info['deprecated'] ? '*Deprecated*' : ''
+        optionals = [required, deprecated, options].join(' ').strip
+
+        "# @option arguments [#{tipo}] :#{name} #{description} #{optionals}\n"
+      end
+
+      def stability_doc_helper(stability)
+        return if stability == 'stable'
+
+        if stability == 'experimental'
+          <<~MSG
+            # This functionality is Experimental and may be changed or removed
+            # completely in a future release. Elastic will take a best effort approach
+            # to fix any issues, but experimental features are not subject to the
+            # support SLA of official GA features.
+          MSG
+        elsif stability == 'beta'
+          <<~MSG
+            # This functionality is in Beta and is subject to change. The design and
+            # code is less mature than official GA features and is being provided
+            # as-is with no warranties. Beta features are not subject to the support
+            # SLA of official GA features.
+          MSG
+        else
+          <<~MSG
+            # This functionality is subject to potential breaking changes within a
+            # minor version, meaning that your referencing code may break when this
+            # library is upgraded.
+          MSG
         end
       end
 
+      def generate_tests
+        copy_file 'templates/test_helper.rb', @output.join('test').join('test_helper.rb')
+
+        @test_directory = @output.join('test/api').join(@module_namespace.join('/'))
+        @test_file      = @test_directory.join("#{@method_name}_test.rb")
+
+        empty_directory @test_directory
+        template 'templates/test.erb', @test_file
+
+        print_source_code(@test_file) if options[:verbose]
+      end
+
+      def print_source_code(path_to_file)
+        colorized_output = CodeRay.scan_file(path_to_file, :ruby).terminal
+        lines            = colorized_output.split("\n")
+        formatted        = lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n")
+
+        say_status('ruby', formatted, :yellow)
+      end
+
+      def print_tree
+        return unless `which tree > /dev/null 2>&1; echo $?`.to_i < 1
+
+        lines = `tree #{@output}`.split("\n")
+        say_status('tree', lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n"))
+      end
+
+      def __utils
+        (@current_api == :xpack) ? 'Elasticsearch::API::Utils' : 'Utils'
+      end
+
+      def run_rubocop(api)
+        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -x #{FilesHelper::output_dir(api)}")
+      end
     end
   end
 end

data/utils/thor/generator/endpoint_specifics.rb

@@ -0,0 +1,181 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Elasticsearch
+  module API
+    # Handles specific exceptional parameters and code snippets that need to be
+    # included in the generated code. This module is included in SourceGenerator
+    # so its methods can be used from the ERB template (method.erb). This will
+    # potentially be refactored into different templates.
+    module EndpointSpecifics
+      # Endpoints that need Utils.__rescue_from_not_found
+      IGNORE_404 = %w[
+        exists
+        indices.exists
+        indices.exists_alias
+        indices.exists_template
+        indices.exists_type
+      ].freeze
+
+      # Endpoints that need Utils.__rescue_from_not_found if the ignore
+      # parameter is included
+      COMPLEX_IGNORE_404 = %w[
+        delete
+        get
+        indices.flush_synced
+        indices.delete_template
+        indices.delete
+        security.get_role
+        security.get_user
+        snapshot.status
+        snapshot.get
+        snapshot.get_repository
+        snapshot.delete_repository
+        snapshot.delete
+        update
+        watcher.delete_watch
+      ].freeze
+
+      # Endpoints that need params[:h] listified
+      H_PARAMS = %w[aliases allocation count health indices nodes pending_tasks
+                    recovery shards thread_pool].freeze
+
+      # Function that adds the listified h param code
+      def specific_params(namespace)
+        params = []
+        if H_PARAMS.include?(@method_name) && namespace == 'cat'
+          if @method_name == 'nodes'
+            params << 'params[:h] = Utils.__listify(params[:h], escape: false) if params[:h]'
+          else
+            params << 'params[:h] = Utils.__listify(params[:h]) if params[:h]'
+          end
+        end
+        params
+      end
+
+      def needs_ignore_404?(endpoint)
+        IGNORE_404.include? endpoint
+      end
+
+      def needs_complex_ignore_404?(endpoint)
+        COMPLEX_IGNORE_404.include? endpoint
+      end
+
+      def module_name_helper(name)
+        return name.upcase if %w[sql ssl].include? name
+
+        name.split('_').map(&:capitalize).join
+      end
+
+      def termvectors_path
+        <<~SRC
+          if _index && _type && _id
+            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{Utils.__listify(_id)}/\#{endpoint}"
+          elsif _index && _type
+            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{endpoint}"
+          elsif _index && _id
+            "\#{Utils.__listify(_index)}/\#{endpoint}/\#{Utils.__listify(_id)}"
+          else
+            "\#{Utils.__listify(_index)}/\#{endpoint}"
+          end
+        SRC
+      end
+
+      def ping_perform_request
+        <<~SRC
+          begin
+            perform_request(method, path, params, body, headers).status == 200 ? true : false
+          rescue Exception => e
+            if e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i
+              false
+            else
+              raise e
+            end
+          end
+        SRC
+      end
+
+      def indices_stats_params_registry
+        <<~SRC
+          ParamsRegistry.register(:stats_params, [
+            #{@spec['params'].keys.map { |k| ":#{k}" }.join(",\n")}
+          ].freeze)
+
+          ParamsRegistry.register(:stats_parts, [
+            #{@parts['metric']['options'].push('metric').map { |k| ":#{k}" }.join(",\n")}
+          ].freeze)
+        SRC
+      end
+
+      def msearch_body_helper
+        <<~SRC
+          case
+          when body.is_a?(Array) && body.any? { |d| d.has_key? :search }
+            payload = body.
+              inject([]) do |sum, item|
+                meta = item
+                data = meta.delete(:search)
+
+                sum << meta
+                sum << data
+                sum
+              end.
+              map { |item| Elasticsearch::API.serializer.dump(item) }
+            payload << "" unless payload.empty?
+            payload = payload.join("\\n")
+          when body.is_a?(Array)
+            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
+            payload << "" unless payload.empty?
+            payload = payload.join("\\n")
+          else
+            payload = body
+          end
+        SRC
+      end
+
+      def msearch_template_body_helper
+        <<~SRC
+          case
+          when body.is_a?(Array)
+            payload = body.map { |d| d.is_a?(String) ? d : Elasticsearch::API.serializer.dump(d) }
+            payload << "" unless payload.empty?
+            payload = payload.join("\n")
+          else
+            payload = body
+          end
+        SRC
+      end
+
+      def bulk_body_helper
+        <<~SRC
+          if body.is_a? Array
+            payload = Elasticsearch::API::Utils.__bulkify(body)
+          else
+            payload = body
+          end
+        SRC
+      end
+
+      def bulk_doc_helper(info)
+        <<~SRC
+          # @option arguments [String|Array] :body #{info}. Array of Strings, Header/Data pairs,
+          # or the conveniency "combined" format can be passed, refer to Elasticsearch::API::Utils.__bulkify documentation.
+        SRC
+      end
+    end
+  end
+end

data/utils/thor/generator/files_helper.rb

@@ -0,0 +1,60 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'pathname'
+require_relative '../../../lib/elasticsearch/api/version.rb'
+
+module Elasticsearch
+  module API
+    module FilesHelper
+      OSS_SRC_PATH   = '../../../../../tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/'.freeze
+      OSS_OUTPUT_DIR = '../../elasticsearch-api/lib/elasticsearch/api/actions'.freeze
+
+      XPACK_SRC_PATH   = '../../../../../tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/api'.freeze
+      XPACK_OUTPUT_DIR = '../../elasticsearch-xpack/lib/elasticsearch/xpack/api/actions'.freeze
+
+      # Path to directory with JSON API specs
+      def self.input_dir(api)
+        input_dir = if api == :xpack
+                      File.expand_path(XPACK_SRC_PATH, __FILE__)
+                    else
+                      File.expand_path(OSS_SRC_PATH, __FILE__)
+                    end
+        Pathname(input_dir)
+      end
+
+      # Path to directory to copy generated files
+      def self.output_dir(api)
+        api == :xpack ? Pathname(XPACK_OUTPUT_DIR) : Pathname(OSS_OUTPUT_DIR)
+      end
+
+      # Only get JSON files and remove hidden files
+      def self.files(api)
+        Dir.entries(input_dir(api).to_s).reject do |f|
+          f.start_with?('.') ||
+            f.start_with?('_') ||
+            File.extname(f) != '.json'
+        end
+      end
+
+      def self.gem_version
+        regex = /([0-9]{1,2}\.[0-9x]{1,2})/
+        Elasticsearch::API::VERSION.match(regex)[0]
+      end
+    end
+  end
+end

data/utils/thor/lister.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: UTF-8
 
 require 'thor'
@@ -5,16 +22,16 @@ require 'thor'
 require 'pathname'
 
 module Elasticsearch
-
   module API
-
     class Lister < Thor
       namespace 'api'
 
+      DEFAULT_PATH = '../../tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/api/'.freeze
+
       desc "list <PATH DIRECTORY WITH JSON SPEC FILES>", "List all the REST API endpoints from the JSON specification"
       method_option :verbose,  type: :boolean, default: false, desc: 'Output more information'
       method_option :format,   default: 'text', desc: 'Output format (text, json)'
-      def list(directory)
+      def list(directory = DEFAULT_PATH)
         input = Pathname(directory).join('*.json')
         apis = Dir[input.to_s].map do |f|
           File.basename(f, '.json')
@@ -36,6 +53,5 @@ module Elasticsearch
         end
       end
     end
-
   end
 end