elasticsearch-api 7.13.3 → 8.14.0

data/utils/thor/endpoint_spec.rb

@@ -0,0 +1,193 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# encoding: UTF-8
+
+require_relative 'endpoint_specifics'
+
+module Elasticsearch
+  module API
+    class EndpointSpec
+      include EndpointSpecifics
+
+      def initialize(filepath)
+        @path = Pathname(filepath)
+        json = MultiJson.load(File.read(@path))
+        @spec = json.values.first
+        @endpoint_name = json.keys.first
+
+        full_namespace = parse_full_namespace
+        @namespace_depth = full_namespace.size > 0 ? full_namespace.size - 1 : 0
+        @module_namespace = full_namespace[0, @namespace_depth]
+        @method_name = full_namespace.last
+
+        @path_parts = parse_endpoint_parts(@spec)
+        @params = @spec['params'] || {}
+        @paths = @spec['url']['paths'].map { |b| b['path'] } if @spec['url']
+        @path_params   = path_variables.flatten.uniq.collect(&:to_sym)
+        @http_method = parse_http_method(@spec)
+        @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
+        @http_path        = parse_http_path(@paths)
+        @required_parts   = parse_required_parts(@spec)
+      end
+
+      attr_reader :module_namespace,
+                  :method_name,
+                  :endpoint_name,
+                  :path,
+                  :path_parts,
+                  :params,
+                  :deprecation_note,
+                  :namespace_depth,
+                  :http_path,
+                  :required_parts,
+                  :http_method,
+                  :path_params,
+                  :perform_request_opts
+
+      def body
+        @spec['body']
+      end
+
+      def documentation
+        @spec['documentation']
+      end
+
+      def stability
+        @spec['stability']
+      end
+
+      # Function that adds the listified h param code
+      def specific_params
+        super(@module_namespace.first, @method_name)
+      end
+
+      private
+
+      def parse_full_namespace
+        names = @endpoint_name.split('.')
+        # Return an array to expand 'ccr', 'ilm', 'ml' and 'slm'
+        names.map do |name|
+          name
+            .gsub(/^ml$/, 'machine_learning')
+            .gsub(/^ilm$/, 'index_lifecycle_management')
+            .gsub(/^ccr/, 'cross_cluster_replication')
+            .gsub(/^slm/, 'snapshot_lifecycle_management')
+        end
+      end
+
+      def parse_endpoint_parts(spec)
+        parts = spec['url']['paths'].select do |a|
+          a.keys.include?('parts')
+        end.map do |path|
+          path&.[]('parts')
+        end
+        (parts.inject(&:merge) || [])
+      end
+
+      def parse_http_method(spec)
+        return '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @endpoint_name == 'index'
+        return '_name ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @method_name == 'create_service_token'
+        return post_and_get if @endpoint_name == 'count'
+
+        default_method = spec['url']['paths'].map { |a| a['methods'] }.flatten.first
+        if spec['body'] && default_method == 'GET'
+          # When default method is GET and body is required, we should always use POST
+          if spec['body']['required']
+            'Elasticsearch::API::HTTP_POST'
+          else
+            post_and_get
+          end
+        else
+          "Elasticsearch::API::HTTP_#{default_method}"
+        end
+      end
+
+      def parse_http_path(paths)
+        return "\"#{parse_path(paths.first)}\"" if paths.size == 1
+
+        result = ''
+        anchor_string = []
+        paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
+          var_string = extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
+          next if anchor_string.include? var_string
+
+          anchor_string << var_string
+          result += if i.zero?
+                      "if #{var_string}\n"
+                    elsif (i == paths.size - 1) || var_string.empty?
+                      "else\n"
+                    else
+                      "elsif #{var_string}\n"
+                    end
+          result += "\"#{parse_path(path)}\"\n"
+        end
+        result += 'end'
+        result
+      end
+
+      def parse_path(path)
+        path.gsub(/^\//, '')
+            .gsub(/\/$/, '')
+            .gsub('{', "\#{Utils.__listify(_")
+            .gsub('}', ')}')
+      end
+
+      def path_variables
+        @paths.map do |path|
+          extract_path_variables(path)
+        end
+      end
+
+      def parse_path_variables
+        @paths.map do |path|
+          extract_path_variables(path)
+        end
+      end
+
+      # extract values that are in the {var} format:
+      def extract_path_variables(path)
+        path.scan(/{(\w+)}/).flatten
+      end
+
+      # Find parts that are definitely required and should raise an error if
+      # they're not present
+      #
+      def parse_required_parts(spec)
+        required = []
+        return required if @endpoint_name == 'tasks.get'
+
+        required << 'body' if (spec['body'] && spec['body']['required'])
+        # Get required variables from paths:
+        req_variables = parse_path_variables.inject(:&) # find intersection
+        required << req_variables unless req_variables.empty?
+        required.flatten
+      end
+
+      def post_and_get
+        # the METHOD is defined after doing arguments.delete(:body), so we need to check for `body`
+        <<~SRC
+          if body
+            Elasticsearch::API::HTTP_POST
+          else
+            Elasticsearch::API::HTTP_GET
+          end
+        SRC
+      end
+    end
+  end
+end

data/utils/thor/{generator/endpoint_specifics.rb → endpoint_specifics.rb}

@@ -28,24 +28,24 @@ module Elasticsearch
         indices.exists
         indices.exists_alias
         indices.exists_template
-        indices.exists_type
       ].freeze
 
       # Endpoints that need Utils.__rescue_from_not_found if the ignore
       # parameter is included
       COMPLEX_IGNORE_404 = %w[
+        clear_scroll
         delete
         get
-        indices.flush_synced
-        indices.delete_template
         indices.delete
+        indices.delete_template
+        indices.flush_synced
         security.get_role
         security.get_user
-        snapshot.status
+        snapshot.delete
+        snapshot.delete_repository
         snapshot.get
         snapshot.get_repository
-        snapshot.delete_repository
-        snapshot.delete
+        snapshot.status
         update
         watcher.delete_watch
       ].freeze
@@ -55,10 +55,10 @@ module Elasticsearch
                     recovery shards thread_pool].freeze
 
       # Function that adds the listified h param code
-      def specific_params(namespace)
+      def specific_params(namespace, method_name)
         params = []
-        if H_PARAMS.include?(@method_name) && namespace == 'cat'
-          if @method_name == 'nodes'
+        if H_PARAMS.include?(method_name) && namespace == 'cat'
+          if method_name == 'nodes'
             params << 'params[:h] = Utils.__listify(params[:h], escape: false) if params[:h]'
           else
             params << 'params[:h] = Utils.__listify(params[:h]) if params[:h]'
@@ -78,27 +78,13 @@ module Elasticsearch
       def module_name_helper(name)
         return name.upcase if %w[sql ssl].include? name
 
-        name.split('_').map(&:capitalize).join
-      end
-
-      def termvectors_path
-        <<~SRC
-          if _index && _type && _id
-            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{Utils.__listify(_id)}/\#{endpoint}"
-          elsif _index && _type
-            "\#{Utils.__listify(_index)}/\#{Utils.__listify(_type)}/\#{endpoint}"
-          elsif _index && _id
-            "\#{Utils.__listify(_index)}/\#{endpoint}/\#{Utils.__listify(_id)}"
-          else
-            "\#{Utils.__listify(_index)}/\#{endpoint}"
-          end
-        SRC
+        name.split('_').map(&:capitalize).map{ |n| n == 'Xpack' ? 'XPack' : n }.join
       end
 
       def ping_perform_request
         <<~SRC
           begin
-            perform_request(method, path, params, body, headers).status == 200 ? true : false
+            perform_request(method, path, params, body, headers, request_opts).status == 200 ? true : false
           rescue Exception => e
             if e.class.to_s =~ /NotFound|ConnectionFailed/ || e.message =~ /Not\s*Found|404|ConnectionFailed/i
               false
@@ -109,18 +95,6 @@ module Elasticsearch
         SRC
       end
 
-      def indices_stats_params_registry
-        <<~SRC
-          ParamsRegistry.register(:stats_params, [
-            #{@spec['params'].keys.map { |k| ":#{k}" }.join(",\n")}
-          ].freeze)
-
-          ParamsRegistry.register(:stats_parts, [
-            #{@parts['metric']['options'].push('metric').map { |k| ":#{k}" }.join(",\n")}
-          ].freeze)
-        SRC
-      end
-
       def msearch_body_helper
         <<~SRC
           case
@@ -162,11 +136,11 @@ module Elasticsearch
 
       def bulk_body_helper
         <<~SRC
-          if body.is_a? Array
-            payload = Elasticsearch::API::Utils.__bulkify(body)
-          else
-            payload = body
-          end
+          payload = if body.is_a? Array
+                      Elasticsearch::API::Utils.__bulkify(body)
+                    else
+                      body
+                    end
         SRC
       end
 

data/utils/thor/generate_source.rb

@@ -19,13 +19,13 @@
 
 require 'thor'
 require 'pathname'
-require 'active_support/core_ext/hash/deep_merge'
-require 'active_support/inflector'
 require 'multi_json'
 require 'coderay'
 require 'pry'
+require_relative 'generator/build_hash_helper'
+require_relative 'generator/docs_helper'
 require_relative 'generator/files_helper'
-require_relative 'generator/endpoint_specifics'
+require_relative './endpoint_spec'
 
 module Elasticsearch
   module API
@@ -40,226 +40,49 @@ module Elasticsearch
     # here.
     #
     class SourceGenerator < Thor
-      namespace 'api:code'
+      namespace 'code'
       include Thor::Actions
       include EndpointSpecifics
-
-      __root = Pathname(File.expand_path('../../..', __FILE__))
+      include DocsHelper
 
       desc 'generate', 'Generate source code and tests from the REST API JSON specification'
       method_option :verbose, type: :boolean, default: false,  desc: 'Output more information'
       method_option :tests,   type: :boolean, default: false,  desc: 'Generate test files'
-      method_option :api,     type: :array,   default: %w[oss xpack], desc: 'APIs to generate (oss, x-pack)'
 
       def generate
+        @build_hash = BuildHashHelper.build_hash
         self.class.source_root File.expand_path(__dir__)
-        @xpack = options[:api].include? 'xpack'
-        @oss   = options[:api].include? 'oss'
+        generate_source
 
-        __generate_source(:xpack) if @xpack
-        __generate_source(:oss) if @oss
         # -- Tree output
         print_tree if options[:verbose]
       end
 
       private
 
-      def __generate_source(api)
-        @current_api = api
-        @output = FilesHelper.output_dir(api)
-
-        FilesHelper.files(api).each do |filepath|
-          @path = Pathname(filepath)
-          @json = MultiJson.load(File.read(@path))
-          @spec = @json.values.first
-          say_status 'json', @path, :yellow
-
-          @spec['url'] ||= {}
+      def generate_source
+        output = FilesHelper.output_dir
+        cleanup_directory!(output)
 
-          @endpoint_name    = @json.keys.first
-          @full_namespace   = __full_namespace
-          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size - 1 : 0
-          @module_namespace = @full_namespace[0, @namespace_depth]
-          @method_name      = @full_namespace.last
-          @parts            = __endpoint_parts
-          @params           = @spec['params'] || {}
-          @specific_params  = specific_params(@module_namespace.first) # See EndpointSpecifics
-          @http_method      = __http_method
-          @paths            = @spec['url']['paths'].map { |b| b['path'] }
-          # Using Ruby's safe operator on array:
-          @deprecation_note = @spec['url']['paths'].last&.[]('deprecated')
-          @http_path        = __http_path
-          @required_parts   = __required_parts
+        FilesHelper.files.each do |filepath|
+          @spec = EndpointSpec.new(filepath)
+          say_status 'json', @spec.path, :yellow
+          # Don't generate code for internal APIs:
+          next if @spec.module_namespace.flatten.first == '_internal'
 
-          @module_namespace.shift if @module_namespace.first == 'xpack'
-          @path_to_file = @output.join(@module_namespace.join('/')).join("#{@method_name}.rb")
-          dir = @output.join(@module_namespace.join('/'))
+          path_to_file = output.join(@spec.module_namespace.join('/')).join("#{@spec.method_name}.rb")
+          dir = output.join(@spec.module_namespace.join('/'))
           empty_directory(dir, verbose: false)
 
           # Write the file with the ERB template:
-          template('templates/method.erb', @path_to_file, force: true)
-
-          print_source_code(@path_to_file) if options[:verbose]
+          template('templates/method.erb', path_to_file, force: true)
 
+          # Optionals:
+          print_source_code(path_to_file) if options[:verbose]
           generate_tests if options[:tests]
-
-          puts
-        end
-
-        run_rubocop(api)
-      end
-
-      def __full_namespace
-        names = @endpoint_name.split('.')
-        if @current_api == :xpack
-          names = (names.first == 'xpack' ? names : ['xpack', names].flatten)
-          # Return an array to expand 'ccr', 'ilm', 'ml' and 'slm'
-          names.map do |name|
-            name
-              .gsub(/^ml$/, 'machine_learning')
-              .gsub(/^ilm$/, 'index_lifecycle_management')
-              .gsub(/^ccr/, 'cross_cluster_replication')
-              .gsub(/^slm/, 'snapshot_lifecycle_management')
-          end
-        else
-          names
-        end
-      end
-
-      # Create the hierarchy of directories based on API namespaces
-      #
-      def __create_directories(key, value)
-        return if value['documentation']
-
-        empty_directory @output.join(key)
-        create_directory_hierarchy * value.to_a.first
-      end
-
-      # Extract parts from each path
-      #
-      def __endpoint_parts
-        parts = @spec['url']['paths'].select do |a|
-          a.keys.include?('parts')
-        end.map do |path|
-          path&.[]('parts')
-        end
-        (parts.inject(&:merge) || [])
-      end
-
-      def __http_method
-        return '_id ? Elasticsearch::API::HTTP_PUT : Elasticsearch::API::HTTP_POST' if @endpoint_name == 'index'
-
-        default_method = @spec['url']['paths'].map { |a| a['methods'] }.flatten.first
-        if @spec['body'] && default_method == 'GET'
-          # When default method is GET and body is required, we should always use POST
-          if @spec['body']['required']
-            'Elasticsearch::API::HTTP_POST'
-          else
-            <<~SRC
-              if arguments[:body]
-                Elasticsearch::API::HTTP_POST
-              else
-                Elasticsearch::API::HTTP_GET
-              end
-            SRC
-          end
-        else
-          "Elasticsearch::API::HTTP_#{default_method}"
-        end
-      end
-
-      def __http_path
-        return "\"#{__parse_path(@paths.first)}\"" if @paths.size == 1
-        return termvectors_path if @method_name == 'termvectors'
-
-        result = ''
-        anchor_string = []
-        @paths.sort { |a, b| b.length <=> a.length }.each_with_index do |path, i|
-          var_string = __extract_path_variables(path).map { |var| "_#{var}" }.join(' && ')
-          next if anchor_string.include? var_string
-
-          anchor_string << var_string
-          result += if i.zero?
-                      "if #{var_string}\n"
-                    elsif (i == @paths.size - 1) || var_string.empty?
-                      "else\n"
-                    else
-                      "elsif #{var_string}\n"
-                    end
-          result += "\"#{__parse_path(path)}\"\n"
-        end
-        result += 'end'
-        result
-      end
-
-      def __parse_path(path)
-        path.gsub(/^\//, '')
-            .gsub(/\/$/, '')
-            .gsub('{', "\#{#{__utils}.__listify(_")
-            .gsub('}', ')}')
-      end
-
-      def __path_variables
-        @paths.map do |path|
-          __extract_path_variables(path)
-        end
-      end
-
-      # extract values that are in the {var} format:
-      def __extract_path_variables(path)
-        path.scan(/{(\w+)}/).flatten
-      end
-
-      # Find parts that are definitely required and should raise an error if
-      # they're not present
-      #
-      def __required_parts
-        required = []
-        return required if @endpoint_name == 'tasks.get'
-
-        required << 'body' if (@spec['body'] && @spec['body']['required'])
-        # Get required variables from paths:
-        req_variables = __path_variables.inject(:&) # find intersection
-        required << req_variables unless req_variables.empty?
-        required.flatten
-      end
-
-      def docs_helper(name, info)
-        info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings'
-        tipo = info['type'] ? info['type'].capitalize : 'String'
-        description = info['description'] ? info['description'].strip : '[TODO]'
-        options = info['options'] ? "(options: #{info['options'].join(', ').strip})" : nil
-        required = info['required'] ? '(*Required*)' : ''
-        deprecated = info['deprecated'] ? '*Deprecated*' : ''
-        optionals = [required, deprecated, options].join(' ').strip
-
-        "# @option arguments [#{tipo}] :#{name} #{description} #{optionals}\n"
-      end
-
-      def stability_doc_helper(stability)
-        return if stability == 'stable'
-
-        if stability == 'experimental'
-          <<~MSG
-            # This functionality is Experimental and may be changed or removed
-            # completely in a future release. Elastic will take a best effort approach
-            # to fix any issues, but experimental features are not subject to the
-            # support SLA of official GA features.
-          MSG
-        elsif stability == 'beta'
-          <<~MSG
-            # This functionality is in Beta and is subject to change. The design and
-            # code is less mature than official GA features and is being provided
-            # as-is with no warranties. Beta features are not subject to the support
-            # SLA of official GA features.
-          MSG
-        else
-          <<~MSG
-            # This functionality is subject to potential breaking changes within a
-            # minor version, meaning that your referencing code may break when this
-            # library is upgraded.
-          MSG
         end
+        run_rubocop
+        BuildHashHelper.add_hash(@build_hash)
       end
 
       def generate_tests
@@ -289,12 +112,15 @@ module Elasticsearch
         say_status('tree', lines.first + "\n" + lines[1, lines.size].map { |l| ' ' * 14 + l }.join("\n"))
       end
 
-      def __utils
-        (@current_api == :xpack) ? 'Elasticsearch::API::Utils' : 'Utils'
+      def cleanup_directory!(output)
+        Dir["#{output}/**/*.rb"].each do |file|
+          # file = File.join(@output, f)
+          File.delete(file) unless (['.', '..'].include? file) || Pathname(file).directory?
+        end
       end
 
-      def run_rubocop(api)
-        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -x #{FilesHelper::output_dir(api)}")
+      def run_rubocop
+        system("rubocop -c ./thor/.rubocop.yml --format autogenconf -a #{FilesHelper::output_dir}")
       end
     end
   end

data/utils/thor/generator/build_hash_helper.rb

@@ -0,0 +1,64 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Elasticsearch
+  module API
+    # Helper for the Elasticsearch build hash in source code docs
+    module BuildHashHelper
+      class << self
+        def build_hash
+          if ENV['BUILD_HASH']
+            File.read(File.expand_path('../../../tmp/rest-api-spec/build_hash',__dir__))
+          else
+            original_build_hash
+          end
+        end
+
+        def add_hash(build_hash)
+          Dir.glob("#{FilesHelper.output_dir}/**/*.rb").each do |file|
+            content = File.read(file)
+            new_content = content.gsub(/(^#\sunder\sthe\sLicense.\n#)/) do |_|
+              match = Regexp.last_match
+              "#{match[1]}\n#{build_hash_comment(build_hash)}"
+            end
+            File.open(file, 'w') { |f| f.puts new_content }
+          end
+        end
+
+        def build_hash_comment(build_hash)
+          [
+            "Auto generated from build hash #{build_hash}",
+            '@see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec',
+            ''
+          ].map { |b| "# #{b}" }.join("\n").strip
+        end
+
+        private
+
+        def original_build_hash
+          content = File.read("#{FilesHelper.output_dir}/info.rb")
+
+          return unless (match = content.match(/Auto generated from build hash ([a-f0-9]+)/))
+
+          match[1]
+        rescue
+          return 'Unavailable'
+        end
+      end
+    end
+  end
+end