elasticsearch-api 1.0.6 → 1.0.7

data/utils/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+gem 'activesupport'
+gem 'rest-client'
+gem 'coderay'
+gem 'multi_json'
+gem 'thor'
+gem 'json'
+gem 'pry'

data/utils/Thorfile

@@ -0,0 +1,3 @@
+require File.expand_path('./thor/generate_api')
+require File.expand_path('./thor/generate_source')
+require File.expand_path('./thor/lister')

data/utils/thor/generate_api.rb

@@ -0,0 +1,189 @@
+require 'thor'
+
+require 'pathname'
+require 'active_support/core_ext/hash/deep_merge'
+require 'active_support/inflector/methods'
+require 'rest_client'
+require 'json'
+require 'pry'
+
+module Elasticsearch
+
+  module API
+    module Utils
+      # controller.registerHandler(RestRequest.Method.GET, "/_cluster/health", this);
+      PATTERN_REST = /.*controller.registerHandler\(.*(?<method>GET|POST|PUT|DELETE|HEAD|OPTIONS|PATCH)\s*,\s*"(?<url>.*)"\s*,\s*.+\);/
+      # request.param("index"), request.paramAsBoolean("docs", indicesStatsRequest.docs()), etc
+      PATTERN_URL_PARAMS = /request.param.*\("(?<param>[a-z_]+)".*/
+      # controller.registerHandler(GET, "/{index}/_refresh", this)
+      PATTERN_URL_PARTS  = /\{(?<part>[a-zA-Z0-9\_\-]+)\}/
+      # request.hasContent()
+      PATTERN_HAS_BODY   = /request\.hasContent()/
+
+      # Parses the Elasticsearch source code and returns a Hash of REST API information/specs.
+      #
+      # Example:
+      #
+      #     {
+      #       "cluster.health" => [
+      #         { "method" => "GET",
+      #           "path"   => "/_cluster/health",
+      #           "parts"  => ["index"],
+      #           "params" => ["index", "local",  ... ],
+      #           "body"   => false
+      #     }
+      #
+      def __parse_java_source(path)
+        path  += '/' unless path =~ /\/$/ # Add trailing slash if missing
+        prefix = "src/main/java/org/elasticsearch/rest/action"
+
+        java_rest_files = Dir["#{path}#{prefix}/**/*.java"]
+
+        map = {}
+
+        java_rest_files.sort.each do |file|
+          content = File.read(file)
+          parts   = file.gsub(path+prefix, '').split('/')
+          name    = parts[0, parts.size-1].reject { |p| p =~ /^\s*$/ }.join('.')
+
+          # Remove the `admin` namespace
+          name.gsub! /admin\./, ''
+
+          # Extract params
+          url_params = content.scan(PATTERN_URL_PARAMS).map { |n| n.first }.sort
+
+          # Extract parts
+          url_parts = content.scan(PATTERN_URL_PARTS).map { |n| n.first }.sort
+
+          # Extract if body allowed
+          has_body  = !!content.match(PATTERN_HAS_BODY)
+
+          # Extract HTTP method and path
+          content.scan(PATTERN_REST) do |method, path|
+            (map[name] ||= []) << { 'method' => method,
+                                    'path'   => path,
+                                    'parts'  => url_parts,
+                                    'params' => url_params,
+                                    'body'   => has_body }
+          end
+
+        end
+
+        map
+      end
+
+      extend self
+    end
+
+    # Contains a generator which will parse the Elasticsearch *.java source files,
+    # extract information about REST API endpoints (URLs, HTTP methods, URL parameters, etc),
+    # and create a skeleton of the JSON API specification file for each endpoint.
+    #
+    # Usage:
+    #
+    #     $ thor help api:generate:spec
+    #
+    # Example:
+    #
+    # time thor api:generate:spec \
+    #             --force \
+    #             --verbose \
+    #             --crawl \
+    #             --elasticsearch=/path/to/elasticsearch/source/code
+    #
+    # Features:
+    #
+    # * Extract the API name from the source filename (eg. `admin/cluster/health/RestClusterHealthAction.java` -> `cluster.health`)
+    # * Extract the URLs from the `registerHandler` statements
+    # * Extract the URL parts (eg. `{index}`) from the URLs
+    # * Extract the URL parameters (eg. `{timeout}`) from the `request.param("ABC")` statements
+    # * Detect whether HTTP body is allowed for the API from `request.hasContent()` statements
+    # * Search the <http://elasticsearch.org> website to get proper documentation URLs
+    # * Assemble the JSON format for the API spec
+    #
+    class JsonGenerator < Thor
+      namespace 'api:spec'
+
+      include Thor::Actions
+
+      __root = Pathname( File.expand_path('../../..', __FILE__) )
+
+      # Usage: thor help api:generate:spec
+      #
+      desc "generate", "Generate JSON API spec files from Elasticsearch source code"
+      method_option :force,     type: :boolean, default: false,            desc: 'Overwrite the output'
+      method_option :verbose,   type: :boolean, default: false,            desc: 'Output more information'
+      method_option :output,    default: __root.join('tmp/out'),           desc: 'Path to output directory'
+      method_option :elasticsearch, default: __root.join('tmp/elasticsearch'), desc: 'Path to directory with Elasticsearch source code'
+      method_option :crawl,     type: :boolean, default: false,            desc: 'Extract URLs from Elasticsearch website'
+
+      def generate
+        self.class.source_root File.expand_path('../', __FILE__)
+
+        @output = Pathname(options[:output])
+
+        rest_actions = Utils.__parse_java_source(options[:elasticsearch].to_s)
+
+        if rest_actions.empty?
+          say_status 'ERROR', 'Cannot find Elasticsearch source in ' + options[:elasticsearch].to_s, :red
+          exit(1)
+        end
+
+        rest_actions.each do |name, info|
+          doc_url  = ""
+          parts    = info.reduce([]) { |sum, n| sum |= n['parts']; sum }.reduce({}) { |sum, n| sum[n] = {}; sum }
+          params   = info.reduce([]) { |sum, n| sum |= n['params']; sum }.reduce({}) { |sum, n| sum[n] = {}; sum }
+
+          if options[:crawl]
+            begin
+              response = RestClient.get "http://search.elasticsearch.org/elastic-search-website/guide/_search?q=#{URI.escape(name.gsub(/\./, ' '))}"
+              hits = JSON.load(response)['hits']['hits']
+              if hit = hits.first
+                if hit['_score'] > 0.2
+                  doc_title = hit['fields']['title']
+                  doc_url   = "http://elasticsearch.org" + hit['fields']['url']
+                end
+              end
+            rescue Exception => e
+              puts "[!] ERROR: #{e.inspect}"
+            end
+          end
+
+          spec     = {
+            name => {
+              'documentation' => doc_url,
+
+              'methods' => info.map { |n| n['method'] }.uniq,
+
+              'url' => {
+                'path'   => info.first['path'],
+                'paths'  => info.map { |n| n['path'] }.uniq,
+                'parts'  => parts,
+                'params' => params
+              },
+
+              'body' => info.first['body'] ? {} : nil
+            }
+          }
+
+          json = JSON.pretty_generate(spec, indent: '  ', array_nl: '', object_nl: "\n", space: ' ', space_before: ' ')
+
+          # Fix JSON array formatting
+          json.gsub!(/\[\s+/, '[')
+          json.gsub!(/, {2,}"/, ', "')
+
+          create_file @output.join( "#{name}.json" ), json + "\n"
+
+          if options[:verbose]
+            lines = json.split("\n")
+            say_status 'JSON',
+                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n")
+          end
+        end
+      end
+
+      private
+
+    end
+  end
+end

data/utils/thor/generate_source.rb

@@ -0,0 +1,122 @@
+# encoding: UTF-8
+
+require 'thor'
+
+require 'pathname'
+require 'active_support/core_ext/hash/deep_merge'
+require 'active_support/inflector'
+require 'multi_json'
+require 'coderay'
+require 'pry'
+
+module Elasticsearch
+
+  module API
+
+    # A command line application based on [Thor](https://github.com/wycats/thor),
+    # which will read the JSON API spec file(s), and generate
+    # the Ruby source code (one file per API endpoint) with correct
+    # module namespace, method names, and RDoc documentation,
+    # as well as test files for each endpoint.
+    #
+    # Currently it only generates Ruby source, but can easily be
+    # extended and adapted to generate source code for other
+    # programming languages.
+    #
+    class SourceGenerator < Thor
+      namespace 'api:code'
+
+      include Thor::Actions
+
+      __root = Pathname( File.expand_path('../../..', __FILE__) )
+
+      desc "generate <PATH TO JSON SPEC FILES>", "Generate source code and tests from the REST API JSON specification"
+      method_option :language,  default: 'ruby',                                          desc: 'Programming language'
+      method_option :force,     type: :boolean, default: false,                           desc: 'Overwrite the output'
+      method_option :verbose,   type: :boolean, default: false,                           desc: 'Output more information'
+      method_option :input,     default: File.expand_path('../../../../tmp/elasticsearch/rest-api-spec/api/**/*.json', __FILE__), desc: 'Path to directory with JSON API specs'
+      method_option :output,    default: File.expand_path('../../../tmp/out', __FILE__),        desc: 'Path to output directory'
+
+      def generate(*files)
+        self.class.source_root File.expand_path('../', __FILE__)
+
+        @input  = Pathname(options[:input])
+        @output = Pathname(options[:output])
+
+        # -- Test helper
+        copy_file "templates/ruby/test_helper.rb", @output.join('test').join('test_helper.rb') if options[:language] == 'ruby'
+
+        Dir[@input].each do |file|
+          @path   = Pathname(file)
+          @json   = MultiJson.load( File.read(@path) )
+          @spec   = @json.values.first
+          say_status 'json', @path, :yellow
+
+          @spec['url'] ||= {}
+          @spec['url']['parts']  ||= []
+          @spec['url']['params'] ||= {}
+
+          # say_status 'JSON', @spec.inspect, options[:verbose]
+
+          @full_namespace   = @json.keys.first.split('.')
+          @namespace_depth  = @full_namespace.size > 0 ? @full_namespace.size-1 : 0
+          @module_namespace = @full_namespace[0, @namespace_depth]
+          @method_name      = @full_namespace.last
+
+          # -- Ruby files
+
+          @path_to_file = @output.join('api').join( @module_namespace.join('/') ).join("#{@method_name}.rb")
+
+          empty_directory @output.join('api').join( @module_namespace.join('/') )
+
+          template "templates/#{options[:language]}/method.erb", @path_to_file
+
+          if options[:verbose]
+            colorized_output   = CodeRay.scan_file(@path_to_file, :ruby).terminal
+            lines              =  colorized_output.split("\n")
+            say_status options[:language].downcase,
+                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"),
+                       :yellow
+          end
+
+          # --- Test files
+
+          @test_directory = @output.join('test/api').join( @module_namespace.join('/') )
+          @test_file      = @test_directory.join("#{@method_name}_test.rb")
+
+          empty_directory @test_directory
+          template "templates/#{options[:language]}/test.erb", @test_file
+
+          if options[:verbose]
+            colorized_output   = colorized_output   = CodeRay.scan_file(@test_file, :ruby).terminal
+            lines              =  colorized_output.split("\n")
+            say_status options[:language].downcase,
+                       lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n"),
+                       :yellow
+            say '▬'*terminal_width
+          end
+        end
+
+        # -- Tree output
+
+        if options[:verbose] && `which tree > /dev/null 2>&1`
+          lines = `tree #{@output}`.split("\n")
+          say_status 'tree',
+                     lines.first  + "\n" + lines[1, lines.size].map { |l| ' '*14 + l }.join("\n")
+        end
+      end
+
+      private
+
+      # Create the hierarchy of directories based on API namespaces
+      #
+      def __create_directories(key, value)
+        unless value['documentation']
+          empty_directory @output.join(key)
+          create_directory_hierarchy *value.to_a.first
+        end
+      end
+
+    end
+  end
+end

data/utils/thor/lister.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+require 'thor'
+
+require 'pathname'
+
+module Elasticsearch
+
+  module API
+
+    class Lister < Thor
+      namespace 'api'
+
+      desc "list <PATH DIRECTORY WITH JSON SPEC FILES>", "List all the REST API endpoints from the JSON specification"
+      method_option :verbose,  type: :boolean, default: false, desc: 'Output more information'
+      method_option :format,   default: 'text', desc: 'Output format (text, json)'
+      def list(directory)
+        input = Pathname(directory).join('*.json')
+        apis = Dir[input.to_s].map do |f|
+          File.basename(f, '.json')
+        end.sort
+
+        if options[:verbose]
+          say_status 'Count', apis.size
+          say '▬'*terminal_width
+        end
+
+        case options[:format]
+          when 'text'
+            apis.each { |a| puts "* #{a}" }
+          when 'json'
+            puts apis.inspect
+          else
+            puts "[!] ERROR: Unknown output format '#{options[:format]}'"
+            exit(1)
+        end
+      end
+    end
+
+  end
+end

data/utils/thor/templates/ruby/method.erb

@@ -0,0 +1,62 @@
+module Elasticsearch
+  module API
+    <%- @module_namespace.each_with_index do |name, i| -%>
+    <%= '  '*i %>module <%= name.capitalize %>
+    <%- end -%>
+    <%= '  '*@namespace_depth %>module Actions
+
+      <%= '  '*@namespace_depth %># <%= @spec['description'] || 'TODO: Description' %>
+      <%= '  '*@namespace_depth %>#
+<%# URL parts -%>
+      <%- @spec['url']['parts'].each do |name,info| -%>
+      <%- info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings' -%>
+      <%= '  '*@namespace_depth + "# @option arguments [#{info['type'] ? info['type'].capitalize : 'String'}] :#{name} #{info['description']}" + ( info['required'] ? ' (*Required*)' : '' ) -%><%= " (options: #{info['options'].join(', ')})" if info['options'] -%>
+      <%= "\n" -%>
+      <%- end -%>
+<%# Body -%>
+<%= '  '*(@namespace_depth+3) + '# @option arguments [Hash] :body ' + (@spec['body']['description'] || 'TODO: Description') + (@spec['body']['required'] ? ' (*Required*)' : '') + "\n" if @spec['body'] -%>
+<%# URL parameters -%>
+      <%- @spec['url']['params'].each do |name,info| -%>
+      <%- info['type'] = 'String' if info['type'] == 'enum' # Rename 'enums' to 'strings' -%>
+      <%= '  '*@namespace_depth + "# @option arguments [#{info['type'] ? info['type'].capitalize : 'String'}] :#{name} #{info['description']}" -%><%= " (options: #{info['options'].join(', ')})" if info['options'] -%>
+      <%=  "\n" -%>
+      <%- end if @spec['url']['parts'] -%>
+      <%= '  '*@namespace_depth -%>#
+<%# Documentation link -%>
+      <%= '  '*@namespace_depth %># @see <%= @spec['documentation'] %>
+      <%= '  '*@namespace_depth %>#
+<%# Method definition -%>
+      <%= '  '*@namespace_depth -%>def <%= @method_name %>(arguments={})
+<%# Required arguments -%>
+      <%- @spec['url']['parts'].select { |name, info| info['required'] }.each do |name, info| -%>
+      <%= '  '*(@namespace_depth+1) + "raise ArgumentError, \"Required argument '#{name}' missing\" unless arguments[:#{name}]" + "\n" -%>
+      <%- end -%>
+<%- if @spec['body'] && @spec['body']['required'] -%>
+      <%= '  '*(@namespace_depth+1) + "raise ArgumentError, \"Required argument 'body' missing\" unless arguments[:body]" + "\n" -%>
+<%- end -%>
+<%# Method, path, params, body  -%>
+      <%= '  '*@namespace_depth %>  valid_params = [
+      <%= '  '*(@namespace_depth+2) %><%= @spec['url']['params'].keys.map { |k| ":#{k}" }.join(",\n#{'  '*(@namespace_depth+5)}") %> ]
+      <%= '  '*@namespace_depth %>  method = '<%= @spec['methods'].first %>'
+      <%- unless @spec['url']['parts'].empty?  -%>
+      <%= '  '*@namespace_depth %>  path   = "<%= @spec['url']['path'].split('/').compact.reject {|p| p =~ /^\s*$/}.map do |p|
+        p =~ /\{/ ? "\#\{arguments[:#{p.tr('{}', '')}]\}" : p
+      end.join('/') %>"
+      <%- else -%>
+      <%= '  '*@namespace_depth %>  path   = "<%= @spec['url']['path'] %>"
+      <%- end -%>
+      <%- unless @spec['url']['params'].keys.empty? -%>
+      <%= '  '*@namespace_depth %>  params = Utils.__validate_and_extract_params arguments, valid_params
+      <%- else -%>
+      <%= '  '*@namespace_depth %>  params = {}
+      <%- end -%>
+      <%= '  '*@namespace_depth %>  body   = <%= @spec['body'].nil? ? 'nil' : 'arguments[:body]' %>
+<%# Perform request %>
+      <%= '  '*@namespace_depth %>  perform_request(method, path, params, body).body
+      <%= '  '*@namespace_depth %>end
+      <%- @namespace_depth.downto(1) do |i| -%>
+      <%= '  '*(i-1) %>end
+      <%- end if @namespace_depth > 0 -%>
+    end
+  end
+end

data/utils/thor/templates/ruby/test.erb

@@ -0,0 +1,26 @@
+require 'test_helper'
+
+module Elasticsearch
+  module Test
+    class <%= @module_namespace.empty? ? @method_name.camelize : @module_namespace.map {|n| n.capitalize}.join + @method_name.camelize %>Test < ::Test::Unit::TestCase
+
+      context "<%= @module_namespace.empty? ? '' : @module_namespace.map {|n| n.capitalize}.join + ': ' %><%= @method_name.humanize %>" do
+        subject { FakeClient.new }
+
+        should "perform correct request" do
+          subject.expects(:perform_request).with do |method, url, params, body|
+            assert_equal 'FAKE', method
+            assert_equal 'test', url
+            assert_equal Hash.new, params
+            <%= @spec['body'].nil? ? 'assert_nil   body' : 'assert_equal Hash.new, body' %>
+            true
+          end.returns(FakeResponse.new)
+
+          subject.<%= @full_namespace.join('.') %>
+        end
+
+      end
+
+    end
+  end
+end