elastomer-client 3.0.1 → 3.1.0

data/lib/elastomer/version_support.rb

@@ -2,21 +2,6 @@ module Elastomer
   # VersionSupport holds methods that (a) encapsulate version differences; or
   # (b) give an intention-revealing name to a conditional check.
   class VersionSupport
-    COMMON_INDEXING_PARAMETER_NAMES = %i[
-      index
-      type
-      id
-      version
-      version_type
-      op_type
-      routing
-      parent
-      refresh
-    ].freeze
-    ES_2_X_INDEXING_PARAMETER_NAMES = %i[consistency ttl timestamp].freeze
-    ES_5_X_INDEXING_PARAMETER_NAMES = %i[wait_for_active_shards].freeze
-    KNOWN_INDEXING_PARAMETER_NAMES =
-      (COMMON_INDEXING_PARAMETER_NAMES + ES_2_X_INDEXING_PARAMETER_NAMES + ES_5_X_INDEXING_PARAMETER_NAMES).freeze
 
     attr_reader :version
 
@@ -43,6 +28,18 @@ module Elastomer
       es_version_5_x?
     end
 
+    # COMPATIBILITY: Return a boolean indicating if this version supports the
+    # `tasks.get` API - https://www.elastic.co/guide/en/elasticsearch/reference/5.x/tasks.html
+    def supports_tasks_get?
+      es_version_5_x?
+    end
+
+    # COMPATIBILITY: Return a boolean indicating if this version supports the
+    # `parent_task_id` param in the tasks API - https://www.elastic.co/guide/en/elasticsearch/reference/5.x/tasks.html
+    def supports_parent_task_id?
+      es_version_5_x?
+    end
+
     # COMPATIBILITY: Return a "text"-type mapping for a field.
     #
     # On ES 2.x, this will be a string field. On ES 5+, it will be a text field.
@@ -78,7 +75,18 @@ module Elastomer
         {enabled: b}
       else
         b
-      end 
+      end
+    end
+
+    # COMPATIBILITY: handle _op_type -> op_type request param conversion for put-if-absent bnehavior
+    # Returns the (possibly mutated) params hash supplied by the caller.
+    #
+    # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#operation-type
+    def op_type(params = {})
+      if es_version_5_x? && (params.key?(:_op_type) || params.key?("_op_type"))
+        params[:op_type] = params.delete(:_op_type)
+      end
+      params
     end
 
     # Elasticsearch 2.0 changed some request formats in a non-backward-compatible
@@ -124,34 +132,22 @@ module Elastomer
     end
 
     # COMPATIBILITY
-    # ES 5.X supports `delete_by_query` natively again.
-    alias :native_delete_by_query? :es_version_5_x?
+    #
+    # ES5 doesn't accept/ignore ambiguous or unexpected req params any more
+    alias :strict_request_params? :es_version_5_x?
 
     # COMPATIBILITY
-    # Return a Hash of indexing request parameters that are valid for this
-    # version of Elasticsearch.
-    def indexing_directives
-      return @indexing_directives if defined?(@indexing_directives)
-
-      @indexing_directives = indexing_parameter_names.each_with_object({}) do |key, h|
-        h[key] = "_#{key}"
-      end
-      @indexing_directives.freeze
-    end
+    # ES 5.X supports `delete_by_query` natively again.
+    alias :native_delete_by_query? :es_version_5_x?
 
     # COMPATIBILITY
-    # Return a Hash of indexing request parameters that are known to
-    # elastomer-client, but not supported by the current version of
-    # Elasticsearch.
-    def unsupported_indexing_directives
-      return @unsupported_indexing_directives if defined?(@unsupported_indexing_directives)
-
-      unsupported_keys = KNOWN_INDEXING_PARAMETER_NAMES - indexing_parameter_names
-
-      @unsupported_indexing_directives = unsupported_keys.each_with_object({}) do |key, h|
-        h[key] = "_#{key}"
+    # Internal: VersionSupport maintains dynamically-created lists of acceptable and unacceptable
+    # request params by ES version. This just shims that list since those params have leading
+    # underscores by default. If we end up with >1 such param, let's make a real thing to handle this.
+    def fix_op_type!(params = {})
+      if es_version_5_x? && params.key?(:op_type)
+        params[:op_type] = "op_type"
       end
-      @unsupported_indexing_directives.freeze
     end
 
     # COMPATIBILITY
@@ -177,12 +173,5 @@ module Elastomer
       end
     end
 
-    def indexing_parameter_names
-      if es_version_2_x?
-        COMMON_INDEXING_PARAMETER_NAMES + ES_2_X_INDEXING_PARAMETER_NAMES
-      else
-        COMMON_INDEXING_PARAMETER_NAMES + ES_5_X_INDEXING_PARAMETER_NAMES
-      end
-    end
   end
 end

data/script/generate-rest-api-spec

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+# Usage:
+#
+#   script/generate-rest-api-spec <elasticsearch-version>
+#
+# Use this script to generate a REST API spec for the given
+# `elasticserach-version`. This will create a new `ApiSpec` class configured
+# to validate the request parameters for the particular Elasticsearch version.
+
+require "erb"
+require "rubygems"
+require "bundler/setup"
+
+$LOAD_PATH.unshift "lib"
+require "elastomer/client"
+
+class RestApiSpecGenerator
+  WORKING_DIR = "vendor/elasticsearch"
+
+  attr_reader :version, :short_version, :class_version
+
+  def initialize(version = "5.6")
+    @version = version
+
+    sliced = @version.split(".").slice(0,2)
+    @short_version = sliced.join(".")
+    @class_version = sliced.join("_")
+  end
+
+  # Setup the working directory and generate the Ruby API spec for the
+  # elasticsearch version.
+  def run
+    setup
+    File.open(ruby_spec_filename, "w") do |fd|
+      fd.puts ERB.new(DATA.read, 0, "-").result(binding)
+    end
+  ensure
+    reset
+  end
+
+  # The name of the Ruby API spec file for this particular Elasticsearch version.
+  def ruby_spec_filename
+    "lib/elastomer/client/rest_api_spec/api_spec_v#{class_version}.rb"
+  end
+
+  # Returns true if the elasticserach working directory exists.
+  def working_dir_exists?
+    File.directory?(WORKING_DIR) && File.exists?(WORKING_DIR)
+  end
+
+  # Iterate over each of the REST API specs yield the name and the descriptor
+  # hash for that particular API spec.
+  def each_api
+    Dir.glob("#{WORKING_DIR}/rest-api-spec/src/main/resources/rest-api-spec/api/*.json").each do |filename|
+      next if filename =~ /\/_common\.json\Z/
+
+      hash = MultiJson.load(File.read(filename))
+      key = hash.keys.first
+      value = hash.values.first
+      yield(key, value)
+    end
+  end
+
+  # Iterate over each of the common request parameters and yield them as key /
+  # value pairs.
+  def each_common
+    filename = "#{WORKING_DIR}/rest-api-spec/src/main/resources/rest-api-spec/api/_common.json"
+    if File.exists? filename
+      hash = MultiJson.load(File.read(filename))
+      hash["params"].each {|k,v| yield(k,v)}
+    end
+  end
+
+  # Perform a sparse checkout of the elasticsearch git repository and then check
+  # out the branch corresponding to the ES version passed to this script.
+  def setup
+    if !working_dir_exists?
+      system <<-SH
+        mkdir -p #{WORKING_DIR} &&
+        cd #{WORKING_DIR} &&
+        git init . &&
+        git remote add -f origin https://github.com/elastic/elasticsearch.git &&
+        git config core.sparsecheckout true &&
+        echo /rest-api-spec/src/main/resources/rest-api-spec/api/ >> .git/info/sparse-checkout &&
+        git pull origin master
+      SH
+    end
+
+    system <<-SH
+      cd #{WORKING_DIR} &&
+      git pull origin master &&
+      git checkout -q origin/#{short_version}
+    SH
+  end
+
+  # Reset the elasticsearch working directory back to the master branch of the
+  # git repository.
+  def reset
+    system <<-SH
+      cd #{WORKING_DIR} &&
+      git checkout master
+    SH
+  end
+end
+
+puts RestApiSpecGenerator.new(*ARGV).run
+
+__END__
+# Generated REST API spec file - DO NOT EDIT!
+# Date: <%= Time.now.strftime("%Y-%m-%d") %>
+# ES version: <%= version %>
+
+module Elastomer::Client::RestApiSpec
+  class ApiSpecV<%= class_version %> < ApiSpec
+    def initialize
+      @rest_apis = {
+<% each_api do |name,data| -%>
+        "<%= name %>" => RestApi.new(
+          documentation: "<%= data["documentation"].to_s %>",
+          methods: <%= Array(data["methods"]).to_s %>,
+          body: <%= data["body"] ? data["body"].to_s : "nil" %>,
+<% url = data["url"] -%>
+          url: {
+            path: "<%= url["path"] %>",
+            paths: <%= Array(url["paths"]).to_s %>,
+<% if (parts = url["parts"]) && !parts.empty? -%>
+            parts: {
+<% parts.each do |k,v| -%>
+              "<%= k %>" => <%= v.to_s %>,
+<% end -%>
+            },
+<% end -%>
+<% if (params = url["params"]) && !params.empty? -%>
+            params: {
+<% params.each do |k,v| -%>
+              "<%= k %>" => <%= v.to_s %>,
+<% end -%>
+            }
+<% end -%>
+          }
+        ),
+<% end -%>
+      }
+      @common_params = {
+<% each_common do |k,v| -%>
+        "<%= k %>" => <%= v.to_s %>,
+<% end -%>
+      }
+      super
+    end
+  end
+end

data/test/client/rest_api_spec/api_spec_test.rb

@@ -0,0 +1,55 @@
+require_relative "../../test_helper"
+
+describe Elastomer::Client::RestApiSpec::ApiSpec do
+  before do
+    @api_spec = Elastomer::Client::RestApiSpec.api_spec("5.6.4")
+  end
+
+  it "selects valid path parts" do
+    parts  = {index: "test", "type" => "doc", foo: "bar"}
+    result = @api_spec.select_parts(api: "search", from: parts)
+
+    assert_equal({index: "test", "type" => "doc"}, result)
+  end
+
+  it "identifies valid path parts" do
+    assert @api_spec.valid_part?(api: "search", part: "index")
+    assert @api_spec.valid_part?(api: "search", part: :type)
+    refute @api_spec.valid_part?(api: "search", part: :id)
+  end
+
+  it "selects valid request params" do
+    params = {explain: true, "preference" => "local", nope: "invalid"}
+    result = @api_spec.select_params(api: "search", from: params)
+
+    assert_equal({explain: true, "preference" => "local"}, result)
+  end
+
+  it "identifies valid request params" do
+    assert @api_spec.valid_param?(api: "search", param: "explain")
+    assert @api_spec.valid_param?(api: "search", param: :preference)
+    assert @api_spec.valid_param?(api: "search", param: :routing)
+    refute @api_spec.valid_param?(api: "search", param: "pretty")
+  end
+
+  it "selects common request params" do
+    params = {pretty: true, "human" => true, nope: "invalid"}
+    result = @api_spec.select_common_params(from: params)
+
+    assert_equal({pretty: true, "human" => true}, result)
+  end
+
+  it "identifies common request params" do
+    assert @api_spec.valid_common_param?("pretty")
+    assert @api_spec.valid_common_param?(:human)
+    assert @api_spec.valid_common_param?(:source)
+    refute @api_spec.valid_common_param?("nope")
+  end
+
+  it "validates request params" do
+    assert_raises(Elastomer::Client::IllegalArgument, "'nope' is not a valid parameter for the 'search' API") {
+      params = {q: "*:*", pretty: true, "nope": false}
+      @api_spec.validate_params!(api: "search", params: params)
+    }
+  end
+end

data/test/client/rest_api_spec/rest_api_test.rb

@@ -0,0 +1,96 @@
+require_relative "../../test_helper"
+
+describe Elastomer::Client::RestApiSpec::RestApi do
+  before do
+    @rest_api = Elastomer::Client::RestApiSpec::RestApi.new \
+        documentation: "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-state.html",
+        methods: ["GET"],
+        body: nil,
+        url: {
+          path: "/_cluster/state",
+          paths: ["/_cluster/state", "/_cluster/state/{metric}", "/_cluster/state/{metric}/{index}"],
+          parts: {
+            "index" => {"type"=>"list", "description"=>"A comma-separated list of index names; use `_all` or empty string to perform the operation on all indices"},
+            "metric" => {"type"=>"list", "options"=>["_all", "blocks", "metadata", "nodes", "routing_table", "routing_nodes", "master_node", "version"], "description"=>"Limit the information returned to the specified metrics"},
+          },
+          params: {
+            "local" => {"type"=>"boolean", "description"=>"Return local information, do not retrieve the state from master node (default: false)"},
+            "master_timeout" => {"type"=>"time", "description"=>"Specify timeout for connection to master"},
+            "flat_settings" => {"type"=>"boolean", "description"=>"Return settings in flat format (default: false)"},
+            "ignore_unavailable" => {"type"=>"boolean", "description"=>"Whether specified concrete indices should be ignored when unavailable (missing or closed)"},
+            "allow_no_indices" => {"type"=>"boolean", "description"=>"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"},
+            "expand_wildcards" => {"type"=>"enum", "options"=>["open", "closed", "none", "all"], "default"=>"open", "description"=>"Whether to expand wildcard expression to concrete indices that are open, closed or both."},
+          }
+        }
+  end
+
+  it "selects valid path parts" do
+    hash = {
+      index: "test",
+      "metric" => "os",
+      nope: "not selected"
+    }
+    selected = @rest_api.select_parts(from: hash)
+
+    refute selected.key?(:nope)
+    assert selected.key?(:index)
+    assert selected.key?("metric")
+  end
+
+  it "identifies valid parts" do
+    assert @rest_api.valid_part? :index
+    assert @rest_api.valid_part? "metric"
+    refute @rest_api.valid_part? :nope
+  end
+
+  it "selects valid request params" do
+    hash = {
+      local: true,
+      "flat_settings" => true,
+      expand_wildcards: "all",
+      nope: "not selected"
+    }
+    selected = @rest_api.select_params(from: hash)
+
+    refute selected.key?(:nope)
+    assert selected.key?(:local)
+    assert selected.key?("flat_settings")
+    assert selected.key?(:expand_wildcards)
+  end
+
+  it "identifies valid params" do
+    assert @rest_api.valid_param? :local
+    assert @rest_api.valid_param? "flat_settings"
+    refute @rest_api.valid_param? :nope
+  end
+
+  it "accesses the documentation url" do
+    assert_equal "https://www.elastic.co/guide/en/elasticsearch/reference/5.x/cluster-state.html", @rest_api.documentation
+  end
+
+  it "exposes the HTTP methods as an Array" do
+    assert_equal %w[GET], @rest_api.methods
+  end
+
+  it "accesses the body settings" do
+    assert_nil @rest_api.body
+  end
+
+  describe "accessing the url" do
+    it "accesses the path" do
+      assert_equal "/_cluster/state", @rest_api.url.path
+    end
+
+    it "exposes the paths as an Array" do
+      assert_equal %w[/_cluster/state /_cluster/state/{metric} /_cluster/state/{metric}/{index}], @rest_api.url.paths
+    end
+
+    it "accesses the path parts" do
+      assert_equal %w[index metric], @rest_api.url.parts.keys
+    end
+
+    it "accesses the request params" do
+      assert_equal %w[local master_timeout flat_settings ignore_unavailable allow_no_indices expand_wildcards], @rest_api.url.params.keys
+    end
+  end
+end

data/test/client/stubbed_client_test.rb

@@ -4,6 +4,7 @@ describe "stubbed client tests" do
   before do
     @stubs  = Faraday::Adapter.lookup_middleware(:test)::Stubs.new
     @client = Elastomer::Client.new :adapter => [:test, @stubs]
+    @client.instance_variable_set(:@version, "5.6.4")
   end
 
   describe Elastomer::Client::Cluster do
@@ -17,24 +18,5 @@ describe "stubbed client tests" do
       h = @client.cluster.reroute commands, :dry_run => true
       assert_acknowledged h
     end
-
-    it "performs a shutdown of the cluster" do
-      @stubs.post("/_shutdown") { [200, {"Content-Type" => "application/json"}, '{"cluster_name":"elasticsearch"}'] }
-      h = @client.cluster.shutdown
-      assert_equal "elasticsearch", h["cluster_name"]
-    end
-  end
-
-  describe Elastomer::Client::Nodes do
-    it "performs a shutdown of the node(s)" do
-      @stubs.post("/_cluster/nodes/_all/_shutdown")  { [200, {"Content-Type" => "application/json"}, '{"nodes":{"1":{"name":"Node1"}}}'] }
-      @stubs.post("/_cluster/nodes/node2/_shutdown") { [200, {"Content-Type" => "application/json"}, '{"nodes":{"2":{"name":"Node2"}}}'] }
-
-      h = @client.nodes("_all").shutdown
-      assert_equal "Node1", h["nodes"]["1"]["name"]
-
-      h = @client.nodes("node2").shutdown
-      assert_equal "Node2", h["nodes"]["2"]["name"]
-    end
   end
 end

data/test/middleware/opaque_id_test.rb

@@ -15,9 +15,6 @@ describe Elastomer::Middleware::OpaqueId do
         ]
       }
 
-      stub.get("/") { |env|
-        [ 200, {}, {"version" => {"number" => "1.0.0"}}  ]
-      }
       stub.get("/_cluster/state") { |env|
         [ 200, {"X-Opaque-Id" => "00000000-0000-0000-0000-000000000000"}, %q[{"foo":"bar"}] ]
       }
@@ -28,6 +25,7 @@ describe Elastomer::Middleware::OpaqueId do
         :adapter   => [:test, stubs]
 
     @client = Elastomer::Client.new opts
+    @client.instance_variable_set(:@version, "5.6.4")
   end
 
   it 'generates an "X-Opaque-Id" header' do

data/test/notifications_test.rb

@@ -39,11 +39,6 @@ describe Elastomer::Notifications do
     nodes.hot_threads; assert_action_event("nodes.hot_threads")
   end
 
-  it "instruments node shutdown" do
-    client = stub_client(:post, "/_cluster/nodes/_shutdown")
-    client.nodes.shutdown; assert_action_event("nodes.shutdown")
-  end
-
   it "instruments index actions" do
     @index.exists?; assert_action_event("index.exists")
     @index.create(default_index_settings)

data/test/test_helper.rb

@@ -30,10 +30,11 @@ require "elastomer/client"
 # we are going to use the same client instance everywhere!
 # the client should always be stateless
 $client_params = {
-  :port => ENV.fetch("ES_PORT", 9200),
-  :read_timeout => 10,
-  :open_timeout => 1,
-  :opaque_id => false
+  port: ENV.fetch("ES_PORT", 9200),
+  read_timeout: 10,
+  open_timeout: 1,
+  opaque_id: false,
+  strict_params: true
 }
 $client = Elastomer::Client.new $client_params
 

data/test/version_support_test.rb

@@ -61,18 +61,6 @@ describe Elastomer::VersionSupport do
         refute version_support.native_delete_by_query?, "ES 2.X does not have native delete_by_query support"
       end
     end
-
-    describe "#indexing_directives" do
-      it "returns a Hash of indexing parameter name to field name" do
-        assert_includes(version_support.indexing_directives.to_a, [:consistency, "_consistency"])
-      end
-    end
-
-    describe "#unsupported_indexing_directives" do
-      it "returns a Hash of indexing parameter name to field name" do
-        assert_includes(version_support.unsupported_indexing_directives.to_a, [:wait_for_active_shards, "_wait_for_active_shards"])
-      end
-    end
   end
 
   describe "ES 5.x" do
@@ -104,15 +92,9 @@ describe Elastomer::VersionSupport do
       end
     end
 
-    describe "#indexing_directives" do
-      it "returns a Hash of indexing parameter name to field name" do
-        assert_includes(version_support.indexing_directives.to_a, [:wait_for_active_shards, "_wait_for_active_shards"])
-      end
-    end
-
-    describe "#unsupported_indexing_directives" do
-      it "returns an Hash of indexing parameter names to field name" do
-        assert_includes(version_support.unsupported_indexing_directives.to_a, [:consistency, "_consistency"])
+    describe "#op_type_param" do
+      it "converts the supplied params key _op_type to op_type, if present" do
+        assert_equal(version_support.op_type(_op_type: "create"), {op_type: "create"})
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastomer-client
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.0
 platform: ruby
 authors:
 - Tim Pease
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-20 00:00:00.000000000 Z
+date: 2018-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -249,6 +249,12 @@ files:
 - lib/elastomer/client/nodes.rb
 - lib/elastomer/client/percolator.rb
 - lib/elastomer/client/repository.rb
+- lib/elastomer/client/rest_api_spec.rb
+- lib/elastomer/client/rest_api_spec/api_spec.rb
+- lib/elastomer/client/rest_api_spec/api_spec_v2_3.rb
+- lib/elastomer/client/rest_api_spec/api_spec_v2_4.rb
+- lib/elastomer/client/rest_api_spec/api_spec_v5_6.rb
+- lib/elastomer/client/rest_api_spec/rest_api.rb
 - lib/elastomer/client/scroller.rb
 - lib/elastomer/client/snapshot.rb
 - lib/elastomer/client/tasks.rb
@@ -268,6 +274,7 @@ files:
 - script/cibuild-elastomer-client-es24
 - script/cibuild-elastomer-client-es56
 - script/console
+- script/generate-rest-api-spec
 - script/poll-for-es
 - test/assertions.rb
 - test/client/app_delete_by_query_test.rb
@@ -283,6 +290,8 @@ files:
 - test/client/nodes_test.rb
 - test/client/percolator_test.rb
 - test/client/repository_test.rb
+- test/client/rest_api_spec/api_spec_test.rb
+- test/client/rest_api_spec/rest_api_test.rb
 - test/client/scroller_test.rb
 - test/client/snapshot_test.rb
 - test/client/stubbed_client_test.rb
@@ -336,6 +345,8 @@ test_files:
 - test/client/nodes_test.rb
 - test/client/percolator_test.rb
 - test/client/repository_test.rb
+- test/client/rest_api_spec/api_spec_test.rb
+- test/client/rest_api_spec/rest_api_test.rb
 - test/client/scroller_test.rb
 - test/client/snapshot_test.rb
 - test/client/stubbed_client_test.rb