elastomer-client 2.3.0 → 3.0.0

data/test/client/nodes_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Nodes do
 

data/test/client/percolator_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Percolator do
 
@@ -14,7 +14,15 @@ describe Elastomer::Client::Percolator do
 
   describe "when an index exists" do
     before do
-      @index.create(nil)
+      # COMPATIBILITY
+      base_mappings =
+        if requires_percolator_mapping?
+          { :mappings => { :percolator => { :properties => { :query => { :type => "percolator" } } } } }
+        else
+          nil
+        end
+
+      @index.create(base_mappings)
       wait_for_index(@index.name)
     end
 

data/test/client/repository_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Repository do
   before do

data/test/client/scroller_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Scroller do
 
@@ -13,16 +13,16 @@ describe Elastomer::Client::Scroller do
           :tweet => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :message => { :type => "string", :analyzer => "standard" },
-              :author  => { :type => "string", :index => "not_analyzed" },
+              :message => $client.version_support.text(analyzer: "standard"),
+              :author  => $client.version_support.keyword,
               :sorter  => { :type => "integer" }
             }
           },
           :book => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :title  => { :type => "string", :analyzer => "standard" },
-              :author => { :type => "string", :index => "not_analyzed" },
+              :title  => $client.version_support.text(analyzer: "standard"),
+              :author => $client.version_support.keyword,
               :sorter  => { :type => "integer" }
             }
           }
@@ -104,7 +104,17 @@ describe Elastomer::Client::Scroller do
     refute_nil h["_scroll_id"], "response is missing a scroll ID"
 
     response = $client.clear_scroll(h["_scroll_id"])
-    assert_empty response
+
+    if returns_cleared_scroll_id_info?
+      assert response["succeeded"]
+      assert_equal 1, response["num_freed"]
+    else
+      assert_empty response
+    end
+  end
+
+  it "raises an exception on existing sort in query" do
+    assert_raises(ArgumentError) { @index.scan :sort => [:_doc] , :query => {} }
   end
 
   def populate!

data/test/client/snapshot_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Snapshot do
   before do

data/test/client/stubbed_client_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe "stubbed client tests" do
   before do

data/test/client/tasks_test.rb

@@ -0,0 +1,139 @@
+require_relative "../test_helper"
+
+describe Elastomer::Client::Tasks do
+  before do
+    @tasks = $client.tasks
+
+    @index = $client.index("elastomer-tasks-test")
+    @index.create(default_index_settings)
+    wait_for_index(@index.name)
+  end
+
+  after do
+    @index.delete if @index.exists?
+  end
+
+  it "list all in-flight tasks" do
+    h = @tasks.get
+    assert h["nodes"].keys.size > 0
+
+    total_tasks = h["nodes"].map { |k, v| v["tasks"].keys.count }.sum
+    assert total_tasks > 0
+  end
+
+  it "groups by parent->child relationships when get-all tasks API is grouped by 'parents'" do
+    unless $client.version_support.es_version_5_x?
+      skip "Tasks API is not supported in ES version #{$client.version}"
+    end
+
+    h = @tasks.get :group_by => "parents"
+    parent_id = h["tasks"].select { |k, v| v.key?("children") }.keys.first
+    childs_parent_ref = h.dig("tasks", parent_id, "children").first["parent_task_id"]
+    assert_equal parent_id, childs_parent_ref
+  end
+
+  it "raises exception when get_by_id is called without required task & node IDs" do
+    assert_raises(ArgumentError) do
+      @tasks.get_by_id
+    end
+  end
+
+  it "raises exception when get_by_id is called w/invalid task ID is supplied" do
+    node_id = @tasks.get["nodes"].map { |k, v| k }.first
+    assert_raises(ArgumentError) do
+      @tasks.get_by_id node_id, "task_id_should_be_integer"
+    end
+  end
+
+  it "raises exception when get_by_id is called w/invalid node ID is supplied" do
+    assert_raises(ArgumentError) do
+        @tasks.get_by_id nil, 42
+    end
+  end
+
+  it "successfully waits for task to complete when wait_for_completion and timeout flags are set" do
+    test_thread = nil
+    begin
+      # poulate the index in a background thread to generate long-running tasks we can query
+      test_thread = populate_background_index!(@index.name)
+
+      # ensure we can wait on completion of a task
+      success = false
+      query_long_running_tasks.each do |ts|
+        t = ts.values.first
+        begin
+          resp = @tasks.wait_by_id t["node"], t["id"], "3s"
+          success = !resp.key?("node_failures")
+        rescue Elastomer::Client::ServerError => e
+          # this means the timeout expired before the task finished, but it's a good thing!
+          success = /Timed out waiting for completion/ =~ e.message
+        end
+        break if success
+      end
+
+      assert success
+    ensure
+      test_thread.join unless test_thread.nil?
+    end
+  end
+
+  it "locates the task properly by ID when valid node and task IDs are supplied" do
+    test_thread = nil
+    begin
+      # make an index with a new client (in this thread, to avoid query check race after)
+      # poulate the index in a background thread to generate long-running tasks we can query
+      test_thread = populate_background_index!(@index.name)
+
+      # look up and verify found task
+      found_by_id = false
+      query_long_running_tasks.each do |ts|
+        t = ts.values.first
+        resp = @tasks.get_by_id t["node"], t["id"]
+
+        # ES 5.x and 2.x responses are structured differently
+        if $client.version_support.tasks_new_response_format?
+          found_by_id = resp["task"]["node"] == t["node"] && resp["task"]["id"] == t["id"]
+        else
+          nid, tid = resp["nodes"][t["node"]]["tasks"].keys.first.split(":")
+          found_by_id = nid == t["node"] && tid.to_i == t["id"]
+        end
+
+        break if found_by_id
+      end
+
+      assert found_by_id
+    ensure
+      test_thread.join unless test_thread.nil?
+    end
+  end
+
+  it "raises exception when cancel_by_id is called without required task & node IDs" do
+    assert_raises(ArgumentError) do
+      @tasks.cancel_by_id
+    end
+  end
+
+  it "raises exception when cancel_by_id is called w/invalid task ID is supplied" do
+    node_id = @tasks.get["nodes"].map { |k, v| k }.first
+    assert_raises(ArgumentError) do
+      @tasks.cancel_by_id node_id, "not_an_integer_id"
+    end
+  end
+
+  it "raises exception when cancel_by_id is called w/invalid node IDs is supplied" do
+    assert_raises(ArgumentError) do
+      @tasks.cancel_by_id nil, 42
+    end
+  end
+
+  # TODO: test this behavior MORE!
+  it "raises exception when cancel_by_id is called w/invalid node and task IDs are supplied" do
+    assert_raises(ArgumentError) do
+      @tasks.cancel_by_id "", "also_should_be_integer_id"
+    end
+  end
+
+  # NOTE: unlike get_by_id, cancellation API doesn't return 404 when valid node_id and task_id
+  # params don't match known nodes/running tasks, so there's no matching test for that here.
+
+end

data/test/client/template_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Cluster do
 

data/test/client/warmer_test.rb

@@ -1,7 +1,11 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Warmer do
   before do
+    unless $client.version_support.supports_warmers?
+      skip "warmers are not supported in ES #{$client.version}"
+    end
+
     @name  = "elastomer-warmer-test"
     @index = $client.index(@name)
 
@@ -12,8 +16,8 @@ describe Elastomer::Client::Warmer do
           :tweet => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :message => { :type => "string", :analyzer => "standard" },
-              :author  => { :type => "string", :index => "not_analyzed" }
+              :message => $client.version_support.text(analyzer: "standard"),
+              :author  => $client.version_support.keyword
             }
           }
         }
@@ -23,7 +27,7 @@ describe Elastomer::Client::Warmer do
   end
 
   after do
-    @index.delete if @index.exists?
+    @index.delete if defined?(@index) && @index.exists?
   end
 
   it "creates warmers" do

data/test/client_test.rb

@@ -1,4 +1,5 @@
 require File.expand_path("../test_helper", __FILE__)
+require "elastomer/notifications"
 
 describe Elastomer::Client do
 
@@ -32,6 +33,14 @@ describe Elastomer::Client do
     end
   end
 
+  it "wraps Faraday errors with our own exceptions" do
+    error = Faraday::TimeoutError.new("it took too long")
+    wrapped = $client.wrap_faraday_error(error, :get, "/_cat/indices")
+
+    assert_instance_of Elastomer::Client::TimeoutError, wrapped
+    assert_equal "it took too long :: GET /_cat/indices", wrapped.message
+  end
+
   it "handles path expansions" do
     uri = $client.expand_path "/{foo}/{bar}", :foo => "_cluster", :bar => "health"
     assert_equal "/_cluster/health", uri
@@ -167,4 +176,94 @@ describe Elastomer::Client do
       assert_equal Semantic::Version.new(version_string), $client.semantic_version
     end
   end
+
+  describe "duplicating a client connection" do
+    it "is configured the same" do
+      client = $client.dup
+
+      refute_same $client, client
+
+      assert_equal $client.host, client.host
+      assert_equal $client.port, client.port
+      assert_equal $client.url, client.url
+      assert_equal $client.read_timeout, client.read_timeout
+      assert_equal $client.open_timeout, client.open_timeout
+      assert_equal $client.max_request_size, client.max_request_size
+    end
+
+    it "has a unique connection" do
+      client = $client.dup
+
+      refute_same $client.connection, client.connection
+    end
+  end
+
+  describe "automatic retry of requests" do
+    before do
+      @events = []
+      @subscriber = ActiveSupport::Notifications.subscribe do |*args|
+        @events << ActiveSupport::Notifications::Event.new(*args)
+      end
+    end
+
+    after do
+      @events.clear
+      ActiveSupport::Notifications.unsubscribe(@subscriber)
+    end
+
+    it "defaults to no retries" do
+      stub_request(:get, $client.url+"/_cat/indices").
+        to_timeout.then.
+        to_return({
+          headers: {"Content-Type" => "text/plain; charset=UTF-8"},
+          body: "green open test-index 1 0 0 0 159b 159b"
+        })
+
+      assert_raises(Elastomer::Client::ConnectionFailed) {
+        $client.get("/_cat/indices")
+      }
+    end
+
+    it "retries up to `max_retries` times" do
+      stub_request(:get, $client.url+"/test-index/_settings").
+        to_timeout.then.
+        to_timeout.then.
+        to_return({body: %q/{"acknowledged": true}/})
+
+      response = $client.index("test-index").settings(max_retries: 2)
+
+      assert_equal 2, @events.first.payload[:retries]
+      assert_equal({"acknowledged" => true}, response)
+    end
+
+    it "does not retry on PUT requests" do
+      stub_request(:put, $client.url+"/test-index").
+        to_timeout.then.
+        to_return({body: %q/{"acknowledged": true}/})
+
+      assert_raises(Elastomer::Client::ConnectionFailed) {
+        $client.index("test-index").create({}, max_retries: 1)
+      }
+    end
+
+    it "does not retry on POST requests" do
+      stub_request(:post, $client.url+"/test-index/_flush").
+        to_timeout.then.
+        to_return({body: %q/{"acknowledged": true}/})
+
+      assert_raises(Elastomer::Client::ConnectionFailed) {
+        $client.index("test-index").flush(max_retries: 1)
+      }
+    end
+
+    it "does not retry on DELETE requests" do
+      stub_request(:delete, $client.url+"/test-index").
+        to_timeout.then.
+        to_return({body: %q/{"acknowledged": true}/})
+
+      assert_raises(Elastomer::Client::ConnectionFailed) {
+        $client.index("test-index").delete(max_retries: 1)
+      }
+    end
+  end
 end

data/test/core_ext/time_test.rb

@@ -13,7 +13,7 @@ describe "JSON conversions for Time" do
           :doc1 => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :title      => { :type => "string", :index => "not_analyzed" },
+              :title      => $client.version_support.keyword,
               :created_at => { :type => "date" }
             }
           }

data/test/notifications_test.rb

@@ -48,9 +48,13 @@ describe Elastomer::Notifications do
     @index.exists?; assert_action_event("index.exists")
     @index.create(default_index_settings)
     assert_action_event("index.create")
+    wait_for_index(@index.name)
+
     @index.get_settings; assert_action_event("index.get_settings")
     @index.update_settings(number_of_replicas: 0)
     assert_action_event("index.get_settings")
+    wait_for_index(@index.name)
+
     @index.close; assert_action_event("index.close")
     @index.open; assert_action_event("index.open")
     @index.delete; assert_action_event("index.delete")

data/test/test_helper.rb

@@ -17,6 +17,11 @@ end
 
 require "minitest/spec"
 require "minitest/autorun"
+require "minitest/focus"
+
+# used in a couple test files, makes them available for all
+require "active_support/core_ext/enumerable"
+require 'active_support/core_ext/hash'
 
 # push the lib folder onto the load path
 $LOAD_PATH.unshift "lib"
@@ -38,7 +43,7 @@ raise "No server available at #{$client.url}" unless $client.available?
 puts "Elasticsearch version is #{$client.version}"
 
 # remove any lingering test indices from the cluster
-MiniTest::Unit.after_tests do
+MiniTest.after_run do
   $client.cluster.indices.keys.each do |name|
     next unless name =~ /^elastomer-/i
     $client.index(name).delete
@@ -84,26 +89,6 @@ def wait_for_index(name, status="yellow")
   )
 end
 
-# Elasticsearch 2.0 changed some request formats in a non-backward-compatible
-# way. Some tests need to know what version is running to structure requests
-# as expected.
-#
-# Returns true if Elasticsearch version is 2.x.
-def es_version_2_x?
-  $client.semantic_version >= "2.0.0" &&
-  $client.semantic_version <  "3.0.0"
-end
-
-# Elasticsearch 5.0 changed some request formats in a non-backward-compatible
-# way. Some tests need to know what version is running to structure requests
-# as expected.
-#
-# Returns true if Elasticsearch version is 5.x.
-def es_version_5_x?
-  $client.semantic_version >= "5.0.0" &&
-  $client.semantic_version <  "6.0.0"
-end
-
 def default_index_settings
   {settings: {index: {number_of_shards: 1, number_of_replicas: 0}}}
 end
@@ -165,3 +150,126 @@ def with_tmp_snapshot(name = SecureRandom.uuid, &block)
     yield repo.snapshot(name), repo
   end
 end
+
+# Just some busy work in the background for tasks API to detect in test cases
+#
+# Returns the thread and index references so caller can join the thread and delete
+# the index after the checks are performed
+def populate_background_index!(name)
+  # make an index with a new client (in this thread, to avoid query check race after)
+  name.freeze
+  index = $client.dup.index(name)
+  docs = index.docs("widget")
+
+  # do some busy work in background thread to generate bulk-indexing tasks we
+  # can query at the caller. return the thread ref so caller can join on it
+  Thread.new do
+    100.times.each do |i|
+      docs.bulk do |d|
+        (1..500).each do |j|
+          d.index \
+            :foo => "foo_#{i}_#{j}",
+            :bar => "bar_#{i}_#{j}",
+            :baz => "baz_#{i}_#{j}"
+        end
+      end
+      index.refresh
+    end
+  end
+end
+
+# when populate_background_index! is running, this query returns healthcheck tasks
+# that are long-running enough to be queried again for verification in test cases
+def query_long_running_tasks
+  Kernel.sleep(0.01)
+  target_tasks = []
+  100.times.each do
+    target_tasks = @tasks.get["nodes"]
+      .map { |k, v| v["tasks"] }
+      .flatten.map { |ts| ts.select { |k, v| /bulk/ =~ v["action"] } }
+      .flatten.reject { |t| t.empty? }
+    break if target_tasks.size > 0
+  end
+
+  target_tasks
+end
+
+# The methods below are to support intention-revealing names about version
+# differences in the tests. If necessary for general operation they can be moved
+# into Elastomer::VersionSupport.
+
+# COMPATIBILITY
+# ES 5.x returns `index` bulk request as `index` responses whether or not the
+# document was created or updated. ES 2.x returns a `create` response if it was
+# created.
+def bulk_index_returns_create_for_new_documents?
+  $client.version_support.es_version_2_x?
+end
+
+# COMPATIBILITY
+# ES 5.x drops support for index-time payloads
+def index_time_payloads?
+  $client.version_support.es_version_2_x?
+end
+
+# COMPATIBILITY
+# ES 2.x returns an empty result when an alias does not exist for a full or partial match
+# ES 5.6 returns an error when an alias does not exist for a full or partial match
+def fetching_non_existent_alias_returns_error?
+  $client.version_support.es_version_5_x?
+end
+
+# COMPATIBILITY
+# ES 5.6 includes a _nodes key in the /_cluster/stats response. Strangely
+# enough, this is not documented in the example response:
+# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/cluster-stats.html
+def cluster_stats_includes_underscore_nodes?
+  $client.version_support.es_version_5_x?
+end
+
+# COMPATIBILITY
+# ES 2.0 deprecated the `filtered` query type. ES 5.0 removed it entirely.
+def filtered_query_removed?
+  $client.version_support.es_version_5_x?
+end
+
+# ES 5.6 percolator queries/document submissions require that an appropriate
+# percolator type and field within that type are defined on the index mappings
+def requires_percolator_mapping?
+  $client.version_support.es_version_5_x?
+end
+
+# COMPATIBILITY
+# ES 5 removes the `output` option for fields.
+# See: https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_suggester.html#_simpler_completion_indexing
+def supports_suggest_output?
+  $client.version_support.es_version_2_x?
+end
+
+# COMPATIBILITY
+# ES 5 returns information about the number of cleared scroll IDs
+def returns_cleared_scroll_id_info?
+  $client.version_support.es_version_5_x?
+end
+
+# COMPATIBILITY
+# Return a Hash with an unsupported indexing directive key/value to test fail-fast.
+def incompatible_indexing_directive
+  if $client.version_support.es_version_2_x?
+    {_wait_for_active_shards: 10}
+  else
+    {_consistency: "all"}
+  end
+end
+
+# COMPATIBILITY
+# Returns true if the Elasticsearch cluster will validate request parameters.
+def parameter_validation?
+  $client.version_support.es_version_5_x?
+end
+
+# ES 5 supports native _delete_by_query, but the output and semantics are
+# different than the plugin which we modeled our delete by query on.
+def supports_native_delete_by_query?
+  $client.version_support.native_delete_by_query?
+end