elastomer-client 2.3.0 → 3.0.0

data/script/cibuild

@@ -1,30 +1,101 @@
-#!/bin/sh
+#!/bin/bash
 # Usage: script/cibuild
-# CI build script.
+# CI build script
 # This is tailored for the janky build machines.
-set -e
+
+set -ue
+
+# Only echo the tags when in CI_MODE
+begin_fold() {
+  local tag="$1"
+  if [ "${CI_MODE:-}" ]; then
+    echo "%%%FOLD {${tag}}%%%"
+  fi
+}
+
+# Only echo the tags when in CI_MODE
+end_fold() {
+  if [ "${CI_MODE:-}" ]; then
+    echo "%%%END FOLD%%%"
+  fi
+}
+
+function cleanup() {
+  echo
+  begin_fold "Shutting down services..."
+  $docker_compose down -v
+  end_fold
+}
+
+# Borrowed from script/bintools in github/ci
+output_fold() {
+  # Exit early if no label provided
+  if [ -z "$1" ]; then
+    echo "output_fold(): requires a label argument."
+    return
+  fi
+
+  exit_value=0  # exit_value is used to record exit status of the given command
+  label=$1      # human-readable label describing what's being folded up
+  shift 1       # having retrieved the output_fold()-specific arguments, strip them off $@
+
+  begin_fold "$label"
+
+  # run the remaining arguments. If the command exits non-0, the `||` will
+  # prevent the `-e` flag from seeing the failure exit code, and we'll see
+  # the second echo execute
+  "$@" || exit_value=$?
+
+  end_fold
+
+  # preserve the exit code from the subcommand.
+  return $exit_value
+}
+
+trap cleanup EXIT
+
+export ES_PORT=${ES_PORT:-19200}
+
+es_version=${ES_VERSION:-24}
+docker_compose="docker-compose --file docker/docker-compose.es${es_version}.yml"
+
+if [ "${CI_MODE:-}" ]; then
+  docker_compose="${docker_compose} --file docker/docker-compose.cibuild.yml"
+#  docker_compose="${docker_compose} --no-ansi"
+fi
 
 # change into root dir and setup path
 cd $(dirname "$0")/..
 PATH="$(pwd)/bin:$(pwd)/script:/usr/share/rbenv/shims:$PATH"
 
+echo "hostname: $(hostname)"
+
 # Write commit we're building at
-git log -n 1 || true
+output_fold "Commit info..." git log -n 1 || true
+echo
+
+output_fold "Bootstrapping container..." $docker_compose build
+output_fold "Bringing up services..." $docker_compose up -d
+output_fold "Waiting for Elasticsearch..." script/poll-for-es
 echo
 
 result=0
 
+begin_fold "Ruby environment setup..."
 export RBENV_VERSION="2.3.3"
 if [ -d /usr/share/rbenv/shims ]; then
-    export PATH=/usr/share/rbenv/shims:$PATH
+  export PATH=/usr/share/rbenv/shims:$PATH
 fi
 ruby -v
 rm -f Gemfile.lock
 script/bootstrap
+end_fold
+echo
+
 bundle exec rake test || result=$?
 
 if [ $result -ne 0 ]; then
-    exit $result
+  exit $result
 fi
 
 # echo

data/script/cibuild-elastomer-client

@@ -0,0 +1 @@
+script/cibuild-elastomer-client-es24

data/script/cibuild-elastomer-client-es24

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Usage: script/cibuild-es24
+# CI build script targeting Elasticsearch 2.4
+
+export ES_VERSION=24
+export CI_MODE=true
+
+script/cibuild

data/script/cibuild-elastomer-client-es56

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Usage: script/cibuild-es56
+# CI build script targeting Elasticsearch 5.6
+
+export ES_VERSION=56
+export CI_MODE=true
+
+script/cibuild

data/script/poll-for-es

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# This script will poll the Elasticsearch health endpoint until the cluster
+# reaches a yellow state which is good enough for testing. This script will poll
+# for up to 30 seconds waiting for Elasticsearch to start. It will give up at
+# that time and return a non-zero exit code.
+
+es_port=${ES_PORT:-9200}
+count=0
+
+until $(curl -s "localhost:${es_port}/_cluster/health?wait_for_status=yellow&timeout=30s" > /dev/null 2>&1); do
+  sleep 0.50
+  count=$(($count+1))
+  if [ "$count" -gt 60 ]; then
+    echo "Timed out waiting for Elasticsearch at localhost:${es_port}"
+    exit 1
+  fi
+done
+
+echo "Elasticsearch is ready at localhost:${es_port}"

data/test/client/{delete_by_query_test.rb → app_delete_by_query_test.rb}

@@ -1,6 +1,6 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
-describe Elastomer::Client::DeleteByQuery do
+describe Elastomer::Client::AppDeleteByQuery do
 
   before do
     @index = $client.index "elastomer-delete-by-query-test"
@@ -23,7 +23,7 @@ describe Elastomer::Client::DeleteByQuery do
       @docs.index({ :_id => 1, :name => "luna" })
 
       @index.refresh
-      response = @index.delete_by_query(nil, :q => "name:mittens")
+      response = @index.app_delete_by_query(nil, :q => "name:mittens")
       assert_equal({
         "_all" => {
           "found" => 1,
@@ -50,7 +50,7 @@ describe Elastomer::Client::DeleteByQuery do
       @docs.index({ :_id => 1, :name => "luna" })
       @index.refresh
 
-      response = @index.delete_by_query(nil, :action_count => 1)
+      response = @index.app_delete_by_query(nil, :action_count => 1)
 
       assert_requested(:post, /_bulk/, :times => 2)
 
@@ -95,7 +95,7 @@ describe Elastomer::Client::DeleteByQuery do
         end)
 
       @index.refresh
-      response = @index.delete_by_query(nil, :action_count => 1)
+      response = @index.app_delete_by_query(nil, :action_count => 1)
       assert_equal({
         "_all" => {
           "found" => 0,
@@ -131,7 +131,7 @@ describe Elastomer::Client::DeleteByQuery do
         end)
 
       @index.refresh
-      response = @index.delete_by_query(nil, :action_count => 1)
+      response = @index.app_delete_by_query(nil, :action_count => 1)
       assert_equal({
         "_all" => {
           "found" => 1,
@@ -160,7 +160,7 @@ describe Elastomer::Client::DeleteByQuery do
       docs.index({ :_id => 1, :_routing => "cat", :name => "luna" })
 
       index.refresh
-      response = index.delete_by_query(nil, :q => "name:mittens")
+      response = index.app_delete_by_query(nil, :q => "name:mittens")
       assert_equal({
         "_all" => {
           "found" => 1,

data/test/client/bulk_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Bulk do
 
@@ -13,15 +13,15 @@ describe Elastomer::Client::Bulk do
           :tweet => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :message => { :type => "string", :analyzer => "standard" },
-              :author  => { :type => "string", :index => "not_analyzed" }
+              :message => $client.version_support.text(analyzer: "standard"),
+              :author  => $client.version_support.keyword
             }
           },
           :book => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :title  => { :type => "string", :analyzer => "standard" },
-              :author => { :type => "string", :index => "not_analyzed" }
+              :title  => $client.version_support.text(analyzer: "standard"),
+              :author => $client.version_support.keyword
             }
           }
         }
@@ -89,16 +89,14 @@ describe Elastomer::Client::Bulk do
 
     assert_equal 2, h["items"].length
 
-    if es_version_2_x?
+    if bulk_index_returns_create_for_new_documents?
       assert_bulk_index h["items"].first
       assert_bulk_create h["items"].last
       book_id = items.last["create"]["_id"]
-    elsif es_version_5_x?
+    else
       assert_bulk_index h["items"].first
       assert_bulk_index h["items"].last
       book_id = items.last["index"]["_id"]
-    else
-      fail "Unknown ES version!"
     end
 
     assert_match %r/^\S{20,22}$/, book_id
@@ -120,18 +118,16 @@ describe Elastomer::Client::Bulk do
 
     assert_equal 2, h["items"].length
 
-    if es_version_2_x?
+    if bulk_index_returns_create_for_new_documents?
       assert_bulk_create h["items"].first, "expected to create a book"
       assert_bulk_delete h["items"].last, "expected to delete a book"
 
       book_id2 = items.first["create"]["_id"]
-    elsif es_version_5_x?
+    else
       assert_bulk_index h["items"].first, "expected to create a book"
       assert_bulk_delete h["items"].last, "expected to delete a book"
 
       book_id2 = items.first["index"]["_id"]
-    else
-      fail "Unknown ES version!"
     end
 
     assert_match %r/^\S{20,22}$/, book_id2

data/test/client/cluster_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Cluster do
 
@@ -69,7 +69,7 @@ describe Elastomer::Client::Cluster do
   it "returns cluster stats" do
     h = @cluster.stats
     expected = %w[cluster_name indices nodes status timestamp]
-    expected.unshift("_nodes") if es_version_5_x?
+    expected.unshift("_nodes") if cluster_stats_includes_underscore_nodes?
     assert_equal expected, h.keys.sort
   end
 

data/test/client/docs_test.rb

@@ -1,4 +1,4 @@
-require File.expand_path("../../test_helper", __FILE__)
+require_relative "../test_helper"
 
 describe Elastomer::Client::Docs do
 
@@ -13,19 +13,24 @@ describe Elastomer::Client::Docs do
           :doc1 => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :title  => { :type => "string", :analyzer => "standard", :term_vector => "with_positions_offsets" },
-              :author => { :type => "string", :index => "not_analyzed" }
+              :title  => $client.version_support.text(analyzer: "standard", term_vector: "with_positions_offsets"),
+              :author => $client.version_support.keyword
             }
           },
           :doc2 => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :title  => { :type => "string", :analyzer => "standard", :term_vector => "with_positions_offsets" },
-              :author => { :type => "string", :index => "not_analyzed" }
+              :title  => $client.version_support.text(analyzer: "standard", term_vector: "with_positions_offsets"),
+              :author => $client.version_support.keyword
             }
           }
         }
 
+      # COMPATIBILITY
+      if requires_percolator_mapping?
+        @index.update_mapping("percolator", { :properties => { :query => { :type => "percolator"}}})
+      end
+
       wait_for_index(@name)
     end
 
@@ -92,26 +97,81 @@ describe Elastomer::Client::Docs do
     assert_match %r/^\S{20,22}$/, h["_id"]
   end
 
-  it "extracts underscore attributes from the document" do
-    doc = {
-      :_id => "12",
-      :_type => "doc2",
-      :_routing => "author",
-      "_consistency" => "all",
-      :title => "The Adventures of Huckleberry Finn",
-      :author => "Mark Twain",
-      :_unknown => "unknown attribute"
-    }
+  describe "indexing directive fields" do
+    it "indexes fields that are not recognized as indexing directives" do
+      doc = {
+        _id: "12",
+        _type: "doc2",
+        title: "The Adventures of Huckleberry Finn",
+        author: "Mark Twain",
+        _unknown_1: "unknown attribute 1",
+        "_unknown_2" => "unknown attribute 2"
+      }
 
-    h = @docs.index doc
-    assert_created h
-    assert_equal "12", h["_id"]
+      h = @docs.index(doc)
+      assert_created h
+      assert_equal "12", h["_id"]
+
+      indexed_doc = @docs.get(type: "doc2", id: "12")
+      expected = {
+        "title" => "The Adventures of Huckleberry Finn",
+        "author" => "Mark Twain",
+        "_unknown_1" => "unknown attribute 1",
+        "_unknown_2" => "unknown attribute 2"
+      }
+      assert_equal expected, indexed_doc["_source"]
+    end
+
+    it "extracts indexing directives from the document" do
+      doc = {
+        _id: "12",
+        "_type" => "doc2",
+        _routing: "author",
+        title: "The Adventures of Huckleberry Finn",
+        author: "Mark Twain"
+      }
+
+      h = @docs.index(doc)
+      assert_created h
+      assert_equal "12", h["_id"]
+
+      # Special keys are removed from the document hash
+      refute doc.key?(:_id)
+      refute doc.key?("_type")
+      refute doc.key?(:_routing)
+
+      indexed_doc = @docs.get(type: "doc2", id: "12")
+      expected = {
+        "title" => "The Adventures of Huckleberry Finn",
+        "author" => "Mark Twain",
+      }
+      assert_equal expected, indexed_doc["_source"]
+    end
 
-    refute doc.key?(:_id)
-    refute doc.key?(:_type)
-    refute doc.key?(:_routing)
-    refute doc.key?("_consistency")
-    assert doc.key?(:_unknown)
+    # COMPATIBILITY: Fail fast on known indexing directives that aren't for this version of ES
+    it "raises an exception when a known indexing directive from an unsupported version is used" do
+      # Symbol keys
+      doc = {
+        _id: "12",
+        _type: "doc2",
+        title: "The Adventures of Huckleberry Finn"
+      }.merge(incompatible_indexing_directive)
+
+      assert_raises(Elastomer::Client::IllegalArgument) do
+        @docs.index(doc)
+      end
+
+      # String keys
+      doc = {
+        "_id" => "12",
+        "_type" => "doc2",
+        "title" => "The Adventures of Huckleberry Finn"
+      }.merge(incompatible_indexing_directive.stringify_keys)
+
+      assert_raises(Elastomer::Client::IllegalArgument) do
+        @docs.index(doc)
+      end
+    end
   end
 
   it "gets documents from the search index" do
@@ -214,33 +274,38 @@ describe Elastomer::Client::Docs do
     assert_equal %w[pea53 grantr], authors
 
     h = @docs.delete_by_query(:q => "author:grantr")
-    assert_equal(h["_indices"], {
-      "_all" => {
-        "found" => 1,
-        "deleted" => 1,
-        "missing" => 0,
-        "failed" => 0,
-      },
-      @name => {
-        "found" => 1,
-        "deleted" => 1,
-        "missing" => 0,
-        "failed" => 0,
-      },
-    })
+
+    if supports_native_delete_by_query?
+      assert_equal(1, h["deleted"])
+    else
+      assert_equal(h["_indices"], {
+        "_all" => {
+          "found" => 1,
+          "deleted" => 1,
+          "missing" => 0,
+          "failed" => 0,
+        },
+        @name => {
+          "found" => 1,
+          "deleted" => 1,
+          "missing" => 0,
+          "failed" => 0,
+        },
+      })
+    end
+
     @index.refresh
     h = @docs.multi_get :ids => [1, 2]
     assert_found h["docs"][0]
     refute_found h["docs"][1]
 
     h = @docs.delete_by_query(
-          :query => {
-            :filtered => {
-              :query => {:match_all => {}},
-              :filter => {:term => {:author => "pea53"}}
-            }
-          }
-        )
+      :query => {
+        :bool => {
+          :filter => {:term => {:author => "pea53"}}
+        }
+      }
+    )
     @index.refresh
     h = @docs.multi_get :ids => [1, 2]
     refute_found h["docs"][0]
@@ -261,7 +326,7 @@ describe Elastomer::Client::Docs do
 
     h = @docs.search({
       :query => {:match_all => {}},
-      :filter => {:term => {:author => "defunkt"}}
+      :post_filter => {:term => {:author => "defunkt"}}
     }, :type => %w[doc1 doc2] )
     assert_equal 1, h["hits"]["total"]
 
@@ -302,8 +367,7 @@ describe Elastomer::Client::Docs do
 
     h = @docs.count({
       :query => {
-        :filtered => {
-          :query => {:match_all => {}},
+        :bool => {
           :filter => {:term => {:author => "defunkt"}}
         }
       }
@@ -340,8 +404,23 @@ describe Elastomer::Client::Docs do
           :filter => {:term => {:author => "defunkt"}}
         }
       }
-    }, :type => %w[doc1 doc2] )
-    assert_equal true, h["valid"]
+    }, :type => %w[doc1 doc2])
+
+    if filtered_query_removed?
+      refute h["valid"]
+    else
+      assert h["valid"]
+    end
+
+    h = @docs.validate({
+      :query => {
+        :bool => {
+          :filter => {:term => {:author => "defunkt"}}
+        }
+      }
+    }, :type => %w[doc1 doc2])
+
+    assert h["valid"]
   end
 
   it "updates documents" do
@@ -463,6 +542,7 @@ describe Elastomer::Client::Docs do
     percolator2 = @index.percolator "2"
     response = percolator2.create :query => { :match => { :author => "defunkt" } }
     assert response["created"], "Couldn't create the percolator query"
+    @index.refresh
 
     response = @index.docs("doc1").percolate(:doc => { :author => "pea53" })
     assert_equal 1, response["matches"].length
@@ -478,6 +558,7 @@ describe Elastomer::Client::Docs do
     percolator2 = @index.percolator "2"
     response = percolator2.create :query => { :match => { :author => "defunkt" } }
     assert response["created"], "Couldn't create the percolator query"
+    @index.refresh
 
     response = @index.docs("doc2").percolate(nil, :id => "1")
     assert_equal 1, response["matches"].length
@@ -493,6 +574,7 @@ describe Elastomer::Client::Docs do
     percolator2 = @index.percolator "2"
     response = percolator2.create :query => { :match => { :author => "defunkt" } }
     assert response["created"], "Couldn't create the percolator query"
+    @index.refresh
 
     count = @index.docs("doc1").percolate_count :doc => { :author => "pea53" }
     assert_equal 1, count
@@ -507,6 +589,7 @@ describe Elastomer::Client::Docs do
     percolator2 = @index.percolator "2"
     response = percolator2.create :query => { :match => { :author => "defunkt" } }
     assert response["created"], "Couldn't create the percolator query"
+    @index.refresh
 
     count = @index.docs("doc2").percolate_count(nil, :id => "1")
     assert_equal 1, count
@@ -515,6 +598,7 @@ describe Elastomer::Client::Docs do
   it "performs multi percolate queries" do
     @index.percolator("1").create :query => { :match_all => { } }
     @index.percolator("2").create :query => { :match => { :author => "pea53" } }
+    @index.refresh
 
     h = @index.docs("doc2").multi_percolate do |m|
       m.percolate :author => "pea53"