elastomer-client 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20de9fda77111f1c5d50ccbbb63dd8b3316f7149
-  data.tar.gz: 97b970b60153c6069c44d866b00c08ba0542ae24
+  metadata.gz: 067c35156016c954c5bffa66b9f1eba92f977410
+  data.tar.gz: a49152e69e1b33ac13f6fef6404cb219a35448e6
 SHA512:
-  metadata.gz: 25dd4d57d43c60e279eaec42708d0ba8f12ad3d085baa64d9b68c489e31f5f152e54b5663fad14afe1c459245e7ca9237082fe41d834a8126be3c09f3d311899
-  data.tar.gz: 751eeeb151a1e582903b84f6383de2f7ff47f39db86afeb582fed185775b6881b915ddaf80c3326930f988bf21d558eb3f66dae82f26b018d4b9c641b64d6b50
+  metadata.gz: 09e1f8f14d809b1c475216620ae2fffc7dd2aa712eead86811dcfd81288078cd25d9507ee66453729a7fe82eba3296525395587044d9bbb9c605e656eae4869d
+  data.tar.gz: 876a476bf1d9cbdb46c4524dad8bc097813eec062d6757e7be046777a162d097f696240a14de83a256ab93d16ab8ec3f53c427ebb68242ee9b9b35ef008ceb96

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.7.0 (2015-09-18)
+- Add streaming bulk functionality via `bulk_stream_items`
+- Make Delete by Query compatible with Elasticsearch 2.0
+
 ## 0.6.0 (2015-09-11)
 - Support all URL parameters when using `Client.#scroll`
 - BREAKING: Moved some `Scroller` reader methods into `Scroller.opts`

data/elastomer-client.gemspec

@@ -27,5 +27,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "activesupport", ">= 3.0"
   spec.add_development_dependency "minitest","~> 4.7"
+  spec.add_development_dependency "webmock","~> 1.21"
   spec.add_development_dependency "rake"
 end

data/lib/elastomer/client/bulk.rb

@@ -43,6 +43,103 @@ module Elastomer
       end
     end
 
+    # Stream bulk actions from an Enumerator.
+    #
+    # Examples
+    #
+    #   ops = [
+    #     [:index, document1, {:_type => "foo", :_id => 1}],
+    #     [:create, document2],
+    #     [:delete, {:_type => "bar", :_id => 42}]
+    #   ]
+    #   bulk_stream_responses(ops, :index => 'default-index').each do |response|
+    #     puts response
+    #   end
+    #
+    # Returns an Enumerator of responses.
+    def bulk_stream_responses(ops, params = {})
+      bulk_obj = Bulk.new(self, params)
+
+      Enumerator.new do |yielder|
+        ops.each do |action, *args|
+          response = bulk_obj.send(action, *args)
+          yielder.yield response unless response.nil?
+        end
+
+        response = bulk_obj.call
+        yielder.yield response unless response.nil?
+      end
+    end
+
+    # Internal: Determine whether or not a response item has an HTTP status code
+    # in the range 200 to 299.
+    #
+    # item - The bulk response item
+    #
+    # Returns a boolean
+    def is_ok?(item)
+      item.values.first["status"].between?(200, 299)
+    end
+
+    # Stream bulk actions from an Enumerator and passes the response items to
+    # the given block.
+    #
+    # Examples
+    #
+    #   ops = [
+    #     [:index, document1, {:_type => "foo", :_id => 1}],
+    #     [:create, document2],
+    #     [:delete, {:_type => "bar", :_id => 42}]
+    #   ]
+    #   bulk_stream_items(ops, :index => 'default-index') do |item|
+    #     puts item
+    #   end
+    #
+    #   # return value:
+    #   # {
+    #   #   "took" => 256,
+    #   #   "errors" => false,
+    #   #   "success" => 3,
+    #   #   "failure" => 0
+    #   # }
+    #
+    #   # sample response item:
+    #   # {
+    #   #   "delete": {
+    #   #     "_index": "foo",
+    #   #     "_type": "bar",
+    #   #     "_id": "42",
+    #   #     "_version": 3,
+    #   #     "status": 200,
+    #   #     "found": true
+    #   #   }
+    #   # }
+    #
+    # Returns a Hash of stats about items from the responses.
+    def bulk_stream_items(ops, params = {})
+      stats = {
+        "took" => 0,
+        "errors" => false,
+        "success" => 0,
+        "failure" => 0
+      }
+
+      bulk_stream_responses(ops, params).each do |response|
+        stats["took"] += response["took"]
+        stats["errors"] |= response["errors"]
+
+        response["items"].each do |item|
+          if is_ok?(item)
+            stats["success"] += 1
+          else
+            stats["failure"] += 1
+          end
+          yield item
+        end
+      end
+
+      stats
+    end
 
     # The Bulk class provides some abstractions and helper methods for working
     # with the ElasticSearch bulk API command. Instances of the Bulk class
@@ -156,7 +253,7 @@ module Elastomer
       end
 
       # Add an update action to the list of bulk actions to be performed when
-      # the bulk API call is made. Parameters can be provided in the parameters 
+      # the bulk API call is made. Parameters can be provided in the parameters
       # hash (underscore prefix optional) or in the document hash (underscore
       # prefix required).
       #
@@ -178,7 +275,7 @@ module Elastomer
       # the bulk API call is made.
       #
       # params - Parameters for the delete action (as a Hash)
-      #      
+      #
       # Examples
       #   delete(:_id => 1, :_type => 'foo')
       #

data/lib/elastomer/client/delete_by_query.rb

@@ -0,0 +1,120 @@
+module Elastomer
+  class Client
+
+    # Delete documents from one or more indices and one or more types based
+    # on a query.
+    #
+    # The return value follows the format returned by the Elasticsearch Delete
+    # by Query plugin: https://github.com/elastic/elasticsearch/blob/master/docs/plugins/delete-by-query.asciidoc#response-body
+    #
+    # Internally, this method uses a combination of scroll and bulk delete
+    # instead of the Delete by Query API, which was removed in Elasticsearch
+    # 2.0.
+    #
+    # query  - The query body as a Hash
+    # params - Parameters Hash
+    #
+    # Examples
+    #
+    #   # request body query
+    #   delete_by_query({:query => {:match_all => {}}}, :type => 'tweet')
+    #
+    #   # same thing but using the URI request method
+    #   delete_by_query(nil, { :q => '*:*', :type => 'tweet' })
+    #
+    # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+    #
+    # Returns a Hash of statistics about the delete operations, for example:
+    #
+    #   {
+    #     "took" : 639,
+    #     "_indices" : {
+    #       "_all" : {
+    #         "found" : 5901,
+    #         "deleted" : 5901,
+    #         "missing" : 0,
+    #         "failed" : 0
+    #       },
+    #       "twitter" : {
+    #         "found" : 5901,
+    #         "deleted" : 5901,
+    #         "missing" : 0,
+    #         "failed" : 0
+    #       }
+    #     },
+    #     "failures" : [ ]
+    #   }
+    def delete_by_query(query, params = {})
+      DeleteByQuery.new(self, query, params).execute
+    end
+
+    class DeleteByQuery
+
+      # Create a new DeleteByQuery command for deleting documents matching a
+      # query
+      #
+      # client - Elastomer::Client used for HTTP requests to the server
+      # query  - The query used to find documents to delete
+      # params - Other URL parameters
+      def initialize(client, query, params = {})
+        @client = client
+        @query = query
+        @params = params
+        @response_stats = { 'took' => 0, '_indices' => { '_all' => {} }, 'failures' => [] }
+      end
+
+      attr_reader :client, :query, :params, :response_stats
+
+      # Internal: Determine whether or not an HTTP status code is in the range
+      # 200 to 299
+      #
+      # status - HTTP status code
+      #
+      # Returns a boolean
+      def is_ok?(status)
+        status.between?(200, 299)
+      end
+
+      # Internal: Tally the contributions of an item to the found, deleted,
+      # missing, and failed counts for the summary statistics
+      #
+      # item - An element of the items array from a bulk response
+      #
+      # Returns a Hash of counts for each category
+      def categorize(item)
+        {
+          "found" => item["found"] || item["status"] == 409 ? 1 : 0,
+          "deleted" => is_ok?(item["status"]) ? 1 : 0,
+          "missing" => !item["found"]  && !item.key?("error") ? 1 : 0,
+          "failed" => item.key?("error") ? 1 : 0,
+        }
+      end
+
+      # Internal: Combine a response item with the existing statistics
+      #
+      # item - A bulk response item
+      def accumulate(item)
+        item = item["delete"]
+        (@response_stats['_indices'][item['_index']] ||= {}).merge!(categorize(item)) { |_, n, m| n + m }
+        @response_stats['_indices']['_all'].merge!(categorize(item)) { |_, n, m| n + m }
+        @response_stats['failures'] << item unless is_ok? item['status']
+      end
+
+      # Perform the Delete by Query action
+      #
+      # Returns a Hash of statistics about the bulk operation
+      def execute
+        ops = Enumerator.new do |yielder|
+          @client.scan(@query, @params).each_document do |hit|
+            yielder.yield([:delete, { _id: hit["_id"], _type: hit["_type"], _index: hit["_index"] }])
+          end
+        end
+
+        stats = @client.bulk_stream_items(ops, @params) { |item| accumulate(item) }
+        @response_stats['took'] = stats['took']
+        @response_stats
+      end
+
+    end  # DeleteByQuery
+  end  # Client
+end  # Elastomer

data/lib/elastomer/client/docs.rb

@@ -252,25 +252,13 @@ module Elastomer
       # hash must contain the :query key. Otherwise we assume a URI request is
       # being made.
       #
-      # query  - The query body as a Hash
-      # params - Parameters Hash
-      #
-      # Examples
-      #
-      #   # request body query
-      #   delete_by_query({:query => {:match_all => {}}}, :type => 'tweet')
+      # See Client#delete_by_query for more information.
       #
-      #   # same thing but using the URI request method
-      #   delete_by_query(:q => '*:*', :type => 'tweet')
-      #
-      # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
-      #
-      # Returns the response body as a hash
-      def delete_by_query( query, params = nil )
+      # Returns a Hash of statistics about the delete operations
+      def delete_by_query(query, params = nil)
         query, params = extract_params(query) if params.nil?
 
-        response = client.delete '/{index}{/type}/_query', update_params(params, :body => query, :action => 'docs.delete_by_query')
-        response.body
+        @client.delete_by_query(query, update_params(params))
       end
 
       # Returns information and statistics on terms in the fields of a

data/lib/elastomer/client/index.rb

@@ -523,6 +523,16 @@ module Elastomer
         client.warmer(name, warmer_name)
       end
 
+      # Delete documents from one or more indices and one or more types based
+      # on a query.
+      #
+      # See Client#delete_by_query for more information.
+      #
+      # Returns a Hash of statistics about the delete operations
+      def delete_by_query(query, params = nil)
+        docs.delete_by_query(query, params)
+      end
+
       # Internal: Add default parameters to the `params` Hash and then apply
       # `overrides` to the params if any are given.
       #

data/lib/elastomer/version.rb

@@ -1,5 +1,5 @@
 module Elastomer
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 
   def self.version
     VERSION

data/test/client/bulk_test.rb

@@ -293,4 +293,31 @@ describe Elastomer::Client::Bulk do
     assert_equal 'tweet', items[0]['index']['_type']
     assert_equal 1, items[0]['index']['_version']
   end
+
+  it 'streams bulk responses' do
+    ops = [
+      [:index, { :message => 'tweet 1' }, { :_id => 1, :_type => 'book', :_index => @index.name }],
+      [:index, { :message => 'tweet 2' }, { :_id => 2, :_type => 'book', :_index => @index.name }],
+      [:index, { :message => 'tweet 3' }, { :_id => 3, :_type => 'book', :_index => @index.name }]
+    ]
+    responses = $client.bulk_stream_responses(ops, { :action_count => 2 }).to_a
+    assert_equal(2, responses.length)
+    assert_bulk_index(responses[0]["items"][0])
+    assert_bulk_index(responses[0]["items"][1])
+    assert_bulk_index(responses[1]["items"][0])
+  end
+
+  it 'streams bulk items' do
+    ops = [
+      [:index, { :message => 'tweet 1' }, { :_id => 1, :_type => 'book', :_index => @index.name }],
+      [:index, { :message => 'tweet 2' }, { :_id => 2, :_type => 'book', :_index => @index.name }],
+      [:index, { :message => 'tweet 3' }, { :_id => 3, :_type => 'book', :_index => @index.name }]
+    ]
+    items = []
+    stats = $client.bulk_stream_items(ops, { :action_count => 2 }) { |item| items << item }
+    assert_equal(3, items.length)
+    assert_bulk_index(items[0])
+    assert_bulk_index(items[1])
+    assert_bulk_index(items[2])
+  end
 end

data/test/client/delete_by_query_test.rb

@@ -0,0 +1,151 @@
+require File.expand_path('../../test_helper', __FILE__)
+
+describe Elastomer::Client::DeleteByQuery do
+
+  before do
+    @index = $client.index "elastomer-delete-by-query-test"
+    @index.delete if @index.exists?
+    @docs = @index.docs("docs")
+  end
+
+  after do
+    @index.delete if @index.exists?
+  end
+
+  describe "when an index with documents exists" do
+    before do
+      @index.create(nil)
+      wait_for_index(@index_name)
+    end
+
+    it 'deletes by query' do
+      @docs.index({ :_id => 0, :name => "mittens" })
+      @docs.index({ :_id => 1, :name => "luna" })
+
+      @index.refresh
+      response = $client.delete_by_query(nil, :q => "name:mittens")
+      assert_equal({
+        '_all' => {
+          'found' => 1,
+          'deleted' => 1,
+          'missing' => 0,
+          'failed' => 0,
+        },
+        @index.name => {
+          'found' => 1,
+          'deleted' => 1,
+          'missing' => 0,
+          'failed' => 0,
+        },
+      }, response['_indices'])
+
+      @index.refresh
+      response = @docs.multi_get :ids => [0, 1]
+      refute_found response['docs'][0]
+      assert_found response['docs'][1]
+    end
+
+    it 'respects action_count' do
+      @docs.index({ :_id => 0, :name => "mittens" })
+      @docs.index({ :_id => 1, :name => "luna" })
+      @index.refresh
+
+      response = $client.delete_by_query(nil, :action_count => 1)
+
+      assert_requested(:post, /_bulk/, :times => 2)
+
+      assert_equal({
+        '_all' => {
+          'found' => 2,
+          'deleted' => 2,
+          'missing' => 0,
+          'failed' => 0,
+        },
+        @index.name => {
+          'found' => 2,
+          'deleted' => 2,
+          'missing' => 0,
+          'failed' => 0,
+        },
+      }, response['_indices'])
+
+      @index.refresh
+      response = @docs.multi_get :ids => [0, 1]
+      refute_found response['docs'][0]
+      refute_found response['docs'][1]
+    end
+
+    it 'counts missing documents' do
+      @docs.index({ :_id => 0 })
+
+      stub_request(:post, /_bulk/).
+        to_return(lambda do |request|
+          {
+            :body => MultiJson.dump({
+              "took" => 0,
+              "errors" => false,
+              "items" => [{
+                "delete" => {
+                  "_index" => @index.name,
+                  "_type" => @docs.name,
+                  "_id" => 0,
+                  "_version" => 1,
+                  "status" => 404,
+                  "found" => false } }] }) }
+        end)
+
+      @index.refresh
+      response = $client.delete_by_query(nil, :action_count => 1)
+      assert_equal({
+        '_all' => {
+          'found' => 0,
+          'deleted' => 0,
+          'missing' => 1,
+          'failed' => 0,
+        },
+        @index.name => {
+          'found' => 0,
+          'deleted' => 0,
+          'missing' => 1,
+          'failed' => 0,
+        },
+      }, response['_indices'])
+    end
+
+    it 'counts failed operations' do
+      @docs.index({ :_id => 0 })
+
+      stub_request(:post, /_bulk/).
+        to_return(lambda do |request|
+          {
+            :body => MultiJson.dump({
+              "took" => 0,
+              "errors" => false,
+              "items" => [{
+                "delete" => {
+                  "_index" => @index.name,
+                  "_type" => @docs.name,
+                  "_id" => 0,
+                  "status" => 409,
+                  "error" => "VersionConflictEngineException" } }] }) }
+        end)
+
+      @index.refresh
+      response = $client.delete_by_query(nil, :action_count => 1)
+      assert_equal({
+        '_all' => {
+          'found' => 1,
+          'deleted' => 0,
+          'missing' => 0,
+          'failed' => 1,
+        },
+        @index.name => {
+          'found' => 1,
+          'deleted' => 0,
+          'missing' => 0,
+          'failed' => 1,
+        },
+      }, response['_indices'])
+    end
+  end
+end

data/test/client/docs_test.rb

@@ -214,6 +214,20 @@ describe Elastomer::Client::Docs do
     assert_equal %w[pea53 grantr], authors
 
     h = @docs.delete_by_query(:q => "author:grantr")
+    assert_equal(h['_indices'], {
+      '_all' => {
+        'found' => 1,
+        'deleted' => 1,
+        'missing' => 0,
+        'failed' => 0,
+      },
+      @name => {
+        'found' => 1,
+        'deleted' => 1,
+        'missing' => 0,
+        'failed' => 0,
+      },
+    })
     @index.refresh
     h = @docs.multi_get :ids => [1, 2]
     assert_found h['docs'][0]

data/test/client/index_test.rb

@@ -362,5 +362,25 @@ describe Elastomer::Client::Index do
       response = @index.segments
       assert_includes response["indices"], "elastomer-index-test"
     end
+
+    it 'deletes by query' do
+      @index.docs('foo').index("foo" => "bar")
+      @index.refresh
+      r = @index.delete_by_query(:q => '*')
+      assert_equal({
+        '_all' => {
+          'found' => 1,
+          'deleted' => 1,
+          'missing' => 0,
+          'failed' => 0,
+        },
+        @name => {
+          'found' => 1,
+          'deleted' => 1,
+          'missing' => 0,
+          'failed' => 0,
+        }
+      }, r['_indices'])
+    end
   end
 end

data/test/test_helper.rb

@@ -1,3 +1,6 @@
+require 'webmock/minitest'
+WebMock.allow_net_connect!
+
 require 'securerandom'
 require 'rubygems' unless defined? Gem
 require 'bundler'
@@ -24,7 +27,7 @@ $client_params = {
   :port => ENV['BOXEN_ELASTICSEARCH_PORT'] || 9200,
   :read_timeout => 2,
   :open_timeout => 1,
-  :opaque_id => true
+  :opaque_id => false
 }
 $client = Elastomer::Client.new $client_params
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastomer-client
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Tim Pease
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-11 00:00:00.000000000 Z
+date: 2015-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -109,6 +109,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -156,6 +170,7 @@ files:
 - lib/elastomer/client.rb
 - lib/elastomer/client/bulk.rb
 - lib/elastomer/client/cluster.rb
+- lib/elastomer/client/delete_by_query.rb
 - lib/elastomer/client/docs.rb
 - lib/elastomer/client/errors.rb
 - lib/elastomer/client/index.rb
@@ -179,6 +194,7 @@ files:
 - test/assertions.rb
 - test/client/bulk_test.rb
 - test/client/cluster_test.rb
+- test/client/delete_by_query_test.rb
 - test/client/docs_test.rb
 - test/client/errors_test.rb
 - test/client/index_test.rb
@@ -225,6 +241,7 @@ test_files:
 - test/assertions.rb
 - test/client/bulk_test.rb
 - test/client/cluster_test.rb
+- test/client/delete_by_query_test.rb
 - test/client/docs_test.rb
 - test/client/errors_test.rb
 - test/client/index_test.rb