scalastic 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 658c178987891b8d35d5e53453957d166be292e8
-  data.tar.gz: c01b6636034a41ea44c2c02216d53d6c7fb801e7
+  metadata.gz: bce6be5082989888ebb029cc6491c89e9ac426c8
+  data.tar.gz: db64fde0ec4e69129a08d16022820f1fd7a2c04f
 SHA512:
-  metadata.gz: 85c8a15a5b37793884bd143493ceea7d75bfebdb9afa16f4aa67c19b8d500d8a53863190c7050bd20907b42b5ff8cd994718c765d40f00c3fe2b4c732f2f6440
-  data.tar.gz: 3d87aebccf6648854df7f4ac582fe183432cba07723847665313bc8456180ac87bdae5a3ae06aabec9f6b5cd404b201e7e210cb0a6643104110cecd96e5b817d
+  metadata.gz: 4c8bdbdd6c7819ccb1aac88061039fc433ba3b590d37ed0a3648659beb9f5073f908cf54f0f23bf5f41c076230433660ae713aa63faef84c8eaa1f68b95e9574
+  data.tar.gz: 25947a75533799e222ce26325e99dcfb9c8f4fe441e261f25ecd5cb15a0e4ab4741b32c4f61a8d58e5a98a5f17a1e8495decabb156802383e520a839dd7416c6

data/README.md

@@ -215,6 +215,29 @@ count = partition.search(search_type: 'count')['hits']['total']
 raise 'Some documents were not removed' unless count == 0
 ```
 
+### Scan/scroll
+Partitions provide `scroll` method for scrolling through results of the query.
+```ruby
+# Connect to Elasticsearch
+client = Elasticsearch::Client.new
+client.indices.create index: 'scrolling'
+partitions = client.partitions
+partitions.prepare_index index: 'scrolling'
+
+p = partitions.create id: 1, index: 'scrolling'
+
+# Create some test data
+10.times.each do |i|
+  p.index id: i + 1, type: 'test', body: {subject: "Test ##{i + 1}"}
+end
+
+# Get the hits. Size is set to 7 to test multiple calls to scroll
+actual_hits = p.scroll(type: 'test', size: 7).to_a.sort{|h1, h2| h1['_id'].to_i <=> h2['_id'].to_i}
+expected_hits = 10.times.map{|i| {'_id' => "#{i + }", '_type' => 'test', '_source' => {'_subject' => "Test ##{i + 1}"} }}
+
+raise "Expected: #{expected_hits}, got: #{actual_hits}" unless expected_hits == actual_hits
+```
+
 ### Deleting by query
 Scalastic partitions support delete_by_query, but because it is no longer available in Elasticsearch core, we use our own implementation which uses scan/scroll searches and bulk operations for deletion.
 ```ruby

data/lib/scalastic/partition.rb

@@ -1,5 +1,6 @@
 require 'scalastic/es_actions_generator'
 require 'scalastic/partition_selector'
+require 'scalastic/scroller'
 
 module Scalastic
   class Partition
@@ -123,6 +124,11 @@ module Scalastic
       get_endpoints.index.nil?
     end
 
+    def scroll(args)
+      args = args.merge(index: config.search_endpoint(id))
+      Scroller.new(es_client, args)
+    end
+
     private
 
     def operation_name(entry)

data/lib/scalastic/scroller.rb

@@ -0,0 +1,32 @@
+module Scalastic
+  class Scroller
+    include Enumerable
+
+    def initialize(es_client, args)
+      @es_client = es_client
+      @args = args
+      @scroll = '1m'
+    end
+
+    def scroll=(value)
+      raise(ArgumentError, "scroll cannot be empty!") if value.nil? || value.empty?
+      @scroll = value
+    end
+
+    attr_reader(:scroll)
+
+    def each(&block)
+      Enumerator.new do |enum|
+        args = @args.merge(search_type: 'scan', scroll: scroll)
+        res = @es_client.search(args)
+        loop do
+          scroll_id = res['_scroll_id']
+          res = @es_client.scroll(body: scroll_id, scroll: scroll)
+          hits = res['hits']['hits']
+          break unless hits.any?
+          hits.each{|h| enum << h}
+        end
+      end.each(&block)
+    end
+  end
+end

data/lib/scalastic/version.rb

@@ -1,3 +1,3 @@
 module Scalastic
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/regression/regression_tests/scroll.rb

@@ -0,0 +1,31 @@
+module RegressionTests
+  module Scroll
+    extend self
+
+    def cleanup
+      client = Elasticsearch::Client.new
+      client.indices.delete index: 'scrolling' if client.indices.exists? index: 'scrolling'
+    end
+
+    def run
+      # Connect to Elasticsearch
+      client = Elasticsearch::Client.new
+      client.indices.create index: 'scrolling'
+      partitions = client.partitions
+      partitions.prepare_index index: 'scrolling'
+
+      p = partitions.create id: 1, index: 'scrolling'
+
+      # Create some test data
+      10.times.each do |i|
+        p.index id: i + 1, type: 'test', body: {subject: "Test ##{i + 1}"}
+      end
+
+      # Get the hits. Size is set to 7 to test multiple calls to scroll
+      actual_hits = p.scroll(type: 'test', size: 7).to_a.sort{|h1, h2| h1['_id'].to_i <=> h2['_id'].to_i}
+      expected_hits = 10.times.map{|i| {'_id' => "#{i + }", '_type' => 'test', '_source' => {'_subject' => "Test ##{i + 1}"} }}
+
+      raise "Expected: #{expected_hits}, got: #{actual_hits}" unless expected_hits == actual_hits
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scalastic
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Aliaksei Baturytski
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-21 00:00:00.000000000 Z
+date: 2016-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,7 @@ files:
 - lib/scalastic/partition.rb
 - lib/scalastic/partition_selector.rb
 - lib/scalastic/partitions_client.rb
+- lib/scalastic/scroller.rb
 - lib/scalastic/version.rb
 - regression/regression.rake
 - regression/regression_tests.rb
@@ -120,6 +121,7 @@ files:
 - regression/regression_tests/nested_selector.rb
 - regression/regression_tests/nested_selector_bulk.rb
 - regression/regression_tests/partition_operations.rb
+- regression/regression_tests/scroll.rb
 - regression/regression_tests/string_selector_field.rb
 - scalastic.gemspec
 homepage: https://github.com/aliakb/scalastic