trample_search 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20ffd69885c3eeda15f53ab9d4b6dd4bbc27f7be
-  data.tar.gz: 71f0bc852acc52e6cd9e3f71edc74336e26b0c85
+  metadata.gz: d6aae72f53b824559e0bcf84a7c544737bc8b180
+  data.tar.gz: 50bca159319464c8cc994d0f87b7aec131b279c4
 SHA512:
-  metadata.gz: 54309a47baec66899dc8e2481e37d3c4406394f8e8b86de457e084c468a352ce351b7988f0509daa6a46feab57f11d8975eea381efc5db53ed9642f8bfecaa63
-  data.tar.gz: 9c25d165d404140577558b19e5a9b776c5c55091f12a51ddb95ade05ea96293f299c24b7225d3935d9eb7c4dcbc567aa233e14005d1bab5a961626925f9d6c70
+  metadata.gz: da90c4512a189ee0909d685eed2b4ea4e4f90408925efaa03dd78584055f07ae5c01f17b98052f11aa3207203e1320b885d419a99d59a7753f1ea23a8bf100b0
+  data.tar.gz: 5bc12aaff9e5665ec1b50c86272495d85622d016bee5c81cbb7fd3a03a0c8ce9ef3289fb4628d6ed44cdb519463855f92d2d35bf3e4123a887757bf21fea7eba

data/lib/trample/backend/searchkick.rb

@@ -10,19 +10,55 @@ module Trample
       end
 
       def query!(conditions, aggregations)
-        query = build_query(conditions, aggregations, @metadata, @_models)
-        results = @_models.first.search(keywords(conditions), query)
-        parse_response_aggs!(results.aggs, aggregations) if results.response.has_key?('aggregations')
+        results = resume_or_execute(conditions, aggregations)
 
         {
-          total:   results.total_count,
-          took:    results.response['took'],
-          results: results.results
+          total:     results.total_count,
+          took:      results.response['took'],
+          results:   results.results,
+          scroll_id: results.response['_scroll_id']
         }
       end
 
+      # There are 3 types of searches
+      #
+      # * A) Vanilla
+      # * B) Vanilla, but tell ES we are scrolling
+      # * C) A scroll search, using the scroll_id from B
+      def resume_or_execute(conditions, aggregations)
+        payload = build_payload(conditions, aggregations, @metadata, @_models)
+        query = ::Searchkick::Query.new(@_models.first, keywords(conditions), payload)
+
+        response = if @metadata.scroll_id
+          resume_search(@metadata.scroll_id)
+        else
+          execute_search(query)
+        end
+        results = query.handle_response(response)
+        parse_response_aggs!(results.aggs, aggregations) if results.response.has_key?('aggregations')
+        results
+      end
+
+      def execute_search(query)
+        search_params = raw_search_params(query, scroll: @metadata.scroll)
+        ::Searchkick.client.search(search_params)
+      end
+
       private
 
+      def resume_search(scroll_id)
+        search_params = { scroll_id: scroll_id, scroll: '1m' }
+        ::Searchkick.client.scroll(search_params)
+      end
+
+      def raw_search_params(query, scroll: false)
+        query.params.tap do |search_params|
+          if scroll
+            search_params[:scroll] = '5m'
+          end
+        end
+      end
+
       def keywords(conditions)
         if conditions[:keywords] and conditions[:keywords].values.first != ''
           conditions[:keywords].values.first
@@ -31,7 +67,7 @@ module Trample
         end
       end
 
-      def build_query(conditions, aggregations, metadata, models)
+      def build_payload(conditions, aggregations, metadata, models)
         clauses = build_condition_clauses(conditions, aggregations)
         query   = searchkick_payload(conditions[:keywords], clauses, metadata, aggregations)
         query.merge!(index_name: models.map { |m| m.searchkick_index.name }) if models.length > 1

data/lib/trample/metadata.rb

@@ -33,6 +33,8 @@ module Trample
     attribute :pagination, Pagination, default: ->(_,_) { Pagination.new }
     attribute :took, Integer
     attribute :sort, Array[Sort]
+    attribute :scroll, Boolean, default: false
+    attribute :scroll_id, String
 
     def_delegators :pagination, :total, :current_page, :per_page
     def_delegator :sort, :att, :sort_att

data/lib/trample/search.rb

@@ -137,6 +137,7 @@ module Trample
       @records = nil
       hash = backend.query!(conditions, aggregations)
       self.metadata.took = hash[:took]
+      self.metadata.scroll_id = hash[:scroll_id]
       self.metadata.pagination.total = hash[:total]
       self.results = hash[:results]
       if !!metadata.records[:load]
@@ -166,21 +167,22 @@ module Trample
       records
     end
 
-    #This implementation is not using scroll and scan of elastic
-    #https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
-    #In future versions we hope to replace this with elastic in-built implementation
+    # Uses elasticsearch scroll, as ES will blow up > 10_000 results,
+    # even if you are paginating.
     #
+    # 1. Execute a search, telling ES we are scrolling
+    # 2. metadata.scroll_id is set on response
+    # 3. Use the scroll_id to fetch the next resultset
+    # 4. Repeat until results are exhausted
     def find_in_batches(batch_size: 10_000)
-      page_number = 1
-
-      loop do
-        paginate(size: batch_size, number: page_number)
+      metadata.scroll = true
+      metadata.pagination.per_page = batch_size
+      query!
+      while !self.results.empty?
+        yield self.results
         query!
-        yield results
-        offset = metadata.pagination.current_page * metadata.pagination.per_page
-        break unless  metadata.pagination.next?
-        page_number += 1
       end
+      self.metadata = Metadata.new
     end
 
     private

data/lib/trample/version.rb

@@ -1,3 +1,3 @@
 module Trample
-  VERSION = "0.16.0"
+  VERSION = "0.17.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: trample_search
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - richmolj
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-27 00:00:00.000000000 Z
+date: 2016-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus