couch_potato 1.10.1 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e0a3143ad59fbb9f3391e381841412de37afda869b90c3b105b3efe41859fa2
-  data.tar.gz: 452e6d3cc546b0d45feb5ea94ea81779400adf6474e22580e6637d15f58858bb
+  metadata.gz: fc65386f8f391b222c93733fbf83cb00050bc37b2fa1787be8d73e4847b2fa43
+  data.tar.gz: a2c6fe4017c646b3bceaa9baaa236044ffe286fd3380095296a602034511d817
 SHA512:
-  metadata.gz: f517b4ecfaf2c57eceade400298c68ef2d189f1274709c95f7bd54d97721b9131b79430956e5c517cd2fbd3781149ff9fe54c4640eaea8e82b940e97c696fd04
-  data.tar.gz: a668623b7cd70ab1acb86aff299f8729f8645625b560f3b45247447117c9ac69d5aa335d57ff9dacaa980cd021d2f01f5e21c0a88f5faa45d5add8045e016e21
+  metadata.gz: 5f7c933270982e090183aacdb7fdd52e76e2739476e7e4d87cbcad1239c62b4973f31ce62278f181c33612bc6c2dc8ee7abbc2ffca0bef4560b919b8a37359c8
+  data.tar.gz: 7f79c9a6b3c31f5f1fc0eea3f4a16da6e7cb6ace5a68232a9a127d7d056b5144d88714e12e128a64eba34a9377e073311408077958c14b818c0f4037853a59a5

data/CHANGES.md

@@ -1,5 +1,9 @@
 ## Changes
 
+# 1.11.0
+
+- improve view_in_batches performance by switching to using startkey_docid over skip
+
 ### 1.10.1
 
 - support passing an empty array to CouchPotato::Database#load

data/lib/couch_potato/database.rb

@@ -65,12 +65,27 @@ module CouchPotato
     # to a given block in batches of the given size, making multiple
     # requests with according skip/limit params sent to CouchDB.
     def view_in_batches(spec, batch_size: default_batch_size)
+      rows = nil
       batch = 0
       loop do
-        spec.view_parameters = spec.view_parameters.merge({ skip: batch * batch_size, limit: batch_size })
-        results = view(spec)
-        yield results
-        break if results.size < batch_size
+        spec.view_parameters = spec
+          .view_parameters
+          .merge({limit: batch_size})
+          .merge(
+            if rows
+              {
+                startkey: rows&.last&.dig('key'),
+                startkey_docid: rows&.last&.dig('id'),
+                skip: 1
+              }
+            else
+              {}
+            end
+          )
+        result = raw_view(spec)
+        rows = result['rows']
+        yield process_view_results(result, spec)
+        break if rows.size < batch_size
 
         batch += 1
       end
@@ -198,27 +213,34 @@ module CouchPotato
 
     def view_without_caching(spec)
       ActiveSupport::Notifications.instrument('couch_potato.view', name: "#{spec.design_document}/#{spec.view_name}") do
-        results = CouchPotato::View::ViewQuery.new(
-          couchrest_database,
-          spec.design_document,
-          { spec.view_name => {
-            map: spec.map_function,
-            reduce: spec.reduce_function
-          } },
-          ({ spec.list_name => spec.list_function } unless spec.list_name.nil?),
-          spec.lib,
-          spec.language
-        ).query_view!(spec.view_parameters)
-        processed_results = spec.process_results results
-        if processed_results.respond_to?(:database=)
-          processed_results.database = self
-        elsif processed_results.respond_to?(:each)
-          processed_results.each do |document|
-            document.database = self if document.respond_to?(:database=)
-          end
+        process_view_results(raw_view(spec), spec)
+      end
+    end
+
+    def process_view_results(results, spec)
+      processed_results = spec.process_results results
+      if processed_results.respond_to?(:database=)
+        processed_results.database = self
+      elsif processed_results.respond_to?(:each)
+        processed_results.each do |document|
+          document.database = self if document.respond_to?(:database=)
         end
-        processed_results
       end
+      processed_results
+    end
+
+    def raw_view(spec)
+      CouchPotato::View::ViewQuery.new(
+        couchrest_database,
+        spec.design_document,
+        { spec.view_name => {
+          map: spec.map_function,
+          reduce: spec.reduce_function
+        } },
+        ({ spec.list_name => spec.list_function } unless spec.list_name.nil?),
+        spec.lib,
+        spec.language
+      ).query_view!(spec.view_parameters)
     end
 
     def load_document_without_caching(id)

data/lib/couch_potato/version.rb

@@ -1,4 +1,4 @@
 module CouchPotato
-  VERSION = '1.10.1'.freeze
+  VERSION = '1.11.0'.freeze
   RSPEC_VERSION = '4.0.2'.freeze
 end

data/spec/unit/database_spec.rb

@@ -430,11 +430,11 @@ describe CouchPotato::Database, '#view_in_batches' do
   let(:view_query) do
     instance_double(
       CouchPotato::View::ViewQuery,
-      query_view!: { 'rows' => [result] }
+      query_view!: { 'rows' => [] }
     )
   end
-  let(:result) { double('result') }
-  let(:spec) { double('view spec', process_results: [result]).as_null_object }
+  let(:processed_result) { double(:processed_result) }
+  let(:spec) { double('view spec', process_results: processed_result).as_null_object }
   let(:couchrest_db) { double('couchrest db').as_null_object }
   let(:db) { CouchPotato::Database.new(couchrest_db) }
 
@@ -443,22 +443,41 @@ describe CouchPotato::Database, '#view_in_batches' do
       .to receive_messages(new: view_query)
   end
 
-  it 'sets skip/limit for each batch' do
-    allow(spec).to receive(:process_results).and_return([result, result], [result]) # run twice
+  it 'sets no skip/startkey/startkey_docid for the first batch' do
     allow(spec).to receive(:view_parameters) { { key: 'x' } }
 
     expect(spec).to receive(:view_parameters=)
-      .with({key: 'x', skip: 0, limit: 2})
+      .with({key: 'x', limit: 2})
+
+    db.view_in_batches(spec, batch_size: 2) { |results| }
+  end
+
+  it 'sets skip/startkey/startkey_docid for each other batch' do
+    allow(spec).to receive(:view_parameters) { { key: 'x' } }
+    allow(view_query).to receive(:query_view!)
+      .and_return({'rows' => [{}, {'key' => 'k1', 'id' => 'id1'}]}, {'rows' => [{}]})
+    allow(spec).to receive(:view_parameters=)
+
     expect(spec).to receive(:view_parameters=)
-      .with({key: 'x', skip: 2, limit: 2})
+      .with({key: 'x', limit: 2, startkey: 'k1', startkey_docid: 'id1', skip: 1})
 
     db.view_in_batches(spec, batch_size: 2) { |results| }
   end
 
+  it 'yields processed results to the block' do
+    allow(view_query).to receive(:query_view!)
+      .and_return({'rows' => [{'key' => 'k1', 'id' => 'id1'}]})
+    allow(spec).to receive(:view_parameters=)
+
+    expect { |x| db.view_in_batches(spec, batch_size: 2, &x) }.to yield_with_args(processed_result)
+  end
+
   it 'yields batches until running out of data' do
-    allow(spec).to receive(:process_results).and_return([result, result], [result])
+    allow(view_query).to receive(:query_view!)
+      .and_return({'rows' => [{}, {}]}, {'rows' => [{}]})
+    allow(spec).to receive(:process_results).and_return([processed_result, processed_result], [processed_result])
 
-    expect { |b| db.view_in_batches(spec, batch_size: 2, &b) }.to yield_successive_args([result, result], [result])
+    expect { |b| db.view_in_batches(spec, batch_size: 2, &b) }.to yield_successive_args([processed_result, processed_result], [processed_result])
   end
 end
 

data/spec/views_spec.rb

@@ -405,4 +405,15 @@ describe 'views' do
       expect(@db.view(Build.timeline(stale: 'ok'))).to be_empty
     end
   end
+
+  describe 'view_in_batches' do
+    it 'yields docs in batches until all gone' do
+      build1 = Build.new(time: 1).tap {|b| @db.save!(b) }
+      build2 = Build.new(time: 2).tap {|b| @db.save!(b) }
+      build3 = Build.new(time: 3).tap {|b| @db.save!(b) }
+
+      expect {|block| @db.view_in_batches(Build.timeline, batch_size: 2, &block)}
+        .to yield_successive_args([build1, build2], [build3])
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: couch_potato
 version: !ruby/object:Gem::Version
-  version: 1.10.1
+  version: 1.11.0
 platform: ruby
 authors:
 - Alexander Lang
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-07 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel