couch_potato 1.10.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 699af51806a63fca2975c8eff17636647a5ca7c12f9131df7e30a5f843ab3028
-  data.tar.gz: 1e305be23ca0606bef97e7f89b06f166f694a7ec937a2641c782274dc9a8cf51
+  metadata.gz: fc65386f8f391b222c93733fbf83cb00050bc37b2fa1787be8d73e4847b2fa43
+  data.tar.gz: a2c6fe4017c646b3bceaa9baaa236044ffe286fd3380095296a602034511d817
 SHA512:
-  metadata.gz: bee39cf067a428f14b646948dc0ceea5e5b0bfde2f0eafd9e1ea025dbd5833bcfc8c328f8457aaa4ecb72aabf07a2e327f329296d1224dafb8c7967cf37d1eeb
-  data.tar.gz: 4f371624eb17e5d03fb25a13ae9375705f1b143fccb30c8e7cbfdeaeb98d8549b7cc29f9b7a7fe83ec0ac06abef1200670ea092aebd5e46bb7aa94265ee48191
+  metadata.gz: 5f7c933270982e090183aacdb7fdd52e76e2739476e7e4d87cbcad1239c62b4973f31ce62278f181c33612bc6c2dc8ee7abbc2ffca0bef4560b919b8a37359c8
+  data.tar.gz: 7f79c9a6b3c31f5f1fc0eea3f4a16da6e7cb6ace5a68232a9a127d7d056b5144d88714e12e128a64eba34a9377e073311408077958c14b818c0f4037853a59a5

data/.github/workflows/ruby.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.6, 2.7, "jruby"]
+        ruby: [2.7, '3.0', "jruby"]
         gemfile:
           - "active_support_5_0"
           - "active_support_5_1"
@@ -25,10 +25,16 @@ jobs:
           - "active_support_6_1"
           - "active_support_7_0"
         exclude:
-          - ruby: 2.6
-            gemfile: "active_support_7_0"
           - ruby: "jruby"
             gemfile: "active_support_7_0"
+          - ruby: "3.0"
+            gemfile: "active_support_5_0"
+          - ruby: "3.0"
+            gemfile: "active_support_5_1"
+          - ruby: "3.0"
+            gemfile: "active_support_5_2"
+          - ruby: "3.0"
+            gemfile: "active_support_6_0"
     steps:
       - uses: actions/checkout@v2
       - name: Set up CouchDB

data/CHANGES.md

@@ -1,5 +1,13 @@
 ## Changes
 
+# 1.11.0
+
+- improve view_in_batches performance by switching to using startkey_docid over skip
+
+### 1.10.1
+
+- support passing an empty array to CouchPotato::Database#load
+
 ### 1.10.0
 
 - add spec/support for Rails 7

data/couch_potato.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'json', '~> 2.3'
 
   s.add_development_dependency 'rake', '~>12.0'
-  s.add_development_dependency 'rspec', '~>3.5.0'
+  s.add_development_dependency 'rspec', '~>3.10.0'
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'tzinfo'
 

data/lib/couch_potato/database.rb

@@ -65,12 +65,27 @@ module CouchPotato
     # to a given block in batches of the given size, making multiple
     # requests with according skip/limit params sent to CouchDB.
     def view_in_batches(spec, batch_size: default_batch_size)
+      rows = nil
       batch = 0
       loop do
-        spec.view_parameters = spec.view_parameters.merge({ skip: batch * batch_size, limit: batch_size })
-        results = view(spec)
-        yield results
-        break if results.size < batch_size
+        spec.view_parameters = spec
+          .view_parameters
+          .merge({limit: batch_size})
+          .merge(
+            if rows
+              {
+                startkey: rows&.last&.dig('key'),
+                startkey_docid: rows&.last&.dig('id'),
+                skip: 1
+              }
+            else
+              {}
+            end
+          )
+        result = raw_view(spec)
+        rows = result['rows']
+        yield process_view_results(result, spec)
+        break if rows.size < batch_size
 
         batch += 1
       end
@@ -198,27 +213,34 @@ module CouchPotato
 
     def view_without_caching(spec)
       ActiveSupport::Notifications.instrument('couch_potato.view', name: "#{spec.design_document}/#{spec.view_name}") do
-        results = CouchPotato::View::ViewQuery.new(
-          couchrest_database,
-          spec.design_document,
-          { spec.view_name => {
-            map: spec.map_function,
-            reduce: spec.reduce_function
-          } },
-          ({ spec.list_name => spec.list_function } unless spec.list_name.nil?),
-          spec.lib,
-          spec.language
-        ).query_view!(spec.view_parameters)
-        processed_results = spec.process_results results
-        if processed_results.respond_to?(:database=)
-          processed_results.database = self
-        elsif processed_results.respond_to?(:each)
-          processed_results.each do |document|
-            document.database = self if document.respond_to?(:database=)
-          end
+        process_view_results(raw_view(spec), spec)
+      end
+    end
+
+    def process_view_results(results, spec)
+      processed_results = spec.process_results results
+      if processed_results.respond_to?(:database=)
+        processed_results.database = self
+      elsif processed_results.respond_to?(:each)
+        processed_results.each do |document|
+          document.database = self if document.respond_to?(:database=)
         end
-        processed_results
       end
+      processed_results
+    end
+
+    def raw_view(spec)
+      CouchPotato::View::ViewQuery.new(
+        couchrest_database,
+        spec.design_document,
+        { spec.view_name => {
+          map: spec.map_function,
+          reduce: spec.reduce_function
+        } },
+        ({ spec.list_name => spec.list_function } unless spec.list_name.nil?),
+        spec.lib,
+        spec.language
+      ).query_view!(spec.view_parameters)
     end
 
     def load_document_without_caching(id)
@@ -269,6 +291,8 @@ module CouchPotato
     end
 
     def bulk_load(ids)
+      return [] if ids.empty?
+
       response = couchrest_database.bulk_load ids
       docs = response['rows'].map { |row| row['doc'] }.compact
       docs.each do |doc|

data/lib/couch_potato/version.rb

@@ -1,4 +1,4 @@
 module CouchPotato
-  VERSION = '1.10.0'.freeze
+  VERSION = '1.11.0'.freeze
   RSPEC_VERSION = '4.0.2'.freeze
 end

data/spec/railtie_spec.rb

@@ -87,7 +87,7 @@ describe "railtie" do
     it 'assigns additional_databases to config' do
       allow(File).to receive_messages(:read => "test:\n  database: test\n  additional_databases:\n    db2: test2")
 
-      expect(CouchPotato::Config).to receive(:additional_databases=).with('db2' => 'test2')
+      expect(CouchPotato::Config).to receive(:additional_databases=).with({'db2' => 'test2'})
 
       CouchPotato.rails_init
     end

data/spec/unit/database_spec.rb

@@ -98,6 +98,10 @@ describe CouchPotato::Database, 'load' do
 
       db.load(['1'])
     end
+
+    it 'returns an empty array when passing an empty array' do
+      expect(db.load([])).to eq([])
+    end
   end
 end
 
@@ -426,11 +430,11 @@ describe CouchPotato::Database, '#view_in_batches' do
   let(:view_query) do
     instance_double(
       CouchPotato::View::ViewQuery,
-      query_view!: { 'rows' => [result] }
+      query_view!: { 'rows' => [] }
     )
   end
-  let(:result) { double('result') }
-  let(:spec) { double('view spec', process_results: [result]).as_null_object }
+  let(:processed_result) { double(:processed_result) }
+  let(:spec) { double('view spec', process_results: processed_result).as_null_object }
   let(:couchrest_db) { double('couchrest db').as_null_object }
   let(:db) { CouchPotato::Database.new(couchrest_db) }
 
@@ -439,22 +443,41 @@ describe CouchPotato::Database, '#view_in_batches' do
       .to receive_messages(new: view_query)
   end
 
-  it 'sets skip/limit for each batch' do
-    allow(spec).to receive(:process_results).and_return([result, result], [result]) # run twice
+  it 'sets no skip/startkey/startkey_docid for the first batch' do
     allow(spec).to receive(:view_parameters) { { key: 'x' } }
 
     expect(spec).to receive(:view_parameters=)
-      .with(key: 'x', skip: 0, limit: 2)
+      .with({key: 'x', limit: 2})
+
+    db.view_in_batches(spec, batch_size: 2) { |results| }
+  end
+
+  it 'sets skip/startkey/startkey_docid for each other batch' do
+    allow(spec).to receive(:view_parameters) { { key: 'x' } }
+    allow(view_query).to receive(:query_view!)
+      .and_return({'rows' => [{}, {'key' => 'k1', 'id' => 'id1'}]}, {'rows' => [{}]})
+    allow(spec).to receive(:view_parameters=)
+
     expect(spec).to receive(:view_parameters=)
-      .with(key: 'x', skip: 2, limit: 2)
+      .with({key: 'x', limit: 2, startkey: 'k1', startkey_docid: 'id1', skip: 1})
 
     db.view_in_batches(spec, batch_size: 2) { |results| }
   end
 
+  it 'yields processed results to the block' do
+    allow(view_query).to receive(:query_view!)
+      .and_return({'rows' => [{'key' => 'k1', 'id' => 'id1'}]})
+    allow(spec).to receive(:view_parameters=)
+
+    expect { |x| db.view_in_batches(spec, batch_size: 2, &x) }.to yield_with_args(processed_result)
+  end
+
   it 'yields batches until running out of data' do
-    allow(spec).to receive(:process_results).and_return([result, result], [result])
+    allow(view_query).to receive(:query_view!)
+      .and_return({'rows' => [{}, {}]}, {'rows' => [{}]})
+    allow(spec).to receive(:process_results).and_return([processed_result, processed_result], [processed_result])
 
-    expect { |b| db.view_in_batches(spec, batch_size: 2, &b) }.to yield_successive_args([result, result], [result])
+    expect { |b| db.view_in_batches(spec, batch_size: 2, &b) }.to yield_successive_args([processed_result, processed_result], [processed_result])
   end
 end
 

data/spec/unit/view_query_spec.rb

@@ -15,10 +15,12 @@ describe CouchPotato::View::ViewQuery, 'query_view!' do
 
   it 'updates a view if it does not exist' do
     expect(db).to receive(:save_doc).with(
-      'views' => {'view' => {'map' => '<map_code>', 'reduce' => '<reduce_code>'}, 'lib' => {'test' => '<lib_code>'}},
-      'lists' => {},
-      "_id" => "_design/design",
-      "language" => "javascript"
+      {
+        'views' => {'view' => {'map' => '<map_code>', 'reduce' => '<reduce_code>'}, 'lib' => {'test' => '<lib_code>'}},
+        'lists' => {},
+        "_id" => "_design/design",
+        "language" => "javascript"
+        }
     )
 
     CouchPotato::View::ViewQuery.new(db, 'design', {:view => {:map => '<map_code>', :reduce => '<reduce_code>'}}, nil, {'test' => "<lib_code>"}).query_view!
@@ -46,8 +48,11 @@ describe CouchPotato::View::ViewQuery, 'query_view!' do
 
   it 'updates a view in erlang if it does not exist' do
     expect(db).to receive(:save_doc).with(
-      'views' => {'view' => {'map' => '<map_code>', 'reduce' => '<reduce_code>'}},
-      'lists' => {}, "_id" => "_design/design", "language" => "erlang")
+      {
+        'views' => {'view' => {'map' => '<map_code>', 'reduce' => '<reduce_code>'}},
+        'lists' => {}, "_id" => "_design/design", "language" => "erlang"
+      }
+    )
 
     CouchPotato::View::ViewQuery.new(db, 'design',
       {:view => {:map => '<map_code>', :reduce => '<reduce_code>'}},
@@ -120,7 +125,7 @@ describe CouchPotato::View::ViewQuery, 'query_view!' do
 
   it 'does not pass in reduce or lib keys if there is no lib or reduce object' do
     allow(db).to receive(:get).and_return({'views' => {}})
-    expect(db).to receive(:save_doc).with('views' => {'view7' => {'map' => '<map code>'}})
+    expect(db).to receive(:save_doc).with({'views' => {'view7' => {'map' => '<map code>'}}})
     CouchPotato::View::ViewQuery.new(db, 'design', :view7 => {:map => '<map code>', :reduce => nil}).query_view!
   end
 

data/spec/views_spec.rb

@@ -405,4 +405,15 @@ describe 'views' do
       expect(@db.view(Build.timeline(stale: 'ok'))).to be_empty
     end
   end
+
+  describe 'view_in_batches' do
+    it 'yields docs in batches until all gone' do
+      build1 = Build.new(time: 1).tap {|b| @db.save!(b) }
+      build2 = Build.new(time: 2).tap {|b| @db.save!(b) }
+      build3 = Build.new(time: 3).tap {|b| @db.save!(b) }
+
+      expect {|block| @db.view_in_batches(Build.timeline, batch_size: 2, &block)}
+        .to yield_successive_args([build1, build2], [build3])
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: couch_potato
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Alexander Lang
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-28 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -78,14 +78,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 3.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 3.10.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -219,7 +219,7 @@ files:
 homepage: http://github.com/langalex/couch_potato
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -234,8 +234,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: Ruby persistence layer for CouchDB
 test_files: