caster 0.9.3 → 0.9.4

data/lib/caster.rb

@@ -13,7 +13,8 @@ module Caster
         :database => nil,
         :id_prefix => 'caster',
         :type => 'caster_metadoc'
-    }
+    },
+    :batch_size => 2000,
   }
 
   @valid_config_keys = @config.keys
@@ -48,4 +49,4 @@ module Caster
   def self.log
     @logger
   end
-end
+end

data/lib/caster/execution.rb

@@ -47,19 +47,51 @@ module Caster
 
     def from scope, query = {}
       if scope.scan('/').length == 1
-        return Reference.new @db, scope, query
+        Reference.new @db, scope, query
       else
         database_name, view = scope.split('/', 2)
         db = CouchRest.database "http://#{Caster.config[:host]}:#{Caster.config[:port]}/#{database_name}"
-        return Reference.new db, view, query
+        Reference.new db, view, query
       end
     end
 
     def execute
       Caster.log.info "executing query on '#{@db.name}' over '#{@view}' with params #{@query.inspect}"
-      rdocs = @db.view(@view, @query)['rows']
+
+      limit = @query['limit'] || 1.0/0.0
+      if Caster.config[:batch_size] == nil or limit < Caster.config[:batch_size]
+        execute_batch @db.view(@view, @query)['rows']
+        return
+      end
+
+      @query['limit'] = Caster.config[:batch_size] + 1
+      saved_docs = 0
+      while saved_docs < limit do
+        docs = @db.view(@view, @query)['rows']
+        return if docs.length == 0
+
+        @query['startkey_docid'] = docs.last['id']
+        @query['startkey'] = docs.last['key']
+
+        if docs.length <= Caster.config[:batch_size]
+          execute_batch docs
+          return
+        elsif saved_docs + docs.length - 1 > limit
+          execute_batch docs.slice(0, limit - saved_docs)
+          return
+        else
+          docs.pop
+          execute_batch docs
+          saved_docs += docs.length
+        end
+      end
+    end
+
+    private
+    def execute_batch docs
       db_docs_map = Hash.new { |k, v| k[v] = [] }
-      rdocs.each do |rdoc|
+
+      docs.each do |rdoc|
         doc = rdoc.has_key?('doc')? rdoc['doc'] : rdoc['value']
 
         @operations = []
@@ -68,9 +100,10 @@ module Caster
           db_docs_map[op.db_handle] << op.transformation.execute(doc)
         end
       end
-      db_docs_map.each do |db, docs|
-        db.bulk_save docs
+      db_docs_map.each do |db, db_docs|
+        db.bulk_save db_docs
       end
     end
+
   end
-end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: caster
 version: !ruby/object:Gem::Version 
-  hash: 61
+  hash: 51
   prerelease: 
   segments: 
   - 0
   - 9
-  - 3
-  version: 0.9.3
+  - 4
+  version: 0.9.4
 platform: ruby
 authors: 
 - Manohar Akula
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-01-10 00:00:00 Z
+date: 2013-01-13 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: couchrest