parallel_batch 0.0.1 → 0.0.2

data/README

@@ -1,4 +1,45 @@
-ParallelBatch runs concurrent batches on tables with huge amount of rows.
+Parallel batch distributes your batches across servers.
+
+It is based on ActiveRecord and relies on the database to ensure records are processed only once.
+
+To get started include Parallel batch into your Gemfile and run bundle install.
+
+  gem 'parallel_batch'
+
+Then run the following migration to create the parallel_batches table.
+
+  class CreateParallelBatches < ActiveRecord::Migration
+    def self.up
+      create_table :parallel_batches do |t|
+        t.string :type
+        t.string :offset # It works either with integer or string IDs.
+        t.timestamps
+      end
+
+      add_index :parallel_batches, :type, :unique: true
+    end
+
+    def self.down
+      drop_table :parallel_batches
+    end
+  end
+
+Your parallel batch must be a sub class of ParallelBatch. It must implement scope and perform methods. Perform is called for each record selected by your defined scope.
+
+    # File: app/models/track_batch.rb
+    class TrackBatch < ParallelBatch
+      def scope
+        Track.not_encoded
+      end
+
+      def perform(track)
+        track.encode_mp3
+      end
+    end
+
+For starting your batches SSH into your servers and run the following command.
+
+    TrackBatch.start(5) # Create 5 workers in their own process.
 
 To Contribute:
 

data/lib/parallel_batch.rb

@@ -2,36 +2,31 @@
 
 class ParallelBatch < ActiveRecord::Base
 
-  #################
-  ### Constants ###
-  #################
-
-  VERSION = "0.0.1"
-
   #####################
   ### Class methods ###
   #####################
 
   def self.find_or_create!
-    first || create!(offset: 0)
+    first || create!
   # When starting many batches at the same time we are pretty sure to get a MySQL
   # error reporting a duplicated entry. That's why we are retrying one time only.
   rescue ActiveRecord::StatementInvalid
-    first || create!(offset: 0)
+    first || create!
   end
 
   def self.start(concurrency = 1)
-    concurrency.times { fork { start_fork } }
+    concurrency.times { Process.detach(fork { start_fork }) }
   end
 
   def self.start_fork
     puts "#{self} has started with pid #{Process.pid}"
     ActiveRecord::Base.connection.reconnect!
+    Process.daemon(false)
     find_or_create!.run
   end
 
   def self.reset
-    find_or_create!.update_attributes!(offset: null)
+    find_or_create!.update_attributes!(offset: nil)
   end
 
   ########################
@@ -53,7 +48,7 @@ class ParallelBatch < ActiveRecord::Base
 
   def run
     while records = next_batch
-      records.each { |record| perform(record) }
+      records.each { |record| perform(record) rescue nil }
     end
   end
 
@@ -66,7 +61,7 @@ class ParallelBatch < ActiveRecord::Base
   end
 
   def batch_size
-    1000
+    100
   end
   
 end

data/lib/parallel_batch/version.rb

@@ -0,0 +1,5 @@
+# -*- encoding: utf-8 -*-
+
+class ParallelBatch
+  VERSION = '0.0.2'.freeze
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallel_batch
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-17 00:00:00.000000000 Z
+date: 2012-11-02 00:00:00.000000000 Z
 dependencies: []
 description: Run safely concurent batches
 email:
@@ -24,6 +24,7 @@ files:
 - README
 - Rakefile
 - lib/parallel_batch.rb
+- lib/parallel_batch/version.rb
 - parallel_batch.gemspec
 - test/parallel_batch_test.rb
 homepage: https://github.com/officialfm/parallel_batch