jober 0.0.1 → 0.2

data/lib/jober/ar_loop.rb

@@ -0,0 +1,38 @@
+class Jober::ARLoop < Jober::Task
+  class << self
+    def batch_size(bs)
+      @batch_size = bs
+    end
+
+    def get_batch_size
+      @batch_size ||= 1000
+    end
+  end
+
+  def proxy
+    raise "implement me, should return AR.proxy, ex: User.where('years > 18')"
+  end
+
+  def run
+    prox = proxy
+
+    if @worker_id && @workers_count && !@opts[:no_auto_proxy]
+      prox = prox.where("id % #{@workers_count} = #{@worker_id}")
+    end
+
+    prox = prox.where(@opts[:where]) if @opts[:where]
+
+    cnt = 0
+    count = prox.count
+    info { "full count to process #{count}" }
+
+    prox.find_in_batches(:batch_size => self.class.get_batch_size) do |batch|
+      res = perform(batch)
+      cnt += batch.size
+      info { "process batch #{res.inspect}, #{cnt} from #{count}, lastid #{batch.last.id}" }
+      break if stopped
+    end
+
+    info { "processed total #{cnt}" }
+  end
+end

data/lib/jober/logger.rb

@@ -9,10 +9,26 @@ module Jober::Logger
     @logger = logger
   end
 
+  def logger_tag
+    @logger_tag ||= begin
+      tag = '[' + self.class.to_s
+      tag += " #{@worker_id}-#{@workers_count}" if @worker_id && @workers_count && @workers_count > 1
+      tag += ']'
+      tag
+    end
+  end
+
   Logger::Severity.constants.each do |level|
     method_name = level.to_s.downcase
-    define_method method_name do |msg = nil, &block|
-      logger.send(method_name, msg, &block)
-    end
+
+    class_eval <<-Q
+      def #{method_name}(msg = nil, &block)
+        if block
+          logger.send(:#{method_name}) { "\#{logger_tag} \#{block.call}" }
+        else
+          logger.send(:#{method_name}, "\#{logger_tag} \#{msg}", &block)
+        end
+      end
+    Q
   end
 end

data/lib/jober/manager.rb

@@ -21,11 +21,8 @@ class Jober::Manager
 
   def run!
     @allowed_classes.each do |klass|
-      workers = klass.workers
-      workers = [[5 * 60, :perform]] if workers.empty?
-
-      workers.each do |interval, method|
-        Thread.new { start_worker(klass, interval, method) }
+      klass.get_workers.times do |idx|
+        Thread.new { start_worker(klass, klass.get_interval, idx, klass.get_workers) }
       end
     end
   end
@@ -71,17 +68,16 @@ class Jober::Manager
     yield
     true
   rescue Object => ex
-    p "exception #{ex.inspect} #{ex.backtrace}"
     Jober.exception(ex)
     nil
   end
 
-  def start_worker(klass, interval, method)
-    debug { "start worker for #{klass.to_s} #{method}" }
+  def start_worker(klass, interval, idx, count)
+    debug { "start worker for #{klass.to_s}" }
     loop do
       pid = nil
       res = catch do
-        pid = run_task_fork(klass, method)
+        pid = run_task_fork(klass, idx, count)
         add_pid(pid)
         Process.wait(pid)
         del_pid(pid)
@@ -94,15 +90,15 @@ class Jober::Manager
     end
   end
 
-  def run_task_fork(klass, method)
-    info "invoke #{klass} #{method}"
+  def run_task_fork(klass, idx, count)
+    info "invoke #{klass}"
     fork do
       $0 = "#{@name} manager #{klass}"
       #$0 += " #{index}" if index > 0
       Jober.call_after_fork
       Jober.reset_redis
 
-      inst = klass.new # class_name parent of Jober::Task
+      inst = klass.new(:worker_id => idx, :workers_count => count) # class_name parent of Jober::Task
 
       if @logger_path
         logger_path = File.join(@logger_path, "#{klass.short_name}.log")
@@ -112,7 +108,9 @@ class Jober::Manager
         inst.logger = ::Logger.new(logger_path)
       end
 
-      inst.execute(method)
+      catch do
+        inst.execute
+      end
     end
   end
 

data/lib/jober/queue.rb

@@ -6,10 +6,11 @@ class Jober::Queue < Jober::Task
   end
 
   class << self
-    attr_accessor :queue_name
+    attr_accessor :queue_name, :queue_name_base
 
     def set_queue_name(q)
-      @queue_name = "Jober:queue:#{q}"
+      @queue_name_base = q
+      @queue_name = Jober.key("queue:#{q}")
     end
   end
 
@@ -18,22 +19,26 @@ class Jober::Queue < Jober::Task
   end
 
   def self.enqueue(*args)
-    Jober.redis.rpush(queue_name, Jober.dump(args))
+    Jober.redis.rpush(queue_name, Jober.dump_args(*args))
   end
 
   def self.len
     Jober.redis.llen(self.queue_name)
   end
 
+  def len
+    self.class.len
+  end
+
   def pop
     res = Jober.redis.lpop(queue_name)
     Jober.load(res) if res
   end
 
-  def run(method)
+  def run
     cnt = 0
     while args = pop
-      send(method, *args)
+      perform(*args)
       cnt += 1
 
       if stopped
@@ -42,6 +47,6 @@ class Jober::Queue < Jober::Task
 
       info { "processed #{cnt}" } if cnt % 1000 == 0
     end
-    info { "processed #{cnt}" }
+    info { "processed total #{cnt}" }
   end
 end

data/lib/jober/queue_batch.rb

@@ -1,5 +1,10 @@
-class Jober::QueueBatch < Jober::Queue
-  class << self
+module Jober::QueueBatchFeature
+  def self.included(base)
+    base.send(:extend, ClassMethods)
+    base.send(:include, InstanceMethods)
+  end
+
+  module ClassMethods
     def batch_size(bs)
       @batch_size = bs
     end
@@ -9,39 +14,47 @@ class Jober::QueueBatch < Jober::Queue
     end
   end
 
-  def execute(method = :perform)
-    cnt = 0
-    batch = []
-    while args = pop
-      batch << args
-      if batch.length >= self.class.get_batch_size
-        execute_batch(method, batch)
-        batch = []
+  module InstanceMethods
+    def run
+      cnt = 0
+      batch = []
+      while args = pop
+        batch << args
+        if batch.length >= self.class.get_batch_size
+          execute_batch(batch)
+          info { "execute batch #{batch.length}, #{cnt} from #{len}" }
+          batch = []
+        end
+        break if stopped
+        cnt += 1
       end
-      break if stopped
-      cnt += 1
-    end
-    if batch.length > 0
-      if stopped
-      	reschedule_batch(batch)
-      else
-        execute_batch(method, batch)
+      if batch.length > 0
+        if stopped
+          reschedule_batch(batch)
+        else
+          execute_batch(batch)
+        end
       end
+      info { "processes total #{cnt} " }
+      self
     end
-    self
-  end
 
-  def execute_batch(method, batch)
-    send(method, batch)
-  rescue Object => ex
-    reschedule_batch(batch)
-    Jober.exception(ex)
-  end
+    def execute_batch(batch)
+      perform(batch)
+    rescue Object => ex
+      reschedule_batch(batch)
+      Jober.exception(ex)
+    end
+
+  private
 
-private
+    def reschedule_batch(batch)
+      batch.reverse_each { |ev| Jober.redis.lpush(queue_name, Jober.dump(ev)) }
+    end
 
-  def reschedule_batch(batch)
-  	batch.reverse_each { |ev| Jober.redis.lpush(queue_name, Jober.dump(ev)) }
   end
+end
 
+class Jober::QueueBatch < Jober::Queue
+  include Jober::QueueBatchFeature
 end

data/lib/jober/shared_object.rb

@@ -1,16 +1,29 @@
 class Jober::SharedObject
   class << self
     def set(name, obj, conn = Jober.redis)
-      conn.set(key(name), Marshal.dump(obj))
+      Jober.catch do
+        conn.set(key(name), Marshal.dump(obj))
+      end
     end
 
     def get(name, conn = Jober.redis)
-      r = conn.get(key(name))
-      Marshal.load(r) if r
+      Jober.catch do
+        r = conn.get(key(name))
+        Marshal.load(r) if r
+      end
+    end
+
+    def get_by_pure_key(name, conn = Jober.redis)
+      Jober.catch do
+        r = conn.get(name)
+        Marshal.load(r) if r
+      end
     end
 
     def raw_get(name)
-      Jober.redis.get(key(name))
+      Jober.catch do
+        Jober.redis.get(key(name))
+      end
     end
 
     def [](name)
@@ -22,27 +35,42 @@ class Jober::SharedObject
     end
 
     def inc(name, by = 1)
-      Jober.redis.incrby(key(name), by)
+      Jober.catch do
+        Jober.redis.incrby(key(name), by)
+      end
     end
 
     def del(name)
-      Jober.redis.del key(name)
+      Jober.catch do
+        Jober.redis.del key(name)
+      end
     end
 
     def clear
-      Jober.redis.keys(key('*')).each { |k| del(k) }
+      Jober.catch do
+        Jober.redis.keys(key('*')).each { |k| Jober.redis.del(k) }
+      end
+    end
+
+    def keys(mask)
+      Jober.catch do
+        Jober.redis.keys(key(mask))
+      end
     end
 
     def values(mask)
-      Jober.redis.keys(key(mask)).map { |key| get(key) }
+      Jober.catch do
+        Jober.redis.keys(key(mask)).map { |key| get_by_pure_key(key) }
+      end
     end
 
     def key(name)
-      if name.start_with?('shared:')
+      k = if name.start_with?('shared:')
         name
       else
         "shared:#{name}"
       end
+      Jober.key(k)
     end
   end
 end

data/lib/jober/task.rb

@@ -1,7 +1,7 @@
 class Jober::Task < Jober::AbstractTask
 
   def self.extract_name(name)
-    Jober.underscore(name).gsub(/_?queue_?/, '').gsub(/_?fk_?/, '').gsub(/_?task_?/, '').split('::').last.gsub('/', '-')
+    Jober.underscore(name).gsub(/[_\/]?queue[_\/]?/, '').gsub(/[_\/]?task[_\/]?/, '').gsub(/[_\/]?jober[_\/]?/, '')
   end
 
   def self.inherited(base)
@@ -13,8 +13,8 @@ class Jober::Task < Jober::AbstractTask
     raise "implement me"
   end
 
-  def run(method)
-    send(method)
+  def run
+    perform
   end
 
 end

data/lib/jober/unique_queue.rb

@@ -5,7 +5,7 @@ class Jober::UniqueQueue < Jober::Queue
   end
 
   def self.enqueue(*args)
-    Jober.redis.sadd(queue_name, Jober.dump(args))
+    Jober.redis.sadd(queue_name, Jober.dump_args(*args))
   end
 
   def pop

data/lib/jober/unique_queue_batch.rb

@@ -0,0 +1,5 @@
+require "#{File.dirname(__FILE__)}/queue_batch"
+
+class Jober::UniqueQueueBatch < Jober::UniqueQueue
+  include Jober::QueueBatchFeature
+end

data/lib/jober/version.rb

@@ -1,3 +1,3 @@
 module Jober
-  VERSION = "0.0.1"
+  VERSION = "0.2"
 end

data/spec/ar_loop_spec.rb

@@ -0,0 +1,69 @@
+require "#{File.dirname(__FILE__)}/spec_helper"
+require "active_record"
+
+class MyAR < Jober::ARLoop
+  batch_size 10
+
+  def proxy
+    User.where("years > 18")
+  end
+
+  def perform(batch)
+    SO["names"] += batch.map(&:name)
+    batch.size
+  end
+end
+
+conn = { 'adapter' => 'sqlite3', 'database' => File.dirname(__FILE__) + "/test.db" }
+ActiveRecord::Base.establish_connection conn
+
+def pg_create_schema
+  ActiveRecord::Migration.verbose = false
+  ActiveRecord::Migration.create_table :users do |t|
+    t.string :name
+    t.integer :years
+  end
+end
+
+def pg_drop_data
+  ActiveRecord::Migration.verbose = false
+  ActiveRecord::Migration.drop_table :users
+end
+
+def create_data
+  100.times do |i|
+    User.create! :name => "unknown #{i}", :years => i % 36 + 1
+  end
+end
+
+class User < ActiveRecord::Base; end
+
+describe "ARLoop" do
+  before :all do
+    pg_drop_data rescue nil
+    pg_create_schema
+    create_data
+  end
+
+  before :each do
+    SO["names"] = []
+  end
+
+  it "should work" do
+    MyAR.new.execute
+    SO["names"].size.should == 46
+    SO["names"].last.should == "unknown 99"
+  end
+
+  it "use auto proxy sharding" do
+    MyAR.new(:worker_id => 1, :workers_count => 4).execute
+    SO["names"].size.should == 10
+    SO["names"].last.should == "unknown 96"
+  end
+
+  it "where" do
+    MyAR.new(:where => "years < 24").execute
+    SO["names"].size.should == 15
+    SO["names"].last.should == "unknown 94"
+  end
+end