jober 0.2 → 0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b38442d4fe738b159e68bb21a37d42d6343058ab
-  data.tar.gz: 3d7ea0e3cd642cfa93545dd486e296c3b4f2a150
+  metadata.gz: ddcfe15a02c6711edc74697daa7f399569561ee1
+  data.tar.gz: 7d1fb585d8bd3e1307231d2a4eb7944102cd9fac
 SHA512:
-  metadata.gz: c8bb7b73c5725f34b19934b602026f91914b9cf666a5688ebcf249ea8fb4fd7ac9b0598700fba3e44a830476d335f9c2260c087fd5e79f7aa1be2653d3b65980
-  data.tar.gz: 867f8e4a94b2f591c6ebc2f7fd9ff0f251c6ff0fadd8db3ae9ff7c5f8984f34ca06fd71856b87bced9b6ebf786c444af2d762c54c803e988c716d319fd399cfa
+  metadata.gz: 274b9cf0c10120c4e0b0f20ef198b54d6d449c9e0be5248a88c59bc0e05d21adeff4aa33a54d923c3ecd6964b319b0e8ec70cc5d7d76a2c95480a82c80e787ee
+  data.tar.gz: c90c327dfa205e59dcf083d07c040e0d12a102a2fc0fcfec6705ebd0fd13dfc4b426df64a329bd7ba0ce007c95ffc6ae4698c33a4625b7537084f827cdb3b7b6

data/.travis.yml

@@ -2,11 +2,8 @@ language: ruby
 services:
   - redis-server
 rvm:
-  - "ree"
-  - "1.8.7"
   - "1.9.3"
   - "2.0.0"
   - "2.1"
   - "2.2"
   - "rbx"
-  - "jruby"

data/bench/queue.rb

@@ -13,7 +13,13 @@ end
 Jober.logger = Logger.new nil
 
 t = Time.now
-50000.times { |i| Bench.enqueue(i, -i / 2.0) }
+threads = []
+5.times do |ti|
+  threads << Thread.new do
+    10000.times { |i| Bench.enqueue(i + ti * 10000, -(i + ti * 10000 ) / 2.0) }
+  end
+end
+threads.map(&:join)
 puts Time.now - t
 puts Bench.len
 

data/examples/tman.rb

@@ -0,0 +1,6 @@
+require 'bundler/setup'
+Bundler.require :default
+require_relative 'classes'
+
+man = Jober::ThreadedManager.new
+man.run_loop

data/lib/jober.rb

@@ -5,6 +5,7 @@ require 'logger'
 
 module Jober
   autoload :Manager,          'jober/manager'
+  autoload :ThreadedManager,  'jober/threaded_manager'
   autoload :AbstractTask,     'jober/abstract_task'
   autoload :Task,             'jober/task'
   autoload :Queue,            'jober/queue'
@@ -14,6 +15,7 @@ module Jober
   autoload :SharedObject,     'jober/shared_object'
   autoload :UniqueQueueBatch, 'jober/unique_queue_batch'
   autoload :ARLoop,           'jober/ar_loop'
+  autoload :Exception,        'jober/exception'
 
   class << self
     def logger
@@ -25,22 +27,31 @@ module Jober
     end
 
     def redis
-      @redis ||= Redis.new
+      Thread.current[:__jober_redis__] ||= (@redis || Redis.new).dup
     end
 
     def redis=(r)
       @redis = r
+      reset_redis
+    end
+
+    def reset_redis
+      Thread.current[:__jober_redis__] = nil
     end
 
     def internal_classes_names
-      @internal_classes_names ||= (%w{Manager AbstractTask Task Queue} +
-        %w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop}).map { |k| "Jober::#{k}" }
+      @internal_classes_names ||= (%w{Manager ThreadedManager AbstractTask Task Queue} +
+        %w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop Exception}).map { |k| "Jober::#{k}" }
     end
 
     def classes
       @classes ||= []
     end
 
+    def names
+      classes.map { |k| underscore(k.to_s.gsub('Jober::', '')) }
+    end
+
     def add_class(klass)
       classes << klass unless internal_classes_names.include?(klass.to_s)
     end
@@ -53,10 +64,6 @@ module Jober
       @after_fork.call if @after_fork
     end
 
-    def reset_redis
-      redis.client.reconnect
-    end
-
     def dump(obj)
       Marshal.dump(obj)
     end

data/lib/jober/abstract_task.rb

@@ -2,6 +2,7 @@ require 'timeout'
 
 class Jober::AbstractTask
   include Jober::Logger
+  include Jober::Exception
 
   class << self
     def interval(interval)
@@ -24,6 +25,7 @@ class Jober::AbstractTask
   end
 
   attr_accessor :stopped
+  attr_reader :worker_id, :workers_count
 
   def self.inherited(base)
     Jober.add_class(base)
@@ -94,6 +96,10 @@ class Jober::AbstractTask
   rescue Timeout::Error
   end
 
+  def stop!
+    @stopped = true
+  end
+
 private
 
   def self.timestamp_key(type)
@@ -133,4 +139,21 @@ private
     end
   end
 
+  def store_key(name)
+    Jober.key("store:#{self.class.short_name}-#{self.worker_id}-#{self.workers_count}:#{name}")
+  end
+
+  def set_store(name, obj)
+    self.catch do
+      Jober.redis.set(store_key(name), Jober.dump(obj))
+    end
+  end
+
+  def get_store(name)
+    self.catch do
+      r = Jober.redis.get(store_key(name))
+      Jober.load(r) if r
+    end
+  end
+
 end

data/lib/jober/ar_loop.rb

@@ -16,8 +16,15 @@ class Jober::ARLoop < Jober::Task
   def run
     prox = proxy
 
-    if @worker_id && @workers_count && !@opts[:no_auto_proxy]
-      prox = prox.where("id % #{@workers_count} = #{@worker_id}")
+    if @worker_id && @workers_count && @workers_count > 1 && !@opts[:no_auto_proxy]
+      cond = "id % #{@workers_count} = #{@worker_id}"
+      prox = prox.where(cond)
+      info { "sharding enabled '#{cond}'" }
+    end
+
+    last_batch_id = if (_last_batch_id = get_store("lastbatch")) && !@opts[:no_last_batch]
+      info { "found last batch id #{last_batch_id} so start with it!" }
+      _last_batch_id
     end
 
     prox = prox.where(@opts[:where]) if @opts[:where]
@@ -26,13 +33,20 @@ class Jober::ARLoop < Jober::Task
     count = prox.count
     info { "full count to process #{count}" }
 
-    prox.find_in_batches(:batch_size => self.class.get_batch_size) do |batch|
+    h = {:batch_size => self.class.get_batch_size}
+    h[:start] = last_batch_id + 1 if last_batch_id
+    prox.find_in_batches(h) do |batch|
       res = perform(batch)
       cnt += batch.size
       info { "process batch #{res.inspect}, #{cnt} from #{count}, lastid #{batch.last.id}" }
+      set_store("lastbatch", batch.last.id)
       break if stopped
     end
 
     info { "processed total #{cnt}" }
   end
+
+  def reset_last_batch_id
+    Jober.redis.del(store_key('lastbatch'))
+  end
 end

data/lib/jober/exception.rb

@@ -0,0 +1,16 @@
+module Jober::Exception
+
+  def exception(ex)
+    msg = self.respond_to?(:logger_tag) ? "#{self.logger_tag} #{ex.message}" : ex.message
+    ex2 = ex.class.new(msg)
+    ex2.set_backtrace(ex.backtrace)
+    Jober.exception(ex2)
+  end
+
+  def catch(&block)
+    yield
+  rescue Object => ex
+    exception(ex)
+    nil
+  end
+end

data/lib/jober/manager.rb

@@ -81,7 +81,7 @@ class Jober::Manager
         add_pid(pid)
         Process.wait(pid)
         del_pid(pid)
-        sleep interval unless stopped
+        sleep interval.to_f unless stopped
       end
       del_pid(pid)
       break if stopped

data/lib/jober/queue_batch.rb

@@ -43,7 +43,7 @@ module Jober::QueueBatchFeature
       perform(batch)
     rescue Object => ex
       reschedule_batch(batch)
-      Jober.exception(ex)
+      exception(ex)
     end
 
   private

data/lib/jober/threaded_manager.rb

@@ -0,0 +1,85 @@
+class Jober::ThreadedManager
+  include Jober::Logger
+  include Jober::Exception
+
+  def default_sleep=(ds)
+    @default_sleep = ds
+  end
+
+  def default_sleep
+    @default_sleep ||= 10
+  end
+
+  def initialize(klasses = nil, opts = {})
+    @klasses = klasses || Jober.classes
+    @stopped = false
+    @objects = @klasses.map do |klass|
+      if klass.is_a?(String)
+        klass_str = klass
+        klass = Jober.find_class(klass_str)
+        raise "unknown class #{klass_str}" unless klass
+      end
+      klass.new(opts)
+    end
+  end
+
+  def run_loop
+    info { "run loop for #{@klasses}, in threads: #{@objects.length}" }
+    @threads = @objects.map { |obj| make_thread(obj) }
+
+    # set signals
+    trap("INT") { @stopped = true }
+    trap("QUIT") { @stopped = true }
+
+    # sleeping infinitely
+    sleeping
+
+    info { "prepare quit ..." }
+
+    # send stop to all objects
+    @objects.each(&:stop!)
+
+    # sleep a little to give time for threads to quit
+    wait_for_kill(default_sleep.to_f)
+
+    info { "quit!" }
+
+    # kill all threads, if they still alive
+    @threads.select(&:alive?).each(&:kill)
+  end
+
+  def stop!
+    @stopped = true
+  end
+
+private
+
+  def sleeping
+    loop do
+      sleep 0.5
+      return if @stopped
+    end
+  end
+
+  def wait_for_kill(interval)
+    info { "waiting quiting jobs for %.1fm ..." % [interval / 60.0] }
+    Timeout.timeout(interval.to_f) do
+      loop do
+        sleep 0.3
+        return if @threads.none? &:alive?
+      end
+    end
+  rescue Timeout::Error
+  end
+
+  def make_thread(obj)
+    Thread.new do
+      loop do
+        break if @stopped
+        obj.catch { obj.run_loop }
+        break if @stopped
+        sleep 1.0
+      end
+    end
+  end
+end

data/lib/jober/version.rb

@@ -1,3 +1,3 @@
 module Jober
-  VERSION = "0.2"
+  VERSION = "0.3"
 end

data/spec/ar_loop_spec.rb

@@ -14,6 +14,20 @@ class MyAR < Jober::ARLoop
   end
 end
 
+class MyAR2 < Jober::ARLoop
+  batch_size 10
+
+  def proxy
+    User.where("years > 18")
+  end
+
+  def perform(batch)
+    sleep 1
+    SO["names"] += batch.map(&:name)
+    batch.size
+  end
+end
+
 conn = { 'adapter' => 'sqlite3', 'database' => File.dirname(__FILE__) + "/test.db" }
 ActiveRecord::Base.establish_connection conn
 
@@ -66,4 +80,39 @@ describe "ARLoop" do
     SO["names"].size.should == 15
     SO["names"].last.should == "unknown 94"
   end
+
+  it "should use lastbatch" do
+    my = MyAR2.new
+    Thread.new { my.execute }
+    sleep 1.5
+    my.stop!
+    sleep 0.6
+
+    SO["names"].size.should == 20
+    SO["names"].last.should == "unknown 55"
+
+    # should start from last ID
+    my = MyAR2.new
+    my.execute
+    SO["names"].size.should == 46
+    SO["names"].last.should == "unknown 99"
+  end
+
+  it "should not use lastbatch, if it was dropped" do
+    my = MyAR2.new
+    Thread.new { my.execute }
+    sleep 1.5
+    my.stop!
+    sleep 0.6
+
+    SO["names"].size.should == 20
+    SO["names"].last.should == "unknown 55"
+
+    # should start from last ID
+    my = MyAR2.new
+    my.reset_last_batch_id
+    my.execute
+    SO["names"].size.should == 66
+  end
+
 end

data/spec/spec_helper.rb

@@ -33,3 +33,20 @@ def run_manager_for(timeout, classes, &block)
   sleep 0.1
   m
 end
+
+def run_threaded_manager_for(timeout, classes, &block)
+  m = Jober::ThreadedManager.new classes
+  t = Thread.new do
+    m.run_loop
+  end
+
+  if block
+    block.call(m)
+  else
+    sleep timeout
+  end
+
+  m.stop!
+  t.join
+  m
+end

data/spec/threaded_manager_spec.rb

@@ -0,0 +1,94 @@
+require "#{File.dirname(__FILE__)}/spec_helper"
+
+class T1 < Jober::Task
+  def perform
+    SO["t1"] += 1
+  end
+end
+
+class T2 < Jober::Task
+  def perform
+    SO["t2"] += 1
+  end
+end
+
+class T3 < Jober::Task
+  def perform
+    SO["t3"] += 1
+  end
+end
+
+class T4 < Jober::Task
+  def perform
+    loop do
+      sleep 0.1
+      break if stopped
+    end
+    SO["t4"] = "yahoo"
+  end
+end
+
+class T41 < Jober::Task
+  interval 1
+
+  def perform
+    sleep 1
+    SO["t41"] += 1
+  end
+end
+
+class T5 < Jober::Task
+  def perform
+    SO["t5"] += 1
+    1 + "bla" # raising
+  end
+end
+
+class T6 < Jober::Task
+  def perform
+    sleep
+  end
+end
+
+describe Jober::ThreadedManager do
+  before :each do
+    SO["t1"] = 0
+    SO["t2"] = 0
+    SO["t3"] = 0
+    SO["t41"] = 0
+    SO["t5"] = 0
+  end
+
+  it "just work" do
+    run_threaded_manager_for(0.1, [T1])
+    SO["t1"].should == 1
+  end
+
+  xit "run multiple identical tasks" do
+    run_threaded_manager_for(0.2, [T1, T1, T1])
+    SO["t"].should == 3
+  end
+
+  it "run multiple tasks" do
+    run_threaded_manager_for(0.2, [T1, T2, T3])
+    SO["t1"].should == 1
+    SO["t2"].should == 1
+    SO["t3"].should == 1
+  end
+
+  it "task should be stopped by stopped flag" do
+    run_threaded_manager_for(0.5, [T4])
+    SO["t4"].should == "yahoo"
+  end
+
+  it "run multi tasks, but one raised" do
+    run_threaded_manager_for(2, [T41, T5])
+    SO["t5"].should == 2
+  end
+
+  it "infinite task should be stopped by kill" do
+    run_threaded_manager_for(2.5, [T6, T41])
+    SO["t41"].should == 2
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jober
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.3'
 platform: ruby
 authors:
 - "'Konstantin Makarchev'"
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-27 00:00:00.000000000 Z
+date: 2015-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -114,16 +114,19 @@ files:
 - bin/jober_manager
 - examples/classes.rb
 - examples/man.rb
+- examples/tman.rb
 - jober.gemspec
 - lib/jober.rb
 - lib/jober/abstract_task.rb
 - lib/jober/ar_loop.rb
+- lib/jober/exception.rb
 - lib/jober/logger.rb
 - lib/jober/manager.rb
 - lib/jober/queue.rb
 - lib/jober/queue_batch.rb
 - lib/jober/shared_object.rb
 - lib/jober/task.rb
+- lib/jober/threaded_manager.rb
 - lib/jober/unique_queue.rb
 - lib/jober/unique_queue_batch.rb
 - lib/jober/version.rb
@@ -142,6 +145,7 @@ files:
 - spec/stats_spec.rb
 - spec/task_loop_spec.rb
 - spec/task_spec.rb
+- spec/threaded_manager_spec.rb
 - spec/unique_queue_batch_spec.rb
 - spec/unique_queue_spec.rb
 homepage: ''
@@ -184,5 +188,6 @@ test_files:
 - spec/stats_spec.rb
 - spec/task_loop_spec.rb
 - spec/task_spec.rb
+- spec/threaded_manager_spec.rb
 - spec/unique_queue_batch_spec.rb
 - spec/unique_queue_spec.rb