jober 0.3 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ddcfe15a02c6711edc74697daa7f399569561ee1
-  data.tar.gz: 7d1fb585d8bd3e1307231d2a4eb7944102cd9fac
+  metadata.gz: 9dda52df02d094768a2a3dd014f9a26f2858cdd1
+  data.tar.gz: 15dc813ba49402330112935dffee45b5e9d454c5
 SHA512:
-  metadata.gz: 274b9cf0c10120c4e0b0f20ef198b54d6d449c9e0be5248a88c59bc0e05d21adeff4aa33a54d923c3ecd6964b319b0e8ec70cc5d7d76a2c95480a82c80e787ee
-  data.tar.gz: c90c327dfa205e59dcf083d07c040e0d12a102a2fc0fcfec6705ebd0fd13dfc4b426df64a329bd7ba0ce007c95ffc6ae4698c33a4625b7537084f827cdb3b7b6
+  metadata.gz: b15bae2a5b73f36802a68b857079a98568f84da046fdbcb2cd562d0dd2bfcf209a9cebd45d680138c6bbdb373572ad80a1b3d579324ce9f228a9f7151f2b1629
+  data.tar.gz: 15fc13a7c10848a56b426928aaead2c0277ffe13696703e6b02a146b7212682c5f6b5d5f55a7b7848ed1b80086f0dc387fbc7c5136c92896527a714a56e9e860

data/jober.gemspec

@@ -20,9 +20,9 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'redis'
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "activerecord"
-  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "activerecord", '~> 3.2'
+  spec.add_development_dependency "sqlite3-ruby"
+  spec.add_development_dependency "i18n", '~> 0.6.0'
 end

data/lib/jober/abstract_task.rb

@@ -24,8 +24,7 @@ class Jober::AbstractTask
     attr_accessor :short_name
   end
 
-  attr_accessor :stopped
-  attr_reader :worker_id, :workers_count
+  attr_reader :finished, :stopped, :worker_id, :workers_count
 
   def self.inherited(base)
     Jober.add_class(base)
@@ -53,7 +52,13 @@ class Jober::AbstractTask
     run
     self.class.write_timestamp(:finished)
     self.class.del_timestamp(:crashed)
+    if @stopped
+      self.class.write_timestamp(:stopped)
+    else
+      self.class.del_timestamp(:stopped)
+    end
     info "<= end (in #{Time.now - @start_at})"
+    @finished = true
     self
   rescue Object
     self.class.write_timestamp(:crashed)
@@ -68,7 +73,8 @@ class Jober::AbstractTask
         (finished = self.class.read_timestamp(:finished)) &&
         (Time.now - finished < self.class.get_interval) &&
         !self.class.pop_skip_delay_flag! &&
-        !@skip_delay
+        !@skip_delay &&
+        !self.class.read_timestamp(:stopped)
 
       sleeping(self.class.get_interval - (Time.now - finished))
     end
@@ -98,6 +104,7 @@ class Jober::AbstractTask
 
   def stop!
     @stopped = true
+    self.class.write_timestamp(:stopped)
   end
 
 private
@@ -156,4 +163,8 @@ private
     end
   end
 
+  def self.stats
+    Jober.stats[self.short_name]
+  end
+
 end

data/lib/jober/ar_loop.rb

@@ -16,34 +16,40 @@ class Jober::ARLoop < Jober::Task
   def run
     prox = proxy
 
+    prefix = ''
+
     if @worker_id && @workers_count && @workers_count > 1 && !@opts[:no_auto_proxy]
       cond = "id % #{@workers_count} = #{@worker_id}"
       prox = prox.where(cond)
+      prefix += "(#{cond}) "
       info { "sharding enabled '#{cond}'" }
     end
 
     last_batch_id = if (_last_batch_id = get_store("lastbatch")) && !@opts[:no_last_batch]
-      info { "found last batch id #{last_batch_id} so start with it!" }
+      info { "Finded last_batch_id #{_last_batch_id} so start with it!" }
+      prefix += "(#{_last_batch_id}:) "
       _last_batch_id
     end
 
     prox = prox.where(@opts[:where]) if @opts[:where]
 
     cnt = 0
-    count = prox.count
-    info { "full count to process #{count}" }
+    count = last_batch_id ? prox.where("id > ?", last_batch_id).count : prox.count
+    info { "#{prefix}full count to process #{count}" }
 
     h = {:batch_size => self.class.get_batch_size}
     h[:start] = last_batch_id + 1 if last_batch_id
     prox.find_in_batches(h) do |batch|
       res = perform(batch)
       cnt += batch.size
-      info { "process batch #{res.inspect}, #{cnt} from #{count}, lastid #{batch.last.id}" }
+      info { "#{prefix}process batch #{res.inspect}, #{cnt} from #{count}, lastid #{batch.last.id}" }
       set_store("lastbatch", batch.last.id)
       break if stopped
     end
 
-    info { "processed total #{cnt}" }
+    reset_last_batch_id unless stopped
+
+    info { "#{prefix}processed total #{cnt}" }
   end
 
   def reset_last_batch_id

data/lib/jober/threaded_manager.rb

@@ -11,7 +11,7 @@ class Jober::ThreadedManager
   end
 
   def initialize(klasses = nil, opts = {})
-    @klasses = klasses || Jober.classes
+    @klasses = Array(klasses || Jober.classes)
     @stopped = false
     @objects = @klasses.map do |klass|
       if klass.is_a?(String)
@@ -24,7 +24,7 @@ class Jober::ThreadedManager
   end
 
   def run_loop
-    info { "run loop for #{@klasses}, in threads: #{@objects.length}" }
+    info { "run loop for #{@klasses.inspect}, in threads: #{@objects.length}" }
     @threads = @objects.map { |obj| make_thread(obj) }
 
     # set signals
@@ -39,10 +39,18 @@ class Jober::ThreadedManager
     # send stop to all objects
     @objects.each(&:stop!)
 
+    # sleep a little
+    sleep(0.2)
+
     # sleep a little to give time for threads to quit
     wait_for_kill(default_sleep.to_f)
 
-    info { "quit!" }
+    names = not_finished_objects_names
+    if names.empty?
+      info { "quit!" }
+    else
+      info { "quit! and force killing #{names.inspect}" }
+    end
 
     # kill all threads, if they still alive
     @threads.select(&:alive?).each(&:kill)
@@ -61,8 +69,12 @@ private
     end
   end
 
+  def not_finished_objects_names
+    @objects.reject(&:finished).map { |o| o.class.name }
+  end
+
   def wait_for_kill(interval)
-    info { "waiting quiting jobs for %.1fm ..." % [interval / 60.0] }
+    info { "waiting quiting jobs (#{not_finished_objects_names.inspect}) for %.1fm ..." % [interval / 60.0] }
     Timeout.timeout(interval.to_f) do
       loop do
         sleep 0.3

data/lib/jober/version.rb

@@ -1,3 +1,3 @@
 module Jober
-  VERSION = "0.3"
+  VERSION = "0.3.5"
 end

data/spec/ar_loop_spec.rb

@@ -30,6 +30,7 @@ end
 
 conn = { 'adapter' => 'sqlite3', 'database' => File.dirname(__FILE__) + "/test.db" }
 ActiveRecord::Base.establish_connection conn
+#ActiveRecord::Base.logger = Logger.new(STDOUT)
 
 def pg_create_schema
   ActiveRecord::Migration.verbose = false
@@ -115,4 +116,14 @@ describe "ARLoop" do
     SO["names"].size.should == 66
   end
 
+  it "should not start from lastbatch if task was finished by itself, not by stop" do
+    my = MyAR2.new
+    my.execute
+    SO["names"].size.should == 46
+
+    # should start from zero
+    my = MyAR2.new
+    my.execute
+    SO["names"].size.should == 46 + 46
+  end
 end

data/spec/stats_spec.rb

@@ -31,5 +31,7 @@ describe "Stats" do
     h['my2'][:started].should be
     h['my2'][:finished].should be
     h['my2'][:duration].should be_within(0.1).of(0.0)
+
+    MyQueue2.stats[:started].should be
   end
 end

data/spec/task_loop_spec.rb

@@ -31,6 +31,16 @@ class Loop2 < Jober::Task
   end
 end
 
+class Loop3 < Jober::Task
+  interval 3
+  def perform
+    c = 0
+    loop { sleep 1; c += 1; break if stopped || c > 4 }
+    SO["y"] ||= 0
+    SO["y"] += 1
+  end
+end
+
 describe "Task" do
   it "interval interval should be inherited" do
     AA1.get_interval.should == 10
@@ -68,7 +78,7 @@ describe "Task" do
     l.execute
     t = Thread.new { l.run_loop }
     sleep 2
-    l.stopped = true
+    l.stop!
     sleep 0.3
     SO["y"].should == 1
   end
@@ -79,7 +89,7 @@ describe "Task" do
     Loop2.skip_delay!
     t = Thread.new { l.run_loop }
     sleep 2
-    l.stopped = true
+    l.stop!
     sleep 0.3
     SO["y"].should == 2
   end
@@ -89,11 +99,44 @@ describe "Task" do
     l.execute
     t = Thread.new { l.run_loop }
     sleep 2
-    l.stopped = true
+    l.stop!
     sleep 0.3
     SO["y"].should == 2
   end
 
+  describe "task was finished by itself or by stop!" do
+    it "if was finished by itself" do
+      l = Loop3.new
+      l.execute
+      SO["y"].should == 1
+
+      # next run should wait for 3 seconds
+      l = Loop3.new
+      t = Thread.new { l.run_loop }
+      sleep 2
+      l.stop!
+      sleep 0.3
+      SO["y"].should == 1
+    end
+
+    it "was stopped in middle, should start next imidiately" do
+      l = Loop3.new
+      t = Thread.new { l.execute }
+      sleep 2
+      l.stop!
+      sleep 0.3
+      SO["y"].should == 1
+
+      # next run should wait for 3 seconds
+      l = Loop3.new
+      t = Thread.new { l.run_loop }
+      sleep 2
+      l.stop!
+      sleep 0.3
+      SO["y"].should == 2
+    end
+  end
+
   it "skip_delay!" do
     Loop2.pop_skip_delay_flag!.should == false
     Loop2.skip_delay!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jober
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: 0.3.5
 platform: ruby
 authors:
 - "'Konstantin Makarchev'"
@@ -25,49 +25,49 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: sqlite3-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,19 +81,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: i18n
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.0
 description: Simple background jobs, queues.
 email:
 - "'kostya27@gmail.com'"