jober 0.2 → 0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b38442d4fe738b159e68bb21a37d42d6343058ab
4
- data.tar.gz: 3d7ea0e3cd642cfa93545dd486e296c3b4f2a150
3
+ metadata.gz: ddcfe15a02c6711edc74697daa7f399569561ee1
4
+ data.tar.gz: 7d1fb585d8bd3e1307231d2a4eb7944102cd9fac
5
5
  SHA512:
6
- metadata.gz: c8bb7b73c5725f34b19934b602026f91914b9cf666a5688ebcf249ea8fb4fd7ac9b0598700fba3e44a830476d335f9c2260c087fd5e79f7aa1be2653d3b65980
7
- data.tar.gz: 867f8e4a94b2f591c6ebc2f7fd9ff0f251c6ff0fadd8db3ae9ff7c5f8984f34ca06fd71856b87bced9b6ebf786c444af2d762c54c803e988c716d319fd399cfa
6
+ metadata.gz: 274b9cf0c10120c4e0b0f20ef198b54d6d449c9e0be5248a88c59bc0e05d21adeff4aa33a54d923c3ecd6964b319b0e8ec70cc5d7d76a2c95480a82c80e787ee
7
+ data.tar.gz: c90c327dfa205e59dcf083d07c040e0d12a102a2fc0fcfec6705ebd0fd13dfc4b426df64a329bd7ba0ce007c95ffc6ae4698c33a4625b7537084f827cdb3b7b6
@@ -2,11 +2,8 @@ language: ruby
2
2
  services:
3
3
  - redis-server
4
4
  rvm:
5
- - "ree"
6
- - "1.8.7"
7
5
  - "1.9.3"
8
6
  - "2.0.0"
9
7
  - "2.1"
10
8
  - "2.2"
11
9
  - "rbx"
12
- - "jruby"
@@ -13,7 +13,13 @@ end
13
13
  Jober.logger = Logger.new nil
14
14
 
15
15
  t = Time.now
16
- 50000.times { |i| Bench.enqueue(i, -i / 2.0) }
16
+ threads = []
17
+ 5.times do |ti|
18
+ threads << Thread.new do
19
+ 10000.times { |i| Bench.enqueue(i + ti * 10000, -(i + ti * 10000 ) / 2.0) }
20
+ end
21
+ end
22
+ threads.map(&:join)
17
23
  puts Time.now - t
18
24
  puts Bench.len
19
25
 
@@ -0,0 +1,6 @@
1
+ require 'bundler/setup'
2
+ Bundler.require :default
3
+ require_relative 'classes'
4
+
5
+ man = Jober::ThreadedManager.new
6
+ man.run_loop
@@ -5,6 +5,7 @@ require 'logger'
5
5
 
6
6
  module Jober
7
7
  autoload :Manager, 'jober/manager'
8
+ autoload :ThreadedManager, 'jober/threaded_manager'
8
9
  autoload :AbstractTask, 'jober/abstract_task'
9
10
  autoload :Task, 'jober/task'
10
11
  autoload :Queue, 'jober/queue'
@@ -14,6 +15,7 @@ module Jober
14
15
  autoload :SharedObject, 'jober/shared_object'
15
16
  autoload :UniqueQueueBatch, 'jober/unique_queue_batch'
16
17
  autoload :ARLoop, 'jober/ar_loop'
18
+ autoload :Exception, 'jober/exception'
17
19
 
18
20
  class << self
19
21
  def logger
@@ -25,22 +27,31 @@ module Jober
25
27
  end
26
28
 
27
29
  def redis
28
- @redis ||= Redis.new
30
+ Thread.current[:__jober_redis__] ||= (@redis || Redis.new).dup
29
31
  end
30
32
 
31
33
  def redis=(r)
32
34
  @redis = r
35
+ reset_redis
36
+ end
37
+
38
+ def reset_redis
39
+ Thread.current[:__jober_redis__] = nil
33
40
  end
34
41
 
35
42
  def internal_classes_names
36
- @internal_classes_names ||= (%w{Manager AbstractTask Task Queue} +
37
- %w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop}).map { |k| "Jober::#{k}" }
43
+ @internal_classes_names ||= (%w{Manager ThreadedManager AbstractTask Task Queue} +
44
+ %w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop Exception}).map { |k| "Jober::#{k}" }
38
45
  end
39
46
 
40
47
  def classes
41
48
  @classes ||= []
42
49
  end
43
50
 
51
+ def names
52
+ classes.map { |k| underscore(k.to_s.gsub('Jober::', '')) }
53
+ end
54
+
44
55
  def add_class(klass)
45
56
  classes << klass unless internal_classes_names.include?(klass.to_s)
46
57
  end
@@ -53,10 +64,6 @@ module Jober
53
64
  @after_fork.call if @after_fork
54
65
  end
55
66
 
56
- def reset_redis
57
- redis.client.reconnect
58
- end
59
-
60
67
  def dump(obj)
61
68
  Marshal.dump(obj)
62
69
  end
@@ -2,6 +2,7 @@ require 'timeout'
2
2
 
3
3
  class Jober::AbstractTask
4
4
  include Jober::Logger
5
+ include Jober::Exception
5
6
 
6
7
  class << self
7
8
  def interval(interval)
@@ -24,6 +25,7 @@ class Jober::AbstractTask
24
25
  end
25
26
 
26
27
  attr_accessor :stopped
28
+ attr_reader :worker_id, :workers_count
27
29
 
28
30
  def self.inherited(base)
29
31
  Jober.add_class(base)
@@ -94,6 +96,10 @@ class Jober::AbstractTask
94
96
  rescue Timeout::Error
95
97
  end
96
98
 
99
+ def stop!
100
+ @stopped = true
101
+ end
102
+
97
103
  private
98
104
 
99
105
  def self.timestamp_key(type)
@@ -133,4 +139,21 @@ private
133
139
  end
134
140
  end
135
141
 
142
+ def store_key(name)
143
+ Jober.key("store:#{self.class.short_name}-#{self.worker_id}-#{self.workers_count}:#{name}")
144
+ end
145
+
146
+ def set_store(name, obj)
147
+ self.catch do
148
+ Jober.redis.set(store_key(name), Jober.dump(obj))
149
+ end
150
+ end
151
+
152
+ def get_store(name)
153
+ self.catch do
154
+ r = Jober.redis.get(store_key(name))
155
+ Jober.load(r) if r
156
+ end
157
+ end
158
+
136
159
  end
@@ -16,8 +16,15 @@ class Jober::ARLoop < Jober::Task
16
16
  def run
17
17
  prox = proxy
18
18
 
19
- if @worker_id && @workers_count && !@opts[:no_auto_proxy]
20
- prox = prox.where("id % #{@workers_count} = #{@worker_id}")
19
+ if @worker_id && @workers_count && @workers_count > 1 && !@opts[:no_auto_proxy]
20
+ cond = "id % #{@workers_count} = #{@worker_id}"
21
+ prox = prox.where(cond)
22
+ info { "sharding enabled '#{cond}'" }
23
+ end
24
+
25
+ last_batch_id = if (_last_batch_id = get_store("lastbatch")) && !@opts[:no_last_batch]
26
+ info { "found last batch id #{last_batch_id} so start with it!" }
27
+ _last_batch_id
21
28
  end
22
29
 
23
30
  prox = prox.where(@opts[:where]) if @opts[:where]
@@ -26,13 +33,20 @@ class Jober::ARLoop < Jober::Task
26
33
  count = prox.count
27
34
  info { "full count to process #{count}" }
28
35
 
29
- prox.find_in_batches(:batch_size => self.class.get_batch_size) do |batch|
36
+ h = {:batch_size => self.class.get_batch_size}
37
+ h[:start] = last_batch_id + 1 if last_batch_id
38
+ prox.find_in_batches(h) do |batch|
30
39
  res = perform(batch)
31
40
  cnt += batch.size
32
41
  info { "process batch #{res.inspect}, #{cnt} from #{count}, lastid #{batch.last.id}" }
42
+ set_store("lastbatch", batch.last.id)
33
43
  break if stopped
34
44
  end
35
45
 
36
46
  info { "processed total #{cnt}" }
37
47
  end
48
+
49
+ def reset_last_batch_id
50
+ Jober.redis.del(store_key('lastbatch'))
51
+ end
38
52
  end
@@ -0,0 +1,16 @@
1
+ module Jober::Exception
2
+
3
+ def exception(ex)
4
+ msg = self.respond_to?(:logger_tag) ? "#{self.logger_tag} #{ex.message}" : ex.message
5
+ ex2 = ex.class.new(msg)
6
+ ex2.set_backtrace(ex.backtrace)
7
+ Jober.exception(ex2)
8
+ end
9
+
10
+ def catch(&block)
11
+ yield
12
+ rescue Object => ex
13
+ exception(ex)
14
+ nil
15
+ end
16
+ end
@@ -81,7 +81,7 @@ class Jober::Manager
81
81
  add_pid(pid)
82
82
  Process.wait(pid)
83
83
  del_pid(pid)
84
- sleep interval unless stopped
84
+ sleep interval.to_f unless stopped
85
85
  end
86
86
  del_pid(pid)
87
87
  break if stopped
@@ -43,7 +43,7 @@ module Jober::QueueBatchFeature
43
43
  perform(batch)
44
44
  rescue Object => ex
45
45
  reschedule_batch(batch)
46
- Jober.exception(ex)
46
+ exception(ex)
47
47
  end
48
48
 
49
49
  private
@@ -0,0 +1,85 @@
1
+ class Jober::ThreadedManager
2
+ include Jober::Logger
3
+ include Jober::Exception
4
+
5
+ def default_sleep=(ds)
6
+ @default_sleep = ds
7
+ end
8
+
9
+ def default_sleep
10
+ @default_sleep ||= 10
11
+ end
12
+
13
+ def initialize(klasses = nil, opts = {})
14
+ @klasses = klasses || Jober.classes
15
+ @stopped = false
16
+ @objects = @klasses.map do |klass|
17
+ if klass.is_a?(String)
18
+ klass_str = klass
19
+ klass = Jober.find_class(klass_str)
20
+ raise "unknown class #{klass_str}" unless klass
21
+ end
22
+ klass.new(opts)
23
+ end
24
+ end
25
+
26
+ def run_loop
27
+ info { "run loop for #{@klasses}, in threads: #{@objects.length}" }
28
+ @threads = @objects.map { |obj| make_thread(obj) }
29
+
30
+ # set signals
31
+ trap("INT") { @stopped = true }
32
+ trap("QUIT") { @stopped = true }
33
+
34
+ # sleeping infinitely
35
+ sleeping
36
+
37
+ info { "prepare quit ..." }
38
+
39
+ # send stop to all objects
40
+ @objects.each(&:stop!)
41
+
42
+ # sleep a little to give time for threads to quit
43
+ wait_for_kill(default_sleep.to_f)
44
+
45
+ info { "quit!" }
46
+
47
+ # kill all threads, if they still alive
48
+ @threads.select(&:alive?).each(&:kill)
49
+ end
50
+
51
+ def stop!
52
+ @stopped = true
53
+ end
54
+
55
+ private
56
+
57
+ def sleeping
58
+ loop do
59
+ sleep 0.5
60
+ return if @stopped
61
+ end
62
+ end
63
+
64
+ def wait_for_kill(interval)
65
+ info { "waiting quiting jobs for %.1fm ..." % [interval / 60.0] }
66
+ Timeout.timeout(interval.to_f) do
67
+ loop do
68
+ sleep 0.3
69
+ return if @threads.none? &:alive?
70
+ end
71
+ end
72
+ rescue Timeout::Error
73
+ end
74
+
75
+ def make_thread(obj)
76
+ Thread.new do
77
+ loop do
78
+ break if @stopped
79
+ obj.catch { obj.run_loop }
80
+ break if @stopped
81
+ sleep 1.0
82
+ end
83
+ end
84
+ end
85
+ end
@@ -1,3 +1,3 @@
1
1
  module Jober
2
- VERSION = "0.2"
2
+ VERSION = "0.3"
3
3
  end
@@ -14,6 +14,20 @@ class MyAR < Jober::ARLoop
14
14
  end
15
15
  end
16
16
 
17
+ class MyAR2 < Jober::ARLoop
18
+ batch_size 10
19
+
20
+ def proxy
21
+ User.where("years > 18")
22
+ end
23
+
24
+ def perform(batch)
25
+ sleep 1
26
+ SO["names"] += batch.map(&:name)
27
+ batch.size
28
+ end
29
+ end
30
+
17
31
  conn = { 'adapter' => 'sqlite3', 'database' => File.dirname(__FILE__) + "/test.db" }
18
32
  ActiveRecord::Base.establish_connection conn
19
33
 
@@ -66,4 +80,39 @@ describe "ARLoop" do
66
80
  SO["names"].size.should == 15
67
81
  SO["names"].last.should == "unknown 94"
68
82
  end
83
+
84
+ it "should use lastbatch" do
85
+ my = MyAR2.new
86
+ Thread.new { my.execute }
87
+ sleep 1.5
88
+ my.stop!
89
+ sleep 0.6
90
+
91
+ SO["names"].size.should == 20
92
+ SO["names"].last.should == "unknown 55"
93
+
94
+ # should start from last ID
95
+ my = MyAR2.new
96
+ my.execute
97
+ SO["names"].size.should == 46
98
+ SO["names"].last.should == "unknown 99"
99
+ end
100
+
101
+ it "should not use lastbatch, if it was dropped" do
102
+ my = MyAR2.new
103
+ Thread.new { my.execute }
104
+ sleep 1.5
105
+ my.stop!
106
+ sleep 0.6
107
+
108
+ SO["names"].size.should == 20
109
+ SO["names"].last.should == "unknown 55"
110
+
111
+ # should start from last ID
112
+ my = MyAR2.new
113
+ my.reset_last_batch_id
114
+ my.execute
115
+ SO["names"].size.should == 66
116
+ end
117
+
69
118
  end
@@ -33,3 +33,20 @@ def run_manager_for(timeout, classes, &block)
33
33
  sleep 0.1
34
34
  m
35
35
  end
36
+
37
+ def run_threaded_manager_for(timeout, classes, &block)
38
+ m = Jober::ThreadedManager.new classes
39
+ t = Thread.new do
40
+ m.run_loop
41
+ end
42
+
43
+ if block
44
+ block.call(m)
45
+ else
46
+ sleep timeout
47
+ end
48
+
49
+ m.stop!
50
+ t.join
51
+ m
52
+ end
@@ -0,0 +1,94 @@
1
+ require "#{File.dirname(__FILE__)}/spec_helper"
2
+
3
+ class T1 < Jober::Task
4
+ def perform
5
+ SO["t1"] += 1
6
+ end
7
+ end
8
+
9
+ class T2 < Jober::Task
10
+ def perform
11
+ SO["t2"] += 1
12
+ end
13
+ end
14
+
15
+ class T3 < Jober::Task
16
+ def perform
17
+ SO["t3"] += 1
18
+ end
19
+ end
20
+
21
+ class T4 < Jober::Task
22
+ def perform
23
+ loop do
24
+ sleep 0.1
25
+ break if stopped
26
+ end
27
+ SO["t4"] = "yahoo"
28
+ end
29
+ end
30
+
31
+ class T41 < Jober::Task
32
+ interval 1
33
+
34
+ def perform
35
+ sleep 1
36
+ SO["t41"] += 1
37
+ end
38
+ end
39
+
40
+ class T5 < Jober::Task
41
+ def perform
42
+ SO["t5"] += 1
43
+ 1 + "bla" # raising
44
+ end
45
+ end
46
+
47
+ class T6 < Jober::Task
48
+ def perform
49
+ sleep
50
+ end
51
+ end
52
+
53
+ describe Jober::ThreadedManager do
54
+ before :each do
55
+ SO["t1"] = 0
56
+ SO["t2"] = 0
57
+ SO["t3"] = 0
58
+ SO["t41"] = 0
59
+ SO["t5"] = 0
60
+ end
61
+
62
+ it "just work" do
63
+ run_threaded_manager_for(0.1, [T1])
64
+ SO["t1"].should == 1
65
+ end
66
+
67
+ xit "run multiple identical tasks" do
68
+ run_threaded_manager_for(0.2, [T1, T1, T1])
69
+ SO["t"].should == 3
70
+ end
71
+
72
+ it "run multiple tasks" do
73
+ run_threaded_manager_for(0.2, [T1, T2, T3])
74
+ SO["t1"].should == 1
75
+ SO["t2"].should == 1
76
+ SO["t3"].should == 1
77
+ end
78
+
79
+ it "task should be stopped by stopped flag" do
80
+ run_threaded_manager_for(0.5, [T4])
81
+ SO["t4"].should == "yahoo"
82
+ end
83
+
84
+ it "run multi tasks, but one raised" do
85
+ run_threaded_manager_for(2, [T41, T5])
86
+ SO["t5"].should == 2
87
+ end
88
+
89
+ it "infinite task should be stopped by kill" do
90
+ run_threaded_manager_for(2.5, [T6, T41])
91
+ SO["t41"].should == 2
92
+ end
93
+
94
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: jober
3
3
  version: !ruby/object:Gem::Version
4
- version: '0.2'
4
+ version: '0.3'
5
5
  platform: ruby
6
6
  authors:
7
7
  - "'Konstantin Makarchev'"
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-27 00:00:00.000000000 Z
11
+ date: 2015-04-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -114,16 +114,19 @@ files:
114
114
  - bin/jober_manager
115
115
  - examples/classes.rb
116
116
  - examples/man.rb
117
+ - examples/tman.rb
117
118
  - jober.gemspec
118
119
  - lib/jober.rb
119
120
  - lib/jober/abstract_task.rb
120
121
  - lib/jober/ar_loop.rb
122
+ - lib/jober/exception.rb
121
123
  - lib/jober/logger.rb
122
124
  - lib/jober/manager.rb
123
125
  - lib/jober/queue.rb
124
126
  - lib/jober/queue_batch.rb
125
127
  - lib/jober/shared_object.rb
126
128
  - lib/jober/task.rb
129
+ - lib/jober/threaded_manager.rb
127
130
  - lib/jober/unique_queue.rb
128
131
  - lib/jober/unique_queue_batch.rb
129
132
  - lib/jober/version.rb
@@ -142,6 +145,7 @@ files:
142
145
  - spec/stats_spec.rb
143
146
  - spec/task_loop_spec.rb
144
147
  - spec/task_spec.rb
148
+ - spec/threaded_manager_spec.rb
145
149
  - spec/unique_queue_batch_spec.rb
146
150
  - spec/unique_queue_spec.rb
147
151
  homepage: ''
@@ -184,5 +188,6 @@ test_files:
184
188
  - spec/stats_spec.rb
185
189
  - spec/task_loop_spec.rb
186
190
  - spec/task_spec.rb
191
+ - spec/threaded_manager_spec.rb
187
192
  - spec/unique_queue_batch_spec.rb
188
193
  - spec/unique_queue_spec.rb