jober 0.2 → 0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +0 -3
- data/bench/queue.rb +7 -1
- data/examples/tman.rb +6 -0
- data/lib/jober.rb +14 -7
- data/lib/jober/abstract_task.rb +23 -0
- data/lib/jober/ar_loop.rb +17 -3
- data/lib/jober/exception.rb +16 -0
- data/lib/jober/manager.rb +1 -1
- data/lib/jober/queue_batch.rb +1 -1
- data/lib/jober/threaded_manager.rb +85 -0
- data/lib/jober/version.rb +1 -1
- data/spec/ar_loop_spec.rb +49 -0
- data/spec/spec_helper.rb +17 -0
- data/spec/threaded_manager_spec.rb +94 -0
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ddcfe15a02c6711edc74697daa7f399569561ee1
|
4
|
+
data.tar.gz: 7d1fb585d8bd3e1307231d2a4eb7944102cd9fac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 274b9cf0c10120c4e0b0f20ef198b54d6d449c9e0be5248a88c59bc0e05d21adeff4aa33a54d923c3ecd6964b319b0e8ec70cc5d7d76a2c95480a82c80e787ee
|
7
|
+
data.tar.gz: c90c327dfa205e59dcf083d07c040e0d12a102a2fc0fcfec6705ebd0fd13dfc4b426df64a329bd7ba0ce007c95ffc6ae4698c33a4625b7537084f827cdb3b7b6
|
data/.travis.yml
CHANGED
data/bench/queue.rb
CHANGED
@@ -13,7 +13,13 @@ end
|
|
13
13
|
Jober.logger = Logger.new nil
|
14
14
|
|
15
15
|
t = Time.now
|
16
|
-
|
16
|
+
threads = []
|
17
|
+
5.times do |ti|
|
18
|
+
threads << Thread.new do
|
19
|
+
10000.times { |i| Bench.enqueue(i + ti * 10000, -(i + ti * 10000 ) / 2.0) }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
threads.map(&:join)
|
17
23
|
puts Time.now - t
|
18
24
|
puts Bench.len
|
19
25
|
|
data/examples/tman.rb
ADDED
data/lib/jober.rb
CHANGED
@@ -5,6 +5,7 @@ require 'logger'
|
|
5
5
|
|
6
6
|
module Jober
|
7
7
|
autoload :Manager, 'jober/manager'
|
8
|
+
autoload :ThreadedManager, 'jober/threaded_manager'
|
8
9
|
autoload :AbstractTask, 'jober/abstract_task'
|
9
10
|
autoload :Task, 'jober/task'
|
10
11
|
autoload :Queue, 'jober/queue'
|
@@ -14,6 +15,7 @@ module Jober
|
|
14
15
|
autoload :SharedObject, 'jober/shared_object'
|
15
16
|
autoload :UniqueQueueBatch, 'jober/unique_queue_batch'
|
16
17
|
autoload :ARLoop, 'jober/ar_loop'
|
18
|
+
autoload :Exception, 'jober/exception'
|
17
19
|
|
18
20
|
class << self
|
19
21
|
def logger
|
@@ -25,22 +27,31 @@ module Jober
|
|
25
27
|
end
|
26
28
|
|
27
29
|
def redis
|
28
|
-
@redis
|
30
|
+
Thread.current[:__jober_redis__] ||= (@redis || Redis.new).dup
|
29
31
|
end
|
30
32
|
|
31
33
|
def redis=(r)
|
32
34
|
@redis = r
|
35
|
+
reset_redis
|
36
|
+
end
|
37
|
+
|
38
|
+
def reset_redis
|
39
|
+
Thread.current[:__jober_redis__] = nil
|
33
40
|
end
|
34
41
|
|
35
42
|
def internal_classes_names
|
36
|
-
@internal_classes_names ||= (%w{Manager AbstractTask Task Queue} +
|
37
|
-
%w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop}).map { |k| "Jober::#{k}" }
|
43
|
+
@internal_classes_names ||= (%w{Manager ThreadedManager AbstractTask Task Queue} +
|
44
|
+
%w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop Exception}).map { |k| "Jober::#{k}" }
|
38
45
|
end
|
39
46
|
|
40
47
|
def classes
|
41
48
|
@classes ||= []
|
42
49
|
end
|
43
50
|
|
51
|
+
def names
|
52
|
+
classes.map { |k| underscore(k.to_s.gsub('Jober::', '')) }
|
53
|
+
end
|
54
|
+
|
44
55
|
def add_class(klass)
|
45
56
|
classes << klass unless internal_classes_names.include?(klass.to_s)
|
46
57
|
end
|
@@ -53,10 +64,6 @@ module Jober
|
|
53
64
|
@after_fork.call if @after_fork
|
54
65
|
end
|
55
66
|
|
56
|
-
def reset_redis
|
57
|
-
redis.client.reconnect
|
58
|
-
end
|
59
|
-
|
60
67
|
def dump(obj)
|
61
68
|
Marshal.dump(obj)
|
62
69
|
end
|
data/lib/jober/abstract_task.rb
CHANGED
@@ -2,6 +2,7 @@ require 'timeout'
|
|
2
2
|
|
3
3
|
class Jober::AbstractTask
|
4
4
|
include Jober::Logger
|
5
|
+
include Jober::Exception
|
5
6
|
|
6
7
|
class << self
|
7
8
|
def interval(interval)
|
@@ -24,6 +25,7 @@ class Jober::AbstractTask
|
|
24
25
|
end
|
25
26
|
|
26
27
|
attr_accessor :stopped
|
28
|
+
attr_reader :worker_id, :workers_count
|
27
29
|
|
28
30
|
def self.inherited(base)
|
29
31
|
Jober.add_class(base)
|
@@ -94,6 +96,10 @@ class Jober::AbstractTask
|
|
94
96
|
rescue Timeout::Error
|
95
97
|
end
|
96
98
|
|
99
|
+
def stop!
|
100
|
+
@stopped = true
|
101
|
+
end
|
102
|
+
|
97
103
|
private
|
98
104
|
|
99
105
|
def self.timestamp_key(type)
|
@@ -133,4 +139,21 @@ private
|
|
133
139
|
end
|
134
140
|
end
|
135
141
|
|
142
|
+
def store_key(name)
|
143
|
+
Jober.key("store:#{self.class.short_name}-#{self.worker_id}-#{self.workers_count}:#{name}")
|
144
|
+
end
|
145
|
+
|
146
|
+
def set_store(name, obj)
|
147
|
+
self.catch do
|
148
|
+
Jober.redis.set(store_key(name), Jober.dump(obj))
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def get_store(name)
|
153
|
+
self.catch do
|
154
|
+
r = Jober.redis.get(store_key(name))
|
155
|
+
Jober.load(r) if r
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
136
159
|
end
|
data/lib/jober/ar_loop.rb
CHANGED
@@ -16,8 +16,15 @@ class Jober::ARLoop < Jober::Task
|
|
16
16
|
def run
|
17
17
|
prox = proxy
|
18
18
|
|
19
|
-
if @worker_id && @workers_count && !@opts[:no_auto_proxy]
|
20
|
-
|
19
|
+
if @worker_id && @workers_count && @workers_count > 1 && !@opts[:no_auto_proxy]
|
20
|
+
cond = "id % #{@workers_count} = #{@worker_id}"
|
21
|
+
prox = prox.where(cond)
|
22
|
+
info { "sharding enabled '#{cond}'" }
|
23
|
+
end
|
24
|
+
|
25
|
+
last_batch_id = if (_last_batch_id = get_store("lastbatch")) && !@opts[:no_last_batch]
|
26
|
+
info { "found last batch id #{last_batch_id} so start with it!" }
|
27
|
+
_last_batch_id
|
21
28
|
end
|
22
29
|
|
23
30
|
prox = prox.where(@opts[:where]) if @opts[:where]
|
@@ -26,13 +33,20 @@ class Jober::ARLoop < Jober::Task
|
|
26
33
|
count = prox.count
|
27
34
|
info { "full count to process #{count}" }
|
28
35
|
|
29
|
-
|
36
|
+
h = {:batch_size => self.class.get_batch_size}
|
37
|
+
h[:start] = last_batch_id + 1 if last_batch_id
|
38
|
+
prox.find_in_batches(h) do |batch|
|
30
39
|
res = perform(batch)
|
31
40
|
cnt += batch.size
|
32
41
|
info { "process batch #{res.inspect}, #{cnt} from #{count}, lastid #{batch.last.id}" }
|
42
|
+
set_store("lastbatch", batch.last.id)
|
33
43
|
break if stopped
|
34
44
|
end
|
35
45
|
|
36
46
|
info { "processed total #{cnt}" }
|
37
47
|
end
|
48
|
+
|
49
|
+
def reset_last_batch_id
|
50
|
+
Jober.redis.del(store_key('lastbatch'))
|
51
|
+
end
|
38
52
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Jober::Exception
|
2
|
+
|
3
|
+
def exception(ex)
|
4
|
+
msg = self.respond_to?(:logger_tag) ? "#{self.logger_tag} #{ex.message}" : ex.message
|
5
|
+
ex2 = ex.class.new(msg)
|
6
|
+
ex2.set_backtrace(ex.backtrace)
|
7
|
+
Jober.exception(ex2)
|
8
|
+
end
|
9
|
+
|
10
|
+
def catch(&block)
|
11
|
+
yield
|
12
|
+
rescue Object => ex
|
13
|
+
exception(ex)
|
14
|
+
nil
|
15
|
+
end
|
16
|
+
end
|
data/lib/jober/manager.rb
CHANGED
data/lib/jober/queue_batch.rb
CHANGED
@@ -0,0 +1,85 @@
|
|
1
|
+
class Jober::ThreadedManager
|
2
|
+
include Jober::Logger
|
3
|
+
include Jober::Exception
|
4
|
+
|
5
|
+
def default_sleep=(ds)
|
6
|
+
@default_sleep = ds
|
7
|
+
end
|
8
|
+
|
9
|
+
def default_sleep
|
10
|
+
@default_sleep ||= 10
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(klasses = nil, opts = {})
|
14
|
+
@klasses = klasses || Jober.classes
|
15
|
+
@stopped = false
|
16
|
+
@objects = @klasses.map do |klass|
|
17
|
+
if klass.is_a?(String)
|
18
|
+
klass_str = klass
|
19
|
+
klass = Jober.find_class(klass_str)
|
20
|
+
raise "unknown class #{klass_str}" unless klass
|
21
|
+
end
|
22
|
+
klass.new(opts)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def run_loop
|
27
|
+
info { "run loop for #{@klasses}, in threads: #{@objects.length}" }
|
28
|
+
@threads = @objects.map { |obj| make_thread(obj) }
|
29
|
+
|
30
|
+
# set signals
|
31
|
+
trap("INT") { @stopped = true }
|
32
|
+
trap("QUIT") { @stopped = true }
|
33
|
+
|
34
|
+
# sleeping infinitely
|
35
|
+
sleeping
|
36
|
+
|
37
|
+
info { "prepare quit ..." }
|
38
|
+
|
39
|
+
# send stop to all objects
|
40
|
+
@objects.each(&:stop!)
|
41
|
+
|
42
|
+
# sleep a little to give time for threads to quit
|
43
|
+
wait_for_kill(default_sleep.to_f)
|
44
|
+
|
45
|
+
info { "quit!" }
|
46
|
+
|
47
|
+
# kill all threads, if they still alive
|
48
|
+
@threads.select(&:alive?).each(&:kill)
|
49
|
+
end
|
50
|
+
|
51
|
+
def stop!
|
52
|
+
@stopped = true
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def sleeping
|
58
|
+
loop do
|
59
|
+
sleep 0.5
|
60
|
+
return if @stopped
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def wait_for_kill(interval)
|
65
|
+
info { "waiting quiting jobs for %.1fm ..." % [interval / 60.0] }
|
66
|
+
Timeout.timeout(interval.to_f) do
|
67
|
+
loop do
|
68
|
+
sleep 0.3
|
69
|
+
return if @threads.none? &:alive?
|
70
|
+
end
|
71
|
+
end
|
72
|
+
rescue Timeout::Error
|
73
|
+
end
|
74
|
+
|
75
|
+
def make_thread(obj)
|
76
|
+
Thread.new do
|
77
|
+
loop do
|
78
|
+
break if @stopped
|
79
|
+
obj.catch { obj.run_loop }
|
80
|
+
break if @stopped
|
81
|
+
sleep 1.0
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
data/lib/jober/version.rb
CHANGED
data/spec/ar_loop_spec.rb
CHANGED
@@ -14,6 +14,20 @@ class MyAR < Jober::ARLoop
|
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
+
class MyAR2 < Jober::ARLoop
|
18
|
+
batch_size 10
|
19
|
+
|
20
|
+
def proxy
|
21
|
+
User.where("years > 18")
|
22
|
+
end
|
23
|
+
|
24
|
+
def perform(batch)
|
25
|
+
sleep 1
|
26
|
+
SO["names"] += batch.map(&:name)
|
27
|
+
batch.size
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
17
31
|
conn = { 'adapter' => 'sqlite3', 'database' => File.dirname(__FILE__) + "/test.db" }
|
18
32
|
ActiveRecord::Base.establish_connection conn
|
19
33
|
|
@@ -66,4 +80,39 @@ describe "ARLoop" do
|
|
66
80
|
SO["names"].size.should == 15
|
67
81
|
SO["names"].last.should == "unknown 94"
|
68
82
|
end
|
83
|
+
|
84
|
+
it "should use lastbatch" do
|
85
|
+
my = MyAR2.new
|
86
|
+
Thread.new { my.execute }
|
87
|
+
sleep 1.5
|
88
|
+
my.stop!
|
89
|
+
sleep 0.6
|
90
|
+
|
91
|
+
SO["names"].size.should == 20
|
92
|
+
SO["names"].last.should == "unknown 55"
|
93
|
+
|
94
|
+
# should start from last ID
|
95
|
+
my = MyAR2.new
|
96
|
+
my.execute
|
97
|
+
SO["names"].size.should == 46
|
98
|
+
SO["names"].last.should == "unknown 99"
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should not use lastbatch, if it was dropped" do
|
102
|
+
my = MyAR2.new
|
103
|
+
Thread.new { my.execute }
|
104
|
+
sleep 1.5
|
105
|
+
my.stop!
|
106
|
+
sleep 0.6
|
107
|
+
|
108
|
+
SO["names"].size.should == 20
|
109
|
+
SO["names"].last.should == "unknown 55"
|
110
|
+
|
111
|
+
# should start from last ID
|
112
|
+
my = MyAR2.new
|
113
|
+
my.reset_last_batch_id
|
114
|
+
my.execute
|
115
|
+
SO["names"].size.should == 66
|
116
|
+
end
|
117
|
+
|
69
118
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -33,3 +33,20 @@ def run_manager_for(timeout, classes, &block)
|
|
33
33
|
sleep 0.1
|
34
34
|
m
|
35
35
|
end
|
36
|
+
|
37
|
+
def run_threaded_manager_for(timeout, classes, &block)
|
38
|
+
m = Jober::ThreadedManager.new classes
|
39
|
+
t = Thread.new do
|
40
|
+
m.run_loop
|
41
|
+
end
|
42
|
+
|
43
|
+
if block
|
44
|
+
block.call(m)
|
45
|
+
else
|
46
|
+
sleep timeout
|
47
|
+
end
|
48
|
+
|
49
|
+
m.stop!
|
50
|
+
t.join
|
51
|
+
m
|
52
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
require "#{File.dirname(__FILE__)}/spec_helper"
|
2
|
+
|
3
|
+
class T1 < Jober::Task
|
4
|
+
def perform
|
5
|
+
SO["t1"] += 1
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
class T2 < Jober::Task
|
10
|
+
def perform
|
11
|
+
SO["t2"] += 1
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class T3 < Jober::Task
|
16
|
+
def perform
|
17
|
+
SO["t3"] += 1
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class T4 < Jober::Task
|
22
|
+
def perform
|
23
|
+
loop do
|
24
|
+
sleep 0.1
|
25
|
+
break if stopped
|
26
|
+
end
|
27
|
+
SO["t4"] = "yahoo"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class T41 < Jober::Task
|
32
|
+
interval 1
|
33
|
+
|
34
|
+
def perform
|
35
|
+
sleep 1
|
36
|
+
SO["t41"] += 1
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class T5 < Jober::Task
|
41
|
+
def perform
|
42
|
+
SO["t5"] += 1
|
43
|
+
1 + "bla" # raising
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class T6 < Jober::Task
|
48
|
+
def perform
|
49
|
+
sleep
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe Jober::ThreadedManager do
|
54
|
+
before :each do
|
55
|
+
SO["t1"] = 0
|
56
|
+
SO["t2"] = 0
|
57
|
+
SO["t3"] = 0
|
58
|
+
SO["t41"] = 0
|
59
|
+
SO["t5"] = 0
|
60
|
+
end
|
61
|
+
|
62
|
+
it "just work" do
|
63
|
+
run_threaded_manager_for(0.1, [T1])
|
64
|
+
SO["t1"].should == 1
|
65
|
+
end
|
66
|
+
|
67
|
+
xit "run multiple identical tasks" do
|
68
|
+
run_threaded_manager_for(0.2, [T1, T1, T1])
|
69
|
+
SO["t"].should == 3
|
70
|
+
end
|
71
|
+
|
72
|
+
it "run multiple tasks" do
|
73
|
+
run_threaded_manager_for(0.2, [T1, T2, T3])
|
74
|
+
SO["t1"].should == 1
|
75
|
+
SO["t2"].should == 1
|
76
|
+
SO["t3"].should == 1
|
77
|
+
end
|
78
|
+
|
79
|
+
it "task should be stopped by stopped flag" do
|
80
|
+
run_threaded_manager_for(0.5, [T4])
|
81
|
+
SO["t4"].should == "yahoo"
|
82
|
+
end
|
83
|
+
|
84
|
+
it "run multi tasks, but one raised" do
|
85
|
+
run_threaded_manager_for(2, [T41, T5])
|
86
|
+
SO["t5"].should == 2
|
87
|
+
end
|
88
|
+
|
89
|
+
it "infinite task should be stopped by kill" do
|
90
|
+
run_threaded_manager_for(2.5, [T6, T41])
|
91
|
+
SO["t41"].should == 2
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jober
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.3'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- "'Konstantin Makarchev'"
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -114,16 +114,19 @@ files:
|
|
114
114
|
- bin/jober_manager
|
115
115
|
- examples/classes.rb
|
116
116
|
- examples/man.rb
|
117
|
+
- examples/tman.rb
|
117
118
|
- jober.gemspec
|
118
119
|
- lib/jober.rb
|
119
120
|
- lib/jober/abstract_task.rb
|
120
121
|
- lib/jober/ar_loop.rb
|
122
|
+
- lib/jober/exception.rb
|
121
123
|
- lib/jober/logger.rb
|
122
124
|
- lib/jober/manager.rb
|
123
125
|
- lib/jober/queue.rb
|
124
126
|
- lib/jober/queue_batch.rb
|
125
127
|
- lib/jober/shared_object.rb
|
126
128
|
- lib/jober/task.rb
|
129
|
+
- lib/jober/threaded_manager.rb
|
127
130
|
- lib/jober/unique_queue.rb
|
128
131
|
- lib/jober/unique_queue_batch.rb
|
129
132
|
- lib/jober/version.rb
|
@@ -142,6 +145,7 @@ files:
|
|
142
145
|
- spec/stats_spec.rb
|
143
146
|
- spec/task_loop_spec.rb
|
144
147
|
- spec/task_spec.rb
|
148
|
+
- spec/threaded_manager_spec.rb
|
145
149
|
- spec/unique_queue_batch_spec.rb
|
146
150
|
- spec/unique_queue_spec.rb
|
147
151
|
homepage: ''
|
@@ -184,5 +188,6 @@ test_files:
|
|
184
188
|
- spec/stats_spec.rb
|
185
189
|
- spec/task_loop_spec.rb
|
186
190
|
- spec/task_spec.rb
|
191
|
+
- spec/threaded_manager_spec.rb
|
187
192
|
- spec/unique_queue_batch_spec.rb
|
188
193
|
- spec/unique_queue_spec.rb
|