jober 0.0.1 → 0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rspec +2 -0
- data/.travis.yml +12 -0
- data/Rakefile +6 -0
- data/bench/queue.rb +25 -0
- data/bin/jober +50 -5
- data/bin/{manager → jober_manager} +0 -0
- data/examples/classes.rb +3 -8
- data/jober.gemspec +2 -0
- data/lib/jober.rb +90 -26
- data/lib/jober/abstract_task.rb +105 -15
- data/lib/jober/ar_loop.rb +38 -0
- data/lib/jober/logger.rb +19 -3
- data/lib/jober/manager.rb +11 -13
- data/lib/jober/queue.rb +11 -6
- data/lib/jober/queue_batch.rb +42 -29
- data/lib/jober/shared_object.rb +37 -9
- data/lib/jober/task.rb +3 -3
- data/lib/jober/unique_queue.rb +1 -1
- data/lib/jober/unique_queue_batch.rb +5 -0
- data/lib/jober/version.rb +1 -1
- data/spec/ar_loop_spec.rb +69 -0
- data/spec/chain_spec.rb +2 -2
- data/spec/integration_spec.rb +17 -6
- data/spec/jober_spec.rb +25 -0
- data/spec/kill_task_spec.rb +1 -1
- data/spec/namespace_spec.rb +16 -0
- data/spec/pids_spec.rb +10 -3
- data/spec/queue_batch_spec.rb +2 -2
- data/spec/queue_parallel_spec.rb +3 -2
- data/spec/queue_spec.rb +2 -3
- data/spec/shared_object_spec.rb +1 -1
- data/spec/spec_helper.rb +2 -0
- data/spec/stats_spec.rb +6 -4
- data/spec/task_loop_spec.rb +103 -0
- data/spec/task_spec.rb +10 -9
- data/spec/unique_queue_batch_spec.rb +28 -0
- data/spec/unique_queue_spec.rb +2 -2
- metadata +47 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b38442d4fe738b159e68bb21a37d42d6343058ab
|
4
|
+
data.tar.gz: 3d7ea0e3cd642cfa93545dd486e296c3b4f2a150
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c8bb7b73c5725f34b19934b602026f91914b9cf666a5688ebcf249ea8fb4fd7ac9b0598700fba3e44a830476d335f9c2260c087fd5e79f7aa1be2653d3b65980
|
7
|
+
data.tar.gz: 867f8e4a94b2f591c6ebc2f7fd9ff0f251c6ff0fadd8db3ae9ff7c5f8984f34ca06fd71856b87bced9b6ebf786c444af2d762c54c803e988c716d319fd399cfa
|
data/.gitignore
CHANGED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Rakefile
CHANGED
data/bench/queue.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
Bundler.require
|
4
|
+
|
5
|
+
$summary = 0
|
6
|
+
|
7
|
+
class Bench < Jober::Queue
|
8
|
+
def perform(x, y)
|
9
|
+
$summary += x + y
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
Jober.logger = Logger.new nil
|
14
|
+
|
15
|
+
t = Time.now
|
16
|
+
50000.times { |i| Bench.enqueue(i, -i / 2.0) }
|
17
|
+
puts Time.now - t
|
18
|
+
puts Bench.len
|
19
|
+
|
20
|
+
t = Time.now
|
21
|
+
Bench.new.execute
|
22
|
+
puts Time.now - t
|
23
|
+
|
24
|
+
puts $summary
|
25
|
+
puts Bench.len
|
data/bin/jober
CHANGED
@@ -1,10 +1,55 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
$:.unshift File.expand_path(File.join(File.dirname(__FILE__), %w[.. lib]))
|
3
|
-
require "
|
3
|
+
require "jober"
|
4
4
|
|
5
|
-
|
6
|
-
recursive = true
|
5
|
+
require 'optparse'
|
7
6
|
|
8
|
-
|
9
|
-
|
7
|
+
options = {}
|
8
|
+
|
9
|
+
OptionParser.new do |opts|
|
10
|
+
opts.on( '-h', '--help', 'Display this screen' ) do
|
11
|
+
puts opts
|
12
|
+
exit
|
13
|
+
end
|
14
|
+
|
15
|
+
opts.on( '-c', '--class CLASS', 'run for class' ) do |klass|
|
16
|
+
options[:klass] = klass
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on( '-r', '--require PATH', 'require some path, usually config/environment' ) do |req|
|
20
|
+
options[:require] = req
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on( '-o', '--once', 'run once, instead of recursive' ) do |o|
|
24
|
+
options[:once] = o
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on( '-i', '--worker_id', 'worker_id for multiple workers' ) do |o|
|
28
|
+
options[:worker_id] = o
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on( '-w', '--workers_count', 'workers_count for multiple workers' ) do |o|
|
32
|
+
options[:workers_count] = o
|
33
|
+
end
|
34
|
+
|
35
|
+
end.parse!
|
36
|
+
|
37
|
+
if req = options[:require]
|
38
|
+
require File.expand_path(req)
|
39
|
+
end
|
40
|
+
|
41
|
+
klass_name = options[:klass]
|
42
|
+
klass = Jober.find_class(klass_name)
|
43
|
+
|
44
|
+
unless klass
|
45
|
+
puts "not found class #{klass_name}"
|
46
|
+
exit(1)
|
47
|
+
end
|
48
|
+
|
49
|
+
inst = klass.new(options)
|
50
|
+
|
51
|
+
if options[:once]
|
52
|
+
inst.execute
|
53
|
+
else
|
54
|
+
inst.run_loop
|
10
55
|
end
|
File without changes
|
data/examples/classes.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
require 'bundler/setup'
|
2
2
|
Bundler.require :default
|
3
3
|
|
4
|
+
Jober.default_interval = 10
|
5
|
+
|
4
6
|
class A < Jober::Task
|
5
|
-
every 10
|
6
7
|
def perform
|
7
8
|
10.times do |i|
|
8
9
|
info "enqueue to b #{i}"
|
@@ -12,7 +13,6 @@ class A < Jober::Task
|
|
12
13
|
end
|
13
14
|
|
14
15
|
class B < Jober::Queue
|
15
|
-
every 10
|
16
16
|
def perform(x)
|
17
17
|
10.times do |i|
|
18
18
|
info "enqueue to c #{x} #{i}"
|
@@ -22,7 +22,6 @@ class B < Jober::Queue
|
|
22
22
|
end
|
23
23
|
|
24
24
|
class C < Jober::Queue
|
25
|
-
every 10
|
26
25
|
def perform(x, i)
|
27
26
|
10.times do |j|
|
28
27
|
info "enqueue to d #{x} #{i} #{j}"
|
@@ -32,7 +31,6 @@ class C < Jober::Queue
|
|
32
31
|
end
|
33
32
|
|
34
33
|
class D < Jober::QueueBatch
|
35
|
-
every 10
|
36
34
|
batch_size 200
|
37
35
|
|
38
36
|
def perform(*batch)
|
@@ -41,7 +39,7 @@ class D < Jober::QueueBatch
|
|
41
39
|
end
|
42
40
|
|
43
41
|
class E < Jober::Task
|
44
|
-
|
42
|
+
workers 5
|
45
43
|
|
46
44
|
def perform
|
47
45
|
puts "start e :)"
|
@@ -50,15 +48,12 @@ class E < Jober::Task
|
|
50
48
|
end
|
51
49
|
|
52
50
|
class G < Jober::Task
|
53
|
-
every 10
|
54
|
-
|
55
51
|
def perform
|
56
52
|
"asdfsdf" + 1
|
57
53
|
end
|
58
54
|
end
|
59
55
|
|
60
56
|
class F < Jober::Task
|
61
|
-
every 5
|
62
57
|
def perform
|
63
58
|
sleep 100
|
64
59
|
end
|
data/jober.gemspec
CHANGED
@@ -23,4 +23,6 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_development_dependency "bundler", "~> 1.7"
|
24
24
|
spec.add_development_dependency "rake", "~> 10.0"
|
25
25
|
spec.add_development_dependency "rspec"
|
26
|
+
spec.add_development_dependency "activerecord"
|
27
|
+
spec.add_development_dependency "sqlite3"
|
26
28
|
end
|
data/lib/jober.rb
CHANGED
@@ -1,39 +1,48 @@
|
|
1
|
-
require "
|
1
|
+
require "jober/version"
|
2
2
|
|
3
3
|
require 'redis'
|
4
4
|
require 'logger'
|
5
5
|
|
6
6
|
module Jober
|
7
|
-
autoload :Manager,
|
8
|
-
autoload :AbstractTask,
|
9
|
-
autoload :Task,
|
10
|
-
autoload :Queue,
|
11
|
-
autoload :QueueBatch,
|
12
|
-
autoload :UniqueQueue,
|
13
|
-
autoload :Logger,
|
14
|
-
autoload :SharedObject,
|
7
|
+
autoload :Manager, 'jober/manager'
|
8
|
+
autoload :AbstractTask, 'jober/abstract_task'
|
9
|
+
autoload :Task, 'jober/task'
|
10
|
+
autoload :Queue, 'jober/queue'
|
11
|
+
autoload :QueueBatch, 'jober/queue_batch'
|
12
|
+
autoload :UniqueQueue, 'jober/unique_queue'
|
13
|
+
autoload :Logger, 'jober/logger'
|
14
|
+
autoload :SharedObject, 'jober/shared_object'
|
15
|
+
autoload :UniqueQueueBatch, 'jober/unique_queue_batch'
|
16
|
+
autoload :ARLoop, 'jober/ar_loop'
|
15
17
|
|
16
18
|
class << self
|
17
|
-
attr_accessor :redis_config
|
18
|
-
attr_reader :classes
|
19
|
-
|
20
19
|
def logger
|
21
|
-
::Logger.new(STDOUT)
|
20
|
+
@logger ||= ::Logger.new(STDOUT)
|
21
|
+
end
|
22
|
+
|
23
|
+
def logger=(l)
|
24
|
+
@logger = l
|
22
25
|
end
|
23
26
|
|
24
27
|
def redis
|
25
|
-
|
28
|
+
@redis ||= Redis.new
|
26
29
|
end
|
27
30
|
|
28
|
-
def
|
29
|
-
|
31
|
+
def redis=(r)
|
32
|
+
@redis = r
|
33
|
+
end
|
34
|
+
|
35
|
+
def internal_classes_names
|
36
|
+
@internal_classes_names ||= (%w{Manager AbstractTask Task Queue} +
|
37
|
+
%w{QueueBatch UniqueQueue UniqueQueueBatch Logger SharedObject ARLoop}).map { |k| "Jober::#{k}" }
|
38
|
+
end
|
39
|
+
|
40
|
+
def classes
|
41
|
+
@classes ||= []
|
30
42
|
end
|
31
43
|
|
32
44
|
def add_class(klass)
|
33
|
-
unless klass.to_s
|
34
|
-
@classes ||= []
|
35
|
-
@classes << klass
|
36
|
-
end
|
45
|
+
classes << klass unless internal_classes_names.include?(klass.to_s)
|
37
46
|
end
|
38
47
|
|
39
48
|
def after_fork(&block)
|
@@ -44,6 +53,10 @@ module Jober
|
|
44
53
|
@after_fork.call if @after_fork
|
45
54
|
end
|
46
55
|
|
56
|
+
def reset_redis
|
57
|
+
redis.client.reconnect
|
58
|
+
end
|
59
|
+
|
47
60
|
def dump(obj)
|
48
61
|
Marshal.dump(obj)
|
49
62
|
end
|
@@ -52,8 +65,13 @@ module Jober
|
|
52
65
|
Marshal.load(obj)
|
53
66
|
end
|
54
67
|
|
68
|
+
def dump_args(*args)
|
69
|
+
dump(args)
|
70
|
+
end
|
71
|
+
|
55
72
|
def exception(ex)
|
56
73
|
# redefine me
|
74
|
+
logger.error "#{ex.message} #{ex.backtrace}"
|
57
75
|
end
|
58
76
|
|
59
77
|
def underscore(str)
|
@@ -71,7 +89,7 @@ module Jober
|
|
71
89
|
h = {}
|
72
90
|
@classes.each do |klass|
|
73
91
|
next unless klass.ancestors.include?(Jober::Queue)
|
74
|
-
h[klass.
|
92
|
+
h[klass.queue_name_base] = klass.len
|
75
93
|
end
|
76
94
|
h
|
77
95
|
end
|
@@ -79,13 +97,59 @@ module Jober
|
|
79
97
|
def stats
|
80
98
|
h = {}
|
81
99
|
@classes.each do |klass|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
100
|
+
started = klass.read_timestamp(:started)
|
101
|
+
finished = klass.read_timestamp(:finished)
|
102
|
+
crashed = klass.read_timestamp(:crashed)
|
103
|
+
h[klass.short_name] = {
|
104
|
+
:started => started,
|
105
|
+
:finished => finished,
|
106
|
+
:crashed => crashed,
|
107
|
+
:duration => (finished && started && finished >= started) ? (finished - started) : nil
|
108
|
+
}
|
87
109
|
end
|
88
110
|
h
|
89
111
|
end
|
112
|
+
|
113
|
+
attr_accessor :namespace
|
114
|
+
|
115
|
+
def key(k)
|
116
|
+
"Jober:#{@namespace}:#{k}"
|
117
|
+
end
|
118
|
+
|
119
|
+
def find_class(klass_name)
|
120
|
+
names = classes.map(&:to_s)
|
121
|
+
return eval(klass_name) if names.include?(klass_name)
|
122
|
+
klass_name = "Jober::#{klass_name}"
|
123
|
+
return eval(klass_name) if names.include?(klass_name)
|
124
|
+
klass_name = "Jobs::#{klass_name}"
|
125
|
+
return eval(klass_name) if names.include?(klass_name)
|
126
|
+
end
|
127
|
+
|
128
|
+
def skip_delay!
|
129
|
+
classes.each &:skip_delay!
|
130
|
+
end
|
131
|
+
|
132
|
+
def default_interval
|
133
|
+
@default_interval ||= 5 * 60
|
134
|
+
end
|
135
|
+
|
136
|
+
def default_interval=(di)
|
137
|
+
@default_interval = di
|
138
|
+
end
|
139
|
+
|
140
|
+
def enqueue(queue_name, *args)
|
141
|
+
Jober.redis.rpush(queue_name, Jober.dump_args(*args))
|
142
|
+
end
|
143
|
+
|
144
|
+
def catch(&block)
|
145
|
+
yield
|
146
|
+
rescue Object => ex
|
147
|
+
Jober.exception(ex)
|
148
|
+
nil
|
149
|
+
end
|
90
150
|
end
|
91
151
|
end
|
152
|
+
|
153
|
+
# just empty module for store tasks
|
154
|
+
module Jobs
|
155
|
+
end
|
data/lib/jober/abstract_task.rb
CHANGED
@@ -1,13 +1,23 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
1
3
|
class Jober::AbstractTask
|
2
4
|
include Jober::Logger
|
3
5
|
|
4
6
|
class << self
|
5
|
-
def
|
6
|
-
|
7
|
+
def interval(interval)
|
8
|
+
@interval = interval
|
9
|
+
end
|
10
|
+
|
11
|
+
def get_interval
|
12
|
+
@interval || Jober.default_interval
|
13
|
+
end
|
14
|
+
|
15
|
+
def workers(n)
|
16
|
+
@workers = n
|
7
17
|
end
|
8
18
|
|
9
|
-
def
|
10
|
-
@workers
|
19
|
+
def get_workers
|
20
|
+
@workers || 1
|
11
21
|
end
|
12
22
|
|
13
23
|
attr_accessor :short_name
|
@@ -17,30 +27,110 @@ class Jober::AbstractTask
|
|
17
27
|
|
18
28
|
def self.inherited(base)
|
19
29
|
Jober.add_class(base)
|
30
|
+
base.interval(self.get_interval)
|
20
31
|
end
|
21
32
|
|
22
|
-
|
33
|
+
# opts:
|
34
|
+
# :worker_id
|
35
|
+
# :workers_count
|
36
|
+
# :skip_delay
|
37
|
+
def initialize(opts = {})
|
38
|
+
@opts = opts
|
23
39
|
@stopped = false
|
24
40
|
trap("QUIT") { @stopped = true }
|
25
41
|
trap("INT") { @stopped = true }
|
42
|
+
@worker_id = opts[:worker_id] || 0
|
43
|
+
@workers_count = opts[:workers_count] || 1
|
44
|
+
@skip_delay = opts[:skip_delay]
|
26
45
|
end
|
27
46
|
|
28
|
-
def execute
|
29
|
-
info "=>
|
47
|
+
def execute
|
48
|
+
info "=> start"
|
30
49
|
@start_at = Time.now
|
31
|
-
write_timestamp(:
|
32
|
-
run
|
33
|
-
write_timestamp(:
|
34
|
-
|
50
|
+
self.class.write_timestamp(:started)
|
51
|
+
run
|
52
|
+
self.class.write_timestamp(:finished)
|
53
|
+
self.class.del_timestamp(:crashed)
|
54
|
+
info "<= end (in #{Time.now - @start_at})"
|
35
55
|
self
|
56
|
+
rescue Object
|
57
|
+
self.class.write_timestamp(:crashed)
|
58
|
+
raise
|
59
|
+
end
|
60
|
+
|
61
|
+
def run_loop
|
62
|
+
info { "running loop" }
|
63
|
+
|
64
|
+
# wait until interval + last end
|
65
|
+
if self.class.get_workers <= 1 &&
|
66
|
+
(finished = self.class.read_timestamp(:finished)) &&
|
67
|
+
(Time.now - finished < self.class.get_interval) &&
|
68
|
+
!self.class.pop_skip_delay_flag! &&
|
69
|
+
!@skip_delay
|
70
|
+
|
71
|
+
sleeping(self.class.get_interval - (Time.now - finished))
|
72
|
+
end
|
73
|
+
|
74
|
+
# main loop
|
75
|
+
loop do
|
76
|
+
break if stopped
|
77
|
+
execute
|
78
|
+
break if stopped
|
79
|
+
sleeping
|
80
|
+
break if stopped
|
81
|
+
end
|
82
|
+
|
83
|
+
info { "quit loop" }
|
84
|
+
end
|
85
|
+
|
86
|
+
def sleeping(int = self.class.get_interval)
|
87
|
+
info { "sleeping for %.1fm ..." % [int / 60.0] }
|
88
|
+
Timeout.timeout(int.to_f) do
|
89
|
+
loop do
|
90
|
+
sleep 0.3
|
91
|
+
return if stopped
|
92
|
+
end
|
93
|
+
end
|
94
|
+
rescue Timeout::Error
|
36
95
|
end
|
37
96
|
|
38
97
|
private
|
39
98
|
|
40
|
-
def
|
41
|
-
Jober.
|
42
|
-
|
43
|
-
|
99
|
+
def self.timestamp_key(type)
|
100
|
+
Jober.key("stats:#{short_name}:#{type}")
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.pop_skip_delay_flag!
|
104
|
+
Jober.catch do
|
105
|
+
res = Jober.redis.get(timestamp_key(:skip))
|
106
|
+
Jober.redis.del(timestamp_key(:skip)) if res
|
107
|
+
!!res
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.skip_delay!
|
112
|
+
Jober.catch do
|
113
|
+
Jober.redis.set(timestamp_key(:skip), '1')
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.read_timestamp(type)
|
118
|
+
Jober.catch do
|
119
|
+
res = Jober.redis.get(timestamp_key(type))
|
120
|
+
Time.at(res.to_i) if res
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.write_timestamp(type)
|
125
|
+
Jober.catch do
|
126
|
+
Jober.redis.set(timestamp_key(type), Time.now.to_i.to_s)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.del_timestamp(type)
|
131
|
+
Jober.catch do
|
132
|
+
Jober.redis.del(timestamp_key(type))
|
133
|
+
end
|
44
134
|
end
|
45
135
|
|
46
136
|
end
|