pipa-monkeyjob 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +1 -1
- data/lib/monkeyjob.rb +4 -0
- data/lib/monkeyjob/broker/active_record.rb +57 -28
- data/lib/monkeyjob/broker/active_record/job_model.rb +5 -9
- data/lib/monkeyjob/client.rb +2 -0
- data/lib/monkeyjob/client/active_record.rb +15 -3
- data/lib/monkeyjob/job.rb +93 -11
- data/lib/monkeyjob/juggler.rb +12 -6
- data/lib/monkeyjob/logger.rb +33 -0
- data/lib/monkeyjob/runner.rb +4 -0
- data/lib/monkeyjob/runner/fork.rb +85 -0
- data/lib/monkeyjob/runner/fork/rails.rb +32 -0
- data/lib/monkeyjob/runner/thread_pool.rb +21 -6
- data/rails/init.rb +2 -0
- metadata +6 -22
data/README.rdoc
CHANGED
data/lib/monkeyjob.rb
CHANGED
@@ -1,11 +1,15 @@
|
|
1
1
|
require 'activesupport'
|
2
2
|
|
3
3
|
module Monkey
|
4
|
+
mattr_accessor :logger
|
5
|
+
@@logger = Logger.new(STDERR)
|
6
|
+
|
4
7
|
autoload :Client, 'monkeyjob/client'
|
5
8
|
autoload :Broker, 'monkeyjob/broker'
|
6
9
|
autoload :Runner, 'monkeyjob/runner'
|
7
10
|
autoload :Juggler, 'monkeyjob/juggler'
|
8
11
|
end
|
9
12
|
|
13
|
+
require 'monkeyjob/logger'
|
10
14
|
require 'monkeyjob/job'
|
11
15
|
|
@@ -3,51 +3,80 @@ require 'monitor'
|
|
3
3
|
begin
|
4
4
|
require 'activerecord'
|
5
5
|
rescue MissingSourceFile => e
|
6
|
-
raise
|
6
|
+
raise $!.exception('missing activerecord!')
|
7
7
|
end
|
8
8
|
|
9
9
|
module Monkey
|
10
10
|
class Broker
|
11
11
|
class ActiveRecord < Broker
|
12
|
-
|
13
|
-
|
12
|
+
class Cookie < Job::Cookie::Impl
|
13
|
+
attr_reader :queue, :id
|
14
|
+
|
15
|
+
def initialize(broker, jm)
|
16
|
+
super(jm.handler)
|
17
|
+
@broker, @queue, @id = broker, jm.queue, jm.id
|
18
|
+
end
|
14
19
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
+
def client
|
21
|
+
@broker.client
|
22
|
+
end
|
23
|
+
|
24
|
+
def to_s
|
25
|
+
"#{id}#{queue && '@' + queue} #{job}"
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
def on_state_changed(was)
|
30
|
+
super
|
31
|
+
@broker.send(:on_finish, self) if finished?
|
20
32
|
end
|
21
33
|
end
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@
|
29
|
-
@poller.run
|
34
|
+
|
35
|
+
attr_reader :client
|
36
|
+
|
37
|
+
def initialize(options = {})
|
38
|
+
extend MonitorMixin
|
39
|
+
@queues = Array(options[:queue] || options[:queues])
|
40
|
+
@client = Client::ActiveRecord.new
|
30
41
|
end
|
31
42
|
|
32
|
-
|
33
|
-
def poll
|
43
|
+
def start(&callback)
|
34
44
|
@jobs = Set.new
|
35
45
|
loop do
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
46
|
+
synchronize do
|
47
|
+
JobModel.logger.hush do |old_logger_level|
|
48
|
+
JobModel.transaction do
|
49
|
+
for jm in JobModel.candidates(@queues).all(:lock => true)
|
50
|
+
JobModel.logger.with_level(old_logger_level) do
|
51
|
+
if @jobs.add?(jm.id) && !callback.call(Cookie.new(self, jm))
|
52
|
+
@jobs.delete(jm.id)
|
53
|
+
break
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
JobModel.update_all({:run_at => 5.minutes.from_now}, :id => @jobs.to_a)
|
41
58
|
end
|
42
|
-
JobModel.update_all({:run_at => 5.minutes.from_now}, :id => @jobs.to_a)
|
43
59
|
end
|
44
60
|
end
|
45
|
-
break if @dead
|
46
61
|
sleep(5)
|
47
|
-
break if @dead
|
48
62
|
end
|
49
|
-
|
50
|
-
|
63
|
+
end
|
64
|
+
|
65
|
+
def close
|
66
|
+
self
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
def on_finish(cookie)
|
71
|
+
synchronize do
|
72
|
+
@jobs.delete(cookie.id)
|
73
|
+
if cookie.success?
|
74
|
+
JobModel.delete(cookie.id)
|
75
|
+
Monkey.logger.debug "Deleted job #{cookie}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
rescue
|
79
|
+
Monkey.logger.error $!
|
51
80
|
end
|
52
81
|
end
|
53
82
|
end
|
@@ -2,17 +2,13 @@ module Monkey
|
|
2
2
|
class Broker
|
3
3
|
class ActiveRecord
|
4
4
|
class JobModel < ::ActiveRecord::Base
|
5
|
-
extend ::ActiveSupport::Memoizable
|
6
|
-
|
7
5
|
set_table_name 'monkey_jobs'
|
6
|
+
serialize :handler
|
8
7
|
|
9
|
-
named_scope :candidates,
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
memoize :handler_object
|
8
|
+
named_scope :candidates, proc {|queues|
|
9
|
+
{:conditions => ["#{queues.blank? ? 'queue IS NULL' : 'queue IN (:queues)'} AND run_at <= :now",
|
10
|
+
{:now => Time.now, :queues => queues}], :order => 'run_at, id'}
|
11
|
+
}
|
16
12
|
end
|
17
13
|
end
|
18
14
|
end
|
data/lib/monkeyjob/client.rb
CHANGED
@@ -1,8 +1,20 @@
|
|
1
1
|
module Monkey
|
2
2
|
class Client
|
3
|
-
class ActiveRecord
|
4
|
-
def
|
5
|
-
|
3
|
+
class ActiveRecord < Client
|
4
|
+
def initialize(options = {})
|
5
|
+
@queue = options[:queue]
|
6
|
+
end
|
7
|
+
|
8
|
+
def submit(job, options = {})
|
9
|
+
jm = Broker::ActiveRecord::JobModel.silence do
|
10
|
+
Broker::ActiveRecord::JobModel.new {|m|
|
11
|
+
m.run_at = options[:run_at] || Time.now
|
12
|
+
m.handler = job
|
13
|
+
m.queue = options.fetch(:queue, @queue)
|
14
|
+
}.tap {|jm| jm.save! }
|
15
|
+
end
|
16
|
+
Monkey.logger.info { "Submitted #{job} to #{jm.queue}:#{jm.id}" }
|
17
|
+
self
|
6
18
|
end
|
7
19
|
end
|
8
20
|
end
|
data/lib/monkeyjob/job.rb
CHANGED
@@ -1,20 +1,102 @@
|
|
1
1
|
module Monkey
|
2
2
|
class Job
|
3
|
-
|
3
|
+
def to_proc
|
4
|
+
method(:run).to_proc
|
5
|
+
end
|
6
|
+
|
7
|
+
def to_s
|
8
|
+
self.class.name
|
9
|
+
end
|
10
|
+
|
11
|
+
def submit(options = {})
|
12
|
+
self.class.submit(self, options)
|
13
|
+
end
|
4
14
|
|
5
|
-
def
|
6
|
-
|
15
|
+
def submit_to(client, options = {})
|
16
|
+
client.submit(self, options)
|
7
17
|
end
|
8
18
|
|
9
|
-
def
|
10
|
-
|
11
|
-
rescue => @exception
|
12
|
-
ensure
|
13
|
-
@callback.call(self, *@callback_args)
|
19
|
+
def self.submit(handler, options = {})
|
20
|
+
Client.default.submit(handler, options)
|
14
21
|
end
|
15
|
-
|
16
|
-
|
17
|
-
|
22
|
+
|
23
|
+
class Cookie
|
24
|
+
attr_reader :state, :result, :exception, :started_at, :finished_at
|
25
|
+
|
26
|
+
def running?
|
27
|
+
state == :running
|
28
|
+
end
|
29
|
+
|
30
|
+
def success?
|
31
|
+
state == :success
|
32
|
+
end
|
33
|
+
|
34
|
+
def error?
|
35
|
+
state == :error
|
36
|
+
end
|
37
|
+
|
38
|
+
def finished?
|
39
|
+
success? || error?
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
def start!
|
44
|
+
Monkey.logger.info "Running #{self}"
|
45
|
+
@result, @exception, @started_at, @finished_at, self.state = nil, nil, Time.now, nil, :running
|
46
|
+
end
|
47
|
+
|
48
|
+
def unstart!
|
49
|
+
@result, @exception, @started_at, @finished_at, self.state = nil, nil, nil, nil, nil
|
50
|
+
end
|
51
|
+
|
52
|
+
def success!(result = nil)
|
53
|
+
@result, @exception, @finished_at, self.state = result, nil, Time.now, :success
|
54
|
+
Monkey.logger.info "Job #{self} succeeded after #{finished_at - started_at} sec."
|
55
|
+
end
|
56
|
+
|
57
|
+
def error!(exception = nil)
|
58
|
+
@result, @exception, @finished_at, self.state = nil, exception, Time.now, :error
|
59
|
+
Monkey.logger.error(exception) if exception
|
60
|
+
Monkey.logger.error "Job #{self} failed after #{finished_at - started_at} sec."
|
61
|
+
end
|
62
|
+
|
63
|
+
attr_writer :state
|
64
|
+
|
65
|
+
class Impl < Cookie
|
66
|
+
attr_reader :job
|
67
|
+
|
68
|
+
def initialize(job)
|
69
|
+
@job = job
|
70
|
+
end
|
71
|
+
|
72
|
+
def to_s
|
73
|
+
job.to_s
|
74
|
+
end
|
75
|
+
|
76
|
+
def to_proc
|
77
|
+
method(:run).to_proc
|
78
|
+
end
|
79
|
+
|
80
|
+
def submit(job, options = {})
|
81
|
+
client.submit(job, {:queue => queue}.update(options))
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
def state=(v)
|
86
|
+
was = state
|
87
|
+
super.tap { on_state_changed(was) }
|
88
|
+
end
|
89
|
+
|
90
|
+
def on_state_changed(was)
|
91
|
+
end
|
92
|
+
|
93
|
+
def run
|
94
|
+
start!
|
95
|
+
success!(job.to_proc.call(self))
|
96
|
+
rescue
|
97
|
+
error!($!)
|
98
|
+
end
|
99
|
+
end
|
18
100
|
end
|
19
101
|
end
|
20
102
|
end
|
data/lib/monkeyjob/juggler.rb
CHANGED
@@ -1,19 +1,25 @@
|
|
1
1
|
module Monkey
|
2
2
|
class Juggler
|
3
|
-
def initialize
|
4
|
-
@broker =
|
5
|
-
@runner =
|
3
|
+
def initialize(options = {})
|
4
|
+
@broker = options[:broker] || raise(ArgumentError ':broker is mandatory')
|
5
|
+
@runner = options[:runner] || raise(ArgumentError ':runner is mandatory')
|
6
6
|
end
|
7
7
|
|
8
8
|
def start
|
9
|
-
@broker.start
|
10
|
-
|
11
|
-
|
9
|
+
@broker.start {|cookie| @runner.run?(cookie) }
|
10
|
+
self
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
start
|
15
|
+
ensure
|
16
|
+
close
|
12
17
|
end
|
13
18
|
|
14
19
|
def close
|
15
20
|
@broker.close
|
16
21
|
@runner.close
|
22
|
+
self
|
17
23
|
end
|
18
24
|
end
|
19
25
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Monkey
|
2
|
+
module LoggerPatch
|
3
|
+
# def included?(other)
|
4
|
+
# other.send :alias_method, :global_level, :level
|
5
|
+
# other.send :alias_method, :global_level=, :level=
|
6
|
+
# end
|
7
|
+
|
8
|
+
# def level
|
9
|
+
# Thread.current["logger[#{object_id}].level"] || global_level
|
10
|
+
# end
|
11
|
+
|
12
|
+
# def level=(level)
|
13
|
+
# Thread.current["logger[#{object_id}].level"]
|
14
|
+
# end
|
15
|
+
|
16
|
+
def with_level(temp_level)
|
17
|
+
return yield old_level unless silencer
|
18
|
+
begin
|
19
|
+
old_level, self.level = self.level, temp_level
|
20
|
+
yield old_level
|
21
|
+
ensure
|
22
|
+
self.level = old_level
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def hush(temp_level = Logger::ERROR, &block)
|
27
|
+
with_level(temp_level, &block)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
[Logger, ActiveSupport::BufferedLogger].each {|m| m.send :include, LoggerPatch }
|
32
|
+
end
|
33
|
+
|
data/lib/monkeyjob/runner.rb
CHANGED
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'monitor'
|
2
|
+
|
3
|
+
module Monkey
|
4
|
+
class Runner
|
5
|
+
class Fork < Runner
|
6
|
+
autoload :Rails, 'monkeyjob/runner/fork/rails'
|
7
|
+
|
8
|
+
def initialize(options = {})
|
9
|
+
extend MonitorMixin
|
10
|
+
@max = options[:max] || 4
|
11
|
+
@workers = Set.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def run(cookie)
|
15
|
+
run_core(:run_local, cookie)
|
16
|
+
end
|
17
|
+
|
18
|
+
def run?(cookie)
|
19
|
+
run_core(:run_local?, cookie)
|
20
|
+
end
|
21
|
+
|
22
|
+
def run_local(*args)
|
23
|
+
raise ArgumentError, 'block must be provided' unless block_given?
|
24
|
+
pid = fork do
|
25
|
+
trap('TERM') do
|
26
|
+
Monkey.logger.warn "Process #{Process.pid} received SIGTERM, will exit!"
|
27
|
+
exit!(2)
|
28
|
+
end
|
29
|
+
ActiveRecord::Base.spawn_reconnect
|
30
|
+
begin
|
31
|
+
yield(*args)
|
32
|
+
exit!(0)
|
33
|
+
rescue
|
34
|
+
Monkey.logger.error $!
|
35
|
+
exit!(1)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
(spid = Process.detach(pid))[:pid] = pid
|
39
|
+
spid
|
40
|
+
end
|
41
|
+
|
42
|
+
def run_local?(*args, &block)
|
43
|
+
synchronize do
|
44
|
+
return nil unless @workers.size < @max
|
45
|
+
@sweeper ||= Thread.new { loop { sleep(1); synchronize { sweep } } }
|
46
|
+
run_local(*args, &block).tap {|spid| @workers << spid }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def close
|
51
|
+
synchronize do
|
52
|
+
@sweeper.try :kill
|
53
|
+
@workers.each do |spid|
|
54
|
+
begin
|
55
|
+
Monkey.logger.warn { "Killing process #{spid[:pid]}" }
|
56
|
+
Process.kill('TERM', spid[:pid])
|
57
|
+
rescue
|
58
|
+
Monkey.logger.debug $!
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
def run_core(method, cookie)
|
66
|
+
synchronize do
|
67
|
+
send(method, cookie, &cookie.job).tap do |spid|
|
68
|
+
(spid[:cookie] = cookie).send(:start!) if spid
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def sweep
|
74
|
+
@workers.delete_if {|spid| finish!(spid) }
|
75
|
+
rescue
|
76
|
+
Monkey.logger.error($!)
|
77
|
+
end
|
78
|
+
|
79
|
+
def finish!(spid)
|
80
|
+
!spid.alive?.tap {|alive| spid[:cookie].try(spid.value.success? ? :success! : :error!) unless alive }
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Monkey
|
2
|
+
class Runner
|
3
|
+
class Fork
|
4
|
+
class Rails < Fork
|
5
|
+
def run_local(*args)
|
6
|
+
super(*args) do
|
7
|
+
begin
|
8
|
+
ActiveRecord::Base.dirty_reconnect!
|
9
|
+
yield(*args)
|
10
|
+
ensure
|
11
|
+
ActiveRecord::Base.connection_handler.clear_all_connections!
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class Monkey::Runner::Fork::Rails
|
21
|
+
module ActiveRecord
|
22
|
+
module BasePatch
|
23
|
+
def dirty_reconnect!
|
24
|
+
@@connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
|
25
|
+
establish_connection
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
::ActiveRecord::Base.extend Monkey::Runner::Fork::Rails::ActiveRecord::BasePatch
|
32
|
+
|
@@ -3,17 +3,32 @@ require 'threadpool'
|
|
3
3
|
module Monkey
|
4
4
|
class Runner
|
5
5
|
class ThreadPool < Runner
|
6
|
-
def initialize
|
7
|
-
@pool = ::ThreadPool.new
|
6
|
+
def initialize(options = {})
|
7
|
+
@pool = ::ThreadPool.new(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def run(cookie)
|
11
|
+
run_local(&cookie)
|
8
12
|
end
|
9
13
|
|
10
|
-
def
|
11
|
-
|
14
|
+
def run?(cookie)
|
15
|
+
run_local?(&cookie)
|
16
|
+
end
|
17
|
+
|
18
|
+
def run_local?(*args, &block)
|
19
|
+
@pool.run(*args, &block)
|
20
|
+
self
|
12
21
|
end
|
13
22
|
|
14
|
-
def
|
15
|
-
@pool.
|
23
|
+
def run_local?(*args, &block)
|
24
|
+
@pool.run?(*args, &block) && self
|
25
|
+
end
|
26
|
+
|
27
|
+
def close
|
28
|
+
@pool.close
|
29
|
+
self
|
16
30
|
end
|
17
31
|
end
|
18
32
|
end
|
19
33
|
end
|
34
|
+
|
data/rails/init.rb
ADDED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pipa-monkeyjob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Igor Gunko
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-06-
|
12
|
+
date: 2009-06-17 00:00:00 -07:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -22,26 +22,6 @@ dependencies:
|
|
22
22
|
- !ruby/object:Gem::Version
|
23
23
|
version: 2.2.2
|
24
24
|
version:
|
25
|
-
- !ruby/object:Gem::Dependency
|
26
|
-
name: activerecord
|
27
|
-
type: :runtime
|
28
|
-
version_requirement:
|
29
|
-
version_requirements: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 2.2.2
|
34
|
-
version:
|
35
|
-
- !ruby/object:Gem::Dependency
|
36
|
-
name: pipa-threadpool
|
37
|
-
type: :runtime
|
38
|
-
version_requirement:
|
39
|
-
version_requirements: !ruby/object:Gem::Requirement
|
40
|
-
requirements:
|
41
|
-
- - ">="
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
version: 0.2.1
|
44
|
-
version:
|
45
25
|
- !ruby/object:Gem::Dependency
|
46
26
|
name: thoughtbot-shoulda
|
47
27
|
type: :development
|
@@ -77,6 +57,7 @@ files:
|
|
77
57
|
- Rakefile
|
78
58
|
- lib/monkeyjob.rb
|
79
59
|
- lib/pipa-monkeyjob.rb
|
60
|
+
- lib/monkeyjob/logger.rb
|
80
61
|
- lib/monkeyjob/job.rb
|
81
62
|
- lib/monkeyjob/juggler.rb
|
82
63
|
- lib/monkeyjob/client.rb
|
@@ -86,6 +67,9 @@ files:
|
|
86
67
|
- lib/monkeyjob/broker/active_record.rb
|
87
68
|
- lib/monkeyjob/broker/active_record/job_model.rb
|
88
69
|
- lib/monkeyjob/runner/thread_pool.rb
|
70
|
+
- lib/monkeyjob/runner/fork.rb
|
71
|
+
- lib/monkeyjob/runner/fork/rails.rb
|
72
|
+
- rails/init.rb
|
89
73
|
has_rdoc: true
|
90
74
|
homepage: http://github.com/pipa/monkeyjob
|
91
75
|
post_install_message:
|