active_worker 0.50.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.rvmrc +1 -0
- data/Gemfile +22 -0
- data/Gemfile.lock +77 -0
- data/LICENSE.txt +20 -0
- data/README.rdoc +19 -0
- data/Rakefile +46 -0
- data/VERSION +1 -0
- data/active_worker.gemspec +108 -0
- data/lib/active_worker.rb +29 -0
- data/lib/active_worker/behavior/acts_as_root_object.rb +80 -0
- data/lib/active_worker/behavior/can_be_notified.rb +23 -0
- data/lib/active_worker/behavior/create_from_error.rb +21 -0
- data/lib/active_worker/behavior/execute_concurrently.rb +142 -0
- data/lib/active_worker/behavior/has_modes.rb +44 -0
- data/lib/active_worker/behavior/has_root_object.rb +50 -0
- data/lib/active_worker/behavior/hashable.rb +79 -0
- data/lib/active_worker/configuration.rb +143 -0
- data/lib/active_worker/controller.rb +112 -0
- data/lib/active_worker/event.rb +68 -0
- data/lib/active_worker/expandable.rb +77 -0
- data/lib/active_worker/failure_event.rb +16 -0
- data/lib/active_worker/finished_event.rb +9 -0
- data/lib/active_worker/host_information.rb +13 -0
- data/lib/active_worker/job_queue/job_executer.rb +52 -0
- data/lib/active_worker/job_queue/queue_manager.rb +46 -0
- data/lib/active_worker/job_queue/run_remotely.rb +52 -0
- data/lib/active_worker/modes_map.rb +37 -0
- data/lib/active_worker/notification_event.rb +9 -0
- data/lib/active_worker/parent_event.rb +5 -0
- data/lib/active_worker/started_event.rb +10 -0
- data/lib/active_worker/templatable.rb +46 -0
- data/lib/active_worker/template.rb +41 -0
- data/lib/active_worker/termination_event.rb +21 -0
- data/test/mongoid.yml +28 -0
- data/test/test_acts_as_root_object.rb +123 -0
- data/test/test_can_be_notified.rb +44 -0
- data/test/test_configuration.rb +281 -0
- data/test/test_controller.rb +205 -0
- data/test/test_event.rb +75 -0
- data/test/test_execute_concurrently.rb +134 -0
- data/test/test_expandable.rb +113 -0
- data/test/test_failure_event.rb +69 -0
- data/test/test_finished_event.rb +35 -0
- data/test/test_has_modes.rb +56 -0
- data/test/test_helper.rb +120 -0
- data/test/test_integration.rb +56 -0
- data/test/test_job_executer.rb +65 -0
- data/test/test_queue_manager.rb +106 -0
- data/test/test_run_remotely.rb +63 -0
- data/test/test_started_event.rb +23 -0
- data/test/test_templatable.rb +45 -0
- data/test/test_template.rb +29 -0
- data/test/test_termination_event.rb +28 -0
- metadata +201 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module CreateFromError
|
4
|
+
|
5
|
+
def self.extended(base)
|
6
|
+
base.field :stack_trace
|
7
|
+
base.field :error_type
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_error_from_configuration(configuration, error)
|
11
|
+
constructor_options = {
|
12
|
+
:message => "#{configuration.event_name} FAILED: #{error.message}",
|
13
|
+
:stack_trace => error.backtrace.join("\n"),
|
14
|
+
:configuration => configuration,
|
15
|
+
:error_type => error.class.name
|
16
|
+
}
|
17
|
+
create! constructor_options
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
|
3
|
+
THREADED_MODE = :local_worker_thread
|
4
|
+
FORKING_MODE = :local_worker_fork
|
5
|
+
DEFAULT_MODE = FORKING_MODE
|
6
|
+
|
7
|
+
PARENT = :parent
|
8
|
+
FORKED = :forked
|
9
|
+
|
10
|
+
module Behavior
|
11
|
+
module ExecuteConcurrently
|
12
|
+
|
13
|
+
def local_worker_mode=(mode)
|
14
|
+
@@local_worker_mode = mode
|
15
|
+
end
|
16
|
+
|
17
|
+
def local_worker_mode
|
18
|
+
@@local_worker_mode ||= FORKING_MODE
|
19
|
+
end
|
20
|
+
|
21
|
+
def threaded?
|
22
|
+
local_worker_mode == THREADED_MODE
|
23
|
+
end
|
24
|
+
|
25
|
+
def forking?
|
26
|
+
local_worker_mode == FORKING_MODE
|
27
|
+
end
|
28
|
+
|
29
|
+
def role
|
30
|
+
@@role ||= PARENT
|
31
|
+
end
|
32
|
+
|
33
|
+
def role=(role)
|
34
|
+
@@role = role
|
35
|
+
end
|
36
|
+
|
37
|
+
def parent?
|
38
|
+
role == PARENT
|
39
|
+
end
|
40
|
+
|
41
|
+
def forked?
|
42
|
+
role == FORKED
|
43
|
+
end
|
44
|
+
|
45
|
+
def pids
|
46
|
+
@pids ||= []
|
47
|
+
end
|
48
|
+
|
49
|
+
def threads
|
50
|
+
@threads ||= []
|
51
|
+
end
|
52
|
+
|
53
|
+
def execute_concurrently(params)
|
54
|
+
new_threads = params.map do |param|
|
55
|
+
case local_worker_mode
|
56
|
+
when THREADED_MODE
|
57
|
+
execute_thread param
|
58
|
+
when FORKING_MODE
|
59
|
+
execute_fork param
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
threads.concat new_threads
|
64
|
+
new_threads
|
65
|
+
end
|
66
|
+
|
67
|
+
def wait_for_children
|
68
|
+
threads.each do |thread|
|
69
|
+
thread.join if thread
|
70
|
+
end
|
71
|
+
|
72
|
+
cleanup_after_children
|
73
|
+
end
|
74
|
+
|
75
|
+
def cleanup_after_children
|
76
|
+
@pids = []
|
77
|
+
@threads = []
|
78
|
+
end
|
79
|
+
|
80
|
+
def kill_children
|
81
|
+
pids.each do |pid|
|
82
|
+
begin
|
83
|
+
Process.kill("TERM", pid) if pid
|
84
|
+
rescue Errno::ESRCH
|
85
|
+
puts "PID: #{pid} did not exist when we went to kill it"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def execute_thread(param)
|
91
|
+
Thread.new do
|
92
|
+
in_thread(param)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def execute_fork(param)
|
97
|
+
pid = fork do
|
98
|
+
in_fork(param)
|
99
|
+
end
|
100
|
+
pids << pid
|
101
|
+
Process.detach(pid)
|
102
|
+
end
|
103
|
+
|
104
|
+
def in_thread(param)
|
105
|
+
execute(param)
|
106
|
+
end
|
107
|
+
|
108
|
+
def in_fork(param)
|
109
|
+
after_fork(param)
|
110
|
+
execute(param)
|
111
|
+
rescue SignalException
|
112
|
+
self.handle_termination([param.to_param])
|
113
|
+
exit
|
114
|
+
rescue Exception => e
|
115
|
+
self.handle_error(e, :in_fork, [param.to_param])
|
116
|
+
end
|
117
|
+
|
118
|
+
def after_fork(param)
|
119
|
+
self.role = FORKED
|
120
|
+
cleanup_after_children
|
121
|
+
set_process_name(param)
|
122
|
+
reset_mongoid
|
123
|
+
reset_resque
|
124
|
+
end
|
125
|
+
|
126
|
+
def set_process_name(param)
|
127
|
+
$0 = "ActiveWorker Forked from #{Process.ppid} for #{param}"
|
128
|
+
end
|
129
|
+
|
130
|
+
def reset_mongoid
|
131
|
+
Mongoid::Sessions.clear
|
132
|
+
end
|
133
|
+
|
134
|
+
def reset_resque
|
135
|
+
Resque.redis.client.reconnect
|
136
|
+
trap("TERM", "DEFAULT")
|
137
|
+
end
|
138
|
+
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module HasModes
|
4
|
+
|
5
|
+
class ModeNotSupportedException < StandardError
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassExtensions
|
9
|
+
|
10
|
+
def modes_map
|
11
|
+
@modes_map ||= ModesMap.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def modes
|
15
|
+
modes_map.modes
|
16
|
+
end
|
17
|
+
|
18
|
+
def add_mode(*args)
|
19
|
+
modes_map.add_mode(*args)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.included(base)
|
24
|
+
base.extend(ClassExtensions)
|
25
|
+
base.template_field :mode
|
26
|
+
base.before_save :set_mode_defined_fields
|
27
|
+
end
|
28
|
+
|
29
|
+
def set_mode_defined_fields
|
30
|
+
unless mode.nil? || mode.empty?
|
31
|
+
if self.class.modes_map.supports? mode
|
32
|
+
self.class.modes_map.mode(mode).each_pair do |field, value|
|
33
|
+
write_attribute(field, value) unless read_attribute(field)
|
34
|
+
end
|
35
|
+
else
|
36
|
+
raise ModeNotSupportedException, "Mode \"#{mode}\" not in Modes List: #{self.class.modes} for #{self.class}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module HasRootObject
|
4
|
+
|
5
|
+
module ClassExtensions
|
6
|
+
def root_object_relation(relation)
|
7
|
+
belongs_to :root_object, :polymorphic => true, inverse_of: relation
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.included(base)
|
12
|
+
base.extend(ClassExtensions)
|
13
|
+
base.before_save :set_root_object
|
14
|
+
base.index({:root_object_id => -1}, {:background => true})
|
15
|
+
end
|
16
|
+
|
17
|
+
def root_owner
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def notify_root_of_child_started
|
22
|
+
if root_object
|
23
|
+
root_object.child_started
|
24
|
+
root_object.save!
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def notify_root_of_finished
|
29
|
+
if root_object
|
30
|
+
root_object.child_finished
|
31
|
+
root_object.save!
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_root_object_id
|
36
|
+
return root_object_id if root_object_id
|
37
|
+
set_root_object
|
38
|
+
root_object_id
|
39
|
+
end
|
40
|
+
|
41
|
+
def set_root_object
|
42
|
+
return unless root_owner
|
43
|
+
self.root_object_id = root_owner.get_root_object_id
|
44
|
+
self.root_object_type = root_owner.root_object_type
|
45
|
+
true
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module Hashable
|
4
|
+
|
5
|
+
def get_as_flat_hash_by_root_object(root_object)
|
6
|
+
configs = []
|
7
|
+
col = get_mongoid_collection
|
8
|
+
col.find("root_object_id" => root_object.id).each do |config|
|
9
|
+
canonicalize(config, col)
|
10
|
+
configs << config
|
11
|
+
end
|
12
|
+
configs
|
13
|
+
end
|
14
|
+
|
15
|
+
def get_as_hash_by_root_object(root_object)
|
16
|
+
configs = []
|
17
|
+
col = get_mongoid_collection
|
18
|
+
col.find("root_object_id" => root_object.id, "parent_configuration_id" => nil).each do |config|
|
19
|
+
config["configurations"] = get_children_for(col, config["_id"])
|
20
|
+
canonicalize(config, col)
|
21
|
+
configs << config
|
22
|
+
end
|
23
|
+
configs
|
24
|
+
end
|
25
|
+
|
26
|
+
def get_children_for(col, parent_configuration_id)
|
27
|
+
configs = []
|
28
|
+
col.find("parent_configuration_id" => parent_configuration_id).each do |config|
|
29
|
+
config["configurations"] = get_children_for(col, config["_id"])
|
30
|
+
canonicalize(config, col)
|
31
|
+
configs << config
|
32
|
+
end
|
33
|
+
configs
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
|
38
|
+
def get_renderable_hash_by_root_object(root_object)
|
39
|
+
configs = []
|
40
|
+
col = get_mongoid_collection
|
41
|
+
col.find("root_object_id" => root_object.id, "parent_configuration_id" => nil, "renderable" => true).each do |config|
|
42
|
+
config["configurations"] = get_renderable_children_for(col, config["_id"])
|
43
|
+
canonicalize(config, col)
|
44
|
+
configs << config
|
45
|
+
end
|
46
|
+
configs
|
47
|
+
end
|
48
|
+
|
49
|
+
def renderable_hash_for_configuration(configuration_id)
|
50
|
+
col = get_mongoid_collection
|
51
|
+
config = col.find("_id" => configuration_id).first
|
52
|
+
config["configurations"] = get_renderable_children_for(col, config["_id"])
|
53
|
+
canonicalize(config, col)
|
54
|
+
config
|
55
|
+
end
|
56
|
+
|
57
|
+
def get_renderable_children_for(col, parent_configuration_id)
|
58
|
+
configs = []
|
59
|
+
col.find("parent_configuration_id" => parent_configuration_id, "renderable" => true).each do |config|
|
60
|
+
config["configurations"] = get_renderable_children_for(col, config["_id"])
|
61
|
+
canonicalize(config, col)
|
62
|
+
configs << config
|
63
|
+
end
|
64
|
+
configs
|
65
|
+
end
|
66
|
+
|
67
|
+
def canonicalize(config, col)
|
68
|
+
config["_type"] ||= col.name.classify
|
69
|
+
config["_id"] = config["_id"].to_s
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
def get_mongoid_collection
|
74
|
+
Mongoid.default_session.collections.select {|c| c.name == self.collection_name.to_s }.first
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
class Configuration
|
3
|
+
include Mongoid::Document
|
4
|
+
include Behavior::HasRootObject
|
5
|
+
extend Behavior::Hashable
|
6
|
+
POLLING_INTERVAL = 0.1
|
7
|
+
|
8
|
+
has_many :events, :class_name => "ActiveWorker::Event"
|
9
|
+
|
10
|
+
has_many :configurations,
|
11
|
+
:as => :parent_configuration,
|
12
|
+
:class_name => 'ActiveWorker::Configuration',
|
13
|
+
:autosave => true
|
14
|
+
|
15
|
+
belongs_to :parent_configuration,
|
16
|
+
:polymorphic => true
|
17
|
+
|
18
|
+
alias_method :root_owner, :parent_configuration
|
19
|
+
root_object_relation :configurations
|
20
|
+
|
21
|
+
field :polling_interval, :type => Integer, :default => 1
|
22
|
+
field :renderable, :type => Boolean, :default => false
|
23
|
+
field :flags, type: Hash, default: {}
|
24
|
+
|
25
|
+
before_save :set_flags
|
26
|
+
|
27
|
+
scope :mine, ->(parent_config) { where(parent_configuration_id: parent_config.id) }
|
28
|
+
|
29
|
+
def launch
|
30
|
+
configurations = expand_for_workers
|
31
|
+
configurations.each(&:enqueue_job)
|
32
|
+
configurations += collect_after_launch_configurations(configurations)
|
33
|
+
configurations
|
34
|
+
end
|
35
|
+
|
36
|
+
def collect_after_launch_configurations(configurations)
|
37
|
+
self.class.after_launch_methods.map { |method| send(method, configurations) }.flatten
|
38
|
+
end
|
39
|
+
|
40
|
+
def expand_for_workers
|
41
|
+
[self]
|
42
|
+
end
|
43
|
+
|
44
|
+
def expand_for_threads
|
45
|
+
[self]
|
46
|
+
end
|
47
|
+
|
48
|
+
def configurations_for_events
|
49
|
+
[self]
|
50
|
+
end
|
51
|
+
|
52
|
+
def enqueue_job
|
53
|
+
self.class.controller_class.run_remotely.execute_expanded(self.id)
|
54
|
+
self
|
55
|
+
end
|
56
|
+
|
57
|
+
def supported_child_configurations
|
58
|
+
[]
|
59
|
+
end
|
60
|
+
|
61
|
+
def defined_fields
|
62
|
+
attributes.select { |k, v| self.class.config_fields.include? k.to_sym }
|
63
|
+
end
|
64
|
+
|
65
|
+
def event_name
|
66
|
+
parts = self.class.name.split("::")
|
67
|
+
parts.pop
|
68
|
+
parts.join(" ")
|
69
|
+
end
|
70
|
+
|
71
|
+
def renderable_configurations
|
72
|
+
configurations.select { |c| c.renderable }
|
73
|
+
end
|
74
|
+
|
75
|
+
def completed?
|
76
|
+
FinishedEvent.where(configuration_id: id).count > 0
|
77
|
+
end
|
78
|
+
|
79
|
+
def wait_until_completed
|
80
|
+
sleep(POLLING_INTERVAL) until completed?
|
81
|
+
end
|
82
|
+
|
83
|
+
def started
|
84
|
+
StartedEvent.create(configuration: self)
|
85
|
+
end
|
86
|
+
|
87
|
+
def finished
|
88
|
+
FinishedEvent.create(configuration: self)
|
89
|
+
end
|
90
|
+
|
91
|
+
def notify
|
92
|
+
NotificationEvent.create(configuration: self)
|
93
|
+
end
|
94
|
+
|
95
|
+
def notified?
|
96
|
+
NotificationEvent.where(configuration_id: id).count > 0
|
97
|
+
end
|
98
|
+
|
99
|
+
def set_flags
|
100
|
+
self.flags = parent_configuration.flags if parent_configuration
|
101
|
+
true
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.controller_class
|
105
|
+
"#{self.parent}::Controller".constantize
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.display_name
|
109
|
+
name.split("::").join(" ")
|
110
|
+
end
|
111
|
+
|
112
|
+
def self.css_name
|
113
|
+
name.split("::").join("_")
|
114
|
+
end
|
115
|
+
|
116
|
+
def self.template_field(name, *args)
|
117
|
+
config_field(name, *args)
|
118
|
+
template_fields << name
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.config_field(name, *args)
|
122
|
+
field name, *args
|
123
|
+
config_fields << name
|
124
|
+
end
|
125
|
+
|
126
|
+
def self.template_fields
|
127
|
+
@template_fields ||= []
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.config_fields
|
131
|
+
@config_fields ||= []
|
132
|
+
end
|
133
|
+
|
134
|
+
def self.after_launch(method)
|
135
|
+
after_launch_methods << method
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.after_launch_methods
|
139
|
+
@after_launch_methods ||= []
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
143
|
+
end
|