active_worker 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rvmrc +1 -0
- data/Gemfile +22 -0
- data/Gemfile.lock +77 -0
- data/LICENSE.txt +20 -0
- data/README.rdoc +19 -0
- data/Rakefile +46 -0
- data/VERSION +1 -0
- data/active_worker.gemspec +108 -0
- data/lib/active_worker.rb +29 -0
- data/lib/active_worker/behavior/acts_as_root_object.rb +80 -0
- data/lib/active_worker/behavior/can_be_notified.rb +23 -0
- data/lib/active_worker/behavior/create_from_error.rb +21 -0
- data/lib/active_worker/behavior/execute_concurrently.rb +142 -0
- data/lib/active_worker/behavior/has_modes.rb +44 -0
- data/lib/active_worker/behavior/has_root_object.rb +50 -0
- data/lib/active_worker/behavior/hashable.rb +79 -0
- data/lib/active_worker/configuration.rb +143 -0
- data/lib/active_worker/controller.rb +112 -0
- data/lib/active_worker/event.rb +68 -0
- data/lib/active_worker/expandable.rb +77 -0
- data/lib/active_worker/failure_event.rb +16 -0
- data/lib/active_worker/finished_event.rb +9 -0
- data/lib/active_worker/host_information.rb +13 -0
- data/lib/active_worker/job_queue/job_executer.rb +52 -0
- data/lib/active_worker/job_queue/queue_manager.rb +46 -0
- data/lib/active_worker/job_queue/run_remotely.rb +52 -0
- data/lib/active_worker/modes_map.rb +37 -0
- data/lib/active_worker/notification_event.rb +9 -0
- data/lib/active_worker/parent_event.rb +5 -0
- data/lib/active_worker/started_event.rb +10 -0
- data/lib/active_worker/templatable.rb +46 -0
- data/lib/active_worker/template.rb +41 -0
- data/lib/active_worker/termination_event.rb +21 -0
- data/test/mongoid.yml +28 -0
- data/test/test_acts_as_root_object.rb +123 -0
- data/test/test_can_be_notified.rb +44 -0
- data/test/test_configuration.rb +281 -0
- data/test/test_controller.rb +205 -0
- data/test/test_event.rb +75 -0
- data/test/test_execute_concurrently.rb +134 -0
- data/test/test_expandable.rb +113 -0
- data/test/test_failure_event.rb +69 -0
- data/test/test_finished_event.rb +35 -0
- data/test/test_has_modes.rb +56 -0
- data/test/test_helper.rb +120 -0
- data/test/test_integration.rb +56 -0
- data/test/test_job_executer.rb +65 -0
- data/test/test_queue_manager.rb +106 -0
- data/test/test_run_remotely.rb +63 -0
- data/test/test_started_event.rb +23 -0
- data/test/test_templatable.rb +45 -0
- data/test/test_template.rb +29 -0
- data/test/test_termination_event.rb +28 -0
- metadata +201 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module CreateFromError
|
4
|
+
|
5
|
+
def self.extended(base)
|
6
|
+
base.field :stack_trace
|
7
|
+
base.field :error_type
|
8
|
+
end
|
9
|
+
|
10
|
+
def create_error_from_configuration(configuration, error)
|
11
|
+
constructor_options = {
|
12
|
+
:message => "#{configuration.event_name} FAILED: #{error.message}",
|
13
|
+
:stack_trace => error.backtrace.join("\n"),
|
14
|
+
:configuration => configuration,
|
15
|
+
:error_type => error.class.name
|
16
|
+
}
|
17
|
+
create! constructor_options
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
|
3
|
+
THREADED_MODE = :local_worker_thread
|
4
|
+
FORKING_MODE = :local_worker_fork
|
5
|
+
DEFAULT_MODE = FORKING_MODE
|
6
|
+
|
7
|
+
PARENT = :parent
|
8
|
+
FORKED = :forked
|
9
|
+
|
10
|
+
module Behavior
|
11
|
+
module ExecuteConcurrently
|
12
|
+
|
13
|
+
def local_worker_mode=(mode)
|
14
|
+
@@local_worker_mode = mode
|
15
|
+
end
|
16
|
+
|
17
|
+
def local_worker_mode
|
18
|
+
@@local_worker_mode ||= FORKING_MODE
|
19
|
+
end
|
20
|
+
|
21
|
+
def threaded?
|
22
|
+
local_worker_mode == THREADED_MODE
|
23
|
+
end
|
24
|
+
|
25
|
+
def forking?
|
26
|
+
local_worker_mode == FORKING_MODE
|
27
|
+
end
|
28
|
+
|
29
|
+
def role
|
30
|
+
@@role ||= PARENT
|
31
|
+
end
|
32
|
+
|
33
|
+
def role=(role)
|
34
|
+
@@role = role
|
35
|
+
end
|
36
|
+
|
37
|
+
def parent?
|
38
|
+
role == PARENT
|
39
|
+
end
|
40
|
+
|
41
|
+
def forked?
|
42
|
+
role == FORKED
|
43
|
+
end
|
44
|
+
|
45
|
+
def pids
|
46
|
+
@pids ||= []
|
47
|
+
end
|
48
|
+
|
49
|
+
def threads
|
50
|
+
@threads ||= []
|
51
|
+
end
|
52
|
+
|
53
|
+
def execute_concurrently(params)
|
54
|
+
new_threads = params.map do |param|
|
55
|
+
case local_worker_mode
|
56
|
+
when THREADED_MODE
|
57
|
+
execute_thread param
|
58
|
+
when FORKING_MODE
|
59
|
+
execute_fork param
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
threads.concat new_threads
|
64
|
+
new_threads
|
65
|
+
end
|
66
|
+
|
67
|
+
def wait_for_children
|
68
|
+
threads.each do |thread|
|
69
|
+
thread.join if thread
|
70
|
+
end
|
71
|
+
|
72
|
+
cleanup_after_children
|
73
|
+
end
|
74
|
+
|
75
|
+
def cleanup_after_children
|
76
|
+
@pids = []
|
77
|
+
@threads = []
|
78
|
+
end
|
79
|
+
|
80
|
+
def kill_children
|
81
|
+
pids.each do |pid|
|
82
|
+
begin
|
83
|
+
Process.kill("TERM", pid) if pid
|
84
|
+
rescue Errno::ESRCH
|
85
|
+
puts "PID: #{pid} did not exist when we went to kill it"
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def execute_thread(param)
|
91
|
+
Thread.new do
|
92
|
+
in_thread(param)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def execute_fork(param)
|
97
|
+
pid = fork do
|
98
|
+
in_fork(param)
|
99
|
+
end
|
100
|
+
pids << pid
|
101
|
+
Process.detach(pid)
|
102
|
+
end
|
103
|
+
|
104
|
+
def in_thread(param)
|
105
|
+
execute(param)
|
106
|
+
end
|
107
|
+
|
108
|
+
def in_fork(param)
|
109
|
+
after_fork(param)
|
110
|
+
execute(param)
|
111
|
+
rescue SignalException
|
112
|
+
self.handle_termination([param.to_param])
|
113
|
+
exit
|
114
|
+
rescue Exception => e
|
115
|
+
self.handle_error(e, :in_fork, [param.to_param])
|
116
|
+
end
|
117
|
+
|
118
|
+
def after_fork(param)
|
119
|
+
self.role = FORKED
|
120
|
+
cleanup_after_children
|
121
|
+
set_process_name(param)
|
122
|
+
reset_mongoid
|
123
|
+
reset_resque
|
124
|
+
end
|
125
|
+
|
126
|
+
def set_process_name(param)
|
127
|
+
$0 = "ActiveWorker Forked from #{Process.ppid} for #{param}"
|
128
|
+
end
|
129
|
+
|
130
|
+
def reset_mongoid
|
131
|
+
Mongoid::Sessions.clear
|
132
|
+
end
|
133
|
+
|
134
|
+
def reset_resque
|
135
|
+
Resque.redis.client.reconnect
|
136
|
+
trap("TERM", "DEFAULT")
|
137
|
+
end
|
138
|
+
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module HasModes
|
4
|
+
|
5
|
+
class ModeNotSupportedException < StandardError
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassExtensions
|
9
|
+
|
10
|
+
def modes_map
|
11
|
+
@modes_map ||= ModesMap.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def modes
|
15
|
+
modes_map.modes
|
16
|
+
end
|
17
|
+
|
18
|
+
def add_mode(*args)
|
19
|
+
modes_map.add_mode(*args)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.included(base)
|
24
|
+
base.extend(ClassExtensions)
|
25
|
+
base.template_field :mode
|
26
|
+
base.before_save :set_mode_defined_fields
|
27
|
+
end
|
28
|
+
|
29
|
+
def set_mode_defined_fields
|
30
|
+
unless mode.nil? || mode.empty?
|
31
|
+
if self.class.modes_map.supports? mode
|
32
|
+
self.class.modes_map.mode(mode).each_pair do |field, value|
|
33
|
+
write_attribute(field, value) unless read_attribute(field)
|
34
|
+
end
|
35
|
+
else
|
36
|
+
raise ModeNotSupportedException, "Mode \"#{mode}\" not in Modes List: #{self.class.modes} for #{self.class}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module HasRootObject
|
4
|
+
|
5
|
+
module ClassExtensions
|
6
|
+
def root_object_relation(relation)
|
7
|
+
belongs_to :root_object, :polymorphic => true, inverse_of: relation
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.included(base)
|
12
|
+
base.extend(ClassExtensions)
|
13
|
+
base.before_save :set_root_object
|
14
|
+
base.index({:root_object_id => -1}, {:background => true})
|
15
|
+
end
|
16
|
+
|
17
|
+
def root_owner
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def notify_root_of_child_started
|
22
|
+
if root_object
|
23
|
+
root_object.child_started
|
24
|
+
root_object.save!
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def notify_root_of_finished
|
29
|
+
if root_object
|
30
|
+
root_object.child_finished
|
31
|
+
root_object.save!
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_root_object_id
|
36
|
+
return root_object_id if root_object_id
|
37
|
+
set_root_object
|
38
|
+
root_object_id
|
39
|
+
end
|
40
|
+
|
41
|
+
def set_root_object
|
42
|
+
return unless root_owner
|
43
|
+
self.root_object_id = root_owner.get_root_object_id
|
44
|
+
self.root_object_type = root_owner.root_object_type
|
45
|
+
true
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
module Behavior
|
3
|
+
module Hashable
|
4
|
+
|
5
|
+
def get_as_flat_hash_by_root_object(root_object)
|
6
|
+
configs = []
|
7
|
+
col = get_mongoid_collection
|
8
|
+
col.find("root_object_id" => root_object.id).each do |config|
|
9
|
+
canonicalize(config, col)
|
10
|
+
configs << config
|
11
|
+
end
|
12
|
+
configs
|
13
|
+
end
|
14
|
+
|
15
|
+
def get_as_hash_by_root_object(root_object)
|
16
|
+
configs = []
|
17
|
+
col = get_mongoid_collection
|
18
|
+
col.find("root_object_id" => root_object.id, "parent_configuration_id" => nil).each do |config|
|
19
|
+
config["configurations"] = get_children_for(col, config["_id"])
|
20
|
+
canonicalize(config, col)
|
21
|
+
configs << config
|
22
|
+
end
|
23
|
+
configs
|
24
|
+
end
|
25
|
+
|
26
|
+
def get_children_for(col, parent_configuration_id)
|
27
|
+
configs = []
|
28
|
+
col.find("parent_configuration_id" => parent_configuration_id).each do |config|
|
29
|
+
config["configurations"] = get_children_for(col, config["_id"])
|
30
|
+
canonicalize(config, col)
|
31
|
+
configs << config
|
32
|
+
end
|
33
|
+
configs
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
|
38
|
+
def get_renderable_hash_by_root_object(root_object)
|
39
|
+
configs = []
|
40
|
+
col = get_mongoid_collection
|
41
|
+
col.find("root_object_id" => root_object.id, "parent_configuration_id" => nil, "renderable" => true).each do |config|
|
42
|
+
config["configurations"] = get_renderable_children_for(col, config["_id"])
|
43
|
+
canonicalize(config, col)
|
44
|
+
configs << config
|
45
|
+
end
|
46
|
+
configs
|
47
|
+
end
|
48
|
+
|
49
|
+
def renderable_hash_for_configuration(configuration_id)
|
50
|
+
col = get_mongoid_collection
|
51
|
+
config = col.find("_id" => configuration_id).first
|
52
|
+
config["configurations"] = get_renderable_children_for(col, config["_id"])
|
53
|
+
canonicalize(config, col)
|
54
|
+
config
|
55
|
+
end
|
56
|
+
|
57
|
+
def get_renderable_children_for(col, parent_configuration_id)
|
58
|
+
configs = []
|
59
|
+
col.find("parent_configuration_id" => parent_configuration_id, "renderable" => true).each do |config|
|
60
|
+
config["configurations"] = get_renderable_children_for(col, config["_id"])
|
61
|
+
canonicalize(config, col)
|
62
|
+
configs << config
|
63
|
+
end
|
64
|
+
configs
|
65
|
+
end
|
66
|
+
|
67
|
+
def canonicalize(config, col)
|
68
|
+
config["_type"] ||= col.name.classify
|
69
|
+
config["_id"] = config["_id"].to_s
|
70
|
+
end
|
71
|
+
|
72
|
+
|
73
|
+
def get_mongoid_collection
|
74
|
+
Mongoid.default_session.collections.select {|c| c.name == self.collection_name.to_s }.first
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
module ActiveWorker
|
2
|
+
class Configuration
|
3
|
+
include Mongoid::Document
|
4
|
+
include Behavior::HasRootObject
|
5
|
+
extend Behavior::Hashable
|
6
|
+
POLLING_INTERVAL = 0.1
|
7
|
+
|
8
|
+
has_many :events, :class_name => "ActiveWorker::Event"
|
9
|
+
|
10
|
+
has_many :configurations,
|
11
|
+
:as => :parent_configuration,
|
12
|
+
:class_name => 'ActiveWorker::Configuration',
|
13
|
+
:autosave => true
|
14
|
+
|
15
|
+
belongs_to :parent_configuration,
|
16
|
+
:polymorphic => true
|
17
|
+
|
18
|
+
alias_method :root_owner, :parent_configuration
|
19
|
+
root_object_relation :configurations
|
20
|
+
|
21
|
+
field :polling_interval, :type => Integer, :default => 1
|
22
|
+
field :renderable, :type => Boolean, :default => false
|
23
|
+
field :flags, type: Hash, default: {}
|
24
|
+
|
25
|
+
before_save :set_flags
|
26
|
+
|
27
|
+
scope :mine, ->(parent_config) { where(parent_configuration_id: parent_config.id) }
|
28
|
+
|
29
|
+
def launch
|
30
|
+
configurations = expand_for_workers
|
31
|
+
configurations.each(&:enqueue_job)
|
32
|
+
configurations += collect_after_launch_configurations(configurations)
|
33
|
+
configurations
|
34
|
+
end
|
35
|
+
|
36
|
+
def collect_after_launch_configurations(configurations)
|
37
|
+
self.class.after_launch_methods.map { |method| send(method, configurations) }.flatten
|
38
|
+
end
|
39
|
+
|
40
|
+
def expand_for_workers
|
41
|
+
[self]
|
42
|
+
end
|
43
|
+
|
44
|
+
def expand_for_threads
|
45
|
+
[self]
|
46
|
+
end
|
47
|
+
|
48
|
+
def configurations_for_events
|
49
|
+
[self]
|
50
|
+
end
|
51
|
+
|
52
|
+
def enqueue_job
|
53
|
+
self.class.controller_class.run_remotely.execute_expanded(self.id)
|
54
|
+
self
|
55
|
+
end
|
56
|
+
|
57
|
+
def supported_child_configurations
|
58
|
+
[]
|
59
|
+
end
|
60
|
+
|
61
|
+
def defined_fields
|
62
|
+
attributes.select { |k, v| self.class.config_fields.include? k.to_sym }
|
63
|
+
end
|
64
|
+
|
65
|
+
def event_name
|
66
|
+
parts = self.class.name.split("::")
|
67
|
+
parts.pop
|
68
|
+
parts.join(" ")
|
69
|
+
end
|
70
|
+
|
71
|
+
def renderable_configurations
|
72
|
+
configurations.select { |c| c.renderable }
|
73
|
+
end
|
74
|
+
|
75
|
+
def completed?
|
76
|
+
FinishedEvent.where(configuration_id: id).count > 0
|
77
|
+
end
|
78
|
+
|
79
|
+
def wait_until_completed
|
80
|
+
sleep(POLLING_INTERVAL) until completed?
|
81
|
+
end
|
82
|
+
|
83
|
+
def started
|
84
|
+
StartedEvent.create(configuration: self)
|
85
|
+
end
|
86
|
+
|
87
|
+
def finished
|
88
|
+
FinishedEvent.create(configuration: self)
|
89
|
+
end
|
90
|
+
|
91
|
+
def notify
|
92
|
+
NotificationEvent.create(configuration: self)
|
93
|
+
end
|
94
|
+
|
95
|
+
def notified?
|
96
|
+
NotificationEvent.where(configuration_id: id).count > 0
|
97
|
+
end
|
98
|
+
|
99
|
+
def set_flags
|
100
|
+
self.flags = parent_configuration.flags if parent_configuration
|
101
|
+
true
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.controller_class
|
105
|
+
"#{self.parent}::Controller".constantize
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.display_name
|
109
|
+
name.split("::").join(" ")
|
110
|
+
end
|
111
|
+
|
112
|
+
def self.css_name
|
113
|
+
name.split("::").join("_")
|
114
|
+
end
|
115
|
+
|
116
|
+
def self.template_field(name, *args)
|
117
|
+
config_field(name, *args)
|
118
|
+
template_fields << name
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.config_field(name, *args)
|
122
|
+
field name, *args
|
123
|
+
config_fields << name
|
124
|
+
end
|
125
|
+
|
126
|
+
def self.template_fields
|
127
|
+
@template_fields ||= []
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.config_fields
|
131
|
+
@config_fields ||= []
|
132
|
+
end
|
133
|
+
|
134
|
+
def self.after_launch(method)
|
135
|
+
after_launch_methods << method
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.after_launch_methods
|
139
|
+
@after_launch_methods ||= []
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
143
|
+
end
|