active_worker 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. data/.document +5 -0
  2. data/.rvmrc +1 -0
  3. data/Gemfile +22 -0
  4. data/Gemfile.lock +77 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.rdoc +19 -0
  7. data/Rakefile +46 -0
  8. data/VERSION +1 -0
  9. data/active_worker.gemspec +108 -0
  10. data/lib/active_worker.rb +29 -0
  11. data/lib/active_worker/behavior/acts_as_root_object.rb +80 -0
  12. data/lib/active_worker/behavior/can_be_notified.rb +23 -0
  13. data/lib/active_worker/behavior/create_from_error.rb +21 -0
  14. data/lib/active_worker/behavior/execute_concurrently.rb +142 -0
  15. data/lib/active_worker/behavior/has_modes.rb +44 -0
  16. data/lib/active_worker/behavior/has_root_object.rb +50 -0
  17. data/lib/active_worker/behavior/hashable.rb +79 -0
  18. data/lib/active_worker/configuration.rb +143 -0
  19. data/lib/active_worker/controller.rb +112 -0
  20. data/lib/active_worker/event.rb +68 -0
  21. data/lib/active_worker/expandable.rb +77 -0
  22. data/lib/active_worker/failure_event.rb +16 -0
  23. data/lib/active_worker/finished_event.rb +9 -0
  24. data/lib/active_worker/host_information.rb +13 -0
  25. data/lib/active_worker/job_queue/job_executer.rb +52 -0
  26. data/lib/active_worker/job_queue/queue_manager.rb +46 -0
  27. data/lib/active_worker/job_queue/run_remotely.rb +52 -0
  28. data/lib/active_worker/modes_map.rb +37 -0
  29. data/lib/active_worker/notification_event.rb +9 -0
  30. data/lib/active_worker/parent_event.rb +5 -0
  31. data/lib/active_worker/started_event.rb +10 -0
  32. data/lib/active_worker/templatable.rb +46 -0
  33. data/lib/active_worker/template.rb +41 -0
  34. data/lib/active_worker/termination_event.rb +21 -0
  35. data/test/mongoid.yml +28 -0
  36. data/test/test_acts_as_root_object.rb +123 -0
  37. data/test/test_can_be_notified.rb +44 -0
  38. data/test/test_configuration.rb +281 -0
  39. data/test/test_controller.rb +205 -0
  40. data/test/test_event.rb +75 -0
  41. data/test/test_execute_concurrently.rb +134 -0
  42. data/test/test_expandable.rb +113 -0
  43. data/test/test_failure_event.rb +69 -0
  44. data/test/test_finished_event.rb +35 -0
  45. data/test/test_has_modes.rb +56 -0
  46. data/test/test_helper.rb +120 -0
  47. data/test/test_integration.rb +56 -0
  48. data/test/test_job_executer.rb +65 -0
  49. data/test/test_queue_manager.rb +106 -0
  50. data/test/test_run_remotely.rb +63 -0
  51. data/test/test_started_event.rb +23 -0
  52. data/test/test_templatable.rb +45 -0
  53. data/test/test_template.rb +29 -0
  54. data/test/test_termination_event.rb +28 -0
  55. metadata +201 -0
@@ -0,0 +1,21 @@
1
+ module ActiveWorker
2
+ module Behavior
3
+ module CreateFromError
4
+
5
+ def self.extended(base)
6
+ base.field :stack_trace
7
+ base.field :error_type
8
+ end
9
+
10
+ def create_error_from_configuration(configuration, error)
11
+ constructor_options = {
12
+ :message => "#{configuration.event_name} FAILED: #{error.message}",
13
+ :stack_trace => error.backtrace.join("\n"),
14
+ :configuration => configuration,
15
+ :error_type => error.class.name
16
+ }
17
+ create! constructor_options
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,142 @@
1
+ module ActiveWorker
2
+
3
+ THREADED_MODE = :local_worker_thread
4
+ FORKING_MODE = :local_worker_fork
5
+ DEFAULT_MODE = FORKING_MODE
6
+
7
+ PARENT = :parent
8
+ FORKED = :forked
9
+
10
+ module Behavior
11
+ module ExecuteConcurrently
12
+
13
+ def local_worker_mode=(mode)
14
+ @@local_worker_mode = mode
15
+ end
16
+
17
+ def local_worker_mode
18
+ @@local_worker_mode ||= FORKING_MODE
19
+ end
20
+
21
+ def threaded?
22
+ local_worker_mode == THREADED_MODE
23
+ end
24
+
25
+ def forking?
26
+ local_worker_mode == FORKING_MODE
27
+ end
28
+
29
+ def role
30
+ @@role ||= PARENT
31
+ end
32
+
33
+ def role=(role)
34
+ @@role = role
35
+ end
36
+
37
+ def parent?
38
+ role == PARENT
39
+ end
40
+
41
+ def forked?
42
+ role == FORKED
43
+ end
44
+
45
+ def pids
46
+ @pids ||= []
47
+ end
48
+
49
+ def threads
50
+ @threads ||= []
51
+ end
52
+
53
+ def execute_concurrently(params)
54
+ new_threads = params.map do |param|
55
+ case local_worker_mode
56
+ when THREADED_MODE
57
+ execute_thread param
58
+ when FORKING_MODE
59
+ execute_fork param
60
+ end
61
+ end
62
+
63
+ threads.concat new_threads
64
+ new_threads
65
+ end
66
+
67
+ def wait_for_children
68
+ threads.each do |thread|
69
+ thread.join if thread
70
+ end
71
+
72
+ cleanup_after_children
73
+ end
74
+
75
+ def cleanup_after_children
76
+ @pids = []
77
+ @threads = []
78
+ end
79
+
80
+ def kill_children
81
+ pids.each do |pid|
82
+ begin
83
+ Process.kill("TERM", pid) if pid
84
+ rescue Errno::ESRCH
85
+ puts "PID: #{pid} did not exist when we went to kill it"
86
+ end
87
+ end
88
+ end
89
+
90
+ def execute_thread(param)
91
+ Thread.new do
92
+ in_thread(param)
93
+ end
94
+ end
95
+
96
+ def execute_fork(param)
97
+ pid = fork do
98
+ in_fork(param)
99
+ end
100
+ pids << pid
101
+ Process.detach(pid)
102
+ end
103
+
104
+ def in_thread(param)
105
+ execute(param)
106
+ end
107
+
108
+ def in_fork(param)
109
+ after_fork(param)
110
+ execute(param)
111
+ rescue SignalException
112
+ self.handle_termination([param.to_param])
113
+ exit
114
+ rescue Exception => e
115
+ self.handle_error(e, :in_fork, [param.to_param])
116
+ end
117
+
118
+ def after_fork(param)
119
+ self.role = FORKED
120
+ cleanup_after_children
121
+ set_process_name(param)
122
+ reset_mongoid
123
+ reset_resque
124
+ end
125
+
126
+ def set_process_name(param)
127
+ $0 = "ActiveWorker Forked from #{Process.ppid} for #{param}"
128
+ end
129
+
130
+ def reset_mongoid
131
+ Mongoid::Sessions.clear
132
+ end
133
+
134
+ def reset_resque
135
+ Resque.redis.client.reconnect
136
+ trap("TERM", "DEFAULT")
137
+ end
138
+
139
+ end
140
+ end
141
+
142
+ end
@@ -0,0 +1,44 @@
1
+ module ActiveWorker
2
+ module Behavior
3
+ module HasModes
4
+
5
+ class ModeNotSupportedException < StandardError
6
+ end
7
+
8
+ module ClassExtensions
9
+
10
+ def modes_map
11
+ @modes_map ||= ModesMap.new
12
+ end
13
+
14
+ def modes
15
+ modes_map.modes
16
+ end
17
+
18
+ def add_mode(*args)
19
+ modes_map.add_mode(*args)
20
+ end
21
+ end
22
+
23
+ def self.included(base)
24
+ base.extend(ClassExtensions)
25
+ base.template_field :mode
26
+ base.before_save :set_mode_defined_fields
27
+ end
28
+
29
+ def set_mode_defined_fields
30
+ unless mode.nil? || mode.empty?
31
+ if self.class.modes_map.supports? mode
32
+ self.class.modes_map.mode(mode).each_pair do |field, value|
33
+ write_attribute(field, value) unless read_attribute(field)
34
+ end
35
+ else
36
+ raise ModeNotSupportedException, "Mode \"#{mode}\" not in Modes List: #{self.class.modes} for #{self.class}"
37
+ end
38
+ end
39
+ end
40
+
41
+ end
42
+ end
43
+
44
+ end
@@ -0,0 +1,50 @@
1
+ module ActiveWorker
2
+ module Behavior
3
+ module HasRootObject
4
+
5
+ module ClassExtensions
6
+ def root_object_relation(relation)
7
+ belongs_to :root_object, :polymorphic => true, inverse_of: relation
8
+ end
9
+ end
10
+
11
+ def self.included(base)
12
+ base.extend(ClassExtensions)
13
+ base.before_save :set_root_object
14
+ base.index({:root_object_id => -1}, {:background => true})
15
+ end
16
+
17
+ def root_owner
18
+ nil
19
+ end
20
+
21
+ def notify_root_of_child_started
22
+ if root_object
23
+ root_object.child_started
24
+ root_object.save!
25
+ end
26
+ end
27
+
28
+ def notify_root_of_finished
29
+ if root_object
30
+ root_object.child_finished
31
+ root_object.save!
32
+ end
33
+ end
34
+
35
+ def get_root_object_id
36
+ return root_object_id if root_object_id
37
+ set_root_object
38
+ root_object_id
39
+ end
40
+
41
+ def set_root_object
42
+ return unless root_owner
43
+ self.root_object_id = root_owner.get_root_object_id
44
+ self.root_object_type = root_owner.root_object_type
45
+ true
46
+ end
47
+
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,79 @@
1
+ module ActiveWorker
2
+ module Behavior
3
+ module Hashable
4
+
5
+ def get_as_flat_hash_by_root_object(root_object)
6
+ configs = []
7
+ col = get_mongoid_collection
8
+ col.find("root_object_id" => root_object.id).each do |config|
9
+ canonicalize(config, col)
10
+ configs << config
11
+ end
12
+ configs
13
+ end
14
+
15
+ def get_as_hash_by_root_object(root_object)
16
+ configs = []
17
+ col = get_mongoid_collection
18
+ col.find("root_object_id" => root_object.id, "parent_configuration_id" => nil).each do |config|
19
+ config["configurations"] = get_children_for(col, config["_id"])
20
+ canonicalize(config, col)
21
+ configs << config
22
+ end
23
+ configs
24
+ end
25
+
26
+ def get_children_for(col, parent_configuration_id)
27
+ configs = []
28
+ col.find("parent_configuration_id" => parent_configuration_id).each do |config|
29
+ config["configurations"] = get_children_for(col, config["_id"])
30
+ canonicalize(config, col)
31
+ configs << config
32
+ end
33
+ configs
34
+ end
35
+
36
+
37
+
38
+ def get_renderable_hash_by_root_object(root_object)
39
+ configs = []
40
+ col = get_mongoid_collection
41
+ col.find("root_object_id" => root_object.id, "parent_configuration_id" => nil, "renderable" => true).each do |config|
42
+ config["configurations"] = get_renderable_children_for(col, config["_id"])
43
+ canonicalize(config, col)
44
+ configs << config
45
+ end
46
+ configs
47
+ end
48
+
49
+ def renderable_hash_for_configuration(configuration_id)
50
+ col = get_mongoid_collection
51
+ config = col.find("_id" => configuration_id).first
52
+ config["configurations"] = get_renderable_children_for(col, config["_id"])
53
+ canonicalize(config, col)
54
+ config
55
+ end
56
+
57
+ def get_renderable_children_for(col, parent_configuration_id)
58
+ configs = []
59
+ col.find("parent_configuration_id" => parent_configuration_id, "renderable" => true).each do |config|
60
+ config["configurations"] = get_renderable_children_for(col, config["_id"])
61
+ canonicalize(config, col)
62
+ configs << config
63
+ end
64
+ configs
65
+ end
66
+
67
+ def canonicalize(config, col)
68
+ config["_type"] ||= col.name.classify
69
+ config["_id"] = config["_id"].to_s
70
+ end
71
+
72
+
73
+ def get_mongoid_collection
74
+ Mongoid.default_session.collections.select {|c| c.name == self.collection_name.to_s }.first
75
+ end
76
+
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,143 @@
1
+ module ActiveWorker
2
+ class Configuration
3
+ include Mongoid::Document
4
+ include Behavior::HasRootObject
5
+ extend Behavior::Hashable
6
+ POLLING_INTERVAL = 0.1
7
+
8
+ has_many :events, :class_name => "ActiveWorker::Event"
9
+
10
+ has_many :configurations,
11
+ :as => :parent_configuration,
12
+ :class_name => 'ActiveWorker::Configuration',
13
+ :autosave => true
14
+
15
+ belongs_to :parent_configuration,
16
+ :polymorphic => true
17
+
18
+ alias_method :root_owner, :parent_configuration
19
+ root_object_relation :configurations
20
+
21
+ field :polling_interval, :type => Integer, :default => 1
22
+ field :renderable, :type => Boolean, :default => false
23
+ field :flags, type: Hash, default: {}
24
+
25
+ before_save :set_flags
26
+
27
+ scope :mine, ->(parent_config) { where(parent_configuration_id: parent_config.id) }
28
+
29
+ def launch
30
+ configurations = expand_for_workers
31
+ configurations.each(&:enqueue_job)
32
+ configurations += collect_after_launch_configurations(configurations)
33
+ configurations
34
+ end
35
+
36
+ def collect_after_launch_configurations(configurations)
37
+ self.class.after_launch_methods.map { |method| send(method, configurations) }.flatten
38
+ end
39
+
40
+ def expand_for_workers
41
+ [self]
42
+ end
43
+
44
+ def expand_for_threads
45
+ [self]
46
+ end
47
+
48
+ def configurations_for_events
49
+ [self]
50
+ end
51
+
52
+ def enqueue_job
53
+ self.class.controller_class.run_remotely.execute_expanded(self.id)
54
+ self
55
+ end
56
+
57
+ def supported_child_configurations
58
+ []
59
+ end
60
+
61
+ def defined_fields
62
+ attributes.select { |k, v| self.class.config_fields.include? k.to_sym }
63
+ end
64
+
65
+ def event_name
66
+ parts = self.class.name.split("::")
67
+ parts.pop
68
+ parts.join(" ")
69
+ end
70
+
71
+ def renderable_configurations
72
+ configurations.select { |c| c.renderable }
73
+ end
74
+
75
+ def completed?
76
+ FinishedEvent.where(configuration_id: id).count > 0
77
+ end
78
+
79
+ def wait_until_completed
80
+ sleep(POLLING_INTERVAL) until completed?
81
+ end
82
+
83
+ def started
84
+ StartedEvent.create(configuration: self)
85
+ end
86
+
87
+ def finished
88
+ FinishedEvent.create(configuration: self)
89
+ end
90
+
91
+ def notify
92
+ NotificationEvent.create(configuration: self)
93
+ end
94
+
95
+ def notified?
96
+ NotificationEvent.where(configuration_id: id).count > 0
97
+ end
98
+
99
+ def set_flags
100
+ self.flags = parent_configuration.flags if parent_configuration
101
+ true
102
+ end
103
+
104
+ def self.controller_class
105
+ "#{self.parent}::Controller".constantize
106
+ end
107
+
108
+ def self.display_name
109
+ name.split("::").join(" ")
110
+ end
111
+
112
+ def self.css_name
113
+ name.split("::").join("_")
114
+ end
115
+
116
+ def self.template_field(name, *args)
117
+ config_field(name, *args)
118
+ template_fields << name
119
+ end
120
+
121
+ def self.config_field(name, *args)
122
+ field name, *args
123
+ config_fields << name
124
+ end
125
+
126
+ def self.template_fields
127
+ @template_fields ||= []
128
+ end
129
+
130
+ def self.config_fields
131
+ @config_fields ||= []
132
+ end
133
+
134
+ def self.after_launch(method)
135
+ after_launch_methods << method
136
+ end
137
+
138
+ def self.after_launch_methods
139
+ @after_launch_methods ||= []
140
+ end
141
+
142
+ end
143
+ end