active_worker 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. data/.document +5 -0
  2. data/.rvmrc +1 -0
  3. data/Gemfile +22 -0
  4. data/Gemfile.lock +77 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.rdoc +19 -0
  7. data/Rakefile +46 -0
  8. data/VERSION +1 -0
  9. data/active_worker.gemspec +108 -0
  10. data/lib/active_worker.rb +29 -0
  11. data/lib/active_worker/behavior/acts_as_root_object.rb +80 -0
  12. data/lib/active_worker/behavior/can_be_notified.rb +23 -0
  13. data/lib/active_worker/behavior/create_from_error.rb +21 -0
  14. data/lib/active_worker/behavior/execute_concurrently.rb +142 -0
  15. data/lib/active_worker/behavior/has_modes.rb +44 -0
  16. data/lib/active_worker/behavior/has_root_object.rb +50 -0
  17. data/lib/active_worker/behavior/hashable.rb +79 -0
  18. data/lib/active_worker/configuration.rb +143 -0
  19. data/lib/active_worker/controller.rb +112 -0
  20. data/lib/active_worker/event.rb +68 -0
  21. data/lib/active_worker/expandable.rb +77 -0
  22. data/lib/active_worker/failure_event.rb +16 -0
  23. data/lib/active_worker/finished_event.rb +9 -0
  24. data/lib/active_worker/host_information.rb +13 -0
  25. data/lib/active_worker/job_queue/job_executer.rb +52 -0
  26. data/lib/active_worker/job_queue/queue_manager.rb +46 -0
  27. data/lib/active_worker/job_queue/run_remotely.rb +52 -0
  28. data/lib/active_worker/modes_map.rb +37 -0
  29. data/lib/active_worker/notification_event.rb +9 -0
  30. data/lib/active_worker/parent_event.rb +5 -0
  31. data/lib/active_worker/started_event.rb +10 -0
  32. data/lib/active_worker/templatable.rb +46 -0
  33. data/lib/active_worker/template.rb +41 -0
  34. data/lib/active_worker/termination_event.rb +21 -0
  35. data/test/mongoid.yml +28 -0
  36. data/test/test_acts_as_root_object.rb +123 -0
  37. data/test/test_can_be_notified.rb +44 -0
  38. data/test/test_configuration.rb +281 -0
  39. data/test/test_controller.rb +205 -0
  40. data/test/test_event.rb +75 -0
  41. data/test/test_execute_concurrently.rb +134 -0
  42. data/test/test_expandable.rb +113 -0
  43. data/test/test_failure_event.rb +69 -0
  44. data/test/test_finished_event.rb +35 -0
  45. data/test/test_has_modes.rb +56 -0
  46. data/test/test_helper.rb +120 -0
  47. data/test/test_integration.rb +56 -0
  48. data/test/test_job_executer.rb +65 -0
  49. data/test/test_queue_manager.rb +106 -0
  50. data/test/test_run_remotely.rb +63 -0
  51. data/test/test_started_event.rb +23 -0
  52. data/test/test_templatable.rb +45 -0
  53. data/test/test_template.rb +29 -0
  54. data/test/test_termination_event.rb +28 -0
  55. metadata +201 -0
@@ -0,0 +1,69 @@
1
+ require_relative "test_helper"
2
+
3
+ module ActiveWorker
4
+ class FailureEventTest < ActiveSupport::TestCase
5
+
6
+ test "can create a failure event from error and basic params" do
7
+ exception = create_exception
8
+
9
+ config = ActiveWorker::Configuration.create
10
+
11
+ original_events = FailureEvent.from_error(config, exception)
12
+ assert_equal 1, original_events.size
13
+ original_event = original_events.first
14
+
15
+ event = FailureEvent.where(configuration_id: config.id).first
16
+
17
+ assert_equal original_event, event
18
+ assert_equal exception.backtrace.join("\n"), event.stack_trace
19
+ assert_equal "Mocha::Mock", event.error_type
20
+ end
21
+
22
+ test "event name and error message are used in event display" do
23
+ exception = create_exception
24
+ config = ActiveWorker::Configuration.create
25
+
26
+ original_event = FailureEvent.from_error(config, exception).first
27
+
28
+ assert_match /#{config.event_name}/, original_event.message
29
+ assert_match /#{exception.message}/, original_event.message
30
+ end
31
+
32
+ test "can use failure events as finished events" do
33
+ exception = create_exception
34
+ config = ActiveWorker::Configuration.create
35
+ original_event = FailureEvent.from_error(config,exception).first
36
+ event = FinishedEvent.where(configuration_id: config.id).first
37
+
38
+ assert_equal original_event, event
39
+ end
40
+
41
+ test "expands for threads unless completed" do
42
+ exception = create_exception
43
+
44
+ config = ExpandableConfig.create number_of_threads: 2
45
+
46
+ config.expand_for_threads
47
+
48
+ events = FailureEvent.from_error(config, exception)
49
+
50
+ assert_equal 2, events.size
51
+ end
52
+
53
+ test "expands for threads" do
54
+ exception = create_exception
55
+
56
+ config = ExpandableConfig.create number_of_threads: 2
57
+
58
+ config.expand_for_threads
59
+ config.finished
60
+
61
+ events = FailureEvent.from_error(config, exception)
62
+
63
+ assert_equal 1, events.size
64
+ end
65
+
66
+
67
+
68
+ end
69
+ end
@@ -0,0 +1,35 @@
1
+ require_relative "test_helper"
2
+
3
+ module ActiveWorker
4
+ class FinishedEventTest < ActiveSupport::TestCase
5
+
6
+ test "exists_for_configurations scopes to finished events" do
7
+ config1 = Configuration.create
8
+ config2 = Configuration.create
9
+ config3 = Configuration.create
10
+
11
+ FinishedEvent.create configuration: config1
12
+ FinishedEvent.create configuration: config2
13
+
14
+ configs = [config1, config2, config3]
15
+
16
+ assert_equal false, FinishedEvent.exists_for_configurations?(configs)
17
+
18
+ Event.create configuration: config3
19
+
20
+ assert_equal false, FinishedEvent.exists_for_configurations?(configs)
21
+
22
+ FinishedEvent.create configuration: config3
23
+
24
+ assert FinishedEvent.exists_for_configurations?(configs)
25
+ end
26
+
27
+ test "finished message" do
28
+ configuration = Configuration.create
29
+ event = FinishedEvent.create(configuration: configuration)
30
+
31
+ assert_match /finished/, event.message
32
+ end
33
+
34
+ end
35
+ end
@@ -0,0 +1,56 @@
1
+ require_relative "test_helper"
2
+
3
+ module ActiveWorker
4
+ class Modeable < Configuration
5
+ include ActiveWorker::Behavior::HasModes
6
+
7
+ field :custom_field
8
+
9
+ add_mode :first, custom_field: "mode1", other_field: "default"
10
+ add_mode :second, custom_field: "mode2"
11
+ end
12
+
13
+ class ModeableWithNoModes < Configuration
14
+ include ActiveWorker::Behavior::HasModes
15
+ field :custom_field
16
+ end
17
+
18
+ class HasModesTest < ActiveSupport::TestCase
19
+ test "can specify modes" do
20
+ mode_config = Modeable.create mode: :first
21
+
22
+ assert_equal [:first, :second], Modeable.modes
23
+ assert_equal :first, mode_config.mode
24
+ end
25
+
26
+ test "sets custom fields with mode map" do
27
+ mode_config = Modeable.create mode: :first
28
+ assert_equal "mode1", mode_config.custom_field
29
+ assert_equal "default", mode_config.other_field
30
+ end
31
+
32
+ test "mode does not override already set field" do
33
+ mode_config = Modeable.create mode: :first, custom_field: "set"
34
+ assert_equal "set", mode_config.custom_field
35
+ end
36
+
37
+ test "does not blow up with unsupported mode" do
38
+ assert_raise ActiveWorker::Behavior::HasModes::ModeNotSupportedException do
39
+ Modeable.create mode: "foo"
40
+ end
41
+ end
42
+
43
+ test "allows no mode to be set" do
44
+ mode_config = Modeable.create custom_field: "set"
45
+ assert_equal "set", mode_config.custom_field
46
+ end
47
+
48
+ test "allows no modes to be added" do
49
+ assert_equal [], ModeableWithNoModes.modes
50
+
51
+ end
52
+
53
+ end
54
+
55
+
56
+ end
@@ -0,0 +1,120 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+
11
+ require 'active_support/test_case'
12
+ require 'test/unit'
13
+
14
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
15
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
16
+ require 'active_worker'
17
+
18
+ require 'mongoid'
19
+ ENV["MONGOID_ENV"]="test"
20
+ Mongoid.load!("#{File.dirname(__FILE__)}/mongoid.yml")
21
+
22
+
23
+ Resque.redis.namespace = "resque:active_worker_test"
24
+
25
+
26
+ module ActiveWorker
27
+
28
+ class Rootable
29
+ include Mongoid::Document
30
+ include ActiveWorker::Behavior::ActsAsRootObject
31
+ end
32
+
33
+
34
+ class TopConfig < Configuration
35
+ config_field :top_field
36
+
37
+ def child_configs
38
+ ChildConfig.mine(self).all
39
+ end
40
+ end
41
+
42
+ class ChildConfig < Configuration
43
+ config_field :child_field
44
+ end
45
+
46
+ class TemplatableTopConfig < Configuration
47
+ include Templatable
48
+ template_field :top_field
49
+ config_field :other_top_field
50
+ end
51
+
52
+ class TemplatableChildConfig < Configuration
53
+ include Templatable
54
+ template_field :child_field
55
+ config_field :other_child_field
56
+ end
57
+
58
+ class AfterLaunchConfig < ActiveWorker::Configuration
59
+ after_launch :after_launch_method
60
+ end
61
+
62
+ class ExpandableConfig < Configuration
63
+ include Expandable
64
+ include Templatable
65
+ field :foo
66
+ template_field :name
67
+ config_field :size
68
+ end
69
+
70
+ class MappedExpandableConfig < ExpandableConfig
71
+ def expansion_maps_for(number_of_configurations)
72
+ maps = []
73
+ number_of_configurations.times do |value|
74
+ maps << {size: value}
75
+ end
76
+ maps
77
+ end
78
+ end
79
+
80
+ end
81
+
82
+
83
+ class ActiveSupport::TestCase
84
+
85
+ ActiveWorker::JobQueue::RunRemotely.worker_mode = ActiveWorker::JobQueue::RunRemotely::THREADED
86
+
87
+ def create_exception
88
+ error_message = "Error message"
89
+ error_backtrace = ["line 1", "line 2"]
90
+
91
+ error = mock
92
+ error.stubs(:message).returns(error_message)
93
+ error.stubs(:backtrace).returns(error_backtrace)
94
+ error
95
+ end
96
+
97
+ def wait_for_all_configurations
98
+ ActiveWorker::Configuration.all.each(&:wait_until_completed)
99
+ end
100
+
101
+ def assert_no_failures
102
+ assert_equal 0, ActiveWorker::FailureEvent.count, "Failures: #{ActiveWorker::FailureEvent.all.map { |e| e.message + "\n" + e.stack_trace }.join}"
103
+ end
104
+
105
+ setup :clear_database
106
+ teardown :reset_default_worker_mode
107
+
108
+ def reset_default_worker_mode
109
+ ActiveWorker::Controller::local_worker_mode = ActiveWorker::DEFAULT_MODE
110
+ end
111
+
112
+ private
113
+
114
+ def clear_database
115
+ Mongoid.default_session.collections.select { |c| c.name != 'system.indexes' }.each(&:drop)
116
+ end
117
+
118
+ end
119
+
120
+ ActiveWorker::JobQueue::JobExecuter.stubs(:log_error)
@@ -0,0 +1,56 @@
1
+ require_relative "test_helper"
2
+
3
+ module ActiveWorker
4
+ class IntegrationTest < ActiveSupport::TestCase
5
+
6
+ test "can create correct templates" do
7
+
8
+ parent_config = TemplatableTopConfig.create(top_field: "top field", other_top_field: "other top field")
9
+ child_config0 = TemplatableChildConfig.create(child_field: "child field", other_child_field: "other child field", parent_configuration: parent_config)
10
+ child_config1 = TemplatableChildConfig.create(child_field: "different child field",other_child_field: "different other child field", parent_configuration: parent_config)
11
+
12
+ top_template = parent_config.find_template
13
+
14
+ assert_kind_of Template, top_template
15
+ assert_equal TemplatableTopConfig.name, top_template.configuration_type
16
+ assert_equal 2, top_template.child_templates.count
17
+
18
+ assert_equal parent_config.top_field,top_template[:top_field]
19
+ assert_nil top_template[:other_top_field]
20
+
21
+ child_template0 = top_template.child_templates[0]
22
+ child_template1 = top_template.child_templates[1]
23
+
24
+ assert_equal child_config0.child_field, child_template0[:child_field]
25
+ assert_equal child_config1.child_field, child_template1[:child_field]
26
+
27
+ assert_equal TemplatableChildConfig.name, child_template0.configuration_type
28
+ assert_equal TemplatableChildConfig.name, child_template0.configuration_type
29
+
30
+ end
31
+
32
+ test "can recreate correct configurations from templates" do
33
+ parent_config = TemplatableTopConfig.create(top_field: "top field", other_top_field: "other top field")
34
+ child_config0 = TemplatableChildConfig.create(child_field: "child field", other_child_field: "other child field", parent_configuration: parent_config)
35
+ child_config1 = TemplatableChildConfig.create(child_field: "different child field",other_child_field: "different other child field", parent_configuration: parent_config)
36
+
37
+ top_template = parent_config.find_template
38
+
39
+ created_config = top_template.build_configuration
40
+
41
+ assert_kind_of TemplatableTopConfig, created_config
42
+ assert_equal parent_config.top_field, created_config.top_field
43
+ assert_nil created_config.other_top_field
44
+
45
+ created_child_config0 = created_config.configurations[0]
46
+ created_child_config1 = created_config.configurations[1]
47
+
48
+ assert_equal child_config0.child_field, created_child_config0.child_field
49
+ assert_equal child_config1.child_field, created_child_config1.child_field
50
+
51
+ assert_nil created_child_config0.other_child_field
52
+ assert_nil created_child_config1.other_child_field
53
+ end
54
+
55
+ end
56
+ end
@@ -0,0 +1,65 @@
1
+ require_relative "test_helper"
2
+ module ActiveWorker
3
+ module JobQueue
4
+ class JobExecuterTest < ActiveSupport::TestCase
5
+
6
+ class TestClass
7
+ def self.test_method(param1, param2)
8
+
9
+ end
10
+
11
+ def self.raise_method
12
+ raise SignalException.new "SIGHUP"
13
+ end
14
+
15
+ def self.handle_error(e, method, params)
16
+
17
+ end
18
+ end
19
+
20
+ setup do
21
+ JobExecuter.stubs(:log_error)
22
+ end
23
+
24
+ test "can execute command from args" do
25
+ param1 = 1
26
+ param2 = 2
27
+ TestClass.expects(:test_method).with(param1, param2)
28
+
29
+ class_name = TestClass.to_s
30
+ method = :test_method
31
+ params = [param1,param2]
32
+
33
+ args = {}
34
+ args["class_name"] = class_name
35
+ args["method"] = method
36
+ args["params"] = params
37
+
38
+ JobExecuter.execute_task_from_args(args)
39
+ end
40
+
41
+ test "handles and reraises signal exception" do
42
+ class_name = TestClass.to_s
43
+ method = :raise_method
44
+
45
+ args = {}
46
+ args["class_name"] = class_name
47
+ args["method"] = method
48
+
49
+ assert_raise SignalException do
50
+ JobExecuter.execute_task_from_args(args)
51
+ end
52
+ end
53
+
54
+ test "can handle termination" do
55
+ params = [1,2]
56
+ fake_class = mock
57
+ fake_class.expects(:handle_termination).with(params)
58
+
59
+ JobExecuter.expects(:exit)
60
+ JobExecuter.handle_termination(fake_class, params)
61
+ end
62
+
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,106 @@
1
+ require_relative "test_helper"
2
+ module ActiveWorker
3
+ module JobQueue
4
+ class QueueManagerTest < ActiveSupport::TestCase
5
+
6
+ test "can extract configuration id from worker" do
7
+ config_id = 5
8
+ worker = mock_worker(config_id)
9
+
10
+ extracted_id = QueueManager.new.configuration_id_from_worker(worker)
11
+ assert_equal config_id, extracted_id
12
+ end
13
+
14
+ test "can create job hash from worker" do
15
+ config_id = 5
16
+ worker = mock_worker(config_id)
17
+
18
+ job_hash = QueueManager.new.create_job_hash_from_worker(worker)
19
+
20
+ expected_hash = {"host" => "localhost",
21
+ "queues" => ["execute", "localhost_execute"],
22
+ "pid" => config_id.to_s,
23
+ "args" => {"params" => [config_id]}}
24
+
25
+ assert_equal expected_hash, job_hash
26
+ end
27
+
28
+ test "can get list of active jobs for configurations" do
29
+ manager = QueueManager.new
30
+
31
+ Resque.expects(:working).returns(mock_workers)
32
+
33
+ jobs = manager.active_jobs_for_configurations(configuration_ids)
34
+
35
+ assert_equal 4, jobs.size
36
+ assert_equal [1, 2, 3, 4], jobs.map { |j| j["pid"].to_i }
37
+
38
+ end
39
+
40
+ test "does not report on jobless workers" do
41
+ config_id = 5
42
+ worker = mock_worker(config_id, {})
43
+
44
+ job_hash = QueueManager.new.create_job_hash_from_worker(worker)
45
+
46
+ assert_nil job_hash
47
+
48
+ end
49
+
50
+ test "active workers can lose their jobs" do
51
+ manager = QueueManager.new
52
+
53
+ workers = mock_workers + [mock_worker(11, {})]
54
+
55
+ Resque.expects(:working).returns(workers)
56
+
57
+ ids = configuration_ids + [11]
58
+
59
+ jobs = manager.active_jobs_for_configurations(ids)
60
+
61
+ assert_equal 4, jobs.size
62
+ assert_equal [1, 2, 3, 4], jobs.map { |j| j["pid"].to_i }
63
+ end
64
+
65
+ test "does not return workers without jobs" do
66
+ manager = QueueManager.new
67
+
68
+ workers = mock_workers(4) + [mock_worker(11, {})]
69
+
70
+ Resque.expects(:working).returns(workers)
71
+
72
+ jobs = manager.active_jobs
73
+
74
+ assert_equal 4, jobs.size
75
+ assert_equal [0, 1, 2, 3], jobs.map { |j| j["pid"].to_i }
76
+ end
77
+
78
+ def mock_workers(num_workers = 10)
79
+ workers = []
80
+ num_workers.times do |num|
81
+ workers << mock_worker(num)
82
+ end
83
+ workers
84
+ end
85
+
86
+ def configuration_ids
87
+ [1, 2, 3, 4, 20]
88
+ end
89
+
90
+ def mock_worker(config_id, job = mock_job(config_id))
91
+ worker = mock
92
+ worker.stubs(:job).returns(job)
93
+ worker.stubs(:hostname).returns("bad_hostname")
94
+ worker.stubs(:to_s).returns("localhost:#{config_id}:execute,localhost_execute")
95
+ worker
96
+ end
97
+
98
+ def mock_job(config_id)
99
+ args = [{"params" => [config_id]}]
100
+ job = {"payload" => {"args" => args}}
101
+ end
102
+
103
+
104
+ end
105
+ end
106
+ end