toro 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +7 -0
  3. data/Appraisals +7 -0
  4. data/Gemfile +3 -0
  5. data/app/assets/javascripts/toro/monitor/abstract_jobs_table.js.coffee +188 -0
  6. data/app/assets/javascripts/toro/monitor/application.js.coffee +14 -0
  7. data/app/assets/javascripts/toro/monitor/chart.js.coffee +203 -0
  8. data/app/assets/javascripts/toro/monitor/initialize.js.coffee.erb +25 -0
  9. data/app/assets/javascripts/toro/monitor/jobs_table.js.coffee +63 -0
  10. data/app/assets/javascripts/toro/monitor/processes_table.js.coffee +96 -0
  11. data/app/assets/javascripts/toro/monitor/queue_jobs_table.js.coffee +91 -0
  12. data/app/assets/stylesheets/toro/monitor/application.css.sass +5 -0
  13. data/app/assets/stylesheets/toro/monitor/jobs_table.css.sass +69 -0
  14. data/app/assets/stylesheets/toro/monitor/layout.css.sass +5 -0
  15. data/app/controllers/toro/monitor/api/jobs_controller.rb +65 -0
  16. data/app/controllers/toro/monitor/api/processes_controller.rb +13 -0
  17. data/app/controllers/toro/monitor/api/queues_controller.rb +24 -0
  18. data/app/controllers/toro/monitor/base_controller.rb +11 -0
  19. data/app/controllers/toro/monitor/jobs_controller.rb +11 -0
  20. data/app/controllers/toro/monitor/processes_controller.rb +8 -0
  21. data/app/controllers/toro/monitor/queues_controller.rb +9 -0
  22. data/app/datatables/toro/monitor/abstract_datatable.rb +71 -0
  23. data/app/datatables/toro/monitor/jobs_datatable.rb +60 -0
  24. data/app/datatables/toro/monitor/processes_datatable.rb +60 -0
  25. data/app/helpers/toro/monitor/toro_monitor_helper.rb +13 -0
  26. data/app/views/toro/monitor/jobs/_jobs.slim +14 -0
  27. data/app/views/toro/monitor/jobs/chart.slim +1 -0
  28. data/app/views/toro/monitor/jobs/index.slim +1 -0
  29. data/app/views/toro/monitor/layouts/application.slim +32 -0
  30. data/app/views/toro/monitor/processes/index.slim +14 -0
  31. data/app/views/toro/monitor/queues/_jobs.slim +14 -0
  32. data/app/views/toro/monitor/queues/index.slim +5 -0
  33. data/config/routes.rb +16 -0
  34. data/examples/chart.png +0 -0
  35. data/examples/job.png +0 -0
  36. data/examples/jobs.png +0 -0
  37. data/examples/processes.png +0 -0
  38. data/examples/queues.png +0 -0
  39. data/gemfiles/rails_3.gemfile +7 -0
  40. data/gemfiles/rails_3.gemfile.lock +146 -0
  41. data/gemfiles/rails_4.gemfile +7 -0
  42. data/gemfiles/rails_4.gemfile.lock +147 -0
  43. data/lib/toro/version.rb +1 -1
  44. data/spec/config/database.yml.example +6 -0
  45. data/spec/lib/cli_spec.rb +28 -0
  46. data/spec/lib/client_spec.rb +26 -0
  47. data/spec/lib/fetcher_spec.rb +105 -0
  48. data/spec/lib/job_spec.rb +17 -0
  49. data/spec/lib/listener_spec.rb +72 -0
  50. data/spec/lib/manager_spec.rb +78 -0
  51. data/spec/lib/middleware/server/error_spec.rb +18 -0
  52. data/spec/lib/middleware/server/error_storage_spec.rb +22 -0
  53. data/spec/lib/middleware/server/properties_spec.rb +18 -0
  54. data/spec/lib/middleware/server/retry_spec.rb +38 -0
  55. data/spec/lib/processor_spec.rb +33 -0
  56. data/spec/lib/worker_spec.rb +50 -0
  57. data/spec/spec_helper.rb +31 -0
  58. data/spec/support/active_record_spec_helper.rb +18 -0
  59. data/spec/support/jobs_helper.rb +4 -0
  60. data/spec/support/workers_helper.rb +56 -0
  61. data/toro.gemspec +29 -0
  62. data/vendor/assets/javascripts/toro/monitor/bootstrap-select.min.js +1 -0
  63. data/vendor/assets/javascripts/toro/monitor/bootstrap.min.js +6 -0
  64. data/vendor/assets/javascripts/toro/monitor/d3.js +8795 -0
  65. data/vendor/assets/javascripts/toro/monitor/datatables.bootstrap.js +96 -0
  66. data/vendor/assets/javascripts/toro/monitor/datatables.js +256 -0
  67. data/vendor/assets/javascripts/toro/monitor/datatables.standing_redraw.js.coffee +5 -0
  68. data/vendor/assets/javascripts/toro/monitor/jquery.timeago.js +27 -0
  69. data/vendor/assets/stylesheets/toro/monitor/bootstrap-select.min.css +1 -0
  70. data/vendor/assets/stylesheets/toro/monitor/bootstrap.min.css +9 -0
  71. metadata +103 -5
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "4.1.0"
6
+
7
+ gemspec :path => "../"
@@ -0,0 +1,147 @@
1
+ PATH
2
+ remote: ..
3
+ specs:
4
+ toro (0.2.2)
5
+ activerecord-postgres-hstore
6
+ celluloid (>= 0.15.2)
7
+ jquery-datatables-rails (>= 2.1.10.0.2)
8
+ nested-hstore
9
+ pg
10
+ rails (>= 3.0)
11
+ rails-datatables
12
+ slim
13
+
14
+ GEM
15
+ remote: https://rubygems.org/
16
+ specs:
17
+ actionmailer (4.1.0)
18
+ actionpack (= 4.1.0)
19
+ actionview (= 4.1.0)
20
+ mail (~> 2.5.4)
21
+ actionpack (4.1.0)
22
+ actionview (= 4.1.0)
23
+ activesupport (= 4.1.0)
24
+ rack (~> 1.5.2)
25
+ rack-test (~> 0.6.2)
26
+ actionview (4.1.0)
27
+ activesupport (= 4.1.0)
28
+ builder (~> 3.1)
29
+ erubis (~> 2.7.0)
30
+ activemodel (4.1.0)
31
+ activesupport (= 4.1.0)
32
+ builder (~> 3.1)
33
+ activerecord (4.1.0)
34
+ activemodel (= 4.1.0)
35
+ activesupport (= 4.1.0)
36
+ arel (~> 5.0.0)
37
+ activerecord-postgres-hstore (0.7.7)
38
+ activerecord (>= 3.1)
39
+ pg-hstore (>= 1.1.5)
40
+ rake
41
+ activesupport (4.1.0)
42
+ i18n (~> 0.6, >= 0.6.9)
43
+ json (~> 1.7, >= 1.7.7)
44
+ minitest (~> 5.1)
45
+ thread_safe (~> 0.1)
46
+ tzinfo (~> 1.1)
47
+ appraisal (1.0.0)
48
+ bundler
49
+ rake
50
+ thor (>= 0.14.0)
51
+ arel (5.0.1.20140414130214)
52
+ builder (3.2.2)
53
+ celluloid (0.16.0)
54
+ timers (~> 4.0.0)
55
+ diff-lcs (1.2.5)
56
+ erubis (2.7.0)
57
+ hike (1.2.3)
58
+ hitimes (1.2.2)
59
+ i18n (0.6.9)
60
+ jquery-datatables-rails (3.1.1)
61
+ actionpack (>= 3.1)
62
+ jquery-rails
63
+ railties (>= 3.1)
64
+ sass-rails
65
+ jquery-rails (3.1.2)
66
+ railties (>= 3.0, < 5.0)
67
+ thor (>= 0.14, < 2.0)
68
+ json (1.8.1)
69
+ mail (2.5.4)
70
+ mime-types (~> 1.16)
71
+ treetop (~> 1.4.8)
72
+ mime-types (1.25.1)
73
+ minitest (5.3.3)
74
+ multi_json (1.9.2)
75
+ nested-hstore (0.1.1)
76
+ activerecord
77
+ activesupport
78
+ pg (0.18.1)
79
+ pg-hstore (1.2.0)
80
+ polyglot (0.3.4)
81
+ rack (1.5.2)
82
+ rack-test (0.6.2)
83
+ rack (>= 1.0)
84
+ rails (4.1.0)
85
+ actionmailer (= 4.1.0)
86
+ actionpack (= 4.1.0)
87
+ actionview (= 4.1.0)
88
+ activemodel (= 4.1.0)
89
+ activerecord (= 4.1.0)
90
+ activesupport (= 4.1.0)
91
+ bundler (>= 1.3.0, < 2.0)
92
+ railties (= 4.1.0)
93
+ sprockets-rails (~> 2.0)
94
+ rails-datatables (0.0.2)
95
+ railties (4.1.0)
96
+ actionpack (= 4.1.0)
97
+ activesupport (= 4.1.0)
98
+ rake (>= 0.8.7)
99
+ thor (>= 0.18.1, < 2.0)
100
+ rake (10.3.1)
101
+ rspec (2.14.1)
102
+ rspec-core (~> 2.14.0)
103
+ rspec-expectations (~> 2.14.0)
104
+ rspec-mocks (~> 2.14.0)
105
+ rspec-core (2.14.8)
106
+ rspec-expectations (2.14.5)
107
+ diff-lcs (>= 1.1.3, < 2.0)
108
+ rspec-mocks (2.14.6)
109
+ sass (3.4.11)
110
+ sass-rails (5.0.1)
111
+ railties (>= 4.0.0, < 5.0)
112
+ sass (~> 3.1)
113
+ sprockets (>= 2.8, < 4.0)
114
+ sprockets-rails (>= 2.0, < 4.0)
115
+ tilt (~> 1.1)
116
+ slim (3.0.1)
117
+ temple (~> 0.7.3)
118
+ tilt (>= 1.3.3, < 2.1)
119
+ sprockets (2.12.1)
120
+ hike (~> 1.2)
121
+ multi_json (~> 1.0)
122
+ rack (~> 1.0)
123
+ tilt (~> 1.1, != 1.3.0)
124
+ sprockets-rails (2.1.3)
125
+ actionpack (>= 3.0)
126
+ activesupport (>= 3.0)
127
+ sprockets (~> 2.8)
128
+ temple (0.7.5)
129
+ thor (0.19.1)
130
+ thread_safe (0.3.3)
131
+ tilt (1.4.1)
132
+ timers (4.0.1)
133
+ hitimes
134
+ treetop (1.4.15)
135
+ polyglot
136
+ polyglot (>= 0.3.1)
137
+ tzinfo (1.1.0)
138
+ thread_safe (~> 0.1)
139
+
140
+ PLATFORMS
141
+ ruby
142
+
143
+ DEPENDENCIES
144
+ appraisal
145
+ rails (= 4.1.0)
146
+ rspec
147
+ toro!
@@ -1,3 +1,3 @@
1
1
  module Toro
2
- VERSION = '0.2.1'
2
+ VERSION = '0.2.2'
3
3
  end
@@ -0,0 +1,6 @@
1
+ test:
2
+ adapter: postgresql
3
+ database: my_database
4
+ pool: 25
5
+ username: my_username
6
+ password: my_password
@@ -0,0 +1,28 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toro::CLI do
4
+ describe '.new' do
5
+ it 'sets the default options' do
6
+ cli = Toro::CLI.new(['toro:work'])
7
+ cli.options.should == {}
8
+ end
9
+
10
+ it 'sets the queue option' do
11
+ cli = Toro::CLI.new(['toro:work', '-q', 'queue1'])
12
+ cli.options.should include(queues: ['queue1'])
13
+ end
14
+
15
+ it 'sets the concurrency option' do
16
+ cli = Toro::CLI.new(['toro:work', '-c', '5'])
17
+ cli.options.should include(concurrency: 5)
18
+ end
19
+
20
+ it 'sets multiple options' do
21
+ cli = Toro::CLI.new(['toro:work', '-q', 'queue1', '-q', 'queue2', '-c', '5'])
22
+ cli.options.should include(
23
+ queues: ['queue1', 'queue2'],
24
+ concurrency: 5
25
+ )
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toro::Client do
4
+ describe '.create_job' do
5
+ it 'creates a job' do
6
+ attributes = {
7
+ queue: 'default',
8
+ class_name: 'DefaultQueueWorker',
9
+ args: [],
10
+ status: 'queued'
11
+ }
12
+ Toro::Client.create_job(attributes)
13
+ job_should_have_attributes(Toro::Job.first, attributes)
14
+ end
15
+
16
+ it 'sets the status as queued' do
17
+ attributes = {
18
+ queue: 'default',
19
+ class_name: 'DefaultQueueWorker',
20
+ args: []
21
+ }
22
+ Toro::Client.create_job(attributes)
23
+ job_should_have_attributes(Toro::Job.first, { status: 'queued' })
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,105 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toro::Fetcher do
4
+ describe '#retrieve' do
5
+ it 'retrieves the first job' do
6
+ fetcher = Toro::Fetcher.new(manager: Toro::Manager.new)
7
+ job = Toro::Job.create!(
8
+ queue: 'default',
9
+ class_name: 'DefaultQueueWorker',
10
+ args: [],
11
+ status: 'queued'
12
+ )
13
+ Toro::Job.create!(
14
+ queue: 'default',
15
+ class_name: 'DefaultQueueWorker',
16
+ args: [],
17
+ status: 'queued'
18
+ )
19
+ retrieved = fetcher.retrieve
20
+ retrieved.id.should == job.id
21
+ end
22
+
23
+ it 'only retrieves jobs in its queue' do
24
+ fetcher = Toro::Fetcher.new(manager: Toro::Manager.new)
25
+ Toro::Job.create!(
26
+ queue: 'foo',
27
+ class_name: 'FooQueueWorker',
28
+ args: [],
29
+ status: 'queued'
30
+ )
31
+ job = Toro::Job.create!(
32
+ queue: 'default',
33
+ class_name: 'DefaultQueueWorker',
34
+ args: [],
35
+ status: 'queued'
36
+ )
37
+ retrieved = fetcher.retrieve
38
+ retrieved.id.should == job.id
39
+ end
40
+
41
+ it 'only retrieves queued jobs' do
42
+ fetcher = Toro::Fetcher.new(manager: Toro::Manager.new)
43
+ Toro::Job.create!(
44
+ queue: 'default',
45
+ class_name: 'DefaultQueueWorker',
46
+ args: [],
47
+ status: 'complete'
48
+ )
49
+ job = Toro::Job.create!(
50
+ queue: 'default',
51
+ class_name: 'DefaultQueueWorker',
52
+ args: [],
53
+ status: 'queued'
54
+ )
55
+ retrieved = fetcher.retrieve
56
+ retrieved.id.should == job.id
57
+ end
58
+
59
+ it 'retrieves scheduled jobs' do
60
+ fetcher = Toro::Fetcher.new(manager: Toro::Manager.new)
61
+ job = Toro::Job.create!(
62
+ queue: 'default',
63
+ class_name: 'DefaultQueueWorker',
64
+ args: [],
65
+ status: 'scheduled',
66
+ scheduled_at: Time.now - 1.minute
67
+ )
68
+ Toro::Job.create!(
69
+ queue: 'default',
70
+ class_name: 'DefaultQueueWorker',
71
+ args: [],
72
+ status: 'queued'
73
+ )
74
+ retrieved = fetcher.retrieve
75
+ retrieved.id.should == job.id
76
+ end
77
+
78
+ it "does not retrieve scheduled jobs that aren't scheduled yet" do
79
+ fetcher = Toro::Fetcher.new(manager: Toro::Manager.new)
80
+ Toro::Job.create!(
81
+ queue: 'default',
82
+ class_name: 'DefaultQueueWorker',
83
+ args: [],
84
+ status: 'scheduled',
85
+ scheduled_at: Time.now + 1.minute
86
+ )
87
+ fetcher.retrieve.should be_nil
88
+ end
89
+
90
+ it "sets the running job's attributes" do
91
+ Toro::Job.create!(
92
+ queue: 'default',
93
+ class_name: 'DefaultQueueWorker',
94
+ args: [],
95
+ status: 'queued'
96
+ )
97
+ fetcher = Toro::Fetcher.new(manager: Toro::Manager.new)
98
+ fetcher.retrieve
99
+ job_should_have_attributes(Toro::Job.first, {
100
+ status: 'running',
101
+ started_by: Toro.process_identity
102
+ })
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,17 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toro::Job do
4
+ describe '#set_properties' do
5
+ it 'sets the properties' do
6
+ job = Toro::Job.new
7
+ job.set_properties(foo: 'bar')
8
+ job.properties.should == { 'foo' => 'bar' }
9
+ end
10
+
11
+ it "doesn't save the job" do
12
+ job = Toro::Job.new
13
+ job.set_properties(foo: 'bar')
14
+ job.new_record?.should be_true
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,72 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toro::Listener do
4
+ class DummyListenerManager
5
+ include Toro::ActorManager
6
+
7
+ def stop
8
+ actors[:listener].stop
9
+ end
10
+ end
11
+
12
+ describe '#start' do
13
+ it 'notifies the fetcher when a job is created' do
14
+ manager = DummyListenerManager.new
15
+ fetcher = double('fetcher').stub(:notify)
16
+ fetcher.should_receive(:notify)
17
+ listener = Toro::Listener.new(fetcher: fetcher, manager: manager)
18
+ listener.async.start
19
+ # Wait for the listener to start listening
20
+ sleep 0.1
21
+ Toro::Job.create!(
22
+ queue: 'default',
23
+ class_name: 'DefaultQueueWorker',
24
+ args: [],
25
+ status: 'queued'
26
+ )
27
+ manager.stop
28
+ # Wait for the NOTIFY to be received
29
+ sleep 0.1
30
+ end
31
+
32
+ it 'notifies the fetcher three times when three jobs are created' do
33
+ manager = DummyListenerManager.new
34
+ fetcher = double('fetcher').stub(:notify)
35
+ fetcher.should_receive(:notify).exactly(3).times
36
+ listener = Toro::Listener.new(fetcher: fetcher, manager: manager)
37
+ listener.async.start
38
+ # Wait for the listener to start listening
39
+ sleep 0.1
40
+ 3.times do
41
+ Toro::Job.create!(
42
+ queue: 'default',
43
+ class_name: 'DefaultQueueWorker',
44
+ args: [],
45
+ status: 'queued'
46
+ )
47
+ end
48
+ manager.stop
49
+ # Wait for the NOTIFY to be received
50
+ sleep 0.1
51
+ end
52
+
53
+ it "doesn't notify the fetcher when a job in a different queue is created" do
54
+ manager = DummyListenerManager.new
55
+ fetcher = double('fetcher').stub(:notify)
56
+ fetcher.should_not_receive(:notify)
57
+ listener = Toro::Listener.new(fetcher: fetcher, manager: manager)
58
+ listener.async.start
59
+ # Wait for the listener to start listening
60
+ sleep 0.1
61
+ Toro::Job.create!(
62
+ queue: 'foo',
63
+ class_name: 'FooQueueWorker',
64
+ args: [],
65
+ status: 'queued'
66
+ )
67
+ manager.stop
68
+ # Wait for the NOTIFY to be received
69
+ sleep 0.1
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,78 @@
1
+ require 'spec_helper'
2
+
3
+ describe Toro::Manager do
4
+ describe '.new' do
5
+ it 'creates N processors' do
6
+ concurrency = 5
7
+ manager = Toro::Manager.new(concurrency: concurrency)
8
+ manager.ready.length.should == concurrency
9
+ end
10
+ end
11
+
12
+ describe '#start' do
13
+ it 'performs N dispatches' do
14
+ concurrency = 5
15
+ manager = Toro::Manager.new(concurrency: concurrency)
16
+ manager.bare_object.should_receive(:dispatch).exactly(concurrency).times
17
+ manager.start
18
+ manager.stop
19
+ end
20
+ end
21
+
22
+ describe '#assign' do
23
+ it 'assigns a job to a processor' do
24
+ job = Toro::Job.create!(
25
+ queue: 'default',
26
+ class_name: 'DefaultQueueWorker',
27
+ args: [],
28
+ status: 'queued'
29
+ )
30
+ manager = Toro::Manager.new(concurrency: 5)
31
+ manager.assign(job)
32
+ manager.busy.length.should == 1
33
+ manager.stop
34
+ end
35
+ end
36
+
37
+ describe '#stop' do
38
+ it 'shuts down' do
39
+ manager = Toro::Manager.new(concurrency: 5)
40
+ manager.stop
41
+ manager.busy.length.should == 0
42
+ manager.ready.length.should == 0
43
+ end
44
+ end
45
+
46
+ describe '#processor_complete' do
47
+ it 'returns the processor to the ready pool' do
48
+ manager = Toro::Manager.new(concurrency: 5)
49
+ initial_ready_length = manager.ready.length
50
+ processor = manager.ready.pop
51
+ manager.busy << processor
52
+ manager.processor_complete(processor)
53
+ manager.busy.length.should == 0
54
+ manager.ready.length.should == initial_ready_length
55
+ manager.stop
56
+ end
57
+ end
58
+
59
+ describe '#requeue' do
60
+ it "requeues the process's running jobs" do
61
+ Toro::Job.create!(
62
+ queue: 'default',
63
+ class_name: 'DefaultQueueWorker',
64
+ args: [],
65
+ status: 'running',
66
+ started_by: Toro.process_identity,
67
+ started_at: Time.now
68
+ )
69
+ manager = Toro::Manager.new
70
+ manager.requeue
71
+ job_should_have_attributes(Toro::Job.first, {
72
+ status: 'queued',
73
+ started_by: nil,
74
+ started_at: nil
75
+ })
76
+ end
77
+ end
78
+ end