resque_manager 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/MIT-LICENSE +21 -0
- data/README.markdown +417 -0
- data/Rakefile +41 -0
- data/app/assets/images/resque_manager/idle.png +0 -0
- data/app/assets/images/resque_manager/poll.png +0 -0
- data/app/assets/images/resque_manager/working.png +0 -0
- data/app/assets/javascripts/resque_manager/application.js +15 -0
- data/app/assets/javascripts/resque_manager/jquery-1.3.2.min.js +19 -0
- data/app/assets/javascripts/resque_manager/jquery.relatize_date.js +95 -0
- data/app/assets/javascripts/resque_manager/ranger.js +24 -0
- data/app/assets/stylesheets/resque_manager/application.css +14 -0
- data/app/assets/stylesheets/resque_manager/resque/resque.css +93 -0
- data/app/assets/stylesheets/resque_manager/resque/resque_reset.css +48 -0
- data/app/assets/stylesheets/resque_manager/resque_cleaner/cleaner.css +62 -0
- data/app/controllers/resque_manager/resque_controller.rb +313 -0
- data/app/helpers/resque_manager/application_helper.rb +4 -0
- data/app/helpers/resque_manager/resque_helper.rb +142 -0
- data/app/models/resque_manager/paginate.rb +54 -0
- data/app/views/layouts/resque_manager/application.html.erb +37 -0
- data/app/views/resque_manager/resque/_key.html.erb +17 -0
- data/app/views/resque_manager/resque/_limiter.html.erb +12 -0
- data/app/views/resque_manager/resque/_next_more.html.erb +10 -0
- data/app/views/resque_manager/resque/_paginate.html.erb +53 -0
- data/app/views/resque_manager/resque/_queues.html.erb +59 -0
- data/app/views/resque_manager/resque/_status_styles.erb +98 -0
- data/app/views/resque_manager/resque/_workers.html.erb +138 -0
- data/app/views/resque_manager/resque/_working.html.erb +69 -0
- data/app/views/resque_manager/resque/cleaner.html.erb +41 -0
- data/app/views/resque_manager/resque/cleaner_exec.html.erb +6 -0
- data/app/views/resque_manager/resque/cleaner_list.html.erb +172 -0
- data/app/views/resque_manager/resque/delayed.html.erb +35 -0
- data/app/views/resque_manager/resque/delayed_timestamp.html.erb +26 -0
- data/app/views/resque_manager/resque/error.erb +1 -0
- data/app/views/resque_manager/resque/overview.html.erb +4 -0
- data/app/views/resque_manager/resque/schedule.html.erb +96 -0
- data/app/views/resque_manager/resque/stats.html.erb +62 -0
- data/app/views/resque_manager/resque/status.html.erb +57 -0
- data/app/views/resque_manager/resque/statuses.html.erb +72 -0
- data/app/views/resque_manager/resque/workers.html.erb +1 -0
- data/config/routes.rb +38 -0
- data/config/sample_redis.yml +43 -0
- data/config/sample_resque_manager.yml +23 -0
- data/lib/resque_manager/engine.rb +9 -0
- data/lib/resque_manager/overrides/resque/failure/redis.rb +11 -0
- data/lib/resque_manager/overrides/resque/job.rb +69 -0
- data/lib/resque_manager/overrides/resque/resque.rb +8 -0
- data/lib/resque_manager/overrides/resque/worker.rb +291 -0
- data/lib/resque_manager/overrides/resque_scheduler/resque_scheduler.rb +58 -0
- data/lib/resque_manager/overrides/resque_status/chained_status.rb +46 -0
- data/lib/resque_manager/overrides/resque_status/hash.rb +12 -0
- data/lib/resque_manager/overrides/resque_status/status.rb +161 -0
- data/lib/resque_manager/recipes.rb +185 -0
- data/lib/resque_manager/version.rb +3 -0
- data/lib/resque_manager.rb +47 -0
- data/lib/tasks/failure.rake +8 -0
- data/lib/tasks/scheduler.rake +11 -0
- data/lib/tasks/worker.rake +129 -0
- data/test/dummy/README.rdoc +261 -0
- data/test/dummy/Rakefile +7 -0
- data/test/dummy/app/assets/javascripts/application.js +15 -0
- data/test/dummy/app/assets/stylesheets/application.css +13 -0
- data/test/dummy/app/controllers/application_controller.rb +3 -0
- data/test/dummy/app/helpers/application_helper.rb +2 -0
- data/test/dummy/app/views/layouts/application.html.erb +14 -0
- data/test/dummy/config/application.rb +65 -0
- data/test/dummy/config/boot.rb +10 -0
- data/test/dummy/config/environment.rb +5 -0
- data/test/dummy/config/environments/development.rb +37 -0
- data/test/dummy/config/environments/production.rb +67 -0
- data/test/dummy/config/environments/test.rb +37 -0
- data/test/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/test/dummy/config/initializers/inflections.rb +15 -0
- data/test/dummy/config/initializers/mime_types.rb +5 -0
- data/test/dummy/config/initializers/secret_token.rb +7 -0
- data/test/dummy/config/initializers/session_store.rb +8 -0
- data/test/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/test/dummy/config/locales/en.yml +5 -0
- data/test/dummy/config/routes.rb +4 -0
- data/test/dummy/config.ru +4 -0
- data/test/dummy/log/development.log +5045 -0
- data/test/dummy/public/404.html +26 -0
- data/test/dummy/public/422.html +26 -0
- data/test/dummy/public/500.html +25 -0
- data/test/dummy/public/favicon.ico +0 -0
- data/test/dummy/script/rails +6 -0
- data/test/dummy/tmp/cache/assets/C2A/A10/sprockets%2Fb2e622954654f415590723e9b882063e +0 -0
- data/test/dummy/tmp/cache/assets/C60/1D0/sprockets%2F8ed12e4193473760f95b973567a8c206 +0 -0
- data/test/dummy/tmp/cache/assets/CA1/970/sprockets%2Fc387148880e015d1eab0dc838b326022 +0 -0
- data/test/dummy/tmp/cache/assets/CAE/930/sprockets%2Fe227278d3c65d8aa1159da720263f771 +0 -0
- data/test/dummy/tmp/cache/assets/CD8/370/sprockets%2F357970feca3ac29060c1e3861e2c0953 +0 -0
- data/test/dummy/tmp/cache/assets/CDC/E30/sprockets%2Fe1207380d69eeee3284e02636c26f24a +0 -0
- data/test/dummy/tmp/cache/assets/CF1/720/sprockets%2Fd91a5918f5aa43a43c8135a67c78e989 +0 -0
- data/test/dummy/tmp/cache/assets/D0E/820/sprockets%2F00c6cc9dc46bf64347b3775d7d15541b +0 -0
- data/test/dummy/tmp/cache/assets/D16/180/sprockets%2F73d6fa09352cb76ac81e1683e832b93f +0 -0
- data/test/dummy/tmp/cache/assets/D27/170/sprockets%2Fec164819553e2e5b28f1efc9bd970978 +0 -0
- data/test/dummy/tmp/cache/assets/D2B/DA0/sprockets%2F989465d3ea8575dd0b54981a9e8add38 +0 -0
- data/test/dummy/tmp/cache/assets/D32/A10/sprockets%2F13fe41fee1fe35b49d145bcc06610705 +0 -0
- data/test/dummy/tmp/cache/assets/D37/1F0/sprockets%2F97119b908ebed2633edfd00ac90d9011 +0 -0
- data/test/dummy/tmp/cache/assets/D38/FB0/sprockets%2F74e5ba1cca7a1470d53c54fb60368b78 +0 -0
- data/test/dummy/tmp/cache/assets/D42/4E0/sprockets%2F0fa6e3c14356aa527d68a8d56fa37f28 +0 -0
- data/test/dummy/tmp/cache/assets/D43/C20/sprockets%2F1efd074fd1074b3dc88145b480ff961f +0 -0
- data/test/dummy/tmp/cache/assets/D46/CD0/sprockets%2F67f1ef70e7ede542318b8d55e25b16c3 +0 -0
- data/test/dummy/tmp/cache/assets/D4E/1B0/sprockets%2Ff7cbd26ba1d28d48de824f0e94586655 +0 -0
- data/test/dummy/tmp/cache/assets/D5A/EA0/sprockets%2Fd771ace226fc8215a3572e0aa35bb0d6 +0 -0
- data/test/dummy/tmp/cache/assets/D68/080/sprockets%2Fa26f2ae225aa4b87c462d540c7cf43f9 +0 -0
- data/test/dummy/tmp/cache/assets/D9A/B20/sprockets%2F0eddc19d46318e2e286cc171ae4cc73e +0 -0
- data/test/dummy/tmp/cache/assets/DA4/900/sprockets%2F515bf984438c6ec4b8a515fcc13baf8e +0 -0
- data/test/dummy/tmp/cache/assets/DBD/070/sprockets%2F60ffef45ddefd5c7746d17977fff0717 +0 -0
- data/test/dummy/tmp/cache/assets/DD7/AC0/sprockets%2Fc7c983c5c607dbfdb726eecc36146ca9 +0 -0
- data/test/dummy/tmp/cache/assets/DDC/400/sprockets%2Fcffd775d018f68ce5dba1ee0d951a994 +0 -0
- data/test/dummy/tmp/cache/assets/DF5/480/sprockets%2Fea4f3c726fc1046cad1ad243faf84e7d +0 -0
- data/test/dummy/tmp/cache/assets/E04/890/sprockets%2F2f5173deea6c795b8fdde723bb4b63af +0 -0
- data/test/dummy/tmp/cache/assets/E2B/7A0/sprockets%2Fd44ef07be0aa6d5b5dea4d37d7f72b4f +0 -0
- data/test/functional/resque_manager/resque_controller_test.rb +9 -0
- data/test/integration/navigation_test.rb +10 -0
- data/test/resque_manager_test.rb +7 -0
- data/test/test_helper.rb +15 -0
- data/test/unit/helpers/resque_manager/resque_helper_test.rb +6 -0
- metadata +307 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2dc1842d613d5bea7b454adf023b49640044354b
|
4
|
+
data.tar.gz: 1a4539c2703f96879d128047b7279332905c7336
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1b8bfb94ace1d479420b8be913c3b4efae77cd0d32b1e0a419fe96b1d15c4c272a162333fdf482bdd920a707ed228e19fb354fbe6aaf3d6ba00059628ffc918e
|
7
|
+
data.tar.gz: c7ca79eadf83075237a5990b2223e4a190ede2abeba24fa6ded8906552fc1afcb44dae2e689554f0bb7dd6884fe102fe48a4c43b37322647b990ed6a940fa283
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2009 Chris Wanstrath
|
2
|
+
Copyright (c) 2013 Kevin Tyll
|
3
|
+
|
4
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
a copy of this software and associated documentation files (the
|
6
|
+
"Software"), to deal in the Software without restriction, including
|
7
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be
|
13
|
+
included in all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.markdown
ADDED
@@ -0,0 +1,417 @@
|
|
1
|
+
Resque UI has been renamed to Resque Manager to better reflect what this engine really does. It manages your resque workers...through the UI.
|
2
|
+
|
3
|
+
ResqueManager
|
4
|
+
--------
|
5
|
+
|
6
|
+
Resque Manager is a Rails engine port of the Sinatra app that is included in Chris Wanstrath's resque gem. We love the gem and love the UI,
|
7
|
+
but just didn't want to add Sinatra to our stack and wanted to be able to manage the queues from the UI.
|
8
|
+
|
9
|
+
|
10
|
+
Installation
|
11
|
+
------------
|
12
|
+
|
13
|
+
sudo gem install resque_manager
|
14
|
+
|
15
|
+
Or just add it to your Gemfile
|
16
|
+
|
17
|
+
gem 'resque_manager'
|
18
|
+
|
19
|
+
If you have your default routes disabled, which you should if you have a RESTful API, then you'll need to add this to
|
20
|
+
the bottom of your routes.rb file.
|
21
|
+
|
22
|
+
# Resque Manager
|
23
|
+
mount ResqueManager::Engine => 'resque'
|
24
|
+
|
25
|
+
Once installed, you now have a resque controller, so you can get to the ui with: http://your_domain/resque.
|
26
|
+
|
27
|
+
Dependencies
|
28
|
+
------------
|
29
|
+
|
30
|
+
This engine now requires rails 3.2.0 or greater.
|
31
|
+
|
32
|
+
This engine now requires redis 2.0 or greater for the expiration of keys to work correctly.
|
33
|
+
|
34
|
+
This engine requires the redis 3.0 or higher gem
|
35
|
+
|
36
|
+
This engine requires the resque 1.24 or higher gem.
|
37
|
+
|
38
|
+
This engine now requires the resque-status 0.4.0 or higher gem.
|
39
|
+
|
40
|
+
This engine now requires the resque-cleaner 0.2 or higher gem.
|
41
|
+
|
42
|
+
These gems will all be installed for you automatically when you install resque_manager.
|
43
|
+
|
44
|
+
Configuration
|
45
|
+
--------------
|
46
|
+
|
47
|
+
There are a few things you need to configure, and a few more you can if you like. The easiest is to add an initializer:
|
48
|
+
|
49
|
+
ResqueManager.configure do |config|
|
50
|
+
# configure redis
|
51
|
+
config.redis_config = YAML.load(IO.read(Rails.root.join("config", "redis.yml")))["#{Rails.env}_resque"]
|
52
|
+
resque_manager_config = YAML.load(IO.read(Rails.root.join('config', 'resque_manager.yml')))[Rails.env]
|
53
|
+
# optional - set when you want your status keys to expire. Once expired, jobs will no longer show on the status page.
|
54
|
+
config.key_expiration = resque_manager_config['key_expiration']
|
55
|
+
# optional - Tell whether to run jobs inline or not.
|
56
|
+
config.inline = resque_manager_config['inline']
|
57
|
+
# optional - If you have workers in multiple applications that you want to control through a single app's UI, this this
|
58
|
+
# to a hash where the keys are your application names, and the values are the paths where the app is deployed.
|
59
|
+
config.applications = resque_manager_config['applications']
|
60
|
+
end
|
61
|
+
|
62
|
+
See the sample .yml files in config.
|
63
|
+
|
64
|
+
Capistrano Configuration
|
65
|
+
--------------
|
66
|
+
|
67
|
+
Added the ability to stop, start, and restart workers from the workers page. This requires capistrano, and capistrano-ext to be installed on all deployed servers.
|
68
|
+
|
69
|
+
![Manage Workers](http://img.skitch.com/20100308-ds6bgsnwqe6j9jn9yx8x7cxre3.png)
|
70
|
+
|
71
|
+
|
72
|
+
The controller calls cap tasks to manage the workers. To include the recipes in your application, add this line to your deploy.rb file:
|
73
|
+
|
74
|
+
require 'resque_manager/recipes'
|
75
|
+
|
76
|
+
You will also need to define the :resque role in your deploy/<environment>.rb file with the servers that will run your workers.
|
77
|
+
|
78
|
+
role :resque, 'server1', 'server2'
|
79
|
+
|
80
|
+
You will also need to make sure you have your rake path set in the deploy.rb file.
|
81
|
+
|
82
|
+
set :rake, "/opt/ruby-enterprise-1.8.6-20090421/bin/rake"
|
83
|
+
|
84
|
+
|
85
|
+
...using your own path of course.
|
86
|
+
|
87
|
+
The cap tasks included are:
|
88
|
+
|
89
|
+
cap resque:work # start a resque worker.
|
90
|
+
cap resque:workers # start multiple resque workers.
|
91
|
+
cap resque:quit_worker # Gracefully kill a worker. If the worker is working, it will finish before shutting down.
|
92
|
+
cap resque:quit_workers # Gracefully kill all workers on all servers. If the worker is working, it will finish before shutting down.
|
93
|
+
cap resque:kill_worker_with_impunity # Kill a rogue worker. If the worker is working, it will not finish and the job will go to the Failed queue as a DirtyExit. arg: host=ip pid=pid
|
94
|
+
cap resque:kill_workera_with_impunity # Kill all rogue workers on all servers. If the worker is working, it will not finish and the job will go to the Failed queue as a DirtyExit.
|
95
|
+
cap resque:restart_workers # Restart all workers on all servers
|
96
|
+
|
97
|
+
Times Fixed
|
98
|
+
-----------
|
99
|
+
|
100
|
+
The displaying of times is fixed. Some browsers displayed "Nan days ago" for all process times in the original Sinatra app.
|
101
|
+
Fixed the stats/keys page. The page wasn't showing the keys' types, size or values.
|
102
|
+
|
103
|
+
Display Process Status
|
104
|
+
----------------------
|
105
|
+
|
106
|
+
Added the ability to display process status in the worker "Processing" column on the overview, workers and working pages.
|
107
|
+
To do this, set the overview_status to the status message in your perform method.
|
108
|
+
|
109
|
+
Class YourClass
|
110
|
+
|
111
|
+
def perform(arg)
|
112
|
+
...your code here
|
113
|
+
overview_status = "Your status message"
|
114
|
+
...more code
|
115
|
+
overview_status = "Another status message"
|
116
|
+
...more code
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
This is really handy for those long running jobs to give you assurance the the job is really running or not.
|
121
|
+
|
122
|
+
![Status Messages](http://img.skitch.com/20100308-8mk5hrwnu462q2d23d51n8cjxp.png)
|
123
|
+
|
124
|
+
Restart Failed Jobs
|
125
|
+
-------------------
|
126
|
+
|
127
|
+
Added the Resque Cleaner gem to manage the failed queue. Have complete control over the failed queue now with the
|
128
|
+
Cleaner tab by querying and restarting failed jobs.
|
129
|
+
|
130
|
+
See what all you can do on the github page: https://github.com/ono/resque-cleaner
|
131
|
+
|
132
|
+
Remove Items from the Queue
|
133
|
+
---------------------------
|
134
|
+
|
135
|
+
Added the ability to remove jobs, from a queue
|
136
|
+
|
137
|
+
![Remove Items from the Queue](http://img.skitch.com/20100308-qukiw7bpsnr9y1saap7f8276qx.png)
|
138
|
+
|
139
|
+
View Processed Job Info
|
140
|
+
-----------------------
|
141
|
+
|
142
|
+
![Job Status](https://img.skitch.com/20110309-r9mhfgnn8w63ep29sik4c6khcj.png)
|
143
|
+
|
144
|
+
resque_manager now incorporates the resque-status gem and replaced the Processed tab with the Status tab. You can read about
|
145
|
+
what you can do with resque-status [here](https://github.com/quirkey/resque-status).
|
146
|
+
|
147
|
+
I've added some additional functionality to the resque-status gem. Namely, I've added a Resque::Plugins::ChainedStatus module. We process a lot of data files. Each part of the file's process is handled by a different worker. One worker may convert
|
148
|
+
a file into a different format, then another will parse that file and peel each record off the file and put each individual
|
149
|
+
record on a separate queue. A separate worker may then do any post processing when the file is complete.
|
150
|
+
|
151
|
+
I wanted all of that to show under a single status. So to do that, the very first worker class includes
|
152
|
+
Resque::Plugins::Status, and everything after that includes from Resque::Plugins::ChainedStatus. When you call #create on
|
153
|
+
the chained job from the preceding job, you just need to pass {'uuid' => uuid} as one of the hash arguments.
|
154
|
+
|
155
|
+
class DataContributionFile
|
156
|
+
Resque::Plugins::Status
|
157
|
+
|
158
|
+
@queue = :data_contribution
|
159
|
+
|
160
|
+
def perform
|
161
|
+
...your code here
|
162
|
+
tick "Retrieving file."
|
163
|
+
...more code
|
164
|
+
tick "Peeling #{file_path}"
|
165
|
+
SingleRecordLoader.create({'uuid' => uuid, 'row_data' => hash_of_row_data, 'rows_in_file' => total_rows})
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
class SingleRecordLoader
|
170
|
+
Resque::Plugins::ChainedStatus
|
171
|
+
@queue = :single_record_loader
|
172
|
+
|
173
|
+
def completed(*messages)
|
174
|
+
if counter(:processed) >= options[:rows_in_file].to_i
|
175
|
+
super("#{options[:rows_in_file]} records processed: Started(#{status.time.to_s(:eastern_time_zone_long)}) Finished(#{Time.now.to_s(:eastern_time_zone_long)})")
|
176
|
+
post_process
|
177
|
+
end
|
178
|
+
|
179
|
+
def perform
|
180
|
+
...your code here
|
181
|
+
|
182
|
+
incr_counter(:processed)
|
183
|
+
at((self.processed), options[:rows_in_file], "#{(self.processed)} of #{options[:rows_in_file]} completed.")
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
So now, the data_contribution worker and the single_record_loader workers will update the same status on the status page.
|
188
|
+
You can call #tick or #set_status to add messages along the way too. Note: These statuses are shown in the Messages column
|
189
|
+
of the status page. When you set the overview_status, those messages appear in the Processing column of the Overview, Workers,
|
190
|
+
and Working pages.
|
191
|
+
|
192
|
+
You will want to override the completed method so that it isn't called until the very end of the entire process.
|
193
|
+
|
194
|
+
I've also added two more methods, #incr_counter(:counter) and #count(:counter). We have dozens of single_record_loader
|
195
|
+
workers processing records at a time. You encounter a race condition when they are all calling #at at the same time to
|
196
|
+
update the :num attribute. So I created these two methods to atomically increment a dedicated counter. Just call #incr_counter
|
197
|
+
and pass in a symbol for what you want to call the counter. You can create any number of different counters
|
198
|
+
for different purposes. We keep track of different validation issues for each record. Use #counter and pass it the same
|
199
|
+
symbol to read the integer back. The redis entries created by these methods all get cleaned up with a call to Resque::Plugins::Status::Hash.clear(uuid)
|
200
|
+
|
201
|
+
When you kill a job on the UI, it will kill all the workers in the chain.
|
202
|
+
|
203
|
+
Pause a Worker
|
204
|
+
--------------
|
205
|
+
|
206
|
+
The workers page now has a button for every worker to pause that worker.
|
207
|
+
|
208
|
+
### Regular Workers
|
209
|
+
|
210
|
+
For workers that do not include Resque::Plugins::Status, this will pause the worker, but not the job. So if the worker is in
|
211
|
+
the middle of a job when it is paused, it will finish it's process, but then will not pick anything else up off the queue.
|
212
|
+
|
213
|
+
You can manually pause the processing though using the worker object.
|
214
|
+
|
215
|
+
Class YourClass
|
216
|
+
|
217
|
+
def perform(arg)
|
218
|
+
...your code here
|
219
|
+
|
220
|
+
if worker && worker.paused?
|
221
|
+
loop do
|
222
|
+
break unless worker.paused?
|
223
|
+
sleep 60
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
...continue
|
229
|
+
end
|
230
|
+
|
231
|
+
### Resque::Plugins::Status Workers
|
232
|
+
|
233
|
+
For workers that do include Resque::Plugins::Status or Resque::Plugins::ChainedStatus, this will pause the worker, and will automatically pause the job it is processing
|
234
|
+
on the next call to #tick. The worker is also available to the class that includes Resque::Plugins::Status so you can manually check it's status as well.
|
235
|
+
|
236
|
+
Class YourClass
|
237
|
+
|
238
|
+
Resque::Plugins::Status
|
239
|
+
|
240
|
+
def perform
|
241
|
+
...your code here
|
242
|
+
tick "Retrieving file." #You're process will pause here automatically and the status on the Status tab will be set to paused if the worker is paused.
|
243
|
+
|
244
|
+
#Alternatively, you have access to the worker, so you can pause the process yourself too.
|
245
|
+
if worker && worker.paused?
|
246
|
+
# There could be workers in a chained job still doing work.
|
247
|
+
loop do
|
248
|
+
pause! unless status.paused?
|
249
|
+
break unless worker.paused?
|
250
|
+
sleep 60
|
251
|
+
end
|
252
|
+
tick("Job resumed at #{Time.now}")
|
253
|
+
end
|
254
|
+
|
255
|
+
...continue
|
256
|
+
end
|
257
|
+
|
258
|
+
This will only pause the work being processed by the worker that was paused. If the job is paused by the call to #tick,
|
259
|
+
the job will sleep for 60 seconds before checking the status again.
|
260
|
+
|
261
|
+
You may have a series of classes that include Resque::Plugins::ChainedStatus and you want all processing in the chain stopped.
|
262
|
+
|
263
|
+
Class YourClass
|
264
|
+
|
265
|
+
Resque::Plugins::ChainedStatus
|
266
|
+
|
267
|
+
def perform
|
268
|
+
...your code here
|
269
|
+
tick "Retrieving file." #You're process will pause here automatically and the status on the Status tab will be set to paused if the worker is paused.
|
270
|
+
|
271
|
+
#Alternatively, you have access to the worker, so you can pause the process yourself too.
|
272
|
+
if (worker && worker.paused?) || status.paused?
|
273
|
+
# There could be workers in a chained job still doing work.
|
274
|
+
loop do
|
275
|
+
pause! unless status.paused?
|
276
|
+
break unless worker.paused?
|
277
|
+
sleep 60
|
278
|
+
end
|
279
|
+
tick("Job resumed at #{Time.now}")
|
280
|
+
end
|
281
|
+
|
282
|
+
...continue
|
283
|
+
end
|
284
|
+
|
285
|
+
By looking at the status.paused? method too, this process will stop, even if it's worker has not been paused.
|
286
|
+
But be aware, if this worker does other jobs, it will not process anything else and it's queue could get backed up.
|
287
|
+
This is where pausing one worker, could affect other, unrelated workers and jobs from getting backed up as well.
|
288
|
+
|
289
|
+
Throttle a Queue
|
290
|
+
----------------
|
291
|
+
|
292
|
+
A throttle method has been added. This is useful if you have a queue that tends to have very high volume, for example,
|
293
|
+
the queue that process all the individual records of a file. You don't want to load that queue up with 1 million entries,
|
294
|
+
possibly blowing out the memory of your Redis server.
|
295
|
+
|
296
|
+
CSV.foreach(self.file_path, :headers => true, :quote_char => '"') do |row|
|
297
|
+
Resque.throttle(:single_record_loader, 10000, 30)
|
298
|
+
SingleRecordLoader.create({'uuid' => uuid, 'row_data' => row.to_hash, 'rows_in_file' => total_rows})
|
299
|
+
end
|
300
|
+
|
301
|
+
Putting the throttle before enqueing the SingleRecordLoader will check the single_record_loader queue to make sure it has
|
302
|
+
less than 10000 entries in it before proceeding. If is has 10000 or more entries, it will sleep for 30 seconds before checking again.
|
303
|
+
|
304
|
+
|
305
|
+
Multi-Threaded Workers
|
306
|
+
---------------------
|
307
|
+
With Jruby, you have to specify the amount of memory to allocate when you start up the jvm. This has proven inefficient for us because different workers
|
308
|
+
require different amounts of memory. We have standardized our jvm configuration for the workers, which means we have to start each worker with
|
309
|
+
the maximum amount of memory needed by the most memory intensive worker. This means we are wasting a lot of resources for the workers that don't
|
310
|
+
require as much memory.
|
311
|
+
|
312
|
+
Our answer was to make the workers multi-threaded. Now you can pass multiple workers and queues into the rake task, each worker will be started in separate
|
313
|
+
threads within the same process.
|
314
|
+
|
315
|
+
NOTE: The convention to identify which queues are monitored by which worker is to prefix each worker with a '#' in the rake task argument.
|
316
|
+
|
317
|
+
rake QUEUE=#file_loader#file_loader,email resque:work
|
318
|
+
|
319
|
+
This will start up 2 workers, 1 will work the file_loader queue, and one will work the file_loader and email queue.
|
320
|
+
|
321
|
+
Be aware that when you stop a worker, it will stop all the workers within that process.
|
322
|
+
|
323
|
+
Workers in Multiple Applications
|
324
|
+
--------------------------------
|
325
|
+
|
326
|
+
You now have the ability to manage workers in multiple applications from a single app's UI. In other words, the workers
|
327
|
+
can be in a different app than the UI itself. I did this becase our app had become very large. I did not want to have to
|
328
|
+
load up our huge monolithic app just to run a small worker. So by splitting the workers out into a separate application, I
|
329
|
+
save server resources, but only our main app mounts resque_manager in the routes.rb file and manages all the workers.
|
330
|
+
|
331
|
+
To do this, you'll need to add resque_manager to the Gemfile of all apps containing workers as well as the app with the UI.
|
332
|
+
|
333
|
+
You will also need to tell the app that mounts resque_manager UI what applications contain workers and the paths where they are deployed to.
|
334
|
+
|
335
|
+
ResqueManager.applications = {application1: '/Users/ktyll/rails_sites/git/application1',
|
336
|
+
application2: '/Users/ktyll/rails_sites/git/application2'}
|
337
|
+
|
338
|
+
See the sample initializer above.
|
339
|
+
|
340
|
+
By setting the applications hash, a select box will display on the workers page so you can select the application where the
|
341
|
+
worker is you want to start.
|
342
|
+
|
343
|
+
![Multiple Apps](https://www.evernote.com/shard/s198/sh/04538799-da21-48b4-ab63-ac19c815dc85/62f73190d926844e5f97ea74f0bedb6c/deep/0/Pasted%20Image%207/30/13%2010:30%20AM.png)
|
344
|
+
|
345
|
+
You do not need to include the application that has mounted the resque_manager UI in this hash. If you do not select an application
|
346
|
+
from the drop down, it will assume the worker is in the same app.
|
347
|
+
|
348
|
+
After Deploy Hooks
|
349
|
+
------------------
|
350
|
+
|
351
|
+
The resque:restart_workers cap task can be added as an after deploy task to refresh your workers. Without this, your workers will
|
352
|
+
continue to run your old code base after a deployment.
|
353
|
+
|
354
|
+
To make it work add the callbacks in your deploy.rb file:
|
355
|
+
|
356
|
+
after "deploy", "resque:restart_workers"
|
357
|
+
after "deploy:migrations", "resque:restart_workers"
|
358
|
+
|
359
|
+
|
360
|
+
Resque Scheduler
|
361
|
+
----------------
|
362
|
+
|
363
|
+
If resque-scheduler is installed, the Schedule and Delayed tabs will display.
|
364
|
+
|
365
|
+
Be sure you add the resque_scheduler gem before the resque_manager gem in your Gemfile:
|
366
|
+
|
367
|
+
gem 'resque-scheduler', :require => 'resque_scheduler'
|
368
|
+
gem 'resque_manager'
|
369
|
+
|
370
|
+
The Schedule tab functionality has been enhanced to be able to add jobs to the scheduler from the UI. This means you don't
|
371
|
+
need to edit a static file that gets loaded on initialization. This also means you don't have to deploy that file every time
|
372
|
+
you edit your schedule.
|
373
|
+
|
374
|
+
You can also create different schedules on different servers in your farm. You specify
|
375
|
+
the IP address you want to schedule a job to run on, and it will add the job to the schedule on that server. You can also start and
|
376
|
+
stop the scheduler on each server from the Schedule tab.
|
377
|
+
|
378
|
+
![Resque Scheduler](http://img.skitch.com/20100308-quccysfiwtgubpw286ka2enr9m.png)
|
379
|
+
|
380
|
+
The caveat to this is the Arguments value must be entered in the text box as JSON in order for the arguments to get parsed and stored
|
381
|
+
in the schedule correctly. I find the easiest thing to do is to perform a Resque.encode on my parameters list in script/console. If
|
382
|
+
we have a method that takes 3 parameters:
|
383
|
+
|
384
|
+
>> Resque.encode([['300'],1,{"start_date"=>"2010-02-01","end_date"=>"2010-02-28"}])
|
385
|
+
=> "[["300"],1,{"end_date":"2010-02-28","start_date":"2010-02-01"}]"
|
386
|
+
|
387
|
+
The first parameter is an array of strings, the second parameter is an integer, and the third parameter is a hash.
|
388
|
+
Remembering that the arguments are stored in an array, all the parameters need to be in an array when there is more than one.
|
389
|
+
|
390
|
+
Any string arguments need to be quoted in the text box:
|
391
|
+
|
392
|
+
>> Resque.encode("Hello World")
|
393
|
+
=> ""Hello World""
|
394
|
+
|
395
|
+
### Additional cap tasks added:
|
396
|
+
|
397
|
+
cap resque:quit_scheduler # Gracefully kill the scheduler on a server.
|
398
|
+
cap resque:scheduler # start a resque worker.
|
399
|
+
cap resque:scheduler_status # Determine if the scheduler is running or not
|
400
|
+
|
401
|
+
Delayed Tab
|
402
|
+
-----------
|
403
|
+
|
404
|
+
I have not tested or added any functionality to the Delayed tab. I get a RuntimeError: -ERR invalid bulk write count
|
405
|
+
any time I try to do a Resque.enqueue_at. I've spent some time researching, and assume it's something with my version combinations.
|
406
|
+
I believe it's a Redis issue and not a Resque-Scheduler issue. But since I'm not using it, I haven't put a great deal of
|
407
|
+
time into resolving it.
|
408
|
+
|
409
|
+
Copyright (c) 2009 Chris Wanstrath
|
410
|
+
Copyright (c) 2010 Ben VandenBos
|
411
|
+
Copyright (c) 2010 Aaron Quint
|
412
|
+
Copyright (c) 2011 Tatsuya Ono
|
413
|
+
Copyright (c) 2013 Kevin Tyll, released under the MIT license
|
414
|
+
|
415
|
+
Thanks to Karl Baum for doing the original heavy lifting for converting this to a rails engine for rails 3.
|
416
|
+
|
417
|
+
Much thanks goes to Brian Ketelsen for the ideas for the improved functionality for the UI.
|
data/Rakefile
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
begin
|
3
|
+
require 'bundler/setup'
|
4
|
+
rescue LoadError
|
5
|
+
puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
|
6
|
+
end
|
7
|
+
begin
|
8
|
+
require 'rdoc/task'
|
9
|
+
rescue LoadError
|
10
|
+
require 'rdoc/rdoc'
|
11
|
+
require 'rake/rdoctask'
|
12
|
+
RDoc::Task = Rake::RDocTask
|
13
|
+
end
|
14
|
+
|
15
|
+
RDoc::Task.new(:rdoc) do |rdoc|
|
16
|
+
rdoc.rdoc_dir = 'rdoc'
|
17
|
+
rdoc.title = 'ResqueManager'
|
18
|
+
rdoc.options << '--line-numbers'
|
19
|
+
rdoc.rdoc_files.include('README.markdown')
|
20
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
21
|
+
rdoc.rdoc_files.include('app/**/*.rb')
|
22
|
+
end
|
23
|
+
|
24
|
+
APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
|
25
|
+
load 'rails/tasks/engine.rake'
|
26
|
+
|
27
|
+
|
28
|
+
|
29
|
+
Bundler::GemHelper.install_tasks
|
30
|
+
|
31
|
+
require 'rake/testtask'
|
32
|
+
|
33
|
+
Rake::TestTask.new(:test) do |t|
|
34
|
+
t.libs << 'lib'
|
35
|
+
t.libs << 'test'
|
36
|
+
t.pattern = 'test/**/*_test.rb'
|
37
|
+
t.verbose = false
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
task :default => :test
|
Binary file
|
Binary file
|
Binary file
|
@@ -0,0 +1,15 @@
|
|
1
|
+
// This is a manifest file that'll be compiled into application.js, which will include all the files
|
2
|
+
// listed below.
|
3
|
+
//
|
4
|
+
// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
|
5
|
+
// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
|
6
|
+
//
|
7
|
+
// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
|
8
|
+
// the compiled file.
|
9
|
+
//
|
10
|
+
// WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD
|
11
|
+
// GO AFTER THE REQUIRES BELOW.
|
12
|
+
//
|
13
|
+
//= require resque_manager/jquery-1.3.2.min
|
14
|
+
//= require resque_manager/jquery.relatize_date
|
15
|
+
//= require resque_manager/ranger
|