mobilize-base 1.0.5 → 1.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/README.md +30 -14
- data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb +22 -11
- data/lib/mobilize-base/handlers/resque.rb +5 -5
- data/lib/mobilize-base/jobtracker.rb +22 -8
- data/lib/mobilize-base/models/job.rb +27 -3
- data/lib/mobilize-base/models/runner.rb +18 -10
- data/lib/mobilize-base/models/task.rb +25 -12
- data/lib/mobilize-base/rakes.rb +32 -18
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/samples/resque_web.rb +4 -0
- data/mobilize-base.gemspec +0 -1
- data/test/mobilize-base_test.rb +14 -5
- metadata +5 -21
- data/lib/mobilize-base/tasks/mobilize-base.rake +0 -2
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -27,10 +27,11 @@ Table Of Contents
|
|
27
27
|
* [Google Drive](#section_Configure_Google_Drive)
|
28
28
|
* [Jobtracker](#section_Configure_Jobtracker)
|
29
29
|
* [Resque](#section_Configure_Resque)
|
30
|
+
* [Resque-Web](#section_Configure_Resque-Web)
|
30
31
|
* [Gridfs](#section_Configure_Gridfs)
|
31
32
|
* [Mongoid](#section_Configure_Mongoid)
|
32
33
|
* [Start](#section_Start)
|
33
|
-
* [Start
|
34
|
+
* [Start Resque-Web](#section_Start_Start_Resque-Web)
|
34
35
|
* [Set Environment](#section_Start_Set_Environment)
|
35
36
|
* [Create User](#section_Start_Create_User)
|
36
37
|
* [Start Workers](#section_Start_Start_Workers)
|
@@ -262,22 +263,42 @@ It needs the below parameters, which can be found in the [lib/samples][git_sampl
|
|
262
263
|
production, but feel free to adjust depending on your hardware.
|
263
264
|
* redis_port - you should probably leave this alone, it specifies the
|
264
265
|
default port for dev and prod and a separate one for testing.
|
266
|
+
* web_port - this specifies the port under which resque-web operates
|
265
267
|
|
266
268
|
``` yml
|
267
269
|
development:
|
268
270
|
queue_name: 'mobilize'
|
269
271
|
max_workers: 4
|
270
272
|
redis_port: 6379
|
273
|
+
web_port: 8282
|
271
274
|
test:
|
272
275
|
queue_name: 'mobilize'
|
273
276
|
max_workers: 4
|
274
277
|
redis_port: 9736
|
278
|
+
web_port: 8282
|
275
279
|
production:
|
276
280
|
queue_name: 'mobilize'
|
277
281
|
max_workers: 36
|
278
282
|
redis_port: 6379
|
283
|
+
web_port: 8282
|
279
284
|
```
|
280
285
|
|
286
|
+
<a name='section_Configure_Resque-Web'></a>
|
287
|
+
### Configure Resque-Web
|
288
|
+
|
289
|
+
Please change your default username and password in the resque_web.rb
|
290
|
+
file in your config folder, reproduced below:
|
291
|
+
|
292
|
+
``` ruby
|
293
|
+
#comment out the below if you want no authentication on your web portal (not recommended)
|
294
|
+
Resque::Server.use(Rack::Auth::Basic) do |user, password|
|
295
|
+
[user, password] == ['admin', 'changeyourpassword']
|
296
|
+
end
|
297
|
+
```
|
298
|
+
|
299
|
+
This file is passed as a config file argument to
|
300
|
+
mobilize_base:resque_web task, as detailed in [Start Resque-Web](#section_Start_Start_Resque-Web).
|
301
|
+
|
281
302
|
<a name='section_Configure_Gridfs'></a>
|
282
303
|
### Configure Gridfs
|
283
304
|
|
@@ -350,22 +371,17 @@ A Mobilize instance can be considered "started" or "running" when you have:
|
|
350
371
|
<a name='section_Start_Start_resque-web'></a>
|
351
372
|
### Start resque-web
|
352
373
|
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
``` ruby
|
357
|
-
gem install resque
|
358
|
-
```
|
374
|
+
Mobilize ships with its own rake task to start resque web -- you can do
|
375
|
+
the following:
|
359
376
|
|
360
|
-
then, you can do
|
361
377
|
|
362
|
-
$
|
378
|
+
$ MOBILIZE_ENV=<environment> rake mobilize_base:resque_web
|
363
379
|
|
364
|
-
|
380
|
+
This will start a resque_web instance with the port specified in your
|
381
|
+
resque.yml and the config/auth scheme specified in your resque_web.rb.
|
365
382
|
|
366
|
-
|
367
|
-
|
368
|
-
[Resque Standalone section][resque-web].
|
383
|
+
More detail on the
|
384
|
+
[Resque-Web Standalone section][resque-web].
|
369
385
|
|
370
386
|
<a name='section_Start_Set_Environment'></a>
|
371
387
|
### Set Environment
|
@@ -374,7 +390,7 @@ Mobilize takes the environment from your Rails.env if you're running
|
|
374
390
|
Rails, or assumes "development." You can specify "development", "test",
|
375
391
|
or "production," as per the yml files.
|
376
392
|
|
377
|
-
Otherwise, it takes it from MOBILIZE_ENV parameter,
|
393
|
+
Otherwise, it takes it from MOBILIZE_ENV parameter, as in:
|
378
394
|
|
379
395
|
``` ruby
|
380
396
|
> ENV['MOBILIZE_ENV'] = 'production'
|
@@ -1,6 +1,7 @@
|
|
1
1
|
module GoogleDrive
|
2
2
|
class ClientLoginFetcher
|
3
3
|
def request_raw(method, url, data, extra_header, auth)
|
4
|
+
clf = self
|
4
5
|
#this is patched to handle server errors due to http chaos
|
5
6
|
uri = URI.parse(url)
|
6
7
|
response = nil
|
@@ -14,22 +15,32 @@ module GoogleDrive
|
|
14
15
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
15
16
|
#set 600 to allow for large downloads
|
16
17
|
http.read_timeout = 600
|
17
|
-
response =
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
18
|
+
response = begin
|
19
|
+
clf.http_call(http, method, uri, data, extra_header, auth)
|
20
|
+
rescue
|
21
|
+
#timeouts etc.
|
22
|
+
nil
|
23
|
+
end
|
24
|
+
if response.nil?
|
25
|
+
attempts +=1
|
26
|
+
else
|
27
|
+
if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
|
28
|
+
if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
|
29
|
+
if sleep_time
|
30
|
+
sleep_time = sleep_time * attempts
|
31
|
+
else
|
32
|
+
sleep_time = (rand*100).to_i
|
33
|
+
end
|
22
34
|
else
|
23
|
-
sleep_time =
|
35
|
+
sleep_time = 10
|
24
36
|
end
|
25
|
-
|
26
|
-
sleep_time
|
37
|
+
attempts += 1
|
38
|
+
puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
|
39
|
+
sleep sleep_time
|
27
40
|
end
|
28
|
-
attempts += 1
|
29
|
-
puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
|
30
|
-
sleep sleep_time
|
31
41
|
end
|
32
42
|
end
|
43
|
+
raise "No response after 5 attempts" if response.nil?
|
33
44
|
raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
|
34
45
|
return response
|
35
46
|
end
|
@@ -17,7 +17,6 @@ module Mobilize
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def Resque.workers(state="all")
|
20
|
-
raise "invalid state #{state}" unless ['all','idle','working','timeout'].include?(state)
|
21
20
|
workers = ::Resque.workers.select{|w| w.queues.first == Resque.queue_name}
|
22
21
|
return workers if state == 'all'
|
23
22
|
working_workers = workers.select{|w| w.job['queue']== Resque.queue_name}
|
@@ -28,6 +27,7 @@ module Mobilize
|
|
28
27
|
return stale_workers if state == 'stale'
|
29
28
|
timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
|
30
29
|
return timeout_workers if state == 'timeout'
|
30
|
+
raise "invalid state #{state}"
|
31
31
|
end
|
32
32
|
|
33
33
|
def Resque.failures
|
@@ -36,7 +36,6 @@ module Mobilize
|
|
36
36
|
|
37
37
|
#active state refers to jobs that are either queued or working
|
38
38
|
def Resque.jobs(state="active")
|
39
|
-
raise "invalid state #{state}" unless ['all','queued','working','active','timeout','failed'].include?(state)
|
40
39
|
working_jobs = Resque.workers('working').map{|w| w.job['payload']}
|
41
40
|
return working_jobs if state == 'working'
|
42
41
|
queued_jobs = ::Resque.peek(Resque.queue_name,0,0).to_a
|
@@ -47,6 +46,7 @@ module Mobilize
|
|
47
46
|
timeout_jobs = Resque.workers("timeout").map{|w| w.job['payload']}
|
48
47
|
return timeout_jobs if state == 'timeout'
|
49
48
|
return working_jobs + queued_jobs + failed_jobs if state == 'all'
|
49
|
+
raise "invalid state #{state}"
|
50
50
|
end
|
51
51
|
|
52
52
|
def Resque.active_paths
|
@@ -125,13 +125,13 @@ module Mobilize
|
|
125
125
|
dir_envs = "MOBILIZE_ENV=#{Base.env} " +
|
126
126
|
"MOBILIZE_CONFIG_DIR=#{Base.config_dir} " +
|
127
127
|
"MOBILIZE_LOG_DIR=#{Base.log_dir}"
|
128
|
-
"(cd #{Base.root};rake #{dir_envs}
|
128
|
+
"(cd #{Base.root};rake #{dir_envs} mobilize_base:work) >> #{Resque.log_path} 2>&1 &".bash
|
129
129
|
end
|
130
130
|
end
|
131
131
|
|
132
132
|
def Resque.kill_idle_workers(count=nil)
|
133
133
|
idle_pids = Resque.workers('idle').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
|
134
|
-
if count>idle_pids.length or count == 0
|
134
|
+
if count.to_i > idle_pids.length or count == 0
|
135
135
|
return false
|
136
136
|
elsif count
|
137
137
|
"kill #{idle_pids[0..count-1].join(" ")}".bash
|
@@ -141,7 +141,7 @@ module Mobilize
|
|
141
141
|
return true
|
142
142
|
end
|
143
143
|
|
144
|
-
def Resque.
|
144
|
+
def Resque.kill_idle_and_stale_workers
|
145
145
|
idle_pids = Resque.workers('idle').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
|
146
146
|
stale_pids = Resque.workers('stale').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
|
147
147
|
idle_stale_pids = (idle_pids & stale_pids)
|
@@ -75,6 +75,10 @@ module Mobilize
|
|
75
75
|
Resque.kill_idle_workers
|
76
76
|
end
|
77
77
|
|
78
|
+
def Jobtracker.kill_idle_and_stale_workers
|
79
|
+
Resque.kill_idle_and_stale_workers
|
80
|
+
end
|
81
|
+
|
78
82
|
def Jobtracker.prep_workers
|
79
83
|
Resque.prep_workers
|
80
84
|
end
|
@@ -202,15 +206,25 @@ module Mobilize
|
|
202
206
|
|
203
207
|
def Jobtracker.deployed_at
|
204
208
|
#assumes deploy is as of last commit, or as of last deploy time
|
205
|
-
#as given by the
|
209
|
+
#as given by the REVISION file in the root folder
|
206
210
|
deploy_time = begin
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
211
|
+
%{git log -1 --format="%cd"}.bash
|
212
|
+
rescue
|
213
|
+
revision_path = "#{ENV['PWD']}/REVISION"
|
214
|
+
"touch #{revision_path}".bash unless File.exists?(revision_path)
|
215
|
+
revision_string = "ls -l #{revision_path}".bash
|
216
|
+
revision_rows = revision_string.split("\n").map{|lss| lss.strip.split(" ")}
|
217
|
+
mod_time = revision_rows.map do |lsr|
|
218
|
+
if lsr.length == 8
|
219
|
+
#ubuntu
|
220
|
+
lsr[5..6].join(" ")
|
221
|
+
elsif lsr.length == 9
|
222
|
+
#osx
|
223
|
+
lsr[5..7].join(" ")
|
224
|
+
end
|
225
|
+
end.first
|
226
|
+
mod_time
|
227
|
+
end.to_s.strip
|
214
228
|
Time.parse(deploy_time)
|
215
229
|
end
|
216
230
|
|
@@ -5,8 +5,6 @@ module Mobilize
|
|
5
5
|
field :path, type: String
|
6
6
|
field :active, type: Boolean
|
7
7
|
field :trigger, type: String
|
8
|
-
field :status, type: String
|
9
|
-
field :last_completed_at, type: Time
|
10
8
|
|
11
9
|
index({ path: 1})
|
12
10
|
|
@@ -26,6 +24,32 @@ module Mobilize
|
|
26
24
|
return j
|
27
25
|
end
|
28
26
|
|
27
|
+
def status
|
28
|
+
#last task status
|
29
|
+
j = self
|
30
|
+
j.active_task.status
|
31
|
+
end
|
32
|
+
|
33
|
+
def active_task
|
34
|
+
j = self
|
35
|
+
#latest started at or first
|
36
|
+
j.tasks.select{|t| t.started_at}.sort_by{|t| t.started_at}.last || j.tasks.first
|
37
|
+
end
|
38
|
+
|
39
|
+
def completed_at
|
40
|
+
j = self
|
41
|
+
j.tasks.last.completed_at
|
42
|
+
end
|
43
|
+
|
44
|
+
def failed_at
|
45
|
+
j = self
|
46
|
+
j.tasks.last.failed_at
|
47
|
+
end
|
48
|
+
|
49
|
+
def status_at
|
50
|
+
j.active_task.status_at
|
51
|
+
end
|
52
|
+
|
29
53
|
#convenience methods
|
30
54
|
def runner
|
31
55
|
j = self
|
@@ -41,7 +65,7 @@ module Mobilize
|
|
41
65
|
def is_due?
|
42
66
|
j = self
|
43
67
|
return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
|
44
|
-
last_run = j.
|
68
|
+
last_run = j.completed_at
|
45
69
|
#check trigger
|
46
70
|
trigger = j.trigger
|
47
71
|
return true if trigger == 'once'
|
@@ -5,7 +5,9 @@ module Mobilize
|
|
5
5
|
field :path, type: String
|
6
6
|
field :active, type: Boolean
|
7
7
|
field :status, type: String
|
8
|
-
field :
|
8
|
+
field :started_at, type: Time
|
9
|
+
field :status_at, type: Time
|
10
|
+
field :completed_at, type: Time
|
9
11
|
|
10
12
|
index({ path: 1})
|
11
13
|
|
@@ -13,9 +15,9 @@ module Mobilize
|
|
13
15
|
%w{name active trigger status task1 task2 task3 task4 task5}
|
14
16
|
end
|
15
17
|
|
16
|
-
def
|
18
|
+
def cached_at
|
17
19
|
r = self
|
18
|
-
Dataset.find_or_create_by_path(r.path).
|
20
|
+
Dataset.find_or_create_by_path(r.path).cached_at
|
19
21
|
end
|
20
22
|
|
21
23
|
def worker
|
@@ -51,12 +53,12 @@ module Mobilize
|
|
51
53
|
j.tasks.first.enqueue!
|
52
54
|
end
|
53
55
|
rescue ScriptError, StandardError => exc
|
54
|
-
|
56
|
+
r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
|
55
57
|
j.update_attributes(:active=>false)
|
56
58
|
end
|
57
59
|
end
|
58
60
|
r.update_gsheet(gdrive_slot)
|
59
|
-
r.update_attributes(:
|
61
|
+
r.update_attributes(:completed_at=>Time.now.utc)
|
60
62
|
end
|
61
63
|
|
62
64
|
def dataset
|
@@ -77,7 +79,7 @@ module Mobilize
|
|
77
79
|
r = self
|
78
80
|
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
79
81
|
jobs_sheet.add_headers(r.headers)
|
80
|
-
jobs_sheet.delete_sheet1
|
82
|
+
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
81
83
|
return jobs_sheet
|
82
84
|
end
|
83
85
|
|
@@ -128,7 +130,12 @@ module Mobilize
|
|
128
130
|
def update_gsheet(gdrive_slot)
|
129
131
|
r = self
|
130
132
|
jobs_gsheet = r.gsheet(gdrive_slot)
|
131
|
-
|
133
|
+
upd_jobs = r.jobs.select do |j|
|
134
|
+
j.completed_at.nil? ||
|
135
|
+
j.completed_at > j.runner.completed_at ||
|
136
|
+
(j.failed_at and j.failed_at > j.runner.completed_at)
|
137
|
+
end
|
138
|
+
upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
|
132
139
|
jobs_gsheet.add_or_update_rows(upd_rows)
|
133
140
|
r.update_status("gsheet updated")
|
134
141
|
return true
|
@@ -152,7 +159,7 @@ module Mobilize
|
|
152
159
|
|
153
160
|
def update_status(msg)
|
154
161
|
r = self
|
155
|
-
r.update_attributes(:status=>msg)
|
162
|
+
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
156
163
|
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
157
164
|
return true
|
158
165
|
end
|
@@ -165,12 +172,13 @@ module Mobilize
|
|
165
172
|
def is_due?
|
166
173
|
r = self.reload
|
167
174
|
return false if r.is_working?
|
168
|
-
|
169
|
-
return true if r.
|
175
|
+
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
176
|
+
return true if r.started_at.nil? or r.started_at < prev_due_time
|
170
177
|
end
|
171
178
|
|
172
179
|
def enqueue!
|
173
180
|
r = self
|
181
|
+
r.update_attributes(:started_at=>Time.now.utc)
|
174
182
|
::Resque::Job.create("mobilize",Runner,r.path,{})
|
175
183
|
return true
|
176
184
|
end
|
@@ -7,8 +7,10 @@ module Mobilize
|
|
7
7
|
field :call, type: String
|
8
8
|
field :param_string, type: Array
|
9
9
|
field :status, type: String
|
10
|
-
field :
|
11
|
-
field :
|
10
|
+
field :completed_at, type: Time
|
11
|
+
field :started_at, type: Time
|
12
|
+
field :failed_at, type: Time
|
13
|
+
field :status_at, type: Time
|
12
14
|
|
13
15
|
index({ path: 1})
|
14
16
|
|
@@ -27,9 +29,14 @@ module Mobilize
|
|
27
29
|
Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
|
28
30
|
end
|
29
31
|
|
32
|
+
def log_dataset
|
33
|
+
t = self
|
34
|
+
Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/log")
|
35
|
+
end
|
36
|
+
|
30
37
|
def params
|
31
38
|
t = self
|
32
|
-
t.param_string.split(",").map do |p|
|
39
|
+
t.param_string.split(",").map do |p|
|
33
40
|
ps = p.strip
|
34
41
|
ps = ps[1..-1] if ['"',"'"].include?(ps[0])
|
35
42
|
ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
|
@@ -69,12 +76,16 @@ module Mobilize
|
|
69
76
|
t.update_status(%{Starting at #{Time.now.utc}})
|
70
77
|
stdout, stderr = [nil,nil]
|
71
78
|
begin
|
72
|
-
stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
|
79
|
+
stdout,log = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
|
80
|
+
#write to log if method returns an array w 2 members
|
81
|
+
t.log_dataset.write_cache(log) if log
|
73
82
|
rescue ScriptError, StandardError => exc
|
74
83
|
stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
|
75
|
-
#record the failure in Job so it appears on Runner
|
76
|
-
|
77
|
-
|
84
|
+
#record the failure in Job so it appears on Runner, turn it off
|
85
|
+
#so it doesn't run again
|
86
|
+
j.update_attributes(:active=>false)
|
87
|
+
t.update_attributes(:failed_at=>Time.now.utc)
|
88
|
+
t.update_status("Failed at #{Time.now.utc.to_s}")
|
78
89
|
raise exc
|
79
90
|
end
|
80
91
|
if stdout == false
|
@@ -86,12 +97,13 @@ module Mobilize
|
|
86
97
|
t.stdout_dataset.write_cache(stdout)
|
87
98
|
t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
|
88
99
|
if t.idx == j.tasks.length
|
89
|
-
|
90
|
-
j.update_attributes(:active=>false) if j.trigger.strip == "once"
|
91
|
-
t.update_attributes(:
|
100
|
+
#job has completed
|
101
|
+
j.update_attributes(:active=>false) if j.trigger.strip.downcase == "once"
|
102
|
+
t.update_attributes(:completed_at=>Time.now.utc)
|
103
|
+
t.update_status("Completed at #{Time.now.utc.to_s}")
|
92
104
|
#check for any dependent jobs, if there are, enqueue them
|
93
105
|
r = j.runner
|
94
|
-
dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
|
106
|
+
dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger.strip.downcase == "after #{j.name}"}
|
95
107
|
#put begin/rescue so all dependencies run
|
96
108
|
dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
|
97
109
|
else
|
@@ -103,6 +115,7 @@ module Mobilize
|
|
103
115
|
|
104
116
|
def enqueue!
|
105
117
|
t = self
|
118
|
+
t.update_attributes(:started_at=>Time.now.utc)
|
106
119
|
::Resque::Job.create("mobilize",Task,t.path,{})
|
107
120
|
return true
|
108
121
|
end
|
@@ -124,7 +137,7 @@ module Mobilize
|
|
124
137
|
|
125
138
|
def update_status(msg)
|
126
139
|
t = self
|
127
|
-
t.update_attributes(:status=>msg)
|
140
|
+
t.update_attributes(:status=>msg,:status_at=>Time.now.utc)
|
128
141
|
Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
|
129
142
|
return true
|
130
143
|
end
|
data/lib/mobilize-base/rakes.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
|
-
|
2
|
-
# will give you the resque tasks
|
3
|
-
|
4
|
-
namespace :mobilize do
|
5
|
-
require 'mobilize-base'
|
1
|
+
namespace :mobilize_base do
|
6
2
|
desc "Start a Resque worker"
|
7
3
|
task :work do
|
4
|
+
require 'mobilize-base'
|
8
5
|
begin
|
9
6
|
#require specified mobilize gems
|
10
7
|
Mobilize::Base.config('jobtracker')['extensions'].each do |e|
|
@@ -23,17 +20,41 @@ namespace :mobilize do
|
|
23
20
|
|
24
21
|
worker.work(ENV['INTERVAL'] || 5) # interval, will block
|
25
22
|
end
|
23
|
+
desc "Kill all Resque workers"
|
24
|
+
task :kill_workers do
|
25
|
+
require 'mobilize-base'
|
26
|
+
Mobilize::Jobtracker.kill_workers
|
27
|
+
end
|
26
28
|
desc "Kill idle workers not in sync with repo"
|
27
|
-
task :
|
28
|
-
|
29
|
+
task :kill_idle_and_stale_workers do
|
30
|
+
require 'mobilize-base'
|
31
|
+
Mobilize::Jobtracker.kill_idle_and_stale_workers
|
29
32
|
end
|
30
|
-
desc "Make sure
|
33
|
+
desc "Make sure there are the correct # of workers, kill if too many"
|
31
34
|
task :prep_workers do
|
35
|
+
require 'mobilize-base'
|
32
36
|
Mobilize::Jobtracker.prep_workers
|
33
37
|
end
|
34
|
-
|
35
|
-
|
36
|
-
|
38
|
+
desc "kill all old resque web processes, start new one with resque_web.rb extension file"
|
39
|
+
task :resque_web do
|
40
|
+
require 'mobilize-base'
|
41
|
+
port = Mobilize::Base.config('resque')['web_port']
|
42
|
+
config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
|
43
|
+
full_config_dir = "#{ENV['PWD']}/#{config_dir}"
|
44
|
+
resque_web_extension_path = "#{full_config_dir}resque_web.rb"
|
45
|
+
#kill any resque-web for now
|
46
|
+
`ps aux | grep resque-web | awk '{print $2}' | xargs kill`
|
47
|
+
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path}"
|
48
|
+
`#{command}`
|
49
|
+
end
|
50
|
+
desc "create indexes for all base models in mongodb"
|
51
|
+
task :create_indexes do
|
52
|
+
require 'mobilize-base'
|
53
|
+
["Dataset","Job","Runner","Task","User"].each do |m|
|
54
|
+
"Mobilize::#{m}".constantize.create_indexes
|
55
|
+
end
|
56
|
+
end
|
57
|
+
desc "Set up config and log folders and files, populate from samples"
|
37
58
|
task :setup do
|
38
59
|
config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
|
39
60
|
log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
|
@@ -56,11 +77,4 @@ namespace :mobilize_base do
|
|
56
77
|
end
|
57
78
|
end
|
58
79
|
end
|
59
|
-
desc "create indexes for all base modelsin mongodb"
|
60
|
-
task :create_indexes do
|
61
|
-
require 'mobilize-base'
|
62
|
-
["Dataset","Job","Runner","Task","User"].each do |m|
|
63
|
-
"Mobilize::#{m}".constantize.create_indexes
|
64
|
-
end
|
65
|
-
end
|
66
80
|
end
|
data/mobilize-base.gemspec
CHANGED
@@ -29,7 +29,6 @@ Gem::Specification.new do |s|
|
|
29
29
|
s.add_runtime_dependency 'redis',"~>3.0.0"
|
30
30
|
s.add_runtime_dependency 'resque','1.21.0'
|
31
31
|
s.add_runtime_dependency 'google_drive','0.3.2'
|
32
|
-
s.add_runtime_dependency 'bluepill','0.0.60'
|
33
32
|
s.add_runtime_dependency 'popen4','0.1.2'
|
34
33
|
s.add_runtime_dependency 'actionmailer','3.1.1'
|
35
34
|
end
|
data/test/mobilize-base_test.rb
CHANGED
@@ -44,15 +44,24 @@ describe "Mobilize" do
|
|
44
44
|
puts "add row to jobs sheet, wait 120s"
|
45
45
|
test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
|
46
46
|
jobs_sheet.add_or_update_rows(test_job_rows)
|
47
|
-
|
48
|
-
puts "job row added, force enqueued runner"
|
49
|
-
r.enqueue!
|
50
47
|
sleep 120
|
51
48
|
|
52
49
|
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
53
|
-
|
50
|
+
test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
51
|
+
test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
52
|
+
|
53
|
+
|
54
|
+
assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
|
54
55
|
|
55
|
-
|
56
|
+
puts "delete both output sheets, set first job to active=true"
|
57
|
+
test_target_sheet_1.delete
|
58
|
+
|
59
|
+
jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
|
60
|
+
sleep 90
|
61
|
+
|
62
|
+
test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
63
|
+
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
64
|
+
assert test_target_sheet_2.to_tsv == test_source_sheet.to_tsv
|
56
65
|
|
57
66
|
end
|
58
67
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-12-
|
12
|
+
date: 2012-12-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -139,22 +139,6 @@ dependencies:
|
|
139
139
|
- - '='
|
140
140
|
- !ruby/object:Gem::Version
|
141
141
|
version: 0.3.2
|
142
|
-
- !ruby/object:Gem::Dependency
|
143
|
-
name: bluepill
|
144
|
-
requirement: !ruby/object:Gem::Requirement
|
145
|
-
none: false
|
146
|
-
requirements:
|
147
|
-
- - '='
|
148
|
-
- !ruby/object:Gem::Version
|
149
|
-
version: 0.0.60
|
150
|
-
type: :runtime
|
151
|
-
prerelease: false
|
152
|
-
version_requirements: !ruby/object:Gem::Requirement
|
153
|
-
none: false
|
154
|
-
requirements:
|
155
|
-
- - '='
|
156
|
-
- !ruby/object:Gem::Version
|
157
|
-
version: 0.0.60
|
158
142
|
- !ruby/object:Gem::Dependency
|
159
143
|
name: popen4
|
160
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -226,7 +210,6 @@ files:
|
|
226
210
|
- lib/mobilize-base/models/task.rb
|
227
211
|
- lib/mobilize-base/models/user.rb
|
228
212
|
- lib/mobilize-base/rakes.rb
|
229
|
-
- lib/mobilize-base/tasks/mobilize-base.rake
|
230
213
|
- lib/mobilize-base/version.rb
|
231
214
|
- lib/samples/gdrive.yml
|
232
215
|
- lib/samples/gridfs.yml
|
@@ -234,6 +217,7 @@ files:
|
|
234
217
|
- lib/samples/jobtracker.yml
|
235
218
|
- lib/samples/mongoid.yml
|
236
219
|
- lib/samples/resque.yml
|
220
|
+
- lib/samples/resque_web.rb
|
237
221
|
- mobilize-base.gemspec
|
238
222
|
- test/base1_task1.yml
|
239
223
|
- test/base_job_rows.yml
|
@@ -254,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
254
238
|
version: '0'
|
255
239
|
segments:
|
256
240
|
- 0
|
257
|
-
hash:
|
241
|
+
hash: -3010357704785525623
|
258
242
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
259
243
|
none: false
|
260
244
|
requirements:
|
@@ -263,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
263
247
|
version: '0'
|
264
248
|
segments:
|
265
249
|
- 0
|
266
|
-
hash:
|
250
|
+
hash: -3010357704785525623
|
267
251
|
requirements: []
|
268
252
|
rubyforge_project: mobilize-base
|
269
253
|
rubygems_version: 1.8.24
|