mobilize-base 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/README.md +30 -14
- data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb +22 -11
- data/lib/mobilize-base/handlers/resque.rb +5 -5
- data/lib/mobilize-base/jobtracker.rb +22 -8
- data/lib/mobilize-base/models/job.rb +27 -3
- data/lib/mobilize-base/models/runner.rb +18 -10
- data/lib/mobilize-base/models/task.rb +25 -12
- data/lib/mobilize-base/rakes.rb +32 -18
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/samples/resque_web.rb +4 -0
- data/mobilize-base.gemspec +0 -1
- data/test/mobilize-base_test.rb +14 -5
- metadata +5 -21
- data/lib/mobilize-base/tasks/mobilize-base.rake +0 -2
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -27,10 +27,11 @@ Table Of Contents
|
|
27
27
|
* [Google Drive](#section_Configure_Google_Drive)
|
28
28
|
* [Jobtracker](#section_Configure_Jobtracker)
|
29
29
|
* [Resque](#section_Configure_Resque)
|
30
|
+
* [Resque-Web](#section_Configure_Resque-Web)
|
30
31
|
* [Gridfs](#section_Configure_Gridfs)
|
31
32
|
* [Mongoid](#section_Configure_Mongoid)
|
32
33
|
* [Start](#section_Start)
|
33
|
-
* [Start
|
34
|
+
* [Start Resque-Web](#section_Start_Start_Resque-Web)
|
34
35
|
* [Set Environment](#section_Start_Set_Environment)
|
35
36
|
* [Create User](#section_Start_Create_User)
|
36
37
|
* [Start Workers](#section_Start_Start_Workers)
|
@@ -262,22 +263,42 @@ It needs the below parameters, which can be found in the [lib/samples][git_sampl
|
|
262
263
|
production, but feel free to adjust depending on your hardware.
|
263
264
|
* redis_port - you should probably leave this alone, it specifies the
|
264
265
|
default port for dev and prod and a separate one for testing.
|
266
|
+
* web_port - this specifies the port under which resque-web operates
|
265
267
|
|
266
268
|
``` yml
|
267
269
|
development:
|
268
270
|
queue_name: 'mobilize'
|
269
271
|
max_workers: 4
|
270
272
|
redis_port: 6379
|
273
|
+
web_port: 8282
|
271
274
|
test:
|
272
275
|
queue_name: 'mobilize'
|
273
276
|
max_workers: 4
|
274
277
|
redis_port: 9736
|
278
|
+
web_port: 8282
|
275
279
|
production:
|
276
280
|
queue_name: 'mobilize'
|
277
281
|
max_workers: 36
|
278
282
|
redis_port: 6379
|
283
|
+
web_port: 8282
|
279
284
|
```
|
280
285
|
|
286
|
+
<a name='section_Configure_Resque-Web'></a>
|
287
|
+
### Configure Resque-Web
|
288
|
+
|
289
|
+
Please change your default username and password in the resque_web.rb
|
290
|
+
file in your config folder, reproduced below:
|
291
|
+
|
292
|
+
``` ruby
|
293
|
+
#comment out the below if you want no authentication on your web portal (not recommended)
|
294
|
+
Resque::Server.use(Rack::Auth::Basic) do |user, password|
|
295
|
+
[user, password] == ['admin', 'changeyourpassword']
|
296
|
+
end
|
297
|
+
```
|
298
|
+
|
299
|
+
This file is passed as a config file argument to
|
300
|
+
mobilize_base:resque_web task, as detailed in [Start Resque-Web](#section_Start_Start_Resque-Web).
|
301
|
+
|
281
302
|
<a name='section_Configure_Gridfs'></a>
|
282
303
|
### Configure Gridfs
|
283
304
|
|
@@ -350,22 +371,17 @@ A Mobilize instance can be considered "started" or "running" when you have:
|
|
350
371
|
<a name='section_Start_Start_resque-web'></a>
|
351
372
|
### Start resque-web
|
352
373
|
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
``` ruby
|
357
|
-
gem install resque
|
358
|
-
```
|
374
|
+
Mobilize ships with its own rake task to start resque web -- you can do
|
375
|
+
the following:
|
359
376
|
|
360
|
-
then, you can do
|
361
377
|
|
362
|
-
$
|
378
|
+
$ MOBILIZE_ENV=<environment> rake mobilize_base:resque_web
|
363
379
|
|
364
|
-
|
380
|
+
This will start a resque_web instance with the port specified in your
|
381
|
+
resque.yml and the config/auth scheme specified in your resque_web.rb.
|
365
382
|
|
366
|
-
|
367
|
-
|
368
|
-
[Resque Standalone section][resque-web].
|
383
|
+
More detail on the
|
384
|
+
[Resque-Web Standalone section][resque-web].
|
369
385
|
|
370
386
|
<a name='section_Start_Set_Environment'></a>
|
371
387
|
### Set Environment
|
@@ -374,7 +390,7 @@ Mobilize takes the environment from your Rails.env if you're running
|
|
374
390
|
Rails, or assumes "development." You can specify "development", "test",
|
375
391
|
or "production," as per the yml files.
|
376
392
|
|
377
|
-
Otherwise, it takes it from MOBILIZE_ENV parameter,
|
393
|
+
Otherwise, it takes it from MOBILIZE_ENV parameter, as in:
|
378
394
|
|
379
395
|
``` ruby
|
380
396
|
> ENV['MOBILIZE_ENV'] = 'production'
|
@@ -1,6 +1,7 @@
|
|
1
1
|
module GoogleDrive
|
2
2
|
class ClientLoginFetcher
|
3
3
|
def request_raw(method, url, data, extra_header, auth)
|
4
|
+
clf = self
|
4
5
|
#this is patched to handle server errors due to http chaos
|
5
6
|
uri = URI.parse(url)
|
6
7
|
response = nil
|
@@ -14,22 +15,32 @@ module GoogleDrive
|
|
14
15
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
15
16
|
#set 600 to allow for large downloads
|
16
17
|
http.read_timeout = 600
|
17
|
-
response =
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
18
|
+
response = begin
|
19
|
+
clf.http_call(http, method, uri, data, extra_header, auth)
|
20
|
+
rescue
|
21
|
+
#timeouts etc.
|
22
|
+
nil
|
23
|
+
end
|
24
|
+
if response.nil?
|
25
|
+
attempts +=1
|
26
|
+
else
|
27
|
+
if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
|
28
|
+
if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
|
29
|
+
if sleep_time
|
30
|
+
sleep_time = sleep_time * attempts
|
31
|
+
else
|
32
|
+
sleep_time = (rand*100).to_i
|
33
|
+
end
|
22
34
|
else
|
23
|
-
sleep_time =
|
35
|
+
sleep_time = 10
|
24
36
|
end
|
25
|
-
|
26
|
-
sleep_time
|
37
|
+
attempts += 1
|
38
|
+
puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
|
39
|
+
sleep sleep_time
|
27
40
|
end
|
28
|
-
attempts += 1
|
29
|
-
puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
|
30
|
-
sleep sleep_time
|
31
41
|
end
|
32
42
|
end
|
43
|
+
raise "No response after 5 attempts" if response.nil?
|
33
44
|
raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
|
34
45
|
return response
|
35
46
|
end
|
@@ -17,7 +17,6 @@ module Mobilize
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def Resque.workers(state="all")
|
20
|
-
raise "invalid state #{state}" unless ['all','idle','working','timeout'].include?(state)
|
21
20
|
workers = ::Resque.workers.select{|w| w.queues.first == Resque.queue_name}
|
22
21
|
return workers if state == 'all'
|
23
22
|
working_workers = workers.select{|w| w.job['queue']== Resque.queue_name}
|
@@ -28,6 +27,7 @@ module Mobilize
|
|
28
27
|
return stale_workers if state == 'stale'
|
29
28
|
timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
|
30
29
|
return timeout_workers if state == 'timeout'
|
30
|
+
raise "invalid state #{state}"
|
31
31
|
end
|
32
32
|
|
33
33
|
def Resque.failures
|
@@ -36,7 +36,6 @@ module Mobilize
|
|
36
36
|
|
37
37
|
#active state refers to jobs that are either queued or working
|
38
38
|
def Resque.jobs(state="active")
|
39
|
-
raise "invalid state #{state}" unless ['all','queued','working','active','timeout','failed'].include?(state)
|
40
39
|
working_jobs = Resque.workers('working').map{|w| w.job['payload']}
|
41
40
|
return working_jobs if state == 'working'
|
42
41
|
queued_jobs = ::Resque.peek(Resque.queue_name,0,0).to_a
|
@@ -47,6 +46,7 @@ module Mobilize
|
|
47
46
|
timeout_jobs = Resque.workers("timeout").map{|w| w.job['payload']}
|
48
47
|
return timeout_jobs if state == 'timeout'
|
49
48
|
return working_jobs + queued_jobs + failed_jobs if state == 'all'
|
49
|
+
raise "invalid state #{state}"
|
50
50
|
end
|
51
51
|
|
52
52
|
def Resque.active_paths
|
@@ -125,13 +125,13 @@ module Mobilize
|
|
125
125
|
dir_envs = "MOBILIZE_ENV=#{Base.env} " +
|
126
126
|
"MOBILIZE_CONFIG_DIR=#{Base.config_dir} " +
|
127
127
|
"MOBILIZE_LOG_DIR=#{Base.log_dir}"
|
128
|
-
"(cd #{Base.root};rake #{dir_envs}
|
128
|
+
"(cd #{Base.root};rake #{dir_envs} mobilize_base:work) >> #{Resque.log_path} 2>&1 &".bash
|
129
129
|
end
|
130
130
|
end
|
131
131
|
|
132
132
|
def Resque.kill_idle_workers(count=nil)
|
133
133
|
idle_pids = Resque.workers('idle').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
|
134
|
-
if count>idle_pids.length or count == 0
|
134
|
+
if count.to_i > idle_pids.length or count == 0
|
135
135
|
return false
|
136
136
|
elsif count
|
137
137
|
"kill #{idle_pids[0..count-1].join(" ")}".bash
|
@@ -141,7 +141,7 @@ module Mobilize
|
|
141
141
|
return true
|
142
142
|
end
|
143
143
|
|
144
|
-
def Resque.
|
144
|
+
def Resque.kill_idle_and_stale_workers
|
145
145
|
idle_pids = Resque.workers('idle').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
|
146
146
|
stale_pids = Resque.workers('stale').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
|
147
147
|
idle_stale_pids = (idle_pids & stale_pids)
|
@@ -75,6 +75,10 @@ module Mobilize
|
|
75
75
|
Resque.kill_idle_workers
|
76
76
|
end
|
77
77
|
|
78
|
+
def Jobtracker.kill_idle_and_stale_workers
|
79
|
+
Resque.kill_idle_and_stale_workers
|
80
|
+
end
|
81
|
+
|
78
82
|
def Jobtracker.prep_workers
|
79
83
|
Resque.prep_workers
|
80
84
|
end
|
@@ -202,15 +206,25 @@ module Mobilize
|
|
202
206
|
|
203
207
|
def Jobtracker.deployed_at
|
204
208
|
#assumes deploy is as of last commit, or as of last deploy time
|
205
|
-
#as given by the
|
209
|
+
#as given by the REVISION file in the root folder
|
206
210
|
deploy_time = begin
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
211
|
+
%{git log -1 --format="%cd"}.bash
|
212
|
+
rescue
|
213
|
+
revision_path = "#{ENV['PWD']}/REVISION"
|
214
|
+
"touch #{revision_path}".bash unless File.exists?(revision_path)
|
215
|
+
revision_string = "ls -l #{revision_path}".bash
|
216
|
+
revision_rows = revision_string.split("\n").map{|lss| lss.strip.split(" ")}
|
217
|
+
mod_time = revision_rows.map do |lsr|
|
218
|
+
if lsr.length == 8
|
219
|
+
#ubuntu
|
220
|
+
lsr[5..6].join(" ")
|
221
|
+
elsif lsr.length == 9
|
222
|
+
#osx
|
223
|
+
lsr[5..7].join(" ")
|
224
|
+
end
|
225
|
+
end.first
|
226
|
+
mod_time
|
227
|
+
end.to_s.strip
|
214
228
|
Time.parse(deploy_time)
|
215
229
|
end
|
216
230
|
|
@@ -5,8 +5,6 @@ module Mobilize
|
|
5
5
|
field :path, type: String
|
6
6
|
field :active, type: Boolean
|
7
7
|
field :trigger, type: String
|
8
|
-
field :status, type: String
|
9
|
-
field :last_completed_at, type: Time
|
10
8
|
|
11
9
|
index({ path: 1})
|
12
10
|
|
@@ -26,6 +24,32 @@ module Mobilize
|
|
26
24
|
return j
|
27
25
|
end
|
28
26
|
|
27
|
+
def status
|
28
|
+
#last task status
|
29
|
+
j = self
|
30
|
+
j.active_task.status
|
31
|
+
end
|
32
|
+
|
33
|
+
def active_task
|
34
|
+
j = self
|
35
|
+
#latest started at or first
|
36
|
+
j.tasks.select{|t| t.started_at}.sort_by{|t| t.started_at}.last || j.tasks.first
|
37
|
+
end
|
38
|
+
|
39
|
+
def completed_at
|
40
|
+
j = self
|
41
|
+
j.tasks.last.completed_at
|
42
|
+
end
|
43
|
+
|
44
|
+
def failed_at
|
45
|
+
j = self
|
46
|
+
j.tasks.last.failed_at
|
47
|
+
end
|
48
|
+
|
49
|
+
def status_at
|
50
|
+
j.active_task.status_at
|
51
|
+
end
|
52
|
+
|
29
53
|
#convenience methods
|
30
54
|
def runner
|
31
55
|
j = self
|
@@ -41,7 +65,7 @@ module Mobilize
|
|
41
65
|
def is_due?
|
42
66
|
j = self
|
43
67
|
return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
|
44
|
-
last_run = j.
|
68
|
+
last_run = j.completed_at
|
45
69
|
#check trigger
|
46
70
|
trigger = j.trigger
|
47
71
|
return true if trigger == 'once'
|
@@ -5,7 +5,9 @@ module Mobilize
|
|
5
5
|
field :path, type: String
|
6
6
|
field :active, type: Boolean
|
7
7
|
field :status, type: String
|
8
|
-
field :
|
8
|
+
field :started_at, type: Time
|
9
|
+
field :status_at, type: Time
|
10
|
+
field :completed_at, type: Time
|
9
11
|
|
10
12
|
index({ path: 1})
|
11
13
|
|
@@ -13,9 +15,9 @@ module Mobilize
|
|
13
15
|
%w{name active trigger status task1 task2 task3 task4 task5}
|
14
16
|
end
|
15
17
|
|
16
|
-
def
|
18
|
+
def cached_at
|
17
19
|
r = self
|
18
|
-
Dataset.find_or_create_by_path(r.path).
|
20
|
+
Dataset.find_or_create_by_path(r.path).cached_at
|
19
21
|
end
|
20
22
|
|
21
23
|
def worker
|
@@ -51,12 +53,12 @@ module Mobilize
|
|
51
53
|
j.tasks.first.enqueue!
|
52
54
|
end
|
53
55
|
rescue ScriptError, StandardError => exc
|
54
|
-
|
56
|
+
r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
|
55
57
|
j.update_attributes(:active=>false)
|
56
58
|
end
|
57
59
|
end
|
58
60
|
r.update_gsheet(gdrive_slot)
|
59
|
-
r.update_attributes(:
|
61
|
+
r.update_attributes(:completed_at=>Time.now.utc)
|
60
62
|
end
|
61
63
|
|
62
64
|
def dataset
|
@@ -77,7 +79,7 @@ module Mobilize
|
|
77
79
|
r = self
|
78
80
|
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
79
81
|
jobs_sheet.add_headers(r.headers)
|
80
|
-
jobs_sheet.delete_sheet1
|
82
|
+
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
81
83
|
return jobs_sheet
|
82
84
|
end
|
83
85
|
|
@@ -128,7 +130,12 @@ module Mobilize
|
|
128
130
|
def update_gsheet(gdrive_slot)
|
129
131
|
r = self
|
130
132
|
jobs_gsheet = r.gsheet(gdrive_slot)
|
131
|
-
|
133
|
+
upd_jobs = r.jobs.select do |j|
|
134
|
+
j.completed_at.nil? ||
|
135
|
+
j.completed_at > j.runner.completed_at ||
|
136
|
+
(j.failed_at and j.failed_at > j.runner.completed_at)
|
137
|
+
end
|
138
|
+
upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
|
132
139
|
jobs_gsheet.add_or_update_rows(upd_rows)
|
133
140
|
r.update_status("gsheet updated")
|
134
141
|
return true
|
@@ -152,7 +159,7 @@ module Mobilize
|
|
152
159
|
|
153
160
|
def update_status(msg)
|
154
161
|
r = self
|
155
|
-
r.update_attributes(:status=>msg)
|
162
|
+
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
156
163
|
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
157
164
|
return true
|
158
165
|
end
|
@@ -165,12 +172,13 @@ module Mobilize
|
|
165
172
|
def is_due?
|
166
173
|
r = self.reload
|
167
174
|
return false if r.is_working?
|
168
|
-
|
169
|
-
return true if r.
|
175
|
+
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
176
|
+
return true if r.started_at.nil? or r.started_at < prev_due_time
|
170
177
|
end
|
171
178
|
|
172
179
|
def enqueue!
|
173
180
|
r = self
|
181
|
+
r.update_attributes(:started_at=>Time.now.utc)
|
174
182
|
::Resque::Job.create("mobilize",Runner,r.path,{})
|
175
183
|
return true
|
176
184
|
end
|
@@ -7,8 +7,10 @@ module Mobilize
|
|
7
7
|
field :call, type: String
|
8
8
|
field :param_string, type: Array
|
9
9
|
field :status, type: String
|
10
|
-
field :
|
11
|
-
field :
|
10
|
+
field :completed_at, type: Time
|
11
|
+
field :started_at, type: Time
|
12
|
+
field :failed_at, type: Time
|
13
|
+
field :status_at, type: Time
|
12
14
|
|
13
15
|
index({ path: 1})
|
14
16
|
|
@@ -27,9 +29,14 @@ module Mobilize
|
|
27
29
|
Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
|
28
30
|
end
|
29
31
|
|
32
|
+
def log_dataset
|
33
|
+
t = self
|
34
|
+
Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/log")
|
35
|
+
end
|
36
|
+
|
30
37
|
def params
|
31
38
|
t = self
|
32
|
-
t.param_string.split(",").map do |p|
|
39
|
+
t.param_string.split(",").map do |p|
|
33
40
|
ps = p.strip
|
34
41
|
ps = ps[1..-1] if ['"',"'"].include?(ps[0])
|
35
42
|
ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
|
@@ -69,12 +76,16 @@ module Mobilize
|
|
69
76
|
t.update_status(%{Starting at #{Time.now.utc}})
|
70
77
|
stdout, stderr = [nil,nil]
|
71
78
|
begin
|
72
|
-
stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
|
79
|
+
stdout,log = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
|
80
|
+
#write to log if method returns an array w 2 members
|
81
|
+
t.log_dataset.write_cache(log) if log
|
73
82
|
rescue ScriptError, StandardError => exc
|
74
83
|
stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
|
75
|
-
#record the failure in Job so it appears on Runner
|
76
|
-
|
77
|
-
|
84
|
+
#record the failure in Job so it appears on Runner, turn it off
|
85
|
+
#so it doesn't run again
|
86
|
+
j.update_attributes(:active=>false)
|
87
|
+
t.update_attributes(:failed_at=>Time.now.utc)
|
88
|
+
t.update_status("Failed at #{Time.now.utc.to_s}")
|
78
89
|
raise exc
|
79
90
|
end
|
80
91
|
if stdout == false
|
@@ -86,12 +97,13 @@ module Mobilize
|
|
86
97
|
t.stdout_dataset.write_cache(stdout)
|
87
98
|
t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
|
88
99
|
if t.idx == j.tasks.length
|
89
|
-
|
90
|
-
j.update_attributes(:active=>false) if j.trigger.strip == "once"
|
91
|
-
t.update_attributes(:
|
100
|
+
#job has completed
|
101
|
+
j.update_attributes(:active=>false) if j.trigger.strip.downcase == "once"
|
102
|
+
t.update_attributes(:completed_at=>Time.now.utc)
|
103
|
+
t.update_status("Completed at #{Time.now.utc.to_s}")
|
92
104
|
#check for any dependent jobs, if there are, enqueue them
|
93
105
|
r = j.runner
|
94
|
-
dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
|
106
|
+
dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger.strip.downcase == "after #{j.name}"}
|
95
107
|
#put begin/rescue so all dependencies run
|
96
108
|
dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
|
97
109
|
else
|
@@ -103,6 +115,7 @@ module Mobilize
|
|
103
115
|
|
104
116
|
def enqueue!
|
105
117
|
t = self
|
118
|
+
t.update_attributes(:started_at=>Time.now.utc)
|
106
119
|
::Resque::Job.create("mobilize",Task,t.path,{})
|
107
120
|
return true
|
108
121
|
end
|
@@ -124,7 +137,7 @@ module Mobilize
|
|
124
137
|
|
125
138
|
def update_status(msg)
|
126
139
|
t = self
|
127
|
-
t.update_attributes(:status=>msg)
|
140
|
+
t.update_attributes(:status=>msg,:status_at=>Time.now.utc)
|
128
141
|
Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
|
129
142
|
return true
|
130
143
|
end
|
data/lib/mobilize-base/rakes.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
|
-
|
2
|
-
# will give you the resque tasks
|
3
|
-
|
4
|
-
namespace :mobilize do
|
5
|
-
require 'mobilize-base'
|
1
|
+
namespace :mobilize_base do
|
6
2
|
desc "Start a Resque worker"
|
7
3
|
task :work do
|
4
|
+
require 'mobilize-base'
|
8
5
|
begin
|
9
6
|
#require specified mobilize gems
|
10
7
|
Mobilize::Base.config('jobtracker')['extensions'].each do |e|
|
@@ -23,17 +20,41 @@ namespace :mobilize do
|
|
23
20
|
|
24
21
|
worker.work(ENV['INTERVAL'] || 5) # interval, will block
|
25
22
|
end
|
23
|
+
desc "Kill all Resque workers"
|
24
|
+
task :kill_workers do
|
25
|
+
require 'mobilize-base'
|
26
|
+
Mobilize::Jobtracker.kill_workers
|
27
|
+
end
|
26
28
|
desc "Kill idle workers not in sync with repo"
|
27
|
-
task :
|
28
|
-
|
29
|
+
task :kill_idle_and_stale_workers do
|
30
|
+
require 'mobilize-base'
|
31
|
+
Mobilize::Jobtracker.kill_idle_and_stale_workers
|
29
32
|
end
|
30
|
-
desc "Make sure
|
33
|
+
desc "Make sure there are the correct # of workers, kill if too many"
|
31
34
|
task :prep_workers do
|
35
|
+
require 'mobilize-base'
|
32
36
|
Mobilize::Jobtracker.prep_workers
|
33
37
|
end
|
34
|
-
|
35
|
-
|
36
|
-
|
38
|
+
desc "kill all old resque web processes, start new one with resque_web.rb extension file"
|
39
|
+
task :resque_web do
|
40
|
+
require 'mobilize-base'
|
41
|
+
port = Mobilize::Base.config('resque')['web_port']
|
42
|
+
config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
|
43
|
+
full_config_dir = "#{ENV['PWD']}/#{config_dir}"
|
44
|
+
resque_web_extension_path = "#{full_config_dir}resque_web.rb"
|
45
|
+
#kill any resque-web for now
|
46
|
+
`ps aux | grep resque-web | awk '{print $2}' | xargs kill`
|
47
|
+
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path}"
|
48
|
+
`#{command}`
|
49
|
+
end
|
50
|
+
desc "create indexes for all base models in mongodb"
|
51
|
+
task :create_indexes do
|
52
|
+
require 'mobilize-base'
|
53
|
+
["Dataset","Job","Runner","Task","User"].each do |m|
|
54
|
+
"Mobilize::#{m}".constantize.create_indexes
|
55
|
+
end
|
56
|
+
end
|
57
|
+
desc "Set up config and log folders and files, populate from samples"
|
37
58
|
task :setup do
|
38
59
|
config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
|
39
60
|
log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
|
@@ -56,11 +77,4 @@ namespace :mobilize_base do
|
|
56
77
|
end
|
57
78
|
end
|
58
79
|
end
|
59
|
-
desc "create indexes for all base modelsin mongodb"
|
60
|
-
task :create_indexes do
|
61
|
-
require 'mobilize-base'
|
62
|
-
["Dataset","Job","Runner","Task","User"].each do |m|
|
63
|
-
"Mobilize::#{m}".constantize.create_indexes
|
64
|
-
end
|
65
|
-
end
|
66
80
|
end
|
data/mobilize-base.gemspec
CHANGED
@@ -29,7 +29,6 @@ Gem::Specification.new do |s|
|
|
29
29
|
s.add_runtime_dependency 'redis',"~>3.0.0"
|
30
30
|
s.add_runtime_dependency 'resque','1.21.0'
|
31
31
|
s.add_runtime_dependency 'google_drive','0.3.2'
|
32
|
-
s.add_runtime_dependency 'bluepill','0.0.60'
|
33
32
|
s.add_runtime_dependency 'popen4','0.1.2'
|
34
33
|
s.add_runtime_dependency 'actionmailer','3.1.1'
|
35
34
|
end
|
data/test/mobilize-base_test.rb
CHANGED
@@ -44,15 +44,24 @@ describe "Mobilize" do
|
|
44
44
|
puts "add row to jobs sheet, wait 120s"
|
45
45
|
test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
|
46
46
|
jobs_sheet.add_or_update_rows(test_job_rows)
|
47
|
-
|
48
|
-
puts "job row added, force enqueued runner"
|
49
|
-
r.enqueue!
|
50
47
|
sleep 120
|
51
48
|
|
52
49
|
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
53
|
-
|
50
|
+
test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
51
|
+
test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
52
|
+
|
53
|
+
|
54
|
+
assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
|
54
55
|
|
55
|
-
|
56
|
+
puts "delete both output sheets, set first job to active=true"
|
57
|
+
test_target_sheet_1.delete
|
58
|
+
|
59
|
+
jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
|
60
|
+
sleep 90
|
61
|
+
|
62
|
+
test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
63
|
+
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
64
|
+
assert test_target_sheet_2.to_tsv == test_source_sheet.to_tsv
|
56
65
|
|
57
66
|
end
|
58
67
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-12-
|
12
|
+
date: 2012-12-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -139,22 +139,6 @@ dependencies:
|
|
139
139
|
- - '='
|
140
140
|
- !ruby/object:Gem::Version
|
141
141
|
version: 0.3.2
|
142
|
-
- !ruby/object:Gem::Dependency
|
143
|
-
name: bluepill
|
144
|
-
requirement: !ruby/object:Gem::Requirement
|
145
|
-
none: false
|
146
|
-
requirements:
|
147
|
-
- - '='
|
148
|
-
- !ruby/object:Gem::Version
|
149
|
-
version: 0.0.60
|
150
|
-
type: :runtime
|
151
|
-
prerelease: false
|
152
|
-
version_requirements: !ruby/object:Gem::Requirement
|
153
|
-
none: false
|
154
|
-
requirements:
|
155
|
-
- - '='
|
156
|
-
- !ruby/object:Gem::Version
|
157
|
-
version: 0.0.60
|
158
142
|
- !ruby/object:Gem::Dependency
|
159
143
|
name: popen4
|
160
144
|
requirement: !ruby/object:Gem::Requirement
|
@@ -226,7 +210,6 @@ files:
|
|
226
210
|
- lib/mobilize-base/models/task.rb
|
227
211
|
- lib/mobilize-base/models/user.rb
|
228
212
|
- lib/mobilize-base/rakes.rb
|
229
|
-
- lib/mobilize-base/tasks/mobilize-base.rake
|
230
213
|
- lib/mobilize-base/version.rb
|
231
214
|
- lib/samples/gdrive.yml
|
232
215
|
- lib/samples/gridfs.yml
|
@@ -234,6 +217,7 @@ files:
|
|
234
217
|
- lib/samples/jobtracker.yml
|
235
218
|
- lib/samples/mongoid.yml
|
236
219
|
- lib/samples/resque.yml
|
220
|
+
- lib/samples/resque_web.rb
|
237
221
|
- mobilize-base.gemspec
|
238
222
|
- test/base1_task1.yml
|
239
223
|
- test/base_job_rows.yml
|
@@ -254,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
254
238
|
version: '0'
|
255
239
|
segments:
|
256
240
|
- 0
|
257
|
-
hash:
|
241
|
+
hash: -3010357704785525623
|
258
242
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
259
243
|
none: false
|
260
244
|
requirements:
|
@@ -263,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
263
247
|
version: '0'
|
264
248
|
segments:
|
265
249
|
- 0
|
266
|
-
hash:
|
250
|
+
hash: -3010357704785525623
|
267
251
|
requirements: []
|
268
252
|
rubyforge_project: mobilize-base
|
269
253
|
rubygems_version: 1.8.24
|