mobilize-base 1.3 → 1.21
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +11 -10
- data/lib/mobilize-base/extensions/google_drive/file.rb +7 -7
- data/lib/mobilize-base/extensions/google_drive/worksheet.rb +2 -7
- data/lib/mobilize-base/extensions/string.rb +4 -12
- data/lib/mobilize-base/extensions/yaml.rb +7 -10
- data/lib/mobilize-base/handlers/gbook.rb +31 -24
- data/lib/mobilize-base/handlers/gfile.rb +3 -5
- data/lib/mobilize-base/handlers/gridfs.rb +24 -19
- data/lib/mobilize-base/handlers/gsheet.rb +20 -25
- data/lib/mobilize-base/handlers/resque.rb +4 -16
- data/lib/mobilize-base/jobtracker.rb +5 -13
- data/lib/mobilize-base/models/job.rb +48 -36
- data/lib/mobilize-base/models/runner.rb +123 -24
- data/lib/mobilize-base/models/stage.rb +43 -14
- data/lib/mobilize-base/tasks.rb +3 -16
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +1 -5
- data/lib/samples/gridfs.yml +3 -0
- data/lib/samples/gsheet.yml +4 -4
- data/mobilize-base.gemspec +5 -4
- data/test/mobilize-base_test.rb +0 -1
- metadata +31 -20
- data/lib/mobilize-base/extensions/resque-server/views/queues.erb +0 -59
- data/lib/mobilize-base/extensions/resque-server/views/working.erb +0 -85
- data/lib/mobilize-base/helpers/job_helper.rb +0 -54
- data/lib/mobilize-base/helpers/runner_helper.rb +0 -83
- data/lib/mobilize-base/helpers/stage_helper.rb +0 -38
@@ -1,6 +1,5 @@
|
|
1
1
|
module Mobilize
|
2
2
|
class Runner
|
3
|
-
include Mobilize::RunnerHelper
|
4
3
|
include Mongoid::Document
|
5
4
|
include Mongoid::Timestamps
|
6
5
|
field :path, type: String
|
@@ -12,6 +11,20 @@ module Mobilize
|
|
12
11
|
|
13
12
|
index({ path: 1})
|
14
13
|
|
14
|
+
def headers
|
15
|
+
%w{name active trigger status stage1 stage2 stage3 stage4 stage5}
|
16
|
+
end
|
17
|
+
|
18
|
+
def title
|
19
|
+
r = self
|
20
|
+
r.path.split("/").first
|
21
|
+
end
|
22
|
+
|
23
|
+
def worker
|
24
|
+
r = self
|
25
|
+
Mobilize::Resque.find_worker_by_path(r.path)
|
26
|
+
end
|
27
|
+
|
15
28
|
def Runner.find_by_path(path)
|
16
29
|
Runner.where(:path=>path).first
|
17
30
|
end
|
@@ -19,7 +32,6 @@ module Mobilize
|
|
19
32
|
def Runner.find_by_title(title)
|
20
33
|
Runner.where(:path=>"#{title}/jobs").first
|
21
34
|
end
|
22
|
-
|
23
35
|
def Runner.perform(id,*args)
|
24
36
|
r = Runner.find_by_path(id)
|
25
37
|
#get gdrive slot for read
|
@@ -29,18 +41,12 @@ module Mobilize
|
|
29
41
|
return false
|
30
42
|
end
|
31
43
|
r.update_attributes(:started_at=>Time.now.utc)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
#queue up the jobs that are due and active
|
39
|
-
rescue => exc
|
40
|
-
#log the exception, but continue w job processing
|
41
|
-
#This ensures jobs are still processed if google drive goes down
|
42
|
-
r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
|
43
|
-
end
|
44
|
+
#make sure any updates to activity are processed first
|
45
|
+
#as in when someone runs a "once" job that has completed
|
46
|
+
r.update_gsheet(gdrive_slot)
|
47
|
+
#read the jobs in the gsheet and update models with news
|
48
|
+
r.read_gsheet(gdrive_slot)
|
49
|
+
#queue up the jobs that are due and active
|
44
50
|
r.jobs.each do |j|
|
45
51
|
begin
|
46
52
|
if j.is_due?
|
@@ -58,32 +64,90 @@ module Mobilize
|
|
58
64
|
r.update_attributes(:completed_at=>Time.now.utc)
|
59
65
|
end
|
60
66
|
|
67
|
+
def dataset
|
68
|
+
r = self
|
69
|
+
Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
|
70
|
+
end
|
71
|
+
|
61
72
|
def Runner.find_or_create_by_path(path)
|
62
73
|
Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
|
63
74
|
end
|
64
75
|
|
76
|
+
def gbook(gdrive_slot)
|
77
|
+
r = self
|
78
|
+
title = r.path.split("/").first
|
79
|
+
Gbook.find_all_by_path(title,gdrive_slot).first
|
80
|
+
end
|
81
|
+
|
82
|
+
def gsheet(gdrive_slot)
|
83
|
+
r = self
|
84
|
+
u = r.user
|
85
|
+
jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
|
86
|
+
#make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
|
87
|
+
unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
|
88
|
+
#only give the user edit permissions if they're the ones
|
89
|
+
#creating it
|
90
|
+
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
91
|
+
unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
|
92
|
+
jobs_sheet.spreadsheet.update_acl(u.email,"writer")
|
93
|
+
end
|
94
|
+
end
|
95
|
+
jobs_sheet.add_headers(r.headers)
|
96
|
+
#add url to dataset
|
97
|
+
Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
|
98
|
+
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
99
|
+
return jobs_sheet
|
100
|
+
end
|
101
|
+
|
65
102
|
def read_gsheet(gdrive_slot)
|
66
103
|
r = self
|
67
|
-
|
68
|
-
gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
|
104
|
+
gsheet_tsv = r.gsheet(gdrive_slot).read(Gdrive.owner_name)
|
69
105
|
#turn it into a hash array
|
70
|
-
|
106
|
+
gsheet_jobs = gsheet_tsv.tsv_to_hash_array
|
71
107
|
#go through each job, update relevant job with its params
|
72
108
|
done_jobs = []
|
73
109
|
#parse out the jobs and update the Job collection
|
74
|
-
|
110
|
+
gsheet_jobs.each_with_index do |rj,rj_i|
|
75
111
|
#skip non-jobs or jobs without required values
|
76
|
-
next if (
|
77
|
-
j = Job.
|
112
|
+
next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
|
113
|
+
j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
|
114
|
+
#update top line params
|
115
|
+
j.update_attributes(:active => rj['active'],
|
116
|
+
:trigger => rj['trigger'])
|
117
|
+
(1..5).to_a.each do |s_idx|
|
118
|
+
stage_string = rj["stage#{s_idx.to_s}"]
|
119
|
+
s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
|
120
|
+
if stage_string.to_s.length==0
|
121
|
+
#delete this stage and all stages after
|
122
|
+
if s
|
123
|
+
j = s.job
|
124
|
+
j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
|
125
|
+
#just in case
|
126
|
+
s.delete
|
127
|
+
end
|
128
|
+
break
|
129
|
+
elsif s.nil?
|
130
|
+
#create this stage
|
131
|
+
s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
|
132
|
+
end
|
133
|
+
#parse command string, update stage with it
|
134
|
+
s_handler, call, param_string = [""*3]
|
135
|
+
stage_string.split(" ").ie do |spls|
|
136
|
+
s_handler = spls.first.split(".").first
|
137
|
+
call = spls.first.split(".").last
|
138
|
+
param_string = spls[1..-1].join(" ").strip
|
139
|
+
end
|
140
|
+
s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
|
141
|
+
end
|
78
142
|
r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
|
79
143
|
#add this job to list of read ones
|
80
144
|
done_jobs << j
|
81
145
|
end
|
82
146
|
#delete user jobs that are not included in Runner
|
83
|
-
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |
|
84
|
-
j = Job.where(:path=>
|
147
|
+
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
|
148
|
+
j = Job.where(:path=>rj_path).first
|
85
149
|
j.delete if j
|
86
|
-
r.update_status("Deleted job:#{
|
150
|
+
r.update_status("Deleted job:#{rj_path}")
|
87
151
|
end
|
88
152
|
r.update_status("jobs read at #{Time.now.utc}")
|
89
153
|
return true
|
@@ -94,13 +158,48 @@ module Mobilize
|
|
94
158
|
#there's nothing to update if runner has never had a completed at
|
95
159
|
return false unless r.completed_at
|
96
160
|
jobs_gsheet = r.gsheet(gdrive_slot)
|
97
|
-
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at
|
161
|
+
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
|
98
162
|
upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
|
99
163
|
jobs_gsheet.add_or_update_rows(upd_rows)
|
100
164
|
r.update_status("gsheet updated")
|
101
165
|
return true
|
102
166
|
end
|
103
167
|
|
168
|
+
def jobs(jname=nil)
|
169
|
+
r = self
|
170
|
+
js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
|
171
|
+
if jname
|
172
|
+
return js.sel{|j| j.name == jname}.first
|
173
|
+
else
|
174
|
+
return js
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def user
|
179
|
+
r = self
|
180
|
+
user_name = r.path.split("_").second.split("(").first.split("/").first
|
181
|
+
User.where(:name=>user_name).first
|
182
|
+
end
|
183
|
+
|
184
|
+
def update_status(msg)
|
185
|
+
r = self
|
186
|
+
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
187
|
+
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
188
|
+
return true
|
189
|
+
end
|
190
|
+
|
191
|
+
def is_working?
|
192
|
+
r = self
|
193
|
+
Mobilize::Resque.active_paths.include?(r.path)
|
194
|
+
end
|
195
|
+
|
196
|
+
def is_due?
|
197
|
+
r = self.reload
|
198
|
+
return false if r.is_working?
|
199
|
+
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
200
|
+
return true if r.started_at.nil? or r.started_at < prev_due_time
|
201
|
+
end
|
202
|
+
|
104
203
|
def enqueue!
|
105
204
|
r = self
|
106
205
|
::Resque::Job.create("mobilize",Runner,r.path,{})
|
@@ -2,7 +2,6 @@ module Mobilize
|
|
2
2
|
class Stage
|
3
3
|
include Mongoid::Document
|
4
4
|
include Mongoid::Timestamps
|
5
|
-
include Mobilize::StageHelper
|
6
5
|
field :path, type: String
|
7
6
|
field :handler, type: String
|
8
7
|
field :call, type: String
|
@@ -17,6 +16,40 @@ module Mobilize
|
|
17
16
|
|
18
17
|
index({ path: 1})
|
19
18
|
|
19
|
+
def idx
|
20
|
+
s = self
|
21
|
+
s.path.split("/").last.gsub("stage","").to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
def out_dst
|
25
|
+
#this gives a dataset that points to the output
|
26
|
+
#allowing you to determine its size
|
27
|
+
#before committing to a read or write
|
28
|
+
s = self
|
29
|
+
Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
|
30
|
+
end
|
31
|
+
|
32
|
+
def err_dst
|
33
|
+
#this gives a dataset that points to the output
|
34
|
+
#allowing you to determine its size
|
35
|
+
#before committing to a read or write
|
36
|
+
s = self
|
37
|
+
Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
|
38
|
+
end
|
39
|
+
|
40
|
+
def params
|
41
|
+
s = self
|
42
|
+
p = YAML.easy_load(s.param_string)
|
43
|
+
raise "Must resolve to Hash" unless p.class==Hash
|
44
|
+
return p
|
45
|
+
end
|
46
|
+
|
47
|
+
def job
|
48
|
+
s = self
|
49
|
+
job_path = s.path.split("/")[0..-2].join("/")
|
50
|
+
Job.where(:path=>job_path).first
|
51
|
+
end
|
52
|
+
|
20
53
|
def Stage.find_or_create_by_path(path)
|
21
54
|
s = Stage.where(:path=>path).first
|
22
55
|
s = Stage.create(:path=>path) unless s
|
@@ -47,11 +80,7 @@ module Mobilize
|
|
47
80
|
s.update_attributes(:started_at=>Time.now.utc)
|
48
81
|
s.update_status(%{Starting at #{Time.now.utc}})
|
49
82
|
#get response by running method
|
50
|
-
response =
|
51
|
-
"Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
|
52
|
-
rescue => exc
|
53
|
-
{'err_str'=>"#{exc.to_s}\n#{exc.backtrace.to_a.join("\n")}", 'signal'=>500}
|
54
|
-
end
|
83
|
+
response = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
|
55
84
|
unless response
|
56
85
|
#re-queue self if no response
|
57
86
|
s.enqueue!
|
@@ -63,10 +92,10 @@ module Mobilize
|
|
63
92
|
#retry
|
64
93
|
s.update_attributes(:retries_done => s.retries_done.to_i + 1, :response => response)
|
65
94
|
s.update_status(%{Retry #{s.retries_done.to_s} at #{Time.now.utc}})
|
66
|
-
sleep s['delay'].to_i
|
67
95
|
s.enqueue!
|
68
96
|
else
|
69
97
|
#sleep as much as user specifies
|
98
|
+
sleep s['delay'].to_i
|
70
99
|
s.fail(response)
|
71
100
|
end
|
72
101
|
return true
|
@@ -82,7 +111,7 @@ module Mobilize
|
|
82
111
|
r = j.runner
|
83
112
|
dep_jobs = r.jobs.select do |dj|
|
84
113
|
dj.active==true and
|
85
|
-
dj.trigger.strip.downcase == "after #{j.name
|
114
|
+
dj.trigger.strip.downcase == "after #{j.name}"
|
86
115
|
end
|
87
116
|
#put begin/rescue so all dependencies run
|
88
117
|
dep_jobs.each do |dj|
|
@@ -112,7 +141,7 @@ module Mobilize
|
|
112
141
|
j = s.job
|
113
142
|
r = j.runner
|
114
143
|
u = r.user
|
115
|
-
j.update_attributes(:active=>false)
|
144
|
+
j.update_attributes(:active=>false)
|
116
145
|
s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
|
117
146
|
stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
|
118
147
|
target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
|
@@ -129,9 +158,10 @@ module Mobilize
|
|
129
158
|
err_txt = ["response","\n",err_txt].join
|
130
159
|
err_sheet.write(err_txt,u.name)
|
131
160
|
#exception will be first row below "response" header
|
161
|
+
exc_to_s,backtrace = err_txt.split("\n").ie{|ea| [ea[1], ea[2..-1]]}
|
132
162
|
s.update_status(status_msg)
|
133
163
|
#raise the exception so it bubbles up to resque
|
134
|
-
raise Exception,
|
164
|
+
raise Exception,exc_to_s,backtrace
|
135
165
|
end
|
136
166
|
|
137
167
|
def enqueue!
|
@@ -178,15 +208,14 @@ module Mobilize
|
|
178
208
|
raise "incompatible target handler #{handler} for #{s.handler} stage"
|
179
209
|
else
|
180
210
|
begin
|
181
|
-
|
182
|
-
return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path,nil)
|
211
|
+
return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path)
|
183
212
|
rescue => exc
|
184
213
|
raise "Could not get #{target_path} with error: #{exc.to_s}"
|
185
214
|
end
|
186
215
|
end
|
187
216
|
end
|
188
217
|
|
189
|
-
def sources
|
218
|
+
def sources
|
190
219
|
#returns an array of Datasets corresponding to
|
191
220
|
#items listed as sources in the stage params
|
192
221
|
s = self
|
@@ -215,7 +244,7 @@ module Mobilize
|
|
215
244
|
end
|
216
245
|
begin
|
217
246
|
stage_path = s.path
|
218
|
-
dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path
|
247
|
+
dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path)
|
219
248
|
rescue => exc
|
220
249
|
raise "Could not get #{source_path} with error: #{exc.to_s}"
|
221
250
|
end
|
data/lib/mobilize-base/tasks.rb
CHANGED
@@ -40,17 +40,17 @@ namespace :mobilize_base do
|
|
40
40
|
Mobilize::Jobtracker.prep_workers
|
41
41
|
end
|
42
42
|
desc "Stop Jobtracker"
|
43
|
-
task :
|
43
|
+
task :stop do
|
44
44
|
require 'mobilize-base'
|
45
45
|
Mobilize::Jobtracker.stop!
|
46
46
|
end
|
47
47
|
desc "Start Jobtracker"
|
48
|
-
task :
|
48
|
+
task :start do
|
49
49
|
require 'mobilize-base'
|
50
50
|
Mobilize::Jobtracker.start
|
51
51
|
end
|
52
52
|
desc "Restart Jobtracker"
|
53
|
-
task :
|
53
|
+
task :restart do
|
54
54
|
require 'mobilize-base'
|
55
55
|
Mobilize::Jobtracker.restart!
|
56
56
|
end
|
@@ -66,19 +66,6 @@ namespace :mobilize_base do
|
|
66
66
|
resque_redis_port_args = if Mobilize::Base.env == 'test'
|
67
67
|
" -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
|
68
68
|
end.to_s
|
69
|
-
#determine view folder and override queues and working erbs
|
70
|
-
require 'resque/server'
|
71
|
-
view_dir = ::Resque::Server.views + "/"
|
72
|
-
old_queues_erb_path = view_dir + "queues.erb"
|
73
|
-
old_working_erb_path = view_dir + "working.erb"
|
74
|
-
gem_dir = Gem::Specification.find_by_name("mobilize-base").gem_dir
|
75
|
-
new_queues_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/queues.erb"
|
76
|
-
new_working_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/working.erb"
|
77
|
-
[old_queues_erb_path,old_working_erb_path].each{|p| File.delete(p) if File.exists?(p)}
|
78
|
-
require 'fileutils'
|
79
|
-
FileUtils.copy(new_queues_erb_path,old_queues_erb_path)
|
80
|
-
FileUtils.copy(new_working_erb_path,old_working_erb_path)
|
81
|
-
sleep 5 #give them time to die
|
82
69
|
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
|
83
70
|
`#{command}`
|
84
71
|
end
|
data/lib/mobilize-base.rb
CHANGED
@@ -60,18 +60,14 @@ module Mobilize
|
|
60
60
|
end
|
61
61
|
mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
|
62
62
|
if File.exists?(mongoid_config_path)
|
63
|
+
require 'mongo'
|
63
64
|
require 'mongoid'
|
64
|
-
require 'mongoid-grid_fs'
|
65
65
|
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
66
66
|
require "mobilize-base/models/dataset"
|
67
67
|
require "mobilize-base/models/user"
|
68
|
-
require "mobilize-base/helpers/runner_helper"
|
69
68
|
require "mobilize-base/models/runner"
|
70
|
-
require "mobilize-base/helpers/job_helper"
|
71
69
|
require "mobilize-base/models/job"
|
72
|
-
require "mobilize-base/helpers/stage_helper"
|
73
70
|
require "mobilize-base/models/stage"
|
74
|
-
|
75
71
|
end
|
76
72
|
require 'google_drive'
|
77
73
|
require 'resque'
|
data/lib/samples/gridfs.yml
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
3
4
|
max_compressed_write_size: 1000000000 #~1GB
|
4
5
|
test:
|
6
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
5
7
|
max_compressed_write_size: 1000000000 #~1GB
|
6
8
|
production:
|
9
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
7
10
|
max_compressed_write_size: 1000000000 #~1GB
|
data/lib/samples/gsheet.yml
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
-
max_cells:
|
3
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
4
4
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
5
5
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
6
6
|
test:
|
7
|
-
max_cells:
|
7
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
8
8
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
9
9
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
10
10
|
staging:
|
11
|
-
max_cells:
|
11
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
12
12
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
13
13
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
14
14
|
production:
|
15
|
-
max_cells:
|
15
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
16
16
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
17
17
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
data/mobilize-base.gemspec
CHANGED
@@ -22,11 +22,12 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.require_paths = ["lib"]
|
23
23
|
|
24
24
|
s.add_runtime_dependency 'rake'
|
25
|
-
s.add_runtime_dependency 'bson','1.
|
26
|
-
s.add_runtime_dependency 'bson_ext','1.
|
25
|
+
s.add_runtime_dependency 'bson','1.6.1'
|
26
|
+
s.add_runtime_dependency 'bson_ext','1.6.1'
|
27
|
+
s.add_runtime_dependency 'mongo', '1.6.1'
|
27
28
|
s.add_runtime_dependency "mongoid", "~>3.0.0"
|
28
|
-
s.add_runtime_dependency '
|
29
|
-
s.add_runtime_dependency 'resque',
|
29
|
+
s.add_runtime_dependency 'redis',"~>3.0.0"
|
30
|
+
s.add_runtime_dependency 'resque','1.21.0'
|
30
31
|
s.add_runtime_dependency 'google_drive','0.3.2'
|
31
32
|
s.add_runtime_dependency 'popen4','0.1.2'
|
32
33
|
s.add_runtime_dependency 'actionmailer','3.1.1'
|
data/test/mobilize-base_test.rb
CHANGED
@@ -64,7 +64,6 @@ describe "Mobilize" do
|
|
64
64
|
test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
|
65
65
|
test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
|
66
66
|
|
67
|
-
assert test_1_tsv.to_s.length>0
|
68
67
|
assert test_1_tsv == test_2_tsv
|
69
68
|
|
70
69
|
puts "change first job to fail, wait for stages"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.21'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-03-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - '='
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.
|
37
|
+
version: 1.6.1
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - '='
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.
|
45
|
+
version: 1.6.1
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: bson_ext
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - '='
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 1.
|
53
|
+
version: 1.6.1
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -58,7 +58,23 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - '='
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.
|
61
|
+
version: 1.6.1
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: mongo
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - '='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.6.1
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - '='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.6.1
|
62
78
|
- !ruby/object:Gem::Dependency
|
63
79
|
name: mongoid
|
64
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -76,21 +92,21 @@ dependencies:
|
|
76
92
|
- !ruby/object:Gem::Version
|
77
93
|
version: 3.0.0
|
78
94
|
- !ruby/object:Gem::Dependency
|
79
|
-
name:
|
95
|
+
name: redis
|
80
96
|
requirement: !ruby/object:Gem::Requirement
|
81
97
|
none: false
|
82
98
|
requirements:
|
83
|
-
- -
|
99
|
+
- - ~>
|
84
100
|
- !ruby/object:Gem::Version
|
85
|
-
version:
|
101
|
+
version: 3.0.0
|
86
102
|
type: :runtime
|
87
103
|
prerelease: false
|
88
104
|
version_requirements: !ruby/object:Gem::Requirement
|
89
105
|
none: false
|
90
106
|
requirements:
|
91
|
-
- -
|
107
|
+
- - ~>
|
92
108
|
- !ruby/object:Gem::Version
|
93
|
-
version:
|
109
|
+
version: 3.0.0
|
94
110
|
- !ruby/object:Gem::Dependency
|
95
111
|
name: resque
|
96
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -98,7 +114,7 @@ dependencies:
|
|
98
114
|
requirements:
|
99
115
|
- - '='
|
100
116
|
- !ruby/object:Gem::Version
|
101
|
-
version: 1.
|
117
|
+
version: 1.21.0
|
102
118
|
type: :runtime
|
103
119
|
prerelease: false
|
104
120
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -106,7 +122,7 @@ dependencies:
|
|
106
122
|
requirements:
|
107
123
|
- - '='
|
108
124
|
- !ruby/object:Gem::Version
|
109
|
-
version: 1.
|
125
|
+
version: 1.21.0
|
110
126
|
- !ruby/object:Gem::Dependency
|
111
127
|
name: google_drive
|
112
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -179,8 +195,6 @@ files:
|
|
179
195
|
- lib/mobilize-base/extensions/google_drive/worksheet.rb
|
180
196
|
- lib/mobilize-base/extensions/hash.rb
|
181
197
|
- lib/mobilize-base/extensions/object.rb
|
182
|
-
- lib/mobilize-base/extensions/resque-server/views/queues.erb
|
183
|
-
- lib/mobilize-base/extensions/resque-server/views/working.erb
|
184
198
|
- lib/mobilize-base/extensions/string.rb
|
185
199
|
- lib/mobilize-base/extensions/yaml.rb
|
186
200
|
- lib/mobilize-base/handlers/email.rb
|
@@ -190,9 +204,6 @@ files:
|
|
190
204
|
- lib/mobilize-base/handlers/gridfs.rb
|
191
205
|
- lib/mobilize-base/handlers/gsheet.rb
|
192
206
|
- lib/mobilize-base/handlers/resque.rb
|
193
|
-
- lib/mobilize-base/helpers/job_helper.rb
|
194
|
-
- lib/mobilize-base/helpers/runner_helper.rb
|
195
|
-
- lib/mobilize-base/helpers/stage_helper.rb
|
196
207
|
- lib/mobilize-base/jobtracker.rb
|
197
208
|
- lib/mobilize-base/models/dataset.rb
|
198
209
|
- lib/mobilize-base/models/job.rb
|
@@ -228,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
228
239
|
version: '0'
|
229
240
|
segments:
|
230
241
|
- 0
|
231
|
-
hash: -
|
242
|
+
hash: -779257039154498150
|
232
243
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
233
244
|
none: false
|
234
245
|
requirements:
|
@@ -237,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
248
|
version: '0'
|
238
249
|
segments:
|
239
250
|
- 0
|
240
|
-
hash: -
|
251
|
+
hash: -779257039154498150
|
241
252
|
requirements: []
|
242
253
|
rubyforge_project: mobilize-base
|
243
254
|
rubygems_version: 1.8.25
|
@@ -1,59 +0,0 @@
|
|
1
|
-
<% @subtabs = resque.queues unless partial? || params[:id].nil? %>
|
2
|
-
|
3
|
-
<% if queue = params[:id] %>
|
4
|
-
|
5
|
-
<h1>Pending jobs on <span class='hl'><%= queue %></span></h1>
|
6
|
-
<form method="POST" action="<%=u "/queues/#{queue}/remove" %>" class='remove-queue'>
|
7
|
-
<input type='submit' name='' value='Remove Queue' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
|
8
|
-
</form>
|
9
|
-
<p class='sub'>Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%=size = resque.size(queue)%></b> jobs</p>
|
10
|
-
<table class='jobs'>
|
11
|
-
<tr>
|
12
|
-
<th>Class</th>
|
13
|
-
<th>Args</th>
|
14
|
-
</tr>
|
15
|
-
<% for job in (jobs = resque.peek(queue, start, 20)) %>
|
16
|
-
<tr>
|
17
|
-
<td class='class'><%= job['class'] %></td>
|
18
|
-
<td class='args'><%=h job['args'].inspect %></td>
|
19
|
-
</tr>
|
20
|
-
<% end %>
|
21
|
-
<% if jobs.empty? %>
|
22
|
-
<tr>
|
23
|
-
<td class='no-data' colspan='2'>There are no pending jobs in this queue</td>
|
24
|
-
</tr>
|
25
|
-
<% end %>
|
26
|
-
</table>
|
27
|
-
<%= partial :next_more, :start => start, :size => size, :per_page => 20 %>
|
28
|
-
<% else %>
|
29
|
-
|
30
|
-
<h1 class='wi'>Queues</h1>
|
31
|
-
<p class='intro'>The list below contains all the registered queues with the number of jobs currently in the queue. Select a queue from above to view all jobs currently pending on the queue.</p>
|
32
|
-
<table class='queues'>
|
33
|
-
<tr>
|
34
|
-
<th>Name</th>
|
35
|
-
<th>Jobs</th>
|
36
|
-
</tr>
|
37
|
-
<!-- only show nonzero length queues-->
|
38
|
-
<% resque.queues.select{|q| resque.size(q)>0}.sort_by { |q| q.to_s }.each do |queue| %>
|
39
|
-
<tr>
|
40
|
-
<td class='queue'><a class="queue" href="<%= u "queues/#{queue}" %>"><%= queue %></a></td>
|
41
|
-
<td class='size'><%= resque.size queue %></td>
|
42
|
-
</tr>
|
43
|
-
<% end %>
|
44
|
-
<% if failed_multiple_queues? %>
|
45
|
-
<% Resque::Failure.queues.sort_by { |q| q.to_s }.each_with_index do |queue, i| %>
|
46
|
-
<tr class="<%= Resque::Failure.count(queue).zero? ? "failed" : "failure" %><%= " first_failure" if i.zero? %>">
|
47
|
-
<td class='queue failed'><a class="queue" href="<%= u "failed/#{queue}" %>"><%= queue %></a></td>
|
48
|
-
<td class='size'><%= Resque::Failure.count(queue) %></td>
|
49
|
-
</tr>
|
50
|
-
<% end %>
|
51
|
-
<% else %>
|
52
|
-
<tr class="<%= Resque::Failure.count.zero? ? "failed" : "failure" %>">
|
53
|
-
<td class='queue failed'><a class="queue" href="<%= u :failed %>">failed</a></td>
|
54
|
-
<td class='size'><%= Resque::Failure.count %></td>
|
55
|
-
</tr>
|
56
|
-
<% end %>
|
57
|
-
</table>
|
58
|
-
|
59
|
-
<% end %>
|