mobilize-base 1.3 → 1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +11 -10
- data/lib/mobilize-base/extensions/google_drive/file.rb +7 -7
- data/lib/mobilize-base/extensions/google_drive/worksheet.rb +2 -7
- data/lib/mobilize-base/extensions/string.rb +4 -12
- data/lib/mobilize-base/extensions/yaml.rb +7 -10
- data/lib/mobilize-base/handlers/gbook.rb +31 -24
- data/lib/mobilize-base/handlers/gfile.rb +3 -5
- data/lib/mobilize-base/handlers/gridfs.rb +24 -19
- data/lib/mobilize-base/handlers/gsheet.rb +20 -25
- data/lib/mobilize-base/handlers/resque.rb +4 -16
- data/lib/mobilize-base/jobtracker.rb +5 -13
- data/lib/mobilize-base/models/job.rb +48 -36
- data/lib/mobilize-base/models/runner.rb +123 -24
- data/lib/mobilize-base/models/stage.rb +43 -14
- data/lib/mobilize-base/tasks.rb +3 -16
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +1 -5
- data/lib/samples/gridfs.yml +3 -0
- data/lib/samples/gsheet.yml +4 -4
- data/mobilize-base.gemspec +5 -4
- data/test/mobilize-base_test.rb +0 -1
- metadata +31 -20
- data/lib/mobilize-base/extensions/resque-server/views/queues.erb +0 -59
- data/lib/mobilize-base/extensions/resque-server/views/working.erb +0 -85
- data/lib/mobilize-base/helpers/job_helper.rb +0 -54
- data/lib/mobilize-base/helpers/runner_helper.rb +0 -83
- data/lib/mobilize-base/helpers/stage_helper.rb +0 -38
@@ -1,6 +1,5 @@
|
|
1
1
|
module Mobilize
|
2
2
|
class Runner
|
3
|
-
include Mobilize::RunnerHelper
|
4
3
|
include Mongoid::Document
|
5
4
|
include Mongoid::Timestamps
|
6
5
|
field :path, type: String
|
@@ -12,6 +11,20 @@ module Mobilize
|
|
12
11
|
|
13
12
|
index({ path: 1})
|
14
13
|
|
14
|
+
def headers
|
15
|
+
%w{name active trigger status stage1 stage2 stage3 stage4 stage5}
|
16
|
+
end
|
17
|
+
|
18
|
+
def title
|
19
|
+
r = self
|
20
|
+
r.path.split("/").first
|
21
|
+
end
|
22
|
+
|
23
|
+
def worker
|
24
|
+
r = self
|
25
|
+
Mobilize::Resque.find_worker_by_path(r.path)
|
26
|
+
end
|
27
|
+
|
15
28
|
def Runner.find_by_path(path)
|
16
29
|
Runner.where(:path=>path).first
|
17
30
|
end
|
@@ -19,7 +32,6 @@ module Mobilize
|
|
19
32
|
def Runner.find_by_title(title)
|
20
33
|
Runner.where(:path=>"#{title}/jobs").first
|
21
34
|
end
|
22
|
-
|
23
35
|
def Runner.perform(id,*args)
|
24
36
|
r = Runner.find_by_path(id)
|
25
37
|
#get gdrive slot for read
|
@@ -29,18 +41,12 @@ module Mobilize
|
|
29
41
|
return false
|
30
42
|
end
|
31
43
|
r.update_attributes(:started_at=>Time.now.utc)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
#queue up the jobs that are due and active
|
39
|
-
rescue => exc
|
40
|
-
#log the exception, but continue w job processing
|
41
|
-
#This ensures jobs are still processed if google drive goes down
|
42
|
-
r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
|
43
|
-
end
|
44
|
+
#make sure any updates to activity are processed first
|
45
|
+
#as in when someone runs a "once" job that has completed
|
46
|
+
r.update_gsheet(gdrive_slot)
|
47
|
+
#read the jobs in the gsheet and update models with news
|
48
|
+
r.read_gsheet(gdrive_slot)
|
49
|
+
#queue up the jobs that are due and active
|
44
50
|
r.jobs.each do |j|
|
45
51
|
begin
|
46
52
|
if j.is_due?
|
@@ -58,32 +64,90 @@ module Mobilize
|
|
58
64
|
r.update_attributes(:completed_at=>Time.now.utc)
|
59
65
|
end
|
60
66
|
|
67
|
+
def dataset
|
68
|
+
r = self
|
69
|
+
Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
|
70
|
+
end
|
71
|
+
|
61
72
|
def Runner.find_or_create_by_path(path)
|
62
73
|
Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
|
63
74
|
end
|
64
75
|
|
76
|
+
def gbook(gdrive_slot)
|
77
|
+
r = self
|
78
|
+
title = r.path.split("/").first
|
79
|
+
Gbook.find_all_by_path(title,gdrive_slot).first
|
80
|
+
end
|
81
|
+
|
82
|
+
def gsheet(gdrive_slot)
|
83
|
+
r = self
|
84
|
+
u = r.user
|
85
|
+
jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
|
86
|
+
#make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
|
87
|
+
unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
|
88
|
+
#only give the user edit permissions if they're the ones
|
89
|
+
#creating it
|
90
|
+
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
91
|
+
unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
|
92
|
+
jobs_sheet.spreadsheet.update_acl(u.email,"writer")
|
93
|
+
end
|
94
|
+
end
|
95
|
+
jobs_sheet.add_headers(r.headers)
|
96
|
+
#add url to dataset
|
97
|
+
Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
|
98
|
+
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
99
|
+
return jobs_sheet
|
100
|
+
end
|
101
|
+
|
65
102
|
def read_gsheet(gdrive_slot)
|
66
103
|
r = self
|
67
|
-
|
68
|
-
gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
|
104
|
+
gsheet_tsv = r.gsheet(gdrive_slot).read(Gdrive.owner_name)
|
69
105
|
#turn it into a hash array
|
70
|
-
|
106
|
+
gsheet_jobs = gsheet_tsv.tsv_to_hash_array
|
71
107
|
#go through each job, update relevant job with its params
|
72
108
|
done_jobs = []
|
73
109
|
#parse out the jobs and update the Job collection
|
74
|
-
|
110
|
+
gsheet_jobs.each_with_index do |rj,rj_i|
|
75
111
|
#skip non-jobs or jobs without required values
|
76
|
-
next if (
|
77
|
-
j = Job.
|
112
|
+
next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
|
113
|
+
j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
|
114
|
+
#update top line params
|
115
|
+
j.update_attributes(:active => rj['active'],
|
116
|
+
:trigger => rj['trigger'])
|
117
|
+
(1..5).to_a.each do |s_idx|
|
118
|
+
stage_string = rj["stage#{s_idx.to_s}"]
|
119
|
+
s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
|
120
|
+
if stage_string.to_s.length==0
|
121
|
+
#delete this stage and all stages after
|
122
|
+
if s
|
123
|
+
j = s.job
|
124
|
+
j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
|
125
|
+
#just in case
|
126
|
+
s.delete
|
127
|
+
end
|
128
|
+
break
|
129
|
+
elsif s.nil?
|
130
|
+
#create this stage
|
131
|
+
s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
|
132
|
+
end
|
133
|
+
#parse command string, update stage with it
|
134
|
+
s_handler, call, param_string = [""*3]
|
135
|
+
stage_string.split(" ").ie do |spls|
|
136
|
+
s_handler = spls.first.split(".").first
|
137
|
+
call = spls.first.split(".").last
|
138
|
+
param_string = spls[1..-1].join(" ").strip
|
139
|
+
end
|
140
|
+
s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
|
141
|
+
end
|
78
142
|
r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
|
79
143
|
#add this job to list of read ones
|
80
144
|
done_jobs << j
|
81
145
|
end
|
82
146
|
#delete user jobs that are not included in Runner
|
83
|
-
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |
|
84
|
-
j = Job.where(:path=>
|
147
|
+
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
|
148
|
+
j = Job.where(:path=>rj_path).first
|
85
149
|
j.delete if j
|
86
|
-
r.update_status("Deleted job:#{
|
150
|
+
r.update_status("Deleted job:#{rj_path}")
|
87
151
|
end
|
88
152
|
r.update_status("jobs read at #{Time.now.utc}")
|
89
153
|
return true
|
@@ -94,13 +158,48 @@ module Mobilize
|
|
94
158
|
#there's nothing to update if runner has never had a completed at
|
95
159
|
return false unless r.completed_at
|
96
160
|
jobs_gsheet = r.gsheet(gdrive_slot)
|
97
|
-
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at
|
161
|
+
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
|
98
162
|
upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
|
99
163
|
jobs_gsheet.add_or_update_rows(upd_rows)
|
100
164
|
r.update_status("gsheet updated")
|
101
165
|
return true
|
102
166
|
end
|
103
167
|
|
168
|
+
def jobs(jname=nil)
|
169
|
+
r = self
|
170
|
+
js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
|
171
|
+
if jname
|
172
|
+
return js.sel{|j| j.name == jname}.first
|
173
|
+
else
|
174
|
+
return js
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def user
|
179
|
+
r = self
|
180
|
+
user_name = r.path.split("_").second.split("(").first.split("/").first
|
181
|
+
User.where(:name=>user_name).first
|
182
|
+
end
|
183
|
+
|
184
|
+
def update_status(msg)
|
185
|
+
r = self
|
186
|
+
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
187
|
+
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
188
|
+
return true
|
189
|
+
end
|
190
|
+
|
191
|
+
def is_working?
|
192
|
+
r = self
|
193
|
+
Mobilize::Resque.active_paths.include?(r.path)
|
194
|
+
end
|
195
|
+
|
196
|
+
def is_due?
|
197
|
+
r = self.reload
|
198
|
+
return false if r.is_working?
|
199
|
+
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
200
|
+
return true if r.started_at.nil? or r.started_at < prev_due_time
|
201
|
+
end
|
202
|
+
|
104
203
|
def enqueue!
|
105
204
|
r = self
|
106
205
|
::Resque::Job.create("mobilize",Runner,r.path,{})
|
@@ -2,7 +2,6 @@ module Mobilize
|
|
2
2
|
class Stage
|
3
3
|
include Mongoid::Document
|
4
4
|
include Mongoid::Timestamps
|
5
|
-
include Mobilize::StageHelper
|
6
5
|
field :path, type: String
|
7
6
|
field :handler, type: String
|
8
7
|
field :call, type: String
|
@@ -17,6 +16,40 @@ module Mobilize
|
|
17
16
|
|
18
17
|
index({ path: 1})
|
19
18
|
|
19
|
+
def idx
|
20
|
+
s = self
|
21
|
+
s.path.split("/").last.gsub("stage","").to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
def out_dst
|
25
|
+
#this gives a dataset that points to the output
|
26
|
+
#allowing you to determine its size
|
27
|
+
#before committing to a read or write
|
28
|
+
s = self
|
29
|
+
Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
|
30
|
+
end
|
31
|
+
|
32
|
+
def err_dst
|
33
|
+
#this gives a dataset that points to the output
|
34
|
+
#allowing you to determine its size
|
35
|
+
#before committing to a read or write
|
36
|
+
s = self
|
37
|
+
Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
|
38
|
+
end
|
39
|
+
|
40
|
+
def params
|
41
|
+
s = self
|
42
|
+
p = YAML.easy_load(s.param_string)
|
43
|
+
raise "Must resolve to Hash" unless p.class==Hash
|
44
|
+
return p
|
45
|
+
end
|
46
|
+
|
47
|
+
def job
|
48
|
+
s = self
|
49
|
+
job_path = s.path.split("/")[0..-2].join("/")
|
50
|
+
Job.where(:path=>job_path).first
|
51
|
+
end
|
52
|
+
|
20
53
|
def Stage.find_or_create_by_path(path)
|
21
54
|
s = Stage.where(:path=>path).first
|
22
55
|
s = Stage.create(:path=>path) unless s
|
@@ -47,11 +80,7 @@ module Mobilize
|
|
47
80
|
s.update_attributes(:started_at=>Time.now.utc)
|
48
81
|
s.update_status(%{Starting at #{Time.now.utc}})
|
49
82
|
#get response by running method
|
50
|
-
response =
|
51
|
-
"Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
|
52
|
-
rescue => exc
|
53
|
-
{'err_str'=>"#{exc.to_s}\n#{exc.backtrace.to_a.join("\n")}", 'signal'=>500}
|
54
|
-
end
|
83
|
+
response = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
|
55
84
|
unless response
|
56
85
|
#re-queue self if no response
|
57
86
|
s.enqueue!
|
@@ -63,10 +92,10 @@ module Mobilize
|
|
63
92
|
#retry
|
64
93
|
s.update_attributes(:retries_done => s.retries_done.to_i + 1, :response => response)
|
65
94
|
s.update_status(%{Retry #{s.retries_done.to_s} at #{Time.now.utc}})
|
66
|
-
sleep s['delay'].to_i
|
67
95
|
s.enqueue!
|
68
96
|
else
|
69
97
|
#sleep as much as user specifies
|
98
|
+
sleep s['delay'].to_i
|
70
99
|
s.fail(response)
|
71
100
|
end
|
72
101
|
return true
|
@@ -82,7 +111,7 @@ module Mobilize
|
|
82
111
|
r = j.runner
|
83
112
|
dep_jobs = r.jobs.select do |dj|
|
84
113
|
dj.active==true and
|
85
|
-
dj.trigger.strip.downcase == "after #{j.name
|
114
|
+
dj.trigger.strip.downcase == "after #{j.name}"
|
86
115
|
end
|
87
116
|
#put begin/rescue so all dependencies run
|
88
117
|
dep_jobs.each do |dj|
|
@@ -112,7 +141,7 @@ module Mobilize
|
|
112
141
|
j = s.job
|
113
142
|
r = j.runner
|
114
143
|
u = r.user
|
115
|
-
j.update_attributes(:active=>false)
|
144
|
+
j.update_attributes(:active=>false)
|
116
145
|
s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
|
117
146
|
stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
|
118
147
|
target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
|
@@ -129,9 +158,10 @@ module Mobilize
|
|
129
158
|
err_txt = ["response","\n",err_txt].join
|
130
159
|
err_sheet.write(err_txt,u.name)
|
131
160
|
#exception will be first row below "response" header
|
161
|
+
exc_to_s,backtrace = err_txt.split("\n").ie{|ea| [ea[1], ea[2..-1]]}
|
132
162
|
s.update_status(status_msg)
|
133
163
|
#raise the exception so it bubbles up to resque
|
134
|
-
raise Exception,
|
164
|
+
raise Exception,exc_to_s,backtrace
|
135
165
|
end
|
136
166
|
|
137
167
|
def enqueue!
|
@@ -178,15 +208,14 @@ module Mobilize
|
|
178
208
|
raise "incompatible target handler #{handler} for #{s.handler} stage"
|
179
209
|
else
|
180
210
|
begin
|
181
|
-
|
182
|
-
return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path,nil)
|
211
|
+
return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path)
|
183
212
|
rescue => exc
|
184
213
|
raise "Could not get #{target_path} with error: #{exc.to_s}"
|
185
214
|
end
|
186
215
|
end
|
187
216
|
end
|
188
217
|
|
189
|
-
def sources
|
218
|
+
def sources
|
190
219
|
#returns an array of Datasets corresponding to
|
191
220
|
#items listed as sources in the stage params
|
192
221
|
s = self
|
@@ -215,7 +244,7 @@ module Mobilize
|
|
215
244
|
end
|
216
245
|
begin
|
217
246
|
stage_path = s.path
|
218
|
-
dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path
|
247
|
+
dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path)
|
219
248
|
rescue => exc
|
220
249
|
raise "Could not get #{source_path} with error: #{exc.to_s}"
|
221
250
|
end
|
data/lib/mobilize-base/tasks.rb
CHANGED
@@ -40,17 +40,17 @@ namespace :mobilize_base do
|
|
40
40
|
Mobilize::Jobtracker.prep_workers
|
41
41
|
end
|
42
42
|
desc "Stop Jobtracker"
|
43
|
-
task :
|
43
|
+
task :stop do
|
44
44
|
require 'mobilize-base'
|
45
45
|
Mobilize::Jobtracker.stop!
|
46
46
|
end
|
47
47
|
desc "Start Jobtracker"
|
48
|
-
task :
|
48
|
+
task :start do
|
49
49
|
require 'mobilize-base'
|
50
50
|
Mobilize::Jobtracker.start
|
51
51
|
end
|
52
52
|
desc "Restart Jobtracker"
|
53
|
-
task :
|
53
|
+
task :restart do
|
54
54
|
require 'mobilize-base'
|
55
55
|
Mobilize::Jobtracker.restart!
|
56
56
|
end
|
@@ -66,19 +66,6 @@ namespace :mobilize_base do
|
|
66
66
|
resque_redis_port_args = if Mobilize::Base.env == 'test'
|
67
67
|
" -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
|
68
68
|
end.to_s
|
69
|
-
#determine view folder and override queues and working erbs
|
70
|
-
require 'resque/server'
|
71
|
-
view_dir = ::Resque::Server.views + "/"
|
72
|
-
old_queues_erb_path = view_dir + "queues.erb"
|
73
|
-
old_working_erb_path = view_dir + "working.erb"
|
74
|
-
gem_dir = Gem::Specification.find_by_name("mobilize-base").gem_dir
|
75
|
-
new_queues_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/queues.erb"
|
76
|
-
new_working_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/working.erb"
|
77
|
-
[old_queues_erb_path,old_working_erb_path].each{|p| File.delete(p) if File.exists?(p)}
|
78
|
-
require 'fileutils'
|
79
|
-
FileUtils.copy(new_queues_erb_path,old_queues_erb_path)
|
80
|
-
FileUtils.copy(new_working_erb_path,old_working_erb_path)
|
81
|
-
sleep 5 #give them time to die
|
82
69
|
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
|
83
70
|
`#{command}`
|
84
71
|
end
|
data/lib/mobilize-base.rb
CHANGED
@@ -60,18 +60,14 @@ module Mobilize
|
|
60
60
|
end
|
61
61
|
mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
|
62
62
|
if File.exists?(mongoid_config_path)
|
63
|
+
require 'mongo'
|
63
64
|
require 'mongoid'
|
64
|
-
require 'mongoid-grid_fs'
|
65
65
|
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
66
66
|
require "mobilize-base/models/dataset"
|
67
67
|
require "mobilize-base/models/user"
|
68
|
-
require "mobilize-base/helpers/runner_helper"
|
69
68
|
require "mobilize-base/models/runner"
|
70
|
-
require "mobilize-base/helpers/job_helper"
|
71
69
|
require "mobilize-base/models/job"
|
72
|
-
require "mobilize-base/helpers/stage_helper"
|
73
70
|
require "mobilize-base/models/stage"
|
74
|
-
|
75
71
|
end
|
76
72
|
require 'google_drive'
|
77
73
|
require 'resque'
|
data/lib/samples/gridfs.yml
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
3
4
|
max_compressed_write_size: 1000000000 #~1GB
|
4
5
|
test:
|
6
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
5
7
|
max_compressed_write_size: 1000000000 #~1GB
|
6
8
|
production:
|
9
|
+
max_versions: 10 #number of versions of cache to keep in gridfs
|
7
10
|
max_compressed_write_size: 1000000000 #~1GB
|
data/lib/samples/gsheet.yml
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
-
max_cells:
|
3
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
4
4
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
5
5
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
6
6
|
test:
|
7
|
-
max_cells:
|
7
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
8
8
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
9
9
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
10
10
|
staging:
|
11
|
-
max_cells:
|
11
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
12
12
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
13
13
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
14
14
|
production:
|
15
|
-
max_cells:
|
15
|
+
max_cells: 400000 #current google limit is 400k cells per book
|
16
16
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
17
17
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
data/mobilize-base.gemspec
CHANGED
@@ -22,11 +22,12 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.require_paths = ["lib"]
|
23
23
|
|
24
24
|
s.add_runtime_dependency 'rake'
|
25
|
-
s.add_runtime_dependency 'bson','1.
|
26
|
-
s.add_runtime_dependency 'bson_ext','1.
|
25
|
+
s.add_runtime_dependency 'bson','1.6.1'
|
26
|
+
s.add_runtime_dependency 'bson_ext','1.6.1'
|
27
|
+
s.add_runtime_dependency 'mongo', '1.6.1'
|
27
28
|
s.add_runtime_dependency "mongoid", "~>3.0.0"
|
28
|
-
s.add_runtime_dependency '
|
29
|
-
s.add_runtime_dependency 'resque',
|
29
|
+
s.add_runtime_dependency 'redis',"~>3.0.0"
|
30
|
+
s.add_runtime_dependency 'resque','1.21.0'
|
30
31
|
s.add_runtime_dependency 'google_drive','0.3.2'
|
31
32
|
s.add_runtime_dependency 'popen4','0.1.2'
|
32
33
|
s.add_runtime_dependency 'actionmailer','3.1.1'
|
data/test/mobilize-base_test.rb
CHANGED
@@ -64,7 +64,6 @@ describe "Mobilize" do
|
|
64
64
|
test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
|
65
65
|
test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
|
66
66
|
|
67
|
-
assert test_1_tsv.to_s.length>0
|
68
67
|
assert test_1_tsv == test_2_tsv
|
69
68
|
|
70
69
|
puts "change first job to fail, wait for stages"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.21'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-03-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - '='
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.
|
37
|
+
version: 1.6.1
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - '='
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.
|
45
|
+
version: 1.6.1
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: bson_ext
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - '='
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 1.
|
53
|
+
version: 1.6.1
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -58,7 +58,23 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - '='
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.
|
61
|
+
version: 1.6.1
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: mongo
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - '='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.6.1
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - '='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.6.1
|
62
78
|
- !ruby/object:Gem::Dependency
|
63
79
|
name: mongoid
|
64
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -76,21 +92,21 @@ dependencies:
|
|
76
92
|
- !ruby/object:Gem::Version
|
77
93
|
version: 3.0.0
|
78
94
|
- !ruby/object:Gem::Dependency
|
79
|
-
name:
|
95
|
+
name: redis
|
80
96
|
requirement: !ruby/object:Gem::Requirement
|
81
97
|
none: false
|
82
98
|
requirements:
|
83
|
-
- -
|
99
|
+
- - ~>
|
84
100
|
- !ruby/object:Gem::Version
|
85
|
-
version:
|
101
|
+
version: 3.0.0
|
86
102
|
type: :runtime
|
87
103
|
prerelease: false
|
88
104
|
version_requirements: !ruby/object:Gem::Requirement
|
89
105
|
none: false
|
90
106
|
requirements:
|
91
|
-
- -
|
107
|
+
- - ~>
|
92
108
|
- !ruby/object:Gem::Version
|
93
|
-
version:
|
109
|
+
version: 3.0.0
|
94
110
|
- !ruby/object:Gem::Dependency
|
95
111
|
name: resque
|
96
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -98,7 +114,7 @@ dependencies:
|
|
98
114
|
requirements:
|
99
115
|
- - '='
|
100
116
|
- !ruby/object:Gem::Version
|
101
|
-
version: 1.
|
117
|
+
version: 1.21.0
|
102
118
|
type: :runtime
|
103
119
|
prerelease: false
|
104
120
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -106,7 +122,7 @@ dependencies:
|
|
106
122
|
requirements:
|
107
123
|
- - '='
|
108
124
|
- !ruby/object:Gem::Version
|
109
|
-
version: 1.
|
125
|
+
version: 1.21.0
|
110
126
|
- !ruby/object:Gem::Dependency
|
111
127
|
name: google_drive
|
112
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -179,8 +195,6 @@ files:
|
|
179
195
|
- lib/mobilize-base/extensions/google_drive/worksheet.rb
|
180
196
|
- lib/mobilize-base/extensions/hash.rb
|
181
197
|
- lib/mobilize-base/extensions/object.rb
|
182
|
-
- lib/mobilize-base/extensions/resque-server/views/queues.erb
|
183
|
-
- lib/mobilize-base/extensions/resque-server/views/working.erb
|
184
198
|
- lib/mobilize-base/extensions/string.rb
|
185
199
|
- lib/mobilize-base/extensions/yaml.rb
|
186
200
|
- lib/mobilize-base/handlers/email.rb
|
@@ -190,9 +204,6 @@ files:
|
|
190
204
|
- lib/mobilize-base/handlers/gridfs.rb
|
191
205
|
- lib/mobilize-base/handlers/gsheet.rb
|
192
206
|
- lib/mobilize-base/handlers/resque.rb
|
193
|
-
- lib/mobilize-base/helpers/job_helper.rb
|
194
|
-
- lib/mobilize-base/helpers/runner_helper.rb
|
195
|
-
- lib/mobilize-base/helpers/stage_helper.rb
|
196
207
|
- lib/mobilize-base/jobtracker.rb
|
197
208
|
- lib/mobilize-base/models/dataset.rb
|
198
209
|
- lib/mobilize-base/models/job.rb
|
@@ -228,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
228
239
|
version: '0'
|
229
240
|
segments:
|
230
241
|
- 0
|
231
|
-
hash: -
|
242
|
+
hash: -779257039154498150
|
232
243
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
233
244
|
none: false
|
234
245
|
requirements:
|
@@ -237,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
248
|
version: '0'
|
238
249
|
segments:
|
239
250
|
- 0
|
240
|
-
hash: -
|
251
|
+
hash: -779257039154498150
|
241
252
|
requirements: []
|
242
253
|
rubyforge_project: mobilize-base
|
243
254
|
rubygems_version: 1.8.25
|
@@ -1,59 +0,0 @@
|
|
1
|
-
<% @subtabs = resque.queues unless partial? || params[:id].nil? %>
|
2
|
-
|
3
|
-
<% if queue = params[:id] %>
|
4
|
-
|
5
|
-
<h1>Pending jobs on <span class='hl'><%= queue %></span></h1>
|
6
|
-
<form method="POST" action="<%=u "/queues/#{queue}/remove" %>" class='remove-queue'>
|
7
|
-
<input type='submit' name='' value='Remove Queue' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
|
8
|
-
</form>
|
9
|
-
<p class='sub'>Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%=size = resque.size(queue)%></b> jobs</p>
|
10
|
-
<table class='jobs'>
|
11
|
-
<tr>
|
12
|
-
<th>Class</th>
|
13
|
-
<th>Args</th>
|
14
|
-
</tr>
|
15
|
-
<% for job in (jobs = resque.peek(queue, start, 20)) %>
|
16
|
-
<tr>
|
17
|
-
<td class='class'><%= job['class'] %></td>
|
18
|
-
<td class='args'><%=h job['args'].inspect %></td>
|
19
|
-
</tr>
|
20
|
-
<% end %>
|
21
|
-
<% if jobs.empty? %>
|
22
|
-
<tr>
|
23
|
-
<td class='no-data' colspan='2'>There are no pending jobs in this queue</td>
|
24
|
-
</tr>
|
25
|
-
<% end %>
|
26
|
-
</table>
|
27
|
-
<%= partial :next_more, :start => start, :size => size, :per_page => 20 %>
|
28
|
-
<% else %>
|
29
|
-
|
30
|
-
<h1 class='wi'>Queues</h1>
|
31
|
-
<p class='intro'>The list below contains all the registered queues with the number of jobs currently in the queue. Select a queue from above to view all jobs currently pending on the queue.</p>
|
32
|
-
<table class='queues'>
|
33
|
-
<tr>
|
34
|
-
<th>Name</th>
|
35
|
-
<th>Jobs</th>
|
36
|
-
</tr>
|
37
|
-
<!-- only show nonzero length queues-->
|
38
|
-
<% resque.queues.select{|q| resque.size(q)>0}.sort_by { |q| q.to_s }.each do |queue| %>
|
39
|
-
<tr>
|
40
|
-
<td class='queue'><a class="queue" href="<%= u "queues/#{queue}" %>"><%= queue %></a></td>
|
41
|
-
<td class='size'><%= resque.size queue %></td>
|
42
|
-
</tr>
|
43
|
-
<% end %>
|
44
|
-
<% if failed_multiple_queues? %>
|
45
|
-
<% Resque::Failure.queues.sort_by { |q| q.to_s }.each_with_index do |queue, i| %>
|
46
|
-
<tr class="<%= Resque::Failure.count(queue).zero? ? "failed" : "failure" %><%= " first_failure" if i.zero? %>">
|
47
|
-
<td class='queue failed'><a class="queue" href="<%= u "failed/#{queue}" %>"><%= queue %></a></td>
|
48
|
-
<td class='size'><%= Resque::Failure.count(queue) %></td>
|
49
|
-
</tr>
|
50
|
-
<% end %>
|
51
|
-
<% else %>
|
52
|
-
<tr class="<%= Resque::Failure.count.zero? ? "failed" : "failure" %>">
|
53
|
-
<td class='queue failed'><a class="queue" href="<%= u :failed %>">failed</a></td>
|
54
|
-
<td class='size'><%= Resque::Failure.count %></td>
|
55
|
-
</tr>
|
56
|
-
<% end %>
|
57
|
-
</table>
|
58
|
-
|
59
|
-
<% end %>
|