mobilize-base 1.29 → 1.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  module Mobilize
2
2
  class Runner
3
+ include Mobilize::RunnerHelper
3
4
  include Mongoid::Document
4
5
  include Mongoid::Timestamps
5
6
  field :path, type: String
@@ -11,20 +12,6 @@ module Mobilize
11
12
 
12
13
  index({ path: 1})
13
14
 
14
- def headers
15
- %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
16
- end
17
-
18
- def title
19
- r = self
20
- r.path.split("/").first
21
- end
22
-
23
- def worker
24
- r = self
25
- Mobilize::Resque.find_worker_by_path(r.path)
26
- end
27
-
28
15
  def Runner.find_by_path(path)
29
16
  Runner.where(:path=>path).first
30
17
  end
@@ -32,6 +19,7 @@ module Mobilize
32
19
  def Runner.find_by_title(title)
33
20
  Runner.where(:path=>"#{title}/jobs").first
34
21
  end
22
+
35
23
  def Runner.perform(id,*args)
36
24
  r = Runner.find_by_path(id)
37
25
  #get gdrive slot for read
@@ -41,12 +29,18 @@ module Mobilize
41
29
  return false
42
30
  end
43
31
  r.update_attributes(:started_at=>Time.now.utc)
44
- #make sure any updates to activity are processed first
45
- #as in when someone runs a "once" job that has completed
46
- r.update_gsheet(gdrive_slot)
47
- #read the jobs in the gsheet and update models with news
48
- r.read_gsheet(gdrive_slot)
49
- #queue up the jobs that are due and active
32
+ begin
33
+ #make sure any updates to activity are processed first
34
+ #as in when someone runs a "once" job that has completed
35
+ r.update_gsheet(gdrive_slot)
36
+ #read the jobs in the gsheet and update models with news
37
+ r.read_gsheet(gdrive_slot)
38
+ #queue up the jobs that are due and active
39
+ rescue => exc
40
+ #log the exception, but continue w job processing
41
+ #This ensures jobs are still processed if google drive goes down
42
+ r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
43
+ end
50
44
  r.jobs.each do |j|
51
45
  begin
52
46
  if j.is_due?
@@ -64,91 +58,32 @@ module Mobilize
64
58
  r.update_attributes(:completed_at=>Time.now.utc)
65
59
  end
66
60
 
67
- def dataset
68
- r = self
69
- Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
70
- end
71
-
72
61
  def Runner.find_or_create_by_path(path)
73
62
  Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
74
63
  end
75
64
 
76
- def gbook(gdrive_slot)
77
- r = self
78
- title = r.path.split("/").first
79
- Gbook.find_by_path(title,gdrive_slot)
80
- end
81
-
82
- def gsheet(gdrive_slot)
83
- r = self
84
- u = r.user
85
- jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
86
- #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
87
- unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
88
- #only give the user edit permissions if they're the ones
89
- #creating it
90
- jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
91
- unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
92
- jobs_sheet.spreadsheet.update_acl(u.email,"writer")
93
- end
94
- end
95
- jobs_sheet.add_headers(r.headers)
96
- #add url to dataset
97
- Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
98
- begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
99
- return jobs_sheet
100
- end
101
-
102
65
  def read_gsheet(gdrive_slot)
103
66
  r = self
104
67
  #argument converts line breaks in cells to spaces
105
68
  gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
106
69
  #turn it into a hash array
107
- gsheet_jobs = gsheet_tsv.tsv_to_hash_array
70
+ gsheet_hashes = gsheet_tsv.tsv_to_hash_array
108
71
  #go through each job, update relevant job with its params
109
72
  done_jobs = []
110
73
  #parse out the jobs and update the Job collection
111
- gsheet_jobs.each_with_index do |rj,rj_i|
74
+ gsheet_hashes.each do |gsheet_hash|
112
75
  #skip non-jobs or jobs without required values
113
- next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
114
- j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
115
- #update top line params
116
- j.update_attributes(:active => rj['active'],
117
- :trigger => rj['trigger'])
118
- (1..5).to_a.each do |s_idx|
119
- stage_string = rj["stage#{s_idx.to_s}"]
120
- s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
121
- if stage_string.to_s.length==0
122
- #delete this stage and all stages after
123
- if s
124
- j = s.job
125
- j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
126
- #just in case
127
- s.delete
128
- end
129
- break
130
- elsif s.nil?
131
- #create this stage
132
- s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
133
- end
134
- #parse command string, update stage with it
135
- s_handler, call, param_string = [""*3]
136
- stage_string.split(" ").ie do |spls|
137
- s_handler = spls.first.split(".").first
138
- call = spls.first.split(".").last
139
- param_string = spls[1..-1].join(" ").strip
140
- end
141
- s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
142
- end
76
+ next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
77
+ j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
143
78
  r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
144
79
  #add this job to list of read ones
145
80
  done_jobs << j
146
81
  end
147
82
  #delete user jobs that are not included in Runner
148
- (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
149
- j = Job.where(:path=>rj_path).first
83
+ (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
84
+ j = Job.where(:path=>gsheet_hash_path).first
150
85
  j.delete if j
151
- r.update_status("Deleted job:#{rj_path}")
86
+ r.update_status("Deleted job:#{gsheet_hash_path}")
152
87
  end
153
88
  r.update_status("jobs read at #{Time.now.utc}")
154
89
  return true
@@ -159,48 +94,13 @@ module Mobilize
159
94
  #there's nothing to update if runner has never had a completed at
160
95
  return false unless r.completed_at
161
96
  jobs_gsheet = r.gsheet(gdrive_slot)
162
- upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
97
+ upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
163
98
  upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
164
99
  jobs_gsheet.add_or_update_rows(upd_rows)
165
100
  r.update_status("gsheet updated")
166
101
  return true
167
102
  end
168
103
 
169
- def jobs(jname=nil)
170
- r = self
171
- js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
172
- if jname
173
- return js.sel{|j| j.name == jname}.first
174
- else
175
- return js
176
- end
177
- end
178
-
179
- def user
180
- r = self
181
- user_name = r.path.split("_").second.split("(").first.split("/").first
182
- User.where(:name=>user_name).first
183
- end
184
-
185
- def update_status(msg)
186
- r = self
187
- r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
188
- Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
189
- return true
190
- end
191
-
192
- def is_working?
193
- r = self
194
- Mobilize::Resque.active_paths.include?(r.path)
195
- end
196
-
197
- def is_due?
198
- r = self.reload
199
- return false if r.is_working?
200
- prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
201
- return true if r.started_at.nil? or r.started_at < prev_due_time
202
- end
203
-
204
104
  def enqueue!
205
105
  r = self
206
106
  ::Resque::Job.create("mobilize",Runner,r.path,{})
@@ -2,6 +2,7 @@ module Mobilize
2
2
  class Stage
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::StageHelper
5
6
  field :path, type: String
6
7
  field :handler, type: String
7
8
  field :call, type: String
@@ -16,40 +17,6 @@ module Mobilize
16
17
 
17
18
  index({ path: 1})
18
19
 
19
- def idx
20
- s = self
21
- s.path.split("/").last.gsub("stage","").to_i
22
- end
23
-
24
- def out_dst
25
- #this gives a dataset that points to the output
26
- #allowing you to determine its size
27
- #before committing to a read or write
28
- s = self
29
- Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
30
- end
31
-
32
- def err_dst
33
- #this gives a dataset that points to the output
34
- #allowing you to determine its size
35
- #before committing to a read or write
36
- s = self
37
- Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
38
- end
39
-
40
- def params
41
- s = self
42
- p = YAML.easy_load(s.param_string)
43
- raise "Must resolve to Hash" unless p.class==Hash
44
- return p
45
- end
46
-
47
- def job
48
- s = self
49
- job_path = s.path.split("/")[0..-2].join("/")
50
- Job.where(:path=>job_path).first
51
- end
52
-
53
20
  def Stage.find_or_create_by_path(path)
54
21
  s = Stage.where(:path=>path).first
55
22
  s = Stage.create(:path=>path) unless s
@@ -77,6 +44,14 @@ module Mobilize
77
44
 
78
45
  def Stage.perform(id,*args)
79
46
  s = Stage.where(:path=>id).first
47
+ #check to make sure params are parsable
48
+ begin
49
+ param_hash = s.params
50
+ raise ScriptError if param_hash.class!=Hash
51
+ rescue StandardError, ScriptError
52
+ s.fail({'signal'=>500,
53
+ 'err_str'=>"Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."})
54
+ end
80
55
  s.update_attributes(:started_at=>Time.now.utc)
81
56
  s.update_status(%{Starting at #{Time.now.utc}})
82
57
  #get response by running method
@@ -145,7 +120,11 @@ module Mobilize
145
120
  j = s.job
146
121
  r = j.runner
147
122
  u = r.user
148
- j.update_attributes(:active=>false) unless s.params['always_on']
123
+ begin
124
+ j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
125
+ rescue StandardError, ScriptError
126
+ #skip due to parse error on params
127
+ end
149
128
  s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
150
129
  stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
151
130
  target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
@@ -40,17 +40,17 @@ namespace :mobilize_base do
40
40
  Mobilize::Jobtracker.prep_workers
41
41
  end
42
42
  desc "Stop Jobtracker"
43
- task :stop do
43
+ task :stop_jobtracker do
44
44
  require 'mobilize-base'
45
45
  Mobilize::Jobtracker.stop!
46
46
  end
47
47
  desc "Start Jobtracker"
48
- task :start do
48
+ task :start_jobtracker do
49
49
  require 'mobilize-base'
50
50
  Mobilize::Jobtracker.start
51
51
  end
52
52
  desc "Restart Jobtracker"
53
- task :restart do
53
+ task :restart_jobtracker do
54
54
  require 'mobilize-base'
55
55
  Mobilize::Jobtracker.restart!
56
56
  end
@@ -66,6 +66,19 @@ namespace :mobilize_base do
66
66
  resque_redis_port_args = if Mobilize::Base.env == 'test'
67
67
  " -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
68
68
  end.to_s
69
+ #determine view folder and override queues and working erbs
70
+ require 'resque/server'
71
+ view_dir = ::Resque::Server.views + "/"
72
+ old_queues_erb_path = view_dir + "queues.erb"
73
+ old_working_erb_path = view_dir + "working.erb"
74
+ gem_dir = Gem::Specification.find_by_name("mobilize-base").gem_dir
75
+ new_queues_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/queues.erb"
76
+ new_working_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/working.erb"
77
+ [old_queues_erb_path,old_working_erb_path].each{|p| File.delete(p) if File.exists?(p)}
78
+ require 'fileutils'
79
+ FileUtils.copy(new_queues_erb_path,old_queues_erb_path)
80
+ FileUtils.copy(new_working_erb_path,old_working_erb_path)
81
+ sleep 5 #give them time to die
69
82
  command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
70
83
  `#{command}`
71
84
  end
@@ -1,5 +1,5 @@
1
1
  module Mobilize
2
2
  module Base
3
- VERSION = "1.29"
3
+ VERSION = "1.33"
4
4
  end
5
5
  end
data/lib/mobilize-base.rb CHANGED
@@ -60,14 +60,18 @@ module Mobilize
60
60
  end
61
61
  mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
62
62
  if File.exists?(mongoid_config_path)
63
- require 'mongo'
64
63
  require 'mongoid'
64
+ require 'mongoid-grid_fs'
65
65
  Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
66
66
  require "mobilize-base/models/dataset"
67
67
  require "mobilize-base/models/user"
68
+ require "mobilize-base/helpers/runner_helper"
68
69
  require "mobilize-base/models/runner"
70
+ require "mobilize-base/helpers/job_helper"
69
71
  require "mobilize-base/models/job"
72
+ require "mobilize-base/helpers/stage_helper"
70
73
  require "mobilize-base/models/stage"
74
+
71
75
  end
72
76
  require 'google_drive'
73
77
  require 'resque'
@@ -4,8 +4,10 @@ development:
4
4
  owner:
5
5
  name: owner_development
6
6
  pw: google_drive_password
7
+ admin_group_name: admins_development
7
8
  admins:
8
9
  - name: admin
10
+ worker_group_name: workers_development
9
11
  workers:
10
12
  - name: worker_development001
11
13
  pw: worker001_google_drive_password
@@ -16,8 +18,10 @@ test:
16
18
  owner:
17
19
  name: owner_test
18
20
  pw: google_drive_password
21
+ admin_group_name: admins_test
19
22
  admins:
20
23
  - name: admin
24
+ worker_group_name: workers_test
21
25
  workers:
22
26
  - name: worker_test001
23
27
  pw: worker001_google_drive_password
@@ -28,8 +32,10 @@ production:
28
32
  owner:
29
33
  name: owner_production
30
34
  pw: google_drive_password
35
+ admin_group_name: admins_production
31
36
  admins:
32
37
  - name: admin
38
+ worker_group_name: workers_production
33
39
  workers:
34
40
  - name: worker_production001
35
41
  pw: worker001_google_drive_password
@@ -1,10 +1,7 @@
1
1
  ---
2
2
  development:
3
- max_versions: 10 #number of versions of cache to keep in gridfs
4
3
  max_compressed_write_size: 1000000000 #~1GB
5
4
  test:
6
- max_versions: 10 #number of versions of cache to keep in gridfs
7
5
  max_compressed_write_size: 1000000000 #~1GB
8
6
  production:
9
- max_versions: 10 #number of versions of cache to keep in gridfs
10
7
  max_compressed_write_size: 1000000000 #~1GB
@@ -1,17 +1,17 @@
1
1
  ---
2
2
  development:
3
- max_cells: 400000 #current google limit is 400k cells per book
3
+ max_cells: 50000 #current google limit is 400k cells per book but fails earlier
4
4
  read_date_format: "%Y-%m-%d" #format to record when reading sheets
5
5
  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
6
6
  test:
7
- max_cells: 400000 #current google limit is 400k cells per book
7
+ max_cells: 50000 #current google limit is 400k cells per book but fails earlier
8
8
  read_date_format: "%Y-%m-%d" #format to record when reading sheets
9
9
  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
10
10
  staging:
11
- max_cells: 400000 #current google limit is 400k cells per book
11
+ max_cells: 50000 #current google limit is 400k cells per book but fails earlier
12
12
  read_date_format: "%Y-%m-%d" #format to record when reading sheets
13
13
  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
14
14
  production:
15
- max_cells: 400000 #current google limit is 400k cells per book
15
+ max_cells: 50000 #current google limit is 400k cells per book but fails earlier
16
16
  read_date_format: "%Y-%m-%d" #format to record when reading sheets
17
17
  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
@@ -22,10 +22,10 @@ Gem::Specification.new do |s|
22
22
  s.require_paths = ["lib"]
23
23
 
24
24
  s.add_runtime_dependency 'rake'
25
- s.add_runtime_dependency 'bson','1.6.1'
26
- s.add_runtime_dependency 'bson_ext','1.6.1'
27
- s.add_runtime_dependency 'mongo', '1.6.1'
25
+ s.add_runtime_dependency 'bson','1.8.4'
26
+ s.add_runtime_dependency 'bson_ext','1.8.4'
28
27
  s.add_runtime_dependency "mongoid", "~>3.0.0"
28
+ s.add_runtime_dependency 'mongoid-grid_fs'
29
29
  s.add_runtime_dependency 'resque', '1.24.0'
30
30
  s.add_runtime_dependency 'google_drive','0.3.2'
31
31
  s.add_runtime_dependency 'popen4','0.1.2'
@@ -3,9 +3,13 @@
3
3
  trigger: once
4
4
  status: ""
5
5
  stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
6
-
7
6
  - name: base2
8
7
  active: true
9
8
  trigger: after base1
10
9
  status: ""
11
10
  stage1: gsheet.write source:base1.out, target:base2.out
11
+ - name: base3
12
+ active: true
13
+ trigger: after base2
14
+ status: ""
15
+ stage1: gsheet.write source:"base1."_"", target:base2.out
@@ -1,43 +1,34 @@
1
1
  require 'test_helper'
2
2
 
3
3
  describe "Mobilize" do
4
-
5
- def before
6
- puts 'nothing before'
7
- end
8
-
9
- # enqueues 4 workers on Resque
10
4
  it "runs integration test" do
11
5
 
12
6
  puts "restart test redis"
13
- Mobilize::Jobtracker.restart_test_redis
7
+ TestHelper.restart_test_redis
14
8
 
15
9
  puts "clear out test db"
16
- Mobilize::Jobtracker.drop_test_db
10
+ TestHelper.drop_test_db
17
11
 
18
12
  puts "restart workers"
19
13
  Mobilize::Jobtracker.restart_workers!
20
14
 
21
- puts "build test runner"
22
- gdrive_slot = Mobilize::Gdrive.owner_email
23
- puts "create user 'mobilize'"
24
- user_name = gdrive_slot.split("@").first
25
- u = Mobilize::User.find_or_create_by_name(user_name)
26
- assert u.email == gdrive_slot
15
+ u = TestHelper.owner_user
16
+ user_name = u.name
17
+ gdrive_slot = u.email
27
18
 
28
- Mobilize::Jobtracker.build_test_runner(user_name)
19
+ puts "build test runner"
20
+ TestHelper.build_test_runner(user_name)
29
21
  assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
30
22
 
31
23
  puts "Jobtracker created runner with 'jobs' sheet?"
24
+
32
25
  r = u.runner
33
26
  jobs_sheet_url = "gsheet://#{r.path}"
34
27
  jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
35
28
  jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
36
29
  jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
37
- assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
38
30
 
39
- #stop Jobtracker, if you're doing this by queueing runners
40
- #Mobilize::Jobtracker.stop!
31
+ assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
41
32
 
42
33
  puts "add base1 input file"
43
34
  test_filename = "test_base_1"
@@ -53,54 +44,22 @@ describe "Mobilize" do
53
44
  jobs_sheet.reload
54
45
  jobs_sheet.add_or_update_rows(test_job_rows)
55
46
  #wait for stages to complete
56
- #r.enqueue!
57
- wait_for_stages
47
+ TestHelper.wait_for_stages
58
48
 
59
49
  puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
60
- test_target_sheet_1_url = "gsheet://#{r.title}/base1.out"
61
- test_target_sheet_2_url = "gsheet://#{r.title}/base2.out"
62
- test_error_sheet_url = "gsheet://#{r.title}/base1_stage1.err"
63
-
64
- test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
65
- test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
50
+ tsv_hash = {}
51
+ ["base1.out", "base2.out", "base3_stage1.err"].each do |sheet_name|
52
+ url = "gsheet://#{r.title}/#{sheet_name}"
53
+ data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
54
+ tsv_hash[sheet_name] = data
55
+ end
66
56
 
67
- assert test_1_tsv == test_2_tsv
57
+ assert tsv_hash["base1.out"].to_s.length>0
58
+ assert tsv_hash["base2.out"] == tsv_hash["base1.out"]
68
59
 
69
- puts "change first job to fail, wait for stages"
70
- test_job_rows.first['stage1'] = %{gsheet.write source:"gfile://test_base_1.fail", target:base1.out, retries:3}
71
- Mobilize::Dataset.write_by_url(test_error_sheet_url," ",user_name,gdrive_slot)
72
- jobs_sheet.add_or_update_rows(test_job_rows)
60
+ base3_response = tsv_hash["base3_stage1.err"].tsv_to_hash_array.first['response']
61
+ assert base3_response == "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
73
62
 
74
- #wait for stages to complete
75
- wait_for_stages
76
-
77
- test_error_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.err",gdrive_slot)
78
- puts "jobtracker posted failing test error to sheet "
79
- error_rows = test_error_sheet.read(user_name).tsv_to_hash_array
80
- assert error_rows.first['response'] == "Could not get gfile://test_base_1.fail with error: unable to find test_base_1.fail"
81
63
  Mobilize::Jobtracker.stop!
82
64
  end
83
-
84
- def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
85
- time = 0
86
- time_since_stage = 0
87
- #check for 10 min
88
- while time < time_limit and time_since_stage < stage_limit
89
- sleep wait_length
90
- job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
91
- if job_classes.include?("Mobilize::Stage")
92
- time_since_stage = 0
93
- puts "saw stage at #{time.to_s} seconds"
94
- else
95
- time_since_stage += wait_length
96
- puts "#{time_since_stage.to_s} seconds since stage seen"
97
- end
98
- time += wait_length
99
- puts "total wait time #{time.to_s} seconds"
100
- end
101
-
102
- if time >= time_limit
103
- raise "Timed out before stage completion"
104
- end
105
- end
106
65
  end
data/test/test_helper.rb CHANGED
@@ -8,3 +8,82 @@ $dir = File.dirname(File.expand_path(__FILE__))
8
8
  ENV['MOBILIZE_ENV'] = 'test'
9
9
  require 'mobilize-base'
10
10
  $TESTING = true
11
+ module TestHelper
12
+ def TestHelper.wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
13
+ time = 0
14
+ time_since_stage = 0
15
+ #check for 10 min
16
+ while time < time_limit and time_since_stage < stage_limit
17
+ sleep wait_length
18
+ job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
19
+ if job_classes.include?("Mobilize::Stage")
20
+ time_since_stage = 0
21
+ puts "saw stage at #{time.to_s} seconds"
22
+ else
23
+ time_since_stage += wait_length
24
+ puts "#{time_since_stage.to_s} seconds since stage seen"
25
+ end
26
+ time += wait_length
27
+ puts "total wait time #{time.to_s} seconds"
28
+ end
29
+
30
+ if time >= time_limit
31
+ raise "Timed out before stage completion"
32
+ end
33
+ end
34
+
35
+ #test methods
36
+ def TestHelper.restart_test_redis
37
+ TestHelper.stop_test_redis
38
+ if !system("which redis-server")
39
+ raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
40
+ end
41
+ "redis-server #{Mobilize::Base.root}/test/redis-test.conf".bash
42
+ end
43
+
44
+ def TestHelper.stop_test_redis
45
+ processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
46
+ pids = processes.map { |process| process.split(" ")[0] }
47
+ puts "Killing test redis server..."
48
+ pids.each { |pid| Process.kill("TERM", pid.to_i) }
49
+ puts "removing redis db dump file"
50
+ sleep 5
51
+ `rm -f #{Mobilize::Base.root}/test/dump.rdb #{Mobilize::Base.root}/test/dump-cluster.rdb`
52
+ end
53
+
54
+ def TestHelper.set_test_env
55
+ ENV['MOBILIZE_ENV']='test'
56
+ ::Resque.redis="localhost:9736"
57
+ mongoid_config_path = "#{Mobilize::Base.root}/config/mobilize/mongoid.yml"
58
+ Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
59
+ end
60
+
61
+ def TestHelper.drop_test_db
62
+ TestHelper.set_test_env
63
+ Mongoid.session(:default).collections.each do |collection|
64
+ unless collection.name =~ /^system\./
65
+ collection.drop
66
+ end
67
+ end
68
+ end
69
+
70
+ def TestHelper.build_test_runner(user_name)
71
+ TestHelper.set_test_env
72
+ u = Mobilize::User.where(:name=>user_name).first
73
+ Mobilize::Jobtracker.update_status("delete old books and datasets")
74
+ # delete any old runner from previous test runs
75
+ gdrive_slot = Mobilize::Gdrive.owner_email
76
+ u.runner.gsheet(gdrive_slot).spreadsheet.delete
77
+ Mobilize::Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
78
+ Mobilize::Jobtracker.update_status("enqueue jobtracker, wait 45s")
79
+ Mobilize::Jobtracker.start
80
+ sleep 45
81
+ end
82
+
83
+ def TestHelper.owner_user
84
+ gdrive_slot = Mobilize::Gdrive.owner_email
85
+ puts "create user 'mobilize'"
86
+ user_name = gdrive_slot.split("@").first
87
+ return Mobilize::User.find_or_create_by_name(user_name)
88
+ end
89
+ end