mobilize-base 1.29 → 1.33

data/lib/mobilize-base/models/runner.rb

@@ -1,5 +1,6 @@
 module Mobilize
   class Runner
+    include Mobilize::RunnerHelper
     include Mongoid::Document
     include Mongoid::Timestamps
     field :path, type: String
@@ -11,20 +12,6 @@ module Mobilize
 
     index({ path: 1})
 
-    def headers
-      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
-    end
-
-    def title
-      r = self
-      r.path.split("/").first
-    end
-
-    def worker
-      r = self
-      Mobilize::Resque.find_worker_by_path(r.path)
-    end
-
     def Runner.find_by_path(path)
       Runner.where(:path=>path).first
     end
@@ -32,6 +19,7 @@ module Mobilize
     def Runner.find_by_title(title)
       Runner.where(:path=>"#{title}/jobs").first
     end
+
     def Runner.perform(id,*args)
       r = Runner.find_by_path(id)
       #get gdrive slot for read
@@ -41,12 +29,18 @@ module Mobilize
         return false
       end
       r.update_attributes(:started_at=>Time.now.utc)
-      #make sure any updates to activity are processed first
-      #as in when someone runs a "once" job that has completed
-      r.update_gsheet(gdrive_slot) 
-      #read the jobs in the gsheet and update models with news
-      r.read_gsheet(gdrive_slot)
-      #queue up the jobs that are due and active
+      begin
+        #make sure any updates to activity are processed first
+        #as in when someone runs a "once" job that has completed
+        r.update_gsheet(gdrive_slot) 
+        #read the jobs in the gsheet and update models with news
+        r.read_gsheet(gdrive_slot)
+        #queue up the jobs that are due and active
+      rescue => exc
+        #log the exception, but continue w job processing
+        #This ensures jobs are still processed if google drive goes down
+        r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
+      end
       r.jobs.each do |j|
         begin
           if j.is_due?
@@ -64,91 +58,32 @@ module Mobilize
       r.update_attributes(:completed_at=>Time.now.utc)
     end
 
-    def dataset
-      r = self
-      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
-    end
-
     def Runner.find_or_create_by_path(path)
       Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
     end
 
-    def gbook(gdrive_slot)
-      r = self
-      title = r.path.split("/").first
-      Gbook.find_by_path(title,gdrive_slot)
-    end
-
-    def gsheet(gdrive_slot)
-      r = self
-      u = r.user
-      jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
-      #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
-      unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
-        #only give the user edit permissions if they're the ones
-        #creating it
-        jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
-        unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
-          jobs_sheet.spreadsheet.update_acl(u.email,"writer")
-        end
-      end
-      jobs_sheet.add_headers(r.headers)
-      #add url to dataset
-      Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
-      begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
-      return jobs_sheet
-    end
-
     def read_gsheet(gdrive_slot)
       r = self
       #argument converts line breaks in cells to spaces
       gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
       #turn it into a hash array
-      gsheet_jobs = gsheet_tsv.tsv_to_hash_array
+      gsheet_hashes = gsheet_tsv.tsv_to_hash_array
       #go through each job, update relevant job with its params
       done_jobs = []
       #parse out the jobs and update the Job collection
-      gsheet_jobs.each_with_index do |rj,rj_i|
+      gsheet_hashes.each do |gsheet_hash|
         #skip non-jobs or jobs without required values
-        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
-        j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
-        #update top line params
-        j.update_attributes(:active => rj['active'],
-                            :trigger => rj['trigger'])
-        (1..5).to_a.each do |s_idx|
-          stage_string = rj["stage#{s_idx.to_s}"]
-          s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
-          if stage_string.to_s.length==0
-            #delete this stage and all stages after
-            if s
-              j = s.job
-              j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
-              #just in case
-              s.delete
-            end
-            break
-          elsif s.nil?
-            #create this stage
-            s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
-          end
-          #parse command string, update stage with it
-          s_handler, call, param_string = [""*3]
-          stage_string.split(" ").ie do |spls|
-            s_handler = spls.first.split(".").first
-            call = spls.first.split(".").last
-            param_string = spls[1..-1].join(" ").strip
-          end
-          s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
-        end
+        next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
+        j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
         r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
         #add this job to list of read ones
         done_jobs << j
       end
       #delete user jobs that are not included in Runner
-      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
-        j = Job.where(:path=>rj_path).first
+      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
+        j = Job.where(:path=>gsheet_hash_path).first
         j.delete if j
-        r.update_status("Deleted job:#{rj_path}")
+        r.update_status("Deleted job:#{gsheet_hash_path}")
       end
       r.update_status("jobs read at #{Time.now.utc}")
       return true
@@ -159,48 +94,13 @@ module Mobilize
       #there's nothing to update if runner has never had a completed at
       return false unless r.completed_at
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
+      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
       upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
       return true
     end
 
-    def jobs(jname=nil)
-      r = self
-      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
-      if jname
-        return js.sel{|j| j.name == jname}.first
-      else
-        return js
-      end
-    end
-
-    def user
-      r = self
-      user_name = r.path.split("_").second.split("(").first.split("/").first
-      User.where(:name=>user_name).first
-    end
-
-    def update_status(msg)
-      r = self
-      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
-      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
-      return true
-    end
-
-    def is_working?
-      r = self
-      Mobilize::Resque.active_paths.include?(r.path)
-    end
-
-    def is_due?
-      r = self.reload
-      return false if r.is_working?
-      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
-      return true if r.started_at.nil? or r.started_at < prev_due_time
-    end
-
     def enqueue!
       r = self
       ::Resque::Job.create("mobilize",Runner,r.path,{})

data/lib/mobilize-base/models/stage.rb

@@ -2,6 +2,7 @@ module Mobilize
   class Stage
     include Mongoid::Document
     include Mongoid::Timestamps
+    include Mobilize::StageHelper
     field :path, type: String
     field :handler, type: String
     field :call, type: String
@@ -16,40 +17,6 @@ module Mobilize
 
     index({ path: 1})
 
-    def idx
-      s = self
-      s.path.split("/").last.gsub("stage","").to_i
-    end
-
-    def out_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
-    end
-
-    def err_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
-    end
-
-    def params
-      s = self
-      p = YAML.easy_load(s.param_string)
-      raise "Must resolve to Hash" unless p.class==Hash
-      return p
-    end
-
-    def job
-      s = self
-      job_path = s.path.split("/")[0..-2].join("/")
-      Job.where(:path=>job_path).first
-    end
-
     def Stage.find_or_create_by_path(path)
       s = Stage.where(:path=>path).first
       s = Stage.create(:path=>path) unless s
@@ -77,6 +44,14 @@ module Mobilize
 
     def Stage.perform(id,*args)
       s = Stage.where(:path=>id).first
+      #check to make sure params are parsable
+      begin
+        param_hash = s.params
+        raise ScriptError if param_hash.class!=Hash
+      rescue StandardError, ScriptError
+        s.fail({'signal'=>500,
+                'err_str'=>"Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."})
+      end
       s.update_attributes(:started_at=>Time.now.utc)
       s.update_status(%{Starting at #{Time.now.utc}})
       #get response by running method
@@ -145,7 +120,11 @@ module Mobilize
       j = s.job
       r = j.runner
       u = r.user
-      j.update_attributes(:active=>false) unless s.params['always_on']
+      begin
+        j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
+      rescue StandardError, ScriptError
+        #skip due to parse error on params
+      end
       s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
       stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
       target_path =  (r.path.split("/")[0..-2] + [stage_name]).join("/")

data/lib/mobilize-base/tasks.rb

@@ -40,17 +40,17 @@ namespace :mobilize_base do
     Mobilize::Jobtracker.prep_workers
   end
   desc "Stop Jobtracker"
-  task :stop do
+  task :stop_jobtracker do
     require 'mobilize-base'
     Mobilize::Jobtracker.stop!
   end
   desc "Start Jobtracker"
-  task :start do
+  task :start_jobtracker do
     require 'mobilize-base'
     Mobilize::Jobtracker.start
   end
   desc "Restart Jobtracker"
-  task :restart do
+  task :restart_jobtracker do
     require 'mobilize-base'
     Mobilize::Jobtracker.restart!
   end
@@ -66,6 +66,19 @@ namespace :mobilize_base do
     resque_redis_port_args = if Mobilize::Base.env == 'test'
                                " -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
                              end.to_s
+    #determine view folder and override queues and working erbs
+    require 'resque/server'
+    view_dir = ::Resque::Server.views + "/"
+    old_queues_erb_path = view_dir + "queues.erb"
+    old_working_erb_path = view_dir + "working.erb"
+    gem_dir = Gem::Specification.find_by_name("mobilize-base").gem_dir
+    new_queues_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/queues.erb"
+    new_working_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/working.erb"
+    [old_queues_erb_path,old_working_erb_path].each{|p| File.delete(p) if File.exists?(p)}
+    require 'fileutils'
+    FileUtils.copy(new_queues_erb_path,old_queues_erb_path)
+    FileUtils.copy(new_working_erb_path,old_working_erb_path)
+    sleep 5 #give them time to die
     command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
     `#{command}`
   end

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.29"
+    VERSION = "1.33"
   end
 end

data/lib/mobilize-base.rb

@@ -60,14 +60,18 @@ module Mobilize
 end
 mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
 if File.exists?(mongoid_config_path)
-  require 'mongo'
   require 'mongoid'
+  require 'mongoid-grid_fs'
   Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
   require "mobilize-base/models/dataset"
   require "mobilize-base/models/user"
+  require "mobilize-base/helpers/runner_helper"
   require "mobilize-base/models/runner"
+  require "mobilize-base/helpers/job_helper"
   require "mobilize-base/models/job"
+  require "mobilize-base/helpers/stage_helper"
   require "mobilize-base/models/stage"
+
 end
 require 'google_drive'
 require 'resque'

data/lib/samples/gdrive.yml

@@ -4,8 +4,10 @@ development:
   owner:
     name: owner_development
     pw: google_drive_password
+  admin_group_name: admins_development
   admins:
     - name: admin
+  worker_group_name: workers_development
   workers:
     - name: worker_development001 
       pw: worker001_google_drive_password
@@ -16,8 +18,10 @@ test:
   owner:
     name: owner_test
     pw: google_drive_password
+  admin_group_name: admins_test
   admins:
     - name: admin
+  worker_group_name: workers_test
   workers:
     - name: worker_test001 
       pw: worker001_google_drive_password
@@ -28,8 +32,10 @@ production:
   owner:
     name: owner_production
     pw: google_drive_password
+  admin_group_name: admins_production
   admins:
     - name: admin
+  worker_group_name: workers_production
   workers:
     - name: worker_production001 
       pw: worker001_google_drive_password

data/lib/samples/gridfs.yml

@@ -1,10 +1,7 @@
 ---
 development:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 test:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 production:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB

data/lib/samples/gsheet.yml

@@ -1,17 +1,17 @@
 ---
 development:
-  max_cells: 400000 #current google limit is 400k cells per book
+  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 test:
-  max_cells: 400000 #current google limit is 400k cells per book
+  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 staging:
-  max_cells: 400000 #current google limit is 400k cells per book
+  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 production:
-  max_cells: 400000 #current google limit is 400k cells per book
+  max_cells: 50000 #current google limit is 400k cells per book but fails earlier
   read_date_format: "%Y-%m-%d" #format to record when reading sheets
   sheet_date_format: "%m/%d/%Y" #format to use to parse sheets

data/mobilize-base.gemspec

@@ -22,10 +22,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency 'rake'
-  s.add_runtime_dependency 'bson','1.6.1'
-  s.add_runtime_dependency 'bson_ext','1.6.1'
-  s.add_runtime_dependency 'mongo', '1.6.1'
+  s.add_runtime_dependency 'bson','1.8.4'
+  s.add_runtime_dependency 'bson_ext','1.8.4'
   s.add_runtime_dependency "mongoid", "~>3.0.0"
+  s.add_runtime_dependency 'mongoid-grid_fs'
   s.add_runtime_dependency 'resque', '1.24.0'
   s.add_runtime_dependency 'google_drive','0.3.2'
   s.add_runtime_dependency 'popen4','0.1.2'

data/test/base_job_rows.yml

@@ -3,9 +3,13 @@
   trigger: once
   status: ""
   stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
-
 - name: base2
   active: true
   trigger: after base1
   status: ""
   stage1: gsheet.write source:base1.out, target:base2.out
+- name: base3
+  active: true
+  trigger: after base2
+  status: ""
+  stage1: gsheet.write source:"base1."_"", target:base2.out

data/test/mobilize-base_test.rb

@@ -1,43 +1,34 @@
 require 'test_helper'
 
 describe "Mobilize" do
-
-  def before
-    puts 'nothing before'
-  end
-
-  # enqueues 4 workers on Resque
   it "runs integration test" do
 
     puts "restart test redis"
-    Mobilize::Jobtracker.restart_test_redis
+    TestHelper.restart_test_redis
 
     puts "clear out test db"
-    Mobilize::Jobtracker.drop_test_db
+    TestHelper.drop_test_db
 
     puts "restart workers"
     Mobilize::Jobtracker.restart_workers!
 
-    puts "build test runner"
-    gdrive_slot = Mobilize::Gdrive.owner_email
-    puts "create user 'mobilize'"
-    user_name = gdrive_slot.split("@").first
-    u = Mobilize::User.find_or_create_by_name(user_name)
-    assert u.email == gdrive_slot
+    u = TestHelper.owner_user
+    user_name = u.name
+    gdrive_slot = u.email
 
-    Mobilize::Jobtracker.build_test_runner(user_name)
+    puts "build test runner"
+    TestHelper.build_test_runner(user_name)
     assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
 
     puts "Jobtracker created runner with 'jobs' sheet?"
+
     r = u.runner
     jobs_sheet_url = "gsheet://#{r.path}"
     jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
     jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
     jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
-    assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
 
-    #stop Jobtracker, if you're doing this by queueing runners
-    #Mobilize::Jobtracker.stop!
+    assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
 
     puts "add base1 input file"
     test_filename = "test_base_1"
@@ -53,54 +44,22 @@ describe "Mobilize" do
     jobs_sheet.reload
     jobs_sheet.add_or_update_rows(test_job_rows)
     #wait for stages to complete
-    #r.enqueue!
-    wait_for_stages
+    TestHelper.wait_for_stages
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    test_target_sheet_1_url = "gsheet://#{r.title}/base1.out"
-    test_target_sheet_2_url = "gsheet://#{r.title}/base2.out"
-    test_error_sheet_url = "gsheet://#{r.title}/base1_stage1.err"
-
-    test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
-    test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
+    tsv_hash = {}
+    ["base1.out", "base2.out", "base3_stage1.err"].each do |sheet_name|
+      url = "gsheet://#{r.title}/#{sheet_name}"
+      data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
+      tsv_hash[sheet_name] = data
+    end
 
-    assert test_1_tsv == test_2_tsv
+    assert tsv_hash["base1.out"].to_s.length>0
+    assert tsv_hash["base2.out"] == tsv_hash["base1.out"]
 
-    puts "change first job to fail, wait for stages"
-    test_job_rows.first['stage1'] = %{gsheet.write source:"gfile://test_base_1.fail", target:base1.out, retries:3}
-    Mobilize::Dataset.write_by_url(test_error_sheet_url," ",user_name,gdrive_slot)
-    jobs_sheet.add_or_update_rows(test_job_rows)
+    base3_response = tsv_hash["base3_stage1.err"].tsv_to_hash_array.first['response']
+    assert base3_response == "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
 
-    #wait for stages to complete
-    wait_for_stages
-
-    test_error_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.err",gdrive_slot)
-    puts "jobtracker posted failing test error to sheet "
-    error_rows = test_error_sheet.read(user_name).tsv_to_hash_array
-    assert error_rows.first['response'] == "Could not get gfile://test_base_1.fail with error: unable to find test_base_1.fail"
     Mobilize::Jobtracker.stop!
   end
-
-  def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
-    time = 0
-    time_since_stage = 0
-    #check for 10 min
-    while time < time_limit and time_since_stage < stage_limit
-      sleep wait_length
-      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
-      if job_classes.include?("Mobilize::Stage")
-        time_since_stage = 0
-        puts "saw stage at #{time.to_s} seconds"
-      else
-        time_since_stage += wait_length
-        puts "#{time_since_stage.to_s} seconds since stage seen"
-      end
-      time += wait_length
-      puts "total wait time #{time.to_s} seconds"
-    end
-
-    if time >= time_limit
-      raise "Timed out before stage completion"
-    end
-  end
 end

data/test/test_helper.rb

@@ -8,3 +8,82 @@ $dir = File.dirname(File.expand_path(__FILE__))
 ENV['MOBILIZE_ENV'] = 'test'
 require 'mobilize-base'
 $TESTING = true
+module TestHelper
+  def TestHelper.wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
+    time = 0
+    time_since_stage = 0
+    #check for 10 min
+    while time < time_limit and time_since_stage < stage_limit
+      sleep wait_length
+      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
+      if job_classes.include?("Mobilize::Stage")
+        time_since_stage = 0
+        puts "saw stage at #{time.to_s} seconds"
+      else
+        time_since_stage += wait_length
+        puts "#{time_since_stage.to_s} seconds since stage seen"
+      end
+      time += wait_length
+      puts "total wait time #{time.to_s} seconds"
+    end
+
+    if time >= time_limit
+      raise "Timed out before stage completion"
+    end
+  end
+
+  #test methods
+  def TestHelper.restart_test_redis
+    TestHelper.stop_test_redis
+    if !system("which redis-server")
+      raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
+    end
+    "redis-server #{Mobilize::Base.root}/test/redis-test.conf".bash
+  end
+
+  def TestHelper.stop_test_redis
+    processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
+    pids = processes.map { |process| process.split(" ")[0] }
+    puts "Killing test redis server..."
+    pids.each { |pid| Process.kill("TERM", pid.to_i) }
+    puts "removing redis db dump file"
+    sleep 5
+    `rm -f #{Mobilize::Base.root}/test/dump.rdb #{Mobilize::Base.root}/test/dump-cluster.rdb`
+  end
+
+  def TestHelper.set_test_env
+    ENV['MOBILIZE_ENV']='test'
+    ::Resque.redis="localhost:9736"
+    mongoid_config_path = "#{Mobilize::Base.root}/config/mobilize/mongoid.yml"
+    Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
+  end
+
+  def TestHelper.drop_test_db
+    TestHelper.set_test_env
+    Mongoid.session(:default).collections.each do |collection| 
+      unless collection.name =~ /^system\./
+        collection.drop
+      end
+    end
+  end
+
+  def TestHelper.build_test_runner(user_name)
+    TestHelper.set_test_env
+    u = Mobilize::User.where(:name=>user_name).first
+    Mobilize::Jobtracker.update_status("delete old books and datasets")
+    # delete any old runner from previous test runs
+    gdrive_slot = Mobilize::Gdrive.owner_email
+    u.runner.gsheet(gdrive_slot).spreadsheet.delete
+    Mobilize::Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
+    Mobilize::Jobtracker.update_status("enqueue jobtracker, wait 45s")
+    Mobilize::Jobtracker.start
+    sleep 45
+  end
+
+  def TestHelper.owner_user
+    gdrive_slot = Mobilize::Gdrive.owner_email
+    puts "create user 'mobilize'"
+    user_name = gdrive_slot.split("@").first
+    return Mobilize::User.find_or_create_by_name(user_name)
+  end
+end