mobilize-base 1.298 → 1.351

data/README.md

@@ -162,9 +162,15 @@ the same domain, and all Users should have emails in this domain.
 * an owner name and password. You can set up separate owners
   for different environments as in the below file, which will keep your
 mission critical workers from getting rate-limit errors.
+* one admin_group_name, which the owner and all admins should be added to -- this
+group will need read permissions to read from and edit permissions to write
+to files.
 * one or more admins with email attributes -- these will be for people
   who should be given write permissions to all Mobilize books in the
 environment for maintenance purposes.
+* one worker_group_name, which the owner and all workers should be added to -- this
+group will need read permissions to read from and edit permissions to write
+to files.
 * one or more workers with name and pw attributes -- they will be used
   to queue up google reads and writes. This can be the same as the owner
 account for testing purposes or low-volume environments. 
@@ -182,8 +188,10 @@ development:
   owner:
     name: owner_development
     pw: google_drive_password
+  admin_group_name: admins_development
   admins:
     - name: admin
+  worker_group_name: workers_development
   workers:
     - name: worker_development001 
       pw: worker001_google_drive_password
@@ -194,8 +202,10 @@ test:
   owner:
     name: owner_test
     pw: google_drive_password
+  admin_group_name: admins_test
   admins:
     - name: admin
+  worker_group_name: workers_test
   workers:
     - name: worker_test001 
       pw: worker001_google_drive_password
@@ -206,8 +216,10 @@ production:
   owner:
     name: owner_production
     pw: google_drive_password
+  admin_group_name: admins_production
   admins:
     - name: admin
+  worker_group_name: workers_production
   workers:
     - name: worker_production001 
       pw: worker001_google_drive_password
@@ -266,6 +278,9 @@ The Jobtracker sits on your Resque and does 2 things:
 
 Emails are sent using ActionMailer, through the owner Google Drive
 account.
+  * errors are sent to the owner of the job/stage as well as the admin group.
+  * errors not specific to a job/stage are sent to the admin group only, as
+    given in gdrive.yml
 
 To this end, it needs these parameters, for which there is a sample
 below and in the [lib/samples][git_samples] folder:
@@ -278,24 +293,18 @@ development:
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
-  admins: #emails to send notifications to
-  - email: admin@host.com
 test:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
-  admins: #emails to send notifications to
-  - email: admin@host.com
 production:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
-  admins: #emails to send notifications to
-  - email: admin@host.com
 ```
 
 <a name='section_Configure_Resque'></a>

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -3,42 +3,16 @@ module GoogleDrive
 
     def add_worker_acl
       f = self
-      return true if f.has_worker_acl?
-      Mobilize::Gdrive.worker_emails.each do |a| 
-        f.update_acl(a)
-      end
+      email = "#{Mobilize::Gdrive.worker_group_name}@#{Mobilize::Gdrive.domain}"
+      f.update_acl(email,"group")
     end
 
     def add_admin_acl
       f = self
-      #admin includes workers
-      return true if f.has_admin_acl?
-      accounts = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).uniq
-      accounts.each do |email|
-        f.update_acl(email)
-      end
-    end
-
-    def has_admin_acl?
-      f = self
-      curr_emails = f.acls.map{|a| a.scope}.compact.sort
-      admin_emails = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).uniq
-      if curr_emails == admin_emails or (curr_emails & admin_emails) == admin_emails
-        return true
-      else
-        return false
-      end
-    end
-
-    def has_worker_acl?
-      f = self
-      curr_emails = f.acls.map{|a| a.scope}.compact.sort
-      worker_emails = Mobilize::Gdrive.worker_emails.sort
-      if curr_emails == worker_emails or (curr_emails & worker_emails) == worker_emails
-        return true
-      else
-        return false
-      end
+      email = "#{Mobilize::Gdrive.admin_group_name}@#{Mobilize::Gdrive.domain}"
+      f.update_acl(email,"group")
+      #if adding acl ,must currently add workers as well
+      f.add_worker_acl
     end
 
     def read(user_name)
@@ -51,7 +25,7 @@ module GoogleDrive
       end
     end
 
-    def update_acl(email,role="writer")
+    def update_acl(email,scope_type="user",role="writer")
       f = self
       #need these flags for HTTP retries
       #create req_acl hash to add to current acl
@@ -64,16 +38,16 @@ module GoogleDrive
           if entry.role != role
             #for whatever reason
             f.acl.delete(entry)
-            f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
+            f.acl.push({:scope_type=>scope_type,:scope=>email,:role=>role})
           end
         elsif !['reader','writer','owner'].include?(role)
           raise "Invalid role #{role}"
         end
       else
         begin
-          f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
+          f.acl.push({:scope_type=>scope_type,:scope=>email,:role=>role})
         rescue => exc
-          raise exc unless exc.to_s.index("user already has access")
+          raise exc unless exc.to_s.index("already has access")
         end
       end
       return true

data/lib/mobilize-base/handlers/gdrive.rb

@@ -37,12 +37,16 @@ module Mobilize
       end
     end
 
-    def Gdrive.worker_emails
-      Gdrive.workers.map{|w| [w['name'],Gdrive.domain].join("@")}
+    def Gdrive.worker_group_name
+      Gdrive.config['worker_group_name']
+    end
+
+    def Gdrive.admin_group_name
+      Gdrive.config['admin_group_name']
     end
 
-    def Gdrive.admin_emails
-      Gdrive.admins.map{|w| [w['name'],Gdrive.domain].join("@")}
+    def Gdrive.worker_emails
+      Gdrive.workers.map{|w| [w['name'],Gdrive.domain].join("@")}
     end
 
     #email management - used to make sure not too many emails get used at the same time

data/lib/mobilize-base/handlers/gfile.rb

@@ -58,7 +58,7 @@ module Mobilize
     def Gfile.update_acl_by_path(path,gdrive_slot,role="writer",target_email=nil)
       file = Gfile.find_by_path(path,target_email)
       raise "File #{path} not found" unless file
-      file.update_acl(gdrive_slot,role)
+      file.update_acl(gdrive_slot,"user",role)
     end
 
     def Gfile.find_by_path(path)

data/lib/mobilize-base/handlers/gsheet.rb

@@ -109,7 +109,7 @@ module Mobilize
         #only give the user edit permissions if they're the ones
         #creating it
         target_sheet = Gsheet.find_or_create_by_path(target_path,gdrive_slot)
-        target_sheet.spreadsheet.update_acl(u.email,"writer") unless target_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
+        target_sheet.spreadsheet.update_acl(u.email) unless target_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
         target_sheet.delete_sheet1
       end
       #pass it crop param to determine whether to shrink target sheet to fit data

data/lib/mobilize-base/handlers/resque.rb

@@ -19,13 +19,17 @@ module Mobilize
     def Resque.workers(state="all")
       workers = ::Resque.workers.select{|w| w.queues.first == Resque.queue_name}
       return workers if state == 'all'
-      working_workers = workers.select{|w| w.job['queue']== Resque.queue_name}
+      working_workers = workers.select{|w| w.job['queue'] == Resque.queue_name}
       return working_workers if state == 'working'
       idle_workers = workers.select{|w| w.job['queue'].nil?}
       return idle_workers if state == 'idle'
       stale_workers = workers.select{|w| Time.parse(w.started) < Jobtracker.deployed_at}
       return stale_workers if state == 'stale'
-      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)}
+      timeout_workers = workers.select do |w|
+                                         w.job['payload'] and
+                                         w.job['payload']['class']!='Jobtracker' and
+                                         w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)
+                                       end
       return timeout_workers if state == 'timeout'
       raise "invalid state #{state}"
     end
@@ -115,14 +119,14 @@ module Mobilize
                   s = Stage.where(:path=>stage_path).first
                   if s.params['notify'].to_s=="false"
                     next
-                  elsif s.params['notify'].index("@")
+                  elsif s.params['notify'].to_s.index("@")
                     s.params['notify']
                   else
                     s.job.runner.user.email
                   end
-                rescue
-                  #jobs without stages are sent to first admin
-                  Jobtracker.admin_emails.first
+                rescue ScriptError, StandardError
+                  #jobs without stages are sent to admins
+                  [Gdrive.admin_group_name,Gdrive.domain].join("@")
                 end
         exc_to_s = f['error']
         if fjobs[email].nil?

data/lib/mobilize-base/helpers/jobtracker_helper.rb

@@ -0,0 +1,143 @@
+module Mobilize
+  module Jobtracker
+    def Jobtracker.config
+      Base.config('jobtracker')
+    end
+
+    #modify this to increase the frequency of request cycles
+    def Jobtracker.cycle_freq
+      Jobtracker.config['cycle_freq']
+    end
+
+    #frequency of notifications
+    def Jobtracker.notification_freq
+      Jobtracker.config['notification_freq']
+    end
+
+    def Jobtracker.runner_read_freq
+      Jobtracker.config['runner_read_freq']
+    end
+
+    #long running tolerance
+    def Jobtracker.max_run_time
+      Jobtracker.config['max_run_time']
+    end
+
+    def Jobtracker.admins
+      Jobtracker.config['admins']
+    end
+
+    def Jobtracker.worker
+      Resque.find_worker_by_path("jobtracker")
+    end
+
+    def Jobtracker.workers(state="all")
+      Resque.workers(state)
+    end
+
+    def Jobtracker.status
+      args = Jobtracker.get_args
+      return args['status'] if args
+      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
+      return 'queued' if job
+      return 'stopped'
+    end
+
+    def Jobtracker.update_status(msg)
+      #this is to keep jobtracker from resisting stop commands
+      return false if Jobtracker.status=="stopping"
+      #Jobtracker has no persistent database state
+      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
+      return true
+    end
+
+    def Jobtracker.restart
+      Jobtracker.stop!
+      Jobtracker.start
+    end
+
+    def Jobtracker.set_args(args)
+      Resque.set_worker_args(Jobtracker.worker,args)
+      return true
+    end
+
+    def Jobtracker.get_args
+      Resque.get_worker_args(Jobtracker.worker)
+    end
+
+    def Jobtracker.kill_workers
+      Resque.kill_workers
+    end
+
+    def Jobtracker.kill_idle_workers
+      Resque.kill_idle_workers
+    end
+
+    def Jobtracker.kill_idle_and_stale_workers
+      Resque.kill_idle_and_stale_workers
+    end
+
+    def Jobtracker.prep_workers
+      Resque.prep_workers
+    end
+
+    def Jobtracker.failures
+      Resque.failures
+    end
+
+    def Jobtracker.start
+      if Jobtracker.status!='stopped'
+        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
+      else
+        #make sure that workers are running and at the right number
+        #Resque.prep_workers
+        #queue up the jobtracker (starts the perform method)
+        Jobtracker.enqueue!
+      end
+      return true
+    end
+
+    def Jobtracker.enqueue!
+      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
+    end
+
+    def Jobtracker.restart!
+      Jobtracker.stop!
+      Jobtracker.start
+      return true
+    end
+
+    def Jobtracker.restart_workers!
+      Jobtracker.kill_workers
+      sleep 10
+      Jobtracker.prep_workers
+      Jobtracker.update_status("put workers back on the queue")
+    end
+
+    def Jobtracker.stop!
+      #send signal for Jobtracker to check for
+      Jobtracker.update_status('stopping')
+      sleep 5
+      i=0
+      while Jobtracker.status=='stopping'
+        puts "#{Jobtracker.to_s} still on queue, waiting"
+        sleep 5
+        i+=1
+      end
+      return true
+    end
+
+    def Jobtracker.last_notification
+      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
+    end
+
+    def Jobtracker.last_notification=(time)
+      Jobtracker.set_args({"last_notification"=>time})
+    end
+
+    def Jobtracker.notif_due?
+      last_duetime = Time.now.utc - Jobtracker.notification_freq
+      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
+    end
+  end
+end

data/lib/mobilize-base/helpers/runner_helper.rb

@@ -36,7 +36,7 @@ module Mobilize
         #creating it
         jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
         unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
-          jobs_sheet.spreadsheet.update_acl(u.email,"writer")
+          jobs_sheet.spreadsheet.update_acl(u.email)
         end
         jobs_sheet.add_headers(r.headers)
         begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails

data/lib/mobilize-base/jobtracker.rb

@@ -1,148 +1,7 @@
 module Mobilize
   module Jobtracker
-    def Jobtracker.config
-      Base.config('jobtracker')
-    end
-
-    #modify this to increase the frequency of request cycles
-    def Jobtracker.cycle_freq
-      Jobtracker.config['cycle_freq']
-    end
-
-    #frequency of notifications
-    def Jobtracker.notification_freq
-      Jobtracker.config['notification_freq']
-    end
-
-    def Jobtracker.runner_read_freq
-      Jobtracker.config['runner_read_freq']
-    end
-
-    #long running tolerance
-    def Jobtracker.max_run_time
-      Jobtracker.config['max_run_time']
-    end
-
-    def Jobtracker.admins
-      Jobtracker.config['admins']
-    end
-
-    def Jobtracker.admin_emails
-      Jobtracker.admins.map{|a| a['email'] }
-    end
-
-    def Jobtracker.worker
-      Resque.find_worker_by_path("jobtracker")
-    end
-
-    def Jobtracker.workers(state="all")
-      Resque.workers(state)
-    end
-
-    def Jobtracker.status
-      args = Jobtracker.get_args
-      return args['status'] if args
-      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
-      return 'queued' if job
-      return 'stopped'
-    end
-
-    def Jobtracker.update_status(msg)
-      #this is to keep jobtracker from resisting stop commands
-      return false if Jobtracker.status=="stopping"
-      #Jobtracker has no persistent database state
-      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
-      return true
-    end
-
-    def Jobtracker.restart
-      Jobtracker.stop!
-      Jobtracker.start
-    end
-
-    def Jobtracker.set_args(args)
-      Resque.set_worker_args(Jobtracker.worker,args)
-      return true
-    end
-
-    def Jobtracker.get_args
-      Resque.get_worker_args(Jobtracker.worker)
-    end
-
-    def Jobtracker.kill_workers
-      Resque.kill_workers
-    end
-
-    def Jobtracker.kill_idle_workers
-      Resque.kill_idle_workers
-    end
-
-    def Jobtracker.kill_idle_and_stale_workers
-      Resque.kill_idle_and_stale_workers
-    end
-
-    def Jobtracker.prep_workers
-      Resque.prep_workers
-    end
-
-    def Jobtracker.failures
-      Resque.failures
-    end
-
-    def Jobtracker.start
-      if Jobtracker.status!='stopped'
-        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
-      else
-        #make sure that workers are running and at the right number
-        #Resque.prep_workers
-        #queue up the jobtracker (starts the perform method)
-        Jobtracker.enqueue!
-      end
-      return true
-    end
-
-    def Jobtracker.enqueue!
-      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
-    end
-
-    def Jobtracker.restart!
-      Jobtracker.stop!
-      Jobtracker.start
-      return true
-    end
-
-    def Jobtracker.restart_workers!
-      Jobtracker.kill_workers
-      sleep 10
-      Jobtracker.prep_workers
-      Jobtracker.update_status("put workers back on the queue")
-    end
-
-    def Jobtracker.stop!
-      #send signal for Jobtracker to check for
-      Jobtracker.update_status('stopping')
-      sleep 5
-      i=0
-      while Jobtracker.status=='stopping'
-        puts "#{Jobtracker.to_s} still on queue, waiting"
-        sleep 5
-        i+=1
-      end
-      return true
-    end
-
-    def Jobtracker.last_notification
-      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
-    end
-
-    def Jobtracker.last_notification=(time)
-      Jobtracker.set_args({"last_notification"=>time})
-    end
-
-    def Jobtracker.notif_due?
-      last_duetime = Time.now.utc - Jobtracker.notification_freq
-      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
-    end
+    #adds convenience methods
+    require "#{File.dirname(__FILE__)}/helpers/jobtracker_helper"
 
     def Jobtracker.max_run_time_workers
       #return workers who have been cranking away for 6+ hours
@@ -176,19 +35,26 @@ module Mobilize
                                           end
                                         end.flatten.join("\n\n")
             u = User.where(:name=>email.split("@").first).first
-            runner_dst = Dataset.find_by_url("gsheet://#{u.runner.path}")
-            n['body'] += "\n\n#{runner_dst.http_url}" if runner_dst and runner_dst.http_url
+            if u
+              runner_dst = Dataset.find_by_url("gsheet://#{u.runner.path}")
+              n['body'] += "\n\n#{runner_dst.http_url}" if runner_dst and runner_dst.http_url
+            end
             n['to'] = email
-            n['bcc'] = Jobtracker.admin_emails.join(",")
+            n['bcc'] = [Gdrive.admin_group_name,Gdrive.domain].join("@")
             notifs << n
           end
         end
         lws = Jobtracker.max_run_time_workers
         if lws.length>0
+          bod = begin
+                  lws.map{|w| w.job['payload']['args']}.first.join("\n")
+                rescue
+                  "Failed to get job names"
+                end
           n = {}
           n['subject'] = "#{lws.length.to_s} max run time jobs"
-          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
-          n['to'] = Jobtracker.admin_emails.join(",")
+          n['body'] = bod
+          n['to'] = [Gdrive.admin_group_name,Gdrive.domain].join("@")
           notifs << n
         end
         #deliver each email generated
@@ -250,53 +116,5 @@ module Mobilize
                     end.to_s.strip
       Time.parse(deploy_time)
     end
-
-    #test methods
-    def Jobtracker.restart_test_redis
-      Jobtracker.stop_test_redis
-      if !system("which redis-server")
-        raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
-      end
-      "redis-server #{Base.root}/test/redis-test.conf".bash
-    end
-
-    def Jobtracker.stop_test_redis
-      processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
-      pids = processes.map { |process| process.split(" ")[0] }
-      puts "Killing test redis server..."
-      pids.each { |pid| Process.kill("TERM", pid.to_i) }
-      puts "removing redis db dump file"
-      sleep 5
-      `rm -f #{Base.root}/test/dump.rdb #{Base.root}/test/dump-cluster.rdb`
-    end
-
-    def Jobtracker.set_test_env
-      ENV['MOBILIZE_ENV']='test'
-      ::Resque.redis="localhost:9736"
-      mongoid_config_path = "#{Base.root}/config/mobilize/mongoid.yml"
-      Mongoid.load!(mongoid_config_path, Base.env)
-    end
-
-    def Jobtracker.drop_test_db
-      Jobtracker.set_test_env
-      Mongoid.session(:default).collections.each do |collection| 
-        unless collection.name =~ /^system\./
-          collection.drop
-        end
-      end
-    end
-
-    def Jobtracker.build_test_runner(user_name)
-      Jobtracker.set_test_env
-      u = User.where(:name=>user_name).first
-      Jobtracker.update_status("delete old books and datasets")
-      # delete any old runner from previous test runs
-      gdrive_slot = Gdrive.owner_email
-      u.runner.gsheet(gdrive_slot).spreadsheet.delete
-      Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
-      Jobtracker.update_status("enqueue jobtracker, wait 45s")
-      Mobilize::Jobtracker.start
-      sleep 45
-    end
   end
 end

data/lib/mobilize-base/models/runner.rb

@@ -101,6 +101,11 @@ module Mobilize
       return true
     end
 
+    def worker
+      r = self
+      Mobilize::Resque.find_worker_by_path(r.path)
+    end
+
     def enqueue!
       r = self
       ::Resque::Job.create("mobilize",Runner,r.path,{})

data/lib/mobilize-base/models/stage.rb

@@ -44,6 +44,14 @@ module Mobilize
 
     def Stage.perform(id,*args)
       s = Stage.where(:path=>id).first
+      #check to make sure params are parsable
+      begin
+        param_hash = s.params
+        raise ScriptError if param_hash.class!=Hash
+      rescue StandardError, ScriptError
+        s.fail({'signal'=>500,
+                'err_str'=>"Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."})
+      end
       s.update_attributes(:started_at=>Time.now.utc)
       s.update_status(%{Starting at #{Time.now.utc}})
       #get response by running method
@@ -112,7 +120,11 @@ module Mobilize
       j = s.job
       r = j.runner
       u = r.user
-      j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
+      begin
+        j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
+      rescue StandardError, ScriptError
+        #skip due to parse error on params
+      end
       s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
       stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
       target_path =  (r.path.split("/")[0..-2] + [stage_name]).join("/")

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.298"
+    VERSION = "1.351"
   end
 end

data/lib/samples/gdrive.yml

@@ -4,8 +4,10 @@ development:
   owner:
     name: owner_development
     pw: google_drive_password
+  admin_group_name: admins_development
   admins:
     - name: admin
+  worker_group_name: workers_development
   workers:
     - name: worker_development001 
       pw: worker001_google_drive_password
@@ -16,8 +18,10 @@ test:
   owner:
     name: owner_test
     pw: google_drive_password
+  admin_group_name: admins_test
   admins:
     - name: admin
+  worker_group_name: workers_test
   workers:
     - name: worker_test001 
       pw: worker001_google_drive_password
@@ -28,8 +32,10 @@ production:
   owner:
     name: owner_production
     pw: google_drive_password
+  admin_group_name: admins_production
   admins:
     - name: admin
+  worker_group_name: workers_production
   workers:
     - name: worker_production001 
       pw: worker001_google_drive_password

data/lib/samples/jobtracker.yml

@@ -5,21 +5,15 @@ development:
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
-  admins: #emails to send notifications to
-  - email: admin@host.com
 test:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
-  admins: #emails to send notifications to
-  - email: admin@host.com
 production:
   cycle_freq: 10 #time between Jobtracker sweeps
   notification_freq: 3600 #1 hour between failure/timeout notifications
   runner_read_freq: 300 #5 min between runner reads
   max_run_time: 14400 # if a job runs for 4h+, notification will be sent
   extensions: [] #additional Mobilize modules to load workers with
-  admins: #emails to send notifications to
-  - email: admin@host.com

data/test/base_job_rows.yml

@@ -3,9 +3,13 @@
   trigger: once
   status: ""
   stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
-
 - name: base2
   active: true
   trigger: after base1
   status: ""
   stage1: gsheet.write source:base1.out, target:base2.out
+- name: base3
+  active: true
+  trigger: after base2
+  status: ""
+  stage1: gsheet.write source:"base1."_"", target:base2.out

data/test/mobilize-base_test.rb

@@ -1,43 +1,34 @@
 require 'test_helper'
 
 describe "Mobilize" do
-
-  def before
-    puts 'nothing before'
-  end
-
-  # enqueues 4 workers on Resque
   it "runs integration test" do
 
     puts "restart test redis"
-    Mobilize::Jobtracker.restart_test_redis
+    TestHelper.restart_test_redis
 
     puts "clear out test db"
-    Mobilize::Jobtracker.drop_test_db
+    TestHelper.drop_test_db
 
     puts "restart workers"
     Mobilize::Jobtracker.restart_workers!
 
-    puts "build test runner"
-    gdrive_slot = Mobilize::Gdrive.owner_email
-    puts "create user 'mobilize'"
-    user_name = gdrive_slot.split("@").first
-    u = Mobilize::User.find_or_create_by_name(user_name)
-    assert u.email == gdrive_slot
+    u = TestHelper.owner_user
+    user_name = u.name
+    gdrive_slot = u.email
 
-    Mobilize::Jobtracker.build_test_runner(user_name)
+    puts "build test runner"
+    TestHelper.build_test_runner(user_name)
     assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
 
     puts "Jobtracker created runner with 'jobs' sheet?"
+
     r = u.runner
     jobs_sheet_url = "gsheet://#{r.path}"
     jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
     jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
     jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
-    assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
 
-    #stop Jobtracker, if you're doing this by queueing runners
-    #Mobilize::Jobtracker.stop!
+    assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
 
     puts "add base1 input file"
     test_filename = "test_base_1"
@@ -53,55 +44,22 @@ describe "Mobilize" do
     jobs_sheet.reload
     jobs_sheet.add_or_update_rows(test_job_rows)
     #wait for stages to complete
-    #r.enqueue!
-    wait_for_stages
+    TestHelper.wait_for_stages
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    test_target_sheet_1_url = "gsheet://#{r.title}/base1.out"
-    test_target_sheet_2_url = "gsheet://#{r.title}/base2.out"
-    test_error_sheet_url = "gsheet://#{r.title}/base1_stage1.err"
-
-    test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
-    test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
+    tsv_hash = {}
+    ["base1.out", "base2.out", "base3_stage1.err"].each do |sheet_name|
+      url = "gsheet://#{r.title}/#{sheet_name}"
+      data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
+      tsv_hash[sheet_name] = data
+    end
 
-    assert test_1_tsv.to_s.length>0
-    assert test_1_tsv == test_2_tsv
+    assert tsv_hash["base1.out"].to_s.length>0
+    assert tsv_hash["base2.out"] == tsv_hash["base1.out"]
 
-    puts "change first job to fail, wait for stages"
-    test_job_rows.first['stage1'] = %{gsheet.write source:"gfile://test_base_1.fail", target:base1.out, retries:3}
-    Mobilize::Dataset.write_by_url(test_error_sheet_url," ",user_name,gdrive_slot)
-    jobs_sheet.add_or_update_rows(test_job_rows)
+    base3_response = tsv_hash["base3_stage1.err"].tsv_to_hash_array.first['response']
+    assert base3_response == "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
 
-    #wait for stages to complete
-    wait_for_stages
-
-    test_error_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.err",gdrive_slot)
-    puts "jobtracker posted failing test error to sheet "
-    error_rows = test_error_sheet.read(user_name).tsv_to_hash_array
-    assert error_rows.first['response'] == "Could not get gfile://test_base_1.fail with error: unable to find test_base_1.fail"
     Mobilize::Jobtracker.stop!
   end
-
-  def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
-    time = 0
-    time_since_stage = 0
-    #check for 10 min
-    while time < time_limit and time_since_stage < stage_limit
-      sleep wait_length
-      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
-      if job_classes.include?("Mobilize::Stage")
-        time_since_stage = 0
-        puts "saw stage at #{time.to_s} seconds"
-      else
-        time_since_stage += wait_length
-        puts "#{time_since_stage.to_s} seconds since stage seen"
-      end
-      time += wait_length
-      puts "total wait time #{time.to_s} seconds"
-    end
-
-    if time >= time_limit
-      raise "Timed out before stage completion"
-    end
-  end
 end

data/test/test_helper.rb

@@ -8,3 +8,82 @@ $dir = File.dirname(File.expand_path(__FILE__))
 ENV['MOBILIZE_ENV'] = 'test'
 require 'mobilize-base'
 $TESTING = true
+module TestHelper
+  def TestHelper.wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
+    time = 0
+    time_since_stage = 0
+    #check for 10 min
+    while time < time_limit and time_since_stage < stage_limit
+      sleep wait_length
+      job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
+      if job_classes.include?("Mobilize::Stage")
+        time_since_stage = 0
+        puts "saw stage at #{time.to_s} seconds"
+      else
+        time_since_stage += wait_length
+        puts "#{time_since_stage.to_s} seconds since stage seen"
+      end
+      time += wait_length
+      puts "total wait time #{time.to_s} seconds"
+    end
+
+    if time >= time_limit
+      raise "Timed out before stage completion"
+    end
+  end
+
+  #test methods
+  def TestHelper.restart_test_redis
+    TestHelper.stop_test_redis
+    if !system("which redis-server")
+      raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
+    end
+    "redis-server #{Mobilize::Base.root}/test/redis-test.conf".bash
+  end
+
+  def TestHelper.stop_test_redis
+    processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
+    pids = processes.map { |process| process.split(" ")[0] }
+    puts "Killing test redis server..."
+    pids.each { |pid| Process.kill("TERM", pid.to_i) }
+    puts "removing redis db dump file"
+    sleep 5
+    `rm -f #{Mobilize::Base.root}/test/dump.rdb #{Mobilize::Base.root}/test/dump-cluster.rdb`
+  end
+
+  def TestHelper.set_test_env
+    ENV['MOBILIZE_ENV']='test'
+    ::Resque.redis="localhost:9736"
+    mongoid_config_path = "#{Mobilize::Base.root}/config/mobilize/mongoid.yml"
+    Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
+  end
+
+  def TestHelper.drop_test_db
+    TestHelper.set_test_env
+    Mongoid.session(:default).collections.each do |collection| 
+      unless collection.name =~ /^system\./
+        collection.drop
+      end
+    end
+  end
+
+  def TestHelper.build_test_runner(user_name)
+    TestHelper.set_test_env
+    u = Mobilize::User.where(:name=>user_name).first
+    Mobilize::Jobtracker.update_status("delete old books and datasets")
+    # delete any old runner from previous test runs
+    gdrive_slot = Mobilize::Gdrive.owner_email
+    u.runner.gsheet(gdrive_slot).spreadsheet.delete
+    Mobilize::Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
+    Mobilize::Jobtracker.update_status("enqueue jobtracker, wait 45s")
+    Mobilize::Jobtracker.start
+    sleep 45
+  end
+
+  def TestHelper.owner_user
+    gdrive_slot = Mobilize::Gdrive.owner_email
+    puts "create user 'mobilize'"
+    user_name = gdrive_slot.split("@").first
+    return Mobilize::User.find_or_create_by_name(user_name)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: '1.298'
+  version: '1.351'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-17 00:00:00.000000000 Z
+date: 2013-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -191,6 +191,7 @@ files:
 - lib/mobilize-base/handlers/gsheet.rb
 - lib/mobilize-base/handlers/resque.rb
 - lib/mobilize-base/helpers/job_helper.rb
+- lib/mobilize-base/helpers/jobtracker_helper.rb
 - lib/mobilize-base/helpers/runner_helper.rb
 - lib/mobilize-base/helpers/stage_helper.rb
 - lib/mobilize-base/jobtracker.rb
@@ -228,7 +229,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2488340168623489502
+      hash: 4420073828570035753
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -237,7 +238,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2488340168623489502
+      hash: 4420073828570035753
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.25