mobilize-base 1.2 → 1.3

data/lib/mobilize-base/helpers/job_helper.rb

@@ -0,0 +1,54 @@
+#this module adds convenience methods to the Job model
+module Mobilize
+  module JobHelper
+    def name
+      j = self
+      j.path.split("/").last
+    end
+
+    def stages
+      j = self
+      #starts with the job path, followed by a slash
+      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
+    end
+
+    def status
+      #last stage status
+      j = self
+      j.active_stage.status if j.active_stage
+    end
+
+    def active_stage
+      j = self
+      #latest started at or first
+      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
+    end
+
+    def completed_at
+      j = self
+      j.stages.last.completed_at if j.stages.last
+    end
+
+    def failed_at
+      j = self
+      j.active_stage.failed_at if j.active_stage
+    end
+
+    def status_at
+      j = self
+      j.active_stage.status_at if j.active_stage
+    end
+
+    #convenience methods
+    def runner
+      j = self
+      runner_path = j.path.split("/")[0..-2].join("/")
+      return Runner.where(:path=>runner_path).first
+    end
+
+    def is_working?
+      j = self
+      j.stages.select{|s| s.is_working?}.compact.length>0
+    end
+  end
+end

data/lib/mobilize-base/helpers/runner_helper.rb

@@ -0,0 +1,83 @@
+#this module adds convenience methods to the Runner model
+module Mobilize
+  module RunnerHelper
+    def headers
+      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
+    end
+
+    def title
+      r = self
+      r.path.split("/").first
+    end
+
+    def worker
+      r = self
+      Mobilize::Resque.find_worker_by_path(r.path)
+    end
+
+    def dataset
+      r = self
+      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
+    end
+
+    def gbook(gdrive_slot)
+      r = self
+      title = r.path.split("/").first
+      Gbook.find_by_path(title,gdrive_slot)
+    end
+
+    def gsheet(gdrive_slot)
+      r = self
+      u = r.user
+      jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
+      #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
+      unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
+        #only give the user edit permissions if they're the ones
+        #creating it
+        jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
+        unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
+          jobs_sheet.spreadsheet.update_acl(u.email,"writer")
+        end
+        jobs_sheet.add_headers(r.headers)
+        begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
+      end
+      return jobs_sheet
+    end
+
+    def jobs(jname=nil)
+      r = self
+      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
+      if jname
+        return js.sel{|j| j.name == jname}.first
+      else
+        return js
+      end
+    end
+
+    def user
+      r = self
+      user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
+      User.where(:name=>user_name).first
+    end
+
+    def update_status(msg)
+      r = self
+      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
+      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      r = self
+      Mobilize::Resque.active_paths.include?(r.path)
+    end
+
+    def is_due?
+      r = self.reload
+      return false if r.is_working?
+      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
+      return true if r.started_at.nil? or r.started_at < prev_due_time
+    end
+
+  end
+end

data/lib/mobilize-base/helpers/stage_helper.rb

@@ -0,0 +1,38 @@
+#this module adds convenience methods to the Stage model
+module Mobilize
+  module StageHelper
+    def idx
+      s = self
+      s.path.split("/").last.gsub("stage","").to_i
+    end
+
+    def out_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
+      s = self
+      Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
+    end
+
+    def err_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
+      s = self
+      Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
+    end
+
+    def params
+      s = self
+      p = YAML.easy_load(s.param_string)
+      raise "Must resolve to Hash" unless p.class==Hash
+      return p
+    end
+
+    def job
+      s = self
+      job_path = s.path.split("/")[0..-2].join("/")
+      Job.where(:path=>job_path).first
+    end
+  end
+end

data/lib/mobilize-base/jobtracker.rb

@@ -48,6 +48,8 @@ module Mobilize
     end
 
     def Jobtracker.update_status(msg)
+      #this is to keep jobtracker from resisting stop commands
+      return false if Jobtracker.status=="stopping"
       #Jobtracker has no persistent database state
       Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
       return true
@@ -122,7 +124,7 @@ module Mobilize
       sleep 5
       i=0
       while Jobtracker.status=='stopping'
-        Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
+        puts "#{Jobtracker.to_s} still on queue, waiting"
         sleep 5
         i+=1
       end
@@ -145,8 +147,8 @@ module Mobilize
     def Jobtracker.max_run_time_workers
       #return workers who have been cranking away for 6+ hours
         workers = Jobtracker.workers('working').select do |w|
-            w.job['runat'].to_s.length>0 and 
-              (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
+            w.job['run_at'].to_s.length>0 and 
+              (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
         end
         return workers
     end
@@ -185,13 +187,18 @@ module Mobilize
         if lws.length>0
           n = {}
           n['subject'] = "#{lws.length.to_s} max run time jobs"
-          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
+          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
           n['to'] = Jobtracker.admin_emails.join(",")
           notifs << n
         end
         #deliver each email generated
         notifs.each do |notif|
-          Email.write(notif).deliver
+          begin
+            Email.write(notif).deliver
+          rescue
+            #log email on failure
+            Jobtracker.update_status("Failed to deliver #{notif.to_s}")
+          end
         end
         #update notification time so JT knows to wait a while
         Jobtracker.last_notification = Time.now.utc.to_s
@@ -286,6 +293,7 @@ module Mobilize
       # delete any old runner from previous test runs
       gdrive_slot = Gdrive.owner_email
       u.runner.gsheet(gdrive_slot).spreadsheet.delete
+      Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
       Jobtracker.update_status("enqueue jobtracker, wait 45s")
       Mobilize::Jobtracker.start
       sleep 45

data/lib/mobilize-base/models/job.rb

@@ -2,66 +2,54 @@ module Mobilize
   class Job
     include Mongoid::Document
     include Mongoid::Timestamps
+    include Mobilize::JobHelper
     field :path, type: String
     field :active, type: Boolean
     field :trigger, type: String
 
     index({ path: 1})
 
-    def name
-      j = self
-      j.path.split("/").last
-    end
-
-    def stages
-      j = self
-      #starts with the job path, followed by a slash
-      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
-    end
-
     def Job.find_or_create_by_path(path)
       j = Job.where(:path=>path).first
       j = Job.create(:path=>path) unless j
       return j
     end
 
-    def status
-      #last stage status
-      j = self
-      j.active_stage.status if j.active_stage
-    end
-
-    def active_stage
-      j = self
-      #latest started at or first
-      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
-    end
-
-    def completed_at
-      j = self
-      j.stages.last.completed_at if j.stages.last
-    end
-
-    def failed_at
-      j = self
-      j.active_stage.failed_at if j.active_stage
-    end
-
-    def status_at
-      j = self
-      j.active_stage.status_at if j.active_stage
-    end
-
-    #convenience methods
-    def runner
-      j = self
-      runner_path = j.path.split("/")[0..-2].join("/")
-      return Runner.where(:path=>runner_path).first
-    end
-
-    def is_working?
-      j = self
-      j.stages.select{|s| s.is_working?}.compact.length>0
+    #takes a hash of job parameters (name, active, trigger, stages)
+    #and creates/updates a job with it
+    def Job.update_by_user_name_and_hash(user_name,hash)
+      u = User.where(name: user_name).first
+      r = u.runner
+      j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
+      #update top line params
+      j.update_attributes(:active => hash['active'],
+                          :trigger => hash['trigger'])
+      (1..5).to_a.each do |s_idx|
+        stage_string = hash["stage#{s_idx.to_s}"]
+        s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
+        if stage_string.to_s.length==0
+          #delete this stage and all stages after
+          if s
+            j = s.job
+            j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
+            #just in case
+            s.delete
+          end
+          break
+        elsif s.nil?
+          #create this stage
+          s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
+        end
+        #parse command string, update stage with it
+        s_handler, call, param_string = [""*3]
+        stage_string.split(" ").ie do |spls|
+          s_handler = spls.first.split(".").first
+          call = spls.first.split(".").last
+          param_string = spls[1..-1].join(" ").strip
+        end
+        s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
+      end
+      return j.reload
     end
 
     def is_due?

data/lib/mobilize-base/models/runner.rb

@@ -1,5 +1,6 @@
 module Mobilize
   class Runner
+    include Mobilize::RunnerHelper
     include Mongoid::Document
     include Mongoid::Timestamps
     field :path, type: String
@@ -11,20 +12,6 @@ module Mobilize
 
     index({ path: 1})
 
-    def headers
-      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
-    end
-
-    def title
-      r = self
-      r.path.split("/").first
-    end
-
-    def worker
-      r = self
-      Mobilize::Resque.find_worker_by_path(r.path)
-    end
-
     def Runner.find_by_path(path)
       Runner.where(:path=>path).first
     end
@@ -32,6 +19,7 @@ module Mobilize
     def Runner.find_by_title(title)
       Runner.where(:path=>"#{title}/jobs").first
     end
+
     def Runner.perform(id,*args)
       r = Runner.find_by_path(id)
       #get gdrive slot for read
@@ -41,12 +29,18 @@ module Mobilize
         return false
       end
       r.update_attributes(:started_at=>Time.now.utc)
-      #make sure any updates to activity are processed first
-      #as in when someone runs a "once" job that has completed
-      r.update_gsheet(gdrive_slot) 
-      #read the jobs in the gsheet and update models with news
-      r.read_gsheet(gdrive_slot)
-      #queue up the jobs that are due and active
+      begin
+        #make sure any updates to activity are processed first
+        #as in when someone runs a "once" job that has completed
+        r.update_gsheet(gdrive_slot) 
+        #read the jobs in the gsheet and update models with news
+        r.read_gsheet(gdrive_slot)
+        #queue up the jobs that are due and active
+      rescue => exc
+        #log the exception, but continue w job processing
+        #This ensures jobs are still processed if google drive goes down
+        r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
+      end
       r.jobs.each do |j|
         begin
           if j.is_due?
@@ -64,90 +58,32 @@ module Mobilize
       r.update_attributes(:completed_at=>Time.now.utc)
     end
 
-    def dataset
-      r = self
-      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
-    end
-
     def Runner.find_or_create_by_path(path)
       Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
     end
 
-    def gbook(gdrive_slot)
-      r = self
-      title = r.path.split("/").first
-      Gbook.find_all_by_path(title,gdrive_slot).first
-    end
-
-    def gsheet(gdrive_slot)
-      r = self
-      u = r.user
-      jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
-      #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
-      unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
-        #only give the user edit permissions if they're the ones
-        #creating it
-        jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
-        unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
-          jobs_sheet.spreadsheet.update_acl(u.email,"writer")
-        end
-      end
-      jobs_sheet.add_headers(r.headers)
-      #add url to dataset
-      Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
-      begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
-      return jobs_sheet
-    end
-
     def read_gsheet(gdrive_slot)
       r = self
-      gsheet_tsv = r.gsheet(gdrive_slot).read(Gdrive.owner_name)
+      #argument converts line breaks in cells to spaces
+      gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
       #turn it into a hash array
-      gsheet_jobs = gsheet_tsv.tsv_to_hash_array
+      gsheet_hashes = gsheet_tsv.tsv_to_hash_array
       #go through each job, update relevant job with its params
       done_jobs = []
       #parse out the jobs and update the Job collection
-      gsheet_jobs.each_with_index do |rj,rj_i|
+      gsheet_hashes.each do |gsheet_hash|
         #skip non-jobs or jobs without required values
-        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
-        j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
-        #update top line params
-        j.update_attributes(:active => rj['active'],
-                            :trigger => rj['trigger'])
-        (1..5).to_a.each do |s_idx|
-          stage_string = rj["stage#{s_idx.to_s}"]
-          s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
-          if stage_string.to_s.length==0
-            #delete this stage and all stages after
-            if s
-              j = s.job
-              j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
-              #just in case
-              s.delete
-            end
-            break
-          elsif s.nil?
-            #create this stage
-            s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
-          end
-          #parse command string, update stage with it
-          s_handler, call, param_string = [""*3]
-          stage_string.split(" ").ie do |spls|
-            s_handler = spls.first.split(".").first
-            call = spls.first.split(".").last
-            param_string = spls[1..-1].join(" ").strip
-          end
-          s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
-        end
+        next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
+        j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
         r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
         #add this job to list of read ones
         done_jobs << j
       end
       #delete user jobs that are not included in Runner
-      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
-        j = Job.where(:path=>rj_path).first
+      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
+        j = Job.where(:path=>gsheet_hash_path).first
         j.delete if j
-        r.update_status("Deleted job:#{rj_path}")
+        r.update_status("Deleted job:#{gsheet_hash_path}")
       end
       r.update_status("jobs read at #{Time.now.utc}")
       return true
@@ -158,48 +94,13 @@ module Mobilize
       #there's nothing to update if runner has never had a completed at
       return false unless r.completed_at
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
+      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
       upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
       return true
     end
 
-    def jobs(jname=nil)
-      r = self
-      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
-      if jname
-        return js.sel{|j| j.name == jname}.first
-      else
-        return js
-      end
-    end
-
-    def user
-      r = self
-      user_name = r.path.split("_").second.split("(").first.split("/").first
-      User.where(:name=>user_name).first
-    end
-
-    def update_status(msg)
-      r = self
-      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
-      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
-      return true
-    end
-
-    def is_working?
-      r = self
-      Mobilize::Resque.active_paths.include?(r.path)
-    end
-
-    def is_due?
-      r = self.reload
-      return false if r.is_working?
-      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
-      return true if r.started_at.nil? or r.started_at < prev_due_time
-    end
-
     def enqueue!
       r = self
       ::Resque::Job.create("mobilize",Runner,r.path,{})

data/lib/mobilize-base/models/stage.rb

@@ -2,6 +2,7 @@ module Mobilize
   class Stage
     include Mongoid::Document
     include Mongoid::Timestamps
+    include Mobilize::StageHelper
     field :path, type: String
     field :handler, type: String
     field :call, type: String
@@ -16,40 +17,6 @@ module Mobilize
 
     index({ path: 1})
 
-    def idx
-      s = self
-      s.path.split("/").last.gsub("stage","").to_i
-    end
-
-    def out_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
-    end
-
-    def err_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
-    end
-
-    def params
-      s = self
-      p = YAML.easy_load(s.param_string)
-      raise "Must resolve to Hash" unless p.class==Hash
-      return p
-    end
-
-    def job
-      s = self
-      job_path = s.path.split("/")[0..-2].join("/")
-      Job.where(:path=>job_path).first
-    end
-
     def Stage.find_or_create_by_path(path)
       s = Stage.where(:path=>path).first
       s = Stage.create(:path=>path) unless s
@@ -80,7 +47,11 @@ module Mobilize
       s.update_attributes(:started_at=>Time.now.utc)
       s.update_status(%{Starting at #{Time.now.utc}})
       #get response by running method
-      response = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
+      response = begin
+                   "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
+                 rescue => exc
+                   {'err_str'=>"#{exc.to_s}\n#{exc.backtrace.to_a.join("\n")}", 'signal'=>500}
+                 end
       unless response
         #re-queue self if no response
         s.enqueue!
@@ -92,10 +63,10 @@ module Mobilize
         #retry
         s.update_attributes(:retries_done => s.retries_done.to_i + 1, :response => response)
         s.update_status(%{Retry #{s.retries_done.to_s} at #{Time.now.utc}})
+        sleep s['delay'].to_i
         s.enqueue!
       else
         #sleep as much as user specifies
-        sleep s['delay'].to_i
         s.fail(response)
       end
       return true
@@ -111,7 +82,7 @@ module Mobilize
         r = j.runner
         dep_jobs = r.jobs.select do |dj|
                                    dj.active==true and
-                                     dj.trigger.strip.downcase == "after #{j.name}"
+                                     dj.trigger.strip.downcase == "after #{j.name.downcase}"
                                  end
         #put begin/rescue so all dependencies run
         dep_jobs.each do |dj|
@@ -141,7 +112,7 @@ module Mobilize
       j = s.job
       r = j.runner
       u = r.user
-      j.update_attributes(:active=>false)
+      j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
       s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
       stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
       target_path =  (r.path.split("/")[0..-2] + [stage_name]).join("/")
@@ -158,10 +129,9 @@ module Mobilize
       err_txt = ["response","\n",err_txt].join
       err_sheet.write(err_txt,u.name)
       #exception will be first row below "response" header
-      exc_to_s,backtrace = err_txt.split("\n").ie{|ea| [ea[1], ea[2..-1]]}
       s.update_status(status_msg)
       #raise the exception so it bubbles up to resque
-      raise Exception,exc_to_s,backtrace
+      raise Exception,err_txt
     end
 
     def enqueue!
@@ -208,14 +178,15 @@ module Mobilize
         raise "incompatible target handler #{handler} for #{s.handler} stage"
       else
         begin
-          return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path)
+          #nil gdrive_slot for targets since there is no verification
+          return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path,nil)
         rescue => exc
           raise "Could not get #{target_path} with error: #{exc.to_s}"
         end
       end
     end
 
-    def sources
+    def sources(gdrive_slot)
       #returns an array of Datasets corresponding to
       #items listed as sources in the stage params
       s = self
@@ -244,7 +215,7 @@ module Mobilize
                     end
           begin
             stage_path = s.path
-            dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path)
+            dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path,gdrive_slot)
           rescue => exc
             raise "Could not get #{source_path} with error: #{exc.to_s}"
           end