mobilize-base 1.36 → 1.293

data/lib/mobilize-base/models/job.rb

@@ -2,153 +2,123 @@ module Mobilize
   class Job
     include Mongoid::Document
     include Mongoid::Timestamps
-    include Mobilize::JobHelper
     field :path, type: String
     field :active, type: Boolean
     field :trigger, type: String
 
     index({ path: 1})
 
+    def name
+      j = self
+      j.path.split("/").last
+    end
+
+    def stages
+      j = self
+      #starts with the job path, followed by a slash
+      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
+    end
+
     def Job.find_or_create_by_path(path)
       j = Job.where(:path=>path).first
       j = Job.create(:path=>path) unless j
       return j
     end
 
-    def parent
+    def status
+      #last stage status
       j = self
-      u = j.runner.user
-      if j.trigger.strip[0..4].downcase == "after"
-        parent_name = j.trigger[5..-1].to_s.strip
-        parent_j = u.jobs.select{|job| job.name == parent_name}.first
-        return parent_j
-      else
-        return nil
-      end
+      j.active_stage.status if j.active_stage
     end
 
-    def children
+    def active_stage
       j = self
-      u = j.runner.user
-      u.jobs.select do |job|
-        parent_name = job.trigger[5..-1].to_s.strip
-        job.trigger.strip[0..4].downcase == "after" and
-          parent_name == j.name
-      end
+      #latest started at or first
+      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
     end
 
-    #takes a hash of job parameters (name, active, trigger, stages)
-    #and creates/updates a job with it
-    def update_from_hash(hash)
+    def completed_at
       j = self
-      #update top line params
-      j.update_attributes(:active => hash['active'],
-                          :trigger => hash['trigger'])
-      (1..5).to_a.each do |s_idx|
-        stage_string = hash["stage#{s_idx.to_s}"]
-        s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
-        if stage_string.to_s.length==0
-          #delete this stage and all stages after
-          if s
-            j = s.job
-            j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
-            #just in case
-            s.delete
-          end
-          break
-        elsif s.nil?
-          #create this stage
-          s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
-        end
-        #parse command string, update stage with it
-        s_handler, call, param_string = [""*3]
-        stage_string.split(" ").ie do |spls|
-          s_handler = spls.first.split(".").first
-          call = spls.first.split(".").last
-          param_string = spls[1..-1].join(" ").strip
-        end
-        s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
-      end
-      return j.reload
+      j.stages.last.completed_at if j.stages.last
     end
 
-    def is_due?
+    def failed_at
       j = self
-      #working or inactive jobs are not due
-      if j.is_working? or j.active == false
-        return false
-      end
+      j.active_stage.failed_at if j.active_stage
+    end
 
-      #if job contains handlers not loaded by jobtracker, not due
-      loaded_handlers = Jobtracker.config['extensions'].map{|m| m.split("-").last}
-      job_handlers = j.stages.map{|s| s.handler}.uniq
-      #base handlers are the ones in mobilize-base/handlers
-      if (job_handlers - loaded_handlers - Base.handlers).length>0
-        return false
-      end
+    def status_at
+      j = self
+      j.active_stage.status_at if j.active_stage
+    end
 
-      #once
-      if j.trigger.strip.downcase=='once'
-        #active and once means due
-        return true
-      end
+    #convenience methods
+    def runner
+      j = self
+      runner_path = j.path.split("/")[0..-2].join("/")
+      return Runner.where(:path=>runner_path).first
+    end
 
-      #depedencies
-      if j.parent
-        #if parent completed more recently than self, is due
-        if j.parent.completed_at and (j.completed_at.nil? or j.parent.completed_at > j.completed_at)
+    def is_working?
+      j = self
+      j.stages.select{|s| s.is_working?}.compact.length>0
+    end
+
+    def is_due?
+      j = self
+      return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
+      last_run = j.completed_at
+      #check trigger
+      trigger = j.trigger
+      return true if trigger == 'once'
+      #strip the "every" from the front if present
+      trigger = trigger.gsub("every","").gsub("."," ").strip
+      value,unit,operator,job_utctime = trigger.split(" ")
+      curr_utctime = Time.now.utc
+      curr_utcdate = curr_utctime.to_date.strftime("%Y-%m-%d")
+      if job_utctime
+        job_utctime = job_utctime.split(" ").first
+        job_utctime = Time.parse([curr_utcdate,job_utctime,"UTC"].join(" "))
+      end
+      #after is the only operator
+      raise "Unknown #{operator.to_s} operator" if operator and operator != "after"
+      if ["hour","hours"].include?(unit)
+        #if it's later than the last run + hour tolerance, is due
+        if last_run.nil? or curr_utctime > (last_run + value.to_i.hour)
           return true
-        else
-          return false
         end
-      end
-
-      #time based
-      last_comp_time = j.completed_at
-      #check trigger; strip the "every" from the front if present, change dot to space
-      trigger = j.trigger.strip.gsub("every","").gsub("."," ").strip
-      number, unit, operator, mark = trigger.split(" ").map{|t_node| t_node.downcase}
-      #operator is not used
-      operator = nil
-      #get time for time-based evaluations
-      curr_time = Time.now.utc
-      if ["hour","hours","day","days"].include?(unit)
-        if mark
-          last_mark_time = Time.at_marks_ago(number,unit,mark)
-          if last_comp_time.nil? or last_comp_time < last_mark_time
-            return true
+      elsif ["day","days"].include?(unit)
+        if last_run.nil? or curr_utctime.to_date >= (last_run.to_date + value.to_i.day)
+          if operator and job_utctime
+            if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
+              return true
+            end
+          elsif operator || job_utctime
+            raise "Please specify both an operator and a time in UTC, or neither"
           else
-            return false
+            return true
           end
-        elsif last_comp_time.nil? or last_comp_time < (curr_time - number.to_i.send(unit))
-          return true
-        else
-          return false
         end
       elsif unit == "day_of_week"
-        if curr_time.wday==number and (last_comp_time.nil? or last_comp_time.to_date != curr_time.to_date)
-          if mark
-            #check if it already ran today
-            last_mark_time = Time.at_marks_ago(1,"day",mark)
-            if last_comp_time < last_mark_time
+        if curr_utctime.wday==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
+          if operator and job_utctime
+            if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
               return true
-            else
-              return false
             end
+          elsif operator || job_utctime
+            raise "Please specify both an operator and a time in UTC, or neither"
           else
             return true
           end
         end
       elsif unit == "day_of_month"
-        if curr_time.day==number and (last_comp_time.nil? or last_comp_time.to_date != curr_time.to_date)
-          if mark
-            #check if it already ran today
-            last_mark_time = Time.at_marks_ago(1,"day",mark)
-            if last_comp_time < last_mark_time
+        if curr_utctime.day==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
+          if operator and job_utctime
+            if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
               return true
-            else
-              return false
             end
+          elsif operator || job_utctime
+            raise "Please specify both an operator and a time in UTC, or neither"
           else
             return true
           end

data/lib/mobilize-base/models/runner.rb

@@ -1,6 +1,5 @@
 module Mobilize
   class Runner
-    include Mobilize::RunnerHelper
     include Mongoid::Document
     include Mongoid::Timestamps
     field :path, type: String
@@ -12,6 +11,20 @@ module Mobilize
 
     index({ path: 1})
 
+    def headers
+      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
+    end
+
+    def title
+      r = self
+      r.path.split("/").first
+    end
+
+    def worker
+      r = self
+      Mobilize::Resque.find_worker_by_path(r.path)
+    end
+
     def Runner.find_by_path(path)
       Runner.where(:path=>path).first
     end
@@ -19,7 +32,6 @@ module Mobilize
     def Runner.find_by_title(title)
       Runner.where(:path=>"#{title}/jobs").first
     end
-
     def Runner.perform(id,*args)
       r = Runner.find_by_path(id)
       #get gdrive slot for read
@@ -29,18 +41,12 @@ module Mobilize
         return false
       end
       r.update_attributes(:started_at=>Time.now.utc)
-      begin
-        #make sure any updates to activity are processed first
-        #as in when someone runs a "once" job that has completed
-        r.update_gsheet(gdrive_slot) 
-        #read the jobs in the gsheet and update models with news
-        r.read_gsheet(gdrive_slot)
-        #queue up the jobs that are due and active
-      rescue => exc
-        #log the exception, but continue w job processing
-        #This ensures jobs are still processed if google drive goes down
-        r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
-      end
+      #make sure any updates to activity are processed first
+      #as in when someone runs a "once" job that has completed
+      r.update_gsheet(gdrive_slot) 
+      #read the jobs in the gsheet and update models with news
+      r.read_gsheet(gdrive_slot)
+      #queue up the jobs that are due and active
       r.jobs.each do |j|
         begin
           if j.is_due?
@@ -50,41 +56,99 @@ module Mobilize
             s.enqueue!
           end
         rescue ScriptError, StandardError => exc
-          r.update_status("Failed to enqueue #{j.path}")
+          r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
+          j.update_attributes(:active=>false)
         end
       end
       r.update_gsheet(gdrive_slot)
       r.update_attributes(:completed_at=>Time.now.utc)
     end
 
+    def dataset
+      r = self
+      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
+    end
+
     def Runner.find_or_create_by_path(path)
       Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
     end
 
+    def gbook(gdrive_slot)
+      r = self
+      title = r.path.split("/").first
+      Gbook.find_by_path(title,gdrive_slot)
+    end
+
+    def gsheet(gdrive_slot)
+      r = self
+      u = r.user
+      jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
+      #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
+      unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
+        #only give the user edit permissions if they're the ones
+        #creating it
+        jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
+        unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
+          jobs_sheet.spreadsheet.update_acl(u.email,"writer")
+        end
+      end
+      jobs_sheet.add_headers(r.headers)
+      #add url to dataset
+      Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
+      begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
+      return jobs_sheet
+    end
+
     def read_gsheet(gdrive_slot)
       r = self
       #argument converts line breaks in cells to spaces
       gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
       #turn it into a hash array
-      gsheet_hashes = gsheet_tsv.tsv_to_hash_array
+      gsheet_jobs = gsheet_tsv.tsv_to_hash_array
       #go through each job, update relevant job with its params
       done_jobs = []
       #parse out the jobs and update the Job collection
-      gsheet_hashes.each do |gsheet_hash|
+      gsheet_jobs.each_with_index do |rj,rj_i|
         #skip non-jobs or jobs without required values
-        next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
-        #find job w this name, or make one
-        j = r.jobs.select{|rj| rj.name == gsheet_hash['name']}.first || Job.find_or_create_by_path("#{r.path}/#{gsheet_hash['name']}")
-        j.update_from_hash(gsheet_hash)
+        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
+        j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
+        #update top line params
+        j.update_attributes(:active => rj['active'],
+                            :trigger => rj['trigger'])
+        (1..5).to_a.each do |s_idx|
+          stage_string = rj["stage#{s_idx.to_s}"]
+          s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
+          if stage_string.to_s.length==0
+            #delete this stage and all stages after
+            if s
+              j = s.job
+              j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
+              #just in case
+              s.delete
+            end
+            break
+          elsif s.nil?
+            #create this stage
+            s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
+          end
+          #parse command string, update stage with it
+          s_handler, call, param_string = [""*3]
+          stage_string.split(" ").ie do |spls|
+            s_handler = spls.first.split(".").first
+            call = spls.first.split(".").last
+            param_string = spls[1..-1].join(" ").strip
+          end
+          s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
+        end
         r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
         #add this job to list of read ones
         done_jobs << j
       end
       #delete user jobs that are not included in Runner
-      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
-        j = Job.where(:path=>gsheet_hash_path).first
+      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
+        j = Job.where(:path=>rj_path).first
         j.delete if j
-        r.update_status("Deleted job:#{gsheet_hash_path}")
+        r.update_status("Deleted job:#{rj_path}")
       end
       r.update_status("jobs read at #{Time.now.utc}")
       return true
@@ -95,24 +159,46 @@ module Mobilize
       #there's nothing to update if runner has never had a completed at
       return false unless r.completed_at
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
-      upd_rows = upd_rows = upd_jobs.map do |j|
-        uj = {'name'=>j.name, 'status'=>j.status}
-        #jobs can only be turned off
-        #automatically, not back on
-        if j.active==false
-          uj['active'] = false
-        end
-        uj
-      end
+      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
+      upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
       return true
     end
 
-    def worker
+    def jobs(jname=nil)
       r = self
-      Mobilize::Resque.find_worker_by_path(r.path)
+      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
+      if jname
+        return js.sel{|j| j.name == jname}.first
+      else
+        return js
+      end
+    end
+
+    def user
+      r = self
+      user_name = r.path.split("_").second.split("(").first.split("/").first
+      User.where(:name=>user_name).first
+    end
+
+    def update_status(msg)
+      r = self
+      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
+      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      r = self
+      Mobilize::Resque.active_paths.include?(r.path)
+    end
+
+    def is_due?
+      r = self.reload
+      return false if r.is_working?
+      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
+      return true if r.started_at.nil? or r.started_at < prev_due_time
     end
 
     def enqueue!