mobilize-base 1.29 → 1.33

data/lib/mobilize-base/helpers/jobtracker_helper.rb

@@ -0,0 +1,147 @@
+module Mobilize
+  module Jobtracker
+    def Jobtracker.config
+      Base.config('jobtracker')
+    end
+
+    #modify this to increase the frequency of request cycles
+    def Jobtracker.cycle_freq
+      Jobtracker.config['cycle_freq']
+    end
+
+    #frequency of notifications
+    def Jobtracker.notification_freq
+      Jobtracker.config['notification_freq']
+    end
+
+    def Jobtracker.runner_read_freq
+      Jobtracker.config['runner_read_freq']
+    end
+
+    #long running tolerance
+    def Jobtracker.max_run_time
+      Jobtracker.config['max_run_time']
+    end
+
+    def Jobtracker.admins
+      Jobtracker.config['admins']
+    end
+
+    def Jobtracker.admin_emails
+      Jobtracker.admins.map{|a| a['email'] }
+    end
+
+    def Jobtracker.worker
+      Resque.find_worker_by_path("jobtracker")
+    end
+
+    def Jobtracker.workers(state="all")
+      Resque.workers(state)
+    end
+
+    def Jobtracker.status
+      args = Jobtracker.get_args
+      return args['status'] if args
+      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
+      return 'queued' if job
+      return 'stopped'
+    end
+
+    def Jobtracker.update_status(msg)
+      #this is to keep jobtracker from resisting stop commands
+      return false if Jobtracker.status=="stopping"
+      #Jobtracker has no persistent database state
+      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
+      return true
+    end
+
+    def Jobtracker.restart
+      Jobtracker.stop!
+      Jobtracker.start
+    end
+
+    def Jobtracker.set_args(args)
+      Resque.set_worker_args(Jobtracker.worker,args)
+      return true
+    end
+
+    def Jobtracker.get_args
+      Resque.get_worker_args(Jobtracker.worker)
+    end
+
+    def Jobtracker.kill_workers
+      Resque.kill_workers
+    end
+
+    def Jobtracker.kill_idle_workers
+      Resque.kill_idle_workers
+    end
+
+    def Jobtracker.kill_idle_and_stale_workers
+      Resque.kill_idle_and_stale_workers
+    end
+
+    def Jobtracker.prep_workers
+      Resque.prep_workers
+    end
+
+    def Jobtracker.failures
+      Resque.failures
+    end
+
+    def Jobtracker.start
+      if Jobtracker.status!='stopped'
+        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
+      else
+        #make sure that workers are running and at the right number
+        #Resque.prep_workers
+        #queue up the jobtracker (starts the perform method)
+        Jobtracker.enqueue!
+      end
+      return true
+    end
+
+    def Jobtracker.enqueue!
+      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
+    end
+
+    def Jobtracker.restart!
+      Jobtracker.stop!
+      Jobtracker.start
+      return true
+    end
+
+    def Jobtracker.restart_workers!
+      Jobtracker.kill_workers
+      sleep 10
+      Jobtracker.prep_workers
+      Jobtracker.update_status("put workers back on the queue")
+    end
+
+    def Jobtracker.stop!
+      #send signal for Jobtracker to check for
+      Jobtracker.update_status('stopping')
+      sleep 5
+      i=0
+      while Jobtracker.status=='stopping'
+        puts "#{Jobtracker.to_s} still on queue, waiting"
+        sleep 5
+        i+=1
+      end
+      return true
+    end
+
+    def Jobtracker.last_notification
+      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
+    end
+
+    def Jobtracker.last_notification=(time)
+      Jobtracker.set_args({"last_notification"=>time})
+    end
+
+    def Jobtracker.notif_due?
+      last_duetime = Time.now.utc - Jobtracker.notification_freq
+      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
+    end
+  end
+end

data/lib/mobilize-base/helpers/runner_helper.rb

@@ -0,0 +1,83 @@
+#this module adds convenience methods to the Runner model
+module Mobilize
+  module RunnerHelper
+    def headers
+      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
+    end
+
+    def title
+      r = self
+      r.path.split("/").first
+    end
+
+    def worker
+      r = self
+      Mobilize::Resque.find_worker_by_path(r.path)
+    end
+
+    def dataset
+      r = self
+      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
+    end
+
+    def gbook(gdrive_slot)
+      r = self
+      title = r.path.split("/").first
+      Gbook.find_by_path(title,gdrive_slot)
+    end
+
+    def gsheet(gdrive_slot)
+      r = self
+      u = r.user
+      jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
+      #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
+      unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
+        #only give the user edit permissions if they're the ones
+        #creating it
+        jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
+        unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
+          jobs_sheet.spreadsheet.update_acl(u.email,"writer")
+        end
+        jobs_sheet.add_headers(r.headers)
+        begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
+      end
+      return jobs_sheet
+    end
+
+    def jobs(jname=nil)
+      r = self
+      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
+      if jname
+        return js.sel{|j| j.name == jname}.first
+      else
+        return js
+      end
+    end
+
+    def user
+      r = self
+      user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
+      User.where(:name=>user_name).first
+    end
+
+    def update_status(msg)
+      r = self
+      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
+      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      r = self
+      Mobilize::Resque.active_paths.include?(r.path)
+    end
+
+    def is_due?
+      r = self.reload
+      return false if r.is_working?
+      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
+      return true if r.started_at.nil? or r.started_at < prev_due_time
+    end
+
+  end
+end

data/lib/mobilize-base/helpers/stage_helper.rb

@@ -0,0 +1,38 @@
+#this module adds convenience methods to the Stage model
+module Mobilize
+  module StageHelper
+    def idx
+      s = self
+      s.path.split("/").last.gsub("stage","").to_i
+    end
+
+    def out_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
+      s = self
+      Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
+    end
+
+    def err_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
+      s = self
+      Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
+    end
+
+    def params
+      s = self
+      p = YAML.easy_load(s.param_string)
+      raise "Must resolve to Hash" unless p.class==Hash
+      return p
+    end
+
+    def job
+      s = self
+      job_path = s.path.split("/")[0..-2].join("/")
+      Job.where(:path=>job_path).first
+    end
+  end
+end

data/lib/mobilize-base/jobtracker.rb

@@ -1,152 +1,13 @@
 module Mobilize
   module Jobtracker
-    def Jobtracker.config
-      Base.config('jobtracker')
-    end
-
-    #modify this to increase the frequency of request cycles
-    def Jobtracker.cycle_freq
-      Jobtracker.config['cycle_freq']
-    end
-
-    #frequency of notifications
-    def Jobtracker.notification_freq
-      Jobtracker.config['notification_freq']
-    end
-
-    def Jobtracker.runner_read_freq
-      Jobtracker.config['runner_read_freq']
-    end
-
-    #long running tolerance
-    def Jobtracker.max_run_time
-      Jobtracker.config['max_run_time']
-    end
-
-    def Jobtracker.admins
-      Jobtracker.config['admins']
-    end
-
-    def Jobtracker.admin_emails
-      Jobtracker.admins.map{|a| a['email'] }
-    end
-
-    def Jobtracker.worker
-      Resque.find_worker_by_path("jobtracker")
-    end
-
-    def Jobtracker.workers(state="all")
-      Resque.workers(state)
-    end
-
-    def Jobtracker.status
-      args = Jobtracker.get_args
-      return args['status'] if args
-      job = Resque.jobs.select{|j| j['args'].first=='jobtracker'}.first
-      return 'queued' if job
-      return 'stopped'
-    end
-
-    def Jobtracker.update_status(msg)
-      #Jobtracker has no persistent database state
-      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
-      return true
-    end
-
-    def Jobtracker.restart
-      Jobtracker.stop!
-      Jobtracker.start
-    end
-
-    def Jobtracker.set_args(args)
-      Resque.set_worker_args(Jobtracker.worker,args)
-      return true
-    end
-
-    def Jobtracker.get_args
-      Resque.get_worker_args(Jobtracker.worker)
-    end
-
-    def Jobtracker.kill_workers
-      Resque.kill_workers
-    end
-
-    def Jobtracker.kill_idle_workers
-      Resque.kill_idle_workers
-    end
-
-    def Jobtracker.kill_idle_and_stale_workers
-      Resque.kill_idle_and_stale_workers
-    end
-
-    def Jobtracker.prep_workers
-      Resque.prep_workers
-    end
-
-    def Jobtracker.failures
-      Resque.failures
-    end
-
-    def Jobtracker.start
-      if Jobtracker.status!='stopped'
-        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
-      else
-        #make sure that workers are running and at the right number
-        #Resque.prep_workers
-        #queue up the jobtracker (starts the perform method)
-        Jobtracker.enqueue!
-      end
-      return true
-    end
-
-    def Jobtracker.enqueue!
-      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
-    end
-
-    def Jobtracker.restart!
-      Jobtracker.stop!
-      Jobtracker.start
-      return true
-    end
-
-    def Jobtracker.restart_workers!
-      Jobtracker.kill_workers
-      sleep 10
-      Jobtracker.prep_workers
-      Jobtracker.update_status("put workers back on the queue")
-    end
-
-    def Jobtracker.stop!
-      #send signal for Jobtracker to check for
-      Jobtracker.update_status('stopping')
-      sleep 5
-      i=0
-      while Jobtracker.status=='stopping'
-        Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
-        sleep 5
-        i+=1
-      end
-      return true
-    end
-
-    def Jobtracker.last_notification
-      return Jobtracker.get_args["last_notification"] if Jobtracker.get_args
-    end
-
-    def Jobtracker.last_notification=(time)
-      Jobtracker.set_args({"last_notification"=>time})
-    end
-
-    def Jobtracker.notif_due?
-      last_duetime = Time.now.utc - Jobtracker.notification_freq
-      return (Jobtracker.last_notification.to_s.length==0 || Jobtracker.last_notification.to_datetime < last_duetime)
-    end
+    #adds convenience methods
+    require "#{File.dirname(__FILE__)}/helpers/jobtracker_helper"
 
     def Jobtracker.max_run_time_workers
       #return workers who have been cranking away for 6+ hours
         workers = Jobtracker.workers('working').select do |w|
-            w.job['runat'].to_s.length>0 and 
-              (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
+            w.job['run_at'].to_s.length>0 and 
+              (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
         end
         return workers
     end
@@ -183,15 +44,25 @@ module Mobilize
         end
         lws = Jobtracker.max_run_time_workers
         if lws.length>0
+          bod = begin
+                  lws.map{|w| w.job['payload']['args']}.first.join("\n")
+                rescue
+                  "Failed to get job names"
+                end
           n = {}
           n['subject'] = "#{lws.length.to_s} max run time jobs"
-          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
+          n['body'] = bod
           n['to'] = Jobtracker.admin_emails.join(",")
           notifs << n
         end
         #deliver each email generated
         notifs.each do |notif|
-          Email.write(notif).deliver
+          begin
+            Email.write(notif).deliver
+          rescue
+            #log email on failure
+            Jobtracker.update_status("Failed to deliver #{notif.to_s}")
+          end
         end
         #update notification time so JT knows to wait a while
         Jobtracker.last_notification = Time.now.utc.to_s
@@ -243,52 +114,5 @@ module Mobilize
                     end.to_s.strip
       Time.parse(deploy_time)
     end
-
-    #test methods
-    def Jobtracker.restart_test_redis
-      Jobtracker.stop_test_redis
-      if !system("which redis-server")
-        raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
-      end
-      "redis-server #{Base.root}/test/redis-test.conf".bash
-    end
-
-    def Jobtracker.stop_test_redis
-      processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
-      pids = processes.map { |process| process.split(" ")[0] }
-      puts "Killing test redis server..."
-      pids.each { |pid| Process.kill("TERM", pid.to_i) }
-      puts "removing redis db dump file"
-      sleep 5
-      `rm -f #{Base.root}/test/dump.rdb #{Base.root}/test/dump-cluster.rdb`
-    end
-
-    def Jobtracker.set_test_env
-      ENV['MOBILIZE_ENV']='test'
-      ::Resque.redis="localhost:9736"
-      mongoid_config_path = "#{Base.root}/config/mobilize/mongoid.yml"
-      Mongoid.load!(mongoid_config_path, Base.env)
-    end
-
-    def Jobtracker.drop_test_db
-      Jobtracker.set_test_env
-      Mongoid.session(:default).collections.each do |collection| 
-        unless collection.name =~ /^system\./
-          collection.drop
-        end
-      end
-    end
-
-    def Jobtracker.build_test_runner(user_name)
-      Jobtracker.set_test_env
-      u = User.where(:name=>user_name).first
-      Jobtracker.update_status("delete old books and datasets")
-      # delete any old runner from previous test runs
-      gdrive_slot = Gdrive.owner_email
-      u.runner.gsheet(gdrive_slot).spreadsheet.delete
-      Jobtracker.update_status("enqueue jobtracker, wait 45s")
-      Mobilize::Jobtracker.start
-      sleep 45
-    end
   end
 end

data/lib/mobilize-base/models/job.rb

@@ -2,66 +2,54 @@ module Mobilize
   class Job
     include Mongoid::Document
     include Mongoid::Timestamps
+    include Mobilize::JobHelper
     field :path, type: String
     field :active, type: Boolean
     field :trigger, type: String
 
     index({ path: 1})
 
-    def name
-      j = self
-      j.path.split("/").last
-    end
-
-    def stages
-      j = self
-      #starts with the job path, followed by a slash
-      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
-    end
-
     def Job.find_or_create_by_path(path)
       j = Job.where(:path=>path).first
       j = Job.create(:path=>path) unless j
       return j
     end
 
-    def status
-      #last stage status
-      j = self
-      j.active_stage.status if j.active_stage
-    end
-
-    def active_stage
-      j = self
-      #latest started at or first
-      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
-    end
-
-    def completed_at
-      j = self
-      j.stages.last.completed_at if j.stages.last
-    end
-
-    def failed_at
-      j = self
-      j.active_stage.failed_at if j.active_stage
-    end
-
-    def status_at
-      j = self
-      j.active_stage.status_at if j.active_stage
-    end
-
-    #convenience methods
-    def runner
-      j = self
-      runner_path = j.path.split("/")[0..-2].join("/")
-      return Runner.where(:path=>runner_path).first
-    end
-
-    def is_working?
-      j = self
-      j.stages.select{|s| s.is_working?}.compact.length>0
+    #takes a hash of job parameters (name, active, trigger, stages)
+    #and creates/updates a job with it
+    def Job.update_by_user_name_and_hash(user_name,hash)
+      u = User.where(name: user_name).first
+      r = u.runner
+      j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
+      #update top line params
+      j.update_attributes(:active => hash['active'],
+                          :trigger => hash['trigger'])
+      (1..5).to_a.each do |s_idx|
+        stage_string = hash["stage#{s_idx.to_s}"]
+        s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
+        if stage_string.to_s.length==0
+          #delete this stage and all stages after
+          if s
+            j = s.job
+            j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
+            #just in case
+            s.delete
+          end
+          break
+        elsif s.nil?
+          #create this stage
+          s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
+        end
+        #parse command string, update stage with it
+        s_handler, call, param_string = [""*3]
+        stage_string.split(" ").ie do |spls|
+          s_handler = spls.first.split(".").first
+          call = spls.first.split(".").last
+          param_string = spls[1..-1].join(" ").strip
+        end
+        s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
+      end
+      return j.reload
     end
 
     def is_due?