mobilize-base 1.0.2 → 1.0.4

data/lib/mobilize-base/jobtracker.rb

@@ -1,7 +1,7 @@
 module Mobilize
   module Jobtracker
     def Jobtracker.config
-      Base.config('jobtracker')[Base.env]
+      Base.config('jobtracker')
     end
 
     #modify this to increase the frequency of request cycles
@@ -14,8 +14,8 @@ module Mobilize
       Jobtracker.config['notification_freq']
     end
 
-    def Jobtracker.requestor_refresh_freq
-      Jobtracker.config['requestor_refresh_freq']
+    def Jobtracker.runner_read_freq
+      Jobtracker.config['runner_read_freq']
     end
 
     #long running tolerance
@@ -28,11 +28,11 @@ module Mobilize
     end
 
     def Jobtracker.admin_emails
-      Jobtracker.admins.map{|a| a['email']}
+      Jobtracker.admins.map{|a| a['email'] }
     end
 
     def Jobtracker.worker
-      Resque.find_worker_by_mongo_id("jobtracker")
+      Resque.find_worker_by_path("jobtracker")
     end
 
     def Jobtracker.workers(state="all")
@@ -49,7 +49,7 @@ module Mobilize
 
     def Jobtracker.update_status(msg)
       #Jobtracker has no persistent database state
-      Resque.update_job_status("jobtracker",msg)
+      Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
       return true
     end
 
@@ -96,7 +96,7 @@ module Mobilize
     end
 
     def Jobtracker.enqueue!
-      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{'status'=>'working'})
+      ::Resque::Job.create(Resque.queue_name, Jobtracker, 'jobtracker',{})
     end
 
     def Jobtracker.restart!
@@ -107,8 +107,9 @@ module Mobilize
 
     def Jobtracker.restart_workers!
       Jobtracker.kill_workers
-      sleep 5
+      sleep 10
       Jobtracker.prep_workers
+      Jobtracker.update_status("put workers back on the queue")
     end
 
     def Jobtracker.stop!
@@ -117,7 +118,7 @@ module Mobilize
       sleep 5
       i=0
       while Jobtracker.status=='stopping'
-        puts "#{Jobtracker.to_s} still on queue, waiting"
+        Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
         sleep 5
         i+=1
       end
@@ -175,7 +176,7 @@ module Mobilize
         notifs.each do |notif|
           Email.write(n['subj'],notif['body']).deliver
           Jobtracker.last_notification=Time.now.utc.to_s
-          "Sent notification at #{Jobtracker.last_notification}".oputs
+          Jobtracker.update_status("Sent notification at #{Jobtracker.last_notification}")
         end
       end
       return true
@@ -183,13 +184,14 @@ module Mobilize
 
     def Jobtracker.perform(id,*args)
       while Jobtracker.status != 'stopping'
-        requestors = Requestor.all
+        users = User.all
         Jobtracker.run_notifications
-        requestors.each do |r|
-          Jobtracker.update_status("Checking requestor #{r.name}")
+        users.each do |u|
+          r = u.runner
+          Jobtracker.update_status("Checking #{r.path}")
           if r.is_due?
             r.enqueue!
-            Jobtracker.update_status("Enqueued requestor #{r.name}")
+            Jobtracker.update_status("Enqueued #{r.path}")
           end
         end
         sleep 5
@@ -221,7 +223,7 @@ module Mobilize
     def Jobtracker.set_test_env
       ENV['MOBILIZE_ENV']='test'
       ::Resque.redis="localhost:9736"
-      mongoid_config_path = "#{Base.root}/config/mongoid.yml"
+      mongoid_config_path = "#{Base.root}/config/mobilize/mongoid.yml"
       Mongoid.load!(mongoid_config_path, Base.env)
     end
 
@@ -233,5 +235,17 @@ module Mobilize
         end
       end
     end
+
+    def Jobtracker.build_test_runner(user_name)
+      Jobtracker.set_test_env
+      u = User.where(:name=>user_name).first
+      Jobtracker.update_status("delete old books and datasets")
+      # delete any old runner from previous test runs
+      gdrive_slot = Gdrive.owner_email
+      u.runner.gsheet(gdrive_slot).spreadsheet.delete
+      Jobtracker.update_status("enqueue jobtracker, wait 45s")
+      Mobilize::Jobtracker.start
+      sleep 45
+    end
   end
 end

data/lib/mobilize-base/models/dataset.rb

@@ -2,69 +2,67 @@ module Mobilize
   class Dataset
     include Mongoid::Document
     include Mongoid::Timestamps
-    field :requestor_id, type: String
     field :handler, type: String
-    field :name, type: String
+    field :path, type: String
     field :url, type: String
-    field :size, type: Fixnum
+    field :raw_size, type: Fixnum
     field :last_cached_at, type: Time
+    field :last_cache_handler, type: String
     field :last_read_at, type: Time
     field :cache_expire_at, type: Time
 
-    index({ requestor_id: 1})
-    index({ handler: 1})
-    index({ name: 1})
-
-    before_destroy :destroy_cache
+    index({ handler: 1, path: 1}, { unique: true})
 
     def read
       dst = self
-      if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
-        return dst.read_cache
-      else
-        return "Mobilize::#{dst.handler.humanize}".constantize.read_by_dst_id(dst.id.to_s)
-      end
+      return "Mobilize::#{dst.handler.humanize}".constantize.read_by_path(dst.path)
     end
 
-    def Dataset.find_by_handler_and_name(handler,name)
-      Dataset.where(handler: handler, name: name).first
-    end
-
-    def Dataset.find_or_create_by_handler_and_name(handler,name)
-      dst = Dataset.where(handler: handler, name: name).first
-      dst = Dataset.create(handler: handler, name: name) unless dst
-      return dst
+    def Dataset.find_by_handler_and_path(handler,path)
+      Dataset.where(handler: handler, path: path).first
     end
 
-    def Dataset.find_or_create_by_requestor_id_and_handler_and_name(requestor_id,handler,name)
-      dst = Dataset.where(requestor_id: requestor_id, handler: handler, name: name).first
-      dst = Dataset.create(requestor_id: requestor_id, handler: handler, name: name) unless dst
+    def Dataset.find_or_create_by_handler_and_path(handler,path)
+      dst = Dataset.where(handler: handler, path: path).first
+      dst = Dataset.create(handler: handler, path: path) unless dst
       return dst
     end
 
-    def write(data)
+    def write(string)
       dst = self
-      dst.handler.humanize.constantize.write_by_dst_id(dst.id.to_s,data)
+      "Mobilize::#{dst.handler.humanize}".constantize.write_by_path(dst.path,string)
+      dst.raw_size = string.length
       dst.save!
       return true
     end
 
-    def read_cache
+    def cache_valid?
       dst = self
-      dst.update_attributes(:last_read_at=>Time.now.utc)
-      return Mongodb.read_by_filename(dst.id.to_s)
+      return true if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
     end
 
-    def write_cache(string,expire_at=nil)
+    def read_cache(cache_handler="gridfs")
       dst = self
-      Mongodb.write_by_filename(dst.id.to_s,string)
-      dst.update_attributes(:last_cached_at=>Time.now.utc,:cache_expire_at=>expire_at,:size=>string.length)
-      return true
+      if cache_valid?
+        dst.update_attributes(:last_read_at=>Time.now.utc)
+        return "Mobilize::#{cache_handler.humanize}".constantize.read([dst.handler,dst.path].join("://"))
+      else
+        raise "Cache invalid or not found for #{cache_handler}://#{dst.path}"
+      end
     end
 
-    def delete_cache
-      return Mongodb.delete_by_filename(dst.id.to_s)
+    def write_cache(string,expire_at=nil,cache_handler="gridfs")
+      dst = self
+      "Mobilize::#{cache_handler.humanize}".constantize.write([dst.handler,dst.path].join("://"),string)
+      dst.update_attributes(:last_cached_at=>Time.now.utc,
+                            :last_cache_handler=>cache_handler.to_s.downcase,
+                            :cache_expire_at=>expire_at,
+                            :size=>string.length)
+      return true
     end
 
+    def delete_cache(cache_handler="gridfs")
+      return "Mobilize::#{cache_handler.humanize}".constantize.delete(dst.handler, dst.path)
+    end
   end
 end

data/lib/mobilize-base/models/job.rb

@@ -2,199 +2,52 @@ module Mobilize
   class Job
     include Mongoid::Document
     include Mongoid::Timestamps
-    field :requestor_id, type: String
-    field :name, type: String
-    field :active, type: Boolean #active, inactive
-    field :schedule, type: String
-    field :active_task, type: String
-    field :tasks, type: String
+    field :path, type: String
+    field :active, type: Boolean
+    field :trigger, type: String
     field :status, type: String
-    field :last_error, type: String
-    field :last_trace, type: String
     field :last_completed_at, type: Time
-    field :datasets, type: String #name of data sources
-    field :params, type: String #JSON
-    field :destination, type: String #output destination - could be file, could be sheet
 
-    index({ requestor_id: 1})
-    index({ name: 1})
+    index({ path: 1})
 
-    before_destroy :destroy_output_dst_ids
-
-    def worker
-      j = self
-      Mobilize::Resque.find_worker_by_mongo_id(j.id.to_s)
-    end
-
-    def dataset_array
+    def name
       j = self
-      r = j.requestor
-      dsts = j.datasets.split(",").map{|dst| dst.strip}
-      dsts.map do |ps|
-        #prepend jobspec title if there is no path separator
-        full_ps = ps.index("/") ? ps : [r.jobspec_title,ps].join("/")
-        #find or create dataset for this sheet
-        dst = Dataset.find_or_create_by_handler_and_name("gsheet",full_ps)
-        dst.update_attributes(:requestor_id=>r.id.to_s) unless dst.requestor_id
-        dst
-      end
-    end
-
-    def task_array
-      self.tasks.split(",").map{|t| t.strip}
-    end
-
-    def task_output_dsts
-      j = self
-      r = j.requestor
-      dst_names = j.task_array.map{|tname| [r.name,j.name,tname].join("/")}
-      dst_names.map do |dst_name|
-        Dataset.find_or_create_by_requestor_id_and_handler_and_name(r.id.to_s,'mongodb',dst_name)
-      end
+      j.path.split("/").last
     end
 
-    def task_idx
+    def tasks
       j = self
-      j.task_array.index(j.active_task)
+      Task.where(:path=>/^#{j.path.escape_regex}/).to_a.sort_by{|t| t.path}
     end
 
-    def Job.find_by_name(name)
-      Job.where(:name=>name).first
-    end
-
-    def Job.find_all_by_requestor_id(requestor_id)
-      Job.where(:requestor_id=>requestor_id).to_a
-    end
-
-    def Job.find_or_create_by_requestor_id_and_name(requestor_id,name)
-      j = Job.where(:requestor_id=>requestor_id, :name=>name).first
-      j = Job.create(:requestor_id=>requestor_id, :name=>name) unless j
+    def Job.find_or_create_by_path(path)
+      j = Job.where(:path=>path).first
+      j = Job.create(:path=>path) unless j
       return j
     end
 
-    #called by Resque
-    def Job.perform(id,*args)
-      j = Job.find(id)
-      r = j.requestor
-      handler,method_name = j.active_task.split(".")
-      begin
-        j.update_status(%{Starting #{j.active_task} task at #{Time.now.utc}})
-        task_output = "Mobilize::#{handler.humanize}".constantize.send("#{method_name}_by_job_id",id)
-        #this allows user to return false if the stage didn't go as expected and needs to retry
-        #e.g. tried to write to Google but all the accounts were in use
-        return false if task_output == false
-        task_output_dst = j.task_output_dsts[j.task_idx]
-        task_output = task_output.to_s unless task_output.class == String
-        task_output_dst.write_cache(task_output)
-        if j.active_task == j.task_array.last
-          j.active_task = nil
-          j.last_error = ""
-          j.last_trace = ""
-          j.last_completed_at = Time.now.utc
-          j.status = %{Completed all tasks at #{Time.now.utc}}
-          j.save!
-          #check for any dependent jobs, if there are, enqueue them
-          r = j.requestor
-          dep_jobs = Job.where(:active=>true, :requestor_id=>r.id.to_s, :schedule=>"after #{j.name}").to_a
-          dep_jobs += Job.where(:active=>true, :schedule=>"after #{r.name}/#{j.name}").to_a
-          #put begin/rescue so all dependencies run
-          dep_jobs.each{|dj| begin;dj.enqueue! unless dj.is_working?;rescue;end}
-        else
-          #set next task
-          j.active_task = j.task_array[j.task_idx+1]
-          j.save!
-          #queue up next task
-          j.enqueue!
-        end
-      rescue ScriptError,StandardError => exc
-        #record the failure in Job so it appears on spec sheets
-        j.status='failed'
-        j.save!
-        j.update_status("Failed at #{Time.now.utc.to_s}")
-        j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
-        [exc.to_s,exc.backtrace.to_s].join("=>").oputs
-        #raising here will cause the failure to show on the Resque UI
-        raise exc
-      end
-      return true
-    end
-
-    def enqueue!
-      j = self
-      r = j.requestor
-      j.update_attributes(:active_task=>j.task_array.first) if j.active_task.blank?
-      ::Resque::Job.create("mobilize",Job,j.id.to_s,%{#{r.name}=>#{j.name}})
-      return true
-    end
-
     #convenience methods
-    def requestor
-      j = self
-      return Requestor.find(j.requestor_id)
-    end
-
-    def restart
-      j = self
-      j.update_attributes(:last_completed_at=>nil)
-      return true
-    end
-
-    def prior_task
-      j = self
-      return nil if j.active_task.nil?
-      task_idx = j.task_array.index(j.active_task)
-      return nil if task_idx==0
-      return j.task_array[task_idx-1]
-    end
-
-    def destination_url
-      j = self
-      return nil if j.destination.nil?
-      destination = j.destination
-      dst = if j.task_array.last == 'gsheet.write'
-              destination = [j.requestor.jobspec_title,j.destination].join("/") if destination.split("/").length==1
-              Dataset.find_by_handler_and_name('gsheet',destination)
-            elsif j.task_array.last == 'gfile.write'
-              #all gfiles must end in gz
-              destination += ".gz" unless destination.ends_with?(".gz")
-              destination = [s.requestor.name,"_"].join + destination unless destination.starts_with?([s.requestor.name,"_"].join)
-              Dataset.find_by_handler_and_name('gfile',destination)
-            end
-      return dst.url if dst
-    end
-
-    def worker_args
-      j = self
-      Jobtracker.get_worker_args(j.worker)
-    end
-
-    def set_worker_args(args)
-      j = self
-      Jobtracker.set_worker_args(j.worker,args)
-    end
-
-    def update_status(msg)
+    def runner
       j = self
-      j.update_attributes(:status=>msg)
-      Mobilize::Resque.update_job_status(j.id.to_s,msg)
-      return true
+      runner_path = j.path.split("/")[0..-2].join("/")
+      return Runner.where(:path=>runner_path).first
     end
 
     def is_working?
       j = self
-      Mobilize::Resque.active_mongo_ids.include?(j.id.to_s)
+      j.tasks.select{|t| t.is_working?}.compact.length>0
     end
 
     def is_due?
       j = self
-      return false if j.is_working? or j.active == false or j.schedule.to_s.starts_with?("after")
+      return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
       last_run = j.last_completed_at
-      #check schedule
-      schedule = j.schedule
-      return true if schedule == 'once'
+      #check trigger
+      trigger = j.trigger
+      return true if trigger == 'once'
       #strip the "every" from the front if present
-      schedule = schedule.gsub("every","").gsub("."," ").strip
-      value,unit,operator,job_utctime = schedule.split(" ")
+      trigger = trigger.gsub("every","").gsub("."," ").strip
+      value,unit,operator,job_utctime = trigger.split(" ")
       curr_utctime = Time.now.utc
       curr_utcdate = curr_utctime.to_date.strftime("%Y-%m-%d")
       if job_utctime

data/lib/mobilize-base/models/runner.rb

@@ -0,0 +1,178 @@
+module Mobilize
+  class Runner
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    field :path, type: String
+    field :active, type: Boolean
+    field :status, type: String
+    field :last_run, type: Time
+
+    index({ path: 1})
+
+    def headers
+      %w{name active trigger status task1 task2 task3 task4 task5}
+    end
+
+    def last_cached_at
+      r = self
+      Dataset.find_or_create_by_path(r.path).last_cached_at
+    end
+
+    def worker
+      r = self
+      Mobilize::Resque.find_worker_by_path(r.path)
+    end
+
+    def Runner.find_by_email(email)
+      Runner.where(:email=>email).first
+    end
+
+    def Runner.find_by_path(path)
+      Runner.where(:path=>path).first
+    end
+
+    def Runner.perform(id,*args)
+      r = Runner.find_by_path(id)
+      #get gdrive slot for read
+      gdrive_slot = Gdrive.slot_worker_by_path(r.path)
+      unless gdrive_slot
+        r.update_status("no gdrive slot available")
+        return false
+      end
+      #make sure any updates to activity are processed first
+      #as in when someone runs a "once" job that has completed
+      r.update_gsheet(gdrive_slot)
+      #read the jobs in the gsheet and update models with news
+      r.read_gsheet(gdrive_slot)
+      #queue up the jobs that are due and active
+      r.jobs.each do |j|
+        begin
+          if j.is_due?
+            j.tasks.first.enqueue!
+          end
+        rescue ScriptError, StandardError => exc
+          j.update_status("Failed to enqueue with #{exc.to_s}")
+          j.update_attributes(:active=>false)
+        end
+      end
+      r.update_gsheet(gdrive_slot)
+      r.update_attributes(:last_run=>Time.now.utc)
+    end
+
+    def dataset
+      r = self
+      Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
+    end
+
+    def Runner.find_or_create_by_path(path)
+      Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
+    end
+
+    def read_cache
+      r = self
+      r.dataset.read_cache
+    end
+
+    def gsheet(gdrive_slot)
+      r = self
+      jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
+      jobs_sheet.add_headers(r.headers)
+      jobs_sheet.delete_sheet1
+      return jobs_sheet
+    end
+
+    def read_gsheet(gdrive_slot)
+      r = self
+      gsheet_tsv = r.gsheet(gdrive_slot).to_tsv
+      #cache in DB
+      r.dataset.write_cache(gsheet_tsv)
+      #turn it into a hash array
+      gsheet_jobs = gsheet_tsv.tsv_to_hash_array
+      #go through each job, update relevant job with its params
+      done_jobs = []
+      #parse out the jobs and update the Job collection
+      gsheet_jobs.each_with_index do |rj,rj_i|
+        #skip non-jobs or jobs without required values
+        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','task1'].select{|c| rj[c].to_s.strip==""}.length>0)
+        j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
+        #update top line params
+        j.update_attributes(:active => rj['active'],
+                            :trigger => rj['trigger'])
+        (1..5).to_a.each do |t_idx|
+          task_string = rj["task#{t_idx.to_s}"]
+          break if task_string.to_s.length==0
+          t = Task.find_or_create_by_path("#{j.path}/task#{t_idx.to_s}")
+          #parse command string, update task with it
+          t_handler, call, param_string = [""*3]
+          task_string.split(" ").ie do |spls|
+            t_handler = spls.first.split(".").first
+            call = spls.first.split(".").last
+            param_string = spls[1..-1].join(" ").strip
+          end
+          t.update_attributes(:call=>call, :handler=>t_handler, :param_string=>param_string)
+        end
+        r.update_status("Updated #{j.path} tasks at #{Time.now.utc}")
+        #add this job to list of read ones
+        done_jobs << j
+      end
+      #delete user jobs that are not included in Runner
+      (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
+        j = Job.where(:path=>rj_path).first
+        j.delete if j
+        r.update_status("Deleted job:#{rj_path}")
+      end
+      r.update_status("jobs read at #{Time.now.utc}")
+      return true
+    end
+
+    def update_gsheet(gdrive_slot)
+      r = self
+      jobs_gsheet = r.gsheet(gdrive_slot)
+      upd_rows = r.jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status} }
+      jobs_gsheet.add_or_update_rows(upd_rows)
+      r.update_status("gsheet updated")
+      return true
+    end
+
+    def jobs(jname=nil)
+      r = self
+      js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
+      if jname
+        return js.sel{|j| j.name == jname}.first
+      else
+        return js
+      end
+    end
+
+    def user
+      r = self
+      user_name = r.path.split("_").second.split("(").first.split("/").first
+      User.where(:email=>[user_name,Gdrive.domain].join("@")).first
+    end
+
+    def update_status(msg)
+      r = self
+      r.update_attributes(:status=>msg)
+      Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      r = self
+      Mobilize::Resque.active_paths.include?(r.path)
+    end
+
+    def is_due?
+      r = self.reload
+      return false if r.is_working?
+      last_due_time = Time.now.utc - Jobtracker.runner_read_freq
+      return true if r.last_run.nil? or r.last_run < last_due_time
+    end
+
+    def enqueue!
+      r = self
+      ::Resque::Job.create("mobilize",Runner,r.path,{})
+      return true
+    end
+  end
+end