mobilize-base 1.0.84 → 1.0.91

data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb

@@ -8,7 +8,7 @@ module GoogleDrive
       attempts = 0
       sleep_time = nil
       #try 5 times to make the call
-      while (response.nil? or response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}) and attempts < 5
+      while (response.nil? or response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}) and attempts < 20
         #instantiate http object, set params
         http = @proxy.new(uri.host, uri.port)
         http.use_ssl = true
@@ -40,7 +40,7 @@ module GoogleDrive
           end
         end
       end
-      raise "No response after 5 attempts" if response.nil?
+      raise "No response after 20 attempts" if response.nil?
       raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
       return response
     end

data/lib/mobilize-base/handlers/gbook.rb

@@ -8,10 +8,10 @@ module Mobilize
       dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       #there should only be one book with each path, otherwise we have fail
       book = nil
-      if books.length>1 and dst.url.to_s.length>0
+      if books.length>1 and dst.http_url.to_s.length>0
         #some idiot process created a duplicate book.
         #Fix by renaming all but one with dst entry's key
-        dkey = dst.url.split("key=").last
+        dkey = dst.http_url.split("key=").last
         books.each do |b|
           bkey = b.resource_id.split(":").last
           if bkey == dkey
@@ -30,9 +30,9 @@ module Mobilize
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
       end
-      #always make sure book dataset URL is up to date
+      #always make sure book dataset http URL is up to date
       #and that book has admin acl
-      dst.update_attributes(:url=>book.human_url)
+      dst.update_attributes(:http_url=>book.human_url)
       book.add_admin_acl
       return book
     end

data/lib/mobilize-base/handlers/gfile.rb

@@ -28,7 +28,10 @@ module Mobilize
       return false unless gdrive_slot
       s = Stage.where(:path=>stage_path)
       gfile_path = s.params['file']
-      Gfile.find_by_path(gfile_path,gdrive_slot).read
+      out_tsv = Gfile.find_by_path(gfile_path,gdrive_slot).read
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,out_tsv)
     end
   end
 end

data/lib/mobilize-base/handlers/gridfs.rb

@@ -11,29 +11,29 @@ module Mobilize
       return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
     end
 
-    def Gridfs.read(path)
+    def Gridfs.read_by_dataset_path(dst_path)
       begin
-        zs=Gridfs.grid.open(path.gridsafe,'r').read
+        zs=Gridfs.grid.open(dst_path,'r').read
         return ::Zlib::Inflate.inflate(zs)
       rescue
         return nil
       end
     end
 
-    def Gridfs.write(path,string)
+    def Gridfs.write_by_dataset_path(dst_path,string)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      curr_zs = Gridfs.read(path.gridsafe).to_s
+      curr_zs = Gridfs.read_by_dataset_path(dst_path).to_s
       #write a new version when there is a change
       if curr_zs != zs
-        Gridfs.grid.open(path.gridsafe,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
+        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
       end
       return true
     end
 
-    def Gridfs.delete(path)
+    def Gridfs.delete(dst_path)
       begin
-        Gridfs.grid.delete(path.gridsafe)
+        Gridfs.grid.delete(dst_path)
         return true
       rescue
         return nil

data/lib/mobilize-base/handlers/gsheet.rb

@@ -38,13 +38,16 @@ module Mobilize
       return false unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
       gsheet_path = s.params['source']
-      Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
+      out_tsv = Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,out_tsv)
     end
 
     def Gsheet.write_by_stage_path(stage_path)
       gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-      #return false if there are no emails available
-      return false unless gdrive_slot
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
       source = s.params['source']
       target_path = s.params['target']
@@ -55,7 +58,7 @@ module Mobilize
                                           end
       source_stage_path = "#{s.job.runner.path}/#{source_job_name || s.job.name}/#{source_stage_name}"
       source_stage = Stage.where(:path=>source_stage_path).first
-      tsv = source_stage.stdout_dataset.read_cache
+      tsv = source_stage.out_dst.read
       sheet_name = target_path.split("/").last
       temp_path = [stage_path.gridsafe,sheet_name].join("/")
       temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
@@ -65,8 +68,11 @@ module Mobilize
       target_sheet.merge(temp_sheet)
       #delete the temp sheet's book
       temp_sheet.spreadsheet.delete
-      "Write successful for #{target_path}".oputs
-      return true
+      status = "Write successful for #{target_path}"
+      s.update_status(status)
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,status)
     end
   end
 end

data/lib/mobilize-base/handlers/resque.rb

@@ -51,12 +51,12 @@ module Mobilize
 
     def Resque.active_paths
       #first argument of the payload is the runner / stage path unless the worker is Jobtracker
-      Resque.jobs('active').map{|j| j['args'].first unless (j.nil? or j['class']=='Jobtracker')}.compact
+      Resque.jobs('active').compact.map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
     end
 
     #Resque workers and methods to find
     def Resque.find_worker_by_path(path)
-      Resque.workers('working').select{|w| w.job['payload'] and w.job['payload']['args'].first == path}.first
+      Resque.workers('working').select{|w| w.job and w.job['payload'] and w.job['payload']['args'].first == path}.first
     end
 
     def Resque.set_worker_args_by_path(path,args)
@@ -106,7 +106,9 @@ module Mobilize
     def Resque.failure_report
       fjobs = {}
       excs = Hash.new(0)
-      Resque.failures.each do |f|
+      Resque.failures.each_with_index do |f,f_i|
+        #skip if already notified
+        next if f['notified']
         sname = f['payload']['args'].first
         excs = f['error']
         if fjobs[sname].nil?
@@ -116,6 +118,9 @@ module Mobilize
         else
           fjobs[sname][excs] += 1
         end
+        #add notified flag to redis
+        f['notified'] = true
+        ::Resque.redis.lset(:failed, f_i, ::Resque.encode(f))
       end
       return fjobs
     end
@@ -134,9 +139,9 @@ module Mobilize
       if count.to_i > idle_pids.length or count == 0
         return false
       elsif count
-        "kill #{idle_pids[0..count-1].join(" ")}".bash
+        "kill #{idle_pids[0..count-1].join(" ")}".bash(false)
       else
-        "kill #{idle_pids.join(" ")}".bash
+        "kill #{idle_pids.join(" ")}".bash(false)
       end
       return true
     end
@@ -148,7 +153,7 @@ module Mobilize
       if idle_stale_pids.length == 0
         return false
       else
-        "kill #{idle_stale_pids.join(" ")}".bash
+        "kill #{idle_stale_pids.join(" ")}".bash(false)
       end
       return true
     end

data/lib/mobilize-base/jobtracker.rb

@@ -163,12 +163,14 @@ module Mobilize
       if Jobtracker.notif_due?
         notifs = []
         if Jobtracker.failures.length>0
-          n = {}
           jfcs = Resque.failure_report
-          n['subj'] = "#{jfcs.keys.length.to_s} failed jobs, #{jfcs.values.map{|v| v.values}.flatten.sum.to_s} failures"
-          #one row per exception type, with the job name
-          n['body'] = jfcs.map{|key,val| val.map{|b,name| [key," : ",b,", ",name," times"].join}}.flatten.join("\n\n")
-          notifs << n
+          unless jfcs=={} #no new failures
+            n = {}
+            n['subj'] = "#{jfcs.keys.length.to_s} new failed jobs, #{jfcs.values.map{|v| v.values}.flatten.sum.to_s} failures"
+            #one row per exception type, with the job name
+            n['body'] = jfcs.map{|key,val| val.map{|b,name| [key," : ",b,", ",name," times"].join}}.flatten.join("\n\n")
+            notifs << n
+          end
         end
         lws = Jobtracker.max_run_time_workers
         if lws.length>0

data/lib/mobilize-base/models/dataset.rb

@@ -4,7 +4,7 @@ module Mobilize
     include Mongoid::Timestamps
     field :handler, type: String
     field :path, type: String
-    field :url, type: String
+    field :http_url, type: String
     field :raw_size, type: Fixnum
     field :last_cached_at, type: Time
     field :last_cache_handler, type: String
@@ -18,6 +18,16 @@ module Mobilize
       return "Mobilize::#{dst.handler.humanize}".constantize.read_by_path(dst.path)
     end
 
+    def Dataset.find_by_url(url)
+      handler,path = url.split("://")
+      Dataset.find_by_handler_and_path(handler,path)
+    end
+
+    def Dataset.find_or_create_by_url(url)
+      handler,path = url.split("://")
+      Dataset.find_or_create_by_handler_and_path(handler,path)
+    end
+
     def Dataset.find_by_handler_and_path(handler,path)
       Dataset.where(handler: handler, path: path).first
     end
@@ -28,41 +38,30 @@ module Mobilize
       return dst
     end
 
-    def write(string)
-      dst = self
-      "Mobilize::#{dst.handler.humanize}".constantize.write_by_path(dst.path,string)
-      dst.raw_size = string.length
-      dst.save!
-      return true
+    def Dataset.write_to_url(url,string)
+      dst = Dataset.find_or_create_by_url(url)
+      dst.write(string)
+      url
     end
 
-    def cache_valid?
+    def read
       dst = self
-      return true if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
+      dst.update_attributes(:last_read_at=>Time.now.utc)
+      "Mobilize::#{dst.handler.humanize}".constantize.read_by_dataset_path(dst.path)
     end
 
-    def read_cache(cache_handler="gridfs")
+    def write(string)
       dst = self
-      if cache_valid?
-        dst.update_attributes(:last_read_at=>Time.now.utc)
-        return "Mobilize::#{cache_handler.humanize}".constantize.read([dst.handler,dst.path].join("://"))
-      else
-        raise "Cache invalid or not found for #{cache_handler}://#{dst.path}"
-      end
+      "Mobilize::#{dst.handler.humanize}".constantize.write_by_dataset_path(dst.path,string)
+      dst.raw_size = string.length
+      dst.save!
+      return true
     end
 
-    def write_cache(string,expire_at=nil,cache_handler="gridfs")
+    def delete
       dst = self
-      "Mobilize::#{cache_handler.humanize}".constantize.write([dst.handler,dst.path].join("://"),string)
-      dst.update_attributes(:last_cached_at=>Time.now.utc,
-                            :last_cache_handler=>cache_handler.to_s.downcase,
-                            :cache_expire_at=>expire_at,
-                            :size=>string.length)
+      "Mobilize::#{dst.handler.humanize}".constantize.delete_by_dataset_path(dst.path)
       return true
     end
-
-    def delete_cache(cache_handler="gridfs")
-      return "Mobilize::#{cache_handler.humanize}".constantize.delete(dst.handler, dst.path)
-    end
   end
 end

data/lib/mobilize-base/models/runner.rb

@@ -71,9 +71,9 @@ module Mobilize
       Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
     end
 
-    def read_cache
+    def cache
       r = self
-      r.dataset.read_cache
+      Dataset.find_or_create_by_url("gridfs://#{r.path}")
     end
 
     def gsheet(gdrive_slot)
@@ -88,7 +88,7 @@ module Mobilize
       r = self
       gsheet_tsv = r.gsheet(gdrive_slot).to_tsv
       #cache in DB
-      r.dataset.write_cache(gsheet_tsv)
+      r.cache.write(gsheet_tsv)
       #turn it into a hash array
       gsheet_jobs = gsheet_tsv.tsv_to_hash_array
       #go through each job, update relevant job with its params

data/lib/mobilize-base/models/stage.rb

@@ -7,6 +7,7 @@ module Mobilize
     field :call, type: String
     field :param_string, type: Array
     field :status, type: String
+    field :out_url, type: String
     field :completed_at, type: Time
     field :started_at, type: Time
     field :failed_at, type: Time
@@ -19,19 +20,12 @@ module Mobilize
       s.path.split("/").last.gsub("stage","").to_i
     end
 
-    def stdout_dataset
+    def out_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
       s = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/stdout")
-    end
-
-    def stderr_dataset
-      s = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/stderr")
-    end
-
-    def log_dataset
-      s = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/log")
+      Dataset.find_by_url(s.out_url) if s.out_url
     end
 
     def params
@@ -79,27 +73,21 @@ module Mobilize
       j = s.job
       s.update_attributes(:started_at=>Time.now.utc)
       s.update_status(%{Starting at #{Time.now.utc}})
-      stdout, stderr = [nil,nil]
       begin
-        stdout,log = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path).to_s
-        #write to log if method returns an array w 2 members
-        s.log_dataset.write_cache(log) if log
+        #get response by running method
+        s.out_url = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
+        s.save!
+        unless s.out_url
+          #re-queue self if no response
+          s.enqueue!
+          return false
+        end
       rescue ScriptError, StandardError => exc
-        stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
-        #record the failure in Job so it appears on Runner, turn it off
-        #so it doesn't run again
         j.update_attributes(:active=>false)
         s.update_attributes(:failed_at=>Time.now.utc)
         s.update_status("Failed at #{Time.now.utc.to_s}")
         raise exc
       end
-      if stdout == false
-        #re-queue self if output is false
-        s.enqueue!
-        return false
-      end
-      #write output to cache
-      s.stdout_dataset.write_cache(stdout)
       s.update_attributes(:completed_at=>Time.now.utc)
       s.update_status("Completed at #{Time.now.utc.to_s}")
       if s.idx == j.stages.length

data/lib/mobilize-base/models/user.rb

@@ -24,7 +24,7 @@ module Mobilize
 
     def jobs(jname=nil)
       u = self
-      return u.runners.map{|r| r.jobs(jname)}.flatten
+      return u.runner.jobs
     end
 
     def runner_path

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.84"
+    VERSION = "1.0.91"
   end
 end

data/test/mobilize-base_test.rb

@@ -41,10 +41,10 @@ describe "Mobilize" do
     test_source_tsv = test_source_ha.hash_array_to_tsv
     test_source_sheet.write(test_source_tsv)
 
-    puts "add row to jobs sheet, wait 120s"
+    puts "add row to jobs sheet, wait 150s"
     test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
     jobs_sheet.add_or_update_rows(test_job_rows)
-    sleep 120
+    sleep 150
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
     test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
@@ -52,11 +52,11 @@ describe "Mobilize" do
 
     assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
 
-    puts "delete both output sheets, set first job to active=true"
+    puts "delete both output sheets, set first job to active=true, wait 120s"
     [test_target_sheet_1,test_target_sheet_2].each{|s| s.delete}
 
     jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
-    sleep 90
+    sleep 120
     
     test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.84
+  version: 1.0.91
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-19 00:00:00.000000000 Z
+date: 2013-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -238,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1770893729203810846
+      hash: -136156851409089699
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -247,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1770893729203810846
+      hash: -136156851409089699
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.24