mobilize-base 1.0.8 → 1.0.9

data/lib/mobilize-base/extensions/google_drive/worksheet.rb

@@ -137,8 +137,9 @@ module GoogleDrive
             if loc_v != rem_v
               if ['true','false'].include?(loc_v.downcase)
                 #google sheet upcases true and false. ignore
-              elsif loc_v.starts_with?('rp') and rem_v.starts_with?('Rp')
-                # some other math bs
+              elsif loc_v.to_s.downcase.gsub("-","").gsub(" ","")==rem_v.to_s.downcase.gsub("-","").gsub(" ","")
+                #supported currency, silently converted whether it's an actual currency or not
+                #put a backtick on it.
                 sheet[row_i+1,col_i+1] = %{'#{loc_v}}
                 re_col_vs << {'row_i'=>row_i+1,'col_i'=>col_i+1,'col_v'=>%{'#{loc_v}}}
               elsif (loc_v.to_s.count('e')==1 or loc_v.to_s.count('e')==0) and

data/lib/mobilize-base/handlers/gbook.rb

@@ -8,10 +8,10 @@ module Mobilize
       dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       #there should only be one book with each path, otherwise we have fail
       book = nil
-      if books.length>1 and dst.url.to_s.length>0
+      if books.length>1 and dst.http_url.to_s.length>0
         #some idiot process created a duplicate book.
         #Fix by renaming all but one with dst entry's key
-        dkey = dst.url.split("key=").last
+        dkey = dst.http_url.split("key=").last
         books.each do |b|
           bkey = b.resource_id.split(":").last
           if bkey == dkey
@@ -30,9 +30,9 @@ module Mobilize
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
       end
-      #always make sure book dataset URL is up to date
+      #always make sure book dataset http URL is up to date
       #and that book has admin acl
-      dst.update_attributes(:url=>book.human_url)
+      dst.update_attributes(:http_url=>book.human_url)
       book.add_admin_acl
       return book
     end

data/lib/mobilize-base/handlers/gfile.rb

@@ -28,7 +28,10 @@ module Mobilize
       return false unless gdrive_slot
       s = Stage.where(:path=>stage_path)
       gfile_path = s.params['file']
-      Gfile.find_by_path(gfile_path,gdrive_slot).read
+      out_tsv = Gfile.find_by_path(gfile_path,gdrive_slot).read
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,out_tsv)
     end
   end
 end

data/lib/mobilize-base/handlers/gridfs.rb

@@ -11,29 +11,29 @@ module Mobilize
       return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
     end
 
-    def Gridfs.read(path)
+    def Gridfs.read_by_dataset_path(dst_path)
       begin
-        zs=Gridfs.grid.open(path.gridsafe,'r').read
+        zs=Gridfs.grid.open(dst_path,'r').read
         return ::Zlib::Inflate.inflate(zs)
       rescue
         return nil
       end
     end
 
-    def Gridfs.write(path,string)
+    def Gridfs.write_by_dataset_path(dst_path,string)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      curr_zs = Gridfs.read(path.gridsafe).to_s
+      curr_zs = Gridfs.read_by_dataset_path(dst_path).to_s
       #write a new version when there is a change
       if curr_zs != zs
-        Gridfs.grid.open(path.gridsafe,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
+        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
       end
       return true
     end
 
-    def Gridfs.delete(path)
+    def Gridfs.delete(dst_path)
       begin
-        Gridfs.grid.delete(path.gridsafe)
+        Gridfs.grid.delete(dst_path)
         return true
       rescue
         return nil

data/lib/mobilize-base/handlers/gsheet.rb

@@ -38,13 +38,16 @@ module Mobilize
       return false unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
       gsheet_path = s.params['source']
-      Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
+      out_tsv = Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,out_tsv)
     end
 
     def Gsheet.write_by_stage_path(stage_path)
       gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-      #return false if there are no emails available
-      return false unless gdrive_slot
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
       s = Stage.where(:path=>stage_path).first
       source = s.params['source']
       target_path = s.params['target']
@@ -55,7 +58,7 @@ module Mobilize
                                           end
       source_stage_path = "#{s.job.runner.path}/#{source_job_name || s.job.name}/#{source_stage_name}"
       source_stage = Stage.where(:path=>source_stage_path).first
-      tsv = source_stage.stdout_dataset.read_cache
+      tsv = source_stage.out_dst.read
       sheet_name = target_path.split("/").last
       temp_path = [stage_path.gridsafe,sheet_name].join("/")
       temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
@@ -65,8 +68,11 @@ module Mobilize
       target_sheet.merge(temp_sheet)
       #delete the temp sheet's book
       temp_sheet.spreadsheet.delete
-      "Write successful for #{target_path}".oputs
-      return true
+      status = "Write successful for #{target_path}"
+      s.update_status(status)
+      #use Gridfs to cache result
+      out_url = "gridfs://#{s.path}/out"
+      Dataset.write_to_url(out_url,status)
     end
   end
 end

data/lib/mobilize-base/handlers/resque.rb

@@ -51,12 +51,12 @@ module Mobilize
 
     def Resque.active_paths
       #first argument of the payload is the runner / stage path unless the worker is Jobtracker
-      Resque.jobs('active').map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
+      Resque.jobs('active').compact.map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
     end
 
     #Resque workers and methods to find
     def Resque.find_worker_by_path(path)
-      Resque.workers('working').select{|w| w.job['payload'] and w.job['payload']['args'].first == path}.first
+      Resque.workers('working').select{|w| w.job and w.job['payload'] and w.job['payload']['args'].first == path}.first
     end
 
     def Resque.set_worker_args_by_path(path,args)

data/lib/mobilize-base/jobtracker.rb

@@ -89,7 +89,7 @@ module Mobilize
 
     def Jobtracker.start
       if Jobtracker.status!='stopped'
-        raise "Jobtracker still #{Jobtracker.status}"
+        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
       else
         #make sure that workers are running and at the right number
         #Resque.prep_workers
@@ -190,7 +190,9 @@ module Mobilize
       while Jobtracker.status != 'stopping'
         users = User.all
         Jobtracker.run_notifications
-        users.each do |u|
+        #run throush all users randomly
+        #so none are privileged on JT restarts
+        users.sort_by{rand}.each do |u|
           r = u.runner
           Jobtracker.update_status("Checking #{r.path}")
           if r.is_due?

data/lib/mobilize-base/models/dataset.rb

@@ -4,7 +4,7 @@ module Mobilize
     include Mongoid::Timestamps
     field :handler, type: String
     field :path, type: String
-    field :url, type: String
+    field :http_url, type: String
     field :raw_size, type: Fixnum
     field :last_cached_at, type: Time
     field :last_cache_handler, type: String
@@ -18,6 +18,16 @@ module Mobilize
       return "Mobilize::#{dst.handler.humanize}".constantize.read_by_path(dst.path)
     end
 
+    def Dataset.find_by_url(url)
+      handler,path = url.split("://")
+      Dataset.find_by_handler_and_path(handler,path)
+    end
+
+    def Dataset.find_or_create_by_url(url)
+      handler,path = url.split("://")
+      Dataset.find_or_create_by_handler_and_path(handler,path)
+    end
+
     def Dataset.find_by_handler_and_path(handler,path)
       Dataset.where(handler: handler, path: path).first
     end
@@ -28,41 +38,30 @@ module Mobilize
       return dst
     end
 
-    def write(string)
-      dst = self
-      "Mobilize::#{dst.handler.humanize}".constantize.write_by_path(dst.path,string)
-      dst.raw_size = string.length
-      dst.save!
-      return true
+    def Dataset.write_to_url(url,string)
+      dst = Dataset.find_or_create_by_url(url)
+      dst.write(string)
+      url
     end
 
-    def cache_valid?
+    def read
       dst = self
-      return true if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
+      dst.update_attributes(:last_read_at=>Time.now.utc)
+      "Mobilize::#{dst.handler.humanize}".constantize.read_by_dataset_path(dst.path)
     end
 
-    def read_cache(cache_handler="gridfs")
+    def write(string)
       dst = self
-      if cache_valid?
-        dst.update_attributes(:last_read_at=>Time.now.utc)
-        return "Mobilize::#{cache_handler.humanize}".constantize.read([dst.handler,dst.path].join("://"))
-      else
-        raise "Cache invalid or not found for #{cache_handler}://#{dst.path}"
-      end
+      "Mobilize::#{dst.handler.humanize}".constantize.write_by_dataset_path(dst.path,string)
+      dst.raw_size = string.length
+      dst.save!
+      return true
     end
 
-    def write_cache(string,expire_at=nil,cache_handler="gridfs")
+    def delete
       dst = self
-      "Mobilize::#{cache_handler.humanize}".constantize.write([dst.handler,dst.path].join("://"),string)
-      dst.update_attributes(:last_cached_at=>Time.now.utc,
-                            :last_cache_handler=>cache_handler.to_s.downcase,
-                            :cache_expire_at=>expire_at,
-                            :size=>string.length)
+      "Mobilize::#{dst.handler.humanize}".constantize.delete_by_dataset_path(dst.path)
       return true
     end
-
-    def delete_cache(cache_handler="gridfs")
-      return "Mobilize::#{cache_handler.humanize}".constantize.delete(dst.handler, dst.path)
-    end
   end
 end

data/lib/mobilize-base/models/runner.rb

@@ -71,9 +71,9 @@ module Mobilize
       Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
     end
 
-    def read_cache
+    def cache
       r = self
-      r.dataset.read_cache
+      Dataset.find_or_create_by_url("gridfs://#{r.path}")
     end
 
     def gsheet(gdrive_slot)
@@ -88,7 +88,7 @@ module Mobilize
       r = self
       gsheet_tsv = r.gsheet(gdrive_slot).to_tsv
       #cache in DB
-      r.dataset.write_cache(gsheet_tsv)
+      r.cache.write(gsheet_tsv)
       #turn it into a hash array
       gsheet_jobs = gsheet_tsv.tsv_to_hash_array
       #go through each job, update relevant job with its params
@@ -151,7 +151,7 @@ module Mobilize
     def user
       r = self
       user_name = r.path.split("_").second.split("(").first.split("/").first
-      User.where(:email=>[user_name,Gdrive.domain].join("@")).first
+      User.where(:name=>user_name).first
     end
 
     def update_status(msg)

data/lib/mobilize-base/models/stage.rb

@@ -7,6 +7,7 @@ module Mobilize
     field :call, type: String
     field :param_string, type: Array
     field :status, type: String
+    field :out_url, type: String
     field :completed_at, type: Time
     field :started_at, type: Time
     field :failed_at, type: Time
@@ -19,19 +20,12 @@ module Mobilize
       s.path.split("/").last.gsub("stage","").to_i
     end
 
-    def stdout_dataset
+    def out_dst
+      #this gives a dataset that points to the output
+      #allowing you to determine its size
+      #before committing to a read or write
       s = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/stdout")
-    end
-
-    def stderr_dataset
-      s = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/stderr")
-    end
-
-    def log_dataset
-      s = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/log")
+      Dataset.find_by_url(s.out_url) if s.out_url
     end
 
     def params
@@ -79,27 +73,21 @@ module Mobilize
       j = s.job
       s.update_attributes(:started_at=>Time.now.utc)
       s.update_status(%{Starting at #{Time.now.utc}})
-      stdout, stderr = [nil,nil]
       begin
-        stdout,log = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path).to_s
-        #write to log if method returns an array w 2 members
-        s.log_dataset.write_cache(log) if log
+        #get response by running method
+        s.out_url = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
+        s.save!
+        unless s.out_url
+          #re-queue self if no response
+          s.enqueue!
+          return false
+        end
       rescue ScriptError, StandardError => exc
-        stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
-        #record the failure in Job so it appears on Runner, turn it off
-        #so it doesn't run again
         j.update_attributes(:active=>false)
         s.update_attributes(:failed_at=>Time.now.utc)
         s.update_status("Failed at #{Time.now.utc.to_s}")
         raise exc
       end
-      if stdout == false
-        #re-queue self if output is false
-        s.enqueue!
-        return false
-      end
-      #write output to cache
-      s.stdout_dataset.write_cache(stdout)
       s.update_attributes(:completed_at=>Time.now.utc)
       s.update_status("Completed at #{Time.now.utc.to_s}")
       if s.idx == j.stages.length

data/lib/mobilize-base/models/user.rb

@@ -24,7 +24,7 @@ module Mobilize
 
     def jobs(jname=nil)
       u = self
-      return u.runners.map{|r| r.jobs(jname)}.flatten
+      return u.runner.jobs
     end
 
     def runner_path

data/lib/mobilize-base/tasks.rb

@@ -8,6 +8,7 @@ namespace :mobilize_base do
         require e
       end
     rescue Exception=>exc
+      #do nothing
     end
 
     begin
@@ -29,6 +30,11 @@ namespace :mobilize_base do
   task :kill_idle_and_stale_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_idle_and_stale_workers
+  end
+    desc "Kill idle workers"
+    task :kill_idle_workers do
+    require 'mobilize-base'
+    Mobilize::Jobtracker.kill_idle_workers
   end
   desc "Make sure there are the correct # of workers, kill if too many"
   task :prep_workers do
@@ -45,6 +51,11 @@ namespace :mobilize_base do
     require 'mobilize-base'
     Mobilize::Jobtracker.start
   end
+  desc "Restart Jobtracker"
+  task :restart do
+    require 'mobilize-base'
+    Mobilize::Jobtracker.restart!
+  end
   desc "kill all old resque web processes, start new one with resque_web.rb extension file"
   task :resque_web do
     require 'mobilize-base'

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.8"
+    VERSION = "1.0.9"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-16 00:00:00.000000000 Z
+date: 2012-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -238,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 469924355258174246
+      hash: 576426973092652786
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -247,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 469924355258174246
+      hash: 576426973092652786
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.24