mobilize-base 1.3 → 1.21

data/README.md

@@ -220,8 +220,9 @@ production:
 
 gsheet.yml needs:
 * max_cells, which is the number of cells a sheet is allowed to have
-  written to it at one time. Default is 50k cells, which is about how
-much you can write before things start breaking.
+  written to it at one time. Default is 400k cells, which is the max per
+  book. Google Drive will throw its own exception if
+  you try to write more than that.
 * Because Google Docs ties date formatting to the Locale for the
   spreadsheet, there are 2 date format parameters:
   * read_date_format, which is the format that should be read FROM google
@@ -355,16 +356,22 @@ mobilize_base:resque_web task, as detailed in [Start Resque-Web](#section_Start_
 Mobilize stores cached data in MongoDB Gridfs. 
 It needs the below parameters, which can be found in the [lib/samples][git_samples] folder. 
 
+* max_versions - the number of __different__ versions of data to keep
+for a given cache. Default is 10. This is meant mostly to allow you to
+restore Runners from cache if necessary.
 * max_compressed_write_size - the amount of compressed data Gridfs will
 allow. If you try to write more than this, an exception will be thrown.
 
 ``` yml
 ---
 development:
+  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 test:
+  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 production:
+  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 ```
 
@@ -557,14 +564,8 @@ the Runner itself.
 and "base1.out" for the second test. The first
 takes the output from the first stage and the second reads it straight
 from the referenced sheet.
-    * All stages accept retry parameters:
-      * retries: an integer specifying the number of times that the system will try it again before giving up.
-      * delay: an integer specifying the number of seconds between retries.
-      * always_on: if false, turns the job off on stage failures.
-Otherwise the job will retry from the beginning with the same frequency as the Runner refresh rate.
-      * notify: by default, the stage owner will be notified on failure.
-          * if false, will not notify the stage owner in the event of a failure. 
-          * If it's an email address, will email the specified person.
+    * All stages accept a "retries" parameter, which is an integer specifying the number of times that the system will try it again before
+giving up.
     * If a stage fails after all retries, it will output its standard error to a tab in the Runner with the name of the job, the name of the stage, and a ".err" extension
       * The tab will be headed "response" and will contain the exception and backtrace for the error.
     * The test uses "Requestor_mobilize(test)/base1.out" and

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -13,7 +13,7 @@ module GoogleDrive
       f = self
       #admin includes workers
       return true if f.has_admin_acl?
-      accounts = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).uniq
+      accounts = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails)
       accounts.each do |email|
         f.update_acl(email)
       end
@@ -21,9 +21,9 @@ module GoogleDrive
 
     def has_admin_acl?
       f = self
-      curr_emails = f.acls.map{|a| a.scope}.compact.sort
-      admin_emails = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).uniq
-      if curr_emails == admin_emails or (curr_emails & admin_emails) == admin_emails
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      admin_emails = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails)
+      if (curr_emails & admin_emails) == admin_emails
         return true
       else
         return false
@@ -32,9 +32,9 @@ module GoogleDrive
 
     def has_worker_acl?
       f = self
-      curr_emails = f.acls.map{|a| a.scope}.compact.sort
+      curr_emails = f.acls.map{|a| a.scope}.sort
       worker_emails = Mobilize::Gdrive.worker_emails.sort
-      if curr_emails == worker_emails or (curr_emails & worker_emails) == worker_emails
+      if (curr_emails & worker_emails) == worker_emails
         return true
       else
         return false
@@ -84,7 +84,7 @@ module GoogleDrive
     end
     def acl_entry(email)
       f = self
-      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope and a.scope == email}.first
+      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
     end
     def entry_hash
       f = self

data/lib/mobilize-base/extensions/google_drive/worksheet.rb

@@ -1,6 +1,6 @@
 module GoogleDrive
   class Worksheet
-    def to_tsv(gsub_line_breaks="\n")
+    def to_tsv
       sheet = self
       rows = sheet.rows
       header = rows.first
@@ -8,12 +8,7 @@ module GoogleDrive
       #look for blank cols to indicate end of row
       col_last_i = (header.index("") || header.length)-1
       #ignore user-entered line breaks for purposes of tsv reads
-      out_tsv = rows.map do |r|
-                             row = r[0..col_last_i].join("\t")
-                             row.gsub!("\n",gsub_line_breaks)
-                             row = row + "\n"
-                             row
-                         end.join + "\n"
+      out_tsv = rows.map{|r| r[0..col_last_i].join("\t").gsub("\n","")+"\n"}.join + "\n"
       out_tsv.tsv_convert_dates(Mobilize::Gsheet.config['sheet_date_format'],
                                 Mobilize::Gsheet.config['read_date_format'])
     end

data/lib/mobilize-base/extensions/string.rb

@@ -11,19 +11,11 @@ class String
   def opp
     pp self
   end
-  def to_md5
-    Digest::MD5.hexdigest(self)
-  end
   def bash(except=true)
-    str = self
-    out_str,err_str = []
-    status = Open4.popen4(str) do |pid,stdin,stdout,stderr|
-      out_str = stdout.read
-      err_str = stderr.read
-    end
-    exit_status = status.exitstatus
-    raise err_str if (exit_status !=0 and except==true)
-    return out_str
+    pid,stdin,stdout,stderr = Open4.popen4(self)
+    pid,stdin = [nil,nil]
+    raise stderr.read if (stderr.read.length>0 and except==true)
+    return stdout.read
   end
   def escape_regex
     str = self

data/lib/mobilize-base/extensions/yaml.rb

@@ -10,16 +10,13 @@ module YAML
       #make sure urls have their colon spaces fixed
       result_hash={}
       easy_hash.each do |k,v|
-        #fucking yaml puts spaces in front of the key
-        #or something
-        strip_k = k.strip
-        result_hash[strip_k] = if v.class==String
-                                 v.gsub(": //","://")
-                               elsif v.class==Array
-                                 v.map{|av| av.to_s.gsub(": //","://")}
-                               else
-                                 v
-                               end
+        result_hash[k] = if v.class==String
+                           v.gsub(": //","://")
+                         elsif v.class==Array
+                           v.map{|av| av.to_s.gsub(": //","://")}
+                         else
+                           v
+                         end
       end
       return result_hash
     end

data/lib/mobilize-base/handlers/gbook.rb

@@ -14,44 +14,51 @@ module Mobilize
       dst = Dataset.find_by_handler_and_path('gbook',path)
       if dst and dst.http_url.to_s.length>0
         book = Gbook.find_by_http_url(dst.http_url,gdrive_slot)
-        if book
-          return book
+        #doesn't count if it's deleted
+        if book.entry_hash[:deleted]
+          book = nil
         else
-          raise "Could not find book #{path} with url #{dst.http_url}, please check dataset"
+          return book
         end
       end
-      #try to find books by title
       books = Gbook.find_all_by_path(path,gdrive_slot)
-      #sort by publish date; if entry hash retrieval fails (as it does)
-      #assume the book was published now
-      book = books.sort_by{|b| begin b.entry_hash[:published];rescue;Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z");end;}.first
-      if book
-        #we know dataset will have blank url since it wasn't picked up above
-        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
-        api_url = book.human_url.split("&").first
-        dst.update_attributes(:http_url=>api_url)
+      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+      book = nil
+      if books.length>1 and dst.http_url.to_s.length>0
+        #some idiot process or malicious user created a duplicate book.
+        #Fix by deleting all but the one with dst entry's key
+        dkey = dst.http_url.split("key=").last
+        books.each do |b|
+          bkey = b.resource_id.split(":").last
+          if bkey == dkey
+            book = b
+            dst.update_attributes(:http_url=>book.human_url)
+          else
+            #delete the invalid book
+            b.delete
+            ("Deleted duplicate book #{path}").oputs
+          end
+        end
+      else
+        #If it's a new dst or if there are multiple books
+        #take the first
+        book = books.first
+        dst.update_attributes(:http_url=>book.human_url) if book
       end
       return book
     end
-
     def Gbook.find_or_create_by_path(path,gdrive_slot)
       book = Gbook.find_by_path(path,gdrive_slot)
+      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       if book.nil?
         #always use owner email to make sure all books are owned by owner account
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
-        #check to make sure the dataset has a blank url; if not, error out
-        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
-        if dst.http_url.to_s.length>0
-          #add acls to book regardless
-          book.add_admin_acl
-          raise "Book #{path} is already assigned to #{dst.http_url}; please update the record with #{book.human_url}"
-        else
-          api_url = book.human_url.split("&").first
-          dst.update_attributes(:http_url=>api_url)
-          book.add_admin_acl
-        end
       end
+      #always make sure book dataset http URL is up to date
+      #and that book has admin acl
+      dst.update_attributes(:http_url=>book.human_url)
+      book.add_admin_acl
       return book
     end
   end

data/lib/mobilize-base/handlers/gfile.rb

@@ -1,6 +1,6 @@
 module Mobilize
   module Gfile
-    def Gfile.path_to_dst(path,stage_path,gdrive_slot)
+    def Gfile.path_to_dst(path,stage_path)
       #don't need the ://
       path = path.split("://").last if path.index("://")
       if Gfile.find_by_path(path)
@@ -38,8 +38,7 @@ module Mobilize
       end
       #update http url for file
       dst = Dataset.find_by_handler_and_path("gfile",dst_path)
-      api_url = file.human_url.split("&").first
-      dst.update_attributes(:http_url=>api_url)
+      dst.update_attributes(:http_url=>file.human_url)
       true
     end
 
@@ -87,8 +86,7 @@ module Mobilize
       #always make sure dataset http URL is up to date
       #and that it has admin acl
       if file
-        api_url = file.human_url.split("&").first
-        dst.update_attributes(:http_url=>api_url)
+        dst.update_attributes(:http_url=>file.human_url)
         file.add_admin_acl
       end
       return file

data/lib/mobilize-base/handlers/gridfs.rb

@@ -1,38 +1,43 @@
-require 'tempfile'
 module Mobilize
   module Gridfs
     def Gridfs.config
       Base.config('gridfs')
     end
 
-    def Gridfs.read_by_dataset_path(dst_path,*args)
-      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
-      zs = curr_file.data if curr_file
-      return ::Zlib::Inflate.inflate(zs) if zs.to_s.length>0
+    def Gridfs.grid
+      session = ::Mongoid.configure.sessions['default']
+      database_name = session['database']
+      host,port = session['hosts'].first.split(":")
+      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
     end
 
-    def Gridfs.write_by_dataset_path(dst_path,string,*args)
+    def Gridfs.read_by_dataset_path(dst_path,user_name,*args)
+      begin
+        zs=Gridfs.grid.open(dst_path,'r').read
+        return ::Zlib::Inflate.inflate(zs)
+      rescue
+        return nil
+      end
+    end
+
+    def Gridfs.write_by_dataset_path(dst_path,string,user_name,*args)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      #find and delete existing file
-      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
-      curr_zs =  curr_file.data if curr_file
-      #overwrite when there is a change
+      curr_zs = Gridfs.read_by_dataset_path(dst_path,user_name).to_s
+      #write a new version when there is a change
       if curr_zs != zs
-        Mongoid::GridFs.delete(curr_file.id) if curr_file
-        #create temp file w zstring
-        temp_file = ::Tempfile.new("#{string}#{Time.now.to_f}".to_md5)
-        temp_file.print(zs)
-        temp_file.close
-        #put data in file
-        Mongoid::GridFs.put(temp_file.path,:filename=>dst_path)
+        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
       end
       return true
     end
 
     def Gridfs.delete(dst_path)
-      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
-      curr_file.delete
+      begin
+        Gridfs.grid.delete(dst_path)
+        return true
+      rescue
+        return nil
+      end
     end
   end
 end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -10,10 +10,12 @@ module Mobilize
     end
 
     # converts a source path or target path to a dst in the context of handler and stage
-    def Gsheet.path_to_dst(path,stage_path,gdrive_slot)
+    def Gsheet.path_to_dst(path,stage_path)
       s = Stage.where(:path=>stage_path).first
       params = s.params
       target_path = params['target']
+      #take random slot if one is not available
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path) || Gdrive.worker_emails.sort_by{rand}.first
       #if this is the target, it doesn't have to exist already
       is_target = true if path == target_path
       #don't need the ://
@@ -44,7 +46,9 @@ module Mobilize
 
     def Gsheet.read_by_dataset_path(dst_path,user_name,*args)
       #expects gdrive slot as first arg, otherwise chooses random
-      gdrive_slot = args.to_a.first
+      gdrive_slot = args
+      worker_emails = Gdrive.worker_emails.sort_by{rand}
+      gdrive_slot = worker_emails.first unless worker_emails.include?(gdrive_slot)
       sheet = Gsheet.find_by_path(dst_path,gdrive_slot)
       sheet.read(user_name) if sheet
     end
@@ -52,6 +56,8 @@ module Mobilize
     def Gsheet.write_by_dataset_path(dst_path,tsv,user_name,*args)
       #expects gdrive slot as first arg, otherwise chooses random
       gdrive_slot,crop = args
+      worker_emails = Gdrive.worker_emails.sort_by{rand}
+      gdrive_slot = worker_emails.first unless worker_emails.include?(gdrive_slot)
       crop ||= true
       Gsheet.write_target(dst_path,tsv,user_name,gdrive_slot,crop)
     end
@@ -81,16 +87,15 @@ module Mobilize
 
     def Gsheet.write_temp(target_path,gdrive_slot,tsv)
       #find and delete temp sheet, if any
-      temp_book_title = target_path.gridsafe
-      #create book and sheet
-      temp_book = Gdrive.root(gdrive_slot).create_spreadsheet(temp_book_title)
-      rows, cols = tsv.split("\n").ie{|t| [t.length,t.first.split("\t").length]}
-      temp_sheet = temp_book.add_worksheet("temp",rows,cols)
+      temp_path = [target_path.gridsafe,"temp"].join("/")
+      temp_sheet = Gsheet.find_by_path(temp_path,gdrive_slot)
+      temp_sheet.delete if temp_sheet
+      #write data to temp sheet
+      temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
       #this step has a tendency to fail; if it does,
       #don't fail the stage, mark it as false
       begin
-        gdrive_user = gdrive_slot.split("@").first
-        temp_sheet.write(tsv,gdrive_user)
+        temp_sheet.write(tsv,Gdrive.owner_name)
       rescue
         return nil
       end
@@ -109,7 +114,7 @@ module Mobilize
         #only give the user edit permissions if they're the ones
         #creating it
         target_sheet = Gsheet.find_or_create_by_path(target_path,gdrive_slot)
-        target_sheet.spreadsheet.update_acl(u.email,"writer") unless target_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
+        target_sheet.spreadsheet.update_acl(user_email,"writer") unless target_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
         target_sheet.delete_sheet1
       end
       #pass it crop param to determine whether to shrink target sheet to fit data
@@ -129,24 +134,14 @@ module Mobilize
       crop = s.params['crop'] || true
       begin
         #get tsv to write from stage
-        source = s.sources(gdrive_slot).first
+        source = s.sources.first
         raise "Need source for gsheet write" unless source
         tsv = source.read(u.name,gdrive_slot)
-        raise "No data source found for #{source.url}" unless tsv
-        tsv_row_count = tsv.to_s.split("\n").length
-        tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
-        tsv_cell_count = tsv_row_count * tsv_col_count
-        stdout = if tsv_row_count == 0
-                   #soft error; no data to write. Stage will complete.
-                   "Write skipped for #{s.target.url}"
-                 elsif tsv_cell_count > Gsheet.max_cells
-                   raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
-                 else
-                   Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
-                   #update status
-                   "Write successful for #{s.target.url}"
-                 end
+        raise "No data found in #{source.url}" unless tsv
+        Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
         Gdrive.unslot_worker_by_path(stage_path)
+        #update status
+        stdout = "Write successful for #{s.target.url}"
         stderr = nil
         s.update_status(stdout)
         signal = 0

data/lib/mobilize-base/handlers/resque.rb

@@ -25,7 +25,7 @@ module Mobilize
       return idle_workers if state == 'idle'
       stale_workers = workers.select{|w| Time.parse(w.started) < Jobtracker.deployed_at}
       return stale_workers if state == 'stale'
-      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)}
+      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
       return timeout_workers if state == 'timeout'
       raise "invalid state #{state}"
     end
@@ -109,28 +109,16 @@ module Mobilize
       Resque.failures.each_with_index do |f,f_i|
         #skip if already notified
         next if f['notified']
-        #try to send message to stage owner, where appropriate
         stage_path = f['payload']['args'].first
-        email = begin
-                  s = Stage.where(:path=>stage_path).first
-                  if s.params['notify'].to_s=="false"
-                    next
-                  elsif s.params['notify'].index("@")
-                    s.params['notify']
-                  else
-                    s.job.runner.user.email
-                  end
-                rescue
-                  #jobs without stages are sent to first admin
-                  Jobtracker.admin_emails.first
-                end
+        s = Stage.where(:path=>stage_path).first
+        email = s.job.runner.user.email
         exc_to_s = f['error']
         if fjobs[email].nil?
           fjobs[email] = {stage_path => {exc_to_s => 1}}
         elsif fjobs[email][stage_path].nil?
           fjobs[email][stage_path] = {exc_to_s => 1}
         elsif fjobs[email][stage_path][exc_to_s].nil?
-          fjobs[email][stage_path][exc_to_s] = 1
+          fjobs[email][stage_path][exc_to_s] = 1        
         else
           fjobs[email][stage_path][exc_to_s] += 1
         end

data/lib/mobilize-base/jobtracker.rb

@@ -48,8 +48,6 @@ module Mobilize
     end
 
     def Jobtracker.update_status(msg)
-      #this is to keep jobtracker from resisting stop commands
-      return false if Jobtracker.status=="stopping"
       #Jobtracker has no persistent database state
       Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
       return true
@@ -124,7 +122,7 @@ module Mobilize
       sleep 5
       i=0
       while Jobtracker.status=='stopping'
-        puts "#{Jobtracker.to_s} still on queue, waiting"
+        Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
         sleep 5
         i+=1
       end
@@ -147,8 +145,8 @@ module Mobilize
     def Jobtracker.max_run_time_workers
       #return workers who have been cranking away for 6+ hours
         workers = Jobtracker.workers('working').select do |w|
-            w.job['run_at'].to_s.length>0 and 
-              (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
+            w.job['runat'].to_s.length>0 and 
+              (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
         end
         return workers
     end
@@ -187,18 +185,13 @@ module Mobilize
         if lws.length>0
           n = {}
           n['subject'] = "#{lws.length.to_s} max run time jobs"
-          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
+          n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
           n['to'] = Jobtracker.admin_emails.join(",")
           notifs << n
         end
         #deliver each email generated
         notifs.each do |notif|
-          begin
-            Email.write(notif).deliver
-          rescue
-            #log email on failure
-            Jobtracker.update_status("Failed to deliver #{notif.to_s}")
-          end
+          Email.write(notif).deliver
         end
         #update notification time so JT knows to wait a while
         Jobtracker.last_notification = Time.now.utc.to_s
@@ -293,7 +286,6 @@ module Mobilize
       # delete any old runner from previous test runs
       gdrive_slot = Gdrive.owner_email
       u.runner.gsheet(gdrive_slot).spreadsheet.delete
-      Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
       Jobtracker.update_status("enqueue jobtracker, wait 45s")
       Mobilize::Jobtracker.start
       sleep 45

data/lib/mobilize-base/models/job.rb

@@ -2,54 +2,66 @@ module Mobilize
   class Job
     include Mongoid::Document
     include Mongoid::Timestamps
-    include Mobilize::JobHelper
     field :path, type: String
     field :active, type: Boolean
     field :trigger, type: String
 
     index({ path: 1})
 
+    def name
+      j = self
+      j.path.split("/").last
+    end
+
+    def stages
+      j = self
+      #starts with the job path, followed by a slash
+      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
+    end
+
     def Job.find_or_create_by_path(path)
       j = Job.where(:path=>path).first
       j = Job.create(:path=>path) unless j
       return j
     end
 
-    #takes a hash of job parameters (name, active, trigger, stages)
-    #and creates/updates a job with it
-    def Job.update_by_user_name_and_hash(user_name,hash)
-      u = User.where(name: user_name).first
-      r = u.runner
-      j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
-      #update top line params
-      j.update_attributes(:active => hash['active'],
-                          :trigger => hash['trigger'])
-      (1..5).to_a.each do |s_idx|
-        stage_string = hash["stage#{s_idx.to_s}"]
-        s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
-        if stage_string.to_s.length==0
-          #delete this stage and all stages after
-          if s
-            j = s.job
-            j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
-            #just in case
-            s.delete
-          end
-          break
-        elsif s.nil?
-          #create this stage
-          s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
-        end
-        #parse command string, update stage with it
-        s_handler, call, param_string = [""*3]
-        stage_string.split(" ").ie do |spls|
-          s_handler = spls.first.split(".").first
-          call = spls.first.split(".").last
-          param_string = spls[1..-1].join(" ").strip
-        end
-        s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
-      end
-      return j.reload
+    def status
+      #last stage status
+      j = self
+      j.active_stage.status if j.active_stage
+    end
+
+    def active_stage
+      j = self
+      #latest started at or first
+      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
+    end
+
+    def completed_at
+      j = self
+      j.stages.last.completed_at if j.stages.last
+    end
+
+    def failed_at
+      j = self
+      j.active_stage.failed_at if j.active_stage
+    end
+
+    def status_at
+      j = self
+      j.active_stage.status_at if j.active_stage
+    end
+
+    #convenience methods
+    def runner
+      j = self
+      runner_path = j.path.split("/")[0..-2].join("/")
+      return Runner.where(:path=>runner_path).first
+    end
+
+    def is_working?
+      j = self
+      j.stages.select{|s| s.is_working?}.compact.length>0
     end
 
     def is_due?