mobilize-base 1.29 → 1.33

data/README.md

@@ -162,9 +162,15 @@ the same domain, and all Users should have emails in this domain.
 * an owner name and password. You can set up separate owners
   for different environments as in the below file, which will keep your
 mission critical workers from getting rate-limit errors.
+* one admin_group_name, which the owner and all admins should be added to -- this
+group will need read permissions to read from and edit permissions to write
+to files.
 * one or more admins with email attributes -- these will be for people
   who should be given write permissions to all Mobilize books in the
 environment for maintenance purposes.
+* one worker_group_name, which the owner and all workers should be added to -- this
+group will need read permissions to read from and edit permissions to write
+to files.
 * one or more workers with name and pw attributes -- they will be used
   to queue up google reads and writes. This can be the same as the owner
 account for testing purposes or low-volume environments. 
@@ -182,8 +188,10 @@ development:
   owner:
     name: owner_development
     pw: google_drive_password
+  admin_group_name: admins_development
   admins:
     - name: admin
+  worker_group_name: workers_development
   workers:
     - name: worker_development001 
       pw: worker001_google_drive_password
@@ -194,8 +202,10 @@ test:
   owner:
     name: owner_test
     pw: google_drive_password
+  admin_group_name: admins_test
   admins:
     - name: admin
+  worker_group_name: workers_test
   workers:
     - name: worker_test001 
       pw: worker001_google_drive_password
@@ -206,8 +216,10 @@ production:
   owner:
     name: owner_production
     pw: google_drive_password
+  admin_group_name: admins_production
   admins:
     - name: admin
+  worker_group_name: workers_production
   workers:
     - name: worker_production001 
       pw: worker001_google_drive_password
@@ -220,9 +232,8 @@ production:
 
 gsheet.yml needs:
 * max_cells, which is the number of cells a sheet is allowed to have
-  written to it at one time. Default is 400k cells, which is the max per
-  book. Google Drive will throw its own exception if
-  you try to write more than that.
+  written to it at one time. Default is 50k cells, which is about how
+much you can write before things start breaking.
 * Because Google Docs ties date formatting to the Locale for the
   spreadsheet, there are 2 date format parameters:
   * read_date_format, which is the format that should be read FROM google
@@ -356,22 +367,16 @@ mobilize_base:resque_web task, as detailed in [Start Resque-Web](#section_Start_
 Mobilize stores cached data in MongoDB Gridfs. 
 It needs the below parameters, which can be found in the [lib/samples][git_samples] folder. 
 
-* max_versions - the number of __different__ versions of data to keep
-for a given cache. Default is 10. This is meant mostly to allow you to
-restore Runners from cache if necessary.
 * max_compressed_write_size - the amount of compressed data Gridfs will
 allow. If you try to write more than this, an exception will be thrown.
 
 ``` yml
 ---
 development:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 test:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 production:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 ```
 
@@ -567,7 +572,11 @@ from the referenced sheet.
     * All stages accept retry parameters:
       * retries: an integer specifying the number of times that the system will try it again before giving up.
       * delay: an integer specifying the number of seconds between retries.
-      * always_on: if true, keeps the job on regardless of stage failures. The job will retry from the beginning with the same frequency as the Runner refresh rate.
+      * always_on: if false, turns the job off on stage failures.
+Otherwise the job will retry from the beginning with the same frequency as the Runner refresh rate.
+      * notify: by default, the stage owner will be notified on failure.
+          * if false, will not notify the stage owner in the event of a failure. 
+          * If it's an email address, will email the specified person.
     * If a stage fails after all retries, it will output its standard error to a tab in the Runner with the name of the job, the name of the stage, and a ".err" extension
       * The tab will be headed "response" and will contain the exception and backtrace for the error.
     * The test uses "Requestor_mobilize(test)/base1.out" and

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -13,7 +13,7 @@ module GoogleDrive
       f = self
       #admin includes workers
       return true if f.has_admin_acl?
-      accounts = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails)
+      accounts = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).uniq
       accounts.each do |email|
         f.update_acl(email)
       end
@@ -21,9 +21,9 @@ module GoogleDrive
 
     def has_admin_acl?
       f = self
-      curr_emails = f.acls.map{|a| a.scope}.sort
-      admin_emails = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails)
-      if (curr_emails & admin_emails) == admin_emails
+      curr_emails = f.acls.map{|a| a.scope}.compact.sort
+      admin_emails = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).uniq
+      if curr_emails == admin_emails or (curr_emails & admin_emails) == admin_emails
         return true
       else
         return false
@@ -32,9 +32,9 @@ module GoogleDrive
 
     def has_worker_acl?
       f = self
-      curr_emails = f.acls.map{|a| a.scope}.sort
+      curr_emails = f.acls.map{|a| a.scope}.compact.sort
       worker_emails = Mobilize::Gdrive.worker_emails.sort
-      if (curr_emails & worker_emails) == worker_emails
+      if curr_emails == worker_emails or (curr_emails & worker_emails) == worker_emails
         return true
       else
         return false
@@ -84,7 +84,7 @@ module GoogleDrive
     end
     def acl_entry(email)
       f = self
-      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
+      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope and a.scope == email}.first
     end
     def entry_hash
       f = self

data/lib/mobilize-base/extensions/resque-server/views/queues.erb

@@ -0,0 +1,59 @@
+<% @subtabs = resque.queues unless partial? || params[:id].nil? %>
+
+<% if queue = params[:id] %>
+
+  <h1>Pending jobs on <span class='hl'><%= queue %></span></h1>
+  <form method="POST" action="<%=u "/queues/#{queue}/remove" %>" class='remove-queue'>
+    <input type='submit' name='' value='Remove Queue' onclick='return confirm("Are you absolutely sure? This cannot be undone.");' />
+  </form>
+  <p class='sub'>Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%=size = resque.size(queue)%></b> jobs</p>
+  <table class='jobs'>
+    <tr>
+      <th>Class</th>
+      <th>Args</th>
+    </tr>
+    <% for job in (jobs = resque.peek(queue, start, 20)) %>
+    <tr>
+      <td class='class'><%= job['class'] %></td>
+      <td class='args'><%=h job['args'].inspect %></td>
+    </tr>
+    <% end %>
+    <% if jobs.empty? %>
+    <tr>
+      <td class='no-data' colspan='2'>There are no pending jobs in this queue</td>
+    </tr>
+    <% end %>
+  </table>
+  <%= partial :next_more, :start => start, :size => size, :per_page => 20 %>
+<% else %>
+
+  <h1 class='wi'>Queues</h1>
+  <p class='intro'>The list below contains all the registered queues with the number of jobs currently in the queue. Select a queue from above to view all jobs currently pending on the queue.</p>
+  <table class='queues'>
+    <tr>
+      <th>Name</th>
+      <th>Jobs</th>
+    </tr>
+    <!-- only show nonzero length queues-->
+    <% resque.queues.select{|q| resque.size(q)>0}.sort_by { |q| q.to_s }.each do |queue| %>
+    <tr>
+      <td class='queue'><a class="queue" href="<%= u "queues/#{queue}" %>"><%= queue %></a></td>
+      <td class='size'><%= resque.size queue %></td>
+    </tr>
+    <% end %>
+    <% if failed_multiple_queues? %>
+      <% Resque::Failure.queues.sort_by { |q| q.to_s }.each_with_index do |queue, i| %>
+      <tr class="<%= Resque::Failure.count(queue).zero? ? "failed" : "failure" %><%= " first_failure" if i.zero? %>">
+        <td class='queue failed'><a class="queue" href="<%= u "failed/#{queue}" %>"><%= queue %></a></td>
+        <td class='size'><%= Resque::Failure.count(queue) %></td>
+      </tr>
+      <% end %>
+    <% else %>
+      <tr class="<%= Resque::Failure.count.zero? ? "failed" : "failure" %>">
+        <td class='queue failed'><a class="queue" href="<%= u :failed %>">failed</a></td>
+        <td class='size'><%= Resque::Failure.count %></td>
+      </tr>
+    <% end %>
+  </table>
+
+<% end %>

data/lib/mobilize-base/extensions/resque-server/views/working.erb

@@ -0,0 +1,85 @@
+<% if params[:id] && (worker = Resque::Worker.find(params[:id])) && worker.job %>
+  <h1><%= worker %>'s job</h1>
+
+  <table>
+    <tr>
+      <th>&nbsp;</th>
+      <th>Where</th>
+      <th>Queue</th>
+      <th>Started</th>
+      <th>Class</th>
+      <th>Args</th>
+    </tr>
+      <tr>
+        <td><img src="<%=u 'working.png' %>" alt="working" title="working"></td>
+        <% host, pid, _ = worker.to_s.split(':') %>
+        <td><a href="<%=u "/workers/#{worker}" %>"><%= host %>:<%= pid %></a></td>
+        <% data = worker.job %>
+        <% queue = data['queue'] %>
+        <td><a class="queue" href="<%=u "/queues/#{queue}" %>"><%= queue %></a></td>
+        <td><span class="time"><%= data['run_at'] %></span></td>
+        <td>
+          <code><%= data['payload']['class'] %></code>
+        </td>
+        <td><%=h data['payload']['args'].inspect %></td>
+      </tr>
+  </table>
+
+<% else %>
+
+  <%
+    workers = resque.working
+    jobs = workers.collect {|w| w.job }
+    worker_jobs = workers.zip(jobs)
+    worker_jobs = worker_jobs.reject { |w, j| w.idle? }
+  %>
+
+  <h1 class='wi'><%= worker_jobs.size %> of <%= resque.workers.size %> Workers Working</h1>
+  <p class='intro'>The list below contains all workers which are currently running a job.</p>
+  <table class='workers'>
+    <tr>
+      <th>&nbsp;</th>
+      <th>Where</th>
+      <th>Queue</th>
+      <th>Processing</th>
+    </tr>
+    <% if worker_jobs.empty? %>
+    <tr>
+      <td colspan="4" class='no-data'>Nothing is happening right now...</td>
+    </tr>
+    <% end %>
+
+    <% worker_jobs.sort_by {|w, j| j['run_at'] ? j['run_at'] : '' }.each do |worker, job| %>
+      <tr>
+        <td class='icon'><img src="<%=u state = worker.state %>.png" alt="<%= state %>" title="<%= state %>"></td>
+        <% host, pid, queues = worker.to_s.split(':') %>
+        <td class='where'><a href="<%=u "/workers/#{worker}" %>"><%= host %>:<%= pid %></a></td>
+        <td class='queues queue'>
+          <a class="queue-tag" href="<%=u "/queues/#{job['queue']}" %>"><%= job['queue'] %></a>
+        </td>
+        <td class='process'>
+          <% if job['queue'] 
+               job_stats = begin
+                             j = job
+                             args_hash = j['payload']['args'][1]
+                             args_array = args_hash.map{|k,v| "#{k} : #{v}" }.join("</code><br><code>") if args_hash.class==Hash
+                             args = [args_array].compact.join("") 
+                             path = j['payload']['args'].first
+                             [path,args].join("</code><br><code>")
+                           rescue => exc
+                             [exc.to_s,exc.backtrace.join("<br>")].join("<br>")
+                           end
+           %>
+           <%=job_stats%>
+            </code>
+            <br>
+            <small><a class="queue time" href="<%=u "/working/#{worker}" %>"><%= job['run_at'] %></a></small>
+          <% else %>
+            <span class='waiting'>Waiting for a job...</span>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+<% end %>

data/lib/mobilize-base/extensions/string.rb

@@ -11,11 +11,19 @@ class String
   def opp
     pp self
   end
+  def to_md5
+    Digest::MD5.hexdigest(self)
+  end
   def bash(except=true)
-    pid,stdin,stdout,stderr = Open4.popen4(self)
-    pid,stdin = [nil,nil]
-    raise stderr.read if (stderr.read.length>0 and except==true)
-    return stdout.read
+    str = self
+    out_str,err_str = []
+    status = Open4.popen4(str) do |pid,stdin,stdout,stderr|
+      out_str = stdout.read
+      err_str = stderr.read
+    end
+    exit_status = status.exitstatus
+    raise err_str if (exit_status !=0 and except==true)
+    return out_str
   end
   def escape_regex
     str = self

data/lib/mobilize-base/extensions/yaml.rb

@@ -10,13 +10,16 @@ module YAML
       #make sure urls have their colon spaces fixed
       result_hash={}
       easy_hash.each do |k,v|
-        result_hash[k] = if v.class==String
-                           v.gsub(": //","://")
-                         elsif v.class==Array
-                           v.map{|av| av.to_s.gsub(": //","://")}
-                         else
-                           v
-                         end
+        #fucking yaml puts spaces in front of the key
+        #or something
+        strip_k = k.strip
+        result_hash[strip_k] = if v.class==String
+                                 v.gsub(": //","://")
+                               elsif v.class==Array
+                                 v.map{|av| av.to_s.gsub(": //","://")}
+                               else
+                                 v
+                               end
       end
       return result_hash
     end

data/lib/mobilize-base/handlers/gbook.rb

@@ -14,57 +14,44 @@ module Mobilize
       dst = Dataset.find_by_handler_and_path('gbook',path)
       if dst and dst.http_url.to_s.length>0
         book = Gbook.find_by_http_url(dst.http_url,gdrive_slot)
-        begin
-          #doesn't count if it's deleted
-          #or if its name can't be accessed
-          if book.entry_hash[:deleted]
-            book = nil
-          else
-            return book
-          end
-        rescue
-          #use regular process if book entry hash fails
-          book = nil
+        if book
+          return book
+        else
+          raise "Could not find book #{path} with url #{dst.http_url}, please check dataset"
         end
       end
+      #try to find books by title
       books = Gbook.find_all_by_path(path,gdrive_slot)
-      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
-      book = nil
-      if books.length>1 and dst.http_url.to_s.length>0
-        #some idiot process or malicious user created a duplicate book.
-        #Fix by deleting all but the one with dst entry's key
-        dkey = dst.http_url.split("key=").last
-        books.each do |b|
-          bkey = b.resource_id.split(":").last
-          if bkey == dkey
-            book = b
-            dst.update_attributes(:http_url=>book.human_url)
-          else
-            #delete the invalid book
-            b.delete
-            ("Deleted duplicate book #{path}").oputs
-          end
-        end
-      else
-        #If it's a new dst or if there are multiple books
-        #take the first
-        book = books.first
-        dst.update_attributes(:http_url=>book.human_url) if book
+      #sort by publish date; if entry hash retrieval fails (as it does)
+      #assume the book was published now
+      book = books.sort_by{|b| begin b.entry_hash[:published];rescue;Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z");end;}.first
+      if book
+        #we know dataset will have blank url since it wasn't picked up above
+        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+        api_url = book.human_url.split("&").first
+        dst.update_attributes(:http_url=>api_url)
       end
       return book
     end
+
     def Gbook.find_or_create_by_path(path,gdrive_slot)
       book = Gbook.find_by_path(path,gdrive_slot)
-      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       if book.nil?
         #always use owner email to make sure all books are owned by owner account
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
+        #check to make sure the dataset has a blank url; if not, error out
+        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+        if dst.http_url.to_s.length>0
+          #add acls to book regardless
+          book.add_admin_acl
+          raise "Book #{path} is already assigned to #{dst.http_url}; please update the record with #{book.human_url}"
+        else
+          api_url = book.human_url.split("&").first
+          dst.update_attributes(:http_url=>api_url)
+          book.add_admin_acl
+        end
       end
-      #always make sure book dataset http URL is up to date
-      #and that book has admin acl
-      dst.update_attributes(:http_url=>book.human_url)
-      book.add_admin_acl
       return book
     end
   end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -37,6 +37,14 @@ module Mobilize
       end
     end
 
+    def Gdrive.worker_group_name
+      Gdrive.config['worker_group_name']
+    end
+
+    def Gdrive.admin_group_name
+      Gdrive.config['admin_group_name']
+    end
+
     def Gdrive.worker_emails
       Gdrive.workers.map{|w| [w['name'],Gdrive.domain].join("@")}
     end

data/lib/mobilize-base/handlers/gfile.rb

@@ -38,7 +38,8 @@ module Mobilize
       end
       #update http url for file
       dst = Dataset.find_by_handler_and_path("gfile",dst_path)
-      dst.update_attributes(:http_url=>file.human_url)
+      api_url = file.human_url.split("&").first
+      dst.update_attributes(:http_url=>api_url)
       true
     end
 
@@ -86,7 +87,8 @@ module Mobilize
       #always make sure dataset http URL is up to date
       #and that it has admin acl
       if file
-        dst.update_attributes(:http_url=>file.human_url)
+        api_url = file.human_url.split("&").first
+        dst.update_attributes(:http_url=>api_url)
         file.add_admin_acl
       end
       return file

data/lib/mobilize-base/handlers/gridfs.rb

@@ -1,43 +1,38 @@
+require 'tempfile'
 module Mobilize
   module Gridfs
     def Gridfs.config
       Base.config('gridfs')
     end
 
-    def Gridfs.grid
-      session = ::Mongoid.configure.sessions['default']
-      database_name = session['database']
-      host,port = session['hosts'].first.split(":")
-      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
+    def Gridfs.read_by_dataset_path(dst_path,*args)
+      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
+      zs = curr_file.data if curr_file
+      return ::Zlib::Inflate.inflate(zs) if zs.to_s.length>0
     end
 
-    def Gridfs.read_by_dataset_path(dst_path,user_name,*args)
-      begin
-        zs=Gridfs.grid.open(dst_path,'r').read
-        return ::Zlib::Inflate.inflate(zs)
-      rescue
-        return nil
-      end
-    end
-
-    def Gridfs.write_by_dataset_path(dst_path,string,user_name,*args)
+    def Gridfs.write_by_dataset_path(dst_path,string,*args)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      curr_zs = Gridfs.read_by_dataset_path(dst_path,user_name).to_s
-      #write a new version when there is a change
+      #find and delete existing file
+      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
+      curr_zs =  curr_file.data if curr_file
+      #overwrite when there is a change
       if curr_zs != zs
-        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
+        Mongoid::GridFs.delete(curr_file.id) if curr_file
+        #create temp file w zstring
+        temp_file = ::Tempfile.new("#{string}#{Time.now.to_f}".to_md5)
+        temp_file.print(zs)
+        temp_file.close
+        #put data in file
+        Mongoid::GridFs.put(temp_file.path,:filename=>dst_path)
       end
       return true
     end
 
     def Gridfs.delete(dst_path)
-      begin
-        Gridfs.grid.delete(dst_path)
-        return true
-      rescue
-        return nil
-      end
+      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
+      curr_file.delete
     end
   end
 end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -81,15 +81,16 @@ module Mobilize
 
     def Gsheet.write_temp(target_path,gdrive_slot,tsv)
       #find and delete temp sheet, if any
-      temp_path = [target_path.gridsafe,"temp"].join("/")
-      temp_sheet = Gsheet.find_by_path(temp_path,gdrive_slot)
-      temp_sheet.delete if temp_sheet
-      #write data to temp sheet
-      temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
+      temp_book_title = target_path.gridsafe
+      #create book and sheet
+      temp_book = Gdrive.root(gdrive_slot).create_spreadsheet(temp_book_title)
+      rows, cols = tsv.split("\n").ie{|t| [t.length,t.first.split("\t").length]}
+      temp_sheet = temp_book.add_worksheet("temp",rows,cols)
       #this step has a tendency to fail; if it does,
       #don't fail the stage, mark it as false
       begin
-        temp_sheet.write(tsv,Gdrive.owner_name)
+        gdrive_user = gdrive_slot.split("@").first
+        temp_sheet.write(tsv,gdrive_user)
       rescue
         return nil
       end
@@ -132,9 +133,14 @@ module Mobilize
         raise "Need source for gsheet write" unless source
         tsv = source.read(u.name,gdrive_slot)
         raise "No data source found for #{source.url}" unless tsv
-        stdout = if tsv.to_s.length == 0
+        tsv_row_count = tsv.to_s.split("\n").length
+        tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
+        tsv_cell_count = tsv_row_count * tsv_col_count
+        stdout = if tsv_row_count == 0
                    #soft error; no data to write. Stage will complete.
                    "Write skipped for #{s.target.url}"
+                 elsif tsv_cell_count > Gsheet.max_cells
+                   raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
                  else
                    Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
                    #update status

data/lib/mobilize-base/handlers/resque.rb

@@ -25,7 +25,7 @@ module Mobilize
       return idle_workers if state == 'idle'
       stale_workers = workers.select{|w| Time.parse(w.started) < Jobtracker.deployed_at}
       return stale_workers if state == 'stale'
-      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
+      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)}
       return timeout_workers if state == 'timeout'
       raise "invalid state #{state}"
     end
@@ -113,8 +113,14 @@ module Mobilize
         stage_path = f['payload']['args'].first
         email = begin
                   s = Stage.where(:path=>stage_path).first
-                  s.job.runner.user.email
-                rescue
+                  if s.params['notify'].to_s=="false"
+                    next
+                  elsif s.params['notify'].index("@")
+                    s.params['notify']
+                  else
+                    s.job.runner.user.email
+                  end
+                rescue ScriptError, StandardError
                   #jobs without stages are sent to first admin
                   Jobtracker.admin_emails.first
                 end

data/lib/mobilize-base/helpers/job_helper.rb

@@ -0,0 +1,54 @@
+#this module adds convenience methods to the Job model
+module Mobilize
+  module JobHelper
+    def name
+      j = self
+      j.path.split("/").last
+    end
+
+    def stages
+      j = self
+      #starts with the job path, followed by a slash
+      Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
+    end
+
+    def status
+      #last stage status
+      j = self
+      j.active_stage.status if j.active_stage
+    end
+
+    def active_stage
+      j = self
+      #latest started at or first
+      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
+    end
+
+    def completed_at
+      j = self
+      j.stages.last.completed_at if j.stages.last
+    end
+
+    def failed_at
+      j = self
+      j.active_stage.failed_at if j.active_stage
+    end
+
+    def status_at
+      j = self
+      j.active_stage.status_at if j.active_stage
+    end
+
+    #convenience methods
+    def runner
+      j = self
+      runner_path = j.path.split("/")[0..-2].join("/")
+      return Runner.where(:path=>runner_path).first
+    end
+
+    def is_working?
+      j = self
+      j.stages.select{|s| s.is_working?}.compact.length>0
+    end
+  end
+end