mobilize-base 1.0.9 → 1.0.51

data/.gitignore

@@ -11,5 +11,4 @@ config
 log
 tmp
 .DS_Store
-REVISION
 test/dump.rdb

data/README.md

@@ -120,7 +120,7 @@ same one that contains your Rakefile)
 Inside the Rakefile in your project's root folder, make sure you have:
 
 ``` ruby
-require 'mobilize-base/tasks'
+require 'mobilize-base/rakes'
 ```
 
 This defines rake tasks essential to run the environment.
@@ -390,7 +390,7 @@ Mobilize takes the environment from your Rails.env if you're running
 Rails, or assumes "development." You can specify "development", "test",
 or "production," as per the yml files.
 
-Otherwise, it takes it from MOBILIZE_ENV parameter, as in:
+Otherwise, it takes it from MOBILIZE_ENV parameter, set from irb, as in:
 
 ``` ruby
 > ENV['MOBILIZE_ENV'] = 'production'
@@ -489,28 +489,26 @@ name>))` and enter values under each header:
 
 * status	Mobilize writes this field with the last status returned by the job
 
-* stage1..stage5 List of stages to be performed by the job. 
-  * Stages have this syntax: <handler>.<call> <params>.
-    * handler specifies the file that should receive the stage
+* task1..task5 List of tasks to be performed by the job. 
+  * Tasks have this syntax: <handler>.<call> <params>.
+    * handler specifies the file that should receive the task
     * the call specifies the method within the file. The method should
-be called `"<handler>.<call>_by_stage_path"`
+be called `"<Handler>.<call>_by_task_path"`
     * the params the method accepts, which are custom to each
-stage. These should be of the for `<key1>: <value1>, <key2>: <value2>`, where
-`<key>` is an unquoted string and `<value>` is a quoted string, an
-integer, an array (delimited by square braces), or a hash (delimited by
-curly braces).
-    * For mobilize-base, the following stages are available:
-      * gsheet.read `source: <input_gsheet_full_path>`, which reads the sheet. 
-        * The gsheet_full_path should be of the form `<gbook_name>/<gsheet_name>`. The test uses
-"Requestor_mobilize(test)/base1_stage1.in".
-      * gsheet.write `source: <stage_relative_path>`,`target: <target_gsheet_path>`,
-which writes the specified stage output to the target_gsheet. 
-        * The stage_relative_path should be of the form `<stage_column>` or
-`<job_name/stage_column>`. The test uses "base1/stage1" for the first test
-and simply "stage1" for the second test. Both of these take the output
-from the first stage.
+task. These should be a comma-delimited list, with each param in
+quotes.
+    * For mobilize-base, the following tasks are available:
+      * gsheet.read `<input_gsheet_full_path>`, which reads the sheet. 
+        * The gsheet_path should be of the form `<gbook_name>/<gsheet_name>`. The test uses
+"Requestor_mobilize(test)/base1_task1.in".
+      * gsheet.write `<task_relative_path>`,`<output_gsheet_path>`,
+which writes the specified task output to the output_gsheet. 
+        * The task_path should be of the form `<task_column>` or
+`<job_name/task_column>`. The test uses "base1/task1" for the first test
+and simply "task1" for the second test. Both of these take the output
+from the first task.
         * The test uses "Requestor_mobilize(test)/base1.out" and
-"Requestor_mobilize(test)/base2.out" for target sheets.
+"Requestor_mobilize(test)/base2.out" for output sheets.
 
 <a name='section_Start_Run_Test'></a>
 ### Run Test

data/Rakefile

@@ -17,7 +17,7 @@ require "bundler/gem_tasks"
 # Setup
 #
 $LOAD_PATH.unshift 'lib'
-require 'mobilize-base/tasks'
+require 'mobilize-base/rakes'
 
 #
 # Tests

data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb

@@ -1,7 +1,6 @@
 module GoogleDrive
   class ClientLoginFetcher
     def request_raw(method, url, data, extra_header, auth)
-      clf = self
       #this is patched to handle server errors due to http chaos
       uri = URI.parse(url)
       response = nil
@@ -15,32 +14,22 @@ module GoogleDrive
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         #set 600  to allow for large downloads
         http.read_timeout = 600
-        response = begin
-                     clf.http_call(http, method, uri, data, extra_header, auth)
-                   rescue
-                     #timeouts etc.
-                     nil
-                   end
-        if response.nil?
-          attempts +=1
-        else
-          if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
-            if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
-              if sleep_time
-                sleep_time = sleep_time * attempts
-              else
-                sleep_time = (rand*100).to_i
-              end
+        response = self.http_call(http, method, uri, data, extra_header, auth)
+        if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
+          if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
+            if sleep_time
+              sleep_time = sleep_time * attempts
             else
-              sleep_time = 10
+              sleep_time = (rand*100).to_i
             end
-            attempts += 1
-            puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
-            sleep sleep_time
+          else
+            sleep_time = 10
           end
+          attempts += 1
+          puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
+          sleep sleep_time
         end
       end
-      raise "No response after 5 attempts" if response.nil?
       raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
       return response
     end

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -54,11 +54,7 @@ module GoogleDrive
           raise "Invalid role #{role}"
         end
       else
-        begin
-          f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
-        rescue => exc
-          raise exc unless exc.to_s.index("user already has access")
-        end
+        f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
       end
       return true
     end

data/lib/mobilize-base/extensions/google_drive/worksheet.rb

@@ -137,9 +137,8 @@ module GoogleDrive
             if loc_v != rem_v
               if ['true','false'].include?(loc_v.downcase)
                 #google sheet upcases true and false. ignore
-              elsif loc_v.to_s.downcase.gsub("-","").gsub(" ","")==rem_v.to_s.downcase.gsub("-","").gsub(" ","")
-                #supported currency, silently converted whether it's an actual currency or not
-                #put a backtick on it.
+              elsif loc_v.starts_with?('rp') and rem_v.starts_with?('Rp')
+                # some other math bs
                 sheet[row_i+1,col_i+1] = %{'#{loc_v}}
                 re_col_vs << {'row_i'=>row_i+1,'col_i'=>col_i+1,'col_v'=>%{'#{loc_v}}}
               elsif (loc_v.to_s.count('e')==1 or loc_v.to_s.count('e')==0) and

data/lib/mobilize-base/handlers/gbook.rb

@@ -8,10 +8,10 @@ module Mobilize
       dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       #there should only be one book with each path, otherwise we have fail
       book = nil
-      if books.length>1 and dst.http_url.to_s.length>0
+      if books.length>1 and dst.url.to_s.length>0
         #some idiot process created a duplicate book.
         #Fix by renaming all but one with dst entry's key
-        dkey = dst.http_url.split("key=").last
+        dkey = dst.url.split("key=").last
         books.each do |b|
           bkey = b.resource_id.split(":").last
           if bkey == dkey
@@ -30,9 +30,9 @@ module Mobilize
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
       end
-      #always make sure book dataset http URL is up to date
+      #always make sure book dataset URL is up to date
       #and that book has admin acl
-      dst.update_attributes(:http_url=>book.human_url)
+      dst.update_attributes(:url=>book.human_url)
       book.add_admin_acl
       return book
     end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -54,15 +54,6 @@ module Mobilize
       return false
     end
 
-    def Gdrive.unslot_worker_by_path(path)
-      begin
-        Mobilize::Resque.set_worker_args_by_path(path,{'gdrive_slot'=>nil})
-        return true
-      rescue
-        return false
-      end
-    end
-
     def Gdrive.root(gdrive_slot=nil)
       pw = Gdrive.password(gdrive_slot)
       GoogleDrive.login(gdrive_slot,pw)

data/lib/mobilize-base/handlers/gfile.rb

@@ -22,16 +22,13 @@ module Mobilize
       Gdrive.files(gdrive_slot,{"title"=>path,"title-exact"=>"true"}).first
     end
 
-    def Gfile.read_by_stage_path(stage_path)
+    def Gfile.read_by_task_path(task_path)
       #reserve gdrive_slot account for read
-      gdrive_slot = Gdrive.slot_worker_by_path(s.path)
+      gdrive_slot = Gdrive.slot_worker_by_path(t.path)
       return false unless gdrive_slot
-      s = Stage.where(:path=>stage_path)
-      gfile_path = s.params['file']
-      out_tsv = Gfile.find_by_path(gfile_path,gdrive_slot).read
-      #use Gridfs to cache result
-      out_url = "gridfs://#{s.path}/out"
-      Dataset.write_to_url(out_url,out_tsv)
+      t = Task.where(:path=>task_path)
+      gfile_path = t.params.first
+      Gfile.find_by_path(gfile_path,gdrive_slot).read
     end
   end
 end

data/lib/mobilize-base/handlers/gridfs.rb

@@ -11,29 +11,29 @@ module Mobilize
       return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
     end
 
-    def Gridfs.read_by_dataset_path(dst_path)
+    def Gridfs.read(path)
       begin
-        zs=Gridfs.grid.open(dst_path,'r').read
+        zs=Gridfs.grid.open(path.gridsafe,'r').read
         return ::Zlib::Inflate.inflate(zs)
       rescue
         return nil
       end
     end
 
-    def Gridfs.write_by_dataset_path(dst_path,string)
+    def Gridfs.write(path,string)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      curr_zs = Gridfs.read_by_dataset_path(dst_path).to_s
+      curr_zs = Gridfs.read(path.gridsafe).to_s
       #write a new version when there is a change
       if curr_zs != zs
-        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
+        Gridfs.grid.open(path.gridsafe,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
       end
       return true
     end
 
-    def Gridfs.delete(dst_path)
+    def Gridfs.delete(path)
       begin
-        Gridfs.grid.delete(dst_path)
+        Gridfs.grid.delete(path.gridsafe)
         return true
       rescue
         return nil

data/lib/mobilize-base/handlers/gsheet.rb

@@ -32,35 +32,32 @@ module Mobilize
       return sheet
     end
 
-    def Gsheet.read_by_stage_path(stage_path)
+    def Gsheet.read_by_task_path(task_path)
       #reserve gdrive_slot account for read
-      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(task_path)
       return false unless gdrive_slot
-      s = Stage.where(:path=>stage_path).first
-      gsheet_path = s.params['source']
-      out_tsv = Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
-      #use Gridfs to cache result
-      out_url = "gridfs://#{s.path}/out"
-      Dataset.write_to_url(out_url,out_tsv)
+      t = Task.where(:path=>task_path).first
+      gsheet_path = t.params.first
+      Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
     end
 
-    def Gsheet.write_by_stage_path(stage_path)
-      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-      #return blank response if there are no slots available
-      return nil unless gdrive_slot
-      s = Stage.where(:path=>stage_path).first
-      source = s.params['source']
-      target_path = s.params['target']
-      source_job_name, source_stage_name = if source.index("/")
+    def Gsheet.write_by_task_path(task_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(task_path)
+      #return false if there are no emails available
+      return false unless gdrive_slot
+      t = Task.where(:path=>task_path).first
+      source = t.params.first
+      target_path = t.params.second
+      source_job_name, source_task_name = if source.index("/")
                                             source.split("/")
                                           else
                                             [nil, source]
                                           end
-      source_stage_path = "#{s.job.runner.path}/#{source_job_name || s.job.name}/#{source_stage_name}"
-      source_stage = Stage.where(:path=>source_stage_path).first
-      tsv = source_stage.out_dst.read
+      source_task_path = "#{t.job.runner.path}/#{source_job_name || t.job.name}/#{source_task_name}"
+      source_task = Task.where(:path=>source_task_path).first
+      tsv = source_task.stdout_dataset.read_cache
       sheet_name = target_path.split("/").last
-      temp_path = [stage_path.gridsafe,sheet_name].join("/")
+      temp_path = [task_path.gridsafe,sheet_name].join("/")
       temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
       temp_sheet.write(tsv)
       temp_sheet.check_and_fix(tsv)
@@ -68,11 +65,8 @@ module Mobilize
       target_sheet.merge(temp_sheet)
       #delete the temp sheet's book
       temp_sheet.spreadsheet.delete
-      status = "Write successful for #{target_path}"
-      s.update_status(status)
-      #use Gridfs to cache result
-      out_url = "gridfs://#{s.path}/out"
-      Dataset.write_to_url(out_url,status)
+      "Write successful for #{target_path}".oputs
+      return true
     end
   end
 end

data/lib/mobilize-base/handlers/resque.rb

@@ -17,6 +17,7 @@ module Mobilize
     end
 
     def Resque.workers(state="all")
+      raise "invalid state #{state}" unless ['all','idle','working','timeout'].include?(state)
       workers = ::Resque.workers.select{|w| w.queues.first == Resque.queue_name}
       return workers if state == 'all'
       working_workers = workers.select{|w| w.job['queue']== Resque.queue_name}
@@ -27,7 +28,6 @@ module Mobilize
       return stale_workers if state == 'stale'
       timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
       return timeout_workers if state == 'timeout'
-      raise "invalid state #{state}"
     end
 
     def Resque.failures
@@ -36,6 +36,7 @@ module Mobilize
 
     #active state refers to jobs that are either queued or working
     def Resque.jobs(state="active")
+      raise "invalid state #{state}" unless ['all','queued','working','active','timeout','failed'].include?(state)
       working_jobs =  Resque.workers('working').map{|w| w.job['payload']}
       return working_jobs if state == 'working'
       queued_jobs = ::Resque.peek(Resque.queue_name,0,0).to_a
@@ -46,17 +47,16 @@ module Mobilize
       timeout_jobs = Resque.workers("timeout").map{|w| w.job['payload']}
       return timeout_jobs if state == 'timeout'
       return working_jobs + queued_jobs + failed_jobs if state == 'all'
-      raise "invalid state #{state}"
     end
 
     def Resque.active_paths
-      #first argument of the payload is the runner / stage path unless the worker is Jobtracker
-      Resque.jobs('active').compact.map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
+      #first argument of the payload is the runner / task path unless the worker is Jobtracker
+      Resque.jobs('active').map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
     end
 
     #Resque workers and methods to find
     def Resque.find_worker_by_path(path)
-      Resque.workers('working').select{|w| w.job and w.job['payload'] and w.job['payload']['args'].first == path}.first
+      Resque.workers('working').select{|w| w.job['payload'] and w.job['payload']['args'].first == path}.first
     end
 
     def Resque.set_worker_args_by_path(path,args)
@@ -141,7 +141,7 @@ module Mobilize
       return true
     end
 
-    def Resque.kill_idle_and_stale_workers
+    def Resque.kill_idle_stale_workers
       idle_pids = Resque.workers('idle').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
       stale_pids = Resque.workers('stale').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
       idle_stale_pids = (idle_pids & stale_pids)

data/lib/mobilize-base/jobtracker.rb

@@ -89,7 +89,7 @@ module Mobilize
 
     def Jobtracker.start
       if Jobtracker.status!='stopped'
-        Jobtracker.update_status("Jobtracker still #{Jobtracker.status}")
+        raise "Jobtracker still #{Jobtracker.status}"
       else
         #make sure that workers are running and at the right number
         #Resque.prep_workers
@@ -190,9 +190,7 @@ module Mobilize
       while Jobtracker.status != 'stopping'
         users = User.all
         Jobtracker.run_notifications
-        #run throush all users randomly
-        #so none are privileged on JT restarts
-        users.sort_by{rand}.each do |u|
+        users.each do |u|
           r = u.runner
           Jobtracker.update_status("Checking #{r.path}")
           if r.is_due?
@@ -208,25 +206,15 @@ module Mobilize
 
     def Jobtracker.deployed_at
       #assumes deploy is as of last commit, or as of last deploy time
-      #as given by the REVISION file in the root folder
+      #as given by the least recently updated file in the root folder
       deploy_time = begin
-                      %{git log -1 --format="%cd"}.bash
-                    rescue
-                      revision_path = "#{ENV['PWD']}/REVISION"
-                      "touch #{revision_path}".bash unless File.exists?(revision_path)
-                      revision_string = "ls -l #{revision_path}".bash
-                      revision_rows = revision_string.split("\n").map{|lss| lss.strip.split(" ")}
-                      mod_time = revision_rows.map do |lsr| 
-                        if lsr.length == 8
-                          #ubuntu
-                          lsr[5..6].join(" ")
-                        elsif lsr.length == 9
-                          #osx
-                          lsr[5..7].join(" ")
-                        end
-                      end.first
-                      mod_time
-                    end.to_s.strip
+                     %{git log -1 --format="%cd"}.bash
+                   rescue
+                     ls_string = "ls -l #{ENV['PWD']}/*".bash
+                     ls_rows = ls_string.split("\n").map{|lss| lss.strip.split(" ")}
+                     mod_times = ls_rows.select{|lsr| lsr.length == 8}.map{|lsr| lsr[5..6].join(" ")}
+                     mod_times.min
+                   end
       Time.parse(deploy_time)
     end
 

data/lib/mobilize-base/models/dataset.rb

@@ -4,7 +4,7 @@ module Mobilize
     include Mongoid::Timestamps
     field :handler, type: String
     field :path, type: String
-    field :http_url, type: String
+    field :url, type: String
     field :raw_size, type: Fixnum
     field :last_cached_at, type: Time
     field :last_cache_handler, type: String
@@ -18,16 +18,6 @@ module Mobilize
       return "Mobilize::#{dst.handler.humanize}".constantize.read_by_path(dst.path)
     end
 
-    def Dataset.find_by_url(url)
-      handler,path = url.split("://")
-      Dataset.find_by_handler_and_path(handler,path)
-    end
-
-    def Dataset.find_or_create_by_url(url)
-      handler,path = url.split("://")
-      Dataset.find_or_create_by_handler_and_path(handler,path)
-    end
-
     def Dataset.find_by_handler_and_path(handler,path)
       Dataset.where(handler: handler, path: path).first
     end
@@ -38,30 +28,41 @@ module Mobilize
       return dst
     end
 
-    def Dataset.write_to_url(url,string)
-      dst = Dataset.find_or_create_by_url(url)
-      dst.write(string)
-      url
+    def write(string)
+      dst = self
+      "Mobilize::#{dst.handler.humanize}".constantize.write_by_path(dst.path,string)
+      dst.raw_size = string.length
+      dst.save!
+      return true
     end
 
-    def read
+    def cache_valid?
       dst = self
-      dst.update_attributes(:last_read_at=>Time.now.utc)
-      "Mobilize::#{dst.handler.humanize}".constantize.read_by_dataset_path(dst.path)
+      return true if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
     end
 
-    def write(string)
+    def read_cache(cache_handler="gridfs")
       dst = self
-      "Mobilize::#{dst.handler.humanize}".constantize.write_by_dataset_path(dst.path,string)
-      dst.raw_size = string.length
-      dst.save!
-      return true
+      if cache_valid?
+        dst.update_attributes(:last_read_at=>Time.now.utc)
+        return "Mobilize::#{cache_handler.humanize}".constantize.read([dst.handler,dst.path].join("://"))
+      else
+        raise "Cache invalid or not found for #{cache_handler}://#{dst.path}"
+      end
     end
 
-    def delete
+    def write_cache(string,expire_at=nil,cache_handler="gridfs")
       dst = self
-      "Mobilize::#{dst.handler.humanize}".constantize.delete_by_dataset_path(dst.path)
+      "Mobilize::#{cache_handler.humanize}".constantize.write([dst.handler,dst.path].join("://"),string)
+      dst.update_attributes(:last_cached_at=>Time.now.utc,
+                            :last_cache_handler=>cache_handler.to_s.downcase,
+                            :cache_expire_at=>expire_at,
+                            :size=>string.length)
       return true
     end
+
+    def delete_cache(cache_handler="gridfs")
+      return "Mobilize::#{cache_handler.humanize}".constantize.delete(dst.handler, dst.path)
+    end
   end
 end

data/lib/mobilize-base/models/job.rb

@@ -5,6 +5,8 @@ module Mobilize
     field :path, type: String
     field :active, type: Boolean
     field :trigger, type: String
+    field :status, type: String
+    field :last_completed_at, type: Time
 
     index({ path: 1})
 
@@ -13,9 +15,9 @@ module Mobilize
       j.path.split("/").last
     end
 
-    def stages
+    def tasks
       j = self
-      Stage.where(:path=>/^#{j.path.escape_regex}/).to_a.sort_by{|s| s.path}
+      Task.where(:path=>/^#{j.path.escape_regex}/).to_a.sort_by{|t| t.path}
     end
 
     def Job.find_or_create_by_path(path)
@@ -24,33 +26,6 @@ module Mobilize
       return j
     end
 
-    def status
-      #last stage status
-      j = self
-      j.active_stage.status if j.active_stage
-    end
-
-    def active_stage
-      j = self
-      #latest started at or first
-      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
-    end
-
-    def completed_at
-      j = self
-      j.stages.last.completed_at if j.stages.last
-    end
-
-    def failed_at
-      j = self
-      j.active_stage.failed_at if j.active_stage
-    end
-
-    def status_at
-      j = self
-      j.active_stage.status_at if j.active_stage
-    end
-
     #convenience methods
     def runner
       j = self
@@ -60,13 +35,13 @@ module Mobilize
 
     def is_working?
       j = self
-      j.stages.select{|s| s.is_working?}.compact.length>0
+      j.tasks.select{|t| t.is_working?}.compact.length>0
     end
 
     def is_due?
       j = self
       return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
-      last_run = j.completed_at
+      last_run = j.last_completed_at
       #check trigger
       trigger = j.trigger
       return true if trigger == 'once'

data/lib/mobilize-base/models/runner.rb

@@ -5,19 +5,17 @@ module Mobilize
     field :path, type: String
     field :active, type: Boolean
     field :status, type: String
-    field :started_at, type: Time
-    field :status_at, type: Time
-    field :completed_at, type: Time
+    field :last_run, type: Time
 
     index({ path: 1})
 
     def headers
-      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
+      %w{name active trigger status task1 task2 task3 task4 task5}
     end
 
-    def cached_at
+    def last_cached_at
       r = self
-      Dataset.find_or_create_by_path(r.path).cached_at
+      Dataset.find_or_create_by_path(r.path).last_cached_at
     end
 
     def worker
@@ -41,7 +39,6 @@ module Mobilize
         r.update_status("no gdrive slot available")
         return false
       end
-      r.update_attributes(:started_at=>Time.now.utc)
       #make sure any updates to activity are processed first
       #as in when someone runs a "once" job that has completed
       r.update_gsheet(gdrive_slot)
@@ -51,15 +48,15 @@ module Mobilize
       r.jobs.each do |j|
         begin
           if j.is_due?
-            j.stages.first.enqueue!
+            j.tasks.first.enqueue!
           end
         rescue ScriptError, StandardError => exc
-          r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
+          j.update_status("Failed to enqueue with #{exc.to_s}")
           j.update_attributes(:active=>false)
         end
       end
       r.update_gsheet(gdrive_slot)
-      r.update_attributes(:completed_at=>Time.now.utc)
+      r.update_attributes(:last_run=>Time.now.utc)
     end
 
     def dataset
@@ -71,16 +68,16 @@ module Mobilize
       Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
     end
 
-    def cache
+    def read_cache
       r = self
-      Dataset.find_or_create_by_url("gridfs://#{r.path}")
+      r.dataset.read_cache
     end
 
     def gsheet(gdrive_slot)
       r = self
       jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
       jobs_sheet.add_headers(r.headers)
-      begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
+      jobs_sheet.delete_sheet1
       return jobs_sheet
     end
 
@@ -88,7 +85,7 @@ module Mobilize
       r = self
       gsheet_tsv = r.gsheet(gdrive_slot).to_tsv
       #cache in DB
-      r.cache.write(gsheet_tsv)
+      r.dataset.write_cache(gsheet_tsv)
       #turn it into a hash array
       gsheet_jobs = gsheet_tsv.tsv_to_hash_array
       #go through each job, update relevant job with its params
@@ -96,25 +93,25 @@ module Mobilize
       #parse out the jobs and update the Job collection
       gsheet_jobs.each_with_index do |rj,rj_i|
         #skip non-jobs or jobs without required values
-        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
+        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','task1'].select{|c| rj[c].to_s.strip==""}.length>0)
         j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
         #update top line params
         j.update_attributes(:active => rj['active'],
                             :trigger => rj['trigger'])
-        (1..5).to_a.each do |s_idx|
-          stage_string = rj["stage#{s_idx.to_s}"]
-          break if stage_string.to_s.length==0
-          s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
-          #parse command string, update stage with it
-          s_handler, call, param_string = [""*3]
-          stage_string.split(" ").ie do |spls|
-            s_handler = spls.first.split(".").first
+        (1..5).to_a.each do |t_idx|
+          task_string = rj["task#{t_idx.to_s}"]
+          break if task_string.to_s.length==0
+          t = Task.find_or_create_by_path("#{j.path}/task#{t_idx.to_s}")
+          #parse command string, update task with it
+          t_handler, call, param_string = [""*3]
+          task_string.split(" ").ie do |spls|
+            t_handler = spls.first.split(".").first
             call = spls.first.split(".").last
             param_string = spls[1..-1].join(" ").strip
           end
-          s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
+          t.update_attributes(:call=>call, :handler=>t_handler, :param_string=>param_string)
         end
-        r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
+        r.update_status("Updated #{j.path} tasks at #{Time.now.utc}")
         #add this job to list of read ones
         done_jobs << j
       end
@@ -131,8 +128,7 @@ module Mobilize
     def update_gsheet(gdrive_slot)
       r = self
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
-      upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
+      upd_rows = r.jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status} }
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
       return true
@@ -151,12 +147,12 @@ module Mobilize
     def user
       r = self
       user_name = r.path.split("_").second.split("(").first.split("/").first
-      User.where(:name=>user_name).first
+      User.where(:email=>[user_name,Gdrive.domain].join("@")).first
     end
 
     def update_status(msg)
       r = self
-      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
+      r.update_attributes(:status=>msg)
       Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
       return true
     end
@@ -169,8 +165,8 @@ module Mobilize
     def is_due?
       r = self.reload
       return false if r.is_working?
-      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
-      return true if r.started_at.nil? or r.started_at < prev_due_time
+      last_due_time = Time.now.utc - Jobtracker.runner_read_freq
+      return true if r.last_run.nil? or r.last_run < last_due_time
     end
 
     def enqueue!

data/lib/mobilize-base/models/task.rb

@@ -0,0 +1,137 @@
+module Mobilize
+  class Task
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    field :path, type: String
+    field :handler, type: String
+    field :call, type: String
+    field :param_string, type: Array
+    field :status, type: String
+    field :last_completed_at, type: Time
+    field :last_run_at, type: Time
+
+    index({ path: 1})
+
+    def idx
+      t = self
+      t.path.split("/").last.gsub("task","").to_i
+    end
+
+    def stdout_dataset
+      t = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stdout")
+    end
+
+    def stderr_dataset
+      t = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
+    end
+
+    def params
+      t = self
+      t.param_string.split(",").map do |p| 
+        ps = p.strip
+        ps = ps[1..-1] if ['"',"'"].include?(ps[0])
+        ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
+        ps
+      end
+    end
+
+    def job
+      t = self
+      job_path = t.path.split("/")[0..-2].join("/")
+      Job.where(:path=>job_path).first
+    end
+
+    def Task.find_or_create_by_path(path)
+      t = Task.where(:path=>path).first
+      t = Task.create(:path=>path) unless t
+      return t
+    end
+
+    def prior
+      t = self
+      j = t.job
+      return nil if t.idx==1
+      return j.tasks[t.idx-2]
+    end
+
+    def next
+      t = self
+      j = t.job
+      return nil if t.idx == j.tasks.length
+      return j.tasks[t.idx]
+    end
+
+    def Task.perform(id,*args)
+      t = Task.where(:path=>id).first
+      j = t.job
+      t.update_status(%{Starting at #{Time.now.utc}})
+      stdout, stderr = [nil,nil]
+      begin
+        stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
+      rescue ScriptError, StandardError => exc
+        stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
+        #record the failure in Job so it appears on Runner
+        j.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
+        t.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
+        raise exc
+      end
+      if stdout == false
+        #re-queue self if output is false
+        t.enqueue!
+        return false
+      end
+      #write output to cache
+      t.stdout_dataset.write_cache(stdout)
+      t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
+      if t.idx == j.tasks.length
+        j.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
+        j.update_attributes(:active=>false) if j.trigger.strip == "once"
+        t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
+        #check for any dependent jobs, if there are, enqueue them
+        r = j.runner
+        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
+        #put begin/rescue so all dependencies run
+        dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
+      else
+        #queue up next task
+        t.next.enqueue!
+      end
+      return true
+    end
+
+    def enqueue!
+      t = self
+      ::Resque::Job.create("mobilize",Task,t.path,{})
+      return true
+    end
+
+    def worker
+      t = self
+      Mobilize::Resque.find_worker_by_path(t.path)
+    end
+
+    def worker_args
+      t = self
+      Jobtracker.get_worker_args(t.worker)
+    end
+
+    def set_worker_args(args)
+      t = self
+      Jobtracker.set_worker_args(t.worker,args)
+    end
+
+    def update_status(msg)
+      t = self
+      t.update_attributes(:status=>msg)
+      Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      t = self
+      Mobilize::Resque.active_paths.include?(t.path)
+    end
+  end
+end

data/lib/mobilize-base/models/user.rb

@@ -24,7 +24,7 @@ module Mobilize
 
     def jobs(jname=nil)
       u = self
-      return u.runner.jobs
+      return u.runners.map{|r| r.jobs(jname)}.flatten
     end
 
     def runner_path

data/lib/mobilize-base/{tasks.rb → rakes.rb}

@@ -1,3 +1,6 @@
+# require 'resque/tasks'
+# will give you the resque tasks
+
 namespace :mobilize_base do
   desc "Start a Resque worker"
   task :work do
@@ -8,7 +11,6 @@ namespace :mobilize_base do
         require e
       end
     rescue Exception=>exc
-      #do nothing
     end
 
     begin
@@ -21,42 +23,17 @@ namespace :mobilize_base do
 
     worker.work(ENV['INTERVAL'] || 5) # interval, will block
   end
-  desc "Kill all Resque workers"
-  task :kill_workers do
-    require 'mobilize-base'
-    Mobilize::Jobtracker.kill_workers
-  end
   desc "Kill idle workers not in sync with repo"
   task :kill_idle_and_stale_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_idle_and_stale_workers
   end
-    desc "Kill idle workers"
-    task :kill_idle_workers do
-    require 'mobilize-base'
-    Mobilize::Jobtracker.kill_idle_workers
-  end
-  desc "Make sure there are the correct # of workers, kill if too many"
+  desc "Make sure workers are prepped"
   task :prep_workers do
     require 'mobilize-base'
     Mobilize::Jobtracker.prep_workers
   end
-  desc "Stop Jobtracker"
-  task :stop do
-    require 'mobilize-base'
-    Mobilize::Jobtracker.stop!
-  end
-  desc "Start Jobtracker"
-  task :start do
-    require 'mobilize-base'
-    Mobilize::Jobtracker.start
-  end
-  desc "Restart Jobtracker"
-  task :restart do
-    require 'mobilize-base'
-    Mobilize::Jobtracker.restart!
-  end
-  desc "kill all old resque web processes, start new one with resque_web.rb extension file"
+  desc "kill all old resque web, start new one with env params"
   task :resque_web do
     require 'mobilize-base'
     port = Mobilize::Base.config('resque')['web_port']
@@ -75,7 +52,9 @@ namespace :mobilize_base do
       "Mobilize::#{m}".constantize.create_indexes
     end
   end
-  desc "Set up config and log folders and files, populate from samples"
+end
+namespace :mobilize do
+  desc "Set up config and log folders and files"
   task :setup do
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
     log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")

data/lib/mobilize-base/tasks/mobilize-base.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'mobilize-base/tasks'

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.9"
+    VERSION = "1.0.51"
   end
 end

data/lib/mobilize-base.rb

@@ -66,7 +66,7 @@ if File.exists?(mongoid_config_path)
   require "mobilize-base/models/user"
   require "mobilize-base/models/runner"
   require "mobilize-base/models/job"
-  require "mobilize-base/models/stage"
+  require "mobilize-base/models/task"
 end
 require 'google_drive'
 require 'resque'

data/mobilize-base.gemspec

@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'redis',"~>3.0.0"
   s.add_runtime_dependency 'resque','1.21.0'
   s.add_runtime_dependency 'google_drive','0.3.2'
+  s.add_runtime_dependency 'bluepill','0.0.60'
   s.add_runtime_dependency 'popen4','0.1.2'
   s.add_runtime_dependency 'actionmailer','3.1.1'
 end

data/test/base1_task1.yml

@@ -0,0 +1,3 @@
+- {test_header: t1, test_header2: t1, test_header3: t1}
+- {test_header: t2, test_header2: t2, test_header3: t2}
+- {test_header: t3, test_header2: t3, test_header3: t3}

data/test/base_job_rows.yml

@@ -2,12 +2,12 @@
   active: true
   trigger: once
   status: ""
-  stage1: 'gsheet.read source:"Runner_mobilize(test)/base1_stage1.in"'
-  stage2: 'gsheet.write source:"base1/stage1", target:"Runner_mobilize(test)/base1.out"'
+  task1: 'gsheet.read "Runner_mobilize(test)/base1_task1.in"'
+  task2: 'gsheet.write "base1/task1", "Runner_mobilize(test)/base1.out"'
 
 - name: "base2"
   active: true
   trigger: "after base1"
   status: ""
-  stage1: 'gsheet.read source:"Runner_mobilize(test)/base1.out"'
-  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/base2.out"'
+  task1: 'gsheet.read "Runner_mobilize(test)/base1.out"'
+  task2: 'gsheet.write "task1", "Runner_mobilize(test)/base2.out"'

data/test/mobilize-base_test.rb

@@ -32,35 +32,27 @@ describe "Mobilize" do
     r = u.runner
     jobs_sheet = r.gsheet(gdrive_slot)
     tsv = jobs_sheet.to_tsv
-    assert tsv.length == 61 #headers only
+    assert tsv.length == 56 #headers only
 
-    puts "add base1_stage1 input sheet"
-    test_source_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.in",gdrive_slot)
+    puts "add base1_task1 input sheet"
+    test_source_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_task1.in",gdrive_slot)
 
-    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_stage1.yml")*40
+    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_task1.yml")*40
     test_source_tsv = test_source_ha.hash_array_to_tsv
     test_source_sheet.write(test_source_tsv)
 
     puts "add row to jobs sheet, wait 120s"
     test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
     jobs_sheet.add_or_update_rows(test_job_rows)
+
+    puts "job row added, force enqueued runner"
+    r.enqueue!
     sleep 120
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
-    test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base2.out",gdrive_slot)
-
-    assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
+    test_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
 
-    puts "delete both output sheets, set first job to active=true"
-    [test_target_sheet_1,test_target_sheet_2].each{|s| s.delete}
-
-    jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
-    sleep 90
-    
-    test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
-    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    assert test_target_sheet_2.to_tsv == test_source_sheet.to_tsv
+    assert test_target_sheet.to_tsv == test_source_sheet.to_tsv
 
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.9
+  version: 1.0.51
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-22 00:00:00.000000000 Z
+date: 2012-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -139,6 +139,22 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.3.2
+- !ruby/object:Gem::Dependency
+  name: bluepill
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.60
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.60
 - !ruby/object:Gem::Dependency
   name: popen4
   requirement: !ruby/object:Gem::Requirement
@@ -207,9 +223,10 @@ files:
 - lib/mobilize-base/models/dataset.rb
 - lib/mobilize-base/models/job.rb
 - lib/mobilize-base/models/runner.rb
-- lib/mobilize-base/models/stage.rb
+- lib/mobilize-base/models/task.rb
 - lib/mobilize-base/models/user.rb
-- lib/mobilize-base/tasks.rb
+- lib/mobilize-base/rakes.rb
+- lib/mobilize-base/tasks/mobilize-base.rake
 - lib/mobilize-base/version.rb
 - lib/samples/gdrive.yml
 - lib/samples/gridfs.yml
@@ -219,7 +236,7 @@ files:
 - lib/samples/resque.yml
 - lib/samples/resque_web.rb
 - mobilize-base.gemspec
-- test/base1_stage1.yml
+- test/base1_task1.yml
 - test/base_job_rows.yml
 - test/mobilize-base_test.rb
 - test/redis-test.conf
@@ -238,7 +255,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 576426973092652786
+      hash: -989061027210314242
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -247,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 576426973092652786
+      hash: -989061027210314242
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.24
@@ -256,7 +273,7 @@ specification_version: 3
 summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
   Docs
 test_files:
-- test/base1_stage1.yml
+- test/base1_task1.yml
 - test/base_job_rows.yml
 - test/mobilize-base_test.rb
 - test/redis-test.conf

data/lib/mobilize-base/models/stage.rb

@@ -1,141 +0,0 @@
-module Mobilize
-  class Stage
-    include Mongoid::Document
-    include Mongoid::Timestamps
-    field :path, type: String
-    field :handler, type: String
-    field :call, type: String
-    field :param_string, type: Array
-    field :status, type: String
-    field :out_url, type: String
-    field :completed_at, type: Time
-    field :started_at, type: Time
-    field :failed_at, type: Time
-    field :status_at, type: Time
-
-    index({ path: 1})
-
-    def idx
-      s = self
-      s.path.split("/").last.gsub("stage","").to_i
-    end
-
-    def out_dst
-      #this gives a dataset that points to the output
-      #allowing you to determine its size
-      #before committing to a read or write
-      s = self
-      Dataset.find_by_url(s.out_url) if s.out_url
-    end
-
-    def params
-      s = self
-      #evaluates param_string to ruby hash
-      #using YAML parser
-      #TODO: eliminate ridiculousness
-      begin
-        YAML.load(s.param_string)
-        raise "Must resolve to Hash" unless result.class==Hash
-      rescue
-        sub_param_string = s.param_string.gsub(":\"",": \"").gsub(":'",": '").gsub(":[",": [").gsub(":{",": {").gsub(/(:[0-9])/,'stageparamsgsub\1').gsub('stageparamsgsub:',': ')
-        YAML.load("{#{sub_param_string}}")
-      end
-    end
-
-    def job
-      s = self
-      job_path = s.path.split("/")[0..-2].join("/")
-      Job.where(:path=>job_path).first
-    end
-
-    def Stage.find_or_create_by_path(path)
-      s = Stage.where(:path=>path).first
-      s = Stage.create(:path=>path) unless s
-      return s
-    end
-
-    def prior
-      s = self
-      j = s.job
-      return nil if s.idx==1
-      return j.stages[s.idx-2]
-    end
-
-    def next
-      s = self
-      j = s.job
-      return nil if s.idx == j.stages.length
-      return j.stages[s.idx]
-    end
-
-    def Stage.perform(id,*args)
-      s = Stage.where(:path=>id).first
-      j = s.job
-      s.update_attributes(:started_at=>Time.now.utc)
-      s.update_status(%{Starting at #{Time.now.utc}})
-      begin
-        #get response by running method
-        s.out_url = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
-        s.save!
-        unless s.out_url
-          #re-queue self if no response
-          s.enqueue!
-          return false
-        end
-      rescue ScriptError, StandardError => exc
-        j.update_attributes(:active=>false)
-        s.update_attributes(:failed_at=>Time.now.utc)
-        s.update_status("Failed at #{Time.now.utc.to_s}")
-        raise exc
-      end
-      s.update_attributes(:completed_at=>Time.now.utc)
-      s.update_status("Completed at #{Time.now.utc.to_s}")
-      if s.idx == j.stages.length
-        #job has completed
-        j.update_attributes(:active=>false) if j.trigger.strip.downcase == "once"
-        #check for any dependent jobs, if there are, enqueue them
-        r = j.runner
-        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger.strip.downcase == "after #{j.name}"}
-        #put begin/rescue so all dependencies run
-        dep_jobs.each{|dj| begin;dj.stages.first.enqueue! unless dj.is_working?;rescue;end}
-      else
-        #queue up next stage
-        s.next.enqueue!
-      end
-      return true
-    end
-
-    def enqueue!
-      s = self
-      ::Resque::Job.create("mobilize",Stage,s.path,{})
-      return true
-    end
-
-    def worker
-      s = self
-      Mobilize::Resque.find_worker_by_path(s.path)
-    end
-
-    def worker_args
-      s = self
-      Jobtracker.get_worker_args(s.worker)
-    end
-
-    def set_worker_args(args)
-      s = self
-      Jobtracker.set_worker_args(s.worker,args)
-    end
-
-    def update_status(msg)
-      s = self
-      s.update_attributes(:status=>msg,:status_at=>Time.now.utc)
-      Mobilize::Resque.set_worker_args_by_path(s.path,{'status'=>msg})
-      return true
-    end
-
-    def is_working?
-      s = self
-      Mobilize::Resque.active_paths.include?(s.path)
-    end
-  end
-end

data/test/base1_stage1.yml

@@ -1,3 +0,0 @@
-- {test_header: s1, test_header2: s1, test_header3: s1}
-- {test_header: s2, test_header2: s2, test_header3: s2}
-- {test_header: s3, test_header2: s3, test_header3: s3}