mobilize-base 1.0.75 → 1.0.81

data/README.md

@@ -120,7 +120,7 @@ same one that contains your Rakefile)
 Inside the Rakefile in your project's root folder, make sure you have:
 
 ``` ruby
-require 'mobilize-base/rakes'
+require 'mobilize-base/tasks'
 ```
 
 This defines rake tasks essential to run the environment.
@@ -489,26 +489,26 @@ name>))` and enter values under each header:
 
 * status	Mobilize writes this field with the last status returned by the job
 
-* task1..task5 List of tasks to be performed by the job. 
-  * Tasks have this syntax: <handler>.<call> <params>.
-    * handler specifies the file that should receive the task
+* stage1..stage5 List of stages to be performed by the job. 
+  * Stages have this syntax: <handler>.<call> <params>.
+    * handler specifies the file that should receive the stage
     * the call specifies the method within the file. The method should
-be called `"<handler>.<call>_by_task_path"`
+be called `"<handler>.<call>_by_stage_path"`
     * the params the method accepts, which are custom to each
-task. These should be of the for `<key1>: <value1>, <key2>: <value2>`, where
+stage. These should be of the for `<key1>: <value1>, <key2>: <value2>`, where
 `<key>` is an unquoted string and `<value>` is a quoted string, an
 integer, an array (delimited by square braces), or a hash (delimited by
 curly braces).
-    * For mobilize-base, the following tasks are available:
+    * For mobilize-base, the following stages are available:
       * gsheet.read `source: <input_gsheet_full_path>`, which reads the sheet. 
         * The gsheet_full_path should be of the form `<gbook_name>/<gsheet_name>`. The test uses
-"Requestor_mobilize(test)/base1_task1.in".
-      * gsheet.write `source: <task_relative_path>`,`target: <target_gsheet_path>`,
-which writes the specified task output to the target_gsheet. 
-        * The task_relative_path should be of the form `<task_column>` or
-`<job_name/task_column>`. The test uses "base1/task1" for the first test
-and simply "task1" for the second test. Both of these take the output
-from the first task.
+"Requestor_mobilize(test)/base1_stage1.in".
+      * gsheet.write `source: <stage_relative_path>`,`target: <target_gsheet_path>`,
+which writes the specified stage output to the target_gsheet. 
+        * The stage_relative_path should be of the form `<stage_column>` or
+`<job_name/stage_column>`. The test uses "base1/stage1" for the first test
+and simply "stage1" for the second test. Both of these take the output
+from the first stage.
         * The test uses "Requestor_mobilize(test)/base1.out" and
 "Requestor_mobilize(test)/base2.out" for target sheets.
 

data/Rakefile

@@ -17,7 +17,7 @@ require "bundler/gem_tasks"
 # Setup
 #
 $LOAD_PATH.unshift 'lib'
-require 'mobilize-base/rakes'
+require 'mobilize-base/tasks'
 
 #
 # Tests

data/lib/mobilize-base.rb

@@ -66,7 +66,7 @@ if File.exists?(mongoid_config_path)
   require "mobilize-base/models/user"
   require "mobilize-base/models/runner"
   require "mobilize-base/models/job"
-  require "mobilize-base/models/task"
+  require "mobilize-base/models/stage"
 end
 require 'google_drive'
 require 'resque'

data/lib/mobilize-base/extensions/google_drive/worksheet.rb

@@ -137,8 +137,9 @@ module GoogleDrive
             if loc_v != rem_v
               if ['true','false'].include?(loc_v.downcase)
                 #google sheet upcases true and false. ignore
-              elsif loc_v.starts_with?('rp') and rem_v.starts_with?('Rp')
-                # some other math bs
+              elsif loc_v.to_s.downcase.gsub("-","").gsub(" ","")==rem_v.to_s.downcase.gsub("-","").gsub(" ","")
+                #supported currency, silently converted whether it's an actual currency or not
+                #put a backtick on it.
                 sheet[row_i+1,col_i+1] = %{'#{loc_v}}
                 re_col_vs << {'row_i'=>row_i+1,'col_i'=>col_i+1,'col_v'=>%{'#{loc_v}}}
               elsif (loc_v.to_s.count('e')==1 or loc_v.to_s.count('e')==0) and

data/lib/mobilize-base/handlers/gfile.rb

@@ -22,12 +22,12 @@ module Mobilize
       Gdrive.files(gdrive_slot,{"title"=>path,"title-exact"=>"true"}).first
     end
 
-    def Gfile.read_by_task_path(task_path)
+    def Gfile.read_by_stage_path(stage_path)
       #reserve gdrive_slot account for read
-      gdrive_slot = Gdrive.slot_worker_by_path(t.path)
+      gdrive_slot = Gdrive.slot_worker_by_path(s.path)
       return false unless gdrive_slot
-      t = Task.where(:path=>task_path)
-      gfile_path = t.params['file']
+      s = Stage.where(:path=>stage_path)
+      gfile_path = s.params['file']
       Gfile.find_by_path(gfile_path,gdrive_slot).read
     end
   end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -32,32 +32,32 @@ module Mobilize
       return sheet
     end
 
-    def Gsheet.read_by_task_path(task_path)
+    def Gsheet.read_by_stage_path(stage_path)
       #reserve gdrive_slot account for read
-      gdrive_slot = Gdrive.slot_worker_by_path(task_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
       return false unless gdrive_slot
-      t = Task.where(:path=>task_path).first
-      gsheet_path = t.params['source']
+      s = Stage.where(:path=>stage_path).first
+      gsheet_path = s.params['source']
       Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
     end
 
-    def Gsheet.write_by_task_path(task_path)
-      gdrive_slot = Gdrive.slot_worker_by_path(task_path)
+    def Gsheet.write_by_stage_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
       #return false if there are no emails available
       return false unless gdrive_slot
-      t = Task.where(:path=>task_path).first
-      source = t.params['source']
-      target_path = t.params['target']
-      source_job_name, source_task_name = if source.index("/")
+      s = Stage.where(:path=>stage_path).first
+      source = s.params['source']
+      target_path = s.params['target']
+      source_job_name, source_stage_name = if source.index("/")
                                             source.split("/")
                                           else
                                             [nil, source]
                                           end
-      source_task_path = "#{t.job.runner.path}/#{source_job_name || t.job.name}/#{source_task_name}"
-      source_task = Task.where(:path=>source_task_path).first
-      tsv = source_task.stdout_dataset.read_cache
+      source_stage_path = "#{s.job.runner.path}/#{source_job_name || s.job.name}/#{source_stage_name}"
+      source_stage = Stage.where(:path=>source_stage_path).first
+      tsv = source_stage.stdout_dataset.read_cache
       sheet_name = target_path.split("/").last
-      temp_path = [task_path.gridsafe,sheet_name].join("/")
+      temp_path = [stage_path.gridsafe,sheet_name].join("/")
       temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
       temp_sheet.write(tsv)
       temp_sheet.check_and_fix(tsv)

data/lib/mobilize-base/handlers/resque.rb

@@ -50,7 +50,7 @@ module Mobilize
     end
 
     def Resque.active_paths
-      #first argument of the payload is the runner / task path unless the worker is Jobtracker
+      #first argument of the payload is the runner / stage path unless the worker is Jobtracker
       Resque.jobs('active').map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
     end
 

data/lib/mobilize-base/jobtracker.rb

@@ -190,7 +190,9 @@ module Mobilize
       while Jobtracker.status != 'stopping'
         users = User.all
         Jobtracker.run_notifications
-        users.each do |u|
+        #run throush all users randomly
+        #so none are privileged on JT restarts
+        users.sort_by{rand}.each do |u|
           r = u.runner
           Jobtracker.update_status("Checking #{r.path}")
           if r.is_due?

data/lib/mobilize-base/models/job.rb

@@ -13,9 +13,9 @@ module Mobilize
       j.path.split("/").last
     end
 
-    def tasks
+    def stages
       j = self
-      Task.where(:path=>/^#{j.path.escape_regex}/).to_a.sort_by{|t| t.path}
+      Stage.where(:path=>/^#{j.path.escape_regex}/).to_a.sort_by{|s| s.path}
     end
 
     def Job.find_or_create_by_path(path)
@@ -25,29 +25,30 @@ module Mobilize
     end
 
     def status
-      #last task status
+      #last stage status
       j = self
-      j.active_task.status
+      j.active_stage.status if j.active_stage
     end
 
-    def active_task
+    def active_stage
       j = self
       #latest started at or first
-      j.tasks.select{|t| t.started_at}.sort_by{|t| t.started_at}.last || j.tasks.first
+      j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
     end
 
     def completed_at
       j = self
-      j.tasks.last.completed_at
+      j.stages.last.completed_at if j.stages.last
     end
 
     def failed_at
       j = self
-      j.tasks.last.failed_at
+      j.active_stage.failed_at if j.active_stage
     end
 
     def status_at
-      j.active_task.status_at
+      j = self
+      j.active_stage.status_at if j.active_stage
     end
 
     #convenience methods
@@ -59,7 +60,7 @@ module Mobilize
 
     def is_working?
       j = self
-      j.tasks.select{|t| t.is_working?}.compact.length>0
+      j.stages.select{|s| s.is_working?}.compact.length>0
     end
 
     def is_due?

data/lib/mobilize-base/models/runner.rb

@@ -12,7 +12,7 @@ module Mobilize
     index({ path: 1})
 
     def headers
-      %w{name active trigger status task1 task2 task3 task4 task5}
+      %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
     end
 
     def cached_at
@@ -41,6 +41,7 @@ module Mobilize
         r.update_status("no gdrive slot available")
         return false
       end
+      r.update_attributes(:started_at=>Time.now.utc)
       #make sure any updates to activity are processed first
       #as in when someone runs a "once" job that has completed
       r.update_gsheet(gdrive_slot)
@@ -50,7 +51,7 @@ module Mobilize
       r.jobs.each do |j|
         begin
           if j.is_due?
-            j.tasks.first.enqueue!
+            j.stages.first.enqueue!
           end
         rescue ScriptError, StandardError => exc
           r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
@@ -95,25 +96,25 @@ module Mobilize
       #parse out the jobs and update the Job collection
       gsheet_jobs.each_with_index do |rj,rj_i|
         #skip non-jobs or jobs without required values
-        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','task1'].select{|c| rj[c].to_s.strip==""}.length>0)
+        next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
         j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
         #update top line params
         j.update_attributes(:active => rj['active'],
                             :trigger => rj['trigger'])
-        (1..5).to_a.each do |t_idx|
-          task_string = rj["task#{t_idx.to_s}"]
-          break if task_string.to_s.length==0
-          t = Task.find_or_create_by_path("#{j.path}/task#{t_idx.to_s}")
-          #parse command string, update task with it
-          t_handler, call, param_string = [""*3]
-          task_string.split(" ").ie do |spls|
-            t_handler = spls.first.split(".").first
+        (1..5).to_a.each do |s_idx|
+          stage_string = rj["stage#{s_idx.to_s}"]
+          break if stage_string.to_s.length==0
+          s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
+          #parse command string, update stage with it
+          s_handler, call, param_string = [""*3]
+          stage_string.split(" ").ie do |spls|
+            s_handler = spls.first.split(".").first
             call = spls.first.split(".").last
             param_string = spls[1..-1].join(" ").strip
           end
-          t.update_attributes(:call=>call, :handler=>t_handler, :param_string=>param_string)
+          s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
         end
-        r.update_status("Updated #{j.path} tasks at #{Time.now.utc}")
+        r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
         #add this job to list of read ones
         done_jobs << j
       end
@@ -130,11 +131,7 @@ module Mobilize
     def update_gsheet(gdrive_slot)
       r = self
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_jobs = r.jobs.select do |j|
-                                 j.completed_at.nil? ||
-                                 j.completed_at > j.runner.completed_at ||
-                                 (j.failed_at and j.failed_at > j.runner.completed_at)
-                               end
+      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
       upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
@@ -178,7 +175,6 @@ module Mobilize
 
     def enqueue!
       r = self
-      r.update_attributes(:started_at=>Time.now.utc)
       ::Resque::Job.create("mobilize",Runner,r.path,{})
       return true
     end

data/lib/mobilize-base/models/stage.rb

@@ -0,0 +1,153 @@
+module Mobilize
+  class Stage
+    include Mongoid::Document
+    include Mongoid::Timestamps
+    field :path, type: String
+    field :handler, type: String
+    field :call, type: String
+    field :param_string, type: Array
+    field :status, type: String
+    field :completed_at, type: Time
+    field :started_at, type: Time
+    field :failed_at, type: Time
+    field :status_at, type: Time
+
+    index({ path: 1})
+
+    def idx
+      s = self
+      s.path.split("/").last.gsub("stage","").to_i
+    end
+
+    def stdout_dataset
+      s = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/stdout")
+    end
+
+    def stderr_dataset
+      s = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/stderr")
+    end
+
+    def log_dataset
+      s = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{s.path}/log")
+    end
+
+    def params
+      s = self
+      #evaluates param_string to ruby hash
+      #using YAML parser
+      #TODO: eliminate ridiculousness
+      begin
+        YAML.load(s.param_string)
+        raise "Must resolve to Hash" unless result.class==Hash
+      rescue
+        sub_param_string = s.param_string.gsub(":\"",": \"").gsub(":'",": '").gsub(":[",": [").gsub(":{",": {").gsub(/(:[0-9])/,'stageparamsgsub\1').gsub('stageparamsgsub:',': ')
+        YAML.load("{#{sub_param_string}}")
+      end
+    end
+
+    def job
+      s = self
+      job_path = s.path.split("/")[0..-2].join("/")
+      Job.where(:path=>job_path).first
+    end
+
+    def Stage.find_or_create_by_path(path)
+      s = Stage.where(:path=>path).first
+      s = Stage.create(:path=>path) unless s
+      return s
+    end
+
+    def prior
+      s = self
+      j = s.job
+      return nil if s.idx==1
+      return j.stages[s.idx-2]
+    end
+
+    def next
+      s = self
+      j = s.job
+      return nil if s.idx == j.stages.length
+      return j.stages[s.idx]
+    end
+
+    def Stage.perform(id,*args)
+      s = Stage.where(:path=>id).first
+      j = s.job
+      s.update_attributes(:started_at=>Time.now.utc)
+      s.update_status(%{Starting at #{Time.now.utc}})
+      stdout, stderr = [nil,nil]
+      begin
+        stdout,log = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path).to_s
+        #write to log if method returns an array w 2 members
+        s.log_dataset.write_cache(log) if log
+      rescue ScriptError, StandardError => exc
+        stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
+        #record the failure in Job so it appears on Runner, turn it off
+        #so it doesn't run again
+        j.update_attributes(:active=>false)
+        s.update_attributes(:failed_at=>Time.now.utc)
+        s.update_status("Failed at #{Time.now.utc.to_s}")
+        raise exc
+      end
+      if stdout == false
+        #re-queue self if output is false
+        s.enqueue!
+        return false
+      end
+      #write output to cache
+      s.stdout_dataset.write_cache(stdout)
+      s.update_attributes(:completed_at=>Time.now.utc)
+      s.update_status("Completed at #{Time.now.utc.to_s}")
+      if s.idx == j.stages.length
+        #job has completed
+        j.update_attributes(:active=>false) if j.trigger.strip.downcase == "once"
+        #check for any dependent jobs, if there are, enqueue them
+        r = j.runner
+        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger.strip.downcase == "after #{j.name}"}
+        #put begin/rescue so all dependencies run
+        dep_jobs.each{|dj| begin;dj.stages.first.enqueue! unless dj.is_working?;rescue;end}
+      else
+        #queue up next stage
+        s.next.enqueue!
+      end
+      return true
+    end
+
+    def enqueue!
+      s = self
+      ::Resque::Job.create("mobilize",Stage,s.path,{})
+      return true
+    end
+
+    def worker
+      s = self
+      Mobilize::Resque.find_worker_by_path(s.path)
+    end
+
+    def worker_args
+      s = self
+      Jobtracker.get_worker_args(s.worker)
+    end
+
+    def set_worker_args(args)
+      s = self
+      Jobtracker.set_worker_args(s.worker,args)
+    end
+
+    def update_status(msg)
+      s = self
+      s.update_attributes(:status=>msg,:status_at=>Time.now.utc)
+      Mobilize::Resque.set_worker_args_by_path(s.path,{'status'=>msg})
+      return true
+    end
+
+    def is_working?
+      s = self
+      Mobilize::Resque.active_paths.include?(s.path)
+    end
+  end
+end

data/lib/mobilize-base/{rakes.rb → tasks.rb}

@@ -35,6 +35,21 @@ namespace :mobilize_base do
     require 'mobilize-base'
     Mobilize::Jobtracker.prep_workers
   end
+  desc "Stop Jobtracker"
+  task :stop do
+    require 'mobilize-base'
+    Mobilize::Jobtracker.stop!
+  end
+  desc "Start Jobtracker"
+  task :start do
+    require 'mobilize-base'
+    Mobilize::Jobtracker.start
+  end
+  desc "Restart Jobtracker"
+  task :restart do
+    require 'mobilize-base'
+    Mobilize::Jobtracker.restart!
+  end
   desc "kill all old resque web processes, start new one with resque_web.rb extension file"
   task :resque_web do
     require 'mobilize-base'

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.75"
+    VERSION = "1.0.81"
   end
 end

data/test/base1_stage1.yml

@@ -0,0 +1,3 @@
+- {test_header: s1, test_header2: s1, test_header3: s1}
+- {test_header: s2, test_header2: s2, test_header3: s2}
+- {test_header: s3, test_header2: s3, test_header3: s3}

data/test/base_job_rows.yml

@@ -2,12 +2,12 @@
   active: true
   trigger: once
   status: ""
-  task1: 'gsheet.read source:"Runner_mobilize(test)/base1_task1.in"'
-  task2: 'gsheet.write source:"base1/task1", target:"Runner_mobilize(test)/base1.out"'
+  stage1: 'gsheet.read source:"Runner_mobilize(test)/base1_stage1.in"'
+  stage2: 'gsheet.write source:"base1/stage1", target:"Runner_mobilize(test)/base1.out"'
 
 - name: "base2"
   active: true
   trigger: "after base1"
   status: ""
-  task1: 'gsheet.read source:"Runner_mobilize(test)/base1.out"'
-  task2: 'gsheet.write source:"task1", target:"Runner_mobilize(test)/base2.out"'
+  stage1: 'gsheet.read source:"Runner_mobilize(test)/base1.out"'
+  stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/base2.out"'

data/test/mobilize-base_test.rb

@@ -32,12 +32,12 @@ describe "Mobilize" do
     r = u.runner
     jobs_sheet = r.gsheet(gdrive_slot)
     tsv = jobs_sheet.to_tsv
-    assert tsv.length == 56 #headers only
+    assert tsv.length == 61 #headers only
 
-    puts "add base1_task1 input sheet"
-    test_source_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_task1.in",gdrive_slot)
+    puts "add base1_stage1 input sheet"
+    test_source_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.in",gdrive_slot)
 
-    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_task1.yml")*40
+    test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_stage1.yml")*40
     test_source_tsv = test_source_ha.hash_array_to_tsv
     test_source_sheet.write(test_source_tsv)
 
@@ -48,13 +48,12 @@ describe "Mobilize" do
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
     test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
-    test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
-
+    test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base2.out",gdrive_slot)
 
     assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
 
     puts "delete both output sheets, set first job to active=true"
-    test_target_sheet_1.delete
+    [test_target_sheet_1,test_target_sheet_2].each{|s| s.delete}
 
     jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
     sleep 90

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.75
+  version: 1.0.81
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-15 00:00:00.000000000 Z
+date: 2012-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -207,9 +207,9 @@ files:
 - lib/mobilize-base/models/dataset.rb
 - lib/mobilize-base/models/job.rb
 - lib/mobilize-base/models/runner.rb
-- lib/mobilize-base/models/task.rb
+- lib/mobilize-base/models/stage.rb
 - lib/mobilize-base/models/user.rb
-- lib/mobilize-base/rakes.rb
+- lib/mobilize-base/tasks.rb
 - lib/mobilize-base/version.rb
 - lib/samples/gdrive.yml
 - lib/samples/gridfs.yml
@@ -219,7 +219,7 @@ files:
 - lib/samples/resque.yml
 - lib/samples/resque_web.rb
 - mobilize-base.gemspec
-- test/base1_task1.yml
+- test/base1_stage1.yml
 - test/base_job_rows.yml
 - test/mobilize-base_test.rb
 - test/redis-test.conf
@@ -238,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2930191387184106525
+      hash: -1990483530432854751
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -247,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2930191387184106525
+      hash: -1990483530432854751
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.24
@@ -256,7 +256,7 @@ specification_version: 3
 summary: Moves datasets and schedules data transfers using MongoDB, Resque and Google
   Docs
 test_files:
-- test/base1_task1.yml
+- test/base1_stage1.yml
 - test/base_job_rows.yml
 - test/mobilize-base_test.rb
 - test/redis-test.conf

data/lib/mobilize-base/models/task.rb

@@ -1,154 +0,0 @@
-module Mobilize
-  class Task
-    include Mongoid::Document
-    include Mongoid::Timestamps
-    field :path, type: String
-    field :handler, type: String
-    field :call, type: String
-    field :param_string, type: Array
-    field :status, type: String
-    field :completed_at, type: Time
-    field :started_at, type: Time
-    field :failed_at, type: Time
-    field :status_at, type: Time
-
-    index({ path: 1})
-
-    def idx
-      t = self
-      t.path.split("/").last.gsub("task","").to_i
-    end
-
-    def stdout_dataset
-      t = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stdout")
-    end
-
-    def stderr_dataset
-      t = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
-    end
-
-    def log_dataset
-      t = self
-      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/log")
-    end
-
-    def params
-      t = self
-      #evaluates param_string to ruby hash
-      #using YAML parser
-      #TODO: eliminate ridiculousness
-      begin
-        YAML.load(t.param_string)
-        raise "Must resolve to Hash" unless result.class==Hash
-      rescue
-        sub_param_string = t.param_string.gsub(":\"",": \"").gsub(":'",": '").gsub(":[",": [").gsub(":{",": {").gsub(/(:[0-9])/,'taskparamsgsub\1').gsub('taskparamsgsub:',': ')
-        YAML.load("{#{sub_param_string}}")
-      end
-    end
-
-    def job
-      t = self
-      job_path = t.path.split("/")[0..-2].join("/")
-      Job.where(:path=>job_path).first
-    end
-
-    def Task.find_or_create_by_path(path)
-      t = Task.where(:path=>path).first
-      t = Task.create(:path=>path) unless t
-      return t
-    end
-
-    def prior
-      t = self
-      j = t.job
-      return nil if t.idx==1
-      return j.tasks[t.idx-2]
-    end
-
-    def next
-      t = self
-      j = t.job
-      return nil if t.idx == j.tasks.length
-      return j.tasks[t.idx]
-    end
-
-    def Task.perform(id,*args)
-      t = Task.where(:path=>id).first
-      j = t.job
-      t.update_status(%{Starting at #{Time.now.utc}})
-      stdout, stderr = [nil,nil]
-      begin
-        stdout,log = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
-        #write to log if method returns an array w 2 members
-        t.log_dataset.write_cache(log) if log
-      rescue ScriptError, StandardError => exc
-        stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
-        #record the failure in Job so it appears on Runner, turn it off
-        #so it doesn't run again
-        j.update_attributes(:active=>false)
-        t.update_attributes(:failed_at=>Time.now.utc)
-        t.update_status("Failed at #{Time.now.utc.to_s}")
-        raise exc
-      end
-      if stdout == false
-        #re-queue self if output is false
-        t.enqueue!
-        return false
-      end
-      #write output to cache
-      t.stdout_dataset.write_cache(stdout)
-      t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
-      if t.idx == j.tasks.length
-        #job has completed
-        j.update_attributes(:active=>false) if j.trigger.strip.downcase == "once"
-        t.update_attributes(:completed_at=>Time.now.utc)
-        t.update_status("Completed at #{Time.now.utc.to_s}")
-        #check for any dependent jobs, if there are, enqueue them
-        r = j.runner
-        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger.strip.downcase == "after #{j.name}"}
-        #put begin/rescue so all dependencies run
-        dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
-      else
-        #queue up next task
-        t.next.enqueue!
-      end
-      return true
-    end
-
-    def enqueue!
-      t = self
-      t.update_attributes(:started_at=>Time.now.utc)
-      ::Resque::Job.create("mobilize",Task,t.path,{})
-      return true
-    end
-
-    def worker
-      t = self
-      Mobilize::Resque.find_worker_by_path(t.path)
-    end
-
-    def worker_args
-      t = self
-      Jobtracker.get_worker_args(t.worker)
-    end
-
-    def set_worker_args(args)
-      t = self
-      Jobtracker.set_worker_args(t.worker,args)
-    end
-
-    def update_status(msg)
-      t = self
-      t.update_attributes(:status=>msg,:status_at=>Time.now.utc)
-      Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
-      return true
-    end
-
-    def is_working?
-      t = self
-      Mobilize::Resque.active_paths.include?(t.path)
-    end
-  end
-end

data/test/base1_task1.yml

@@ -1,3 +0,0 @@
-- {test_header: t1, test_header2: t1, test_header3: t1}
-- {test_header: t2, test_header2: t2, test_header3: t2}
-- {test_header: t3, test_header2: t3, test_header3: t3}