mobilize-base 1.0.55 → 1.0.75

data/README.md

@@ -390,7 +390,7 @@ Mobilize takes the environment from your Rails.env if you're running
 Rails, or assumes "development." You can specify "development", "test",
 or "production," as per the yml files.
 
-Otherwise, it takes it from MOBILIZE_ENV parameter, set from irb, as in:
+Otherwise, it takes it from MOBILIZE_ENV parameter, as in:
 
 ``` ruby
 > ENV['MOBILIZE_ENV'] = 'production'
@@ -493,22 +493,24 @@ name>))` and enter values under each header:
   * Tasks have this syntax: <handler>.<call> <params>.
     * handler specifies the file that should receive the task
     * the call specifies the method within the file. The method should
-be called `"<Handler>.<call>_by_task_path"`
+be called `"<handler>.<call>_by_task_path"`
     * the params the method accepts, which are custom to each
-task. These should be a comma-delimited list, with each param in
-quotes.
+task. These should be of the for `<key1>: <value1>, <key2>: <value2>`, where
+`<key>` is an unquoted string and `<value>` is a quoted string, an
+integer, an array (delimited by square braces), or a hash (delimited by
+curly braces).
     * For mobilize-base, the following tasks are available:
-      * gsheet.read `<input_gsheet_full_path>`, which reads the sheet. 
-        * The gsheet_path should be of the form `<gbook_name>/<gsheet_name>`. The test uses
+      * gsheet.read `source: <input_gsheet_full_path>`, which reads the sheet. 
+        * The gsheet_full_path should be of the form `<gbook_name>/<gsheet_name>`. The test uses
 "Requestor_mobilize(test)/base1_task1.in".
-      * gsheet.write `<task_relative_path>`,`<output_gsheet_path>`,
-which writes the specified task output to the output_gsheet. 
-        * The task_path should be of the form `<task_column>` or
+      * gsheet.write `source: <task_relative_path>`,`target: <target_gsheet_path>`,
+which writes the specified task output to the target_gsheet. 
+        * The task_relative_path should be of the form `<task_column>` or
 `<job_name/task_column>`. The test uses "base1/task1" for the first test
 and simply "task1" for the second test. Both of these take the output
 from the first task.
         * The test uses "Requestor_mobilize(test)/base1.out" and
-"Requestor_mobilize(test)/base2.out" for output sheets.
+"Requestor_mobilize(test)/base2.out" for target sheets.
 
 <a name='section_Start_Run_Test'></a>
 ### Run Test

data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb

@@ -1,6 +1,7 @@
 module GoogleDrive
   class ClientLoginFetcher
     def request_raw(method, url, data, extra_header, auth)
+      clf = self
       #this is patched to handle server errors due to http chaos
       uri = URI.parse(url)
       response = nil
@@ -14,22 +15,32 @@ module GoogleDrive
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         #set 600  to allow for large downloads
         http.read_timeout = 600
-        response = self.http_call(http, method, uri, data, extra_header, auth)
-        if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
-          if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
-            if sleep_time
-              sleep_time = sleep_time * attempts
+        response = begin
+                     clf.http_call(http, method, uri, data, extra_header, auth)
+                   rescue
+                     #timeouts etc.
+                     nil
+                   end
+        if response.nil?
+          attempts +=1
+        else
+          if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
+            if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
+              if sleep_time
+                sleep_time = sleep_time * attempts
+              else
+                sleep_time = (rand*100).to_i
+              end
             else
-              sleep_time = (rand*100).to_i
+              sleep_time = 10
             end
-          else
-            sleep_time = 10
+            attempts += 1
+            puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
+            sleep sleep_time
           end
-          attempts += 1
-          puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
-          sleep sleep_time
         end
       end
+      raise "No response after 5 attempts" if response.nil?
       raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
       return response
     end

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -54,7 +54,11 @@ module GoogleDrive
           raise "Invalid role #{role}"
         end
       else
-        f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
+        begin
+          f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
+        rescue => exc
+          raise exc unless exc.to_s.index("user already has access")
+        end
       end
       return true
     end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -54,6 +54,15 @@ module Mobilize
       return false
     end
 
+    def Gdrive.unslot_worker_by_path(path)
+      begin
+        Mobilize::Resque.set_worker_args_by_path(path,{'gdrive_slot'=>nil})
+        return true
+      rescue
+        return false
+      end
+    end
+
     def Gdrive.root(gdrive_slot=nil)
       pw = Gdrive.password(gdrive_slot)
       GoogleDrive.login(gdrive_slot,pw)

data/lib/mobilize-base/handlers/gfile.rb

@@ -27,7 +27,7 @@ module Mobilize
       gdrive_slot = Gdrive.slot_worker_by_path(t.path)
       return false unless gdrive_slot
       t = Task.where(:path=>task_path)
-      gfile_path = t.params.first
+      gfile_path = t.params['file']
       Gfile.find_by_path(gfile_path,gdrive_slot).read
     end
   end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -37,7 +37,7 @@ module Mobilize
       gdrive_slot = Gdrive.slot_worker_by_path(task_path)
       return false unless gdrive_slot
       t = Task.where(:path=>task_path).first
-      gsheet_path = t.params.first
+      gsheet_path = t.params['source']
       Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
     end
 
@@ -46,8 +46,8 @@ module Mobilize
       #return false if there are no emails available
       return false unless gdrive_slot
       t = Task.where(:path=>task_path).first
-      source = t.params.first
-      target_path = t.params.second
+      source = t.params['source']
+      target_path = t.params['target']
       source_job_name, source_task_name = if source.index("/")
                                             source.split("/")
                                           else

data/lib/mobilize-base/models/job.rb

@@ -5,8 +5,6 @@ module Mobilize
     field :path, type: String
     field :active, type: Boolean
     field :trigger, type: String
-    field :status, type: String
-    field :last_completed_at, type: Time
 
     index({ path: 1})
 
@@ -26,6 +24,32 @@ module Mobilize
       return j
     end
 
+    def status
+      #last task status
+      j = self
+      j.active_task.status
+    end
+
+    def active_task
+      j = self
+      #latest started at or first
+      j.tasks.select{|t| t.started_at}.sort_by{|t| t.started_at}.last || j.tasks.first
+    end
+
+    def completed_at
+      j = self
+      j.tasks.last.completed_at
+    end
+
+    def failed_at
+      j = self
+      j.tasks.last.failed_at
+    end
+
+    def status_at
+      j.active_task.status_at
+    end
+
     #convenience methods
     def runner
       j = self
@@ -41,7 +65,7 @@ module Mobilize
     def is_due?
       j = self
       return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
-      last_run = j.last_completed_at
+      last_run = j.completed_at
       #check trigger
       trigger = j.trigger
       return true if trigger == 'once'

data/lib/mobilize-base/models/runner.rb

@@ -5,7 +5,9 @@ module Mobilize
     field :path, type: String
     field :active, type: Boolean
     field :status, type: String
-    field :last_run, type: Time
+    field :started_at, type: Time
+    field :status_at, type: Time
+    field :completed_at, type: Time
 
     index({ path: 1})
 
@@ -13,9 +15,9 @@ module Mobilize
       %w{name active trigger status task1 task2 task3 task4 task5}
     end
 
-    def last_cached_at
+    def cached_at
       r = self
-      Dataset.find_or_create_by_path(r.path).last_cached_at
+      Dataset.find_or_create_by_path(r.path).cached_at
     end
 
     def worker
@@ -51,12 +53,12 @@ module Mobilize
             j.tasks.first.enqueue!
           end
         rescue ScriptError, StandardError => exc
-          j.update_status("Failed to enqueue with #{exc.to_s}")
+          r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
           j.update_attributes(:active=>false)
         end
       end
       r.update_gsheet(gdrive_slot)
-      r.update_attributes(:last_run=>Time.now.utc)
+      r.update_attributes(:completed_at=>Time.now.utc)
     end
 
     def dataset
@@ -77,7 +79,7 @@ module Mobilize
       r = self
       jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
       jobs_sheet.add_headers(r.headers)
-      jobs_sheet.delete_sheet1
+      begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
       return jobs_sheet
     end
 
@@ -128,7 +130,12 @@ module Mobilize
     def update_gsheet(gdrive_slot)
       r = self
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_rows = r.jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status} }
+      upd_jobs = r.jobs.select do |j|
+                                 j.completed_at.nil? ||
+                                 j.completed_at > j.runner.completed_at ||
+                                 (j.failed_at and j.failed_at > j.runner.completed_at)
+                               end
+      upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
       return true
@@ -152,7 +159,7 @@ module Mobilize
 
     def update_status(msg)
       r = self
-      r.update_attributes(:status=>msg)
+      r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
       Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
       return true
     end
@@ -165,12 +172,13 @@ module Mobilize
     def is_due?
       r = self.reload
       return false if r.is_working?
-      last_due_time = Time.now.utc - Jobtracker.runner_read_freq
-      return true if r.last_run.nil? or r.last_run < last_due_time
+      prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
+      return true if r.started_at.nil? or r.started_at < prev_due_time
     end
 
     def enqueue!
       r = self
+      r.update_attributes(:started_at=>Time.now.utc)
       ::Resque::Job.create("mobilize",Runner,r.path,{})
       return true
     end

data/lib/mobilize-base/models/task.rb

@@ -7,8 +7,10 @@ module Mobilize
     field :call, type: String
     field :param_string, type: Array
     field :status, type: String
-    field :last_completed_at, type: Time
-    field :last_run_at, type: Time
+    field :completed_at, type: Time
+    field :started_at, type: Time
+    field :failed_at, type: Time
+    field :status_at, type: Time
 
     index({ path: 1})
 
@@ -27,13 +29,22 @@ module Mobilize
       Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/stderr")
     end
 
+    def log_dataset
+      t = self
+      Dataset.find_or_create_by_handler_and_path("gridfs","#{t.path}/log")
+    end
+
     def params
       t = self
-      t.param_string.split(",").map do |p| 
-        ps = p.strip
-        ps = ps[1..-1] if ['"',"'"].include?(ps[0])
-        ps = ps[0..-2] if ['"',"'"].include?(ps[-1])
-        ps
+      #evaluates param_string to ruby hash
+      #using YAML parser
+      #TODO: eliminate ridiculousness
+      begin
+        YAML.load(t.param_string)
+        raise "Must resolve to Hash" unless result.class==Hash
+      rescue
+        sub_param_string = t.param_string.gsub(":\"",": \"").gsub(":'",": '").gsub(":[",": [").gsub(":{",": {").gsub(/(:[0-9])/,'taskparamsgsub\1').gsub('taskparamsgsub:',': ')
+        YAML.load("{#{sub_param_string}}")
       end
     end
 
@@ -69,12 +80,16 @@ module Mobilize
       t.update_status(%{Starting at #{Time.now.utc}})
       stdout, stderr = [nil,nil]
       begin
-        stdout = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
+        stdout,log = "Mobilize::#{t.handler.humanize}".constantize.send("#{t.call}_by_task_path",t.path).to_s
+        #write to log if method returns an array w 2 members
+        t.log_dataset.write_cache(log) if log
       rescue ScriptError, StandardError => exc
         stderr = [exc.to_s,exc.backtrace.to_s].join("\n")
-        #record the failure in Job so it appears on Runner
-        j.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
-        t.update_attributes(:status=>"Failed at #{Time.now.utc.to_s}")
+        #record the failure in Job so it appears on Runner, turn it off
+        #so it doesn't run again
+        j.update_attributes(:active=>false)
+        t.update_attributes(:failed_at=>Time.now.utc)
+        t.update_status("Failed at #{Time.now.utc.to_s}")
         raise exc
       end
       if stdout == false
@@ -86,12 +101,13 @@ module Mobilize
       t.stdout_dataset.write_cache(stdout)
       t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}")
       if t.idx == j.tasks.length
-        j.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
-        j.update_attributes(:active=>false) if j.trigger.strip == "once"
-        t.update_attributes(:status=>"Completed at #{Time.now.utc.to_s}",:last_completed_at=>Time.now.utc)
+        #job has completed
+        j.update_attributes(:active=>false) if j.trigger.strip.downcase == "once"
+        t.update_attributes(:completed_at=>Time.now.utc)
+        t.update_status("Completed at #{Time.now.utc.to_s}")
         #check for any dependent jobs, if there are, enqueue them
         r = j.runner
-        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger=="after #{j.name}"}
+        dep_jobs = r.jobs.select{|dj| dj.active==true and dj.trigger.strip.downcase == "after #{j.name}"}
         #put begin/rescue so all dependencies run
         dep_jobs.each{|dj| begin;dj.tasks.first.enqueue! unless dj.is_working?;rescue;end}
       else
@@ -103,6 +119,7 @@ module Mobilize
 
     def enqueue!
       t = self
+      t.update_attributes(:started_at=>Time.now.utc)
       ::Resque::Job.create("mobilize",Task,t.path,{})
       return true
     end
@@ -124,7 +141,7 @@ module Mobilize
 
     def update_status(msg)
       t = self
-      t.update_attributes(:status=>msg)
+      t.update_attributes(:status=>msg,:status_at=>Time.now.utc)
       Mobilize::Resque.set_worker_args_by_path(t.path,{'status'=>msg})
       return true
     end

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.55"
+    VERSION = "1.0.75"
   end
 end

data/mobilize-base.gemspec

@@ -29,7 +29,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'redis',"~>3.0.0"
   s.add_runtime_dependency 'resque','1.21.0'
   s.add_runtime_dependency 'google_drive','0.3.2'
-  s.add_runtime_dependency 'bluepill','0.0.60'
   s.add_runtime_dependency 'popen4','0.1.2'
   s.add_runtime_dependency 'actionmailer','3.1.1'
 end

data/test/base_job_rows.yml

@@ -2,12 +2,12 @@
   active: true
   trigger: once
   status: ""
-  task1: 'gsheet.read "Runner_mobilize(test)/base1_task1.in"'
-  task2: 'gsheet.write "base1/task1", "Runner_mobilize(test)/base1.out"'
+  task1: 'gsheet.read source:"Runner_mobilize(test)/base1_task1.in"'
+  task2: 'gsheet.write source:"base1/task1", target:"Runner_mobilize(test)/base1.out"'
 
 - name: "base2"
   active: true
   trigger: "after base1"
   status: ""
-  task1: 'gsheet.read "Runner_mobilize(test)/base1.out"'
-  task2: 'gsheet.write "task1", "Runner_mobilize(test)/base2.out"'
+  task1: 'gsheet.read source:"Runner_mobilize(test)/base1.out"'
+  task2: 'gsheet.write source:"task1", target:"Runner_mobilize(test)/base2.out"'

data/test/mobilize-base_test.rb

@@ -44,15 +44,24 @@ describe "Mobilize" do
     puts "add row to jobs sheet, wait 120s"
     test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
     jobs_sheet.add_or_update_rows(test_job_rows)
-
-    puts "job row added, force enqueued runner"
-    r.enqueue!
     sleep 120
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    test_target_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
+    test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
+    test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
+
+
+    assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
 
-    assert test_target_sheet.to_tsv == test_source_sheet.to_tsv
+    puts "delete both output sheets, set first job to active=true"
+    test_target_sheet_1.delete
+
+    jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
+    sleep 90
+    
+    test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
+    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
+    assert test_target_sheet_2.to_tsv == test_source_sheet.to_tsv
 
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.55
+  version: 1.0.75
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-13 00:00:00.000000000 Z
+date: 2012-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -139,22 +139,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.3.2
-- !ruby/object:Gem::Dependency
-  name: bluepill
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.0.60
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.0.60
 - !ruby/object:Gem::Dependency
   name: popen4
   requirement: !ruby/object:Gem::Requirement
@@ -254,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3858742184331837226
+      hash: 2930191387184106525
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -263,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3858742184331837226
+      hash: 2930191387184106525
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.24