mobilize-base 1.0.0 → 1.0.1

data/README.md

@@ -440,19 +440,19 @@ name>)` and enter values under each header:
 * write_handler	This is where the job writes its data to. For
   mobilize-base, you should enter "gsheet"
 
-* param_source This is the path to an array of data, as read from a google sheet,
-  that is relayed to the job.
+* param_sheets This is a comma-delimited list of sheets, relayed to the job,
+which can be used for parameters.
   The format is `<google docs book>/<google docs sheet>`, so if you
-wanted to read from the "output" sheet on the "monthly results" book you
-would write in `<monthly results>/<output>`. For a sheet in the Jobspec
+wanted to read from the "output" sheet on the "monthly_results" book you
+would write in `monthly_results/output`. For a sheet in the Jobspec
 itself you could write simply `<output>`.
 
-* params This is a hash of data, expressed in a JSON. Not relevant to
-  mobilize-base
+* params This is a hash of data, expressed in a JSON, which can be used
+for parameters.
 
 * destination This is the destination for the data, relayed to the job.
   For a gsheet write_handler, this would be the name of the sheet to be
-written to, similar to param_source.
+written to, similar to param_sheets.
 
 <a name='section_Start_Run_Test'></a>
 ### Run Test

data/lib/mobilize-base/handlers/gdriver.rb

@@ -231,17 +231,11 @@ module Mobilize
 
     def Gsheeter.read_by_job_id(job_id)
       j = Job.find(job_id)
-      r = j.requestor
       #reserve email account for read
       email = Gdriver.get_worker_email_by_mongo_id(job_id)
       return false unless email
-      source = j.param_source
-      book,sheet = source.split("/")
-      #assume jobspec source if none given
-      source = [r.jobspec_title,source].join("/") if sheet.nil?
-      tsv = Gsheeter.find_or_create_by_name(source,email).to_tsv
-      book = nil
-      return tsv
+      #pull tsv from cache
+      j.param_sheet_dsts.first.read_cache
     end
 
     def Gsheeter.read_by_dst_id(dst_id,email=nil)
@@ -277,32 +271,32 @@ module Mobilize
     def Gsheeter.write_by_job_id(job_id)
       j = Job.find(job_id)
       r = j.requestor
-      dest_name = if j.destination.split("/").length==1
+      tgt_name = if j.destination.split("/").length==1
                     "#{r.jobspec_title}#{"/"}#{j.destination}"
                   else
                     j.destination
                   end
-      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',dest_name)
+      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',tgt_name)
       sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
       email = Gdriver.get_worker_email_by_mongo_id(job_id)
       #return false if there are no emails available
       return false unless email
       #create temp tab, write data to it, checksum it against the source
-      tempsheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',"#{dest_name}_temp")
+      tempsheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',"#{tgt_name}_temp")
       tempsheet_dst.update_attributes(:requestor_id=>r.id.to_s) if tempsheet_dst.requestor_id.nil?
       tempsheet = Gsheeter.find_or_create_by_dst_id(tempsheet_dst.id.to_s)
       #tsv is the second to last stage's output (the last is the write)
       tsv = Dataset.find(j.tasks[j.prior_task]['output_dst_id']).read
       tempsheet.write(tsv,true,job_id)
       #delete current sheet, replace it with temp one
-      sheet = Gsheeter.find_or_create_by_name(dest_name,email)
+      sheet = Gsheeter.find_or_create_by_name(tgt_name,email)
       title = sheet.title
       #http
       sheet.delete
       tempsheet.title = title
       tempsheet.save
       sheet_dst.update_attributes(:url=>tempsheet.spreadsheet.human_url)
-      "Write successful for #{dest_name}".oputs
+      "Write successful for #{tgt_name}".oputs
       return true
     end
   end

data/lib/mobilize-base/jobtracker.rb

@@ -193,7 +193,7 @@ module Mobilize
         requestors = Requestor.all
         Jobtracker.run_notifications
         requestors.each do |r|
-          Jobtracker.update_status("Running requestor #{r.name}")
+          Jobtracker.update_status("Checking requestor #{r.name}")
           if r.is_due?
             r.enqueue!
             Jobtracker.update_status("Enqueued requestor #{r.name}")

data/lib/mobilize-base/models/job.rb

@@ -14,8 +14,8 @@ module Mobilize
     field :last_completed_at, type: Time
     field :read_handler, type: String
     field :write_handler, type: String
-    field :param_source, type: String #name of sheet on doc
-    field :param_hash, type: String #JSON
+    field :files, type: String #name of sheet(s) on doc
+    field :params, type: String #JSON
     field :destination, type: String #output destination - could be file, could be sheet
 
     index({ requestor_id: 1})
@@ -28,6 +28,19 @@ module Mobilize
       Mobilize::Resque.find_worker_by_mongo_id(j.id.to_s)
     end
 
+    def param_sheet_dsts
+      j = self
+      r = j.requestor
+      j.param_sheets.split(",").map do |ps|
+        #prepend jobspec title if there is no path separator
+        full_ps = ps.index("/") ? ps : [r.jobspec_title,ps].join("/")
+        #find or create dataset for this sheet
+        dst = Dataset.find_or_create_by_handler_and_name("gsheeter",full_ps)
+        dst.update_attributes(:requestor_id=>r.id.to_s) unless dst.requestor_id
+        dst
+      end
+    end
+
     def Job.find_by_name(name)
       Job.where(:name=>name).first
     end
@@ -186,11 +199,11 @@ module Mobilize
 
     def is_due?
       j = self
-      return false if j.is_working? or j.schedule.to_s.starts_with?("after")
+      return false if j.is_working? or j.active == false or j.schedule.to_s.starts_with?("after")
       last_run = j.last_completed_at
       #check schedule
       schedule = j.schedule
-      return true if schedule == 'once' and j.active
+      return true if schedule == 'once'
       #strip the "every" from the front if present
       schedule = schedule.gsub("every","").gsub("."," ").strip
       value,unit,operator,job_utctime = schedule.split(" ")

data/lib/mobilize-base/models/requestor.rb

@@ -31,14 +31,17 @@ module Mobilize
     end
 
     def Requestor.jobs_sheet_headers
-      %w{name active schedule status last_error destination_url read_handler write_handler param_source params destination}
+      %w{name active schedule status last_error destination_url read_handler write_handler param_sheets params destination}
     end
 
     def Requestor.perform(id,*args)
       r = Requestor.find(id.to_s)
       #reserve email account for read
       gdrive_email = Gdriver.get_worker_email_by_mongo_id(id)
-      return false unless gdrive_email
+      unless gdrive_email
+        "no gdrive_email available for #{r.name}".oputs
+        return false
+      end
       jobs_sheet = r.jobs_sheet(gdrive_email)
       #write headers to sheet
       Requestor.jobs_sheet_headers.each_with_index do |h,h_i|
@@ -52,7 +55,16 @@ module Mobilize
       #queue up the jobs that are due and active
       r.jobs.each do |j|
         begin
-          j.enqueue! if j.active and j.is_due?
+          if j.active and j.is_due?
+            #cache all param_sheets
+            j.param_sheet_dsts.each do |psdst|
+             #read tsv, write to cache for job to use
+              tsv = Gsheeter.find_or_create_by_name(psdst.name,gdrive_email).to_tsv
+              r.update_status("caching #{psdst.name}")
+              psdst.write_cache(tsv)
+            end
+            j.enqueue!
+          end
         rescue ScriptError,StandardError => exc
           #update errors
           j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
@@ -85,7 +97,7 @@ module Mobilize
                             :schedule => rj['schedule'],
                             :read_handler => rj['read_handler'],
                             :write_handler => rj['write_handler'],
-                            :param_source => rj['param_source'],
+                            :param_sheets => rj['param_sheets'],
                             :params => rj['params'],
                             :destination => rj['destination'])
         #update laststatus with "Created job for" if job is due
@@ -207,7 +219,7 @@ module Mobilize
     end
 
     def is_due?
-      r = self
+      r = self.reload
       return false if r.is_working?
       last_due_time = Time.now.utc - Jobtracker.requestor_refresh_freq
       return true if r.last_run.nil? or r.last_run < last_due_time

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.0"
+    VERSION = "1.0.1"
   end
 end

data/test/mobilize_test.rb

@@ -9,6 +9,13 @@ describe "Mobilize" do
 
   # enqueues 4 workers on Resque
   it "runs integration test" do
+    puts "clear out test db"
+    Mongoid.session(:default).collections.each do |collection| 
+      unless collection.name =~ /^system\./
+        collection.drop
+      end
+    end
+
     email = Mobilize::Gdriver.owner_email
 
     #kill all workers
@@ -19,9 +26,6 @@ describe "Mobilize" do
     sleep 20
     assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
 
-    #make sure old one is deleted
-    Mobilize::Requestor.find_or_create_by_email(email).delete
-
     puts "create requestor 'mobilize'"
     requestor = Mobilize::Requestor.find_or_create_by_email(email)
     assert requestor.email == email
@@ -31,12 +35,10 @@ describe "Mobilize" do
     jobspec_title = requestor.jobspec_title
     books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
     books.each{|book| book.delete}
-    #delete old datasets for this specbook
-    Mobilize::Dataset.all.select{|d| d.name.starts_with?(jobspec_title)}.each{|d| d.delete}
 
-    puts "enqueue jobtracker, wait 60s"
+    puts "enqueue jobtracker, wait 45s"
     Mobilize::Jobtracker.start
-    sleep 60
+    sleep 45
     puts "jobtracker status: #{Mobilize::Jobtracker.status}" 
     puts "status:#{Mobilize::Jobtracker.status}" #!= 'stopped'
 
@@ -62,10 +64,7 @@ describe "Mobilize" do
     test_source_tsv = test_source_rows.map{|r| r.join("\t")}.join("\n")
     test_source_sheet.write(test_source_tsv)
 
-    puts "add row to jobs sheet, wait 100s"
-
-    #delete existing Jobs from the db
-    Mobilize::Job.each{|j| j.delete}
+    puts "add row to jobs sheet, wait 60s"
 
     jobs_sheet = jobs_sheets.first
 
@@ -77,7 +76,7 @@ describe "Mobilize" do
               "destination_url" => "",
                  "read_handler" => "gsheeter",
                 "write_handler" => "gsheeter",
-                 "param_source" => "test_source",
+                 "param_sheets" => "test_source",
                        "params" => "",
                   "destination" => "test_destination"},
                   #run after the first
@@ -89,7 +88,7 @@ describe "Mobilize" do
               "destination_url" => "",
                  "read_handler" => "gsheeter",
                 "write_handler" => "gsheeter",
-                 "param_source" => "test_source",
+                 "param_sheets" => "test_source",
                        "params" => "",
                   "destination" => "test_destination2"}
     ]
@@ -105,9 +104,9 @@ describe "Mobilize" do
 
     puts "job row added, force enqueued requestor"
     requestor.enqueue!
-    sleep 100
+    sleep 60
 
-    puts "jobtracker posted test source data to test destination, and checksum succeeded?"
+    puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
     test_destination_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_destination",email)
 
     assert test_destination_sheet.to_tsv == test_source_sheet.to_tsv

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-24 00:00:00.000000000 Z
+date: 2012-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake