mobilize-base 1.0.1 → 1.0.2

data/lib/mobilize-base/models/job.rb

@@ -7,14 +7,12 @@ module Mobilize
     field :active, type: Boolean #active, inactive
     field :schedule, type: String
     field :active_task, type: String
-    field :tasks, type: Hash
+    field :tasks, type: String
     field :status, type: String
     field :last_error, type: String
     field :last_trace, type: String
     field :last_completed_at, type: Time
-    field :read_handler, type: String
-    field :write_handler, type: String
-    field :files, type: String #name of sheet(s) on doc
+    field :datasets, type: String #name of data sources
     field :params, type: String #JSON
     field :destination, type: String #output destination - could be file, could be sheet
 
@@ -28,19 +26,38 @@ module Mobilize
       Mobilize::Resque.find_worker_by_mongo_id(j.id.to_s)
     end
 
-    def param_sheet_dsts
+    def dataset_array
       j = self
       r = j.requestor
-      j.param_sheets.split(",").map do |ps|
+      dsts = j.datasets.split(",").map{|dst| dst.strip}
+      dsts.map do |ps|
         #prepend jobspec title if there is no path separator
         full_ps = ps.index("/") ? ps : [r.jobspec_title,ps].join("/")
         #find or create dataset for this sheet
-        dst = Dataset.find_or_create_by_handler_and_name("gsheeter",full_ps)
+        dst = Dataset.find_or_create_by_handler_and_name("gsheet",full_ps)
         dst.update_attributes(:requestor_id=>r.id.to_s) unless dst.requestor_id
         dst
       end
     end
 
+    def task_array
+      self.tasks.split(",").map{|t| t.strip}
+    end
+
+    def task_output_dsts
+      j = self
+      r = j.requestor
+      dst_names = j.task_array.map{|tname| [r.name,j.name,tname].join("/")}
+      dst_names.map do |dst_name|
+        Dataset.find_or_create_by_requestor_id_and_handler_and_name(r.id.to_s,'mongodb',dst_name)
+      end
+    end
+
+    def task_idx
+      j = self
+      j.task_array.index(j.active_task)
+    end
+
     def Job.find_by_name(name)
       Job.where(:name=>name).first
     end
@@ -58,24 +75,18 @@ module Mobilize
     #called by Resque
     def Job.perform(id,*args)
       j = Job.find(id)
-      task_params = j.tasks[j.active_task]
+      r = j.requestor
+      handler,method_name = j.active_task.split(".")
       begin
         j.update_status(%{Starting #{j.active_task} task at #{Time.now.utc}})
-        task_output = "Mobilize::#{task_params['handler'].humanize}".constantize.send(j.active_task,id)
+        task_output = "Mobilize::#{handler.humanize}".constantize.send("#{method_name}_by_job_id",id)
         #this allows user to return false if the stage didn't go as expected and needs to retry
         #e.g. tried to write to Google but all the accounts were in use
         return false if task_output == false
-        task_output_dst_id = if task_output.to_s.length==24 and Dataset.find(task_output.to_s)
-                              #user has returned dst as output from task
-                              task_output
-                            else
-                              #store the output in a cache
-                              dst = Dataset.find_or_create_by_requestor_id_and_handler_and_name(j.requestor.id.to_s,'mongoer',"#{j.id.to_s}/#{j.active_task}")
-                              dst.write_cache(task_output.to_s)
-                              dst.id.to_s
-                            end
-        j.tasks[j.active_task]['output_dst_id'] = task_output_dst_id
-        if j.active_task == j.tasks.keys.last
+        task_output_dst = j.task_output_dsts[j.task_idx]
+        task_output = task_output.to_s unless task_output.class == String
+        task_output_dst.write_cache(task_output)
+        if j.active_task == j.task_array.last
           j.active_task = nil
           j.last_error = ""
           j.last_trace = ""
@@ -89,9 +100,8 @@ module Mobilize
           #put begin/rescue so all dependencies run
           dep_jobs.each{|dj| begin;dj.enqueue! unless dj.is_working?;rescue;end}
         else
-          task_names = j.tasks.keys
-          stage_idx = task_names.index(j.active_task) + 1
-          j.active_task = j.tasks.keys[stage_idx]
+          #set next task
+          j.active_task = j.task_array[j.task_idx+1]
           j.save!
           #queue up next task
           j.enqueue!
@@ -111,14 +121,9 @@ module Mobilize
 
     def enqueue!
       j = self
-      #assign first task if none assigned
-      if j.tasks.blank?
-        #make a hash with the read/write tasks
-        j.update_attributes(:tasks=>{"read_by_job_id"=>{'handler'=>j.read_handler},
-                                     "write_by_job_id"=>{'handler'=>j.write_handler}})
-      end
-      j.update_attributes(:active_task=>"read_by_job_id") if j.active_task.blank?
-      ::Resque::Job.create("mobilize",Job,j.id.to_s,%{#{j.requestor.name}=>#{j.name}})
+      r = j.requestor
+      j.update_attributes(:active_task=>j.task_array.first) if j.active_task.blank?
+      ::Resque::Job.create("mobilize",Job,j.id.to_s,%{#{r.name}=>#{j.name}})
       return true
     end
 
@@ -137,23 +142,23 @@ module Mobilize
     def prior_task
       j = self
       return nil if j.active_task.nil?
-      task_idx = j.tasks.keys.index(j.active_task)
+      task_idx = j.task_array.index(j.active_task)
       return nil if task_idx==0
-      return j.tasks.keys[task_idx-1]
+      return j.task_array[task_idx-1]
     end
 
     def destination_url
       j = self
       return nil if j.destination.nil?
       destination = j.destination
-      dst = if j.write_handler == 'gsheet'
+      dst = if j.task_array.last == 'gsheet.write'
               destination = [j.requestor.jobspec_title,j.destination].join("/") if destination.split("/").length==1
-              Dataset.find_by_handler_and_name('gsheeter',destination)
-            elsif j.write_handler == 'gtxt'
-              #all gtxt files must end in gz
+              Dataset.find_by_handler_and_name('gsheet',destination)
+            elsif j.task_array.last == 'gfile.write'
+              #all gfiles must end in gz
               destination += ".gz" unless destination.ends_with?(".gz")
               destination = [s.requestor.name,"_"].join + destination unless destination.starts_with?([s.requestor.name,"_"].join)
-              Dataset.find_by_handler_and_name('gtxter',destination)
+              Dataset.find_by_handler_and_name('gfile',destination)
             end
       return dst.url if dst
     end
@@ -168,23 +173,6 @@ module Mobilize
       Jobtracker.set_worker_args(j.worker,args)
     end
 
-    def cache_params
-      j = self
-      #go to paramsheet and read
-      param_path = if j.paramsheet.split("/").length==1
-                     [j.requestor.jobspec_title,j.paramsheet].join("/")
-                   else
-                     j.paramsheet
-                   end
-      param_sheet = j.requestor.find_or_create_gsheet_by_path(param_path)
-      param_tsv = param_sheet.to_tsv
-      param_dst = j.requestor.gsheets.select{|s| s.path == param_sheet.path}
-      param_dst.cache.write(param_tsv)
-      s.update_attributes(:param_dst_id=>paramdst.id.to_s)
-      (s.requestor.name + "'s #{s.name} params cached").oputs
-      return true
-    end
-
     def update_status(msg)
       j = self
       j.update_attributes(:status=>msg)

data/lib/mobilize-base/models/requestor.rb

@@ -31,13 +31,13 @@ module Mobilize
     end
 
     def Requestor.jobs_sheet_headers
-      %w{name active schedule status last_error destination_url read_handler write_handler param_sheets params destination}
+      %w{name active schedule status last_error destination_url tasks datasets params destination}
     end
 
     def Requestor.perform(id,*args)
       r = Requestor.find(id.to_s)
       #reserve email account for read
-      gdrive_email = Gdriver.get_worker_email_by_mongo_id(id)
+      gdrive_email = Gdrive.get_worker_email_by_mongo_id(id)
       unless gdrive_email
         "no gdrive_email available for #{r.name}".oputs
         return false
@@ -56,12 +56,12 @@ module Mobilize
       r.jobs.each do |j|
         begin
           if j.active and j.is_due?
-            #cache all param_sheets
-            j.param_sheet_dsts.each do |psdst|
+            #cache all datasets
+            j.dataset_array.each do |dst|
              #read tsv, write to cache for job to use
-              tsv = Gsheeter.find_or_create_by_name(psdst.name,gdrive_email).to_tsv
-              r.update_status("caching #{psdst.name}")
-              psdst.write_cache(tsv)
+              tsv = Gsheet.find_or_create_by_name(dst.name,gdrive_email).to_tsv
+              r.update_status("caching #{dst.name}")
+              dst.write_cache(tsv)
             end
             j.enqueue!
           end
@@ -90,14 +90,13 @@ module Mobilize
       loc_jobs = []
       rem_jobs.each_with_index do |rj,rj_i|
         #skip bad rows
-        next if (rj['name'].to_s.first == "#" or ['name','schedule','read_handler','write_handler','active'].select{|c| rj[c].to_s.strip==""}.length>0)
+        next if (rj['name'].to_s.first == "#" or ['name','schedule','tasks','active'].select{|c| rj[c].to_s.strip==""}.length>0)
         j = Job.find_or_create_by_requestor_id_and_name(r.id.to_s,rj['name'])
         #update top line params
         j.update_attributes(:active => rj['active'],
                             :schedule => rj['schedule'],
-                            :read_handler => rj['read_handler'],
-                            :write_handler => rj['write_handler'],
-                            :param_sheets => rj['param_sheets'],
+                            :tasks => rj['tasks'],
+                            :datasets => rj['datasets'],
                             :params => rj['params'],
                             :destination => rj['destination'])
         #update laststatus with "Created job for" if job is due
@@ -125,20 +124,19 @@ module Mobilize
       #go through each job, update relevant job with its params
       headers = Requestor.jobs_sheet_headers
       #write headers
-      headers.each_with_index do |h,h_i|
-        jobs_sheet[1,h_i+1] = h
-      end
+      jobs_sheet.add_headers(headers)
       #write rows
       rem_jobs.each_with_index do |rj,rj_i|
         #skip bad rows
-        next if (rj['name'].to_s.first == "#" or ['name','schedule','read_handler','write_handler','active'].select{|c| rj[c].to_s.strip==""}.length>0)
-        j = r.jobs(rj['name'])
-        #update active to false if this was a run once
-        j.update_attributes(:active=>false) if j.schedule.to_s == 'once'
-        jobs_sheet[rj_i+2,headers.index('active')+1] = j.active.to_s
-        jobs_sheet[rj_i+2,headers.index('status')+1] = j.status.to_s.gsub("\n",";").gsub("\t"," ")
-        jobs_sheet[rj_i+2,headers.index('last_error')+1] = j.last_error.to_s.gsub("\n",";").gsub("\t"," ")
-        jobs_sheet[rj_i+2,headers.index('destination_url')+1] = j.destination_url.to_s
+        next if (rj['name'].to_s.first == "#" or ['name','schedule','tasks','active'].select{|c| rj[c].to_s.strip==""}.length>0)
+        if j = r.jobs(rj['name'])
+          #update active to false if this was a run once
+          j.update_attributes(:active=>false) if j.schedule.to_s == 'once'
+          jobs_sheet[rj_i+2,headers.index('active')+1] = j.active.to_s
+          jobs_sheet[rj_i+2,headers.index('status')+1] = j.status.to_s.gsub("\n",";").gsub("\t"," ")
+          jobs_sheet[rj_i+2,headers.index('last_error')+1] = j.last_error.to_s.gsub("\n",";").gsub("\t"," ")
+          jobs_sheet[rj_i+2,headers.index('destination_url')+1] = j.destination_url.to_s
+        end
       end
       jobs_sheet.save
       r.update_status(r.name + " jobs written")
@@ -166,18 +164,18 @@ module Mobilize
 
     def find_or_create_gbook_by_title(title,gdrive_email)
       r = self
-      book_dst = Dataset.find_or_create_by_handler_and_name('gbooker',title)
+      book_dst = Dataset.find_or_create_by_handler_and_name('gbook',title)
       #give dst this requestor if none
       book_dst.update_attributes(:requestor_id=>r.id.to_s) if book_dst.requestor_id.nil?
-      book = Gbooker.find_or_create_by_dst_id(book_dst.id.to_s,gdrive_email)
+      book = Gbook.find_or_create_by_dst_id(book_dst.id.to_s,gdrive_email)
       return book
     end
 
     def find_or_create_gsheet_by_name(name,gdrive_email)
       r = self
-      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',name)
+      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheet',name)
       sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
-      sheet = Gsheeter.find_or_create_by_dst_id(sheet_dst.id.to_s,gdrive_email)
+      sheet = Gsheet.find_or_create_by_dst_id(sheet_dst.id.to_s,gdrive_email)
       return sheet
     end
 
@@ -230,6 +228,5 @@ module Mobilize
       ::Resque::Job.create("mobilize",Requestor,r.id.to_s,{"name"=>r.name})
       return true
     end
-
   end
 end

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.0.1"
+    VERSION = "1.0.2"
   end
 end

data/lib/mobilize-base.rb

@@ -67,10 +67,10 @@ require "mobilize-base/extensions/resque"
 Resque.redis = "127.0.0.1:#{Mobilize::Resque.config['redis_port']}"
 require 'popen4'
 require "mobilize-base/jobtracker"
-require "mobilize-base/handlers/gdriver"
-require "mobilize-base/extensions/google_drive.rb"
-require "mobilize-base/handlers/mongoer"
-require "mobilize-base/handlers/emailer"
-
-#require "mobilize-base/handlers/*"
-#require "mobilize-base/models/*"
+require "mobilize-base/handlers/gdrive"
+require "mobilize-base/handlers/gfile"
+require "mobilize-base/handlers/gbook"
+require "mobilize-base/handlers/gsheet"
+require "mobilize-base/extensions/google_drive"
+require "mobilize-base/handlers/mongodb"
+require "mobilize-base/handlers/email"

data/lib/samples/gdrive.yml

@@ -1,4 +1,5 @@
 development:
+  domain: 'host.com'
   owner:
     email: 'owner_development@host.com'
     pw: "google_drive_password"
@@ -8,6 +9,7 @@ development:
     - {email: 'worker_development001@host.com', pw: "worker001_google_drive_password"}
     - {email: 'worker_development002@host.com', pw: "worker002_google_drive_password"}
 test:
+  domain: 'host.com'
   owner:
     email: 'owner_test@host.com'
     pw: "google_drive_password"
@@ -17,6 +19,7 @@ test:
     - {email: 'worker_test001@host.com', pw: "worker001_google_drive_password"}
     - {email: 'worker_test002@host.com', pw: "worker002_google_drive_password"}
 production:
+  domain: 'host.com'
   owner:
     email: 'owner_production@host.com'
     pw: "google_drive_password"

data/test/mobilize_test.rb

@@ -3,20 +3,19 @@ require 'test_helper'
 describe "Mobilize" do
 
   def before
-    puts 'before'
-
+    puts 'nothing before'
   end
 
   # enqueues 4 workers on Resque
   it "runs integration test" do
+
+    puts "restart test redis"
+    Mobilize::Jobtracker.restart_test_redis
+
     puts "clear out test db"
-    Mongoid.session(:default).collections.each do |collection| 
-      unless collection.name =~ /^system\./
-        collection.drop
-      end
-    end
+    Mobilize::Jobtracker.drop_test_db
 
-    email = Mobilize::Gdriver.owner_email
+    email = Mobilize::Gdrive.owner_email
 
     #kill all workers
     Mobilize::Jobtracker.kill_workers
@@ -31,9 +30,9 @@ describe "Mobilize" do
     assert requestor.email == email
 
     puts "delete old books and datasets"
-    # delete any old specbooks from previous test runs
+    # delete any old jobspec from previous test runs
     jobspec_title = requestor.jobspec_title
-    books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
+    books = Mobilize::Gbook.find_all_by_title(jobspec_title)
     books.each{|book| book.delete}
 
     puts "enqueue jobtracker, wait 45s"
@@ -42,12 +41,12 @@ describe "Mobilize" do
     puts "jobtracker status: #{Mobilize::Jobtracker.status}" 
     puts "status:#{Mobilize::Jobtracker.status}" #!= 'stopped'
 
-    puts "requestor created specbook?"
-    books = Mobilize::Gbooker.find_all_by_title(jobspec_title)
+    puts "requestor created jobspec?"
+    books = Mobilize::Gbook.find_all_by_title(jobspec_title)
     assert books.length == 1
 
     puts "Jobtracker created jobspec with 'jobs' sheet?"
-    jobs_sheets = Mobilize::Gsheeter.find_all_by_name("#{jobspec_title}/Jobs",email)
+    jobs_sheets = Mobilize::Gsheet.find_all_by_name("#{jobspec_title}/Jobs",email)
     assert jobs_sheets.length == 1
 
     puts "add test_source data"
@@ -59,12 +58,12 @@ describe "Mobilize" do
     ]
 
     book = books.first
-    test_source_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_source",email)
+    test_source_sheet = Mobilize::Gsheet.find_or_create_by_name("#{jobspec_title}/test_source",email)
 
     test_source_tsv = test_source_rows.map{|r| r.join("\t")}.join("\n")
     test_source_sheet.write(test_source_tsv)
 
-    puts "add row to jobs sheet, wait 60s"
+    puts "add row to jobs sheet, wait 120s"
 
     jobs_sheet = jobs_sheets.first
 
@@ -74,9 +73,8 @@ describe "Mobilize" do
                        "status" => "",
                    "last_error" => "",
               "destination_url" => "",
-                 "read_handler" => "gsheeter",
-                "write_handler" => "gsheeter",
-                 "param_sheets" => "test_source",
+                        "tasks" => "gsheet.read, gsheet.write",
+                     "datasets" => "test_source",
                        "params" => "",
                   "destination" => "test_destination"},
                   #run after the first
@@ -86,39 +84,25 @@ describe "Mobilize" do
                        "status" => "",
                    "last_error" => "",
               "destination_url" => "",
-                 "read_handler" => "gsheeter",
-                "write_handler" => "gsheeter",
-                 "param_sheets" => "test_source",
+                        "tasks" => "gsheet.read, gsheet.write",
+                     "datasets" => "test_source",
                        "params" => "",
                   "destination" => "test_destination2"}
     ]
 
-    #update second row w details
-    test_job_rows.each_with_index do |r,r_i|
-      r.values.each_with_index do |v,v_i|
-      jobs_sheet[r_i+2,v_i+1] = v
-      end
-    end
-
-    jobs_sheet.save
+    jobs_sheet.add_or_update_rows(test_job_rows)
 
     puts "job row added, force enqueued requestor"
     requestor.enqueue!
-    sleep 60
+    sleep 120
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    test_destination_sheet = Mobilize::Gsheeter.find_or_create_by_name("#{jobspec_title}/test_destination",email)
+    test_destination_sheet = Mobilize::Gsheet.find_or_create_by_name("#{jobspec_title}/test_destination",email)
 
     assert test_destination_sheet.to_tsv == test_source_sheet.to_tsv
-  end
 
-  after do
-    processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
-    pids = processes.map { |process| process.split(" ")[0] }
-    puts "Killing test redis server..."
-    pids.each { |pid| Process.kill("TERM", pid.to_i) }
-    puts "removing redis db dump file"
-    sleep 5
-    `rm -f #{$dir}/dump.rdb #{$dir}/dump-cluster.rdb`
+    puts "stop test redis"
+    Mobilize::Jobtracker.stop_test_redis
   end
+
 end

data/test/test_helper.rb

@@ -8,16 +8,3 @@ $dir = File.dirname(File.expand_path(__FILE__))
 ENV['MOBILIZE_ENV'] = 'test'
 require 'mobilize-base'
 $TESTING = true
-
-#
-# make sure we can run redis
-#
-
-if !system("which redis-server")
-  puts '', "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
-  abort ''
-end
-
-#start test redis
-puts "Starting redis for testing at 127.0.0.1:#{Mobilize::Resque.config['redis_port']}..."
-`redis-server #{$dir}/redis-test.conf`

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-27 00:00:00.000000000 Z
+date: 2012-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -210,9 +210,12 @@ files:
 - lib/mobilize-base/extensions/object.rb
 - lib/mobilize-base/extensions/resque.rb
 - lib/mobilize-base/extensions/string.rb
-- lib/mobilize-base/handlers/emailer.rb
-- lib/mobilize-base/handlers/gdriver.rb
-- lib/mobilize-base/handlers/mongoer.rb
+- lib/mobilize-base/handlers/email.rb
+- lib/mobilize-base/handlers/gbook.rb
+- lib/mobilize-base/handlers/gdrive.rb
+- lib/mobilize-base/handlers/gfile.rb
+- lib/mobilize-base/handlers/gsheet.rb
+- lib/mobilize-base/handlers/mongodb.rb
 - lib/mobilize-base/jobtracker.rb
 - lib/mobilize-base/models/dataset.rb
 - lib/mobilize-base/models/job.rb