mobilize-base 1.351 → 1.361

data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb

@@ -8,7 +8,7 @@ module GoogleDrive
       attempts = 0
       sleep_time = nil
       #try 5 times to make the call
-      while (response.nil? or response.code.starts_with?("5")) and attempts < 20
+      while (response.nil? or response.code.starts_with?("5")) and attempts < Mobilize::Gdrive.max_api_retries
         #instantiate http object, set params
         http = @proxy.new(uri.host, uri.port)
         http.use_ssl = true
@@ -16,6 +16,7 @@ module GoogleDrive
         #set 600  to allow for large downloads
         http.read_timeout = 600
         response = begin
+                     puts "#{Time.now.utc} Gdrive API #{method.to_s}: #{uri.to_s} #{extra_header.to_s}"
                      clf.http_call(http, method, uri, data, extra_header, auth)
                    rescue
                      #timeouts etc.

data/lib/mobilize-base/extensions/string.rb

@@ -43,7 +43,7 @@ class String
     return v if v.to_s.strip==""
     #normalize numbers by removing '$', '%', ',', ' '
     vnorm = v.to_s.norm_num
-    vdigits = vnorm.split(".").last.length
+    vdigits = vnorm.split(".").last.to_s.length
     if vnorm.to_f.to_s=="Infinity"
       #do nothing
     elsif ("%.#{vdigits}f" % vnorm.to_f.to_s)==vnorm

data/lib/mobilize-base/extensions/time.rb

@@ -0,0 +1,20 @@
+class Time
+  def Time.at_marks_ago(number=1, unit='day', mark='0000')
+    curr_time = Time.now.utc
+    #strip out non-numerical characters from mark, add colon
+    mark = mark.gsub(/[^0-9]/i,"").rjust(4,'0').ie{|m| [m[0..1],":",m[-2..-1]].join}
+    #if user passes in 0 for the number, make it 1
+    number = (number.to_i <= 0 ? 1 : number.to_i)
+    if unit == 'day'
+      curr_mark_time = Time.parse(curr_time.strftime("%Y-%m-%d #{mark} UTC"))
+    elsif unit == 'hour'
+      if curr_time.strftime("%H%M").to_i > mark.to_i
+        curr_mark_time = Time.parse(curr_time.strftime("%Y-%m-%d %H:#{mark[-2..-1]} UTC"))
+      end
+    end
+    #last mark time is 
+    mark_ago_increment = (curr_time > curr_mark_time ? (number-1).send(unit) : number.send(unit))
+    last_mark_time = curr_mark_time - mark_ago_increment
+    return last_mark_time
+  end
+end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -25,6 +25,18 @@ module Mobilize
       end
     end
 
+    def Gdrive.max_api_retries
+      Gdrive.config['max_api_retries']
+    end
+
+    def Gdrive.max_file_write_retries
+      Gdrive.config['max_file_write_retries']
+    end
+
+    def Gdrive.file_write_retry_delay
+      Gdrive.config['file_write_retry_delay']
+    end
+
     def Gdrive.admins
       Gdrive.config['admins']
     end

data/lib/mobilize-base/handlers/gfile.rb

@@ -1,9 +1,23 @@
 module Mobilize
   module Gfile
+
+    def Gfile.config
+      Base.config('gfile')
+    end
+
+    def Gfile.max_length
+      Gfile.config['max_length']
+    end
+   
     def Gfile.path_to_dst(path,stage_path,gdrive_slot)
+      s = Stage.where(:path=>stage_path).first
+      params = s.params
+      target_path = params['target']
+      #if this is the target, it doesn't have to exist already
+      is_target = true if path == target_path
       #don't need the ://
       path = path.split("://").last if path.index("://")
-      if Gfile.find_by_path(path)
+      if is_target or Gfile.find_by_path(path)
         handler = "gfile"
         Dataset.find_or_create_by_url("#{handler}://#{path}")
       else
@@ -30,12 +44,6 @@ module Mobilize
                                     :content_type=>"test/plain",
                                     :convert=>false)
       file.add_admin_acl
-      #make sure user is owner or can edit
-      u = User.where(:name=>user_name).first
-      entry = file.acl_entry(u.email)
-      unless entry and ['writer','owner'].include?(entry.role)
-        file.update_acl(u.email)
-      end
       #update http url for file
       dst = Dataset.find_by_handler_and_path("gfile",dst_path)
       api_url = file.human_url.split("&").first
@@ -93,5 +101,49 @@ module Mobilize
       end
       return file
     end
+
+    def Gfile.write_by_stage_path(stage_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
+      #return blank response if there are no slots available
+      return nil unless gdrive_slot
+      s = Stage.where(:path=>stage_path).first
+      u = s.job.runner.user
+      retries = 0
+      stdout,stderr = []
+      while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
+        begin
+          #get tsv to write from stage
+          source = s.sources(gdrive_slot).first
+          raise "Need source for gfile write" unless source
+          tsv = source.read(u.name,gdrive_slot)
+          raise "No data source found for #{source.url}" unless tsv.to_s.length>0
+          if tsv.length > Gfile.max_length
+            raise "Too much data; you have #{tsv.length.to_s}, max is #{Gfile.max_length.to_s}"
+          end
+          stdout = if tsv.length == 0
+               #soft error; no data to write. Stage will complete.
+               "Write skipped for #{s.target.url}"
+             else
+               Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot)
+               #update status
+               "Write successful for #{s.target.url}"
+             end
+          Gdrive.unslot_worker_by_path(stage_path)
+          stderr = nil
+          s.update_status(stdout)
+          signal = 0
+        rescue => exc
+          if retries < Gdrive.max_file_write_retries
+            retries +=1
+            sleep Gdrive.file_write_retry_delay
+          else
+            stdout = nil
+            stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
+            signal = 500
+          end
+        end
+      end
+      return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
+    end
   end
 end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -127,33 +127,43 @@ module Mobilize
       s = Stage.where(:path=>stage_path).first
       u = s.job.runner.user
       crop = s.params['crop'] || true
-      begin
-        #get tsv to write from stage
-        source = s.sources(gdrive_slot).first
-        raise "Need source for gsheet write" unless source
-        tsv = source.read(u.name,gdrive_slot)
-        raise "No data source found for #{source.url}" unless tsv
-        tsv_row_count = tsv.to_s.split("\n").length
-        tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
-        tsv_cell_count = tsv_row_count * tsv_col_count
-        stdout = if tsv_row_count == 0
-                   #soft error; no data to write. Stage will complete.
-                   "Write skipped for #{s.target.url}"
-                 elsif tsv_cell_count > Gsheet.max_cells
-                   raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
-                 else
-                   Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
-                   #update status
-                   "Write successful for #{s.target.url}"
-                 end
-        Gdrive.unslot_worker_by_path(stage_path)
-        stderr = nil
-        s.update_status(stdout)
-        signal = 0
-      rescue => exc
-        stdout = nil
-        stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
-        signal = 500
+      retries = 0
+      stdout,stderr = []
+      while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
+        begin
+          #get tsv to write from stage
+          source = s.sources(gdrive_slot).first
+          raise "Need source for gsheet write" unless source
+          tsv = source.read(u.name,gdrive_slot)
+          raise "No data source found for #{source.url}" unless tsv
+          tsv_row_count = tsv.to_s.split("\n").length
+          tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
+          tsv_cell_count = tsv_row_count * tsv_col_count
+          if tsv_cell_count > Gsheet.max_cells
+            raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
+          end
+          stdout = if tsv_row_count == 0
+                     #soft error; no data to write. Stage will complete.
+                     "Write skipped for #{s.target.url}"
+                   else
+                     Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
+                     #update status
+                     "Write successful for #{s.target.url}"
+                   end
+          Gdrive.unslot_worker_by_path(stage_path)
+          stderr = nil
+          s.update_status(stdout)
+          signal = 0
+        rescue => exc
+          if retries < Gdrive.max_file_write_retries
+            retries +=1
+            sleep Gdrive.file_write_retry_delay
+          else
+            stdout = nil
+            stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
+            signal = 500
+          end
+        end
       end
       return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
     end

data/lib/mobilize-base/handlers/resque.rb

@@ -148,10 +148,7 @@ module Mobilize
 
     def Resque.start_workers(count=1)
       count.times do
-        dir_envs = "MOBILIZE_ENV=#{Base.env} " +
-                   "MOBILIZE_CONFIG_DIR=#{Base.config_dir} " +
-                   "MOBILIZE_LOG_DIR=#{Base.log_dir}"
-        "(cd #{Base.root};rake #{dir_envs} mobilize_base:work) >> #{Resque.log_path} 2>&1 &".bash
+        "(cd #{Base.root};rake mobilize:work[#{Base.env}]) >> #{Resque.log_path} 2>&1 &".bash
       end
     end
 

data/lib/mobilize-base/models/job.rb

@@ -15,12 +15,32 @@ module Mobilize
       return j
     end
 
+    def parent
+      j = self
+      u = j.runner.user
+      if j.trigger.strip[0..4].downcase == "after"
+        parent_name = j.trigger[5..-1].to_s.strip
+        parent_j = u.jobs.select{|job| job.name == parent_name}.first
+        return parent_j
+      else
+        return nil
+      end
+    end
+
+    def children
+      j = self
+      u = j.runner.user
+      u.jobs.select do |job|
+        parent_name = job.trigger[5..-1].to_s.strip
+        job.trigger.strip[0..4].downcase == "after" and
+          parent_name == j.name
+      end
+    end
+
     #takes a hash of job parameters (name, active, trigger, stages)
     #and creates/updates a job with it
-    def Job.update_by_user_name_and_hash(user_name,hash)
-      u = User.where(name: user_name).first
-      r = u.runner
-      j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
+    def update_from_hash(hash)
+      j = self
       #update top line params
       j.update_attributes(:active => hash['active'],
                           :trigger => hash['trigger'])
@@ -54,59 +74,82 @@ module Mobilize
 
     def is_due?
       j = self
-      return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
-      last_run = j.completed_at
-      #check trigger
-      trigger = j.trigger
-      return true if trigger == 'once'
-      #strip the "every" from the front if present
-      trigger = trigger.gsub("every","").gsub("."," ").strip
-      value,unit,operator,job_utctime = trigger.split(" ")
-      curr_utctime = Time.now.utc
-      curr_utcdate = curr_utctime.to_date.strftime("%Y-%m-%d")
-      if job_utctime
-        job_utctime = job_utctime.split(" ").first
-        job_utctime = Time.parse([curr_utcdate,job_utctime,"UTC"].join(" "))
+      #working or inactive jobs are not due
+      if j.is_working? or j.active == false
+        return false
+      end
+
+      #if job contains handlers not loaded by jobtracker, not due
+      loaded_handlers = Jobtracker.config['extensions'].map{|m| m.split("-").last}
+      job_handlers = j.stages.map{|s| s.handler}.uniq
+      #base handlers are the ones in mobilize-base/handlers
+      if (job_handlers - loaded_handlers - Base.handlers).length>0
+        return false
+      end
+
+      #once
+      if j.trigger.strip.downcase=='once'
+        #active and once means due
+        return true
       end
-      #after is the only operator
-      raise "Unknown #{operator.to_s} operator" if operator and operator != "after"
-      if ["hour","hours"].include?(unit)
-        #if it's later than the last run + hour tolerance, is due
-        if last_run.nil? or curr_utctime > (last_run + value.to_i.hour)
+
+      #depedencies
+      if j.parent
+        #if parent is not working and completed more recently than self, is due
+        if !j.parent.is_working? and
+          j.parent.completed_at and (j.completed_at.nil? or j.parent.completed_at > j.completed_at)
           return true
+        else
+          return false
         end
-      elsif ["day","days"].include?(unit)
-        if last_run.nil? or curr_utctime.to_date >= (last_run.to_date + value.to_i.day)
-          if operator and job_utctime
-            if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
-              return true
-            end
-          elsif operator || job_utctime
-            raise "Please specify both an operator and a time in UTC, or neither"
-          else
+      end
+
+      #time based
+      last_comp_time = j.completed_at
+      #check trigger; strip the "every" from the front if present, change dot to space
+      trigger = j.trigger.strip.gsub("every","").gsub("."," ").strip
+      number, unit, operator, mark = trigger.split(" ").map{|t_node| t_node.downcase}
+      #operator is not used
+      operator = nil
+      #get time for time-based evaluations
+      curr_time = Time.now.utc
+      if ["hour","hours","day","days"].include?(unit)
+        if mark
+          last_mark_time = Time.at_marks_ago(number,unit,mark)
+          if last_comp_time.nil? or last_comp_time < last_mark_time
             return true
+          else
+            return false
           end
+        elsif last_comp_time.nil? or last_comp_time < (curr_time - number.to_i.send(unit))
+          return true
+        else
+          return false
         end
       elsif unit == "day_of_week"
-        if curr_utctime.wday==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
-          if operator and job_utctime
-            if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
+        if curr_time.wday==number and (last_comp_time.nil? or last_comp_time.to_date != curr_time.to_date)
+          if mark
+            #check if it already ran today
+            last_mark_time = Time.at_marks_ago(1,"day",mark)
+            if last_comp_time < last_mark_time
               return true
+            else
+              return false
             end
-          elsif operator || job_utctime
-            raise "Please specify both an operator and a time in UTC, or neither"
           else
             return true
           end
         end
       elsif unit == "day_of_month"
-        if curr_utctime.day==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
-          if operator and job_utctime
-            if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
+        if curr_time.day==number and (last_comp_time.nil? or last_comp_time.to_date != curr_time.to_date)
+          if mark
+            #check if it already ran today
+            last_mark_time = Time.at_marks_ago(1,"day",mark)
+            if last_comp_time < last_mark_time
               return true
+            else
+              return false
             end
-          elsif operator || job_utctime
-            raise "Please specify both an operator and a time in UTC, or neither"
           else
             return true
           end

data/lib/mobilize-base/models/runner.rb

@@ -50,8 +50,7 @@ module Mobilize
             s.enqueue!
           end
         rescue ScriptError, StandardError => exc
-          r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
-          j.update_attributes(:active=>false)
+          r.update_status("Failed to enqueue #{j.path}")
         end
       end
       r.update_gsheet(gdrive_slot)
@@ -74,7 +73,9 @@ module Mobilize
       gsheet_hashes.each do |gsheet_hash|
         #skip non-jobs or jobs without required values
         next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
-        j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
+        #find job w this name, or make one
+        j = r.jobs.select{|rj| rj.name == gsheet_hash['name']}.first || Job.find_or_create_by_path("#{r.path}/#{gsheet_hash['name']}")
+        j.update_from_hash(gsheet_hash)
         r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
         #add this job to list of read ones
         done_jobs << j
@@ -95,7 +96,15 @@ module Mobilize
       return false unless r.completed_at
       jobs_gsheet = r.gsheet(gdrive_slot)
       upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
-      upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
+      upd_rows = upd_rows = upd_jobs.map do |j|
+        uj = {'name'=>j.name, 'status'=>j.status}
+        #jobs can only be turned off
+        #automatically, not back on
+        if j.active==false
+          uj['active'] = false
+        end
+        uj
+      end
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
       return true

data/lib/mobilize-base/models/stage.rb

@@ -82,8 +82,6 @@ module Mobilize
 
     def complete(response)
       s = self
-      s.update_attributes(:completed_at=>Time.now.utc,:response=>response)
-      s.update_status("Completed at #{Time.now.utc.to_s}")
       j = s.job
       if s.idx == j.stages.length
         #check for any dependent jobs, if there are, enqueue them
@@ -110,6 +108,10 @@ module Mobilize
         s.next.update_attributes(:retries_done=>0)
         s.next.enqueue!
       end
+      #complete after dependencies are processed
+      #to make sure it doesn't enqueue due to runner check
+      s.update_attributes(:completed_at=>Time.now.utc,:response=>response)
+      s.update_status("Completed at #{Time.now.utc.to_s}")
       true
     end
 

data/lib/mobilize-base/tasks.rb

@@ -1,6 +1,7 @@
-namespace :mobilize_base do
+namespace :mobilize do
   desc "Start a Resque worker"
-  task :work do
+  task :work, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Base.config('jobtracker')['extensions'].each do |e|
       begin
@@ -20,42 +21,77 @@ namespace :mobilize_base do
     worker.work(ENV['INTERVAL'] || 5) # interval, will block
   end
   desc "Kill all Resque workers"
-  task :kill_workers do
+  task :kill_workers, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_workers
   end
   desc "Kill idle workers not in sync with repo"
-  task :kill_idle_and_stale_workers do
+  task :kill_idle_and_stale_workers, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_idle_and_stale_workers
   end
-    desc "Kill idle workers"
-    task :kill_idle_workers do
+  desc "Kill idle workers"
+  task :kill_idle_workers, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.kill_idle_workers
   end
   desc "Make sure there are the correct # of workers, kill if too many"
-  task :prep_workers do
+  task :prep_workers, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.prep_workers
   end
+  desc "Restart Resque workers"
+  task :restart_workers, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
+    require 'mobilize-base'
+    Mobilize::Jobtracker.kill_workers
+    sleep 5
+    Mobilize::Jobtracker.prep_workers
+  end
   desc "Stop Jobtracker"
-  task :stop_jobtracker do
+  task :stop_jobtracker, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.stop!
   end
   desc "Start Jobtracker"
-  task :start_jobtracker do
+  task :start_jobtracker, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.start
   end
   desc "Restart Jobtracker"
-  task :restart_jobtracker do
+  task :restart_jobtracker, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     Mobilize::Jobtracker.restart!
   end
+  desc "Add a user"
+  task :add_user, :name, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
+    require 'mobilize-base'
+    Mobilize::User.find_or_create_by_name(args.name)
+  end
+  desc "Enqueue a user's runner"
+  task :enqueue_user, :name, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
+    require 'mobilize-base'
+    Mobilize::User.where(name: args.name).first.runner.enqueue!
+  end
+  desc "Enqueue a stage"
+  task :enqueue_stage, :path, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
+    require 'mobilize-base'
+    user,job,stage = args.path.split("/")
+    Mobilize::Stage.where(path: "Runner_#{user}/jobs/#{job}/#{stage}").first.en
+  end
   desc "kill all old resque web processes, start new one with resque_web.rb extension file"
-  task :resque_web do
+  task :resque_web, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     port = Mobilize::Base.config('resque')['web_port']
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
@@ -83,14 +119,15 @@ namespace :mobilize_base do
     `#{command}`
   end
   desc "create indexes for all base models in mongodb"
-  task :create_indexes do
+  task :create_indexes, :env do |t,args|
+    ENV['MOBILIZE_ENV']=args.env
     require 'mobilize-base'
     ["Dataset","Job","Runner","Task","User"].each do |m|
       "Mobilize::#{m}".constantize.create_indexes
     end
   end
   desc "Set up config and log folders and files, populate from samples"
-  task :setup do
+  task :setup_base do
     config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
     log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
     sample_dir = File.dirname(__FILE__) + '/../samples/'

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.351"
+    VERSION = "1.361"
   end
 end

data/lib/mobilize-base.rb

@@ -3,6 +3,7 @@ require "mobilize-base/extensions/array"
 require "mobilize-base/extensions/hash"
 require "mobilize-base/extensions/object"
 require "mobilize-base/extensions/string"
+require "mobilize-base/extensions/time"
 require "mobilize-base/extensions/yaml"
 #this is the base of the mobilize object, any methods that should be
 #made available application-wide go over here
@@ -16,6 +17,9 @@ module Mobilize
         ENV['PWD']
       end
     end
+    def Base.home_dir
+      File.expand_path('..',File.dirname(__FILE__))
+    end
     def Base.config_dir
       ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/"
     end
@@ -56,6 +60,9 @@ module Mobilize
         raise "Could not find #{log_dir} folder for logs"
       end
     end
+    def Base.handlers
+      Dir.entries(File.dirname(__FILE__) + "/mobilize-base/handlers").select{|e| e.ends_with?(".rb")}.map{|e| e.split(".").first}
+    end
   end
 end
 mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"

data/lib/samples/gdrive.yml

@@ -1,6 +1,9 @@
 ---
 development:
   domain: host.com
+  max_api_retries: 10
+  file_write_retries: 5
+  file_write_retry_delay: 30
   owner:
     name: owner_development
     pw: google_drive_password
@@ -15,6 +18,9 @@ development:
       pw: worker002_google_drive_password
 test:
   domain: host.com
+  max_api_retries: 10
+  file_write_retries: 5
+  file_write_retry_delay: 30
   owner:
     name: owner_test
     pw: google_drive_password
@@ -29,6 +35,9 @@ test:
       pw: worker002_google_drive_password
 production:
   domain: host.com
+  max_api_retries: 10
+  file_write_retries: 5
+  file_write_retry_delay: 30
   owner:
     name: owner_production
     pw: google_drive_password

data/lib/samples/gfile.yml

@@ -0,0 +1,9 @@
+---
+development:
+  max_length: 100000000 #about 100MB
+test:
+  max_length: 100000000 #about 100MB
+staging:
+  max_length: 100000000 #about 100MB
+production:
+  max_length: 100000000 #about 100MB

data/test/fixtures/base1_stage1.in.yml

@@ -0,0 +1,10 @@
+---
+- date: 2013-01-01
+  snake_date: 2013-02-01
+  camelDate: 2013-03-01
+- date: 2013-01-01
+  snake_date: 2013-02-02
+  camelDate: 2013-03-02
+- date: 2013-01-01
+  snake_date: 2013-02-03
+  camelDate: 2013-03-03