mobilize-base 1.1.06 → 1.1.07

data/README.md

@@ -25,6 +25,7 @@ Table Of Contents
   * [Default Folders and Files](#section_Install_Folders_and_Files)
 * [Configure](#section_Configure)
   * [Google Drive](#section_Configure_Google_Drive)
+  * [Google Sheets](#section_Configure_Google_Sheets)
   * [Jobtracker](#section_Configure_Jobtracker)
   * [Resque](#section_Configure_Resque)
   * [Resque-Web](#section_Configure_Resque-Web)
@@ -214,6 +215,47 @@ production:
       pw: worker002_google_drive_password
 ```
 
+<a name='section_Configure_Google_Sheets'></a>
+### Configure Google Sheets
+
+gsheet.yml needs:
+* max_cells, which is the number of cells a sheet is allowed to have
+  written to it at one time. Default is 400k cells, which is the max per
+  book. Google Drive will throw its own exception if
+  you try to write more than that.
+* Because Google Docs ties date formatting to the Locale for the
+  spreadsheet, there are 2 date format parameters:
+  * read_date_format, which is the format that should be read FROM google
+    sheets for date columns.
+  * sheet_date_format, which is the format that the google sheet is in.
+  * A date column is defined as one where the column header = "date" or "Date", or ends with "_date" or "Date".
+  * The defaults are set to US locale for sheet_date_format, because in 'Murica (US) we
+    use %m/%d/%Y for some reason, and to %Y-%m-%d format for
+    reading, which is more standard and sorts well as a string. If your
+    locale is NOT 'Murica you will want to change these.
+
+Sample gsheet.yml
+
+``` yml
+---
+development:
+  max_cells: 400000
+  read_date_format: "%Y-%m-%d"
+  sheet_date_format: "%m/%d/%Y"
+test:
+  max_cells: 400000
+  read_date_format: "%Y-%m-%d"
+  sheet_date_format: "%m/%d/%Y"
+staging:
+  max_cells: 400000
+  read_date_format: "%Y-%m-%d"
+  sheet_date_format: "%m/%d/%Y"
+production:
+  max_cells: 400000
+  read_date_format: "%Y-%m-%d"
+  sheet_date_format: "%m/%d/%Y"
+```
+
 <a name='section_Configure_Jobtracker'></a>
 ### Configure Jobtracker
 

data/lib/mobilize-base/extensions/google_drive/worksheet.rb

@@ -7,7 +7,10 @@ module GoogleDrive
       return nil unless header and header.first.to_s.length>0
       #look for blank cols to indicate end of row
       row_last_i = (header.index("") || header.length)-1
-      rows.map{|r| r[0..row_last_i]}.map{|r| r.join("\t")}.join("\n")
+      out_tsv = rows.map{|r| r[0..row_last_i]}.map{|r| r.join("\t")}.join("\n")
+      out_tsv.tsv_convert_dates(Mobilize::Gsheet.config['sheet_date_format'],
+                                Mobilize::Gsheet.config['read_date_format'])
+
     end
     def add_headers(headers)
       headers.each_with_index do |h,h_i|

data/lib/mobilize-base/extensions/string.rb

@@ -105,4 +105,32 @@ class String
     end
     return row_hash_arr
   end
+  def tsv_header_array(delim="\t")
+    str = self
+    #up to right before first line break, or whole thing
+    str[0..(str.index("\n") || 0)-1].split(delim)
+  end
+  def tsv_convert_dates(from_date_format,to_date_format)
+    str = self
+    hash_array = str.tsv_to_hash_array
+    headers = str.tsv_header_array.join("\t")
+    rows = hash_array.map do |h|
+      row = h.map do |k,v|
+              if k.to_s.downcase=="date" or
+                k.to_s.downcase.ends_with?("_date") or
+                k.to_s.ends_with?("Date")
+                begin
+                  Date.strptime(v,from_date_format).strftime(to_date_format)
+                rescue
+                  v
+                end
+              else
+                v
+              end
+      end
+      row.join("\t")
+    end
+    rows = [headers] + rows
+    rows.join("\n")
+  end
 end

data/lib/mobilize-base/models/runner.rb

@@ -45,7 +45,7 @@ module Mobilize
       r.update_attributes(:started_at=>Time.now.utc)
       #make sure any updates to activity are processed first
       #as in when someone runs a "once" job that has completed
-      r.update_gsheet(gdrive_slot)
+      r.update_gsheet(gdrive_slot) 
       #read the jobs in the gsheet and update models with news
       r.read_gsheet(gdrive_slot)
       #queue up the jobs that are due and active
@@ -153,6 +153,8 @@ module Mobilize
 
     def update_gsheet(gdrive_slot)
       r = self
+      #there's nothing to update if runner has never had a completed at
+      return false unless r.completed_at
       jobs_gsheet = r.gsheet(gdrive_slot)
       upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
       upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.1.06"
+    VERSION = "1.1.07"
   end
 end

data/lib/samples/gsheet.yml

@@ -1,7 +1,17 @@
 ---
 development:
-  max_cells: 400000
+  max_cells: 400000 #current google limit is 400k cells per book
+  read_date_format: "%Y-%m-%d" #format to record when reading sheets
+  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 test:
-  max_cells: 400000
+  max_cells: 400000 #current google limit is 400k cells per book
+  read_date_format: "%Y-%m-%d" #format to record when reading sheets
+  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
+staging:
+  max_cells: 400000 #current google limit is 400k cells per book
+  read_date_format: "%Y-%m-%d" #format to record when reading sheets
+  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
 production:
-  max_cells: 400000
+  max_cells: 400000 #current google limit is 400k cells per book
+  read_date_format: "%Y-%m-%d" #format to record when reading sheets
+  sheet_date_format: "%m/%d/%Y" #format to use to parse sheets

data/test/base1_stage1.yml

@@ -1,3 +1,3 @@
-- {test_header: s1, test_header2: s1, test_header3: s1}
-- {test_header: s2, test_header2: s2, test_header3: s2}
-- {test_header: s3, test_header2: s3, test_header3: s3}
+- {date: 2013-01-01, snake_date: 2013-02-01, camelDate: 2013-03-01}
+- {date: 2013-01-01, snake_date: 2013-02-02, camelDate: 2013-03-02}
+- {date: 2013-01-01, snake_date: 2013-02-03, camelDate: 2013-03-03}

data/test/mobilize-base_test.rb

@@ -31,36 +31,36 @@ describe "Mobilize" do
     puts "Jobtracker created runner with 'jobs' sheet?"
     r = u.runner
     jobs_sheet = r.gsheet(gdrive_slot)
-    tsv = jobs_sheet.read(Mobilize::Gdrive.owner_name)
-    assert tsv.length == 61 #headers only
+    tsv = jobs_sheet.read(user_name)
+    assert tsv.tsv_header_array.join.length == 53 #total header length
 
     puts "add base1_stage1 input sheet"
     test_source_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.in",gdrive_slot)
 
     test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_stage1.yml")*40
     test_source_tsv = test_source_ha.hash_array_to_tsv
-    test_source_sheet.write(test_source_tsv,Mobilize::Gdrive.owner_name)
+    test_source_sheet.write(test_source_tsv,user_name)
 
-    puts "add row to jobs sheet, wait 150s"
+    puts "add row to jobs sheet, wait 300s"
     test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
     jobs_sheet.add_or_update_rows(test_job_rows)
-    sleep 150
+    sleep 300
 
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
     test_target_sheet_1 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
     test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base2.out",gdrive_slot)
 
-    assert test_target_sheet_1.to_tsv == test_source_sheet.to_tsv
+    assert test_target_sheet_1.read(user_name) == test_source_sheet.read(user_name)
 
-    puts "delete both output sheets, set first job to active=true, wait 120s"
+    puts "delete both output sheets, set first job to active=true, wait 300s"
     [test_target_sheet_1,test_target_sheet_2].each{|s| s.delete}
 
     jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
-    sleep 120
+    sleep 300
 
-    test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
+    test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base2.out",gdrive_slot)
     puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
-    assert test_target_sheet_2.to_tsv == test_source_sheet.to_tsv
+    assert test_target_sheet_2.read(user_name)  == test_source_sheet.read(user_name)
 
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: 1.1.06
+  version: 1.1.07
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-04 00:00:00.000000000 Z
+date: 2013-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -239,7 +239,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2443012398121616343
+      hash: 1956526465748167234
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -248,7 +248,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2443012398121616343
+      hash: 1956526465748167234
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.24