mobilize-base 1.0.2 → 1.0.4

data/Rakefile

@@ -17,8 +17,7 @@ require "bundler/gem_tasks"
 # Setup
 #
 $LOAD_PATH.unshift 'lib'
-require 'mobilize-base/tasks'
-
+require 'mobilize-base/rakes'
 
 #
 # Tests

data/lib/mobilize-base/extensions/google_drive/acl.rb

@@ -0,0 +1,25 @@
+module GoogleDrive
+  class Acl
+    def update_role(entry, role) #:nodoc:
+      #do not send email notifications
+      url_suffix = "?send-notification-emails=false"
+      header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
+      doc = @session.request(
+          :put, %{#{entry.edit_url}#{url_suffix}}, :data => entry.to_xml(), :header => header, :auth => :writely)
+
+      entry.params = entry_to_params(doc.root)
+      return entry
+    end
+
+    def push(entry)
+      #do not send email notifications
+      entry = AclEntry.new(entry) if entry.is_a?(Hash)
+      url_suffix = "?send-notification-emails=false"
+      header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
+      doc = @session.request(:post, "#{@acls_feed_url}#{url_suffix}", :data => entry.to_xml(), :header => header, :auth => :writely)
+      entry.params = entry_to_params(doc.root)
+      @acls.push(entry)
+      return entry
+    end
+  end
+end

data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb

@@ -0,0 +1,49 @@
+module GoogleDrive
+  class ClientLoginFetcher
+    def request_raw(method, url, data, extra_header, auth)
+      #this is patched to handle server errors due to http chaos
+      uri = URI.parse(url)
+      response = nil
+      attempts = 0
+      sleep_time = nil
+      #try 5 times to make the call
+      while (response.nil? or response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}) and attempts < 5
+        #instantiate http object, set params
+        http = @proxy.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        #set 600  to allow for large downloads
+        http.read_timeout = 600
+        response = self.http_call(http, method, uri, data, extra_header, auth)
+        if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
+          if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
+            if sleep_time
+              sleep_time = sleep_time * attempts
+            else
+              sleep_time = (rand*100).to_i
+            end
+          else
+            sleep_time = 10
+          end
+          attempts += 1
+          puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
+          sleep sleep_time
+        end
+      end
+      raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
+      return response
+    end
+    def http_call(http, method, uri, data, extra_header, auth)
+      http.read_timeout = 600
+      http.start() do
+        path = uri.path + (uri.query ? "?#{uri.query}" : "")
+        header = auth_header(auth).merge(extra_header)
+        if method == :delete || method == :get
+          http.__send__(method, path, header)
+        else
+          http.__send__(method, path, data, header)
+        end
+      end
+    end
+  end
+end

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -0,0 +1,80 @@
+module GoogleDrive
+  class File
+
+    def add_worker_acl
+      f = self
+      return true if f.has_worker_acl?
+      Mobilize::Gdrive.worker_emails.each do |a| 
+        f.update_acl(a)
+      end
+    end
+
+    def add_admin_acl
+      f = self
+      #admin includes workers
+      return true if f.has_admin_acl?
+      (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).each do |a| 
+        f.update_acl(a)
+      end
+    end
+
+    def has_admin_acl?
+      f = self
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      admin_emails = Mobilize::Gdrive.admin_emails.sort
+      if (curr_emails & admin_emails) == admin_emails
+        return true
+      else
+        return false
+      end
+    end
+
+    def has_worker_acl?
+      f = self
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      worker_emails = Mobilize::Gdrive.worker_emails.sort
+      if (curr_emails & worker_emails) == worker_emails
+        return true
+      else
+        return false
+      end
+    end
+
+    def update_acl(email,role="writer")
+      f = self
+      #need these flags for HTTP retries
+      #create req_acl hash to add to current acl
+      if entry = f.acl_entry(email)
+        if [nil,"none","delete"].include?(role)
+          f.acl.delete(entry)
+        elsif entry.role != role and ['reader','writer','owner'].include?(role)
+          entry.role=role
+          f.acl.update_role(entry,entry.role)
+        elsif !['reader','writer','owner'].include?(role)
+          raise "Invalid role #{role}"
+        end
+      else
+        f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
+      end
+      return true
+    end
+    def acls
+      f = self
+      f.acl.to_enum.to_a
+    end
+    def acl_entry(email)
+      f = self
+      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
+    end
+
+    def entry_hash
+      f = self
+      dfe_xml = f.document_feed_entry.to_xml
+      begin
+        Hash.from_xml(dfe_xml)[:entry]
+      rescue
+        {}
+      end
+    end
+  end
+end

data/lib/mobilize-base/extensions/{google_drive.rb → google_drive/worksheet.rb}

@@ -1,154 +1,4 @@
 module GoogleDrive
-  class ClientLoginFetcher
-    def request_raw(method, url, data, extra_header, auth)
-      #this is patched to handle server errors due to http chaos
-      uri = URI.parse(url)
-      response = nil
-      attempts = 0
-      sleep_time = nil
-      #try 5 times to make the call
-      while (response.nil? or response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}) and attempts < 5
-        #instantiate http object, set params
-        http = @proxy.new(uri.host, uri.port)
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        #set 600  to allow for large downloads
-        http.read_timeout = 600
-        response = self.http_call(http, method, uri, data, extra_header, auth)
-        if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
-          if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
-            if sleep_time
-              sleep_time = sleep_time * attempts
-            else
-              sleep_time = (rand*100).to_i
-            end
-          else
-            sleep_time = 10
-          end
-          attempts += 1
-          puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
-          sleep sleep_time
-        end
-      end
-      raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
-      return response
-    end
-    def http_call(http, method, uri, data, extra_header, auth)
-      http.read_timeout = 600
-      http.start() do
-        path = uri.path + (uri.query ? "?#{uri.query}" : "")
-        header = auth_header(auth).merge(extra_header)
-        if method == :delete || method == :get
-          http.__send__(method, path, header)
-        else
-          http.__send__(method, path, data, header)
-        end
-      end
-    end
-  end
-  class Acl
-    def update_role(entry, role) #:nodoc:
-      #do not send email notifications
-      url_suffix = "?send-notification-emails=false"
-      header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
-      doc = @session.request(
-          :put, %{#{entry.edit_url}#{url_suffix}}, :data => entry.to_xml(), :header => header, :auth => :writely)
-
-      entry.params = entry_to_params(doc.root)
-      return entry
-    end
-
-    def push(entry)
-      #do not send email notifications
-      entry = AclEntry.new(entry) if entry.is_a?(Hash)
-      url_suffix = "?send-notification-emails=false"
-      header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
-      doc = @session.request(:post, "#{@acls_feed_url}#{url_suffix}", :data => entry.to_xml(), :header => header, :auth => :writely)
-      entry.params = entry_to_params(doc.root)
-      @acls.push(entry)
-      return entry
-    end
-  end
-
-  class File
-
-    def add_worker_acl
-      f = self
-      return true if f.has_worker_acl?
-      Mobilize::Gdrive.worker_emails.each do |a| 
-        f.update_acl(a)
-      end
-    end
-
-    def add_admin_acl
-      f = self
-      #admin includes workers
-      return true if f.has_admin_acl?
-      (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).each do |a| 
-        f.update_acl(a)
-      end
-    end
-
-    def has_admin_acl?
-      f = self
-      curr_emails = f.acls.map{|a| a.scope}.sort
-      admin_emails = Mobilize::Gdrive.admin_emails.sort
-      if (curr_emails & admin_emails) == admin_emails
-        return true
-      else
-        return false
-      end
-    end
-
-    def has_worker_acl?
-      f = self
-      curr_emails = f.acls.map{|a| a.scope}.sort
-      worker_emails = Mobilize::Gdrive.worker_emails.sort
-      if (curr_emails & worker_emails) == worker_emails
-        return true
-      else
-        return false
-      end
-    end
-
-    def update_acl(email,role="writer")
-      f = self
-      #need these flags for HTTP retries
-      #create req_acl hash to add to current acl
-      if entry = f.acl_entry(email)
-        if [nil,"none","delete"].include?(role)
-          f.acl.delete(entry)
-        elsif entry.role != role and ['reader','writer','owner'].include?(role)
-          entry.role=role
-          f.acl.update_role(entry,entry.role)
-        elsif !['reader','writer','owner'].include?(role)
-          raise "Invalid role #{role}"
-        end
-      else
-        f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
-      end
-      return true
-    end
-    def acls
-      f = self
-      f.acl.to_enum.to_a
-    end
-    def acl_entry(email)
-      f = self
-      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
-    end
-
-    def entry_hash
-      f = self
-      dfe_xml = f.document_feed_entry.to_xml
-      begin
-        Hash.from_xml(dfe_xml)[:entry]
-      rescue
-        {}
-      end
-    end
-  end
-
   class Worksheet
     def to_tsv
       sheet = self
@@ -163,8 +13,17 @@ module GoogleDrive
       headers.each_with_index do |h,h_i|
         self[1,h_i+1] = h
       end
+      self.save
+    end
+    def delete_sheet1
+      sheet = self
+      #delete sheet1
+      sheet1 = sheet.spreadsheet.worksheet_by_title("Sheet1") || sheet.spreadsheet.worksheet_by_title("Sheet 1")
+      if sheet1
+        sheet1.delete
+        return true
+      end
     end
-
     def add_or_update_rows(upd_rows)
       sheet = self
       curr_rows = sheet.to_tsv.tsv_to_hash_array
@@ -185,7 +44,40 @@ module GoogleDrive
       sheet.save
     end
 
-    def write(tsv,check=true,job_id=nil)
+    def merge(merge_sheet)
+      #write the top left of sheet
+      #with the contents of merge_sheet
+      sheet = self
+      sheet.reload
+      merge_sheet.reload
+      curr_rows = sheet.num_rows
+      curr_cols = sheet.num_cols
+      merge_rows = merge_sheet.num_rows
+      merge_cols = merge_sheet.num_cols
+      #make sure sheet is at least as big as necessary
+      if merge_rows > curr_rows
+        sheet.max_rows = merge_rows
+        sheet.save
+      end
+      if merge_cols > curr_cols
+        sheet.max_cols = merge_cols
+        sheet.save
+      end
+      batch_start = 0
+      batch_length = 80
+      merge_sheet.rows.each_with_index do |row,row_i|
+        row.each_with_index do |val,col_i|
+          sheet[row_i+1,col_i+1] = val
+        end
+        if row_i > batch_start + batch_length
+          sheet.save
+          batch_start += (batch_length+1)
+        end
+      end
+      sheet.save
+    end
+
+    def write(tsv)
       sheet = self
       tsvrows = tsv.split("\n")
       #no rows, no write
@@ -196,8 +88,6 @@ module GoogleDrive
       rows_written = 0
       curr_rows = sheet.num_rows
       curr_cols = sheet.num_cols
-      pct_tens_complete =["0"]
-      curr_pct_complete = "00"
       #make sure sheet is at least as big as necessary
       if tsvrows.length != curr_rows
         sheet.max_rows = tsvrows.length
@@ -220,29 +110,12 @@ module GoogleDrive
         batch_start += (batch_length + 1)
         rows_written+=batch_length
         if batch_start>tsvrows.length+1
-          if job_id
-            newstatus = "100 pct written at #{Time.now.utc}"
-            Mobilize::Job.find(job_id).update_status(newstatus)
-          end
-          break
-        else
-          #pad digit
-          curr_pct_complete = "%02d" % ((rows_written+1).to_f*100/tsvrows.length.to_f).round(0)
-          if !pct_tens_complete.include?(curr_pct_complete.first)
-            if job_id
-              newstatus = "#{curr_pct_complete} pct written at #{Time.now.utc}"
-              Mobilize::Job.find(job_id).update_status(newstatus)
-              newstatus.oputs
-              pct_tens_complete << curr_pct_complete.first
-            end
-          end
+         break
         end
       end
-      #checksum it against the source
-      sheet.checksum(tsv) if check
       true
     end
-    def checksum(tsv)
+    def check_and_fix(tsv)
       sheet = self
       sheet.reload
       #loading remote data for checksum

data/lib/mobilize-base/extensions/resque.rb

@@ -1,7 +1,7 @@
 module Mobilize
   module Resque
     def Resque.config
-      Base.config('resque')[Base.env]
+      Base.config('resque')
     end
 
     def Resque.queue_name
@@ -47,36 +47,27 @@ module Mobilize
       return working_jobs + queued_jobs + failed_jobs if state == 'all'
     end
 
-    def Resque.active_mongo_ids
-      #first argument of the payload is the mongo id in Mongo unless the worker is Jobtracker
+    def Resque.active_paths
+      #first argument of the payload is the runner / task path unless the worker is Jobtracker
       Resque.jobs('active').map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
     end
 
     #Resque workers and methods to find
-    def Resque.find_worker_by_mongo_id(mongo_id)
-      Resque.workers('working').select{|w| w.job['payload']['args'][0] == mongo_id}.first
+    def Resque.find_worker_by_path(path)
+      Resque.workers('working').select{|w| w.job['payload'] and w.job['payload']['args'].first == path}.first
     end
 
-    def Resque.update_job_status(mongo_id,msg)
+    def Resque.set_worker_args_by_path(path,args)
       #this only works on working workers
-      worker = Resque.find_worker_by_mongo_id(mongo_id)
+      worker = Resque.find_worker_by_path(path)
+      args_string = args.map{|k,v| "#{k}: #{v}"}.join(";")
       #also fire a log, cap logfiles at 10 MB
-      if !worker
-        Logger.new(Resque.log_path, 10, 1024*1000*10).info("[no worker for #{mongo_id}: #{Time.now.utc}] status: #{msg}")
-        return false
-      end
-      Resque.set_worker_args(worker,{"status"=>msg})
-      Logger.new(Resque.log_path, 10, 1024*1000*10).info("[#{worker} #{Time.now.utc}] status: #{msg}")
-      return true
-    end
-
-    def Resque.update_job_email(mongo_id,email)
-      #this only works on working workers
-      worker = Resque.find_worker_by_mongo_id(mongo_id)
+      worker_string = worker ? worker.to_s : "no worker"
+      info_msg = "[#{worker_string} for #{path}: #{Time.now.utc}] #{args_string}"
+      Logger.new(Resque.log_path, 10, 1024*1000*10).info(info_msg)
       return false unless worker
-      Resque.set_worker_args(worker,{"email"=>email})
-      #also fire a log, cap logfiles at 10 MB
-      Logger.new(Resque.log_path, 10, 1024*1000*10).info("[#{worker} #{Time.now.utc}] email: #{email}")
+      Resque.set_worker_args(worker,args)
+      return true
     end
 
     def Resque.get_worker_args(worker)
@@ -114,7 +105,7 @@ module Mobilize
       fjobs = {}
       excs = Hash.new(0)
       Resque.failures.each do |f|
-        sname = f['payload']['class'] + ("=>" + f['payload']['args'].second['name'].to_s if f['payload']['args'].second).to_s
+        sname = f['payload']['args'].first
         excs = f['error']
         if fjobs[sname].nil?
           fjobs[sname] = {excs => 1} 
@@ -129,7 +120,10 @@ module Mobilize
 
     def Resque.start_workers(count=1)
       count.times do
-        "(cd #{Base.root};rake MOBILIZE_ENV=#{Base.env} mobilize:work) >> #{Resque.log_path} 2>&1 &".bash
+        dir_envs = "MOBILIZE_ENV=#{Base.env} " +
+                   "MOBILIZE_CONFIG_DIR=#{Base.config_dir} " +
+                   "MOBILIZE_LOG_DIR=#{Base.log_dir}"
+        "(cd #{Base.root};rake #{dir_envs} mobilize:work) >> #{Resque.log_path} 2>&1 &".bash
       end
     end