mobilize-base 1.0.0

data/lib/mobilize-base/extensions/google_drive.rb

@@ -0,0 +1,296 @@
+module GoogleDrive
+  class ClientLoginFetcher
+    def request_raw(method, url, data, extra_header, auth)
+      #this is patched to handle server errors due to http chaos
+      uri = URI.parse(url)
+      response = nil
+      attempts = 0
+      sleep_time = nil
+      #try 5 times to make the call
+      while (response.nil? or response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}) and attempts < 5
+        #instantiate http object, set params
+        http = @proxy.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        #set 600  to allow for large downloads
+        http.read_timeout = 600
+        response = self.http_call(http, method, uri, data, extra_header, auth)
+        if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
+          if response.body.downcase.index("rate limit") or response.body.downcase.index("captcha")
+            if sleep_time
+              sleep_time = sleep_time * attempts
+            else
+              sleep_time = (rand*100).to_i
+            end
+          else
+            sleep_time = 10
+          end
+          attempts += 1
+          puts "Sleeping for #{sleep_time.to_s} due to #{response.body}"
+          sleep sleep_time
+        end
+      end
+      raise response.body if response.code.ie{|rcode| rcode.starts_with?("4") or rcode.starts_with?("5")}
+      return response
+    end
+    def http_call(http, method, uri, data, extra_header, auth)
+      http.read_timeout = 600
+      http.start() do
+        path = uri.path + (uri.query ? "?#{uri.query}" : "")
+        header = auth_header(auth).merge(extra_header)
+        if method == :delete || method == :get
+          http.__send__(method, path, header)
+        else
+          http.__send__(method, path, data, header)
+        end
+      end
+    end
+  end
+  class Acl
+    def update_role(entry, role) #:nodoc:
+      #do not send email notifications
+      url_suffix = "?send-notification-emails=false"
+      header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
+      doc = @session.request(
+          :put, %{#{entry.edit_url}#{url_suffix}}, :data => entry.to_xml(), :header => header, :auth => :writely)
+
+      entry.params = entry_to_params(doc.root)
+      return entry
+    end
+
+    def push(entry)
+      #do not send email notifications
+      entry = AclEntry.new(entry) if entry.is_a?(Hash)
+      url_suffix = "?send-notification-emails=false"
+      header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
+      doc = @session.request(:post, "#{@acls_feed_url}#{url_suffix}", :data => entry.to_xml(), :header => header, :auth => :writely)
+      entry.params = entry_to_params(doc.root)
+      @acls.push(entry)
+      return entry
+    end
+  end
+
+  class File
+
+    def add_worker_acl
+      f = self
+      return true if f.has_worker_acl?
+      Mobilize::Gdriver.worker_emails.each do |a| 
+        f.update_acl(a)
+      end
+    end
+
+    def add_admin_acl
+      f = self
+      #admin includes workers
+      return true if f.has_admin_acl?
+      (Mobilize::Gdriver.admin_emails + Mobilize::Gdriver.worker_emails).each do |a| 
+        f.update_acl(a)
+      end
+    end
+
+    def has_admin_acl?
+      f = self
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      admin_emails = Mobilize::Gdriver.admin_emails.sort
+      if (curr_emails & admin_emails) == admin_emails
+        return true
+      else
+        return false
+      end
+    end
+
+    def has_worker_acl?
+      f = self
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      worker_emails = Mobilize::Gdriver.worker_emails.sort
+      if (curr_emails & worker_emails) == worker_emails
+        return true
+      else
+        return false
+      end
+    end
+
+    def update_acl(email,role="writer")
+      f = self
+      #need these flags for HTTP retries
+      #create req_acl hash to add to current acl
+      if entry = f.acl_entry(email)
+        if [nil,"none","delete"].include?(role)
+          f.acl.delete(entry)
+        elsif entry.role != role and ['reader','writer','owner'].include?(role)
+          entry.role=role
+          f.acl.update_role(entry,entry.role)
+        elsif !['reader','writer','owner'].include?(role)
+          raise "Invalid role #{role}"
+        end
+      else
+        f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
+      end
+      return true
+    end
+    def acls
+      f = self
+      f.acl.to_enum.to_a
+    end
+    def acl_entry(email)
+      f = self
+      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
+    end
+
+    def entry_hash
+      f = self
+      dfe_xml = f.document_feed_entry.to_xml
+      begin
+        Hash.from_xml(dfe_xml)[:entry]
+      rescue
+        {}
+      end
+    end
+  end
+
+  class Worksheet
+    def to_tsv
+      sheet = self
+      rows = sheet.rows
+      header = rows.first
+      return nil unless header and header.first.to_s.length>0
+      #look for blank cols to indicate end of row
+      row_last_i = (header.index("") || header.length)-1
+      rows.map{|r| r[0..row_last_i]}.map{|r| r.join("\t")}.join("\n")
+    end
+    def write(tsv,check=true,job_id=nil)
+      sheet = self
+      tsvrows = tsv.split("\n")
+      #no rows, no write
+      return true if tsvrows.length==0
+      headers = tsvrows.first.split("\t")
+      batch_start = 0
+      batch_length = 80
+      rows_written = 0
+      curr_rows = sheet.num_rows
+      curr_cols = sheet.num_cols
+      pct_tens_complete =["0"]
+      curr_pct_complete = "00"
+      #make sure sheet is at least as big as necessary
+      if tsvrows.length != curr_rows
+        sheet.max_rows = tsvrows.length
+        sheet.save
+      end
+      if headers.length != curr_cols
+        sheet.max_cols = headers.length
+        sheet.save
+      end
+      #write to sheet in batches of batch_length
+      while batch_start < tsvrows.length
+        batch_end = batch_start + batch_length
+        tsvrows[batch_start..batch_end].each_with_index do |row,row_i|
+          rowcols = row.split("\t")
+          rowcols.each_with_index do |col_v,col_i|
+            sheet[row_i+batch_start+1,col_i+1]= %{#{col_v}}
+          end
+        end
+        sheet.save
+        batch_start += (batch_length + 1)
+        rows_written+=batch_length
+        if batch_start>tsvrows.length+1
+          if job_id
+            newstatus = "100 pct written at #{Time.now.utc}"
+            Mobilize::Job.find(job_id).update_status(newstatus)
+            newstatus.oputs
+          end
+          break
+        else
+          #pad digit
+          curr_pct_complete = "%02d" % ((rows_written+1).to_f*100/tsvrows.length.to_f).round(0)
+          if !pct_tens_complete.include?(curr_pct_complete.first)
+            if job_id
+              newstatus = "#{curr_pct_complete} pct written at #{Time.now.utc}"
+              Mobilize::Job.find(job_id).update_status(newstatus)
+              newstatus.oputs
+              pct_tens_complete << curr_pct_complete.first
+            end
+          end
+        end
+      end
+      #checksum it against the source
+      sheet.checksum(tsv) if check
+      true
+    end
+    def checksum(tsv)
+      sheet = self
+      sheet.reload
+      #loading remote data for checksum
+      rem_tsv = sheet.to_tsv
+      rem_table = rem_tsv.split("\n").map{|r| r.split("\t").map{|v| v.googlesafe}}
+      loc_table = tsv.split("\n").map{|r| r.split("\t").map{|v| v.googlesafe}}
+      re_col_vs = []
+      errcnt = 0
+      #checking cells
+      loc_table.each_with_index do |loc_row,row_i|
+        loc_row.each_with_index do |loc_v,col_i|
+          rem_row = rem_table[row_i]
+          if rem_row.nil?
+            errcnt+=1
+            "No Row #{row_i} for Write Dst".oputs
+            break
+          else
+            rem_v = rem_table[row_i][col_i]
+            if loc_v != rem_v
+              if ['true','false'].include?(loc_v.downcase)
+                #google sheet upcases true and false. ignore
+              elsif loc_v.starts_with?('rp') and rem_v.starts_with?('Rp')
+                # some other math bs
+                sheet[row_i+1,col_i+1] = %{'#{loc_v}}
+                re_col_vs << {'row_i'=>row_i+1,'col_i'=>col_i+1,'col_v'=>%{'#{loc_v}}}
+              elsif (loc_v.to_s.count('e')==1 or loc_v.to_s.count('e')==0) and
+                loc_v.to_s.sub('e','').to_i.to_s==loc_v.to_s.sub('e','').gsub(/\A0+/,"") #trim leading zeroes
+                #this is a string in scentific notation, or a numerical string with a leading zero
+                #GDocs handles this poorly, need to rewrite write_dst cells by hand with a leading apostrophe for text
+                sheet[row_i+1,col_i+1] = %{'#{loc_v}}
+                re_col_vs << {'row_i'=>row_i+1,'col_i'=>col_i+1,'col_v'=>%{'#{loc_v}}}
+              elsif loc_v.class==Float or loc_v.class==Fixnum
+                if (loc_v - rem_v.to_f).abs>0.0001
+                  "row #{row_i.to_s} col #{col_i.to_s}: Local=>#{loc_v.to_s} , Remote=>#{rem_v.to_s}".oputs
+                  errcnt+=1
+                end
+              elsif rem_v.class==Float or rem_v.class==Fixnum
+                if (rem_v - loc_v.to_f).abs>0.0001
+                  "row #{row_i.to_s} col #{col_i.to_s}: Local=>#{loc_v.to_s} , Remote=>#{rem_v.to_s}".oputs
+                  errcnt+=1
+                end
+              elsif loc_v.to_s.is_time?
+                rem_time = begin
+                             Time.parse(rem_v.to_s)
+                           rescue
+                             nil
+                           end
+                if rem_time.nil? || ((loc_v - rem_time).abs>1)
+                  "row #{row_i.to_s} col #{col_i.to_s}: Local=>#{loc_v} , Remote=>#{rem_v}".oputs
+                  errcnt+=1
+                end
+              else
+                #"loc_v=>#{loc_v.to_s},rem_v=>#{rem_v.to_s}".oputs
+                if loc_v.force_encoding("UTF-8") != rem_v.force_encoding("UTF-8")
+                #make sure it's not an ecoding issue
+                  "row #{row_i.to_s} col #{col_i.to_s}: Local=>#{loc_v} , Remote=>#{rem_v}".oputs
+                  errcnt+=1
+                end
+              end
+            end
+          end
+        end
+      end
+      if errcnt==0
+        if re_col_vs.length>0
+          sheet.save
+          "rewrote:#{re_col_vs.to_s}".oputs
+        else
+          true
+        end
+      else
+        raise "#{errcnt} errors found in checksum"
+      end
+    end
+  end
+end

data/lib/mobilize-base/extensions/hash.rb

@@ -0,0 +1,86 @@
+
+class Hash
+  #defined read and write methods for hashes to get around annoying mkpath issues
+  def read_path(array)
+    result=self
+    array.each_with_index do |p,i|
+      if result.class==Hash and result[p]
+        result=result[p]
+      elsif i==array.length-1
+        return result[p]
+      else
+        return nil
+      end
+    end
+    return result
+  end
+  #write array of arbitrary depth to the hash
+  def write_path(array,data)
+    return self[array.first.to_s]=data if array.length==1
+    array.each_with_index do |m,i|
+      arr_s = array[0..i]
+      if !self.read_path(arr_s)
+        eval(%{self["#{arr_s.join('"]["')}"]={}})
+      elsif self.read_path(arr_s).class != Hash
+        raise "There is data at #{arr_s.join("=>")}"
+      end
+    end
+    eval(%{self["#{array[0..-2].join('"]["')}"]}).store(array[-1].to_s,data)
+    return self
+  end
+  # BEGIN methods to create hash from XML
+  class << self
+    def from_xml(xml_io) 
+      begin
+        result = Nokogiri::XML(xml_io)
+        return { result.root.name.to_sym => xml_node_to_hash(result.root)} 
+      rescue Exception => e
+        # raise your custom exception here
+      end
+    end 
+    def xml_node_to_hash(node) 
+      # If we are at the root of the document, start the hash 
+      if node.element?
+        result_hash = {}
+        if node.attributes != {}
+          result_hash[:attributes] = {}
+          node.attributes.keys.each do |key|
+            result_hash[:attributes][node.attributes[key].name.to_sym] = prepare(node.attributes[key].value)
+          end
+        end
+        if node.children.size > 0
+          node.children.each do |child| 
+            result = xml_node_to_hash(child) 
+
+            if child.name == "text"
+              unless child.next_sibling || child.previous_sibling
+                return prepare(result)
+              end
+            elsif result_hash[child.name.to_sym]
+              if result_hash[child.name.to_sym].is_a?(Object::Array)
+                result_hash[child.name.to_sym] << prepare(result)
+              else
+                result_hash[child.name.to_sym] = [result_hash[child.name.to_sym]] << prepare(result)
+              end
+            else 
+              result_hash[child.name.to_sym] = prepare(result)
+            end
+          end
+
+          return result_hash 
+        else 
+          return result_hash
+        end 
+      else 
+        return prepare(node.content.to_s) 
+      end 
+    end          
+    def prepare(data)
+      (data.class == String && data.to_i.to_s == data) ? data.to_i : data
+    end
+  end
+  def to_struct(struct_name)
+    Struct.new(struct_name,*keys).new(*values)
+  end
+  # END methods to create hash from XML  
+end

data/lib/mobilize-base/extensions/object.rb

@@ -0,0 +1,6 @@
+class Object
+  #alias for instance_eval
+  def ie(&blk)
+    self.instance_eval(&blk)
+  end
+end

data/lib/mobilize-base/extensions/resque.rb

@@ -0,0 +1,180 @@
+module Mobilize
+  module Resque
+    def Resque.config
+      Base.config('resque')[Base.env]
+    end
+
+    def Resque.queue_name
+      Resque.config['queue_name']
+    end
+
+    def Resque.queues
+      Base.queues
+    end
+
+    def Resque.log_path
+      Base.log_path("mobilize-resque-#{::Mobilize::Base.env}")
+    end
+
+    def Resque.workers(state="all")
+      raise "invalid state #{state}" unless ['all','idle','working','timeout'].include?(state)
+      workers = ::Resque.workers.select{|w| w.queues.first == Resque.queue_name}
+      return workers if state == 'all'
+      working_workers = workers.select{|w| w.job['queue']== Resque.queue_name}
+      return working_workers if state == 'working'
+      idle_workers = workers.select{|w| w.job['queue'].nil?}
+      return idle_workers if state == 'idle'
+      timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
+      return timeout_workers if state == 'timeout'
+    end
+
+    def Resque.failures
+      ::Resque::Failure.all(0,0).select{|f| f['queue'] == Resque.queue_name}
+    end
+
+    #active state refers to jobs that are either queued or working
+    def Resque.jobs(state="active")
+      raise "invalid state #{state}" unless ['all','queued','working','active','timeout','failed'].include?(state)
+      working_jobs =  Resque.workers('working').map{|w| w.job['payload']}
+      return working_jobs if state == 'working'
+      queued_jobs = ::Resque.peek(Resque.queue_name,0,0).to_a
+      return queued_jobs if state == 'queued'
+      return working_jobs + queued_jobs if state == 'active'
+      failed_jobs = Resque.failures.map{|f| f['payload']}
+      return failed_jobs if state == 'failed'
+      timeout_jobs = Resque.workers("timeout").map{|w| w.job['payload']}
+      return timeout_jobs if state == 'timeout'
+      return working_jobs + queued_jobs + failed_jobs if state == 'all'
+    end
+
+    def Resque.active_mongo_ids
+      #first argument of the payload is the mongo id in Mongo unless the worker is Jobtracker
+      Resque.jobs('active').map{|j| j['args'].first unless j['class']=='Jobtracker'}.compact
+    end
+
+    #Resque workers and methods to find
+    def Resque.find_worker_by_mongo_id(mongo_id)
+      Resque.workers('working').select{|w| w.job['payload']['args'][0] == mongo_id}.first
+    end
+
+    def Resque.update_job_status(mongo_id,msg)
+      #this only works on working workers
+      worker = Resque.find_worker_by_mongo_id(mongo_id)
+      #also fire a log, cap logfiles at 10 MB
+      if !worker
+        Logger.new(Resque.log_path, 10, 1024*1000*10).info("[no worker for #{mongo_id}: #{Time.now.utc}] status: #{msg}")
+        return false
+      end
+      Resque.set_worker_args(worker,{"status"=>msg})
+      Logger.new(Resque.log_path, 10, 1024*1000*10).info("[#{worker} #{Time.now.utc}] status: #{msg}")
+      return true
+    end
+
+    def Resque.update_job_email(mongo_id,email)
+      #this only works on working workers
+      worker = Resque.find_worker_by_mongo_id(mongo_id)
+      return false unless worker
+      Resque.set_worker_args(worker,{"email"=>email})
+      #also fire a log, cap logfiles at 10 MB
+      Logger.new(Resque.log_path, 10, 1024*1000*10).info("[#{worker} #{Time.now.utc}] email: #{email}")
+    end
+
+    def Resque.get_worker_args(worker)
+      key = "worker:#{worker}"
+      json = ::Resque.redis.get(key)
+      if json
+        hash = JSON.parse(json)
+        hash['payload']['args'].last
+      end
+    end
+
+    #takes a worker and invokes redis to set the last value in its second arg array element
+    #by our convention this is a Hash
+    def Resque.set_worker_args(worker,args)
+      key = "worker:#{worker}"
+      json = ::Resque.redis.get(key)
+      if json
+        hash = JSON.parse(json)
+        payload_args = hash['payload']['args']
+        #jobmaster only gets one arg
+        if payload_args[1].nil?
+          payload_args[1] = args
+        else
+          payload_args[1] = {} unless payload_args[1].class==Hash
+          args.keys.each{|k,v| payload_args[1][k] = args[k]}
+        end
+        ::Resque.redis.set(key,hash.to_json)
+        return true
+      else
+        return false
+      end
+    end
+
+    def Resque.failure_report
+      fjobs = {}
+      excs = Hash.new(0)
+      Resque.failures.each do |f|
+        sname = f['payload']['class'] + ("=>" + f['payload']['args'].second['name'].to_s if f['payload']['args'].second).to_s
+        excs = f['error']
+        if fjobs[sname].nil?
+          fjobs[sname] = {excs => 1} 
+        elsif fjobs[sname][excs].nil?
+          fjobs[sname][excs] = 1
+        else
+          fjobs[sname][excs] += 1
+        end
+      end
+      return fjobs
+    end
+
+    def Resque.start_workers(count=1)
+      count.times do
+        "(cd #{Base.root};rake MOBILIZE_ENV=#{Base.env} mobilize:work) >> #{Resque.log_path} 2>&1 &".bash
+      end
+    end
+
+    def Resque.kill_idle_workers(count=nil)
+      idle_pids = Resque.workers('idle').select{|w| w.job=={}}.map{|w| w.to_s.split(":").second}
+      if count>idle_pids.length or count == 0
+        return false
+      elsif count
+        "kill #{idle_pids[0..count-1].join(" ")}".bash
+      else
+        "kill #{idle_pids.join(" ")}".bash
+      end
+      return true
+    end
+
+    def Resque.kill_workers(count=nil)
+      pids = Resque.workers.map{|w| w.to_s.split(":").second}
+      if count.to_i > pids.length or count == 0
+        return false
+      elsif count
+        "kill #{pids[0..count-1].join(" ")}".bash(false)
+      elsif pids.length>0
+        "kill #{pids.join(" ")}".bash(false)
+      else
+        return false
+      end
+      return true
+    end
+
+    def Resque.prep_workers(max_workers=Resque.config['max_workers'])
+      curr_workers = Resque.workers.length
+      if curr_workers > max_workers
+        #kill as many idlers as necessary
+        Resque.kill_idle_workers(curr_workers - max_workers)
+        #wait a few secs for these guys to die
+        sleep 10
+        curr_workers = Resque.workers.length
+        if curr_workers > max_workers
+          #kill working workers
+          Resque.kill_workers(curr_workers - max_workers)
+        end
+      else
+        Resque.start_workers(max_workers-curr_workers)
+      end
+      return true
+    end
+  end
+end

data/lib/mobilize-base/extensions/string.rb

@@ -0,0 +1,94 @@
+class String
+  def to_a
+    return [self]
+  end
+  def oputs
+    STDOUT.puts self
+  end
+  def eputs
+    STDERR.puts self
+  end
+  def opp
+    pp self
+  end
+  def bash(except=true)
+    pid,stdin,stdout,stderr = Open4.popen4(self)
+    raise stderr.read if (stderr.read.length>0 and except==true)
+    return stdout.read
+  end
+  def googlesafe
+    v=self
+    return "" if v.to_s==""
+    #normalize numbers by removing '$', '%', ',', ' '
+    vnorm = v.to_s.norm_num
+    vdigits = vnorm.split(".").last.length
+    if vnorm.to_f.to_s=="Infinity"
+      #do nothing
+    elsif ("%.#{vdigits}f" % vnorm.to_f.to_s)==vnorm
+      #round floats to 5 sig figs
+      v=vnorm.to_f.round(5)
+    elsif vnorm.to_i.to_s==vnorm
+      #make sure integers are cast as such
+      v=vnorm.to_i
+    elsif v.is_time?
+      begin
+        time_vs = v.split("/")
+        if time_vs.first.length<=2 and time_vs.second.length<=2
+          #date is of the form mm/dd/yyyy or mm/dd/yy
+          v=Time.parse("#{time_vs[2][0..3]}/#{time_vs[0]}/#{time_vs[1]}#{time_vs[2][4..-1]}")
+        else
+          v=Time.parse(v)
+        end
+      rescue
+        #do nothing
+      end
+    end
+    return v
+  end
+  def norm_num
+    return self.gsub(",","").gsub("$","").gsub("%","").gsub(" ","")
+  end
+  def is_float?
+    return self.norm_num.to_f.to_s == self.norm_num.to_s
+  end
+  def is_fixnum?
+    return self.norm_num.to_i.to_s == self.norm_num.to_s
+  end
+  def is_time?
+    if ((self.count("-")==2 or self.count("/")==2) and self.length>=8 and self.length<=20)
+      return true
+    end
+    split_str = self.split(" ")
+    if split_str.length==3 and
+      split_str.first.count("-")==2 and
+      split_str.last.first=="-" and
+      split_str.second.count(":")==2
+      return true
+    end
+  end
+  def json_to_hash
+    begin
+      return JSON.parse(self)
+    rescue => exc
+      return {}
+    end
+  end
+  def tsv_to_hash_array
+    rows = self.split("\n")
+    return [] if rows.first.nil?
+    return [{rows.first=>nil}] if (rows.length==2 and rows.second==nil)
+    headers = rows.first.split("\t")
+    if rows.length==1
+      #return single member hash with all nil values
+      return [headers.map{|k| {k=>nil}}.inject{|k,h| h.merge(k)}]
+    end
+    row_hash_arr =[]
+    rows[1..-1].each do |row|
+      cols = row.split("\t")
+      row_hash = {}
+      headers.each_with_index{|h,h_i| row_hash[h] = cols[h_i]}
+      row_hash_arr << row_hash
+    end
+    return row_hash_arr
+  end
+end

data/lib/mobilize-base/handlers/emailer.rb

@@ -0,0 +1,24 @@
+module Mobilize
+  require 'action_mailer'
+  class Emailer < ActionMailer::Base
+    ActionMailer::Base.delivery_method = :smtp
+    
+    ActionMailer::Base.smtp_settings = {
+    :address              => "smtp.gmail.com",
+    :port                 => 587,
+    :domain               => 'ngmoco.com',
+    :user_name            => Gdriver.owner_email,
+    :password             => Gdriver.password(Gdriver.owner_email),
+    :authentication       => 'plain',
+    :enable_starttls_auto => true  }
+
+    def write(subj, 
+                      bod="", 
+                      recipient=Jobtracker.admin_emails.join(","))
+      mail(:from=>Gdriver.owner_email,
+           :to=>recipient, 
+           :subject=>subj, 
+           :body=>bod)
+    end
+  end
+end