mobilize-base 1.0.1 → 1.0.2

data/lib/mobilize-base/extensions/google_drive.rb

@@ -75,7 +75,7 @@ module GoogleDrive
     def add_worker_acl
       f = self
       return true if f.has_worker_acl?
-      Mobilize::Gdriver.worker_emails.each do |a| 
+      Mobilize::Gdrive.worker_emails.each do |a| 
         f.update_acl(a)
       end
     end
@@ -84,7 +84,7 @@ module GoogleDrive
       f = self
       #admin includes workers
       return true if f.has_admin_acl?
-      (Mobilize::Gdriver.admin_emails + Mobilize::Gdriver.worker_emails).each do |a| 
+      (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails).each do |a| 
         f.update_acl(a)
       end
     end
@@ -92,7 +92,7 @@ module GoogleDrive
     def has_admin_acl?
       f = self
       curr_emails = f.acls.map{|a| a.scope}.sort
-      admin_emails = Mobilize::Gdriver.admin_emails.sort
+      admin_emails = Mobilize::Gdrive.admin_emails.sort
       if (curr_emails & admin_emails) == admin_emails
         return true
       else
@@ -103,7 +103,7 @@ module GoogleDrive
     def has_worker_acl?
       f = self
       curr_emails = f.acls.map{|a| a.scope}.sort
-      worker_emails = Mobilize::Gdriver.worker_emails.sort
+      worker_emails = Mobilize::Gdrive.worker_emails.sort
       if (curr_emails & worker_emails) == worker_emails
         return true
       else
@@ -159,6 +159,32 @@ module GoogleDrive
       row_last_i = (header.index("") || header.length)-1
       rows.map{|r| r[0..row_last_i]}.map{|r| r.join("\t")}.join("\n")
     end
+    def add_headers(headers)
+      headers.each_with_index do |h,h_i|
+        self[1,h_i+1] = h
+      end
+    end
+
+    def add_or_update_rows(upd_rows)
+      sheet = self
+      curr_rows = sheet.to_tsv.tsv_to_hash_array
+      headers = curr_rows.first.keys
+      curr_rows = [] if curr_rows.length==1 and curr_rows.first['name'].nil?
+      curr_row_names = curr_rows.map{|r| r['name']}
+      upd_rows.each_with_index do |row,urow_i|
+        crow_i = curr_row_names.index(row['name'])
+        if crow_i.nil?
+          curr_row_names << row['name']
+          crow_i = curr_row_names.length-1
+        end
+        row.each do |col_n,col_v|
+          col_v_i = headers.index(col_n)
+          sheet[crow_i+2,col_v_i+1] = col_v
+        end
+      end
+      sheet.save
+    end
+
     def write(tsv,check=true,job_id=nil)
       sheet = self
       tsvrows = tsv.split("\n")
@@ -197,7 +223,6 @@ module GoogleDrive
           if job_id
             newstatus = "100 pct written at #{Time.now.utc}"
             Mobilize::Job.find(job_id).update_status(newstatus)
-            newstatus.oputs
           end
           break
         else

data/lib/mobilize-base/extensions/string.rb

@@ -13,6 +13,7 @@ class String
   end
   def bash(except=true)
     pid,stdin,stdout,stderr = Open4.popen4(self)
+    pid,stdin = [nil,nil]
     raise stderr.read if (stderr.read.length>0 and except==true)
     return stdout.read
   end
@@ -70,6 +71,7 @@ class String
     begin
       return JSON.parse(self)
     rescue => exc
+      exc = nil
       return {}
     end
   end
@@ -80,7 +82,7 @@ class String
     headers = rows.first.split("\t")
     if rows.length==1
       #return single member hash with all nil values
-      return [headers.map{|k| {k=>nil}}.inject{|k,h| h.merge(k)}]
+      return [headers.map{|k| {k=>nil}}.inject{|k,h| k.merge(h)}]
     end
     row_hash_arr =[]
     rows[1..-1].each do |row|

data/lib/mobilize-base/handlers/{emailer.rb → email.rb}

@@ -1,21 +1,21 @@
 module Mobilize
   require 'action_mailer'
-  class Emailer < ActionMailer::Base
+  class Email < ActionMailer::Base
     ActionMailer::Base.delivery_method = :smtp
     
     ActionMailer::Base.smtp_settings = {
     :address              => "smtp.gmail.com",
     :port                 => 587,
-    :domain               => 'ngmoco.com',
-    :user_name            => Gdriver.owner_email,
-    :password             => Gdriver.password(Gdriver.owner_email),
+    :domain               => Gdrive.domain,
+    :user_name            => Gdrive.owner_email,
+    :password             => Gdrive.password(Gdrive.owner_email),
     :authentication       => 'plain',
     :enable_starttls_auto => true  }
 
     def write(subj, 
                       bod="", 
                       recipient=Jobtracker.admin_emails.join(","))
-      mail(:from=>Gdriver.owner_email,
+      mail(:from=>Gdrive.owner_email,
            :to=>recipient, 
            :subject=>subj, 
            :body=>bod)

data/lib/mobilize-base/handlers/gbook.rb

@@ -0,0 +1,73 @@
+module Mobilize
+  module Gbook
+    def Gbook.find_all_by_title(title,email=nil)
+      Gdrive.books(email,{"title"=>title,"title-exact"=>"true"})
+    end
+    def Gbook.find_or_create_by_title(title,email)
+      books = Gdrive.books(email,{"title"=>title,"title-exact"=>"true"})
+      #there should only be one book with each title, otherwise we have fail
+      book = nil
+      if books.length>1
+        #some idiot process created a duplicate book.
+        #Fix by renaming all but one with dst entry's key
+        dst = Dataset.find_by_handler_and_name('gbook',title)
+        dkey = dst.url.split("key=").last
+        books.each do |b|
+          bkey = b.resource_id.split(":").last
+          if bkey == dkey
+            book = b
+          else
+            #delete the invalid book
+            b.delete
+            ("Deleted duplicate book #{title}").oputs
+          end
+        end
+      else
+        book = books.first
+      end
+      if book.nil?
+        #add book using owner email
+        #http
+        book = Gdrive.root.create_spreadsheet(title)
+        ("Created book #{title} at #{Time.now.utc.to_s}").oputs
+      end
+      #delete Sheet1 if there are other sheets
+      #http
+      if (sheets = book.worksheets).length>1
+        sheet1 = sheets.select{|s| s.title == "Sheet1"}.first
+        #http
+        sheet1.delete if sheet1
+      end
+      #always make sure books have admin acl
+      book.add_admin_acl
+      return book
+    end
+
+    def Gbook.find_or_create_by_dst_id(dst_id,email=nil)
+      #creates by title, updates acl, updates dataset with url
+      dst = Dataset.find(dst_id)
+      r = Requestor.find(dst.requestor_id)
+      book = nil
+      #http
+      book = Gdrive.root.spreadsheet_by_url(dst.url) if dst.url
+      #manually try 5 times to validate sheet since we can't just try again and again
+      5.times.each do
+        begin
+          book.resource_id
+          #if no error then break loop
+          break
+        rescue=>exc
+          if book.nil? or exc.to_s.index('Invalid document id')
+            book = Gbook.find_or_create_by_title(dst.name,email)
+            #if invalid doc then update url w new book and break loop
+            dst.update_attributes(:url=>book.human_url)
+            break
+          end
+        end
+      end
+      #add requestor write access
+      book.update_acl(r.email)
+      return book
+    end
+  end
+end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -0,0 +1,72 @@
+module Mobilize
+  module Gdrive
+    def Gdrive.config
+      Base.config('gdrive')[Base.env]
+    end
+
+    def Gdrive.domain
+      Gdrive.config['domain']
+    end
+
+    def Gdrive.owner_email
+      Gdrive.config['owner']['email']
+    end
+
+    def Gdrive.password(email)
+      if email == Gdrive.owner_email
+        Gdrive.config['owner']['pw']
+      else
+        worker = Gdrive.workers(email)
+        return worker['pw'] if worker
+      end
+    end
+
+    def Gdrive.admins
+      Gdrive.config['admins']
+    end
+
+    def Gdrive.workers(email=nil)
+      if email.nil?
+        Gdrive.config['workers']
+      else
+        Gdrive.workers.select{|w| w['email'] == email}.first
+      end
+    end
+
+    def Gdrive.worker_emails
+      Gdrive.workers.map{|w| w['email']}
+    end
+
+    def Gdrive.admin_emails
+      Gdrive.admins.map{|w| w['email']}
+    end
+
+    #email management - used to make sure not too many emails get used at the same time
+    def Gdrive.get_worker_email_by_mongo_id(mongo_id)
+      active_emails = Mobilize::Resque.jobs('working').map{|j| j['email'] if j['email']}.compact
+      Gdrive.workers.sort_by{rand}.each do |w|
+        if !(active_emails.include?(w['email']))
+          Mobilize::Resque.update_job_email(mongo_id,w['email'])
+          return w['email']
+        end
+      end
+      #return false if none are available
+      return false
+    end
+
+    def Gdrive.root(email=nil)
+      email ||= Gdrive.owner_email
+      pw = Gdrive.password(email)
+      GoogleDrive.login(email,pw)
+    end
+
+    def Gdrive.files(email=nil,params={})
+      root = Gdrive.root(email)
+      root.files(params)
+    end
+
+    def Gdrive.books(email=nil,params={})
+      Gdrive.files(email,params).select{|f| f.class==GoogleDrive::Spreadsheet}
+    end
+  end
+end

data/lib/mobilize-base/handlers/gfile.rb

@@ -0,0 +1,55 @@
+module Mobilize
+  class Gfile
+    def Gfile.find_by_title(title,email=nil)
+      Gdriver.files(email).select{|f| f.title==title}.first
+    end
+
+    def Gfile.find_by_dst_id(dst_id,email=nil)
+      dst = Dataset.find(dst_id)
+      Gfile.find_by_title(dst.path,email)
+    end
+
+    def Gfile.add_admin_acl_by_dst_id(dst_id)
+      #adds admins and workers as writers
+      file = Gfile.find_by_dst_id(dst_id)
+      file.add_admin_acl
+      return true
+    end
+
+    def Gfile.add_admin_acl_by_title(title)
+      file = Gfile.find_by_title(title)
+      file.add_admin_acl
+      return true
+    end
+
+    def Gfile.add_worker_acl_by_title(title)
+      file = Gfile.find_by_title(title)
+      file.add_worker_acl
+      return true
+    end
+
+    def Gfile.update_acl_by_dst_id(dst_id,email,role="writer",edit_email=nil)
+      dst = Dataset.find(dst_id)
+      Gfile.update_acl_by_title(dst.path,email,role,edit_email)
+    end
+
+    def Gfile.update_acl_by_title(title,email,role="writer",edit_email=nil)
+      file = Gfile.find_by_title(title,edit_email)
+      raise "File #{title} not found" unless file
+      file.update_acl(email,role)
+    end
+
+    def Gfile.read_by_name()
+      
+    end
+
+    def Gfile.read_by_url()
+      
+    end
+
+    def Gfile.read_by_job_id(job_id)
+      j = Job.find(job_id)
+    end
+
+  end
+end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -0,0 +1,123 @@
+module Mobilize
+  module Gsheet
+
+    def Gsheet.max_cells
+      400000
+    end
+
+    def Gsheet.read(name,email=nil)
+      sheet = Gsheet.find_or_create_by_name(name,email)
+      sheet.to_tsv
+    end
+
+    def Gsheet.write(name,tsv,email=nil)
+      sheet = Gsheet.find_or_create_by_name(name,email)
+      sheet.write(tsv)
+    end
+
+    def Gsheet.find_all_by_name(name,email)
+      book_title,sheet_title = name.split("/")
+      books = Gdrive.books(email,{"title"=>book_title,"title-exact"=>"true"})
+      sheets = books.map{|b| b.worksheets}.flatten.select{|w| w.title == sheet_title }
+      sheets
+    end
+
+    def Gsheet.find_or_create_by_name(name,email=nil,rows=100,cols=20)
+      book_title,sheet_title = name.split("/")
+      book = Gbook.find_or_create_by_title(book_title,email)
+      #http
+      sheet = book.worksheets.select{|w| w.title==sheet_title}.first
+      if sheet.nil?
+        #http
+        sheet = book.add_worksheet(sheet_title,rows,cols)
+        ("Created sheet #{name} at #{Time.now.utc.to_s}").oputs
+      end
+      return sheet
+    end
+
+    def Gsheet.find_or_create_by_dst_id(dst_id,email=nil)
+      #creates by title, updates acl, updates dataset with url
+      dst = Dataset.find(dst_id)
+      r = Requestor.find(dst.requestor_id)
+      name = dst.name
+      book_title,sheet_title = name.split("/")
+      #make sure book exists and is assigned to this user
+      r.find_or_create_gbook_by_title(book_title,email)
+      #add admin write access
+      sheet = Gsheet.find_or_create_by_name(name)
+      sheet_title = nil
+      return sheet
+    end
+
+    def Gsheet.read_by_dst_id(dst_id,email=nil)
+      dst = Dataset.find(dst_id)
+      name = dst.name
+      sheet = Gsheet.find_or_create_by_name(name,email)
+      output = sheet.to_tsv
+      return output
+    end
+
+    def Gsheet.read_by_job_id(job_id)
+      j = Job.find(job_id)
+      #reserve email account for read
+      email = Gdrive.get_worker_email_by_mongo_id(job_id)
+      return false unless email
+      #pull tsv from cache
+      j.dataset_array.first.read_cache
+    end
+
+    def Gsheet.write_by_dst_id(dst_id,tsv,email=nil)
+      dst = Dataset.find(dst_id)
+      #see if this is a specific cell
+      name = dst.name
+      return false unless email
+      #create temp tab, write data to it, checksum it against the source
+      temp_sheet = Gsheet.find_or_create_by_name("#{name}_temp")
+      temp_sheet.write(tsv)
+      #delete current sheet, replace it with temp one
+      sheet = Gsheet.find_or_create_by_name(dst.name)
+      title = sheet.title
+      #http
+      sheet.delete
+      begin
+        temp_sheet.rename(title)
+      rescue
+        #need this because sometimes it gets confused and tries to rename twice
+      end
+      "Write successful for #{write_name}".oputs
+      return true
+    end
+
+    def Gsheet.write_by_job_id(job_id)
+      j = Job.find(job_id)
+      r = j.requestor
+      dest_name = if j.destination.split("/").length==1
+                    "#{r.jobspec_title}#{"/"}#{j.destination}"
+                  else
+                    j.destination
+                  end
+      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheet',dest_name)
+      sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
+      email = Gdrive.get_worker_email_by_mongo_id(job_id)
+      #return false if there are no emails available
+      return false unless email
+      #create temp tab, write data to it, checksum it against the source
+      temp_sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheet',"#{dest_name}_temp")
+      temp_sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if temp_sheet_dst.requestor_id.nil?
+      temp_sheet = Gsheet.find_or_create_by_name(temp_sheet_dst.name,email)
+      #tsv is the prior task's output
+      tsv = j.task_output_dsts[j.task_idx-1].read
+      temp_sheet.write(tsv,true,job_id)
+      #delete current sheet, replace it with temp one
+      sheet = Gsheet.find_or_create_by_name(dest_name,email)
+      title = sheet.title
+      #http
+      sheet.delete
+      temp_sheet.title = title
+      temp_sheet.save
+      sheet_dst.update_attributes(:url=>temp_sheet.spreadsheet.human_url)
+      "Write successful for #{dest_name}".oputs
+      return true
+    end
+  end
+end

data/lib/mobilize-base/handlers/{mongoer.rb → mongodb.rb}

@@ -1,16 +1,16 @@
 module Mobilize
-  class Mongoer
+  class Mongodb
 
-    def Mongoer.grid
+    def Mongodb.grid
       session = ::Mongoid.configure.sessions['default']
       database_name = session['database']
       host,port = session['hosts'].first.split(":")
       return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
     end
 
-    def Mongoer.read_by_filename(filename)
+    def Mongodb.read_by_filename(filename)
       begin
-        zs=Mongoer.grid.open(filename,'r').read
+        zs=Mongodb.grid.open(filename,'r').read
         return ::Zlib::Inflate.inflate(zs)
       rescue
         "failed Mongo read for filename #{filename}".oputs
@@ -18,14 +18,14 @@ module Mobilize
       end
     end
 
-    def Mongoer.write_by_filename(filename,string)
+    def Mongodb.write_by_filename(filename,string)
       zs = ::Zlib::Deflate.deflate(string)
-      Mongoer.grid.open(filename,'w',:delete_old => true){|f| f.write(zs)}
+      Mongodb.grid.open(filename,'w',:delete_old => true){|f| f.write(zs)}
       return true
     end
 
-    def Mongoer.delete_by_filename(filename)
-      Mongoer.grid.delete(filename)
+    def Mongodb.delete_by_filename(filename)
+      Mongodb.grid.delete(filename)
       return true
     end
   end

data/lib/mobilize-base/jobtracker.rb

@@ -154,13 +154,6 @@ module Mobilize
       Resque.kill_workers(count)
     end
 
-    def Jobtracker.set_test_env
-      ENV['MOBILIZE_ENV']='test'
-      ::Resque.redis="localhost:9736"
-      mongoid_config_path = "#{Mobilize::Base.root}/config/mongoid.yml"
-      Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
-    end
-
     def Jobtracker.run_notifications
       if Jobtracker.notif_due?
         notifs = []
@@ -180,7 +173,7 @@ module Mobilize
           notifs << n
         end
         notifs.each do |notif|
-          Emailer.write(n['subj'],notif['body']).deliver
+          Email.write(n['subj'],notif['body']).deliver
           Jobtracker.last_notification=Time.now.utc.to_s
           "Sent notification at #{Jobtracker.last_notification}".oputs
         end
@@ -204,5 +197,41 @@ module Mobilize
       Jobtracker.update_status("told to stop")
       return true
     end
+
+
+    #test methods
+    def Jobtracker.restart_test_redis
+      Jobtracker.stop_test_redis
+      if !system("which redis-server")
+        raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
+      end
+      "redis-server #{Base.root}/test/redis-test.conf".bash
+    end
+
+    def Jobtracker.stop_test_redis
+      processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
+      pids = processes.map { |process| process.split(" ")[0] }
+      puts "Killing test redis server..."
+      pids.each { |pid| Process.kill("TERM", pid.to_i) }
+      puts "removing redis db dump file"
+      sleep 5
+      `rm -f #{Base.root}/test/dump.rdb #{Base.root}/test/dump-cluster.rdb`
+    end
+
+    def Jobtracker.set_test_env
+      ENV['MOBILIZE_ENV']='test'
+      ::Resque.redis="localhost:9736"
+      mongoid_config_path = "#{Base.root}/config/mongoid.yml"
+      Mongoid.load!(mongoid_config_path, Base.env)
+    end
+
+    def Jobtracker.drop_test_db
+      Jobtracker.set_test_env
+      Mongoid.session(:default).collections.each do |collection| 
+        unless collection.name =~ /^system\./
+          collection.drop
+        end
+      end
+    end
   end
 end

data/lib/mobilize-base/models/dataset.rb

@@ -22,7 +22,7 @@ module Mobilize
       if dst.last_cached_at and (dst.cache_expire_at.nil? or dst.cache_expire_at > Time.now.utc)
         return dst.read_cache
       else
-        return dst.handler.humanize.constantize.read_by_dst_id(dst.id.to_s)
+        return "Mobilize::#{dst.handler.humanize}".constantize.read_by_dst_id(dst.id.to_s)
       end
     end
 
@@ -52,18 +52,18 @@ module Mobilize
     def read_cache
       dst = self
       dst.update_attributes(:last_read_at=>Time.now.utc)
-      return Mongoer.read_by_filename(dst.id.to_s)
+      return Mongodb.read_by_filename(dst.id.to_s)
     end
 
     def write_cache(string,expire_at=nil)
       dst = self
-      Mongoer.write_by_filename(dst.id.to_s,string)
+      Mongodb.write_by_filename(dst.id.to_s,string)
       dst.update_attributes(:last_cached_at=>Time.now.utc,:cache_expire_at=>expire_at,:size=>string.length)
       return true
     end
 
     def delete_cache
-      return Mongoer.delete_by_filename(dst.id.to_s)
+      return Mongodb.delete_by_filename(dst.id.to_s)
     end
 
   end