mobilize-base 1.0.2 → 1.0.4

data/lib/mobilize-base/extensions/string.rb

@@ -17,6 +17,18 @@ class String
     raise stderr.read if (stderr.read.length>0 and except==true)
     return stdout.read
   end
+  def escape_regex
+    str = self
+    new_str = str.clone
+    char_string = "[\/^$. |?*+()"
+    char_string.chars.to_a.each{|c|
+    new_str.gsub!(c,"\\#{c}")}
+    new_str
+  end
+  def gridsafe
+    str = self
+    str.downcase.gsub(/[^a-z0-9]/,"")
+  end
   def googlesafe
     v=self
     return "" if v.to_s==""

data/lib/mobilize-base/handlers/gbook.rb

@@ -1,16 +1,16 @@
 module Mobilize
   module Gbook
-    def Gbook.find_all_by_title(title,email=nil)
-      Gdrive.books(email,{"title"=>title,"title-exact"=>"true"})
+    def Gbook.find_all_by_path(path,gdrive_slot)
+      Gdrive.books(gdrive_slot,{"title"=>path,"title-exact"=>"true"})
     end
-    def Gbook.find_or_create_by_title(title,email)
-      books = Gdrive.books(email,{"title"=>title,"title-exact"=>"true"})
-      #there should only be one book with each title, otherwise we have fail
+    def Gbook.find_or_create_by_path(path,gdrive_slot)
+      books = Gdrive.books(gdrive_slot,{"title"=>path,"title-exact"=>"true"})
+      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+      #there should only be one book with each path, otherwise we have fail
       book = nil
-      if books.length>1
+      if books.length>1 and dst.url.to_s.length>0
         #some idiot process created a duplicate book.
         #Fix by renaming all but one with dst entry's key
-        dst = Dataset.find_by_handler_and_name('gbook',title)
         dkey = dst.url.split("key=").last
         books.each do |b|
           bkey = b.resource_id.split(":").last
@@ -19,55 +19,22 @@ module Mobilize
           else
             #delete the invalid book
             b.delete
-            ("Deleted duplicate book #{title}").oputs
+            ("Deleted duplicate book #{path}").oputs
           end
         end
       else
         book = books.first
       end
       if book.nil?
-        #add book using owner email
-        #http
-        book = Gdrive.root.create_spreadsheet(title)
-        ("Created book #{title} at #{Time.now.utc.to_s}").oputs
+        #always use owner email to make sure all books are owned by owner account
+        book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
+        ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
       end
-      #delete Sheet1 if there are other sheets
-      #http
-      if (sheets = book.worksheets).length>1
-        sheet1 = sheets.select{|s| s.title == "Sheet1"}.first
-        #http
-        sheet1.delete if sheet1
-      end
-      #always make sure books have admin acl
+      #always make sure book dataset URL is up to date
+      #and that book has admin acl
+      dst.update_attributes(:url=>book.human_url)
       book.add_admin_acl
       return book
     end
-
-    def Gbook.find_or_create_by_dst_id(dst_id,email=nil)
-      #creates by title, updates acl, updates dataset with url
-      dst = Dataset.find(dst_id)
-      r = Requestor.find(dst.requestor_id)
-      book = nil
-      #http
-      book = Gdrive.root.spreadsheet_by_url(dst.url) if dst.url
-      #manually try 5 times to validate sheet since we can't just try again and again
-      5.times.each do
-        begin
-          book.resource_id
-          #if no error then break loop
-          break
-        rescue=>exc
-          if book.nil? or exc.to_s.index('Invalid document id')
-            book = Gbook.find_or_create_by_title(dst.name,email)
-            #if invalid doc then update url w new book and break loop
-            dst.update_attributes(:url=>book.human_url)
-            break
-          end
-        end
-      end
-      #add requestor write access
-      book.update_acl(r.email)
-      return book
-    end
   end
 end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -1,7 +1,7 @@
 module Mobilize
   module Gdrive
     def Gdrive.config
-      Base.config('gdrive')[Base.env]
+      Base.config('gdrive')
     end
 
     def Gdrive.domain
@@ -9,7 +9,7 @@ module Mobilize
     end
 
     def Gdrive.owner_email
-      Gdrive.config['owner']['email']
+      [Gdrive.config['owner']['name'],Gdrive.domain].join("@")
     end
 
     def Gdrive.password(email)
@@ -29,44 +29,43 @@ module Mobilize
       if email.nil?
         Gdrive.config['workers']
       else
-        Gdrive.workers.select{|w| w['email'] == email}.first
+        Gdrive.workers.select{|w| [w['name'],Gdrive.domain].join("@") == email}.first
       end
     end
 
     def Gdrive.worker_emails
-      Gdrive.workers.map{|w| w['email']}
+      Gdrive.workers.map{|w| [w['name'],Gdrive.domain].join("@")}
     end
 
     def Gdrive.admin_emails
-      Gdrive.admins.map{|w| w['email']}
+      Gdrive.admins.map{|w| [w['name'],Gdrive.domain].join("@")}
     end
 
     #email management - used to make sure not too many emails get used at the same time
-    def Gdrive.get_worker_email_by_mongo_id(mongo_id)
-      active_emails = Mobilize::Resque.jobs('working').map{|j| j['email'] if j['email']}.compact
+    def Gdrive.slot_worker_by_path(path)
+      working_slots = Mobilize::Resque.jobs('working').map{|j| j['gdrive_slot'] if j['gdrive_slot']}.compact
       Gdrive.workers.sort_by{rand}.each do |w|
-        if !(active_emails.include?(w['email']))
-          Mobilize::Resque.update_job_email(mongo_id,w['email'])
-          return w['email']
+        unless working_slots.include?([w['name'],Gdrive.domain].join("@"))
+          Mobilize::Resque.set_worker_args_by_path(path,{'gdrive_slot'=>[w['name'],Gdrive.domain].join("@")})
+          return [w['name'],Gdrive.domain].join("@")
         end
       end
       #return false if none are available
       return false
     end
 
-    def Gdrive.root(email=nil)
-      email ||= Gdrive.owner_email
-      pw = Gdrive.password(email)
-      GoogleDrive.login(email,pw)
+    def Gdrive.root(gdrive_slot=nil)
+      pw = Gdrive.password(gdrive_slot)
+      GoogleDrive.login(gdrive_slot,pw)
     end
 
-    def Gdrive.files(email=nil,params={})
-      root = Gdrive.root(email)
+    def Gdrive.files(gdrive_slot=nil,params={})
+      root = Gdrive.root(gdrive_slot)
       root.files(params)
     end
 
-    def Gdrive.books(email=nil,params={})
-      Gdrive.files(email,params).select{|f| f.class==GoogleDrive::Spreadsheet}
+    def Gdrive.books(gdrive_slot=nil,params={})
+      Gdrive.files(gdrive_slot,params).select{|f| f.class==GoogleDrive::Spreadsheet}
     end
   end
 end

data/lib/mobilize-base/handlers/gfile.rb

@@ -1,55 +1,34 @@
 module Mobilize
-  class Gfile
-    def Gfile.find_by_title(title,email=nil)
-      Gdriver.files(email).select{|f| f.title==title}.first
-    end
-
-    def Gfile.find_by_dst_id(dst_id,email=nil)
-      dst = Dataset.find(dst_id)
-      Gfile.find_by_title(dst.path,email)
-    end
-
-    def Gfile.add_admin_acl_by_dst_id(dst_id)
-      #adds admins and workers as writers
-      file = Gfile.find_by_dst_id(dst_id)
+  module Gfile
+    def Gfile.add_admin_acl_by_path(path)
+      file = Gfile.find_by_path(path)
       file.add_admin_acl
       return true
     end
 
-    def Gfile.add_admin_acl_by_title(title)
-      file = Gfile.find_by_title(title)
-      file.add_admin_acl
-      return true
-    end
-
-    def Gfile.add_worker_acl_by_title(title)
-      file = Gfile.find_by_title(title)
+    def Gfile.add_worker_acl_by_path(path)
+      file = Gfile.find_by_path(path)
       file.add_worker_acl
       return true
     end
 
-    def Gfile.update_acl_by_dst_id(dst_id,email,role="writer",edit_email=nil)
-      dst = Dataset.find(dst_id)
-      Gfile.update_acl_by_title(dst.path,email,role,edit_email)
-    end
-
-    def Gfile.update_acl_by_title(title,email,role="writer",edit_email=nil)
-      file = Gfile.find_by_title(title,edit_email)
-      raise "File #{title} not found" unless file
-      file.update_acl(email,role)
+    def Gfile.update_acl_by_path(path,gdrive_slot,role="writer",target_email=nil)
+      file = Gfile.find_by_path(path,target_email)
+      raise "File #{path} not found" unless file
+      file.update_acl(gdrive_slot,role)
     end
 
-    def Gfile.read_by_name()
-      
+    def Gfile.find_by_path(path,gdrive_slot)
+      Gdrive.files(gdrive_slot,{"title"=>path,"title-exact"=>"true"}).first
     end
 
-    def Gfile.read_by_url()
-      
+    def Gfile.read_by_task_path(task_path)
+      #reserve gdrive_slot account for read
+      gdrive_slot = Gdrive.slot_worker_by_path(t.path)
+      return false unless gdrive_slot
+      t = Task.where(:path=>task_path)
+      gfile_path = t.params.first
+      Gfile.find_by_path(gfile_path,gdrive_slot).read
     end
-
-    def Gfile.read_by_job_id(job_id)
-      j = Job.find(job_id)
-    end
-
   end
 end

data/lib/mobilize-base/handlers/gridfs.rb

@@ -0,0 +1,43 @@
+module Mobilize
+  module Gridfs
+    def Gridfs.config
+      Base.config('gridfs')
+    end
+
+    def Gridfs.grid
+      session = ::Mongoid.configure.sessions['default']
+      database_name = session['database']
+      host,port = session['hosts'].first.split(":")
+      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
+    end
+
+    def Gridfs.read(path)
+      begin
+        zs=Gridfs.grid.open(path.gridsafe,'r').read
+        return ::Zlib::Inflate.inflate(zs)
+      rescue
+        return nil
+      end
+    end
+
+    def Gridfs.write(path,string)
+      zs = ::Zlib::Deflate.deflate(string)
+      raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
+      curr_zs = Gridfs.read(path.gridsafe).to_s
+      #write a new version when there is a change
+      if curr_zs != zs
+        Gridfs.grid.open(path.gridsafe,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
+      end
+      return true
+    end
+
+    def Gridfs.delete(path)
+      begin
+        Gridfs.grid.delete(path.gridsafe)
+        return true
+      rescue
+        return nil
+      end
+    end
+  end
+end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -1,122 +1,71 @@
 module Mobilize
   module Gsheet
 
-    def Gsheet.max_cells
-      400000
+    def Gsheet.config
+      Base.config('gsheet')
     end
 
-    def Gsheet.read(name,email=nil)
-      sheet = Gsheet.find_or_create_by_name(name,email)
-      sheet.to_tsv
+    def Gsheet.max_cells
+      Gsheet.config['max_cells']
     end
 
-    def Gsheet.write(name,tsv,email=nil)
-      sheet = Gsheet.find_or_create_by_name(name,email)
+    def Gsheet.write(path,tsv,gdrive_slot)
+      sheet = Gsheet.find_or_create_by_path(path,gdrive_slot)
       sheet.write(tsv)
     end
 
-    def Gsheet.find_all_by_name(name,email)
-      book_title,sheet_title = name.split("/")
-      books = Gdrive.books(email,{"title"=>book_title,"title-exact"=>"true"})
-      sheets = books.map{|b| b.worksheets}.flatten.select{|w| w.title == sheet_title }
-      sheets
+    def Gsheet.find_by_path(path,gdrive_slot)
+      book_path,sheet_name = path.split("/")
+      book = Gdrive.books(gdrive_slot,{"title"=>book_path,"title-exact"=>"true"}).first
+      return book.worksheet_by_title(sheet_name) if book
     end
 
-    def Gsheet.find_or_create_by_name(name,email=nil,rows=100,cols=20)
-      book_title,sheet_title = name.split("/")
-      book = Gbook.find_or_create_by_title(book_title,email)
-      #http
-      sheet = book.worksheets.select{|w| w.title==sheet_title}.first
+    def Gsheet.find_or_create_by_path(path,gdrive_slot,rows=100,cols=20)
+      book_path,sheet_name = path.split("/")
+      book = Gbook.find_or_create_by_path(book_path,gdrive_slot)
+      sheet = book.worksheet_by_title(sheet_name)
       if sheet.nil?
-        #http
-        sheet = book.add_worksheet(sheet_title,rows,cols)
-        ("Created sheet #{name} at #{Time.now.utc.to_s}").oputs
+        sheet = book.add_worksheet(sheet_name,rows,cols)
+        ("Created gsheet #{path} at #{Time.now.utc.to_s}").oputs
       end
+      Dataset.find_or_create_by_handler_and_path("gsheet",path)
       return sheet
     end
 
-    def Gsheet.find_or_create_by_dst_id(dst_id,email=nil)
-      #creates by title, updates acl, updates dataset with url
-      dst = Dataset.find(dst_id)
-      r = Requestor.find(dst.requestor_id)
-      name = dst.name
-      book_title,sheet_title = name.split("/")
-      #make sure book exists and is assigned to this user
-      r.find_or_create_gbook_by_title(book_title,email)
-      #add admin write access
-      sheet = Gsheet.find_or_create_by_name(name)
-      sheet_title = nil
-      return sheet
-    end
-
-    def Gsheet.read_by_dst_id(dst_id,email=nil)
-      dst = Dataset.find(dst_id)
-      name = dst.name
-      sheet = Gsheet.find_or_create_by_name(name,email)
-      output = sheet.to_tsv
-      return output
-    end
-
-    def Gsheet.read_by_job_id(job_id)
-      j = Job.find(job_id)
-      #reserve email account for read
-      email = Gdrive.get_worker_email_by_mongo_id(job_id)
-      return false unless email
-      #pull tsv from cache
-      j.dataset_array.first.read_cache
+    def Gsheet.read_by_task_path(task_path)
+      #reserve gdrive_slot account for read
+      gdrive_slot = Gdrive.slot_worker_by_path(task_path)
+      return false unless gdrive_slot
+      t = Task.where(:path=>task_path).first
+      gsheet_path = t.params.first
+      Gsheet.find_by_path(gsheet_path,gdrive_slot).to_tsv
     end
 
-    def Gsheet.write_by_dst_id(dst_id,tsv,email=nil)
-      dst = Dataset.find(dst_id)
-      #see if this is a specific cell
-      name = dst.name
-      return false unless email
-      #create temp tab, write data to it, checksum it against the source
-      temp_sheet = Gsheet.find_or_create_by_name("#{name}_temp")
-      temp_sheet.write(tsv)
-      #delete current sheet, replace it with temp one
-      sheet = Gsheet.find_or_create_by_name(dst.name)
-      title = sheet.title
-      #http
-      sheet.delete
-      begin
-        temp_sheet.rename(title)
-      rescue
-        #need this because sometimes it gets confused and tries to rename twice
-      end
-      "Write successful for #{write_name}".oputs
-      return true
-    end
-
-    def Gsheet.write_by_job_id(job_id)
-      j = Job.find(job_id)
-      r = j.requestor
-      dest_name = if j.destination.split("/").length==1
-                    "#{r.jobspec_title}#{"/"}#{j.destination}"
-                  else
-                    j.destination
-                  end
-      sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheet',dest_name)
-      sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
-      email = Gdrive.get_worker_email_by_mongo_id(job_id)
+    def Gsheet.write_by_task_path(task_path)
+      gdrive_slot = Gdrive.slot_worker_by_path(task_path)
       #return false if there are no emails available
-      return false unless email
-      #create temp tab, write data to it, checksum it against the source
-      temp_sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheet',"#{dest_name}_temp")
-      temp_sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if temp_sheet_dst.requestor_id.nil?
-      temp_sheet = Gsheet.find_or_create_by_name(temp_sheet_dst.name,email)
-      #tsv is the prior task's output
-      tsv = j.task_output_dsts[j.task_idx-1].read
-      temp_sheet.write(tsv,true,job_id)
-      #delete current sheet, replace it with temp one
-      sheet = Gsheet.find_or_create_by_name(dest_name,email)
-      title = sheet.title
-      #http
-      sheet.delete
-      temp_sheet.title = title
-      temp_sheet.save
-      sheet_dst.update_attributes(:url=>temp_sheet.spreadsheet.human_url)
-      "Write successful for #{dest_name}".oputs
+      return false unless gdrive_slot
+      t = Task.where(:path=>task_path).first
+      source = t.params.first
+      target_path = t.params.second
+      source_job_name, source_task_name = if source.index("/")
+                                            source.split("/")
+                                          else
+                                            [nil, source]
+                                          end
+      source_task_path = "#{t.job.runner.path}/#{source_job_name || t.job.name}/#{source_task_name}"
+      source_task = Task.where(:path=>source_task_path).first
+      tsv = source_task.stdout_dataset.read_cache
+      sheet_name = target_path.split("/").last
+      temp_path = [task_path.gridsafe,sheet_name].join("/")
+      temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
+      temp_sheet.write(tsv)
+      temp_sheet.check_and_fix(tsv)
+      target_sheet = Gsheet.find_or_create_by_path(target_path,gdrive_slot)
+      target_sheet.merge(temp_sheet)
+      #delete the temp sheet's book
+      temp_sheet.spreadsheet.delete
+      "Write successful for #{target_path}".oputs
       return true
     end
   end