mobilize-base 1.36 → 1.293

data/lib/mobilize-base.rb

@@ -3,7 +3,6 @@ require "mobilize-base/extensions/array"
 require "mobilize-base/extensions/hash"
 require "mobilize-base/extensions/object"
 require "mobilize-base/extensions/string"
-require "mobilize-base/extensions/time"
 require "mobilize-base/extensions/yaml"
 #this is the base of the mobilize object, any methods that should be
 #made available application-wide go over here
@@ -17,9 +16,6 @@ module Mobilize
         ENV['PWD']
       end
     end
-    def Base.home_dir
-      File.expand_path('..',File.dirname(__FILE__))
-    end
     def Base.config_dir
       ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/"
     end
@@ -60,25 +56,18 @@ module Mobilize
         raise "Could not find #{log_dir} folder for logs"
       end
     end
-    def Base.handlers
-      Dir.entries(File.dirname(__FILE__) + "/mobilize-base/handlers").select{|e| e.ends_with?(".rb")}.map{|e| e.split(".").first}
-    end
   end
 end
 mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
 if File.exists?(mongoid_config_path)
+  require 'mongo'
   require 'mongoid'
-  require 'mongoid-grid_fs'
   Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
   require "mobilize-base/models/dataset"
   require "mobilize-base/models/user"
-  require "mobilize-base/helpers/runner_helper"
   require "mobilize-base/models/runner"
-  require "mobilize-base/helpers/job_helper"
   require "mobilize-base/models/job"
-  require "mobilize-base/helpers/stage_helper"
   require "mobilize-base/models/stage"
-
 end
 require 'google_drive'
 require 'resque'

data/lib/mobilize-base/extensions/array.rb

@@ -12,16 +12,11 @@ class Array
     return self.inject{|sum,x| sum + x }
   end
   def hash_array_to_tsv
-    ha = self
-    if ha.first.nil? or ha.first.class!=Hash
+    if self.first.nil? or self.first.class!=Hash
       return ""
     end
-    max_row_length = ha.map{|h| h.keys.length}.max
-    header_keys = ha.select{|h| h.keys.length==max_row_length}.first.keys
-    header = header_keys.join("\t")
-    rows = ha.map do |r|
-      header_keys.map{|k| r[k]}.join("\t")
-    end
+    header = self.first.keys.join("\t")
+    rows = self.map{|r| r.values.join("\t")}
     ([header] + rows).join("\n")
   end
 end

data/lib/mobilize-base/extensions/google_drive/acl.rb

@@ -14,7 +14,7 @@ module GoogleDrive
     def push(entry)
       #do not send email notifications
       entry = AclEntry.new(entry) if entry.is_a?(Hash)
-      url_suffix = ((@acls_feed_url.index("?") ? "&" : "?") + "send-notification-emails=false")
+      url_suffix = "?send-notification-emails=false"
       header = {"GData-Version" => "3.0", "Content-Type" => "application/atom+xml"}
       doc = @session.request(:post, "#{@acls_feed_url}#{url_suffix}", :data => entry.to_xml(), :header => header, :auth => :writely)
       entry.params = entry_to_params(doc.root)

data/lib/mobilize-base/extensions/google_drive/client_login_fetcher.rb

@@ -8,7 +8,7 @@ module GoogleDrive
       attempts = 0
       sleep_time = nil
       #try 5 times to make the call
-      while (response.nil? or response.code.starts_with?("5")) and attempts < Mobilize::Gdrive.max_api_retries
+      while (response.nil? or response.code.starts_with?("5")) and attempts < 20
         #instantiate http object, set params
         http = @proxy.new(uri.host, uri.port)
         http.use_ssl = true
@@ -16,7 +16,6 @@ module GoogleDrive
         #set 600  to allow for large downloads
         http.read_timeout = 600
         response = begin
-                     puts "#{Time.now.utc} Gdrive API #{method.to_s}: #{uri.to_s} #{extra_header.to_s}"
                      clf.http_call(http, method, uri, data, extra_header, auth)
                    rescue
                      #timeouts etc.

data/lib/mobilize-base/extensions/google_drive/file.rb

@@ -3,16 +3,42 @@ module GoogleDrive
 
     def add_worker_acl
       f = self
-      email = "#{Mobilize::Gdrive.worker_group_name}@#{Mobilize::Gdrive.domain}"
-      f.update_acl(email,"group")
+      return true if f.has_worker_acl?
+      Mobilize::Gdrive.worker_emails.each do |a| 
+        f.update_acl(a)
+      end
     end
 
     def add_admin_acl
       f = self
-      email = "#{Mobilize::Gdrive.admin_group_name}@#{Mobilize::Gdrive.domain}"
-      f.update_acl(email,"group")
-      #if adding acl ,must currently add workers as well
-      f.add_worker_acl
+      #admin includes workers
+      return true if f.has_admin_acl?
+      accounts = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails)
+      accounts.each do |email|
+        f.update_acl(email)
+      end
+    end
+
+    def has_admin_acl?
+      f = self
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      admin_emails = (Mobilize::Gdrive.admin_emails + Mobilize::Gdrive.worker_emails)
+      if (curr_emails & admin_emails) == admin_emails
+        return true
+      else
+        return false
+      end
+    end
+
+    def has_worker_acl?
+      f = self
+      curr_emails = f.acls.map{|a| a.scope}.sort
+      worker_emails = Mobilize::Gdrive.worker_emails.sort
+      if (curr_emails & worker_emails) == worker_emails
+        return true
+      else
+        return false
+      end
     end
 
     def read(user_name)
@@ -25,7 +51,7 @@ module GoogleDrive
       end
     end
 
-    def update_acl(email,scope_type="user",role="writer")
+    def update_acl(email,role="writer")
       f = self
       #need these flags for HTTP retries
       #create req_acl hash to add to current acl
@@ -38,16 +64,16 @@ module GoogleDrive
           if entry.role != role
             #for whatever reason
             f.acl.delete(entry)
-            f.acl.push({:scope_type=>scope_type,:scope=>email,:role=>role})
+            f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
           end
         elsif !['reader','writer','owner'].include?(role)
           raise "Invalid role #{role}"
         end
       else
         begin
-          f.acl.push({:scope_type=>scope_type,:scope=>email,:role=>role})
+          f.acl.push({:scope_type=>"user",:scope=>email,:role=>role})
         rescue => exc
-          raise exc unless exc.to_s.index("already has access")
+          raise exc unless exc.to_s.index("user already has access")
         end
       end
       return true
@@ -58,7 +84,7 @@ module GoogleDrive
     end
     def acl_entry(email)
       f = self
-      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope and a.scope == email}.first
+      f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
     end
     def entry_hash
       f = self

data/lib/mobilize-base/extensions/string.rb

@@ -11,18 +11,13 @@ class String
   def opp
     pp self
   end
-  def to_md5
-    Digest::MD5.hexdigest(self)
-  end
   def bash(except=true)
     str = self
-    out_str,err_str = []
-    status = Open4.popen4(str) do |pid,stdin,stdout,stderr|
-      out_str = stdout.read
-      err_str = stderr.read
-    end
-    exit_status = status.exitstatus
-    raise err_str if (exit_status !=0 and except==true)
+    pid,stdin,stdout,stderr = Open4.popen4(str)
+    pid,stdin = [nil,nil]
+    err_str = stderr.read if stderr
+    out_str = stdout.read if stdout
+    raise err_str if (err_str.length>0 and except==true)
     return out_str
   end
   def escape_regex
@@ -43,7 +38,7 @@ class String
     return v if v.to_s.strip==""
     #normalize numbers by removing '$', '%', ',', ' '
     vnorm = v.to_s.norm_num
-    vdigits = vnorm.split(".").last.to_s.length
+    vdigits = vnorm.split(".").last.length
     if vnorm.to_f.to_s=="Infinity"
       #do nothing
     elsif ("%.#{vdigits}f" % vnorm.to_f.to_s)==vnorm

data/lib/mobilize-base/extensions/yaml.rb

@@ -10,16 +10,13 @@ module YAML
       #make sure urls have their colon spaces fixed
       result_hash={}
       easy_hash.each do |k,v|
-        #fucking yaml puts spaces in front of the key
-        #or something
-        strip_k = k.strip
-        result_hash[strip_k] = if v.class==String
-                                 v.gsub(": //","://")
-                               elsif v.class==Array
-                                 v.map{|av| av.to_s.gsub(": //","://")}
-                               else
-                                 v
-                               end
+        result_hash[k] = if v.class==String
+                           v.gsub(": //","://")
+                         elsif v.class==Array
+                           v.map{|av| av.to_s.gsub(": //","://")}
+                         else
+                           v
+                         end
       end
       return result_hash
     end

data/lib/mobilize-base/handlers/gbook.rb

@@ -14,44 +14,57 @@ module Mobilize
       dst = Dataset.find_by_handler_and_path('gbook',path)
       if dst and dst.http_url.to_s.length>0
         book = Gbook.find_by_http_url(dst.http_url,gdrive_slot)
-        if book
-          return book
-        else
-          raise "Could not find book #{path} with url #{dst.http_url}, please check dataset"
+        begin
+          #doesn't count if it's deleted
+          #or if its name can't be accessed
+          if book.entry_hash[:deleted]
+            book = nil
+          else
+            return book
+          end
+        rescue
+          #use regular process if book entry hash fails
+          book = nil
         end
       end
-      #try to find books by title
       books = Gbook.find_all_by_path(path,gdrive_slot)
-      #sort by publish date; if entry hash retrieval fails (as it does)
-      #assume the book was published now
-      book = books.sort_by{|b| begin b.entry_hash[:published];rescue;Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z");end;}.first
-      if book
-        #we know dataset will have blank url since it wasn't picked up above
-        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
-        api_url = book.human_url.split("&").first
-        dst.update_attributes(:http_url=>api_url)
+      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+      book = nil
+      if books.length>1 and dst.http_url.to_s.length>0
+        #some idiot process or malicious user created a duplicate book.
+        #Fix by deleting all but the one with dst entry's key
+        dkey = dst.http_url.split("key=").last
+        books.each do |b|
+          bkey = b.resource_id.split(":").last
+          if bkey == dkey
+            book = b
+            dst.update_attributes(:http_url=>book.human_url)
+          else
+            #delete the invalid book
+            b.delete
+            ("Deleted duplicate book #{path}").oputs
+          end
+        end
+      else
+        #If it's a new dst or if there are multiple books
+        #take the first
+        book = books.first
+        dst.update_attributes(:http_url=>book.human_url) if book
       end
       return book
     end
-
     def Gbook.find_or_create_by_path(path,gdrive_slot)
       book = Gbook.find_by_path(path,gdrive_slot)
+      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       if book.nil?
         #always use owner email to make sure all books are owned by owner account
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
-        #check to make sure the dataset has a blank url; if not, error out
-        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
-        if dst.http_url.to_s.length>0
-          #add acls to book regardless
-          book.add_admin_acl
-          raise "Book #{path} is already assigned to #{dst.http_url}; please update the record with #{book.human_url}"
-        else
-          api_url = book.human_url.split("&").first
-          dst.update_attributes(:http_url=>api_url)
-          book.add_admin_acl
-        end
       end
+      #always make sure book dataset http URL is up to date
+      #and that book has admin acl
+      dst.update_attributes(:http_url=>book.human_url)
+      book.add_admin_acl
       return book
     end
   end

data/lib/mobilize-base/handlers/gdrive.rb

@@ -25,18 +25,6 @@ module Mobilize
       end
     end
 
-    def Gdrive.max_api_retries
-      Gdrive.config['max_api_retries']
-    end
-
-    def Gdrive.max_file_write_retries
-      Gdrive.config['max_file_write_retries']
-    end
-
-    def Gdrive.file_write_retry_delay
-      Gdrive.config['file_write_retry_delay']
-    end
-
     def Gdrive.admins
       Gdrive.config['admins']
     end
@@ -49,18 +37,14 @@ module Mobilize
       end
     end
 
-    def Gdrive.worker_group_name
-      Gdrive.config['worker_group_name']
-    end
-
-    def Gdrive.admin_group_name
-      Gdrive.config['admin_group_name']
-    end
-
     def Gdrive.worker_emails
       Gdrive.workers.map{|w| [w['name'],Gdrive.domain].join("@")}
     end
 
+    def Gdrive.admin_emails
+      Gdrive.admins.map{|w| [w['name'],Gdrive.domain].join("@")}
+    end
+
     #email management - used to make sure not too many emails get used at the same time
     def Gdrive.slot_worker_by_path(path)
       working_slots = Mobilize::Resque.jobs.map{|j| begin j['args'][1]['gdrive_slot'];rescue;nil;end}.compact.uniq

data/lib/mobilize-base/handlers/gfile.rb

@@ -1,23 +1,9 @@
 module Mobilize
   module Gfile
-
-    def Gfile.config
-      Base.config('gfile')
-    end
-
-    def Gfile.max_length
-      Gfile.config['max_length']
-    end
-   
     def Gfile.path_to_dst(path,stage_path,gdrive_slot)
-      s = Stage.where(:path=>stage_path).first
-      params = s.params
-      target_path = params['target']
-      #if this is the target, it doesn't have to exist already
-      is_target = true if path == target_path
       #don't need the ://
       path = path.split("://").last if path.index("://")
-      if is_target or Gfile.find_by_path(path)
+      if Gfile.find_by_path(path)
         handler = "gfile"
         Dataset.find_or_create_by_url("#{handler}://#{path}")
       else
@@ -44,10 +30,15 @@ module Mobilize
                                     :content_type=>"test/plain",
                                     :convert=>false)
       file.add_admin_acl
+      #make sure user is owner or can edit
+      u = User.where(:name=>user_name).first
+      entry = file.acl_entry(u.email)
+      unless entry and ['writer','owner'].include?(entry.role)
+        file.update_acl(u.email)
+      end
       #update http url for file
       dst = Dataset.find_by_handler_and_path("gfile",dst_path)
-      api_url = file.human_url.split("&").first
-      dst.update_attributes(:http_url=>api_url)
+      dst.update_attributes(:http_url=>file.human_url)
       true
     end
 
@@ -66,7 +57,7 @@ module Mobilize
     def Gfile.update_acl_by_path(path,gdrive_slot,role="writer",target_email=nil)
       file = Gfile.find_by_path(path,target_email)
       raise "File #{path} not found" unless file
-      file.update_acl(gdrive_slot,"user",role)
+      file.update_acl(gdrive_slot,role)
     end
 
     def Gfile.find_by_path(path)
@@ -95,55 +86,10 @@ module Mobilize
       #always make sure dataset http URL is up to date
       #and that it has admin acl
       if file
-        api_url = file.human_url.split("&").first
-        dst.update_attributes(:http_url=>api_url)
+        dst.update_attributes(:http_url=>file.human_url)
         file.add_admin_acl
       end
       return file
     end
-
-    def Gfile.write_by_stage_path(stage_path)
-      gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
-      #return blank response if there are no slots available
-      return nil unless gdrive_slot
-      s = Stage.where(:path=>stage_path).first
-      u = s.job.runner.user
-      retries = 0
-      stdout,stderr = []
-      while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
-        begin
-          #get tsv to write from stage
-          source = s.sources(gdrive_slot).first
-          raise "Need source for gfile write" unless source
-          tsv = source.read(u.name,gdrive_slot)
-          raise "No data source found for #{source.url}" unless tsv.to_s.length>0
-          if tsv.length > Gfile.max_length
-            raise "Too much data; you have #{tsv.length.to_s}, max is #{Gfile.max_length.to_s}"
-          end
-          stdout = if tsv.length == 0
-               #soft error; no data to write. Stage will complete.
-               "Write skipped for #{s.target.url}"
-             else
-               Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot)
-               #update status
-               "Write successful for #{s.target.url}"
-             end
-          Gdrive.unslot_worker_by_path(stage_path)
-          stderr = nil
-          s.update_status(stdout)
-          signal = 0
-        rescue => exc
-          if retries < Gdrive.max_file_write_retries
-            retries +=1
-            sleep Gdrive.file_write_retry_delay
-          else
-            stdout = nil
-            stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
-            signal = 500
-          end
-        end
-      end
-      return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
-    end
   end
 end