mobilize-base 1.294 → 1.295

data/README.md

@@ -355,22 +355,16 @@ mobilize_base:resque_web task, as detailed in [Start Resque-Web](#section_Start_
 Mobilize stores cached data in MongoDB Gridfs. 
 It needs the below parameters, which can be found in the [lib/samples][git_samples] folder. 
 
-* max_versions - the number of __different__ versions of data to keep
-for a given cache. Default is 10. This is meant mostly to allow you to
-restore Runners from cache if necessary.
 * max_compressed_write_size - the amount of compressed data Gridfs will
 allow. If you try to write more than this, an exception will be thrown.
 
 ``` yml
 ---
 development:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 test:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 production:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 ```
 

data/lib/mobilize-base/extensions/string.rb

@@ -11,6 +11,9 @@ class String
   def opp
     pp self
   end
+  def to_md5
+    Digest::MD5.hexdigest(self)
+  end
   def bash(except=true)
     str = self
     pid,stdin,stdout,stderr = Open4.popen4(str)

data/lib/mobilize-base/extensions/yaml.rb

@@ -10,13 +10,16 @@ module YAML
       #make sure urls have their colon spaces fixed
       result_hash={}
       easy_hash.each do |k,v|
-        result_hash[k] = if v.class==String
-                           v.gsub(": //","://")
-                         elsif v.class==Array
-                           v.map{|av| av.to_s.gsub(": //","://")}
-                         else
-                           v
-                         end
+        #fucking yaml puts spaces in front of the key
+        #or something
+        strip_k = k.strip
+        result_hash[strip_k] = if v.class==String
+                                 v.gsub(": //","://")
+                               elsif v.class==Array
+                                 v.map{|av| av.to_s.gsub(": //","://")}
+                               else
+                                 v
+                               end
       end
       return result_hash
     end

data/lib/mobilize-base/handlers/gbook.rb

@@ -14,57 +14,44 @@ module Mobilize
       dst = Dataset.find_by_handler_and_path('gbook',path)
       if dst and dst.http_url.to_s.length>0
         book = Gbook.find_by_http_url(dst.http_url,gdrive_slot)
-        begin
-          #doesn't count if it's deleted
-          #or if its name can't be accessed
-          if book.entry_hash[:deleted]
-            book = nil
-          else
-            return book
-          end
-        rescue
-          #use regular process if book entry hash fails
-          book = nil
+        if book
+          return book
+        else
+          raise "Could not find book #{path} with url #{dst.http_url}, please check dataset"
         end
       end
+      #try to find books by title
       books = Gbook.find_all_by_path(path,gdrive_slot)
-      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
-      book = nil
-      if books.length>1 and dst.http_url.to_s.length>0
-        #some idiot process or malicious user created a duplicate book.
-        #Fix by deleting all but the one with dst entry's key
-        dkey = dst.http_url.split("key=").last
-        books.each do |b|
-          bkey = b.resource_id.split(":").last
-          if bkey == dkey
-            book = b
-            dst.update_attributes(:http_url=>book.human_url)
-          else
-            #delete the invalid book
-            b.delete
-            ("Deleted duplicate book #{path}").oputs
-          end
-        end
-      else
-        #If it's a new dst or if there are multiple books
-        #take the first
-        book = books.first
-        dst.update_attributes(:http_url=>book.human_url) if book
+      #sort by publish date; if entry hash retrieval fails (as it does)
+      #assume the book was published now
+      book = books.sort_by{|b| begin b.entry_hash[:published];rescue;Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z");end;}.first
+      if book
+        #we know dataset will have blank url since it wasn't picked up above
+        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+        api_url = book.human_url.split("&").first
+        dst.update_attributes(:http_url=>api_url)
       end
       return book
     end
+
     def Gbook.find_or_create_by_path(path,gdrive_slot)
       book = Gbook.find_by_path(path,gdrive_slot)
-      dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
       if book.nil?
         #always use owner email to make sure all books are owned by owner account
         book = Gdrive.root(Gdrive.owner_email).create_spreadsheet(path)
         ("Created book #{path} at #{Time.now.utc.to_s}; Access at #{book.human_url}").oputs
+        #check to make sure the dataset has a blank url; if not, error out
+        dst = Dataset.find_or_create_by_handler_and_path('gbook',path)
+        if dst.http_url.to_s.length>0
+          #add acls to book regardless
+          book.add_admin_acl
+          raise "Book #{path} is already assigned to #{dst.http_url}; please update the record with #{book.human_url}"
+        else
+          api_url = book.human_url.split("&").first
+          dst.update_attributes(:http_url=>api_url)
+          book.add_admin_acl
+        end
       end
-      #always make sure book dataset http URL is up to date
-      #and that book has admin acl
-      dst.update_attributes(:http_url=>book.human_url)
-      book.add_admin_acl
       return book
     end
   end

data/lib/mobilize-base/handlers/gfile.rb

@@ -38,7 +38,8 @@ module Mobilize
       end
       #update http url for file
       dst = Dataset.find_by_handler_and_path("gfile",dst_path)
-      dst.update_attributes(:http_url=>file.human_url)
+      api_url = file.human_url.split("&").first
+      dst.update_attributes(:http_url=>api_url)
       true
     end
 
@@ -86,7 +87,8 @@ module Mobilize
       #always make sure dataset http URL is up to date
       #and that it has admin acl
       if file
-        dst.update_attributes(:http_url=>file.human_url)
+        api_url = file.human_url.split("&").first
+        dst.update_attributes(:http_url=>api_url)
         file.add_admin_acl
       end
       return file

data/lib/mobilize-base/handlers/gridfs.rb

@@ -4,40 +4,34 @@ module Mobilize
       Base.config('gridfs')
     end
 
-    def Gridfs.grid
-      session = ::Mongoid.configure.sessions['default']
-      database_name = session['database']
-      host,port = session['hosts'].first.split(":")
-      return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
+    def Gridfs.read_by_dataset_path(dst_path,*args)
+      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
+      zs = curr_file.data if curr_file
+      return ::Zlib::Inflate.inflate(zs) if zs.to_s.length>0
     end
 
-    def Gridfs.read_by_dataset_path(dst_path,user_name,*args)
-      begin
-        zs=Gridfs.grid.open(dst_path,'r').read
-        return ::Zlib::Inflate.inflate(zs)
-      rescue
-        return nil
-      end
-    end
-
-    def Gridfs.write_by_dataset_path(dst_path,string,user_name,*args)
+    def Gridfs.write_by_dataset_path(dst_path,string,*args)
       zs = ::Zlib::Deflate.deflate(string)
       raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
-      curr_zs = Gridfs.read_by_dataset_path(dst_path,user_name).to_s
-      #write a new version when there is a change
+      #find and delete existing file
+      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
+      curr_zs =  curr_file.data if curr_file
+      #overwrite when there is a change
       if curr_zs != zs
-        Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
+        curr_file.delete if curr_file
+        #create temp file w zstring
+        temp_file = Tempfile.new("#{string}#{Time.now.to_f}".to_md5)
+        temp_file.print(zs)
+        temp_file.close
+        #put data in file
+        Mongoid::GridFs.put(temp_file.path,:filename=>dst_path)
       end
       return true
     end
 
     def Gridfs.delete(dst_path)
-      begin
-        Gridfs.grid.delete(dst_path)
-        return true
-      rescue
-        return nil
-      end
+      curr_file = Mongoid::GridFs::Fs::File.where(:filename=>dst_path).first
+      curr_file.delete
     end
   end
 end

data/lib/mobilize-base/handlers/gsheet.rb

@@ -86,6 +86,10 @@ module Mobilize
       temp_sheet.delete if temp_sheet
       #write data to temp sheet
       temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
+      #delete the temp sheet's datasets, they won't be needed again
+      temp_sheet_dst = Dataset.find_by_handler_and_path("gsheet",temp_path)
+      temp_book_dst = Dataset.find_by_handler_and_path("gbook",target_path.gridsafe)
+      [temp_sheet_dst, temp_book_dst].compact.each{|s| s.delete}
       #this step has a tendency to fail; if it does,
       #don't fail the stage, mark it as false
       begin

data/lib/mobilize-base/jobtracker.rb

@@ -286,6 +286,7 @@ module Mobilize
       # delete any old runner from previous test runs
       gdrive_slot = Gdrive.owner_email
       u.runner.gsheet(gdrive_slot).spreadsheet.delete
+      Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
       Jobtracker.update_status("enqueue jobtracker, wait 45s")
       Mobilize::Jobtracker.start
       sleep 45

data/lib/mobilize-base/models/runner.rb

@@ -93,8 +93,6 @@ module Mobilize
         end
       end
       jobs_sheet.add_headers(r.headers)
-      #add url to dataset
-      Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
       begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
       return jobs_sheet
     end
@@ -159,7 +157,7 @@ module Mobilize
       #there's nothing to update if runner has never had a completed at
       return false unless r.completed_at
       jobs_gsheet = r.gsheet(gdrive_slot)
-      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
+      upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
       upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
       jobs_gsheet.add_or_update_rows(upd_rows)
       r.update_status("gsheet updated")
@@ -178,7 +176,7 @@ module Mobilize
 
     def user
       r = self
-      user_name = r.path.split("_").second.split("(").first.split("/").first
+      user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
       User.where(:name=>user_name).first
     end
 

data/lib/mobilize-base/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Base
-    VERSION = "1.294"
+    VERSION = "1.295"
   end
 end

data/lib/mobilize-base.rb

@@ -60,8 +60,8 @@ module Mobilize
 end
 mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
 if File.exists?(mongoid_config_path)
-  require 'mongo'
   require 'mongoid'
+  require 'mongoid-grid_fs'
   Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
   require "mobilize-base/models/dataset"
   require "mobilize-base/models/user"

data/lib/samples/gridfs.yml

@@ -1,10 +1,7 @@
 ---
 development:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 test:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB
 production:
-  max_versions: 10 #number of versions of cache to keep in gridfs
   max_compressed_write_size: 1000000000 #~1GB

data/mobilize-base.gemspec

@@ -22,10 +22,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_runtime_dependency 'rake'
-  s.add_runtime_dependency 'bson','1.6.1'
-  s.add_runtime_dependency 'bson_ext','1.6.1'
-  s.add_runtime_dependency 'mongo', '1.6.1'
+  s.add_runtime_dependency 'bson','1.8.4'
+  s.add_runtime_dependency 'bson_ext','1.8.4'
   s.add_runtime_dependency "mongoid", "~>3.0.0"
+  s.add_runtime_dependency 'mongoid-grid_fs'
   s.add_runtime_dependency 'resque', '1.24.0'
   s.add_runtime_dependency 'google_drive','0.3.2'
   s.add_runtime_dependency 'popen4','0.1.2'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-base
 version: !ruby/object:Gem::Version
-  version: '1.294'
+  version: '1.295'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-01 00:00:00.000000000 Z
+date: 2013-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.8.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.8.4
 - !ruby/object:Gem::Dependency
   name: bson_ext
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.8.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,39 +58,39 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.8.4
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: mongoid
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
-  name: mongoid
+  name: mongoid-grid_fs
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
@@ -225,7 +225,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1508381292879500596
+      hash: -2613537944604284453
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -234,7 +234,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1508381292879500596
+      hash: -2613537944604284453
 requirements: []
 rubyforge_project: mobilize-base
 rubygems_version: 1.8.25