umlaut 3.0.0alpha8 → 3.0.0alpha9

data/lib/tasks/umlaut_migrate_permalinks.rake

@@ -2,7 +2,12 @@ namespace :umlaut do
     
     desc "Migrate permalinks from an umlaut 2.x installation"
     task :migrate_permalinks, [:connection] => [:environment] do |t,args|
-      old_connection_name = args[:connection] || "umlaut2_source"            
+      old_connection_name = args[:connection] || "umlaut2_source"    
+      from_id = ENV['FROM_ID']
+      
+      import_batch = (ENV['IMPORT_BATCH'] || 20000).to_i
+      export_batch = (ENV['EXPORT_BATCH'] || 20000).to_i
+      gc_batch = (ENV['GC_BATCH'] || 40000).to_i
       
       begin        
         require 'activerecord-import'        
@@ -20,13 +25,16 @@ namespace :umlaut do
         raise Exception.new("You must have a connection for '#{old_connection_name}' configured in database.yml, pointing to the Umlaut 2.x source db.")
       end
             
-      puts "\nWARNING: We will delete all existing permalinks from the '#{Rails.env}' database:"   
+      puts "\nWARNING: We will delete all existing permalinks " + (from_id ? "(id >= #{from_id})" : "") +  " from the '#{Rails.env}' database:"   
       puts "  " + {}.merge(Permalink.connection_config).inspect
-      print "Sure you want to continue? (yes to continue) "
+      print "\nSure you want to continue? (yes to continue) "
       continue = $stdin.gets
       raise Exception.new("Cancelled by user") unless continue.chomp == "yes" 
       
-      Permalink.delete_all
+      puts "\nDeleting...\n"
+      rel = Permalink
+      rel = rel.where("id >= #{from_id}") if from_id      
+      rel.delete_all
       
       # dynamically create a model for Permalinks in old
       # db, set up with connection info to old db. 
@@ -53,56 +61,80 @@ namespace :umlaut do
         :highest_id => 0,
         :latest_date => 0
       }
+      highest_id = 0
       bulk_queue = []
       i = 0
+      
+      
       ActiveRecord::Base.uncached do 
-        OldPermalink.find_each(:batch_size => 20000) do |old_p|
+        rel = OldPermalink
+        rel = rel.where("id >= #{from_id}") if from_id
+
+        rel.find_each(:batch_size => import_batch) do |old_p|
           i += 1
           
+          highest_id = [highest_id, old_p.id].max
+          
           if old_p.context_obj_serialized.blank?
             could_not_migrate[:count] += 1
             count_not_migrate[:highest_id] = [count_not_migrate[:highest_id], old_p.id].max
             could_not_migrate[:latest_date] = [could_not_migrate[:latest_date], old_p.created_on].max
-          else        
-            new_p = Permalink.new
-            new_p.id = old_p.id # keep the id the same!
-            new_p.created_on = old_p.created_on # why not keep it the same?
-            new_p.orig_rfr_id = old_p.orig_rfr_id # why not
-            
-            # the important thing to be able to actually resolve it
-            new_p.context_obj_serialized = old_p.context_obj_serialized
-            
+          else                    
             if ar_import
-              bulk_queue << new_p
+              values =  [ old_p.id,
+                          old_p.created_on,
+                          old_p.orig_rfr_id,
+                          old_p.context_obj_serialized
+                         ]
+                         
+              bulk_queue << values
+              
             else
+              new_p = Permalink.new
+              new_p.id = old_p.id # keep the id the same!
+              new_p.created_on = old_p.created_on # why not keep it the same?
+              new_p.orig_rfr_id = old_p.orig_rfr_id # why not            
+              # the important thing to be able to actually resolve it
+              new_p.context_obj_serialized = old_p.context_obj_serialized
+              
               new_p.save!
             end
           end
   
-          print(".") if i % 1000 == 0        
-          
-          if ar_import && i % 10000 == 0
+          print(".") if i % 1000 == 0 
+                    
+          if ar_import && i % export_batch == 0
+            print "-" 
+            Permalink.import( [:id, :created_on, :orig_rfr_id, :context_obj_serialized],                            
+              bulk_queue, :validate => false, :timestamps => false)
             print "+"
-            Permalink.import(bulk_queue, :validate => false, :timestamps => false)
-            bulk_queue.clear
+            bulk_queue.clear            
+          end
+          
+          if i % gc_batch == 0
+            GC.start 
+            print "*"
           end
           
         end
       end
       
       unless bulk_queue.empty?
+        print "-" 
+        Permalink.import( [:id, :created_on, :orig_rfr_id, :context_obj_serialized],                            
+          bulk_queue, :validate => false, :timestamps => false)
         print "+"
-        Permalink.import(bulk_queue, :validate => false, :timestamps => false)
+        bulk_queue.clear
       end
-      
-      puts "\nDone."
-      
+                  
       if could_not_migrate[:count] > 0
         puts "\n\nCould not migrate #{could_not_migrate[:count]} permalinks"
         puts "   Ending at permalink #{could_not_migrate[:highest_id]}, "
         puts "   created at #{could_not_migrate[:latest_date]}"
       end
       
+      puts "\nDone. Highest ID migrated: #{highest_id}"
+      
     end
     
   end

data/lib/term_color.rb

@@ -0,0 +1,34 @@
+# Just some methods for outputting colorized text to a terminal
+# with ansi codes, cribbed from rails. 
+
+module TermColor
+  mattr_accessor :colorize_logging
+  self.colorize_logging = Rails.application.config.colorize_logging
+  
+    # Embed in a String to clear all previous ANSI sequences.
+    CLEAR   = "\e[0m"
+    BOLD    = "\e[1m"
+
+    # Colors
+    BLACK   = "\e[30m"
+    RED     = "\e[31m"
+    GREEN   = "\e[32m"
+    YELLOW  = "\e[33m"
+    BLUE    = "\e[34m"
+    MAGENTA = "\e[35m"
+    CYAN    = "\e[36m"
+    WHITE   = "\e[37m"
+  
+    # Set color by using a string or one of the defined constants. If a third
+    # option is set to true, it also adds bold to the string. This is based
+    # on the Highline implementation and will automatically append CLEAR to the
+    # end of the returned String.
+    #
+    def self.color(text, color, bold=false)
+      return text unless colorize_logging
+      color = self.const_get(color.to_s.upcase) if color.is_a?(Symbol)
+      bold  = bold ? BOLD : ""
+      "#{bold}#{color}#{text}#{CLEAR}"
+    end
+  
+end

data/lib/umlaut/version.rb

@@ -1,3 +1,3 @@
 module Umlaut
-  VERSION = "3.0.0alpha8"
+  VERSION = "3.0.0alpha9"
 end

data/lib/umlaut_configurable.rb

@@ -96,7 +96,7 @@ module UmlautConfigurable
     
       # A pretty typical direct-linking setup, excludes queries that come
       # from citation linker/azlist/opensearch from direct linking. 
-      # AppConfig::Base.skip_resolve_menu = {:service_types => ['fulltext'],
+      # skip_resolve_menu  {:service_types => ['fulltext'],
       #:services=>['JH_SFX'], :excluded_rfr_ids => ["info:sid/sfxit.com:citation",
       #'info:sid/umlaut.code4lib.org:citation',
       #'info:sid/umlaut.code4lib.org:azlist',  
@@ -167,7 +167,7 @@ module UmlautConfigurable
         sfx_az_profile "default"    
         
         # Use your own custom search view? mention it here.   
-        #search_view = "my_search"
+        #search_view  "my_search"
         
         # can set to "_blank" etc. 
         result_link_target nil
@@ -190,21 +190,21 @@ module UmlautConfigurable
         # urls with urls from catalog. But sometimes it misses some, or
         # alternate hostnames for some. Regexps matching against
         # urls can be included here. Eg,
-        # AppConfig::Base.additional_sfx_controlled_urls = [
+        # additional_sfx_controlled_urls  [
         #    %r{^http://([^\.]\.)*pubmedcentral\.com}
         #  ]    
-        additional_sfx_controlled_urls = []
+        additional_sfx_controlled_urls  []
         
         # "web.archive.org" is listed in SFX, but that causes suppression
         # of MARC 856 tags from our catalog pointing to archive.org, which are
         # being used for some digitized books. We'd like to ignore that archive.org
         # is in SFX. Same for netlibrary.
-        #sfx_load_ignore_hosts = [/.*\.archive\.org/, /www\.netlibrary\.com/, 'www.loc.gov']
-        sfx_load_ignore_hosts = []        
+        #sfx_load_ignore_hosts  [/.*\.archive\.org/, /www\.netlibrary\.com/, 'www.loc.gov']
+        sfx_load_ignore_hosts  []        
       end
     
       # Output timing of service execution to logs
-      log_service_timing = true if Rails.env == "development"
+      log_service_timing (Rails.env == "development")
       
       
       #####

data/test/dummy/config/environments/development.rb

@@ -31,4 +31,13 @@ Dummy::Application.configure do
 
   # Expands the lines which load the assets
   config.assets.debug = true
+  
+  # Raise exception on mass assignment protection for Active Record models
+  config.active_record.mass_assignment_sanitizer = :strict
+ 
+  # Log the query plan for queries taking more than this (works
+  # with SQLite, MySQL, and PostgreSQL)
+  # This seems to not work right in multi-threaded AR use as we're doing,
+  # leave off, at least for now. 
+  #config.active_record.auto_explain_threshold_in_seconds = 0.5
 end

data/test/dummy/config/environments/test.rb

@@ -36,4 +36,7 @@ Dummy::Application.configure do
 
   # Print deprecation notices to the stderr
   config.active_support.deprecation = :stderr
+  
+  # Raise exception on mass assignment protection for Active Record models
+  config.active_record.mass_assignment_sanitizer = :strict  
 end

data/test/dummy/out

@@ -0,0 +1,5 @@
+[2012-01-12 15:01:37] INFO  WEBrick 1.3.1
+[2012-01-12 15:01:37] INFO  ruby 1.8.7 (2011-06-30) [x86_64-linux]
+[2012-01-12 15:01:37] INFO  WEBrick::HTTPServer#start: pid=10034 port=3000
+[2012-01-12 15:05:11] INFO  going to shutdown ...
+[2012-01-12 15:05:11] INFO  WEBrick::HTTPServer#start done.

data/test/dummy/tmp/cache/assets/{DF1/AA0/sprockets%2Fedd37b0fdb6157fccb2297a8a10acc90 → CAA/EB0/sprockets%2F1d179210e8b76f1ea63c802688a015e4}

@@ -1,15 +1,4 @@
-o: ActiveSupport::Cache::Entry	:@compressedF:@expires_in0:@created_atf1325695617.852983:@value{I"
-class:EFI"BundledAsset;
-FI"id;
-F"%fc5b3729d62948fd14a866ac23320895I"logical_path;
-F"jquery.jsI"
-F"h/home/rochkind/.rvm/gems/ruby-1.9.3-p0/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery.jsI"content_type;
-FI"application/javascript;
-FI"
-mtime;
-FI"2012-01-03T17:59:30-05:00;
-FI"	body;
-FI"��/*!
+o: ActiveSupport::Cache::Entry	:@compressedF:@value{"asset_paths["r/home/rochkind/.rvm/gems/ruby-1.8.7-p352@umlaut3/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery.js"	body"��/*!
  * jQuery JavaScript Library v1.7.1
  * http://jquery.com/
  *
@@ -9276,13 +9265,7 @@ if ( typeof define === "function" && define.amd && define.amd.jQuery ) {
 
 
 })( window );
-;
-FI"asset_paths;
-F["h/home/rochkind/.rvm/gems/ruby-1.9.3-p0/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery.jsI"dependency_paths;
-F[{I"	path;
-F"h/home/rochkind/.rvm/gems/ruby-1.9.3-p0/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery.jsI"
-mtime;
-FIu:	Time
-T:offseti���I"hexdigest;
-F"%683679b346b30943f8589b640966202aI"
-F"%46dde6621c301f4928e3b34efee9e3b5
+"content_type"application/javascript"
+class"BundledAsset"
+mtime"2011-12-06T17:02:51-05:00"
+mtimeu:	Time

data/test/dummy/tmp/cache/assets/D54/ED0/sprockets%2F71c9fa01091d432b131da3bb73faf3d4

@@ -1,6 +1,4 @@
-o: ActiveSupport::Cache::Entry	:@compressedF:@expires_in0:@created_atf1325695617.8294442:@value{	I"length:EFi<I"digest;
-F"%2893cd4a723c50a374d18513d6308b0fI"source;
-FI"</* TODO: Clean up this CSS a LOT to make it easier for local apps to
+o: ActiveSupport::Cache::Entry	:@compressedF:@value{	"
  * selectively override. Quite likely turn it into one or more sass-ified .scss
  * in ./umlaut, with "umlaut.css" being a manifest file that includes
  * them in a way that can be over-ridden. See Blacklight. 
@@ -872,6 +870,4 @@ body.banner {
 
   @import url(http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.16/themes/ui-lightness/jquery-ui.css);
           
-;
-FI"
-F"%366451ff3afe7af8c14d27d7751dbc52
+:@created_atf1326403366.3184049`�:@expires_in0

data/test/dummy/tmp/cache/assets/{DD0/140/sprockets%2F1656d1d8f4c13fe42aff6553c6cdcda7 → D7A/E40/sprockets%2F9622ffcc499a57627cd1bb18fe31b8e4}

@@ -1,15 +1,4 @@
-o: ActiveSupport::Cache::Entry	:@compressedF:@expires_in0:@created_atf1325695617.864653:@value{I"
-class:EFI"BundledAsset;
-FI"id;
-F"%24d1e7408f551c742179d2c0a6103dcfI"logical_path;
-F"jquery-ui.jsI"
-F"k/home/rochkind/.rvm/gems/ruby-1.9.3-p0/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery-ui.jsI"content_type;
-FI"application/javascript;
-FI"
-mtime;
-FI"2012-01-03T17:59:30-05:00;
-FI"	body;
-FI"�/*!
+o: ActiveSupport::Cache::Entry	:@compressedF:@value{"asset_paths["u/home/rochkind/.rvm/gems/ruby-1.8.7-p352@umlaut3/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery-ui.js"	body"�/*!
  * jQuery UI 1.8.16
  *
  * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
@@ -11777,13 +11766,7 @@ $.effects.transfer = function(o) {
 };
 
 })(jQuery);
-;
-FI"asset_paths;
-F["k/home/rochkind/.rvm/gems/ruby-1.9.3-p0/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery-ui.jsI"dependency_paths;
-F[{I"	path;
-F"k/home/rochkind/.rvm/gems/ruby-1.9.3-p0/gems/jquery-rails-1.0.19/vendor/assets/javascripts/jquery-ui.jsI"
-mtime;
-FIu:	Time
-T:offseti���I"hexdigest;
-F"%e230f739946310b8a860334c61f3c9a4I"
-F"%46dde6621c301f4928e3b34efee9e3b5
+"content_type"application/javascript"
+class"BundledAsset"
+mtime"2011-12-06T17:02:51-05:00"
+mtimeu:	Time