umlaut 3.0.5 → 3.1.0.pre1

data/lib/generators/umlaut/install_generator.rb

@@ -69,6 +69,18 @@ module Umlaut
 #  password:
 #  pool: 5
 #  encoding: utf8
+# 
+# UMLAUT: for Sfx4Solr functionality.  If you're not using SFX 4 indexed in 
+# Solr via Sunspot, feel free to ignore this.
+#sfx4_global:
+#  adapter: mysql2
+#  host: my_sfx_host.u.edu
+#  port: 3310 # 3310 is defualt SFX embedded mysql port
+#  database: sfxglb41 # or other sfx global db
+#  username:
+#  password:
+#  pool: 5
+#  encoding: utf8
 # 
           eos
         end

data/lib/tasks/umlaut.rake

@@ -6,40 +6,24 @@ namespace :umlaut do
 
     desc "Loads sfx_urls from SFX installation. SFX mysql login needs to be set in config."
     task :load_sfx_urls => :environment do
-
-      if SfxDb.connection_configured?
-    
-        puts "Loading SFXUrls via direct access to SFX db."
-        #sfxlcl41.TARGET_SERVICE
-        # Check if we have an SFX3 schema, or if not use SFX4
-        sfx3 = true
-        begin
-          SfxDb::Object.connection.select_all("SHOW FIELDS FROM TARGET_SERVICE")
-        rescue ActiveRecord::StatementInvalid
-          sfx3 = false
-        end
-        
-        if sfx3
-          urls = SfxDb::SfxDbBase.fetch_sfx_urls
-        else
-          urls = SearchMethods::Sfx4.fetch_sfx_urls
-        end
-        
+      # Get the configured searcher
+      searcher = UmlautController.umlaut_config.lookup!("search.az_search_method", SearchMethods::Sfx4)
+      if searcher.fetch_urls?
+        puts "Loading SfxUrls (e.g. via direct access to SFX db)."
+        # Grab the urls
+        urls = searcher.fetch_urls
         ignore_urls = UmlautController.umlaut_config.lookup!("sfx.sfx_load_ignore_hosts", [])
-        
         # We only want the hostnames
         hosts = urls.collect do |u|
           begin
-          uri = URI.parse(u)
-          uri.host
+            uri = URI.parse(u)
+            uri.host
           rescue Exception
           end
         end
         hosts.uniq!
-        
         SfxUrl.transaction do
           SfxUrl.delete_all
-          
           hosts.each {|h| SfxUrl.new({:url => h}).save! unless ignore_urls.find {|ignore| ignore === h }}      
         end
       else

data/lib/umlaut.rb

@@ -3,28 +3,12 @@ require 'umlaut/routes'
 # not sure why including openurl gem doesn't do the require, but it
 # seems to need this. 
 require 'openurl'
+require 'bootstrap-sass'
 
 module Umlaut
   class Engine < Rails::Engine
     engine_name "umlaut"
     
-    # we store some things in non-standard subdirs, add em to path.
-    #
-    # We EAGER load em all to try and handle threading issues. 
-    
-    config.autoload_paths << File.join(self.root, "lib", "referent_filters")
-    config.eager_load_paths << File.join(self.root, "lib", "referent_filters")
-    
-    config.autoload_paths << File.join(self.root, "lib", "service_adaptors")
-    config.eager_load_paths << File.join(self.root, "lib", "service_adaptors")
-    
-    # Ane make 'lib' dir auto-loaded, because we have legacy Rails2 code
-    # that assumes it.
-    config.autoload_paths << File.join(self.root, "lib")
-    # Sadly including eager_load_paths here makes weird things happen, apparently
-    # I don't entirely understand what's going on. 
-    #config.eager_load_paths << File.join(self.root, "lib")
-    
     # We need the update_html.js script to be available as it's own
     # JS file too, not just compiled into application.js, so we can
     # deliver it to external apps using it (JQuery Content Utility).
@@ -41,9 +25,5 @@ module Umlaut
     #initializer("#{engine_name}.patch_connection_pool", :before => "active_record.initialize_database") do |app|
       load File.join(self.root, "active_record_patch", "connection_pool.rb")
     #end
-    
-         
-    
-    
   end
 end

data/lib/umlaut/version.rb

@@ -1,3 +1,3 @@
 module Umlaut
-  VERSION = "3.0.5"
+  VERSION = "3.1.0.pre1"
 end

data/test/README.md

@@ -0,0 +1,75 @@
+#Testing Umlaut
+
+Testing coverage in Umlaut is not yet comprehensive, but is continually improving.  
+There is more about this topic on the [Umlaut Wiki](https://github.com/team-umlaut/umlaut/wiki/Developing)
+
+##Testing Principles
+
+When writing tests for Umlaut, there are a few things you should keep in mind.
+
+### Dummy app
+
+We used rails 3.1 `rails plugin new` generator to create a gem plugin skeleton for Umlaut. 
+One thing this did was create a `./test/dummy` dummy app in umlaut source. 
+This is meant for running tests against. Tests you write in `/test/*` will run against the dummy app. 
+The dummy app can have configuration customized (with those changes committed) if needed to exercise certain features 
+(although changing config 'live' and temporarily in a test itself would be preferable where feasible). 
+The dummy app doesn't have a `database.yml`, you'll need to add one pointing at a local mysql 
+(not sqlite3, umlaut won't work against sqlite3). 
+
+### Test::Unit/Minitest please
+
+Please use straight Test::Unit and/or Minitest.  Please do not use rspec, cucumber, etc. Put tests in `./test`.  There's already a `rake test` to run all tests.
+
+### Tests should pass locally for everyone
+
+Tests should pass for everyone with a straight umlaut checkout without modifying any files but adding a database.yml. 
+
+Please don't add tests that raise or fail without a private API key or access to a private server. 
+We've started to use the [VCR](https://github.com/myronmarston/vcr) gem to record HTTP responses to provide deterministic testing results.
+VCR "cassettes" are committed to the repo in `./test/vcr_cassettes/<module>`.
+To run VCR tests, you can leverage the `TestWithCassette` support module following the example below.
+
+    class GoogleBookSearchTest < ActiveSupport::TestCase  
+      extend TestWithCassette
+
+      # Use VCR to provide a deterministic GBS search. 
+      test_with_cassette("frankenstein by OCLC number", :google_book_search) do
+        hashified_response = @gbs_default.do_query('OCLC2364071', requests(:frankenstein))
+        assert_not_nil hashified_response
+        assert_not_nil hashified_response["totalItems"]
+        assert_operator hashified_response["totalItems"], :>, 0 
+      end
+    end
+    
+We still want to figure out a way for someone with access to the necessary third party platforms to 
+choose to run against 'live' too (for only specified services?) 
+(and then commit the new versions of the responses back to repo if they want). 
+There's some stuff to figure out, yeah.
+
+If you really must write tests that require private api keys or access to private servers to pass, 
+then I guess that's better than no tests at all, but figure out a way to:
+
+* Clearly document where the developer should put those private api keys or local server URLs to get tests to pass. 
+* Keep from interfering with running the rest of the test suite for a developer who does not have that private information filled out. 
+  Ideally tests that require it would output a clear 'skipped' or 'pending' message 
+  (explaining what needs to be configured to run them_ rather than failing.)
+  **In no cases should they raise exceptions in such a way that prevents the rest of the test suite from continuing.**
+
+### Test should pass on Travis, too
+
+[Travis](http://travis-ci.org) is a continuous integration service for the open source community that Umlaut leverages to run tests.
+One reason to use Test::Unit/Minitest is that [Travis](https://travis-ci.org/#!/team-umlaut/umlaut) will automatically run your tests.
+
+Travis creates the necessary MySQL databases and runs tests against the dummy app based on the database configuration in
+`./test/dummy/config/travis_database.yml`.
+
+One specific test Travis runs is for SFX search functionality.  
+Travis uses the configuration in `./test/dummy/config/travis_database.yml` to create a mock instance of an 
+SFX global and local database, migrates the SFX database schema and the test populates the database with 
+deterministic SFX Object and AZ Title fixtures.  
+It tests basic search functionality and serves as a sanity check to make sure SFX searching is working as expected.
+This test make @jrochkind very nervous, and **you probably shouldn't ever run this test locally**.  The migrations and test code
+only run in the case that the SFX database configuration is specified as a 'mock instance' in order to prevent overwriting
+a real SFX database. (Yet another reason the Umlaut user that queries SFX should only have read permissions.)
+

data/test/dummy/app/controllers/umlaut_controller.rb

@@ -65,10 +65,10 @@ class UmlautController < ApplicationController
       # and citation entry. 
       search do
         # Is your SFX database connection, defined in database.yml under
-        # sfx_db and used for A-Z searches, Sfx3 or Sfx4?  Other SearchMethods
-        # in addition to SFX direct db may be provided later. 
+        # sfx_db and used for A-Z searches, Sfx4 or do you want to use Sfx4Solr?  
+        # Other SearchMethods in addition to SFX direct db may be provided later. 
         #az_search_method  SearchMethods::Sfx4
-        #az_search_method  SearchMethods::Sfx3
+        #az_search_method  SearchMethods::Sfx4Solr::Local
         
         # When talking directly to the SFX A-Z list database, you may
         # need to set this, if you have multiple A-Z profiles configured

data/test/dummy/config/application.rb

@@ -4,6 +4,9 @@ require 'rails/all'
 
 Bundler.require
 require "umlaut"
+require "sunspot_rails"
+
+require 'jquery-rails' # Oddly neccesary in our dummy app see http://www.ruby-forum.com/topic/2484569#1021529
 
 module Dummy
   class Application < Rails::Application

data/test/dummy/config/sunspot.yml

@@ -0,0 +1,20 @@
+# Dummy config for testing sfx4solr search module.
+# Solr is not running on localhost. Instead we use a
+# VCR cassette to test the expected result from Solr.
+production:
+  solr:
+    hostname: localhost
+    port: 8983
+    log_level: WARNING
+
+development:
+  solr:
+    hostname: localhost
+    port: 8982
+    log_level: INFO
+
+test:
+  solr:
+    hostname: localhost
+    port: 8981
+    log_level: WARNING

data/test/dummy/config/travis_database.yml

@@ -0,0 +1,96 @@
+# This file is used by travis to provide continuous
+# integration for umlaut.  It contains "mock instances"
+# of the SFX4 Global and Local databases to provide
+# sanity checks for SFX searching.  The file is 
+# renamed to database.yml as part of the travis before script 
+# hook.
+# 
+# MySQL.  Versions 4.1 and 5.0 are recommended.
+# 
+# Install the MYSQL driver
+#   gem install mysql2
+#
+# Ensure the MySQL gem is defined in your Gemfile
+#   gem 'mysql2'
+#
+# And be sure to use new-style password hashing:
+#   http://dev.mysql.com/doc/refman/5.0/en/old-client.html
+#
+# UMLAUT: mysql db with mysql2 adapter strongly recommended for Umlaut, in both 
+# production and development. sqlite3 has unclear semantics under threaded 
+# concurrency which umlaut uses, and in many cases simply does not work. 
+#
+# A high (connection) pool setting is also, sadly, generally
+# required at present. 
+#
+# development:
+#   adapter: mysql2
+#   host: 
+#   username:
+#   password:
+#   database:
+#   pool: 30
+
+development:
+  adapter: <%= defined?(JRUBY_VERSION) ? "mysql" : "mysql2" %>
+  encoding: utf8
+  reconnect: false
+  database: umlaut3_development
+  pool: 5
+  username: root
+  password: 
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  adapter: <%= defined?(JRUBY_VERSION) ? "mysql" : "mysql2" %>
+  database: umlaut3_test
+  pool: 5
+  username: root
+  encoding: utf8
+
+production:
+  adapter: <%= defined?(JRUBY_VERSION) ? "mysql" : "mysql2" %>
+  encoding: utf8
+  reconnect: false
+  database: umlaut3
+  pool: 30
+  username: 
+  password: 
+          
+#
+# UMLAUT: for the 'search' functions (A-Z title lookup) to work, you need
+# a direct database connection to the SFX database, under 'sfx_db' key. 
+# You should manually set up a new read-only MySQL account in the SFX db
+# for this purpose, rather than use one of the full-access existing SFX
+# mysql accounts. 
+#
+# In this dummy app, the sfx_db configuation is used for SFX search testing.
+# DO NOT SET THIS FOR ANYTHING LIKE A REAL SFX DATABASE.
+sfx_db:
+  adapter: <%= defined?(JRUBY_VERSION) ? "mysql" : "mysql2" %>
+  database: sfxlcl41_test # or other sfx instance db
+  username: root
+  password: 
+  pool: 5
+  encoding: utf8
+  # :mock_instance is used for SFX search testing.
+  # DO NOT SET :mock_instance TO TRUE FOR ANYTHING LIKE A REAL SFX DATABASE.
+  # If in doubt set :mock_instance => false
+  mock_instance: true
+
+# This is used for SFX search testing.
+# DO NOT SET THIS FOR ANYTHING LIKE A REAL SFX DATABASE.
+sfx4_global:
+  adapter: <%= defined?(JRUBY_VERSION) ? "mysql" : "mysql2" %>
+  database: sfxglb41_test
+  username: root
+  password: 
+  pool: 5
+  encoding: utf8
+  # :mock_instance is used for SFX search testing.
+  # DO NOT SET :mock_instance TO TRUE FOR ANYTHING LIKE A REAL SFX DATABASE.
+  # If in doubt set :mock_instance => false
+  mock_instance: true
+

data/test/dummy/config/umlaut_services.yml

@@ -22,32 +22,17 @@ default:
       
       SFX:
         type: Sfx
-        disabled: true
         display_name: Find It       
-        base_url: YOUR_SFX_BASE_URL
+        base_url: http://sfx.library.edu/local
         priority: 3      
+        roll_up_prefixes:
+          - 'EBSCOHOST_'
+          - 'JSTOR_'
+          - 'PROQUEST_'
+          - 'SPRINGER_LINK_'
+          - 'ELSEVIER_SD_'
+          - 'HIGHWIRE_'
       
-      Primo:
-        type: PrimoService
-        disabled: false
-        display_name: Primo
-        priority: 3      
-        base_url: http://bobcat.library.nyu.edu
-        vid: NYU
-        institution: NYU
-        holding_search_institution: NYU
-        holding_search_text: Search for this title in Primo.
-        suppress_holdings: [ !ruby/regexp '/\$\$LBWEB/', !ruby/regexp '/\$\$LNWEB/', !ruby/regexp '/\$\$LTWEB/', !ruby/regexp '/\$\$LWEB/', !ruby/regexp '/\$\$1Restricted Internet Resources/' ]
-        ez_proxy: !ruby/regexp '/https\:\/\/ezproxy\.library\.nyu\.edu\/login\?url=/'
-        service_types:
-          - holding
-          - holding_search
-          - fulltext
-          - table_of_contents
-          - referent_enhance
-          - cover_image
-
-        
       # First half of Amazon, run in foreground, get metadata and cover images.
       Amazon:
         disabled: true
@@ -107,6 +92,14 @@ default:
         priority: c        
         #api_key: 'YOUR_GBS_API_KEY'
                 
+      DummyService:
+        type: DummyService
+        priority: c      
+        responses:
+          - service_type_value: holding
+            display_text: Library holding
+            url: http://holding.library.edu/DOCID
+
       Ulrichs:
         display_name: "Ulrich's Directory"
         type: UlrichsLink

data/test/dummy/db/migrate/20120927163304_sfx4_global.rb

@@ -0,0 +1,26 @@
+# This is used for SFX search testing.
+# DO NOT USE THIS FOR ANYTHING LIKE A REAL SFX DATABASE.
+class Sfx4Global < ActiveRecord::Migration
+  def connection
+    if sfx4_mock_instance?
+      Sfx4::Global::Base.connection.initialize_schema_migrations_table
+      return Sfx4::Global::Base.connection
+    end
+  end
+
+  def change
+    if sfx4_mock_instance?
+      create_table "KB_OBJECTS", {:id => false} do |t|
+        t.integer "OBJECT_ID", :default => 0, :null => false, :limit => 8
+      end
+      execute "ALTER TABLE KB_OBJECTS ADD PRIMARY KEY (OBJECT_ID);"
+    else
+      puts "Skipping SFX Global migration since SFX global DB specified is not a mock instance."
+    end
+  end
+
+  def sfx4_mock_instance?
+    (ActiveRecord::Base.configurations["sfx4_global"] and
+      ActiveRecord::Base.configurations["sfx4_global"]["mock_instance"])
+  end
+end

data/test/dummy/db/migrate/20120927164040_sfx4_local.rb

@@ -0,0 +1,54 @@
+# This is used for SFX search testing.
+# DO NOT USE THIS FOR ANYTHING LIKE A REAL SFX DATABASE.
+class Sfx4Local < ActiveRecord::Migration
+  def connection
+    if sfx4_mock_instance?
+      Sfx4::Local::Base.connection.initialize_schema_migrations_table
+      return Sfx4::Local::Base.connection
+    end
+  end
+
+  def change
+    if sfx4_mock_instance?
+      create_table "AZ_TITLE", {:id => false} do |t|
+        t.integer "AZ_TITLE_ID", :default => 0, :null => false
+        t.string "AZ_PROFILE", :limit => 100, :null => false
+        t.integer "OBJECT_ID", :default => 0, :null => false, :limit => 8
+        t.string "TITLE_DISPLAY", :limit => 255, :null => false
+        t.string "TITLE_SORT", :limit => 200, :null => false
+        t.string "SCRIPT", :limit => 20, :null => false
+      end
+      execute "ALTER TABLE AZ_TITLE ADD PRIMARY KEY (AZ_TITLE_ID);"
+
+      create_table "AZ_EXTRA_INFO", {:id => false} do |t|
+        t.integer "AZ_EXTRA_INFO_ID", :default => 0, :null => false
+        t.string "AZ_PROFILE", :limit => 100, :null => false
+        t.integer "OBJECT_ID", :default => 0, :null => false, :limit => 8
+        t.text "EXTRA_INFO_XML", :limit => 16777215
+      end
+      execute "ALTER TABLE AZ_EXTRA_INFO ADD PRIMARY KEY (AZ_EXTRA_INFO_ID);"
+
+      create_table "AZ_TITLE_SEARCH", {:id => false} do |t|
+        t.integer "AZ_TITLE_SEARCH_ID", :default => 0, :null => false
+        t.string "AZ_PROFILE", :limit => 100, :null => false
+        t.integer "AZ_TITLE_ID", :default => 0, :null => false
+        t.text "TITLE_SEARCH", :null => false
+      end
+      execute "ALTER TABLE AZ_TITLE_SEARCH ADD PRIMARY KEY (AZ_TITLE_SEARCH_ID);"
+
+      create_table "AZ_LETTER_GROUP", {:id => false} do |t|
+        t.integer "AZ_LETTER_GROUP_ID", :default => 0, :null => false
+        t.integer "AZ_TITLE_ID", :default => 0, :null => false
+        t.string "AZ_LETTER_GROUP_NAME", :limit => 10, :null => false
+      end
+      execute "ALTER TABLE AZ_LETTER_GROUP ADD PRIMARY KEY (AZ_LETTER_GROUP_ID);"
+    else
+      puts "Skipping SFX DB migration since the SFX DB specified is not a mock instance."
+    end
+  end
+
+  def sfx4_mock_instance?
+    (ActiveRecord::Base.configurations["sfx_db"] and
+      ActiveRecord::Base.configurations["sfx_db"]["mock_instance"])
+  end
+end