revs-utils 1.0.8

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NGY1MmUzYTdjMjU1OTI2MGU5NzA0YTIzZDVkOGY0Nzg1OWYwMzZjOA==
+  data.tar.gz: !binary |-
+    ZjQ0OWRiOTlmZGQ2YzY1YjU4Zjc1NWU3MWIyODU4NjRiMTg4Nzg5MQ==
+SHA512:
+  metadata.gz: !binary |-
+    Y2RkOTIyYTVhNmRlODRmYWViNzMwMzkyNGM3MjVmYTc1MzNjYWMyMDgyZDk0
+    OTNiYjE2NjQ2YzJiMGY2NjA0OGNjY2UzZDdhZTc0ZWRhMjhkNWRlYmY5ZTdi
+    M2I5NzJhMjY2OTMyYTg3NDYyZTFiYWQzYTQ3MjhiZDg5NjMxYjA=
+  data.tar.gz: !binary |-
+    MGNhMDEwMTM5Y2Q3YmNiODcxMTU5NzI3MjU5MTZkYjY4YWI1YmZmNWZhMzA0
+    MzhjMWY0ZjExZjg3MTdlNTBiMjJjMGVjYjhhZjk5MGIwOGYxMDBhOWU1MzEz
+    ZDUwNWNmOGU4YzEyNmI0NjY0MzM4OTEwYzgzNjJiOTVhMTQzMGI=

data/.gitignore

@@ -0,0 +1,10 @@
+.rvmrc
+*.gem
+.DS_Store
+.bundle
+doc/*
+log/*
+pkg/*
+tags
+tmp
+.yardoc/*

data/.rvmrc.example

@@ -0,0 +1 @@
+rvm use 1.9.3@assembly-image --create

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+source "http://sul-gems.stanford.edu"
+
+# Specify your gem's dependencies in revs-utils.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,75 @@
+PATH
+  remote: .
+  specs:
+    revs-utils (1.0.8)
+      actionpack (~> 3)
+      countries (= 0.9.2)
+      rdf
+
+GEM
+  remote: http://rubygems.org/
+  remote: http://sul-gems.stanford.edu/
+  specs:
+    actionpack (3.2.19)
+      activemodel (= 3.2.19)
+      activesupport (= 3.2.19)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      journey (~> 1.0.4)
+      rack (~> 1.4.5)
+      rack-cache (~> 1.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.2.1)
+    activemodel (3.2.19)
+      activesupport (= 3.2.19)
+      builder (~> 3.0.0)
+    activesupport (3.2.19)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
+    builder (3.0.4)
+    countries (0.9.2)
+      currencies (>= 0.4.0)
+    currencies (0.4.2)
+    diff-lcs (1.2.4)
+    erubis (2.7.0)
+    hike (1.2.3)
+    i18n (0.6.11)
+    journey (1.0.4)
+    lyberteam-gems-devel (1.0.1)
+      rake (>= 0.8.7)
+      rest-client
+    mime-types (2.0)
+    multi_json (1.10.1)
+    rack (1.4.5)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rake (10.1.0)
+    rdf (1.1.4.1)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.3)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    sprockets (2.2.2)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    tilt (1.4.1)
+    yard (0.8.7.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  lyberteam-gems-devel (> 1.0.0)
+  revs-utils!
+  rspec (~> 2.6)
+  yard

data/LICENSE

@@ -0,0 +1,14 @@
+#############################################################################################################
+# Copyright (c) 2013-2014 by The Board of Trustees of the Leland Stanford Junior University.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

data/README.rdoc

@@ -0,0 +1,54 @@
+= Revs::Utils
+
+Shared methods and functions used by revs-indexer, pre-assembly and bulk metadata loading code.
+
+== Releases
+- <b>0.0.1</b> Initial release
+- <b>0.0.3</b> Add collection name cleaning method
+- <b>0.0.5</b> Lock down the version of countries gem to avoid problems
+- <b>0.0.5</b> Lock down the version of countries gem to avoid problems
+- <b>0.0.7</b> Don't use this version, it uses the wrong method of Dir.pwd to find the root directory for loading assets.
+- <b>0.0.8</b> Add in helper functions to check the .csv files for registration and metadata updates
+- <b>0.0.9</b> Add in a clean marque function
+- <b>1.0.0</b> Make format fixing case insensitive; add a method for loading CSV files with UTF-8 compliance
+- <b>1.0.1</b> Remove a couple methods from Revs code and update the methods to make them consistent in gem
+- <b>1.0.2 and 1.0.3</b> Make invalid two digit years in dates return as false
+- <b>1.0.4</b> Revs-Utils now assumes .csv files are in UTF-8 format instead of Latin-1
+- <b>1.0.5</b> Parse location using commas as well as pipes
+- <b>1.0.6</b> Add some more conditions to CSV header checks
+- <b>1.0.7</b> Label column needs to be there but does not need to have a value to register
+- <b>1.0.8</b> Update clean_collection_name method to deal with other possible names
+
+== Running tests
+
+rake spec
+
+== Release the gem to the gemserver
+
+1. Bump the version number in lib/revs-utils/version.rb
+2. Add to the release notes in this readme.
+3. Ensure the tests pass.
+4. Git commit and push
+5. rake dlss_release
+
+== Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'revs-utils'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install revs-utils
+
+== Updating LC Automobile Terms in Gem
+
+1. Check out Gem code.
+2. CD into Gem directory
+3. ruby bin/revs_lc_automobile_terms.rb 
+4. Update Gem in git, bump version number and rake dlss_release
+

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+
+require 'dlss/rake/dlss_release'
+Dlss::Release.new
+
+require 'rspec/core/rake_task'
+
+desc "Run specs"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,16 @@
+#! /usr/bin/env ruby
+
+require 'rubygems'
+require 'irb'
+
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+
+
+class RevsUtils
+  
+  include Revs::Utils
+  extend Revs::Utils
+  
+end
+
+IRB.start

data/bin/revs_lc_automobile_terms.rb

@@ -0,0 +1,26 @@
+# This file will generate a hash of LC more specific terms from the to-level term defined below (currently "Automobiles").
+# It then dumps this hash to a file, so it can be loaded with each pre-assembly run and used when generating Revs Descriptive Metadata
+# This method can be run periodically to refresh the list of terms.  It will generate a new file in the "lib/pre_assembly/project" folder
+# which can be updated in git.
+
+# Peter Mangiafico
+# May 16, 2013
+
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+require 'rdf'
+require 'rdf/ntriples'
+
+automobile_term='http://id.loc.gov/authorities/subjects/sh85010201' # the top-level LC term to get RDF for, "Automobiles"
+term_predicate='http://www.w3.org/2004/02/skos/core#prefLabel' # the predicate which tells us when we have a term defined 
+
+results={} # the hash we will write with the terms and their LC URLs
+
+RDF::Reader.open("#{automobile_term}.nt") do |reader|
+  reader.each_statement do |statement|
+    if statement.predicate.to_s.strip == term_predicate
+      results.merge!({statement.object.to_s=>statement.subject.to_s})
+    end
+  end
+end
+
+File.open(REVS_LC_TERMS_FILENAME, "wb") {|f| Marshal.dump(results, f)}

data/bin/run_all_tests

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+bundle exec rspec spec

data/config/boot.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+project_root = File.expand_path(File.dirname(__FILE__) + '/..')
+# Load config for current environment.
+$LOAD_PATH.unshift(project_root + '/lib')
+require 'revs-utils'
+

data/config/manifest_headers.yml

@@ -0,0 +1,34 @@
+register:
+  label: label
+  sourceid: sourceid
+  filename: filename
+metadata:
+  marque: marque
+  model: model
+  people: people
+  entrant: entrant
+  photographer: photographer
+  current_owner: current_owner
+  venue: venue
+  track: track
+  event: event
+  location: location
+  state: state
+  country: country
+  city: city
+  date: date
+  year: year
+  description: description
+  model_year: model_year
+  group_or_class: group_or_class
+  race_data: race_data
+  metadata_sources: metadata_sources
+  vehicle_markings: vehicle_markings
+  inst_notes: inst_notes
+  prod_notes: prod_notes
+  has_more_metadata: has_more_metadata
+  hide: hide
+  format: format
+  collection_name: collection_name
+  
+  

data/lib/revs-utils/version.rb

@@ -0,0 +1,5 @@
+module Revs
+  module Utils
+    VERSION = "1.0.8"
+  end
+end

data/lib/revs-utils.rb

@@ -0,0 +1,299 @@
+# encoding: UTF-8
+
+require "revs-utils/version"
+require "countries"
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/hash'
+require 'csv'
+
+PROJECT_ROOT = File.expand_path(File.dirname(__FILE__) + '/..')
+
+
+REVS_LC_TERMS_FILENAME=File.join(PROJECT_ROOT,'files','revs-lc-marque-terms.obj')
+REVS_MANIFEST_HEADERS_FILEPATH = File.join(PROJECT_ROOT,'config',"manifest_headers.yml")
+REGISTER = "register"
+METADATA = "metadata"
+
+
+module Revs
+  module Utils
+    
+      
+      # a hash of LC Subject Heading terms and their IDs for linking for "Automobiles" http://id.loc.gov/authorities/subjects/sh85010201.html
+      # this is cached and loaded from disk and deserialized back into a hash for performance reasons, then stored as a module
+      # level constant so it can be reused throughout the pre-assembly run as a constant
+      #  This cached set of terms can be re-generated with "ruby devel/revs_lc_automobile_terms.rb"
+      AUTOMOBILE_LC_TERMS= File.open(REVS_LC_TERMS_FILENAME,'rb'){|io| Marshal.load(io)} if File.exists?(REVS_LC_TERMS_FILENAME)
+      REVS_MANIFEST_HEADERS_FILE = File.open(REVS_MANIFEST_HEADERS_FILEPATH)
+      REVS_MANIFEST_HEADERS = YAML.load( REVS_MANIFEST_HEADERS_FILE)
+      
+      
+      def get_manifest_section(section)
+        return REVS_MANIFEST_HEADERS[section]
+      end
+      
+      def manifest_headers_file()
+        return REVS_MANIFEST_HEADERS_FILE
+      end
+      
+      def manifest_headers_path()
+        return MAINFEST_HEADERS_FILEPATH
+      end
+      
+      def manifest_register_section_name()
+        return REGISTER
+      end
+      
+      def manifest_metadata_section_name()
+        return METADATA
+      end
+      
+      def read_csv_with_headers(file)
+        # load CSV into an array of hashes, allowing UTF-8 to pass through, deleting blank columns
+        #file_contents = IO.read(file).force_encoding("ISO-8859-1").encode("utf-8", replace: nil) 
+        file_contents = IO.read(file)
+        csv = CSV.parse(file_contents, :headers => true)
+        return csv.map { |row| row.to_hash.with_indifferent_access }
+      end
+      
+      #Pass this function a list of all CSVs containing metadata for files you are about to register and it will ensure each sourceid is unique 
+      def unique_source_ids(file_paths)
+        files = Array.new
+        file_paths.each do |fp|
+          files << read_csv_with_headers(fp)
+        end
+        
+        sources = Array.new
+        files.each do |file|
+          file.each do |row|
+            #Make sure the sourcid and filename are the same
+            fname = row[get_manifest_section(REGISTER)['filename']].chomp(File.extname(row[get_manifest_section(REGISTER)['filename']]))
+            return false if row[get_manifest_section(REGISTER)['sourceid']] != fname
+            sources << row[get_manifest_section(REGISTER)['sourceid']]
+          end
+          
+          
+         
+        end
+        return sources.uniq.size == sources.size
+      
+      end
+      
+      
+      #Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in
+      def valid_to_register(file_path)
+        
+        file = read_csv_with_headers(file_path)
+        #Make sure all the required headers are there
+        return false if not get_manifest_section(REGISTER).values-file[0].keys == []
+        
+        #Make sure all files have entries for those required headers
+        file.each do |row|
+          get_manifest_section(REGISTER).keys.each do |header| # label should be there as a column but does not always need a value
+            return false if header.downcase !='label' && row[header].blank? #Alternatively consider row[header].class != String or row[header].size <= 0
+          end
+        end
+       return true
+      end
+      
+      #Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in.  
+      def valid_for_metadata(file_path)
+        file = read_csv_with_headers(file_path)
+        file_headers=file[0].keys
+        #The file doesn't need to have all the metadata values, it just can't have headers that aren't used for metadata or registration
+        if file_headers.include?('date') && file_headers.include?('year') # can't have both date and year 
+          return false
+        elsif file_headers.include?('location') && file_headers.include?('state') && file_headers.include?('city') && file_headers.include?('country') # can't have both location and the specific fields
+          return false
+        else
+          return file_headers-get_manifest_section(METADATA).values-get_manifest_section(REGISTER).values == []
+        end
+      end
+
+      def clean_collection_name(name)
+        return "" if name.blank? || name.nil?
+        name=name.to_s
+        name.gsub!(/\A(the )/i,'')
+        name.gsub!(/( of the revs institute)\z/i,'')
+        name.gsub!(/( of the revs institute for automotive research)\z/i,'')
+        name.gsub!(/( of the revs institute for automotive research, inc)\z/i,'')
+        name.gsub!(/( of the revs institute for automotive research, inc.)\z/i,'')
+        return name.strip
+      end
+
+      def clean_marque_name(name)
+        return "" if name.blank? || name.nil?
+        name=name.to_s
+        name.gsub!(/(automobiles)\z/i,'')
+        name.gsub!(/(automobile)\z/i,'')
+        return name.strip
+      end
+            
+      def parse_location(row, location)
+        row[location].split(/[,|]/).reverse.each do |local|
+          country = revs_get_country(local)
+          city_state = revs_get_city_state(local) 
+          row['country'] = country.strip if country 
+          if city_state
+            row['state'] = revs_get_state_name(city_state[1].strip)
+            row['city'] = city_state[0].strip
+          end
+          if not city_state and not country
+            row['city_section'] = local
+          end
+        end
+
+        return row
+      end
+
+      def revs_check_format(format)
+        return revs_check_formats([format]).first
+      end
+      
+      # check the incoming format and fix some common issues
+      def revs_check_formats(format)
+        known_fixes = {"black-and-white negative"=>"black-and-white negatives",
+                       "color negative"=>"color negatives",
+                       "slides/color transparency"=>"color transparencies",
+                       "color negatives/slides"=>"color negatives",
+                       "black-and-white negative strips"=>"black-and-white negatives",
+                       "color transparency"=>"color transparencies",
+                       "slide"=>"slides"
+                     }
+        count = 0 
+        format.each do |f|
+          format[count] = known_fixes[f.downcase] || f.downcase
+          count += 1
+        end
+        return format
+      end
+
+      # lookup the marque sent to see if it matches any known LC terms, trying a few varieties; returns a hash of the term and its ID if match is found, else returns false
+      def revs_lookup_marque(marque)
+        result=false
+        variants1=[marque,marque.capitalize,marque.singularize,marque.pluralize,marque.capitalize.singularize,marque.capitalize.pluralize]
+        variants2=[]
+        variants1.each do |name| 
+          variants2 << "#{name} automobile" 
+          variants2 << "#{name} automobiles"
+        end
+        (variants1+variants2).each do |variant|
+          lookup_term=AUTOMOBILE_LC_TERMS[variant]
+          if lookup_term
+            result={'url'=>lookup_term,'value'=>variant}
+            break
+          end
+        end
+        return result
+      end # revs_lookup_marque
+
+      # check if the string passed is a country name or code -- if so, return the country name, if not a recognized country, return false
+      def revs_get_country(name)
+        name='US' if name=='USA' # special case; USA is not recognized by the country gem, but US is
+        country=Country.find_country_by_name(name.strip) # find it by name
+        code=Country.new(name.strip) # find it by code
+        if country.nil? && code.data.nil? 
+          return false
+        else
+          return (code.data.nil? ? country.name : code.name)
+        end
+      end # revs_get_country
+
+      # parse a string like this: "San Mateo (Calif.)" to try and figure out if there is any state in there; if found, return the city and state as an array, if none found, return false
+      def revs_get_city_state(name)
+        state_match=name.match(/[(]\S+[)]/)
+        if state_match.nil?
+          return false
+        else
+          first_match=state_match[0]
+          state=first_match.gsub(/[()]/,'').strip # remove parens and strip
+          city=name.gsub(first_match,'').strip # remove state name from input string and strip
+          return [city,state]
+        end
+      end # revs_get_city_state
+
+      # given an abbreviated state name (e.g. "Calif." or "CA") return the full state name (e.g. "California")
+      def revs_get_state_name(name)
+        test_name=name.gsub('.','').strip.downcase
+        us=Country.new('US')
+        us.states.each do |key,value|
+          if value['name'].downcase.start_with?(test_name) || key.downcase == test_name
+            return value['name']
+            break
+          end
+        end
+        return name
+      end # revs_get_state_name
+
+
+      # tell us if the string passed is a valid year
+      def is_valid_year?(date_string,starting_year=1800)
+        date_string.to_s.strip.scan(/\D/).empty? and (starting_year..Date.today.year).include?(date_string.to_i)
+      end
+
+      # tell us if the string passed is in is a full date of the format M/D/YYYY, and returns the date object if it is valid
+      def get_full_date(date_string)
+        begin
+          date_obj=Date.strptime(date_string.gsub('-','/').delete(' '), '%m/%d/%Y')
+          return (is_valid_year?(date_obj.year.to_s) ? date_obj : false)
+        rescue
+          false
+        end
+      end
+
+      # given a string with dates separated by commas, split into an array
+      # also, parse dates like "195x" and "1961-62" into all dates in that range
+      def parse_years(date_string)
+        date_string.delete!(' ')
+        if date_string.include?('|')
+          result=date_string.split('|')
+        else
+          result=date_string.split(',')
+        end
+        years_to_add=[]
+        result.each do |year|
+
+          if year.scan(/[1-2][0-9][0-9][0-9][-][0-9][0-9]/).size > 0 # if we have a year that looks like "1961-62" or "1961-73", lets deal with it turning it into [1961,1962] or [1961,1962,1963,1964,1965,1966,1967...etc]
+            start_year=year[2..3]
+            end_year=year[5..6]
+            stem=year[0..1] 
+            for n in start_year..end_year
+              years_to_add << "#{stem}#{n}"
+            end
+          elsif year.scan(/[1-2][0-9][0-9][0-9][-][1-9]/).size > 0 # if we have a year that lloks like "1961-2" or "1961-3", lets deal with it turning it into [1961,1962] or [1961,1962,1963]
+            start_year=year[3..3]
+            end_year=year[5..5]
+            stem=year[0..2]
+            for n in start_year..end_year
+              years_to_add << "#{stem}#{n}"
+            end
+          end
+
+          if year.scan(/[1-2][0-9][0-9][0](('s)|s)/).size > 0 || year.scan(/[1-2][0-9][0-9][x_]/).size > 0 # if we have a year that looks like "195x", let's deal with it by turning it into [1950,1951,1952..etc]
+            result.delete(year) # first delete the year itself from the list
+            stem=year[0..2] # next get the stem, and expand into the whole decade
+            %w{0 1 2 3 4 5 6 7 8 9}.each {|n| years_to_add << "#{stem}#{n}"} # add each year in that decade to the output array
+          end
+
+          if year.scan(/[1-2][0-9][0-9][0-9][-][1-2][0-9][0-9][0-9]/).size > 0 # if we have a year that lloks like "1961-1962" or "1930-1955", lets deal with it turning it into [1961,1962] or [1961,1962,1963]
+            start_year=year[0..3]
+            end_year=year[5..8]
+            if end_year.to_i - start_year.to_i < 10 # let's only do the expansion if we don't have some really large date range, like "1930-1985" .. only ranges less than 9 years will be split into separate years
+              for n in start_year..end_year
+                years_to_add << n
+              end
+            end
+          end
+
+        end
+
+        result = result.uniq
+        result.each do |year|
+          result.delete(year) if not year.scan(/\A[1-2][0-9][0-9][0-9]\z/).size == 1  #If it doesn't fit the format #### remove it
+        end
+        return result.concat(years_to_add).uniq.sort
+
+      end
+
+  end
+end

data/revs-utils.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'revs-utils/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "revs-utils"
+  gem.version       = Revs::Utils::VERSION
+  gem.authors       = ["Peter Mangiafico"]
+  gem.email         = ["pmangiafico@stanford.edu"]
+  gem.description   = "Shared methods and functions used by revs-indexer, pre-assembly and bulk metadata loading code."
+  gem.summary       = "Shared methods and functions used by revs-indexer, pre-assembly and bulk metadata loading code."
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_dependency "countries", "0.9.2"
+  gem.add_dependency "rdf"
+  gem.add_dependency "actionpack", '~> 3'
+  
+  gem.add_development_dependency "rspec", "~> 2.6"
+  gem.add_development_dependency "lyberteam-gems-devel", "> 1.0.0"
+  gem.add_development_dependency "yard"
+  
+end