revs-utils 2.0.7 → 2.0.8

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjgyNzZkYTI2YTNmODVlNTZjOTRkZDNhMWUzNjE3ZjY0NzI5NmNlOQ==
+    MTk1MjU3MmRiNzgwMjNlNmRmMjRjMGQ2NmEwOTIxZTFiNTA3YjEyNQ==
   data.tar.gz: !binary |-
-    N2U1NzkxNjNmNzE0MmQ3OWQ5ZTA1YWMzMTE1M2IwMTNhNTQ1Yzk5OA==
+    NzMzOWNmODZlYjU2NDE2ZmQyZWUwNjgxMjcwMjk5ZTNmYmU5OWQ2Nw==
 SHA512:
   metadata.gz: !binary |-
-    ZmMzODNjM2Q3MTM1NDhlMWNjNjkyZmViNzNmMmZhMzZlNGE1OTUxOGI2YjY5
-    Zjk3MzMwMzMxMjQ0ZWY3YjliNTlkNjE0YjY5YTI1YTY1NDhmNTg2YWZmMzIx
-    M2JhNmM4YTU1YTFhMThiNjgyNTc2YzI5Y2NlZDUxNWU5MjNlYmM=
+    OTkzOTQ5ZmEyNjYxMWJlZTNmOTZkOWZhMjBjOWNmYmI1OGQ4MzM5Y2IzZmE4
+    ZWNhNDI5M2Y2YTUxZmIwMDExM2Q3YWVkYmFjNDg5OTM1ZDVkMGVjOWRjODI3
+    YjM3ZjU3MTg4NWViMGZkODUwYjgyMWU1OWRkYTkzZTJiYmE1MGM=
   data.tar.gz: !binary |-
-    ODVlYTBiNzYzMTY3NDc4N2JmMzVlMjM0NzM4N2VhZDNhNjQwYTk4OGUyNzY1
-    MWY2ZjYzMTZlODFiZmU4N2E5Y2U3NTE1MzQ2ZDJmNTQxOTVmYmQ2NGNlZmVh
-    NWIwNjM5NTk0MGUwMTRiMzVmOWY4N2FmMzQ1N2NmMzY5ZmMyYzg=
+    YTBmMjY5YzIyMzVhOWQ1NjA3NTkxYWNjNDUzZDA3MWIyODRhZmU5NDZhYmY0
+    MTUxZDcyNjlkNzFmNWZmNGJhNzU2OWYzYzRhM2QxNzVhZWUwNGVjM2NiODNk
+    ZDY1NzcxOGNjMWQ1ZDZlODM1YTJjYzgzM2FiZTg2NDE4NmQ4ZTg=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    revs-utils (2.0.6)
+    revs-utils (2.0.8)
       actionpack (~> 4.1.6)
       chronic
       countries (= 0.9.2)
@@ -11,16 +11,16 @@ GEM
   remote: http://rubygems.org/
   remote: http://sul-gems.stanford.edu/
   specs:
-    actionpack (4.1.9)
-      actionview (= 4.1.9)
-      activesupport (= 4.1.9)
+    actionpack (4.1.10)
+      actionview (= 4.1.10)
+      activesupport (= 4.1.10)
       rack (~> 1.5.2)
       rack-test (~> 0.6.2)
-    actionview (4.1.9)
-      activesupport (= 4.1.9)
+    actionview (4.1.10)
+      activesupport (= 4.1.10)
       builder (~> 3.1)
       erubis (~> 2.7.0)
-    activesupport (4.1.9)
+    activesupport (4.1.10)
       i18n (~> 0.6, >= 0.6.9)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
@@ -32,23 +32,28 @@ GEM
       currencies (>= 0.4.0)
     currencies (0.4.2)
     diff-lcs (1.2.5)
+    domain_name (0.5.23)
+      unf (>= 0.0.5, < 1.0.0)
     erubis (2.7.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
     i18n (0.7.0)
     json (1.8.2)
     link_header (0.0.8)
     lyberteam-gems-devel (1.0.1)
       rake (>= 0.8.7)
       rest-client
-    mime-types (2.3)
+    mime-types (2.4.3)
     minitest (5.5.1)
-    netrc (0.7.7)
+    netrc (0.10.3)
     rack (1.5.2)
     rack-test (0.6.3)
       rack (>= 1.0)
-    rake (10.3.2)
-    rdf (1.1.7)
+    rake (10.4.2)
+    rdf (1.1.11)
       link_header (~> 0.0, >= 0.0.8)
-    rest-client (1.7.2)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
     rspec (2.99.0)
@@ -58,11 +63,14 @@ GEM
     rspec-core (2.99.2)
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.99.2)
-    thread_safe (0.3.4)
+    rspec-mocks (2.99.3)
+    thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    yard (0.8.7.4)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    yard (0.8.7.6)
 
 PLATFORMS
   ruby

data/README.rdoc

@@ -18,21 +18,14 @@ Shared methods and functions used by revs-indexer, pre-assembly and bulk metadat
 - <b>1.0.6</b> Add some more conditions to CSV header checks
 - <b>1.0.7</b> Label column needs to be there but does not need to have a value to register
 - <b>1.0.8</b> Update clean_collection_name method to deal with other possible names
-<<<<<<< HEAD
 - <b>2.0.0</b> Updating to use ActionPack 4 for Rails 4 applications.  For Rails 3, continue to use 1.x.y releases.  
 - <b>2.0.1</b> Add more common format corrections
 - <b>2.0.2 and 2.0.3</b> Update valid for metadata method so it is not sensitive to blank or uppercase columns
 - <b>2.0.4</b> Fix issues with year parsing
-=======
-- <b>1.0.9</b> Add more common format corrections
-- <b>1.0.10</b> Update valid for metadata method so it is not sensitive to blank or uppercase columns
-- <b>1.0.11</b> Fix issues with year parsing
-- <b>1.0.12</b> Allow two digit years in year formatting
->>>>>>> 9db8640... Allow two digit years in year formatting
 
 == Running tests
 
-bundle exec rake spec
+bundle exec rspec spec
 
 == Release the gem to the gemserver
 
@@ -48,6 +41,7 @@ gem build revs-utils.gemspec
 
 gem push revs-utils-2.0.1.gem  # substitute actual version number
 
+
 == Installation
 
 Add this line to your application's Gemfile:

data/config/manifest_headers.yml

@@ -30,5 +30,15 @@ metadata:
   hide: hide
   format: format
   collection_name: collection_name
-  
-  
+known_formats: 
+  - black-and-white film
+  - color film
+  - slides
+  - foldouts
+  - pamphlets
+  - oversize items
+  - photographic prints
+  - black-and-white negatives
+  - color negatives
+  - black-and-white transparencies
+  - color transparencies

data/lib/revs-utils/version.rb

@@ -1,5 +1,5 @@
 module Revs
   module Utils
-    VERSION = "2.0.7"
+    VERSION = "2.0.8"
   end
 end

data/lib/revs-utils.rb

@@ -14,7 +14,7 @@ REVS_LC_TERMS_FILENAME=File.join(PROJECT_ROOT,'files','revs-lc-marque-terms.obj'
 REVS_MANIFEST_HEADERS_FILEPATH = File.join(PROJECT_ROOT,'config',"manifest_headers.yml")
 REGISTER = "register"
 METADATA = "metadata"
-
+FORMATS = "known_formats"
 
 module Revs
   module Utils
@@ -28,6 +28,9 @@ module Revs
       REVS_MANIFEST_HEADERS_FILE = File.open(REVS_MANIFEST_HEADERS_FILEPATH)
       REVS_MANIFEST_HEADERS = YAML.load( REVS_MANIFEST_HEADERS_FILE)
       
+      def revs_known_formats
+        get_manifest_section(FORMATS)
+      end
       
       def get_manifest_section(section)
         return REVS_MANIFEST_HEADERS[section]
@@ -67,40 +70,59 @@ module Revs
         sources = Array.new
         files.each do |file|
           file.each do |row|
-            #Make sure the sourcid and filename are the same
+            #Make sure the sourceid and filename are the same
             fname = row[get_manifest_section(REGISTER)['filename']].chomp(File.extname(row[get_manifest_section(REGISTER)['filename']]))
             return false if row[get_manifest_section(REGISTER)['sourceid']] != fname
             sources << row[get_manifest_section(REGISTER)['sourceid']]
-          end
-          
-          
-         
+          end         
         end
         return sources.uniq.size == sources.size
       
       end
-      
-      
+            
       #Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in
       def valid_to_register(file_path)
-        
         file = read_csv_with_headers(file_path)
+        return check_valid_to_register(file)
+      end
+      
+      #Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in.  
+      def valid_for_metadata(file_path)
+        file = read_csv_with_headers(file_path)
+        return check_headers(file)
+      end
+      
+      # pass in csv data and it will tell if you everything is safe to register based on having labels, unique sourceIDs and filenames matching sourceIDs
+      def check_valid_to_register(csv_data)
         #Make sure all the required headers are there
-        return false if not get_manifest_section(REGISTER).values-file[0].keys == []
-        
+        return false if not get_manifest_section(REGISTER).values-csv_data[0].keys == []
+        sources=Array.new
         #Make sure all files have entries for those required headers
-        file.each do |row|
+        csv_data.each do |row|
           get_manifest_section(REGISTER).keys.each do |header| # label should be there as a column but does not always need a value
             return false if header.downcase !='label' && row[header].blank? #Alternatively consider row[header].class != String or row[header].size <= 0
           end
+          fname = row[get_manifest_section(REGISTER)['filename']].chomp(File.extname(row[get_manifest_section(REGISTER)['filename']]))
+          return false if row[get_manifest_section(REGISTER)['sourceid']] != fname
+          sources << row[get_manifest_section(REGISTER)['sourceid']]
         end
-       return true
+        return sources.uniq.size == sources.size
       end
       
-      #Pass this function a CSV file and it will return true if the proper headers are there and each entry has the required fields filled in.  
-      def valid_for_metadata(file_path)
-        file = read_csv_with_headers(file_path)
-        file_headers=file[0].keys.reject(&:blank?).collect(&:downcase)
+      # looks at certain metadata fields in manifest to confirm validity (such as dates and formats)
+      def check_metadata(csv_data)
+        bad_rows=0
+        csv_data.each do |row|
+          valid_date=revs_is_valid_datestring?(row[get_manifest_section(METADATA)['year']] || row[get_manifest_section(METADATA)['date']])
+          valid_format=revs_is_valid_format?(row[get_manifest_section(METADATA)['format']])
+          bad_rows+=1 unless (valid_date && valid_format)
+        end
+        return bad_rows
+      end
+      
+      # pass in csv data from a file read in and it will tell you if the headers are valid
+      def check_headers(csv_data)
+        file_headers=csv_data[0].keys.reject(&:blank?).collect(&:downcase)
         #The file doesn't need to have all the metadata values, it just can't have headers that aren't used for metadata or registration
         if file_headers.include?('date') && file_headers.include?('year') # can't have both date and year 
           return false
@@ -110,7 +132,7 @@ module Revs
           return file_headers-get_manifest_section(METADATA).values-get_manifest_section(REGISTER).values == []
         end
       end
-
+      
       def clean_collection_name(name)
         return "" if name.blank? || name.nil?
         name=name.to_s
@@ -147,11 +169,19 @@ module Revs
         return row
       end
 
+      # checks to see if we have a valid format
+      def revs_is_valid_format?(format)
+        return true if format.nil? || format.blank?
+        formats=format.split("|").collect{|f| f.strip}
+        !formats.collect {|f| revs_known_formats.include?(f)}.uniq.include?(false)
+      end
+      
+      # check a single format and fix some common issues
       def revs_check_format(format)
         return revs_check_formats([format]).first
       end
       
-      # check the incoming format and fix some common issues
+      # check the incoming array of formats and fix some common issues
       def revs_check_formats(format)
         known_fixes = {"black-and-white negative"=>"black-and-white negatives",
                        "color negative"=>"color negatives",
@@ -163,7 +193,8 @@ module Revs
                        "black and white negative"=>"black-and-white negatives",
                        "black and white negatives"=>"black-and-white negatives",
                        "color transparency"=>"color transparencies",
-                       "slide"=>"slides"
+                       "slide"=>"slides",
+                       "color transparancies"=>"color transparencies"
                      }
         count = 0 
         format.each do |f|
@@ -236,6 +267,14 @@ module Revs
         date_string.to_s.strip.scan(/\D/).empty? and (starting_year..Date.today.year).include?(date_string.to_i)
       end
 
+      # tell us if the incoming datestring supplied in the manifest column is a valid date, year or list of years
+      def revs_is_valid_datestring?(date_string)
+        return true if date_string.nil? || date_string.empty?
+        is_full_date=(get_full_date(date_string) != false)
+        is_year=!parse_years(date_string).empty?
+        return is_year || is_full_date
+      end
+      
       # tell us if the string passed is in is a full date of the format M/D/YYYY or m-d-yyyy or m-d-yy or M/D/YY, and returns the date object if it is valid
       def get_full_date(date_string)
         begin

data/revs-utils.gemspec

@@ -11,7 +11,8 @@ Gem::Specification.new do |gem|
   gem.description   = "Shared methods and functions used by revs-indexer, pre-assembly and bulk metadata loading code."
   gem.summary       = "Shared methods and functions used by revs-indexer, pre-assembly and bulk metadata loading code."
   gem.homepage      = ""
-
+  gem.license       = "All rights reserved, Stanford University."
+  
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})

data/spec/revs-utils_spec.rb

@@ -61,6 +61,7 @@ describe "Revs-Utils" do
    end
    
    it "should clean up some common format errors from an array" do 
+     @revs.revs_check_formats(['black-and-white negative','color negative','leave alone']).should == ['black-and-white negatives','color negatives','leave alone']
      @revs.revs_check_formats(['black and white','color negative','black-and-white negative']).should == ['black-and-white negatives','color negatives','black-and-white negatives']
    end
 
@@ -98,6 +99,7 @@ describe "Revs-Utils" do
      @revs.get_full_date('1965|1968').should be_falsey# multiple years
      @revs.get_full_date('1965-1968').should be_falsey# multiple years
      @revs.get_full_date('1965-8').should be_falsey# multiple years
+
    end
 
    it "should indicate if we have a valid year" do
@@ -106,7 +108,29 @@ describe "Revs-Utils" do
      @revs.is_valid_year?('1700').should be_falsey # too old! no cars even existed yet
      @revs.is_valid_year?('1700',1600).should be_truthy # unless we allow it to be ok
   end
-   
+
+  it "should indicate if we have unknown formats" do
+    @revs.revs_is_valid_format?(nil).should be_truthy
+    @revs.revs_is_valid_format?('').should be_truthy    
+    @revs.revs_is_valid_format?('slides').should be_truthy
+    @revs.revs_is_valid_format?('slide').should be_falsey
+    @revs.revs_is_valid_format?('slides | slide').should be_falsey
+    @revs.revs_is_valid_format?('slides | black-and-white negatives').should be_truthy
+    @revs.revs_is_valid_format?('black-and-white-negatives').should be_falsey
+    @revs.revs_is_valid_format?('black-and-white negatives').should be_truthy   
+  end
+  
+  it "should indicate if we have a valid datestring" do
+    @revs.revs_is_valid_datestring?('1959').should be_truthy
+    @revs.revs_is_valid_datestring?('bogus').should be_falsey
+    @revs.revs_is_valid_datestring?('').should be_truthy
+    @revs.revs_is_valid_datestring?(nil).should be_truthy
+    @revs.revs_is_valid_datestring?([]).should be_truthy
+    @revs.revs_is_valid_datestring?('2/2/1950').should be_truthy
+    @revs.revs_is_valid_datestring?('2/31/1950').should be_falsey
+    @revs.revs_is_valid_datestring?('2/2/50').should be_truthy
+    @revs.revs_is_valid_datestring?('195x').should be_truthy
+ end   
    
    it "should lookup the country correctly" do
      @revs.revs_get_country('USA').should == "United States"
@@ -132,9 +156,10 @@ describe "Revs-Utils" do
      @revs.revs_get_state_name('IN').should == "Indiana"       
    end
   
-  it "should parse 1950s correctly" do
+  it "should parse 1950s and 1950's correctly" do
     
     @revs.parse_years('1950s').should == ['1950','1951','1952','1953','1954','1955','1956','1957','1958','1959']    
+    @revs.parse_years("1950's").should == ['1950','1951','1952','1953','1954','1955','1956','1957','1958','1959']    
     
   end
 
@@ -156,6 +181,12 @@ describe "Revs-Utils" do
     
   end
 
+  it "should parse 1800-1802" do
+    
+    @revs.parse_years('1800-1802').should == ['1800','1801','1802']    
+    
+  end
+  
   it "should parse 1955-1957 | 1955 | 1955 and not produce duplicate years" do
     
     @revs.parse_years('1955-1957 | 1955 | 1955').should == ['1955','1956','1957']    

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: revs-utils
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.0.8
 platform: ruby
 authors:
 - Peter Mangiafico
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2015-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: countries
@@ -149,7 +149,8 @@ files:
 - spec/sample-csv-files/no-sourceid.csv
 - spec/spec_helper.rb
 homepage: ''
-licenses: []
+licenses:
+- All rights reserved, Stanford University.
 metadata: {}
 post_install_message: 
 rdoc_options: []