bpl_enrich 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41994a22b0ef539c9938525b17ad475e6eaafce7
-  data.tar.gz: 9c16b1cbb137f1c770d72190b3916a2ac6f703ae
+  metadata.gz: 63ca261fe49ac43d3aad9df73b3caa2ce33dce29
+  data.tar.gz: 70f1ae1ef1a0b9650b75093cff8d3916023cc3cf
 SHA512:
-  metadata.gz: 06560416487c4a58d16c0ebed6716429b44aa73bd92c62f6d99093ce4d130a416adf8fa8f5eaeae0e91f2f371fed67535288109265c1a1c8ea44c3690b15d78c
-  data.tar.gz: 54f601dd8de34b18f9c018c09ea48fe1ac4de300e719c07aa34b6f595cbab50c337eecf24d19cde70c2cde2c5f786af492346581027fc30872a1cb3de31db311
+  metadata.gz: 48f5af0440326806c69d638ee8a3cb4b2cb5c28e9b7d4e17128a48b413542a59fb8e063b6d31521089334adb1791b90216e3f10b68b86b176dca0f289b9132ec
+  data.tar.gz: cedfcdcc85a1bb2d55e7f97117081f45fd5fc77c1b031fe108a4aded7ea258d7e6f35c12e1a0139f4828264369d52bfb1baaa4e0b1933aad71177d18db793aae

data/lib/bpl_enrich/dates.rb

@@ -1,6 +1,43 @@
 module BplEnrich
   class Dates
 
+    def self.is_numeric? (string)
+      true if Float(string) rescue false
+    end
+
+    def self.convert_month_words(date_string)
+      return_date_string = date_string.clone
+
+      date_string = date_string.gsub(/[,\/\.]/, ' ').squeeze #switch periods, slashes, and commas that can seperate dates with spaces
+      if date_string.split(' ').any? { |word| Date::MONTHNAMES.include?(word.humanize) || Date::ABBR_MONTHNAMES.include?(word.gsub('.', '').humanize) }
+        return_date_string = ''
+        was_numeric = false
+
+        date_string.split(' ').each do |date_word|
+          if Date::MONTHNAMES.include?(date_word.humanize)
+              current_value = Date::MONTHNAMES.index(date_word).to_s.rjust(2, '0')
+          elsif Date::ABBR_MONTHNAMES.include?(date_word.humanize)
+            current_value = Date::ABBR_MONTHNAMES.index(date_word).to_s.rjust(2, '0')
+          else
+              current_value = date_word
+          end
+          if is_numeric?(current_value)
+            if was_numeric
+              return_date_string += "/#{current_value.to_s.rjust(2, '0')}"
+            else
+              was_numeric = true
+              return_date_string += " #{current_value.to_s.rjust(2, '0')}"
+            end
+          else
+            was_numeric = false
+            return_date_string += " #{current_value}"
+          end
+        end
+      end
+
+      return return_date_string
+    end
+
     # a function to convert date data from OAI feeds into MODS-usable date data
     # assumes date values containing ";" have already been split
     # returns hash with :single_date, :date_range, :date_qualifier, and/or :date_note values
@@ -9,6 +46,8 @@ module BplEnrich
       date_data = {} # create the hash to hold all the data
       source_date_string = value.strip # variable to hold original value
 
+      value = convert_month_words(value) #Stuff like April 7, 1983
+
       # weed out obvious bad dates before processing
       if (value.match(/([Pp]re|[Pp]ost|[Bb]efore|[Aa]fter|[Uu]nknown|[Uu]ndated|n\.d\.)/)) ||
           (value.match(/\d\d\d\d-\z/)) || # 1975-
@@ -194,22 +233,26 @@ module BplEnrich
           # try to automatically parse single dates with YYYY && MM && DD values
           if Timeliness.parse(value).nil?
             # start further processing
-            if value.match(/\A[12]\d\d\d-[01][0-9]\z/) # yyyy-mm
-              date_data[:single_date] = value
-            elsif value.match(/\A[01]?[1-9][-\/][12]\d\d\d\z/) # mm-yyyy || m-yyyy || mm/yyyy
-              value = '0' + value if value.match(/\A[1-9][-\/][12]\d\d\d\z/) # m-yyyy || m/yyyy
-              date_data[:single_date] = value[3..6] + '-' + value[0..1]
-            elsif value.match(/\A[A-Za-z]{3,} [12]\d\d\d\z/) # April 1987 || Apr. 1987
-              value = value.split(' ')
-              if value[0].length == 3
-                value_month = '%02d' % Date::ABBR_MONTHNAMES.index(value[0])
-              else
-                value_month = '%02d' % Date::MONTHNAMES.index(value[0])
+            value.split(' ').each do |split_value|
+              if split_value.match(/\A[12]\d\d\d[-\/\.][01][0-9]\z/) # yyyy-mm || yyyy/mm || yyyy.mm
+                split_value = split_value.gsub(/[,\/\.]/, '-').squeeze
+                date_data[:single_date] = split_value
+              elsif split_value.match(/\A[12]\d\d\d[-\/\.][01][0-9][-\/\.][01][0-9]\z/) # yyyy-mm-dd || yyyy/mm/dd || yyyy.mm.dd
+                split_value = split_value.gsub(/[,\/\.]/, '-').squeeze
+                date_data[:single_date] = split_value
+              elsif split_value.match(/\A[01]?[1-9][-\/][12]\d\d\d\z/) # mm-yyyy || m-yyyy || mm/yyyy
+                split_value = '0' + split_value if split_value.match(/\A[1-9][-\/\.][12]\d\d\d\z/) # m-yyyy || m/yyyy
+                date_data[:single_date] = split_value[3..6] + '-' + split_value[0..1]
+              elsif split_value.match(/\A[12]\d\d\d\z/) # 1999
+                date_data[:single_date] = split_value
+              elsif split_value.match(/\A[01]?[1-9][-\/\.][01]?[1-9][-\/\.][12]\d\d\d\z/) # mm-dd-yyyy || m-dd-yyyy || mm/dd/yyyy
+                split_value = split_value.gsub(/[,\/\.]/, '/').squeeze
+                date_data[:single_date] = "#{split_value.split('/')[2]}-#{split_value.split('/')[0]}-#{split_value.split('/')[1]}"
               end
-              date_data[:single_date] = value_month ? value[1] + '-' + value_month : value[1]
-            elsif value.match(/\A[12]\d\d\d\z/) # 1999
-              date_data[:single_date] = value
-            else
+
+            end
+
+            if value.split(' ').length > 1 || date_data[:single_date].blank?
               date_data[:date_note] = source_date_string
             end
           else

data/lib/bpl_enrich/version.rb

@@ -1,3 +1,3 @@
 module BplEnrich
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/bpl_enrich.rb

@@ -8,6 +8,11 @@ module BplEnrich
   require "htmlentities"
   require "qa"
 
+  # add some formats to Timeliness gem for better parsing
+  Timeliness.add_formats(:date, 'm-d-yy', :before => 'd-m-yy')
+  Timeliness.add_formats(:date, 'mmm[\.]? d[a-z]?[a-z]?[,]? yyyy')
+  Timeliness.add_formats(:date, 'yyyy mmm d')
+
   def self.strip_value(value)
     if(value.blank?)
       return nil

data/test/dates_test.rb

@@ -2,10 +2,33 @@ require 'test_helper'
 
 class DatesTest < ActiveSupport::TestCase
   def test_date_standardizer
+
+    #Month dates
     result = BplEnrich::Dates.standardize('April 1983')
     assert_equal '1983-04', result[:single_date]
     assert_equal nil, result[:date_range]
     assert_equal nil, result[:date_note]
+
+    result = BplEnrich::Dates.standardize('April 7, 1983')
+    assert_equal '1983-04-07', result[:single_date]
+    assert_equal nil, result[:date_range]
+    assert_equal nil, result[:date_note]
+
+    result = BplEnrich::Dates.standardize('April 7.1983 (Easter)')
+    assert_equal '1983-04-07', result[:single_date]
+    assert_equal nil, result[:date_range]
+    assert_equal 'April 7.1983 (Easter)', result[:date_note]
+
+    result = BplEnrich::Dates.standardize('1983.April.7 (Easter)')
+    assert_equal '1983-04-07', result[:single_date]
+    assert_equal nil, result[:date_range]
+    assert_equal '1983.April.7 (Easter)', result[:date_note]
+
+    result = BplEnrich::Dates.standardize('between April 2014 and May 2014')
+    assert_equal nil, result[:single_date]
+    assert_equal '2014-04', result[:date_range][:start]
+    assert_equal '2014-05', result[:date_range][:end]
+    assert_equal nil, result[:date_note]
   end
 
 

data/test/dummy/log/test.log

@@ -176,5 +176,125 @@ DatesTest: test_date_standardizer
    (0.1ms)  begin transaction
 --------------------------------
 LCSHTest: test_lcsh_standardizer
+--------------------------------
+   (0.1ms)  rollback transaction
+   (0.7ms)  begin transaction
+------------------------------------
+AuthoritiesTest: test_parse_language
+------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-----------------------------------------
+AuthoritiesTest: test_parse_name_for_role
+-----------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+--------------------------------
+AuthoritiesTest: test_parse_role
+--------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-------------------------------
+BplEnrichTest: test_strip_value
+-------------------------------
+   (0.1ms)  rollback transaction
+   (0.2ms)  begin transaction
+---------------------------------
+DatesTest: test_date_standardizer
+---------------------------------
+   (0.2ms)  rollback transaction
+   (0.1ms)  begin transaction
+--------------------------------
+LCSHTest: test_lcsh_standardizer
+--------------------------------
+   (0.1ms)  rollback transaction
+   (0.5ms)  begin transaction
+------------------------------------
+AuthoritiesTest: test_parse_language
+------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-----------------------------------------
+AuthoritiesTest: test_parse_name_for_role
+-----------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+--------------------------------
+AuthoritiesTest: test_parse_role
+--------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-------------------------------
+BplEnrichTest: test_strip_value
+-------------------------------
+   (0.2ms)  rollback transaction
+   (0.1ms)  begin transaction
+---------------------------------
+DatesTest: test_date_standardizer
+---------------------------------
+   (0.2ms)  rollback transaction
+   (0.1ms)  begin transaction
+--------------------------------
+LCSHTest: test_lcsh_standardizer
+--------------------------------
+   (0.1ms)  rollback transaction
+   (0.4ms)  begin transaction
+------------------------------------
+AuthoritiesTest: test_parse_language
+------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-----------------------------------------
+AuthoritiesTest: test_parse_name_for_role
+-----------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+--------------------------------
+AuthoritiesTest: test_parse_role
+--------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-------------------------------
+BplEnrichTest: test_strip_value
+-------------------------------
+   (0.2ms)  rollback transaction
+   (0.5ms)  begin transaction
+---------------------------------
+DatesTest: test_date_standardizer
+---------------------------------
+   (0.1ms)  rollback transaction
+   (0.1ms)  begin transaction
+--------------------------------
+LCSHTest: test_lcsh_standardizer
+--------------------------------
+   (0.1ms)  rollback transaction
+   (0.8ms)  begin transaction
+------------------------------------
+AuthoritiesTest: test_parse_language
+------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+-----------------------------------------
+AuthoritiesTest: test_parse_name_for_role
+-----------------------------------------
+   (0.3ms)  rollback transaction
+   (0.2ms)  begin transaction
+--------------------------------
+AuthoritiesTest: test_parse_role
+--------------------------------
+   (0.1ms)  rollback transaction
+   (0.1ms)  begin transaction
+-------------------------------
+BplEnrichTest: test_strip_value
+-------------------------------
+   (0.1ms)  rollback transaction
+   (0.2ms)  begin transaction
+---------------------------------
+DatesTest: test_date_standardizer
+---------------------------------
+   (0.3ms)  rollback transaction
+   (0.1ms)  begin transaction
+--------------------------------
+LCSHTest: test_lcsh_standardizer
 --------------------------------
    (0.1ms)  rollback transaction

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bpl_enrich
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Boston Public Library
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-11 00:00:00.000000000 Z
+date: 2014-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails