traject_horizon 1.0.0 → 1.1.0

data/lib/traject/horizon_reader.rb

@@ -117,6 +117,11 @@ module Traject
   #   codepoint escaping to actual UTF-8 bytes. Defaults to true. Will be ignored
   #   unless horizon.destination_encoding is UTF8 though.
   #
+  # [horizon.character_reference_translate] Default true. Convert HTML/XML-style
+  #   character references like "‏" to actual UTF-8 bytes, when converting
+  #   to UTF8. These character references are oddly legal representations of UTF8 in
+  #   MARC8. http://www.loc.gov/marc/specifications/speccharconversion.html#lossless
+  #
   # == Misc
   #
   # [horizon.batch_size] Batch size to use for fetching item/copy info on each bib. Default 400.
@@ -248,14 +253,27 @@ module Traject
     end
 
     # Converts from Marc8 to UTF8 if neccesary.
-    # Also replaces horizon <U+nnnn> codes if needed.
+    # Also replaces horizon <U+nnnn> codes if needed, as well as weird Horizon HTML-escaped rlm
     def convert_text!(text, error_handler)
       text = AnselToUnicode.new(error_handler, true).convert(text) if convert_marc8_to_utf8?
 
       # Turn Horizon's weird escaping into UTF8: <U+nnnn> where nnnn is a hex unicode
       # codepoint, turn it UTF8 for that codepoint
-      if settings["horizon.codepoint_translate"].to_s == "true" && settings["horizon.destination_encoding"] == "UTF8"
-        text.gsub!(/\<U\+([0-9A-Fa-f]{4})\>/) do
+      if settings["horizon.destination_encoding"] == "UTF8" &&
+          settings["horizon.codepoint_translate"].to_s == "true" || settings["horizon.character_reference_translate"]
+
+          regexp = if settings["horizon.codepoint_translate"].to_s == "true" && settings["horizon.character_reference_translate"].to_s == "true"
+            # unicode codepoint in either HTML char reference form OR
+            # weird horizon form. 
+             /(?:\<U\+|&#x)([0-9A-Fa-f]{4})(?:\>|;)/
+          elsif settings["horizon.codepoint_translate"].to_s == "true"
+            # just weird horizon form
+            /\<U\+([0-9A-Fa-f]{4})\>/
+          else # just character references
+            /&#x([0-9A-Fa-f]{4});/
+          end
+          
+        text.gsub!(regexp) do
           [$1.hex].pack("U")
         end
       end
@@ -682,6 +700,7 @@ module Traject
         "horizon.source_encoding"      => "MARC8",
         "horizon.destination_encoding" => "UTF8",
         "horizon.codepoint_translate"  => true,
+        "horizon.character_reference_translate" => true, 
 
         "horizon.item_tag"          => "991",
         # Crazy isnull() in the call_type join to join to call_type directly on item

data/lib/traject_horizon/version.rb

@@ -1,3 +1,3 @@
 module TrajectHorizon
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/test/horizon_unicode_escape_test.rb

@@ -12,8 +12,13 @@ describe "turning weird Horizon escape sequences into unicode" do
     end
 
     it "converts" do
-      converted = @reader.convert_text!("A hangul character: <U+1112>, okay<!>", org.marc4j.ErrorHandler.new)
-      assert_equal "A hangul character: ᄒ, okay<!>", converted
+      converted = @reader.convert_text!("A hangul character: <U+1112>, okay<!> U+1000>", org.marc4j.ErrorHandler.new)
+      assert_equal "A hangul character: ᄒ, okay<!> U+1000>", converted
+    end
+
+    it "converts rlm" do
+      converted = @reader.convert_text!("Weird &#x200F; but these aren't changed #x2000; &#200F etc.", org.marc4j.ErrorHandler.new)
+      assert_equal "Weird \u200F but these aren't changed #x2000; &#200F etc.", converted
     end
 
   end

metadata

@@ -1,86 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: traject_horizon
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
+  prerelease: 
 platform: ruby
 authors:
 - Jonathan Rochkind
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-21 00:00:00.000000000 Z
+date: 2013-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: traject
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: marc-marc4j
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: marc-marc4j
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: bundler
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: minitest
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-description:
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
 email:
 - jonathan@dnil.net
 executables: []
@@ -105,28 +116,30 @@ files:
 homepage: http://github.com/jrochkind/traject_horizon
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.5
-signing_key:
-specification_version: 4
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
 summary: Horizon ILS MARC Exporter, a plugin for the traject tool
 test_files:
 - test/horizon_bib_auth_merge_test.rb
 - test/horizon_unicode_escape_test.rb
 - test/test_helper.rb
+has_rdoc: 

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 2f8ac4a546bb69032c0da921fa9a749ef83a6d96
-  data.tar.gz: 530cc90de2a1eb2ffb52eda7601bbc5076be3bf7
-SHA512:
-  metadata.gz: d5e490f9706b94fa747b21c689255320b9cd96dfe6cd1b93c63d32952496e293a90f550d5a8449d87d4a05c39a8a59cd049399e004bd75f031a9a55048ffb718
-  data.tar.gz: a766c0d3a2418017d0dcfe562737b4b647bb28e8e9d05dad9fc55be617e6de69c53dcbdd415bb153e4e56de1ea8bdd3cb0599728ef5dc729f16c23049a00764b