traject_horizon 1.1.0 → 1.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c445af099fb059edc31fcf8a82737032e97ea149
+  data.tar.gz: 3db072af3d1202b9235d4a0dbd9c3fcd2c29699f
+SHA512:
+  metadata.gz: 53780c5c0e8eb1d3f3929f4bb3b42f688decfafddd0c1e38ad2920b2f9e5ec690cbd656255d3d9d4f44e10bcc73b3868ce203d201e54f212360d0ffff613fca6
+  data.tar.gz: af84bb6b374d0bf2f4489361c9cf524b2670511369eefe0ec615f3d5b7bf0fde5950ac17466d5659fde59192e68be0f28bc531214b19faf5a993d05a9f70d49b

data/lib/traject/horizon_reader.rb

@@ -122,6 +122,10 @@ module Traject
   #   to UTF8. These character references are oddly legal representations of UTF8 in
   #   MARC8. http://www.loc.gov/marc/specifications/speccharconversion.html#lossless
   #
+  # Note HorizonReader will also remove control chars from output (except for ones
+  # with legal meaning in binary MARC) -- these are errors in Horizon db which mean
+  # nothing, are illegal in MARC binary serialization, and can mess things up. 
+  #
   # == Misc
   #
   # [horizon.batch_size] Batch size to use for fetching item/copy info on each bib. Default 400.
@@ -253,14 +257,17 @@ module Traject
     end
 
     # Converts from Marc8 to UTF8 if neccesary.
-    # Also replaces horizon <U+nnnn> codes if needed, as well as weird Horizon HTML-escaped rlm
+    #
+    # Also replaces escaped unicode codepoints using custom Horizon "<U+nnnn>" format
+    # Or standard MARC 'lossless encoding' "&#xHHHH;" format. 
     def convert_text!(text, error_handler)
       text = AnselToUnicode.new(error_handler, true).convert(text) if convert_marc8_to_utf8?
 
       # Turn Horizon's weird escaping into UTF8: <U+nnnn> where nnnn is a hex unicode
       # codepoint, turn it UTF8 for that codepoint
       if settings["horizon.destination_encoding"] == "UTF8" &&
-          settings["horizon.codepoint_translate"].to_s == "true" || settings["horizon.character_reference_translate"]
+          (settings["horizon.codepoint_translate"].to_s == "true" ||
+           settings["horizon.character_reference_translate"].to_s == "true")
 
           regexp = if settings["horizon.codepoint_translate"].to_s == "true" && settings["horizon.character_reference_translate"].to_s == "true"
             # unicode codepoint in either HTML char reference form OR
@@ -278,6 +285,12 @@ module Traject
         end
       end
 
+      # eliminate illegal control chars. All ASCII less than 0x20
+      # _except_ for four legal ones (including MARC delimiters). 
+      # http://www.loc.gov/marc/specifications/specchargeneral.html#controlfunction
+      # this is all bytes from 0x00 to 0x19 except for the allowed 1B, 1D, 1E, 1F. 
+      text.gsub!(/[\x00-\x1A\x1C]/, '')
+
       return text
     end
 

data/lib/traject_horizon/version.rb

@@ -1,3 +1,3 @@
 module TrajectHorizon
-  VERSION = "1.1.0"
+  VERSION = "1.1.1"
 end

data/test/{horizon_unicode_escape_test.rb → horizon_convert_text_test.rb}

@@ -21,6 +21,12 @@ describe "turning weird Horizon escape sequences into unicode" do
       assert_equal "Weird \u200F but these aren't changed #x2000; &#200F etc.", converted
     end
 
+    it "erases illegal control chars" do
+      converted = @reader.convert_text!("Some illegal control chars: '\x07' '\x02'; and a legal one: '\x1F'.", org.marc4j.ErrorHandler.new)
+
+      assert_equal "Some illegal control chars: '' ''; and a legal one: '\x1F'.", converted
+    end
+
   end
 
 end

data/traject_horizon.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "traject"
 
   spec.add_dependency "marc-marc4j" # for marc4j jar files
+  spec.add_dependency "ensure_valid_encoding", ">= 0.5.3"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

metadata

@@ -1,97 +1,100 @@
 --- !ruby/object:Gem::Specification
 name: traject_horizon
 version: !ruby/object:Gem::Version
-  version: 1.1.0
-  prerelease: 
+  version: 1.1.1
 platform: ruby
 authors:
 - Jonathan Rochkind
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-11-06 00:00:00.000000000 Z
+date: 2013-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: traject
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: marc-marc4j
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: marc-marc4j
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: ensure_valid_encoding
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
+        version: 0.5.3
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :development
+        version: 0.5.3
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bundler
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '1.3'
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
   prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: minitest
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+  prerelease: false
+  type: :development
+description:
 email:
 - jonathan@dnil.net
 executables: []
@@ -108,7 +111,7 @@ files:
 - lib/traject_horizon.rb
 - lib/traject_horizon/version.rb
 - test/horizon_bib_auth_merge_test.rb
-- test/horizon_unicode_escape_test.rb
+- test/horizon_convert_text_test.rb
 - test/test_helper.rb
 - traject_horizon.gemspec
 - vendor/jtds/.DS_Store
@@ -116,30 +119,28 @@ files:
 homepage: http://github.com/jrochkind/traject_horizon
 licenses:
 - MIT
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.23
-signing_key: 
-specification_version: 3
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
 summary: Horizon ILS MARC Exporter, a plugin for the traject tool
 test_files:
 - test/horizon_bib_auth_merge_test.rb
-- test/horizon_unicode_escape_test.rb
+- test/horizon_convert_text_test.rb
 - test/test_helper.rb
-has_rdoc: