traject 1.0.0.beta.7 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ccc1ca6c023f01b9e21148a3ca37d86858df8065
-  data.tar.gz: 4360b687f3ba8d957d4a14d39bc6cefbd443bcdd
+  metadata.gz: 31243d453f43fbc8f8634c2511340d8fb96c606f
+  data.tar.gz: a843a583f235920931304fc6baa7ff75ce91dfab
 SHA512:
-  metadata.gz: dba976f724960b4048c2db65827780a46a2c64c442b2f48aa077375775f9f3a8901f446f226fd26f8e69cddbb696b3c114403c592b20cdc362bc43161a25f608
-  data.tar.gz: 7216d83c97aa75698d2c03dec2bcc263decf4cc3d39a62187e9b0b09856fa1cac45e4fcb524d633f8a77acbf474343a69fec9b0324b66f056eb888915396e95b
+  metadata.gz: 5a0e6f3c695a5fe497f4cadd14fba7a2a10a3ea1ff40e42a6625949227d1eb3f995dc4d0ecb228f5e98c95f737df1c1e0e8089b59e17fcb0fedca8de5208b535
+  data.tar.gz: 3598e91ed10b039c4382d56ddc69939713425c4f059eb27adfddfdc272e25f2c52e8f900a31b94ceda6576d716c2155188dc03d4697156314e71aadf99a92471

data/README.md

@@ -6,7 +6,7 @@ Might be used to index MARC data for a Solr-based discovery product like [Blackl
 Traject might also be generalized to a set of tools for getting structured data from a source, and transforming it to a hash-like object to send to a destination. 
 
 
-**Traject is nearing 1.0, it is robust, feature-rich and being used in production by authors -- feedback invited**
+**Traject is stable, mature software, that is already being used in production by its authors.**
 
 [![Gem Version](https://badge.fury.io/rb/traject.png)](http://badge.fury.io/rb/traject)
 [![Build Status](https://travis-ci.org/traject-project/traject.png)](https://travis-ci.org/traject-project/traject)
@@ -89,13 +89,11 @@ settings do
   # various others...
   provide "solrj_writer.commit_on_close", "true"
 
-  # By default, we use the Traject::Marc4JReader, which
-  # can read marc8 and ISO8859_1 -- if your records are all in UTF8,
-  # the pure-ruby MarcReader may be faster...
-  # provide "reader_class_name", "Traject::MarcReader"
-  # If you ARE using the Marc4JReader, it defaults to "BESTGUESS"
-  # as to encoding when reading binary, you may want to tell it instead
-  provide "marc4j_reader.source_encoding", "MARC8" # or UTF-8 or ISO8859_1
+  # By default, we use the Traject::MarcReader
+  # One altenrnative is the Marc4JReader, using Marc4J. 
+  # provide "reader_class_name", "Traject::Marc4Reader"
+  # If we're reading binary MARC, it's best to tell it the encoding. 
+  provide "marc4j_reader.source_encoding", "MARC-8" # or 'UTF-8' or 'ISO-8859-1' or whatever. 
 end
 ~~~
 

data/doc/batch_execution.md

@@ -224,7 +224,7 @@ object you configure yourself however you like:
 ~~~ruby
   # inside a traject configuration file
 
-  logger = Yell.new do |l|
+  self.logger = Yell.new do |l|
     l.level = 'gte.info' # will only pass :info and above to the adapters
 
     l.adapter :datefile, 'production.log', level: 'lte.warn' # anything lower or equal to :warn
@@ -232,6 +232,10 @@ object you configure yourself however you like:
   end
 ~~~
 
+**note** it's important to use to use `self.logger =`, or due to
+ruby idiosyncracies you'll just be setting a local variable, not the Indexer's
+logger attribute. 
+
 See [yell](https://github.com/rudionrails/yell)  docs for more, you can
 do whatever you can make yell, just write ruby.
 

data/doc/settings.md

@@ -47,8 +47,10 @@ settings are applied first of all. It's recommended you use `provide`.
 * `log.level`:  Log this level and above. Default 'info', set to eg 'debug' to get potentially more logging info,
               or 'error' to get less. https://github.com/rudionrails/yell/wiki/101-setting-the-log-level
 
-* `log.batch_size`: If set to a number N (or string representation), will output a progress line to DEBUG
-   log, every N records. (use -d to turn logging to DEBUG to see.)
+* `log.batch_size`: If set to a number N (or string representation), will output a progress line to
+   log. (by default as INFO, but see log.batch_size.severity)
+
+* `log.batch_size.severity`: If `log.batch_size` is set, what logger severity level to log to. Default "INFO", set to "DEBUG" etc if desired. 
 
 * `marc_source.type`: default 'binary'. Can also set to 'xml' or (not yet implemented todo) 'json'. Command line shortcut `-t`
 

data/lib/traject/indexer.rb

@@ -332,7 +332,7 @@ class Traject::Indexer
       if log_batch_size && (count % log_batch_size == 0)
         batch_rps = log_batch_size / (Time.now - batch_start_time)
         overall_rps = count / (Time.now - start_time)
-        logger.debug "Traject::Indexer#process, read #{count} records at id:#{id_string(record)}; #{'%.0f' % batch_rps}/s this batch, #{'%.0f' % overall_rps}/s overall"
+        logger.send(settings["log.batch_size.severity"].downcase.to_sym, "Traject::Indexer#process, read #{count} records at id:#{id_string(record)}; #{'%.0f' % batch_rps}/s this batch, #{'%.0f' % overall_rps}/s overall")
         batch_start_time = Time.now
       end
 

data/lib/traject/indexer/settings.rb

@@ -62,14 +62,14 @@ class Traject::Indexer
 
     def self.defaults
       @@defaults ||= {
-      "reader_class_name"         => "Traject::Marc4JReader",
+      "reader_class_name"         => "Traject::MarcReader",
       "writer_class_name"         => "Traject::SolrJWriter",
-      "marc_source.type"          => "binary",
+      "marc_source.type"          => "binary",            
       "marc4j_reader.permissive"  => true,
-      "marc4j_reader.source_encoding" => "BESTGUESS",
       "solrj_writer.batch_size"   => 200,
       "solrj_writer.thread_pool"  => 1,
-      "processing_thread_pool"    => 3
+      "processing_thread_pool"    => 3,
+      "log.batch_size.severity"   => "info"
       }
     end
 

data/lib/traject/marc4j_reader.rb

@@ -3,9 +3,8 @@ require 'marc'
 require 'marc/marc4j'
 
 # `Traject::Marc4JReader` uses the marc4j java package to parse the MARC records
-# into standard ruby-marc MARC::Record objects. This reader is often faster than
-# Traject::MarcReader, especially for XML, and offers support for reading Marc8
-# encoded records and transcoding to UTF8.
+# into standard ruby-marc MARC::Record objects. This reader may be faster than
+# Traject::MarcReader, especially for XML.
 #
 # Marc4JReader can read MARC ISO 2709 ("binary") or MARCXML. We use the Marc4J MarcPermissiveStreamReader
 # for reading binary, but sometimes in non-permissive mode, according to settings. We use the Marc4j MarcXmlReader
@@ -24,13 +23,15 @@ require 'marc/marc4j'
 #                             value to 'permissive' arg of MarcPermissiveStreamReader constructor.
 #                             Only used for 'binary'
 #
-# * marc4j_reader.source_encoding: Only used for 'binary', otherwise always UTF-8.
+# * marc_source.encoding: Only used for 'binary', otherwise always UTF-8.
 #         String of the values MarcPermissiveStreamReader accepts:
-#         * BESTGUESS  (tries to use MARC leader and believe it, I think)
-#         * ISO8859_1
+#         * BESTGUESS  (default: not entirely clear what Marc4J does with this)
+#         * ISO-8859-1 (also accepted: ISO8859_1)
 #         * UTF-8
-#         * MARC8
-#         Default 'BESTGUESS', but marc records in the wild are so wrong here, recommend setting.
+#         * MARC-8 (also accepted: MARC8)
+#         Default 'BESTGUESS', but HIGHLY recommend setting
+#         to avoid some Marc4J unpredictability, Marc4J "BESTGUESS" can be unpredictable
+#         in a variety of ways. 
 #         (will ALWAYS be transcoded to UTF-8 on the way out. We insist.)
 #
 # * marc4j_reader.jar_dir: Path to a directory containing Marc4J jar file to use. All .jar's in dir will
@@ -54,7 +55,7 @@ require 'marc/marc4j'
 #
 #       # Or instead for binary:
 #       provide "marc4j_reader.permissive", true
-#       provide "marc4j_reader.source_encoding", "MARC8"
+#       provide "marc_source.encoding", "MARC8"
 #     end
 class Traject::Marc4JReader
   include Enumerable
@@ -94,6 +95,20 @@ class Traject::Marc4JReader
     settings["marc_source.type"]
   end
 
+  def specified_source_encoding
+    #settings["marc4j_reader.source_encoding"]
+    enc = settings["marc_source.encoding"]
+
+    # one is standard for ruby and we want to support,
+    # the other is used by Marc4J and we have to pass it to Marc4J
+    enc = "ISO8859_1" if enc == "ISO-8859-1"
+
+    # default
+    enc = "BESTGUESS" if enc.nil? || enc.empty?
+
+    return enc
+  end
+
   def create_marc_reader!
     case input_type
     when "binary"
@@ -101,7 +116,7 @@ class Traject::Marc4JReader
 
       # #to_inputstream turns our ruby IO into a Java InputStream
       # third arg means 'convert to UTF-8, yes'
-      MarcPermissiveStreamReader.new(input_stream.to_inputstream, permissive, true, settings["marc4j_reader.source_encoding"])
+      MarcPermissiveStreamReader.new(input_stream.to_inputstream, permissive, true, specified_source_encoding)
     when "xml"
       MarcXmlReader.new(input_stream.to_inputstream)
     else

data/lib/traject/marc_reader.rb

@@ -4,22 +4,33 @@ require 'traject/ndj_reader'
 # `Traject::MarcReader` uses pure ruby marc gem to parse MARC records. It
 # can read MARC ISO 2709 ('binary'), MARC-XML, and Marc-in-json (newline-delimited-json).
 #
-# MarcReader can not currently read binary MARC in the MARC8 encoding, see
-# the Traject::Marc4JReader instead.
+# Marc4JReader is an alternative to this class, powered by Marc4J. You may be interested
+# in comparing for performance, under your particular use case. 
 #
 # By default assumes binary MARC encoding, please set marc_source.type setting
-# for XML or json.
+# for XML or json. If binary, please set marc_source.encoding with char encoding. 
 #
 # ## Settings
 
 # * "marc_source.type":  serialization type. default 'binary'
-#       * "binary". standard ISO 2709 "binary" MARC format.
-#       * "xml", MarcXML
+#       * "binary". standard ISO 2709 "binary" MARC format, 
+#           will use ruby-marc MARC::Reader (Note, if you are using
+#          type 'binary', you probably want to also set 'marc_source.encoding')
+#       * "xml", MarcXML, will use ruby-marc MARC::XMLReader
 #       * "json" The "marc-in-json" format, encoded as newline-separated
 #         json. (synonym 'ndj'). A simplistic newline-separated json, with no comments
 #         allowed, and no unescpaed internal newlines allowed in the json
 #         objects -- we just read line by line, and assume each line is a
 #         marc-in-json. http://dilettantes.code4lib.org/blog/2010/09/a-proposal-to-serialize-marc-in-json/
+#         will use Traject::NDJReader which uses MARC::Record.new_from_hash. 
+# * "marc_source.encoding": Only used for marc_source.type 'binary', character encoding
+#         of the source marc records. Can be any
+#         encoding recognized by ruby, OR 'MARC-8'.  For 'MARC-8', content will
+#         be transcoded (by ruby-marc) to UTF-8 in internal MARC::Record Strings. 
+#         Default nil, meaning let MARC::Reader use it's default, which will
+#         probably be Encoding.default_internal, which will probably be UTF-8. 
+#         Right now Traject::MarcReader is hard-coded to transcode to UTF-8 as
+#         an internal encoding. 
 # * "marc_reader.xml_parser": For XML type, which XML parser to tell Marc::Reader
 #         to use. Anything recognized by [Marc::Reader :parser
 #         argument](http://rdoc.info/github/ruby-marc/ruby-marc/MARC/XMLReader).
@@ -62,7 +73,9 @@ class Traject::MarcReader
         when 'json'
           Traject::NDJReader.new(self.input_stream, settings)
         else
-          MARC::Reader.new(self.input_stream)
+          args = { :invalid => :replace }
+          args[:external_encoding] = settings["marc_source.encoding"]          
+          MARC::Reader.new(self.input_stream, args)
         end
     end
     return @internal_reader

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "1.0.0.beta.7"
+  VERSION = "1.0.0"
 end

data/test/marc4j_reader_test.rb

@@ -20,7 +20,7 @@ describe "Marc4JReader" do
     first = array.first
 
     assert_kind_of MARC::Record, first
-    assert first['245']['a'].encoding.name, "UTF-8"
+    assert_equal first['245']['a'].encoding.name, "UTF-8"
   end
 
   it "can skip a bad subfield code" do
@@ -37,7 +37,7 @@ describe "Marc4JReader" do
 
   it "reads Marc binary in Marc8 encoding" do
     file = File.new(support_file_path("one-marc8.mrc"))
-    settings = Traject::Indexer::Settings.new("marc4j_reader.source_encoding" => "MARC8")
+    settings = Traject::Indexer::Settings.new("marc_source.encoding" => "MARC8")
     reader = Traject::Marc4JReader.new(file, settings)
 
     array = reader.to_a
@@ -84,5 +84,53 @@ describe "Marc4JReader" do
     assert_kind_of Java::org.marc4j.marc.impl::RecordImpl, record.original_marc4j
   end
 
+  it "replaces unicode character reference in Marc8 transcode" do
+    file = File.new(support_file_path "escaped_character_reference.marc8.marc")
+    # due to marc4j idiosyncracies, this test will NOT pass with default source_encoding
+    # of "BESTGUESS", it only works if you explicitly set to MARC8. Doh. 
+    settings = Traject::Indexer::Settings.new("marc_source.encoding" => "MARC8") # binary type is default
+    record = Traject::Marc4JReader.new(file, settings).to_a.first
+
+    assert_equal "Rio de Janeiro escaped replacement char: \uFFFD .", record['260']['a']
+  end
+
+  describe "Marc4J Java Permissive Stream Reader" do 
+    # needed for sanity check when our tests fail to see if Marc4J
+    # is not behaving how we think it should. 
+    it "converts character references" do
+      file = File.new(support_file_path "escaped_character_reference.marc8.marc")
+      reader = MarcPermissiveStreamReader.new(file.to_inputstream, true, true, "MARC-8")
+      record = reader.next
+
+      field = record.getVariableField("260")
+      subfield = field.getSubfield('a'.ord)
+      value = subfield.getData
+      
+      assert_equal "Rio de Janeiro escaped replacement char: \uFFFD .", value        
+    end
+  end
+
+  it "replaces bad byte in UTF8 marc" do
+    skip "Marc4J needs fixing on it's end" # Marc4J won't do this in 'permissive' mode, gah. 
+
+    # Note this only works because the marc file DOES correctly
+    # have leader byte 9 set to 'a' for UTF8, otherwise Marc4J can't do it. 
+    file = File.new(support_file_path "bad_utf_byte.utf8.marc")
+
+    settings = Traject::Indexer::Settings.new() # binary UTF8 type is default
+    reader = Traject::Marc4JReader.new(file, settings)
+
+    record = reader.to_a.first
+    
+    value = record['300']['a']
+
+    assert_equal value.encoding.name, "UTF-8"
+    assert value.valid_encoding?, "Has valid encoding"
+    assert_equal "This is a bad byte: '\uFFFD' and another: '\uFFFD'", record['300']['a']
+  end
+
+
+
+
 
 end

data/test/marc_reader_test.rb

@@ -17,21 +17,74 @@ describe "Traject::MarcReader" do
     assert_equal 30, array.length
   end
 
-  it "reads Marc binary" do
-    file = File.new(support_file_path "test_data.utf8.mrc")
-    settings = Traject::Indexer::Settings.new() # binary type is default
-    reader = Traject::MarcReader.new(file, settings)
 
-    array = reader.to_a
+  describe "MARC binary" do
+    it "reads" do
+      file = File.new(support_file_path "test_data.utf8.mrc")
+      settings = Traject::Indexer::Settings.new() # binary type is default
+      reader = Traject::MarcReader.new(file, settings)
 
-    assert_equal 30, array.length
+      array = reader.to_a
 
-    first = array.first
+      assert_equal 30, array.length
 
-    assert_kind_of MARC::Record, first
+      first = array.first
 
-    assert first['245']['a'].encoding.name, "UTF-8"
-    assert_equal "Fikr-i Ayāz /", first['245']['a']
+      assert_kind_of MARC::Record, first
+
+      assert first['245']['a'].encoding.name, "UTF-8"
+      assert_equal "Fikr-i Ayāz /", first['245']['a']
+    end
+
+    it "reads Marc binary in Marc8 encoding, transcoding to UTF-8" do
+      file = File.new(support_file_path("one-marc8.mrc"))
+      settings = Traject::Indexer::Settings.new("marc_source.encoding" => "MARC-8")
+      reader = Traject::MarcReader.new(file, settings)
+
+      array = reader.to_a
+
+      assert_length 1, array
+
+
+      assert_kind_of MARC::Record, array.first
+      a245a = array.first['245']['a']
+
+      assert a245a.encoding.name, "UTF-8"
+      assert a245a.valid_encoding?    
+      assert_equal "Por uma outra globalização :", a245a
+    end
+
+    it "replaces unicode character reference in Marc8 transcode" do
+      file = File.new(support_file_path("escaped_character_reference.marc8.marc"))
+      
+      settings = Traject::Indexer::Settings.new("marc_source.encoding" => "MARC-8") # binary type is default
+      record = Traject::MarcReader.new(file, settings).to_a.first
+
+      assert_equal "Rio de Janeiro escaped replacement char: \uFFFD .", record['260']['a']
+    end
+
+    it "raises on unrecognized encoding for binary type" do
+      file = File.new(support_file_path "one-marc8.mrc")
+      settings = Traject::Indexer::Settings.new("marc_source.encoding" => "ADFADFADF")
+      assert_raises(ArgumentError) do
+        record = Traject::MarcReader.new(file, settings).to_a.first
+      end
+    end
+
+    it "replaces bad byte in UTF8 marc binary" do
+      file = File.new(support_file_path "bad_utf_byte.utf8.marc")
+
+      settings = Traject::Indexer::Settings.new() # binary type is default
+      reader = Traject::MarcReader.new(file, settings)
+
+      record = reader.to_a.first
+      
+      value = record['300']['a']
+
+      assert_equal value.encoding.name, "UTF-8"
+      assert value.valid_encoding?, "Has valid encoding"
+      assert_equal "This is a bad byte: '\uFFFD' and another: '\uFFFD'", value
+    end
   end
 
   it "reads JSON" do
@@ -52,4 +105,6 @@ describe "Traject::MarcReader" do
 
 
 
+
+
 end

data/test/test_support/bad_utf_byte.utf8.marc

@@ -0,0 +1 @@
+00083    a2200037   4500300004500000  aThis is a bad byte: '�' and another: '�'

data/test/test_support/escaped_character_reference.marc8.marc

@@ -0,0 +1 @@
+00138cam  2200049Ia 45000010008000002600080000082196384  aRio de Janeiro escaped replacement char: � .bEditora Record,c2000.

data/traject.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.extra_rdoc_files = spec.files.grep(%r{^doc/})
 
 
-  spec.add_dependency "marc", ">= 0.7.1"
+  spec.add_dependency "marc", ">= 0.8.0"
   spec.add_dependency "marc-marc4j", ">=0.1.1" # use and convert marc4j
   spec.add_dependency "hashie", ">= 2.0.5", "< 2.1" # used for Indexer#settings
   spec.add_dependency "slop", ">= 3.4.5", "< 4.0"   # command line parsing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta.7
+  version: 1.0.0
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-11-08 00:00:00.000000000 Z
+date: 2013-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: marc
@@ -17,12 +17,12 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -211,11 +211,13 @@ files:
 - test/test_support/245_no_ab.marc
 - test/test_support/880_with_no_6.utf8.marc
 - test/test_support/bad_subfield_code.marc
+- test/test_support/bad_utf_byte.utf8.marc
 - test/test_support/date_resort_to_260.marc
 - test/test_support/date_type_r_missing_date2.marc
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb
 - test/test_support/emptyish_record.marc
+- test/test_support/escaped_character_reference.marc8.marc
 - test/test_support/george_eliot.marc
 - test/test_support/hebrew880s.marc
 - test/test_support/louis_armstrong.marc
@@ -281,12 +283,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.1.11
 signing_key:
 specification_version: 4
 summary: Index MARC to Solr; or generally process source records to hash-like structures
@@ -309,11 +311,13 @@ test_files:
 - test/test_support/245_no_ab.marc
 - test/test_support/880_with_no_6.utf8.marc
 - test/test_support/bad_subfield_code.marc
+- test/test_support/bad_utf_byte.utf8.marc
 - test/test_support/date_resort_to_260.marc
 - test/test_support/date_type_r_missing_date2.marc
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb
 - test/test_support/emptyish_record.marc
+- test/test_support/escaped_character_reference.marc8.marc
 - test/test_support/george_eliot.marc
 - test/test_support/hebrew880s.marc
 - test/test_support/louis_armstrong.marc