berkeley_library-tind 0.4.0 → 0.5.0

data/lib/berkeley_library/tind/marc/xml_reader.rb

@@ -1,6 +1,7 @@
 require 'nokogiri'
 require 'marc/xml_parsers'
 require 'marc_extensions'
+require 'berkeley_library/util/files'
 
 module BerkeleyLibrary
   module TIND
@@ -9,6 +10,7 @@ module BerkeleyLibrary
       class XMLReader
         include Enumerable
         include ::MARC::NokogiriReader
+        include BerkeleyLibrary::Util::Files
 
         # ############################################################
         # Constant
@@ -43,10 +45,10 @@ module BerkeleyLibrary
         # ############################################################
         # Initializer
 
-        # Reads MARC records from an XML datasource given either as a file path,
+        # Reads MARC records from an XML datasource given either as an XML string, a file path,
         # or as an IO object.
         #
-        # @param source [String, Pathname, IO] the path to a file, or an IO to read from directly
+        # @param source [String, Pathname, IO] an XML string, the path to a file, or an IO to read from directly
         # @param freeze [Boolean] whether to freeze each record after reading
         def initialize(source, freeze: false)
           @handle = ensure_io(source)
@@ -55,14 +57,26 @@ module BerkeleyLibrary
         end
 
         class << self
-          include MARCExtensions::XMLReaderClassExtensions
+          # Reads MARC records from an XML datasource given either as an XML string, a file path,
+          # or as an IO object.
+          #
+          # @param source [String, Pathname, IO] an XML string, the path to a file, or an IO to read from directly
+          # @param freeze [Boolean] whether to freeze each record after reading
+          def read(source, freeze: false)
+            new(source, freeze: freeze)
+          end
         end
 
         # ############################################################
         # MARC::GenericPullParser overrides
 
         def yield_record
-          @record[:record].freeze if @freeze
+          @record[:record].tap do |record|
+            clean_cf_values(record)
+            move_cf000_to_leader(record)
+            record.freeze if @freeze
+          end
+
           super
         ensure
           increment_records_yielded!
@@ -113,26 +127,25 @@ module BerkeleyLibrary
 
         private
 
-        def ensure_io(file)
-          return file if io_like?(file)
-          return File.new(file) if file_exists?(file)
-          return StringIO.new(file) if file =~ /^\s*</x
+        # TIND uses <controlfield tag="000"/> instead of <leader/>
+        def move_cf000_to_leader(record)
+          return unless (cf_000 = record['000'])
 
-          raise ArgumentError, "Don't know how to read XML from #{file.inspect}: not an IO, file path, or XML text"
+          record.leader = cf_000.value
+          record.fields.delete(cf_000)
         end
 
-        # Returns true if `obj` is close enough to an IO object for Nokogiri
-        # to parse as one.
-        #
-        # @param obj [Object] the object that might be an IO
-        # @see https://github.com/sparklemotion/nokogiri/blob/v1.11.1/lib/nokogiri/xml/sax/parser.rb#L81 Nokogiri::XML::SAX::Parser#parse
-        def io_like?(obj)
-          obj.respond_to?(:read) && obj.respond_to?(:close)
+        # TIND uses \ (0x5c), not space (0x32), for unspecified values in positional fields
+        def clean_cf_values(record)
+          record.each_control_field { |cf| cf.value = cf.value&.gsub('\\', ' ') }
         end
 
-        def file_exists?(path)
-          (path.respond_to?(:exist?) && path.exist?) ||
-            (path.respond_to?(:to_str) && File.exist?(path))
+        def ensure_io(file)
+          return file if reader_like?(file)
+          return File.new(file) if file_exists?(file)
+          return StringIO.new(file) if file =~ /^\s*</x
+
+          raise ArgumentError, "Don't know how to read XML from #{file.inspect}: not an IO, file path, or XML text"
         end
 
         def increment_records_yielded!

data/lib/berkeley_library/tind/marc/xml_writer.rb

@@ -0,0 +1,152 @@
+require 'nokogiri'
+require 'marc_extensions'
+require 'berkeley_library/tind/marc/xml_builder'
+
+module BerkeleyLibrary
+  module TIND
+    module MARC
+      class XMLWriter
+        include BerkeleyLibrary::Util::Files
+        include BerkeleyLibrary::Logging
+
+        # ------------------------------------------------------------
+        # Constants
+
+        UTF_8 = Encoding::UTF_8.name
+
+        EMPTY_COLLECTION_DOC = Nokogiri::XML::Builder.new(encoding: UTF_8) do |xml|
+          xml.collection(xmlns: ::MARC::MARC_NS)
+        end.doc.freeze
+
+        COLLECTION_CLOSING_TAG = '</collection>'.freeze
+
+        DEFAULT_NOKOGIRI_OPTS = { encoding: UTF_8 }.freeze
+
+        # ------------------------------------------------------------
+        # Fields
+
+        attr_reader :out
+        attr_reader :nokogiri_options
+
+        # ------------------------------------------------------------
+        # Initializer
+
+        # Initializes a new {XMLWriter}.
+        #
+        # ```ruby
+        # File.open('marc.xml', 'wb') do |f|
+        #   w = XMLWriter.new(f)
+        #   marc_records.each { |r| w.write(r) }
+        #   w.close
+        # end
+        # ```
+        #
+        # @param out [IO, String] an IO, or the name of a file
+        # @param nokogiri_options [Hash] Options passed to
+        #   {https://nokogiri.org/rdoc/Nokogiri/XML/Node.html#method-i-write_to Nokogiri::XML::Node#write_to}
+        #   Note that the `encoding` option is ignored, except insofar as
+        #   passing an encoding other than UTF-8 will raise an `ArgumentError`.
+        # @raise ArgumentError if `out` is not an IO or a string, or is a string referencing
+        #   a file path that cannot be opened for writing; or if an encoding other than UTF-8
+        #   is specified in `nokogiri-options`
+        # @see #open
+        def initialize(out, **nokogiri_options)
+          @nokogiri_options = valid_nokogiri_options(nokogiri_options)
+          @out = ensure_io(out)
+        end
+
+        # ------------------------------------------------------------
+        # Class methods
+
+        class << self
+
+          # Opens a new {XMLWriter} with the specified output destination and
+          # Nokogiri options, writes the XML prolog and opening `<collection>`
+          # tag, yields the writer to write one or more MARC records, and closes
+          # the writer.
+          #
+          # ```ruby
+          # XMLWriter.open('marc.xml') do |w|
+          #   marc_records.each { |r| w.write(r) }
+          # end
+          # ```
+          #
+          # Note that unlike initializing a writer with {#new} and closing it
+          # immediately, this will write an XML document with an empty
+          # `<collection></collection>` tag even if no records are written.
+          #
+          # @yieldparam writer [XMLWriter] the writer
+          # @see #new
+          # @see #close
+          def open(out, **nokogiri_options)
+            writer = new(out, **nokogiri_options)
+            writer.send(:ensure_open!)
+            yield writer if block_given?
+            writer.close
+          end
+        end
+
+        # ------------------------------------------------------------
+        # Instance methods
+
+        # Writes the specified record to the underlying stream, writing the
+        # XML prolog and opening `<collection>` tag if they have not yet
+        # been written.
+        #
+        # @param record [::MARC::Record] the MARC record to write.
+        # @raise IOError if the underlying stream has already been closed.
+        def write(record)
+          ensure_open!
+          record_element = XMLBuilder.new(record).build
+          record_element.write_to(out, nokogiri_options)
+          out.write("\n")
+        end
+
+        # Closes the underlying stream. If the XML prolog and opening `<collection>`
+        # tag have already been written, the closing `<collection/>` tag is written
+        # first.
+        def close
+          out.write(COLLECTION_CLOSING_TAG) if @open
+          out.close
+        end
+
+        # ------------------------------------------------------------
+        # Private
+
+        private
+
+        def ensure_open!
+          return if @open
+
+          out.write(prolog_and_opening_tag)
+          @open = true
+        end
+
+        def prolog_and_opening_tag
+          StringIO.open do |tmp|
+            EMPTY_COLLECTION_DOC.write_to(tmp, nokogiri_options)
+            result = tmp.string
+            result.sub!(%r{/>\s*$}, ">\n")
+            result
+          end
+        end
+
+        def ensure_io(file)
+          return file if writer_like?(file)
+          return File.open(file, 'wb') if parent_exists?(file)
+
+          raise ArgumentError, "Don't know how to write XML to #{file.inspect}: not an IO or file path"
+        end
+
+        def valid_nokogiri_options(opts)
+          if (encoding = opts.delete(:encoding)) && encoding != UTF_8
+            raise ArgumentError, "#{self.class.name} only supports #{UTF_8}; unable to use specified encoding #{encoding}"
+          end
+
+          DEFAULT_NOKOGIRI_OPTS.merge(opts)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/module_info.rb

@@ -7,7 +7,7 @@ module BerkeleyLibrary
       SUMMARY = 'TIND DA utilities for the UC Berkeley Library'.freeze
       DESCRIPTION = 'UC Berkeley Library utility gem for working with the TIND DA digital archive.'.freeze
       LICENSE = 'MIT'.freeze
-      VERSION = '0.4.0'.freeze
+      VERSION = '0.5.0'.freeze
       HOMEPAGE = 'https://github.com/BerkeleyLibrary/tind'.freeze
     end
   end

data/lib/berkeley_library/util/files.rb

@@ -0,0 +1,39 @@
+module BerkeleyLibrary
+  module Util
+    # TODO: Move this to `berkeley_library-util`
+    module Files
+      class << self
+        include Files
+      end
+
+      def file_exists?(path)
+        (path.respond_to?(:exist?) && path.exist?) ||
+          (path.respond_to?(:to_str) && File.exist?(path))
+      end
+
+      def parent_exists?(path)
+        path.respond_to?(:parent) && path.parent.exist? ||
+          path.respond_to?(:to_str) && Pathname.new(path).parent.exist?
+      end
+
+      # Returns true if `obj` is close enough to an IO object for Nokogiri
+      # to parse as one.
+      #
+      # @param obj [Object] the object that might be an IO
+      # @see https://github.com/sparklemotion/nokogiri/blob/v1.11.1/lib/nokogiri/xml/sax/parser.rb#L81 Nokogiri::XML::SAX::Parser#parse
+      def reader_like?(obj)
+        obj.respond_to?(:read) && obj.respond_to?(:close)
+      end
+
+      # Returns true if `obj` is close enough to an IO object for Nokogiri
+      # to write to.
+      #
+      # @param obj [Object] the object that might be an IO
+      def writer_like?(obj)
+        # TODO: is it possible/desirable to loosen this? how strict is libxml2?
+        obj.is_a?(IO) || obj.is_a?(StringIO)
+      end
+
+    end
+  end
+end

data/lib/berkeley_library/util/ods/spreadsheet.rb

@@ -1,6 +1,6 @@
 require 'fileutils'
 require 'zip'
-require 'berkeley_library/util/logging'
+require 'berkeley_library/logging'
 require 'berkeley_library/util/ods/xml/content_doc'
 require 'berkeley_library/util/ods/xml/styles_doc'
 require 'berkeley_library/util/ods/xml/manifest_doc'

data/lib/berkeley_library/util/ods/xml/element_node.rb

@@ -1,5 +1,5 @@
 require 'nokogiri'
-require 'berkeley_library/util/logging'
+require 'berkeley_library/logging'
 require 'berkeley_library/util/ods/xml/namespace'
 
 module BerkeleyLibrary

data/spec/berkeley_library/tind/export/export_spec.rb

@@ -130,7 +130,9 @@ module BerkeleyLibrary
 
         before(:each) do
           search = instance_double(BerkeleyLibrary::TIND::API::Search)
-          empty_enumerator = Enumerator.new({})
+          # rubocop:disable Lint/EmptyBlock
+          empty_enumerator = Enumerator.new {}
+          # rubocop:enable Lint/EmptyBlock
           allow(search).to receive(:each_result).and_return(empty_enumerator)
           allow(BerkeleyLibrary::TIND::API::Search).to receive(:new).with(collection: collection).and_return(search)
         end

data/spec/berkeley_library/tind/export/table_spec.rb

@@ -24,6 +24,8 @@ module BerkeleyLibrary
             tag = '245'
             ind_bad = '!'
 
+            records = MARC::XMLReader.read('spec/data/records-manual-search.xml', freeze: false).to_a
+
             record = records.first
             record[tag].indicator1 = ind_bad
             expect { table << record }.to raise_error(Export::ExportException) do |e|

data/spec/berkeley_library/tind/marc/xml_reader_spec.rb

@@ -16,6 +16,25 @@ module BerkeleyLibrary
           expect(record0['024']['a']).to eq('BANC PIC 1982.078:15--ALB')
         end
 
+        describe 'freeze: true' do
+          it 'freezes the records' do
+            reader = XMLReader.new('spec/data/records-api-search.xml', freeze: true)
+            records = reader.to_a
+            expect(records).not_to be_empty # just to be sure
+            records.each { |record| expect(record).to be_frozen }
+          end
+        end
+
+        describe :records_yielded do
+          it 'counts the records' do
+            reader = XMLReader.new('spec/data/records-api-search.xml')
+            reader.each_with_index do |_, i|
+              expect(reader.records_yielded).to eq(i)
+            end
+            expect(reader.records_yielded).to eq(5)
+          end
+        end
+
         describe :new do
           it 'accepts a string path' do
             path = 'spec/data/records-api-search.xml'
@@ -59,7 +78,7 @@ module BerkeleyLibrary
 
           it 'raises ArgumentError if passed something random' do
             non_xml = Object.new
-            # noinspection RubyYardParamTypeMatch
+            # noinspection RubyMismatchedArgumentType
             expect { XMLReader.new(non_xml) }.to raise_error(ArgumentError)
           end
         end
@@ -87,6 +106,28 @@ module BerkeleyLibrary
           end
         end
 
+        describe 'TIND peculiarities' do
+          attr_reader :record
+
+          before(:each) do
+            reader = XMLReader.new('spec/data/new-records.xml')
+            records = reader.to_a
+            expect(records.size).to eq(1) # just to be sure
+            @record = records.first
+          end
+
+          it 'converts backslashes in control fields to spaces' do
+            cf_008 = record['008']
+            expect(cf_008).to be_a(::MARC::ControlField)
+            expect(cf_008.value).to eq('190409s2015    xx                  eng  ')
+          end
+
+          it 'parses CF 000 as the leader' do
+            expect(record.leader).to eq('00287cam a2200313   4500')
+            expect(record['000']).to be_nil
+          end
+        end
+
       end
     end
   end

data/spec/berkeley_library/tind/marc/xml_writer_spec.rb

@@ -0,0 +1,156 @@
+require 'spec_helper'
+require 'equivalent-xml'
+
+module BerkeleyLibrary
+  module TIND
+    module MARC
+      describe XMLWriter do
+        let(:input_path) { 'spec/data/new-records.xml' }
+        attr_reader :record
+
+        before(:each) do
+          reader = XMLReader.new(input_path)
+          @record = reader.first
+        end
+
+        describe :open do
+
+          it 'writes a MARC record to a file as XML' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              output_path = File.join(dir, 'marc.xml')
+              XMLWriter.open(output_path) { |w| w.write(record) }
+
+              expected = File.open(input_path) { |f| Nokogiri::XML(f) }
+              actual = File.open(output_path) { |f| Nokogiri::XML(f) }
+
+              aggregate_failures do
+                EquivalentXml.equivalent?(expected, actual) do |n1, n2, result|
+                  expect(n2.to_s).to eq(n1.to_s) unless result
+                end
+              end
+            end
+          end
+
+          it 'writes a MARC record to a StringIO' do
+            out = StringIO.new
+            XMLWriter.open(out) { |w| w.write(record) }
+            expected = File.open(input_path) { |f| Nokogiri::XML(f) }
+            actual = Nokogiri::XML(out.string)
+            aggregate_failures do
+              EquivalentXml.equivalent?(expected, actual) do |n1, n2, result|
+                expect(n2.to_s).to eq(n1.to_s) unless result
+              end
+            end
+          end
+
+          it 'accepts Nokogiri options' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              expected_path = File.join(dir, 'expected.xml')
+              XMLWriter.open(expected_path) { |w| w.write(record) }
+
+              actual_path = File.join(dir, 'actual.xml')
+              XMLWriter.open(actual_path, indent_text: "\t") { |w| w.write(record) }
+
+              expected = File.read(expected_path).gsub(%r{ (?= *<)(?!/)}, "\t")
+              actual = File.read(actual_path)
+              expect(actual).to eq(expected)
+            end
+          end
+
+          it 'accepts an explicit UTF-8 argument' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              output_path = File.join(dir, 'marc.xml')
+              XMLWriter.open(output_path, encoding: 'UTF-8') { |w| w.write(record) }
+
+              expected = File.open(input_path) { |f| Nokogiri::XML(f) }
+              actual = File.open(output_path) { |f| Nokogiri::XML(f) }
+
+              aggregate_failures do
+                EquivalentXml.equivalent?(expected, actual) do |n1, n2, result|
+                  expect(n2.to_s).to eq(n1.to_s) unless result
+                end
+              end
+            end
+          end
+
+          it 'only writes UTF-8' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              output_path = File.join(dir, 'marc.xml')
+              expect { XMLWriter.open(output_path, encoding: 'UTF-16') }.to raise_error(ArgumentError)
+              expect(File.exist?(output_path)).to eq(false)
+            end
+          end
+
+          it 'rejects an invalid file path' do
+            bad_directory = Dir.mktmpdir(File.basename(__FILE__, '.rb')) { |dir| dir }
+            expect(File.directory?(bad_directory)).to eq(false)
+            output_path = File.join(bad_directory, 'marc.xml')
+            expect { XMLWriter.open(output_path) }.to raise_error(ArgumentError)
+          end
+
+          it 'rejects a non-IO, non-String argument' do
+            invalid_target = Object.new
+            expect { XMLWriter.open(invalid_target) }.to raise_error(ArgumentError)
+          end
+        end
+
+        describe :close do
+          it 'closes without writing the closing tag if nothing has been written' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              output_path = File.join(dir, 'marc.xml')
+              w = XMLWriter.new(output_path)
+              w.close
+
+              stat = File.stat(output_path)
+              expect(stat.size).to eq(0)
+            end
+          end
+
+          it 'writes the closing tag if the opening tag has been written' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              output_path = File.join(dir, 'marc.xml')
+              XMLWriter.open(output_path)
+              expect(File.exist?(output_path)).to eq(true)
+
+              doc = File.open(output_path) { |f| Nokogiri::XML(f) }
+              expect(doc.root.name).to eq('collection')
+            end
+          end
+        end
+
+        describe :write do
+          it 'raises an IOError if the writer has already been closed' do
+            Dir.mktmpdir(File.basename(__FILE__, '.rb')) do |dir|
+              output_path = File.join(dir, 'marc.xml')
+              w = XMLWriter.new(output_path)
+              w.close
+
+              expect { w.write(record) }.to raise_error(IOError)
+
+              stat = File.stat(output_path)
+              expect(stat.size).to eq(0)
+            end
+          end
+
+          it 'does not write a nil leader' do
+            record.leader = nil
+            marc_xml = StringIO.open do |out|
+              XMLWriter.open(out) { |w| w.write(record) }
+              out.string
+            end
+            expect(marc_xml).not_to include('leader')
+          end
+
+          it 'does not write a blank leader' do
+            record.leader = ''
+            marc_xml = StringIO.open do |out|
+              XMLWriter.open(out) { |w| w.write(record) }
+              out.string
+            end
+            expect(marc_xml).not_to include('leader')
+          end
+        end
+      end
+    end
+  end
+end

data/spec/data/new-records.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+  Source: "Batch Uploader: Caveats, common errors and example metadata files", docs.tind.io
+-->
+<collection xmlns="http://www.loc.gov/MARC21/slim">
+  <record>
+
+    <!-- The Leader is encoded in the `000` control field.
+	 If you want to edit the leader in software such as
+	 MarcEdit, you will need to change these fields to a
+	 leader tag. Then, before import into the repository
+	 you will need to change the fields back to controlfields
+	 with tag `000`.
+    -->
+    <controlfield tag="000">00287cam\a2200313\\\4500</controlfield>
+
+    <!-- All whitespace in control fields need to be replaced with
+	 backspaces.
+    -->
+    <controlfield tag="008">190409s2015\\\\xx\\\\\\\\\\\\\\\\\\eng\\</controlfield>
+
+    <!-- Regular fields are encoded in datafield elements. -->
+    <datafield tag="100" ind1="0" ind2=" ">
+      <subfield code="a">Aristotle</subfield>
+      <subfield code="0">580897</subfield>
+    </datafield>
+    
+    <datafield tag="245" ind1="0" ind2="0">
+      <subfield code="a">Metaphysics</subfield>
+      <subfield code="c">Aristotle</subfield>
+    </datafield>
+    
+    <datafield tag="260" ind1=" " ind2=" ">
+      <subfield code="a">Narnia</subfield>
+      <subfield code="b">Fictive Books</subfield>
+      <subfield code="c">2015</subfield>
+    </datafield>
+
+    <!-- Make sure to include a collection when uploading new
+	 records, so that the record will be searchable.
+    -->
+    <datafield tag="980" ind1=" " ind2=" ">
+      <subfield code="a">BIB</subfield>
+    </datafield>
+  </record>
+</collection>