berkeley_library-tind 0.4.3 → 0.6.0

data/lib/berkeley_library/tind/mapping/util.rb

@@ -0,0 +1,117 @@
+require 'csv'
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module Util
+
+        class << self
+
+          def csv_rows(file)
+            rows = []
+            CSV.foreach(file, headers: true, header_converters: :symbol, encoding: 'bom|utf-8') do |row|
+              # puts row.headers
+              rows << clr_row(row)
+            end
+            rows
+          end
+
+          def concatenation_symbol(str)
+            str ? str.gsub('a', ' ') : ' '
+          end
+
+          def indicator(str)
+            return [] unless str
+
+            str.strip.gsub('_', ' ').split(',')
+          end
+
+          def tag_symbol(tag)
+            "tag_#{tag}".to_sym
+          end
+
+          def datafield(tag, indicator, subfields)
+            datafield = ::MARC::DataField.new(tag, indicator[0], indicator[1])
+            subfields.each { |sf| datafield.append(sf) }
+            datafield
+          end
+
+          def subfield(code, value)
+            ::MARC::Subfield.new(code, value)
+          end
+
+          def subfield_hash(field)
+            code_value_arr = field.to_hash[field.tag]['subfields']
+            {}.tap { |i| code_value_arr.each(&i.method(:update)) }
+          end
+
+          # input an array of rules, example: [["a,b,c,d", "b", "--"],["o,p,q", "b", ""]]
+          def symbols(rules)
+            rules.map { |rule| concatenation_symbol(rule[2]).strip }
+          end
+
+          def remove_extra_symbol(rules, val)
+            symbols = symbols(rules)
+            symbols.each { |s| val = val.strip.delete_suffix(s) }
+            val
+          end
+
+          def alma_datafield(tag, record)
+            record.fields.each { |f| return f if f.tag.to_s == tag }
+            nil
+          end
+
+          def qualified_alma_record?(alma_record)
+            f_245 = alma_datafield('245', alma_record)
+            f_245_a = f_245['a'].downcase
+
+            val = 'Host bibliographic record'.downcase
+            !f_245_a.start_with? val
+          end
+
+          # From DM - get testing MARC from xml file
+          # @param xml [String] the XML to parse
+          # @return [MARC::Record, nil] the MARC record from the specified XML
+          def from_xml(xml)
+            # noinspection RubyYardReturnMatch,RubyMismatchedReturnType
+            all_from_xml(xml).first
+          end
+
+          private
+
+          def clr_row(row)
+            row[:tag_origin] = format_tag(row[:tag_origin])
+            row[:tag_destination] = format_tag(row[:tag_destination])
+            row[:map_if_no_this_tag_existed] = format_tag(row[:map_if_no_this_tag_existed])
+            row
+          end
+
+          # To ensure tag from csv file in three digits
+          def format_tag(tag)
+            # LDR is not a numeric tag
+            return tag unless numeric? tag
+
+            format('%03d', tag)
+          end
+
+          def numeric?(str)
+            return false if str.nil?
+
+            str.scan(/\D/).empty?
+          end
+
+          # Parses MARCXML.
+          #
+          # @param xml [String] the XML to parse
+          # @return [MARC::XMLReader] the MARC records
+          def all_from_xml(xml)
+            input = StringIO.new(xml.scrub)
+            MARC::XMLReader.new(input)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping.rb

@@ -0,0 +1 @@
+Dir.glob(File.expand_path('mapping/*.rb', __dir__)).sort.each(&method(:require))

data/lib/berkeley_library/tind/marc/xml_builder.rb

@@ -0,0 +1,70 @@
+require 'nokogiri'
+
+module BerkeleyLibrary
+  module TIND
+    module MARC
+      class XMLBuilder
+        attr_reader :marc_record
+
+        def initialize(marc_record)
+          @marc_record = marc_record
+        end
+
+        def build
+          builder.doc.root.tap(&:unlink)
+        end
+
+        private
+
+        def builder
+          Nokogiri::XML::Builder.new do |xml|
+            xml.record do
+              add_leader(xml)
+              marc_record.each { |f| add_field(xml, f) }
+            end
+          end
+        end
+
+        def add_leader(xml)
+          leader = marc_record.leader
+          return if leader.nil? || leader == ''
+
+          # TIND uses <controlfield tag="000"/> instead of <leader/>
+          leader_as_cf = ::MARC::ControlField.new('000', clean_leader(leader))
+          add_control_field(xml, leader_as_cf)
+        end
+
+        def add_data_field(xml, df)
+          xml.datafield(tag: df.tag, ind1: df.indicator1, ind2: df.indicator2) do
+            df.subfields.each do |sf|
+              xml.subfield(sf.value, code: sf.code)
+            end
+          end
+        end
+
+        def add_control_field(xml, cf)
+          # TIND uses \ (0x5c), not space (0x32), for unspecified values in positional fields
+          value = cf.value&.gsub(' ', '\\')
+          xml.controlfield(value, tag: cf.tag)
+        end
+
+        def add_field(xml, f)
+          case f
+          when ::MARC::ControlField
+            add_control_field(xml, f)
+          when ::MARC::DataField
+            add_data_field(xml, f)
+          end
+        end
+
+        def clean_leader(leader)
+          leader.gsub(/[^\w|^\s]/, 'Z').tap do |ldr|
+            ldr[20..23] = '4500' unless ldr[20..23] == '4500'
+            ldr[6..6] = 'Z' if ldr[6..6] == ' '
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/marc/xml_reader.rb

@@ -1,6 +1,7 @@
 require 'nokogiri'
 require 'marc/xml_parsers'
 require 'marc_extensions'
+require 'berkeley_library/util/files'
 
 module BerkeleyLibrary
   module TIND
@@ -9,7 +10,7 @@ module BerkeleyLibrary
       class XMLReader
         include Enumerable
         include ::MARC::NokogiriReader
-        prepend MARCExtensions::YieldFrozenRecord
+        include BerkeleyLibrary::Util::Files
 
         # ############################################################
         # Constant
@@ -70,6 +71,12 @@ module BerkeleyLibrary
         # MARC::GenericPullParser overrides
 
         def yield_record
+          @record[:record].tap do |record|
+            clean_cf_values(record)
+            move_cf000_to_leader(record)
+            record.freeze if @freeze
+          end
+
           super
         ensure
           increment_records_yielded!
@@ -120,26 +127,25 @@ module BerkeleyLibrary
 
         private
 
-        def ensure_io(file)
-          return file if io_like?(file)
-          return File.new(file) if file_exists?(file)
-          return StringIO.new(file) if file =~ /^\s*</x
+        # TIND uses <controlfield tag="000"/> instead of <leader/>
+        def move_cf000_to_leader(record)
+          return unless (cf_000 = record['000'])
 
-          raise ArgumentError, "Don't know how to read XML from #{file.inspect}: not an IO, file path, or XML text"
+          record.leader = cf_000.value
+          record.fields.delete(cf_000)
         end
 
-        # Returns true if `obj` is close enough to an IO object for Nokogiri
-        # to parse as one.
-        #
-        # @param obj [Object] the object that might be an IO
-        # @see https://github.com/sparklemotion/nokogiri/blob/v1.11.1/lib/nokogiri/xml/sax/parser.rb#L81 Nokogiri::XML::SAX::Parser#parse
-        def io_like?(obj)
-          obj.respond_to?(:read) && obj.respond_to?(:close)
+        # TIND uses \ (0x5c), not space (0x32), for unspecified values in positional fields
+        def clean_cf_values(record)
+          record.each_control_field { |cf| cf.value = cf.value&.gsub('\\', ' ') }
         end
 
-        def file_exists?(path)
-          (path.respond_to?(:exist?) && path.exist?) ||
-            (path.respond_to?(:to_str) && File.exist?(path))
+        def ensure_io(file)
+          return file if reader_like?(file)
+          return File.new(file) if file_exists?(file)
+          return StringIO.new(file) if file =~ /^\s*</x
+
+          raise ArgumentError, "Don't know how to read XML from #{file.inspect}: not an IO, file path, or XML text"
         end
 
         def increment_records_yielded!

data/lib/berkeley_library/tind/marc/xml_writer.rb

@@ -0,0 +1,152 @@
+require 'nokogiri'
+require 'marc_extensions'
+require 'berkeley_library/tind/marc/xml_builder'
+
+module BerkeleyLibrary
+  module TIND
+    module MARC
+      class XMLWriter
+        include BerkeleyLibrary::Util::Files
+        include BerkeleyLibrary::Logging
+
+        # ------------------------------------------------------------
+        # Constants
+
+        UTF_8 = Encoding::UTF_8.name
+
+        EMPTY_COLLECTION_DOC = Nokogiri::XML::Builder.new(encoding: UTF_8) do |xml|
+          xml.collection(xmlns: ::MARC::MARC_NS)
+        end.doc.freeze
+
+        COLLECTION_CLOSING_TAG = '</collection>'.freeze
+
+        DEFAULT_NOKOGIRI_OPTS = { encoding: UTF_8 }.freeze
+
+        # ------------------------------------------------------------
+        # Fields
+
+        attr_reader :out
+        attr_reader :nokogiri_options
+
+        # ------------------------------------------------------------
+        # Initializer
+
+        # Initializes a new {XMLWriter}.
+        #
+        # ```ruby
+        # File.open('marc.xml', 'wb') do |f|
+        #   w = XMLWriter.new(f)
+        #   marc_records.each { |r| w.write(r) }
+        #   w.close
+        # end
+        # ```
+        #
+        # @param out [IO, String] an IO, or the name of a file
+        # @param nokogiri_options [Hash] Options passed to
+        #   {https://nokogiri.org/rdoc/Nokogiri/XML/Node.html#method-i-write_to Nokogiri::XML::Node#write_to}
+        #   Note that the `encoding` option is ignored, except insofar as
+        #   passing an encoding other than UTF-8 will raise an `ArgumentError`.
+        # @raise ArgumentError if `out` is not an IO or a string, or is a string referencing
+        #   a file path that cannot be opened for writing; or if an encoding other than UTF-8
+        #   is specified in `nokogiri-options`
+        # @see #open
+        def initialize(out, **nokogiri_options)
+          @nokogiri_options = valid_nokogiri_options(nokogiri_options)
+          @out = ensure_io(out)
+        end
+
+        # ------------------------------------------------------------
+        # Class methods
+
+        class << self
+
+          # Opens a new {XMLWriter} with the specified output destination and
+          # Nokogiri options, writes the XML prolog and opening `<collection>`
+          # tag, yields the writer to write one or more MARC records, and closes
+          # the writer.
+          #
+          # ```ruby
+          # XMLWriter.open('marc.xml') do |w|
+          #   marc_records.each { |r| w.write(r) }
+          # end
+          # ```
+          #
+          # Note that unlike initializing a writer with {#new} and closing it
+          # immediately, this will write an XML document with an empty
+          # `<collection></collection>` tag even if no records are written.
+          #
+          # @yieldparam writer [XMLWriter] the writer
+          # @see #new
+          # @see #close
+          def open(out, **nokogiri_options)
+            writer = new(out, **nokogiri_options)
+            writer.send(:ensure_open!)
+            yield writer if block_given?
+            writer.close
+          end
+        end
+
+        # ------------------------------------------------------------
+        # Instance methods
+
+        # Writes the specified record to the underlying stream, writing the
+        # XML prolog and opening `<collection>` tag if they have not yet
+        # been written.
+        #
+        # @param record [::MARC::Record] the MARC record to write.
+        # @raise IOError if the underlying stream has already been closed.
+        def write(record)
+          ensure_open!
+          record_element = XMLBuilder.new(record).build
+          record_element.write_to(out, nokogiri_options)
+          out.write("\n")
+        end
+
+        # Closes the underlying stream. If the XML prolog and opening `<collection>`
+        # tag have already been written, the closing `<collection/>` tag is written
+        # first.
+        def close
+          out.write(COLLECTION_CLOSING_TAG) if @open
+          out.close
+        end
+
+        # ------------------------------------------------------------
+        # Private
+
+        private
+
+        def ensure_open!
+          return if @open
+
+          out.write(prolog_and_opening_tag)
+          @open = true
+        end
+
+        def prolog_and_opening_tag
+          StringIO.open do |tmp|
+            EMPTY_COLLECTION_DOC.write_to(tmp, nokogiri_options)
+            result = tmp.string
+            result.sub!(%r{/>\s*$}, ">\n")
+            result
+          end
+        end
+
+        def ensure_io(file)
+          return file if writer_like?(file)
+          return File.open(file, 'wb') if parent_exists?(file)
+
+          raise ArgumentError, "Don't know how to write XML to #{file.inspect}: not an IO or file path"
+        end
+
+        def valid_nokogiri_options(opts)
+          if (encoding = opts.delete(:encoding)) && encoding != UTF_8
+            raise ArgumentError, "#{self.class.name} only supports #{UTF_8}; unable to use specified encoding #{encoding}"
+          end
+
+          DEFAULT_NOKOGIRI_OPTS.merge(opts)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/module_info.rb

@@ -7,7 +7,7 @@ module BerkeleyLibrary
       SUMMARY = 'TIND DA utilities for the UC Berkeley Library'.freeze
       DESCRIPTION = 'UC Berkeley Library utility gem for working with the TIND DA digital archive.'.freeze
       LICENSE = 'MIT'.freeze
-      VERSION = '0.4.3'.freeze
+      VERSION = '0.6.0'.freeze
       HOMEPAGE = 'https://github.com/BerkeleyLibrary/tind'.freeze
     end
   end

data/lib/berkeley_library/util/files.rb

@@ -0,0 +1,38 @@
+module BerkeleyLibrary
+  module Util
+    # TODO: Move this to `berkeley_library-util`
+    module Files
+      class << self
+        include Files
+      end
+
+      def file_exists?(path)
+        (path.respond_to?(:exist?) && path.exist?) ||
+          (path.respond_to?(:to_str) && File.exist?(path))
+      end
+
+      def parent_exists?(path)
+        path.respond_to?(:parent) && path.parent.exist? ||
+          path.respond_to?(:to_str) && Pathname.new(path).parent.exist?
+      end
+
+      # Returns true if `obj` is close enough to an IO object for Nokogiri
+      # to parse as one.
+      #
+      # @param obj [Object] the object that might be an IO
+      # @see https://github.com/sparklemotion/nokogiri/blob/v1.11.1/lib/nokogiri/xml/sax/parser.rb#L81 Nokogiri::XML::SAX::Parser#parse
+      def reader_like?(obj)
+        obj.respond_to?(:read) && obj.respond_to?(:close)
+      end
+
+      # Returns true if `obj` is close enough to an IO object for Nokogiri
+      # to write to.
+      #
+      # @param obj [Object] the object that might be an IO
+      def writer_like?(obj)
+        obj.respond_to?(:write) && obj.respond_to?(:close)
+      end
+
+    end
+  end
+end

data/spec/berkeley_library/tind/mapping/additional_datafield_process_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      describe AdditionalDatafieldProcess do
+
+        let(:qualified_alma_obj) { Alma.new('spec/data/mapping/record.xml') }
+        let(:qualified_alm_record) { qualified_alma_obj.record }
+
+        let(:tind_marc) { TindMarc.new(qualified_alm_record) }
+        let(:tindfields) { tind_marc.tindfields }
+        let(:additona_245_field) { [Util.datafield('245', [' ', ' '], [Util.subfield('a', 'fake 245 a')])] }
+        let(:with_repeated_fields) { tindfields.concat additona_245_field }
+
+        context '# Process 1: Remove duplicated fields' do
+          # two 245 fields inputed, one is removed
+          it 'Input two 245 fields, return only one 245 field' do
+            expect(tind_marc.remove_repeats(with_repeated_fields).map(&:tag).count('245')).to eq 1
+          end
+
+          # one subfield 'a' is kept, subfield 'a' with a value of 'fake 245 a' is ignored
+          it 'Input two subfield $a, keeping the first one' do
+            without_repeated_fields = tind_marc.remove_repeats(with_repeated_fields)
+            single_field_245 = tind_marc.field_on_tag('245', without_repeated_fields)
+            expect(single_field_245['a']).to eq 'Cang jie pian'
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/spec/berkeley_library/tind/mapping/alma_base_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+require 'tempfile'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      describe AlmaBase do
+        extend BerkeleyLibrary::Logging
+        let(:dummy_obj) { Class.new { extend AlmaBase } }
+        let(:alma_record) { ::MARC::Record.new }
+        let(:save_to_file) { Tempfile.new('xml') }
+        let(:record_id) { Class.new { extend BerkeleyLibrary::Alma::RecordId } }
+        let(:hash) { { '980' => ['pre_1912'] } }
+
+        let(:qualified_alma_obj) { Alma.new('spec/data/mapping/record.xml') }
+        let(:qualified_alm_record) { qualified_alma_obj.record }
+
+        let(:un_qualified_alma_obj) { Alma.new('spec/data/mapping/record_not_qualified.xml') }
+        let(:un_qualified_alm_record) { un_qualified_alma_obj.record }
+
+        describe '# base_tind_record' do
+          it 'input a qualified Alma record - return a tind record' do
+            allow(dummy_obj).to receive(:tind_record).with('C084093187', qualified_alm_record, []).and_return(::MARC::Record.new)
+            expect(dummy_obj.base_tind_record('C084093187', [], qualified_alm_record)).to be_a ::MARC::Record
+          end
+
+          it 'input an unqualified Alma record - return nil' do
+            expect { dummy_obj.base_tind_record('C084093187', [], un_qualified_alm_record) }.to raise_error(ArgumentError)
+          end
+
+          it 'no input Alma record with a nil (record) from id  - return nil' do
+            allow(dummy_obj).to receive(:alma_record_from).with('C084093187').and_return(nil)
+            expect { dummy_obj.base_tind_record('C084093187', []) }.to raise_error(ArgumentError)
+          end
+        end
+
+        describe '# base_save' do
+          it 'save tind record' do
+            dummy_obj.base_save('C084093187', qualified_alm_record, save_to_file)
+            expect(File.open(save_to_file.path).readlines[0]).to eq "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+          end
+        end
+
+        describe '# alma_record' do
+          it 'get Alma record' do
+            allow(record_id).to receive(:get_marc_record).and_return(::MARC::Record.new)
+            allow(dummy_obj).to receive(:get_record_id).with('C084093187').and_return(record_id)
+            expect(dummy_obj.send(:alma_record_from, 'C084093187')).to be_instance_of ::MARC::Record
+          end
+        end
+
+        describe '# get_record_id' do
+          it 'return BerkeleyLibrary::Alma::BarCode' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_barcode = true
+            expect(dummy_obj.send(:get_record_id, 'C084093187')).to be_instance_of BerkeleyLibrary::Alma::BarCode
+          end
+
+          it 'return BerkeleyLibrary::Alma::MMSID' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_barcode = false
+            expect(dummy_obj.send(:get_record_id, '991085821143406532')).to be_instance_of BerkeleyLibrary::Alma::MMSID
+
+          end
+
+          it 'return BerkeleyLibrary::Alma::BibNumber' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_barcode = false
+            expect(dummy_obj.send(:get_record_id, 'b11082434')).to be_instance_of BerkeleyLibrary::Alma::BibNumber
+
+          end
+        end
+
+        describe '# add_f_035' do
+          it 'Add 035 from mms_id' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_035_from_mms_id = true
+            expect(dummy_obj.send(:add_f_035, '991085821143406532', hash).tag).to eq '035'
+          end
+
+          it 'Not to add 035 from mms_id' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_035_from_mms_id = false
+            expect(dummy_obj.send(:add_f_035, '991085821143406532', hash)).to eq nil
+          end
+        end
+
+        describe '# tind_record, # derived_tind_fields' do
+          BerkeleyLibrary::TIND::Mapping::AlmaBase.collection_parameter_hash = {
+            '336' => ['Image'],
+            '852' => ['East Asian Library'],
+            '980' => ['pre_1912'],
+            '982' => ['Pre 1912 Chinese Materials - short name', 'Pre 1912 Chinese Materials - long name'],
+            '991' => []
+          }
+          let(:id) { '991085821143406532' }
+          let(:tags) { %w[902 336 852 980 982 901 856] }
+          let(:tags_with_035) { %w[902 336 852 980 982 901 856 035] }
+
+          it ' get tind_record' do
+            datafields = []
+            marc_record = qualified_alm_record
+            expect(dummy_obj.send(:tind_record, id, marc_record, datafields)).to be_instance_of ::MARC::Record
+          end
+
+          it 'get derived fields without 035' do
+            expect(dummy_obj.send(:derived_tind_fields, id).map(&:tag)).to eq tags
+          end
+
+          it 'get derived fields with 035' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_035_from_mms_id = true
+            expect(dummy_obj.send(:derived_tind_fields, id).map(&:tag)).to eq tags_with_035
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/spec/berkeley_library/tind/mapping/alma_multiple_tind_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      describe AlmaMultipleTIND do
+        let(:additona_245_field) { [Util.datafield('245', [' ', ' '], [Util.subfield('a', 'fake 245 a')])] }
+        let(:marc_obj) { (::MARC::Record.new).append(additona_245_field) }
+
+        it ' get tind record' do
+          allow_any_instance_of(BerkeleyLibrary::TIND::Mapping::AlmaMultipleTIND).to receive(:alma_record_from).with('991085821143406532').and_return(marc_obj)
+          alma_multiple_tind = BerkeleyLibrary::TIND::Mapping::AlmaMultipleTIND.new('991085821143406532')
+
+          allow(alma_multiple_tind).to receive(:base_tind_record).with('991085821143406532', additona_245_field, marc_obj).and_return(marc_obj)
+          expect(alma_multiple_tind.record(additona_245_field)).to be marc_obj
+        end
+      end
+    end
+  end
+end