berkeley_library-tind 0.5.1 → 0.6.0

data/lib/berkeley_library/tind/mapping/tind_field_util.rb

@@ -0,0 +1,112 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module TindFieldUtil
+
+        # tag - regular alma field
+        # referred tag - got tag from subfield6 value of a 880 field
+        # nil rule caused by nil referred tag - eg. 880 subfild6 has a value in wrong format
+        def rule(field)
+          tag = origin_mapping_tag(field)
+          return nil unless tag
+
+          rules[Util.tag_symbol(tag)]
+        end
+
+        def tindfield_existed?(field, fields)
+          return false unless field_has_rule?(field)
+
+          field_rule = rule(field)
+          mapping_to_tag = field_rule.pre_existed_tag
+          return false unless mapping_to_tag
+
+          map_to_tag_existed_in_fields?(field, fields, mapping_to_tag)
+        end
+
+        # To check TIND datafield and the specific subfield  from rule existed
+        def tindfield_subfield_existed?(field, fields)
+          return false unless field_has_rule?(field)
+
+          field_rule = rule(field)
+          return false unless pre_existed_tag_subfield_in_rule?(field_rule)
+
+          tag_subfield = field_rule.pre_existed_tag_subfield
+          mapping_to_tag = tag_subfield[0]
+          return false unless map_to_tag_existed_in_fields?(field, fields, mapping_to_tag)
+
+          existed_datafield = field_pre_existed(mapping_to_tag, field, fields)
+          return false unless existed_datafield
+
+          subfield_name = tag_subfield[1]
+          existed_datafield[subfield_name] ? true : false
+        end
+
+        def field_880_on_subfield6_tag(tag, fields)
+          datafield_on_tag(tag, fields) { |f| referred_tag(f) == tag }
+        end
+
+        def field_on_tag(tag, fields)
+          datafield_on_tag(tag, fields) { |f| f.tag == tag }
+        end
+
+        private
+
+        def field_has_rule?(field)
+          field_rule = rule(field)
+          return false unless field_rule
+
+          true
+        end
+
+        def pre_existed_tag_subfield_in_rule?(rule)
+          tag_subfield = rule.pre_existed_tag_subfield
+          return false unless tag_subfield
+
+          return false unless tag_subfield.length == 2
+
+          true
+        end
+
+        def map_to_tag_existed_in_fields?(field, fields, mapping_to_tag)
+          existed_tags = if is_880_field?(field)
+                           tags_from_fields(fields) { |f| tag_from_880_subfield6(f) }
+                         else
+                           tags_from_fields(fields, &:tag)
+                         end
+
+          existed_tags.include? mapping_to_tag
+        end
+
+        # field, fields be both regular fields
+        # or field, fields be both 880 fields
+        # since a field may mapped to another one in TIND, mapping_to_tag is not always the same as field.tag
+        def field_pre_existed(mapping_to_tag, field, fields)
+          if is_880_field?(field)
+            field_880_on_subfield6_tag(mapping_to_tag,
+                                       fields)
+          else
+            field_on_tag(mapping_to_tag, fields)
+          end
+        end
+
+        def datafield_on_tag(tag, fields)
+          fields.find do |f|
+            yield(f, tag)
+          end
+        end
+
+        def tags_from_fields(fields, &block)
+          fields.map(&block)
+        end
+
+        def tag_from_880_subfield6(field)
+          field['6'].split('-')[0]
+        end
+
+      end
+
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/tind_marc.rb

@@ -0,0 +1,134 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class TindMarc
+        include CsvMapper
+        include Util
+        include TindSubfieldUtil
+        include Misc
+        include TindFieldUtil
+        include AdditionalDatafieldProcess
+        include BerkeleyLibrary::Logging
+        include MatchTindField
+
+        attr_accessor :source_marc_record
+        attr_writer :tind_external_datafields
+        attr_reader :field_catalog
+        attr_reader :mms_id
+
+        # input an alma record
+        def initialize(record)
+          @source_marc_record = record
+          @field_catalog = DataFieldsCatalog.new(@source_marc_record)
+          @tind_external_datafields = []
+        end
+
+        # return mapped tind datafields
+        # keep the order of different mapping
+        def tindfields
+          fields = []
+          fields.concat tindfields_group
+          fields.concat tindfields_group_880
+          fields
+        end
+
+        # return a TIND Marc record
+        # add external datafields
+        # flag to do additional TIND datafield process before generating a TIND Marc record
+        def tind_record
+          fields = tindfields
+          fields.concat @tind_external_datafields
+          more_process(fields)
+          record = ::MARC::Record.new
+          fields.each { |f| record.append(f) }
+          record
+        end
+
+        private
+
+        # return mapped tind datafields
+        # keep the order of different mapping
+        def tindfields_group
+          fields_from_normal = tindfields_from_normal(@field_catalog.data_fields_group[:normal])
+          fields_from_leader = TindFieldFromLeader.new(@source_marc_record).to_datafields
+
+          temp_fields = fields_from_normal.concat fields_from_leader
+          temp_fields.concat tindfields_from_control(temp_fields)
+          temp_fields.concat tindfields_with_pre_existed_field(@field_catalog.data_fields_group[:pre_tag], temp_fields)
+          temp_fields.concat tindfields_with_pre_existed_subfield(@field_catalog.data_fields_group[:pre_tag_subfield], temp_fields)
+
+          temp_fields
+        end
+
+        # return mapped tind 880 datafields
+        # keep the order of different mapping
+        def tindfields_group_880
+          fields = []
+          fields.concat tindfields_from_normal(@field_catalog.data_fields_880_group[:normal])
+          fields.concat tindfields_with_pre_existed_field(@field_catalog.data_fields_880_group[:pre_tag], fields)
+          fields.concat tindfields_with_pre_existed_subfield(@field_catalog.data_fields_880_group[:pre_tag_subfield], fields)
+          fields.concat @field_catalog.data_fields_880_00
+          fields
+        end
+
+        # # Return TIND datafields mapped in a normal way
+        # # Normal way mapping: one regular datafield is mapped one TIND datafield
+        def tindfields_from_normal(alma_fields)
+          new_fls = []
+          alma_fields.each do |f|
+            add_tindfield(new_fls, f, excluding_subfield: false)
+          end
+          new_fls
+        end
+
+        # Return TIND datafield mapped from a control datafield
+        # One control datafield could be mapped to multiple TIND datafields
+        def tindfields_from_control(currentfields)
+          new_fls = []
+          @field_catalog.control_fields.each do |f|
+            add_tindcontrolfield(new_fls, f, currentfields)
+          end
+          new_fls
+        end
+
+        # Return TIND datafields if no pre_existed field
+        def tindfields_with_pre_existed_field(alma_fields, currentfields)
+          new_fls = []
+          alma_fields.each do |f|
+            add_tindfield(new_fls, f, excluding_subfield: false) unless tindfield_existed?(f, currentfields)
+          end
+          new_fls
+        end
+
+        # Return TIND datafields considering excluding pre_existing subfields
+        def tindfields_with_pre_existed_subfield(alma_fields, currentfields)
+          new_fls = []
+          alma_fields.each do |f|
+            excluding_subfield = tindfield_subfield_existed?(f, currentfields)
+            add_tindfield(new_fls, f, excluding_subfield: excluding_subfield)
+          end
+          new_fls
+        end
+
+        def add_tindfield(fls, f, excluding_subfield: false)
+          tindfield = TindFieldFromSingleMap.new(f, excluding_subfield).to_datafield
+          fls << tindfield if tindfield
+        end
+
+        def add_tindcontrolfield(fls, f, currentfields)
+          fls.concat TindFieldFromMultipleMap.new(f, currentfields).to_datafields
+        end
+
+        # Additional processes - run in a sequence
+        def more_process(fields)
+          remove_repeats(fields)
+          clean_subfields(fields)
+          un_matched_fields_880(fields, @field_catalog.mms_id)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/tind_subfield_util.rb

@@ -0,0 +1,154 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module TindSubfieldUtil
+
+        def clean_subfield(subfield)
+          new_value = clr_value(subfield.value)
+          subfield.value = new_value
+        end
+
+        def fields_880_subfield6(datafields_880)
+          formated_subfield6_value_arr(fields_by(datafields_880) { |f| is_880_field?(f) })
+        end
+
+        # From all subfield 6 gotten for repeated fields - with the same tag
+        # return the first subfield 6
+        def the_first_subfield6(fields)
+          values = subfield_6_values(fields)
+          return nil if values.empty?
+
+          # keep it here, in case needed in future: this can make sure
+          # 880 and regular fields having matching sequence number
+          # subfield6_with_small_no(values)
+
+          # new implementation: keep the first subfield 6 value
+          logger.warn("#{f[0].tag} have multiple datafields with multiple subfield 6, the first subfield 6 is kept") if values.length > 1
+          Util.subfield('6', values[0])
+        end
+
+        private
+
+        def subfield6?(f)
+          !f['6'].nil?
+        end
+
+        # f.tag can be a string or integer
+        def is_880_field?(f)
+          f.tag.to_s == '880'
+        end
+
+        # return subfield6 value formated the same way for both 880 and regular datafields
+        def formated_subfield6_value(f)
+          is_880_field?(f) ? formated_subfield6_from_880(f) : formated_subfield6_from_regular_field(f)
+        end
+
+        def fields_by(fields, &block)
+          fields.select(&block)
+        end
+
+        # return array of formated subfield6 values on inputted datafields
+        def formated_subfield6_value_arr(fields_source)
+          # fields_source = is_880_field ? fields_880(datafields) : fields_regular(datafields)
+          ls = []
+          fields_source.map do |f|
+            next unless subfield6?(f)
+
+            ls << formated_subfield6_value(f)
+          end
+          ls.compact
+        end
+
+        # return formated field6 for a regular datafield
+        # tag:246, subfields: 880-02 => 880-246-02
+        def formated_subfield6_from_regular_field(f)
+          return nil unless f['6']
+
+          tag = f.tag
+          arr = f['6'].strip.split('-')
+          "#{arr[0]}-#{tag}-#{arr[1]}"
+        end
+
+        # return formated subfield6 for a 880 datafield
+        # tag: 880, subfield6: 245-01/$1  => 880-245-01
+        def formated_subfield6_from_880(f)
+          return nil unless f['6']
+
+          "#{f.tag}-#{f['6'].strip[0, 6].strip}"
+        end
+
+        # return subfields of specific code
+        # Datafield returned from Alma may have mutiple subfields with the same code
+        def subfields(field, code)
+          sf = []
+          field.each do |s|
+            next unless s.code == code
+
+            captilize_subfield(s) if code == '3'
+            sf << s
+          end
+          sf
+        end
+
+        def captilize_subfield(subfield)
+          new_value = subfield.value.capitalize
+          subfield.value = new_value
+        end
+
+        # Combine multiple subfield values based on definition from csv file
+        def combined_subfield_value(field, code, symbol)
+          sf_arr = subfields(field, code)
+          return '' if sf_arr.empty?
+
+          sf_arr.map(&:value).join(symbol) << symbol # symbol exmaple ' -- '
+        end
+
+        # Mutilpe subfields in one field may have the same name
+        # Get all subfields on 'from subfield name'
+        # Change subfield names with 'to subfield name'
+        def subfields_from_to(field, from, to)
+          subfield_arr = subfields(field, from)
+          subfield_arr.each { |sf| sf.code = to } if !subfield_arr.empty? && (from != to)
+          subfield_arr
+        end
+
+        def subfield6_endwith_00?(f)
+          return false unless is_880_field?(f)
+
+          two_digits = f['6'].strip.split('-')[1][0..1]
+          two_digits.to_s == '00'
+        end
+
+        def fields_with_subfield6(fields)
+          fields.select { |f| subfield6?(f) }
+        end
+
+        # return subfield 6 with the smallest sequence number
+        def subfield_6_values(fields)
+          fields_with_subfield6(fields).map(&:value)
+        end
+
+        def subfield6_value_with_lowest_seq_no(values)
+          seq = 9999
+          txt = nil
+          values.each do |val|
+            num = seq_no(val)
+            if (num > 0) && (num < seq)
+              seq = num
+              txt = val
+            end
+          end
+          txt
+        end
+
+        # return all subfields except subfield6
+        def subfields_without_subfield6(field)
+          field.subfields.reject { |sf| sf.code == '6' }
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/util.rb

@@ -0,0 +1,117 @@
+require 'csv'
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module Util
+
+        class << self
+
+          def csv_rows(file)
+            rows = []
+            CSV.foreach(file, headers: true, header_converters: :symbol, encoding: 'bom|utf-8') do |row|
+              # puts row.headers
+              rows << clr_row(row)
+            end
+            rows
+          end
+
+          def concatenation_symbol(str)
+            str ? str.gsub('a', ' ') : ' '
+          end
+
+          def indicator(str)
+            return [] unless str
+
+            str.strip.gsub('_', ' ').split(',')
+          end
+
+          def tag_symbol(tag)
+            "tag_#{tag}".to_sym
+          end
+
+          def datafield(tag, indicator, subfields)
+            datafield = ::MARC::DataField.new(tag, indicator[0], indicator[1])
+            subfields.each { |sf| datafield.append(sf) }
+            datafield
+          end
+
+          def subfield(code, value)
+            ::MARC::Subfield.new(code, value)
+          end
+
+          def subfield_hash(field)
+            code_value_arr = field.to_hash[field.tag]['subfields']
+            {}.tap { |i| code_value_arr.each(&i.method(:update)) }
+          end
+
+          # input an array of rules, example: [["a,b,c,d", "b", "--"],["o,p,q", "b", ""]]
+          def symbols(rules)
+            rules.map { |rule| concatenation_symbol(rule[2]).strip }
+          end
+
+          def remove_extra_symbol(rules, val)
+            symbols = symbols(rules)
+            symbols.each { |s| val = val.strip.delete_suffix(s) }
+            val
+          end
+
+          def alma_datafield(tag, record)
+            record.fields.each { |f| return f if f.tag.to_s == tag }
+            nil
+          end
+
+          def qualified_alma_record?(alma_record)
+            f_245 = alma_datafield('245', alma_record)
+            f_245_a = f_245['a'].downcase
+
+            val = 'Host bibliographic record'.downcase
+            !f_245_a.start_with? val
+          end
+
+          # From DM - get testing MARC from xml file
+          # @param xml [String] the XML to parse
+          # @return [MARC::Record, nil] the MARC record from the specified XML
+          def from_xml(xml)
+            # noinspection RubyYardReturnMatch,RubyMismatchedReturnType
+            all_from_xml(xml).first
+          end
+
+          private
+
+          def clr_row(row)
+            row[:tag_origin] = format_tag(row[:tag_origin])
+            row[:tag_destination] = format_tag(row[:tag_destination])
+            row[:map_if_no_this_tag_existed] = format_tag(row[:map_if_no_this_tag_existed])
+            row
+          end
+
+          # To ensure tag from csv file in three digits
+          def format_tag(tag)
+            # LDR is not a numeric tag
+            return tag unless numeric? tag
+
+            format('%03d', tag)
+          end
+
+          def numeric?(str)
+            return false if str.nil?
+
+            str.scan(/\D/).empty?
+          end
+
+          # Parses MARCXML.
+          #
+          # @param xml [String] the XML to parse
+          # @return [MARC::XMLReader] the MARC records
+          def all_from_xml(xml)
+            input = StringIO.new(xml.scrub)
+            MARC::XMLReader.new(input)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping.rb

@@ -0,0 +1 @@
+Dir.glob(File.expand_path('mapping/*.rb', __dir__)).sort.each(&method(:require))

data/lib/berkeley_library/tind/module_info.rb

@@ -7,7 +7,7 @@ module BerkeleyLibrary
       SUMMARY = 'TIND DA utilities for the UC Berkeley Library'.freeze
       DESCRIPTION = 'UC Berkeley Library utility gem for working with the TIND DA digital archive.'.freeze
       LICENSE = 'MIT'.freeze
-      VERSION = '0.5.1'.freeze
+      VERSION = '0.6.0'.freeze
       HOMEPAGE = 'https://github.com/BerkeleyLibrary/tind'.freeze
     end
   end

data/lib/berkeley_library/util/files.rb

@@ -30,8 +30,7 @@ module BerkeleyLibrary
       #
       # @param obj [Object] the object that might be an IO
       def writer_like?(obj)
-        # TODO: is it possible/desirable to loosen this? how strict is libxml2?
-        obj.is_a?(IO) || obj.is_a?(StringIO)
+        obj.respond_to?(:write) && obj.respond_to?(:close)
       end
 
     end

data/spec/berkeley_library/tind/mapping/additional_datafield_process_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      describe AdditionalDatafieldProcess do
+
+        let(:qualified_alma_obj) { Alma.new('spec/data/mapping/record.xml') }
+        let(:qualified_alm_record) { qualified_alma_obj.record }
+
+        let(:tind_marc) { TindMarc.new(qualified_alm_record) }
+        let(:tindfields) { tind_marc.tindfields }
+        let(:additona_245_field) { [Util.datafield('245', [' ', ' '], [Util.subfield('a', 'fake 245 a')])] }
+        let(:with_repeated_fields) { tindfields.concat additona_245_field }
+
+        context '# Process 1: Remove duplicated fields' do
+          # two 245 fields inputed, one is removed
+          it 'Input two 245 fields, return only one 245 field' do
+            expect(tind_marc.remove_repeats(with_repeated_fields).map(&:tag).count('245')).to eq 1
+          end
+
+          # one subfield 'a' is kept, subfield 'a' with a value of 'fake 245 a' is ignored
+          it 'Input two subfield $a, keeping the first one' do
+            without_repeated_fields = tind_marc.remove_repeats(with_repeated_fields)
+            single_field_245 = tind_marc.field_on_tag('245', without_repeated_fields)
+            expect(single_field_245['a']).to eq 'Cang jie pian'
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/spec/berkeley_library/tind/mapping/alma_base_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+require 'tempfile'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      describe AlmaBase do
+        extend BerkeleyLibrary::Logging
+        let(:dummy_obj) { Class.new { extend AlmaBase } }
+        let(:alma_record) { ::MARC::Record.new }
+        let(:save_to_file) { Tempfile.new('xml') }
+        let(:record_id) { Class.new { extend BerkeleyLibrary::Alma::RecordId } }
+        let(:hash) { { '980' => ['pre_1912'] } }
+
+        let(:qualified_alma_obj) { Alma.new('spec/data/mapping/record.xml') }
+        let(:qualified_alm_record) { qualified_alma_obj.record }
+
+        let(:un_qualified_alma_obj) { Alma.new('spec/data/mapping/record_not_qualified.xml') }
+        let(:un_qualified_alm_record) { un_qualified_alma_obj.record }
+
+        describe '# base_tind_record' do
+          it 'input a qualified Alma record - return a tind record' do
+            allow(dummy_obj).to receive(:tind_record).with('C084093187', qualified_alm_record, []).and_return(::MARC::Record.new)
+            expect(dummy_obj.base_tind_record('C084093187', [], qualified_alm_record)).to be_a ::MARC::Record
+          end
+
+          it 'input an unqualified Alma record - return nil' do
+            expect { dummy_obj.base_tind_record('C084093187', [], un_qualified_alm_record) }.to raise_error(ArgumentError)
+          end
+
+          it 'no input Alma record with a nil (record) from id  - return nil' do
+            allow(dummy_obj).to receive(:alma_record_from).with('C084093187').and_return(nil)
+            expect { dummy_obj.base_tind_record('C084093187', []) }.to raise_error(ArgumentError)
+          end
+        end
+
+        describe '# base_save' do
+          it 'save tind record' do
+            dummy_obj.base_save('C084093187', qualified_alm_record, save_to_file)
+            expect(File.open(save_to_file.path).readlines[0]).to eq "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+          end
+        end
+
+        describe '# alma_record' do
+          it 'get Alma record' do
+            allow(record_id).to receive(:get_marc_record).and_return(::MARC::Record.new)
+            allow(dummy_obj).to receive(:get_record_id).with('C084093187').and_return(record_id)
+            expect(dummy_obj.send(:alma_record_from, 'C084093187')).to be_instance_of ::MARC::Record
+          end
+        end
+
+        describe '# get_record_id' do
+          it 'return BerkeleyLibrary::Alma::BarCode' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_barcode = true
+            expect(dummy_obj.send(:get_record_id, 'C084093187')).to be_instance_of BerkeleyLibrary::Alma::BarCode
+          end
+
+          it 'return BerkeleyLibrary::Alma::MMSID' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_barcode = false
+            expect(dummy_obj.send(:get_record_id, '991085821143406532')).to be_instance_of BerkeleyLibrary::Alma::MMSID
+
+          end
+
+          it 'return BerkeleyLibrary::Alma::BibNumber' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_barcode = false
+            expect(dummy_obj.send(:get_record_id, 'b11082434')).to be_instance_of BerkeleyLibrary::Alma::BibNumber
+
+          end
+        end
+
+        describe '# add_f_035' do
+          it 'Add 035 from mms_id' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_035_from_mms_id = true
+            expect(dummy_obj.send(:add_f_035, '991085821143406532', hash).tag).to eq '035'
+          end
+
+          it 'Not to add 035 from mms_id' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_035_from_mms_id = false
+            expect(dummy_obj.send(:add_f_035, '991085821143406532', hash)).to eq nil
+          end
+        end
+
+        describe '# tind_record, # derived_tind_fields' do
+          BerkeleyLibrary::TIND::Mapping::AlmaBase.collection_parameter_hash = {
+            '336' => ['Image'],
+            '852' => ['East Asian Library'],
+            '980' => ['pre_1912'],
+            '982' => ['Pre 1912 Chinese Materials - short name', 'Pre 1912 Chinese Materials - long name'],
+            '991' => []
+          }
+          let(:id) { '991085821143406532' }
+          let(:tags) { %w[902 336 852 980 982 901 856] }
+          let(:tags_with_035) { %w[902 336 852 980 982 901 856 035] }
+
+          it ' get tind_record' do
+            datafields = []
+            marc_record = qualified_alm_record
+            expect(dummy_obj.send(:tind_record, id, marc_record, datafields)).to be_instance_of ::MARC::Record
+          end
+
+          it 'get derived fields without 035' do
+            expect(dummy_obj.send(:derived_tind_fields, id).map(&:tag)).to eq tags
+          end
+
+          it 'get derived fields with 035' do
+            BerkeleyLibrary::TIND::Mapping::AlmaBase.is_035_from_mms_id = true
+            expect(dummy_obj.send(:derived_tind_fields, id).map(&:tag)).to eq tags_with_035
+          end
+
+        end
+
+      end
+    end
+  end
+end