berkeley_library-tind 0.5.1 → 0.7.0

data/lib/berkeley_library/tind/mapping/additional_datafield_process.rb

@@ -0,0 +1,128 @@
+# 1. Combine repeated fields
+# 2. Remove characters pre_defined
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module AdditionalDatafieldProcess
+
+        # process 1: remove and combine repeated fields - defined in Config module
+        def remove_repeats(fields)
+          Config.no_duplicated_tags.each { |tag| remove_repeated_fields(tag, fields) }
+          fields
+        end
+
+        # process 2: remove charaters in subfields defined in Config module
+        def clean_subfields(fields)
+          fields.each do |f|
+            next unless field_in_tags?(f, Config.clean_tags)
+
+            clean_subfields_in_field(f)
+          end
+          fields
+        end
+
+        # 1. Find all datafield with the tag,
+        #    if more than one found, combine repeated datafields which have a same tag into one
+        # 2. Find all 880 datafields with subfield6 referring to the tag,
+        #    if more than one found, combine repeated datafields with one subfield 6,
+        #    a related datafield will be modified to have the same sequence in subfield 6 as in this 880 subfield 6
+        def remove_repeated_fields(tag, fields)
+          repeated_fields = fields_on_tag(tag, fields)
+          remove_repeated(repeated_fields, fields)
+
+          repeated_880_fields = fields_880_on_subfield6_referredtag(tag, fields)
+          remove_repeated(repeated_880_fields, fields)
+          fields
+        end
+
+        # clean subfields of a datafield
+        def clean_subfields_in_field(field)
+          field.subfields.each { |sf| clean_subfield(sf) }
+        end
+
+        private
+
+        def field_in_tags?(field, tags)
+          tag = field.tag
+          tag_to_match = tag == '880' ? referred_tag(field) : tag
+          return false unless tag_to_match
+
+          tags.include? tag_to_match
+        end
+
+        def fields_on_tag(tag, fields)
+          fields.select { |f| f.tag == tag }
+        end
+
+        def fields_880_on_subfield6_referredtag(tag, fields)
+          fields.select { |f| field_880_has_referred_tag?(tag, f) }
+        end
+
+        # Marc with tag in 'Config.no_duplicated_tags' are not supposed to have the same multiple subfields
+        # but sometime, sourc data error happens
+        # solution:
+        # taking the first subfield 6
+        # taking the first other subfield when there are multiple same subfields
+        # if 880 and regular field having unmatching subfield 6 in sequnce number, they will go to log file
+        # A user will check the log and correct data in Alma or TIND accordingly
+        def combined_subfields(fields)
+          sf_arr = []
+          subfield_6 = the_first_subfield6(fields)
+          sf_arr << subfield_6 if subfield_6
+          identical_subfields = no_duplicated_first_subfields(fields)
+          sf_arr.concat identical_subfields
+          sf_arr
+        end
+
+        # if there are multiple same subfields, pick up the first one, ignore the others
+        def no_duplicated_first_subfields(fields)
+          indentical_subfields = []
+
+          fields.each do |f|
+            subfields = subfields_without_subfield6(f)
+            subfields.each { |sf| indentical_subfields << sf if not_in_identical_subfields?(indentical_subfields, sf) }
+          end
+
+          indentical_subfields
+        end
+
+        def subfield_codes(subfields)
+          subfields.map(&:code)
+        end
+
+        def not_in_identical_subfields?(subfields, subfield)
+          !subfield_codes(subfields).include? subfield.code
+        end
+
+        # return a combined datafield from repeated datafields - with the same tag
+        def identical_field(repeated_fields)
+          tag = repeated_fields[0].tag
+          indicator = first_indicator(repeated_fields)
+          subfield_arr = combined_subfields(repeated_fields)
+          Util.datafield(tag, indicator, subfield_arr)
+        end
+
+        # remove repeated fields
+        def rm_fields(fields_tobe_removed, fields)
+          fields.delete_if { |f| fields_tobe_removed.include? f }
+        end
+
+        # suppose all repeated datafield have the same indictor
+        def first_indicator(fields)
+          indicator1 = fields[0].indicator1
+          indicator2 = fields[0].indicator2
+          [indicator1, indicator2]
+        end
+
+        # remove and keep one repeated datafield
+        def remove_repeated(repeated_fields, fields)
+          return fields unless repeated_fields.length > 1
+
+          rm_fields(repeated_fields, fields)
+          fields << identical_field(repeated_fields)
+          fields
+        end
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma.rb

@@ -0,0 +1,42 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class Alma
+        include Util
+
+        attr_reader :record
+
+        def initialize(xml_file)
+          @record = alma_record(xml_file)
+        end
+
+        def control_field
+          @record.fields.each { |f| return f if f.tag.to_s == '008' }
+          nil
+        end
+
+        def field_880(subfield6_value)
+          @record.fields.each { |f| return f if f.tag.to_s == '880' && f['6'] == subfield6_value }
+          nil
+        end
+
+        def field(tag)
+          @record.fields.each { |f| return f if f.tag.to_s == tag }
+          nil
+        end
+
+        private
+
+        def alma_record(xml_file)
+          file = File.open(xml_file)
+          content = file.readlines.map(&:chomp)
+          xml = content.join(' ')
+          Util.from_xml(xml)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma_base.rb

@@ -0,0 +1,108 @@
+# 902__$d derived from current date is added in this module
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module AlmaBase
+        include BerkeleyLibrary::Logging
+
+        @collection_parameter_hash = {}
+        @is_barcode = false
+        @is_035_from_mms_id = false
+        @including_origin_tags = []
+        @excluding_origin_tags = []
+
+        class << self
+          attr_accessor :collection_parameter_hash
+          attr_accessor :is_barcode
+          attr_accessor :is_035_from_mms_id
+          attr_accessor :including_origin_tags
+          attr_accessor :excluding_origin_tags
+
+        end
+
+        # id can be:
+        # 1) Alma mms id
+        # 2) Oskicat No
+        # 3) BarCode No
+        # When alma record is nil or un-qualified, raise error
+        # Input datafields - an array of record specific datafields:  for example, fft datafields, datafield 035 etc.
+
+        def base_tind_record(id, datafields, alma_record = nil)
+          marc_record = alma_record || alma_record_from(id)
+
+          raise ArgumentError, "#{id} has no Alma record." unless marc_record
+
+          unless Util.qualified_alma_record?(marc_record)
+            raise ArgumentError,
+                  "#{id} belong to a host bibliographic record which should not be uploaded to TIND."
+          end
+
+          tind_record(id, marc_record, datafields)
+        end
+
+        # This is mainly for testing purpose, each collection can have a function to save it's record
+        def base_save(id, tind_record, file)
+          raise ArgumentError, "#{id} has no TIND record or not a qualified TIND record." unless tind_record
+
+          BerkeleyLibrary::TIND::MARC::XMLWriter.open(file) do |writer|
+            writer.write(tind_record)
+          end
+        end
+
+        private
+
+        def alma_record_from(id)
+          record_id = get_record_id(id)
+          raise ArgumentError, "#{id} gets no BarCode or RecordId from Alma module." unless record_id
+
+          record_id.get_marc_record
+        end
+
+        def get_record_id(id)
+          AlmaBase.is_barcode ? BerkeleyLibrary::Alma::BarCode.new(id) : BerkeleyLibrary::Alma::RecordId.parse(id)
+        end
+
+        def derived_tind_fields(mms_id)
+          tind_fields = []
+          tind_fields << TindField.f_902_d
+
+          hash = AlmaBase.collection_parameter_hash
+          tind_fields.concat ExternalTindField.tind_fields_from_collection_information(hash)
+
+          tind_fields.concat ExternalTindField.tind_mms_id_fields(mms_id)
+
+          f_035 = add_f_035(mms_id, hash)
+          tind_fields << f_035 if f_035
+
+          tind_fields
+        end
+
+        def tind_record(id, marc_record, datafields)
+          return nil unless Util.collection_config_correct?
+
+          tindmarc = TindMarc.new(marc_record)
+          # get all derived tind_fields: 1) from collection information; 2) from id
+          mms_id = tindmarc.field_catalog.mms_id
+          logger.warn("#{id} has no Control Field 001") unless mms_id
+
+          tind_fields = derived_tind_fields(mms_id)
+          # add inputted record specific datafields
+          tind_fields.concat datafields
+          tindmarc.tind_external_datafields = tind_fields
+
+          # creete a tind marc record
+          tindmarc.tind_record
+        end
+
+        def add_f_035(mms_id, hash)
+          return nil unless mms_id && AlmaBase.is_035_from_mms_id
+
+          val_980 = hash['980'][0].strip
+          TindField.f_035_from_alma_id(mms_id, val_980)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma_multiple_tind.rb

@@ -0,0 +1,31 @@
+require 'marc'
+require 'berkeley_library/tind'
+require 'berkeley_library/alma'
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+
+      class AlmaMultipleTIND
+        include Util
+        include AlmaBase
+        include BerkeleyLibrary::Logging
+
+        # id can be
+        # 1) Alma mms id
+        # 2) Oskicat No
+        # 3) BarCode No
+        def initialize(id)
+          @id = id
+          @marc_record = alma_record_from(id)
+        end
+
+        # If alma record is nil or un-qualified, it returns nil
+        # Input datafields - an array of record specific datafields:  for example, fft datafields, datafield 035 etc.
+        def record(datafields)
+          base_tind_record(@id, datafields, @marc_record)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma_single_tind.rb

@@ -0,0 +1,28 @@
+require 'marc'
+require 'berkeley_library/tind'
+require 'berkeley_library/alma'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class AlmaSingleTIND
+        include Util
+        include AlmaBase
+        include BerkeleyLibrary::Logging
+
+        def initialize; end
+
+        # id can be
+        # 1) Alma mms id
+        # 2) Oskicat No
+        # 3) BarCode No
+        # If alma record is nil or un-qualified, it returns nil
+        # Input datafields - an array of record specific datafields:  for example, fft datafields, datafield 035 etc.
+        def record(id, datafields)
+          base_tind_record(id, datafields)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/config.rb

@@ -0,0 +1,44 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class Config
+
+        class << self
+
+          def one_to_one_map_file
+            ENV.fetch('ONE_TO_ONE_MAP_FILE', File.expand_path('data/one_to_one_mapping.csv', __dir__))
+          end
+
+          def one_to_multiple_map_file
+            ENV.fetch('ONE_TO_ONE_MAP_FILE', File.expand_path('data/one_to_multiple_mapping.csv', __dir__))
+          end
+
+          def no_duplicated_tags
+            %w[245 260 852 901 902 980].freeze
+          end
+
+          def punctuations
+            %w[, : ; / =].freeze
+          end
+
+          def clean_tags
+            %w[245 260 300].freeze
+          end
+
+          def collection_subfield_names
+            {
+              '336' => ['a'],
+              '852' => ['c'],
+              '980' => ['a'],
+              '982' => ['a', 'b'],
+              '991' => ['a']
+            }.freeze
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/csv_mapper.rb

@@ -0,0 +1,35 @@
+require 'csv'
+require 'berkeley_library/tind/mapping/util'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module CsvMapper
+        @rows = []
+        class << self
+          attr_accessor :rows
+        end
+
+        CsvMapper.rows = Util.csv_rows(Config.one_to_one_map_file)
+
+        def from_tags
+          CsvMapper.rows.map { |row| row[:tag_origin] }.compact
+        end
+
+        def rules
+          CsvMapper.rows.to_h { |row| ["tag_#{row[:tag_origin]}".to_sym, SingleRule.new(row)] }
+        end
+
+        # tags allow to keep the first datafield from original marc record
+        def one_occurrence_tags
+          tags = []
+          CsvMapper.rows.each do |row|
+            tags << row[:tag_origin] if row[:keep_one_if_multiple_available]
+          end
+          tags.compact
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/csv_multiple_mapper.rb

@@ -0,0 +1,41 @@
+require 'csv'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module CsvMultipleMapper
+        @rows = []
+        class << self
+          attr_accessor :rows
+        end
+
+        CsvMultipleMapper.rows = Util.csv_rows(Config.one_to_multiple_map_file)
+        def from_tags
+          tags = []
+          CsvMultipleMapper.rows.each do |row|
+            tag = row[:tag_origin]
+            tags << tag unless tags.include?(tag)
+          end
+          tags
+        end
+
+        def rules
+          from_tags.to_h { |tag| [Util.tag_symbol(tag), rules_on_tag(tag)] }
+        end
+
+        private
+
+        def rules_on_tag(tag)
+          rules = []
+          CsvMultipleMapper.rows.each do |row|
+            origin_tag = row[:tag_origin]
+            rules << MultipleRule.new(row) if origin_tag == tag
+          end
+
+          rules
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/data/one_to_multiple_mapping.csv

@@ -0,0 +1,4 @@
+ID,tag_origin,tag_destination,map_if_no_this_tag_existed,subfield_key,new_indecator,value_from,value_to
+1,8,41,41,a,"_,_",35,37
+5,8,269,,a,"_,_",7,10
+6,LDR,903,,b,"_,_",7,7

data/lib/berkeley_library/tind/mapping/data/one_to_one_mapping.csv

@@ -0,0 +1,39 @@
+tag_origin,tag_destination,subfield_single_from,subfield_single_to,order,subfield_combined_from_1,subfield_combined_to_1,symbol_1 ,subfield_combined_from_2,subfield_combined_to_2,symbol_2,subfield_combined_from_3,subfield_combined_to_3,symbol_3,New_indecator,Position_from,Position_to,Map_if_no_this_tag_existed,Map_if_no_this_tag_subfield_existed, Keep_one_if_multiple_available
+100,700,"6,e","6,e","6,a,e","a,b,c,d,q",a,,,,,,,,"1,_",,,,,
+110,710,"6,e","6,e","6,a,e","a,b",a,,,,,,,,"2,_",,,,,
+111,711,6,6,,"a,c,d,e,f,n,p,t",a,,,,,,,,"_,_",,,,,
+242,246,6,6,,"a,b,p,n",a,,,,,,,,,,,,,
+245,245,"6,a","6,a","6,a,b,p","n,p",p,,,,,"b,f,k",b,,"_,_",,,,,
+246,246,6,6,,"a,b,p,n",a,,,,,,,,"_,_",,,,,
+250,250,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+255,255,"6,a,b,c","6,a,b,c",,,,,,,,,,,"_,_",,,,,
+260,260,"6,a,b,c","6,a,b,c",,,,,,,,,,,"_,_",,,,,
+264,260,"6,a,b,c","6,a,b,c",,,,,,,,,,,"_,_",,,260,,1
+300,300,"6,b,c","6,b,c","6,a,b,c","3,a",a,,,,,,,,"_,_",,,,,
+351,505,6,6,,"a,b",a,,,,,,,,"_,_",,,,,
+490,490,6,6,,"a,v",a,,,,,,,,"_,_",,,,,
+500,500,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+502,502,6,6,,"a,b,c,d,g,o",a,,,,,,,,"_,_",,,,,
+505,505,6,6,,"a,g,r,t",a,,,,,,,,"_,_",,,,,
+507,255,"6,a","6,a",,,,,,,,,,,"_,_",,,,255__a,
+520,520,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+522,522,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+524,524,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+536,536,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+541,541,6,6,,"c,a,d",a,,,,,,,,"_,_",,,,,
+545,545,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+546,546,"6,a","6,a",,,,,,,,,,,"_,_",,,,,
+600,600,6,6,,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+610,610,6,6,,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+611,611,6,6,,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+630,630,6,6,,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+650,650,6,6,,"a,b,c",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+651,651,6,6,,a,a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+655,655,"6,2","6,2",,a,a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+700,700,"6,e","6,e","6,a,e","a,b,c,d,q",a,,,,,,,,"1,_",,,,,
+710,710,"6,e","6,e","6,a,e","a,b",a,,,,,,,,"2,_",,,,,
+711,711,6,6,,"a,c,d,e,f,n,p,t",a,,,,,,,,"_,_",,,,,
+720,700,"6,a,e","6,a,e",,,,,,,,,,,"1,_",,,,,
+752,651,6,6,,,,,"a,b,c,d,f,g",a,a--a,,,,"_,_",,,,,
+773,773,"6,t,j,k,o","6,t,j,k,o",,,,,,,,,,,"_,_",,,,,
+907,901,"6,a","6,m",,,,,,,,,,,"_,_",,,,,

data/lib/berkeley_library/tind/mapping/external_tind_field.rb

@@ -0,0 +1,103 @@
+require 'marc'
+
+# Derived fields from this module
+# 1) From collection information:
+#  336__$a
+#  852__$c
+#  982__$a,b
+#  991__$a
+# 2) From mms_id
+# 901__$m
+# 85641$u,y
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module ExternalTindField
+        class << self
+          include BerkeleyLibrary::Logging
+
+          def tind_fields_from_collection_information(hash)
+            raise ArgumentError, 'Collection parameters are incorrect.' unless valid_collection_hash?(hash)
+
+            collection_fields(hash)
+          end
+
+          def tind_mms_id_fields(mms_id)
+            raise ArgumentError, 'mms_id is nil' unless mms_id
+
+            mms_id_fields(mms_id)
+          end
+
+          private
+
+          def collection_fields(hash)
+            fields = []
+            hash.each_key do |tag|
+              tindfield = tindfield_on_tag(tag, hash)
+              fields << tindfield if tindfield
+            end
+
+            fields
+          end
+
+          def mms_id_fields(mms_id)
+            fields = []
+            fields << tind_field_901_m(mms_id)
+            fields << tind_field_856_4_1(mms_id)
+          end
+
+          def tind_field_901_m(alma_id)
+            ::MARC::DataField.new('901', ' ', ' ', ['m', alma_id])
+          end
+
+          def tind_field_856_4_1(alma_id)
+            u = "https://search.library.berkeley.edu/discovery/fulldisplay?context=L&vid=01UCS_BER:UCB&docid=alma#{alma_id}"
+            y = 'View library catalog record.'
+            subfield1 = Util.subfield('u', u)
+            subfield2 = Util.subfield('y', y)
+
+            ::MARC::DataField.new('856', '4', '1', subfield1, subfield2)
+          end
+
+          def tindfield_on_tag(tag, hash)
+            subfield_names = Config.collection_subfield_names[tag]
+            subfield_values = hash[tag]
+            return nil if subfield_values.empty?
+
+            subfields = tind_subfields(subfield_values, subfield_names)
+            Util.datafield(tag, [' ', ' '], subfields)
+          end
+
+          def tind_subfields(subfield_values, subfield_names)
+            subfields = []
+            subfield_values.each_with_index do |value, i|
+              name = subfield_names[i]
+              subfield = Util.subfield(name, value.strip)
+              subfields << subfield
+            end
+            subfields
+          end
+
+          def valid_collection_hash?(hash)
+            return false unless valid_item?(hash, '336', 1) &&
+            valid_item?(hash, '852', 1) && valid_item?(hash, '980', 1) && valid_item?(hash, '982', 2) && valid_991(hash)
+
+            true
+          end
+
+          def valid_item?(hash, key, num)
+            (hash.key? key) && (hash[key].length == num)
+          end
+
+          def valid_991(hash)
+            cout = hash['991'].length
+            [0, 1].include?(cout)
+          end
+
+        end
+      end
+    end
+
+  end
+end