berkeley_library-tind 0.5.1 → 0.6.0

data/lib/berkeley_library/tind/mapping/alma.rb

@@ -0,0 +1,42 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class Alma
+        include Util
+
+        attr_reader :record
+
+        def initialize(xml_file)
+          @record = alma_record(xml_file)
+        end
+
+        def control_field
+          @record.fields.each { |f| return f if f.tag.to_s == '008' }
+          nil
+        end
+
+        def field_880(subfield6_value)
+          @record.fields.each { |f| return f if f.tag.to_s == '880' && f['6'] == subfield6_value }
+          nil
+        end
+
+        def field(tag)
+          @record.fields.each { |f| return f if f.tag.to_s == tag }
+          nil
+        end
+
+        private
+
+        def alma_record(xml_file)
+          file = File.open(xml_file)
+          content = file.readlines.map(&:chomp)
+          xml = content.join(' ')
+          Util.from_xml(xml)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma_base.rb

@@ -0,0 +1,101 @@
+# 902__$d derived from current date is added in this module
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module AlmaBase
+        include BerkeleyLibrary::Logging
+
+        @collection_parameter_hash = {}
+        @is_barcode = false
+        @is_035_from_mms_id = false
+
+        class << self
+          attr_accessor :collection_parameter_hash
+          attr_accessor :is_barcode
+          attr_accessor :is_035_from_mms_id
+        end
+
+        # id can be:
+        # 1) Alma mms id
+        # 2) Oskicat No
+        # 3) BarCode No
+        # When alma record is nil or un-qualified, raise error
+        # Input datafields - an array of record specific datafields:  for example, fft datafields, datafield 035 etc.
+
+        def base_tind_record(id, datafields, alma_record = nil)
+          marc_record = alma_record || alma_record_from(id)
+
+          raise ArgumentError, "#{id} has no Alma record." unless marc_record
+
+          unless Util.qualified_alma_record?(marc_record)
+            raise ArgumentError,
+                  "#{id} belong to a host bibliographic record which should not be uploaded to TIND."
+          end
+
+          tind_record(id, marc_record, datafields)
+        end
+
+        # This is mainly for testing purpose, each collection can have a function to save it's record
+        def base_save(id, tind_record, file)
+          raise ArgumentError, "#{id} has no TIND record or not a qualified TIND record." unless tind_record
+
+          BerkeleyLibrary::TIND::MARC::XMLWriter.open(file) do |writer|
+            writer.write(tind_record)
+          end
+        end
+
+        private
+
+        def alma_record_from(id)
+          record_id = get_record_id(id)
+          raise ArgumentError, "#{id} gets no BarCode or RecordId from Alma module." unless record_id
+
+          record_id.get_marc_record
+        end
+
+        def get_record_id(id)
+          AlmaBase.is_barcode ? BerkeleyLibrary::Alma::BarCode.new(id) : BerkeleyLibrary::Alma::RecordId.parse(id)
+        end
+
+        def derived_tind_fields(mms_id)
+          tind_fields = []
+          tind_fields << TindField.f_902_d
+
+          hash = AlmaBase.collection_parameter_hash
+          tind_fields.concat ExternalTindField.tind_fields_from_collection_information(hash)
+
+          tind_fields.concat ExternalTindField.tind_mms_id_fields(mms_id)
+
+          f_035 = add_f_035(mms_id, hash)
+          tind_fields << f_035 if f_035
+
+          tind_fields
+        end
+
+        def tind_record(id, marc_record, datafields)
+          tindmarc = TindMarc.new(marc_record)
+          # get all derived tind_fields: 1) from collection information; 2) from id
+          mms_id = tindmarc.field_catalog.mms_id
+          logger.warn("#{id} has no Control Field 001") unless mms_id
+
+          tind_fields = derived_tind_fields(mms_id)
+          # add inputted record specific datafields
+          tind_fields.concat datafields
+          tindmarc.tind_external_datafields = tind_fields
+
+          # creete a tind marc record
+          tindmarc.tind_record
+        end
+
+        def add_f_035(mms_id, hash)
+          return nil unless mms_id && AlmaBase.is_035_from_mms_id
+
+          val_980 = hash['980'][0].strip
+          TindField.f_035_from_alma_id(mms_id, val_980)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma_multiple_tind.rb

@@ -0,0 +1,31 @@
+require 'marc'
+require 'berkeley_library/tind'
+require 'berkeley_library/alma'
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+
+      class AlmaMultipleTIND
+        include Util
+        include AlmaBase
+        include BerkeleyLibrary::Logging
+
+        # id can be
+        # 1) Alma mms id
+        # 2) Oskicat No
+        # 3) BarCode No
+        def initialize(id)
+          @id = id
+          @marc_record = alma_record_from(id)
+        end
+
+        # If alma record is nil or un-qualified, it returns nil
+        # Input datafields - an array of record specific datafields:  for example, fft datafields, datafield 035 etc.
+        def record(datafields)
+          base_tind_record(@id, datafields, @marc_record)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/alma_single_tind.rb

@@ -0,0 +1,28 @@
+require 'marc'
+require 'berkeley_library/tind'
+require 'berkeley_library/alma'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class AlmaSingleTIND
+        include Util
+        include AlmaBase
+        include BerkeleyLibrary::Logging
+
+        def initialize; end
+
+        # id can be
+        # 1) Alma mms id
+        # 2) Oskicat No
+        # 3) BarCode No
+        # If alma record is nil or un-qualified, it returns nil
+        # Input datafields - an array of record specific datafields:  for example, fft datafields, datafield 035 etc.
+        def record(id, datafields)
+          base_tind_record(id, datafields)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/config.rb

@@ -0,0 +1,44 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      class Config
+
+        class << self
+
+          def one_to_one_map_file
+            ENV.fetch('ONE_TO_ONE_MAP_FILE', File.expand_path('data/one_to_one_mapping.csv', __dir__))
+          end
+
+          def one_to_multiple_map_file
+            ENV.fetch('ONE_TO_ONE_MAP_FILE', File.expand_path('data/one_to_multiple_mapping.csv', __dir__))
+          end
+
+          def no_duplicated_tags
+            %w[245 260 852 901 902 980].freeze
+          end
+
+          def punctuations
+            %w[, : ; / =].freeze
+          end
+
+          def clean_tags
+            %w[245 260 300].freeze
+          end
+
+          def collection_subfield_names
+            {
+              '336' => ['a'],
+              '852' => ['c'],
+              '980' => ['a'],
+              '982' => ['a', 'b'],
+              '991' => ['a']
+            }.freeze
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/csv_mapper.rb

@@ -0,0 +1,35 @@
+require 'csv'
+require 'berkeley_library/tind/mapping/util'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module CsvMapper
+        @rows = []
+        class << self
+          attr_accessor :rows
+        end
+
+        CsvMapper.rows = Util.csv_rows(Config.one_to_one_map_file)
+
+        def from_tags
+          CsvMapper.rows.map { |row| row[:tag_origin] }.compact
+        end
+
+        def rules
+          CsvMapper.rows.to_h { |row| ["tag_#{row[:tag_origin]}".to_sym, SingleRule.new(row)] }
+        end
+
+        # tags allow to keep the first datafield from original marc record
+        def one_occurrence_tags
+          tags = []
+          CsvMapper.rows.each do |row|
+            tags << row[:tag_origin] if row[:keep_one_if_multiple_available]
+          end
+          tags.compact
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/csv_multiple_mapper.rb

@@ -0,0 +1,41 @@
+require 'csv'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module CsvMultipleMapper
+        @rows = []
+        class << self
+          attr_accessor :rows
+        end
+
+        CsvMultipleMapper.rows = Util.csv_rows(Config.one_to_multiple_map_file)
+        def from_tags
+          tags = []
+          CsvMultipleMapper.rows.each do |row|
+            tag = row[:tag_origin]
+            tags << tag unless tags.include?(tag)
+          end
+          tags
+        end
+
+        def rules
+          from_tags.to_h { |tag| [Util.tag_symbol(tag), rules_on_tag(tag)] }
+        end
+
+        private
+
+        def rules_on_tag(tag)
+          rules = []
+          CsvMultipleMapper.rows.each do |row|
+            origin_tag = row[:tag_origin]
+            rules << MultipleRule.new(row) if origin_tag == tag
+          end
+
+          rules
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/data/one_to_multiple_mapping.csv

@@ -0,0 +1,4 @@
+ID,tag_origin,tag_destination,map_if_no_this_tag_existed,subfield_key,new_indecator,value_from,value_to
+1,8,41,41,a,"_,_",35,37
+5,8,269,,a,"_,_",7,10
+6,LDR,903,,b,"_,_",7,7

data/lib/berkeley_library/tind/mapping/data/one_to_one_mapping.csv

@@ -0,0 +1,39 @@
+tag_origin,tag_destination,subfield_single_from,subfield_single_to,subfield_combined_from_1,subfield_combined_to_1,symbol_1 ,subfield_combined_from_2,subfield_combined_to_2,symbol_2,subfield_combined_from_3,subfield_combined_to_3,symbol_3,New_indecator,Position_from,Position_to,Map_if_no_this_tag_existed,Map_if_no_this_tag_subfield_existed, Keep_one_if_multiple_available
+100,700,"6,e","6,e","a,b,c,d,q",a,,,,,,,,"1,_",,,,,
+110,710,"6,e","6,e","a,b",a,,,,,,,,"2,_",,,,,
+111,711,6,6,"a,c,d,e,f,n,p,t",a,,,,,,,,"_,_",,,,,
+242,246,6,6,"a,b,p,n",a,,,,,,,,,,,,,
+245,245,"6,a","6,a","n,p",p,,,,,"b,f,k",b,,"_,_",,,,,
+246,246,6,6,"a,b,p,n",a,,,,,,,,"_,_",,,,,
+250,250,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+255,255,"6,a,b,c","6,a,b,c",,,,,,,,,,"_,_",,,,,
+260,260,"6,a,b,c","6,a,b,c",,,,,,,,,,"_,_",,,,,
+264,260,"6,a,b,c","6,a,b,c",,,,,,,,,,"_,_",,,260,,1
+300,300,"6,b,c","6,b,c","3,a",a,,,,,,,,"_,_",,,,,
+351,505,6,6,"a,b",a,,,,,,,,"_,_",,,,,
+490,490,6,6,"a,v",a,,,,,,,,"_,_",,,,,
+500,500,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+502,502,6,6,"a,b,c,d,g,o",a,,,,,,,,"_,_",,,,,
+505,505,6,6,"a,g,r,t",a,,,,,,,,"_,_",,,,,
+507,255,"6,a","6,a",,,,,,,,,,"_,_",,,,255__a,
+520,520,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+522,522,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+524,524,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+536,536,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+541,541,6,6,"c,a,d",a,,,,,,,,"_,_",,,,,
+545,545,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+546,546,"6,a","6,a",,,,,,,,,,"_,_",,,,,
+600,600,6,6,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+610,610,6,6,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+611,611,6,6,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+630,630,6,6,"a,b,c,d,f,j,k,l,m,n,o,p,q,r,s,t",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+650,650,6,6,"a,b,c",a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+651,651,6,6,a,a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+655,655,"6,2","6,2",a,a,,"x,y,z,v",a,a--a,,,,"_,_",,,,,
+700,700,"6,e","6,e","a,b,c,d,q",a,,,,,,,,"1,_",,,,,
+710,710,"6,e","6,e","a,b",a,,,,,,,,"2,_",,,,,
+711,711,6,6,"a,c,d,e,f,n,p,t",a,,,,,,,,"_,_",,,,,
+720,700,"6,a,e","6,a,e",,,,,,,,,,"1,_",,,,,
+752,651,6,6,,,,"a,b,c,d,f,g",a,a--a,,,,"_,_",,,,,
+773,773,"6,t,j,k,o","6,t,j,k,o",,,,,,,,,,"_,_",,,,,
+907,901,"6,a","6,m",,,,,,,,,,"_,_",,,,,

data/lib/berkeley_library/tind/mapping/external_tind_field.rb

@@ -0,0 +1,103 @@
+require 'marc'
+
+# Derived fields from this module
+# 1) From collection information:
+#  336__$a
+#  852__$c
+#  982__$a,b
+#  991__$a
+# 2) From mms_id
+# 901__$m
+# 85641$u,y
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module ExternalTindField
+        class << self
+          include BerkeleyLibrary::Logging
+
+          def tind_fields_from_collection_information(hash)
+            raise ArgumentError, 'Collection parameters are incorrect.' unless valid_collection_hash?(hash)
+
+            collection_fields(hash)
+          end
+
+          def tind_mms_id_fields(mms_id)
+            raise ArgumentError, 'mms_id is nil' unless mms_id
+
+            mms_id_fields(mms_id)
+          end
+
+          private
+
+          def collection_fields(hash)
+            fields = []
+            hash.each_key do |tag|
+              tindfield = tindfield_on_tag(tag, hash)
+              fields << tindfield if tindfield
+            end
+
+            fields
+          end
+
+          def mms_id_fields(mms_id)
+            fields = []
+            fields << tind_field_901_m(mms_id)
+            fields << tind_field_856_4_1(mms_id)
+          end
+
+          def tind_field_901_m(alma_id)
+            ::MARC::DataField.new('901', ' ', ' ', ['m', alma_id])
+          end
+
+          def tind_field_856_4_1(alma_id)
+            u = "https://search.library.berkeley.edu/discovery/fulldisplay?context=L&vid=01UCS_BER:UCB&docid=alma#{alma_id}"
+            y = 'View library catalog record.'
+            subfield1 = Util.subfield('u', u)
+            subfield2 = Util.subfield('y', y)
+
+            ::MARC::DataField.new('856', '4', '1', subfield1, subfield2)
+          end
+
+          def tindfield_on_tag(tag, hash)
+            subfield_names = Config.collection_subfield_names[tag]
+            subfield_values = hash[tag]
+            return nil if subfield_values.empty?
+
+            subfields = tind_subfields(subfield_values, subfield_names)
+            Util.datafield(tag, [' ', ' '], subfields)
+          end
+
+          def tind_subfields(subfield_values, subfield_names)
+            subfields = []
+            subfield_values.each_with_index do |value, i|
+              name = subfield_names[i]
+              subfield = Util.subfield(name, value.strip)
+              subfields << subfield
+            end
+            subfields
+          end
+
+          def valid_collection_hash?(hash)
+            return false unless valid_item?(hash, '336', 1) &&
+            valid_item?(hash, '852', 1) && valid_item?(hash, '980', 1) && valid_item?(hash, '982', 2) && valid_991(hash)
+
+            true
+          end
+
+          def valid_item?(hash, key, num)
+            (hash.key? key) && (hash[key].length == num)
+          end
+
+          def valid_991(hash)
+            cout = hash['991'].length
+            [0, 1].include?(cout)
+          end
+
+        end
+      end
+    end
+
+  end
+end

data/lib/berkeley_library/tind/mapping/field_catalog.rb

@@ -0,0 +1,146 @@
+require 'berkeley_library/tind/mapping/tind_subfield_util'
+require 'berkeley_library/tind/mapping/misc'
+require 'berkeley_library/tind/mapping/field_catalog_util'
+require 'berkeley_library/tind/mapping/match_tind_field'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+
+      class DataFieldsCatalog
+        include Misc
+        include TindSubfieldUtil
+        include CsvMapper
+        include Util
+        include AdditionalDatafieldProcess
+        include FieldCatalogUtil
+        include MatchTindField
+        include BerkeleyLibrary::Logging
+
+        attr_reader :control_fields
+        attr_reader :data_fields_group
+        attr_reader :data_fields_880_group
+        attr_reader :data_fields_880_00
+        attr_reader :mms_id
+
+        def initialize(record)
+          @control_fields = []
+          @data_fields_group = []
+          @data_fields_880_group = []
+          @data_fields_880_00 = []
+          @mms_id = ''
+
+          @data_fields = []
+          @data_fields_880 = []
+          @alma_field_tags = []
+
+          init(record)
+        end
+
+        def init(record)
+          prepare_catalog(record)
+          @data_fields_group = prepare_group(@data_fields)
+          @data_fields_880_group = prepare_group(@data_fields_880)
+          @mms_id = alma_mms_id
+        end
+
+        def prepare_catalog(record)
+          clean_fields = clean_subfields(record.fields)
+          check_abnormal_formated_subfield6(clean_fields)
+          allocate_fields(clean_fields)
+          remove_fields_with_subject_fast
+        end
+
+        def remove_fields_with_subject_fast
+          @data_fields = exluding_fields_with_fast_subject(@data_fields)
+          @data_fields_880 = exluding_fields_with_fast_subject(@data_fields_880)
+        end
+
+        private
+
+        def allocate_fields(fields)
+          fields.each do |f|
+            next if added_control_field?(f)
+            next if added_880_field?(f)
+
+            tag = f.tag
+            next unless (found_in_mapper? tag) && (no_pre_existed_field? tag)
+
+            @data_fields << f
+            @alma_field_tags << f.tag
+          end
+        end
+
+        # 880 field with a subfield6 including a tag belong to origin tags defined in csv file
+        def qualified_880_field?(f)
+          return false unless referred_tag(f)
+
+          found_in_mapper?(referred_tag(f))
+        end
+
+        def added_control_field?(f)
+          return false unless ::MARC::ControlField.control_tag?(f.tag)
+
+          @control_fields << f
+          true
+        end
+
+        def added_880_field?(f)
+          return false unless f.tag == '880'
+
+          # adding 880 datafield with "non-subfield6" to "00" group for keeping this record in TIND
+          # with log information, to let users correcting or removing this datafield from TIND record
+          @data_fields_880_00 << f unless valid_subfield6?(f)
+
+          if qualified_880_field?(f)
+            subfield6_endwith_00?(f) ? @data_fields_880_00 << f : @data_fields_880 << f
+          end
+
+          true
+        end
+
+        def valid_subfield6?(f)
+          return true if subfield6?(f)
+
+          logger.warn("880 field has no subfield 6 #{f.inspect}")
+
+          false
+        end
+
+        # Is the origin_tag of a field has related from_tag in csv file?
+        def found_in_mapper?(tag)
+          from_tags.include? tag
+        end
+
+        # If tag is listed in csv_mapper.one_occurrence_tags
+        # Check pre_existed field of this tag
+        # make sure to keep the first datafield for an one_occurrence_tag defined in csv mapping file
+        # def no_pre_existed_field?(tag)
+        #   # no one-occurrence defined in csv
+        #   return true unless one_occurrence_tags.include? tag
+
+        #   # Checking the exsisting regular fields include the one-occurrence field defined in the csv
+        #   return false if @alma_field_tags.compact.include? tag
+
+        #   true
+        # end
+
+        def no_pre_existed_field?(tag)
+          # no one-occurrence defined in csv
+          return true unless one_occurrence_tags.include? tag
+
+          # Checking the exsisting regular fields include the one-occurrence field defined in the csv
+          !(@alma_field_tags.compact.include? tag)
+        end
+
+        def alma_mms_id
+          f_001 = @control_fields.find { |f| f if f.tag == '001' }
+          return nil unless f_001
+
+          f_001.value
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/field_catalog_util.rb

@@ -0,0 +1,59 @@
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module FieldCatalogUtil
+
+        # Excluding fields: subfield2 = 'fast' and tag or refered tag(880 fields) started with '6':
+        def exluding_fields_with_fast_subject(fields)
+          fields.reject { |f| excluding_field?(f) }
+        end
+
+        def prepare_group(from_fields)
+          datafields_hash = { normal: [], pre_tag: [], pre_tag_subfield: [] }
+          from_fields.each do |f|
+            # a regular field tag, or a tag value from 880 field captured from subfield6
+            tag = origin_mapping_tag(f)
+            next unless tag
+
+            rule = rules[Util.tag_symbol(tag)]
+
+            assing_field(rule, f, datafields_hash)
+          end
+
+          datafields_hash
+        end
+
+        private
+
+        def excluding_field?(f)
+          return false unless field_6xx?(f)
+          return false unless subfield2_fast(f)
+
+          true
+        end
+
+        # Both regular and 880 field: tag or refered tag started with '6'
+        def field_6xx?(f)
+          tag = origin_mapping_tag(f)
+          tag =~ /^6\d{2}$/
+        end
+
+        def subfield2_fast(f)
+          subject = f['2']
+          return false unless subject
+
+          subject.downcase == 'fast'
+        end
+
+        # f is either from field whose tag having a match in csv mapping file - 'from tag' column
+        def assing_field(rule, f, datafields_hash)
+          if rule.pre_existed_tag then datafields_hash[:pre_tag] << f
+          elsif rule.pre_existed_tag_subfield then datafields_hash[:pre_tag_subfield] << f
+          else  datafields_hash[:normal] << f
+          end
+        end
+
+      end
+    end
+  end
+end