berkeley_library-tind 0.5.1 → 0.7.0

data/lib/berkeley_library/tind/mapping/field_catalog.rb

@@ -0,0 +1,137 @@
+require 'berkeley_library/tind/mapping/tind_subfield_util'
+require 'berkeley_library/tind/mapping/misc'
+require 'berkeley_library/tind/mapping/field_catalog_util'
+require 'berkeley_library/tind/mapping/match_tind_field'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+
+      class DataFieldsCatalog
+        include Misc
+        include TindSubfieldUtil
+        include CsvMapper
+        include Util
+        include AdditionalDatafieldProcess
+        include FieldCatalogUtil
+        include MatchTindField
+        include BerkeleyLibrary::Logging
+
+        attr_reader :control_fields
+        attr_reader :data_fields_group
+        attr_reader :data_fields_880_group
+        attr_reader :data_fields_880_00
+        attr_reader :mms_id
+
+        def initialize(record)
+          @control_fields = []
+          @data_fields_group = []
+          @data_fields_880_group = []
+          @data_fields_880_00 = []
+          @mms_id = ''
+
+          @data_fields = []
+          @data_fields_880 = []
+          @alma_field_tags = []
+
+          init(record)
+        end
+
+        def init(record)
+          prepare_catalog(record)
+          @data_fields_group = prepare_group(@data_fields)
+          @data_fields_880_group = prepare_group(@data_fields_880)
+          @mms_id = alma_mms_id
+        end
+
+        def prepare_catalog(record)
+          clean_fields = clean_subfields(record.fields)
+          check_abnormal_formated_subfield6(clean_fields)
+          final_fields_to_map = fields_to_map(clean_fields)
+          allocate_fields(final_fields_to_map)
+          remove_fields_with_subject_fast
+        end
+
+        def remove_fields_with_subject_fast
+          @data_fields = exluding_fields_with_fast_subject(@data_fields)
+          @data_fields_880 = exluding_fields_with_fast_subject(@data_fields_880)
+        end
+
+        private
+
+        def allocate_fields(fields)
+          fields.each do |f|
+            next if added_control_field?(f)
+            next if added_880_field?(f)
+
+            tag = f.tag
+            next unless (found_in_mapper? tag) && (no_pre_existed_field? tag)
+
+            @data_fields << f
+            @alma_field_tags << f.tag
+          end
+        end
+
+        # 880 field with a subfield6 including a tag belong to origin tags defined in csv file
+        def qualified_880_field?(f)
+          return false unless referred_tag(f)
+
+          found_in_mapper?(referred_tag(f))
+        end
+
+        def added_control_field?(f)
+          return false unless ::MARC::ControlField.control_tag?(f.tag)
+
+          @control_fields << f
+          true
+        end
+
+        def added_880_field?(f)
+          return false unless f.tag == '880'
+
+          # adding 880 datafield with "non-subfield6" to "00" group for keeping this record in TIND
+          # with log information, to let users correcting or removing this datafield from TIND record
+          @data_fields_880_00 << f unless valid_subfield6?(f)
+
+          if qualified_880_field?(f)
+            subfield6_endwith_00?(f) ? @data_fields_880_00 << f : @data_fields_880 << f
+          end
+
+          true
+        end
+
+        def valid_subfield6?(f)
+          return true if subfield6?(f)
+
+          logger.warn("880 field has no subfield 6 #{f.inspect}")
+
+          false
+        end
+
+        # Is the origin_tag of a field has related from_tag in csv file?
+        def found_in_mapper?(tag)
+          from_tags.include? tag
+        end
+
+        # If tag is listed in csv_mapper.one_occurrence_tags
+        # Check pre_existed field of this tag
+        # make sure to keep the first datafield for an one_occurrence_tag defined in csv mapping file
+        def no_pre_existed_field?(tag)
+          # no one-occurrence defined in csv
+          return true unless one_occurrence_tags.include? tag
+
+          # Checking the exsisting regular fields include the one-occurrence field defined in the csv
+          !(@alma_field_tags.compact.include? tag)
+        end
+
+        def alma_mms_id
+          f_001 = @control_fields.find { |f| f if f.tag == '001' }
+          return nil unless f_001
+
+          f_001.value
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/field_catalog_util.rb

@@ -0,0 +1,105 @@
+require 'berkeley_library/tind/mapping/alma_base'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module FieldCatalogUtil
+        include AlmaBase
+
+        # Excluding fields: subfield2 = 'fast' and tag or refered tag(880 fields) started with '6':
+        def exluding_fields_with_fast_subject(fields)
+          fields.reject { |f| excluding_field?(f) }
+        end
+
+        def prepare_group(from_fields)
+          datafields_hash = { normal: [], pre_tag: [], pre_tag_subfield: [] }
+          from_fields.each do |f|
+            # a regular field tag, or a tag value from 880 field captured from subfield6
+            tag = origin_mapping_tag(f)
+            next unless tag
+
+            rule = rules[Util.tag_symbol(tag)]
+
+            assign_field(rule, f, datafields_hash)
+          end
+
+          datafields_hash
+        end
+
+        # Defining a list of fields from Alma to be mapped to TIND fields based on
+        # collection configuration:
+        # 1) BerkeleyLibrary::TIND::Mapping::AlmaBase.including_origin_tags
+        # 2) BerkeleyLibrary::TIND::Mapping::AlmaBase.excluding_origin_tags
+        def fields_to_map(fields)
+          including_defined = !BerkeleyLibrary::TIND::Mapping::AlmaBase.including_origin_tags.empty?
+          excluding_defined = !BerkeleyLibrary::TIND::Mapping::AlmaBase.excluding_origin_tags.empty?
+          return [] if including_defined && excluding_defined # not allow to define both including and excluding tags
+          return fields unless including_defined || excluding_defined # Neither including nor excluding tags are defined
+          return fields_included(fields) if including_defined # including tags defined
+          return fields_exclued(fields) if excluding_defined  # excluding tags defined
+        end
+
+        private
+
+        def fields_included(fields)
+          fields.select { |f| inclduing?(f) }
+        end
+
+        def inclduing?(f)
+          return true if %w[001 008].include? f.tag # always keeping 001, 008 field since it include almid
+
+          tag = origin_mapping_tag(f)
+          BerkeleyLibrary::TIND::Mapping::AlmaBase.including_origin_tags.include? tag
+        end
+
+        def fields_exclued(fields)
+          new_fields = []
+          exclude_tags = BerkeleyLibrary::TIND::Mapping::AlmaBase.excluding_origin_tags
+          fields.each do |f|
+            tag = tag(f, exclude_tags)
+            new_fields << f unless exclude_tags.include? tag
+          end
+          new_fields
+        end
+
+        # 1. exluding tags have '880' :  all 880 fields will be excluded
+        # 2. excluding tags have no '880', then excluding 880 fields whoses refered tag located in excluding tags
+        def tag(field, exclude_tags)
+          return field.tag if exclude_tags.include? '880' #  Case: excluding all 880 fields
+
+          origin_mapping_tag(field)
+        end
+
+        def excluding_field?(f)
+          return false unless field_6xx?(f)
+          return false unless subfield2_fast(f)
+
+          true
+        end
+
+        # Both regular and 880 field: tag or refered tag started with '6'
+        def field_6xx?(f)
+          tag = origin_mapping_tag(f)
+          tag =~ /^6\d{2}$/
+        end
+
+        def subfield2_fast(f)
+          subject = f['2']
+          return false unless subject
+
+          subject.downcase == 'fast'
+        end
+
+        # f is either from field whose tag having a match in csv mapping file - 'from tag' column
+        def assign_field(rule, f, datafields_hash)
+          if rule.pre_existed_tag then datafields_hash[:pre_tag] << f
+          elsif rule.pre_existed_tag_subfield then datafields_hash[:pre_tag_subfield] << f
+          else
+            datafields_hash[:normal] << f
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/match_tind_field.rb

@@ -0,0 +1,77 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module MatchTindField
+
+        # return regular fields without matched 880 fields
+        # return 880 fields without matched regular fields
+        def un_matched_fields_880(fields, mms_id)
+          unmached_fields = []
+
+          str_arr_from_880 = subfield6_values_from_880_fields(fields)
+          str_arr_from_regular = subfield6_values_from_regular_fields(fields)
+
+          fields_tobe_validated = fields_need_880_validation(fields)
+
+          fields_880_tobe_validated = fields_tobe_validated.select { |f| is_880_field?(f) }
+          fields_regular_tobe_validated = fields_tobe_validated.reject { |f| is_880_field?(f) }
+
+          unmached_fields.concat un_matched_fields(fields_880_tobe_validated, str_arr_from_regular)
+          unmached_fields.concat un_matched_fields(fields_regular_tobe_validated, str_arr_from_880)
+
+          log_warning(unmached_fields, mms_id)
+        end
+
+        def check_abnormal_formated_subfield6(fields)
+          fields.each { |f| check_subfield6_format(f) if check_subfield6?(f) }
+        end
+
+        private
+
+        def subfield6_values_from_880_fields(fields)
+          formated_subfield6_value_arr(fields_by(fields) { |f| is_880_field?(f) })
+        end
+
+        def subfield6_values_from_regular_fields(fields)
+          formated_subfield6_value_arr(fields_by(fields) { |f| !is_880_field?(f) })
+        end
+
+        def fields_need_880_validation(fields)
+          fields_with_subfield6(fields).reject { |f| subfield6_endwith_00?(f) }
+        end
+
+        # return true when field has a matched 880 field,
+        # or an 880 field has a matched regular field
+        def match?(f, arr)
+          str = formated_subfield6_value(f)
+          arr.include? str
+        end
+
+        def un_matched_fields(fields, arr)
+          fds = []
+          fields.each { |f| fds << f unless match?(f, arr) }
+          fds
+        end
+
+        def log_warning(fields, mms_id)
+          warning_message_for_rspec = []
+          fields.each do |f|
+            msg = "Please check 880 matching: mms_id: #{mms_id}, tag: #{f.tag}, value: #{f['6']} "
+            warning_message_for_rspec << msg
+            logger.warn(msg)
+          end
+          warning_message_for_rspec
+        end
+
+        def check_subfield6?(f)
+          return false if ::MARC::ControlField.control_tag?(f.tag)
+
+          f['6'] ? true : false
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/misc.rb

@@ -0,0 +1,69 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module Misc
+
+        #### referred tag ###
+        def origin_mapping_tag(f)
+          is_880_field?(f) ? referred_tag(f) : f.tag
+        end
+
+        # get the 880 referred tag.
+        # An example $6 value: '650-05/$1', referred tag is 650
+        def referred_tag(field)
+          return nil unless subfield6?(field)
+
+          field['6'].strip.split('-')[0]
+        end
+
+        # check a tag in subfield 6 of a 880 datafield
+        def field_880_has_referred_tag?(tag, field)
+          referred_tag_from_880 = referred_tag(field)
+          return false unless referred_tag_from_880
+
+          referred_tag_from_880 == tag
+        end
+
+        ### referred tag end ###
+
+        # add subfield6 validation
+        def check_subfield6_format(f)
+          val = f['6']
+          reg1 = %r{^\d{3}-\d{2}/}
+          reg2 = /^\d{3}-\d{2}$/
+
+          logger.warn("Unusual subfield6 format: #{val}; correct format examples: 1) 880-02 ; 2)246-02/$1") unless reg1.match(val) || reg2.match(val)
+        end
+
+        private
+
+        # manipulate original values
+        # Delete characters when occuring at the end of a subfield value
+        def rm_punctuation(str)
+          return str if str.empty? || str.nil?
+
+          punctuations = Config.punctuations
+          char = str[-1]
+          return str unless punctuations.include? char
+
+          rm_punctuation(str.delete_suffix!(char))
+        end
+
+        def clr_value(value)
+          new_value = rm_punctuation(value)
+          ['[', ']'].each { |v| value.gsub!(v, ' ') }
+          new_value.strip
+        end
+
+        # input example: 1) 880-02 ; 2)246-02/$1
+        def seq_no(value)
+          # logger if not started with ***-** format
+          value.split('/')[0].split('-')[1].to_i # nil.to_i => 0, ''.to_i = >0
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/multiple_rule.rb

@@ -0,0 +1,36 @@
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+
+      class MultipleRule
+        include Util
+
+        attr_reader :tag_origin
+        attr_reader :tag_destination
+        attr_reader :indicator
+        attr_reader :pre_existed_tag
+        attr_reader :subfield_key
+        attr_reader :position_from_to
+
+        def initialize(row)
+          @tag_origin = row[:tag_origin]
+          @tag_destination = row[:tag_destination]
+          @indicator = Util.indicator(row[:new_indecator])
+          @pre_existed_tag = row[:map_if_no_this_tag_existed]
+          @subfield_key = row[:subfield_key]
+          @position_from_to = extract_position(row[:value_from], row[:value_to])
+        end
+
+        private
+
+        # return an array with string positons for extracting value
+        def extract_position(f, t)
+          return nil unless f && t
+
+          [f.to_i, t.to_i]
+        end
+      end
+
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/single_rule.rb

@@ -0,0 +1,149 @@
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      # TODO: ADD CSV VALIDATION, WHEN NEED ADD A NEW MAPPING CSV FILE
+      # 1. not empty row
+      # 2. a subfield name can appear in either normal mapping or combine mapping, not both
+      # 3. single map has the same amount of from names and to names
+      # 4. Combine mappping should have tree columns, validate more?
+      # 5. Combine from_subfield, to_subfield should have values, no empty
+      # 6. Tag from row[:map_if_no_this_tag_subfield_existed]), row[:map_if_no_this_tag_existed] # This tag should be the same as destination tag
+      # 7. In single map csv, one row cannot have both "map_if_no_this_tag_existed" (245) and ":map_if_no_this_tag_subfield_existed"
+      #      (245__b) because tag in these two column are  identical
+      # 8. csv file validation - a row should have coulumns:  tag origin and destintation ? single rule
+      # 9. Validating these column names
+      # 10. csv file validation - a row should have coulumns:  tag origin and destintation ? single rule
+      # 11. validating headers
+      # 12.  Formats for some of the columns
+
+      class SingleRule
+        include Util
+        attr_reader :tag_origin
+        attr_reader :tag_destination
+        attr_reader :indicator
+        attr_reader :pre_existed_tag
+        attr_reader :pre_existed_tag_subfield
+        attr_reader :single_rule_hash
+        attr_reader :single_rule_subfield_excluded_hash
+        attr_reader :combined_rules
+        attr_reader :subfields_order
+
+        def initialize(row)
+          @tag_origin = row[:tag_origin]
+          @tag_destination = row[:tag_destination]
+          @indicator = Util.indicator(row[:new_indecator])
+          @pre_existed_tag = row[:map_if_no_this_tag_existed]
+          @pre_existed_tag_subfield = existed_tag_subfield(row[:map_if_no_this_tag_subfield_existed]) # This tag should be the same as destination tag
+          @single_rule_hash = single_map_dic(row[:subfield_single_from], row[:subfield_single_to])
+          @single_rule_subfield_excluded_hash = single_map_subfield_excluded_dic
+          @combined_rules = rules_with_same_subfield_name(row)
+          @subfields_order = order(row[:order])
+        end
+
+        # 1. Return an array of combined rules, an item in the array
+        #    is an array of rules which have the same 'to subfield name'
+        # 2. An example: [[["a,b,c,d", "b", "--"],["o,p,q", "b", ""]],[["x,y,z", "a", "--"]]]
+        def rules_with_same_subfield_name(row)
+          rules = all_combined_rules(row)
+          identical_to_subfield_names = unique_tosubfield_names(rules)
+          identical_to_subfield_names.each_with_object([]) do |name, result|
+            result << rules_with_sametosubfield(name, rules)
+          end
+        end
+
+        private
+
+        def order(str)
+          str.nil? ? nil : str.split(',')
+        end
+
+        # return an array of tag and subfield name, example '255__a' => ['255','a']
+        def existed_tag_subfield(str)
+          str.nil? ? nil : str.split('__')
+        end
+
+        # list identical 'to subfield name's from combined mapping rules
+        # (an example rule ['a,b,c', 'b', ' -- '])
+        def unique_tosubfield_names(rules)
+          names = rules.map { |rule| rule[1] }
+          names.uniq
+        end
+
+        # numbers of combined rules
+        def combined_rule_counts(row)
+          headers = row.headers
+          headers.select { |h| h.to_s.include? 'subfield_combined_from_' }.count
+        end
+
+        # Three coulumns 'subfield_combined_from_*','subfield_combined_to_*','symbol_*'
+        # define a combined mapping rule
+        def combined_rule(row, i)
+          from_subfield = row["subfield_combined_from_#{i}".to_sym]
+          to_subfield = row["subfield_combined_to_#{i}".to_sym]
+          s = row["symbol_#{i}".to_sym]
+          from_subfield.nil? || to_subfield.nil? ? nil : [from_subfield, to_subfield, s] # add validation rule , such as not empty later
+        end
+
+        def all_combined_rules(row)
+          rules = []
+          n = combined_rule_counts(row)
+          (1..n).each do |i|
+            rule = combined_rule(row, i)
+            rules << rule if rule
+          end
+          rules
+        end
+
+        # list all combined rules with the same 'to subfield name'
+        def rules_with_sametosubfield(name, rules)
+          rules.each_with_object([]) do |rule, result|
+            result << rule if rule[1] == name
+          end
+        end
+
+        # Define a hash for single map rules, key is 'from subfield name',
+        # value is 'to subfield name'
+        def single_map_dic(str_from, str_to)
+          dic = {}
+          if  should_single_map?(str_from, str_to)
+            arr_from = str_from.strip.split(',')
+            arr_to = str_to.strip.split(',')
+            arr_from.each_with_index { |from_name, i| dic[from_name.to_s] = arr_to[i].to_s }
+          end
+          dic
+        end
+
+        # Hash removed the excluding subfield
+        def single_map_subfield_excluded_dic
+          dic = @single_rule_hash
+          return dic unless excluding_subfield?
+
+          excluding_subfield_name = @pre_existed_tag_subfield[1].to_s
+          dic.delete(excluding_subfield_name) if dic.key? excluding_subfield_name
+          dic
+        end
+
+        # Check excluding subfield
+        def excluding_subfield?
+          return false unless @pre_existed_tag_subfield
+          return false unless @pre_existed_tag_subfield[1]
+
+          true
+        end
+
+        # add this to validation
+        def should_single_map?(str_from, str_to)
+          return false unless str_from && str_to
+
+          arr_from = str_from.split(',')
+          arr_to = str_to.split(',')
+          return false unless arr_from.count == arr_to.count
+
+          true
+        end
+
+      end
+
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/tind_control_subfield.rb

@@ -0,0 +1,59 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module TindControlSubfield
+
+        def extract_value(rule, value)
+          pos = rule.position_from_to
+          return nil unless pos
+
+          value[pos[0]..pos[1]]
+        end
+
+        # return a mapped datafield based on rule and extract value
+        def extracted_field(rule, sub_value)
+          subname = rule.subfield_key
+          destiantion_tag = rule.tag_destination
+          indicator = rule.indicator
+          return nil unless subname && destiantion_tag && indicator
+
+          new_sub_value = clean_subfield_value(destiantion_tag, sub_value)
+          return nil unless new_sub_value
+
+          new_sub_value = clean_subfield_value(destiantion_tag, sub_value)
+          subfields = [Util.subfield(subname, new_sub_value)]
+          Util.datafield(destiantion_tag, indicator, subfields)
+        end
+
+        # pass in rules, a string value; return datafields based on rules
+        def extracted_fields_from_leader(leader_rules, leader_value)
+          new_fls = []
+          leader_rules.each do |rule|
+            sub_value = extract_value(rule, leader_value)
+            next unless sub_value
+
+            newfield = extracted_field(rule, sub_value)
+            new_fls << newfield if newfield
+          end
+          new_fls
+        end
+
+        private
+
+        def clean_subfield_value(tag, val)
+          return val if tag != '269'
+
+          new_val = val.downcase.sub(/u$/, '0')
+          qualified_269?(new_val) ? new_val : nil
+        end
+
+        def qualified_269?(val)
+          val =~ /^\d{4}$/
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/mapping/tind_field.rb

@@ -0,0 +1,49 @@
+require 'marc'
+
+module BerkeleyLibrary
+  module TIND
+    module Mapping
+      module TindField
+        class << self
+
+          def f_035_from_alma_id(alma_id, value_980)
+            val = "(#{value_980})#{alma_id}"
+            f('035', 'a', val)
+          end
+
+          def f_035(val)
+            f('035', 'a', val)
+          end
+
+          def f_245_p(val)
+            f('245', 'p', val)
+          end
+
+          def f_fft(url, txt = None)
+            return f('FFT', 'a', url) unless txt
+
+            ::MARC::DataField.new('FFT', ' ', ' ', ['d', txt], ['a', url])
+          end
+
+          def f_902_d
+            f('902', 'd', Time.now.strftime('%F'))
+          end
+
+          def f_902_n(name_initial)
+            f('902', 'n', name_initial)
+          end
+
+          def f_982_p(val)
+            f('982', 'p', val)
+          end
+
+          def f(tag, code, value)
+            ::MARC::DataField.new(tag, ' ', ' ', [code, value])
+          end
+
+        end
+      end
+    end
+
+  end
+end