cul_hydra 1.0.0

data/lib/cul_hydra/solrizer.rb

@@ -0,0 +1,10 @@
+module Cul
+  module Hydra
+    module Solrizer
+      autoload :Extractor, "cul_hydra/solrizer/extractor"
+      autoload :TerminologyBasedSolrizer, "cul_hydra/solrizer/terminology_based_solrizer"
+      autoload :ValueMapper, "cul_hydra/solrizer/value_mapper"
+      autoload :ModsFieldable, "cul_hydra/solrizer/mods_fieldable"
+    end
+  end
+end

data/lib/cul_hydra/solrizer/extractor.rb

@@ -0,0 +1,27 @@
+module Cul::Hydra::Solrizer
+class Extractor < ::Solrizer::Extractor
+  # Insert +field_value+ for +field_name+ into +solr_doc+
+  # Handles inserting new values into a Hash while ensuring that you don't destroy or overwrite any existing values in the hash.
+  # Ensures that field values are always appended to arrays within the values hash. 
+  # Ensures that values are run through format_node_value
+  # Also ensures that values are unique if specified
+  # @param [Hash] solr_doc
+  # @param [String] field_name
+  # @param [String] field_value
+  # @param [boolean] unique
+  def self.insert_solr_field_value(solr_doc, field_name, field_value, unique=false)
+    formatted_value = self.format_node_value(field_value)
+    if solr_doc.has_key?(field_name)
+      solr_doc[field_name] << formatted_value unless (unique and solr_doc[field_name].include? formatted_value)
+    else
+      solr_doc.merge!( {field_name => [formatted_value]} ) 
+    end
+    return solr_doc
+  end
+
+  # Instance Methods
+  def insert_solr_field_value(solr_doc, field_name, field_value, unique=false)
+    Cul::Hydra::Solrizer::Extractor.insert_solr_field_value(solr_doc, field_name, field_value, unique)
+  end
+end
+end

data/lib/cul_hydra/solrizer/mods_fieldable.rb

@@ -0,0 +1,473 @@
+module Cul::Hydra::Solrizer
+	module ModsFieldable
+    extend ActiveSupport::Concern
+    include Solrizer::DefaultDescriptors::Normal
+
+    MODS_NS = {'mods'=>'http://www.loc.gov/mods/v3'}
+
+    module ClassMethods
+      def value_mapper(maps=nil)
+        @value_mapper ||= ValueMapper.new(maps)
+      end
+
+      def map_field(field_key, map_key)
+        value_mapper.map_field(field_key, map_key)
+      end
+
+      def map_value(field_key, value_key)
+        value_mapper.map_value(field_key, value_key)
+      end
+
+      def maps_field?(field_key)
+        value_mapper.maps_field? field_key
+      end
+      def normalize(t, strip_punctuation=false)
+        # strip whitespace
+        n_t = t.dup.strip
+        # collapse intermediate whitespace
+        n_t.gsub!(/\s+/, ' ')
+        # pull off paired punctuation, and any leading punctuation
+        if strip_punctuation
+          n_t = n_t.sub(/^\((.*)\)$/, "\\1")
+          n_t = n_t.sub(/^\{(.*)\}$/, "\\1")
+          n_t = n_t.sub(/^\[(.*)\]$/, "\\1")
+          n_t = n_t.sub(/^"(.*)"$/, "\\1")
+          n_t = n_t.sub(/^'(.*)'$/, "\\1")
+          n_t = n_t.sub(/^<(.*)>$/, "\\1")
+          #n_t = n_t.sub(/^\p{Ps}(.*)\p{Pe}/u, "\\1")
+          n_t = n_t.sub(/^[[:punct:]]+/, '')
+          # this may have 'created' leading/trailing space, so strip
+          n_t.strip!
+        end
+        n_t
+      end
+    end
+
+    extend ClassMethods
+
+    def mods
+      ng_xml.xpath('/mods:mods', MODS_NS).first
+    end
+
+    def projects
+      mods.xpath("./mods:relatedItem[@type='host' and @displayLabel='Project']", MODS_NS).collect do |p_node|
+        ModsFieldable.normalize(main_title(p_node), true)
+      end
+    end
+
+    def collections
+      mods.xpath("./mods:relatedItem[@type='host' and @displayLabel='Collection']", MODS_NS).collect do |p_node|
+        ModsFieldable.normalize(main_title(p_node), true)
+      end
+    end
+
+    def sort_title(node=mods)
+      # include only the untyped [!@type] titleInfo, exclude noSort
+      base_text = ''
+      t = node.xpath('./mods:titleInfo[not(@type)]', MODS_NS).first
+      if t
+        t.children.each do |child|
+          base_text << child.text unless child.name == 'nonSort'
+        end
+      end
+      base_text = ModsFieldable.normalize(base_text, true)
+      base_text = nil if base_text.empty?
+      base_text
+    end
+
+    def main_title(node=mods)
+      # include only the untyped [!@type] titleInfo
+      t = node.xpath('./mods:titleInfo[not(@type)]', MODS_NS).first
+      if t
+        ModsFieldable.normalize(t.text)
+      else
+        nil
+      end
+    end
+
+    def titles(node=mods)
+      # all titles without descending into relatedItems
+      # For now, this only includes the main title and selected alternate_titles
+      all_titles = []
+      all_titles << main_title unless main_title.nil?
+      all_titles += alternative_titles unless alternative_titles.nil?
+    end
+
+    def alternative_titles(node=mods)
+      node.xpath('./mods:titleInfo[@type and (@type="alternative" or @type="abbreviated" or @type="translated" or @type="uniform")]', MODS_NS).collect do |t|
+        ModsFieldable.normalize(t.text)
+      end
+    end
+
+    def names(role_authority=nil, role=nil)
+      # get all the name nodes
+      # keep all child text except the role terms
+      xpath = "./mods:name"
+      unless role_authority.nil?
+        xpath << "/mods:role/mods:roleTerm[@authority='#{role_authority.to_s}'"
+        unless role.nil?
+          xpath << " and normalize-space(text()) = '#{role.to_s.strip}'"
+        end
+        xpath << "]/ancestor::mods:name"
+      end
+      names = mods.xpath(xpath, MODS_NS).collect do |node|
+        base_text = node.xpath('./mods:namePart', MODS_NS).collect { |c| c.text }.join(' ')
+        ModsFieldable.normalize(base_text, true)
+      end
+
+      # Note: Removing subject names from name field extraction.
+      # See: https://issues.cul.columbia.edu/browse/DCV-231 and https://issues.cul.columbia.edu/browse/SCV-102
+      #xpath = "./mods:subject" + xpath[1,xpath.length]
+      #mods.xpath(xpath, MODS_NS).each do |node|
+      #  base_text = node.xpath('./mods:namePart', MODS_NS).collect { |c| c.text }.join(' ')
+      #  names << ModsFieldable.normalize(base_text, true)
+      #end
+
+      names
+    end
+
+    def dates(node=mods)
+      # get all the dateIssued with keyDate = 'yes', but not point = 'end'
+    end
+
+    def formats(node=mods)
+      # get all the form values with authority != 'marcform'
+      node.xpath("./mods:physicalDescription/mods:form[@authority != 'marcform']", MODS_NS).collect do |n|
+        ModsFieldable.normalize(n.text)
+      end
+    end
+
+    def repository_code(node=mods)
+      # get the location/physicalLocation[@authority = 'marcorg']
+      repo_code_node = node.xpath("./mods:location/mods:physicalLocation[@authority = 'marcorg']", MODS_NS).first
+
+      if repo_code_node
+        ModsFieldable.normalize(repo_code_node.text)
+      else
+        return nil
+      end
+    end
+
+    def repository_text(node=mods)
+      # get the location/physicalLocation[not(@authority)]
+      repo_text_node = node.xpath("./mods:location/mods:physicalLocation[not(@authority)]", MODS_NS).first
+
+      if repo_text_node
+        ModsFieldable.normalize(repo_text_node.text)
+      else
+        return nil
+      end
+    end
+
+    def translate_repo_marc_code(code, type)
+      #code = ModsFieldable.normalize(code)
+
+      if type == 'short'
+        return translate_with_default(SHORT_REPO, code, 'Non-Columbia Location')
+      elsif type == 'long'
+        return translate_with_default(LONG_REPO, code, 'Non-Columbia Location')
+      elsif type == 'full'
+        return translate_with_default(FULL_REPO, code, 'Non-Columbia Location')
+      end
+
+      return nil
+    end
+
+    def translate_project_title(project_title, type)
+      normalized_project_title = ModsFieldable.normalize(project_title)
+
+      if type == 'short'
+        return translate_with_default(SHORT_PROJ, normalized_project_title, normalized_project_title)
+      elsif type == 'full'
+        return translate_with_default(FULL_PROJ, normalized_project_title, normalized_project_title)
+      end
+
+      return nil
+    end
+
+    def shelf_locators(node=mods)
+      node.xpath("./mods:location/mods:shelfLocator", MODS_NS).collect do |n|
+        ModsFieldable.normalize(n.text, true)
+      end
+    end
+
+    def textual_dates(node=mods)
+      dates = []
+      node.xpath("./mods:originInfo/mods:dateCreated[not(@keyDate) and not(@point) and not(@w3cdtf)]", MODS_NS).collect do |n|
+        dates << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:originInfo/mods:dateIssued[not(@keyDate) and not(@point) and not(@w3cdtf)]", MODS_NS).collect do |n|
+        dates << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:originInfo/mods:dateOther[not(@keyDate) and not(@point) and not(@w3cdtf)]", MODS_NS).collect do |n|
+        dates << ModsFieldable.normalize(n.text, true)
+      end
+      return dates
+    end
+
+    def date_range_to_textual_date(start_year, end_year)
+      start_year = start_year.to_i.to_s # Remove zero-padding if present
+      end_year = end_year.to_i.to_s # Remove zero-padding if present
+
+      if start_year == end_year
+        return [start_year]
+      else
+        return [('Between ' +
+          (start_year.to_i > 0 ? start_year : start_year[1,start_year.length] + ' BCE') +
+          ' and ' +
+          (end_year.to_i > 0 ? (start_year.to_i > 0 ? end_year : end_year + ' CE') : end_year[1,end_year.length] + ' BCE')
+        )]
+      end
+    end
+
+    def date_notes(node=mods)
+      date_notes = []
+      node.xpath("./mods:note[@type = 'date' or @type = 'date source']", MODS_NS).collect do |n|
+        date_notes << ModsFieldable.normalize(n.text, true)
+      end
+      return date_notes
+    end
+
+    def non_date_notes(node=mods)
+      non_date_notes = []
+      node.xpath("./mods:note[not(@type) or (@type != 'date' and @type != 'date source')]", MODS_NS).collect do |n|
+				if n.attr('type') == 'view direction'
+					non_date_notes << 'View Direction: ' + ModsFieldable.normalize(n.text, true)
+				else
+					non_date_notes << ModsFieldable.normalize(n.text, true)
+				end
+
+
+      end
+      return non_date_notes
+    end
+
+    def item_in_context_url(node=mods)
+      item_in_context_url_val = []
+      node.xpath("./mods:location/mods:url[@access='object in context' and @usage='primary display']", MODS_NS).collect do |n|
+        item_in_context_url_val << ModsFieldable.normalize(n.text, true)
+      end
+      item_in_context_url_val
+    end
+
+    def non_item_in_context_url(node=mods)
+			non_item_in_context_url_val = []
+      node.xpath("./mods:location/mods:url[not(@access='object in context')]", MODS_NS).collect do |n|
+        non_item_in_context_url_val << ModsFieldable.normalize(n.text, true)
+      end
+      non_item_in_context_url_val
+		end
+
+    def project_url(node=mods)
+      project_url_val = []
+      node.xpath("./mods:relatedItem[@type='host' and @displayLabel='Project']/mods:location/mods:url", MODS_NS).collect do |n|
+        project_url_val << ModsFieldable.normalize(n.text, true)
+      end
+      project_url_val
+    end
+
+    def all_subjects(node=mods)
+      list_of_subjects = []
+
+      node.xpath("./mods:subject[not(@authority) or @authority != 'Durst']/mods:topic", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:subject/mods:geographic", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:subject/mods:name", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:subject/mods:temporal", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:subject/mods:titleInfo", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:subject/mods:genre", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+
+      return list_of_subjects
+    end
+
+    def durst_subjects(node=mods)
+      list_of_subjects = []
+      node.xpath("./mods:subject[@authority='Durst']/mods:topic", MODS_NS).collect do |n|
+        list_of_subjects << ModsFieldable.normalize(n.text, true)
+      end
+      return list_of_subjects
+    end
+
+    def origin_info_place(node=mods)
+      places = []
+      node.xpath("./mods:originInfo/mods:place/mods:placeTerm", MODS_NS).collect do |n|
+        places << ModsFieldable.normalize(n.text, true)
+      end
+      return places
+    end
+
+    def origin_info_place_for_display(node=mods)
+      # If there are multiple origin_info place elements, choose only the ones without valueURI attributes.  Otherwise show the others.
+      places_with_uri = []
+      places_without_uri = []
+      node.xpath("./mods:originInfo/mods:place/mods:placeTerm[@valueURI]", MODS_NS).collect do |n|
+        places_with_uri << ModsFieldable.normalize(n.text, true)
+      end
+      node.xpath("./mods:originInfo/mods:place/mods:placeTerm[not(@valueURI)]", MODS_NS).collect do |n|
+        places_without_uri << ModsFieldable.normalize(n.text, true)
+      end
+
+      return (places_without_uri.length > 0 ? places_without_uri : places_with_uri)
+    end
+
+    def coordinates(node=mods)
+      coordinate_values = []
+      node.xpath("./mods:subject/mods:cartographics/mods:coordinates", MODS_NS).collect do |n|
+        n = ModsFieldable.normalize(n.text, true)
+        if n.match(/-*\d+\.\d+\s*,\s*-*\d+\.\d+\s*/) # Expected coordinate format: 40.123456,-73.5678
+          coordinate_values << n
+        end
+      end
+      coordinate_values
+    end
+
+    def to_solr(solr_doc={})
+      solr_doc = (defined? super) ? super : solr_doc
+
+      return solr_doc if mods.nil? 	# There is no mods.  Return because there is nothing to process, otherwise NoMethodError will be raised by subsequent lines.
+
+      solr_doc["all_text_teim"] ||= []
+
+      solr_doc["title_si"] = sort_title
+      solr_doc["title_ssm"] = titles
+      solr_doc["alternative_title_ssm"] = alternative_titles
+      solr_doc["all_text_teim"] += solr_doc["alternative_title_ssm"]
+      solr_doc["lib_collection_sim"] = collections
+      solr_doc["lib_name_sim"] = names
+      solr_doc["lib_name_teim"] = solr_doc["lib_name_sim"]
+      solr_doc["all_text_teim"] += solr_doc["lib_name_teim"]
+      solr_doc["lib_all_subjects_ssm"] = all_subjects
+      solr_doc["durst_subjects_ssim"] = durst_subjects
+      solr_doc["lib_all_subjects_teim"] = solr_doc["lib_all_subjects_ssm"]
+      solr_doc["all_text_teim"] += solr_doc["lib_all_subjects_teim"]
+      solr_doc["lib_name_ssm"] = solr_doc["lib_name_sim"]
+      solr_doc["lib_author_sim"] = names(:marcrelator, 'aut')
+      solr_doc["lib_recipient_sim"] = names(:marcrelator, 'rcp')
+      solr_doc["lib_format_sim"] = formats
+      solr_doc["lib_shelf_sim"] = shelf_locators
+      solr_doc["lib_date_textual_ssm"] = textual_dates
+      solr_doc["lib_date_notes_ssm"] = date_notes
+      solr_doc["lib_non_date_notes_ssm"] = non_date_notes
+      solr_doc["lib_item_in_context_url_ssm"] = item_in_context_url
+      solr_doc["lib_non_item_in_context_url_ssm"] = non_item_in_context_url
+      solr_doc["lib_project_url_ssm"] = project_url
+      solr_doc["origin_info_place_ssm"] = origin_info_place
+      solr_doc["origin_info_place_for_display_ssm"] = origin_info_place_for_display
+
+      repo_marc_code = repository_code
+      unless repo_marc_code.nil?
+        solr_doc["lib_repo_short_ssim"] = [translate_repo_marc_code(repo_marc_code, 'short')]
+        solr_doc["lib_repo_long_sim"] = [translate_repo_marc_code(repo_marc_code, 'long')]
+        solr_doc["lib_repo_full_ssim"] = [translate_repo_marc_code(repo_marc_code, 'full')]
+      end
+      solr_doc["lib_repo_text_ssm"] = repository_text
+
+      project_titles = projects
+      unless project_titles.nil?
+        solr_doc["lib_project_short_ssim"] = []
+        solr_doc["lib_project_full_ssim"] = []
+        project_titles.each {|project_title|
+          solr_doc["lib_project_short_ssim"] << translate_project_title(project_title, 'short')
+          solr_doc["lib_project_full_ssim"] << translate_project_title(project_title, 'full')
+        }
+        solr_doc["lib_project_short_ssim"].uniq!
+        solr_doc["lib_project_full_ssim"].uniq!
+      end
+
+      # Create convenient start and end date values based on one of the many possible originInfo/dateX elements.
+      possible_start_date_fields = ['origin_info_date_issued_ssm', 'origin_info_date_issued_start_ssm', 'origin_info_date_created_ssm', 'origin_info_date_created_start_ssm', 'origin_info_date_other_ssm', 'origin_info_date_other_start_ssm']
+      possible_end_date_fields = ['origin_info_date_issued_end_ssm', 'origin_info_date_created_end_ssm', 'origin_info_date_other_end_ssm']
+      start_date = nil
+      end_date = nil
+      start_year = nil
+      end_year = nil
+      possible_start_date_fields.each{|key|
+        if solr_doc.has_key?(key)
+            start_date = solr_doc[key][0]
+          break
+        end
+      }
+      possible_end_date_fields.each{|key|
+        if solr_doc.has_key?(key)
+            end_date = solr_doc[key][0]
+          break
+        end
+      }
+
+      if start_date.present?
+
+				start_year = nil
+				end_year = nil
+
+				start_date = nil if start_date == 'uuuu'
+				end_date = nil if end_date == 'uuuu'
+				start_date = start_date.gsub('u', '0') unless start_date.nil?
+				end_date = end_date.gsub('u', '0') unless end_date.nil?
+
+        end_date = start_date if end_date.blank?
+        start_date = end_date if start_date.blank?
+
+        year_regex = /^(-?\d{1,4}).*/
+
+				unless start_date.blank?
+					start_year_match = start_date.match(year_regex)
+					if start_year_match && start_year_match.captures.length > 0
+						start_year = start_year_match.captures[0]
+						start_year = zero_pad_year(start_year)
+						solr_doc["lib_start_date_year_itsi"] = start_year.to_i # TrieInt version for searches
+					end
+				end
+
+				unless end_date.blank?
+					end_year_match = end_date.match(year_regex)
+					if end_year_match && end_year_match.captures.length > 0
+						end_year = end_year_match.captures[0]
+						end_year = zero_pad_year(end_year)
+						solr_doc["lib_end_date_year_itsi"] = end_year.to_i # TrieInt version for searches
+					end
+				end
+
+        solr_doc["lib_date_year_range_si"] = start_year + '-' + end_year if start_year && end_year
+        solr_doc["lib_date_year_range_ss"] = solr_doc["lib_date_year_range_si"]
+
+        # When no textual date is available, fall back to other date data (if available)
+        if solr_doc["lib_date_textual_ssm"].blank?
+          solr_doc["lib_date_textual_ssm"] = date_range_to_textual_date(start_year.to_i, end_year.to_i)
+        end
+      end
+
+      # Geo data
+      solr_doc["geo"] = coordinates
+
+      solr_doc.each do |k, v|
+        if self.class.maps_field? k
+          solr_doc[k] = self.class.map_value(k, v)
+        end
+      end
+
+      solr_doc
+    end
+
+    def zero_pad_year(year)
+      year = year.to_s
+      is_negative = year.start_with?('-')
+      year_without_sign = (is_negative ? year[1, year.length]: year)
+      if year_without_sign.length < 4
+        year_without_sign = year_without_sign.rjust(4, '0')
+      end
+
+      return (is_negative ? '-' : '') + year_without_sign
+    end
+  end
+end