search_solr_tools 6.1.0 → 6.2.0

data/lib/search_solr_tools/helpers/iso_to_solr.rb

@@ -1,97 +0,0 @@
-require_relative 'selectors'
-require 'nokogiri'
-
-module SearchSolrTools
-  module Helpers
-    # Translates ISO nokogiri documents into solr nokogiri documents using a hash driver object.
-    # This class should be constructed passing the selector file hash as a parameter (see selectors.rb).
-    # After creating an instance we call translate with a nokogiri iso document as a parameter.
-    class IsoToSolr
-      def initialize(selector)
-        @fields = SELECTORS[selector]
-        @multiple_whitespace = /\s{2,}/ # save the regex so it is not recompiled every time format_field() is called
-      end
-
-      # this will return a nodeset with all the elements that matched the xpath
-      def eval_xpath(iso_xml_doc, xpath, multivalue, reduce)
-        fields = []
-        begin
-          iso_xml_doc.xpath(xpath, IsoNamespaces.namespaces(iso_xml_doc)).each do |f|
-            fields.push(f)
-            break if multivalue == false && reduce.nil?
-          end
-        rescue
-          fields = []
-        end
-        fields
-      end
-
-      def get_default_values(selector)
-        selector.key?(:default_values) ? selector[:default_values] : ['']
-      end
-
-      def format_text(field)
-        field.respond_to?(:text) ? field.text : field
-      end
-
-      def format_field(selector, field)
-        formatted = selector.key?(:format) ? selector[:format].call(field) : format_text(field) rescue format_text(field)
-        formatted = strip_invalid_utf8_bytes(formatted)
-        formatted.strip! if formatted.respond_to?(:strip!)
-        formatted.gsub!(@multiple_whitespace, ' ') if formatted.respond_to?(:gsub!)
-        formatted
-      end
-
-      def format_fields(selector, fields, reduce = nil)
-        formatted = fields.map { |f| format_field(selector, f) }.flatten
-        formatted = [reduce.call(formatted)] unless reduce.nil?
-        selector[:unique] ? formatted.uniq : formatted
-      end
-
-      def create_solr_fields(iso_xml_doc, selector)
-        selector[:xpaths].each do |xpath|
-          fields = eval_xpath(iso_xml_doc, xpath, selector[:multivalue], selector[:reduce])
-
-          # stop evaluating xpaths once we find data in one of them
-          if fields.size > 0 && fields.any? { |f| strip_invalid_utf8_bytes(f.text).strip.length > 0 }
-            return format_fields(selector, fields, selector[:reduce])
-          end
-        end
-        format_fields(selector, get_default_values(selector))
-      end
-
-      def translate(iso_xml_doc)
-        solr_xml_doc = Nokogiri::XML::Builder.new do |xml|
-          xml.doc_ do
-            build_fields(xml, iso_xml_doc)
-          end
-        end
-        solr_xml_doc.doc
-      end
-
-      def build_fields(xml, iso_xml_doc)
-        @fields.each do |field_name, selector|
-          create_solr_fields(iso_xml_doc, selector).each do |value|
-            if value.is_a? Array
-              value.each do |v|
-                xml.field_({ name: field_name }, v) unless v.nil? || v.eql?('')
-              end
-            else
-              xml.field_({ name: field_name }, value) unless value.nil? || value.eql?('')
-            end
-          end
-        end
-      end
-
-      def strip_invalid_utf8_bytes(text)
-        if text.respond_to?(:encode) && !text.valid_encoding?
-          text.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
-        end
-
-        text.delete!("\u00BF") if text.respond_to?(:delete!)
-
-        text
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/iso_to_solr_format.rb

@@ -1,197 +0,0 @@
-require 'date'
-
-require_relative 'iso_namespaces'
-require_relative 'solr_format'
-
-module SearchSolrTools
-  module Helpers
-    # Methods for generating formatted strings from ISO xml nodes that can be indexed by SOLR
-    # rubocop:disable ClassLength
-    class IsoToSolrFormat
-      KEYWORDS = proc { |keywords| build_keyword_list keywords }
-
-      SPATIAL_DISPLAY = proc { |node| IsoToSolrFormat.spatial_display_str(node) }
-      SPATIAL_INDEX = proc { |node| IsoToSolrFormat.spatial_index_str(node) }
-      SPATIAL_AREA = proc { |node| IsoToSolrFormat.spatial_area_str(node) }
-      MAX_SPATIAL_AREA = proc { |values| IsoToSolrFormat.get_max_spatial_area(values) }
-
-      FACET_SPONSORED_PROGRAM = proc { |node| IsoToSolrFormat.sponsored_program_facet node }
-      FACET_SPATIAL_COVERAGE = proc { |node| IsoToSolrFormat.get_spatial_facet(node) }
-      FACET_SPATIAL_SCOPE = proc { |node| IsoToSolrFormat.get_spatial_scope_facet(node) }
-      FACET_TEMPORAL_DURATION = proc { |node| IsoToSolrFormat.get_temporal_duration_facet(node) }
-
-      TEMPORAL_DURATION = proc { |node| IsoToSolrFormat.get_temporal_duration(node) }
-      TEMPORAL_INDEX_STRING = proc { |node| IsoToSolrFormat.temporal_index_str node }
-      TEMPORAL_DISPLAY_STRING = proc { |node| IsoToSolrFormat.temporal_display_str node }
-      TEMPORAL_DISPLAY_STRING_FORMATTED = proc { |node| IsoToSolrFormat.temporal_display_str(node, true) }
-
-      DATASET_URL = proc { |node| IsoToSolrFormat.dataset_url(node) }
-      ICES_DATASET_URL = proc { |node| IsoToSolrFormat.ices_dataset_url(node) }
-      EOL_AUTHOR_FORMAT = proc { |node| IsoToSolrFormat.eol_author_format(node) }
-
-      def self.spatial_display_str(box_node)
-        box = bounding_box(box_node)
-        "#{box[:south]} #{box[:west]} #{box[:north]} #{box[:east]}"
-      end
-
-      def self.spatial_index_str(box_node)
-        box = bounding_box(box_node)
-        if box[:west] == box[:east] && box[:south] == box[:north]
-          [box[:west], box[:south]]
-        else
-          [box[:west], box[:south], box[:east], box[:north]]
-        end.join(' ')
-      end
-
-      def self.spatial_area_str(box_node)
-        box = bounding_box(box_node)
-        area = box[:north].to_f - box[:south].to_f
-        area
-      end
-
-      def self.get_max_spatial_area(values)
-        values.map(&:to_f).max
-      end
-
-      def self.get_spatial_facet(box_node)
-        box = bounding_box(box_node)
-
-        if BoundingBoxUtil.box_invalid?(box)
-          facet = nil
-        elsif BoundingBoxUtil.box_global?(box)
-          facet = 'Global'
-        else
-          facet = 'Non Global'
-        end
-        facet
-      end
-
-      def self.get_spatial_scope_facet(box_node)
-        box = bounding_box(box_node)
-        SolrFormat.get_spatial_scope_facet_with_bounding_box(box)
-      end
-
-      def self.temporal_display_str(temporal_node, formatted = false)
-        SolrFormat.temporal_display_str(date_range(temporal_node, formatted))
-      end
-
-      def self.get_temporal_duration(temporal_node)
-        dr = date_range(temporal_node)
-        end_time = dr[:end].to_s.empty? ? Time.now : Time.parse(dr[:end])
-        SolrFormat.get_temporal_duration(Time.parse(dr[:start]), end_time) unless dr[:start].to_s.empty?
-      end
-
-      def self.get_temporal_duration_facet(temporal_node)
-        duration = get_temporal_duration(temporal_node)
-        SolrFormat.get_temporal_duration_facet(duration)
-      end
-
-      def self.temporal_index_str(temporal_node)
-        dr = date_range(temporal_node)
-        SolrFormat.temporal_index_str(dr)
-      end
-
-      def self.sponsored_program_facet(node)
-        long_name = node.xpath('.//gmd:organisationName', IsoNamespaces.namespaces(node)).text.strip
-        short_name = node.xpath('.//gmd:organisationShortName', IsoNamespaces.namespaces(node)).text.strip
-
-        [long_name, short_name].join(' | ')
-      end
-
-      def self.build_keyword_list(keywords)
-        category = keywords.xpath('.//CategoryKeyword').text
-        topic = keywords.xpath('.//TopicKeyword').text
-        term = keywords.xpath('.//TermKeyword').text
-        category << ' > ' << topic << ' > ' << term
-      end
-
-      def self.date_range(temporal_node, formatted = false)
-        start_date = get_first_matching_child(
-          temporal_node,
-          ['.//gml:beginPosition', './/BeginningDateTime', './/gco:Date', './/dif:Start_Date']
-        )
-        start_date = '' unless SolrFormat.date?(start_date)
-        start_date = SolrFormat.date_str(start_date) if formatted
-
-        end_date = get_first_matching_child(
-          temporal_node,
-          ['.//gml:endPosition', './/EndingDateTime', './/gco:Date', './/dif:Stop_Date']
-        )
-        end_date = '' unless SolrFormat.date?(end_date)
-        end_date = SolrFormat.date_str(end_date) if formatted
-
-        {
-          start: start_date,
-          end: end_date
-        }
-      end
-
-      # Met.no sometimes has bad metadata, such as <gmd:URL>SU-1 (planned activity)</gmd:URL>
-      def self.dataset_url(url_node)
-        url_node.text.strip =~ %r{http[s]?://} ? url_node.text.strip : ''
-      end
-
-      def self.ices_dataset_url(auth_id)
-        'http://geo.ices.dk/geonetwork/srv/en/main.home?uuid=' + auth_id
-      end
-
-      def self.get_first_matching_child(node, paths)
-        matching_nodes = node.at_xpath(paths.join(' | '), IsoNamespaces.namespaces(node))
-        matching_nodes.nil? ? '' : matching_nodes.text
-      end
-
-      def self.bounding_box(box_node)
-        {
-          west:  get_bound(box_node, :west),
-          south: get_bound(box_node, :south),
-          east:  get_bound(box_node, :east),
-          north: get_bound(box_node, :north)
-        }
-      end
-
-      def self.axis_label(direction)
-        {
-          north: 'Latitude',
-          south: 'Latitude',
-          east: 'Longitude',
-          west: 'Longitude'
-        }[direction]
-      end
-
-      def self.coordinate_boundary(lat_lon)
-        {
-          'Latitude' => 90,
-          'Longitude' => 180
-        }[lat_lon]
-      end
-
-      def self.node_values(box_node, direction, lat_lon)
-        get_first_matching_child(
-          box_node,
-          [
-            "./gmd:#{direction.to_s.downcase}Bounding#{lat_lon}/gco:Decimal",
-            "./gmd:#{direction.to_s.downcase}Bound#{lat_lon}/gco:Decimal",
-            "./#{direction.to_s.capitalize}BoundingCoordinate",
-            "./dif:#{direction.to_s.capitalize}ernmost_#{lat_lon}"
-          ]
-        ).split(' ')
-      end
-
-      def self.get_bound(box_node, direction)
-        lat_lon = axis_label(direction)
-
-        vals = node_values(box_node, direction, lat_lon)
-        val = vals.first
-
-        boundary = coordinate_boundary(lat_lon)
-        out_of_bounds = boundary < val.to_f.abs
-
-        return '' if vals.empty? || out_of_bounds
-
-        val = -val.to_f if %w(West South).include?(vals.last)
-
-        val.to_f.to_s
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/ncdc_paleo_format.rb

@@ -1,61 +0,0 @@
-require 'date'
-
-require_relative 'iso_namespaces'
-require_relative 'solr_format'
-require_relative 'iso_to_solr_format'
-
-module SearchSolrTools
-  module Helpers
-    class NcdcPaleoFormat < IsoToSolrFormat
-      def self.bounding_box(node)
-        east, north = node.xpath('./ows:UpperCorner').text.split
-        west, south = node.xpath('./ows:LowerCorner').text.split
-        { north: north, south: south, east: east, west: west }
-      end
-
-      def self.date_range(node, _formatted = false)
-        if node.text.include?('START YEAR')
-          if node.text.include?('AD')
-            format_ad_time(node.text)
-          elsif node.text.include?('yr BP')
-            format_cal_yr_bp_time(node.text)
-          end
-        end
-      end
-
-      def self.format_ad_time(node_text)
-        match = node_text.match(/START YEAR:(?<start>[^*]*)AD\s*\* END YEAR:(?<end>[^*]*)AD/)
-        {
-          start: DateTime.strptime(match[:start].strip, '%Y'),
-          end: DateTime.strptime(match[:end].strip, '%Y')
-        }
-      end
-
-      def self.format_cal_yr_bp_time(node_text)
-        zero_year = 1950
-        match = node_text.match(/START YEAR:(?<start>[^*]*)... yr BP\s*\* END YEAR:(?<end>[^*]*)... yr BP/)
-        {
-          start: DateTime.strptime((-(match[:start].strip.to_i) - zero_year).to_s, '%Y'),
-          end: DateTime.strptime((-(match[:end].strip.to_i) - zero_year).to_s, '%Y')
-        }
-      end
-
-      def self.temporal_index_str(node)
-        range = date_range(node)
-        SolrFormat.temporal_index_str(start: range[:start].to_s, end: range[:end].to_s) unless range.nil?
-      end
-
-      def self.get_temporal_duration(node)
-        range = date_range(node)
-        return if range.to_s.empty?
-        (range[:start] - range[:end]).to_i.abs
-      end
-
-      def self.author(node)
-        return node if node == ''
-        return if node.text.include? ';'
-        node.text
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/query_builder.rb

@@ -1,13 +0,0 @@
-module SearchSolrTools
-  module Helpers
-    # Class to build a query string based on a hash of params
-    class QueryBuilder
-      class << self
-        def build(params)
-          param_str = params.map { |k, v| "#{k}=#{v}" }.join('&')
-          "?#{param_str}"
-        end
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/r2r_format.rb

@@ -1,25 +0,0 @@
-require_relative 'iso_namespaces'
-require_relative 'iso_to_solr_format'
-require_relative 'solr_format'
-
-module SearchSolrTools
-  module Helpers
-    class R2RFormat < IsoToSolrFormat
-      TEMPORAL_INDEX_STRING = proc { |node| R2RFormat.temporal_index_str(node) }
-      TEMPORAL_DISPLAY_STRING = proc { |node| R2RFormat.temporal_display_str(node) }
-      TEMPORAL_DURATION = proc { |node| R2RFormat.get_temporal_duration(node) }
-      FACET_TEMPORAL_DURATION = proc { |node| R2RFormat.get_temporal_duration_facet(node) }
-
-      def self.date_range(temporal_node, _formatted = false)
-        xpath_start = './/gmd:temporalElement/gmd:EX_SpatialTemporalExtent/gmd:extent/'\
-                      'gml:TimeInstant[@gml:id="start"]/gml:timePosition'
-        xpath_end = xpath_start.gsub('start', 'end')
-
-        {
-          start: temporal_node.xpath(xpath_start).text,
-          end:   temporal_node.xpath(xpath_end).text
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/selectors.rb

@@ -1,22 +0,0 @@
-Dir[File.join(__dir__, '..', 'selectors', '*.rb')].each { |file| require file }
-
-module SearchSolrTools
-  module Helpers
-    # This hash grabs all the selector files inside the selectors directory,
-    # to add a new source we need to create a selector file and add it to this hash.
-    SELECTORS = {
-      adc:         Selectors::ADC,
-      data_one:    Selectors::DATA_ONE,
-      echo:        Selectors::ECHO,
-      ices:        Selectors::ICES,
-      nmi:         Selectors::NMI,
-      ncdc_paleo:  Selectors::NCDC_PALEO,
-      nodc:        Selectors::NODC,
-      pdc:         Selectors::PDC,
-      r2r:         Selectors::R2R,
-      rda:         Selectors::RDA,
-      tdar:        Selectors::TDAR,
-      usgs:        Selectors::USGS
-    }
-  end
-end

data/lib/search_solr_tools/helpers/tdar_format.rb

@@ -1,70 +0,0 @@
-require_relative 'iso_namespaces'
-require_relative 'iso_to_solr_format'
-require_relative 'solr_format'
-
-module SearchSolrTools
-  module Helpers
-    # Special formatter for dealing with temporal metadata issues in the TDAR feed
-    class TdarFormat < IsoToSolrFormat
-      SPATIAL_DISPLAY = proc { |node| TdarFormat.spatial_display_str(node) }
-      SPATIAL_INDEX = proc { |node| TdarFormat.spatial_index_str(node) }
-      FACET_SPATIAL_SCOPE = proc { |node| TdarFormat.get_spatial_scope_facet(node) }
-
-      TEMPORAL_INDEX_STRING = proc { |node| TdarFormat.temporal_index_str(node) }
-      TEMPORAL_DISPLAY_STRING = proc { |node| TdarFormat.temporal_display_str(node) }
-      TEMPORAL_DISPLAY_STRING_FORMATTED = proc { |node| TdarFormat.temporal_display_str(node, true) }
-      TEMPORAL_DURATION = proc { |node| TdarFormat.get_temporal_duration(node) }
-      FACET_TEMPORAL_DURATION = proc { |node| TdarFormat.get_temporal_duration_facet(node) }
-
-      def self.get_spatial_scope_facet(node)
-        box = bounding_box(node)
-        SolrFormat.get_spatial_scope_facet_with_bounding_box(box)
-      end
-
-      def self.date_range(temporal_node, formatted = false)
-        xpath = '.'
-        namespaces = IsoNamespaces.namespaces(temporal_node)
-
-        temporal_node_count = temporal_node.xpath(xpath, namespaces).size
-        date_str = temporal_node.at_xpath(xpath, namespaces).text
-
-        super if temporal_node_count != 1
-
-        case date_str
-        when /^[0-9]{4}$/
-          year_to_range(date_str)
-        when /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/
-          single_date_to_range(date_str)
-        else
-          super
-        end
-      end
-
-      def self.single_date_to_range(date)
-        {
-          start: date,
-          end: date
-        }
-      end
-
-      def self.year_to_range(year)
-        {
-          start: "#{year}-01-01",
-          end: "#{year}-12-31"
-        }
-      end
-
-      # Bounding box is defined by two coordinates to create a point.
-      # Create a bounding box from this point.
-      def self.bounding_box(node)
-        point = node.text.split(' ')
-        {
-          west: point[1],
-          south: point[0],
-          east: point[3],
-          north: point[2]
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/usgs_format.rb

@@ -1,50 +0,0 @@
-require_relative 'iso_namespaces'
-require_relative 'iso_to_solr_format'
-
-module SearchSolrTools
-  module Helpers
-    # Special formatter for dealing with temporal metadata issues in the USGS feed
-    class UsgsFormat < IsoToSolrFormat
-      TEMPORAL_INDEX_STRING = proc { |node| UsgsFormat.temporal_index_str(node) }
-      TEMPORAL_DISPLAY_STRING = proc { |node| UsgsFormat.temporal_display_str(node) }
-      TEMPORAL_DURATION = proc { |node| UsgsFormat.get_temporal_duration(node) }
-      FACET_TEMPORAL_DURATION = proc { |node| UsgsFormat.get_temporal_duration_facet(node) }
-
-      # for USGS, a single date entry (i.e., missing either start or end date, and
-      # the value that is present is not clearly labeled) means the whole year if
-      # just a year is given, or just a single day if just a single day is given
-      def self.date_range(temporal_node, formatted = false)
-        xpath = './/gco:Date'
-        namespaces = IsoNamespaces.namespaces(temporal_node)
-
-        temporal_node_count = temporal_node.xpath(xpath, namespaces).size
-        date_str = temporal_node.at_xpath(xpath, namespaces).text
-
-        super if temporal_node_count != 1
-
-        case date_str
-        when /^[0-9]{4}$/
-          year_to_range(date_str)
-        when /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/
-          single_date_to_range(date_str)
-        else
-          super
-        end
-      end
-
-      def self.single_date_to_range(date)
-        {
-          start: date,
-          end: date
-        }
-      end
-
-      def self.year_to_range(year)
-        {
-          start: "#{year}-01-01",
-          end: "#{year}-12-31"
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/selectors/adc.rb

@@ -1,96 +0,0 @@
-require_relative '../helpers/solr_format'
-require_relative '../helpers/data_one_format'
-
-module SearchSolrTools
-  module Selectors
-    ADC = {
-      authoritative_id: {
-        xpaths: ['.//str[@name="id"]'],
-        multivalue: false
-      },
-      title: {
-        xpaths: ['.//str[@name="title"]'],
-        multivalue: false
-      },
-      summary: {
-        xpaths: ['.//str[@name="abstract"]'],
-        multivalue: false
-      },
-      data_centers: {
-        xpaths: [''],
-        default_values: [Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]],
-        multivalue: false
-      },
-      authors: {
-        xpaths: ['.//str[@name="author"]'],
-        multivalue: false
-      },
-      keywords: {
-        xpaths: ['.//arr[@name="keywords"]/str'],
-        multivalue: true
-      },
-      last_revision_date: {
-        xpaths: ['.//date[@name="updateDate"]'],
-        default_values: [Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
-        multivalue: false,
-        format: Helpers::SolrFormat::DATE
-      },
-      dataset_url: {
-        xpaths: ['.//str[@name="dataUrl"]'],
-        default_values: [''],
-        multivalue: false
-      },
-      spatial_coverages: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:spatial_display)
-      },
-      spatial: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:spatial_index)
-      },
-      spatial_area: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:spatial_area)
-      },
-      temporal_coverages: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:temporal_coverage)
-      },
-      temporal_duration: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:temporal_duration)
-      },
-      temporal: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:temporal_index_string)
-      },
-      source: {
-        xpaths: [''],
-        default_values: ['ADE'],
-        multivalue: false
-      },
-      facet_data_center: {
-        xpaths: [''],
-        default_values: ["#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:short_name]}"],
-        multivalue: false
-      },
-      facet_spatial_scope: {
-        xpaths: ['.'],
-        multivalue: false,
-        format: Helpers::DataOneFormat.method(:facet_spatial_scope)
-      },
-      facet_temporal_duration: {
-        xpaths: ['.'],
-        default_values: [Helpers::SolrFormat::NOT_SPECIFIED],
-        format: Helpers::DataOneFormat.method(:facet_temporal_duration),
-        multivalue: false
-      }
-    }
-  end
-end