search_solr_tools 3.1.2

data/lib/search_solr_tools/helpers/bounding_box_util.rb

@@ -0,0 +1,37 @@
+require_relative './iso_namespaces'
+
+module SearchSolrTools
+  module Helpers
+    # Utility methods for dealing with bounding boxes.
+    module BoundingBoxUtil
+      SOUTHERN_GLOBAL_BOUNDARY = -85.0
+      NORTHERN_GLOBAL_BOUNDARY = 85.0
+
+      def self.bounding_box_hash_from_geo_json(geometry)
+        if geometry_is_point?(geometry)
+          return { west: geometry.x.to_s, south: geometry.y.to_s, east: geometry.x.to_s, north: geometry.y.to_s }
+        else
+          bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geometry)
+          return { west: bbox.min_x.to_s, south: bbox.min_y.to_s, east: bbox.max_x.to_s, north: bbox.max_y.to_s }
+        end
+      end
+
+      def self.geometry_is_point?(geometry)
+        geometry.geometry_type.to_s.downcase.eql?('point')
+      end
+
+      def self.box_global?(box)
+        box[:south].to_f < SOUTHERN_GLOBAL_BOUNDARY && box[:north].to_f > NORTHERN_GLOBAL_BOUNDARY
+      end
+
+      def self.box_local?(box)
+        distance = box[:north].to_f - box[:south].to_f
+        distance < 1
+      end
+
+      def self.box_invalid?(box)
+        [:north, :south, :east, :west].any? { |d| box[d].to_s.empty? }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/csw_iso_query_builder.rb

@@ -0,0 +1,30 @@
+require 'search_solr_tools/helpers/query_builder'
+
+module SearchSolrTools
+  module Helpers
+    # Constructs the string to query a CSW endpoint
+    class CswIsoQueryBuilder
+      DEFAULT_PARAMS = {
+        service: 'CSW',
+        version: '2.0.2',
+        request: 'GetRecords',
+        'TypeNames' => 'gmd:MD_Metadata',
+        'ElementSetName' => 'full',
+        'resultType' => 'results',
+        'outputFormat' => 'application/xml',
+        'maxRecords' => '25',
+        'startPosition' => '1',
+        'outputSchema' => 'http://www.isotc211.org/2005/gmd'
+      }
+
+      def self.get_query_string(url, query_params = {})
+        all_params = query_params(query_params)
+        QueryBuilder.build(all_params).prepend(url)
+      end
+
+      def self.query_params(query_params = {})
+        DEFAULT_PARAMS.merge(query_params)
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/facet_configuration.rb

@@ -0,0 +1,19 @@
+require 'json'
+require 'rest_client'
+require 'singleton'
+
+module SearchSolrTools
+  module Helpers
+    ## Singleton configuration class to get and parse the binning configuration from the catalog services endpoint
+    class FacetConfiguration
+      include Singleton
+      def self.import_bin_configuration(env)
+        @bin_configuration = JSON.parse(RestClient.get(SolrEnvironments[env][:nsidc_dataset_metadata_url] + '/binConfiguration')) if @bin_configuration.nil?
+      end
+
+      def self.get_facet_bin(facet_name)
+        @bin_configuration.select { |x| x['facet_name'] == facet_name }.sort_by! { |x| x['order_value'] }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/iso_namespaces.rb

@@ -0,0 +1,30 @@
+module SearchSolrTools
+  module Helpers
+    # Helper class to provide default namespaces for XML document parsing.
+    class IsoNamespaces
+      def self.namespaces(doc = nil)
+        ISO_NAMESPACES.merge(doc.nil? ? {} : doc.namespaces)
+      end
+
+      ISO_NAMESPACES = {
+        'csw' => 'http://www.opengis.net/cat/csw/2.0.2',
+        'gmd' => 'http://www.isotc211.org/2005/gmd',
+        'gco' => 'http://www.isotc211.org/2005/gco',
+        'gml' => 'http://www.opengis.net/gml/3.2',
+        'gmi' => 'http://www.isotc211.org/2005/gmi',
+        'gmx' => 'http://www.isotc211.org/2005/gmx',
+        'gsr' => 'http://www.isotc211.org/2005/gsr',
+        'gss' => 'http://www.isotc211.org/2005/gss',
+        'gts' => 'http://www.isotc211.org/2005/gts',
+        'srv' => 'http://www.isotc211.org/2005/srv',
+        'xlink' => 'http://www.w3.org/1999/xlink',
+        'xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
+        'oai' => 'http://www.openarchives.org/OAI/2.0/',
+        'dif' => 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/',
+        'atom' => 'http://www.w3.org/2005/Atom',
+        'dc' => 'http://purl.org/dc/elements/1.1/',
+        'georss' => 'http://www.georss.org/georss'
+      }
+    end
+  end
+end

data/lib/search_solr_tools/helpers/iso_to_solr.rb

@@ -0,0 +1,96 @@
+require 'nokogiri'
+
+module SearchSolrTools
+  module Helpers
+    # Translates ISO nokogiri documents into solr nokogiri documents using a hash driver object
+    # This class should be constructed passing the selector file hash as a parameter (see selectors.rb)
+    # after creating an instance we call transtale with a nokogiri iso document as a parameter.
+    class IsoToSolr
+      def initialize(selector)
+        @fields = SELECTORS[selector]
+        @multiple_whitespace = /\s{2,}/ # save the regex so it is not recompiled every time format_field() is called
+      end
+
+      # this will return a nodeset with all the elements that matched the xpath
+      def eval_xpath(iso_xml_doc, xpath, multivalue, reduce)
+        fields = []
+        begin
+          iso_xml_doc.xpath(xpath, IsoNamespaces.namespaces(iso_xml_doc)).each do |f|
+            fields.push(f)
+            break if multivalue == false && reduce.nil?
+          end
+        rescue
+          fields = []
+        end
+        fields
+      end
+
+      def get_default_values(selector)
+        selector.key?(:default_values) ? selector[:default_values] : ['']
+      end
+
+      def format_text(field)
+        field.respond_to?(:text) ? field.text : field
+      end
+
+      def format_field(selector, field)
+        formatted = selector.key?(:format) ? selector[:format].call(field) : format_text(field) rescue format_text(field)
+        formatted = strip_invalid_utf8_bytes(formatted)
+        formatted.strip! if formatted.respond_to?(:strip!)
+        formatted.gsub!(@multiple_whitespace, ' ') if formatted.respond_to?(:gsub!)
+        formatted
+      end
+
+      def format_fields(selector, fields, reduce = nil)
+        formatted = fields.map { |f| format_field(selector, f) }.flatten
+        formatted = [reduce.call(formatted)] unless reduce.nil?
+        selector[:unique] ? formatted.uniq : formatted
+      end
+
+      def create_solr_fields(iso_xml_doc, selector)
+        selector[:xpaths].each do |xpath|
+          fields = eval_xpath(iso_xml_doc, xpath, selector[:multivalue], selector[:reduce])
+
+          # stop evaluating xpaths once we find data in one of them
+          if fields.size > 0 && fields.any? { |f| strip_invalid_utf8_bytes(f.text).strip.length > 0 }
+            return format_fields(selector, fields, selector[:reduce])
+          end
+        end
+        format_fields(selector, get_default_values(selector))
+      end
+
+      def translate(iso_xml_doc)
+        solr_xml_doc = Nokogiri::XML::Builder.new do |xml|
+          xml.doc_ do
+            build_fields(xml, iso_xml_doc)
+          end
+        end
+        solr_xml_doc.doc
+      end
+
+      def build_fields(xml, iso_xml_doc)
+        @fields.each do |field_name, selector|
+          create_solr_fields(iso_xml_doc, selector).each do |value|
+            if value.is_a? Array
+              value.each do |v|
+                xml.field_({ name: field_name }, v) unless v.nil? || v.eql?('')
+              end
+            else
+              xml.field_({ name: field_name }, value) unless value.nil? || value.eql?('')
+            end
+          end
+        end
+      end
+
+      def strip_invalid_utf8_bytes(text)
+        if text.respond_to?(:encode) && (!text.valid_encoding?)
+          text.encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+        end
+
+        text.gsub!("\u00BF", '') if text.respond_to?(:gsub!)
+
+        text
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/iso_to_solr_format.rb

@@ -0,0 +1,198 @@
+require 'date'
+
+require_relative './iso_namespaces'
+require_relative './solr_format'
+
+module SearchSolrTools
+  module Helpers
+    # Methods for generating formatted strings from ISO xml nodes that can be indexed by SOLR
+    # rubocop:disable ClassLength
+    class IsoToSolrFormat
+      KEYWORDS = proc { |keywords| build_keyword_list keywords }
+
+      SPATIAL_DISPLAY = proc { |node| IsoToSolrFormat.spatial_display_str(node) }
+      SPATIAL_INDEX = proc { |node| IsoToSolrFormat.spatial_index_str(node) }
+      SPATIAL_AREA = proc { |node| IsoToSolrFormat.spatial_area_str(node) }
+      MAX_SPATIAL_AREA = proc { |values| IsoToSolrFormat.get_max_spatial_area(values) }
+
+      FACET_SPONSORED_PROGRAM = proc { |node| IsoToSolrFormat.sponsored_program_facet node }
+      FACET_SPATIAL_COVERAGE = proc { |node| IsoToSolrFormat.get_spatial_facet(node) }
+      FACET_SPATIAL_SCOPE = proc { |node| IsoToSolrFormat.get_spatial_scope_facet(node) }
+      FACET_TEMPORAL_DURATION = proc { |node| IsoToSolrFormat.get_temporal_duration_facet(node) }
+
+      TEMPORAL_DURATION = proc { |node| IsoToSolrFormat.get_temporal_duration(node) }
+      TEMPORAL_INDEX_STRING = proc { |node| IsoToSolrFormat.temporal_index_str node }
+      TEMPORAL_DISPLAY_STRING = proc { |node| IsoToSolrFormat.temporal_display_str node }
+      TEMPORAL_DISPLAY_STRING_FORMATTED = proc { |node| IsoToSolrFormat.temporal_display_str(node, true) }
+
+      DATASET_URL = proc { |node| IsoToSolrFormat.dataset_url(node) }
+      ICES_DATASET_URL = proc { |node| IsoToSolrFormat.ices_dataset_url(node) }
+      EOL_AUTHOR_FORMAT = proc { |node| IsoToSolrFormat.eol_author_format(node) }
+
+      def self.spatial_display_str(box_node)
+        box = bounding_box(box_node)
+        "#{box[:south]} #{box[:west]} #{box[:north]} #{box[:east]}"
+      end
+
+      def self.spatial_index_str(box_node)
+        box = bounding_box(box_node)
+        if box[:west] == box[:east] && box[:south] == box[:north]
+          [box[:west], box[:south]]
+        else
+          [box[:west], box[:south], box[:east], box[:north]]
+        end.join(' ')
+      end
+
+      def self.spatial_area_str(box_node)
+        box = bounding_box(box_node)
+        area = box[:north].to_f - box[:south].to_f
+        area
+      end
+
+      def self.get_max_spatial_area(values)
+        values.map(&:to_f).max
+      end
+
+      def self.get_spatial_facet(box_node)
+        box = bounding_box(box_node)
+
+        if BoundingBoxUtil.box_invalid?(box)
+          facet = nil
+        elsif BoundingBoxUtil.box_global?(box)
+          facet = 'Global'
+        else
+          facet = 'Non Global'
+        end
+        facet
+      end
+
+      def self.get_spatial_scope_facet(box_node)
+        box = bounding_box(box_node)
+        SolrFormat.get_spatial_scope_facet_with_bounding_box(box)
+      end
+
+      def self.temporal_display_str(temporal_node, formatted = false)
+        SolrFormat.temporal_display_str(date_range(temporal_node, formatted))
+      end
+
+      def self.get_temporal_duration(temporal_node)
+        dr = date_range(temporal_node)
+        dr[:end].to_s.empty? ? end_time = Time.now : end_time = Time.parse(dr[:end])
+        dr[:start].to_s.empty? ? duration = nil : duration = SolrFormat.get_temporal_duration(Time.parse(dr[:start]), end_time)
+        duration
+      end
+
+      def self.get_temporal_duration_facet(temporal_node)
+        duration = get_temporal_duration(temporal_node)
+        SolrFormat.get_temporal_duration_facet(duration)
+      end
+
+      def self.temporal_index_str(temporal_node)
+        dr = date_range(temporal_node)
+        SolrFormat.temporal_index_str(dr)
+      end
+
+      def self.sponsored_program_facet(node)
+        long_name = node.xpath('.//gmd:organisationName', IsoNamespaces.namespaces(node)).text.strip
+        short_name = node.xpath('.//gmd:organisationShortName', IsoNamespaces.namespaces(node)).text.strip
+
+        [long_name, short_name].join(' | ')
+      end
+
+      def self.build_keyword_list(keywords)
+        category = keywords.xpath('.//CategoryKeyword').text
+        topic = keywords.xpath('.//TopicKeyword').text
+        term = keywords.xpath('.//TermKeyword').text
+        category << ' > ' << topic << ' > ' << term
+      end
+
+      def self.date_range(temporal_node, formatted = false)
+        start_date = get_first_matching_child(
+          temporal_node,
+          ['.//gml:beginPosition', './/BeginningDateTime', './/gco:Date', './/dif:Start_Date']
+        )
+        start_date = '' unless SolrFormat.date?(start_date)
+        start_date = SolrFormat.date_str(start_date) if formatted
+
+        end_date = get_first_matching_child(
+          temporal_node,
+          ['.//gml:endPosition', './/EndingDateTime', './/gco:Date', './/dif:Stop_Date']
+        )
+        end_date = '' unless SolrFormat.date?(end_date)
+        end_date = SolrFormat.date_str(end_date) if formatted
+
+        {
+          start: start_date,
+          end: end_date
+        }
+      end
+
+      # Met.no sometimes has bad metadata, such as <gmd:URL>SU-1 (planned activity)</gmd:URL>
+      def self.dataset_url(url_node)
+        url_node.text.strip =~ %r{http[s]?://} ? url_node.text.strip : ''
+      end
+
+      def self.ices_dataset_url(auth_id)
+        'http://geo.ices.dk/geonetwork/srv/en/main.home?uuid=' + auth_id
+      end
+
+      def self.get_first_matching_child(node, paths)
+        matching_nodes = node.at_xpath(paths.join(' | '), IsoNamespaces.namespaces(node))
+        matching_nodes.nil? ? '' : matching_nodes.text
+      end
+
+      def self.bounding_box(box_node)
+        {
+          west:  get_bound(box_node, :west),
+          south: get_bound(box_node, :south),
+          east:  get_bound(box_node, :east),
+          north: get_bound(box_node, :north)
+        }
+      end
+
+      def self.axis_label(direction)
+        {
+          north: 'Latitude',
+          south: 'Latitude',
+          east: 'Longitude',
+          west: 'Longitude'
+        }[direction]
+      end
+
+      def self.coordinate_boundary(lat_lon)
+        {
+          'Latitude' => 90,
+          'Longitude' => 180
+        }[lat_lon]
+      end
+
+      def self.node_values(box_node, direction, lat_lon)
+        get_first_matching_child(
+          box_node,
+          [
+            "./gmd:#{direction.to_s.downcase}Bounding#{lat_lon}/gco:Decimal",
+            "./gmd:#{direction.to_s.downcase}Bound#{lat_lon}/gco:Decimal",
+            "./#{direction.to_s.capitalize}BoundingCoordinate",
+            "./dif:#{direction.to_s.capitalize}ernmost_#{lat_lon}"
+          ]
+        ).split(' ')
+      end
+
+      def self.get_bound(box_node, direction)
+        lat_lon = axis_label(direction)
+
+        vals = node_values(box_node, direction, lat_lon)
+        val = vals.first
+
+        boundary = coordinate_boundary(lat_lon)
+        out_of_bounds = boundary < val.to_f.abs
+
+        return '' if vals.empty? || out_of_bounds
+
+        val = (-val.to_f) if %w(West South).include?(vals.last)
+
+        val.to_f.to_s
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/query_builder.rb

@@ -0,0 +1,13 @@
+module SearchSolrTools
+  module Helpers
+    # Class to build a query string based on a hash of params
+    class QueryBuilder
+      class << self
+        def build(params)
+          param_str = params.map { |k, v| "#{k}=#{v}" }.join('&')
+          "?#{param_str}"
+        end
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/selectors.rb

@@ -0,0 +1,20 @@
+require 'require_all'
+require_rel '../selectors'
+
+module SearchSolrTools
+  module Helpers
+    # This hash grabs all the selector files inside the selectors directory,
+    # to add a new source we need to create a selector file and add it to this hash.
+    SELECTORS = {
+      cisl:   Selectors::CISL,
+      echo:   Selectors::ECHO,
+      ices:   Selectors::ICES,
+      nmi:    Selectors::NMI,
+      nodc:   Selectors::NODC,
+      pdc:    Selectors::PDC,
+      rda:    Selectors::RDA,
+      tdar:   Selectors::TDAR,
+      usgs:   Selectors::USGS
+    }
+  end
+end