search_solr_tools 3.1.2

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -0,0 +1,260 @@
+require 'date'
+require 'iso8601'
+
+require_relative 'bounding_box_util'
+
+module SearchSolrTools
+  module Helpers
+    #  Methods for generating formatted values that can be indexed by SOLR
+    # rubocop:disable Metrics/ModuleLength
+    module SolrFormat
+      DATA_CENTER_NAMES = {
+        NSIDC: { short_name: 'NSIDC', long_name: 'National Snow and Ice Data Center' },
+        CISL: { short_name: 'ACADIS Gateway', long_name: 'Advanced Cooperative Arctic Data and Information Service' },
+        ECHO: { short_name: 'NASA ECHO', long_name: 'NASA Earth Observing System (EOS) Clearing House (ECHO)' },
+        EOL: { short_name: 'UCAR NCAR EOL', long_name: 'UCAR NCAR - Earth Observing Laboratory' },
+        ICES: { short_name: 'ICES', long_name: 'International Council for the Exploration of the Sea' },
+        NMI: { short_name: 'Met.no', long_name: 'Norwegian Meteorological Institute' },
+        NODC: { short_name: 'NOAA NODC', long_name: 'NOAA National Oceanographic Data Center' },
+        RDA: { short_name: 'UCAR NCAR RDA', long_name: 'UCAR NCAR Research Data Archive' },
+        USGS: { short_name: 'USGS ScienceBase', long_name: 'U.S. Geological Survey ScienceBase' },
+        BCODMO: { short_name: 'BCO-DMO', long_name: 'Biological and Chemical Oceanography Data Management Office' },
+        TDAR: { short_name: 'tDAR', long_name: 'tDAR: The Digital Archaeological Record' },
+        PDC: { short_name: 'PDC', long_name: 'Polar Data Catalogue' }
+      }
+
+      NOT_SPECIFIED = 'Not specified'
+
+      TEMPORAL_RESOLUTION_FACET_VALUES = %w(Subhourly Hourly Subdaily Daily Weekly Submonthly Monthly Subyearly Yearly Multiyearly)
+      SUBHOURLY_INDEX = 0
+      HOURLY_INDEX = 1
+      SUBDAILY_INDEX = 2
+      DAILY_INDEX = 3
+      WEEKLY_INDEX = 4
+      SUBMONTHLY_INDEX = 5
+      MONTHLY_INDEX = 6
+      SUBYEARLY_INDEX = 7
+      YEARLY_INDEX = 8
+      MULTIYEARLY_INDEX = 9
+
+      SPATIAL_RESOLUTION_FACET_VALUES = ['0 - 500 m', '501 m - 1 km', '2 - 5 km', '6 - 15 km', '16 - 30 km', '>30 km']
+      SPATIAL_0_500_INDEX = 0
+      SPATIAL_501_1_INDEX = 1
+      SPATIAL_2_5_INDEX = 2
+      SPATIAL_6_15_INDEX = 3
+      SPATIAL_16_30_INDEX = 4
+      SPATIAL_GREATER_30_INDEX = 5
+
+      REDUCE_TEMPORAL_DURATION = proc { |values| reduce_temporal_duration(values) }
+      DATE = proc { |date| date_str date.text }
+
+      HTTP_URL_FORMAT = proc do |url_node|
+        url = url_node.text
+        url =~ %r{//} ? url : "http://#{url}"
+      end
+
+      def self.temporal_display_str(date_range)
+        temporal_str = "#{date_range[:start]}"
+        temporal_str += ",#{date_range[:end]}" unless date_range[:end].nil?
+        temporal_str
+      end
+
+      # returns the temporal duration in days; returns -1 if there is not a valid
+      # start date
+      def self.get_temporal_duration(start_time, end_time)
+        if start_time.to_s.empty?
+          duration = nil
+        else
+          end_time = Time.now if end_time.to_s.empty?
+          # datasets that cover just one day would have end_date - start_date = 0,
+          # so we need to add 1 to make sure the duration is the actual number of
+          # days; if the end date and start date are flipped in the metadata, a
+          # negative duration doesn't make sense so use the absolute value
+          duration = Integer((end_time - start_time).abs / 86_400) + 1
+        end
+        duration
+      end
+
+      def self.get_temporal_duration_facet(duration)
+        return NOT_SPECIFIED if duration.nil?
+        years = duration.to_i / 365
+        temporal_duration_range(years)
+      end
+
+      # We are indexing date ranges a spatial coordinates.
+      # This means we have to convert dates into the format YY.YYMMDD which can be stored in the standard lat/long space
+      # For example: 2013-01-01T00:00:00Z to 2013-01-31T00:00:00Z will be converted to 20.130101, 20.130131.
+      # See http://wiki.apache.org/solr/SpatialForTimeDurations
+      def self.temporal_index_str(date_range)
+        "#{format_date_for_index date_range[:start], MIN_DATE} #{format_date_for_index(date_range[:end], MAX_DATE)}"
+      end
+
+      def self.reduce_temporal_duration(values)
+        values.map { |v| Integer(v) rescue nil }.compact.max
+      end
+
+      def self.facet_binning(type, format_string)
+        binned_facet = bin(FacetConfiguration.get_facet_bin(type), format_string)
+        if binned_facet.nil?
+          return format_string
+        elsif binned_facet.eql?('exclude')
+          return nil
+        else
+          return binned_facet
+        end
+
+        nil
+      end
+
+      def self.parameter_binning(parameter_string)
+        binned_parameter = bin(FacetConfiguration.get_facet_bin('parameter'), parameter_string)
+        # use variable_level_1 if no mapping exists
+        if binned_parameter.nil?
+          parts = parameter_string.split '>'
+          return parts[3].strip if parts.length >= 4
+        else
+          return binned_parameter
+        end
+
+        nil
+      end
+
+      def self.resolution_value(resolution, find_index_method, resolution_values)
+        return NOT_SPECIFIED if self.resolution_not_specified? resolution
+        if resolution['type'] == 'single'
+          i = send(find_index_method, resolution['resolution'])
+          return resolution_values[i]
+        end
+        if resolution['type'] == 'range'
+          i = send(find_index_method, resolution['min_resolution'])
+          j = send(find_index_method, resolution['max_resolution'])
+          return resolution_values[i..j]
+        end
+        fail "Invalid resolution #{resolution['type']}"
+      end
+
+      def self.resolution_not_specified?(resolution)
+        return true if resolution.to_s.empty?
+        return true unless %w(single range).include? resolution['type']
+        return true if resolution['type'] == 'single' && resolution['resolution'].to_s.empty?
+        return true if resolution['type'] == 'range' && resolution['min_resolution'].to_s.empty?
+      end
+
+      def self.get_spatial_scope_facet_with_bounding_box(bbox)
+        if bbox.nil? || BoundingBoxUtil.box_invalid?(bbox)
+          return nil
+        elsif BoundingBoxUtil.box_global?(bbox)
+          facet = 'Coverage from over 85 degrees North to -85 degrees South | Global'
+        elsif BoundingBoxUtil.box_local?(bbox)
+          facet = 'Less than 1 degree of latitude change | Local'
+        else
+          facet = 'Between 1 and 170 degrees of latitude change | Regional'
+        end
+        facet
+      end
+
+      def self.date_str(date)
+        d = if date.is_a? String
+              DateTime.parse(date.strip) rescue nil
+            else
+              date
+            end
+        "#{d.iso8601[0..-7]}Z" unless d.nil?
+      end
+
+      private
+
+      MIN_DATE = '00010101'
+      MAX_DATE = Time.now.strftime('%Y%m%d')
+
+      def self.bin(mappings, term)
+        mappings.each do |mapping|
+          term.match(mapping['pattern']) do
+            return mapping['mapping']
+          end
+        end
+        nil
+      end
+
+      # rubocop:disable CyclomaticComplexity
+      def self.find_index_for_single_temporal_resolution_value(string_duration)
+        iso8601_duration = ISO8601::Duration.new(string_duration)
+
+        dur_sec = iso8601_duration.to_seconds
+
+        case dur_sec
+        when 0..3_599              then SUBHOURLY_INDEX
+        when 3600                  then HOURLY_INDEX
+        when 3601..86_399          then SUBDAILY_INDEX
+        when 86_400..172_800       then DAILY_INDEX
+        when 172_801..691_200      then WEEKLY_INDEX
+        when 691_201..1_728_000    then SUBMONTHLY_INDEX
+        when 1_728_001..2_678_400  then MONTHLY_INDEX
+        when 2_678_400..31_535_999 then SUBYEARLY_INDEX
+        when 31_536_000            then YEARLY_INDEX
+        else
+          MULTIYEARLY_INDEX
+        end
+      end
+      # rubocop:enable CyclomaticComplexity
+
+      def self.find_index_for_single_spatial_resolution_value(string_duration)
+        value, units = string_duration.split(' ')
+
+        if units == 'deg'
+          spatial_resolution_index_degrees(value)
+        elsif units == 'm'
+          spatial_resolution_index_meters(value)
+        end
+      end
+
+      def self.spatial_resolution_index_degrees(degrees)
+        if degrees.to_f <= 0.05
+          SPATIAL_2_5_INDEX
+        elsif degrees.to_f < 0.5
+          SPATIAL_16_30_INDEX
+        else
+          SPATIAL_GREATER_30_INDEX
+        end
+      end
+
+      def self.spatial_resolution_index_meters(meters)
+        case meters.to_f
+        when 0..500 then SPATIAL_0_500_INDEX
+        when 500..1_000 then SPATIAL_501_1_INDEX
+        when 1_000..5_000 then SPATIAL_2_5_INDEX
+        when 5_000..15_000 then SPATIAL_6_15_INDEX
+        when 15_000..30_000 then SPATIAL_16_30_INDEX
+        else
+          SPATIAL_GREATER_30_INDEX
+        end
+      end
+
+      # takes a temporal_duration in years, returns a string representing the range
+      # for faceting
+      def self.temporal_duration_range(years)
+        range = []
+
+        range.push '< 1 year' if years >= 0 && years < 1
+        range.push '1+ years' if years >= 1
+        range.push '5+ years' if years >= 5
+        range.push '10+ years' if years >= 10
+
+        range
+      end
+
+      def self.date?(date)
+        valid_date = if date.is_a? String
+                       d = DateTime.parse(date.strip) rescue false
+                       DateTime.valid_date?(d.year, d.mon, d.day) unless d.eql?(false)
+                     end
+        valid_date
+      end
+
+      def self.format_date_for_index(date_str, default)
+        date_str = default unless date? date_str
+        DateTime.parse(date_str).strftime('%C.%y%m%d')
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/tdar_format.rb

@@ -0,0 +1,70 @@
+require_relative './iso_namespaces'
+require_relative './iso_to_solr_format'
+require_relative './solr_format'
+
+module SearchSolrTools
+  module Helpers
+    # Special formatter for dealing with temporal metadata issues in the TDAR feed
+    class TdarFormat < IsoToSolrFormat
+      SPATIAL_DISPLAY = proc { |node| TdarFormat.spatial_display_str(node) }
+      SPATIAL_INDEX = proc { |node| TdarFormat.spatial_index_str(node) }
+      FACET_SPATIAL_SCOPE = proc { |node| TdarFormat.get_spatial_scope_facet(node) }
+
+      TEMPORAL_INDEX_STRING = proc { |node| TdarFormat.temporal_index_str(node) }
+      TEMPORAL_DISPLAY_STRING = proc { |node| TdarFormat.temporal_display_str(node) }
+      TEMPORAL_DISPLAY_STRING_FORMATTED = proc { |node| TdarFormat.temporal_display_str(node, true) }
+      TEMPORAL_DURATION = proc { |node| TdarFormat.get_temporal_duration(node) }
+      FACET_TEMPORAL_DURATION = proc { |node| TdarFormat.get_temporal_duration_facet(node) }
+
+      def self.get_spatial_scope_facet(node)
+        box = bounding_box(node)
+        SolrFormat.get_spatial_scope_facet_with_bounding_box(box)
+      end
+
+      def self.date_range(temporal_node, formatted = false)
+        xpath = '.'
+        namespaces = IsoNamespaces.namespaces(temporal_node)
+
+        temporal_node_count = temporal_node.xpath(xpath, namespaces).size
+        date_str = temporal_node.at_xpath(xpath, namespaces).text
+
+        super if temporal_node_count != 1
+
+        case date_str
+        when /^[0-9]{4}$/
+          year_to_range(date_str)
+        when /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$/
+          single_date_to_range(date_str)
+        else
+          super
+        end
+      end
+
+      def self.single_date_to_range(date)
+        {
+          start: date,
+          end: date
+        }
+      end
+
+      def self.year_to_range(year)
+        {
+          start: "#{year}-01-01",
+          end: "#{year}-12-31"
+        }
+      end
+
+      # Bounding box is defined by two coordinates to create a point.
+      # Create a bounding box from this point.
+      def self.bounding_box(node)
+        point = node.text.split(' ')
+        {
+          west: point[1],
+          south: point[0],
+          east: point[3],
+          north: point[2]
+        }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/translate_spatial_coverage.rb

@@ -0,0 +1,77 @@
+require 'rgeo/geo_json'
+
+require 'search_solr_tools/helpers/bounding_box_util'
+require 'search_solr_tools/helpers/iso_to_solr_format'
+
+module SearchSolrTools
+  module Helpers
+    # Methods to translate list of geoJson objects to solr format values
+    module TranslateSpatialCoverage
+      def self.geojson_to_spatial_display_str(spatial_coverage_geom)
+        spatial_coverage_geom = convert_multipoint_to_point(spatial_coverage_geom)
+        spatial_coverage_geom.map do |geom|
+          bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geom)
+          "#{bbox.min_y} #{bbox.min_x} #{bbox.max_y} #{bbox.max_x}"
+        end
+      end
+
+      def self.convert_multipoint_to_point(spatial_coverage_geom)
+        return_geom = []
+        spatial_coverage_geom.each do |geom|
+          if geom.geometry_type.to_s.downcase.eql?('multipoint')
+            geom.each do |point|
+              return_geom << point
+            end
+          else
+            return_geom << geom
+          end
+        end
+        return_geom
+      end
+
+      def self.geojson_to_spatial_index_str(spatial_coverage_geom)
+        spatial_coverage_geom = convert_multipoint_to_point(spatial_coverage_geom)
+        spatial_coverage_geom.map do |geo_json|
+          if geo_json.geometry_type.to_s.downcase.eql?('point')
+            "#{geo_json.x} #{geo_json.y}"
+          else
+            bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geo_json)
+            "#{bbox.min_x} #{bbox.min_y} #{bbox.max_x} #{bbox.max_y}"
+          end
+        end
+      end
+
+      def self.geojson_to_spatial_area(spatial_coverage_geom)
+        spatial_areas = spatial_coverage_geom.map do |geo_json|
+          if %w(point).include?(geo_json.geometry_type.to_s.downcase)
+            0.0
+          else
+            bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geo_json)
+            bbox.max_y - bbox.min_y
+          end
+        end
+        return nil if spatial_areas.empty?
+        spatial_areas.sort.last
+      end
+
+      def self.geojson_to_global_facet(spatial_coverage_geom)
+        return nil if spatial_coverage_geom.nil?
+        spatial_coverage_geom.each do |geo_json|
+          bbox_hash = BoundingBoxUtil.bounding_box_hash_from_geo_json(geo_json)
+          return 'Show Global Only' if BoundingBoxUtil.box_global?(bbox_hash)
+        end
+        nil
+      end
+
+      def self.geojson_to_spatial_scope_facet(spatial_coverage_geom)
+        unless spatial_coverage_geom.nil?
+          spatial_coverage_geom.map do |geo_json|
+            bbox_hash = BoundingBoxUtil.bounding_box_hash_from_geo_json(geo_json)
+            scope = SolrFormat.get_spatial_scope_facet_with_bounding_box(bbox_hash)
+            scope unless scope.nil?
+          end.uniq
+        end
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/translate_temporal_coverage.rb

@@ -0,0 +1,40 @@
+require 'rgeo/geo_json'
+
+require 'search_solr_tools/helpers/solr_format'
+
+module SearchSolrTools
+  module Helpers
+    # Methods to translate temporal coverage object to solr format values
+    module TranslateTemporalCoverage
+      def self.translate_coverages(temporal_coverages_json)
+        temporal_coverages = temporal_coverages_json.to_a.map do |coverage|
+          start_time = time_string(coverage, 'start')
+          end_time = time_string(coverage, 'end')
+
+          [
+            SolrFormat.temporal_index_str(start: start_time.to_s, end: end_time.to_s),
+            SolrFormat.temporal_display_str(start: format_string(start_time), end: format_string(end_time)),
+            SolrFormat.get_temporal_duration(start_time, end_time)
+          ]
+        end.transpose
+
+        temporal_index_str = temporal_coverages[0] || []
+        temporal_display   = temporal_coverages[1] || []
+        temporal_durations = temporal_coverages[2] || []
+
+        max_temporal_duration = SolrFormat.reduce_temporal_duration(temporal_durations)
+        facet = SolrFormat.get_temporal_duration_facet(max_temporal_duration)
+
+        { 'temporal_coverages' => temporal_display, 'temporal_duration' => max_temporal_duration, 'temporal' => temporal_index_str, 'facet_temporal_duration' => facet  }
+      end
+
+      def self.format_string(value)
+        value.to_s.empty? ? nil : value.strftime('%Y-%m-%d')
+      end
+
+      def self.time_string(coverage, key)
+        Time.parse(coverage[key]) unless coverage[key].to_s.empty?
+      end
+    end
+  end
+end