search_solr_tools 3.1.2

data/lib/search_solr_tools/helpers/usgs_format.rb

@@ -0,0 +1,50 @@
+require_relative './iso_namespaces'
+require_relative './iso_to_solr_format'
+
+module SearchSolrTools
+  module Helpers
+    # Special formatter for dealing with temporal metadata issues in the USGS feed
+    class UsgsFormat < IsoToSolrFormat
+      TEMPORAL_INDEX_STRING = proc { |node| UsgsFormat.temporal_index_str(node) }
+      TEMPORAL_DISPLAY_STRING = proc { |node| UsgsFormat.temporal_display_str(node) }
+      TEMPORAL_DURATION = proc { |node| UsgsFormat.get_temporal_duration(node) }
+      FACET_TEMPORAL_DURATION = proc { |node| UsgsFormat.get_temporal_duration_facet(node) }
+
+      # for USGS, a single date entry (i.e., missing either start or end date, and
+      # the value that is present is not clearly labeled) means the whole year if
+      # just a year is given, or just a single day if just a single day is given
+      def self.date_range(temporal_node, formatted = false)
+        xpath = './/gco:Date'
+        namespaces = IsoNamespaces.namespaces(temporal_node)
+
+        temporal_node_count = temporal_node.xpath(xpath, namespaces).size
+        date_str = temporal_node.at_xpath(xpath, namespaces).text
+
+        super if temporal_node_count != 1
+
+        case date_str
+        when /^[0-9]{4}$/
+          year_to_range(date_str)
+        when /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/
+          single_date_to_range(date_str)
+        else
+          super
+        end
+      end
+
+      def self.single_date_to_range(date)
+        {
+          start: date,
+          end: date
+        }
+      end
+
+      def self.year_to_range(year)
+        {
+          start: "#{year}-01-01",
+          end: "#{year}-12-31"
+        }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/selectors/cisl.rb

@@ -0,0 +1,112 @@
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Selectors
+    # The hash contains keys that should map to the fields in the solr schema,
+    # the keys are called selectors and are in charge of selecting the nodes
+    # from the ISO document, applying the default value if none of the xpaths
+    # resolved to a value and formatting the field.  xpaths and multivalue are
+    # required, default_value, format, and reduce are optional.
+    #
+    # reduce takes the formatted result of multiple nodes and produces a single
+    #   result. This is for fields that are not multivalued, but their value
+    #   should consider information from all the nodes (for example, storing
+    #   only the maximum duration from multiple temporal coverage fields, taking
+    #   the sum of multiple spatial areas)
+    CISL = {
+      authoritative_id: {
+        xpaths: ['.//oai:header/oai:identifier'],
+        multivalue: false
+      },
+      title: {
+        xpaths: ['.//dif:Entry_Title'],
+        multivalue: false
+      },
+      summary: {
+        xpaths: ['.//dif:Summary/dif:Abstract'],
+        multivalue: false
+      },
+      data_centers: {
+        xpaths: [''],
+        default_values: [SearchSolrTools::Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:long_name]],
+        multivalue: false
+      },
+      authors: {
+        xpaths: [''],
+        multivalue: true
+      },
+      keywords: {
+        xpaths: [
+          './/dif:Parameters/dif:Category',
+          './/dif:Parameters/dif:Topic',
+          './/dif:Parameters/dif:Term',
+          './/dif:Parameters/dif:Variable_Level_1'
+        ].reverse,
+        multivalue: true
+      },
+      last_revision_date: {
+        xpaths: ['.//dif:Last_DIF_Revision_Date'],
+        default_values: [SearchSolrTools::Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
+        multivalue: false,
+        format: SearchSolrTools::Helpers::SolrFormat::DATE
+      },
+      dataset_url: {
+        xpaths: ['.//dif:Related_URL/dif:URL'],
+        multivalue: false
+      },
+      spatial_coverages: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: true,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::SPATIAL_DISPLAY
+      },
+      spatial: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: true,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::SPATIAL_INDEX
+      },
+      spatial_area: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: false,
+        reduce: SearchSolrTools::Helpers::IsoToSolrFormat::MAX_SPATIAL_AREA,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::SPATIAL_AREA
+      },
+      temporal: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        multivalue: true,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::TEMPORAL_INDEX_STRING
+      },
+      temporal_coverages: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        multivalue: true,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::TEMPORAL_DISPLAY_STRING
+      },
+      temporal_duration: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        multivalue: false,
+        reduce: SearchSolrTools::Helpers::SolrFormat::REDUCE_TEMPORAL_DURATION,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::TEMPORAL_DURATION
+      },
+      source: {
+        xpaths: [''],
+        default_values: ['ADE'],
+        multivalue: false
+      },
+      facet_data_center: {
+        xpaths: [''],
+        default_values: ["#{SearchSolrTools::Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:short_name]}"],
+        multivalue: false
+      },
+      facet_spatial_scope: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: true,
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::FACET_SPATIAL_SCOPE
+      },
+      facet_temporal_duration: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        default_values: [SearchSolrTools::Helpers::SolrFormat::NOT_SPECIFIED],
+        format: SearchSolrTools::Helpers::IsoToSolrFormat::FACET_TEMPORAL_DURATION,
+        multivalue: true
+      }
+    }
+  end
+end

data/lib/search_solr_tools/selectors/echo_iso.rb

@@ -0,0 +1,111 @@
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Selectors
+    # The hash contains keys that should map to the fields in the solr schema,
+    # the keys are called selectors and are in charge of selecting the nodes
+    # from the ISO document, applying the default value if none of the xpaths
+    # resolved to a value and formatting the field. xpaths and multivalue are
+    # required, default_value and format are optional
+    ECHO = {
+      authoritative_id: {
+        xpaths: ['.//@echo_dataset_id'],
+        multivalue: false
+      },
+      title: {
+        xpaths: ['.//Collection/LongName'],
+        multivalue: false
+      },
+      summary: {
+        xpaths: ['.//Collection/Description'],
+        multivalue: false
+      },
+      data_centers: {
+        xpaths: [''],
+        default_values: [Helpers::SolrFormat::DATA_CENTER_NAMES[:ECHO][:long_name]],
+        multivalue: false
+      },
+      authors: {
+        xpaths: [''],
+        multivalue: true
+      },
+      keywords: {
+        xpaths: ['.//Collection/ScienceKeywords/ScienceKeyword'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::KEYWORDS
+      },
+      last_revision_date: {
+        xpaths: ['.//Collection/LastUpdate'],
+        default_values: [Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
+        multivalue: false,
+        format: Helpers::SolrFormat::DATE
+      },
+      dataset_url: {
+        xpaths: ['.//Collection/OnlineResources/OnlineResource[contains(./Type/text(),"static URL")]/URL',
+                 './/Collection/OnlineResources/OnlineResource[contains(./Type/text(), "VIEW RELATED INFORMATION")]/URL',
+                 './/Collection/OnlineAccessURLs/OnlineAccessURL/[contains(./URLDescription/text(), "Data Access")]/URL',
+                 './/Collection/OnlineResources/OnlineResource[contains(./Type/text(),"Guide Document for this product at NSIDC")]/URL',
+                 './/Collection/OnlineResources/OnlineResource[contains(./Type/text(),"DOI URL")]/URL',
+                 './/Collection/OnlineResources/OnlineResource[contains(./Type/text(),"ECSCollGuide")]/URL',
+                 './/Collection/OnlineResources/OnlineResource[contains(./Type/text(),"GET DATA : ON-LINE ARCHIVE")]/URL',
+                 './/Collection/OnlineResources/OnlineResource/URL',
+                 './/Collection/OnlineAccessURLs/OnlineAccessURL/URL'],
+        default_values: ['https://earthdata.nasa.gov/echo'],
+        multivalue: false
+      },
+      spatial_coverages: {
+        xpaths: ['.//Collection/Spatial/HorizontalSpatialDomain/Geometry/BoundingRectangle'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::SPATIAL_DISPLAY
+      },
+      spatial: {
+        xpaths: ['.//Collection/Spatial/HorizontalSpatialDomain/Geometry/BoundingRectangle'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::SPATIAL_INDEX
+      },
+      spatial_area: {
+        xpaths: ['.//Collection/Spatial/HorizontalSpatialDomain/Geometry/BoundingRectangle'],
+        multivalue: false,
+        reduce: Helpers::IsoToSolrFormat::MAX_SPATIAL_AREA,
+        format: Helpers::IsoToSolrFormat::SPATIAL_AREA
+      },
+      temporal_coverages: {
+        xpaths: ['.//Collection/Temporal/RangeDateTime'],
+        multivalue: true,
+        format:  Helpers::IsoToSolrFormat::TEMPORAL_DISPLAY_STRING_FORMATTED
+      },
+      temporal_duration: {
+        xpaths: ['.//Collection/Temporal/RangeDateTime'],
+        multivalue: false,
+        reduce: Helpers::SolrFormat::REDUCE_TEMPORAL_DURATION,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_DURATION
+      },
+      temporal: {
+        xpaths: ['.//Collection/Temporal/RangeDateTime'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_INDEX_STRING
+      },
+      source: {
+        xpaths: [''],
+        default_values: ['ADE'],
+        multivalue: false
+      },
+      facet_data_center: {
+        xpaths: [''],
+        default_values: ["#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ECHO][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:ECHO][:short_name]}"],
+        multivalue: false
+      },
+      facet_spatial_scope: {
+        xpaths: ['.//Collection/Spatial/HorizontalSpatialDomain/Geometry/BoundingRectangle'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::FACET_SPATIAL_SCOPE
+      },
+      facet_temporal_duration: {
+        xpaths: ['.//Collection/Temporal/RangeDateTime'],
+        default_values: [Helpers::SolrFormat::NOT_SPECIFIED],
+        format: Helpers::IsoToSolrFormat::FACET_TEMPORAL_DURATION,
+        multivalue: true
+      }
+    }
+  end
+end

data/lib/search_solr_tools/selectors/ices_iso.rb

@@ -0,0 +1,107 @@
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Selectors
+    # The hash contains keys that should map to the fields in the solr schema,
+    # the keys are called selectors and are in charge of selecting the nodes
+    # from the ISO document, applying the default value if none of the xpaths
+    # resolved to a value and formatting the field.  xpaths and multivalue are
+    # required, default_value and format are optional
+    ICES = {
+      authoritative_id: {
+        xpaths: ['.//gmd:fileIdentifier/gco:CharacterString'],
+        multivalue: false
+      },
+      title: {
+        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:title/gco:CharacterString'],
+        multivalue: false
+      },
+      summary: {
+        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:abstract/gco:CharacterString'],
+        multivalue: false
+      },
+      data_centers: {
+        xpaths: [''],
+        default_values: [Helpers::SolrFormat::DATA_CENTER_NAMES[:ICES][:long_name]],
+        multivalue: false
+      },
+      authors: {
+        xpaths: [".//gmd:CI_ResponsibleParty[./gmd:role/gmd:CI_RoleCode[@codeListValue='principalInvestigator']]/gmd:individualName/gco:CharacterString"],
+        multivalue: true
+      },
+      keywords: {
+        xpaths: ['.//gmd:descriptiveKeywords/gmd:MD_Keywords/gmd:keyword/gco:CharacterString',
+                 './/gmd:descriptiveKeywords/gmd:MD_Keywords/gmd:keyword/gmx:Anchor'],
+        multivalue: true
+      },
+      last_revision_date: {
+        xpaths: ['.//gmd:dateStamp/gco:Date', './/gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date/gmd:date/gco:DateTime'],
+        default_values: [Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
+        multivalue: false,
+        format: Helpers::SolrFormat::DATE
+      },
+      dataset_url: {
+        xpaths: ['.//gmd:fileIdentifier/gco:CharacterString'],
+        multivalue: false,
+        format: Helpers::IsoToSolrFormat:: ICES_DATASET_URL
+      },
+      spatial_coverages: {
+        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::SPATIAL_DISPLAY
+      },
+      spatial: {
+        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::SPATIAL_INDEX
+      },
+      spatial_area: {
+        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
+        multivalue: false,
+        reduce: Helpers::IsoToSolrFormat::MAX_SPATIAL_AREA,
+        format: Helpers::IsoToSolrFormat::SPATIAL_AREA
+      },
+      temporal_coverages: {
+        xpaths: ['.//gmd:EX_TemporalExtent'],
+        multivalue: false,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_DISPLAY_STRING_FORMATTED
+      },
+      temporal_duration: {
+        xpaths: ['.//gmd:EX_TemporalExtent'],
+        multivalue: false,
+        reduce: Helpers::SolrFormat::REDUCE_TEMPORAL_DURATION,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_DURATION
+      },
+      temporal: {
+        xpaths: ['.//gmd:EX_TemporalExtent'],
+        multivalue: true,
+        format:  Helpers::IsoToSolrFormat::TEMPORAL_INDEX_STRING
+      },
+      sensors: {
+        xpaths: ['.//gmi:acquisitionInformation/gmi:MI_AcquisitionInformation/gmi:instrument/gmi:MI_Instrument/gmi:citation/gmd:CI_Citation/gmd:title/gco:CharacterString'],
+        multivalue: true
+      },
+      source: {
+        xpaths: [''],
+        default_values: ['ADE'],
+        multivalue: false
+      },
+      facet_data_center: {
+        xpaths: [''],
+        default_values: ["#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ICES][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:ICES][:short_name]}"],
+        multivalue: false
+      },
+      facet_spatial_scope: {
+        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::FACET_SPATIAL_SCOPE
+      },
+      facet_temporal_duration: {
+        xpaths: ['.//gmd:EX_TemporalExtent'],
+        default_values: [Helpers::SolrFormat::NOT_SPECIFIED],
+        format: Helpers::IsoToSolrFormat::FACET_TEMPORAL_DURATION,
+        multivalue: true
+      }
+    }
+  end
+end

data/lib/search_solr_tools/selectors/nmi.rb

@@ -0,0 +1,106 @@
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Selectors
+    # The hash contains keys that should map to the fields in the solr schema,
+    # the keys are called selectors and are in charge of selecting the nodes
+    # from the ISO document, applying the default value if none of the xpaths
+    # resolved to a value and formatting the field.  xpaths and multivalue are
+    # required, default_value and format are optional.
+    NMI = {
+      authoritative_id: {
+        xpaths: ['.//oai:header/oai:identifier'],
+        multivalue: false
+      },
+      title: {
+        xpaths: ['.//dif:Entry_Title'],
+        multivalue: false
+      },
+      summary: {
+        xpaths: ['.//dif:Summary'],
+        multivalue: false
+      },
+      data_centers: {
+        xpaths: [''],
+        default_values: [Helpers::SolrFormat::DATA_CENTER_NAMES[:NMI][:long_name]],
+        multivalue: false
+      },
+      authors: {
+        xpaths: [''],
+        multivalue: true
+      },
+      keywords: {
+        xpaths: [
+          './/dif:Parameters/dif:Category',
+          './/dif:Parameters/dif:Topic',
+          './/dif:Parameters/dif:Term',
+          './/dif:Parameters/dif:Variable_Level_1'
+        ].reverse,
+        multivalue: true
+      },
+      last_revision_date: {
+        xpaths: ['.//dif:Last_DIF_Revision_Date'],
+        default_values: [Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
+        multivalue: false,
+        format: Helpers::SolrFormat::DATE
+      },
+      dataset_url: {
+        xpaths: ['.//dif:Related_URL/dif:URL'],
+        multivalue: false
+      },
+      spatial_coverages: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::SPATIAL_DISPLAY
+      },
+      spatial: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::SPATIAL_INDEX
+      },
+      spatial_area: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: false,
+        reduce: Helpers::IsoToSolrFormat::MAX_SPATIAL_AREA,
+        format: Helpers::IsoToSolrFormat::SPATIAL_AREA
+      },
+      temporal: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_INDEX_STRING
+      },
+      temporal_coverages: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_DISPLAY_STRING
+      },
+      temporal_duration: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        multivalue: false,
+        reduce: Helpers::SolrFormat::REDUCE_TEMPORAL_DURATION,
+        format: Helpers::IsoToSolrFormat::TEMPORAL_DURATION
+      },
+      source: {
+        xpaths: [''],
+        default_values: ['ADE'],
+        multivalue: false
+      },
+      facet_data_center: {
+        xpaths: [''],
+        default_values: ["#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NMI][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:NMI][:short_name]}"],
+        multivalue: false
+      },
+      facet_spatial_scope: {
+        xpaths: ['.//dif:Spatial_Coverage'],
+        multivalue: true,
+        format: Helpers::IsoToSolrFormat::FACET_SPATIAL_SCOPE
+      },
+      facet_temporal_duration: {
+        xpaths: ['.//dif:Temporal_Coverage'],
+        default_values: [Helpers::SolrFormat::NOT_SPECIFIED],
+        format: Helpers::IsoToSolrFormat::FACET_TEMPORAL_DURATION,
+        multivalue: true
+      }
+    }
+  end
+end