search_solr_tools 6.1.0 → 6.3.0

data/lib/search_solr_tools/selectors/usgs_iso.rb

@@ -1,107 +0,0 @@
-require_relative '../helpers/solr_format'
-require_relative '../helpers/iso_to_solr_format'
-require_relative '../helpers/usgs_format'
-
-module SearchSolrTools
-  module Selectors
-    # The hash contains keys that should map to the fields in the solr schema,
-    # the keys are called selectors and are in charge of selecting the nodes
-    # from the ISO document, applying the default value if none of the xpaths
-    # resolved to a value and formatting the field. xpaths and multivalue are
-    # required, default_value and format are optional
-    USGS = {
-      authoritative_id: {
-        xpaths: ['.//gmd:fileIdentifier/gco:CharacterString'],
-        multivalue: false
-      },
-      title: {
-        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:title/gco:CharacterString'],
-        multivalue: false
-      },
-      summary: {
-        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:abstract/gco:CharacterString'],
-        multivalue: false
-      },
-      data_centers: {
-        xpaths: [''],
-        default_values: [Helpers::SolrFormat::DATA_CENTER_NAMES[:USGS][:long_name]],
-        multivalue: false
-      },
-      authors: {
-        xpaths: [".//gmd:contact/gmd:CI_ResponsibleParty[./gmd:role/gmd:CI_RoleCode[@codeListValue='originator']]/gmd:organisationName/gco:CharacterString"],
-        multivalue: true
-      },
-      keywords: {
-        xpaths: ['.//gmd:descriptiveKeywords/gmd:MD_Keywords/gmd:keyword/gco:CharacterString'],
-        multivalue: true
-      },
-      last_revision_date: {
-        xpaths: ['.//gmd:dateStamp/gco:DateTime'],
-        default_values: [Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
-        multivalue: false,
-        format: Helpers::SolrFormat::DATE
-      },
-      dataset_url: {
-        xpaths: ['.//gmd:transferOptions/gmd:MD_DigitalTransferOptions/gmd:onLine/gmd:CI_OnlineResource[contains(./gmd:name/gco:CharacterString/text(),"Summary")]/gmd:linkage/gmd:URL'],
-        multivalue: false
-      },
-      spatial_coverages: {
-        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
-        multivalue: true,
-        format: Helpers::IsoToSolrFormat::SPATIAL_DISPLAY
-      },
-      spatial: {
-        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
-        multivalue: true,
-        format: Helpers::IsoToSolrFormat::SPATIAL_INDEX
-      },
-      spatial_area: {
-        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
-        multivalue: false,
-        reduce: Helpers::IsoToSolrFormat::MAX_SPATIAL_AREA,
-        format: Helpers::IsoToSolrFormat::SPATIAL_AREA
-      },
-      temporal: {
-        xpaths: [".//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date[./gmd:dateType/gmd:CI_DateTypeCode[@codeListValue='Time Period']]/gmd:date"],
-        multivalue: true,
-        format: Helpers::UsgsFormat::TEMPORAL_INDEX_STRING
-      },
-      temporal_coverages: {
-        xpaths: [".//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date[./gmd:dateType/gmd:CI_DateTypeCode[@codeListValue='Time Period']]/gmd:date"],
-        multivalue: true,
-        format: Helpers::UsgsFormat::TEMPORAL_DISPLAY_STRING
-      },
-      temporal_duration: {
-        xpaths: [".//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date[./gmd:dateType/gmd:CI_DateTypeCode[@codeListValue='Time Period']]/gmd:date"],
-        multivalue: false,
-        reduce: Helpers::SolrFormat::REDUCE_TEMPORAL_DURATION,
-        format: Helpers::UsgsFormat::TEMPORAL_DURATION
-      },
-      sensors: {
-        xpaths: [''],
-        multivalue: true
-      },
-      source: {
-        xpaths: [''],
-        default_values: ['ADE'],
-        multivalue: false
-      },
-      facet_data_center: {
-        xpaths: [''],
-        default_values: ["#{Helpers::SolrFormat::DATA_CENTER_NAMES[:USGS][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:USGS][:short_name]}"],
-        multivalue: false
-      },
-      facet_spatial_scope: {
-        xpaths: ['.//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:extent/gmd:EX_Extent/gmd:geographicElement/gmd:EX_GeographicBoundingBox'],
-        multivalue: true,
-        format: Helpers::IsoToSolrFormat::FACET_SPATIAL_SCOPE
-      },
-      facet_temporal_duration: {
-        xpaths: [".//gmd:identificationInfo/gmd:MD_DataIdentification/gmd:citation/gmd:CI_Citation/gmd:date/gmd:CI_Date[./gmd:dateType/gmd:CI_DateTypeCode[@codeListValue='Time Period']]/gmd:date"],
-        default_values: [Helpers::SolrFormat::NOT_SPECIFIED],
-        format: Helpers::UsgsFormat::FACET_TEMPORAL_DURATION,
-        multivalue: true
-      }
-    }
-  end
-end

data/lib/search_solr_tools/translators/bcodmo_json.rb

@@ -1,89 +0,0 @@
-require 'json'
-require 'rest-client'
-require 'rgeo/geo_json'
-require 'rgeo/wkrep/wkt_parser'
-
-require 'search_solr_tools'
-require_relative '../helpers/solr_format'
-require_relative '../helpers/translate_temporal_coverage'
-require_relative '../helpers/translate_spatial_coverage'
-
-module SearchSolrTools
-  module Translators
-    # Translates Bcodmo json to solr json format
-    class BcodmoJsonToSolr
-      # rubocop:disable MethodLength
-      # rubocop:disable AbcSize
-      def translate(json_doc, json_record, geometry)
-        originators = json_doc.key?('people') ? JSON.parse(RestClient.get((json_doc['people']))) : []
-        spatial_values = translate_geometry geometry
-        temporal_coverage_values = Helpers::TranslateTemporalCoverage.translate_coverages [{ 'start' => "#{json_record['startDate']}", 'end' => "#{json_record['endDate']}" }]
-        {
-          'title' => json_doc['dataset_name'],
-          'authoritative_id' => json_record['id'] + json_doc['dataset_nid'],
-          'dataset_version' => translate_dataset_version(json_doc['dataset_version']),
-          'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:long_name],
-          'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:short_name]}",
-          'summary' => json_doc['dataset_description'].to_s.empty? ? json_doc['dataset_brief_description'] : json_doc['dataset_description'],
-          'temporal_coverages' => temporal_coverage_values['temporal_coverages'],
-          'temporal_duration' => temporal_coverage_values['temporal_duration'],
-          'temporal' => temporal_coverage_values['temporal'],
-          'facet_temporal_duration' => temporal_coverage_values['facet_temporal_duration'],
-          'last_revision_date' => json_doc['dataset_deployment_version_date'].to_s.empty? ? nil : Time.parse(json_doc['dataset_deployment_version_date']).strftime('%Y-%m-%dT%H:%M:%SZ'),
-          'dataset_url' => json_doc['dataset_url'],
-          'source' => 'ADE',
-          'facet_spatial_coverage' => spatial_values[:global_facet],
-          'facet_spatial_scope' => spatial_values[:spatial_scope_facet],
-          'spatial_coverages' => spatial_values[:spatial_display],
-          'spatial_area' => spatial_values[:spatial_area],
-          'spatial' => spatial_values[:spatial_index],
-          'data_access_urls' => json_doc.key?('dataset_deployment_url') ? json_doc['dataset_deployment_url'] : [],
-          'authors' => parse_people(originators)
-        }
-      end
-      # rubocop:enable MethodLength
-
-      def translate_dataset_version(dataset_version)
-        version_translation = dataset_version.to_s.gsub(/\D/, '')
-        version_translation.empty? ? nil : version_translation
-      end
-
-      def parse_people(people_json)
-        people_json.map { |entry| entry['person_name'] } unless people_json.empty?
-      end
-
-      def translate_geometry(wkt_geom)
-        if wkt_geom['geometry']['type'] == 'LineString'
-          wkt_geom['geometry']['type'] = 'MultiPoint'
-        end
-        geometry = RGeo::GeoJSON.decode(wkt_geom).geometry
-        geometry = RGeo::Feature.cast(geometry, RGeo::Feature::MultiPoint)
-
-        # This feed sometimes returns MultiLineString but wrongly calls them 'LineString'
-        # If the above fails, we assume this is why. If the feed gets fixed, this code
-        # should still handle that.
-        if geometry.nil? || geometry.num_geometries == 0
-          # Try to decode as an actual MultiLineString.
-          wkt_geom['geometry']['type'] = 'MultiLineString'
-          geometry = RGeo::GeoJSON.decode(wkt_geom).geometry
-
-          # Convert to a MultiPoint, for passing into the helper functions below.
-          coords = geometry.coordinates.flatten
-          coords = coords.each_slice(2).to_a
-          f = RGeo::Geos.factory
-          points = []
-          coords.each { |x, y| points << f.point(x, y) }
-          geometry = f.multi_point(points)
-        end
-
-        {
-          spatial_display: Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str([geometry]),
-          spatial_index: Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str([geometry]),
-          spatial_area: Helpers::TranslateSpatialCoverage.geojson_to_spatial_area([geometry]),
-          global_facet: Helpers::TranslateSpatialCoverage.geojson_to_global_facet([geometry]),
-          spatial_scope_facet: Helpers::TranslateSpatialCoverage.geojson_to_spatial_scope_facet([geometry])
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/translators/eol_to_solr.rb

@@ -1,84 +0,0 @@
-require 'search_solr_tools'
-require_relative '../helpers/solr_format'
-require_relative '../helpers/translate_temporal_coverage'
-require_relative '../helpers/translate_spatial_coverage'
-require_relative '../helpers/bounding_box_util'
-
-module SearchSolrTools
-  module Translators
-    # Translates an EOL THREDDS dataset link set into a SOLR json ingest record
-    class EolToSolr
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/MethodLength
-
-      def translate(title_metadata, dataset_metadata)
-        temporal_coverage_values = Helpers::TranslateTemporalCoverage.translate_coverages get_time_coverages(dataset_metadata)
-        rev_date = dataset_metadata.xpath('//xmlns:date[@type="metadataCreated"]').text
-        geospatial_coverage = parse_geospatial_coverages(dataset_metadata)
-        {
-          'title' => title_metadata.xpath('//xmlns:dataset').first['name'],
-          'authoritative_id' => title_metadata.xpath('//xmlns:dataset').first['ID'],
-          'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name],
-          'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:short_name]}",
-          'summary' => dataset_metadata.xpath('//xmlns:documentation[@type="summary"]').text,
-          'temporal_coverages' => temporal_coverage_values['temporal_coverages'],
-          'temporal_duration' => temporal_coverage_values['temporal_duration'],
-          'temporal' => temporal_coverage_values['temporal'],
-          'facet_temporal_duration' => temporal_coverage_values['facet_temporal_duration'],
-          'last_revision_date' => rev_date.empty? ? Helpers::SolrFormat.date_str(DateTime.now) : Helpers::SolrFormat.date_str(rev_date),
-          'source' => 'ADE',
-          'keywords' => dataset_metadata.xpath('//xmlns:keyword').map(&:text),
-          'authors' => dataset_metadata.xpath('//xmlns:contributor[@role="author"]').map { |node| parse_eol_authors(node.text) }.join(', '),
-          'dataset_url' => eol_dataset_url(dataset_metadata),
-          'facet_spatial_coverage' => Helpers::BoundingBoxUtil.box_global?(geospatial_coverage),
-          'facet_spatial_scope' => Helpers::SolrFormat.get_spatial_scope_facet_with_bounding_box(geospatial_coverage),
-          'spatial_coverages' => %i(south west north east).map { |d| geospatial_coverage[d] }.join(' '),
-          'spatial_area' => spatial_coverage_to_spatial_area(geospatial_coverage),
-          'spatial' => %i(west south east north).map { |d| geospatial_coverage[d] }.join(' ')
-        }
-      end
-
-      def eol_dataset_url(node)
-        node.xpath('//xmlns:documentation[@xlink:href]').each do |doc|
-          return doc['xlink:href'] if doc['xlink:href'].match('http://data.eol.ucar.edu/codiac/dss/id=(\S*)')
-        end
-      rescue Nokogiri::XML::XPath::SyntaxError
-        puts "Warning - no documentation URL found in the following node: #{node.to_html}"
-      end
-
-      def parse_eol_authors(author)
-        if author.include?(' AT ') && author.include?(' dot ')
-          author = author[0..author.rindex(',') - 1]
-        end
-        author
-      end
-
-      def get_time_coverages(doc)
-        doc.xpath('//xmlns:timeCoverage').map do |node|
-          { 'start' => node.xpath('./xmlns:start').text,
-            'end' => node.xpath('./xmlns:end').text }
-        end
-      end
-
-      def spatial_coverage_to_spatial_area(coverage)
-        return if [:north, :south].any? { |x| coverage[x].nil? }
-        coverage[:north].abs - coverage[:south].abs
-      end
-
-      def parse_geospatial_coverages(doc)
-        node = doc.xpath('//xmlns:geospatialCoverage')
-        south = node.xpath('./xmlns:northsouth/xmlns:start').text.to_f
-        north = south + node.xpath('./xmlns:northsouth/xmlns:size').text.to_f
-        west = node.xpath('./xmlns:eastwest/xmlns:start').text.to_f
-        east = west + node.xpath('./xmlns:eastwest/xmlns:size').text.to_f
-        # EOL uses out-of-range east-west values to represent bounding boxes
-        # that cross the date line.   For any box with a value out of range,
-        # adjust the east/west value to lie within the -180 to 180 range.
-        east -= 360 if east > 180
-        west += 360 if west < -180
-
-        { east: east, west: west, north: north, south: south }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/translators/gtnp_json.rb

@@ -1,59 +0,0 @@
-require 'json'
-require 'rest-client'
-require 'rgeo/geo_json'
-
-require 'search_solr_tools'
-require_relative '../helpers/solr_format'
-require_relative '../helpers/translate_temporal_coverage'
-require_relative '../helpers/translate_spatial_coverage'
-
-module SearchSolrTools
-  module Translators
-    # Translates GTN-P json to solr json format
-    class GtnpJsonToSolr
-      # rubocop:disable AbcSize
-      def translate(json_doc, json_record)
-        json_geo = json_doc['geo'].nil? ? json_doc['coordinates'] : json_doc['geo']['coordinates']
-        concatenated_name = "#{json_record['title']} - #{json_doc['name']}"
-        spatial_values = translate_geometry json_geo
-        {
-          'title' => concatenated_name,
-          'authoritative_id' => concatenated_name,
-          'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name],
-          'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:short_name]}",
-          'summary' => json_record['abstract'].to_s,
-          'dataset_url' => json_doc['link'],
-          'source' => 'ADE',
-          'facet_spatial_scope' => spatial_values[:spatial_scope_facet],
-          'spatial_coverages' => spatial_values[:spatial_display],
-          'spatial_area' => spatial_values[:spatial_area],
-          'spatial' => spatial_values[:spatial_index],
-          'temporal_coverages' => Helpers::SolrFormat::NOT_SPECIFIED,
-          'facet_temporal_duration' => Helpers::SolrFormat::NOT_SPECIFIED,
-          'authors' => parse_people(json_doc)
-        }
-      end
-
-      def parse_people(json_doc)
-        people_found = []
-        return people_found unless json_doc.key?('citation') && json_doc['citation'].key?('contacts')
-        citation = json_doc['citation']
-        citation['contacts'].each do |person|
-          people_found << "#{person['givenName']} #{person['familyName']}"
-        end
-        people_found
-      end
-
-      def translate_geometry(json_geom)
-        geo_string = "{\"type\":\"Point\",\"coordinates\":[#{json_geom['longitude']},#{json_geom['latitude']}]}"
-        geometry = RGeo::GeoJSON.decode(geo_string, json_parser: :json)
-        {
-          spatial_display: Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str([geometry]),
-          spatial_index: Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str([geometry]),
-          spatial_area: Helpers::TranslateSpatialCoverage.geojson_to_spatial_area([geometry]),
-          spatial_scope_facet: Helpers::TranslateSpatialCoverage.geojson_to_spatial_scope_facet([geometry])
-        }
-      end
-    end
-  end
-end