search_solr_tools 6.1.0 → 6.2.0

data/lib/search_solr_tools/harvesters/ncdc_paleo.rb

@@ -1,62 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from NODC PALEO and inserts it into Solr after it has been translated
-    class NcdcPaleo < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 50
-        @translator = Helpers::IsoToSolr.new :ncdc_paleo
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of NCDC Paleo catalog from #{ncdc_paleo_url}"
-        super(method(:harvest_ncdc_paleo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NCDC_PALEO][:long_name]}\"")
-      end
-
-      def harvest_ncdc_paleo_into_solr
-        start_index = 1
-        while (entries = get_results_from_ncdc_paleo_url(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_ncdc_paleo(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def ncdc_paleo_url
-        SolrEnvironments[@environment][:ncdc_paleo_url]
-      end
-
-      def get_results_from_ncdc_paleo_url(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//csw:Record'
-      end
-
-      def get_docs_with_translated_entries_from_ncdc_paleo(entries)
-        auth_ids = entries.map { |e| e.xpath("./dc:identifier[@scheme='urn:x-esri:specification:ServiceType:ArcIMS:Metadata:DocID']").text }
-
-        auth_ids.map do |record|
-          result_xml = get_results("https://gis.ncdc.noaa.gov/gptpaleo/csw?getxml=#{record}",
-                                   '/rdf:RDF/rdf:Description').first
-          solr_doc = create_new_solr_add_doc_with_child(@translator.translate(result_xml).root)
-          insert_node = solr_doc.at_xpath('//doc')
-          insert_node.add_child("<field name='authoritative_id'>#{record}</field>")
-          insert_node.add_child("<field name='dataset_url'>https://gis.ncdc.noaa.gov/gptpaleo/catalog/search/resource/details.page?uuid=#{record}")
-          solr_doc.root
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '1000', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(ncdc_paleo_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition)
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/nmi.rb

@@ -1,34 +0,0 @@
-require_relative 'oai'
-
-module SearchSolrTools
-  module Harvesters
-    class Nmi < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:NMI][:long_name]
-        @translator = Helpers::IsoToSolr.new :nmi
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:nmi_url]
-      end
-
-      # resumption_token must be empty to stop the harvest loop; NMI's feed does not
-      # provide any resumption token and gets all the records in just one go
-      def results
-        @resumption_token = ''
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        {
-          verb: 'ListRecords',
-          metadataPrefix: 'dif'
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/nodc.rb

@@ -1,75 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from NODC and inserts it into Solr after it has been translated
-    class Nodc < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 50
-        @translator = Helpers::IsoToSolr.new :nodc
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of NODC catalog from #{nodc_url}"
-        super(method(:harvest_nodc_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NODC][:long_name]}\"")
-      end
-
-      # get translated entries from NODC and add them to Solr
-      # this is the main entry point for the class
-      def harvest_nodc_into_solr
-        start_index = 1
-        while (entries = get_results_from_nodc(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_nodc(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def nodc_url
-        SolrEnvironments[@environment][:nodc_url]
-      end
-
-      def get_results_from_nodc(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//gmi:MI_Metadata'
-      end
-
-      def get_docs_with_translated_entries_from_nodc(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(nodc_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition,
-                                                     'constraint' => bbox_constraint,
-                                                     'outputSchema' => 'http://www.isotc211.org/2005/gmd')
-      end
-
-      def bbox_constraint
-        bbox = {
-          west: '-180',
-          south: '45',
-          east: '180',
-          north: '90'
-        }
-
-        URI.encode '<Filter xmlns:ogc="http://www.opengis.net/ogc" ' \
-                   'xmlns:gml="http://www.opengis.net/gml" ' \
-                   'xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0">' \
-                   '<ogc:BBOX><PropertyName>apiso:BoundingBox</PropertyName><gml:Envelope>' \
-                   '<gml:lowerCorner>' + bbox[:west] + ' ' + bbox[:south] + '</gml:lowerCorner>' \
-                                                                            '<gml:upperCorner>' + bbox[:east] + ' ' + bbox[:north] + '</gml:upperCorner>' \
-                                                                                                                                     '</gml:Envelope></ogc:BBOX></Filter>'
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/oai.rb

@@ -1,62 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/query_builder'
-
-require 'json'
-
-module SearchSolrTools
-  module Harvesters
-    # Base class for harvesting Oai feeds into SOLR
-    class Oai < Base
-      # Used in query string params, resumptionToken
-
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        # This is updated when we harvest based on the response
-        # from the server.
-        @resumption_token = nil
-      end
-
-      def encode_data_provider_url(url)
-        URI.encode(url)
-      end
-
-      def harvest_and_delete
-        puts "Running #{self.class.name} at #{metadata_url}"
-        super(method(:harvest), %(data_centers:"#{@data_centers}"))
-      end
-
-      def harvest
-        while @resumption_token.nil? || !@resumption_token.empty?
-          begin
-            insert_solr_docs(translated_docs(results))
-          rescue => e
-            puts "ERROR: #{e.class} #{e}"
-            raise e if @die_on_failure
-          end
-        end
-      end
-
-      def results
-        fail NotImplementedError
-      end
-
-      def metadata_url
-        fail NotImplementedError
-      end
-
-      def translated_docs(entries)
-        entries.map { |e| create_new_solr_add_doc_with_child(@translator.translate(e).root) }
-      end
-
-      private
-
-      def request_params
-        fail NotImplementedError
-      end
-
-      def request_string
-        "#{metadata_url}#{Helpers::QueryBuilder.build(request_params)}"
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/pdc.rb

@@ -1,40 +0,0 @@
-require_relative 'oai'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from Polar data catalogue and inserts it into
-    # Solr after it has been translated
-    class Pdc < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:PDC][:long_name]
-        @translator = Helpers::IsoToSolr.new :pdc
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:pdc_url]
-      end
-
-      def results
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-
-        @resumption_token = list_records_oai_response.xpath('.//oai:resumptionToken', Helpers::IsoNamespaces.namespaces).first.text
-
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        # If a 'resumptionToken' is supplied with any arguments other than 'verb',
-        # the response from PDC gives a badArgument error, saying "The argument
-        # 'resumptionToken' must be supplied without other arguments"
-        {
-          verb: 'ListRecords',
-          metadataPrefix: @resumption_token.nil? ? 'iso' : nil,
-          resumptionToken: @resumption_token
-        }.delete_if { |_k, v| v.nil? }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/r2r.rb

@@ -1,61 +0,0 @@
-require_relative 'base'
-
-require 'nokogiri'
-require 'rest-client'
-
-module SearchSolrTools
-  module Harvesters
-    class R2R < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:R2R][:long_name]
-        @translator = Helpers::IsoToSolr.new :r2r
-        @metadata_url = SolrEnvironments[@environment][:r2r_url]
-      end
-
-      def harvest_and_delete
-        puts "Running #{self.class.name} at #{@metadata_url}"
-        super(method(:harvest), %(data_centers:"#{@data_centers}"))
-      end
-
-      # rubocop: disable MethodLength
-      # rubocop: disable AbcSize
-      def harvest
-        # first fetch list of available records at http://get.rvdata.us/services/cruise/
-        # then loop through each one of those, using the root <gmi:MI_Metadata> tag
-        puts "Getting list of records from #{@data_centers}"
-        RestClient.get(@metadata_url) do |resp, _req, _result, &_block|
-          unless resp.code == 200
-            puts "Got code #{resp.code} from #{@metadata_url}, skipping R2R harvest."
-            next
-          end
-
-          doc = Nokogiri::HTML(resp.body)
-
-          urls = doc.xpath('//a').map do |node|
-            "#{@metadata_url}#{node.attr('href')}"
-          end
-
-          urls.each_slice(50) do |url_subset|
-            # each result is a nokogirii doc with root element
-            # <gmi:MI_Metadata>
-            results = url_subset.map do |url|
-              get_results(url, '//gmi:MI_Metadata').first
-            end
-
-            begin
-              translated = results.map do |e|
-                create_new_solr_add_doc_with_child(@translator.translate(e).root)
-              end
-
-              insert_solr_docs(translated)
-            rescue => e
-              puts "ERROR: #{e}"
-              raise e if @die_on_failure
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/rda.rb

@@ -1,35 +0,0 @@
-require_relative 'oai'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests the RDA feed
-    class Rda < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:RDA][:long_name]
-        @translator = Helpers::IsoToSolr.new :rda
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:rda_url]
-      end
-
-      # resumption_token must be empty to stop the harvest loop; RDA's feed does not
-      # provide any resumption token and gets all the records in just one go
-      def results
-        @resumption_token = ''
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        {
-          verb: 'ListRecords',
-          metadataPrefix: 'dif'
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/tdar.rb

@@ -1,71 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from TDAR and inserts it into Solr after it has been translated
-    class Tdar < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :tdar
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of TDAR catalog from #{tdar_url}"
-        super(method(:harvest_tdar_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:TDAR][:long_name]}\"")
-      end
-
-      def harvest_tdar_into_solr
-        start_record = 0
-        total_harvested = 0
-        total_expected = total_results
-        while (entries = get_results_from_tdar(start_record)) && (entries.length > 0)
-          begin
-            insert_solr_docs(get_docs_with_translated_entries_from_tdar(entries))
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-
-          # if we have all the records we expect, don't attempt another request;
-          # it would result in an error
-          total_harvested += entries.length
-          break if total_harvested >= total_expected
-
-          start_record += @page_size
-        end
-      end
-
-      def tdar_url
-        SolrEnvironments[@environment][:tdar_url]
-      end
-
-      def get_results_from_tdar(start_record)
-        get_results(build_request(@page_size, start_record), './/atom:entry', 'application/xml')
-      end
-
-      def get_docs_with_translated_entries_from_tdar(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_request(max_records = '25', start_record = '0')
-        request_url = tdar_url + '?_tDAR.searchType=ACADIS_RSS&'\
-                                 'resourceTypes=DATASET&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].maximumLongitude=180&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].minimumLatitude=45&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].minimumLongitude=-180&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].maximumLatitude=90&'\
-                                 'geoMode=ENVELOPE&'\
-                                 'recordsPerPage=' + max_records.to_s + '&startRecord=' + start_record.to_s
-
-        request_url
-      end
-
-      def total_results
-        get_results(build_request(0, 0), './/opensearch:totalResults').text.to_i
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/usgs.rb

@@ -1,76 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from USGS and inserts it into Solr after it has been translated
-    class Usgs < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :usgs
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of USGS catalog from #{usgs_url}"
-        super(method(:harvest_usgs_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:USGS][:long_name]}\"")
-      end
-
-      # get translated entries from USGS and add them to Solr
-      # this is the main entry point for the class
-      def harvest_usgs_into_solr
-        start_index = 1
-        while (entries = get_results_from_usgs(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_usgs(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def usgs_url
-        SolrEnvironments[@environment][:usgs_url]
-      end
-
-      def get_results_from_usgs(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//gmd:MD_Metadata', ''
-      end
-
-      def get_docs_with_translated_entries_from_usgs(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(usgs_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition,
-                                                     'TypeNames' => '',
-                                                     'constraint' => bbox_constraint,
-                                                     'outputSchema' => 'http://www.isotc211.org/2005/gmd')
-      end
-
-      def bbox_constraint
-        bbox = {
-          west: '-180',
-          south: '45',
-          east: '180',
-          north: '90'
-        }
-
-        URI.encode '<Filter xmlns:ogc="http://www.opengis.net/ogc" ' \
-                   'xmlns:gml="http://www.opengis.net/gml" ' \
-                   'xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0">' \
-                   '<ogc:BBOX><PropertyName>apiso:BoundingBox</PropertyName><gml:Envelope>' \
-                   '<gml:lowerCorner>' + bbox[:west] + ' ' + bbox[:south] + '</gml:lowerCorner>' \
-                                                                            '<gml:upperCorner>' + bbox[:east] + ' ' + bbox[:north] + '</gml:upperCorner>' \
-                                                                                                                                     '</gml:Envelope></ogc:BBOX></Filter>'
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/csw_iso_query_builder.rb

@@ -1,29 +0,0 @@
-require 'search_solr_tools/helpers/query_builder'
-
-module SearchSolrTools
-  module Helpers
-    # Constructs the string to query a CSW endpoint
-    class CswIsoQueryBuilder
-      DEFAULT_PARAMS = {
-        service: 'CSW',
-        version: '2.0.2',
-        request: 'GetRecords',
-        'TypeNames' => 'gmd:MD_Metadata',
-        'ElementSetName' => 'full',
-        'resultType' => 'results',
-        'outputFormat' => 'application/xml',
-        'maxRecords' => '25',
-        'startPosition' => '1'
-      }
-
-      def self.get_query_string(url, query_params = {})
-        all_params = query_params(query_params)
-        QueryBuilder.build(all_params).prepend(url)
-      end
-
-      def self.query_params(query_params = {})
-        DEFAULT_PARAMS.merge(query_params)
-      end
-    end
-  end
-end

data/lib/search_solr_tools/helpers/data_one_format.rb

@@ -1,74 +0,0 @@
-require_relative 'iso_namespaces'
-require_relative 'iso_to_solr_format'
-require_relative 'solr_format'
-
-module SearchSolrTools
-  module Helpers
-    class DataOneFormat < IsoToSolrFormat
-      class << self
-        def date_range(node)
-          {
-            start: SolrFormat.date_str(node.xpath('.//date[@name="beginDate"]').text.strip),
-            end: SolrFormat.date_str(node.xpath('.//date[@name="endDate"]').text.strip)
-          }
-        end
-
-        def bounding_box(node)
-          {
-            north: node.xpath('.//float[@name="northBoundCoord"]').text.strip,
-            south: node.xpath('.//float[@name="southBoundCoord"]').text.strip,
-            east: node.xpath('.//float[@name="eastBoundCoord"]').text.strip,
-            west: node.xpath('.//float[@name="westBoundCoord"]').text.strip
-          }
-        end
-
-        def spatial_display(node)
-          box = bounding_box(node)
-
-          [box[:south], box[:west], box[:north], box[:east]].join(' ')
-        end
-
-        def spatial_index(node)
-          box = bounding_box(node)
-
-          if box[:west] == box[:east] && box[:south] == box[:north]
-            [box[:west], box[:south]]
-          else
-            [box[:west], box[:south], box[:east], box[:north]]
-          end.join(' ')
-        end
-
-        def spatial_area(node)
-          box = bounding_box(node)
-
-          box[:north].to_f - box[:south].to_f
-        end
-
-        def temporal_coverage(node)
-          SolrFormat.temporal_display_str(date_range(node))
-        end
-
-        def temporal_duration(node)
-          dr = date_range(node)
-          end_time = dr[:end].to_s.empty? ? Time.now : Time.parse(dr[:end])
-          SolrFormat.get_temporal_duration(Time.parse(dr[:start]), end_time) unless dr[:start].to_s.empty?
-        end
-
-        def temporal_index_string(node)
-          dr = date_range(node)
-          SolrFormat.temporal_index_str(dr)
-        end
-
-        def facet_spatial_scope(node)
-          box = bounding_box(node)
-          SolrFormat.get_spatial_scope_facet_with_bounding_box(box)
-        end
-
-        def facet_temporal_duration(node)
-          duration = temporal_duration(node)
-          SolrFormat.get_temporal_duration_facet(duration)
-        end
-      end
-    end
-  end
-end