search_solr_tools 6.0.0 → 6.2.0

data/lib/search_solr_tools/harvesters/eol.rb

@@ -1,51 +0,0 @@
-require_relative 'base'
-require 'json'
-require 'rgeo/geo_json'
-
-module SearchSolrTools
-  module Harvesters
-    class Eol < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @translator = SearchSolrTools::Translators::EolToSolr.new
-      end
-
-      def harvest_and_delete
-        puts 'Running harvest of EOL catalog using the following configured EOL URLs:'
-        SearchSolrTools::SolrEnvironments[:common][:eol].each { |x| puts x }
-        super(method(:harvest_eol_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name]}\"")
-      end
-
-      def harvest_eol_into_solr
-        solr_add_queries = eol_dataset_urls.map do |dataset|
-          begin
-            doc = open_xml_document(dataset)
-            if doc.xpath('//xmlns:metadata').size > 1
-              # THREDDS allows for a dataset of datasests, EOL should not utilize this
-              fail "Complex dataset encountered at #{doc.xpath('//xmlns:catalog').to_html}"
-            end
-            metadata_doc = open_xml_document(doc.xpath('//xmlns:metadata')[0]['xlink:href'])
-            { 'add' => { 'doc' => @translator.translate(doc, metadata_doc) } }
-          rescue => e
-            puts "ERROR: #{e}"
-            puts "Failed to translate this record: #{doc} -> #{metadata_doc}"
-            raise e if @die_on_failure
-            next
-          end
-        end
-        insert_solr_docs solr_add_queries, Base::JSON_CONTENT_TYPE
-      end
-
-      def eol_dataset_urls
-        SearchSolrTools::SolrEnvironments[:common][:eol].flat_map do |endpoint|
-          doc = open_xml_document(endpoint)
-          doc.xpath('//xmlns:catalogRef').map { |node| node['xlink:href'] }
-        end
-      end
-
-      def open_xml_document(url)
-        Nokogiri::XML(open(url), &:strict)
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/gtnp.rb

@@ -1,67 +0,0 @@
-require_relative 'base'
-require 'json'
-require 'rest-client'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from GTN-P endpoints, translates and adds it to solr
-    class GtnP < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @translator = Translators::GtnpJsonToSolr.new
-      end
-
-      def gtnp_service_urls
-        json_records = []
-        SearchSolrTools::SolrEnvironments[:common][:gtnp].flat_map do |endpoint|
-          record = request_json(endpoint)
-          json_records << record
-        end
-        json_records
-      end
-
-      def harvest_and_delete
-        puts 'Running harvest of GTN-P catalog using the following configured GTN-P URLs:'
-        SearchSolrTools::SolrEnvironments[:common][:gtnp].each { |x| puts x }
-        super(method(:harvest_gtnp_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name]}\"")
-      end
-
-      def harvest_gtnp_into_solr
-        result = translate_gtnp
-        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        fail 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
-      end
-
-      def translate_gtnp
-        documents = []
-        failure_ids = []
-        gtnp_records = gtnp_service_urls
-        gtnp_records.each do |record|
-          results = parse_record(record)
-          results[:documents].each { |d| documents << d }
-          results[:failure_ids].each { |id| failure_ids << id }
-        end
-        { add_docs: documents, failure_ids: failure_ids }
-      end
-
-      def request_json(url)
-        JSON.parse(RestClient.get(url))
-      end
-
-      def parse_record(record)
-        documents = []
-        failure_ids = []
-        begin
-          record.drop(1).each do |dataset|
-            trans_doc = @translator.translate(dataset, record[0])
-            documents << { 'add' => { 'doc' => trans_doc } }
-          end
-        rescue => e
-          puts "Failed to add record #{record[0][:title]} with error #{e} (#{e.message}) : #{e.backtrace.join("\n")}"
-          failure_ids << record[0][:title]
-        end
-        { documents: documents, failure_ids: failure_ids }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/ices.rb

@@ -1,58 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from ICES and inserts it into Solr after it has been translated
-    class Ices < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :ices
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of ICES catalog from #{ices_url}"
-        super(method(:harvest_ices_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ICES][:long_name]}\"")
-      end
-
-      # get translated entries from ICES and add them to Solr
-      # this is the main entry point for the class
-      def harvest_ices_into_solr
-        start_index = 1
-        while (entries = get_results_from_ices(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_ices(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def ices_url
-        SolrEnvironments[@environment][:ices_url]
-      end
-
-      def get_results_from_ices(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//gmd:MD_Metadata'
-      end
-
-      def get_docs_with_translated_entries_from_ices(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(ices_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition,
-                                                     'constraintLanguage' => 'CQL_TEXT',
-                                                     'outputSchema' => 'http://www.isotc211.org/2005/gmd')
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/ncdc_paleo.rb

@@ -1,62 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from NODC PALEO and inserts it into Solr after it has been translated
-    class NcdcPaleo < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 50
-        @translator = Helpers::IsoToSolr.new :ncdc_paleo
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of NCDC Paleo catalog from #{ncdc_paleo_url}"
-        super(method(:harvest_ncdc_paleo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NCDC_PALEO][:long_name]}\"")
-      end
-
-      def harvest_ncdc_paleo_into_solr
-        start_index = 1
-        while (entries = get_results_from_ncdc_paleo_url(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_ncdc_paleo(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def ncdc_paleo_url
-        SolrEnvironments[@environment][:ncdc_paleo_url]
-      end
-
-      def get_results_from_ncdc_paleo_url(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//csw:Record'
-      end
-
-      def get_docs_with_translated_entries_from_ncdc_paleo(entries)
-        auth_ids = entries.map { |e| e.xpath("./dc:identifier[@scheme='urn:x-esri:specification:ServiceType:ArcIMS:Metadata:DocID']").text }
-
-        auth_ids.map do |record|
-          result_xml = get_results("https://gis.ncdc.noaa.gov/gptpaleo/csw?getxml=#{record}",
-                                   '/rdf:RDF/rdf:Description').first
-          solr_doc = create_new_solr_add_doc_with_child(@translator.translate(result_xml).root)
-          insert_node = solr_doc.at_xpath('//doc')
-          insert_node.add_child("<field name='authoritative_id'>#{record}</field>")
-          insert_node.add_child("<field name='dataset_url'>https://gis.ncdc.noaa.gov/gptpaleo/catalog/search/resource/details.page?uuid=#{record}")
-          solr_doc.root
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '1000', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(ncdc_paleo_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition)
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/nmi.rb

@@ -1,34 +0,0 @@
-require_relative 'oai'
-
-module SearchSolrTools
-  module Harvesters
-    class Nmi < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:NMI][:long_name]
-        @translator = Helpers::IsoToSolr.new :nmi
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:nmi_url]
-      end
-
-      # resumption_token must be empty to stop the harvest loop; NMI's feed does not
-      # provide any resumption token and gets all the records in just one go
-      def results
-        @resumption_token = ''
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        {
-          verb: 'ListRecords',
-          metadataPrefix: 'dif'
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/nodc.rb

@@ -1,75 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from NODC and inserts it into Solr after it has been translated
-    class Nodc < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 50
-        @translator = Helpers::IsoToSolr.new :nodc
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of NODC catalog from #{nodc_url}"
-        super(method(:harvest_nodc_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NODC][:long_name]}\"")
-      end
-
-      # get translated entries from NODC and add them to Solr
-      # this is the main entry point for the class
-      def harvest_nodc_into_solr
-        start_index = 1
-        while (entries = get_results_from_nodc(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_nodc(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def nodc_url
-        SolrEnvironments[@environment][:nodc_url]
-      end
-
-      def get_results_from_nodc(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//gmi:MI_Metadata'
-      end
-
-      def get_docs_with_translated_entries_from_nodc(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(nodc_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition,
-                                                     'constraint' => bbox_constraint,
-                                                     'outputSchema' => 'http://www.isotc211.org/2005/gmd')
-      end
-
-      def bbox_constraint
-        bbox = {
-          west: '-180',
-          south: '45',
-          east: '180',
-          north: '90'
-        }
-
-        URI.encode '<Filter xmlns:ogc="http://www.opengis.net/ogc" ' \
-                   'xmlns:gml="http://www.opengis.net/gml" ' \
-                   'xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0">' \
-                   '<ogc:BBOX><PropertyName>apiso:BoundingBox</PropertyName><gml:Envelope>' \
-                   '<gml:lowerCorner>' + bbox[:west] + ' ' + bbox[:south] + '</gml:lowerCorner>' \
-                                                                            '<gml:upperCorner>' + bbox[:east] + ' ' + bbox[:north] + '</gml:upperCorner>' \
-                                                                                                                                     '</gml:Envelope></ogc:BBOX></Filter>'
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/oai.rb

@@ -1,62 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/query_builder'
-
-require 'json'
-
-module SearchSolrTools
-  module Harvesters
-    # Base class for harvesting Oai feeds into SOLR
-    class Oai < Base
-      # Used in query string params, resumptionToken
-
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        # This is updated when we harvest based on the response
-        # from the server.
-        @resumption_token = nil
-      end
-
-      def encode_data_provider_url(url)
-        URI.encode(url)
-      end
-
-      def harvest_and_delete
-        puts "Running #{self.class.name} at #{metadata_url}"
-        super(method(:harvest), %(data_centers:"#{@data_centers}"))
-      end
-
-      def harvest
-        while @resumption_token.nil? || !@resumption_token.empty?
-          begin
-            insert_solr_docs(translated_docs(results))
-          rescue => e
-            puts "ERROR: #{e.class} #{e}"
-            raise e if @die_on_failure
-          end
-        end
-      end
-
-      def results
-        fail NotImplementedError
-      end
-
-      def metadata_url
-        fail NotImplementedError
-      end
-
-      def translated_docs(entries)
-        entries.map { |e| create_new_solr_add_doc_with_child(@translator.translate(e).root) }
-      end
-
-      private
-
-      def request_params
-        fail NotImplementedError
-      end
-
-      def request_string
-        "#{metadata_url}#{Helpers::QueryBuilder.build(request_params)}"
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/pdc.rb

@@ -1,40 +0,0 @@
-require_relative 'oai'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from Polar data catalogue and inserts it into
-    # Solr after it has been translated
-    class Pdc < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:PDC][:long_name]
-        @translator = Helpers::IsoToSolr.new :pdc
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:pdc_url]
-      end
-
-      def results
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-
-        @resumption_token = list_records_oai_response.xpath('.//oai:resumptionToken', Helpers::IsoNamespaces.namespaces).first.text
-
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        # If a 'resumptionToken' is supplied with any arguments other than 'verb',
-        # the response from PDC gives a badArgument error, saying "The argument
-        # 'resumptionToken' must be supplied without other arguments"
-        {
-          verb: 'ListRecords',
-          metadataPrefix: @resumption_token.nil? ? 'iso' : nil,
-          resumptionToken: @resumption_token
-        }.delete_if { |_k, v| v.nil? }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/r2r.rb

@@ -1,61 +0,0 @@
-require_relative 'base'
-
-require 'nokogiri'
-require 'rest-client'
-
-module SearchSolrTools
-  module Harvesters
-    class R2R < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:R2R][:long_name]
-        @translator = Helpers::IsoToSolr.new :r2r
-        @metadata_url = SolrEnvironments[@environment][:r2r_url]
-      end
-
-      def harvest_and_delete
-        puts "Running #{self.class.name} at #{@metadata_url}"
-        super(method(:harvest), %(data_centers:"#{@data_centers}"))
-      end
-
-      # rubocop: disable MethodLength
-      # rubocop: disable AbcSize
-      def harvest
-        # first fetch list of available records at http://get.rvdata.us/services/cruise/
-        # then loop through each one of those, using the root <gmi:MI_Metadata> tag
-        puts "Getting list of records from #{@data_centers}"
-        RestClient.get(@metadata_url) do |resp, _req, _result, &_block|
-          unless resp.code == 200
-            puts "Got code #{resp.code} from #{@metadata_url}, skipping R2R harvest."
-            next
-          end
-
-          doc = Nokogiri::HTML(resp.body)
-
-          urls = doc.xpath('//a').map do |node|
-            "#{@metadata_url}#{node.attr('href')}"
-          end
-
-          urls.each_slice(50) do |url_subset|
-            # each result is a nokogirii doc with root element
-            # <gmi:MI_Metadata>
-            results = url_subset.map do |url|
-              get_results(url, '//gmi:MI_Metadata').first
-            end
-
-            begin
-              translated = results.map do |e|
-                create_new_solr_add_doc_with_child(@translator.translate(e).root)
-              end
-
-              insert_solr_docs(translated)
-            rescue => e
-              puts "ERROR: #{e}"
-              raise e if @die_on_failure
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/rda.rb

@@ -1,35 +0,0 @@
-require_relative 'oai'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests the RDA feed
-    class Rda < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:RDA][:long_name]
-        @translator = Helpers::IsoToSolr.new :rda
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:rda_url]
-      end
-
-      # resumption_token must be empty to stop the harvest loop; RDA's feed does not
-      # provide any resumption token and gets all the records in just one go
-      def results
-        @resumption_token = ''
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        {
-          verb: 'ListRecords',
-          metadataPrefix: 'dif'
-        }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/tdar.rb

@@ -1,71 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from TDAR and inserts it into Solr after it has been translated
-    class Tdar < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :tdar
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of TDAR catalog from #{tdar_url}"
-        super(method(:harvest_tdar_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:TDAR][:long_name]}\"")
-      end
-
-      def harvest_tdar_into_solr
-        start_record = 0
-        total_harvested = 0
-        total_expected = total_results
-        while (entries = get_results_from_tdar(start_record)) && (entries.length > 0)
-          begin
-            insert_solr_docs(get_docs_with_translated_entries_from_tdar(entries))
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-
-          # if we have all the records we expect, don't attempt another request;
-          # it would result in an error
-          total_harvested += entries.length
-          break if total_harvested >= total_expected
-
-          start_record += @page_size
-        end
-      end
-
-      def tdar_url
-        SolrEnvironments[@environment][:tdar_url]
-      end
-
-      def get_results_from_tdar(start_record)
-        get_results(build_request(@page_size, start_record), './/atom:entry', 'application/xml')
-      end
-
-      def get_docs_with_translated_entries_from_tdar(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_request(max_records = '25', start_record = '0')
-        request_url = tdar_url + '?_tDAR.searchType=ACADIS_RSS&'\
-                                 'resourceTypes=DATASET&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].maximumLongitude=180&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].minimumLatitude=45&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].minimumLongitude=-180&'\
-                                 'groups[0].latitudeLongitudeBoxes[0].maximumLatitude=90&'\
-                                 'geoMode=ENVELOPE&'\
-                                 'recordsPerPage=' + max_records.to_s + '&startRecord=' + start_record.to_s
-
-        request_url
-      end
-
-      def total_results
-        get_results(build_request(0, 0), './/opensearch:totalResults').text.to_i
-      end
-    end
-  end
-end