search_solr_tools 3.1.2

data/lib/search_solr_tools/harvesters/nmi.rb

@@ -0,0 +1,32 @@
+module SearchSolrTools
+  module Harvesters
+    class Nmi < Oai
+      def initialize(env = 'development', die_on_failure = false)
+        super
+        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:NMI][:long_name]
+        @translator = Helpers::IsoToSolr.new :nmi
+      end
+
+      def metadata_url
+        SolrEnvironments[@environment][:nmi_url]
+      end
+
+      # resumption_token must be empty to stop the harvest loop; NMI's feed does not
+      # provide any resumption token and gets all the records in just one go
+      def results
+        @resumption_token = ''
+        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
+        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
+      end
+
+      private
+
+      def request_params
+        {
+          verb: 'ListRecords',
+          metadataPrefix: 'dif'
+        }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/nodc.rb

@@ -0,0 +1,72 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from NODC and inserts it into Solr after it has been translated
+    class Nodc < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @page_size = 100
+        @translator = Helpers::IsoToSolr.new :nodc
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of NODC catalog from #{nodc_url}"
+        super(method(:harvest_nodc_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NODC][:long_name]}\"")
+      end
+
+      # get translated entries from NODC and add them to Solr
+      # this is the main entry point for the class
+      def harvest_nodc_into_solr
+        start_index = 1
+        while (entries = get_results_from_nodc(start_index)) && (entries.length > 0)
+          begin
+            insert_solr_docs get_docs_with_translated_entries_from_nodc(entries)
+          rescue => e
+            puts "ERROR: #{e}"
+            raise e if @die_on_failure
+          end
+          start_index += @page_size
+        end
+      end
+
+      def nodc_url
+        SolrEnvironments[@environment][:nodc_url]
+      end
+
+      def get_results_from_nodc(start_index)
+        get_results build_csw_request('results', @page_size, start_index), '//gmi:MI_Metadata'
+      end
+
+      def get_docs_with_translated_entries_from_nodc(entries)
+        docs = []
+        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
+        docs
+      end
+
+      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
+        Helpers::CswIsoQueryBuilder.get_query_string(nodc_url,
+                                                     'resultType' => resultType,
+                                                     'maxRecords' => maxRecords,
+                                                     'startPosition' => startPosition,
+                                                     'constraint' => bbox_constraint
+                                                    )
+      end
+
+      def bbox_constraint
+        bbox = {
+          west: '-180',
+          south: '45',
+          east: '180',
+          north: '90'
+        }
+
+        URI.encode '<Filter xmlns:ogc="http://www.opengis.net/ogc" ' \
+                   'xmlns:gml="http://www.opengis.net/gml" ' \
+                   'xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0">' \
+                   '<ogc:BBOX><PropertyName>apiso:BoundingBox</PropertyName><gml:Envelope>' \
+                   '<gml:lowerCorner>' + bbox[:west] + ' ' + bbox[:south] + '</gml:lowerCorner>' \
+                                                                            '<gml:upperCorner>' + bbox[:east] + ' ' + bbox[:north] + '</gml:upperCorner>' \
+                                                                                                                                     '</gml:Envelope></ogc:BBOX></Filter>'
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/nsidc_auto_suggest.rb

@@ -0,0 +1,33 @@
+module SearchSolrTools
+  module Harvesters
+    class NsidcAutoSuggest < AutoSuggest
+      def harvest_and_delete
+        puts 'Building auto-suggest indexes for NSIDC'
+        super(method(:harvest), "source:\"NSIDC\"", @env_settings[:auto_suggest_collection_name])
+      end
+
+      def harvest
+        url = "#{solr_url}/#{@env_settings[:collection_name]}/select?q=*%3A*&fq=source%3ANSIDC&rows=0&wt=json&indent=true&facet=true&facet.mincount=1&facet.sort=count&facet.limit=-1"
+        super url, fields
+      end
+
+      def fields
+        { 'authoritative_id' => { weight: 1, source: 'NSIDC', creator: method(:standard_add_creator) },
+          'full_title' => { weight: 2, source: 'NSIDC', creator: method(:standard_add_creator) },
+          'copy_parameters' => { weight: 5, source: 'NSIDC', creator: method(:standard_add_creator) },
+          'full_platforms' => { weight: 2, source: 'NSIDC', creator: method(:short_full_split_add_creator) },
+          'full_sensors' => { weight: 2, source: 'NSIDC', creator: method(:short_full_split_add_creator) },
+          'full_authors' => { weight: 1, source: 'NSIDC', creator: method(:standard_add_creator) }
+        }
+      end
+
+      def short_full_split_add_creator(value, count, field_weight, source)
+        add_docs = []
+        value.split(' > ').each do |v|
+          add_docs.concat(standard_add_creator(v, count, field_weight, source))
+        end
+        add_docs
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -0,0 +1,60 @@
+require 'json'
+require 'rest-client'
+
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from NSIDC OAI and inserts it into Solr after it has been translated
+    class NsidcJson < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @translator = Translators::NsidcJsonToSolr.new
+        Helpers::FacetConfiguration.import_bin_configuration(env)
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of NSIDC catalog from #{nsidc_json_url}"
+        super(method(:harvest_nsidc_json_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name]}\"")
+      end
+
+      # get translated entries from NSIDC OAI and add them to Solr
+      # this is the main entry point for the class
+      def harvest_nsidc_json_into_solr
+        result = docs_with_translated_entries_from_nsidc
+        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
+        fail 'Failed to harvest and insert some authoritative IDs' if result[:failure_ids].length > 0
+      end
+
+      def nsidc_json_url
+        SolrEnvironments[@environment][:nsidc_dataset_metadata_url]
+      end
+
+      def result_ids_from_nsidc
+        get_results SolrEnvironments[@environment][:nsidc_oai_identifiers_url], '//xmlns:identifier'
+      end
+
+      def fetch_json_from_nsidc(id)
+        json_response = RestClient.get(nsidc_json_url + id + '.json')
+        JSON.parse(json_response)
+      end
+
+      def docs_with_translated_entries_from_nsidc
+        docs = []
+        failure_ids = []
+
+        result_ids_from_nsidc.each do |r|
+          id = r.text.split('/').last
+          begin
+            docs << { 'add' => { 'doc' => @translator.translate(fetch_json_from_nsidc(id)) } }
+          rescue => e
+            puts "Failed to fetch #{id} with error #{e}: #{e.backtrace}"
+            failure_ids << id
+          end
+        end
+
+        { add_docs: docs, failure_ids: failure_ids }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/oai.rb

@@ -0,0 +1,59 @@
+require 'json'
+
+module SearchSolrTools
+  module Harvesters
+    # Base class for harvesting Oai feeds into SOLR
+    class Oai < Base
+      # Used in query string params, resumptionToken
+
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        # This is updated when we harvest based on the response
+        # from the server.
+        @resumption_token = nil
+      end
+
+      def encode_data_provider_url(url)
+        URI.encode(url)
+      end
+
+      def harvest_and_delete
+        puts "Running #{self.class.name} at #{metadata_url}"
+        super(method(:harvest), %(data_centers:"#{@data_centers}"))
+      end
+
+      def harvest
+        while @resumption_token.nil? || !@resumption_token.empty?
+          begin
+            insert_solr_docs(translated_docs(results))
+          rescue => e
+            puts "ERROR: #{e}"
+            raise e if @die_on_failure
+          end
+        end
+      end
+
+      def results
+        fail NotImplementedError
+      end
+
+      def metadata_url
+        fail NotImplementedError
+      end
+
+      def translated_docs(entries)
+        entries.map { |e| create_new_solr_add_doc_with_child(@translator.translate(e).root) }
+      end
+
+      private
+
+      def request_params
+        fail NotImplementedError
+      end
+
+      def request_string
+        "#{metadata_url}#{Helpers::QueryBuilder.build(request_params)}"
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/pdc.rb

@@ -0,0 +1,38 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from Polar data catalogue and inserts it into
+    # Solr after it has been translated
+    class Pdc < Oai
+      def initialize(env = 'development', die_on_failure = false)
+        super
+        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:PDC][:long_name]
+        @translator = Helpers::IsoToSolr.new :pdc
+      end
+
+      def metadata_url
+        SolrEnvironments[@environment][:pdc_url]
+      end
+
+      def results
+        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
+
+        @resumption_token = list_records_oai_response.xpath('.//oai:resumptionToken', Helpers::IsoNamespaces.namespaces).first.text
+
+        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
+      end
+
+      private
+
+      def request_params
+        # If a 'resumptionToken' is supplied with any arguments other than 'verb',
+        # the response from PDC gives a badArgument error, saying "The argument
+        # 'resumptionToken' must be supplied without other arguments"
+        {
+          verb: 'ListRecords',
+          metadataPrefix: @resumption_token.nil? ? 'iso' : nil,
+          resumptionToken: @resumption_token
+        }.delete_if { |_k, v| v.nil? }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/rda.rb

@@ -0,0 +1,33 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests the RDA feed
+    class Rda < Oai
+      def initialize(env = 'development', die_on_failure = false)
+        super
+        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:RDA][:long_name]
+        @translator = Helpers::IsoToSolr.new :rda
+      end
+
+      def metadata_url
+        SolrEnvironments[@environment][:rda_url]
+      end
+
+      # resumption_token must be empty to stop the harvest loop; RDA's feed does not
+      # provide any resumption token and gets all the records in just one go
+      def results
+        @resumption_token = ''
+        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
+        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
+      end
+
+      private
+
+      def request_params
+        {
+          verb: 'ListRecords',
+          metadataPrefix: 'dif'
+        }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/tdar.rb

@@ -0,0 +1,57 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from TDAR and inserts it into Solr after it has been translated
+    class Tdar < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @page_size = 100
+        @translator = Helpers::IsoToSolr.new :tdar
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of TDAR catalog from #{tdar_url}"
+        super(method(:harvest_tdar_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:TDAR][:long_name]}\"")
+      end
+
+      def harvest_tdar_into_solr
+        start_record = 1
+        while (entries = get_results_from_tdar(start_record)) && (entries.length > 0)
+          begin
+            insert_solr_docs get_docs_with_translated_entries_from_tdar(entries)
+          rescue => e
+            puts "ERROR: #{e}\n\n"
+            raise e if @die_on_failure
+          end
+          start_record += @page_size
+        end
+      end
+
+      def tdar_url
+        SolrEnvironments[@environment][:tdar_url]
+      end
+
+      def get_results_from_tdar(start_record)
+        get_results build_request(@page_size, start_record), './/atom:entry', 'application/xml'
+      end
+
+      def get_docs_with_translated_entries_from_tdar(entries)
+        docs = []
+        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
+        docs
+      end
+
+      def build_request(max_records = '25', start_record = '1')
+        request_url = tdar_url + '?_tDAR.searchType=ACADIS_RSS&'\
+                                 'resourceTypes=DATASET&'\
+                                 'groups[0].latitudeLongitudeBoxes[0].maximumLongitude=180&'\
+                                 'groups[0].latitudeLongitudeBoxes[0].minimumLatitude=45&'\
+                                 'groups[0].latitudeLongitudeBoxes[0].minimumLongitude=-180&'\
+                                 'groups[0].latitudeLongitudeBoxes[0].maximumLatitude=90&'\
+                                 'geoMode=ENVELOPE&'\
+                                 'recordsPerPage=' + max_records.to_s + '&startRecord=' + start_record.to_s
+
+        request_url
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/usgs.rb

@@ -0,0 +1,74 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from USGS and inserts it into Solr after it has been translated
+    class Usgs < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @page_size = 100
+        @translator = Helpers::IsoToSolr.new :usgs
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of USGS catalog from #{usgs_url}"
+        super(method(:harvest_usgs_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:USGS][:long_name]}\"")
+      end
+
+      # get translated entries from USGS and add them to Solr
+      # this is the main entry point for the class
+      def harvest_usgs_into_solr
+        start_index = 1
+        while (entries = get_results_from_usgs(start_index)) && (entries.length > 0)
+          begin
+            insert_solr_docs get_docs_with_translated_entries_from_usgs(entries)
+          rescue => e
+            puts "ERROR: #{e}"
+            raise e if @die_on_failure
+          end
+          start_index += @page_size
+        end
+      end
+
+      def usgs_url
+        SolrEnvironments[@environment][:usgs_url]
+      end
+
+      def get_results_from_usgs(start_index)
+        get_results build_csw_request('results', @page_size, start_index), '//gmd:MD_Metadata', ''
+      end
+
+      def get_docs_with_translated_entries_from_usgs(entries)
+        docs = []
+        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
+        docs
+      end
+
+      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
+        Helpers::CswIsoQueryBuilder.get_query_string(usgs_url,
+                                                     'resultType' => resultType,
+                                                     'maxRecords' => maxRecords,
+                                                     'startPosition' => startPosition,
+                                                     'TypeNames' => '',
+                                                     'constraint' => bbox_constraint
+
+                                                    )
+      end
+
+      def bbox_constraint
+        bbox = {
+          west: '-180',
+          south: '45',
+          east: '180',
+          north: '90'
+        }
+
+        URI.encode '<Filter xmlns:ogc="http://www.opengis.net/ogc" ' \
+                   'xmlns:gml="http://www.opengis.net/gml" ' \
+                   'xmlns:apiso="http://www.opengis.net/cat/csw/apiso/1.0">' \
+                   '<ogc:BBOX><PropertyName>apiso:BoundingBox</PropertyName><gml:Envelope>' \
+                   '<gml:lowerCorner>' + bbox[:west] + ' ' + bbox[:south] + '</gml:lowerCorner>' \
+                                                                            '<gml:upperCorner>' + bbox[:east] + ' ' + bbox[:north] + '</gml:upperCorner>' \
+                                                                                                                                     '</gml:Envelope></ogc:BBOX></Filter>'
+      end
+    end
+  end
+end