search_solr_tools 3.1.2

data/lib/search_solr_tools/harvesters/base.rb

@@ -0,0 +1,183 @@
+require 'multi_json'
+require 'nokogiri'
+require 'open-uri'
+require 'rest-client'
+require 'rsolr'
+require 'time'
+
+module SearchSolrTools
+  module Harvesters
+    # base class for solr harvesters
+    class Base
+      attr_accessor :environment
+
+      DELETE_DOCUMENTS_RATIO = 0.1
+      XML_CONTENT_TYPE = 'text/xml; charset=utf-8'
+      JSON_CONTENT_TYPE = 'application/json; charset=utf-8'
+
+      def initialize(env = 'development', die_on_failure = false)
+        @environment = env
+        @die_on_failure = die_on_failure
+      end
+
+      def solr_url
+        env = SolrEnvironments[@environment]
+        "http://#{env[:host]}:#{env[:port]}/#{env[:collection_path]}"
+      end
+
+      # Some data providers require encoding (such as URI.encode),
+      # while others barf on encoding.  The default is to just
+      # return url, override this in the subclass if special
+      # encoding is needed.
+      def encode_data_provider_url(url)
+        url
+      end
+
+      def harvest_and_delete(harvest_method, delete_constraints, solr_core = SolrEnvironments[@environment][:collection_name])
+        start_time = Time.now.utc.iso8601
+        harvest_method.call
+        delete_old_documents start_time, delete_constraints, solr_core
+      end
+
+      def delete_old_documents(timestamp, constraints, solr_core, force = false)
+        constraints = sanitize_data_centers_constraints(constraints)
+        delete_query = "last_update:[* TO #{timestamp}] AND #{constraints}"
+        solr = RSolr.connect url: solr_url + "/#{solr_core}"
+        unchanged_count = (solr.get 'select', params: { q: delete_query, rows: 0 })['response']['numFound'].to_i
+        if unchanged_count == 0
+          puts "All documents were updated after #{timestamp}, nothing to delete"
+        else
+          puts "Begin removing documents older than #{timestamp}"
+          remove_documents(solr, delete_query, constraints, force, unchanged_count)
+        end
+      end
+
+      def sanitize_data_centers_constraints(query_string)
+        # Remove lucene special characters, preserve the query parameter and compress whitespace
+        query_string.gsub!(/[:&|!~\-\(\)\{\}\[\]\^\*\?\+]+/, ' ')
+        query_string.gsub!(/data_centers /, 'data_centers:')
+        query_string.squeeze(' ').strip
+      end
+
+      def remove_documents(solr, delete_query, constraints, force, numfound)
+        all_response_count = (solr.get 'select', params: { q: constraints, rows: 0 })['response']['numFound']
+        if force || (numfound / all_response_count.to_f < DELETE_DOCUMENTS_RATIO)
+          puts "Deleting #{numfound} documents for #{constraints}"
+          solr.delete_by_query delete_query
+          solr.commit
+        else
+          puts "Failed to delete records older than current harvest start because they exceeded #{DELETE_DOCUMENTS_RATIO} of the total records for this data center."
+          puts "\tTotal records: #{all_response_count}"
+          puts "\tNon-updated records: #{numfound}"
+        end
+      end
+
+      # Update Solr with an array of Nokogiri xml documents, report number of successfully added documents
+      def insert_solr_docs(docs, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])
+        success = 0
+        failure = 0
+        docs.each do |doc|
+          insert_solr_doc(doc, content_type, core) ? success += 1 : failure += 1
+        end
+        puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."
+        puts "#{failure} document#{failure == 1 ? '' : 's'} not added to Solr."
+        fail 'Some documents failed to be inserted into Solr' if failure > 0
+      end
+
+      def insert_solr_doc(doc, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])
+        url = solr_url + "/#{core}/update?commit=true"
+        success = false
+
+        # Some of the docs will cause Solr to crash - CPU goes to 195% with `top` and it
+        # doesn't seem to recover.
+        return success unless doc_valid?(doc) if content_type == XML_CONTENT_TYPE
+
+        doc_serialized = get_serialized_doc(doc, content_type)
+
+        # Some docs will cause solr to time out during the POST
+        begin
+          RestClient.post(url, doc_serialized,  content_type: content_type) do |response, _request, _result|
+            success = response.code == 200
+            puts "Error for #{doc_serialized}\n\n response: #{response.body}" unless success
+          end
+        rescue => e
+          puts "Rest exception while POSTing to Solr: #{e}, for doc: #{doc_serialized}"
+        end
+
+        success
+      end
+
+      def get_serialized_doc(doc, content_type)
+        if content_type.eql?(XML_CONTENT_TYPE)
+          return doc.respond_to?(:to_xml) ? doc.to_xml : doc
+        elsif content_type.eql?(JSON_CONTENT_TYPE)
+          return MultiJson.dump(doc)
+        else
+          return doc
+        end
+      end
+
+      # Get results from some ISO end point specified in the query string
+      def get_results(request_url, metadata_path, content_type = 'application/xml')
+        timeout = 300
+        retries_left = 3
+
+        request_url = encode_data_provider_url(request_url)
+
+        begin
+          puts "Request: #{request_url}"
+          response = open(request_url, read_timeout: timeout, 'Content-Type' => content_type)
+        rescue OpenURI::HTTPError, Timeout::Error => e
+          retries_left -= 1
+          puts "## REQUEST FAILED ## Retrying #{retries_left} more times..."
+
+          retry if retries_left > 0
+
+          raise e if @die_on_failure
+          return
+        end
+        doc = Nokogiri.XML(response)
+        doc.xpath(metadata_path, Helpers::IsoNamespaces.namespaces(doc))
+      end
+
+      # returns Nokogiri XML document with content
+      # '<?xml version="1.0"?><add/>'
+      def create_new_solr_add_doc
+        doc = Nokogiri::XML::Document.new
+        doc.root = Nokogiri::XML::Node.new('add', doc)
+        doc
+      end
+
+      # returns a Nokogiri XML document with content
+      # '<?xml version="1.0"?><add> <child /> </add>'
+      def create_new_solr_add_doc_with_child(child)
+        doc = create_new_solr_add_doc
+        doc.root.add_child(child)
+        doc
+      end
+
+      # Make sure that Solr is able to accept this doc in a POST
+      def doc_valid?(doc)
+        spatial_coverages = doc.xpath(".//field[@name='spatial_coverages']").first
+        return true if spatial_coverages.nil?
+
+        spatial_coverages = spatial_coverages.text.split(' ')
+
+        # We've only seen the failure with 4 spatial coverage values
+        return true if spatial_coverages.size < 4
+
+        valid_solr_spatial_coverage?(spatial_coverages)
+      end
+
+      # spatial_coverages is an array with length 4:
+      # [North, East, South, West]
+      def valid_solr_spatial_coverage?(spatial_coverages)
+        north, east, south, west = spatial_coverages
+
+        polar_point = (north == south) && (north.to_f.abs == 90)
+
+        (east == west) || !polar_point
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/bcodmo.rb

@@ -0,0 +1,55 @@
+require 'json'
+require 'rest-client'
+
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from BcoDmo endpoint, translates and adds it to solr
+    class BcoDmo < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @translator = Translators::BcodmoJsonToSolr.new
+        @wkt_parser = RGeo::WKRep::WKTParser.new(nil, {})   # (factory_generator_=nil,
+      end
+
+      def harvest_and_delete
+        super(method(:harvest_bcodmo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:long_name]}\"")
+      end
+
+      def harvest_bcodmo_into_solr
+        result = translate_bcodmo
+        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
+        fail 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
+      end
+
+      def translate_bcodmo
+        documents = []
+        failure_ids = []
+        request_json(SolrEnvironments[@environment][:bcodmo_url]).each do |record|
+          geometry = request_json(record['geometryUrl'])
+          results = parse_record(record, geometry)
+          results[:documents].each { |d| documents << d }
+          results[:failure_ids].each { |id| failure_ids << id }
+        end
+        { add_docs: documents, failure_ids: failure_ids }
+      end
+
+      def request_json(url)
+        JSON.parse(RestClient.get(url))
+      end
+
+      def parse_record(record, geometry)
+        documents = []
+        failure_ids = []
+        begin
+          JSON.parse(RestClient.get(record['datasets'])).each do |dataset|
+            documents << { 'add' => { 'doc' => @translator.translate(dataset, record, geometry) } }
+          end
+        rescue => e
+          puts "Failed to add record #{record['id']} with error #{e} (#{e.message}) : #{e.backtrace.join("\n")}"
+          failure_ids << record['id']
+        end
+        { documents: documents, failure_ids: failure_ids }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/cisl.rb

@@ -0,0 +1,63 @@
+
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from CISL and inserts it into Solr after it has been translated
+    class Cisl < Oai
+      def initialize(env = 'development', die_on_failure = false)
+        super
+        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:long_name]
+        @translator = Helpers::IsoToSolr.new :cisl
+
+        # Used in query string params, resumptionToken
+        @dataset = '0bdd2d39-3493-4fa2-98f9-6766596bdc50'
+      end
+
+      def metadata_url
+        SolrEnvironments[@environment][:cisl_url]
+      end
+
+      def results
+        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
+
+        @resumption_token = list_records_oai_response.xpath('.//oai:resumptionToken', Helpers::IsoNamespaces.namespaces)
+        @resumption_token = format_resumption_token(@resumption_token.first.text)
+
+        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
+      end
+
+      private
+
+      def request_params
+        {
+          verb: 'ListRecords',
+          metadataPrefix: 'dif',
+          set: @dataset,
+          resumptionToken: @resumption_token
+        }.delete_if { |_k, v| v.nil? }
+      end
+
+      # The ruby response is lacking quotes, which the token requires in order to work...
+      # Also, the response back seems to be inconsistent - sometimes it adds &quot; instead of '"',
+      # which makes the token fail to work.
+      # To get around this I'd prefer to make assumptions about the token and let it break if
+      # they change the formatting.  For now, all fields other than offset should be able to be
+      # assumed to remain constant.
+      # If the input is empty, then we are done - return an empty string, which is checked for
+      # in the harvest loop.
+      def format_resumption_token(resumption_token)
+        return '' if resumption_token.empty?
+
+        resumption_token =~ /offset:(\d+)/
+        offset = Regexp.last_match(1)
+
+        {
+          from: nil,
+          until: nil,
+          set: @dataset,
+          metadataPrefix: 'dif',
+          offset: offset
+        }.to_json
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/echo.rb

@@ -0,0 +1,50 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from ECHO and inserts it into Solr after it has been translated
+    class Echo < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @page_size = 1000
+        @translator = Helpers::IsoToSolr.new :echo
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of ECHO catalog from #{echo_url}"
+        super(method(:harvest_echo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ECHO][:long_name]}\"")
+      end
+
+      # get translated entries from ECHO and add them to Solr
+      # this is the main entry point for the class
+      def harvest_echo_into_solr
+        page_num = 1
+        while (entries = get_results_from_echo(page_num)) && (entries.length > 0)
+          begin
+            insert_solr_docs get_docs_with_translated_entries_from_echo(entries)
+          rescue => e
+            puts "ERROR: #{e}\n\n"
+            raise e if @die_on_failure
+          end
+          page_num += 1
+        end
+      end
+
+      def echo_url
+        SolrEnvironments[@environment][:echo_url]
+      end
+
+      def get_results_from_echo(page_num)
+        get_results build_request(@page_size, page_num), './/results/result', 'application/echo10+xml'
+      end
+
+      def get_docs_with_translated_entries_from_echo(entries)
+        docs = []
+        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
+        docs
+      end
+
+      def build_request(max_records = '25', page_num = '1')
+        echo_url + '?page_size=' + max_records.to_s + '&page_num=' + page_num.to_s
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/eol.rb

@@ -0,0 +1,53 @@
+require_relative 'base'
+require 'json'
+require 'rgeo/geo_json'
+
+module SearchSolrTools
+  module Harvesters
+    class Eol < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @translator = SearchSolrTools::Translators::EolToSolr.new
+      end
+
+      def harvest_and_delete
+        puts 'Running harvest of EOL catalog using the following configured EOL URLs:'
+        SearchSolrTools::SolrEnvironments[:common][:eol].each { |x| puts x }
+        super(method(:harvest_eol_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name]}\"")
+      end
+
+      def harvest_eol_into_solr
+        solr_add_queries = eol_dataset_urls.map do |dataset|
+          begin
+            doc = open_xml_document(dataset)
+            if doc.xpath('//xmlns:metadata').size > 1
+              # THREDDS allows for a dataset of datasests, EOL should not utilize this
+              fail "Complex dataset encountered at #{doc.xpath('//xmlns:catalog').to_html}"
+            end
+            metadata_doc = open_xml_document(doc.xpath('//xmlns:metadata')[0]['xlink:href'])
+            { 'add' => { 'doc' => @translator.translate(doc, metadata_doc) } }
+          rescue => e
+            puts "ERROR: #{e}"
+            puts "Failed to translate this record: #{doc} -> #{metadata_doc}"
+            raise e if @die_on_failure
+            next
+          end
+        end
+        insert_solr_docs solr_add_queries, Base::JSON_CONTENT_TYPE
+      end
+
+      def eol_dataset_urls
+        SearchSolrTools::SolrEnvironments[:common][:eol].flat_map do |endpoint|
+          doc = open_xml_document(endpoint)
+          doc.xpath('//xmlns:catalogRef').map { |node| node['xlink:href'] }
+        end
+      end
+
+      def open_xml_document(url)
+        Nokogiri::XML(open(url)) do |config|
+          config.strict
+        end
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/ices.rb

@@ -0,0 +1,55 @@
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from ICES and inserts it into Solr after it has been translated
+    class Ices < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @page_size = 100
+        @translator = Helpers::IsoToSolr.new :ices
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of ICES catalog from #{ices_url}"
+        super(method(:harvest_ices_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ICES][:long_name]}\"")
+      end
+
+      # get translated entries from ICES and add them to Solr
+      # this is the main entry point for the class
+      def harvest_ices_into_solr
+        start_index = 1
+        while (entries = get_results_from_ices(start_index)) && (entries.length > 0)
+          begin
+            insert_solr_docs get_docs_with_translated_entries_from_ices(entries)
+          rescue => e
+            puts "ERROR: #{e}"
+            raise e if @die_on_failure
+          end
+          start_index += @page_size
+        end
+      end
+
+      def ices_url
+        SolrEnvironments[@environment][:ices_url]
+      end
+
+      def get_results_from_ices(start_index)
+        get_results build_csw_request('results', @page_size, start_index), '//gmd:MD_Metadata'
+      end
+
+      def get_docs_with_translated_entries_from_ices(entries)
+        docs = []
+        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
+        docs
+      end
+
+      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
+        Helpers::CswIsoQueryBuilder.get_query_string(ices_url,
+                                                     'resultType' => resultType,
+                                                     'maxRecords' => maxRecords,
+                                                     'startPosition' => startPosition,
+                                                     'constraintLanguage' => 'CQL_TEXT'
+                                                    )
+      end
+    end
+  end
+end