search_solr_tools 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cded5837c68e69c4dd22f9aea6b3efab1d4cc21570daba1b91b9561d7cde1216
-  data.tar.gz: 73a840e47c0cbc41a3fb13e62ca85735e20d87dc9f4b0acfd3d522ef1874ad34
+  metadata.gz: afbece000e765162dcb622479c3b9da58b534e4609b83ccb194a719ae5bb9c03
+  data.tar.gz: 012cb04bdced985254701d6f50fc99340c909fcff8000263c184dd9519282bbf
 SHA512:
-  metadata.gz: 5056c01da21c54b3b4c86ddef264d2e0f9dbe83db472677ac1076a31a5e42e97a6ad0cd0a037b0431374a2d243bfc398dfcfa631863d3d4f3b3b334751b6e2c6
-  data.tar.gz: 39418c870ebada693b8ef2b75460d04e8ed0d1f9764e534d2195fa59fb5daaec4ae002ae4f016d05449a1a8a593b7e931261965b5756c664c6edfc7c94e6e3b1
+  metadata.gz: 9b0c53d6ff652e840a8014f1ee0e31db62ad5faba634cd7aebf66ac90cd9a6865953ed081169f4b52ca96bf3ac135954a26d8aa6e5e7aa2bc1cf3298f946f260
+  data.tar.gz: 3975437ccf9540013f5c74e71a8e6f676273aab0e4b8a9a78df853b0010120b660f06a7a6ae630d6b40c6c6b8e56fc01b7562a6e2127508218c716e5ff4b41fc

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## v6.2.0 (2022-07-18)
+
+  - Remove deprecated harvesters and associated tests, helpers, etc.
+
 ## v6.1.0 (2022-07-14)
 
   - Updated a few other dependencies that weren't at the newest versions.

data/bin/search_solr_tools

@@ -120,20 +120,8 @@ class SolrHarvestCLI < Thor
   no_tasks do
     def harvester_map
       {
-        'bco_dmo' => SearchSolrTools::Harvesters::BcoDmo,
-        'adc' => SearchSolrTools::Harvesters::Adc,
-        'echo' => SearchSolrTools::Harvesters::Echo,
-        'ices' => SearchSolrTools::Harvesters::Ices,
-        'nmi' => SearchSolrTools::Harvesters::Nmi,
-        'nodc' => SearchSolrTools::Harvesters::Nodc,
-        'r2r' => SearchSolrTools::Harvesters::R2R,
-        'rda' => SearchSolrTools::Harvesters::Rda,
-        'usgs' => SearchSolrTools::Harvesters::Usgs,
-        'tdar' => SearchSolrTools::Harvesters::Tdar,
-        'pdc' => SearchSolrTools::Harvesters::Pdc,
         'nsidc' => SearchSolrTools::Harvesters::NsidcJson,
-        'nsidc_auto_suggest' => SearchSolrTools::Harvesters::NsidcAutoSuggest,
-        'ade_auto_suggest' => SearchSolrTools::Harvesters::AdeAutoSuggest
+        'nsidc_auto_suggest' => SearchSolrTools::Harvesters::NsidcAutoSuggest
       }
     end
 

data/lib/search_solr_tools/config/environments.yaml

@@ -3,38 +3,6 @@
   :collection_name: nsidc_oai
   :collection_path: solr
   :port: 8983
-  :bcodmo_url: http://www.bco-dmo.org/nsidc/arctic-deployments.json
-  :adc_url: https://arcticdata.io/metacat/d1/mn/v2/query/solr/select?q=northBoundCoord:%5B45.0%20TO%2090.0%5D
-  :data_one_url: https://cn.dataone.org/cn/v1/query/solr/select?q=northBoundCoord:%5B45.0%20TO%2090.0%5D
-  :echo_url: https://api.echo.nasa.gov/catalog-rest/echo_catalog/datasets.echo10?bounding_box=-180,45,180,90
-  :gtnp:
-    - http://www.gtnpdatabase.org/rest/boreholes/json
-    - http://www.gtnpdatabase.org/rest/activelayers/json
-  :ices_url: http://geo.ices.dk/geonetwork/srv/en/csw
-  :ncdc_paleo_url: https://gis.ncdc.noaa.gov/gptpaleo/csw
-  :nmi_url: http://arcticdata.met.no/metamod/oai
-  :nodc_url: https://data.nodc.noaa.gov/geoportal/csw
-  :pdc_url: http://www.polardata.ca/oai/provider
-  :rda_url: https://rda.ucar.edu/cgi-bin/oai
-  :tdar_url: http://core.tdar.org/search/rss
-  :usgs_url: https://www.sciencebase.gov/catalog/item/527cf4ede4b0850ea05182ee/csw
-  :eol:
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.SHEBA.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.SBI.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.PacMARS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BASE.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ATLAS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ARC_MIP.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.AMTS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BOREAS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BeringSea.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ARCSS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BEST.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BSIERP.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BARROW.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.DBO.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ITEX.thredds.xml
-  :r2r_url: http://get.rvdata.us/services/cruise/
 
   # Not using DCS API v2 here because not all retired datasets have their "retired"
   # flag checked. For example, GLA01.033 is retired; GLA01.018 is not, but it

data/lib/search_solr_tools/harvesters/base.rb

@@ -8,7 +8,6 @@ require 'time'
 require 'search_solr_tools'
 require_relative '../helpers/iso_namespaces'
 require_relative '../helpers/solr_format'
-require_relative '../helpers/iso_to_solr'
 
 
 module SearchSolrTools

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -10,22 +10,7 @@ module SearchSolrTools
     # rubocop:disable Metrics/ModuleLength
     module SolrFormat
       DATA_CENTER_NAMES = {
-        BCODMO: { short_name: 'BCO-DMO', long_name: 'Biological and Chemical Oceanography Data Management Office' },
-        ADC: { short_name: 'NSF ADC', long_name: 'NSF Arctic Data Center' },
-        DATA_ONE: { short_name: 'DataONE', long_name: 'DataONE' },
-        ECHO: { short_name: 'NASA ECHO', long_name: 'NASA Earth Observing System (EOS) Clearing House (ECHO)' },
-        EOL: { short_name: 'UCAR NCAR EOL', long_name: 'UCAR NCAR - Earth Observing Laboratory' },
-        GTNP: { short_name: 'GTN-P', long_name: 'Global Terrestrial Network for Permafrost' },
-        ICES: { short_name: 'ICES', long_name: 'International Council for the Exploration of the Sea' },
-        NCDC_PALEO: { short_name: 'NOAA WDS Paleo', long_name: 'NOAA’s National Centers for Environmental Information, World Data Service for Paleoclimatology' },
-        NMI: { short_name: 'Met.no', long_name: 'Norwegian Meteorological Institute' },
-        NODC: { short_name: 'NOAA NODC', long_name: 'NOAA National Oceanographic Data Center' },
         NSIDC: { short_name: 'NSIDC', long_name: 'National Snow and Ice Data Center' },
-        PDC: { short_name: 'PDC', long_name: 'Polar Data Catalogue' },
-        R2R: { short_name: 'R2R', long_name: 'Rolling Deck to Repository' },
-        RDA: { short_name: 'UCAR NCAR RDA', long_name: 'UCAR NCAR Research Data Archive' },
-        TDAR: { short_name: 'tDAR', long_name: 'tDAR: The Digital Archaeological Record' },
-        USGS: { short_name: 'USGS ScienceBase', long_name: 'U.S. Geological Survey ScienceBase' }
       }
 
       NOT_SPECIFIED = 'Not specified'

data/lib/search_solr_tools/helpers/translate_spatial_coverage.rb

@@ -1,7 +1,6 @@
 require 'rgeo/geo_json'
 
 require_relative 'bounding_box_util'
-require_relative 'iso_to_solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '6.1.0'
+  VERSION = '6.2.0'
 end

data/lib/search_solr_tools.rb

@@ -1,10 +1,9 @@
 require_relative 'search_solr_tools/config/environments'
 require_relative 'search_solr_tools/version'
 
-require_relative 'search_solr_tools/helpers/selectors'
 require_relative 'search_solr_tools/helpers/harvest_status'
 require_relative 'search_solr_tools/errors/harvest_error'
 
-%w( selectors harvesters translators ).each do |subdir|
+%w( harvesters translators ).each do |subdir|
   Dir[File.join(__dir__, 'search_solr_tools', subdir, '*.rb')].each { |file| require file }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-17 00:00:00.000000000 Z
+date: 2023-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi-geos
@@ -321,59 +321,17 @@ files:
 - lib/search_solr_tools/config/environments.rb
 - lib/search_solr_tools/config/environments.yaml
 - lib/search_solr_tools/errors/harvest_error.rb
-- lib/search_solr_tools/harvesters/adc.rb
-- lib/search_solr_tools/harvesters/ade_auto_suggest.rb
 - lib/search_solr_tools/harvesters/auto_suggest.rb
 - lib/search_solr_tools/harvesters/base.rb
-- lib/search_solr_tools/harvesters/bcodmo.rb
-- lib/search_solr_tools/harvesters/data_one.rb
-- lib/search_solr_tools/harvesters/echo.rb
-- lib/search_solr_tools/harvesters/eol.rb
-- lib/search_solr_tools/harvesters/gtnp.rb
-- lib/search_solr_tools/harvesters/ices.rb
-- lib/search_solr_tools/harvesters/ncdc_paleo.rb
-- lib/search_solr_tools/harvesters/nmi.rb
-- lib/search_solr_tools/harvesters/nodc.rb
 - lib/search_solr_tools/harvesters/nsidc_auto_suggest.rb
 - lib/search_solr_tools/harvesters/nsidc_json.rb
-- lib/search_solr_tools/harvesters/oai.rb
-- lib/search_solr_tools/harvesters/pdc.rb
-- lib/search_solr_tools/harvesters/r2r.rb
-- lib/search_solr_tools/harvesters/rda.rb
-- lib/search_solr_tools/harvesters/tdar.rb
-- lib/search_solr_tools/harvesters/usgs.rb
 - lib/search_solr_tools/helpers/bounding_box_util.rb
-- lib/search_solr_tools/helpers/csw_iso_query_builder.rb
-- lib/search_solr_tools/helpers/data_one_format.rb
 - lib/search_solr_tools/helpers/facet_configuration.rb
 - lib/search_solr_tools/helpers/harvest_status.rb
 - lib/search_solr_tools/helpers/iso_namespaces.rb
-- lib/search_solr_tools/helpers/iso_to_solr.rb
-- lib/search_solr_tools/helpers/iso_to_solr_format.rb
-- lib/search_solr_tools/helpers/ncdc_paleo_format.rb
-- lib/search_solr_tools/helpers/query_builder.rb
-- lib/search_solr_tools/helpers/r2r_format.rb
-- lib/search_solr_tools/helpers/selectors.rb
 - lib/search_solr_tools/helpers/solr_format.rb
-- lib/search_solr_tools/helpers/tdar_format.rb
 - lib/search_solr_tools/helpers/translate_spatial_coverage.rb
 - lib/search_solr_tools/helpers/translate_temporal_coverage.rb
-- lib/search_solr_tools/helpers/usgs_format.rb
-- lib/search_solr_tools/selectors/adc.rb
-- lib/search_solr_tools/selectors/data_one.rb
-- lib/search_solr_tools/selectors/echo_iso.rb
-- lib/search_solr_tools/selectors/ices_iso.rb
-- lib/search_solr_tools/selectors/ncdc_paleo.rb
-- lib/search_solr_tools/selectors/nmi.rb
-- lib/search_solr_tools/selectors/nodc_iso.rb
-- lib/search_solr_tools/selectors/pdc_iso.rb
-- lib/search_solr_tools/selectors/r2r.rb
-- lib/search_solr_tools/selectors/rda.rb
-- lib/search_solr_tools/selectors/tdar_opensearch.rb
-- lib/search_solr_tools/selectors/usgs_iso.rb
-- lib/search_solr_tools/translators/bcodmo_json.rb
-- lib/search_solr_tools/translators/eol_to_solr.rb
-- lib/search_solr_tools/translators/gtnp_json.rb
 - lib/search_solr_tools/translators/nsidc_json.rb
 - lib/search_solr_tools/version.rb
 - search_solr_tools.gemspec

data/lib/search_solr_tools/harvesters/adc.rb

@@ -1,49 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    class Adc < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @page_size = 250
-        @translator = Helpers::IsoToSolr.new :adc
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of adc catalog from #{metadata_url}"
-        super(method(:harvest_adc_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]}\"")
-      end
-
-      def harvest_adc_into_solr
-        start = 0
-        while (entries = get_results_from_adc(start)) && (entries.length > 0)
-          begin
-            insert_solr_docs(get_docs_with_translated_entries_from_adc(entries))
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-          start += @page_size
-        end
-      end
-
-      def get_results_from_adc(start)
-        get_results(build_request(start, @page_size), './response/result/doc')
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:adc_url]
-      end
-
-      def get_docs_with_translated_entries_from_adc(entries)
-        entries.map do |e|
-          create_new_solr_add_doc_with_child(@translator.translate(e).root)
-        end
-      end
-
-      def build_request(start = 0, max_records = 100)
-        "#{metadata_url}&start=#{start}&rows=#{max_records}"
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/ade_auto_suggest.rb

@@ -1,46 +0,0 @@
-require_relative 'auto_suggest'
-
-module SearchSolrTools
-  module Harvesters
-    class AdeAutoSuggest < AutoSuggest
-      def harvest_and_delete
-        puts 'Building auto-suggest indexes for ADE'
-        super(method(:harvest), 'source:"ADE"', @env_settings[:auto_suggest_collection_name])
-      end
-
-      def harvest
-        url = "#{solr_url}/#{@env_settings[:collection_name]}/select?q=*%3A*&fq=source%3AADE&fq=spatial:[45.0,-180.0+TO+90.0,180.0]&rows=0&wt=json&indent=true&facet=true&facet.mincount=1&facet.sort=count&facet.limit=-1"
-        super url, fields
-      end
-
-      def fields
-        {
-          'full_keywords_and_parameters' => { weight: 2, source: 'ADE', creator: method(:keyword_creator) },
-          'full_authors' => { weight: 1, source: 'ADE', creator: method(:author_creator) }
-        }
-      end
-
-      def split_creator(value, count, field_weight, source, split_regex)
-        add_docs = []
-        value.downcase.split(split_regex).each do |v|
-          v = v.strip.chomp('/')
-          add_docs.concat(ade_length_limit_creator(v, count, field_weight, source)) unless v.nil? || v.empty?
-        end
-        add_docs
-      end
-
-      def keyword_creator(value, count, field_weight, source)
-        split_creator value, count, field_weight, source, %r{/ [\/ \>]+ /}
-      end
-
-      def author_creator(value, count, field_weight, source)
-        split_creator value, count, field_weight, source, %r{/;/}
-      end
-
-      def ade_length_limit_creator(value, count, field_weight, source)
-        return [] if value.length > 80
-        standard_add_creator value, count, field_weight, source
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/bcodmo.rb

@@ -1,64 +0,0 @@
-require_relative 'base'
-require 'json'
-require 'rest-client'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from BcoDmo endpoint, translates and adds it to solr
-    class BcoDmo < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @translator = Translators::BcodmoJsonToSolr.new
-        @wkt_parser = RGeo::WKRep::WKTParser.new(nil, {}) # (factory_generator_=nil,
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of BCO-DMO catalog from #{bcodmo_url}"
-        super(method(:harvest_bcodmo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:long_name]}\"")
-      end
-
-      def bcodmo_url
-        SolrEnvironments[@environment][:bcodmo_url]
-      end
-
-      def harvest_bcodmo_into_solr
-        result = translate_bcodmo
-        insert_solr_docs(result[:add_docs], Base::JSON_CONTENT_TYPE)
-
-        errors_exist = result[:failure_ids].length > 0
-        fail 'Failed to harvest some records from BCO-DMO' if errors_exist && @die_on_failure
-      end
-
-      def translate_bcodmo
-        documents = []
-        failure_ids = []
-        request_json(SolrEnvironments[@environment][:bcodmo_url]).each do |record|
-          geometry = request_json(record['geometryUrl'])
-          results = parse_record(record, geometry)
-          results[:documents].each { |d| documents << d }
-          results[:failure_ids].each { |id| failure_ids << id }
-        end
-        { add_docs: documents, failure_ids: failure_ids }
-      end
-
-      def request_json(url)
-        puts "Request: #{url}"
-        JSON.parse(RestClient.get(url))
-      end
-
-      def parse_record(record, geometry)
-        documents = []
-        failure_ids = []
-        begin
-          JSON.parse(RestClient.get(record['datasets'])).each do |dataset|
-            documents << { 'add' => { 'doc' => @translator.translate(dataset, record, geometry) } }
-          end
-        rescue => e
-          puts "Failed to add record #{record['id']} with error #{e} (#{e.message}) : #{e.backtrace.join("\n")}"
-          failure_ids << record['id']
-        end
-        { documents: documents, failure_ids: failure_ids }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/data_one.rb

@@ -1,49 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    class DataOne < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @page_size = 250
-        @translator = Helpers::IsoToSolr.new :data_one
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of dataONE catalog from #{metadata_url}"
-        super(method(:harvest_data_one_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:DATA_ONE][:long_name]}\"")
-      end
-
-      def harvest_data_one_into_solr
-        start = 0
-        while (entries = get_results_from_data_one(start)) && (entries.length > 0)
-          begin
-            insert_solr_docs(get_docs_with_translated_entries_from_data_one(entries))
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-          start += @page_size
-        end
-      end
-
-      def get_results_from_data_one(start)
-        get_results(build_request(start, @page_size), './response/result/doc')
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:data_one_url]
-      end
-
-      def get_docs_with_translated_entries_from_data_one(entries)
-        entries.map do |e|
-          create_new_solr_add_doc_with_child(@translator.translate(e).root)
-        end
-      end
-
-      def build_request(start = 0, max_records = 100)
-        "#{metadata_url}&start=#{start}&rows=#{max_records}"
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/echo.rb

@@ -1,52 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from ECHO and inserts it into Solr after it has been translated
-    class Echo < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :echo
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of ECHO catalog from #{echo_url}"
-        super(method(:harvest_echo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ECHO][:long_name]}\"")
-      end
-
-      # get translated entries from ECHO and add them to Solr
-      # this is the main entry point for the class
-      def harvest_echo_into_solr
-        page_num = 1
-        while (entries = get_results_from_echo(page_num)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_echo(entries)
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-          page_num += 1
-        end
-      end
-
-      def echo_url
-        SolrEnvironments[@environment][:echo_url]
-      end
-
-      def get_results_from_echo(page_num)
-        get_results build_request(@page_size, page_num), './/results/result', 'application/echo10+xml'
-      end
-
-      def get_docs_with_translated_entries_from_echo(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_request(max_records = '25', page_num = '1')
-        echo_url + '&page_size=' + max_records.to_s + '&page_num=' + page_num.to_s
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/eol.rb

@@ -1,51 +0,0 @@
-require_relative 'base'
-require 'json'
-require 'rgeo/geo_json'
-
-module SearchSolrTools
-  module Harvesters
-    class Eol < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @translator = SearchSolrTools::Translators::EolToSolr.new
-      end
-
-      def harvest_and_delete
-        puts 'Running harvest of EOL catalog using the following configured EOL URLs:'
-        SearchSolrTools::SolrEnvironments[:common][:eol].each { |x| puts x }
-        super(method(:harvest_eol_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name]}\"")
-      end
-
-      def harvest_eol_into_solr
-        solr_add_queries = eol_dataset_urls.map do |dataset|
-          begin
-            doc = open_xml_document(dataset)
-            if doc.xpath('//xmlns:metadata').size > 1
-              # THREDDS allows for a dataset of datasests, EOL should not utilize this
-              fail "Complex dataset encountered at #{doc.xpath('//xmlns:catalog').to_html}"
-            end
-            metadata_doc = open_xml_document(doc.xpath('//xmlns:metadata')[0]['xlink:href'])
-            { 'add' => { 'doc' => @translator.translate(doc, metadata_doc) } }
-          rescue => e
-            puts "ERROR: #{e}"
-            puts "Failed to translate this record: #{doc} -> #{metadata_doc}"
-            raise e if @die_on_failure
-            next
-          end
-        end
-        insert_solr_docs solr_add_queries, Base::JSON_CONTENT_TYPE
-      end
-
-      def eol_dataset_urls
-        SearchSolrTools::SolrEnvironments[:common][:eol].flat_map do |endpoint|
-          doc = open_xml_document(endpoint)
-          doc.xpath('//xmlns:catalogRef').map { |node| node['xlink:href'] }
-        end
-      end
-
-      def open_xml_document(url)
-        Nokogiri::XML(open(url), &:strict)
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/gtnp.rb

@@ -1,67 +0,0 @@
-require_relative 'base'
-require 'json'
-require 'rest-client'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from GTN-P endpoints, translates and adds it to solr
-    class GtnP < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @translator = Translators::GtnpJsonToSolr.new
-      end
-
-      def gtnp_service_urls
-        json_records = []
-        SearchSolrTools::SolrEnvironments[:common][:gtnp].flat_map do |endpoint|
-          record = request_json(endpoint)
-          json_records << record
-        end
-        json_records
-      end
-
-      def harvest_and_delete
-        puts 'Running harvest of GTN-P catalog using the following configured GTN-P URLs:'
-        SearchSolrTools::SolrEnvironments[:common][:gtnp].each { |x| puts x }
-        super(method(:harvest_gtnp_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name]}\"")
-      end
-
-      def harvest_gtnp_into_solr
-        result = translate_gtnp
-        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        fail 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
-      end
-
-      def translate_gtnp
-        documents = []
-        failure_ids = []
-        gtnp_records = gtnp_service_urls
-        gtnp_records.each do |record|
-          results = parse_record(record)
-          results[:documents].each { |d| documents << d }
-          results[:failure_ids].each { |id| failure_ids << id }
-        end
-        { add_docs: documents, failure_ids: failure_ids }
-      end
-
-      def request_json(url)
-        JSON.parse(RestClient.get(url))
-      end
-
-      def parse_record(record)
-        documents = []
-        failure_ids = []
-        begin
-          record.drop(1).each do |dataset|
-            trans_doc = @translator.translate(dataset, record[0])
-            documents << { 'add' => { 'doc' => trans_doc } }
-          end
-        rescue => e
-          puts "Failed to add record #{record[0][:title]} with error #{e} (#{e.message}) : #{e.backtrace.join("\n")}"
-          failure_ids << record[0][:title]
-        end
-        { documents: documents, failure_ids: failure_ids }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/ices.rb

@@ -1,58 +0,0 @@
-require_relative 'base'
-require_relative '../helpers/csw_iso_query_builder'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from ICES and inserts it into Solr after it has been translated
-    class Ices < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :ices
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of ICES catalog from #{ices_url}"
-        super(method(:harvest_ices_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ICES][:long_name]}\"")
-      end
-
-      # get translated entries from ICES and add them to Solr
-      # this is the main entry point for the class
-      def harvest_ices_into_solr
-        start_index = 1
-        while (entries = get_results_from_ices(start_index)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_ices(entries)
-          rescue => e
-            puts "ERROR: #{e}"
-            raise e if @die_on_failure
-          end
-          start_index += @page_size
-        end
-      end
-
-      def ices_url
-        SolrEnvironments[@environment][:ices_url]
-      end
-
-      def get_results_from_ices(start_index)
-        get_results build_csw_request('results', @page_size, start_index), '//gmd:MD_Metadata'
-      end
-
-      def get_docs_with_translated_entries_from_ices(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')
-        Helpers::CswIsoQueryBuilder.get_query_string(ices_url,
-                                                     'resultType' => resultType,
-                                                     'maxRecords' => maxRecords,
-                                                     'startPosition' => startPosition,
-                                                     'constraintLanguage' => 'CQL_TEXT',
-                                                     'outputSchema' => 'http://www.isotc211.org/2005/gmd')
-      end
-    end
-  end
-end