search_solr_tools 6.0.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab316e3a683c1fc5d532a32c6325757777bde4afb817c2790c58df9d839216f5
-  data.tar.gz: e87963292111c382f1e4fae6d26cf629e328d8bd15cad8c373f87ed35cc68712
+  metadata.gz: afbece000e765162dcb622479c3b9da58b534e4609b83ccb194a719ae5bb9c03
+  data.tar.gz: 012cb04bdced985254701d6f50fc99340c909fcff8000263c184dd9519282bbf
 SHA512:
-  metadata.gz: b63f30bdecebf58e9f9d95400166740a04d3fe6dd738c652960a5de9e9dab0bfa34a4dfe897ce843cdee2f3c21b48a8258c544320446091960fb05ddf4259248
-  data.tar.gz: 9cd7b65d29ceb21fc4e5f0940f856a26ad08fb04e4fbdd421460f1f0518d19b9c15192657bae61549f393e30b0c924776f22e3cceee37ab04a29996491a59ab8
+  metadata.gz: 9b0c53d6ff652e840a8014f1ee0e31db62ad5faba634cd7aebf66ac90cd9a6865953ed081169f4b52ca96bf3ac135954a26d8aa6e5e7aa2bc1cf3298f946f260
+  data.tar.gz: 3975437ccf9540013f5c74e71a8e6f676273aab0e4b8a9a78df853b0010120b660f06a7a6ae630d6b40c6c6b8e56fc01b7562a6e2127508218c716e5ff4b41fc

data/CHANGELOG.md

@@ -1,4 +1,12 @@
-## v6.0.0 (2022-07-13)
+## v6.2.0 (2022-07-18)
+
+  - Remove deprecated harvesters and associated tests, helpers, etc.
+
+## v6.1.0 (2022-07-14)
+
+  - Updated a few other dependencies that weren't at the newest versions.
+
+## v6.0.0 (2022-07-14)
 
   - Updated Ruby to 3.2.2, updated gem dependencies to more recent versions.
 

data/bin/search_solr_tools

@@ -120,20 +120,8 @@ class SolrHarvestCLI < Thor
   no_tasks do
     def harvester_map
       {
-        'bco_dmo' => SearchSolrTools::Harvesters::BcoDmo,
-        'adc' => SearchSolrTools::Harvesters::Adc,
-        'echo' => SearchSolrTools::Harvesters::Echo,
-        'ices' => SearchSolrTools::Harvesters::Ices,
-        'nmi' => SearchSolrTools::Harvesters::Nmi,
-        'nodc' => SearchSolrTools::Harvesters::Nodc,
-        'r2r' => SearchSolrTools::Harvesters::R2R,
-        'rda' => SearchSolrTools::Harvesters::Rda,
-        'usgs' => SearchSolrTools::Harvesters::Usgs,
-        'tdar' => SearchSolrTools::Harvesters::Tdar,
-        'pdc' => SearchSolrTools::Harvesters::Pdc,
         'nsidc' => SearchSolrTools::Harvesters::NsidcJson,
-        'nsidc_auto_suggest' => SearchSolrTools::Harvesters::NsidcAutoSuggest,
-        'ade_auto_suggest' => SearchSolrTools::Harvesters::AdeAutoSuggest
+        'nsidc_auto_suggest' => SearchSolrTools::Harvesters::NsidcAutoSuggest
       }
     end
 

data/lib/search_solr_tools/config/environments.yaml

@@ -3,38 +3,6 @@
   :collection_name: nsidc_oai
   :collection_path: solr
   :port: 8983
-  :bcodmo_url: http://www.bco-dmo.org/nsidc/arctic-deployments.json
-  :adc_url: https://arcticdata.io/metacat/d1/mn/v2/query/solr/select?q=northBoundCoord:%5B45.0%20TO%2090.0%5D
-  :data_one_url: https://cn.dataone.org/cn/v1/query/solr/select?q=northBoundCoord:%5B45.0%20TO%2090.0%5D
-  :echo_url: https://api.echo.nasa.gov/catalog-rest/echo_catalog/datasets.echo10?bounding_box=-180,45,180,90
-  :gtnp:
-    - http://www.gtnpdatabase.org/rest/boreholes/json
-    - http://www.gtnpdatabase.org/rest/activelayers/json
-  :ices_url: http://geo.ices.dk/geonetwork/srv/en/csw
-  :ncdc_paleo_url: https://gis.ncdc.noaa.gov/gptpaleo/csw
-  :nmi_url: http://arcticdata.met.no/metamod/oai
-  :nodc_url: https://data.nodc.noaa.gov/geoportal/csw
-  :pdc_url: http://www.polardata.ca/oai/provider
-  :rda_url: https://rda.ucar.edu/cgi-bin/oai
-  :tdar_url: http://core.tdar.org/search/rss
-  :usgs_url: https://www.sciencebase.gov/catalog/item/527cf4ede4b0850ea05182ee/csw
-  :eol:
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.SHEBA.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.SBI.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.PacMARS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BASE.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ATLAS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ARC_MIP.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.AMTS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BOREAS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BeringSea.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ARCSS.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BEST.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BSIERP.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.BARROW.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.DBO.thredds.xml
-    - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ITEX.thredds.xml
-  :r2r_url: http://get.rvdata.us/services/cruise/
 
   # Not using DCS API v2 here because not all retired datasets have their "retired"
   # flag checked. For example, GLA01.033 is retired; GLA01.018 is not, but it

data/lib/search_solr_tools/harvesters/base.rb

@@ -8,7 +8,6 @@ require 'time'
 require 'search_solr_tools'
 require_relative '../helpers/iso_namespaces'
 require_relative '../helpers/solr_format'
-require_relative '../helpers/iso_to_solr'
 
 
 module SearchSolrTools

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -10,22 +10,7 @@ module SearchSolrTools
     # rubocop:disable Metrics/ModuleLength
     module SolrFormat
       DATA_CENTER_NAMES = {
-        BCODMO: { short_name: 'BCO-DMO', long_name: 'Biological and Chemical Oceanography Data Management Office' },
-        ADC: { short_name: 'NSF ADC', long_name: 'NSF Arctic Data Center' },
-        DATA_ONE: { short_name: 'DataONE', long_name: 'DataONE' },
-        ECHO: { short_name: 'NASA ECHO', long_name: 'NASA Earth Observing System (EOS) Clearing House (ECHO)' },
-        EOL: { short_name: 'UCAR NCAR EOL', long_name: 'UCAR NCAR - Earth Observing Laboratory' },
-        GTNP: { short_name: 'GTN-P', long_name: 'Global Terrestrial Network for Permafrost' },
-        ICES: { short_name: 'ICES', long_name: 'International Council for the Exploration of the Sea' },
-        NCDC_PALEO: { short_name: 'NOAA WDS Paleo', long_name: 'NOAA’s National Centers for Environmental Information, World Data Service for Paleoclimatology' },
-        NMI: { short_name: 'Met.no', long_name: 'Norwegian Meteorological Institute' },
-        NODC: { short_name: 'NOAA NODC', long_name: 'NOAA National Oceanographic Data Center' },
         NSIDC: { short_name: 'NSIDC', long_name: 'National Snow and Ice Data Center' },
-        PDC: { short_name: 'PDC', long_name: 'Polar Data Catalogue' },
-        R2R: { short_name: 'R2R', long_name: 'Rolling Deck to Repository' },
-        RDA: { short_name: 'UCAR NCAR RDA', long_name: 'UCAR NCAR Research Data Archive' },
-        TDAR: { short_name: 'tDAR', long_name: 'tDAR: The Digital Archaeological Record' },
-        USGS: { short_name: 'USGS ScienceBase', long_name: 'U.S. Geological Survey ScienceBase' }
       }
 
       NOT_SPECIFIED = 'Not specified'

data/lib/search_solr_tools/helpers/translate_spatial_coverage.rb

@@ -1,7 +1,6 @@
 require 'rgeo/geo_json'
 
 require_relative 'bounding_box_util'
-require_relative 'iso_to_solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '6.0.0'
+  VERSION = '6.2.0'
 end

data/lib/search_solr_tools.rb

@@ -1,10 +1,9 @@
 require_relative 'search_solr_tools/config/environments'
 require_relative 'search_solr_tools/version'
 
-require_relative 'search_solr_tools/helpers/selectors'
 require_relative 'search_solr_tools/helpers/harvest_status'
 require_relative 'search_solr_tools/errors/harvest_error'
 
-%w( selectors harvesters translators ).each do |subdir|
+%w( harvesters translators ).each do |subdir|
   Dir[File.join(__dir__, 'search_solr_tools', subdir, '*.rb')].each { |file| require file }
 end

data/search_solr_tools.gemspec

@@ -28,9 +28,9 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'ffi-geos', '~> 2.4.0'
   spec.add_runtime_dependency 'iso8601', '~> 0.13.0'
   spec.add_runtime_dependency 'multi_json', '~> 1.15.0'
-  spec.add_runtime_dependency 'nokogiri', '~> 1.13.10'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.15.3'
   spec.add_runtime_dependency 'rest-client', '~> 2.1.0'
-  spec.add_runtime_dependency 'rgeo', '~> 2.4.0'
+  spec.add_runtime_dependency 'rgeo', '~> 3.0.0'
   spec.add_runtime_dependency 'rgeo-geojson', '~> 2.1.1'
   spec.add_runtime_dependency 'rsolr', '~> 2.5.0'
   spec.add_runtime_dependency 'thor', '~> 1.2.2'
@@ -39,11 +39,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'guard', '~> 2.18.0'
   spec.add_development_dependency 'guard-rspec', '~> 4.7.3'
   spec.add_development_dependency 'guard-rubocop', '~> 1.5.0'
-  spec.add_development_dependency 'listen', '3.7.1'
+  spec.add_development_dependency 'listen', '3.8.0'
   spec.add_development_dependency 'rake', '~> 13.0.6'
-  spec.add_development_dependency 'rspec', '~> 3.11.0'
-  spec.add_development_dependency 'rubocop', '~> 1.36.0'
+  spec.add_development_dependency 'rspec', '~> 3.12.0'
+  spec.add_development_dependency 'rubocop', '~> 1.54.2'
   spec.add_development_dependency 'rubocop-rake', '~> 0.6.0'
-  spec.add_development_dependency 'rubocop-rspec', '~> 2.17.1'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.22.0'
   spec.add_development_dependency 'webmock', '~> 3.18.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-14 00:00:00.000000000 Z
+date: 2023-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi-geos
@@ -64,14 +64,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.13.10
+        version: 1.15.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.13.10
+        version: 1.15.3
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
@@ -92,14 +92,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: rgeo-geojson
   requirement: !ruby/object:Gem::Requirement
@@ -204,14 +204,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.7.1
+        version: 3.8.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.7.1
+        version: 3.8.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -232,28 +232,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.11.0
+        version: 3.12.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.11.0
+        version: 3.12.0
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.36.0
+        version: 1.54.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.36.0
+        version: 1.54.2
 - !ruby/object:Gem::Dependency
   name: rubocop-rake
   requirement: !ruby/object:Gem::Requirement
@@ -274,14 +274,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.17.1
+        version: 2.22.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.17.1
+        version: 2.22.0
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -321,59 +321,17 @@ files:
 - lib/search_solr_tools/config/environments.rb
 - lib/search_solr_tools/config/environments.yaml
 - lib/search_solr_tools/errors/harvest_error.rb
-- lib/search_solr_tools/harvesters/adc.rb
-- lib/search_solr_tools/harvesters/ade_auto_suggest.rb
 - lib/search_solr_tools/harvesters/auto_suggest.rb
 - lib/search_solr_tools/harvesters/base.rb
-- lib/search_solr_tools/harvesters/bcodmo.rb
-- lib/search_solr_tools/harvesters/data_one.rb
-- lib/search_solr_tools/harvesters/echo.rb
-- lib/search_solr_tools/harvesters/eol.rb
-- lib/search_solr_tools/harvesters/gtnp.rb
-- lib/search_solr_tools/harvesters/ices.rb
-- lib/search_solr_tools/harvesters/ncdc_paleo.rb
-- lib/search_solr_tools/harvesters/nmi.rb
-- lib/search_solr_tools/harvesters/nodc.rb
 - lib/search_solr_tools/harvesters/nsidc_auto_suggest.rb
 - lib/search_solr_tools/harvesters/nsidc_json.rb
-- lib/search_solr_tools/harvesters/oai.rb
-- lib/search_solr_tools/harvesters/pdc.rb
-- lib/search_solr_tools/harvesters/r2r.rb
-- lib/search_solr_tools/harvesters/rda.rb
-- lib/search_solr_tools/harvesters/tdar.rb
-- lib/search_solr_tools/harvesters/usgs.rb
 - lib/search_solr_tools/helpers/bounding_box_util.rb
-- lib/search_solr_tools/helpers/csw_iso_query_builder.rb
-- lib/search_solr_tools/helpers/data_one_format.rb
 - lib/search_solr_tools/helpers/facet_configuration.rb
 - lib/search_solr_tools/helpers/harvest_status.rb
 - lib/search_solr_tools/helpers/iso_namespaces.rb
-- lib/search_solr_tools/helpers/iso_to_solr.rb
-- lib/search_solr_tools/helpers/iso_to_solr_format.rb
-- lib/search_solr_tools/helpers/ncdc_paleo_format.rb
-- lib/search_solr_tools/helpers/query_builder.rb
-- lib/search_solr_tools/helpers/r2r_format.rb
-- lib/search_solr_tools/helpers/selectors.rb
 - lib/search_solr_tools/helpers/solr_format.rb
-- lib/search_solr_tools/helpers/tdar_format.rb
 - lib/search_solr_tools/helpers/translate_spatial_coverage.rb
 - lib/search_solr_tools/helpers/translate_temporal_coverage.rb
-- lib/search_solr_tools/helpers/usgs_format.rb
-- lib/search_solr_tools/selectors/adc.rb
-- lib/search_solr_tools/selectors/data_one.rb
-- lib/search_solr_tools/selectors/echo_iso.rb
-- lib/search_solr_tools/selectors/ices_iso.rb
-- lib/search_solr_tools/selectors/ncdc_paleo.rb
-- lib/search_solr_tools/selectors/nmi.rb
-- lib/search_solr_tools/selectors/nodc_iso.rb
-- lib/search_solr_tools/selectors/pdc_iso.rb
-- lib/search_solr_tools/selectors/r2r.rb
-- lib/search_solr_tools/selectors/rda.rb
-- lib/search_solr_tools/selectors/tdar_opensearch.rb
-- lib/search_solr_tools/selectors/usgs_iso.rb
-- lib/search_solr_tools/translators/bcodmo_json.rb
-- lib/search_solr_tools/translators/eol_to_solr.rb
-- lib/search_solr_tools/translators/gtnp_json.rb
 - lib/search_solr_tools/translators/nsidc_json.rb
 - lib/search_solr_tools/version.rb
 - search_solr_tools.gemspec

data/lib/search_solr_tools/harvesters/adc.rb

@@ -1,49 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    class Adc < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @page_size = 250
-        @translator = Helpers::IsoToSolr.new :adc
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of adc catalog from #{metadata_url}"
-        super(method(:harvest_adc_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]}\"")
-      end
-
-      def harvest_adc_into_solr
-        start = 0
-        while (entries = get_results_from_adc(start)) && (entries.length > 0)
-          begin
-            insert_solr_docs(get_docs_with_translated_entries_from_adc(entries))
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-          start += @page_size
-        end
-      end
-
-      def get_results_from_adc(start)
-        get_results(build_request(start, @page_size), './response/result/doc')
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:adc_url]
-      end
-
-      def get_docs_with_translated_entries_from_adc(entries)
-        entries.map do |e|
-          create_new_solr_add_doc_with_child(@translator.translate(e).root)
-        end
-      end
-
-      def build_request(start = 0, max_records = 100)
-        "#{metadata_url}&start=#{start}&rows=#{max_records}"
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/ade_auto_suggest.rb

@@ -1,46 +0,0 @@
-require_relative 'auto_suggest'
-
-module SearchSolrTools
-  module Harvesters
-    class AdeAutoSuggest < AutoSuggest
-      def harvest_and_delete
-        puts 'Building auto-suggest indexes for ADE'
-        super(method(:harvest), 'source:"ADE"', @env_settings[:auto_suggest_collection_name])
-      end
-
-      def harvest
-        url = "#{solr_url}/#{@env_settings[:collection_name]}/select?q=*%3A*&fq=source%3AADE&fq=spatial:[45.0,-180.0+TO+90.0,180.0]&rows=0&wt=json&indent=true&facet=true&facet.mincount=1&facet.sort=count&facet.limit=-1"
-        super url, fields
-      end
-
-      def fields
-        {
-          'full_keywords_and_parameters' => { weight: 2, source: 'ADE', creator: method(:keyword_creator) },
-          'full_authors' => { weight: 1, source: 'ADE', creator: method(:author_creator) }
-        }
-      end
-
-      def split_creator(value, count, field_weight, source, split_regex)
-        add_docs = []
-        value.downcase.split(split_regex).each do |v|
-          v = v.strip.chomp('/')
-          add_docs.concat(ade_length_limit_creator(v, count, field_weight, source)) unless v.nil? || v.empty?
-        end
-        add_docs
-      end
-
-      def keyword_creator(value, count, field_weight, source)
-        split_creator value, count, field_weight, source, %r{/ [\/ \>]+ /}
-      end
-
-      def author_creator(value, count, field_weight, source)
-        split_creator value, count, field_weight, source, %r{/;/}
-      end
-
-      def ade_length_limit_creator(value, count, field_weight, source)
-        return [] if value.length > 80
-        standard_add_creator value, count, field_weight, source
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/bcodmo.rb

@@ -1,64 +0,0 @@
-require_relative 'base'
-require 'json'
-require 'rest-client'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from BcoDmo endpoint, translates and adds it to solr
-    class BcoDmo < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @translator = Translators::BcodmoJsonToSolr.new
-        @wkt_parser = RGeo::WKRep::WKTParser.new(nil, {}) # (factory_generator_=nil,
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of BCO-DMO catalog from #{bcodmo_url}"
-        super(method(:harvest_bcodmo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:long_name]}\"")
-      end
-
-      def bcodmo_url
-        SolrEnvironments[@environment][:bcodmo_url]
-      end
-
-      def harvest_bcodmo_into_solr
-        result = translate_bcodmo
-        insert_solr_docs(result[:add_docs], Base::JSON_CONTENT_TYPE)
-
-        errors_exist = result[:failure_ids].length > 0
-        fail 'Failed to harvest some records from BCO-DMO' if errors_exist && @die_on_failure
-      end
-
-      def translate_bcodmo
-        documents = []
-        failure_ids = []
-        request_json(SolrEnvironments[@environment][:bcodmo_url]).each do |record|
-          geometry = request_json(record['geometryUrl'])
-          results = parse_record(record, geometry)
-          results[:documents].each { |d| documents << d }
-          results[:failure_ids].each { |id| failure_ids << id }
-        end
-        { add_docs: documents, failure_ids: failure_ids }
-      end
-
-      def request_json(url)
-        puts "Request: #{url}"
-        JSON.parse(RestClient.get(url))
-      end
-
-      def parse_record(record, geometry)
-        documents = []
-        failure_ids = []
-        begin
-          JSON.parse(RestClient.get(record['datasets'])).each do |dataset|
-            documents << { 'add' => { 'doc' => @translator.translate(dataset, record, geometry) } }
-          end
-        rescue => e
-          puts "Failed to add record #{record['id']} with error #{e} (#{e.message}) : #{e.backtrace.join("\n")}"
-          failure_ids << record['id']
-        end
-        { documents: documents, failure_ids: failure_ids }
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/data_one.rb

@@ -1,49 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    class DataOne < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @page_size = 250
-        @translator = Helpers::IsoToSolr.new :data_one
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of dataONE catalog from #{metadata_url}"
-        super(method(:harvest_data_one_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:DATA_ONE][:long_name]}\"")
-      end
-
-      def harvest_data_one_into_solr
-        start = 0
-        while (entries = get_results_from_data_one(start)) && (entries.length > 0)
-          begin
-            insert_solr_docs(get_docs_with_translated_entries_from_data_one(entries))
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-          start += @page_size
-        end
-      end
-
-      def get_results_from_data_one(start)
-        get_results(build_request(start, @page_size), './response/result/doc')
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:data_one_url]
-      end
-
-      def get_docs_with_translated_entries_from_data_one(entries)
-        entries.map do |e|
-          create_new_solr_add_doc_with_child(@translator.translate(e).root)
-        end
-      end
-
-      def build_request(start = 0, max_records = 100)
-        "#{metadata_url}&start=#{start}&rows=#{max_records}"
-      end
-    end
-  end
-end

data/lib/search_solr_tools/harvesters/echo.rb

@@ -1,52 +0,0 @@
-require_relative 'base'
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from ECHO and inserts it into Solr after it has been translated
-    class Echo < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
-        @page_size = 100
-        @translator = Helpers::IsoToSolr.new :echo
-      end
-
-      def harvest_and_delete
-        puts "Running harvest of ECHO catalog from #{echo_url}"
-        super(method(:harvest_echo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ECHO][:long_name]}\"")
-      end
-
-      # get translated entries from ECHO and add them to Solr
-      # this is the main entry point for the class
-      def harvest_echo_into_solr
-        page_num = 1
-        while (entries = get_results_from_echo(page_num)) && (entries.length > 0)
-          begin
-            insert_solr_docs get_docs_with_translated_entries_from_echo(entries)
-          rescue => e
-            puts "ERROR: #{e}\n\n"
-            raise e if @die_on_failure
-          end
-          page_num += 1
-        end
-      end
-
-      def echo_url
-        SolrEnvironments[@environment][:echo_url]
-      end
-
-      def get_results_from_echo(page_num)
-        get_results build_request(@page_size, page_num), './/results/result', 'application/echo10+xml'
-      end
-
-      def get_docs_with_translated_entries_from_echo(entries)
-        entries.map do |entry|
-          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
-        end
-      end
-
-      def build_request(max_records = '25', page_num = '1')
-        echo_url + '&page_size=' + max_records.to_s + '&page_num=' + page_num.to_s
-      end
-    end
-  end
-end