search_solr_tools 3.7.1 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 187e12f711b38e3274387d3575728d8d5250bf48
-  data.tar.gz: db79a42d420a46430dcf32e4bf442bf790d0d18b
+  metadata.gz: 2c338059d0ec2049e415517a2e6b9b89df18cdd1
+  data.tar.gz: 3cb8c2c60e269ec9b66c305730522b7da3ed8243
 SHA512:
-  metadata.gz: a69bcf39d6c09009d399cd2d029f8691ed6d00b3760a217f7ed7c5324c4171d0332b6ed2982c12caa9f663bc489a4edd5ce5ce0a8717a8dde2df78cdf2215c5c
-  data.tar.gz: 613a293d8808099b554551932c37b8c3a709cdbb94488c61cd4f65bd5d0737ba7cd30537e217beccb032d3792195759496e785f4f1ffeb6560e1f1a0560ca7e0
+  metadata.gz: 5334498934992c587a414e3672f98130b4125d7013035a1d35cbd335eecf248a5438696255da3f456fd51b6f7fef48b3e79ecd199a5af9200cc06c4aca7f563d
+  data.tar.gz: 303c74385390406d79ff379efbc44a22405427d512d8e369a685f851cc50a77d54dc07229eb105529059e06f3b5f4fb4b7a70e1902a3817f9ccad424c2a9d71d

data/CHANGELOG.md

@@ -1,24 +1,85 @@
-## v3.3.3 (2015-05-10)
+## v3.8.0 (2017-03-28)
+
+Changes
+
+  - Change ECHO harvester to harvest 100 records at a time, rather than 1000 to
+    avoid timeout/hanging issues with the large requests.
+  - Change "CISL"/ACADIS Gateway harvester to "NSF Arctic Data Center";
+    aoncadis.org redirects to another site, and the data center's name was
+    changed. The feed format was also changed; the harvester was updated to
+    consume the new feed.
+
+Bugfixes
+
+  - Update NODC feed URL to use https.
+  - Update RDA feed URL to use https.
+  - Update handling of geometries to match new format provided by BCO-DMO feed.
+  - Update NMI feed URL; the feed was relocated.
+  - Harvesting tDAR starts from record 0 instead of record 1.
+  - tDAR harvester no longer attempts to obtain another page of records after
+    all the records have been harvested; where other feeds return an empty
+    response that our harvester handles without issue, tDAR throws an error if
+    the "startRecord" parameter is higher than their last record.
+  - Exit with a non-0 status when a problem with the whole feed is encountered,
+    even if `--die-on-failure` is not passed. That flag should only cause
+    failures when there are issues with individual records; we don't want
+    harvesting to stop due to a metadata issue with a small number of
+    records.
+  - Include BCO-DMO URL in the harvester output the same way all the other URLs
+    are displayed.
+
+## v3.7.1 (2016-05-18)
+
+  - RuboCop fixes.
+
+## v3.7.0 (2016-05-18)
 
 New Features
 
+  - Add sponsored programs to NSIDC harvesting.
   - Add support for ingesting Data Access Links from NSIDC JSON
 
+Bugfixes
+
+  - Fix dependency issue with gem "listen".
+  - Fix bad configuration for OAI feed URLs.
+
+## v3.5.1 (2016-02-15)
+
+Bugfixes
+
+  - Add temporal duration facet for GTN-P data center.
+
+## v3.5.0 (2016-02-11)
+
+Changes
+
+  - Update long name for GTN-P data center.
+
+## v3.4.0 (2016-02-11)
+
+New Features
+
+  - Add harvester for GTN-P.
+
+## v3.3.4 (2016-02-08)
+
+See v3.4.0.
 
-## v3.3.2
+## v3.3.3 (2016-01-14)
 
 Bugfix
 
   - Added quote checking for cisl offset parsing check
 
-## v3.3.1
+## v3.3.1 (2015-09-25)
 
 Bugfix
 
   - Remove strange facet string for temporal duration from NOAA Paleo search
     results.
 
-## v3.3.0
+## v3.3.0 (2015-09-24)
 
 New Features
 

data/bin/search_solr_tools

@@ -19,11 +19,11 @@ class SolrHarvestCLI < Thor
       puts target
       begin
         harvest_class = get_harvester_class(target)
-        harvester = harvest_class.new options[:environment], die_on_failure
+        harvester = harvest_class.new(options[:environment], die_on_failure)
         harvester.harvest_and_delete
       rescue => e
         puts "harvest failed for #{target}: #{e.message}"
-        raise e if die_on_failure
+        raise e
       end
     end
   end
@@ -62,11 +62,10 @@ class SolrHarvestCLI < Thor
   end
 
   no_tasks do
-    # rubocop: disable MethodLength
     def harvester_map
       {
         'bco_dmo'            => SearchSolrTools::Harvesters::BcoDmo,
-        'cisl'               => SearchSolrTools::Harvesters::Cisl,
+        'adc'                => SearchSolrTools::Harvesters::Adc,
         'data_one'           => SearchSolrTools::Harvesters::DataOne,
         'echo'               => SearchSolrTools::Harvesters::Echo,
         'eol'                => SearchSolrTools::Harvesters::Eol,
@@ -85,11 +84,10 @@ class SolrHarvestCLI < Thor
         'ade_auto_suggest'   => SearchSolrTools::Harvesters::AdeAutoSuggest
       }
     end
-    # rubocop: enable MethodLength
 
     def get_harvester_class(data_center_name)
       name = data_center_name.downcase.to_s
-      raise("Invalid data center #{name}") unless harvester_map.key?(name)
+      fail("Invalid data center #{name}") unless harvester_map.key?(name)
 
       harvester_map[name]
     end

data/lib/search_solr_tools/config/environments.yaml

@@ -4,7 +4,7 @@
   :collection_path: solr
   :port: 8983
   :bcodmo_url: http://www.bco-dmo.org/nsidc/arctic-deployments.json
-  :cisl_url: https://www.aoncadis.org/oai/repository
+  :adc_url: https://arcticdata.io/metacat/d1/mn/v2/query/solr/
   :data_one_url: https://cn.dataone.org/cn/v1/query/solr/select?q=northBoundCoord:%5B45.0%20TO%2090.0%5D
   :echo_url: https://api.echo.nasa.gov/catalog-rest/echo_catalog/datasets.echo10
   :gtnp:
@@ -12,10 +12,10 @@
     - http://www.gtnpdatabase.org/rest/activelayers/json
   :ices_url: http://geo.ices.dk/geonetwork/srv/en/csw
   :ncdc_paleo_url: http://gis.ncdc.noaa.gov/gptpaleo/csw
-  :nmi_url: http://access.met.no/metamod/oai
-  :nodc_url: http://data.nodc.noaa.gov/geoportal/csw
+  :nmi_url: http://arcticdata.met.no/metamod/oai
+  :nodc_url: https://data.nodc.noaa.gov/geoportal/csw
   :pdc_url: http://www.polardata.ca/oai/provider
-  :rda_url: http://rda.ucar.edu/cgi-bin/oai
+  :rda_url: https://rda.ucar.edu/cgi-bin/oai
   :tdar_url: http://core.tdar.org/search/rss
   :usgs_url: https://www.sciencebase.gov/catalog/item/527cf4ede4b0850ea05182ee/csw
   :eol:

data/lib/search_solr_tools/harvesters/adc.rb

@@ -0,0 +1,47 @@
+module SearchSolrTools
+  module Harvesters
+    class Adc < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super
+        @page_size = 250
+        @translator = Helpers::IsoToSolr.new :adc
+      end
+
+      def harvest_and_delete
+        puts "Running harvest of adc catalog from #{metadata_url}"
+        super(method(:harvest_adc_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]}\"")
+      end
+
+      def harvest_adc_into_solr
+        start = 0
+        while (entries = get_results_from_adc(start)) && (entries.length > 0)
+          begin
+            insert_solr_docs(get_docs_with_translated_entries_from_adc(entries))
+          rescue => e
+            puts "ERROR: #{e}\n\n"
+            raise e if @die_on_failure
+          end
+          start += @page_size
+        end
+      end
+
+      def get_results_from_adc(start)
+        get_results(build_request(start, @page_size), './response/result/doc')
+      end
+
+      def metadata_url
+        SolrEnvironments[@environment][:adc_url]
+      end
+
+      def get_docs_with_translated_entries_from_adc(entries)
+        entries.map do |e|
+          create_new_solr_add_doc_with_child(@translator.translate(e).root)
+        end
+      end
+
+      def build_request(start = 0, max_records = 100)
+        "#{metadata_url}?q=*:*&start=#{start}&rows=#{max_records}"
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/base.rb

@@ -81,7 +81,7 @@ module SearchSolrTools
         end
         puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."
         puts "#{failure} document#{failure == 1 ? '' : 's'} not added to Solr."
-        raise 'Some documents failed to be inserted into Solr' if failure > 0
+        fail 'Some documents failed to be inserted into Solr' if failure > 0
       end
 
       def insert_solr_doc(doc, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])

data/lib/search_solr_tools/harvesters/bcodmo.rb

@@ -12,13 +12,18 @@ module SearchSolrTools
       end
 
       def harvest_and_delete
+        puts "Running harvest of BCO-DMO catalog from #{bcodmo_url}"
         super(method(:harvest_bcodmo_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:BCODMO][:long_name]}\"")
       end
 
+      def bcodmo_url
+        SolrEnvironments[@environment][:bcodmo_url]
+      end
+
       def harvest_bcodmo_into_solr
         result = translate_bcodmo
         insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        raise 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
+        fail 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
       end
 
       def translate_bcodmo
@@ -34,6 +39,7 @@ module SearchSolrTools
       end
 
       def request_json(url)
+        puts "Request: #{url}"
         JSON.parse(RestClient.get(url))
       end
 

data/lib/search_solr_tools/harvesters/echo.rb

@@ -4,7 +4,7 @@ module SearchSolrTools
     class Echo < Base
       def initialize(env = 'development', die_on_failure = false)
         super env, die_on_failure
-        @page_size = 1000
+        @page_size = 100
         @translator = Helpers::IsoToSolr.new :echo
       end
 
@@ -37,9 +37,9 @@ module SearchSolrTools
       end
 
       def get_docs_with_translated_entries_from_echo(entries)
-        docs = []
-        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
-        docs
+        entries.map do |entry|
+          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
+        end
       end
 
       def build_request(max_records = '25', page_num = '1')

data/lib/search_solr_tools/harvesters/eol.rb

@@ -22,7 +22,7 @@ module SearchSolrTools
             doc = open_xml_document(dataset)
             if doc.xpath('//xmlns:metadata').size > 1
               # THREDDS allows for a dataset of datasests, EOL should not utilize this
-              raise "Complex dataset encountered at #{doc.xpath('//xmlns:catalog').to_html}"
+              fail "Complex dataset encountered at #{doc.xpath('//xmlns:catalog').to_html}"
             end
             metadata_doc = open_xml_document(doc.xpath('//xmlns:metadata')[0]['xlink:href'])
             { 'add' => { 'doc' => @translator.translate(doc, metadata_doc) } }

data/lib/search_solr_tools/harvesters/gtnp.rb

@@ -28,7 +28,7 @@ module SearchSolrTools
       def harvest_gtnp_into_solr
         result = translate_gtnp
         insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        raise 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
+        fail 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
       end
 
       def translate_gtnp

data/lib/search_solr_tools/harvesters/ices.rb

@@ -37,9 +37,9 @@ module SearchSolrTools
       end
 
       def get_docs_with_translated_entries_from_ices(entries)
-        docs = []
-        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
-        docs
+        entries.map do |entry|
+          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
+        end
       end
 
       def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')

data/lib/search_solr_tools/harvesters/nodc.rb

@@ -37,9 +37,9 @@ module SearchSolrTools
       end
 
       def get_docs_with_translated_entries_from_nodc(entries)
-        docs = []
-        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
-        docs
+        entries.map do |entry|
+          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
+        end
       end
 
       def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -23,7 +23,7 @@ module SearchSolrTools
       def harvest_nsidc_json_into_solr
         result = docs_with_translated_entries_from_nsidc
         insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        raise 'Failed to harvest and insert some authoritative IDs' if result[:failure_ids].length > 0
+        fail 'Failed to harvest and insert some authoritative IDs' if result[:failure_ids].length > 0
       end
 
       def nsidc_json_url

data/lib/search_solr_tools/harvesters/oai.rb

@@ -34,11 +34,11 @@ module SearchSolrTools
       end
 
       def results
-        raise NotImplementedError
+        fail NotImplementedError
       end
 
       def metadata_url
-        raise NotImplementedError
+        fail NotImplementedError
       end
 
       def translated_docs(entries)
@@ -48,7 +48,7 @@ module SearchSolrTools
       private
 
       def request_params
-        raise NotImplementedError
+        fail NotImplementedError
       end
 
       def request_string

data/lib/search_solr_tools/harvesters/tdar.rb

@@ -14,14 +14,22 @@ module SearchSolrTools
       end
 
       def harvest_tdar_into_solr
-        start_record = 1
+        start_record = 0
+        total_harvested = 0
+        total_expected = total_results
         while (entries = get_results_from_tdar(start_record)) && (entries.length > 0)
           begin
-            insert_solr_docs get_docs_with_translated_entries_from_tdar(entries)
+            insert_solr_docs(get_docs_with_translated_entries_from_tdar(entries))
           rescue => e
             puts "ERROR: #{e}\n\n"
             raise e if @die_on_failure
           end
+
+          # if we have all the records we expect, don't attempt another request;
+          # it would result in an error
+          total_harvested += entries.length
+          break if total_harvested >= total_expected
+
           start_record += @page_size
         end
       end
@@ -31,16 +39,16 @@ module SearchSolrTools
       end
 
       def get_results_from_tdar(start_record)
-        get_results build_request(@page_size, start_record), './/atom:entry', 'application/xml'
+        get_results(build_request(@page_size, start_record), './/atom:entry', 'application/xml')
       end
 
       def get_docs_with_translated_entries_from_tdar(entries)
-        docs = []
-        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
-        docs
+        entries.map do |entry|
+          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
+        end
       end
 
-      def build_request(max_records = '25', start_record = '1')
+      def build_request(max_records = '25', start_record = '0')
         request_url = tdar_url + '?_tDAR.searchType=ACADIS_RSS&'\
                                  'resourceTypes=DATASET&'\
                                  'groups[0].latitudeLongitudeBoxes[0].maximumLongitude=180&'\
@@ -52,6 +60,10 @@ module SearchSolrTools
 
         request_url
       end
+
+      def total_results
+        get_results(build_request(0, 0), './/opensearch:totalResults').text.to_i
+      end
     end
   end
 end

data/lib/search_solr_tools/harvesters/usgs.rb

@@ -37,9 +37,9 @@ module SearchSolrTools
       end
 
       def get_docs_with_translated_entries_from_usgs(entries)
-        docs = []
-        entries.each { |r| docs.push(create_new_solr_add_doc_with_child(@translator.translate(r).root)) }
-        docs
+        entries.map do |entry|
+          create_new_solr_add_doc_with_child(@translator.translate(entry).root)
+        end
       end
 
       def build_csw_request(resultType = 'results', maxRecords = '25', startPosition = '1')

data/lib/search_solr_tools/helpers/selectors.rb

@@ -6,7 +6,7 @@ module SearchSolrTools
     # This hash grabs all the selector files inside the selectors directory,
     # to add a new source we need to create a selector file and add it to this hash.
     SELECTORS = {
-      cisl:        Selectors::CISL,
+      adc:         Selectors::ADC,
       data_one:    Selectors::DATA_ONE,
       echo:        Selectors::ECHO,
       ices:        Selectors::ICES,

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -10,7 +10,7 @@ module SearchSolrTools
     module SolrFormat
       DATA_CENTER_NAMES = {
         BCODMO: { short_name: 'BCO-DMO', long_name: 'Biological and Chemical Oceanography Data Management Office' },
-        CISL: { short_name: 'ACADIS Gateway', long_name: 'Advanced Cooperative Arctic Data and Information Service' },
+        ADC: { short_name: 'NSF ADC', long_name: 'NSF Arctic Data Center' },
         DATA_ONE: { short_name: 'DataONE', long_name: 'DataONE' },
         ECHO: { short_name: 'NASA ECHO', long_name: 'NASA Earth Observing System (EOS) Clearing House (ECHO)' },
         EOL: { short_name: 'UCAR NCAR EOL', long_name: 'UCAR NCAR - Earth Observing Laboratory' },
@@ -134,7 +134,7 @@ module SearchSolrTools
           j = send(find_index_method, resolution['max_resolution'])
           return resolution_values[i..j]
         end
-        raise "Invalid resolution #{resolution['type']}"
+        fail "Invalid resolution #{resolution['type']}"
       end
 
       def self.resolution_not_specified?(resolution)

data/lib/search_solr_tools/selectors/adc.rb

@@ -0,0 +1,95 @@
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Selectors
+    ADC = {
+      authoritative_id: {
+        xpaths: ['.//str[@name="id"]'],
+        multivalue: false
+      },
+      title: {
+        xpaths: ['.//str[@name="title"]'],
+        multivalue: false
+      },
+      summary: {
+        xpaths: ['.//str[@name="abstract"]'],
+        multivalue: false
+      },
+      data_centers: {
+        xpaths: [''],
+        default_values: [Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]],
+        multivalue: false
+      },
+      authors: {
+        xpaths: ['.//str[@name="author"]'],
+        multivalue: false
+      },
+      keywords: {
+        xpaths: ['.//arr[@name="keywords"]/str'],
+        multivalue: true
+      },
+      last_revision_date: {
+        xpaths: ['.//date[@name="updateDate"]'],
+        default_values: [Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
+        multivalue: false,
+        format: Helpers::SolrFormat::DATE
+      },
+      dataset_url: {
+        xpaths: ['.//str[@name="dataUrl"]'],
+        default_values: [''],
+        multivalue: false
+      },
+      spatial_coverages: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:spatial_display)
+      },
+      spatial: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:spatial_index)
+      },
+      spatial_area: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:spatial_area)
+      },
+      temporal_coverages: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:temporal_coverage)
+      },
+      temporal_duration: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:temporal_duration)
+      },
+      temporal: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:temporal_index_string)
+      },
+      source: {
+        xpaths: [''],
+        default_values: ['ADE'],
+        multivalue: false
+      },
+      facet_data_center: {
+        xpaths: [''],
+        default_values: ["#{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:ADC][:short_name]}"],
+        multivalue: false
+      },
+      facet_spatial_scope: {
+        xpaths: ['.'],
+        multivalue: false,
+        format: Helpers::DataOneFormat.method(:facet_spatial_scope)
+      },
+      facet_temporal_duration: {
+        xpaths: ['.'],
+        default_values: [Helpers::SolrFormat::NOT_SPECIFIED],
+        format: Helpers::DataOneFormat.method(:facet_temporal_duration),
+        multivalue: false
+      }
+    }
+  end
+end

data/lib/search_solr_tools/translators/bcodmo_json.rb

@@ -50,12 +50,29 @@ module SearchSolrTools
       end
 
       def translate_geometry(wkt_geom)
-        wkt_geom['geometry'].sub! '<http://www.opengis.net/def/crs/OGC/1.3/CRS84> ', ''
-        # Consider all linestring and polygon geometries to be multipoint for this provider
-        wkt_geom['geometry'].sub! 'LINESTRING', 'MULTIPOINT'
-        wkt_geom['geometry'].sub! 'POLYGON', 'MULTIPOINT'
-        parser = RGeo::WKRep::WKTParser.new(nil, {})
-        geometry = parser.parse(wkt_geom['geometry'])
+        if wkt_geom['geometry']['type'] == 'LineString'
+          wkt_geom['geometry']['type'] = 'MultiPoint'
+        end
+        geometry = RGeo::GeoJSON.decode(wkt_geom).geometry
+        geometry = RGeo::Feature.cast(geometry, RGeo::Feature::MultiPoint)
+
+        # This feed sometimes returns MultiLineString but wrongly calls them 'LineString'
+        # If the above fails, we assume this is why. If the feed gets fixed, this code
+        # should still handle that.
+        if geometry.nil? || geometry.num_geometries == 0
+          # Try to decode as an actual MultiLineString.
+          wkt_geom['geometry']['type'] = 'MultiLineString'
+          geometry = RGeo::GeoJSON.decode(wkt_geom).geometry
+
+          # Convert to a MultiPoint, for passing into the helper functions below.
+          coords = geometry.coordinates.flatten
+          coords = coords.each_slice(2).to_a
+          f = RGeo::Geos.factory
+          points = []
+          coords.each { |x, y| points << f.point(x, y) }
+          geometry = f.multi_point(points)
+        end
+
         {
           spatial_display: Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str([geometry]),
           spatial_index: Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str([geometry]),

data/lib/search_solr_tools/translators/gtnp_json.rb

@@ -8,7 +8,6 @@ module SearchSolrTools
   module Translators
     # Translates GTN-P json to solr json format
     class GtnpJsonToSolr
-      # rubocop:disable Metrics/MethodLength
       # rubocop:disable AbcSize
       def translate(json_doc, json_record)
         json_geo = json_doc['geo'].nil? ? json_doc['coordinates'] : json_doc['geo']['coordinates']

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '3.7.1'
+  VERSION = '3.8.0'
 end

data/search_solr_tools.gemspec

@@ -40,7 +40,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'guard-rubocop', '~> 1.2'
   spec.add_development_dependency 'rake', '~> 10.4'
   spec.add_development_dependency 'rspec', '~> 3.2'
-  spec.add_development_dependency 'rubocop', '~> 0.32'
+  spec.add_development_dependency 'rubocop', '~> 0.32.1'
   spec.add_development_dependency 'webmock', '~> 1.13'
   spec.add_development_dependency 'listen', '3.0.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 3.7.1
+  version: 3.8.0
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-18 00:00:00.000000000 Z
+date: 2017-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iso8601
@@ -230,14 +230,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.32'
+        version: 0.32.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.32'
+        version: 0.32.1
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -288,11 +288,11 @@ files:
 - lib/search_solr_tools.rb
 - lib/search_solr_tools/config/environments.rb
 - lib/search_solr_tools/config/environments.yaml
+- lib/search_solr_tools/harvesters/adc.rb
 - lib/search_solr_tools/harvesters/ade_auto_suggest.rb
 - lib/search_solr_tools/harvesters/auto_suggest.rb
 - lib/search_solr_tools/harvesters/base.rb
 - lib/search_solr_tools/harvesters/bcodmo.rb
-- lib/search_solr_tools/harvesters/cisl.rb
 - lib/search_solr_tools/harvesters/data_one.rb
 - lib/search_solr_tools/harvesters/echo.rb
 - lib/search_solr_tools/harvesters/eol.rb
@@ -325,7 +325,7 @@ files:
 - lib/search_solr_tools/helpers/translate_spatial_coverage.rb
 - lib/search_solr_tools/helpers/translate_temporal_coverage.rb
 - lib/search_solr_tools/helpers/usgs_format.rb
-- lib/search_solr_tools/selectors/cisl.rb
+- lib/search_solr_tools/selectors/adc.rb
 - lib/search_solr_tools/selectors/data_one.rb
 - lib/search_solr_tools/selectors/echo_iso.rb
 - lib/search_solr_tools/selectors/ices_iso.rb
@@ -363,7 +363,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Tools to harvest and manage various scientific dataset feeds in a Solr instance.

data/lib/search_solr_tools/harvesters/cisl.rb

@@ -1,65 +0,0 @@
-
-module SearchSolrTools
-  module Harvesters
-    # Harvests data from CISL and inserts it into Solr after it has been translated
-    class Cisl < Oai
-      def initialize(env = 'development', die_on_failure = false)
-        super
-        @data_centers = Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:long_name]
-        @translator = Helpers::IsoToSolr.new :cisl
-
-        # Used in query string params, resumptionToken
-        @dataset = '0bdd2d39-3493-4fa2-98f9-6766596bdc50'
-      end
-
-      def metadata_url
-        SolrEnvironments[@environment][:cisl_url]
-      end
-
-      def results
-        list_records_oai_response = get_results(request_string, '//oai:ListRecords', '')
-
-        @resumption_token = list_records_oai_response.xpath('.//oai:resumptionToken', Helpers::IsoNamespaces.namespaces)
-        @resumption_token = format_resumption_token(@resumption_token.first.text)
-
-        list_records_oai_response.xpath('.//oai:record', Helpers::IsoNamespaces.namespaces)
-      end
-
-      private
-
-      def request_params
-        {
-          verb: 'ListRecords',
-          metadataPrefix: 'dif',
-          set: @dataset,
-          resumptionToken: @resumption_token
-        }.delete_if { |_k, v| v.nil? }
-      end
-
-      # The ruby response is lacking quotes, which the token requires in order to work...
-      # Also, the response back seems to be inconsistent - sometimes it adds &quot; instead of '"',
-      # which makes the token fail to work.
-      # To get around this I'd prefer to make assumptions about the token and let it break if
-      # they change the formatting.  For now, all fields other than offset should be able to be
-      # assumed to remain constant.
-      # glewis 2016-01-15: It broke, offset has quotes around it, so I updated the regex to account for
-      # the possibility, including '"' or '&quot;'
-      # If the input is empty, then we are done - return an empty string, which is checked for
-      # in the harvest loop.
-      def format_resumption_token(resumption_token)
-        return '' if resumption_token.empty?
-
-        resumption_token =~ /offset(?:"|&quot;)?:(\d+)/
-        offset = Regexp.last_match(1)
-
-        {
-          from: nil,
-          until: nil,
-          set: @dataset,
-          metadataPrefix: 'dif',
-          offset: offset
-        }.to_json
-      end
-    end
-  end
-end

data/lib/search_solr_tools/selectors/cisl.rb

@@ -1,112 +0,0 @@
-require 'search_solr_tools'
-
-module SearchSolrTools
-  module Selectors
-    # The hash contains keys that should map to the fields in the solr schema,
-    # the keys are called selectors and are in charge of selecting the nodes
-    # from the ISO document, applying the default value if none of the xpaths
-    # resolved to a value and formatting the field.  xpaths and multivalue are
-    # required, default_value, format, and reduce are optional.
-    #
-    # reduce takes the formatted result of multiple nodes and produces a single
-    #   result. This is for fields that are not multivalued, but their value
-    #   should consider information from all the nodes (for example, storing
-    #   only the maximum duration from multiple temporal coverage fields, taking
-    #   the sum of multiple spatial areas)
-    CISL = {
-      authoritative_id: {
-        xpaths: ['.//oai:header/oai:identifier'],
-        multivalue: false
-      },
-      title: {
-        xpaths: ['.//dif:Entry_Title'],
-        multivalue: false
-      },
-      summary: {
-        xpaths: ['.//dif:Summary/dif:Abstract'],
-        multivalue: false
-      },
-      data_centers: {
-        xpaths: [''],
-        default_values: [SearchSolrTools::Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:long_name]],
-        multivalue: false
-      },
-      authors: {
-        xpaths: [''],
-        multivalue: true
-      },
-      keywords: {
-        xpaths: [
-          './/dif:Parameters/dif:Category',
-          './/dif:Parameters/dif:Topic',
-          './/dif:Parameters/dif:Term',
-          './/dif:Parameters/dif:Variable_Level_1'
-        ].reverse,
-        multivalue: true
-      },
-      last_revision_date: {
-        xpaths: ['.//dif:Last_DIF_Revision_Date'],
-        default_values: [SearchSolrTools::Helpers::SolrFormat.date_str(DateTime.now)], # formats the date into ISO8601 as in http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/schema/DateField.html
-        multivalue: false,
-        format: SearchSolrTools::Helpers::SolrFormat::DATE
-      },
-      dataset_url: {
-        xpaths: ['.//dif:Related_URL/dif:URL'],
-        multivalue: false
-      },
-      spatial_coverages: {
-        xpaths: ['.//dif:Spatial_Coverage'],
-        multivalue: true,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::SPATIAL_DISPLAY
-      },
-      spatial: {
-        xpaths: ['.//dif:Spatial_Coverage'],
-        multivalue: true,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::SPATIAL_INDEX
-      },
-      spatial_area: {
-        xpaths: ['.//dif:Spatial_Coverage'],
-        multivalue: false,
-        reduce: SearchSolrTools::Helpers::IsoToSolrFormat::MAX_SPATIAL_AREA,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::SPATIAL_AREA
-      },
-      temporal: {
-        xpaths: ['.//dif:Temporal_Coverage'],
-        multivalue: true,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::TEMPORAL_INDEX_STRING
-      },
-      temporal_coverages: {
-        xpaths: ['.//dif:Temporal_Coverage'],
-        multivalue: true,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::TEMPORAL_DISPLAY_STRING
-      },
-      temporal_duration: {
-        xpaths: ['.//dif:Temporal_Coverage'],
-        multivalue: false,
-        reduce: SearchSolrTools::Helpers::SolrFormat::REDUCE_TEMPORAL_DURATION,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::TEMPORAL_DURATION
-      },
-      source: {
-        xpaths: [''],
-        default_values: ['ADE'],
-        multivalue: false
-      },
-      facet_data_center: {
-        xpaths: [''],
-        default_values: ["#{SearchSolrTools::Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:CISL][:short_name]}"],
-        multivalue: false
-      },
-      facet_spatial_scope: {
-        xpaths: ['.//dif:Spatial_Coverage'],
-        multivalue: true,
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::FACET_SPATIAL_SCOPE
-      },
-      facet_temporal_duration: {
-        xpaths: ['.//dif:Temporal_Coverage'],
-        default_values: [SearchSolrTools::Helpers::SolrFormat::NOT_SPECIFIED],
-        format: SearchSolrTools::Helpers::IsoToSolrFormat::FACET_TEMPORAL_DURATION,
-        multivalue: true
-      }
-    }
-  end
-end