search_solr_tools 4.0.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0382657017a7d0cc8bc41326d81f5c5bb4119fba404f6dbbdc903080ebb88722'
-  data.tar.gz: 0ca907ef6d78f9212c1af2d421c8c0280614958c1e9c2694df3522230ae4c10a
+  metadata.gz: 5929841e7bf672bf499a73bbfd49a4f7b13db0238cfd57c3705d59513351fc30
+  data.tar.gz: f4a8809af8058bb43f24a0a0842b89e5158ca96ae16145d65f63dd8a6030f8bd
 SHA512:
-  metadata.gz: b1289b38c0ef0f39f66ca5ed913be6db354a50d8a4001dc86db098090c6f27513c62997205d370da769d517879b6f603df381ebfa3d016102b72657298794f9a
-  data.tar.gz: eb9b7978b9b7a92f3757320256e42696062ed8a06f478ff2fddb987a911ca159836716ce66e0fc1a199a33f315c2acda4426829f41d47e11b6b1088f53d27fdd
+  metadata.gz: 867c37d6e1b39d2a7de7a001c0aec4cc0891338f0d3504e7de8062b3ff88aeef1e8794f312f97b038bd4b1a4936e58e5629ea97907c417d13e9ead01d502cfb4
+  data.tar.gz: d0eb4c0cdd073d4c34c8460a72e767347d52dcc0d03de007c829e7d6b2a20075e8013d1764db39e74e4d62a62e43537599b28d98b8ab0cbe163499663b578e64

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## v4.1.0 (2019-08-12)
+
+  - Update dataset-catalog-services URL to only fetch current (*not* retired)
+    metadata records.
+  - Add a few more gem release notes to README.
+
+## v4.0.1 (2019-07-08)
+
+  - Update CHANGELOG and release instructions.
+  - Fix README typo.
+
 ## v4.0.0 (2019-07-08)
 
 Changes:

data/README.md

@@ -134,6 +134,19 @@ tagging, and publishing to RubyGems.
 * the changes are pushed
 * the tagged version is built and published to RubyGems
 
+You will need to have a current Rubygems API key for the _NSIDC developer user_ account in
+order to publish a new version of the gem to Rubygems. To get the lastest API key:
+
+`curl -u <username> https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials`
+
+## Release steps (summary)
+
+- Confirm no errors are returned by `bundle exec rubocop`
+- Confirm all tests pass (`bundle exec rake spec:unit`)
+- Update the version number and date manually in `CHANGELOG.md` and commit the
+  changes.
+- Run the appropriate `bundle exec rake release:*` task
+
 ### SOLR
 
 To harvest data utilizing the gem, you will need an installed instance of [Solr

data/bin/search_solr_tools

@@ -12,13 +12,13 @@ class SolrHarvestCLI < Thor
     puts SearchSolrTools::VERSION
   end
 
-  desc 'harvest', 'Harvest from one of the ADE harvesters'
+  desc 'harvest', 'Harvest from the specified data centers'
   option :data_center, type: :array, required: true
   option :environment, required: true
   option :die_on_failure, type: :boolean
   def harvest(die_on_failure = options[:die_on_failure] || false)
     options[:data_center].each do |target|
-      puts target
+      puts "Target: #{target}"
       begin
         harvest_class = get_harvester_class(target)
         harvester = harvest_class.new(options[:environment], die_on_failure)

data/lib/search_solr_tools/config/environments.yaml

@@ -36,39 +36,38 @@
     - http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ITEX.thredds.xml
   :r2r_url: http://get.rvdata.us/services/cruise/
 
+  # Not using DCS API v2 here because not all retired datasets have their "retired"
+  # flag checked. For example, GLA01.033 is retired; GLA01.018 is not, but it
+  # should be. GLA01.018 will show up if we use DCS API v2.
+  :nsidc_oai_identifiers_url: oai?verb=ListIdentifiers&metadata_prefix=iso&retired=false
+
 :local:
   :host: localhost
   :nsidc_dataset_metadata_url: http://integration.nsidc.org/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://integration.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
 
 :dev:
   ## For the below, you'll need to instantiate your own search-solr instance, and point host to that.
   :host: dev.search-solr.USERNAME.dev.int.nsidc.org
-  ## For these, either set up your own instance of dataset search catalog or change it to point to integration
+  ## For the metadata content, either set up your own instance of dataset-catalog-services
+  ## or change the URL below to point to integration
   :nsidc_dataset_metadata_url: http://dev.dcs.USERNAME.dev.int.nsidc.org:1580/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://dev.dcs.USERNAME.dev.int.nsidc.org:1580/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
 
 :integration:
   :host: integration.search-solr.apps.int.nsidc.org
   :nsidc_dataset_metadata_url: http://integration.nsidc.org/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://integration.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
 
 :qa:
   :host: qa.search-solr.apps.int.nsidc.org
   :nsidc_dataset_metadata_url: http://qa.nsidc.org/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://qa.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
 
 :staging:
   :host: staging.search-solr.apps.int.nsidc.org
   :nsidc_dataset_metadata_url: http://staging.nsidc.org/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://staging.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
 
 :blue:
   :host: blue.search-solr.apps.int.nsidc.org
   :nsidc_dataset_metadata_url: http://nsidc.org/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
 
 :production:
   :host: search-solr.apps.int.nsidc.org
   :nsidc_dataset_metadata_url: http://nsidc.org/api/dataset/metadata/
-  :nsidc_oai_identifiers_url: http://nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -31,9 +31,14 @@ module SearchSolrTools
       end
 
       def result_ids_from_nsidc
-        get_results SolrEnvironments[@environment][:nsidc_oai_identifiers_url], '//xmlns:identifier'
+        url = SolrEnvironments[@environment][:nsidc_dataset_metadata_url] +
+              SolrEnvironments[@environment][:nsidc_oai_identifiers_url]
+        get_results url, '//xmlns:identifier'
       end
 
+      # Fetch a JSON representation of a dataset's metadata
+      # @param id [String] NSIDC authoritative ID for the dataset
+      # @return [Hash] Parsed version of the JSON response
       def fetch_json_from_nsidc(id)
         json_response = RestClient.get(nsidc_json_url + id + '.json')
         JSON.parse(json_response)
@@ -44,6 +49,8 @@ module SearchSolrTools
         failure_ids = []
 
         result_ids_from_nsidc.each do |r|
+          # Each result looks like:
+          # oai:nsidc.org/AE_L2A
           id = r.text.split('/').last
           begin
             docs << { 'add' => { 'doc' => @translator.translate(fetch_json_from_nsidc(id)) } }

data/lib/search_solr_tools/helpers/facet_configuration.rb

@@ -8,7 +8,7 @@ module SearchSolrTools
     class FacetConfiguration
       include Singleton
       def self.import_bin_configuration(env)
-        @bin_configuration = JSON.parse(RestClient.get(SolrEnvironments[env][:nsidc_dataset_metadata_url] + '/binConfiguration')) if @bin_configuration.nil?
+        @bin_configuration = JSON.parse(RestClient.get(SolrEnvironments[env][:nsidc_dataset_metadata_url] + 'binConfiguration')) if @bin_configuration.nil?
       end
 
       def self.get_facet_bin(facet_name)

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '4.0.1'
+  VERSION = '4.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.2.0
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-08 00:00:00.000000000 Z
+date: 2019-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi-geos