search_solr_tools 3.3.3 → 3.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ede056ad3f242ee3181914e874335216920d6fb
-  data.tar.gz: 778b003b040565581be421b43cccfb49f4086952
+  metadata.gz: b659f88de85b827d7a4f56883f1c0b781cc4fea8
+  data.tar.gz: 68c5c487467f2d2cffd12dfc780ce35248cc456f
 SHA512:
-  metadata.gz: 3e9665f5671a2b1f195ec51902d396dd302b262dd848776259db3ba176e6eea53f716f4d1557ca6e8cc1d3954dc5034ae7d66664faa25db112b0a78c12f9733f
-  data.tar.gz: 3a632303193cec2b4784dce4cf0311ba69826b6d1b00c8cbe8da43748bce11434299cc378fdff507411b5003f943da37326b880d860118c51319d849c6f7a90a
+  metadata.gz: 4192221ae83802c9411d6af1aaa80e61bdeb8d67a96cf2bf8746c70199359e768e9b198be1d6ea09b0e098cb8116739fe2219d90a9e80a489b8ec2b20e8bea33
+  data.tar.gz: 515caf93f828b00e9c843dbb3447ad39b8b9bc7f73c0e33823d2c6e9be6a53c1aa63d23979962c2b9f3f15f76fcbab1ca3c15790d9d2ee7807c0a43cd4e3f9be

data/CHANGELOG.md

@@ -1,9 +1,3 @@
-## v3.3.2
-
-Bugfix
-
-  - Added quote checking for cisl offset parsing check
-
 ## v3.3.1
 
 Bugfix

data/bin/search_solr_tools

@@ -71,6 +71,7 @@ class SolrHarvestCLI < Thor
         'data_one'           => SearchSolrTools::Harvesters::DataOne,
         'echo'               => SearchSolrTools::Harvesters::Echo,
         'eol'                => SearchSolrTools::Harvesters::Eol,
+        'gtnp'               => SearchSolrTools::Harvesters::GtnP,
         'ices'               => SearchSolrTools::Harvesters::Ices,
         'ncdc_paleo'         => SearchSolrTools::Harvesters::NcdcPaleo,
         'nmi'                => SearchSolrTools::Harvesters::Nmi,

data/lib/search_solr_tools/config/environments.yaml

@@ -7,6 +7,9 @@
   :cisl_url: https://www.aoncadis.org/oai/repository
   :data_one_url: https://cn.dataone.org/cn/v1/query/solr/select?q=northBoundCoord:%5B45.0%20TO%2090.0%5D
   :echo_url: https://api.echo.nasa.gov/catalog-rest/echo_catalog/datasets.echo10
+  :gtnp:
+    - http://www.gtnpdatabase.org/rest/boreholes/json
+    - http://www.gtnpdatabase.org/rest/activelayers/json
   :ices_url: http://geo.ices.dk/geonetwork/srv/en/csw
   :ncdc_paleo_url: http://gis.ncdc.noaa.gov/gptpaleo/csw
   :nmi_url: http://access.met.no/metamod/oai

data/lib/search_solr_tools/harvesters/gtnp.rb

@@ -0,0 +1,66 @@
+require 'json'
+require 'rest-client'
+
+module SearchSolrTools
+  module Harvesters
+    # Harvests data from GTN-P endpoints, translates and adds it to solr
+    class GtnP < Base
+      def initialize(env = 'development', die_on_failure = false)
+        super env, die_on_failure
+        @translator = Translators::GtnpJsonToSolr.new
+      end
+
+      def gtnp_service_urls
+        json_records = []
+        SearchSolrTools::SolrEnvironments[:common][:gtnp].flat_map do |endpoint|
+          record = request_json(endpoint)
+          json_records << record
+        end
+        json_records
+      end
+
+      def harvest_and_delete
+        puts 'Running harvest of GTN-P catalog using the following configured GTN-P URLs:'
+        SearchSolrTools::SolrEnvironments[:common][:gtnp].each { |x| puts x }
+        super(method(:harvest_gtnp_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name]}\"")
+      end
+
+      def harvest_gtnp_into_solr
+        result = translate_gtnp
+        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
+        fail 'Failed to harvest some records from the provider' if result[:failure_ids].length > 0
+      end
+
+      def translate_gtnp
+        documents = []
+        failure_ids = []
+        gtnp_records = gtnp_service_urls
+        gtnp_records.each do |record|
+          results = parse_record(record)
+          results[:documents].each { |d| documents << d }
+          results[:failure_ids].each { |id| failure_ids << id }
+        end
+        { add_docs: documents, failure_ids: failure_ids }
+      end
+
+      def request_json(url)
+        JSON.parse(RestClient.get(url))
+      end
+
+      def parse_record(record)
+        documents = []
+        failure_ids = []
+        begin
+          record.drop(1).each do |dataset|
+            trans_doc = @translator.translate(dataset, record[0])
+            documents << { 'add' => { 'doc' => trans_doc } }
+          end
+        rescue => e
+          puts "Failed to add record #{record[0][:title]} with error #{e} (#{e.message}) : #{e.backtrace.join("\n")}"
+          failure_ids << record[0][:title]
+        end
+        { documents: documents, failure_ids: failure_ids }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -14,6 +14,7 @@ module SearchSolrTools
         DATA_ONE: { short_name: 'DataONE', long_name: 'DataONE' },
         ECHO: { short_name: 'NASA ECHO', long_name: 'NASA Earth Observing System (EOS) Clearing House (ECHO)' },
         EOL: { short_name: 'UCAR NCAR EOL', long_name: 'UCAR NCAR - Earth Observing Laboratory' },
+        GTNP: { short_name: 'GTN-P', long_name: 'Global Terrestrial Network for Permafrost (GTN-P)' },
         ICES: { short_name: 'ICES', long_name: 'International Council for the Exploration of the Sea' },
         NCDC_PALEO: { short_name: 'NOAA WDS Paleo', long_name: 'NOAA’s National Centers for Environmental Information, World Data Service for Paleoclimatology' },
         NMI: { short_name: 'Met.no', long_name: 'Norwegian Meteorological Institute' },

data/lib/search_solr_tools/translators/gtnp_json.rb

@@ -0,0 +1,55 @@
+require 'json'
+require 'rest-client'
+require 'rgeo/geo_json'
+
+require 'search_solr_tools'
+
+module SearchSolrTools
+  module Translators
+    # Translates GTN-P json to solr json format
+    class GtnpJsonToSolr
+      # rubocop:disable AbcSize
+      def translate(json_doc, json_record)
+        json_geo = json_doc['geo'].nil? ? json_doc['coordinates'] : json_doc['geo']['coordinates']
+        concatenated_name = "#{json_record['title']} - #{json_doc['name']}"
+        spatial_values = translate_geometry json_geo
+        {
+          'title' => concatenated_name,
+          'authoritative_id' => concatenated_name,
+          'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name],
+          'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:short_name]}",
+          'summary' => json_record['abstract'].to_s,
+          'dataset_url' => json_doc['link'],
+          'source' => 'ADE',
+          'facet_spatial_scope' => spatial_values[:spatial_scope_facet],
+          'spatial_coverages' => spatial_values[:spatial_display],
+          'spatial_area' => spatial_values[:spatial_area],
+          'spatial' => spatial_values[:spatial_index],
+          'temporal_coverages' => Helpers::SolrFormat::NOT_SPECIFIED,
+          'authors' => parse_people(json_doc)
+        }
+      end
+
+      def parse_people(json_doc)
+        people_found = []
+        return people_found unless json_doc.key?('citation') && json_doc['citation'].key?('contacts')
+        citation = json_doc['citation']
+        citation['contacts'].each do |person|
+          people_found << "#{person['givenName']} #{person['familyName']}"
+        end
+        people_found
+      end
+
+      def translate_geometry(json_geom)
+        geo_string = "{\"type\":\"Point\",\"coordinates\":[#{json_geom['longitude']},#{json_geom['latitude']}]}"
+        geometry = RGeo::GeoJSON.decode(geo_string, json_parser: :json)
+        {
+          spatial_display: Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str([geometry]),
+          spatial_index: Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str([geometry]),
+          spatial_area: Helpers::TranslateSpatialCoverage.geojson_to_spatial_area([geometry]),
+          spatial_scope_facet: Helpers::TranslateSpatialCoverage.geojson_to_spatial_scope_facet([geometry])
+        }
+      end
+    end
+  end
+end

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '3.3.3'
+  VERSION = '3.3.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 3.3.3
+  version: 3.3.4
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-14 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iso8601
@@ -282,6 +282,7 @@ files:
 - lib/search_solr_tools/harvesters/data_one.rb
 - lib/search_solr_tools/harvesters/echo.rb
 - lib/search_solr_tools/harvesters/eol.rb
+- lib/search_solr_tools/harvesters/gtnp.rb
 - lib/search_solr_tools/harvesters/ices.rb
 - lib/search_solr_tools/harvesters/ncdc_paleo.rb
 - lib/search_solr_tools/harvesters/nmi.rb
@@ -324,6 +325,7 @@ files:
 - lib/search_solr_tools/selectors/usgs_iso.rb
 - lib/search_solr_tools/translators/bcodmo_json.rb
 - lib/search_solr_tools/translators/eol_to_solr.rb
+- lib/search_solr_tools/translators/gtnp_json.rb
 - lib/search_solr_tools/translators/nsidc_json.rb
 - lib/search_solr_tools/version.rb
 - search_solr_tools.gemspec
@@ -347,7 +349,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Tools to harvest and manage various scientific dataset feeds in a Solr instance.