geo_combine 0.5.0 → 0.7.0

data/lib/geo_combine/geoblacklight.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/hash/except'
 require 'open-uri'
@@ -10,8 +12,8 @@ module GeoCombine
 
     attr_reader :metadata
 
-    GEOBLACKLIGHT_VERSION = 'v1.1.0'
-    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/#{GEOBLACKLIGHT_VERSION}/schema/geoblacklight-schema.json".freeze
+    GEOBLACKLIGHT_VERSION = '1.0'
+    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/main/schema/geoblacklight-schema-#{GEOBLACKLIGHT_VERSION}.json"
     DEPRECATED_KEYS_V1 = %w[
       uuid
       georss_polygon_s
@@ -29,7 +31,6 @@ module GeoCombine
     # @param [Hash] fields enhancements to metadata that are merged with @metadata
     def initialize(metadata, fields = {})
       @metadata = JSON.parse(metadata).merge(fields)
-      @schema = nil
     end
 
     ##
@@ -58,8 +59,7 @@ module GeoCombine
     # Validates a GeoBlacklight-Schema json document
     # @return [Boolean]
     def valid?
-      @schema ||= JSON.parse(open(SCHEMA_JSON_URL).read)
-      JSON::Validator.validate!(@schema, to_json, fragment: '#/properties/layer') &&
+      JSON::Validator.validate!(schema, to_json, fragment: '#/definitions/layer') &&
         dct_references_validate! &&
         spatial_validate!
     end
@@ -69,9 +69,14 @@ module GeoCombine
     # @return [Boolean]
     def dct_references_validate!
       return true unless metadata.key?('dct_references_s') # TODO: shouldn't we require this field?
+
       begin
         ref = JSON.parse(metadata['dct_references_s'])
-        raise GeoCombine::Exceptions::InvalidDCTReferences, 'dct_references must be parsed to a Hash' unless ref.is_a?(Hash)
+        unless ref.is_a?(Hash)
+          raise GeoCombine::Exceptions::InvalidDCTReferences,
+                'dct_references must be parsed to a Hash'
+        end
+
         true
       rescue JSON::ParserError => e
         raise e, "Invalid JSON in dct_references_s: #{e.message}"
@@ -89,6 +94,7 @@ module GeoCombine
     # GeoBlacklight-Schema format
     def translate_formats(key, value)
       return unless key == 'dc_format_s' && formats.include?(value)
+
       metadata[key] = formats[value]
     end
 
@@ -96,6 +102,7 @@ module GeoCombine
     # Enhances the 'layer_geom_type_s' field by translating from known types
     def translate_geometry_type(key, value)
       return unless key == 'layer_geom_type_s' && geometry_types.include?(value)
+
       metadata[key] = geometry_types[value]
     end
 
@@ -104,6 +111,7 @@ module GeoCombine
     # categories
     def enhance_subjects(key, value)
       return unless key == 'dc_subject_sm'
+
       metadata[key] = value.map do |val|
         if subjects.include?(val)
           subjects[val]
@@ -118,11 +126,13 @@ module GeoCombine
     # and ISO8601 (for indexing into Solr)
     def format_proper_date(key, value)
       return unless key == 'layer_modified_dt'
+
       metadata[key] = Time.parse(value).utc.iso8601
     end
 
     def fields_should_be_array(key, value)
       return unless should_be_array.include?(key) && !value.is_a?(Array)
+
       metadata[key] = [value]
     end
 
@@ -155,5 +165,9 @@ module GeoCombine
       # ensure we have a proper v1 record
       valid?
     end
+
+    def schema
+      @schema ||= JSON.parse(URI.open(SCHEMA_JSON_URL).read)
+    end
   end
 end

data/lib/geo_combine/geometry_types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   module GeometryTypes
     def geometry_types

data/lib/geo_combine/iso19139.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class Iso19139 < Metadata
-
     ##
     # Returns a Nokogiri::XSLT object containing the ISO19139 to GeoBlacklight
     # XSL

data/lib/geo_combine/ogp.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/core_ext/object/blank'
 require 'cgi'
 
@@ -77,15 +79,13 @@ module GeoCombine
     end
 
     def date
-      begin
-        DateTime.rfc3339(metadata['ContentDate'])
-      rescue
-        nil
-      end
+      DateTime.rfc3339(metadata['ContentDate'])
+    rescue StandardError
+      nil
     end
 
     def year
-      date.year unless date.nil?
+      date&.year
     end
 
     ##
@@ -104,9 +104,9 @@ module GeoCombine
     def ogp_formats
       case metadata['DataType']
       when 'Paper Map', 'Raster'
-        return 'GeoTIFF'
+        'GeoTIFF'
       when 'Polygon', 'Point', 'Line'
-        return 'Shapefile'
+        'Shapefile'
       else
         raise ArgumentError, metadata['DataType']
       end
@@ -128,6 +128,7 @@ module GeoCombine
                                  north >= -90 && north <= 90 &&
                                  south >= -90 && south <= 90 &&
                                  west <= east && south <= north
+
       "ENVELOPE(#{west}, #{east}, #{north}, #{south})"
     end
 
@@ -165,6 +166,7 @@ module GeoCombine
 
     def download_uri
       return 'http://schema.org/DownloadAction' if institution == 'Harvard'
+
       'http://schema.org/downloadUrl'
     end
 
@@ -205,7 +207,7 @@ module GeoCombine
       sluggify(filter_name(name))
     end
 
-    SLUG_BLACKLIST = %w[
+    SLUG_STRIP_VALUES = %w[
       SDE_DATA.
       SDE.
       SDE2.
@@ -216,8 +218,8 @@ module GeoCombine
 
     def filter_name(name)
       # strip out schema and usernames
-      SLUG_BLACKLIST.each do |blacklisted|
-        name.sub!(blacklisted, '')
+      SLUG_STRIP_VALUES.each do |strip_val|
+        name.sub!(strip_val, '')
       end
       unless name.size > 1
         # use first word of title is empty name

data/lib/geo_combine/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class Railtie < Rails::Railtie
     rake_tasks do

data/lib/geo_combine/subjects.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Translation dictionary to ISO topics

data/lib/geo_combine/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
-  VERSION = '0.5.0'
+  VERSION = '0.7.0'
 end

data/lib/geo_combine.rb

@@ -1,10 +1,11 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 require 'json'
 require 'json-schema'
 require 'sanitize'
 
 module GeoCombine
-
   ##
   # TODO: Create a parse method that can interpret the type of metadata being
   # passed in.
@@ -23,7 +24,7 @@ module GeoCombine
     # @param [String] metadata can be a File path
     # "./tmp/edu.stanford.purl/bb/338/jh/0716/iso19139.xml" or a String of XML
     # metadata
-    def initialize metadata
+    def initialize(metadata)
       metadata = File.read metadata if File.readable? metadata
       metadata = Nokogiri::XML(metadata) if metadata.instance_of? String
       @metadata = metadata
@@ -35,7 +36,7 @@ module GeoCombine
     # GeoCombine::Geoblacklight on its instantiation
     # @return [GeoCombine::Geoblacklight] the data transformed into
     # geoblacklight schema, returned as a GeoCombine::Geoblacklight
-    def to_geoblacklight fields = {}
+    def to_geoblacklight(fields = {})
       GeoCombine::Geoblacklight.new(xsl_geoblacklight.apply_to(@metadata), fields)
     end
 

data/lib/tasks/geo_combine.rake

@@ -1,17 +1,12 @@
-require 'net/http'
+# frozen_string_literal: true
+
 require 'json'
 require 'rsolr'
 require 'find'
 require 'geo_combine/geo_blacklight_harvester'
+require 'faraday/net_http_persistent'
 
 namespace :geocombine do
-  commit_within = (ENV['SOLR_COMMIT_WITHIN'] || 5000).to_i
-  ogm_path = ENV['OGM_PATH'] || 'tmp/opengeometadata'
-  solr_url = ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
-  whitelist = %w[
-    https://github.com/OpenGeoMetadata/big-ten.git
-  ]
-
   desc 'Clone OpenGeoMetadata repositories'
   task :clone, [:repo] do |_t, args|
     if args.repo
@@ -19,44 +14,44 @@ namespace :geocombine do
     else
       ogm_api_uri = URI('https://api.github.com/orgs/opengeometadata/repos')
       ogm_repos = JSON.parse(Net::HTTP.get(ogm_api_uri)).map do |repo|
-        repo['clone_url'] if repo['size'] > 0
+        repo['clone_url'] if (repo['size']).positive?
       end.compact
-      ogm_repos.select! { |repo| whitelist.include?(repo) || repo =~ /(edu|org|uk)\..*\.git$/ }
+      ogm_repos.reject! { |repo| GeoCombineRake.denylist.include?(repo) }
     end
     ogm_repos.each do |repo|
-      system "echo #{repo} && mkdir -p #{ogm_path} && cd #{ogm_path} && git clone --depth 1 #{repo}"
+      Kernel.system "echo #{repo} && mkdir -p #{GeoCombineRake.ogm_path} && cd #{GeoCombineRake.ogm_path} && git clone --depth 1 #{repo}"
     end
   end
 
   desc '"git pull" OpenGeoMetadata repositories'
   task :pull, [:repo] do |_t, args|
     paths = if args.repo
-              [File.join(ogm_path, args.repo)]
+              [File.join(GeoCombineRake.ogm_path, args.repo)]
             else
-              Dir.glob("#{ogm_path}/*")
+              Dir.glob("#{GeoCombineRake.ogm_path}/*")
             end
     paths.each do |path|
       next unless File.directory?(path)
-      system "echo #{path} && cd #{path} && git pull origin"
+
+      Kernel.system "echo #{path} && cd #{path} && git pull origin"
     end
   end
 
-  desc 'Index all of the GeoBlacklight JSON documents'
+  desc 'Index all JSON documents except Layers.json'
   task :index do
-    puts "Indexing #{ogm_path} into #{solr_url}"
-    solr = RSolr.connect url: solr_url, adapter: :net_http_persistent
-    Find.find(ogm_path) do |path|
-      next unless File.basename(path) == 'geoblacklight.json'
+    puts "Indexing #{GeoCombineRake.ogm_path} into #{GeoCombineRake.solr_url}"
+    solr = RSolr.connect url: GeoCombineRake.solr_url, adapter: :net_http_persistent
+    Find.find(GeoCombineRake.ogm_path) do |path|
+      next unless File.basename(path).include?('.json') && File.basename(path) != 'layers.json'
+
       doc = JSON.parse(File.read(path))
       [doc].flatten.each do |record|
-        begin
-          puts "Indexing #{record['layer_slug_s']}: #{path}" if $DEBUG
-          solr.update params: { commitWithin: commit_within, overwrite: true },
-                      data: [record].to_json,
-                      headers: { 'Content-Type' => 'application/json' }
-        rescue RSolr::Error::Http => error
-          puts error
-        end
+        puts "Indexing #{record['layer_slug_s']}: #{path}" if $DEBUG
+        solr.update params: { commitWithin: GeoCombineRake.commit_within, overwrite: true },
+                    data: [record].to_json,
+                    headers: { 'Content-Type' => 'application/json' }
+      rescue RSolr::Error::Http => e
+        puts e
       end
     end
     solr.commit
@@ -64,10 +59,36 @@ namespace :geocombine do
 
   namespace :geoblacklight_harvester do
     desc 'Harvest documents from a configured GeoBlacklight instance'
-    task :index, [:site] do |_t, args|
+    task :index, [:site] => [:environment] do |_t, args|
       raise ArgumentError, 'A site argument is required' unless args.site
 
-      GeoCombine::GeoBlacklightHarvester.new(args.site).index
+      GeoCombine::GeoBlacklightHarvester.new(args.site.to_sym).index
     end
   end
 end
+
+# Class to hold helper methods for use in GeoCombine rake tasks
+class GeoCombineRake
+  def self.commit_within
+    (ENV['SOLR_COMMIT_WITHIN'] || 5000).to_i
+  end
+
+  def self.denylist
+    [
+      'https://github.com/OpenGeoMetadata/GeoCombine.git',
+      'https://github.com/OpenGeoMetadata/aardvark.git',
+      'https://github.com/OpenGeoMetadata/metadata-issues.git',
+      'https://github.com/OpenGeoMetadata/ogm_utils-python.git',
+      'https://github.com/OpenGeoMetadata/opengeometadata.github.io.git',
+      'https://github.com/OpenGeoMetadata/opengeometadata-rails.git'
+    ]
+  end
+
+  def self.ogm_path
+    ENV['OGM_PATH'] || 'tmp/opengeometadata'
+  end
+
+  def self.solr_url
+    ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
+  end
+end

data/lib/xslt/fgdc2html.xsl

@@ -93,7 +93,7 @@
                 <xsl:value-of select="." />
               </dd>
             </xsl:for-each>
-            <xsl:for-each select="descrip/supplinf">
+            <xsl:for-each select="descript/supplinf">
               <dt>Supplemental Information</dt>
               <dd>
                 <xsl:value-of select="." />
@@ -204,6 +204,22 @@
                 </dd>
               </xsl:if>
             </xsl:for-each>
+            <dt>Temporal Keyword</dt>
+            <xsl:for-each select="keywords/temporal/tempkey">
+              <dd>
+                <xsl:value-of select="." />
+              </dd>
+              <xsl:if test="position()=last()">
+                <dd>
+                  <dl>
+                    <dt>Temporal Keyword Thesaurus</dt>
+                    <dd>
+                      <xsl:value-of select="ancestor-or-self::*/tempkt" />
+                    </dd>
+                  </dl>
+                </dd>
+              </xsl:if>
+            </xsl:for-each>
             <xsl:for-each select="accconst">
               <dt>Access Restrictions</dt>
               <dd>
@@ -234,6 +250,9 @@
                 <dl>
                   <xsl:for-each select="cntinfo/cntperp/cntper">
                     <dt>Contact Person</dt>
+                    <dd>
+                      <xsl:value-of select="." />
+                    </dd>
                   </xsl:for-each>
                   <xsl:for-each select="cntinfo/cntorgp/cntorg">
                     <dt>Contact Organization</dt>
@@ -277,31 +296,31 @@
                       <xsl:value-of select="." />
                     </dd>
                   </xsl:for-each>
-                  <xsl:for-each select="cntvoice">
+                  <xsl:for-each select="cntinfo/cntvoice">
                     <dt>Contact Telephone</dt>
                     <dd>
                       <xsl:value-of select="." />
                     </dd>
                   </xsl:for-each>
-                  <xsl:for-each select="cntfax">
+                  <xsl:for-each select="cntinfo/cntfax">
                     <dt>Contact Facsimile Telephone</dt>
                     <dd>
                       <xsl:value-of select="." />
                     </dd>
                   </xsl:for-each>
-                  <xsl:for-each select="cntemail">
+                  <xsl:for-each select="cntinfo/cntemail">
                     <dt>Contact Electronic Mail Address</dt>
                     <dd>
                       <xsl:value-of select="." />
                     </dd>
                   </xsl:for-each>
-                  <xsl:for-each select="hours">
+                  <xsl:for-each select="cntinfo/hours">
                     <dt>Hours of Service</dt>
                     <dd>
                       <xsl:value-of select="." />
                     </dd>
                   </xsl:for-each>
-                  <xsl:for-each select="cntinst">
+                  <xsl:for-each select="cntfo/cntinst">
                     <dt>Contact Instructions</dt>
                     <dd>
                       <xsl:value-of select="." />
@@ -475,6 +494,12 @@
                         </dl>
                       </dd>
                     </xsl:for-each>
+                    <xsl:for-each select="srccontr">
+                      <dt>Contribution</dt>
+                      <dd>
+                        <xsl:value-of select="." />
+                      </dd>
+                    </xsl:for-each>
                   </dl>
                   <xsl:for-each select="procstep">
                     <dt>Process Step</dt>
@@ -1186,6 +1211,12 @@
               <xsl:value-of select="attrdef" />
               <xsl:apply-templates select="attrdomv" />
               <dl>
+                <xsl:for-each select="attrdefs">
+                  <dt>Definition Source</dt>
+                  <dd>
+                    <xsl:value-of select="." />
+                  </dd>
+                </xsl:for-each>
                 <xsl:for-each select="begdatea">
                   <dt>Beginning Date of Attribute Values</dt>
                   <dd>
@@ -1234,9 +1265,7 @@
   </xsl:template>
 
   <xsl:template match="attrdomv[edom]">
-    <br />
-    <button onclick="this.nextElementSibling.style.display = (this.nextElementSibling.style.display==='none') ? '' : 'none';">show/hide coded values</button>
-    <dl style="display:none">
+    <dl>
       <xsl:for-each select="edom">
         <dt><xsl:value-of select="edomv" /></dt>
         <dd><xsl:value-of select="edomvd" /></dd>

data/spec/features/fgdc2html_spec.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 require 'spec_helper'
 
-#TODO Provide additional expectations on html structure
+# TODO: Provide additional expectations on html structure
 describe 'FGDC to html' do
   include XmlDocs
   let(:page) { GeoCombine::Fgdc.new(tufts_fgdc).to_html }
+
   describe 'Identification Information' do
     it 'has sections' do
       expect(page).to have_tag '#fgdc-identification-info' do
@@ -14,29 +17,78 @@ describe 'FGDC to html' do
       end
     end
   end
+
   describe 'Data Quality Information' do
     it 'has sections' do
       expect(page).to have_tag '#fgdc-data-quality-info'
     end
   end
+
   describe 'Spatial Data Organization Information' do
     it 'has sections' do
       expect(page).to have_tag '#fgdc-spatialdataorganization-info'
     end
   end
+
   describe 'Entity and Attribute Information' do
     it 'has sections' do
       expect(page).to have_tag '#fgdc-spatialreference-info'
     end
   end
+
   describe 'Distribution Information' do
     it 'has sections' do
       expect(page).to have_tag '#fgdc-distribution-info'
     end
   end
+
   describe 'Metadata Reference Information' do
     it 'has sections' do
       expect(page).to have_tag '#fgdc-metadata-reference-info'
     end
   end
+
+  describe 'Point of Contact' do
+    it 'has contact info' do
+      expect(page).to have_tag '#fgdc-identification-info'
+    end
+  end
+
+  context 'with fgdc metadata from another institution' do
+    let(:page) { GeoCombine::Fgdc.new(princeton_fgdc).to_html }
+
+    it 'has temporal keywords' do
+      expect(page).to have_tag 'dt', text: 'Temporal Keyword'
+      expect(page).to have_tag 'dd', text: '2030'
+    end
+
+    it 'has supplemental information' do
+      expect(page).to have_tag 'dt', text: 'Supplemental Information'
+      expect(page).to have_tag 'dd', text: /The E\+ scenario/
+    end
+
+    it 'has a contact person' do
+      expect(page).to have_tag 'dt', text: 'Contact Person'
+      expect(page).to have_tag 'dd', text: 'Andrew Pascale'
+    end
+
+    it 'has a contact telephone' do
+      expect(page).to have_tag 'dt', text: 'Contact Telephone'
+      expect(page).to have_tag 'dd', text: '609-258-1097'
+    end
+
+    it 'has an attribute description source and a list of values' do
+      expect(page).to have_tag 'dt', text: 'Definition Source'
+      expect(page).to have_tag 'dd', text: 'Andlinger Center/HMEI'
+
+      # Attribute elements are show by default
+      expect(page).not_to have_tag 'button'
+      expect(page).to have_tag 'dd', text: 'Desert Southwest'
+    end
+
+    it 'has a attribute source contribution' do
+      expect(page).to have_tag 'dt', text: 'Contribution'
+      expect(page).to have_tag 'dd', text: 'Net-Zero America report, 2020'
+    end
+  end
 end

data/spec/features/iso2html_spec.rb

@@ -1,9 +1,12 @@
+# frozen_string_literal: true
+
 require 'spec_helper'
 
-#TODO Provide additional expectations on html structure
+# TODO: Provide additional expectations on html structure
 describe 'ISO 19139 to html' do
   include XmlDocs
   let(:page) { GeoCombine::Iso19139.new(stanford_iso).to_html }
+
   describe 'Identification Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-identification-info' do
@@ -17,36 +20,42 @@ describe 'ISO 19139 to html' do
       end
     end
   end
+
   describe 'Spatial Reference Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-spatial-reference-info'
       expect(page).to have_tag 'h2', text: 'Identification Information'
     end
   end
+
   describe 'Data Quality Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-data-quality-info'
       expect(page).to have_tag 'h2', text: 'Data Quality Information'
     end
   end
+
   describe 'Distribution Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-distribution-info'
       expect(page).to have_tag 'h2', text: 'Distribution Information'
     end
   end
+
   describe 'Content Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-content-info'
       expect(page).to have_tag 'h2', text: 'Content Information'
     end
   end
+
   describe 'Spatial Representation Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-spatial-representation-info'
       expect(page).to have_tag 'h2', text: 'Spatial Representation Information'
     end
   end
+
   describe 'Metadata Reference Information' do
     it 'has sections' do
       expect(page).to have_tag '#iso-metadata-reference-info'