geo_combine 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d47d9cff1e3bf0e3ec2237c65554524b9a624d8c9faf833d1c99c6fdc4f7c94
-  data.tar.gz: 7069057b0b5166f2ed5496af51a270cefdd6ee5aa3937456f10a54b2f5f32536
+  metadata.gz: c91fb8acc12d958eb450bd5cebc525073f30b6df664bd55f80622a942d8e6703
+  data.tar.gz: abcc395544fe5530d2fa9b15d1f26f2e758f5b48983fcab35f8fe7305e424e92
 SHA512:
-  metadata.gz: 7208f9b13e73b183571861a7d40bcd6a1c1a466b4af7ff6fead7174bdb992a51955ebb986b07094ce195dd2e1d520a3c8b0b51b7c18178b57b3b0eb6db0c6e4e
-  data.tar.gz: 7daf75a1d31036a3659d4e96f261f4f1ce20bfea2083827f32e767ee8f15c14d6c112053cd009ca17cdfc6c433b9f7b64ce524fbf4fdaf7fe56877b1b7c07360
+  metadata.gz: 8ad8cee3e3aa481816dffe885db30217daaa3d5167240ff8395954f02226c8d54a8ce9227efa602127ac55f08106a77f1f1ca3ac0a3203fd51c92fe284327ecd
+  data.tar.gz: 523b156be9684071cd06ddcd6a10dbb14b93f3b35970ad976787edb7363667301c72963075563cccc7e330f4c9a0acad26df271f63a799e1eeaa5dc72373f574

data/.github/workflows/ruby.yml

@@ -7,12 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby
+    - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: 2.7
-    - name: Install dependencies
-      run: bundle install
+        bundler-cache: true
     - name: Run linter
       run: bundle exec rubocop
 
@@ -20,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [2.7, 3.0, 3.1]
+        ruby: [2.7, '3.0', 3.1]
         faraday_version: [''] # Defaults to whatever's the most recent version.
         include:
           - ruby: 2.7
@@ -28,16 +27,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Ruby
+    - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
-
-    - name: Install bundler
-      run: gem install bundler -v 2.1.1
-
-    - name: Install dependencies
-      run: bundle _2.1.1_ install
+        bundler-cache: true
       env:
         FARADAY_VERSION: ${{ matrix.faraday_version }}
 

data/.gitignore

@@ -14,3 +14,4 @@
 mkmf.log
 .tool-versions
 .byebug_history
+.ruby-version

data/.rubocop.yml

@@ -11,6 +11,7 @@ AllCops:
   Exclude:
   - 'geo_combine.gemspec'
   - 'tmp/**/*'
+  - 'vendor/bundle/**/*'
 
 RSpec/DescribeClass:
   Enabled: false

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2022-02-17 18:38:52 UTC using RuboCop version 1.25.1.
+# on 2023-03-27 19:15:05 UTC using RuboCop version 1.48.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -17,50 +17,45 @@ Lint/UselessAssignment:
     - 'spec/helpers.rb'
 
 # Offense count: 7
-# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
 Metrics/AbcSize:
   Max: 33
 
-# Offense count: 25
-# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
-# IgnoredMethods: refine
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode.
+# AllowedMethods: refine
 Metrics/BlockLength:
-  Max: 181
+  Max: 27
 
 # Offense count: 1
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
   Max: 152
 
-# Offense count: 3
-# Configuration parameters: IgnoredMethods.
+# Offense count: 4
+# Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/CyclomaticComplexity:
   Max: 11
 
-# Offense count: 10
-# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# Offense count: 11
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 21
 
 # Offense count: 1
-# Configuration parameters: CountComments, CountAsOne.
-Metrics/ModuleLength:
-  Max: 1657
-
-# Offense count: 1
-# Configuration parameters: IgnoredMethods.
+# Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/PerceivedComplexity:
   Max: 11
 
 # Offense count: 9
-# Configuration parameters: Prefixes.
+# Configuration parameters: Prefixes, AllowedPatterns.
 # Prefixes: when, with, without
 RSpec/ContextWording:
   Exclude:
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
     - 'spec/lib/geo_combine/ogp_spec.rb'
 
-# Offense count: 9
+# Offense count: 11
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 12
@@ -78,10 +73,10 @@ RSpec/FilePath:
   Exclude:
     - 'spec/lib/geo_combine_spec.rb'
 
-# Configuration parameters: .
+# Offense count: 23
+# Configuration parameters: EnforcedStyle.
 # SupportedStyles: have_received, receive
 RSpec/MessageSpies:
-  EnforcedStyle: have_received
   Exclude:
     - 'spec/lib/geo_combine/esri_open_data_spec.rb'
     - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
@@ -89,7 +84,7 @@ RSpec/MessageSpies:
     - 'spec/lib/geo_combine/ogp_spec.rb'
     - 'spec/lib/geo_combine_spec.rb'
 
-# Offense count: 39
+# Offense count: 48
 RSpec/MultipleExpectations:
   Max: 5
 
@@ -99,12 +94,14 @@ RSpec/MultipleMemoizedHelpers:
   Max: 7
 
 # Offense count: 5
-# Configuration parameters: IgnoreSharedExamples.
+# Configuration parameters: EnforcedStyle, IgnoreSharedExamples.
+# SupportedStyles: always, named_only
 RSpec/NamedSubject:
   Exclude:
     - 'spec/lib/geo_combine/formatting_spec.rb'
 
 # Offense count: 8
+# Configuration parameters: AllowedGroups.
 RSpec/NestedGroups:
   Max: 4
 
@@ -113,6 +110,11 @@ RSpec/OverwritingSetup:
   Exclude:
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
 
+# Offense count: 1
+RSpec/PendingWithoutReason:
+  Exclude:
+    - 'spec/lib/geo_combine/migrators/v1_aardvark_migrator_spec.rb'
+
 # Offense count: 2
 RSpec/RepeatedExampleGroupBody:
   Exclude:
@@ -151,15 +153,14 @@ Style/Documentation:
     - 'test/**/*'
     - 'lib/geo_combine/bounding_box.rb'
     - 'lib/geo_combine/ckan_metadata.rb'
-    - 'lib/geo_combine/cli.rb'
     - 'lib/geo_combine/geo_blacklight_harvester.rb'
     - 'lib/geo_combine/geoblacklight.rb'
     - 'lib/geo_combine/geometry_types.rb'
     - 'lib/geo_combine/iso19139.rb'
 
-# Offense count: 7
-# Cop supports --auto-correct.
-# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# Offense count: 8
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
 # URISchemes: http, https
 Layout/LineLength:
   Max: 159

data/geo_combine.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sanitize'
   spec.add_dependency 'thor'
   spec.add_dependency 'faraday-net_http_persistent', '~> 2.0'
+  spec.add_dependency 'git'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/lib/geo_combine/formatting.rb

@@ -30,7 +30,7 @@ module GeoCombine
 
     # slugs should be lowercase and only have a-z, A-Z, 0-9, and -
     def sluggify(slug)
-      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/-+/, '-').downcase
+      slug.gsub(/[^a-zA-Z0-9-]/, '-').gsub(/-+/, '-').downcase
     end
   end
 end

data/lib/geo_combine/geoblacklight.rb

@@ -13,7 +13,7 @@ module GeoCombine
     attr_reader :metadata
 
     GEOBLACKLIGHT_VERSION = '1.0'
-    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/main/schema/geoblacklight-schema-#{GEOBLACKLIGHT_VERSION}.json"
+    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/OpenGeoMetadata/opengeometadata.github.io/main/docs/schema/geoblacklight-schema-#{GEOBLACKLIGHT_VERSION}.json"
     DEPRECATED_KEYS_V1 = %w[
       uuid
       georss_polygon_s

data/lib/geo_combine/harvester.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'find'
+require 'git'
+
+module GeoCombine
+  # Harvests Geoblacklight documents from OpenGeoMetadata for indexing
+  class Harvester
+    attr_reader :ogm_path, :schema_version
+
+    # Non-metadata repositories that shouldn't be harvested
+    def self.denylist
+      [
+        'GeoCombine',
+        'aardvark',
+        'metadata-issues',
+        'ogm_utils-python',
+        'opengeometadata.github.io',
+        'opengeometadata-rails',
+        'gbl-1_to_aardvark'
+      ]
+    end
+
+    # GitHub API endpoint for OpenGeoMetadata repositories
+    def self.ogm_api_uri
+      URI('https://api.github.com/orgs/opengeometadata/repos?per_page=1000')
+    end
+
+    def initialize(
+      ogm_path: ENV.fetch('OGM_PATH', 'tmp/opengeometadata'),
+      schema_version: ENV.fetch('SCHEMA_VERSION', '1.0')
+    )
+      @ogm_path = ogm_path
+      @schema_version = schema_version
+    end
+
+    # Enumerable of docs to index, for passing to an indexer
+    def docs_to_index
+      return to_enum(:docs_to_index) unless block_given?
+
+      Find.find(@ogm_path) do |path|
+        # skip non-json and layers.json files
+        next unless File.basename(path).include?('.json') && File.basename(path) != 'layers.json'
+
+        doc = JSON.parse(File.read(path))
+        [doc].flatten.each do |record|
+          # skip indexing if this record has a different schema version than what we want
+          record_schema = record['gbl_mdVersion_s'] || record['geoblacklight_version']
+          next unless record_schema == @schema_version
+
+          yield record, path
+        end
+      end
+    end
+
+    # Update a repository via git
+    # If the repository doesn't exist, clone it.
+    def pull(repo)
+      repo_path = File.join(@ogm_path, repo)
+      clone(repo) unless File.directory? repo_path
+
+      Git.open(repo_path).pull
+      puts "Updated #{repo}"
+      1
+    end
+
+    # Update all repositories
+    # Return the count of repositories updated
+    def pull_all
+      repositories.map(&method(:pull)).reduce(:+)
+    end
+
+    # Clone a repository via git
+    # If the repository already exists, skip it.
+    def clone(repo)
+      repo_path = File.join(@ogm_path, repo)
+      repo_info = repository_info(repo)
+
+      # Skip if exists; warn if archived or empty
+      if File.directory? repo_path
+        puts "Skipping clone to #{repo_path}; directory exists"
+        return 0
+      end
+      puts "WARNING: repository '#{repo}' is archived" if repo_info['archived']
+      puts "WARNING: repository '#{repo}' is empty" if repo_info['size'].zero?
+
+      repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
+      Git.clone(repo_url, nil, path: ogm_path, depth: 1)
+      puts "Cloned #{repo_url}"
+      1
+    end
+
+    # Clone all repositories via git
+    # Return the count of repositories cloned.
+    def clone_all
+      repositories.map(&method(:clone)).reduce(:+)
+    end
+
+    private
+
+    # List of repository names to harvest
+    def repositories
+      @repositories ||= JSON.parse(Net::HTTP.get(self.class.ogm_api_uri))
+                            .filter { |repo| repo['size'].positive? }
+                            .reject { |repo| repo['archived'] }
+                            .map { |repo| repo['name'] }
+                            .reject { |name| self.class.denylist.include? name }
+    end
+
+    def repository_info(repo_name)
+      JSON.parse(Net::HTTP.get(URI("https://api.github.com/repos/opengeometadata/#{repo_name}")))
+    end
+  end
+end

data/lib/geo_combine/indexer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'rsolr'
+require 'faraday/net_http_persistent'
+
+module GeoCombine
+  # Indexes Geoblacklight documents into Solr
+  class Indexer
+    attr_reader :solr
+
+    def self.solr(url: ENV.fetch('SOLR_URL', 'http://127.0.0.1:8983/solr/blacklight-core'))
+      RSolr.connect url: url, adapter: :net_http_persistent
+    end
+
+    def initialize(solr: GeoCombine::Indexer.solr)
+      @solr = solr
+    end
+
+    def solr_url
+      @solr.options[:url]
+    end
+
+    # Index everything and return the number of docs successfully indexed
+    def index(docs, commit_within: ENV.fetch('SOLR_COMMIT_WITHIN', 5000).to_i)
+      indexed_count = 0
+
+      docs.each do |record, path|
+        # log the unique identifier for the record for debugging
+        id = record['id'] || record['dc_identifier_s']
+        puts "Indexing #{id}: #{path}" if $DEBUG
+
+        # index the record into solr
+        @solr.update params: { commitWithin: commit_within, overwrite: true },
+                     data: [record].to_json,
+                     headers: { 'Content-Type' => 'application/json' }
+
+        # count the number of records successfully indexed
+        indexed_count += 1
+      rescue RSolr::Error::Http => e
+        puts e
+      end
+
+      @solr.commit
+      indexed_count
+    end
+  end
+end

data/lib/geo_combine/migrators/v1_aardvark_migrator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module GeoCombine
+  module Migrators
+    # TODO: WARNING! This class is not fully implemented and should not be used in
+    # production. See https://github.com/OpenGeoMetadata/GeoCombine/issues/121
+    # for remaining work.
+    #
+    # migrates the v1 schema to the aardvark schema
+    class V1AardvarkMigrator
+      attr_reader :v1_hash
+
+      # @param v1_hash [Hash] parsed json in the v1 schema
+      def initialize(v1_hash:)
+        @v1_hash = v1_hash
+      end
+
+      def run
+        v2_hash = convert_keys
+        v2_hash['gbl_mdVersion_s'] = 'Aardvark'
+        v2_hash
+      end
+
+      def convert_keys
+        v1_hash.transform_keys do |k|
+          SCHEMA_FIELD_MAP[k] || k
+        end
+      end
+
+      SCHEMA_FIELD_MAP = {
+        'dc_title_s' => 'dct_title_s', # new namespace
+        'dc_description_s' => 'dct_description_sm', # new namespace; single to multi-valued
+        'dc_language_s' => 'dct_language_sm', # new namespace; single to multi-valued
+        'dc_language_sm' => 'dct_language_sm', # new namespace; single to multi-valued
+        'dc_creator_sm' => 'dct_creator_sm', # new namespace
+        'dc_publisher_s' => 'dct_publisher_sm', # new namespace; single to multi-valued
+        'dct_provenance_s' => 'schema_provider_s', # new URI name
+        'dc_subject_sm' => 'dct_subject_sm', # new namespace
+        'solr_year_i' => 'gbl_indexYear_im', # new URI name; single to multi-valued
+        'dc_source_sm' => 'dct_source_sm', # new namespace
+        'dc_rights_s' => 'dct_accessRights_s', # new URI name
+        'dc_format_s' => 'dct_format_s', # new namespace
+        'layer_id_s' => 'gbl_wxsIdentifier_s', # new URI name
+        'layer_slug_s' => 'id', # new URI name
+        'dc_identifier_s' => 'dct_identifier_sm', # new namespace; single to multi-valued
+        'layer_modified_dt' => 'gbl_mdModified_dt', # new URI name
+        'geoblacklight_version' => 'gbl_mdVersion_s', # new URI name
+        'suppressed_b' => 'gbl_suppressed_b' # new namespace
+      }.freeze
+    end
+  end
+end

data/lib/geo_combine/railtie.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module GeoCombine
+  # Railtie for rake tasks
   class Railtie < Rails::Railtie
     rake_tasks do
       load 'tasks/geo_combine.rake'

data/lib/geo_combine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GeoCombine
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/lib/geo_combine.rb

@@ -72,6 +72,9 @@ require 'geo_combine/ogp'
 # Require harvesting/indexing files
 require 'geo_combine/geo_blacklight_harvester'
 
+# Migrators
+require 'geo_combine/migrators/v1_aardvark_migrator'
+
 # Require gem files
 require 'geo_combine/version'
 require 'geo_combine/railtie' if defined?(Rails)

data/lib/tasks/geo_combine.rake

@@ -3,58 +3,33 @@
 require 'json'
 require 'rsolr'
 require 'find'
-require 'geo_combine/geo_blacklight_harvester'
 require 'faraday/net_http_persistent'
+require 'geo_combine/harvester'
+require 'geo_combine/indexer'
+require 'geo_combine/geo_blacklight_harvester'
 
 namespace :geocombine do
   desc 'Clone OpenGeoMetadata repositories'
   task :clone, [:repo] do |_t, args|
-    if args.repo
-      ogm_repos = ["https://github.com/OpenGeoMetadata/#{args.repo}.git"]
-    else
-      ogm_api_uri = URI('https://api.github.com/orgs/opengeometadata/repos')
-      ogm_repos = JSON.parse(Net::HTTP.get(ogm_api_uri)).map do |repo|
-        repo['clone_url'] if (repo['size']).positive?
-      end.compact
-      ogm_repos.reject! { |repo| GeoCombineRake.denylist.include?(repo) }
-    end
-    ogm_repos.each do |repo|
-      Kernel.system "echo #{repo} && mkdir -p #{GeoCombineRake.ogm_path} && cd #{GeoCombineRake.ogm_path} && git clone --depth 1 #{repo}"
-    end
+    harvester = GeoCombine::Harvester.new
+    total = args[:repo] ? harvester.clone(args.repo) : harvester.clone_all
+    puts "Cloned #{total} repositories"
   end
 
   desc '"git pull" OpenGeoMetadata repositories'
   task :pull, [:repo] do |_t, args|
-    paths = if args.repo
-              [File.join(GeoCombineRake.ogm_path, args.repo)]
-            else
-              Dir.glob("#{GeoCombineRake.ogm_path}/*")
-            end
-    paths.each do |path|
-      next unless File.directory?(path)
-
-      Kernel.system "echo #{path} && cd #{path} && git pull origin"
-    end
+    harvester = GeoCombine::Harvester.new
+    total = args[:repo] ? harvester.pull(args.repo) : harvester.pull_all
+    puts "Updated #{total} repositories"
   end
 
   desc 'Index all JSON documents except Layers.json'
   task :index do
-    puts "Indexing #{GeoCombineRake.ogm_path} into #{GeoCombineRake.solr_url}"
-    solr = RSolr.connect url: GeoCombineRake.solr_url, adapter: :net_http_persistent
-    Find.find(GeoCombineRake.ogm_path) do |path|
-      next unless File.basename(path).include?('.json') && File.basename(path) != 'layers.json'
-
-      doc = JSON.parse(File.read(path))
-      [doc].flatten.each do |record|
-        puts "Indexing #{record['layer_slug_s']}: #{path}" if $DEBUG
-        solr.update params: { commitWithin: GeoCombineRake.commit_within, overwrite: true },
-                    data: [record].to_json,
-                    headers: { 'Content-Type' => 'application/json' }
-      rescue RSolr::Error::Http => e
-        puts e
-      end
-    end
-    solr.commit
+    harvester = GeoCombine::Harvester.new
+    indexer = GeoCombine::Indexer.new
+    puts "Indexing #{harvester.ogm_path} into #{indexer.solr_url}"
+    total = indexer.index(harvester.docs_to_index)
+    puts "Indexed #{total} documents"
   end
 
   namespace :geoblacklight_harvester do
@@ -66,29 +41,3 @@ namespace :geocombine do
     end
   end
 end
-
-# Class to hold helper methods for use in GeoCombine rake tasks
-class GeoCombineRake
-  def self.commit_within
-    (ENV['SOLR_COMMIT_WITHIN'] || 5000).to_i
-  end
-
-  def self.denylist
-    [
-      'https://github.com/OpenGeoMetadata/GeoCombine.git',
-      'https://github.com/OpenGeoMetadata/aardvark.git',
-      'https://github.com/OpenGeoMetadata/metadata-issues.git',
-      'https://github.com/OpenGeoMetadata/ogm_utils-python.git',
-      'https://github.com/OpenGeoMetadata/opengeometadata.github.io.git',
-      'https://github.com/OpenGeoMetadata/opengeometadata-rails.git'
-    ]
-  end
-
-  def self.ogm_path
-    ENV['OGM_PATH'] || 'tmp/opengeometadata'
-  end
-
-  def self.solr_url
-    ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
-  end
-end

data/spec/fixtures/docs/full_geoblacklight_aardvark.json

@@ -0,0 +1,33 @@
+{
+  "gbl_mdVersion_s":"Aardvark",
+  "dct_identifier_sm":"http://purl.stanford.edu/cz128vq0535",
+  "dct_title_s":"2005 Rural Poverty GIS Database: Uganda",
+  "dct_description_sm":"This polygon shapefile contains 2005 poverty data for 855 rural subcounties in Uganda. These data are intended for researchers, students, policy makers and the general public for reference and mapping purposes, and may be used for basic applications such as viewing, querying, and map output production.",
+  "dct_accessRights_s":"Public",
+  "schema_provider_s":"Stanford",
+  "dct_references_s":"{\"http://schema.org/url\":\"http://purl.stanford.edu/cz128vq0535\",\"http://schema.org/downloadUrl\":\"http://stacks.stanford.edu/file/druid:cz128vq0535/data.zip\",\"http://www.loc.gov/mods/v3\":\"http://purl.stanford.edu/cz128vq0535.mods\",\"http://www.isotc211.org/schemas/2005/gmd/\":\"http://opengeometadata.stanford.edu/metadata/edu.stanford.purl/druid:cz128vq0535/iso19139.xml\",\"http://www.w3.org/1999/xhtml\":\"http://opengeometadata.stanford.edu/metadata/edu.stanford.purl/druid:cz128vq0535/default.html\",\"http://www.opengis.net/def/serviceType/ogc/wfs\":\"https://geowebservices.stanford.edu/geoserver/wfs\",\"http://www.opengis.net/def/serviceType/ogc/wms\":\"https://geowebservices.stanford.edu/geoserver/wms\"}",
+  "gbl_wxsIdentifier_s":"druid:cz128vq0535",
+  "id":"stanford-cz128vq0535",
+  "layer_geom_type_s":"Polygon",
+  "gbl_mdModified_dt":"2015-01-13T18:46:38Z",
+  "dct_format_s":"Shapefile",
+  "dct_language_sm":"English",
+  "dc_type_s":"Dataset",
+  "dct_publisher_sm":"Uganda Bureau of Statistics",
+  "dct_creator_sm":[
+    "Uganda Bureau of Statistics"
+  ],
+  "dct_subject_sm":[
+    "Poverty",
+    "Statistics"
+  ],
+  "dct_issued_s":"2005",
+  "dct_temporal_sm":[
+    "2005"
+  ],
+  "dct_spatial_sm":[
+    "Uganda"
+  ],
+  "solr_geom":"ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)",
+  "gbl_indexYear_im":2005
+}

data/spec/fixtures/indexing/aardvark.json

@@ -0,0 +1,57 @@
+{
+  "dct_title_s": "A description of the coast & city of Goa.",
+  "dct_alternative_sm": [
+    "A description of the coast & city of Goa."
+  ],
+  "dct_description_sm": [
+    "Photocopy.   Some relief shown pictorially.   North oriented to the left.   \"The city of Goa & all its dependencies doth justly belong to the crown of England by the mariage [sic] of King Charles the Second with Queen Catherine\"--upper right.   \"21\"--upper right.   Outside of original margin is image of British Museum rule and \"7-Tab-125.\" Dimensions: 51 x 61 centimeters"
+  ],
+  "dct_language_sm": [
+    "eng"
+  ],
+  "dct_publisher_sm": [
+    "British Museum? (London?)"
+  ],
+  "schema_provider_s": "University of Minnesota",
+  "gbl_resourceClass_sm": [
+    "Maps"
+  ],
+  "dcat_keyword_sm": [
+    "Velha Goa (India) Maps",
+    "Maps"
+  ],
+  "dct_temporal_sm": [
+    "1900-1999"
+  ],
+  "dct_issued_s": "1900 - 1999?",
+  "gbl_indexYear_im": [
+    "1900"
+  ],
+  "gbl_dateRange_drsim": [
+    "[1900 TO 1999]"
+  ],
+  "dct_spatial_sm": [
+    "India"
+  ],
+  "locn_geometry": "ENVELOPE(-2.36,-2.06,11.73,11.5101)",
+  "dcat_bbox": "ENVELOPE(-2.36,-2.06,11.73,11.5101)",
+  "dcat_centroid": "11.620049999999999,-2.21",
+  "pcdm_memberOf_sm": [
+    "64bd8c4c-8e60-4956-b43d-bdc3f93db488"
+  ],
+  "dct_isPartOf_sm": [
+    "05d-01"
+  ],
+  "dct_rights_sm": [
+    "Use of this item may be governed by US and international copyright laws. You may be able to use this item, but copyright and other considerations may apply. For possible additional information or guidance on your use, please contact the contributing organization."
+  ],
+  "dct_accessRights_s": "Public",
+  "dct_format_s": "JPEG",
+  "dct_references_s": "{\"http://schema.org/downloadUrl\":\"http://cdm16022.contentdm.oclc.org/utils/getfile/collection/p16022coll205/id/236/filename/print/page/download/fparams/forcedownload\",\"http://schema.org/url\":\"https://umedia.lib.umn.edu/item/p16022coll205:236\",\"http://iiif.io/api/presentation#manifest\":\"https://cdm16022.contentdm.oclc.org/iiif/info/p16022coll205/236/manifest.json\"}",
+  "id": "p16022coll205:236",
+  "dct_identifier_sm": [
+    "UMN_ALMA:9949551790001701"
+  ],
+  "gbl_mdModified_dt": "2022-04-01T15:27:13Z",
+  "gbl_mdVersion_s": "Aardvark"
+}

data/spec/fixtures/json_docs.rb

@@ -15,6 +15,12 @@ module JsonDocs
     File.read(File.join(File.dirname(__FILE__), './docs/full_geoblacklight.json'))
   end
 
+  ##
+  #  full_geoblacklight fixture converted to the aardvark schema
+  def full_geoblacklight_aardvark
+    File.read(File.join(File.dirname(__FILE__), './docs/full_geoblacklight_aardvark.json'))
+  end
+
   ##
   # A sample Esri OpenData metadata record
   def esri_opendata_metadata

data/spec/lib/geo_combine/bounding_box_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe GeoCombine::BoundingBox do
 
   describe '#valid?' do
     context 'when valid' do
-      it { valid.valid? }
+      it { expect(valid.valid?).to be true }
     end
 
     context 'when south > north' do

data/spec/lib/geo_combine/geoblacklight_spec.rb

@@ -151,8 +151,8 @@ RSpec.describe GeoCombine::Geoblacklight do
       let(:unparseable_json) do
         <<-JSON
           {
-            \"http://schema.org/url\":\"http://example.com/abc123\",,
-            \"http://schema.org/downloadUrl\":\"http://example.com/abc123/data.zip\"
+            "http://schema.org/url":"http://example.com/abc123",,
+            "http://schema.org/downloadUrl":"http://example.com/abc123/data.zip"
           }
         JSON
       end
@@ -184,7 +184,7 @@ RSpec.describe GeoCombine::Geoblacklight do
 
   describe 'spatial_validate!' do
     context 'when valid' do
-      it { full_geobl.spatial_validate! }
+      it { expect { full_geobl.spatial_validate! }.not_to raise_error }
     end
 
     context 'when invalid' do

data/spec/lib/geo_combine/harvester_spec.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'git'
+require 'geo_combine/harvester'
+require 'spec_helper'
+
+RSpec.describe GeoCombine::Harvester do
+  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing') }
+
+  let(:repo_name) { 'my-institution' }
+  let(:repo_path) { File.join(harvester.ogm_path, repo_name) }
+  let(:repo_url) { "https://github.com/OpenGeoMetadata/#{repo_name}.git" }
+  let(:stub_repo) { instance_double(Git::Base) }
+  let(:stub_gh_api) do
+    [
+      { name: repo_name, size: 100 },
+      { name: 'another-institution', size: 100 },
+      { name: 'outdated-institution', size: 100, archived: true }, # archived
+      { name: 'aardvark', size: 300 },                             # on denylist
+      { name: 'empty', size: 0 }                                   # no data
+    ]
+  end
+
+  before do
+    # stub github API requests
+    # use the whole org response, or just a portion for particular repos
+    allow(Net::HTTP).to receive(:get) do |uri|
+      if uri == described_class.ogm_api_uri
+        stub_gh_api.to_json
+      else
+        repo_name = uri.path.split('/').last.gsub('.git', '')
+        stub_gh_api.find { |repo| repo[:name] == repo_name }.to_json
+      end
+    end
+
+    # stub git commands
+    allow(Git).to receive(:open).and_return(stub_repo)
+    allow(Git).to receive(:clone).and_return(stub_repo)
+    allow(stub_repo).to receive(:pull).and_return(stub_repo)
+  end
+
+  describe '#docs_to_index' do
+    it 'yields each JSON record with its path, skipping layers.JSON' do
+      expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
+        [JSON.parse(File.read('spec/fixtures/indexing/basic_geoblacklight.json')), 'spec/fixtures/indexing/basic_geoblacklight.json'],
+        [JSON.parse(File.read('spec/fixtures/indexing/geoblacklight.json')), 'spec/fixtures/indexing/geoblacklight.json']
+      )
+    end
+
+    it 'skips records with a different schema version' do
+      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: 'Aardvark')
+      expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
+        [JSON.parse(File.read('spec/fixtures/indexing/aardvark.json')), 'spec/fixtures/indexing/aardvark.json']
+      )
+    end
+  end
+
+  describe '#pull' do
+    it 'can pull a single repository' do
+      harvester.pull(repo_name)
+      expect(Git).to have_received(:open).with(repo_path)
+      expect(stub_repo).to have_received(:pull)
+    end
+
+    it 'clones a repo before pulling if it does not exist' do
+      harvester.pull(repo_name)
+      expect(Git).to have_received(:clone)
+    end
+  end
+
+  describe '#pull_all' do
+    it 'can pull all repositories' do
+      harvester.pull_all
+      expect(Git).to have_received(:open).exactly(2).times
+      expect(stub_repo).to have_received(:pull).exactly(2).times
+    end
+
+    it 'returns the count of repositories pulled' do
+      expect(harvester.pull_all).to eq(2)
+    end
+
+    it 'skips repositories in the denylist' do
+      harvester.pull_all
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/aardvark.git')
+    end
+
+    it 'skips archived repositories' do
+      harvester.pull_all
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/outdated-institution.git')
+    end
+  end
+
+  describe '#clone' do
+    it 'can clone a single repository' do
+      harvester.clone(repo_name)
+      expect(Git).to have_received(:clone).with(
+        repo_url,
+        nil, {
+          depth: 1, # shallow clone
+          path: harvester.ogm_path
+        }
+      )
+    end
+
+    it 'skips repositories that already exist' do
+      allow(File).to receive(:directory?).with(repo_path).and_return(true)
+      harvester.clone(repo_name)
+      expect(Git).not_to have_received(:clone)
+    end
+
+    it 'warns if a repository is empty' do
+      allow(Net::HTTP).to receive(:get).with('https://api.github.com/repos/opengeometadata/empty').and_return('{"size": 0}')
+      expect do
+        harvester.clone('empty')
+      end.to output(/repository 'empty' is empty/).to_stdout
+    end
+
+    it 'warns if a repository is archived' do
+      allow(Net::HTTP).to receive(:get).with('https://api.github.com/repos/opengeometadata/empty').and_return('{"archived": true}')
+      expect do
+        harvester.clone('outdated-institution')
+      end.to output(/repository 'outdated-institution' is archived/).to_stdout
+    end
+  end
+
+  describe '#clone_all' do
+    it 'can clone all repositories' do
+      harvester.clone_all
+      expect(Git).to have_received(:clone).exactly(2).times
+    end
+
+    it 'skips repositories in the denylist' do
+      harvester.clone_all
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/aardvark.git')
+    end
+
+    it 'returns the count of repositories cloned' do
+      expect(harvester.clone_all).to eq(2)
+    end
+  end
+
+  describe '#ogm_api_uri' do
+    it 'includes a per_page param' do
+      expect(described_class.send('ogm_api_uri').to_s).to include('per_page')
+    end
+  end
+end

data/spec/lib/geo_combine/indexer_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'geo_combine/indexer'
+require 'spec_helper'
+
+RSpec.describe GeoCombine::Indexer do
+  subject(:indexer) { described_class.new(solr: solr) }
+
+  let(:solr) { instance_double(RSolr::Client) }
+  let(:docs) do
+    [
+      [{ 'id' => '1' }, 'path/to/record1.json'],              # v1.0 schema
+      [{ 'dc_identifier_s' => '2' }, 'path/to/record2.json']  # aardvark schema
+    ]
+  end
+
+  before do
+    allow(solr).to receive(:update)
+    allow(solr).to receive(:commit)
+  end
+
+  describe '#initialize' do
+    before do
+      stub_const('ENV', 'SOLR_URL' => 'http://localhost:8983/solr/geoblacklight')
+      allow(RSolr).to receive(:connect).and_return(solr)
+    end
+
+    it 'connects to a solr instance if set in the environment' do
+      described_class.new
+      expect(RSolr).to have_received(:connect).with(
+        url: 'http://localhost:8983/solr/geoblacklight',
+        adapter: :net_http_persistent
+      )
+    end
+  end
+
+  describe '#index' do
+    it 'posts each record to solr as JSON' do
+      indexer.index([docs[0]], commit_within: 1)
+      expect(solr).to have_received(:update).with(
+        params: { commitWithin: 1, overwrite: true },
+        data: [docs[0][0]].to_json,
+        headers: { 'Content-Type' => 'application/json' }
+      )
+    end
+
+    it 'prints the id and path of each record in debug mode' do
+      $DEBUG = true
+      expect { indexer.index([docs[0]]) }.to output("Indexing 1: path/to/record1.json\n").to_stdout
+      expect { indexer.index([docs[1]]) }.to output("Indexing 2: path/to/record2.json\n").to_stdout
+      $DEBUG = false
+    end
+
+    it 'commits changes to solr after indexing' do
+      indexer.index(docs)
+      expect(solr).to have_received(:commit).once
+    end
+
+    it 'returns the count of records successfully indexed' do
+      expect(indexer.index(docs)).to eq 2
+    end
+  end
+end

data/spec/lib/geo_combine/migrators/v1_aardvark_migrator_spec.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe GeoCombine::Migrators::V1AardvarkMigrator do
+  include JsonDocs
+
+  describe '#run' do
+    it 'migrates keys' do
+      input_hash = JSON.parse(full_geoblacklight)
+      # TODO: Note that this fixture has not yet been fully converted to
+      # aardvark. See https://github.com/OpenGeoMetadata/GeoCombine/issues/121
+      # for remaining work.
+      expected_output = JSON.parse(full_geoblacklight_aardvark)
+      expect(described_class.new(v1_hash: input_hash).run).to eq(expected_output)
+    end
+
+    context 'when the given record is already in aardvark schema' do
+      xit 'returns the record unchanged'
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -7,7 +7,6 @@ SimpleCov.start 'rails' do
   add_filter 'lib/tasks/geo_combine.rake'
   add_filter 'lib/geo_combine/version.rb'
   add_filter 'lib/geo_combine/railtie.rb'
-  add_filter 'lib/geo_combine/cli.rb'
   minimum_coverage 95 # When updating this value, update the README badge value
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: geo_combine
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jack Reed
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-07 00:00:00.000000000 Z
+date: 2023-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -237,8 +251,7 @@ dependencies:
 description: A Ruby toolkit for managing geospatial metadata
 email:
 - pjreed@stanford.edu
-executables:
-- geocombine
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -251,12 +264,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/geocombine
 - geo_combine.gemspec
 - lib/geo_combine.rb
 - lib/geo_combine/bounding_box.rb
 - lib/geo_combine/ckan_metadata.rb
-- lib/geo_combine/cli.rb
 - lib/geo_combine/esri_open_data.rb
 - lib/geo_combine/exceptions.rb
 - lib/geo_combine/fgdc.rb
@@ -265,7 +276,10 @@ files:
 - lib/geo_combine/geo_blacklight_harvester.rb
 - lib/geo_combine/geoblacklight.rb
 - lib/geo_combine/geometry_types.rb
+- lib/geo_combine/harvester.rb
+- lib/geo_combine/indexer.rb
 - lib/geo_combine/iso19139.rb
+- lib/geo_combine/migrators/v1_aardvark_migrator.rb
 - lib/geo_combine/ogp.rb
 - lib/geo_combine/railtie.rb
 - lib/geo_combine/subjects.rb
@@ -294,6 +308,7 @@ files:
 - spec/fixtures/docs/ckan.json
 - spec/fixtures/docs/esri_open_data.json
 - spec/fixtures/docs/full_geoblacklight.json
+- spec/fixtures/docs/full_geoblacklight_aardvark.json
 - spec/fixtures/docs/geoblacklight_pre_v1.json
 - spec/fixtures/docs/ogp_harvard_line.json
 - spec/fixtures/docs/ogp_harvard_raster.json
@@ -304,6 +319,7 @@ files:
 - spec/fixtures/docs/simple_xslt.xsl
 - spec/fixtures/docs/stanford_iso.xml
 - spec/fixtures/docs/tufts_fgdc.xml
+- spec/fixtures/indexing/aardvark.json
 - spec/fixtures/indexing/basic_geoblacklight.json
 - spec/fixtures/indexing/geoblacklight.json
 - spec/fixtures/indexing/layers.json
@@ -318,10 +334,12 @@ files:
 - spec/lib/geo_combine/formatting_spec.rb
 - spec/lib/geo_combine/geo_blacklight_harvester_spec.rb
 - spec/lib/geo_combine/geoblacklight_spec.rb
+- spec/lib/geo_combine/harvester_spec.rb
+- spec/lib/geo_combine/indexer_spec.rb
 - spec/lib/geo_combine/iso19139_spec.rb
+- spec/lib/geo_combine/migrators/v1_aardvark_migrator_spec.rb
 - spec/lib/geo_combine/ogp_spec.rb
 - spec/lib/geo_combine_spec.rb
-- spec/lib/tasks/geo_combine_spec.rb
 - spec/spec_helper.rb
 - spec/support/fixtures.rb
 homepage: ''
@@ -343,7 +361,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: A Ruby toolkit for managing geospatial metadata
@@ -354,6 +372,7 @@ test_files:
 - spec/fixtures/docs/ckan.json
 - spec/fixtures/docs/esri_open_data.json
 - spec/fixtures/docs/full_geoblacklight.json
+- spec/fixtures/docs/full_geoblacklight_aardvark.json
 - spec/fixtures/docs/geoblacklight_pre_v1.json
 - spec/fixtures/docs/ogp_harvard_line.json
 - spec/fixtures/docs/ogp_harvard_raster.json
@@ -364,6 +383,7 @@ test_files:
 - spec/fixtures/docs/simple_xslt.xsl
 - spec/fixtures/docs/stanford_iso.xml
 - spec/fixtures/docs/tufts_fgdc.xml
+- spec/fixtures/indexing/aardvark.json
 - spec/fixtures/indexing/basic_geoblacklight.json
 - spec/fixtures/indexing/geoblacklight.json
 - spec/fixtures/indexing/layers.json
@@ -378,9 +398,11 @@ test_files:
 - spec/lib/geo_combine/formatting_spec.rb
 - spec/lib/geo_combine/geo_blacklight_harvester_spec.rb
 - spec/lib/geo_combine/geoblacklight_spec.rb
+- spec/lib/geo_combine/harvester_spec.rb
+- spec/lib/geo_combine/indexer_spec.rb
 - spec/lib/geo_combine/iso19139_spec.rb
+- spec/lib/geo_combine/migrators/v1_aardvark_migrator_spec.rb
 - spec/lib/geo_combine/ogp_spec.rb
 - spec/lib/geo_combine_spec.rb
-- spec/lib/tasks/geo_combine_spec.rb
 - spec/spec_helper.rb
 - spec/support/fixtures.rb

data/bin/geocombine

@@ -1,6 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'geo_combine/cli'
-
-GeoCombine::CLI.start(ARGV)

data/lib/geo_combine/cli.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require 'thor'
-require 'rake'
-
-root = Gem::Specification.find_by_name('geo_combine').gem_dir
-tasks = File.join(root, 'lib/tasks/*.rake')
-Dir.glob(tasks).each { |r| load r }
-
-module GeoCombine
-  class CLI < Thor
-    desc 'clone', 'Clone all OpenGeoMetadata repositories'
-    def clone
-      Rake::Task['geocombine:clone'].invoke
-    end
-
-    desc 'pull', '"git pull" OpenGeoMetadata repositories'
-    def pull
-      Rake::Task['geocombine:pull'].invoke
-    end
-
-    desc 'index', 'Index all of the GeoBlacklight documents'
-    def index
-      Rake::Task['geocombine:index'].invoke
-    end
-  end
-end

data/spec/lib/tasks/geo_combine_spec.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require 'rake'
-
-describe 'geo_combine.rake' do
-  before(:all) do
-    load File.expand_path('../../../lib/tasks/geo_combine.rake', __dir__)
-    Rake::Task.define_task(:environment)
-  end
-
-  before do
-    allow(ENV).to receive(:[]).and_call_original
-    allow(ENV).to receive(:[]).with('OGM_PATH').and_return(File.join(fixture_dir, 'indexing'))
-  end
-
-  describe 'geocombine:clone' do
-    before do
-      WebMock.disable_net_connect!
-    end
-
-    after do
-      WebMock.allow_net_connect!
-    end
-
-    it 'does not clone repos on deny list' do
-      stub_request(:get, 'https://api.github.com/orgs/opengeometadata/repos').to_return(status: 200, body: read_fixture('docs/repos.json'))
-      allow(Kernel).to receive(:system)
-      Rake::Task['geocombine:clone'].invoke
-      expect(Kernel).to have_received(:system).exactly(21).times
-    end
-  end
-
-  describe 'geocombine:index' do
-    it 'only indexes .json files but not layers.json' do
-      rsolr_mock = instance_double(RSolr::Client)
-      allow(rsolr_mock).to receive(:update)
-      allow(rsolr_mock).to receive(:commit)
-      allow(RSolr).to receive(:connect).and_return(rsolr_mock)
-      Rake::Task['geocombine:index'].invoke
-      # We expect 2 files to index
-      expect(rsolr_mock).to have_received(:update).exactly(2).times
-    end
-  end
-end