geo_combine 0.8.0 → 0.9.0

data/lib/geo_combine/indexer.rb

@@ -1,47 +1,126 @@
 # frozen_string_literal: true
 
 require 'rsolr'
+require 'faraday/retry'
 require 'faraday/net_http_persistent'
+require 'geo_combine/logger'
 
 module GeoCombine
   # Indexes Geoblacklight documents into Solr
   class Indexer
     attr_reader :solr
 
-    def self.solr(url: ENV.fetch('SOLR_URL', 'http://127.0.0.1:8983/solr/blacklight-core'))
-      RSolr.connect url: url, adapter: :net_http_persistent
+    def initialize(solr: nil, logger: GeoCombine::Logger.logger)
+      @logger = logger
+      @batch_size = ENV.fetch('SOLR_BATCH_SIZE', 100).to_i
+
+      # If SOLR_URL is set, use it; if in a Geoblacklight app, use its solr core
+      solr_url = ENV.fetch('SOLR_URL', nil)
+      solr_url ||= Blacklight.default_index.connection.base_uri.to_s if defined? Blacklight
+
+      # If neither, warn and try to use local Blacklight default solr core
+      if solr_url.nil?
+        @logger.warn 'SOLR_URL not set; using Blacklight default'
+        solr_url = 'http://localhost:8983/solr/blacklight-core'
+      end
+
+      @solr = solr || RSolr.connect(client, url: solr_url)
     end
 
-    def initialize(solr: GeoCombine::Indexer.solr)
-      @solr = solr
+    # Index everything and return the number of docs successfully indexed
+    def index(docs)
+      # Track total indexed and time spent
+      @logger.info "indexing into #{solr_url}"
+      total_indexed = 0
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      # Index in batches; set batch size via BATCH_SIZE
+      batch = []
+      docs.each do |doc, path|
+        if batch.size < @batch_size
+          batch << [doc, path]
+        else
+          total_indexed += index_batch(batch)
+          batch = []
+        end
+      end
+      total_indexed += index_batch(batch) unless batch.empty?
+
+      # Issue a commit to make sure all documents are indexed
+      @solr.commit
+      end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      sec = end_time - start_time
+      @logger.info format('indexed %<total_indexed>d documents in %<sec>.2f seconds', total_indexed:, sec:)
+      total_indexed
     end
 
+    # URL to the solr instance being used
     def solr_url
       @solr.options[:url]
     end
 
-    # Index everything and return the number of docs successfully indexed
-    def index(docs, commit_within: ENV.fetch('SOLR_COMMIT_WITHIN', 5000).to_i)
-      indexed_count = 0
-
-      docs.each do |record, path|
-        # log the unique identifier for the record for debugging
-        id = record['id'] || record['dc_identifier_s']
-        puts "Indexing #{id}: #{path}" if $DEBUG
-
-        # index the record into solr
-        @solr.update params: { commitWithin: commit_within, overwrite: true },
-                     data: [record].to_json,
-                     headers: { 'Content-Type' => 'application/json' }
-
-        # count the number of records successfully indexed
-        indexed_count += 1
-      rescue RSolr::Error::Http => e
-        puts e
-      end
+    private
 
-      @solr.commit
-      indexed_count
+    # Index a batch of documents; if we fail, index them all individually
+    def index_batch(batch)
+      docs = batch.map(&:first)
+      @solr.update(data: batch_json(docs), params:, headers:)
+      @logger.debug "indexed batch (#{batch.size} docs)"
+      batch.size
+    rescue RSolr::Error::Http => e
+      @logger.error "error indexing batch (#{batch.size} docs): #{format_error(e)}"
+      @logger.warn 'retrying documents individually'
+      batch.map { |doc, path| index_single(doc, path) }.compact.size
+    end
+
+    # Index a single document; if it fails, log the error and continue
+    def index_single(doc, path)
+      @solr.add(doc, params:, headers:)
+      @logger.debug "indexed #{path}"
+      doc
+    rescue RSolr::Error::Http => e
+      @logger.error "error indexing #{path}: #{format_error(e)}"
+      nil
+    end
+
+    # Generate a JSON string to send to solr update API for a batch of documents
+    def batch_json(batch)
+      batch.map { |doc| "add: { doc: #{doc.to_json} }" }.join(",\n").prepend('{ ').concat(' }')
+    end
+
+    # Generate a friendly error message for logging including status code and message
+    def format_error(error)
+      code = error.response[:status]
+      status_info = "#{code} #{RSolr::Error::Http::STATUS_CODES[code.to_i]}"
+      error_info = parse_solr_error(error)
+      [status_info, error_info].compact.join(' - ')
+    end
+
+    # Extract the specific error message from a solr JSON error response, if any
+    def parse_solr_error(error)
+      JSON.parse(error.response[:body]).dig('error', 'msg')
+    rescue StandardError
+      nil
+    end
+
+    def headers
+      { 'Content-Type' => 'application/json' }
+    end
+
+    def params
+      { overwrite: true }
+    end
+
+    def client
+      @client ||= Faraday.new do |conn|
+        conn.request :retry, max: 3, interval: 1, backoff_factor: 2, exceptions: [
+          Faraday::TimeoutError,
+          Faraday::ConnectionFailed,
+          Faraday::TooManyRequestsError
+        ]
+        conn.response :raise_error
+        conn.adapter :net_http_persistent
+      end
     end
   end
 end

data/lib/geo_combine/logger.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module GeoCombine
+  # Logger for gem
+  class Logger
+    def self.logger
+      @logger ||= ::Logger.new(
+        $stderr,
+        progname: 'GeoCombine',
+        level: ENV.fetch('LOG_LEVEL', 'info').to_sym
+      )
+    end
+  end
+end

data/lib/geo_combine/migrators/v1_aardvark_migrator.rb

@@ -1,41 +1,86 @@
 # frozen_string_literal: true
 
+require 'active_support'
+
 module GeoCombine
   module Migrators
-    # TODO: WARNING! This class is not fully implemented and should not be used in
-    # production. See https://github.com/OpenGeoMetadata/GeoCombine/issues/121
-    # for remaining work.
-    #
     # migrates the v1 schema to the aardvark schema
     class V1AardvarkMigrator
       attr_reader :v1_hash
 
       # @param v1_hash [Hash] parsed json in the v1 schema
-      def initialize(v1_hash:)
+      # @param collection_id_map [Hash] a hash mapping collection names to ids for converting dct_isPartOf_sm
+      def initialize(v1_hash:, collection_id_map: {})
         @v1_hash = v1_hash
+        @v2_hash = v1_hash
+        @collection_id_map = collection_id_map
       end
 
       def run
-        v2_hash = convert_keys
-        v2_hash['gbl_mdVersion_s'] = 'Aardvark'
-        v2_hash
+        # Return unchanged if already in the aardvark schema
+        return @v2_hash if @v2_hash['gbl_mdVersion_s'] == 'Aardvark'
+
+        # Convert the record
+        convert_keys
+        convert_single_to_multi_valued_fields
+        convert_non_crosswalked_fields
+        remove_deprecated_fields
+
+        # Mark the record as converted and return it
+        @v2_hash['gbl_mdVersion_s'] = 'Aardvark'
+        @v2_hash
       end
 
+      # Namespace and URI changes to fields
       def convert_keys
-        v1_hash.transform_keys do |k|
+        @v2_hash.transform_keys! do |k|
           SCHEMA_FIELD_MAP[k] || k
         end
       end
 
+      # Fields that need to be converted from single to multi-valued
+      def convert_single_to_multi_valued_fields
+        @v2_hash = @v2_hash.each_with_object({}) do |(k, v), h|
+          h[k] = if !v.is_a?(Array) && k.match?(/.*_[s|i]m/)
+                   [v]
+                 else
+                   v
+                 end
+        end
+      end
+
+      # Convert non-crosswalked fields via lookup tables
+      def convert_non_crosswalked_fields
+        # Keys may or may not include whitespace, so we normalize them.
+        # Resource class is required so we default to "Other"; resource type is not required.
+        @v2_hash['gbl_resourceClass_sm'] = RESOURCE_CLASS_MAP[@v1_hash['dc_type_s']&.gsub(/\s+/, '')] || ['Other']
+        resource_type = RESOURCE_TYPE_MAP[@v1_hash['layer_geom_type_s']&.gsub(/\s+/, '')]
+        @v2_hash['gbl_resourceType_sm'] = resource_type unless resource_type.nil?
+
+        # If the user specified a collection id map, use it to convert the collection names to ids
+        is_part_of = @v1_hash['dct_isPartOf_sm']&.map { |name| @collection_id_map[name] }&.compact
+        if is_part_of.present?
+          @v2_hash['dct_isPartOf_sm'] = is_part_of
+        else
+          @v2_hash.delete('dct_isPartOf_sm')
+        end
+      end
+
+      # Remove fields that are no longer used
+      def remove_deprecated_fields
+        @v2_hash = @v2_hash.except(*SCHEMA_FIELD_MAP.keys, 'dc_type_s', 'layer_geom_type_s')
+      end
+
       SCHEMA_FIELD_MAP = {
         'dc_title_s' => 'dct_title_s', # new namespace
         'dc_description_s' => 'dct_description_sm', # new namespace; single to multi-valued
         'dc_language_s' => 'dct_language_sm', # new namespace; single to multi-valued
-        'dc_language_sm' => 'dct_language_sm', # new namespace; single to multi-valued
+        'dc_language_sm' => 'dct_language_sm', # new namespace
         'dc_creator_sm' => 'dct_creator_sm', # new namespace
         'dc_publisher_s' => 'dct_publisher_sm', # new namespace; single to multi-valued
         'dct_provenance_s' => 'schema_provider_s', # new URI name
         'dc_subject_sm' => 'dct_subject_sm', # new namespace
+        'solr_geom' => 'dcat_bbox', # new URI name
         'solr_year_i' => 'gbl_indexYear_im', # new URI name; single to multi-valued
         'dc_source_sm' => 'dct_source_sm', # new namespace
         'dc_rights_s' => 'dct_accessRights_s', # new URI name
@@ -47,6 +92,27 @@ module GeoCombine
         'geoblacklight_version' => 'gbl_mdVersion_s', # new URI name
         'suppressed_b' => 'gbl_suppressed_b' # new namespace
       }.freeze
+
+      # Map Dublin Core types to Aardvark resource class sets
+      # See: https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/ogm-aardvark/resource-class.md
+      RESOURCE_CLASS_MAP = {
+        'Collection' => ['Collections'],
+        'Dataset' => ['Datasets'],
+        'Image' => ['Imagery'],
+        'InteractiveResource' => ['Websites'],
+        'Service' => ['Web services'],
+        'StillImage' => ['Imagery']
+      }.freeze
+
+      # Map geometry types to Aardvark resource type sets
+      # See: https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/ogm-aardvark/resource-type.md
+      RESOURCE_TYPE_MAP = {
+        'Point' => ['Point data'],
+        'Line' => ['Line data'],
+        'Polygon' => ['Polygon data'],
+        'Raster' => ['Raster data'],
+        'Table' => ['Table data']
+      }.freeze
     end
   end
 end

data/lib/geo_combine/ogp.rb

@@ -75,7 +75,7 @@ module GeoCombine
         dc_publisher_s: metadata['Publisher'],
         dc_subject_sm: subjects,
         dc_type_s: 'Dataset'
-      }.delete_if { |_k, v| v.nil? }
+      }.compact
     end
 
     def date

data/lib/geo_combine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GeoCombine
-  VERSION = '0.8.0'
+  VERSION = '0.9.0'
 end

data/lib/tasks/geo_combine.rake

@@ -12,24 +12,20 @@ namespace :geocombine do
   desc 'Clone OpenGeoMetadata repositories'
   task :clone, [:repo] do |_t, args|
     harvester = GeoCombine::Harvester.new
-    total = args[:repo] ? harvester.clone(args.repo) : harvester.clone_all
-    puts "Cloned #{total} repositories"
+    args[:repo] ? harvester.clone(args.repo) : harvester.clone_all
   end
 
   desc '"git pull" OpenGeoMetadata repositories'
   task :pull, [:repo] do |_t, args|
     harvester = GeoCombine::Harvester.new
-    total = args[:repo] ? harvester.pull(args.repo) : harvester.pull_all
-    puts "Updated #{total} repositories"
+    args[:repo] ? harvester.pull(args.repo) : harvester.pull_all
   end
 
   desc 'Index all JSON documents except Layers.json'
   task :index do
     harvester = GeoCombine::Harvester.new
     indexer = GeoCombine::Indexer.new
-    puts "Indexing #{harvester.ogm_path} into #{indexer.solr_url}"
-    total = indexer.index(harvester.docs_to_index)
-    puts "Indexed #{total} documents"
+    indexer.index(harvester.docs_to_index)
   end
 
   namespace :geoblacklight_harvester do

data/spec/fixtures/docs/full_geoblacklight.json

@@ -28,6 +28,13 @@
   "dct_spatial_sm":[
     "Uganda"
   ],
+  "dct_isPartOf_sm":[
+    "Uganda GIS Maps and Data, 2000-2010"
+  ],
+  "dc_source_sm": [
+    "stanford-rb371kw9607"
+  ],
   "solr_geom":"ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)",
-  "solr_year_i":2005
+  "solr_year_i":2005,
+  "suppressed_b":false
 }

data/spec/fixtures/docs/full_geoblacklight_aardvark.json

@@ -1,19 +1,31 @@
 {
   "gbl_mdVersion_s":"Aardvark",
-  "dct_identifier_sm":"http://purl.stanford.edu/cz128vq0535",
+  "dct_identifier_sm":[
+    "http://purl.stanford.edu/cz128vq0535"
+  ],
   "dct_title_s":"2005 Rural Poverty GIS Database: Uganda",
-  "dct_description_sm":"This polygon shapefile contains 2005 poverty data for 855 rural subcounties in Uganda. These data are intended for researchers, students, policy makers and the general public for reference and mapping purposes, and may be used for basic applications such as viewing, querying, and map output production.",
+  "dct_description_sm":[
+    "This polygon shapefile contains 2005 poverty data for 855 rural subcounties in Uganda. These data are intended for researchers, students, policy makers and the general public for reference and mapping purposes, and may be used for basic applications such as viewing, querying, and map output production."
+  ],
   "dct_accessRights_s":"Public",
   "schema_provider_s":"Stanford",
   "dct_references_s":"{\"http://schema.org/url\":\"http://purl.stanford.edu/cz128vq0535\",\"http://schema.org/downloadUrl\":\"http://stacks.stanford.edu/file/druid:cz128vq0535/data.zip\",\"http://www.loc.gov/mods/v3\":\"http://purl.stanford.edu/cz128vq0535.mods\",\"http://www.isotc211.org/schemas/2005/gmd/\":\"http://opengeometadata.stanford.edu/metadata/edu.stanford.purl/druid:cz128vq0535/iso19139.xml\",\"http://www.w3.org/1999/xhtml\":\"http://opengeometadata.stanford.edu/metadata/edu.stanford.purl/druid:cz128vq0535/default.html\",\"http://www.opengis.net/def/serviceType/ogc/wfs\":\"https://geowebservices.stanford.edu/geoserver/wfs\",\"http://www.opengis.net/def/serviceType/ogc/wms\":\"https://geowebservices.stanford.edu/geoserver/wms\"}",
   "gbl_wxsIdentifier_s":"druid:cz128vq0535",
   "id":"stanford-cz128vq0535",
-  "layer_geom_type_s":"Polygon",
+  "gbl_resourceType_sm": [
+    "Polygon data"
+  ],
   "gbl_mdModified_dt":"2015-01-13T18:46:38Z",
   "dct_format_s":"Shapefile",
-  "dct_language_sm":"English",
-  "dc_type_s":"Dataset",
-  "dct_publisher_sm":"Uganda Bureau of Statistics",
+  "dct_language_sm":[
+    "English"
+  ],
+  "gbl_resourceClass_sm":[
+    "Datasets"
+  ],
+  "dct_publisher_sm":[
+    "Uganda Bureau of Statistics"
+  ],
   "dct_creator_sm":[
     "Uganda Bureau of Statistics"
   ],
@@ -28,6 +40,12 @@
   "dct_spatial_sm":[
     "Uganda"
   ],
-  "solr_geom":"ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)",
-  "gbl_indexYear_im":2005
+  "dct_source_sm": [
+    "stanford-rb371kw9607"
+  ],
+  "dcat_bbox":"ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)",
+  "gbl_indexYear_im":[
+    2005
+  ],
+  "gbl_suppressed_b":false
 }

data/spec/lib/geo_combine/geo_blacklight_harvester_spec.rb

@@ -5,8 +5,9 @@ require 'spec_helper'
 require 'rsolr'
 
 RSpec.describe GeoCombine::GeoBlacklightHarvester do
-  subject(:harvester) { described_class.new(site_key) }
+  subject(:harvester) { described_class.new(site_key, logger:) }
 
+  let(:logger) { instance_double(Logger, warn: nil, info: nil, error: nil, debug: nil) }
   let(:site_key) { :INSTITUTION }
   let(:stub_json_response) { '{}' }
   let(:stub_solr_connection) { double('RSolr::Connection') }
@@ -40,7 +41,7 @@ RSpec.describe GeoCombine::GeoBlacklightHarvester do
 
     let(:docs) { [{ layer_slug_s: 'abc-123' }, { layer_slug_s: 'abc-321' }] }
     let(:stub_json_response) do
-      { response: { docs: docs, pages: { current_page: 1, total_pages: 1 } } }.to_json
+      { response: { docs:, pages: { current_page: 1, total_pages: 1 } } }.to_json
     end
 
     it 'adds documents returned to solr' do
@@ -142,7 +143,7 @@ RSpec.describe GeoCombine::GeoBlacklightHarvester do
         ).and_return(stub_second_response.to_json)
         base_url = 'https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100'
         docs = described_class::LegacyBlacklightResponse.new(response: stub_first_response,
-                                                             base_url: base_url).documents
+                                                             base_url:).documents
 
         expect(docs.to_a).to eq([first_docs, second_docs])
       end
@@ -182,7 +183,7 @@ RSpec.describe GeoCombine::GeoBlacklightHarvester do
 
         base_url = 'https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100'
         docs = described_class::ModernBlacklightResponse.new(response: first_results_response,
-                                                             base_url: base_url).documents
+                                                             base_url:).documents
 
         expect(docs.to_a).to eq([
                                   [{ 'layer_slug_s' => 'abc-123' }, { 'layer_slug_s' => 'abc-321' }],

data/spec/lib/geo_combine/harvester_spec.rb

@@ -5,8 +5,9 @@ require 'geo_combine/harvester'
 require 'spec_helper'
 
 RSpec.describe GeoCombine::Harvester do
-  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing') }
+  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', logger:) }
 
+  let(:logger) { instance_double(Logger, warn: nil, info: nil, error: nil, debug: nil) }
   let(:repo_name) { 'my-institution' }
   let(:repo_path) { File.join(harvester.ogm_path, repo_name) }
   let(:repo_url) { "https://github.com/OpenGeoMetadata/#{repo_name}.git" }
@@ -34,8 +35,7 @@ RSpec.describe GeoCombine::Harvester do
     end
 
     # stub git commands
-    allow(Git).to receive(:open).and_return(stub_repo)
-    allow(Git).to receive(:clone).and_return(stub_repo)
+    allow(Git).to receive_messages(open: stub_repo, clone: stub_repo)
     allow(stub_repo).to receive(:pull).and_return(stub_repo)
   end
 
@@ -48,7 +48,7 @@ RSpec.describe GeoCombine::Harvester do
     end
 
     it 'skips records with a different schema version' do
-      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: 'Aardvark')
+      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: 'Aardvark', logger:)
       expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
         [JSON.parse(File.read('spec/fixtures/indexing/aardvark.json')), 'spec/fixtures/indexing/aardvark.json']
       )
@@ -75,8 +75,8 @@ RSpec.describe GeoCombine::Harvester do
       expect(stub_repo).to have_received(:pull).exactly(2).times
     end
 
-    it 'returns the count of repositories pulled' do
-      expect(harvester.pull_all).to eq(2)
+    it 'returns the names of repositories pulled' do
+      expect(harvester.pull_all).to eq(%w[my-institution another-institution])
     end
 
     it 'skips repositories in the denylist' do
@@ -107,20 +107,6 @@ RSpec.describe GeoCombine::Harvester do
       harvester.clone(repo_name)
       expect(Git).not_to have_received(:clone)
     end
-
-    it 'warns if a repository is empty' do
-      allow(Net::HTTP).to receive(:get).with('https://api.github.com/repos/opengeometadata/empty').and_return('{"size": 0}')
-      expect do
-        harvester.clone('empty')
-      end.to output(/repository 'empty' is empty/).to_stdout
-    end
-
-    it 'warns if a repository is archived' do
-      allow(Net::HTTP).to receive(:get).with('https://api.github.com/repos/opengeometadata/empty').and_return('{"archived": true}')
-      expect do
-        harvester.clone('outdated-institution')
-      end.to output(/repository 'outdated-institution' is archived/).to_stdout
-    end
   end
 
   describe '#clone_all' do
@@ -134,8 +120,8 @@ RSpec.describe GeoCombine::Harvester do
       expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/aardvark.git')
     end
 
-    it 'returns the count of repositories cloned' do
-      expect(harvester.clone_all).to eq(2)
+    it 'returns the names of repositories cloned' do
+      expect(harvester.clone_all).to eq(%w[my-institution another-institution])
     end
   end
 

data/spec/lib/geo_combine/indexer_spec.rb

@@ -3,10 +3,22 @@
 require 'geo_combine/indexer'
 require 'spec_helper'
 
+# Mock an available Blacklight installation
+class FakeBlacklight
+  def self.default_index
+    Repository
+  end
+
+  class Repository
+    def self.connection; end
+  end
+end
+
 RSpec.describe GeoCombine::Indexer do
-  subject(:indexer) { described_class.new(solr: solr) }
+  subject(:indexer) { described_class.new(solr:, logger:) }
 
-  let(:solr) { instance_double(RSolr::Client) }
+  let(:logger) { instance_double(Logger, warn: nil, info: nil, error: nil, debug: nil) }
+  let(:solr) { instance_double(RSolr::Client, options: { url: 'TEST' }) }
   let(:docs) do
     [
       [{ 'id' => '1' }, 'path/to/record1.json'],              # v1.0 schema
@@ -21,36 +33,69 @@ RSpec.describe GeoCombine::Indexer do
 
   describe '#initialize' do
     before do
-      stub_const('ENV', 'SOLR_URL' => 'http://localhost:8983/solr/geoblacklight')
       allow(RSolr).to receive(:connect).and_return(solr)
     end
 
-    it 'connects to a solr instance if set in the environment' do
-      described_class.new
-      expect(RSolr).to have_received(:connect).with(
-        url: 'http://localhost:8983/solr/geoblacklight',
-        adapter: :net_http_persistent
-      )
+    context 'when solr url is set in the environment' do
+      before do
+        stub_const('ENV', 'SOLR_URL' => 'http://localhost:8983/solr/geoblacklight')
+      end
+
+      it 'connects to the solr instance' do
+        described_class.new(logger:)
+        expect(RSolr).to have_received(:connect).with(
+          be_a(Faraday::Connection),
+          url: 'http://localhost:8983/solr/geoblacklight'
+        )
+      end
+    end
+
+    context 'when there is a configured Blacklight connection' do
+      before do
+        stub_const('Blacklight', FakeBlacklight)
+        allow(FakeBlacklight::Repository).to receive(:connection).and_return(
+          instance_double(RSolr::Client, base_uri: URI('http://localhost:8983/solr/geoblacklight'))
+        )
+      end
+
+      it 'connects to the solr instance' do
+        described_class.new(logger:)
+        expect(RSolr).to have_received(:connect).with(
+          be_a(Faraday::Connection),
+          url: 'http://localhost:8983/solr/geoblacklight'
+        )
+      end
+    end
+
+    context 'when solr url is not set' do
+      before do
+        stub_const('ENV', {})
+      end
+
+      it 'falls back to the Blacklight default' do
+        described_class.new(logger:)
+        expect(RSolr).to have_received(:connect).with(
+          be_a(Faraday::Connection),
+          url: 'http://localhost:8983/solr/blacklight-core'
+        )
+      end
     end
   end
 
   describe '#index' do
-    it 'posts each record to solr as JSON' do
-      indexer.index([docs[0]], commit_within: 1)
+    let(:solr_error_msg) {  { error: { msg: 'error message' } }.to_json }
+    let(:solr_response) { { status: '400', body: solr_error_msg } }
+    let(:error) { RSolr::Error::Http.new({ uri: URI('') }, solr_response) }
+
+    it 'sends records in batches to solr' do
+      indexer.index(docs)
       expect(solr).to have_received(:update).with(
-        params: { commitWithin: 1, overwrite: true },
-        data: [docs[0][0]].to_json,
-        headers: { 'Content-Type' => 'application/json' }
+        data: "{ add: { doc: {\"id\":\"1\"} },\nadd: { doc: {\"dc_identifier_s\":\"2\"} } }",
+        headers: { 'Content-Type' => 'application/json' },
+        params: { overwrite: true }
       )
     end
 
-    it 'prints the id and path of each record in debug mode' do
-      $DEBUG = true
-      expect { indexer.index([docs[0]]) }.to output("Indexing 1: path/to/record1.json\n").to_stdout
-      expect { indexer.index([docs[1]]) }.to output("Indexing 2: path/to/record2.json\n").to_stdout
-      $DEBUG = false
-    end
-
     it 'commits changes to solr after indexing' do
       indexer.index(docs)
       expect(solr).to have_received(:commit).once
@@ -59,5 +104,31 @@ RSpec.describe GeoCombine::Indexer do
     it 'returns the count of records successfully indexed' do
       expect(indexer.index(docs)).to eq 2
     end
+
+    context 'when an error occurs during batch indexing' do
+      before do
+        allow(solr).to receive(:update).and_raise(error)
+        allow(solr).to receive(:add)
+      end
+
+      it 'attempts to index records individually' do
+        total = indexer.index(docs)
+        expect(solr).to have_received(:add).twice
+        expect(total).to eq 2
+      end
+    end
+
+    context 'when an error occurs during individual indexing' do
+      before do
+        allow(solr).to receive(:update).and_raise(error)
+        allow(solr).to receive(:add).with(docs[0][0], anything).and_raise(error)
+        allow(solr).to receive(:add).with(docs[1][0], anything)
+      end
+
+      it 'continues indexing' do
+        total = indexer.index(docs)
+        expect(total).to eq 1
+      end
+    end
   end
 end