bulkrax 5.2.1 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a811bef32bb83948c7ea8fe9cbc3aead7e9fa3e09f88fe6d3ec45e4cdef7461
-  data.tar.gz: 253314c9d1a35505b50ad70e6fb0f9dec0f5331a55cdee6555cb137fff697e08
+  metadata.gz: 1bdc3299c331e345417117694e5da25031db99e181701d6b52db630b5c297470
+  data.tar.gz: 105119be729e2ea30a5fceb5b2f47557d5edb933b05663b7905e49665ceb3dfa
 SHA512:
-  metadata.gz: 3c8281a0c12778d8db9a6e3e8dc39a9c591e2d81e9d668b9b52da400b65e48012f40b51b86aa75182fc26b7bf8d20e761af7a00b6c1c798df34dba35da7890fc
-  data.tar.gz: d71d16a09cfa1d9b0c3954bf2be9f2778bacdaa98e0a99d301c0b0762d8be04b0213f99463e05e6aab33321b64a9284551b32a1435e97569097e32c23e74f95d
+  metadata.gz: 027d2ea6005df040daebf48bf431e471b1386141e2e13ae5de149f25125f8d2306576accb8300869ab16fb500f5f2b4d3f88ea28e299648bf588bf5f4d9c8264
+  data.tar.gz: a3924a66c3ca3d8e3a243573cbb44ca41f11f687ca02f17bbfad6403883a516b917248cf0f807088e351cc206a6b18d99d9434643ccebebd06dbb126319c9445

data/README.md

@@ -17,6 +17,7 @@ And then execute:
 ```bash
 $ bundle install
 $ rails generate bulkrax:install
+$ rails db:migrate
 ```
 
 If using Sidekiq, set up queues for `import` and `export`.
@@ -32,6 +33,7 @@ gem 'bulkrax'
 And then execute:
 ```bash
 $ bundle install
+$ rails db:migrate
 ```
 
 Mount the engine in your routes file
@@ -61,7 +63,7 @@ If using Sidekiq, set up queues for `import` and `export`.
 *= require 'bulkrax/application'
 ```
 
-You'll want to add an intializer to configure the importer to your needs:
+You'll want to add an initializer to configure the importer to your needs:
 
 ```ruby
 # config/initializers/bulkrax.rb
@@ -112,13 +114,13 @@ An Import needs to know what Work Type to create. The importer looks for:
 
 If it does not find either of these, or the data they contain is not a valid Work Type in the repository, the `default_work_type` will be used.
 
-The install generator sets `default_work_type` to the first Work Type returned by `Hyrax.config.curation_concerns` but this can be overriden by setting `default_work_type` in `config/initializer/bulkrax.rb` as shown above.
+The install generator sets `default_work_type` to the first Work Type returned by `Hyrax.config.curation_concerns` (stringified), but this can be overwritten by setting `default_work_type` in `config/initializer/bulkrax.rb` as shown above.
 
 ## Configuring Field Mapping
 
 It's unlikely that the incoming import data has fields that exactly match those in your repository. Field mappings allow you to tell bulkrax how to map field in the incoming data to a field in your application.
 
-By default, a mapping for the OAI parser has been added to map standard oai_dc fields to Hyrax basic_metadata. The other parsers have no default mapping, and will map any incoming fields to Hyrax properties with the same name. Configurations can be added in `config/intializers/bulkrax.rb`
+By default, a mapping for the OAI parser has been added to map standard oai_dc fields to Hyrax basic_metadata. The other parsers have no default mapping, and will map any incoming fields to Hyrax properties with the same name. Configurations can be added in `config/initializers/bulkrax.rb`
 
 Configuring field mappings is documented in the [Bulkrax Configuration Guide](https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax).
 
@@ -176,7 +178,7 @@ To edit an importer or exporter, select the edit icon (pencil) and complete the
 To delete an importer or exporter, select the delete (x) icon.
 
 ### Downloading an export
-Once your the exporter has run, a download icon will apear on the exporters menu page.
+Once your the exporter has run, a download icon will appear on the exporters menu page.
 
 ## Contributing
 If you're working on a PR for this project, create a feature branch off of `main`.

data/app/assets/javascripts/bulkrax/navtabs.js.erb

@@ -0,0 +1,9 @@
+<% unless defined?(::Hyku) %>
+  // enables the tabs in the importers/exporters pages.
+  $(document).ready(function() {
+    $('.nav-tabs a').click(function (e) {
+      e.preventDefault();
+      $(this).tab('show');
+    });
+  });
+<% end %>

data/app/controllers/bulkrax/exporters_controller.rb

@@ -13,7 +13,8 @@ module Bulkrax
 
     # GET /exporters
     def index
-      @exporters = Exporter.all
+      # NOTE: We're paginating this in the browser.
+      @exporters = Exporter.order(created_at: :desc).all
 
       add_exporter_breadcrumbs if defined?(::Hyrax)
     end

data/app/controllers/bulkrax/importers_controller.rb

@@ -20,7 +20,8 @@ module Bulkrax
 
     # GET /importers
     def index
-      @importers = Importer.all
+      # NOTE: We're paginating this in the browser.
+      @importers = Importer.order(created_at: :desc).all
       if api_request?
         json_response('index')
       elsif defined?(::Hyrax)

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -40,7 +40,7 @@ module Bulkrax
 
     include DynamicRecordLookup
 
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
     # @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
@@ -61,38 +61,82 @@ module Bulkrax
       number_of_successes = 0
       number_of_failures = 0
       errors = []
-
-      ActiveRecord::Base.uncached do
-        Bulkrax::PendingRelationship.where(parent_id: parent_identifier, importer_run_id: importer_run_id)
-                                    .ordered.find_each do |rel|
-          process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
-          number_of_successes += 1
-        rescue => e
-          number_of_failures += 1
-          errors << e
+      @parent_record_members_added = false
+      @child_members_added = []
+
+      if parent_record
+        conditionally_acquire_lock_for(parent_record.id) do
+          ActiveRecord::Base.uncached do
+            Bulkrax::PendingRelationship.where(parent_id: parent_identifier, importer_run_id: importer_run_id)
+                                        .ordered.find_each do |rel|
+              process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
+              number_of_successes += 1
+            rescue => e
+              number_of_failures += 1
+              errors << e
+            end
+          end
+
+          # save record if members were added
+          if @parent_record_members_added
+            parent_record.save!
+            # Ensure that the new relationship gets indexed onto the children
+            @child_members_added.each(&:update_index)
+          end
         end
+      else
+        # In moving the check of the parent record "up" we've exposed a hidden reporting foible.
+        # Namely we were reporting one error per child record when the parent record was itself
+        # unavailable.
+        #
+        # We have chosen not to duplicate that "number of errors" as it does not seem like the
+        # correct pattern for reporting a singular error (the previous pattern being one error per
+        # child who's parent is not yet created).
+        number_of_failures = 1
+        errors = ["Parent record not yet available for creating relationships with children records."]
       end
 
-      # save record if members were added
-      parent_record.save! if @parent_record_members_added
-
-      # rubocop:disable Rails/SkipsModelValidations
       if errors.present?
+        # rubocop:disable Rails/SkipsModelValidations
         importer_run.increment!(:failed_relationships, number_of_failures)
+        # rubocop:enable Rails/SkipsModelValidations
+
         parent_entry&.set_status_info(errors.last, importer_run)
 
         # TODO: This can create an infinite job cycle, consider a time to live tracker.
         reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
       else
+        # rubocop:disable Rails/SkipsModelValidations
         Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships, number_of_successes)
+        # rubocop:enable Rails/SkipsModelValidations
       end
-      # rubocop:enable Rails/SkipsModelValidations
     end
     # rubocop:enable Metrics/MethodLength
 
     private
 
+    ##
+    # We can use Hyrax's lock manager when we have one available.
+    if defined?(::Hyrax)
+      include Hyrax::Lockable
+
+      def conditionally_acquire_lock_for(*args, &block)
+        if Bulkrax.use_locking?
+          acquire_lock_for(*args, &block)
+        else
+          yield
+        end
+      end
+    else
+      # Otherwise, we're providing no meaningful lock manager at this time.
+      def acquire_lock_for(*)
+        yield
+      end
+
+      alias conditionally_acquire_lock_for acquire_lock_for
+    end
+
     def process(relationship:, importer_run_id:, parent_record:, ability:)
       raise "#{relationship} needs a child to create relationship" if relationship.child_id.nil?
       raise "#{relationship} needs a parent to create relationship" if relationship.parent_id.nil?
@@ -124,8 +168,7 @@ module Bulkrax
 
       parent_record.ordered_members << child_record
       @parent_record_members_added = true
-      # TODO: Do we need to save the child record?
-      child_record.save!
+      @child_members_added << child_record
     end
 
     def reschedule(parent_identifier:, importer_run_id:)

data/app/jobs/bulkrax/delete_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class DeleteJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # rubocop:disable Rails/SkipsModelValidations
     def perform(entry, importer_run)

data/app/jobs/bulkrax/download_cloud_file_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class DownloadCloudFileJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # Retrieve cloud file and write to the imports directory
     # Note: if using the file system, the mounted directory in

data/app/jobs/bulkrax/import_collection_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImportCollectionJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # rubocop:disable Rails/SkipsModelValidations
     def perform(*args)

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -6,7 +6,7 @@ module Bulkrax
   class ImportFileSetJob < ApplicationJob
     include DynamicRecordLookup
 
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     attr_reader :importer_run_id
 

data/app/jobs/bulkrax/import_work_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImportWorkJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # rubocop:disable Rails/SkipsModelValidations
     #

data/app/jobs/bulkrax/importer_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImporterJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     def perform(importer_id, only_updates_since_last_import = false)
       importer = Importer.find(importer_id)

data/app/matchers/bulkrax/application_matcher.rb

@@ -26,7 +26,8 @@ module Bulkrax
 
       # @result will evaluate to an empty string for nil content values
       @result = content.to_s.gsub(/\s/, ' ').strip # remove any line feeds and tabs
-      process_split if @result.present?
+      # blank needs to be based to split, only skip nil
+      process_split unless @result.nil?
       @result = @result[0] if @result.is_a?(Array) && @result.size == 1
       process_parse
       return @result
@@ -36,8 +37,8 @@ module Bulkrax
       if self.split.is_a?(TrueClass)
         @result = @result.split(Bulkrax.multi_value_element_split_on)
       elsif self.split
-        result = @result.split(Regexp.new(self.split))
-        @result = result.map(&:strip)
+        @result = @result.split(Regexp.new(self.split))
+        @result = @result.map(&:strip).select(&:present?)
       end
     end
 

data/app/models/bulkrax/csv_entry.rb

@@ -20,7 +20,7 @@ module Bulkrax
       raise StandardError, 'CSV path empty' if path.blank?
       options = {
         headers: true,
-        header_converters: ->(h) { h.to_sym },
+        header_converters: ->(h) { h.to_s.strip.to_sym },
         encoding: 'utf-8'
       }.merge(csv_read_data_options)
 
@@ -243,20 +243,17 @@ module Bulkrax
       object_metadata(Array.wrap(data))
     end
 
-    def build_value(key, value)
-      return unless hyrax_record.respond_to?(key.to_s)
+    def build_value(property_name, mapping_config)
+      return unless hyrax_record.respond_to?(property_name.to_s)
 
-      data = hyrax_record.send(key.to_s)
-      if data.is_a?(ActiveTriples::Relation)
-        if value['join']
-          self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
-        else
-          data.each_with_index do |d, i|
-            self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
-          end
-        end
+      data = hyrax_record.send(property_name.to_s)
+
+      if mapping_config['join'] || !data.is_a?(Enumerable)
+        self.parsed_metadata[key_for_export(property_name)] = prepare_export_data_with_join(data)
       else
-        self.parsed_metadata[key_for_export(key)] = prepare_export_data(data)
+        data.each_with_index do |d, i|
+          self.parsed_metadata["#{key_for_export(property_name)}_#{i + 1}"] = prepare_export_data(d)
+        end
       end
     end
 
@@ -269,6 +266,14 @@ module Bulkrax
       "#{unnumbered_key}#{key.sub(clean_key, '')}"
     end
 
+    def prepare_export_data_with_join(data)
+      # Yes...it's possible we're asking to coerce a multi-value but only have a single value.
+      return data.to_s unless data.is_a?(Enumerable)
+      return "" if data.empty?
+
+      data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
+    end
+
     def prepare_export_data(datum)
       if datum.is_a?(ActiveTriples::Resource)
         datum.to_uri.to_s

data/app/models/bulkrax/importer.rb

@@ -24,13 +24,12 @@ module Bulkrax
     attr_writer :current_run
 
     def self.safe_uri_filename(uri)
-      uri = URI.parse(uri) unless uri.is_a?(URI)
       r = Faraday.head(uri.to_s)
       return CGI.parse(r.headers['content-disposition'])["filename"][0].delete("\"")
     rescue
-      filename = File.basename(uri.path)
+      filename = File.basename(uri.to_s)
       filename.delete!('/')
-      filename.presence || file_set.id
+      filename.presence || SecureRandom.uuid
     end
 
     def status

data/app/parsers/bulkrax/csv_parser.rb

@@ -33,9 +33,9 @@ module Bulkrax
           model_field_mappings.map(&:to_sym).each do |model_mapping|
             next unless r.key?(model_mapping)
 
-            if r[model_mapping].casecmp('collection').zero?
+            if r[model_mapping].strip.casecmp('collection').zero?
               @collections << r
-            elsif r[model_mapping].casecmp('fileset').zero?
+            elsif r[model_mapping].strip.casecmp('fileset').zero?
               @file_sets << r
             else
               @works << r

data/app/parsers/bulkrax/parser_export_record_set.rb

@@ -21,6 +21,30 @@ module Bulkrax
       "Bulkrax::ParserExportRecordSet::#{export_from.classify}".constantize.new(parser: parser)
     end
 
+    SOLR_QUERY_PAGE_SIZE = 512
+
+    ##
+    # A helper method for handling querying large batches of IDs.  By default SOLR has a max of 1024
+    # `OR` clauses per query.  This method helps chunk large sets of IDs into batches.
+    #
+    # @param array [Array<Object>]
+    # @param page_size [Integer]
+    # @yieldparam [Array<Object>] slice of the original arrays which are yielded.  The results of
+    #             the yield are merged into the return value.
+    #
+    # @return [Array<Object>]
+    #
+    # @see https://github.com/samvera-labs/bulkrax/issues/776
+    def self.in_batches(array, page_size: SOLR_QUERY_PAGE_SIZE)
+      array = Array.wrap(array)
+      return [] if array.empty?
+      results = []
+      array.each_slice(page_size) do |slice|
+        results += Array.wrap(yield(slice))
+      end
+      results
+    end
+
     # @abstract
     #
     # @note This has {#each} and {#count} but is not an Enumerable.  But because it has these two
@@ -36,6 +60,7 @@ module Bulkrax
       delegate :limit_reached?, :work_entry_class, :collection_entry_class, :file_set_entry_class, :importerexporter, to: :parser
       private :limit_reached?, :work_entry_class, :collection_entry_class, :file_set_entry_class, :importerexporter
 
+      ##
       # @return [Integer]
       def count
         sum = works.count + collections.count + file_sets.count
@@ -44,6 +69,7 @@ module Bulkrax
         return sum
       end
 
+      ##
       # Yield first the works, then collections, then file sets.  Once we've yielded as many times
       # as the parser's limit, we break the iteration and return.
       #
@@ -134,8 +160,6 @@ module Bulkrax
                          end
       end
 
-      SOLR_QUERY_PAGE_SIZE = 512
-
       # @note In most cases, when we don't have any candidate file sets, there is no need to query SOLR.
       #
       # @see Bulkrax::ParserExportRecordSet::Importer#file_sets
@@ -148,20 +172,14 @@ module Bulkrax
       # @see https://github.com/scientist-softserv/britishlibrary/issues/289
       # @see https://github.com/samvera/hyrax/blob/64c0bbf0dc0d3e1b49f040b50ea70d177cc9d8f6/app/indexers/hyrax/work_indexer.rb#L15-L18
       def file_sets
-        @file_sets ||= if candidate_file_set_ids.empty?
-                         []
-                       else
-                         results = []
-                         candidate_file_set_ids.each_slice(SOLR_QUERY_PAGE_SIZE) do |ids|
-                           fsq = "has_model_ssim:#{Bulkrax.file_model_class} AND id:(\"" + ids.join('" OR "') + "\")"
-                           fsq += extra_filters if extra_filters.present?
-                           results += ActiveFedora::SolrService.query(
-                             fsq,
-                             { fl: "id", method: :post, rows: ids.size }
-                           )
-                         end
-                         results
-                       end
+        @file_sets ||= ParserExportRecordSet.in_batches(candidate_file_set_ids) do |batch_of_ids|
+          fsq = "has_model_ssim:#{Bulkrax.file_model_class} AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
+          fsq += extra_filters if extra_filters.present?
+          ActiveFedora::SolrService.query(
+            fsq,
+            { fl: "id", method: :post, rows: batch_of_ids.size }
+          )
+        end
       end
 
       def solr_name(base_name)
@@ -227,32 +245,34 @@ module Bulkrax
           end
       end
 
-      def works_query_kwargs
-        query_kwargs.merge(
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
-          ],
-          fl: 'id'
-        )
-      end
-
-      def works_query
-        extra_filters.to_s
-      end
-
-      def collections_query_kwargs
-        query_kwargs.merge(
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:Collection"
-          ],
-          fl: 'id'
-        )
+      def works
+        @works ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
+          ActiveFedora::SolrService.query(
+            extra_filters.to_s,
+            **query_kwargs.merge(
+              fq: [
+                %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
+                "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
+              ],
+              fl: 'id'
+            )
+          )
+        end
       end
 
-      def collections_query
-        "has_model_ssim:Collection #{extra_filters}"
+      def collections
+        @collections ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
+          ActiveFedora::SolrService.query(
+            "has_model_ssim:Collection #{extra_filters}",
+            **query_kwargs.merge(
+              fq: [
+                %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
+                "has_model_ssim:Collection"
+              ],
+              fl: "id"
+            )
+          )
+        end
       end
 
       # This is an exception; we don't know how many candidate file sets there might be.  So we will instead
@@ -260,21 +280,18 @@ module Bulkrax
       #
       # @see Bulkrax::ParserExportRecordSet::Base#file_sets
       def file_sets
-        @file_sets ||= ActiveFedora::SolrService.query(file_sets_query, **file_sets_query_kwargs)
-      end
-
-      def file_sets_query_kwargs
-        query_kwargs.merge(
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:#{Bulkrax.file_model_class}"
-          ],
-          fl: 'id'
-        )
-      end
-
-      def file_sets_query
-        extra_filters
+        @file_sets ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
+          ActiveFedora::SolrService.query(
+            extra_filters,
+            query_kwargs.merge(
+              fq: [
+                %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
+                "has_model_ssim:#{Bulkrax.file_model_class}"
+              ],
+              fl: 'id'
+            )
+          )
+        end
       end
     end
   end

data/app/views/bulkrax/importers/edit.html.erb

@@ -10,7 +10,9 @@
         <%= render 'form', importer: @importer, form: form %>
         <div class="panel-footer">
           <div class='pull-right'>
-            <%= link_to 'Update Importer', '#bulkraxModal', class: "btn btn-primary", data: { toggle: 'modal' } %>
+            <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#bulkraxModal">
+              Update Importer
+            </button>
             <%= render 'edit_form_buttons', form: form %>
            <% cancel_path = form.object.persisted? ? importer_path(form.object) : importers_path %>
             | <%= link_to t('.cancel'), cancel_path, class: 'btn btn-default ' %>
@@ -19,4 +21,4 @@
       <% end %>
     </div>
   </div>
-</div>
+</div>

data/app/views/bulkrax/importers/new.html.erb

@@ -9,8 +9,10 @@
         <%= render 'form', importer: @importer, form: form %>
         <div class="panel-footer">
           <div class='pull-right'>
-            <%= form.button :submit, value: 'Create and Validate', class: 'btn btn-primary' %>
-            |
+            <% if ENV['SHOW_CREATE_AND_VALIDATE'] == 'true' %>
+              <%= form.button :submit, value: 'Create and Validate', class: 'btn btn-primary' %>
+              |
+            <% end  %>
             <%= form.button :submit, value: 'Create and Import', class: 'btn btn-primary' %>
             |
             <%= form.button :submit, value: 'Create', class: 'btn btn-primary' %>

data/app/views/hyrax/dashboard/sidebar/_bulkrax_sidebar_additions.html.erb

@@ -1,10 +1,12 @@
 <% if current_ability.can_import_works? %>
-  <%= menu.nav_link(bulkrax.importers_path) do %>
+  <%= menu.nav_link(bulkrax.importers_path,
+                    title: t('bulkrax.admin.sidebar.importers')) do %>
     <span class="fa fa-cloud-upload" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.importers') %></span>
   <% end %>
 <% end %>
 <% if current_ability.can_export_works? %>
-  <%= menu.nav_link(bulkrax.exporters_path) do %>
+  <%= menu.nav_link(bulkrax.exporters_path,
+                    title: t('bulkrax.admin.sidebar.exporters')) do %>
     <span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
   <% end %>
 <% end %>

data/app/views/hyrax/dashboard/sidebar/_repository_content.html.erb

@@ -3,14 +3,16 @@
 <%= menu.nav_link(hyrax.my_collections_path,
                   class: "nav-link",
                   onclick: "dontChangeAccordion(event);",
-                  also_active_for: hyrax.dashboard_collections_path) do %>
+                  also_active_for: hyrax.dashboard_collections_path,
+                  title: t('hyrax.admin.sidebar.collections')) do %>
   <span class="fa fa-folder-open" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.collections') %></span>
 <% end %>
 
 <%= menu.nav_link(hyrax.my_works_path,
                   class: "nav-link",
                   onclick: "dontChangeAccordion(event);",
-                  also_active_for: hyrax.dashboard_works_path) do %>
+                  also_active_for: hyrax.dashboard_works_path,
+                  title: t('hyrax.admin.sidebar.works')) do %>
   <span class="fa fa-file" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.works') %></span>
 <% end %>
 

data/db/migrate/20230608153601_add_indices_to_bulkrax.rb

@@ -0,0 +1,14 @@
+class AddIndicesToBulkrax < ActiveRecord::Migration[5.1]
+  def change
+    add_index :bulkrax_entries, :identifier
+    add_index :bulkrax_entries, :type
+    add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx'
+
+    add_index :bulkrax_pending_relationships, :parent_id
+    add_index :bulkrax_pending_relationships, :child_id
+
+    add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx'
+    add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx'
+    add_index :bulkrax_statuses, :error_class
+  end
+end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '5.2.1'
+  VERSION = '5.3.1'
 end

data/lib/bulkrax.rb

@@ -1,16 +1,16 @@
-# frozen_string_literal: true
+  # frozen_string_literal: true
 
-require "bulkrax/version"
-require "bulkrax/engine"
-require 'active_support/all'
+    require "bulkrax/version"
+    require "bulkrax/engine"
+    require 'active_support/all'
 
-# rubocop:disable Metrics/ModuleLength
+    # rubocop:disable Metrics/ModuleLength
 module Bulkrax
-  extend self # rubocop:disable Style/ModuleFunction
-  extend Forwardable
+    extend self # rubocop:disable Style/ModuleFunction
+    extend Forwardable
 
-  ##
-  # @api public
+    ##
+    # @api public
   class Configuration
     attr_accessor :api_definition,
                   :curation_concerns,
@@ -34,60 +34,81 @@ module Bulkrax
                   :required_elements,
                   :reserved_properties,
                   :server_name
-  end
 
-  def config
-    @config ||= Configuration.new
-    yield @config if block_given?
-    @config
+    attr_writer :ingest_queue_name
+    ##
+    # @return [String, Proc]
+    def ingest_queue_name
+      return @ingest_queue_name if @ingest_queue_name.present?
+      return Hyrax.config.ingest_queue_name if defined?(Hyrax)
+      :import
+    end
+
+    attr_writer :use_locking
+
+    def use_locking
+      return @use_locking if defined?(@use_locking)
+
+      ENV.key?("REDIS_HOST")
+    end
+    alias use_locking? use_locking
   end
-  alias setup config
 
-  def_delegators :@config,
-                 :api_definition,
-                 :api_definition=,
-                 :curation_concerns,
-                 :curation_concerns=,
-                 :default_field_mapping,
-                 :default_field_mapping=,
-                 :default_work_type,
-                 :default_work_type=,
-                 :export_path,
-                 :export_path=,
-                 :field_mappings,
-                 :field_mappings=,
-                 :file_model_class,
-                 :file_model_class=,
-                 :fill_in_blank_source_identifiers,
-                 :fill_in_blank_source_identifiers=,
-                 :generated_metadata_mapping,
-                 :generated_metadata_mapping=,
-                 :import_path,
-                 :import_path=,
-                 :multi_value_element_join_on,
-                 :multi_value_element_join_on=,
-                 :multi_value_element_split_on,
-                 :multi_value_element_split_on=,
-                 :object_factory,
-                 :object_factory=,
-                 :parsers,
-                 :parsers=,
-                 :qa_controlled_properties,
-                 :qa_controlled_properties=,
-                 :related_children_field_mapping,
-                 :related_children_field_mapping=,
-                 :related_parents_field_mapping,
-                 :related_parents_field_mapping=,
-                 :relationship_job_class,
-                 :relationship_job_class=,
-                 :removed_image_path,
-                 :removed_image_path=,
-                 :required_elements,
-                 :required_elements=,
-                 :reserved_properties,
-                 :reserved_properties=,
-                 :server_name,
-                 :server_name=
+    def config
+      @config ||= Configuration.new
+      yield @config if block_given?
+      @config
+    end
+    alias setup config
+
+    def_delegators :@config,
+                   :api_definition,
+                   :api_definition=,
+                   :curation_concerns,
+                   :curation_concerns=,
+      :default_field_mapping,
+      :default_field_mapping=,
+      :default_work_type,
+      :default_work_type=,
+      :export_path,
+      :export_path=,
+      :field_mappings,
+      :field_mappings=,
+      :file_model_class,
+      :file_model_class=,
+      :fill_in_blank_source_identifiers,
+      :fill_in_blank_source_identifiers=,
+      :generated_metadata_mapping,
+      :generated_metadata_mapping=,
+      :import_path,
+      :import_path=,
+      :multi_value_element_join_on,
+      :multi_value_element_join_on=,
+      :multi_value_element_split_on,
+      :multi_value_element_split_on=,
+      :object_factory,
+      :object_factory=,
+      :parsers,
+      :parsers=,
+      :qa_controlled_properties,
+      :qa_controlled_properties=,
+      :related_children_field_mapping,
+      :related_children_field_mapping=,
+      :related_parents_field_mapping,
+      :related_parents_field_mapping=,
+      :relationship_job_class,
+      :relationship_job_class=,
+      :removed_image_path,
+      :removed_image_path=,
+      :required_elements,
+      :required_elements=,
+      :reserved_properties,
+      :reserved_properties=,
+      :server_name,
+      :server_name=,
+      :use_locking,
+      :use_locking=,
+    :use_locking?
 
   config do |conf|
     conf.parsers = [
@@ -126,149 +147,149 @@ module Bulkrax
     # Based on Hyrax CoreMetadata && BasicMetadata
     # Override at application level to change
     conf.field_mappings = {
-      "Bulkrax::OaiDcParser" => {
-        "contributor" => { from: ["contributor"] },
-        # no appropriate mapping for coverage (based_near needs id)
-        #  ""=>{:from=>["coverage"]},
-        "creator" => { from: ["creator"] },
-        "date_created" => { from: ["date"] },
-        "description" => { from: ["description"] },
-        # no appropriate mapping for format
-        # ""=>{:from=>["format"]},
-        "identifier" => { from: ["identifier"] },
-        "language" => { from: ["language"], parsed: true },
-        "publisher" => { from: ["publisher"] },
-        "related_url" => { from: ["relation"] },
-        "rights_statement" => { from: ["rights"] },
-        "source" => { from: ["source"] },
-        "subject" => { from: ["subject"], parsed: true },
-        "title" => { from: ["title"] },
-        "resource_type" => { from: ["type"], parsed: true },
-        "remote_files" => { from: ["thumbnail_url"], parsed: true }
+    "Bulkrax::OaiDcParser" => {
+    "contributor" => { from: ["contributor"] },
+    # no appropriate mapping for coverage (based_near needs id)
+    #  ""=>{:from=>["coverage"]},
+    "creator" => { from: ["creator"] },
+    "date_created" => { from: ["date"] },
+    "description" => { from: ["description"] },
+    # no appropriate mapping for format
+    # ""=>{:from=>["format"]},
+    "identifier" => { from: ["identifier"] },
+    "language" => { from: ["language"], parsed: true },
+    "publisher" => { from: ["publisher"] },
+    "related_url" => { from: ["relation"] },
+    "rights_statement" => { from: ["rights"] },
+    "source" => { from: ["source"] },
+    "subject" => { from: ["subject"], parsed: true },
+    "title" => { from: ["title"] },
+    "resource_type" => { from: ["type"], parsed: true },
+    "remote_files" => { from: ["thumbnail_url"], parsed: true }
       },
-      "Bulkrax::OaiQualifiedDcParser" => {
-        "abstract" => { from: ["abstract"] },
-        "alternative_title" => { from: ["alternative"] },
+                                                         "Bulkrax::OaiQualifiedDcParser" => {
+    "abstract" => { from: ["abstract"] },
+      "alternative_title" => { from: ["alternative"] },
         "bibliographic_citation" => { from: ["bibliographicCitation"] },
-        "contributor" => { from: ["contributor"] },
-        "creator" => { from: ["creator"] },
-        "date_created" => { from: ["created"] },
-        "description" => { from: ["description"] },
-        "language" => { from: ["language"] },
-        "license" => { from: ["license"] },
-        "publisher" => { from: ["publisher"] },
-        "related_url" => { from: ["relation"] },
-        "rights_holder" => { from: ["rightsHolder"] },
-        "rights_statement" => { from: ["rights"] },
-        "source" => { from: ["source"] },
-        "subject" => { from: ["subject"], parsed: true },
-        "title" => { from: ["title"] },
-        "resource_type" => { from: ["type"], parsed: true },
-        "remote_files" => { from: ["thumbnail_url"], parsed: true }
-      },
-      # When empty, a default_field_mapping will be generated
-      "Bulkrax::CsvParser" => {},
-      'Bulkrax::BagitParser' => {},
-      'Bulkrax::XmlParser' => {}
-    }
+          "contributor" => { from: ["contributor"] },
+            "creator" => { from: ["creator"] },
+              "date_created" => { from: ["created"] },
+                "description" => { from: ["description"] },
+                  "language" => { from: ["language"] },
+                    "license" => { from: ["license"] },
+                      "publisher" => { from: ["publisher"] },
+                        "related_url" => { from: ["relation"] },
+                          "rights_holder" => { from: ["rightsHolder"] },
+                            "rights_statement" => { from: ["rights"] },
+                              "source" => { from: ["source"] },
+                                "subject" => { from: ["subject"], parsed: true },
+                                  "title" => { from: ["title"] },
+                                    "resource_type" => { from: ["type"], parsed: true },
+                                      "remote_files" => { from: ["thumbnail_url"], parsed: true }
+                                        },
+                                        # When empty, a default_field_mapping will be generated
+                                        "Bulkrax::CsvParser" => {},
+                                          'Bulkrax::BagitParser' => {},
+                                          'Bulkrax::XmlParser' => {}
+                                          }
 
-    # Lambda to set the default field mapping
-    conf.default_field_mapping = lambda do |field|
-      return if field.blank?
-      {
-        field.to_s =>
-        {
-          from: [field.to_s],
-          split: false,
-          parsed: Bulkrax::ApplicationMatcher.method_defined?("parse_#{field}"),
-          if: nil,
-          excluded: false
-        }
-      }
-    end
+                                          # Lambda to set the default field mapping
+                                          conf.default_field_mapping = lambda do |field|
+                                          return if field.blank?
+                                          {
+                                          field.to_s =>
+                                          {
+                                          from: [field.to_s],
+                                          split: false,
+                                          parsed: Bulkrax::ApplicationMatcher.method_defined?("parse_#{field}"),
+                                            if: nil,
+                                            excluded: false
+                                            }
+                                            }
+                                            end
 
-    # Properties that should not be used in imports. They are reserved for use by Hyrax.
-    conf.reserved_properties = %w[
-      create_date
-      modified_date
-      date_modified
-      date_uploaded
-      depositor
-      arkivo_checksum
-      has_model
-      head
-      label
-      import_url
-      on_behalf_of
-      proxy_depositor
-      owner
-      state
-      tail
-      original_url
-      relative_path
-    ]
+                                            # Properties that should not be used in imports. They are reserved for use by Hyrax.
+                                            conf.reserved_properties = %w[
+                                            create_date
+                                            modified_date
+                                            date_modified
+                                            date_uploaded
+                                            depositor
+                                            arkivo_checksum
+                                            has_model
+                                            head
+                                            label
+                                            import_url
+                                            on_behalf_of
+                                            proxy_depositor
+                                            owner
+                                            state
+                                            tail
+                                            original_url
+                                            relative_path
+                                            ]
 
-    # List of Questioning Authority properties that are controlled via YAML files in
-    # the config/authorities/ directory. For example, the :rights_statement property
-    # is controlled by the active terms in config/authorities/rights_statements.yml
-    conf.qa_controlled_properties = %w[rights_statement license]
-  end
+                                            # List of Questioning Authority properties that are controlled via YAML files in
+                                            # the config/authorities/ directory. For example, the :rights_statement property
+                                            # is controlled by the active terms in config/authorities/rights_statements.yml
+                                            conf.qa_controlled_properties = %w[rights_statement license]
+                                            end
 
-  def api_definition
-    @api_definition ||= ActiveSupport::HashWithIndifferentAccess.new(
-      YAML.safe_load(
-        ERB.new(
-          File.read(Rails.root.join('config', 'bulkrax_api.yml'))
-        ).result
-      )
-    )
-  end
+                                            def api_definition
+                                            @api_definition ||= ActiveSupport::HashWithIndifferentAccess.new(
+                                            YAML.safe_load(
+                                            ERB.new(
+                                            File.read(Rails.root.join('config', 'bulkrax_api.yml'))
+                                            ).result
+                                            )
+                                            )
+                                            end
 
-  DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON = ' | '
-  # Specify the delimiter for joining an attribute's multi-value array into a string.
-  #
-  # @note the specific delimiter should likely be present in the multi_value_element_split_on
-  #       expression.
-  def multi_value_element_join_on
-    @multi_value_element_join_on ||= DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
-  end
+                                            DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON = ' | '
+                                            # Specify the delimiter for joining an attribute's multi-value array into a string.
+                                            #
+                                            # @note the specific delimiter should likely be present in the multi_value_element_split_on
+                                            #       expression.
+                                            def multi_value_element_join_on
+                                            @multi_value_element_join_on ||= DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
+                                            end
 
-  DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON = /\s*[:;|]\s*/.freeze
-  # @return [RegexClass] the regular express to use to "split" an attribute's values.  If set to
-  # `true` use the DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON.
-  #
-  # @note The "true" value is to preserve backwards compatibility.
-  # @see DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
+                                            DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON = /\s*[:;|]\s*/.freeze
+                                              # @return [RegexClass] the regular express to use to "split" an attribute's values.  If set to
+                                              # `true` use the DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON.
+                                              #
+                                              # @note The "true" value is to preserve backwards compatibility.
+                                              # @see DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
   def multi_value_element_split_on
-    if @multi_value_element_join_on.is_a?(TrueClass)
-      DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
+                                              if @multi_value_element_join_on.is_a?(TrueClass)
+    DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
     else
-      @multi_value_element_split_on ||= DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
+    @multi_value_element_split_on ||= DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
     end
   end
 
-  # Responsible for stripping hidden characters from the given string.
-  #
-  # @param value [#to_s]
-  # @return [String] with hidden characters removed
-  #
-  # @see https://github.com/samvera-labs/bulkrax/issues/688
-  def normalize_string(value)
-    # Removing [Byte Order Mark (BOM)](https://en.wikipedia.org/wiki/Byte_order_mark)
-    value.to_s.delete("\xEF\xBB\xBF")
-  end
+    # Responsible for stripping hidden characters from the given string.
+    #
+    # @param value [#to_s]
+    # @return [String] with hidden characters removed
+    #
+    # @see https://github.com/samvera-labs/bulkrax/issues/688
+    def normalize_string(value)
+      # Removing [Byte Order Mark (BOM)](https://en.wikipedia.org/wiki/Byte_order_mark)
+      value.to_s.delete("\xEF\xBB\xBF")
+    end
 
-  def fallback_user_for_importer_exporter_processing
-    return User.batch_user if defined?(Hyrax) && User.respond_to?(:batch_user)
+      def fallback_user_for_importer_exporter_processing
+        return User.batch_user if defined?(Hyrax) && User.respond_to?(:batch_user)
 
-    raise "We have no fallback user available for Bulkrax.fallback_user_for_importer_exporter_processing"
-  end
+                                 raise "We have no fallback user available for Bulkrax.fallback_user_for_importer_exporter_processing"
+                               end
 
-  # This class confirms to the Active::Support.serialize interface.  It's job is to ensure that we
-  # don't have keys with the tricksy Byte Order Mark character.
-  #
-  # @see https://api.rubyonrails.org/classes/ActiveRecord/AttributeMethods/Serialization/ClassMethods.html#method-i-serialize
-  class NormalizedJson
+      # This class confirms to the Active::Support.serialize interface.  It's job is to ensure that we
+      # don't have keys with the tricksy Byte Order Mark character.
+      #
+      # @see https://api.rubyonrails.org/classes/ActiveRecord/AttributeMethods/Serialization/ClassMethods.html#method-i-serialize
+      class NormalizedJson
     def self.normalize_keys(hash)
       return hash unless hash.respond_to?(:each_pair)
       returning_value = {}
@@ -280,18 +301,18 @@ module Bulkrax
 
     # When we write the serialized data to the database, we "dump" the value into that database
     # column.
-    def self.dump(value)
-      JSON.dump(normalize_keys(value))
-    end
+        def self.dump(value)
+          JSON.dump(normalize_keys(value))
+        end
 
     # When we load the serialized data from the database, we pass the database's value into "load"
     # function.
     #
     # rubocop:disable Security/JSONLoad
-    def self.load(string)
-      normalize_keys(JSON.load(string))
-    end
+        def self.load(string)
+          normalize_keys(JSON.load(string))
+        end
     # rubocop:enable Security/JSONLoad
-  end
+      end
 end
-# rubocop:disable Metrics/ModuleLength
+    # rubocop:disable Metrics/ModuleLength

data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -7,8 +7,8 @@ Bulkrax.setup do |config|
   # ]
 
   # WorkType to use as the default if none is specified in the import
-  # Default is the first returned by Hyrax.config.curation_concerns
-  # config.default_work_type = MyWork
+  # Default is the first returned by Hyrax.config.curation_concerns, stringified
+  # config.default_work_type = "MyWork"
 
   # Factory Class to use when generating and saving objects
   config.object_factory = Bulkrax::ObjectFactory

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.3.1
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-24 00:00:00.000000000 Z
+date: 2024-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-monads
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: iso8601
   requirement: !ruby/object:Gem::Requirement
@@ -238,6 +252,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: psych
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 description: Bulkrax is a batteries included importer for Samvera applications. It
   currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the
   box. It is also designed to be extensible, allowing you to easily add new importers
@@ -258,6 +300,7 @@ files:
 - app/assets/javascripts/bulkrax/entries.js
 - app/assets/javascripts/bulkrax/exporters.js
 - app/assets/javascripts/bulkrax/importers.js.erb
+- app/assets/javascripts/bulkrax/navtabs.js.erb
 - app/assets/stylesheets/bulkrax/accordion.scss
 - app/assets/stylesheets/bulkrax/application.css
 - app/assets/stylesheets/bulkrax/coderay.scss
@@ -395,6 +438,7 @@ files:
 - db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb
 - db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb
 - db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb
+- db/migrate/20230608153601_add_indices_to_bulkrax.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/entry_spec_helper.rb
@@ -427,7 +471,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku