bulkrax 5.2.1 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a811bef32bb83948c7ea8fe9cbc3aead7e9fa3e09f88fe6d3ec45e4cdef7461
-  data.tar.gz: 253314c9d1a35505b50ad70e6fb0f9dec0f5331a55cdee6555cb137fff697e08
+  metadata.gz: 56ac96f944d880492d7f7aa50870557ba81cf2ed23875751509b3ec23411644b
+  data.tar.gz: f18c9fa1d5a968d9214587476f2f949e04015e1402f6cb2ac516616669567c0e
 SHA512:
-  metadata.gz: 3c8281a0c12778d8db9a6e3e8dc39a9c591e2d81e9d668b9b52da400b65e48012f40b51b86aa75182fc26b7bf8d20e761af7a00b6c1c798df34dba35da7890fc
-  data.tar.gz: d71d16a09cfa1d9b0c3954bf2be9f2778bacdaa98e0a99d301c0b0762d8be04b0213f99463e05e6aab33321b64a9284551b32a1435e97569097e32c23e74f95d
+  metadata.gz: 0556a64e91eb41a27597b717fbf9e1de0682d666f116397e64a1b58a64fffaaadfd2dac645fbd256e3bd14ffb9065ce5f0ceaac920296f6623629199a6e130b5
+  data.tar.gz: dada612e0ee38d0b74ceacedc5c034cb39e26111edc90c39afd0d04338f77c1e0c0857a4d983bd7bd24fe8d7e601e2b8a6c32242e3142b2faad3a3369cbf82c1

data/README.md

@@ -17,6 +17,7 @@ And then execute:
 ```bash
 $ bundle install
 $ rails generate bulkrax:install
+$ rails db:migrate
 ```
 
 If using Sidekiq, set up queues for `import` and `export`.
@@ -32,6 +33,7 @@ gem 'bulkrax'
 And then execute:
 ```bash
 $ bundle install
+$ rails db:migrate
 ```
 
 Mount the engine in your routes file
@@ -61,7 +63,7 @@ If using Sidekiq, set up queues for `import` and `export`.
 *= require 'bulkrax/application'
 ```
 
-You'll want to add an intializer to configure the importer to your needs:
+You'll want to add an initializer to configure the importer to your needs:
 
 ```ruby
 # config/initializers/bulkrax.rb
@@ -112,13 +114,13 @@ An Import needs to know what Work Type to create. The importer looks for:
 
 If it does not find either of these, or the data they contain is not a valid Work Type in the repository, the `default_work_type` will be used.
 
-The install generator sets `default_work_type` to the first Work Type returned by `Hyrax.config.curation_concerns` but this can be overriden by setting `default_work_type` in `config/initializer/bulkrax.rb` as shown above.
+The install generator sets `default_work_type` to the first Work Type returned by `Hyrax.config.curation_concerns` (stringified), but this can be overwritten by setting `default_work_type` in `config/initializer/bulkrax.rb` as shown above.
 
 ## Configuring Field Mapping
 
 It's unlikely that the incoming import data has fields that exactly match those in your repository. Field mappings allow you to tell bulkrax how to map field in the incoming data to a field in your application.
 
-By default, a mapping for the OAI parser has been added to map standard oai_dc fields to Hyrax basic_metadata. The other parsers have no default mapping, and will map any incoming fields to Hyrax properties with the same name. Configurations can be added in `config/intializers/bulkrax.rb`
+By default, a mapping for the OAI parser has been added to map standard oai_dc fields to Hyrax basic_metadata. The other parsers have no default mapping, and will map any incoming fields to Hyrax properties with the same name. Configurations can be added in `config/initializers/bulkrax.rb`
 
 Configuring field mappings is documented in the [Bulkrax Configuration Guide](https://github.com/samvera-labs/bulkrax/wiki/Configuring-Bulkrax).
 
@@ -176,7 +178,7 @@ To edit an importer or exporter, select the edit icon (pencil) and complete the
 To delete an importer or exporter, select the delete (x) icon.
 
 ### Downloading an export
-Once your the exporter has run, a download icon will apear on the exporters menu page.
+Once your the exporter has run, a download icon will appear on the exporters menu page.
 
 ## Contributing
 If you're working on a PR for this project, create a feature branch off of `main`.

data/app/assets/javascripts/bulkrax/navtabs.js.erb

@@ -0,0 +1,9 @@
+<% unless defined?(::Hyku) %>
+  // enables the tabs in the importers/exporters pages.
+  $(document).ready(function() {
+    $('.nav-tabs a').click(function (e) {
+      e.preventDefault();
+      $(this).tab('show');
+    });
+  });
+<% end %>

data/app/controllers/bulkrax/exporters_controller.rb

@@ -13,7 +13,8 @@ module Bulkrax
 
     # GET /exporters
     def index
-      @exporters = Exporter.all
+      # NOTE: We're paginating this in the browser.
+      @exporters = Exporter.order(created_at: :desc).all
 
       add_exporter_breadcrumbs if defined?(::Hyrax)
     end

data/app/controllers/bulkrax/importers_controller.rb

@@ -20,7 +20,8 @@ module Bulkrax
 
     # GET /importers
     def index
-      @importers = Importer.all
+      # NOTE: We're paginating this in the browser.
+      @importers = Importer.order(created_at: :desc).all
       if api_request?
         json_response('index')
       elsif defined?(::Hyrax)

data/app/factories/bulkrax/object_factory.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class ObjectFactory
+  class ObjectFactory # rubocop:disable Metrics/ClassLength
     extend ActiveModel::Callbacks
     include Bulkrax::FileFactory
     include DynamicRecordLookup
@@ -87,7 +87,8 @@ module Bulkrax
     end
 
     def find
-      return find_by_id if attributes[:id].present?
+      found = find_by_id if attributes[:id].present?
+      return found if found.present?
       return search_by_identifier if attributes[work_identifier].present?
     end
 
@@ -102,7 +103,8 @@ module Bulkrax
     end
 
     def search_by_identifier
-      query = { work_identifier =>
+      work_index = ::ActiveFedora.index_field_mapper.solr_name(work_identifier, :facetable)
+      query = { work_index =>
                 source_identifier_value }
       # Query can return partial matches (something6 matches both something6 and something68)
       # so we need to weed out any that are not the correct full match. But other items might be

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -61,38 +61,82 @@ module Bulkrax
       number_of_successes = 0
       number_of_failures = 0
       errors = []
-
-      ActiveRecord::Base.uncached do
-        Bulkrax::PendingRelationship.where(parent_id: parent_identifier, importer_run_id: importer_run_id)
-                                    .ordered.find_each do |rel|
-          process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
-          number_of_successes += 1
-        rescue => e
-          number_of_failures += 1
-          errors << e
+      @parent_record_members_added = false
+      @child_members_added = []
+
+      if parent_record
+        conditionally_acquire_lock_for(parent_record.id) do
+          ActiveRecord::Base.uncached do
+            Bulkrax::PendingRelationship.where(parent_id: parent_identifier, importer_run_id: importer_run_id)
+                                        .ordered.find_each do |rel|
+              process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
+              number_of_successes += 1
+            rescue => e
+              number_of_failures += 1
+              errors << e
+            end
+          end
+
+          # save record if members were added
+          if @parent_record_members_added
+            parent_record.save!
+            # Ensure that the new relationship gets indexed onto the children
+            @child_members_added.each(&:update_index)
+          end
         end
+      else
+        # In moving the check of the parent record "up" we've exposed a hidden reporting foible.
+        # Namely we were reporting one error per child record when the parent record was itself
+        # unavailable.
+        #
+        # We have chosen not to duplicate that "number of errors" as it does not seem like the
+        # correct pattern for reporting a singular error (the previous pattern being one error per
+        # child who's parent is not yet created).
+        number_of_failures = 1
+        errors = ["Parent record not yet available for creating relationships with children records."]
       end
 
-      # save record if members were added
-      parent_record.save! if @parent_record_members_added
-
-      # rubocop:disable Rails/SkipsModelValidations
       if errors.present?
+        # rubocop:disable Rails/SkipsModelValidations
         importer_run.increment!(:failed_relationships, number_of_failures)
+        # rubocop:enable Rails/SkipsModelValidations
+
         parent_entry&.set_status_info(errors.last, importer_run)
 
         # TODO: This can create an infinite job cycle, consider a time to live tracker.
         reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
       else
+        # rubocop:disable Rails/SkipsModelValidations
         Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships, number_of_successes)
+        # rubocop:enable Rails/SkipsModelValidations
       end
-      # rubocop:enable Rails/SkipsModelValidations
     end
     # rubocop:enable Metrics/MethodLength
 
     private
 
+    ##
+    # We can use Hyrax's lock manager when we have one available.
+    if defined?(::Hyrax)
+      include Hyrax::Lockable
+
+      def conditionally_acquire_lock_for(*args, &block)
+        if Bulkrax.use_locking?
+          acquire_lock_for(*args, &block)
+        else
+          yield
+        end
+      end
+    else
+      # Otherwise, we're providing no meaningful lock manager at this time.
+      def acquire_lock_for(*)
+        yield
+      end
+
+      alias conditionally_acquire_lock_for acquire_lock_for
+    end
+
     def process(relationship:, importer_run_id:, parent_record:, ability:)
       raise "#{relationship} needs a child to create relationship" if relationship.child_id.nil?
       raise "#{relationship} needs a parent to create relationship" if relationship.parent_id.nil?
@@ -124,8 +168,7 @@ module Bulkrax
 
       parent_record.ordered_members << child_record
       @parent_record_members_added = true
-      # TODO: Do we need to save the child record?
-      child_record.save!
+      @child_members_added << child_record
     end
 
     def reschedule(parent_identifier:, importer_run_id:)

data/app/matchers/bulkrax/application_matcher.rb

@@ -26,7 +26,8 @@ module Bulkrax
 
       # @result will evaluate to an empty string for nil content values
       @result = content.to_s.gsub(/\s/, ' ').strip # remove any line feeds and tabs
-      process_split if @result.present?
+      # blank needs to be based to split, only skip nil
+      process_split unless @result.nil?
       @result = @result[0] if @result.is_a?(Array) && @result.size == 1
       process_parse
       return @result
@@ -36,8 +37,8 @@ module Bulkrax
       if self.split.is_a?(TrueClass)
         @result = @result.split(Bulkrax.multi_value_element_split_on)
       elsif self.split
-        result = @result.split(Regexp.new(self.split))
-        @result = result.map(&:strip)
+        @result = @result.split(Regexp.new(self.split))
+        @result = @result.map(&:strip).select(&:present?)
       end
     end
 

data/app/models/bulkrax/csv_entry.rb

@@ -20,7 +20,7 @@ module Bulkrax
       raise StandardError, 'CSV path empty' if path.blank?
       options = {
         headers: true,
-        header_converters: ->(h) { h.to_sym },
+        header_converters: ->(h) { h.to_s.strip.to_sym },
         encoding: 'utf-8'
       }.merge(csv_read_data_options)
 
@@ -243,20 +243,17 @@ module Bulkrax
       object_metadata(Array.wrap(data))
     end
 
-    def build_value(key, value)
-      return unless hyrax_record.respond_to?(key.to_s)
+    def build_value(property_name, mapping_config)
+      return unless hyrax_record.respond_to?(property_name.to_s)
 
-      data = hyrax_record.send(key.to_s)
-      if data.is_a?(ActiveTriples::Relation)
-        if value['join']
-          self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
-        else
-          data.each_with_index do |d, i|
-            self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
-          end
-        end
+      data = hyrax_record.send(property_name.to_s)
+
+      if mapping_config['join'] || !data.is_a?(Enumerable)
+        self.parsed_metadata[key_for_export(property_name)] = prepare_export_data_with_join(data)
       else
-        self.parsed_metadata[key_for_export(key)] = prepare_export_data(data)
+        data.each_with_index do |d, i|
+          self.parsed_metadata["#{key_for_export(property_name)}_#{i + 1}"] = prepare_export_data(d)
+        end
       end
     end
 
@@ -269,6 +266,14 @@ module Bulkrax
       "#{unnumbered_key}#{key.sub(clean_key, '')}"
     end
 
+    def prepare_export_data_with_join(data)
+      # Yes...it's possible we're asking to coerce a multi-value but only have a single value.
+      return data.to_s unless data.is_a?(Enumerable)
+      return "" if data.empty?
+
+      data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
+    end
+
     def prepare_export_data(datum)
       if datum.is_a?(ActiveTriples::Resource)
         datum.to_uri.to_s

data/app/models/bulkrax/importer.rb

@@ -24,13 +24,12 @@ module Bulkrax
     attr_writer :current_run
 
     def self.safe_uri_filename(uri)
-      uri = URI.parse(uri) unless uri.is_a?(URI)
       r = Faraday.head(uri.to_s)
       return CGI.parse(r.headers['content-disposition'])["filename"][0].delete("\"")
     rescue
-      filename = File.basename(uri.path)
+      filename = File.basename(uri.to_s)
       filename.delete!('/')
-      filename.presence || file_set.id
+      filename.presence || SecureRandom.uuid
     end
 
     def status

data/app/parsers/bulkrax/csv_parser.rb

@@ -33,9 +33,9 @@ module Bulkrax
           model_field_mappings.map(&:to_sym).each do |model_mapping|
             next unless r.key?(model_mapping)
 
-            if r[model_mapping].casecmp('collection').zero?
+            if r[model_mapping].strip.casecmp('collection').zero?
               @collections << r
-            elsif r[model_mapping].casecmp('fileset').zero?
+            elsif r[model_mapping].strip.casecmp('fileset').zero?
               @file_sets << r
             else
               @works << r

data/app/parsers/bulkrax/parser_export_record_set.rb

@@ -21,6 +21,30 @@ module Bulkrax
       "Bulkrax::ParserExportRecordSet::#{export_from.classify}".constantize.new(parser: parser)
     end
 
+    SOLR_QUERY_PAGE_SIZE = 512
+
+    ##
+    # A helper method for handling querying large batches of IDs.  By default SOLR has a max of 1024
+    # `OR` clauses per query.  This method helps chunk large sets of IDs into batches.
+    #
+    # @param array [Array<Object>]
+    # @param page_size [Integer]
+    # @yieldparam [Array<Object>] slice of the original arrays which are yielded.  The results of
+    #             the yield are merged into the return value.
+    #
+    # @return [Array<Object>]
+    #
+    # @see https://github.com/samvera-labs/bulkrax/issues/776
+    def self.in_batches(array, page_size: SOLR_QUERY_PAGE_SIZE)
+      array = Array.wrap(array)
+      return [] if array.empty?
+      results = []
+      array.each_slice(page_size) do |slice|
+        results += Array.wrap(yield(slice))
+      end
+      results
+    end
+
     # @abstract
     #
     # @note This has {#each} and {#count} but is not an Enumerable.  But because it has these two
@@ -36,6 +60,7 @@ module Bulkrax
       delegate :limit_reached?, :work_entry_class, :collection_entry_class, :file_set_entry_class, :importerexporter, to: :parser
       private :limit_reached?, :work_entry_class, :collection_entry_class, :file_set_entry_class, :importerexporter
 
+      ##
       # @return [Integer]
       def count
         sum = works.count + collections.count + file_sets.count
@@ -44,6 +69,7 @@ module Bulkrax
         return sum
       end
 
+      ##
       # Yield first the works, then collections, then file sets.  Once we've yielded as many times
       # as the parser's limit, we break the iteration and return.
       #
@@ -134,8 +160,6 @@ module Bulkrax
                          end
       end
 
-      SOLR_QUERY_PAGE_SIZE = 512
-
       # @note In most cases, when we don't have any candidate file sets, there is no need to query SOLR.
       #
       # @see Bulkrax::ParserExportRecordSet::Importer#file_sets
@@ -148,20 +172,14 @@ module Bulkrax
       # @see https://github.com/scientist-softserv/britishlibrary/issues/289
       # @see https://github.com/samvera/hyrax/blob/64c0bbf0dc0d3e1b49f040b50ea70d177cc9d8f6/app/indexers/hyrax/work_indexer.rb#L15-L18
       def file_sets
-        @file_sets ||= if candidate_file_set_ids.empty?
-                         []
-                       else
-                         results = []
-                         candidate_file_set_ids.each_slice(SOLR_QUERY_PAGE_SIZE) do |ids|
-                           fsq = "has_model_ssim:#{Bulkrax.file_model_class} AND id:(\"" + ids.join('" OR "') + "\")"
-                           fsq += extra_filters if extra_filters.present?
-                           results += ActiveFedora::SolrService.query(
-                             fsq,
-                             { fl: "id", method: :post, rows: ids.size }
-                           )
-                         end
-                         results
-                       end
+        @file_sets ||= ParserExportRecordSet.in_batches(candidate_file_set_ids) do |batch_of_ids|
+          fsq = "has_model_ssim:#{Bulkrax.file_model_class} AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
+          fsq += extra_filters if extra_filters.present?
+          ActiveFedora::SolrService.query(
+            fsq,
+            { fl: "id", method: :post, rows: batch_of_ids.size }
+          )
+        end
       end
 
       def solr_name(base_name)
@@ -227,32 +245,34 @@ module Bulkrax
           end
       end
 
-      def works_query_kwargs
-        query_kwargs.merge(
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
-          ],
-          fl: 'id'
-        )
-      end
-
-      def works_query
-        extra_filters.to_s
-      end
-
-      def collections_query_kwargs
-        query_kwargs.merge(
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:Collection"
-          ],
-          fl: 'id'
-        )
+      def works
+        @works ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
+          ActiveFedora::SolrService.query(
+            extra_filters.to_s,
+            **query_kwargs.merge(
+              fq: [
+                %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
+                "has_model_ssim:(#{Bulkrax.curation_concerns.join(' OR ')})"
+              ],
+              fl: 'id'
+            )
+          )
+        end
       end
 
-      def collections_query
-        "has_model_ssim:Collection #{extra_filters}"
+      def collections
+        @collections ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
+          ActiveFedora::SolrService.query(
+            "has_model_ssim:Collection #{extra_filters}",
+            **query_kwargs.merge(
+              fq: [
+                %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
+                "has_model_ssim:Collection"
+              ],
+              fl: "id"
+            )
+          )
+        end
       end
 
       # This is an exception; we don't know how many candidate file sets there might be.  So we will instead
@@ -260,21 +280,18 @@ module Bulkrax
       #
       # @see Bulkrax::ParserExportRecordSet::Base#file_sets
       def file_sets
-        @file_sets ||= ActiveFedora::SolrService.query(file_sets_query, **file_sets_query_kwargs)
-      end
-
-      def file_sets_query_kwargs
-        query_kwargs.merge(
-          fq: [
-            %(#{solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:#{Bulkrax.file_model_class}"
-          ],
-          fl: 'id'
-        )
-      end
-
-      def file_sets_query
-        extra_filters
+        @file_sets ||= ParserExportRecordSet.in_batches(complete_entry_identifiers) do |ids|
+          ActiveFedora::SolrService.query(
+            extra_filters,
+            query_kwargs.merge(
+              fq: [
+                %(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
+                "has_model_ssim:#{Bulkrax.file_model_class}"
+              ],
+              fl: 'id'
+            )
+          )
+        end
       end
     end
   end

data/app/views/bulkrax/entries/show.html.erb

@@ -2,17 +2,17 @@
 <div class="panel panel-default">
   <div class="panel-body">
     <p class='bulkrax-p-align'>
-      <strong>Identifier:</strong>
+      <strong><%= t('bulkrax.importer.labels.identifier') %>:</strong>
       <%= @entry.identifier %>
     </p>
 
     <p class='bulkrax-p-align'>
-      <strong>Entry ID:</strong>
+      <strong><%= t('bulkrax.importer.labels.entry_id') %>:</strong>
       <%= @entry.id %>
     </p>
 
     <p class='bulkrax-p-align'>
-      <strong>Type:</strong>
+      <strong><%= t('bulkrax.importer.labels.type') %>:</strong>
       <%= @entry.factory_class || 'Unknown' %>
     </p>
     <%= render partial: 'raw_metadata'%>
@@ -23,10 +23,10 @@
 
     <p class="bulkrax-p-align">
       <% if @importer.present? %>
-        <strong>Importer:</strong>
+        <strong><%= t('bulkrax.importer.labels.importer') %>:</strong>
         <%= link_to @importer.name, importer_path(@importer) %>
       <% elsif @exporter.present? %>
-        <strong>Exporter:</strong>
+        <strong><%= t('bulkrax.importer.labels.exporter') %>:</strong>
         <%= link_to @exporter.name, exporter_path(@exporter) %>
       <% end %>
     </p>

data/app/views/bulkrax/importers/_bagit_fields.html.erb

@@ -1,15 +1,15 @@
 <div class='bagit_fields'>
 
-  <%#= fi.input :metadata_type, 
-    collection:  importer.import_metadata_type, 
+  <%#= fi.input :metadata_type,
+    collection:  importer.import_metadata_type,
     selected: importer.parser_fields['metadata_type'],
     include_blank: true,
     input_html: { class: 'form-control' }
     %>
   <%= fi.input :metadata_file_name, as: :string, input_html: { value: importer.parser_fields['metadata_file_name'] } %>
 
-  <%= fi.input :metadata_format, 
-    collection:  importer.import_metadata_format, 
+  <%= fi.input :metadata_format,
+    collection:  importer.import_metadata_format,
     selected: importer.parser_fields['metadata_format'],
     include_blank: true,
     input_html: { class: 'form-control' }
@@ -18,7 +18,8 @@
   <%= fi.input :visibility,
     collection: [
       ['Public', 'open'],
-      ['Private', 'restricted']
+      ['Private', 'restricted'],
+      ['Institution', 'authenticated']
     ],
     selected: importer.parser_fields['visibility'] || 'open',
     input_html: { class: 'form-control' }

data/app/views/bulkrax/importers/_csv_fields.html.erb

@@ -3,7 +3,8 @@
   <%= fi.input :visibility,
     collection: [
       ['Public', 'open'],
-      ['Private', 'restricted']
+      ['Private', 'restricted'],
+      ['Institution', 'authenticated']
     ],
     selected: importer.parser_fields['visibility'] || 'open',
     input_html: { class: 'form-control' }

data/app/views/bulkrax/importers/_oai_fields.html.erb

@@ -1,16 +1,17 @@
 <div class='oai_fields'>
   <%= fi.input :base_url, as: :string, input_html: { value: importer.parser_fields['base_url'] } %>
-  
+
   <%= fi.input :metadata_prefix, as: :string, hint: 'Such as oai_dc, dcterms or oai_qdc', input_html: { value: importer.parser_fields['metadata_prefix'] } %>
-  
+
   <%= fi.input :set, collection: [importer.parser_fields['set']], label: 'Set (source)', selected: importer.parser_fields['set'] %>
   <button type="button" class="btn btn-default refresh-set-source">Refresh Sets</button>
 
   <%= fi.input :visibility,
     collection: [
       ['Public', 'open'],
-      ['Private', 'restricted']
-    ],
+      ['Private', 'restricted'],
+      ['Institution', 'authenticated']
+    ],x
     selected: importer.parser_fields['visibility'] || 'open',
     input_html: { class: 'form-control' }
   %>

data/app/views/bulkrax/importers/_xml_fields.html.erb

@@ -1,31 +1,32 @@
 <div class='xml_fields'>
 
-  <%# @todo improve on this implementation. 
-    As it stands, it's a hostage to namespaces, 
-    eg. dc:title 
+  <%# @todo improve on this implementation.
+    As it stands, it's a hostage to namespaces,
+    eg. dc:title
     if namespaces aren't in the xml, we would have to specify dc:title
     but if the namespaces ARE present, we remove them so we would need title
   %>
-  <%= fi.input :record_element, 
-    hint: 'Provide the xml element name to use to identify the record, or records, eg. ROW - each record in the attached XML is wrapped in a <ROW> tag.', 
+  <%= fi.input :record_element,
+    hint: 'Provide the xml element name to use to identify the record, or records, eg. ROW - each record in the attached XML is wrapped in a <ROW> tag.',
     input_html: { value: importer.parser_fields['record_element'] }
     %>
 
-  <%= fi.input :import_type, 
+  <%= fi.input :import_type,
 		collection: [
       ['Single Work per Metadata File', 'single'],
-			['Multiple Works per Metadata File', 'multiple'] 
-		], 
+			['Multiple Works per Metadata File', 'multiple']
+		],
     selected: importer.parser_fields['import_type'],
     input_html: { class: 'form-control' }
     %>
-  
+
   <h4>Visiblity</h4>
 
   <%= fi.input :visibility,
     collection: [
       ['Public', 'open'],
-      ['Private', 'restricted']
+      ['Private', 'restricted'],
+      ['Institution', 'authenticated']
     ],
     selected: importer.parser_fields['visibility'] || 'open',
     input_html: { class: 'form-control' }

data/app/views/bulkrax/importers/edit.html.erb

@@ -10,7 +10,9 @@
         <%= render 'form', importer: @importer, form: form %>
         <div class="panel-footer">
           <div class='pull-right'>
-            <%= link_to 'Update Importer', '#bulkraxModal', class: "btn btn-primary", data: { toggle: 'modal' } %>
+            <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#bulkraxModal">
+              Update Importer
+            </button>
             <%= render 'edit_form_buttons', form: form %>
            <% cancel_path = form.object.persisted? ? importer_path(form.object) : importers_path %>
             | <%= link_to t('.cancel'), cancel_path, class: 'btn btn-default ' %>
@@ -19,4 +21,4 @@
       <% end %>
     </div>
   </div>
-</div>
+</div>

data/app/views/bulkrax/importers/new.html.erb

@@ -9,8 +9,10 @@
         <%= render 'form', importer: @importer, form: form %>
         <div class="panel-footer">
           <div class='pull-right'>
-            <%= form.button :submit, value: 'Create and Validate', class: 'btn btn-primary' %>
-            |
+            <% if ENV['SHOW_CREATE_AND_VALIDATE'] == 'true' %>
+              <%= form.button :submit, value: 'Create and Validate', class: 'btn btn-primary' %>
+              |
+            <% end  %>
             <%= form.button :submit, value: 'Create and Import', class: 'btn btn-primary' %>
             |
             <%= form.button :submit, value: 'Create', class: 'btn btn-primary' %>

data/app/views/hyrax/dashboard/sidebar/_bulkrax_sidebar_additions.html.erb

@@ -1,10 +1,12 @@
 <% if current_ability.can_import_works? %>
-  <%= menu.nav_link(bulkrax.importers_path) do %>
+  <%= menu.nav_link(bulkrax.importers_path,
+                    title: t('bulkrax.admin.sidebar.importers')) do %>
     <span class="fa fa-cloud-upload" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.importers') %></span>
   <% end %>
 <% end %>
 <% if current_ability.can_export_works? %>
-  <%= menu.nav_link(bulkrax.exporters_path) do %>
+  <%= menu.nav_link(bulkrax.exporters_path,
+                    title: t('bulkrax.admin.sidebar.exporters')) do %>
     <span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
   <% end %>
 <% end %>

data/app/views/hyrax/dashboard/sidebar/_repository_content.html.erb

@@ -3,14 +3,16 @@
 <%= menu.nav_link(hyrax.my_collections_path,
                   class: "nav-link",
                   onclick: "dontChangeAccordion(event);",
-                  also_active_for: hyrax.dashboard_collections_path) do %>
+                  also_active_for: hyrax.dashboard_collections_path,
+                  title: t('hyrax.admin.sidebar.collections')) do %>
   <span class="fa fa-folder-open" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.collections') %></span>
 <% end %>
 
 <%= menu.nav_link(hyrax.my_works_path,
                   class: "nav-link",
                   onclick: "dontChangeAccordion(event);",
-                  also_active_for: hyrax.dashboard_works_path) do %>
+                  also_active_for: hyrax.dashboard_works_path,
+                  title: t('hyrax.admin.sidebar.works')) do %>
   <span class="fa fa-file" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.works') %></span>
 <% end %>
 

data/config/locales/bulkrax.en.yml

@@ -46,18 +46,23 @@ en:
         generated_metadata: "These exported fields currently cannot be imported."
     importer:
       labels:
-        name: Name
-        user: User
         admin_set: Admin set
+        collection_entries: Collection Entries
+        entry_id: Entry ID
+        exporter: Exporter
+        file_set_entries: File Set Entries
         frequency: Frequency
-        parser_klass: Parser klass
+        identifier: Identifier
+        importer: Importer
         limit: Limit
-        total_work_entries: Total Works
+        name: Name
+        parser_klass: Parser klass
         total_collections: Total Collections
         total_file_sets: Total File Sets
+        total_work_entries: Total Works
+        type: Type
+        user: User
         work_entries: Work Entries
-        collection_entries: Collection Entries
-        file_set_entries: File Set Entries
     table_header:
       labels:
         identifier: Identifier

data/db/migrate/20210806044408_remove_unused_last_error.rb

@@ -1,7 +1,7 @@
 class RemoveUnusedLastError < ActiveRecord::Migration[5.1]
   def change
-    remove_column :bulkrax_entries, :last_error
-    remove_column :bulkrax_exporters, :last_error
-    remove_column :bulkrax_importers, :last_error
+    remove_column :bulkrax_entries, :last_error if column_exists?(:bulkrax_entries, :last_error)
+    remove_column :bulkrax_exporters, :last_error if column_exists?(:bulkrax_exporters, :last_error)
+    remove_column :bulkrax_importers, :last_error if column_exists?(:bulkrax_importers, :last_error)
   end
 end

data/db/migrate/20230608153601_add_indices_to_bulkrax.rb

@@ -0,0 +1,14 @@
+class AddIndicesToBulkrax < ActiveRecord::Migration[5.1]
+  def change
+    add_index :bulkrax_entries, :identifier unless index_exists?(:bulkrax_entries, :identifier)
+    add_index :bulkrax_entries, :type unless index_exists?(:bulkrax_entries, :type)
+    add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx' unless index_exists?(:bulkrax_entries, [:importerexporter_id, :importerexporter_type], name: 'bulkrax_entries_importerexporter_idx')
+
+    add_index :bulkrax_pending_relationships, :parent_id unless index_exists?(:bulkrax_pending_relationships, :parent_id)
+    add_index :bulkrax_pending_relationships, :child_id unless index_exists?(:bulkrax_pending_relationships, :child_id) 
+
+    add_index :bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx' unless index_exists?(:bulkrax_statuses, [:statusable_id, :statusable_type], name: 'bulkrax_statuses_statusable_idx')
+    add_index :bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx' unless index_exists?(:bulkrax_statuses, [:runnable_id, :runnable_type], name: 'bulkrax_statuses_runnable_idx')
+    add_index :bulkrax_statuses, :error_class unless index_exists?(:bulkrax_statuses, :error_class)
+  end
+end

data/lib/bulkrax/engine.rb

@@ -23,15 +23,16 @@ module Bulkrax
     end
 
     config.after_initialize do
-      my_engine_root = Bulkrax::Engine.root.to_s
-      paths = ActionController::Base.view_paths.collect(&:to_s)
-      hyrax_path = paths.detect { |path| path.match(/\/hyrax-[\d\.]+.*/) }
-      paths = if hyrax_path
-                paths.insert(paths.index(hyrax_path), my_engine_root + '/app/views')
-              else
-                paths.insert(0, my_engine_root + '/app/views')
-              end
-      ActionController::Base.view_paths = paths
+      # We want to ensure that Bulkrax is earlier in the lookup for view_paths than Hyrax.  That is
+      # we favor view in Bulkrax over those in Hyrax.
+      if defined?(Hyrax)
+        my_engine_root = Bulkrax::Engine.root.to_s
+        hyrax_engine_root = Hyrax::Engine.root.to_s
+        paths = ActionController::Base.view_paths.collect(&:to_s)
+        hyrax_view_path = paths.detect { |path| path.match(%r{^#{hyrax_engine_root}}) }
+        paths.insert(paths.index(hyrax_view_path), File.join(my_engine_root, 'app', 'views')) if hyrax_view_path
+        ActionController::Base.view_paths = paths.uniq
+      end
     end
   end
 end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '5.2.1'
+  VERSION = '5.4.0'
 end

data/lib/bulkrax.rb

@@ -34,6 +34,15 @@ module Bulkrax
                   :required_elements,
                   :reserved_properties,
                   :server_name
+
+    attr_writer :use_locking
+
+    def use_locking
+      return @use_locking if defined?(@use_locking)
+
+      ENV.key?("REDIS_HOST")
+    end
+    alias use_locking? use_locking
   end
 
   def config
@@ -87,7 +96,10 @@ module Bulkrax
                  :reserved_properties,
                  :reserved_properties=,
                  :server_name,
-                 :server_name=
+                 :server_name=,
+                 :use_locking,
+                 :use_locking=,
+                 :use_locking?
 
   config do |conf|
     conf.parsers = [

data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -7,8 +7,8 @@ Bulkrax.setup do |config|
   # ]
 
   # WorkType to use as the default if none is specified in the import
-  # Default is the first returned by Hyrax.config.curation_concerns
-  # config.default_work_type = MyWork
+  # Default is the first returned by Hyrax.config.curation_concerns, stringified
+  # config.default_work_type = "MyWork"
 
   # Factory Class to use when generating and saving objects
   config.object_factory = Bulkrax::ObjectFactory

data/lib/tasks/bulkrax_tasks.rake

@@ -1,6 +1,108 @@
 # frozen_string_literal: true
 
 namespace :bulkrax do
+  # Usage example: rails bulkrax:generate_test_csvs['5','100','GenericWork']
+  desc 'Generate CSVs with fake data for testing purposes'
+  task :generate_test_csvs, [:num_of_csvs, :csv_rows, :record_type] => :environment do |_t, args|
+    # NOTE: If this line throws an error, run `gem install faker` inside your Docker container
+    require 'faker'
+    require 'csv'
+
+    FileUtils.mkdir_p(Rails.root.join('tmp', 'imports'))
+
+    IGNORED_PROPERTIES = %w[
+      admin_set_id
+      alternate_ids
+      arkivo_checksum
+      created_at
+      date_modified
+      date_uploaded
+      depositor
+      embargo
+      has_model
+      head
+      internal_resource
+      label
+      lease
+      member_ids
+      member_of_collection_ids
+      modified_date
+      new_record
+      on_behalf_of
+      owner
+      proxy_depositor
+      rendering_ids
+      representative_id
+      state
+      tail
+      thumbnail_id
+      updated_at
+    ].freeze
+
+    BULKRAX_PROPERTIES = %w[
+      source_identifier
+      model
+    ].freeze
+
+    num_of_csvs = args.num_of_csvs.presence&.to_i || 5
+    csv_rows = args.csv_rows.presence&.to_i || 100
+    record_type = args.record_type.presence&.constantize || GenericWork
+
+    csv_header = if Hyrax.config.try(:use_valkyrie?)
+                   record_type.schema.map { |k| k.name.to_s }
+                 else
+                   record_type.properties.keys
+                 end
+
+    csv_header -= IGNORED_PROPERTIES
+    csv_header.unshift(*BULKRAX_PROPERTIES)
+
+    num_of_csvs.times do |i|
+      CSV.open(Rails.root.join('tmp', 'imports', "importer_#{i}.csv"), 'wb') do |csv|
+        csv << csv_header
+        csv_rows.times do |_index|
+          row = []
+          csv_header.each do |prop_name|
+            row << case prop_name
+                   when 'id', 'source_identifier'
+                     Faker::Number.number(digits: 4)
+                   when 'model'
+                     record_type.to_s
+                   when 'rights_statement'
+                     'http://rightsstatements.org/vocab/CNE/1.0/'
+                   when 'license'
+                     'https://creativecommons.org/licenses/by-nc/4.0/'
+                   when 'based_near'
+                     # FIXME: Set a proper :based_near value
+                     nil
+                   else
+                     Faker::Lorem.sentence
+                   end
+          end
+          csv << row
+        end
+      end
+    end
+
+    num_of_csvs.times do |i|
+      Bulkrax::Importer.create(
+        name: "Generated CSV #{i}",
+        admin_set_id: 'admin_set/default',
+        user_id: User.find_by(email: 'admin@example.com').id,
+        frequency: 'PT0S',
+        parser_klass: 'Bulkrax::CsvParser',
+        parser_fields: {
+          'visibility' => 'open',
+          'rights_statement' => '',
+          'override_rights_statement' => '0',
+          'file_style' => 'Specify a Path on the Server',
+          'import_file_path' => "tmp/imports/importer_#{i}.csv",
+          'update_files' => false
+        }
+      )
+    end
+  end
+
   desc "Remove old exported zips and create new ones with the new file structure"
   task rerun_all_exporters: :environment do
     # delete the existing folders and zip files

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.4.0
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-24 00:00:00.000000000 Z
+date: 2023-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-monads
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: iso8601
   requirement: !ruby/object:Gem::Requirement
@@ -238,6 +252,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: psych
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 description: Bulkrax is a batteries included importer for Samvera applications. It
   currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the
   box. It is also designed to be extensible, allowing you to easily add new importers
@@ -258,6 +300,7 @@ files:
 - app/assets/javascripts/bulkrax/entries.js
 - app/assets/javascripts/bulkrax/exporters.js
 - app/assets/javascripts/bulkrax/importers.js.erb
+- app/assets/javascripts/bulkrax/navtabs.js.erb
 - app/assets/stylesheets/bulkrax/accordion.scss
 - app/assets/stylesheets/bulkrax/application.css
 - app/assets/stylesheets/bulkrax/coderay.scss
@@ -395,6 +438,7 @@ files:
 - db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb
 - db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb
 - db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb
+- db/migrate/20230608153601_add_indices_to_bulkrax.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/entry_spec_helper.rb
@@ -427,7 +471,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku