bulkrax 8.0.0 → 8.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ca767b1e461d5830774bb4d7c49eade799d14298625bdbdccdcbdc0ac997f88
-  data.tar.gz: b5024ea772132123fcb922fbe183d22950428ea50b7a8b4152c6bd781b94510f
+  metadata.gz: 7886fb5eefc500cc8e567d9097e6930e67cee2908fa7fa0d79ce05fc714db392
+  data.tar.gz: c6e720816dd152931df91e06be37977216a7ef06875a98386350706e4df1e3ea
 SHA512:
-  metadata.gz: 91004f88a97054107a3bbbcbe7f3b3dd84226a286df47992c13bfc5fb483c3bfd4aaa9190e08e567258e5105a317a37bba58fa7857ee6c534153ba930b608ad7
-  data.tar.gz: 89a32c155c3931e486b2d41215942b291eb724c59833a031fdc4815a72d26a2df13b015c1b1e2c0dafd608e58ec481a574e20da7bcf95ed76b3e45c3d5f19287
+  metadata.gz: 62971f4eab58de239643d016ec8f8701b6e71953bbb8ccc4bf1d74fd8a88d3798c506004704203107e6f676218bc0dcd337776ecccc423b6c932aeddcd0dacf5
+  data.tar.gz: f3710a5e394a470753ea83d5e007505afd91a8fcdc3c383465fdaa7f2eb9c0901b3bef7cb69ae2c9bfa6d508231095e44ac7781cc9c52565162adfe2e58a5c3d

data/app/assets/javascripts/bulkrax/datatables.js

@@ -67,7 +67,7 @@ Blacklight.onLoad(function() {
         { "data": "name" },
         { "data": "status_message" },
         { "data": "created_at" },
-        { "data": "download" },
+        { "data": "download", "orderable": false },
         { "data": "actions", "orderable": false }
       ],
       initComplete: function () {

data/app/concerns/loggable.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Loggable
+  extend ActiveSupport::Concern
+
+  def log_created(obj)
+    log_action('Created', obj)
+  end
+
+  def log_updated(obj)
+    log_action('Updated', obj)
+  end
+
+  def log_deleted_fs(obj)
+    msg = "Deleted All Files from #{obj.id}"
+    Rails.logger.info("#{msg} (#{Array(obj.attributes[work_identifier]).first})")
+  end
+
+  private
+
+  def log_action(action, obj)
+    msg = "#{action} #{obj.class.model_name.human} #{obj.id}"
+    Rails.logger.info("#{msg} (#{Array(obj.attributes[work_identifier]).first})")
+  end
+end

data/app/controllers/bulkrax/importers_controller.rb

@@ -13,7 +13,7 @@ module Bulkrax
     before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
     before_action :authenticate_user!, unless: -> { api_request? }
     before_action :check_permissions
-    before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy]
+    before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy, :original_file]
     with_themed_layout 'dashboard' if defined?(::Hyrax)
 
     # GET /importers
@@ -201,6 +201,14 @@ module Bulkrax
       end
     end
 
+    def original_file
+      if @importer.original_file?
+        send_file @importer.original_file
+      else
+        redirect_to @importer, alert: 'Importer does not support file re-download or the imported file is not found on the server.'
+      end
+    end
+
     # GET /importers/1/export_errors
     def export_errors
       @importer = Importer.find(params[:importer_id])

data/app/factories/bulkrax/object_factory.rb

@@ -12,7 +12,7 @@ module Bulkrax
     # @note This does not save either object.  We need to do that in another
     #       loop.  Why?  Because we might be adding many items to the parent.
     def self.add_child_to_parent_work(parent:, child:)
-      return true if parent.ordered_members.to_a.include?(child_record)
+      return true if parent.ordered_members.to_a.include?(child)
 
       parent.ordered_members << child
     end
@@ -228,33 +228,29 @@ module Bulkrax
       actor.create_metadata(attrs)
       actor.create_content(uploaded_file) if uploaded_file
       actor.attach_to_work(work, attrs)
-      handle_remote_file(remote_file: remote_file, actor: actor, update: false) if remote_file
+      handle_remote_file(remote_file: remote_file, actor: actor) if remote_file
     end
 
     def update_file_set(attrs)
       file_set_attrs = attrs.slice(*object.attributes.keys)
       actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
       attrs['remote_files']&.each do |remote_file|
-        handle_remote_file(remote_file: remote_file, actor: actor, update: true)
+        handle_remote_file(remote_file: remote_file, actor: actor)
       end
       actor.update_metadata(file_set_attrs)
     end
 
-    def handle_remote_file(remote_file:, actor:, update: false)
+    def handle_remote_file(remote_file:, actor:)
       actor.file_set.label = remote_file['file_name']
       actor.file_set.import_url = remote_file['url']
+      auth_header = remote_file.fetch('auth_header', {})
 
-      url = remote_file['url']
-      tmp_file = Tempfile.new(remote_file['file_name'].split('.').first)
-      tmp_file.binmode
-
-      URI.open(url) do |url_file|
-        tmp_file.write(url_file.read)
-      end
+      ImportUrlJob.perform_now(actor.file_set, file_set_operation_for(user: @user), auth_header)
+    end
 
-      tmp_file.rewind
-      update == true ? actor.update_content(tmp_file) : actor.create_content(tmp_file, from_url: true)
-      tmp_file.close
+    def file_set_operation_for(user:)
+      Hyrax::Operation.create!(user: user,
+                               operation_type: "Attach Remote File")
     end
   end
   # rubocop:enable Metrics/ClassLength

data/app/factories/bulkrax/object_factory_interface.rb

@@ -20,6 +20,7 @@ module Bulkrax
   class ObjectFactoryInterface
     extend ActiveModel::Callbacks
     include DynamicRecordLookup
+    include Loggable
 
     # We're inheriting from an ActiveRecord exception as that is something we
     # know will be here; and something that the main_app will be expect to be
@@ -224,6 +225,12 @@ module Bulkrax
                       id
                       read_groups
                       visibility
+                      visibility_during_embargo
+                      embargo_release_date
+                      visibility_after_embargo
+                      visibility_during_lease
+                      lease_expiration_date
+                      visibility_after_lease
                       work_members_attributes
                     ]
 
@@ -398,21 +405,6 @@ module Bulkrax
       self.class.add_user_to_collection_permissions(**arguments)
     end
 
-    def log_created(obj)
-      msg = "Created #{klass.model_name.human} #{obj.id}"
-      Rails.logger.info("#{msg} (#{Array(attributes[work_identifier]).first})")
-    end
-
-    def log_updated(obj)
-      msg = "Updated #{klass.model_name.human} #{obj.id}"
-      Rails.logger.info("#{msg} (#{Array(attributes[work_identifier]).first})")
-    end
-
-    def log_deleted_fs(obj)
-      msg = "Deleted All Files from #{obj.id}"
-      Rails.logger.info("#{msg} (#{Array(attributes[work_identifier]).first})")
-    end
-
     private
 
     def apply_depositor_metadata

data/app/factories/bulkrax/valkyrie_object_factory.rb

@@ -217,7 +217,7 @@ module Bulkrax
       run
       # reload the object
       object = find
-      return object if object.persisted?
+      return object if object&.persisted?
 
       raise(ObjectFactoryInterface::RecordInvalid, object)
     end
@@ -225,10 +225,10 @@ module Bulkrax
     private
 
     def apply_depositor_metadata
-      return if object.depositor.present?
+      return if @object.depositor.present?
 
-      object.depositor = @user.email
-      object = Hyrax.persister.save(resource: object)
+      @object.depositor = @user.email
+      object = Hyrax.persister.save(resource: @object)
       self.class.publish(event: "object.metadata.updated", object: object, user: @user)
       object
     end
@@ -249,19 +249,77 @@ module Bulkrax
     end
 
     def create_work(attrs)
-      # NOTE: We do not add relationships here; that is part of the create
-      # relationships job.
+      # NOTE: We do not add relationships here; that is part of the create relationships job.
+      attrs = HashWithIndifferentAccess.new(attrs)
       perform_transaction_for(object: object, attrs: attrs) do
+        uploaded_files, file_set_params = prep_fileset_content(attrs)
         transactions["change_set.create_work"]
           .with_step_args(
-            'work_resource.add_file_sets' => { uploaded_files: uploaded_files_from(attrs) },
+            'work_resource.add_file_sets' => { uploaded_files: uploaded_files, file_set_params: file_set_params },
             "change_set.set_user_as_depositor" => { user: @user },
             "work_resource.change_depositor" => { user: @user },
-            'work_resource.save_acl' => { permissions_params: [attrs['visibility'] || 'open'].compact }
+            'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
           )
       end
     end
 
+    ## Prepare fileset data in the required format for creating or updating a work
+    # TODO: Determine why attrs is different from attributes?
+    # TODO: Disabled s3 until we get additional details
+    def prep_fileset_content(attrs)
+      # combine remote_files + thumbnail_url [Array < { url:, file_name:, * }]
+      thumbnail_url = HashWithIndifferentAccess.new(self.attributes)['thumbnail_url']
+      all_remote_files = merge_thumbnails(remote_files: attrs["remote_files"], thumbnail_url: thumbnail_url)
+      # combine local & remote files [Array < Hash &/or String]
+      all_local_files = self.attributes['file'] || []
+      all_files = all_local_files + all_remote_files
+
+      # collect all uploaded files [Array < Hyrax::UploadedFile]
+      uploaded_local = uploaded_local_files(uploaded_files: attrs[:uploaded_files])
+      uploaded_remote = uploaded_remote_files(remote_files: all_remote_files)
+      # uploaded_s3 = uploaded_s3_files(remote_files: attrs[:remote_files])
+      uploaded_files = uploaded_local + uploaded_remote
+
+      # add in other attributes
+      file_set_params = file_set_params_for(uploads: uploaded_files, files: all_files)
+      # return data for filesets
+      [uploaded_files, file_set_params]
+    end
+
+    # supports using thumbnail_url to import a thumbnail separately from other remote_files
+    # in the format thumbnail_url: { url:, file_name: }
+    def merge_thumbnails(remote_files:, thumbnail_url:)
+      r = remote_files || []
+      thumbnail_url.present? ? r + [thumbnail_url] : r
+    end
+
+    # formats file info and facilitates additional custom file_set attributes
+    # To have the additional attributes appear on the file_set, they must be:
+    # - included in the file_set_metadata.yaml
+    # - overridden in file_set_args from Hyrax::WorkUploadsHandler
+    # @param uploads [Array < Hyrax::UploadedFile]
+    # @param files [Array < Hash or String]
+    # @return [Array < Hash]
+    def file_set_params_for(uploads:, files:)
+      # remove url, file_name and paths from attributes
+      additional_attributes = files.map do |f|
+        case f
+        when String
+          {}
+        else
+          temp = f.reject { |key, _| key.to_s == 'url' || key.to_s == 'file_name' }
+          temp['import_url'] = f['url']
+          temp
+        end
+      end
+
+      file_attrs = []
+      uploads.each_with_index do |f, index|
+        file_attrs << ({ uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }).merge(additional_attributes[index])
+      end
+      file_attrs.compact.uniq
+    end
+
     def create_collection(attrs)
       # TODO: Handle Collection Type
       #
@@ -328,10 +386,12 @@ module Bulkrax
     end
 
     def update_work(attrs)
+      attrs = HashWithIndifferentAccess.new(attrs)
       perform_transaction_for(object: object, attrs: attrs) do
+        uploaded_files, file_set_params = prep_fileset_content(attrs)
         transactions["change_set.update_work"]
           .with_step_args(
-            'work_resource.add_file_sets' => { uploaded_files: uploaded_files_from(attrs) },
+            'work_resource.add_file_sets' => { uploaded_files: uploaded_files, file_set_params: file_set_params },
             'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
           )
       end
@@ -349,17 +409,13 @@ module Bulkrax
       # TODO: Make it work
     end
 
-    def uploaded_files_from(attrs)
-      uploaded_local_files(uploaded_files: attrs[:uploaded_files]) + uploaded_s3_files(remote_files: attrs[:remote_files])
-    end
-
     def uploaded_local_files(uploaded_files: [])
       Array.wrap(uploaded_files).map do |file_id|
         Hyrax::UploadedFile.find(file_id)
       end
     end
 
-    def uploaded_s3_files(remote_files: {})
+    def uploaded_s3_files(remote_files: [])
       return [] if remote_files.blank?
 
       s3_bucket_name = ENV.fetch("STAGING_AREA_S3_BUCKET", "comet-staging-area-#{Rails.env}")
@@ -371,6 +427,41 @@ module Bulkrax
       end.compact
     end
 
+    def uploaded_remote_files(remote_files: [])
+      remote_files.map do |r|
+        file_path = download_file(r["url"])
+        next unless file_path
+
+        create_uploaded_file(file_path, r["file_name"])
+      end.compact
+    end
+
+    def download_file(url)
+      require 'open-uri'
+      require 'tempfile'
+
+      begin
+        file = Tempfile.new
+        file.binmode
+        file.write(URI.open(url).read)
+        file.rewind
+        file.path
+      rescue => e
+        Rails.logger.debug "Failed to download file from #{url}: #{e.message}"
+        nil
+      end
+    end
+
+    def create_uploaded_file(file_path, file_name)
+      file = File.open(file_path)
+      uploaded_file = Hyrax::UploadedFile.create(file: file, user: @user, filename: file_name)
+      file.close
+      uploaded_file
+    rescue => e
+      Rails.logger.debug "Failed to create Hyrax::UploadedFile for #{file_name}: #{e.message}"
+      nil
+    end
+
     # @Override Destroy existing files with Hyrax::Transactions
     def destroy_existing_files
       existing_files = Hyrax.custom_queries.find_child_file_sets(resource: object)

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -38,11 +38,13 @@ module Bulkrax
     #
     # @see https://github.com/scientist-softserv/louisville-hyku/commit/128a9ef
     class_attribute :update_child_records_works_file_sets, default: false
+    class_attribute :max_failure_count, default: 5
 
     include DynamicRecordLookup
 
     queue_as Bulkrax.config.ingest_queue_name
 
+    attr_accessor :user, :importer_run, :errors
     ##
     # @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
     # @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
@@ -54,9 +56,10 @@ module Bulkrax
     # is the child in the relationship, and vice versa if a child_identifier is passed.
     #
     # rubocop:disable Metrics/MethodLength
-    def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
-      @importer_run = Bulkrax::ImporterRun.find(importer_run_id)
-      ability = Ability.new(importer_run.user)
+    def perform(parent_identifier:, importer_run_id: nil, run_user: nil, failure_count: 0) # rubocop:disable Metrics/AbcSize
+      importer_run = Bulkrax::ImporterRun.find(importer_run_id) if importer_run_id
+      user = run_user || importer_run&.user
+      ability = Ability.new(user)
 
       parent_entry, parent_record = find_record(parent_identifier, importer_run_id)
 
@@ -69,19 +72,21 @@ module Bulkrax
       if parent_record
         conditionally_acquire_lock_for(parent_record.id) do
           ActiveRecord::Base.uncached do
-            Bulkrax::PendingRelationship.where(parent_id: parent_identifier, importer_run_id: importer_run_id)
+            Bulkrax::PendingRelationship.where(parent_id: parent_identifier)
                                         .ordered.find_each do |rel|
               process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
               number_of_successes += 1
+              @parent_record_members_added = true
             rescue => e
               number_of_failures += 1
+              rel.set_status_info(e, importer_run)
               errors << e
             end
           end
 
           # save record if members were added
           if @parent_record_members_added
-            Bulkrax.object_factory.save!(resource: parent_record, user: importer_run.user)
+            Bulkrax.object_factory.save!(resource: parent_record, user: user)
             Bulkrax.object_factory.publish(event: 'object.membership.updated', object: parent_record)
             Bulkrax.object_factory.update_index(resources: @child_members_added)
           end
@@ -104,10 +109,17 @@ module Bulkrax
         # rubocop:enable Rails/SkipsModelValidations
 
         parent_entry&.set_status_info(errors.last, importer_run)
-
-        # TODO: This can create an infinite job cycle, consider a time to live tracker.
-        reschedule(parent_identifier: parent_identifier, importer_run_id: importer_run_id)
-        return false # stop current job from continuing to run after rescheduling
+        failure_count += 1
+
+        if failure_count < max_failure_count
+          reschedule(
+            parent_identifier: parent_identifier,
+            importer_run_id: importer_run_id,
+            run_user: run_user,
+            failure_count: failure_count
+          )
+        end
+        return errors # stop current job from continuing to run after rescheduling
       else
         # rubocop:disable Rails/SkipsModelValidations
         ImporterRun.update_counters(importer_run_id, processed_relationships: number_of_successes)
@@ -116,8 +128,6 @@ module Bulkrax
     end
     # rubocop:enable Metrics/MethodLength
 
-    attr_reader :importer_run
-
     private
 
     ##
@@ -170,7 +180,7 @@ module Bulkrax
       Bulkrax.object_factory.add_resource_to_collection(
         collection: parent_record,
         resource: child_record,
-        user: importer_run.user
+        user: user
       )
     end
 
@@ -183,11 +193,8 @@ module Bulkrax
       )
     end
 
-    def reschedule(parent_identifier:, importer_run_id:)
-      CreateRelationshipsJob.set(wait: 10.minutes).perform_later(
-        parent_identifier: parent_identifier,
-        importer_run_id: importer_run_id
-      )
+    def reschedule(**kargs)
+      CreateRelationshipsJob.set(wait: 10.minutes).perform_later(**kargs)
     end
   end
 end

data/app/jobs/bulkrax/delete_file_set_job.rb

@@ -1,5 +1,23 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class DeleteFileSetJob < DeleteJob; end
+  class DeleteFileSetJob < DeleteJob
+    def perform(entry, importer_run)
+      file_set = entry.factory.find
+      if file_set
+        parent = file_set.parent
+        if parent&.respond_to?(:ordered_members)
+          om = parent.ordered_members.to_a
+          om.delete(file_set)
+          parent.ordered_members = om
+        elsif parent.respond_to?(:member_ids)
+          parent.member_ids.delete(file_set.id)
+          Hyrax.persister.save(resource: parent)
+        end
+        parent.save
+      end
+
+      super
+    end
+  end
 end

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -16,7 +16,12 @@ module Bulkrax
       # e.g. "parents" or "parents_1"
       parent_identifier = (entry.raw_metadata[entry.related_parents_raw_mapping] || entry.raw_metadata["#{entry.related_parents_raw_mapping}_1"])&.strip
 
-      validate_parent!(parent_identifier)
+      begin
+        validate_parent!(parent_identifier)
+      rescue MissingParentError => e
+        handle_retry(entry, importer_run_id, e)
+        return
+      end
 
       entry.build
       if entry.succeeded?
@@ -32,17 +37,6 @@ module Bulkrax
       entry.save!
       entry.importer.current_run = ImporterRun.find(importer_run_id)
       entry.importer.record_status
-
-    rescue MissingParentError => e
-      # try waiting for the parent record to be created
-      entry.import_attempts += 1
-      entry.save!
-      if entry.import_attempts < 5
-        ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry_id, importer_run_id)
-      else
-        ImporterRun.decrement_counter(:enqueued_records, importer_run_id) # rubocop:disable Rails/SkipsModelValidations
-        entry.set_status_info(e)
-      end
     end
 
     private
@@ -54,14 +48,9 @@ module Bulkrax
       return if parent_identifier.blank?
 
       find_parent_record(parent_identifier)
-      check_parent_exists!(parent_identifier)
       check_parent_is_a_work!(parent_identifier)
     end
 
-    def check_parent_exists!(parent_identifier)
-      raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") if parent_record.nil?
-    end
-
     def check_parent_is_a_work!(parent_identifier)
       case parent_record
       when Bulkrax.collection_model_class, Bulkrax.file_model_class
@@ -72,6 +61,18 @@ module Bulkrax
 
     def find_parent_record(parent_identifier)
       _, @parent_record = find_record(parent_identifier, importer_run_id)
+      raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") unless parent_record
+    end
+
+    def handle_retry(entry, importer_run_id, e)
+      entry.import_attempts += 1
+      entry.save!
+      if entry.import_attempts < 5
+        ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry.id, importer_run_id)
+      else
+        ImporterRun.decrement_counter(:enqueued_records, importer_run_id) # rubocop:disable Rails/SkipsModelValidations
+        entry.set_status_info(e)
+      end
     end
   end
 end

data/app/models/bulkrax/importer.rb

@@ -18,6 +18,9 @@ module Bulkrax
 
     delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
 
+    after_save :set_last_imported_at_from_importer_run
+    after_save :set_next_import_at_from_importer_run
+
     attr_accessor :only_updates, :file_style, :file
     attr_writer :current_run
 
@@ -149,6 +152,18 @@ module Bulkrax
       @seen ||= {}
     end
 
+    def import_file_path
+      self.parser_fields['import_file_path']
+    end
+
+    def original_file?
+      import_file_path && File.exist?(import_file_path)
+    end
+
+    def original_file
+      import_file_path if original_file?
+    end
+
     def replace_files
       self.parser_fields['replace_files']
     end
@@ -235,5 +250,27 @@ module Bulkrax
     rescue
       "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
     end
+
+    private
+
+    # Adding this here since we can update the importer without running the importer.
+    # When we simply save the importer (as in just updating the importer from the options),
+    # it does not trigger the after_save callback in the importer_run.
+    def set_last_imported_at_from_importer_run
+      return if @skip_set_last_imported_at # Prevent infinite loop
+
+      @skip_set_last_imported_at = true
+      importer_runs.last&.set_last_imported_at
+      @skip_set_last_imported_at = false
+    end
+
+    # @see #set_last_imported_at_from_importer_run
+    def set_next_import_at_from_importer_run
+      return if @skip_set_next_import_at # Prevent infinite loop
+
+      @skip_set_next_import_at = true
+      importer_runs.last&.set_next_import_at
+      @skip_set_next_import_at = false
+    end
   end
 end

data/app/models/bulkrax/importer_run.rb

@@ -6,6 +6,9 @@ module Bulkrax
     has_many :statuses, as: :runnable, dependent: :destroy
     has_many :pending_relationships, dependent: :destroy
 
+    after_save :set_last_imported_at
+    after_save :set_next_import_at
+
     def parents
       pending_relationships.pluck(:parent_id).uniq
     end
@@ -15,5 +18,13 @@ module Bulkrax
       # fallback to the configured user.
       importer.user || Bulkrax.fallback_user_for_importer_exporter_processing
     end
+
+    def set_last_imported_at
+      importer.update(last_imported_at: importer.last_imported_at)
+    end
+
+    def set_next_import_at
+      importer.update(next_import_at: importer.next_import_at)
+    end
   end
 end

data/app/models/bulkrax/pending_relationship.rb

@@ -2,6 +2,8 @@
 
 module Bulkrax
   class PendingRelationship < ApplicationRecord
+    include Bulkrax::StatusInfo
+
     belongs_to :importer_run
 
     # Ideally we wouldn't have a column named "order", as it is a reserved SQL term.  However, if we

data/app/models/concerns/bulkrax/file_factory.rb

@@ -32,6 +32,8 @@ module Bulkrax
     end
 
     class InnerWorkings
+      include Loggable
+
       def initialize(object_factory:)
         @object_factory = object_factory
       end
@@ -119,7 +121,7 @@ module Bulkrax
       def destroy_existing_files
         return unless object.present? && object.file_sets.present?
         object.file_sets.each do |fs|
-          Hyrax::Actors::FileSetActor.new(fs, @user).destroy
+          Hyrax::Actors::FileSetActor.new(fs, user).destroy
         end
         @object = object.reload
         log_deleted_fs(object)
@@ -155,6 +157,8 @@ module Bulkrax
       end
 
       def ordered_file_sets
+        return [] if object.blank?
+
         Bulkrax.object_factory.ordered_file_sets_for(object)
       end
 

data/app/models/concerns/bulkrax/has_matchers.rb

@@ -65,6 +65,7 @@ module Bulkrax
 
       parsed_metadata[name] ||= []
       parsed_metadata[name] += Array.wrap(value).flatten
+      parsed_metadata[name].uniq!
     end
 
     def set_parsed_object_data(object_multiple, object_name, name, index, value)
@@ -148,7 +149,18 @@ module Bulkrax
     end
 
     def fields_that_are_always_multiple
-      %w[id delete model visibility]
+      @fields_that_are_always_multiple = %w[
+        id
+        delete
+        model
+        visibility
+        visibility_during_embargo
+        embargo_release_date
+        visibility_after_embargo
+        visibility_during_lease
+        lease_expiration_date
+        visibility_after_lease
+      ]
     end
 
     def fields_that_are_always_singular

data/app/parsers/bulkrax/application_parser.rb

@@ -150,7 +150,7 @@ module Bulkrax
       end
     end
 
-    # The visibility of the record.  Acceptable values are: "open", "embaro", "lease", "authenticated", "restricted".  The default is "open"
+    # The visibility of the record.  Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted".  The default is "open"
     #
     # @return [String]
     # @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.

data/app/views/bulkrax/importers/_csv_fields.html.erb

@@ -1,13 +1,15 @@
 <div class='csv_fields'>
 
   <%= fi.input :visibility,
+    label: 'Default Visibility',
     collection: [
       ['Public', 'open'],
       ['Private', 'restricted'],
       ['Institution', 'authenticated']
     ],
     selected: importer.parser_fields['visibility'] || 'open',
-    input_html: { class: 'form-control' }
+    input_html: { class: 'form-control' },
+    hint: 'If your CSV includes the visibility field, it will override the default setting.'
   %>
 
   <% if defined?(::Hyrax) %>

data/app/views/bulkrax/importers/show.html.erb

@@ -1,12 +1,12 @@
 <div class="col-xs-12 main-header">
   <h1><span class="fa fa-cloud-upload" aria-hidden="true"></span> Importer: <%= @importer.name %></h1>
-
-  <% if @importer.failed_entries? %>
-    <div class="pull-right">
+  <div class="pull-right">
+    <%= link_to 'Download Original File', importer_original_file_path(@importer.id), class: 'btn btn-primary', data: { turbolinks: false } if @importer.original_file %>
+    <% if @importer.failed_entries? %>
       <%= link_to 'Export Errored Entries', importer_export_errors_path(@importer.id), class: 'btn btn-primary', data: { turbolinks: false }%>
       <%= link_to 'Upload Corrected Entries', importer_upload_corrected_entries_path(@importer.id), class: 'btn btn-primary' if @importer.parser.is_a?(Bulkrax::CsvParser) %>
-    </div>
-  <% end %>
+    <% end %>
+  </div>
 </div>
 <div class="panel panel-default bulkrax-align-text">
   <div class="panel-body">

data/config/routes.rb

@@ -11,6 +11,7 @@ Bulkrax::Engine.routes.draw do
   end
   resources :importers do
     put :continue
+    get :original_file
     get :entry_table
     get :export_errors
     collection do

data/db/migrate/20240307053156_add_index_to_metadata_bulkrax_identifier.rb

@@ -1,4 +1,4 @@
-class AddIndexToMetadataBulkraxIdentifier < ActiveRecord::Migration[6.1]
+class AddIndexToMetadataBulkraxIdentifier < ActiveRecord::Migration[5.2]
   def up
     return unless table_exists?(:orm_resources)
     return if index_exists?(:orm_resources, "(((metadata -> 'bulkrax_identifier'::text) ->> 0))", name: 'index_on_bulkrax_identifier')

data/db/migrate/20240806161142_add_file_name_to_uploaded_files.rb

@@ -0,0 +1,5 @@
+class AddFileNameToUploadedFiles < ActiveRecord::Migration[5.2]
+  def change
+    add_column :uploaded_files, :filename, :string unless column_exists?(:uploaded_files, :filename)
+  end
+end

data/db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb

@@ -0,0 +1,5 @@
+class AddErrorTrackingToPendingRelationships < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_pending_relationships, :status_message, :string, default: 'Pending' unless column_exists?(:bulkrax_pending_relationships, :status_message)
+  end
+end

data/db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb

@@ -0,0 +1,5 @@
+class AddLastImportedAtToBulkraxImporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importers, :last_imported_at, :datetime unless column_exists?(:bulkrax_importers, :last_imported_at)
+  end
+end

data/db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb

@@ -0,0 +1,5 @@
+class AddNextImportAtToBulkraxImporters < ActiveRecord::Migration[5.1]
+  def change
+    add_column :bulkrax_importers, :next_import_at, :datetime unless column_exists?(:bulkrax_importers, :next_import_at)
+  end
+end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '8.0.0'
+  VERSION = '8.2.0'
 end

data/lib/tasks/bulkrax_tasks.rake

@@ -141,4 +141,25 @@ namespace :bulkrax do
   rescue => e
     puts "(#{e.message})"
   end
+
+  desc "Resave importers"
+  task resave_importers: :environment do
+    if defined?(::Hyku)
+      Account.find_each do |account|
+        next if account.name == "search"
+        switch!(account)
+        puts "=============== updating #{account.name} ============"
+
+        resave_importers
+
+        puts "=============== finished updating #{account.name} ============"
+      end
+    else
+      resave_importers
+    end
+  end
+
+  def resave_importers
+    Bulkrax::Importer.find_each(&:save!)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.2.0
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-04-02 00:00:00.000000000 Z
+date: 2024-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -334,6 +334,7 @@ files:
 - app/assets/stylesheets/bulkrax/application.css
 - app/assets/stylesheets/bulkrax/coderay.scss
 - app/assets/stylesheets/bulkrax/import_export.scss
+- app/concerns/loggable.rb
 - app/controllers/bulkrax/application_controller.rb
 - app/controllers/bulkrax/entries_controller.rb
 - app/controllers/bulkrax/exporters_controller.rb
@@ -481,6 +482,10 @@ files:
 - db/migrate/20240208005801_denormalize_status_message.rb
 - db/migrate/20240209070952_update_identifier_index.rb
 - db/migrate/20240307053156_add_index_to_metadata_bulkrax_identifier.rb
+- db/migrate/20240806161142_add_file_name_to_uploaded_files.rb
+- db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb
+- db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb
+- db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/entry_spec_helper.rb
@@ -513,7 +518,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.5
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku