bulkrax 7.0.0 → 8.0.0

data/app/factories/bulkrax/valkyrie_object_factory.rb

@@ -0,0 +1,402 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # rubocop:disable Metrics/ClassLength
+  class ValkyrieObjectFactory < ObjectFactoryInterface
+    class FileFactoryInnerWorkings < Bulkrax::FileFactory::InnerWorkings
+      def remove_file_set(file_set:)
+        file_metadata = Hyrax.custom_queries.find_files(file_set: file_set).first
+        raise "No file metadata records found for #{file_set.class} ID=#{file_set.id}" unless file_metadata
+
+        Hyrax::VersioningService.create(file_metadata, user, File.new(Bulkrax.removed_image_path))
+
+        ::ValkyrieCreateDerivativesJob.set(wait: 1.minute).perform_later(file_set.id, file_metadata.id)
+      end
+
+      ##
+      # Replace an existing :file_set's file with the :uploaded file.
+      #
+      # @param file_set [Hyrax::FileSet, Object]
+      # @param uploaded [Hyrax::UploadedFile]
+      #
+      # @return [NilClass]
+      def update_file_set(file_set:, uploaded:)
+        file_metadata = Hyrax.custom_queries.find_files(file_set: file_set).first
+        raise "No file metadata records found for #{file_set.class} ID=#{file_set.id}" unless file_metadata
+
+        uploaded_file = uploaded.file
+
+        # TODO: Is this accurate?  We'll need to interrogate the file_metadata
+        # object.  Should it be `file_metadata.checksum.first.to_s` Or something
+        # else?
+        return nil if file_metadata.checksum.first == Digest::SHA1.file(uploaded_file.path).to_s
+
+        Hyrax::VersioningService.create(file_metadata, user, uploaded_file)
+
+        ::ValkyrieCreateDerivativesJob.set(wait: 1.minute).perform_later(file_set.id, file_metadata.id)
+        nil
+      end
+    end
+
+    # TODO: the following module needs revisiting for Valkyrie work.
+    #       proposal is to create Bulkrax::ValkyrieFileFactory.
+    include Bulkrax::FileFactory
+
+    self.file_set_factory_inner_workings_class = Bulkrax::ValkyrieObjectFactory::FileFactoryInnerWorkings
+
+    ##
+    # When you want a different set of transactions you can change the
+    # container.
+    #
+    # @note Within {Bulkrax::ValkyrieObjectFactory} there are several calls to
+    #       transactions; so you'll need your container to register those
+    #       transactions.
+    def self.transactions
+      @transactions || Hyrax::Transactions::Container
+    end
+
+    def transactions
+      self.class.transactions
+    end
+
+    ##
+    # @!group Class Method Interface
+
+    ##
+    # @note This does not save either object.  We need to do that in another
+    #       loop.  Why?  Because we might be adding many items to the parent.
+    def self.add_child_to_parent_work(parent:, child:)
+      return true if parent.member_ids.include?(child.id)
+
+      parent.member_ids << child.id
+      parent.save
+    end
+
+    def self.add_resource_to_collection(collection:, resource:, user:)
+      resource.member_of_collection_ids << collection.id
+      save!(resource: resource, user: user)
+    end
+
+    def self.field_multi_value?(field:, model:)
+      return false unless field_supported?(field: field, model: model)
+
+      if model.respond_to?(:schema)
+        dry_type = model.schema.key(field.to_sym)
+        return true if dry_type.respond_to?(:primitive) && dry_type.primitive == Array
+
+        false
+      else
+        Bulkrax::ObjectFactory.field_multi_value?(field: field, model: model)
+      end
+    end
+
+    def self.field_supported?(field:, model:)
+      if model.respond_to?(:schema)
+        schema_properties(model).include?(field)
+      else
+        # We *might* have a Fedora object, so we need to consider that approach as
+        # well.
+        Bulkrax::ObjectFactory.field_supported?(field: field, model: model)
+      end
+    end
+
+    def self.file_sets_for(resource:)
+      return [] if resource.blank?
+      return [resource] if resource.is_a?(Bulkrax.file_model_class)
+
+      Hyrax.query_service.custom_queries.find_child_file_sets(resource: resource)
+    end
+
+    def self.find(id)
+      Hyrax.query_service.find_by(id: id)
+      # Because Hyrax is not a hard dependency, we need to transform the Hyrax exception into a
+      # common exception so that callers can handle a generalize exception.
+    rescue Hyrax::ObjectNotFoundError => e
+      raise ObjectFactoryInterface::ObjectNotFoundError, e.message
+    end
+
+    def self.find_or_create_default_admin_set
+      Hyrax::AdminSetCreateService.find_or_create_default_admin_set
+    end
+
+    def self.solr_name(field_name)
+      # It's a bit unclear what this should be if we can't rely on Hyrax.
+      raise NotImplementedError, "#{self}.#{__method__}" unless defined?(Hyrax)
+      Hyrax.config.index_field_mapper.solr_name(field_name)
+    end
+
+    def self.publish(event:, **kwargs)
+      Hyrax.publisher.publish(event, **kwargs)
+    end
+
+    def self.query(q, **kwargs)
+      # Someone could choose ActiveFedora::SolrService.  But I think we're
+      # assuming Valkyrie is specifcally working for Hyrax.  Someone could make
+      # another object factory.
+      raise NotImplementedError, "#{self}.#{__method__}" unless defined?(Hyrax)
+      Hyrax::SolrService.query(q, **kwargs)
+    end
+
+    def self.save!(resource:, user:)
+      if resource.respond_to?(:save!)
+        resource.save!
+      else
+        result = Hyrax.persister.save(resource: resource)
+        raise Valkyrie::Persistence::ObjectNotFoundError unless result
+        Hyrax.index_adapter.save(resource: result)
+        if result.collection?
+          publish('collection.metadata.updated', collection: result, user: user)
+        else
+          publish('object.metadata.updated', object: result, user: user)
+        end
+        resource
+      end
+    end
+
+    def self.update_index(resources:)
+      Array(resources).each do |resource|
+        Hyrax.index_adapter.save(resource: resource)
+      end
+    end
+
+    def self.update_index_for_file_sets_of(resource:)
+      file_sets = Hyrax.query_service.custom_queries.find_child_file_sets(resource: resource)
+      update_index(resources: file_sets)
+    end
+
+    ##
+    # @param value [String]
+    # @param klass [Class, #where]
+    # @param field [String, Symbol] A convenience parameter where we pass the
+    #        same value to search_field and name_field.
+    # @param name_field [String] the ActiveFedora::Base property name
+    #        (e.g. "title")
+    # @return [NilClass] when no object is found.
+    # @return [Valkyrie::Resource] when a match is found, an instance of given
+    #         :klass
+    # rubocop:disable Metrics/ParameterLists
+    def self.search_by_property(value:, klass:, field: nil, name_field: nil, **)
+      name_field ||= field
+      raise "Expected named_field or field got nil" if name_field.blank?
+      return if value.blank?
+
+      # Return nil or a single object.
+      Hyrax.query_service.custom_query.find_by_model_and_property_value(model: klass, property: name_field, value: value)
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    ##
+    # Retrieve properties from M3 model
+    # @param klass the model
+    # @return [Array<String>]
+    def self.schema_properties(klass)
+      @schema_properties_map ||= {}
+
+      klass_key = klass.name
+      @schema_properties_map[klass_key] = klass.schema.map { |k| k.name.to_s } unless @schema_properties_map.key?(klass_key)
+
+      @schema_properties_map[klass_key]
+    end
+
+    def self.ordered_file_sets_for(object)
+      return [] if object.blank?
+
+      Hyrax.custom_queries.find_child_file_sets(resource: object)
+    end
+
+    def delete(user)
+      obj = find
+      return false unless obj
+
+      Hyrax.persister.delete(resource: obj)
+      Hyrax.index_adapter.delete(resource: obj)
+      self.class.publish(event: 'object.deleted', object: obj, user: user)
+    end
+
+    def run!
+      run
+      # reload the object
+      object = find
+      return object if object.persisted?
+
+      raise(ObjectFactoryInterface::RecordInvalid, object)
+    end
+
+    private
+
+    def apply_depositor_metadata
+      return if object.depositor.present?
+
+      object.depositor = @user.email
+      object = Hyrax.persister.save(resource: object)
+      self.class.publish(event: "object.metadata.updated", object: object, user: @user)
+      object
+    end
+
+    def conditionall_apply_depositor_metadata
+      # We handle this in transactions
+      nil
+    end
+
+    def conditionally_set_reindex_extent
+      # Valkyrie does not concern itself with the reindex extent; no nesting
+      # indexers here!
+      nil
+    end
+
+    def create_file_set(attrs)
+      # TODO: Make it work for Valkyrie
+    end
+
+    def create_work(attrs)
+      # NOTE: We do not add relationships here; that is part of the create
+      # relationships job.
+      perform_transaction_for(object: object, attrs: attrs) do
+        transactions["change_set.create_work"]
+          .with_step_args(
+            'work_resource.add_file_sets' => { uploaded_files: uploaded_files_from(attrs) },
+            "change_set.set_user_as_depositor" => { user: @user },
+            "work_resource.change_depositor" => { user: @user },
+            'work_resource.save_acl' => { permissions_params: [attrs['visibility'] || 'open'].compact }
+          )
+      end
+    end
+
+    def create_collection(attrs)
+      # TODO: Handle Collection Type
+      #
+      # NOTE: We do not add relationships here; that is part of the create
+      # relationships job.
+      perform_transaction_for(object: object, attrs: attrs) do
+        transactions['change_set.create_collection']
+          .with_step_args(
+            'change_set.set_user_as_depositor' => { user: @user },
+            'collection_resource.apply_collection_type_permissions' => { user: @user }
+          )
+      end
+    end
+
+    def find_by_id
+      Hyrax.query_service.find_by(id: attributes[:id]) if attributes.key? :id
+    end
+
+    ##
+    # @param object [Valkyrie::Resource]
+    # @param attrs [Valkyrie::Resource]
+    # @return [Valkyrie::Resource] when we successfully processed the
+    #         transaction (e.g. the transaction's data was valid according to
+    #         the derived form)
+    #
+    # @yield the returned value of the yielded block should be a
+    #        {Hyrax::Transactions::Transaction}.  We yield because the we first
+    #        want to check if the attributes are valid.  And if so, then process
+    #        the transaction, which is something that could trigger expensive
+    #        operations.  Put another way, don't do something expensive if the
+    #        data is invalid.
+    #
+    # TODO What do we return when the calculated form fails?
+    # @raise [StandardError] when there was a failure calling the translation.
+    def perform_transaction_for(object:, attrs:)
+      form = Hyrax::Forms::ResourceForm.for(object).prepopulate!
+
+      # TODO: Handle validations
+      form.validate(attrs)
+
+      transaction = yield
+
+      result = transaction.call(form)
+
+      result.value_or do
+        msg = result.failure[0].to_s
+        msg += " - #{result.failure[1].full_messages.join(',')}" if result.failure[1].respond_to?(:full_messages)
+        raise StandardError, msg, result.trace
+      end
+    end
+
+    ##
+    # We accept attributes based on the model schema
+    #
+    # @return [Array<Symbols>]
+    def permitted_attributes
+      @permitted_attributes ||= (
+        base_permitted_attributes + if klass.respond_to?(:schema)
+                                      Bulkrax::ValkyrieObjectFactory.schema_properties(klass)
+                                    else
+                                      klass.properties.keys.map(&:to_sym)
+                                    end
+      ).uniq
+    end
+
+    def update_work(attrs)
+      perform_transaction_for(object: object, attrs: attrs) do
+        transactions["change_set.update_work"]
+          .with_step_args(
+            'work_resource.add_file_sets' => { uploaded_files: uploaded_files_from(attrs) },
+            'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
+          )
+      end
+    end
+
+    def update_collection(attrs)
+      # NOTE: We do not add relationships here; that is part of the create
+      # relationships job.
+      perform_transaction_for(object: object, attrs: attrs) do
+        transactions['change_set.update_collection']
+      end
+    end
+
+    def update_file_set(attrs)
+      # TODO: Make it work
+    end
+
+    def uploaded_files_from(attrs)
+      uploaded_local_files(uploaded_files: attrs[:uploaded_files]) + uploaded_s3_files(remote_files: attrs[:remote_files])
+    end
+
+    def uploaded_local_files(uploaded_files: [])
+      Array.wrap(uploaded_files).map do |file_id|
+        Hyrax::UploadedFile.find(file_id)
+      end
+    end
+
+    def uploaded_s3_files(remote_files: {})
+      return [] if remote_files.blank?
+
+      s3_bucket_name = ENV.fetch("STAGING_AREA_S3_BUCKET", "comet-staging-area-#{Rails.env}")
+      s3_bucket = Rails.application.config.staging_area_s3_connection
+                       .directories.get(s3_bucket_name)
+
+      remote_files.map { |r| r["url"] }.map do |key|
+        s3_bucket.files.get(key)
+      end.compact
+    end
+
+    # @Override Destroy existing files with Hyrax::Transactions
+    def destroy_existing_files
+      existing_files = Hyrax.custom_queries.find_child_file_sets(resource: object)
+
+      existing_files.each do |fs|
+        transactions["file_set.destroy"]
+          .with_step_args("file_set.remove_from_work" => { user: @user },
+                          "file_set.delete" => { user: @user })
+          .call(fs)
+          .value!
+      end
+
+      @object.member_ids = @object.member_ids.reject { |m| existing_files.detect { |f| f.id == m } }
+      @object.rendering_ids = []
+      @object.representative_id = nil
+      @object.thumbnail_id = nil
+    end
+
+    def transform_attributes(update: false)
+      attrs = super.merge(alternate_ids: [source_identifier_value])
+                   .symbolize_keys
+
+      attrs[:title] = [''] if attrs[:title].blank?
+      attrs[:creator] = [''] if attrs[:creator].blank?
+      attrs
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/app/helpers/bulkrax/importers_helper.rb

@@ -6,7 +6,7 @@ module Bulkrax
     def available_admin_sets
       # Restrict available_admin_sets to only those current user can deposit to.
       @available_admin_sets ||= Hyrax::Collections::PermissionsService.source_ids_for_deposit(ability: current_ability, source_type: 'admin_set').map do |admin_set_id|
-        [AdminSet.find(admin_set_id).title.first, admin_set_id]
+        [Bulkrax.object_factory.find_or_nil(admin_set_id)&.title&.first || admin_set_id, admin_set_id]
       end
     end
   end

data/app/helpers/bulkrax/validation_helper.rb

@@ -20,14 +20,14 @@ module Bulkrax
       return unless defined?(::Hyrax)
 
       if params[:importer][:admin_set_id].blank?
-        params[:importer][:admin_set_id] = AdminSet::DEFAULT_ID
+        params[:importer][:admin_set_id] = Bulkrax.object_factory.default_admin_set_id
       else
-        AdminSet.find(params[:importer][:admin_set_id])
+        Bulkrax.object_factory.find(params[:importer][:admin_set_id])
       end
       return true
-    rescue ActiveFedora::ObjectNotFoundError
+    rescue ActiveFedora::ObjectNotFoundError, Bulkrax::ObjectFactoryInterface::ObjectNotFoundError
       logger.warn("AdminSet #{params[:importer][:admin_set_id]} not found. Using default admin set.")
-      params[:importer][:admin_set_id] = AdminSet::DEFAULT_ID
+      params[:importer][:admin_set_id] = Bulkrax.object_factory.default_admin_set_id
       return true
     end
 

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Bulkrax
+  ##
   # Responsible for creating parent-child relationships between Works and Collections.
   #
   # Handles three kinds of relationships:
@@ -42,6 +43,7 @@ module Bulkrax
 
     queue_as Bulkrax.config.ingest_queue_name
 
+    ##
     # @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
     # @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
     #
@@ -53,7 +55,7 @@ module Bulkrax
     #
     # rubocop:disable Metrics/MethodLength
     def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
-      importer_run = Bulkrax::ImporterRun.find(importer_run_id)
+      @importer_run = Bulkrax::ImporterRun.find(importer_run_id)
       ability = Ability.new(importer_run.user)
 
       parent_entry, parent_record = find_record(parent_identifier, importer_run_id)
@@ -79,9 +81,9 @@ module Bulkrax
 
           # save record if members were added
           if @parent_record_members_added
-            parent_record.save!
-            # Ensure that the new relationship gets indexed onto the children
-            @child_members_added.each(&:update_index)
+            Bulkrax.object_factory.save!(resource: parent_record, user: importer_run.user)
+            Bulkrax.object_factory.publish(event: 'object.membership.updated', object: parent_record)
+            Bulkrax.object_factory.update_index(resources: @child_members_added)
           end
         end
       else
@@ -104,7 +106,7 @@ module Bulkrax
         parent_entry&.set_status_info(errors.last, importer_run)
 
         # TODO: This can create an infinite job cycle, consider a time to live tracker.
-        reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
+        reschedule(parent_identifier: parent_identifier, importer_run_id: importer_run_id)
         return false # stop current job from continuing to run after rescheduling
       else
         # rubocop:disable Rails/SkipsModelValidations
@@ -114,6 +116,8 @@ module Bulkrax
     end
     # rubocop:enable Metrics/MethodLength
 
+    attr_reader :importer_run
+
     private
 
     ##
@@ -151,25 +155,32 @@ module Bulkrax
       # We could do this outside of the loop, but that could lead to odd counter failures.
       ability.authorize!(:edit, parent_record)
 
-      parent_record.is_a?(Collection) ? add_to_collection(child_record, parent_record) : add_to_work(child_record, parent_record)
+      if parent_record.is_a?(Bulkrax.collection_model_class)
+        add_to_collection(child_record, parent_record)
+      else
+        add_to_work(child_record, parent_record)
+      end
+
+      Bulkrax.object_factory.update_index_for_file_sets_of(resource: child_record) if update_child_records_works_file_sets?
 
-      child_record.file_sets.each(&:update_index) if update_child_records_works_file_sets? && child_record.respond_to?(:file_sets)
       relationship.destroy
     end
 
     def add_to_collection(child_record, parent_record)
-      parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX) if
-        defined?(Hyrax::Adapters::NestingIndexAdapter)
-      child_record.member_of_collections << parent_record
-      child_record.save!
+      Bulkrax.object_factory.add_resource_to_collection(
+        collection: parent_record,
+        resource: child_record,
+        user: importer_run.user
+      )
     end
 
     def add_to_work(child_record, parent_record)
-      return true if parent_record.ordered_members.to_a.include?(child_record)
-
-      parent_record.ordered_members << child_record
-      @parent_record_members_added = true
-      @child_members_added << child_record
+      # NOTE: The .add_child_to_parent_work should not persist changes to the
+      #       child nor parent.  We'll do that elsewhere in this loop.
+      Bulkrax.object_factory.add_child_to_parent_work(
+        parent: parent_record,
+        child: child_record
+      )
     end
 
     def reschedule(parent_identifier:, importer_run_id:)

data/app/jobs/bulkrax/delete_job.rb

@@ -5,8 +5,9 @@ module Bulkrax
     queue_as Bulkrax.config.ingest_queue_name
 
     def perform(entry, importer_run)
-      obj = entry.factory.find
-      obj&.delete
+      user = importer_run.importer.user
+      entry.factory.delete(user)
+
       # rubocop:disable Rails/SkipsModelValidations
       ImporterRun.increment_counter(:deleted_records, importer_run.id)
       ImporterRun.decrement_counter(:enqueued_records, importer_run.id)

data/app/jobs/bulkrax/download_cloud_file_job.rb

@@ -1,18 +1,31 @@
 # frozen_string_literal: true
-
 module Bulkrax
   class DownloadCloudFileJob < ApplicationJob
     queue_as Bulkrax.config.ingest_queue_name
 
+    include ActionView::Helpers::NumberHelper
+
     # Retrieve cloud file and write to the imports directory
     # Note: if using the file system, the mounted directory in
     #   browse_everything MUST be shared by web and worker servers
     def perform(file, target_file)
       retriever = BrowseEverything::Retriever.new
+      last_logged_time = Time.zone.now
+      log_interval = 3.seconds
+
       retriever.download(file, target_file) do |filename, retrieved, total|
-        # The block is still useful for showing progress, but the
-        # first argument is the filename instead of a chunk of data.
+        percentage = (retrieved.to_f / total.to_f) * 100
+        current_time = Time.zone.now
+
+        if (current_time - last_logged_time) >= log_interval
+          # Use number_to_human_size for formatting
+          readable_retrieved = number_to_human_size(retrieved)
+          readable_total = number_to_human_size(total)
+          Rails.logger.info "Downloaded #{readable_retrieved} of #{readable_total}, #{filename}: #{percentage.round}% complete"
+          last_logged_time = current_time
+        end
       end
+      Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
     end
   end
 end

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -63,8 +63,11 @@ module Bulkrax
     end
 
     def check_parent_is_a_work!(parent_identifier)
-      error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
-      raise ::StandardError, error_msg unless curation_concern?(parent_record)
+      case parent_record
+      when Bulkrax.collection_model_class, Bulkrax.file_model_class
+        error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
+        raise ::StandardError, error_msg
+      end
     end
 
     def find_parent_record(parent_identifier)

data/app/jobs/bulkrax/importer_job.rb

@@ -6,6 +6,7 @@ module Bulkrax
 
     def perform(importer_id, only_updates_since_last_import = false)
       importer = Importer.find(importer_id)
+      return schedule(importer, Time.zone.now + 3.minutes, 'Rescheduling: cloud files are not ready yet') unless all_files_completed?(importer)
 
       importer.current_run
       unzip_imported_file(importer.parser)
@@ -16,6 +17,8 @@ module Bulkrax
       importer.set_status_info(e)
     end
 
+    private
+
     def import(importer, only_updates_since_last_import)
       importer.only_updates = only_updates_since_last_import || false
       return unless importer.valid_import?
@@ -36,8 +39,21 @@ module Bulkrax
       importer.current_run.save!
     end
 
-    def schedule(importer)
-      ImporterJob.set(wait_until: importer.next_import_at).perform_later(importer.id, true)
+    def schedule(importer, wait_until = importer.next_import_at, message = nil)
+      Rails.logger.info message if message
+      ImporterJob.set(wait_until: wait_until).perform_later(importer.id, true)
+    end
+
+    # checks the file sizes of the download files to match the original files
+    def all_files_completed?(importer)
+      cloud_files = importer.parser_fields['cloud_file_paths']
+      original_files = importer.parser_fields['original_file_paths']
+      return true unless cloud_files.present? && original_files.present?
+
+      imported_file_sizes = cloud_files.map { |_, v| v['file_size'].to_i }
+      original_file_sizes = original_files.map { |imported_file| File.size(imported_file) }
+
+      original_file_sizes == imported_file_sizes
     end
   end
 end

data/app/matchers/bulkrax/application_matcher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'language_list'
-
 module Bulkrax
   class ApplicationMatcher
     attr_accessor :to, :from, :parsed, :if, :split, :excluded, :nested_type

data/app/models/bulkrax/csv_collection_entry.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class CsvCollectionEntry < CsvEntry
-    self.default_work_type = "Collection"
+    self.default_work_type = Bulkrax.collection_model_class.to_s
 
     # Use identifier set by CsvParser#unique_collection_identifier, which falls back
     # on the Collection's first title if record[source_identifier] is not present