bulkrax 6.0.1 → 8.0.0

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -6,7 +6,7 @@ module Bulkrax
   class ImportFileSetJob < ApplicationJob
     include DynamicRecordLookup
 
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     attr_reader :importer_run_id
 
@@ -63,8 +63,11 @@ module Bulkrax
     end
 
     def check_parent_is_a_work!(parent_identifier)
-      error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
-      raise ::StandardError, error_msg unless curation_concern?(parent_record)
+      case parent_record
+      when Bulkrax.collection_model_class, Bulkrax.file_model_class
+        error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
+        raise ::StandardError, error_msg
+      end
     end
 
     def find_parent_record(parent_identifier)

data/app/jobs/bulkrax/import_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImportJob < ApplicationJob
+    queue_as :import
+  end
+end

data/app/jobs/bulkrax/import_work_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImportWorkJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # rubocop:disable Rails/SkipsModelValidations
     #

data/app/jobs/bulkrax/importer_job.rb

@@ -2,10 +2,11 @@
 
 module Bulkrax
   class ImporterJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     def perform(importer_id, only_updates_since_last_import = false)
       importer = Importer.find(importer_id)
+      return schedule(importer, Time.zone.now + 3.minutes, 'Rescheduling: cloud files are not ready yet') unless all_files_completed?(importer)
 
       importer.current_run
       unzip_imported_file(importer.parser)
@@ -16,6 +17,8 @@ module Bulkrax
       importer.set_status_info(e)
     end
 
+    private
+
     def import(importer, only_updates_since_last_import)
       importer.only_updates = only_updates_since_last_import || false
       return unless importer.valid_import?
@@ -36,8 +39,21 @@ module Bulkrax
       importer.current_run.save!
     end
 
-    def schedule(importer)
-      ImporterJob.set(wait_until: importer.next_import_at).perform_later(importer.id, true)
+    def schedule(importer, wait_until = importer.next_import_at, message = nil)
+      Rails.logger.info message if message
+      ImporterJob.set(wait_until: wait_until).perform_later(importer.id, true)
+    end
+
+    # checks the file sizes of the download files to match the original files
+    def all_files_completed?(importer)
+      cloud_files = importer.parser_fields['cloud_file_paths']
+      original_files = importer.parser_fields['original_file_paths']
+      return true unless cloud_files.present? && original_files.present?
+
+      imported_file_sizes = cloud_files.map { |_, v| v['file_size'].to_i }
+      original_file_sizes = original_files.map { |imported_file| File.size(imported_file) }
+
+      original_file_sizes == imported_file_sizes
     end
   end
 end

data/app/matchers/bulkrax/application_matcher.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'language_list'
-
 module Bulkrax
   class ApplicationMatcher
     attr_accessor :to, :from, :parsed, :if, :split, :excluded, :nested_type

data/app/models/bulkrax/csv_collection_entry.rb

@@ -2,9 +2,7 @@
 
 module Bulkrax
   class CsvCollectionEntry < CsvEntry
-    def factory_class
-      Collection
-    end
+    self.default_work_type = Bulkrax.collection_model_class.to_s
 
     # Use identifier set by CsvParser#unique_collection_identifier, which falls back
     # on the Collection's first title if record[source_identifier] is not present

data/app/models/bulkrax/csv_entry.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'csv'
-
 module Bulkrax
   # TODO: We need to rework this class some to address the Metrics/ClassLength rubocop offense.
   # We do too much in these entry classes. We need to extract the common logic from the various
@@ -16,11 +14,12 @@ module Bulkrax
     class_attribute(:csv_read_data_options, default: {})
 
     # there's a risk that this reads the whole file into memory and could cause a memory leak
+    # we strip any special characters out of the headers. looking at you Excel
     def self.read_data(path)
       raise StandardError, 'CSV path empty' if path.blank?
       options = {
         headers: true,
-        header_converters: ->(h) { h.to_s.strip.to_sym },
+        header_converters: ->(h) { h.to_s.gsub(/[^\w\d\. -]+/, '').strip.to_sym },
         encoding: 'utf-8'
       }.merge(csv_read_data_options)
 
@@ -105,7 +104,7 @@ module Bulkrax
     end
 
     def add_metadata_for_model
-      if defined?(::Collection) && factory_class == ::Collection
+      if factory_class.present? && factory_class == Bulkrax.collection_model_class
         add_collection_type_gid if defined?(::Hyrax)
         # add any additional collection metadata methods here
       elsif factory_class == Bulkrax.file_model_class
@@ -145,7 +144,7 @@ module Bulkrax
       self.parsed_metadata = {}
 
       build_system_metadata
-      build_files_metadata if defined?(Collection) && !hyrax_record.is_a?(Collection)
+      build_files_metadata if Bulkrax.collection_model_class.present? && !hyrax_record.is_a?(Bulkrax.collection_model_class)
       build_relationship_metadata
       build_mapping_metadata
       self.save!
@@ -157,9 +156,12 @@ module Bulkrax
     def build_system_metadata
       self.parsed_metadata['id'] = hyrax_record.id
       source_id = hyrax_record.send(work_identifier)
-      source_id = source_id.to_a.first if source_id.is_a?(ActiveTriples::Relation)
+      # Because ActiveTriples::Relation does not respond to #to_ary we can't rely on Array.wrap universally
+      source_id = source_id.to_a if source_id.is_a?(ActiveTriples::Relation)
+      source_id = Array.wrap(source_id).first
       self.parsed_metadata[source_identifier] = source_id
-      self.parsed_metadata[key_for_export('model')] = hyrax_record.has_model.first
+      model_name = hyrax_record.respond_to?(:to_rdf_representation) ? hyrax_record.to_rdf_representation : hyrax_record.has_model.first
+      self.parsed_metadata[key_for_export('model')] = model_name
     end
 
     def build_files_metadata

data/app/models/bulkrax/entry.rb

@@ -8,6 +8,8 @@ module Bulkrax
   class Entry < ApplicationRecord
     include Bulkrax::HasMatchers
     include Bulkrax::ImportBehavior
+    self.class_attribute :default_work_type, default: Bulkrax.default_work_type
+
     include Bulkrax::ExportBehavior
     include Bulkrax::StatusInfo
     include Bulkrax::HasLocalProcessing
@@ -101,22 +103,18 @@ module Bulkrax
       self.importerexporter_type == 'Bulkrax::Exporter'
     end
 
-    def valid_system_id(model_class)
-      return true if model_class.properties.keys.include?(work_identifier)
-      raise(
-        "#{model_class} does not implement the system_identifier_field: #{work_identifier}"
-      )
-    end
-
     def last_run
       self.importerexporter&.last_run
     end
 
     def find_collection(collection_identifier)
-      return unless Collection.properties.keys.include?(work_identifier)
-      Collection.where(
-        work_identifier => collection_identifier
-      ).detect { |m| m.send(work_identifier).include?(collection_identifier) }
+      Bulkrax.object_factory.search_by_property(
+        klass: Bulkrax.collection_model_class,
+        value: collection_identifier,
+        search_field: work_identifier,
+        name_field: work_identifier,
+        verify_property: true
+      )
     end
   end
 end

data/app/models/bulkrax/exporter.rb

@@ -23,6 +23,10 @@ module Bulkrax
       set_status_info(e)
     end
 
+    def remove_and_rerun
+      self.parser_fields['remove_and_rerun']
+    end
+
     # #export_source accessors
     # Used in form to prevent it from getting confused as to which value to populate #export_source with.
     # Also, used to display the correct selected value when rendering edit form.
@@ -102,9 +106,12 @@ module Bulkrax
       Importer.all.map { |i| [i.name, i.id] }
     end
 
-    def current_run
+    def current_run(skip_counts: false)
+      @current_run ||= self.exporter_runs.create! if skip_counts
+      return @current_run if @current_run
+
       total = self.limit || parser.total
-      @current_run ||= self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
+      @current_run = self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
     end
 
     def last_run
@@ -130,8 +137,8 @@ module Bulkrax
     end
 
     def export_properties
-      properties = Bulkrax.curation_concerns.map { |work| work.properties.keys }.flatten.uniq.sort
-      properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
+      # TODO: Does this work for Valkyrie?
+      Bulkrax.object_factory.export_properties
     end
 
     def metadata_only?

data/app/models/bulkrax/importer.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require 'iso8601'
-
 module Bulkrax
-  class Importer < ApplicationRecord
+  class Importer < ApplicationRecord # rubocop:disable Metrics/ClassLength
     include Bulkrax::ImporterExporterBehavior
     include Bulkrax::StatusInfo
 
@@ -18,7 +16,7 @@ module Bulkrax
     validates :admin_set_id, presence: true if defined?(::Hyrax)
     validates :parser_klass, presence: true
 
-    delegate :valid_import?, :write_errored_entries_file, :visibility, to: :parser
+    delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
 
     attr_accessor :only_updates, :file_style, :file
     attr_writer :current_run
@@ -103,11 +101,12 @@ module Bulkrax
       frequency.to_seconds != 0
     end
 
-    def current_run
+    def current_run(skip_counts: false)
       return @current_run if @current_run.present?
 
       @current_run = self.importer_runs.create!
       return @current_run if file? && zip?
+      return @current_run if skip_counts
 
       entry_counts = {
         total_work_entries: self.limit || parser.works_total,
@@ -123,6 +122,29 @@ module Bulkrax
       @last_run ||= self.importer_runs.last
     end
 
+    def failed_entries?
+      entries.failed.any?
+    end
+
+    def failed_statuses
+      @failed_statuses ||= Bulkrax::Status.latest_by_statusable
+                                          .includes(:statusable)
+                                          .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Failed')
+    end
+
+    def failed_messages
+      failed_statuses.each_with_object({}) do |e, i|
+        i[e.error_message] ||= []
+        i[e.error_message] << e.id
+      end
+    end
+
+    def completed_statuses
+      @completed_statuses ||= Bulkrax::Status.latest_by_statusable
+                                             .includes(:statusable)
+                                             .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Complete')
+    end
+
     def seen
       @seen ||= {}
     end
@@ -135,6 +157,18 @@ module Bulkrax
       self.parser_fields['update_files']
     end
 
+    def remove_and_rerun
+      self.parser_fields['remove_and_rerun']
+    end
+
+    def metadata_only?
+      parser.parser_fields['metadata_only'] == true
+    end
+
+    def existing_entries?
+      parser.parser_fields['file_style']&.match(/Existing Entries/)
+    end
+
     def import_works
       import_objects(['work'])
     end
@@ -157,11 +191,20 @@ module Bulkrax
       self.only_updates ||= false
       self.save if self.new_record? # Object needs to be saved for statuses
       types = types_array || DEFAULT_OBJECT_TYPES
-      parser.create_objects(types)
+      existing_entries? ? parser.rebuild_entries(types) : parser.create_objects(types)
+      mark_unseen_as_skipped
     rescue StandardError => e
       set_status_info(e)
     end
 
+    # After an import any entries we did not touch are skipped.
+    # They are not really pending, complete for the last run, or failed
+    def mark_unseen_as_skipped
+      entries.where.not(identifier: seen.keys).find_each do |entry|
+        entry.set_status_info('Skipped')
+      end
+    end
+
     # Prepend the base_url to ensure unique set identifiers
     # @todo - move to parser, as this is OAI specific
     def unique_collection_identifier(id)
@@ -192,9 +235,5 @@ module Bulkrax
     rescue
       "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
     end
-
-    def metadata_only?
-      parser.parser_fields['metadata_only'] == true
-    end
   end
 end

data/app/models/bulkrax/oai_entry.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'erb'
-require 'ostruct'
-
 module Bulkrax
   class OaiEntry < Entry
     serialize :raw_metadata, Bulkrax::NormalizedJson

data/app/models/bulkrax/oai_set_entry.rb

@@ -2,9 +2,7 @@
 
 module Bulkrax
   class OaiSetEntry < OaiEntry
-    def factory_class
-      Collection
-    end
+    self.default_work_type = Bulkrax.collection_model_class.to_s
 
     def build_metadata
       self.parsed_metadata = self.raw_metadata

data/app/models/bulkrax/rdf_collection_entry.rb

@@ -2,6 +2,7 @@
 
 module Bulkrax
   class RdfCollectionEntry < RdfEntry
+    self.default_work_type = Bulkrax.collection_model_class.to_s
     def record
       @record ||= self.raw_metadata
     end
@@ -11,9 +12,5 @@ module Bulkrax
       add_local
       return self.parsed_metadata
     end
-
-    def factory_class
-      Collection
-    end
   end
 end

data/app/models/bulkrax/rdf_entry.rb

@@ -1,91 +1,92 @@
 # frozen_string_literal: true
 
-require 'rdf'
-module Bulkrax
-  class RdfEntry < Entry
-    serialize :raw_metadata, Bulkrax::NormalizedJson
+unless ENV.fetch('BULKRAX_NO_RDF', 'false').to_s == 'true'
+  module Bulkrax
+    class RdfEntry < Entry
+      serialize :raw_metadata, Bulkrax::NormalizedJson
 
-    def self.read_data(path)
-      RDF::Reader.open(path)
-    end
+      def self.read_data(path)
+        RDF::Reader.open(path)
+      end
 
-    def self.fields_from_data(data)
-      data.predicates.map(&:to_s)
-    end
+      def self.fields_from_data(data)
+        data.predicates.map(&:to_s)
+      end
 
-    def self.data_for_entry(data, source_id, parser)
-      reader = data
-      format = reader.class.format.to_sym
-      collections = []
-      children = []
-      delete = nil
-      data = RDF::Writer.for(format).buffer do |writer|
-        reader.each_statement do |statement|
-          collections << statement.object.to_s if parent_field(parser).present? && parent_field(parser) == statement.predicate.to_s
-          children << statement.object.to_s if related_children_parsed_mapping.present? && related_children_parsed_mapping == statement.predicate.to_s
-          delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
-          writer << statement
+      def self.data_for_entry(data, source_id, parser)
+        reader = data
+        format = reader.class.format.to_sym
+        collections = []
+        children = []
+        delete = nil
+        data = RDF::Writer.for(format).buffer do |writer|
+          reader.each_statement do |statement|
+            collections << statement.object.to_s if parent_field(parser).present? && parent_field(parser) == statement.predicate.to_s
+            children << statement.object.to_s if related_children_parsed_mapping.present? && related_children_parsed_mapping == statement.predicate.to_s
+            delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
+            writer << statement
+          end
         end
+        return {
+          source_id => reader.subjects.first.to_s,
+          delete: delete,
+          format: format,
+          data: data,
+          collection: collections,
+          children: children
+        }
       end
-      return {
-        source_id => reader.subjects.first.to_s,
-        delete: delete,
-        format: format,
-        data: data,
-        collection: collections,
-        children: children
-      }
-    end
 
-    def self.related_children_parsed_mapping
-      return @related_children_parsed_mapping if @related_children_parsed_mapping.present?
+      def self.related_children_parsed_mapping
+        return @related_children_parsed_mapping if @related_children_parsed_mapping.present?
 
-      rdf_related_children_field_mapping = Bulkrax.field_mappings['Bulkrax::RdfParser']&.select { |_, h| h.key?('related_children_field_mapping') }
-      return if rdf_related_children_field_mapping.blank?
+        rdf_related_children_field_mapping = Bulkrax.field_mappings['Bulkrax::RdfParser']&.select { |_, h| h.key?('related_children_field_mapping') }
+        return if rdf_related_children_field_mapping.blank?
 
-      @related_children_parsed_mapping = rdf_related_children_field_mapping&.keys&.first
-    end
+        @related_children_parsed_mapping = rdf_related_children_field_mapping&.keys&.first
+      end
 
-    def record
-      @record ||= RDF::Reader.for(self.raw_metadata['format'].to_sym).new(self.raw_metadata['data'])
-    end
+      def record
+        @record ||= RDF::Reader.for(self.raw_metadata['format'].to_sym).new(self.raw_metadata['data'])
+      end
 
-    def build_metadata
-      raise StandardError, 'Record not found' if record.nil?
-      raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
+      def build_metadata
+        raise StandardError, 'Record not found' if record.nil?
+        raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
 
-      self.parsed_metadata = {}
-      self.parsed_metadata[work_identifier] = [self.raw_metadata[source_identifier]]
+        self.parsed_metadata = {}
+        self.parsed_metadata[work_identifier] = [self.raw_metadata[source_identifier]]
 
-      record.each_statement do |statement|
-        # Only process the subject for our record (in case other data is in the file)
-        next unless statement.subject.to_s == self.raw_metadata[source_identifier]
-        add_metadata(statement.predicate.to_s, statement.object.to_s)
-      end
-      add_visibility
-      add_rights_statement
-      add_admin_set_id
-      add_collections
-      add_local
-      self.parsed_metadata['file'] = self.raw_metadata['file']
+        record.each_statement do |statement|
+          # Only process the subject for our record (in case other data is in the file)
+          next unless statement.subject.to_s == self.raw_metadata[source_identifier]
+          add_metadata(statement.predicate.to_s, statement.object.to_s)
+        end
+        add_visibility
+        add_rights_statement
+        add_admin_set_id
+        add_collections
+        add_local
+        self.parsed_metadata['file'] = self.raw_metadata['file']
 
-      self.parsed_metadata
-    end
+        self.parsed_metadata
+      end
 
-    def collections_created?
-      return true if self.raw_metadata['collection'].blank?
-      self.raw_metadata['collection'].length == self.collection_ids.length
-    end
+      def collections_created?
+        return true if self.raw_metadata['collection'].blank?
+        self.raw_metadata['collection'].length == self.collection_ids.length
+      end
 
-    def find_collection_ids
-      return self.collection_ids if collections_created?
-      if self.raw_metadata['collection'].present?
-        self.raw_metadata['collection'].each do |collection|
-          c = find_collection(collection)
-          self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
+      def find_collection_ids
+        return self.collection_ids if collections_created?
+        if self.raw_metadata['collection'].present?
+          self.raw_metadata['collection'].each do |collection|
+            c = find_collection(collection)
+            self.collection_ids << c.id unless c.blank? || self.collection_ids.include?(c.id)
+          end
         end
+        return self.collection_ids
       end
-      return self.collection_ids
     end
   end
 end

data/app/models/bulkrax/status.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class Status < ApplicationRecord
-    belongs_to :statusable, polymorphic: true
+    belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message], if: :latest? }
     belongs_to :runnable, polymorphic: true
     serialize :error_backtrace, Array
 
@@ -21,5 +21,14 @@ module Bulkrax
       status_table.join(latest_status_query.as(latest_status_table.name.to_s), Arel::Nodes::InnerJoin)
                   .on(status_table[:id].eq(latest_status_table[:latest_status_id]))
     end
+
+    def latest?
+      # TODO: remove if statment when we stop supporting Hyrax < 4
+      self.id == if Gem::Version.new(Rails::VERSION::STRING) >= Gem::Version.new('6.0.0')
+                   self.class.where(statusable_id: self.statusable_id, statusable_type: self.statusable_type).order('id desc').pick(:id)
+                 else
+                   self.class.where(statusable_id: self.statusable_id, statusable_type: self.statusable_type).order('id desc').pluck(:id).first # rubocop:disable Rails/Pick
+                 end
+    end
   end
 end

data/app/models/bulkrax/xml_entry.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'nokogiri'
 module Bulkrax
   # Generic XML Entry
   class XmlEntry < Entry

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -18,9 +18,9 @@ module Bulkrax
       begin
         # the identifier parameter can be a :source_identifier or the id of an object
         record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
-        record ||= ActiveFedora::Base.find(identifier)
+        record ||= Bulkrax.object_factory.find(identifier)
       # NameError for if ActiveFedora isn't installed
-      rescue NameError, ActiveFedora::ObjectNotFoundError
+      rescue NameError, ActiveFedora::ObjectNotFoundError, Bulkrax::ObjectFactoryInterface::ObjectNotFoundError
         record = nil
       end
 
@@ -28,22 +28,5 @@ module Bulkrax
       # also accounts for when the found entry isn't a part of this importer
       record.is_a?(Entry) ? [record, record.factory.find] : [nil, record]
     end
-
-    # Check if the record is a Work
-    def curation_concern?(record)
-      available_work_types.include?(record.class)
-    end
-
-    private
-
-    # @return [Array<Class>] list of work type classes
-    def available_work_types
-      # If running in a Hyku app, do not include disabled work types
-      @available_work_types ||= if defined?(::Hyku)
-                                  ::Site.instance.available_works.map(&:constantize)
-                                else
-                                  Bulkrax.curation_concerns
-                                end
-    end
   end
 end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'marcel'
 
 module Bulkrax
   module ExportBehavior
@@ -22,11 +21,12 @@ module Bulkrax
     end
 
     def hyrax_record
-      @hyrax_record ||= ActiveFedora::Base.find(self.identifier)
+      @hyrax_record ||= Bulkrax.object_factory.find(self.identifier)
     end
 
     # Prepend the file_set id to ensure a unique filename and also one that is not longer than 255 characters
     def filename(file_set)
+      # NOTE: Will this work with Valkyrie?
       return if file_set.original_file.blank?
       fn = file_set.original_file.file_name.first
       mime = ::Marcel::MimeType.for(file_set.original_file.mime_type)