bulkrax 5.0.0 → 5.2.0

data/app/models/bulkrax/csv_entry.rb

@@ -13,15 +13,49 @@ module Bulkrax
       data.headers.flatten.compact.uniq
     end
 
+    class_attribute(:csv_read_data_options, default: {})
+
     # there's a risk that this reads the whole file into memory and could cause a memory leak
     def self.read_data(path)
       raise StandardError, 'CSV path empty' if path.blank?
-      CSV.read(path,
+      options = {
         headers: true,
         header_converters: ->(h) { h.to_sym },
-        encoding: 'utf-8')
+        encoding: 'utf-8'
+      }.merge(csv_read_data_options)
+
+      results = CSV.read(path, **options)
+      csv_wrapper_class.new(results)
     end
 
+    # The purpose of this class is to reject empty lines.  This causes lots of grief in importing.
+    # But why not use {CSV.read}'s `skip_lines` option?  Because for some CSVs, it will never finish
+    # reading the file.
+    #
+    # There is a spec that demonstrates this approach works.
+    class CsvWrapper
+      include Enumerable
+      def initialize(original)
+        @original = original
+      end
+
+      delegate :headers, to: :@original
+
+      def each
+        @original.each do |row|
+          next if all_fields_are_empty_for(row: row)
+          yield(row)
+        end
+      end
+
+      private
+
+      def all_fields_are_empty_for(row:)
+        row.to_hash.values.all?(&:blank?)
+      end
+    end
+    class_attribute :csv_wrapper_class, default: CsvWrapper
+
     def self.data_for_entry(data, _source_id, parser)
       # If a multi-line CSV data is passed, grab the first row
       data = data.first if data.is_a?(CSV::Table)
@@ -35,11 +69,7 @@ module Bulkrax
     end
 
     def build_metadata
-      raise StandardError, 'Record not found' if record.nil?
-      unless importerexporter.parser.required_elements?(keys_without_numbers(record.keys))
-        raise StandardError,
-"Missing required elements, missing element(s) are: #{importerexporter.parser.missing_elements(keys_without_numbers(record.keys)).join(', ')}"
-      end
+      validate_record
 
       self.parsed_metadata = {}
       add_identifier
@@ -56,6 +86,12 @@ module Bulkrax
       self.parsed_metadata
     end
 
+    def validate_record
+      raise StandardError, 'Record not found' if record.nil?
+      raise StandardError, "Missing required elements, missing element(s) are: "\
+"#{importerexporter.parser.missing_elements(record).join(', ')}" unless importerexporter.parser.required_elements?(record)
+    end
+
     def add_identifier
       self.parsed_metadata[work_identifier] = [record[source_identifier]]
     end
@@ -67,9 +103,10 @@ module Bulkrax
     end
 
     def add_metadata_for_model
-      if factory_class == Collection
-        add_collection_type_gid
-      elsif factory_class == FileSet
+      if defined?(::Collection) && factory_class == ::Collection
+        add_collection_type_gid if defined?(::Hyrax)
+        # add any additional collection metadata methods here
+      elsif factory_class == Bulkrax.file_model_class
         validate_presence_of_filename!
         add_path_to_file
         validate_presence_of_parent!
@@ -106,7 +143,7 @@ module Bulkrax
       self.parsed_metadata = {}
 
       build_system_metadata
-      build_files_metadata unless hyrax_record.is_a?(Collection)
+      build_files_metadata if defined?(Collection) && !hyrax_record.is_a?(Collection)
       build_relationship_metadata
       build_mapping_metadata
       self.save!
@@ -157,26 +194,48 @@ module Bulkrax
       end
     end
 
+    # The purpose of this helper module is to make easier the testing of the rather complex
+    # switching logic for determining the method we use for building the value.
+    module AttributeBuilderMethod
+      # @param key [Symbol]
+      # @param value [Hash<String, Object>]
+      # @param entry [Bulkrax::Entry]
+      #
+      # @return [NilClass] when we won't be processing this field
+      # @return [Symbol] (either :build_value or :build_object)
+      def self.for(key:, value:, entry:)
+        return if key == 'model'
+        return if key == 'file'
+        return if key == entry.related_parents_parsed_mapping
+        return if key == entry.related_children_parsed_mapping
+        return if value['excluded'] || value[:excluded]
+        return if Bulkrax.reserved_properties.include?(key) && !entry.field_supported?(key)
+
+        object_key = key if value.key?('object') || value.key?(:object)
+        return unless entry.hyrax_record.respond_to?(key.to_s) || object_key.present?
+
+        models_to_skip = Array.wrap(value['skip_object_for_model_names'] || value[:skip_object_for_model_names] || [])
+
+        return :build_value if models_to_skip.detect { |model| entry.factory_class.model_name.name == model }
+        return :build_object if object_key.present?
+
+        :build_value
+      end
+    end
+
     def build_mapping_metadata
       mapping = fetch_field_mapping
       mapping.each do |key, value|
-        # these keys are handled by other methods
-        next if ['model', 'file', related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
-        next if value['excluded']
-        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
-
-        object_key = key if value.key?('object')
-        next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
+        method_name = AttributeBuilderMethod.for(key: key, value: value, entry: self)
+        next unless method_name
 
-        if object_key.present?
-          build_object(value)
-        else
-          build_value(key, value)
-        end
+        send(method_name, key, value)
       end
     end
 
-    def build_object(value)
+    def build_object(_key, value)
+      return unless hyrax_record.respond_to?(value['object'])
+
       data = hyrax_record.send(value['object'])
       return if data.empty?
 
@@ -185,6 +244,8 @@ module Bulkrax
     end
 
     def build_value(key, value)
+      return unless hyrax_record.respond_to?(key.to_s)
+
       data = hyrax_record.send(key.to_s)
       if data.is_a?(ActiveTriples::Relation)
         if value['join']
@@ -217,6 +278,14 @@ module Bulkrax
     end
 
     def object_metadata(data)
+      # NOTE: What is `d` in this case:
+      #
+      #  "[{\"single_object_first_name\"=>\"Fake\", \"single_object_last_name\"=>\"Fakerson\", \"single_object_position\"=>\"Leader, Jester, Queen\", \"single_object_language\"=>\"english\"}]"
+      #
+      # The above is a stringified version of a Ruby string.  Using eval is a very bad idea as it
+      # will execute the value of `d` within the full Ruby interpreter context.
+      #
+      # TODO: Would it be possible to store this as a non-string?  Maybe the actual Ruby Array and Hash?
       data = data.map { |d| eval(d) }.flatten # rubocop:disable Security/Eval
 
       data.each_with_index do |obj, index|

data/app/models/bulkrax/exporter.rb

@@ -18,18 +18,9 @@ module Bulkrax
 
     def export
       current_run && setup_export_path
-      case self.export_from
-      when 'collection'
-        create_from_collection
-      when 'importer'
-        create_from_importer
-      when 'worktype'
-        create_from_worktype
-      when 'all'
-        create_from_all
-      end
+      send("create_from_#{self.export_from}")
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     end
 
     # #export_source accessors
@@ -69,7 +60,7 @@ module Bulkrax
     end
 
     def workflow_status_list
-      Sipity::WorkflowState.all.map { |s| [s.name&.titleize, s.name] }.uniq
+      Sipity::WorkflowState.all.map { |s| [s.name&.titleize, s.name] }.uniq if defined?(::Hyrax)
     end
 
     # If field_mapping is empty, setup a default based on the export_properties
@@ -84,12 +75,20 @@ module Bulkrax
     end
 
     def export_from_list
-      [
-        [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
-        [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
-        [I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
-        [I18n.t('bulkrax.exporter.labels.all'), 'all']
-      ]
+      if defined?(::Hyrax)
+        [
+          [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
+          [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
+          [I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
+          [I18n.t('bulkrax.exporter.labels.all'), 'all']
+        ]
+      else
+        [
+          [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
+          [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
+          [I18n.t('bulkrax.exporter.labels.all'), 'all']
+        ]
+      end
     end
 
     def export_type_list
@@ -131,7 +130,7 @@ module Bulkrax
     end
 
     def export_properties
-      properties = Hyrax.config.registered_curation_concern_types.map { |work| work.constantize.properties.keys }.flatten.uniq.sort
+      properties = Bulkrax.curation_concerns.map { |work| work.properties.keys }.flatten.uniq.sort
       properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
     end
 

data/app/models/bulkrax/importer.rb

@@ -15,7 +15,7 @@ module Bulkrax
     has_many :entries, as: :importerexporter, dependent: :destroy
 
     validates :name, presence: true
-    validates :admin_set_id, presence: true
+    validates :admin_set_id, presence: true if defined?(::Hyrax)
     validates :parser_klass, presence: true
 
     delegate :valid_import?, :write_errored_entries_file, :visibility, to: :parser
@@ -47,12 +47,12 @@ module Bulkrax
       if importer_run.failed_records.positive?
         if importer_run.invalid_records.present?
           e = Bulkrax::ImportFailed.new('Failed with Invalid Records', importer_run.invalid_records.split("\n"))
-          importer_run.importer.status_info(e)
+          importer_run.importer.set_status_info(e)
         else
-          importer_run.importer.status_info('Complete (with failures)')
+          importer_run.importer.set_status_info('Complete (with failures)')
         end
       else
-        importer_run.importer.status_info('Complete')
+        importer_run.importer.set_status_info('Complete')
       end
     end
 
@@ -160,7 +160,7 @@ module Bulkrax
       types = types_array || DEFAULT_OBJECT_TYPES
       parser.create_objects(types)
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     end
 
     # Prepend the base_url to ensure unique set identifiers

data/app/models/bulkrax/importer_run.rb

@@ -9,5 +9,11 @@ module Bulkrax
     def parents
       pending_relationships.pluck(:parent_id).uniq
     end
+
+    def user
+      # An importer might not have a user, the CLI ingest need not assign a user.  As such, we
+      # fallback to the configured user.
+      importer.user || Bulkrax.fallback_user_for_importer_exporter_processing
+    end
   end
 end

data/app/models/bulkrax/oai_entry.rb

@@ -9,6 +9,15 @@ module Bulkrax
 
     delegate :record, to: :raw_record
 
+    # @api private
+    #
+    # Included to assist in testing; namely so that you can copy down an OAI entry, store it locally,
+    # and then manually construct an {OAI::GetRecordResponse}.
+    #
+    # @see Bulkrax::EntrySpecHelper.oai_entry_for
+    attr_writer :raw_record
+
+    # @return [OAI::GetRecordResponse]
     def raw_record
       @raw_record ||= client.get_record(identifier: identifier, metadata_prefix: parser.parser_fields['metadata_prefix'])
     end
@@ -28,7 +37,7 @@ module Bulkrax
     def build_metadata
       self.parsed_metadata = {}
       self.parsed_metadata[work_identifier] = [record.header.identifier]
-      self.raw_metadata = { xml: record.metadata.to_s }
+      self.raw_metadata = { record: record.metadata.to_s, header: record.header.to_s }
 
       # We need to establish the #factory_class before we proceed with the metadata.  See
       # https://github.com/samvera-labs/bulkrax/issues/702 for further details.
@@ -91,7 +100,8 @@ module Bulkrax
     # If OAI-PMH doesn't return setSpec in the headers for GetRecord, use parser.collection_name
     #   in this case, if 'All' is selected, records will not be added to a collection.
     def find_collection_ids
-      return self.collection_ids if collections_created?
+      return self.collection_ids if defined?(@called_find_collection_ids)
+
       if sets.blank? || parser.collection_name != 'all'
         collection = find_collection(importerexporter.unique_collection_identifier(parser.collection_name))
         self.collection_ids << collection.id if collection.present? && !self.collection_ids.include?(collection.id)
@@ -101,6 +111,8 @@ module Bulkrax
           self.collection_ids << c.id if c.present? && !self.collection_ids.include?(c.id)
         end
       end
+
+      @called_find_collection_ids = true
       return self.collection_ids
     end
   end

data/app/models/bulkrax/pending_relationship.rb

@@ -3,5 +3,9 @@
 module Bulkrax
   class PendingRelationship < ApplicationRecord
     belongs_to :importer_run
+
+    # Ideally we wouldn't have a column named "order", as it is a reserved SQL term.  However, if we
+    # quote the column, all is well...for the application.
+    scope :ordered, -> { order(Arel.sql("#{quoted_table_name}.#{connection.quote_column_name('order')}")) }
   end
 end

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Bulkrax
+  # TODO: Extract methods to class methods; there's no reason for these methods to be a mixin.
+  # TODO: Add specs to test in isolation
   module DynamicRecordLookup
     # Search entries, collections, and every available work type for a record that
     # has the provided identifier.
@@ -40,7 +42,7 @@ module Bulkrax
       @available_work_types ||= if defined?(::Hyku)
                                   ::Site.instance.available_works.map(&:constantize)
                                 else
-                                  ::Hyrax.config.curation_concerns
+                                  Bulkrax.curation_concerns
                                 end
     end
   end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'marcel'
+
 module Bulkrax
   module ExportBehavior
     extend ActiveSupport::Concern
@@ -10,9 +12,9 @@ module Bulkrax
     rescue RSolr::Error::Http, CollectionsCreatedError => e
       raise e
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     else
-      status_info
+      set_status_info
     end
 
     def build_export_metadata
@@ -27,8 +29,8 @@ module Bulkrax
     def filename(file_set)
       return if file_set.original_file.blank?
       fn = file_set.original_file.file_name.first
-      mime = Mime::Type.lookup(file_set.original_file.mime_type)
-      ext_mime = MIME::Types.of(file_set.original_file.file_name).first
+      mime = ::Marcel::MimeType.for(file_set.original_file.mime_type)
+      ext_mime = ::Marcel::MimeType.for(file_set.original_file.file_name)
       if fn.include?(file_set.id) || importerexporter.metadata_only?
         filename = "#{fn}.#{mime.to_sym}"
         filename = fn if mime.to_s == ext_mime.to_s

data/app/models/concerns/bulkrax/has_matchers.rb

@@ -147,6 +147,7 @@ module Bulkrax
         %W[
           file
           remote_files
+          rights_statement
           #{related_parents_parsed_mapping}
           #{related_children_parsed_mapping}
         ]

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -11,16 +11,16 @@ module Bulkrax
         unless self.importerexporter.validate_only
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
-          add_user_to_permission_templates! if self.class.to_s.include?("Collection")
+          add_user_to_permission_templates! if self.class.to_s.include?("Collection") && defined?(::Hyrax)
           parent_jobs if self.parsed_metadata[related_parents_parsed_mapping]&.join.present?
           child_jobs if self.parsed_metadata[related_children_parsed_mapping]&.join.present?
         end
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e
       rescue StandardError => e
-        status_info(e)
+        set_status_info(e)
       else
-        status_info
+        set_status_info
       ensure
         self.save!
       end
@@ -93,6 +93,8 @@ module Bulkrax
     end
 
     def add_admin_set_id
+      return unless defined?(::Hyrax)
+
       self.parsed_metadata['admin_set_id'] = importerexporter.admin_set_id if self.parsed_metadata['admin_set_id'].blank?
     end
 
@@ -165,6 +167,7 @@ module Bulkrax
     # @param field [String] name of the controlled property
     # @return [Boolean] provided value is a present, active authority ID for the provided field
     def active_id_for_authority?(value, field)
+      return false unless defined?(::Hyrax)
       field_service = ('Hyrax::' + "#{field}_service".camelcase).constantize
       active_authority_ids = field_service.new.active_elements.map { |ae| ae['id'] }
 

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'marcel'
 
 module Bulkrax
   module ImporterExporterBehavior
@@ -50,7 +51,14 @@ module Bulkrax
 
     # Is this a zip file?
     def zip?
-      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
+      filename = parser_fields&.[]('import_file_path')
+      return false unless filename
+      return false unless File.file?(filename)
+      returning_value = false
+      File.open(filename) do |file|
+        returning_value = ::Marcel::MimeType.for(file).include?('application/zip')
+      end
+      returning_value
     end
   end
 end

data/app/models/concerns/bulkrax/status_info.rb

@@ -33,16 +33,21 @@ module Bulkrax
       current_status&.created_at
     end
 
-    def status_info(e = nil, current_run = nil)
+    def set_status_info(e = nil, current_run = nil)
+      runnable = current_run || last_run
       if e.nil?
-        self.statuses.create!(status_message: 'Complete', runnable: current_run || last_run)
+        self.statuses.create!(status_message: 'Complete', runnable: runnable)
       elsif e.is_a?(String)
-        self.statuses.create!(status_message: e, runnable: current_run || last_run)
+        self.statuses.create!(status_message: e, runnable: runnable)
       else
-        self.statuses.create!(status_message: 'Failed', runnable: current_run || last_run, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
+        self.statuses.create!(status_message: 'Failed', runnable: runnable, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
       end
     end
 
+    alias status_info set_status_info
+
+    deprecation_deprecate status_info: "Favor Bulkrax::StatusInfo.set_status_info.  We will be removing .status_info in Bulkrax v6.0.0"
+
     # api compatible with previous error structure
     def last_error
       return unless current_status && current_status.error_class.present?

data/app/parsers/bulkrax/application_parser.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'zip'
+require 'marcel'
 
 module Bulkrax
   # An abstract class that establishes the API for Bulkrax's import and export parsing.
@@ -10,10 +11,11 @@ module Bulkrax
     alias importer importerexporter
     alias exporter importerexporter
     delegate :only_updates, :limit, :current_run, :errors, :mapping,
-      :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
-      :key_without_numbers, :status, :status_info, :status_at,
-      :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
-      to: :importerexporter
+             :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
+             :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
+             :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
+             :zip?, :file?,
+             to: :importerexporter
 
     # @todo Convert to `class_attribute :parser_fiels, default: {}`
     def self.parser_fields
@@ -275,10 +277,16 @@ module Bulkrax
 
     # @return [Array<String>]
     def required_elements
+      matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
+      unless matched_elements.count == Bulkrax.required_elements.count
+        missing_elements = Bulkrax.required_elements - matched_elements
+        error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
+        raise StandardError, error_alert
+      end
       if Bulkrax.fill_in_blank_source_identifiers
-        ['title']
+        Bulkrax.required_elements
       else
-        ['title', source_identifier]
+        Bulkrax.required_elements + [source_identifier]
       end
     end
 
@@ -351,16 +359,6 @@ module Bulkrax
       end
     end
 
-    # Is this a file?
-    def file?
-      parser_fields&.[]('import_file_path') && File.file?(parser_fields['import_file_path'])
-    end
-
-    # Is this a zip file?
-    def zip?
-      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
-    end
-
     # Path for the import
     # @return [String]
     def import_file_path

data/app/parsers/bulkrax/bagit_parser.rb

@@ -11,7 +11,7 @@ module Bulkrax
     def valid_import?
       return true if import_fields.present?
     rescue => e
-      status_info(e)
+      set_status_info(e)
       false
     end
 
@@ -51,7 +51,7 @@ module Bulkrax
           record_data = entry_class.data_for_entry(data_row, source_identifier, self)
           next record_data if importerexporter.metadata_only?
 
-          record_data[:file] = bag.bag_files.join('|') if ::Hyrax.config.curation_concerns.include? record_data[:model]&.constantize
+          record_data[:file] = bag.bag_files.join('|') if Bulkrax.curation_concerns.include? record_data[:model]&.constantize
           record_data
         end
       else
@@ -82,19 +82,7 @@ module Bulkrax
       end
       importer.record_status
     rescue StandardError => e
-      status_info(e)
-    end
-
-    def total
-      @total = importer.parser_fields['total'] || 0 if importer?
-
-      @total = if exporter?
-                 limit.nil? || limit.zero? ? current_record_ids.count : limit
-               end
-
-      return @total || 0
-    rescue StandardError
-      @total = 0
+      set_status_info(e)
     end
 
     # export methods
@@ -143,8 +131,8 @@ module Bulkrax
           begin
             bag.add_file(file_name, file.path) if bag.bag_files.select { |b| b.include?(file_name) }.blank?
           rescue => e
-            entry.status_info(e)
-            status_info(e)
+            entry.set_status_info(e)
+            set_status_info(e)
           end
         end
 
@@ -185,6 +173,7 @@ module Bulkrax
       File.join(path, id)
     end
 
+    # @todo(bjustice) - remove hyrax reference
     def write_triples(folder_count, e)
       sd = SolrDocument.find(e.identifier)
       return if sd.nil?