bulkrax 5.1.0 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 490e0f170cc1128c00c467c3cc344db627d027a3b857d53dfa33b97805567d4b
-  data.tar.gz: 7290801bacea707b7398e674a17acf56e7a770cfb3bea20958169588a4404175
+  metadata.gz: c9794c69e891f2397ac94398676f49db008e83b7a6df16e7a08f0ab393c148ca
+  data.tar.gz: fd2d48507add6bcbc7557f9240951d174c488e456579af65e20bb6cdc6f3c080
 SHA512:
-  metadata.gz: a6f5486405e2d2eb7f6c0c49b17ed0926e55a701368e42c93db9b009a5f663682ec4141fe1dd58d0dde132fa747010ada7cd22187b81d60a7e8b6b23cbf2e24d
-  data.tar.gz: d56a8780ef074d412ac7406d3f3ddb39b34b17bbe43c61a64bebe06f39952a1d62cbf2dbd01a7012bcd776686def03c5a5e6c556cc1384284c2dea8a89f3eec2
+  metadata.gz: 3b07a4178650201c48602c9bb9df29135a5cc51bc6954355732bd6ae5fa918c5fd4c88d187e305b7ee2877ca995818d7ec3a476e92ec173c7569c0a15ca229a2
+  data.tar.gz: 324393aeb341d82b23e6391cbe1ec2a64d3e55a61400330f1656f46b3bb22cd0c6e7335cfeaf1b3b12e6ca5faefea5405b96e1100b1df138dd4c5a87fd522d76

data/app/controllers/bulkrax/importers_controller.rb

@@ -35,11 +35,10 @@ module Bulkrax
       elsif defined?(::Hyrax)
         add_importer_breadcrumbs
         add_breadcrumb @importer.name
-
-        @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
-        @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
-        @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
       end
+      @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
+      @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
+      @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
     end
 
     # GET /importers/new

data/app/helpers/bulkrax/validation_helper.rb

@@ -5,10 +5,8 @@ module Bulkrax
     def valid_create_params?
       check_admin_set
       check_user
-      return true if valid_importer? &&
-                     valid_commit? &&
-                     valid_name? &&
-                     valid_parser_klass? &&
+      return true if valid_importer? && valid_commit? &&
+                     valid_name? && valid_parser_klass? &&
                      valid_parser_fields?
     end
 
@@ -19,6 +17,8 @@ module Bulkrax
     end
 
     def check_admin_set
+      return unless defined?(::Hyrax)
+
       if params[:importer][:admin_set_id].blank?
         params[:importer][:admin_set_id] = AdminSet::DEFAULT_ID
       else

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -16,6 +16,8 @@ module Bulkrax
   #       to either an instance of a Work or an instance of a Collection.
   # NOTE: In the context of this job, "identifier" is used to generically refer
   #       to either a record's ID or an Bulkrax::Entry's source_identifier.
+  # Please override with your own job for custom/non-hyrax applications
+  # set Bulkrax config variable :relationship_job to your custom class
   class CreateRelationshipsJob < ApplicationJob
     ##
     # @api public
@@ -112,6 +114,7 @@ module Bulkrax
     end
 
     def add_to_collection(child_record, parent_record)
+      parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
       child_record.member_of_collections << parent_record
       child_record.save!
     end

data/app/jobs/bulkrax/import_work_job.rb

@@ -5,24 +5,37 @@ module Bulkrax
     queue_as :import
 
     # rubocop:disable Rails/SkipsModelValidations
+    #
+    # @note Yes, we are calling {ImporterRun.find} each time.  these were on purpose to prevent race
+    #       conditions on the database update. If you do not re-find (or at least reload) the object
+    #       on each increment, the count can get messed up. Let's say there are two jobs A and B and
+    #       a counter set to 2.
+    #
+    #       - A grabs the importer_run (line 10)
+    #       - B grabs the importer_run (line 10)
+    #       - A Finishes the build, does the increment (now the counter is 3)
+    #       - B Finishes the build, does the increment (now the counter is 3 again) and thus a count
+    #         is lost.
+    #
+    # @see https://codingdeliberately.com/activerecord-increment/
+    # @see https://github.com/samvera-labs/bulkrax/commit/5c2c795452e13a98c9217fdac81ae2f5aea031a0#r105848236
     def perform(entry_id, run_id, time_to_live = 3, *)
       entry = Entry.find(entry_id)
-      importer_run = ImporterRun.find(run_id)
       entry.build
       if entry.status == "Complete"
-        importer_run.increment!(:processed_records)
-        importer_run.increment!(:processed_works)
+        ImporterRun.find(run_id).increment!(:processed_records)
+        ImporterRun.find(run_id).increment!(:processed_works)
       else
         # do not retry here because whatever parse error kept you from creating a work will likely
         # keep preventing you from doing so.
-        importer_run.increment!(:failed_records)
-        importer_run.increment!(:failed_works)
+        ImporterRun.find(run_id).increment!(:failed_records)
+        ImporterRun.find(run_id).increment!(:failed_works)
       end
       # Regardless of completion or not, we want to decrement the enqueued records.
-      importer_run.decrement!(:enqueued_records) unless importer_run.enqueued_records <= 0
+      ImporterRun.find(run_id).decrement!(:enqueued_records) unless ImporterRun.find(run_id).enqueued_records <= 0
 
       entry.save!
-      entry.importer.current_run = importer_run
+      entry.importer.current_run = ImporterRun.find(run_id)
       entry.importer.record_status
     rescue Bulkrax::CollectionsCreatedError => e
       Rails.logger.warn("#{self.class} entry_id: #{entry_id}, run_id: #{run_id} encountered #{e.class}: #{e.message}")

data/app/jobs/bulkrax/importer_job.rb

@@ -12,7 +12,7 @@ module Bulkrax
       import(importer, only_updates_since_last_import)
       update_current_run_counters(importer)
       schedule(importer) if importer.schedulable?
-    rescue CSV::MalformedCSVError => e
+    rescue ::CSV::MalformedCSVError => e
       importer.set_status_info(e)
     end
 

data/app/jobs/bulkrax/schedule_relationships_job.rb

@@ -9,7 +9,8 @@ module Bulkrax
       return reschedule(importer_id) unless pending_num.zero?
 
       importer.last_run.parents.each do |parent_id|
-        CreateRelationshipsJob.perform_later(parent_identifier: parent_id, importer_run_id: importer.last_run.id)
+        Bulkrax.relationship_job_class.constantize.perform_later(parent_identifier: parent_id,
+                                                                 importer_run_id: importer.last_run.id)
       end
     end
 

data/app/matchers/bulkrax/application_matcher.rb

@@ -102,6 +102,7 @@ module Bulkrax
 
     # Only add valid resource types
     def parse_resource_type(src)
+      ActiveSupport::Deprecation.warn('#parse_resource_type will be removed in Bulkrax v6.0.0')
       Hyrax::ResourceTypesService.label(src.to_s.strip.titleize)
     rescue KeyError
       nil

data/app/models/bulkrax/csv_entry.rb

@@ -13,15 +13,49 @@ module Bulkrax
       data.headers.flatten.compact.uniq
     end
 
+    class_attribute(:csv_read_data_options, default: {})
+
     # there's a risk that this reads the whole file into memory and could cause a memory leak
     def self.read_data(path)
       raise StandardError, 'CSV path empty' if path.blank?
-      CSV.read(path,
+      options = {
         headers: true,
         header_converters: ->(h) { h.to_sym },
-        encoding: 'utf-8')
+        encoding: 'utf-8'
+      }.merge(csv_read_data_options)
+
+      results = CSV.read(path, **options)
+      csv_wrapper_class.new(results)
     end
 
+    # The purpose of this class is to reject empty lines.  This causes lots of grief in importing.
+    # But why not use {CSV.read}'s `skip_lines` option?  Because for some CSVs, it will never finish
+    # reading the file.
+    #
+    # There is a spec that demonstrates this approach works.
+    class CsvWrapper
+      include Enumerable
+      def initialize(original)
+        @original = original
+      end
+
+      delegate :headers, to: :@original
+
+      def each
+        @original.each do |row|
+          next if all_fields_are_empty_for(row: row)
+          yield(row)
+        end
+      end
+
+      private
+
+      def all_fields_are_empty_for(row:)
+        row.to_hash.values.all?(&:blank?)
+      end
+    end
+    class_attribute :csv_wrapper_class, default: CsvWrapper
+
     def self.data_for_entry(data, _source_id, parser)
       # If a multi-line CSV data is passed, grab the first row
       data = data.first if data.is_a?(CSV::Table)
@@ -35,11 +69,7 @@ module Bulkrax
     end
 
     def build_metadata
-      raise StandardError, 'Record not found' if record.nil?
-      unless importerexporter.parser.required_elements?(keys_without_numbers(record.keys))
-        raise StandardError,
-"Missing required elements, missing element(s) are: #{importerexporter.parser.missing_elements(keys_without_numbers(record.keys)).join(', ')}"
-      end
+      validate_record
 
       self.parsed_metadata = {}
       add_identifier
@@ -56,6 +86,12 @@ module Bulkrax
       self.parsed_metadata
     end
 
+    def validate_record
+      raise StandardError, 'Record not found' if record.nil?
+      raise StandardError, "Missing required elements, missing element(s) are: "\
+"#{importerexporter.parser.missing_elements(record).join(', ')}" unless importerexporter.parser.required_elements?(record)
+    end
+
     def add_identifier
       self.parsed_metadata[work_identifier] = [record[source_identifier]]
     end
@@ -67,9 +103,10 @@ module Bulkrax
     end
 
     def add_metadata_for_model
-      if factory_class == Collection
-        add_collection_type_gid
-      elsif factory_class == FileSet
+      if defined?(::Collection) && factory_class == ::Collection
+        add_collection_type_gid if defined?(::Hyrax)
+        # add any additional collection metadata methods here
+      elsif factory_class == Bulkrax.file_model_class
         validate_presence_of_filename!
         add_path_to_file
         validate_presence_of_parent!
@@ -106,7 +143,7 @@ module Bulkrax
       self.parsed_metadata = {}
 
       build_system_metadata
-      build_files_metadata unless hyrax_record.is_a?(Collection)
+      build_files_metadata if defined?(Collection) && !hyrax_record.is_a?(Collection)
       build_relationship_metadata
       build_mapping_metadata
       self.save!
@@ -157,26 +194,48 @@ module Bulkrax
       end
     end
 
+    # The purpose of this helper module is to make easier the testing of the rather complex
+    # switching logic for determining the method we use for building the value.
+    module AttributeBuilderMethod
+      # @param key [Symbol]
+      # @param value [Hash<String, Object>]
+      # @param entry [Bulkrax::Entry]
+      #
+      # @return [NilClass] when we won't be processing this field
+      # @return [Symbol] (either :build_value or :build_object)
+      def self.for(key:, value:, entry:)
+        return if key == 'model'
+        return if key == 'file'
+        return if key == entry.related_parents_parsed_mapping
+        return if key == entry.related_children_parsed_mapping
+        return if value['excluded'] || value[:excluded]
+        return if Bulkrax.reserved_properties.include?(key) && !entry.field_supported?(key)
+
+        object_key = key if value.key?('object') || value.key?(:object)
+        return unless entry.hyrax_record.respond_to?(key.to_s) || object_key.present?
+
+        models_to_skip = Array.wrap(value['skip_object_for_model_names'] || value[:skip_object_for_model_names] || [])
+
+        return :build_value if models_to_skip.detect { |model| entry.factory_class.model_name.name == model }
+        return :build_object if object_key.present?
+
+        :build_value
+      end
+    end
+
     def build_mapping_metadata
       mapping = fetch_field_mapping
       mapping.each do |key, value|
-        # these keys are handled by other methods
-        next if ['model', 'file', related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
-        next if value['excluded']
-        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
-
-        object_key = key if value.key?('object')
-        next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
+        method_name = AttributeBuilderMethod.for(key: key, value: value, entry: self)
+        next unless method_name
 
-        if object_key.present?
-          build_object(value)
-        else
-          build_value(key, value)
-        end
+        send(method_name, key, value)
       end
     end
 
-    def build_object(value)
+    def build_object(_key, value)
+      return unless hyrax_record.respond_to?(value['object'])
+
       data = hyrax_record.send(value['object'])
       return if data.empty?
 
@@ -185,6 +244,8 @@ module Bulkrax
     end
 
     def build_value(key, value)
+      return unless hyrax_record.respond_to?(key.to_s)
+
       data = hyrax_record.send(key.to_s)
       if data.is_a?(ActiveTriples::Relation)
         if value['join']
@@ -217,6 +278,14 @@ module Bulkrax
     end
 
     def object_metadata(data)
+      # NOTE: What is `d` in this case:
+      #
+      #  "[{\"single_object_first_name\"=>\"Fake\", \"single_object_last_name\"=>\"Fakerson\", \"single_object_position\"=>\"Leader, Jester, Queen\", \"single_object_language\"=>\"english\"}]"
+      #
+      # The above is a stringified version of a Ruby string.  Using eval is a very bad idea as it
+      # will execute the value of `d` within the full Ruby interpreter context.
+      #
+      # TODO: Would it be possible to store this as a non-string?  Maybe the actual Ruby Array and Hash?
       data = data.map { |d| eval(d) }.flatten # rubocop:disable Security/Eval
 
       data.each_with_index do |obj, index|

data/app/models/bulkrax/exporter.rb

@@ -18,18 +18,9 @@ module Bulkrax
 
     def export
       current_run && setup_export_path
-      case self.export_from
-      when 'collection'
-        create_from_collection
-      when 'importer'
-        create_from_importer
-      when 'worktype'
-        create_from_worktype
-      when 'all'
-        create_from_all
-      end
+      send("create_from_#{self.export_from}")
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     end
 
     # #export_source accessors
@@ -139,7 +130,7 @@ module Bulkrax
     end
 
     def export_properties
-      properties = Hyrax.config.registered_curation_concern_types.map { |work| work.constantize.properties.keys }.flatten.uniq.sort
+      properties = Bulkrax.curation_concerns.map { |work| work.properties.keys }.flatten.uniq.sort
       properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
     end
 

data/app/models/bulkrax/importer.rb

@@ -160,7 +160,7 @@ module Bulkrax
       types = types_array || DEFAULT_OBJECT_TYPES
       parser.create_objects(types)
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     end
 
     # Prepend the base_url to ensure unique set identifiers

data/app/models/bulkrax/pending_relationship.rb

@@ -6,6 +6,6 @@ module Bulkrax
 
     # Ideally we wouldn't have a column named "order", as it is a reserved SQL term.  However, if we
     # quote the column, all is well...for the application.
-    scope :ordered, -> { order("#{quoted_table_name}.#{connection.quote_column_name('order')}") }
+    scope :ordered, -> { order(Arel.sql("#{quoted_table_name}.#{connection.quote_column_name('order')}")) }
   end
 end

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -42,7 +42,7 @@ module Bulkrax
       @available_work_types ||= if defined?(::Hyku)
                                   ::Site.instance.available_works.map(&:constantize)
                                 else
-                                  ::Hyrax.config.curation_concerns
+                                  Bulkrax.curation_concerns
                                 end
     end
   end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'marcel'
+
 module Bulkrax
   module ExportBehavior
     extend ActiveSupport::Concern
@@ -10,9 +12,9 @@ module Bulkrax
     rescue RSolr::Error::Http, CollectionsCreatedError => e
       raise e
     rescue StandardError => e
-      status_info(e)
+      set_status_info(e)
     else
-      status_info
+      set_status_info
     end
 
     def build_export_metadata
@@ -27,8 +29,8 @@ module Bulkrax
     def filename(file_set)
       return if file_set.original_file.blank?
       fn = file_set.original_file.file_name.first
-      mime = Mime::Type.lookup(file_set.original_file.mime_type)
-      ext_mime = MIME::Types.of(file_set.original_file.file_name).first
+      mime = ::Marcel::MimeType.for(file_set.original_file.mime_type)
+      ext_mime = ::Marcel::MimeType.for(file_set.original_file.file_name)
       if fn.include?(file_set.id) || importerexporter.metadata_only?
         filename = "#{fn}.#{mime.to_sym}"
         filename = fn if mime.to_s == ext_mime.to_s

data/app/models/concerns/bulkrax/has_matchers.rb

@@ -147,6 +147,7 @@ module Bulkrax
         %W[
           file
           remote_files
+          rights_statement
           #{related_parents_parsed_mapping}
           #{related_children_parsed_mapping}
         ]

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -11,16 +11,16 @@ module Bulkrax
         unless self.importerexporter.validate_only
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
-          add_user_to_permission_templates! if self.class.to_s.include?("Collection")
+          add_user_to_permission_templates! if self.class.to_s.include?("Collection") && defined?(::Hyrax)
           parent_jobs if self.parsed_metadata[related_parents_parsed_mapping]&.join.present?
           child_jobs if self.parsed_metadata[related_children_parsed_mapping]&.join.present?
         end
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e
       rescue StandardError => e
-        status_info(e)
+        set_status_info(e)
       else
-        status_info
+        set_status_info
       ensure
         self.save!
       end
@@ -93,6 +93,8 @@ module Bulkrax
     end
 
     def add_admin_set_id
+      return unless defined?(::Hyrax)
+
       self.parsed_metadata['admin_set_id'] = importerexporter.admin_set_id if self.parsed_metadata['admin_set_id'].blank?
     end
 
@@ -165,6 +167,7 @@ module Bulkrax
     # @param field [String] name of the controlled property
     # @return [Boolean] provided value is a present, active authority ID for the provided field
     def active_id_for_authority?(value, field)
+      return false unless defined?(::Hyrax)
       field_service = ('Hyrax::' + "#{field}_service".camelcase).constantize
       active_authority_ids = field_service.new.active_elements.map { |ae| ae['id'] }
 

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'marcel'
 
 module Bulkrax
   module ImporterExporterBehavior
@@ -50,7 +51,14 @@ module Bulkrax
 
     # Is this a zip file?
     def zip?
-      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
+      filename = parser_fields&.[]('import_file_path')
+      return false unless filename
+      return false unless File.file?(filename)
+      returning_value = false
+      File.open(filename) do |file|
+        returning_value = ::Marcel::MimeType.for(file).include?('application/zip')
+      end
+      returning_value
     end
   end
 end

data/app/parsers/bulkrax/application_parser.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'zip'
+require 'marcel'
 
 module Bulkrax
   # An abstract class that establishes the API for Bulkrax's import and export parsing.
@@ -10,10 +11,11 @@ module Bulkrax
     alias importer importerexporter
     alias exporter importerexporter
     delegate :only_updates, :limit, :current_run, :errors, :mapping,
-      :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
-      :key_without_numbers, :status, :status_info, :status_at,
-      :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
-      to: :importerexporter
+             :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
+             :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
+             :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
+             :zip?, :file?,
+             to: :importerexporter
 
     # @todo Convert to `class_attribute :parser_fiels, default: {}`
     def self.parser_fields
@@ -275,10 +277,16 @@ module Bulkrax
 
     # @return [Array<String>]
     def required_elements
+      matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
+      unless matched_elements.count == Bulkrax.required_elements.count
+        missing_elements = Bulkrax.required_elements - matched_elements
+        error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
+        raise StandardError, error_alert
+      end
       if Bulkrax.fill_in_blank_source_identifiers
-        ['title']
+        Bulkrax.required_elements
       else
-        ['title', source_identifier]
+        Bulkrax.required_elements + [source_identifier]
       end
     end
 
@@ -351,16 +359,6 @@ module Bulkrax
       end
     end
 
-    # Is this a file?
-    def file?
-      parser_fields&.[]('import_file_path') && File.file?(parser_fields['import_file_path'])
-    end
-
-    # Is this a zip file?
-    def zip?
-      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
-    end
-
     # Path for the import
     # @return [String]
     def import_file_path

data/app/parsers/bulkrax/bagit_parser.rb

@@ -11,7 +11,7 @@ module Bulkrax
     def valid_import?
       return true if import_fields.present?
     rescue => e
-      status_info(e)
+      set_status_info(e)
       false
     end
 
@@ -51,7 +51,7 @@ module Bulkrax
           record_data = entry_class.data_for_entry(data_row, source_identifier, self)
           next record_data if importerexporter.metadata_only?
 
-          record_data[:file] = bag.bag_files.join('|') if ::Hyrax.config.curation_concerns.include? record_data[:model]&.constantize
+          record_data[:file] = bag.bag_files.join('|') if Bulkrax.curation_concerns.include? record_data[:model]&.constantize
           record_data
         end
       else
@@ -82,19 +82,7 @@ module Bulkrax
       end
       importer.record_status
     rescue StandardError => e
-      status_info(e)
-    end
-
-    def total
-      @total = importer.parser_fields['total'] || 0 if importer?
-
-      @total = if exporter?
-                 limit.nil? || limit.zero? ? current_record_ids.count : limit
-               end
-
-      return @total || 0
-    rescue StandardError
-      @total = 0
+      set_status_info(e)
     end
 
     # export methods
@@ -144,7 +132,7 @@ module Bulkrax
             bag.add_file(file_name, file.path) if bag.bag_files.select { |b| b.include?(file_name) }.blank?
           rescue => e
             entry.set_status_info(e)
-            status_info(e)
+            set_status_info(e)
           end
         end
 
@@ -185,6 +173,7 @@ module Bulkrax
       File.join(path, id)
     end
 
+    # @todo(bjustice) - remove hyrax reference
     def write_triples(folder_count, e)
       sd = SolrDocument.find(e.identifier)
       return if sd.nil?