bulkrax 1.0.2 → 2.1.0

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module DynamicRecordLookup
+    # Search entries, collections, and every available work type for a record that
+    # has the provided identifier.
+    #
+    # @param identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
+    # @return [Work, Collection, nil] Work or Collection if found, otherwise nil
+    def find_record(identifier)
+      record = Entry.find_by(identifier: identifier)
+      record ||= ::Collection.where(id: identifier).first # rubocop:disable Rails/FindBy
+      if record.blank?
+        available_work_types.each do |work_type|
+          record ||= work_type.where(id: identifier).first # rubocop:disable Rails/FindBy
+        end
+      end
+
+      record.is_a?(Entry) ? record.factory.find : record
+    end
+
+    # Check if the record is a Work
+    def curation_concern?(record)
+      available_work_types.include?(record.class)
+    end
+
+    private
+
+    # @return [Array<Class>] list of work type classes
+    def available_work_types
+      # If running in a Hyku app, do not include disabled work types
+      @available_work_types ||= if defined?(::Hyku)
+                                  ::Site.instance.available_works.map(&:constantize)
+                                else
+                                  ::Hyrax.config.curation_concerns
+                                end
+    end
+  end
+end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -40,13 +40,13 @@ module Bulkrax
       end
     end
 
-    # Append the file_set id to ensure a unique filename
+    # Prepend the file_set id to ensure a unique filename
     def filename(file_set)
       return if file_set.original_file.blank?
       fn = file_set.original_file.file_name.first
       mime = Mime::Type.lookup(file_set.original_file.mime_type)
       ext_mime = MIME::Types.of(file_set.original_file.file_name).first
-      if fn.include?(file_set.id)
+      if fn.include?(file_set.id) || importerexporter.metadata_only?
         return fn if mime.to_s == ext_mime.to_s
         return "#{fn}.#{mime.to_sym}"
       else

data/app/models/concerns/bulkrax/has_matchers.rb

@@ -43,7 +43,6 @@ module Bulkrax
 
         value = if matcher
                   result = matcher.result(self, node_content)
-                  next unless result
                   matched_metadata(multiple, name, result, object_multiple)
                 elsif multiple
                   Rails.logger.info("Bulkrax Column automatically matched #{node_name}, #{node_content}")
@@ -53,12 +52,20 @@ module Bulkrax
                   single_metadata(node_content)
                 end
 
-        set_parsed_data(object_multiple, object_name, name, index, value) if value
+        object_name.present? ? set_parsed_object_data(object_multiple, object_name, name, index, value) : set_parsed_data(name, value)
       end
     end
 
-    def set_parsed_data(object_multiple, object_name, name, index, value)
+    def set_parsed_data(name, value)
+      return parsed_metadata[name] = value unless multiple?(name)
+
+      parsed_metadata[name] ||= []
+      parsed_metadata[name] += Array.wrap(value).flatten
+    end
+
+    def set_parsed_object_data(object_multiple, object_name, name, index, value)
       if object_multiple
+        index ||= 0
         parsed_metadata[object_name][index] ||= {}
         parsed_metadata[object_name][index][name] ||= []
         if value.is_a?(Array)
@@ -66,20 +73,13 @@ module Bulkrax
         else
           parsed_metadata[object_name][index][name] = value
         end
-      elsif object_name
+      else
         parsed_metadata[object_name][name] ||= []
         if value.is_a?(Array)
           parsed_metadata[object_name][name] += value
         else
           parsed_metadata[object_name][name] = value
         end
-      else
-        parsed_metadata[name] ||= []
-        if value.is_a?(Array)
-          parsed_metadata[name] += value
-        else
-          parsed_metadata[name] = value
-        end
       end
     end
 
@@ -124,12 +124,43 @@ module Bulkrax
       field = field.gsub('_attributes', '')
 
       return false if excluded?(field)
-      return true if ['collections', 'file', 'remote_files', 'model', 'delete'].include?(field)
+      return true if supported_bulkrax_fields.include?(field)
       return factory_class.method_defined?(field) && factory_class.properties[field].present?
     end
 
+    def supported_bulkrax_fields
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      @supported_bulkrax_fields ||=
+        %W[
+          id
+          file
+          remote_files
+          model
+          delete
+          #{parser.collection_field_mapping}
+          #{related_parents_parsed_mapping}
+          #{related_children_parsed_mapping}
+        ]
+    end
+
     def multiple?(field)
-      return true if field == 'file' || field == 'remote_files' || field == 'collections'
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      @multiple_bulkrax_fields ||=
+        %W[
+          file
+          remote_files
+          #{parser.collection_field_mapping}
+          #{related_parents_parsed_mapping}
+          #{related_children_parsed_mapping}
+        ]
+
+      return true if @multiple_bulkrax_fields.include?(field)
       return false if field == 'model'
 
       field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -11,6 +11,8 @@ module Bulkrax
         unless self.importerexporter.validate_only
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
+          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping].present?
+          child_jobs if self.parsed_metadata[related_children_parsed_mapping].present?
         end
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e
@@ -22,7 +24,23 @@ module Bulkrax
       return @item
     end
 
-    def find_or_create_collection_ids
+    def parent_jobs
+      self.parsed_metadata[related_parents_parsed_mapping].each do |parent_identifier|
+        next if parent_identifier.blank?
+
+        CreateRelationshipsJob.perform_later(entry_identifier: self.identifier, parent_identifier: parent_identifier, importer_run: self.last_run)
+      end
+    end
+
+    def child_jobs
+      self.parsed_metadata[related_children_parsed_mapping].each do |child_identifier|
+        next if child_identifier.blank?
+
+        CreateRelationshipsJob.perform_later(entry_identifier: self.identifier, child_identifier: child_identifier, importer_run: self.last_run)
+      end
+    end
+
+    def find_collection_ids
       self.collection_ids
     end
 
@@ -57,15 +75,28 @@ module Bulkrax
     end
 
     def add_collections
-      return if find_or_create_collection_ids.blank?
-      self.parsed_metadata['collections'] = []
-      self.parsed_metadata['collections'] += find_or_create_collection_ids.map { |c| { id: c } }
+      return if find_collection_ids.blank?
+
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      self.parsed_metadata['member_of_collections_attributes'] = {}
+      find_collection_ids.each_with_index do |c, i|
+        self.parsed_metadata['member_of_collections_attributes'][i.to_s] = { id: c }
+      end
     end
 
     def factory
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
       @factory ||= Bulkrax::ObjectFactory.new(attributes: self.parsed_metadata,
                                               source_identifier_value: identifier,
                                               work_identifier: parser.work_identifier,
+                                              collection_field_mapping: parser.collection_field_mapping,
+                                              related_parents_parsed_mapping: related_parents_parsed_mapping,
                                               replace_files: replace_files,
                                               user: user,
                                               klass: factory_class,
@@ -80,7 +111,11 @@ module Bulkrax
            else
              Bulkrax.default_work_type
            end
-      fc.constantize
+
+      # return the name of the collection or work
+      fc.tr!(' ', '_')
+      fc.downcase! if fc.match?(/[-_]/)
+      fc.camelcase.constantize
     rescue NameError
       nil
     rescue

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -20,15 +20,36 @@ module Bulkrax
       (last_imported_at || Time.current) + frequency.to_seconds if schedulable? && last_imported_at.present?
     end
 
-    def increment_counters(index, collection = false)
+    def increment_counters(index, collection: false, file_set: false)
       # Only set the totals if they were not set on initialization
       if collection
         current_run.total_collection_entries = index + 1 unless parser.collections_total.positive?
+      elsif file_set
+        current_run.total_file_set_entries = index + 1 unless parser.file_sets_total.positive?
       else
+        # TODO: differentiate between work and collection counts for exporters
         current_run.total_work_entries = index + 1 unless limit.to_i.positive? || parser.total.positive?
       end
-      current_run.enqueued_records = index + 1
+      current_run.enqueued_records += 1
       current_run.save!
     end
+
+    def keys_without_numbers(keys)
+      keys.map { |key| key_without_numbers(key) }
+    end
+
+    def key_without_numbers(key)
+      key.gsub(/_\d+/, '').sub(/^\d+_/, '')
+    end
+
+    # Is this a file?
+    def file?
+      parser_fields&.[]('import_file_path') && File.file?(parser_fields['import_file_path'])
+    end
+
+    # Is this a zip file?
+    def zip?
+      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
+    end
   end
 end

data/app/models/concerns/bulkrax/status_info.rb

@@ -33,13 +33,13 @@ module Bulkrax
       current_status&.created_at
     end
 
-    def status_info(e = nil)
+    def status_info(e = nil, current_run = nil)
       if e.nil?
-        self.statuses.create!(status_message: 'Complete', runnable: last_run)
+        self.statuses.create!(status_message: 'Complete', runnable: current_run || last_run)
       elsif e.is_a?(String)
-        self.statuses.create!(status_message: e, runnable: last_run)
+        self.statuses.create!(status_message: e, runnable: current_run || last_run)
       else
-        self.statuses.create!(status_message: 'Failed', runnable: last_run, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
+        self.statuses.create!(status_message: 'Failed', runnable: current_run || last_run, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
       end
     end
 

data/app/parsers/bulkrax/application_parser.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class ApplicationParser
-    attr_accessor :importerexporter
+  class ApplicationParser # rubocop:disable Metrics/ClassLength
+    attr_accessor :importerexporter, :headers
     alias importer importerexporter
     alias exporter importerexporter
-    delegate :only_updates, :limit, :current_run, :errors,
-             :seen, :increment_counters, :parser_fields, :user,
-             :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
-             :status, :status_info, :status_at,
-             to: :importerexporter
+    delegate :only_updates, :limit, :current_run, :errors, :mapping,
+      :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
+      :key_without_numbers, :status, :status_info, :status_at,
+      :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
+      to: :importerexporter
 
     def self.parser_fields
       {}
@@ -25,6 +25,7 @@ module Bulkrax
 
     def initialize(importerexporter)
       @importerexporter = importerexporter
+      @headers = []
     end
 
     # @api
@@ -43,20 +44,54 @@ module Bulkrax
     end
 
     def source_identifier
-      @source_identifier ||= identifier_hash.values.first&.[]("from")&.first&.to_sym || :source_identifier
+      @source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
     end
 
     def work_identifier
-      @work_identifier ||= identifier_hash.keys.first&.to_sym || :source
+      @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
     end
 
-    def identifier_hash
-      @identifier_hash ||= importerexporter.mapping.select do |_, h|
-        h.key?("source_identifier")
-      end
-      raise StandardError, "more than one source_identifier declared: #{@identifier_hash.keys.join(', ')}" if @identifier_hash.length > 1
+    def related_parents_raw_mapping
+      @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
+    end
+
+    def related_parents_parsed_mapping
+      @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first
+    end
+
+    def related_children_raw_mapping
+      @related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
+    end
+
+    def related_children_parsed_mapping
+      @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first
+    end
+
+    def get_field_mapping_hash_for(key)
+      return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
+
+      instance_variable_set(
+        "@#{key}_hash",
+        importerexporter.mapping.with_indifferent_access.select { |_, h| h.key?(key) }
+      )
+      raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
+
+      instance_variable_get("@#{key}_hash")
+    end
+
+    def collection_field_mapping
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      Bulkrax.collection_field_mapping[self.entry_class.to_s]&.to_sym || :collection
+    end
+
+    def model_field_mappings
+      model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
+      model_mappings |= ['model']
 
-      @identifier_hash
+      model_mappings
     end
 
     def perform_method
@@ -79,6 +114,10 @@ module Bulkrax
       raise StandardError, 'must be defined' if importer?
     end
 
+    def create_file_sets
+      raise StandardError, 'must be defined' if importer?
+    end
+
     # Optional, define if using browse everything for file upload
     def retrieve_cloud_files(files); end
 
@@ -91,76 +130,19 @@ module Bulkrax
       path
     end
 
+    # Base path for imported and exported files
+    def base_path(type = 'import')
+      ENV['HYKU_MULTITENANT'] ? File.join(Bulkrax.send("#{type}_path"), Site.instance.account.name) : Bulkrax.send("#{type}_path")
+    end
+
     # Path where we'll store the import metadata and files
     #  this is used for uploaded and cloud files
     def path_for_import
-      @path_for_import = File.join(Bulkrax.import_path, importerexporter.path_string)
+      @path_for_import = File.join(base_path, importerexporter.path_string)
       FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
       @path_for_import
     end
 
-    # Optional, only used by certain parsers
-    # Other parsers should override with a custom or empty method
-    # Will be skipped unless the #record is a Hash
-    def create_parent_child_relationships
-      parents.each do |key, value|
-        parent = entry_class.where(
-          identifier: key,
-          importerexporter_id: importerexporter.id,
-          importerexporter_type: 'Bulkrax::Importer'
-        ).first
-
-        # not finding the entries here indicates that the given identifiers are incorrect
-        # in that case we should log that
-        children = value.map do |child|
-          entry_class.where(
-            identifier: child,
-            importerexporter_id: importerexporter.id,
-            importerexporter_type: 'Bulkrax::Importer'
-          ).first
-        end.compact.uniq
-
-        if parent.present? && (children.length != value.length)
-          # Increment the failures for the number we couldn't find
-          # Because all of our entries have been created by now, if we can't find them, the data is wrong
-          Rails.logger.error("Expected #{value.length} children for parent entry #{parent.id}, found #{children.length}")
-          break if children.empty?
-          Rails.logger.warn("Adding #{children.length} children to parent entry #{parent.id} (expected #{value.length})")
-        end
-        parent_id = parent.id
-        child_entry_ids = children.map(&:id)
-        ChildRelationshipsJob.perform_later(parent_id, child_entry_ids, current_run.id)
-      end
-    rescue StandardError => e
-      status_info(e)
-    end
-
-    def parents
-      @parents ||= setup_parents
-    end
-
-    def setup_parents
-      pts = []
-      records.each do |record|
-        r = if record.respond_to?(:to_h)
-              record.to_h
-            else
-              record
-            end
-        next unless r.is_a?(Hash)
-        children = if r[:children].is_a?(String)
-                     r[:children].split(/\s*[:;|]\s*/)
-                   else
-                     r[:children]
-                   end
-        next if children.blank?
-        pts << {
-          r[source_identifier] => children
-        }
-      end
-      pts.blank? ? pts : pts.inject(:merge)
-    end
-
     def setup_export_file
       raise StandardError, 'must be defined' if exporter?
     end
@@ -256,6 +238,10 @@ module Bulkrax
       0
     end
 
+    def file_sets_total
+      0
+    end
+
     def write
       write_files
       zip
@@ -288,12 +274,9 @@ module Bulkrax
     private
 
     def real_import_file_path
-      if file? && zip?
-        unzip(parser_fields['import_file_path'])
-        return importer_unzip_path
-      else
-        parser_fields['import_file_path']
-      end
+      return importer_unzip_path if file? && zip?
+
+      parser_fields['import_file_path']
     end
   end
 end

data/app/parsers/bulkrax/bagit_parser.rb

@@ -40,7 +40,7 @@ module Bulkrax
         raise StandardError, 'No metadata files were found' if path.blank?
         data = entry_class.read_data(path)
         data = entry_class.data_for_entry(data, source_identifier)
-        data[:file] = bag.bag_files.join('|')
+        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
         data
       end
     end
@@ -58,8 +58,8 @@ module Bulkrax
           collection_type_gid: Hyrax::CollectionType.find_or_create_default_collection_type.gid
         }
         new_entry = find_or_create_entry(collection_entry_class, collection, 'Bulkrax::Importer', metadata)
-        ImportWorkCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, true)
+        ImportCollectionJob.perform_now(new_entry.id, current_run.id)
+        increment_counters(index, collection: true)
       end
     end
 
@@ -83,13 +83,22 @@ module Bulkrax
     end
 
     def collections
-      records.map { |r| r[:collection].split(/\s*[;|]\s*/) if r[:collection].present? }.flatten.compact.uniq
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      records.map { |r| r[collection_field_mapping].split(/\s*[;|]\s*/) if r[collection_field_mapping].present? }.flatten.compact.uniq
     end
 
     def collections_total
       collections.size
     end
 
+    # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
+    def works_total
+      total
+    end
+
     def total
       metadata_paths.count
     end