bulkrax 1.0.1 → 2.0.2

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -11,6 +11,8 @@ module Bulkrax
         unless self.importerexporter.validate_only
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
+          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping].present?
+          child_jobs if self.parsed_metadata[related_children_parsed_mapping].present?
         end
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e
@@ -22,7 +24,23 @@ module Bulkrax
       return @item
     end
 
-    def find_or_create_collection_ids
+    def parent_jobs
+      self.parsed_metadata[related_parents_parsed_mapping].each do |parent_identifier|
+        next if parent_identifier.blank?
+
+        CreateRelationshipsJob.perform_later(entry_identifier: self.identifier, parent_identifier: parent_identifier, importer_run: self.last_run)
+      end
+    end
+
+    def child_jobs
+      self.parsed_metadata[related_children_parsed_mapping].each do |child_identifier|
+        next if child_identifier.blank?
+
+        CreateRelationshipsJob.perform_later(entry_identifier: self.identifier, child_identifier: child_identifier, importer_run: self.last_run)
+      end
+    end
+
+    def find_collection_ids
       self.collection_ids
     end
 
@@ -57,15 +75,27 @@ module Bulkrax
     end
 
     def add_collections
-      return if find_or_create_collection_ids.blank?
-      self.parsed_metadata['collections'] = []
-      self.parsed_metadata['collections'] += find_or_create_collection_ids.map { |c| { id: c } }
+      return if find_collection_ids.blank?
+
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      self.parsed_metadata['member_of_collections_attributes'] = {}
+      find_collection_ids.each_with_index do |c, i|
+        self.parsed_metadata['member_of_collections_attributes'][i.to_s] = { id: c }
+      end
     end
 
     def factory
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
       @factory ||= Bulkrax::ObjectFactory.new(attributes: self.parsed_metadata,
                                               source_identifier_value: identifier,
                                               work_identifier: parser.work_identifier,
+                                              collection_field_mapping: parser.collection_field_mapping,
                                               replace_files: replace_files,
                                               user: user,
                                               klass: factory_class,
@@ -80,7 +110,11 @@ module Bulkrax
            else
              Bulkrax.default_work_type
            end
-      fc.constantize
+
+      # return the name of the collection or work
+      fc.tr!(' ', '_')
+      fc.downcase! if fc.match?(/[-_]/)
+      fc.camelcase.constantize
     rescue NameError
       nil
     rescue

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -25,10 +25,29 @@ module Bulkrax
       if collection
         current_run.total_collection_entries = index + 1 unless parser.collections_total.positive?
       else
+        # TODO: differentiate between work and collection counts for exporters
         current_run.total_work_entries = index + 1 unless limit.to_i.positive? || parser.total.positive?
       end
       current_run.enqueued_records = index + 1
       current_run.save!
     end
+
+    def keys_without_numbers(keys)
+      keys.map { |key| key_without_numbers(key) }
+    end
+
+    def key_without_numbers(key)
+      key.gsub(/_\d+/, '').sub(/^\d+_/, '')
+    end
+
+    # Is this a file?
+    def file?
+      parser_fields&.[]('import_file_path') && File.file?(parser_fields['import_file_path'])
+    end
+
+    # Is this a zip file?
+    def zip?
+      parser_fields&.[]('import_file_path') && MIME::Types.type_for(parser_fields['import_file_path']).include?('application/zip')
+    end
   end
 end

data/app/models/concerns/bulkrax/status_info.rb

@@ -33,13 +33,13 @@ module Bulkrax
       current_status&.created_at
     end
 
-    def status_info(e = nil)
+    def status_info(e = nil, current_run = nil)
       if e.nil?
-        self.statuses.create!(status_message: 'Complete', runnable: last_run)
+        self.statuses.create!(status_message: 'Complete', runnable: current_run || last_run)
       elsif e.is_a?(String)
-        self.statuses.create!(status_message: e, runnable: last_run)
+        self.statuses.create!(status_message: e, runnable: current_run || last_run)
       else
-        self.statuses.create!(status_message: 'Failed', runnable: last_run, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
+        self.statuses.create!(status_message: 'Failed', runnable: current_run || last_run, error_class: e.class.to_s, error_message: e.message, error_backtrace: e.backtrace)
       end
     end
 

data/app/parsers/bulkrax/application_parser.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class ApplicationParser
-    attr_accessor :importerexporter
+  class ApplicationParser # rubocop:disable Metrics/ClassLength
+    attr_accessor :importerexporter, :headers
     alias importer importerexporter
     alias exporter importerexporter
-    delegate :only_updates, :limit, :current_run, :errors,
-             :seen, :increment_counters, :parser_fields, :user,
-             :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
-             :status, :status_info, :status_at,
-             to: :importerexporter
+    delegate :only_updates, :limit, :current_run, :errors, :mapping,
+      :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
+      :key_without_numbers, :status, :status_info, :status_at,
+      :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
+      to: :importerexporter
 
     def self.parser_fields
       {}
@@ -25,6 +25,7 @@ module Bulkrax
 
     def initialize(importerexporter)
       @importerexporter = importerexporter
+      @headers = []
     end
 
     # @api
@@ -43,20 +44,54 @@ module Bulkrax
     end
 
     def source_identifier
-      @source_identifier ||= identifier_hash.values.first&.[]("from")&.first&.to_sym || :source_identifier
+      @source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
     end
 
     def work_identifier
-      @work_identifier ||= identifier_hash.keys.first&.to_sym || :source
+      @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
     end
 
-    def identifier_hash
-      @identifier_hash ||= importerexporter.mapping.select do |_, h|
-        h.key?("source_identifier")
-      end
-      raise StandardError, "more than one source_identifier declared: #{@identifier_hash.keys.join(', ')}" if @identifier_hash.length > 1
+    def related_parents_raw_mapping
+      @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
+    end
+
+    def related_parents_parsed_mapping
+      @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first
+    end
+
+    def related_children_raw_mapping
+      @related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
+    end
+
+    def related_children_parsed_mapping
+      @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first
+    end
+
+    def get_field_mapping_hash_for(key)
+      return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
+
+      instance_variable_set(
+        "@#{key}_hash",
+        importerexporter.mapping.with_indifferent_access.select { |_, h| h.key?(key) }
+      )
+      raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
 
-      @identifier_hash
+      instance_variable_get("@#{key}_hash")
+    end
+
+    def collection_field_mapping
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      Bulkrax.collection_field_mapping[self.entry_class.to_s]&.to_sym || :collection
+    end
+
+    def model_field_mappings
+      model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
+      model_mappings |= ['model']
+
+      model_mappings
     end
 
     def perform_method
@@ -91,76 +126,19 @@ module Bulkrax
       path
     end
 
+    # Base path for imported and exported files
+    def base_path(type = 'import')
+      ENV['HYKU_MULTITENANT'] ? File.join(Bulkrax.send("#{type}_path"), Site.instance.account.name) : Bulkrax.send("#{type}_path")
+    end
+
     # Path where we'll store the import metadata and files
     #  this is used for uploaded and cloud files
     def path_for_import
-      @path_for_import = File.join(Bulkrax.import_path, importerexporter.path_string)
+      @path_for_import = File.join(base_path, importerexporter.path_string)
       FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
       @path_for_import
     end
 
-    # Optional, only used by certain parsers
-    # Other parsers should override with a custom or empty method
-    # Will be skipped unless the #record is a Hash
-    def create_parent_child_relationships
-      parents.each do |key, value|
-        parent = entry_class.where(
-          identifier: key,
-          importerexporter_id: importerexporter.id,
-          importerexporter_type: 'Bulkrax::Importer'
-        ).first
-
-        # not finding the entries here indicates that the given identifiers are incorrect
-        # in that case we should log that
-        children = value.map do |child|
-          entry_class.where(
-            identifier: child,
-            importerexporter_id: importerexporter.id,
-            importerexporter_type: 'Bulkrax::Importer'
-          ).first
-        end.compact.uniq
-
-        if parent.present? && (children.length != value.length)
-          # Increment the failures for the number we couldn't find
-          # Because all of our entries have been created by now, if we can't find them, the data is wrong
-          Rails.logger.error("Expected #{value.length} children for parent entry #{parent.id}, found #{children.length}")
-          break if children.empty?
-          Rails.logger.warn("Adding #{children.length} children to parent entry #{parent.id} (expected #{value.length})")
-        end
-        parent_id = parent.id
-        child_entry_ids = children.map(&:id)
-        ChildRelationshipsJob.perform_later(parent_id, child_entry_ids, current_run.id)
-      end
-    rescue StandardError => e
-      status_info(e)
-    end
-
-    def parents
-      @parents ||= setup_parents
-    end
-
-    def setup_parents
-      pts = []
-      records.each do |record|
-        r = if record.respond_to?(:to_h)
-              record.to_h
-            else
-              record
-            end
-        next unless r.is_a?(Hash)
-        children = if r[:children].is_a?(String)
-                     r[:children].split(/\s*[:;|]\s*/)
-                   else
-                     r[:children]
-                   end
-        next if children.blank?
-        pts << {
-          r[source_identifier] => children
-        }
-      end
-      pts.blank? ? pts : pts.inject(:merge)
-    end
-
     def setup_export_file
       raise StandardError, 'must be defined' if exporter?
     end
@@ -288,12 +266,9 @@ module Bulkrax
     private
 
     def real_import_file_path
-      if file? && zip?
-        unzip(parser_fields['import_file_path'])
-        return importer_unzip_path
-      else
-        parser_fields['import_file_path']
-      end
+      return importer_unzip_path if file? && zip?
+
+      parser_fields['import_file_path']
     end
   end
 end

data/app/parsers/bulkrax/bagit_parser.rb

@@ -40,7 +40,7 @@ module Bulkrax
         raise StandardError, 'No metadata files were found' if path.blank?
         data = entry_class.read_data(path)
         data = entry_class.data_for_entry(data, source_identifier)
-        data[:file] = bag.bag_files.join('|')
+        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
         data
       end
     end
@@ -58,7 +58,7 @@ module Bulkrax
           collection_type_gid: Hyrax::CollectionType.find_or_create_default_collection_type.gid
         }
         new_entry = find_or_create_entry(collection_entry_class, collection, 'Bulkrax::Importer', metadata)
-        ImportWorkCollectionJob.perform_now(new_entry.id, current_run.id)
+        ImportCollectionJob.perform_now(new_entry.id, current_run.id)
         increment_counters(index, true)
       end
     end
@@ -83,13 +83,22 @@ module Bulkrax
     end
 
     def collections
-      records.map { |r| r[:collection].split(/\s*[;|]\s*/) if r[:collection].present? }.flatten.compact.uniq
+      ActiveSupport::Deprecation.warn(
+        'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+        ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+      )
+      records.map { |r| r[collection_field_mapping].split(/\s*[;|]\s*/) if r[collection_field_mapping].present? }.flatten.compact.uniq
     end
 
     def collections_total
       collections.size
     end
 
+    # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
+    def works_total
+      total
+    end
+
     def total
       metadata_paths.count
     end