bulkrax 2.0.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56a6005830d733023f9455ebb9e0565081dacbfbf1a36ebad290d594d74499f4
-  data.tar.gz: dbfe501d7c62a47091e23fbc9b7b7e8aff3147f88c999eb9d55bc6e6bf27ae05
+  metadata.gz: a35c9f6307c21a9933950a3d383ab04e5a49e15b9980d76f6cfb52a5378f5dd0
+  data.tar.gz: d5a58eeb2b755c68e6c66103afb5dbc649205aed73b2d85167246ca912bb07bf
 SHA512:
-  metadata.gz: 1054dbc43420ad3fba9f75e8c88cec4b795c55a1dc5416b5b37a7a75eff994f629aa7740dda54f635be406ec2b7c4f913b1de7426cbd1a378573cd42d526e5c7
-  data.tar.gz: 6eb81924d7587caddcea1ec4b7d71b41324336a85ea2ae6662662028ccc3c5b92f93b5aa02fe7d0a9c19cb5e16db560ff95cfc74b85f475bae0c425ecb96824c
+  metadata.gz: d01b9ecd45171f2b20a5c33605fafe645b7c389a141ed70b41a04ecb7bcf1d1f1c4f430ea50fda7d7ef702fd12123cfd904868ba6468a306a41eca5643061e45
+  data.tar.gz: 943eb873c86e40adf96cd7b0badb1ccce719f088d01aada4e18df90ec5a4f4df5b5ebfda5bfe6f74943c6fa371632937a0918f6aed70a4e14a75bcef0b7eaa0b

data/app/controllers/bulkrax/importers_controller.rb

@@ -37,6 +37,7 @@ module Bulkrax
 
         @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
         @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
+        @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
       end
     end
 

data/app/factories/bulkrax/object_factory.rb

@@ -4,11 +4,13 @@ module Bulkrax
   class ObjectFactory
     extend ActiveModel::Callbacks
     include Bulkrax::FileFactory
+    include DynamicRecordLookup
+
     define_model_callbacks :save, :create
-    attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :collection_field_mapping
+    attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :collection_field_mapping, :related_parents_parsed_mapping
 
     # rubocop:disable Metrics/ParameterLists
-    def initialize(attributes:, source_identifier_value:, work_identifier:, collection_field_mapping:, replace_files: false, user: nil, klass: nil, update_files: false)
+    def initialize(attributes:, source_identifier_value:, work_identifier:, collection_field_mapping:, related_parents_parsed_mapping: nil, replace_files: false, user: nil, klass: nil, update_files: false)
       ActiveSupport::Deprecation.warn(
         'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
         ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
@@ -19,6 +21,7 @@ module Bulkrax
       @user = user || User.batch_user
       @work_identifier = work_identifier
       @collection_field_mapping = collection_field_mapping
+      @related_parents_parsed_mapping = related_parents_parsed_mapping
       @source_identifier_value = source_identifier_value
       @klass = klass || Bulkrax.default_work_type.constantize
     end
@@ -33,7 +36,7 @@ module Bulkrax
       arg_hash = { id: attributes[:id], name: 'UPDATE', klass: klass }
       @object = find
       if object
-        object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
+        object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent)
         ActiveSupport::Notifications.instrument('import.importer', arg_hash) { update }
       else
         ActiveSupport::Notifications.instrument('import.importer', arg_hash.merge(name: 'CREATE')) { create }
@@ -51,10 +54,16 @@ module Bulkrax
 
     def update
       raise "Object doesn't exist" unless object
-      destroy_existing_files if @replace_files && klass != Collection
+      destroy_existing_files if @replace_files && ![Collection, FileSet].include?(klass)
       attrs = attribute_update
       run_callbacks :save do
-        klass == Collection ? update_collection(attrs) : work_actor.update(environment(attrs))
+        if klass == Collection
+          update_collection(attrs)
+        elsif klass == FileSet
+          update_file_set(attrs)
+        else
+          work_actor.update(environment(attrs))
+        end
       end
       log_updated(object)
     end
@@ -90,10 +99,16 @@ module Bulkrax
     def create
       attrs = create_attributes
       @object = klass.new
-      object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
+      object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent)
       run_callbacks :save do
         run_callbacks :create do
-          klass == Collection ? create_collection(attrs) : work_actor.create(environment(attrs))
+          if klass == Collection
+            create_collection(attrs)
+          elsif klass == FileSet
+            create_file_set(attrs)
+          else
+            work_actor.create(environment(attrs))
+          end
         end
       end
       log_created(object)
@@ -150,6 +165,35 @@ module Bulkrax
       object.save!
     end
 
+    # This method is heavily inspired by Hyrax's AttachFilesToWorkJob
+    def create_file_set(attrs)
+      work = find_record(attributes[related_parents_parsed_mapping].first)
+      work_permissions = work.permissions.map(&:to_hash)
+      file_set_attrs = attrs.slice(*object.attributes.keys)
+      object.assign_attributes(file_set_attrs)
+
+      attrs['uploaded_files'].each do |uploaded_file_id|
+        uploaded_file = ::Hyrax::UploadedFile.find(uploaded_file_id)
+        next if uploaded_file.file_set_uri.present?
+
+        actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
+        uploaded_file.update(file_set_uri: actor.file_set.uri)
+        actor.file_set.permissions_attributes = work_permissions
+        actor.create_metadata
+        actor.create_content(uploaded_file)
+        actor.attach_to_work(work)
+      end
+
+      object.save!
+    end
+
+    def update_file_set(attrs)
+      file_set_attrs = attrs.slice(*object.attributes.keys)
+      actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
+
+      actor.update_metadata(file_set_attrs)
+    end
+
     # Add child to parent's #member_collections
     # Add parent to child's #member_of_collections
     def persist_collection_memberships(parent:, child:)

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -17,6 +17,8 @@ module Bulkrax
   # NOTE: In the context of this job, "identifier" is used to generically refer
   #       to either a record's ID or an Bulkrax::Entry's source_identifier.
   class CreateRelationshipsJob < ApplicationJob
+    include DynamicRecordLookup
+
     queue_as :import
 
     attr_accessor :base_entry, :child_record, :parent_record, :importer_run
@@ -74,39 +76,6 @@ module Bulkrax
       end
     end
 
-    # This method allows us to create relationships with preexisting records (by their ID) OR
-    # with records that are concurrently being imported (by their Bulkrax::Entry source_identifier).
-    #
-    # @param identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
-    # @return [Work, Collection, nil] Work or Collection if found, otherwise nil
-    def find_record(identifier)
-      record = Entry.find_by(identifier: identifier)
-      record ||= ::Collection.where(id: identifier).first
-      if record.blank?
-        available_work_types.each do |work_type|
-          record ||= work_type.where(id: identifier).first
-        end
-      end
-      record = record.factory.find if record.is_a?(Entry)
-
-      record
-    end
-
-    # Check if the record is a Work
-    def curation_concern?(record)
-      available_work_types.include?(record.class)
-    end
-
-    # @return [Array<Class>] list of work type classes
-    def available_work_types
-      # If running in a Hyku app, do not reference disabled work types
-      @available_work_types ||= if defined?(::Hyku)
-                                  ::Site.instance.available_works.map(&:constantize)
-                                else
-                                  ::Hyrax.config.curation_concerns
-                                end
-    end
-
     def user
       @user ||= importer_run.importer.user
     end

data/app/jobs/bulkrax/import_collection_job.rb

@@ -11,9 +11,11 @@ module Bulkrax
         entry.build
         entry.save
         add_user_to_permission_template!(entry)
+        ImporterRun.find(args[1]).increment!(:processed_records)
         ImporterRun.find(args[1]).increment!(:processed_collections)
         ImporterRun.find(args[1]).decrement!(:enqueued_records)
       rescue => e
+        ImporterRun.find(args[1]).increment!(:failed_records)
         ImporterRun.find(args[1]).increment!(:failed_collections)
         ImporterRun.find(args[1]).decrement!(:enqueued_records)
         raise e

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class MissingParentError < ::StandardError; end
+  class ImportFileSetJob < ApplicationJob
+    include DynamicRecordLookup
+
+    queue_as :import
+
+    def perform(entry_id, importer_run_id)
+      entry = Entry.find(entry_id)
+      parent_identifier = entry.raw_metadata[entry.related_parents_raw_mapping]&.strip
+
+      validate_parent!(parent_identifier)
+
+      entry.build
+      if entry.succeeded?
+        # rubocop:disable Rails/SkipsModelValidations
+        ImporterRun.find(importer_run_id).increment!(:processed_records)
+        ImporterRun.find(importer_run_id).increment!(:processed_file_sets)
+      else
+        ImporterRun.find(importer_run_id).increment!(:failed_records)
+        ImporterRun.find(importer_run_id).increment!(:failed_file_sets)
+        # rubocop:enable Rails/SkipsModelValidations
+      end
+      ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
+      entry.save!
+
+    rescue MissingParentError => e
+      # try waiting for the parent record to be created
+      entry.import_attempts += 1
+      entry.save!
+      if entry.import_attempts < 5
+        ImportFileSetJob
+          .set(wait: (entry.import_attempts + 1).minutes)
+          .perform_later(entry_id, importer_run_id)
+      else
+        ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
+        entry.status_info(e)
+      end
+    end
+
+    private
+
+    attr_reader :parent_record
+
+    def validate_parent!(parent_identifier)
+      # if parent_identifier is missing, it will be caught by #validate_presence_of_parent!
+      return if parent_identifier.blank?
+
+      find_parent_record(parent_identifier)
+      check_parent_exists!(parent_identifier)
+      check_parent_is_a_work!(parent_identifier)
+    end
+
+    def check_parent_exists!(parent_identifier)
+      raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") if parent_record.blank?
+    end
+
+    def check_parent_is_a_work!(parent_identifier)
+      error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
+      raise ::StandardError, error_msg unless curation_concern?(parent_record)
+    end
+
+    def find_parent_record(parent_identifier)
+      @parent_record ||= find_record(parent_identifier)
+    end
+  end
+end

data/app/jobs/bulkrax/import_work_job.rb

@@ -10,11 +10,13 @@ module Bulkrax
       entry.build
       if entry.status == "Complete"
         ImporterRun.find(args[1]).increment!(:processed_records)
+        ImporterRun.find(args[1]).increment!(:processed_works)
         ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
       else
         # do not retry here because whatever parse error kept you from creating a work will likely
         # keep preventing you from doing so.
         ImporterRun.find(args[1]).increment!(:failed_records)
+        ImporterRun.find(args[1]).increment!(:failed_works)
         ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
       end
       entry.save!

data/app/jobs/bulkrax/importer_job.rb

@@ -20,6 +20,7 @@ module Bulkrax
 
       importer.import_collections
       importer.import_works
+      importer.import_file_sets
     end
 
     def unzip_imported_file(parser)
@@ -31,6 +32,7 @@ module Bulkrax
     def update_current_run_counters(importer)
       importer.current_run.total_work_entries = importer.limit || importer.parser.works_total
       importer.current_run.total_collection_entries = importer.parser.collections_total
+      importer.current_run.total_file_set_entries = importer.parser.file_sets_total
       importer.current_run.save!
     end
 

data/app/models/bulkrax/csv_collection_entry.rb

@@ -9,7 +9,7 @@ module Bulkrax
     # Use identifier set by CsvParser#unique_collection_identifier, which falls back
     # on the Collection's first title if record[source_identifier] is not present
     def add_identifier
-      self.parsed_metadata[work_identifier] = self.identifier
+      self.parsed_metadata[work_identifier] = [self.identifier].flatten
     end
 
     def add_collection_type_gid

data/app/models/bulkrax/csv_entry.rb

@@ -40,9 +40,9 @@ module Bulkrax
 
       self.parsed_metadata = {}
       add_identifier
-      add_metadata_for_model
       add_visibility
       add_ingested_metadata
+      add_metadata_for_model
       add_rights_statement
       add_collections
       add_local
@@ -57,6 +57,9 @@ module Bulkrax
     def add_metadata_for_model
       if factory_class == Collection
         add_collection_type_gid
+      elsif factory_class == FileSet
+        add_path_to_file
+        validate_presence_of_parent!
       else
         add_file unless importerexporter.metadata_only?
         add_admin_set_id
@@ -68,7 +71,9 @@ module Bulkrax
         'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
         ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
       )
-      record.sort.each do |key, value|
+      # we do not want to sort the values in the record before adding the metadata.
+      # if we do, the factory_class will be set to the default_work_type for all values that come before "model" or "work type"
+      record.each do |key, value|
         next if self.parser.collection_field_mapping.to_s == key_without_numbers(key)
 
         index = key[/\d+/].to_i - 1 if key[/\d+/].to_i != 0
@@ -83,7 +88,11 @@ module Bulkrax
       elsif record['file'].is_a?(Array)
         self.parsed_metadata['file'] = record['file']
       end
-      self.parsed_metadata['file'] = self.parsed_metadata['file'].map { |f| path_to_file(f.tr(' ', '_')) }
+      self.parsed_metadata['file'] = self.parsed_metadata['file'].map do |f|
+        next if f.blank?
+
+        path_to_file(f.tr(' ', '_'))
+      end.compact
     end
 
     def build_export_metadata

data/app/models/bulkrax/csv_file_set_entry.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class CsvFileSetEntry < CsvEntry
+    def factory_class
+      ::FileSet
+    end
+
+    def add_path_to_file
+      parsed_metadata['file'].each_with_index do |filename, i|
+        path_to_file = ::File.join(parser.path_to_files, filename)
+
+        parsed_metadata['file'][i] = path_to_file
+      end
+      raise ::StandardError, 'one or more file paths are invalid' unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
+
+      parsed_metadata['file']
+    end
+
+    def validate_presence_of_parent!
+      return if parsed_metadata[related_parents_parsed_mapping]&.map(&:present?)&.any?
+
+      raise StandardError, 'File set must be related to at least one work'
+    end
+  end
+end

data/app/models/bulkrax/importer.rb

@@ -99,7 +99,12 @@ module Bulkrax
       @current_run ||= if file? && zip?
                          self.importer_runs.create!
                        else
-                         self.importer_runs.create!(total_work_entries: self.limit || parser.works_total, total_collection_entries: parser.collections_total)
+                         entry_counts = {
+                           total_work_entries: self.limit || parser.works_total,
+                           total_collection_entries: parser.collections_total,
+                           total_file_set_entries: parser.file_sets_total
+                         }
+                         self.importer_runs.create!(entry_counts)
                        end
     end
 
@@ -134,6 +139,13 @@ module Bulkrax
       status_info(e)
     end
 
+    def import_file_sets
+      self.save if self.new_record? # Object needs to be saved for statuses
+      parser.create_file_sets
+    rescue StandardError => e
+      status_info(e)
+    end
+
     # Prepend the base_url to ensure unique set identifiers
     # @todo - move to parser, as this is OAI specific
     def unique_collection_identifier(id)

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module DynamicRecordLookup
+    # Search entries, collections, and every available work type for a record that
+    # has the provided identifier.
+    #
+    # @param identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
+    # @return [Work, Collection, nil] Work or Collection if found, otherwise nil
+    def find_record(identifier)
+      record = Entry.find_by(identifier: identifier)
+      record ||= ::Collection.where(id: identifier).first # rubocop:disable Rails/FindBy
+      if record.blank?
+        available_work_types.each do |work_type|
+          record ||= work_type.where(id: identifier).first # rubocop:disable Rails/FindBy
+        end
+      end
+
+      record.is_a?(Entry) ? record.factory.find : record
+    end
+
+    # Check if the record is a Work
+    def curation_concern?(record)
+      available_work_types.include?(record.class)
+    end
+
+    private
+
+    # @return [Array<Class>] list of work type classes
+    def available_work_types
+      # If running in a Hyku app, do not include disabled work types
+      @available_work_types ||= if defined?(::Hyku)
+                                  ::Site.instance.available_works.map(&:constantize)
+                                else
+                                  ::Hyrax.config.curation_concerns
+                                end
+    end
+  end
+end

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -11,9 +11,9 @@ module Bulkrax
         unless self.importerexporter.validate_only
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
+          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping].present?
+          child_jobs if self.parsed_metadata[related_children_parsed_mapping].present?
         end
-        parent_jobs if self.parsed_metadata[related_parents_parsed_mapping].present?
-        child_jobs if self.parsed_metadata[related_children_parsed_mapping].present?
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e
       rescue StandardError => e
@@ -26,12 +26,16 @@ module Bulkrax
 
     def parent_jobs
       self.parsed_metadata[related_parents_parsed_mapping].each do |parent_identifier|
+        next if parent_identifier.blank?
+
         CreateRelationshipsJob.perform_later(entry_identifier: self.identifier, parent_identifier: parent_identifier, importer_run: self.last_run)
       end
     end
 
     def child_jobs
       self.parsed_metadata[related_children_parsed_mapping].each do |child_identifier|
+        next if child_identifier.blank?
+
         CreateRelationshipsJob.perform_later(entry_identifier: self.identifier, child_identifier: child_identifier, importer_run: self.last_run)
       end
     end
@@ -92,6 +96,7 @@ module Bulkrax
                                               source_identifier_value: identifier,
                                               work_identifier: parser.work_identifier,
                                               collection_field_mapping: parser.collection_field_mapping,
+                                              related_parents_parsed_mapping: related_parents_parsed_mapping,
                                               replace_files: replace_files,
                                               user: user,
                                               klass: factory_class,

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -20,15 +20,17 @@ module Bulkrax
       (last_imported_at || Time.current) + frequency.to_seconds if schedulable? && last_imported_at.present?
     end
 
-    def increment_counters(index, collection = false)
+    def increment_counters(index, collection: false, file_set: false)
       # Only set the totals if they were not set on initialization
       if collection
         current_run.total_collection_entries = index + 1 unless parser.collections_total.positive?
+      elsif file_set
+        current_run.total_file_set_entries = index + 1 unless parser.file_sets_total.positive?
       else
         # TODO: differentiate between work and collection counts for exporters
         current_run.total_work_entries = index + 1 unless limit.to_i.positive? || parser.total.positive?
       end
-      current_run.enqueued_records = index + 1
+      current_run.enqueued_records += 1
       current_run.save!
     end
 

data/app/parsers/bulkrax/application_parser.rb

@@ -114,6 +114,10 @@ module Bulkrax
       raise StandardError, 'must be defined' if importer?
     end
 
+    def create_file_sets
+      raise StandardError, 'must be defined' if importer?
+    end
+
     # Optional, define if using browse everything for file upload
     def retrieve_cloud_files(files); end
 
@@ -234,6 +238,10 @@ module Bulkrax
       0
     end
 
+    def file_sets_total
+      0
+    end
+
     def write
       write_files
       zip

data/app/parsers/bulkrax/bagit_parser.rb

@@ -59,7 +59,7 @@ module Bulkrax
         }
         new_entry = find_or_create_entry(collection_entry_class, collection, 'Bulkrax::Importer', metadata)
         ImportCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, true)
+        increment_counters(index, collection: true)
       end
     end
 

data/app/parsers/bulkrax/csv_parser.rb

@@ -25,7 +25,7 @@ module Bulkrax
       # retrieve a list of unique collections
       records.map do |r|
         collections = []
-        r[collection_field_mapping].split(/\s*[;|]\s*/).each { |title| collections << { title: title } } if r[collection_field_mapping].present?
+        r[collection_field_mapping].split(/\s*[;|]\s*/).each { |title| collections << { title: title, from_collection_field_mapping: true } } if r[collection_field_mapping].present?
         model_field_mappings.each do |model_mapping|
           collections << r if r[model_mapping.to_sym]&.downcase == 'collection'
         end
@@ -38,13 +38,27 @@ module Bulkrax
     end
 
     def works
-      records - collections
+      records - collections - file_sets
     end
 
     def works_total
       works.size
     end
 
+    def file_sets
+      records.map do |r|
+        file_sets = []
+        model_field_mappings.each do |model_mapping|
+          file_sets << r if r[model_mapping.to_sym]&.downcase == 'fileset'
+        end
+        file_sets
+      end.flatten.compact.uniq
+    end
+
+    def file_sets_total
+      file_sets.size
+    end
+
     # We could use CsvEntry#fields_from_data(data) but that would mean re-reading the data
     def import_fields
       @import_fields ||= records.inject(:merge).keys.compact.uniq
@@ -74,11 +88,31 @@ module Bulkrax
       collections.each_with_index do |collection, index|
         next if collection.blank?
         break if records.find_index(collection).present? && limit_reached?(limit, records.find_index(collection))
-
-        new_entry = find_or_create_entry(collection_entry_class, unique_collection_identifier(collection), 'Bulkrax::Importer', collection.to_h)
+        ActiveSupport::Deprecation.warn(
+          'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
+          ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
+        )
+
+        ## BEGIN
+        # Add required metadata to collections being imported using the collection_field_mapping, which only have a :title
+        # TODO: Remove once collection_field_mapping is removed
+        metadata = if collection.delete(:from_collection_field_mapping)
+                     uci = unique_collection_identifier(collection)
+                     {
+                       title: collection[:title],
+                       work_identifier => uci,
+                       source_identifier => uci,
+                       visibility: 'open',
+                       collection_type_gid: ::Hyrax::CollectionType.find_or_create_default_collection_type.gid
+                     }
+                   end
+        collection_hash = metadata.presence || collection
+        ## END
+
+        new_entry = find_or_create_entry(collection_entry_class, collection_hash[source_identifier], 'Bulkrax::Importer', collection_hash)
         # TODO: add support for :delete option
         ImportCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, true)
+        increment_counters(index, collection: true)
       end
       importer.record_status
     rescue StandardError => e
@@ -104,6 +138,20 @@ module Bulkrax
       status_info(e)
     end
 
+    def create_file_sets
+      file_sets.each_with_index do |file_set, index|
+        next unless record_has_source_identifier(file_set, records.find_index(file_set))
+        break if limit_reached?(limit, records.find_index(file_set))
+
+        new_entry = find_or_create_entry(file_set_entry_class, file_set[source_identifier], 'Bulkrax::Importer', file_set.to_h)
+        ImportFileSetJob.perform_later(new_entry.id, current_run.id)
+        increment_counters(index, file_set: true)
+      end
+      importer.record_status
+    rescue StandardError => e
+      status_info(e)
+    end
+
     def write_partial_import_file(file)
       import_filename = import_file_path.split('/').last
       partial_import_filename = "#{File.basename(import_filename, '.csv')}_corrected_entries.csv"
@@ -179,6 +227,10 @@ module Bulkrax
       CsvCollectionEntry
     end
 
+    def file_set_entry_class
+      CsvFileSetEntry
+    end
+
     # See https://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
     #   Changed to grep as wc -l counts blank lines, and ignores the final unescaped line (which may or may not contain data)
     def total

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -76,7 +76,7 @@ module Bulkrax
         new_entry = collection_entry_class.where(importerexporter: importerexporter, identifier: unique_collection_identifier, raw_metadata: metadata).first_or_create!
         # perform now to ensure this gets created before work imports start
         ImportCollectionJob.perform_now(new_entry.id, importerexporter.current_run.id)
-        increment_counters(index, true)
+        increment_counters(index, collection: true)
       end
     end
 

data/app/views/bulkrax/importers/index.html.erb

@@ -24,6 +24,7 @@
               <th scope="col">Entries Deleted Upstream</th>
               <th scope="col">Total Collection Entries</th>
               <th scope="col">Total Work Entries</th>
+              <th scope="col">Total File Set Entries</th>
               <th scope="col"></th>
               <th scope="col"></th>
               <th scope="col"></th>
@@ -36,12 +37,13 @@
                 <td><%= importer.status %></td>
                 <td><%= importer.last_imported_at.strftime("%b %d, %Y") if importer.last_imported_at %></td>
                 <td><%= importer.next_import_at.strftime("%b %d, %Y") if importer.next_import_at %></td>
-                <td><%= importer.importer_runs.last&.enqueued_records %></td>
-                <td><%= (importer.importer_runs.last&.processed_collections || 0) + (importer.importer_runs.last&.processed_records || 0) %></td>
-                <td><%= (importer.importer_runs.last&.failed_collections || 0) + (importer.importer_runs.last&.failed_records || 0) %></td>
-                <td><%= importer.importer_runs.last&.deleted_records %></td>
-                <td><%= importer.importer_runs.last&.total_collection_entries %></td>
-                <td><%= importer.importer_runs.last&.total_work_entries %></td>
+                <td><%= importer.last_run&.enqueued_records %></td>
+                <td><%= (importer.last_run&.processed_records || 0) %></td>
+                <td><%= (importer.last_run&.failed_records || 0) %></td>
+                <td><%= importer.last_run&.deleted_records %></td>
+                <td><%= importer.last_run&.total_collection_entries %></td>
+                <td><%= importer.last_run&.total_work_entries %></td>
+                <td><%= importer.last_run&.total_file_set_entries %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), importer_path(importer) %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(importer) %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>

data/app/views/bulkrax/importers/show.html.erb

@@ -56,14 +56,19 @@
 
     <%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @importer} %>
 
-    <p class="bulkrax-p-align">
+    <p class="bulkrax-p-align" title="<%= @importer.last_run&.processed_works %> processed, <%= @importer.last_run&.failed_works %> failed">
       <strong>Total Works:</strong>
-      <%= @importer.importer_runs.last&.total_work_entries %>
+      <%= @importer.last_run&.total_work_entries %>
     </p>
 
-    <p class="bulkrax-p-align">
+    <p class="bulkrax-p-align" title="<%= @importer.last_run&.processed_collections %> processed, <%= @importer.last_run&.failed_collections %> failed">
       <strong>Total Collections:</strong>
-      <%= @importer.importer_runs.last&.total_collection_entries %>
+      <%= @importer.last_run&.total_collection_entries %>
+    </p>
+
+    <p class="bulkrax-p-align" title="<%= @importer.last_run&.processed_file_sets %> processed, <%= @importer.last_run&.failed_file_sets %> failed">
+      <strong>Total File Sets:</strong>
+      <%= @importer.last_run&.total_file_set_entries %>
     </p>
 
     <div class="bulkrax-nav-tab-bottom-margin">
@@ -71,6 +76,7 @@
       <ul class="bulkrax-nav-tab-top-margin tab-nav nav nav-tabs" role="tablist">
         <li role="presentation" class='active'><a href="#work-entries" aria-controls="work-entries" role="tab" data-toggle="tab">Work Entries</a></li>
         <li role="presentation"><a href="#collection-entries" aria-controls="collection-entries" role="tab" data-toggle="tab">Collection Entries</a></li>
+        <li role="presentation"><a href="#file-set-entries" aria-controls="file-set-entries" role="tab" data-toggle="tab">File Set Entries</a></li>
       </ul>
       <!-- Tab panes -->
       <div class="tab-content outline">
@@ -158,6 +164,42 @@
           <%= page_entries_info(@collection_entries) %><br />
           <%= paginate(@collection_entries, theme: 'blacklight', param_name: :collections_entries_page, params: {anchor: 'collection-entries'}) %>
         </div>
+        <div role="tabpanel" class="tab-pane bulkrax-nav-tab-table-left-align" id="file-set-entries">
+          <table class='table table-striped'>
+            <thead>
+              <tr>
+                <th>Identifier</th>
+                <th>Entry ID</th>
+                <th>Status</th>
+                <th>Errors</th>
+                <th>Status Set At</th>
+                <th>Actions</th>
+              </tr>
+            </thead>
+            <tbody>
+              <% @file_set_entries.each do |e| %>
+                <tr>
+                  <td><%= link_to e.identifier, bulkrax.importer_entry_path(@importer.id, e.id) %></td>
+                  <td><%= e.id %></td>
+                  <% if e.status == "Complete" %>
+                    <td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
+                  <% else %>
+                    <td><span class="glyphicon glyphicon-remove" style="color: red;"></span> <%= e.status %></td>
+                  <% end %>
+                  <% if e.last_error.present? %>
+                    <td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
+                  <% else %>
+                    <td></td>
+                  <% end %>
+                  <td><%= e.status_at %></td>
+                  <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+          <%= page_entries_info(@file_set_entries) %><br />
+          <%= paginate(@file_set_entries, theme: 'blacklight', param_name: :file_set_entries_page, params: {anchor: 'file-set-entries'}) %>
+        </div>
       </div>
     </div>
 

data/db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb

@@ -0,0 +1,7 @@
+class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_importer_runs, :processed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_file_sets)
+    add_column :bulkrax_importer_runs, :failed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_file_sets)
+    add_column :bulkrax_importer_runs, :total_file_set_entries, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :total_file_set_entries)
+  end
+end

data/db/migrate/20220118001339_add_import_attempts_to_entries.rb

@@ -0,0 +1,5 @@
+class AddImportAttemptsToEntries < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_entries, :import_attempts, :integer, default: 0 unless column_exists?(:bulkrax_entries, :import_attempts)
+  end
+end

data/db/migrate/20220119213325_add_work_counters_to_importer_runs.rb

@@ -0,0 +1,6 @@
+class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_importer_runs, :processed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_works)
+    add_column :bulkrax_importer_runs, :failed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_works)
+  end
+end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '2.0.0'
+  VERSION = '2.1.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-14 00:00:00.000000000 Z
+date: 2022-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -224,7 +224,11 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Description of Bulkrax.
+description: Bulkrax is a batteries included importer for Samvera applications. It
+  currently includes support for OAI-PMH (DC and Qualified DC) and CSV out of the
+  box. It is also designed to be extensible, allowing you to easily add new importers
+  in to your application or to include them with other gems. Bulkrax provides a full
+  admin interface including creating, editing, scheduling and reviewing imports.
 email:
 - rob@notch8.com
 executables: []
@@ -261,6 +265,7 @@ files:
 - app/jobs/bulkrax/export_work_job.rb
 - app/jobs/bulkrax/exporter_job.rb
 - app/jobs/bulkrax/import_collection_job.rb
+- app/jobs/bulkrax/import_file_set_job.rb
 - app/jobs/bulkrax/import_work_job.rb
 - app/jobs/bulkrax/importer_job.rb
 - app/mailers/bulkrax/application_mailer.rb
@@ -271,6 +276,7 @@ files:
 - app/models/bulkrax/application_record.rb
 - app/models/bulkrax/csv_collection_entry.rb
 - app/models/bulkrax/csv_entry.rb
+- app/models/bulkrax/csv_file_set_entry.rb
 - app/models/bulkrax/entry.rb
 - app/models/bulkrax/exporter.rb
 - app/models/bulkrax/exporter_run.rb
@@ -286,6 +292,7 @@ files:
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
 - app/models/concerns/bulkrax/download_behavior.rb
+- app/models/concerns/bulkrax/dynamic_record_lookup.rb
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
 - app/models/concerns/bulkrax/file_factory.rb
@@ -353,6 +360,9 @@ files:
 - db/migrate/20210806065737_increase_text_sizes.rb
 - db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb
 - db/migrate/20211203195233_rename_children_counters_to_relationships.rb
+- db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb
+- db/migrate/20220118001339_add_import_attempts_to_entries.rb
+- db/migrate/20220119213325_add_work_counters_to_importer_runs.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/version.rb
@@ -368,7 +378,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -383,8 +393,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
-summary: Summary of Bulkrax.
+summary: Import and export tool for Hyrax and Hyku
 test_files: []