bulkrax 2.0.2 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c862fc4eef8cf2d892ab2c3dd635ed6070d887b87104b007dc5dd7fffdd5fd51
-  data.tar.gz: 7e735123366e3e7998d2e99ccdaf97047560e2e8fc8695a69e21da8abb6234a0
+  metadata.gz: b581b6768e5d7300546aec47d8cdfdde60cf462f37b51f6b434e92d1f9018334
+  data.tar.gz: 8d82be5129abaea32753bebc862cd5da8960cda8865e9749cbf5cfff1b49b012
 SHA512:
-  metadata.gz: 5b6bb1cd44e421a22cb93df50aea16292da25e3d4e124797ec2024ad17fb8d239033c1c89d9cec7984fd3dff575278443489b59523fe46e328f63a8f265be826
-  data.tar.gz: 43d65c3c6ef09d07cb677dfd35b2476eaa8e32c8d5ddcbddfd1a55b25541a882f3611414d38d351b134763723181ce2a15ab926e0a306cb8157e1417763b97b8
+  metadata.gz: f98c57a345d2bf08789877cc854bcc50680a97b7b2638965cd7940e311e4ff72c0db86ff4ca71e57c7058f2609035fa2ec402f298df439c5899190cfd8efc8c8
+  data.tar.gz: '09c7d9d8d51db04f5f01356c22f09e9d5041880effec9352b853d609668a58809dbb0f151d93f8c95c3f53bafcf3bbb57a6a4ab036616583dec1ed124d6d101b'

data/app/controllers/bulkrax/importers_controller.rb

@@ -37,6 +37,7 @@ module Bulkrax
 
         @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
         @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
+        @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
       end
     end
 

data/app/factories/bulkrax/object_factory.rb

@@ -4,11 +4,13 @@ module Bulkrax
   class ObjectFactory
     extend ActiveModel::Callbacks
     include Bulkrax::FileFactory
+    include DynamicRecordLookup
+
     define_model_callbacks :save, :create
-    attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :collection_field_mapping
+    attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :collection_field_mapping, :related_parents_parsed_mapping
 
     # rubocop:disable Metrics/ParameterLists
-    def initialize(attributes:, source_identifier_value:, work_identifier:, collection_field_mapping:, replace_files: false, user: nil, klass: nil, update_files: false)
+    def initialize(attributes:, source_identifier_value:, work_identifier:, collection_field_mapping:, related_parents_parsed_mapping: nil, replace_files: false, user: nil, klass: nil, update_files: false)
       ActiveSupport::Deprecation.warn(
         'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
         ' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
@@ -19,6 +21,7 @@ module Bulkrax
       @user = user || User.batch_user
       @work_identifier = work_identifier
       @collection_field_mapping = collection_field_mapping
+      @related_parents_parsed_mapping = related_parents_parsed_mapping
       @source_identifier_value = source_identifier_value
       @klass = klass || Bulkrax.default_work_type.constantize
     end
@@ -33,7 +36,7 @@ module Bulkrax
       arg_hash = { id: attributes[:id], name: 'UPDATE', klass: klass }
       @object = find
       if object
-        object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
+        object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent)
         ActiveSupport::Notifications.instrument('import.importer', arg_hash) { update }
       else
         ActiveSupport::Notifications.instrument('import.importer', arg_hash.merge(name: 'CREATE')) { create }
@@ -51,10 +54,16 @@ module Bulkrax
 
     def update
       raise "Object doesn't exist" unless object
-      destroy_existing_files if @replace_files && klass != Collection
+      destroy_existing_files if @replace_files && ![Collection, FileSet].include?(klass)
       attrs = attribute_update
       run_callbacks :save do
-        klass == Collection ? update_collection(attrs) : work_actor.update(environment(attrs))
+        if klass == Collection
+          update_collection(attrs)
+        elsif klass == FileSet
+          update_file_set(attrs)
+        else
+          work_actor.update(environment(attrs))
+        end
       end
       log_updated(object)
     end
@@ -90,10 +99,16 @@ module Bulkrax
     def create
       attrs = create_attributes
       @object = klass.new
-      object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
+      object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent)
       run_callbacks :save do
         run_callbacks :create do
-          klass == Collection ? create_collection(attrs) : work_actor.create(environment(attrs))
+          if klass == Collection
+            create_collection(attrs)
+          elsif klass == FileSet
+            create_file_set(attrs)
+          else
+            work_actor.create(environment(attrs))
+          end
         end
       end
       log_created(object)
@@ -150,6 +165,35 @@ module Bulkrax
       object.save!
     end
 
+    # This method is heavily inspired by Hyrax's AttachFilesToWorkJob
+    def create_file_set(attrs)
+      work = find_record(attributes[related_parents_parsed_mapping].first)
+      work_permissions = work.permissions.map(&:to_hash)
+      file_set_attrs = attrs.slice(*object.attributes.keys)
+      object.assign_attributes(file_set_attrs)
+
+      attrs['uploaded_files'].each do |uploaded_file_id|
+        uploaded_file = ::Hyrax::UploadedFile.find(uploaded_file_id)
+        next if uploaded_file.file_set_uri.present?
+
+        actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
+        uploaded_file.update(file_set_uri: actor.file_set.uri)
+        actor.file_set.permissions_attributes = work_permissions
+        actor.create_metadata
+        actor.create_content(uploaded_file)
+        actor.attach_to_work(work)
+      end
+
+      object.save!
+    end
+
+    def update_file_set(attrs)
+      file_set_attrs = attrs.slice(*object.attributes.keys)
+      actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
+
+      actor.update_metadata(file_set_attrs)
+    end
+
     # Add child to parent's #member_collections
     # Add parent to child's #member_of_collections
     def persist_collection_memberships(parent:, child:)

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -17,6 +17,8 @@ module Bulkrax
   # NOTE: In the context of this job, "identifier" is used to generically refer
   #       to either a record's ID or an Bulkrax::Entry's source_identifier.
   class CreateRelationshipsJob < ApplicationJob
+    include DynamicRecordLookup
+
     queue_as :import
 
     attr_accessor :base_entry, :child_record, :parent_record, :importer_run
@@ -74,39 +76,6 @@ module Bulkrax
       end
     end
 
-    # This method allows us to create relationships with preexisting records (by their ID) OR
-    # with records that are concurrently being imported (by their Bulkrax::Entry source_identifier).
-    #
-    # @param identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
-    # @return [Work, Collection, nil] Work or Collection if found, otherwise nil
-    def find_record(identifier)
-      record = Entry.find_by(identifier: identifier)
-      record ||= ::Collection.where(id: identifier).first
-      if record.blank?
-        available_work_types.each do |work_type|
-          record ||= work_type.where(id: identifier).first
-        end
-      end
-      record = record.factory.find if record.is_a?(Entry)
-
-      record
-    end
-
-    # Check if the record is a Work
-    def curation_concern?(record)
-      available_work_types.include?(record.class)
-    end
-
-    # @return [Array<Class>] list of work type classes
-    def available_work_types
-      # If running in a Hyku app, do not reference disabled work types
-      @available_work_types ||= if defined?(::Hyku)
-                                  ::Site.instance.available_works.map(&:constantize)
-                                else
-                                  ::Hyrax.config.curation_concerns
-                                end
-    end
-
     def user
       @user ||= importer_run.importer.user
     end

data/app/jobs/bulkrax/import_collection_job.rb

@@ -11,9 +11,11 @@ module Bulkrax
         entry.build
         entry.save
         add_user_to_permission_template!(entry)
+        ImporterRun.find(args[1]).increment!(:processed_records)
         ImporterRun.find(args[1]).increment!(:processed_collections)
         ImporterRun.find(args[1]).decrement!(:enqueued_records)
       rescue => e
+        ImporterRun.find(args[1]).increment!(:failed_records)
         ImporterRun.find(args[1]).increment!(:failed_collections)
         ImporterRun.find(args[1]).decrement!(:enqueued_records)
         raise e

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class MissingParentError < ::StandardError; end
+  class ImportFileSetJob < ApplicationJob
+    include DynamicRecordLookup
+
+    queue_as :import
+
+    def perform(entry_id, importer_run_id)
+      entry = Entry.find(entry_id)
+      parent_identifier = entry.raw_metadata[entry.related_parents_raw_mapping]&.strip
+
+      validate_parent!(parent_identifier)
+
+      entry.build
+      if entry.succeeded?
+        # rubocop:disable Rails/SkipsModelValidations
+        ImporterRun.find(importer_run_id).increment!(:processed_records)
+        ImporterRun.find(importer_run_id).increment!(:processed_file_sets)
+      else
+        ImporterRun.find(importer_run_id).increment!(:failed_records)
+        ImporterRun.find(importer_run_id).increment!(:failed_file_sets)
+        # rubocop:enable Rails/SkipsModelValidations
+      end
+      ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
+      entry.save!
+
+    rescue MissingParentError => e
+      # try waiting for the parent record to be created
+      entry.import_attempts += 1
+      entry.save!
+      if entry.import_attempts < 5
+        ImportFileSetJob
+          .set(wait: (entry.import_attempts + 1).minutes)
+          .perform_later(entry_id, importer_run_id)
+      else
+        ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
+        entry.status_info(e)
+      end
+    end
+
+    private
+
+    attr_reader :parent_record
+
+    def validate_parent!(parent_identifier)
+      # if parent_identifier is missing, it will be caught by #validate_presence_of_parent!
+      return if parent_identifier.blank?
+
+      find_parent_record(parent_identifier)
+      check_parent_exists!(parent_identifier)
+      check_parent_is_a_work!(parent_identifier)
+    end
+
+    def check_parent_exists!(parent_identifier)
+      raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") if parent_record.blank?
+    end
+
+    def check_parent_is_a_work!(parent_identifier)
+      error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
+      raise ::StandardError, error_msg unless curation_concern?(parent_record)
+    end
+
+    def find_parent_record(parent_identifier)
+      @parent_record ||= find_record(parent_identifier)
+    end
+  end
+end

data/app/jobs/bulkrax/import_work_job.rb

@@ -10,11 +10,13 @@ module Bulkrax
       entry.build
       if entry.status == "Complete"
         ImporterRun.find(args[1]).increment!(:processed_records)
+        ImporterRun.find(args[1]).increment!(:processed_works)
         ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
       else
         # do not retry here because whatever parse error kept you from creating a work will likely
         # keep preventing you from doing so.
         ImporterRun.find(args[1]).increment!(:failed_records)
+        ImporterRun.find(args[1]).increment!(:failed_works)
         ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
       end
       entry.save!

data/app/jobs/bulkrax/importer_job.rb

@@ -20,6 +20,7 @@ module Bulkrax
 
       importer.import_collections
       importer.import_works
+      importer.import_file_sets
     end
 
     def unzip_imported_file(parser)
@@ -31,6 +32,7 @@ module Bulkrax
     def update_current_run_counters(importer)
       importer.current_run.total_work_entries = importer.limit || importer.parser.works_total
       importer.current_run.total_collection_entries = importer.parser.collections_total
+      importer.current_run.total_file_set_entries = importer.parser.file_sets_total
       importer.current_run.save!
     end
 

data/app/models/bulkrax/csv_entry.rb

@@ -40,9 +40,9 @@ module Bulkrax
 
       self.parsed_metadata = {}
       add_identifier
-      add_metadata_for_model
       add_visibility
       add_ingested_metadata
+      add_metadata_for_model
       add_rights_statement
       add_collections
       add_local
@@ -57,6 +57,9 @@ module Bulkrax
     def add_metadata_for_model
       if factory_class == Collection
         add_collection_type_gid
+      elsif factory_class == FileSet
+        add_path_to_file
+        validate_presence_of_parent!
       else
         add_file unless importerexporter.metadata_only?
         add_admin_set_id
@@ -85,7 +88,11 @@ module Bulkrax
       elsif record['file'].is_a?(Array)
         self.parsed_metadata['file'] = record['file']
       end
-      self.parsed_metadata['file'] = self.parsed_metadata['file'].map { |f| path_to_file(f.tr(' ', '_')) }
+      self.parsed_metadata['file'] = self.parsed_metadata['file'].map do |f|
+        next if f.blank?
+
+        path_to_file(f.tr(' ', '_'))
+      end.compact
     end
 
     def build_export_metadata

data/app/models/bulkrax/csv_file_set_entry.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class CsvFileSetEntry < CsvEntry
+    def factory_class
+      ::FileSet
+    end
+
+    def add_path_to_file
+      parsed_metadata['file'].each_with_index do |filename, i|
+        path_to_file = ::File.join(parser.path_to_files, filename)
+
+        parsed_metadata['file'][i] = path_to_file
+      end
+      raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
+
+      parsed_metadata['file']
+    end
+
+    def validate_presence_of_parent!
+      return if parsed_metadata[related_parents_parsed_mapping]&.map(&:present?)&.any?
+
+      raise StandardError, 'File set must be related to at least one work'
+    end
+  end
+end

data/app/models/bulkrax/importer.rb

@@ -99,7 +99,12 @@ module Bulkrax
       @current_run ||= if file? && zip?
                          self.importer_runs.create!
                        else
-                         self.importer_runs.create!(total_work_entries: self.limit || parser.works_total, total_collection_entries: parser.collections_total)
+                         entry_counts = {
+                           total_work_entries: self.limit || parser.works_total,
+                           total_collection_entries: parser.collections_total,
+                           total_file_set_entries: parser.file_sets_total
+                         }
+                         self.importer_runs.create!(entry_counts)
                        end
     end
 
@@ -134,6 +139,13 @@ module Bulkrax
       status_info(e)
     end
 
+    def import_file_sets
+      self.save if self.new_record? # Object needs to be saved for statuses
+      parser.create_file_sets
+    rescue StandardError => e
+      status_info(e)
+    end
+
     # Prepend the base_url to ensure unique set identifiers
     # @todo - move to parser, as this is OAI specific
     def unique_collection_identifier(id)

data/app/models/concerns/bulkrax/download_behavior.rb

@@ -30,7 +30,7 @@ module Bulkrax
 
     # Create some headers for the datastream
     def content_options
-      { disposition: 'inline', type: download_content_type, filename: file_name }
+      { disposition: 'attachment', type: download_content_type, filename: file_name }
     end
 
     # render an HTTP HEAD response

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  module DynamicRecordLookup
+    # Search entries, collections, and every available work type for a record that
+    # has the provided identifier.
+    #
+    # @param identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
+    # @return [Work, Collection, nil] Work or Collection if found, otherwise nil
+    def find_record(identifier)
+      record = Entry.find_by(identifier: identifier)
+      record ||= ::Collection.where(id: identifier).first # rubocop:disable Rails/FindBy
+      if record.blank?
+        available_work_types.each do |work_type|
+          record ||= work_type.where(id: identifier).first # rubocop:disable Rails/FindBy
+        end
+      end
+
+      record.is_a?(Entry) ? record.factory.find : record
+    end
+
+    # Check if the record is a Work
+    def curation_concern?(record)
+      available_work_types.include?(record.class)
+    end
+
+    private
+
+    # @return [Array<Class>] list of work type classes
+    def available_work_types
+      # If running in a Hyku app, do not include disabled work types
+      @available_work_types ||= if defined?(::Hyku)
+                                  ::Site.instance.available_works.map(&:constantize)
+                                else
+                                  ::Hyrax.config.curation_concerns
+                                end
+    end
+  end
+end

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -96,6 +96,7 @@ module Bulkrax
                                               source_identifier_value: identifier,
                                               work_identifier: parser.work_identifier,
                                               collection_field_mapping: parser.collection_field_mapping,
+                                              related_parents_parsed_mapping: related_parents_parsed_mapping,
                                               replace_files: replace_files,
                                               user: user,
                                               klass: factory_class,

data/app/models/concerns/bulkrax/importer_exporter_behavior.rb

@@ -20,15 +20,17 @@ module Bulkrax
       (last_imported_at || Time.current) + frequency.to_seconds if schedulable? && last_imported_at.present?
     end
 
-    def increment_counters(index, collection = false)
+    def increment_counters(index, collection: false, file_set: false)
       # Only set the totals if they were not set on initialization
       if collection
         current_run.total_collection_entries = index + 1 unless parser.collections_total.positive?
+      elsif file_set
+        current_run.total_file_set_entries = index + 1 unless parser.file_sets_total.positive?
       else
         # TODO: differentiate between work and collection counts for exporters
         current_run.total_work_entries = index + 1 unless limit.to_i.positive? || parser.total.positive?
       end
-      current_run.enqueued_records = index + 1
+      current_run.enqueued_records += 1
       current_run.save!
     end
 

data/app/parsers/bulkrax/application_parser.rb

@@ -114,6 +114,10 @@ module Bulkrax
       raise StandardError, 'must be defined' if importer?
     end
 
+    def create_file_sets
+      raise StandardError, 'must be defined' if importer?
+    end
+
     # Optional, define if using browse everything for file upload
     def retrieve_cloud_files(files); end
 
@@ -234,6 +238,10 @@ module Bulkrax
       0
     end
 
+    def file_sets_total
+      0
+    end
+
     def write
       write_files
       zip

data/app/parsers/bulkrax/bagit_parser.rb

@@ -59,7 +59,7 @@ module Bulkrax
         }
         new_entry = find_or_create_entry(collection_entry_class, collection, 'Bulkrax::Importer', metadata)
         ImportCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, true)
+        increment_counters(index, collection: true)
       end
     end
 

data/app/parsers/bulkrax/csv_parser.rb

@@ -38,13 +38,27 @@ module Bulkrax
     end
 
     def works
-      records - collections
+      records - collections - file_sets
     end
 
     def works_total
       works.size
     end
 
+    def file_sets
+      records.map do |r|
+        file_sets = []
+        model_field_mappings.each do |model_mapping|
+          file_sets << r if r[model_mapping.to_sym]&.downcase == 'fileset'
+        end
+        file_sets
+      end.flatten.compact.uniq
+    end
+
+    def file_sets_total
+      file_sets.size
+    end
+
     # We could use CsvEntry#fields_from_data(data) but that would mean re-reading the data
     def import_fields
       @import_fields ||= records.inject(:merge).keys.compact.uniq
@@ -85,7 +99,7 @@ module Bulkrax
         metadata = if collection.delete(:from_collection_field_mapping)
                      uci = unique_collection_identifier(collection)
                      {
-                       title: [collection[:title]],
+                       title: collection[:title],
                        work_identifier => uci,
                        source_identifier => uci,
                        visibility: 'open',
@@ -96,9 +110,9 @@ module Bulkrax
         ## END
 
         new_entry = find_or_create_entry(collection_entry_class, collection_hash[source_identifier], 'Bulkrax::Importer', collection_hash)
+        increment_counters(index, collection: true)
         # TODO: add support for :delete option
         ImportCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, true)
       end
       importer.record_status
     rescue StandardError => e
@@ -124,6 +138,20 @@ module Bulkrax
       status_info(e)
     end
 
+    def create_file_sets
+      file_sets.each_with_index do |file_set, index|
+        next unless record_has_source_identifier(file_set, records.find_index(file_set))
+        break if limit_reached?(limit, records.find_index(file_set))
+
+        new_entry = find_or_create_entry(file_set_entry_class, file_set[source_identifier], 'Bulkrax::Importer', file_set.to_h)
+        ImportFileSetJob.perform_later(new_entry.id, current_run.id)
+        increment_counters(index, file_set: true)
+      end
+      importer.record_status
+    rescue StandardError => e
+      status_info(e)
+    end
+
     def write_partial_import_file(file)
       import_filename = import_file_path.split('/').last
       partial_import_filename = "#{File.basename(import_filename, '.csv')}_corrected_entries.csv"
@@ -149,40 +177,74 @@ module Bulkrax
     end
 
     def current_work_ids
+      ActiveSupport::Deprication.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
+      current_record_ids
+    end
+
+    def current_record_ids
+      @work_ids = []
+      @collection_ids = []
+      @file_set_ids = []
+
       case importerexporter.export_from
       when 'all'
-        ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", rows: 2_147_483_647).map(&:id)
+        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", rows: 2_147_483_647).map(&:id)
+        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", rows: 2_147_483_647).map(&:id)
+        @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", rows: 2_147_483_647).map(&:id)
       when 'collection'
-        ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
+        @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
       when 'worktype'
-        ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
+        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
       when 'importer'
-        entry_ids = Bulkrax::Importer.find(importerexporter.export_source).entries.pluck(:id)
-        complete_statuses = Bulkrax::Status.latest_by_statusable
-                                           .includes(:statusable)
-                                           .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
+        set_ids_for_exporting_from_importer
+      end
+
+      @work_ids + @collection_ids + @file_set_ids
+    end
 
-        complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
-        extra_filters = extra_filters.presence || '*:*'
+    # Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
+    # @see #current_record_ids
+    def set_ids_for_exporting_from_importer
+      entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
+      complete_statuses = Status.latest_by_statusable
+                                .includes(:statusable)
+                                .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
 
-        ActiveFedora::SolrService.get(
+      complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
+      extra_filters = extra_filters.presence || '*:*'
+
+      { :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
+        instance_variable_set(instance_var, ActiveFedora::SolrService.get(
           extra_filters.to_s,
-          fq: "#{work_identifier}_sim:(#{complete_entry_identifiers.join(' OR ')})",
+          fq: [
+            "#{work_identifier}_sim:(#{complete_entry_identifiers.join(' OR ')})",
+            "has_model_ssim:(#{models_to_search.join(' OR ')})"
+          ],
           fl: 'id',
           rows: 2_000_000_000
-        )['response']['docs'].map { |obj| obj['id'] }
+        )['response']['docs'].map { |obj| obj['id'] })
       end
     end
 
     def create_new_entries
-      current_work_ids.each_with_index do |wid, index|
+      current_record_ids.each_with_index do |id, index|
         break if limit_reached?(limit, index)
-        new_entry = find_or_create_entry(entry_class, wid, 'Bulkrax::Exporter')
+
+        this_entry_class = if @collection_ids.include?(id)
+                             collection_entry_class
+                           elsif @file_set_ids.include?(id)
+                             file_set_entry_class
+                           else
+                             entry_class
+                           end
+        new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
+
         begin
-          entry = Bulkrax::ExportWorkJob.perform_now(new_entry.id, current_run.id)
+          entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
         rescue => e
           Rails.logger.info("#{e.message} was detected during export")
         end
+
         self.headers |= entry.parsed_metadata.keys if entry
       end
     end
@@ -199,6 +261,10 @@ module Bulkrax
       CsvCollectionEntry
     end
 
+    def file_set_entry_class
+      CsvFileSetEntry
+    end
+
     # See https://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
     #   Changed to grep as wc -l counts blank lines, and ignores the final unescaped line (which may or may not contain data)
     def total
@@ -235,7 +301,7 @@ module Bulkrax
 
     def write_files
       CSV.open(setup_export_file, "w", headers: export_headers, write_headers: true) do |csv|
-        importerexporter.entries.where(identifier: current_work_ids)[0..limit || total].each do |e|
+        importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |e|
           csv << e.parsed_metadata
         end
       end

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -76,7 +76,7 @@ module Bulkrax
         new_entry = collection_entry_class.where(importerexporter: importerexporter, identifier: unique_collection_identifier, raw_metadata: metadata).first_or_create!
         # perform now to ensure this gets created before work imports start
         ImportCollectionJob.perform_now(new_entry.id, importerexporter.current_run.id)
-        increment_counters(index, true)
+        increment_counters(index, collection: true)
       end
     end
 

data/app/views/bulkrax/importers/index.html.erb

@@ -24,6 +24,7 @@
               <th scope="col">Entries Deleted Upstream</th>
               <th scope="col">Total Collection Entries</th>
               <th scope="col">Total Work Entries</th>
+              <th scope="col">Total File Set Entries</th>
               <th scope="col"></th>
               <th scope="col"></th>
               <th scope="col"></th>
@@ -36,12 +37,13 @@
                 <td><%= importer.status %></td>
                 <td><%= importer.last_imported_at.strftime("%b %d, %Y") if importer.last_imported_at %></td>
                 <td><%= importer.next_import_at.strftime("%b %d, %Y") if importer.next_import_at %></td>
-                <td><%= importer.importer_runs.last&.enqueued_records %></td>
-                <td><%= (importer.importer_runs.last&.processed_collections || 0) + (importer.importer_runs.last&.processed_records || 0) %></td>
-                <td><%= (importer.importer_runs.last&.failed_collections || 0) + (importer.importer_runs.last&.failed_records || 0) %></td>
-                <td><%= importer.importer_runs.last&.deleted_records %></td>
-                <td><%= importer.importer_runs.last&.total_collection_entries %></td>
-                <td><%= importer.importer_runs.last&.total_work_entries %></td>
+                <td><%= importer.last_run&.enqueued_records %></td>
+                <td><%= (importer.last_run&.processed_records || 0) %></td>
+                <td><%= (importer.last_run&.failed_records || 0) %></td>
+                <td><%= importer.last_run&.deleted_records %></td>
+                <td><%= importer.last_run&.total_collection_entries %></td>
+                <td><%= importer.last_run&.total_work_entries %></td>
+                <td><%= importer.last_run&.total_file_set_entries %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), importer_path(importer) %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(importer) %></td>
                 <td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), importer, method: :delete, data: { confirm: 'Are you sure?' } %></td>

data/app/views/bulkrax/importers/show.html.erb

@@ -56,14 +56,19 @@
 
     <%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @importer} %>
 
-    <p class="bulkrax-p-align">
+    <p class="bulkrax-p-align" title="<%= @importer.last_run&.processed_works %> processed, <%= @importer.last_run&.failed_works %> failed">
       <strong>Total Works:</strong>
-      <%= @importer.importer_runs.last&.total_work_entries %>
+      <%= @importer.last_run&.total_work_entries %>
     </p>
 
-    <p class="bulkrax-p-align">
+    <p class="bulkrax-p-align" title="<%= @importer.last_run&.processed_collections %> processed, <%= @importer.last_run&.failed_collections %> failed">
       <strong>Total Collections:</strong>
-      <%= @importer.importer_runs.last&.total_collection_entries %>
+      <%= @importer.last_run&.total_collection_entries %>
+    </p>
+
+    <p class="bulkrax-p-align" title="<%= @importer.last_run&.processed_file_sets %> processed, <%= @importer.last_run&.failed_file_sets %> failed">
+      <strong>Total File Sets:</strong>
+      <%= @importer.last_run&.total_file_set_entries %>
     </p>
 
     <div class="bulkrax-nav-tab-bottom-margin">
@@ -71,6 +76,7 @@
       <ul class="bulkrax-nav-tab-top-margin tab-nav nav nav-tabs" role="tablist">
         <li role="presentation" class='active'><a href="#work-entries" aria-controls="work-entries" role="tab" data-toggle="tab">Work Entries</a></li>
         <li role="presentation"><a href="#collection-entries" aria-controls="collection-entries" role="tab" data-toggle="tab">Collection Entries</a></li>
+        <li role="presentation"><a href="#file-set-entries" aria-controls="file-set-entries" role="tab" data-toggle="tab">File Set Entries</a></li>
       </ul>
       <!-- Tab panes -->
       <div class="tab-content outline">
@@ -158,6 +164,42 @@
           <%= page_entries_info(@collection_entries) %><br />
           <%= paginate(@collection_entries, theme: 'blacklight', param_name: :collections_entries_page, params: {anchor: 'collection-entries'}) %>
         </div>
+        <div role="tabpanel" class="tab-pane bulkrax-nav-tab-table-left-align" id="file-set-entries">
+          <table class='table table-striped'>
+            <thead>
+              <tr>
+                <th>Identifier</th>
+                <th>Entry ID</th>
+                <th>Status</th>
+                <th>Errors</th>
+                <th>Status Set At</th>
+                <th>Actions</th>
+              </tr>
+            </thead>
+            <tbody>
+              <% @file_set_entries.each do |e| %>
+                <tr>
+                  <td><%= link_to e.identifier, bulkrax.importer_entry_path(@importer.id, e.id) %></td>
+                  <td><%= e.id %></td>
+                  <% if e.status == "Complete" %>
+                    <td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
+                  <% else %>
+                    <td><span class="glyphicon glyphicon-remove" style="color: red;"></span> <%= e.status %></td>
+                  <% end %>
+                  <% if e.last_error.present? %>
+                    <td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
+                  <% else %>
+                    <td></td>
+                  <% end %>
+                  <td><%= e.status_at %></td>
+                  <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), bulkrax.importer_entry_path(@importer.id, e.id) %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+          <%= page_entries_info(@file_set_entries) %><br />
+          <%= paginate(@file_set_entries, theme: 'blacklight', param_name: :file_set_entries_page, params: {anchor: 'file-set-entries'}) %>
+        </div>
       </div>
     </div>
 

data/db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb

@@ -0,0 +1,7 @@
+class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_importer_runs, :processed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_file_sets)
+    add_column :bulkrax_importer_runs, :failed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_file_sets)
+    add_column :bulkrax_importer_runs, :total_file_set_entries, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :total_file_set_entries)
+  end
+end

data/db/migrate/20220118001339_add_import_attempts_to_entries.rb

@@ -0,0 +1,5 @@
+class AddImportAttemptsToEntries < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_entries, :import_attempts, :integer, default: 0 unless column_exists?(:bulkrax_entries, :import_attempts)
+  end
+end

data/db/migrate/20220119213325_add_work_counters_to_importer_runs.rb

@@ -0,0 +1,6 @@
+class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_importer_runs, :processed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_works)
+    add_column :bulkrax_importer_runs, :failed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_works)
+  end
+end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '2.0.2'
+  VERSION = '2.2.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.2.1
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-31 00:00:00.000000000 Z
+date: 2022-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -265,6 +265,7 @@ files:
 - app/jobs/bulkrax/export_work_job.rb
 - app/jobs/bulkrax/exporter_job.rb
 - app/jobs/bulkrax/import_collection_job.rb
+- app/jobs/bulkrax/import_file_set_job.rb
 - app/jobs/bulkrax/import_work_job.rb
 - app/jobs/bulkrax/importer_job.rb
 - app/mailers/bulkrax/application_mailer.rb
@@ -275,6 +276,7 @@ files:
 - app/models/bulkrax/application_record.rb
 - app/models/bulkrax/csv_collection_entry.rb
 - app/models/bulkrax/csv_entry.rb
+- app/models/bulkrax/csv_file_set_entry.rb
 - app/models/bulkrax/entry.rb
 - app/models/bulkrax/exporter.rb
 - app/models/bulkrax/exporter_run.rb
@@ -290,6 +292,7 @@ files:
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
 - app/models/concerns/bulkrax/download_behavior.rb
+- app/models/concerns/bulkrax/dynamic_record_lookup.rb
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
 - app/models/concerns/bulkrax/file_factory.rb
@@ -357,6 +360,9 @@ files:
 - db/migrate/20210806065737_increase_text_sizes.rb
 - db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb
 - db/migrate/20211203195233_rename_children_counters_to_relationships.rb
+- db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb
+- db/migrate/20220118001339_add_import_attempts_to_entries.rb
+- db/migrate/20220119213325_add_work_counters_to_importer_runs.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/version.rb
@@ -372,7 +378,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -387,8 +393,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku
 test_files: []