bulkrax 3.1.2 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1738f148a6fe4a7a4bc3b97a39e08169f04540c9814dc1ba6c4c14350591440
-  data.tar.gz: 995db6ebd622ccef4fe95c73e240b85bd3e303f9a00fe164c95f602b96c694b2
+  metadata.gz: 2525e4aed8b31a897668d556c311b29189ec1b7a951b4303604acd3ccb11fcc8
+  data.tar.gz: d642de2af38c8c82108b113641ab1e191ae05b96b2ce3528dde7d7c908639291
 SHA512:
-  metadata.gz: e5d7c2dfc70ddd26c80c51e580a2fb9b067cd3752df172066d68e26d0a6c5e7aa593a3779d9dad99e7d344fb632a723a13e8c663a2cf944e3bd491d3071bcc21
-  data.tar.gz: 5408270ea11b4d1d185daaf266d9de8a16a11b27777e45c9443ab6de9fb02600c1899ac08fd57267c909fe1584fb7a32fbf9b628086157ce39bbbcf2824fe646
+  metadata.gz: 6547c0283f75626cda95a8cc09cceff84ac576b7ca4034b72d729ab16d50c71657dac0e78739b5ef6f2d83aa85e1ddc8c45646f6c44408a0a989b4b33b93c487
+  data.tar.gz: 9ae11d35d0b3caf9d45237590e42541c0004268e6c86e8d44e6754900b253894813c7e817b471ed9379b7eefad1ec3a27d2cfebbe94865727ecb69b6b8652480

data/app/controllers/bulkrax/exporters_controller.rb

@@ -101,12 +101,12 @@ module Bulkrax
     def exporter_params
       params[:exporter][:export_source] = params[:exporter]["export_source_#{params[:exporter][:export_from]}".to_sym]
       if params[:exporter][:date_filter] == "1"
-        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
-                                       :parser_klass, :limit, :start_date, :finish_date, :work_visibility,
+        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type, :generated_metadata,
+                                       :include_thumbnails, :parser_klass, :limit, :start_date, :finish_date, :work_visibility,
                                        :workflow_status, field_mapping: {})
       else
-        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
-                                       :parser_klass, :limit, :work_visibility, :workflow_status,
+        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type, :generated_metadata,
+                                       :include_thumbnails, :parser_klass, :limit, :work_visibility, :workflow_status,
                                        field_mapping: {}).merge(start_date: nil, finish_date: nil)
       end
     end

data/app/{models → controllers}/concerns/bulkrax/download_behavior.rb

@@ -42,7 +42,7 @@ module Bulkrax
     def send_file_contents
       self.status = 200
       prepare_file_headers
-      stream_body file.read
+      send_file file
     end
 
     def prepare_file_headers
@@ -53,9 +53,5 @@ module Bulkrax
       response.headers['Last-Modified'] = File.mtime(file_path).utc.strftime("%a, %d %b %Y %T GMT")
       self.content_type = download_content_type
     end
-
-    def stream_body(iostream)
-      self.response_body = iostream
-    end
   end
 end

data/app/factories/bulkrax/object_factory.rb

@@ -61,6 +61,7 @@ module Bulkrax
           work_actor.update(environment(attrs))
         end
       end
+      object.apply_depositor_metadata(@user) && object.save! if object.depositor.nil?
       log_updated(object)
     end
 
@@ -107,6 +108,7 @@ module Bulkrax
           end
         end
       end
+      object.apply_depositor_metadata(@user) && object.save! if object.depositor.nil?
       log_created(object)
     end
 
@@ -141,7 +143,6 @@ module Bulkrax
       attrs = clean_attrs(attrs)
       attrs = collection_type(attrs)
       object.attributes = attrs
-      object.apply_depositor_metadata(@user)
       object.save!
     end
 

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -33,7 +33,7 @@ module Bulkrax
     # is the child in the relationship, and vice versa if a child_identifier is passed.
     def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
       pending_relationships = Bulkrax::PendingRelationship.find_each.select do |rel|
-        rel.bulkrax_importer_run_id == importer_run_id && rel.parent_id == parent_identifier
+        rel.importer_run_id == importer_run_id && rel.parent_id == parent_identifier
       end.sort_by(&:order)
 
       @importer_run_id = importer_run_id

data/app/jobs/bulkrax/delete_collection_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteCollectionJob < DeleteJob; end
+end

data/app/jobs/bulkrax/delete_file_set_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteFileSetJob < DeleteJob; end
+end

data/app/jobs/bulkrax/delete_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteJob < ApplicationJob
+    queue_as :import
+
+    # rubocop:disable Rails/SkipsModelValidations
+    def perform(entry, importer_run)
+      obj = entry.factory.find
+      obj&.delete
+      ImporterRun.find(importer_run.id).increment!(:deleted_records)
+      ImporterRun.find(importer_run.id).decrement!(:enqueued_records)
+      entry.save!
+      entry.importer.current_run = ImporterRun.find(importer_run.id)
+      entry.importer.record_status
+      entry.status_info("Deleted", ImporterRun.find(importer_run.id))
+    end
+    # rubocop:enable Rails/SkipsModelValidations
+  end
+end

data/app/jobs/bulkrax/delete_work_job.rb

@@ -1,20 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class DeleteWorkJob < ApplicationJob
-    queue_as :import
-
-    # rubocop:disable Rails/SkipsModelValidations
-    def perform(entry, importer_run)
-      work = entry.factory.find
-      work&.delete
-      ImporterRun.find(importer_run.id).increment!(:deleted_records)
-      ImporterRun.find(importer_run.id).decrement!(:enqueued_records)
-      entry.save!
-      entry.importer.current_run = ImporterRun.find(importer_run.id)
-      entry.importer.record_status
-      entry.status_info("Deleted", ImporterRun.find(importer_run.id))
-    end
-    # rubocop:enable Rails/SkipsModelValidations
-  end
+  class DeleteWorkJob < DeleteJob; end
 end

data/app/models/bulkrax/csv_entry.rb

@@ -93,17 +93,32 @@ module Bulkrax
     end
 
     def build_export_metadata
-      # make_round_trippable
       self.parsed_metadata = {}
-      self.parsed_metadata['id'] = hyrax_record.id
-      self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
-      self.parsed_metadata['model'] = hyrax_record.has_model.first
+
+      build_system_metadata
+      build_files_metadata unless hyrax_record.is_a?(Collection)
       build_relationship_metadata
       build_mapping_metadata
-      build_files unless hyrax_record.is_a?(Collection)
+      self.save!
+
       self.parsed_metadata
     end
 
+    # Metadata required by Bulkrax for round-tripping
+    def build_system_metadata
+      self.parsed_metadata['id'] = hyrax_record.id
+      self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
+      self.parsed_metadata[key_for_export('model')] = hyrax_record.has_model.first
+    end
+
+    def build_files_metadata
+      file_mapping = key_for_export('file')
+      file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
+      filenames = file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+
+      handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
+    end
+
     def build_relationship_metadata
       # Includes all relationship methods for all exportable record types (works, Collections, FileSets)
       relationship_methods = {
@@ -126,13 +141,12 @@ module Bulkrax
     end
 
     def build_mapping_metadata
+      mapping = fetch_field_mapping
       mapping.each do |key, value|
-        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
-        next if key == "model"
-        # relationships handled by #build_relationship_metadata
-        next if [related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
-        next if key == 'file' # handled by #build_files
+        # these keys are handled by other methods
+        next if ['model', 'file', related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
         next if value['excluded']
+        next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
 
         object_key = key if value.key?('object')
         next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
@@ -209,8 +223,19 @@ module Bulkrax
       file_mapping = mapping['file']&.[]('from')&.first || 'file'
       file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
 
-      filenames = file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+      filenames = map_file_sets(file_sets)
       handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
+      build_thumbnail_files if hyrax_record.work?
+    end
+
+    def build_thumbnail_files
+      return unless importerexporter.include_thumbnails
+
+      thumbnail_mapping = 'thumbnail_file'
+      file_sets = Array.wrap(hyrax_record.thumbnail)
+
+      filenames = map_file_sets(file_sets)
+      handle_join_on_export(thumbnail_mapping, filenames, false)
     end
 
     def handle_join_on_export(key, values, join)
@@ -224,16 +249,6 @@ module Bulkrax
       end
     end
 
-    # In order for the existing exported hyrax_record, to be updated by a re-import
-    # we need a unique value in system_identifier
-    # add the existing hyrax_record id to system_identifier
-    def make_round_trippable
-      values = hyrax_record.send(work_identifier.to_s).to_a
-      values << hyrax_record.id
-      hyrax_record.send("#{work_identifier}=", values)
-      hyrax_record.save
-    end
-
     def record
       @record ||= raw_metadata
     end
@@ -258,12 +273,12 @@ module Bulkrax
         raise ::StandardError, 'Only expected to find one matching entry' if matching_collection_entries.count > 1
         identifiers << matching_collection_entries.first&.identifier
       end
-
       @collection_identifiers = identifiers.compact.presence || []
     end
 
     def collections_created?
-      collection_identifiers.length == self.collection_ids.length
+      # TODO: look into if this method is still needed after new relationships code
+      true
     end
 
     def find_collection_ids
@@ -288,5 +303,11 @@ module Bulkrax
       return f if File.exist?(f)
       raise "File #{f} does not exist"
     end
+
+    private
+
+    def map_file_sets(file_sets)
+      file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+    end
   end
 end

data/app/models/bulkrax/entry.rb

@@ -4,6 +4,8 @@ module Bulkrax
   # Custom error class for collections_created?
   class CollectionsCreatedError < RuntimeError; end
   class OAIError < RuntimeError; end
+  # TODO: remove when ApplicationParser#bagit_zip_file_size_check is removed
+  class BagitZipError < RuntimeError; end
   class Entry < ApplicationRecord
     include Bulkrax::HasMatchers
     include Bulkrax::ImportBehavior
@@ -34,6 +36,7 @@ module Bulkrax
     delegate :client,
       :collection_name,
       :user,
+      :generated_metadata_mapping,
       :related_parents_raw_mapping,
       :related_parents_parsed_mapping,
       :related_children_raw_mapping,
@@ -70,6 +73,15 @@ module Bulkrax
       parser&.work_identifier&.to_s || 'source'
     end
 
+    # Returns field_mapping hash based on whether or not generated metadata should be included
+    def fetch_field_mapping
+      return self.mapping if importerexporter.generated_metadata
+
+      self.mapping.each do |key, value|
+        self.mapping.delete(key) if value[generated_metadata_mapping]
+      end
+    end
+
     def self.parent_field(parser)
       parser.related_parents_parsed_mapping
     end

data/app/models/bulkrax/exporter.rb

@@ -51,6 +51,14 @@ module Bulkrax
       self.start_date.present? || self.finish_date.present?
     end
 
+    def include_thumbnails?
+      self.include_thumbnails
+    end
+
+    def generated_metadata?
+      self.generated_metadata
+    end
+
     def work_visibility_list
       [
         ['Any', ''],

data/app/models/bulkrax/importer_run.rb

@@ -4,9 +4,10 @@ module Bulkrax
   class ImporterRun < ApplicationRecord
     belongs_to :importer
     has_many :statuses, as: :runnable, dependent: :destroy
+    has_many :pending_relationships, dependent: :destroy
 
     def parents
-      PendingRelationship.where(bulkrax_importer_run_id: id).pluck(:parent_id).uniq
+      pending_relationships.pluck(:parent_id).uniq
     end
   end
 end

data/app/models/bulkrax/pending_relationship.rb

@@ -2,6 +2,6 @@
 
 module Bulkrax
   class PendingRelationship < ApplicationRecord
-    belongs_to :bulkrax_importer_run, class_name: "::Bulkrax::ImporterRun"
+    belongs_to :importer_run
   end
 end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -7,6 +7,8 @@ module Bulkrax
 
     def build_for_exporter
       build_export_metadata
+      # TODO(alishaevn): determine if the line below is still necessary
+      # the csv and bagit parsers also have write_files methods
       write_files if export_type == 'full' && !importerexporter.parser_klass.include?('Bagit')
     rescue RSolr::Error::Http, CollectionsCreatedError => e
       raise e
@@ -28,6 +30,7 @@ module Bulkrax
       return if hyrax_record.is_a?(Collection)
 
       file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
+      file_sets << hyrax_record.thumbnail if hyrax_record.thumbnail.present? && hyrax_record.work? && exporter.include_thumbnails
       file_sets.each do |fs|
         path = File.join(exporter_export_path, 'files')
         FileUtils.mkdir_p(path)
@@ -48,7 +51,7 @@ module Bulkrax
       fn = file_set.original_file.file_name.first
       mime = Mime::Type.lookup(file_set.original_file.mime_type)
       ext_mime = MIME::Types.of(file_set.original_file.file_name).first
-      if fn.include?(file_set.id) || importerexporter.metadata_only?
+      if fn.include?(file_set.id) || importerexporter.metadata_only? || importerexporter.parser_klass.include?('Bagit')
         filename = "#{fn}.#{mime.to_sym}"
         filename = fn if mime.to_s == ext_mime.to_s
       else
@@ -57,7 +60,7 @@ module Bulkrax
       end
       # Remove extention truncate and reattach
       ext = File.extname(filename)
-      "#{File.basename(filename, ext)[0...(255 - ext.length)]}#{ext}"
+      "#{File.basename(filename, ext)[0...(220 - ext.length)]}#{ext}"
     end
   end
 end

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -28,5 +28,13 @@ module Bulkrax
 
       raise StandardError, 'File set must be related to at least one work'
     end
+
+    def parent_jobs
+      false # FileSet relationships are handled in ObjectFactory#create_file_set
+    end
+
+    def child_jobs
+      raise ::StandardError, 'A FileSet cannot be a parent of a Collection, Work, or other FileSet'
+    end
   end
 end

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -50,7 +50,7 @@ module Bulkrax
       self.parsed_metadata[related_parents_parsed_mapping].each do |parent_identifier|
         next if parent_identifier.blank?
 
-        PendingRelationship.create!(child_id: self.identifier, parent_id: parent_identifier, bulkrax_importer_run_id: importerexporter.last_run.id, order: self.id)
+        PendingRelationship.create!(child_id: self.identifier, parent_id: parent_identifier, importer_run_id: importerexporter.last_run.id, order: self.id)
       end
     end
 
@@ -58,7 +58,7 @@ module Bulkrax
       self.parsed_metadata[related_children_parsed_mapping].each do |child_identifier|
         next if child_identifier.blank?
 
-        PendingRelationship.create!(parent_id: self.identifier, child_id: child_identifier, bulkrax_importer_run_id: importerexporter.last_run.id, order: self.id)
+        PendingRelationship.create!(parent_id: self.identifier, child_id: child_identifier, importer_run_id: importerexporter.last_run.id, order: self.id)
       end
     end
 

data/app/parsers/bulkrax/application_parser.rb

@@ -51,6 +51,10 @@ module Bulkrax
       @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
     end
 
+    def generated_metadata_mapping
+      @generated_metadata_mapping ||= 'generated'
+    end
+
     def related_parents_raw_mapping
       @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
     end
@@ -242,15 +246,49 @@ module Bulkrax
     def write
       write_files
       zip
+      # uncomment next line to debug for faulty zipping during bagit export
+      bagit_zip_file_size_check if importerexporter.parser_klass.include?('Bagit')
     end
 
     def unzip(file_to_unzip)
-      WillowSword::ZipPackage.new(file_to_unzip, importer_unzip_path).unzip_file
+      Zip::File.open(file_to_unzip) do |zip_file|
+        zip_file.each do |entry|
+          entry_path = File.join(importer_unzip_path, entry.name)
+          FileUtils.mkdir_p(File.dirname(entry_path))
+          zip_file.extract(entry, entry_path) unless File.exist?(entry_path)
+        end
+      end
     end
 
     def zip
+      require 'zip'
       FileUtils.rm_rf(exporter_export_zip_path)
-      WillowSword::ZipPackage.new(exporter_export_path, exporter_export_zip_path).create_zip
+      Zip::File.open(exporter_export_zip_path, create: true) do |zip_file|
+        Dir["#{exporter_export_path}/**/**"].each do |file|
+          zip_file.add(file.sub("#{exporter_export_path}/", ''), file)
+        end
+      end
+    end
+
+    # TODO: remove Entry::BagitZipError as well as this method when we're sure it's not needed
+    def bagit_zip_file_size_check
+      Zip::File.open(exporter_export_zip_path) do |zip_file|
+        zip_file.select { |entry| entry.name.include?('data/') && entry.file? }.each do |zipped_file|
+          Dir["#{exporter_export_path}/**/data/*"].select { |file| file.include?(zipped_file.name) }.each do |file|
+            begin
+              raise BagitZipError, "Invalid Bag, file size mismatch for #{file.sub("#{exporter_export_path}/", '')}" if File.size(file) != zipped_file.size
+            rescue BagitZipError => e
+              matched_entry_ids = importerexporter.entry_ids.select do |id|
+                Bulkrax::Entry.find(id).identifier.include?(zipped_file.name.split('/').first)
+              end
+              matched_entry_ids.each do |entry_id|
+                Bulkrax::Entry.find(entry_id).status_info(e)
+                status_info('Complete (with failures)')
+              end
+            end
+          end
+        end
+      end
     end
 
     # Is this a file?
@@ -272,7 +310,6 @@ module Bulkrax
 
     def real_import_file_path
       return importer_unzip_path if file? && zip?
-
       parser_fields['import_file_path']
     end
   end

data/app/parsers/bulkrax/bagit_parser.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class BagitParser < ApplicationParser
+  class BagitParser < ApplicationParser # rubocop:disable Metrics/ClassLength
+    include ExportBehavior
+
     def self.export_supported?
-      false # @todo will be supported
+      true
     end
 
     def valid_import?
@@ -14,19 +16,11 @@ module Bulkrax
     end
 
     def entry_class
-      parser_fields['metadata_format'].constantize
-    end
-
-    def collection_entry_class
-      parser_fields['metadata_format'].gsub('Entry', 'CollectionEntry').constantize
-    rescue
-      Entry
-    end
-
-    def file_set_entry_class
-      csv_format = Bulkrax::Importer.last.parser_fields['metadata_format'] == "Bulkrax::CsvEntry"
-      csv_format ? CsvFileSetEntry : RdfFileSetEntry
+      rdf_format = parser_fields&.[]('metadata_format') == "Bulkrax::RdfEntry"
+      rdf_format ? RdfEntry : CsvEntry
     end
+    alias collection_entry_class entry_class
+    alias file_set_entry_class entry_class
 
     # Take a random sample of 10 metadata_paths and work out the import fields from that
     def import_fields
@@ -101,7 +95,185 @@ module Bulkrax
     end
 
     def total
-      metadata_paths.count
+      importerexporter.entries.count
+    end
+
+    def extra_filters
+      output = ""
+      if importerexporter.start_date.present?
+        start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
+        finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
+        output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
+      end
+      output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
+      output += importerexporter.workflow_status.present? ? " AND workflow_state_name_ssim:#{importerexporter.workflow_status}" : ""
+      output
+    end
+
+    def current_record_ids
+      @work_ids = []
+      @collection_ids = []
+      @file_set_ids = []
+
+      case importerexporter.export_from
+      when 'all'
+        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+        @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+      when 'collection'
+        @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+      when 'worktype'
+        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+      when 'importer'
+        set_ids_for_exporting_from_importer
+      end
+
+      @work_ids + @collection_ids + @file_set_ids
+    end
+
+    # Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
+    # @see #current_record_ids
+    def set_ids_for_exporting_from_importer
+      entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
+      complete_statuses = Status.latest_by_statusable
+                                .includes(:statusable)
+                                .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
+
+      complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
+      extra_filters = extra_filters.presence || '*:*'
+
+      { :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
+        instance_variable_set(instance_var, ActiveFedora::SolrService.post(
+          extra_filters.to_s,
+          fq: [
+            %(#{::Solrizer.solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
+            "has_model_ssim:(#{models_to_search.join(' OR ')})"
+          ],
+          fl: 'id',
+          rows: 2_000_000_000
+        )['response']['docs'].map { |obj| obj['id'] })
+      end
+    end
+
+    def create_new_entries
+      current_record_ids.each_with_index do |id, index|
+        break if limit_reached?(limit, index)
+
+        this_entry_class = if @collection_ids.include?(id)
+                             collection_entry_class
+                           elsif @file_set_ids.include?(id)
+                             file_set_entry_class
+                           else
+                             entry_class
+                           end
+        new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
+
+        begin
+          entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
+        rescue => e
+          Rails.logger.info("#{e.message} was detected during export")
+        end
+
+        self.headers |= entry.parsed_metadata.keys if entry
+      end
+    end
+    alias create_from_collection create_new_entries
+    alias create_from_importer create_new_entries
+    alias create_from_worktype create_new_entries
+    alias create_from_all create_new_entries
+
+    # export methods
+
+    # rubocop:disable Metrics/AbcSize
+    def write_files
+      require 'open-uri'
+      require 'socket'
+      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |e|
+        bag = BagIt::Bag.new setup_bagit_folder(e.identifier)
+        w = ActiveFedora::Base.find(e.identifier)
+        next unless Hyrax.config.curation_concerns.include?(w.class)
+
+        w.file_sets.each do |fs|
+          file_name = filename(fs)
+          next if file_name.blank?
+          io = open(fs.original_file.uri)
+          file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
+          file.write(io.read)
+          file.close
+          bag.add_file(file_name, file.path)
+        end
+        CSV.open(setup_csv_metadata_export_file(e.identifier), "w", headers: export_headers, write_headers: true) do |csv|
+          csv << e.parsed_metadata
+        end
+        write_triples(e)
+        bag.manifest!(algo: 'sha256')
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def setup_csv_metadata_export_file(id)
+      File.join(importerexporter.exporter_export_path, id, 'metadata.csv')
+    end
+
+    def key_allowed(key)
+      !Bulkrax.reserved_properties.include?(key) &&
+        new_entry(entry_class, 'Bulkrax::Exporter').field_supported?(key) &&
+        key != source_identifier.to_s
+    end
+
+    # All possible column names
+    def export_headers
+      headers = sort_headers(self.headers)
+
+      # we don't want access_control_id exported and we want file at the end
+      headers.delete('access_control_id') if headers.include?('access_control_id')
+
+      # add the headers below at the beginning or end to maintain the preexisting export behavior
+      headers.prepend('model')
+      headers.prepend(source_identifier.to_s)
+      headers.prepend('id')
+
+      headers.uniq
+    end
+
+    def object_names
+      return @object_names if @object_names
+
+      @object_names = mapping.values.map { |value| value['object'] }
+      @object_names.uniq!.delete(nil)
+
+      @object_names
+    end
+
+    def sort_headers(headers)
+      # converting headers like creator_name_1 to creator_1_name so they get sorted by numerical order
+      # while keeping objects grouped together
+      headers.sort_by do |item|
+        number = item.match(/\d+/)&.[](0) || 0.to_s
+        sort_number = number.rjust(4, "0")
+        object_prefix = object_names.detect { |o| item.match(/^#{o}/) } || item
+        remainder = item.gsub(/^#{object_prefix}_/, '').gsub(/_#{number}/, '')
+        "#{object_prefix}_#{sort_number}_#{remainder}"
+      end
+    end
+
+    def setup_triple_metadata_export_file(id)
+      File.join(importerexporter.exporter_export_path, id, 'metadata.nt')
+    end
+
+    def setup_bagit_folder(id)
+      File.join(importerexporter.exporter_export_path, id)
+    end
+
+    def write_triples(e)
+      sd = SolrDocument.find(e.identifier)
+      return if sd.nil?
+
+      req = ActionDispatch::Request.new({ 'HTTP_HOST' => Socket.gethostname })
+      rdf = Hyrax::GraphExporter.new(sd, req).fetch.dump(:ntriples)
+      File.open(setup_triple_metadata_export_file(e.identifier), "w") do |triples|
+        triples.write(rdf)
+      end
     end
 
     def required_elements?(keys)
@@ -126,11 +298,7 @@ module Bulkrax
     def bags
       return @bags if @bags.present?
       new_bag = bag(import_file_path)
-      @bags = if new_bag
-                [new_bag]
-              else
-                Dir.glob("#{import_file_path}/**/*").map { |d| bag(d) }
-              end
+      @bags = new_bag ? [new_bag] : Dir.glob("#{import_file_path}/**/*").map { |d| bag(d) }
       @bags.delete(nil)
       raise StandardError, 'No valid bags found' if @bags.blank?
       return @bags

data/app/parsers/bulkrax/csv_parser.rb

@@ -11,12 +11,15 @@ module Bulkrax
     end
 
     def records(_opts = {})
+      return @records if @records.present?
+
       file_for_import = only_updates ? parser_fields['partial_import_file_path'] : import_file_path
       # data for entry does not need source_identifier for csv, because csvs are read sequentially and mapped after raw data is read.
       csv_data = entry_class.read_data(file_for_import)
       importer.parser_fields['total'] = csv_data.count
       importer.save
-      @records ||= csv_data.map { |record_data| entry_class.data_for_entry(record_data, nil, self) }
+
+      @records = csv_data.map { |record_data| entry_class.data_for_entry(record_data, nil, self) }
     end
 
     def build_records
@@ -145,7 +148,6 @@ module Bulkrax
                                        'Bulkrax::Importer',
                                        current_record.to_h)
       if current_record[:delete].present?
-        # TODO: create a "Delete" job for file_sets and collections
         "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
       else
         "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)

data/app/views/bulkrax/exporters/_form.html.erb

@@ -60,6 +60,16 @@
     hint: 'leave blank or 0 for all records',
     label: t('bulkrax.exporter.labels.limit') %>
 
+  <%= form.input :generated_metadata?,
+                 as: :boolean,
+                 label: t('bulkrax.exporter.labels.generated_metadata'),
+                 hint: t('bulkrax.exporter.hints.generated_metadata') %>
+
+  <%= form.input :include_thumbnails?,
+                 as: :boolean,
+                 label: t('bulkrax.exporter.labels.include_thumbnails'),
+                 hint: t('bulkrax.exporter.hints.include_thumbnails') %>
+
   <%= form.input :date_filter,
                  as: :boolean,
                  label: t('bulkrax.exporter.labels.filter_by_date') %>

data/app/views/bulkrax/exporters/show.html.erb

@@ -57,6 +57,18 @@
       <strong><%= t('bulkrax.exporter.labels.limit') %>:</strong>
       <%= @exporter.limit %>
     </p>
+
+    <p class='bulkrax-p-align'>
+      <strong><%= t('bulkrax.exporter.labels.generated_metadata') %>:</strong>
+      <%= @exporter.generated_metadata %>
+    </p>
+
+    <p class='bulkrax-p-align'>
+      <strong><%= t('bulkrax.exporter.labels.include_thumbnails') %>:</strong>
+      <%= @exporter.include_thumbnails %>
+    </p>
+
+
     <%= render partial: 'bulkrax/shared/bulkrax_errors', locals: {item: @exporter} %>
 
     <%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @exporter} %>

data/app/views/bulkrax/importers/show.html.erb

@@ -178,7 +178,7 @@
                   <% elsif e.status == "Pending" %>
                     <td><span class="glyphicon glyphicon-option-horizontal" style="color: blue;"></span> <%= e.status %></td>
                   <% else %>
-                    <td><span class="glyphicon glyphicon-remove" style="color: red;"></span> <%= e.status %></td>
+                    <td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
                   <% end %>
                   <% if e.last_error.present? %>
                     <td><%= link_to e.last_error.dig("error_class"), bulkrax.importer_entry_path(@importer.id, e.id) %></td>

data/config/locales/bulkrax.en.yml

@@ -16,6 +16,8 @@ en:
         filter_by_date: Filter By Date
         finish_date: End Date
         full: Metadata and Files
+        include_thumbnails: Include Thumbnails?
+        generated_metadata: Include Generated Metadata?
         importer: Importer
         limit: Limit
         metadata: Metadata Only
@@ -35,3 +37,6 @@ en:
           ingested: "Ingested"
           unapproved: "Unapproved"
           needs_repair: "Needs Repair"
+      hints:
+        include_thumbnails: "These exported fields currently cannot be imported."
+        generated_metadata: "These exported fields currently cannot be imported."

data/db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb

@@ -0,0 +1,5 @@
+class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_exporters, :include_thumbnails, :boolean, default: false unless column_exists?(:bulkrax_exporters, :include_thumbnails)
+  end
+end

data/db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb

@@ -0,0 +1,5 @@
+class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_exporters, :generated_metadata, :boolean, default: false unless column_exists?(:bulkrax_exporters, :generated_metadata)
+  end
+end

data/db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb

@@ -0,0 +1,7 @@
+class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.2]
+  def change
+    if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
+      rename_column :bulkrax_pending_relationships, :bulkrax_importer_run_id, :importer_run_id
+    end
+  end
+end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.1.2'
+  VERSION = '3.3.1'
 end

data/lib/bulkrax.rb

@@ -9,6 +9,7 @@ module Bulkrax
                    :default_work_type,
                    :default_field_mapping,
                    :fill_in_blank_source_identifiers,
+                   :generated_metadata_mapping,
                    :related_children_field_mapping,
                    :related_parents_field_mapping,
                    :reserved_properties,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.3.1
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-24 00:00:00.000000000 Z
+date: 2022-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -253,6 +253,7 @@ files:
 - app/controllers/bulkrax/exporters_controller.rb
 - app/controllers/bulkrax/importers_controller.rb
 - app/controllers/concerns/bulkrax/api.rb
+- app/controllers/concerns/bulkrax/download_behavior.rb
 - app/factories/bulkrax/object_factory.rb
 - app/helpers/bulkrax/application_helper.rb
 - app/helpers/bulkrax/exporters_helper.rb
@@ -260,6 +261,9 @@ files:
 - app/helpers/bulkrax/validation_helper.rb
 - app/jobs/bulkrax/application_job.rb
 - app/jobs/bulkrax/create_relationships_job.rb
+- app/jobs/bulkrax/delete_collection_job.rb
+- app/jobs/bulkrax/delete_file_set_job.rb
+- app/jobs/bulkrax/delete_job.rb
 - app/jobs/bulkrax/delete_work_job.rb
 - app/jobs/bulkrax/download_cloud_file_job.rb
 - app/jobs/bulkrax/export_work_job.rb
@@ -294,7 +298,6 @@ files:
 - app/models/bulkrax/rdf_file_set_entry.rb
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
-- app/models/concerns/bulkrax/download_behavior.rb
 - app/models/concerns/bulkrax/dynamic_record_lookup.rb
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
@@ -369,6 +372,9 @@ files:
 - db/migrate/20220119213325_add_work_counters_to_importer_runs.rb
 - db/migrate/20220301001839_create_bulkrax_pending_relationships.rb
 - db/migrate/20220303212810_add_order_to_bulkrax_pending_relationships.rb
+- db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb
+- db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb
+- db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/version.rb
@@ -384,7 +390,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -399,8 +405,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku
 test_files: []