bulkrax 3.2.0 → 3.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49b7cfb404f1878429bbac77aad6d1167e97377fe28b54e2042cee7f6ca04e67
-  data.tar.gz: c3789c193dc1610c99117e5e2c450ed477a7390995916c4b6b419900f5cf539b
+  metadata.gz: b136a8742a7f9b953b4c3ef86d700540b931b6fbf22798d719e9aa693ea61fa9
+  data.tar.gz: e1cc32eda55a606285cf6e080340db608a8867f0bd4de8ab1361ca4a3d21adf9
 SHA512:
-  metadata.gz: a669e9b566770ad21e3a6242d24779f5c73847b2b97d4f7334f06209ac7de3634e7667550ba35ce6930f9ceaffda8efccb29cf3277b40f29d808ddbd9623a1b8
-  data.tar.gz: 794e6de65d4ebb5665ab73fdac3eba2d4157ff1b07f3da6cc63a6323b2127dc40021d69ea193da2a356e276f08dde8ccc51dc1d8c563e5ba3e8ecb592e4b4c88
+  metadata.gz: 87e35f340faa583a9ae6ac156f95fa4958fcaf9d3ed09a9963a8dda39649a66307a8c494375a989e2103c2d1edef62343f5b69878eae2249d63cc4b8f65240a8
+  data.tar.gz: 21ddafc671eda822144b6b73abfb89892ffac954f0692952985299f40f7d0e1aba472077aac72a1f681da315ba35de795f43d95df323c2178576716d02eafc75

data/app/controllers/bulkrax/exporters_controller.rb

@@ -101,12 +101,12 @@ module Bulkrax
     def exporter_params
       params[:exporter][:export_source] = params[:exporter]["export_source_#{params[:exporter][:export_from]}".to_sym]
       if params[:exporter][:date_filter] == "1"
-        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
-                                       :parser_klass, :limit, :start_date, :finish_date, :work_visibility,
+        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type, :generated_metadata,
+                                       :include_thumbnails, :parser_klass, :limit, :start_date, :finish_date, :work_visibility,
                                        :workflow_status, field_mapping: {})
       else
-        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type,
-                                       :parser_klass, :limit, :work_visibility, :workflow_status,
+        params.fetch(:exporter).permit(:name, :user_id, :export_source, :export_from, :export_type, :generated_metadata,
+                                       :include_thumbnails, :parser_klass, :limit, :work_visibility, :workflow_status,
                                        field_mapping: {}).merge(start_date: nil, finish_date: nil)
       end
     end

data/app/{models → controllers}/concerns/bulkrax/download_behavior.rb

@@ -42,7 +42,7 @@ module Bulkrax
     def send_file_contents
       self.status = 200
       prepare_file_headers
-      stream_body file.read
+      send_file file
     end
 
     def prepare_file_headers
@@ -53,9 +53,5 @@ module Bulkrax
       response.headers['Last-Modified'] = File.mtime(file_path).utc.strftime("%a, %d %b %Y %T GMT")
       self.content_type = download_content_type
     end
-
-    def stream_body(iostream)
-      self.response_body = iostream
-    end
   end
 end

data/app/factories/bulkrax/object_factory.rb

@@ -61,6 +61,7 @@ module Bulkrax
           work_actor.update(environment(attrs))
         end
       end
+      object.apply_depositor_metadata(@user) && object.save! if object.depositor.nil?
       log_updated(object)
     end
 
@@ -107,6 +108,7 @@ module Bulkrax
           end
         end
       end
+      object.apply_depositor_metadata(@user) && object.save! if object.depositor.nil?
       log_created(object)
     end
 
@@ -141,7 +143,6 @@ module Bulkrax
       attrs = clean_attrs(attrs)
       attrs = collection_type(attrs)
       object.attributes = attrs
-      object.apply_depositor_metadata(@user)
       object.save!
     end
 

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -33,7 +33,7 @@ module Bulkrax
     # is the child in the relationship, and vice versa if a child_identifier is passed.
     def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
       pending_relationships = Bulkrax::PendingRelationship.find_each.select do |rel|
-        rel.bulkrax_importer_run_id == importer_run_id && rel.parent_id == parent_identifier
+        rel.importer_run_id == importer_run_id && rel.parent_id == parent_identifier
       end.sort_by(&:order)
 
       @importer_run_id = importer_run_id

data/app/jobs/bulkrax/importer_job.rb

@@ -12,6 +12,8 @@ module Bulkrax
       import(importer, only_updates_since_last_import)
       update_current_run_counters(importer)
       schedule(importer) if importer.schedulable?
+    rescue CSV::MalformedCSVError => e
+      importer.status_info(e)
     end
 
     def import(importer, only_updates_since_last_import)

data/app/models/bulkrax/csv_entry.rb

@@ -99,6 +99,7 @@ module Bulkrax
       build_files_metadata unless hyrax_record.is_a?(Collection)
       build_relationship_metadata
       build_mapping_metadata
+      self.save!
 
       self.parsed_metadata
     end
@@ -113,9 +114,10 @@ module Bulkrax
     def build_files_metadata
       file_mapping = key_for_export('file')
       file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
-      filenames = file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+      filenames = map_file_sets(file_sets)
 
       handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
+      build_thumbnail_files if hyrax_record.work?
     end
 
     def build_relationship_metadata
@@ -140,6 +142,7 @@ module Bulkrax
     end
 
     def build_mapping_metadata
+      mapping = fetch_field_mapping
       mapping.each do |key, value|
         # these keys are handled by other methods
         next if ['model', 'file', related_parents_parsed_mapping, related_children_parsed_mapping].include?(key)
@@ -217,6 +220,16 @@ module Bulkrax
       end
     end
 
+    def build_thumbnail_files
+      return unless importerexporter.include_thumbnails
+
+      thumbnail_mapping = 'thumbnail_file'
+      file_sets = Array.wrap(hyrax_record.thumbnail)
+
+      filenames = map_file_sets(file_sets)
+      handle_join_on_export(thumbnail_mapping, filenames, false)
+    end
+
     def handle_join_on_export(key, values, join)
       if join
         parsed_metadata[key] = values.join(' | ') # TODO: make split char dynamic
@@ -252,7 +265,6 @@ module Bulkrax
         raise ::StandardError, 'Only expected to find one matching entry' if matching_collection_entries.count > 1
         identifiers << matching_collection_entries.first&.identifier
       end
-
       @collection_identifiers = identifiers.compact.presence || []
     end
 
@@ -283,5 +295,11 @@ module Bulkrax
       return f if File.exist?(f)
       raise "File #{f} does not exist"
     end
+
+    private
+
+    def map_file_sets(file_sets)
+      file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+    end
   end
 end

data/app/models/bulkrax/entry.rb

@@ -4,6 +4,8 @@ module Bulkrax
   # Custom error class for collections_created?
   class CollectionsCreatedError < RuntimeError; end
   class OAIError < RuntimeError; end
+  # TODO: remove when ApplicationParser#bagit_zip_file_size_check is removed
+  class BagitZipError < RuntimeError; end
   class Entry < ApplicationRecord
     include Bulkrax::HasMatchers
     include Bulkrax::ImportBehavior
@@ -34,6 +36,7 @@ module Bulkrax
     delegate :client,
       :collection_name,
       :user,
+      :generated_metadata_mapping,
       :related_parents_raw_mapping,
       :related_parents_parsed_mapping,
       :related_children_raw_mapping,
@@ -70,6 +73,15 @@ module Bulkrax
       parser&.work_identifier&.to_s || 'source'
     end
 
+    # Returns field_mapping hash based on whether or not generated metadata should be included
+    def fetch_field_mapping
+      return self.mapping if importerexporter.generated_metadata
+
+      self.mapping.each do |key, value|
+        self.mapping.delete(key) if value[generated_metadata_mapping]
+      end
+    end
+
     def self.parent_field(parser)
       parser.related_parents_parsed_mapping
     end

data/app/models/bulkrax/exporter.rb

@@ -51,6 +51,14 @@ module Bulkrax
       self.start_date.present? || self.finish_date.present?
     end
 
+    def include_thumbnails?
+      self.include_thumbnails
+    end
+
+    def generated_metadata?
+      self.generated_metadata
+    end
+
     def work_visibility_list
       [
         ['Any', ''],

data/app/models/bulkrax/importer.rb

@@ -96,16 +96,19 @@ module Bulkrax
     end
 
     def current_run
-      @current_run ||= if file? && zip?
-                         self.importer_runs.create!
-                       else
-                         entry_counts = {
-                           total_work_entries: self.limit || parser.works_total,
-                           total_collection_entries: parser.collections_total,
-                           total_file_set_entries: parser.file_sets_total
-                         }
-                         self.importer_runs.create!(entry_counts)
-                       end
+      return @current_run if @current_run.present?
+
+      @current_run = self.importer_runs.create!
+      return @current_run if file? && zip?
+
+      entry_counts = {
+        total_work_entries: self.limit || parser.works_total,
+        total_collection_entries: parser.collections_total,
+        total_file_set_entries: parser.file_sets_total
+      }
+      @current_run.update!(entry_counts)
+
+      @current_run
     end
 
     def last_run

data/app/models/bulkrax/importer_run.rb

@@ -4,9 +4,10 @@ module Bulkrax
   class ImporterRun < ApplicationRecord
     belongs_to :importer
     has_many :statuses, as: :runnable, dependent: :destroy
+    has_many :pending_relationships, dependent: :destroy
 
     def parents
-      PendingRelationship.where(bulkrax_importer_run_id: id).pluck(:parent_id).uniq
+      pending_relationships.pluck(:parent_id).uniq
     end
   end
 end

data/app/models/bulkrax/pending_relationship.rb

@@ -2,6 +2,6 @@
 
 module Bulkrax
   class PendingRelationship < ApplicationRecord
-    belongs_to :bulkrax_importer_run, class_name: "::Bulkrax::ImporterRun"
+    belongs_to :importer_run
   end
 end

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -7,6 +7,8 @@ module Bulkrax
 
     def build_for_exporter
       build_export_metadata
+      # TODO(alishaevn): determine if the line below is still necessary
+      # the csv and bagit parsers also have write_files methods
       write_files if export_type == 'full' && !importerexporter.parser_klass.include?('Bagit')
     rescue RSolr::Error::Http, CollectionsCreatedError => e
       raise e
@@ -28,6 +30,7 @@ module Bulkrax
       return if hyrax_record.is_a?(Collection)
 
       file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
+      file_sets << hyrax_record.thumbnail if hyrax_record.thumbnail.present? && hyrax_record.work? && exporter.include_thumbnails
       file_sets.each do |fs|
         path = File.join(exporter_export_path, 'files')
         FileUtils.mkdir_p(path)
@@ -48,7 +51,7 @@ module Bulkrax
       fn = file_set.original_file.file_name.first
       mime = Mime::Type.lookup(file_set.original_file.mime_type)
       ext_mime = MIME::Types.of(file_set.original_file.file_name).first
-      if fn.include?(file_set.id) || importerexporter.metadata_only?
+      if fn.include?(file_set.id) || importerexporter.metadata_only? || importerexporter.parser_klass.include?('Bagit')
         filename = "#{fn}.#{mime.to_sym}"
         filename = fn if mime.to_s == ext_mime.to_s
       else

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -28,5 +28,13 @@ module Bulkrax
 
       raise StandardError, 'File set must be related to at least one work'
     end
+
+    def parent_jobs
+      false # FileSet relationships are handled in ObjectFactory#create_file_set
+    end
+
+    def child_jobs
+      raise ::StandardError, 'A FileSet cannot be a parent of a Collection, Work, or other FileSet'
+    end
   end
 end

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -50,7 +50,7 @@ module Bulkrax
       self.parsed_metadata[related_parents_parsed_mapping].each do |parent_identifier|
         next if parent_identifier.blank?
 
-        PendingRelationship.create!(child_id: self.identifier, parent_id: parent_identifier, bulkrax_importer_run_id: importerexporter.last_run.id, order: self.id)
+        PendingRelationship.create!(child_id: self.identifier, parent_id: parent_identifier, importer_run_id: importerexporter.last_run.id, order: self.id)
       end
     end
 
@@ -58,7 +58,7 @@ module Bulkrax
       self.parsed_metadata[related_children_parsed_mapping].each do |child_identifier|
         next if child_identifier.blank?
 
-        PendingRelationship.create!(parent_id: self.identifier, child_id: child_identifier, bulkrax_importer_run_id: importerexporter.last_run.id, order: self.id)
+        PendingRelationship.create!(parent_id: self.identifier, child_id: child_identifier, importer_run_id: importerexporter.last_run.id, order: self.id)
       end
     end
 

data/app/models/concerns/bulkrax/status_info.rb

@@ -18,7 +18,7 @@ module Bulkrax
     end
 
     def failed?
-      current_status&.status_message&.match(/fail/i)
+      current_status&.status_message&.eql?('Failed')
     end
 
     def succeeded?

data/app/parsers/bulkrax/application_parser.rb

@@ -51,6 +51,10 @@ module Bulkrax
       @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
     end
 
+    def generated_metadata_mapping
+      @generated_metadata_mapping ||= 'generated'
+    end
+
     def related_parents_raw_mapping
       @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
     end
@@ -242,15 +246,49 @@ module Bulkrax
     def write
       write_files
       zip
+      # uncomment next line to debug for faulty zipping during bagit export
+      bagit_zip_file_size_check if importerexporter.parser_klass.include?('Bagit')
     end
 
     def unzip(file_to_unzip)
-      WillowSword::ZipPackage.new(file_to_unzip, importer_unzip_path).unzip_file
+      Zip::File.open(file_to_unzip) do |zip_file|
+        zip_file.each do |entry|
+          entry_path = File.join(importer_unzip_path, entry.name)
+          FileUtils.mkdir_p(File.dirname(entry_path))
+          zip_file.extract(entry, entry_path) unless File.exist?(entry_path)
+        end
+      end
     end
 
     def zip
+      require 'zip'
       FileUtils.rm_rf(exporter_export_zip_path)
-      WillowSword::ZipPackage.new(exporter_export_path, exporter_export_zip_path).create_zip
+      Zip::File.open(exporter_export_zip_path, create: true) do |zip_file|
+        Dir["#{exporter_export_path}/**/**"].each do |file|
+          zip_file.add(file.sub("#{exporter_export_path}/", ''), file)
+        end
+      end
+    end
+
+    # TODO: remove Entry::BagitZipError as well as this method when we're sure it's not needed
+    def bagit_zip_file_size_check
+      Zip::File.open(exporter_export_zip_path) do |zip_file|
+        zip_file.select { |entry| entry.name.include?('data/') && entry.file? }.each do |zipped_file|
+          Dir["#{exporter_export_path}/**/data/*"].select { |file| file.include?(zipped_file.name) }.each do |file|
+            begin
+              raise BagitZipError, "Invalid Bag, file size mismatch for #{file.sub("#{exporter_export_path}/", '')}" if File.size(file) != zipped_file.size
+            rescue BagitZipError => e
+              matched_entry_ids = importerexporter.entry_ids.select do |id|
+                Bulkrax::Entry.find(id).identifier.include?(zipped_file.name.split('/').first)
+              end
+              matched_entry_ids.each do |entry_id|
+                Bulkrax::Entry.find(entry_id).status_info(e)
+                status_info('Complete (with failures)')
+              end
+            end
+          end
+        end
+      end
     end
 
     # Is this a file?
@@ -272,7 +310,6 @@ module Bulkrax
 
     def real_import_file_path
       return importer_unzip_path if file? && zip?
-
       parser_fields['import_file_path']
     end
   end

data/app/parsers/bulkrax/bagit_parser.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class BagitParser < ApplicationParser
+  class BagitParser < ApplicationParser # rubocop:disable Metrics/ClassLength
+    include ExportBehavior
+
     def self.export_supported?
-      false # @todo will be supported
+      true
     end
 
     def valid_import?
@@ -14,19 +16,11 @@ module Bulkrax
     end
 
     def entry_class
-      parser_fields['metadata_format'].constantize
-    end
-
-    def collection_entry_class
-      parser_fields['metadata_format'].gsub('Entry', 'CollectionEntry').constantize
-    rescue
-      Entry
-    end
-
-    def file_set_entry_class
-      csv_format = Bulkrax::Importer.last.parser_fields['metadata_format'] == "Bulkrax::CsvEntry"
-      csv_format ? CsvFileSetEntry : RdfFileSetEntry
+      rdf_format = parser_fields&.[]('metadata_format') == "Bulkrax::RdfEntry"
+      rdf_format ? RdfEntry : CsvEntry
     end
+    alias collection_entry_class entry_class
+    alias file_set_entry_class entry_class
 
     # Take a random sample of 10 metadata_paths and work out the import fields from that
     def import_fields
@@ -101,7 +95,185 @@ module Bulkrax
     end
 
     def total
-      metadata_paths.count
+      importerexporter.entries.count
+    end
+
+    def extra_filters
+      output = ""
+      if importerexporter.start_date.present?
+        start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
+        finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
+        output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
+      end
+      output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
+      output += importerexporter.workflow_status.present? ? " AND workflow_state_name_ssim:#{importerexporter.workflow_status}" : ""
+      output
+    end
+
+    def current_record_ids
+      @work_ids = []
+      @collection_ids = []
+      @file_set_ids = []
+
+      case importerexporter.export_from
+      when 'all'
+        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+        @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+      when 'collection'
+        @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+      when 'worktype'
+        @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+      when 'importer'
+        set_ids_for_exporting_from_importer
+      end
+
+      @work_ids + @collection_ids + @file_set_ids
+    end
+
+    # Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
+    # @see #current_record_ids
+    def set_ids_for_exporting_from_importer
+      entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
+      complete_statuses = Status.latest_by_statusable
+                                .includes(:statusable)
+                                .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
+
+      complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
+      extra_filters = extra_filters.presence || '*:*'
+
+      { :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
+        instance_variable_set(instance_var, ActiveFedora::SolrService.post(
+          extra_filters.to_s,
+          fq: [
+            %(#{::Solrizer.solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
+            "has_model_ssim:(#{models_to_search.join(' OR ')})"
+          ],
+          fl: 'id',
+          rows: 2_000_000_000
+        )['response']['docs'].map { |obj| obj['id'] })
+      end
+    end
+
+    def create_new_entries
+      current_record_ids.each_with_index do |id, index|
+        break if limit_reached?(limit, index)
+
+        this_entry_class = if @collection_ids.include?(id)
+                             collection_entry_class
+                           elsif @file_set_ids.include?(id)
+                             file_set_entry_class
+                           else
+                             entry_class
+                           end
+        new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
+
+        begin
+          entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
+        rescue => e
+          Rails.logger.info("#{e.message} was detected during export")
+        end
+
+        self.headers |= entry.parsed_metadata.keys if entry
+      end
+    end
+    alias create_from_collection create_new_entries
+    alias create_from_importer create_new_entries
+    alias create_from_worktype create_new_entries
+    alias create_from_all create_new_entries
+
+    # export methods
+
+    # rubocop:disable Metrics/AbcSize
+    def write_files
+      require 'open-uri'
+      require 'socket'
+      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |e|
+        bag = BagIt::Bag.new setup_bagit_folder(e.identifier)
+        w = ActiveFedora::Base.find(e.identifier)
+        next unless Hyrax.config.curation_concerns.include?(w.class)
+
+        w.file_sets.each do |fs|
+          file_name = filename(fs)
+          next if file_name.blank?
+          io = open(fs.original_file.uri)
+          file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
+          file.write(io.read)
+          file.close
+          bag.add_file(file_name, file.path)
+        end
+        CSV.open(setup_csv_metadata_export_file(e.identifier), "w", headers: export_headers, write_headers: true) do |csv|
+          csv << e.parsed_metadata
+        end
+        write_triples(e)
+        bag.manifest!(algo: 'sha256')
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def setup_csv_metadata_export_file(id)
+      File.join(importerexporter.exporter_export_path, id, 'metadata.csv')
+    end
+
+    def key_allowed(key)
+      !Bulkrax.reserved_properties.include?(key) &&
+        new_entry(entry_class, 'Bulkrax::Exporter').field_supported?(key) &&
+        key != source_identifier.to_s
+    end
+
+    # All possible column names
+    def export_headers
+      headers = sort_headers(self.headers)
+
+      # we don't want access_control_id exported and we want file at the end
+      headers.delete('access_control_id') if headers.include?('access_control_id')
+
+      # add the headers below at the beginning or end to maintain the preexisting export behavior
+      headers.prepend('model')
+      headers.prepend(source_identifier.to_s)
+      headers.prepend('id')
+
+      headers.uniq
+    end
+
+    def object_names
+      return @object_names if @object_names
+
+      @object_names = mapping.values.map { |value| value['object'] }
+      @object_names.uniq!.delete(nil)
+
+      @object_names
+    end
+
+    def sort_headers(headers)
+      # converting headers like creator_name_1 to creator_1_name so they get sorted by numerical order
+      # while keeping objects grouped together
+      headers.sort_by do |item|
+        number = item.match(/\d+/)&.[](0) || 0.to_s
+        sort_number = number.rjust(4, "0")
+        object_prefix = object_names.detect { |o| item.match(/^#{o}/) } || item
+        remainder = item.gsub(/^#{object_prefix}_/, '').gsub(/_#{number}/, '')
+        "#{object_prefix}_#{sort_number}_#{remainder}"
+      end
+    end
+
+    def setup_triple_metadata_export_file(id)
+      File.join(importerexporter.exporter_export_path, id, 'metadata.nt')
+    end
+
+    def setup_bagit_folder(id)
+      File.join(importerexporter.exporter_export_path, id)
+    end
+
+    def write_triples(e)
+      sd = SolrDocument.find(e.identifier)
+      return if sd.nil?
+
+      req = ActionDispatch::Request.new({ 'HTTP_HOST' => Socket.gethostname })
+      rdf = Hyrax::GraphExporter.new(sd, req).fetch.dump(:ntriples)
+      File.open(setup_triple_metadata_export_file(e.identifier), "w") do |triples|
+        triples.write(rdf)
+      end
     end
 
     def required_elements?(keys)
@@ -126,11 +298,7 @@ module Bulkrax
     def bags
       return @bags if @bags.present?
       new_bag = bag(import_file_path)
-      @bags = if new_bag
-                [new_bag]
-              else
-                Dir.glob("#{import_file_path}/**/*").map { |d| bag(d) }
-              end
+      @bags = new_bag ? [new_bag] : Dir.glob("#{import_file_path}/**/*").map { |d| bag(d) }
       @bags.delete(nil)
       raise StandardError, 'No valid bags found' if @bags.blank?
       return @bags

data/app/views/bulkrax/exporters/_form.html.erb

@@ -60,6 +60,16 @@
     hint: 'leave blank or 0 for all records',
     label: t('bulkrax.exporter.labels.limit') %>
 
+  <%= form.input :generated_metadata?,
+                 as: :boolean,
+                 label: t('bulkrax.exporter.labels.generated_metadata'),
+                 hint: t('bulkrax.exporter.hints.generated_metadata') %>
+
+  <%= form.input :include_thumbnails?,
+                 as: :boolean,
+                 label: t('bulkrax.exporter.labels.include_thumbnails'),
+                 hint: t('bulkrax.exporter.hints.include_thumbnails') %>
+
   <%= form.input :date_filter,
                  as: :boolean,
                  label: t('bulkrax.exporter.labels.filter_by_date') %>

data/app/views/bulkrax/exporters/show.html.erb

@@ -57,6 +57,18 @@
       <strong><%= t('bulkrax.exporter.labels.limit') %>:</strong>
       <%= @exporter.limit %>
     </p>
+
+    <p class='bulkrax-p-align'>
+      <strong><%= t('bulkrax.exporter.labels.generated_metadata') %>:</strong>
+      <%= @exporter.generated_metadata %>
+    </p>
+
+    <p class='bulkrax-p-align'>
+      <strong><%= t('bulkrax.exporter.labels.include_thumbnails') %>:</strong>
+      <%= @exporter.include_thumbnails %>
+    </p>
+
+
     <%= render partial: 'bulkrax/shared/bulkrax_errors', locals: {item: @exporter} %>
 
     <%= render partial: 'bulkrax/shared/bulkrax_field_mapping', locals: {item: @exporter} %>

data/config/locales/bulkrax.en.yml

@@ -16,6 +16,8 @@ en:
         filter_by_date: Filter By Date
         finish_date: End Date
         full: Metadata and Files
+        include_thumbnails: Include Thumbnails?
+        generated_metadata: Include Generated Metadata?
         importer: Importer
         limit: Limit
         metadata: Metadata Only
@@ -35,3 +37,6 @@ en:
           ingested: "Ingested"
           unapproved: "Unapproved"
           needs_repair: "Needs Repair"
+      hints:
+        include_thumbnails: "These exported fields currently cannot be imported."
+        generated_metadata: "These exported fields currently cannot be imported."

data/db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb

@@ -0,0 +1,5 @@
+class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_exporters, :include_thumbnails, :boolean, default: false unless column_exists?(:bulkrax_exporters, :include_thumbnails)
+  end
+end

data/db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb

@@ -0,0 +1,5 @@
+class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.2]
+  def change
+    add_column :bulkrax_exporters, :generated_metadata, :boolean, default: false unless column_exists?(:bulkrax_exporters, :generated_metadata)
+  end
+end

data/db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb

@@ -0,0 +1,7 @@
+class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.2]
+  def change
+    if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
+      rename_column :bulkrax_pending_relationships, :bulkrax_importer_run_id, :importer_run_id
+    end
+  end
+end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.2.0'
+  VERSION = '3.3.2'
 end

data/lib/bulkrax.rb

@@ -9,6 +9,7 @@ module Bulkrax
                    :default_work_type,
                    :default_field_mapping,
                    :fill_in_blank_source_identifiers,
+                   :generated_metadata_mapping,
                    :related_children_field_mapping,
                    :related_parents_field_mapping,
                    :reserved_properties,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.2
 platform: ruby
 authors:
 - Rob Kaufman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-27 00:00:00.000000000 Z
+date: 2022-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -253,6 +253,7 @@ files:
 - app/controllers/bulkrax/exporters_controller.rb
 - app/controllers/bulkrax/importers_controller.rb
 - app/controllers/concerns/bulkrax/api.rb
+- app/controllers/concerns/bulkrax/download_behavior.rb
 - app/factories/bulkrax/object_factory.rb
 - app/helpers/bulkrax/application_helper.rb
 - app/helpers/bulkrax/exporters_helper.rb
@@ -297,7 +298,6 @@ files:
 - app/models/bulkrax/rdf_file_set_entry.rb
 - app/models/bulkrax/status.rb
 - app/models/bulkrax/xml_entry.rb
-- app/models/concerns/bulkrax/download_behavior.rb
 - app/models/concerns/bulkrax/dynamic_record_lookup.rb
 - app/models/concerns/bulkrax/errored_entries.rb
 - app/models/concerns/bulkrax/export_behavior.rb
@@ -372,6 +372,9 @@ files:
 - db/migrate/20220119213325_add_work_counters_to_importer_runs.rb
 - db/migrate/20220301001839_create_bulkrax_pending_relationships.rb
 - db/migrate/20220303212810_add_order_to_bulkrax_pending_relationships.rb
+- db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb
+- db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb
+- db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb
 - lib/bulkrax.rb
 - lib/bulkrax/engine.rb
 - lib/bulkrax/version.rb