bulkrax 3.3.1 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2525e4aed8b31a897668d556c311b29189ec1b7a951b4303604acd3ccb11fcc8
-  data.tar.gz: d642de2af38c8c82108b113641ab1e191ae05b96b2ce3528dde7d7c908639291
+  metadata.gz: 0a80674a9f582c3b8e83f442318908edb6ca9f0b615c970d09b17d941cc8027d
+  data.tar.gz: a2a53116ef49e03dde1aa1df14d8259a2b4abf06a82cff63a9d4ba622ba6600a
 SHA512:
-  metadata.gz: 6547c0283f75626cda95a8cc09cceff84ac576b7ca4034b72d729ab16d50c71657dac0e78739b5ef6f2d83aa85e1ddc8c45646f6c44408a0a989b4b33b93c487
-  data.tar.gz: 9ae11d35d0b3caf9d45237590e42541c0004268e6c86e8d44e6754900b253894813c7e817b471ed9379b7eefad1ec3a27d2cfebbe94865727ecb69b6b8652480
+  metadata.gz: af3d75fb03105e37f7374f3a7f863c545d9cc9c95ab2f18bbbf7b4692024e09811f103a372327b4724c836568bad176ed0ad0b7be929ed556259aa9b0793fce6
+  data.tar.gz: 1117a185fbab2bae0746187f464bebea855759a5ecccf0d34f098ac55ad7a2952e663268372262ba8f97820c8c1f02bd29c74a388cfd8ea9cfed84a46dad94cf

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -42,10 +42,12 @@ module Bulkrax
       pending_relationships.each do |rel|
         raise ::StandardError, %("#{rel}" needs either a child or a parent to create a relationship) if rel.child_id.nil? || rel.parent_id.nil?
         @child_entry, child_record = find_record(rel.child_id, importer_run_id)
-        child_record.is_a?(::Collection) ? @child_records[:collections] << child_record : @child_records[:works] << child_record
+        if child_record
+          child_record.is_a?(::Collection) ? @child_records[:collections] << child_record : @child_records[:works] << child_record
+        end
       end
 
-      if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.blank?
+      if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.nil?
         reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
       end

data/app/jobs/bulkrax/export_work_job.rb

@@ -6,26 +6,26 @@ module Bulkrax
 
     def perform(*args)
       entry = Entry.find(args[0])
+      exporter_run = ExporterRun.find(args[1])
       begin
         entry.build
         entry.save
       rescue StandardError
         # rubocop:disable Rails/SkipsModelValidations
-        ExporterRun.find(args[1]).increment!(:failed_records)
-        ExporterRun.find(args[1]).decrement!(:enqueued_records)
+        exporter_run.increment!(:failed_records)
+        exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0
         raise
       else
         if entry.failed?
-          ExporterRun.find(args[1]).increment!(:failed_records)
-          ExporterRun.find(args[1]).decrement!(:enqueued_records)
+          exporter_run.increment!(:failed_records)
+          exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0
           raise entry.reload.current_status.error_class.constantize
         else
-          ExporterRun.find(args[1]).increment!(:processed_records)
-          ExporterRun.find(args[1]).decrement!(:enqueued_records)
+          exporter_run.increment!(:processed_records)
+          exporter_run.decrement!(:enqueued_records) unless exporter_run.enqueued_records <= 0
         end
         # rubocop:enable Rails/SkipsModelValidations
       end
-      exporter_run = ExporterRun.find(args[1])
       return entry if exporter_run.enqueued_records.positive?
 
       if exporter_run.failed_records.positive?

data/app/jobs/bulkrax/importer_job.rb

@@ -12,6 +12,8 @@ module Bulkrax
       import(importer, only_updates_since_last_import)
       update_current_run_counters(importer)
       schedule(importer) if importer.schedulable?
+    rescue CSV::MalformedCSVError => e
+      importer.status_info(e)
     end
 
     def import(importer, only_updates_since_last_import)

data/app/models/bulkrax/csv_entry.rb

@@ -114,9 +114,10 @@ module Bulkrax
     def build_files_metadata
       file_mapping = key_for_export('file')
       file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
-      filenames = file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact
+      filenames = map_file_sets(file_sets)
 
       handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
+      build_thumbnail_files if hyrax_record.work?
     end
 
     def build_relationship_metadata
@@ -219,15 +220,6 @@ module Bulkrax
       end
     end
 
-    def build_files
-      file_mapping = mapping['file']&.[]('from')&.first || 'file'
-      file_sets = hyrax_record.file_set? ? Array.wrap(hyrax_record) : hyrax_record.file_sets
-
-      filenames = map_file_sets(file_sets)
-      handle_join_on_export(file_mapping, filenames, mapping['file']&.[]('join')&.present?)
-      build_thumbnail_files if hyrax_record.work?
-    end
-
     def build_thumbnail_files
       return unless importerexporter.include_thumbnails
 

data/app/models/bulkrax/exporter.rb

@@ -14,13 +14,15 @@ module Bulkrax
     validates :name, presence: true
     validates :parser_klass, presence: true
 
-    delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
+    delegate :write, :create_from_collection, :create_from_collections_metadata, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
 
     def export
       current_run && setup_export_path
       case self.export_from
       when 'collection'
         create_from_collection
+      when 'collections metadata'
+        create_from_collections_metadata
       when 'importer'
         create_from_importer
       when 'worktype'
@@ -87,6 +89,7 @@ module Bulkrax
       [
         [I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
         [I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
+        [I18n.t('bulkrax.exporter.labels.collections_metadata'), 'collections metadata'],
         [I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
         [I18n.t('bulkrax.exporter.labels.all'), 'all']
       ]

data/app/models/bulkrax/importer.rb

@@ -96,16 +96,19 @@ module Bulkrax
     end
 
     def current_run
-      @current_run ||= if file? && zip?
-                         self.importer_runs.create!
-                       else
-                         entry_counts = {
-                           total_work_entries: self.limit || parser.works_total,
-                           total_collection_entries: parser.collections_total,
-                           total_file_set_entries: parser.file_sets_total
-                         }
-                         self.importer_runs.create!(entry_counts)
-                       end
+      return @current_run if @current_run.present?
+
+      @current_run = self.importer_runs.create!
+      return @current_run if file? && zip?
+
+      entry_counts = {
+        total_work_entries: self.limit || parser.works_total,
+        total_collection_entries: parser.collections_total,
+        total_file_set_entries: parser.file_sets_total
+      }
+      @current_run.update!(entry_counts)
+
+      @current_run
     end
 
     def last_run

data/app/models/concerns/bulkrax/dynamic_record_lookup.rb

@@ -12,15 +12,14 @@ module Bulkrax
       # check for our entry in our current importer first
       importer_id = ImporterRun.find(importer_run_id).importer_id
       default_scope = { identifier: identifier, importerexporter_type: 'Bulkrax::Importer' }
-      record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
 
-      # TODO(alishaevn): discuss whether we are only looking for Collection models here
-      # use ActiveFedora::Base.find(identifier) instead?
-      record ||= ::Collection.where(id: identifier).first # rubocop:disable Rails/FindBy
-      if record.blank?
-        available_work_types.each do |work_type|
-          record ||= work_type.where(id: identifier).first # rubocop:disable Rails/FindBy
-        end
+      begin
+        # the identifier parameter can be a :source_identifier or the id of an object
+        record = Entry.find_by(default_scope.merge({ importerexporter_id: importer_id })) || Entry.find_by(default_scope)
+        record ||= ActiveFedora::Base.find(identifier)
+      # NameError for if ActiveFedora isn't installed
+      rescue NameError, ActiveFedora::ObjectNotFoundError
+        record = nil
       end
 
       # return the found entry here instead of searching for it again in the CreateRelationshipsJob

data/app/models/concerns/bulkrax/export_behavior.rb

@@ -51,7 +51,7 @@ module Bulkrax
       fn = file_set.original_file.file_name.first
       mime = Mime::Type.lookup(file_set.original_file.mime_type)
       ext_mime = MIME::Types.of(file_set.original_file.file_name).first
-      if fn.include?(file_set.id) || importerexporter.metadata_only? || importerexporter.parser_klass.include?('Bagit')
+      if fn.include?(file_set.id) || importerexporter.metadata_only?
         filename = "#{fn}.#{mime.to_sym}"
         filename = fn if mime.to_s == ext_mime.to_s
       else

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -8,10 +8,14 @@ module Bulkrax
 
     def add_path_to_file
       parsed_metadata['file'].each_with_index do |filename, i|
-        path_to_file = ::File.join(parser.path_to_files, filename)
+        next if filename.blank?
+
+        path_to_file = parser.path_to_files(filename: filename)
 
         parsed_metadata['file'][i] = path_to_file
       end
+      parsed_metadata['file'].delete('')
+
       raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
 
       parsed_metadata['file']

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -12,8 +12,8 @@ module Bulkrax
           raise CollectionsCreatedError unless collections_created?
           @item = factory.run!
           add_user_to_permission_templates! if self.class.to_s.include?("Collection")
-          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping].present?
-          child_jobs if self.parsed_metadata[related_children_parsed_mapping].present?
+          parent_jobs if self.parsed_metadata[related_parents_parsed_mapping]&.join.present?
+          child_jobs if self.parsed_metadata[related_children_parsed_mapping]&.join.present?
         end
       rescue RSolr::Error::Http, CollectionsCreatedError => e
         raise e

data/app/models/concerns/bulkrax/status_info.rb

@@ -18,7 +18,7 @@ module Bulkrax
     end
 
     def failed?
-      current_status&.status_message&.match(/fail/i)
+      current_status&.status_message&.eql?('Failed')
     end
 
     def succeeded?

data/app/parsers/bulkrax/application_parser.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'zip'
 
 module Bulkrax
   class ApplicationParser # rubocop:disable Metrics/ClassLength
@@ -261,7 +262,6 @@ module Bulkrax
     end
 
     def zip
-      require 'zip'
       FileUtils.rm_rf(exporter_export_zip_path)
       Zip::File.open(exporter_export_zip_path, create: true) do |zip_file|
         Dir["#{exporter_export_path}/**/**"].each do |file|

data/app/parsers/bulkrax/bagit_parser.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  class BagitParser < ApplicationParser # rubocop:disable Metrics/ClassLength
+  class BagitParser < CsvParser # rubocop:disable Metrics/ClassLength
     include ExportBehavior
 
     def self.export_supported?
@@ -19,8 +19,10 @@ module Bulkrax
       rdf_format = parser_fields&.[]('metadata_format') == "Bulkrax::RdfEntry"
       rdf_format ? RdfEntry : CsvEntry
     end
-    alias collection_entry_class entry_class
-    alias file_set_entry_class entry_class
+
+    def path_to_files(filename:)
+      @path_to_files ||= Dir.glob(File.join(import_file_path, '**/data', filename)).first
+    end
 
     # Take a random sample of 10 metadata_paths and work out the import fields from that
     def import_fields
@@ -30,39 +32,41 @@ module Bulkrax
       end.flatten.compact.uniq
     end
 
-    # Assume a single metadata record per path
-    # Create an Array of all metadata records, one per file
+    # Create an Array of all metadata records
     def records(_opts = {})
       raise StandardError, 'No BagIt records were found' if bags.blank?
       @records ||= bags.map do |bag|
         path = metadata_path(bag)
         raise StandardError, 'No metadata files were found' if path.blank?
         data = entry_class.read_data(path)
-        data = entry_class.data_for_entry(data, source_identifier, self)
-        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
-        data
+        get_data(bag, data)
       end
+
+      @records = @records.flatten
     end
 
-    # Find or create collections referenced by works
-    # If the import data also contains records for these works, they will be updated
-    # during create works
-    def create_collections
-      collections.each_with_index do |collection, index|
-        next if collection.blank?
-        metadata = {
-          title: [collection],
-          work_identifier => [collection],
-          visibility: 'open',
-          collection_type_gid: Hyrax::CollectionType.find_or_create_default_collection_type.gid
-        }
-        new_entry = find_or_create_entry(collection_entry_class, collection, 'Bulkrax::Importer', metadata)
-        ImportCollectionJob.perform_now(new_entry.id, current_run.id)
-        increment_counters(index, collection: true)
+    def get_data(bag, data)
+      if entry_class == CsvEntry
+        data = data.map do |data_row|
+          record_data = entry_class.data_for_entry(data_row, source_identifier, self)
+          next record_data if importerexporter.metadata_only?
+
+          record_data[:file] = bag.bag_files.join('|') if ::Hyrax.config.curation_concerns.include? record_data[:model]&.constantize
+          record_data
+        end
+      else
+        data = entry_class.data_for_entry(data, source_identifier, self)
+        data[:file] = bag.bag_files.join('|') unless importerexporter.metadata_only?
       end
+
+      data
     end
 
     def create_works
+      entry_class == CsvEntry ? super : create_rdf_works
+    end
+
+    def create_rdf_works
       records.each_with_index do |record, index|
         next unless record_has_source_identifier(record, index)
         break if limit_reached?(limit, index)
@@ -81,33 +85,16 @@ module Bulkrax
       status_info(e)
     end
 
-    def collections
-      records.map { |r| r[related_parents_parsed_mapping].split(/\s*[;|]\s*/) if r[related_parents_parsed_mapping].present? }.flatten.compact.uniq
-    end
-
-    def collections_total
-      collections.size
-    end
-
-    # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
-    def works_total
-      total
-    end
-
     def total
-      importerexporter.entries.count
-    end
+      @total = importer.parser_fields['total'] || 0 if importer?
 
-    def extra_filters
-      output = ""
-      if importerexporter.start_date.present?
-        start_dt = importerexporter.start_date.to_datetime.strftime('%FT%TZ')
-        finish_dt = importerexporter.finish_date.present? ? importerexporter.finish_date.to_datetime.end_of_day.strftime('%FT%TZ') : "NOW"
-        output += " AND system_modified_dtsi:[#{start_dt} TO #{finish_dt}]"
-      end
-      output += importerexporter.work_visibility.present? ? " AND visibility_ssi:#{importerexporter.work_visibility}" : ""
-      output += importerexporter.workflow_status.present? ? " AND workflow_state_name_ssim:#{importerexporter.workflow_status}" : ""
-      output
+      @total = if exporter?
+                 limit.nil? || limit.zero? ? current_record_ids.count : limit
+               end
+
+      return @total || 0
+    rescue StandardError
+      @total = 0
     end
 
     def current_record_ids
@@ -118,7 +105,6 @@ module Bulkrax
       case importerexporter.export_from
       when 'all'
         @work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
-        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
         @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'collection'
         @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
@@ -127,89 +113,49 @@ module Bulkrax
       when 'importer'
         set_ids_for_exporting_from_importer
       end
-
       @work_ids + @collection_ids + @file_set_ids
     end
 
-    # Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
-    # @see #current_record_ids
-    def set_ids_for_exporting_from_importer
-      entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
-      complete_statuses = Status.latest_by_statusable
-                                .includes(:statusable)
-                                .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
-
-      complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
-      extra_filters = extra_filters.presence || '*:*'
-
-      { :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
-        instance_variable_set(instance_var, ActiveFedora::SolrService.post(
-          extra_filters.to_s,
-          fq: [
-            %(#{::Solrizer.solr_name(work_identifier)}:("#{complete_entry_identifiers.join('" OR "')}")),
-            "has_model_ssim:(#{models_to_search.join(' OR ')})"
-          ],
-          fl: 'id',
-          rows: 2_000_000_000
-        )['response']['docs'].map { |obj| obj['id'] })
-      end
-    end
-
-    def create_new_entries
-      current_record_ids.each_with_index do |id, index|
-        break if limit_reached?(limit, index)
-
-        this_entry_class = if @collection_ids.include?(id)
-                             collection_entry_class
-                           elsif @file_set_ids.include?(id)
-                             file_set_entry_class
-                           else
-                             entry_class
-                           end
-        new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
-
-        begin
-          entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
-        rescue => e
-          Rails.logger.info("#{e.message} was detected during export")
-        end
-
-        self.headers |= entry.parsed_metadata.keys if entry
-      end
-    end
-    alias create_from_collection create_new_entries
-    alias create_from_importer create_new_entries
-    alias create_from_worktype create_new_entries
-    alias create_from_all create_new_entries
-
     # export methods
 
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
     def write_files
       require 'open-uri'
       require 'socket'
-      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |e|
-        bag = BagIt::Bag.new setup_bagit_folder(e.identifier)
-        w = ActiveFedora::Base.find(e.identifier)
-        next unless Hyrax.config.curation_concerns.include?(w.class)
+      importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |entry|
+        record = ActiveFedora::Base.find(entry.identifier)
+        next unless Hyrax.config.curation_concerns.include?(record.class)
+        bag = BagIt::Bag.new setup_bagit_folder(entry.identifier)
+        bag_entries = [entry]
+
+        record.file_sets.each do |fs|
+          if @file_set_ids.present?
+            file_set_entry = Bulkrax::CsvFileSetEntry.where("parsed_metadata LIKE '%#{fs.id}%'").first
+            bag_entries << file_set_entry unless file_set_entry.nil?
+          end
 
-        w.file_sets.each do |fs|
           file_name = filename(fs)
           next if file_name.blank?
           io = open(fs.original_file.uri)
           file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
           file.write(io.read)
           file.close
-          bag.add_file(file_name, file.path)
+          begin
+            bag.add_file(file_name, file.path)
+          rescue => e
+            entry.status_info(e)
+            status_info(e)
+          end
         end
-        CSV.open(setup_csv_metadata_export_file(e.identifier), "w", headers: export_headers, write_headers: true) do |csv|
-          csv << e.parsed_metadata
+
+        CSV.open(setup_csv_metadata_export_file(entry.identifier), "w", headers: export_headers, write_headers: true) do |csv|
+          bag_entries.each { |csv_entry| csv << csv_entry.parsed_metadata }
         end
-        write_triples(e)
+        write_triples(entry)
         bag.manifest!(algo: 'sha256')
       end
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
     def setup_csv_metadata_export_file(id)
       File.join(importerexporter.exporter_export_path, id, 'metadata.csv')
@@ -221,42 +167,6 @@ module Bulkrax
         key != source_identifier.to_s
     end
 
-    # All possible column names
-    def export_headers
-      headers = sort_headers(self.headers)
-
-      # we don't want access_control_id exported and we want file at the end
-      headers.delete('access_control_id') if headers.include?('access_control_id')
-
-      # add the headers below at the beginning or end to maintain the preexisting export behavior
-      headers.prepend('model')
-      headers.prepend(source_identifier.to_s)
-      headers.prepend('id')
-
-      headers.uniq
-    end
-
-    def object_names
-      return @object_names if @object_names
-
-      @object_names = mapping.values.map { |value| value['object'] }
-      @object_names.uniq!.delete(nil)
-
-      @object_names
-    end
-
-    def sort_headers(headers)
-      # converting headers like creator_name_1 to creator_1_name so they get sorted by numerical order
-      # while keeping objects grouped together
-      headers.sort_by do |item|
-        number = item.match(/\d+/)&.[](0) || 0.to_s
-        sort_number = number.rjust(4, "0")
-        object_prefix = object_names.detect { |o| item.match(/^#{o}/) } || item
-        remainder = item.gsub(/^#{object_prefix}_/, '').gsub(/_#{number}/, '')
-        "#{object_prefix}_#{sort_number}_#{remainder}"
-      end
-    end
-
     def setup_triple_metadata_export_file(id)
       File.join(importerexporter.exporter_export_path, id, 'metadata.nt')
     end
@@ -276,11 +186,6 @@ module Bulkrax
       end
     end
 
-    def required_elements?(keys)
-      return if keys.blank?
-      !required_elements.map { |el| keys.map(&:to_s).include?(el) }.include?(false)
-    end
-
     # @todo - investigate getting directory structure
     # @todo - investigate using perform_later, and having the importer check for
     #   DownloadCloudFileJob before it starts
@@ -331,5 +236,11 @@ module Bulkrax
       return nil unless bag.valid?
       bag
     end
+
+    # use the version of this method from the application parser instead
+    def real_import_file_path
+      return importer_unzip_path if file? && zip?
+      parser_fields['import_file_path']
+    end
   end
 end

data/app/parsers/bulkrax/csv_parser.rb

@@ -195,6 +195,9 @@ module Bulkrax
         @file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'collection'
         @work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+        @collection_ids = ActiveFedora::SolrService.query("id:#{importerexporter.export_source} #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
+      when 'collections metadata'
+        @collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
       when 'worktype'
         @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
       when 'importer'
@@ -251,6 +254,7 @@ module Bulkrax
       end
     end
     alias create_from_collection create_new_entries
+    alias create_from_collections_metadata create_new_entries
     alias create_from_importer create_new_entries
     alias create_from_worktype create_new_entries
     alias create_from_all create_new_entries
@@ -268,8 +272,8 @@ module Bulkrax
       CsvFileSetEntry
     end
 
-    # See https://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
-    #   Changed to grep as wc -l counts blank lines, and ignores the final unescaped line (which may or may not contain data)
+    # TODO: figure out why using the version of this method that's in the bagit parser
+    # breaks specs for the "if importer?" line
     def total
       @total = importer.parser_fields['total'] || 0 if importer?
       @total = limit || current_record_ids.count if exporter?
@@ -378,10 +382,11 @@ module Bulkrax
     end
 
     # Retrieve the path where we expect to find the files
-    def path_to_files
+    def path_to_files(**args)
+      filename = args.fetch(:filename, '')
+
       @path_to_files ||= File.join(
-        zip? ? importer_unzip_path : File.dirname(import_file_path),
-        'files'
+        zip? ? importer_unzip_path : File.dirname(import_file_path), 'files', filename
       )
     end
 

data/app/views/bulkrax/exporters/_form.html.erb

@@ -15,20 +15,21 @@
 
   <%= form.hidden_field :user_id, value: current_user.id %>
 
-  <%= form.input :export_type, 
-    collection:  form.object.export_type_list, 
-    label: t('bulkrax.exporter.labels.export_type'), 
+  <%= form.input :export_type,
+    collection:  form.object.export_type_list,
+    label: t('bulkrax.exporter.labels.export_type'),
     required: true,
     prompt: 'Please select an export type' %>
 
-  <%= form.input :export_from, 
-    collection: form.object.export_from_list, 
-    label: t('bulkrax.exporter.labels.export_from'), 
+  <%= form.input :export_from,
+    collection: form.object.export_from_list,
+    label: t('bulkrax.exporter.labels.export_from'),
     required: true,
     prompt: 'Please select an export source' %>
 
   <%= form.input :export_source_importer,
     label: t('bulkrax.exporter.labels.importer'),
+    required: true,
     prompt: 'Select from the list',
     label_html: { class: 'importer export-source-option hidden' },
     input_html: { class: 'importer export-source-option hidden' },
@@ -37,6 +38,7 @@
   <%= form.input :export_source_collection,
     prompt: 'Start typing ...',
     label: t('bulkrax.exporter.labels.collection'),
+    required: true,
     placeholder: @collection&.title&.first,
     label_html: { class: 'collection export-source-option hidden' },
     input_html: {
@@ -50,13 +52,14 @@
 
   <%= form.input :export_source_worktype,
     label: t('bulkrax.exporter.labels.worktype'),
+    required: true,
     prompt: 'Select from the list',
     label_html: { class: 'worktype export-source-option hidden' },
     input_html: { class: 'worktype export-source-option hidden' },
     collection: Hyrax.config.curation_concerns.map {|cc| [cc.to_s, cc.to_s] } %>
 
-  <%= form.input :limit, 
-    as: :integer, 
+  <%= form.input :limit,
+    as: :integer,
     hint: 'leave blank or 0 for all records',
     label: t('bulkrax.exporter.labels.limit') %>
 
@@ -90,8 +93,8 @@
                  collection: form.object.workflow_status_list,
                  label: t('bulkrax.exporter.labels.status') %>
 
-  <%= form.input :parser_klass, 
-    collection: Bulkrax.parsers.map {|p| [p[:name], p[:class_name], {'data-partial' => p[:partial]}] if p[:class_name].constantize.export_supported? }.compact, 
+  <%= form.input :parser_klass,
+    collection: Bulkrax.parsers.map {|p| [p[:name], p[:class_name], {'data-partial' => p[:partial]}] if p[:class_name].constantize.export_supported? }.compact,
     label: t('bulkrax.exporter.labels.export_format') %>
 </div>
 

data/app/views/bulkrax/exporters/show.html.erb

@@ -40,6 +40,11 @@
       <% when 'collection' %>
         <% collection = Collection.find(@exporter.export_source) %>
         <%= link_to collection&.title&.first, hyrax.dashboard_collection_path(collection.id) %>
+      <% when 'collections metadata' %>
+        <% collections = Collection.all %>
+        <% collections.each_with_index do |c, i| %>
+          <%= link_to c&.title&.first, hyrax.dashboard_collection_path(c.id) %><%= ',' if i != collections.count - 1 %>
+        <% end %>
       <% when 'importer' %>
         <% importer = Bulkrax::Importer.find(@exporter.export_source) %>
         <%= link_to importer.name, bulkrax.importer_path(importer.id) %>

data/config/locales/bulkrax.en.yml

@@ -8,6 +8,7 @@ en:
       labels:
         all: All
         collection: Collection
+        collections_metadata: All Collections' Metadata (only)
         export_format: Export Format
         export_from: Export From
         export_source: Export Source

data/db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb

@@ -1,7 +1,17 @@
 class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.2]
-  def change
+  def up
     if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
+      remove_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs
+      remove_index :bulkrax_pending_relationships, column: :bulkrax_importer_run_id
+
       rename_column :bulkrax_pending_relationships, :bulkrax_importer_run_id, :importer_run_id
+
+      add_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs, column: :importer_run_id
+      add_index :bulkrax_pending_relationships, :importer_run_id, name: 'index_bulkrax_pending_relationships_on_importer_run_id'
     end
   end
+
+  def down
+    rename_column :bulkrax_pending_relationships, :importer_run_id, :bulkrax_importer_run_id
+  end
 end

data/lib/bulkrax/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bulkrax
-  VERSION = '3.3.1'
+  VERSION = '3.5.0'
 end

data/lib/generators/bulkrax/install_generator.rb

@@ -10,7 +10,7 @@ class Bulkrax::InstallGenerator < Rails::Generators::Base
   end
 
   def add_to_gemfile
-    gem 'willow_sword', github: 'notch8/willow_sword'
+    gem 'bulkrax'
 
     Bundler.with_clean_env do
       run "bundle install"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bulkrax
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.5.0
 platform: ruby
 authors:
 - Rob Kaufman
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-09 00:00:00.000000000 Z
+date: 2022-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -25,81 +25,81 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.1.6
 - !ruby/object:Gem::Dependency
-  name: loofah
+  name: bagit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: '0.4'
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: coderay
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.6
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: simple_form
+  name: iso8601
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
-  name: iso8601
+  name: kaminari
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: oai
+  name: language_list
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
-    - - "<"
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.x
+        version: 1.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
-    - - "<"
+        version: '1.2'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.x
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: libxml-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -115,61 +115,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
-  name: language_list
+  name: loofah
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 2.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 2.2.3
 - !ruby/object:Gem::Dependency
-  name: rdf
+  name: oai
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '0.4'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: 2.x
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.2
+        version: '0.4'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: 2.x
 - !ruby/object:Gem::Dependency
-  name: bagit
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 2.0.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 2.0.6
 - !ruby/object:Gem::Dependency
-  name: coderay
+  name: rdf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -183,7 +197,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: kaminari
+  name: simple_form
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -390,7 +404,7 @@ homepage: https://github.com/samvera-labs/bulkrax
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -405,8 +419,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Import and export tool for Hyrax and Hyku
 test_files: []