bulkrax 4.2.1 → 4.4.0

data/app/parsers/bulkrax/application_parser.rb

@@ -2,6 +2,9 @@
 require 'zip'
 
 module Bulkrax
+  # An abstract class that establishes the API for Bulkrax's import and export parsing.
+  #
+  # @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
   class ApplicationParser # rubocop:disable Metrics/ClassLength
     attr_accessor :importerexporter, :headers
     alias importer importerexporter
@@ -12,14 +15,21 @@ module Bulkrax
       :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
       to: :importerexporter
 
+    # @todo Convert to `class_attribute :parser_fiels, default: {}`
     def self.parser_fields
       {}
     end
 
+    # @return [TrueClass,FalseClass] this parser does or does not support exports.
+    #
+    # @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
     def self.export_supported?
       false
     end
 
+    # @return [TrueClass,FalseClass] this parser does or does not support imports.
+    #
+    # @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
     def self.import_supported?
       true
     end
@@ -29,49 +39,70 @@ module Bulkrax
       @headers = []
     end
 
-    # @api
+    # @api public
+    # @abstract Subclass and override {#entry_class} to implement behavior for the parser.
     def entry_class
-      raise StandardError, 'must be defined'
+      raise NotImplementedError, 'must be defined'
     end
 
-    # @api
+    # @api public
+    # @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
     def collection_entry_class
-      raise StandardError, 'must be defined'
+      raise NotImplementedError, 'must be defined'
     end
 
-    # @api
+    # @api public
+    # @abstract Subclass and override {#records} to implement behavior for the parser.
     def records(_opts = {})
-      raise StandardError, 'must be defined'
+      raise NotImplementedError, 'must be defined'
     end
 
+    # @return [Symbol] the name of the identifying property in the source system from which we're
+    # importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
+    #
+    # @see #work_identifier
+    # @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
     def source_identifier
       @source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
     end
 
+    # @return [Symbol] the name of the identifying property for the system which we're importing
+    #         into (e.g. the application that mounts *this* Bulkrax engine)
+    # @see #source_identifier
     def work_identifier
       @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
     end
 
+    # @return [String]
     def generated_metadata_mapping
       @generated_metadata_mapping ||= 'generated'
     end
 
+    # @return [String, NilClass]
+    # @see #related_parents_raw_mapping
     def related_parents_raw_mapping
       @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
     end
 
+    # @return [String]
+    # @see #related_parents_field_mapping
     def related_parents_parsed_mapping
       @related_parents_parsed_mapping ||= (get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents')
     end
 
+    # @return [String, NilClass]
+    # @see #related_children_parsed_mapping
     def related_children_raw_mapping
       @related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
     end
 
+    # @return [String]
+    # @see #related_children_raw_mapping
     def related_children_parsed_mapping
       @related_children_parsed_mapping ||= (get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children')
     end
 
+    # @api private
     def get_field_mapping_hash_for(key)
       return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
 
@@ -85,6 +116,7 @@ module Bulkrax
       instance_variable_get("@#{key}_hash")
     end
 
+    # @return [Array<String>]
     def model_field_mappings
       model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
       model_mappings |= ['model']
@@ -92,6 +124,7 @@ module Bulkrax
       model_mappings
     end
 
+    # @return [String]
     def perform_method
       if self.validate_only
         'perform_now'
@@ -100,29 +133,55 @@ module Bulkrax
       end
     end
 
+    # The visibility of the record.  Acceptable values are: "open", "embaro", "lease", "authenticated", "restricted".  The default is "open"
+    #
+    # @return [String]
+    # @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
+    # @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
     def visibility
       @visibility ||= self.parser_fields['visibility'] || 'open'
     end
 
+    # @api public
+    #
+    # @param types [Array<Symbol>] the types of objects that we'll create.
+    #
+    # @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
+    # @see #create_collections
+    # @see #create_works
+    # @see #create_file_sets
+    # @see #create_relationships
+    def create_objects(types = [])
+      types.each do |object_type|
+        parser.send("create_#{object_type.pluralize}")
+      end
+    end
+
+    # @abstract Subclass and override {#create_collections} to implement behavior for the parser.
     def create_collections
-      raise StandardError, 'must be defined' if importer?
+      raise NotImplementedError, 'must be defined' if importer?
     end
 
+    # @abstract Subclass and override {#create_works} to implement behavior for the parser.
     def create_works
-      raise StandardError, 'must be defined' if importer?
+      raise NotImplementedError, 'must be defined' if importer?
     end
 
+    # @abstract Subclass and override {#create_file_sets} to implement behavior for the parser.
     def create_file_sets
-      raise StandardError, 'must be defined' if importer?
+      raise NotImplementedError, 'must be defined' if importer?
     end
 
+    # @abstract Subclass and override {#create_relationships} to implement behavior for the parser.
     def create_relationships
-      raise StandardError, 'must be defined' if importer?
+      raise NotImplementedError, 'must be defined' if importer?
     end
 
     # Optional, define if using browse everything for file upload
     def retrieve_cloud_files(files); end
 
+    # @param file [#path, #original_filename] the file object that with the relevant data for the
+    #        import.
     def write_import_file(file)
       path = File.join(path_for_import, file.original_filename)
       FileUtils.mv(
@@ -133,47 +192,58 @@ module Bulkrax
     end
 
     # Base path for imported and exported files
+    # @param [String]
+    # @return [String] the base path for files that this parser will "parse"
     def base_path(type = 'import')
-      ENV['HYKU_MULTITENANT'] ? File.join(Bulkrax.send("#{type}_path"), Site.instance.account.name) : Bulkrax.send("#{type}_path")
+      # account for multiple versions of hyku
+      is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
+      is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
     end
 
     # Path where we'll store the import metadata and files
     #  this is used for uploaded and cloud files
+    # @return [String]
     def path_for_import
       @path_for_import = File.join(base_path, importerexporter.path_string)
       FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
       @path_for_import
     end
 
+    # @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
     def setup_export_file
-      raise StandardError, 'must be defined' if exporter?
+      raise NotImplementedError, 'must be defined' if exporter?
     end
 
+    # @abstract Subclass and override {#write_files} to implement behavior for the parser.
     def write_files
-      raise StandardError, 'must be defined' if exporter?
+      raise NotImplementedError, 'must be defined' if exporter?
     end
 
+    # @return [TrueClass,FalseClass]
     def importer?
       importerexporter.is_a?(Bulkrax::Importer)
     end
 
+    # @return [TrueClass,FalseClass]
     def exporter?
       importerexporter.is_a?(Bulkrax::Exporter)
     end
 
     # @param limit [Integer] limit set on the importerexporter
     # @param index [Integer] index of current iteration
-    # @return [boolean]
+    # @return [TrueClass,FalseClass]
     def limit_reached?(limit, index)
       return false if limit.nil? || limit.zero? # no limit
       index >= limit
     end
 
     # Override to add specific validations
+    # @return [TrueClass,FalseClass]
     def valid_import?
       true
     end
 
+    # @return [TrueClass,FalseClass]
     def record_has_source_identifier(record, index)
       if record[source_identifier].blank?
         if Bulkrax.fill_in_blank_source_identifiers.present?
@@ -197,6 +267,7 @@ module Bulkrax
     end
     # rubocop:enable Rails/SkipsModelValidations
 
+    # @return [Array<String>]
     def required_elements
       if Bulkrax.fill_in_blank_source_identifiers
         ['title']
@@ -285,12 +356,14 @@ module Bulkrax
     end
 
     # Path for the import
+    # @return [String]
     def import_file_path
       @import_file_path ||= real_import_file_path
     end
 
     private
 
+    # @return [String]
     def real_import_file_path
       return importer_unzip_path if file? && zip?
       parser_fields['import_file_path']

data/app/parsers/bulkrax/bagit_parser.rb

@@ -134,7 +134,8 @@ module Bulkrax
 
         record.file_sets.each do |fs|
           file_name = filename(fs)
-          next if file_name.blank?
+          next if file_name.blank? || fs.original_file.blank?
+
           io = open(fs.original_file.uri)
           file = Tempfile.new([file_name, File.extname(file_name)], binmode: true)
           file.write(io.read)

data/app/parsers/bulkrax/csv_parser.rb

@@ -180,7 +180,7 @@ module Bulkrax
     end
 
     def current_work_ids
-      ActiveSupport::Deprication.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
+      ActiveSupport::Deprecation.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
       current_record_ids
     end
 
@@ -200,14 +200,14 @@ module Bulkrax
         # get the parent collection and child collections
         @collection_ids = ActiveFedora::SolrService.query("id:#{importerexporter.export_source} #{extra_filters}", method: :post, rows: 2_147_483_647).map(&:id)
         @collection_ids += ActiveFedora::SolrService.query("has_model_ssim:Collection AND member_of_collection_ids_ssim:#{importerexporter.export_source}", method: :post, rows: 2_147_483_647).map(&:id)
+        find_child_file_sets(@work_ids)
       when 'worktype'
         @work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", method: :post, rows: 2_000_000_000).map(&:id)
+        find_child_file_sets(@work_ids)
       when 'importer'
         set_ids_for_exporting_from_importer
       end
 
-      find_child_file_sets(@work_ids) if importerexporter.export_from == 'collection'
-
       @work_ids + @collection_ids + @file_set_ids
     end
     # rubocop:enable Metrics/AbcSize
@@ -355,9 +355,9 @@ module Bulkrax
         path = File.join(exporter_export_path, folder_count, 'files')
         FileUtils.mkdir_p(path) unless File.exist? path
         file = filename(fs)
-        io = open(fs.original_file.uri)
-        next if file.blank?
+        next if file.blank? || fs.original_file.blank?
 
+        io = open(fs.original_file.uri)
         File.open(File.join(path, file), 'wb') do |f|
           f.write(io.read)
           f.close
@@ -440,7 +440,7 @@ module Bulkrax
         file_mapping = Bulkrax.field_mappings.dig(self.class.to_s, 'file', :from)&.first&.to_sym || :file
         next if r[file_mapping].blank?
 
-        r[file_mapping].split(/\s*[:;|]\s*/).map do |f|
+        r[file_mapping].split(Bulkrax.multi_value_element_split_on).map do |f|
           file = File.join(path_to_files, f.tr(' ', '_'))
           if File.exist?(file) # rubocop:disable Style/GuardClause
             file
@@ -455,9 +455,10 @@ module Bulkrax
     def path_to_files(**args)
       filename = args.fetch(:filename, '')
 
-      @path_to_files ||= File.join(
-        zip? ? importer_unzip_path : File.dirname(import_file_path), 'files', filename
-      )
+      return @path_to_files if @path_to_files.present? && filename.blank?
+      @path_to_files = File.join(
+          zip? ? importer_unzip_path : File.dirname(import_file_path), 'files', filename
+        )
     end
 
     private
@@ -467,7 +468,7 @@ module Bulkrax
       entry_uid ||= if Bulkrax.fill_in_blank_source_identifiers.present?
                       Bulkrax.fill_in_blank_source_identifiers.call(self, records.find_index(collection_hash))
                     else
-                      collection_hash[:title].split(/\s*[;|]\s*/).first
+                      collection_hash[:title].split(Bulkrax.multi_value_element_split_on).first
                     end
 
       entry_uid

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -13,8 +13,7 @@ module Bulkrax
     def client
       @client ||= OAI::Client.new(importerexporter.parser_fields['base_url'],
                                   headers: headers,
-                                  parser: 'libxml',
-                                  metadata_prefix: importerexporter.parser_fields['metadata_prefix'])
+                                  parser: 'libxml')
     rescue StandardError
       raise OAIError
     end
@@ -32,6 +31,7 @@ module Bulkrax
     end
 
     def records(opts = {})
+      opts[:metadata_prefix] ||= importerexporter.parser_fields['metadata_prefix']
       opts[:set] = collection_name unless collection_name == 'all'
 
       opts[:from] = importerexporter&.last_imported_at&.strftime("%Y-%m-%d") if importerexporter.last_imported_at && only_updates

data/app/services/bulkrax/remove_relationships_for_importer.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+module Bulkrax
+  # This module is rather destructive; it will break relationships between the works, file sets, and
+  # collections that were imported via an importer.  You probably don't want to run this on your
+  # data, except in the case where you have been testing a Bulkrax::Importer, the parsers and
+  # mappings.  Then, you might have relationships that you want to remove.
+  #
+  # tl;dr - Caution this will break things!
+  class RemoveRelationshipsForImporter
+    # @api public
+    #
+    # Remove the relationships of the works and collections for all of the Bulkrax::Entry records
+    # associated with the given Bulkrax::Importer.
+    #
+    # @param importer [Bulkrax::Importer]
+    # @param with_progress_bar [Boolean]
+    def self.break_relationships_for!(importer:, with_progress_bar: false)
+      entries = importer.entries.select(&:succeeded?)
+      progress_bar = build_progress_bar_for(with_progress_bar: with_progress_bar, entries: entries)
+      new(progress_bar: progress_bar, entries: entries).break_relationships!
+    end
+
+    # @api private
+    #
+    # A null object that conforms to this class's use of a progress bar.
+    module NullProgressBar
+      def self.increment; end
+    end
+
+    # @api private
+    #
+    # @return [#increment]
+    def self.build_progress_bar_for(with_progress_bar:, entries:)
+      return NullProgressBar unless with_progress_bar
+
+      begin
+        require 'ruby-progressbar'
+        ProgessBar.create(total: entries.count)
+      rescue LoadError
+        Rails.logger.info("Using NullProgressBar because ProgressBar is not available due to a LoadError.")
+      end
+    end
+
+    # @param entries [#each]
+    # @param progress_bar [#increment]
+    def initialize(entries:, progress_bar:)
+      @progress_bar = progress_bar
+      @entries = entries
+    end
+
+    attr_reader :entries, :progress_bar
+
+    def break_relationships!
+      entries.each do |entry|
+        progress_bar.increment
+
+        obj = entry.factory.find
+        next if obj.is_a?(FileSet) # FileSets must be attached to a Work
+
+        if obj.is_a?(Collection)
+          remove_relationships_from_collection(obj)
+        else
+          remove_relationships_from_work(obj)
+        end
+
+        obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
+        obj.save!
+      end
+    end
+
+    def remove_relationships_from_collection(collection)
+      # Remove child work relationships
+      collection.member_works.each do |work|
+        change = work.member_of_collections.delete(collection)
+        work.save! if change.present?
+      end
+
+      # Remove parent collection relationships
+      collection.member_of_collections.each do |parent_col|
+        Hyrax::Collections::NestedCollectionPersistenceService
+          .remove_nested_relationship_for(parent: parent_col, child: collection)
+      end
+
+      # Remove child collection relationships
+      collection.member_collections.each do |child_col|
+        Hyrax::Collections::NestedCollectionPersistenceService
+          .remove_nested_relationship_for(parent: collection, child: child_col)
+      end
+    end
+
+    def remove_relationships_from_work(work)
+      # Remove parent collection relationships
+      work.member_of_collections = []
+
+      # Remove parent work relationships
+      work.member_of_works.each do |parent_work|
+        parent_work.members.delete(work)
+        parent_work.save!
+      end
+
+      # Remove child work relationships
+      work.member_works.each do |child_work|
+        work.member_works.delete(child_work)
+      end
+    end
+  end
+end

data/app/views/bulkrax/exporters/_form.html.erb

@@ -29,7 +29,7 @@
 
   <%= form.input :export_source_importer,
     label: t('bulkrax.exporter.labels.importer'),
-    # required: true,
+    required: true,
     prompt: 'Select from the list',
     label_html: { class: 'importer export-source-option hidden' },
     input_html: { class: 'importer export-source-option hidden' },
@@ -38,7 +38,7 @@
   <%= form.input :export_source_collection,
     prompt: 'Start typing ...',
     label: t('bulkrax.exporter.labels.collection'),
-    # required: true,
+    required: true,
     placeholder: @collection&.title&.first,
     label_html: { class: 'collection export-source-option hidden' },
     input_html: {
@@ -52,7 +52,7 @@
 
   <%= form.input :export_source_worktype,
     label: t('bulkrax.exporter.labels.worktype'),
-    # required: true,
+    required: true,
     prompt: 'Select from the list',
     label_html: { class: 'worktype export-source-option hidden' },
     input_html: { class: 'worktype export-source-option hidden' },

data/app/views/bulkrax/exporters/show.html.erb

@@ -87,49 +87,25 @@
 
     <p class='bulkrax-p-align'><strong><%= t('bulkrax.exporter.labels.field_mapping') %>:</strong></p>
 
-    <p class='bulkrax-p-align'>
-      <strong><%= t('bulkrax.exporter.labels.total_work_entries') %>:</strong>
-      <%= @exporter.exporter_runs.last&.total_work_entries %>
+    <p class="bulkrax-p-align" title="<%= @exporter.last_run&.total_work_entries %> processed, <%= @exporter.last_run&.failed_records %> failed">
+      <strong>Total Entries:</strong>
+      <%= @exporter.last_run&.total_work_entries %>
     </p>
     <br>
-    <div class='bulkrax-nav-tab-table-left-align'>
-      <h2>Entries</h2>
-      <table class='table table-striped'>
-        <thead>
-          <tr>
-            <th>Identifier</th>
-            <th>Entry ID</th>
-            <th>Status</th>
-            <th>Errors</th>
-            <th>Status Set At</th>
-            <th>Actions</th>
-          </tr>
-        </thead>
-        <tbody>
-          <% @work_entries.each do |e| %>
-            <tr>
-              <td><%= link_to e.identifier, bulkrax.exporter_entry_path(@exporter.id, e.id) %></td>
-              <td><%= e.id %></td>
-              <% if e.status == 'Complete' %>
-                <td><span class='glyphicon glyphicon-ok' style='color: green;'></span> <%= e.status %></td>
-              <% elsif e.status == 'Pending' %>
-                <td><span class='glyphicon glyphicon-option-horizontal' style='color: blue;'></span> <%= e.status %></td>
-              <% else %>
-                <td><span class='glyphicon glyphicon-remove' style='color: red;'></span> <%= e.status %></td>
-              <% end %>
-              <% if e.last_error.present? %>
-                <td><%= link_to e.last_error.dig('error_class'), bulkrax.exporter_entry_path(@exporter.id, e.id) %></td>
-              <% else %>
-                <td></td>
-              <% end %>
-              <td><%= e.status_at %></td>
-              <td><%= link_to raw("<span class='glyphicon glyphicon-info-sign'></span>"), bulkrax.exporter_entry_path(@exporter.id, e.id) %></td>
-            </tr>
-          <% end %>
-        </tbody>
-      </table>
-      <%= page_entries_info(@work_entries) %><br>
-      <%= paginate(@work_entries, param_name: :work_entries_page) %>
+
+    <div class="bulkrax-nav-tab-bottom-margin">
+      <!-- Nav tabs -->
+      <ul class="bulkrax-nav-tab-top-margin tab-nav nav nav-tabs" role="tablist">
+        <li role="presentation" class='active'><a href="#work-entries" aria-controls="work-entries" role="tab" data-toggle="tab">Work Entries</a></li>
+        <li role="presentation"><a href="#collection-entries" aria-controls="collection-entries" role="tab" data-toggle="tab">Collection Entries</a></li>
+        <li role="presentation"><a href="#file-set-entries" aria-controls="file-set-entries" role="tab" data-toggle="tab">File Set Entries</a></li>
+      </ul>
+      <!-- Tab panes -->
+      <div class="tab-content outline">
+        <%= render partial: 'bulkrax/shared/work_entries_tab', locals: { item: @exporter, entries: @work_entries } %>
+        <%= render partial: 'bulkrax/shared/collection_entries_tab', locals: { item: @exporter, entries: @collection_entries } %>
+        <%= render partial: 'bulkrax/shared/file_set_entries_tab', locals: { item: @exporter, entries: @file_set_entries } %>
+      </div>
       <br>
       <%= link_to 'Edit', edit_exporter_path(@exporter) %>
       |

data/app/views/bulkrax/importers/edit.html.erb

@@ -4,7 +4,7 @@
 
 <div class="row">
   <div class="col-md-12">
-    <div class="panel panel-default tabs">
+    <div class="panel panel-default tabs importer-form">
 
       <%= simple_form_for @importer, html: { multipart: true } do |form| %>
         <%= render 'form', importer: @importer, form: form %>

data/app/views/bulkrax/importers/new.html.erb

@@ -4,7 +4,7 @@
 
 <div class="row">
   <div class="col-md-12">
-    <div class="panel panel-default tabs">
+    <div class="panel panel-default tabs importer-form">
       <%= simple_form_for @importer, html: { multipart: true } do |form| %>
         <%= render 'form', importer: @importer, form: form %>
         <div class="panel-footer">