bulkrax 6.0.0 → 7.0.0

data/app/parsers/bulkrax/application_parser.rb

@@ -14,7 +14,7 @@ module Bulkrax
              :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
              :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
              :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
-             :zip?, :file?,
+             :zip?, :file?, :remove_and_rerun,
              to: :importerexporter
 
     # @todo Convert to `class_attribute :parser_fiels, default: {}`
@@ -47,6 +47,10 @@ module Bulkrax
       raise NotImplementedError, 'must be defined'
     end
 
+    def work_entry_class
+      entry_class
+    end
+
     # @api public
     # @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
     def collection_entry_class
@@ -85,7 +89,7 @@ module Bulkrax
     #         defaults to work_identifier value + "_sim"
     # @see #work_identifier
     def work_identifier_search_field
-      @work_identifier_search_field ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field')&.first&.to_s || "#{work_identifier}_sim"
+      @work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim"
     end
 
     # @return [String]
@@ -102,7 +106,7 @@ module Bulkrax
     # @return [String]
     # @see #related_parents_field_mapping
     def related_parents_parsed_mapping
-      @related_parents_parsed_mapping ||= (get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents')
+      @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents'
     end
 
     # @return [String, NilClass]
@@ -114,7 +118,7 @@ module Bulkrax
     # @return [String]
     # @see #related_children_raw_mapping
     def related_children_parsed_mapping
-      @related_children_parsed_mapping ||= (get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children')
+      @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children'
     end
 
     # @api private
@@ -157,6 +161,22 @@ module Bulkrax
       @visibility ||= self.parser_fields['visibility'] || 'open'
     end
 
+    def create_collections
+      create_objects(['collection'])
+    end
+
+    def create_works
+      create_objects(['work'])
+    end
+
+    def create_file_sets
+      create_objects(['file_set'])
+    end
+
+    def create_relationships
+      create_objects(['relationship'])
+    end
+
     # @api public
     #
     # @param types [Array<Symbol>] the types of objects that we'll create.
@@ -166,30 +186,77 @@ module Bulkrax
     # @see #create_works
     # @see #create_file_sets
     # @see #create_relationships
-    def create_objects(types = [])
-      types.each do |object_type|
-        send("create_#{object_type.pluralize}")
+    def create_objects(types_array = nil)
+      index = 0
+      (types_array || %w[collection work file_set relationship]).each do |type|
+        if type.eql?('relationship')
+          ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
+          next
+        end
+        send(type.pluralize).each do |current_record|
+          next unless record_has_source_identifier(current_record, index)
+          break if limit_reached?(limit, index)
+          seen[current_record[source_identifier]] = true
+          create_entry_and_job(current_record, type)
+          increment_counters(index, "#{type}": true)
+          index += 1
+        end
+        importer.record_status
+      end
+      true
+    rescue StandardError => e
+      set_status_info(e)
+    end
+
+    def rebuild_entries(types_array = nil)
+      index = 0
+      (types_array || %w[collection work file_set relationship]).each do |type|
+        # works are not gurneteed to have Work in the type
+
+        importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
+          seen[e.identifier] = true
+          e.status_info('Pending', importer.current_run)
+          if remove_and_rerun
+            delay = calculate_type_delay(type)
+            "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
+          else
+            "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
+          end
+          increment_counters(index)
+          index += 1
+        end
       end
     end
 
-    # @abstract Subclass and override {#create_collections} to implement behavior for the parser.
-    def create_collections
-      raise NotImplementedError, 'must be defined' if importer?
-    end
+    def rebuild_entry_query(type, statuses)
+      type_col = Bulkrax::Entry.arel_table['type']
+      status_col = Bulkrax::Entry.arel_table['status_message']
 
-    # @abstract Subclass and override {#create_works} to implement behavior for the parser.
-    def create_works
-      raise NotImplementedError, 'must be defined' if importer?
+      query = (type == 'work' ? type_col.not.matches(%w[collection file_set]) : type_col.matches(type.camelize))
+      query.and(status_col.in(statuses))
     end
 
-    # @abstract Subclass and override {#create_file_sets} to implement behavior for the parser.
-    def create_file_sets
-      raise NotImplementedError, 'must be defined' if importer?
+    def calculate_type_delay(type)
+      return 2.minutes if type == 'file_set'
+      return 1.minute if type == 'work'
+      return 0
     end
 
-    # @abstract Subclass and override {#create_relationships} to implement behavior for the parser.
-    def create_relationships
-      raise NotImplementedError, 'must be defined' if importer?
+    def create_entry_and_job(current_record, type, identifier = nil)
+      identifier ||= current_record[source_identifier]
+      new_entry = find_or_create_entry(send("#{type}_entry_class"),
+                                       identifier,
+                                       'Bulkrax::Importer',
+                                       current_record.to_h)
+      new_entry.status_info('Pending', importer.current_run)
+      if current_record[:delete].present?
+        "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
+      elsif current_record[:remove_and_rerun].present? || remove_and_rerun
+        delay = calculate_type_delay(type)
+        "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
+      else
+        "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
+      end
     end
 
     # Optional, define if using browse everything for file upload
@@ -305,11 +372,15 @@ module Bulkrax
     end
 
     def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
-      entry = entryclass.where(
+      # limit entry search to just this importer or exporter. Don't go moving them
+      entry = importerexporter.entries.where(
+        identifier: identifier
+      ).first
+      entry ||= entryclass.new(
         importerexporter_id: importerexporter.id,
         importerexporter_type: type,
         identifier: identifier
-      ).first_or_create!
+      )
       entry.raw_metadata = raw_metadata
       entry.save!
       entry

data/app/parsers/bulkrax/bagit_parser.rb

@@ -63,29 +63,6 @@ module Bulkrax
       data
     end
 
-    def create_works
-      entry_class == CsvEntry ? super : create_rdf_works
-    end
-
-    def create_rdf_works
-      records.each_with_index do |record, index|
-        next unless record_has_source_identifier(record, index)
-        break if limit_reached?(limit, index)
-
-        seen[record[source_identifier]] = true
-        new_entry = find_or_create_entry(entry_class, record[source_identifier], 'Bulkrax::Importer', record)
-        if record[:delete].present?
-          DeleteWorkJob.send(perform_method, new_entry, current_run)
-        else
-          ImportWorkJob.send(perform_method, new_entry.id, current_run.id)
-        end
-        increment_counters(index, work: true)
-      end
-      importer.record_status
-    rescue StandardError => e
-      set_status_info(e)
-    end
-
     # export methods
 
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize

data/app/parsers/bulkrax/csv_parser.rb

@@ -113,57 +113,6 @@ module Bulkrax
       false
     end
 
-    def create_collections
-      create_objects(['collection'])
-    end
-
-    def create_works
-      create_objects(['work'])
-    end
-
-    def create_file_sets
-      create_objects(['file_set'])
-    end
-
-    def create_relationships
-      create_objects(['relationship'])
-    end
-
-    def create_objects(types_array = nil)
-      index = 0
-      (types_array || %w[collection work file_set relationship]).each do |type|
-        if type.eql?('relationship')
-          ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
-          next
-        end
-        send(type.pluralize).each do |current_record|
-          next unless record_has_source_identifier(current_record, index)
-          break if limit_reached?(limit, index)
-
-          seen[current_record[source_identifier]] = true
-          create_entry_and_job(current_record, type)
-          increment_counters(index, "#{type}": true)
-          index += 1
-        end
-        importer.record_status
-      end
-      true
-    rescue StandardError => e
-      set_status_info(e)
-    end
-
-    def create_entry_and_job(current_record, type)
-      new_entry = find_or_create_entry(send("#{type}_entry_class"),
-                                       current_record[source_identifier],
-                                       'Bulkrax::Importer',
-                                       current_record.to_h)
-      if current_record[:delete].present?
-        "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
-      else
-        "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
-      end
-    end
-
     def write_partial_import_file(file)
       import_filename = import_file_path.split('/').last
       partial_import_filename = "#{File.basename(import_filename, '.csv')}_corrected_entries.csv"
@@ -204,7 +153,6 @@ module Bulkrax
     def entry_class
       CsvEntry
     end
-    alias work_entry_class entry_class
 
     def collection_entry_class
       CsvCollectionEntry

data/app/parsers/bulkrax/oai_dc_parser.rb

@@ -63,6 +63,12 @@ module Bulkrax
 
     delegate :list_sets, to: :client
 
+    def create_objects(types = [])
+      types.each do |object_type|
+        send("create_#{object_type.pluralize}")
+      end
+    end
+
     def create_collections
       metadata = {
         visibility: 'open'
@@ -86,27 +92,31 @@ module Bulkrax
       results = self.records(quick: true)
       return if results.blank?
       results.full.each_with_index do |record, index|
-        identifier = record.send(source_identifier)
-        if identifier.blank?
-          if Bulkrax.fill_in_blank_source_identifiers.present?
-            identifier = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
-          else
-            invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
-            next
-          end
-        end
-
+        identifier = record_has_source_identifier(record, index)
+        next unless identifier
         break if limit_reached?(limit, index)
+
         seen[identifier] = true
-        new_entry = entry_class.where(importerexporter: self.importerexporter, identifier: identifier).first_or_create!
-        if record.deleted?
-          DeleteWorkJob.send(perform_method, new_entry, importerexporter.current_run)
-        else
-          ImportWorkJob.send(perform_method, new_entry.id, importerexporter.current_run.id)
-        end
+        create_entry_and_job(record, 'work', identifier)
         increment_counters(index, work: true)
       end
       importer.record_status
+    rescue StandardError => e
+      set_status_info(e)
+    end
+
+    # oai records so not let us set the source identifier easily
+    def record_has_source_identifier(record, index)
+      identifier = record.send(source_identifier)
+      if identifier.blank?
+        if Bulkrax.fill_in_blank_source_identifiers.present?
+          identifier = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
+        else
+          invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
+          return false
+        end
+      end
+      identifier
     end
 
     def collections

data/app/parsers/bulkrax/parser_export_record_set.rb

@@ -113,14 +113,14 @@ module Bulkrax
       #
       # @see #file_sets
       def candidate_file_set_ids
-        @candidate_file_set_ids ||= works.flat_map { |work| work.fetch("#{Bulkrax.file_model_class.to_s.underscore}_ids_ssim", []) }
+        @candidate_file_set_ids ||= works.flat_map { |work| work.fetch(Bulkrax.solr_key_for_member_file_ids, []) }
       end
 
       # @note Specifically not memoizing this so we can merge values without changing the object.
       #
       # No sense attempting to query for more than the limit.
       def query_kwargs
-        { fl: "id,#{Bulkrax.file_model_class.to_s.underscore}_ids_ssim", method: :post, rows: row_limit }
+        { fl: "id,#{Bulkrax.solr_key_for_member_file_ids}", method: :post, rows: row_limit }
       end
 
       # If we have a limit, we need not query beyond that limit

data/app/parsers/bulkrax/xml_parser.rb

@@ -11,13 +11,29 @@ module Bulkrax
     def collection_entry_class; end
 
     # @todo not yet supported
-    def create_collections; end
+    def create_collections
+      raise NotImplementedError
+    end
 
     # @todo not yet supported
     def file_set_entry_class; end
 
     # @todo not yet supported
-    def create_file_sets; end
+    def create_file_sets
+      raise NotImplementedError
+    end
+
+    def file_sets
+      raise NotImplementedError
+    end
+
+    def collections
+      raise NotImplementedError
+    end
+
+    def works
+      records
+    end
 
     # TODO: change to differentiate between collection and work records when adding ability to import collection metadata
     def works_total
@@ -92,25 +108,6 @@ module Bulkrax
       %w[.xml .xls .xsd].include?(File.extname(path)) || ::Marcel::MimeType.for(path).include?('application/xml')
     end
 
-    def create_works
-      records.each_with_index do |record, index|
-        next unless record_has_source_identifier(record, index)
-        break if !limit.nil? && index >= limit
-
-        seen[record[source_identifier]] = true
-        new_entry = find_or_create_entry(entry_class, record[source_identifier], 'Bulkrax::Importer', record)
-        if record[:delete].present?
-          DeleteWorkJob.send(perform_method, new_entry, current_run)
-        else
-          ImportWorkJob.send(perform_method, new_entry.id, current_run.id)
-        end
-        increment_counters(index, work: true)
-      end
-      importer.record_status
-    rescue StandardError => e
-      set_status_info(e)
-    end
-
     def total
       records.size
     end

data/app/services/bulkrax/factory_class_finder.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class FactoryClassFinder
+    ##
+    # The v6.0.0 default coercer.  Responsible for converting a factory class name to a constant.
+    module DefaultCoercer
+      ##
+      # @param name [String]
+      # @return [Class] when the name is a coercible constant.
+      # @raise [NameError] when the name is not coercible to a constant.
+      def self.call(name)
+        name.constantize
+      end
+    end
+
+    ##
+    # A name coercer that favors classes that end with "Resource" but will attempt to fallback to
+    # those that don't.
+    module ValkyrieMigrationCoercer
+      SUFFIX = "Resource"
+
+      ##
+      # @param name [String]
+      # @param suffix [String] the suffix we use for a naming convention.
+      #
+      # @return [Class] when the name is a coercible constant.
+      # @raise [NameError] when the name is not coercible to a constant.
+      def self.call(name, suffix: SUFFIX)
+        if name.end_with?(suffix)
+          name.constantize
+        else
+          begin
+            "#{name}#{suffix}".constantize
+          rescue NameError
+            name.constantize
+          end
+        end
+      end
+    end
+
+    ##
+    # @param entry [Bulkrax::Entry]
+    # @return [Class]
+    def self.find(entry:, coercer: Bulkrax.factory_class_name_coercer || DefaultCoercer)
+      new(entry: entry, coercer: coercer).find
+    end
+
+    def initialize(entry:, coercer:)
+      @entry = entry
+      @coercer = coercer
+    end
+    attr_reader :entry, :coercer
+
+    ##
+    # @return [Class] when we are able to derive the class based on the {#name}.
+    # @return [Nil] when we encounter errors with constantizing the {#name}.
+    # @see #name
+    def find
+      coercer.call(name)
+    rescue NameError
+      nil
+    rescue
+      entry.default_work_type.constantize
+    end
+
+    ##
+    # @api private
+    # @return [String]
+    def name
+      fc = if entry.parsed_metadata&.[]('model').present?
+             Array.wrap(entry.parsed_metadata['model']).first
+           elsif entry.importerexporter&.mapping&.[]('work_type').present?
+             # Because of delegation's nil guard, we're reaching rather far into the implementation
+             # details.
+             Array.wrap(entry.parsed_metadata['work_type']).first
+           else
+             entry.default_work_type
+           end
+
+      # Let's coerce this into the right shape; we're not mutating the string because it might well
+      # be frozen.
+      fc = fc.tr(' ', '_')
+      fc = fc.downcase if fc.match?(/[-_]/)
+      fc.camelcase
+    rescue
+      entry.default_work_type
+    end
+  end
+end

data/app/views/bulkrax/exporters/_form.html.erb

@@ -33,8 +33,8 @@
     label: t('bulkrax.exporter.labels.importer'),
     required: true,
     prompt: 'Select from the list',
-    label_html: { class: 'importer export-source-option d-none' },
-    input_html: { class: 'importer export-source-option d-none form-control' },
+    label_html: { class: 'importer export-source-option d-none hidden' },
+    input_html: { class: 'importer export-source-option d-none hidden form-control' },
     collection:  form.object.importers_list.sort %>
 
   <%= form.input :export_source_collection,
@@ -42,9 +42,9 @@
     label: t('bulkrax.exporter.labels.collection'),
     required: true,
     placeholder: @collection&.title&.first,
-    label_html: { class: 'collection export-source-option d-none' },
+    label_html: { class: 'collection export-source-option d-none hidden' },
     input_html: {
-      class: 'collection export-source-option d-none form-control',
+      class: 'collection export-source-option d-none hidden form-control',
       data: {
         'autocomplete-url' => '/authorities/search/collections',
         'autocomplete' => 'collection'
@@ -56,8 +56,8 @@
     label: t('bulkrax.exporter.labels.worktype'),
     required: true,
     prompt: 'Select from the list',
-    label_html: { class: 'worktype export-source-option d-none' },
-    input_html: { class: 'worktype export-source-option d-none form-control' },
+    label_html: { class: 'worktype export-source-option d-none hidden' },
+    input_html: { class: 'worktype export-source-option d-none hidden form-control' },
     collection: Bulkrax.curation_concerns.map { |cc| [cc.to_s, cc.to_s] } %>
 
   <%= form.input :limit,
@@ -80,7 +80,7 @@
                  as: :boolean,
                  label: t('bulkrax.exporter.labels.filter_by_date') %>
 
-  <div id="date_filter_picker" class="d-none">
+  <div id="date_filter_picker" class="d-none hidden">
     <%= form.input :start_date,
                    as: :date,
                    label: t('bulkrax.exporter.labels.start_date'),
@@ -136,13 +136,13 @@
             // get the date filter option and show the corresponding date selectors
             $('.exporter_date_filter').change(function () {
                 if ($('.exporter_date_filter').find(".boolean").is(":checked"))
-                    $('#date_filter_picker').removeClass('d-none');
+                    $('#date_filter_picker').removeClass('d-none hidden');
                 else
-                    $('#date_filter_picker').addClass('d-none');
+                    $('#date_filter_picker').addClass('d-none hidden');
             });
 
             if ($('.exporter_date_filter').find(".boolean").is(":checked"))
-                $('#date_filter_picker').removeClass('d-none');
+                $('#date_filter_picker').removeClass('d-none hidden');
         });
     });
 </script>

data/app/views/bulkrax/exporters/index.html.erb

@@ -13,62 +13,18 @@
 
 <div class="panel panel-default">
   <div class="panel-body">
-    <% if @exporters.present? %>
-      <div class="table-responsive">
-        <table class="table table-striped datatable">
-          <thead>
-            <tr>
-              <th scope="col">Name</th>
-              <th scope="col">Status</th>
-              <th scope="col">Date Exported</th>
-              <th scope="col">Downloadable Files</th>
-              <th scope="col"></th>
-              <th scope="col"></th>
-              <th scope="col"></th>
-            </tr>
-          </thead>
-          <tbody>
-            <% @exporters.each do |exporter| %>
-              <tr>
-                <th scope="row"><%= link_to exporter.name, exporter_path(exporter) %></th>
-                <td><%= exporter.status %></td>
-                <td><%= exporter.created_at %></td>
-                <td>
-                  <% if File.exist?(exporter.exporter_export_zip_path) %>
-                    <%= simple_form_for(exporter, method: :get, url: exporter_download_path(exporter)) do |form| %>
-                      <%= render 'downloads', exporter: exporter, form: form %>
-                      <%= form.button :submit, value: 'Download', data: { disable_with: false } %>
-                    <% end %>
-                  <% end%>
-                </td>
-                <td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), exporter_path(exporter) %></td>
-                <td><%= link_to raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_exporter_path(exporter), data: { turbolinks: false } %></td>
-                <td><%= link_to raw('<span class="glyphicon glyphicon-remove"></span>'), exporter, method: :delete, data: { confirm: 'Are you sure?' } %></td>
-              </tr>
-            <% end %>
-          </tbody>
-        </table>
-      </div>
-      <% else %>
-        <p>No exporters have been created.</p>
-      <% end %>
+    <div class="table-responsive">
+      <table id='exporters-table' class="table table-striped">
+        <thead>
+          <tr>
+            <th scope="col">Name</th>
+            <th scope="col">Status</th>
+            <th scope="col">Date Exported</th>
+            <th scope="col">Downloadable Files</th>
+            <th scope="col">Actions</th>
+          </tr>
+        </thead>
+      </table>
+    </div>
   </div>
 </div>
-
-<script>
-  $(function() {
-    $('#DataTables_Table_0').DataTable({
-      destroy: true, /* Reinitialize DataTable with config below */
-      'columnDefs': [
-          { 'orderable': true, 'targets': [0, 1, 2] },
-          { 'orderable': false, 'targets': [3, 4, 5, 6] }
-      ],
-      'language': {
-        'info': 'Showing _START_ to _END_ of _TOTAL_ exporters',
-        'infoEmpty': 'No exporters to show',
-        'infoFiltered': '(filtered from _MAX_ total exporters)',
-        'lengthMenu': 'Show _MENU_ exporters'
-      }
-    })
-  })
-</script>

data/app/views/bulkrax/exporters/show.html.erb

@@ -95,16 +95,8 @@
 
     <div class="bulkrax-nav-tab-bottom-margin">
       <!-- Nav tabs -->
-      <ul class="bulkrax-nav-tab-top-margin tab-nav nav nav-tabs" role="tablist">
-        <li role="presentation" class="nav-link active"><a href="#work-entries" aria-controls="work-entries" role="tab" data-toggle="tab"><%= t('bulkrax.exporter.labels.work_entries') %></a></li>
-        <li role="presentation" class="nav-link"><a href="#collection-entries" aria-controls="collection-entries" role="tab" data-toggle="tab"><%= t('bulkrax.exporter.labels.collection_entries') %></a></li>
-        <li role="presentation" class="nav-link"><a href="#file-set-entries" aria-controls="file-set-entries" role="tab" data-toggle="tab"><%= t('bulkrax.exporter.labels.file_set_entries') %></a></li>
-      </ul>
-      <!-- Tab panes -->
-      <div class="tab-content outline">
-        <%= render partial: 'bulkrax/shared/work_entries_tab', locals: { item: @exporter, entries: @work_entries } %>
-        <%= render partial: 'bulkrax/shared/collection_entries_tab', locals: { item: @exporter, entries: @collection_entries } %>
-        <%= render partial: 'bulkrax/shared/file_set_entries_tab', locals: { item: @exporter, entries: @file_set_entries } %>
+      <div class="outline">
+        <%= render partial: 'bulkrax/shared/entries_tab', locals: { item: @exporter } %>
       </div>
       <br>
       <%= link_to 'Edit', edit_exporter_path(@exporter) %>