bulkrax 6.0.1 → 7.0.0

data/app/controllers/concerns/bulkrax/datatables_behavior.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  # rubocop:disable Metrics/ModuleLength
+  module DatatablesBehavior
+    extend ActiveSupport::Concern
+
+    def table_per_page
+      per_page = params[:length].to_i
+      per_page < 1 ? 30 : per_page
+    end
+
+    def order_value(column)
+      params['columns']&.[](column)&.[]('data')
+    end
+
+    def table_order
+      "#{order_value(params&.[]('order')&.[]('0')&.[]('column'))} #{params&.[]('order')&.[]('0')&.[]('dir')}" if params&.[]('order')&.[]('0')&.[]('column').present?
+    end
+
+    # convert offset to page number
+    def table_page
+      params[:start].blank? ? 1 : (params[:start].to_i / params[:length].to_i) + 1
+    end
+
+    def entry_table_search
+      return @entry_table_search if @entry_table_search
+      return @entry_table_search = false if params['search']&.[]('value').blank?
+
+      table_search_value = params['search']&.[]('value')&.downcase
+
+      ['identifier', 'id', 'status_message', 'type', 'updated_at'].map do |col|
+        column = Bulkrax::Entry.arel_table[col]
+        column = Arel::Nodes::NamedFunction.new('CAST', [column.as('text')])
+        column = Arel::Nodes::NamedFunction.new('LOWER', [column])
+        @entry_table_search = if @entry_table_search
+                                @entry_table_search.or(column.matches("%#{table_search_value}%"))
+                              else
+                                column.matches("%#{table_search_value}%")
+                              end
+      end
+
+      @entry_table_search
+    end
+
+    def importer_table_search
+      return @importer_table_search if @importer_table_search
+      return @importer_table_search = false if params['search']&.[]('value').blank?
+
+      table_search_value = params['search']&.[]('value')&.downcase
+
+      ['name', 'id', 'status_message', 'last_error_at', 'last_succeeded_at', 'updated_at'].map do |col|
+        column = Bulkrax::Importer.arel_table[col]
+        column = Arel::Nodes::NamedFunction.new('CAST', [column.as('text')])
+        column = Arel::Nodes::NamedFunction.new('LOWER', [column])
+        @importer_table_search = if @importer_table_search
+                                   @importer_table_search.or(column.matches("%#{table_search_value}%"))
+                                 else
+                                   column.matches("%#{table_search_value}%")
+                                 end
+      end
+
+      @importer_table_search
+    end
+
+    def exporter_table_search
+      return @exporter_table_search if @exporter_table_search
+      return @exporter_table_search = false if params['search']&.[]('value').blank?
+
+      table_search_value = params['search']&.[]('value')&.downcase
+
+      ['name', 'status_message', 'created_at'].map do |col|
+        column = Bulkrax::Exporter.arel_table[col]
+        column = Arel::Nodes::NamedFunction.new('CAST', [column.as('text')])
+        column = Arel::Nodes::NamedFunction.new('LOWER', [column])
+        @exporter_table_search = if @exporter_table_search
+                                   @exporter_table_search.or(column.matches("%#{table_search_value}%"))
+                                 else
+                                   column.matches("%#{table_search_value}%")
+                                 end
+      end
+
+      @exporter_table_search
+    end
+
+    def format_importers(importers)
+      result = importers.map do |i|
+        {
+          name: view_context.link_to(i.name, view_context.importer_path(i)),
+          status_message: status_message_for(i),
+          last_imported_at: i.last_imported_at&.strftime("%b %d, %Y"),
+          next_import_at: i.next_import_at&.strftime("%b %d, %Y"),
+          enqueued_records: i.last_run&.enqueued_records,
+          processed_records: i.last_run&.processed_records || 0,
+          failed_records: i.last_run&.failed_records || 0,
+          deleted_records: i.last_run&.deleted_records,
+          total_collection_entries: i.last_run&.total_collection_entries,
+          total_work_entries: i.last_run&.total_work_entries,
+          total_file_set_entries: i.last_run&.total_file_set_entries,
+          actions: importer_util_links(i)
+        }
+      end
+      {
+        data: result,
+        recordsTotal: Bulkrax::Importer.count,
+        recordsFiltered: importers.size
+      }
+    end
+
+    def format_exporters(exporters)
+      result = exporters.map do |e|
+        {
+          name: view_context.link_to(e.name, view_context.exporter_path(e)),
+          status_message: status_message_for(e),
+          created_at: e.created_at,
+          download: download_zip(e),
+          actions: exporter_util_links(e)
+        }
+      end
+      {
+        data: result,
+        recordsTotal: Bulkrax::Exporter.count,
+        recordsFiltered: exporters.size
+      }
+    end
+
+    def format_entries(entries, item)
+      result = entries.map do |e|
+        {
+          identifier: view_context.link_to(e.identifier, view_context.item_entry_path(item, e)),
+          id: e.id,
+          status_message: status_message_for(e),
+          type: e.type,
+          updated_at: e.updated_at,
+          errors: e.latest_status&.error_class&.present? ? view_context.link_to(e.latest_status.error_class, view_context.item_entry_path(item, e), title: e.latest_status.error_message) : "",
+          actions: entry_util_links(e, item)
+        }
+      end
+      {
+        data: result,
+        recordsTotal: item.entries.size,
+        recordsFiltered: item.entries.size
+      }
+    end
+
+    def entry_util_links(e, item)
+      links = []
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-info-sign"></span>'), view_context.item_entry_path(item, e))
+      links << "<a class='glyphicon glyphicon-repeat' data-toggle='modal' data-target='#bulkraxItemModal' data-entry-id='#{e.id}'></a>" if view_context.an_importer?(item)
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-trash"></span>'), view_context.item_entry_path(item, e), method: :delete, data: { confirm: 'This will delete the entry and any work associated with it. Are you sure?' })
+      links.join(" ")
+    end
+
+    def status_message_for(e)
+      if e.status_message == "Complete"
+        "<td><span class='glyphicon glyphicon-ok' style='color: green;'></span> #{e.status_message}</td>"
+      elsif e.status_message == "Pending"
+        "<td><span class='glyphicon glyphicon-option-horizontal' style='color: blue;'></span> #{e.status_message}</td>"
+      elsif e.status_message == "Skipped"
+        "<td><span class='glyphicon glyphicon-step-forward' style='color: yellow;'></span> #{e.status_message}</td>"
+      else
+        "<td><span class='glyphicon glyphicon-remove' style='color: #{e.status == 'Deleted' ? 'green' : 'red'};'></span> #{e.status_message}</td>"
+      end
+    end
+
+    def importer_util_links(i)
+      links = []
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-info-sign"></span>'), importer_path(i))
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_importer_path(i))
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
+      links.join(" ")
+    end
+
+    def exporter_util_links(i)
+      links = []
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-info-sign"></span>'), exporter_path(i))
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-pencil"></span>'), edit_exporter_path(i), data: { turbolinks: false })
+      links << view_context.link_to(view_context.raw('<span class="glyphicon glyphicon-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
+      links.join(" ")
+    end
+
+    def download_zip(e)
+      return unless File.exist?(e.exporter_export_zip_path)
+
+      options_html = e.exporter_export_zip_files.flatten.map do |file_name|
+        "<option value='#{CGI.escapeHTML(file_name)}'>#{CGI.escapeHTML(file_name)}</option>"
+      end.join
+
+      form_html = "<form class='simple_form edit_exporter' id='edit_exporter_#{e.id}' action='#{view_context.exporter_download_path(e)}' accept-charset='UTF-8' method='get'>"
+      form_html += "<input name='utf8' type='hidden' value='✓'>"
+      form_html += "<select class='btn btn-default form-control' style='width: 200px' name='exporter[exporter_export_zip_files]' id='exporter_#{e.id}_exporter_export_zip_files'>"
+      form_html += options_html
+      form_html += "</select>\n" # add newline here to add a space between the dropdown and the download button
+      form_html += "<input type='submit' name='commit' value='Download' class='btn btn-default'>"
+      form_html += "</form>"
+
+      form_html
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end

data/app/helpers/bulkrax/application_helper.rb

@@ -1,9 +1,13 @@
 # frozen_string_literal: true
-require 'coderay'
-
 module Bulkrax
   module ApplicationHelper
-    include ::Hyrax::HyraxHelperBehavior if defined?(::Hyrax)
+    def item_entry_path(item, e, opts = {})
+      an_importer?(item) ? bulkrax.importer_entry_path(item.id, e.id, opts) : bulkrax.exporter_entry_path(item.id, e.id, opts)
+    end
+
+    def an_importer?(item)
+      item.class.to_s.include?('Importer')
+    end
 
     def coderay(value, opts)
       CodeRay

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -40,7 +40,7 @@ module Bulkrax
 
     include DynamicRecordLookup
 
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
     # @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)

data/app/jobs/bulkrax/delete_and_import_collection_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportCollectionJob < DeleteAndImportJob
+    DELETE_CLASS = Bulkrax::DeleteCollectionJob
+    IMPORT_CLASS = Bulkrax::ImportCollectionJob
+  end
+end

data/app/jobs/bulkrax/delete_and_import_file_set_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportFileSetJob < DeleteAndImportJob
+    DELETE_CLASS = Bulkrax::DeleteFileSetJob
+    IMPORT_CLASS = Bulkrax::ImportFileSetJob
+  end
+end

data/app/jobs/bulkrax/delete_and_import_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportJob < ApplicationJob
+    queue_as :import
+
+    def perform(entry, importer_run)
+      status = self.class::DELETE_CLASS.perform_now(entry, importer_run)
+      if status.status_message == "Deleted"
+        entry = Bulkrax::Entry.find(entry.id) # maximum reload
+        self.class::IMPORT_CLASS.perform_now(entry.id, importer_run.id)
+      end
+
+    rescue => e
+      entry.set_status_info(e)
+      # this causes caught exception to be reraised
+      raise
+    end
+  end
+end

data/app/jobs/bulkrax/delete_and_import_work_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class DeleteAndImportWorkJob < DeleteAndImportJob
+    DELETE_CLASS = Bulkrax::DeleteWorkJob
+    IMPORT_CLASS = Bulkrax::ImportWorkJob
+  end
+end

data/app/jobs/bulkrax/delete_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class DeleteJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     def perform(entry, importer_run)
       obj = entry.factory.find
@@ -15,6 +15,10 @@ module Bulkrax
       entry.importer.current_run = ImporterRun.find(importer_run.id)
       entry.importer.record_status
       entry.set_status_info("Deleted", ImporterRun.find(importer_run.id))
+    rescue => e
+      entry.set_status_info(e)
+      # this causes caught exception to be reraised
+      raise
     end
   end
 end

data/app/jobs/bulkrax/download_cloud_file_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class DownloadCloudFileJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # Retrieve cloud file and write to the imports directory
     # Note: if using the file system, the mounted directory in

data/app/jobs/bulkrax/import_collection_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImportCollectionJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # rubocop:disable Rails/SkipsModelValidations
     def perform(*args)

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -6,7 +6,7 @@ module Bulkrax
   class ImportFileSetJob < ApplicationJob
     include DynamicRecordLookup
 
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     attr_reader :importer_run_id
 

data/app/jobs/bulkrax/import_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ImportJob < ApplicationJob
+    queue_as :import
+  end
+end

data/app/jobs/bulkrax/import_work_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImportWorkJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     # rubocop:disable Rails/SkipsModelValidations
     #

data/app/jobs/bulkrax/importer_job.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class ImporterJob < ApplicationJob
-    queue_as :import
+    queue_as Bulkrax.config.ingest_queue_name
 
     def perform(importer_id, only_updates_since_last_import = false)
       importer = Importer.find(importer_id)

data/app/models/bulkrax/csv_collection_entry.rb

@@ -2,9 +2,7 @@
 
 module Bulkrax
   class CsvCollectionEntry < CsvEntry
-    def factory_class
-      Collection
-    end
+    self.default_work_type = "Collection"
 
     # Use identifier set by CsvParser#unique_collection_identifier, which falls back
     # on the Collection's first title if record[source_identifier] is not present

data/app/models/bulkrax/csv_entry.rb

@@ -16,11 +16,12 @@ module Bulkrax
     class_attribute(:csv_read_data_options, default: {})
 
     # there's a risk that this reads the whole file into memory and could cause a memory leak
+    # we strip any special characters out of the headers. looking at you Excel
     def self.read_data(path)
       raise StandardError, 'CSV path empty' if path.blank?
       options = {
         headers: true,
-        header_converters: ->(h) { h.to_s.strip.to_sym },
+        header_converters: ->(h) { h.to_s.gsub(/[^\w\d\. -]+/, '').strip.to_sym },
         encoding: 'utf-8'
       }.merge(csv_read_data_options)
 

data/app/models/bulkrax/entry.rb

@@ -8,6 +8,8 @@ module Bulkrax
   class Entry < ApplicationRecord
     include Bulkrax::HasMatchers
     include Bulkrax::ImportBehavior
+    self.class_attribute :default_work_type, default: Bulkrax.default_work_type
+
     include Bulkrax::ExportBehavior
     include Bulkrax::StatusInfo
     include Bulkrax::HasLocalProcessing

data/app/models/bulkrax/exporter.rb

@@ -23,6 +23,10 @@ module Bulkrax
       set_status_info(e)
     end
 
+    def remove_and_rerun
+      self.parser_fields['remove_and_rerun']
+    end
+
     # #export_source accessors
     # Used in form to prevent it from getting confused as to which value to populate #export_source with.
     # Also, used to display the correct selected value when rendering edit form.
@@ -102,9 +106,12 @@ module Bulkrax
       Importer.all.map { |i| [i.name, i.id] }
     end
 
-    def current_run
+    def current_run(skip_counts: false)
+      @current_run ||= self.exporter_runs.create! if skip_counts
+      return @current_run if @current_run
+
       total = self.limit || parser.total
-      @current_run ||= self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
+      @current_run = self.exporter_runs.create!(total_work_entries: total, enqueued_records: total)
     end
 
     def last_run

data/app/models/bulkrax/importer.rb

@@ -3,7 +3,7 @@
 require 'iso8601'
 
 module Bulkrax
-  class Importer < ApplicationRecord
+  class Importer < ApplicationRecord # rubocop:disable Metrics/ClassLength
     include Bulkrax::ImporterExporterBehavior
     include Bulkrax::StatusInfo
 
@@ -103,11 +103,12 @@ module Bulkrax
       frequency.to_seconds != 0
     end
 
-    def current_run
+    def current_run(skip_counts: false)
       return @current_run if @current_run.present?
 
       @current_run = self.importer_runs.create!
       return @current_run if file? && zip?
+      return @current_run if skip_counts
 
       entry_counts = {
         total_work_entries: self.limit || parser.works_total,
@@ -123,6 +124,29 @@ module Bulkrax
       @last_run ||= self.importer_runs.last
     end
 
+    def failed_entries?
+      entries.failed.any?
+    end
+
+    def failed_statuses
+      @failed_statuses ||= Bulkrax::Status.latest_by_statusable
+                                          .includes(:statusable)
+                                          .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Failed')
+    end
+
+    def failed_messages
+      failed_statuses.each_with_object({}) do |e, i|
+        i[e.error_message] ||= []
+        i[e.error_message] << e.id
+      end
+    end
+
+    def completed_statuses
+      @completed_statuses ||= Bulkrax::Status.latest_by_statusable
+                                             .includes(:statusable)
+                                             .where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', self.entries.pluck(:id), 'Bulkrax::Entry', 'Complete')
+    end
+
     def seen
       @seen ||= {}
     end
@@ -135,6 +159,18 @@ module Bulkrax
       self.parser_fields['update_files']
     end
 
+    def remove_and_rerun
+      self.parser_fields['remove_and_rerun']
+    end
+
+    def metadata_only?
+      parser.parser_fields['metadata_only'] == true
+    end
+
+    def existing_entries?
+      parser.parser_fields['file_style']&.match(/Existing Entries/)
+    end
+
     def import_works
       import_objects(['work'])
     end
@@ -157,11 +193,20 @@ module Bulkrax
       self.only_updates ||= false
       self.save if self.new_record? # Object needs to be saved for statuses
       types = types_array || DEFAULT_OBJECT_TYPES
-      parser.create_objects(types)
+      existing_entries? ? parser.rebuild_entries(types) : parser.create_objects(types)
+      mark_unseen_as_skipped
     rescue StandardError => e
       set_status_info(e)
     end
 
+    # After an import any entries we did not touch are skipped.
+    # They are not really pending, complete for the last run, or failed
+    def mark_unseen_as_skipped
+      entries.where.not(identifier: seen.keys).find_each do |entry|
+        entry.set_status_info('Skipped')
+      end
+    end
+
     # Prepend the base_url to ensure unique set identifiers
     # @todo - move to parser, as this is OAI specific
     def unique_collection_identifier(id)
@@ -192,9 +237,5 @@ module Bulkrax
     rescue
       "#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
     end
-
-    def metadata_only?
-      parser.parser_fields['metadata_only'] == true
-    end
   end
 end

data/app/models/bulkrax/oai_set_entry.rb

@@ -2,9 +2,7 @@
 
 module Bulkrax
   class OaiSetEntry < OaiEntry
-    def factory_class
-      Collection
-    end
+    self.default_work_type = "Collection"
 
     def build_metadata
       self.parsed_metadata = self.raw_metadata

data/app/models/bulkrax/rdf_collection_entry.rb

@@ -2,6 +2,7 @@
 
 module Bulkrax
   class RdfCollectionEntry < RdfEntry
+    self.default_work_type = "Collection"
     def record
       @record ||= self.raw_metadata
     end
@@ -11,9 +12,5 @@ module Bulkrax
       add_local
       return self.parsed_metadata
     end
-
-    def factory_class
-      Collection
-    end
   end
 end

data/app/models/bulkrax/status.rb

@@ -2,7 +2,7 @@
 
 module Bulkrax
   class Status < ApplicationRecord
-    belongs_to :statusable, polymorphic: true
+    belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message], if: :latest? }
     belongs_to :runnable, polymorphic: true
     serialize :error_backtrace, Array
 
@@ -21,5 +21,14 @@ module Bulkrax
       status_table.join(latest_status_query.as(latest_status_table.name.to_s), Arel::Nodes::InnerJoin)
                   .on(status_table[:id].eq(latest_status_table[:latest_status_id]))
     end
+
+    def latest?
+      # TODO: remove if statment when we stop supporting Hyrax < 4
+      self.id == if Gem::Version.new(Rails::VERSION::STRING) >= Gem::Version.new('6.0.0')
+                   self.class.where(statusable_id: self.statusable_id, statusable_type: self.statusable_type).order('id desc').pick(:id)
+                 else
+                   self.class.where(statusable_id: self.statusable_id, statusable_type: self.statusable_type).order('id desc').pluck(:id).first # rubocop:disable Rails/Pick
+                 end
+    end
   end
 end

data/app/models/concerns/bulkrax/file_set_entry_behavior.rb

@@ -2,8 +2,10 @@
 
 module Bulkrax
   module FileSetEntryBehavior
-    def factory_class
-      ::FileSet
+    extend ActiveSupport::Concern
+
+    included do
+      self.default_work_type = "::FileSet"
     end
 
     def file_reference

data/app/models/concerns/bulkrax/import_behavior.rb

@@ -189,22 +189,10 @@ module Bulkrax
     end
 
     def factory_class
-      fc = if self.parsed_metadata&.[]('model').present?
-             self.parsed_metadata&.[]('model').is_a?(Array) ? self.parsed_metadata&.[]('model')&.first : self.parsed_metadata&.[]('model')
-           elsif self.mapping&.[]('work_type').present?
-             self.parsed_metadata&.[]('work_type').is_a?(Array) ? self.parsed_metadata&.[]('work_type')&.first : self.parsed_metadata&.[]('work_type')
-           else
-             Bulkrax.default_work_type
-           end
-
-      # return the name of the collection or work
-      fc.tr!(' ', '_')
-      fc.downcase! if fc.match?(/[-_]/)
-      fc.camelcase.constantize
-    rescue NameError
-      nil
-    rescue
-      Bulkrax.default_work_type.constantize
+      # ATTENTION: Do not memoize this here; tests should catch the problem, but through out the
+      # lifecycle of parsing a CSV row or what not, we end up having different factory classes based
+      # on the encountered metadata.
+      FactoryClassFinder.find(entry: self)
     end
   end
 end

data/app/models/concerns/bulkrax/status_info.rb

@@ -10,6 +10,10 @@ module Bulkrax
               as: :statusable,
               class_name: "Bulkrax::Status",
               inverse_of: :statusable
+      scope :failed, -> { where(status_message: 'Failed') }
+      scope :complete, -> { where(status_message: 'Complete') }
+      scope :pending, -> { where(status_message: 'Pending') }
+      scope :skipped, -> { where(status_message: 'Skipped') }
     end
 
     def current_status
@@ -25,6 +29,10 @@ module Bulkrax
       current_status&.status_message&.match(/^Complete$/)
     end
 
+    def skipped?
+      current_status&.status_message&.match('Skipped')
+    end
+
     def status
       current_status&.status_message || 'Pending'
     end