bulkrax 5.0.0 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49a1f0ccc806cf73a7872a634c8c2819dac62f98f0a9dc163e7ada8931d9b1fe
-  data.tar.gz: ecf5c8ad3e4864110665cfb5c6c5f6329e3b868414815bba163646db839400e7
+  metadata.gz: c9794c69e891f2397ac94398676f49db008e83b7a6df16e7a08f0ab393c148ca
+  data.tar.gz: fd2d48507add6bcbc7557f9240951d174c488e456579af65e20bb6cdc6f3c080
 SHA512:
-  metadata.gz: df394e4fbbc6ca0a71eb595c075c5181d24098b1ab889ff185da10991ec01c2eaf60239658564838b67df4c0a421306de923d2eab4bbeffafb31321acb9ba1ff
-  data.tar.gz: 35b691e96d0e59f83efadc35364283239f7f741ab4bf9d859756dacc8483e380090512cd10bf3b6ec4ce81936d00c88eb4d1ed96742c384e0f3082028ffdcf45
+  metadata.gz: 3b07a4178650201c48602c9bb9df29135a5cc51bc6954355732bd6ae5fa918c5fd4c88d187e305b7ee2877ca995818d7ec3a476e92ec173c7569c0a15ca229a2
+  data.tar.gz: 324393aeb341d82b23e6391cbe1ec2a64d3e55a61400330f1656f46b3bb22cd0c6e7335cfeaf1b3b12e6ca5faefea5405b96e1100b1df138dd4c5a87fd522d76

data/app/controllers/bulkrax/entries_controller.rb

@@ -5,10 +5,10 @@ require_dependency "oai"
 
 module Bulkrax
   class EntriesController < ApplicationController
-    include Hyrax::ThemedLayoutController
+    include Hyrax::ThemedLayoutController if defined?(::Hyrax)
     before_action :authenticate_user!
     before_action :check_permissions
-    with_themed_layout 'dashboard'
+    with_themed_layout 'dashboard' if defined?(::Hyrax)
 
     def show
       if params[:importer_id].present?
@@ -23,6 +23,7 @@ module Bulkrax
       @importer = Importer.find(params[:importer_id])
       @entry = Entry.find(params[:id])
 
+      return unless defined?(::Hyrax)
       add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
       add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
       add_breadcrumb 'Importers', bulkrax.importers_path
@@ -35,6 +36,7 @@ module Bulkrax
       @exporter = Exporter.find(params[:exporter_id])
       @entry = Entry.find(params[:id])
 
+      return unless defined?(::Hyrax)
       add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
       add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
       add_breadcrumb 'Exporters', bulkrax.exporters_path

data/app/controllers/bulkrax/exporters_controller.rb

@@ -4,24 +4,26 @@ require_dependency "bulkrax/application_controller"
 
 module Bulkrax
   class ExportersController < ApplicationController
-    include Hyrax::ThemedLayoutController
+    include Hyrax::ThemedLayoutController if defined?(::Hyrax)
     include Bulkrax::DownloadBehavior
     before_action :authenticate_user!
     before_action :check_permissions
     before_action :set_exporter, only: [:show, :edit, :update, :destroy]
-    with_themed_layout 'dashboard'
+    with_themed_layout 'dashboard' if defined?(::Hyrax)
 
     # GET /exporters
     def index
       @exporters = Exporter.all
 
-      add_exporter_breadcrumbs
+      add_exporter_breadcrumbs if defined?(::Hyrax)
     end
 
     # GET /exporters/1
     def show
-      add_exporter_breadcrumbs
-      add_breadcrumb @exporter.name
+      if defined?(::Hyrax)
+        add_exporter_breadcrumbs
+        add_breadcrumb @exporter.name
+      end
 
       @work_entries = @exporter.entries.where(type: @exporter.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
       @collection_entries = @exporter.entries.where(type: @exporter.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
@@ -31,16 +33,18 @@ module Bulkrax
     # GET /exporters/new
     def new
       @exporter = Exporter.new
-
+      return unless defined?(::Hyrax)
       add_exporter_breadcrumbs
       add_breadcrumb 'New'
     end
 
     # GET /exporters/1/edit
     def edit
-      add_exporter_breadcrumbs
-      add_breadcrumb @exporter.name, bulkrax.exporter_path(@exporter.id)
-      add_breadcrumb 'Edit'
+      if defined?(::Hyrax)
+        add_exporter_breadcrumbs
+        add_breadcrumb @exporter.name, bulkrax.exporter_path(@exporter.id)
+        add_breadcrumb 'Edit'
+      end
 
       # Correctly populate export_source_collection input
       @collection = Collection.find(@exporter.export_source) if @exporter.export_source.present? && @exporter.export_from == 'collection'

data/app/controllers/bulkrax/importers_controller.rb

@@ -6,7 +6,7 @@ require_dependency 'oai'
 module Bulkrax
   # rubocop:disable Metrics/ClassLength
   class ImportersController < ApplicationController
-    include Hyrax::ThemedLayoutController
+    include Hyrax::ThemedLayoutController if defined?(::Hyrax)
     include Bulkrax::DownloadBehavior
     include Bulkrax::API
     include Bulkrax::ValidationHelper
@@ -16,14 +16,14 @@ module Bulkrax
     before_action :authenticate_user!, unless: -> { api_request? }
     before_action :check_permissions
     before_action :set_importer, only: [:show, :edit, :update, :destroy]
-    with_themed_layout 'dashboard'
+    with_themed_layout 'dashboard' if defined?(::Hyrax)
 
     # GET /importers
     def index
       @importers = Importer.all
       if api_request?
         json_response('index')
-      else
+      elsif defined?(::Hyrax)
         add_importer_breadcrumbs
       end
     end
@@ -32,14 +32,13 @@ module Bulkrax
     def show
       if api_request?
         json_response('show')
-      else
+      elsif defined?(::Hyrax)
         add_importer_breadcrumbs
         add_breadcrumb @importer.name
-
-        @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
-        @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
-        @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
       end
+      @work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
+      @collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
+      @file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
     end
 
     # GET /importers/new
@@ -47,7 +46,7 @@ module Bulkrax
       @importer = Importer.new
       if api_request?
         json_response('new')
-      else
+      elsif defined?(::Hyrax)
         add_importer_breadcrumbs
         add_breadcrumb 'New'
       end
@@ -57,7 +56,7 @@ module Bulkrax
     def edit
       if api_request?
         json_response('edit')
-      else
+      elsif defined?(::Hyrax)
         add_importer_breadcrumbs
         add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
         add_breadcrumb 'Edit'
@@ -159,6 +158,7 @@ module Bulkrax
     # GET /importer/1/upload_corrected_entries
     def upload_corrected_entries
       @importer = Importer.find(params[:importer_id])
+      return unless defined?(::Hyrax)
       add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
       add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
       add_breadcrumb 'Importers', bulkrax.importers_path

data/app/helpers/bulkrax/application_helper.rb

@@ -3,7 +3,7 @@ require 'coderay'
 
 module Bulkrax
   module ApplicationHelper
-    include ::Hyrax::HyraxHelperBehavior
+    include ::Hyrax::HyraxHelperBehavior if defined?(::Hyrax)
 
     def coderay(value, opts)
       CodeRay

data/app/helpers/bulkrax/importers_helper.rb

@@ -2,9 +2,9 @@
 
 module Bulkrax
   module ImportersHelper
-    # borrowd from batch-importer https://github.com/samvera-labs/hyrax-batch_ingest/blob/main/app/controllers/hyrax/batch_ingest/batches_controller.rb
+    # borrowed from batch-importer https://github.com/samvera-labs/hyrax-batch_ingest/blob/main/app/controllers/hyrax/batch_ingest/batches_controller.rb
     def available_admin_sets
-      # Restrict available_admin_sets to only those current user can desposit to.
+      # Restrict available_admin_sets to only those current user can deposit to.
       @available_admin_sets ||= Hyrax::Collections::PermissionsService.source_ids_for_deposit(ability: current_ability, source_type: 'admin_set').map do |admin_set_id|
         [AdminSet.find(admin_set_id).title.first, admin_set_id]
       end

data/app/helpers/bulkrax/validation_helper.rb

@@ -5,10 +5,8 @@ module Bulkrax
     def valid_create_params?
       check_admin_set
       check_user
-      return true if valid_importer? &&
-                     valid_commit? &&
-                     valid_name? &&
-                     valid_parser_klass? &&
+      return true if valid_importer? && valid_commit? &&
+                     valid_name? && valid_parser_klass? &&
                      valid_parser_fields?
     end
 
@@ -19,6 +17,8 @@ module Bulkrax
     end
 
     def check_admin_set
+      return unless defined?(::Hyrax)
+
       if params[:importer][:admin_set_id].blank?
         params[:importer][:admin_set_id] = AdminSet::DEFAULT_ID
       else

data/app/jobs/bulkrax/create_relationships_job.rb

@@ -16,13 +16,32 @@ module Bulkrax
   #       to either an instance of a Work or an instance of a Collection.
   # NOTE: In the context of this job, "identifier" is used to generically refer
   #       to either a record's ID or an Bulkrax::Entry's source_identifier.
+  # Please override with your own job for custom/non-hyrax applications
+  # set Bulkrax config variable :relationship_job to your custom class
   class CreateRelationshipsJob < ApplicationJob
+    ##
+    # @api public
+    # @since v5.0.1
+    #
+    # Once we've created the relationships, should we then index the works's file_sets to ensure
+    # that we have the proper indexed values.  This can help set things like `is_page_of_ssim` for
+    # IIIF manifest and search results of file sets.
+    #
+    # @note As of v5.0.1 the default behavior is to not perform this.  That preserves past
+    #       implementations.  However, we might determine that we want to change the default
+    #       behavior.  Which would likely mean a major version change.
+    #
+    # @example
+    #   # In config/initializers/bulkrax.rb
+    #   Bulkrax::CreateRelationshipsJob.update_child_records_works_file_sets = true
+    #
+    # @see https://github.com/scientist-softserv/louisville-hyku/commit/128a9ef
+    class_attribute :update_child_records_works_file_sets, default: false
+
     include DynamicRecordLookup
 
     queue_as :import
 
-    attr_accessor :child_records, :child_entry, :parent_record, :parent_entry, :importer_run_id
-
     # @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
     # @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
     #
@@ -31,82 +50,82 @@ module Bulkrax
     # Whether the @base_entry is the parent or the child in the relationship is determined by the presence of a
     # parent_identifier or child_identifier param. For example, if a parent_identifier is passed, we know @base_entry
     # is the child in the relationship, and vice versa if a child_identifier is passed.
+    #
+    # rubocop:disable Metrics/MethodLength
     def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
-      pending_relationships = Bulkrax::PendingRelationship.find_each.select do |rel|
-        rel.importer_run_id == importer_run_id && rel.parent_id == parent_identifier
-      end.sort_by(&:order)
-
-      @importer_run_id = importer_run_id
-      @parent_entry, @parent_record = find_record(parent_identifier, importer_run_id)
-      @child_records = { works: [], collections: [] }
-      pending_relationships.each do |rel|
-        raise ::StandardError, %("#{rel}" needs either a child or a parent to create a relationship) if rel.child_id.nil? || rel.parent_id.nil?
-        @child_entry, child_record = find_record(rel.child_id, importer_run_id)
-        if child_record
-          child_record.is_a?(::Collection) ? @child_records[:collections] << child_record : @child_records[:works] << child_record
+      importer_run = Bulkrax::ImporterRun.find(importer_run_id)
+      ability = Ability.new(importer_run.user)
+
+      parent_entry, parent_record = find_record(parent_identifier, importer_run_id)
+
+      number_of_successes = 0
+      number_of_failures = 0
+      errors = []
+
+      ActiveRecord::Base.uncached do
+        Bulkrax::PendingRelationship.where(parent_id: parent_identifier, importer_run_id: importer_run_id)
+                                    .ordered.find_each do |rel|
+          process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
+          number_of_successes += 1
+        rescue => e
+          number_of_failures += 1
+          errors << e
         end
       end
 
-      if (child_records[:collections].blank? && child_records[:works].blank?) || parent_record.nil?
+      # save record if members were added
+      parent_record.save! if @parent_record_members_added
+
+      # rubocop:disable Rails/SkipsModelValidations
+      if errors.present?
+        importer_run.increment!(:failed_relationships, number_of_failures)
+        parent_entry&.set_status_info(errors.last, importer_run)
+
+        # TODO: This can create an infinite job cycle, consider a time to live tracker.
         reschedule({ parent_identifier: parent_identifier, importer_run_id: importer_run_id })
         return false # stop current job from continuing to run after rescheduling
+      else
+        Bulkrax::ImporterRun.find(importer_run_id).increment!(:processed_relationships, number_of_successes)
       end
-      @parent_entry ||= Bulkrax::Entry.where(identifier: parent_identifier,
-                                             importerexporter_id: ImporterRun.find(importer_run_id).importer_id,
-                                             importerexporter_type: "Bulkrax::Importer").first
-      create_relationships
-      pending_relationships.each(&:destroy)
-    rescue ::StandardError => e
-      parent_entry ? parent_entry.status_info(e) : child_entry.status_info(e)
-      Bulkrax::ImporterRun.find(importer_run_id).increment!(:failed_relationships) # rubocop:disable Rails/SkipsModelValidations
+      # rubocop:enable Rails/SkipsModelValidations
     end
+    # rubocop:enable Metrics/MethodLength
 
     private
 
-    def create_relationships
-      if parent_record.is_a?(::Collection)
-        collection_parent_work_child unless child_records[:works].empty?
-        collection_parent_collection_child unless child_records[:collections].empty?
-      else
-        work_parent_work_child unless child_records[:works].empty?
-        raise ::StandardError, 'a Collection may not be assigned as a child of a Work' if child_records[:collections].present?
-      end
-    end
+    def process(relationship:, importer_run_id:, parent_record:, ability:)
+      raise "#{relationship} needs a child to create relationship" if relationship.child_id.nil?
+      raise "#{relationship} needs a parent to create relationship" if relationship.parent_id.nil?
 
-    def user
-      @user ||= Bulkrax::ImporterRun.find(importer_run_id).importer.user
-    end
+      _child_entry, child_record = find_record(relationship.child_id, importer_run_id)
+      raise "#{relationship} could not find child record" unless child_record
 
-    # Work-Collection membership is added to the child as member_of_collection_ids
-    # This is adding the reverse relationship, from the child to the parent
-    def collection_parent_work_child
-      child_work_ids = child_records[:works].map(&:id)
-      parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
+      raise "Cannot add child collection (ID=#{relationship.child_id}) to parent work (ID=#{relationship.parent_id})" if child_record.collection? && parent_record.work?
 
-      parent_record.add_member_objects(child_work_ids)
-      ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_work_ids.count) # rubocop:disable Rails/SkipsModelValidations
-    end
+      ability.authorize!(:edit, child_record)
 
-    # Collection-Collection membership is added to the as member_ids
-    def collection_parent_collection_child
-      child_records[:collections].each do |child_record|
-        ::Hyrax::Collections::NestedCollectionPersistenceService.persist_nested_collection_for(parent: parent_record, child: child_record)
-        ImporterRun.find(importer_run_id).increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
-      end
+      # We could do this outside of the loop, but that could lead to odd counter failures.
+      ability.authorize!(:edit, parent_record)
+
+      parent_record.is_a?(Collection) ? add_to_collection(child_record, parent_record) : add_to_work(child_record, parent_record)
+
+      child_record.file_sets.each(&:update_index) if update_child_records_works_file_sets? && child_record.respond_to?(:file_sets)
+      relationship.destroy
     end
 
-    # Work-Work membership is added to the parent as member_ids
-    def work_parent_work_child
-      records_hash = {}
-      child_records[:works].each_with_index do |child_record, i|
-        records_hash[i] = { id: child_record.id }
-      end
-      attrs = { work_members_attributes: records_hash }
+    def add_to_collection(child_record, parent_record)
       parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
-      env = Hyrax::Actors::Environment.new(parent_record, Ability.new(user), attrs)
+      child_record.member_of_collections << parent_record
+      child_record.save!
+    end
+
+    def add_to_work(child_record, parent_record)
+      return true if parent_record.ordered_members.to_a.include?(child_record)
 
-      Hyrax::CurationConcern.actor.update(env)
-      ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_records[:works].count) # rubocop:disable Rails/SkipsModelValidations
+      parent_record.ordered_members << child_record
+      @parent_record_members_added = true
+      # TODO: Do we need to save the child record?
+      child_record.save!
     end
 
     def reschedule(parent_identifier:, importer_run_id:)

data/app/jobs/bulkrax/delete_job.rb

@@ -13,7 +13,7 @@ module Bulkrax
       entry.save!
       entry.importer.current_run = ImporterRun.find(importer_run.id)
       entry.importer.record_status
-      entry.status_info("Deleted", ImporterRun.find(importer_run.id))
+      entry.set_status_info("Deleted", ImporterRun.find(importer_run.id))
     end
     # rubocop:enable Rails/SkipsModelValidations
   end

data/app/jobs/bulkrax/export_work_job.rb

@@ -29,9 +29,9 @@ module Bulkrax
       return entry if exporter_run.enqueued_records.positive?
 
       if exporter_run.failed_records.positive?
-        exporter_run.exporter.status_info('Complete (with failures)')
+        exporter_run.exporter.set_status_info('Complete (with failures)')
       else
-        exporter_run.exporter.status_info('Complete')
+        exporter_run.exporter.set_status_info('Complete')
       end
 
       return entry

data/app/jobs/bulkrax/import_file_set_job.rb

@@ -41,7 +41,7 @@ module Bulkrax
         ImportFileSetJob.set(wait: (entry.import_attempts + 1).minutes).perform_later(entry_id, importer_run_id)
       else
         ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
-        entry.status_info(e)
+        entry.set_status_info(e)
       end
     end
 

data/app/jobs/bulkrax/import_work_job.rb

@@ -5,24 +5,37 @@ module Bulkrax
     queue_as :import
 
     # rubocop:disable Rails/SkipsModelValidations
+    #
+    # @note Yes, we are calling {ImporterRun.find} each time.  these were on purpose to prevent race
+    #       conditions on the database update. If you do not re-find (or at least reload) the object
+    #       on each increment, the count can get messed up. Let's say there are two jobs A and B and
+    #       a counter set to 2.
+    #
+    #       - A grabs the importer_run (line 10)
+    #       - B grabs the importer_run (line 10)
+    #       - A Finishes the build, does the increment (now the counter is 3)
+    #       - B Finishes the build, does the increment (now the counter is 3 again) and thus a count
+    #         is lost.
+    #
+    # @see https://codingdeliberately.com/activerecord-increment/
+    # @see https://github.com/samvera-labs/bulkrax/commit/5c2c795452e13a98c9217fdac81ae2f5aea031a0#r105848236
     def perform(entry_id, run_id, time_to_live = 3, *)
       entry = Entry.find(entry_id)
-      importer_run = ImporterRun.find(run_id)
       entry.build
       if entry.status == "Complete"
-        importer_run.increment!(:processed_records)
-        importer_run.increment!(:processed_works)
+        ImporterRun.find(run_id).increment!(:processed_records)
+        ImporterRun.find(run_id).increment!(:processed_works)
       else
         # do not retry here because whatever parse error kept you from creating a work will likely
         # keep preventing you from doing so.
-        importer_run.increment!(:failed_records)
-        importer_run.increment!(:failed_works)
+        ImporterRun.find(run_id).increment!(:failed_records)
+        ImporterRun.find(run_id).increment!(:failed_works)
       end
       # Regardless of completion or not, we want to decrement the enqueued records.
-      importer_run.decrement!(:enqueued_records) unless importer_run.enqueued_records <= 0
+      ImporterRun.find(run_id).decrement!(:enqueued_records) unless ImporterRun.find(run_id).enqueued_records <= 0
 
       entry.save!
-      entry.importer.current_run = importer_run
+      entry.importer.current_run = ImporterRun.find(run_id)
       entry.importer.record_status
     rescue Bulkrax::CollectionsCreatedError => e
       Rails.logger.warn("#{self.class} entry_id: #{entry_id}, run_id: #{run_id} encountered #{e.class}: #{e.message}")

data/app/jobs/bulkrax/importer_job.rb

@@ -12,8 +12,8 @@ module Bulkrax
       import(importer, only_updates_since_last_import)
       update_current_run_counters(importer)
       schedule(importer) if importer.schedulable?
-    rescue CSV::MalformedCSVError => e
-      importer.status_info(e)
+    rescue ::CSV::MalformedCSVError => e
+      importer.set_status_info(e)
     end
 
     def import(importer, only_updates_since_last_import)

data/app/jobs/bulkrax/schedule_relationships_job.rb

@@ -9,7 +9,8 @@ module Bulkrax
       return reschedule(importer_id) unless pending_num.zero?
 
       importer.last_run.parents.each do |parent_id|
-        CreateRelationshipsJob.perform_later(parent_identifier: parent_id, importer_run_id: importer.last_run.id)
+        Bulkrax.relationship_job_class.constantize.perform_later(parent_identifier: parent_id,
+                                                                 importer_run_id: importer.last_run.id)
       end
     end
 

data/app/matchers/bulkrax/application_matcher.rb

@@ -102,6 +102,7 @@ module Bulkrax
 
     # Only add valid resource types
     def parse_resource_type(src)
+      ActiveSupport::Deprecation.warn('#parse_resource_type will be removed in Bulkrax v6.0.0')
       Hyrax::ResourceTypesService.label(src.to_s.strip.titleize)
     rescue KeyError
       nil