ddr-batch 1.7.2 → 2.0.0.alpha.1

data/app/jobs/ddr/batch/batch_deletion_job.rb

@@ -1,19 +0,0 @@
-module Ddr::Batch
-  class BatchDeletionJob
-    @queue = :batch
-
-    def self.perform(batch_id)
-      batch = Batch.find(batch_id)
-      batch.status = Batch::STATUS_DELETING
-      batch.save!
-      batch.destroy!
-    end
-
-    def self.before_enqueue_set_status(batch_id)
-      batch = Batch.find(batch_id)
-      batch.status = Batch::STATUS_QUEUED_FOR_DELETION
-      batch.save
-    end
-
-  end
-end

data/app/jobs/ddr/batch/batch_objects_processor_job.rb

@@ -1,11 +0,0 @@
-module Ddr::Batch
-  class BatchObjectsProcessorJob
-    @queue = :batch
-
-    def self.perform(batch_object_ids, operator_id)
-      operator = User.find(operator_id)
-      ProcessBatchObjects.new(batch_object_ids: batch_object_ids, operator: operator).execute
-    end
-
-  end
-end

data/app/models/ddr/batch/batch_object_message.rb

@@ -1,8 +0,0 @@
-module Ddr::Batch
-  class BatchObjectMessage < ActiveRecord::Base
-    belongs_to :batch_object, :inverse_of => :batch_object_messages
-
-    validates_presence_of :message
-
-  end
-end

data/app/models/ddr/batch/batch_object_role.rb

@@ -1,26 +0,0 @@
-module Ddr::Batch
-
-  class BatchObjectRole < ActiveRecord::Base
-    belongs_to :batch_object, :inverse_of => :batch_object_roles
-
-    OPERATION_ADD = "ADD".freeze  # Add the principal and role to the object in the indicated scope
-    OPERATIONS = [ OPERATION_ADD ].freeze
-
-    validates :operation, inclusion: { in: OPERATIONS }
-    validate :valid_role, if: :operation_requires_valid_role?
-    validate :policy_role_scope_only_for_collections, if: "role_scope == Ddr::Auth::Roles::POLICY_SCOPE"
-
-    def operation_requires_valid_role?
-      [ OPERATION_ADD ].include? operation
-    end
-
-    def valid_role
-      Ddr::Auth::Roles::Role.build(type: role_type, agent: agent, scope: role_scope).valid?
-    end
-
-    def policy_role_scope_only_for_collections
-      errors.add(:role_scope, "policy role scope is valid only for Collections") unless batch_object.model == "Collection"
-    end
-  end
-
-end

data/app/models/ddr/batch/error.rb

@@ -1,10 +0,0 @@
-module Ddr
-  module Batch
-    # Base class for custom exceptions
-    class Error < StandardError; end
-
-    # Error processing batch object
-    class BatchObjectProcessingError < Error; end
-
-  end
-end

data/app/models/ddr/batch/log.rb

@@ -1,29 +0,0 @@
-module Ddr::Batch
-  class Log
-
-    DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
-
-    class << self
-
-      def logger(batch_id)
-        loggr = Logger.new(File.open(file_path(batch_id), File::WRONLY | File::APPEND | File::CREAT))
-        loggr.level = Ddr::Batch.processor_logging_level
-        loggr.datetime_format = "%Y-%m-%d %H:%M:%S.L"
-        loggr.formatter = proc do |severity, datetime, progname, msg|
-          "#{datetime} #{severity}: #{msg}\n"
-        end
-        loggr
-      end
-
-      def clear_log(batch_id)
-        log_file_path = file_path(batch_id)
-        FileUtils.remove(log_file_path) if File.exists?(log_file_path)
-      end
-
-      def file_path(batch_id)
-        File.join(DEFAULT_LOG_DIR, "batch_#{batch_id}_log.txt")
-      end
-
-    end
-  end
-end

data/app/services/ddr/batch/monitor_batch_finished.rb

@@ -1,87 +0,0 @@
-module Ddr::Batch
-  class MonitorBatchFinished
-
-    class << self
-      def call(*args)
-        event = ActiveSupport::Notifications::Event.new(*args)
-        batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
-        batch_finished(batch)
-        if batch.outcome == Ddr::Batch::Batch::OUTCOME_SUCCESS
-          ActiveSupport::Notifications.instrument('success.batch.batch.ddr', batch_id: batch.id)
-        end
-      end
-
-      private
-
-      def batch_finished(batch)
-        log_batch_finish(batch)
-        update_batch(batch)
-        send_email(batch) if batch.user && batch.user.email
-      end
-
-      def log_batch_finish(batch)
-        logger = Ddr::Batch::Log.logger(batch.id)
-        logger.info "====== Summary ======"
-        results_tracker = results(batch)
-        results_tracker.keys.each do |type|
-          results_tracker[type].keys.each do |model|
-            log_result(results_tracker, type, model, logger)
-          end
-        end
-        logger.close
-      end
-
-      def results(batch)
-        results_tracker = Hash.new
-        batch.batch_objects.each do |batch_object|
-          track_result(results_tracker, batch_object)
-        end
-        results_tracker
-      end
-
-      def track_result(results_tracker, batch_object)
-        type = batch_object.type
-        model = batch_object.model || "Missing Model"
-        results_tracker[type] = Hash.new unless results_tracker.has_key?(type)
-        results_tracker[type][model] = Hash.new unless results_tracker[type].has_key?(model)
-        results_tracker[type][model][:successes] = 0 unless results_tracker[type][model].has_key?(:successes)
-        results_tracker[type][model][:successes] += 1 if batch_object.verified
-      end
-
-      def log_result(results_tracker, type, model, logger)
-        verb = type_verb(type)
-        count = results_tracker[type][model][:successes]
-        logger.info "#{verb} #{ActionController::Base.helpers.pluralize(count, model)}"
-      end
-
-      def type_verb(type)
-        case type
-          when Ddr::Batch::IngestBatchObject.name
-            "Ingested"
-          when Ddr::Batch::UpdateBatchObject.name
-            "Updated"
-        end
-      end
-
-      def update_batch(batch)
-        outcome = batch.success_count.eql?(batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
-        logfile = File.new(Ddr::Batch::Log.file_path(batch.id))
-        batch.update!(stop: DateTime.now,
-                      status: Batch::STATUS_FINISHED,
-                      outcome: outcome,
-                      logfile: logfile)
-      end
-
-      def send_email(batch)
-        begin
-          Ddr::Batch::BatchProcessorRunMailer.send_notification(batch).deliver!
-        rescue => e
-          Rails.logger.error("An error occurred while attempting to send a notification for batch #{batch.id}")
-          Rails.logger.error(e.message)
-          Rails.logger.error(e.backtrace)
-        end
-      end
-    end
-
-  end
-end

data/app/services/ddr/batch/monitor_batch_object_handled.rb

@@ -1,42 +0,0 @@
-module Ddr::Batch
-  class MonitorBatchObjectHandled
-
-    class << self
-      def call(*args)
-        event = ActiveSupport::Notifications::Event.new(*args)
-        batch_object = BatchObject.find(event.payload[:batch_object_id])
-        batch = batch_object.batch
-        if event.payload[:exception].present?
-          record_batch_object_exception(batch_object, event.payload[:exception])
-        end
-        batch_object_handled(batch_object, batch)
-      end
-
-      private
-
-      def record_batch_object_exception(batch_object, exception_info)
-        batch_object_exception_msg = I18n.t('ddr.batch.errors.batch_object_processing', error_msg: exception_info[1])
-        Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
-                                               level: Logger::ERROR,
-                                               message: batch_object_exception_msg)
-      end
-
-      def batch_object_handled(batch_object, batch)
-        log_batch_object_messages(batch_object, batch.id)
-        batch_object.update!(handled: true)
-        unless batch.unhandled_objects?
-          ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
-        end
-      end
-
-      def log_batch_object_messages(batch_object, batch_id)
-        logger = Ddr::Batch::Log.logger(batch_id)
-        batch_object.batch_object_messages.each do |message|
-          logger.add(message.level) { "Batch Object #{batch_object.id}: #{message.message}" }
-        end
-        logger.close
-      end
-    end
-
-  end
-end

data/app/services/ddr/batch/monitor_batch_started.rb

@@ -1,43 +0,0 @@
-module Ddr::Batch
-  class MonitorBatchStarted
-
-    class << self
-      def call(*args)
-        event = ActiveSupport::Notifications::Event.new(*args)
-        batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
-        batch_started(batch)
-      end
-
-      private
-
-      def batch_started(batch)
-        clear_logs(batch)
-        log_batch_start(batch)
-        update_batch(batch)
-      end
-
-      def clear_logs(batch)
-        # delete any previously existing filesystem log file for this batch
-        Ddr::Batch::Log.clear_log(batch.id)
-        # remove any existing attached log file from the Batch ActiveRecord object
-        batch.logfile.clear
-      end
-
-      def log_batch_start(batch)
-        logger = Ddr::Batch::Log.logger(batch.id)
-        logger.info "Collection: #{batch.collection_title}" if batch.collection_title.present?
-        logger.info "Batch id: #{batch.id}"
-        logger.info "Batch name: #{batch.name}" if name
-        logger.info "Batch size: #{batch.batch_objects.size}"
-        logger.close
-      end
-
-      def update_batch(batch)
-        batch.update!(start: DateTime.now,
-                      status: Ddr::Batch::Batch::STATUS_RUNNING,
-                      version: VERSION)
-      end
-    end
-
-  end
-end

data/app/services/ddr/batch/process_batch.rb

@@ -1,60 +0,0 @@
-module Ddr::Batch
-  class ProcessBatch
-
-    attr_accessor :batch, :operator_id
-
-    def initialize(batch_id:, operator_id:)
-      @batch = Ddr::Batch::Batch.find(batch_id)
-      @operator_id = operator_id
-    end
-
-    def execute
-      ActiveSupport::Notifications.instrument('started.batch.batch.ddr', batch_id: batch.id)
-      batch.batch_objects.each do |batch_object|
-        case
-          when batch_object.is_a?(IngestBatchObject)
-            handle_ingest_batch_object(batch_object)
-          when batch_object.is_a?(UpdateBatchObject)
-            handle_update_batch_object(batch_object)
-        end
-      end
-    end
-
-    def handle_ingest_batch_object(batch_object)
-      case batch_object.model
-        when 'Collection'
-          ingest_collection_object(batch_object)
-        when 'Item'
-          enqueue_item_component_ingest(batch_object)
-        when 'Component'
-          # skip -- will be handled along with associated Item
-        when 'Target', 'Attachment'
-          Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
-      end
-    end
-
-    def handle_update_batch_object(batch_object)
-      Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
-    end
-
-    def ingest_collection_object(batch_object)
-      # Collection batch objects are processed synchronously because they need to exist in the repository
-      # prior to the processing of any objects (e.g., Item, Component, Target) associated with them.
-      # If the Collection batch object does not process successfully, consider the batch finished (albeit unsuccessfully)
-      # and raise an exception.
-      unless ProcessBatchObject.new(batch_object_id: batch_object.id, operator: User.find(operator_id)).execute
-        ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
-        raise Ddr::Batch::BatchObjectProcessingError, batch_object.id
-      end
-    end
-
-    def enqueue_item_component_ingest(batch_object)
-      query = [ "object = '#{batch_object.pid}'",
-                "batch_object_relationships.name = '#{Ddr::Batch::BatchObjectRelationship::RELATIONSHIP_PARENT}'",
-                "batches.id = #{batch_object.batch.id}" ].join(' AND ')
-      recs = Ddr::Batch::BatchObjectRelationship.joins(batch_object: :batch).where(query)
-      batch_object_ids = recs.map { |rec| rec.batch_object.id }.unshift(batch_object.id)
-      Resque.enqueue(BatchObjectsProcessorJob, batch_object_ids, operator_id)
-    end
-  end
-end

data/app/services/ddr/batch/process_batch_object.rb

@@ -1,46 +0,0 @@
-module Ddr::Batch
-  class ProcessBatchObject
-
-    attr_reader :batch_object_id, :operator
-
-    def initialize(batch_object_id:, operator:)
-      @batch_object_id = batch_object_id
-      @operator = operator
-    end
-
-    def execute
-      ActiveSupport::Notifications.instrument("handled.batchobject.batch.ddr",
-                                              batch_object_id: batch_object_id) do |payload|
-        batch_object = BatchObject.find(batch_object_id)
-        # Validate batch object
-        errors = batch_object.validate
-        # Process batch object or record validation errors
-        if errors.empty?
-          process(batch_object, operator)
-        else
-          record_errors(batch_object, errors)
-        end
-        # return true if batch_object was processed; otherwise, false
-        batch_object.processed? ? true : false
-      end
-    end
-
-    def process(batch_object, operator)
-      batch_object.update!(validated: true)
-      batch_object.process(operator)
-      batch_object.update!(processed: true)
-      results_message = batch_object.results_message
-      Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
-                                             level: results_message.level,
-                                             message: results_message.message)
-    end
-
-    def record_errors(batch_object, errors)
-      errors.each do |error|
-        Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
-                                               level: Logger::ERROR,
-                                               message: error)
-      end
-    end
-  end
-end

data/app/services/ddr/batch/process_batch_objects.rb

@@ -1,39 +0,0 @@
-module Ddr::Batch
-  class ProcessBatchObjects
-
-    attr_reader :batch_object_ids, :operator
-
-    def initialize(batch_object_ids:, operator:)
-      @batch_object_ids = batch_object_ids
-      @operator = operator
-    end
-
-    def execute
-      # Assume successful processing of all batch objects until proven otherwise.
-      success = true
-      batch_object_ids.each do |batch_object_id|
-        batch_object = Ddr::Batch::BatchObject.find(batch_object_id)
-        # Skip batch objects that have already been successfully processed.  This is useful when this service is
-        # called within the context of a BatchObjectsProcessorJob, that job fails, and the failed job is retried.
-        unless batch_object.verified?
-          # Once any batch object included in this job fails to process successfully, do not attempt to process
-          # any remaining batch objects included in this job.  Instead, mark them as "handled" so the batch knows
-          # it's not waiting on them to be handled before it can consider itself "finished".
-          # The use case prompting this behavior is a job containing an Item ingest batch object plus one or more
-          # associated Component ingest batch objects.  If the Item batch object fails to process correctly, we don't
-          # want to attempt to process the Component batch objects.
-          # In the preceding use case, we could skip the remaining batch objects only if the failed batch object is an
-          # Item but there might be future cases in which we don't want to process the remaining batch objects in the
-          # job regardless of which batch object fails.  The failure of any batch object to process should be rare
-          # enough that it doesn't seem harmful to cover this potential broader use case in the current code.
-          if success
-            success = ProcessBatchObject.new(batch_object_id: batch_object.id, operator: operator).execute
-          else
-            batch_object.update!(handled: true)
-          end
-        end
-      end
-    end
-
-  end
-end

data/config/initializers/subscriptions.rb

@@ -1,9 +0,0 @@
-##
-## Subscriptions to ActiveSupport::Notifications instrumentation events
-##
-
-# Batch Processing events
-ActiveSupport::Notifications.subscribe('started.batch.batch.ddr', Ddr::Batch::MonitorBatchStarted)
-ActiveSupport::Notifications.subscribe('handled.batchobject.batch.ddr', Ddr::Batch::MonitorBatchObjectHandled)
-ActiveSupport::Notifications.subscribe('finished.batch.batch.ddr', Ddr::Batch::MonitorBatchFinished)
-