ddr-batch 1.1.0 → 1.2.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6f10719eff4705f320384ac30a9b9353911e375
-  data.tar.gz: 668bcc32b0d5a85f36443a71436b42b0a20b01cb
+  metadata.gz: e99d993322b0feb42c08dd0a367aa2cbb9a8bd36
+  data.tar.gz: 7d3ea407d8f1cc09f33725c8c54cbbb934967a98
 SHA512:
-  metadata.gz: d4c23913959d6405ca58d0155a98126ffc4e0c7741e7f9d2961f48f411927ff9c48d36c34e72a65bcb0cb961617324c1cbaa3fc7a5d78fa4aee519484d3d3187
-  data.tar.gz: a9fbe5fbee20e38b27f94f9bc5f7f4d067e4dd3938e21372839c3e5ccf43513fe239be531cfb76011bac7c70f03dce4f4942c15f48206cb0ce8362b08581aeb8
+  metadata.gz: 8954284d5fed970fbaa7ee95626bafc28f7e906ba668d657d95279191555f842b6b5832943bc9034fdcf5bf699f27ad2d7c2d412d34cb4d89f6d9fff24bb1eb5
+  data.tar.gz: 2de0336399b640e5b3b85daa2ca65ebd1ac31f85e259907d80701d68198c8969e1dc155024ebe9a3bbe16b50cc61703677fe5cb2a9392d2306e2d5094b914bcc

data/app/jobs/ddr/batch/batch_deletion_job.rb

@@ -0,0 +1,19 @@
+module Ddr::Batch
+  class BatchDeletionJob
+    @queue = :batch
+
+    def self.perform(batch_id)
+      batch = Batch.find(batch_id)
+      batch.status = Batch::STATUS_DELETING
+      batch.save!
+      batch.destroy!
+    end
+
+    def self.before_enqueue_set_status(batch_id)
+      batch = Batch.find(batch_id)
+      batch.status = Batch::STATUS_QUEUED_FOR_DELETION
+      batch.save
+    end
+
+  end
+end

data/app/jobs/ddr/batch/batch_objects_processor_job.rb

@@ -0,0 +1,11 @@
+module Ddr::Batch
+  class BatchObjectsProcessorJob
+    @queue = :batch
+
+    def self.perform(batch_object_ids, operator_id)
+      operator = User.find(operator_id)
+      ProcessBatchObjects.new(batch_object_ids: batch_object_ids, operator: operator).execute
+    end
+
+  end
+end

data/app/jobs/ddr/batch/batch_processor_job.rb

@@ -3,12 +3,7 @@ module Ddr::Batch
     @queue = :batch
 
     def self.perform(batch_id, operator_id)
-      ts = Time.now.strftime("%Y%m%d%H%M%S%L")
-      logfile = "batch_processor_#{ts}_log.txt"
-      batch = Batch.find(batch_id)
-      operator = User.find(operator_id)
-      bp = BatchProcessor.new(batch, operator, log_file: logfile)
-      bp.execute
+      ProcessBatch.new(batch_id: batch_id, operator_id: operator_id).execute
     end
 
     def self.after_enqueue_set_status(batch_id, operator_id)
@@ -18,4 +13,4 @@ module Ddr::Batch
     end
 
   end
-end
+end

data/app/mailers/ddr/batch/batch_processor_run_mailer.rb

@@ -6,7 +6,7 @@ module Ddr::Batch
 
     def send_notification(batch)
       @batch = batch
-      @title = "Batch Processor Run #{@batch.status}"
+      @title = "Batch Processor Run #{@batch.status} #{@batch.outcome}"
       @host = `uname -n`.strip
       @subject = "[#{@host}] #{@title}"
       from = "#{`echo $USER`.strip}@#{@host}"
@@ -16,4 +16,4 @@ module Ddr::Batch
 
   end
 
-end
+end

data/app/models/ddr/batch/batch.rb

@@ -19,30 +19,18 @@ module Ddr::Batch
     STATUS_FINISHED = "FINISHED"
     STATUS_INTERRUPTED = "INTERRUPTED"
     STATUS_RESTARTABLE = "INTERRUPTED - RESTARTABLE"
+    STATUS_QUEUED_FOR_DELETION = "QUEUED FOR DELETION"
+    STATUS_DELETING = "DELETING"
 
-    def validate
-      errors = []
-      begin
-        batch_objects.each do |object|
-          unless object.verified
-            errors << object.validate
-          end
-        end
-      rescue Exception => e
-        errors << "Exception raised during batch validation: #{e.backtrace}"
-      end
-      errors.flatten
-    end
-
-    def completed_count
-      batch_objects.where(verified: true).count
+    def handled_count
+      batch_objects.where(handled: true).count
     end
 
     def time_to_complete
-      unless processing_step_start.nil?
-        if completed_count > 0
-          completed = completed_count
-          ((Time.now - processing_step_start.to_time) / completed) * (batch_objects.count - completed)
+      unless start.nil?
+        if handled_count > 0
+          handled = handled_count
+          ((Time.now - start.to_time) / handled) * (batch_objects.count - handled)
         end
       end
     end
@@ -63,10 +51,14 @@ module Ddr::Batch
       batch_objects.map{ |x| x.pid if x.pid.present? }.compact
     end
 
+    def unhandled_objects?
+      batch_objects.any? { |batch_object| !batch_object.handled? }
+    end
+
     def finished?
       status == STATUS_FINISHED
     end
 
   end
 
-end
+end

data/app/models/ddr/batch/batch_object.rb

@@ -7,6 +7,7 @@ module Ddr::Batch
       belongs_to :batch, inverse_of: :batch_objects
       has_many :batch_object_attributes, -> { order "id ASC" }, inverse_of: :batch_object, dependent: :destroy
       has_many :batch_object_datastreams, inverse_of: :batch_object, dependent: :destroy
+      has_many :batch_object_messages, inverse_of: :batch_object, dependent: :destroy
       has_many :batch_object_relationships, inverse_of: :batch_object, dependent: :destroy
       has_many :batch_object_roles, inverse_of: :batch_object, dependent: :destroy
 
@@ -20,6 +21,8 @@ module Ddr::Batch
   Model: %{model}
       EOS
 
+      ProcessingResultsMessage = Struct.new(:level, :message)
+
       def self.pid_from_identifier(identifier, batch_id)
         query = "identifier = :identifier"
         query << " and batch_id = :batch_id" if batch_id
@@ -112,12 +115,11 @@ module Ddr::Batch
               obj_model = batch.found_pids[r[:object]]
             else
               begin
-                obj = ActiveFedora::Base.find(r[:object], :cast => true)
-                obj_model = obj.class.name
+                obj_model = SolrDocument.find(r[:object]).active_fedora_model
                 if batch.present?
-                  batch.add_found_pid(obj.pid, obj_model)
+                  batch.add_found_pid(r[:object], obj_model)
                 end
-              rescue ActiveFedora::ObjectNotFoundError
+              rescue SolrDocument::NotFound
                 pid_in_batch = false
                 if batch.present?
                   if batch.pre_assigned_pids.include?(r[:object])

data/app/models/ddr/batch/batch_object_message.rb

@@ -0,0 +1,8 @@
+module Ddr::Batch
+  class BatchObjectMessage < ActiveRecord::Base
+    belongs_to :batch_object, :inverse_of => :batch_object_messages
+
+    validates_presence_of :message
+
+  end
+end

data/app/models/ddr/batch/batch_object_relationship.rb

@@ -1,7 +1,6 @@
 module Ddr::Batch
 
   class BatchObjectRelationship < ActiveRecord::Base
-#    attr_accessible :name, :object, :object_type, :operation, :batch_object
     belongs_to :batch_object, :inverse_of => :batch_object_relationships
 
     RELATIONSHIP_ADMIN_POLICY = "admin_policy"

data/app/models/ddr/batch/error.rb

@@ -0,0 +1,10 @@
+module Ddr
+  module Batch
+    # Base class for custom exceptions
+    class Error < StandardError; end
+
+    # Error processing batch object
+    class BatchObjectProcessingError < Error; end
+
+  end
+end

data/app/models/ddr/batch/ingest_batch_object.rb

@@ -20,10 +20,11 @@ module Ddr::Batch
 
     def results_message
       if pid
-        verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
-        message = "Ingested #{model} #{identifier} into #{pid}...#{verification_result}"
+        message_level = verified ? Logger::INFO : Logger::WARN
+        verification_result = verified ? "Verified" : "VERIFICATION FAILURE"
+        ProcessingResultsMessage.new(message_level, "Ingested #{model} #{identifier} into #{pid}...#{verification_result}")
       else
-        message = "Attempt to ingest #{model} #{identifier} FAILED"
+        ProcessingResultsMessagemessage.new(Logger::ERROR, "Attempt to ingest #{model} #{identifier} FAILED")
       end
     end
 
@@ -95,30 +96,21 @@ module Ddr::Batch
       begin
         repo_object = model.constantize.new(:pid => repo_pid)
         repo_object.label = label if label
-        repo_object.save(validate: false)
         batch_object_attributes.each { |a| repo_object = add_attribute(repo_object, a) }
+        repo_object.save(validate: false)
         batch_object_datastreams.each { |d| repo_object = populate_datastream(repo_object, d) }
         batch_object_relationships.each { |r| repo_object = add_relationship(repo_object, r) }
         batch_object_roles.each { |r| repo_object = add_role(repo_object, r) }
-        repo_object.save! # Do not allow batch ingest to successfully create an invalid object
+        repo_object.save!
       rescue Exception => e1
         logger.fatal("Error in creating repository object #{repo_object.pid} for #{identifier} : #{e1}")
-        repo_clean = false
         if repo_object && !repo_object.new_record?
           begin
             logger.info("Deleting potentially incomplete #{repo_object.pid} due to error in ingest batch processing")
             repo_object.destroy
           rescue Exception => e2
             logger.fatal("Error deleting repository object #{repo_object.pid}: #{e2}")
-          else
-            repo_clean = true
           end
-        else
-          repo_clean = true
-        end
-        if batch.present?
-          batch.status = repo_clean ? Batch::STATUS_RESTARTABLE : Batch::STATUS_INTERRUPTED
-          batch.save
         end
         raise e1
       end

data/app/models/ddr/batch/log.rb

@@ -0,0 +1,29 @@
+module Ddr::Batch
+  class Log
+
+    DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
+
+    class << self
+
+      def logger(batch_id)
+        loggr = Logger.new(File.open(file_path(batch_id), File::WRONLY | File::APPEND | File::CREAT))
+        loggr.level = Ddr::Batch.processor_logging_level
+        loggr.datetime_format = "%Y-%m-%d %H:%M:%S.L"
+        loggr.formatter = proc do |severity, datetime, progname, msg|
+          "#{datetime} #{severity}: #{msg}\n"
+        end
+        loggr
+      end
+
+      def clear_log(batch_id)
+        log_file_path = file_path(batch_id)
+        FileUtils.remove(log_file_path) if File.exists?(log_file_path)
+      end
+
+      def file_path(batch_id)
+        File.join(DEFAULT_LOG_DIR, "batch_#{batch_id}_log.txt")
+      end
+
+    end
+  end
+end

data/app/models/ddr/batch/update_batch_object.rb

@@ -43,10 +43,11 @@ module Ddr::Batch
 
     def results_message
       if pid
-        verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
-        message = "Updated #{pid}...#{verification_result}"
+        message_level = verified ? Logger::INFO : Logger::WARN
+        verification_result = verified ? "Verified" : "VERIFICATION FAILURE"
+        ProcessingResultsMessage.new(message_level, "Updated #{pid}...#{verification_result}")
       else
-        message = "Attempt to update #{model} #{identifier} FAILED"
+        ProcessingResultsMessage.new(Logger::ERROR, "Attempt to update #{model} #{identifier} FAILED")
       end
     end
 
@@ -60,6 +61,7 @@ module Ddr::Batch
       repo_object = nil
       begin
         repo_object = ActiveFedora::Base.find(pid)
+        update!(model: repo_object.class.name) unless model.present?
         batch_object_attributes.each do |a|
           repo_object = case
           when a.operation.eql?(BatchObjectAttribute::OPERATION_ADD)
@@ -81,10 +83,6 @@ module Ddr::Batch
         end
       rescue Exception => e
         logger.error("Error in updating repository object #{pid} for #{identifier} : : #{e}")
-        if batch.present?
-          batch.status = Batch::STATUS_RESTARTABLE
-          batch.save
-        end
         raise e
       end
       repo_object

data/app/services/ddr/batch/monitor_batch_finished.rb

@@ -0,0 +1,84 @@
+module Ddr::Batch
+  class MonitorBatchFinished
+
+    class << self
+      def call(*args)
+        event = ActiveSupport::Notifications::Event.new(*args)
+        batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
+        batch_finished(batch)
+      end
+
+      private
+
+      def batch_finished(batch)
+        log_batch_finish(batch)
+        update_batch(batch)
+        send_notification(batch) if batch.user && batch.user.email
+      end
+
+      def log_batch_finish(batch)
+        logger = Ddr::Batch::Log.logger(batch.id)
+        logger.info "====== Summary ======"
+        results_tracker = results(batch)
+        results_tracker.keys.each do |type|
+          results_tracker[type].keys.each do |model|
+            log_result(results_tracker, type, model, logger)
+          end
+        end
+        logger.close
+      end
+
+      def results(batch)
+        results_tracker = Hash.new
+        batch.batch_objects.each do |batch_object|
+          track_result(results_tracker, batch_object)
+        end
+        results_tracker
+      end
+
+      def track_result(results_tracker, batch_object)
+        type, model = [ batch_object.type, batch_object.model ]
+        results_tracker[type] = Hash.new unless results_tracker.has_key?(type)
+        results_tracker[type][model] = Hash.new unless results_tracker[type].has_key?(model)
+        results_tracker[type][model][:successes] = 0 unless results_tracker[type][model].has_key?(:successes)
+        results_tracker[type][model][:successes] += 1 if batch_object.verified
+      end
+
+      def log_result(results_tracker, type, model, logger)
+        verb = type_verb(type)
+        count = results_tracker[type][model][:successes]
+        logger.info "#{verb} #{ActionController::Base.helpers.pluralize(count, model)}"
+      end
+
+      def type_verb(type)
+        case type
+          when IngestBatchObject.name
+            "Ingested"
+          when UpdateBatchObject.name
+            "Updated"
+        end
+      end
+
+      def update_batch(batch)
+        outcome = batch.success.eql?(batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
+        logfile = File.new(Ddr::Batch::Log.file_path(batch.id))
+        batch.update!(stop: DateTime.now,
+                      status: Batch::STATUS_FINISHED,
+                      outcome: outcome,
+                      logfile: logfile)
+      end
+
+      def send_notification(batch)
+        begin
+          BatchProcessorRunMailer.send_notification(batch).deliver!
+        rescue
+          Rails.logger.error("An error occurred while attempting to send a notification for batch #{batch.id}")
+        end
+      end
+    end
+
+  end
+end
+
+
+

data/app/services/ddr/batch/monitor_batch_object_handled.rb

@@ -0,0 +1,36 @@
+module Ddr::Batch
+  class MonitorBatchObjectHandled
+
+    class << self
+      def call(*args)
+        event = ActiveSupport::Notifications::Event.new(*args)
+        batch_object = BatchObject.find(event.payload[:batch_object_id])
+        batch = batch_object.batch
+        batch_object_handled(batch_object, batch)
+      end
+
+      private
+
+      def batch_object_handled(batch_object, batch)
+        log_batch_object_messages(batch_object, batch.id)
+        update_batch(batch_object, batch)
+        unless batch.unhandled_objects?
+          ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
+        end
+      end
+
+      def log_batch_object_messages(batch_object, batch_id)
+        logger = Ddr::Batch::Log.logger(batch_id)
+        batch_object.batch_object_messages.each do |message|
+          logger.add(message.level) { "Batch Object #{batch_object.id}: #{message.message}" }
+        end
+        logger.close
+      end
+
+      def update_batch(batch_object, batch)
+        batch_object.verified? ? batch.update!(success: batch.success + 1) : batch.update!(failure: batch.failure + 1)
+      end
+    end
+
+  end
+end

data/app/services/ddr/batch/monitor_batch_started.rb

@@ -0,0 +1,42 @@
+module Ddr::Batch
+  class MonitorBatchStarted
+
+    class << self
+      def call(*args)
+        event = ActiveSupport::Notifications::Event.new(*args)
+        batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
+        batch_started(batch)
+      end
+
+      private
+
+      def batch_started(batch)
+        clear_logs(batch)
+        log_batch_start(batch)
+        update_batch(batch)
+      end
+
+      def clear_logs(batch)
+        # delete any previously existing filesystem log file for this batch
+        Ddr::Batch::Log.clear_log(batch.id)
+        # remove any existing attached log file from the Batch ActiveRecord object
+        batch.logfile.clear
+      end
+
+      def log_batch_start(batch)
+        logger = Ddr::Batch::Log.logger(batch.id)
+        logger.info "Batch id: #{batch.id}"
+        logger.info "Batch name: #{batch.name}" if name
+        logger.info "Batch size: #{batch.batch_objects.size}"
+        logger.close
+      end
+
+      def update_batch(batch)
+        batch.update!(start: DateTime.now,
+                      status: Ddr::Batch::Batch::STATUS_RUNNING,
+                      version: VERSION)
+      end
+    end
+
+  end
+end

data/app/services/ddr/batch/process_batch.rb

@@ -0,0 +1,59 @@
+module Ddr::Batch
+  class ProcessBatch
+
+    attr_accessor :batch, :operator_id
+
+    def initialize(batch_id:, operator_id:)
+      @batch = Ddr::Batch::Batch.find(batch_id)
+      @operator_id = operator_id
+    end
+
+    def execute
+      ActiveSupport::Notifications.instrument('started.batch.batch.ddr', batch_id: batch.id)
+      batch.batch_objects.each do |batch_object|
+        case
+          when batch_object.is_a?(IngestBatchObject)
+            handle_ingest_batch_object(batch_object)
+          when batch_object.is_a?(UpdateBatchObject)
+            handle_update_batch_object(batch_object)
+        end
+      end
+    end
+
+    def handle_ingest_batch_object(batch_object)
+      case batch_object.model
+        when 'Collection'
+          ingest_collection_object(batch_object)
+        when 'Item'
+          enqueue_item_component_ingest(batch_object)
+        when 'Component'
+          # skip -- will be handled along with associated Item
+        when 'Target', 'Attachment'
+          Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
+      end
+    end
+
+    def handle_update_batch_object(batch_object)
+      Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
+    end
+
+    def ingest_collection_object(batch_object)
+      # Collection batch objects are processed synchronously because they need to exist in the repository
+      # prior to the processing of any objects (e.g., Item, Component, Target) associated with them.
+      # If the Collection batch object does not process successfully, consider the batch finished (albeit unsuccessfully)
+      # and raise an exception.
+      unless ProcessBatchObject.new(batch_object_id: batch_object.id, operator: User.find(operator_id)).execute
+        ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
+        raise Ddr::Batch::BatchObjectProcessingError, batch_object.id
+      end
+    end
+
+    def enqueue_item_component_ingest(batch_object)
+      batch_object_ids = [ batch_object.id ]
+      parent_rel_query = "object = '#{batch_object.pid}' AND name = '#{Ddr::Batch::BatchObjectRelationship::RELATIONSHIP_PARENT}'"
+      parent_rel_recs = Ddr::Batch::BatchObjectRelationship.where(parent_rel_query)
+      parent_rel_recs.each { |parent_rel_rec| batch_object_ids << parent_rel_rec.batch_object_id}
+      Resque.enqueue(BatchObjectsProcessorJob, batch_object_ids, operator_id)
+    end
+  end
+end

data/app/services/ddr/batch/process_batch_object.rb

@@ -0,0 +1,48 @@
+module Ddr::Batch
+  class ProcessBatchObject
+
+    attr_reader :batch_object_id, :operator
+
+    def initialize(batch_object_id:, operator:)
+      @batch_object_id = batch_object_id
+      @operator = operator
+    end
+
+    def execute
+      ActiveSupport::Notifications.instrument("handled.batchobject.batch.ddr",
+                                              batch_object_id: batch_object_id) do |payload|
+        batch_object = BatchObject.find(batch_object_id)
+        # Mark batch object as 'handled'
+        batch_object.update!(handled: true)
+        # Validate batch object
+        errors = batch_object.validate
+        # Process batch object or record validation errors
+        if errors.empty?
+          process(batch_object, operator)
+        else
+          record_errors(batch_object, errors)
+        end
+        # return true if batch_object was processed; otherwise, false
+        batch_object.processed? ? true : false
+      end
+    end
+
+    def process(batch_object, operator)
+      batch_object.update!(validated: true)
+      batch_object.process(operator)
+      batch_object.update!(processed: true)
+      results_message = batch_object.results_message
+      Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
+                                             level: results_message.level,
+                                             message: results_message.message)
+    end
+
+    def record_errors(batch_object, errors)
+      errors.each do |error|
+        Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
+                                               level: Logger::ERROR,
+                                               message: error)
+      end
+    end
+  end
+end

data/app/services/ddr/batch/process_batch_objects.rb

@@ -0,0 +1,35 @@
+module Ddr::Batch
+  class ProcessBatchObjects
+
+    attr_reader :batch_object_ids, :operator
+
+    def initialize(batch_object_ids:, operator:)
+      @batch_object_ids = batch_object_ids
+      @operator = operator
+    end
+
+    def execute
+      # Assume successful processing of all batch objects until proven otherwise.
+      success = true
+      batch_object_ids.each do |batch_object_id|
+        # Once any batch object included in this job fails to process successfully, do not attempt to process
+        # any remaining batch objects included in this job.  Instead, mark them as "handled" so the batch knows
+        # it's not waiting on them to be handled before it can consider itself "finished".
+        # The use case prompting this behavior is a job containing an Item ingest batch object plus one or more
+        # associated Component ingest batch objects.  If the Item batch object fails to process correctly, we don't
+        # want to attempt to process the Component batch objects.
+        # In the preceding use case, we could skip the remaining batch objects only if the failed batch object is an
+        # Item but there might be future cases in which we don't want to process the remaining batch objects in the
+        # job regardless of which batch object fails.  The failure of any batch object to process should be rare
+        # enough that it doesn't seem harmful to cover this potential broader use case in the current code.
+        if success
+          success = ProcessBatchObject.new(batch_object_id: batch_object_id, operator: operator).execute
+        else
+          batch_object = Ddr::Batch::BatchObject.find(batch_object_id)
+          batch_object.update!(handled: true)
+        end
+      end
+    end
+
+  end
+end

data/config/initializers/subscriptions.rb

@@ -0,0 +1,9 @@
+##
+## Subscriptions to ActiveSupport::Notifications instrumentation events
+##
+
+# Batch Processing events
+ActiveSupport::Notifications.subscribe('started.batch.batch.ddr', Ddr::Batch::MonitorBatchStarted)
+ActiveSupport::Notifications.subscribe('handled.batchobject.batch.ddr', Ddr::Batch::MonitorBatchObjectHandled)
+ActiveSupport::Notifications.subscribe('finished.batch.batch.ddr', Ddr::Batch::MonitorBatchFinished)
+

data/config/locales/en.yml

@@ -2,7 +2,7 @@ en:
     ddr:
         batch:
             errors:
-                prefix: "%{identifier} [Database ID: %{id}]:"
+                prefix: "%{identifier}:"
             no_batches: "No %{type} batches found for your user account."
             web:
                 action_names:
@@ -49,4 +49,4 @@ en:
                     finished_batches:
                         label: "Already Run"
                     pending_batches:
-                        label: "Pending"
+                        label: "Pending"

data/db/migrate/20161115191636_add_columns_to_batch_object.rb

@@ -0,0 +1,9 @@
+class AddColumnsToBatchObject < ActiveRecord::Migration
+  def change
+    change_table :batch_objects do |t|
+      t.boolean "handled", default: false
+      t.boolean "processed", default: false
+      t.boolean "validated", default: false
+    end
+  end
+end

data/db/migrate/20161116142512_create_batch_object_messages.rb

@@ -0,0 +1,13 @@
+class CreateBatchObjectMessages < ActiveRecord::Migration
+  def change
+    unless table_exists?(:batch_object_messages)
+      create_table :batch_object_messages do |t|
+        t.integer :batch_object_id
+        t.integer :level,   default: Logger::DEBUG
+        t.text    :message, limit: 65535
+
+        t.timestamps
+      end
+    end
+  end
+end

data/lib/ddr/batch.rb

@@ -9,6 +9,12 @@ module Ddr
     extend ActiveSupport::Autoload
 
     autoload :BatchUser
+    autoload :BatchObjectProcessingError, 'ddr/batch/error'
+
+    # Logging level for batch processing - defaults to Logger::INFO
+    mattr_accessor :processor_logging_level do
+      Logger::INFO
+    end
 
     def self.table_name_prefix
     end

data/lib/ddr/batch/version.rb

@@ -1,5 +1,5 @@
 module Ddr
   module Batch
-    VERSION = "1.1.0"
+    VERSION = "1.2.0.rc1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ddr-batch
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0.rc1
 platform: ruby
 authors:
 - Jim Coble
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-21 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -59,14 +59,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: log4r
   requirement: !ruby/object:Gem::Requirement
@@ -175,6 +175,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- app/jobs/ddr/batch/batch_deletion_job.rb
+- app/jobs/ddr/batch/batch_objects_processor_job.rb
 - app/jobs/ddr/batch/batch_processor_job.rb
 - app/mailers/ddr/batch/batch_processor_run_mailer.rb
 - app/models/ddr/batch/batch.rb
@@ -182,13 +184,22 @@ files:
 - app/models/ddr/batch/batch_object.rb
 - app/models/ddr/batch/batch_object_attribute.rb
 - app/models/ddr/batch/batch_object_datastream.rb
+- app/models/ddr/batch/batch_object_message.rb
 - app/models/ddr/batch/batch_object_relationship.rb
 - app/models/ddr/batch/batch_object_role.rb
+- app/models/ddr/batch/error.rb
 - app/models/ddr/batch/ingest_batch_object.rb
+- app/models/ddr/batch/log.rb
 - app/models/ddr/batch/update_batch_object.rb
-- app/scripts/ddr/batch/batch_processor.rb
+- app/services/ddr/batch/monitor_batch_finished.rb
+- app/services/ddr/batch/monitor_batch_object_handled.rb
+- app/services/ddr/batch/monitor_batch_started.rb
+- app/services/ddr/batch/process_batch.rb
+- app/services/ddr/batch/process_batch_object.rb
+- app/services/ddr/batch/process_batch_objects.rb
 - app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb
 - app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb
+- config/initializers/subscriptions.rb
 - config/locales/en.yml
 - config/routes.rb
 - db/migrate/20150828183839_create_batches.rb
@@ -197,6 +208,8 @@ files:
 - db/migrate/20150828202200_create_batch_object_datastreams.rb
 - db/migrate/20150828202240_create_batch_object_relationships.rb
 - db/migrate/20160816164010_create_batch_object_roles.rb
+- db/migrate/20161115191636_add_columns_to_batch_object.rb
+- db/migrate/20161116142512_create_batch_object_messages.rb
 - lib/ddr-batch.rb
 - lib/ddr/batch.rb
 - lib/ddr/batch/batch_user.rb
@@ -218,9 +231,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.3

data/app/scripts/ddr/batch/batch_processor.rb

@@ -1,152 +0,0 @@
-module Ddr::Batch
-  class BatchProcessor
-
-    LOG_CONFIG_FILEPATH = File.join(Rails.root, 'config', 'log4r_batch_processor.yml')
-    DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
-    DEFAULT_LOG_FILE = "batch_processor_log.txt"
-    PASS = "PASS"
-    FAIL = "FAIL"
-
-    # Options
-    #   :log_dir - optional - directory for log file - default is given in DEFAULT_LOG_DIR
-    #   :log_file - optional - filename of log file - default is given in DEFAULT_LOG_FILE
-    #   :skip_validation - optional - whether to skip batch object validation step when processing - default is false
-    #   :ignore_validation_errors - optional - whether to continue processing even if batch object validation errors occur - default is false
-    def initialize(batch, operator=nil, opts={})
-      @batch = batch
-      @operator = operator
-      @bp_log_dir = opts.fetch(:log_dir, DEFAULT_LOG_DIR)
-      @bp_log_file = opts.fetch(:log_file, DEFAULT_LOG_FILE)
-      @skip_validation = opts.fetch(:skip_validation, false)
-      @ignore_validation_errors = opts.fetch(:ignore_validation_errors, false)
-    end
-
-    def execute
-      config_logger
-      if @batch
-        initiate_batch_run
-        unless @skip_validation
-          valid_batch = validate_batch
-          @batch.update_attributes(status: Batch::STATUS_INVALID) unless valid_batch
-        end
-        if @skip_validation || @ignore_validation_errors || valid_batch
-          process_batch
-        end
-        close_batch_run
-      end
-      save_logfile
-      send_notification if @batch.user && @batch.user.email
-    end
-
-    private
-
-    def validate_batch
-      @batch.update_attributes(status: Batch::STATUS_VALIDATING)
-      valid = true
-      errors = @batch.validate
-      unless errors.empty?
-        valid = false
-        errors.each do |error|
-          message = "Batch Object Validation Error: #{error}"
-          @bp_log.error(message)
-        end
-      end
-      @batch.update_attributes(status: Batch::STATUS_RUNNING)
-      return valid
-    end
-
-    def process_batch
-      @batch.update_attributes(status: Batch::STATUS_PROCESSING, processing_step_start: DateTime.now)
-      @batch.batch_objects.each do |object|
-        begin
-          process_object(object)
-        rescue Exception => e
-          @bp_log.error(e.backtrace)
-          break
-        end
-        sleep 2
-      end
-      @batch.update_attributes(status: Batch::STATUS_RUNNING) if @batch.status == Batch::STATUS_PROCESSING
-    end
-
-    def initiate_batch_run
-      @bp_log.info "Batch id: #{@batch.id}"
-      @bp_log.info "Batch name: #{@batch.name}" if @batch.name
-      @bp_log.info "Batch size: #{@batch.batch_objects.size}"
-      @batch.logfile.clear  # clear out any attached logfile
-      @batch.update_attributes(:start => DateTime.now,
-                               :status => Batch::STATUS_RUNNING,
-                               :version => VERSION)
-      @failures = 0
-      @successes = 0
-      @results_tracker = Hash.new
-    end
-
-    def close_batch_run
-      @batch.reload
-      @batch.failure = @failures
-      @batch.outcome = @successes.eql?(@batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
-      if @batch.status.eql?(Batch::STATUS_RUNNING)
-        @batch.status = Batch::STATUS_FINISHED
-      end
-      @batch.stop = DateTime.now
-      @batch.success = @successes
-      @batch.save
-      @bp_log.info "====== Summary ======"
-      @results_tracker.keys.each do |type|
-        verb = case type
-        when IngestBatchObject.name
-          "Ingested"
-        when UpdateBatchObject.name
-          "Updated"
-        end
-        @results_tracker[type].keys.each do |model|
-          @bp_log.info "#{verb} #{@results_tracker[type][model][:successes]} #{model}"
-        end
-      end
-    end
-
-    def update_results_tracker(type, model, verified)
-      @results_tracker[type] = Hash.new unless @results_tracker.has_key?(type)
-      @results_tracker[type][model] = Hash.new unless @results_tracker[type].has_key?(model)
-      @results_tracker[type][model][:successes] = 0 unless @results_tracker[type][model].has_key?(:successes)
-      @results_tracker[type][model][:successes] += 1 if verified
-    end
-
-    def process_object(object)
-      @bp_log.debug "Processing object: #{object.identifier}"
-      repository_object = object.process(@operator)
-      update_results_tracker(object.type, repository_object.present? ? repository_object.class.name : object.model, object.verified)
-      if object.verified
-        @successes += 1
-      else
-        @failures += 1
-      end
-      message = object.results_message
-      @bp_log.info(message)
-    end
-
-    def config_logger
-      logconfig = Log4r::YamlConfigurator
-      logconfig['LOG_FILE'] = File.join(@bp_log_dir, @bp_log_file)
-      logconfig.load_yaml_file File.join(LOG_CONFIG_FILEPATH)
-      @bp_log = Log4r::Logger['batch_processor']
-    end
-
-    def save_logfile
-      @bp_log.outputters.each do |outputter|
-        @logfilename = outputter.filename if outputter.respond_to?(:filename)
-      end
-      @batch.update!({ logfile: File.new(@logfilename) }) if @logfilename
-    end
-
-    def send_notification
-      begin
-        BatchProcessorRunMailer.send_notification(@batch).deliver!
-      rescue
-        puts "An error occurred while attempting to send the notification."
-      end
-    end
-
-  end
-end