ddr-batch 1.7.2 → 2.0.0.alpha.1

data/app/models/ddr/batch/update_batch_object.rb

@@ -37,29 +37,16 @@ module Ddr::Batch
           verified = false if value.eql?(VERIFICATION_FAIL)
         end
         update_attributes(:verified => verified)
-        Ddr::Events::ValidationEvent.new.tap do |event|
-          event.object = repo_object
-          event.failure! unless verified
-          event.summary = EVENT_SUMMARY % {
-              label: "Object update validation",
-              batch_id: id,
-              identifier: identifier,
-              model: model
-          }
-          event.detail = verification_outcome_detail.join("\n")
-          event.save!
-        end
         repo_object
       end
     end
 
     def results_message
       if pid
-        message_level = verified ? Logger::INFO : Logger::WARN
-        verification_result = verified ? "Verified" : "VERIFICATION FAILURE"
-        ProcessingResultsMessage.new(message_level, "Updated #{pid}...#{verification_result}")
+        verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
+        message = "Updated #{pid}...#{verification_result}"
       else
-        ProcessingResultsMessage.new(Logger::ERROR, "Attempt to update #{model} #{identifier} FAILED")
+        message = "Attempt to update #{model} #{identifier} FAILED"
       end
     end
 
@@ -73,7 +60,6 @@ module Ddr::Batch
       repo_object = nil
       begin
         repo_object = ActiveFedora::Base.find(pid)
-        update!(model: repo_object.class.name) unless model.present?
         batch_object_attributes.each do |a|
           repo_object = case
           when a.operation.eql?(BatchObjectAttribute::OPERATION_ADD)
@@ -90,9 +76,15 @@ module Ddr::Batch
             populate_datastream(repo_object, d)
           end
         end
-        repo_object.save!(user: user, comment: event_log_comment)
+        if repo_object.save
+          repo_object.notify_event(:update, user: user, comment: event_log_comment)
+        end
       rescue Exception => e
         logger.error("Error in updating repository object #{pid} for #{identifier} : : #{e}")
+        if batch.present?
+          batch.status = Batch::STATUS_RESTARTABLE
+          batch.save
+        end
         raise e
       end
       repo_object

data/app/scripts/ddr/batch/batch_processor.rb

@@ -0,0 +1,152 @@
+module Ddr::Batch
+  class BatchProcessor
+
+    LOG_CONFIG_FILEPATH = File.join(Rails.root, 'config', 'log4r_batch_processor.yml')
+    DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
+    DEFAULT_LOG_FILE = "batch_processor_log.txt"
+    PASS = "PASS"
+    FAIL = "FAIL"
+
+    # Options
+    #   :log_dir - optional - directory for log file - default is given in DEFAULT_LOG_DIR
+    #   :log_file - optional - filename of log file - default is given in DEFAULT_LOG_FILE
+    #   :skip_validation - optional - whether to skip batch object validation step when processing - default is false
+    #   :ignore_validation_errors - optional - whether to continue processing even if batch object validation errors occur - default is false
+    def initialize(batch, operator=nil, opts={})
+      @batch = batch
+      @operator = operator
+      @bp_log_dir = opts.fetch(:log_dir, DEFAULT_LOG_DIR)
+      @bp_log_file = opts.fetch(:log_file, DEFAULT_LOG_FILE)
+      @skip_validation = opts.fetch(:skip_validation, false)
+      @ignore_validation_errors = opts.fetch(:ignore_validation_errors, false)
+    end
+
+    def execute
+      config_logger
+      if @batch
+        initiate_batch_run
+        unless @skip_validation
+          valid_batch = validate_batch
+          @batch.update_attributes(status: Batch::STATUS_INVALID) unless valid_batch
+        end
+        if @skip_validation || @ignore_validation_errors || valid_batch
+          process_batch
+        end
+        close_batch_run
+      end
+      save_logfile
+      send_notification if @batch.user && @batch.user.email
+    end
+
+    private
+
+    def validate_batch
+      @batch.update_attributes(status: Batch::STATUS_VALIDATING)
+      valid = true
+      errors = @batch.validate
+      unless errors.empty?
+        valid = false
+        errors.each do |error|
+          message = "Batch Object Validation Error: #{error}"
+          @bp_log.error(message)
+        end
+      end
+      @batch.update_attributes(status: Batch::STATUS_RUNNING)
+      return valid
+    end
+
+    def process_batch
+      @batch.update_attributes(status: Batch::STATUS_PROCESSING, processing_step_start: DateTime.now)
+      @batch.batch_objects.each do |object|
+        begin
+          process_object(object)
+        rescue Exception => e
+          @bp_log.error(e.backtrace)
+          break
+        end
+        sleep 2
+      end
+      @batch.update_attributes(status: Batch::STATUS_RUNNING) if @batch.status == Batch::STATUS_PROCESSING
+    end
+
+    def initiate_batch_run
+      @bp_log.info "Batch id: #{@batch.id}"
+      @bp_log.info "Batch name: #{@batch.name}" if @batch.name
+      @bp_log.info "Batch size: #{@batch.batch_objects.size}"
+      @batch.logfile.clear  # clear out any attached logfile
+      @batch.update_attributes(:start => DateTime.now,
+                               :status => Batch::STATUS_RUNNING,
+                               :version => VERSION)
+      @failures = 0
+      @successes = 0
+      @results_tracker = Hash.new
+    end
+
+    def close_batch_run
+      @batch.reload
+      @batch.failure = @failures
+      @batch.outcome = @successes.eql?(@batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
+      if @batch.status.eql?(Batch::STATUS_RUNNING)
+        @batch.status = Batch::STATUS_FINISHED
+      end
+      @batch.stop = DateTime.now
+      @batch.success = @successes
+      @batch.save
+      @bp_log.info "====== Summary ======"
+      @results_tracker.keys.each do |type|
+        verb = case type
+        when IngestBatchObject.name
+          "Ingested"
+        when UpdateBatchObject.name
+          "Updated"
+        end
+        @results_tracker[type].keys.each do |model|
+          @bp_log.info "#{verb} #{@results_tracker[type][model][:successes]} #{model}"
+        end
+      end
+    end
+
+    def update_results_tracker(type, model, verified)
+      @results_tracker[type] = Hash.new unless @results_tracker.has_key?(type)
+      @results_tracker[type][model] = Hash.new unless @results_tracker[type].has_key?(model)
+      @results_tracker[type][model][:successes] = 0 unless @results_tracker[type][model].has_key?(:successes)
+      @results_tracker[type][model][:successes] += 1 if verified
+    end
+
+    def process_object(object)
+      @bp_log.debug "Processing object: #{object.identifier}"
+      repository_object = object.process(@operator)
+      update_results_tracker(object.type, repository_object.present? ? repository_object.class.name : object.model, object.verified)
+      if object.verified
+        @successes += 1
+      else
+        @failures += 1
+      end
+      message = object.results_message
+      @bp_log.info(message)
+    end
+
+    def config_logger
+      logconfig = Log4r::YamlConfigurator
+      logconfig['LOG_FILE'] = File.join(@bp_log_dir, @bp_log_file)
+      logconfig.load_yaml_file File.join(LOG_CONFIG_FILEPATH)
+      @bp_log = Log4r::Logger['batch_processor']
+    end
+
+    def save_logfile
+      @bp_log.outputters.each do |outputter|
+        @logfilename = outputter.filename if outputter.respond_to?(:filename)
+      end
+      @batch.update!({ logfile: File.new(@logfilename) }) if @logfilename
+    end
+
+    def send_notification
+      begin
+        BatchProcessorRunMailer.send_notification(@batch).deliver!
+      rescue
+        puts "An error occurred while attempting to send the notification."
+      end
+    end
+
+  end
+end

data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb

@@ -22,13 +22,13 @@ td { text-align: right; }
 	  Outcome: <%= @batch.outcome %>
 	</p>
 	<p>
-	  Objects in batch: <%= @size %><br />
-	  Successfully processed: <%= @success %><br />
-	  Processing failures: <%= @handled - @success %><br />
-	  Unprocessed: <%= @size - @handled %>
+	  Objects in batch: <%= @batch.batch_objects.size %><br />
+	  Successfully processed: <%= @batch.success %><br />
+	  Processing failures: <%= @batch.failure %><br />
+	  Unprocessed: <%= @batch.batch_objects.size - @batch.success - @batch.failure %>
 	</p>
 	<p>
 	  See attachment for details of batch run.
 	</p>
   </body>
-</html>
+</html>

data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb

@@ -12,9 +12,9 @@ Stopped at: <%= @batch.stop %>
 Status: <%= @batch.status %>
 Outcome: <%= @batch.outcome %>
 
-Objects in batch: <%= @size %>
-Successfully processed: <%= @success %>
-Processing failures: <%= @handled - @success %>
-Unprocessed: <%= @size - @handled %>
+Objects in batch: <%= @batch.batch_objects.size %>
+Successfully processed: <%= @batch.success %>
+Processing failures: <%= @batch.failure %>
+Unprocessed: <%= @batch.batch_objects.size - @batch.success - @batch.failure %>
 
-See attachment for details of batch run.
+See attachment for details of batch run.

data/config/locales/en.yml

@@ -2,8 +2,7 @@ en:
     ddr:
         batch:
             errors:
-                prefix: "%{identifier}:"
-                batch_object_processing: "Error processing batch object: %{error_msg}"
+                prefix: "%{identifier} [Database ID: %{id}]:"
             no_batches: "No %{type} batches found for your user account."
             web:
                 action_names:
@@ -50,4 +49,4 @@ en:
                     finished_batches:
                         label: "Already Run"
                     pending_batches:
-                        label: "Pending"
+                        label: "Pending"

data/lib/ddr/batch.rb

@@ -9,12 +9,6 @@ module Ddr
     extend ActiveSupport::Autoload
 
     autoload :BatchUser
-    autoload :BatchObjectProcessingError, 'ddr/batch/error'
-
-    # Logging level for batch processing - defaults to Logger::INFO
-    mattr_accessor :processor_logging_level do
-      Logger::INFO
-    end
 
     def self.table_name_prefix
     end

data/lib/ddr/batch/version.rb

@@ -1,5 +1,5 @@
 module Ddr
   module Batch
-    VERSION = "1.7.2"
+    VERSION = "2.0.0.alpha.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ddr-batch
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 2.0.0.alpha.1
 platform: ruby
 authors:
 - Jim Coble
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-31 00:00:00.000000000 Z
+date: 2015-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.1.13
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: 4.1.13
 - !ruby/object:Gem::Dependency
   name: devise
   requirement: !ruby/object:Gem::Requirement
@@ -45,28 +45,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: 4.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: 4.2.0
 - !ruby/object:Gem::Dependency
   name: ddr-models
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.11'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -129,14 +143,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
@@ -151,20 +165,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: byebug
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Batch processing for Duke Digital Repository
 email:
 - lib-drs@duke.edu
@@ -175,30 +175,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- app/jobs/ddr/batch/batch_deletion_job.rb
-- app/jobs/ddr/batch/batch_objects_processor_job.rb
 - app/jobs/ddr/batch/batch_processor_job.rb
 - app/mailers/ddr/batch/batch_processor_run_mailer.rb
 - app/models/ddr/batch/batch.rb
+- app/models/ddr/batch/batch_ability_definitions.rb
 - app/models/ddr/batch/batch_object.rb
 - app/models/ddr/batch/batch_object_attribute.rb
 - app/models/ddr/batch/batch_object_datastream.rb
-- app/models/ddr/batch/batch_object_message.rb
 - app/models/ddr/batch/batch_object_relationship.rb
-- app/models/ddr/batch/batch_object_role.rb
-- app/models/ddr/batch/error.rb
 - app/models/ddr/batch/ingest_batch_object.rb
-- app/models/ddr/batch/log.rb
 - app/models/ddr/batch/update_batch_object.rb
-- app/services/ddr/batch/monitor_batch_finished.rb
-- app/services/ddr/batch/monitor_batch_object_handled.rb
-- app/services/ddr/batch/monitor_batch_started.rb
-- app/services/ddr/batch/process_batch.rb
-- app/services/ddr/batch/process_batch_object.rb
-- app/services/ddr/batch/process_batch_objects.rb
+- app/scripts/ddr/batch/batch_processor.rb
 - app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb
 - app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb
-- config/initializers/subscriptions.rb
 - config/locales/en.yml
 - config/routes.rb
 - db/migrate/20150828183839_create_batches.rb
@@ -206,11 +195,6 @@ files:
 - db/migrate/20150828202118_create_batch_object_attributes.rb
 - db/migrate/20150828202200_create_batch_object_datastreams.rb
 - db/migrate/20150828202240_create_batch_object_relationships.rb
-- db/migrate/20160816164010_create_batch_object_roles.rb
-- db/migrate/20161115191636_add_columns_to_batch_object.rb
-- db/migrate/20161116142512_create_batch_object_messages.rb
-- db/migrate/20161222192611_remove_columns_from_batch.rb
-- db/migrate/20171116183514_add_collection_columns_to_batch.rb
 - lib/ddr-batch.rb
 - lib/ddr/batch.rb
 - lib/ddr/batch/batch_user.rb
@@ -232,12 +216,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Batch processing for Duke Digital Repository