ddr-batch 1.0.0.rc1

data/app/models/ddr/batch/batch_object_attribute.rb

@@ -0,0 +1,57 @@
+module Ddr::Batch
+
+  class BatchObjectAttribute < ActiveRecord::Base
+    belongs_to :batch_object, :inverse_of => :batch_object_attributes
+
+    OPERATION_ADD = "ADD"              # Add the provided value to the attribute
+    OPERATION_DELETE = "DELETE"        # Delete the provided value from the attribute
+    OPERATION_CLEAR = "CLEAR"          # Clear all values from the attribute
+    OPERATION_CLEAR_ALL = "CLEAR_ALL"  # Clear all attributes in the datastream
+
+    OPERATIONS = [ OPERATION_ADD, OPERATION_DELETE, OPERATION_CLEAR, OPERATION_CLEAR_ALL ]
+
+    VALUE_TYPE_STRING = "STRING"
+
+    VALUE_TYPES = [ VALUE_TYPE_STRING ]
+
+    validates :operation, inclusion: { in: OPERATIONS }
+    validates :datastream, presence: true
+    with_options if: :operation_requires_name? do |obj|
+      obj.validates :name, presence: true
+    end
+    validate :valid_datastream_and_attribute_name, if: [ 'batch_object.model', 'datastream', 'name' ]
+    with_options if: :operation_requires_value? do |obj|
+      obj.validates :value, presence: true
+      obj.validates :value_type, inclusion: { in: VALUE_TYPES }
+    end
+
+    def operation_requires_name?
+      [ OPERATION_ADD, OPERATION_DELETE, OPERATION_CLEAR ].include? operation
+    end
+
+    def operation_requires_value?
+      [ OPERATION_ADD, OPERATION_DELETE ].include? operation
+    end
+
+    def valid_datastream_and_attribute_name
+      if datastream_valid?
+        errors.add(:name, "is not valid") unless attribute_name_valid?
+      else
+        errors.add(:datastream, "is not valid")
+      end
+    end
+
+    def datastream_type
+      batch_object.model.constantize.ds_specs[datastream][:type] rescue nil
+    end
+
+    def datastream_valid?
+        datastream_type < ActiveFedora::RDFDatastream rescue false
+    end
+
+    def attribute_name_valid?
+      datastream_type.term_names.include?(name.to_sym)
+    end
+
+  end
+end

data/app/models/ddr/batch/batch_object_datastream.rb

@@ -0,0 +1,23 @@
+module Ddr::Batch
+
+  class BatchObjectDatastream < ActiveRecord::Base
+
+    belongs_to :batch_object, :inverse_of => :batch_object_datastreams
+
+    DATASTREAMS = [ Ddr::Datastreams::CONTENT,
+                    Ddr::Datastreams::DESC_METADATA,
+                    Ddr::Datastreams::RIGHTS_METADATA,
+                    Ddr::Datastreams::STRUCT_METADATA ]
+
+    OPERATION_ADD = "ADD" # add this datastream to the object -- considered an error if datastream already exists
+    OPERATION_ADDUPDATE = "ADDUPDATE" # add this datastream to or update this datastream in the object
+    OPERATION_UPDATE = "UPDATE" # update this datastream in the object -- considered an error if datastream does not already exist
+    OPERATION_DELETE = "DELETE" # delete this datastream from the object -- considered an error if datastream does not exist
+
+    PAYLOAD_TYPE_BYTES = "BYTES"
+    PAYLOAD_TYPE_FILENAME = "FILENAME"
+
+    PAYLOAD_TYPES = [ PAYLOAD_TYPE_BYTES, PAYLOAD_TYPE_FILENAME ]
+  end
+
+end

data/app/models/ddr/batch/batch_object_relationship.rb

@@ -0,0 +1,26 @@
+module Ddr::Batch
+
+  class BatchObjectRelationship < ActiveRecord::Base
+#    attr_accessible :name, :object, :object_type, :operation, :batch_object
+    belongs_to :batch_object, :inverse_of => :batch_object_relationships
+
+    RELATIONSHIP_ADMIN_POLICY = "admin_policy"
+    RELATIONSHIP_COLLECTION = "collection"
+    RELATIONSHIP_PARENT = "parent"
+    RELATIONSHIP_ITEM = "item"
+    RELATIONSHIP_COMPONENT = "component"
+    RELATIONSHIP_ATTACHED_TO = "attached_to"
+
+    RELATIONSHIPS = [ RELATIONSHIP_ADMIN_POLICY, RELATIONSHIP_COLLECTION, RELATIONSHIP_PARENT, RELATIONSHIP_ITEM,
+      RELATIONSHIP_COMPONENT, RELATIONSHIP_ATTACHED_TO ]
+
+    OPERATION_ADD = "ADD"
+    OPERATION_UPDATE = "UPDATE"
+    OPERATION_DELETE = "DELETE"
+
+    OBJECT_TYPE_PID = "PID"
+
+    OBJECT_TYPES = [ OBJECT_TYPE_PID ]
+  end
+
+end

data/app/models/ddr/batch/ingest_batch_object.rb

@@ -0,0 +1,118 @@
+module Ddr::Batch
+
+  class IngestBatchObject < BatchObject
+
+    def local_validations
+      errors = []
+      errors << "#{@error_prefix} Model required for INGEST operation" unless model
+      errors += validate_pre_assigned_pid if pid
+      errors
+    end
+
+    def model_datastream_keys
+      model.constantize.new.datastreams.keys
+    end
+
+    def process(user, opts = {})
+      ingest(user, opts) unless verified
+    end
+
+    def results_message
+      if pid
+        verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
+        message = "Ingested #{model} #{identifier} into #{pid}...#{verification_result}"
+      else
+        message = "Attempt to ingest #{model} #{identifier} FAILED"
+      end
+    end
+
+    private
+
+    def validate_pre_assigned_pid
+      errs = []
+      errs << "#{@error_prefix} #{pid} already exists in repository" if ActiveFedora::Base.exists?(pid)
+      return errs
+    end
+
+    def ingest(user, opts = {})
+      repo_object = create_repository_object
+      if !repo_object.nil? && !repo_object.new_record?
+        ingest_outcome_detail = []
+        ingest_outcome_detail << "Ingested #{model} #{identifier} into #{repo_object.pid}"
+        Ddr::Events::IngestionEvent.new.tap do |event|
+          event.object = repo_object
+          event.user = user
+          event.summary = EVENT_SUMMARY % {
+            :label => "Object ingestion",
+            :batch_id => id,
+            :identifier => identifier,
+            :model => model
+          }
+          event.detail = ingest_outcome_detail.join("\n")
+          event.save!
+        end
+        update_attributes(:pid => repo_object.pid)
+        verifications = verify_repository_object
+        verification_outcome_detail = []
+        verified = true
+        verifications.each do |key, value|
+          verification_outcome_detail << "#{key}...#{value}"
+          verified = false if value.eql?(VERIFICATION_FAIL)
+        end
+        update_attributes(:verified => verified)
+        Ddr::Events::ValidationEvent.new.tap do |event|
+          event.object = repo_object
+          event.failure! unless verified
+          event.summary = EVENT_SUMMARY % {
+            :label => "Object ingestion validation",
+            :batch_id => id,
+            :identifier => identifier,
+            :model => model
+          }
+          event.detail = verification_outcome_detail.join("\n")
+          event.save!
+        end
+      else
+        verifications = nil
+      end
+      repo_object
+    end
+
+    def create_repository_object
+      repo_pid = pid if pid.present?
+      repo_object = nil
+      begin
+        repo_object = model.constantize.new(:pid => repo_pid)
+        repo_object.label = label if label
+        repo_object.save(validate: false)
+        batch_object_attributes.each { |a| repo_object = add_attribute(repo_object, a) }
+        batch_object_datastreams.each { |d| repo_object = populate_datastream(repo_object, d) }
+        batch_object_relationships.each { |r| repo_object = add_relationship(repo_object, r) }
+        repo_object.save
+      rescue Exception => e1
+        logger.fatal("Error in creating repository object #{repo_object.pid} for #{identifier} : #{e1}")
+        repo_clean = false
+        if repo_object && !repo_object.new_record?
+          begin
+            logger.info("Deleting potentially incomplete #{repo_object.pid} due to error in ingest batch processing")
+            repo_object.destroy
+          rescue Exception => e2
+            logger.fatal("Error deleting repository object #{repo_object.pid}: #{e2}")
+          else
+            repo_clean = true
+          end
+        else
+          repo_clean = true
+        end
+        if batch.present?
+          batch.status = repo_clean ? Batch::STATUS_RESTARTABLE : Batch::STATUS_INTERRUPTED
+          batch.save
+        end
+        raise e1
+      end
+      repo_object
+    end
+
+  end
+
+end

data/app/models/ddr/batch/update_batch_object.rb

@@ -0,0 +1,94 @@
+module Ddr::Batch
+
+  class UpdateBatchObject < BatchObject
+
+    def local_validations
+      errs = []
+      errs << "#{@error_prefix} PID required for UPDATE operation" unless pid
+      if pid
+        if ActiveFedora::Base.exists?(pid)
+          errs << "#{@error_prefix} #{batch.user.user_key} not permitted to edit #{pid}" unless batch.user.can?(:edit, ActiveFedora::Base.find(pid, :cast => true))
+        else
+          errs << "#{@error_prefix} PID #{pid} not found in repository" unless ActiveFedora::Base.exists?(pid)
+        end
+      end
+      errs
+    end
+
+    def model_datastream_keys
+      if pid
+        begin
+          obj = ActiveFedora::Base.find(pid, :cast => true)
+          obj.datastreams.keys
+        rescue
+          nil
+        end
+      end
+    end
+
+    def process(user, opts = {})
+      unless verified
+        repo_object = update_repository_object(user, opts)
+        verifications = verify_repository_object
+        verification_outcome_detail = []
+        verified = true
+        verifications.each do |key, value|
+          verification_outcome_detail << "#{key}...#{value}"
+          verified = false if value.eql?(VERIFICATION_FAIL)
+        end
+        update_attributes(:verified => verified)
+        repo_object
+      end
+    end
+
+    def results_message
+      if pid
+        verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
+        message = "Updated #{pid}...#{verification_result}"
+      else
+        message = "Attempt to update #{model} #{identifier} FAILED"
+      end
+    end
+
+    def event_log_comment
+      "Updated by batch process (Batch #{batch.id}, BatchObject #{id})"
+    end
+
+    private
+
+    def update_repository_object(user, opts = {})
+      repo_object = nil
+      begin
+        repo_object = ActiveFedora::Base.find(pid)
+        batch_object_attributes.each do |a|
+          repo_object = case
+          when a.operation.eql?(BatchObjectAttribute::OPERATION_ADD)
+            add_attribute(repo_object, a)
+          when a.operation.eql?(BatchObjectAttribute::OPERATION_CLEAR)
+            clear_attribute(repo_object, a)
+          when a.operation.eql?(BatchObjectAttribute::OPERATION_CLEAR_ALL)
+            clear_attributes(repo_object, a)
+          end
+        end
+        batch_object_datastreams.each do |d|
+          repo_object = case
+          when d.operation.eql?(BatchObjectDatastream::OPERATION_ADDUPDATE)
+            populate_datastream(repo_object, d)
+          end
+        end
+        if repo_object.save
+          repo_object.notify_event(:update, user: user, comment: event_log_comment)
+        end
+      rescue Exception => e
+        logger.error("Error in updating repository object #{pid} for #{identifier} : : #{e}")
+        if batch.present?
+          batch.status = Batch::STATUS_RESTARTABLE
+          batch.save
+        end
+        raise e
+      end
+      repo_object
+    end
+
+  end
+end

data/app/scripts/ddr/batch/batch_processor.rb

@@ -0,0 +1,151 @@
+module Ddr::Batch
+  class BatchProcessor
+
+    LOG_CONFIG_FILEPATH = File.join(Rails.root, 'config', 'log4r_batch_processor.yml')
+    DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
+    DEFAULT_LOG_FILE = "batch_processor_log.txt"
+    PASS = "PASS"
+    FAIL = "FAIL"
+
+    # Options
+    #   :log_dir - optional - directory for log file - default is given in DEFAULT_LOG_DIR
+    #   :log_file - optional - filename of log file - default is given in DEFAULT_LOG_FILE
+    #   :skip_validation - optional - whether to skip batch object validation step when processing - default is false
+    #   :ignore_validation_errors - optional - whether to continue processing even if batch object validation errors occur - default is false
+    def initialize(batch, operator=nil, opts={})
+      @batch = batch
+      @operator = operator
+      @bp_log_dir = opts.fetch(:log_dir, DEFAULT_LOG_DIR)
+      @bp_log_file = opts.fetch(:log_file, DEFAULT_LOG_FILE)
+      @skip_validation = opts.fetch(:skip_validation, false)
+      @ignore_validation_errors = opts.fetch(:ignore_validation_errors, false)
+    end
+
+    def execute
+      config_logger
+      if @batch
+        initiate_batch_run
+        unless @skip_validation
+          valid_batch = validate_batch
+          @batch.update_attributes(status: Batch::STATUS_INVALID) unless valid_batch
+        end
+        if @skip_validation || @ignore_validation_errors || valid_batch
+          process_batch
+        end
+        close_batch_run
+      end
+      save_logfile
+      send_notification if @batch.user && @batch.user.email
+    end
+
+    private
+
+    def validate_batch
+      @batch.update_attributes(status: Batch::STATUS_VALIDATING)
+      valid = true
+      errors = @batch.validate
+      unless errors.empty?
+        valid = false
+        errors.each do |error|
+          message = "Batch Object Validation Error: #{error}"
+          @bp_log.error(message)
+        end
+      end
+      @batch.update_attributes(status: Batch::STATUS_RUNNING)
+      return valid
+    end
+
+    def process_batch
+      @batch.update_attributes(status: Batch::STATUS_PROCESSING, processing_step_start: DateTime.now)
+      @batch.batch_objects.each do |object|
+        begin
+          process_object(object)
+        rescue Exception => e
+          break
+        end
+        sleep 2
+      end
+      @batch.update_attributes(status: Batch::STATUS_RUNNING) if @batch.status == Batch::STATUS_PROCESSING
+    end
+
+    def initiate_batch_run
+      @bp_log.info "Batch id: #{@batch.id}"
+      @bp_log.info "Batch name: #{@batch.name}" if @batch.name
+      @bp_log.info "Batch size: #{@batch.batch_objects.size}"
+      @batch.logfile.clear  # clear out any attached logfile
+      @batch.update_attributes(:start => DateTime.now,
+                               :status => Batch::STATUS_RUNNING,
+                               :version => VERSION)
+      @failures = 0
+      @successes = 0
+      @results_tracker = Hash.new
+    end
+
+    def close_batch_run
+      @batch.reload
+      @batch.failure = @failures
+      @batch.outcome = @successes.eql?(@batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
+      if @batch.status.eql?(Batch::STATUS_RUNNING)
+        @batch.status = Batch::STATUS_FINISHED
+      end
+      @batch.stop = DateTime.now
+      @batch.success = @successes
+      @batch.save
+      @bp_log.info "====== Summary ======"
+      @results_tracker.keys.each do |type|
+        verb = case type
+        when IngestBatchObject.name
+          "Ingested"
+        when UpdateBatchObject.name
+          "Updated"
+        end
+        @results_tracker[type].keys.each do |model|
+          @bp_log.info "#{verb} #{@results_tracker[type][model][:successes]} #{model}"
+        end
+      end
+    end
+
+    def update_results_tracker(type, model, verified)
+      @results_tracker[type] = Hash.new unless @results_tracker.has_key?(type)
+      @results_tracker[type][model] = Hash.new unless @results_tracker[type].has_key?(model)
+      @results_tracker[type][model][:successes] = 0 unless @results_tracker[type][model].has_key?(:successes)
+      @results_tracker[type][model][:successes] += 1 if verified
+    end
+
+    def process_object(object)
+      @bp_log.debug "Processing object: #{object.identifier}"
+      repository_object = object.process(@operator)
+      update_results_tracker(object.type, repository_object.present? ? repository_object.class.name : object.model, object.verified)
+      if object.verified
+        @successes += 1
+      else
+        @failures += 1
+      end
+      message = object.results_message
+      @bp_log.info(message)
+    end
+
+    def config_logger
+      logconfig = Log4r::YamlConfigurator
+      logconfig['LOG_FILE'] = File.join(@bp_log_dir, @bp_log_file)
+      logconfig.load_yaml_file File.join(LOG_CONFIG_FILEPATH)
+      @bp_log = Log4r::Logger['batch_processor']
+    end
+
+    def save_logfile
+      @bp_log.outputters.each do |outputter|
+        @logfilename = outputter.filename if outputter.respond_to?(:filename)
+      end
+      @batch.update!({ logfile: File.new(@logfilename) }) if @logfilename
+    end
+
+    def send_notification
+      begin
+        BatchProcessorRunMailer.send_notification(@batch).deliver!
+      rescue
+        puts "An error occurred while attempting to send the notification."
+      end
+    end
+
+  end
+end