ddr-batch 1.7.2 → 2.0.0.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e356a65a70813c599af04d01d764de1109d98f5e
-  data.tar.gz: 4ac1235826c5fce10d7990f16ad9636d771169ba
+  metadata.gz: f604d559b09d360d0f9357113a314fbbe33d5777
+  data.tar.gz: 4ed2ac035ff250220a8b7b07e6766f338a9da348
 SHA512:
-  metadata.gz: 10ea52dce76f86f9c9965ebe8d594f2a2c5c7ab3329cb7fe759d7fac9f6122adf84d929c350daab37334a86aa1088f48766c407e5541f00b76c5a405257380d3
-  data.tar.gz: f397545eb2d5741fbcb9ece062f0e4c1f82e8eec0e952314de3bdab6ec595e128d3cbdc56406a52b2d372ef86ee497481aeb93281ca3bcea359b6b9eb52cac63
+  metadata.gz: 8a495a43eb4b92a0753a1ac80ca16a08a40693f91de7928ff7a7de7c81dc6b8733c38063526f07cb027bc744adfca8bafd47b7bc9afc6dac33490859e56c51d6
+  data.tar.gz: 3c3abce0902a76666701f79c80f65fd7ef5d6b019cce4927a462a7dbb398b4c6218088584f5a82d3fc5aef4631485c8b542b5c5faf0612fddc634ed4e9fac0e4

data/README.md

@@ -42,6 +42,51 @@ class Ability < Ddr::Auth::Ability
 end
 ```
 
+### Log4r
+
+#### Application.rb
+
+Add the following lines to `config\application.rb` if they are not already there:
+
+```ruby
+require 'log4r'
+require 'log4r/yamlconfigurator'
+require 'log4r/outputter/datefileoutputter'
+include Log4r
+```
+
+#### Configuration
+
+`Ddr::Batch::BatchProcessor` expects a Log4r configuration file at `config\log4r_batch_processor.yml`.
+
+##### Example
+
+```yaml
+log4r_config:
+    loggers:
+        - name      : batch_processor
+          level     : DEBUG
+          trace     : 'false'
+          outputters:
+            - logfile
+    outputters:
+        - type      : StdoutOutputter
+          name      : stdout
+          level     : DEBUG
+          formatter :
+            date_pattern: '%F %T.%L'
+            pattern     : '%d %l: %m'
+            type        : PatternFormatter
+        - type        : FileOutputter
+          name        : logfile
+          trunc       : 'false'
+          filename    : "#{LOG_FILE}"
+          formatter   :
+            date_pattern: '%F %T.%L'
+            pattern     : '%d %l: %m'
+            type        : PatternFormatter
+```
+
 ### Migrations
 
 Install the ddr-batch migrations:

data/app/jobs/ddr/batch/batch_processor_job.rb

@@ -3,7 +3,12 @@ module Ddr::Batch
     @queue = :batch
 
     def self.perform(batch_id, operator_id)
-      ProcessBatch.new(batch_id: batch_id, operator_id: operator_id).execute
+      ts = Time.now.strftime("%Y%m%d%H%M%S%L")
+      logfile = "batch_processor_#{ts}_log.txt"
+      batch = Batch.find(batch_id)
+      operator = User.find(operator_id)
+      bp = BatchProcessor.new(batch, operator, log_file: logfile)
+      bp.execute
     end
 
     def self.after_enqueue_set_status(batch_id, operator_id)
@@ -13,4 +18,4 @@ module Ddr::Batch
     end
 
   end
-end
+end

data/app/mailers/ddr/batch/batch_processor_run_mailer.rb

@@ -2,35 +2,18 @@ module Ddr::Batch
 
   class BatchProcessorRunMailer < ActionMailer::Base
 
+    default :from => "noreply@duke.edu"
+
     def send_notification(batch)
       @batch = batch
-      @title = "Batch Processor Run #{@batch.status} #{@batch.outcome}"
-      @title << " - #{@batch.collection_title}" if @batch.collection_title.present?
+      @title = "Batch Processor Run #{@batch.status}"
       @host = `uname -n`.strip
       @subject = "[#{@host}] #{@title}"
-      @size = @batch.batch_objects.size
-      @handled = @batch.handled_count
-      @success = @batch.success_count
-      attachments[attachment_file_name(@batch)] = File.read(@batch.logfile.path)
-      mail(to: @batch.user.email, subject: @subject)
+      from = "#{`echo $USER`.strip}@#{@host}"
+      attachments["details.txt"] = File.read(@batch.logfile.path)
+      mail(from: from, to: @batch.user.email, subject: @subject)
     end
 
-    private
-
-    def attachment_file_name(batch)
-      if batch.collection_title.present?
-        sanitized_title = sanitize_title_for_filename(batch.collection_title)
-        "details_#{sanitized_title}.txt"
-      else
-        "details.txt"
-      end
-    end
-
-    def sanitize_title_for_filename(title)
-      title
-          .gsub(/[^\w\s_-]+/, '')
-          .gsub(/\s+/, '_')
-    end
   end
 
-end
+end

data/app/models/ddr/batch/batch.rb

@@ -19,22 +19,30 @@ module Ddr::Batch
     STATUS_FINISHED = "FINISHED"
     STATUS_INTERRUPTED = "INTERRUPTED"
     STATUS_RESTARTABLE = "INTERRUPTED - RESTARTABLE"
-    STATUS_QUEUED_FOR_DELETION = "QUEUED FOR DELETION"
-    STATUS_DELETING = "DELETING"
 
-    def handled_count
-      batch_objects.where(handled: true).count
+    def validate
+      errors = []
+      begin
+        batch_objects.each do |object|
+          unless object.verified
+            errors << object.validate
+          end
+        end
+      rescue Exception => e
+        errors << "Exception raised during batch validation: #{e.backtrace}"
+      end
+      errors.flatten
     end
 
-    def success_count
+    def completed_count
       batch_objects.where(verified: true).count
     end
 
     def time_to_complete
-      unless start.nil?
-        if handled_count > 0
-          handled = handled_count
-          ((Time.now - start.to_time) / handled) * (batch_objects.count - handled)
+      unless processing_step_start.nil?
+        if completed_count > 0
+          completed = completed_count
+          ((Time.now - processing_step_start.to_time) / completed) * (batch_objects.count - completed)
         end
       end
     end
@@ -55,20 +63,10 @@ module Ddr::Batch
       batch_objects.map{ |x| x.pid if x.pid.present? }.compact
     end
 
-    def unhandled_objects?
-      batch_objects.any? { |batch_object| !batch_object.handled? }
-    end
-
     def finished?
       status == STATUS_FINISHED
     end
 
-    def deletable?
-      [ nil,
-        Ddr::Batch::Batch::STATUS_READY,
-        Ddr::Batch::Batch::STATUS_VALIDATED,
-        Ddr::Batch::Batch::STATUS_INVALID ].include?(status)
-    end
   end
 
-end
+end

data/app/models/ddr/batch/batch_ability_definitions.rb

@@ -0,0 +1,14 @@
+module Ddr::Batch
+  class BatchAbilityDefinitions < Ddr::Auth::AbilityDefinitions
+
+    def call
+      if authenticated?
+        can :manage, Batch, user_id: user.id
+      end
+      can :manage, Ddr::Batch::BatchObject do |batch_object|
+        can? :manage, batch_object.batch
+      end
+    end
+
+  end
+end

data/app/models/ddr/batch/batch_object.rb

@@ -7,9 +7,7 @@ module Ddr::Batch
       belongs_to :batch, inverse_of: :batch_objects
       has_many :batch_object_attributes, -> { order "id ASC" }, inverse_of: :batch_object, dependent: :destroy
       has_many :batch_object_datastreams, inverse_of: :batch_object, dependent: :destroy
-      has_many :batch_object_messages, inverse_of: :batch_object, dependent: :destroy
       has_many :batch_object_relationships, inverse_of: :batch_object, dependent: :destroy
-      has_many :batch_object_roles, inverse_of: :batch_object, dependent: :destroy
 
       VERIFICATION_PASS = "PASS"
       VERIFICATION_FAIL = "FAIL"
@@ -21,8 +19,6 @@ module Ddr::Batch
   Model: %{model}
       EOS
 
-      ProcessingResultsMessage = Struct.new(:level, :message)
-
       def self.pid_from_identifier(identifier, batch_id)
         query = "identifier = :identifier"
         query << " and batch_id = :batch_id" if batch_id
@@ -79,7 +75,7 @@ module Ddr::Batch
         errs = []
         batch_object_datastreams.each do |d|
           if model_datastream_keys.present?
-            unless model_datastream_keys.include?(d.name)
+            unless model_datastream_keys.include?(d.name.to_sym)
               errs << "#{@error_prefix} Invalid datastream name for #{model}: #{d.name}"
             end
           end
@@ -115,11 +111,12 @@ module Ddr::Batch
               obj_model = batch.found_pids[r[:object]]
             else
               begin
-                obj_model = SolrDocument.find(r[:object]).active_fedora_model
+                obj = ActiveFedora::Base.find(r[:object], :cast => true)
+                obj_model = obj.class.name
                 if batch.present?
-                  batch.add_found_pid(r[:object], obj_model)
+                  batch.add_found_pid(obj.pid, obj_model)
                 end
-              rescue SolrDocument::NotFound
+              rescue ActiveFedora::ObjectNotFoundError
                 pid_in_batch = false
                 if batch.present?
                   if batch.pre_assigned_pids.include?(r[:object])
@@ -156,7 +153,7 @@ module Ddr::Batch
         else
           verifications["Object exists in repository"] = VERIFICATION_PASS
           verifications["Object is correct model"] = verify_model(repo_object) if model
-          verifications["Object has correct label"] = verify_label(repo_object) if label
+          # verifications["Object has correct label"] = verify_label(repo_object) if label
           unless batch_object_attributes.empty?
             batch_object_attributes.each do |a|
               if a.operation == BatchObjectAttribute::OPERATION_ADD
@@ -175,11 +172,6 @@ module Ddr::Batch
               verifications["#{r.name} relationship is correct"] = verify_relationship(repo_object, r)
             end
           end
-          unless batch_object_roles.empty?
-            batch_object_roles.each do |r|
-              verifications["#{r.role_scope} #{r.role_type} #{r.agent} role is correct"] = verify_role(repo_object, r)
-            end
-          end
           result = Ddr::Actions::FixityCheck.execute repo_object
           verifications["Fixity check"] = result.success ? VERIFICATION_PASS : VERIFICATION_FAIL
         end
@@ -198,13 +190,18 @@ module Ddr::Batch
         end
       end
 
-      def verify_label(repo_object)
-        repo_object.label.eql?(label) ? VERIFICATION_PASS : VERIFICATION_FAIL
-      end
-
+      # def verify_label(repo_object)
+      #   repo_object.label.eql?(label) ? VERIFICATION_PASS : VERIFICATION_FAIL
+      # end
+      #
       def verify_attribute(repo_object, attribute)
-        repo_object.datastreams[attribute.datastream].values(attribute.name).include?(attribute.value.strip) ?
-                VERIFICATION_PASS : VERIFICATION_FAIL
+        verified = case attribute.datastream
+          when 'descMetadata'
+            repo_object.descMetadata.values(attribute.name).include?(attribute.value)
+          when 'adminMetadata'
+            repo_object.adminMetadata.values(attribute.name).include?(attribute.value)
+        end
+        verified ? VERIFICATION_PASS : VERIFICATION_FAIL
       end
 
       def verify_datastream(repo_object, datastream)
@@ -236,33 +233,23 @@ module Ddr::Batch
         end
       end
 
-      def verify_role(repo_object, role)
-        role_hash = {  "role_type"=>[ role.role_type ], "agent"=>[ role.agent ], "scope"=>[ role.role_scope ] }
-        repo_object.roles.role_set.map(&:to_h).include?(role_hash) ? VERIFICATION_PASS : VERIFICATION_FAIL
-      end
-
       def add_attribute(repo_object, attribute)
-        repo_object.datastreams[attribute.datastream].add_value(attribute.name, attribute.value)
+        repo_object.send(attribute.datastream).add_value(attribute.name, attribute.value)
         return repo_object
       end
 
       def clear_attribute(repo_object, attribute)
-        repo_object.datastreams[attribute.datastream].set_values(attribute.name, nil)
+        repo_object.send(attribute.datastream).set_values(attribute.name, nil)
         return repo_object
       end
 
       def clear_attributes(repo_object, attribute)
-        repo_object.datastreams[attribute.datastream].class.term_names.each do |term|
-          repo_object.datastreams[attribute.datastream].set_values(term, nil) if repo_object.datastreams[attribute.datastream].values(term)
+        Ddr::Models::DescriptiveMetadata.unqualified_names.each do |term|
+          repo_object.descMetadata.set_values(term, nil) if repo_object.descMetadata.values(term)
         end
         return repo_object
       end
 
-      def add_role(repo_object, role)
-        repo_object.roles.grant(scope: role.role_scope, type: role.role_type, agent: role.agent)
-        return repo_object
-      end
-
       def populate_datastream(repo_object, datastream)
         case datastream[:payload_type]
         when BatchObjectDatastream::PAYLOAD_TYPE_BYTES
@@ -278,7 +265,11 @@ module Ddr::Batch
           else
             ds_content = File.new(datastream[:payload])
           end
-          repo_object.add_file(ds_content, datastream[:name], mime_type: mime_type)
+          file_name = File.basename(datastream[:payload])
+          dsid = datastream[:name]
+          opts = { filename: file_name }
+          opts.merge({ mime_type: mime_type }) if mime_type
+          repo_object.add_file(ds_content, path: dsid)
         end
         return repo_object
       end

data/app/models/ddr/batch/batch_object_attribute.rb

@@ -46,11 +46,16 @@ module Ddr::Batch
     end
 
     def datastream_valid?
-        datastream_type < ActiveFedora::RDFDatastream rescue false
+        [ 'adminMetadata', 'descMetadata' ].include?(datastream)
     end
 
     def attribute_name_valid?
-      datastream_type.term_names.include?(name.to_sym)
+      case datastream
+        when 'adminMetadata'
+          batch_object.model.constantize.properties.include?(name)
+        when 'descMetadata'
+          Ddr::Models::DescriptiveMetadata.unqualified_names.include?(name.to_sym)
+      end
     end
 
   end

data/app/models/ddr/batch/batch_object_datastream.rb

@@ -4,6 +4,11 @@ module Ddr::Batch
 
     belongs_to :batch_object, :inverse_of => :batch_object_datastreams
 
+    DATASTREAMS = [ Ddr::Datastreams::CONTENT,
+                    Ddr::Datastreams::DESC_METADATA,
+                    Ddr::Datastreams::RIGHTS_METADATA,
+                    Ddr::Datastreams::STRUCT_METADATA ]
+
     OPERATION_ADD = "ADD" # add this datastream to the object -- considered an error if datastream already exists
     OPERATION_ADDUPDATE = "ADDUPDATE" # add this datastream to or update this datastream in the object
     OPERATION_UPDATE = "UPDATE" # update this datastream in the object -- considered an error if datastream does not already exist

data/app/models/ddr/batch/batch_object_relationship.rb

@@ -1,6 +1,7 @@
 module Ddr::Batch
 
   class BatchObjectRelationship < ActiveRecord::Base
+#    attr_accessible :name, :object, :object_type, :operation, :batch_object
     belongs_to :batch_object, :inverse_of => :batch_object_relationships
 
     RELATIONSHIP_ADMIN_POLICY = "admin_policy"

data/app/models/ddr/batch/ingest_batch_object.rb

@@ -6,7 +6,6 @@ module Ddr::Batch
       errors = []
       errors << "#{@error_prefix} Model required for INGEST operation" unless model
       errors += validate_pre_assigned_pid if pid
-      errors += validate_collection if model == 'Collection'
       errors
     end
 
@@ -20,11 +19,10 @@ module Ddr::Batch
 
     def results_message
       if pid
-        message_level = verified ? Logger::INFO : Logger::WARN
-        verification_result = verified ? "Verified" : "VERIFICATION FAILURE"
-        ProcessingResultsMessage.new(message_level, "Ingested #{model} #{identifier} into #{pid}...#{verification_result}")
+        verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
+        message = "Ingested #{model} #{identifier} into #{pid}...#{verification_result}"
       else
-        ProcessingResultsMessage.new(Logger::ERROR, "Attempt to ingest #{model} #{identifier} FAILED")
+        message = "Attempt to ingest #{model} #{identifier} FAILED"
       end
     end
 
@@ -36,18 +34,8 @@ module Ddr::Batch
       return errs
     end
 
-    def validate_collection
-      errs = []
-      coll = Collection.new
-      batch_object_attributes.each { |attr| coll = add_attribute(coll, attr) }
-      unless coll.valid?
-        coll.errors.messages.each { |k, v| errs << "#{@error_prefix} Collection #{k} #{v.join(';')}" }
-      end
-      errs
-    end
-
     def ingest(user, opts = {})
-      repo_object = create_repository_object(user)
+      repo_object = create_repository_object
       if !repo_object.nil? && !repo_object.new_record?
         ingest_outcome_detail = []
         ingest_outcome_detail << "Ingested #{model} #{identifier} into #{repo_object.pid}"
@@ -90,27 +78,35 @@ module Ddr::Batch
       repo_object
     end
 
-    def create_repository_object(user)
+    def create_repository_object
       repo_pid = pid if pid.present?
       repo_object = nil
       begin
-        repo_object = model.constantize.new(:pid => repo_pid)
-        repo_object.label = label if label
+        repo_object = model.constantize.new(:id => repo_pid)
+        # repo_object.label = label if label
+        repo_object.save(validate: false)
         batch_object_attributes.each { |a| repo_object = add_attribute(repo_object, a) }
-        repo_object.save(validate: false, skip_structure_updates: true, user: user)
         batch_object_datastreams.each { |d| repo_object = populate_datastream(repo_object, d) }
         batch_object_relationships.each { |r| repo_object = add_relationship(repo_object, r) }
-        batch_object_roles.each { |r| repo_object = add_role(repo_object, r) }
-        repo_object.save!(skip_structure_updates: true)
+        repo_object.save
       rescue Exception => e1
-        logger.fatal("Error in creating repository object #{repo_object.pid} for #{identifier} : #{e1}")
+        logger.fatal("Error in creating repository object #{repo_object.id} for #{identifier} : #{e1}")
+        repo_clean = false
         if repo_object && !repo_object.new_record?
           begin
-            logger.info("Deleting potentially incomplete #{repo_object.pid} due to error in ingest batch processing")
+            logger.info("Deleting potentially incomplete #{repo_object.id} due to error in ingest batch processing")
             repo_object.destroy
           rescue Exception => e2
-            logger.fatal("Error deleting repository object #{repo_object.pid}: #{e2}")
+            logger.fatal("Error deleting repository object #{repo_object.id}: #{e2}")
+          else
+            repo_clean = true
           end
+        else
+          repo_clean = true
+        end
+        if batch.present?
+          batch.status = repo_clean ? Batch::STATUS_RESTARTABLE : Batch::STATUS_INTERRUPTED
+          batch.save
         end
         raise e1
       end