ddr-batch 1.0.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: da0b053081d66d3ec2124f3ff59ad25154475822
+  data.tar.gz: 5be40e70e6c0233ea0f0abe6533743e03f1dfa11
+SHA512:
+  metadata.gz: 9d891c9ece98cd96791fa342bfb11c0824b550618a9a562551984cbdbd2449d526a95fae120ea609f9ae2f28d90ca4a9e765da3c56af4f2a61c8154b9b873b1d
+  data.tar.gz: faed3d3c6dcd10550441aabf4641555f7c4339dff93354ccf5a822c7c529a6931b9cd50c41945677fb2dece04b3d05995c288a4dade814907dbc823c1b9ce7ae

data/LICENSE.txt

@@ -0,0 +1,12 @@
+Copyright (c) 2015, Duke University
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,100 @@
+# ddr-batch
+
+A Rails engine providing batch processing functionality for the Duke Digital Repository.
+
+## Installation
+
+Add to your application's Gemfile:
+
+    gem 'ddr-batch'
+
+and
+
+    bundle install
+
+## Configuration
+
+### User model
+
+Include `Ddr::Batch::BatchUser` in `app/models/user.rb`.
+
+```ruby
+class User < ActiveRecord::Base
+
+  # DO NOT REMOVE:
+  # Blacklight::User
+  # Ddr::Auth::User
+  #
+  include Ddr::Batch::BatchUser
+
+end
+```
+
+### Ability class
+
+Add `Ddr::Batch::BatchAbilityDefinitions` to the list of `ability_definitions`.
+
+```ruby
+class Ability < Ddr::Auth::Ability
+
+  self.ability_definitions += [ Ddr::Batch::BatchAbilityDefinitions ]
+
+end
+```
+
+### Log4r
+
+#### Application.rb
+
+Add the following lines to `config\application.rb` if they are not already there:
+
+```ruby
+require 'log4r'
+require 'log4r/yamlconfigurator'
+require 'log4r/outputter/datefileoutputter'
+include Log4r
+```
+
+#### Configuration
+
+`Ddr::Batch::BatchProcessor` expects a Log4r configuration file at `config\log4r_batch_processor.yml`.
+
+##### Example
+
+```yaml
+log4r_config:
+    loggers:
+        - name      : batch_processor
+          level     : DEBUG
+          trace     : 'false'
+          outputters:
+            - logfile
+    outputters:
+        - type      : StdoutOutputter
+          name      : stdout
+          level     : DEBUG
+          formatter :
+            date_pattern: '%F %T.%L'
+            pattern     : '%d %l: %m'
+            type        : PatternFormatter
+        - type        : FileOutputter
+          name        : logfile
+          trunc       : 'false'
+          filename    : "#{LOG_FILE}"
+          formatter   :
+            date_pattern: '%F %T.%L'
+            pattern     : '%d %l: %m'
+            type        : PatternFormatter
+```
+
+### Migrations
+
+Install the ddr-batch migrations:
+
+`rake ddr_batch:install:migrations`
+
+then
+
+`rake db:migrate`
+
+`rake db:test:prepare`

data/Rakefile

@@ -0,0 +1,36 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+
+APP_ROOT = File.dirname(__FILE__)
+
+JETTY_CONFIG = {
+	jetty_home: File.join(APP_ROOT, "jetty"),
+	jetty_port: 8983,
+	startup_wait: 60,
+	quiet: true,
+	java_opts: ["-Xmx256m", "-XX:MaxPermSize=128m"]
+}
+
+require 'jettywrapper'
+
+Jettywrapper.instance.base_path = APP_ROOT
+
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+require 'rspec/core/rake_task'
+
+desc "Run all specs in spec directory"
+RSpec::Core::RakeTask.new(:spec => ["app:db:migrate", "app:db:test:prepare"])
+
+desc "Run the CI build"
+task :ci => ["jetty:clean"] do
+  Jettywrapper.wrap(JETTY_CONFIG) do
+    Rake::Task['spec'].invoke
+  end
+end

data/app/jobs/ddr/batch/batch_processor_job.rb

@@ -0,0 +1,21 @@
+module Ddr::Batch
+  class BatchProcessorJob
+    @queue = :batch
+
+    def self.perform(batch_id, operator_id)
+      ts = Time.now.strftime("%Y%m%d%H%M%S%L")
+      logfile = "batch_processor_#{ts}_log.txt"
+      batch = Batch.find(batch_id)
+      operator = User.find(operator_id)
+      bp = BatchProcessor.new(batch, operator, log_file: logfile)
+      bp.execute
+    end
+
+    def self.after_enqueue_set_status(batch_id, operator_id)
+      batch = Batch.find(batch_id)
+      batch.status = Batch::STATUS_QUEUED
+      batch.save
+    end
+
+  end
+end

data/app/mailers/ddr/batch/batch_processor_run_mailer.rb

@@ -0,0 +1,19 @@
+module Ddr::Batch
+
+  class BatchProcessorRunMailer < ActionMailer::Base
+
+    default :from => "noreply@duke.edu"
+
+    def send_notification(batch)
+      @batch = batch
+      @title = "Batch Processor Run #{@batch.status}"
+      @host = `uname -n`.strip
+      @subject = "[#{@host}] #{@title}"
+      from = "#{`echo $USER`.strip}@#{@host}"
+      attachments["details.txt"] = File.read(@batch.logfile.path)
+      mail(from: from, to: @batch.user.email, subject: @subject)
+    end
+
+  end
+
+end

data/app/models/ddr/batch/batch.rb

@@ -0,0 +1,72 @@
+module Ddr::Batch
+
+  class Batch < ActiveRecord::Base
+    belongs_to :user, :inverse_of => :batches, class_name: ::User
+    has_many :batch_objects, -> { order("id ASC") }, :inverse_of => :batch, :dependent => :destroy
+    has_attached_file :logfile
+    do_not_validate_attachment_file_type :logfile
+
+    OUTCOME_SUCCESS = "SUCCESS"
+    OUTCOME_FAILURE = "FAILURE"
+
+    STATUS_READY = "READY"
+    STATUS_VALIDATING = "VALIDATING"
+    STATUS_INVALID = "INVALID"
+    STATUS_VALIDATED = "VALIDATED"
+    STATUS_QUEUED = "QUEUED"
+    STATUS_PROCESSING = "PROCESSING"
+    STATUS_RUNNING = "RUNNING"
+    STATUS_FINISHED = "FINISHED"
+    STATUS_INTERRUPTED = "INTERRUPTED"
+    STATUS_RESTARTABLE = "INTERRUPTED - RESTARTABLE"
+
+    def validate
+      errors = []
+      begin
+        batch_objects.each do |object|
+          unless object.verified
+            errors << object.validate
+          end
+        end
+      rescue Exception => e
+        errors << "Exception raised during batch validation: #{e.backtrace}"
+      end
+      errors.flatten
+    end
+
+    def completed_count
+      batch_objects.where(verified: true).count
+    end
+
+    def time_to_complete
+      unless processing_step_start.nil?
+        if completed_count > 0
+          completed = completed_count
+          ((Time.now - processing_step_start.to_time) / completed) * (batch_objects.count - completed)
+        end
+      end
+    end
+
+    def found_pids
+      @found_pids ||= {}
+    end
+
+    def add_found_pid(pid, model)
+      @found_pids[pid] = model
+    end
+
+    def pre_assigned_pids
+      @pre_assigned_pids ||= collect_pre_assigned_pids
+    end
+
+    def collect_pre_assigned_pids
+      batch_objects.map{ |x| x.pid if x.pid.present? }.compact
+    end
+
+    def finished?
+      status == STATUS_FINISHED
+    end
+
+  end
+
+end

data/app/models/ddr/batch/batch_ability_definitions.rb

@@ -0,0 +1,14 @@
+module Ddr::Batch
+  class BatchAbilityDefinitions < Ddr::Auth::AbilityDefinitions
+
+    def call
+      if authenticated?
+        can :manage, Batch, user_id: user.id
+      end
+      can :manage, Ddr::Batch::BatchObject do |batch_object|
+        can? :manage, batch_object.batch
+      end
+    end
+
+  end
+end

data/app/models/ddr/batch/batch_object.rb

@@ -0,0 +1,297 @@
+module Ddr::Batch
+
+    # This is a superclass containing methods common to all batch object objects.  It is not intended to be instantiated directly.
+    # This superclass and its subclasses are designed following the ActiveRecord single-table inheritance pattern.
+    class BatchObject < ActiveRecord::Base
+
+      belongs_to :batch, inverse_of: :batch_objects
+      has_many :batch_object_attributes, -> { order "id ASC" }, inverse_of: :batch_object, dependent: :destroy
+      has_many :batch_object_datastreams, inverse_of: :batch_object, dependent: :destroy
+      has_many :batch_object_relationships, inverse_of: :batch_object, dependent: :destroy
+
+      VERIFICATION_PASS = "PASS"
+      VERIFICATION_FAIL = "FAIL"
+
+      EVENT_SUMMARY = <<-EOS
+  %{label}
+  Batch object database id: %{batch_id}
+  Batch object identifier: %{identifier}
+  Model: %{model}
+      EOS
+
+      def self.pid_from_identifier(identifier, batch_id)
+        query = "identifier = :identifier"
+        query << " and batch_id = :batch_id" if batch_id
+        params = { :identifier => identifier }
+        params[:batch_id] = batch_id if batch_id
+        sort = "updated_at asc"
+        found_objects = BatchObject.where(query, params).order(sort)
+        pids = []
+        found_objects.each { |obj| pids << obj.pid }
+        return pids
+      end
+
+      def validate
+        @error_prefix = I18n.t('ddr.batch.errors.prefix', :identifier => identifier, :id => id)
+        errors = []
+        errors += validate_model if model
+        errors += validate_datastreams if batch_object_datastreams
+        errors += validate_relationships if batch_object_relationships
+        errors += local_validations
+        return errors
+      end
+
+      def local_validations
+        []
+      end
+
+      def model_datastream_keys
+        raise NotImplementedError
+      end
+
+      def process(user, opts = {})
+        raise NotImplementedError
+      end
+
+      def results_message
+        raise NotImplementedError
+      end
+
+      Results = Struct.new(:repository_object, :verified, :verifications)
+
+      private
+
+      def validate_model
+        errs = []
+        begin
+          model.constantize
+        rescue NameError
+          errs << "#{@error_prefix} Invalid model name: #{model}"
+        end
+        return errs
+      end
+
+      def validate_datastreams
+        errs = []
+        batch_object_datastreams.each do |d|
+          if model_datastream_keys.present?
+            unless model_datastream_keys.include?(d.name)
+              errs << "#{@error_prefix} Invalid datastream name for #{model}: #{d.name}"
+            end
+          end
+          unless BatchObjectDatastream::PAYLOAD_TYPES.include?(d.payload_type)
+            errs << "#{@error_prefix} Invalid payload type for #{d.name} datastream: #{d.payload_type}"
+          end
+          if d.payload_type.eql?(BatchObjectDatastream::PAYLOAD_TYPE_FILENAME)
+            unless File.readable?(d.payload)
+              errs << "#{@error_prefix} Missing or unreadable file for #{d[:name]} datastream: #{d[:payload]}"
+            end
+          end
+          if d.checksum && !d.checksum_type
+            errs << "#{@error_prefix} Must specify checksum type if providing checksum for #{d.name} datastream"
+          end
+          if d.checksum_type
+            unless Ddr::Datastreams::CHECKSUM_TYPES.include?(d.checksum_type)
+              errs << "#{@error_prefix} Invalid checksum type for #{d.name} datastream: #{d.checksum_type}"
+            end
+          end
+        end
+        return errs
+      end
+
+      def validate_relationships
+        errs = []
+        batch_object_relationships.each do |r|
+          obj_model = nil
+          unless BatchObjectRelationship::OBJECT_TYPES.include?(r[:object_type])
+            errs << "#{@error_prefix} Invalid object_type for #{r[:name]} relationship: #{r[:object_type]}"
+          end
+          if r[:object_type].eql?(BatchObjectRelationship::OBJECT_TYPE_PID)
+            if batch.present? && batch.found_pids.keys.include?(r[:object])
+              obj_model = batch.found_pids[r[:object]]
+            else
+              begin
+                obj = ActiveFedora::Base.find(r[:object], :cast => true)
+                obj_model = obj.class.name
+                if batch.present?
+                  batch.add_found_pid(obj.pid, obj_model)
+                end
+              rescue ActiveFedora::ObjectNotFoundError
+                pid_in_batch = false
+                if batch.present?
+                  if batch.pre_assigned_pids.include?(r[:object])
+                    pid_in_batch = true
+                  end
+                end
+                unless pid_in_batch
+                  errs << "#{@error_prefix} #{r[:name]} relationship object does not exist: #{r[:object]}"
+                end
+              end
+            end
+            if obj_model
+              relationship_reflection = Ddr::Utils.relationship_object_reflection(model, r[:name])
+              if relationship_reflection
+                klass = Ddr::Utils.reflection_object_class(relationship_reflection)
+                if klass
+                  errs << "#{@error_prefix} #{r[:name]} relationship object #{r[:object]} exists but is not a(n) #{klass}" unless batch.found_pids[r[:object]].eql?(klass.name)
+                end
+              else
+                errs << "#{@error_prefix} #{model} does not define a(n) #{r[:name]} relationship"
+              end
+            end
+          end
+        end
+        return errs
+      end
+
+      def verify_repository_object
+        verifications = {}
+        begin
+          repo_object = ActiveFedora::Base.find(pid, :cast => true)
+        rescue ActiveFedora::ObjectNotFound
+          verifications["Object exists in repository"] = VERIFICATION_FAIL
+        else
+          verifications["Object exists in repository"] = VERIFICATION_PASS
+          verifications["Object is correct model"] = verify_model(repo_object) if model
+          verifications["Object has correct label"] = verify_label(repo_object) if label
+          unless batch_object_attributes.empty?
+            batch_object_attributes.each do |a|
+              if a.operation == BatchObjectAttribute::OPERATION_ADD
+                verifications["#{a.name} attribute set correctly"] = verify_attribute(repo_object, a)
+              end
+            end
+          end
+          unless batch_object_datastreams.empty?
+            batch_object_datastreams.each do |d|
+              verifications["#{d.name} datastream present and not empty"] = verify_datastream(repo_object, d)
+              verifications["#{d.name} external checksum match"] = verify_datastream_external_checksum(repo_object, d) if d.checksum
+            end
+          end
+          unless batch_object_relationships.empty?
+            batch_object_relationships.each do |r|
+              verifications["#{r.name} relationship is correct"] = verify_relationship(repo_object, r)
+            end
+          end
+          result = Ddr::Actions::FixityCheck.execute repo_object
+          verifications["Fixity check"] = result.success ? VERIFICATION_PASS : VERIFICATION_FAIL
+        end
+        verifications
+      end
+
+      def verify_model(repo_object)
+        begin
+          if repo_object.class.eql?(model.constantize)
+            return VERIFICATION_PASS
+          else
+            return VERIFICATION_FAIL
+          end
+        rescue NameError
+          return VERIFICATION_FAIL
+        end
+      end
+
+      def verify_label(repo_object)
+        repo_object.label.eql?(label) ? VERIFICATION_PASS : VERIFICATION_FAIL
+      end
+
+      def verify_attribute(repo_object, attribute)
+        repo_object.datastreams[attribute.datastream].values(attribute.name).include?(attribute.value) ?
+                VERIFICATION_PASS : VERIFICATION_FAIL
+      end
+
+      def verify_datastream(repo_object, datastream)
+        if repo_object.datastreams.include?(datastream.name) &&
+            repo_object.datastreams[datastream.name].has_content?
+          VERIFICATION_PASS
+        else
+          VERIFICATION_FAIL
+        end
+      end
+
+      def verify_datastream_external_checksum(repo_object, datastream)
+        repo_object.datastreams[datastream.name].validate_checksum! datastream.checksum, datastream.checksum_type
+        return VERIFICATION_PASS
+      rescue Ddr::Models::ChecksumInvalid
+        return VERIFICATION_FAIL
+      end
+
+      def verify_relationship(repo_object, relationship)
+        relationship_reflection = Ddr::Utils.relationship_object_reflection(model, relationship.name)
+        relationship_object_class = Ddr::Utils.reflection_object_class(relationship_reflection)
+        relationship_object = repo_object.send(relationship.name)
+        if !relationship_object.nil? &&
+            relationship_object.pid.eql?(relationship.object) &&
+            relationship_object.is_a?(relationship_object_class)
+          VERIFICATION_PASS
+        else
+          VERIFICATION_FAIL
+        end
+      end
+
+      def add_attribute(repo_object, attribute)
+        repo_object.datastreams[attribute.datastream].add_value(attribute.name, attribute.value)
+        return repo_object
+      end
+
+      def clear_attribute(repo_object, attribute)
+        repo_object.datastreams[attribute.datastream].set_values(attribute.name, nil)
+        return repo_object
+      end
+
+      def clear_attributes(repo_object, attribute)
+        repo_object.datastreams[attribute.datastream].class.term_names.each do |term|
+          repo_object.datastreams[attribute.datastream].set_values(term, nil) if repo_object.datastreams[attribute.datastream].values(term)
+        end
+        return repo_object
+      end
+
+      def populate_datastream(repo_object, datastream)
+        case datastream[:payload_type]
+        when BatchObjectDatastream::PAYLOAD_TYPE_BYTES
+          ds_content = datastream[:payload]
+          if repo_object.datastreams[datastream[:name]].is_a? ActiveFedora::RDFDatastream
+            ds_content = set_rdf_subject(repo_object, ds_content)
+          end
+          repo_object.datastreams[datastream[:name]].content = ds_content
+        when BatchObjectDatastream::PAYLOAD_TYPE_FILENAME
+          if repo_object.datastreams[datastream[:name]].is_a? ActiveFedora::RDFDatastream
+            ds_content = set_rdf_subject(repo_object, File.read(datastream[:payload]))
+            mime_type = "application/n-triples"
+          else
+            ds_content = File.new(datastream[:payload])
+          end
+          file_name = File.basename(datastream[:payload])
+          dsid = datastream[:name]
+          opts = { filename: file_name }
+          opts.merge({ mime_type: mime_type }) if mime_type
+          repo_object.add_file(ds_content, dsid, opts)
+        end
+        return repo_object
+      end
+
+      def add_relationship(repo_object, relationship)
+        relationship_object = case relationship[:object_type]
+        when BatchObjectRelationship::OBJECT_TYPE_PID
+          ActiveFedora::Base.find(relationship[:object], :cast => true)
+        end
+        repo_object.send("#{relationship[:name]}=", relationship_object)
+        return repo_object
+      end
+
+      def set_rdf_subject(repo_object, ds_content)
+        graph = RDF::Graph.new
+        RDF::Reader.for(:ntriples).new(ds_content) do |reader|
+          reader.each_statement do |statement|
+            if statement.subject.is_a? RDF::Node
+              statement.subject = RDF::URI(repo_object.internal_uri)
+            end
+            graph.insert(statement)
+          end
+        end
+        graph.dump :ntriples
+      end
+
+    end
+
+
+end