ddr-batch 1.7.2 → 2.0.0.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +45 -0
  3. data/app/jobs/ddr/batch/batch_processor_job.rb +7 -2
  4. data/app/mailers/ddr/batch/batch_processor_run_mailer.rb +7 -24
  5. data/app/models/ddr/batch/batch.rb +18 -20
  6. data/app/models/ddr/batch/batch_ability_definitions.rb +14 -0
  7. data/app/models/ddr/batch/batch_object.rb +26 -35
  8. data/app/models/ddr/batch/batch_object_attribute.rb +7 -2
  9. data/app/models/ddr/batch/batch_object_datastream.rb +5 -0
  10. data/app/models/ddr/batch/batch_object_relationship.rb +1 -0
  11. data/app/models/ddr/batch/ingest_batch_object.rb +21 -25
  12. data/app/models/ddr/batch/update_batch_object.rb +10 -18
  13. data/app/scripts/ddr/batch/batch_processor.rb +152 -0
  14. data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb +5 -5
  15. data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb +5 -5
  16. data/config/locales/en.yml +2 -3
  17. data/lib/ddr/batch.rb +0 -6
  18. data/lib/ddr/batch/version.rb +1 -1
  19. metadata +31 -47
  20. data/app/jobs/ddr/batch/batch_deletion_job.rb +0 -19
  21. data/app/jobs/ddr/batch/batch_objects_processor_job.rb +0 -11
  22. data/app/models/ddr/batch/batch_object_message.rb +0 -8
  23. data/app/models/ddr/batch/batch_object_role.rb +0 -26
  24. data/app/models/ddr/batch/error.rb +0 -10
  25. data/app/models/ddr/batch/log.rb +0 -29
  26. data/app/services/ddr/batch/monitor_batch_finished.rb +0 -87
  27. data/app/services/ddr/batch/monitor_batch_object_handled.rb +0 -42
  28. data/app/services/ddr/batch/monitor_batch_started.rb +0 -43
  29. data/app/services/ddr/batch/process_batch.rb +0 -60
  30. data/app/services/ddr/batch/process_batch_object.rb +0 -46
  31. data/app/services/ddr/batch/process_batch_objects.rb +0 -39
  32. data/config/initializers/subscriptions.rb +0 -9
  33. data/db/migrate/20160816164010_create_batch_object_roles.rb +0 -15
  34. data/db/migrate/20161115191636_add_columns_to_batch_object.rb +0 -9
  35. data/db/migrate/20161116142512_create_batch_object_messages.rb +0 -13
  36. data/db/migrate/20161222192611_remove_columns_from_batch.rb +0 -13
  37. data/db/migrate/20171116183514_add_collection_columns_to_batch.rb +0 -8
@@ -1,19 +0,0 @@
1
- module Ddr::Batch
2
- class BatchDeletionJob
3
- @queue = :batch
4
-
5
- def self.perform(batch_id)
6
- batch = Batch.find(batch_id)
7
- batch.status = Batch::STATUS_DELETING
8
- batch.save!
9
- batch.destroy!
10
- end
11
-
12
- def self.before_enqueue_set_status(batch_id)
13
- batch = Batch.find(batch_id)
14
- batch.status = Batch::STATUS_QUEUED_FOR_DELETION
15
- batch.save
16
- end
17
-
18
- end
19
- end
@@ -1,11 +0,0 @@
1
- module Ddr::Batch
2
- class BatchObjectsProcessorJob
3
- @queue = :batch
4
-
5
- def self.perform(batch_object_ids, operator_id)
6
- operator = User.find(operator_id)
7
- ProcessBatchObjects.new(batch_object_ids: batch_object_ids, operator: operator).execute
8
- end
9
-
10
- end
11
- end
@@ -1,8 +0,0 @@
1
- module Ddr::Batch
2
- class BatchObjectMessage < ActiveRecord::Base
3
- belongs_to :batch_object, :inverse_of => :batch_object_messages
4
-
5
- validates_presence_of :message
6
-
7
- end
8
- end
@@ -1,26 +0,0 @@
1
- module Ddr::Batch
2
-
3
- class BatchObjectRole < ActiveRecord::Base
4
- belongs_to :batch_object, :inverse_of => :batch_object_roles
5
-
6
- OPERATION_ADD = "ADD".freeze # Add the principal and role to the object in the indicated scope
7
- OPERATIONS = [ OPERATION_ADD ].freeze
8
-
9
- validates :operation, inclusion: { in: OPERATIONS }
10
- validate :valid_role, if: :operation_requires_valid_role?
11
- validate :policy_role_scope_only_for_collections, if: "role_scope == Ddr::Auth::Roles::POLICY_SCOPE"
12
-
13
- def operation_requires_valid_role?
14
- [ OPERATION_ADD ].include? operation
15
- end
16
-
17
- def valid_role
18
- Ddr::Auth::Roles::Role.build(type: role_type, agent: agent, scope: role_scope).valid?
19
- end
20
-
21
- def policy_role_scope_only_for_collections
22
- errors.add(:role_scope, "policy role scope is valid only for Collections") unless batch_object.model == "Collection"
23
- end
24
- end
25
-
26
- end
@@ -1,10 +0,0 @@
1
- module Ddr
2
- module Batch
3
- # Base class for custom exceptions
4
- class Error < StandardError; end
5
-
6
- # Error processing batch object
7
- class BatchObjectProcessingError < Error; end
8
-
9
- end
10
- end
@@ -1,29 +0,0 @@
1
- module Ddr::Batch
2
- class Log
3
-
4
- DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
5
-
6
- class << self
7
-
8
- def logger(batch_id)
9
- loggr = Logger.new(File.open(file_path(batch_id), File::WRONLY | File::APPEND | File::CREAT))
10
- loggr.level = Ddr::Batch.processor_logging_level
11
- loggr.datetime_format = "%Y-%m-%d %H:%M:%S.L"
12
- loggr.formatter = proc do |severity, datetime, progname, msg|
13
- "#{datetime} #{severity}: #{msg}\n"
14
- end
15
- loggr
16
- end
17
-
18
- def clear_log(batch_id)
19
- log_file_path = file_path(batch_id)
20
- FileUtils.remove(log_file_path) if File.exists?(log_file_path)
21
- end
22
-
23
- def file_path(batch_id)
24
- File.join(DEFAULT_LOG_DIR, "batch_#{batch_id}_log.txt")
25
- end
26
-
27
- end
28
- end
29
- end
@@ -1,87 +0,0 @@
1
- module Ddr::Batch
2
- class MonitorBatchFinished
3
-
4
- class << self
5
- def call(*args)
6
- event = ActiveSupport::Notifications::Event.new(*args)
7
- batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
8
- batch_finished(batch)
9
- if batch.outcome == Ddr::Batch::Batch::OUTCOME_SUCCESS
10
- ActiveSupport::Notifications.instrument('success.batch.batch.ddr', batch_id: batch.id)
11
- end
12
- end
13
-
14
- private
15
-
16
- def batch_finished(batch)
17
- log_batch_finish(batch)
18
- update_batch(batch)
19
- send_email(batch) if batch.user && batch.user.email
20
- end
21
-
22
- def log_batch_finish(batch)
23
- logger = Ddr::Batch::Log.logger(batch.id)
24
- logger.info "====== Summary ======"
25
- results_tracker = results(batch)
26
- results_tracker.keys.each do |type|
27
- results_tracker[type].keys.each do |model|
28
- log_result(results_tracker, type, model, logger)
29
- end
30
- end
31
- logger.close
32
- end
33
-
34
- def results(batch)
35
- results_tracker = Hash.new
36
- batch.batch_objects.each do |batch_object|
37
- track_result(results_tracker, batch_object)
38
- end
39
- results_tracker
40
- end
41
-
42
- def track_result(results_tracker, batch_object)
43
- type = batch_object.type
44
- model = batch_object.model || "Missing Model"
45
- results_tracker[type] = Hash.new unless results_tracker.has_key?(type)
46
- results_tracker[type][model] = Hash.new unless results_tracker[type].has_key?(model)
47
- results_tracker[type][model][:successes] = 0 unless results_tracker[type][model].has_key?(:successes)
48
- results_tracker[type][model][:successes] += 1 if batch_object.verified
49
- end
50
-
51
- def log_result(results_tracker, type, model, logger)
52
- verb = type_verb(type)
53
- count = results_tracker[type][model][:successes]
54
- logger.info "#{verb} #{ActionController::Base.helpers.pluralize(count, model)}"
55
- end
56
-
57
- def type_verb(type)
58
- case type
59
- when Ddr::Batch::IngestBatchObject.name
60
- "Ingested"
61
- when Ddr::Batch::UpdateBatchObject.name
62
- "Updated"
63
- end
64
- end
65
-
66
- def update_batch(batch)
67
- outcome = batch.success_count.eql?(batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
68
- logfile = File.new(Ddr::Batch::Log.file_path(batch.id))
69
- batch.update!(stop: DateTime.now,
70
- status: Batch::STATUS_FINISHED,
71
- outcome: outcome,
72
- logfile: logfile)
73
- end
74
-
75
- def send_email(batch)
76
- begin
77
- Ddr::Batch::BatchProcessorRunMailer.send_notification(batch).deliver!
78
- rescue => e
79
- Rails.logger.error("An error occurred while attempting to send a notification for batch #{batch.id}")
80
- Rails.logger.error(e.message)
81
- Rails.logger.error(e.backtrace)
82
- end
83
- end
84
- end
85
-
86
- end
87
- end
@@ -1,42 +0,0 @@
1
- module Ddr::Batch
2
- class MonitorBatchObjectHandled
3
-
4
- class << self
5
- def call(*args)
6
- event = ActiveSupport::Notifications::Event.new(*args)
7
- batch_object = BatchObject.find(event.payload[:batch_object_id])
8
- batch = batch_object.batch
9
- if event.payload[:exception].present?
10
- record_batch_object_exception(batch_object, event.payload[:exception])
11
- end
12
- batch_object_handled(batch_object, batch)
13
- end
14
-
15
- private
16
-
17
- def record_batch_object_exception(batch_object, exception_info)
18
- batch_object_exception_msg = I18n.t('ddr.batch.errors.batch_object_processing', error_msg: exception_info[1])
19
- Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
20
- level: Logger::ERROR,
21
- message: batch_object_exception_msg)
22
- end
23
-
24
- def batch_object_handled(batch_object, batch)
25
- log_batch_object_messages(batch_object, batch.id)
26
- batch_object.update!(handled: true)
27
- unless batch.unhandled_objects?
28
- ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
29
- end
30
- end
31
-
32
- def log_batch_object_messages(batch_object, batch_id)
33
- logger = Ddr::Batch::Log.logger(batch_id)
34
- batch_object.batch_object_messages.each do |message|
35
- logger.add(message.level) { "Batch Object #{batch_object.id}: #{message.message}" }
36
- end
37
- logger.close
38
- end
39
- end
40
-
41
- end
42
- end
@@ -1,43 +0,0 @@
1
- module Ddr::Batch
2
- class MonitorBatchStarted
3
-
4
- class << self
5
- def call(*args)
6
- event = ActiveSupport::Notifications::Event.new(*args)
7
- batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
8
- batch_started(batch)
9
- end
10
-
11
- private
12
-
13
- def batch_started(batch)
14
- clear_logs(batch)
15
- log_batch_start(batch)
16
- update_batch(batch)
17
- end
18
-
19
- def clear_logs(batch)
20
- # delete any previously existing filesystem log file for this batch
21
- Ddr::Batch::Log.clear_log(batch.id)
22
- # remove any existing attached log file from the Batch ActiveRecord object
23
- batch.logfile.clear
24
- end
25
-
26
- def log_batch_start(batch)
27
- logger = Ddr::Batch::Log.logger(batch.id)
28
- logger.info "Collection: #{batch.collection_title}" if batch.collection_title.present?
29
- logger.info "Batch id: #{batch.id}"
30
- logger.info "Batch name: #{batch.name}" if name
31
- logger.info "Batch size: #{batch.batch_objects.size}"
32
- logger.close
33
- end
34
-
35
- def update_batch(batch)
36
- batch.update!(start: DateTime.now,
37
- status: Ddr::Batch::Batch::STATUS_RUNNING,
38
- version: VERSION)
39
- end
40
- end
41
-
42
- end
43
- end
@@ -1,60 +0,0 @@
1
- module Ddr::Batch
2
- class ProcessBatch
3
-
4
- attr_accessor :batch, :operator_id
5
-
6
- def initialize(batch_id:, operator_id:)
7
- @batch = Ddr::Batch::Batch.find(batch_id)
8
- @operator_id = operator_id
9
- end
10
-
11
- def execute
12
- ActiveSupport::Notifications.instrument('started.batch.batch.ddr', batch_id: batch.id)
13
- batch.batch_objects.each do |batch_object|
14
- case
15
- when batch_object.is_a?(IngestBatchObject)
16
- handle_ingest_batch_object(batch_object)
17
- when batch_object.is_a?(UpdateBatchObject)
18
- handle_update_batch_object(batch_object)
19
- end
20
- end
21
- end
22
-
23
- def handle_ingest_batch_object(batch_object)
24
- case batch_object.model
25
- when 'Collection'
26
- ingest_collection_object(batch_object)
27
- when 'Item'
28
- enqueue_item_component_ingest(batch_object)
29
- when 'Component'
30
- # skip -- will be handled along with associated Item
31
- when 'Target', 'Attachment'
32
- Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
33
- end
34
- end
35
-
36
- def handle_update_batch_object(batch_object)
37
- Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
38
- end
39
-
40
- def ingest_collection_object(batch_object)
41
- # Collection batch objects are processed synchronously because they need to exist in the repository
42
- # prior to the processing of any objects (e.g., Item, Component, Target) associated with them.
43
- # If the Collection batch object does not process successfully, consider the batch finished (albeit unsuccessfully)
44
- # and raise an exception.
45
- unless ProcessBatchObject.new(batch_object_id: batch_object.id, operator: User.find(operator_id)).execute
46
- ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
47
- raise Ddr::Batch::BatchObjectProcessingError, batch_object.id
48
- end
49
- end
50
-
51
- def enqueue_item_component_ingest(batch_object)
52
- query = [ "object = '#{batch_object.pid}'",
53
- "batch_object_relationships.name = '#{Ddr::Batch::BatchObjectRelationship::RELATIONSHIP_PARENT}'",
54
- "batches.id = #{batch_object.batch.id}" ].join(' AND ')
55
- recs = Ddr::Batch::BatchObjectRelationship.joins(batch_object: :batch).where(query)
56
- batch_object_ids = recs.map { |rec| rec.batch_object.id }.unshift(batch_object.id)
57
- Resque.enqueue(BatchObjectsProcessorJob, batch_object_ids, operator_id)
58
- end
59
- end
60
- end
@@ -1,46 +0,0 @@
1
- module Ddr::Batch
2
- class ProcessBatchObject
3
-
4
- attr_reader :batch_object_id, :operator
5
-
6
- def initialize(batch_object_id:, operator:)
7
- @batch_object_id = batch_object_id
8
- @operator = operator
9
- end
10
-
11
- def execute
12
- ActiveSupport::Notifications.instrument("handled.batchobject.batch.ddr",
13
- batch_object_id: batch_object_id) do |payload|
14
- batch_object = BatchObject.find(batch_object_id)
15
- # Validate batch object
16
- errors = batch_object.validate
17
- # Process batch object or record validation errors
18
- if errors.empty?
19
- process(batch_object, operator)
20
- else
21
- record_errors(batch_object, errors)
22
- end
23
- # return true if batch_object was processed; otherwise, false
24
- batch_object.processed? ? true : false
25
- end
26
- end
27
-
28
- def process(batch_object, operator)
29
- batch_object.update!(validated: true)
30
- batch_object.process(operator)
31
- batch_object.update!(processed: true)
32
- results_message = batch_object.results_message
33
- Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
34
- level: results_message.level,
35
- message: results_message.message)
36
- end
37
-
38
- def record_errors(batch_object, errors)
39
- errors.each do |error|
40
- Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
41
- level: Logger::ERROR,
42
- message: error)
43
- end
44
- end
45
- end
46
- end
@@ -1,39 +0,0 @@
1
- module Ddr::Batch
2
- class ProcessBatchObjects
3
-
4
- attr_reader :batch_object_ids, :operator
5
-
6
- def initialize(batch_object_ids:, operator:)
7
- @batch_object_ids = batch_object_ids
8
- @operator = operator
9
- end
10
-
11
- def execute
12
- # Assume successful processing of all batch objects until proven otherwise.
13
- success = true
14
- batch_object_ids.each do |batch_object_id|
15
- batch_object = Ddr::Batch::BatchObject.find(batch_object_id)
16
- # Skip batch objects that have already been successfully processed. This is useful when this service is
17
- # called within the context of a BatchObjectsProcessorJob, that job fails, and the failed job is retried.
18
- unless batch_object.verified?
19
- # Once any batch object included in this job fails to process successfully, do not attempt to process
20
- # any remaining batch objects included in this job. Instead, mark them as "handled" so the batch knows
21
- # it's not waiting on them to be handled before it can consider itself "finished".
22
- # The use case prompting this behavior is a job containing an Item ingest batch object plus one or more
23
- # associated Component ingest batch objects. If the Item batch object fails to process correctly, we don't
24
- # want to attempt to process the Component batch objects.
25
- # In the preceding use case, we could skip the remaining batch objects only if the failed batch object is an
26
- # Item but there might be future cases in which we don't want to process the remaining batch objects in the
27
- # job regardless of which batch object fails. The failure of any batch object to process should be rare
28
- # enough that it doesn't seem harmful to cover this potential broader use case in the current code.
29
- if success
30
- success = ProcessBatchObject.new(batch_object_id: batch_object.id, operator: operator).execute
31
- else
32
- batch_object.update!(handled: true)
33
- end
34
- end
35
- end
36
- end
37
-
38
- end
39
- end
@@ -1,9 +0,0 @@
1
- ##
2
- ## Subscriptions to ActiveSupport::Notifications instrumentation events
3
- ##
4
-
5
- # Batch Processing events
6
- ActiveSupport::Notifications.subscribe('started.batch.batch.ddr', Ddr::Batch::MonitorBatchStarted)
7
- ActiveSupport::Notifications.subscribe('handled.batchobject.batch.ddr', Ddr::Batch::MonitorBatchObjectHandled)
8
- ActiveSupport::Notifications.subscribe('finished.batch.batch.ddr', Ddr::Batch::MonitorBatchFinished)
9
-