ddr-batch 1.7.2 → 2.0.0.alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +45 -0
- data/app/jobs/ddr/batch/batch_processor_job.rb +7 -2
- data/app/mailers/ddr/batch/batch_processor_run_mailer.rb +7 -24
- data/app/models/ddr/batch/batch.rb +18 -20
- data/app/models/ddr/batch/batch_ability_definitions.rb +14 -0
- data/app/models/ddr/batch/batch_object.rb +26 -35
- data/app/models/ddr/batch/batch_object_attribute.rb +7 -2
- data/app/models/ddr/batch/batch_object_datastream.rb +5 -0
- data/app/models/ddr/batch/batch_object_relationship.rb +1 -0
- data/app/models/ddr/batch/ingest_batch_object.rb +21 -25
- data/app/models/ddr/batch/update_batch_object.rb +10 -18
- data/app/scripts/ddr/batch/batch_processor.rb +152 -0
- data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb +5 -5
- data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb +5 -5
- data/config/locales/en.yml +2 -3
- data/lib/ddr/batch.rb +0 -6
- data/lib/ddr/batch/version.rb +1 -1
- metadata +31 -47
- data/app/jobs/ddr/batch/batch_deletion_job.rb +0 -19
- data/app/jobs/ddr/batch/batch_objects_processor_job.rb +0 -11
- data/app/models/ddr/batch/batch_object_message.rb +0 -8
- data/app/models/ddr/batch/batch_object_role.rb +0 -26
- data/app/models/ddr/batch/error.rb +0 -10
- data/app/models/ddr/batch/log.rb +0 -29
- data/app/services/ddr/batch/monitor_batch_finished.rb +0 -87
- data/app/services/ddr/batch/monitor_batch_object_handled.rb +0 -42
- data/app/services/ddr/batch/monitor_batch_started.rb +0 -43
- data/app/services/ddr/batch/process_batch.rb +0 -60
- data/app/services/ddr/batch/process_batch_object.rb +0 -46
- data/app/services/ddr/batch/process_batch_objects.rb +0 -39
- data/config/initializers/subscriptions.rb +0 -9
- data/db/migrate/20160816164010_create_batch_object_roles.rb +0 -15
- data/db/migrate/20161115191636_add_columns_to_batch_object.rb +0 -9
- data/db/migrate/20161116142512_create_batch_object_messages.rb +0 -13
- data/db/migrate/20161222192611_remove_columns_from_batch.rb +0 -13
- data/db/migrate/20171116183514_add_collection_columns_to_batch.rb +0 -8
@@ -1,19 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class BatchDeletionJob
|
3
|
-
@queue = :batch
|
4
|
-
|
5
|
-
def self.perform(batch_id)
|
6
|
-
batch = Batch.find(batch_id)
|
7
|
-
batch.status = Batch::STATUS_DELETING
|
8
|
-
batch.save!
|
9
|
-
batch.destroy!
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.before_enqueue_set_status(batch_id)
|
13
|
-
batch = Batch.find(batch_id)
|
14
|
-
batch.status = Batch::STATUS_QUEUED_FOR_DELETION
|
15
|
-
batch.save
|
16
|
-
end
|
17
|
-
|
18
|
-
end
|
19
|
-
end
|
@@ -1,11 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class BatchObjectsProcessorJob
|
3
|
-
@queue = :batch
|
4
|
-
|
5
|
-
def self.perform(batch_object_ids, operator_id)
|
6
|
-
operator = User.find(operator_id)
|
7
|
-
ProcessBatchObjects.new(batch_object_ids: batch_object_ids, operator: operator).execute
|
8
|
-
end
|
9
|
-
|
10
|
-
end
|
11
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
|
3
|
-
class BatchObjectRole < ActiveRecord::Base
|
4
|
-
belongs_to :batch_object, :inverse_of => :batch_object_roles
|
5
|
-
|
6
|
-
OPERATION_ADD = "ADD".freeze # Add the principal and role to the object in the indicated scope
|
7
|
-
OPERATIONS = [ OPERATION_ADD ].freeze
|
8
|
-
|
9
|
-
validates :operation, inclusion: { in: OPERATIONS }
|
10
|
-
validate :valid_role, if: :operation_requires_valid_role?
|
11
|
-
validate :policy_role_scope_only_for_collections, if: "role_scope == Ddr::Auth::Roles::POLICY_SCOPE"
|
12
|
-
|
13
|
-
def operation_requires_valid_role?
|
14
|
-
[ OPERATION_ADD ].include? operation
|
15
|
-
end
|
16
|
-
|
17
|
-
def valid_role
|
18
|
-
Ddr::Auth::Roles::Role.build(type: role_type, agent: agent, scope: role_scope).valid?
|
19
|
-
end
|
20
|
-
|
21
|
-
def policy_role_scope_only_for_collections
|
22
|
-
errors.add(:role_scope, "policy role scope is valid only for Collections") unless batch_object.model == "Collection"
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
end
|
data/app/models/ddr/batch/log.rb
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class Log
|
3
|
-
|
4
|
-
DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
|
5
|
-
|
6
|
-
class << self
|
7
|
-
|
8
|
-
def logger(batch_id)
|
9
|
-
loggr = Logger.new(File.open(file_path(batch_id), File::WRONLY | File::APPEND | File::CREAT))
|
10
|
-
loggr.level = Ddr::Batch.processor_logging_level
|
11
|
-
loggr.datetime_format = "%Y-%m-%d %H:%M:%S.L"
|
12
|
-
loggr.formatter = proc do |severity, datetime, progname, msg|
|
13
|
-
"#{datetime} #{severity}: #{msg}\n"
|
14
|
-
end
|
15
|
-
loggr
|
16
|
-
end
|
17
|
-
|
18
|
-
def clear_log(batch_id)
|
19
|
-
log_file_path = file_path(batch_id)
|
20
|
-
FileUtils.remove(log_file_path) if File.exists?(log_file_path)
|
21
|
-
end
|
22
|
-
|
23
|
-
def file_path(batch_id)
|
24
|
-
File.join(DEFAULT_LOG_DIR, "batch_#{batch_id}_log.txt")
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
@@ -1,87 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class MonitorBatchFinished
|
3
|
-
|
4
|
-
class << self
|
5
|
-
def call(*args)
|
6
|
-
event = ActiveSupport::Notifications::Event.new(*args)
|
7
|
-
batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
|
8
|
-
batch_finished(batch)
|
9
|
-
if batch.outcome == Ddr::Batch::Batch::OUTCOME_SUCCESS
|
10
|
-
ActiveSupport::Notifications.instrument('success.batch.batch.ddr', batch_id: batch.id)
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
private
|
15
|
-
|
16
|
-
def batch_finished(batch)
|
17
|
-
log_batch_finish(batch)
|
18
|
-
update_batch(batch)
|
19
|
-
send_email(batch) if batch.user && batch.user.email
|
20
|
-
end
|
21
|
-
|
22
|
-
def log_batch_finish(batch)
|
23
|
-
logger = Ddr::Batch::Log.logger(batch.id)
|
24
|
-
logger.info "====== Summary ======"
|
25
|
-
results_tracker = results(batch)
|
26
|
-
results_tracker.keys.each do |type|
|
27
|
-
results_tracker[type].keys.each do |model|
|
28
|
-
log_result(results_tracker, type, model, logger)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
logger.close
|
32
|
-
end
|
33
|
-
|
34
|
-
def results(batch)
|
35
|
-
results_tracker = Hash.new
|
36
|
-
batch.batch_objects.each do |batch_object|
|
37
|
-
track_result(results_tracker, batch_object)
|
38
|
-
end
|
39
|
-
results_tracker
|
40
|
-
end
|
41
|
-
|
42
|
-
def track_result(results_tracker, batch_object)
|
43
|
-
type = batch_object.type
|
44
|
-
model = batch_object.model || "Missing Model"
|
45
|
-
results_tracker[type] = Hash.new unless results_tracker.has_key?(type)
|
46
|
-
results_tracker[type][model] = Hash.new unless results_tracker[type].has_key?(model)
|
47
|
-
results_tracker[type][model][:successes] = 0 unless results_tracker[type][model].has_key?(:successes)
|
48
|
-
results_tracker[type][model][:successes] += 1 if batch_object.verified
|
49
|
-
end
|
50
|
-
|
51
|
-
def log_result(results_tracker, type, model, logger)
|
52
|
-
verb = type_verb(type)
|
53
|
-
count = results_tracker[type][model][:successes]
|
54
|
-
logger.info "#{verb} #{ActionController::Base.helpers.pluralize(count, model)}"
|
55
|
-
end
|
56
|
-
|
57
|
-
def type_verb(type)
|
58
|
-
case type
|
59
|
-
when Ddr::Batch::IngestBatchObject.name
|
60
|
-
"Ingested"
|
61
|
-
when Ddr::Batch::UpdateBatchObject.name
|
62
|
-
"Updated"
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def update_batch(batch)
|
67
|
-
outcome = batch.success_count.eql?(batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
|
68
|
-
logfile = File.new(Ddr::Batch::Log.file_path(batch.id))
|
69
|
-
batch.update!(stop: DateTime.now,
|
70
|
-
status: Batch::STATUS_FINISHED,
|
71
|
-
outcome: outcome,
|
72
|
-
logfile: logfile)
|
73
|
-
end
|
74
|
-
|
75
|
-
def send_email(batch)
|
76
|
-
begin
|
77
|
-
Ddr::Batch::BatchProcessorRunMailer.send_notification(batch).deliver!
|
78
|
-
rescue => e
|
79
|
-
Rails.logger.error("An error occurred while attempting to send a notification for batch #{batch.id}")
|
80
|
-
Rails.logger.error(e.message)
|
81
|
-
Rails.logger.error(e.backtrace)
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
end
|
87
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class MonitorBatchObjectHandled
|
3
|
-
|
4
|
-
class << self
|
5
|
-
def call(*args)
|
6
|
-
event = ActiveSupport::Notifications::Event.new(*args)
|
7
|
-
batch_object = BatchObject.find(event.payload[:batch_object_id])
|
8
|
-
batch = batch_object.batch
|
9
|
-
if event.payload[:exception].present?
|
10
|
-
record_batch_object_exception(batch_object, event.payload[:exception])
|
11
|
-
end
|
12
|
-
batch_object_handled(batch_object, batch)
|
13
|
-
end
|
14
|
-
|
15
|
-
private
|
16
|
-
|
17
|
-
def record_batch_object_exception(batch_object, exception_info)
|
18
|
-
batch_object_exception_msg = I18n.t('ddr.batch.errors.batch_object_processing', error_msg: exception_info[1])
|
19
|
-
Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
|
20
|
-
level: Logger::ERROR,
|
21
|
-
message: batch_object_exception_msg)
|
22
|
-
end
|
23
|
-
|
24
|
-
def batch_object_handled(batch_object, batch)
|
25
|
-
log_batch_object_messages(batch_object, batch.id)
|
26
|
-
batch_object.update!(handled: true)
|
27
|
-
unless batch.unhandled_objects?
|
28
|
-
ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def log_batch_object_messages(batch_object, batch_id)
|
33
|
-
logger = Ddr::Batch::Log.logger(batch_id)
|
34
|
-
batch_object.batch_object_messages.each do |message|
|
35
|
-
logger.add(message.level) { "Batch Object #{batch_object.id}: #{message.message}" }
|
36
|
-
end
|
37
|
-
logger.close
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
end
|
42
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class MonitorBatchStarted
|
3
|
-
|
4
|
-
class << self
|
5
|
-
def call(*args)
|
6
|
-
event = ActiveSupport::Notifications::Event.new(*args)
|
7
|
-
batch = Ddr::Batch::Batch.find(event.payload[:batch_id])
|
8
|
-
batch_started(batch)
|
9
|
-
end
|
10
|
-
|
11
|
-
private
|
12
|
-
|
13
|
-
def batch_started(batch)
|
14
|
-
clear_logs(batch)
|
15
|
-
log_batch_start(batch)
|
16
|
-
update_batch(batch)
|
17
|
-
end
|
18
|
-
|
19
|
-
def clear_logs(batch)
|
20
|
-
# delete any previously existing filesystem log file for this batch
|
21
|
-
Ddr::Batch::Log.clear_log(batch.id)
|
22
|
-
# remove any existing attached log file from the Batch ActiveRecord object
|
23
|
-
batch.logfile.clear
|
24
|
-
end
|
25
|
-
|
26
|
-
def log_batch_start(batch)
|
27
|
-
logger = Ddr::Batch::Log.logger(batch.id)
|
28
|
-
logger.info "Collection: #{batch.collection_title}" if batch.collection_title.present?
|
29
|
-
logger.info "Batch id: #{batch.id}"
|
30
|
-
logger.info "Batch name: #{batch.name}" if name
|
31
|
-
logger.info "Batch size: #{batch.batch_objects.size}"
|
32
|
-
logger.close
|
33
|
-
end
|
34
|
-
|
35
|
-
def update_batch(batch)
|
36
|
-
batch.update!(start: DateTime.now,
|
37
|
-
status: Ddr::Batch::Batch::STATUS_RUNNING,
|
38
|
-
version: VERSION)
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|
@@ -1,60 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class ProcessBatch
|
3
|
-
|
4
|
-
attr_accessor :batch, :operator_id
|
5
|
-
|
6
|
-
def initialize(batch_id:, operator_id:)
|
7
|
-
@batch = Ddr::Batch::Batch.find(batch_id)
|
8
|
-
@operator_id = operator_id
|
9
|
-
end
|
10
|
-
|
11
|
-
def execute
|
12
|
-
ActiveSupport::Notifications.instrument('started.batch.batch.ddr', batch_id: batch.id)
|
13
|
-
batch.batch_objects.each do |batch_object|
|
14
|
-
case
|
15
|
-
when batch_object.is_a?(IngestBatchObject)
|
16
|
-
handle_ingest_batch_object(batch_object)
|
17
|
-
when batch_object.is_a?(UpdateBatchObject)
|
18
|
-
handle_update_batch_object(batch_object)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
def handle_ingest_batch_object(batch_object)
|
24
|
-
case batch_object.model
|
25
|
-
when 'Collection'
|
26
|
-
ingest_collection_object(batch_object)
|
27
|
-
when 'Item'
|
28
|
-
enqueue_item_component_ingest(batch_object)
|
29
|
-
when 'Component'
|
30
|
-
# skip -- will be handled along with associated Item
|
31
|
-
when 'Target', 'Attachment'
|
32
|
-
Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def handle_update_batch_object(batch_object)
|
37
|
-
Resque.enqueue(BatchObjectsProcessorJob, [ batch_object.id ], operator_id)
|
38
|
-
end
|
39
|
-
|
40
|
-
def ingest_collection_object(batch_object)
|
41
|
-
# Collection batch objects are processed synchronously because they need to exist in the repository
|
42
|
-
# prior to the processing of any objects (e.g., Item, Component, Target) associated with them.
|
43
|
-
# If the Collection batch object does not process successfully, consider the batch finished (albeit unsuccessfully)
|
44
|
-
# and raise an exception.
|
45
|
-
unless ProcessBatchObject.new(batch_object_id: batch_object.id, operator: User.find(operator_id)).execute
|
46
|
-
ActiveSupport::Notifications.instrument('finished.batch.batch.ddr', batch_id: batch.id)
|
47
|
-
raise Ddr::Batch::BatchObjectProcessingError, batch_object.id
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
def enqueue_item_component_ingest(batch_object)
|
52
|
-
query = [ "object = '#{batch_object.pid}'",
|
53
|
-
"batch_object_relationships.name = '#{Ddr::Batch::BatchObjectRelationship::RELATIONSHIP_PARENT}'",
|
54
|
-
"batches.id = #{batch_object.batch.id}" ].join(' AND ')
|
55
|
-
recs = Ddr::Batch::BatchObjectRelationship.joins(batch_object: :batch).where(query)
|
56
|
-
batch_object_ids = recs.map { |rec| rec.batch_object.id }.unshift(batch_object.id)
|
57
|
-
Resque.enqueue(BatchObjectsProcessorJob, batch_object_ids, operator_id)
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,46 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class ProcessBatchObject
|
3
|
-
|
4
|
-
attr_reader :batch_object_id, :operator
|
5
|
-
|
6
|
-
def initialize(batch_object_id:, operator:)
|
7
|
-
@batch_object_id = batch_object_id
|
8
|
-
@operator = operator
|
9
|
-
end
|
10
|
-
|
11
|
-
def execute
|
12
|
-
ActiveSupport::Notifications.instrument("handled.batchobject.batch.ddr",
|
13
|
-
batch_object_id: batch_object_id) do |payload|
|
14
|
-
batch_object = BatchObject.find(batch_object_id)
|
15
|
-
# Validate batch object
|
16
|
-
errors = batch_object.validate
|
17
|
-
# Process batch object or record validation errors
|
18
|
-
if errors.empty?
|
19
|
-
process(batch_object, operator)
|
20
|
-
else
|
21
|
-
record_errors(batch_object, errors)
|
22
|
-
end
|
23
|
-
# return true if batch_object was processed; otherwise, false
|
24
|
-
batch_object.processed? ? true : false
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
def process(batch_object, operator)
|
29
|
-
batch_object.update!(validated: true)
|
30
|
-
batch_object.process(operator)
|
31
|
-
batch_object.update!(processed: true)
|
32
|
-
results_message = batch_object.results_message
|
33
|
-
Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
|
34
|
-
level: results_message.level,
|
35
|
-
message: results_message.message)
|
36
|
-
end
|
37
|
-
|
38
|
-
def record_errors(batch_object, errors)
|
39
|
-
errors.each do |error|
|
40
|
-
Ddr::Batch::BatchObjectMessage.create!(batch_object: batch_object,
|
41
|
-
level: Logger::ERROR,
|
42
|
-
message: error)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
module Ddr::Batch
|
2
|
-
class ProcessBatchObjects
|
3
|
-
|
4
|
-
attr_reader :batch_object_ids, :operator
|
5
|
-
|
6
|
-
def initialize(batch_object_ids:, operator:)
|
7
|
-
@batch_object_ids = batch_object_ids
|
8
|
-
@operator = operator
|
9
|
-
end
|
10
|
-
|
11
|
-
def execute
|
12
|
-
# Assume successful processing of all batch objects until proven otherwise.
|
13
|
-
success = true
|
14
|
-
batch_object_ids.each do |batch_object_id|
|
15
|
-
batch_object = Ddr::Batch::BatchObject.find(batch_object_id)
|
16
|
-
# Skip batch objects that have already been successfully processed. This is useful when this service is
|
17
|
-
# called within the context of a BatchObjectsProcessorJob, that job fails, and the failed job is retried.
|
18
|
-
unless batch_object.verified?
|
19
|
-
# Once any batch object included in this job fails to process successfully, do not attempt to process
|
20
|
-
# any remaining batch objects included in this job. Instead, mark them as "handled" so the batch knows
|
21
|
-
# it's not waiting on them to be handled before it can consider itself "finished".
|
22
|
-
# The use case prompting this behavior is a job containing an Item ingest batch object plus one or more
|
23
|
-
# associated Component ingest batch objects. If the Item batch object fails to process correctly, we don't
|
24
|
-
# want to attempt to process the Component batch objects.
|
25
|
-
# In the preceding use case, we could skip the remaining batch objects only if the failed batch object is an
|
26
|
-
# Item but there might be future cases in which we don't want to process the remaining batch objects in the
|
27
|
-
# job regardless of which batch object fails. The failure of any batch object to process should be rare
|
28
|
-
# enough that it doesn't seem harmful to cover this potential broader use case in the current code.
|
29
|
-
if success
|
30
|
-
success = ProcessBatchObject.new(batch_object_id: batch_object.id, operator: operator).execute
|
31
|
-
else
|
32
|
-
batch_object.update!(handled: true)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
end
|
39
|
-
end
|
@@ -1,9 +0,0 @@
|
|
1
|
-
##
|
2
|
-
## Subscriptions to ActiveSupport::Notifications instrumentation events
|
3
|
-
##
|
4
|
-
|
5
|
-
# Batch Processing events
|
6
|
-
ActiveSupport::Notifications.subscribe('started.batch.batch.ddr', Ddr::Batch::MonitorBatchStarted)
|
7
|
-
ActiveSupport::Notifications.subscribe('handled.batchobject.batch.ddr', Ddr::Batch::MonitorBatchObjectHandled)
|
8
|
-
ActiveSupport::Notifications.subscribe('finished.batch.batch.ddr', Ddr::Batch::MonitorBatchFinished)
|
9
|
-
|