ddr-batch 1.7.2 → 2.0.0.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +45 -0
  3. data/app/jobs/ddr/batch/batch_processor_job.rb +7 -2
  4. data/app/mailers/ddr/batch/batch_processor_run_mailer.rb +7 -24
  5. data/app/models/ddr/batch/batch.rb +18 -20
  6. data/app/models/ddr/batch/batch_ability_definitions.rb +14 -0
  7. data/app/models/ddr/batch/batch_object.rb +26 -35
  8. data/app/models/ddr/batch/batch_object_attribute.rb +7 -2
  9. data/app/models/ddr/batch/batch_object_datastream.rb +5 -0
  10. data/app/models/ddr/batch/batch_object_relationship.rb +1 -0
  11. data/app/models/ddr/batch/ingest_batch_object.rb +21 -25
  12. data/app/models/ddr/batch/update_batch_object.rb +10 -18
  13. data/app/scripts/ddr/batch/batch_processor.rb +152 -0
  14. data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb +5 -5
  15. data/app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb +5 -5
  16. data/config/locales/en.yml +2 -3
  17. data/lib/ddr/batch.rb +0 -6
  18. data/lib/ddr/batch/version.rb +1 -1
  19. metadata +31 -47
  20. data/app/jobs/ddr/batch/batch_deletion_job.rb +0 -19
  21. data/app/jobs/ddr/batch/batch_objects_processor_job.rb +0 -11
  22. data/app/models/ddr/batch/batch_object_message.rb +0 -8
  23. data/app/models/ddr/batch/batch_object_role.rb +0 -26
  24. data/app/models/ddr/batch/error.rb +0 -10
  25. data/app/models/ddr/batch/log.rb +0 -29
  26. data/app/services/ddr/batch/monitor_batch_finished.rb +0 -87
  27. data/app/services/ddr/batch/monitor_batch_object_handled.rb +0 -42
  28. data/app/services/ddr/batch/monitor_batch_started.rb +0 -43
  29. data/app/services/ddr/batch/process_batch.rb +0 -60
  30. data/app/services/ddr/batch/process_batch_object.rb +0 -46
  31. data/app/services/ddr/batch/process_batch_objects.rb +0 -39
  32. data/config/initializers/subscriptions.rb +0 -9
  33. data/db/migrate/20160816164010_create_batch_object_roles.rb +0 -15
  34. data/db/migrate/20161115191636_add_columns_to_batch_object.rb +0 -9
  35. data/db/migrate/20161116142512_create_batch_object_messages.rb +0 -13
  36. data/db/migrate/20161222192611_remove_columns_from_batch.rb +0 -13
  37. data/db/migrate/20171116183514_add_collection_columns_to_batch.rb +0 -8
@@ -37,29 +37,16 @@ module Ddr::Batch
37
37
  verified = false if value.eql?(VERIFICATION_FAIL)
38
38
  end
39
39
  update_attributes(:verified => verified)
40
- Ddr::Events::ValidationEvent.new.tap do |event|
41
- event.object = repo_object
42
- event.failure! unless verified
43
- event.summary = EVENT_SUMMARY % {
44
- label: "Object update validation",
45
- batch_id: id,
46
- identifier: identifier,
47
- model: model
48
- }
49
- event.detail = verification_outcome_detail.join("\n")
50
- event.save!
51
- end
52
40
  repo_object
53
41
  end
54
42
  end
55
43
 
56
44
  def results_message
57
45
  if pid
58
- message_level = verified ? Logger::INFO : Logger::WARN
59
- verification_result = verified ? "Verified" : "VERIFICATION FAILURE"
60
- ProcessingResultsMessage.new(message_level, "Updated #{pid}...#{verification_result}")
46
+ verification_result = (verified ? "Verified" : "VERIFICATION FAILURE")
47
+ message = "Updated #{pid}...#{verification_result}"
61
48
  else
62
- ProcessingResultsMessage.new(Logger::ERROR, "Attempt to update #{model} #{identifier} FAILED")
49
+ message = "Attempt to update #{model} #{identifier} FAILED"
63
50
  end
64
51
  end
65
52
 
@@ -73,7 +60,6 @@ module Ddr::Batch
73
60
  repo_object = nil
74
61
  begin
75
62
  repo_object = ActiveFedora::Base.find(pid)
76
- update!(model: repo_object.class.name) unless model.present?
77
63
  batch_object_attributes.each do |a|
78
64
  repo_object = case
79
65
  when a.operation.eql?(BatchObjectAttribute::OPERATION_ADD)
@@ -90,9 +76,15 @@ module Ddr::Batch
90
76
  populate_datastream(repo_object, d)
91
77
  end
92
78
  end
93
- repo_object.save!(user: user, comment: event_log_comment)
79
+ if repo_object.save
80
+ repo_object.notify_event(:update, user: user, comment: event_log_comment)
81
+ end
94
82
  rescue Exception => e
95
83
  logger.error("Error in updating repository object #{pid} for #{identifier} : : #{e}")
84
+ if batch.present?
85
+ batch.status = Batch::STATUS_RESTARTABLE
86
+ batch.save
87
+ end
96
88
  raise e
97
89
  end
98
90
  repo_object
@@ -0,0 +1,152 @@
1
+ module Ddr::Batch
2
+ class BatchProcessor
3
+
4
+ LOG_CONFIG_FILEPATH = File.join(Rails.root, 'config', 'log4r_batch_processor.yml')
5
+ DEFAULT_LOG_DIR = File.join(Rails.root, 'log')
6
+ DEFAULT_LOG_FILE = "batch_processor_log.txt"
7
+ PASS = "PASS"
8
+ FAIL = "FAIL"
9
+
10
+ # Options
11
+ # :log_dir - optional - directory for log file - default is given in DEFAULT_LOG_DIR
12
+ # :log_file - optional - filename of log file - default is given in DEFAULT_LOG_FILE
13
+ # :skip_validation - optional - whether to skip batch object validation step when processing - default is false
14
+ # :ignore_validation_errors - optional - whether to continue processing even if batch object validation errors occur - default is false
15
+ def initialize(batch, operator=nil, opts={})
16
+ @batch = batch
17
+ @operator = operator
18
+ @bp_log_dir = opts.fetch(:log_dir, DEFAULT_LOG_DIR)
19
+ @bp_log_file = opts.fetch(:log_file, DEFAULT_LOG_FILE)
20
+ @skip_validation = opts.fetch(:skip_validation, false)
21
+ @ignore_validation_errors = opts.fetch(:ignore_validation_errors, false)
22
+ end
23
+
24
+ def execute
25
+ config_logger
26
+ if @batch
27
+ initiate_batch_run
28
+ unless @skip_validation
29
+ valid_batch = validate_batch
30
+ @batch.update_attributes(status: Batch::STATUS_INVALID) unless valid_batch
31
+ end
32
+ if @skip_validation || @ignore_validation_errors || valid_batch
33
+ process_batch
34
+ end
35
+ close_batch_run
36
+ end
37
+ save_logfile
38
+ send_notification if @batch.user && @batch.user.email
39
+ end
40
+
41
+ private
42
+
43
+ def validate_batch
44
+ @batch.update_attributes(status: Batch::STATUS_VALIDATING)
45
+ valid = true
46
+ errors = @batch.validate
47
+ unless errors.empty?
48
+ valid = false
49
+ errors.each do |error|
50
+ message = "Batch Object Validation Error: #{error}"
51
+ @bp_log.error(message)
52
+ end
53
+ end
54
+ @batch.update_attributes(status: Batch::STATUS_RUNNING)
55
+ return valid
56
+ end
57
+
58
+ def process_batch
59
+ @batch.update_attributes(status: Batch::STATUS_PROCESSING, processing_step_start: DateTime.now)
60
+ @batch.batch_objects.each do |object|
61
+ begin
62
+ process_object(object)
63
+ rescue Exception => e
64
+ @bp_log.error(e.backtrace)
65
+ break
66
+ end
67
+ sleep 2
68
+ end
69
+ @batch.update_attributes(status: Batch::STATUS_RUNNING) if @batch.status == Batch::STATUS_PROCESSING
70
+ end
71
+
72
+ def initiate_batch_run
73
+ @bp_log.info "Batch id: #{@batch.id}"
74
+ @bp_log.info "Batch name: #{@batch.name}" if @batch.name
75
+ @bp_log.info "Batch size: #{@batch.batch_objects.size}"
76
+ @batch.logfile.clear # clear out any attached logfile
77
+ @batch.update_attributes(:start => DateTime.now,
78
+ :status => Batch::STATUS_RUNNING,
79
+ :version => VERSION)
80
+ @failures = 0
81
+ @successes = 0
82
+ @results_tracker = Hash.new
83
+ end
84
+
85
+ def close_batch_run
86
+ @batch.reload
87
+ @batch.failure = @failures
88
+ @batch.outcome = @successes.eql?(@batch.batch_objects.size) ? Batch::OUTCOME_SUCCESS : Batch::OUTCOME_FAILURE
89
+ if @batch.status.eql?(Batch::STATUS_RUNNING)
90
+ @batch.status = Batch::STATUS_FINISHED
91
+ end
92
+ @batch.stop = DateTime.now
93
+ @batch.success = @successes
94
+ @batch.save
95
+ @bp_log.info "====== Summary ======"
96
+ @results_tracker.keys.each do |type|
97
+ verb = case type
98
+ when IngestBatchObject.name
99
+ "Ingested"
100
+ when UpdateBatchObject.name
101
+ "Updated"
102
+ end
103
+ @results_tracker[type].keys.each do |model|
104
+ @bp_log.info "#{verb} #{@results_tracker[type][model][:successes]} #{model}"
105
+ end
106
+ end
107
+ end
108
+
109
+ def update_results_tracker(type, model, verified)
110
+ @results_tracker[type] = Hash.new unless @results_tracker.has_key?(type)
111
+ @results_tracker[type][model] = Hash.new unless @results_tracker[type].has_key?(model)
112
+ @results_tracker[type][model][:successes] = 0 unless @results_tracker[type][model].has_key?(:successes)
113
+ @results_tracker[type][model][:successes] += 1 if verified
114
+ end
115
+
116
+ def process_object(object)
117
+ @bp_log.debug "Processing object: #{object.identifier}"
118
+ repository_object = object.process(@operator)
119
+ update_results_tracker(object.type, repository_object.present? ? repository_object.class.name : object.model, object.verified)
120
+ if object.verified
121
+ @successes += 1
122
+ else
123
+ @failures += 1
124
+ end
125
+ message = object.results_message
126
+ @bp_log.info(message)
127
+ end
128
+
129
+ def config_logger
130
+ logconfig = Log4r::YamlConfigurator
131
+ logconfig['LOG_FILE'] = File.join(@bp_log_dir, @bp_log_file)
132
+ logconfig.load_yaml_file File.join(LOG_CONFIG_FILEPATH)
133
+ @bp_log = Log4r::Logger['batch_processor']
134
+ end
135
+
136
+ def save_logfile
137
+ @bp_log.outputters.each do |outputter|
138
+ @logfilename = outputter.filename if outputter.respond_to?(:filename)
139
+ end
140
+ @batch.update!({ logfile: File.new(@logfilename) }) if @logfilename
141
+ end
142
+
143
+ def send_notification
144
+ begin
145
+ BatchProcessorRunMailer.send_notification(@batch).deliver!
146
+ rescue
147
+ puts "An error occurred while attempting to send the notification."
148
+ end
149
+ end
150
+
151
+ end
152
+ end
@@ -22,13 +22,13 @@ td { text-align: right; }
22
22
  Outcome: <%= @batch.outcome %>
23
23
  </p>
24
24
  <p>
25
- Objects in batch: <%= @size %><br />
26
- Successfully processed: <%= @success %><br />
27
- Processing failures: <%= @handled - @success %><br />
28
- Unprocessed: <%= @size - @handled %>
25
+ Objects in batch: <%= @batch.batch_objects.size %><br />
26
+ Successfully processed: <%= @batch.success %><br />
27
+ Processing failures: <%= @batch.failure %><br />
28
+ Unprocessed: <%= @batch.batch_objects.size - @batch.success - @batch.failure %>
29
29
  </p>
30
30
  <p>
31
31
  See attachment for details of batch run.
32
32
  </p>
33
33
  </body>
34
- </html>
34
+ </html>
@@ -12,9 +12,9 @@ Stopped at: <%= @batch.stop %>
12
12
  Status: <%= @batch.status %>
13
13
  Outcome: <%= @batch.outcome %>
14
14
 
15
- Objects in batch: <%= @size %>
16
- Successfully processed: <%= @success %>
17
- Processing failures: <%= @handled - @success %>
18
- Unprocessed: <%= @size - @handled %>
15
+ Objects in batch: <%= @batch.batch_objects.size %>
16
+ Successfully processed: <%= @batch.success %>
17
+ Processing failures: <%= @batch.failure %>
18
+ Unprocessed: <%= @batch.batch_objects.size - @batch.success - @batch.failure %>
19
19
 
20
- See attachment for details of batch run.
20
+ See attachment for details of batch run.
@@ -2,8 +2,7 @@ en:
2
2
  ddr:
3
3
  batch:
4
4
  errors:
5
- prefix: "%{identifier}:"
6
- batch_object_processing: "Error processing batch object: %{error_msg}"
5
+ prefix: "%{identifier} [Database ID: %{id}]:"
7
6
  no_batches: "No %{type} batches found for your user account."
8
7
  web:
9
8
  action_names:
@@ -50,4 +49,4 @@ en:
50
49
  finished_batches:
51
50
  label: "Already Run"
52
51
  pending_batches:
53
- label: "Pending"
52
+ label: "Pending"
data/lib/ddr/batch.rb CHANGED
@@ -9,12 +9,6 @@ module Ddr
9
9
  extend ActiveSupport::Autoload
10
10
 
11
11
  autoload :BatchUser
12
- autoload :BatchObjectProcessingError, 'ddr/batch/error'
13
-
14
- # Logging level for batch processing - defaults to Logger::INFO
15
- mattr_accessor :processor_logging_level do
16
- Logger::INFO
17
- end
18
12
 
19
13
  def self.table_name_prefix
20
14
  end
@@ -1,5 +1,5 @@
1
1
  module Ddr
2
2
  module Batch
3
- VERSION = "1.7.2"
3
+ VERSION = "2.0.0.alpha.1"
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ddr-batch
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.2
4
+ version: 2.0.0.alpha.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jim Coble
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-08-31 00:00:00.000000000 Z
12
+ date: 2015-10-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rails
@@ -17,14 +17,14 @@ dependencies:
17
17
  requirements:
18
18
  - - "~>"
19
19
  - !ruby/object:Gem::Version
20
- version: '4.1'
20
+ version: 4.1.13
21
21
  type: :runtime
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - "~>"
26
26
  - !ruby/object:Gem::Version
27
- version: '4.1'
27
+ version: 4.1.13
28
28
  - !ruby/object:Gem::Dependency
29
29
  name: devise
30
30
  requirement: !ruby/object:Gem::Requirement
@@ -45,28 +45,42 @@ dependencies:
45
45
  requirements:
46
46
  - - "~>"
47
47
  - !ruby/object:Gem::Version
48
- version: '5.2'
48
+ version: 4.2.0
49
49
  type: :runtime
50
50
  prerelease: false
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
53
  - - "~>"
54
54
  - !ruby/object:Gem::Version
55
- version: '5.2'
55
+ version: 4.2.0
56
56
  - !ruby/object:Gem::Dependency
57
57
  name: ddr-models
58
58
  requirement: !ruby/object:Gem::Requirement
59
59
  requirements:
60
- - - "~>"
60
+ - - ">="
61
61
  - !ruby/object:Gem::Version
62
- version: '2.11'
62
+ version: '0'
63
63
  type: :runtime
64
64
  prerelease: false
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
- - - "~>"
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: log4r
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ type: :runtime
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ">="
68
82
  - !ruby/object:Gem::Version
69
- version: '2.11'
83
+ version: '0'
70
84
  - !ruby/object:Gem::Dependency
71
85
  name: sqlite3
72
86
  requirement: !ruby/object:Gem::Requirement
@@ -129,14 +143,14 @@ dependencies:
129
143
  requirements:
130
144
  - - "~>"
131
145
  - !ruby/object:Gem::Version
132
- version: '1.8'
146
+ version: '2.0'
133
147
  type: :development
134
148
  prerelease: false
135
149
  version_requirements: !ruby/object:Gem::Requirement
136
150
  requirements:
137
151
  - - "~>"
138
152
  - !ruby/object:Gem::Version
139
- version: '1.8'
153
+ version: '2.0'
140
154
  - !ruby/object:Gem::Dependency
141
155
  name: database_cleaner
142
156
  requirement: !ruby/object:Gem::Requirement
@@ -151,20 +165,6 @@ dependencies:
151
165
  - - ">="
152
166
  - !ruby/object:Gem::Version
153
167
  version: '0'
154
- - !ruby/object:Gem::Dependency
155
- name: byebug
156
- requirement: !ruby/object:Gem::Requirement
157
- requirements:
158
- - - ">="
159
- - !ruby/object:Gem::Version
160
- version: '0'
161
- type: :development
162
- prerelease: false
163
- version_requirements: !ruby/object:Gem::Requirement
164
- requirements:
165
- - - ">="
166
- - !ruby/object:Gem::Version
167
- version: '0'
168
168
  description: Batch processing for Duke Digital Repository
169
169
  email:
170
170
  - lib-drs@duke.edu
@@ -175,30 +175,19 @@ files:
175
175
  - LICENSE.txt
176
176
  - README.md
177
177
  - Rakefile
178
- - app/jobs/ddr/batch/batch_deletion_job.rb
179
- - app/jobs/ddr/batch/batch_objects_processor_job.rb
180
178
  - app/jobs/ddr/batch/batch_processor_job.rb
181
179
  - app/mailers/ddr/batch/batch_processor_run_mailer.rb
182
180
  - app/models/ddr/batch/batch.rb
181
+ - app/models/ddr/batch/batch_ability_definitions.rb
183
182
  - app/models/ddr/batch/batch_object.rb
184
183
  - app/models/ddr/batch/batch_object_attribute.rb
185
184
  - app/models/ddr/batch/batch_object_datastream.rb
186
- - app/models/ddr/batch/batch_object_message.rb
187
185
  - app/models/ddr/batch/batch_object_relationship.rb
188
- - app/models/ddr/batch/batch_object_role.rb
189
- - app/models/ddr/batch/error.rb
190
186
  - app/models/ddr/batch/ingest_batch_object.rb
191
- - app/models/ddr/batch/log.rb
192
187
  - app/models/ddr/batch/update_batch_object.rb
193
- - app/services/ddr/batch/monitor_batch_finished.rb
194
- - app/services/ddr/batch/monitor_batch_object_handled.rb
195
- - app/services/ddr/batch/monitor_batch_started.rb
196
- - app/services/ddr/batch/process_batch.rb
197
- - app/services/ddr/batch/process_batch_object.rb
198
- - app/services/ddr/batch/process_batch_objects.rb
188
+ - app/scripts/ddr/batch/batch_processor.rb
199
189
  - app/views/ddr/batch/batch_processor_run_mailer/send_notification.html.erb
200
190
  - app/views/ddr/batch/batch_processor_run_mailer/send_notification.text.erb
201
- - config/initializers/subscriptions.rb
202
191
  - config/locales/en.yml
203
192
  - config/routes.rb
204
193
  - db/migrate/20150828183839_create_batches.rb
@@ -206,11 +195,6 @@ files:
206
195
  - db/migrate/20150828202118_create_batch_object_attributes.rb
207
196
  - db/migrate/20150828202200_create_batch_object_datastreams.rb
208
197
  - db/migrate/20150828202240_create_batch_object_relationships.rb
209
- - db/migrate/20160816164010_create_batch_object_roles.rb
210
- - db/migrate/20161115191636_add_columns_to_batch_object.rb
211
- - db/migrate/20161116142512_create_batch_object_messages.rb
212
- - db/migrate/20161222192611_remove_columns_from_batch.rb
213
- - db/migrate/20171116183514_add_collection_columns_to_batch.rb
214
198
  - lib/ddr-batch.rb
215
199
  - lib/ddr/batch.rb
216
200
  - lib/ddr/batch/batch_user.rb
@@ -232,12 +216,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
232
216
  version: '0'
233
217
  required_rubygems_version: !ruby/object:Gem::Requirement
234
218
  requirements:
235
- - - ">="
219
+ - - ">"
236
220
  - !ruby/object:Gem::Version
237
- version: '0'
221
+ version: 1.3.1
238
222
  requirements: []
239
223
  rubyforge_project:
240
- rubygems_version: 2.6.14
224
+ rubygems_version: 2.4.3
241
225
  signing_key:
242
226
  specification_version: 4
243
227
  summary: Batch processing for Duke Digital Repository