darlingtonia 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/lib/darlingtonia.rb +1 -0
- data/lib/darlingtonia/hyrax_record_importer.rb +18 -5
- data/lib/darlingtonia/importer.rb +15 -3
- data/lib/darlingtonia/metadata_only_stack.rb +70 -0
- data/lib/darlingtonia/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63a317635545c8808e56b95fcad54851481952e89268fcf3230eba94a885c203
|
4
|
+
data.tar.gz: 5e76830148cf211c69fab7e7f6b455a9a06a912e73bd023dab6e1369964c8c25
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3e86824312574df7b9630c1fd0f8d42416fe917309cea097912497f443d1e824b998dae3c97af18f76dc53a38e6af3f5ca54053bb3bf2423bc7ab9fd7b19245b
|
7
|
+
data.tar.gz: cbe2deaca3be8d5da290f611c4d22a7d8abf91f2764cb40361da36551b8e0629563f50a30abae2723d2a91ac1c688f06ec24b6b7400f59fb0d61dabdb17b3451
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
3.2.0 - Tue Mar 5, 2019
|
2
|
+
|
3
|
+
* Allow the importer to receive log settings from elsewhere, so it can log
|
4
|
+
success messages with all the other logs.
|
5
|
+
|
6
|
+
* When a record is imported a second time (i.e., when there is already a
|
7
|
+
record matching the deduplication field in the repository), only update
|
8
|
+
its metadata and collection membership. Use a stripped down actor stack
|
9
|
+
that only performs these actions.
|
10
|
+
|
11
|
+
* Log a special message if we attempt an empty import.
|
12
|
+
|
13
|
+
* Fix logging error where id number wasn't printing to logs on updates.
|
14
|
+
|
1
15
|
3.1.0 - Tue Feb 26, 2019
|
2
16
|
|
3
17
|
New Feature: `HyraxRecordImporter` now accepts a `deduplication_field` in the
|
data/lib/darlingtonia.rb
CHANGED
@@ -180,8 +180,18 @@ module Darlingtonia
|
|
180
180
|
|
181
181
|
private
|
182
182
|
|
183
|
+
# Build a pared down actor stack that will not re-attach files,
|
184
|
+
# or set workflow, or do anything except update metadata.
|
185
|
+
# TODO: We should be able to set an environment variable that would allow updates to go through the regular
|
186
|
+
# actor stack instead of the stripped down one, in case we want to re-import files.
|
187
|
+
def metadata_only_middleware
|
188
|
+
terminator = Hyrax::Actors::Terminator.new
|
189
|
+
Darlingtonia::MetadataOnlyStack.build_stack.build(terminator)
|
190
|
+
end
|
191
|
+
|
183
192
|
# Update an existing object using the Hyrax actor stack
|
184
193
|
# We assume the object was created as expected if the actor stack returns true.
|
194
|
+
# Note that for now the update stack will only update metadata and update collection membership, it will not re-import files.
|
185
195
|
def update_for(existing_record:, update_record:)
|
186
196
|
info_stream << "event: record_update_started, batch_id: #{batch_id}, collection_id: #{collection_id}, #{deduplication_field}: #{update_record.respond_to?(deduplication_field) ? update_record.send(deduplication_field) : update_record}"
|
187
197
|
additional_attrs = {
|
@@ -192,13 +202,16 @@ module Darlingtonia
|
|
192
202
|
# Ensure nothing is passed in the files field,
|
193
203
|
# since this is reserved for Hyrax and is where uploaded_files will be attached
|
194
204
|
attrs.delete(:files)
|
205
|
+
|
206
|
+
# We aren't using the attach remote files actor, so make sure any remote files are removed from the params before we try to save the object.
|
207
|
+
attrs.delete(:remote_files)
|
208
|
+
|
195
209
|
based_near = attrs.delete(:based_near)
|
196
210
|
attrs = attrs.merge(based_near_attributes: based_near_attributes(based_near)) unless based_near.nil? || based_near.empty?
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
info_stream << "event: record_updated, batch_id: #{batch_id}, record_id: #{existing_record.id}, collection_id: #{collection_id}, #{deduplication_field}: #{existing_record.respond_to?(deduplication_field) ? existing_record.send(deduplication_field) : existing_record}"
|
211
|
+
|
212
|
+
actor_env = Hyrax::Actors::Environment.new(existing_record, ::Ability.new(@depositor), attrs)
|
213
|
+
if metadata_only_middleware.update(actor_env)
|
214
|
+
info_stream << "event: record_updated, batch_id: #{batch_id}, record_id: #{existing_record.id}, collection_id: #{collection_id}, #{deduplication_field}: #{existing_record.respond_to?(deduplication_field) ? existing_record.send(deduplication_field)&.first : existing_record}"
|
202
215
|
@success_count += 1
|
203
216
|
else
|
204
217
|
existing_record.errors.each do |attr, msg|
|
@@ -31,10 +31,17 @@ module Darlingtonia
|
|
31
31
|
# records.
|
32
32
|
# @param record_importer [RecordImporter] An object to handle import of
|
33
33
|
# each record
|
34
|
-
def initialize(parser:, record_importer: RecordImporter.new)
|
34
|
+
def initialize(parser:, record_importer: RecordImporter.new, info_stream: Darlingtonia.config.default_info_stream, error_stream: Darlingtonia.config.default_error_stream)
|
35
35
|
self.parser = parser
|
36
36
|
self.record_importer = record_importer
|
37
|
-
@info_stream =
|
37
|
+
@info_stream = info_stream
|
38
|
+
@error_stream = error_stream
|
39
|
+
end
|
40
|
+
|
41
|
+
# Do not attempt to run an import if there are no records. Instead, just write to the log.
|
42
|
+
def no_records_message
|
43
|
+
@info_stream << "event: empty_import, batch_id: #{record_importer.batch_id}"
|
44
|
+
@error_stream << "event: empty_import, batch_id: #{record_importer.batch_id}"
|
38
45
|
end
|
39
46
|
|
40
47
|
##
|
@@ -42,8 +49,13 @@ module Darlingtonia
|
|
42
49
|
#
|
43
50
|
# @return [void]
|
44
51
|
def import
|
52
|
+
no_records_message && return unless records.count.positive?
|
53
|
+
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
54
|
+
@info_stream << "event: start_import, batch_id: #{record_importer.batch_id}, expecting to import #{records.count} records."
|
45
55
|
records.each { |record| record_importer.import(record: record) }
|
46
|
-
|
56
|
+
end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
57
|
+
elapsed_time = end_time - start_time
|
58
|
+
@info_stream << "event: finish_import, batch_id: #{record_importer.batch_id}, successful_record_count: #{record_importer.success_count}, failed_record_count: #{record_importer.failure_count}, elapsed_time: #{elapsed_time}, elapsed_time_per_record: #{elapsed_time / records.count}"
|
47
59
|
end
|
48
60
|
end
|
49
61
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
class MetadataOnlyStack
|
5
|
+
def self.build_stack
|
6
|
+
ActionDispatch::MiddlewareStack.new.tap do |middleware|
|
7
|
+
# Wrap everything in a database transaction, if the save of the resource
|
8
|
+
# fails then roll back any database AdminSetChangeSet
|
9
|
+
middleware.use Hyrax::Actors::TransactionalRequest
|
10
|
+
|
11
|
+
# Ensure you are mutating the most recent version
|
12
|
+
# middleware.use Hyrax::Actors::OptimisticLockValidator
|
13
|
+
|
14
|
+
# Attach files from a URI (for BrowseEverything)
|
15
|
+
# middleware.use Hyrax::Actors::CreateWithRemoteFilesActor
|
16
|
+
|
17
|
+
# Attach files uploaded in the form to the UploadsController
|
18
|
+
# In Californica, for command line import,
|
19
|
+
# we are using the CreateWithFilesActor to attach
|
20
|
+
# local files with a file:// url, not via the UploadsController,
|
21
|
+
# so we never use the CreateWithFilesActor
|
22
|
+
# middleware.use Hyrax::Actors::CreateWithFilesActor
|
23
|
+
|
24
|
+
# Add/remove the resource to/from a collection
|
25
|
+
middleware.use Hyrax::Actors::CollectionsMembershipActor
|
26
|
+
|
27
|
+
# Add/remove to parent work
|
28
|
+
# middleware.use Hyrax::Actors::AddToWorkActor
|
29
|
+
|
30
|
+
# Add/remove children (works or file_sets)
|
31
|
+
# middleware.use Hyrax::Actors::AttachMembersActor
|
32
|
+
|
33
|
+
# Set the order of the children (works or file_sets)
|
34
|
+
# middleware.use Hyrax::Actors::ApplyOrderActor
|
35
|
+
|
36
|
+
# Sets the default admin set if they didn't supply one
|
37
|
+
# middleware.use Hyrax::Actors::DefaultAdminSetActor
|
38
|
+
|
39
|
+
# Decode the private/public/institution on the form into permisisons on
|
40
|
+
# the model
|
41
|
+
# We aren't using this in Californica, we're just setting the visibility
|
42
|
+
# at import time
|
43
|
+
# middleware.use Hyrax::Actors::InterpretVisibilityActor
|
44
|
+
|
45
|
+
# Handles transfering ownership of works from one user to another
|
46
|
+
# We aren't using this in Californica
|
47
|
+
# middleware.use Hyrax::Actors::TransferRequestActor
|
48
|
+
|
49
|
+
# Copies default permissions from the PermissionTemplate to the work
|
50
|
+
# middleware.use Hyrax::Actors::ApplyPermissionTemplateActor
|
51
|
+
|
52
|
+
# Remove attached FileSets when destroying a work
|
53
|
+
# middleware.use Hyrax::Actors::CleanupFileSetsActor
|
54
|
+
|
55
|
+
# Destroys the trophies in the database when the work is destroyed
|
56
|
+
# middleware.use Hyrax::Actors::CleanupTrophiesActor
|
57
|
+
|
58
|
+
# Destroys the feature tag in the database when the work is destroyed
|
59
|
+
# middleware.use Hyrax::Actors::FeaturedWorkActor
|
60
|
+
|
61
|
+
# Persist the metadata changes on the resource
|
62
|
+
middleware.use Hyrax::Actors::ModelActor
|
63
|
+
|
64
|
+
# Start the workflow for this work
|
65
|
+
# middleware.use Hyrax::Actors::InitializeWorkflowActor
|
66
|
+
end
|
67
|
+
end
|
68
|
+
# rubocop:enable Metrics/MethodLength
|
69
|
+
end
|
70
|
+
end
|
data/lib/darlingtonia/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: darlingtonia
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Data Curation Experts
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-03-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: active-fedora
|
@@ -177,6 +177,7 @@ files:
|
|
177
177
|
- lib/darlingtonia/input_record.rb
|
178
178
|
- lib/darlingtonia/log_stream.rb
|
179
179
|
- lib/darlingtonia/metadata_mapper.rb
|
180
|
+
- lib/darlingtonia/metadata_only_stack.rb
|
180
181
|
- lib/darlingtonia/parser.rb
|
181
182
|
- lib/darlingtonia/parsers/csv_parser.rb
|
182
183
|
- lib/darlingtonia/record_importer.rb
|