fedora-migrate 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +128 -0
- data/.rubocop_todo.yml +9 -0
- data/.travis.yml +3 -2
- data/Gemfile +2 -0
- data/fedora-migrate.gemspec +1 -0
- data/lib/fedora-migrate.rb +3 -5
- data/lib/fedora_migrate/content_mover.rb +7 -11
- data/lib/fedora_migrate/datastream_mover.rb +21 -24
- data/lib/fedora_migrate/datastream_verification.rb +15 -17
- data/lib/fedora_migrate/dates_mover.rb +0 -2
- data/lib/fedora_migrate/file_configurator.rb +3 -5
- data/lib/fedora_migrate/hooks.rb +0 -2
- data/lib/fedora_migrate/logger.rb +8 -11
- data/lib/fedora_migrate/migration_options.rb +6 -7
- data/lib/fedora_migrate/migration_report.rb +18 -22
- data/lib/fedora_migrate/mover.rb +4 -6
- data/lib/fedora_migrate/object_mover.rb +28 -34
- data/lib/fedora_migrate/permissions.rb +8 -10
- data/lib/fedora_migrate/permissions_mover.rb +7 -11
- data/lib/fedora_migrate/rdf_datastream_mover.rb +1 -2
- data/lib/fedora_migrate/rels_ext_datastream_mover.rb +29 -31
- data/lib/fedora_migrate/repository_migrator.rb +40 -43
- data/lib/fedora_migrate/rights_metadata.rb +109 -114
- data/lib/fedora_migrate/rubydora_connection.rb +4 -5
- data/lib/fedora_migrate/target_constructor.rb +19 -22
- data/lib/fedora_migrate/version.rb +1 -1
- data/spec/integration/content_versions_spec.rb +12 -14
- data/spec/integration/custom_target_spec.rb +44 -0
- data/spec/integration/fedora3_interface_spec.rb +7 -11
- data/spec/integration/missing_relationships_spec.rb +8 -10
- data/spec/integration/object_migration_spec.rb +20 -31
- data/spec/integration/permission_migration_spec.rb +4 -6
- data/spec/integration/rdf_migration_spec.rb +3 -6
- data/spec/integration/relationship_migration_spec.rb +6 -7
- data/spec/integration/repository_migration_spec.rb +14 -19
- data/spec/integration/versions_spec.rb +6 -8
- data/spec/spec_helper.rb +3 -3
- data/spec/support/example_model.rb +23 -25
- data/spec/unit/content_mover_spec.rb +21 -23
- data/spec/unit/datastream_mover_spec.rb +10 -14
- data/spec/unit/datastream_verification_spec.rb +7 -9
- data/spec/unit/dates_mover_spec.rb +3 -4
- data/spec/unit/fedora_migrate_spec.rb +2 -6
- data/spec/unit/file_configurator_spec.rb +4 -8
- data/spec/unit/migration_options_spec.rb +1 -3
- data/spec/unit/migration_report_spec.rb +5 -6
- data/spec/unit/mover_spec.rb +10 -12
- data/spec/unit/object_mover_spec.rb +9 -16
- data/spec/unit/permissions_mover_spec.rb +8 -11
- data/spec/unit/rels_ext_datastream_mover_spec.rb +4 -6
- data/spec/unit/repository_migrator_spec.rb +12 -14
- data/spec/unit/rubydora_connection_spec.rb +3 -5
- data/spec/unit/target_constructor_spec.rb +10 -16
- data/tasks/dev.rake +9 -1
- metadata +21 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 77fed55ecdbf64c57627ad65017d99be9f1424a1
|
4
|
+
data.tar.gz: fb0bfbb69e5d0efd99521bee4c3434218ae8f672
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4943665d395ae6e0d45f152f00eca3d7a9075c876be7757ec146d63b16b420db67ce3be520546bdbd0ce3cff1e10a6fc3071ff05c656459cc68e1a40cbcf5a34
|
7
|
+
data.tar.gz: 23f37bbb636630151e125384ddc00ef2c198209d0fa1aa2983abeb16dc5b07e34a1470ad26b1451ac4b8326708e2c48fbbcd05ba40baabb120300ce1fb915105
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
require: rubocop-rspec
|
2
|
+
|
3
|
+
inherit_from: .rubocop_todo.yml
|
4
|
+
|
5
|
+
AllCops:
|
6
|
+
TargetRubyVersion: 2.2
|
7
|
+
DisplayCopNames: true
|
8
|
+
Include:
|
9
|
+
- '**/Rakefile'
|
10
|
+
|
11
|
+
Metrics/LineLength:
|
12
|
+
Enabled: false
|
13
|
+
|
14
|
+
Metrics/AbcSize:
|
15
|
+
Enabled: false
|
16
|
+
|
17
|
+
Metrics/CyclomaticComplexity:
|
18
|
+
Exclude:
|
19
|
+
- 'lib/fedora_migrate/rights_metadata.rb'
|
20
|
+
|
21
|
+
Metrics/PerceivedComplexity:
|
22
|
+
Exclude:
|
23
|
+
- 'lib/fedora_migrate/rights_metadata.rb'
|
24
|
+
|
25
|
+
Metrics/MethodLength:
|
26
|
+
Enabled: false
|
27
|
+
|
28
|
+
Metrics/ClassLength:
|
29
|
+
Exclude:
|
30
|
+
- 'lib/fedora_migrate/rights_metadata.rb'
|
31
|
+
|
32
|
+
Metrics/ModuleLength:
|
33
|
+
Exclude:
|
34
|
+
|
35
|
+
Style/BlockDelimiters:
|
36
|
+
Exclude:
|
37
|
+
- 'spec/**/*'
|
38
|
+
|
39
|
+
Style/BlockEndNewline:
|
40
|
+
Exclude:
|
41
|
+
- 'spec/**/*'
|
42
|
+
|
43
|
+
Style/MultilineBlockLayout:
|
44
|
+
Exclude:
|
45
|
+
- 'spec/**/*'
|
46
|
+
|
47
|
+
Style/IndentationConsistency:
|
48
|
+
EnforcedStyle: rails
|
49
|
+
|
50
|
+
Style/CollectionMethods:
|
51
|
+
PreferredMethods:
|
52
|
+
collect: 'map'
|
53
|
+
collect!: 'map!'
|
54
|
+
inject: 'reduce'
|
55
|
+
detect: 'find'
|
56
|
+
find_all: 'select'
|
57
|
+
|
58
|
+
Style/WordArray:
|
59
|
+
Enabled: false
|
60
|
+
|
61
|
+
Style/RegexpLiteral:
|
62
|
+
Enabled: false
|
63
|
+
|
64
|
+
Style/StringLiterals:
|
65
|
+
Enabled: false
|
66
|
+
|
67
|
+
Style/ClassAndModuleChildren:
|
68
|
+
Enabled: false
|
69
|
+
|
70
|
+
Style/Documentation:
|
71
|
+
Enabled: false
|
72
|
+
|
73
|
+
Style/HashSyntax:
|
74
|
+
Exclude:
|
75
|
+
|
76
|
+
Style/LineEndConcatenation:
|
77
|
+
Exclude:
|
78
|
+
|
79
|
+
Style/PredicateName:
|
80
|
+
Exclude:
|
81
|
+
|
82
|
+
Style/GlobalVars:
|
83
|
+
Exclude:
|
84
|
+
- 'spec/**/*'
|
85
|
+
|
86
|
+
Style/SingleLineBlockParams:
|
87
|
+
Enabled: false
|
88
|
+
|
89
|
+
Style/ClassVars:
|
90
|
+
Exclude:
|
91
|
+
|
92
|
+
Style/SignalException:
|
93
|
+
Enabled: false
|
94
|
+
|
95
|
+
Rails/Output:
|
96
|
+
Exclude:
|
97
|
+
|
98
|
+
Rails/Date:
|
99
|
+
Enabled: false
|
100
|
+
|
101
|
+
Rails/TimeZone:
|
102
|
+
Enabled: false
|
103
|
+
|
104
|
+
RSpec/ExampleWording:
|
105
|
+
CustomTransform:
|
106
|
+
be: is
|
107
|
+
have: has
|
108
|
+
not: does not
|
109
|
+
NOT: does NOT
|
110
|
+
IgnoredWords:
|
111
|
+
- only
|
112
|
+
|
113
|
+
RSpec/FilePath:
|
114
|
+
Enabled: false
|
115
|
+
|
116
|
+
RSpec/InstanceVariable:
|
117
|
+
Enabled: false
|
118
|
+
|
119
|
+
RSpec/DescribeClass:
|
120
|
+
Exclude:
|
121
|
+
|
122
|
+
Style/FileName:
|
123
|
+
Exclude:
|
124
|
+
- 'lib/fedora-migrate.rb'
|
125
|
+
|
126
|
+
Style/VariableName:
|
127
|
+
Exclude:
|
128
|
+
- 'lib/fedora_migrate/permissions_mover.rb'
|
data/.rubocop_todo.yml
ADDED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/fedora-migrate.gemspec
CHANGED
data/lib/fedora-migrate.rb
CHANGED
@@ -5,7 +5,7 @@ require "hydra/head"
|
|
5
5
|
require "rubydora"
|
6
6
|
|
7
7
|
# Loads rake tasks
|
8
|
-
Dir[File.expand_path(File.join(File.dirname(__FILE__),"tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
|
8
|
+
Dir[File.expand_path(File.join(File.dirname(__FILE__), "tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
|
9
9
|
|
10
10
|
module FedoraMigrate
|
11
11
|
extend ActiveSupport::Autoload
|
@@ -45,19 +45,17 @@ module FedoraMigrate
|
|
45
45
|
@source ||= FedoraMigrate::RubydoraConnection.new(fedora_config.credentials)
|
46
46
|
end
|
47
47
|
|
48
|
-
def find
|
48
|
+
def find(id)
|
49
49
|
FedoraMigrate.source.connection.find(id)
|
50
50
|
end
|
51
51
|
|
52
|
-
def migrate_repository
|
52
|
+
def migrate_repository(args)
|
53
53
|
migrator = FedoraMigrate::RepositoryMigrator.new(args[:namespace], args[:options])
|
54
54
|
migrator.migrate_objects
|
55
55
|
migrator.migrate_relationships
|
56
56
|
migrator
|
57
57
|
end
|
58
|
-
|
59
58
|
end
|
60
59
|
|
61
60
|
self.configurator ||= FedoraMigrate::FileConfigurator.new
|
62
|
-
|
63
61
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class ContentMover < Mover
|
3
|
-
|
4
3
|
include DatastreamVerification
|
5
4
|
|
6
5
|
class Report
|
@@ -36,13 +35,13 @@ module FedoraMigrate
|
|
36
35
|
end
|
37
36
|
|
38
37
|
def insert_date_created_by_application
|
39
|
-
result = perform_sparql_insert
|
38
|
+
result = perform_sparql_insert
|
40
39
|
report.original_date = source.createDate.iso8601
|
41
40
|
report.error = "There was a problem with sparql #{result.status} #{result.body}" unless result.status == 204
|
42
41
|
end
|
43
42
|
|
44
43
|
def sparql_insert
|
45
|
-
<<-EOF
|
44
|
+
<<-EOF
|
46
45
|
PREFIX premis: <http://www.loc.gov/premis/rdf/v1#>
|
47
46
|
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
48
47
|
DELETE WHERE { ?s premis:hasDateCreatedByApplication ?o } ;
|
@@ -55,17 +54,14 @@ EOF
|
|
55
54
|
|
56
55
|
private
|
57
56
|
|
58
|
-
|
59
|
-
|
60
|
-
|
57
|
+
def perform_sparql_insert
|
58
|
+
ActiveFedora.fedora.connection.patch(target.metadata.metadata_uri, sparql_insert, "Content-Type" => "application/sparql-update")
|
59
|
+
end
|
61
60
|
|
62
|
-
|
63
|
-
|
61
|
+
def nil_source
|
62
|
+
return unless source.content.nil?
|
64
63
|
report.error = "Nil source -- it's probably defined in the target but not present in the source"
|
65
64
|
true
|
66
65
|
end
|
67
|
-
end
|
68
|
-
|
69
66
|
end
|
70
|
-
|
71
67
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class DatastreamMover < Mover
|
3
|
-
|
4
3
|
attr_accessor :versionable
|
5
4
|
|
6
5
|
def post_initialize
|
@@ -28,33 +27,31 @@ module FedoraMigrate
|
|
28
27
|
|
29
28
|
private
|
30
29
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
30
|
+
def migrate_datastream
|
31
|
+
if versionable?
|
32
|
+
migrate_versions
|
33
|
+
else
|
34
|
+
migrate_current
|
35
|
+
end
|
36
36
|
end
|
37
|
-
end
|
38
|
-
|
39
|
-
# Reload the target, otherwise the checksum is nil
|
40
|
-
def migrate_current
|
41
|
-
migrate_content
|
42
|
-
target.reload if report.last.success?
|
43
|
-
end
|
44
37
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
target.create_version if report.last.success? && !application_creates_versions?
|
38
|
+
# Reload the target, otherwise the checksum is nil
|
39
|
+
def migrate_current
|
40
|
+
migrate_content
|
41
|
+
target.reload if report.last.success?
|
50
42
|
end
|
51
|
-
end
|
52
43
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
44
|
+
# Rubydora stores the versions array as the most recent first. We explicitly sort them according to createDate
|
45
|
+
def migrate_versions
|
46
|
+
source.versions.sort { |a, b| a.createDate <=> b.createDate }.each do |version|
|
47
|
+
migrate_content(version)
|
48
|
+
target.create_version if report.last.success? && !application_creates_versions?
|
49
|
+
end
|
50
|
+
end
|
57
51
|
|
52
|
+
def migrate_content(datastream = nil)
|
53
|
+
datastream ||= source
|
54
|
+
report << FedoraMigrate::ContentMover.new(datastream, target).migrate
|
55
|
+
end
|
58
56
|
end
|
59
|
-
|
60
57
|
end
|
@@ -1,34 +1,32 @@
|
|
1
1
|
module FedoraMigrate::DatastreamVerification
|
2
|
-
|
3
2
|
attr_accessor :datastream
|
4
3
|
|
5
|
-
def valid?
|
4
|
+
def valid?(datastream = nil)
|
6
5
|
@datastream = datastream || @source
|
7
|
-
|
6
|
+
matching_checksums? || matching_nokogiri_checksums?
|
8
7
|
end
|
9
8
|
|
10
|
-
def
|
9
|
+
def matching_checksums?
|
11
10
|
datastream.checksum == target_checksum || checksum(datastream.content) == target_checksum
|
12
11
|
end
|
13
12
|
|
14
|
-
def
|
13
|
+
def matching_nokogiri_checksums?
|
15
14
|
return false unless datastream.mimeType == "text/xml"
|
16
15
|
checksum(Nokogiri::XML(datastream.content).to_xml) == checksum(Nokogiri::XML(target_content).to_xml)
|
17
16
|
end
|
18
17
|
|
19
|
-
private
|
18
|
+
private
|
20
19
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
# In some cases, the data is in ldp_source but target.content is empty, so we check both places
|
26
|
-
def target_content
|
27
|
-
target.content.empty? ? target.ldp_source.content : target.content
|
28
|
-
end
|
20
|
+
def target_checksum
|
21
|
+
target.digest.first.to_s.split(/:/).last
|
22
|
+
end
|
29
23
|
|
30
|
-
|
31
|
-
|
32
|
-
|
24
|
+
# In some cases, the data is in ldp_source but target.content is empty, so we check both places
|
25
|
+
def target_content
|
26
|
+
target.content.empty? ? target.ldp_source.content : target.content
|
27
|
+
end
|
33
28
|
|
29
|
+
def checksum(content)
|
30
|
+
Digest::SHA1.hexdigest(content)
|
31
|
+
end
|
34
32
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class DatesMover < Mover
|
3
|
-
|
4
3
|
Report = Struct.new(:uploaded, :modified)
|
5
4
|
|
6
5
|
def migrate
|
@@ -22,6 +21,5 @@ module FedoraMigrate
|
|
22
21
|
target.date_modified = source.lastModifiedDate
|
23
22
|
report.modified = source.lastModifiedDate
|
24
23
|
end
|
25
|
-
|
26
24
|
end
|
27
25
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class FileConfigurator < ActiveFedora::FileConfigurator
|
3
|
-
|
4
3
|
def fedora3_config
|
5
4
|
load_fedora3_config
|
6
5
|
@fedora_config
|
@@ -13,8 +12,8 @@ module FedoraMigrate
|
|
13
12
|
|
14
13
|
begin
|
15
14
|
config_erb = ERB.new(IO.read(@fedora_config_path)).result(binding)
|
16
|
-
rescue
|
17
|
-
raise("fedora.yml was found, but could not be parsed with ERB. \n#{
|
15
|
+
rescue StandardError
|
16
|
+
raise("fedora.yml was found, but could not be parsed with ERB. \n#{$ERROR_INFO.inspect}")
|
18
17
|
end
|
19
18
|
|
20
19
|
begin
|
@@ -27,8 +26,7 @@ module FedoraMigrate
|
|
27
26
|
config = fedora_yml.symbolize_keys
|
28
27
|
|
29
28
|
cfg = config[ActiveFedora.environment.to_sym] || {}
|
30
|
-
@fedora_config = cfg.
|
29
|
+
@fedora_config = cfg.is_a?(Array) ? cfg.map(&:symbolize_keys) : cfg.symbolize_keys
|
31
30
|
end
|
32
|
-
|
33
31
|
end
|
34
32
|
end
|
data/lib/fedora_migrate/hooks.rb
CHANGED
@@ -5,7 +5,6 @@
|
|
5
5
|
# you application and substitute methods for the ones listed below
|
6
6
|
module FedoraMigrate
|
7
7
|
module Hooks
|
8
|
-
|
9
8
|
# Called from FedoraMigrate::ObjectMover
|
10
9
|
def before_object_migration
|
11
10
|
end
|
@@ -29,6 +28,5 @@ module FedoraMigrate
|
|
29
28
|
# Called from FedoraMigrate::DatastreamMover
|
30
29
|
def after_datastream_migration
|
31
30
|
end
|
32
|
-
|
33
31
|
end
|
34
32
|
end
|
@@ -1,28 +1,26 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class Logger
|
3
|
-
|
4
3
|
class << self
|
5
|
-
|
6
|
-
def method_missing method_name, *arguments, &block
|
4
|
+
def method_missing(method_name, *arguments, &block)
|
7
5
|
logger.send(method_name, *arguments, &block)
|
8
6
|
rescue
|
9
7
|
super
|
10
8
|
end
|
11
9
|
|
12
|
-
def respond_to?(method_name,
|
10
|
+
def respond_to?(method_name, _include_private = false)
|
13
11
|
logger.respond_to? method_name
|
14
12
|
end
|
15
13
|
|
16
|
-
def info
|
17
|
-
super("FedoraMigrate INFO: ##{caller_locations(1,1)[0].label} " + msg)
|
14
|
+
def info(msg)
|
15
|
+
super("FedoraMigrate INFO: ##{caller_locations(1, 1)[0].label} " + msg)
|
18
16
|
end
|
19
17
|
|
20
|
-
def warn
|
21
|
-
super("FedoraMigrate WARN: ##{caller_locations(1,1)[0].label} " + msg)
|
18
|
+
def warn(msg)
|
19
|
+
super("FedoraMigrate WARN: ##{caller_locations(1, 1)[0].label} " + msg)
|
22
20
|
end
|
23
21
|
|
24
|
-
def fatal
|
25
|
-
super("FedoraMigrate FATAL: ##{caller_locations(1,1)[0].label} " + msg)
|
22
|
+
def fatal(msg)
|
23
|
+
super("FedoraMigrate FATAL: ##{caller_locations(1, 1)[0].label} " + msg)
|
26
24
|
end
|
27
25
|
|
28
26
|
private
|
@@ -30,7 +28,6 @@ module FedoraMigrate
|
|
30
28
|
def logger
|
31
29
|
ActiveFedora::Base.logger || ::Logger.new(STDOUT)
|
32
30
|
end
|
33
|
-
|
34
31
|
end
|
35
32
|
end
|
36
33
|
end
|