fedora-migrate 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +128 -0
- data/.rubocop_todo.yml +9 -0
- data/.travis.yml +3 -2
- data/Gemfile +2 -0
- data/fedora-migrate.gemspec +1 -0
- data/lib/fedora-migrate.rb +3 -5
- data/lib/fedora_migrate/content_mover.rb +7 -11
- data/lib/fedora_migrate/datastream_mover.rb +21 -24
- data/lib/fedora_migrate/datastream_verification.rb +15 -17
- data/lib/fedora_migrate/dates_mover.rb +0 -2
- data/lib/fedora_migrate/file_configurator.rb +3 -5
- data/lib/fedora_migrate/hooks.rb +0 -2
- data/lib/fedora_migrate/logger.rb +8 -11
- data/lib/fedora_migrate/migration_options.rb +6 -7
- data/lib/fedora_migrate/migration_report.rb +18 -22
- data/lib/fedora_migrate/mover.rb +4 -6
- data/lib/fedora_migrate/object_mover.rb +28 -34
- data/lib/fedora_migrate/permissions.rb +8 -10
- data/lib/fedora_migrate/permissions_mover.rb +7 -11
- data/lib/fedora_migrate/rdf_datastream_mover.rb +1 -2
- data/lib/fedora_migrate/rels_ext_datastream_mover.rb +29 -31
- data/lib/fedora_migrate/repository_migrator.rb +40 -43
- data/lib/fedora_migrate/rights_metadata.rb +109 -114
- data/lib/fedora_migrate/rubydora_connection.rb +4 -5
- data/lib/fedora_migrate/target_constructor.rb +19 -22
- data/lib/fedora_migrate/version.rb +1 -1
- data/spec/integration/content_versions_spec.rb +12 -14
- data/spec/integration/custom_target_spec.rb +44 -0
- data/spec/integration/fedora3_interface_spec.rb +7 -11
- data/spec/integration/missing_relationships_spec.rb +8 -10
- data/spec/integration/object_migration_spec.rb +20 -31
- data/spec/integration/permission_migration_spec.rb +4 -6
- data/spec/integration/rdf_migration_spec.rb +3 -6
- data/spec/integration/relationship_migration_spec.rb +6 -7
- data/spec/integration/repository_migration_spec.rb +14 -19
- data/spec/integration/versions_spec.rb +6 -8
- data/spec/spec_helper.rb +3 -3
- data/spec/support/example_model.rb +23 -25
- data/spec/unit/content_mover_spec.rb +21 -23
- data/spec/unit/datastream_mover_spec.rb +10 -14
- data/spec/unit/datastream_verification_spec.rb +7 -9
- data/spec/unit/dates_mover_spec.rb +3 -4
- data/spec/unit/fedora_migrate_spec.rb +2 -6
- data/spec/unit/file_configurator_spec.rb +4 -8
- data/spec/unit/migration_options_spec.rb +1 -3
- data/spec/unit/migration_report_spec.rb +5 -6
- data/spec/unit/mover_spec.rb +10 -12
- data/spec/unit/object_mover_spec.rb +9 -16
- data/spec/unit/permissions_mover_spec.rb +8 -11
- data/spec/unit/rels_ext_datastream_mover_spec.rb +4 -6
- data/spec/unit/repository_migrator_spec.rb +12 -14
- data/spec/unit/rubydora_connection_spec.rb +3 -5
- data/spec/unit/target_constructor_spec.rb +10 -16
- data/tasks/dev.rake +9 -1
- metadata +21 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 77fed55ecdbf64c57627ad65017d99be9f1424a1
|
4
|
+
data.tar.gz: fb0bfbb69e5d0efd99521bee4c3434218ae8f672
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4943665d395ae6e0d45f152f00eca3d7a9075c876be7757ec146d63b16b420db67ce3be520546bdbd0ce3cff1e10a6fc3071ff05c656459cc68e1a40cbcf5a34
|
7
|
+
data.tar.gz: 23f37bbb636630151e125384ddc00ef2c198209d0fa1aa2983abeb16dc5b07e34a1470ad26b1451ac4b8326708e2c48fbbcd05ba40baabb120300ce1fb915105
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,128 @@
|
|
1
|
+
require: rubocop-rspec
|
2
|
+
|
3
|
+
inherit_from: .rubocop_todo.yml
|
4
|
+
|
5
|
+
AllCops:
|
6
|
+
TargetRubyVersion: 2.2
|
7
|
+
DisplayCopNames: true
|
8
|
+
Include:
|
9
|
+
- '**/Rakefile'
|
10
|
+
|
11
|
+
Metrics/LineLength:
|
12
|
+
Enabled: false
|
13
|
+
|
14
|
+
Metrics/AbcSize:
|
15
|
+
Enabled: false
|
16
|
+
|
17
|
+
Metrics/CyclomaticComplexity:
|
18
|
+
Exclude:
|
19
|
+
- 'lib/fedora_migrate/rights_metadata.rb'
|
20
|
+
|
21
|
+
Metrics/PerceivedComplexity:
|
22
|
+
Exclude:
|
23
|
+
- 'lib/fedora_migrate/rights_metadata.rb'
|
24
|
+
|
25
|
+
Metrics/MethodLength:
|
26
|
+
Enabled: false
|
27
|
+
|
28
|
+
Metrics/ClassLength:
|
29
|
+
Exclude:
|
30
|
+
- 'lib/fedora_migrate/rights_metadata.rb'
|
31
|
+
|
32
|
+
Metrics/ModuleLength:
|
33
|
+
Exclude:
|
34
|
+
|
35
|
+
Style/BlockDelimiters:
|
36
|
+
Exclude:
|
37
|
+
- 'spec/**/*'
|
38
|
+
|
39
|
+
Style/BlockEndNewline:
|
40
|
+
Exclude:
|
41
|
+
- 'spec/**/*'
|
42
|
+
|
43
|
+
Style/MultilineBlockLayout:
|
44
|
+
Exclude:
|
45
|
+
- 'spec/**/*'
|
46
|
+
|
47
|
+
Style/IndentationConsistency:
|
48
|
+
EnforcedStyle: rails
|
49
|
+
|
50
|
+
Style/CollectionMethods:
|
51
|
+
PreferredMethods:
|
52
|
+
collect: 'map'
|
53
|
+
collect!: 'map!'
|
54
|
+
inject: 'reduce'
|
55
|
+
detect: 'find'
|
56
|
+
find_all: 'select'
|
57
|
+
|
58
|
+
Style/WordArray:
|
59
|
+
Enabled: false
|
60
|
+
|
61
|
+
Style/RegexpLiteral:
|
62
|
+
Enabled: false
|
63
|
+
|
64
|
+
Style/StringLiterals:
|
65
|
+
Enabled: false
|
66
|
+
|
67
|
+
Style/ClassAndModuleChildren:
|
68
|
+
Enabled: false
|
69
|
+
|
70
|
+
Style/Documentation:
|
71
|
+
Enabled: false
|
72
|
+
|
73
|
+
Style/HashSyntax:
|
74
|
+
Exclude:
|
75
|
+
|
76
|
+
Style/LineEndConcatenation:
|
77
|
+
Exclude:
|
78
|
+
|
79
|
+
Style/PredicateName:
|
80
|
+
Exclude:
|
81
|
+
|
82
|
+
Style/GlobalVars:
|
83
|
+
Exclude:
|
84
|
+
- 'spec/**/*'
|
85
|
+
|
86
|
+
Style/SingleLineBlockParams:
|
87
|
+
Enabled: false
|
88
|
+
|
89
|
+
Style/ClassVars:
|
90
|
+
Exclude:
|
91
|
+
|
92
|
+
Style/SignalException:
|
93
|
+
Enabled: false
|
94
|
+
|
95
|
+
Rails/Output:
|
96
|
+
Exclude:
|
97
|
+
|
98
|
+
Rails/Date:
|
99
|
+
Enabled: false
|
100
|
+
|
101
|
+
Rails/TimeZone:
|
102
|
+
Enabled: false
|
103
|
+
|
104
|
+
RSpec/ExampleWording:
|
105
|
+
CustomTransform:
|
106
|
+
be: is
|
107
|
+
have: has
|
108
|
+
not: does not
|
109
|
+
NOT: does NOT
|
110
|
+
IgnoredWords:
|
111
|
+
- only
|
112
|
+
|
113
|
+
RSpec/FilePath:
|
114
|
+
Enabled: false
|
115
|
+
|
116
|
+
RSpec/InstanceVariable:
|
117
|
+
Enabled: false
|
118
|
+
|
119
|
+
RSpec/DescribeClass:
|
120
|
+
Exclude:
|
121
|
+
|
122
|
+
Style/FileName:
|
123
|
+
Exclude:
|
124
|
+
- 'lib/fedora-migrate.rb'
|
125
|
+
|
126
|
+
Style/VariableName:
|
127
|
+
Exclude:
|
128
|
+
- 'lib/fedora_migrate/permissions_mover.rb'
|
data/.rubocop_todo.yml
ADDED
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/fedora-migrate.gemspec
CHANGED
data/lib/fedora-migrate.rb
CHANGED
@@ -5,7 +5,7 @@ require "hydra/head"
|
|
5
5
|
require "rubydora"
|
6
6
|
|
7
7
|
# Loads rake tasks
|
8
|
-
Dir[File.expand_path(File.join(File.dirname(__FILE__),"tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
|
8
|
+
Dir[File.expand_path(File.join(File.dirname(__FILE__), "tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
|
9
9
|
|
10
10
|
module FedoraMigrate
|
11
11
|
extend ActiveSupport::Autoload
|
@@ -45,19 +45,17 @@ module FedoraMigrate
|
|
45
45
|
@source ||= FedoraMigrate::RubydoraConnection.new(fedora_config.credentials)
|
46
46
|
end
|
47
47
|
|
48
|
-
def find
|
48
|
+
def find(id)
|
49
49
|
FedoraMigrate.source.connection.find(id)
|
50
50
|
end
|
51
51
|
|
52
|
-
def migrate_repository
|
52
|
+
def migrate_repository(args)
|
53
53
|
migrator = FedoraMigrate::RepositoryMigrator.new(args[:namespace], args[:options])
|
54
54
|
migrator.migrate_objects
|
55
55
|
migrator.migrate_relationships
|
56
56
|
migrator
|
57
57
|
end
|
58
|
-
|
59
58
|
end
|
60
59
|
|
61
60
|
self.configurator ||= FedoraMigrate::FileConfigurator.new
|
62
|
-
|
63
61
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class ContentMover < Mover
|
3
|
-
|
4
3
|
include DatastreamVerification
|
5
4
|
|
6
5
|
class Report
|
@@ -36,13 +35,13 @@ module FedoraMigrate
|
|
36
35
|
end
|
37
36
|
|
38
37
|
def insert_date_created_by_application
|
39
|
-
result = perform_sparql_insert
|
38
|
+
result = perform_sparql_insert
|
40
39
|
report.original_date = source.createDate.iso8601
|
41
40
|
report.error = "There was a problem with sparql #{result.status} #{result.body}" unless result.status == 204
|
42
41
|
end
|
43
42
|
|
44
43
|
def sparql_insert
|
45
|
-
<<-EOF
|
44
|
+
<<-EOF
|
46
45
|
PREFIX premis: <http://www.loc.gov/premis/rdf/v1#>
|
47
46
|
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
48
47
|
DELETE WHERE { ?s premis:hasDateCreatedByApplication ?o } ;
|
@@ -55,17 +54,14 @@ EOF
|
|
55
54
|
|
56
55
|
private
|
57
56
|
|
58
|
-
|
59
|
-
|
60
|
-
|
57
|
+
def perform_sparql_insert
|
58
|
+
ActiveFedora.fedora.connection.patch(target.metadata.metadata_uri, sparql_insert, "Content-Type" => "application/sparql-update")
|
59
|
+
end
|
61
60
|
|
62
|
-
|
63
|
-
|
61
|
+
def nil_source
|
62
|
+
return unless source.content.nil?
|
64
63
|
report.error = "Nil source -- it's probably defined in the target but not present in the source"
|
65
64
|
true
|
66
65
|
end
|
67
|
-
end
|
68
|
-
|
69
66
|
end
|
70
|
-
|
71
67
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class DatastreamMover < Mover
|
3
|
-
|
4
3
|
attr_accessor :versionable
|
5
4
|
|
6
5
|
def post_initialize
|
@@ -28,33 +27,31 @@ module FedoraMigrate
|
|
28
27
|
|
29
28
|
private
|
30
29
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
30
|
+
def migrate_datastream
|
31
|
+
if versionable?
|
32
|
+
migrate_versions
|
33
|
+
else
|
34
|
+
migrate_current
|
35
|
+
end
|
36
36
|
end
|
37
|
-
end
|
38
|
-
|
39
|
-
# Reload the target, otherwise the checksum is nil
|
40
|
-
def migrate_current
|
41
|
-
migrate_content
|
42
|
-
target.reload if report.last.success?
|
43
|
-
end
|
44
37
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
target.create_version if report.last.success? && !application_creates_versions?
|
38
|
+
# Reload the target, otherwise the checksum is nil
|
39
|
+
def migrate_current
|
40
|
+
migrate_content
|
41
|
+
target.reload if report.last.success?
|
50
42
|
end
|
51
|
-
end
|
52
43
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
44
|
+
# Rubydora stores the versions array as the most recent first. We explicitly sort them according to createDate
|
45
|
+
def migrate_versions
|
46
|
+
source.versions.sort { |a, b| a.createDate <=> b.createDate }.each do |version|
|
47
|
+
migrate_content(version)
|
48
|
+
target.create_version if report.last.success? && !application_creates_versions?
|
49
|
+
end
|
50
|
+
end
|
57
51
|
|
52
|
+
def migrate_content(datastream = nil)
|
53
|
+
datastream ||= source
|
54
|
+
report << FedoraMigrate::ContentMover.new(datastream, target).migrate
|
55
|
+
end
|
58
56
|
end
|
59
|
-
|
60
57
|
end
|
@@ -1,34 +1,32 @@
|
|
1
1
|
module FedoraMigrate::DatastreamVerification
|
2
|
-
|
3
2
|
attr_accessor :datastream
|
4
3
|
|
5
|
-
def valid?
|
4
|
+
def valid?(datastream = nil)
|
6
5
|
@datastream = datastream || @source
|
7
|
-
|
6
|
+
matching_checksums? || matching_nokogiri_checksums?
|
8
7
|
end
|
9
8
|
|
10
|
-
def
|
9
|
+
def matching_checksums?
|
11
10
|
datastream.checksum == target_checksum || checksum(datastream.content) == target_checksum
|
12
11
|
end
|
13
12
|
|
14
|
-
def
|
13
|
+
def matching_nokogiri_checksums?
|
15
14
|
return false unless datastream.mimeType == "text/xml"
|
16
15
|
checksum(Nokogiri::XML(datastream.content).to_xml) == checksum(Nokogiri::XML(target_content).to_xml)
|
17
16
|
end
|
18
17
|
|
19
|
-
private
|
18
|
+
private
|
20
19
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
# In some cases, the data is in ldp_source but target.content is empty, so we check both places
|
26
|
-
def target_content
|
27
|
-
target.content.empty? ? target.ldp_source.content : target.content
|
28
|
-
end
|
20
|
+
def target_checksum
|
21
|
+
target.digest.first.to_s.split(/:/).last
|
22
|
+
end
|
29
23
|
|
30
|
-
|
31
|
-
|
32
|
-
|
24
|
+
# In some cases, the data is in ldp_source but target.content is empty, so we check both places
|
25
|
+
def target_content
|
26
|
+
target.content.empty? ? target.ldp_source.content : target.content
|
27
|
+
end
|
33
28
|
|
29
|
+
def checksum(content)
|
30
|
+
Digest::SHA1.hexdigest(content)
|
31
|
+
end
|
34
32
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class DatesMover < Mover
|
3
|
-
|
4
3
|
Report = Struct.new(:uploaded, :modified)
|
5
4
|
|
6
5
|
def migrate
|
@@ -22,6 +21,5 @@ module FedoraMigrate
|
|
22
21
|
target.date_modified = source.lastModifiedDate
|
23
22
|
report.modified = source.lastModifiedDate
|
24
23
|
end
|
25
|
-
|
26
24
|
end
|
27
25
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class FileConfigurator < ActiveFedora::FileConfigurator
|
3
|
-
|
4
3
|
def fedora3_config
|
5
4
|
load_fedora3_config
|
6
5
|
@fedora_config
|
@@ -13,8 +12,8 @@ module FedoraMigrate
|
|
13
12
|
|
14
13
|
begin
|
15
14
|
config_erb = ERB.new(IO.read(@fedora_config_path)).result(binding)
|
16
|
-
rescue
|
17
|
-
raise("fedora.yml was found, but could not be parsed with ERB. \n#{
|
15
|
+
rescue StandardError
|
16
|
+
raise("fedora.yml was found, but could not be parsed with ERB. \n#{$ERROR_INFO.inspect}")
|
18
17
|
end
|
19
18
|
|
20
19
|
begin
|
@@ -27,8 +26,7 @@ module FedoraMigrate
|
|
27
26
|
config = fedora_yml.symbolize_keys
|
28
27
|
|
29
28
|
cfg = config[ActiveFedora.environment.to_sym] || {}
|
30
|
-
@fedora_config = cfg.
|
29
|
+
@fedora_config = cfg.is_a?(Array) ? cfg.map(&:symbolize_keys) : cfg.symbolize_keys
|
31
30
|
end
|
32
|
-
|
33
31
|
end
|
34
32
|
end
|
data/lib/fedora_migrate/hooks.rb
CHANGED
@@ -5,7 +5,6 @@
|
|
5
5
|
# you application and substitute methods for the ones listed below
|
6
6
|
module FedoraMigrate
|
7
7
|
module Hooks
|
8
|
-
|
9
8
|
# Called from FedoraMigrate::ObjectMover
|
10
9
|
def before_object_migration
|
11
10
|
end
|
@@ -29,6 +28,5 @@ module FedoraMigrate
|
|
29
28
|
# Called from FedoraMigrate::DatastreamMover
|
30
29
|
def after_datastream_migration
|
31
30
|
end
|
32
|
-
|
33
31
|
end
|
34
32
|
end
|
@@ -1,28 +1,26 @@
|
|
1
1
|
module FedoraMigrate
|
2
2
|
class Logger
|
3
|
-
|
4
3
|
class << self
|
5
|
-
|
6
|
-
def method_missing method_name, *arguments, &block
|
4
|
+
def method_missing(method_name, *arguments, &block)
|
7
5
|
logger.send(method_name, *arguments, &block)
|
8
6
|
rescue
|
9
7
|
super
|
10
8
|
end
|
11
9
|
|
12
|
-
def respond_to?(method_name,
|
10
|
+
def respond_to?(method_name, _include_private = false)
|
13
11
|
logger.respond_to? method_name
|
14
12
|
end
|
15
13
|
|
16
|
-
def info
|
17
|
-
super("FedoraMigrate INFO: ##{caller_locations(1,1)[0].label} " + msg)
|
14
|
+
def info(msg)
|
15
|
+
super("FedoraMigrate INFO: ##{caller_locations(1, 1)[0].label} " + msg)
|
18
16
|
end
|
19
17
|
|
20
|
-
def warn
|
21
|
-
super("FedoraMigrate WARN: ##{caller_locations(1,1)[0].label} " + msg)
|
18
|
+
def warn(msg)
|
19
|
+
super("FedoraMigrate WARN: ##{caller_locations(1, 1)[0].label} " + msg)
|
22
20
|
end
|
23
21
|
|
24
|
-
def fatal
|
25
|
-
super("FedoraMigrate FATAL: ##{caller_locations(1,1)[0].label} " + msg)
|
22
|
+
def fatal(msg)
|
23
|
+
super("FedoraMigrate FATAL: ##{caller_locations(1, 1)[0].label} " + msg)
|
26
24
|
end
|
27
25
|
|
28
26
|
private
|
@@ -30,7 +28,6 @@ module FedoraMigrate
|
|
30
28
|
def logger
|
31
29
|
ActiveFedora::Base.logger || ::Logger.new(STDOUT)
|
32
30
|
end
|
33
|
-
|
34
31
|
end
|
35
32
|
end
|
36
33
|
end
|