fedora-migrate 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/lib/fedora-migrate.rb +3 -5
- data/lib/fedora_migrate/content_mover.rb +32 -10
- data/lib/fedora_migrate/datastream_mover.rb +4 -7
- data/lib/fedora_migrate/datastream_verification.rb +1 -3
- data/lib/fedora_migrate/dates_mover.rb +19 -6
- data/lib/fedora_migrate/migration_report.rb +40 -0
- data/lib/fedora_migrate/mover.rb +11 -14
- data/lib/fedora_migrate/object_mover.rb +25 -9
- data/lib/fedora_migrate/permissions_mover.rb +2 -2
- data/lib/fedora_migrate/rdf_datastream_mover.rb +1 -1
- data/lib/fedora_migrate/rels_ext_datastream_mover.rb +5 -2
- data/lib/fedora_migrate/repository_migrator.rb +35 -22
- data/lib/fedora_migrate/target_constructor.rb +1 -2
- data/lib/fedora_migrate/version.rb +1 -1
- data/lib/tasks/fedora-migrate.rake +8 -2
- data/spec/fixtures/failed-report.json +339 -0
- data/spec/fixtures/sample-report.json +166 -0
- data/spec/integration/missing_relationships_spec.rb +15 -6
- data/spec/integration/repository_migration_spec.rb +25 -6
- data/spec/unit/content_mover_spec.rb +17 -10
- data/spec/unit/datastream_verification_spec.rb +2 -8
- data/spec/unit/migration_report_spec.rb +58 -0
- data/spec/unit/object_mover_spec.rb +2 -2
- data/spec/unit/repository_migrator_spec.rb +39 -10
- data/spec/unit/rubydora_connection_spec.rb +4 -0
- data/tasks/dev.rake +9 -3
- metadata +10 -3
@@ -3,53 +3,63 @@ module FedoraMigrate
|
|
3
3
|
|
4
4
|
include MigrationOptions
|
5
5
|
|
6
|
-
attr_accessor :source_objects, :namespace, :
|
6
|
+
attr_accessor :source_objects, :namespace, :report
|
7
|
+
|
8
|
+
SingleObjectReport = Struct.new(:status, :object, :relationships)
|
7
9
|
|
8
10
|
def initialize namespace = nil, options = {}
|
9
11
|
@namespace = namespace || repository_namespace
|
10
12
|
@options = options
|
11
|
-
@
|
13
|
+
@report = MigrationReport.new(@options.fetch(:report, nil))
|
12
14
|
@source_objects = get_source_objects
|
13
15
|
conversion_options
|
14
16
|
end
|
15
17
|
|
16
|
-
# TODO: need a reporting mechanism for results (issue #4)
|
17
18
|
def migrate_objects
|
18
19
|
source_objects.each { |source| migrate_object(source) }
|
19
|
-
@failed == 0
|
20
20
|
end
|
21
21
|
|
22
|
-
# TODO: need a reporting mechanism for results (issue #4)
|
23
22
|
def migrate_relationships
|
24
|
-
return "Relationship migration halted because #{
|
23
|
+
return "Relationship migration halted because #{failures.to_s} objects didn't migrate successfully." if failures > 0 && not_forced?
|
25
24
|
source_objects.each { |source| migrate_relationship(source) }
|
26
|
-
@failed == 0
|
27
25
|
end
|
28
26
|
|
29
27
|
def get_source_objects
|
30
|
-
|
28
|
+
if report.empty?
|
29
|
+
FedoraMigrate.source.connection.search(nil).collect { |o| qualifying_object(o) }.compact
|
30
|
+
else
|
31
|
+
report.failed_objects.map { |o| FedoraMigrate.source.connection.find(o) }
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def failures
|
36
|
+
report.failed_objects.count
|
31
37
|
end
|
32
38
|
|
33
39
|
private
|
34
40
|
|
35
41
|
def migrate_object source
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
42
|
+
object_report = SingleObjectReport.new
|
43
|
+
begin
|
44
|
+
object_report.object = FedoraMigrate::ObjectMover.new(source, nil, options).migrate
|
45
|
+
object_report.status = true
|
46
|
+
rescue StandardError => e
|
47
|
+
object_report.object = e.inspect
|
48
|
+
object_report.status = false
|
49
|
+
end
|
50
|
+
report.results[source.pid] = object_report
|
41
51
|
end
|
42
52
|
|
43
53
|
def migrate_relationship source
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
[
|
54
|
+
relationship_report = find_or_create_single_object_report(source)
|
55
|
+
begin
|
56
|
+
relationship_report.relationships = FedoraMigrate::RelsExtDatastreamMover.new(source).migrate
|
57
|
+
relationship_report.status = true
|
58
|
+
rescue StandardError => e
|
59
|
+
relationship_report.relationships = e.inspect
|
60
|
+
relationship_report.status = false
|
61
|
+
end
|
62
|
+
report.results[source.pid] = relationship_report
|
53
63
|
end
|
54
64
|
|
55
65
|
def repository_namespace
|
@@ -61,5 +71,8 @@ module FedoraMigrate
|
|
61
71
|
return object if name.match(namespace)
|
62
72
|
end
|
63
73
|
|
74
|
+
def find_or_create_single_object_report source
|
75
|
+
report.results[source.pid] || SingleObjectReport.new
|
76
|
+
end
|
64
77
|
end
|
65
78
|
end
|
@@ -30,9 +30,8 @@ module FedoraMigrate
|
|
30
30
|
|
31
31
|
def vet model
|
32
32
|
@target = FedoraMigrate::Mover.id_component(model).constantize
|
33
|
-
Logger.info "using #{model} for target"
|
34
33
|
rescue NameError
|
35
|
-
Logger.
|
34
|
+
Logger.debug "rejecting #{model} for target"
|
36
35
|
end
|
37
36
|
|
38
37
|
end
|
@@ -8,8 +8,8 @@ namespace :fedora do
|
|
8
8
|
namespace :migrate do
|
9
9
|
desc "Migrates all objects in a Sufia-based application"
|
10
10
|
task sufia: :environment do
|
11
|
-
|
12
|
-
|
11
|
+
migrator = FedoraMigrate.migrate_repository(namespace: "sufia", options: {convert: "descMetadata"})
|
12
|
+
migrator.report.save
|
13
13
|
end
|
14
14
|
|
15
15
|
desc "Migrates only relationships in a Sufia-based application"
|
@@ -39,6 +39,12 @@ namespace :fedora do
|
|
39
39
|
raise "Please provide a pid, example changeme:1234" if args[:pid].nil?
|
40
40
|
FedoraMigrate::RelsExtDatastreamMover.new(FedoraMigrate.source.connection.find(args[:pid])).migrate
|
41
41
|
end
|
42
|
+
|
43
|
+
desc "Report the results of a migration"
|
44
|
+
task :report, [:file] => :environment do |t, args|
|
45
|
+
raise "Please provide a path to a report.json file" if args[:file].nil?
|
46
|
+
FedoraMigrate::MigrationReport.new(args[:file]).report_failures
|
47
|
+
end
|
42
48
|
|
43
49
|
end
|
44
50
|
|
@@ -0,0 +1,339 @@
|
|
1
|
+
{
|
2
|
+
"sufia:5m60qr94g": {
|
3
|
+
"status": false,
|
4
|
+
"object": "Sample object migration failure",
|
5
|
+
"relationships": [
|
6
|
+
|
7
|
+
]
|
8
|
+
},
|
9
|
+
"sufia:5m60qr95r": {
|
10
|
+
"status": true,
|
11
|
+
"object": {
|
12
|
+
"id": "5m60qr95r",
|
13
|
+
"class": "GenericFile",
|
14
|
+
"content_datastreams": [
|
15
|
+
{
|
16
|
+
"ds": "content",
|
17
|
+
"versions": [
|
18
|
+
{
|
19
|
+
"name": "file2.txt",
|
20
|
+
"mime_type": "text/plain",
|
21
|
+
"original_date": "2015-01-19T21:32:47Z"
|
22
|
+
}
|
23
|
+
]
|
24
|
+
},
|
25
|
+
{
|
26
|
+
"ds": "thumbnail",
|
27
|
+
"versions": [
|
28
|
+
{
|
29
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
30
|
+
}
|
31
|
+
]
|
32
|
+
},
|
33
|
+
{
|
34
|
+
"ds": "characterization",
|
35
|
+
"versions": [
|
36
|
+
{
|
37
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
38
|
+
}
|
39
|
+
]
|
40
|
+
}
|
41
|
+
],
|
42
|
+
"rdf_datastreams": [
|
43
|
+
{
|
44
|
+
"ds": "descMetadata",
|
45
|
+
"status": [
|
46
|
+
|
47
|
+
]
|
48
|
+
}
|
49
|
+
],
|
50
|
+
"permissions": [
|
51
|
+
"read_groups = []",
|
52
|
+
"edit_groups = []",
|
53
|
+
"discover_groups = []",
|
54
|
+
"read_users = []",
|
55
|
+
"edit_users = [\"awead@psu.edu\"]",
|
56
|
+
"discover_users = []"
|
57
|
+
],
|
58
|
+
"dates": {
|
59
|
+
"uploaded": null,
|
60
|
+
"modified": null
|
61
|
+
}
|
62
|
+
},
|
63
|
+
"relationships": [
|
64
|
+
"http://localhost:8983/fedora/rest/test/5m60qr95r--info:fedora/fedora-system:def/relations-external#isPartOf--http://localhost:8983/fedora/rest/test/5m60qr94g"
|
65
|
+
]
|
66
|
+
},
|
67
|
+
"sufia:5m60qr961": {
|
68
|
+
"status": true,
|
69
|
+
"object": {
|
70
|
+
"id": "5m60qr961",
|
71
|
+
"class": "GenericFile",
|
72
|
+
"content_datastreams": [
|
73
|
+
{
|
74
|
+
"ds": "content",
|
75
|
+
"versions": [
|
76
|
+
{
|
77
|
+
"name": "file1.txt",
|
78
|
+
"mime_type": "text/plain",
|
79
|
+
"original_date": "2015-01-19T21:32:49Z"
|
80
|
+
}
|
81
|
+
]
|
82
|
+
},
|
83
|
+
{
|
84
|
+
"ds": "thumbnail",
|
85
|
+
"versions": [
|
86
|
+
{
|
87
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
88
|
+
}
|
89
|
+
]
|
90
|
+
},
|
91
|
+
{
|
92
|
+
"ds": "characterization",
|
93
|
+
"versions": [
|
94
|
+
{
|
95
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
96
|
+
}
|
97
|
+
]
|
98
|
+
}
|
99
|
+
],
|
100
|
+
"rdf_datastreams": [
|
101
|
+
{
|
102
|
+
"ds": "descMetadata",
|
103
|
+
"status": [
|
104
|
+
|
105
|
+
]
|
106
|
+
}
|
107
|
+
],
|
108
|
+
"permissions": [
|
109
|
+
"read_groups = []",
|
110
|
+
"edit_groups = []",
|
111
|
+
"discover_groups = []",
|
112
|
+
"read_users = []",
|
113
|
+
"edit_users = [\"awead@psu.edu\"]",
|
114
|
+
"discover_users = []"
|
115
|
+
],
|
116
|
+
"dates": {
|
117
|
+
"uploaded": null,
|
118
|
+
"modified": null
|
119
|
+
}
|
120
|
+
},
|
121
|
+
"relationships": [
|
122
|
+
"http://localhost:8983/fedora/rest/test/5m60qr961--info:fedora/fedora-system:def/relations-external#isPartOf--http://localhost:8983/fedora/rest/test/5m60qr94g"
|
123
|
+
]
|
124
|
+
},
|
125
|
+
"sufia:5m60qr979": {
|
126
|
+
"status": true,
|
127
|
+
"object": {
|
128
|
+
"id": "5m60qr979",
|
129
|
+
"class": "Collection",
|
130
|
+
"content_datastreams": [
|
131
|
+
|
132
|
+
],
|
133
|
+
"rdf_datastreams": [
|
134
|
+
{
|
135
|
+
"ds": "descMetadata",
|
136
|
+
"status": [
|
137
|
+
|
138
|
+
]
|
139
|
+
}
|
140
|
+
],
|
141
|
+
"permissions": [
|
142
|
+
"read_groups = [\"public\"]",
|
143
|
+
"edit_groups = []",
|
144
|
+
"discover_groups = []",
|
145
|
+
"read_users = []",
|
146
|
+
"edit_users = [\"awead@psu.edu\"]",
|
147
|
+
"discover_users = []"
|
148
|
+
],
|
149
|
+
"dates": {
|
150
|
+
"uploaded": "2015-01-19T21:34:23.805Z",
|
151
|
+
"modified": "2015-03-03T18:48:51.16Z"
|
152
|
+
}
|
153
|
+
},
|
154
|
+
"relationships": [
|
155
|
+
"http://localhost:8983/fedora/rest/test/5m60qr979--info:fedora/fedora-system:def/relations-external#hasCollectionMember--http://localhost:8983/fedora/rest/test/5m60qr95r",
|
156
|
+
"http://localhost:8983/fedora/rest/test/5m60qr979--info:fedora/fedora-system:def/relations-external#hasCollectionMember--http://localhost:8983/fedora/rest/test/5m60qr961"
|
157
|
+
]
|
158
|
+
},
|
159
|
+
"sufia:rb68xc089": {
|
160
|
+
"status": false,
|
161
|
+
"object": "Sample object migration failure",
|
162
|
+
"relationships": [
|
163
|
+
|
164
|
+
]
|
165
|
+
},
|
166
|
+
"sufia:rb68xc09k": {
|
167
|
+
"status": true,
|
168
|
+
"object": {
|
169
|
+
"id": "rb68xc09k",
|
170
|
+
"class": "Batch",
|
171
|
+
"content_datastreams": [
|
172
|
+
|
173
|
+
],
|
174
|
+
"rdf_datastreams": [
|
175
|
+
|
176
|
+
],
|
177
|
+
"permissions": null,
|
178
|
+
"dates": {
|
179
|
+
"uploaded": null,
|
180
|
+
"modified": null
|
181
|
+
}
|
182
|
+
},
|
183
|
+
"relationships": [
|
184
|
+
|
185
|
+
]
|
186
|
+
},
|
187
|
+
"sufia:rb68xc10b": {
|
188
|
+
"status": true,
|
189
|
+
"object": {
|
190
|
+
"id": "rb68xc10b",
|
191
|
+
"class": "GenericFile",
|
192
|
+
"content_datastreams": [
|
193
|
+
{
|
194
|
+
"ds": "content",
|
195
|
+
"versions": [
|
196
|
+
|
197
|
+
]
|
198
|
+
},
|
199
|
+
{
|
200
|
+
"ds": "thumbnail",
|
201
|
+
"versions": [
|
202
|
+
{
|
203
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
204
|
+
}
|
205
|
+
]
|
206
|
+
},
|
207
|
+
{
|
208
|
+
"ds": "characterization",
|
209
|
+
"versions": [
|
210
|
+
{
|
211
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
212
|
+
}
|
213
|
+
]
|
214
|
+
}
|
215
|
+
],
|
216
|
+
"rdf_datastreams": [
|
217
|
+
|
218
|
+
],
|
219
|
+
"permissions": [
|
220
|
+
"read_groups = []",
|
221
|
+
"edit_groups = []",
|
222
|
+
"discover_groups = []",
|
223
|
+
"read_users = []",
|
224
|
+
"edit_users = [\"jilluser@example.com\"]",
|
225
|
+
"discover_users = []"
|
226
|
+
],
|
227
|
+
"dates": {
|
228
|
+
"uploaded": null,
|
229
|
+
"modified": null
|
230
|
+
}
|
231
|
+
},
|
232
|
+
"relationships": [
|
233
|
+
"http://localhost:8983/fedora/rest/test/rb68xc10b--info:fedora/fedora-system:def/relations-external#isPartOf--http://localhost:8983/fedora/rest/test/rb68xc09k"
|
234
|
+
]
|
235
|
+
},
|
236
|
+
"sufia:rb68xc11m": {
|
237
|
+
"status": true,
|
238
|
+
"object": {
|
239
|
+
"id": "rb68xc11m",
|
240
|
+
"class": "GenericFile",
|
241
|
+
"content_datastreams": [
|
242
|
+
{
|
243
|
+
"ds": "content",
|
244
|
+
"versions": [
|
245
|
+
|
246
|
+
]
|
247
|
+
},
|
248
|
+
{
|
249
|
+
"ds": "thumbnail",
|
250
|
+
"versions": [
|
251
|
+
{
|
252
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
253
|
+
}
|
254
|
+
]
|
255
|
+
},
|
256
|
+
{
|
257
|
+
"ds": "characterization",
|
258
|
+
"versions": [
|
259
|
+
{
|
260
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
261
|
+
}
|
262
|
+
]
|
263
|
+
}
|
264
|
+
],
|
265
|
+
"rdf_datastreams": [
|
266
|
+
|
267
|
+
],
|
268
|
+
"permissions": [
|
269
|
+
"read_groups = []",
|
270
|
+
"edit_groups = []",
|
271
|
+
"discover_groups = []",
|
272
|
+
"read_users = []",
|
273
|
+
"edit_users = [\"otherUser\"]",
|
274
|
+
"discover_users = []"
|
275
|
+
],
|
276
|
+
"dates": {
|
277
|
+
"uploaded": null,
|
278
|
+
"modified": null
|
279
|
+
}
|
280
|
+
},
|
281
|
+
"relationships": [
|
282
|
+
"http://localhost:8983/fedora/rest/test/rb68xc11m--info:fedora/fedora-system:def/relations-external#isPartOf--http://localhost:8983/fedora/rest/test/rb68xc09k"
|
283
|
+
]
|
284
|
+
},
|
285
|
+
"sufia:xp68km39w": {
|
286
|
+
"status": true,
|
287
|
+
"object": {
|
288
|
+
"id": "xp68km39w",
|
289
|
+
"class": "GenericFile",
|
290
|
+
"content_datastreams": [
|
291
|
+
{
|
292
|
+
"ds": "content",
|
293
|
+
"versions": [
|
294
|
+
|
295
|
+
]
|
296
|
+
},
|
297
|
+
{
|
298
|
+
"ds": "thumbnail",
|
299
|
+
"versions": [
|
300
|
+
{
|
301
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
302
|
+
}
|
303
|
+
]
|
304
|
+
},
|
305
|
+
{
|
306
|
+
"ds": "characterization",
|
307
|
+
"versions": [
|
308
|
+
{
|
309
|
+
"error": "Nil source -- it's probably defined in the target but not present in the source"
|
310
|
+
}
|
311
|
+
]
|
312
|
+
}
|
313
|
+
],
|
314
|
+
"rdf_datastreams": [
|
315
|
+
{
|
316
|
+
"ds": "descMetadata",
|
317
|
+
"status": [
|
318
|
+
|
319
|
+
]
|
320
|
+
}
|
321
|
+
],
|
322
|
+
"permissions": [
|
323
|
+
"read_groups = []",
|
324
|
+
"edit_groups = []",
|
325
|
+
"discover_groups = []",
|
326
|
+
"read_users = []",
|
327
|
+
"edit_users = [\"awead@psu.edu\"]",
|
328
|
+
"discover_users = []"
|
329
|
+
],
|
330
|
+
"dates": {
|
331
|
+
"uploaded": null,
|
332
|
+
"modified": null
|
333
|
+
}
|
334
|
+
},
|
335
|
+
"relationships": [
|
336
|
+
|
337
|
+
]
|
338
|
+
}
|
339
|
+
}
|