ddr-ingesttools 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/migrate_ddr_to_rdr.rb +8 -4
- data/bin/mint_manifest_arks.rb +1 -1
- data/config/locales/en.yml +1 -1
- data/lib/ddr/ingesttools/ddr_rdr_migrator/file_path_adder.rb +36 -3
- data/lib/ddr/ingesttools/ddr_rdr_migrator/migrator.rb +6 -5
- data/lib/ddr/ingesttools/version.rb +1 -1
- data/spec/ddr_rdr_migrator/file_path_adder_spec.rb +12 -1
- data/spec/ddr_rdr_migrator/migrator_spec.rb +7 -2
- data/spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt +3 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4e3de5f3f9741a14f8ee24a8665f02b8685f4b26
|
4
|
+
data.tar.gz: c2782cb6a29ed694230e714e6082587d67e8bdcc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 68380b51a8b4c28ef053d3758064edf1f9e69b70d6e0426460be8e13bfc7629608b5e6c2e5822b8d8c11beb39fd596ec06007362877a4e7477e6f0061144b97b
|
7
|
+
data.tar.gz: 2a81290d1b0b7f9ad41c284a8a78af914548a098ae5213475bcf51f81af8198fb0890df48fe86cb1519150fc29dc1f1cd2fa0edb76b3f04eab090e9940ad83f1
|
data/bin/migrate_ddr_to_rdr.rb
CHANGED
@@ -7,7 +7,7 @@ require 'optparse'
|
|
7
7
|
options = {}
|
8
8
|
|
9
9
|
puts I18n.t('marquee')
|
10
|
-
puts I18n.t('suite.name')
|
10
|
+
puts I18n.t('suite.name', version: Ddr::IngestTools::VERSION)
|
11
11
|
puts I18n.t('ddr_rdr_migrator.name')
|
12
12
|
puts I18n.t('marquee')
|
13
13
|
|
@@ -15,15 +15,19 @@ puts I18n.t('marquee')
|
|
15
15
|
parser = OptionParser.new do |opts|
|
16
16
|
opts.banner = 'Usage: migrate_ddr_to_rdr.rb [options]'
|
17
17
|
|
18
|
+
opts.on('-c', '--checksum_file CHECKSUM_FILE', 'File (with full path) to which checksum file should be written') do |v|
|
19
|
+
options[:checksum] = v
|
20
|
+
end
|
21
|
+
|
18
22
|
opts.on('-f', '--files FILE_PATH', 'Path to exported files') do |v|
|
19
23
|
options[:files] = v
|
20
24
|
end
|
21
25
|
|
22
|
-
opts.on('-m', '--
|
26
|
+
opts.on('-m', '--metadata_file METADATA_FILE', 'DDR metadata export file (with full path)') do |v|
|
23
27
|
options[:metadata] = v
|
24
28
|
end
|
25
29
|
|
26
|
-
opts.on('-o', '--outfile OUTPUT_FILE', '
|
30
|
+
opts.on('-o', '--outfile OUTPUT_FILE', 'File (with full path) to which updated manifest file should be written') do |v|
|
27
31
|
options[:outfile] = v
|
28
32
|
end
|
29
33
|
|
@@ -31,7 +35,7 @@ end
|
|
31
35
|
|
32
36
|
begin
|
33
37
|
parser.parse!
|
34
|
-
mandatory = [ :files, :
|
38
|
+
mandatory = [ :checksum_file, :files, :metadata_file, :outfile ]
|
35
39
|
missing = mandatory.select{ |param| options[param].nil? }
|
36
40
|
unless missing.empty?
|
37
41
|
raise OptionParser::MissingArgument.new(missing.join(', '))
|
data/bin/mint_manifest_arks.rb
CHANGED
data/config/locales/en.yml
CHANGED
@@ -1,17 +1,21 @@
|
|
1
1
|
module Ddr::IngestTools::DdrRdrMigrator
|
2
2
|
class FilePathAdder
|
3
3
|
|
4
|
-
attr_reader :files, :logger, :manifest
|
4
|
+
attr_reader :checksum_file, :files, :import_checksums, :logger, :manifest
|
5
5
|
|
6
|
-
def initialize(files:, logger: nil, manifest:)
|
6
|
+
def initialize(checksum_file:, files:, logger: nil, manifest:)
|
7
|
+
@checksum_file = checksum_file
|
8
|
+
@import_checksums = {}
|
7
9
|
@files = files
|
8
10
|
@logger = logger || Logger.new(STDOUT)
|
9
11
|
@manifest = manifest
|
10
12
|
end
|
11
13
|
|
12
14
|
def call
|
15
|
+
bag_checksums
|
13
16
|
scan_files(files)
|
14
17
|
update_manifest
|
18
|
+
write_checksums
|
15
19
|
manifest
|
16
20
|
end
|
17
21
|
|
@@ -35,7 +39,15 @@ module Ddr::IngestTools::DdrRdrMigrator
|
|
35
39
|
|
36
40
|
def handle_file(file_loc)
|
37
41
|
partial_path = file_partial_path(file_loc)
|
38
|
-
|
42
|
+
if payload_file?(partial_path)
|
43
|
+
add_to_item_files(partial_path)
|
44
|
+
add_to_checksums(partial_path)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def add_to_checksums(partial_path)
|
49
|
+
checksum = bag_checksums[partial_path]
|
50
|
+
import_checksums[File.join(files, partial_path)] = checksum
|
39
51
|
end
|
40
52
|
|
41
53
|
def add_to_item_files(partial_path)
|
@@ -75,5 +87,26 @@ module Ddr::IngestTools::DdrRdrMigrator
|
|
75
87
|
end
|
76
88
|
end
|
77
89
|
|
90
|
+
def write_checksums
|
91
|
+
File.open(checksum_file, 'w') do |f|
|
92
|
+
import_checksums.each do |k,v|
|
93
|
+
f.puts "#{v} #{k}"
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def bag_checksums
|
99
|
+
@bag_checksums ||= load_bag_checksums
|
100
|
+
end
|
101
|
+
|
102
|
+
def load_bag_checksums
|
103
|
+
checksums = {}
|
104
|
+
bag_checksum_file = File.join(files, 'manifest-sha1.txt')
|
105
|
+
File.readlines(bag_checksum_file).each do |line|
|
106
|
+
sum, path = line.chomp.split
|
107
|
+
checksums[path] = sum
|
108
|
+
end
|
109
|
+
checksums
|
110
|
+
end
|
78
111
|
end
|
79
112
|
end
|
@@ -1,13 +1,14 @@
|
|
1
1
|
module Ddr::IngestTools::DdrRdrMigrator
|
2
2
|
class Migrator
|
3
3
|
|
4
|
-
attr_reader :files, :logger, :
|
4
|
+
attr_reader :checksum_file, :files, :logger, :metadata_file, :outfile
|
5
5
|
attr_writer :manifest
|
6
6
|
|
7
|
-
def initialize(files:, logger: nil,
|
7
|
+
def initialize(checksum_file:, files:, logger: nil, metadata_file:, outfile:)
|
8
|
+
@checksum_file = checksum_file
|
8
9
|
@files = files
|
9
10
|
@logger = logger || Logger.new(STDOUT)
|
10
|
-
@
|
11
|
+
@metadata_file = metadata_file
|
11
12
|
@outfile = outfile
|
12
13
|
end
|
13
14
|
|
@@ -25,7 +26,7 @@ module Ddr::IngestTools::DdrRdrMigrator
|
|
25
26
|
private
|
26
27
|
|
27
28
|
def manifest
|
28
|
-
@manifest ||= as_csv_table(
|
29
|
+
@manifest ||= as_csv_table(metadata_file)
|
29
30
|
end
|
30
31
|
|
31
32
|
def sort_manifest
|
@@ -66,7 +67,7 @@ module Ddr::IngestTools::DdrRdrMigrator
|
|
66
67
|
end
|
67
68
|
|
68
69
|
def file_path_adder
|
69
|
-
FilePathAdder.new(files: files, logger: logger, manifest: manifest)
|
70
|
+
FilePathAdder.new(checksum_file: checksum_file, files: files, logger: logger, manifest: manifest)
|
70
71
|
end
|
71
72
|
|
72
73
|
def license_mapper
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
+
require 'tempfile'
|
2
3
|
|
3
4
|
module Ddr::IngestTools::DdrRdrMigrator
|
4
5
|
|
5
6
|
RSpec.describe FilePathAdder do
|
6
7
|
|
7
8
|
subject do
|
8
|
-
described_class.new(files: files, manifest: manifest)
|
9
|
+
described_class.new(checksum_file: checksum_file, files: files, manifest: manifest)
|
9
10
|
end
|
10
11
|
|
11
12
|
let(:files) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664',
|
@@ -15,11 +16,21 @@ module Ddr::IngestTools::DdrRdrMigrator
|
|
15
16
|
let(:expected_manifest_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'manifests',
|
16
17
|
'sorted_manifest_with_file_paths.csv') }
|
17
18
|
let(:expected_manifest) { CSV.read(expected_manifest_file, headers: true) }
|
19
|
+
let(:outdir) { Dir.mktmpdir }
|
20
|
+
let(:checksum_file) { File.join(outdir, 'checksums.txt') }
|
21
|
+
let(:expected_checksum_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
|
22
|
+
'checksums.txt') }
|
23
|
+
|
24
|
+
after { FileUtils.remove_dir outdir }
|
18
25
|
|
19
26
|
it 'adds the appropriate file paths to the manifest' do
|
20
27
|
expect(subject.call).to eq(expected_manifest)
|
21
28
|
end
|
22
29
|
|
30
|
+
it 'creates a checksum file' do
|
31
|
+
subject.call
|
32
|
+
expect(File.read(checksum_file)).to eq(File.read(expected_checksum_file))
|
33
|
+
end
|
23
34
|
end
|
24
35
|
|
25
36
|
end
|
@@ -4,12 +4,16 @@ require 'tempfile'
|
|
4
4
|
module Ddr::IngestTools::DdrRdrMigrator
|
5
5
|
RSpec.describe Migrator do
|
6
6
|
|
7
|
-
subject { described_class.new(
|
7
|
+
subject { described_class.new(checksum_file: checksum_file, files: files, metadata_file: metadata_file,
|
8
|
+
outfile: outfile) }
|
8
9
|
|
9
10
|
let(:files) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664',
|
10
11
|
'changeme-664-export-file') }
|
11
|
-
let(:
|
12
|
+
let(:metadata_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664', 'changeme-664.csv') }
|
12
13
|
let(:outdir) { Dir.mktmpdir }
|
14
|
+
let(:checksum_file) { File.join(outdir, 'checksums.txt') }
|
15
|
+
let(:expected_checksum_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
|
16
|
+
'checksums.txt') }
|
13
17
|
let(:outfile) { File.join(outdir, 'manifest.csv') }
|
14
18
|
let(:expected_outfile) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
|
15
19
|
'manifest.csv') }
|
@@ -19,6 +23,7 @@ module Ddr::IngestTools::DdrRdrMigrator
|
|
19
23
|
it 'produces the expected manifest file' do
|
20
24
|
subject.call
|
21
25
|
expect(CSV.read(outfile, headers: true)).to eq(CSV.read(expected_outfile, headers: true))
|
26
|
+
expect(File.read(checksum_file)).to eq(File.read(expected_checksum_file))
|
22
27
|
end
|
23
28
|
|
24
29
|
end
|
@@ -0,0 +1,3 @@
|
|
1
|
+
4c4665b408134d8f6995d1640a7f2d4eeee5c010 spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_665/data1.csv
|
2
|
+
37a0502601ed54f31d119d5355ade2c29ea530ea spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_665/data3.csv
|
3
|
+
ab84c8b1187123c4d627bea511714dd723b56dbe spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_667/data2.csv
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ddr-ingesttools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jim Coble
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ezid-client
|
@@ -151,6 +151,7 @@ files:
|
|
151
151
|
- spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-md5.txt
|
152
152
|
- spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-sha1.txt
|
153
153
|
- spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664.csv
|
154
|
+
- spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt
|
154
155
|
- spec/fixtures/ddr_rdr_migrator/target/changeme-664/manifest.csv
|
155
156
|
- spec/fixtures/rdr_importer/configs/default.yml
|
156
157
|
- spec/fixtures/rdr_importer/manifests/manifest_with_all_arks.csv
|
@@ -205,6 +206,7 @@ test_files:
|
|
205
206
|
- spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-md5.txt
|
206
207
|
- spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-sha1.txt
|
207
208
|
- spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664.csv
|
209
|
+
- spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt
|
208
210
|
- spec/fixtures/ddr_rdr_migrator/target/changeme-664/manifest.csv
|
209
211
|
- spec/fixtures/rdr_importer/configs/default.yml
|
210
212
|
- spec/fixtures/rdr_importer/manifests/manifest_with_all_arks.csv
|