ddr-ingesttools 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 01e309ab9066ae8ae9c96c95a2760bcee6e22ef1
4
- data.tar.gz: cb6ac427ab1fd8e7bc4914385c1c42c790824bce
3
+ metadata.gz: 4e3de5f3f9741a14f8ee24a8665f02b8685f4b26
4
+ data.tar.gz: c2782cb6a29ed694230e714e6082587d67e8bdcc
5
5
  SHA512:
6
- metadata.gz: 87c4c2df7e0d29e9c55b6d29118b6521729584af3cccf1c74728713ed055f7f905ef1737b954fb4ac2ccd6e52eb06f2c15c21b9170810ef4a8fb6e2847360585
7
- data.tar.gz: 632504a34a960e3df29c744ea2471e2181b4e233a84d3d587c8b0ae9983efadf85bc8b706200faaf4cd4d535fb7ae276e2047184babb63bd205847b64734fdd1
6
+ metadata.gz: 68380b51a8b4c28ef053d3758064edf1f9e69b70d6e0426460be8e13bfc7629608b5e6c2e5822b8d8c11beb39fd596ec06007362877a4e7477e6f0061144b97b
7
+ data.tar.gz: 2a81290d1b0b7f9ad41c284a8a78af914548a098ae5213475bcf51f81af8198fb0890df48fe86cb1519150fc29dc1f1cd2fa0edb76b3f04eab090e9940ad83f1
@@ -7,7 +7,7 @@ require 'optparse'
7
7
  options = {}
8
8
 
9
9
  puts I18n.t('marquee')
10
- puts I18n.t('suite.name')
10
+ puts I18n.t('suite.name', version: Ddr::IngestTools::VERSION)
11
11
  puts I18n.t('ddr_rdr_migrator.name')
12
12
  puts I18n.t('marquee')
13
13
 
@@ -15,15 +15,19 @@ puts I18n.t('marquee')
15
15
  parser = OptionParser.new do |opts|
16
16
  opts.banner = 'Usage: migrate_ddr_to_rdr.rb [options]'
17
17
 
18
+ opts.on('-c', '--checksum_file CHECKSUM_FILE', 'File (with full path) to which checksum file should be written') do |v|
19
+ options[:checksum] = v
20
+ end
21
+
18
22
  opts.on('-f', '--files FILE_PATH', 'Path to exported files') do |v|
19
23
  options[:files] = v
20
24
  end
21
25
 
22
- opts.on('-m', '--metadata METADATA_FILE', 'Path to DDR metadata export file') do |v|
26
+ opts.on('-m', '--metadata_file METADATA_FILE', 'DDR metadata export file (with full path)') do |v|
23
27
  options[:metadata] = v
24
28
  end
25
29
 
26
- opts.on('-o', '--outfile OUTPUT_FILE', 'Path to which updated manifest file should be written') do |v|
30
+ opts.on('-o', '--outfile OUTPUT_FILE', 'File (with full path) to which updated manifest file should be written') do |v|
27
31
  options[:outfile] = v
28
32
  end
29
33
 
@@ -31,7 +35,7 @@ end
31
35
 
32
36
  begin
33
37
  parser.parse!
34
- mandatory = [ :files, :metadata, :outfile ]
38
+ mandatory = [ :checksum_file, :files, :metadata_file, :outfile ]
35
39
  missing = mandatory.select{ |param| options[param].nil? }
36
40
  unless missing.empty?
37
41
  raise OptionParser::MissingArgument.new(missing.join(', '))
@@ -7,7 +7,7 @@ require 'optparse'
7
7
  options = {}
8
8
 
9
9
  puts I18n.t('marquee')
10
- puts I18n.t('suite.name')
10
+ puts I18n.t('suite.name', version: Ddr::IngestTools::VERSION)
11
11
  puts I18n.t('manifest_ark_minter.name')
12
12
  puts I18n.t('marquee')
13
13
 
@@ -1,7 +1,7 @@
1
1
  en:
2
2
  marquee: ==================================================
3
3
  suite:
4
- name: DDR Ingest Tools
4
+ name: "DDR Ingest Tools %{version}"
5
5
  ddr_rdr_migrator:
6
6
  name: DDR to RDR Migrator
7
7
  manifest_ark_minter:
@@ -1,17 +1,21 @@
1
1
  module Ddr::IngestTools::DdrRdrMigrator
2
2
  class FilePathAdder
3
3
 
4
- attr_reader :files, :logger, :manifest
4
+ attr_reader :checksum_file, :files, :import_checksums, :logger, :manifest
5
5
 
6
- def initialize(files:, logger: nil, manifest:)
6
+ def initialize(checksum_file:, files:, logger: nil, manifest:)
7
+ @checksum_file = checksum_file
8
+ @import_checksums = {}
7
9
  @files = files
8
10
  @logger = logger || Logger.new(STDOUT)
9
11
  @manifest = manifest
10
12
  end
11
13
 
12
14
  def call
15
+ bag_checksums
13
16
  scan_files(files)
14
17
  update_manifest
18
+ write_checksums
15
19
  manifest
16
20
  end
17
21
 
@@ -35,7 +39,15 @@ module Ddr::IngestTools::DdrRdrMigrator
35
39
 
36
40
  def handle_file(file_loc)
37
41
  partial_path = file_partial_path(file_loc)
38
- add_to_item_files(partial_path) if payload_file?(partial_path)
42
+ if payload_file?(partial_path)
43
+ add_to_item_files(partial_path)
44
+ add_to_checksums(partial_path)
45
+ end
46
+ end
47
+
48
+ def add_to_checksums(partial_path)
49
+ checksum = bag_checksums[partial_path]
50
+ import_checksums[File.join(files, partial_path)] = checksum
39
51
  end
40
52
 
41
53
  def add_to_item_files(partial_path)
@@ -75,5 +87,26 @@ module Ddr::IngestTools::DdrRdrMigrator
75
87
  end
76
88
  end
77
89
 
90
+ def write_checksums
91
+ File.open(checksum_file, 'w') do |f|
92
+ import_checksums.each do |k,v|
93
+ f.puts "#{v} #{k}"
94
+ end
95
+ end
96
+ end
97
+
98
+ def bag_checksums
99
+ @bag_checksums ||= load_bag_checksums
100
+ end
101
+
102
+ def load_bag_checksums
103
+ checksums = {}
104
+ bag_checksum_file = File.join(files, 'manifest-sha1.txt')
105
+ File.readlines(bag_checksum_file).each do |line|
106
+ sum, path = line.chomp.split
107
+ checksums[path] = sum
108
+ end
109
+ checksums
110
+ end
78
111
  end
79
112
  end
@@ -1,13 +1,14 @@
1
1
  module Ddr::IngestTools::DdrRdrMigrator
2
2
  class Migrator
3
3
 
4
- attr_reader :files, :logger, :metadata, :outfile
4
+ attr_reader :checksum_file, :files, :logger, :metadata_file, :outfile
5
5
  attr_writer :manifest
6
6
 
7
- def initialize(files:, logger: nil, metadata:, outfile:)
7
+ def initialize(checksum_file:, files:, logger: nil, metadata_file:, outfile:)
8
+ @checksum_file = checksum_file
8
9
  @files = files
9
10
  @logger = logger || Logger.new(STDOUT)
10
- @metadata = metadata
11
+ @metadata_file = metadata_file
11
12
  @outfile = outfile
12
13
  end
13
14
 
@@ -25,7 +26,7 @@ module Ddr::IngestTools::DdrRdrMigrator
25
26
  private
26
27
 
27
28
  def manifest
28
- @manifest ||= as_csv_table(metadata)
29
+ @manifest ||= as_csv_table(metadata_file)
29
30
  end
30
31
 
31
32
  def sort_manifest
@@ -66,7 +67,7 @@ module Ddr::IngestTools::DdrRdrMigrator
66
67
  end
67
68
 
68
69
  def file_path_adder
69
- FilePathAdder.new(files: files, logger: logger, manifest: manifest)
70
+ FilePathAdder.new(checksum_file: checksum_file, files: files, logger: logger, manifest: manifest)
70
71
  end
71
72
 
72
73
  def license_mapper
@@ -1,5 +1,5 @@
1
1
  module Ddr
2
2
  module IngestTools
3
- VERSION = '0.5.0'
3
+ VERSION = '0.6.0'
4
4
  end
5
5
  end
@@ -1,11 +1,12 @@
1
1
  require 'spec_helper'
2
+ require 'tempfile'
2
3
 
3
4
  module Ddr::IngestTools::DdrRdrMigrator
4
5
 
5
6
  RSpec.describe FilePathAdder do
6
7
 
7
8
  subject do
8
- described_class.new(files: files, manifest: manifest)
9
+ described_class.new(checksum_file: checksum_file, files: files, manifest: manifest)
9
10
  end
10
11
 
11
12
  let(:files) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664',
@@ -15,11 +16,21 @@ module Ddr::IngestTools::DdrRdrMigrator
15
16
  let(:expected_manifest_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'manifests',
16
17
  'sorted_manifest_with_file_paths.csv') }
17
18
  let(:expected_manifest) { CSV.read(expected_manifest_file, headers: true) }
19
+ let(:outdir) { Dir.mktmpdir }
20
+ let(:checksum_file) { File.join(outdir, 'checksums.txt') }
21
+ let(:expected_checksum_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
22
+ 'checksums.txt') }
23
+
24
+ after { FileUtils.remove_dir outdir }
18
25
 
19
26
  it 'adds the appropriate file paths to the manifest' do
20
27
  expect(subject.call).to eq(expected_manifest)
21
28
  end
22
29
 
30
+ it 'creates a checksum file' do
31
+ subject.call
32
+ expect(File.read(checksum_file)).to eq(File.read(expected_checksum_file))
33
+ end
23
34
  end
24
35
 
25
36
  end
@@ -4,12 +4,16 @@ require 'tempfile'
4
4
  module Ddr::IngestTools::DdrRdrMigrator
5
5
  RSpec.describe Migrator do
6
6
 
7
- subject { described_class.new(files: files, metadata: metadata, outfile: outfile) }
7
+ subject { described_class.new(checksum_file: checksum_file, files: files, metadata_file: metadata_file,
8
+ outfile: outfile) }
8
9
 
9
10
  let(:files) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664',
10
11
  'changeme-664-export-file') }
11
- let(:metadata) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664', 'changeme-664.csv') }
12
+ let(:metadata_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664', 'changeme-664.csv') }
12
13
  let(:outdir) { Dir.mktmpdir }
14
+ let(:checksum_file) { File.join(outdir, 'checksums.txt') }
15
+ let(:expected_checksum_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
16
+ 'checksums.txt') }
13
17
  let(:outfile) { File.join(outdir, 'manifest.csv') }
14
18
  let(:expected_outfile) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
15
19
  'manifest.csv') }
@@ -19,6 +23,7 @@ module Ddr::IngestTools::DdrRdrMigrator
19
23
  it 'produces the expected manifest file' do
20
24
  subject.call
21
25
  expect(CSV.read(outfile, headers: true)).to eq(CSV.read(expected_outfile, headers: true))
26
+ expect(File.read(checksum_file)).to eq(File.read(expected_checksum_file))
22
27
  end
23
28
 
24
29
  end
@@ -0,0 +1,3 @@
1
+ 4c4665b408134d8f6995d1640a7f2d4eeee5c010 spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_665/data1.csv
2
+ 37a0502601ed54f31d119d5355ade2c29ea530ea spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_665/data3.csv
3
+ ab84c8b1187123c4d627bea511714dd723b56dbe spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_667/data2.csv
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ddr-ingesttools
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jim Coble
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-06 00:00:00.000000000 Z
11
+ date: 2018-08-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ezid-client
@@ -151,6 +151,7 @@ files:
151
151
  - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-md5.txt
152
152
  - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-sha1.txt
153
153
  - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664.csv
154
+ - spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt
154
155
  - spec/fixtures/ddr_rdr_migrator/target/changeme-664/manifest.csv
155
156
  - spec/fixtures/rdr_importer/configs/default.yml
156
157
  - spec/fixtures/rdr_importer/manifests/manifest_with_all_arks.csv
@@ -205,6 +206,7 @@ test_files:
205
206
  - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-md5.txt
206
207
  - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-sha1.txt
207
208
  - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664.csv
209
+ - spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt
208
210
  - spec/fixtures/ddr_rdr_migrator/target/changeme-664/manifest.csv
209
211
  - spec/fixtures/rdr_importer/configs/default.yml
210
212
  - spec/fixtures/rdr_importer/manifests/manifest_with_all_arks.csv