ddr-ingesttools 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01e309ab9066ae8ae9c96c95a2760bcee6e22ef1
-  data.tar.gz: cb6ac427ab1fd8e7bc4914385c1c42c790824bce
+  metadata.gz: 4e3de5f3f9741a14f8ee24a8665f02b8685f4b26
+  data.tar.gz: c2782cb6a29ed694230e714e6082587d67e8bdcc
 SHA512:
-  metadata.gz: 87c4c2df7e0d29e9c55b6d29118b6521729584af3cccf1c74728713ed055f7f905ef1737b954fb4ac2ccd6e52eb06f2c15c21b9170810ef4a8fb6e2847360585
-  data.tar.gz: 632504a34a960e3df29c744ea2471e2181b4e233a84d3d587c8b0ae9983efadf85bc8b706200faaf4cd4d535fb7ae276e2047184babb63bd205847b64734fdd1
+  metadata.gz: 68380b51a8b4c28ef053d3758064edf1f9e69b70d6e0426460be8e13bfc7629608b5e6c2e5822b8d8c11beb39fd596ec06007362877a4e7477e6f0061144b97b
+  data.tar.gz: 2a81290d1b0b7f9ad41c284a8a78af914548a098ae5213475bcf51f81af8198fb0890df48fe86cb1519150fc29dc1f1cd2fa0edb76b3f04eab090e9940ad83f1

data/bin/migrate_ddr_to_rdr.rb

@@ -7,7 +7,7 @@ require 'optparse'
 options = {}
 
 puts I18n.t('marquee')
-puts I18n.t('suite.name')
+puts I18n.t('suite.name', version: Ddr::IngestTools::VERSION)
 puts I18n.t('ddr_rdr_migrator.name')
 puts I18n.t('marquee')
 
@@ -15,15 +15,19 @@ puts I18n.t('marquee')
 parser = OptionParser.new do |opts|
   opts.banner = 'Usage: migrate_ddr_to_rdr.rb [options]'
 
+  opts.on('-c', '--checksum_file CHECKSUM_FILE', 'File (with full path) to which checksum file should be written') do |v|
+    options[:checksum] = v
+  end
+
   opts.on('-f', '--files FILE_PATH', 'Path to exported files') do |v|
     options[:files] = v
   end
 
-  opts.on('-m', '--metadata METADATA_FILE', 'Path to DDR metadata export file') do |v|
+  opts.on('-m', '--metadata_file METADATA_FILE', 'DDR metadata export file (with full path)') do |v|
     options[:metadata] = v
   end
 
-  opts.on('-o', '--outfile OUTPUT_FILE', 'Path to which updated manifest file should be written') do |v|
+  opts.on('-o', '--outfile OUTPUT_FILE', 'File (with full path) to which updated manifest file should be written') do |v|
     options[:outfile] = v
   end
 
@@ -31,7 +35,7 @@ end
 
 begin
   parser.parse!
-  mandatory = [ :files, :metadata, :outfile ]
+  mandatory = [ :checksum_file, :files, :metadata_file, :outfile ]
   missing = mandatory.select{ |param| options[param].nil? }
   unless missing.empty?
     raise OptionParser::MissingArgument.new(missing.join(', '))

data/bin/mint_manifest_arks.rb

@@ -7,7 +7,7 @@ require 'optparse'
 options = {}
 
 puts I18n.t('marquee')
-puts I18n.t('suite.name')
+puts I18n.t('suite.name', version: Ddr::IngestTools::VERSION)
 puts I18n.t('manifest_ark_minter.name')
 puts I18n.t('marquee')
 

data/config/locales/en.yml

@@ -1,7 +1,7 @@
 en:
   marquee: ==================================================
   suite:
-    name: DDR Ingest Tools
+    name: "DDR Ingest Tools %{version}"
   ddr_rdr_migrator:
     name: DDR to RDR Migrator
   manifest_ark_minter:

data/lib/ddr/ingesttools/ddr_rdr_migrator/file_path_adder.rb

@@ -1,17 +1,21 @@
 module Ddr::IngestTools::DdrRdrMigrator
   class FilePathAdder
 
-    attr_reader :files, :logger, :manifest
+    attr_reader :checksum_file, :files, :import_checksums, :logger, :manifest
 
-    def initialize(files:, logger: nil, manifest:)
+    def initialize(checksum_file:, files:, logger: nil, manifest:)
+      @checksum_file = checksum_file
+      @import_checksums = {}
       @files = files
       @logger = logger || Logger.new(STDOUT)
       @manifest = manifest
     end
 
     def call
+      bag_checksums
       scan_files(files)
       update_manifest
+      write_checksums
       manifest
     end
 
@@ -35,7 +39,15 @@ module Ddr::IngestTools::DdrRdrMigrator
 
     def handle_file(file_loc)
       partial_path = file_partial_path(file_loc)
-      add_to_item_files(partial_path) if payload_file?(partial_path)
+      if payload_file?(partial_path)
+        add_to_item_files(partial_path)
+        add_to_checksums(partial_path)
+      end
+    end
+
+    def add_to_checksums(partial_path)
+      checksum = bag_checksums[partial_path]
+      import_checksums[File.join(files,  partial_path)] = checksum
     end
 
     def add_to_item_files(partial_path)
@@ -75,5 +87,26 @@ module Ddr::IngestTools::DdrRdrMigrator
       end
     end
 
+    def write_checksums
+      File.open(checksum_file, 'w') do |f|
+        import_checksums.each do |k,v|
+          f.puts "#{v}  #{k}"
+        end
+      end
+    end
+
+    def bag_checksums
+      @bag_checksums ||= load_bag_checksums
+    end
+
+    def load_bag_checksums
+      checksums = {}
+      bag_checksum_file = File.join(files, 'manifest-sha1.txt')
+      File.readlines(bag_checksum_file).each do |line|
+        sum, path = line.chomp.split
+        checksums[path] = sum
+      end
+      checksums
+    end
   end
 end

data/lib/ddr/ingesttools/ddr_rdr_migrator/migrator.rb

@@ -1,13 +1,14 @@
 module Ddr::IngestTools::DdrRdrMigrator
   class Migrator
 
-    attr_reader :files, :logger, :metadata, :outfile
+    attr_reader :checksum_file, :files, :logger, :metadata_file, :outfile
     attr_writer :manifest
 
-    def initialize(files:, logger: nil, metadata:, outfile:)
+    def initialize(checksum_file:, files:, logger: nil, metadata_file:, outfile:)
+      @checksum_file = checksum_file
       @files = files
       @logger = logger || Logger.new(STDOUT)
-      @metadata = metadata
+      @metadata_file = metadata_file
       @outfile = outfile
     end
 
@@ -25,7 +26,7 @@ module Ddr::IngestTools::DdrRdrMigrator
     private
 
     def manifest
-      @manifest ||= as_csv_table(metadata)
+      @manifest ||= as_csv_table(metadata_file)
     end
 
     def sort_manifest
@@ -66,7 +67,7 @@ module Ddr::IngestTools::DdrRdrMigrator
     end
 
     def file_path_adder
-      FilePathAdder.new(files: files, logger: logger, manifest: manifest)
+      FilePathAdder.new(checksum_file: checksum_file, files: files, logger: logger, manifest: manifest)
     end
 
     def license_mapper

data/lib/ddr/ingesttools/version.rb

@@ -1,5 +1,5 @@
 module Ddr
   module IngestTools
-    VERSION = '0.5.0'
+    VERSION = '0.6.0'
   end
 end

data/spec/ddr_rdr_migrator/file_path_adder_spec.rb

@@ -1,11 +1,12 @@
 require 'spec_helper'
+require 'tempfile'
 
 module Ddr::IngestTools::DdrRdrMigrator
 
   RSpec.describe FilePathAdder do
 
     subject do
-      described_class.new(files: files, manifest: manifest)
+      described_class.new(checksum_file: checksum_file, files: files, manifest: manifest)
     end
 
     let(:files) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664',
@@ -15,11 +16,21 @@ module Ddr::IngestTools::DdrRdrMigrator
     let(:expected_manifest_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'manifests',
                                              'sorted_manifest_with_file_paths.csv') }
     let(:expected_manifest) { CSV.read(expected_manifest_file, headers: true) }
+    let(:outdir) { Dir.mktmpdir }
+    let(:checksum_file) { File.join(outdir, 'checksums.txt') }
+    let(:expected_checksum_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
+                                             'checksums.txt') }
+
+    after { FileUtils.remove_dir outdir }
 
     it 'adds the appropriate file paths to the manifest' do
       expect(subject.call).to eq(expected_manifest)
     end
 
+    it 'creates a checksum file' do
+      subject.call
+      expect(File.read(checksum_file)).to eq(File.read(expected_checksum_file))
+    end
   end
 
 end

data/spec/ddr_rdr_migrator/migrator_spec.rb

@@ -4,12 +4,16 @@ require 'tempfile'
 module Ddr::IngestTools::DdrRdrMigrator
   RSpec.describe Migrator do
 
-    subject { described_class.new(files: files, metadata: metadata, outfile: outfile) }
+    subject { described_class.new(checksum_file: checksum_file, files: files, metadata_file: metadata_file,
+                                  outfile: outfile) }
 
     let(:files) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664',
                             'changeme-664-export-file') }
-    let(:metadata) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664', 'changeme-664.csv') }
+    let(:metadata_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'source', 'changeme-664', 'changeme-664.csv') }
     let(:outdir) { Dir.mktmpdir }
+    let(:checksum_file) { File.join(outdir, 'checksums.txt') }
+    let(:expected_checksum_file) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
+                                             'checksums.txt') }
     let(:outfile) { File.join(outdir, 'manifest.csv') }
     let(:expected_outfile) { File.join('spec', 'fixtures', 'ddr_rdr_migrator', 'target', 'changeme-664',
                                        'manifest.csv') }
@@ -19,6 +23,7 @@ module Ddr::IngestTools::DdrRdrMigrator
     it 'produces the expected manifest file' do
       subject.call
       expect(CSV.read(outfile, headers: true)).to eq(CSV.read(expected_outfile, headers: true))
+      expect(File.read(checksum_file)).to eq(File.read(expected_checksum_file))
     end
 
   end

data/spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt

@@ -0,0 +1,3 @@
+4c4665b408134d8f6995d1640a7f2d4eeee5c010  spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_665/data1.csv
+37a0502601ed54f31d119d5355ade2c29ea530ea  spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_665/data3.csv
+ab84c8b1187123c4d627bea511714dd723b56dbe  spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/data/objects/changeme_667/data2.csv

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ddr-ingesttools
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Jim Coble
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-06 00:00:00.000000000 Z
+date: 2018-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ezid-client
@@ -151,6 +151,7 @@ files:
 - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-md5.txt
 - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-sha1.txt
 - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664.csv
+- spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt
 - spec/fixtures/ddr_rdr_migrator/target/changeme-664/manifest.csv
 - spec/fixtures/rdr_importer/configs/default.yml
 - spec/fixtures/rdr_importer/manifests/manifest_with_all_arks.csv
@@ -205,6 +206,7 @@ test_files:
 - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-md5.txt
 - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664-export-file/tagmanifest-sha1.txt
 - spec/fixtures/ddr_rdr_migrator/source/changeme-664/changeme-664.csv
+- spec/fixtures/ddr_rdr_migrator/target/changeme-664/checksums.txt
 - spec/fixtures/ddr_rdr_migrator/target/changeme-664/manifest.csv
 - spec/fixtures/rdr_importer/configs/default.yml
 - spec/fixtures/rdr_importer/manifests/manifest_with_all_arks.csv