ddr-ingesttools 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5171561cdd0df54c62ed927f82edbc2f72e07b2a
-  data.tar.gz: 46e0b9ad341186c05f7558b8b622b5be4b919021
+  metadata.gz: 4c8a03cc7a443dbca08bd1c8cb86d6cc70f18050
+  data.tar.gz: 3f1d806ba1c1b8ccde507157372e30c93c1cd0cd
 SHA512:
-  metadata.gz: bee3fb79556ed7e262be0caa783514181db1abf7384e32946567a83ec2e21e41300cad0c972de24772ddf019a93543f0b2b70ab0d8344d6dc2e1fb1d133e0152
-  data.tar.gz: a28b353c14b0dc32ae08687dc925d2cf37b72e35b0f10d36e30f943ce23a63e3850a98f4db1707e35d155e0b348da38706b2cbfd5ab1979229284396dea91c6f
+  metadata.gz: 361c4c511a7bb10f03087412aa700739e188161f9b6650e0c6eb295560bfa1a5ff9d23a8621db46d552e418bae0ff50bd08b7dbda3fabc5263becb3fc7644c16
+  data.tar.gz: a14ed57c5cbb449f02d1c29c1f4cf5df37960152c7a2c8d2717eea2363ae9de05bdf8e2e7e5478a0e0048b3b82d73e7328fcb00ef0a5b199bc49640c93aa9cd7

data/bin/convert_dpc_folder.rb

@@ -3,8 +3,9 @@
 require 'ddr/ingesttools'
 require 'optparse'
 
-# Parse command line arguments
 options = {}
+
+# Parse command line arguments
 parser = OptionParser.new do |opts|
   opts.banner = 'Usage: convert_dpc_folder.rb [options]'
 
@@ -20,6 +21,20 @@ parser = OptionParser.new do |opts|
           'to use as the local ID of the item of which that file is a component') do |v|
     options[:item_id_length] = v
   end
+
+  opts.on('-c', '--checksums [CHECKSUM_FILE]', 'External checksum file') do |v|
+    options[:checksums] = v
+  end
+
+  opts.on('--[no-]copy_files', 'Copy files to target location instead of using a symlink') do |v|
+    options[:copy_files] = v
+  end
+
+  opts.on('--collection_title [TITLE]', 'Title for collection',
+          'required if intending to create a collection-creating Standard Ingest') do |v|
+    options[:collection_title] = v
+  end
+
 end
 
 begin
@@ -35,6 +50,7 @@ rescue OptionParser::InvalidOption, OptionParser::MissingArgument
   exit(false)
 end
 
-converter_args = [ options[:source], options[:target], options[:item_id_length] ]
-converter = Ddr::IngestTools::DpcFolderConverter::Converter.new(*converter_args)
-converter.call
+converter = Ddr::IngestTools::DpcFolderConverter::Converter.new(options)
+results = converter.call
+puts I18n.translate('errors.count', { count: results.errors.size })
+results.errors.each { |e| puts e }

data/config/locales/en.yml

@@ -0,0 +1,7 @@
+en:
+  errors:
+    count: "%{count} error(s)"
+    checksum_mismatch: |
+      ***** Checksum mismatch:
+              %{c1} %{f1}
+              %{c2} %{f2}

data/ddr-ingesttools.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "bagit", "~> 0.4"
+  spec.add_dependency "i18n", "~> 0.8"
 
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"

data/lib/ddr/ingesttools.rb

@@ -1,8 +1,13 @@
 require_relative 'ingesttools/version'
 require_relative 'ingesttools/dpc_folder_converter'
+require_relative 'ingesttools/checksum_file'
+
+require 'i18n'
 
 module Ddr
   module IngestTools
 
+    I18n.load_path = Dir['config/locales/*.yml']
+
   end
 end

data/lib/ddr/ingesttools/checksum_file.rb

@@ -0,0 +1,28 @@
+module Ddr::IngestTools
+  class ChecksumFile
+
+    attr_reader :digests
+
+    def initialize(checksum_filepath)
+      @digests = digest_hash(checksum_filepath)
+    end
+
+    def digest(filepath)
+      digests[filepath]
+    end
+
+    private
+
+    def digest_hash(checksum_filepath)
+      h = {}
+      File.open(checksum_filepath, 'r') do |file|
+        file.each_line do |line|
+          digest, path = line.chomp.split
+          h[path] = digest
+        end
+      end
+      h
+    end
+
+  end
+end

data/lib/ddr/ingesttools/dpc_folder_converter/converter.rb

@@ -6,54 +6,70 @@ require 'find'
 module Ddr::IngestTools::DpcFolderConverter
     class Converter
 
-      METADATA_HEADERS = [ 'path', 'local_id' ]
+      INTERMEDIATE_FILES_DIRNAME = 'intermediate_files'
+      DPC_TARGETS_DIRNAME = 'targets'
+      SIF_TARGETS_DIRNAME = 'dpc_targets'
+      SIF_METADATA_FILENAME = 'metadata.txt'
+      SIF_MANIFEST_SHA1_FILENAME = 'manifest-sha1.txt'
 
-      attr_reader :source, :target, :data_dir, :item_id_length
-      attr_accessor :local_id_metadata
+      Results = Struct.new(:file_map, :errors)
 
-      def initialize(source, target, item_id_length)
+      attr_reader :source, :target, :data_dir, :item_id_length, :checksums, :copy_files, :collection_title,
+                  :metadata_headers
+      attr_accessor :errors, :file_map, :local_id_metadata, :results
+
+      def initialize(source:, target:, item_id_length:, checksums: nil, copy_files: false, collection_title: nil)
         @source = source
         @target = target
-        @data_dir = File.join(target, 'data')
         @item_id_length = item_id_length
-        @local_id_metadata = {}
+        @checksums = checksums
+        @copy_files = copy_files
+        @collection_title = collection_title
+        @metadata_headers = [ 'path', 'local_id' ]
+        @metadata_headers << 'title' unless collection_title.nil?
       end
 
       def call
-        FileUtils.mkdir_p data_dir
-        find_component_files(source).each { |file| handle_component(file) }
-        find_target_files(source).each { |file| handle_target(file) }
+        setup
+        scan_files(source)
         output_metadata
         bagitup
+        validate_checksums if checksums
+        Results.new(file_map, errors)
       end
 
       private
 
-      def included_extensions
-        Ddr::IngestTools::DpcFolderConverter.config[:included_extensions]
+      def setup
+        @data_dir = File.join(target, 'data')
+        @errors = []
+        @file_map = {}
+        @local_id_metadata = {}
+        FileUtils.mkdir_p data_dir
       end
 
-      def find_component_files(dir)
-        files = []
-        Find.find(dir) do |path|
-          Find.prune if path.include?('targets')
-          Find.prune if path.include?('intermediate_files')
-          next unless File.file?(path)
-          next unless included_extensions.include?(File.extname(path))
-          files << path
-        end
-        files
+      def included_extensions
+        Ddr::IngestTools::DpcFolderConverter.config[:included_extensions]
       end
 
-      def find_target_files(dir)
-        files = []
-        Find.find(dir) do |path|
-          next unless path.include?('targets')
-          next unless File.file?(path)
-          next unless included_extensions.include?(File.extname(path))
-          files << path
+      def scan_files(dirpath, file_handler='handle_component'.to_sym)
+        Dir.foreach(dirpath).each do |entry|
+          next if [ '.', '..' ].include?(entry)
+          path = File.join(dirpath, entry)
+          if File.directory?(path)
+            if entry == DPC_TARGETS_DIRNAME
+              scan_files(path, :handle_target)
+            elsif entry == INTERMEDIATE_FILES_DIRNAME
+              scan_files(path, :handle_intermediate_file)
+            else
+              scan_files(path, file_handler)
+            end
+          else
+            if included_extensions.include?(File.extname(entry))
+              self.send(file_handler, path)
+            end
+          end
         end
-        files
       end
 
       def handle_component(file)
@@ -61,26 +77,45 @@ module Ddr::IngestTools::DpcFolderConverter
         item_id = item_id_length == 0 ? base : base[0, item_id_length]
         FileUtils.mkdir_p(File.join(data_dir, item_id))
         local_id_metadata[item_id] = item_id
-        FileUtils.cp file, File.join(data_dir, item_id)
+        handle_file(file, item_id)
         local_id_metadata[File.join(item_id, File.basename(file))] = base
       end
 
+      def handle_intermediate_file(file)
+        FileUtils.mkdir_p(File.join(data_dir, INTERMEDIATE_FILES_DIRNAME))
+        handle_file(file, INTERMEDIATE_FILES_DIRNAME)
+      end
+
       def handle_target(file)
         base = File.basename(file, File.extname(file))
-        FileUtils.mkdir_p(File.join(data_dir, 'dpc_targets'))
-        FileUtils.cp file, File.join(data_dir, 'dpc_targets')
-        local_id_metadata[File.join('dpc_targets', File.basename(file))] = base
+        FileUtils.mkdir_p(File.join(data_dir, SIF_TARGETS_DIRNAME))
+        handle_file(file, SIF_TARGETS_DIRNAME)
+        local_id_metadata[File.join(SIF_TARGETS_DIRNAME, File.basename(file))] = base
+      end
+
+      def handle_file(file, folder_name)
+        if copy_files
+          FileUtils.cp file, File.join(data_dir, folder_name)
+        else
+          FileUtils.ln_s file, File.join(data_dir, folder_name)
+        end
+        file_map[file] = File.join(data_dir, folder_name, File.basename(file))
       end
 
       def output_metadata
         metadata_rows = []
+        if collection_title
+          metadata_rows << CSV::Row.new(metadata_headers, [ nil, nil, collection_title ])
+        end
         local_id_metadata.each_pair do |k,v|
-          metadata_rows << CSV::Row.new(METADATA_HEADERS, [ k, v ])
+          row_elements = [ k, v ]
+          row_elements << nil if collection_title
+          metadata_rows << CSV::Row.new(metadata_headers, row_elements)
         end
-        File.open(File.join(data_dir, 'metadata.txt'), 'w') do |file|
-          file.puts(METADATA_HEADERS.join(Ddr::IngestTools::DpcFolderConverter.config[:csv_options][:col_sep]))
+        File.open(File.join(data_dir, SIF_METADATA_FILENAME), 'w') do |file|
+          file.puts(metadata_headers.join(Ddr::IngestTools::DpcFolderConverter.config[:csv_options][:col_sep]))
           metadata_rows.each do |row|
-            file.puts(row.to_csv(Ddr::IngestTools::DpcFolderConverter.config[:csv_options]).strip)
+            file.puts(row.to_csv(Ddr::IngestTools::DpcFolderConverter.config[:csv_options]))
           end
         end
       end
@@ -89,5 +124,19 @@ module Ddr::IngestTools::DpcFolderConverter
         bag = BagIt::Bag.new(target)
         bag.manifest!
       end
+
+      def validate_checksums
+        external_checksums = Ddr::IngestTools::ChecksumFile.new(checksums)
+        sif_manifest = Ddr::IngestTools::ChecksumFile.new(File.join(target, SIF_MANIFEST_SHA1_FILENAME))
+        file_map.each do |source_path, target_path|
+          external_checksum = external_checksums.digest(source_path)
+          manifest_path = target_path.sub("#{target}/", '')
+          sif_checksum = sif_manifest.digest(manifest_path)
+          unless external_checksum == sif_checksum
+            errors << I18n.translate('errors.checksum_mismatch', { c1: external_checksum, f1: source_path,
+                                                                   c2: sif_checksum, f2: target_path })
+          end
+        end
+      end
     end
 end

data/lib/ddr/ingesttools/version.rb

@@ -1,5 +1,5 @@
 module Ddr
   module IngestTools
-    VERSION = '0.1.0'
+    VERSION = '0.2.0'
   end
 end

data/spec/fixtures/files/bad-checksums-sha1.txt

@@ -0,0 +1,11 @@
+3201454891f1d63a1493f393e061491e2d65ffcd  SOURCE_DIRECTORY/Thumbs.db
+59ec01f979a76b968bc579e5cd0ceb3bcf3e629f  SOURCE_DIRECTORY/abc001001.tif
+d0a2f2482783ae3c83d06f3cdeaa1a306cc043ad  SOURCE_DIRECTORY/abc001002.tif
+38ee72ab417192589f3a54ef1016131c7d7e9e4e  SOURCE_DIRECTORY/abc002001.tif
+1da7765e3ca71ed76395bc56dae69d64732beff8  SOURCE_DIRECTORY/checksums.txt
+c227abc095d3b758ab1c1c1c9e922494b6b6e0b0  SOURCE_DIRECTORY/g/abc003001.wav
+541af8df7d2c631382bd0ec189476894d0323df5  SOURCE_DIRECTORY/g/abc003002.wav
+49d77d27bf82ec3dafa1967490594883f5e8b432  SOURCE_DIRECTORY/intermediate_files/abc001001.jpg
+260b3c2d20a1726de96671d29f73ba09d13b61ba  SOURCE_DIRECTORY/intermediate_files/abc002001.jpg
+a08c4d5a76d1b8735587be6ffcba66a9baf475c4  SOURCE_DIRECTORY/targets/T001.tif
+40f9993d06945544fa57af88f7c3e9102ecc69e3  SOURCE_DIRECTORY/targets/T002.tif

data/spec/fixtures/files/good-checksums-sha1.txt

@@ -0,0 +1,11 @@
+3201454891f1d63a1493f393e061491e2d65ffcd  SOURCE_DIRECTORY/Thumbs.db
+59ec01f979a76b968bc579e5cd0ceb3bcf3e629f  SOURCE_DIRECTORY/abc001001.tif
+d0a2f2482783ae3c38d06f3cdeaa1a306cc043ad  SOURCE_DIRECTORY/abc001002.tif
+38ee72ab417192589f3a54ef1016131c7d7e9e4e  SOURCE_DIRECTORY/abc002001.tif
+1da7765e3ca71ed76395bc56dae69d64732beff8  SOURCE_DIRECTORY/checksums.txt
+c227abc095d3b758051c1c1c9e922494b6b6e0b0  SOURCE_DIRECTORY/g/abc003001.wav
+541af8df7d2c631382bd0ec189476894d0323df5  SOURCE_DIRECTORY/g/abc003002.wav
+49d77d27bf82ec3dafa1967490594883f5e8b432  SOURCE_DIRECTORY/intermediate_files/abc001001.jpg
+260b3c2d20a7126de96671d29f73ba09d13b61ba  SOURCE_DIRECTORY/intermediate_files/abc002001.jpg
+a08c4d5a76d1b8734487be6ffcba66a9baf475c4  SOURCE_DIRECTORY/targets/T001.tif
+40f9993d06945544fa57af88f7c3e9102ecc69e3  SOURCE_DIRECTORY/targets/T002.tif

data/spec/fixtures/files/manifest-sha1-collection-title.txt

@@ -0,0 +1,10 @@
+59ec01f979a76b968bc579e5cd0ceb3bcf3e629f data/abc001/abc001001.tif
+d0a2f2482783ae3c38d06f3cdeaa1a306cc043ad data/abc001/abc001002.tif
+38ee72ab417192589f3a54ef1016131c7d7e9e4e data/abc002/abc002001.tif
+c227abc095d3b758051c1c1c9e922494b6b6e0b0 data/abc003/abc003001.wav
+541af8df7d2c631382bd0ec189476894d0323df5 data/abc003/abc003002.wav
+a08c4d5a76d1b8734487be6ffcba66a9baf475c4 data/dpc_targets/T001.tif
+40f9993d06945544fa57af88f7c3e9102ecc69e3 data/dpc_targets/T002.tif
+49d77d27bf82ec3dafa1967490594883f5e8b432 data/intermediate_files/abc001001.jpg
+260b3c2d20a7126de96671d29f73ba09d13b61ba data/intermediate_files/abc002001.jpg
+8a7878863fd183436f50060343d3757747772d9f data/metadata.txt

data/spec/fixtures/files/manifest-sha1.txt

@@ -5,4 +5,6 @@ c227abc095d3b758051c1c1c9e922494b6b6e0b0 data/abc003/abc003001.wav
 541af8df7d2c631382bd0ec189476894d0323df5 data/abc003/abc003002.wav
 a08c4d5a76d1b8734487be6ffcba66a9baf475c4 data/dpc_targets/T001.tif
 40f9993d06945544fa57af88f7c3e9102ecc69e3 data/dpc_targets/T002.tif
+49d77d27bf82ec3dafa1967490594883f5e8b432 data/intermediate_files/abc001001.jpg
+260b3c2d20a7126de96671d29f73ba09d13b61ba data/intermediate_files/abc002001.jpg
 913699468893882d1dec463f3df1a405c7f32784 data/metadata.txt

data/spec/integration/dpc_folder_converter_spec.rb

@@ -2,10 +2,49 @@ module Ddr::IngestTools::DpcFolderConverter
 
   RSpec.describe Converter do
 
+    shared_examples 'a conversion to standard ingest format' do
+      subject { described_class.new(converter_args) }
+      it 'produces the correct standard ingest format directory' do
+        results = subject.call
+        # Target directory contains all the expected files and only the expected files
+        expect(Array(Find.find(target_directory))).to match_array(expected_files)
+        # Target content files are same as source content files
+        expect(FileUtils.compare_file(File.join(data_directory, 'abc001', 'abc001001.tif'),
+                                      File.join(source_directory, 'abc001001.tif'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'abc001', 'abc001002.tif'),
+                                      File.join(source_directory, 'abc001002.tif'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'abc002', 'abc002001.tif'),
+                                      File.join(source_directory, 'abc002001.tif'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'abc003', 'abc003001.wav'),
+                                      File.join(source_directory, 'g', 'abc003001.wav'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'abc003', 'abc003002.wav'),
+                                      File.join(source_directory, 'g', 'abc003002.wav'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'dpc_targets', 'T001.tif'),
+                                      File.join(source_directory, 'targets', 'T001.tif'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'dpc_targets', 'T002.tif'),
+                                      File.join(source_directory, 'targets', 'T002.tif'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'intermediate_files', 'abc001001.jpg'),
+                                      File.join(source_directory, 'intermediate_files', 'abc001001.jpg'))).to be true
+        expect(FileUtils.compare_file(File.join(data_directory, 'intermediate_files', 'abc002001.jpg'),
+                                      File.join(source_directory, 'intermediate_files', 'abc002001.jpg'))).to be true
+        # Generated metadata file contains the expected contents
+        metadata_lines = File.readlines(File.join(data_directory, 'metadata.txt')).map(&:chomp)
+        expect(metadata_lines).to match_array(expected_metadata)
+        # Generated manifest contains the expected contents (ignoring line order)
+        generated_manifest = File.readlines(File.join(File.join(target_directory, 'manifest-sha1.txt'))).sort
+        expect(generated_manifest).to match_array(expected_manifest)
+        # Conversion process produces the expected errors
+        expect(results.errors).to match_array(checksum_errors)
+      end
+    end
+
     let(:source_directory) { Dir.mktmpdir('dpc') }
     let(:target_directory) { Dir.mktmpdir('sif') }
     let(:data_directory) { File.join(target_directory, 'data') }
     let(:item_id_length) { 6 }
+    let(:checksums_directory) { Dir.mktmpdir('checksums') }
+    let(:checksums) { File.join(checksums_directory, 'checksums-sha1.txt') }
+    let(:converter_args) { { source: source_directory, target: target_directory, item_id_length: item_id_length } }
     let(:expected_files) { [
         target_directory,
         File.join(target_directory, 'bag-info.txt'),
@@ -22,6 +61,9 @@ module Ddr::IngestTools::DpcFolderConverter
         File.join(data_directory, 'dpc_targets'),
         File.join(data_directory, 'dpc_targets', 'T001.tif'),
         File.join(data_directory, 'dpc_targets', 'T002.tif'),
+        File.join(data_directory, 'intermediate_files'),
+        File.join(data_directory, 'intermediate_files', 'abc001001.jpg'),
+        File.join(data_directory, 'intermediate_files', 'abc002001.jpg'),
         File.join(data_directory, 'metadata.txt'),
         File.join(target_directory, 'manifest-md5.txt'),
         File.join(target_directory, 'manifest-sha1.txt'),
@@ -41,8 +83,9 @@ module Ddr::IngestTools::DpcFolderConverter
         "dpc_targets/T001.tif\tT001",
         "dpc_targets/T002.tif\tT002"
     ] }
-
-    subject { Converter.new(source_directory, target_directory, item_id_length) }
+    let(:expected_manifest) do
+      File.readlines(File.join('spec', 'fixtures', 'files', 'manifest-sha1.txt')).sort
+    end
 
     before do
       File.open(File.join(source_directory, 'Thumbs.db'), 'w') { |f| f.write('Thumbs') }
@@ -53,35 +96,102 @@ module Ddr::IngestTools::DpcFolderConverter
       Dir.mkdir(File.join(source_directory,'g'))
       File.open(File.join(source_directory, 'g', 'abc003001.wav'), 'w') { |f| f.write('abc003001') }
       File.open(File.join(source_directory, 'g', 'abc003002.wav'), 'w') { |f| f.write('abc003002') }
-      Dir.mkdir(File.join(source_directory,'targets'))
+      Dir.mkdir(File.join(source_directory, 'intermediate_files'))
+      File.open(File.join(source_directory, 'intermediate_files', 'abc001001.jpg'), 'w') { |f| f.write('abc001001 jpg')}
+      File.open(File.join(source_directory, 'intermediate_files', 'abc002001.jpg'), 'w') { |f| f.write('abc002001 jpg')}
+      Dir.mkdir(File.join(source_directory, 'targets'))
       File.open(File.join(source_directory, 'targets', 'T001.tif'), 'w') { |f| f.write('T001') }
       File.open(File.join(source_directory, 'targets', 'T002.tif'), 'w') { |f| f.write('T002') }
     end
 
-    it 'produces the correct standard ingest format directory' do
-      subject.call
-      expect(Array(Find.find(target_directory))).to match_array(expected_files)
-      expect(FileUtils.compare_file(File.join(data_directory, 'abc001', 'abc001001.tif'),
-                                    File.join(source_directory, 'abc001001.tif'))).to be true
-      expect(FileUtils.compare_file(File.join(data_directory, 'abc001', 'abc001002.tif'),
-                                    File.join(source_directory, 'abc001002.tif'))).to be true
-      expect(FileUtils.compare_file(File.join(data_directory, 'abc002', 'abc002001.tif'),
-                                    File.join(source_directory, 'abc002001.tif'))).to be true
-      expect(FileUtils.compare_file(File.join(data_directory, 'abc003', 'abc003001.wav'),
-                                    File.join(source_directory, 'g', 'abc003001.wav'))).to be true
-      expect(FileUtils.compare_file(File.join(data_directory, 'abc003', 'abc003002.wav'),
-                                    File.join(source_directory, 'g', 'abc003002.wav'))).to be true
-      expect(FileUtils.compare_file(File.join(data_directory, 'dpc_targets', 'T001.tif'),
-                                    File.join(source_directory, 'targets', 'T001.tif'))).to be true
-      expect(FileUtils.compare_file(File.join(data_directory, 'dpc_targets', 'T002.tif'),
-                                    File.join(source_directory, 'targets', 'T002.tif'))).to be true
-      metadata_lines = File.readlines(File.join(data_directory, 'metadata.txt')).map(&:strip)
-      expect(metadata_lines).to match_array(expected_metadata)
-      # expect(FileUtils.compare_file(File.join(target_directory, 'manifest-sha1.txt'),
-      #                               File.join(File.dirname(__FILE__), '..', 'fixtures', 'files', 'manifest-sha1.txt'))).to be true
-      expect(FileUtils.compare_file(File.join(target_directory, 'manifest-sha1.txt'),
-                                    File.join('spec', 'fixtures', 'files', 'manifest-sha1.txt'))).to be true
+    describe 'external checksum files' do
+      describe 'external checksum file' do
+        before do
+          File.open(checksums, 'w') do |f|
+            f << File.open(checksum_file_template).read.gsub('SOURCE_DIRECTORY', source_directory)
+          end
+          converter_args[:checksums] = checksums
+        end
+        describe 'mismatch' do
+          let(:checksum_file_template) { File.join('spec', 'fixtures', 'files', 'bad-checksums-sha1.txt') }
+          let(:checksum_errors) {
+            [ I18n.translate('errors.checksum_mismatch', { c1: 'd0a2f2482783ae3c83d06f3cdeaa1a306cc043ad',
+                                                           f1: File.join(source_directory, 'abc001002.tif'),
+                                                           c2: 'd0a2f2482783ae3c38d06f3cdeaa1a306cc043ad',
+                                                           f2: File.join(target_directory, 'data/abc001/abc001002.tif') }),
+              I18n.translate('errors.checksum_mismatch', { c1: 'c227abc095d3b758ab1c1c1c9e922494b6b6e0b0',
+                                                           f1: File.join(source_directory, 'g/abc003001.wav'),
+                                                           c2: 'c227abc095d3b758051c1c1c9e922494b6b6e0b0',
+                                                           f2: File.join(target_directory, 'data/abc003/abc003001.wav') }),
+              I18n.translate('errors.checksum_mismatch', { c1: '260b3c2d20a1726de96671d29f73ba09d13b61ba',
+                                                           f1: File.join(source_directory, 'intermediate_files/abc002001.jpg'),
+                                                           c2: '260b3c2d20a7126de96671d29f73ba09d13b61ba',
+                                                           f2: File.join(target_directory, 'data/intermediate_files/abc002001.jpg') }),
+              I18n.translate('errors.checksum_mismatch', { c1: 'a08c4d5a76d1b8735587be6ffcba66a9baf475c4',
+                                                           f1: File.join(source_directory, 'targets/T001.tif'),
+                                                           c2: 'a08c4d5a76d1b8734487be6ffcba66a9baf475c4',
+                                                           f2: File.join(target_directory, 'data/dpc_targets/T001.tif') })
+            ]
+          }
+          describe 'files are copied' do
+            before { converter_args[:copy_files] = true }
+            it_behaves_like 'a conversion to standard ingest format'
+          end
+          describe 'files are not copied' do
+            before { converter_args[:copy_files] = false }
+            it_behaves_like 'a conversion to standard ingest format'
+          end
+        end
+        describe 'no mismatch' do
+          let(:checksum_file_template) { File.join('spec', 'fixtures', 'files', 'good-checksums-sha1.txt') }
+          let(:checksum_errors) { [] }
+          describe 'files are copied' do
+            before { converter_args[:copy_files] = true }
+            it_behaves_like 'a conversion to standard ingest format'
+          end
+          describe 'files are not copied' do
+            before { converter_args[:copy_files] = false }
+            it_behaves_like 'a conversion to standard ingest format'
+          end
+        end
+      end
+
+      describe 'no external checksum file' do
+        let(:checksum_errors) { [] }
+        describe 'files are copied' do
+          before { converter_args[:copy_files] = true }
+          it_behaves_like 'a conversion to standard ingest format'
+        end
+        describe 'files are not copied' do
+          before { converter_args[:copy_files] = false }
+          it_behaves_like 'a conversion to standard ingest format'
+        end
+      end
     end
 
+    describe 'collection titles' do
+      let(:checksum_errors) { [] }
+      describe 'collection title provided' do
+        let(:expected_metadata) { [
+            "path\tlocal_id\ttitle",
+            "\t\tTest Collection",
+            "abc001\tabc001\t",
+            "abc002\tabc002\t",
+            "abc003\tabc003\t",
+            "abc001/abc001001.tif\tabc001001\t",
+            "abc001/abc001002.tif\tabc001002\t",
+            "abc002/abc002001.tif\tabc002001\t",
+            "abc003/abc003001.wav\tabc003001\t",
+            "abc003/abc003002.wav\tabc003002\t",
+            "dpc_targets/T001.tif\tT001\t",
+            "dpc_targets/T002.tif\tT002\t"
+        ] }
+        let(:expected_manifest) do
+          File.readlines(File.join('spec', 'fixtures', 'files', 'manifest-sha1-collection-title.txt')).sort
+        end
+        before { converter_args[:collection_title] = 'Test Collection' }
+        it_behaves_like 'a conversion to standard ingest format'
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'ddr/ingesttools'
+require 'i18n'
 
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.

data/spec/unit/checksum_file_spec.rb

@@ -0,0 +1,17 @@
+module Ddr::IngestTools
+
+  RSpec.describe ChecksumFile do
+
+    subject { described_class.new(checksum_filepath) }
+
+    let(:checksum_filepath) { File.join('spec', 'fixtures', 'files', 'manifest-sha1.txt') }
+
+    describe 'digest' do
+      it 'provides the requested digest' do
+        expect(subject.digest('data/abc001/abc001002.tif')).to eq('d0a2f2482783ae3c38d06f3cdeaa1a306cc043ad')
+        expect(subject.digest('not/in/checksum/file.txt')).to be nil
+      end
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ddr-ingesttools
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jim Coble
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-08 00:00:00.000000000 Z
+date: 2017-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bagit
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -83,14 +97,20 @@ files:
 - README.md
 - Rakefile
 - bin/convert_dpc_folder.rb
+- config/locales/en.yml
 - ddr-ingesttools.gemspec
 - lib/ddr/ingesttools.rb
+- lib/ddr/ingesttools/checksum_file.rb
 - lib/ddr/ingesttools/dpc_folder_converter.rb
 - lib/ddr/ingesttools/dpc_folder_converter/converter.rb
 - lib/ddr/ingesttools/version.rb
+- spec/fixtures/files/bad-checksums-sha1.txt
+- spec/fixtures/files/good-checksums-sha1.txt
+- spec/fixtures/files/manifest-sha1-collection-title.txt
 - spec/fixtures/files/manifest-sha1.txt
 - spec/integration/dpc_folder_converter_spec.rb
 - spec/spec_helper.rb
+- spec/unit/checksum_file_spec.rb
 homepage: https://github.com/duke-libraries/ddr-ingesttools
 licenses:
 - BSD-3-Clause
@@ -116,6 +136,10 @@ signing_key:
 specification_version: 4
 summary: Ruby tools supporting ingest into the Duke Digital Repository.
 test_files:
+- spec/fixtures/files/bad-checksums-sha1.txt
+- spec/fixtures/files/good-checksums-sha1.txt
+- spec/fixtures/files/manifest-sha1-collection-title.txt
 - spec/fixtures/files/manifest-sha1.txt
 - spec/integration/dpc_folder_converter_spec.rb
 - spec/spec_helper.rb
+- spec/unit/checksum_file_spec.rb