sdr-replication 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c04fc63fa050cdbb2c45b3100ce8026ef01cb046
-  data.tar.gz: bd7d2e898a1989494086e08186e68dcd1a02b8ca
+  metadata.gz: 97ad8544df9baab2d69065f1ef6612eae515fc78
+  data.tar.gz: a81bee33c44cd1fa8a28176fa938891bdaecdad1
 SHA512:
-  metadata.gz: c0a39e7551eddff9a371594663fb097ca435d123e904a03e399b3be52a2f73fad4564c672b999e3cf562d558076e233a534f69237ac47787752eac044c29c861
-  data.tar.gz: 59e27ee6f9765c65054f57bab410e3e3d716bf775903c2d92e05ba0597c1a7f28355f2ef5438c5200d052fcfdadd73aa36ba81fe9c3a1e59d702c938bd0f2c00
+  metadata.gz: 05d4d43b5ba1abc93bfd450eb3d93645d3a7d0685d905e1a1541bc1eafde910005af98755bbd9344074bc82f5882b0786a9ec31b1bc0f0428bd586fbc0bd1428
+  data.tar.gz: 27f7d6ca7037724abae331f4f3f16cd4fec143f42f2c292b00503a9acd166869cdf26289fd6dde186b66f92c26f038c858ec33d09099b68e111cbaa5e595c79c

data/lib/replication/sdr_object_version.rb

@@ -152,7 +152,7 @@ module Replication
     # @return [Replica] Copy the object version into a BagIt Bag in tarfile format
     def create_replica
       replica = self.replica
-      bag = BagitBag.create_bag(replica.bag_pathname)
+      bag = Archive::BagitBag.create_bag(replica.bag_pathname)
       bag.bag_checksum_types = [:sha256]
       bag.add_payload_tarfile("#{replica.replica_id}.tar",version_pathname, storage_object.object_pathname.parent)
       bag.write_bag_info_txt
@@ -163,4 +163,4 @@ module Replication
 
   end
 
-end
+end

data/lib/sdr_replication.rb

@@ -1,27 +1,16 @@
 require 'rubygems'
 require 'bundler/setup'
 Bundler.setup
-require 'digest'
-require 'find'
-require 'json/pure'
+require 'archive-utils'
 require 'moab_stanford'
-require 'pathname'
-require 'rest-client'
-require 'systemu'
-
 
 # The classes used for SDR Replication workflows
 module Replication
 end
 
 require 'replication/archive_catalog'
-require 'replication/bagit_bag'
-require 'replication/file_fixity'
-require 'replication/fixity'
-require 'replication/operating_system'
 require 'replication/replica'
 require 'replication/sdr_object'
 require 'replication/sdr_object_version'
-require 'replication/tarfile'
 include Replication
 

metadata

@@ -1,29 +1,44 @@
 --- !ruby/object:Gem::Specification
 name: sdr-replication
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
+- Darren Weber
 - Richard Anderson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json_pure
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: moab-versioning
   requirement: !ruby/object:Gem::Requirement
@@ -39,152 +54,148 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: systemu
+  name: archive-utils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: rest-client
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: fakeweb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.14.1
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.14.1
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
 description: Contains classes to archive and retrieve digital object version content
   and metadata
 email:
-- rnanders@stanford.edu
+- darren.weber@stanford.edu
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/libdir.rb
 - lib/replication/archive_catalog.rb
-- lib/replication/bagit_bag.rb
-- lib/replication/file_fixity.rb
-- lib/replication/fixity.rb
-- lib/replication/operating_system.rb
 - lib/replication/replica.rb
 - lib/replication/sdr_object.rb
 - lib/replication/sdr_object_version.rb
-- lib/replication/tarfile.rb
 - lib/sdr_replication.rb
-homepage: 
-licenses: []
+homepage: https://github.com/sul-dlss/sdr-replication
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -202,7 +213,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 2.2.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Core methods for support of SDR Preservation Core replication

data/lib/replication/bagit_bag.rb

@@ -1,353 +0,0 @@
-require File.join(File.dirname(__FILE__),'../libdir')
-require 'sdr_replication'
-
-module Replication
-
-  # A BagIt bag contains a structured copy of a digital object for storage, transfer, or replication
-  # @see https://tools.ietf.org/html/draft-kunze-bagit-10
-  # This class can be used to create, parse, or validate a bag instance
-  #
-  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
-  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
-  class BagitBag
-
-    # @param [Pathname,String] pathname The location of the bag home directory
-    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
-    def BagitBag.create_bag(pathname)
-      bag = BagitBag.new
-      bag.bag_pathname = pathname
-      bag.payload_pathname.mkpath
-      bag.write_bagit_txt
-      bag
-    end
-
-    # @param [Pathname,String] pathname The location of the bag home directory
-    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
-    def BagitBag.open_bag(pathname)
-      bag = BagitBag.new
-      bag.bag_pathname = pathname
-      raise "No bag found at #{bag.bag_pathname}" unless bag.bag_pathname.exist?
-      bagit_txt = bag.bag_pathname.join("bagit.txt")
-      raise "No bagit.txt file found at #{bagit_txt}" unless bagit_txt.exist?
-      bag
-    end
-
-    # @return [Pathname] The location of the bag home directory
-    def bag_pathname
-      @bag_pathname
-    end
-
-    # @param [Pathname,String] pathname The location of the bag home directory
-    # @return [Void] Set the location of the bag home directory
-    def bag_pathname=(pathname)
-      @bag_pathname = Pathname(pathname)
-    end
-
-    # @return [Pathname] The location of the bag data directory
-    def payload_pathname
-      bag_pathname.join('data')
-    end
-
-    # @return [Pathname] Generate the bagit.txt tag file
-    def write_bagit_txt
-      bagit_txt = bag_pathname.join("bagit.txt")
-      bagit_txt.open('w') do |f|
-       f.puts "Tag-File-Character-Encoding: UTF-8"
-       f.puts "BagIt-Version: 0.97"
-      end
-      bagit_txt
-    end
-
-    # @return [Hash<String,String] A hash containing the properties documented in the bagit.txt tagfile
-    def read_bagit_txt
-      properties = Hash.new
-      bagit_txt = bag_pathname.join("bagit.txt")
-      bagit_txt.readlines.each do |line|
-        line.chomp!.strip!
-        key,value = line.split(':',2)
-        properties[key.strip] = value.strip if value
-      end
-      properties
-    end
-
-    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
-    def bag_checksum_types
-      @bag_checksum_types ||= Fixity.default_checksum_types
-    end
-
-    # @param [Object] types The list of checksum types to be used when generating fixity data
-    # @return [Void] Set the list of checksum types to be used when generating fixity data
-    def bag_checksum_types=(*types)
-      @bag_checksum_types = Fixity.validate_checksum_types(*types)
-    end
-
-    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
-    # @param [Pathname] source_dir The source location of the directory whose contents are to be bagged
-    # @return [Pathname] Generate file_fixity_hash and send it to #add_files_to_payload
-    def add_dir_to_payload (link_mode, source_dir)
-      file_fixity_hash = Fixity.generate_checksums(source_dir, source_dir.find ,bag_checksum_types)
-      add_files_to_payload(link_mode, source_dir, file_fixity_hash)
-      payload_pathname
-    end
-
-    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
-    # @param [Pathname] source_basepath The source location of the directory whose contents are to be ingested
-    # @param [Hash<String,FileFixity>] file_fixity_hash The list of files (with fixity data) to be added to the payload
-    # @return [Pathname] Copy or link the files specified in the file_fixity_hash to the payload directory,
-    #   then update the payload manifest files
-    def add_files_to_payload(link_mode, source_basepath, file_fixity_hash)
-      file_fixity_hash.keys.each do |file_id|
-        source_pathname = source_basepath.join(file_id)
-        target_pathname = payload_pathname.join(file_id)
-        copy_file(link_mode, source_pathname, target_pathname)
-      end
-      write_manifest_checksums('manifest', add_data_prefix(file_fixity_hash))
-      payload_pathname
-    end
-
-    # @param [Hash<String,FileFixity>] file_fixity_hash key is file_id, values are Fixity objects containing checksums
-    # @return [Hash<String,FileFixity>] A revised hash with file_id paths prefixed with 'data/'
-    def add_data_prefix(file_fixity_hash)
-      new_hash = Hash.new
-      file_fixity_hash.values.each do |fixity|
-        fixity.file_id = "data/#{fixity.file_id}"
-        new_hash[fixity.file_id] = fixity
-      end
-      new_hash
-    end
-
-    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
-    # @param [Pathname] source_pathname The source location of the file to be ingested
-    # @param [Pathname] target_pathname The location of the directory in which to place the file
-    # @return [Pathname] link or copy the specified file from source location to the target location
-    def copy_file(link_mode, source_pathname, target_pathname)
-      target_pathname.parent.mkpath
-      case link_mode
-        when :copy, nil
-          FileUtils.copy(source_pathname.to_s, target_pathname.to_s) # automatically dereferences symlinks
-        when :link
-          FileUtils.link(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
-        when :symlink
-          FileUtils.symlink(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
-        else
-          raise "Invalid link_mode: #{link_mode}, expected one of [:copy,:link,:symlink]"
-      end
-      target_pathname
-    end
-
-    # @param [Pathname,String] source_fullpath The location of the directory whose content will be tarred
-    # @param [Pathname,String] source_basepath The location of the directory to change to before doing the tar create
-    # @return [Tarfile] Create a tar archive of a directory into the payload directory,
-    #   generating checksums in parallel processes and recording those checksums in the payload manifests
-    def add_payload_tarfile(tarfile_id,source_fullpath, source_basepath)
-      tarfile = Tarfile.new
-      tarfile.source_basepath = Pathname(source_basepath)
-      tarfile.source_fullpath = Pathname(source_fullpath)
-      tarfile.tarfile_basepath = payload_pathname
-      tarfile.tarfile_fullpath = payload_pathname.join("#{tarfile_id}")
-      tarfile.create_tarfile
-      file_fixity_hash = Fixity.generate_checksums(bag_pathname,[tarfile.tarfile_fullpath],bag_checksum_types)
-      write_manifest_checksums('manifest', file_fixity_hash)
-      tarfile
-    end
-
-    # @return [Pathname] Generate the bag-info.txt tag file to record the payload size
-    def write_bag_info_txt
-      payload_size = bag_payload_size
-      bag_info_txt = bag_pathname.join("bag-info.txt")
-      bag_info_txt.open('w') do |f|
-        f.puts "External-Identifier: #{bag_pathname.basename}"
-        f.puts "Payload-Oxum: #{payload_size[:bytes]}.#{payload_size[:files]}"
-        f.puts "Bag-Size: #{bag_size_human(payload_size[:bytes])}"
-      end
-      bag_info_txt
-    end
-
-    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
-    #   derived from the payload directory contents
-    def bag_payload_size
-      payload_pathname.find.select{|f| f.file?}.inject({bytes: 0, files: 0}) do |hash,file|
-        hash[:bytes] += file.size
-        hash[:files] += 1
-        hash
-      end
-    end
-
-    # @param [Integer] bytes The total number of bytes in the payload
-    # @return [String] Human-readable rendition of the total payload size
-    def bag_size_human(bytes)
-      count = 0
-      size = bytes
-      while ( size >= 1024 and count < 4 )
-        size /= 1024.0
-        count += 1
-      end
-      if (count == 0)
-        return sprintf("%d B", size)
-      else
-        return sprintf("%.2f %s", size, %w[B KB MB GB TB][count] )
-      end
-    end
-
-    # @return [Hash<String,String] A hash containing the properties documented in the bag-info.txt tagfile
-    def read_bag_info_txt
-      properties = Hash.new
-      bag_info = bag_pathname.join("bag-info.txt")
-      bag_info.readlines.each do |line|
-        line.chomp!.strip!
-        key,value = line.split(':',2)
-        properties[key.strip] = value.strip if value
-      end
-      properties
-    end
-
-    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
-    #   derived from the Payload-Oxum property
-    def info_payload_size
-      info = read_bag_info_txt
-      size_array = info['Payload-Oxum'].split('.')
-      size_hash = {:bytes => size_array[0].to_i, :files => size_array[1].to_i}
-      size_hash
-    end
-
-    # @return [Boolean] Compare the actual measured payload size against the value recorded in bag-info.txt
-    def verify_payload_size
-      info_size = info_payload_size
-      bag_size = bag_payload_size
-      if info_size != bag_size
-        raise "Failed payload size verification! Expected: #{info_size}, Found: #{bag_size}"
-      end
-      true
-    end
-
-    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's root directory
-    def generate_tagfile_checksums
-      # get list of all files in the bag home dir, except those starting with 'tagmanifest'
-      tagfiles = bag_pathname.children.reject{|file| file.basename.to_s.start_with?('tagmanifest')}
-      # generate checksums, using bag home dir as the base directory for file ids (per bagit spec)
-      Fixity.generate_checksums(bag_pathname, tagfiles, bag_checksum_types )
-    end
-
-    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's payload
-    def generate_payload_checksums
-      # get list of all files in the data directory
-      path_list = payload_pathname.find
-      # generate checksums, but use bag home dir as the base directory for file ids (per bagit spec)
-      Fixity.generate_checksums(bag_pathname, path_list, bag_checksum_types)
-    end
-
-    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be updated
-    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data
-    # @param [String] open_mode The file open mode (default is 'a')
-    # @return [Hash<Symbol,Pathname] Update each of the manifests with data from the file_fixity_hash
-    def write_manifest_checksums(manifest_type, file_fixity_hash, open_mode='a')
-      manifests = Hash.new
-      self.bag_checksum_types.each do |checksum_type|
-        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
-        manifest_file = manifest_pathname.open(open_mode)
-        file_fixity_hash.values.each do |fixity|
-          checksum = fixity.get_checksum(checksum_type)
-          manifest_file.puts("#{checksum} #{fixity.file_id}") if checksum
-        end
-        manifest_file.close
-        manifests[checksum_type] = manifest_pathname
-      end
-      manifests
-    end
-
-    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
-    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the manifest files
-    def read_manifest_files(manifest_type)
-      file_fixity_hash = Hash.new
-      checksum_type_list = Array.new
-      Fixity.valid_checksum_ids.each do |checksum_type|
-        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
-        if manifest_pathname.file?
-          checksum_type_list << checksum_type
-          manifest_pathname.readlines.each do |line|
-            line.chomp!.strip!
-            checksum,file_id = line.split(/[\s*]+/,2)
-            file_fixity = file_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
-            file_fixity.set_checksum(checksum_type,checksum)
-            file_fixity_hash[file_id] = file_fixity
-          end
-        end
-      end
-      self.bag_checksum_types = self.bag_checksum_types | checksum_type_list
-      file_fixity_hash
-     end
-
-    # @return [Boolean] Compare fixity data from the tag manifest files against the values measured by digesting the files
-    def verify_tagfile_manifests
-      manifest_type = 'tagmanifest'
-      manifest_fixity_hash = read_manifest_files(manifest_type)
-      bag_fixity_hash = generate_tagfile_checksums
-      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
-    end
-
-    # @return [Boolean] Compare fixity data from the payload manifest files against the values measured by digesting the files
-    def verify_payload_manifests
-      manifest_type = 'manifest'
-      manifest_fixity_hash = read_manifest_files(manifest_type)
-      bag_fixity_hash = generate_payload_checksums
-      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
-    end
-
-    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
-    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
-    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
-    # @return [Boolean] Compare fixity data from the manifest files against the values measured by digesting the files,
-    #   returning true if equal or false if not equal
-    def verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
-      diff = manifest_diff(manifest_fixity_hash, bag_fixity_hash)
-      if diff.size > 0
-        raise "Failed #{manifest_type} verification! Differences: \n#{diff.inspect}"
-      end
-      true
-    end
-
-    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
-    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
-    # @return [Hash] A report of the differences between the fixity data from the manifest files
-    #   against the values measured by digesting the files
-    def manifest_diff(manifest_fixity_hash, bag_fixity_hash)
-      diff = Hash.new
-      (manifest_fixity_hash.keys | bag_fixity_hash.keys).each do |file_id|
-        manifest_fixity = manifest_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
-        bag_fixity = bag_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
-        if manifest_fixity != bag_fixity
-          diff[file_id] = manifest_fixity.diff(bag_fixity,'manifest','bag')
-        end
-      end
-      diff
-    end
-
-    # @return [Boolean] Validate the bag containing the digital object
-    def verify_bag
-      verify_bag_structure
-      verify_tagfile_manifests
-      verify_payload_size
-      verify_payload_manifests
-      true
-    end
-
-    # @return [Boolean] Test the existence of expected files, return true if files exist, raise exception if not
-    def verify_bag_structure
-      required_files = ['data','bagit.txt','bag-info.txt','manifest-sha256.txt','tagmanifest-sha256.txt']
-      required_files.each{|filename| verify_pathname(bag_pathname.join(filename))}
-      optional_files = []
-      true
-    end
-
-    # @param [Pathname] pathname The file whose existence should be verified
-    # @return [Boolean] Test the existence of the specified path.  Return true if file exists, raise exception if not
-    def verify_pathname(pathname)
-      raise "#{pathname.basename} not found at #{pathname}" unless pathname.exist?
-      true
-    end
-
-
-  end
-
-
-end

data/lib/replication/file_fixity.rb

@@ -1,98 +0,0 @@
-require File.join(File.dirname(__FILE__),'../libdir')
-require 'sdr_replication'
-
-module Replication
-
-  # The fixity properties of a file, used to determine file content equivalence.
-  # Placing this data in a class by itself facilitates using the MD5, SHA1, etc checksums (and optionally the file size)
-  # as a single key when doing comparisons against other file instances.  The design assumes that this file fixity
-  # is sufficiently unique to act as a comparator for determining file equality or verifying checksum manifests.
-  #
-  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
-  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
-  class FileFixity
-
-    # @param [Hash<Symbol,Object>] options Key,Value pairs specifying initial values of attributes
-    def initialize(options=nil)
-      @checksums=Hash.new
-      options = {} if options.nil?
-      options.each do |key,value|
-        #instance_variable_set("@#{key}", value)
-        send "#{key}=", value
-      end
-    end
-
-    # @return [String] The name of the file, relative to its base directory
-    #   (for payload files, path relative to the data folder.  For tag files, path relative to the bag home folder)
-    attr_accessor :file_id
-
-    # @return [Integer] The size of the file in bytes
-    attr_accessor :bytes
-
-    # @return [Hash<Symbol,String>] The MD5, SHA1, SHA256, etc checksum values of the file
-    attr_accessor :checksums
-
-    # @param [Symbol,String] type The type of checksum (e.g. :md5, :sha1, :sha256)
-    # @return [String] The value of the file digest
-    def get_checksum(type)
-      checksum_type = type.to_s.downcase.to_sym
-      self.checksums[checksum_type]
-    end
-
-    # @param type [Symbol,String] The type of checksum
-    # @param value [String] value of the file digest
-    # @return [void] Set the value for the specified checksum type in the checksum hash
-    def set_checksum(type,value)
-      checksum_type = type.to_s.downcase.to_sym
-      Fixity.validate_checksum_types(checksum_type)
-      self.checksums[checksum_type] = value
-    end
-
-    # @param other [FileFixity] The other file fixity being compared to this fixity
-    # @return [Boolean] Returns true if self and other have comparable fixity data.
-    def eql?(other)
-      matching_checksum_types = self.checksums.keys & other.checksums.keys
-      return false if matching_checksum_types.size == 0
-      matching_checksum_types.each do |type|
-        return false if self.checksums[type] != other.checksums[type]
-      end
-      true
-    end
-
-    # (see #eql?)
-    def ==(other)
-      eql?(other)
-    end
-
-    # @return [Fixnum] Compute a hash-code for the fixity value array.
-    #   Two file instances with the same content will have the same hash code (and will compare using eql?).
-    # @note The hash and eql? methods override the methods inherited from Object.
-    #   These methods ensure that instances of this class can be used as Hash keys.  See
-    #   * {http://www.paulbutcher.com/2007/10/navigating-the-equality-maze/}
-    #   * {http://techbot.me/2011/05/ruby-basics-equality-operators-ruby/}
-    #   Also overriden is {#==} so that equality tests in other contexts will also return the expected result.
-    def hash
-      [self.file_id].hash
-    end
-
-    # @param [FileFixity] other The other FileFixity object being compared to this one
-    # @param [String] left The label to use for values from this base FileFixity object
-    # @param [String] right he label to use for values from the other FileFixity object
-    # @return [Hash<symbol,Hash<String,String>] details of the checksum differences between fixity objects
-    def diff(other,left='base',right='other')
-      diff_hash = Hash.new
-      matching_checksum_types = (self.checksums.keys & other.checksums.keys)
-      matching_checksum_types = (self.checksums.keys | other.checksums.keys) if matching_checksum_types.empty?
-      matching_checksum_types.each do |type|
-        base_checksum = self.checksums[type]
-        other_checksum = other.checksums[type]
-        if base_checksum != other_checksum
-          diff_hash[type] = {left => base_checksum, right => other_checksum }
-        end
-      end
-      return diff_hash.size > 0 ? diff_hash : nil
-    end
-
-  end
-
-end

data/lib/replication/fixity.rb

@@ -1,155 +0,0 @@
-require File.join(File.dirname(__FILE__),'../libdir')
-require 'sdr_replication'
-
-module Replication
-
-  # A Struct to hold properties of a given checksum digest type
-  ChecksumType = Struct.new(:id, :hex_length, :names)
-
-  # A helper class that facilites the generation and processing of checksums
-  #
-  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
-  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
-  class Fixity
-
-    @@default_checksum_types = [:sha1, :sha256]
-
-    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
-    def Fixity.default_checksum_types
-      @@default_checksum_types
-    end
-
-    # @param [Array<Symbol>] types The list of checksum types to be used when generating fixity data
-    # @return [Void] Set the list of checksum types to be used when generating fixity data
-    def Fixity.default_checksum_types=(*types)
-      @@default_checksum_types = Fixity.validate_checksum_types(*types)
-    end
-
-    @@valid_checksum_types = [
-        ChecksumType.new(:md5, 32, ['MD5']),
-        ChecksumType.new(:sha1, 40, ['SHA-1', 'SHA1']),
-        ChecksumType.new(:sha256, 64, ['SHA-256', 'SHA256']),
-        ChecksumType.new(:sha384, 96, ['SHA-384', 'SHA384']),
-        ChecksumType.new(:sha512, 128, ['SHA-512', 'SHA512'])
-    ]
-
-    # @return [Array<ChecksumType>] The list of allowed ChecksumType structs containing the type's properties
-    def Fixity.valid_checksum_types
-      @@valid_checksum_types
-    end
-
-    # @return [Array<Symbol>] The list of allowed checksum types
-    def Fixity.valid_checksum_ids
-      @@valid_checksum_types.map { |type| type.id }
-    end
-
-    # @param [Array<Symbol>] types The list of checksum types being specified by the caller
-    # @return [Object] The list of specified checksum types after being checked for validity
-    def Fixity.validate_checksum_types(*types)
-      checksum_types = types.flatten
-      invalid_types = checksum_types - valid_checksum_ids
-      raise "Invalid digest type specified: #{invalid_types.inspect}" unless invalid_types.empty?
-      checksum_types
-    end
-
-    # @param [Array<Symbol>] checksum_types The list of checksum types being specified by the caller
-    # @return [Array<Digest::Class>] The list of digest implementation objects that will generate the checksums
-    def Fixity.get_digesters(checksum_types=@@default_checksum_types)
-      checksum_types.inject(Hash.new) do |digesters, checksum_type|
-        case checksum_type
-          when :md5
-            digesters[checksum_type] = Digest::MD5.new
-          when :sha1
-            digesters[checksum_type] = Digest::SHA1.new
-          when :sha256
-            digesters[checksum_type] = Digest::SHA2.new(256)
-          when :sha384
-            digesters[checksum_type] = Digest::SHA2.new(384)
-          when :sha512
-            digesters[checksum_type] = Digest::SHA2.new(512)
-          else
-            raise "Unrecognized checksum type: #{checksum_type}"
-        end
-        digesters
-      end
-    end
-
-    # @param pathname [Pathname] The location of the file to be digested
-    # @param [Object] base_pathname The base directory from which relative paths (file IDS) will be derived
-    # @param [Object] checksum_types The list of checksum types being specified by the caller (or default list)
-    # @return [FileFixity] Generate a FileFixity instance containing fixity properties measured from of a physical file
-    def Fixity.fixity_from_file(pathname, base_pathname, checksum_types=@@default_checksum_types)
-      file_fixity = FileFixity.new
-      file_fixity.file_id = pathname.relative_path_from(base_pathname).to_s
-      file_fixity.bytes = pathname.size
-      digesters = Fixity.get_digesters(checksum_types)
-      pathname.open("r") do |stream|
-        while buffer = stream.read(8192)
-          digesters.values.each { |digest| digest.update(buffer) }
-        end
-      end
-      digesters.each { |checksum_type, digest| file_fixity.checksums[checksum_type] = digest.hexdigest }
-      file_fixity
-    end
-
-     # @param [Pathname] base_pathname The directory path used as the base for deriving relative paths (file IDs)
-    # @param [Array<Pathname>] path_list The list of pathnames for files whose fixity will be generated
-    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the actual files
-    def Fixity.generate_checksums(base_pathname, path_list, checksum_types=@@default_checksum_types)
-      path_list = base_pathname.find  if path_list.nil?
-      file_fixity_hash = Hash.new
-      path_list.select{|pathname| pathname.file?}.each  do |file|
-        file_fixity = Fixity.fixity_from_file(file, base_pathname, checksum_types)
-        file_fixity_hash[file_fixity.file_id] = file_fixity
-      end
-      file_fixity_hash
-    end
-
-    # @param [Integer] length The length of the checksum value in hex format
-    # @return [ChecksumType] The ChecksumType struct that contains the properties of the matching checksum type
-    def Fixity.type_for_length(length)
-      @@valid_checksum_types.select {|type| type.hex_length == length}.first
-    end
-
-    # @param [Object] file_id The filename or relative path of the file from its base directory
-    # @param [Object] checksum_values The digest values of the file
-    # @return [FileFixity] Generate a FileFixity instance containing fixity properties supplied by the caller
-    def Fixity.fixity_from_checksum_values(file_id, checksum_values)
-      file_fixity = FileFixity.new
-      file_fixity.file_id = file_id
-      checksum_values.each do |digest|
-        checksum_type = Fixity.type_for_length(digest.length)
-        file_fixity.checksums[checksum_type.id] = digest
-      end
-      file_fixity
-    end
-
-    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data derived from the manifest files
-    # @return [Hash<String,Hash<Symbol,String] A hash containing file ids and checksum data derived from the file_fixity_hash
-    def Fixity.file_checksum_hash(file_fixity_hash)
-      checksum_hash = Hash.new
-      file_fixity_hash.values.each{|file| checksum_hash[file.file_id] = file.checksums}
-      checksum_hash
-    end
-
-    # @param [Symbol,String] checksum_type The type of checksum digest to be generated
-    # @param [Pathname,String] file_pathname The location of the file to digest
-    # @return [String] The operating system shell command that will generate the checksum digest value
-    def Fixity.openssl_digest_command(checksum_type,file_pathname)
-      command = "openssl dgst -#{checksum_type} #{file_pathname}"
-      command
-    end
-
-    # @param [Symbol,String] checksum_type The type of checksum digest to be generated
-    # @param [Pathname,String] file_pathname The location of the file to digest
-    # @return [String] The checksum digest value for the file
-    def Fixity.openssl_digest(checksum_type,file_pathname)
-      command = openssl_digest_command(checksum_type,file_pathname)
-      stdout = OperatingSystem.execute(command)
-      checksum = stdout.scan(/[A-Za-z0-9]+/).last
-      checksum
-    end
-
-  end
-
-end

data/lib/replication/operating_system.rb

@@ -1,33 +0,0 @@
-require File.join(File.dirname(__FILE__),'../libdir')
-require 'sdr_replication'
-
-module Replication
-
-  # A wrapper class around the systemu gem that is used for shelling out to the operating system
-  # and executing a command
-  #
-  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
-  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
-  class OperatingSystem
-
-    # Executes a system command in a subprocess.
-    # The method will return stdout from the command if execution was successful.
-    # The method will raise an exception if if execution fails.
-    # The exception's message will contain the explaination of the failure.
-    # @param [String] command the command to be executed
-    # @return [String] stdout from the command if execution was successful
-    def OperatingSystem.execute(command)
-      status, stdout, stderr = systemu(command)
-      if (status.exitstatus != 0)
-        raise stderr
-      end
-      return stdout
-    rescue
-      msg = "Command failed to execute: [#{command}] caused by <STDERR = #{stderr.split($/).join('; ')}>"
-      msg << " STDOUT = #{stdout.split($/).join('; ')}" if (stdout && (stdout.length > 0))
-      raise msg
-    end
-
-  end
-
-end

data/lib/replication/tarfile.rb

@@ -1,160 +0,0 @@
-require File.join(File.dirname(__FILE__),'../libdir')
-require 'sdr_replication'
-
-module Replication
-
-  # A tar archive file containing a set of digital object files
-  #
-  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
-  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
-  class Tarfile
-
-    # @return [String] create archive of the specified format
-    # * gnu = GNU tar 1.13.x format
-    # * posix = POSIX 1003.1-2001 (pax) format
-    attr_accessor :format
-
-    # @return [Boolean] Follow symlinks and archive the files they point to
-    attr_accessor :dereference
-
-    # @return [Boolean] Verify that files were copied faithfully
-    attr_accessor :verify
-
-    # @return [Boolean] Create/list/extract multi-volume archive (not yet implemented)
-    attr_accessor :multi_volume
-
-    # @param [Hash<Symbol,Object>] options Key,Value pairs specifying initial values of attributes
-    # @return [Tarfile] Initialize a new Tarfile object
-    def initialize(options=nil)
-      options={} if options.nil?
-      # set defaults
-      @format=:posix
-      @dereference = true
-      @verify = false
-      @multi_volume = false
-      # override defaults
-      options.each do |key,value|
-        #instance_variable_set("@#{key}", value)
-        send "#{key}=", value
-      end
-    end
-
-    # @return [Pathname] The full path of the ancestor dir in which the tar file resides
-    def tarfile_basepath
-      raise "Tarfile basepath is nil" unless @tarfile_basepath
-      @tarfile_basepath
-    end
-
-    # @param [Pathname,String] basepath The full path of the ancestor dir in which the tar file resides
-    # @return [Void] Set the full path of the ancestor dir in which the tar file resides
-    def tarfile_basepath=(basepath)
-      raise "No pathname specified" unless basepath
-      @tarfile_basepath = Pathname(basepath).expand_path
-    end
-
-    # @return [Pathname] the full path of the tar archive file to be created or extracted from
-    def tarfile_fullpath
-      @tarfile_fullpath
-    end
-
-    # @param [Pathname,String] fullpath The full path of tar file
-    # @return [Void] Sets the full path of tar file
-    def tarfile_fullpath=(fullpath)
-      @tarfile_fullpath = Pathname(fullpath).expand_path
-    end
-
-    # @return [String] The id (path relative to basepath) of the tar file
-    def tarfile_relative_path
-      @tarfile_fullpath.relative_path_from(@tarfile_basepath).to_s
-    end
-
-    # @return [Pathname] The full path of the source file or directory being archived
-    def source_fullpath
-      raise "Source pathname is nil" unless @source_pathname
-      @source_pathname
-    end
-
-    # @param [Pathname,String] source The full path of the source file or directory being archived
-    # @return [Void] Set the full path of the source file or directory being archived
-    def source_fullpath=(source)
-      raise "No pathname specified" unless source
-      @source_pathname = Pathname(source).expand_path
-    end
-
-    # @return [Pathname] The directory that is the basis of relative paths
-    def source_basepath
-      @source_basepath
-    end
-
-    # @param [Pathname,String] base The directory that is the basis of relative paths
-    # @return [Void] Set the base path of the source file or directory being archived
-    def source_basepath=(base)
-      raise "No pathname specified" unless base
-      @source_basepath = Pathname(base).expand_path
-    end
-
-    # @return [Pathname] The relative path from the source base directory to the source directory
-    def source_relative_path
-        source_fullpath.relative_path_from(source_basepath)
-    end
-
-    # @return [String] The shell command string to be used to create the tarfile
-    def create_cmd
-      command = "tar --create --file=#{tarfile_fullpath} --format=#{@format} "
-      command << "--dereference " if @dereference
-      command << "--verify " if @verify
-      command << "--directory='#{source_basepath}' " if source_basepath
-      command << source_relative_path.to_s
-      command
-    end
-
-    # @return [Tarfile] Shell out to the operating system and create the tar archive file
-    def create_tarfile
-      command = create_cmd
-      OperatingSystem.execute(command)
-      self
-    end
-
-    # @return [String] The shell command that will list the tarfile's contents
-    def list_cmd
-      command = "tar --list --file=#{tarfile_fullpath} "
-      command
-    end
-
-    # @return [String] The list of the tarfile's contents
-    def list_tarfile
-      command = list_cmd
-      list = OperatingSystem.execute(command)
-      list
-    end
-
-    # @return [Pathname] The location of the directory into which the tarfile should be extracted
-    def target_pathname
-      raise "Target pathname is nil" unless @target_pathname
-      @target_pathname
-    end
-
-    # @param [Pathname,String] source The location of the directory into which the tarfile should be extracted
-    # @return [Void] Set the location of the directory into which the tarfile should be extracted
-    def target_pathname=(target)
-      raise "No target pathname specified" unless target
-      @target_pathname = Pathname(target).expand_path
-    end
-
-    # @return [String] The shell command that will extract the tarfile's contents    # @return [Void]
-    def extract_cmd
-      command = "tar --extract --file=#{tarfile_fullpath} "
-      command << "--directory='#{target_pathname}' " if target_pathname
-      command
-    end
-
-    # @return [String] Shell out to the operating system and extract the tar archive file
-    def extract_tarfile
-      command = extract_cmd
-      stdout = OperatingSystem.execute(command)
-      stdout
-    end
-
-  end
-
-end