archive-utils 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 52a10a6b4f10f5b140e6c47e9d88dd25d4407610
+  data.tar.gz: 374996e4a353ff7876c4717bb5b73757bae24420
+SHA512:
+  metadata.gz: 6f60a79a35a425ed109f633a0ed1b3b066c97b806c00e928f33d17fd448b018be18f4f08d1c46c9f91b2f031aa0defadb53f58f622a4502d57a524be55ad6219
+  data.tar.gz: 3498dfacc552e52d3db4d0910b56eaa0da8ceac2b47059e15976744f0e2c18c167c82657894a1cffaba35789b35435f8c6251f5d65f0436a52c860b56d9f9186

data/lib/archive-utils.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'bundler/setup'
+Bundler.setup
+require 'digest'
+require 'find'
+require 'json/pure'
+require 'pathname'
+require 'systemu'
+
+# Should remove these dependencies from sdr-archive
+#require 'moab_stanford'
+#require 'rest-client'
+
+module Archive
+end
+
+require 'archive/bagit_bag'
+require 'archive/file_fixity'
+require 'archive/fixity'
+require 'archive/operating_system'
+require 'archive/tarfile'
+include Archive
+

data/lib/archive/bagit_bag.rb

@@ -0,0 +1,353 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'archive-utils'
+
+module Archive
+
+  # A BagIt bag contains a structured copy of a digital object for storage, transfer, or archive
+  # @see https://tools.ietf.org/html/draft-kunze-bagit-10
+  # This class can be used to create, parse, or validate a bag instance
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class BagitBag
+
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
+    def BagitBag.create_bag(pathname)
+      bag = BagitBag.new
+      bag.bag_pathname = pathname
+      bag.payload_pathname.mkpath
+      bag.write_bagit_txt
+      bag
+    end
+
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
+    def BagitBag.open_bag(pathname)
+      bag = BagitBag.new
+      bag.bag_pathname = pathname
+      raise "No bag found at #{bag.bag_pathname}" unless bag.bag_pathname.exist?
+      bagit_txt = bag.bag_pathname.join("bagit.txt")
+      raise "No bagit.txt file found at #{bagit_txt}" unless bagit_txt.exist?
+      bag
+    end
+
+    # @return [Pathname] The location of the bag home directory
+    def bag_pathname
+      @bag_pathname
+    end
+
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [Void] Set the location of the bag home directory
+    def bag_pathname=(pathname)
+      @bag_pathname = Pathname(pathname)
+    end
+
+    # @return [Pathname] The location of the bag data directory
+    def payload_pathname
+      bag_pathname.join('data')
+    end
+
+    # @return [Pathname] Generate the bagit.txt tag file
+    def write_bagit_txt
+      bagit_txt = bag_pathname.join("bagit.txt")
+      bagit_txt.open('w') do |f|
+       f.puts "Tag-File-Character-Encoding: UTF-8"
+       f.puts "BagIt-Version: 0.97"
+      end
+      bagit_txt
+    end
+
+    # @return [Hash<String,String] A hash containing the properties documented in the bagit.txt tagfile
+    def read_bagit_txt
+      properties = Hash.new
+      bagit_txt = bag_pathname.join("bagit.txt")
+      bagit_txt.readlines.each do |line|
+        line.chomp!.strip!
+        key,value = line.split(':',2)
+        properties[key.strip] = value.strip if value
+      end
+      properties
+    end
+
+    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
+    def bag_checksum_types
+      @bag_checksum_types ||= Fixity.default_checksum_types
+    end
+
+    # @param [Object] types The list of checksum types to be used when generating fixity data
+    # @return [Void] Set the list of checksum types to be used when generating fixity data
+    def bag_checksum_types=(*types)
+      @bag_checksum_types = Fixity.validate_checksum_types(*types)
+    end
+
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_dir The source location of the directory whose contents are to be bagged
+    # @return [Pathname] Generate file_fixity_hash and send it to #add_files_to_payload
+    def add_dir_to_payload (link_mode, source_dir)
+      file_fixity_hash = Fixity.generate_checksums(source_dir, source_dir.find ,bag_checksum_types)
+      add_files_to_payload(link_mode, source_dir, file_fixity_hash)
+      payload_pathname
+    end
+
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_basepath The source location of the directory whose contents are to be ingested
+    # @param [Hash<String,FileFixity>] file_fixity_hash The list of files (with fixity data) to be added to the payload
+    # @return [Pathname] Copy or link the files specified in the file_fixity_hash to the payload directory,
+    #   then update the payload manifest files
+    def add_files_to_payload(link_mode, source_basepath, file_fixity_hash)
+      file_fixity_hash.keys.each do |file_id|
+        source_pathname = source_basepath.join(file_id)
+        target_pathname = payload_pathname.join(file_id)
+        copy_file(link_mode, source_pathname, target_pathname)
+      end
+      write_manifest_checksums('manifest', add_data_prefix(file_fixity_hash))
+      payload_pathname
+    end
+
+    # @param [Hash<String,FileFixity>] file_fixity_hash key is file_id, values are Fixity objects containing checksums
+    # @return [Hash<String,FileFixity>] A revised hash with file_id paths prefixed with 'data/'
+    def add_data_prefix(file_fixity_hash)
+      new_hash = Hash.new
+      file_fixity_hash.values.each do |fixity|
+        fixity.file_id = "data/#{fixity.file_id}"
+        new_hash[fixity.file_id] = fixity
+      end
+      new_hash
+    end
+
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_pathname The source location of the file to be ingested
+    # @param [Pathname] target_pathname The location of the directory in which to place the file
+    # @return [Pathname] link or copy the specified file from source location to the target location
+    def copy_file(link_mode, source_pathname, target_pathname)
+      target_pathname.parent.mkpath
+      case link_mode
+        when :copy, nil
+          FileUtils.copy(source_pathname.to_s, target_pathname.to_s) # automatically dereferences symlinks
+        when :link
+          FileUtils.link(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
+        when :symlink
+          FileUtils.symlink(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
+        else
+          raise "Invalid link_mode: #{link_mode}, expected one of [:copy,:link,:symlink]"
+      end
+      target_pathname
+    end
+
+    # @param [Pathname,String] source_fullpath The location of the directory whose content will be tarred
+    # @param [Pathname,String] source_basepath The location of the directory to change to before doing the tar create
+    # @return [Tarfile] Create a tar archive of a directory into the payload directory,
+    #   generating checksums in parallel processes and recording those checksums in the payload manifests
+    def add_payload_tarfile(tarfile_id,source_fullpath, source_basepath)
+      tarfile = Tarfile.new
+      tarfile.source_basepath = Pathname(source_basepath)
+      tarfile.source_fullpath = Pathname(source_fullpath)
+      tarfile.tarfile_basepath = payload_pathname
+      tarfile.tarfile_fullpath = payload_pathname.join("#{tarfile_id}")
+      tarfile.create_tarfile
+      file_fixity_hash = Fixity.generate_checksums(bag_pathname,[tarfile.tarfile_fullpath],bag_checksum_types)
+      write_manifest_checksums('manifest', file_fixity_hash)
+      tarfile
+    end
+
+    # @return [Pathname] Generate the bag-info.txt tag file to record the payload size
+    def write_bag_info_txt
+      payload_size = bag_payload_size
+      bag_info_txt = bag_pathname.join("bag-info.txt")
+      bag_info_txt.open('w') do |f|
+        f.puts "External-Identifier: #{bag_pathname.basename}"
+        f.puts "Payload-Oxum: #{payload_size[:bytes]}.#{payload_size[:files]}"
+        f.puts "Bag-Size: #{bag_size_human(payload_size[:bytes])}"
+      end
+      bag_info_txt
+    end
+
+    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
+    #   derived from the payload directory contents
+    def bag_payload_size
+      payload_pathname.find.select{|f| f.file?}.inject({bytes: 0, files: 0}) do |hash,file|
+        hash[:bytes] += file.size
+        hash[:files] += 1
+        hash
+      end
+    end
+
+    # @param [Integer] bytes The total number of bytes in the payload
+    # @return [String] Human-readable rendition of the total payload size
+    def bag_size_human(bytes)
+      count = 0
+      size = bytes
+      while ( size >= 1024 and count < 4 )
+        size /= 1024.0
+        count += 1
+      end
+      if (count == 0)
+        return sprintf("%d B", size)
+      else
+        return sprintf("%.2f %s", size, %w[B KB MB GB TB][count] )
+      end
+    end
+
+    # @return [Hash<String,String] A hash containing the properties documented in the bag-info.txt tagfile
+    def read_bag_info_txt
+      properties = Hash.new
+      bag_info = bag_pathname.join("bag-info.txt")
+      bag_info.readlines.each do |line|
+        line.chomp!.strip!
+        key,value = line.split(':',2)
+        properties[key.strip] = value.strip if value
+      end
+      properties
+    end
+
+    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
+    #   derived from the Payload-Oxum property
+    def info_payload_size
+      info = read_bag_info_txt
+      size_array = info['Payload-Oxum'].split('.')
+      size_hash = {:bytes => size_array[0].to_i, :files => size_array[1].to_i}
+      size_hash
+    end
+
+    # @return [Boolean] Compare the actual measured payload size against the value recorded in bag-info.txt
+    def verify_payload_size
+      info_size = info_payload_size
+      bag_size = bag_payload_size
+      if info_size != bag_size
+        raise "Failed payload size verification! Expected: #{info_size}, Found: #{bag_size}"
+      end
+      true
+    end
+
+    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's root directory
+    def generate_tagfile_checksums
+      # get list of all files in the bag home dir, except those starting with 'tagmanifest'
+      tagfiles = bag_pathname.children.reject{|file| file.basename.to_s.start_with?('tagmanifest')}
+      # generate checksums, using bag home dir as the base directory for file ids (per bagit spec)
+      Fixity.generate_checksums(bag_pathname, tagfiles, bag_checksum_types )
+    end
+
+    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's payload
+    def generate_payload_checksums
+      # get list of all files in the data directory
+      path_list = payload_pathname.find
+      # generate checksums, but use bag home dir as the base directory for file ids (per bagit spec)
+      Fixity.generate_checksums(bag_pathname, path_list, bag_checksum_types)
+    end
+
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be updated
+    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data
+    # @param [String] open_mode The file open mode (default is 'a')
+    # @return [Hash<Symbol,Pathname] Update each of the manifests with data from the file_fixity_hash
+    def write_manifest_checksums(manifest_type, file_fixity_hash, open_mode='a')
+      manifests = Hash.new
+      self.bag_checksum_types.each do |checksum_type|
+        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
+        manifest_file = manifest_pathname.open(open_mode)
+        file_fixity_hash.values.each do |fixity|
+          checksum = fixity.get_checksum(checksum_type)
+          manifest_file.puts("#{checksum} #{fixity.file_id}") if checksum
+        end
+        manifest_file.close
+        manifests[checksum_type] = manifest_pathname
+      end
+      manifests
+    end
+
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
+    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the manifest files
+    def read_manifest_files(manifest_type)
+      file_fixity_hash = Hash.new
+      checksum_type_list = Array.new
+      Fixity.valid_checksum_ids.each do |checksum_type|
+        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
+        if manifest_pathname.file?
+          checksum_type_list << checksum_type
+          manifest_pathname.readlines.each do |line|
+            line.chomp!.strip!
+            checksum,file_id = line.split(/[\s*]+/,2)
+            file_fixity = file_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+            file_fixity.set_checksum(checksum_type,checksum)
+            file_fixity_hash[file_id] = file_fixity
+          end
+        end
+      end
+      self.bag_checksum_types = self.bag_checksum_types | checksum_type_list
+      file_fixity_hash
+     end
+
+    # @return [Boolean] Compare fixity data from the tag manifest files against the values measured by digesting the files
+    def verify_tagfile_manifests
+      manifest_type = 'tagmanifest'
+      manifest_fixity_hash = read_manifest_files(manifest_type)
+      bag_fixity_hash = generate_tagfile_checksums
+      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+    end
+
+    # @return [Boolean] Compare fixity data from the payload manifest files against the values measured by digesting the files
+    def verify_payload_manifests
+      manifest_type = 'manifest'
+      manifest_fixity_hash = read_manifest_files(manifest_type)
+      bag_fixity_hash = generate_payload_checksums
+      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+    end
+
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
+    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
+    # @return [Boolean] Compare fixity data from the manifest files against the values measured by digesting the files,
+    #   returning true if equal or false if not equal
+    def verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+      diff = manifest_diff(manifest_fixity_hash, bag_fixity_hash)
+      if diff.size > 0
+        raise "Failed #{manifest_type} verification! Differences: \n#{diff.inspect}"
+      end
+      true
+    end
+
+    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
+    # @return [Hash] A report of the differences between the fixity data from the manifest files
+    #   against the values measured by digesting the files
+    def manifest_diff(manifest_fixity_hash, bag_fixity_hash)
+      diff = Hash.new
+      (manifest_fixity_hash.keys | bag_fixity_hash.keys).each do |file_id|
+        manifest_fixity = manifest_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+        bag_fixity = bag_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+        if manifest_fixity != bag_fixity
+          diff[file_id] = manifest_fixity.diff(bag_fixity,'manifest','bag')
+        end
+      end
+      diff
+    end
+
+    # @return [Boolean] Validate the bag containing the digital object
+    def verify_bag
+      verify_bag_structure
+      verify_tagfile_manifests
+      verify_payload_size
+      verify_payload_manifests
+      true
+    end
+
+    # @return [Boolean] Test the existence of expected files, return true if files exist, raise exception if not
+    def verify_bag_structure
+      required_files = ['data','bagit.txt','bag-info.txt','manifest-sha256.txt','tagmanifest-sha256.txt']
+      required_files.each{|filename| verify_pathname(bag_pathname.join(filename))}
+      optional_files = []
+      true
+    end
+
+    # @param [Pathname] pathname The file whose existence should be verified
+    # @return [Boolean] Test the existence of the specified path.  Return true if file exists, raise exception if not
+    def verify_pathname(pathname)
+      raise "#{pathname.basename} not found at #{pathname}" unless pathname.exist?
+      true
+    end
+
+
+  end
+
+
+end

data/lib/archive/file_fixity.rb

@@ -0,0 +1,98 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'archive-utils'
+
+module Archive
+
+  # The fixity properties of a file, used to determine file content equivalence.
+  # Placing this data in a class by itself facilitates using the MD5, SHA1, etc checksums (and optionally the file size)
+  # as a single key when doing comparisons against other file instances.  The design assumes that this file fixity
+  # is sufficiently unique to act as a comparator for determining file equality or verifying checksum manifests.
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class FileFixity
+
+    # @param [Hash<Symbol,Object>] options Key,Value pairs specifying initial values of attributes
+    def initialize(options=nil)
+      @checksums=Hash.new
+      options = {} if options.nil?
+      options.each do |key,value|
+        #instance_variable_set("@#{key}", value)
+        send "#{key}=", value
+      end
+    end
+
+    # @return [String] The name of the file, relative to its base directory
+    #   (for payload files, path relative to the data folder.  For tag files, path relative to the bag home folder)
+    attr_accessor :file_id
+
+    # @return [Integer] The size of the file in bytes
+    attr_accessor :bytes
+
+    # @return [Hash<Symbol,String>] The MD5, SHA1, SHA256, etc checksum values of the file
+    attr_accessor :checksums
+
+    # @param [Symbol,String] type The type of checksum (e.g. :md5, :sha1, :sha256)
+    # @return [String] The value of the file digest
+    def get_checksum(type)
+      checksum_type = type.to_s.downcase.to_sym
+      self.checksums[checksum_type]
+    end
+
+    # @param type [Symbol,String] The type of checksum
+    # @param value [String] value of the file digest
+    # @return [void] Set the value for the specified checksum type in the checksum hash
+    def set_checksum(type,value)
+      checksum_type = type.to_s.downcase.to_sym
+      Fixity.validate_checksum_types(checksum_type)
+      self.checksums[checksum_type] = value
+    end
+
+    # @param other [FileFixity] The other file fixity being compared to this fixity
+    # @return [Boolean] Returns true if self and other have comparable fixity data.
+    def eql?(other)
+      matching_checksum_types = self.checksums.keys & other.checksums.keys
+      return false if matching_checksum_types.size == 0
+      matching_checksum_types.each do |type|
+        return false if self.checksums[type] != other.checksums[type]
+      end
+      true
+    end
+
+    # (see #eql?)
+    def ==(other)
+      eql?(other)
+    end
+
+    # @return [Fixnum] Compute a hash-code for the fixity value array.
+    #   Two file instances with the same content will have the same hash code (and will compare using eql?).
+    # @note The hash and eql? methods override the methods inherited from Object.
+    #   These methods ensure that instances of this class can be used as Hash keys.  See
+    #   * {http://www.paulbutcher.com/2007/10/navigating-the-equality-maze/}
+    #   * {http://techbot.me/2011/05/ruby-basics-equality-operators-ruby/}
+    #   Also overriden is {#==} so that equality tests in other contexts will also return the expected result.
+    def hash
+      [self.file_id].hash
+    end
+
+    # @param [FileFixity] other The other FileFixity object being compared to this one
+    # @param [String] left The label to use for values from this base FileFixity object
+    # @param [String] right he label to use for values from the other FileFixity object
+    # @return [Hash<symbol,Hash<String,String>] details of the checksum differences between fixity objects
+    def diff(other,left='base',right='other')
+      diff_hash = Hash.new
+      matching_checksum_types = (self.checksums.keys & other.checksums.keys)
+      matching_checksum_types = (self.checksums.keys | other.checksums.keys) if matching_checksum_types.empty?
+      matching_checksum_types.each do |type|
+        base_checksum = self.checksums[type]
+        other_checksum = other.checksums[type]
+        if base_checksum != other_checksum
+          diff_hash[type] = {left => base_checksum, right => other_checksum }
+        end
+      end
+      return diff_hash.size > 0 ? diff_hash : nil
+    end
+
+  end
+
+end

data/lib/archive/fixity.rb

@@ -0,0 +1,155 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'archive-utils'
+
+module Archive
+
+  # A Struct to hold properties of a given checksum digest type
+  ChecksumType = Struct.new(:id, :hex_length, :names)
+
+  # A helper class that facilites the generation and processing of checksums
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class Fixity
+
+    @@default_checksum_types = [:sha1, :sha256]
+
+    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
+    def Fixity.default_checksum_types
+      @@default_checksum_types
+    end
+
+    # @param [Array<Symbol>] types The list of checksum types to be used when generating fixity data
+    # @return [Void] Set the list of checksum types to be used when generating fixity data
+    def Fixity.default_checksum_types=(*types)
+      @@default_checksum_types = Fixity.validate_checksum_types(*types)
+    end
+
+    @@valid_checksum_types = [
+        ChecksumType.new(:md5, 32, ['MD5']),
+        ChecksumType.new(:sha1, 40, ['SHA-1', 'SHA1']),
+        ChecksumType.new(:sha256, 64, ['SHA-256', 'SHA256']),
+        ChecksumType.new(:sha384, 96, ['SHA-384', 'SHA384']),
+        ChecksumType.new(:sha512, 128, ['SHA-512', 'SHA512'])
+    ]
+
+    # @return [Array<ChecksumType>] The list of allowed ChecksumType structs containing the type's properties
+    def Fixity.valid_checksum_types
+      @@valid_checksum_types
+    end
+
+    # @return [Array<Symbol>] The list of allowed checksum types
+    def Fixity.valid_checksum_ids
+      @@valid_checksum_types.map { |type| type.id }
+    end
+
+    # @param [Array<Symbol>] types The list of checksum types being specified by the caller
+    # @return [Object] The list of specified checksum types after being checked for validity
+    def Fixity.validate_checksum_types(*types)
+      checksum_types = types.flatten
+      invalid_types = checksum_types - valid_checksum_ids
+      raise "Invalid digest type specified: #{invalid_types.inspect}" unless invalid_types.empty?
+      checksum_types
+    end
+
+    # @param [Array<Symbol>] checksum_types The list of checksum types being specified by the caller
+    # @return [Array<Digest::Class>] The list of digest implementation objects that will generate the checksums
+    def Fixity.get_digesters(checksum_types=@@default_checksum_types)
+      checksum_types.inject(Hash.new) do |digesters, checksum_type|
+        case checksum_type
+          when :md5
+            digesters[checksum_type] = Digest::MD5.new
+          when :sha1
+            digesters[checksum_type] = Digest::SHA1.new
+          when :sha256
+            digesters[checksum_type] = Digest::SHA2.new(256)
+          when :sha384
+            digesters[checksum_type] = Digest::SHA2.new(384)
+          when :sha512
+            digesters[checksum_type] = Digest::SHA2.new(512)
+          else
+            raise "Unrecognized checksum type: #{checksum_type}"
+        end
+        digesters
+      end
+    end
+
+    # @param pathname [Pathname] The location of the file to be digested
+    # @param [Object] base_pathname The base directory from which relative paths (file IDS) will be derived
+    # @param [Object] checksum_types The list of checksum types being specified by the caller (or default list)
+    # @return [FileFixity] Generate a FileFixity instance containing fixity properties measured from of a physical file
+    def Fixity.fixity_from_file(pathname, base_pathname, checksum_types=@@default_checksum_types)
+      file_fixity = FileFixity.new
+      file_fixity.file_id = pathname.relative_path_from(base_pathname).to_s
+      file_fixity.bytes = pathname.size
+      digesters = Fixity.get_digesters(checksum_types)
+      pathname.open("r") do |stream|
+        while buffer = stream.read(8192)
+          digesters.values.each { |digest| digest.update(buffer) }
+        end
+      end
+      digesters.each { |checksum_type, digest| file_fixity.checksums[checksum_type] = digest.hexdigest }
+      file_fixity
+    end
+
+     # @param [Pathname] base_pathname The directory path used as the base for deriving relative paths (file IDs)
+    # @param [Array<Pathname>] path_list The list of pathnames for files whose fixity will be generated
+    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the actual files
+    def Fixity.generate_checksums(base_pathname, path_list, checksum_types=@@default_checksum_types)
+      path_list = base_pathname.find  if path_list.nil?
+      file_fixity_hash = Hash.new
+      path_list.select{|pathname| pathname.file?}.each  do |file|
+        file_fixity = Fixity.fixity_from_file(file, base_pathname, checksum_types)
+        file_fixity_hash[file_fixity.file_id] = file_fixity
+      end
+      file_fixity_hash
+    end
+
+    # @param [Integer] length The length of the checksum value in hex format
+    # @return [ChecksumType] The ChecksumType struct that contains the properties of the matching checksum type
+    def Fixity.type_for_length(length)
+      @@valid_checksum_types.select {|type| type.hex_length == length}.first
+    end
+
+    # @param [Object] file_id The filename or relative path of the file from its base directory
+    # @param [Object] checksum_values The digest values of the file
+    # @return [FileFixity] Generate a FileFixity instance containing fixity properties supplied by the caller
+    def Fixity.fixity_from_checksum_values(file_id, checksum_values)
+      file_fixity = FileFixity.new
+      file_fixity.file_id = file_id
+      checksum_values.each do |digest|
+        checksum_type = Fixity.type_for_length(digest.length)
+        file_fixity.checksums[checksum_type.id] = digest
+      end
+      file_fixity
+    end
+
+    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @return [Hash<String,Hash<Symbol,String] A hash containing file ids and checksum data derived from the file_fixity_hash
+    def Fixity.file_checksum_hash(file_fixity_hash)
+      checksum_hash = Hash.new
+      file_fixity_hash.values.each{|file| checksum_hash[file.file_id] = file.checksums}
+      checksum_hash
+    end
+
+    # @param [Symbol,String] checksum_type The type of checksum digest to be generated
+    # @param [Pathname,String] file_pathname The location of the file to digest
+    # @return [String] The operating system shell command that will generate the checksum digest value
+    def Fixity.openssl_digest_command(checksum_type,file_pathname)
+      command = "openssl dgst -#{checksum_type} #{file_pathname}"
+      command
+    end
+
+    # @param [Symbol,String] checksum_type The type of checksum digest to be generated
+    # @param [Pathname,String] file_pathname The location of the file to digest
+    # @return [String] The checksum digest value for the file
+    def Fixity.openssl_digest(checksum_type,file_pathname)
+      command = openssl_digest_command(checksum_type,file_pathname)
+      stdout = OperatingSystem.execute(command)
+      checksum = stdout.scan(/[A-Za-z0-9]+/).last
+      checksum
+    end
+
+  end
+
+end

data/lib/archive/operating_system.rb

@@ -0,0 +1,33 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'archive-utils'
+
+module Archive
+
+  # A wrapper class around the systemu gem that is used for shelling out to the operating system
+  # and executing a command
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class OperatingSystem
+
+    # Executes a system command in a subprocess.
+    # The method will return stdout from the command if execution was successful.
+    # The method will raise an exception if if execution fails.
+    # The exception's message will contain the explaination of the failure.
+    # @param [String] command the command to be executed
+    # @return [String] stdout from the command if execution was successful
+    def OperatingSystem.execute(command)
+      status, stdout, stderr = systemu(command)
+      if (status.exitstatus != 0)
+        raise stderr
+      end
+      return stdout
+    rescue
+      msg = "Command failed to execute: [#{command}] caused by <STDERR = #{stderr.split($/).join('; ')}>"
+      msg << " STDOUT = #{stdout.split($/).join('; ')}" if (stdout && (stdout.length > 0))
+      raise msg
+    end
+
+  end
+
+end

data/lib/archive/tarfile.rb

@@ -0,0 +1,160 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'archive-utils'
+
+module Archive
+
+  # A tar archive file containing a set of digital object files
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class Tarfile
+
+    # @return [String] create archive of the specified format
+    # * gnu = GNU tar 1.13.x format
+    # * posix = POSIX 1003.1-2001 (pax) format
+    attr_accessor :format
+
+    # @return [Boolean] Follow symlinks and archive the files they point to
+    attr_accessor :dereference
+
+    # @return [Boolean] Verify that files were copied faithfully
+    attr_accessor :verify
+
+    # @return [Boolean] Create/list/extract multi-volume archive (not yet implemented)
+    attr_accessor :multi_volume
+
+    # @param [Hash<Symbol,Object>] options Key,Value pairs specifying initial values of attributes
+    # @return [Tarfile] Initialize a new Tarfile object
+    def initialize(options=nil)
+      # set defaults
+      @format=:posix
+      @dereference = true
+      @verify = false
+      @multi_volume = false
+      # override defaults
+      options={} if options.nil?
+      options.each do |key,value|
+        #instance_variable_set("@#{key}", value)
+        send "#{key}=", value
+      end
+    end
+
+    # @return [Pathname] The full path of the ancestor dir in which the tar file resides
+    def tarfile_basepath
+      raise "Tarfile basepath is nil" unless @tarfile_basepath
+      @tarfile_basepath
+    end
+
+    # @param [Pathname,String] basepath The full path of the ancestor dir in which the tar file resides
+    # @return [Void] Set the full path of the ancestor dir in which the tar file resides
+    def tarfile_basepath=(basepath)
+      raise "No pathname specified" unless basepath
+      @tarfile_basepath = Pathname(basepath).expand_path
+    end
+
+    # @return [Pathname] the full path of the tar archive file to be created or extracted from
+    def tarfile_fullpath
+      @tarfile_fullpath
+    end
+
+    # @param [Pathname,String] fullpath The full path of tar file
+    # @return [Void] Sets the full path of tar file
+    def tarfile_fullpath=(fullpath)
+      @tarfile_fullpath = Pathname(fullpath).expand_path
+    end
+
+    # @return [String] The id (path relative to basepath) of the tar file
+    def tarfile_relative_path
+      @tarfile_fullpath.relative_path_from(@tarfile_basepath).to_s
+    end
+
+    # @return [Pathname] The full path of the source file or directory being archived
+    def source_fullpath
+      raise "Source pathname is nil" unless @source_pathname
+      @source_pathname
+    end
+
+    # @param [Pathname,String] source The full path of the source file or directory being archived
+    # @return [Void] Set the full path of the source file or directory being archived
+    def source_fullpath=(source)
+      raise "No pathname specified" unless source
+      @source_pathname = Pathname(source).expand_path
+    end
+
+    # @return [Pathname] The directory that is the basis of relative paths
+    def source_basepath
+      @source_basepath
+    end
+
+    # @param [Pathname,String] base The directory that is the basis of relative paths
+    # @return [Void] Set the base path of the source file or directory being archived
+    def source_basepath=(base)
+      raise "No pathname specified" unless base
+      @source_basepath = Pathname(base).expand_path
+    end
+
+    # @return [Pathname] The relative path from the source base directory to the source directory
+    def source_relative_path
+        source_fullpath.relative_path_from(source_basepath)
+    end
+
+    # @return [String] The shell command string to be used to create the tarfile
+    def create_cmd
+      command = "tar --create --file=#{tarfile_fullpath} --format=#{@format} "
+      command << "--dereference " if @dereference
+      command << "--verify " if @verify
+      command << "--directory='#{source_basepath}' " if source_basepath
+      command << source_relative_path.to_s
+      command
+    end
+
+    # @return [Tarfile] Shell out to the operating system and create the tar archive file
+    def create_tarfile
+      command = create_cmd
+      OperatingSystem.execute(command)
+      self
+    end
+
+    # @return [String] The shell command that will list the tarfile's contents
+    def list_cmd
+      command = "tar --list --file=#{tarfile_fullpath} "
+      command
+    end
+
+    # @return [String] The list of the tarfile's contents
+    def list_tarfile
+      command = list_cmd
+      list = OperatingSystem.execute(command)
+      list
+    end
+
+    # @return [Pathname] The location of the directory into which the tarfile should be extracted
+    def target_pathname
+      raise "Target pathname is nil" unless @target_pathname
+      @target_pathname
+    end
+
+    # @param [Pathname,String] source The location of the directory into which the tarfile should be extracted
+    # @return [Void] Set the location of the directory into which the tarfile should be extracted
+    def target_pathname=(target)
+      raise "No target pathname specified" unless target
+      @target_pathname = Pathname(target).expand_path
+    end
+
+    # @return [String] The shell command that will extract the tarfile's contents    # @return [Void]
+    def extract_cmd
+      command = "tar --extract --file=#{tarfile_fullpath} "
+      command << "--directory='#{target_pathname}' " if target_pathname
+      command
+    end
+
+    # @return [String] Shell out to the operating system and extract the tar archive file
+    def extract_tarfile
+      command = extract_cmd
+      stdout = OperatingSystem.execute(command)
+      stdout
+    end
+
+  end
+
+end

data/lib/libdir.rb

@@ -0,0 +1,3 @@
+libdir = File.expand_path(File.join(File.dirname(__FILE__)))
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification
+name: archive-utils
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Darren Weber
+- Richard Anderson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json_pure
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: systemu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: equivalent-xml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: Contains classes to archive and retrieve digital object version content
+  and metadata
+email:
+- darren.weber@stanford.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/archive-utils.rb
+- lib/archive/bagit_bag.rb
+- lib/archive/file_fixity.rb
+- lib/archive/fixity.rb
+- lib/archive/operating_system.rb
+- lib/archive/tarfile.rb
+- lib/libdir.rb
+homepage: https://github.com/sul-dlss/archive-utils
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: Ruby utilities for data archival (BagIt, Fixity, Tarfile).
+test_files: []
+has_rdoc: