sdr-replication 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e99a0814e4383ec6287dec7df41825d786e65919
+  data.tar.gz: 4890c2758dd820f22ce8aa9015621e1b33dec9ae
+SHA512:
+  metadata.gz: deb3400e53fcbdf16cf8263ffe1deed70be83889633b4fecfa0de58e41a076e7f502de61d22f079b09ff6b81ddb29586bb6e898782f22e45010ecfd46d93828d
+  data.tar.gz: a0533d23addcc264e7aeca97f985bdad1d70e8081230e09ab5337a03314b058374541b1587f47a5f89dd2f558115677190d49b320b892a77e7031bb15ac9c3ff

data/lib/libdir.rb

@@ -0,0 +1,3 @@
+libdir = File.expand_path(File.join(File.dirname(__FILE__)))
+$LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
+

data/lib/replication/archive_catalog.rb

@@ -0,0 +1,110 @@
+require 'rubygems'
+require 'rest-client'
+
+module Replication
+
+  # A wrapper class based on {RestClient} used to interface with the Archive Catalog service.
+  # <br>
+  # <br>
+  # The default RestClient behavior is:
+  # * for results code between 200 and 207 a RestClient::Response will be returned
+  # * for results code 301, 302 or 307 the redirection will be followed if the request is a get or a head
+  # * for result code 303 the redirection will be followed and the request transformed into a get
+  # * for other cases a RestClient::Exception holding the Response will be raised
+  #
+  # But we are using a technique that forces RestClient to always provide the response
+  # <br>
+  # <br>
+  # RestClient::Response has these instance methods (some inherited from AbstractResponse):
+  # * args
+  # * body
+  # * code (e.g. 204)
+  # * description (e.g. "204 No Content |  0 bytes")
+  # * headers
+  # * net_http_res
+  #
+  # @see https://github.com/rest-client/rest-client
+  # @see http://rubydoc.info/gems/rest-client/1.6.7/frames
+  class ArchiveCatalog
+
+    @root_uri = 'http://localhost:3000'
+    @timeout = 120
+
+    # @see https://www.google.com/search?q="class+<<+self"+"attr_accessor"
+    class << self
+
+      # @return [String] The base or home URL of the Archive Catalog web service
+      attr_accessor :root_uri
+
+      # @return [Integer] seconds to wait for a response or to open a connection. Value nil disables the timeout.
+      attr_accessor :timeout
+
+      # The base RestClient resource to be used for requests
+      def root_resource
+        RestClient::Resource.new(@root_uri, {:open_timeout => @timeout, :timeout => @timeout})
+      end
+
+      # Get the item record from the specified table for the specified primary key.
+      # @param [String] table name of the database table
+      # @param [String] id primary key for the item in the database table
+      # @return [Hash] the row (in key,value hash) from the specified table for the specified identifier.
+      #    Response body contains the item data in JSON format, which is converted to a hash.
+      # @see http://tools.ietf.org/html/rfc2616#page-53
+      def get_item(table,id)
+        # Don't raise RestClient::Exception but return the response
+        headers = {:accept => 'application/json'}
+        response = root_resource["#{table}/#{id}.json"].get(headers) {|response, request, result| response }
+        case response.code.to_s
+          when '200'
+            JSON.parse(response.body)
+          else
+            raise response.description
+        end
+      end
+
+      # Retrieve an existing database record or add a new one using the data provided.
+      # @param [String] table name of the database table
+      # @param [Hash] hash the item data to be added to the database table
+      # @return [Hash] result containing the item data as if a GET were performed.
+      #    The HTTP response code for success is 201 (Created).
+      # @see http://en.wikipedia.org/wiki/POST_(HTTP)
+      # @see http://tools.ietf.org/html/rfc2616#page-54
+      def find_or_create_item(table,hash)
+        payload = hash.to_json
+        headers = {:content_type => :json, :accept => :json}
+        # Don't raise RestClient::Exception but return the response
+        response = root_resource["#{table}.json"].post(payload, headers) {|response, request, result| response }
+        case response.code.to_s
+          when '201'
+            JSON.parse(response.body)
+          else
+            raise response.description
+        end
+      end
+
+      # Update the database columns for the specified item using the hash data.
+      # @param [String] table name of the database table
+      # @param [String] id primary key for the item in the database table
+      # @param [Hash] hash the item data to be updated in the database table
+      # @return (Boolean) true if the HTTP response code is 204, per specification for PATCH or PUT request types.
+      #    Response body is empty, per same specification.
+      # @see https://tools.ietf.org/html/rfc5789
+      # @see http://stackoverflow.com/questions/797834/should-a-restful-put-operation-return-something/827045#827045
+      def update_item(table,id,hash)
+        payload = hash.to_json
+        headers = {:content_type => :json}
+        # Don't raise RestClient::Exception but return the response
+        response = root_resource["#{table}/#{id}.json"].patch(payload, headers) {|response, request, result| response }
+        case response.code.to_s
+          when '204'
+            true
+          else
+            raise response.description
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/replication/bagit_bag.rb

@@ -0,0 +1,337 @@
+require File.join(File.dirname(__FILE__),'../libdir')
+require 'sdr_replication'
+
+module Replication
+
+  # A BagIt bag contains a structured copy of a digital object for storage, transfer, or replication
+  # @see https://tools.ietf.org/html/draft-kunze-bagit-10
+  # This class can be used to create, parse, or validate a bag instance
+  #
+  # @note Copyright (c) 2014 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class BagitBag
+
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
+    def BagitBag.create_bag(pathname)
+      bag = BagitBag.new
+      bag.bag_pathname = pathname
+      bag.payload_pathname.mkpath
+      bag.write_bagit_txt
+      bag
+    end
+
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [BagitBag] Initialize a new bag, create home and payload folders, write bagit.txt file
+    def BagitBag.open_bag(pathname)
+      bag = BagitBag.new
+      bag.bag_pathname = pathname
+      raise "No bag found at #{bag.bag_pathname}" unless bag.bag_pathname.exist?
+      bagit_txt = bag.bag_pathname.join("bagit.txt")
+      raise "No bagit.txt file found at #{bagit_txt}" unless bagit_txt.exist?
+      bag
+    end
+
+    # @return [Pathname] The location of the bag home directory
+    def bag_pathname
+      @bag_pathname
+    end
+
+    # @param [Pathname,String] pathname The location of the bag home directory
+    # @return [Void] Set the location of the bag home directory
+    def bag_pathname=(pathname)
+      @bag_pathname = Pathname(pathname)
+    end
+
+    # @return [Pathname] The location of the bag data directory
+    def payload_pathname
+      bag_pathname.join('data')
+    end
+
+    # @return [Pathname] Generate the bagit.txt tag file
+    def write_bagit_txt
+      bagit_txt = bag_pathname.join("bagit.txt")
+      bagit_txt.open('w') do |f|
+       f.puts "Tag-File-Character-Encoding: UTF-8"
+       f.puts "BagIt-Version: 0.97"
+      end
+      bagit_txt
+    end
+
+    # @return [Hash<String,String] A hash containing the properties documented in the bagit.txt tagfile
+    def read_bagit_txt
+      properties = Hash.new
+      bagit_txt = bag_pathname.join("bagit.txt")
+      bagit_txt.readlines.each do |line|
+        line.chomp!.strip!
+        key,value = line.split(':',2)
+        properties[key.strip] = value.strip if value
+      end
+      properties
+    end
+
+    # @return [Array<Symbol>] The list of checksum types to be used when generating fixity data
+    def bag_checksum_types
+      @bag_checksum_types ||= Fixity.default_checksum_types
+    end
+
+    # @param [Object] types The list of checksum types to be used when generating fixity data
+    # @return [Void] Set the list of checksum types to be used when generating fixity data
+    def bag_checksum_types=(*types)
+      @bag_checksum_types = Fixity.validate_checksum_types(*types)
+    end
+
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_dir The source location of the directory whose contents are to be ingested
+    # @return [Pathname] Generate file_fixity_hash and send it to #add_payload_files
+    def add_payload_dir (link_mode, source_dir)
+      file_fixity_hash = Fixity.generate_checksums(source_dir, nil ,bag_checksum_types)
+      add_payload_files(link_mode, source_dir, file_fixity_hash)
+      payload_pathname
+    end
+
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_basepath The source location of the directory whose contents are to be ingested
+    # @param [Hash<String,FileFixity>] file_fixity_hash The list of files (with fixity data) to be added to the payload
+    # @return [Pathname] Copy or link the files specified in the file_fixity_hash to the payload directory,
+    #   then update the payload manifest files
+    def add_payload_files(link_mode, source_basepath, file_fixity_hash)
+      file_fixity_hash.keys.each do |file_id|
+        source_pathname = source_basepath.join(file_id)
+        target_pathname = payload_pathname.join(file_id)
+        copy_file(link_mode, source_pathname, target_pathname)
+      end
+      write_manifest_checksums('manifest', file_fixity_hash)
+      payload_pathname
+    end
+
+    # @param [Symbol] link_mode Specifies whether to :copy, :link, or :symlink the files to the payload directory
+    # @param [Pathname] source_pathname The source location of the file to be ingested
+    # @param [Pathname] target_pathname The location of the directory in which to place the file
+    # @return [Pathname] link or copy the specified file from source location to the target location
+    def copy_file(link_mode, source_pathname, target_pathname)
+      target_pathname.parent.mkpath
+      case link_mode
+        when :copy, nil
+          FileUtils.copy(source_pathname.to_s, target_pathname.to_s) # automatically dereferences symlinks
+        when :link
+          FileUtils.link(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
+        when :symlink
+          FileUtils.symlink(source_pathname.to_s, target_pathname.to_s) #, :force => true (false is default)
+        else
+          raise "Invalid link_mode: #{link_mode}, expected one of [:copy,:link,:symlink]"
+      end
+      target_pathname
+    end
+
+    # @param [Pathname,String] source_fullpath The location of the directory whose content will be tarred
+    # @param [Pathname,String] source_basepath The location of the directory to change to before doing the tar create
+    # @return [Tarfile] Create a tar archive of a directory into the payload directory,
+    #   generating checksums in parallel processes and recording those checksums in the payload manifests
+    def add_payload_tarfile(tarfile_id,source_fullpath, source_basepath)
+      tarfile = Tarfile.new
+      tarfile.source_basepath = Pathname(source_basepath)
+      tarfile.source_fullpath = Pathname(source_fullpath)
+      tarfile.tarfile_basepath = payload_pathname
+      tarfile.tarfile_fullpath = payload_pathname.join("#{tarfile_id}")
+      tarfile.create_tarfile
+      file_fixity_hash = Fixity.generate_checksums(tarfile.tarfile_basepath,[tarfile.tarfile_fullpath],bag_checksum_types)
+      write_manifest_checksums('manifest', file_fixity_hash)
+      tarfile
+    end
+
+    # @return [Pathname] Generate the bag-info.txt tag file to record the payload size
+    def write_bag_info_txt
+      payload_size = bag_payload_size
+      bag_info_txt = bag_pathname.join("bag-info.txt")
+      bag_info_txt.open('w') do |f|
+        f.puts "External-Identifier: #{bag_pathname.basename}"
+        f.puts "Payload-Oxum: #{payload_size[:bytes]}.#{payload_size[:files]}"
+        f.puts "Bag-Size: #{bag_size_human(payload_size[:bytes])}"
+      end
+      bag_info_txt
+    end
+
+    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
+    #   derived from the payload directory contents
+    def bag_payload_size
+      payload_pathname.find.select{|f| f.file?}.inject({bytes: 0, files: 0}) do |hash,file|
+        hash[:bytes] += file.size
+        hash[:files] += 1
+        hash
+      end
+    end
+
+    # @param [Integer] bytes The total number of bytes in the payload
+    # @return [String] Human-readable rendition of the total payload size
+    def bag_size_human(bytes)
+      count = 0
+      size = bytes
+      while ( size >= 1024 and count < 4 )
+        size /= 1024.0
+        count += 1
+      end
+      if (count == 0)
+        return sprintf("%d B", size)
+      else
+        return sprintf("%.2f %s", size, %w[B KB MB GB TB][count] )
+      end
+    end
+
+    # @return [Hash<String,String] A hash containing the properties documented in the bag-info.txt tagfile
+    def read_bag_info_txt
+      properties = Hash.new
+      bag_info = bag_pathname.join("bag-info.txt")
+      bag_info.readlines.each do |line|
+        line.chomp!.strip!
+        key,value = line.split(':',2)
+        properties[key.strip] = value.strip if value
+      end
+      properties
+    end
+
+    # @return [Hash<Symbol,Integer>] A hash contining the payload size in bytes, and the number of files,
+    #   derived from the Payload-Oxum property
+    def info_payload_size
+      info = read_bag_info_txt
+      size_array = info['Payload-Oxum'].split('.')
+      size_hash = {:bytes => size_array[0].to_i, :files => size_array[1].to_i}
+      size_hash
+    end
+
+    # @return [Boolean] Compare the actual measured payload size against the value recorded in bag-info.txt
+    def verify_payload_size
+      info_size = info_payload_size
+      bag_size = bag_payload_size
+      if info_size != bag_size
+        raise "Failed payload size verification! Expected: #{info_size}, Found: #{bag_size}"
+      end
+      true
+    end
+
+    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's root directory
+    def generate_tagfile_checksums
+      tagfiles = bag_pathname.children.reject{|file| file.basename.to_s.start_with?('tagmanifest')}
+      Fixity.generate_checksums(bag_pathname, tagfiles, bag_checksum_types )
+    end
+
+    # @return [Hash<String,FileFixity>] create hash containing ids and checksums for all files in the bag's payload
+    def generate_payload_checksums
+      Fixity.generate_checksums(payload_pathname, nil, bag_checksum_types)
+    end
+
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be updated
+    # @param [Hash<String,FileFixity>] file_fixity_hash A hash containing file ids and fixity data
+    # @param [String] open_mode The file open mode (default is 'a')
+    # @return [Hash<Symbol,Pathname] Update each of the manifests with data from the file_fixity_hash
+    def write_manifest_checksums(manifest_type, file_fixity_hash, open_mode='a')
+      manifests = Hash.new
+      self.bag_checksum_types.each do |checksum_type|
+        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
+        manifest_file = manifest_pathname.open(open_mode)
+        file_fixity_hash.values.each do |fixity|
+          checksum = fixity.get_checksum(checksum_type)
+          manifest_file.puts("#{checksum} #{fixity.file_id}") if checksum
+        end
+        manifest_file.close
+        manifests[checksum_type] = manifest_pathname
+      end
+      manifests
+    end
+
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
+    # @return [Hash<String,FileFixity>] A hash containing file ids and fixity data derived from the manifest files
+    def read_manifest_files(manifest_type)
+      file_fixity_hash = Hash.new
+      checksum_type_list = Array.new
+      Fixity.valid_checksum_ids.each do |checksum_type|
+        manifest_pathname = bag_pathname.join("#{manifest_type}-#{checksum_type}.txt")
+        if manifest_pathname.file?
+          checksum_type_list << checksum_type
+          manifest_pathname.readlines.each do |line|
+            line.chomp!.strip!
+            checksum,file_id = line.split(/[\s*]+/,2)
+            file_fixity = file_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+            file_fixity.set_checksum(checksum_type,checksum)
+            file_fixity_hash[file_id] = file_fixity
+          end
+        end
+      end
+      self.bag_checksum_types = self.bag_checksum_types | checksum_type_list
+      file_fixity_hash
+     end
+
+    # @return [Boolean] Compare fixity data from the tag manifest files against the values measured by digesting the files
+    def verify_tagfile_manifests
+      manifest_type = 'tagmanifest'
+      manifest_fixity_hash = read_manifest_files(manifest_type)
+      bag_fixity_hash = generate_tagfile_checksums
+      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+    end
+
+    # @return [Boolean] Compare fixity data from the payload manifest files against the values measured by digesting the files
+    def verify_payload_manifests
+      manifest_type = 'manifest'
+      manifest_fixity_hash = read_manifest_files(manifest_type)
+      bag_fixity_hash = generate_payload_checksums
+      verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+    end
+
+    # @param [String] manifest_type The type of manifest file ('manifest' or 'tagmanifest') to be read
+    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
+    # @return [Boolean] Compare fixity data from the manifest files against the values measured by digesting the files,
+    #   returning true if equal or false if not equal
+    def verify_manifests(manifest_type, manifest_fixity_hash, bag_fixity_hash)
+      diff = manifest_diff(manifest_fixity_hash, bag_fixity_hash)
+      if diff.size > 0
+        raise "Failed #{manifest_type} verification! Differences: \n#{diff.inspect}"
+      end
+      true
+    end
+
+    # @param [Hash<String,FileFixity>] manifest_fixity_hash A hash containing file ids and fixity data derived from the manifest files
+    # @param [Hash<String,FileFixity>] bag_fixity_hash A hash containing file ids and fixity data derived from the actual files
+    # @return [Hash] A report of the differences between the fixity data from the manifest files
+    #   against the values measured by digesting the files
+    def manifest_diff(manifest_fixity_hash, bag_fixity_hash)
+      diff = Hash.new
+      (manifest_fixity_hash.keys | bag_fixity_hash.keys).each do |file_id|
+        manifest_fixity = manifest_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+        bag_fixity = bag_fixity_hash[file_id] || FileFixity.new(file_id: file_id)
+        if manifest_fixity != bag_fixity
+          diff[file_id] = manifest_fixity.diff(bag_fixity,'manifest','bag')
+        end
+      end
+      diff
+    end
+
+    # @return [Boolean] Validate the bag containing the digital object
+    def verify_bag
+      verify_bag_structure
+      verify_tagfile_manifests
+      verify_payload_size
+      verify_payload_manifests
+      true
+    end
+
+    # @return [Boolean] Test the existence of expected files, return true if files exist, raise exception if not
+    def verify_bag_structure
+      required_files = ['data','bagit.txt','bag-info.txt','manifest-sha256.txt','tagmanifest-sha256.txt']
+      required_files.each{|filename| verify_pathname(bag_pathname.join(filename))}
+      optional_files = []
+      true
+    end
+
+    # @param [Pathname] pathname The file whose existence should be verified
+    # @return [Boolean] Test the existence of the specified path.  Return true if file exists, raise exception if not
+    def verify_pathname(pathname)
+      raise "#{pathname.basename} not found at #{pathname}" unless pathname.exist?
+      true
+    end
+
+
+  end
+
+
+end