moab-versioning 1.3.0

data/lib/moab/file_manifestation.rb

@@ -0,0 +1,85 @@
+require 'moab'
+
+module Moab
+
+  # A container for a file signature and all the physical file instances that have that signature
+  # This element has one child {FileSignature} element, and one or more {FileInstance} elements
+  # Regarding the class name, see
+  # * {http://en.wikipedia.org/wiki/Functional_Requirements_for_Bibliographic_Records}
+  # * {http://planets-project.eu/events/copenhagen-2009/pre-reading/docs/Modelling%20Organizational%20Preservation%20Goals_Angela%20Dappert.pdf}
+  #
+  # ====Data Model
+  # * {FileInventory} = container for recording information about a collection of related files
+  #   * {FileGroup} [1..*] = subset allow segregation of content and metadata files.
+  #     * <b>{FileManifestation} [1..*] = snapshot of a file's filesystem characteristics</b>
+  #       * {FileSignature} [1] = file fixity information
+  #       * {FileInstance} [1..*] = filepath and timestamp of any physical file having that signature
+  #
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class FileManifestation < Serializable
+    include HappyMapper
+
+    # The name of the XML element used to serialize this objects data
+    tag 'file'
+
+    # (see Serializable#initialize)
+    def initialize(opts={})
+      @instances = Array.new
+      super(opts)
+    end
+
+    # @attribute
+    # @return [FileSignature] The fixity data of the file instance
+    element :signature, FileSignature, :tag => 'fileSignature'
+
+    def signature
+      @signature.is_a?(Array) ? @signature[0] : @signature
+    end
+
+    def signature=(signature)
+      @signature = signature.is_a?(Array) ? signature[0] : signature
+    end
+
+    # @attribute
+    # @return [Array<FileInstance>] The location(s) of the file manifestation's file instances
+    has_many :instances, FileInstance, :tag => 'fileInstance'
+
+    # @api internal
+    # @return [Array<String>] Create an array from all the file paths of the child {FileInstance} objects
+    def paths
+      instances.collect { |i| i.path}
+    end
+
+    # @api internal
+    # @return [Integer] The total number of {FileInstance} objects in this manifestation.
+    #   (Number of files that share this manifestation's signature)
+    def file_count
+      instances.size
+    end
+
+    # @api internal
+    # @return [Integer] The total size (in bytes) of all files that share this manifestation's signature
+    def byte_count
+      file_count.to_i * signature.size.to_i
+    end
+
+    # @api internal
+    # @return [Integer] The total disk usage (in 1 kB blocks) of all files that share this manifestation's signature
+    #   (estimating du -k result)
+    def block_count
+      block_size=1024
+      instance_blocks = (signature.size.to_i + block_size - 1)/block_size
+      file_count * instance_blocks
+    end
+
+    # @api internal
+    # @param other [FileManifestation] The {FileManifestation} object to compare with self
+    # @return [Boolean] True if {FileManifestation} objects have same content
+    def ==(other)
+      (self.signature == other.signature) && (self.instances == other.instances)
+    end
+
+  end
+
+end

data/lib/moab/file_signature.rb

@@ -0,0 +1,200 @@
+require 'moab'
+
+module Moab
+
+  # The fixity properties of a file, used to determine file content equivalence regardless of filename.
+  # Placing this data in a class by itself facilitates using file size together with the MD5 and SHA1 checksums
+  # as a single key when doing comparisons against other file instances.  The Moab design assumes that this file signature
+  # is sufficiently unique to act as a comparator for determining file equality and eliminating file redundancy.
+  #
+  # The use of signatures for a compare-by-hash mechanism introduces a miniscule (but non-zero) risk
+  # that two non-identical files will have the same checksum.  While this risk is only about 1 in 1048
+  # when using the SHA1 checksum alone, it can be reduced even further (to about 1 in 1086)
+  # if we use the MD5 and SHA1 checksums together.  And we gain a bit more comfort by including a comparison of file sizes.
+  #
+  # Finally, the "collision" risk is reduced by isolation of each digital object's file pool within an object folder,
+  # instead of in a common storage area shared by the whole repository.
+  #
+  # ====Data Model
+  # * {FileInventory} = container for recording information about a collection of related files
+  #   * {FileGroup} [1..*] = subset allow segregation of content and metadata files
+  #     * {FileManifestation} [1..*] = snapshot of a file's filesystem characteristics
+  #       * <b>{FileSignature} [1] = file fixity information</b>
+  #       * {FileInstance} [1..*] = filepath and timestamp of any physical file having that signature
+  #
+  # * {SignatureCatalog} = lookup table containing a cumulative collection of all files ever ingested
+  #   * {SignatureCatalogEntry} [1..*] = an row in the lookup table containing storage information about a single file
+  #     * <b>{FileSignature} [1] = file fixity information</b>
+  #
+  # * {FileInventoryDifference} = compares two {FileInventory} instances based on file signatures and pathnames
+  #   * {FileGroupDifference} [1..*] = performs analysis and reports differences between two matching {FileGroup} objects
+  #     * {FileGroupDifferenceSubset} [1..5] = collects a set of file-level differences of a give change type
+  #       * {FileInstanceDifference} [1..*] = contains difference information at the file level
+  #         * <b>{FileSignature} [1..2] = contains the file signature(s) of two file instances being compared</b>
+  #
+  # @see http://searchstorage.techtarget.com/feature/The-skinny-on-data-deduplication
+  # @see http://www.ibm.com/developerworks/wikis/download/attachments/106987789/TSMDataDeduplication.pdf
+  # @see https://www.redlegg.com/pdf_file/3_1320410927_HowDataDedupeWorks_WP_100809.pdf
+  # @see http://www.library.yale.edu/iac/DPC/AN_DPC_FixityChecksFinal11.pdf
+  #
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class FileSignature < Serializable
+
+    include HappyMapper
+
+    # The name of the XML element used to serialize this objects data
+    tag 'fileSignature'
+
+    # (see Serializable#initialize)
+    def initialize(opts={})
+      super(opts)
+    end
+
+    # @attribute
+    # @return [Integer] The size of the file in bytes
+    attribute :size, Integer, :on_save => Proc.new { |n| n.to_s }
+
+    # @attribute
+    # @return [String] The MD5 checksum value of the file
+    attribute :md5, String, :on_save => Proc.new { |n| n.nil? ? "" : n.to_s }
+
+    # @attribute
+    # @return [String] The SHA1 checksum value of the file
+    attribute :sha1, String, :on_save => Proc.new { |n| n.nil? ? "" : n.to_s }
+
+    # @attribute
+    # @return [String] The SHA256 checksum value of the file
+    attribute :sha256, String, :on_save => Proc.new { |n| n.nil? ? "" : n.to_s }
+
+    # @param type [Symbol,String] The type of checksum
+    # @param value [String] The checksum value
+    # @return [void] Set the value of the specified checksum type
+    def set_checksum(type,value)
+      case type.to_s.downcase.to_sym
+        when :md5
+          @md5 = value
+        when :sha1
+          @sha1 = value
+        when :sha256
+          @sha256 = value
+        else
+          raise "Unknown checksum type '#{type.to_s}'"
+      end
+    end
+
+   # @return [Hash<Symbol,String>] A hash of the checksum data
+    def checksums
+      checksum_hash = OrderedHash.new
+      checksum_hash[:md5] = @md5
+      checksum_hash[:sha1] = @sha1
+      checksum_hash[:sha256] = @sha256
+      checksum_hash.delete_if { |key,value| value.nil? or value.empty?}
+      checksum_hash
+    end
+
+    # @return [Boolean] The signature contains all of the 3 desired checksums
+    def complete?
+      checksums.size == 3
+    end
+
+    # @api internal
+    # @return [Hash<Symbol,String>] A hash of fixity data from this signataure object
+    def fixity
+      fixity_hash = OrderedHash.new
+      fixity_hash[:size] = @size.to_s
+      fixity_hash.merge!(checksums)
+      fixity_hash
+    end
+
+    # @api internal
+    # @param other [FileSignature] The other file signature being compared to this signature
+    # @return [Boolean] Returns true if self and other have comparable fixity data.
+    def eql?(other)
+      return false if self.size.to_i != other.size.to_i
+      self_checksums = self.checksums
+      other_checksums = other.checksums
+      matching_keys = self_checksums.keys & other_checksums.keys
+      return false if matching_keys.size == 0
+      matching_keys.each do |key|
+        return false if self_checksums[key] != other_checksums[key]
+      end
+      true
+    end
+
+    # @api internal
+    # (see #eql?)
+    def ==(other)
+      eql?(other)
+    end
+
+    # @api internal
+    # @return [Fixnum] Compute a hash-code for the fixity value array.
+    #   Two file instances with the same content will have the same hash code (and will compare using eql?).
+    # @note The hash and eql? methods override the methods inherited from Object.
+    #   These methods ensure that instances of this class can be used as Hash keys.  See
+    #   * {http://www.paulbutcher.com/2007/10/navigating-the-equality-maze/}
+    #   * {http://techbot.me/2011/05/ruby-basics-equality-operators-ruby/}
+    #   Also overriden is {#==} so that equality tests in other contexts will also return the expected result.
+    def hash
+      @size.to_i
+    end
+
+    # @api internal
+    # @param pathname [Pathname] The location of the file to be digested
+    # @return [FileSignature] Generate a FileSignature instance containing size and checksums for a physical file
+    def signature_from_file(pathname)
+      @size = pathname.size
+      md5_digest = Digest::MD5.new
+      sha1_digest = Digest::SHA1.new
+      sha256_digest = Digest::SHA2.new(256)
+      pathname.open("r") do |stream|
+        while buffer = stream.read(8192)
+          md5_digest.update(buffer)
+          sha1_digest.update(buffer)
+          sha256_digest.update(buffer)
+        end
+      end
+      @md5 = md5_digest.hexdigest
+      @sha1 = sha1_digest.hexdigest
+      @sha256 = sha256_digest.hexdigest
+      self
+    end
+
+    # @api internal
+    # @param pathname [Pathname] The location of the file whose full signature will be returned
+    # @return [FileSignature] The full signature derived from the file, unless the fixity is inconsistent with current values
+    def normalized_signature(pathname)
+      sig_from_file = FileSignature.new.signature_from_file(pathname)
+      if self.eql?(sig_from_file)
+        # The full signature from file is consistent with current values
+        return sig_from_file
+      else
+        # One or more of the fixity values is inconsistent, so raise an exception
+        raise "Signature inconsistent between inventory and file for #{pathname}: #{self.diff(sig_from_file).inspect}"
+      end
+    end
+
+    # @return [Hash<Symbol,String>] Key is type (e.g. :sha1), value is checksum names (e.g. ['SHA-1', 'SHA1'])
+    def FileSignature.checksum_names_for_type
+      names_for_type = OrderedHash.new
+      names_for_type[:md5] = ['MD5']
+      names_for_type[:sha1] = ['SHA-1', 'SHA1']
+      names_for_type[:sha256] = ['SHA-256', 'SHA256']
+      names_for_type
+    end
+
+   # @return [Hash<String, Symbol>] Key is checksum name (e.g. MD5), value is checksum type (e.g. :md5)
+    def FileSignature.checksum_type_for_name
+      type_for_name = OrderedHash.new
+      self.checksum_names_for_type.each do |type, names|
+        names.each do |name|
+          type_for_name[name] = type
+        end
+      end
+      type_for_name
+    end
+
+  end
+
+end

data/lib/moab/signature_catalog.rb

@@ -0,0 +1,195 @@
+require 'moab'
+
+module Moab
+
+  # A digital object's Signature Catalog is derived from an filtered aggregation of the file inventories
+  # of a digital object's set of versions.  (see {#update})
+  # It has an entry for every file (identified by {FileSignature}) found in any of the versions,
+  # along with a record of the SDR storage location that was used to preserve a single file instance.
+  # Once this catalog has been populated, it has multiple uses:
+  # * The signature index is used to determine which files of a newly submitted object version
+  #   are new additions and which are duplicates of files previously ingested.  (See {#version_additions})
+  #   (When a new version contains a mixture of added files and files carried over from the previous version
+  #   we only need to store the files from the new version that have unique file signatures.)
+  # * Reconstruction of an object version (see {StorageObject#reconstruct_version}) requires a combination
+  #   of a full version's {FileInventory} and the SignatureCatalog.
+  # * The catalog can also be used for performing consistency checks between manifest files and storage
+  #
+  # ====Data Model
+  # * <b>{SignatureCatalog} = lookup table containing a cumulative collection of all files ever ingested</b>
+  #   * {SignatureCatalogEntry} [1..*] = an row in the lookup table containing storage information about a single file
+  #     * {FileSignature} [1] = file fixity information
+  #
+  # @example {include:file:spec/fixtures/derivatives/manifests/v3/signatureCatalog.xml}
+  # @see StorageObject
+  # @see Bagger
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class SignatureCatalog < Manifest
+    include HappyMapper
+
+    # The name of the XML element used to serialize this objects data
+    tag 'signatureCatalog'
+
+    # (see Serializable#initialize)
+    def initialize(opts={})
+      @entries = Array.new
+      @signature_hash = OrderedHash.new
+      super(opts)
+    end
+
+    # @attribute
+    # @return [String] The object ID (druid)
+    attribute :digital_object_id, String, :tag => 'objectId'
+
+    # @attribute
+    # @return [Integer] The ordinal version number
+    attribute :version_id, Integer, :tag => 'versionId', :key => true, :on_save => Proc.new {|n| n.to_s}
+
+    # @return [String] The unique identifier concatenating digital object id with version id
+    def composite_key
+      @digital_object_id + '-' + StorageObject.version_dirname(@version_id)
+    end
+
+    # @attribute
+    # @return [Time] The datetime at which the catalog was updated
+    attribute :catalog_datetime, Time, :tag => 'catalogDatetime', :on_save => Proc.new {|t| t.to_s}
+
+    def catalog_datetime=(datetime)
+      @catalog_datetime=Time.input(datetime)
+    end
+
+    def catalog_datetime
+      Time.output(@catalog_datetime)
+    end
+
+    # @attribute
+    # @return [Integer] The total number of data files (dynamically calculated)
+    attribute :file_count, Integer, :tag => 'fileCount', :on_save => Proc.new {|t| t.to_s}
+
+    def file_count
+      entries.size
+    end
+
+    # @attribute
+    # @return [Integer] The total size (in bytes) of all data files (dynamically calculated)
+    attribute :byte_count, Integer, :tag => 'byteCount', :on_save => Proc.new {|t| t.to_s}
+
+    def byte_count
+      entries.inject(0) { |sum, entry| sum + entry.signature.size.to_i }
+    end
+
+    # @attribute
+    # @return [Integer] The total disk usage (in 1 kB blocks) of all data files (estimating du -k result) (dynamically calculated)
+    attribute :block_count, Integer, :tag => 'blockCount', :on_save => Proc.new {|t| t.to_s}
+
+    def block_count
+      block_size=1024
+      entries.inject(0) { |sum, entry| sum + (entry.signature.size.to_i + block_size - 1)/block_size }
+    end
+
+    # @return [Array<String>] The data fields to include in summary reports
+    def summary_fields
+      %w{digital_object_id version_id catalog_datetime file_count byte_count block_count}
+    end
+
+    # @attribute
+    # @return [Array<SignatureCatalogEntry>] The set of data groups comprising the version
+    has_many :entries, SignatureCatalogEntry, :tag => 'entry'
+
+    def entries=(entry_array)
+      entry_array.each do |entry|
+        add_entry(entry)
+      end
+    end
+
+    # @return [OrderedHash] An index having {FileSignature} objects as keys and {SignatureCatalogEntry} objects as values
+    attr_accessor :signature_hash
+
+    # @api internal
+    # @param entry [SignatureCatalogEntry] The new catalog entry
+    # @return [void] Add a new entry to the catalog and to the {#signature_hash} index
+    def add_entry(entry)
+      @signature_hash[entry.signature] = entry
+      entries << entry
+    end
+
+    # @param [FileSignature] file_signature The signature of the file whose path is sought
+    # @return [String] The object-relative path of the file having the specified signature
+    def catalog_filepath(file_signature)
+      catalog_entry = @signature_hash[file_signature]
+      raise FileNotFoundException, "catalog entry not found for #{file_signature.fixity.inspect} in #{@digital_object_id} - #{@version_id}" if catalog_entry.nil?
+      catalog_entry.storage_path
+    end
+
+    # @param group [FileGroup] A group of the files from a file inventory
+    # @param group_pathname [Pathname] The location of the directory containing the group's files
+    # @return [void] Inspect and upgrade the group's signature data to include all desired checksums
+    def normalize_group_signatures(group, group_pathname=nil)
+      unless  group_pathname.nil?
+        group_pathname = Pathname(group_pathname)
+        raise "Could not locate #{group_pathname}" unless group_pathname.exist?
+      end
+      group.files.each do |file|
+        unless file.signature.complete?
+          if @signature_hash.has_key?(file.signature)
+            file.signature = @signature_hash.find {|k,v| k == file.signature}[0]
+          elsif group_pathname
+            file_pathname = group_pathname.join(file.instances[0].path)
+            file.signature = file.signature.normalized_signature(file_pathname)
+          end
+        end
+      end
+    end
+
+    # @api external
+    # @param version_inventory [FileInventory] The complete inventory of the files comprising a digital object version
+    # @param data_pathname [Pathname] The location of the object's data directory
+    # @return [void] Compares the {FileSignature} entries in the new versions {FileInventory} against the signatures
+    #   in this catalog and create new {SignatureCatalogEntry} addtions to the catalog
+    # @example {include:file:spec/features/catalog/catalog_update_spec.rb}
+    def update(version_inventory, data_pathname)
+      version_inventory.groups.each do |group|
+        group.files.each do |file|
+          unless @signature_hash.has_key?(file.signature)
+            entry = SignatureCatalogEntry.new
+            entry.version_id = version_inventory.version_id
+            entry.group_id = group.group_id
+            entry.path = file.instances[0].path
+            if file.signature.complete?
+              entry.signature = file.signature
+            else
+              file_pathname = data_pathname.join(group.group_id,entry.path)
+              entry.signature = file.signature.normalized_signature(file_pathname)
+            end
+            add_entry(entry)
+          end
+        end
+      end
+      @version_id = version_inventory.version_id
+      @catalog_datetime = Time.now
+    end
+
+    # @api external
+    # @param version_inventory (see #update)
+    # @return [FileInventory] Retrurns a filtered copy of the input inventory
+    #   containing only those files that were added in this version
+    # @example {include:file:spec/features/catalog/version_additions_spec.rb}
+    def version_additions(version_inventory)
+      version_additions = FileInventory.new(:type=>'additions')
+      version_additions.copy_ids(version_inventory)
+      version_inventory.groups.each do |group|
+        group_addtions = FileGroup.new(:group_id => group.group_id)
+        group.files.each do |file|
+          unless @signature_hash.has_key?(file.signature)
+            group_addtions.add_file_instance(file.signature,file.instances[0])
+          end
+        end
+        version_additions.groups << group_addtions if group_addtions.files.size > 0
+      end
+      version_additions
+    end
+
+  end
+
+end