moab-versioning 1.3.0

data/lib/serializer.rb

@@ -0,0 +1,36 @@
+# Serializer is a module containing classes whose methods faciliate serialization
+# of data fields to various formats.  To obtain those benefits, a dependent class
+# should inherit from {Serializable} or {Manifest}
+# depending on whether XML serialization is required.
+#
+# ====Data Model
+# * <b>{Serializable} = utility methods to faciliate serialization to Hash, JSON, or YAML</b>
+#   * {Manifest} = adds methods for marshalling/unmarshalling data to a persistent XML file format
+#
+# @see https://github.com/jnunemaker/happymapper
+# @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+#   All rights reserved.  See {file:LICENSE.rdoc} for details.
+module Serializer
+end
+
+require 'nokogiri'
+require 'happymapper'
+if RUBY_VERSION < '1.9'
+  require 'hashery/ordered_hash'
+  include Hashery
+else
+  require 'psych'
+  OrderedHash = Hash
+end
+require 'json'
+require 'json/pure'
+require 'pathname'
+require 'fileutils'
+require 'time'
+require 'digest/md5'
+require 'digest/sha1'
+
+require 'monkey_patches'
+require 'serializer/serializable'
+require 'serializer/manifest'
+

data/lib/serializer/manifest.rb

@@ -0,0 +1,76 @@
+module Serializer
+
+  # Subclass of {Serializable} that adds methods for marshalling/unmarshalling data
+  # to a persistent XML file format.
+  #
+  # ====Data Model
+  # * {Serializable} = utility methods to faciliate serialization to Hash, JSON, or YAML
+  #   * <b>{Manifest} = subclass adds methods for marshalling/unmarshalling data to XML file format</b>
+  #
+  # @see Serializable
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class Manifest < Serializable
+
+    include HappyMapper
+
+    # @api internal
+    # @param filename [String] Optional filename if one wishes to override the default filename
+    # @return [String] Returns the standard filename (derived from the class name) to be used for serializing an object
+    def self.xml_filename(filename=nil)
+      if filename
+        filename
+      else
+        cname = self.name.split(/::/).last
+        cname[0, 1].downcase + cname[1..-1] + '.xml'
+      end
+    end
+
+    # @api internal
+    # @param parent_dir [Pathname,String] The location of the directory in which the xml file is located
+    # @param filename [String] Optional filename if one wishes to override the default filename
+    # @return [Pathname] The location of the xml file
+    def self.xml_pathname(parent_dir, filename=nil)
+      Pathname.new(parent_dir).join(self.xml_filename(filename))
+    end
+
+    # @api external
+    # @param parent_dir [Pathname,String] The location of the directory in which the xml file is located
+    # @param filename [String] Optional filename if one wishes to override the default filename
+    # @return [Boolean] Returns true if the xml file exists
+    def self.xml_pathname_exist?(parent_dir, filename=nil)
+      self.xml_pathname(parent_dir, filename).exist?
+    end
+
+    # @api external
+    # @param parent_dir [Pathname,String] The location of the directory in which the xml file is located
+    # @param filename [String] Optional filename if one wishes to override the default filename
+    # @return [Serializable] Read the xml file and return the parsed XML
+    # @example {include:file:spec/features/serializer/read_xml_spec.rb}
+    def self.read_xml_file(parent_dir, filename=nil)
+      self.parse(self.xml_pathname(parent_dir, filename).read)
+    end
+
+    # @api external
+    # @param xml_object [Serializable]
+    # @param parent_dir [Pathname,String] The location of the directory in which the xml file is located
+    # @param filename [String] Optional filename if one wishes to override the default filename
+    # @return [void] Serializize the in-memory object to a xml file instance
+    def self.write_xml_file(xml_object, parent_dir, filename=nil)
+      parent_dir.mkpath
+      self.xml_pathname(parent_dir, filename).open('w') { |f| f << xml_object.to_xml }
+      nil
+    end
+
+    # @api external
+    # @param parent_dir [Pathname,String] The location of the directory in which the xml file is located
+    # @param filename [String] Optional filename if one wishes to override the default filename
+    # @return [void] Serializize the in-memory object to a xml file instance
+    # @example {include:file:spec/features/serializer/write_xml_spec.rb}
+    def write_xml_file(parent_dir, filename=nil)
+      self.class.write_xml_file(self, parent_dir, filename)
+    end
+
+  end
+
+end

data/lib/serializer/serializable.rb

@@ -0,0 +1,178 @@
+module Serializer
+
+  # Some utility methods to faciliate serialization of data fields to Hash, JSON, or YAML shared by all subclasses.
+  # This class assumes that HappyMapper is used for declaration of fields to be serialized.
+  #
+  # ====Data Model
+  # * <b>{Serializable} = utility methods to faciliate serialization to Hash, JSON, or YAML</b>
+  #   * {Manifest} = adds methods for marshalling/unmarshalling data to a persistent XML file format
+  #
+  # @see https://github.com/jnunemaker/happymapper
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class Serializable
+
+    include HappyMapper
+
+    # A flexible initializer based on the DataMapper "create factory" design pattern.
+    # @see http://datamapper.org/docs/create_and_destroy.html
+    # @see Serializable#initialize
+    # @param opts [Hash<Symbol,Object>] a hash containing any number of symbol => value pairs.
+    #   The symbols should correspond to attributes declared using HappyMapper syntax
+    def initialize(opts={})
+      opts.each do |key, value|
+        if variable_names.include?(key.to_s) || key == :test
+          instance_variable_set("@#{key}", value)
+        else
+          raise "#{key} is not a variable name in #{self.class.name}"
+        end
+      end
+    end
+
+    # @api internal
+    # @return [Array] A list of HappyMapper xml attribute, element and text nodes declared for the class
+    def variables
+      attributes = self.class.attributes
+      elements   = self.class.elements
+      attributes + elements
+      # text_node enhancement added by unhappymapper, which is not being used
+      # It enables elements having both attributes and a text value
+      #text_node = []
+      #if self.class.instance_variable_defined?("@text_node")
+      #  text_node << self.class.instance_variable_get("@text_node")
+      #end
+      #attributes + elements + text_node
+    end
+
+    # @api internal
+    # @return [Array] Extract the names of the variables
+    def variable_names
+      variables.collect { |variable| variable.name}
+    end
+
+    # @api internal
+    # @return [String] Determine which attribute was marked as an object instance key.
+    #   Keys are indicated by option :key=true when declaring the object's variables.
+    #   This follows the same convention as used by DataMapper
+    # @see http://datamapper.org/docs/properties.html
+    def key_name
+      if not defined?(@key_name)
+        @key_name = nil
+        self.class.attributes.each do |attribute|
+          if attribute.options[:key]
+            @key_name = attribute.name
+            break
+          end
+        end
+      end
+      @key_name
+    end
+
+    # @api internal
+    # @return [String] For the current object instance, return the string to use as a hash key
+    def key
+      return self.send(key_name) if key_name
+      nil
+    end
+
+    # @api internal
+    # @param array [Array] The array to be converted to a hash
+    # @return [OrderedHash] Generate a hash from an array of objects.
+    #   If the array member has a field tagged as a key, that field will be used as the hash.key.
+    #   Otherwise the index position of the array member will be used as the key
+    def array_to_hash(array,summary=false)
+      item_hash = OrderedHash.new
+      array.each_index do |index|
+        item = array[index]
+        ikey = (item.respond_to?(:key) && item.key) ?  item.key : index
+        item_hash[ikey] =  item.respond_to?(:to_hash) ? item.to_hash(summary) : item
+      end
+      item_hash
+    end
+
+    # @api internal
+    # @return [OrderedHash] Recursively generate an OrderedHash containing the object's properties
+    # @param summary [Boolean] Controls the depth and detail of recursion
+    def to_hash(summary=false)
+      oh = OrderedHash.new
+      vars = summary ? variables.select{|v| summary_fields.include?(v.name)} : variables
+      vars.each do |variable|
+        key = variable.options[:tag] || variable.name.to_s
+        value = self.send(variable.name)
+        case value
+          when Array
+            oh[key] = array_to_hash(value,summary)
+          when Serializable
+            oh[key] = value.to_hash
+          else
+            oh[key] = value
+        end
+      end
+      oh
+    end
+
+    # @return [OrderedHash] Calls to_hash(summary=true)
+    def summary
+      self.to_hash(summary=true)
+    end
+
+    # @api internal
+    # @param other [Serializable] The other object being compared
+    # @return [OrderedHash] Generate a hash containing the differences between two objects of the same type
+    def diff(other)
+      raise "Cannot compare different classes" if self.class != other.class
+      left = other.to_hash
+      right = self.to_hash
+      if self.key.nil? or other.key.nil?
+        ltag = :old
+        rtag = :new
+      else
+        ltag = other.key
+        rtag = self.key
+      end
+      Serializable.deep_diff(ltag, left, rtag, right)
+    end
+
+    # @api internal
+    # @param hashes [Array<Hash>] The hashes to be compared, with optional name tags
+    # @return [OrderedHash] Generate a hash containing the differences between two hashes
+    #   (recursively descend parallel trees of hashes)
+    # @see https://gist.github.com/146844
+    def Serializable.deep_diff(*hashes)
+      diff = OrderedHash.new
+      case hashes.length
+        when 4
+          ltag, left, rtag, right = hashes
+        when 2
+          ltag, left, rtag, right = :left, hashes[0], :right, hashes[1]
+        else
+          raise "wrong number of arguments (expected 2 or 4)"
+      end
+      (left.keys | right.keys).each do |k|
+        if left[k] != right[k]
+          if left[k].is_a?(Hash) && right[k].is_a?(Hash)
+            diff[k] = deep_diff(ltag, left[k], rtag, right[k])
+          else
+            diff[k] = OrderedHash.[](ltag, left[k], rtag, right[k])
+          end
+        end
+      end
+      diff
+    end
+
+    # @api internal
+    # @return [String] Generate JSON output from a hash of the object's variables
+    def to_json(summary=false)
+      hash=self.to_hash(summary)
+      JSON.pretty_generate(hash)
+    end
+
+    # @api internal
+    # @return [String] Generate YAML output from a hash of the object's variables
+    def to_yaml(summary=false)
+      self.to_hash(summary).to_yaml
+    end
+
+  end
+
+end

data/lib/stanford/active_fedora_object.rb

@@ -0,0 +1,34 @@
+require 'moab_stanford'
+
+module Stanford
+
+  # Utility Class for extracting content or other information from a Fedora Instance
+  #
+  # ====Data Model
+  # * {DorMetadata} = utility methods for interfacing with Stanford metadata files (esp contentMetadata)
+  #   * {ContentInventory} [1..1] = utilities for transforming contentMetadata to versionInventory and doing comparsions
+  #   * <b>{ActiveFedoraObject} [1..*] = utility for extracting content or other information from a Fedora Instance</b>
+  #
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class ActiveFedoraObject
+
+    # @param fedora_object [Object]  The Active Fedora representation of the Fedora Object
+    # @return [Stanford::ActiveFedoraObject] Create a u
+    def initialize(fedora_object)
+      @fedora_object = fedora_object
+    end
+
+    # @return [Object]  The Active Fedora representation of the Fedora Object
+    attr_accessor :fedora_object
+
+    # @api external
+    # @param ds_id [String] The datastream identifier
+    # @return [String] The content of the specified datastream
+    def get_datastream_content(ds_id)
+      @fedora_object.datastreams[ds_id].content
+    end
+
+  end
+
+end

data/lib/stanford/content_inventory.rb

@@ -0,0 +1,236 @@
+require 'moab_stanford'
+
+module Stanford
+
+  # Stanford-specific utility methods for transforming contentMetadata to versionInventory and doing
+  #
+  # ====Data Model
+  # * {DorMetadata} = utility methods for interfacing with Stanford metadata files (esp contentMetadata)
+  #   * <b>{ContentInventory} [1..1] = utilities for transforming contentMetadata to versionInventory and doing comparsions</b>
+  #   * {ActiveFedoraObject} [1..*] = utility for extracting content or other information from a Fedora Instance
+  #
+  # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
+  #   All rights reserved.  See {file:LICENSE.rdoc} for details.
+  class ContentInventory
+
+    # @param content_metadata [String] The content metadata to be transformed into a versionInventory
+    # @param object_id [String] The identifier of the digital object
+    # @param subset [String] Speciifes which subset of files to list (all|preserve|publish|shelve)
+    # @param version_id [Integer] The ID of the version whosen content metadata is to be transformed
+    # @return [FileInventory] The versionInventory equivalent of the contentMetadata
+    #   if the supplied content_metadata is blank or empty, then a skeletal FileInventory will be returned
+    def inventory_from_cm(content_metadata, object_id, subset, version_id=nil)
+      # The contentMetadata datastream is not required for ingest, since some object types, such as collection or APO do not require one.
+      # Many of these objects have contentMetadata with no child elements, such as this:
+      #    <contentMetadata objectId="bd608mj3166" type="file"/>
+      # but there are also objects that have no datasteam of this name at all
+      cm_inventory = FileInventory.new(:type=>"version",:digital_object_id=>object_id, :version_id=>version_id)
+      content_group = group_from_cm(content_metadata, subset)
+      cm_inventory.groups << content_group
+      cm_inventory
+    end
+
+    # @api external
+    # @param content_metadata [String] The contentMetadata as a string
+    # @param subset [String] Speciifes which subset of files to list (all|preserve|publish|shelve)
+    # @return [FileGroup] The {FileGroup} object generated from a contentMetadata instance
+    # @example {include:file:spec/features/stanford/content_metadata_read_spec.rb}
+    def group_from_cm(content_metadata, subset)
+      ng_doc = Nokogiri::XML(content_metadata)
+      validate_content_metadata(ng_doc)
+      nodeset = case subset.to_s.downcase
+        when 'preserve'
+          ng_doc.xpath("//file[@preserve='yes']")
+        when 'publish'
+          ng_doc.xpath("//file[@publish='yes']")
+        when 'shelve'
+          ng_doc.xpath("//file[@shelve='yes']")
+        when 'all'
+          ng_doc.xpath("//file")
+        else
+            raise "Unknown disposition subset (#{subset})"
+        end
+      content_group = FileGroup.new(:group_id=>'content', :data_source => "contentMetadata-#{subset}")
+      nodeset.each do |file_node|
+        signature = generate_signature(file_node)
+        instance = generate_instance(file_node)
+        content_group.add_file_instance(signature, instance)
+      end
+      content_group
+    end
+
+    # @api internal
+    # @param node [Nokogiri::XML::Node] The XML node containing file information
+    # @return [FileSignature] The {FileSignature} object generated from the XML data
+    def generate_signature(node)
+      signature = FileSignature.new()
+      signature.size = node.attributes['size'].content
+      checksum_nodes = node.xpath('checksum')
+      checksum_nodes.each do |checksum_node|
+        case checksum_node.attributes['type'].content.upcase
+          when 'MD5'
+            signature.md5 = checksum_node.text
+          when 'SHA1', 'SHA-1'
+            signature.sha1 = checksum_node.text
+          when 'SHA256', 'SHA-256'
+            signature.sha256 = checksum_node.text
+        end
+      end
+      signature
+    end
+
+    # @api internal
+    # @param node (see #generate_signature)
+    # @return [FileInstance] The {FileInstance} object generated from the XML data
+    def generate_instance(node)
+      instance = FileInstance.new()
+      instance.path = node.attributes['id'].content
+      instance.datetime = node.attributes['datetime'].content rescue nil
+      instance
+    end
+
+    # @api external
+    # @param file_group [FileGroup] The {FileGroup} object used as the data source
+    # @return [String] The contentMetadata instance generated from the FileGroup
+    # @example {include:file:spec/features/stanford/content_metadata_write_spec.rb}
+    def generate_content_metadata(file_group, object_id, version_id)
+      cm = Nokogiri::XML::Builder.new do |xml|
+        xml.contentMetadata(:type=>"sample", :objectId=>object_id) {
+          xml.resource(:type=>"version", :sequence=>"1", :id=>"version-#{version_id.to_s}") {
+            file_group.files.each do |file_manifestation|
+              signature = file_manifestation.signature
+              file_manifestation.instances.each do |instance|
+                xml.file(
+                    :id=>instance.path,
+                    :size=>signature.size,
+                    :datetime=>instance.datetime,
+                    :shelve=>'yes',
+                    :publish=>'yes',
+                    :preserve=>'yes') {
+                  fixity = signature.fixity
+                  xml.checksum(:type=>"MD5") {xml.text signature.md5 } if fixity[:md5]
+                  xml.checksum(:type=>"SHA-1") {xml.text signature.sha1} if fixity[:sha1]
+                  xml.checksum(:type=>"SHA-256") {xml.text signature.sha256} if fixity[:sha256]
+                }
+              end
+            end
+          }
+        }
+      end
+      cm.to_xml
+    end
+
+    # @param content_metadata [String,Nokogiri::XML::Document] The contentMetadata as a string or XML doc
+    # @return [Boolean] True if contentMetadata has essetial file attributes, else raise exception
+    def validate_content_metadata(content_metadata)
+      result = validate_content_metadata_details(content_metadata)
+      raise Moab::InvalidMetadataException, result[0]+" ..." if result.size > 0
+      true
+    end
+
+    # @param content_metadata [String, Nokogiri::XML::Document] The contentMetadata as a string or XML doc
+    # @return [Array<String>] List of problems found
+    def validate_content_metadata_details(content_metadata)
+      result = []
+      content_metadata_doc =
+        case content_metadata.class.name
+          when "String"
+            Nokogiri::XML(content_metadata)
+          when "Pathname"
+            Nokogiri::XML(content_metadata.read)
+          when "Nokogiri::XML::Document"
+            content_metadata
+          else
+            raise Moab::InvalidMetadataException, "Content Metadata is in unrecognized format"
+          end
+      nodeset = content_metadata_doc.xpath("//file")
+      nodeset.each do |file_node|
+        missing = ['id', 'size','md5','sha1']
+        missing.delete('id') if file_node.has_attribute?('id')
+        missing.delete('size') if file_node.has_attribute?('size')
+        checksum_nodes = file_node.xpath('checksum')
+        checksum_nodes.each do |checksum_node|
+          case checksum_node.attributes['type'].content.upcase
+            when 'MD5'
+              missing.delete('md5')
+            when 'SHA1', 'SHA-1'
+              missing.delete('sha1')
+          end
+        end
+        if missing.include?('id')
+          result << "File node #{nodeset.index(file_node)} is missing #{missing.join(',')}"
+        elsif missing.size > 0
+          id = file_node['id']
+          result << "File node having id='#{id}' is missing #{missing.join(',')}"
+        end
+      end
+      result
+    end
+
+    # @param content_metadata [String] The contentMetadata as a string
+    # @param content_group [FileGroup] The {FileGroup} object used as the fixity data source
+    # @return [String] Returns a remediated copy of the contentMetadata with fixity data filled in
+    # @see http://blog.slashpoundbang.com/post/1454850669/how-to-pretty-print-xml-with-nokogiri
+    def remediate_content_metadata(content_metadata, content_group)
+      return nil if content_metadata.nil?
+      return content_metadata if content_group.nil? or content_group.files.size < 1
+      signature_for_path = content_group.path_hash
+      @type_for_name = FileSignature.checksum_type_for_name
+      @names_for_type = FileSignature.checksum_names_for_type
+      ng_doc = Nokogiri::XML(content_metadata) { |x| x.noblanks }
+      nodeset = ng_doc.xpath("//file")
+      nodeset.each do |file_node|
+        filepath = file_node['id']
+        signature = signature_for_path[filepath]
+        remediate_file_size(file_node, signature)
+        remediate_checksum_nodes(file_node, signature)
+      end
+      ng_doc.to_xml(:indent => 2)
+    end
+
+    # @param [Nokogiri::XML::Element] file_node the File stanza being remediated
+    # @param [FileSignature] signature the fixity data for the file from the FileGroup
+    # @return [void] update the file size attribute if missing, raise exception if inconsistent
+    def remediate_file_size(file_node, signature)
+      file_size = file_node['size']
+      if file_size.nil? or file_size.empty?
+        file_node['size'] = signature.size.to_s
+      elsif file_size != signature.size.to_s
+        raise "Inconsistent size for #{file_node['id']}: #{file_size} != #{signature.size.to_s}"
+      end
+    end
+
+    # @param [Nokogiri::XML::Element] file_node the File stanza being remediated
+    # @param [FileSignature] signature the fixity data for the file from the FileGroup
+    # @return [void] update the file's checksum elements if data missing, raise exception if inconsistent
+    def remediate_checksum_nodes(file_node, signature)
+      # collect <checksum> elements for checksum types that are already present
+      checksum_nodes = OrderedHash.new
+      file_node.xpath('checksum').each do |checksum_node|
+        type = @type_for_name[checksum_node['type']]
+        checksum_nodes[type] = checksum_node
+      end
+      # add new <checksum> elements for the other checksum types that were missing
+      @names_for_type.each do |type, names|
+        unless checksum_nodes.has_key?(type)
+          checksum_node = Nokogiri::XML::Element.new('checksum',file_node.document)
+          checksum_node['type'] = names[0]
+          file_node << checksum_node
+          checksum_nodes[type] = checksum_node
+        end
+      end
+      # make sure the <checksum> element has a content value
+      checksum_nodes.each do |type,checksum_node|
+        cm_checksum = checksum_node.content
+        sig_checksum = signature.checksums[type]
+        if cm_checksum.nil? or cm_checksum.empty?
+          checksum_node.content = sig_checksum
+        elsif cm_checksum != sig_checksum
+          raise "Inconsistent #{type.to_s} for #{file_node['id']}: #{cm_checksum} != #{sig_checksum}"
+        end
+      end
+    end
+
+  end
+
+end