moab-versioning 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b369a2ecc1e65bdcf4a40dfde46176838912ce2f
-  data.tar.gz: 8e9b829361e700e12220e203204433a1c56e7b55
+  metadata.gz: 5c13dd09bef589a3c4a3314737c2d3b6dfb4589f
+  data.tar.gz: 629913d22d1ba0b40f62671aabc7cca01767c276
 SHA512:
-  metadata.gz: b076d84c4bc16759c646234d37625068922cdbbbbe68add37bb0922c21d9963441ce7d280bab4f1d5511a078364004b0c59c9d7b4a376f60fe7f37ed8844f674
-  data.tar.gz: 81a48dbd61d60ca5469cba9d330970cb4b929a003bcea55177294f8426eca1d84e223988fdf70ac95ee818e38787992ae2355d3f3fe7d8c1718d8807a78fa803
+  metadata.gz: fe0076d5a2e868613c4e5f081abdee5fd93e76effccfdf4219d7849d8f9944cc11ff728b407ffac22a0d02b9a71c30a75e2077c3f11698bb815e336b951f33b4
+  data.tar.gz: 6c3e42f3cf596cd16e2d62858da725e93fcf37ab835bfcc4f9a3951eb4bbd569845210f9dfa40f39048bab235162fe2d169494a5469a31229924ad29dbea01db

data/lib/moab.rb

@@ -29,6 +29,7 @@
 # @note Copyright (c) 2012 by The Board of Trustees of the Leland Stanford Junior University.
 #   All rights reserved.  See {file:LICENSE.rdoc} for details.
 module Moab
+  DEFAULT_CHECKSUM_TYPES = [:md5, :sha1, :sha256].freeze
 end
 
 require 'serializer'
@@ -57,3 +58,4 @@ require 'moab/storage_services'
 require 'moab/exceptions'
 require 'moab/verification_result'
 require 'moab/storage_object_validator'
+require 'moab/deposit_bag_validator'

data/lib/moab/bagger.rb

@@ -1,6 +1,3 @@
-require 'moab'
-require 'systemu'
-
 module Moab
 
   # A class used to create a BagIt package from a version inventory and a set of source files.
@@ -60,8 +57,8 @@ module Moab
     # @api internal
     # @return [void] Generate the bagit.txt tag file
     def create_bagit_txt()
-      @bag_pathname.mkpath
-      @bag_pathname.join("bagit.txt").open('w') do |f|
+      bag_pathname.mkpath
+      bag_pathname.join("bagit.txt").open('w') do |f|
         f.puts "Tag-File-Character-Encoding: UTF-8"
         f.puts "BagIt-Version: 0.97"
       end
@@ -70,12 +67,12 @@ module Moab
     # @return [NilClass] Delete the bagit files
     def delete_bag()
       # make sure this looks like a bag before deleting
-      if @bag_pathname.join('bagit.txt').exist?
-        if @bag_pathname.join('data').exist?
-          @bag_pathname.rmtree
+      if bag_pathname.join('bagit.txt').exist?
+        if bag_pathname.join('data').exist?
+          bag_pathname.rmtree
         else
-          @bag_pathname.children.each {|file| file.delete}
-          @bag_pathname.rmdir
+          bag_pathname.children.each {|file| file.delete}
+          bag_pathname.rmdir
         end
       end
       nil
@@ -83,8 +80,8 @@ module Moab
 
     # @param tar_pathname [Pathname] The location of the tar file (default is based on bag location)
     def delete_tarfile()
-      bag_name = @bag_pathname.basename
-      bag_parent = @bag_pathname.parent
+      bag_name = bag_pathname.basename
+      bag_parent = bag_pathname.parent
       tar_pathname = bag_parent.join("#{bag_name}.tar")
       tar_pathname.delete if tar_pathname.exist?
     end
@@ -109,17 +106,17 @@ module Moab
     # @return [FileInventory] Create, write, and return the inventory of the files that will become the payload
     def create_bag_inventory(package_mode)
       @package_mode = package_mode
-      @bag_pathname.mkpath
+      bag_pathname.mkpath
       case package_mode
         when :depositor
-          @version_inventory.write_xml_file(@bag_pathname, 'version')
-          @bag_inventory = @signature_catalog.version_additions(@version_inventory)
-          @bag_inventory.write_xml_file(@bag_pathname, 'additions')
+          version_inventory.write_xml_file(bag_pathname, 'version')
+          @bag_inventory = signature_catalog.version_additions(version_inventory)
+          bag_inventory.write_xml_file(bag_pathname, 'additions')
         when :reconstructor
-          @bag_inventory = @version_inventory
-          @bag_inventory.write_xml_file(@bag_pathname, 'version')
+          @bag_inventory = version_inventory
+          bag_inventory.write_xml_file(bag_pathname, 'version')
       end
-      @bag_inventory
+      bag_inventory
     end
 
     # @api internal
@@ -128,9 +125,9 @@ module Moab
     # This method uses Unix hard links in order to greatly speed up the process.
     # Hard links, however, require that the target bag must be created within the same filesystem as the source files
     def fill_payload(source_base_pathname)
-      @bag_inventory.groups.each do |group|
+      bag_inventory.groups.each do |group|
         group_id = group.group_id
-        case @package_mode
+        case package_mode
           when :depositor
             deposit_group(group_id, source_base_pathname.join(group_id))
           when :reconstructor
@@ -144,9 +141,9 @@ module Moab
     # @return [Boolean] Copy all the files listed in the group inventory to the bag.
     #    Return true if successful or nil if the group was not found in the inventory
     def deposit_group(group_id, source_dir)
-      group = @bag_inventory.group(group_id)
+      group = bag_inventory.group(group_id)
       return nil? if group.nil? or group.files.empty?
-      target_dir = @bag_pathname.join('data',group_id)
+      target_dir = bag_pathname.join('data',group_id)
       group.path_list.each do |relative_path|
         source = source_dir.join(relative_path)
         target = target_dir.join(relative_path)
@@ -161,11 +158,11 @@ module Moab
     # @return [Boolean] Copy all the files listed in the group inventory to the bag.
     #    Return true if successful or nil if the group was not found in the inventory
     def reconstuct_group(group_id, storage_object_dir)
-      group = @bag_inventory.group(group_id)
+      group = bag_inventory.group(group_id)
       return nil? if group.nil? or group.files.empty?
-      target_dir = @bag_pathname.join('data',group_id)
+      target_dir = bag_pathname.join('data',group_id)
       group.files.each do |file|
-        catalog_entry = @signature_catalog.signature_hash[file.signature]
+        catalog_entry = signature_catalog.signature_hash[file.signature]
         source = storage_object_dir.join(catalog_entry.storage_path)
         file.instances.each do |instance|
           target = target_dir.join(instance.path)
@@ -190,24 +187,23 @@ module Moab
     def create_payload_manifests
       manifest_pathname = Hash.new
       manifest_file = Hash.new
-      manifest_types =  [:md5, :sha1, :sha256]
-      manifest_types.each do |type|
-        manifest_pathname[type] = @bag_pathname.join("manifest-#{type}.txt")
+      DEFAULT_CHECKSUM_TYPES.each do |type|
+        manifest_pathname[type] = bag_pathname.join("manifest-#{type}.txt")
         manifest_file[type] = manifest_pathname[type].open('w')
       end
-      @bag_inventory.groups.each do |group|
+      bag_inventory.groups.each do |group|
         group.files.each do |file|
           fixity = file.signature.fixity
           file.instances.each do |instance|
             data_path = File.join('data', group.group_id, instance.path)
-            manifest_types.each do |type|
+            DEFAULT_CHECKSUM_TYPES.each do |type|
               manifest_file[type].puts("#{fixity[type]} #{data_path}") if fixity[type]
             end
           end
         end
       end
     ensure
-      manifest_types.each do |type|
+      DEFAULT_CHECKSUM_TYPES.each do |type|
         if manifest_file[type]
           manifest_file[type].close
           manifest_pathname[type].delete if
@@ -219,10 +215,10 @@ module Moab
     # @api internal
     # @return [void] Generate the bag-info.txt tag file
     def create_bag_info_txt
-      @bag_pathname.join("bag-info.txt").open('w') do |f|
-        f.puts "External-Identifier: #{@bag_inventory.package_id}"
-        f.puts "Payload-Oxum: #{@bag_inventory.byte_count}.#{@bag_inventory.file_count}"
-        f.puts "Bag-Size: #{@bag_inventory.human_size}"
+      bag_pathname.join("bag-info.txt").open('w') do |f|
+        f.puts "External-Identifier: #{bag_inventory.package_id}"
+        f.puts "Payload-Oxum: #{bag_inventory.byte_count}.#{bag_inventory.file_count}"
+        f.puts "Bag-Size: #{bag_inventory.human_size}"
       end
     end
 
@@ -231,22 +227,21 @@ module Moab
     def create_tagfile_manifests()
       manifest_pathname = Hash.new
       manifest_file = Hash.new
-      manifest_types =  [:md5, :sha1, :sha256]
-      manifest_types.each do |type|
-        manifest_pathname[type] = @bag_pathname.join("tagmanifest-#{type}.txt")
+      DEFAULT_CHECKSUM_TYPES.each do |type|
+        manifest_pathname[type] = bag_pathname.join("tagmanifest-#{type}.txt")
         manifest_file[type] = manifest_pathname[type].open('w')
       end
-      @bag_pathname.children.each do |file|
+      bag_pathname.children.each do |file|
         unless file.directory? || file.basename.to_s[0, 11] == 'tagmanifest'
           signature = FileSignature.new.signature_from_file(file)
           fixity = signature.fixity
-          manifest_types.each do |type|
+          DEFAULT_CHECKSUM_TYPES.each do |type|
             manifest_file[type].puts("#{fixity[type]} #{file.basename}") if fixity[type]
           end
         end
       end
     ensure
-      manifest_types.each do |type|
+      DEFAULT_CHECKSUM_TYPES.each do |type|
         if manifest_file[type]
           manifest_file[type].close
           manifest_pathname[type].delete if
@@ -257,8 +252,8 @@ module Moab
 
     # @return [Boolean] Create a tar file containing the bag
     def create_tarfile(tar_pathname=nil)
-      bag_name = @bag_pathname.basename
-      bag_parent = @bag_pathname.parent
+      bag_name = bag_pathname.basename
+      bag_parent = bag_pathname.parent
       tar_pathname ||= bag_parent.join("#{bag_name}.tar")
       tar_cmd="cd '#{bag_parent}'; tar --dereference --force-local -cf  '#{tar_pathname}' '#{bag_name}'"
       begin
@@ -268,27 +263,24 @@ module Moab
       end
       raise "Unable to create tarfile #{tar_pathname}" unless tar_pathname.exist?
       return true
-
     end
 
-    # Executes a system command in a subprocess.
-    # The method will return stdout from the command if execution was successful.
-    # The method will raise an exception if if execution fails.
-    # The exception's message will contain the explaination of the failure.
-    # @param [String] command the command to be executed
-    # @return [String] stdout from the command if execution was successful
+    # Executes a system command in a subprocess
+    # if command isn't successful, grabs stdout and stderr and puts them in ruby exception message
+    # @return stdout if execution was successful
     def shell_execute(command)
-      status, stdout, stderr = systemu(command)
-      if (status.exitstatus != 0)
-        raise stderr
+      require 'open3'
+      stdout, stderr, status = Open3.capture3(command.chomp)
+      if status.success? && status.exitstatus.zero?
+        stdout
+      else
+        msg = "Shell command failed: [#{command}] caused by <STDERR = #{stderr}>"
+        msg << " STDOUT = #{stdout}" if stdout && stdout.length.positive?
+        raise(StandardError, msg)
       end
-      return stdout
-    rescue
-      msg = "Command failed to execute: [#{command}] caused by <STDERR = #{stderr.split($/).join('; ')}>"
-      msg << " STDOUT = #{stdout.split($/).join('; ')}" if (stdout && (stdout.length > 0))
-      raise msg
+    rescue SystemCallError => e
+      msg = "Shell command failed: [#{command}] caused by #{e.inspect}"
+      raise(StandardError, msg)
     end
-
   end
-
 end

data/lib/moab/deposit_bag_validator.rb

@@ -0,0 +1,325 @@
+module Moab
+
+  # Given a deposit bag, ensures the contents valid for becoming a StorageObjectVersion
+  # this is a Shameless Green implementation, combining code from:
+  #  - sdr-preservation-core/lib/sdr_ingest/validate_bag  <-- old preservation robots
+  #  - archive-utils/lib/bagit_bag  <-- gem only used by sdr-preservation-robots
+  #  - archive-utils/lib/file_fixity
+  #  - archive-utils/lib/fixity
+  # this code adds duplication to this gem (see github issue #119);
+  # for example, computing checksums is done
+  #  - deposit_bag_validator
+  #  - file_signature
+  class DepositBagValidator
+
+    BAG_DIR_NOT_FOUND = :bag_dir_not_found
+    CHECKSUM_MISMATCH = :checksum_mismatch
+    CHECKSUM_TYPE_UNRECOGNIZED = :checksum_type_unrecognized
+    INVALID_VERSION_XXX_XML = :invalid_versionXxx_xml
+    PAYLOAD_SIZE_MISMATCH = :payload_size_mismatch
+    REQUIRED_FILE_NOT_FOUND = :required_file_not_found
+    VERSION_MISMATCH_TO_MOAB = :version_mismatch_to_moab
+    VERSION_MISSING_FROM_FILE = :version_missing_from_file
+
+    ERROR_CODE_TO_MESSAGES = {
+      BAG_DIR_NOT_FOUND => "Deposit bag directory %{bag_dir} does not exist",
+      CHECKSUM_MISMATCH => "Failed %{manifest_type} verification. Differences: \n%{diffs}",
+      CHECKSUM_TYPE_UNRECOGNIZED => "Checksum type unrecognized: %{checksum_type}; file: %{filename}",
+      INVALID_VERSION_XXX_XML => "Unable to parse %{file_pathname}: %{err_info}",
+      PAYLOAD_SIZE_MISMATCH => "Failed payload size verification. Expected: %{bag_info_sizes}; found: %{generated_sizes}",
+      REQUIRED_FILE_NOT_FOUND => "Deposit bag required file %{file_pathname} not found",
+      VERSION_MISMATCH_TO_MOAB => "Version mismatch in %{file_pathname}: Moab expected %{new_version}; found %{file_version}",
+      VERSION_MISSING_FROM_FILE => "Version xml file %{version_file} missing data at %{xpath} containing version id"
+    }.freeze
+
+    REQUIRED_MANIFEST_CHECKSUM_TYPE = 'sha256'.freeze
+    RECOGNIZED_CHECKSUM_ALGORITHMS = [:md5, :sha1, :sha256, :sha384, :sha512].freeze
+
+    TAGMANIFEST = 'tagmanifest'.freeze
+    MANIFEST = 'manifest'.freeze
+    DATA_DIR_BASENAME = 'data'.freeze
+    BAG_INFO_TXT_BASENAME = 'bag-info.txt'.freeze
+    VERSION_ADDITIONS_BASENAME = 'versionAdditions.xml'.freeze
+    VERSION_INVENTORY_BASENAME = 'versionInventory.xml'.freeze
+    VERSION_METADATA_PATH = "#{DATA_DIR_BASENAME}/metadata/versionMetadata.xml".freeze
+
+    REQUIRED_BAG_FILES = [
+      DATA_DIR_BASENAME,
+      'bagit.txt'.freeze,
+      BAG_INFO_TXT_BASENAME,
+      "#{MANIFEST}-#{REQUIRED_MANIFEST_CHECKSUM_TYPE}.txt".freeze,
+      "#{TAGMANIFEST}-#{REQUIRED_MANIFEST_CHECKSUM_TYPE}.txt".freeze,
+      VERSION_ADDITIONS_BASENAME,
+      VERSION_INVENTORY_BASENAME,
+      VERSION_METADATA_PATH
+    ].freeze
+
+    attr_reader :deposit_bag_pathname, :expected_new_version, :result_array
+
+    def initialize(storage_object)
+      @deposit_bag_pathname = storage_object.deposit_bag_pathname
+      @expected_new_version = storage_object.current_version_id + 1
+      @result_array = []
+    end
+
+    # returns Array of tiny error hashes, allowing multiple occurrences of a single error code
+    def validation_errors
+      return [single_error_hash(BAG_DIR_NOT_FOUND, bag_dir: deposit_bag_pathname)] unless deposit_bag_pathname.exist?
+      return result_array unless required_bag_files_exist?
+      verify_version
+      verify_tagmanifests
+      verify_payload_size
+      verify_payload_manifests
+      result_array # attr that accumulates any errors encountered along the way
+    end
+
+    private
+
+    def bag_dir_exists?
+      deposit_bag_pathname.exist?
+    end
+
+    # assumes this is called when result_array is empty, as subsequent checks will use these required files
+    def required_bag_files_exist?
+      REQUIRED_BAG_FILES.each do |filename|
+        pathname = deposit_bag_pathname.join(filename)
+        result_array << single_error_hash(REQUIRED_FILE_NOT_FOUND, file_pathname: pathname) unless pathname.exist?
+      end
+      result_array.empty? ? true : false
+    end
+
+    def verify_version
+      version_md_pathname = deposit_bag_pathname.join(VERSION_METADATA_PATH)
+      version_from_file = last_version_id_from_version_md_xml(version_md_pathname)
+      verify_version_from_xml_file(version_md_pathname, version_from_file) if version_from_file
+
+      version_additions_pathname = deposit_bag_pathname.join(VERSION_ADDITIONS_BASENAME)
+      version_from_file = version_id_from_version_manifest_xml(version_additions_pathname)
+      verify_version_from_xml_file(version_additions_pathname, version_from_file) if version_from_file
+
+      version_inventory_pathname = deposit_bag_pathname.join(VERSION_INVENTORY_BASENAME)
+      version_from_file = version_id_from_version_manifest_xml(version_inventory_pathname)
+      verify_version_from_xml_file(version_inventory_pathname, version_from_file) if version_from_file
+    end
+
+    def last_version_id_from_version_md_xml(version_md_pathname)
+      last_version_id_from_xml(version_md_pathname, '/versionMetadata/version/@versionId')
+    end
+
+    def version_id_from_version_manifest_xml(version_manifest_xml_pathname)
+      last_version_id_from_xml(version_manifest_xml_pathname, '/fileInventory/@versionId')
+    end
+
+    def last_version_id_from_xml(pathname, xpath)
+      doc = Nokogiri::XML(File.open(pathname.to_s), &:strict)
+      version_id = doc.xpath(xpath).last.text unless doc.xpath(xpath).empty?
+      return version_id.to_i if version_id
+      err_data = {
+        version_file: pathname,
+        xpath: xpath
+      }
+      result_array << single_error_hash(VERSION_MISSING_FROM_FILE, err_data) unless version_id
+      nil
+    rescue StandardError => e
+      err_data = {
+        file_pathname: pathname,
+        err_info: "#{e}\n#{e.backtrace}"
+      }
+      result_array << single_error_hash(INVALID_VERSION_XXX_XML, err_data)
+      nil
+    end
+
+    def verify_version_from_xml_file(file_pathname, found)
+      return if found == expected_new_version
+      err_data = {
+        file_pathname: file_pathname,
+        new_version: expected_new_version,
+        file_version: found
+      }
+      result_array << single_error_hash(VERSION_MISMATCH_TO_MOAB, err_data)
+    end
+
+    # adds to result_array if tagmanifest checksums don't match generated checksums
+    def verify_tagmanifests
+      tagmanifests_checksums_hash = checksums_hash_from_manifest_files(TAGMANIFEST)
+      types_to_generate = checksum_types_from_manifest_checksums_hash(tagmanifests_checksums_hash)
+      generated_checksums_hash = generate_tagmanifest_checksums_hash(types_to_generate)
+      verify_manifest_checksums(TAGMANIFEST, tagmanifests_checksums_hash, generated_checksums_hash)
+    end
+
+    # adds to result_array if manifest checksums don't match generated checksums
+    def verify_payload_manifests
+      manifests_checksums_hash = checksums_hash_from_manifest_files(MANIFEST)
+      types_to_generate = checksum_types_from_manifest_checksums_hash(manifests_checksums_hash)
+      generated_checksums_hash = generate_payload_checksums(types_to_generate)
+      verify_manifest_checksums(MANIFEST, manifests_checksums_hash, generated_checksums_hash)
+    end
+
+    # construct hash based on manifest_type-alg.txt files in bag home dir
+    # key: file_name, relative to base_path, value: hash of checksum alg => checksum value
+    def checksums_hash_from_manifest_files(manifest_type)
+      checksums_hash = {}
+      deposit_bag_pathname.children.each do |child_pathname|
+        if child_pathname.file?
+          child_fname = child_pathname.basename.to_s
+          match_result = child_fname.match("^#{manifest_type}-(.*).txt")
+          if match_result
+            checksum_type = match_result.captures.first.to_sym
+            if RECOGNIZED_CHECKSUM_ALGORITHMS.include?(checksum_type)
+              child_pathname.readlines.each do |line|
+                line.chomp!.strip!
+                checksum, file_name = line.split(/[\s*]+/, 2)
+                file_checksums = checksums_hash[file_name] || {}
+                file_checksums[checksum_type] = checksum
+                checksums_hash[file_name] = file_checksums
+              end
+            else
+              result_array << single_error_hash(CHECKSUM_TYPE_UNRECOGNIZED, checksum_type: checksum_type, filename: child_pathname)
+            end
+          end
+        end
+      end
+      checksums_hash
+    end
+
+    # generate hash of checksums by file name for bag home dir files
+    def generate_tagmanifest_checksums_hash(types_to_generate)
+      # all names in the bag home dir except those starting with 'tagmanifest'
+      home_dir_pathnames = deposit_bag_pathname.children.reject { |file| file.basename.to_s.start_with?(TAGMANIFEST) }
+      hash_with_full_pathnames = generate_checksums_hash(home_dir_pathnames, types_to_generate)
+      # return hash keys as basenames only
+      hash_with_full_pathnames.map { |k, v| [Pathname.new(k).basename.to_s, v] }.to_h
+    end
+
+    # generate hash of checksums by file name for bag data dir files
+    def generate_payload_checksums(types_to_generate)
+      data_pathnames = deposit_bag_pathname.join(DATA_DIR_BASENAME).find
+      hash_with_full_pathnames = generate_checksums_hash(data_pathnames, types_to_generate)
+      # return hash keys beginning with 'data/'
+      hash_with_full_pathnames.map { |k, v| [Pathname.new(k).relative_path_from(deposit_bag_pathname).to_s, v] }.to_h
+    end
+
+    def generate_checksums_hash(pathnames, types_to_generate)
+      file_checksums_hash = {}
+      pathnames.each do |pathname|
+        file_checksums_hash[pathname.to_s] = generated_checksums(pathname, types_to_generate) if pathname.file?
+      end
+      file_checksums_hash
+    end
+
+    def generated_checksums(pathname, types_to_generate)
+      my_digester_hash = digester_hash(types_to_generate)
+      pathname.open('r') do |stream|
+        while (buffer = stream.read(8192))
+          my_digester_hash.each_value { |digest| digest.update(buffer) }
+        end
+      end
+      file_checksums = {}
+      my_digester_hash.each do |checksum_type, digest|
+        file_checksums[checksum_type] = digest.hexdigest
+      end
+      file_checksums
+    end
+
+    def digester_hash(types_to_generate=DEFAULT_CHECKSUM_TYPES)
+      types_to_generate.each_with_object({}) do |checksum_type, digester_hash|
+        case checksum_type
+        when :md5
+          digester_hash[checksum_type] = Digest::MD5.new
+        when :sha1
+          digester_hash[checksum_type] = Digest::SHA1.new
+        when :sha256
+          digester_hash[checksum_type] = Digest::SHA2.new(256)
+        when :sha384
+          digesters[checksum_type] = Digest::SHA2.new(384)
+        when :sha512
+          digesters[checksum_type] = Digest::SHA2.new(512)
+        else
+          result_array << single_error_hash(CHECKSUM_TYPE_UNRECOGNIZED, checksum_type: checksum_type, filename: nil)
+        end
+        digester_hash
+      end
+    end
+
+    def verify_manifest_checksums(manifest_type, manifests_checksum_hash, generated_checksum_hash)
+      diff_hash = {}
+      # NOTE: this is intentionally | instead of ||
+      (manifests_checksum_hash.keys | generated_checksum_hash.keys).each do |file_name|
+        manifest_checksums = manifests_checksum_hash[file_name] || {}
+        generated_checksums = generated_checksum_hash[file_name] || {}
+        if manifest_checksums != generated_checksums
+          cdh = checksums_diff_hash(manifest_checksums, generated_checksums, manifest_type, 'generated')
+          diff_hash[file_name] = cdh if cdh
+        end
+      end
+      return if diff_hash.empty?
+      err_data = {
+        manifest_type: manifest_type,
+        diffs: diff_hash
+      }
+      result_array << single_error_hash(CHECKSUM_MISMATCH, err_data)
+    end
+
+    def checksums_diff_hash(left_checksums, right_checksums, left_label, right_label)
+      diff_hash = {}
+      # NOTE: these are intentionally & and | instead of && and ||
+      checksum_types_to_compare = (left_checksums.keys & right_checksums.keys)
+      checksum_types_to_compare = (left_checksums.keys | right_checksums.keys) if checksum_types_to_compare.empty?
+      checksum_types_to_compare.each do |type|
+        left_checksum = left_checksums[type]
+        right_checksum = right_checksums[type]
+        if left_checksum != right_checksum
+          diff_hash[type] = { left_label => left_checksum, right_label => right_checksum }
+        end
+      end
+      diff_hash.empty? ? nil : diff_hash
+    end
+
+    def verify_payload_size
+      sizes_from_bag_info_file = bag_info_payload_size
+      generated_sizes = generated_payload_size
+      return if sizes_from_bag_info_file == generated_sizes
+      err_data = {
+        bag_info_sizes: sizes_from_bag_info_file,
+        generated_sizes: generated_sizes
+      }
+      result_array << single_error_hash(PAYLOAD_SIZE_MISMATCH, err_data)
+    end
+
+    def bag_info_payload_size
+      bag_info_txt_pathname = deposit_bag_pathname.join(BAG_INFO_TXT_BASENAME)
+      bag_info_txt_pathname.readlines.each do |line|
+        line.chomp!.strip!
+        key, value = line.split(':', 2)
+        if key.strip == 'Payload-Oxum'
+          num_bytes, num_files = value.strip.split('.') if value
+          return { bytes: num_bytes.to_i, files: num_files.to_i }
+        end
+      end
+    end
+
+    def generated_payload_size
+      payload_pathname = deposit_bag_pathname.join(DATA_DIR_BASENAME)
+      payload_pathname.find.select(&:file?).each_with_object(bytes: 0, files: 0) do |file, hash|
+        hash[:bytes] += file.size
+        hash[:files] += 1
+        hash
+      end
+    end
+
+    # checksums_hash: { fname => {:md5=>"xxx", :sha1=>"yyy"}, fname => ... }
+    def checksum_types_from_manifest_checksums_hash(checksums_hash)
+      types = []
+      checksums_hash.each_value { |v| v.each_key { |k| types << k unless types.include?(k) } }
+      types
+    end
+
+    def single_error_hash(error_code, err_data_hash)
+      { error_code => error_code_msg(error_code, err_data_hash) }
+    end
+
+    def error_code_msg(error_code, err_data_hash)
+      ERROR_CODE_TO_MESSAGES[error_code] % err_data_hash
+    end
+  end
+end

data/lib/moab/file_group.rb

@@ -66,13 +66,12 @@ module Moab
       %w{group_id file_count byte_count block_count}
     end
 
-
     # @attribute
     # @return [Array<FileManifestation>] The set of files comprising the group
     has_many :files, FileManifestation, :tag => 'file'
 
     def files
-      @signature_hash.values
+      signature_hash.values
     end
 
     # @return [Hash<FileSignature, FileManifestation>] The actual in-memory store for the collection
@@ -84,7 +83,7 @@ module Moab
     #   used to test for existence of a filename in this file group
     def path_hash
       path_hash = Hash.new
-      @signature_hash.each do |signature,manifestation|
+      signature_hash.each do |signature,manifestation|
         manifestation.instances.each do |instance|
           path_hash[instance.path] = signature
         end
@@ -103,7 +102,7 @@ module Moab
     def path_hash_subset(signature_subset)
       path_hash = Hash.new
       signature_subset.each do |signature|
-        manifestation = @signature_hash[signature]
+        manifestation = signature_hash[signature]
         manifestation.instances.each do |instance|
           path_hash[instance.path] = signature
         end
@@ -135,12 +134,12 @@ module Moab
     # @return [void] Add a single {FileSignature},{FileInstance} key/value pair to this group.
     #   Data is actually stored in the {#signature_hash}
     def add_file_instance(signature,instance)
-      if @signature_hash.has_key?(signature)
-        manifestation = @signature_hash[signature]
+      if signature_hash.has_key?(signature)
+        manifestation = signature_hash[signature]
       else
         manifestation = FileManifestation.new
         manifestation.signature = signature
-        @signature_hash[signature] = manifestation
+        signature_hash[signature] = manifestation
       end
       manifestation.instances << instance
     end
@@ -150,7 +149,7 @@ module Moab
     # for example, the manifest inventory does not contain a file entry for itself
     def remove_file_having_path(path)
       signature = self.path_hash[path]
-      @signature_hash.delete(signature)
+      signature_hash.delete(signature)
     end
 
     # @return [Pathname] The full path used as the basis of the relative paths reported
@@ -203,13 +202,13 @@ module Moab
     # @see http://stackoverflow.com/questions/3974087/how-to-make-rubys-find-find-follow-symlinks
     # @see http://stackoverflow.com/questions/357754/can-i-traverse-symlinked-directories-in-ruby-with-a-glob
     def harvest_directory(path, recursive, validated=nil)
-      pathname=Pathname.new(path).expand_path
+      pathname = Pathname.new(path).expand_path
       validated ||= is_descendent_of_base?(pathname)
       pathname.children.sort.each do |child|
         if child.basename.to_s == ".DS_Store"
           next
         elsif child.directory?
-          harvest_directory(child,recursive, validated) if recursive
+          harvest_directory(child, recursive, validated) if recursive
         else
           add_physical_file(child, validated)
         end

data/lib/moab/file_group_difference.rb

@@ -49,7 +49,7 @@ module Moab
     # @param change [String] the change type to search for
     # @return [FileGroupDifferenceSubset] Find a specified subset of changes
     def subset(change)
-      @subset_hash[change.to_sym]
+      subset_hash[change.to_sym]
     end
 
     # (see Serializable#initialize)
@@ -69,7 +69,7 @@ module Moab
 
     def difference_count
       count = 0
-      @subset_hash.each do |type,subset|
+      subset_hash.each do |type, subset|
         count += subset.count if type != :identical
       end
       count
@@ -79,49 +79,49 @@ module Moab
     # @return [Integer] How many files were unchanged
     attribute :identical, Integer, :on_save => Proc.new { |n| n.to_s }
     def identical
-      @subset_hash[:identical].count
+      subset_hash[:identical].count
     end
 
     # @attribute
     # @return [Integer] How many duplicate copies of files were added
     attribute :copyadded, Integer, :on_save => Proc.new { |n| n.to_s }
     def copyadded
-      @subset_hash[:copyadded].count
+      subset_hash[:copyadded].count
     end
 
     # @attribute
     # @return [Integer] How many duplicate copies of files were deleted
     attribute :copydeleted, Integer, :on_save => Proc.new { |n| n.to_s }
     def copydeleted
-      @subset_hash[:copydeleted].count
+      subset_hash[:copydeleted].count
     end
 
     # @attribute
     # @return [Integer] How many files were renamed
     attribute :renamed, Integer, :on_save => Proc.new { |n| n.to_s }
     def renamed
-      @subset_hash[:renamed].count
+      subset_hash[:renamed].count
     end
 
     # @attribute
     # @return [Integer] How many files were modified
     attribute :modified, Integer, :on_save => Proc.new { |n| n.to_s }
     def modified
-      @subset_hash[:modified].count
+      subset_hash[:modified].count
     end
 
     # @attribute
     # @return [Integer] How many files were added
     attribute :added, Integer, :on_save => Proc.new { |n| n.to_s }
     def added
-      @subset_hash[:added].count
+      subset_hash[:added].count
     end
 
     # @attribute
     # @return [Integer] How many files were deleted
     attribute :deleted, Integer, :on_save => Proc.new { |n| n.to_s }
     def deleted
-      @subset_hash[:deleted].count
+      subset_hash[:deleted].count
     end
 
     # @attribute
@@ -130,12 +130,12 @@ module Moab
     has_many :subsets, FileGroupDifferenceSubset, :tag => 'subset'
 
     def subsets
-      @subset_hash.values
+      subset_hash.values
     end
 
     def subsets=(array)
       if array
-        array.each{|subset| @subset_hash[subset.change.to_sym] = subset}
+        array.each{|subset| subset_hash[subset.change.to_sym] = subset}
       end
     end
 
@@ -237,13 +237,12 @@ module Moab
           fid.basis_path = path
           fid.other_path = "same"
           fid.signatures << signature
-          @subset_hash[:identical].files << fid
+          subset_hash[:identical].files << fid
         end
       end
       self
     end
 
-
     # @api internal
     # @param matching_signatures [Array<FileSignature>] The file signature of the file manifestations being compared
     # @param basis_signature_hash [Hash<FileSignature, FileManifestation>]
@@ -272,13 +271,12 @@ module Moab
           else
             fid.change = 'renamed'
           end
-          @subset_hash[fid.change.to_sym].files << fid
+          subset_hash[fid.change.to_sym].files << fid
         end
       end
       self
     end
 
-
     # @api internal
     # @param basis_path_hash [Hash<String,FileSignature>]
     #   The file paths and associated signatures for manifestations appearing only in the basis group
@@ -293,7 +291,7 @@ module Moab
         fid.other_path = "same"
         fid.signatures << basis_path_hash[path]
         fid.signatures << other_path_hash[path]
-        @subset_hash[:modified].files << fid
+        subset_hash[:modified].files << fid
       end
       self
     end
@@ -311,7 +309,7 @@ module Moab
         fid.basis_path = ""
         fid.other_path = path
         fid.signatures << other_path_hash[path]
-        @subset_hash[:added].files << fid
+        subset_hash[:added].files << fid
       end
       self
     end
@@ -329,7 +327,7 @@ module Moab
         fid.basis_path = path
         fid.other_path = ""
         fid.signatures << basis_path_hash[path]
-        @subset_hash[:deleted].files << fid
+        subset_hash[:deleted].files << fid
       end
       self
     end
@@ -340,15 +338,15 @@ module Moab
       deltas = Hash.new {|hash, key| hash[key] = []}
       # case where other_path is empty or 'same'.  (create array of strings)
       [:identical, :modified, :deleted, :copydeleted].each do |change|
-        deltas[change].concat(@subset_hash[change].files.collect{ |file| file.basis_path })
+        deltas[change].concat(subset_hash[change].files.collect{ |file| file.basis_path })
       end
       # case where basis_path and other_path are both present.  (create array of arrays)
       [:copyadded, :renamed].each do |change|
-        deltas[change].concat(@subset_hash[change].files.collect { |file| [file.basis_path,file.other_path] })
+        deltas[change].concat(subset_hash[change].files.collect { |file| [file.basis_path, file.other_path] })
       end
       # case where basis_path is empty.  (create array of strings)
       [:added].each do |change|
-        deltas[change].concat(@subset_hash[change].files.collect { |file| file.other_path })
+        deltas[change].concat(subset_hash[change].files.collect { |file| file.other_path })
       end
       deltas
     end
@@ -362,7 +360,7 @@ module Moab
       # Split the filepairs into two arrays
       oldnames = []
       newnames = []
-      filepairs.each do |old,new|
+      filepairs.each do |old, new|
         oldnames << old
         newnames << new
       end
@@ -374,7 +372,7 @@ module Moab
     # @param [Array<Array<String>>] filepairs The set of oldname, newname pairs for all files being renamed
     # @return [Array<Array<String>>] a set of file triples containing oldname, tempname, newname
     def rename_tempfile_triplets(filepairs)
-      filepairs.collect{|old,new| [old, new, "#{new}-#{Time.now.strftime('%Y%m%d%H%H%S')}-tmp"]}
+      filepairs.collect{ |old, new| [old, new, "#{new}-#{Time.now.strftime('%Y%m%d%H%H%S')}-tmp"] }
     end
 
   end

data/lib/moab/file_inventory.rb

@@ -52,11 +52,11 @@ module Moab
 
     # @attribute
     # @return [Integer] The ordinal version number
-    attribute :version_id, Integer, :tag => 'versionId', :key => true, :on_save => Proc.new {|n| n.to_s}
+    attribute :version_id, Integer, :tag => 'versionId', :key => true, :on_save => Proc.new { |n| n.to_s }
 
     # @return [String] The unique identifier concatenating digital object id with version id
     def composite_key
-      @digital_object_id + '-' + StorageObject.version_dirname(@version_id)
+      digital_object_id + '-' + StorageObject.version_dirname(version_id)
     end
 
     # @attribute
@@ -64,7 +64,7 @@ module Moab
     attribute :inventory_datetime, String, :tag => 'inventoryDatetime'
 
     def inventory_datetime=(datetime)
-      @inventory_datetime=Moab::UtcTime.input(datetime)
+      @inventory_datetime = Moab::UtcTime.input(datetime)
     end
 
     def inventory_datetime
@@ -73,7 +73,7 @@ module Moab
 
     # @attribute
     # @return [Integer] The total number of data files in the inventory (dynamically calculated)
-    attribute :file_count, Integer, :tag => 'fileCount', :on_save => Proc.new {|t| t.to_s}
+    attribute :file_count, Integer, :tag => 'fileCount', :on_save => Proc.new { |t| t.to_s }
 
     def file_count
       groups.inject(0) { |sum, group| sum + group.file_count }
@@ -81,7 +81,7 @@ module Moab
 
      # @attribute
     # @return [Integer] The total size (in bytes) in all files of all files in the inventory (dynamically calculated)
-    attribute :byte_count, Integer, :tag => 'byteCount', :on_save => Proc.new {|t| t.to_s}
+    attribute :byte_count, Integer, :tag => 'byteCount', :on_save => Proc.new { |t| t.to_s }
 
     def byte_count
       groups.inject(0) { |sum, group| sum + group.byte_count }
@@ -89,7 +89,7 @@ module Moab
 
     # @attribute
     # @return [Integer] The total disk usage (in 1 kB blocks) of all data files (estimating du -k result) (dynamically calculated)
-    attribute :block_count, Integer, :tag => 'blockCount', :on_save => Proc.new {|t| t.to_s}
+    attribute :block_count, Integer, :tag => 'blockCount', :on_save => Proc.new { |t| t.to_s }
 
     def block_count
       groups.inject(0) { |sum, group| sum + group.block_count }
@@ -101,20 +101,20 @@ module Moab
 
     # @return [Array<FileGroup] The set of data groups that contain files
     def non_empty_groups
-      @groups.select{|group| !group.files.empty?}
+      groups.select { |group| !group.files.empty? }
     end
 
     # @param non_empty [Boolean] if true, return group_id's only for groups having files
     # @return [Array<String>] group identifiers contained in this file inventory
     def group_ids(non_empty=nil)
-      groups = non_empty ? self.non_empty_groups : @groups
-      groups.map{|group| group.group_id}
+      my_groups = non_empty ? self.non_empty_groups : groups
+      my_groups.map { |g| g.group_id }
     end
 
     # @param [String] group_id The identifer of the group to be selected
     # @return [FileGroup] The file group in this inventory for the specified group_id
     def group(group_id)
-      @groups.find{ |group| group.group_id == group_id}
+      groups.find { |group| group.group_id == group_id}
     end
 
     # @param group_id [String] File group identifer (e.g. data, metadata, manifests)
@@ -134,10 +134,10 @@ module Moab
     # @return [FileSignature] The signature of the specified file
     def file_signature(group_id, file_id)
       file_group = group(group_id)
-      errmsg = "group #{group_id} not found for #{@digital_object_id} - #{@version_id}"
+      errmsg = "group #{group_id} not found for #{digital_object_id} - #{version_id}"
       raise FileNotFoundException, errmsg if file_group.nil?
       file_signature = file_group.path_hash[file_id]
-      errmsg = "#{group_id} file #{file_id} not found for #{@digital_object_id} - #{@version_id}"
+      errmsg = "#{group_id} file #{file_id} not found for #{digital_object_id} - #{version_id}"
       raise FileNotFoundException, errmsg if file_signature.nil?
       file_signature
     end
@@ -154,7 +154,7 @@ module Moab
     # @api internal
     # @return [String] Concatenation of the objectId and versionId values
     def package_id
-      "#{@digital_object_id}-v#{@version_id}"
+      "#{digital_object_id}-v#{version_id}"
     end
 
     # @api internal
@@ -174,7 +174,6 @@ module Moab
         else
           data_source
         end
-
       end
     end
 
@@ -186,10 +185,10 @@ module Moab
     # @example {include:file:spec/features/inventory/harvest_inventory_spec.rb}
     def inventory_from_directory(data_dir, group_id=nil)
       if group_id
-        @groups << FileGroup.new(group_id: group_id).group_from_directory(data_dir)
+        groups << FileGroup.new(group_id: group_id).group_from_directory(data_dir)
       else
         ['content', 'metadata'].each do |gid|
-          @groups << FileGroup.new(group_id: gid).group_from_directory(Pathname(data_dir).join(gid))
+          groups << FileGroup.new(group_id: gid).group_from_directory(Pathname(data_dir).join(gid))
         end
       end
       self
@@ -203,7 +202,7 @@ module Moab
       signatures_from_bag = signatures_from_bagit_manifests(bag_pathname)
       bag_data_subdirs = bag_pathname.join('data').children
       bag_data_subdirs.each do |subdir|
-        @groups << FileGroup.new(:group_id=>subdir.basename.to_s).group_from_bagit_subdir(subdir, signatures_from_bag)
+        groups << FileGroup.new(:group_id=>subdir.basename.to_s).group_from_bagit_subdir(subdir, signatures_from_bag)
       end
       self
     end
@@ -212,12 +211,11 @@ module Moab
     # @return [Hash<Pathname,FileSignature>] The fixity data present in the bag's manifest files
     def signatures_from_bagit_manifests(bag_pathname)
       manifest_pathname = Hash.new
-      checksum_types =  [:md5, :sha1, :sha256]
-      checksum_types.each do |type|
+      DEFAULT_CHECKSUM_TYPES.each do |type|
         manifest_pathname[type] = bag_pathname.join("manifest-#{type}.txt")
       end
-      signatures = Hash.new { |hash,path| hash[path] = FileSignature.new }
-      checksum_types.each do |type|
+      signatures = Hash.new { |hash, path| hash[path] = FileSignature.new }
+      DEFAULT_CHECKSUM_TYPES.each do |type|
         if manifest_pathname[type].exist?
           manifest_pathname[type].each_line do |line|
             line.chomp!
@@ -230,7 +228,7 @@ module Moab
           end
         end
       end
-      signatures.each {|file_pathname,signature| signature.size = file_pathname.size}
+      signatures.each { |file_pathname, signature| signature.size = file_pathname.size }
       signatures
     end
 

data/lib/moab/file_inventory_difference.rb

@@ -55,7 +55,7 @@ module Moab
     attribute :report_datetime, String, :tag => 'reportDatetime'
 
     def report_datetime=(datetime)
-      @report_datetime=Moab::UtcTime.input(datetime)
+      @report_datetime = Moab::UtcTime.input(datetime)
     end
 
     def report_datetime
@@ -115,7 +115,7 @@ module Moab
       inv_diff = self.to_hash
       inv_diff["group_differences"].each_value do |group_diff|
         delete_subsets = []
-        group_diff["subsets"].each do |change_type,subset|
+        group_diff["subsets"].each do |change_type, subset|
           delete_subsets << change_type if change_type == "identical" or subset["count"] == 0
         end
         delete_subsets.each do |change_type|

data/lib/moab/file_manifestation.rb

@@ -68,7 +68,7 @@ module Moab
     # @return [Integer] The total disk usage (in 1 kB blocks) of all files that share this manifestation's signature
     #   (estimating du -k result)
     def block_count
-      block_size=1024
+      block_size = 1024
       instance_blocks = (signature.size.to_i + block_size - 1)/block_size
       file_count * instance_blocks
     end

data/lib/moab/storage_object_validator.rb

@@ -9,7 +9,7 @@ module Moab
     METADATA_DIR = "metadata".freeze
     CONTENT_DIR = "content".freeze
     EXPECTED_DATA_SUB_DIRS = [CONTENT_DIR, METADATA_DIR].freeze
-    IMPLICIT_DIRS = ['.', '..', '.keep'].freeze # unlike Find.find, Dir.entries returns the current/parent dirs
+    IMPLICIT_DIRS = ['.', '..'].freeze # unlike Find.find, Dir.entries returns the current/parent dirs
     DATA_DIR = "data".freeze
     MANIFESTS_DIR = 'manifests'.freeze
     EXPECTED_VERSION_SUB_DIRS = [DATA_DIR, MANIFESTS_DIR].freeze

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: moab-versioning
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Darren Weber
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-02 00:00:00.000000000 Z
+date: 2018-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: confstruct
@@ -69,20 +69,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: systemu
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: druid-tools
   requirement: !ruby/object:Gem::Requirement
@@ -97,20 +83,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: awesome_print
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: equivalent-xml
   requirement: !ruby/object:Gem::Requirement
@@ -139,20 +111,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -181,20 +139,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: yard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -247,6 +191,7 @@ files:
 - lib/moab.rb
 - lib/moab/bagger.rb
 - lib/moab/config.rb
+- lib/moab/deposit_bag_validator.rb
 - lib/moab/exceptions.rb
 - lib/moab/file_group.rb
 - lib/moab/file_group_difference.rb
@@ -280,7 +225,6 @@ files:
 - lib/stanford/storage_object_validator.rb
 - lib/stanford/storage_repository.rb
 - lib/stanford/storage_services.rb
-- lib/tasks/yard.rake
 homepage: https://github.com/sul-dlss/moab-versioning
 licenses:
 - Apache-2.0

data/lib/tasks/yard.rake

@@ -1,34 +0,0 @@
-desc "Generate RDoc"
-task :doc => ['doc:generate']
-
-namespace :doc do
-  project_root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
-  doc_destination = File.join(project_root, 'doc')
-
-  begin
-    require 'yard'
-    require 'yard/rake/yardoc_task'
-
-    YARD::Rake::YardocTask.new(:generate) do |yt|
-      yt.files   =  Dir.glob(File.join(project_root, 'lib', '*.rb')) + 
-                    Dir.glob(File.join(project_root, 'lib', 'serializer', '*.rb')) +
-                    Dir.glob(File.join(project_root, 'lib', 'moab', '*.rb')) +
-                    Dir.glob(File.join(project_root, 'lib', 'stanford', '*.rb')) +
-                    ['-'] +
-                   [ File.join(project_root, 'LICENSE.rdoc') ]
-                   
-      yt.options = ['--output-dir', doc_destination, '--hide-void-return']
-    end
-  rescue LoadError
-    desc "Generate YARD Documentation"
-    task :generate do
-      abort "Please install the YARD gem to generate rdoc."
-    end
-  end
-
-  desc "Remove generated documentation"
-  task :clean do
-    rm_r doc_destination if File.exists?(doc_destination)
-  end
-
-end