ocfl-tools 0.9.14

data/lib/ocfl_tools/utils.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module OcflTools
+  module Utils
+    # converts [Integer] version to [String] v0001 format.
+    # Adjust VERSION_FORMAT to format string version to local needs.
+    # @return [String] of version in desired format, starting with 'v'.
+    def self.version_int_to_string(version)
+      result = OcflTools.config.version_format % version.to_i
+    end
+
+    # converts [String] version name to [Integer].
+    # OCFL spec requires string versions to start with 'v'.
+    # Chop off the 'v' at th start, make into integer.
+    # @param [String] version_name string to convert to an integer.
+    # @return [Integer] the version as an integer.
+    def self.version_string_to_int(version_name)
+      result = version_name.split('v')[1].to_i
+    end
+
+    # We sometimes need to make deep (not shallow) copies of objects, mostly hashes.
+    # When we are copying state from a prior version, we don't want our copy to still
+    # be mutable by that prior version hash. So a deep (serialized) copy is called for.
+    # @param [Object] o object to make a deep copy of.
+    # @return [Object] a new object with no links to the previous one.
+    def self.deep_copy(o)
+      # We need this serialize Hashes so they don't shallow'y refer to each other.
+      Marshal.load(Marshal.dump(o))
+    end
+
+    # Given a fully-resolvable file path, calculate and return @digest.
+    # @param [String] file fully-resolvable filesystem path to a file.
+    # @param [String] digest to encode file with.
+    # @return [String] checksum of requested file using specified digest algorithm.
+    def self.generate_file_digest(file, digest)
+      case digest
+      when 'md5'
+      # checksum = Digest::MD5.hexdigest(File.read(file))
+        computed_hash = Digest::MD5.new
+        open(file) do |s|
+          while chunk=s.read(8096)
+            computed_hash.update chunk
+          end
+        end
+        return "#{computed_hash}" # return as a String, not a Digest object.
+      when 'sha1'
+      #  checksum = Digest::SHA1.hexdigest(File.read(file))
+        computed_hash = Digest::SHA1.new
+        open(file) do |s|
+          while chunk=s.read(8096)
+            computed_hash.update chunk
+          end
+        end
+        return "#{computed_hash}" # return as a String, not a Digest object.
+      when 'sha256'
+      # checksum = Digest::SHA256.hexdigest(File.read(file))
+        computed_hash = Digest::SHA256.new
+        open(file) do |s|
+          while chunk=s.read(8096)
+            computed_hash.update chunk
+          end
+        end
+        return "#{computed_hash}" # return as a String, not a Digest object.
+      when 'sha512'
+      #  checksum = Digest::SHA512.hexdigest(File.read(file))
+        computed_hash = Digest::SHA512.new
+        open(file) do |s|
+          while chunk=s.read(8096)
+            computed_hash.update chunk
+          end
+        end
+        return "#{computed_hash}" # return as a String, not a Digest object.
+      else
+        raise 'Unknown digest type!'
+      end
+      checksum
+    end
+
+    # @param [Hash] disk_checksums first hash of [ filepath => digest ] to compare.
+    # @param [Hash] inventory_checksums second hash of [ filepath => digest ] to compare.
+    # @param {OcflTools::OcflResults} results optional results instance to put results into.
+    def self.compare_hash_checksums(disk_checksums:, inventory_checksums:, results: OcflTools::OcflResults.new, context: 'verify_checksums')
+      unless results.is_a?(OcflTools::OcflResults)
+        raise 'You need to give me a results instance!'
+      end
+
+      # 1st check! If everything is perfect, these two Hashs SHOULD BE IDENTICAL!
+      if inventory_checksums == disk_checksums
+        results.ok('O200', context, 'All digests successfully verified.')
+        return results
+      end
+
+      # If they are NOT the same, we have to increment thru the Hashes to work out what's up.
+      # It might be a file in the manifest that's not found on disk
+      # Or a file on disk that's not in the manifest.
+      # Or a file that is on disk and in the manifest, but the checksums don't match.
+
+      disk_files       = disk_checksums.keys
+      inventory_files  = inventory_checksums.keys
+
+      missing_from_inventory = disk_files - inventory_files
+      missing_from_disk      = inventory_files - disk_files
+
+      unless missing_from_inventory.empty?
+        missing_from_inventory.each do |missing|
+          results.error('E111', context, "#{missing} found on disk but missing from inventory.json.")
+        end
+      end
+
+      unless missing_from_disk.empty?
+        missing_from_disk.each do |missing|
+          results.error('E111', context, "#{missing} in inventory but not found on disk.")
+        end
+      end
+
+      # checksum mismatches; requires the file to be in both hashes, so.
+      inventory_checksums.each do |file, digest|
+        next unless disk_checksums.key?(file)
+
+        if disk_checksums[file] != digest
+          results.error('E111', context, "#{file} digest in inventory does not match digest computed from disk")
+        end
+      end
+      results
+    end
+  end
+end

data/lib/ocfl_tools/utils_file.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module OcflTools
+  module Utils
+    module Files
+      # Given a directory, return a list of all files (no dirs or special files) found beneath it.
+      # @param [Pathname] directory full file path to directory to search.
+      # @return [Array] of files found in all sub-directories of given path.
+      def self.get_dir_files(directory)
+        # Don't crash out if the requested dir doesn't exist, just state the obvious: there are no files in it.
+        return [] unless Dir.exist?(directory) == true
+
+        Dir.chdir(directory)
+        directory_files = []
+        Dir.glob('**/*').select do |file|
+          directory_files << file if File.file? file
+        end
+        directory_files
+      end
+
+      # Given an object root and a version, return the files on disk in the appropriate content dir.
+      # @return [Array] of fully-qualified filepaths for this version of the {OcflTools::Ocflinventory}
+      def self.get_version_dir_files(object_root_dir, version)
+        version_format = OcflTools::Utils::Files.get_version_format(object_root_dir)
+        # Int to version format
+        version_name = version_format % version.to_i
+        # Get latest inventory file
+        inventory = OcflTools::Utils::Files.get_latest_inventory(object_root_dir)
+        # Get contentDirectory value from inventory (or use default value)
+        contentDirectory = OcflTools::Utils::Inventory.get_contentDirectory(inventory)
+        # Now bring it together and get the goods.
+        my_files = OcflTools::Utils::Files.get_dir_files("#{object_root_dir}/#{version_name}/#{contentDirectory}")
+        # And expand it to a full file path
+        OcflTools::Utils::Files.expand_filepaths(my_files, "#{object_root_dir}/#{version_name}/#{contentDirectory}")
+      end
+
+      # Given an object root and two versions, get the files on disk for that range of versions (inclusive)
+      def self.get_versions_dir_files(object_root_dir, version1, version2)
+        top_ver = [version1, version2].max
+        bot_ver = [version1, version2].min
+        all_files = []
+        count = bot_ver       # start at the bottom
+        until count > top_ver # count to the top
+          all_files << OcflTools::Utils::Files.get_version_dir_files(object_root_dir, count)
+          count += 1
+        end
+        raise 'No files found in version directories!' if all_files.empty?
+
+        all_files.flatten!
+      end
+
+      # Given an object root directory, deduce and return the version directories by inspecting disk.
+      def self.get_version_directories(object_root_dir)
+        unless Dir.exist?(object_root_dir) == true
+          raise 'Directory does not exist!'
+        end
+
+        object_root_dirs = []
+        version_directories = []
+        Dir.chdir(object_root_dir)
+        Dir.glob('*').select do |file|
+          object_root_dirs << file if File.directory? file
+        end
+        if object_root_dirs.empty?
+          raise "No directories found in #{object_root_dir}!"
+        end
+
+        # Needs to call get version_format method here.
+        object_root_dirs.each do |i|
+          if i =~ /[^"{OcflTools::Utils.Files.get_version_format(object_root_dir)}"$]/
+            version_directories << i
+          end
+        end
+        raise 'No version directories found!' if version_directories.empty?
+
+        version_directories.sort! # sort it, to be nice.
+      end
+
+      # Given an object_root_directory, deduce the format used to describe version directories.
+      def self.get_version_format(object_root_dir)
+        unless Dir.exist?(object_root_dir) == true
+          raise 'Directory does not exist!'
+        end
+
+        # Get all directories starting with 'v', sort them.
+        # Take the top of the sort. Count the number of 0s found.
+        # Raises errors if it can't find an appropriate version 1 directory.
+        version_dirs = []
+        Dir.chdir(object_root_dir)
+        Dir.glob('v*').select do |file|
+          version_dirs << file if File.directory? file
+        end
+        version_dirs.sort!
+        # if there's a verson_dirs that's just 'v', throw it out! It's hot garbage edge case we'll deal with later.
+        version_dirs.delete('v') if version_dirs.include? 'v'
+
+        first_version = version_dirs[0] # the first element should be the first version directory.
+        first_version.slice!(0, 1) # cut the leading 'v' from the string.
+        if first_version.length == 1 # A length of 1 for the first version implies 'v1'
+          unless first_version.to_i == 1
+            raise "#{object_root_dir}/#{first_version} is not the first version directory!"
+          end
+
+          version_format = 'v%d'
+        else
+          # Make sure this is Integer 1.
+          unless first_version.to_i == 1
+            raise "#{object_root_dir}/#{first_version} is not the first version directory!"
+          end
+
+          version_format = "v%0#{first_version.length}d"
+        end
+        version_format
+      end
+
+      # Given a [Hash] of digests and [ filepaths ], flip & expand to unique Filepath => digest.
+      def self.invert_and_expand(digest_hash)
+        raise 'This only works on Hashes, buck-o' unless digest_hash.is_a?(Hash)
+
+        working_hash = OcflTools::Utils.deep_copy(digest_hash)
+        return_hash = {}
+        working_hash.each do |key, value|
+          value.each do |v|
+            return_hash[v] = key
+          end
+        end
+        return_hash
+      end
+
+      # Given a hash of digest => [ Filepaths ], invert and expand, then prepend a string to all filepaths.
+      def self.invert_and_expand_and_prepend(digest_hash, prepend_string)
+        raise 'This only works on Hashes, buck-o' unless digest_hash.is_a?(Hash)
+
+        return_hash = {}
+        filepath_hash = OcflTools::Utils::Files.invert_and_expand(digest_hash)
+        filepath_hash.each do |file, digest|
+          filepaths = OcflTools::Utils::Files.expand_filepaths(file, prepend_string)
+          return_hash[filepaths[0]] = digest
+        end
+        return_hash
+      end
+
+      # Given an array of files and a digestAlgorithm, create digests and return results in a [Hash]
+      def self.create_digests(files, digestAlgorithm)
+        my_digests = {}
+        array = Array(files) # make sure it's an array, so we can handle single files as well.
+        array.each do |file|
+          my_digests[file] = OcflTools::Utils.generate_file_digest(file, digestAlgorithm)
+        end
+        my_digests
+      end
+
+      # Given an array of (relative to object root) filepaths, expand to fully-resovable filesystem paths.
+      # If the object_root_dir is already at the front of the filepath, don't add it again.
+      def self.expand_filepaths(files, object_root_dir)
+        array = Array(files) # make sure whatever we have is an array, so we can handle single files too.
+        my_full_filepaths = []
+        array.each do |f|
+          #  /^#{object_root_dir}/ matches on what we want.
+          unless f =~ /^#{object_root_dir}/
+            my_full_filepaths << "#{object_root_dir}/#{f}"
+          end
+        end
+        my_full_filepaths
+      end
+
+      # Given an object root dir, get the most recent inventory file.
+      def self.get_latest_inventory(object_root_dir)
+        # Tries most recent version dir first, then object root, then other version dirs.
+        # g_v_d returns a sorted array already. Reverse it, so we start with highest version.
+        my_versions = OcflTools::Utils::Files.get_version_directories(object_root_dir).reverse
+        case
+        when File.exist?("#{object_root_dir}/#{my_versions[0]}/inventory.json")
+          return "#{object_root_dir}/#{my_versions[0]}/inventory.json"
+        when File.exist?("#{object_root_dir}/inventory.json")
+          return "#{object_root_dir}/inventory.json"
+        else
+          # Quit out here if there was only 1 version directory
+          unless my_versions.size > 1
+            raise "No inventory file found in #{object_root_dir}!"
+          end
+
+          my_versions.delete_at(0) # drop the first element.
+          my_versions.each do |v|
+            if File.exist?("#{object_root_dir}/#{v}/inventory.json")
+              return "#{object_root_dir}/#{v}/inventory.json"
+            end
+          end
+          # If we get here, no inventory file found!
+          raise "No inventory file found in #{object_root_dir}!"
+        end
+      end
+    end
+  end
+end

data/lib/ocfl_tools/utils_inventory.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module OcflTools
+  module Utils
+    # A module of convenience methods for reading information from an OCFL inventory.json file.
+    # {get_value} and its children are designed to account for reading info from the top few lines of a potentially many-MB size file,
+    # without having to load it all into memory by ingesting it with {OcflTools::OcflInventory}.
+    module Inventory
+      # Given an inventory file and a key to search for, return the value at that key.
+      # @param [Pathname] inventory_file fully-qualified path to a valid OCFL inventory.json.
+      # @param [String] key the JSON key in the inventory file that you want to return a value for.
+      # @return [String or nil] the value of the requested key, or nil if not found.
+      def self.get_value(inventory_file, key)
+        unless %w[contentDirectory digestAlgorithm head type id].include?(key)
+          raise "#{key} is not a valid OCFL inventory header key"
+        end
+
+        inventory = OcflTools::OcflInventory.new.from_file(inventory_file)
+
+        case key
+          when 'contentDirectory'
+            inventory.contentDirectory
+          when 'digestAlgorithm'
+            inventory.digestAlgorithm
+          when 'head'
+            inventory.head
+          when 'type'
+            inventory.type
+          when 'id'
+            inventory.id
+          else
+            raise "Unknown key #{key}"
+        end
+
+      end
+
+      # Given an inventory file, return the value of contentDirectory IF FOUND, or 'content' if contentDirectory is not set.
+      # It explicitly does NOT use the config.content_directory setting for this check.
+      # @param [Pathname] inventory_file fully-qualified path to a valid OCFL inventory.json.
+      # @return [String] the value of content_directory in the JSON, if found, or the OCFL required default value of 'content'.
+      def self.get_contentDirectory(inventory_file)
+        contentDirectory = OcflTools::Utils::Inventory.get_value(inventory_file, 'contentDirectory')
+        contentDirectory = 'content' if contentDirectory.nil?
+        contentDirectory
+      end
+
+      # Given an inventory file, return the name of the digest algorithm used (e.g. 'sha512').
+      # @param [Pathname] inventory_file fully-qualified path to a valid OCFL inventory.json.
+      # @return [String] the string value describing the digest algorithm used in this inventory.
+      def self.get_digestAlgorithm(inventory_file)
+        digestAlgorithm = OcflTools::Utils::Inventory.get_value(inventory_file, 'digestAlgorithm')
+        if digestAlgorithm.nil?
+          # Actually against OCFL spec
+          raise "Unable to find value for digestAlgorithm in #{inventory_file}"
+        end
+
+        digestAlgorithm
+      end
+
+      # Given an inventory file, return the fixity block (if it exists) or nil.
+      # @param [Pathname] inventory_file fully-qualified path to a valid OCFL inventory.json.
+      # @return [Hash or nil] the fixity block from the provided inventory.json, or nil if the inventory does not contain a fixity block.
+      def self.get_fixity(inventory_file)
+        inventory = OcflTools::OcflInventory.new.from_file(inventory_file)
+        return nil if inventory.fixity.empty?
+
+        inventory.fixity
+      end
+
+      # Given an inventory file, return [Array] of the digest types found in the fixity block, or nil.
+      # @param [Pathname] inventory_file fully-qualified path to a valid OCFL inventory.json.
+      # @return [Array or nil] an array of [String] values, with each value being a digest type found in the fixity block, e.g. 'sha1', 'md5', etc, or nil if no fixity block is found.
+      def self.get_fixity_digestAlgorithms(inventory_file)
+        inventory = OcflTools::OcflInventory.new.from_file(inventory_file)
+        return nil if inventory.fixity.empty?
+
+        inventory.fixity.keys
+      end
+
+      # Given an inventory file and a digestAlgorithm, return [Hash] of digests and [ filepaths ], or nil.
+      # @param [Pathname] inventory_file fully-qualified path to a valid OCFL inventory.json.
+      # @param [String] digestAlgorithm the algorithm used in the fixity block that you want digests for.
+      # @return [Hash or nil] a hash of digests and filepaths from the fixity block for the given digest type, or nil if the inventory.json does not contain a fixity block.
+      def self.get_fixity_digests(inventory_file, digestAlgorithm)
+        inventory = OcflTools::OcflInventory.new.from_file(inventory_file)
+        return nil if inventory.fixity.empty?
+
+        inventory.fixity[digestAlgorithm]
+      end
+
+      # Given an inventory & version, return files from that version.
+
+      # Given an inventory and 2 versions, return all files for range of versions.
+    end
+  end
+end

data/ocfl-tools.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |gem|
+  gem.authors       = ['Julian M. Morley']
+  gem.email         = ['jmorley@stanford.edu']
+  gem.description   = 'Tools to create, manipulate and write Oxford Common File Layout (OCFL) preservation objects.'
+  gem.summary       = 'Tools to create, manipulate and write Oxford Common File Layout (OCFL) preservation objects.'
+  gem.homepage      = 'https://github.com/sul-dlss-labs/OCFL-Tools'
+  gem.licenses      = ['Apache-2.0']
+
+  gem.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  # gem.files         = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
+
+  gem.add_runtime_dependency 'anyway_config', '~> 1.0'
+  gem.add_runtime_dependency 'fileutils', '~> 1.3'
+  gem.add_runtime_dependency 'json', '~> 2.2', '>= 2.2.0'
+
+  #  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  #  gem.test_files    = gem.files.grep(%r{^spec/})
+  gem.name          = 'ocfl-tools'
+  gem.require_paths = ['lib']
+  gem.version       = File.read('VERSION').strip
+  # gem.metadata["yard.run"] = "yri" # use "yard" to build full HTML docs.
+  gem.add_development_dependency 'pry-byebug' unless ENV['CI']
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rubocop'
+  gem.add_development_dependency 'rubocop-rspec'
+end