ocfl-tools 0.9.14

data/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+task default: %i[spec]
+
+task all: %i[spec rubocop]

data/VERSION

@@ -0,0 +1 @@
+0.9.14

data/examples/list_files.rb

@@ -0,0 +1,56 @@
+# A simple example to demonstrate the relationship between logical content in an OCFL object
+# and the fully-resolved path to those binaries on the local storage system.
+
+require 'ocfl-tools'
+require 'optparse'
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.on('-d DIRECTORY', '--dir DIRECTORY', 'A directory containing an OCFL object') do |dir|
+    unless Dir.exist?(dir)
+      raise "#{dir} is not a valid directory path."
+    end
+    options[:object_root] = dir
+  end
+
+  opts.on('-v VERSION', '--version VERSION', 'An optional version number') do |ver|
+    options[:version] = ver.to_i
+  end
+
+end
+
+opts.parse(ARGV)
+
+raise OptionParser::MissingArgument if options[:object_root].nil?
+
+object_root = options[:object_root]
+
+# The inventory we're working on might not conform to the site default version format.
+# Inspect the object root to determine what version format we should use, and use it.
+OcflTools.config.version_format = OcflTools::Utils::Files.get_version_format(object_root)
+
+# Get the latest inventory file from the object root.
+inventory_file = OcflTools::Utils::Files.get_latest_inventory(object_root)
+
+# Create an ocfl object from that inventory.
+ocfl_object = OcflTools::OcflInventory.new.from_file(inventory_file)
+
+# If we've been asked for a specific version, use it.
+if options[:version].nil?
+  version = OcflTools::Utils.version_string_to_int(ocfl_object.head)
+else
+  version = options[:version]
+end
+
+local_files = ocfl_object.get_files(version)
+
+# Prepend the object root path to content_path to get fully-resolvable files.
+local_files.each do | logical_path, content_path |
+  local_files[logical_path] = object_root + '/' + content_path
+end
+
+# Output a pretty result, for demo purposes.
+local_files.each do | logical_path, content_path |
+  puts "  #{logical_path} => #{content_path}"
+end

data/examples/validate_object.rb

@@ -0,0 +1,23 @@
+# Usage: ruby ./validate_object.rb /path/to/directory/to/check
+require 'ocfl-tools'
+require 'optparse'
+
+options = {}
+
+opts = OptionParser.new do |opts|
+  opts.on('-d DIRECTORY', '--dir DIRECTORY', 'A directory containing an OCFL object') do |dir|
+    unless Dir.exist?(dir)
+      raise "#{dir} is not a valid directory path."
+    end
+    options[:object_root] = dir
+  end
+end
+
+opts.parse(ARGV)
+
+raise OptionParser::MissingArgument if options[:object_root].nil?
+
+object_root = options[:object_root]
+
+
+OcflTools::OcflValidator.new(object_root).validate_ocfl_object_root.print

data/lib/ocfl-tools.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# OcflTools is a module that provides a distintive namespace for classes that create,
+# maintain and read Oxford Common File Layout preservation objects.
+#
+# ====Data Model
+#
+# * <b>{OcflObject} = an object that models the internal data structures of an OCFL manifest.</b>
+#   * {OcflInventory} = An I/O interface for {OcflObject} allowing the reading and creaton of OCFL inventory.json files.
+#
+# @note Copyright (c) 2019 by The Board of Trustees of the Leland Stanford Junior University.
+
+require 'ocfl_tools'
+require 'json'
+require 'anyway'
+require 'fileutils'
+require 'digest'
+require 'time' # for iso8601 checking.
+require 'uri' # for, well, uri testing.

data/lib/ocfl_tools.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module OcflTools
+  require 'ocfl_tools/ocfl_object'
+  require 'ocfl_tools/ocfl_inventory'
+  require 'ocfl_tools/ocfl_verify'
+  require 'ocfl_tools/ocfl_deposit'
+  require 'ocfl_tools/ocfl_validator'
+  require 'ocfl_tools/ocfl_results'
+  require 'ocfl_tools/ocfl_delta'
+  require 'ocfl_tools/ocfl_actions'
+  require 'ocfl_tools/ocfl_errors'
+  require 'ocfl_tools/config'
+  require 'ocfl_tools/utils'
+  require 'ocfl_tools/utils_file'
+  require 'ocfl_tools/utils_inventory'
+end

data/lib/ocfl_tools/config.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'anyway'
+
+module OcflTools
+  # Site-wide configuration settings for OCFL-Tools, using the 'anyway' gem.
+  # Settings and their default values are:
+  #            version_format: "v%04d",
+  #              content_type: 'https://ocfl.io/1.0/spec/#inventory',
+  #         content_directory: 'content',
+  #          digest_algorithm: 'sha512',
+  #         fixity_algorithms: ['md5', 'sha1', 'sha256']
+  #              ocfl_version: '1.0'
+  class Config < Anyway::Config
+    attr_config version_format: 'v%04d',
+                content_type: 'https://ocfl.io/1.0/spec/#inventory',
+                content_directory: 'content',
+                digest_algorithm: 'sha512',
+                fixity_algorithms: %w[md5 sha1 sha256], # site-specific allowable fixity algorithms
+                ocfl_version: '1.0'
+  end
+
+  # Creates a new config instance if it doesn't already exist.
+  def self.config
+    @config ||= Config.new
+  end
+end

data/lib/ocfl_tools/ocfl_actions.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module OcflTools
+  # Class for collating manifest actions, both for delta reporting and staging new versions.
+  class OcflActions
+    def initialize
+      @my_actions                    = {}
+      @my_actions['update_manifest'] = {}
+      @my_actions['add']             = {}
+      @my_actions['update']          = {}
+      @my_actions['copy']            = {}
+      @my_actions['move']            = {}
+      @my_actions['delete']          = {}
+    end
+
+    # Convenience method for obtaining a hash of recorded actions.
+    # @return [Hash] of actions stored in this instance.
+    def actions
+      # Don't return empty keys.
+      @my_actions.delete_if { |_k, v| v == {} }
+      @my_actions
+    end
+
+    # Convenience method for obtaining a hash recorded of actions.
+    # @return [Hash] of actions stored in this instance.
+    def all
+      # Don't return empty keys.
+      @my_actions.delete_if { |_k, v| v == {} }
+      @my_actions
+    end
+
+    # Creates an 'update_manifest' entry in the actions hash.
+    # @param [String] digest of the filepath being recorded.
+    # @param [Pathname] filepath of file to record.
+    # @return [Hash] of recorded action.
+    def update_manifest(digest, filepath)
+      if @my_actions['update_manifest'].key?(digest) == false
+        @my_actions['update_manifest'][digest] = []
+      end
+      # Only put unique values into filepaths
+      if @my_actions['update_manifest'][digest].include?(filepath)
+        return @my_actions['update_manifest'][digest]
+      else
+        @my_actions['update_manifest'][digest] = (@my_actions['update_manifest'][digest] << filepath)
+      end
+    end
+
+    # Creates an 'add' entry in the actions hash.
+    # @param [String] digest of the filepath being recorded.
+    # @param [Pathname] filepath of file to record.
+    # @return [Hash] of recorded action.
+    def add(digest, filepath)
+      if @my_actions['add'].key?(digest) == false
+        @my_actions['add'][digest] = []
+      end
+      # Only put unique values into filepaths
+      if @my_actions['add'][digest].include?(filepath)
+        return @my_actions['add'][digest]
+      else
+        @my_actions['add'][digest] = (@my_actions['add'][digest] << filepath)
+      end
+    end
+
+    # Creates an 'update' entry in the actions hash.
+    # @param [String] digest of the filepath being recorded.
+    # @param [Pathname] filepath of file to record.
+    # @return [Hash] of recorded action.
+    def update(digest, filepath)
+      if @my_actions['update'].key?(digest) == false
+        @my_actions['update'][digest] = []
+      end
+      # Only put unique values into filepaths
+      if @my_actions['update'][digest].include?(filepath)
+        return @my_actions['update'][digest]
+      else
+        @my_actions['update'][digest] = (@my_actions['update'][digest] << filepath)
+      end
+    end
+
+    # Creates a 'copy' entry in the actions hash.
+    # @param [String] digest of the filepath being recorded.
+    # @param [Pathname] filepath of file to record.
+    # @return [Hash] of recorded action.
+    def copy(digest, filepath)
+      if @my_actions['copy'].key?(digest) == false
+        @my_actions['copy'][digest] = []
+      end
+      # Only put unique values into filepaths
+      if @my_actions['copy'][digest].include?(filepath)
+        return @my_actions['copy'][digest]
+      else
+        @my_actions['copy'][digest] = (@my_actions['copy'][digest] << filepath)
+      end
+    end
+
+    # Creates a 'move' entry in the actions hash.
+    # @param [String] digest of the filepath being recorded.
+    # @param [Pathname] filepath of file to record.
+    # @return [Hash] of recorded action.
+    def move(digest, filepath)
+      if @my_actions['move'].key?(digest) == false
+        @my_actions['move'][digest] = []
+      end
+      # Only put unique values into filepaths
+      if @my_actions['move'][digest].include?(filepath)
+        return @my_actions['move'][digest]
+      else
+        @my_actions['move'][digest] = (@my_actions['move'][digest] << filepath)
+      end
+    end
+
+    # Creates a 'delete' entry in the actions hash.
+    # @param [String] digest of the filepath being recorded.
+    # @param [Pathname] filepath of file to record.
+    # @return [Hash] of recorded action.
+    def delete(digest, filepath)
+      if @my_actions['delete'].key?(digest) == false
+        @my_actions['delete'][digest] = []
+      end
+      # Only put unique values into filepaths
+      if @my_actions['delete'][digest].include?(filepath)
+        return @my_actions['delete'][digest]
+      else
+        @my_actions['delete'][digest] = (@my_actions['delete'][digest] << filepath)
+      end
+    end
+
+    # @param [String] digest of the filepath that is getting additional fixity values.
+    # @param [String] fixity_algorithm of the fixity digest being added (e.g. 'md5', 'sha1').
+    # @param [String] fixity_digest to associate with this digest.
+    # @return [Hash] of recorded fixity block.
+    def fixity(digest, fixity_algorithm, fixity_digest)
+      # Only create this key if used.
+      @my_actions['fixity'] = {} if @my_actions.key?('fixity') == false
+      if @my_actions['fixity'].key?(fixity_algorithm) == false
+        @my_actions['fixity'][fixity_algorithm] = {}
+      end
+      # only add unique fixity digests.
+      if @my_actions['fixity'][fixity_algorithm].include?(digest)
+        return @my_actions['fixity'][fixity_algorithm][digest]
+      else
+        @my_actions['fixity'][fixity_algorithm][digest] = fixity_digest
+      end
+    end
+  end
+end

data/lib/ocfl_tools/ocfl_delta.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+module OcflTools
+  # Given an inventory, show changes from previous versions.
+  # OcflDelta takes in an OCFL Inventory object and creates a delta hash containing
+  # the actions performed to assemble the requested version.
+  class OcflDelta
+    attr_reader :delta
+
+    def initialize(ocfl_object)
+      # Duck sanity check.
+      ['@id', '@head', '@manifest', '@versions', '@fixity'].each do |var|
+        unless ocfl_object.instance_variable_defined?(var)
+          raise "Object #{ocfl_object} does not have instance var #{var} defined"
+        end
+      end
+
+      %w[get_state version_id_list get_digest].each do |mthd|
+        unless ocfl_object.respond_to?(mthd)
+          raise "Object #{ocfl_object} does not respond to #{mthd}"
+        end
+      end
+
+      @ocfl_object = ocfl_object
+      @delta = {}
+      # We need to get version format, for final report-out. Assume that the ocfl_object versions are
+      # formatted correctly (starting with a 'v'). We can't trust the site config setting
+      # for this, as there's no guarantee the inventory we are reading in was created at this site.
+      first_version = @ocfl_object.versions.keys.min # should get us 'v0001' or 'v1'
+      sliced_version = first_version.split('v')[1] # cut the leading 'v' from the string.
+      if sliced_version.length == 1 # A length of 1 for the first version implies 'v1'
+        @version_format = 'v%d'
+      else
+        @version_format = "v%0#{sliced_version.length}d"
+      end
+    end
+
+    # Generates a complete delta hash for all versions of this object.
+    def all
+      @ocfl_object.version_id_list.each do |version|
+        get_version_delta(version)
+      end
+      @delta
+    end
+
+    # Given a version, get the delta from the previous version.
+    # @param [Integer] version of object to get deltas for.
+    # @return [Hash] of actions applied to previous version to create current version.
+    def previous(version)
+      # San check, does version exist in object?
+      if version == 1
+        get_first_version_delta
+      else
+        # verify version exists, then...
+        unless @ocfl_object.version_id_list.include?(version)
+          raise "Version #{version} not found in #{@ocfl_object}!"
+        end
+        get_version_delta(version)
+      end
+    end
+
+    private
+
+    def get_version_delta(version)
+
+      unless version > 1
+        return get_first_version_delta
+      end
+
+      current_digests = @ocfl_object.get_state(version)
+      current_files = OcflTools::Utils::Files.invert_and_expand(current_digests)
+
+      previous_digests = @ocfl_object.get_state((version - 1))
+      previous_files = OcflTools::Utils::Files.invert_and_expand(previous_digests)
+
+      missing_digests = {}
+      missing_files = {}
+
+      new_digests = {}
+      new_files = {}
+
+      unchanged_digests = {}  # digests may not have changed, but filepaths can!
+      unchanged_files = {}    # filepaths may not change, but digests can!
+
+      version_string = @version_format % version.to_i
+      @delta[version_string] = {}
+      @delta[version_string].clear # Always clear out the existing version delta.
+      actions = OcflTools::OcflActions.new
+
+      temp_digests = previous_digests.keys - current_digests.keys
+      unless temp_digests.empty?
+        temp_digests.each do |digest|
+          missing_digests[digest] = previous_digests[digest]
+        end
+      end
+
+      temp_files = previous_files.keys - current_files.keys
+      unless temp_files.empty?
+        temp_files.each do |file|
+          missing_files[file] = previous_files[file]
+        end
+      end
+
+      temp_digests = current_digests.keys - previous_digests.keys
+      unless temp_digests.empty?
+
+        temp_digests.each do |digest|
+          new_digests[digest] = current_digests[digest]
+        end
+      end
+
+      temp_files = current_files.keys - previous_files.keys
+      unless temp_files.empty?
+
+        temp_files.each do |file|
+          new_files[file] = current_files[file]
+        end
+      end
+
+      temp_digests = current_digests.keys - (new_digests.keys + missing_digests.keys)
+      unless temp_digests.empty?
+        temp_digests.each do |digest|
+          unchanged_digests[digest] = current_digests[digest]
+        end
+      end
+
+      temp_files = current_files.keys - (new_files.keys + missing_files.keys)
+      unless temp_files.empty?
+        temp_files.each do |file|
+          unchanged_files[file] = current_files[file]
+        end
+      end
+
+      # 1. ADD is new digest, new filepath.
+      # consult new_digests and new_files
+      unless new_digests.empty?
+        new_digests.each do |digest, filepaths|
+          # If new_files, check for ADD.
+          filepaths.each do |file|
+            if new_files.key?(file)
+              # new digest, new file, it's an ADD!
+              if new_files[file] == digest
+                actions.add(digest, file)
+                update_manifest_action(digest, version, actions)
+                next # need this so we don't also count it as an UPDATE
+              end
+            end
+
+            # 2. UPDATE is new digest, existing filepath
+            # if new_files doesn't have it, check current_files
+            if current_files.key?(file)
+              # New digest, existing file
+              if current_files[file] == digest
+                actions.update(digest, file)
+                update_manifest_action(digest, version, actions)
+              end
+            end
+          end
+        end
+      end
+
+      # 3. COPY is unchanged digest, additional (new) filepath
+      unless unchanged_digests.empty?
+        unchanged_digests.each do |digest, filepaths|
+          # get previous version filepaths, compare to current version filepaths.
+          if filepaths.size > previous_digests[digest].size
+            # Take current array from previous array
+            # What *new* filepaths do we have for this digest in this version?
+            copied_files = filepaths - previous_digests[digest]
+            copied_files.each do |copy_file|
+              actions.copy(digest, copy_file)
+            end
+          end
+
+          # 4. MOVE is unchanged digest, 1 deleted filepath, 1 added filepath.
+          if filepaths.size == previous_digests[digest].size
+            # For it to be a move, this digest must be listed in missing_files AND new_files.
+            if missing_files.value?(digest) && new_files.value?(digest)
+              # look this up in previous_files.
+              old_filename = previous_digests[digest][0]
+              new_filename = current_digests[digest][0]
+              actions.move(digest, old_filename)
+              actions.move(digest, new_filename)
+            end
+          end
+
+          # 5. One possible DELETE is unchanged digest, fewer filepaths.
+          if filepaths.size < previous_digests[digest].size
+
+            # Am I in missing_files ?
+            previous_filepaths = previous_digests[digest]
+            deleted_filepaths = previous_filepaths - filepaths
+            if deleted_filepaths.empty?
+              deleted_filepaths.each do |delete_me|
+                actions.delete(digest, delete_me)
+              end
+            end
+          end
+        end
+      end
+
+      # 6. DELETE of last filepath is where there's a missing_digest && the filepath is gone too.
+      unless missing_digests.empty?
+        missing_digests.each do |digest, filepaths|
+          # For each missing digest, see if any of its filepaths are still referenced in current files.
+          filepaths.each do |filepath|
+            actions.delete(digest, filepath) unless current_files.key?(filepath)
+          end
+        end
+      end
+
+      @delta[version_string] = actions.all
+    end
+
+    def update_manifest_action(digest, version, action)
+      version_string = @version_format % version.to_i
+      # We need to make a deep copy here so content_paths edits don't screw up the ocfl_object's manifest.
+      content_paths  = OcflTools::Utils.deep_copy(@ocfl_object.manifest[digest])
+      # Find any content_path that starts with the current version's directory & contentDirectory;
+      # these are bitstreams that were added to this version directory.
+      content_paths.each do |content_path|
+        if content_path =~ /^#{version_string}\/#{@ocfl_object.contentDirectory}/
+          # Now trim from front of content_path.
+          content_path.slice!("#{version_string}/#{@ocfl_object.contentDirectory}/")
+          action.update_manifest(digest, content_path)
+        end
+      end
+    end
+
+    def get_first_version_delta
+      # Everything in get_state is an 'add'
+      version = 1
+      actions = OcflTools::OcflActions.new
+
+      version_string = @version_format % version.to_i
+      @delta[version_string] = {} # Always clear out the existing version delta.
+      @delta[version_string].clear
+
+      current_digests = @ocfl_object.get_state(version)
+      current_digests.each do |digest, filepaths|
+        filepaths.each do |file|
+          actions.add(digest, file)
+          update_manifest_action(digest, version, actions)
+        end
+      end
+      @delta[version_string] = actions.all
+      # Everything in Fixity is also an 'add'
+    end
+  end
+end