file_pipeline 0.0.1

data/lib/file_pipeline/file_operations/log_data_parser.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # This class parses an object that may be a hash, array other object or
+    # +nil+.
+    #
+    # If it is initialized with an array, that array may contain another array,
+    # a hash, any objects, or +nil+
+    #
+    # The resulting instance will behave like an array and always have two
+    # elements:
+    # * +nil+ or an array containing all arguments that are not a hash at index
+    #   0
+    # * +nil+ or a hash at index 1.
+    #
+    # ===== Examples
+    #
+    # When passed +nil+:
+    #
+    #   LogDataParser.new(nil).to_a
+    #   # => [nil, nil]
+    #
+    # When initialized with individual strings or errors, those will be wrapped
+    # in an array:
+    #
+    #   LogDataParser.new(StandardError.new).to_a
+    #   # => [[#<StandardError: StandardError>], nil]
+    #
+    #   LogDataParser.new('a warning').to_a
+    #   # => [['a warning'], nil]
+    #
+    # This is also true when initialized with individual messages or errors
+    # along with data:
+    #
+    #   LogDataParser.new(['a warning', { a_key: 'some value' }]).to_a
+    #   # => [['a warning'], { a_key: 'some value' }]
+    #
+    #   LogDataParser.new(['a warning', { a_key: 'some value' }, 'error']).to_a
+    #   # => [['a warning', 'error'], { a_key: 'some value' }]
+    #
+    # When initialized with a hash, the array will be +nil+ and the hash:
+    #
+    #   LogDataParser.new(['a warning', { a_key: 'some value' }]).to_a
+    #   # => [nil, { a_key: 'some value' }]
+    #
+    # When initialized with an arry that does contain neither arrays nor hashes,
+    # it will become the first element of the resulting array, with second being
+    # +nil+.
+    #
+    #   LogDataParser.new(['a warning', StandardError.new]).to_a
+    #   # => [['a warning', #<StandardError: StandardError>], nil]
+    #
+    # When initialized with an array containing an array and a hash, the inner
+    # array is will be the first element, the hash the second
+    #
+    #   log = ['a warning', 'another warning']
+    #   data = { a_key: 'some value' }
+    #
+    #   LogDataParser.new([log, data]).to_a
+    #   # => [['a warning', 'another warning'], { a_key: 'some value' }]
+    #
+    #   LogDataParser.new([data, log])
+    #   # => [['a warning', 'another warning'], { a_key: 'some value' }]
+    #
+    # When initialized with an array containing a hash and nil
+    #
+    #   LogDataParser.new([nil, data]).to_a
+    #   # => [nil, { a_key: 'some value' }]
+    #
+    class LogDataParser
+      # :args: object
+      #
+      # Returns a new instance for +object+, which may be +nil+, a hash, another
+      # object, or an array, that may itself contain a hash, an array, or other
+      # objects.
+      def initialize(obj)
+        @log_data = nil
+        parse obj
+        normalize
+      end
+
+      # Returns a trwo element array with an empty array and a hash.
+      def self.template
+        [[], {}]
+      end
+
+      private
+
+      def method_missing(method_name, *args, &block)
+        super unless respond_to_missing? method_name.to_sym
+
+        @log_data.public_send method_name, *args, &block
+      end
+
+      def normalize
+        return unless @log_data[0].is_a? Array
+
+        @log_data[0].compact!
+        @log_data[0] = nil if @log_data[0].empty?
+      end
+
+      def parse(obj)
+        @log_data = case obj
+                    when Array
+                      parse_array obj
+                    when Hash
+                      [nil, obj]
+                    when nil
+                      [nil, nil]
+                    else
+                      [[obj], nil]
+                    end
+      end
+
+      def parse_array(obj)
+        return [obj, nil] if obj.none? { |e| e.respond_to? :each }
+
+        parse_nested obj
+      end
+
+      def parse_nested(obj)
+        obj.each_with_object([]) do |element, ld|
+          case element
+          when Array
+            ld[0] = element
+          when Hash
+            ld[1] = element
+          else
+            (ld[0] ||= []) << element
+          end
+        end
+      end
+
+      def respond_to_missing?(method_name, include_private = false)
+        @log_data.respond_to?(method_name.to_sym) || super
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations/results.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # This class contains the results from a FileOperation being run on a file.
+    # Instances will be returned by the FileOperation#run method.
+    #
+    # Instances contain the file operation opbject that has produced +self+,
+    # a flag for success, and any logs and data the operation may return.
+    class Results
+      # The object (usually an instance of a subclass of FileOperation) that
+      # created +self+
+      attr_reader :operation
+
+      # +true+ if the operation has finished and produced a version file,
+      # or +false+ if it encountered an error that caused it to terminate.
+      attr_reader :success
+
+      # Array with log messages from operations.
+      attr_reader :log
+
+      # Hash with any data returned from an operation.
+      attr_reader :data
+
+      # Returns a new instance.
+      #
+      # ===== Arguments
+      #
+      # * +operation+ - Must respond to +:name+ and +:options+.
+      # * +success+ - +true+ or +false+.
+      # * +log_data+ - A string, error, array, hash, or +nil+.
+      #
+      # ===== Examples
+      #
+      #   error = StandardError.new
+      #   warning = 'a warning occurred'
+      #   log = [error, warning]
+      #   data = { mime_type: 'image/jpeg' }
+      #
+      #   my_op = MyOperation.new
+      #
+      #   Results.new(my_op, false, error)
+      #   # => <Results @data=nil, @log=[error], ..., @success=false>
+      #
+      #   Results.new(my_op, true, warning)
+      #   # => <Results @data=nil, @log=[warning], ..., @success=true>
+      #
+      #   Results.new(my_op, true, data)
+      #   # => <Results @data=data, @log=[], ..., @success=true>
+      #
+      #   Results.new(my_op, true, [warning, data])
+      #   # => <Results @data=data, @log=[warning], ..., @success=true>
+      #
+      #   Results.new(my_op, false, log)
+      #   # => <Results @data=nil, @log=[error, warning], ..., @success=false>
+      #
+      #   Results.new(my_op, false, [log, data])
+      #   # => <Results @data=data, @log=[error, warning], ..., @success=false>
+      #
+      #   Results.new(my_op, false, nil)
+      #   # => <Results @data=nil, @log=nil, ..., @success=false>
+      #
+      def initialize(operation, success, log_data)
+        @operation = operation
+        @success = success
+        @log, @data = LogDataParser.new log_data
+      end
+
+      def self.return_data(obj) # :nodoc:
+        return [nil, obj] if obj.is_a? Hash
+      end
+
+      def self.return_log(obj) # :nodoc:
+        flat_array = obj.is_a?(Array) &&
+                     obj.none? { |i| i.is_a?(Array) || i.is_a?(Hash) }
+        return unless flat_array
+
+        [obj]
+      end
+
+      def self.return_log_and_data(obj) # :nodoc:
+        log = obj.find { |i| !i.is_a? Hash }
+        log = [log] unless log.is_a? Array
+        data = obj.find { |i| i.is_a? Hash }
+        [log, data]
+      end
+
+      def self.return_log_message(obj) # :nodoc:
+        return if obj.is_a?(Array) || obj.is_a?(Hash)
+
+        [[obj]]
+      end
+
+      def self.normalize_log_data(obj)
+        return unless obj
+
+        Results.return_data(obj) ||
+          Results.return_log_message(obj) ||
+          Results.return_log(obj) ||
+          Results.return_log_and_data(obj)
+      end
+
+      # Returns +true+ if the operation was not succesful, +false+ otherwise.
+      def failure
+        !success
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'ruby-vips'
+
+require_relative 'file_operations/captured_data_tags'
+require_relative 'file_operations/exif_manipulable'
+require_relative 'file_operations/file_operation'
+require_relative 'file_operations/log_data_parser'
+require_relative 'file_operations/results'
+
+module FilePipeline
+  # Module that contains FileOperation and subclasses thereof that contain the
+  # logic to perform file modifications, as well as associated classes, for
+  # passing on information that was produced during a file operation.
+  module FileOperations
+  end
+end

data/lib/file_pipeline/pipeline.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  # Instances of Pipeline hold a defined set of operations that perform
+  # modifications of files.
+  #
+  # The operations are applied to a VersionedFile in the order they are added
+  # to the instance. To implement custom operations, it is easiest to write a
+  # subclass of FileOperations::FileOperation.
+  #
+  # The class can be initialized with an optional block to add file
+  # operations:
+  #
+  #   Pipeline.new do |pipeline|
+  #     pipeline.define_operation('scale',
+  #                               :width => 1280, :height => 1024)
+  #     pipeline.define_operation('ptiff_conversion',
+  #                               :tile_width => 64, :tile_height => 64)
+  #   end
+  #
+  class Pipeline
+    # An array of file operations that will be applied to files in the order
+    # they have been added.
+    attr_reader :file_operations
+
+    # Returns a new instance.
+    #
+    # If <tt>src_directories</tt> are provided, they will be added to
+    # FilePipeline.source_directories.
+    #
+    # ===== Arguments
+    #
+    # * <tt>src_directories</tt> - one or more paths to directories where
+    #   classes for file operations are defined.
+    def initialize(*src_directories)
+      src_directories.each { |dir| FilePipeline << dir }
+      @file_operations = []
+      yield(self) if block_given?
+    end
+
+    # Adds a file operation object #file_operations. The object must implement
+    # a _run_ method (see FileOperations::FileOperation#run for details).
+    def <<(file_operation_instance)
+      unless file_operation_instance.respond_to? :run
+        raise TypeError, 'File operations must implement a #run method'
+      end
+
+      @file_operations << file_operation_instance
+    end
+
+    # Applies all #file_operations to a <tt>versioned_file</tt> and returns it.
+    def apply_to(versioned_file)
+      file_operations.each { |job| run job, versioned_file }
+      versioned_file
+    end
+
+    # Applies all #file_operations to <tt>versioned_files</tt> (an array) and
+    # returns it.
+    def batch_apply(versioned_files)
+      versioned_files.map { |file| Thread.new(file) { apply_to(file) } }
+                     .map(&:value)
+    end
+
+    # Initializes the class for <tt>file_operation</tt> (a string in
+    # underscore notation) with +options+, adds it to #file_operations, and
+    # returns +self+.
+    #
+    # If the source file containing the file operation's class definition is not
+    # loaded, this method will try to locate it in the
+    # FilePipeline.source_directories and require it.
+    #
+    # ===== Examples
+    #
+    # Define single operation:
+    #
+    #   pipeline.define_operation('ptiff_conversion', :tile => false)
+    #
+    # Chaining:
+    #
+    #   pipeline.define_operation('scale', width: 1280, height: 1024)
+    #           .define_operation('ptiff_conversion')
+    #
+    def define_operation(file_operation, options = {})
+      operation = FilePipeline.load file_operation
+      self << operation.new(options)
+      self
+    end
+
+    # Returns +true+ if no #file_operations are defined.
+    def empty?
+      file_operations.empty?
+    end
+
+    # Applies +operation+ to <tt>versioned_file</tt>.
+    #
+    # +operation+ must be an object implementing a _run_ method that takes three
+    # arguments (see FileOperations::FileOperation#run ).
+    def run(operation, versioned_file)
+      versioned_file.modify do |version, directory, original|
+        operation.run version, directory, original
+      end
+    end
+  end
+end

data/lib/file_pipeline/versioned_file.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  # VersionedFile creates a directory where it stores any versions of _file_.
+  class VersionedFile
+    include FileOperations::ExifManipulable
+
+    # The basename of the versioned file.
+    attr_reader :basename
+
+    # A hash with file paths as keys, information on the modifications applied
+    # to create the version as values (instances of FileOperations::Results).
+    attr_reader :history
+
+    # The path to the original file of _self_.
+    attr_reader :original
+
+    # A String that is appended to the file basename when the file written
+    # by #finalize is not replacing the original.
+    attr_reader :target_suffix
+
+    # Returns a new instance with +file+ as the #original.
+    #
+    # ===== Arguments
+    #
+    # * +file+ - Path to the file the instance will be based on. That file
+    #   should not be touched unless #finalize is called with the +:overwrite+
+    #   option set to +true+.
+    #
+    # *Caveat* it can not be ruled out that buggy or malignant file operations
+    # modify the original.
+    #
+    #--
+    # FIXME: protect the original
+    #++
+    #
+    # ===== Options
+    #
+    # <tt>target_suffix</ttm> is a string to be appended to the file that
+    # will be written by #finalize (the last version) if #finalize is to
+    # preserve the original. It is recommended to use a UUID (_default_) to
+    # avoid clashes with other files in the directory.
+    def initialize(file, target_suffix: SecureRandom.uuid)
+      raise Errors::MissingVersionFileError, file: file unless File.exist? file
+
+      @original = file
+      @basename = File.basename(file, '.*')
+      @history = {}
+      @directory = nil
+      @target_suffix = target_suffix
+    end
+
+    # Copies the file with path _src_ to <em>/dir/filename</em>.
+    def self.copy(src, dir, filename)
+      dest = FilePipeline.path(dir, filename)
+      FileUtils.cp src, dest
+      dest
+    end
+
+    # Moves the file with path _src_ to <em>/dir/filename</em>.
+    def self.move(src, dir, filename)
+      dest = FilePipeline.path(dir, filename)
+      FileUtils.mv src, dest
+      dest
+    end
+
+    # Adds a new version to #history and returns _self_.
+    #
+    # <tt>version_info</tt> must be a path to an existing file or an array with
+    # the path and optionally a FileOperations::Results instance:
+    # <tt>['path/to/file', results_object]</tt>.
+    # Will move the file to #directory if it is in another directory.
+    def <<(version_info)
+      file, info = version_info
+      raise Errors::FailedModificationError, info: info if info&.failure
+
+      version = validate(file)
+      @history[version] = info
+      self
+    rescue StandardError => e
+      reset
+      raise e
+    end
+
+    # Returns a two-dimesnional array, where each nested array has two items;
+    # the file operation object and data captured by the operartion (if any).
+    #
+    # <tt>[[description_object, data_or_nil], ...]</tt>
+    def captured_data
+      filter_history :data
+    end
+
+    # Returns any data captured by <tt>operation_name</tt>.
+    #
+    # If multiple instances of one operation class have modified the file,
+    # pass any +options+ the specific instance of the operation was initialized
+    # with as the optional second argument.
+    def captured_data_for(operation_name, **options)
+      raw_data = captured_data.filter do |operation, _|
+        operation.name == operation_name &&
+          options.all? { |k, v| operation.options[k] == v }
+      end
+      raw_data.map(&:last)
+    end
+
+    # Returns an array with all data captured by operations with +tag+ has.
+    #
+    # Tags are defined in FileOperations::CapturedDataTags
+    def captured_data_with(tag)
+      return unless changed?
+
+      captured_data.map do |operation, results|
+        next unless operation.captured_data_tag == tag
+
+        results
+      end
+    end
+
+    # Returns +true+ if there are #versions (file has been modified).
+    #
+    # *Warning:* It will also return +true+ if the file has been cloned.
+    def changed?
+      current != original
+    end
+
+    # Creates a new identical version of #current. Will only add the path of
+    # the file to history, but no FileOperations::Results.
+    def clone
+      filename = FilePipeline.new_basename + current_extension
+      clone_file = VersionedFile.copy(current, directory, filename)
+      self << clone_file
+    end
+
+    # Returns the path to the current file or the #original if no versions
+    # have been created.
+    def current
+      versions.last || original
+    end
+
+    # Returns the file extension for the #current file.
+    def current_extension
+      File.extname current
+    end
+
+    # Returns the path to the directory where the versioned of +self+ are
+    # stored. Creates the directory if it does not exist.
+    def directory
+      @directory ||= workdir
+    end
+
+    # Writes the #current version to #basename, optionally the #target_suffix,
+    # and the #current_extension in #original_dir. Deletes all versions and
+    # resets the #history to an empty Hash. Returns the path to the written
+    # file.
+    #
+    # ===== Options
+    #
+    # * +overwrite+ - +true+ or +false+
+    #   * +false+ (_default_) - The #target_suffix will be appended to the
+    #     #basename and the #original will be preserved.
+    #   * +true+ - The finalized version will replace the #original.
+    def finalize(overwrite: false)
+      filename = overwrite ? replacing_trarget : preserving_taget
+      FileUtils.rm original if overwrite
+      @original = VersionedFile.copy(current, original_dir, filename)
+    ensure
+      reset
+    end
+
+    # Returns an array of triplets (arryas with three items each): the name of
+    # the file operation class (a string), options (a hash), and the actual log
+    # (an array).
+    def log
+      filter_history(:log)
+        .map { |operation, info| [operation.name, operation.options, info] }
+    end
+
+    # Returns the Exif metadata
+    #
+    # ===== Options
+    #
+    # * <tt>:for_version</tt> - +current+ or +original+
+    #   * +current+ (_default_) - Metadata for the #current file will be
+    #     returned.
+    #   * +original+ - Metadata for the #original file will be returned.
+    #
+    #--
+    # TODO: when file is not an image file, this should return other metadata
+    # than exif.
+    # TODO: implement the option to return metadata for a specif version index
+    #++
+    def metadata(for_version: :current)
+      file = public_send for_version
+      read_exif(file).first
+    end
+
+    # Creates a new version.
+    # Requires a block that must return a path to an existing file or an array
+    # with the path and optionally a FileOperations::Results instance:
+    # <tt>['path/to/file', results_object]</tt>.
+    #
+    # The actual file modification logic will be in the block.
+    #
+    # The block must take three arguments: for the #current file (from which the
+    # modified version will be created), the work #directory (to where the
+    # modified file will be written), and the #original file (which will only
+    # be used in modifications that need the original file for reference, such
+    # as modifications that restore file metadata that was lost in other
+    # modifications).
+    def modify
+      self << yield(current, directory, original)
+    end
+
+    # Returns the directory where #original is stored.
+    def original_dir
+      File.dirname original
+    end
+
+    # Returns a hash into which all captured data from file operations with the
+    # FileOperations::CapturedDataTags::DROPPED_EXIF_DATA has been merged.
+    def recovered_metadata
+      captured_data_with(FileOperations::CapturedDataTags::DROPPED_EXIF_DATA)
+        &.reduce({}) { |recovered, data| recovered.merge data }
+    end
+
+    # Returns an array with paths to the version files of +self+ (excluding
+    # #original).
+    def versions
+      history.keys
+    end
+
+    alias touch clone
+
+    private
+
+    # item = :data or :log
+    def filter_history(item)
+      history.inject([]) do |results, (_, info)|
+        next results unless info.respond_to?(item) && info.public_send(item)
+
+        results << [info.operation, info.public_send(item)]
+      end
+    end
+
+    # Returns the filename for a target file that will not overwrite the
+    # original.
+    def preserving_taget
+      basename + '_' + target_suffix + current_extension
+    end
+
+    # Returns the filename for a target file that will overwrite the
+    # original.
+    def replacing_trarget
+      basename + current_extension
+    end
+
+    # Deletes the work directory and resets #versions
+    def reset
+      FileUtils.rm_r directory, force: true
+      @history = {}
+    end
+
+    # Validates if file exists and has been stored in #directory.
+    def validate(file)
+      raise Errors::MissingVersionFileError, file: file unless File.exist? file
+
+      return file if File.dirname(file) == directory
+
+      VersionedFile.move file, directory, File.basename(file)
+    end
+
+    # Creates the directory containing all version files. Directory name is
+    # composed of the basename plus '_version'.
+    #
+    # Raises SystemCallError if the directory already exists.
+    def workdir
+      subdir = basename + '_versions'
+      filedir = File.dirname(original)
+      dirname = File.join filedir, subdir
+      FileUtils.mkdir(dirname)
+      File.path dirname
+    end
+  end
+end