file_pipeline 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fe1107c14a49c1372b1d3f62c76db36b262311113d3661b4be3ac3b7275f97b8
+  data.tar.gz: f01ab4be7e0eb246a7da41fa734a2e88491dc90ce01033b3b7a125c012029525
+SHA512:
+  metadata.gz: bc53dff7ae7c5a9955f72e83ee2be935f59db398093b555531b5bdbd619faf4637d6bf5effa1842fcf8b3906a90c1626394b892aa6fd24042063b7e78c2f86fb
+  data.tar.gz: 14f493a7bf76fc55c4e60df8d2f999ae66b5770a064c3cbf03b46e52b8a03602629e399cfb6d395109cbf87d89f7dfc459d3f9911c7b1bdc959fc8b9e49c7881

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Martin Stein
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/lib/file_pipeline/errors/failed_modification_error.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module Errors
+    # Error class for exceptions that are raised when a a FileOperation in a
+    # Pipeline returns failure.
+    class FailedModificationError < StandardError
+      # The file opration that caused the error.
+      attr_reader :info
+
+      def initialize(msg = nil, info: nil)
+        @info = info
+        if info.respond_to?(:operation) && info.respond_to?(:log)
+          msg ||= "#{@info.operation&.name} with options"\
+                  " #{@info.operation&.options} failed, log: #{@info.log}"
+        else
+          msg ||= 'Operation failed' unless info
+        end
+        super msg
+      end
+    end
+  end
+end

data/lib/file_pipeline/errors/missing_version_file_error.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module Errors
+    # Error class for exceptions that are raised when a new version is added,
+    # but no actual file is associated with it.
+    class MissingVersionFileError < StandardError
+      # The file that could not be found.
+      attr_reader :file
+
+      def initialize(msg = nil, file: nil)
+        @file = file
+        msg ||= "File missing for version '#{@file}'"
+        super msg
+      end
+    end
+  end
+end

data/lib/file_pipeline/errors/source_directory_error.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module Errors
+    # Error class for exceptions that are raised when a specified source
+    # directory does not exist (or is not a directory).
+    class SourceDirectoryError < StandardError
+      # The directory that could not be found.
+      attr_reader :directory
+
+      def initialize(msg = nil, dir: nil)
+        @directory = dir
+        msg ||= "The source directory #{@directory} does not exist"
+        super msg
+      end
+    end
+  end
+end

data/lib/file_pipeline/errors/source_file_error.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module Errors
+    # Error class for exceptions that are raised when a specified source
+    # directory does not exist (or is not a directory).
+    class SourceFileError < StandardError
+      # The source file that could not be located.
+      attr_reader :file
+
+      # The directories for source files that were registered with FilePipeline
+      # and searched at the time the error was raises.
+      attr_reader :directories
+
+      def initialize(msg = nil, file: nil, directories: nil)
+        @file = file
+        @directories = directories
+        default_msg = "The source file #{@file} was not found. Searched in:\n"
+        msg ||= @directories.inject(default_msg) do |str, dir|
+          str + "\t- #{dir}\n"
+        end
+        super msg
+      end
+    end
+  end
+end

data/lib/file_pipeline/errors.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative 'errors/failed_modification_error'
+require_relative 'errors/missing_version_file_error'
+require_relative 'errors/source_directory_error'
+require_relative 'errors/source_file_error'
+
+module FilePipeline
+  # Contains error classes for FilePipeline.
+  module Errors
+  end
+end

data/lib/file_pipeline/file_operations/captured_data_tags.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # Module that contains constants used as tags for the kinds of data captured
+    # by file operations.
+    module CapturedDataTags
+      # Tag for operations that do not return data
+      NO_DATA = :no_data
+
+      # Tag for operations that return _Exif_ metadata that has not been
+      # preserved (by accident or intention) in a file.
+      DROPPED_EXIF_DATA = :dropped_exif_data
+    end
+  end
+end

data/lib/file_pipeline/file_operations/default_operations/exif_redaction.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # A FileOperation that will redact (delete) unwanted _Exif_ tags from a
+    # file's metadata.
+    #
+    # This could be tags containing sensitive data, such as e.g. _GPS_ location
+    # data.
+    #
+    # *Caveat:* if this operation is applied to a file together with
+    # ExifRestoration, it should be applied _after_ the latter, to avoid
+    # redacted tags being restored.
+    class ExifRedaction < FileOperation
+      include ExifManipulable
+
+      # :args: options
+      #
+      # Returns a new instance.
+      #
+      # ===== Options
+      #
+      # * <tt>redact_tags</tt> - _Exif_ tags to be deleted.
+      #
+      def initialize(**opts)
+        defaults = { redact_tags: [] }
+        super(opts, defaults)
+      end
+
+      # Returns the DROPPED_EXIF_DATA tag defined in CapturedDataTags.
+      #
+      # This operation will capture all _Exif_ tags and their values are
+      # declared in #options <tt>redact_tags</tt> that are redacted from the
+      # file created by the operation.
+      def captured_data_tag
+        CapturedDataTags::DROPPED_EXIF_DATA
+      end
+
+      # :args: src_file, out_file
+      #
+      # Writes a new version of <tt>src_file</tt> to <tt>out_file</tt> with all
+      # _Exif_ tags provided in the +redact_tags+ option deleted.
+      #
+      # Will return all deleted _Exif_ tags and their values as data.
+      def operation(*args)
+        src_file, out_file = args
+        FileUtils.cp src_file, out_file
+        delete_tags out_file, options[:redact_tags]
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations/default_operations/exif_restoration.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # A FileOperation that compares Exif Metadata in two files and copies tags
+    # missing in one from the other. Used to restore Exif tags that were not
+    # preserved during e.g. a file conversion.
+    #
+    # *Caveat:* if this operation is applied to a file together with
+    # ExifRedaction, it should be applied _before_ the latter, to avoid
+    # redacted tags being restored.
+    class ExifRestoration < FileOperation
+      include ExifManipulable
+
+      # :args: options
+      #
+      # Returns a new instance.
+      #
+      # ===== Options
+      #
+      # * <tt>skip_tags</tt> - _Exif_ tags to be ignored during restoration.
+      #
+      # The ExifManipulable mixin defines a set of _Exif_ tags that will always
+      # be ignored. These are tags relating to the file properties (e.g.
+      # filesize, MIME-type) that will have been altered by any prior operation,
+      # such as file format conversions.
+      def initialize(**opts)
+        defaults = { skip_tags: [] }
+        super(opts, defaults)
+        @options[:skip_tags] += ExifManipulable.file_tags
+      end
+
+      # Returns the DROPPED_EXIF_DATA tag defined in CapturedDataTags.
+      #
+      # This operation will capture any _Exif_ tags and their values that could
+      # not be written to the file created by the operation.
+      def captured_data_tag
+        CapturedDataTags::DROPPED_EXIF_DATA
+      end
+
+      # :args: src_file, out_file
+      #
+      # Writes a new version of <tt>src_file</tt> to <tt>out_file</tt> with all
+      # writable _Exif_ tags from +original+ restored.
+      #
+      # Will return any _Exif_ tags that could not be written and their values
+      # from the +original+ file as data.
+      def operation(src_file, out_file, original)
+        original_exif, src_file_exif = read_exif original, src_file
+        values = missing_exif_fields(src_file_exif, original_exif)
+        FileUtils.cp src_file, out_file
+        write_exif out_file, values
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations/default_operations/ptiff_conversion.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # Saves a file to a <em>tiled multi-resolution TIFF</em> ('pyramid'), as
+    # required by e.g. the IIP image server.
+    #
+    # See https://iipimage.sourceforge.io/documentation/images/ or
+    # https://www.loc.gov/preservation/digital/formats/fdd/fdd000237.shtml
+    # for more information on the format.
+    class PtiffConversion < FileOperation
+      # :args: options
+      #
+      # Returns a new instance.
+      #
+      # ===== Options
+      #
+      # * +:tile+ - Writes a tiled _TIFF_ (_default_ +true+)
+      # * +:tile_width+: Tile width in pixels (_default_ +256+)
+      # * +:tile_height+: Tile height in pixels (_default_ +256+)
+      def initialize(**opts)
+        defaults = {
+          tile: true,
+          tile_width: 256,
+          tile_height: 256
+        }
+        super(opts, defaults)
+        @options[:pyramid] = true
+      end
+
+      # :args: src_file, out_file
+      #
+      # Writes a pyramid tiff version of <tt>src_file</tt> to <tt>out_file</tt>.
+      def operation(*args)
+        src_file, out_file = args
+        image = Vips::Image.new_from_file src_file
+        image.tiffsave(out_file, options)
+        # Return lof if any
+      end
+
+      # Returns <tt>'.tiff'</tt> (all files created by #operation will be _TIFF_
+      # files).
+      def target_extension
+        '.tiff'
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations/default_operations/scale.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # Scale instances are FileOperations that will scale an image to a given
+    # resolution.
+    #
+    # ==== Caveats
+    #
+    # This will scale images smaller than the given width and height up.
+    class Scale < FileOperation
+      include Math
+
+      # :args: options
+      #
+      # Returns a new instance.
+      #
+      # ===== Options
+      #
+      # * +:width+ - The target image width in pixels (_default_ 1024).
+      # * +:height+ - The target image height in pixels (_default_ 768).
+      # * +:method+ - A symbol for the method used to calculate the scale:
+      #   factor.
+      #   * +:scale_by_bounds+ (_default_) - see #scale_by_bounds.
+      #   * +:scale_by_pixels+ - See #scale_by_pixels.
+      def initialize(**opts)
+        defaults = {
+          width: 1024,
+          height: 768,
+          method: :scale_by_bounds
+        }
+        super(opts, defaults)
+      end
+
+      # :args: src_file, out_file
+      #
+      # Writes a scaled version of <tt>src_file</tt> to <tt>out_file</tt>.
+      def operation(*args)
+        src_file, out_file = args
+        image = Vips::Image.new_from_file src_file
+        factor = public_send options[:method], image.size
+        image.resize(factor).write_to_file out_file
+      end
+
+      # Calculatees the scale factor to scale +dimensions+ (an array with image
+      # width and height in pixels) so that it will match the same total pixel
+      # count as +:width+ multiplied by +:height+ given in #options.
+      #
+      # *Warning*: rounding errors may occur.
+      #
+      #--
+      # FIXME: avoid rounding errors.
+      #++
+      def scale_by_pixels(dimensions)
+        out_pixels = sqrt(options[:width] * options[:height]).truncate
+        src_pixels = sqrt(dimensions[0] * dimensions[1]).truncate
+        out_pixels / src_pixels.to_f
+      end
+
+      # Calculates the scale factor to scale +dimensions+ (an array with image
+      # width and height in pixels) so that it will fit inside the bounds
+      # defined by +:width+ and +:height+ given in #options.
+      def scale_by_bounds(dimensions)
+        x = options[:width] / dimensions[0].to_f
+        y = options[:height] / dimensions[1].to_f
+        x * dimensions[1] > options[:height] ? y : x
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations/exif_manipulable.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'multi_exiftool'
+
+module FilePipeline
+  module FileOperations
+    # Mixin with methods to facilitate work with _Exif_ metadata.
+    module ExifManipulable
+      # Returns an Array of tags to be ignored during comparison. These can
+      # be merged with an ExifManipulable including FileOperation's options
+      # to skip tags (e.g. the <tt>skip_tags</tt> option in ExifRestoration).
+      def self.file_tags
+        %w[FileSize FileModifyDate FileAccessDate FileInodeChangeDate
+           FilePermissions FileType FileTypeExtension MIMEType]
+      end
+
+      def self.parse_tag_error(message) # :nodoc:
+        /Warning: Sorry, (?<tag>\w+) is not writable/
+          .match(message) { |match| match[:tag] }
+      end
+
+      def self.strip_path(str) # :nodoc:
+        str.sub(%r{ - \/?(\/|[-:.]+|\w+)+\.\w+$}, '')
+      end
+
+      # Redacts (deletes) all <tt>tags_to_delete</tt> in <tt>out_file</tt>.
+      def delete_tags(out_file, tags_to_delete)
+        exif, = read_exif out_file
+        values = exif.select { |tag| tags_to_delete.include? tag }
+        values_to_delete = values.transform_values { nil }
+        log, = write_exif out_file, values_to_delete
+        [log, values]
+      end
+
+      # Compares to hashes with exif tags and values and returns a hash with
+      # the tags that are present in <tt>other_exif</tt> but absent in
+      # <tt>this_exif</tt>.
+      def missing_exif_fields(this_exif, other_exif)
+        other_exif.delete_if do |tag, _|
+          this_exif.key?(tag) || options[:skip_tags].include?(tag)
+        end
+      end
+
+      # :args: error_messages, exif
+      #
+      # Takes an array of <tt>error_messages</tt> and a hash (+exif+) with tags
+      # and their values and parses errors where tags could not be written.
+      #
+      # Returns an array with a log (any messages that were not errors where a
+      # tag could not be written) and data (a hash with any tags that could not
+      # be written, and the associated values from +exif+)
+      def parse_exif_errors(errs, values)
+        errs.each_with_object(LogDataParser.template) do |message, info|
+          errors, data = info
+          tag = ExifManipulable.parse_tag_error(message)
+          if tag
+            data.store tag, values[tag]
+            next info
+          end
+          errors << ExifManipulable.strip_path(message)
+          info
+        end
+      end
+
+      # Reads exif information for one or more +files+. Returns an array of
+      # hashes, one for each file, with tags and their values.
+      def read_exif(*files)
+        file_paths = files.map { |f| File.expand_path(f) }
+        results, errors = MultiExiftool.read file_paths
+        raise 'Error reading Exif' unless errors.empty?
+
+        results.map(&:to_h)
+      end
+
+      # Writes +values+ (a hash with exif tags as keys) to +out_file+.
+      #
+      # Returns an array with a log (an array of messages - strings) and a
+      # hash with all tags/values that could not be written.
+      def write_exif(out_file, values)
+        writer = MultiExiftool::Writer.new
+        writer.filenames = Dir[File.expand_path(out_file)]
+        writer.overwrite_original = true
+        writer.values = values
+        return if writer.write
+
+        parse_exif_errors(writer.errors, values)
+      end
+    end
+  end
+end

data/lib/file_pipeline/file_operations/file_operation.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module FilePipeline
+  module FileOperations
+    # This is an abstract class to be subclassed when file operations are
+    # implemented.
+    #
+    # For the autoloading mechanism in FilePipeline.load to work, it is required
+    # that subclasses are in defined in the module FilePipeline::FileOperations.
+    #
+    # The constructor (_#initialze_) must accept a doublesplat argument as the
+    # only argument and should call +super+.
+    #
+    # Subclasses must implement an #operation method or override the #run
+    # method.
+    #
+    # If the operation results in file type different than that of the file
+    # that is passed to #run or #operation as <tt>src_file</tt>, the subclass
+    # must have a #target_extension method that returns the appropriate
+    # extension.
+    class FileOperation
+      # A Hash; any options used when performing #operation.
+      attr_reader :options
+
+      # Returns a new instance and sets #options.
+      #
+      # This can be called from subclasses.
+      #
+      # ===== Arguments
+      #
+      # * +defaults+ - Default options for the subclass (hash).
+      # * +opts+ - Options passed to the sublass initializer (hash).
+      def initialize(opts, defaults = {})
+        @options = defaults.update(opts)
+      end
+
+      # Returns the NO_DATA tag.
+      #
+      # If the results returned by a subclass contain data, override this methos
+      # to return the appropriate tag for the data. This tag can be used to
+      # filter data captured by operations.
+      #
+      # Tags are defined in CapturedDataTags.
+      def captured_data_tag
+        CapturedDataTags::NO_DATA
+      end
+
+      # Returns the extension for +file+ (a string). This should be the
+      # extension for the type the file created by #operation will have.
+      #
+      # If the #operation of a subclass will result in a different extension of
+      # predictable type, define a #target_extension method.
+      def extension(file)
+        target_extension || File.extname(file)
+      end
+
+      # Returns a Results object with the Results#success set to +false+ and
+      # any information returned by the operation in <tt>log_data</tt> (a string
+      # error, array, or hash).
+      def failure(log_data = nil)
+        results false, log_data
+      end
+
+      # Returns the class name (string) of +self+ _without_ the names of the
+      # modules that the class is nested in.
+      def name
+        self.class.name.split('::').last
+      end
+
+      # :args: src_file, out_file, original = nil
+      #
+      # To be implemented in subclasses. Should return any logged errors or data
+      # produced (a string, error, array, or hash) or +nil+.
+      #
+      # ===== Arguments
+      #
+      # * <tt>src_file</tt> - Path for the file the operation will use as the
+      #   basis for the new version it will create.
+      # * <tt>out_file</tt> - Path the file created by the operation will be
+      #   written to.
+      # * +original+ - Path to the original, unmodified, file (optional).
+      def operation(*_)
+        raise 'not implemented'
+      end
+
+      # Returns a new Results object with the #descrip1tion of +self+,
+      # +success+, and any information returned by the operation as
+      # <tt>log_data</tt> (a string, error, array, or hash.)
+      #
+      # ===== Examples
+      #
+      #   error = StandardError.new
+      #   warning = 'a warning occurred'
+      #   log = [error, warning]
+      #   data = { mime_type: 'image/jpeg' }
+      #
+      #   my_op = MyOperation.new
+      #
+      #   my_op.results(false, error)
+      #   # => <Results @data=nil, @log=[error], ..., @success=false>
+      #
+      #   my_op.results(true, warning)
+      #   # => <Results @data=nil, @log=[warning], ..., @success=true>
+      #
+      #   my_op.results(true, data)
+      #   # => <Results @data=data, @log=[], ..., @success=true>
+      #
+      #   my_op.results(true, [warning, data])
+      #   # => <Results @data=data, @log=[warning], ..., @success=true>
+      #
+      #   my_op.results(false, log)
+      #   # => <Results @data=nil, @log=[error, warning], ..., @success=false>
+      #
+      #   my_op.results(false, [log, data])
+      #   # => <Results @data=data, @log=[error, warning], ..., @success=false>
+      #
+      def results(success, log_data = nil)
+        Results.new(self, success, log_data)
+      end
+
+      # Runs the operation on <tt>src_file</tt> and retunes an array with a
+      # path for the file created by the operation and a Results object.
+      #
+      # Subclasses of FileOperation must either implement an #operation method,
+      # or override the #run method, making sure it has the same signature and
+      # kind of return value.
+      #
+      # The method will create a new path for the file produced by #operation to
+      # be written to. This path will consist of +directory+ and a new basename.
+      #
+      # The optional +original+ argument can be used to reference another file,
+      # e.g. when exif metadata tags missing in the <tt>src_file</tt> are to
+      # be copied over from another file.
+      def run(src_file, directory, original = nil)
+        out_file = target directory, extension(src_file)
+        log_data = operation src_file, out_file, original
+        [out_file, success(log_data)]
+      rescue StandardError => e
+        FileUtils.rm out_file if File.exist? out_file
+        [out_file, failure(e)]
+      end
+
+      # Returns a Results object with the Results#success set to +true+ and
+      # any information returned by the operation in <tt>log_data</tt> (a string
+      # error, array, or hash).
+      def success(log_data = nil)
+        results true, log_data
+      end
+
+      # Returns a new path to which the file created by the operation can be
+      # written. The path will be in +directory+, with a new basename determined
+      # by +kind+ and have the specified file +extension+.
+      #
+      # There are two options for the +kind+ of basename to be created:
+      # * +:timestamp+ (_default_) - Creates a timestamp basename.
+      # * +:random+ - Creates a UUID basename.
+      #
+      # The timestamp format is <tt>YYYY-MM-DDTHH:MM:SS.NNNNNNNNN</TT>.
+      #
+      # ===== Examples
+      #
+      #   file_operation.target('path/to/dir', '.png', :timestamp)
+      #   # => 'path/to/dir/2019-07-24T09:30:12:638935761.png'
+      #
+      #   file_operation.target('path/to/dir', '.png', :random)
+      #   # => 'path/to/dir/123e4567-e89b-12d3-a456-426655440000.png'
+      def target(directory, extension, kind = :timestamp)
+        filename = FilePipeline.new_basename(kind) + extension
+        File.join directory, filename
+      end
+
+      # Returns +nil+.
+      #
+      # If the #operation of a subclass will result in a different extension of
+      # predictable type, override this method to return the appropriate type.
+      #
+      # If, for instance, the operation will always create a <em>TIFF</em> file,
+      # the implementation could be:
+      #
+      #   # Returns '.tiff'
+      #   def target_extension
+      #     '.tiff'
+      #   end
+      #
+      def target_extension; end
+    end
+  end
+end