bpl-derivatives 0.2.0

data/lib/bpl/derivatives/logger.rb

@@ -0,0 +1,25 @@
+module BPL::Derivatives
+  class Logger
+    class << self
+      def method_missing(method_name, *arguments, &block)
+        logger.send(method_name, *arguments, &block)
+      rescue StandardError
+        super
+      end
+
+      def respond_to?(method_name, _include_private = false)
+        logger.respond_to? method_name
+      end
+
+      def respond_to_missing?(method_name, _include_private = false)
+        logger.respond_to_missing? method_name
+      end
+
+      private
+
+      def logger
+        BPL::Derivatives.config.base_logger
+      end
+    end
+  end
+end

data/lib/bpl/derivatives/output_object_decorator.rb

@@ -0,0 +1,12 @@
+require 'delegate'
+
+module BPL::Derivatives
+  class OutputObjectDecorator < SimpleDelegator
+    attr_accessor :content, :original_object
+    def initialize(content, original_object = nil)
+      super(content)
+      self.content = StringIO.new(content)
+      self.original_object = original_object
+    end
+  end
+end

data/lib/bpl/derivatives/processors.rb

@@ -0,0 +1,18 @@
+module BPL::Derivatives
+  module Processors
+    extend ActiveSupport::Autoload
+
+    eager_autoload do
+      autoload :Processor
+    end
+
+    autoload :Audio
+    autoload :Document
+    autoload :Ffmpeg
+    autoload :Image
+    autoload :Jpeg2kImage
+    autoload :RawImage
+    autoload :ShellBasedProcessor
+    autoload :Video
+  end
+end

data/lib/bpl/derivatives/processors/audio.rb

@@ -0,0 +1,5 @@
+module BPL::Derivatives::Processors
+  class Audio < Processor
+    include Ffmpeg
+  end
+end

data/lib/bpl/derivatives/processors/document.rb

@@ -0,0 +1,45 @@
+module BPL::Derivatives::Processors
+  class Document < Processor
+    include ShellBasedProcessor
+
+    def self.encode(path, format, outdir)
+      execute "#{BPL::Derivatives.config.libreoffice_path} --invisible --headless --convert-to #{format} --outdir #{outdir} #{Shellwords.escape(path)}"
+    end
+
+    # Converts the document to the format specified in the directives hash.
+    # TODO: file_suffix and options are passed from ShellBasedProcessor.process but are not needed.
+    #       A refactor could simplify this.
+    def encode_file(_file_suffix, _options = {})
+      convert_to_format
+    ensure
+      FileUtils.rm_f(converted_file)
+    end
+
+    private
+
+      # For jpeg files, a pdf is created from the original source and then passed to the Image processor class
+      # so we can get a better conversion with resizing options. Otherwise, the ::encode method is used.
+      def convert_to_format
+        if directives.fetch(:format) == "jpg"
+          object.source_path = converted_file
+          BPL::Derivatives::Processors::Image.new(object, directives).process
+        else
+          finalize_derivative_output(File.read(converted_file))
+        end
+      end
+
+
+      def converted_file
+        @converted_file ||= if directives.fetch(:format) == "jpg"
+                              convert_to("pdf")
+                            else
+                              convert_to(directives.fetch(:format))
+                            end
+      end
+
+      def convert_to(format)
+        self.class.encode(source_path, format, BPL::Derivatives.config.temp_file_base)
+        File.join(BPL::Derivatives.config.temp_file_base, [File.basename(source_path, ".*"), format].join('.'))
+      end
+  end
+end

data/lib/bpl/derivatives/processors/ffmpeg.rb

@@ -0,0 +1,21 @@
+# An abstract class for asyncronous jobs that transcode files using FFMpeg
+module BPL::Derivatives::Processors
+  module Ffmpeg
+    extend ActiveSupport::Concern
+
+    INPUT_OPTIONS = :input_options
+    OUTPUT_OPTIONS = :output_options
+
+    included do
+      include ShellBasedProcessor
+    end
+
+    module ClassMethods
+      def encode(path, options, output_file)
+        inopts = options[INPUT_OPTIONS] ||= "-y"
+        outopts = options[OUTPUT_OPTIONS] ||= ""
+        execute "#{BPL::Derivatives.ffmpeg_path} #{inopts} -i #{Shellwords.escape(path)} #{outopts} #{output_file}"
+      end
+    end
+  end
+end

data/lib/bpl/derivatives/processors/image.rb

@@ -0,0 +1,76 @@
+require 'mini_magick'
+
+module BPL::Derivatives::Processors
+  class Image < Processor
+    cattr_accessor :timeout
+
+    def process
+      timeout ? process_with_timeout : create_resized_image
+    end
+
+    def process_with_timeout
+      Timeout.timeout(timeout) { create_resized_image }
+    rescue Timeout::Error
+      raise BPL::Derivatives::TimeoutError, "Unable to process image derivative\nThe command took longer than #{timeout} seconds to execute"
+    end
+
+    protected
+
+      # When resizing images, it is necessary to flatten any layers, otherwise the background
+      # may be completely black. This happens especially with PDFs. See #110
+      def create_resized_image
+        create_image do |xfrm|
+          if size
+            xfrm.flatten
+            xfrm.resize(size)
+          end
+        end
+      end
+
+      def create_image
+        xfrm = selected_layers(load_image_transformer)
+        yield(xfrm) if block_given?
+        xfrm.format(directives.fetch(:format))
+        xfrm.quality(quality.to_s) if quality
+        xfrm.density(density.to_s) if density
+        write_image(xfrm)
+      end
+
+      def write_image(xfrm)
+        output_io = StringIO.new
+        xfrm.write(output_io)
+        output_io.rewind
+        finalize_derivative_output(output_io.read)
+      end
+
+      # Override this method if you want a different transformer, or need to load the
+      # raw image from a different source (e.g. external file)
+      def load_image_transformer
+        MiniMagick::Image.open(source_path)
+      end
+
+    private
+
+      def size
+        directives.fetch(:size, nil)
+      end
+
+      def quality
+        directives.fetch(:quality, nil)
+      end
+
+      def density
+        directives.fetch(:density, nil)
+      end
+
+      def selected_layers(image)
+        if image.type =~ /pdf/i
+          image.layers[directives.fetch(:layer, 0)]
+        elsif directives.fetch(:layer, false)
+          image.layers[directives.fetch(:layer)]
+        else
+          image
+        end
+      end
+  end
+end

data/lib/bpl/derivatives/processors/jpeg2k_image.rb

@@ -0,0 +1,127 @@
+require 'mini_magick'
+require 'nokogiri'
+
+module BPL::Derivatives::Processors
+  class Jpeg2kImage < Processor
+    include ShellBasedProcessor
+
+    class << self
+      def srgb_profile_path
+        File.join [
+          File.expand_path('../../../../', __FILE__),
+          'color_profiles',
+          'sRGB_IEC61966-2-1_no_black_scaling.icc'
+        ]
+      end
+
+      def kdu_compress_recipe(args, quality, long_dim)
+        if args[:recipe].is_a? Symbol
+          recipe = [args[:recipe].to_s, quality].join('_').to_sym
+          return BPL::Derivatives.config.kdu_compress_recipes[recipe] if BPL::Derivatives.config.kdu_compress_recipes.key? recipe
+          BPL::Derivatives.config.base_logger.warn "No JP2 recipe for :#{args[:recipe]} ('#{recipe}') found in configuration. Using best guess."
+          calculate_recipe(args, quality, long_dim)
+        elsif args[:recipe].is_a? String
+          args[:recipe]
+        else
+          calculate_recipe(args, quality, long_dim)
+        end
+      end
+
+      def calculate_recipe(args, quality, long_dim)
+        levels_arg = args.fetch(:levels, level_count_for_size(long_dim))
+        rates_arg = layer_rates(args.fetch(:layers, 8), args.fetch(:compression, 10))
+        tile_size = args.fetch(:tile_size, 1024)
+        tiles_arg = "#{tile_size},#{tile_size}"
+        jp2_space_arg = quality == 'gray' ? 'sLUM' : 'sRGB'
+
+        %(-rate #{rates_arg}
+            -jp2_space #{jp2_space_arg}
+            -double_buffering 10
+            -num_threads 4
+            -no_weights
+            Clevels=#{levels_arg}
+            "Stiles={#{tiles_arg}}"
+            "Cblk={64,64}"
+            Cuse_sop=yes
+            Cuse_eph=yes
+            Corder=RPCL
+            ORGgen_plt=yes
+            ORGtparts=R  ).gsub(/\s+/, " ").strip
+      end
+
+      def level_count_for_size(long_dim)
+        levels = 0
+        level_size = long_dim
+        while level_size >= 96
+          level_size /= 2
+          levels += 1
+        end
+        levels - 1
+      end
+
+      def layer_rates(layer_count, compression_numerator)
+        # e.g. if compression_numerator = 10 then compression is 10:1
+        rates = []
+        cmp = 24.0 / compression_numerator
+        layer_count.times do
+          rates << cmp
+          cmp = (cmp / 1.618).round(8)
+        end
+        rates.map(&:to_s).join(',')
+      end
+
+      def encode(path, recipe, output_file)
+        kdu_compress = BPL::Derivatives.config.kdu_compress_path
+        execute "#{kdu_compress} -quiet -i #{Shellwords.escape(path)} -o #{output_file} #{recipe}"
+      end
+
+      def tmp_file(ext)
+        Dir::Tmpname.create(['bpl-derivative', ext], BPL::Derivatives.config.temp_file_base) {}
+      end
+
+      def long_dim(image)
+        [image[:width], image[:height]].max
+      end
+    end
+
+    def process
+      image = MiniMagick::Image.open(source_path)
+      quality = image['%[channels]'] == 'gray' ? 'gray' : 'color'
+      long_dim = self.class.long_dim(image)
+      file_path = self.class.tmp_file('.tif')
+      to_srgb = directives.fetch(:to_srgb, true)
+      if directives[:resize] || to_srgb
+        preprocess(image, resize: directives[:resize], to_srgb: to_srgb, src_quality: quality)
+      end
+      image.write file_path
+      recipe = self.class.kdu_compress_recipe(directives, quality, long_dim)
+      encode_file(recipe, file_path: file_path)
+      File.unlink(file_path) unless file_path.nil?
+    end
+
+    def encode_file(recipe, opts = {})
+      output_file = self.class.tmp_file('.jp2')
+      if opts[:file_path]
+        self.class.encode(opts[:file_path], recipe, output_file)
+      else
+        BPL::Derivatives::TempfileService.create(source_file) do |f|
+          self.class.encode(f.path, recipe, output_file)
+        end
+      end
+      finalize_derivative_output(File.open(output_file, "rb", &:read))
+      File.unlink(output_file)
+    end
+
+
+    protected
+
+    def preprocess(image, opts = {})
+      # resize: <geometry>, to_srgb: <bool>, src_quality: 'color'|'gray'
+      image.combine_options do |c|
+        c.resize(opts[:resize]) if opts[:resize]
+        c.profile self.class.srgb_profile_path if opts[:src_quality] == 'color' && opts[:to_srgb]
+      end
+      image
+    end
+  end
+end

data/lib/bpl/derivatives/processors/processor.rb

@@ -0,0 +1,43 @@
+module BPL::Derivatives::Processors
+  # Processors take a single input and produce a single output
+  class Processor
+    attr_accessor :object, :source_path, :directives, :output_file_service
+
+    # @param [BPL::Derivatives::InputObjectDecorator,BPL::Derivatives::OutputObjectDelegator]
+    # @param [Hash] directives directions for creating the output
+    # @option [String] :format the format of the output
+    # @option [String] :url the location to put the output
+    # @param [Hash] opts
+    # @option [#call] :output_file_service An output file service to call
+    def initialize(object, directives, opts = {})
+      self.object = object
+      self.source_path = object.source_path
+      self.directives = directives
+      self.output_file_service = opts.fetch(:output_file_service, BPL::Derivatives.config.output_file_service)
+    end
+
+    def process
+      raise "Processor is an abstract class. Implement `process' on #{self.class.name}"
+    end
+
+    # This governs the output key sent to the persist file service
+    # while this is adequate for storing in Fedora, it's not a great name for saving
+    # to the file system.
+    def output_file_id(name)
+      [out_prefix, name].join('_')
+    end
+
+    def output_filename_for(_name)
+      File.basename(source_path)
+    end
+
+    def finalize_derivative_output(output_io)
+      output_object = BPL::Derivatives::OutputObjectDecorator.new(output_io, object)
+      output_file_service.call(output_object, directives)
+    end
+    # @deprecated Please use a PersistOutputFileService class to save an object
+    def output_file
+      raise NotImplementedError, "Processor is an abstract class. Utilize an implementation of a PersistOutputFileService class in #{self.class.name}"
+    end
+  end
+end

data/lib/bpl/derivatives/processors/raw_image.rb

@@ -0,0 +1,37 @@
+require 'mini_magick'
+
+module BPL::Derivatives::Processors
+  class RawImage < Image
+    class_attribute :timeout
+
+    protected
+
+      def create_image(destination_name, format, quality = nil)
+        xfrm = load_image_transformer
+        # Transpose format and scaling due to the fact that ImageMagick can
+        # read but not write RAW files and this will otherwise cause many
+        # cryptic segmentation faults
+        xfrm.format(format)
+        yield(xfrm) if block_given?
+        xfrm.quality(quality.to_s) if quality
+        write_image(destination_name, format, xfrm)
+        remove_temp_files(xfrm)
+      end
+
+      # Delete any temp files that might clutter up the disk if
+      # you are doing a batch or don't touch your temporary storage
+      # for a long time
+      def remove_temp_files(xfrm)
+        xfrm.destroy!
+      end
+
+      # Override this method if you want a different transformer, or # need to load the raw image from a different source (e.g.
+      # external file).
+      #
+      # In this case always add an extension to help out MiniMagick
+      # with RAW files
+      def load_image_transformer
+        MiniMagick::Image.open(source_path)
+      end
+  end
+end

data/lib/bpl/derivatives/processors/shell_based_processor.rb

@@ -0,0 +1,103 @@
+# An abstract class for asyncronous jobs that transcode files using FFMpeg
+
+require 'tmpdir'
+require 'open3'
+
+module BPL::Derivatives::Processors
+  module ShellBasedProcessor
+    extend ActiveSupport::Concern
+
+    BLOCK_SIZE = 1024
+
+    included do
+      cattr_accessor :timeout
+      extend Open3
+    end
+
+    def process
+      format = directives[:format]
+      raise ArgumentError, "You must provide the :format you want to transcode into. You provided #{directives}" unless format
+      # TODO: if the source is in the correct format, we could just copy it and skip transcoding.
+      encode_file(format, options_for(format))
+    end
+
+    # override this method in subclass if you want to provide specific options.
+    # returns a hash of options that the specific processors use
+    def options_for(_format)
+      {}
+    end
+
+    def encode_file(file_suffix, options)
+      temp_file_name = output_file(file_suffix)
+      self.class.encode(source_path, options, temp_file_name)
+      finalize_derivative_output(File.read(temp_file_name))
+      File.unlink(temp_file_name)
+    end
+
+    def output_file(file_suffix)
+      Dir::Tmpname.create(['sufia', ".#{file_suffix}"], BPL::Derivatives.temp_file_base) {}
+    end
+
+    module ClassMethods
+      def execute(command)
+        context = {}
+        if timeout
+          execute_with_timeout(timeout, command, context)
+        else
+          execute_without_timeout(command, context)
+        end
+      end
+
+      def execute_with_timeout(timeout, command, context)
+        Timeout.timeout(timeout) do
+          execute_without_timeout(command, context)
+        end
+      rescue Timeout::Error
+        pid = context[:pid]
+        Process.kill("KILL", pid)
+        raise BPL::Derivatives::TimeoutError, "Unable to execute command \"#{command}\"\nThe command took longer than #{timeout} seconds to execute"
+      end
+
+      def execute_without_timeout(command, context)
+        err_str = ''
+        stdin, stdout, stderr, wait_thr = popen3(command)
+        context[:pid] = wait_thr[:pid]
+        files = [stderr, stdout]
+        stdin.close
+
+        until all_eof?(files)
+          ready = IO.select(files, nil, nil, 60)
+
+          next unless ready
+          readable = ready[0]
+          readable.each do |f|
+            fileno = f.fileno
+
+            begin
+              data = f.read_nonblock(BLOCK_SIZE)
+
+              case fileno
+              when stderr.fileno
+                err_str << data
+              end
+            rescue EOFError => e
+              BPL::Derivatives::Logger.debug "Caught an eof error in ShellBasedProcessor"
+              BPL::Derivatives::Logger.debug "#{e.message}"
+              # No big deal.
+            end
+          end
+        end
+
+        stdout.close
+        stderr.close
+        exit_status = wait_thr.value
+
+        raise "Unable to execute command \"#{command}\". Exit code: #{exit_status}\nError message: #{err_str}" unless exit_status.success?
+      end
+
+      def all_eof?(files)
+        files.find { |f| !f.eof }.nil?
+      end
+    end
+  end
+end