hydra-derivatives 2.0.0 → 3.0.0

data/lib/hydra/derivatives/processors/full_text.rb

@@ -0,0 +1,60 @@
+module Hydra::Derivatives::Processors
+  # Extract the full text from the content using Solr's extract handler
+  class FullText < Processor
+    # Run the full text extraction and save the result
+    # @return [TrueClass,FalseClass] was the process successful.
+    def process
+      output_file_service.call(extract, directives)
+    end
+
+    private
+
+      ##
+      # Extract full text from the content using Solr's extract handler.
+      # This will extract text from the file
+      #
+      # @return [String] The extracted text
+      def extract
+        JSON.parse(fetch)[''].rstrip
+      end
+
+      # send the request to the extract service and return the response if it was successful.
+      # TODO: this pulls the whole file into memory. We should stream it from Fedora instead
+      # @return [String] the result of calling the extract service
+      def fetch
+        req = Net::HTTP.new(uri.host, uri.port)
+        resp = req.post(uri.to_s, file_content, request_headers)
+        raise "Solr Extract service was unsuccessful. '#{uri}' returned code #{resp.code} for #{source_path}\n#{resp.body}" unless resp.code == '200'
+        file_content.rewind if file_content.respond_to?(:rewind)
+
+        resp.body
+      end
+
+      def file_content
+        @content ||= File.open(source_path).read
+      end
+
+      # @return [Hash] the request headers to send to the Solr extract service
+      def request_headers
+        { Faraday::Request::UrlEncoded::CONTENT_TYPE => "#{mime_type}",
+          Faraday::Adapter::CONTENT_LENGTH => original_size.to_s }
+      end
+
+      def mime_type
+        Hydra::Derivatives::MimeTypeService.mime_type(source_path)
+      end
+
+      def original_size
+        File.size(source_path)
+      end
+
+      # @returns [URI] path to the extract service
+      def uri
+        @uri ||= URI("#{connection_url}/update/extract?extractOnly=true&wt=json&extractFormat=text")
+      end
+
+      def connection_url
+        ActiveFedora.solr_config[:url]
+      end
+  end
+end

data/lib/hydra/derivatives/processors/image.rb

@@ -0,0 +1,58 @@
+require 'mini_magick'
+
+module Hydra::Derivatives::Processors
+  class Image < Processor
+    class_attribute :timeout
+
+    def process
+      timeout ? process_with_timeout : process_without_timeout
+    end
+
+    def process_with_timeout
+      status = Timeout::timeout(timeout) do
+        process_without_timeout
+      end
+    rescue Timeout::Error => ex
+      raise Hydra::Derivatives::TimeoutError, "Unable to process image derivative\nThe command took longer than #{timeout} seconds to execute"
+    end
+
+    def process_without_timeout
+      format = directives.fetch(:format)
+      name = directives.fetch(:label, format)
+      destination_name = output_filename_for(name)
+      size = directives.fetch(:size, nil)
+      quality = directives.fetch(:quality, nil)
+      create_resized_image(destination_name, size, format, quality)
+    end
+
+    protected
+
+    def create_resized_image(destination_name, size, format, quality=nil)
+      create_image(destination_name, format, quality) do |xfrm|
+        xfrm.resize(size) if size.present?
+      end
+    end
+
+    def create_image(destination_name, format, quality=nil)
+      xfrm = load_image_transformer
+      yield(xfrm) if block_given?
+      xfrm.format(format)
+      xfrm.quality(quality.to_s) if quality
+      write_image(destination_name, format, xfrm)
+    end
+
+    def write_image(destination_name, format, xfrm)
+      output_io = StringIO.new
+      xfrm.write(output_io)
+      output_io.rewind
+
+      output_file_service.call(output_io, directives)
+    end
+
+    # Override this method if you want a different transformer, or need to load the
+    # raw image from a different source (e.g. external file)
+    def load_image_transformer
+      MiniMagick::Image.open(source_path)
+    end
+  end
+end

data/lib/hydra/derivatives/processors/jpeg2k_image.rb

@@ -0,0 +1,129 @@
+require 'mini_magick'
+require 'nokogiri'
+
+
+module Hydra::Derivatives::Processors
+  class Jpeg2kImage < Processor
+    include ShellBasedProcessor
+
+    def process
+      image = MiniMagick::Image.open(source_path)
+      quality = image['%[channels]'] == 'gray' ? 'gray' : 'color'
+      name = directives.fetch(:label)
+      long_dim = self.class.long_dim(image)
+      file_path = self.class.tmp_file('.tif')
+      to_srgb = directives.fetch(:to_srgb, true)
+      if directives[:resize] || to_srgb
+        preprocess(image, resize: directives[:resize], to_srgb: to_srgb, src_quality: quality)
+      end
+      image.write file_path
+      recipe = self.class.kdu_compress_recipe(directives, quality, long_dim)
+      encode_file(recipe, file_path: file_path)
+      File.unlink(file_path) unless file_path.nil?
+    end
+
+    def encode_file(recipe, opts={})
+      output_file = self.class.tmp_file('.jp2')
+      if opts[:file_path]
+        self.class.encode(opts[:file_path], recipe, output_file)
+      else
+        Hydra::Derivatives::TempfileService.create(source_file) do |f|
+          self.class.encode(f.path, recipe, output_file)
+        end
+      end
+      output_file_service.call(File.open(output_file, 'rb'), directives)
+      File.unlink(output_file)
+    end
+
+    protected
+    def preprocess(image, opts={})
+      # resize: <geometry>, to_srgb: <bool>, src_quality: 'color'|'gray'
+      image.combine_options do |c|
+        c.resize(opts[:resize]) if opts[:resize]
+        c.profile self.class.srgb_profile_path if opts[:src_quality] == 'color' && opts[:to_srgb]
+      end
+      image
+    end
+
+    def self.encode(path, recipe, output_file)
+      kdu_compress = Hydra::Derivatives.kdu_compress_path
+      execute "#{kdu_compress} -i #{path} -o #{output_file} #{recipe}"
+    end
+
+    def self.srgb_profile_path
+      File.join [
+        File.expand_path('../../../', __FILE__),
+        'color_profiles',
+        'sRGB_IEC61966-2-1_no_black_scaling.icc'
+      ]
+    end
+
+    def self.tmp_file(ext)
+      Dir::Tmpname.create(['sufia', ext], Hydra::Derivatives.temp_file_base){}
+    end
+
+    def self.long_dim(image)
+      [image[:width], image[:height]].max
+    end
+
+    def self.kdu_compress_recipe(args, quality, long_dim)
+      if args[:recipe].is_a? Symbol
+        recipe = [args[:recipe].to_s, quality].join('_').to_sym
+        if Hydra::Derivatives.kdu_compress_recipes.has_key? recipe
+          return Hydra::Derivatives.kdu_compress_recipes[recipe]
+        else
+          ActiveFedora::Base.logger.warn "No JP2 recipe for :#{args[:recipe].to_s} ('#{recipe}') found in configuration. Using best guess."
+          return calculate_recipe(args,quality,long_dim)
+        end
+      elsif args[:recipe].is_a? String
+        return args[:recipe]
+      else
+        return calculate_recipe(args, quality, long_dim)
+      end
+    end
+
+    def self.calculate_recipe(args, quality, long_dim)
+      levels_arg = args.fetch(:levels, level_count_for_size(long_dim))
+      rates_arg = layer_rates(args.fetch(:layers, 8), args.fetch(:compression, 10))
+      tile_size = args.fetch(:tile_size, 1024)
+      tiles_arg = "#{tile_size},#{tile_size}"
+      jp2_space_arg = quality == 'gray' ? 'sLUM' : 'sRGB'
+
+      %Q{-rate #{rates_arg}
+          -jp2_space #{jp2_space_arg}
+          -double_buffering 10
+          -num_threads 4
+          -no_weights
+          Clevels=#{levels_arg}
+          "Stiles={#{tiles_arg}}"
+          "Cblk={64,64}"
+          Cuse_sop=yes
+          Cuse_eph=yes
+          Corder=RPCL
+          ORGgen_plt=yes
+          ORGtparts=R  }.gsub(/\s+/, " ").strip
+    end
+
+    def self.level_count_for_size(long_dim)
+      levels = 0
+      level_size = long_dim
+      while level_size >= 96
+        level_size = level_size/2
+        levels+=1
+      end
+      levels-1
+    end
+
+    def self.layer_rates(layer_count,compression_numerator)
+      #e.g. if compression_numerator = 10 then compression is 10:1
+      rates = []
+      cmp = 24.0/compression_numerator
+      layer_count.times do
+        rates << cmp
+        cmp = (cmp/1.618).round(8)
+      end
+      rates.map(&:to_s ).join(',')
+    end
+
+  end
+end

data/lib/hydra/derivatives/processors/processor.rb

@@ -0,0 +1,38 @@
+module Hydra::Derivatives::Processors
+  # Processors take a single input and produce a single output
+  class Processor
+    attr_accessor :source_path, :directives, :output_file_service
+
+    # @param [String] source_path path to the file on disk
+    # @param [Hash] directives directions for creating the output
+    # @option [String] :format the format of the output
+    # @option [String] :url the location to put the output
+    # @param [Hash] opts
+    # @option [#call] :output_file_service An output file service to call
+    def initialize(source_path, directives, opts={})
+      self.source_path = source_path
+      self.directives = directives
+      self.output_file_service = opts.fetch(:output_file_service, Hydra::Derivatives.output_file_service)
+    end
+
+    def process
+      raise "Processor is an abstract class. Implement `process' on #{self.class.name}"
+    end
+
+    # This governs the output key sent to the persist file service
+    # while this is adequate for storing in Fedora, it's not a great name for saving
+    # to the file system.
+    def output_file_id(name)
+      [out_prefix, name].join('_')
+    end
+
+    def output_filename_for(_name)
+      File.basename(source_path)
+    end
+
+    # @deprecated Please use a PersistOutputFileService class to save an object
+    def output_file
+      raise NotImplementedError, "Processor is an abstract class. Utilize an implementation of a PersistOutputFileService class in #{self.class.name}"
+    end
+  end
+end

data/lib/hydra/derivatives/processors/raw_image.rb

@@ -0,0 +1,37 @@
+require 'mini_magick'
+
+module Hydra::Derivatives::Processors
+  class RawImage < Image
+    class_attribute :timeout
+
+    protected
+
+    def create_image(destination_name, format, quality=nil)
+      xfrm = load_image_transformer
+      # Transpose format and scaling due to the fact that ImageMagick can
+      # read but not write RAW files and this will otherwise cause many
+      # cryptic segmentation faults
+      xfrm.format(format)
+      yield(xfrm) if block_given?
+      xfrm.quality(quality.to_s) if quality
+      write_image(destination_name, format, xfrm)
+      remove_temp_files(xfrm)
+    end
+
+    # Delete any temp files that might clutter up the disk if
+    # you are doing a batch or don't touch your temporary storage
+    # for a long time
+    def remove_temp_files(xfrm)
+      xfrm.destroy!
+    end
+
+    # Override this method if you want a different transformer, or # need to load the raw image from a different source (e.g.
+    # external file).
+    #
+    # In this case always add an extension to help out MiniMagick
+    # with RAW files
+    def load_image_transformer
+      MiniMagick::Image.open(source_path)
+    end
+  end
+end

data/lib/hydra/derivatives/processors/shell_based_processor.rb

@@ -0,0 +1,108 @@
+# An abstract class for asyncronous jobs that transcode files using FFMpeg
+
+require 'tmpdir'
+require 'open3'
+
+module Hydra::Derivatives::Processors
+  module ShellBasedProcessor
+    extend ActiveSupport::Concern
+
+    BLOCK_SIZE = 1024
+
+    included do
+      class_attribute :timeout
+      extend Open3
+    end
+
+    def process
+      name = directives.fetch(:label)
+      format = directives[:format]
+      raise ArgumentError, "You must provide the :format you want to transcode into. You provided #{directives}" unless format
+      # TODO if the source is in the correct format, we could just copy it and skip transcoding.
+      encode_file(format, options_for(format))
+    end
+
+    # override this method in subclass if you want to provide specific options.
+    # returns a hash of options that the specific processors use
+    def options_for(format)
+      {}
+    end
+
+    def encode_file(file_suffix, options)
+      out_file = nil
+      temp_file_name = output_file(file_suffix)
+      self.class.encode(source_path, options, temp_file_name)
+      output_file_service.call(File.open(temp_file_name, 'rb'), directives)
+      File.unlink(temp_file_name)
+    end
+
+    def output_file(file_suffix)
+      Dir::Tmpname.create(['sufia', ".#{file_suffix}"], Hydra::Derivatives.temp_file_base){}
+    end
+
+    module ClassMethods
+
+      def execute(command)
+        context = {}
+        if timeout
+          execute_with_timeout(timeout, command, context)
+        else
+          execute_without_timeout(command, context)
+        end
+      end
+
+      def execute_with_timeout(timeout, command, context)
+        begin
+          status = Timeout::timeout(timeout) do
+            execute_without_timeout(command, context)
+          end
+        rescue Timeout::Error => ex
+          pid = context[:pid]
+          Process.kill("KILL", pid)
+          raise Hydra::Derivatives::TimeoutError, "Unable to execute command \"#{command}\"\nThe command took longer than #{timeout} seconds to execute"
+        end
+
+      end
+
+      def execute_without_timeout(command, context)
+        exit_status = nil
+        err_str = ''
+        stdin, stdout, stderr, wait_thr = popen3(command)
+        context[:pid] = wait_thr[:pid]
+        stdin.close
+        stdout.close
+        files = [stderr]
+
+        until all_eof?(files) do
+          ready = IO.select(files, nil, nil, 60)
+
+          if ready
+            readable = ready[0]
+            readable.each do |f|
+              fileno = f.fileno
+
+              begin
+                data = f.read_nonblock(BLOCK_SIZE)
+
+                case fileno
+                  when stderr.fileno
+                    err_str << data
+                end
+              rescue EOFError
+                Rails.logger "Caught an eof error in ShellBasedProcessor"
+                # No big deal.
+              end
+            end
+          end
+        end
+        exit_status = wait_thr.value
+
+        raise "Unable to execute command \"#{command}\". Exit code: #{exit_status}\nError message: #{err_str}" unless exit_status.success?
+      end
+
+      def all_eof?(files)
+        files.find { |f| !f.eof }.nil?
+      end
+    end
+  end
+end

data/lib/hydra/derivatives/{video.rb → processors/video.rb}

@@ -1,4 +1,4 @@
-module Hydra::Derivatives
+module Hydra::Derivatives::Processors
   module Video
     extend ActiveSupport::Autoload
 

data/lib/hydra/derivatives/{video → processors/video}/config.rb

@@ -1,4 +1,4 @@
-module Hydra::Derivatives::Video
+module Hydra::Derivatives::Processors::Video
   class Config
     attr_writer :video_bitrate, :video_attributes, :size_attributes, :audio_attributes