image_vise 0.0.16

data/lib/image_vise/pipeline.rb

@@ -0,0 +1,54 @@
+class ImageVise::Pipeline
+  def self.operator_by_name(name)
+    operator = ImageVise.operator_from(name)  or raise "Unknown operator #{name}"
+  end
+
+  def self.from_param(array_of_operator_names_to_operator_params)
+    operators = array_of_operator_names_to_operator_params.map do |(operator_name, operator_params)|
+      operator_class = operator_by_name(operator_name)
+      if operator_params.any? && operator_class.method(:new).arity.nonzero?
+        operator_class.new(**operator_params)
+      else
+        operator_class.new
+      end
+    end
+    new(operators)
+  end
+
+  def initialize(operators = [])
+    @ops = operators.to_a
+  end
+
+  def <<(image_operator)
+    @ops << image_operator; self
+  end
+
+  def empty?
+    @ops.empty?
+  end
+
+  def method_missing(method_name, *a, &blk)
+    operator_builder = ImageVise.operator_from(method_name)
+    self << operator_builder.new(*a)
+  end
+  
+  def respond_to_missing?(method_name, *a)
+    ImageVise.defined_operators.include?(method_name.to_s)
+  end
+
+  def to_params
+    @ops.map do |operator|
+      operator_name = ImageVise.operator_name_for(operator)
+      operator_params = operator.respond_to?(:to_h) ? operator.to_h : {}
+      [operator_name, operator_params]
+    end
+  end
+
+  def apply!(magick_image)
+    @ops.each{|e| e.apply!(magick_image) }
+  end
+  
+  def each(&b)
+    @ops.each(&b)
+  end
+end

data/lib/image_vise/render_engine.rb

@@ -0,0 +1,210 @@
+class ImageVise::RenderEngine
+  require_relative 'image_request'
+  require_relative 'file_response'
+  class UnsupportedInputFormat < StandardError; end
+  class EmptyRender < StandardError; end
+
+  # Codes that have to be sent through to the requester
+  PASSTHROUGH_STATUS_CODES = [404, 403, 503, 504, 500]
+  
+  DEFAULT_HEADERS = {
+    'Allow' => "GET"
+  }.freeze
+  
+  # To prevent some string allocations
+  JSON_ERROR_HEADERS = DEFAULT_HEADERS.merge({
+    'Content-Type' => 'application/json',
+    'Cache-Control' => 'private, max-age=0, no-cache'
+  }).freeze
+  
+  # How long is a render (the ImageMagick/write part) is allowed to
+  # take before we kill it
+  RENDER_TIMEOUT_SECONDS = 10
+  
+  # Which input files we permit (based on extensions stored in MagicBytes)
+  PERMITTED_EXTENSIONS = %w( gif png jpg )
+  
+  # How long should we wait when fetching the image from the external host
+  EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS = 4
+  
+  # The default file type for images with alpha
+  PNG_FILE_TYPE = Class.new do
+    def self.mime; 'image/png'; end
+    def self.ext; 'png'; end
+  end
+  
+  # Fetch the given URL into a Tempfile and return the File object
+  def fetch_url_into_tempfile(source_image_uri)
+    tf = Tempfile.new('source-imagevise-image')
+    s = Patron::Session.new
+    s.automatic_content_encoding = true
+    s.timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
+    s.connect_timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
+    response = s.get_file(source_image_uri, tf.path)
+    if PASSTHROUGH_STATUS_CODES.include?(response.status)
+      tf.close; tf.unlink;
+      bail response.status, "Unfortunate upstream response: #{response.status}" 
+    end
+    tf
+  rescue Exception => e
+    tf.close; tf.unlink;
+    raise e
+  end
+  
+  def bail(status, *errors_array)
+    h = JSON_ERROR_HEADERS.dup # Needed because some upstream middleware migh be modifying headers
+    response = [status.to_i, h, [JSON.pretty_generate({errors: errors_array})]]
+    throw :__bail, response
+  end
+  
+  # The main entry point URL, at the index so that the Sinatra app can be used
+  # in-place of a Rails controller (as opposed to having to mount it at the root
+  # of the Rails app or having all the URLs refer to a subpath)
+  def call(env)
+    catch(:__bail) { handle_request(env) }
+  end
+  
+  def handle_request(env)
+    setup_error_handling(env)
+    render_destination_file = binary_tempfile
+    
+    # Assume that if _any_ ETag is given the image is being requested anew as a refetch,
+    # and the client already has it. Just respond with a 304.
+    return [304, DEFAULT_HEADERS.dup, []] if env['HTTP_IF_NONE_MATCH']
+
+    req = Rack::Request.new(env)
+    bail(405, 'Only GET supported') unless req.get?
+
+    # Validate the inputs
+    image_request = ImageVise::ImageRequest.to_request(qs_params: req.params,
+        secrets: ImageVise.secret_keys,
+        permitted_source_hosts: ImageVise.allowed_hosts)
+
+    # Recover the source image URL and the pipeline instructions (all the image ops)
+    source_image_uri, pipeline = image_request.src_url, image_request.pipeline
+    raise 'Image pipeline has no operators' if pipeline.empty?
+
+    # Compute an ETag which describes this image transform + image source location.
+    # Assume the image URL contents does _never_ change.
+    etag = image_request.cache_etag
+    
+    # Download the original into a Tempfile
+    source_file = fetch_url_into_tempfile(source_image_uri)
+    
+    # Make sure we do not try to process something...questionable
+    source_file_type = detect_file_type(source_file)
+    
+    # Perform the processing
+    if enable_forking?
+      require 'exceptional_fork'
+      ExceptionalFork.fork_and_wait { apply_pipeline(source_file.path, pipeline, source_file_type, render_destination_file.path) }
+    else
+      apply_pipeline(source_file.path, pipeline, source_file_type, render_destination_file.path)
+    end
+    
+    # Catch this one early
+    raise EmptyRender, "The rendered image was empty" if render_destination_file.size.zero?
+
+    render_destination_file.rewind
+    render_file_type = detect_file_type(render_destination_file)
+
+    response_headers = DEFAULT_HEADERS.merge({
+      'Content-Type' => render_file_type.mime,
+      'Content-Length' => '%d' % render_destination_file.size,
+      'Cache-Control' => 'public',
+      'ETag' => etag
+    })
+
+    # Wrap the body Tempfile with a self-closing response.
+    # Once the response is read in full, the tempfile is going to be closed and unlinked.
+    [200, response_headers, ImageVise::FileResponse.new(render_destination_file)]
+  rescue *permanent_failures => e
+    handle_request_error(e)
+    raise_exception_or_error_response(e, 422)
+  rescue Exception => e
+    handle_generic_error(e)
+    raise_exception_or_error_response(e, 500)
+  ensure
+    close_and_unlink(source_file)
+  end
+  
+  def raise_exception_or_error_response(exception, status_code)
+    if raise_exceptions? 
+      raise exception
+    else
+      bail status_code, exception.message
+    end
+  end
+  
+  def close_and_unlink(f)
+    return unless f
+    f.close unless f.closed?
+    f.unlink
+  end
+  
+  def binary_tempfile
+    Tempfile.new('imagevise-tmp').tap{|f| f.binmode }
+  end
+  
+  def detect_file_type(tempfile)
+    tempfile.rewind
+    
+    file_info = MagicBytes.read_and_detect(tempfile)
+    return file_info if PERMITTED_EXTENSIONS.include?(file_info.ext)
+    raise UnsupportedInputFormat.new("Unsupported/unknown input file format .%s" %
+       file_info.ext)
+  end
+  
+  # Lists exceptions that should lead to the request being flagged
+  # as invalid (and not 5xx). Decent clients should _not_ retry those requests.
+  def permanent_failures
+    [
+      Magick::ImageMagickError,
+      UnsupportedInputFormat,
+      ImageVise::ImageRequest::InvalidRequest
+    ]
+  end
+  
+  # Is meant to be overridden by subclasses,
+  # will be called at the start of each reauest
+  def setup_error_handling(rack_env)
+  end
+
+  # Is meant to be overridden by subclasses,
+  # will be called when a request fails due to a malformed query string,
+  # unrecognized signature or other client-induced problems
+  def handle_request_error(err)
+  end
+
+  # Is meant to be overridden by subclasses,
+  # will be called when a request fails due to an error on the server
+  # (like an unexpected error in an image operator)
+  def handle_generic_error(err)
+  end
+  
+  # Tells whether the engine must raise the exceptions further up the Rack stack,
+  # or they should be suppressed and a JSON response must be returned.
+  def raise_exceptions?
+    false
+  end
+  
+  def enable_forking?
+    ENV.key? 'IMAGE_VISE_ENABLE_FORK'
+  end
+  
+  def apply_pipeline(source_file_path, pipeline, source_file_type, render_to_path)
+    render_file_type = source_file_type
+    magick_image = Magick::Image.read(source_file_path)[0]
+    pipeline.apply!(magick_image)
+    
+    # If processing the image has created an alpha channel, use PNG always.
+    # Otherwise, keep the original format for as far as the supported formats list goes.
+    render_file_type = PNG_FILE_TYPE if magick_image.alpha?
+    
+    magick_image.format = render_file_type.ext
+    magick_image.write(render_to_path)
+  ensure
+    ImageVise.destroy(magick_image)
+  end
+  
+end

data/lib/image_vise/sharpen.rb

@@ -0,0 +1,21 @@
+# Applies a sharpening filter to the image.
+#
+# The corresponding Pipeline method is `sharpen`.
+class ImageVise::Sharpen < Ks.strict(:radius, :sigma)
+  def initialize(*)
+    super
+    self.radius = radius.to_f
+    self.sigma = sigma.to_f
+    raise ArgumentError, ":radius must positive" unless sigma > 0
+    raise ArgumentError, ":sigma must positive" unless sigma > 0
+  end
+  
+  def apply!(magick_image)
+    sharpened_image = magick_image.sharpen(radius, sigma)
+    magick_image.composite!(sharpened_image, Magick::CenterGravity, Magick::CopyCompositeOp)
+  ensure
+    ImageVise.destroy(sharpened_image)
+  end
+end
+
+ImageVise.add_operator 'sharpen', ImageVise::Sharpen

data/lib/image_vise.rb

@@ -0,0 +1,118 @@
+require 'ks'
+require 'json'
+require 'patron'
+require 'rmagick'
+require 'magic_bytes'
+
+class ImageVise
+  VERSION = '0.0.16'
+
+  @allowed_hosts = Set.new
+  @keys = Set.new
+  @operators = {}
+  
+  class << self
+    # Resets all allowed hosts
+    def reset_allowed_hosts!
+      @allowed_hosts.clear
+    end
+    
+    # Add an allowed host
+    def add_allowed_host!(hostname)
+      @allowed_hosts << hostname
+    end
+    
+    # Returns both the allowed hosts added at runtime and the ones set in the constant
+    def allowed_hosts
+      @allowed_hosts.to_a
+    end
+    
+    # Removes all set keys
+    def reset_secret_keys!
+      @keys.clear
+    end
+    
+    # Adds a key against which the parameters are going to be verified.
+    # Multiple applications may have their own different keys, 
+    # so we need to have multiple keys.
+    def add_secret_key!(key)
+      @keys << key; self
+    end
+    
+    # Returns the array of defined keys or raises an exception if no keys have been set yet
+    def secret_keys
+      (@keys.any? && @keys.to_a) or raise "No keys set, add a key using `ImageVise.add_secret_key!(key)'"
+    end
+    
+    # Generate a set of querystring params for a resized image. Yields a Pipeline object that
+    # will receive method calls for adding image operations to a stack.
+    #
+    #   ImageVise.image_params(src_url: image_url_on_s3, secret: '...') do |p|
+    #      p.center_fit width: 128, height: 128 
+    #      p.elliptic_stencil
+    #   end #=> {q: '...', sig: '...'}
+    #
+    # The query string elements can be then passed on to RenderEngine for validation and execution.
+    #
+    # @yields {ImageVise::Pipeline}
+    # @return [Hash]
+    def image_params(src_url:, secret:)
+      p = Pipeline.new
+      yield(p)
+      raise ArgumentError, "Image pipeline has no steps defined" if p.empty?
+      ImageRequest.new(src_url: src_url, pipeline: p).to_query_string_params(secret)
+    end
+
+    # Adds an operator
+    def add_operator(operator_name, object_responding_to_new)
+      @operators[operator_name.to_s] = object_responding_to_new
+    end
+
+    # Gets an operator by name
+    def operator_from(operator_name)
+      @operators.fetch(operator_name.to_s)
+    end
+    
+    def defined_operator_names
+      @operators.keys
+    end
+    
+    def operator_name_for(operator)
+      @operators.key(operator.class) or raise "Operator #{operator.inspect} not registered using ImageVise.add_operator"
+    end
+  end
+  
+  # Made available since the object that is used with `mount()` in Rails
+  # has to, by itself, to respond to `call`.
+  #
+  # Thanks to this method you can do this:
+  #
+  #   mount ImageVise => '/thumbnails'
+  #
+  # instead of having to do
+  #
+  #   mount ImageVise.new => '/thumbnails'
+  #
+  def self.call(rack_env)
+    ImageVise::RenderEngine.new.call(rack_env)
+  end
+  
+  def call(rack_env)
+    ImageVise::RenderEngine.new.call(rack_env)
+  end
+  
+  # Used as a shorthand to force-destroy Magick images in ensure() blocks. Since
+  # ensure blocks sometimes deal with variables in inconsistent states (variable
+  # in scope but not yet set to an image) we take the possibility of nils into account.
+  # We also deal with Magick::Image objects that already have been destroyed in a clean manner.
+  def self.destroy(maybe_image)
+    return unless maybe_image
+    return unless maybe_image.respond_to?(:destroy!)
+    return if maybe_image.destroyed?
+    maybe_image.destroy!
+  end
+end
+
+Dir.glob(__dir__ + '/**/*.rb').sort.each do |f|
+  require f unless f == File.expand_path(__FILE__)
+end

data/spec/image_vise/auto_orient_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe ImageVise::AutoOrient do
+  it 'applies auto orient to the image' do
+    image = Magick::Image.read(test_image_path)[0]
+    orient = described_class.new
+    expect(image).to receive(:auto_orient!).and_call_original
+    orient.apply!(image)
+  end
+end

data/spec/image_vise/crop_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe ImageVise::Crop do
+  it 'refuses invalid parameters' do
+    expect { described_class.new(width: 0, height: -1, gravity: '') }.to raise_error(ArgumentError)
+  end
+  
+  it 'applies the crop with different gravities' do
+    %w( s sw se n ne nw c).each do |gravity|
+      image = Magick::Image.read(test_image_path)[0]
+      crop = described_class.new(width: 120, height: 220, gravity: gravity)
+
+      crop.apply!(image)
+
+      expect(image.columns).to eq(120)
+      expect(image.rows).to eq(220)
+      examine_image(image, "gravity-%s-" % gravity)
+    end
+  end
+end

data/spec/image_vise/ellipse_stencil_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe ImageVise::EllipseStencil do
+  it 'applies the circle stencil' do
+    image = Magick::Image.read(test_image_path)[0]
+    stencil = described_class.new
+    stencil.apply!(image)
+    examine_image(image, "circle-stencil")
+  end
+end

data/spec/image_vise/file_response_spec.rb

@@ -0,0 +1,45 @@
+require_relative '../spec_helper'
+
+describe ImageVise::FileResponse do
+  it 'reads the file in binary mode, closes and unlinks the tempfile when close() is called' do
+    random_data = SecureRandom.random_bytes(1024 * 2048)
+    f = Tempfile.new("experiment")
+    f.binmode
+    f << random_data
+    
+    response = described_class.new(f)
+    readback = ''.encode(Encoding::BINARY)
+    response.each do | chunk |
+      expect(chunk.encoding).to eq(Encoding::BINARY)
+      readback << chunk
+    end
+    
+    response.close
+    
+    expect(readback).to eq(random_data)
+    expect(f).to be_closed
+    expect(f.path).to be_nil
+  end
+  
+  it 'only asks for the path of the tempfile and uses a separate file descriptor' do
+    f = Tempfile.new("experiment")
+    f.binmode
+    f << SecureRandom.random_bytes(2048)
+    f.flush
+    
+    # Use a double so that all the methods except the ones we mock raise an assertion
+    double = double(path: f.path)
+    expect(double).to receive(:flush)
+    
+    read_from_response = ''.encode(Encoding::BINARY)
+    response = described_class.new(double)
+    response.each{|b| read_from_response << b }
+    
+    f.rewind
+    
+    expect(f.read).to eq(read_from_response)
+    
+    f.close
+    f.unlink
+  end
+end

data/spec/image_vise/fit_crop_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe ImageVise::FitCrop do
+  it 'refuses invalid arguments' do
+    expect { described_class.new(width: 0, height: -1, gravity: '') }.to raise_error(ArgumentError)
+  end
+  
+  it 'applies the crop with different gravities' do
+    %w( s sw se n ne nw c).each do |gravity|
+      image = Magick::Image.read(test_image_path)[0]
+      crop = described_class.new(width: 120, height: 220, gravity: gravity)
+
+      crop.apply!(image)
+
+      expect(image.columns).to eq(120)
+      expect(image.rows).to eq(220)
+      examine_image(image, "gravity-%s-" % gravity)
+    end
+  end
+end

data/spec/image_vise/geom_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe ImageVise::Geom do
+  it 'refuses invalid parameters' do
+    expect { described_class.new(geometry_string: nil) }.to raise_error(ArgumentError)
+  end
+
+  it 'applies various geometry strings' do
+    %w( ^220x110 !20x20 !10x100 ).each do |geom_string|
+      image = Magick::Image.read(test_image_path)[0]
+      crop = described_class.new(geometry_string: geom_string)
+
+      crop.apply!(image)
+      examine_image(image, 'geom-%s' % geom_string)
+    end
+  end
+end

data/spec/image_vise/image_request_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe ImageVise::ImageRequest do
+  it 'accepts a set of params, secrets and permitted hosts and returns a Pipeline' do
+    img_params = {src_url: 'http://bucket.s3.aws.com/image.jpg', pipeline: [[:crop, {width: 10, height: 10, gravity: 's'}]]}
+    img_params_json = JSON.dump(img_params)
+    signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, 'this is a secret', img_params_json)
+    params = {
+      q: Base64.encode64(img_params_json),
+      sig: signature
+    }
+
+    image_request = described_class.to_request(qs_params: params, secrets: ['this is a secret'], permitted_source_hosts: ['bucket.s3.aws.com'])
+    request_qs_params = image_request.to_query_string_params('this is a secret')
+    expect(request_qs_params).to be_kind_of(Hash)
+
+    image_request_roundtrip = described_class.to_request(qs_params: request_qs_params, secrets: ['this is a secret'], permitted_source_hosts: ['bucket.s3.aws.com'])
+  end
+
+  describe 'fails with an invalid pipeline' do
+    it 'when the pipe param is missing'
+    it 'when the pipe param is empty'
+    it 'when the pipe param cannot be parsed into a Pipeline'
+    it 'when the pipe param parses into a Pipeline with zero operators'
+  end
+
+  describe 'fails with an invalid URL' do
+    it 'when the URL param is missing'
+    it 'when the URL param is empty'
+    it 'when the URL is referencing a non-permitted host'
+    it 'when the URL refers to a non-HTTP(S) scheme'
+  end
+
+  describe 'fails with an invalid signature' do
+    it 'when the sig param is missing'
+    it 'when the sig param is empty'
+    it 'when the sig is invalid' do
+      img_params = {src_url: 'http://bucket.s3.aws.com/image.jpg',
+          pipeline: [[:crop, {width: 10, height: 10, gravity: 's'}]]}
+      img_params_json = JSON.dump(img_params)
+      signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, 'a', img_params_json)
+      params = {
+        q: Base64.encode64(img_params_json),
+        sig: signature
+      }
+      
+      expect {
+        described_class.to_request(qs_params: params,
+          secrets: ['b'], permitted_source_hosts: ['bucket.s3.aws.com'])
+      }.to raise_error(/Invalid or missing signature/)
+    end
+  end
+end

data/spec/image_vise/pipeline_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe ImageVise::Pipeline do
+  it 'is empty by default' do
+    expect(subject).to be_empty
+  end
+  
+  it 'reinstates the pipeline from the operator list parameters' do
+    params = [
+      ["geom", {:geometry_string=>"10x10"}],
+      ["crop", {:width=>5, :height=>5, :gravity=>"se"}],
+      ["auto_orient", {}],
+      ["fit_crop", {:width=>10, :height=>32, :gravity=>"c"}]
+    ]
+    pipeline = described_class.from_param(params)
+    expect(pipeline).not_to be_empty
+  end
+
+  it 'produces a usable operator parameter list that can be roundtripped' do
+    operator_list = subject.geom(geometry_string: '10x10').
+      crop(width: 5, height: 5, gravity: 'se').
+      auto_orient.
+      fit_crop(width: 10, height: 32, gravity: 'c').to_params
+    
+    expect(operator_list).to eq([
+      ["geom", {:geometry_string=>"10x10"}],
+      ["crop", {:width=>5, :height=>5, :gravity=>"se"}],
+      ["auto_orient", {}],
+      ["fit_crop", {:width=>10, :height=>32, :gravity=>"c"}]
+    ])
+
+    pipeline = described_class.from_param(operator_list)
+    expect(pipeline).not_to be_empty
+  end
+  
+  it 'applies itself to the image' do
+    pipeline = subject.
+      auto_orient.
+      fit_crop(width: 48, height: 48, gravity: 'c').
+      sharpen(radius: 2, sigma: 0.5).
+      ellipse_stencil
+    
+    image = Magick::Image.read(test_image_path)[0]
+    pipeline.apply! image
+    examine_image(image, "stenciled")
+  end
+  
+  it 'raises an exception when an attempt is made to serialize an unknown operator' do
+    unknown_op_class = Class.new
+    subject << unknown_op_class.new
+    expect {
+      subject.to_params
+    }.to raise_error(/not registered/)
+  end
+
+  it 'composes parameters even if one of the operators does not support to_h' do
+    class AnonOp
+    end
+    class ParametricOp
+      def to_h; {a: 133}; end
+    end
+    
+    ImageVise.add_operator('t_anon', AnonOp)
+    ImageVise.add_operator('t_parametric', ParametricOp)
+    
+    subject << AnonOp.new
+    subject << ParametricOp.new
+    expect(subject.to_params).to eq([["t_anon", {}], ["t_parametric", {:a=>133}]])
+  end
+end