image_vise 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: b65aacfbdbada4bb3d52c9883955ff2204ee84aa66bc82d05fc81320259ac900
-  data.tar.gz: d8576cfba12f8793a75dc36f8a5c2c206f7650f402d048432f763de82ddeb0b0
+SHA1:
+  metadata.gz: 1be0d4d8682d0436356b4ea5e0023869b50709dd
+  data.tar.gz: de463d93ee8e7e55c0cfa59db9a5c0dee9b91a25
 SHA512:
-  metadata.gz: 767f28b82f290cfefdf023f83a6794901744d585eff9f0f2af92a6c408718b4663162156270d51f0ea7def4de2e14339a0bc0183b80ba62d4d16f11b69359e11
-  data.tar.gz: bb8534e292d19e51015b24586d0db5c472da2b02c3463180c35f42d8cff1d2d39aca875d5446255ea5b67b1bf59f849dda08065792438324a6ed3c9a753d9549
+  metadata.gz: e93df7c9c8723663ab09a7ba09916f5327ec7b2a71e63818b7e4bf72d851dcb119fd21c9133bd6e97158fc672e9ac64043db9f6f897a5dd9b051dd7e706c5315
+  data.tar.gz: 15e293b3e33026a78f486ba2921a8df5a1af1b51fb08c3984864b81316697c14979715148e39eaa00f0b85a7228cc0a79f19a7b8f41002558f6e86ceaec507cd

data/SECURITY.md

@@ -31,9 +31,9 @@ CDN cache because the query string params contain extra data.
 
 ## Protection for remote URLs from HTTP(s) origins
 
-Only URLs referring to permitted hosts are going to be permitted for fetching. If there are no hosts added,
+Only URLs on whitelisted hosts are going to be fetched. If there are no hosts added,
 any remote URL is going to cause an exception. No special verification for whether the upstream must be HTTP
-or HTTPS is performed at this time.
+or HTTPS is performed at this time, but HTTPS upstreams' SSL certificats _will_ be verified.
 
 ## Protection for "file:/" URLs
 

data/image_vise.gemspec

@@ -27,12 +27,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'patron', '~> 0.6'
+  spec.add_dependency 'patron', '~> 0.9'
   spec.add_dependency 'rmagick', '~> 2.15'
   spec.add_dependency 'ks'
-  spec.add_dependency 'magic_bytes', '~> 1'
   spec.add_dependency 'rack', '>= 1', '< 3'
+  spec.add_dependency 'format_parser', '>= 0.12.1', '< 1.0'
 
+  spec.add_development_dependency 'magic_bytes', '~> 1'
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 12.2"
   spec.add_development_dependency "rack-test"

data/lib/image_vise.rb

@@ -6,6 +6,7 @@ require 'magic_bytes'
 require 'thread'
 require 'base64'
 require 'rack'
+require 'format_parser'
 
 class ImageVise
   require_relative 'image_vise/version'
@@ -16,6 +17,10 @@ class ImageVise
   # The default cache liftime is 30 days, and will be used if no custom lifetime is set.
   DEFAULT_CACHE_LIFETIME = 2_592_000
 
+  # The default limit on how large may a file loaded for processing be, in bytes. This
+  # is in addition to the constraints on the file format.
+  DEFAULT_MAXIMUM_SOURCE_FILE_SIZE = 48 * 1024 * 1024
+
   @allowed_hosts = Set.new
   @keys = Set.new
   @operators = {}
@@ -172,7 +177,7 @@ class ImageVise
     return unless maybe_tempfile
     ImageVise::Measurometer.instrument('image_vise.tempfile_unlink') do
       maybe_tempfile.close unless maybe_tempfile.closed?
-      maybe_tempfile.unlink
+      maybe_tempfile.unlink if maybe_tempfile.respond_to?(:unlink)
     end
   end
 end

data/lib/image_vise/fetchers/fetcher_file.rb

@@ -2,11 +2,16 @@ class ImageVise::FetcherFile
   class AccessError < StandardError
     def http_status; 403; end
   end
+
+  class SizeError < AccessError
+    def http_status; 400; end
+  end
+
   def self.fetch_uri_to_tempfile(uri)
     tf = Tempfile.new 'imagevise-localfs-copy'
-    real_path_on_filesystem = File.expand_path(URI.decode(uri.path))
-    verify_filesystem_access! real_path_on_filesystem
-    # Do the checks
+    real_path_on_filesystem = uri_to_path(uri)
+    verify_filesystem_access!(real_path_on_filesystem)
+    verify_file_size_within_limit!(real_path_on_filesystem)
     File.open(real_path_on_filesystem, 'rb') do |f|
       IO.copy_stream(f, tf)
     end
@@ -16,6 +21,10 @@ class ImageVise::FetcherFile
     raise e
   end
 
+  def self.uri_to_path(uri)
+    File.expand_path(URI.decode(uri.path))
+  end
+
   def self.verify_filesystem_access!(path_on_filesystem)
     patterns = ImageVise.allowed_filesystem_sources
     matches = patterns.any? { |glob_pattern| File.fnmatch?(glob_pattern, path_on_filesystem) }
@@ -23,5 +32,16 @@ class ImageVise::FetcherFile
     raise AccessError, "#{path_on_filesystem} is not on the path whitelist" unless matches
   end
 
+  def self.verify_file_size_within_limit!(path_on_filesystem)
+    file_size = File.size(path_on_filesystem)
+    if file_size > maximum_source_file_size_bytes
+      raise SizeError, "#{path_on_filesystem} is too large to process (#{file_size} bytes)"
+    end
+  end
+
+  def self.maximum_source_file_size_bytes
+    ImageVise::DEFAULT_MAXIMUM_SOURCE_FILE_SIZE
+  end
+
   ImageVise.register_fetcher 'file', self
 end

data/lib/image_vise/fetchers/fetcher_http.rb

@@ -14,23 +14,35 @@ class ImageVise::FetcherHTTP
   def self.fetch_uri_to_tempfile(uri)
     tf = Tempfile.new 'imagevise-http-download'
     verify_uri_access!(uri)
+
     s = Patron::Session.new
-    s.automatic_content_encoding = true
-    s.timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
-    s.connect_timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
-    
+    configure_patron_session!(s)
     response = s.get_file(uri.to_s, tf.path)
-    
+
     if response.status != 200
       raise UpstreamError.new(response.status, "Unfortunate upstream response #{response.status} on #{uri}")
     end
-    
+
     tf
+  rescue Patron::Aborted # File size exceeds permitted size
+    ImageVise.close_and_unlink(tf)
+    raise UpstreamError.new(400, "Upstream resource at #{uri} is too large to load")
   rescue Exception => e
     ImageVise.close_and_unlink(tf)
     raise e
   end
-  
+
+  def self.maximum_response_size_bytes
+    ImageVise::DEFAULT_MAXIMUM_SOURCE_FILE_SIZE
+  end
+
+  def self.configure_patron_session!(session)
+    session.automatic_content_encoding = true
+    session.timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
+    session.connect_timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
+    session.download_byte_limit = maximum_response_size_bytes
+  end
+
   def self.verify_uri_access!(uri)
     host = uri.host
     return if ImageVise.allowed_hosts.include?(uri.host)

data/lib/image_vise/render_engine.rb

@@ -2,6 +2,16 @@
   class UnsupportedInputFormat < StandardError; end
   class EmptyRender < StandardError; end
 
+  class Filetype < Struct.new(:format_parser_format)
+    def mime
+      Rack::Mime.mime_type(ext)
+    end
+    
+    def ext
+      ".#{format_parser_format}"
+    end
+  end
+
   DEFAULT_HEADERS = {
     'Allow' => "GET"
   }.freeze
@@ -28,12 +38,8 @@
   # decoding for example).
   IMAGE_CACHE_CONTROL = "public, no-transform, max-age=%d"
 
-  # How long is a render (the ImageMagick/write part) is allowed to
-  # take before we kill it
-  RENDER_TIMEOUT_SECONDS = 10
-
-  # Which input files we permit (based on extensions stored in MagicBytes)
-  PERMITTED_SOURCE_FILE_EXTENSIONS = %w( gif png jpg psd tif)
+  # Which input files we permit (based on format identifiers in format_parser, which are symbols)
+  PERMITTED_SOURCE_FORMATS = [:bmp, :tif, :jpg, :psd, :gif, :png]
 
   # How long should we wait when fetching the image from the external host
   EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS = 4
@@ -85,7 +91,7 @@
     handle_request_error(e)
     http_status_code = e.respond_to?(:http_status) ? e.http_status : 400
     raise_exception_or_error_response(e, http_status_code)
-  rescue Exception => e
+  rescue => e
     if http_status_code = (e.respond_to?(:http_status) && e.http_status)
       handle_request_error(e)
       raise_exception_or_error_response(e, http_status_code)
@@ -134,7 +140,7 @@
   # representing the processing pipeline
   #
   # @param image_request[ImageVise::ImageRequest] the request for the image
-  # @return [Array<File, MagicBytes::FileType, String]
+  # @return [Array<File, FileType, String]
   def process_image_request(image_request)
     # Recover the source image URL and the pipeline instructions (all the image ops)
     source_image_uri, pipeline = image_request.src_url, image_request.pipeline
@@ -146,24 +152,26 @@
 
     # Download/copy the original into a Tempfile
     fetcher = ImageVise.fetcher_for(source_image_uri.scheme)
-    source_file = fetcher.fetch_uri_to_tempfile(source_image_uri)
-
-    # Make sure we do not try to process something...questionable
-    source_file_type = detect_file_type(source_file)
-    unless source_file_type_permitted?(source_file_type)
-      raise UnsupportedInputFormat.new("Unsupported/unknown input file format .%s" % source_file_type.ext)
+    source_file = ImageVise::Measurometer.instrument('image_vise.fetch') do
+      fetcher.fetch_uri_to_tempfile(source_image_uri)
     end
+    file_format = FormatParser.parse(source_file, natures: [:image]).tap { source_file.rewind }
+    raise UnsupportedInputFormat.new("%s has an unknown input file format" % source_image_uri) unless file_format
+    raise UnsupportedInputFormat.new("%s does not pass file constraints" % source_image_uri) unless permitted_format?(file_format)
 
     render_destination_file = Tempfile.new('imagevise-render').tap{|f| f.binmode }
 
     # Do the actual imaging stuff
-    expire_after = apply_pipeline(source_file.path, pipeline, source_file_type, render_destination_file.path)
+    expire_after = ImageVise::Measurometer.instrument('image_vise.render_engine.apply_pipeline') do
+      apply_pipeline(source_file.path, pipeline, file_format, render_destination_file.path)
+    end
 
     # Catch this one early
     render_destination_file.rewind
     raise EmptyRender, "The rendered image was empty" if render_destination_file.size.zero?
 
     render_file_type = detect_file_type(render_destination_file)
+
     [render_destination_file, render_file_type, etag, expire_after]
   ensure
     ImageVise.close_and_unlink(source_file)
@@ -210,18 +218,23 @@
   # the MIME type.
   #
   # @param tempfile[File] the file to perform detection on
-  # @return [MagicBytes::FileType] the detected file type
+  # @return [Symbol] the detected file format symbol that can be used as an extension
   def detect_file_type(tempfile)
     tempfile.rewind
-    MagicBytes.read_and_detect(tempfile).tap { tempfile.rewind }
+    parser_result = FormatParser.parse(tempfile, natures: :image).tap { tempfile.rewind }
+    raise "Rendered file type detection failed" unless parser_result
+    Filetype.new(parser_result.format)
   end
 
-  # Tells whether the given file type may be loaded into the image processor.
+  # Tells whether the file described by the given FormatParser result object
+  # can be accepted for processing
   #
-  # @param magic_bytes_file_info[MagicBytes::FileType] the filetype
+  # @param format_parser_result[FormatParser::Image] file information descriptor
   # @return [Boolean]
-  def source_file_type_permitted?(magic_bytes_file_info)
-    PERMITTED_SOURCE_FILE_EXTENSIONS.include?(magic_bytes_file_info.ext)
+  def permitted_format?(format_parser_result)
+    return false unless PERMITTED_SOURCE_FORMATS.include?(format_parser_result.format)
+    return false if format_parser_result.has_multiple_frames
+    true
   end
 
   # Lists exceptions that should lead to the request being flagged
@@ -279,8 +292,7 @@
   # @param pipeline[#apply!(Magick::Image)] the processing pipeline
   # @param render_to_path[String] the path to write the rendered image to
   # @return [void]
-  def apply_pipeline(source_file_path, pipeline, source_file_type, render_to_path)
-    render_file_type = source_file_type
+  def apply_pipeline(source_file_path, pipeline, source_format_parser_result, render_to_path)
 
     # Load the first frame of the animated GIF _or_ the blended compatibility layer from Photoshop
     image_list = ImageVise::Measurometer.instrument('image_vise.load_pixbuf') do
@@ -290,7 +302,7 @@
     magick_image = image_list.first # Picks up the "precomp" PSD layer in compatibility mode, or the first frame of a GIF
 
     # If any operators want to stash some data for downstream use we use this Hash
-    metadata = {}
+    metadata = {format_parser_result: source_format_parser_result}
 
     # Apply the pipeline (all the image operators)
     pipeline.apply!(magick_image, metadata)

data/lib/image_vise/version.rb

@@ -1,3 +1,3 @@
 class ImageVise
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

data/lib/image_vise/writers/auto_writer.rb

@@ -3,21 +3,13 @@
 # be chosen. Since ImageVise URLs do not contain a file extension we are free to pick
 # the suitable format at render time
 class ImageVise::AutoWriter
-  # The default file type for images with alpha
-  PNG_FILE_TYPE = MagicBytes::FileType.new('png','image/png').freeze
-
-  # Renders the file as a jpg if the custom output filetype operator is used
-  JPG_FILE_TYPE = MagicBytes::FileType.new('jpg','image/jpeg').freeze
-  
+  PNG_EXT = 'png'
+  JPG_EXT = 'jpg'
   def write_image!(magick_image, _, render_to_path)
     # If processing the image has created an alpha channel, use PNG always.
     # Otherwise, keep the original format for as far as the supported formats list goes.
-    render_file_type = if magick_image.alpha?
-      PNG_FILE_TYPE
-    else
-      JPG_FILE_TYPE
-    end
-    magick_image.format = render_file_type.ext
+    extension = magick_image.alpha? ? PNG_EXT : JPG_EXT
+    magick_image.format = extension
     magick_image.write(render_to_path)
   end
 end

data/lib/image_vise/writers/jpeg_writer.rb

@@ -1,9 +1,9 @@
 class ImageVise::JPGWriter < Ks.strict(:quality)
-  JPG_FILE_TYPE = MagicBytes::FileType.new('jpg','image/jpeg').freeze
+  JPG_EXT = 'jpg'
 
   def write_image!(magick_image, _, render_to_path)
     q = self.quality  # to avoid the changing "self" context
-    magick_image.format = JPG_FILE_TYPE.ext
+    magick_image.format = JPG_EXT
     magick_image.write(render_to_path) { self.quality = q }
   end
 end

data/spec/image_vise/fetcher_file_spec.rb

@@ -32,7 +32,7 @@ describe ImageVise::FetcherFile do
       ImageVise::FetcherFile.fetch_uri_to_tempfile(uri)
     }.to raise_error(ImageVise::FetcherFile::AccessError)
   end
-  
+
   it 'raises a meaningful exception if this file is not permitted as source' do
     path = File.expand_path(__FILE__)
 
@@ -45,4 +45,21 @@ describe ImageVise::FetcherFile do
       ImageVise::FetcherFile.fetch_uri_to_tempfile(uri)
     }.to raise_error(ImageVise::FetcherFile::AccessError)
   end
+
+  it 'raises a meaningful exception if the image exceeds the maximum permitted size' do
+    path = File.expand_path(__FILE__)
+    ruby_files_in_this_directory = __dir__ + '/*.rb'
+    ImageVise.allow_filesystem_source! ruby_files_in_this_directory
+
+    uri = URI('file://' + URI.encode(path))
+    expect(ImageVise::FetcherFile).to receive(:maximum_source_file_size_bytes).and_return(1)
+
+    expect {
+      ImageVise::FetcherFile.fetch_uri_to_tempfile(uri)
+    }.to raise_error {|e|
+      expect(e).to be_kind_of(ImageVise::FetcherFile::AccessError)
+      expect(e.message).to match(/is too large to process/)
+      expect(e.http_status).to eq(400)
+    }
+  end
 end

data/spec/image_vise/fetcher_http_spec.rb

@@ -31,6 +31,21 @@ describe ImageVise::FetcherHTTP do
     }
   end
 
+  it 'raises an error if the image exceeds the maximum permitted size' do
+    uri = URI(public_url_psd)
+    ImageVise.add_allowed_host! 'localhost'
+    expect(ImageVise::FetcherHTTP).to receive(:maximum_response_size_bytes).and_return(10)
+
+    expect {
+      ImageVise::FetcherHTTP.fetch_uri_to_tempfile(uri)
+    }.to raise_error {|e|
+      expect(e).to be_kind_of(ImageVise::FetcherHTTP::UpstreamError)
+      expect(e.message).to include(uri.to_s)
+      expect(e.message).to match(/is too large to load/)
+      expect(e.http_status).to eq(400)
+    }
+  end
+
   it 'fetches the image into a Tempfile' do
     uri = URI(public_url_psd)
     ImageVise.add_allowed_host! 'localhost'

data/spec/image_vise/render_engine_spec.rb

@@ -51,16 +51,15 @@ describe ImageVise::RenderEngine do
       p = ImageVise::Pipeline.new.crop(width: 10, height: 10, gravity: 'c')
       image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
 
-      expect_any_instance_of(Patron::Session).to receive(:get_file) {|_self, url, path|
-        File.open(path, 'wb') {|f| f << 'totally not an image' }
-        double(status: 200)
-      }
+      bad_data = StringIO.new('totally not an image')
+      expect(ImageVise::FetcherHTTP).to receive(:fetch_uri_to_tempfile).and_return(bad_data)
       expect(app).to receive(:handle_request_error).and_call_original
 
       get image_request.to_path_params('l33tness')
+
       expect(last_response.status).to eq(400)
       expect(last_response['Cache-Control']).to match(/public/)
-      expect(last_response.body).to include('Unsupported/unknown')
+      expect(last_response.body).to include('unknown')
     end
 
     it 'halts with 400 when a file:// URL is given and filesystem access is not enabled' do
@@ -224,7 +223,7 @@ describe ImageVise::RenderEngine do
       expect(app).to receive(:process_image_request).and_call_original
       expect(app).to receive(:extract_params_from_request).and_call_original
       expect(app).to receive(:image_rack_response).and_call_original
-      expect(app).to receive(:source_file_type_permitted?).and_call_original
+      expect(app).to receive(:permitted_format?).and_call_original
 
       get image_request.to_path_params('l33tness')
       expect(last_response.status).to eq(200)

data/spec/image_vise_spec.rb

@@ -102,6 +102,39 @@ describe ImageVise do
     end
   end
 
+  describe '.close_and_unlink' do
+    it 'closes and unlinks a Tempfile' do
+      tf = Tempfile.new('x')
+      tf << "foo"
+      expect(tf).to receive(:close).and_call_original
+      expect(tf).to receive(:unlink).and_call_original
+
+      ImageVise.close_and_unlink(tf)
+
+      expect(tf).to be_closed
+    end
+
+    it 'unlinks a closed Tempfile' do
+      tf = Tempfile.new('x')
+      tf << "foo"
+      tf.close
+      expect(tf).to receive(:unlink).and_call_original
+
+      ImageVise.close_and_unlink(tf)
+    end
+
+    it 'works on a nil since it gets used in ensure blocks, where the variable might be empty' do
+      ImageVise.close_and_unlink(nil) # Should not raise anything
+    end
+
+    it 'works for a StringIO which does not have unlink' do
+      sio = StringIO.new('some gunk')
+      expect(sio).not_to be_closed
+      ImageVise.close_and_unlink(sio)
+      expect(sio).to be_closed
+    end
+  end
+
   describe 'methods dealing with the operator list' do
     it 'have the basic operators already set up' do
       oplist = ImageVise.defined_operator_names

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: image_vise
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Julik Tarkhanov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-19 00:00:00.000000000 Z
+date: 2018-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: patron
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rmagick
   requirement: !ruby/object:Gem::Requirement
@@ -53,39 +53,59 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: magic_bytes
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: format_parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: 0.12.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: 0.12.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: magic_bytes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -292,7 +312,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Runtime thumbnailing proxy