image_vise 0.0.17 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6b6177fda88bf4453fffa272bf1f824865827fb0
-  data.tar.gz: a2c9d53049369496b9dd1ec781bda29ca809397b
+  metadata.gz: 5fd7e2b6f4a49ab832e600a46708d2974b74ed17
+  data.tar.gz: 1b13d15d9e99308a001ee62aeed21bf3e45c7429
 SHA512:
-  metadata.gz: 585a4c85b023af887d0e4d508956a864addef958d634baaebf6d84a7639e4d423a16b30f1c59e12e2fd4cb03017118fbb5721af0be7b04a366e6114bc7924fbb
-  data.tar.gz: 049f638ebf0f16a6c8f70df0df0e6ed55392c347e5467ada234b09962f164841f8bfda4741c4e5c767f7b70f2c1a6b0d55de7439783269d3c7140eb9bbdcdde5
+  metadata.gz: 04fcaa4b8fba508845ba29634636edf9f67758d21687754d9d2117d82eefb22282cb22a678de26f4f0e31980a509beb9f6e62b229e08ebfe6dc1b2d5718f83b6
+  data.tar.gz: 6ac6c1bda81dbfa6b2ce27106277d016ff725ac80a51a116cba447a51fc878cb89393110e53f4c06befc65d617f041d6fc865a824103c1b9d8803ec776408989

data/README.md

@@ -57,7 +57,6 @@ You might want to define a helper method for generating signed URLs as well, whi
 
     def thumb_url(source_image_url)
       qs_params = ImageVise.image_params(src_url: source_image_url, secret: ENV.fetch('IMAGE_VISE_SECRET')) do |pipe|
-        # For example, you can also yield `pipeline` to the caller
         pipe.fit_crop width: 256, height: 256, gravity: 'c'
         pipe.sharpen sigma: 0.5, radius: 2
         pipe.ellipse_stencil
@@ -66,6 +65,15 @@ You might want to define a helper method for generating signed URLs as well, whi
       '/images?' + Rack::Utils.build_query(image_request)
     end
 
+
+## Processing files on the local filesystem instead of remote ones
+
+If you want to grab a local file, compose a `file://` URL (mind the endcoding!)
+
+    src_url = 'file://' + URI.encode(File.expand_path(my_pic))
+
+Note that you need to permit certain glob patterns as sources before this will work, see below.
+
 ## Operators and pipelining
 
 ImageVise processes an image using _operators_. Each operator is just like an adjustment layer in Photoshop, except
@@ -160,26 +168,25 @@ multiple applications all using different keys for their signatures. Every reque
 each key and if at least one key generates the same signature for the same given parameters, it is going to be
 accepted and the request will be allowed to go through.
 
-## Hostname validation
+## Hostname and filesystem validation
 
 By default, `ImageVise` will refuse to process images from URLs on "unknown" hosts. To mark a host as "known"
 tell `ImageVise` to
 
     ImageVise.add_allowed_host!('my-image-store.ourcompany.co.uk')
 
+If you want to permit images from the local server filesystem to be accessed, add the glob pattern
+to the set of allowed filesystem patterns:
+
+    ImageVise.allow_filesystem_source!(Rails.root + '/public/*.jpg')
+
+Note that these are _glob_ patterns. The image path will be checked against them using `File.fnmatch`.
+
 ## State
 
 Except for the HTTP cache for redirects et.al no state is stored (`ImageVise` does not care whether you store
 your images using Dragonfly, CarrierWave or some custom handling code). All the app needs is the full URL.
 
-## FAQ
-
-* _Yo dawg, I thought you like URLs so I have put encoded URL in your URL so you can..._ - well, the only alternative
-  is also managing image storage, and this something we want to avoid to keep `ImageVise` stateless
-* _But the URLs can be exploited_ - this is highly unlikely if you pick strong keys for the HMAC signatures
-* _I can load any image into the thumbnailer_ - in fact, no. First you have the URL checks, and then - all the URLs
-  are supposed to be coming from the sources you trust since they are signed.
-
 ## Running the tests, versioning, contributing
 
 By default, `bundle exec rake` will run RSpec and will also open the generated images using the `$ open` command available

data/image_vise.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: image_vise 0.0.17 ruby lib
+# stub: image_vise 0.0.19 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "image_vise"
-  s.version = "0.0.17"
+  s.version = "0.0.19"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]

data/lib/image_vise.rb

@@ -3,45 +3,63 @@ require 'json'
 require 'patron'
 require 'rmagick'
 require 'magic_bytes'
+require 'thread'
 
 class ImageVise
-  VERSION = '0.0.17'
-
+  VERSION = '0.0.19'
+  S_MUTEX = Mutex.new
+  private_constant :S_MUTEX
+  
   @allowed_hosts = Set.new
   @keys = Set.new
   @operators = {}
-  
+  @allowed_glob_patterns = Set.new
+
   class << self
     # Resets all allowed hosts
     def reset_allowed_hosts!
-      @allowed_hosts.clear
+      S_MUTEX.synchronize { @allowed_hosts.clear }
     end
     
     # Add an allowed host
     def add_allowed_host!(hostname)
-      @allowed_hosts << hostname
+      S_MUTEX.synchronize { @allowed_hosts << hostname }
     end
     
     # Returns both the allowed hosts added at runtime and the ones set in the constant
     def allowed_hosts
-      @allowed_hosts.to_a
+      S_MUTEX.synchronize { @allowed_hosts.to_a }
     end
     
     # Removes all set keys
     def reset_secret_keys!
-      @keys.clear
+      S_MUTEX.synchronize { @keys.clear }
     end
     
+    def allow_filesystem_source!(glob_pattern)
+      S_MUTEX.synchronize { @allowed_glob_patterns << glob_pattern }
+    end
+
+    def allowed_filesystem_sources
+      S_MUTEX.synchronize { @allowed_glob_patterns.to_a }
+    end
+
+    def deny_filesystem_sources!
+      S_MUTEX.synchronize { @allowed_glob_patterns.clear }
+    end
+
     # Adds a key against which the parameters are going to be verified.
     # Multiple applications may have their own different keys, 
     # so we need to have multiple keys.
     def add_secret_key!(key)
-      @keys << key; self
+      S_MUTEX.synchronize { @keys << key }
+      self
     end
     
     # Returns the array of defined keys or raises an exception if no keys have been set yet
     def secret_keys
-      (@keys.any? && @keys.to_a) or raise "No keys set, add a key using `ImageVise.add_secret_key!(key)'"
+      keys = S_MUTEX.synchronize { @keys.any? && @keys.to_a }
+      keys or raise "No keys set, add a key using `ImageVise.add_secret_key!(key)'"
     end
     
     # Generate a set of querystring params for a resized image. Yields a Pipeline object that

data/lib/image_vise/image_request.rb

@@ -1,4 +1,5 @@
 require 'base64'
+require 'rack'
 
 class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
   class InvalidRequest < ArgumentError; end
@@ -8,7 +9,7 @@ class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
   
   # Initializes a new ParamsChecker from given HTTP server framework
   # params. The params can be symbol- or string-keyed, does not matter.
-  def self.to_request(qs_params:, secrets:, permitted_source_hosts:)
+  def self.to_request(qs_params:, secrets:, permitted_source_hosts:, allowed_filesystem_patterns:)
     base64_encoded_params = qs_params.fetch(:q) rescue qs_params.fetch('q')
     given_signature = qs_params.fetch(:sig) rescue qs_params.fetch('sig')
     
@@ -24,12 +25,19 @@ class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
     # Pick up the URL and validate it
     src_url = params.fetch(:src_url).to_s
     raise URLError, "the :src_url parameter must be non-empty" if src_url.empty?
-    raise URLError, "#{src_url} is not permitted as source" unless valid_host?(src_url, permitted_source_hosts)
 
+    src_url = URI.parse(src_url)
+    if src_url.scheme == 'file'
+      raise URLError, "#{src_url} not permitted since filesystem access is disabled" if allowed_filesystem_patterns.empty?
+      raise URLError, "#{src_url} is not on the path whitelist" unless allowed_path?(allowed_filesystem_patterns, src_url.path)
+    elsif src_url.scheme != 'file'
+      raise URLError, "#{src_url} is not permitted as source" unless permitted_source_hosts.include?(src_url.host)
+    end
+    
     # Build out the processing pipeline
     pipeline_definition = params.fetch(:pipeline)
 
-    new(src_url: src_url, pipeline: ImageVise::Pipeline.from_param(pipeline_definition))
+    new(src_url: src_url.to_s, pipeline: ImageVise::Pipeline.from_param(pipeline_definition))
   rescue KeyError => e
     raise InvalidRequest.new(e.message)
   end
@@ -49,6 +57,11 @@ class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
 
   private
 
+  def self.allowed_path?(filesystem_glob_patterns, path_to_check)
+    expanded_path = File.expand_path(path_to_check)
+    filesystem_glob_patterns.any? {|pattern| File.fnmatch?(pattern, expanded_path) }
+  end
+
   def self.valid_signature?(for_payload, given_signature, secrets)
     # Check the signature against every key that we have,
     # since different apps might be using different keys
@@ -60,9 +73,4 @@ class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
     end
     seen_valid_signature
   end
-
-  def self.valid_host?(src_url, permitted_hosts)
-    parsed_url = URI.parse(src_url)
-    permitted_hosts.include?(parsed_url.host)
-  end
 end

data/lib/image_vise/render_engine.rb

@@ -35,20 +35,12 @@ class ImageVise::RenderEngine
   
   # Fetch the given URL into a Tempfile and return the File object
   def fetch_url_into_tempfile(source_image_uri)
-    tf = Tempfile.new('source-imagevise-image')
-    s = Patron::Session.new
-    s.automatic_content_encoding = true
-    s.timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
-    s.connect_timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
-    response = s.get_file(source_image_uri, tf.path)
-    if PASSTHROUGH_STATUS_CODES.include?(response.status)
-      tf.close; tf.unlink;
-      bail response.status, "Unfortunate upstream response: #{response.status}" 
+    parsed = URI.parse(source_image_uri)
+    if parsed.scheme == 'file'
+      copy_path_into_tempfile(URI.decode(parsed.path))
+    else
+      fetch_url(source_image_uri)
     end
-    tf
-  rescue Exception => e
-    tf.close; tf.unlink;
-    raise e
   end
   
   def bail(status, *errors_array)
@@ -76,9 +68,7 @@ class ImageVise::RenderEngine
     bail(405, 'Only GET supported') unless req.get?
 
     # Validate the inputs
-    image_request = ImageVise::ImageRequest.to_request(qs_params: req.params,
-        secrets: ImageVise.secret_keys,
-        permitted_source_hosts: ImageVise.allowed_hosts)
+    image_request = ImageVise::ImageRequest.to_request(qs_params: req.params, **image_request_options)
 
     # Recover the source image URL and the pipeline instructions (all the image ops)
     source_image_uri, pipeline = image_request.src_url, image_request.pipeline
@@ -206,5 +196,43 @@ class ImageVise::RenderEngine
   ensure
     ImageVise.destroy(magick_image)
   end
-  
+
+  def image_request_options
+    {
+      secrets: ImageVise.secret_keys,
+      permitted_source_hosts: ImageVise.allowed_hosts,
+      allowed_filesystem_patterns: ImageVise.allowed_filesystem_sources,
+    }
+  end
+
+  def fetch_url(source_image_uri)
+    tf = binary_tempfile
+    s = Patron::Session.new
+    s.automatic_content_encoding = true
+    s.timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
+    s.connect_timeout = EXTERNAL_IMAGE_FETCH_TIMEOUT_SECONDS
+    response = s.get_file(source_image_uri, tf.path)
+    if PASSTHROUGH_STATUS_CODES.include?(response.status)
+      tf.close; tf.unlink;
+      bail response.status, "Unfortunate upstream response: #{response.status}" 
+    end
+    tf
+  rescue Exception => e
+    tf.close; tf.unlink;
+    raise e
+  end
+
+  def copy_path_into_tempfile(path_on_filesystem)
+    tf = binary_tempfile
+    File.open(path_on_filesystem, 'rb') do |f|
+      IO.copy_stream(f, tf)
+    end
+    tf.rewind; tf
+  rescue Errno::ENOENT
+    bail 404, "Image file not found" 
+  rescue Exception => e
+    tf.close; tf.unlink;
+    raise e
+  end
+
 end

data/spec/image_vise/image_request_spec.rb

@@ -10,11 +10,43 @@ describe ImageVise::ImageRequest do
       sig: signature
     }
 
-    image_request = described_class.to_request(qs_params: params, secrets: ['this is a secret'], permitted_source_hosts: ['bucket.s3.aws.com'])
+    image_request = described_class.to_request(qs_params: params, secrets: ['this is a secret'],
+      permitted_source_hosts: ['bucket.s3.aws.com'], allowed_filesystem_patterns: [])
     request_qs_params = image_request.to_query_string_params('this is a secret')
     expect(request_qs_params).to be_kind_of(Hash)
 
-    image_request_roundtrip = described_class.to_request(qs_params: request_qs_params, secrets: ['this is a secret'], permitted_source_hosts: ['bucket.s3.aws.com'])
+    image_request_roundtrip = described_class.to_request(qs_params: request_qs_params,
+      secrets: ['this is a secret'], permitted_source_hosts: ['bucket.s3.aws.com'], allowed_filesystem_patterns: [])
+  end
+
+  it 'forbids a file:// URL if the flag is not enabled' do
+    img_params = {src_url: 'file:///etc/passwd', pipeline: [[:auto_orient]]}
+    img_params_json = JSON.dump(img_params)
+    signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, 'this is a secret', img_params_json)
+    params = {
+      q: Base64.encode64(img_params_json),
+      sig: signature
+    }
+    
+    expect {
+      described_class.to_request(qs_params: params, secrets: ['this is a secret'],
+        permitted_source_hosts: ['bucket.s3.aws.com'], allowed_filesystem_patterns: [])
+    }.to raise_error(/filesystem access is disabled/)
+  end
+
+  it 'allows a file:// URL if its path is within the permit list' do
+    img_params = {src_url: 'file:///etc/passwd', pipeline: [[:auto_orient, {}]]}
+    img_params_json = JSON.dump(img_params)
+    signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, 'this is a secret', img_params_json)
+    params = {
+      q: Base64.encode64(img_params_json),
+      sig: signature
+    }
+    
+    image_request = described_class.to_request(qs_params: params, secrets: ['this is a secret'],
+      permitted_source_hosts: ['bucket.s3.aws.com'], allowed_filesystem_patterns: %w( /etc/* ))
+    request_qs_params = image_request.to_query_string_params('this is a secret')
+    expect(request_qs_params).to be_kind_of(Hash)
   end
 
   describe 'fails with an invalid pipeline' do
@@ -46,7 +78,7 @@ describe ImageVise::ImageRequest do
       
       expect {
         described_class.to_request(qs_params: params,
-          secrets: ['b'], permitted_source_hosts: ['bucket.s3.aws.com'])
+          secrets: ['b'], permitted_source_hosts: ['bucket.s3.aws.com'], allowed_filesystem_patterns: [])
       }.to raise_error(/Invalid or missing signature/)
     end
   end

data/spec/image_vise/render_engine_spec.rb

@@ -30,38 +30,50 @@ describe ImageVise::RenderEngine do
   end
 
   context 'when requesting an image' do
+    before :each do
+      parsed_url = Addressable::URI.parse(public_url)
+      ImageVise.add_allowed_host!(parsed_url.host)
+    end
+
     after :each do
       ImageVise.reset_allowed_hosts!
       ImageVise.reset_secret_keys!
     end
     
-    context 'halts with 422' do
-      before :each do
-        parsed_url = Addressable::URI.parse(public_url)
-        ImageVise.add_allowed_host!(parsed_url.host)
-      end
+    it 'halts with 422 when the requested image cannot be opened by ImageMagick' do
+      uri = Addressable::URI.parse(public_url)
+      ImageVise.add_allowed_host!(uri.host)
+      ImageVise.add_secret_key!('l33tness')
+      uri.path = '/___nonexistent_image.jpg'
+      
+      p = ImageVise::Pipeline.new.crop(width: 10, height: 10, gravity: 'c')
+      image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
+      params = image_request.to_query_string_params('l33tness')
+      
+      expect_any_instance_of(Patron::Session).to receive(:get_file) {|_self, url, path|
+        File.open(path, 'wb') {|f| f << 'totally not an image' }
+        double(status: 200)
+      }
+      expect(app).to receive(:handle_request_error).and_call_original
+      
+      get '/', params
+      expect(last_response.status).to eq(422)
+      expect(last_response['Cache-Control']).to eq("private, max-age=0, no-cache")
+      expect(last_response.body).to include('Unsupported/unknown')
+    end
 
-      it 'when the requested image cannot be opened by ImageMagick' do
-        uri = Addressable::URI.parse(public_url)
-        ImageVise.add_allowed_host!(uri.host)
-        ImageVise.add_secret_key!('l33tness')
-        uri.path = '/___nonexistent_image.jpg'
-        
-        p = ImageVise::Pipeline.new.crop(width: 10, height: 10, gravity: 'c')
-        image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
-        params = image_request.to_query_string_params('l33tness')
-        
-        expect_any_instance_of(Patron::Session).to receive(:get_file) {|_self, url, path|
-          File.open(path, 'wb') {|f| f << 'totally not an image' }
-          double(status: 200)
-        }
-        expect(app).to receive(:handle_request_error).and_call_original
-        
-        get '/', params
-        expect(last_response.status).to eq(422)
-        expect(last_response['Cache-Control']).to eq("private, max-age=0, no-cache")
-        expect(last_response.body).to include('Unsupported/unknown')
-      end
+    it 'halts with 422 when a file:// URL is given and filesystem access is not enabled' do
+      uri = 'file://' + test_image_path
+      ImageVise.deny_filesystem_sources!
+      ImageVise.add_secret_key!('l33tness')
+
+      p = ImageVise::Pipeline.new.fit_crop(width: 10, height: 10, gravity: 'c')
+      image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
+      params = image_request.to_query_string_params('l33tness')
+
+      get '/', params
+      expect(last_response.status).to eq(422)
+      expect(last_response.body).to include('filesystem access is disabled')
     end
     
     it 'responds with 403 when upstream returns it' do
@@ -145,7 +157,41 @@ describe ImageVise::RenderEngine do
       parsed_image = Magick::Image.from_blob(last_response.body)[0]
       expect(parsed_image.columns).to eq(10)
     end
-    
+
+    it 'picks the image from the filesystem if that is permitted' do
+      uri = 'file://' + test_image_path
+      ImageVise.allow_filesystem_source!(File.dirname(test_image_path) + '/*.*')
+      ImageVise.add_secret_key!('l33tness')
+
+      p = ImageVise::Pipeline.new.fit_crop(width: 10, height: 10, gravity: 'c')
+      image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
+      params = image_request.to_query_string_params('l33tness')
+
+      get '/', params
+      expect(last_response.status).to eq(200)
+      expect(last_response.headers['Content-Type']).to eq('image/jpeg')
+    end
+
+    it 'expands and forbids a path outside of the permitted sources'
+
+    it 'URI-decodes the path in a file:// URL for a file with a Unicode path' do
+      utf8_file_path = File.dirname(test_image_path) + '/картинка.jpg'
+      FileUtils.cp_r(test_image_path, utf8_file_path)
+      uri = 'file://' + URI.encode(utf8_file_path)
+      
+      ImageVise.allow_filesystem_source!(File.dirname(test_image_path) + '/*.*')
+      ImageVise.add_secret_key!('l33tness')
+
+      p = ImageVise::Pipeline.new.fit_crop(width: 10, height: 10, gravity: 'c')
+      image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
+      params = image_request.to_query_string_params('l33tness')
+
+      get '/', params
+      File.unlink(utf8_file_path)
+      expect(last_response.status).to eq(200)
+      expect(last_response.headers['Content-Type']).to eq('image/jpeg')
+    end
+
     it 'returns the processed JPEG image as a PNG if it had to get an alpha channel during processing' do
       uri = Addressable::URI.parse(public_url)
       ImageVise.add_allowed_host!(uri.host)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: image_vise
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.19
 platform: ruby
 authors:
 - Julik Tarkhanov