httpimagestore 1.0.0 → 1.1.0

data/README.md

@@ -89,7 +89,7 @@ Variables:
 * `extension` - file extension determined from `path`
 * `mimeextension` - image extension based on mime type; mime type will be updated based on information from [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) for input image and output thumbnails - content determined
 * `imagename` - name of the image that is being stored or sourced
-* URL matches - other variables can be matched from URL pattern - see API configuration
+* URL matches and query string parameters - other variables can be matched from URL pattern and query string parameters - see API configuration
 
 Example:
 
@@ -119,6 +119,7 @@ Statement should start with one of the following HTTP verbs in lowercase: `get`,
 * `:symbol/regexp/` - in this format symbol will be matched only if `/` surrounded [regular expression](http://rubular.com) matches the URI section
 
 Note that any remaining URI are is stored in `path` variable.
+Also any query string parameters are available as variables. Additionally `query_string_options` is build from query string parameters and can be used to specify options to [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer).
 
 Example:
 
@@ -540,6 +541,14 @@ get "v1" "thumbnail" ":path" ":operation" ":width" ":height" ":options?" {
 
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
+
+get "v2" "thumbnail" ":operation" ":width" ":height" {
+	source_s3 "original" bucket="mybucket_v1" path="path"
+
+	thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="#{query_string_options}" quality=84 format="png"
+
+	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
+}
 ```
 
 Compatibility API works by storing input image and selected (via URI) classes of thumbnails generated during image upload. Once the image is uploaded thumbnails can be served directly from S3. There are two endpoints defined for that API to handle URIs that contain optional image storage name that results in usage of different storage key.
@@ -617,6 +626,31 @@ $ curl 10.1.1.24:3000/v1/thumbnail/4006450256177f4a.jpg/fit/100/1000 -v -s -o /t
 
 $ identify /tmp/test.jpeg
 /tmp/test.jpeg JPEG 100x141 100x141+0+0 8-bit sRGB 4.68KB 0.000u 0:00.000
+
+# Also form with query string passed options can be used to retrieve thumbnails
+$ curl 10.1.1.24:3000/v2/thumbnail/pad/100/100/4006450256177f4a.jpg?background-color=green -v -s -o /tmp/test.jpg
+* About to connect() to 10.1.1.24 port 3000 (#0)
+*   Trying 10.1.1.24... connected
+> GET /v2/thumbnail/pad/100/100/4006450256177f4a.jpg?background-color=green HTTP/1.1
+> User-Agent: curl/7.22.0 (x86_64-apple-darwin10.8.0) libcurl/7.22.0 OpenSSL/1.0.1c zlib/1.2.7 libidn/1.25
+> Host: 10.1.1.24:3000
+> Accept: */*
+> 
+< HTTP/1.1 200 OK
+< Server: nginx/1.2.9
+< Date: Wed, 24 Jul 2013 11:38:39 GMT
+< Content-Type: image/jpeg
+< Content-Length: 3310
+< Connection: keep-alive
+< Status: 200 OK
+< Cache-Control: public, max-age=31557600, s-maxage=0
+< 
+{ [data not shown]
+* Connection #0 to host 10.1.1.24 left intact
+* Closing connection #0
+
+$ identify /tmp/test.jpg
+/tmp/test.jpg JPEG 100x100 100x100+0+0 8-bit sRGB 3.31KB 0.000u 0:00.000
 ```
 
 ## Usage
@@ -680,7 +714,8 @@ httpthumbnailer --pid-file /var/run/httpthumbnailer/pidfile --log-file /var/log/
 ```
 
 To start [nginx](http://nginx.org) we need to configure it to run as reverse HTTP proxy for our UNIX socket based `httpimagestore` backend.
-Also we set it up so that it does request and response buffering.
+Also we set it up so that it does request and response buffering and on disk caching of GET requests.
+You may want to disable caching if your GET URL resource is not immutable.
 Here is the example `/etc/nginx/nginx.conf` file:
 
 ```nginx
@@ -716,6 +751,9 @@ http {
 		server unix:/var/run/httpimagestore.sock fail_timeout=0;
 	}
 
+	# cache GET requests up to 256MiB in RAM and 130GiB on disk for up to 30 days of no access
+	proxy_cache_path	/var/cache/nginx/httpimagestore levels=2:2 keys_zone=httpimagestore:256m max_size=130g inactive=30d;
+
 	server {
 		listen		*:3000;
 		server_name	localhost;
@@ -737,6 +775,9 @@ http {
 			proxy_read_timeout			120s;
 			proxy_connect_timeout		10s;
 
+			proxy_cache			httpimagestore;
+			proxy_cache_key		"$request_uri";
+
 			proxy_pass http://httpimagestore;
 		}
 	}

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

data/bin/httpimagestore

@@ -68,17 +68,21 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
 
 				env['app.configuration'].handlers.each do |handler|
 					on eval(handler.http_method), *handler.uri_matchers.map{|m| instance_eval(&m.matcher)} do |*args|
-						locals = {}
-
-						{
-							path: env["PATH_INFO"][1..-1] || ''
-						}.merge(
-							Hash[handler.uri_matchers.select{|m| m.name}.map{|m| m.name}.zip(args)]
-						).each do |name, value|
-							locals[name] = URI.decode(value).force_encoding('UTF-8')
+						# map and decode matched URI componetns
+						matches = {}
+						Hash[handler.uri_matchers.select{|m| m.name}.map{|m| m.name}.zip(args)].each do |name, value|
+							matches[name] = URI.decode(value).force_encoding('UTF-8')
 						end
 
-						state = Configuration::RequestState.new(req.body.read, locals, memory_limit)
+						# decode remaining URI components
+						path = (env["PATH_INFO"][1..-1] || '').split('/').map do |part|
+							URI.decode(part).force_encoding('UTF-8')
+						end.join('/')
+
+						# query string already doceded by Rack
+						query_string = req.GET
+
+						state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit)
 
 						handler.image_sources.each do |image_source|
 							image_source.realize(state) unless image_source.respond_to? :excluded? and image_source.excluded?(state)

data/features/s3-store-and-thumbnail.feature

@@ -28,6 +28,13 @@ Feature: Store limited original image in S3 and thumbnail based on request
 			output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 		}
 
+		get "thumbnail" "v2" ":operation" ":width" ":height" {
+			source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="path"
+
+			thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="#{query_string_options}" quality=84 format="png"
+
+			output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
+		}
 		"""
 		Given httpthumbnailer server is running at http://localhost:3100/
 
@@ -63,3 +70,20 @@ Feature: Store limited original image in S3 and thumbnail based on request
 		Then response body will contain PNG image of size 50x50
 		And that image pixel at 2x2 should be of color green
 
+	@s3-store-and-thumbnail @v2
+	Scenario: Getting thumbnail to spec based on uploaded S3 image - v2
+		Given test.jpg file content is stored in S3 under 4006450256177f4a.jpg
+		When I do GET request http://localhost:3000/thumbnail/v2/pad/50/50/4006450256177f4a.jpg
+		Then response status will be 200
+		And response content type will be image/png
+		Then response body will contain PNG image of size 50x50
+
+	@s3-store-and-thumbnail @v2
+	Scenario: Getting thumbnail to spec based on uploaded S3 image - with options passed
+		Given test.jpg file content is stored in S3 under 4006450256177f4a.jpg
+		When I do GET request http://localhost:3000/thumbnail/v2/pad/50/50/4006450256177f4a.jpg?background-color=green
+		Then response status will be 200
+		And response content type will be image/png
+		Then response body will contain PNG image of size 50x50
+		And that image pixel at 2x2 should be of color green
+

data/httpimagestore.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "httpimagestore"
-  s.version = "1.0.0"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jakub Pastuszek"]
-  s.date = "2013-07-16"
+  s.date = "2013-07-24"
   s.description = "Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)"
   s.email = "jpastuszek@gmail.com"
   s.executables = ["httpimagestore"]

data/lib/httpimagestore/configuration/handler.rb

@@ -14,6 +14,8 @@ module Configuration
 	end
 
 	class RequestState
+		include ClassLogging
+
 		class Images < Hash
 			def initialize(memory_limit)
 				@memory_limit = memory_limit
@@ -31,10 +33,18 @@ module Configuration
 				fetch(name){|image_name| raise ImageNotLoadedError.new(image_name)}
 			end
 		end
-		
-		def initialize(body = '', locals = {}, memory_limit = MemoryLimit.new)
+
+		def initialize(body = '', matches = {}, path = '', query_string = {}, memory_limit = MemoryLimit.new)
+			@locals = {}
+			@locals.merge! query_string
+			@locals[:path] = path
+			@locals.merge! matches
+			@locals[:query_string_options] = query_string.sort.map{|kv| kv.join(':')}.join(',')
+			log.debug "processing request with body length: #{body.bytesize} bytes and locals: #{@locals} "
+
+			@locals[:body] = body
+
 			@images = Images.new(memory_limit)
-			@locals = {body: body}.merge(locals)
 			@memory_limit = memory_limit
 			@output_callback = nil
 		end
@@ -234,6 +244,8 @@ module Configuration
 			log.warn 'no handlers configured' if configuration.handlers.empty?
 		end
 	end
+	RequestState.logger = Global.logger_for(RequestState)
+
 	Global.register_node_parser Handler
 end
 

data/spec/configuration_file_spec.rb

@@ -146,7 +146,7 @@ describe Configuration do
 
 		describe 'memory limit' do
 			let :state do
-				Configuration::RequestState.new('abc', {}, MemoryLimit.new(1))
+				Configuration::RequestState.new('abc', {}, '', {}, MemoryLimit.new(1))
 			end
 
 			it 'should rais MemoryLimit::MemoryLimitedExceededError error if limit exceeded runing file sourcing' do
@@ -214,7 +214,7 @@ describe Configuration do
 			end
 
 			let :state do
-				Configuration::RequestState.new('abc', list: 'input1,input3')
+				Configuration::RequestState.new('abc', {list: 'input1,input3'})
 			end
 
 			it 'should mark stores to ib included when image name match if-image-name-on list' do

data/spec/configuration_handler_spec.rb

@@ -47,7 +47,7 @@ describe Configuration do
 
 			it 'should free memory limit if overwritting image' do
 				limit = MemoryLimit.new(2)
-				request_state = Configuration::RequestState.new('abc', {}, limit)
+				request_state = Configuration::RequestState.new('abc', {}, '', {}, limit)
 
 				limit.borrow 1
 				request_state.images['test'] = Configuration::Image.new('x')

data/spec/configuration_output_spec.rb

@@ -155,7 +155,7 @@ describe Configuration do
 
 			describe 'conditional inclusion support' do
 				let :state do
-					Configuration::RequestState.new('abc', list: 'input,image2')
+					Configuration::RequestState.new('abc', {list: 'input,image2'})
 				end
 
 				subject do

data/spec/configuration_s3_spec.rb

@@ -68,7 +68,7 @@ else
 
 		describe Configuration::S3Source do
 			let :state do
-				Configuration::RequestState.new('abc', test_image: 'test.jpg')
+				Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
 			end
 
 			subject do
@@ -259,7 +259,7 @@ else
 
 			describe 'memory limit' do
 				let :state do
-					Configuration::RequestState.new('abc', {test_image: 'test.jpg'}, MemoryLimit.new(10))
+					Configuration::RequestState.new('abc', {test_image: 'test.jpg'}, '', {}, MemoryLimit.new(10))
 				end
 
 				it 'should raise MemoryLimit::MemoryLimitedExceededError when sourcing bigger image than limit' do
@@ -272,7 +272,7 @@ else
 
 		describe Configuration::S3Store do
 			let :state do
-				Configuration::RequestState.new(@test_data, test_image: 'test_out.jpg')
+				Configuration::RequestState.new(@test_data, {test_image: 'test_out.jpg'})
 			end
 
 			subject do
@@ -435,7 +435,7 @@ else
 
 			describe 'conditional inclusion support' do
 				let :state do
-					Configuration::RequestState.new(@test_data, test_image: 'test_out.jpg', list: 'input,input2')
+					Configuration::RequestState.new(@test_data, {test_image: 'test_out.jpg', list: 'input,input2'})
 				end
 
 				subject do

data/spec/configuration_thumbnailer_spec.rb

@@ -136,11 +136,12 @@ describe Configuration do
 			let :state do
 				Configuration::RequestState.new(
 					(support_dir + 'compute.jpg').read,
-					operation: 'pad',
-					width: '10',
-					height: '10',
-					options: 'background-color:green',
-					path: nil
+					{
+						operation: 'pad',
+						width: '10',
+						height: '10',
+						options: 'background-color:green'
+					}
 				)
 			end
 
@@ -187,11 +188,14 @@ describe Configuration do
 					let :state do
 						Configuration::RequestState.new(
 							(support_dir + 'compute.jpg').read,
-							{operation: 'pad',
-							width: '10',
-							height: '10',
-							options: 'background-color:green',
-							path: nil},
+							{
+								operation: 'pad',
+								width: '10',
+								height: '10',
+								options: 'background-color:green'
+							},
+							'',
+							{},
 							MemoryLimit.new(10)
 						)
 					end
@@ -207,11 +211,12 @@ describe Configuration do
 					it 'should raise Thumbnail::ThumbnailingError on realization of bad thumbnail sepc' do
 						state = Configuration::RequestState.new(
 							(support_dir + 'compute.jpg').read,
-							operation: 'pad',
-							width: '0',
-							height: '10',
-							options: 'background-color:green',
-							path: nil
+							{
+								operation: 'pad',
+								width: '0',
+								height: '10',
+								options: 'background-color:green'
+							}
 						)
 
 						expect {
@@ -293,11 +298,14 @@ describe Configuration do
 					let :state do
 						Configuration::RequestState.new(
 							(support_dir + 'compute.jpg').read,
-							{operation: 'pad',
-							width: '10',
-							height: '10',
-							options: 'background-color:green',
-							path: nil},
+							{
+								operation: 'pad',
+								width: '10',
+								height: '10',
+								options: 'background-color:green'
+							},
+							'',
+							{},
 							MemoryLimit.new(10)
 						)
 					end
@@ -325,12 +333,13 @@ describe Configuration do
 					let :state do
 						Configuration::RequestState.new(
 							(support_dir + 'compute.jpg').read,
-							operation: 'pad',
-							width: '10',
-							height: '10',
-							options: 'background-color:green',
-							path: nil,
-							list: 'small,padded'
+							{
+								operation: 'pad',
+								width: '10',
+								height: '10',
+								options: 'background-color:green',
+								list: 'small,padded'
+							}
 						)
 					end
 
@@ -346,11 +355,12 @@ describe Configuration do
 					it 'should raise Thumbnail::ThumbnailingError on realization of bad thumbnail sepc' do
 						state = Configuration::RequestState.new(
 							(support_dir + 'compute.jpg').read,
-							operation: 'pad',
-							width: '0',
-							height: '10',
-							options: 'background-color:green',
-							path: nil
+							{
+								operation: 'pad',
+								width: '0',
+								height: '10',
+								options: 'background-color:green'
+							}
 						)
 
 						subject.handlers[0].image_sources[0].realize(state)
@@ -378,7 +388,9 @@ describe Configuration do
 			let :state do
 				Configuration::RequestState.new(
 					(support_dir + 'compute.jpg').read,
-					list: 'thumbnail1,input4,thumbnail5,input6'
+					{
+						list: 'thumbnail1,input4,thumbnail5,input6'
+					}
 				)
 			end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: httpimagestore
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-16 00:00:00.000000000 Z
+date: 2013-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicorn-cuba-base
@@ -291,7 +291,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 801711138329567989
+      hash: 1029095012750983706
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: