httpimagestore 0.5.0 → 1.0.0

data/features/error-reporting.feature

@@ -0,0 +1,207 @@
+Feature: Image list based thumbnailing and S3 storage
+	Storage based on URL specified image names to be generated and stored using two different path formats.
+	This configuration should be mostly compatible with pre v1.0 release.
+
+	Background:
+		Given S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables
+		Given httpimagestore server is running at http://localhost:3000/ with the following configuration
+		"""
+		s3 key="@AWS_ACCESS_KEY_ID@" secret="@AWS_SECRET_ACCESS_KEY@" ssl=false
+
+		path "structured-name"	"#{dirname}/#{digest}/#{basename}-#{imagename}.#{mimeextension}"
+		path "missing"		"blah"
+		path "zero"		"zero"
+
+		put "multipart" ":name_list" {
+			thumbnail "input" {
+				"small"		operation="crop"	width=128	height=128				if-image-name-on="#{name_list}"
+				"bad"		operation="crop"	width=0		height=0				if-image-name-on="#{name_list}"
+				"bad_dim"	operation="crop"	width="128x"	height=128				if-image-name-on="#{name_list}"
+				"superlarge"	operation="crop"	width=16000	height=16000				if-image-name-on="#{name_list}"
+				"large_png"	operation="crop"	width=7000	height=7000	format="png"		if-image-name-on="#{name_list}"
+				"bad_opts"	operation="crop"	width=128	height=128	options="foo=bar"	if-image-name-on="#{name_list}"
+			}
+		}
+
+		put "singlepart" ":name_list" {
+			thumbnail "input"	"small"		operation="crop"	width=128	height=128				if-image-name-on="#{name_list}"
+			thumbnail "input"	"bad"		operation="crop"	width=0		height=0				if-image-name-on="#{name_list}"
+			thumbnail "input"	"bad_dim"	operation="crop"	width="128x"	height=128				if-image-name-on="#{name_list}"
+			thumbnail "input"	"superlarge"	operation="crop"	width=16000	height=16000				if-image-name-on="#{name_list}"
+			thumbnail "input"	"large_png"	operation="crop"	width=7000	height=7000	format="png"		if-image-name-on="#{name_list}"
+			thumbnail "input"	"bad_opts"	operation="crop"	width=128	height=128	options="foo=bar"	if-image-name-on="#{name_list}"
+		}
+
+		get "s3" {
+			source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="missing"
+		}
+
+		get "file" {
+			source_file "original" root="/tmp" path="missing"
+		}
+
+		get "zero" {
+			source_file "original" root="/dev" path="zero"
+		}
+		"""
+		Given httpthumbnailer server is running at http://localhost:3100/
+
+	@error-reporting
+	Scenario: Reporting of missing resource
+		When I do GET request http://localhost:3000/blah
+		Then response status will be 404
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		request for URI '/blah' was not handled by the server
+		"""
+
+	@error-reporting
+	Scenario: Reporting of missing S3 resource
+		When I do GET request http://localhost:3000/s3
+		Then response status will be 404
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		S3 bucket 'httpimagestoretest' does not contain key 'blah'
+		"""
+
+	@error-reporting
+	Scenario: Reporting of missing file resource
+		When I do GET request http://localhost:3000/file
+		Then response status will be 404
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		error while processing image 'original': file 'blah' not found
+		"""
+
+	@error-reporting
+	Scenario: Reporting of unsupported media type
+		Given test.txt file content as request body
+		When I do PUT request http://localhost:3000/multipart/small,tiny
+		Then response status will be 415
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' failed: unsupported media type: no decode delegate for this image format
+		"""
+		When I do PUT request http://localhost:3000/singlepart/small,tiny
+		Then response status will be 415
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'small' failed: unsupported media type: no decode delegate for this image format
+		"""
+
+	@error-reporting
+	Scenario: Reporting and handling of thumbnailing errors
+		Given test.jpg file content as request body
+		When I do PUT request http://localhost:3000/multipart/small,bad
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'bad' failed: at least one image dimension is zero: 0x0
+		"""
+		When I do PUT request http://localhost:3000/singlepart/small,bad
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'bad' failed: at least one image dimension is zero: 0x0
+		"""
+
+	@error-reporting
+	Scenario: Reporting and handling of thumbnailing errors - bad options format
+		Given test.jpg file content as request body
+		When I do PUT request http://localhost:3000/multipart/small,bad_opts
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		thumbnailing of 'input' into 'bad_opts' failed: missing option value for key 'foo=bar'
+		"""
+		When I do PUT request http://localhost:3000/singlepart/small,bad_opts
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		thumbnailing of 'input' into 'bad_opts' failed: missing option value for key 'foo=bar'
+		"""
+
+	@error-reporting @test
+	Scenario: Bad dimension
+		Given test.jpg file content as request body
+		When I do PUT request http://localhost:3000/multipart/bad_dim
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'bad_dim' failed: bad dimension value: 128x
+		"""
+		When I do PUT request http://localhost:3000/singlepart/bad_dim
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'bad_dim' failed: bad dimension value: 128x
+		"""
+
+	@error-reporting
+	Scenario: Too large image - uploaded image too big to fit in memory limit
+		Given test-large.jpg file content as request body
+		When I do PUT request http://localhost:3000/multipart/large_png
+		Then response status will be 413
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' failed: image too large: cache resources exhausted
+		"""
+		When I do PUT request http://localhost:3000/singlepart/large_png
+		Then response status will be 413
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'large_png' failed: image too large: cache resources exhausted
+		"""
+
+	@error-reporting
+	Scenario: Too large image - memory exhausted when thmbnailing
+		Given test.jpg file content as request body
+		When I do PUT request http://localhost:3000/multipart/superlarge
+		Then response status will be 413
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'superlarge' failed: image too large: cache resources exhausted
+		"""
+		When I do PUT request http://localhost:3000/singlepart/superlarge
+		Then response status will be 413
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		thumbnailing of 'input' into 'superlarge' failed: image too large: cache resources exhausted
+		"""
+
+	@error-reporting
+	Scenario: Zero body length
+		Given test.empty file content as request body
+		When I do PUT request http://localhost:3000/multipart/small
+		Then response status will be 400
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		empty body - expected image data
+		"""
+
+	@error-reporting
+	Scenario: Memory limit exceeded
+		When I do GET request http://localhost:3000/zero
+		Then response status will be 413
+		And response content type will be text/plain
+		And response body will be CRLF ended lines like
+		"""
+		memory limit exceeded
+		"""
+

data/features/health-check.feature

@@ -0,0 +1,30 @@
+Feature: Health check URL
+	Server can be tested with GET request to '/health_check'.
+
+	Background:
+		Given httpimagestore server is running at http://localhost:3000/ with the following configuration
+		"""
+		"""
+
+	@health-check
+	Scenario: Passing health check when thumbnailer is running
+		Given httpthumbnailer server is running at http://localhost:3100/
+		When I do GET request http://localhost:3000/health_check
+		Then response status will be 200
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		HTTP Image Store OK
+		"""
+
+	@health-check
+	Scenario: Failing health check when thumbnailer is not running
+		Given httpthumbnailer server is not running
+		When I do GET request http://localhost:3000/health_check
+		Then response status will be 502
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		Connection refused - connect(2) (http://localhost:3100)
+		"""
+

data/features/s3-store-and-thumbnail.feature

@@ -0,0 +1,65 @@
+Feature: Store limited original image in S3 and thumbnail based on request
+	Posted image will be converted to JPEG and resized if it is bigger that given dimensions.
+	Than it will get stored on S3.
+	Get interface will allow to fetch the image from S3 and thumbnailing to given parameters.
+
+	Background:
+		Given S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables
+		Given httpimagestore server is running at http://localhost:3000/ with the following configuration
+		"""
+		s3 key="@AWS_ACCESS_KEY_ID@" secret="@AWS_SECRET_ACCESS_KEY@" ssl=false
+
+		path "original-hash"	"#{digest}.#{mimeextension}"
+		path "path"		"#{path}"
+
+		put "original" {
+			thumbnail "input" "original" operation="limit" width=100 height=100 format="jpeg" quality=95
+
+			store_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="original-hash"
+
+			output_store_path "original"
+		}
+
+		get "thumbnail" "v1" ":path" ":operation" ":width" ":height" ":options?" {
+			source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="path"
+
+			thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="#{options}" quality=84 format="png"
+
+			output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
+		}
+
+		"""
+		Given httpthumbnailer server is running at http://localhost:3100/
+
+	@s3-store-and-thumbnail
+	Scenario: Putting original to S3 bucket
+		Given there is no 4006450256177f4a.jpg file in S3 bucket
+		Given test.jpg file content as request body
+		When I do PUT request http://localhost:3000/original
+		Then response status will be 200
+		And response content type will be text/plain
+		And response body will be CRLF ended lines
+		"""
+		4006450256177f4a.jpg
+		"""
+		Then S3 object 4006450256177f4a.jpg will contain JPEG image of size 71x100
+		When I do GET request http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/4006450256177f4a.jpg
+		Then response status will be 403
+
+	@s3-store-and-thumbnail
+	Scenario: Getting thumbnail to spec based on uploaded S3 image
+		Given test.jpg file content is stored in S3 under 4006450256177f4a.jpg
+		When I do GET request http://localhost:3000/thumbnail/v1/4006450256177f4a.jpg/pad/50/50
+		Then response status will be 200
+		And response content type will be image/png
+		Then response body will contain PNG image of size 50x50
+
+	@s3-store-and-thumbnail
+	Scenario: Getting thumbnail to spec based on uploaded S3 image - with options passed
+		Given test.jpg file content is stored in S3 under 4006450256177f4a.jpg
+		When I do GET request http://localhost:3000/thumbnail/v1/4006450256177f4a.jpg/pad/50/50/background-color:green
+		Then response status will be 200
+		And response content type will be image/png
+		Then response body will contain PNG image of size 50x50
+		And that image pixel at 2x2 should be of color green
+

data/features/step_definitions/httpimagestore_steps.rb

@@ -3,15 +3,19 @@ Given /httpimagestore argument (.*)/ do |arg|
 end
 
 Given /httpimagestore server is running at (.*) with the following configuration/ do |url, config|
-	cfile = Tempfile.new('httpimagestore.conf')
-	cfile.write(config)
-	cfile.close
+	$temp_dir = Pathname.new(Dir.mktmpdir) unless $temp_dir
+	
+	cfile = $temp_dir + Digest.hexencode(Digest::MD5.digest(config))
+	cfile.open('w') do |io|
+		io.write(config.replace_s3_variables)
+	end
 
 	begin
+		log = support_dir + 'server.log'
 		start_server(
-			"bundle exec #{script('httpimagestore')} #{(@httpimagestore_args ||= []).join(' ')} #{cfile.path}",
+			"bundle exec #{script('httpimagestore')} -f -d -l #{log} -w 1 #{(@httpimagestore_args ||= []).join(' ')} #{cfile.to_s}",
 			'/tmp/httpimagestore.pid',
-			support_dir + 'server.log',
+			log,
 			url
 		)
 	ensure
@@ -20,14 +24,19 @@ Given /httpimagestore server is running at (.*) with the following configuration
 end
 
 Given /httpthumbnailer server is running at (.*)/ do |url|
+	log = support_dir + 'thumbniler.log'
 	start_server(
-		"httpthumbnailer",
+		"httpthumbnailer -f -d -l #{log} -w 1",
 		'/tmp/httpthumbnailer.pid',
-		support_dir + 'thumbniler.log',
+		log,
 		url
 	)
 end
 
+Given /httpthumbnailer server is not running/ do
+	stop_server('/tmp/httpthumbnailer.pid') 
+end
+
 Given /httpimagestore log is empty/ do
 		log = support_dir + 'server.log'
 		log.truncate(0) if log.exist?
@@ -38,16 +47,21 @@ Given /httpthumbnailer log is empty/ do
 		log.truncate(0) if log.exist?
 end
 
-Given /(.*) file content as request body/ do |file|
+Given /^([^ ]*) file content as request body/ do |file|
 	@request_body = File.open(support_dir + file){|f| f.read }
 end
 
-Given /(.*) S3 bucket with key (.*) and secret (.*)/ do |bucket, key_id, key_secret|
-		@bucket = S3::Service.new(:access_key_id => key_id, :secret_access_key => key_secret).buckets.find(bucket)
+Given /S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables/ do
+	
+	unless ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY'] and ENV['AWS_S3_TEST_BUCKET']
+		fail "AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY or AWS_S3_TEST_BUCKET environment variables not set"
+	end
+
+	@bucket = AWS::S3.new(access_key_id: ENV['AWS_ACCESS_KEY_ID'], secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'], use_ssl: false).buckets[ENV['AWS_S3_TEST_BUCKET']]
 end
 
 Given /there is no (.*) file in S3 bucket/ do |path|
-	@bucket.objects.find(path).destroy rescue S3::Error::NoSuchKey
+	@bucket.objects[path].delete # rescue S3::Error::NoSuchKey
 end
 
 Given /(.*) header set to (.*)/ do |header, value|
@@ -55,8 +69,16 @@ Given /(.*) header set to (.*)/ do |header, value|
 	@request_headers[header] = value
 end
 
+Given /(.*) file content is stored in S3 under (.*)/ do |file, key|
+	@bucket.objects[key].write(File.open(support_dir + file){|f| f.read }, content_type: 'image/jpeg')
+end
+
+Then /S3 bucket will not contain (.*)/ do |key|
+	@bucket.objects[key].exists?.should_not be_true
+end
+
 When /I do (.*) request (.*)/ do |method, uri|
-	@response = HTTPClient.new.request(method, URI.encode(uri), nil, @request_body, (@request_headers or {}))
+	@response = HTTPClient.new.request(method, URI.encode(uri.replace_s3_variables), nil, @request_body, (@request_headers or {}))
 end
 
 Then /response status will be (.*)/ do |status|
@@ -68,30 +90,30 @@ Then /response content type will be (.*)/ do |content_type|
 end
 
 Then /response body will be CRLF ended lines like/ do |body|	
-	@response.body.should match(body)
+	@response.body.should match(body.replace_s3_variables)
 	@response.body.each_line do |line|
 		line[-2,2].should == "\r\n"
 	end
 end
 
 Then /response body will be CRLF ended lines$/ do |body|	
-	@response.body.should == body.gsub("\n", "\r\n") + "\r\n"
+	@response.body.should == body.replace_s3_variables.gsub("\n", "\r\n") + "\r\n"
 end
 
 Then /(http.*) content type will be (.*)/ do |url, content_type|
-	get_headers(url)['Content-Type'].should == content_type
+	get_headers(url.replace_s3_variables)['Content-Type'].should == content_type
 end
 
 Then /(http.*) ([^ ]+) header will be (.*)/ do |url, header, value|
-	get_headers(url)[header].should == value
+	get_headers(url.replace_s3_variables)[header].should == value
 end
 
 Then /(http.*) ([^ ]+) header will not be set/ do |url, header|
-	get_headers(url)[header].should be_nil
+	get_headers(url.replace_s3_variables)[header].should be_nil
 end
 
-Then /(.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, height|
-	data = get(url)
+Then /(http.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, height|
+	data = get(url.replace_s3_variables)
 	
 	@image.destroy! if @image
 	@image = Magick::Image.from_blob(data).first
@@ -101,12 +123,30 @@ Then /(.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, he
 	@image.rows.should == height.to_i
 end
 
-Then /S3 bucket will not contain (.*)/ do |path|
-	begin
-		@bucket.objects.find(path)
-		true.should eq(false, "object #{path} found in bucket")
-	rescue S3::Error::NoSuchKey
-		true.should == true
-	end
+Then /S3 object (.*) will contain (.*) image of size (.*)x(.*)/ do |key, format, width, height|
+	data = @bucket.objects[key].read
+	
+	@image.destroy! if @image
+	@image = Magick::Image.from_blob(data).first
+
+	@image.format.should == format
+	@image.columns.should == width.to_i
+	@image.rows.should == height.to_i
+end
+
+Then /response body will contain (.*) image of size (.*)x(.*)/ do |format, width, height|
+	data = @response.body
+	Pathname.new('/tmp/out.jpg').open('w'){|io| io.write data}
+	
+	@image.destroy! if @image
+	@image = Magick::Image.from_blob(data).first
+
+	@image.format.should == format
+	@image.columns.should == width.to_i
+	@image.rows.should == height.to_i
+end
+
+And /that image pixel at (.*)x(.*) should be of color (.*)/ do |x, y, color|
+	@image.pixel_color(x.to_i, y.to_i).to_color.sub(/^#/, '0x').should == color
 end
 

data/features/support/env.rb

@@ -17,7 +17,19 @@ require "open3"
 require "thread"
 require 'tempfile'
 require 'RMagick'
-require 's3'
+require 'aws-sdk'
+require 'httpimagestore/aws_sdk_regions_hack'
+require 'digest'
+
+class String
+	def replace_s3_variables
+		string = self.dup
+		string.gsub!(/@AWS_ACCESS_KEY_ID@/, ENV['AWS_ACCESS_KEY_ID'])
+		string.gsub!(/@AWS_SECRET_ACCESS_KEY@/, ENV['AWS_SECRET_ACCESS_KEY'])
+		string.gsub!(/@AWS_S3_TEST_BUCKET@/, ENV['AWS_S3_TEST_BUCKET'])
+		string
+	end
+end
 
 def gem_dir
 		Pathname.new(__FILE__).dirname + '..' + '..'
@@ -35,22 +47,35 @@ def script(file)
 		gem_dir + 'bin' + file
 end
 
+def http_client
+	client = HTTPClient.new
+	#client.debug_dev = STDOUT
+	client
+end
+
 def get(url)
-	HTTPClient.new.get_content(URI.encode(url))
+	http_client.get_content(URI.encode(url))
 end
 
 def get_headers(url)
-	HTTPClient.new.get(URI.encode(url)).headers
+	http_client.get(URI.encode(url)).headers
 end
 
+@@running_cmd = {}
 def start_server(cmd, pid_file, log_file, test_url)
-	stop_server(pid_file)
+	if @@running_cmd[pid_file]
+		return if @@running_cmd[pid_file] == cmd
+		stop_server(pid_file) 
+	end
 
 	fork do
 		Daemon.daemonize(pid_file, log_file)
+		log_file = Pathname.new(log_file)
+		log_file.truncate(0) if log_file.exist?
 		exec(cmd)
 	end
-	Process.wait
+
+	@@running_cmd[pid_file] = cmd
 
 	ppid = Process.pid
 	at_exit do
@@ -71,6 +96,7 @@ def stop_server(pid_file)
 	pid_file = Pathname.new(pid_file)
 	return unless pid_file.exist?
 
+	STDERR.puts http_client.get_content("http://localhost:3000/stats") if pid_file.to_s.include? 'httpimagestore'
 	pid = pid_file.read.strip.to_i
 
 	Timeout.timeout(20) do
@@ -80,6 +106,7 @@ def stop_server(pid_file)
 				sleep 0.1
 			end
 		rescue Errno::ESRCH
+			@@running_cmd.delete pid_file.to_s
 			pid_file.unlink
 		end
 	end