httpthumbnailer 0.3.1 → 1.0.0

data/httpthumbnailer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "httpthumbnailer"
-  s.version = "0.3.1"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jakub Pastuszek"]
-  s.date = "2012-02-07"
+  s.date = "2013-07-16"
   s.description = "Provides HTTP API for thumbnailing images"
   s.email = "jpastuszek@gmail.com"
   s.executables = ["httpthumbnailer"]
@@ -35,81 +35,80 @@ Gem::Specification.new do |s|
     "features/support/test.jpg",
     "features/support/test.png",
     "features/support/test.txt",
+    "features/thumbnail.feature",
+    "features/thumbnails.feature",
     "httpthumbnailer.gemspec",
-    "lib/httpthumbnailer/multipart_response.rb",
+    "lib/httpthumbnailer/error_reporter.rb",
+    "lib/httpthumbnailer/plugin/thumbnailer.rb",
     "lib/httpthumbnailer/thumbnail_specs.rb",
     "lib/httpthumbnailer/thumbnailer.rb",
     "load_test/extralarge.jpg",
     "load_test/large.jpg",
     "load_test/large.png",
+    "load_test/load_test-cd9679c.csv",
+    "load_test/load_test-v0.3.1.csv",
     "load_test/load_test.jmx",
     "load_test/medium.jpg",
     "load_test/small.jpg",
+    "load_test/soak_test-ac0c6bcbe5e-broken-libjpeg-tatoos.csv",
+    "load_test/soak_test-cd9679c.csv",
+    "load_test/soak_test-f98334a-tatoos.csv",
+    "load_test/soak_test.jmx",
     "load_test/tiny.jpg",
     "load_test/v0.0.13-loading.csv",
     "load_test/v0.0.13.csv",
     "load_test/v0.0.14-no-optimization.csv",
     "load_test/v0.0.14.csv",
-    "spec/multipart_response_spec.rb",
+    "spec/image_processing_spec.rb",
     "spec/spec_helper.rb",
-    "spec/thumbnail_specs_spec.rb",
-    "spec/thumbnailer_spec.rb"
+    "spec/thumbnail_specs_spec.rb"
   ]
   s.homepage = "http://github.com/jpastuszek/httpthumbnailer"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.25"
   s.summary = "HTTP thumbnailing server"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<sinatra>, [">= 1.2.6"])
-      s.add_runtime_dependency(%q<mongrel>, [">= 1.2.0.pre2"])
+      s.add_runtime_dependency(%q<unicorn-cuba-base>, ["~> 1.0"])
       s.add_runtime_dependency(%q<rmagick>, ["~> 2"])
-      s.add_runtime_dependency(%q<haml>, ["~> 3"])
-      s.add_runtime_dependency(%q<ruby-ip>, ["~> 0.9"])
-      s.add_runtime_dependency(%q<cli>, ["~> 1.1.0"])
-      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.13"])
+      s.add_development_dependency(%q<rspec-mocks>, ["~> 2.13"])
       s.add_development_dependency(%q<cucumber>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
-      s.add_development_dependency(%q<daemon>, ["~> 1"])
-      s.add_development_dependency(%q<httpclient>, ["~> 2.2"])
+      s.add_development_dependency(%q<capybara>, ["~> 1.1"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
+      s.add_development_dependency(%q<httpclient>, ["~> 2.3"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.9"])
+      s.add_development_dependency(%q<multipart-parser>, ["~> 0.1.1"])
+      s.add_development_dependency(%q<daemon>, ["~> 1.1"])
     else
-      s.add_dependency(%q<sinatra>, [">= 1.2.6"])
-      s.add_dependency(%q<mongrel>, [">= 1.2.0.pre2"])
+      s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.0"])
       s.add_dependency(%q<rmagick>, ["~> 2"])
-      s.add_dependency(%q<haml>, ["~> 3"])
-      s.add_dependency(%q<ruby-ip>, ["~> 0.9"])
-      s.add_dependency(%q<cli>, ["~> 1.1.0"])
-      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<rspec>, ["~> 2.13"])
+      s.add_dependency(%q<rspec-mocks>, ["~> 2.13"])
       s.add_dependency(%q<cucumber>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
-      s.add_dependency(%q<daemon>, ["~> 1"])
-      s.add_dependency(%q<httpclient>, ["~> 2.2"])
+      s.add_dependency(%q<capybara>, ["~> 1.1"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+      s.add_dependency(%q<httpclient>, ["~> 2.3"])
       s.add_dependency(%q<rdoc>, ["~> 3.9"])
+      s.add_dependency(%q<multipart-parser>, ["~> 0.1.1"])
+      s.add_dependency(%q<daemon>, ["~> 1.1"])
     end
   else
-    s.add_dependency(%q<sinatra>, [">= 1.2.6"])
-    s.add_dependency(%q<mongrel>, [">= 1.2.0.pre2"])
+    s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.0"])
     s.add_dependency(%q<rmagick>, ["~> 2"])
-    s.add_dependency(%q<haml>, ["~> 3"])
-    s.add_dependency(%q<ruby-ip>, ["~> 0.9"])
-    s.add_dependency(%q<cli>, ["~> 1.1.0"])
-    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<rspec>, ["~> 2.13"])
+    s.add_dependency(%q<rspec-mocks>, ["~> 2.13"])
     s.add_dependency(%q<cucumber>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
-    s.add_dependency(%q<daemon>, ["~> 1"])
-    s.add_dependency(%q<httpclient>, ["~> 2.2"])
+    s.add_dependency(%q<capybara>, ["~> 1.1"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+    s.add_dependency(%q<httpclient>, ["~> 2.3"])
     s.add_dependency(%q<rdoc>, ["~> 3.9"])
+    s.add_dependency(%q<multipart-parser>, ["~> 0.1.1"])
+    s.add_dependency(%q<daemon>, ["~> 1.1"])
   end
 end
 

data/lib/httpthumbnailer/error_reporter.rb

@@ -0,0 +1,38 @@
+class ErrorReporter < Controler
+	self.define do
+		on error Rack::UnhandledRequest::UnhandledRequestError do |error|
+			write_error 404, error
+		end
+
+		on error Plugin::Thumbnailer::UnsupportedMediaTypeError do |error|
+			write_error 415, error
+		end
+
+		on error(
+			Plugin::Thumbnailer::ImageTooLargeError,
+			MemoryLimit::MemoryLimitedExceededError
+		)	do |error|
+			write_error 413, error
+		end
+
+		on error(
+			ThumbnailSpec::BadThubnailSpecError,
+			Plugin::Thumbnailer::ZeroSizedImageError,
+			Plugin::Thumbnailer::UnsupportedMethodError
+		) do |error|
+			write_error 400, error
+		end
+
+		on error StandardError do |error|
+			log.error "unhandled error while processing request: #{env['REQUEST_METHOD']} #{env['SCRIPT_NAME']}[#{env["PATH_INFO"]}]", error
+			log.debug {
+				out = StringIO.new
+				PP::pp(env, out, 200)
+				"Request: \n" + out.string
+			}
+
+			write_error 500, error
+		end
+	end
+end
+

data/lib/httpthumbnailer/plugin/thumbnailer.rb

@@ -0,0 +1,396 @@
+require 'RMagick'
+require 'forwardable'
+
+module Plugin
+	module Thumbnailer
+		class UnsupportedMethodError < ArgumentError
+			def initialize(method)
+				super("thumbnail method '#{method}' is not supported")
+			end
+		end
+
+		class UnsupportedMediaTypeError < ArgumentError
+			def initialize(error)
+				super("unsupported media type: #{error}")
+			end
+		end
+
+		class ImageTooLargeError < ArgumentError
+			def initialize(error)
+				super("image too large: #{error}")
+			end
+		end
+
+		class ZeroSizedImageError < ArgumentError
+			def initialize(width, height)
+				super("at least one image dimension is zero: #{width}x#{height}")
+			end
+		end
+
+		module ImageProcessing
+			def replace
+				@use_count ||= 0
+				processed = nil
+				begin
+					processed = yield self
+					processed = self unless processed
+					fail 'got destroyed image' if processed.destroyed?
+				ensure
+					self.destroy! if @use_count <= 0 unless processed.equal? self
+				end
+				processed
+			end
+
+			def use
+				@use_count ||= 0
+				@use_count += 1
+				begin
+					yield self
+					self
+				ensure
+					@use_count -=1
+					self.destroy! if @use_count <= 0
+				end
+			end
+		end
+
+		class InputImage
+			include ClassLogging
+			extend Forwardable
+
+			def initialize(image, processing_methods, options = {})
+				@image = image
+				@processing_methods = processing_methods
+			end
+
+			def thumbnail(spec)
+				spec = spec.dup
+				# default backgraud is white
+				spec.options['background-color'] = spec.options.fetch('background-color', 'white').sub(/^0x/, '#')
+
+				width = spec.width == :input ? @image.columns : spec.width
+				height = spec.height == :input ? @image.rows : spec.height
+
+				raise ZeroSizedImageError.new(width, height) if width == 0 or height == 0
+
+				begin
+					process_image(spec.method, width, height, spec.options).replace do |image|
+						if image.alpha?
+							log.info 'thumbnail has alpha, rendering on background'
+							image.render_on_background(spec.options['background-color'])
+						end
+					end.use do |image|
+						Service.stats.incr_total_thumbnails_created
+						image_format = spec.format == :input ? @image.format : spec.format
+
+						yield Thumbnail.new(image, image_format, spec.options)
+					end
+				rescue Magick::ImageMagickError => error
+					raise ImageTooLargeError, error.message if error.message =~ /cache resources exhausted/
+					raise
+				end
+			end
+
+			def process_image(method, width, height, options)
+				@image.replace do |image|
+					impl = @processing_methods[method] or raise UnsupportedMethodError, method
+					impl.call(image, width, height, options)
+				end
+			end
+
+			# behave as @image in processing
+			def use
+				@image.use do |image|
+					yield self
+				end
+			end
+
+			def_delegators :@image, :destroy!, :destroyed?, :mime_type
+
+			# needs to be seen as @image when returned in replace block
+			def equal?(image)
+				super image or @image.equal? image
+			end
+		end
+
+		class Thumbnail
+			include ClassLogging
+
+			def initialize(image, format, options = {})
+				@image = image
+				@format = format
+				@quality = (options['quality'] or default_quality(format))
+				@quality &&= @quality.to_i
+			end
+
+			def data
+					format = @format
+					quality = @quality
+					@image.to_blob do
+						self.format = format
+						self.quality = quality if quality
+					end
+			end
+
+			def mime_type
+				#@image.mime_type cannot be used since it is raw crated image
+				#TODO: how do I do it better?
+				mime = case @format
+					when 'JPG' then 'jpeg'
+					else @format.downcase
+				end
+				"image/#{mime}"
+			end
+
+			private
+
+			def default_quality(format)
+				case format
+				when /png/i
+					95 # max zlib compression, adaptive filtering (photo)
+				when /jpeg|jpg/i
+					85
+				else
+					nil
+				end
+			end
+		end
+
+		class Service
+			include ClassLogging
+
+			extend Stats
+			def_stats(
+				:total_images_loaded, 
+				:total_images_prescaled, 
+				:total_thumbnails_created, 
+				:images_loaded, 
+				:max_images_loaded, 
+				:max_images_loaded_worker, 
+				:total_images_created,
+				:total_images_destroyed,
+				:total_images_created_from_blob,
+				:total_images_created_initialize,
+				:total_images_created_resize,
+				:total_images_created_crop,
+				:total_images_created_sample
+			)
+
+			def self.input_formats
+				Magick.formats.select do |name, mode|
+					mode.include? 'r'
+				end.keys.map(&:downcase)
+			end
+
+			def self.output_formats
+				Magick.formats.select do |name, mode|
+					mode.include? 'w'
+				end.keys.map(&:downcase)
+			end
+
+			def self.rmagick_version
+				Magick::Version
+			end
+
+			def self.magick_version
+				Magick::Magick_version
+			end
+
+			def initialize(options = {})
+				@processing_methods = {}
+				@options = options
+				@images_loaded = 0
+
+				log.info "initializing thumbnailer: #{self.class.rmagick_version} #{self.class.magick_version}"
+
+				set_limit(:area, options[:limit_area]) if options.member?(:limit_area)
+				set_limit(:memory, options[:limit_memory]) if options.member?(:limit_memory)
+				set_limit(:map, options[:limit_map]) if options.member?(:limit_map)
+				set_limit(:disk, options[:limit_disk]) if options.member?(:limit_disk)
+
+				Magick.trace_proc = lambda do |which, description, id, method|
+					case which
+					when :c
+						Service.stats.incr_images_loaded
+						@images_loaded += 1
+						Service.stats.max_images_loaded = Service.stats.images_loaded if Service.stats.images_loaded > Service.stats.max_images_loaded
+						Service.stats.max_images_loaded_worker = @images_loaded if @images_loaded > Service.stats.max_images_loaded_worker
+						Service.stats.incr_total_images_created
+						case method
+						when :from_blob
+							Service.stats.incr_total_images_created_from_blob
+						when :initialize
+							Service.stats.incr_total_images_created_initialize
+						when :resize
+							Service.stats.incr_total_images_created_resize
+						when :resize!
+							Service.stats.incr_total_images_created_resize
+						when :crop
+							Service.stats.incr_total_images_created_crop
+						when :crop!
+							Service.stats.incr_total_images_created_crop
+						when :sample
+							Service.stats.incr_total_images_created_sample
+						else
+							log.warn "uncounted image creation method: #{method}"
+						end
+					when :d
+						Service.stats.decr_images_loaded
+						@images_loaded -= 1
+						Service.stats.incr_total_images_destroyed
+					end
+					log.debug{"image event: #{which}, #{description}, #{id}, #{method}: loaded images: #{Service.stats.images_loaded}"}
+				end
+			end
+
+			def load(io, options = {})
+				mw = options['max-width']
+				mh = options['max-height']
+				if mw and mh
+					mw = mw.to_i
+					mh = mh.to_i
+					log.info "using max size hint of: #{mw}x#{mh}"
+				end
+
+				begin
+					blob = io.read
+
+					old_memory_limit = nil
+					borrowed_memory_limit = nil
+					if options.member?(:limit_memory)
+						borrowed_memory_limit = options[:limit_memory].borrow(options[:limit_memory].limit)
+						old_memory_limit = set_limit(:memory, borrowed_memory_limit)
+					end
+
+					images = Magick::Image.from_blob(blob) do |info|
+						if mw and mh
+							define('jpeg', 'size', "#{mw*2}x#{mh*2}")
+							define('jbig', 'size', "#{mw*2}x#{mh*2}")
+						end
+					end
+
+					image = images.first
+					if image.columns > image.base_columns or image.rows > image.base_rows
+						log.warn "input image got upscaled from: #{image.base_columns}x#{image.base_rows} to #{image.columns}x#{image.rows}: reloading without max size hint!"
+						images.each do |other|
+							other.destroy!
+						end
+						images = Magick::Image.from_blob(blob)
+					end
+					blob = nil
+
+					images.shift.replace do |image|
+						images.each do |other|
+							other.destroy!
+						end
+						log.info "loaded image: #{image.inspect}"
+						Service.stats.incr_total_images_loaded
+						image.strip!
+					end.replace do |image|
+						if mw and mh
+							f = image.find_prescale_factor(mw, mh)
+							if f > 1
+								image = image.prescale(f)
+								log.info "prescaled image by factor of #{f}: #{image.inspect}"
+								Service.stats.incr_total_images_prescaled
+							end
+						end
+						InputImage.new(image, @processing_methods)
+					end
+				rescue Magick::ImageMagickError => error
+					raise ImageTooLargeError, error if error.message =~ /cache resources exhausted/
+					raise UnsupportedMediaTypeError, error
+				ensure
+					if old_memory_limit
+						set_limit(:memory, old_memory_limit)
+						options[:limit_memory].return(borrowed_memory_limit)
+					end
+				end
+			end
+
+			def processing_method(method, &impl)
+				@processing_methods[method] = impl
+			end
+
+			def set_limit(limit, value)
+				old = Magick.limit_resource(limit, value)
+				log.info "changed #{limit} limit from #{old} to #{value} bytes"
+				old
+			end
+		end
+
+		def self.setup(app)
+			Service.logger = app.logger_for(Service)
+			InputImage.logger = app.logger_for(InputImage)
+			Thumbnail.logger = app.logger_for(Thumbnail)
+
+			@@service = Service.new(
+				limit_memory: app.settings[:limit_memory],
+				limit_map: app.settings[:limit_map],
+				limit_disk: app.settings[:limit_disk]
+			)
+
+			@@service.processing_method('crop') do |image, width, height, options|
+				image.resize_to_fill(width, height) if image.columns != width or image.rows != height
+			end
+
+			@@service.processing_method('fit') do |image, width, height, options|
+				image.resize_to_fit(width, height) if image.columns != width or image.rows != height
+			end
+
+			@@service.processing_method('pad') do |image, width, height, options|
+				image.resize_to_fit(width, height).replace do |resize|
+					resize.render_on_background(options['background-color'], width, height)
+				end if image.columns != width or image.rows != height
+			end
+
+			@@service.processing_method('limit') do |image, width, height, options|
+				image.resize_to_fit(width, height) if image.columns > width or image.rows > height
+			end
+		end
+
+		def thumbnailer
+			@@service
+		end
+	end
+end
+
+class Magick::Image
+	include Plugin::Thumbnailer::ImageProcessing
+
+	def render_on_background(background_color, width = nil, height = nil)
+		Magick::Image.new(width || self.columns, height || self.rows) {
+			self.background_color = background_color
+			self.depth = 8
+		}.replace do |background|
+			background.composite!(self, Magick::CenterGravity, Magick::OverCompositeOp)
+		end
+	end
+
+	# non coping version
+	def resize_to_fill(ncols, nrows = nil, gravity = Magick::CenterGravity)
+		nrows ||= ncols
+		if ncols != columns or nrows != rows
+			scale = [ncols / columns.to_f, nrows / rows.to_f].max
+			resize(scale * columns + 0.5, scale * rows + 0.5).replace do |image|
+				image.crop(gravity, ncols, nrows, true) if ncols != columns or nrows != rows
+			end
+		else
+			crop(gravity, ncols, nrows, true) if ncols != columns or nrows != rows
+		end
+	end
+
+	def prescale(f)
+		sample(columns / f, rows / f)
+	end
+
+	def find_prescale_factor(max_width, max_height, factor = 1)
+		new_factor = factor * 2
+		if columns / new_factor > max_width * 2 and rows / new_factor > max_height * 2
+			find_prescale_factor(max_width, max_height, factor * 2)
+		else
+			factor
+		end
+	end
+end
+

data/lib/httpthumbnailer/thumbnail_specs.rb

@@ -1,6 +1,28 @@
-require 'httpthumbnailer/thumbnailer'
-
 class ThumbnailSpecs < Array
+	def self.from_uri(specs)
+		ts = ThumbnailSpecs.new
+		specs.split('/').each do |spec|
+			ts << ThumbnailSpec.from_uri(spec)
+		end
+		ts
+	end
+
+	def max_width
+		map do |spec|
+			return nil unless spec.width.is_a? Integer
+			spec.width
+		end.max
+	end
+
+	def max_height
+		map do |spec|
+			return nil unless spec.height.is_a? Integer
+			spec.height
+		end.max
+	end
+end
+
+class ThumbnailSpec
 	class BadThubnailSpecError < ArgumentError
 		class MissingArgumentError < BadThubnailSpecError
 			def initialize(spec)
@@ -14,67 +36,48 @@ class ThumbnailSpecs < Array
 			end
 		end
 
-		class BadDimmensionValueError < BadThubnailSpecError
+		class BadDimensionValueError < BadThubnailSpecError
 			def initialize(value)
-				super "bad dimmension value: #{value}"
+				super "bad dimension value: #{value}"
 			end
 		end
 	end
 
-	class ThumbnailSpec
-		def initialize(method, width, height, format, options = {})
-			@method = method
-			@width = cast_dimmension(width)
-			@height = cast_dimmension(height)
-			@format = (format == 'INPUT' ? :input : format.upcase)
-			@options = options
-		end
+	def initialize(method, width, height, format, options = {})
+		@method = method
+		@width = cast_dimension(width)
+		@height = cast_dimension(height)
+		@format = (format == 'input' ? :input : format.upcase)
+		@options = options
+	end
 
-		attr_reader :method, :width, :height, :format, :options
+	def self.from_uri(spec)
+		method, width, height, format, *options = *spec.split(',')
+		raise BadThubnailSpecError::MissingArgumentError.new(spec) unless method and width and height and format
 
-		def to_s
-			"#{method} #{width}x#{height} (#{format}) #{options.inspect}"
+		opts = {}
+		options.each do |option|
+			key, value = option.split(':')
+			raise BadThubnailSpecError::MissingOptionKeyOrValueError.new(option) unless key and value
+			opts[key] = value
 		end
 
-		private
-
-		def cast_dimmension(string)
-			return :input if string == 'INPUT'
-			raise BadThubnailSpecError::BadDimmensionValueError.new(string) unless string =~ /^\d+$/
-			string.to_i
-		end
+		ThumbnailSpec.new(method, width, height, format, opts)
 	end
 
-	def self.from_uri(specs)
-		ts = ThumbnailSpecs.new
-		specs.split('/').each do |spec|
-			method, width, height, format, *options = *spec.split(',')
-			raise BadThubnailSpecError::MissingArgumentError.new(spec) unless method and width and height and format
 
-			opts = {}
-			options.each do |option|
-				key, value = option.split(':')
-				raise BadThubnailSpecError::MissingOptionKeyOrValueError.new(option) unless key and value
-				opts[key] = value
-			end
+	attr_reader :method, :width, :height, :format, :options
 
-			ts << ThumbnailSpec.new(method, width, height, format, opts)
-		end
-		ts
+	def to_s
+		"#{method} #{width}x#{height} (#{format.downcase}) #{options.inspect}"
 	end
 
-	def max_width
-		map do |spec|
-			return nil unless spec.width.is_a? Integer
-			spec.width
-		end.max
-	end
+	private
 
-	def max_height
-		map do |spec|
-			return nil unless spec.height.is_a? Integer
-			spec.height
-		end.max
+	def cast_dimension(string)
+		return :input if string == 'input'
+		raise BadThubnailSpecError::BadDimensionValueError.new(string) unless string =~ /^\d+$/
+		string.to_i
 	end
 end