httpthumbnailer 1.2.0 → 1.3.0

data/Rakefile

@@ -17,10 +17,16 @@ Jeweler::Tasks.new do |gem|
   gem.name = "httpthumbnailer"
   gem.homepage = "http://github.com/jpastuszek/httpthumbnailer"
   gem.license = "MIT"
-  gem.summary = %Q{HTTP thumbnailing server}
-  gem.description = %Q{Provides HTTP API for thumbnailing images}
+  gem.summary = %Q{HTTP API server for image thumbnailing, editing and format conversion}
+  gem.description = %Q{Statless HTTP server that provides API for thumbnailing images with different aspect ratio keeping methods, applying image edits (like rotate, crop, blur, pixelate, etc.), identification of image format and size and more. It is using ImageMagick or GraphicsMagick via RMagick gem as the image processing library.}
   gem.email = "jpastuszek@gmail.com"
   gem.authors = ["Jakub Pastuszek"]
+  gem.files.exclude "features/**/*"
+  gem.files.exclude "gatling/**/*"
+  gem.files.exclude "spec/**/*"
+  gem.files.exclude "test_plugins/**/*"
+  gem.files.exclude "*.gemspec"
+  gem.files.exclude ".rspec"
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-1.2.0
+1.3.0

data/bin/httpthumbnailer

@@ -13,8 +13,18 @@ Application.new('httpthumbnailer', port: 3100) do
 			cast: Integer,
 			description: 'image cache temporary file size limit in MiB',
 			default: 1024
+		options :plugins,
+			cast: Pathname,
+			description: 'path to directory from which plugins will be loaded (files with .rb extension)',
+			default: '/usr/share/httpthumbnailer/plugins'
 		switch :no_optimization,
-			description: 'disable load time size hinting and prescaling optimizations'
+			description: 'disable load time size hinting and downsampling optimizations all together'
+		switch :reload,
+			description: 'reload input images without size hint that got upscaled instead of downsampling (broken JPEG lib)'
+		switch :no_upscale_fix,
+			description: 'do nothing if image got upscaled when using size hint (broken JPEG lib)'
+		switch :no_downsample,
+			description: 'disable downsampling of input image before processing'
 		version (Pathname.new(__FILE__).dirname + '..' + 'VERSION').read
 	end
 
@@ -37,6 +47,9 @@ Application.new('httpthumbnailer', port: 3100) do
 		end
 
 		Controller.settings[:optimization] = (not settings.no_optimization)
+		Controller.settings[:reload] = settings.reload
+		Controller.settings[:no_upscale_fix] = settings.no_upscale_fix
+		Controller.settings[:no_downsample] = settings.no_downsample
 		Controller.settings[:limit_memory] = settings.limit_memory * 1024**2
 		Controller.settings[:limit_map] = settings.limit_disk * 1024**2
 		Controller.settings[:limit_disk] = settings.limit_disk * 1024**2
@@ -46,7 +59,22 @@ Application.new('httpthumbnailer', port: 3100) do
 		require 'httpthumbnailer/error_reporter'
 		require 'httpthumbnailer/thumbnailer'
 
-		class HTTPThumbniler < Controller
+		settings.plugins.map do |dir|
+			begin
+				dir.realpath
+			rescue Errno::ENOENT => error
+				log.warn "plugin directory '#{dir}' is not accessible: #{error}"
+				nil
+			end
+		end.compact.map do |dir|
+			Pathname::glob(dir + '**/*.rb')
+		end.each do |plugin_files|
+			plugin_files.sort.each do |plugin_file|
+				Plugin::Thumbnailer.setup_plugin_from_file(plugin_file)
+			end
+		end
+
+		class HTTPThumbnailer < Controller
 			extend Stats
 			def_stats(
 				:workers,
@@ -57,15 +85,15 @@ Application.new('httpthumbnailer', port: 3100) do
 			raindrops_stats = Raindrops::Middleware::Stats.new
 			self.use Raindrops::Middleware, stats: raindrops_stats
 
-			StatsReporter << HTTPThumbniler.stats
+			StatsReporter << HTTPThumbnailer.stats
 			StatsReporter << raindrops_stats
 			StatsReporter << Plugin::Thumbnailer::Service.stats
 			StatsReporter << Plugin::ResponseHelpers.stats
 
 			self.define do
-				HTTPThumbniler.stats.incr_total_requests
+				HTTPThumbnailer.stats.incr_total_requests
 				on error? do
-					HTTPThumbniler.stats.incr_total_errors
+					HTTPThumbnailer.stats.incr_total_errors
 					run ErrorReporter
 				end
 
@@ -87,11 +115,11 @@ Application.new('httpthumbnailer', port: 3100) do
 			end
 		end
 
-		HTTPThumbniler
+		HTTPThumbnailer
 	end
 
 	after_fork do |server, worker|
-		HTTPThumbniler.stats.incr_workers
+		HTTPThumbnailer.stats.incr_workers
 	end
 end
 

data/lib/httpthumbnailer/error_reporter.rb

@@ -12,10 +12,12 @@ class ErrorReporter < Controller
 		end
 
 		on error(
-			ThumbnailSpec::BadThubnailSpecError,
+			ThumbnailSpec::InvalidFormatError,
 			Plugin::Thumbnailer::ZeroSizedImageError,
 			Plugin::Thumbnailer::UnsupportedMethodError,
-			Plugin::Thumbnailer::InvalidColorNameError
+			Plugin::Thumbnailer::InvalidColorNameError,
+			Plugin::Thumbnailer::ThumbnailArgumentError,
+			Plugin::Thumbnailer::EditArgumentError
 		) do |error|
 			write_error 400, error
 		end

data/lib/httpthumbnailer/ownership.rb

@@ -0,0 +1,54 @@
+module Ownership
+	UseDestroyedError = Class.new(RuntimeError)
+	BorrowingDestoryedError = Class.new(RuntimeError)
+	BorrowingNotOwnedError = Class.new(RuntimeError)
+
+	def owned?
+		@owned
+	end
+
+	def borrowed?
+		@borrowed
+	end
+
+	def borrow
+		@destroyed and Kernel::raise BorrowingDestoryedError, "cannot borrow a destroyed obejct '#{self}'"
+		@owned or Kernel::raise BorrowingNotOwnedError, "cannot borrow not owned object '#{self}'"
+		was_borrowed = @borrowed
+		begin
+			@borrowed = true
+			yield self
+		ensure
+			@borrowed = was_borrowed
+		end
+	end
+
+	def get(&block)
+		if @borrowed
+			borrow(&block)
+		else
+			@destroyed and Kernel::raise UseDestroyedError, "cannot own a destoryed object '#{self}'"
+			# take ownership; it may be owned already
+			@owned = true
+			begin
+				ret = yield self
+				# give up ownership if nothing happened with the obejct
+				# NOTE: we use equal here sice == may actually test pixel by pixel which is not the point!
+				if ret.equal?(self) or ret.nil?
+					@owned = nil
+					return self
+				end
+				ret
+			ensure
+				# if I am still an owner destroy and give up ownership
+				if @owned
+					destroy!
+					@destroyed = true
+					@owned = nil
+				end
+			end
+		end
+	end
+end
+
+

data/lib/httpthumbnailer/plugin.rb

@@ -0,0 +1,87 @@
+class PluginContext
+	include ClassLogging
+	PluginArgumentError = Class.new ArgumentError
+	include PerfStats
+
+	attr_reader :thumbnailing_methods
+	attr_reader :edits
+
+	def initialize(&block)
+		@thumbnailing_methods = []
+		@edits = []
+		instance_eval(&block)
+	end
+
+	def self.from_file(file)
+		self.new do
+			instance_eval file.read, file.to_s
+		end
+	end
+
+	def thumbnailing_method(name, &block)
+		name.kind_of? String or fail "thumbnailing method name must ba a string; got: #{name.class.name}"
+		block.kind_of? Proc or fail "thumbnailing method '#{name}' needs to provide an implementation; got: #{name.class.name}"
+		@thumbnailing_methods << [name, block]
+	end
+
+	def edit(name, &block)
+		name.kind_of? String or fail "edit name must ba a string; got: #{name.class.name}"
+		block.kind_of? Proc or fail "edit '#{name}' needs to provide an implementation; got: #{name.class.name}"
+		@edits << [name, block]
+	end
+
+	def with_default(arg, default = nil)
+		return default if arg.nil? or arg == ''
+		arg
+	end
+
+	# static helpers
+	def int!(name, arg, default = nil)
+		value = with_default(arg, default) or raise PluginArgumentError, "expected argument '#{name}' to be an integer but got no value"
+		begin
+			Integer(value)
+		rescue ArgumentError
+			raise PluginArgumentError, "expected argument '#{name}' to be an integer, got: #{arg.inspect}"
+		end
+	end
+
+	def uint!(name, arg, default = nil)
+		ret = int!(name, arg, default)
+	 	ret < 0 and raise PluginArgumentError, "expected argument '#{name}' to be an unsigned integer, got negative value: #{arg}"
+		ret
+	end
+
+	def float!(name, arg, default = nil)
+		value = with_default(arg, default) or raise PluginArgumentError, "expected argument '#{name}' to be a float but got no value"
+		begin
+			Float(value)
+		rescue ArgumentError
+			raise PluginArgumentError, "expected argument '#{name}' to be a float, got: #{arg}"
+		end
+	end
+
+	def ufloat!(name, arg, default = nil)
+		ret = float!(name, arg, default)
+		ret < 0 and raise PluginArgumentError, "expected argument '#{name}' to be an unsigned float, got negative value: #{arg}"
+		ret
+	end
+
+	def offset_to_center(x, y, w, h)
+		[x + w / 2, y + h / 2]
+	end
+
+	def center_to_offset(center_x, center_y, w, h)
+		[center_x - w / 2, center_y - h / 2]
+	end
+
+	def normalize_region(x, y, width, height)
+		x = 0.0 if x < 0
+		y = 0.0 if y < 0
+		width = 1.0 - x if width + x > 1
+		height = 1.0 - y if height + y > 1
+		width = Float::EPSILON if width < 0
+		height = Float::EPSILON if height < 0
+		[x, y, width, height]
+	end
+end
+

data/lib/httpthumbnailer/plugin/thumbnailer.rb

@@ -1,37 +1,23 @@
-require 'RMagick'
 require 'forwardable'
-
-module MetaData
-	def width
-		@image.columns
-	end
-
-	def height
-		@image.rows
-	end
-
-	# ImageMagick Image.mime_type is absolutely bunkers! It goes over file system to look for some strange files WTF?!
-	# Also it cannot be used for thumbnails since they are not yet rendered to desired format
-	# Here is stupid implementation
-	def mime_type
-		#TODO: how do I do it better?
-		format = @format || @image.format
-		mime = case format
-			when 'JPG' then 'jpeg'
-			else format.downcase
-		end
-		"image/#{mime}"
-	end
-end
+require 'httpthumbnailer/plugin'
+require_relative 'thumbnailer/service'
 
 module Plugin
 	module Thumbnailer
+		include ClassLogging
+
 		class UnsupportedMethodError < ArgumentError
 			def initialize(method)
 				super("thumbnail method '#{method}' is not supported")
 			end
 		end
 
+		class UnsupportedEditError < ArgumentError
+			def initialize(name)
+				super("no edit with name '#{name}' is supported")
+			end
+		end
+
 		class UnsupportedMediaTypeError < ArgumentError
 			def initialize(error)
 				super("unsupported media type: #{error}")
@@ -56,355 +42,33 @@ module Plugin
 			end
 		end
 
-		module ImageProcessing
-			def replace
-				@use_count ||= 0
-				processed = nil
-				begin
-					processed = yield self
-					processed = self unless processed
-					fail 'got destroyed image' if processed.destroyed?
-				ensure
-					self.destroy! if @use_count <= 0 unless processed.equal? self
-				end
-				processed
-			end
-
-			def use
-				@use_count ||= 0
-				@use_count += 1
-				begin
-					yield self
-					self
-				ensure
-					@use_count -=1
-					self.destroy! if @use_count <= 0
-				end
-			end
-		end
-
-		class InputImage
-			include ClassLogging
-			extend Forwardable
-
-			def initialize(image, processing_methods, options = {})
-				@image = image
-				@processing_methods = processing_methods
-			end
-
-			def thumbnail(spec)
-				spec = spec.dup
-				# default background is white
-				spec.options['background-color'] = spec.options.fetch('background-color', 'white').sub(/^0x/, '#')
-
-				width = spec.width == :input ? @image.columns : spec.width
-				height = spec.height == :input ? @image.rows : spec.height
-
-				raise ZeroSizedImageError.new(width, height) if width == 0 or height == 0
-
-				begin
-					process_image(spec.method, width, height, spec.options).replace do |image|
-						if image.alpha?
-							log.info 'thumbnail has alpha, rendering on background'
-							image.render_on_background(spec.options['background-color'])
-						end
-					end.use do |image|
-						Service.stats.incr_total_thumbnails_created
-						image_format = spec.format == :input ? @image.format : spec.format
-
-						yield Thumbnail.new(image, image_format, spec.options)
-					end
-				rescue Magick::ImageMagickError => error
-					raise ImageTooLargeError, error.message if error.message =~ /cache resources exhausted/
-					raise
-				end
-			end
-
-			def process_image(method, width, height, options)
-				@image.replace do |image|
-					impl = @processing_methods[method] or raise UnsupportedMethodError, method
-					impl.call(image, width, height, options)
-				end
-			end
-
-			# behave as @image in processing
-			def use
-				@image.use do |image|
-					yield self
-				end
-			end
-
-			def_delegators :@image, :destroy!, :destroyed?, :format
-
-			include MetaData
-
-			# We use base values since it might have been loaded with size hint and prescaled
-			def width
-				@image.base_columns
-			end
-
-			def height
-				@image.base_rows
-			end
-
-			# needs to be seen as @image when returned in replace block
-			def equal?(image)
-				super image or @image.equal? image
-			end
-		end
-
-		class Thumbnail
-			include ClassLogging
-			extend Forwardable
-
-			def initialize(image, format, options = {})
-				@image = image
-				@format = format
-
-				@quality = (options['quality'] or default_quality(format))
-				@quality &&= @quality.to_i
-
-				@interlace = (options['interlace'] or 'NoInterlace')
-				fail "unsupported interlace: #{@interlace}" unless Magick::InterlaceType.values.map(&:to_s).include? @interlace
-				@interlace = Magick.const_get @interlace.to_sym
-			end
-
-			def_delegators :@image, :format
-
-			def data
-				# export class variables to local scope
-				format = @format
-				quality = @quality
-				interlace = @interlace
-
-				@image.to_blob do
-					self.format = format
-					self.quality = quality if quality
-					self.interlace = interlace
-				end
-			end
-
-			include MetaData
-
-			private
-
-			def default_quality(format)
-				case format
-				when /png/i
-					95 # max zlib compression, adaptive filtering (photo)
-				when /jpeg|jpg/i
-					85
-				else
-					nil
-				end
+		class ThumbnailArgumentError < ArgumentError
+			def initialize(method, msg)
+				super("error while thumbnailing with method '#{method}': #{msg}")
 			end
 		end
 
-		class Service
-			include ClassLogging
-
-			extend Stats
-			def_stats(
-				:total_images_loaded,
-				:total_images_reloaded,
-				:total_images_downscaled,
-				:total_thumbnails_created,
-				:images_loaded,
-				:max_images_loaded,
-				:max_images_loaded_worker,
-				:total_images_created,
-				:total_images_destroyed,
-				:total_images_created_from_blob,
-				:total_images_created_initialize,
-				:total_images_created_resize,
-				:total_images_created_crop,
-				:total_images_created_sample
-			)
-
-			def self.input_formats
-				Magick.formats.select do |name, mode|
-					mode.include? 'r'
-				end.keys.map(&:downcase)
-			end
-
-			def self.output_formats
-				Magick.formats.select do |name, mode|
-					mode.include? 'w'
-				end.keys.map(&:downcase)
-			end
-
-			def self.rmagick_version
-				Magick::Version
-			end
-
-			def self.magick_version
-				Magick::Magick_version
-			end
-
-			def initialize(options = {})
-				@processing_methods = {}
-				@options = options
-				@images_loaded = 0
-
-				log.info "initializing thumbnailer: #{self.class.rmagick_version} #{self.class.magick_version}"
-
-				set_limit(:area, options[:limit_area]) if options.member?(:limit_area)
-				set_limit(:memory, options[:limit_memory]) if options.member?(:limit_memory)
-				set_limit(:map, options[:limit_map]) if options.member?(:limit_map)
-				set_limit(:disk, options[:limit_disk]) if options.member?(:limit_disk)
-
-				Magick.trace_proc = lambda do |which, description, id, method|
-					case which
-					when :c
-						Service.stats.incr_images_loaded
-						@images_loaded += 1
-						Service.stats.max_images_loaded = Service.stats.images_loaded if Service.stats.images_loaded > Service.stats.max_images_loaded
-						Service.stats.max_images_loaded_worker = @images_loaded if @images_loaded > Service.stats.max_images_loaded_worker
-						Service.stats.incr_total_images_created
-						case method
-						when :from_blob
-							Service.stats.incr_total_images_created_from_blob
-						when :initialize
-							Service.stats.incr_total_images_created_initialize
-						when :resize
-							Service.stats.incr_total_images_created_resize
-						when :resize!
-							Service.stats.incr_total_images_created_resize
-						when :crop
-							Service.stats.incr_total_images_created_crop
-						when :crop!
-							Service.stats.incr_total_images_created_crop
-						when :sample
-							Service.stats.incr_total_images_created_sample
-						else
-							log.warn "uncounted image creation method: #{method}"
-						end
-					when :d
-						Service.stats.decr_images_loaded
-						@images_loaded -= 1
-						Service.stats.incr_total_images_destroyed
-					end
-					log.debug{"image event: #{which}, #{description}, #{id}, #{method}: loaded images: #{Service.stats.images_loaded}"}
-				end
-			end
-
-			def load(io, options = {})
-				mw = options[:max_width]
-				mh = options[:max_height]
-				if mw and mh
-					mw = mw.to_i
-					mh = mh.to_i
-					log.info "using max size hint of: #{mw}x#{mh}"
-				end
-
-				begin
-					blob = io.read
-
-					old_memory_limit = nil
-					borrowed_memory_limit = nil
-					if options.member?(:limit_memory)
-						borrowed_memory_limit = options[:limit_memory].borrow(options[:limit_memory].limit, 'image magick')
-						old_memory_limit = set_limit(:memory, borrowed_memory_limit)
-					end
-
-					images = Magick::Image.from_blob(blob) do |info|
-						if mw and mh
-							define('jpeg', 'size', "#{mw*2}x#{mh*2}")
-							define('jbig', 'size', "#{mw*2}x#{mh*2}")
-						end
-					end
-
-					image = images.first
-					if image.columns > image.base_columns or image.rows > image.base_rows and not options[:no_reload]
-						log.warn "input image got upscaled from: #{image.base_columns}x#{image.base_rows} to #{image.columns}x#{image.rows}: reloading without max size hint!"
-						images.each do |other|
-							other.destroy!
-						end
-						images = Magick::Image.from_blob(blob)
-						Service.stats.incr_total_images_reloaded
-					end
-					blob = nil
-
-					images.shift.replace do |image|
-						images.each do |other|
-							other.destroy!
-						end
-						log.info "loaded image: #{image.inspect}"
-						Service.stats.incr_total_images_loaded
-
-						# clean up the image
-						image.strip!
-						image.properties do |key, value|
-							log.debug "deleting user propertie '#{key}'"
-							image[key] = nil
-						end
-
-						image
-					end.replace do |image|
-						if mw and mh and not options[:no_downscale]
-							f = image.find_downscale_factor(mw, mh)
-							if f > 1
-								image = image.downscale(f)
-								log.info "downscaled image by factor of #{f}: #{image.inspect}"
-								Service.stats.incr_total_images_downscaled
-							end
-						end
-						InputImage.new(image, @processing_methods)
-					end
-				rescue Magick::ImageMagickError => error
-					raise ImageTooLargeError, error if error.message =~ /cache resources exhausted/
-					raise UnsupportedMediaTypeError, error
-				ensure
-					if old_memory_limit
-						set_limit(:memory, old_memory_limit)
-						options[:limit_memory].return(borrowed_memory_limit, 'image magick')
-					end
-				end
-			end
-
-			def processing_method(method, &impl)
-				@processing_methods[method] = impl
-			end
-
-			def set_limit(limit, value)
-				old = Magick.limit_resource(limit, value)
-				log.info "changed #{limit} limit from #{old} to #{value} bytes"
-				old
-			end
-
-			def setup_default_methods
-				processing_method('crop') do |image, width, height, options|
-					image.resize_to_fill(width, height, (Float(options['float-x']) rescue 0.5), (Float(options['float-y']) rescue 0.5)) if image.columns != width or image.rows != height
-				end
-
-				processing_method('fit') do |image, width, height, options|
-					image.resize_to_fit(width, height) if image.columns != width or image.rows != height
-				end
-
-				processing_method('pad') do |image, width, height, options|
-					image.resize_to_fit(width, height).replace do |resize|
-						resize.render_on_background(options['background-color'], width, height, (Float(options['float-x']) rescue 0.5), (Float(options['float-y']) rescue 0.5))
-					end if image.columns != width or image.rows != height
-				end
-
-				processing_method('limit') do |image, width, height, options|
-					image.resize_to_fit(width, height) if image.columns > width or image.rows > height
-				end
+		class EditArgumentError < ArgumentError
+			def initialize(name, msg)
+				super("error while applying edit '#{name}': #{msg}")
 			end
 		end
 
 		def self.setup(app)
 			Service.logger = app.logger_for(Service)
-			InputImage.logger = app.logger_for(InputImage)
-			Thumbnail.logger = app.logger_for(Thumbnail)
+			PluginContext.logger = app.logger_for(PluginContext)
 
 			@@service = Service.new(
 				limit_memory: app.settings[:limit_memory],
 				limit_map: app.settings[:limit_map],
 				limit_disk: app.settings[:limit_disk]
 			)
+			@@service.setup_built_in_plugins
+		end
 
-			@@service.setup_default_methods
+		def self.setup_plugin_from_file(file)
+			log.info("loading plugin from: #{file}")
+			@@service.load_plugin(PluginContext.from_file(file))
 		end
 
 		def thumbnailer
@@ -413,72 +77,3 @@ module Plugin
 	end
 end
 
-class Magick::Image
-	include Plugin::Thumbnailer::ImageProcessing
-
-	def render_on_background(background_color, width = nil, height = nil, float_x = 0.5, float_y = 0.5)
-		# default to image size
-		width ||= self.columns
-		height ||= self.rows
-
-		# make sure we have enough background to fit image on top of it
-		width = self.columns if width < self.columns
-		height = self.rows if height < self.rows
-
-		Magick::Image.new(width, height) {
-			begin
-				self.background_color = background_color
-			rescue ArgumentError
-				raise Plugin::Thumbnailer::InvalidColorNameError.new(background_color)
-			end
-			self.depth = 8
-		}.replace do |background|
-			background.composite!(self, *background.float_to_offset(self.columns, self.rows, float_x, float_y), Magick::OverCompositeOp)
-		end
-	end
-
-	# non coping version
-	def resize_to_fill(width, height = nil, float_x = 0.5, float_y = 0.5)
-		# default to square
-		height ||= width
-
-		return if width == columns and height == rows
-
-		scale = [width / columns.to_f, height / rows.to_f].max
-
-		resize((scale * columns).ceil, (scale * rows).ceil).replace do |image|
-			next if width == image.columns and height == image.rows
-			image.crop(*image.float_to_offset(width, height, float_x, float_y), width, height, true)
-		end
-	end
-
-	def downscale(f)
-		sample(columns / f, rows / f)
-	end
-
-	def find_downscale_factor(max_width, max_height, factor = 1)
-		new_factor = factor * 2
-		if columns / new_factor > max_width * 2 and rows / new_factor > max_height * 2
-			find_downscale_factor(max_width, max_height, factor * 2)
-		else
-			factor
-		end
-	end
-
-	def float_to_offset(float_width, float_height, float_x = 0.5, float_y = 0.5)
-		base_width = self.columns
-		base_height = self.rows
-
-		x = ((base_width - float_width) * float_x).ceil
-		y = ((base_height - float_height) * float_y).ceil
-
-		x = 0 if x < 0
-		x = (base_width - float_width) if x > (base_width - float_width)
-
-		y = 0 if y < 0
-		y = (base_height - float_height) if y > (base_height - float_height)
-
-		[x, y]
-	end
-end
-