httpimagestore 1.8.1 → 1.9.0

data/Rakefile

@@ -17,10 +17,15 @@ Jeweler::Tasks.new do |gem|
   gem.name = "httpimagestore"
   gem.homepage = "http://github.com/jpastuszek/httpimagestore"
   gem.license = "MIT"
-  gem.summary = %Q{HTTP based image storage and thumbnailer}
-  gem.description = %Q{Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)}
+  gem.summary = %Q{HTTP API server for image thumbnailing and storage}
+  gem.description = %Q{Configurable S3 or file system image storage and processing HTTP API server. It is using HTTP Thumbnailer as image processing backend.}
   gem.email = "jpastuszek@gmail.com"
   gem.authors = ["Jakub Pastuszek"]
+  gem.files.exclude "features/**/*"
+  gem.files.exclude "gatling/**/*"
+  gem.files.exclude "spec/**/*"
+  gem.files.exclude "*.gemspec"
+  gem.files.exclude ".rspec"
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-1.8.1
+1.9.0

data/bin/httpimagestore

@@ -10,7 +10,7 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
 		argument :config,
 			cast: Pathname,
 			description: 'configuration file path'
-		version (Pathname.new(__FILE__).dirname + '..' + 'VERSION').read
+		version((Pathname.new(__FILE__).dirname + '..' + 'VERSION').read)
 	end
 
 	settings do |settings|
@@ -21,6 +21,7 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
 		require 'httpimagestore/error_reporter'
 
 		class HTTPImageStore < Controller
+			include PerfStats
 			extend Stats
 			def_stats(
 				:workers,
@@ -68,48 +69,79 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
 
 				log.debug{"got request: #{env["REQUEST_METHOD"]} #{env["REQUEST_URI"]}"}
 				env['app.configuration'].handlers.each do |handler|
-					log.debug{"trying handler: #{handler.http_method}, #{handler.uri_matchers.join(', ')}"}
+					log.debug{"trying handler: #{handler}"}
 					on eval(handler.http_method), *handler.uri_matchers.map{|m| instance_eval(&m.matcher)} do |*args|
-						log.debug{"matched handler: #{handler.http_method}, #{handler.uri_matchers.join(', ')}"}
-						# map and decode matched URI segments
-						matches = {}
-						names = handler.uri_matchers
-						.map do |matcher|
-							matcher.names
-						end
-						.flatten
-
-						fail "matched more arguments than named (#{args.length} for #{names.length})" if args.length > names.length
-						fail "matched less arguments than named (#{args.length} for #{names.length})" if args.length < names.length
-
-						names.zip(args)
-						.each do |name, value|
-							fail "name should be a symbol" unless name.is_a? Symbol
-							matches[name] = URI.utf_decode(value)
-						end
-
-						# decode remaining URI components
-						path = (env["PATH_INFO"][1..-1] || '').split('/').map do |part|
-							URI.utf_decode(part)
-						end.join('/')
-
-						# query string already decoded by Rack
-						query_string = req.GET
-
-						state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit, env['xid'] || {})
-
-						handler.sources.each do |source|
-							source.realize(state) unless source.respond_to? :excluded? and source.excluded?(state)
-						end
-						handler.processors.each do |processor|
-							processor.realize(state) unless processor.respond_to? :excluded? and processor.excluded?(state)
-						end
-						handler.stores.each do |store|
-							store.realize(state) unless store.respond_to? :excluded? and store.excluded?(state)
+						log.debug{"matched handler: #{handler}"}
+						log.with_meta_context api_method: handler.http_method.upcase, api_handler: handler.to_s do
+							measure "handling request", handler.to_s do
+								# map and decode matched URI segments
+								matches = {}
+								names = handler.uri_matchers
+								.map do |matcher|
+									matcher.names
+								end
+								.flatten
+
+								fail "matched more arguments than named (#{args.length} for #{names.length})" if args.length > names.length
+								fail "matched less arguments than named (#{args.length} for #{names.length})" if args.length < names.length
+
+								names.zip(args)
+								.each do |name, value|
+									fail "name should be a symbol" unless name.is_a? Symbol
+									matches[name] = URI.utf_decode(value)
+								end
+
+								# decode remaining URI components
+								path = (env['PATH_INFO'][1..-1] || '').split('/').map do |part|
+									URI.utf_decode(part)
+								end.join('/')
+
+								# query string already decoded by Rack
+								query_string = req.GET
+
+								# actual request URI
+								request_uri = env['REQUEST_URI']
+								request_headers = env.select{|k,v| k.start_with? 'HTTP_'}.map do |pair|
+									[
+										pair[0].sub(/^HTTP_/, '').gsub('_', '-'),
+										pair[1]
+									]
+								end
+								request_headers = Hash[request_headers]
+								request_headers.delete('VERSION')
+
+								body = measure "reading request body" do
+									req.body.read
+								end
+
+								state = Configuration::RequestState.new(body, matches, path, query_string, request_uri, request_headers, memory_limit, env['xid'] || {})
+
+								measure "validating request" do
+									handler.validators.each do |validator|
+										validator.realize(state) unless validator.respond_to? :excluded? and validator.excluded?(state)
+									end
+								end unless handler.validators.empty?
+								measure "sourcing images" do
+									handler.sources.each do |source|
+										source.realize(state) unless source.respond_to? :excluded? and source.excluded?(state)
+									end
+								end
+								measure "processing images" do
+									handler.processors.each do |processor|
+										processor.realize(state) unless processor.respond_to? :excluded? and processor.excluded?(state)
+									end
+								end unless handler.processors.empty?
+								measure "storing images" do
+									handler.stores.each do |store|
+										store.realize(state) unless store.respond_to? :excluded? and store.excluded?(state)
+									end
+								end unless handler.stores.empty?
+								measure "sending response" do
+									handler.output.realize(state)
+									instance_eval(&state.output_callback)
+								end
+							end
 						end
-						handler.output.realize(state)
-
-						instance_eval &state.output_callback
 					end
 				end
 
@@ -144,6 +176,7 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
 		require 'httpimagestore/configuration/file'
 		require 'httpimagestore/configuration/output'
 		require 'httpimagestore/configuration/s3'
+		require 'httpimagestore/configuration/validate_hmac'
 
 		HTTPImageStore.use Configurator, Configuration.from_file(settings.config)
 		HTTPImageStore

data/lib/httpimagestore/configuration/file.rb

@@ -1,5 +1,5 @@
 require 'httpimagestore/configuration/path'
-require 'httpimagestore/configuration/handler'
+require 'httpimagestore/configuration/handler/source_store_base'
 require 'httpimagestore/configuration/source_failover'
 require 'pathname'
 require 'addressable/uri'
@@ -18,6 +18,7 @@ module Configuration
 	end
 
 	class FileSourceStoreBase < SourceStoreBase
+		include PerfStats
 		extend Stats
 		def_stats(
 			:total_file_store,
@@ -29,20 +30,24 @@ module Configuration
 		def self.parse(configuration, node)
 			image_name = node.grab_values('image name').first
 			node.required_attributes('root', 'path')
-			root_dir, path_spec, if_image_name_on = *node.grab_attributes('root', 'path', 'if-image-name-on')
-			matcher = InclusionMatcher.new(image_name, if_image_name_on)
 
-			self.new(
+			# TODO: it should be possible to compact that
+			root_dir, path_spec, remaining = *node.grab_attributes_with_remaining('root', 'path')
+			conditions, remaining = *ConditionalInclusion.grab_conditions_with_remaining(remaining)
+			remaining.empty? or raise UnexpectedAttributesError.new(node, remaining)
+
+			file = self.new(
 				configuration.global,
 				image_name,
-				matcher,
 				root_dir,
 				path_spec
 			)
+			file.with_conditions(conditions)
+			file
 		end
 
-		def initialize(global, image_name, matcher, root_dir, path_spec)
-			super global, image_name, matcher, path_spec
+		def initialize(global, image_name, root_dir, path_spec)
+			super(global, image_name, path_spec)
 			@root_dir = Pathname.new(root_dir).cleanpath
 		end
 
@@ -83,9 +88,11 @@ module Configuration
 
 				log.info "sourcing '#{image_name}' from file '#{storage_path}'"
 				begin
-					data = storage_path.open('rb') do |io|
-						request_state.memory_limit.io io
-						io.read
+					data = measure "sourcing image from file", image_name do
+						storage_path.open('rb') do |io|
+							request_state.memory_limit.io io
+							io.read
+						end
 					end
 					FileSourceStoreBase.stats.incr_total_file_source
 					FileSourceStoreBase.stats.incr_total_file_source_bytes(data.bytesize)
@@ -120,7 +127,9 @@ module Configuration
 				image.store_url = file_url(rendered_path)
 
 				log.info "storing '#{image_name}' in file '#{storage_path}' (#{image.data.length} bytes)"
-				storage_path.open('wb'){|io| io.write image.data}
+				measure "storing image in file", image_name do
+					storage_path.open('wb'){|io| io.write image.data}
+				end
 				FileSourceStoreBase.stats.incr_total_file_store
 				FileSourceStoreBase.stats.incr_total_file_store_bytes(image.data.bytesize)
 			end

data/lib/httpimagestore/configuration/handler.rb

@@ -1,6 +1,5 @@
-require 'mime/types'
-require 'digest/sha2'
-require 'securerandom'
+require 'httpimagestore/configuration/request_state'
+require 'httpimagestore/ruby_string_template'
 
 module Configuration
 	class ImageNotLoadedError < ConfigurationError
@@ -39,123 +38,6 @@ module Configuration
 		end
 	end
 
-	class RequestState < Hash
-		include ClassLogging
-
-		class Images < Hash
-			def initialize(memory_limit)
-				@memory_limit = memory_limit
-				super
-			end
-
-			def []=(name, image)
-				if member?(name)
-					@memory_limit.return fetch(name).data.bytesize
-				end
-				super
-			end
-
-			def [](name)
-				fetch(name){|image_name| raise ImageNotLoadedError.new(image_name)}
-			end
-		end
-
-		def initialize(body = '', matches = {}, path = '', query_string = {}, memory_limit = MemoryLimit.new, headers = {})
-			super() do |request_state, name|
-				# note that request_state may be different object when useing with_locals that creates duplicate
-				request_state[name] = request_state.generate_meta_variable(name) or raise VariableNotDefinedError.new(name)
-			end
-
-			self[:path] = path
-			merge! matches
-			self[:query_string_options] = query_string.sort.map{|kv| kv.join(':')}.join(',')
-
-			log.debug "processing request with body length: #{body.bytesize} bytes and variables: #{map{|k,v| "#{k}: '#{v}'"}.join(', ')}"
-
-			@body = body
-			@images = Images.new(memory_limit)
-			@memory_limit = memory_limit
-			@output_callback = nil
-
-			@headers = headers
-		end
-
-		attr_reader :body
-		attr_reader :images
-		attr_reader :memory_limit
-		attr_reader :headers
-
-		def with_locals(*locals)
-			locals = locals.reduce{|a, b| a.merge(b)}
-			log.debug "using additional local variables: #{locals}"
-			self.dup.merge!(locals)
-		end
-
-		def output(&callback)
-			@output_callback = callback
-		end
-
-		def output_callback
-			@output_callback or fail 'no output callback'
-		end
-
-		def fetch_base_variable(name, base_name)
-			fetch(base_name, nil) or generate_meta_variable(base_name) or raise NoVariableToGenerateMetaVariableError.new(base_name, name)
-		end
-
-		def generate_meta_variable(name)
-			log.debug  "generating meta variable: #{name}"
-			val = case name
-			when :basename
-				path = Pathname.new(fetch_base_variable(name, :path))
-				path.basename(path.extname).to_s
-			when :dirname
-				Pathname.new(fetch_base_variable(name, :path)).dirname.to_s
-			when :extension
-				Pathname.new(fetch_base_variable(name, :path)).extname.delete('.')
-			when :digest # deprecated
-				@body.empty? and raise NoRequestBodyToGenerateMetaVariableError.new(name)
-				Digest::SHA2.new.update(@body).to_s[0,16]
-			when :input_digest
-				@body.empty? and raise NoRequestBodyToGenerateMetaVariableError.new(name)
-				Digest::SHA2.new.update(@body).to_s[0,16]
-			when :input_sha256
-				@body.empty? and raise NoRequestBodyToGenerateMetaVariableError.new(name)
-				Digest::SHA2.new.update(@body).to_s
-			when :input_image_width
-				@images['input'].width or raise NoImageDataForVariableError.new('input', name)
-			when :input_image_height
-				@images['input'].height or raise NoImageDataForVariableError.new('input', name)
-			when :input_image_mime_extension
-				@images['input'].mime_extension or raise NoImageDataForVariableError.new('input', name)
-			when :image_digest
-				Digest::SHA2.new.update(@images[fetch_base_variable(name, :image_name)].data).to_s[0,16]
-			when :image_sha256
-				Digest::SHA2.new.update(@images[fetch_base_variable(name, :image_name)].data).to_s
-			when :mimeextension # deprecated
-				image_name = fetch_base_variable(name, :image_name)
-				@images[image_name].mime_extension or raise NoImageDataForVariableError.new(image_name, name)
-			when :image_mime_extension
-				image_name = fetch_base_variable(name, :image_name)
-				@images[image_name].mime_extension or raise NoImageDataForVariableError.new(image_name, name)
-			when :image_width
-				image_name = fetch_base_variable(name, :image_name)
-				@images[image_name].width or raise NoImageDataForVariableError.new(image_name, name)
-			when :image_height
-				image_name = fetch_base_variable(name, :image_name)
-				@images[image_name].height or raise NoImageDataForVariableError.new(image_name, name)
-			when :uuid
-				SecureRandom.uuid
-			end
-			if val
-				log.debug  "generated meta variable '#{name}': #{val}"
-			else
-				log.debug  "could not generated meta variable '#{name}'"
-			end
-			val
-		end
-	end
-
 	module ImageMetaData
 		attr_accessor :source_path
 		attr_accessor :source_url
@@ -163,9 +45,16 @@ module Configuration
 		attr_accessor :store_url
 
 		def mime_extension
-			return nil unless mime_type
-			mime = MIME::Types[mime_type].first
-			mime.extensions.select{|e| e.length == 3}.first or mime.extensions.first
+			case mime_type
+			when nil then nil
+			when 'image/jpeg' then 'jpg'
+			when 'image/png' then 'png'
+			when 'image/gif' then 'gif'
+			else
+				# TODO: this does not work well; the resoult may not the most common extension (like 'jpeg')
+				mime = MIME::Types[mime_type].first or return nil
+				mime.preferred_extension
+			end
 		end
 	end
 
@@ -180,114 +69,6 @@ module Configuration
 		end
 	end
 
-	class InclusionMatcher
-		def initialize(value, template)
-			@value = value
-			@template = RubyStringTemplate.new(template) if template
-		end
-
-		def included?(request_state)
-			return true if not @template
-			@template.render(request_state).split(',').include? @value
-		end
-	end
-
-	class HandlerStatement
-		module ImageName
-			attr_reader :image_name
-
-			def initialize(global, *args)
-				@image_name = args.pop
-
-				super(global, *args)
-
-				config_local :imagename, @image_name # deprecated
-				config_local :image_name, @image_name
-			end
-		end
-
-		module PathSpec
-			attr_reader :path_spec
-
-			def initialize(global, *args)
-				@path_spec = args.pop
-				super(global, *args)
-			end
-
-			def path_template
-				@global.paths[@path_spec]
-			end
-		end
-
-		module ConditionalInclusion
-			def initialize(global, *args)
-				@matchers = []
-				matcher = args.pop
-				@matchers << matcher if matcher
-				super(global, *args)
-			end
-
-			def inclusion_matcher(matcher)
-				@matchers << matcher
-			end
-
-			def included?(request_state)
-				return true if @matchers.empty?
-				@matchers.any? do |matcher|
-					matcher.included?(request_state)
-				end
-			end
-
-			def excluded?(request_state)
-				not included? request_state
-			end
-		end
-
-		def initialize(global, *args)
-			@global = global
-			@config_locals = {}
-			@module_args = args
-		end
-
-		attr_reader :config_locals
-		def config_local(name, value)
-			@config_locals[name] = value
-		end
-
-		def path_template(path_spec)
-			@global.paths[path_spec]
-		end
-	end
-
-	class SourceStoreBase < HandlerStatement
-		include ImageName
-		include PathSpec
-		include ConditionalInclusion
-
-		def initialize(global, image_name, matcher, path_spec)
-			super(global, image_name, path_spec, matcher)
-		end
-
-		private
-
-		def put_sourced_named_image(request_state)
-			rendered_path = path_template.render(request_state.with_locals(config_locals))
-
-			image = yield @image_name, rendered_path
-
-			image.source_path = rendered_path
-			request_state.images[@image_name] = image
-		end
-
-		def get_named_image_for_storage(request_state)
-			image = request_state.images[@image_name]
-			rendered_path = path_template.render(request_state.with_locals(config_locals))
-			image.store_path = rendered_path
-
-			yield @image_name, image, rendered_path
-		end
-	end
-
 	class Matcher
 		def initialize(names, debug_type = '', debug_value = '', &matcher)
 			@names = names
@@ -296,8 +77,10 @@ module Configuration
 			@debug_value = case debug_value
 			when Regexp
 				"/#{debug_value.source}/"
+			when nil
+				nil
 			else
-				debug_value.inspect
+				debug_value.to_s
 			end
 		end
 
@@ -305,15 +88,38 @@ module Configuration
 		attr_reader :matcher
 
 		def to_s
-			if @names.empty?
-				"#{@debug_type}(#{@debug_value})"
+			if @debug_value
+				if @names.empty?
+					"#{@debug_type}(#{@debug_value})"
+				else
+					"#{@debug_type}(#{@names.join(',')} => #{@debug_value})"
+				end
 			else
-				"#{@debug_type}(#{@names.join(',')} => #{@debug_value})"
+				@debug_type
 			end
 		end
 	end
 
 	class Handler < Scope
+		class HandlerConfiguration
+			def initialize(global, http_method, uri_matchers)
+				@global = global
+				@http_method = http_method
+				@uri_matchers = uri_matchers
+				@validators = []
+				@sources = []
+				@processors = []
+				@stores = []
+				@output = nil
+			end
+
+			attr_accessor :global, :http_method, :uri_matchers, :validators, :sources, :processors, :stores, :output
+
+			def to_s
+				"#{@http_method} #{@uri_matchers.join(', ')}"
+			end
+		end
+
 		def self.match(node)
 			node.name == 'put' or
 			node.name == 'post' or
@@ -325,20 +131,7 @@ module Configuration
 		end
 
 		def self.parse(configuration, node)
-			handler_configuration =
-				Struct.new(
-					:global,
-					:http_method,
-					:uri_matchers,
-					:sources,
-					:processors,
-					:stores,
-					:output
-				).new
-
-			handler_configuration.global = configuration
-			handler_configuration.http_method = node.name
-			handler_configuration.uri_matchers = node.values.map do |matcher|
+			uri_matchers = node.values.map do |matcher|
 				case matcher
 				# URI matchers
 				when %r{^:([^/]+)/(.*)/$} # :foobar/.*/
@@ -381,31 +174,29 @@ module Configuration
 				when /^\&([^=]+)=(.+)$/# ?foo=bar
 					name = $1.to_sym
 					value = $2
-					Matcher.new([name], 'QueryKeyValue', "#{name}=#{value}") do
+					Matcher.new([name], 'QueryKeyValue', "#{value}") do
 						->{req.GET[name.to_s] && req.GET[name.to_s] == value && captures.push(req.GET[name.to_s])}
 					end
 				when /^\&:(.+)\?(.*)$/# &:foo?bar
 					name = $1.to_sym
 					default = $2
-					Matcher.new([name], 'QueryKeyDefault', "#{name}=<key>|#{default}") do
+					Matcher.new([name], 'QueryKeyDefault', "<key>|#{default}") do
 						->{captures.push(req.GET[name.to_s] || default)}
 					end
 				when /^\&:(.+)$/# &:foo
 					name = $1.to_sym
-					Matcher.new([name], 'QueryKey', "#{name}=<key>") do
+					Matcher.new([name], 'QueryKey', "<key>") do
 						->{req.GET[name.to_s] && captures.push(req.GET[name.to_s])}
 					end
 				# Literal URI segment matcher
 				else # foobar
-					Matcher.new([], "Literal", matcher) do
+					Matcher.new([], matcher, nil) do
 						Regexp.escape(matcher)
 					end
 				end
 			end
-			handler_configuration.sources = []
-			handler_configuration.processors = []
-			handler_configuration.stores = []
-			handler_configuration.output = nil
+
+			handler_configuration = HandlerConfiguration.new(configuration, node.name, uri_matchers)
 
 			node.grab_attributes
 
@@ -426,6 +217,54 @@ module Configuration
 	end
 	RequestState.logger = Global.logger_for(RequestState)
 
+	class OutputText < Scope
+		def self.match(node)
+			node.name == 'output_text'
+		end
+
+		def self.parse(configuration, node)
+			configuration.output and raise StatementCollisionError.new(node, 'output')
+			text = node.grab_values('text').first
+			status, cache_control = *node.grab_attributes('status', 'cache-control')
+			configuration.output = OutputText.new(text, status || 200, cache_control)
+		end
+
+		def initialize(text, status, cache_control)
+			@text = RubyStringTemplate.new(text || fail("no text?!"))
+			@status = status || 200
+			@cache_control = cache_control
+		end
+
+		def realize(request_state)
+			# make sure variables are available in request context
+			status = @status
+			text = @text.render(request_state)
+			cache_control = @cache_control
+			request_state.output do
+				res['Cache-Control'] = cache_control if cache_control
+				write_plain status.to_i, text.to_s
+			end
+		end
+	end
+
+	class OutputOK < OutputText
+		def self.match(node)
+			node.name == 'output_ok'
+		end
+
+		def self.parse(configuration, node)
+			configuration.output and raise StatementCollisionError.new(node, 'output')
+			cache_control = node.grab_attributes('cache-control').first
+			configuration.output = OutputOK.new(cache_control)
+		end
+
+		def initialize(cache_control = nil)
+			super 'OK', 200, cache_control
+		end
+	end
+
 	Global.register_node_parser Handler
+	Handler::register_node_parser OutputText
+	Handler::register_node_parser OutputOK
 end