httpimagestore 1.6.0 → 1.7.0

data/httpimagestore.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "httpimagestore"
-  s.version = "1.6.0"
+  s.version = "1.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jakub Pastuszek"]
-  s.date = "2013-11-04"
+  s.date = "2014-07-28"
   s.description = "Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)"
   s.email = "jpastuszek@gmail.com"
   s.executables = ["httpimagestore"]
@@ -19,17 +19,6 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".document",
-    ".idea/.name",
-    ".idea/.rakeTasks",
-    ".idea/codeStyleSettings.xml",
-    ".idea/dictionaries/wcc.xml",
-    ".idea/encodings.xml",
-    ".idea/httpimagestore.iml",
-    ".idea/jenkinsSettings.xml",
-    ".idea/misc.xml",
-    ".idea/modules.xml",
-    ".idea/scopes/scope_settings.xml",
-    ".idea/vcs.xml",
     ".rspec",
     "Gemfile",
     "Gemfile.lock",
@@ -40,11 +29,13 @@ Gem::Specification.new do |s|
     "bin/httpimagestore",
     "features/cache-control.feature",
     "features/compatibility.feature",
+    "features/data-uri.feature",
     "features/error-reporting.feature",
     "features/flexi.feature",
     "features/health-check.feature",
     "features/request-matching.feature",
     "features/s3-store-and-thumbnail.feature",
+    "features/source-failover.feature",
     "features/step_definitions/httpimagestore_steps.rb",
     "features/storage.feature",
     "features/support/env.rb",
@@ -53,6 +44,8 @@ Gem::Specification.new do |s|
     "features/support/test.jpg",
     "features/support/test.png",
     "features/support/test.txt",
+    "features/support/tiny.png",
+    "features/xid-forwarding.feature",
     "httpimagestore.gemspec",
     "lib/httpimagestore/aws_sdk_regions_hack.rb",
     "lib/httpimagestore/configuration.rb",
@@ -62,6 +55,7 @@ Gem::Specification.new do |s|
     "lib/httpimagestore/configuration/output.rb",
     "lib/httpimagestore/configuration/path.rb",
     "lib/httpimagestore/configuration/s3.rb",
+    "lib/httpimagestore/configuration/source_failover.rb",
     "lib/httpimagestore/configuration/thumbnailer.rb",
     "lib/httpimagestore/error_reporter.rb",
     "lib/httpimagestore/ruby_string_template.rb",
@@ -69,12 +63,14 @@ Gem::Specification.new do |s|
     "load_test/load_test.1k.ec9bde794.m1.small.csv",
     "load_test/load_test.jmx",
     "load_test/thumbnail_specs.csv",
+    "load_test/thumbnail_specs_v2.csv",
     "spec/configuration_file_spec.rb",
     "spec/configuration_handler_spec.rb",
     "spec/configuration_identify_spec.rb",
     "spec/configuration_output_spec.rb",
     "spec/configuration_path_spec.rb",
     "spec/configuration_s3_spec.rb",
+    "spec/configuration_source_failover_spec.rb",
     "spec/configuration_spec.rb",
     "spec/configuration_thumbnailer_spec.rb",
     "spec/ruby_string_template_spec.rb",
@@ -86,15 +82,15 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/jpastuszek/httpimagestore"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.25"
+  s.rubygems_version = "1.8.23"
   s.summary = "HTTP based image storage and thumbnailer"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<unicorn-cuba-base>, ["~> 1.1.2"])
-      s.add_runtime_dependency(%q<httpthumbnailer-client>, ["~> 1.1.1"])
+      s.add_runtime_dependency(%q<unicorn-cuba-base>, ["~> 1.2.0"])
+      s.add_runtime_dependency(%q<httpthumbnailer-client>, ["~> 1.2.0"])
       s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.10"])
       s.add_runtime_dependency(%q<mime-types>, ["~> 1.17"])
       s.add_runtime_dependency(%q<sdl4r>, ["~> 0.9"])
@@ -106,10 +102,10 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<rdoc>, ["~> 3.9"])
       s.add_development_dependency(%q<daemon>, ["~> 1"])
       s.add_development_dependency(%q<prawn>, ["= 0.8.4"])
-      s.add_development_dependency(%q<httpthumbnailer>, [">= 0"])
+      s.add_development_dependency(%q<httpthumbnailer>, ["~> 1.2.0"])
     else
-      s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.1.2"])
-      s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.1.1"])
+      s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.2.0"])
+      s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.2.0"])
       s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
       s.add_dependency(%q<mime-types>, ["~> 1.17"])
       s.add_dependency(%q<sdl4r>, ["~> 0.9"])
@@ -121,11 +117,11 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rdoc>, ["~> 3.9"])
       s.add_dependency(%q<daemon>, ["~> 1"])
       s.add_dependency(%q<prawn>, ["= 0.8.4"])
-      s.add_dependency(%q<httpthumbnailer>, [">= 0"])
+      s.add_dependency(%q<httpthumbnailer>, ["~> 1.2.0"])
     end
   else
-    s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.1.2"])
-    s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.1.1"])
+    s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.2.0"])
+    s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.2.0"])
     s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
     s.add_dependency(%q<mime-types>, ["~> 1.17"])
     s.add_dependency(%q<sdl4r>, ["~> 0.9"])
@@ -137,7 +133,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rdoc>, ["~> 3.9"])
     s.add_dependency(%q<daemon>, ["~> 1"])
     s.add_dependency(%q<prawn>, ["= 0.8.4"])
-    s.add_dependency(%q<httpthumbnailer>, [">= 0"])
+    s.add_dependency(%q<httpthumbnailer>, ["~> 1.2.0"])
   end
 end
 

data/lib/httpimagestore/configuration/file.rb

@@ -1,5 +1,6 @@
 require 'httpimagestore/configuration/path'
 require 'httpimagestore/configuration/handler'
+require 'httpimagestore/configuration/source_failover'
 require 'pathname'
 require 'uri'
 
@@ -54,6 +55,10 @@ module Configuration
 
 			storage_path
 		end
+
+		def to_s
+			"FileSource[image_name: '#{@image_name}' root_dir: '#{@root_dir}' path_spec: '#{@path_spec}']"
+		end
 	end
 
 	class FileSource < FileSourceStoreBase
@@ -90,6 +95,7 @@ module Configuration
 		end
 	end
 	Handler::register_node_parser FileSource
+	SourceFailover::register_node_parser FileSource
 
 	class FileStore < FileSourceStoreBase
 		include ClassLogging

data/lib/httpimagestore/configuration/handler.rb

@@ -60,7 +60,7 @@ module Configuration
 			end
 		end
 
-		def initialize(body = '', matches = {}, path = '', query_string = {}, memory_limit = MemoryLimit.new)
+		def initialize(body = '', matches = {}, path = '', query_string = {}, memory_limit = MemoryLimit.new, headers = {})
 			super() do |request_state, name|
 				# note that request_state may be different object when useing with_locals that creates duplicate
 				request_state[name] = request_state.generate_meta_variable(name) or raise VariableNotDefinedError.new(name)
@@ -76,11 +76,14 @@ module Configuration
 			@images = Images.new(memory_limit)
 			@memory_limit = memory_limit
 			@output_callback = nil
+
+			@headers = headers
 		end
 
 		attr_reader :body
 		attr_reader :images
 		attr_reader :memory_limit
+		attr_reader :headers
 
 		def with_locals(locals)
 			log.debug "using additional local variables: #{locals}"

data/lib/httpimagestore/configuration/identify.rb

@@ -8,7 +8,7 @@ module Configuration
 
 		extend Stats
 		def_stats(
-			:total_identify_requests, 
+			:total_identify_requests,
 			:total_identify_requests_bytes
 		)
 
@@ -42,7 +42,7 @@ module Configuration
 			Identify.stats.incr_total_identify_requests
 			Identify.stats.incr_total_identify_requests_bytes image.data.bytesize
 
-			id = client.identify(image.data)
+			id = client.with_headers(request_state.headers).identify(image.data)
 
 			image.mime_type = id.mime_type if id.mime_type
 			image.width = id.width if id.width

data/lib/httpimagestore/configuration/output.rb

@@ -1,6 +1,7 @@
 require 'httpimagestore/configuration/handler'
 require 'httpimagestore/ruby_string_template'
 require 'uri'
+require 'base64'
 
 module Configuration
 	class StorePathNotSetForImage < ConfigurationError
@@ -131,7 +132,7 @@ module Configuration
 			configuration.output and raise StatementCollisionError.new(node, 'output')
 			image_name = node.grab_values('image name').first
 			cache_control = node.grab_attributes('cache-control').first
-			configuration.output = OutputImage.new(image_name, cache_control)
+			configuration.output = self.new(image_name, cache_control)
 		end
 
 		def initialize(name, cache_control)
@@ -158,6 +159,24 @@ module Configuration
 	end
 	Handler::register_node_parser OutputImage
 
+	class OutputDataURIImage < OutputImage
+		def self.match(node)
+			node.name == 'output_data_uri_image'
+		end
+
+		def realize(request_state)
+			image = request_state.images[@name]
+			fail "image '#{@name}' needs to be identified first to be used in data URI output" unless image.mime_type
+
+			cache_control = @cache_control
+			request_state.output do
+				res['Cache-Control'] = cache_control if cache_control
+				write 200, 'text/uri-list', "data:#{image.mime_type};base64,#{Base64.strict_encode64(image.data)}"
+			end
+		end
+	end
+	Handler::register_node_parser OutputDataURIImage
+
 	class OutputStorePath < OutputMultiBase
 		def self.match(node)
 			node.name == 'output_store_path'

data/lib/httpimagestore/configuration/s3.rb

@@ -4,6 +4,7 @@ require 'msgpack'
 require 'httpimagestore/aws_sdk_regions_hack'
 require 'httpimagestore/configuration/path'
 require 'httpimagestore/configuration/handler'
+require 'httpimagestore/configuration/source_failover'
 
 module Configuration
 	class S3NotConfiguredError < ConfigurationError
@@ -43,7 +44,7 @@ module Configuration
 			node.grab_values
 			node.required_attributes('key', 'secret')
 			node.valid_attribute_values('ssl', true, false, nil)
-			
+
 			key, secret, ssl = node.grab_attributes('key', 'secret', 'ssl')
 			ssl = true if ssl.nil?
 
@@ -191,7 +192,7 @@ module Configuration
 		class CacheObject < S3Object
 			extend Stats
 			def_stats(
-				:total_s3_cache_hits, 
+				:total_s3_cache_hits,
 				:total_s3_cache_misses,
 				:total_s3_cache_errors,
 			)
@@ -274,7 +275,7 @@ module Configuration
 
 		extend Stats
 		def_stats(
-			:total_s3_store, 
+			:total_s3_store,
 			:total_s3_store_bytes,
 			:total_s3_source,
 			:total_s3_source_bytes
@@ -286,18 +287,18 @@ module Configuration
 			node.required_attributes('bucket', 'path')
 			node.valid_attribute_values('public_access', true, false, nil)
 
-			bucket, path_spec, public_access, cache_control, prefix, cache_root, if_image_name_on = 
+			bucket, path_spec, public_access, cache_control, prefix, cache_root, if_image_name_on =
 				*node.grab_attributes('bucket', 'path', 'public', 'cache-control', 'prefix', 'cache-root', 'if-image-name-on')
 			public_access = false if public_access.nil?
 			prefix = '' if prefix.nil?
 
 			self.new(
-				configuration.global, 
-				image_name, 
+				configuration.global,
+				image_name,
 				InclusionMatcher.new(image_name, if_image_name_on),
-				bucket, 
-				path_spec, 
-				public_access, 
+				bucket,
+				path_spec,
+				public_access,
 				cache_control,
 				prefix,
 				cache_root
@@ -395,8 +396,13 @@ module Configuration
 				end
 			end
 		end
+
+		def to_s
+			"S3Source[image_name: '#{@image_name}' bucket: '#{@bucket}' prefix: '#{@prefix}' path_spec: '#{@path_spec}']"
+		end
 	end
 	Handler::register_node_parser S3Source
+	SourceFailover::register_node_parser S3Source
 
 	class S3Store < S3SourceStoreBase
 		def self.match(node)

data/lib/httpimagestore/configuration/source_failover.rb

@@ -0,0 +1,51 @@
+require 'httpimagestore/configuration/handler'
+
+module Configuration
+	class SourceFailoverAllFailedError < ConfigurationError
+		attr_reader :sources, :errors
+
+		def initialize(sources, errors)
+			@sources = sources
+			@errors = errors
+			super "all sources failed: #{sources.zip(errors).map{|s, e| "#{s}(#{e.class.name}: #{e.message})"}.join(', ')}"
+		end
+	end
+
+	class SourceFailover < Scope
+		include ClassLogging
+
+		def self.match(node)
+			node.name == 'source_failover'
+		end
+
+		def self.parse(configuration, node)
+			# support only sources
+			handler_configuration = Struct.new(
+				:global,
+				:sources
+			).new
+			handler_configuration.global = configuration.global
+			handler_configuration.sources = []
+
+			failover = self.new(handler_configuration)
+			configuration.sources << failover
+			failover.parse(node)
+		end
+
+		def realize(request_state)
+			errors = []
+			@configuration.sources.each do |source|
+				begin
+					log.debug "trying source: #{source}"
+					return source.realize(request_state) unless source.respond_to? :excluded? and source.excluded?(request_state)
+				rescue => error
+					errors << error
+					log.warn "source #{source} failed; trying next source", error
+				end
+			end
+			log.error "all sources: #{@configuration.sources.map(&:to_s).join(', ')} failed; giving up"
+			raise SourceFailoverAllFailedError.new(@configuration.sources.to_a, errors)
+		end
+	end
+	Handler::register_node_parser SourceFailover
+end

data/lib/httpimagestore/configuration/thumbnailer.rb

@@ -36,7 +36,7 @@ module Configuration
 
 		extend Stats
 		def_stats(
-			:total_thumbnail_requests, 
+			:total_thumbnail_requests,
 			:total_thumbnail_requests_bytes,
 			:total_thumbnail_thumbnails,
 			:total_thumbnail_thumbnails_bytes
@@ -134,9 +134,9 @@ module Configuration
 			matcher = InclusionMatcher.new(source_image_name, node.grab_attributes('if-image-name-on').first) if use_multipart_api
 
 			configuration.processors << self.new(
-				configuration.global, 
-				source_image_name, 
-				specs, 
+				configuration.global,
+				source_image_name,
+				specs,
 				use_multipart_api,
 				matcher
 			)
@@ -177,7 +177,7 @@ module Configuration
 				logger = log
 
 				begin
-					thumbnails = client.thumbnail(source_image.data) do
+					thumbnails = client.with_headers(request_state.headers).thumbnail(source_image.data) do
 						rendered_specs.each_pair do |name, spec|
 							begin
 								thumbnail(*spec)
@@ -202,7 +202,7 @@ module Configuration
 					if thumbnail.kind_of? HTTPThumbnailerClient::HTTPThumbnailerClientError
 						error = thumbnail
 						log.warn 'got single thumbnail error', error
-						raise ThumbnailingError.new(@source_image_name, name, error) 
+						raise ThumbnailingError.new(@source_image_name, name, error)
 					end
 				end
 
@@ -215,7 +215,7 @@ module Configuration
 				log.info "thumbnailing '#{@source_image_name}' to '#{name}' with spec: #{rendered_spec}"
 
 				begin
-					thumbnail = client.thumbnail(source_image.data, *rendered_spec)
+					thumbnail = client.with_headers(request_state.headers).thumbnail(source_image.data, *rendered_spec)
 					request_state.memory_limit.borrow(thumbnail.data.bytesize, "thumbnail '#{name}'")
 
 					input_mime_type = thumbnail.input_mime_type

data/lib/httpimagestore/error_reporter.rb

@@ -1,4 +1,4 @@
-class ErrorReporter < Controler
+class ErrorReporter < Controller
 	self.define do
 		on error(
 			Configuration::S3NoSuchKeyError,
@@ -20,6 +20,14 @@ class ErrorReporter < Controler
 			write_error 400, error
 		end
 
+		on error Configuration::SourceFailoverAllFailedError do |error|
+			if [Configuration::S3NoSuchKeyError, Configuration::NoSuchFileError].member? error.errors.first.class
+				write_error 404, error
+			else
+				write_error 500, error
+			end
+		end
+
 		run DefaultErrorReporter
 	end
 end

data/lib/httpimagestore/ruby_string_template.rb

@@ -4,22 +4,27 @@ class RubyStringTemplate
 			super "no value for '\#{#{name}}' in template '#{template}'"
 		end
 	end
-	
+
 	def initialize(template, &resolver)
 		@template = template.to_s
 		@resolver = resolver ? resolver : ->(locals, name){locals[name]}
 	end
 
 	def render(locals = {})
-		template = @template
-		while tag = template.match(/(#\{[^\}]+\})/m)
-			tag = tag.captures.first
-			name = tag.match(/#\{([^\}]*)\}/).captures.first.to_sym
+		template = @template.dup
+		values = {}
+
+		template.scan(/#\{[^\}]+\}/um).uniq.each do |placeholder|
+			name = placeholder.match(/#\{([^\}]*)\}/u).captures.first.to_sym
 			value = @resolver.call(locals, name)
 			value or fail NoValueForTemplatePlaceholderError.new(name, @template)
-			value = value.to_s
-			template = template.gsub(tag, value)
+			values[placeholder] = value.to_s
 		end
+
+		values.each_pair do |placeholder, value|
+			template.gsub!(placeholder, value)
+		end
+
 		template
 	end
 end