httpimagestore 1.8.1 → 1.9.0

data/lib/httpimagestore/configuration/s3.rb

@@ -4,7 +4,7 @@ require 'msgpack'
 require 'addressable/uri'
 require 'httpimagestore/aws_sdk_regions_hack'
 require 'httpimagestore/configuration/path'
-require 'httpimagestore/configuration/handler'
+require 'httpimagestore/configuration/handler/source_store_base'
 require 'httpimagestore/configuration/source_failover'
 
 module Configuration
@@ -123,7 +123,7 @@ module Configuration
 							io.write [header.length].pack('L') # header length
 							io.write header
 							io.write data
-						rescue => error
+						rescue
 							unlink # remove broken cache file
 							raise
 						end
@@ -296,15 +296,17 @@ module Configuration
 			node.required_attributes('bucket', 'path')
 			node.valid_attribute_values('public_access', true, false, nil)
 
-			bucket, path_spec, public_access, cache_control, prefix, cache_root, if_image_name_on =
-				*node.grab_attributes('bucket', 'path', 'public', 'cache-control', 'prefix', 'cache-root', 'if-image-name-on')
+			bucket, path_spec, public_access, cache_control, prefix, cache_root, remaining =
+				*node.grab_attributes_with_remaining('bucket', 'path', 'public', 'cache-control', 'prefix', 'cache-root')
+			conditions, remaining = *ConditionalInclusion.grab_conditions_with_remaining(remaining)
+			remaining.empty? or raise UnexpectedAttributesError.new(node, remaining)
+
 			public_access = false if public_access.nil?
 			prefix = '' if prefix.nil?
 
-			self.new(
+			s3 = self.new(
 				configuration.global,
 				image_name,
-				InclusionMatcher.new(image_name, if_image_name_on),
 				bucket,
 				path_spec,
 				public_access,
@@ -312,10 +314,13 @@ module Configuration
 				prefix,
 				cache_root
 			)
+			s3.with_conditions(conditions)
+			s3
 		end
 
-		def initialize(global, image_name, matcher, bucket, path_spec, public_access, cache_control, prefix, cache_root)
-			super global, image_name, matcher, path_spec
+		def initialize(global, image_name, bucket, path_spec, public_access, cache_control, prefix, cache_root)
+			super(global, image_name, path_spec)
+
 			@bucket = bucket
 			@public_access = public_access
 			@cache_control = cache_control
@@ -379,6 +384,8 @@ module Configuration
 	end
 
 	class S3Source < S3SourceStoreBase
+		include PerfStats
+
 		def self.match(node)
 			node.name == 'source_s3'
 		end
@@ -391,16 +398,18 @@ module Configuration
 			put_sourced_named_image(request_state) do |image_name, rendered_path|
 				log.info "sourcing '#{image_name}' image from S3 '#{@bucket}' bucket under '#{rendered_path}' key"
 
-				object(rendered_path) do |object|
-					data = request_state.memory_limit.get do |limit|
-						object.read(limit + 1)
-					end
-					S3SourceStoreBase.stats.incr_total_s3_source
-					S3SourceStoreBase.stats.incr_total_s3_source_bytes(data.bytesize)
+				measure "sourcing image from S3", "image: '#{image_name}' bucket: '#{@bucket}'" do
+					object(rendered_path) do |object|
+						data = request_state.memory_limit.get do |limit|
+							object.read(limit + 1)
+						end
+						S3SourceStoreBase.stats.incr_total_s3_source
+						S3SourceStoreBase.stats.incr_total_s3_source_bytes(data.bytesize)
 
-					image = Image.new(data, object.content_type)
-					image.source_url = url(object)
-					image
+						image = Image.new(data, object.content_type)
+						image.source_url = url(object)
+						image
+					end
 				end
 			end
 		end
@@ -413,6 +422,8 @@ module Configuration
 	SourceFailover::register_node_parser S3Source
 
 	class S3Store < S3SourceStoreBase
+		include PerfStats
+
 		def self.match(node)
 			node.name == 'store_s3'
 		end
@@ -426,21 +437,22 @@ module Configuration
 				acl = @public_access ?  :public_read : :private
 
 				log.info "storing '#{image_name}' image in S3 '#{@bucket}' bucket under '#{rendered_path}' key with #{acl} access"
+				measure "storing image in S3", "image: '#{image_name}' bucket: '#{@bucket}'" do
+					object(rendered_path) do |object|
+						image.mime_type or log.warn "storing '#{image_name}' in S3 '#{@bucket}' bucket under '#{rendered_path}' key with unknown mime type"
 
-				object(rendered_path) do |object|
-					image.mime_type or log.warn "storing '#{image_name}' in S3 '#{@bucket}' bucket under '#{rendered_path}' key with unknown mime type"
+						options = {}
+						options[:single_request] = true
+						options[:content_type] = image.mime_type if image.mime_type
+						options[:acl] = acl
+						options[:cache_control] = @cache_control if @cache_control
 
-					options = {}
-					options[:single_request] = true
-					options[:content_type] = image.mime_type if image.mime_type
-					options[:acl] = acl
-					options[:cache_control] = @cache_control if @cache_control
+						object.write(image.data, options)
+						S3SourceStoreBase.stats.incr_total_s3_store
+						S3SourceStoreBase.stats.incr_total_s3_store_bytes(image.data.bytesize)
 
-					object.write(image.data, options)
-					S3SourceStoreBase.stats.incr_total_s3_store
-					S3SourceStoreBase.stats.incr_total_s3_store_bytes(image.data.bytesize)
-
-					image.store_url = url(object)
+						image.store_url = url(object)
+					end
 				end
 			end
 		end

data/lib/httpimagestore/configuration/thumbnailer.rb

@@ -1,6 +1,6 @@
 require 'httpthumbnailer-client'
 require 'httpimagestore/ruby_string_template'
-require 'httpimagestore/configuration/handler'
+require 'httpimagestore/configuration/handler/statement'
 
 module Configuration
 	class Thumnailer
@@ -25,20 +25,49 @@ module Configuration
 	end
 	Global.register_node_parser Thumnailer
 
-	class NoValueForSpecTemplatePlaceholerError < ConfigurationError
+	class NoValueForSpecTemplatePlaceholderError < ConfigurationError
 		def initialize(image_name, spec_name, value_name, template)
 			super "cannot generate specification for thumbnail '#{image_name}': cannot generate value for attribute '#{spec_name}' from template '#{template}': no value for \#{#{value_name}}"
 		end
 	end
 
+	class UnknownThumbnailingDirectiveError < ConfigurationError
+		def initialize(source_image_name, image_name, directive)
+			super "unknown directive '#{directive}' for thumbnail specification '#{image_name}' for image '#{source_image_name}'"
+		end
+	end
+
 	class NoSpecSelectedError < RuntimeError
 		def initialize(specs)
-			super "no thumbnailing specs were selected, please use at least one of: #{specs.join(', ')}"
+			super "no thumbnail specs were selected, please use at least one of: #{specs.join(', ')}"
+		end
+	end
+
+	class InvalidSpecError < RuntimeError
+		def initialize(spec_name, cause)
+			super "thumbnail spec '#{spec_name}' is invalid: #{cause}"
+		end
+	end
+
+	class InvalidOptionsSpecError < InvalidSpecError
+		def initialize(spec_name, spec, cause)
+			super spec_name, "options spec '#{spec}' format is invalid: #{cause}"
+		end
+	end
+
+	class InvalidEditsSpecError < InvalidSpecError
+		def initialize(spec_name, spec, cause)
+			super spec_name, "edits spec '#{spec}' format is invalid: #{cause}"
 		end
 	end
 
 	class Thumbnail < HandlerStatement
 		include ClassLogging
+		include ImageName
+		include LocalConfiguration
+		include GlobalConfiguration
+		include ConditionalInclusion
+		include PerfStats
 
 		extend Stats
 		def_stats(
@@ -63,33 +92,89 @@ module Configuration
 
 		class ThumbnailSpec < HandlerStatement
 			include ImageName
+			include LocalConfiguration
 			include ConditionalInclusion
 
-			def initialize(image_name, method, width, height, format, options = {}, matcher = nil)
-				super(nil, image_name, matcher)
-				@method = method.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholerError.new(image_name, 'method', key, method)}
-				@width =  width.to_s.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholerError.new(image_name, 'width', key, width)}
-				@height = height.to_s.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholerError.new(image_name, 'height', key, height)}
-				@format = format.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholerError.new(image_name, 'format', key, format)}
+			class EditSpec
+				include HandlerStatement::ConditionalInclusion
+
+				def initialize(name, args, options, edit_no)
+					@name = name
+					@args = args.map.with_index do |arg, arg_no|
+						arg.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholderError.new(image_name, "edit #{edit_no + 1} argument #{arg_no + 1}", key, arg)}
+					end
+
+					@options = options.merge(options) do |option, old, template|
+						template.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholderError.new(image_name, "edit #{edit_no + 1} option '#{option}' value", key, arg)}
+					end
+				end
+
+				def render(request_state = {})
+					args = @args.map.with_index do |arg, arg_no|
+						arg.render(request_state)
+					end
+
+					options = @options.merge(@options) do |option, old, template|
+						template.render(request_state)
+					end
+
+					HTTPThumbnailerClient::ThumbnailSpec::EditSpec.new(@name, args, options)
+				end
+			end
+
+			def initialize(image_name, method, width, height, format, options = {}, edits = [])
+				with_image_name(image_name)
+				@method = method.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholderError.new(image_name, 'method', key, method)}
+				@width =  width.to_s.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholderError.new(image_name, 'width', key, width)}
+				@height = height.to_s.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholderError.new(image_name, 'height', key, height)}
+				@format = format.to_template.with_missing_resolver{|locals, key| raise NoValueForSpecTemplatePlaceholderError.new(image_name, 'format', key, format)}
 
 				@options = options.merge(options) do |option, old, template|
-					template.to_s.to_template.with_missing_resolver{|locals, field| raise NoValueForSpecTemplatePlaceholerError.new(image_name, option, field, template)}
+					template.to_s.to_template.with_missing_resolver{|locals, field| raise NoValueForSpecTemplatePlaceholderError.new(image_name, option, field, template)}
 				end
+
+				@edits = edits
 			end
 
-			def render(locals = {})
-				options = @options.inject({}){|h, v| h[v.first] = v.last.render(locals); h}
-				nested_options = options['options'] ? Hash[options.delete('options').to_s.split(',').map{|pair| pair.split(':', 2)}] : {}
-				{
-					image_name =>
-						[
-							@method.render(locals),
-							@width.render(locals),
-							@height.render(locals),
-							@format.render(locals),
-							nested_options.merge(options)
-						]
-				}
+			def render(request_state = {})
+				options = @options.inject({}){|h, v| h[v.first] = v.last.render(request_state); h}
+
+				# NOTE: normally options will be passed as options=String; but may be supplied each by each as in the configuration with key=value pairs
+				nested_options = begin
+					opts = options.delete('options') || ''
+					HTTPThumbnailerClient::ThumbnailSpec.parse_options(HTTPThumbnailerClient::ThumbnailSpec.split_args(opts))
+				rescue HTTPThumbnailerClient::ThumbnailSpec::InvalidFormatError => error
+					raise InvalidOptionsSpecError.new(image_name, opts, error)
+				end
+
+				edits_option = HTTPThumbnailerClient::ThumbnailSpec.split_edits(options.delete('edits') || '').map do |edit|
+					begin
+						HTTPThumbnailerClient::ThumbnailSpec::EditSpec.from_string(edit)
+					rescue HTTPThumbnailerClient::ThumbnailSpec::InvalidFormatError => error
+						raise InvalidEditsSpecError.new(image_name, edit, error)
+					end
+				end
+
+				edits = @edits.select do |edit|
+					edit.included?(request_state)
+				end.map do |edit|
+					edit.render(request_state)
+				end
+
+				spec = begin
+					HTTPThumbnailerClient::ThumbnailSpec.new(
+						@method.render(request_state),
+						@width.render(request_state),
+						@height.render(request_state),
+						@format.render(request_state),
+						nested_options.merge(options),
+						edits | edits_option
+					)
+				rescue HTTPThumbnailerClient::ThumbnailSpec::InvalidFormatError => error
+					raise InvalidSpecError.new(image_name, error)
+				end
+
+				Struct.new(:name, :spec).new(image_name, spec)
 			end
 		end
 
@@ -98,13 +183,19 @@ module Configuration
 		end
 
 		def self.parse(configuration, node)
+			# if we don't have any subnodes use this node as single subnode and parse as if they were subnodes
 			use_multipart_api = node.values.length == 1 ? true : false
 
-			nodes = use_multipart_api ?  node.children : [node]
+			nodes = use_multipart_api ? node.children : [node]
 			source_image_name = use_multipart_api ? node.grab_values('source image name').first : nil # parsed later
 
+			general_conditions = []
+			if use_multipart_api
+				general_conditions, remaining = *ConditionalInclusion.grab_conditions_with_remaining(node.attributes)
+				remaining.empty? or raise UnexpectedAttributesError.new(node, remaining)
+			end
+
 			nodes.empty? and raise NoValueError.new(node, 'thumbnail image name')
-			matcher = nil
 
 			specs = nodes.map do |node|
 				if use_multipart_api
@@ -113,9 +204,26 @@ module Configuration
 					source_image_name, image_name = *node.grab_values('source image name', 'thumbnail image name')
 				end
 
-				operation, width, height, format, if_image_name_on, remaining = *node.grab_attributes_with_remaining('operation', 'width', 'height', 'format', 'if-image-name-on')
+				operation, width, height, format, remaining = *node.grab_attributes_with_remaining('operation', 'width', 'height', 'format')
+
+				conditions, remaining = *ConditionalInclusion.grab_conditions_with_remaining(remaining)
+
+				edits = []
+				# check for subnodes
+				subnodes = node.children.group_by{|node| node.name}
+				edit_nodes = subnodes.delete('edit')
+				edit_nodes and edit_nodes.each.with_index do |node, edit_no|
+					edit_conditions, edit_options = *ConditionalInclusion.grab_conditions_with_remaining(node.attributes)
+
+					args = node.values.dup
+
+					edit = ThumbnailSpec::EditSpec.new(args.shift, args, edit_options, edit_no)
+					edit.with_conditions(edit_conditions)
+
+					edits << edit
+				end
 
-				matcher = InclusionMatcher.new(image_name, if_image_name_on) if if_image_name_on
+				subnodes.keys.empty? or raise UnknownThumbnailingDirectiveError.new(source_image_name, image_name, subnodes.keys.join(' and '))
 
 				ThumbnailSpec.new(
 					image_name,
@@ -124,41 +232,45 @@ module Configuration
 					height || 'input',
 					format || 'input',
 					remaining || {},
-					matcher
-				)
+					edits
+				).with_conditions(conditions)
 			end
 
-			matcher = InclusionMatcher.new(source_image_name, node.grab_attributes('if-image-name-on').first) if use_multipart_api
-
-			configuration.processors << self.new(
+			thum = self.new(
 				configuration.global,
 				source_image_name,
 				specs,
 				use_multipart_api,
-				matcher
 			)
+			thum.with_conditions(general_conditions)
+			configuration.processors << thum
 		end
 
-		include ConditionalInclusion
-
-		def initialize(global, source_image_name, specs, use_multipart_api, matcher)
-			super(global, matcher)
+		def initialize(global, source_image_name, specs, use_multipart_api)
+			with_global_configuration(global)
+			with_image_name(source_image_name)
 			@source_image_name = source_image_name
-			@specs = specs
+			@specs = specs.freeze
 			@use_multipart_api = use_multipart_api
 		end
 
 		def realize(request_state)
 			client = @global.thumbnailer or fail 'thumbnailer configuration'
 
-			rendered_specs = {}
-			@specs.select do |spec|
+			specs = @specs.select do |spec|
 				spec.included?(request_state)
-			end.each do |spec|
-				rendered_specs.merge! spec.render(request_state)
 			end
 
-			rendered_specs.empty? and raise NoSpecSelectedError.new(@specs.map(&:image_name))
+			if specs.empty?
+				# in single part form we are OK to skip the thumbnailing all thogether
+				return if not @use_multipart_api
+				# in multipart for at least one thumbnail spec should be selected
+				raise NoSpecSelectedError.new(@specs.map(&:image_name))
+			end
+
+			rendered_specs = specs.map do |spec|
+				spec.render(request_state)
+			end
 
 			source_image = request_state.images[@source_image_name]
 
@@ -167,69 +279,38 @@ module Configuration
 			input_width = nil
 			input_height = nil
 
+			log.info "thumbnailing '#{@source_image_name}' to specs: #{rendered_specs.map{|s| "#{s.name} -> #{s.spec}"}.join('; ')}"
 			Thumbnail.stats.incr_total_thumbnail_requests
 			Thumbnail.stats.incr_total_thumbnail_requests_bytes source_image.data.bytesize
 
-			if @use_multipart_api
-				log.info "thumbnailing '#{@source_image_name}' to multiple specs: #{rendered_specs}"
-
-				# need to reference to local so they are available within thumbnail() block context
-				source_image_name = @source_image_name
-				logger = log
-
-				begin
-					thumbnails = client.with_headers(request_state.headers).thumbnail(source_image.data) do
-						rendered_specs.each_pair do |name, spec|
-							begin
-								thumbnail(*spec)
-							rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
-								logger.warn 'got thumbnailer error while passing specs', error
-								raise ThumbnailingError.new(source_image_name, name, error)
-							end
-						end
-					end
-				rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
-					logger.warn 'got thumbnailer error while sending input data', error
-					raise ThumbnailingError.new(source_image_name, nil, error)
-				end
-
-				input_mime_type = thumbnails.input_mime_type
-				input_width = thumbnails.input_width
-				input_height = thumbnails.input_height
-
-				# check each thumbnail for errors
-				thumbnails = Hash[rendered_specs.keys.zip(thumbnails)]
-				thumbnails.each do |name, thumbnail|
-					if thumbnail.kind_of? HTTPThumbnailerClient::HTTPThumbnailerClientError
-						error = thumbnail
-						log.warn 'got single thumbnail error', error
-						raise ThumbnailingError.new(@source_image_name, name, error)
-					end
+			thumbnails = begin
+				measure "thumbnailing image", "'#{@source_image_name}' to specs: #{rendered_specs.map{|s| "#{s.name} -> #{s.spec}"}.join('; ')}" do
+					client.with_headers(request_state.forward_headers).thumbnail(source_image.data, *rendered_specs.map(&:spec))
 				end
+			rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
+				log.warn 'got thumbnailer error', error
+				raise ThumbnailingError.new(@source_image_name, rendered_specs.length == 1 ? rendered_specs.first.name : nil, error)
+			end
 
-				# borrow from memory limit - note that we might have already used too much memory
-				thumbnails.each do |name, thumbnail|
-					request_state.memory_limit.borrow(thumbnail.data.bytesize, "thumbnail '#{name}'")
-				end
-			else
-				name, rendered_spec = *rendered_specs.first
-				log.info "thumbnailing '#{@source_image_name}' to '#{name}' with spec: #{rendered_spec}"
-
-				begin
-					thumbnail = client.with_headers(request_state.headers).thumbnail(source_image.data, *rendered_spec)
-					request_state.memory_limit.borrow(thumbnail.data.bytesize, "thumbnail '#{name}'")
-
-					input_mime_type = thumbnail.input_mime_type
-					input_width = thumbnail.input_width
-					input_height = thumbnail.input_height
+			input_mime_type = thumbnails.input_mime_type
+			input_width = thumbnails.input_width
+			input_height = thumbnails.input_height
 
-					thumbnails[name] = thumbnail
-				rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
-					log.warn 'got thumbnailer error', error
+			# check each thumbnail for errors
+			thumbnails = Hash[rendered_specs.map(&:name).zip(thumbnails)]
+			thumbnails.each do |name, thumbnail|
+				if thumbnail.kind_of? HTTPThumbnailerClient::HTTPThumbnailerClientError
+					error = thumbnail
+					log.warn 'got single thumbnail error', error
 					raise ThumbnailingError.new(@source_image_name, name, error)
 				end
 			end
 
+			# borrow from memory limit - note that we might have already used too much memory
+			thumbnails.each do |name, thumbnail|
+				request_state.memory_limit.borrow(thumbnail.data.bytesize, "thumbnail '#{name}'")
+			end
+
 			# copy input source path and url
 			thumbnails.each do |name, thumbnail|
 				thumbnail.extend ImageMetaData
@@ -247,7 +328,19 @@ module Configuration
 
 			request_state.images.merge! thumbnails
 		end
+
+		def included?(request_state)
+			# TODO this is to complicated!
+			if @use_multipart_api
+				super(request_state)
+			else
+				@specs.any? do |spec|
+					spec.included?(request_state)
+				end
+			end
+		end
 	end
+
 	Handler::register_node_parser Thumbnail
 	StatsReporter << Thumbnail.stats
 end