httpimagestore 0.5.0 → 1.0.0

data/lib/httpimagestore/configuration/path.rb

@@ -0,0 +1,77 @@
+require 'httpimagestore/ruby_string_template'
+require 'digest/sha2'
+
+module Configuration
+	class PathNotDefinedError < ConfigurationError
+		def initialize(path_name)
+			super "path '#{path_name}' not defined"
+		end
+	end
+
+	class PathRenderingError < ConfigurationError
+		def initialize(path_name, template, message)
+			super "cannot generate path '#{path_name}' from template '#{template}': #{message}"
+		end
+	end
+
+	class NoValueForPathTemplatePlaceholerError < PathRenderingError
+		def initialize(path_name, template, value_name)
+			super path_name, template, "no value for '\#{#{value_name}}'"
+		end
+	end
+
+	class NoMetaValueForPathTemplatePlaceholerError < PathRenderingError
+		def initialize(path_name, template, value_name, meta_value)
+			super path_name, template, "need '#{value_name}' to generate value for '\#{#{meta_value}}'"
+		end
+	end
+
+	class Path < RubyStringTemplate
+		def self.match(node)
+			node.name == 'path'
+		end
+
+		def self.pre(configuration)
+			configuration.paths ||= Hash.new{|hash, path_name| raise PathNotDefinedError.new(path_name)}
+		end
+
+		def self.parse(configuration, node)
+			nodes = []
+			nodes << node unless node.values.empty?
+			nodes |= node.children
+
+			nodes.empty? and raise NoValueError.new(node, 'path name')
+			nodes.each do |node|
+				path_name, template = *node.grab_values('path name', 'path template')
+				configuration.paths[path_name] = Path.new(path_name, template)
+			end
+		end
+
+		def initialize(path_name, template)
+			super(template) do |locals, name|
+				case name
+				when :basename
+					path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
+					path = Pathname.new(path)
+					path.basename(path.extname).to_s
+				when :dirname
+					path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
+					Pathname.new(path).dirname.to_s
+				when :extension
+					path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
+					Pathname.new(path).extname.delete('.')
+				when :digest
+					return locals[:_digest] if locals.include? :_digest
+					data = locals[:body] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :body, name) 
+					digest = Digest::SHA2.new.update(data).to_s[0,16]
+					# cache digest in request locals
+					locals[:_digest] = digest
+				else
+					locals[name] or raise NoValueForPathTemplatePlaceholerError.new(path_name, template, name)
+				end
+			end
+		end
+	end
+	Global.register_node_parser Path
+end
+

data/lib/httpimagestore/configuration/s3.rb

@@ -0,0 +1,194 @@
+require 'aws-sdk'
+require 'httpimagestore/aws_sdk_regions_hack'
+require 'httpimagestore/configuration/path'
+require 'httpimagestore/configuration/handler'
+
+module Configuration
+	class S3NotConfiguredError < ConfigurationError
+		def initialize
+			super "S3 client not configured"
+		end
+	end
+
+	class S3NoSuchBucketError < ConfigurationError
+		def initialize(bucket)
+			super "S3 bucket '#{bucket}' does not exist"
+		end
+	end
+
+	class S3NoSuchKeyError < ConfigurationError
+		def initialize(bucket, path)
+			super "S3 bucket '#{bucket}' does not contain key '#{path}'"
+		end
+	end
+
+	class S3AccessDenied < ConfigurationError
+		def initialize(bucket, path)
+			super "access to S3 bucket '#{bucket}' or key '#{path}' was denied"
+		end
+	end
+
+	class S3
+		include ClassLogging
+
+		def self.match(node)
+			node.name == 's3'
+		end
+
+		def self.parse(configuration, node)
+			configuration.s3 and raise StatementCollisionError.new(node, 's3')
+
+			node.grab_values
+			node.required_attributes('key', 'secret')
+			node.valid_attribute_values('ssl', true, false, nil)
+			
+			key, secret, ssl = node.grab_attributes('key', 'secret', 'ssl')
+			ssl = true if ssl.nil?
+
+			configuration.s3 = AWS::S3.new(
+				access_key_id: key,
+				secret_access_key: secret,
+				logger: logger_for(AWS::S3),
+				log_level: :debug,
+				use_ssl: ssl
+			)
+
+			log.info "S3 client using '#{key}' key and #{ssl ? 'HTTPS' : 'HTTP'} connections"
+		end
+	end
+	Global.register_node_parser S3
+
+	class S3SourceStoreBase < SourceStoreBase
+		include ClassLogging
+
+		extend Stats
+		def_stats(
+			:total_s3_store, 
+			:total_s3_store_bytes,
+			:total_s3_source,
+			:total_s3_source_bytes
+		)
+
+		def self.parse(configuration, node)
+			image_name = node.grab_values('image name').first
+
+			node.required_attributes('bucket', 'path')
+			node.valid_attribute_values('public_access', true, false, nil)
+
+			bucket, path_spec, cache_control, public_access, if_image_name_on = 
+				*node.grab_attributes('bucket', 'path', 'cache-control', 'public', 'if-image-name-on')
+			public_access = false if public_access.nil?
+
+			self.new(
+				configuration.global, 
+				image_name, 
+				InclusionMatcher.new(image_name, if_image_name_on),
+				bucket, 
+				path_spec, 
+				public_access, 
+				cache_control
+			)
+		end
+
+		def initialize(global, image_name, matcher, bucket, path_spec, public_access, cache_control)
+			super global, image_name, matcher
+			@bucket = bucket
+			@path_spec = path_spec
+			@public_access = public_access
+			@cache_control = cache_control
+			local :bucket, @bucket
+		end
+
+		def client
+			@global.s3 or raise S3NotConfiguredError
+		end
+
+		def url(object)
+			if @public_access
+				object.public_url.to_s
+			else
+				object.url_for(:read, expires: 60 * 60 * 24 * 365 * 20).to_s # expire in 20 years
+			end
+		end
+
+		def object(path)
+			begin
+				bucket = client.buckets[@bucket]
+				yield bucket.objects[path]
+			rescue AWS::S3::Errors::AccessDenied
+				raise S3AccessDenied.new(@bucket, path)
+			rescue AWS::S3::Errors::NoSuchBucket
+				raise S3NoSuchBucketError.new(@bucket)
+			rescue AWS::S3::Errors::NoSuchKey
+				 raise S3NoSuchKeyError.new(@bucket, path)
+			end
+		end
+	end
+
+	class S3Source < S3SourceStoreBase
+		def self.match(node)
+			node.name == 'source_s3'
+		end
+
+		def self.parse(configuration, node)
+			configuration.image_sources << super
+		end
+
+		def realize(request_state)
+			put_sourced_named_image(request_state) do |image_name, rendered_path|
+				log.info "sourcing '#{image_name}' image from S3 '#{@bucket}' bucket under '#{rendered_path}' key"
+
+				object(rendered_path) do |object|
+					data = request_state.memory_limit.get do |limit|
+						object.read range: 0..(limit + 1)
+					end
+					S3SourceStoreBase.stats.incr_total_s3_source
+					S3SourceStoreBase.stats.incr_total_s3_source_bytes(data.bytesize)
+
+					image = Image.new(data, object.head[:content_type])
+					image.source_url = url(object)
+					image
+				end
+			end
+		end
+	end
+	Handler::register_node_parser S3Source
+
+	class S3Store < S3SourceStoreBase
+		def self.match(node)
+			node.name == 'store_s3'
+		end
+
+		def self.parse(configuration, node)
+			configuration.stores << super
+		end
+
+		def realize(request_state)
+			get_named_image_for_storage(request_state) do |image_name, image, rendered_path|
+				acl = @public_access ?  :public_read : :private
+
+				log.info "storing '#{image_name}' image in S3 '#{@bucket}' bucket under '#{rendered_path}' key with #{acl} access"
+
+				object(rendered_path) do |object|
+					image.mime_type or log.warn "storing '#{image_name}' in S3 '#{@bucket}' bucket under '#{rendered_path}' key with unknown mime type"
+
+					options = {}
+					options[:single_request] = true
+					options[:content_type] = image.mime_type
+					options[:acl] = acl
+					options[:cache_control] = @cache_control if @cache_control
+
+					object.write(image.data, options)
+					S3SourceStoreBase.stats.incr_total_s3_store
+					S3SourceStoreBase.stats.incr_total_s3_store_bytes(image.data.bytesize)
+
+					image.store_url = url(object)
+				end
+			end
+		end
+	end
+	Handler::register_node_parser S3Store
+	StatsReporter << S3SourceStoreBase.stats
+end
+
+

data/lib/httpimagestore/configuration/thumbnailer.rb

@@ -0,0 +1,244 @@
+require 'unicorn-cuba-base'
+require 'httpthumbnailer-client'
+require 'httpimagestore/ruby_string_template'
+require 'httpimagestore/configuration/handler'
+
+module Configuration
+	class Thumnailer
+		include ClassLogging
+
+		def self.match(node)
+			node.name == 'thumbnailer'
+		end
+
+		def self.parse(configuration, node)
+			configuration.thumbnailer and raise StatementCollisionError.new(node, 'thumbnailer')
+			node.required_attributes('url')
+			configuration.thumbnailer = HTTPThumbnailerClient.new(node.grab_attributes('url').first)
+		end
+
+		def self.post(configuration)
+			if not configuration.thumbnailer
+				configuration.thumbnailer = HTTPThumbnailerClient.new(configuration.defaults[:thumbnailer_url] || 'http://localhost:3100')
+			end
+			log.info "using thumbnailer at #{configuration.thumbnailer.server_url}"
+		end
+	end
+	Global.register_node_parser Thumnailer
+
+	class NoValueForSpecTemplatePlaceholerError < ConfigurationError
+		def initialize(image_name, spec_name, value_name, template)
+			super "cannot generate specification for thumbnail '#{image_name}': cannot generate value for attribute '#{spec_name}' from template '#{template}': no value for \#{#{value_name}}"
+		end
+	end
+
+	class Thumbnail
+		include ClassLogging
+
+		extend Stats
+		def_stats(
+			:total_thumbnail_requests, 
+			:total_thumbnail_requests_bytes,
+			:total_thumbnail_thumbnails,
+			:total_thumbnail_thumbnails_bytes
+		)
+
+		class ThumbnailingError < RuntimeError
+			def initialize(input_image_name, output_image_name, remote_error)
+				@remote_error = remote_error
+				if output_image_name
+					super "thumbnailing of '#{input_image_name}' into '#{output_image_name}' failed: #{remote_error.message}"
+				else
+					super "thumbnailing of '#{input_image_name}' failed: #{remote_error.message}"
+				end
+			end
+
+			attr_reader :remote_error
+		end
+
+		class ThumbnailSpec
+			class Spec < RubyStringTemplate
+				def initialize(image_name, sepc_name, template)
+					super(template) do |locals, name|
+						locals[name] or raise NoValueForSpecTemplatePlaceholerError.new(image_name, sepc_name, name, template)
+					end
+				end
+			end
+
+			include ConditionalInclusion
+
+			def initialize(image_name, method, width, height, format, options = {}, matcher = nil)
+				@image_name = image_name
+				@method = Spec.new(image_name, 'method', method)
+				@width = Spec.new(image_name, 'width', width)
+				@height = Spec.new(image_name, 'height', height)
+				@format = Spec.new(image_name, 'format', format)
+				@options = options.inject({}){|h, v| h[v.first] = Spec.new(image_name, v.first, v.last); h}
+				inclusion_matcher matcher if matcher
+			end
+
+			attr_reader :image_name
+
+			def render(locals = {})
+				options = @options.inject({}){|h, v| h[v.first] = v.last.render(locals); h}
+				nested_options = options['options'] ? Hash[options.delete('options').to_s.split(',').map{|pair| pair.split(':', 2)}] : {}
+				{
+					@image_name =>
+						[
+							@method.render(locals),
+							@width.render(locals),
+							@height.render(locals),
+							@format.render(locals),
+							nested_options.merge(options)
+						]
+				}
+			end
+		end
+
+		include ConditionalInclusion
+
+		def self.match(node)
+			node.name == 'thumbnail'
+		end
+
+		def self.parse(configuration, node)
+			use_multipart_api = node.values.length == 1 ? true : false
+
+			nodes = use_multipart_api ?  node.children : [node]
+			source_image_name = use_multipart_api ? node.grab_values('source image name').first : nil # parsed later
+
+			nodes.empty? and raise NoValueError.new(node, 'thumbnail image name')
+			matcher = nil
+
+			specs = nodes.map do |node|
+				if use_multipart_api
+					image_name = node.grab_values('thumbnail image name').first
+				else
+					source_image_name, image_name = *node.grab_values('source image name', 'thumbnail image name')
+				end
+
+				operation, width, height, format, if_image_name_on, remaining = *node.grab_attributes_with_remaining('operation', 'width', 'height', 'format', 'if-image-name-on')
+
+				matcher = InclusionMatcher.new(image_name, if_image_name_on) if if_image_name_on
+
+				ThumbnailSpec.new(
+					image_name,
+					operation || 'fit',
+					width || 'input',
+					height || 'input',
+					format || 'jpeg',
+					remaining || {},
+					matcher
+				)
+			end
+
+			matcher = InclusionMatcher.new(source_image_name, node.grab_attributes('if-image-name-on').first) if use_multipart_api
+
+			configuration.image_sources << self.new(
+				configuration.global, 
+				source_image_name, 
+				specs, 
+				use_multipart_api,
+				matcher
+			)
+		end
+
+		def initialize(global, source_image_name, specs, use_multipart_api, matcher)
+			@global = global
+			@source_image_name = source_image_name
+			@specs = specs
+			@use_multipart_api = use_multipart_api
+			inclusion_matcher matcher
+		end
+
+		def realize(request_state)
+			client = @global.thumbnailer or fail 'thumbnailer configuration'
+
+			rendered_specs = {}
+			@specs.select do |spec|
+				spec.included?(request_state)
+			end.each do |spec|
+				rendered_specs.merge! spec.render(request_state.locals)
+			end
+			source_image = request_state.images[@source_image_name]
+
+			thumbnails = {}
+			input_mime_type = nil
+
+			Thumbnail.stats.incr_total_thumbnail_requests
+			Thumbnail.stats.incr_total_thumbnail_requests_bytes source_image.data.bytesize
+
+			if @use_multipart_api
+				log.info "thumbnailing '#{@source_image_name}' to multiple specs: #{rendered_specs}"
+
+				# need to reference to local so they are available within thumbnail() block context
+				source_image_name = @source_image_name
+				logger = log
+
+				begin
+					thumbnails = client.thumbnail(source_image.data) do
+						rendered_specs.each_pair do |name, spec|
+							begin
+								thumbnail(*spec)
+							rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
+								logger.warn 'got thumbnailer error while passing specs', error
+								raise ThumbnailingError.new(source_image_name, name, error)
+							end
+						end
+					end
+				rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
+					logger.warn 'got thumbnailer error while sending input data', error
+					raise ThumbnailingError.new(source_image_name, nil, error)
+				end
+
+				input_mime_type = thumbnails.input_mime_type
+
+				# check each thumbnail for errors
+				thumbnails = Hash[rendered_specs.keys.zip(thumbnails)]
+				thumbnails.each do |name, thumbnail|
+					if thumbnail.kind_of? HTTPThumbnailerClient::HTTPThumbnailerClientError
+						error = thumbnail
+						log.warn 'got single thumbnail error', error
+						raise ThumbnailingError.new(@source_image_name, name, error) 
+					end
+				end
+
+				# borrow from memory limit - note that we might have already used too much memory
+				thumbnails.each do |name, thumbnail|
+					request_state.memory_limit.borrow thumbnail.data.bytesize
+				end
+			else
+				name, rendered_spec = *rendered_specs.first
+				log.info "thumbnailing '#{@source_image_name}' to '#{name}' with spec: #{rendered_spec}"
+
+				begin
+					thumbnail = client.thumbnail(source_image.data, *rendered_spec)
+					request_state.memory_limit.borrow thumbnail.data.bytesize
+					input_mime_type = thumbnail.input_mime_type
+					thumbnails[name] = thumbnail
+				rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
+					log.warn 'got thumbnailer error', error
+					raise ThumbnailingError.new(@source_image_name, name, error)
+				end
+			end
+
+			# copy input source path and url
+			thumbnails.each do |name, thumbnail|
+				thumbnail.extend ImageMetaData
+				thumbnail.source_path = source_image.source_path
+				thumbnail.source_url = source_image.source_url
+
+				Thumbnail.stats.incr_total_thumbnail_thumbnails
+				Thumbnail.stats.incr_total_thumbnail_thumbnails_bytes thumbnail.data.bytesize
+			end
+
+			# update input image mime type from httpthumbnailer provided information
+			source_image.mime_type = input_mime_type unless source_image.mime_type
+
+			request_state.images.merge! thumbnails
+		end
+	end
+	Handler::register_node_parser Thumbnail
+	StatsReporter << Thumbnail.stats
+end
+

data/lib/httpimagestore/configuration.rb

@@ -1,55 +1,152 @@
-require 'httpimagestore/thumbnail_class'
+require 'sdl4r'
 require 'pathname'
+require 'ostruct'
+require 'unicorn-cuba-base'
 
-class Configuration
-	class ThumbnailClassDoesNotExistError < RuntimeError
-		def initialize(name)
-			super "Class '#{name}' does not exist"
+module Configuration
+	# parsing errors
+	class SyntaxError < ArgumentError
+		def initialize(node, message)
+			super "syntax error while parsing '#{node}': #{message}"
 		end
 	end
 
-	def initialize(&block)
-		@thumbnail_classes = Hash.new do |h, k|
-			raise ThumbnailClassDoesNotExistError, k
+	class NoAttributeError < SyntaxError
+		def initialize(node, attribute)
+			super node, "expected '#{attribute}' attribute to be set"
 		end
+	end
 
-		@thumbnailer_url = "http://localhost:3100"
+	class NoValueError < SyntaxError
+		def initialize(node, value)
+			super node, "expected #{value}"
+		end
+	end
 
-		instance_eval &block
+	class BadAttributeValueError < SyntaxError
+		def initialize(node, attribute, value, valid)
+			super node, "expected '#{attribute}' attribute value to be #{valid.map(&:inspect).join(' or ')}; got: #{value.inspect}"
+		end
 	end
 
-	def self.from_file(file)
-		file = Pathname.pwd + file
-		Configuration.new do
-			 eval(file.read, nil, file.to_s)
+	class UnexpectedValueError < SyntaxError
+		def initialize(node, values)
+			super node, "unexpected values: #{values.map(&:inspect).join(', ')}"
 		end
 	end
 
-	def thumbnail_class(name, method, width, height, format = 'JPEG', options = {})
-		@thumbnail_classes[name] = ThumbnailClass.new(name, method, width, height, format, options)
+	class UnexpectedAttributesError < SyntaxError
+		def initialize(node, attributes)
+			super node, "unexpected attributes: #{attributes.keys.map{|a| "'#{a}'"}.join(', ')}"
+		end
 	end
 
-	def s3_key(id, secret)
-		@s3_key_id = id
-		@s3_key_secret = secret
+	class StatementCollisionError < SyntaxError
+		def initialize(node, type)
+			super node, "only one #{type} type statement can be specified within context"
+		end
 	end
 
-	def s3_bucket(bucket)
-		@s3_bucket = bucket
+	# runtime errors
+	ConfigurationError = Class.new ArgumentError
+
+	module SDL4RTagExtensions
+		def required_attributes(*list)
+			list.each do |attribute|
+				attribute(attribute) or raise NoAttributeError.new(self, attribute)
+			end
+			true
+		end
+
+		def grab_attributes_with_remaining(*list)
+			attributes = self.attributes.dup
+			values = list.map do |attribute|
+				attributes.delete(attribute)
+			end
+			values + [attributes]
+		end
+
+		def grab_attributes(*list)
+			*values, remaining = *grab_attributes_with_remaining(*list)
+			remaining.empty? or raise UnexpectedAttributesError.new(self, remaining)
+			values
+		end
+
+		def valid_attribute_values(attribute, *valid)
+			value = self.attribute(attribute)
+			valid.include? value or raise BadAttributeValueError.new(self, attribute, value, valid)
+		end
+
+		def grab_values(*list)
+			values = self.values.dup
+			out = []
+			list.each do |name|
+				val = values.shift or raise NoValueError.new(self, name)
+				out << val
+			end
+			values.empty? or raise UnexpectedValueError.new(self, values)
+			out
+		end
 	end
 
-	def thumbnailer_url(url)
-		@thumbnailer_url = url
+	class Scope
+		include ClassLogging
+
+		def self.node_parsers
+			@node_parsers ||= []
+		end
+
+		def self.register_node_parser(parser)
+			parser.logger = logger_for(parser) if parser.respond_to? :logger=
+			node_parsers << parser
+		end
+
+		def initialize(configuration)
+			@configuration = configuration
+		end
+
+		def parse(node)
+			self.class.node_parsers.each do |parser|
+				parser.pre(@configuration) if parser.respond_to? :pre
+			end
+
+			node.children.each do |node|
+				parser = self.class.node_parsers.find do |parser|
+					parser.match node
+				end
+				if parser
+					parser.parse(@configuration, node)
+				else
+					log.warn "unexpected statement: #{node.name}"
+				end
+			end
+
+			self.class.node_parsers.each do |parser|
+				parser.post(@configuration) if parser.respond_to? :post
+			end
+			@configuration
+		end
 	end
 
-	def get
-		Struct.new(:thumbnail_classes, :s3_key_id, :s3_key_secret, :s3_bucket, :thumbnailer_url).new(@thumbnail_classes, @s3_key_id, @s3_key_secret, @s3_bucket, @thumbnailer_url)
+	class Global < Scope
 	end
 
-	def put(sinatra)
-		get.each_pair do |key, value|
-			sinatra.set key, value
-		end
+	def self.from_file(config_file, defaults = {})
+		read Pathname.new(config_file), defaults
+	end
+
+	def self.read(config, defaults = {})
+		parse SDL4R::read(config), defaults
 	end
+
+	def self.parse(root, defaults = {})
+		configuration = OpenStruct.new
+		configuration.defaults = defaults
+		Global.new(configuration).parse(root)
+	end
+end
+
+class SDL4R::Tag
+	include Configuration::SDL4RTagExtensions
 end