httpimagestore 1.2.0 → 1.3.0

data/lib/httpimagestore/configuration/handler.rb

@@ -1,4 +1,6 @@
 require 'mime/types'
+require 'digest/sha2'
+require 'securerandom'
 
 module Configuration
 	class ImageNotLoadedError < ConfigurationError
@@ -13,7 +15,31 @@ module Configuration
 		end
 	end
 
-	class RequestState
+	class VariableNotDefinedError < ConfigurationError
+		def initialize(name)
+			super "variable '#{name}' not defined"
+		end
+	end
+
+	class NoRequestBodyToGenerateMetaVariableError < ConfigurationError
+		def initialize(meta_value)
+			super "need not empty request body to generate value for '#{meta_value}'"
+		end
+	end
+
+	class NoVariableToGenerateMetaVariableError < ConfigurationError
+		def initialize(value_name, meta_value)
+			super "need '#{value_name}' variable to generate value for '#{meta_value}'"
+		end
+	end
+
+	class NoImageDataForVariableError < ConfigurationError
+		def initialize(image_name, meta_value)
+			super "image '#{image_name}' does not have data for variable '#{meta_value}'"
+		end
+	end
+
+	class RequestState < Hash
 		include ClassLogging
 
 		class Images < Hash
@@ -35,24 +61,33 @@ module Configuration
 		end
 
 		def initialize(body = '', matches = {}, path = '', query_string = {}, memory_limit = MemoryLimit.new)
-			@locals = {}
-			@locals.merge! query_string
-			@locals[:path] = path
-			@locals.merge! matches
-			@locals[:query_string_options] = query_string.sort.map{|kv| kv.join(':')}.join(',')
-			log.debug "processing request with body length: #{body.bytesize} bytes and locals: #{@locals} "
+			super() do |request_state, name|
+				# note that request_state may be different object when useing with_locals that creates duplicate
+				request_state[name] = request_state.generate_meta_variable(name) or raise VariableNotDefinedError.new(name)
+			end
+
+			merge! query_string
+			self[:path] = path
+			merge! matches
+			self[:query_string_options] = query_string.sort.map{|kv| kv.join(':')}.join(',')
 
-			@locals[:_body] = body
+			log.debug "processing request with body length: #{body.bytesize} bytes and variables: #{self} "
 
+			@body = body
 			@images = Images.new(memory_limit)
 			@memory_limit = memory_limit
 			@output_callback = nil
 		end
 
+		attr_reader :body
 		attr_reader :images
-		attr_reader :locals
 		attr_reader :memory_limit
 
+		def with_locals(locals)
+			log.debug "using additional local variables: #{locals}"
+			self.dup.merge!(locals)
+		end
+
 		def output(&callback)
 			@output_callback = callback
 		end
@@ -60,6 +95,62 @@ module Configuration
 		def output_callback
 			@output_callback or fail 'no output callback'
 		end
+
+		def fetch_base_variable(name, base_name)
+			fetch(base_name, nil) or generate_meta_variable(base_name) or raise NoVariableToGenerateMetaVariableError.new(base_name, name)
+		end
+
+		def generate_meta_variable(name)
+			log.debug  "generating meta variable: #{name}"
+			val = case name
+			when :basename
+				path = Pathname.new(fetch_base_variable(name, :path))
+				path.basename(path.extname).to_s
+			when :dirname
+				Pathname.new(fetch_base_variable(name, :path)).dirname.to_s
+			when :extension
+				Pathname.new(fetch_base_variable(name, :path)).extname.delete('.')
+			when :digest # deprecated
+				@body.empty? and raise NoRequestBodyToGenerateMetaVariableError.new(name)
+				Digest::SHA2.new.update(@body).to_s[0,16]
+			when :input_digest
+				@body.empty? and raise NoRequestBodyToGenerateMetaVariableError.new(name)
+				Digest::SHA2.new.update(@body).to_s[0,16]
+			when :input_sha256
+				@body.empty? and raise NoRequestBodyToGenerateMetaVariableError.new(name)
+				Digest::SHA2.new.update(@body).to_s
+			when :input_image_width
+				@images['input'].width or raise NoImageDataForVariableError.new('input', name)
+			when :input_image_height
+				@images['input'].height or raise NoImageDataForVariableError.new('input', name)
+			when :input_image_mime_extension
+				@images['input'].mime_extension or raise NoImageDataForVariableError.new('input', name)
+			when :image_digest
+				Digest::SHA2.new.update(@images[fetch_base_variable(name, :image_name)].data).to_s[0,16]
+			when :image_sha256
+				Digest::SHA2.new.update(@images[fetch_base_variable(name, :image_name)].data).to_s
+			when :mimeextension # deprecated
+				image_name = fetch_base_variable(name, :image_name)
+				@images[image_name].mime_extension or raise NoImageDataForVariableError.new(image_name, name)
+			when :image_mime_extension
+				image_name = fetch_base_variable(name, :image_name)
+				@images[image_name].mime_extension or raise NoImageDataForVariableError.new(image_name, name)
+			when :image_width
+				image_name = fetch_base_variable(name, :image_name)
+				@images[image_name].width or raise NoImageDataForVariableError.new(image_name, name)
+			when :image_height
+				image_name = fetch_base_variable(name, :image_name)
+				@images[image_name].height or raise NoImageDataForVariableError.new(image_name, name)
+			when :uuid
+				SecureRandom.uuid
+			end
+			if val
+				log.debug  "generated meta variable '#{name}': #{val}"
+			else
+				log.debug  "could not generated meta variable '#{name}'"
+			end
+			val
+		end
 	end
 
 	module ImageMetaData
@@ -75,14 +166,14 @@ module Configuration
 		end
 	end
 
-	class Image < Struct.new(:data, :mime_type)
+	class Image < Struct.new(:data, :mime_type, :width, :height)
 		include ImageMetaData
 	end
 
 	class InputSource
 		def realize(request_state)
-			request_state.locals[:_body].empty? and raise ZeroBodyLengthError
-			request_state.images['input'] = Image.new(request_state.locals[:_body])
+			request_state.body.empty? and raise ZeroBodyLengthError
+			request_state.images['input'] = Image.new(request_state.body)
 		end
 	end
 
@@ -102,7 +193,7 @@ module Configuration
 
 		def included?(request_state)
 			return true if not @template
-			@template.render(request_state.locals).split(',').include? @value
+			@template.render(request_state).split(',').include? @value
 		end
 	end
 
@@ -129,8 +220,11 @@ module Configuration
 		def initialize(global, image_name, matcher)
 			@global = global
 			@image_name = image_name
-			@locals = {imagename: @image_name}
+			@locals = {}
+
 			inclusion_matcher matcher
+			local :imagename, @image_name # deprecated
+			local :image_name, @image_name
 		end
 
 		private
@@ -143,7 +237,7 @@ module Configuration
 
 		def rendered_path(request_state)
 			path = @global.paths[@path_spec]
-			Pathname.new(path.render(@locals.merge(request_state.locals))).cleanpath.to_s
+			Pathname.new(path.render(request_state.with_locals(@locals))).cleanpath.to_s
 		end
 
 		def put_sourced_named_image(request_state)
@@ -157,8 +251,6 @@ module Configuration
 
 		def get_named_image_for_storage(request_state)
 			image = request_state.images[@image_name]
-			local :mimeextension, image.mime_extension
-
 			rendered_path = rendered_path(request_state)
 			image.store_path = rendered_path
 
@@ -193,7 +285,8 @@ module Configuration
 					:global,
 					:http_method,
 					:uri_matchers,
-					:image_sources,
+					:sources,
+					:processors,
 					:stores,
 					:output
 				).new
@@ -245,14 +338,15 @@ module Configuration
 					end
 				end
 			end
-			handler_configuration.image_sources = []
+			handler_configuration.sources = []
+			handler_configuration.processors = []
 			handler_configuration.stores = []
 			handler_configuration.output = nil
 
 			node.grab_attributes
 
 			if handler_configuration.http_method != 'get'
-				handler_configuration.image_sources << InputSource.new
+				handler_configuration.sources << InputSource.new
 			end
 
 			configuration.handlers << handler_configuration

data/lib/httpimagestore/configuration/identify.rb

@@ -0,0 +1,56 @@
+require 'httpthumbnailer-client'
+require 'httpimagestore/ruby_string_template'
+require 'httpimagestore/configuration/handler'
+
+module Configuration
+	class Identify
+		include ClassLogging
+
+		extend Stats
+		def_stats(
+			:total_identify_requests, 
+			:total_identify_requests_bytes
+		)
+
+		include ConditionalInclusion
+
+		def self.match(node)
+			node.name == 'identify'
+		end
+
+		def self.parse(configuration, node)
+			image_name = node.grab_values('image name').first
+			if_image_name_on = node.grab_attributes('if-image-name-on').first
+
+			matcher = InclusionMatcher.new(image_name, if_image_name_on) if if_image_name_on
+
+			configuration.processors << self.new(configuration.global, image_name, matcher)
+		end
+
+		def initialize(global, image_name, matcher = nil)
+			@global = global
+			@image_name = image_name
+			inclusion_matcher matcher if matcher
+		end
+
+		def realize(request_state)
+			client = @global.thumbnailer or fail 'thumbnailer configuration'
+			image = request_state.images[@image_name]
+
+			log.info "identifying '#{@image_name}'"
+
+			Identify.stats.incr_total_identify_requests
+			Identify.stats.incr_total_identify_requests_bytes image.data.bytesize
+
+			id = client.identify(image.data)
+
+			image.mime_type = id.mime_type if id.mime_type
+			image.width = id.width if id.width
+			image.height = id.height if id.height
+			log.info "image '#{@image_name}' identified as '#{id.mime_type}' #{image.width}x#{image.height}"
+		end
+	end
+	Handler::register_node_parser Identify
+	StatsReporter << Identify.stats
+end
+

data/lib/httpimagestore/configuration/path.rb

@@ -15,14 +15,8 @@ module Configuration
 	end
 
 	class NoValueForPathTemplatePlaceholerError < PathRenderingError
-		def initialize(path_name, template, value_name)
-			super path_name, template, "no value for '\#{#{value_name}}'"
-		end
-	end
-
-	class NoMetaValueForPathTemplatePlaceholerError < PathRenderingError
-		def initialize(path_name, template, value_name, meta_value)
-			super path_name, template, "need '#{value_name}' to generate value for '\#{#{meta_value}}'"
+		def initialize(path_name, template, placeholder)
+			super path_name, template, "no value for '\#{#{placeholder}}'"
 		end
 	end
 
@@ -49,26 +43,11 @@ module Configuration
 
 		def initialize(path_name, template)
 			super(template) do |locals, name|
-				case name
-				when :basename
-					path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
-					path = Pathname.new(path)
-					path.basename(path.extname).to_s
-				when :dirname
-					path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
-					Pathname.new(path).dirname.to_s
-				when :extension
-					path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
-					Pathname.new(path).extname.delete('.')
-				when :digest
-					return locals[:_digest] if locals.include? :_digest
-					data = locals[:_body] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :body, name) 
-					digest = Digest::SHA2.new.update(data).to_s[0,16]
-					# cache digest in request locals
-					locals[:_digest] = digest
-				else
-					locals[name] or raise NoValueForPathTemplatePlaceholerError.new(path_name, template, name)
-				end
+				begin
+					locals[name]
+				rescue ConfigurationError => error
+					raise PathRenderingError.new(path_name, template, error.message)
+				end or raise NoValueForPathTemplatePlaceholerError.new(path_name, template, name)
 			end
 		end
 	end

data/lib/httpimagestore/configuration/s3.rb

@@ -1,4 +1,6 @@
 require 'aws-sdk'
+require 'digest/sha2'
+require 'msgpack'
 require 'httpimagestore/aws_sdk_regions_hack'
 require 'httpimagestore/configuration/path'
 require 'httpimagestore/configuration/handler'
@@ -61,6 +63,190 @@ module Configuration
 	class S3SourceStoreBase < SourceStoreBase
 		include ClassLogging
 
+		class CacheRoot
+			CacheRootError = Class.new ArgumentError
+			class CacheRootNotDirError < CacheRootError
+				def initialize(root_dir)
+					super "S3 object cache directory '#{root_dir}' does not exist or not a directory"
+				end
+			end
+
+			class CacheRootNotWritableError < CacheRootError
+				def initialize(root_dir)
+					super "S3 object cache directory '#{root_dir}' is not writable"
+				end
+			end
+
+			class CacheRootNotAccessibleError < CacheRootError
+				def initialize(root_dir)
+					super "S3 object cache directory '#{root_dir}' is not readable"
+				end
+			end
+
+			def initialize(root_dir)
+				@root = Pathname.new(root_dir)
+				@root.directory? or raise CacheRootNotDirError.new(root_dir)
+				@root.executable? or raise CacheRootNotAccessibleError.new(root_dir)
+				@root.writable? or raise CacheRootNotWritableError.new(root_dir)
+			end
+
+			def cache_file(bucket, key)
+				File.join(Digest::SHA2.new.update("#{bucket}/#{key}").to_s[0,32].match(/(..)(..)(.*)/).captures)
+			end
+
+			def open(bucket, key)
+				# TODO: locking
+				file = @root + cache_file(bucket, key)
+
+				file.dirname.directory? or file.dirname.mkpath
+				if file.exist?
+					file.open('r+') do |io|
+						yield io
+					end
+				else
+					file.open('w+') do |io|
+						yield io
+					end
+				end
+			end
+		end
+
+		class S3Object
+			def initialize(client, bucket, key)
+				@client = client
+				@bucket = bucket
+				@key = key
+			end
+
+			def s3_object
+				return @s3_object if @s3_object
+				@s3_object = @client.buckets[@bucket].objects[@key]
+			end
+
+			def read(max_bytes = nil)
+				options = {}
+				options[:range] = 0..max_bytes if max_bytes
+				s3_object.read(options)
+			end
+
+			def write(data, options = {})
+				s3_object.write(data, options)
+			end
+
+			def private_url
+				s3_object.url_for(:read, expires: 60 * 60 * 24 * 365 * 20).to_s # expire in 20 years
+			end
+
+			def public_url
+				s3_object.public_url.to_s
+			end
+
+			def content_type
+				s3_object.head[:content_type]
+			end
+		end
+
+		class CacheObject < S3Object
+			include ClassLogging
+
+			def initialize(io, client, bucket, key)
+				@io = io
+				super(client, bucket, key)
+
+				@header = {}
+				@have_cache = false
+				@dirty = false
+
+				begin
+					head_length = @io.read(4)
+
+					if head_length and head_length.length == 4
+						head_length = head_length.unpack('L').first
+						@header = MessagePack.unpack(@io.read(head_length))
+						@have_cache = true
+
+						log.debug{"S3 object cache hit; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]: header: #{@header}"}
+					else
+						log.debug{"S3 object cache miss; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]"}
+					end
+				rescue => error
+					log.warn "cannot use cached S3 object; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]: #{error}"
+					# not usable
+					io.seek 0
+					io.truncate 0
+				end
+
+				yield self
+
+				# save object as was used if no error happened and there were changes
+				write_cache if dirty?
+			end
+
+			def read(max_bytes = nil)
+				if @have_cache
+					data_location = @io.seek(0, IO::SEEK_CUR)
+					begin
+						return @data = @io.read(max_bytes)
+					ensure
+						@io.seek(data_location, IO::SEEK_SET)
+					end
+				else
+					dirty! :read
+					return @data = super
+				end
+			end
+
+			def write(data, options = {})
+				out = super
+				@data = data
+				dirty! :write
+				out
+			end
+
+			def private_url
+				@header['private_url'] ||= (dirty! :private_url; super)
+			end
+
+			def public_url
+				@header['public_url'] ||= (dirty! :public_url; super)
+			end
+
+			def content_type
+				@header['content_type'] ||= (dirty! :content_type; super)
+			end
+
+			private
+
+			def write_cache
+				begin
+					log.debug{"S3 object is dirty, wirting cache file; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]; header: #{@header}"}
+
+					raise 'nil data!' unless @data
+					# rewrite
+					@io.seek(0, IO::SEEK_SET)
+					@io.truncate 0
+
+					header = MessagePack.pack(@header)
+					@io.write [header.length].pack('L') # header length
+					@io.write header
+					@io.write @data
+				rescue => error
+					log.warn "cannot store S3 object in cache: bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]: #{error}"
+				ensure
+					@dirty = false
+				end
+			end
+
+			def dirty!(reason = :unknown)
+				log.debug{"marking cache dirty for reason: #{reason}"}
+				@dirty = true
+			end
+
+			def dirty?
+				@dirty
+			end
+		end
+
 		extend Stats
 		def_stats(
 			:total_s3_store, 
@@ -75,8 +261,8 @@ module Configuration
 			node.required_attributes('bucket', 'path')
 			node.valid_attribute_values('public_access', true, false, nil)
 
-			bucket, path_spec, public_access, cache_control, prefix, if_image_name_on = 
-				*node.grab_attributes('bucket', 'path', 'public', 'cache-control', 'prefix', 'if-image-name-on')
+			bucket, path_spec, public_access, cache_control, prefix, cache_root, if_image_name_on = 
+				*node.grab_attributes('bucket', 'path', 'public', 'cache-control', 'prefix', 'cache-root', 'if-image-name-on')
 			public_access = false if public_access.nil?
 			prefix = '' if prefix.nil?
 
@@ -88,17 +274,31 @@ module Configuration
 				path_spec, 
 				public_access, 
 				cache_control,
-				prefix
+				prefix,
+				cache_root
 			)
 		end
 
-		def initialize(global, image_name, matcher, bucket, path_spec, public_access, cache_control, prefix)
+		def initialize(global, image_name, matcher, bucket, path_spec, public_access, cache_control, prefix, cache_root)
 			super global, image_name, matcher
 			@bucket = bucket
 			@path_spec = path_spec
 			@public_access = public_access
 			@cache_control = cache_control
 			@prefix = prefix
+
+			@cache_root = nil
+			begin
+				if cache_root
+					@cache_root = CacheRoot.new(cache_root)
+					log.info "using S3 object cache directory '#{cache_root}' for image '#{image_name}'"
+				else
+					log.info "S3 object cache not configured (no cache-root) for image '#{image_name}'"
+				end
+			rescue CacheRoot::CacheRootNotDirError => error
+				log.warn "not using S3 object cache for image '#{image_name}': #{error}"
+			end
+
 			local :bucket, @bucket
 		end
 
@@ -108,16 +308,31 @@ module Configuration
 
 		def url(object)
 			if @public_access
-				object.public_url.to_s
+				object.public_url
 			else
-				object.url_for(:read, expires: 60 * 60 * 24 * 365 * 20).to_s # expire in 20 years
+				object.private_url
 			end
 		end
 
 		def object(path)
 			begin
-				bucket = client.buckets[@bucket]
-				yield bucket.objects[@prefix + path]
+				key = @prefix + path
+				image = nil
+
+				if @cache_root
+					begin
+						@cache_root.open(@bucket, key) do |cahce_file_io|
+							CacheObject.new(cahce_file_io, client, @bucket, key) do |obj|
+								image = yield obj
+							end
+						end
+					rescue IOError => error
+						log.warn "cannot use S3 object cache '#{@cache_root.cache_file(@bucket, key)}': #{error}"
+						image = yield obj
+					end
+				else
+					image = yield S3Object.new(client, @bucket, key)
+				end
 			rescue AWS::S3::Errors::AccessDenied
 				raise S3AccessDenied.new(@bucket, path)
 			rescue AWS::S3::Errors::NoSuchBucket
@@ -125,7 +340,11 @@ module Configuration
 			rescue AWS::S3::Errors::NoSuchKey
 				 raise S3NoSuchKeyError.new(@bucket, path)
 			end
+			image
 		end
+
+		S3SourceStoreBase.logger = Handler.logger_for(S3SourceStoreBase)
+		CacheObject.logger = S3SourceStoreBase.logger_for(CacheObject)
 	end
 
 	class S3Source < S3SourceStoreBase
@@ -134,7 +353,7 @@ module Configuration
 		end
 
 		def self.parse(configuration, node)
-			configuration.image_sources << super
+			configuration.sources << super
 		end
 
 		def realize(request_state)
@@ -143,12 +362,12 @@ module Configuration
 
 				object(rendered_path) do |object|
 					data = request_state.memory_limit.get do |limit|
-						object.read range: 0..(limit + 1)
+						object.read(limit + 1)
 					end
 					S3SourceStoreBase.stats.incr_total_s3_source
 					S3SourceStoreBase.stats.incr_total_s3_source_bytes(data.bytesize)
 
-					image = Image.new(data, object.head[:content_type])
+					image = Image.new(data, object.content_type)
 					image.source_url = url(object)
 					image
 				end
@@ -194,4 +413,3 @@ module Configuration
 	StatsReporter << S3SourceStoreBase.stats
 end
 
-