httpimagestore 0.5.0 → 1.0.0

data/httpimagestore.gemspec

@@ -5,17 +5,17 @@
 
 Gem::Specification.new do |s|
   s.name = "httpimagestore"
-  s.version = "0.5.0"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jakub Pastuszek"]
-  s.date = "2012-11-14"
+  s.date = "2013-07-16"
   s.description = "Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)"
   s.email = "jpastuszek@gmail.com"
   s.executables = ["httpimagestore"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
@@ -23,88 +23,101 @@ Gem::Specification.new do |s|
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/httpimagestore",
     "features/cache-control.feature",
-    "features/httpimagestore.feature",
+    "features/compatibility.feature",
+    "features/error-reporting.feature",
+    "features/health-check.feature",
+    "features/s3-store-and-thumbnail.feature",
     "features/step_definitions/httpimagestore_steps.rb",
     "features/support/env.rb",
     "features/support/test-large.jpg",
+    "features/support/test.empty",
     "features/support/test.jpg",
     "features/support/test.txt",
     "httpimagestore.gemspec",
+    "lib/httpimagestore/aws_sdk_regions_hack.rb",
     "lib/httpimagestore/configuration.rb",
-    "lib/httpimagestore/image_path.rb",
-    "lib/httpimagestore/s3_service.rb",
-    "lib/httpimagestore/thumbnail_class.rb",
+    "lib/httpimagestore/configuration/file.rb",
+    "lib/httpimagestore/configuration/handler.rb",
+    "lib/httpimagestore/configuration/output.rb",
+    "lib/httpimagestore/configuration/path.rb",
+    "lib/httpimagestore/configuration/s3.rb",
+    "lib/httpimagestore/configuration/thumbnailer.rb",
+    "lib/httpimagestore/error_reporter.rb",
+    "lib/httpimagestore/ruby_string_template.rb",
+    "load_test/load_test.1k.23a022f6e.m1.small-comp.csv",
+    "load_test/load_test.1k.ec9bde794.m1.small.csv",
+    "load_test/load_test.jmx",
+    "load_test/thumbnail_specs.csv",
+    "spec/configuration_file_spec.rb",
+    "spec/configuration_handler_spec.rb",
+    "spec/configuration_output_spec.rb",
+    "spec/configuration_path_spec.rb",
+    "spec/configuration_s3_spec.rb",
     "spec/configuration_spec.rb",
-    "spec/image_path_spec.rb",
+    "spec/configuration_thumbnailer_spec.rb",
+    "spec/ruby_string_template_spec.rb",
     "spec/spec_helper.rb",
-    "spec/test.cfg"
+    "spec/support/compute.jpg",
+    "spec/support/cuba_response_env.rb",
+    "spec/support/full.cfg"
   ]
   s.homepage = "http://github.com/jpastuszek/httpimagestore"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.25"
   s.summary = "HTTP based image storage and thumbnailer"
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<sinatra>, [">= 1.2.6"])
-      s.add_runtime_dependency(%q<mongrel>, [">= 1.2.0.pre2"])
-      s.add_runtime_dependency(%q<s3>, ["~> 0.3"])
-      s.add_runtime_dependency(%q<httpthumbnailer-client>, ["~> 0.1.0"])
-      s.add_runtime_dependency(%q<ruby-ip>, ["~> 0.9"])
-      s.add_runtime_dependency(%q<cli>, ["~> 1.1.0"])
-      s.add_runtime_dependency(%q<mime-types>, ["~> 1.17.2"])
-      s.add_runtime_dependency(%q<retry-this>, ["~> 1.1"])
-      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_runtime_dependency(%q<unicorn-cuba-base>, ["~> 1.0"])
+      s.add_runtime_dependency(%q<httpthumbnailer-client>, ["~> 1.0"])
+      s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.10"])
+      s.add_runtime_dependency(%q<mime-types>, ["~> 1.17"])
+      s.add_runtime_dependency(%q<sdl4r>, ["~> 0.9"])
+      s.add_development_dependency(%q<httpclient>, [">= 2.3"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.13"])
       s.add_development_dependency(%q<cucumber>, [">= 0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.9"])
       s.add_development_dependency(%q<daemon>, ["~> 1"])
-      s.add_development_dependency(%q<httpthumbnailer>, ["~> 0.2"])
       s.add_development_dependency(%q<prawn>, ["= 0.8.4"])
+      s.add_development_dependency(%q<httpthumbnailer>, [">= 0"])
     else
-      s.add_dependency(%q<sinatra>, [">= 1.2.6"])
-      s.add_dependency(%q<mongrel>, [">= 1.2.0.pre2"])
-      s.add_dependency(%q<s3>, ["~> 0.3"])
-      s.add_dependency(%q<httpthumbnailer-client>, ["~> 0.1.0"])
-      s.add_dependency(%q<ruby-ip>, ["~> 0.9"])
-      s.add_dependency(%q<cli>, ["~> 1.1.0"])
-      s.add_dependency(%q<mime-types>, ["~> 1.17.2"])
-      s.add_dependency(%q<retry-this>, ["~> 1.1"])
-      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.0"])
+      s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.0"])
+      s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
+      s.add_dependency(%q<mime-types>, ["~> 1.17"])
+      s.add_dependency(%q<sdl4r>, ["~> 0.9"])
+      s.add_dependency(%q<httpclient>, [">= 2.3"])
+      s.add_dependency(%q<rspec>, ["~> 2.13"])
       s.add_dependency(%q<cucumber>, [">= 0"])
-      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
       s.add_dependency(%q<rdoc>, ["~> 3.9"])
       s.add_dependency(%q<daemon>, ["~> 1"])
-      s.add_dependency(%q<httpthumbnailer>, ["~> 0.2"])
       s.add_dependency(%q<prawn>, ["= 0.8.4"])
+      s.add_dependency(%q<httpthumbnailer>, [">= 0"])
     end
   else
-    s.add_dependency(%q<sinatra>, [">= 1.2.6"])
-    s.add_dependency(%q<mongrel>, [">= 1.2.0.pre2"])
-    s.add_dependency(%q<s3>, ["~> 0.3"])
-    s.add_dependency(%q<httpthumbnailer-client>, ["~> 0.1.0"])
-    s.add_dependency(%q<ruby-ip>, ["~> 0.9"])
-    s.add_dependency(%q<cli>, ["~> 1.1.0"])
-    s.add_dependency(%q<mime-types>, ["~> 1.17.2"])
-    s.add_dependency(%q<retry-this>, ["~> 1.1"])
-    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+    s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.0"])
+    s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.0"])
+    s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
+    s.add_dependency(%q<mime-types>, ["~> 1.17"])
+    s.add_dependency(%q<sdl4r>, ["~> 0.9"])
+    s.add_dependency(%q<httpclient>, [">= 2.3"])
+    s.add_dependency(%q<rspec>, ["~> 2.13"])
     s.add_dependency(%q<cucumber>, [">= 0"])
-    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
     s.add_dependency(%q<rdoc>, ["~> 3.9"])
     s.add_dependency(%q<daemon>, ["~> 1"])
-    s.add_dependency(%q<httpthumbnailer>, ["~> 0.2"])
     s.add_dependency(%q<prawn>, ["= 0.8.4"])
+    s.add_dependency(%q<httpthumbnailer>, [">= 0"])
   end
 end
 

data/lib/httpimagestore/aws_sdk_regions_hack.rb

@@ -0,0 +1,23 @@
+## HACK: Auto select region based on location_constraint
+module AWS
+	class S3
+		class BucketCollection
+			def [](name)
+				# if name is DNS compatible we still cannot use it for writes if it does contain dots
+				return S3::Bucket.new(name.to_s, :owner => nil, :config => config) if client.dns_compatible_bucket_name?(name) and not name.include? '.'
+
+				# save region mapping for bucket for futher requests
+				@@location_cache = {} unless defined? @@location_cache
+				# if we have it cased use it; else try to fetch it and if it is nil bucket is in standard region
+				region = @@location_cache[name] || @@location_cache[name] = S3::Bucket.new(name.to_s, :owner => nil, :config => config).location_constraint || @@location_cache[name] = :standard
+
+				# no need to specify region if bucket is in standard region
+				return S3::Bucket.new(name.to_s, :owner => nil, :config => config) if region == :standard
+
+				# use same config but with region specified for buckets that are not DNS compatible or have dots and are not in standard region
+				S3::Bucket.new(name.to_s, :owner => nil, :config => config.with(region: region))
+			end
+		end
+	end
+end
+

data/lib/httpimagestore/configuration/file.rb

@@ -0,0 +1,120 @@
+require 'httpimagestore/configuration/path'
+require 'httpimagestore/configuration/handler'
+require 'pathname'
+
+module Configuration
+	class FileStorageOutsideOfRootDirError < ConfigurationError
+		def initialize(image_name, file_path)
+			super "error while processing image '#{image_name}': file storage path '#{file_path.to_s}' outside of root direcotry"
+		end
+	end
+
+	class NoSuchFileError < ConfigurationError
+		def initialize(image_name, file_path)
+			super "error while processing image '#{image_name}': file '#{file_path.to_s}' not found"
+		end
+	end
+
+	class FileSourceStoreBase < SourceStoreBase
+		extend Stats
+		def_stats(
+			:total_file_store, 
+			:total_file_store_bytes,
+			:total_file_source,
+			:total_file_source_bytes
+		)
+
+		def self.parse(configuration, node)
+			image_name = node.grab_values('image name').first
+			node.required_attributes('root', 'path')
+			root_dir, path_spec, if_image_name_on = *node.grab_attributes('root', 'path', 'if-image-name-on')
+			matcher = InclusionMatcher.new(image_name, if_image_name_on)
+
+			self.new(
+				configuration.global, 
+				image_name, 
+				matcher,
+				root_dir, 
+				path_spec 
+			)
+		end
+
+		def initialize(global, image_name, matcher, root_dir, path_spec)
+			super global, image_name, matcher
+			@root_dir = Pathname.new(root_dir).cleanpath
+			@path_spec = path_spec
+		end
+
+		def storage_path(rendered_path)
+			path = Pathname.new(rendered_path)
+
+			storage_path = (@root_dir + path).cleanpath
+			storage_path.to_s =~ /^#{@root_dir.to_s}/ or raise FileStorageOutsideOfRootDirError.new(@image_name, path)
+
+			storage_path
+		end
+	end
+
+	class FileSource < FileSourceStoreBase
+		include ClassLogging
+
+		def self.match(node)
+			node.name == 'source_file'
+		end
+
+		def self.parse(configuration, node)
+			configuration.image_sources << super
+		end
+
+		def realize(request_state)
+			put_sourced_named_image(request_state) do |image_name, rendered_path|
+				storage_path = storage_path(rendered_path)
+
+				log.info "sourcing '#{image_name}' from file '#{storage_path}'"
+				begin
+					data = storage_path.open('r') do |io|
+						request_state.memory_limit.io io
+						io.read
+					end
+					FileSourceStoreBase.stats.incr_total_file_source
+					FileSourceStoreBase.stats.incr_total_file_source_bytes(data.bytesize)
+
+					image = Image.new(data)
+					image.source_url = "file://#{rendered_path}"
+					image
+				rescue Errno::ENOENT
+					raise NoSuchFileError.new(image_name, rendered_path)
+				end
+			end
+		end
+	end
+	Handler::register_node_parser FileSource
+	
+	class FileStore < FileSourceStoreBase
+		include ClassLogging
+
+		def self.match(node)
+			node.name == 'store_file'
+		end
+
+		def self.parse(configuration, node)
+			configuration.stores << super
+		end
+
+		def realize(request_state)
+			get_named_image_for_storage(request_state) do |image_name, image, rendered_path|
+				storage_path = storage_path(rendered_path)
+
+				image.store_url = "file://#{rendered_path.to_s}"
+
+				log.info "storing '#{image_name}' in file '#{storage_path}' (#{image.data.length} bytes)"
+				storage_path.open('w'){|io| io.write image.data}
+				FileSourceStoreBase.stats.incr_total_file_store
+				FileSourceStoreBase.stats.incr_total_file_store_bytes(image.data.bytesize)
+			end
+		end
+	end
+	Handler::register_node_parser FileStore
+	StatsReporter << FileSourceStoreBase.stats
+end
+

data/lib/httpimagestore/configuration/handler.rb

@@ -0,0 +1,239 @@
+require 'mime/types'
+
+module Configuration
+	class ImageNotLoadedError < ConfigurationError
+		def initialize(image_name)
+			super "image '#{image_name}' not loaded"
+		end
+	end
+
+	class ZeroBodyLengthError < ConfigurationError
+		def initialize
+			super 'empty body - expected image data'
+		end
+	end
+
+	class RequestState
+		class Images < Hash
+			def initialize(memory_limit)
+				@memory_limit = memory_limit
+				super
+			end
+
+			def []=(name, image)
+				if member?(name)
+					@memory_limit.return fetch(name).data.bytesize
+				end
+				super
+			end
+
+			def [](name)
+				fetch(name){|image_name| raise ImageNotLoadedError.new(image_name)}
+			end
+		end
+		
+		def initialize(body = '', locals = {}, memory_limit = MemoryLimit.new)
+			@images = Images.new(memory_limit)
+			@locals = {body: body}.merge(locals)
+			@memory_limit = memory_limit
+			@output_callback = nil
+		end
+
+		attr_reader :images
+		attr_reader :locals
+		attr_reader :memory_limit
+
+		def output(&callback)
+			@output_callback = callback
+		end
+
+		def output_callback
+			@output_callback or fail 'no output callback'
+		end
+	end
+
+	module ImageMetaData
+		attr_accessor :source_path
+		attr_accessor :source_url
+		attr_accessor :store_path
+		attr_accessor :store_url
+
+		def mime_extension
+			return nil unless mime_type
+			mime = MIME::Types[mime_type].first
+			mime.extensions.select{|e| e.length == 3}.first or mime.extensions.first
+		end
+	end
+
+	class Image < Struct.new(:data, :mime_type)
+		include ImageMetaData
+	end
+
+	class InputSource
+		def realize(request_state)
+			request_state.locals[:body].empty? and raise ZeroBodyLengthError
+			request_state.images['input'] = Image.new(request_state.locals[:body])
+		end
+	end
+
+	class OutputOK
+		def realize(request_state)
+			request_state.output do
+				write_plain 200, 'OK'
+			end
+		end
+	end
+
+	class InclusionMatcher
+		def initialize(value, template)
+			@value = value
+			@template = RubyStringTemplate.new(template) if template
+		end
+
+		def included?(request_state)
+			return true if not @template
+			@template.render(request_state.locals).split(',').include? @value
+		end
+	end
+
+	module ConditionalInclusion
+		def inclusion_matcher(matcher)
+			(@matchers ||= []) << matcher if matcher
+		end
+
+		def included?(request_state)
+			return true unless @matchers
+			@matchers.any? do |matcher|
+				matcher.included?(request_state)
+			end
+		end
+
+		def excluded?(request_state)
+			not included? request_state
+		end
+	end
+
+	class SourceStoreBase
+		include ConditionalInclusion
+
+		def initialize(global, image_name, matcher)
+			@global = global
+			@image_name = image_name
+			@locals = {imagename: @image_name}
+			inclusion_matcher matcher
+		end
+
+		private
+
+		attr_accessor :image_name
+
+		def local(name, value)
+			@locals[name] = value
+		end
+
+		def rendered_path(request_state)
+			path = @global.paths[@path_spec]
+			Pathname.new(path.render(@locals.merge(request_state.locals))).cleanpath.to_s
+		end
+
+		def put_sourced_named_image(request_state)
+			rendered_path = rendered_path(request_state)
+
+			image = yield @image_name, rendered_path
+
+			image.source_path = rendered_path
+			request_state.images[@image_name] = image
+		end
+
+		def get_named_image_for_storage(request_state)
+			image = request_state.images[@image_name]
+			local :mimeextension, image.mime_extension
+
+			rendered_path = rendered_path(request_state)
+			image.store_path = rendered_path
+
+			yield @image_name, image, rendered_path
+		end
+	end
+
+	class Matcher
+		def initialize(name, &matcher)
+			@name = name
+			@matcher = matcher
+		end
+
+		attr_reader :name
+		attr_reader :matcher
+	end
+
+	class Handler < Scope
+		def self.match(node)
+			node.name == 'put' or
+			node.name == 'post' or
+			node.name == 'get'
+		end
+
+		def self.pre(configuration)
+			configuration.handlers ||= []
+		end
+
+		def self.parse(configuration, node)
+			handler_configuration = 
+				Struct.new(
+					:global,
+					:http_method,
+					:uri_matchers,
+					:image_sources,
+					:stores,
+					:output
+				).new
+
+			handler_configuration.global = configuration
+			handler_configuration.http_method = node.name
+			handler_configuration.uri_matchers = node.values.map do |matcher|
+				case matcher
+				when %r{^:[^/]+/.*/$}
+					name, regexp = *matcher.match(%r{^:([^/]+)/(.*)/$}).captures
+					Matcher.new(name.to_sym) do
+						Regexp.new("(#{regexp})")
+					end
+				when /^:.+\?$/
+					name = matcher.sub(/^:(.+)\?$/, '\1').to_sym
+					Matcher.new(name) do
+						->{match(name) || captures.push('')}
+					end
+				when /^:/
+					name = matcher.sub(/^:/, '').to_sym
+					Matcher.new(name) do
+						name
+					end
+				else
+					Matcher.new(nil) do
+						matcher
+					end
+				end
+			end
+			handler_configuration.image_sources = []
+			handler_configuration.stores = []
+			handler_configuration.output = nil
+
+			node.grab_attributes
+
+			if handler_configuration.http_method != 'get'
+				handler_configuration.image_sources << InputSource.new
+			end
+
+			configuration.handlers << handler_configuration
+
+			self.new(handler_configuration).parse(node)
+
+			handler_configuration.output = OutputOK.new unless handler_configuration.output
+		end
+
+		def self.post(configuration)
+			log.warn 'no handlers configured' if configuration.handlers.empty?
+		end
+	end
+	Global.register_node_parser Handler
+end
+

data/lib/httpimagestore/configuration/output.rb

@@ -0,0 +1,119 @@
+require 'httpimagestore/configuration/handler'
+
+module Configuration
+	class StorePathNotSetForImage < ConfigurationError
+		def initialize(image_name)
+			super "store path not set for image '#{image_name}'"
+		end
+	end
+
+	class StoreURLNotSetForImage < ConfigurationError
+		def initialize(image_name)
+			super "store URL not set for image '#{image_name}'"
+		end
+	end
+
+	class OutputMultiBase
+		class ImageName < String
+			include ConditionalInclusion
+
+			def initialize(name, matcher)
+				super name
+				inclusion_matcher matcher
+			end
+		end
+
+		def self.parse(configuration, node)
+			nodes = node.values.empty? ? node.children : [node]
+			names = nodes.map do |node|
+				image_name = node.grab_values('image name').first
+				matcher = InclusionMatcher.new(image_name, node.grab_attributes('if-image-name-on').first)
+				ImageName.new(image_name, matcher)
+			end
+
+			configuration.output and raise StatementCollisionError.new(node, 'output')
+			configuration.output = self.new(names)
+		end
+
+		def initialize(names)
+			@names = names
+		end
+	end
+	
+	class OutputImage
+		include ClassLogging
+
+		def self.match(node)
+			node.name == 'output_image'
+		end
+
+		def self.parse(configuration, node)
+			configuration.output and raise StatementCollisionError.new(node, 'output')
+			image_name = node.grab_values('image name').first
+			cache_control = node.grab_attributes('cache-control').first
+			configuration.output = OutputImage.new(image_name, cache_control)
+		end
+
+		def initialize(name, cache_control)
+			@name = name
+			@cache_control = cache_control
+		end
+
+		def realize(request_state)
+			image = request_state.images[@name]
+			mime_type = 
+				if image.mime_type
+					image.mime_type
+				else
+					log.warn "image '#{@name}' has no mime type; sending 'application/octet-stream' content type"
+					'application/octet-stream'
+				end
+
+			cache_control = @cache_control
+			request_state.output do
+				res['Cache-Control'] = cache_control if cache_control
+				write 200, mime_type, image.data
+			end
+		end
+	end
+	Handler::register_node_parser OutputImage
+
+	class OutputStorePath < OutputMultiBase
+		def self.match(node)
+			node.name == 'output_store_path'
+		end
+
+		def realize(request_state)
+			paths = @names.select do |name|
+				name.included?(request_state)
+			end.map do |name|
+				request_state.images[name].store_path or raise StorePathNotSetForImage.new(name)
+			end
+
+			request_state.output do
+				write_plain 200, paths
+			end
+		end
+	end
+	Handler::register_node_parser OutputStorePath
+
+	class OutputStoreURL < OutputMultiBase
+		def self.match(node)
+			node.name == 'output_store_url'
+		end
+
+		def realize(request_state)
+			urls = @names.select do |name|
+				name.included?(request_state)
+			end.map do |name|
+				request_state.images[name].store_url or raise StoreURLNotSetForImage.new(name)
+			end
+
+			request_state.output do
+				write_url_list 200, urls
+			end
+		end
+	end
+	Handler::register_node_parser OutputStoreURL
+end
+