httpimagestore 1.3.0 → 1.4.0

data/Gemfile

@@ -1,10 +1,8 @@
 source "http://rubygems.org"
 ruby "1.9.3"
 
-#gem "unicorn-cuba-base", "~> 1.0"
-gem "unicorn-cuba-base", path: '../unicorn-cuba-base'
-#gem "httpthumbnailer-client", "~> 1.0"
-gem "httpthumbnailer-client", path: '../httpthumbnailer-client'
+gem "unicorn-cuba-base", "~> 1.1"
+gem "httpthumbnailer-client", "~> 1.1"
 gem "aws-sdk", "~> 1.10"
 gem "mime-types", "~> 1.17"
 gem "sdl4r", "~> 0.9"

data/Gemfile.lock

@@ -1,63 +1,79 @@
 PATH
   remote: ../httpthumbnailer
   specs:
-    httpthumbnailer (1.0.0)
+    httpthumbnailer (1.1.0)
       rmagick (~> 2)
-      unicorn-cuba-base (~> 1.0)
-
-PATH
-  remote: ../httpthumbnailer-client
-  specs:
-    httpthumbnailer-client (1.0.0)
-      cli (~> 1.1.0)
-      httpclient (>= 2.3)
-      multipart-parser (~> 0.1.1)
-
-PATH
-  remote: ../unicorn-cuba-base
-  specs:
-    unicorn-cuba-base (1.0.0)
-      cli (~> 1.1.0)
-      cuba (~> 3.0)
-      facter (~> 1.6.11)
-      raindrops (~> 0.11)
-      ruby-ip (~> 0.9)
-      unicorn (>= 4.6.2)
+      unicorn-cuba-base (~> 1.1)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    aws-sdk (1.10.0)
+    addressable (2.3.5)
+    aws-sdk (1.18.0)
       json (~> 1.4)
-      nokogiri (>= 1.4.4)
+      nokogiri (< 1.6.0)
       uuidtools (~> 2.1)
-    builder (3.0.0)
+    builder (3.2.2)
     cli (1.1.1)
     cuba (3.1.0)
       rack
-    cucumber (1.2.1)
+    cucumber (1.3.8)
       builder (>= 2.1.2)
       diff-lcs (>= 1.1.3)
-      gherkin (~> 2.11.0)
-      json (>= 1.4.6)
+      gherkin (~> 2.12.1)
+      multi_json (>= 1.7.5, < 2.0)
+      multi_test (>= 0.0.2)
     daemon (1.1.0)
-    diff-lcs (1.1.3)
+    diff-lcs (1.2.4)
     facter (1.6.18)
-    gherkin (2.11.2)
-      json (>= 1.4.6)
-    git (1.2.5)
-    httpclient (2.3.3)
-    jeweler (1.8.4)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    gherkin (2.12.1)
+      multi_json (~> 1.3)
+    git (1.2.6)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (2.0.5)
+    highline (1.6.19)
+    httpauth (0.2.0)
+    httpclient (2.3.4.1)
+    httpthumbnailer-client (1.1.0)
+      cli (~> 1.1.0)
+      httpclient (>= 2.3)
+      multipart-parser (~> 0.1.1)
+    jeweler (1.8.7)
+      builder
       bundler (~> 1.0)
       git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
       rake
       rdoc
-    json (1.7.5)
-    kgio (2.8.0)
-    mime-types (1.17.2)
+    json (1.8.0)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    kgio (2.8.1)
+    mime-types (1.25)
     msgpack (0.5.5)
+    multi_json (1.8.0)
+    multi_test (0.0.2)
+    multi_xml (0.5.5)
     multipart-parser (0.1.1)
-    nokogiri (1.5.9)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
     prawn (0.8.4)
       prawn-core (>= 0.8.4, < 0.9)
       prawn-layout (>= 0.8.4, < 0.9)
@@ -66,25 +82,32 @@ GEM
     prawn-layout (0.8.4)
     prawn-security (0.8.4)
     rack (1.5.2)
-    raindrops (0.11.0)
-    rake (10.0.4)
-    rdoc (3.12)
+    raindrops (0.12.0)
+    rake (10.1.0)
+    rdoc (3.12.2)
       json (~> 1.4)
     rmagick (2.13.2)
-    rspec (2.13.0)
-      rspec-core (~> 2.13.0)
-      rspec-expectations (~> 2.13.0)
-      rspec-mocks (~> 2.13.0)
-    rspec-core (2.13.1)
-    rspec-expectations (2.13.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.5)
+    rspec-expectations (2.14.2)
       diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.13.1)
+    rspec-mocks (2.14.3)
     ruby-ip (0.9.1)
     sdl4r (0.9.11)
     unicorn (4.6.3)
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
+    unicorn-cuba-base (1.1.0)
+      cli (~> 1.1.0)
+      cuba (~> 3.0)
+      facter (~> 1.6.11)
+      raindrops (~> 0.11)
+      ruby-ip (~> 0.9)
+      unicorn (>= 4.6.2)
     uuidtools (2.1.4)
 
 PLATFORMS
@@ -96,7 +119,7 @@ DEPENDENCIES
   daemon (~> 1)
   httpclient (>= 2.3)
   httpthumbnailer!
-  httpthumbnailer-client!
+  httpthumbnailer-client (~> 1.1)
   jeweler (~> 1.8.4)
   mime-types (~> 1.17)
   msgpack (~> 0.5)
@@ -104,4 +127,4 @@ DEPENDENCIES
   rdoc (~> 3.9)
   rspec (~> 2.13)
   sdl4r (~> 0.9)
-  unicorn-cuba-base!
+  unicorn-cuba-base (~> 1.1)

data/README.md

@@ -11,11 +11,15 @@ It is using [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) as
 * sourcing and storage of images on [Amazon S3](http://aws.amazon.com/s3/)
 * image output with Cache-Control header
 * S3 public or private and http:// or https:// URL list output for stored images
+* S3 read-through and write-through object cache
 * storage under custom paths including image hash, content determined extension or used URL path
 * based on [Unicorn HTTP server](http://unicorn.bogomips.org) with UNIX socket communication support
 
 ## Changelog
 
+### 1.4.0
+* read-through and write-through S3 object cache support
+
 ### 1.3.0
 
 * `identify` statement support (requires [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) v1.1.0 or higher)
@@ -233,7 +237,8 @@ Options:
 
 * `bucket` - name of bucket to source image from
 * `path` - name of predefined path that will be used to generate key to object to source
-* `prefix` - prefix object key with given prefix value; this does not affect fromat of output URL; prefix will not be included in source path output; default: ``
+* `prefix` - prefix object key with given prefix value; this does not affect format of output URL; prefix will not be included in source path output; default: ``
+* `cache-root` - path to directory where S3 objects and meta-data will be cached when sourced from S3; read-through mode; if required information was found in cache object no S3 requests will be made; same directory can be used with different buckets since cache key consists of S3 bucket name and object key
 
 Example:
 
@@ -361,6 +366,7 @@ Options:
 * `path` - name of predefined path that will be used to generate key to store object under
 * `public` - if set to `true` the image will be readable by everybody; this affects fromat of output URL; default: `false`
 * `prefix` - prefix storeage key with given prefix value; this does not affect fromat of output URL; prefix will not be included in storage path output; default: ``
+* `cache-root` - path to directory where stored S3 objects and meta-data will be cached for future sourcing with `source_s3`; write-through mode; note that same directory needs to be configured with corresponding `source_s3` statement
 
 Example:
 
@@ -526,11 +532,12 @@ This option is useful when building API that works on predefined set of image op
 
 ### Flexible API example
 
-Features two storage apporaches: with JPEG conversion and limiting in size - for user provided content - and storing literaly.
+Features two storage approaches: with JPEG conversion and limiting in size - for user provided content - and storing literally.
 POST requests will end up with server side generated storage key based on input data digest.
-PUT requsts can be used to store image under provided storage key.
-Thumbnail GET API is similart to described in [Facebook APIs](https://developers.facebook.com/docs/reference/api/using-pictures/#sizes) for thumbnailing.
+PUT requests can be used to store image under provided storage key.
+Thumbnail GET API is similar to described in [Facebook APIs](https://developers.facebook.com/docs/reference/api/using-pictures/#sizes) for thumbnailing.
 Stored object extension and content type is determined from image data.
+S3 objects are cached on storage and on read if not cached already (read-through/write-through cache).
 
 ```sdl
 s3 key="AIAITCKMELYWQZPJP7HQ" secret="V37lCu0F48Tv9s7QVqIT/sLf/wwqhNSB4B0Em7Ei" ssl=false
@@ -541,77 +548,77 @@ path "path" "#{path}"
 ## User uploaded content - always JPEG converted, not bigger than 2160x2160 and in hight quality compression
 post "pictures" {
 	thumbnail "input" "original" operation="limit" width=2160 height=2160 format="jpeg" quality=95
-	store_s3 "original" bucket="mybucket" path="hash"
+	store_s3 "original" bucket="mybucket" path="hash" cache-root="/var/cache/httpimagestore"
 	output_store_path "original"
 }
 
 put "pictures" {
 	thumbnail "input" "original" operation="limit" width=2160 height=2160 format="jpeg" quality=95
-	store_s3 "original" bucket="mybucket" path="path"
+	store_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	output_store_path "original"
 }
 
 ## Uploaded by admin for use on the website for example - store whatever was send
 post "images" {
 	identify "input"
-	store_s3 "input" bucket="mybucket" path="hash"
+	store_s3 "input" bucket="mybucket" path="hash" cache-root="/var/cache/httpimagestore"
 	output_store_path "input"
 }
 
 put "images" {
 	identify "input"
-	store_s3 "input" bucket="mybucket" path="path"
+	store_s3 "input" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	output_store_path "input"
 }
 
 ## Thumbailing - keep input format; default JPEG quality is 85
 ### Thumbnail specification from query string paramaters
 get "pictures" "&:width" "&:height" "&:operation?crop" "&:background-color?white" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="background-color:#{background-color}"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 get "pictures" "&:width" "&:height?1080" "&:operation?fit" "&:background-color?white" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="background-color:#{background-color}"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 get "pictures" "&:height" "&:width?1080" "&:operation?fit" "&:background-color?white" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="background-color:#{background-color}"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 ### Predefined thumbnailing specification
 get "pictures" "&type=square" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="crop" width="50" height="50"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 get "pictures" "&type=small" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="fit" width="50" height="2000"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 get "pictures" "&type=normall" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="fit" width="100" height="2000"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 get "pictures" "&type=large" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	thumbnail "original" "thumbnail" operation="fit" width="200" height="2000"
 	output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
 }
 
 ## By default serve original image as is - JPEG for user content and what was send for admin uploaded images
 get "pictures" {
-	source_s3 "original" bucket="mybucket" path="path"
+	source_s3 "original" bucket="mybucket" path="path" cache-root="/var/cache/httpimagestore"
 	output_image "original" cache-control="public, max-age=31557600, s-maxage=0"
 }
 ```
@@ -719,7 +726,7 @@ Compatibility API works by storing input image and selected (via URI) classes of
 
 With thumbnail on demand API user uploads original image. It is converted to JPEG and if it is too large also scaled down. Than that processed version is stored in S3 under key composed from hash of input image data and final image extension. Client will receive storage key for further reference in the response body. To obtain thumbnail **GET** request with obtained key and thumbnail parameters encoded in the URI needs to be send to the sever. It will read parameters from the URI and source selected image from S3. That image is then thumbnailed in the backend and sent back to client with custom Cache-Control header.
 
-Note that Compatibility API will also store "migarion" image in bucket used by on demand API. This allows for migration from that API to on demand API.
+Note that Compatibility API will also store "migration" image in bucket used by on demand API. This allows for migration from that API to on demand API.
 
 Compatibility API example:
 

data/VERSION

@@ -1 +1 @@
-1.2.0
+1.4.0

data/features/s3-store-and-thumbnail.feature

@@ -16,7 +16,7 @@ Feature: Store limited original image in S3 and thumbnail based on request
 		put "original" {
 			thumbnail "input" "original" operation="limit" width=100 height=100 format="jpeg" quality=95
 
-			store_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="original-hash"
+			store_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="original-hash" cache-root="/tmp"
 
 			output_store_path "original"
 		}
@@ -30,7 +30,7 @@ Feature: Store limited original image in S3 and thumbnail based on request
 		}
 
 		get "thumbnail" "v2" ":operation" ":width" ":height" {
-			source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="path"
+			source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="path" cache-root="/tmp"
 
 			thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="#{query_string_options}" quality=84 format="png"
 

data/httpimagestore.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "httpimagestore"
-  s.version = "1.3.0"
+  s.version = "1.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jakub Pastuszek"]
-  s.date = "2013-09-11"
+  s.date = "2013-10-04"
   s.description = "Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)"
   s.email = "jpastuszek@gmail.com"
   s.executables = ["httpimagestore"]
@@ -86,6 +86,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.10"])
       s.add_runtime_dependency(%q<mime-types>, ["~> 1.17"])
       s.add_runtime_dependency(%q<sdl4r>, ["~> 0.9"])
+      s.add_runtime_dependency(%q<msgpack>, ["~> 0.5"])
       s.add_development_dependency(%q<httpclient>, [">= 2.3"])
       s.add_development_dependency(%q<rspec>, ["~> 2.13"])
       s.add_development_dependency(%q<cucumber>, [">= 0"])
@@ -100,6 +101,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
       s.add_dependency(%q<mime-types>, ["~> 1.17"])
       s.add_dependency(%q<sdl4r>, ["~> 0.9"])
+      s.add_dependency(%q<msgpack>, ["~> 0.5"])
       s.add_dependency(%q<httpclient>, [">= 2.3"])
       s.add_dependency(%q<rspec>, ["~> 2.13"])
       s.add_dependency(%q<cucumber>, [">= 0"])
@@ -115,6 +117,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
     s.add_dependency(%q<mime-types>, ["~> 1.17"])
     s.add_dependency(%q<sdl4r>, ["~> 0.9"])
+    s.add_dependency(%q<msgpack>, ["~> 0.5"])
     s.add_dependency(%q<httpclient>, [">= 2.3"])
     s.add_dependency(%q<rspec>, ["~> 2.13"])
     s.add_dependency(%q<cucumber>, [">= 0"])

data/lib/httpimagestore/configuration/file.rb

@@ -72,7 +72,7 @@ module Configuration
 
 				log.info "sourcing '#{image_name}' from file '#{storage_path}'"
 				begin
-					data = storage_path.open('r') do |io|
+					data = storage_path.open('rb') do |io|
 						request_state.memory_limit.io io
 						io.read
 					end
@@ -108,7 +108,7 @@ module Configuration
 				image.store_url = "file://#{rendered_path.to_s}"
 
 				log.info "storing '#{image_name}' in file '#{storage_path}' (#{image.data.length} bytes)"
-				storage_path.open('w'){|io| io.write image.data}
+				storage_path.open('wb'){|io| io.write image.data}
 				FileSourceStoreBase.stats.incr_total_file_store
 				FileSourceStoreBase.stats.incr_total_file_store_bytes(image.data.bytesize)
 			end

data/lib/httpimagestore/configuration/s3.rb

@@ -83,6 +83,64 @@ module Configuration
 				end
 			end
 
+			class CacheFile < Pathname
+				def initialize(path)
+					super
+					@header = nil
+				end
+
+				def header
+					begin
+						read(0)
+					rescue
+						@header = {}
+					end unless @header
+					@header or fail 'no header data'
+				end
+
+				def read(max_bytes = nil)
+					open('rb') do |io|
+						io.flock(File::LOCK_SH)
+						@header = read_header(io)
+						return io.read(max_bytes)
+					end
+				end
+
+				def write(data)
+					dirname.directory? or dirname.mkpath
+					open('ab') do |io|
+						# opened but not truncated before lock can be obtained
+						io.flock(File::LOCK_EX)
+
+						# now get rid of the old content if any
+						io.seek 0, IO::SEEK_SET
+						io.truncate 0
+
+						begin
+							header = MessagePack.pack(@header)
+							io.write [header.length].pack('L') # header length
+							io.write header
+							io.write data
+						rescue => error
+							unlink # remove broken cache file
+							raise
+						end
+					end
+				end
+
+				private
+
+				def read_header_length(io)
+					head_length = io.read(4)
+					fail 'no header length' unless head_length and head_length.length == 4
+					head_length.unpack('L').first
+				end
+
+				def read_header(io)
+					MessagePack.unpack(io.read(read_header_length(io)))
+				end
+			end
+
 			def initialize(root_dir)
 				@root = Pathname.new(root_dir)
 				@root.directory? or raise CacheRootNotDirError.new(root_dir)
@@ -91,23 +149,7 @@ module Configuration
 			end
 
 			def cache_file(bucket, key)
-				File.join(Digest::SHA2.new.update("#{bucket}/#{key}").to_s[0,32].match(/(..)(..)(.*)/).captures)
-			end
-
-			def open(bucket, key)
-				# TODO: locking
-				file = @root + cache_file(bucket, key)
-
-				file.dirname.directory? or file.dirname.mkpath
-				if file.exist?
-					file.open('r+') do |io|
-						yield io
-					end
-				else
-					file.open('w+') do |io|
-						yield io
-					end
-				end
+				CacheFile.new(File.join(@root.to_s, *Digest::SHA2.new.update("#{bucket}/#{key}").to_s[0,32].match(/(..)(..)(.*)/).captures))
 			end
 		end
 
@@ -147,91 +189,74 @@ module Configuration
 		end
 
 		class CacheObject < S3Object
+			extend Stats
+			def_stats(
+				:total_s3_cache_hits, 
+				:total_s3_cache_misses,
+				:total_s3_cache_errors,
+			)
+
 			include ClassLogging
 
-			def initialize(io, client, bucket, key)
-				@io = io
+			def initialize(cache_file, client, bucket, key)
 				super(client, bucket, key)
 
-				@header = {}
-				@have_cache = false
+				@cache_file = cache_file
 				@dirty = false
 
-				begin
-					head_length = @io.read(4)
-
-					if head_length and head_length.length == 4
-						head_length = head_length.unpack('L').first
-						@header = MessagePack.unpack(@io.read(head_length))
-						@have_cache = true
-
-						log.debug{"S3 object cache hit; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]: header: #{@header}"}
-					else
-						log.debug{"S3 object cache miss; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]"}
-					end
-				rescue => error
-					log.warn "cannot use cached S3 object; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]: #{error}"
-					# not usable
-					io.seek 0
-					io.truncate 0
-				end
-
 				yield self
 
-				# save object as was used if no error happened and there were changes
+				# save object if new data was read/written to/from S3 and no error happened
 				write_cache if dirty?
 			end
 
 			def read(max_bytes = nil)
-				if @have_cache
-					data_location = @io.seek(0, IO::SEEK_CUR)
-					begin
-						return @data = @io.read(max_bytes)
-					ensure
-						@io.seek(data_location, IO::SEEK_SET)
-					end
-				else
-					dirty! :read
-					return @data = super
+				begin
+					@data = @cache_file.read(max_bytes)
+					CacheObject.stats.incr_total_s3_cache_hits
+					log.debug{"S3 object cache hit for bucket: '#{@bucket}' key: '#{@key}' [#{@cache_file}]: header: #{@cache_file.header}"}
+					return @data
+				rescue Errno::ENOENT
+					CacheObject.stats.incr_total_s3_cache_misses
+					log.debug{"S3 object cache miss for bucket: '#{@bucket}' key: '#{@key}' [#{@cache_file}]"}
+				rescue => error
+					CacheObject.stats.incr_total_s3_cache_errors
+					log.warn "cannot use cached S3 object for bucket: '#{@bucket}' key: '#{@key}' [#{@cache_file}]", error
 				end
+				@data = super
+				dirty! :read
+				return @data
 			end
 
 			def write(data, options = {})
-				out = super
+				super
 				@data = data
+				@cache_file.header['content_type'] = options[:content_type] if options[:content_type]
 				dirty! :write
-				out
 			end
 
 			def private_url
-				@header['private_url'] ||= (dirty! :private_url; super)
+				@cache_file.header['private_url'] ||= (dirty! :private_url; super)
 			end
 
 			def public_url
-				@header['public_url'] ||= (dirty! :public_url; super)
+				@cache_file.header['public_url'] ||= (dirty! :public_url; super)
 			end
 
 			def content_type
-				@header['content_type'] ||= (dirty! :content_type; super)
+				@cache_file.header['content_type'] ||= (dirty! :content_type; super)
 			end
 
 			private
 
 			def write_cache
 				begin
-					log.debug{"S3 object is dirty, wirting cache file; bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]; header: #{@header}"}
+					log.debug{"S3 object is dirty, wirting cache file for bucket: '#{@bucket}' key: '#{@key}' [#{@cache_file}]; header: #{@cache_file.header}"}
 
 					raise 'nil data!' unless @data
-					# rewrite
-					@io.seek(0, IO::SEEK_SET)
-					@io.truncate 0
-
-					header = MessagePack.pack(@header)
-					@io.write [header.length].pack('L') # header length
-					@io.write header
-					@io.write @data
+					@cache_file.write(@data)
 				rescue => error
-					log.warn "cannot store S3 object in cache: bucket: '#{@bucket}' key: '#{@key}' [#{@io.path}]: #{error}"
+					log.warn "cannot store S3 object in cache for bucket: '#{@bucket}' key: '#{@key}' [#{@cache_file}]", error
 				ensure
 					@dirty = false
 				end
@@ -296,7 +321,7 @@ module Configuration
 					log.info "S3 object cache not configured (no cache-root) for image '#{image_name}'"
 				end
 			rescue CacheRoot::CacheRootNotDirError => error
-				log.warn "not using S3 object cache for image '#{image_name}': #{error}"
+				log.warn "not using S3 object cache for image '#{image_name}'", error
 			end
 
 			local :bucket, @bucket
@@ -321,18 +346,16 @@ module Configuration
 
 				if @cache_root
 					begin
-						@cache_root.open(@bucket, key) do |cahce_file_io|
-							CacheObject.new(cahce_file_io, client, @bucket, key) do |obj|
-								image = yield obj
-							end
+						cache_file = @cache_root.cache_file(@bucket, key)
+						CacheObject.new(cache_file, client, @bucket, key) do |obj|
+							image = yield obj
 						end
-					rescue IOError => error
-						log.warn "cannot use S3 object cache '#{@cache_root.cache_file(@bucket, key)}': #{error}"
-						image = yield obj
+						return image
+					rescue Errno::EACCES, IOError => error
+						log.warn "cannot use S3 object cache for bucket: '#{@bucket}' key: '#{key}' [#{cache_file}]", error
 					end
-				else
-					image = yield S3Object.new(client, @bucket, key)
 				end
+				return yield S3Object.new(client, @bucket, key)
 			rescue AWS::S3::Errors::AccessDenied
 				raise S3AccessDenied.new(@bucket, path)
 			rescue AWS::S3::Errors::NoSuchBucket
@@ -340,7 +363,6 @@ module Configuration
 			rescue AWS::S3::Errors::NoSuchKey
 				 raise S3NoSuchKeyError.new(@bucket, path)
 			end
-			image
 		end
 
 		S3SourceStoreBase.logger = Handler.logger_for(S3SourceStoreBase)
@@ -396,7 +418,7 @@ module Configuration
 
 					options = {}
 					options[:single_request] = true
-					options[:content_type] = image.mime_type
+					options[:content_type] = image.mime_type if image.mime_type
 					options[:acl] = acl
 					options[:cache_control] = @cache_control if @cache_control
 
@@ -411,5 +433,6 @@ module Configuration
 	end
 	Handler::register_node_parser S3Store
 	StatsReporter << S3SourceStoreBase.stats
+	StatsReporter << S3SourceStoreBase::CacheObject.stats
 end
 

data/spec/configuration_s3_spec.rb

@@ -72,9 +72,9 @@ else
 			end
 
 			before do
-				@cached_object = Pathname.new('/tmp/0d/bf/50c256d6b6efe55d11d0b6b50600')
-				@cached_object.dirname.mkpath
-				@cached_object.open('w') do |io|
+				@cache_file = Pathname.new('/tmp/0d/bf/50c256d6b6efe55d11d0b6b50600')
+				@cache_file.dirname.mkpath
+				@cache_file.open('w') do |io|
 					io.write 'abc'
 				end
 
@@ -83,19 +83,9 @@ else
 			end
 
 			it 'should build cache file location for storage location from bucket and key' do
-				subject.cache_file('mybucket', 'hello/world.jpg').should == "0d/bf/50c256d6b6efe55d11d0b6b50600"
-			end
-
-			it 'should look up object stored on disk by bucket and key' do
-				subject.open('mybucket', 'hello/world.jpg') do |io|
-					io.read.should == 'abc'
-				end
-			end
-
-			it 'should create cache object for bucket and key if it does not exist' do
-				subject.open('mybucket', 'hello/world2.jpg') do |io|
-					io.read.should == ''
-				end
+				cache_file = subject.cache_file('mybucket', 'hello/world.jpg')
+				cache_file.should be_a Configuration::S3SourceStoreBase::CacheRoot::CacheFile
+				cache_file.to_s.should == "/tmp/0d/bf/50c256d6b6efe55d11d0b6b50600"
 			end
 		end
 
@@ -190,13 +180,17 @@ else
 						source_s3 "original_cached_public" bucket="#{ENV['AWS_S3_TEST_BUCKET']}" path="hash" cache-root="/tmp" public="true"
 						source_s3 "original_cached_public2" bucket="#{ENV['AWS_S3_TEST_BUCKET']}" path="hash" cache-root="/tmp" public="true"
 					}
+
+					post {
+						store_s3 "input" bucket="#{ENV['AWS_S3_TEST_BUCKET']}" path="hash" cache-root="/tmp"
+					}
 					EOF
 				end
 
 				before do
-					@cached_object = Pathname.new('/tmp/ce/26/b196585e28aa99f55b1260b629e2')
-					@cached_object.dirname.mkpath
-					@cached_object.open('w') do |io|
+					@cache_file = Pathname.new('/tmp/ce/26/b196585e28aa99f55b1260b629e2')
+					@cache_file.dirname.mkpath
+					@cache_file.open('wb') do |io|
 						header = MessagePack.pack(
 							'private_url' => 'https://s3-eu-west-1.amazonaws.com/test/ghost.jpg?' + ENV['AWS_ACCESS_KEY_ID'],
 							'public_url' => 'https://s3-eu-west-1.amazonaws.com/test/ghost.jpg',
@@ -208,7 +202,7 @@ else
 					end
 				end
 
-				it 'should use cache when configured' do
+				it 'should use cache when configured and object in cache' do
 					subject.handlers[0].sources[0].should be_a Configuration::S3Source
 					subject.handlers[0].sources[0].realize(state)
 
@@ -240,64 +234,165 @@ else
 					state.images['original'].source_url.should == 'https://s3-eu-west-1.amazonaws.com/test/ghost.jpg'
 				end
 
-				it 'shluld use object stored in S3 when not found in the cache' do
-					cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
-					cache_file.unlink if cache_file.exist?
+				describe 'read-through' do
+					it 'shluld use object stored in S3 when not found in the cache' do
+						cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
+						cache_file.unlink if cache_file.exist?
 
-					state = Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
-					subject.handlers[0].sources[0].realize(state)
+						state = Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
+						subject.handlers[0].sources[0].realize(state)
 
-					cache_file.should exist
-				end
+						cache_file.should exist
+					end
 
-				it 'should use cached object writen when it was initialy read' do
-					cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
-					cache_file.unlink if cache_file.exist?
+					it 'should write cache on read and be able to use it on next read' do
+						cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
+						cache_file.unlink if cache_file.exist?
 
-					state = Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
-					subject.handlers[0].sources[0].realize(state)
+						state = Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
+						subject.handlers[0].sources[0].realize(state)
 
-					cache_file.should exist
+						cache_file.should exist
 
-					subject.handlers[0].sources[1].realize(state)
+						subject.handlers[0].sources[1].realize(state)
 
-					state.images['original'].data.should == @test_data
-					state.images['original'].mime_type.should == 'image/jpeg'
+						state.images['original'].data.should == @test_data
+						state.images['original'].mime_type.should == 'image/jpeg'
 
-					state.images['original_cached'].data.should == @test_data
-					state.images['original_cached'].mime_type.should == 'image/jpeg'
-				end
+						state.images['original_cached'].data.should == @test_data
+						state.images['original_cached'].mime_type.should == 'image/jpeg'
+					end
 
-				it 'should use update cached object when new properties are read from S3' do
-					cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
-					cache_file.unlink if cache_file.exist?
+					it 'should update cached object with new properties read from S3' do
+						cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
+						cache_file.unlink if cache_file.exist?
 
-					state = Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
+						state = Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
 
-					## cache with private URL
-					subject.handlers[0].sources[0].realize(state)
+						## cache with private URL
+						subject.handlers[0].sources[0].realize(state)
 
-					cache_file.should exist
-					sum = Digest::SHA2.new.update(cache_file.read).to_s
+						cache_file.should exist
+						sum = Digest::SHA2.new.update(cache_file.read).to_s
 
-					## read from cache with private URL
-					subject.handlers[0].sources[1].realize(state)
+						## read from cache with private URL
+						subject.handlers[0].sources[1].realize(state)
 
-					# no change
-					Digest::SHA2.new.update(cache_file.read).to_s.should == sum
+						# no change
+						Digest::SHA2.new.update(cache_file.read).to_s.should == sum
 
-					## read from cache; add public URL
-					subject.handlers[0].sources[2].realize(state)
+						## read from cache; add public URL
+						subject.handlers[0].sources[2].realize(state)
 
-					# should get updated
-					Digest::SHA2.new.update(cache_file.read).to_s.should_not == sum
-					
-					sum = Digest::SHA2.new.update(cache_file.read).to_s
-					## read from cahce
-					subject.handlers[0].sources[3].realize(state)
+						# should get updated
+						Digest::SHA2.new.update(cache_file.read).to_s.should_not == sum
+						
+						sum = Digest::SHA2.new.update(cache_file.read).to_s
+						## read from cahce
+						subject.handlers[0].sources[3].realize(state)
 
-					# no change
-					Digest::SHA2.new.update(cache_file.read).to_s.should == sum
+						# no change
+						Digest::SHA2.new.update(cache_file.read).to_s.should == sum
+					end
+
+					describe 'error handling' do
+						let :state do
+							Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
+						end
+						
+						before :each do
+							@cache_file = Pathname.new('/tmp/af/a3/5eaf0a614693e2d1ed455ac1cedb')
+							@cache_file.dirname.mkpath
+							@cache_file.open('wb') do |io|
+								header = 'xyz'
+								io.write [header.length].pack('L') # header length
+								io.write header
+								io.write 'abc'
+							end
+						end
+
+						it 'should rewrite cached object when corrupted' do
+							subject.handlers[0].sources[0].realize(state)
+							state.images['original'].data.should == @test_data
+
+							cache = @cache_file.read.force_encoding('ASCII-8BIT')
+							cache.should_not include 'xyz'
+							cache.should include @test_data
+						end
+
+						it 'should use S3 object when cache file is not accessible' do
+							@cache_file.chmod(0000)
+							begin
+								subject.handlers[0].sources[0].realize(state)
+								state.images['original'].data.should == @test_data
+							ensure
+								@cache_file.chmod(0644)
+
+								cache = @cache_file.read.force_encoding('ASCII-8BIT')
+								cache.should include 'xyz'
+								cache.should_not include @test_data
+							end
+						end
+
+						it 'should use S3 object when cache direcotry is not accessible' do
+							@cache_file.dirname.chmod(0000)
+							begin
+								subject.handlers[0].sources[0].realize(state)
+								state.images['original'].data.should == @test_data
+							ensure
+								@cache_file.dirname.chmod(0755)
+
+								cache = @cache_file.read.force_encoding('ASCII-8BIT')
+								cache.should include 'xyz'
+								cache.should_not include @test_data
+							end
+						end
+
+						it 'should not store cache file for S3 objects that does not exist' do
+							cache_file = Pathname.new('/tmp/a2/fd/4261e9a7586ed772d0c78bb51c9d')
+							cache_file.unlink if cache_file.exist?
+
+							state = Configuration::RequestState.new('abc', {test_image: 'bogous.jpg'})
+
+							expect {
+								subject.handlers[0].sources[0].realize(state)
+							}.to raise_error Configuration::S3NoSuchKeyError
+
+							cache_file.should_not exist
+						end
+					end
+				end
+
+				describe 'write-through' do
+					let :state do
+						Configuration::RequestState.new(@test_data, {test_image: 'test_cache.jpg'})
+					end
+
+					before :each do
+					end
+
+					it 'should cache S3 object during write' do
+						cache_file = Pathname.new('/tmp/31/f6/d48147b9981bb880fb1861539e3f')
+						cache_file.unlink if cache_file.exist?
+
+						subject.handlers[1].sources[0].realize(state)
+						state.images['input'].mime_type = 'image/jpeg'
+						subject.handlers[1].stores[0].realize(state)
+
+						# we have cache
+						cache_file.should exist
+
+						# but delete S3 so it will fail if cache was not used fully
+						s3_client = AWS::S3.new(use_ssl: false)
+						s3_test_bucket = s3_client.buckets[ENV['AWS_S3_TEST_BUCKET']]
+						s3_test_bucket.objects['test_cache.jpg'].delete
+
+						state = Configuration::RequestState.new('', {test_image: 'test_cache.jpg'})
+						expect {
+							subject.handlers[0].sources[0].realize(state)
+						}.not_to raise_error
+							state.images['original'].data.should == @test_data
+					end
 				end
 			end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: httpimagestore
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-11 00:00:00.000000000 Z
+date: 2013-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unicorn-cuba-base
@@ -91,6 +91,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
@@ -296,7 +312,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1883353100319806536
+      hash: -2918091045365099838
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: