httpimagestore 1.7.0 → 1.8.0

data/lib/httpimagestore/configuration/output.rb

@@ -69,9 +69,12 @@ module Configuration
 			attr_reader :image_name
 			attr_reader :path_spec
 
-			def initialize(global, image_name, path_spec, matcher)
+			def initialize(global, image_name, scheme, host, port, path_spec, matcher)
 				@global = global
 				@image_name = image_name
+				@scheme = scheme
+				@host = host
+				@port = port
 				@path_spec = path_spec
 				inclusion_matcher matcher
 			end
@@ -79,26 +82,42 @@ module Configuration
 			def store_path(request_state)
 				store_path = request_state.images[@image_name].store_path or raise StorePathNotSetForImage.new(@image_name)
 				return store_path unless @path_spec
-				rendered_path(store_path, request_state)
-			end
 
-			def store_url(request_state)
-				store_url = request_state.images[@image_name].store_url or raise StoreURLNotSetForImage.new(@image_name)
-				return store_url unless @path_spec
-				uri = URI(store_url)
-				uri.path = '/' + URI.encode(rendered_path(URI.decode(uri.path), request_state))
-				uri
+				locals = {
+					image_name: @image_name,
+					path: store_path
+				}
+
+				rendered_path(request_state.with_locals(locals))
 			end
 
-		private
+			def store_url(request_state)
+				url = request_state.images[@image_name].store_url or raise StoreURLNotSetForImage.new(@image_name)
+				url = url.dup
 
-			def rendered_path(store_path, request_state)
-				path = @global.paths[@path_spec]
 				locals = {
-					path: store_path,
-					image_name: @image_name
+					image_name: @image_name,
+					path: URI.utf_decode(url.path),
+					url: url.to_s
 				}
-				Pathname.new(path.render(request_state.with_locals(locals))).cleanpath.to_s
+				locals[:scheme] = url.scheme if url.scheme
+				locals[:host] = url.host if url.host
+				locals[:port] = url.port if url.port
+
+				request_state = request_state.with_locals(locals)
+
+				# optional rewrites
+				url.scheme = request_state.render_template(@scheme) if @scheme
+				url.host = request_state.render_template(@host) if @host
+				url.port = request_state.render_template(@port).to_i if @port
+				url.path = URI.encode(rendered_path(request_state)).tap{|path| path.replace('/' + path) if path[0] != '/'} if @path_spec
+
+				url
+			end
+
+		private
+			def rendered_path(request_state)
+				Pathname.new(@global.paths[@path_spec].render(request_state)).cleanpath.to_s
 			end
 		end
 
@@ -106,9 +125,9 @@ module Configuration
 			nodes = node.values.empty? ? node.children : [node]
 			output_specs = nodes.map do |node|
 				image_name = node.grab_values('image name').first
-				path_spec, if_image_name_on = *node.grab_attributes('path', 'if-image-name-on')
+				scheme, host, port, path_spec, if_image_name_on = *node.grab_attributes('scheme', 'host', 'port', 'path', 'if-image-name-on')
 				matcher = InclusionMatcher.new(image_name, if_image_name_on)
-				OutputSpec.new(configuration.global, image_name, path_spec, matcher)
+				OutputSpec.new(configuration.global, image_name, scheme, host, port, path_spec, matcher)
 			end
 
 			configuration.output and raise StatementCollisionError.new(node, 'output')
@@ -214,5 +233,24 @@ module Configuration
 		end
 	end
 	Handler::register_node_parser OutputStoreURL
+
+	class OutputStoreURI < OutputMultiBase
+		def self.match(node)
+			node.name == 'output_store_uri'
+		end
+
+		def realize(request_state)
+			urls = @output_specs.select do |output_spec|
+				output_spec.included?(request_state)
+			end.map do |output_spec|
+				output_spec.store_url(request_state).path
+			end
+
+			request_state.output do
+				write_url_list 200, urls
+			end
+		end
+	end
+	Handler::register_node_parser OutputStoreURI
 end
 

data/lib/httpimagestore/configuration/s3.rb

@@ -177,11 +177,11 @@ module Configuration
 			end
 
 			def private_url
-				s3_object.url_for(:read, expires: 60 * 60 * 24 * 365 * 20).to_s # expire in 20 years
+				s3_object.url_for(:read, expires: 60 * 60 * 24 * 365 * 20) # expire in 20 years
 			end
 
 			def public_url
-				s3_object.public_url.to_s
+				s3_object.public_url
 			end
 
 			def content_type
@@ -237,11 +237,19 @@ module Configuration
 			end
 
 			def private_url
-				@cache_file.header['private_url'] ||= (dirty! :private_url; super)
+				url = @cache_file.header['private_url'] and return URI(url)
+				dirty! :private_url
+				url = super
+				@cache_file.header['private_url'] = url.to_s
+				url
 			end
 
 			def public_url
-				@cache_file.header['public_url'] ||= (dirty! :public_url; super)
+				url = @cache_file.header['public_url'] and return URI(url)
+				dirty! :public_url
+				url = super
+				@cache_file.header['public_url'] = url.to_s
+				url
 			end
 
 			def content_type

data/spec/configuration_file_spec.rb

@@ -1,10 +1,9 @@
 require_relative 'spec_helper'
 require 'httpimagestore/configuration'
-Configuration::Scope.logger = Logger.new('/dev/null')
+MemoryLimit.logger = Configuration::Scope.logger = RootLogger.new('/dev/null')
 
 require 'httpimagestore/configuration/file'
 require 'httpimagestore/configuration/output'
-MemoryLimit.logger = Logger.new('/dev/null')
 
 describe Configuration do
 	let :state do
@@ -54,7 +53,7 @@ describe Configuration do
 			subject.handlers[0].sources[0].realize(state)
 
 			state.images['original'].source_path.should == "test.in"
-			state.images['original'].source_url.should == "file://test.in"
+			state.images['original'].source_url.to_s.should == "file:/test.in"
 		end
 
 		describe 'context locals' do
@@ -220,7 +219,7 @@ describe Configuration do
 			subject.handlers[0].stores[0].realize(state)
 
 			state.images['input'].store_path.should == "test.out"
-			state.images['input'].store_url.should == "file://test.out"
+			state.images['input'].store_url.to_s.should == "file:/test.out"
 		end
 
 		describe 'conditional inclusion support' do

data/spec/configuration_handler_spec.rb

@@ -2,11 +2,10 @@ require_relative 'spec_helper'
 require_relative 'support/cuba_response_env'
 
 require 'httpimagestore/configuration'
-Configuration::Scope.logger = Logger.new('/dev/null')
+MemoryLimit.logger = Configuration::Scope.logger = RootLogger.new('/dev/null')
 
 require 'httpimagestore/configuration/handler'
 require 'httpimagestore/configuration/output'
-MemoryLimit.logger = Logger.new('/dev/null')
 
 describe Configuration do
 	describe Configuration::Handler do

data/spec/configuration_identify_spec.rb

@@ -1,11 +1,10 @@
 require_relative 'spec_helper'
 require 'httpimagestore/configuration'
-Configuration::Scope.logger = Logger.new('/dev/null')
+MemoryLimit.logger = Configuration::Scope.logger = RootLogger.new('/dev/null')
 
 require 'httpimagestore/configuration/output'
 require 'httpimagestore/configuration/thumbnailer'
 require 'httpimagestore/configuration/identify'
-MemoryLimit.logger = Logger.new('/dev/null')
 
 describe Configuration do
 	describe 'identify' do

data/spec/configuration_output_spec.rb

@@ -2,11 +2,10 @@ require_relative 'spec_helper'
 require_relative 'support/cuba_response_env'
 
 require 'httpimagestore/configuration'
-Configuration::Scope.logger = Logger.new('/dev/null')
+MemoryLimit.logger = Configuration::Scope.logger = RootLogger.new('/dev/null')
 
 require 'httpimagestore/configuration/output'
 require 'httpimagestore/configuration/file'
-MemoryLimit.logger = Logger.new('/dev/null')
 
 describe Configuration do
 	let :state do
@@ -159,6 +158,10 @@ describe Configuration do
 	end
 
 	describe 'output store paths and URLs' do
+		let :utf_string do
+			(support_dir + 'utf_string.txt').read.strip
+		end
+
 		let :in_file do
 			Pathname.new("/tmp/test.in")
 		end
@@ -179,8 +182,12 @@ describe Configuration do
 			Pathname.new('/tmp/abc/t e s t.out')
 		end
 
+		let :utf_test_file do
+			Pathname.new("/tmp/abc/#{utf_string}.out")
+		end
+
 		before :each do
-			test_file.dirname.mkdir
+			test_file.dirname.mkdir unless test_file.dirname.directory?
 			test_file.open('w'){|io| io.write('abc')}
 			space_test_file.open('w'){|io| io.write('abc')}
 			in_file.open('w'){|io| io.write('abc')}
@@ -191,6 +198,7 @@ describe Configuration do
 		after :each do
 			test_file.exist? and test_file.unlink
 			space_test_file.exist? and space_test_file.unlink
+			utf_test_file.exist? and utf_test_file.unlink
 			test_file.dirname.exist? and test_file.dirname.rmdir
 			out_file.unlink if out_file.exist?
 			out2_file.unlink if out2_file.exist?
@@ -391,7 +399,7 @@ describe Configuration do
 
 				env.instance_eval &state.output_callback
 				env.res['Content-Type'].should == 'text/uri-list'
-				env.res.data.should == "file://test.out\r\n"
+				env.res.data.should == "file:/test.out\r\n"
 			end
 
 			it 'should provide multiple file store URLs' do
@@ -423,7 +431,7 @@ describe Configuration do
 
 				env.instance_eval &state.output_callback
 				env.res['Content-Type'].should == 'text/uri-list'
-				env.res.data.should == "file://test.out\r\nfile://test.out2\r\n"
+				env.res.data.should == "file:/test.out\r\nfile:/test.out2\r\n"
 			end
 
 			describe 'conditional inclusion support' do
@@ -468,12 +476,12 @@ describe Configuration do
 
 					env.instance_eval &state.output_callback
 					env.res['Content-Type'].should == 'text/uri-list'
-					env.res.data.should == "file://test.out1\r\nfile://test.out3\r\n"
+					env.res.data.should == "file:/test.out1\r\nfile:/test.out3\r\n"
 				end
 			end
 
-			describe 'custom formatting' do
-				it 'should provide formatted file store URL' do
+			describe 'URL rewrites' do
+				it 'should allow using path spec to rewrite URL path component' do
 					subject = Configuration.read(<<-'EOF')
 					path  "out"	  "abc/test.out"
 
@@ -492,7 +500,93 @@ describe Configuration do
 
 					env.instance_eval &state.output_callback
 					env.res['Content-Type'].should == 'text/uri-list'
-					env.res.data.should == "file://abc/hello/world/test-xyz.out\r\n"
+					env.res.data.should == "file:/hello/abc/world/test-xyz.out\r\n"
+				end
+
+				it 'should allow rewriting scheme component' do
+					subject = Configuration.read(<<-'EOF')
+					path  "out"	  "abc/test.out"
+
+					post "single" {
+						store_file "input" root="/tmp" path="out"
+
+						output_store_url "input" scheme="ftp"
+					}
+					EOF
+
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "ftp:/abc/test.out\r\n"
+				end
+
+				it 'should allow rewriting host component' do
+					subject = Configuration.read(<<-'EOF')
+					path  "out"	  "abc/test.out"
+
+					post "single" {
+						store_file "input" root="/tmp" path="out"
+
+						output_store_url "input" host="localhost"
+					}
+					EOF
+
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "file://localhost/abc/test.out\r\n"
+				end
+
+				it 'should allow rewriting port component' do
+					subject = Configuration.read(<<-'EOF')
+					path  "out"	  "abc/test.out"
+
+					post "single" {
+						store_file "input" root="/tmp" path="out"
+
+						output_store_url "input" port="21"
+					}
+					EOF
+
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "file::21/abc/test.out\r\n"
+				end
+
+				it 'should allow using variables for all supported rewrites' do
+					state = Configuration::RequestState.new('abc',
+						remote: 'example.com',
+						remote_port: 21,
+						proto: 'ftp'
+					)
+					subject = Configuration.read(<<-'EOF')
+					path  "out"	  "abc/test.out"
+					path  "formatted"	  "hello/#{dirname}/world/#{basename}-xyz.#{extension}"
+
+					post "single" {
+						store_file "input" root="/tmp" path="out"
+
+						output_store_url "input" scheme="#{proto}" host="#{remote}" port="#{remote_port}" path="formatted"
+					}
+					EOF
+
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "ftp://example.com:21/hello/abc/world/test-xyz.out\r\n"
 				end
 			end
 
@@ -518,7 +612,7 @@ describe Configuration do
 
 					env.instance_eval &state.output_callback
 					env.res['Content-Type'].should == 'text/uri-list'
-					env.res.data.should == "file://abc/t%20e%20s%20t.out\r\nfile://abc/hello/world/t%20e%20s%20t-xyz.out\r\n"
+					env.res.data.should == "file:/abc/t%20e%20s%20t.out\r\nfile:/hello/abc/world/t%20e%20s%20t-xyz.out\r\n"
 				end
 			end
 
@@ -538,5 +632,190 @@ describe Configuration do
 				end
 			end
 		end
+
+		describe Configuration::OutputStoreURI do
+			it 'should provide file store URI' do
+				subject = Configuration.read(<<-EOF)
+				path {
+					"out"	"test.out"
+				}
+
+				post "single" {
+					store_file "input" root="/tmp" path="out"
+
+					output_store_uri "input"
+				}
+				EOF
+
+				subject.handlers[0].sources[0].realize(state)
+				subject.handlers[0].stores[0].realize(state)
+				subject.handlers[0].output.realize(state)
+
+				env.instance_eval &state.output_callback
+				env.res['Content-Type'].should == 'text/uri-list'
+				env.res.data.should == "/test.out\r\n"
+			end
+
+			it 'should provide multiple file store URIs' do
+				subject = Configuration.read(<<-EOF)
+				path {
+					"in"	"test.in"
+					"out"	"test.out"
+					"out2"	"test.out2"
+				}
+
+				post "multi" {
+					source_file "original" root="/tmp" path="in"
+
+					store_file "input" root="/tmp" path="out"
+					store_file "original" root="/tmp" path="out2"
+
+					output_store_uri {
+						"input"
+						"original"
+					}
+				}
+				EOF
+
+				subject.handlers[0].sources[0].realize(state)
+				subject.handlers[0].sources[1].realize(state)
+				subject.handlers[0].stores[0].realize(state)
+				subject.handlers[0].stores[1].realize(state)
+				subject.handlers[0].output.realize(state)
+
+				env.instance_eval &state.output_callback
+				env.res['Content-Type'].should == 'text/uri-list'
+				env.res.data.should == "/test.out\r\n/test.out2\r\n"
+			end
+
+			describe 'conditional inclusion support' do
+				let :state do
+					Configuration::RequestState.new('abc', list: 'input,image2')
+				end
+
+				subject do
+					Configuration.read(<<-'EOF')
+					path {
+						"in"		"test.in"
+						"out1"	"test.out1"
+						"out2"	"test.out2"
+						"out3"	"test.out3"
+					}
+
+					post "multi" {
+						source_file "image1" root="/tmp" path="in"
+						source_file "image2" root="/tmp" path="in"
+
+						store_file "input" root="/tmp" path="out1"
+						store_file "image1" root="/tmp" path="out2"
+						store_file "image2" root="/tmp" path="out3"
+
+						output_store_uri {
+							"input"		if-image-name-on="#{list}"
+							"image1"	if-image-name-on="#{list}"
+							"image2"	if-image-name-on="#{list}"
+						}
+					}
+					EOF
+				end
+
+				it 'should output store url only for images that names match if-image-name-on list' do
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].sources[1].realize(state)
+					subject.handlers[0].sources[2].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].stores[1].realize(state)
+					subject.handlers[0].stores[2].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "/test.out1\r\n/test.out3\r\n"
+				end
+			end
+
+			describe 'URI rewrites' do
+				it 'should allow using path spec to rewrite URI path' do
+					subject = Configuration.read(<<-'EOF')
+					path  "out"	  "abc/test.out"
+
+					path  "formatted"	  "hello/#{dirname}/world/#{basename}-xyz.#{extension}"
+
+					post "single" {
+						store_file "input" root="/tmp" path="out"
+
+						output_store_uri "input" path="formatted"
+					}
+					EOF
+
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "/hello/abc/world/test-xyz.out\r\n"
+				end
+			end
+
+			describe 'URI encoding' do
+				let :subject do
+					Configuration.read(<<-'EOF')
+					path  "out"         "abc/#{name}.out"
+					path  "formatted"   "hello/#{dirname}/world/#{basename}-xyz.#{extension}"
+
+					post "single" {
+						store_file "input" root="/tmp" path="out"
+
+						output_store_uri {
+							"input"
+							"input" path="formatted"
+						}
+					}
+					EOF
+				end
+
+				it 'should provide properly encoded file store URI' do
+					state = Configuration::RequestState.new('abc', name: 't e s t')
+
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					env.res.data.should == "/abc/t%20e%20s%20t.out\r\n/hello/abc/world/t%20e%20s%20t-xyz.out\r\n"
+				end
+
+				it 'should handle UTF-8 characters' do
+					state = Configuration::RequestState.new('abc', name: utf_string)
+					subject.handlers[0].sources[0].realize(state)
+					subject.handlers[0].stores[0].realize(state)
+					subject.handlers[0].output.realize(state)
+
+					env.instance_eval &state.output_callback
+					env.res['Content-Type'].should == 'text/uri-list'
+					l1, l2 = *env.res.data.split("\r\n")
+					URI.utf_decode(l1).should == "/abc/#{utf_string}.out"
+					URI.utf_decode(l2).should == "/hello/abc/world/#{utf_string}-xyz.out"
+				end
+			end
+
+			describe 'error handling' do
+				it 'should raise StoreURLNotSetForImage for output of not stored image' do
+					subject = Configuration.read(<<-EOF)
+					post "single" {
+						output_store_uri "input"
+					}
+					EOF
+
+					subject.handlers[0].sources[0].realize(state)
+
+					expect {
+						subject.handlers[0].output.realize(state)
+					}.to raise_error Configuration::StoreURLNotSetForImage, %{store URL not set for image 'input'}
+				end
+			end
+		end
 	end
 end

data/spec/configuration_path_spec.rb

@@ -1,10 +1,9 @@
 require_relative 'spec_helper'
 require 'httpimagestore/configuration'
-Configuration::Scope.logger = Logger.new('/dev/null')
+MemoryLimit.logger = Configuration::Scope.logger = RootLogger.new('/dev/null')
 
 require 'httpimagestore/configuration/handler'
 require 'httpimagestore/configuration/path'
-MemoryLimit.logger = Logger.new('/dev/null')
 
 describe Configuration do
 	describe 'path rendering' do
@@ -62,7 +61,7 @@ describe Configuration do
 
 				expect {
 					subject.paths['test'].render
-				}.to raise_error Configuration::NoValueForPathTemplatePlaceholerError, %q{cannot generate path 'test' from template '#{abc}#{xyz}': no value for '#{abc}'} 
+				}.to raise_error Configuration::NoValueForPathTemplatePlaceholerError, %q{cannot generate path 'test' from template '#{abc}#{xyz}': no value for '#{abc}'}
 			end
 		end