swiftiply 0.6.1.1 → 1.0.0

data/src/swiftcore/Swiftiply/rest_based_cluster_manager.rb

@@ -0,0 +1,15 @@
+require 'em-http'
+module Swiftcore
+	module Swiftiply
+		module ManagerProtocols
+			class RestBasedClusterManager
+				def self.call(callsite, params)
+					EventMachine::HttpRequest.new(callsite).get
+					true
+				rescue Exception
+					false
+				end
+			end
+		end		
+	end
+end

data/src/swiftcore/Swiftiply/splay_cache_base.rb

@@ -0,0 +1,21 @@
+require 'swiftcore/Swiftiply/cache_base_mixin'
+
+module Swiftcore
+	# Use Array instead of Deque, if Deque wasn't available.
+	Deque = Array unless HasDeque or const_defined?(:Deque)
+	
+	module Swiftiply
+		class CacheBase < Swiftcore::SplayTreeMap
+			include CacheBaseMixin
+			def initialize(vw = 900, time_limit = 0.05, maxsize = 100)
+				@vw = vw
+				@tl = time_limit
+				@vwtl = vw * time_limit
+				@old_vql = 0
+				@vq = Deque.new
+				super()
+				self.max_size = maxsize
+			end
+		end
+	end
+end

data/src/swiftcore/Swiftiply/support_pagecache.rb

@@ -33,16 +33,19 @@ module Swiftcore
 				
 				def find_static_file(dr,path_info,client_name)
 					path = File.join(dr,path_info)
-					if FileTest.exist?(path) and FileTest.file?(path)
+					puts path					
+					if FileTest.exist?(path) and FileTest.file?(path) and File.expand_path(path).index(dr) == 0 and !(x = static_mask(client_name) and path =~ x)
 						path
 					elsif @suffix_list.has_key?(client_name)
 						path = File.join(dr,@cache_dir[client_name],path_info)
-						if FileTest.exist?(path) and FileTest.file?(path)
+						if FileTest.exist?(path) and FileTest.file?(path) and File.expand_path(path).index(dr) == 0 and !(x = static_mask(client_name) and path =~ x)
 							path
 						else
 							for suffix in @suffix_list[client_name] do
 								p = "#{path}.#{suffix}"
-								return p if FileTest.exist?(p) and FileTest.file?(p)
+								if FileTest.exist?(p) and FileTest.file?(p) and File.expand_path(p).index(dr) == 0 and !(x = static_mask(client_name) and p =~ x)
+									return p
+								end
 							end
 							nil
 						end

data/src/swiftcore/Swiftiply/swiftiply_2_http_proxy.rb

@@ -0,0 +1,7 @@
+module Swiftiply
+  class Swiftiply2HttpProxy
+    def self.run(config)
+    end
+
+  end
+end

data/src/swiftcore/Swiftiply/swiftiply_client.rb

@@ -1,6 +1,20 @@
-# -*- coding: ISO-8859-1 -*-
-require 'eventmachine'
-require 'socket'
+# Encoding:ascii-8bit
+
+begin
+	load_attempted ||= false
+	require 'eventmachine'
+	require 'socket'
+rescue LoadError => e
+	unless load_attempted
+		load_attempted = true
+		# Ugh.  Everything gets slower once rubygems are used.  So, for the
+		# best speed possible, don't install EventMachine or Swiftiply via
+		# gems.
+		require 'rubygems'
+		retry
+	end
+	raise e
+end
 
 # This is a basic Swiftiply client implementation.
 
@@ -22,7 +36,8 @@ class SwiftiplyClientProtocol < EventMachine::Connection
 			conn.port = port
 			conn.key = key
 			ip = conn.ip = conn.__get_ip(hostname)
-			conn.id = 'swiftclient' << ip.collect {|x| sprintf('%02x',x.to_i)}.join << sprintf('%04x',port.to_i)<< sprintf('%02x',key.length) << key
+			#conn.id = 'swiftclient' << ip.collect {|x| sprintf('%02x',x.to_i)}.join << sprintf('%04x',port.to_i)<< sprintf('%02x',key.length) << key
+			conn.id = 'swiftclient' << ip.collect {|x| sprintf('%02x',x.to_i)}.join << sprintf('%04x',$$)<< sprintf('%02x',key.length) << key
 			conn.set_comm_inactivity_timeout inactivity_timeout
 		end
 	end
@@ -54,4 +69,4 @@ class SwiftiplyClientProtocol < EventMachine::Connection
 	def __get_ip hostname
 		Socket.gethostbyname(hostname)[3].unpack(CCCCC) rescue ip = C0s
 	end
-end
+end

data/src/swiftcore/Swiftiply/version.rb

@@ -0,0 +1,5 @@
+module Swiftcore
+  module Swiftiply
+    VERSION = '1.0.0'
+  end
+end

data/src/swiftcore/evented_mongrel.rb

@@ -19,6 +19,10 @@ require 'mongrel'
 
 module Mongrel
 	class MongrelProtocol < EventMachine::Connection
+		
+		Cblank = ''.freeze
+		C400Header = "HTTP/1.0 400 Bad Request\r\nContent-Type: text/plain\r\nServer: Swiftiplied Mongrel 0.6.5\r\nConnection: close\r\n\r\n"
+		
 		def post_init
 			@parser = HttpParser.new
 			@params = HttpParams.new
@@ -34,11 +38,16 @@ module Mongrel
 			if @parser.finished?
 				if @request_len.nil?
 					@request_len = @params[::Mongrel::Const::CONTENT_LENGTH].to_i
-					script_name, path_info, handlers = ::Mongrel::HttpServer::Instance.classifier.resolve(@params[::Mongrel::Const::REQUEST_PATH])
+					script_name, path_info, handlers = ::Mongrel::HttpServer::Instance.classifier.resolve(@params[::Mongrel::Const::REQUEST_PATH] || Cblank)
 					if handlers
 						@params[::Mongrel::Const::PATH_INFO] = path_info
 						@params[::Mongrel::Const::SCRIPT_NAME] = script_name
-						@params[::Mongrel::Const::REMOTE_ADDR] = @params[::Mongrel::Const::HTTP_X_FORWARDED_FOR] || ::Socket.unpack_sockaddr_in(get_peername)[1]
+						# The previous behavior of this line set REMOTE_ADDR equal to HTTP_X_FORWARDED_FOR
+						# if it was defined.  This behavior seems inconsistent with the intention of
+						# http://www.ietf.org/rfc/rfc3875 so it has been changed.  REMOTE_ADDR now always
+						# contains the address of the immediate source of the connection.  Check
+						# @params[::Mongrel::Const::HTTP_X_FORWARDED_FOR] if you need that information.
+						@params[::Mongrel::Const::REMOTE_ADDR] = ::Socket.unpack_sockaddr_in(get_peername)[1] rescue nil
 						@notifiers = handlers.select { |h| h.request_notify }
 					end
 					if @request_len > ::Mongrel::Const::MAX_BODY
@@ -65,10 +74,12 @@ module Mongrel
 				STDERR.puts "#{Time.now}: BAD CLIENT (#{params[Const::HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #$!"
 				STDERR.puts "#{Time.now}: REQUEST DATA: #{data.inspect}\n---\nPARAMS: #{params.inspect}\n---\n"
 			end
-			close_connection
+			send_data C400Header
+			close_connection_after_writing
 		rescue Exception => e
-			close_connection
 			raise e
+			send_data C400Header
+			close_connection_after_writing
 		end
 
 		def write data
@@ -102,23 +113,28 @@ module Mongrel
 		def run
 			trap('INT') { raise StopServer }
 			trap('TERM') { raise StopServer }
-			@acceptor = Thread.new do
+			#@acceptor = Thread.new do
+			@acceptor = Thread.current
+				# SHOULD NOT DO THIS AUTOMATICALLY.
+				# There either needs to be a way to configure this, or to detect
+				# when it is safe or when kqueue needs to run.
 				EventMachine.epoll
 				EventMachine.set_descriptor_table_size(4096)
 				EventMachine.run do
+					EM.set_timer_quantum(5)
 					begin
-						EventMachine.start_server(@host,@port,MongrelProtocol)
+						EventMachine.start_server(@host,@port.to_i,MongrelProtocol)
 					rescue StopServer
 						EventMachine.stop_event_loop
 					end
 				end
-			end
+			#end
 		end
 
 		def process_http_request(params,linebuffer,client)
 			if not params[Const::REQUEST_PATH]
 				uri = URI.parse(params[Const::REQUEST_URI])
-				params[Const::REQUEST_PATH] = uri.request_uri
+				params[Const::REQUEST_PATH] = uri.path
 			end
 
  			raise "No REQUEST PATH" if not params[Const::REQUEST_PATH]
@@ -129,6 +145,8 @@ module Mongrel
 				notifiers = handlers.select { |h| h.request_notify }
 				request = HttpRequest.new(params, linebuffer, notifiers)
 
+				# TODO: Add Keep-Alive support
+
 				# request is good so far, continue processing the response
 				response = HttpResponse.new(client)
 

data/src/swiftcore/hash.rb

@@ -0,0 +1,43 @@
+class Hash
+
+	# Hash#merge doesn't work right if any of the values in the hash are themselves
+	# hashes.  This creates frowns on my normally smiling face.  This method
+	# will walk a hash, merging it, while preserving default_proc's that may
+	# exist on the hashes involved.  The code was adopted from IOWA.  It's also
+	# quite likely that it sucks.  I wrote it years ago, and have left it alone
+	# since then.  If you need similar functionality in your code somewhere,
+	# though, feel free to use this.  If you make it better, I'd appreciate it
+	# if you would send me your patches, though.
+	
+	def rmerge!(h)
+		h.each do |k,v|
+			if v.kind_of?(::Hash)
+				if self[k].kind_of?(::Hash)
+					unless self[k].respond_to?(:rmerge!)
+						if dp = self[k].default_proc
+							self[k] = Hash.new {|h,k| dp.call(h,k)}.rmerge!(self[k])
+						else
+							osk = self[k]
+							self[k] = Hash.new
+							self[k].rmerge!(osk)
+						end
+					end
+					self[k].rmerge!(v)
+				else
+					if self.default_proc
+						self.delete k
+						self[k]
+					end
+					unless self[k].kind_of?(::Hash)
+						self.delete k
+						self[k] = Hash.new
+					end
+					self[k].rmerge!(v)
+				end
+			else
+				self[k] = v
+			end
+		end
+	end
+	
+end

data/src/swiftcore/method_builder.rb

@@ -0,0 +1,28 @@
+# Mixin to allow dynamic construction of instance methods.
+
+module MethodBuilder
+	def method_builder(method_name, method_args, chunks, callback_args = [])
+		if Array === method_args
+			pma = [] method_args.each do |ma|
+				if Array === ma
+					pma << "#{ma[0]}=#{ma[1]}"
+				else
+					pma << ma.to_s
+				end
+			end
+			parsed_method_args = pma.join(',')
+		else
+			parsed_method_args = method_args.to_s
+		end
+
+		m = "def #{method_name}(#{parsed_method_args})\n" chunks.each do |chunk|
+			if chunk.respond_to?(:call)
+				m << chunk.call(*args)
+			else
+				m << chunk.to_s
+			end
+		end m << "\nend"
+
+		class_eval m
+	end
+end

data/src/swiftcore/streamer.rb

@@ -0,0 +1,46 @@
+begin
+	load_attempted ||= false
+	require 'em/streamer'
+rescue LoadError
+	unless load_attempted
+		load_attempted = true
+		require 'rubygems'
+		retry
+	end
+end
+
+module EventMachine
+	class FileStreamer
+
+		def stream_one_chunk
+			loop {
+				if @position < @size
+					if @connection.get_outbound_data_size > BackpressureLevel
+						EventMachine::next_tick {stream_one_chunk}
+						break
+					else
+						len = @size - @position
+						len = ChunkSize if (len > ChunkSize)
+
+						#@connection.send_data( "#{format("%x",len)}\r\n" ) if @http_chunks
+						if @http_chunks
+							@connection.send_data( "#{len.to_s(16)}\r\n" ) if @http_chunks
+							@connection.send_data( @mapping.get_chunk( @position, len ))
+							@connection.send_data("\r\n") if @http_chunks
+						else
+							@connection.send_data( @mapping.get_chunk( @position, len ))
+						end
+
+						@position += len
+					end
+				else
+					@connection.send_data "0\r\n\r\n" if @http_chunks
+					@mapping.close
+					succeed
+					break
+				end
+			}
+		end
+
+	end
+end

data/src/swiftcore/swiftiplied_mongrel.rb

@@ -3,7 +3,7 @@
 # application running inside an EventMachine event loop.  It should
 # be compatible with the existing Mongrel handlers for Rails,
 # Camping, Nitro, etc....
- 
+
 begin
 	load_attempted ||= false
 	require 'eventmachine'
@@ -20,6 +20,9 @@ end
 module Mongrel
 	C0s = [0,0,0,0].freeze unless const_defined?(:C0s)
 	CCCCC = 'CCCC'.freeze unless const_defined?(:CCCCC)
+	Cblank = ''.freeze
+
+	C400Header = "HTTP/1.0 400 Bad Request\r\nContent-Type: text/plain\r\nServer: Swiftiplied Mongrel 0.6.5\r\nConnection: close\r\n\r\n"
 
 	class MongrelProtocol < SwiftiplyClientProtocol
 
@@ -34,15 +37,26 @@ module Mongrel
 
 		def receive_data data
 			@linebuffer << data
+
 			@nparsed = @parser.execute(@params, @linebuffer, @nparsed) unless @parser.finished?
 			if @parser.finished?
-				if @request_len.nil?
+
+				unless @params[::Mongrel::Const::REQUEST_PATH]
+					params[::Mongrel::Const::REQUEST_PATH] = URI.parse(params[::Mongrel::Const::REQUEST_URI]).path
+				end
+
+				unless @request_len
 					@request_len = @params[::Mongrel::Const::CONTENT_LENGTH].to_i
 					script_name, path_info, handlers = ::Mongrel::HttpServer::Instance.classifier.resolve(@params[::Mongrel::Const::REQUEST_PATH])
 					if handlers
-						@params[::Mongrel::Const::PATH_INFO] = path_info
+						@params[::Mongrel::Const::PATH_INFO] = path_info || Cblank # path_info shouldn't be nil, but just in case it somehow is, let's make sure we don't crash later because of it.
 						@params[::Mongrel::Const::SCRIPT_NAME] = script_name
-						@params[::Mongrel::Const::REMOTE_ADDR] = @params[::Mongrel::Const::HTTP_X_FORWARDED_FOR] || ::Socket.unpack_sockaddr_in(get_peername)[1]
+						# The previous behavior of this line set REMOTE_ADDR equal to HTTP_X_FORWARDED_FOR
+						# if it was defined.  This behavior seems inconsistent with the intention of
+						# http://www.ietf.org/rfc/rfc3875 so it has been changed.  REMOTE_ADDR now always
+						# contains the address of the immediate source of the connection.  Check
+						# @params[::Mongrel::Const::HTTP_X_FORWARDED_FOR] if you need that information.
+						@params[::Mongrel::Const::REMOTE_ADDR] = ::Socket.unpack_sockaddr_in(get_peername)[1] rescue nil
 						@notifiers = handlers.select { |h| h.request_notify }
 					end
 					if @request_len > ::Mongrel::Const::MAX_BODY
@@ -58,7 +72,7 @@ module Mongrel
 				if @linebuffer.length >= @request_len
 					@linebuffer.rewind
 					::Mongrel::HttpServer::Instance.process_http_request(@params,@linebuffer,self)
-					@linebuffer.delete if Tempfile === @linebuffer
+					@linebuffer.delete if Tempfile === @linebuffer && FileTest.exist?(@linebuffer.path.to_s)
 					post_init
 				end
 			elsif @linebuffer.length > ::Mongrel::Const::MAX_HEADER
@@ -70,10 +84,17 @@ module Mongrel
 				STDERR.puts "#{Time.now}: BAD CLIENT (#{params[Const::HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #$!"
 				STDERR.puts "#{Time.now}: REQUEST DATA: #{data.inspect}\n---\nPARAMS: #{params.inspect}\n---\n"
 			end
-			close_connection
+			send_data C400Header
+			close_connection_after_writing
 		rescue Exception => e
-			close_connection
+			# This isn't a parse error; if this rescue caught an exception, then something
+			# significant happened.  
 			raise e
+			send_data C400Header
+			close_connection_after_writing
+		ensure
+			# Make sure to cleanup the Tempfile.
+			@linebuffer.delete if Tempfile === @linebuffer && FileTest.exist?(@linebuffer.path.to_s)
 		end
 
 		def write data
@@ -84,6 +105,7 @@ module Mongrel
 			false
 		end
 
+    def flush; end # EM handles flushing, not us, so this is just here for compatibility.
 	end
 
 	class HttpServer
@@ -93,7 +115,12 @@ module Mongrel
 		# The support is here, for swiftiplied_mongrels which are secured by
 		# a key.  If someone want to donate any patches.  Otherwise, this won't
 		# really be useful to most people until 0.7.0.
-		
+
+		CHTTP_CONNECTION = 'HTTP_CONNECTION'.freeze
+		CHTTP_VERSION = 'HTTP_VERSION'.freeze
+		CKEEP_ALIVE = 'KEEP_ALIVE'.freeze
+		C1_1 = '1.1'.freeze
+
 		def initialize(host, port, num_processors=950, x=0, y=nil,key='') # Deal with Mongrel 1.0.1 or earlier, as well as later.
 			@socket = nil
 			@classifier = URIClassifier.new
@@ -115,24 +142,26 @@ module Mongrel
 		def run
 			trap('INT') { EventMachine.stop_event_loop }
 			trap('TERM') { EventMachine.stop_event_loop }
-			@acceptor = Thread.new do
+			#@acceptor = Thread.new do
+			@acceptor = Thread.current
 				EventMachine.run do
+					EM.set_timer_quantum(5)
 					begin
-						MongrelProtocol.connect(@host,@port,@key)
+						MongrelProtocol.connect(@host,@port.to_i,@key)
 					rescue StopServer
 						EventMachine.stop_event_loop
 					end
 				end
-			end
+			#end
 		end
 
 		def process_http_request(params,linebuffer,client)
 			if not params[Const::REQUEST_PATH]
 				uri = URI.parse(params[Const::REQUEST_URI])
-				params[Const::REQUEST_PATH] = uri.request_uri
+				params[Const::REQUEST_PATH] = uri.path
 			end
 
- 			raise "No REQUEST PATH" if not params[Const::REQUEST_PATH]
+			raise "No REQUEST PATH" if not params[Const::REQUEST_PATH]
 
 			script_name, path_info, handlers = @classifier.resolve(params[Const::REQUEST_PATH])
 
@@ -140,8 +169,15 @@ module Mongrel
 				notifiers = handlers.select { |h| h.request_notify }
 				request = HttpRequest.new(params, linebuffer, notifiers)
 
+				http_version = params[CHTTP_VERSION]
+				if http_version == C1_1
+					keep_alive = params[CHTTP_CONNECTION] =~ /close/i ? false : true
+				else
+					keep_alive = params[CHTTP_CONNECTION] =~ /alive/i ? true : false
+				end
+
 				# request is good so far, continue processing the response
-				response = HttpResponse.new(client)
+				response = HttpResponse.new(client,http_version,keep_alive)
 
 				# Process each handler in registered order until we run out or one finalizes the response.
 				dispatch_to_handlers(handlers,request,response)
@@ -156,19 +192,20 @@ module Mongrel
 				# header MUST accompany all HTTP responses that go into a swiftiply
 				# keepalive connection, so just use the Response object to construct the
 				# 404 response.
+				response = HttpResponse.new(client)
 				response.status = 404
-				response.body = "#{params[Const::REQUEST_PATH]} not found"
+				response.body << "#{params[Const::REQUEST_PATH]} not found"
 				response.finished
 			end
 		end
-		
+
 		def dispatch_to_handlers(handlers,request,response)
 			handlers.each do |handler|
 				handler.process(request, response)
 				break if response.done
 			end
 		end
-		
+
 	end
 
 	class HttpRequest
@@ -180,6 +217,25 @@ module Mongrel
 	end
 
 	class HttpResponse
+
+		CContentLength = 'Content-Length'.freeze
+		C1_1 = '1.1'.freeze
+
+		def initialize(socket,http_version = C1_1, keepalive = false)
+			@socket = socket
+			@body = StringIO.new
+			@status = 404
+			@reason = nil
+			@header = HeaderOut.new(StringIO.new)
+			@header[Const::DATE] = Time.now.httpdate
+			@body_sent = false
+			@header_sent = false
+			@status_sent = false
+			@http_version = http_version
+			#@keepalive = keepalive
+      @keepalive = false ## DISABLE ALL KEEPALIVE
+		end
+
 		def send_file(path, small_file = false)
 			File.open(path, "rb") do |f|
 				while chunk = f.read(Const::CHUNK_SIZE) and chunk.length > 0
@@ -193,6 +249,18 @@ module Mongrel
 			@body_sent = true
 		end
 
+		def send_status(content_length=@body.length)
+			unless @status_sent
+				@header[CContentLength] = content_length if content_length && @status != 304
+				if @keepalive
+					write("HTTP/1.1 #{@status} #{@reason || HTTP_STATUS_CODES[@status]}\r\nConnection: Keep-Alive\r\n")
+				else
+					write("HTTP/1.1 #{@status} #{@reason || HTTP_STATUS_CODES[@status]}\r\nConnection: Close\r\n")
+				end
+				@status_sent = true
+			end
+		end
+
 		def write(data)
 			@socket.send_data data
 		end
@@ -204,12 +272,12 @@ module Mongrel
 		end
 
 		def finished
-			send_status
-			send_header
-			send_body
+  		send_status
+	  	send_header
+		 	send_body
 		end
 	end
-	
+
 	class Configurator
 		# This version fixes a bug in the regular Mongrel version by adding
 		# initialization of groups.
@@ -218,12 +286,12 @@ module Mongrel
 				log "Initializing groups for {#user}:{#group}."
 				Process.initgroups(user,Etc.getgrnam(group).gid)
 			end
-			
+
 			if group
 				log "Changing group to #{group}."
 				Process::GID.change_privilege(Etc.getgrnam(group).gid)
 			end
-			
+
 			if user
 				log "Changing user to #{user}."
 				Process::UID.change_privilege(Etc.getpwnam(user).uid)