swiftiply 0.6.1.1 → 1.0.0

data/external/httpclient.rb

@@ -0,0 +1,231 @@
+# This is a replacement for the regular EventMachine HttpClient library.
+# It removes a few lines that seek to protect the user against himself.
+# This makes it a much more useful tool for sending wacky request lines
+# to a web server.
+
+
+module EventMachine
+  module Protocols
+
+    class HttpClient < Connection
+      include EventMachine::Deferrable
+
+      remove_const :MaxPostContentLength if MaxPostContentLength
+      MaxPostContentLength = 20 * 1024 * 1024
+
+      # USAGE SAMPLE:
+      #
+      # EventMachine.run {
+      #   http = EventMachine::Protocols::HttpClient.request(
+      #     :host => server,
+      #     :port => 80,
+      #     :request => "/index.html",
+      #     :query_string => "parm1=value1&parm2=value2"
+      #   )
+      #   http.callback {|response|
+      #     puts response[:status]
+      #     puts response[:headers]
+      #     puts response[:content]
+      #   }
+      # }
+      #
+
+      # TODO:
+      # Add streaming so we can support enormous POSTs. Current max is 20meg.
+      # Timeout for connections that run too long or hang somewhere in the middle.
+      # Persistent connections (HTTP/1.1), may need a associated delegate object.
+      # DNS: Some way to cache DNS lookups for hostnames we connect to. Ruby's
+      # DNS lookups are unbelievably slow.
+      # HEAD requests.
+      # Chunked transfer encoding.
+      # Convenience methods for requests. get, post, url, etc.
+      # SSL.
+      # Handle status codes like 304, 100, etc.
+      # Refactor this code so that protocol errors all get handled one way (an exception?),
+      # instead of sprinkling set_deferred_status :failed calls everywhere.
+
+      def self.request( args = {} )
+        args[:port] ||= 80
+        EventMachine.connect( args[:host], args[:port], self ) {|c|
+          # According to the docs, we will get here AFTER post_init is called.
+          c.instance_eval {@args = args}
+        }
+      end
+
+      def post_init
+        @start_time = Time.now
+        @data = ""
+        @read_state = :base
+      end
+
+      # We send the request when we get a connection.
+      # AND, we set an instance variable to indicate we passed through here.
+      # That allows #unbind to know whether there was a successful connection.
+      # NB: This naive technique won't work when we have to support multiple
+      # requests on a single connection.
+      def connection_completed
+        @connected = true
+        send_request @args
+      end
+
+      def send_request args
+        args[:verb] ||= args[:method] # Support :method as an alternative to :verb.
+        args[:verb] ||= :get # IS THIS A GOOD IDEA, to default to GET if nothing was specified?
+
+        verb = args[:verb].to_s.upcase
+        unless ["GET", "POST", "PUT", "DELETE", "HEAD"].include?(verb)
+          set_deferred_status :failed, {:status => 0} # TODO, not signalling the error type
+          return # NOTE THE EARLY RETURN, we're not sending any data.
+        end
+
+        request = args[:request] || "/"
+    #    unless request[0,1] == "/"
+    #      request = "/" + request
+    #    end
+
+        qs = args[:query_string] || ""
+        if qs.length > 0 and qs[0,1] != '?'
+          qs = "?" + qs
+        end
+
+        # Allow an override for the host header if it's not the connect-string.
+        host = args[:host_header] || args[:host] || "_"
+        # For now, ALWAYS tuck in the port string, although we may want to omit it if it's the default.
+        port = args[:port]
+
+        # POST items.
+        postcontenttype = args[:contenttype] || "application/octet-stream"
+        postcontent = args[:content] || ""
+        raise "oversized content in HTTP POST" if postcontent.length > MaxPostContentLength
+
+        # ESSENTIAL for the request's line-endings to be CRLF, not LF. Some servers misbehave otherwise.
+        # TODO: We ASSUME the caller wants to send a 1.1 request. May not be a good assumption.
+        req = [
+          "#{verb} #{request}#{qs} HTTP/1.1",
+          "Host: #{host}:#{port}",
+          "User-agent: Ruby EventMachine",
+        ]
+
+        if verb == "POST" || verb == "PUT"
+          req << "Content-type: #{postcontenttype}"
+          req << "Content-length: #{postcontent.length}"
+        end
+
+        # TODO, this cookie handler assumes it's getting a single, semicolon-delimited string.
+        # Eventually we will want to deal intelligently with arrays and hashes.
+        if args[:cookie]
+          req << "Cookie: #{args[:cookie]}"
+        end
+
+        req << ""
+        reqstring = req.map {|l| "#{l}\r\n"}.join
+        send_data reqstring
+
+        if verb == "POST" || verb == "PUT"
+          send_data postcontent
+        end
+      end
+
+
+      def receive_data data
+        while data and data.length > 0
+          case @read_state
+          when :base
+            # Perform any per-request initialization here and don't consume any data.
+            @data = ""
+            @headers = []
+            @content_length = nil # not zero
+            @content = ""
+            @status = nil
+            @read_state = :header
+          when :header
+            ary = data.split( /\r?\n/m, 2 )
+            if ary.length == 2
+              data = ary.last
+              if ary.first == ""
+                if @content_length and @content_length > 0
+  	              @read_state = :content
+	              else
+                  dispatch_response
+                  @read_state = :base
+                end
+              else
+                @headers << ary.first
+                if @headers.length == 1
+                  parse_response_line
+                elsif ary.first =~ /\Acontent-length:\s*/i
+                  # Only take the FIRST content-length header that appears,
+                  # which we can distinguish because @content_length is nil.
+                  # TODO, it's actually a fatal error if there is more than one
+                  # content-length header, because the caller is presumptively
+                  # a bad guy. (There is an exploit that depends on multiple
+                  # content-length headers.)
+                  @content_length ||= $'.to_i
+                end
+              end
+            else
+              @data << data
+              data = ""
+            end
+          when :content
+            # If there was no content-length header, we have to wait until the connection
+            # closes. Everything we get until that point is content.
+            # TODO: Must impose a content-size limit, and also must implement chunking.
+            # Also, must support either temporary files for large content, or calling
+            # a content-consumer block supplied by the user.
+            if @content_length
+              bytes_needed = @content_length - @content.length
+              @content += data[0, bytes_needed]
+              data = data[bytes_needed..-1] || ""
+              if @content_length == @content.length
+                dispatch_response
+                @read_state = :base
+              end
+            else
+              @content << data
+              data = ""
+            end
+          end
+        end
+      end
+
+
+      # We get called here when we have received an HTTP response line.
+      # It's an opportunity to throw an exception or trigger other exceptional
+      # handling.
+      def parse_response_line
+        if @headers.first =~ /\AHTTP\/1\.[01] ([\d]{3})/
+          @status = $1.to_i
+        else
+          set_deferred_status :failed, {
+            :status => 0 # crappy way of signifying an unrecognized response. TODO, find a better way to do this.
+          }
+          close_connection
+        end
+      end
+      private :parse_response_line
+
+      def dispatch_response
+        @read_state = :base
+        set_deferred_status :succeeded, {
+          :content => @content,
+          :headers => @headers,
+          :status => @status
+        }
+        # TODO, we close the connection for now, but this is wrong for persistent clients.
+        close_connection
+      end
+
+      def unbind
+        if !@connected
+          set_deferred_status :failed, {:status => 0} # YECCCCH. Find a better way to signal no-connect/network error.
+        elsif (@read_state == :content and @content_length == nil)
+          dispatch_response
+        end
+      end
+    end
+
+  end
+end
+
+

data/external/package.rb

@@ -36,9 +36,9 @@ end
 
 def self.config(name)
 	# XXX use pathname
-	prefix = Regexp.quote(Config::CONFIG["prefix"])
-	exec_prefix = Regexp.quote(Config::CONFIG["exec_prefix"])
-	Config::CONFIG[name].gsub(/\A\/?(#{prefix}|#{exec_prefix})\/?/, '')
+	prefix = Regexp.quote(RbConfig::CONFIG["prefix"])
+	exec_prefix = Regexp.quote(RbConfig::CONFIG["exec_prefix"])
+	RbConfig::CONFIG[name].gsub(/\A\/?(#{prefix}|#{exec_prefix})\/?/, '')
 end
 
 SITE_DIRS = {
@@ -70,7 +70,7 @@ MODES = {
 
 
 SETUP_OPTIONS = {:parse_cmdline => true, :load_conf => true, :run_tasks => true}
-RUBY_BIN = File.join(::Config::CONFIG['bindir'],::Config::CONFIG['ruby_install_name']) << ::Config::CONFIG['EXEEXT']
+RUBY_BIN = File.join(::RbConfig::CONFIG['bindir'],::RbConfig::CONFIG['ruby_install_name']) << ::RbConfig::CONFIG['EXEEXT']
 
 def self.setup(version, options = {}, &instructions)
 	prefixes = dirs = nil
@@ -118,7 +118,7 @@ module Actions
 			d = File.expand_path(File.join(dirs)).w32
 			FileUtils.install @source, d,
 				{:verbose => @options.verbose,
-				:noop => @options.noop, :mode => @mode }
+				:noop => @options.noop, :mode => @mode } rescue Errno::ENOENT
 		end
 
 		def hash
@@ -197,9 +197,9 @@ module Actions
 						File.open(tmpfile, 'w', 0755) do |w|
 							first = r.gets
 							return unless SHEBANG_RE =~ first
-							ruby = File.join(::Config::CONFIG['bindir'],::Config::CONFIG['ruby_install_name'])
-							ruby << ::Config::CONFIG['EXEEXT']
-							#w.print first.sub(SHEBANG_RE, '#!' + Config::CONFIG['ruby-prog'])
+							ruby = File.join(::RbConfig::CONFIG['bindir'],::RbConfig::CONFIG['ruby_install_name'])
+							ruby << ::RbConfig::CONFIG['EXEEXT']
+							#w.print first.sub(SHEBANG_RE, '#!' + RbConfig::CONFIG['ruby-prog'])
 							w.print first.sub(SHEBANG_RE, '#!' + ruby)
 							w.write r.read
 						end
@@ -308,7 +308,7 @@ class PackageSpecification_1_0
 			end
 			#TODO: refactor
 			self.class.declare_file_type(args) do |files, ignore_p, opt_rename_info|
-				files.each do |file|
+				(Array === files ? files : [files]).each do |file|
 					next if ignore_p && IGNORE_FILES.any?{|re| re.match(file)}
 					add_file(kind, file, opt_rename_info, &bin_callback)
 				end
@@ -397,7 +397,7 @@ class PackageSpecification_1_0
 	end
 
 	def initialize(prefixes = nil, dirs = nil)
-		@prefix = Config::CONFIG["prefix"].gsub(/\A\//, '')
+		@prefix = RbConfig::CONFIG["prefix"].gsub(/\A\//, '')
 		@translate = {}
 		@prefixes = (prefixes || {}).dup
 		KINDS.each do |kind|
@@ -429,7 +429,7 @@ class PackageSpecification_1_0
 			__send__ kind, Dir["#{kind}/**/*"]
 		end
 		translate(:ext, "ext/*" => "", :inherit => true)
-		ext Dir["ext/**/*.#{Config::CONFIG['DLEXT']}"]
+		ext Dir["ext/**/*.#{RbConfig::CONFIG['DLEXT']}"]
 	end
 
 	# Builds any needed extensions.
@@ -660,7 +660,7 @@ class PackageSpecification_1_0
 	end
 
 	def run_tasks
-		@tasks.each { |task| __send__ task }
+		@tasks.each { |task| puts "Doing #{task}"; __send__ task }
 	end
 end
 
@@ -668,5 +668,5 @@ end # module Package
 
 require 'rbconfig'
 def config(x)
-	Config::CONFIG[x]
+	RbConfig::CONFIG[x]
 end

data/setup.rb

@@ -11,12 +11,25 @@ end
 
 Dir.chdir(basedir)
 Package.setup("1.0") {
-	name "Swiftcore Swiftiply"
+	# TODO pull version right from the code's version.rb.
+	name "Swiftcore Swiftiply v. 0.6.5"
 
 	build_ext "fastfilereader"
 	translate(:ext, 'ext/fastfilereader/' => '/')
 	#translate(:ext, 'ext/http11/' => 'iowa/')
+
 	ext "ext/fastfilereader/fastfilereaderext.so"
+	ext "ext/fastfilereader/fastfilereaderext.bundle"
+	
+	build_ext "deque"
+	translate(:ext, 'ext/deque/' => '/swiftcore/')
+	ext "ext/deque/deque.so"
+	ext "ext/deque/deque.bundle"
+
+	build_ext "splaytree"
+	translate(:ext, 'ext/splaytree/' => '/swiftcore/')
+	ext "ext/splaytree/splaytreemap.so"
+	ext "ext/splaytree/splaytreemap.bundle"
 
 	translate(:lib, 'src/' => '')
 	translate(:bin, 'bin/' => '')
@@ -28,9 +41,12 @@ Package.setup("1.0") {
 	bin "bin/swiftiply"
 	bin "bin/swiftiply_mongrel_rails"
 	#File.rename("#{Config::CONFIG["bindir"]}/mongrel_rails","#{Config::CONFIG["bindir"]}/mongrel_rails.orig")
-	bin "bin/mongrel_rails"
+	bin "bin/swiftiplied_mongrel_rails"
+	bin "bin/evented_mongrel_rails"
+	bin "bin/swiftiplyctl"
 
 	unit_test "test/TC_ProxyBag.rb"
 	unit_test "test/TC_Swiftiply.rb"
+	unit_test "test/TC_Deque.rb"
 	true
 }

data/src/swiftcore/Swiftiply.rb

@@ -1,776 +1,420 @@
-begin
-	load_attempted ||= false
-	require 'digest/sha2'
-	require 'eventmachine'
-	require 'fastfilereaderext'
-	require 'swiftcore/types'
-rescue LoadError => e
-	unless load_attempted
-		load_attempted = true
-		# Ugh.  Everything gets slower once rubygems are used.  So, for the
-		# best speed possible, don't install EventMachine or Swiftiply via
-		# gems.
-		require 'rubygems'
-		retry
-	end
-	raise e
-end
-
 module Swiftcore
-	module Swiftiply
-		Version = '0.6.1.1'
-
-		# Yeah, these constants look kind of tacky.  Inside of tight loops,
-		# though, using them makes a small but measurable difference, and those
-		# small differences add up....
-		C_empty = ''.freeze
-		C_slash = '/'.freeze
-		C_slashindex_html = '/index.html'.freeze
-		Caos = 'application/octet-stream'.freeze
-		Ccache_directory = 'cache_directory'.freeze
-		Ccache_extensions = 'cache_extensions'.freeze
-		Ccluster_address = 'cluster_address'.freeze
-		Ccluster_port = 'cluster_port'.freeze
-		Ccluster_server = 'cluster_server'.freeze
-		CBackendAddress = 'BackendAddress'.freeze
-		CBackendPort = 'BackendPort'.freeze
-		Cchunked_encoding_threshold = 'chunked_encoding_threshold'.freeze
-		Cdaemonize = 'daemonize'.freeze
-		Cdefault = 'default'.freeze
-		Cdocroot = 'docroot'.freeze
-		Cepoll = 'epoll'.freeze
-		Cepoll_descriptors = 'epoll_descriptors'.freeze
-		Cgroup = 'group'.freeze
-		Chost = 'host'.freeze
-		Cincoming = 'incoming'.freeze
-		Ckeepalive = 'keepalive'.freeze
-		Ckey = 'key'.freeze
-		Cmap = 'map'.freeze
-		Cmsg_expired = 'browser connection expired'.freeze
-		Coutgoing = 'outgoing'.freeze
-		Cport = 'port'.freeze
-		Credeployable = 'redeployable'.freeze
-		Credeployment_sizelimit = 'redeployment_sizelimit'.freeze
-		Cswiftclient = 'swiftclient'.freeze
-		Ctimeout = 'timeout'.freeze
-		Curl = 'url'.freeze
-		Cuser = 'user'.freeze
-
-		C_fsep = File::SEPARATOR
-		
-		RunningConfig = {}
-
-		class EMStartServerError < RuntimeError; end
-		
-		# The ProxyBag is a class that holds the client and the server queues,
-		# and that is responsible for managing them, matching them, and expiring
-		# them, if necessary.
-
-		class ProxyBag
-			@client_q = Hash.new {|h,k| h[k] = []}
-			@server_q = Hash.new {|h,k| h[k] = []}
-			@ctime = Time.now
-			@server_unavailable_timeout = 6
-			@id_map = {}
-			@reverse_id_map = {}
-			@incoming_map = {}
-			@docroot_map = {}
-			@log_map = {}
-			@redeployable_map = {}
-			@keys = {}
-			@demanding_clients = Hash.new {|h,k| h[k] = []}
-			@hitcounters = Hash.new {|h,k| h[k] = 0}
-			# Kids, don't do this at home.  It's gross.
-			@typer = MIME::Types.instance_variable_get('@__types__')
-
-			class << self
-
-				def now
-					@ctime
-				end
-
-				# Returns the access key.  If an access key is set, then all new backend
-				# connections must send the correct access key before being added to
-				# the cluster as a valid backend.
-
-				def get_key(h)
-					@keys[h] || C_empty
-				end
-
-				def set_key(h,val)
-					@keys[h] = val
-				end
-
-				def add_id(who,what)
-					@id_map[who] = what
-					@reverse_id_map[what] = who
-				end
-
-				def remove_id(who)
-					what = @id_map.delete(who)
-					@reverse_id_map.delete(what)
-				end
-
-				def incoming_mapping(name)
-					@incoming_map[name]
-				end
-				
-				def add_incoming_mapping(hashcode,name)
-					@incoming_map[name] = hashcode
-				end
-				
-				def remove_incoming_mapping(name)
-					@incoming_map.delete(name)
-				end
-
-				def add_incoming_docroot(path,name)
-					@docroot_map[name] = path
-				end
-
-				def remove_incoming_docroot(name)
-					@docroot_map.delete(name)
-				end
-
-				def add_incoming_redeployable(limit,name)
-					@redeployable_map[name] = limit
-				end
-
-				def remove_incoming_redeployable(name)
-					@redeployable_map.delete(name)
-				end
-				
-				def add_log(log,name)
-					@log_map[name] = log
-				end
-
-				# Sets the default proxy destination, if requests are received
-				# which do not match a defined destination.
-				
-				def default_name
-					@default_name
-				end
-
-				def default_name=(val)
-					@default_name = val
-				end
-
-				# This timeout is the amount of time a connection will sit in queue
-				# waiting for a backend to process it.  A client connection that
-				# sits for longer than this timeout receives a 503 response and
-				# is dropped.
-
-				def server_unavailable_timeout
-					@server_unavailable_timeout
-				end
-
-				def server_unavailable_timeout=(val)
-					@server_unavailable_timeout = val
-				end
-
-				# The chunked_encoding_threshold is a file size limit.  Files
-				# which fall below this limit are sent in one chunk of data.
-				# Files which hit or exceed this limit are delivered via chunked
-				# encoding.
-				
-				def chunked_encoding_threshold
-					@chunked_enconding_threshold
-				end
-				
-				def chunked_encoding_threshold=(val)
-					@chunked_encoding_threshold = val
-				end
-				
-				# Handle static files.  It employs an extension to efficiently
-				# handle large files, and depends on an addition to
-				# EventMachine, send_file_data(), to efficiently handle small
-				# files.  In my tests, it streams in excess of 120 megabytes of
-				# data per second for large files, and does 8000+ to 9000+
-				# requests per second with small files (i.e. under 4k).  I think
-				# this can still be improved upon for small files.
-				#
-				# Todo for 0.7.0 -- add etag/if-modified/if-modified-since
-				# support.
-				#
-				# TODO: Add support for logging static file delivery if wanted.
-				#   The ideal logging would probably be to Analogger since it'd
-				#   limit the performance impact of the the logging.
-				#
-				
-				def serve_static_file(clnt)
-					path_info = clnt.uri
-					client_name = clnt.name
-					dr = @docroot_map[client_name]
-					if path = find_static_file(dr,path_info,client_name)
-						#ct = ::MIME::Types.type_for(path).first || Caos
-						ct = @typer.simple_type_for(path) || Caos
-						fsize = File.size?(path)
-						if fsize > @chunked_encoding_threshold
-							clnt.send_data "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: #{ct}\r\nTransfer-encoding: chunked\r\n\r\n"
-							EM::Deferrable.future(clnt.stream_file_data(path, :http_chunks=>true)) {clnt.close_connection_after_writing}
-						else
-							clnt.send_data "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: #{ct}\r\nContent-length: #{fsize}\r\n\r\n"
-							clnt.send_file_data path
-							clnt.close_connection_after_writing
-						end
-						true
-					else
-						false
-					end
-					# The exception is going to be eaten here, because some
-					# dumb file IO error shouldn't take Swiftiply down.
-					# TODO: It should log these errors, though.
-				rescue Object
-					clnt.close_connection_after_writing
-					false
-				end
-
-				# Determine if the requested file, in the given docroot, exists
-				# and is a file (i.e. not a directory).
-				#
-				# If Rails style page caching is enabled, this method will be
-				# dynamically replaced by a more sophisticated version.
-				
-				def find_static_file(docroot,path_info,client_name)
-					path = File.join(docroot,path_info)
-					path if FileTest.exist?(path) and FileTest.file?(path) and File.expand_path(path).index(docroot) == 0
-				end
-				
-				# Pushes a front end client (web browser) into the queue of clients
-				# waiting to be serviced if there's no server available to handle
-				# it right now.
-
-				def add_frontend_client(clnt,data_q,data)
-					clnt.create_time = @ctime
-					clnt.data_pos = clnt.data_len = 0 if clnt.redeployable = @redeployable_map[clnt.name]
-
-					unless @docroot_map.has_key?(clnt.name) and serve_static_file(clnt)
-						data_q.unshift data
-						unless match_client_to_server_now(clnt)
-							if clnt.uri =~ /\w+-\w+-\w+\.\w+\.[\w\.]+-(\w+)?$/
-								# NOTE: I hate using unshift and delete on arrays
-								# in this code.  Look at switching to something
-								# with a faster profile for push/pull from both
-								# ends as well as deletes.  There has to be
-								# something.  A linked list solves the push/pull
-								# which might be good enough.
-								@demanding_clients[$1].unshift clnt
-							else
-								@client_q[@incoming_map[clnt.name]].unshift(clnt)
-							end
-						end
-						#clnt.push ## wasted call, yes?
-					end
-				end
-				
-				def rebind_frontend_client(clnt)
-					clnt.create_time = @ctime
-					clnt.data_pos = clnt.data_len = 0
-					
-					unless match_client_to_server_now(clnt)
-						if clnt.uri =~ /\w+-\w+-\w+\.\w+\.[\w\.]+-(\w+)?$/
-							# NOTE: I hate using unshift and delete on arrays
-							# in this code.  Look at switching to something
-							# with a faster profile for push/pull from both
-							# ends as well as deletes.  There has to be
-							# something.
-							@demanding_clients[$1].unshift clnt
-						else
-							@client_q[@incoming_map[clnt.name]].unshift(clnt)
-						end
-					end
-				end
-
-				# Pushes a backend server into the queue of servers waiting for a
-				# client to service if there are no clients waiting to be serviced.
-
-				def add_server srvr
-					@server_q[srvr.name].unshift(srvr) unless match_server_to_client_now(srvr)
-				end
-
-				# Deletes the provided server from the server queue.
-
-				def remove_server srvr
-					@server_q[srvr.name].delete srvr
-				end
-
-				# Removes the named client from the client queue.
-				# TODO: Try replacing this with ...something.  Performance
-				# here has to be bad when the list is long.
-				
-				def remove_client clnt
-					@client_q[clnt.name].delete clnt
-				end
-
-				# Tries to match the client (passed as an argument) to a
-				# server.
-
-				def match_client_to_server_now(client)
-					sq = @server_q[@incoming_map[client.name]]
-					if client.uri =~ /\w+-\w+-\w+\.\w+\.[\w\.]+-(\w+)?$/
-						if sidx = sq.index(@reverse_id_map[$1])
-							server = sq.delete_at(sidx)
-							server.associate = client
-							client.associate = server
-							client.push
-							true
-						else
-							false
-						end
-					elsif server = sq.pop
-						server.associate = client
-						client.associate = server
-						client.push
-						true
-					else
-						false
-					end
-				end
-	
-				# Tries to match the server (passed as an argument) to a
-				# client.
-
-				def match_server_to_client_now(server)
-					if client = @demanding_clients[server.id].pop
-						server.associate = client
-						client.associate = server
-						client.push
-						true
-					elsif client = @client_q[server.name].pop
-						server.associate = client
-						client.associate = server
-						client.push
-						true
-					else
-						false
-					end
-				end
-
-				# Walk through the waiting clients if there is no server
-				# available to process clients and expire any clients that
-				# have been waiting longer than @server_unavailable_timeout
-				# seconds.  Clients which are expired will receive a 503
-				# response.  If this is happening, either you need more
-				# backend processes, or you @server_unavailable_timeout is
-				# too short.
-
-				def expire_clients
-					now = Time.now
-
-					@server_q.each_key do |name|
-						unless @server_q[name].first
-							while c = @client_q[name].pop
-								if (now - c.create_time) >= @server_unavailable_timeout
-									c.send_503_response
-								else
-									@client_q[name].push c
-									break
-								end
-							end
-						end
-					end
-				end
-
-				# This is called by a periodic timer once a second to update
-				# the time.
-
-				def update_ctime
-					@ctime = Time.now
-				end
-
-			end
-		end
-
-		# The ClusterProtocol is the subclass of EventMachine::Connection used
-		# to communicate between Swiftiply and the web browser clients.
-
-		class ClusterProtocol < EventMachine::Connection
-
-			attr_accessor :create_time, :associate, :name, :redeployable, :data_pos, :data_len
-
-			Crn = "\r\n".freeze
-			Crnrn = "\r\n\r\n".freeze
-			C_blank = ''.freeze
-
-			# Initialize the @data array, which is the temporary storage for blocks
-			# of data received from the web browser client, then invoke the superclass
-			# initialization.
-
-			def initialize *args
-				@data = []
-				@data_pos = 0
-				@name = @uri = nil
-				super
-			end
-
-			def receive_data data
-				if @name
-					@data.unshift data
-					push
-				else
-					# Note the \0 below.  intern() blows up when passed a \0.  People who are trying to break a server like to pass \0s.  This should cope with that.
-					if data =~ /^Host:\s*([^\r\0:]*)/
-						# NOTE: Should I be using intern for this?  It might not
-						# be a good idea.
-						@name = $1.intern
-						
-						data =~ /\s([^\s\?]*)/
-						@uri = $1
-						@name = ProxyBag.default_name unless ProxyBag.incoming_mapping(@name)
-						ProxyBag.add_frontend_client(self,@data,data)
-					elsif data =~ /\r\n\r\n/
-						@name = ProxyBag.default_name
-						ProxyBag.add_frontend_client(self,@data,data)
-					end
-				end
-			end
-
-			# Hardcoded 503 response that is sent if a connection is timed out while
-			# waiting for a backend to handle it.
-
-			def send_503_response
-				send_data [
-					"HTTP/1.0 503 Server Unavailable\r\n",
-					"Content-type: text/plain\r\n",
-					"Connection: close\r\n",
-					"\r\n",
-					"Server Unavailable"
-				].join
-				close_connection_after_writing
-			end
-	
-			# Push data from the web browser client to the backend server process.
-
-			def push
-				if @associate
-					unless @redeployable
-						# normal data push
-						data = nil
-						@associate.send_data data while data = @data.pop
-					else
-						# redeployable data push; just send the stuff that has
-						# not already been sent.
-						(@data.length - 1 - @data_pos).downto(0) do |p|
-							d = @data[p]
-							@associate.send_data d
-							@data_len += d.length
-						end
-						@data_pos = @data.length
-
-						# If the request size crosses the size limit, then
-						# disallow redeployent of this request.
-						if @data_len > @redeployable
-							@redeployable = false
-							@data.clear
-						end
-					end
-				end
-			end
-
-			# The connection with the web browser client has been closed, so the
-			# object must be removed from the ProxyBag's queue if it is has not
-			# been associated with a backend.  If it has already been associated
-			# with a backend, then it will not be in the queue and need not be
-			# removed.
-
-			def unbind
-				ProxyBag.remove_client(self) unless @associate
-			end
-
-			def uri
-				@uri
-			end
-
-			def setup_for_redeployment
-				@data_pos = 0
-			end
-
-		end
-
-		# The BackendProtocol is the EventMachine::Connection subclass that
-		# handles the communications between Swiftiply and the backend process
-		# it is proxying to.
-
-		class BackendProtocol < EventMachine::Connection
-			attr_accessor :associate, :id
-
-			C0rnrn = "0\r\n\r\n".freeze
-			Crnrn = "\r\n\r\n".freeze
-
-			def initialize *args
-				@name = self.class.bname
-				super
-			end
-
-			def name
-				@name
-			end
-
-			# Call setup() and add the backend to the ProxyBag queue.
-
-			def post_init
-				setup
-				@initialized = nil
-				ProxyBag.add_server self
-			end
-
-			# Setup the initial variables for receiving headers and content.
-
-			def setup
-				@headers = ''
-				@headers_completed = false
-				#@content_length = nil
-				@content_sent = 0
-			end
-			
-			# Receive data from the backend process.  Headers are parsed from
-			# the rest of the content.  If a Content-Length header is present,
-			# that is used to determine how much data to expect.  Otherwise,
-			# if 'Transfer-encoding: chunked' is present, assume chunked
-			# encoding.  Otherwise be paranoid; something isn't the way we like
-			# it to be.
-
-			def receive_data data
-				unless @initialized
-					preamble = data.slice!(0..24)
-					
-					keylen = preamble[23..24].to_i(16)
-					keylen = 0 if keylen < 0
-					key = keylen > 0 ? data.slice!(0..(keylen - 1)) : C_empty
-					if preamble[0..10] == Cswiftclient and key == ProxyBag.get_key(@name)
-						@id = preamble[11..22]
-						ProxyBag.add_id(self,@id)
-						@initialized = true
-					else
-						close_connection
-						return
-					end
-				end
-				
-				unless @headers_completed 
-					if data =~ /\r\n\r\n/
-						@headers_completed = true
-						h,data = data.split(/\r\n\r\n/,2)
-						@headers << h << Crnrn
-						if @headers =~ /Content-[Ll]ength:\s*([^\r]+)/
-							@content_length = $1.to_i
-						elsif @headers =~ /Transfer-encoding:\s*chunked/
-							@content_length = nil
-						else
-							@content_length = 0
-						end
-						@associate.send_data @headers
-					else
-						@headers << data
-					end
-				end
-
-				if @headers_completed
-					@associate.send_data data
-					@content_sent += data.length
-					if @content_length and @content_sent >= @content_length or data[-6..-1] == C0rnrn
-						@associate.close_connection_after_writing
-						@associate = nil
-						@headers = ''
-						@headers_completed = false
-						#@content_length = nil
-						@content_sent = 0
-						#setup
-						ProxyBag.add_server self
-					end
-				end
-			# TODO: Log these errors!
-			rescue
-				@associate.close_connection_after_writing if @associate
-				@associate = nil
-				setup
-				ProxyBag.add_server self
-			end
-
-			# This is called when the backend disconnects from the proxy.
-			# If the backend is currently associated with a web browser client,
-			# that connection will be closed.  Otherwise, the backend will be
-			# removed from the ProxyBag's backend queue.
-
-			def unbind
-				if @associate
-					if !@associate.redeployable or @content_length
-						@associate.close_connection_after_writing
-					else
-						@associate.associate = nil
-						@associate.setup_for_redeployment
-						ProxyBag.rebind_frontend_client(@associate)
-					end
-				else
-					ProxyBag.remove_server(self)
-				end
-				ProxyBag.remove_id(self)
-			end
-
-			def self.bname=(val)
-				@bname = val
-			end
-
-			def self.bname
-				@bname
-			end
-		end
-
-		# Start the EventMachine event loop and create the front end and backend
-		# handlers, then create the timers that are used to expire unserviced
-		# clients and to update the Proxy's clock.
-
-		def self.run(config)
-			@existing_backends = {}
-			
-			# Default is to assume we want to try to turn epoll support on.  EM
-			# ignores this on platforms that don't support it, so this is safe.
-			EventMachine.epoll unless config.has_key?(Cepoll) and !config[Cepoll]
-			EventMachine.set_descriptor_table_size(4096 || config[Cepoll_descriptors]) if config[Cepoll]
-			EventMachine.run do
-				trap("HUP") {em_config(Swiftcore::SwiftiplyExec.parse_options); GC.start}
-				trap("INT") {EventMachine.stop_event_loop}
-				em_config(config)
-				GC.start
-			end
-		end
-		
-		def self.em_config(config)
-			new_config = {}
-			if RunningConfig[Ccluster_address] != config[Ccluster_address] or RunningConfig[Ccluster_port] != config[Ccluster_port]
-				begin
-			    	new_config[Ccluster_server] = EventMachine.start_server(
-                	config[Ccluster_address],
-                	config[Ccluster_port],
-                	ClusterProtocol)
-				rescue RuntimeError => e
-					advice = ''
-					if config[Ccluster_port] < 1024
-						advice << 'Make sure you have the correct permissions to use that port, and make sure there is nothing else running on that port.'
-					else
-						advice << 'Make sure there is nothing else running on that port.'
-					end
-					advice << "  The original error was:  #{e}\n"
-					raise EMStartServerError.new("The listener on #{config[Ccluster_address]}:#{config[Ccluster_port]} could not be started.\n#{advice}")
-				end
-				new_config[Ccluster_address] = config[Ccluster_address]
-				new_config[Ccluster_port] = config[Ccluster_port]
-		    	RunningConfig[Ccluster_server].stop_server if RunningConfig.has_key?(Ccluster_server)
-			else
-				new_config[Ccluster_server] = RunningConfig[Ccluster_server]
-				new_config[Ccluster_address] = RunningConfig[Ccluster_address]
-				new_config[Ccluster_port] = RunningConfig[Ccluster_port]
-			end
-		    	
-			new_config[Coutgoing] = {}
-
-			config[Cmap].each do |m|
-				if m[Ckeepalive]
-					# keepalive requests are standard Swiftiply requests.
-					
-					# The hash of the "outgoing" config section.  It is used to
-					# uniquely identify a section.
-					hash = Digest::SHA256.hexdigest(m[Cincoming].sort.join('|')).intern
-					
-					# For each incoming entry, do setup.
-					new_config[Cincoming] = {}
-					m[Cincoming].each do |p_|
-						p = p_.intern
-						new_config[Cincoming][p] = {}
-						ProxyBag.add_incoming_mapping(hash,p)
-						
-						if m.has_key?(Cdocroot)
-							ProxyBag.add_incoming_docroot(m[Cdocroot],p)
-						else
-							ProxyBag.remove_incoming_docroot(p)
-						end
-						
-						if m[Credeployable]
-							ProxyBag.add_incoming_redeployable(m[Credeployment_sizelimit] || 16384,p)
-						else
-							ProxyBag.remove_incoming_redeployable(p)
-						end
-						
-						if m.has_key?(Ckey)
-							ProxyBag.set_key(hash,m[Ckey])
-						else
-							ProxyBag.set_key(hash,C_empty)
-						end
-						
-						if m.has_key?(Ccache_extensions) or m.has_key?(Ccache_directory)
-							require 'swiftcore/Swiftiply/support_pagecache'
-							ProxyBag.add_suffix_list((m[Ccache_extensions] || ProxyBag.const_get(:DefaultSuffixes)),p)
-							ProxyBag.add_cache_dir((m[Ccache_directory] || ProxyBag.const_get(:DefaultCacheDir)),p)
-						else
-							ProxyBag.remove_suffix_list(p) if ProxyBag.respond_to?(:remove_suffix_list)
-							ProxyBag.remove_cache_dir(p) if ProxyBag.respond_to?(:remove_cache_dir)
-						end
-							
-						m[Coutgoing].each do |o|
-							ProxyBag.default_name = p if m[Cdefault]
-							if @existing_backends.has_key?(o)
-								new_config[Coutgoing][o] ||= RunningConfig[Coutgoing][o]
-								next
-							else
-								@existing_backends[o] = true
-								backend_class = Class.new(BackendProtocol)
-								backend_class.bname = hash
-								host, port = o.split(/:/,2)
-								begin
-									new_config[Coutgoing][o] = EventMachine.start_server(host, port.to_i, backend_class)
-								rescue RuntimeError => e
-									advice = ''
-									if port.to_i < 1024
-										advice << 'Make sure you have the correct permissions to use that port, and make sure there is nothing else running on that port.'
-									else
-										advice << 'Make sure there is nothing else running on that port.'
-									end
-									advice << "  The original error was:  #{e}\n"
-									raise EMStartServerError.new("The listener on #{host}:#{port} could not be started.\n#{advice}")
-								end
-							end
-						end
-						
-						# Now stop everything that is still running but which isn't needed.
-						if RunningConfig.has_key?(Coutgoing)
-							(RunningConfig[Coutgoing].keys - new_config[Coutgoing].keys).each do |unneeded_server_key|
-								RunningConfig[Coutgoing][unneeded_server_key].stop_server
-							end
-						end
-					end
-				else
-					# This is where the code goes that sets up traditional proxy destinations.
-					# This is a future TODO item.	
-				end
-			end
-
-			#EventMachine.set_effective_user = config[Cuser] if config[Cuser] and RunningConfig[Cuser] != config[Cuser]
-			run_as(config[Cuser],config[Cgroup]) if (config[Cuser] and RunningConfig[Cuser] != config[Cuser]) or (config[Cgroup] and RunningConfig[Cgroup] != config[Cgroup])
-			new_config[Cuser] = config[Cuser]
-			new_config[Cgroup] = config[Cgroup]
-			
-			ProxyBag.server_unavailable_timeout ||= config[Ctimeout]
-			ProxyBag.chunked_encoding_threshold = config[Cchunked_encoding_threshold] || 16384
-			
-			unless RunningConfig[:initialized]
-				EventMachine.add_periodic_timer(2) { ProxyBag.expire_clients }
-				EventMachine.add_periodic_timer(1) { ProxyBag.update_ctime }
-				new_config[:initialized] = true
-			end
-			
-			RunningConfig.replace new_config
-		end
-		
-		
-		# This can be used to change the effective user and group that
-		# Swiftiply is running as.
-		
-		def self.run_as(user = "nobody", group = "nobody")
-			Process.initgroups(user,Etc.getgrnam(group).gid) if user and group
-			::Process::GID.change_privilege(Etc.getgrnam(group).gid) if group
-			::Process::UID.change_privilege(Etc.getpwnam(user).uid) if user
-		rescue Errno::EPERM
-			raise "Failed to change the effective user to #{user} and the group to #{group}"
-		end
-	end
+  # TODO:
+  #
+  # 1) Basic HTTP Authentication
+  # 2) Stats
+  #   Stats will be recorded in aggregate and for each incoming section, and may
+  #   accessed through a separate stats port via a RESTful HTTP request which
+  #   identifies the section to pull stats for, and the authentication key for
+  #   access to those stats.
+  #   http://127.0.0.1:8082
+  #
+  # To track:
+  #   Total connections
+  #   400s served
+  #   404s served
+  #
+  #   Per config section:
+  #     backends connected
+  #     backends busy
+  #     backend disconnects
+  #     backend errors
+  #     static bytes served
+  #     static requests handled
+  #     static requests 304'd
+  #     cache hits for static files
+  #     dynamic bytes returned
+  #     dynamic requests handled
+  #     
+  #   
+  #
+  # 3) Maintenance Page Support
+  #   This is a path to a static file which will be returned on a 503 error.
+  # 4) GZip compression
+  #   Can be toggled on or off.  Configure mime types to compress.  Implemented
+  #   via an extension.
+  # 5) Make one "SwiftiplyCplusplus" and one "SwiftiplC" extension that,
+  #    respectively, encapsulate all of the C++ and C extensions into just
+  #    two.
+
+  # A little statemachine for loading requirements.  The intention is to
+  # only load rubygems if necessary, and to load the Deque and SplayTreeMap
+  # classes if they are available, setting a constant accordingly so that
+  # the fallbacks (Array and Hash) can be used if they are not.
+
+  begin
+    load_state ||= :start
+    rubygems_loaded ||= false
+    require 'socket'
+    require 'digest/sha2'
+    require 'eventmachine'
+    require 'swiftcore/hash'
+    require 'swiftcore/types'
+    require 'swiftcore/Swiftiply/mocklog'
+    require 'swiftcore/Swiftiply/version'
+
+    load_state = :deque
+    require 'swiftcore/deque' unless const_defined?(:HasDeque)
+    HasDeque = true unless const_defined?(:HasDeque)
+
+    load_state = :splaytreemap
+    require 'swiftcore/splaytreemap' unless const_defined?(:HasSplayTree)
+    HasSplayTree = true unless const_defined?(:HasSplayTree)
+
+    load_state = :helpers
+    require 'swiftcore/streamer'
+    require 'swiftcore/Swiftiply/etag_cache'
+    require 'swiftcore/Swiftiply/file_cache'
+    require 'swiftcore/Swiftiply/dynamic_request_cache'
+    require 'time'
+
+    load_state = :core
+    require 'swiftcore/Swiftiply/constants'
+    require 'swiftcore/Swiftiply/proxy_bag'
+    require 'swiftcore/Swiftiply/cluster_protocol'
+    require 'swiftcore/Swiftiply/proxy'
+
+  rescue LoadError => e
+    unless rubygems_loaded
+      # Everything gets slower once rubygems are used (though this
+      # effect is not so profound as it once was).  So, for the
+      # best speed possible, don't install EventMachine or Swiftiply via
+      # gems.
+      begin
+        require 'rubygems'
+        rubygems_loaded = true
+      rescue LoadError
+        raise e
+      end
+      retry
+    end
+    case load_state
+    when :deque
+      HasDeque = false unless const_defined?(:HasDeque)
+      retry
+    when :splaytreemap
+      HasSplayTree = false unless const_defined?(:HasSplayTree)
+      retry
+    end
+    raise e
+  end
+
+  GC.start
+
+  module Swiftiply
+
+    Updaters = {
+            'rest' => ['swiftcore/Swiftiply/config/rest_updater','::Swiftcore::Swiftiply::Config::RestUpdater']
+    }
+
+    def self.existing_backends
+      @existing_backends
+    end
+
+    def self.existing_backends=(val)
+      @existing_backends = val
+    end
+
+    # Start the EventMachine event loop and create the front end and backend
+    # handlers, then create the timers that are used to expire unserviced
+    # clients and to update the Proxy's clock.
+
+    def self.run(config)
+      self.existing_backends = {}
+
+      # Default is to assume we want to try to turn epoll/kqueue support on.
+      EventMachine.epoll unless config.has_key?(Cepoll) and !config[Cepoll] rescue nil
+      EventMachine.kqueue unless config.has_key?(Ckqueue) and !config[Ckqueue] rescue nil
+      EventMachine.set_descriptor_table_size(config[Cepoll_descriptors] || config[Cdescriptors] || 4096) rescue nil
+
+      EventMachine.run do
+        EM.set_timer_quantum(5)
+        trap("HUP") {em_config(Swiftcore::SwiftiplyExec.parse_options); GC.start}
+        trap("INT") {EventMachine.stop_event_loop}
+        GC.start
+        em_config(config)
+        GC.start # We just want to make sure all the junk created during
+                 # configuration is purged prior to real work starting.
+        #RubyProf.start
+      end
+      #result = RubyProf.stop
+
+      #printer = RubyProf::TextPrinter.new(result)
+      #File.open('/tmp/swprof','w+') {|fh| printer = printer.print(fh,0)}
+    end
+
+    def self.log_level
+      @log_level
+    end
+
+    def self.log_level=(val)
+      @log_level = val
+    end
+
+    # TODO: This method is absurdly long, and should be refactored.
+    def self.em_config(config)
+      new_config = {Ccluster_address => [],Ccluster_port => [],Ccluster_server => {}}
+      defaults = config['defaults'] || {}
+
+      new_log = _config_loggers(config,defaults)
+      self.log_level = ProxyBag.log_level
+      ssl_addresses = _config_determine_ssl_addresses(config)
+
+      addresses = (Array === config[Ccluster_address]) ? config[Ccluster_address] : [config[Ccluster_address]]
+      ports = (Array === config[Ccluster_port]) ? config[Ccluster_port] : [config[Ccluster_port]]
+      addrports = []
+
+      addresses.each do |address|
+        ports.each do |port|
+          addrport = "#{address}:#{port}"
+          addrports << addrport
+
+          if (!RunningConfig.has_key?(Ccluster_address)) ||
+            (RunningConfig.has_key?(Ccluster_address) && !RunningConfig[Ccluster_address].include?(address)) ||
+            (RunningConfig.has_key?(Ccluster_port) && !RunningConfig[Ccluster_port].include?(port))
+            begin
+              # If this particular address/port runs SSL, check that the certificate and the
+              # key files exist and are readable, then create a special protocol class
+              # that embeds the certificate and key information.
+
+              if ssl_addresses.has_key?(addrport)
+                # TODO: LOG that the certfiles are missing instead of silently ignoring it.
+                next unless exists_and_is_readable(ssl_addresses[addrport][Ccertfile])
+                next unless exists_and_is_readable(ssl_addresses[addrport][Ckeyfile])
+
+                # Create a customized protocol object for each different address/port combination.
+                ssl_protocol = Class.new(ClusterProtocol)
+                ssl_protocol.class_eval <<EOC
+def post_init
+  start_tls({:cert_chain_file => "#{ssl_addresses[addrport][Ccertfile]}", :private_key_file => "#{ssl_addresses[addrport][Ckeyfile]}"})
+end
+EOC
+                  ProxyBag.logger.log(Cinfo,"Opening SSL server on #{address}:#{port}") if log_level > 0 and log_level < 3
+                  ProxyBag.logger.log(Cinfo,"Opening SSL server on #{address}:#{port} using key at #{ssl_addresses[addrport][Ckeyfile]} and certificate at #{ssl_addresses[addrport][Ccertfile]}")
+                  new_config[Ccluster_server][addrport] = EventMachine.start_server(
+                  address,
+                  port,
+                  ssl_protocol)
+              else
+                standard_protocol = Class.new(ClusterProtocol)
+                standard_protocol.init_class_variables
+                ProxyBag.logger.log(Cinfo,"Opening server on #{address}:#{port}") if ProxyBag.log_level > 0
+                new_config[Ccluster_server][addrport] = EventMachine.start_server(
+                  address,
+                  port,
+                  standard_protocol)
+              end
+            rescue RuntimeError => e
+              advice = ''
+              if port < 1024
+                advice << 'Make sure you have the correct permissions to use that port, and make sure there is nothing else running on that port.'
+              else
+                advice << 'Make sure there is nothing else running on that port.'
+              end
+              advice << "  The original error was:  #{e}\n"
+              msg = "The listener on #{address}:#{port} could not be started.\n#{advice}\n"
+              ProxyBag.logger.log('fatal',msg)
+              raise EMStartServerError.new(msg)
+            end
+
+            new_config[Ccluster_address] << address
+            new_config[Ccluster_port] << port unless new_config[Ccluster_port].include?(port)
+          else
+            new_config[Ccluster_server][addrport] = RunningConfig[Ccluster_server][addrport]
+            new_config[Ccluster_address] << address
+            new_config[Ccluster_port] << port unless new_config[Ccluster_port].include?(port)
+          end
+        end
+      end
+
+      # Stop anything that is no longer in the config.
+      if RunningConfig.has_key?(Ccluster_server)
+        (RunningConfig[Ccluster_server].keys - addrports).each do |s|
+          ProxyBag.logger.log(Cinfo,"Stopping unused incoming server #{s.inspect} out of #{RunningConfig[Ccluster_server].keys.inspect - RunningConfig[Ccluster_server].keys.inspect}")
+          EventMachine.stop_server(s)
+        end
+      end
+
+      new_config[Coutgoing] = {}
+      config[Cmap].each do |mm|
+        m = defaults.dup
+        m.rmerge!(mm)
+        Swiftcore::Swiftiply::Proxy.config(m,new_config)
+      end
+
+      updater = nil
+      if config[Cupdates]
+        uconf = config[Cupdates]
+        require Updaters[uconf[Cupdater]].first
+        updater = Updaters[uconf[Cupdater]].last
+      end
+
+      updater_class = Swiftcore::Swiftiply::class_by_name(updater) if updater
+      self.class.const_set(:Updater, updater_class.new(uconf)) if updater
+
+      #EventMachine.set_effective_user = config[Cuser] if config[Cuser] and RunningConfig[Cuser] != config[Cuser]
+      run_as(config[Cuser],config[Cgroup]) if (config[Cuser] and RunningConfig[Cuser] != config[Cuser]) or (config[Cgroup] and RunningConfig[Cgroup] != config[Cgroup])
+      new_config[Cuser] = config[Cuser]
+      new_config[Cgroup] = config[Cgroup]
+
+      ProxyBag.server_unavailable_timeout ||= config[Ctimeout]
+
+      # By default any file over 16k will be sent via chunked encoding
+      # if the client supports HTTP 1.1.  Generally there is no reason
+      # to change this, but it is configurable.
+
+puts "CHUNKED ENCODING THRESHOLD: #{config[Cchunked_encoding_threshold] || 16384}"
+      ProxyBag.chunked_encoding_threshold = config[Cchunked_encoding_threshold] || 16384
+
+      # The default cache_threshold is set to 100k.  Files above this size
+      # will not be cached.  Customize this value in your configurations
+      # as necessary for the best performance on your site.
+
+      ProxyBag.cache_threshold = config['cache_threshold'] || 102400
+
+      unless RunningConfig[:initialized]
+        EventMachine.add_periodic_timer(0.1) { ProxyBag.recheck_or_expire_clients }
+        #EventMachine.next_tick { ProxyBag.do_and_requeue_recheck_or_expire_clients }
+        EventMachine.add_periodic_timer(1) { ProxyBag.update_ctime }
+        EventMachine.add_periodic_timer(1) { ProxyBag.request_worker_resources }
+        new_config[:initialized] = true
+      end
+
+      Updater.start if updater
+      RunningConfig.replace new_config
+    end
+
+    def self._config_loggers(config,defaults)
+      if defaults['logger']
+        if config['logger']
+          config['logger'].rmerge!(defaults['logger'])
+        else
+          config['logger'] = {}.rmerge!(defaults['logger'])
+        end
+      else
+        config['logger'] = {'log_level' => 0, 'type' => 'stderror'} unless config['logger']
+      end
+
+      new_log = handle_logger_config(config['logger']) if config['logger']
+      ProxyBag.logger = new_log[:logger] if new_log
+      ProxyBag.log_level = new_log[:log_level] if new_log
+      new_log
+    end
+
+    def self._config_determine_ssl_addresses(config)
+      ssl_addresses = {}
+      # Determine which address/port combos should be running SSL.
+      (config[Cssl] || []).each do |sa|
+        if sa.has_key?(Cat)
+          ssl_addresses[sa[Cat]] = {Ccertfile => sa[Ccertfile], Ckeyfile => sa[Ckeyfile]}
+        end
+      end
+      ssl_addresses
+    end
+
+    # This can be used to change the effective user and group that
+    # Swiftiply is running as.
+
+    def self.run_as(user = "nobody", group = "nobody")
+      Process.initgroups(user,Etc.getgrnam(group).gid) if user and group
+      ::Process::GID.change_privilege(Etc.getgrnam(group).gid) if group
+      ::Process::UID.change_privilege(Etc.getpwnam(user).uid) if user
+    rescue Errno::EPERM
+      raise "Failed to change the effective user to #{user} and the group to #{group}"
+    end
+
+    def self.exists_and_is_readable(file)
+      FileTest.exist?(file) and FileTest.readable?(file)
+    end
+
+    # There are 4 levels of logging supported.
+    #   :disabled or 0 means no logging
+    #   :minimal or 1 logs only essential items
+    #   :normal or 2 logs everything useful/interesting
+    #   :full or 3 logs all major events
+    #
+    def self.determine_log_level(lvl)
+      case lvl.to_s
+      when /^d|0/
+        0
+      when /^m|1/
+        1
+      when /^n|2/
+        2
+      when /^f|3/
+        3
+      else
+        1
+      end
+    end
+
+    def self.get_const_from_name(name,space)
+      r = nil
+      space.constants.each do |c|
+        if c =~ /#{name}/i
+          r = c
+          break
+        end
+      end
+      "#{space.name}::#{r}".split('::').inject(Object) { |o,n| o.const_get n }
+    end
+
+    def self.class_by_name(name)
+      klass = Object
+      name.sub(/^::/,'').split('::').each {|n| klass = klass.const_get n}
+      klass
+    end
+
+    def self.handle_logger_config(logger_config = nil,handle_default = true)
+      new_logger = {}
+      if logger_config
+        type = logger_config['type'] || 'Analogger'
+        begin
+          load_attempted ||= false
+          require "swiftcore/Swiftiply/loggers/#{type}"
+        rescue LoadError
+          if load_attempted
+            raise SwiftiplyLoggerNotFound.new("The logger that was specified, #{type}, could not be found.")
+          else
+            load_attempted = true
+            require 'rubygems'
+            retry
+          end
+        end
+        new_logger[:log_level] = determine_log_level(logger_config['level'] || logger_config['log_level'])
+        begin
+          log_class = get_const_from_name(type,::Swiftcore::Swiftiply::Loggers)
+
+          new_logger[:logger] = log_class.new(logger_config)
+          new_logger[:logger].log(Cinfo,"Logger type #{type} started; log level is #{new_logger[:log_level]}.") if new_logger[:log_level] > 0
+        rescue NameError
+          raise SwiftiplyLoggerNameError.new("The logger class specified, Swiftcore::Swiftiply::Loggers::#{type} was not defined.")
+        end
+      elsif handle_default
+        # Default to the stderror logger with a log level of 0
+        begin
+          load_attempted ||= false
+          require "swiftcore/Swiftiply/loggers/stderror"
+        rescue LoadError
+          if load_attempted
+            raise SwiftiplyLoggerNotFound.new("The attempt to load the default logger (swiftcore/Swiftiply/loggers/stderror.rb) failed.  This should not happen.  Please double check your Swiftiply installation.")
+          else
+            load_attempted = true
+            require 'rubygems'
+            retry
+          end
+        end
+
+        log_class = get_const_from_name('stderror',::Swiftcore::Swiftiply::Loggers)
+        new_logger[:logger] = log_class.new(logger_config)
+        new_logger[:log_level] = log_level
+      else
+        new_logger = nil
+      end
+
+      new_logger
+    end
+
+  end
 end