neverblock 0.1.6.2

data/lib/never_block/servers/mongrel.rb

@@ -0,0 +1,236 @@
+# This module rewrites pieces of the very good Mongrel web server in
+# order to change it from a threaded application to an event based
+# application running inside an EventMachine event loop.  It should
+# be compatible with the existing Mongrel handlers for Rails,
+# Camping, Nitro, etc....
+
+# NeverBlock support added
+
+begin
+	load_attempted ||= false
+	require 'eventmachine'
+rescue LoadError
+	unless load_attempted
+		load_attempted = true
+		require 'rubygems'
+		retry
+	end
+end
+
+require 'rubygems'
+require 'neverblock'
+require 'mongrel'
+
+module Mongrel
+	class MongrelProtocol < EventMachine::Connection
+		def post_init
+			@parser = HttpParser.new
+			@params = HttpParams.new
+			@nparsed = 0
+			@request = nil
+			@request_len = nil
+			@linebuffer = ''
+		end
+
+		def receive_data data
+			@linebuffer << data
+			@nparsed = @parser.execute(@params, @linebuffer, @nparsed) unless @parser.finished?
+			if @parser.finished?
+				if @request_len.nil?
+					@request_len = @params[::Mongrel::Const::CONTENT_LENGTH].to_i
+					script_name, path_info, handlers = ::Mongrel::HttpServer::Instance.classifier.resolve(@params[::Mongrel::Const::REQUEST_PATH])
+					if handlers
+						@params[::Mongrel::Const::PATH_INFO] = path_info
+						@params[::Mongrel::Const::SCRIPT_NAME] = script_name
+						@params[::Mongrel::Const::REMOTE_ADDR] = @params[::Mongrel::Const::HTTP_X_FORWARDED_FOR] || ::Socket.unpack_sockaddr_in(get_peername)[1]
+						@notifiers = handlers.select { |h| h.request_notify }
+					end
+					if @request_len > ::Mongrel::Const::MAX_BODY
+						new_buffer = Tempfile.new(::Mongrel::Const::MONGREL_TMP_BASE)
+						new_buffer.binmode
+						new_buffer << @linebuffer[@nparsed..-1]
+						@linebuffer = new_buffer
+					else
+						@linebuffer = StringIO.new(@linebuffer[@nparsed..-1])
+						@linebuffer.pos = @linebuffer.length
+					end
+				end
+				if @linebuffer.length >= @request_len
+					@linebuffer.rewind
+					::Mongrel::HttpServer::Instance.process_http_request(@params,@linebuffer,self)
+					@linebuffer.delete if Tempfile === @linebuffer
+				end
+			elsif @linebuffer.length > ::Mongrel::Const::MAX_HEADER
+				close_connection
+				raise ::Mongrel::HttpParserError.new("HEADER is longer than allowed, aborting client early.")
+			end
+		rescue ::Mongrel::HttpParserError
+			if $mongrel_debug_client
+				STDERR.puts "#{Time.now}: BAD CLIENT (#{params[Const::HTTP_X_FORWARDED_FOR] || client.peeraddr.last}): #$!"
+				STDERR.puts "#{Time.now}: REQUEST DATA: #{data.inspect}\n---\nPARAMS: #{params.inspect}\n---\n"
+			end
+			close_connection
+		rescue Exception => e
+			close_connection
+			raise e
+		end
+
+		def write data
+			send_data data
+		end
+
+		def closed?
+			false
+		end
+
+	end
+
+	class HttpServer
+    DEFAULT_FIBER_POOL_SIZE = 20
+		def initialize(host, port, num_processors=950, x=0, y=nil) # Deal with Mongrel 1.0.1 or earlier, as well as later.
+			@socket = nil
+			@classifier = URIClassifier.new
+			@host = host
+			@port = port
+			@workers = ThreadGroup.new
+			if y
+				@throttle = x
+				@timeout = y || 60
+			else
+				@timeout = x
+			end
+			@num_processors = num_processors #num_processors is pointless for evented....
+			@death_time = 60
+			self.class.const_set(:Instance,self)
+		end
+
+    def fiber_pool
+      @fiber_pool ||= NB::Pool::FiberPool.new(DEFAULT_FIBER_POOL_SIZE)
+    end
+
+		def run
+			trap('INT') { raise StopServer }
+			trap('TERM') { raise StopServer }
+			@acceptor = Thread.new do
+				EventMachine.epoll
+				EventMachine.set_descriptor_table_size(4096)
+				EventMachine.run do
+					begin
+						EventMachine.start_server(@host,@port,MongrelProtocol)
+					rescue StopServer
+						EventMachine.stop_event_loop
+					end
+				end
+			end
+		end
+
+		def process_http_request(params,linebuffer,client)
+			if not params[Const::REQUEST_PATH]
+				uri = URI.parse(params[Const::REQUEST_URI])
+				params[Const::REQUEST_PATH] = uri.request_uri
+			end
+
+      raise "No REQUEST PATH" if not params[Const::REQUEST_PATH]
+
+			script_name, path_info, handlers = @classifier.resolve(params[Const::REQUEST_PATH])
+
+			if handlers
+				notifiers = handlers.select { |h| h.request_notify }
+				request = HttpRequest.new(params, linebuffer, notifiers)
+
+				# request is good so far, continue processing the response
+				response = HttpResponse.new(client)
+
+				# Process each handler in registered order until we run out or one finalizes the response.
+				fiber_pool.spawn do
+				  dispatch_to_handlers(handlers,request,response)
+        end
+				# And finally, if nobody closed the response off, we finalize it.
+				unless response.done
+					response.finished
+				else
+					response.close_connection_after_writing
+				end
+			else
+				# Didn't find it, return a stock 404 response.
+				client.send_data(Const::ERROR_404_RESPONSE)
+				client.close_connection_after_writing
+			end
+		end
+
+		def dispatch_to_handlers(handlers,request,response)
+			handlers.each do |handler|
+				handler.process(request, response)
+				break if response.done
+			end
+		end
+	end
+
+	class HttpRequest
+		def initialize(params, linebuffer, dispatchers)
+			@params = params
+			@dispatchers = dispatchers
+			@body = linebuffer
+		end
+	end
+
+	class HttpResponse
+		def send_file(path, small_file = false)
+			File.open(path, "rb") do |f|
+				while chunk = f.read(Const::CHUNK_SIZE) and chunk.length > 0
+					begin
+						write(chunk)
+					rescue Object => exc
+						break
+					end
+				end
+			end
+			@body_sent = true
+		end
+
+		def write(data)
+			@socket.send_data data
+		end
+
+		def close_connection_after_writing
+			@socket.close_connection_after_writing
+		end
+
+		def socket_error(details)
+			@socket.close_connection
+			done = true
+			raise details
+		end
+
+		def finished
+			send_status
+			send_header
+			send_body
+			@socket.close_connection_after_writing
+		end
+	end
+
+	class Configurator
+		# This version fixes a bug in the regular Mongrel version by adding
+		# initialization of groups.
+		def change_privilege(user, group)
+			if user and group
+				log "Initializing groups for {#user}:{#group}."
+				Process.initgroups(user,Etc.getgrnam(group).gid)
+			end
+
+			if group
+				log "Changing group to #{group}."
+				Process::GID.change_privilege(Etc.getgrnam(group).gid)
+			end
+
+			if user
+				log "Changing user to #{user}."
+				Process::UID.change_privilege(Etc.getpwnam(user).uid)
+			end
+		rescue Errno::EPERM
+			log "FAILED to change user:group #{user}:#{group}: #$!"
+			exit 1
+		end
+	end
+end

data/lib/never_block/servers/thin.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'neverblock' unless defined?(NeverBlock)
+require 'thin'
+
+module Thin
+  
+  # Patch the thin server to use NeverBlock::Pool::FiberPool to be able to
+  # wrap requests in fibers
+  class Server
+
+    DEFAULT_FIBER_POOL_SIZE = 20
+
+    def fiber_pool
+      @fiber_pool ||= NB::Pool::FiberPool.new(DEFAULT_FIBER_POOL_SIZE)
+    end  
+
+  end # Server
+
+  # A request is processed by wrapping it in a fiber from the fiber pool.
+  # If all the fibers are busy the request will wait in a queue to be picked up
+  # later. Meanwhile, the server will still be processing requests
+  class Connection < EventMachine::Connection
+
+    def process
+        @request.threaded = false
+        @backend.server.fiber_pool.spawn {post_process(pre_process)}
+    end
+    
+  end # Connection
+
+
+end # Thin

data/lib/neverblock-mysql.rb

@@ -0,0 +1,5 @@
+$:.unshift File.expand_path(File.dirname(__FILE__))
+require 'neverblock'
+require 'never_block/db/fibered_db_connection'
+require 'never_block/db/fibered_mysql_connection'
+require 'never_block/db/pooled_db_connection'

data/lib/neverblock-pg.rb

@@ -0,0 +1,5 @@
+$:.unshift File.expand_path(File.dirname(__FILE__))
+require 'neverblock'
+require 'never_block/db/fibered_db_connection'
+require 'never_block/db/pooled_db_connection'
+require 'never_block/db/fibered_postgres_connection'

data/lib/neverblock.rb

@@ -0,0 +1,8 @@
+# Author::    Mohammad A. Ali  (mailto:oldmoe@gmail.com)
+# Copyright:: Copyright (c) 2008 eSpace, Inc.
+# License::   Distributes under the same terms as Ruby
+
+$:.unshift File.expand_path(File.dirname(__FILE__))
+
+require 'never_block'
+

data/neverblock.gemspec

@@ -0,0 +1,73 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{neverblock}
+  s.version = "0.1.6.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Muhammad A. Ali", "Ahmed Sobhi", "Osama Brekaa", "Nicholas Silva"]
+  s.date = %q{2010-02-03}
+  s.description = %q{NeverBlock is a collection of classes and modules that help you write evented non-blocking applications in a seemingly blocking mannner.}
+  s.email = %q{conickal@gmail.com}
+  s.extra_rdoc_files = [
+    "README"
+  ]
+  s.files = [
+    "README",
+     "Rakefile",
+     "VERSION",
+     "lib/active_record/connection_adapters/neverblock_mysql_adapter.rb",
+     "lib/active_record/connection_adapters/neverblock_postgresql_adapter.rb",
+     "lib/never_block.rb",
+     "lib/never_block/db/fibered_db_connection.rb",
+     "lib/never_block/db/fibered_mysql_connection.rb",
+     "lib/never_block/db/fibered_postgres_connection.rb",
+     "lib/never_block/db/pooled_db_connection.rb",
+     "lib/never_block/extensions/fiber_extensions.rb",
+     "lib/never_block/frameworks/activerecord.rb",
+     "lib/never_block/frameworks/rails.rb",
+     "lib/never_block/pool/fiber_pool.rb",
+     "lib/never_block/pool/fibered_connection_pool.rb",
+     "lib/never_block/servers/mongrel.rb",
+     "lib/never_block/servers/thin.rb",
+     "lib/neverblock-mysql.rb",
+     "lib/neverblock-pg.rb",
+     "lib/neverblock.rb",
+     "neverblock.gemspec",
+     "spec/fiber_extensions_spec.rb",
+     "spec/fiber_pool_spec.rb",
+     "spec/fibered_connection_pool_spec.rb",
+     "tasks/spec.rake",
+     "test/test_mysql.rb",
+     "test/test_pg.rb"
+  ]
+  s.homepage = %q{http://github.com/conickal/neverblock}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Utilities for non-blocking stack components}
+  s.test_files = [
+    "spec/fiber_extensions_spec.rb",
+     "spec/fiber_pool_spec.rb",
+     "spec/fibered_connection_pool_spec.rb",
+     "test/test_mysql.rb",
+     "test/test_pg.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.2"])
+    else
+      s.add_dependency(%q<eventmachine>, [">= 0.12.2"])
+    end
+  else
+    s.add_dependency(%q<eventmachine>, [">= 0.12.2"])
+  end
+end
+

data/spec/fiber_extensions_spec.rb

@@ -0,0 +1,24 @@
+$:.unshift File.expand_path('..')
+require 'lib/neverblock'
+
+describe Fiber do
+  before(:all) do
+    @fiber = Fiber.new {puts "I'm a new fiber"}
+  end
+
+  it "should be able to set fiber local variable" do
+    @fiber[:x] = "wow"
+  end
+
+  it "should be able to retrieve an already set fiber local variable" do
+    @fiber[:x].should == "wow"
+  end
+
+  it "should return nil when trying to retrieve an unset fiber local variable" do
+    @fiber[:y].should == nil
+  end
+
+  after(:all) do
+    @fiber = nil
+  end
+end

data/spec/fiber_pool_spec.rb

@@ -0,0 +1,60 @@
+$:.unshift File.expand_path('..')
+require 'lib/neverblock'
+
+describe NeverBlock::Pool::FiberPool do
+  before(:each) do
+    @fiber_pool = NeverBlock::Pool::FiberPool.new(10)
+  end
+
+  it "should have all fibers ready and an empty queue initially" do
+    @fiber_pool.fibers.length.should == 10
+    @fiber_pool.instance_variable_get(:@queue).length.should == 0
+  end
+
+  it "should have fibers with :neverblock fiber variable set to true" do
+    @fiber_pool.fibers.each {|f| f[:neverblock].should == true}
+  end
+
+  it "should process a new block if there are available fibers" do
+    x = false
+    @fiber_pool.spawn {x  = true}
+    x.should == true
+  end
+
+  it "should queue requests if requests are more than fibers" do
+    progress = Array.new(15, false)
+    fibers = []
+    @fiber_pool.fibers.each {|f| fibers << f}
+    10.times do |i|
+      @fiber_pool.spawn {Fiber.yield; progress[i] = true}
+    end
+    @fiber_pool.fibers.length.should == 0
+    @fiber_pool.instance_variable_get(:@queue).length.should == 0
+
+    #it should now queue
+     (10..14).each {|i| @fiber_pool.spawn {progress[i] = true}}
+    @fiber_pool.fibers.length.should == 0
+    @fiber_pool.instance_variable_get(:@queue).length.should == 5
+
+    #resume the first fiber, this should also process the queued requests
+    fibers[0].resume
+    @fiber_pool.fibers.length.should == 1
+    @fiber_pool.instance_variable_get(:@queue).length.should == 0
+    [0,*10..14].each {|i| progress[i].should == true}
+     (1..9).to_a.each {|i| progress[i].should == false}
+    fibers_count = 1
+     (1..9).to_a.each do |i|
+      fibers[i].resume
+      fibers_count = fibers_count + 1
+      progress[i].should == true
+      @fiber_pool.fibers.length.should == fibers_count
+    end
+    @fiber_pool.instance_variable_get(:@queue).length.should == 0
+  end
+
+  after(:each) do
+    @fiber_pool = nil
+  end
+
+end
+