oldmoe-neverblock 0.1.6 → 1.0

data/lib/neverblock/io/db/drivers/postgres.rb

@@ -0,0 +1,63 @@
+require File.expand_path(File.dirname(__FILE__)+'/../lib/neverblock/io/db/drivers/postgres')
+require File.expand_path(File.dirname(__FILE__)+'/../lib/neverblock/io/db/connection')
+
+module NeverBlock
+
+  module DB
+  
+    # A modified postgres connection driver
+    # builds on the original pg driver.
+    # This driver is able to register the socket
+    # at a certain backend (Reacotr)
+    # and then whenever the query is executed
+    # within the scope of a friendly fiber (NB::Fiber)
+    # it will be done in async mode and the fiber
+    # will yield
+	  class Postgres < PGconn	                    
+
+      # Assuming the use of NeverBlock fiber extensions and that the exec is run in
+      # the context of a fiber. One that have the value :neverblock set to true.
+      # All neverblock IO classes check this value, setting it to false will force
+      # the execution in a blocking way.
+      def exec(sql)
+        # TODO Still not "killing the query process"-proof
+        # In some cases, the query is simply sent but the fiber never yields
+        if NB.neverblocking? && NB.reactor.running?
+          send_query sql
+          while is_busy
+            NB.wait(:read, IO.new(socket))
+            consume_input
+          end
+          res, data = 0, []
+          while res != nil
+            res = self.get_result
+            data << res unless res.nil?
+          end
+          data.last
+        else
+          super(sql)
+        end
+      end
+
+      alias_method :query, :exec
+            
+    end #Postgres 
+
+    class PooledPostgres < ::NeverBlock::DB::Connection
+
+      def initialize(*args)
+        options = {}
+        if args && (options = args.last).is_a? Hash
+          size = options[:size] || 4
+          eager = options[:eager] || true
+          args.pop
+        end
+        @pool = NB::DB::Pool.new(:size=>size, :eager=>eager) do
+          Postgres.new(*args)
+        end
+      end
+
+    end #PooledMySQL
+  end #DB
+
+end #NeverBlock

data/lib/neverblock/io/db/fibered_connection_pool.rb

@@ -0,0 +1,130 @@
+module NeverBlock
+  module Pool
+    # This class represents a pool of connections,
+    # you hold or release conncetions from the pool
+    # hold requests that cannot be fullfiled will be queued
+    # the fiber will be paused and resumed later when
+    # a connection is avaialble
+    #
+    # Large portions of this class were copied and pasted
+    # form Sequel's threaded connection pool
+    #
+    # Example:
+    #
+    # pool = NeverBlock::Pool::FiberedConnectionPool.new(:size=>16)do
+    # # connection creation code goes here
+    # end
+    # 32.times do
+    # Fiber.new do
+    # # acquire a connection from the pool
+    # pool.hold do |conn|
+    # conn.execute('something') # you can use the connection normally now
+    # end
+    # end.resume
+    # end
+    #
+    # The pool has support for transactions, just pass true to the
+    # pool#hold method and the connection will not be released after the block
+    # is finished
+    # It is the responsibility of client code to release the connection
+    class FiberedConnectionPool
+ 
+      attr_reader :size
+ 
+      # initialize the connection pool using the supplied proc to create
+      # the connections
+      # You can choose to start them eagerly or lazily (lazy by default)
+      # Available options are
+      # :size => the maximum number of connections to be created in the pool
+      # :eager => (true|false) indicates whether connections should be
+      # created initially or when need
+      def initialize(options = {}, &block)
+        @connections, @busy_connections, @queue = [], {},[]
+        @connection_proc = block
+        @size = options[:size] || 8
+        if options[:eager]
+         @size.times do
+         @connections << @connection_proc.call
+         end
+        end
+      end
+ 
+      def replace_acquired_connection
+        fiber = Fiber.current
+        conn = @connection_proc.call
+        @busy_connections[fiber] = conn
+        fiber[connection_pool_key] = conn
+      end
+ 
+      # If a connection is available, pass it to the block, otherwise pass
+      # the fiber to the queue till a connection is available
+      def hold()
+       fiber = Fiber.current
+        conn = acquire(fiber)
+        yield conn
+      end
+ 
+      def all_connections
+        (@connections + @busy_connections.values).each {|conn| yield(conn)}
+      end
+ 
+      private
+ 
+      # Can we find a connection?
+      # Can we create one?
+      # Wait in the queue then
+      def acquire(fiber)
+        # A special case for rails when doing ActiveRecord stuff when not yet
+        # running in the context of a request (fiber) like in the case of AR
+        # queries in environment.rb (Root Fiber)
+        return @connections.first unless fiber[:callbacks]
+ 
+        fiber[:current_pool_key] = connection_pool_key
+        return fiber[connection_pool_key] if fiber[connection_pool_key]
+        conn = if !@connections.empty?
+          @connections.shift
+        elsif (@connections.length + @busy_connections.length) < @size
+          @connection_proc.call
+        else
+          Fiber.yield @queue << fiber
+        end
+ 
+        # They're called in reverse order i.e. release then process_queue
+        fiber[:callbacks] << self.method(:process_queue)
+        fiber[:callbacks] << self.method(:release)
+ 
+        @busy_connections[fiber] = conn
+        fiber[connection_pool_key] = conn
+      end
+ 
+      # Give the fiber's connection back to the pool
+      def release()
+        fiber = Fiber.current
+        if fiber[connection_pool_key]
+         @busy_connections.delete(fiber)
+         @connections << fiber[connection_pool_key]
+          fiber[connection_pool_key] = nil
+        end
+      end
+ 
+      # Check if there are waiting fibers and
+      # try to process them
+      def process_queue
+        while !@connections.empty? and !@queue.empty?
+          fiber = @queue.shift
+          # What is really happening here?
+          # we are resuming a fiber from within
+          # another, should we call transfer instead?
+          fiber.resume @connections.shift
+        end
+      end
+      
+      def connection_pool_key
+       @connection_pool_key ||= "connection_pool_#{object_id}".intern
+     end
+      
+    end #FiberedConnectionPool
+    
+  end #Pool
+ 
+end #NeverBlock

data/lib/{never_block → neverblock/io}/db/fibered_mysql_connection.rb

@@ -7,10 +7,8 @@ module NeverBlock
     # This driver is able to register the socket at a certain backend (EM)
     # and then whenever the query is executed within the scope of a friendly
     # fiber. It will be done in async mode and the fiber will yield
-	  class FiberedMysqlConnection < Mysql	      
+	  class FiberedMysqlConnection < Mysql
                 
-      include FiberedDBConnection
-
       # Initializes the connection and remembers the connection params
       def initialize(*args)
         @connection_params = args
@@ -31,21 +29,10 @@ module NeverBlock
       # All neverblock IO classes check this value, setting it to false will force
       # the execution in a blocking way.
       def query(sql)
-        if NB.event_loop_available? && NB.neverblocking?
-          raise ::NB::NBError.new("FiberedMysqlConnection: The running fiber is attached to a connection other than the current one") if (c = Fiber.current[Fiber.current[:current_pool_key]]) && c != self
-          begin
-            send_query sql
-            Fiber.yield register_with_event_loop
-            get_result
-          rescue Exception => e
-            if error = ['not connected', 'gone away', 'Lost connection'].detect{|msg| e.message.include? msg}
-              event_loop_connection_close
-              unregister_from_event_loop
-              remove_unregister_from_event_loop_callbacks
-              #connect
-            end
-            raise e
-          end
+        if NB.neverblocking? && NB.reactor.running?
+          send_query sql
+          NB.wait(:read, IO.new(socket))
+          get_result
         else
           super(sql)
         end

data/lib/{never_block/pool/fibered_connection_pool.rb → neverblock/io/db/pool.rb}

@@ -1,5 +1,7 @@
+require File.expand_path(File.dirname(__FILE__)+'/../../../neverblock')
+
 module NeverBlock
-  module Pool
+  module DB
 		# This class represents a pool of connections, 
 		# you hold or release conncetions from the pool
 		# hold requests that cannot be fullfiled will be queued
@@ -11,7 +13,7 @@ module NeverBlock
     #
     # Example:
     #
-    # pool = NeverBlock::Pool::FiberedConnectionPool.new(:size=>16)do
+    # pool = NeverBlock::Pool.new(:size=>16)do
     #   # connection creation code goes here   
     # end
     # 32.times do
@@ -23,11 +25,9 @@ module NeverBlock
     #   end.resume
     # end
     # 
-    # The pool has support for transactions, just pass true to the
-    # pool#hold method and the connection will not be released after the block
-    # is finished
     # It is the responsibility of client code to release the connection
-		class FiberedConnectionPool
+    # using pool.release(conn)
+		class Pool
 
       attr_reader :size
 
@@ -50,7 +50,7 @@ module NeverBlock
 			end
 
       def replace_acquired_connection
-        fiber = Fiber.current
+        fiber = NB::Fiber.current
         conn = @connection_proc.call
         @busy_connections[fiber] = conn
         fiber[connection_pool_key] = conn
@@ -58,10 +58,22 @@ module NeverBlock
 
       # If a connection is available, pass it to the block, otherwise pass
       # the fiber to the queue till a connection is available
-			def hold()
-			  fiber = Fiber.current
+			def hold
+			  fiber = NB::Fiber.current
         conn = acquire(fiber)
-        yield conn
+        if block_given?
+          yield conn
+          release(conn)
+        else
+          conn
+        end
+			end
+
+      # Give the fiber's connection back to the pool
+			def release(conn)
+			  @busy_connections.delete(conn.object_id)
+			  @connections << conn unless @connections.include? conn
+        process_queue
 			end
 
       def all_connections
@@ -74,37 +86,14 @@ module NeverBlock
       # Can we create one?
       # Wait in the queue then
 			def acquire(fiber)
-        # A special case for rails when doing ActiveRecord stuff when not yet
-        # running in the context of a request (fiber) like in the case of AR
-        # queries in environment.rb (Root Fiber)
-        return @connections.first unless fiber[:callbacks]
-
-        fiber[:current_pool_key] = connection_pool_key
-        return fiber[connection_pool_key] if fiber[connection_pool_key]
         conn =  if !@connections.empty?
           @connections.shift
         elsif (@connections.length + @busy_connections.length) < @size
           @connection_proc.call
         else
-					Fiber.yield @queue << fiber
-        end
-
-        # They're called in reverse order i.e. release then process_queue
-        fiber[:callbacks] << self.method(:process_queue)
-        fiber[:callbacks] << self.method(:release)
-
-        @busy_connections[fiber] = conn
-        fiber[connection_pool_key] = conn
-			end
-
-      # Give the fiber's connection back to the pool
-			def release()
-        fiber = Fiber.current
-        if fiber[connection_pool_key]
-				  @busy_connections.delete(fiber)
-				  @connections << fiber[connection_pool_key]
-          fiber[connection_pool_key] = nil
+					NB::Fiber.yield @queue << fiber
         end
+        @busy_connections[conn.object_id] = conn
 			end
 
       # Check if there are waiting fibers and
@@ -118,11 +107,7 @@ module NeverBlock
 					fiber.resume @connections.shift
 				end
 			end
-			
-			def connection_pool_key
-			  @connection_pool_key ||= "connection_pool_#{object_id}".intern
-		  end
-			
+						
 		end #FiberedConnectionPool
 		
 	end #Pool

data/lib/neverblock/io/file.rb

@@ -0,0 +1,24 @@
+# Author::    Mohammad A. Ali  (mailto:oldmoe@gmail.com)
+# Copyright:: Copyright (c) 2009 eSpace, Inc.
+# License::   Distributes under the same terms as Ruby
+
+require 'thread'
+require File.expand_path(File.dirname(__FILE__)+'/io')
+
+class File < IO
+  
+  def self.neverblock(*methods)
+    methods.each do |method|  
+      class_eval %{
+        def #{method}(*args)
+          return rb_#{method}(*args) unless NB.neverblocking?
+          NB.defer(self, :#{method}, args)
+        end
+      }
+    end
+  end
+  
+  neverblock :syswrite, :sysread, :write, :read, :readline, 
+             :readlines, :readchar, :gets, :getc, :print
+               
+end

data/lib/neverblock/io/io.rb

@@ -0,0 +1,219 @@
+require 'fcntl'
+
+# This is an extention to the Ruby IO class that makes it compatable with
+#	NeverBlocks event loop to avoid blocking IO calls. That's done by delegating
+# Author::    Mohammad A. Ali  (mailto:oldmoe@gmail.com)
+# Copyright:: Copyright (c) 2009 eSpace, Inc.
+# License::   Distributes under the same terms as Ruby
+
+$:.unshift 
+
+require File.expand_path(File.dirname(__FILE__)+'/../../neverblock')
+
+class IO
+
+	NB_BUFFER_LENGTH = 128*1024
+	
+  alias_method :rb_sysread,   :sysread
+  alias_method :rb_syswrite,  :syswrite
+  alias_method :rb_read,      :read
+  alias_method :rb_write,     :write
+  alias_method :rb_gets,      :gets
+  alias_method :rb_getc,      :getc
+  alias_method :rb_readchar,  :readchar
+  alias_method :rb_readline,  :readline
+  alias_method :rb_readlines, :readlines
+  alias_method :rb_print,     :print
+  
+	#	This method is the delegation method which reads using read_nonblock()
+	#	and registers the IO call with event loop if the call blocks. The value
+	# @immediate_result is used to get the value that method got before it was blocked.
+
+  def read_neverblock(*args)
+		res = ""
+		begin
+      old_flags = get_flags			
+			res << read_nonblock(*args)
+      set_flags(old_flags)
+		rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::EINTR
+      set_flags(old_flags)
+  		NB.wait(:read, self)
+  		retry
+		end
+		res
+  end
+
+	#	The is the main reading method that all other methods use.
+	#	If the mode is set to neverblock it uses the delegation method.
+	#	Otherwise it uses the original ruby read method.
+
+  def sysread(length)
+    self.neverblock? ? read_neverblock(length) : rb_sysread(length)
+  end
+  
+  def read(length=nil, sbuffer=nil)
+    return rb_read(length, sbuffer) if self.file?
+    return '' if length == 0
+    if sbuffer.nil?
+      sbuffer = '' 
+    else
+     sbuffer = sbuffer.to_str
+     sbuffer.delete!(sbuffer)
+    end    
+    if length.nil?
+      # we need to read till end of stream
+      loop do
+        begin 
+          sbuffer << sysread(NB_BUFFER_LENGTH)
+        rescue EOFError
+      	  break
+        end
+      end
+      return sbuffer 
+    else # length != nil
+      if self.buffer.length >= length
+        sbuffer << self.buffer.slice!(0, length)
+        return sbuffer
+      elsif self.buffer.length > 0
+        sbuffer << self.buffer
+      end
+      self.buffer = ''
+      remaining_length = length - sbuffer.length
+	    while sbuffer.length < length && remaining_length > 0 
+		    begin 
+			    sbuffer << sysread(NB_BUFFER_LENGTH < remaining_length ? remaining_length : NB_BUFFER_LENGTH)
+          remaining_length = remaining_length - sbuffer.length   
+		    rescue EOFError
+          break
+		    end #begin
+	    end	#while	  
+    end #if length 
+    return nil if sbuffer.length.zero? && length > 0
+    return sbuffer if sbuffer.length <= length
+		self.buffer << sbuffer.slice!(length, sbuffer.length-1)
+    return sbuffer
+  end
+
+ def readpartial(length=nil,sbuffer=nil)
+   raise ArgumentError if !length.nil? && length < 0
+    if sbuffer.nil?
+      sbuffer = '' 
+    else
+     sbuffer = sbuffer.to_str
+     sbuffer.delete!(sbuffer)
+    end
+   
+    if self.buffer.length >= length
+       sbuffer << self.buffer.slice!(0, length)
+    elsif self.buffer.length > 0 
+      sbuffer << self.buffer.slice!(0, self.buffer.length-1)
+    else
+      sbuffer << rb_sysread(length)
+    end  
+     return sbuffer  
+  end
+
+  def write_neverblock(data)
+		written = 0
+		begin
+      old_flags = get_flags			
+			written = written + write_nonblock(data[written,data.length])
+      set_flags(old_flags)
+			raise Errno::EAGAIN if written < data.length
+		rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::EINTR
+      set_flags(old_flags)
+  		NB.wait(:write, self)
+			retry
+	  end
+		written
+  end
+
+	def syswrite(*args)	
+		return rb_syswrite(*args) unless self.neverblock?
+			write_neverblock(*args)
+	end
+  
+	def write(data)
+    return 0 if data.to_s.empty?
+    return rb_write(data) if self.file?
+		syswrite(data)
+	end 
+	
+	def gets(sep=$/)
+    return rb_gets(sep) if self.file?
+		res = ""
+		sep = "\n\n" if sep == ""
+		sep = $/ if sep.nil?
+		while res.index(sep).nil?
+		  break if (c = read(1)).nil?
+		  res << c
+		end
+		$_ = res
+		res
+	end
+	
+	def readlines(sep=$/)
+    return rb_readlines(sep) if self.file?
+		res = []
+		begin
+			loop{res << readline(sep)}
+		rescue EOFError
+		end
+		res
+	end
+	
+	def readchar
+    return rb_readchar if self.file?
+		ch = read(1)
+		raise EOFError if ch.nil?
+		ch
+	end
+	
+	def getc
+    return rb_getc if self.file?
+		begin
+			res = readchar
+		rescue EOFError
+			res = nil
+		end
+	end
+
+	def readline(sep = $/)
+    return rb_readline(sep) if self.file?
+		res = gets(sep)
+		raise EOFError if res == nil
+		res
+	end
+	
+	def print(*args)
+    return rb_print if self.file?
+		args.each{|element|syswrite(element)}
+	end
+
+	protected
+
+  def get_flags
+    self.fcntl(Fcntl::F_GETFL, 0)
+  end
+
+  def set_flags(flags)
+    self.fcntl(Fcntl::F_SETFL, flags)
+  end
+
+	def buffer
+	 @buffer ||= ""
+	end		
+
+	def buffer=(value)
+	 @buffer = value
+	end		
+	
+	def file?
+	  @file ||= self.stat.file?
+	end
+
+  def neverblock?
+    !file? && NB.neverblocking?
+	end	
+
+end