michaelyta-neverblock 1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +21 -0
- data/lib/never_block.rb +29 -0
- data/lib/neverblock.rb +10 -0
- data/lib/neverblock/core/fiber.rb +45 -0
- data/lib/neverblock/core/pool.rb +72 -0
- data/lib/neverblock/core/reactor.rb +52 -0
- data/lib/neverblock/core/system/system.rb +38 -0
- data/lib/neverblock/core/system/timeout.rb +60 -0
- data/lib/neverblock/io/db/connection.rb +57 -0
- data/lib/neverblock/io/db/drivers/mysql.rb +69 -0
- data/lib/neverblock/io/db/drivers/postgres.rb +63 -0
- data/lib/neverblock/io/db/pool.rb +115 -0
- data/lib/neverblock/io/file.rb +24 -0
- data/lib/neverblock/io/io.rb +219 -0
- data/lib/neverblock/io/socket.rb +75 -0
- data/lib/neverblock_io.rb +6 -0
- data/lib/system.rb +4 -0
- data/neverblock.gemspec +37 -0
- metadata +82 -0
data/README
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
== NeverBlock
|
2
|
+
Never, ever!
|
3
|
+
|
4
|
+
NeverBlock is a collection of classes and modules that help you write evented non-blocking applications in a seemingly blocking mannner.
|
5
|
+
|
6
|
+
NeverBlock currently provides the following Libraries:
|
7
|
+
|
8
|
+
=== FiberExtensions
|
9
|
+
A set of extenstions to the standard Fiber implementation
|
10
|
+
|
11
|
+
=== NeverBlock::Pool::FiberPool
|
12
|
+
A pool of fibers that can be used to provide an upper limit to the numbers of active fibers in an application
|
13
|
+
|
14
|
+
=== NeverBlock::Pool::FiberedConnectionPool
|
15
|
+
A generic fibered connection pool for all sorts of connections with support for transactions. This was mostly copied from Sequel::ConnectionPool
|
16
|
+
|
17
|
+
NeverBlock should be the basis for providing completely async Ruby application development that does not require the usual twisted style of evented programming. For example, you will be able to develop in Rails in the usual style and deploy to a NeverBlock server which will do all the IO in an evented manner without you even noticing this.
|
18
|
+
|
19
|
+
=== License
|
20
|
+
Ruby License, http://www.ruby-lang.org/en/LICENSE.txt.
|
21
|
+
|
data/lib/never_block.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
2
|
+
# Copyright:: Copyright (c) 2009 eSpace, Inc.
|
3
|
+
# License:: Distributes under the same terms as Ruby
|
4
|
+
|
5
|
+
$:.unshift File.expand_path(File.dirname(__FILE__))
|
6
|
+
|
7
|
+
module NeverBlock
|
8
|
+
|
9
|
+
# Checks if we should be working in a non-blocking mode
|
10
|
+
def self.neverblocking?
|
11
|
+
NB::Fiber.respond_to?(:current) && NB::Fiber.current.respond_to?('[]') && NB::Fiber.current[:neverblock] && NB.reactor.running?
|
12
|
+
end
|
13
|
+
|
14
|
+
# The given block will run its queries either in blocking or non-blocking
|
15
|
+
# mode based on the first parameter
|
16
|
+
def self.neverblock(nb = true, &block)
|
17
|
+
status = NB::Fiber.current[:neverblock]
|
18
|
+
NB::Fiber.current[:neverblock] = nb
|
19
|
+
block.call
|
20
|
+
NB::Fiber.current[:neverblock] = status
|
21
|
+
end
|
22
|
+
|
23
|
+
# Exception to be thrown for all neverblock internal errors
|
24
|
+
class NBError < StandardError
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
NB = NeverBlock
|
data/lib/neverblock.rb
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
2
|
+
# Copyright:: Copyright (c) 2008 eSpace, Inc.
|
3
|
+
# License:: Distributes under the same terms as Ruby
|
4
|
+
|
5
|
+
$:.unshift File.expand_path(File.dirname(__FILE__))
|
6
|
+
|
7
|
+
require 'never_block'
|
8
|
+
require 'neverblock/core/reactor'
|
9
|
+
require 'neverblock/core/fiber'
|
10
|
+
require 'neverblock/core/pool'
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
2
|
+
# Copyright:: Copyright (c) 2009 eSpace, Inc.
|
3
|
+
# License:: Distributes under the same terms as Ruby
|
4
|
+
|
5
|
+
require 'fiber'
|
6
|
+
require File.expand_path(File.dirname(__FILE__)+'/../../never_block')
|
7
|
+
|
8
|
+
class NeverBlock::Fiber < Fiber
|
9
|
+
|
10
|
+
def initialize(neverblock = true, &block)
|
11
|
+
self[:neverblock] = neverblock
|
12
|
+
super()
|
13
|
+
end
|
14
|
+
|
15
|
+
|
16
|
+
#Attribute Reference--Returns the value of a fiber-local variable, using
|
17
|
+
#either a symbol or a string name. If the specified variable does not exist,
|
18
|
+
#returns nil.
|
19
|
+
def [](key)
|
20
|
+
local_fiber_variables[key]
|
21
|
+
end
|
22
|
+
|
23
|
+
#Attribute Assignment--Sets or creates the value of a fiber-local variable,
|
24
|
+
#using either a symbol or a string. See also Fiber#[].
|
25
|
+
def []=(key,value)
|
26
|
+
local_fiber_variables[key] = value
|
27
|
+
end
|
28
|
+
|
29
|
+
#Sending an exception instance to resume will yield the fiber
|
30
|
+
#and then raise the exception. This is necessary to raise exceptions
|
31
|
+
#in their correct context.
|
32
|
+
def self.yield(*args)
|
33
|
+
result = super
|
34
|
+
raise result if result.is_a? Exception
|
35
|
+
result
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def local_fiber_variables
|
41
|
+
@local_fiber_variables ||= {}
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
@@ -0,0 +1,72 @@
|
|
1
|
+
|
2
|
+
require File.expand_path(File.dirname(__FILE__)+'/fiber')
|
3
|
+
|
4
|
+
module NeverBlock
|
5
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
6
|
+
# Copyright:: Copyright (c) 2008 eSpace, Inc.
|
7
|
+
# License:: Distributes under the same terms as Ruby
|
8
|
+
#
|
9
|
+
# A pool of initialized fibers
|
10
|
+
# It does not grow in size or create transient fibers
|
11
|
+
# It will queue code blocks when needed (if all its fibers are busy)
|
12
|
+
#
|
13
|
+
# This class is particulary useful when you use the fibers
|
14
|
+
# to connect to evented back ends. It also does not generate
|
15
|
+
# transient objects and thus saves memory.
|
16
|
+
#
|
17
|
+
# Example:
|
18
|
+
# fiber_pool = NeverBlock::FiberPool.new(150)
|
19
|
+
#
|
20
|
+
# loop do
|
21
|
+
# fiber_pool.spawn do
|
22
|
+
# #fiber body goes here
|
23
|
+
# end
|
24
|
+
# end
|
25
|
+
#
|
26
|
+
class FiberPool
|
27
|
+
|
28
|
+
# gives access to the currently free fibers
|
29
|
+
attr_reader :fibers
|
30
|
+
|
31
|
+
# Prepare a list of fibers that are able to run different blocks of code
|
32
|
+
# every time. Once a fiber is done with its block, it attempts to fetch
|
33
|
+
# another one from the queue
|
34
|
+
def initialize(count = 50)
|
35
|
+
@fibers,@busy_fibers,@queue = [],{},[]
|
36
|
+
count.times do |i|
|
37
|
+
fiber = NB::Fiber.new do |block|
|
38
|
+
loop do
|
39
|
+
block.call
|
40
|
+
# callbacks are called in a reverse order, much like c++ destructor
|
41
|
+
NB::Fiber.current[:callbacks].pop.call while NB::Fiber.current[:callbacks].length > 0
|
42
|
+
unless @queue.empty?
|
43
|
+
block = @queue.shift
|
44
|
+
else
|
45
|
+
@busy_fibers.delete(NB::Fiber.current.object_id)
|
46
|
+
@fibers << NB::Fiber.current
|
47
|
+
block = NB::Fiber.yield
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
fiber[:callbacks] = []
|
52
|
+
@fibers << fiber
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# If there is an available fiber use it, otherwise, leave it to linger
|
57
|
+
# in a queue
|
58
|
+
def spawn(evented = true, &block)
|
59
|
+
if fiber = @fibers.shift
|
60
|
+
fiber[:callbacks] = []
|
61
|
+
@busy_fibers[fiber.object_id] = fiber
|
62
|
+
fiber[:neverblock] = evented
|
63
|
+
fiber.resume(block)
|
64
|
+
else
|
65
|
+
@queue << block
|
66
|
+
end
|
67
|
+
self # we are keen on hiding our queue
|
68
|
+
end
|
69
|
+
|
70
|
+
end # FiberPool
|
71
|
+
end # NeverBlock
|
72
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
$:.unshift
|
2
|
+
|
3
|
+
require 'reactor'
|
4
|
+
require 'thread'
|
5
|
+
require File.expand_path(File.dirname(__FILE__)+'/fiber')
|
6
|
+
|
7
|
+
module NeverBlock
|
8
|
+
|
9
|
+
@@reactors = {}
|
10
|
+
|
11
|
+
@@queue = Queue.new
|
12
|
+
|
13
|
+
@@thread_pool = []
|
14
|
+
|
15
|
+
20.times do
|
16
|
+
@@thread_pool << Thread.new do
|
17
|
+
loop do
|
18
|
+
io, method, params, fiber, reactor = *(@@queue.shift)
|
19
|
+
begin
|
20
|
+
reactor.next_tick{fiber.resume(io.__send__(method, *params))} if fiber.alive?
|
21
|
+
rescue Exception => e
|
22
|
+
reactor.next_tick{fiber.resume(e)} if fiber.alive?
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.reactor
|
29
|
+
@@reactors[Thread.current.object_id] ||= ::Reactor::Base.new
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.wait(mode, io)
|
33
|
+
fiber = NB::Fiber.current
|
34
|
+
NB.reactor.attach(mode, io){fiber.resume}
|
35
|
+
NB::Fiber.yield
|
36
|
+
NB.reactor.detach(mode, io)
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.sleep(time)
|
40
|
+
NB::Fiber.yield if time.nil?
|
41
|
+
return if time <= 0
|
42
|
+
fiber = NB::Fiber.current
|
43
|
+
NB.reactor.add_timer(time){fiber.resume}
|
44
|
+
NB::Fiber.yield
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.defer(io, action, args)
|
48
|
+
@@queue << [io, action, args, NB::Fiber.current, NB.reactor]
|
49
|
+
NB::Fiber.yield
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
|
2
|
+
require File.expand_path(File.dirname(__FILE__)+'/../../../neverblock')
|
3
|
+
|
4
|
+
module Kernel
|
5
|
+
|
6
|
+
alias_method :rb_sleep, :sleep
|
7
|
+
|
8
|
+
def sleep(time=nil)
|
9
|
+
return rb_sleep(time) unless NB.neverblocking?
|
10
|
+
NB.sleep(time)
|
11
|
+
end
|
12
|
+
|
13
|
+
alias_method :rb_system, :system
|
14
|
+
|
15
|
+
def system(cmd, *args)
|
16
|
+
return rb_system(cmd, *args) unless NB.neverblocking?
|
17
|
+
begin
|
18
|
+
backticks(cmd, *args)
|
19
|
+
result = $?.exitstatus
|
20
|
+
return true if result.zero?
|
21
|
+
return nil if result == 127
|
22
|
+
return false
|
23
|
+
rescue Errno::ENOENT => e
|
24
|
+
return nil
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def backticks(cmd, *args)
|
29
|
+
myargs = "#{cmd} "
|
30
|
+
myargs << args.join(' ') if args
|
31
|
+
res = ''
|
32
|
+
IO.popen(myargs) do |f|
|
33
|
+
res << f.read
|
34
|
+
end
|
35
|
+
res
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require File.expand_path(File.dirname(__FILE__)+'/../../../neverblock')
|
3
|
+
|
4
|
+
module Timeout
|
5
|
+
|
6
|
+
alias_method :rb_timeout, :timeout
|
7
|
+
|
8
|
+
def timeout(time, klass=Timeout::Error)
|
9
|
+
return rb_timeout(time, klass) unless NB.neverblocking?
|
10
|
+
if time.nil? || time <= 0
|
11
|
+
yield
|
12
|
+
else
|
13
|
+
fiber = NB::Fiber.current
|
14
|
+
timer = NB.reactor.add_timer(time) do
|
15
|
+
fiber[:timeouts].last.each do |event|
|
16
|
+
if event.is_a? Reactor::Timer
|
17
|
+
event.cancel
|
18
|
+
else
|
19
|
+
NB.reactor.detach(event[0], event[1])
|
20
|
+
end
|
21
|
+
end
|
22
|
+
fiber.resume(klass.new)
|
23
|
+
end
|
24
|
+
(fiber[:timeouts] ||= []) << []
|
25
|
+
begin
|
26
|
+
yield
|
27
|
+
rescue Exception => e
|
28
|
+
raise e
|
29
|
+
ensure
|
30
|
+
timer.cancel
|
31
|
+
fiber[:timeouts].pop
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
module_function :timeout
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
NB.reactor.on_add_timer do |timer|
|
41
|
+
timeouts = NB::Fiber.current[:timeouts]
|
42
|
+
unless timeouts.nil? || timeouts.empty?
|
43
|
+
timeouts.last << timer
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
NB.reactor.on_attach do |mode, io|
|
48
|
+
timeouts = NB::Fiber.current[:timeouts]
|
49
|
+
unless timeouts.nil? || timeouts.empty?
|
50
|
+
timeouts.last << [mode, io]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
NB.reactor.on_detach do |mode, io|
|
55
|
+
timeouts = NB::Fiber.current[:timeouts]
|
56
|
+
unless timeouts.nil? || timeouts.empty?
|
57
|
+
timeouts.delete_if{|to|to.is_a? Array && to[0] == mode && to[1] == io}
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__)+'/pool')
|
2
|
+
|
3
|
+
module NeverBlock
|
4
|
+
|
5
|
+
module DB
|
6
|
+
# a proxy for pooled fibered connections
|
7
|
+
class Connection
|
8
|
+
# Requires a block with connection parameters
|
9
|
+
# and a pool size (defaults to 4)
|
10
|
+
def initialize(size=4, &block)
|
11
|
+
@pool = NB::DB::Pool.new(:size=>size, :eager=>true) do
|
12
|
+
yield
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# A proxy for the connection's query method
|
17
|
+
# quries the pool to get a connection first
|
18
|
+
def query(query)
|
19
|
+
@pool.hold do |conn|
|
20
|
+
conn.query(query)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
alias :exec :query
|
25
|
+
|
26
|
+
# Replaces the current connection with a brand new one
|
27
|
+
def replace_acquired_connection
|
28
|
+
@pool.replace_acquired_connection
|
29
|
+
end
|
30
|
+
|
31
|
+
# Pass unknown methods to the connection
|
32
|
+
def method_missing(method, *args)
|
33
|
+
@pool.hold do |conn|
|
34
|
+
conn.send(method, *args)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# Pass method queries to the connection
|
39
|
+
def respond_to?(method)
|
40
|
+
@pool.hold do |conn|
|
41
|
+
conn.respond_to?(method)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
protected
|
46
|
+
|
47
|
+
# are we in a transaction?
|
48
|
+
# if no then just hold a connection and run the block
|
49
|
+
# else get a connection, pass it to the block
|
50
|
+
# and move away
|
51
|
+
def hold_connection
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
$:.unshift File.expand_path(File.dirname(__FILE__))
|
2
|
+
|
3
|
+
require 'mysqlplus'
|
4
|
+
require 'neverblock/db/connection'
|
5
|
+
|
6
|
+
module NeverBlock
|
7
|
+
|
8
|
+
module DB
|
9
|
+
# A modified mysql connection driver. It builds on the original pg driver.
|
10
|
+
# This driver is able to register the socket at a certain backend (EM)
|
11
|
+
# and then whenever the query is executed within the scope of a friendly
|
12
|
+
# fiber. It will be done in async mode and the fiber will yield
|
13
|
+
class Mysql < ::Mysql
|
14
|
+
|
15
|
+
# Initializes the connection and remembers the connection params
|
16
|
+
def initialize(*args)
|
17
|
+
@connection_params = args
|
18
|
+
super(*@connection_params)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Does a normal real_connect if arguments are passed. If no arguments are
|
22
|
+
# passed it uses the ones it remembers
|
23
|
+
def real_connect(*args)
|
24
|
+
@connection_params = args unless args.empty?
|
25
|
+
super(*@connection_params)
|
26
|
+
end
|
27
|
+
|
28
|
+
alias_method :connect, :real_connect
|
29
|
+
|
30
|
+
# Assuming the use of NeverBlock fiber extensions and that the exec is run in
|
31
|
+
# the context of a fiber. One that have the value :neverblock set to true.
|
32
|
+
# All neverblock IO classes check this value, setting it to false will force
|
33
|
+
# the execution in a blocking way.
|
34
|
+
def query(sql)
|
35
|
+
if NB.neverblocking? && NB.reactor.running?
|
36
|
+
send_query sql
|
37
|
+
NB.wait(:read, IO.new(socket))
|
38
|
+
get_result
|
39
|
+
else
|
40
|
+
super(sql)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
alias_method :exec, :query
|
45
|
+
|
46
|
+
end #MySQL
|
47
|
+
|
48
|
+
class PooledMySQL < ::NeverBlock::DB::Connection
|
49
|
+
|
50
|
+
def initialize(*args)
|
51
|
+
options = {}
|
52
|
+
if args && (options = args.last).is_a? Hash
|
53
|
+
size = options[:size] || 4
|
54
|
+
eager = options[:eager] || true
|
55
|
+
args.pop
|
56
|
+
end
|
57
|
+
@pool = NB::DB::Pool.new(:size=>size, :eager=>eager) do
|
58
|
+
MySQL.new(*args)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end #PooledMySQL
|
63
|
+
|
64
|
+
end #DB
|
65
|
+
|
66
|
+
end #NeverBlock
|
67
|
+
|
68
|
+
|
69
|
+
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__)+'/pg')
|
2
|
+
require File.expand_path(File.dirname(__FILE__)+'/../connection')
|
3
|
+
|
4
|
+
module NeverBlock
|
5
|
+
|
6
|
+
module DB
|
7
|
+
|
8
|
+
# A modified postgres connection driver
|
9
|
+
# builds on the original pg driver.
|
10
|
+
# This driver is able to register the socket
|
11
|
+
# at a certain backend (Reacotr)
|
12
|
+
# and then whenever the query is executed
|
13
|
+
# within the scope of a friendly fiber (NB::Fiber)
|
14
|
+
# it will be done in async mode and the fiber
|
15
|
+
# will yield
|
16
|
+
class Postgres < PGconn
|
17
|
+
|
18
|
+
# Assuming the use of NeverBlock fiber extensions and that the exec is run in
|
19
|
+
# the context of a fiber. One that have the value :neverblock set to true.
|
20
|
+
# All neverblock IO classes check this value, setting it to false will force
|
21
|
+
# the execution in a blocking way.
|
22
|
+
def exec(sql)
|
23
|
+
# TODO Still not "killing the query process"-proof
|
24
|
+
# In some cases, the query is simply sent but the fiber never yields
|
25
|
+
if NB.neverblocking? && NB.reactor.running?
|
26
|
+
send_query sql
|
27
|
+
while is_busy
|
28
|
+
NB.wait(:read, IO.new(socket))
|
29
|
+
consume_input
|
30
|
+
end
|
31
|
+
res, data = 0, []
|
32
|
+
while res != nil
|
33
|
+
res = self.get_result
|
34
|
+
data << res unless res.nil?
|
35
|
+
end
|
36
|
+
data.last
|
37
|
+
else
|
38
|
+
super(sql)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
alias_method :query, :exec
|
43
|
+
|
44
|
+
end #Postgres
|
45
|
+
|
46
|
+
class PooledPostgres < ::NeverBlock::DB::Connection
|
47
|
+
|
48
|
+
def initialize(*args)
|
49
|
+
options = {}
|
50
|
+
if args && (options = args.last).is_a? Hash
|
51
|
+
size = options[:size] || 4
|
52
|
+
eager = options[:eager] || true
|
53
|
+
args.pop
|
54
|
+
end
|
55
|
+
@pool = NB::DB::Pool.new(:size=>size, :eager=>eager) do
|
56
|
+
Postgres.new(*args)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end #PooledMySQL
|
61
|
+
end #DB
|
62
|
+
|
63
|
+
end #NeverBlock
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__)+'/../../../neverblock')
|
2
|
+
|
3
|
+
module NeverBlock
|
4
|
+
module DB
|
5
|
+
# This class represents a pool of connections,
|
6
|
+
# you hold or release conncetions from the pool
|
7
|
+
# hold requests that cannot be fullfiled will be queued
|
8
|
+
# the fiber will be paused and resumed later when
|
9
|
+
# a connection is avaialble
|
10
|
+
#
|
11
|
+
# Large portions of this class were copied and pasted
|
12
|
+
# form Sequel's threaded connection pool
|
13
|
+
#
|
14
|
+
# Example:
|
15
|
+
#
|
16
|
+
# pool = NeverBlock::Pool.new(:size=>16)do
|
17
|
+
# # connection creation code goes here
|
18
|
+
# end
|
19
|
+
# 32.times do
|
20
|
+
# Fiber.new do
|
21
|
+
# # acquire a connection from the pool
|
22
|
+
# pool.hold do |conn|
|
23
|
+
# conn.execute('something') # you can use the connection normally now
|
24
|
+
# end
|
25
|
+
# end.resume
|
26
|
+
# end
|
27
|
+
#
|
28
|
+
# It is the responsibility of client code to release the connection
|
29
|
+
# using pool.release(conn)
|
30
|
+
class Pool
|
31
|
+
|
32
|
+
attr_reader :size
|
33
|
+
|
34
|
+
# initialize the connection pool using the supplied proc to create
|
35
|
+
# the connections
|
36
|
+
# You can choose to start them eagerly or lazily (lazy by default)
|
37
|
+
# Available options are
|
38
|
+
# :size => the maximum number of connections to be created in the pool
|
39
|
+
# :eager => (true|false) indicates whether connections should be
|
40
|
+
# created initially or when need
|
41
|
+
def initialize(options = {}, &block)
|
42
|
+
@connections, @busy_connections, @queue = [], {},[]
|
43
|
+
@connection_proc = block
|
44
|
+
@size = options[:size] || 8
|
45
|
+
if options[:eager]
|
46
|
+
@size.times do
|
47
|
+
@connections << @connection_proc.call
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def replace_acquired_connection
|
53
|
+
fiber = NB::Fiber.current
|
54
|
+
conn = @connection_proc.call
|
55
|
+
@busy_connections[fiber] = conn
|
56
|
+
fiber[connection_pool_key] = conn
|
57
|
+
end
|
58
|
+
|
59
|
+
# If a connection is available, pass it to the block, otherwise pass
|
60
|
+
# the fiber to the queue till a connection is available
|
61
|
+
def hold
|
62
|
+
fiber = NB::Fiber.current
|
63
|
+
conn = acquire(fiber)
|
64
|
+
if block_given?
|
65
|
+
yield conn
|
66
|
+
release(conn)
|
67
|
+
else
|
68
|
+
conn
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# Give the fiber's connection back to the pool
|
73
|
+
def release(conn)
|
74
|
+
@busy_connections.delete(conn.object_id)
|
75
|
+
@connections << conn unless @connections.include? conn
|
76
|
+
process_queue
|
77
|
+
end
|
78
|
+
|
79
|
+
def all_connections
|
80
|
+
(@connections + @busy_connections.values).each {|conn| yield(conn)}
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
# Can we find a connection?
|
86
|
+
# Can we create one?
|
87
|
+
# Wait in the queue then
|
88
|
+
def acquire(fiber)
|
89
|
+
conn = if !@connections.empty?
|
90
|
+
@connections.shift
|
91
|
+
elsif (@connections.length + @busy_connections.length) < @size
|
92
|
+
@connection_proc.call
|
93
|
+
else
|
94
|
+
NB::Fiber.yield @queue << fiber
|
95
|
+
end
|
96
|
+
@busy_connections[conn.object_id] = conn
|
97
|
+
end
|
98
|
+
|
99
|
+
# Check if there are waiting fibers and
|
100
|
+
# try to process them
|
101
|
+
def process_queue
|
102
|
+
while !@connections.empty? and !@queue.empty?
|
103
|
+
fiber = @queue.shift
|
104
|
+
# What is really happening here?
|
105
|
+
# we are resuming a fiber from within
|
106
|
+
# another, should we call transfer instead?
|
107
|
+
fiber.resume @connections.shift
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
end #FiberedConnectionPool
|
112
|
+
|
113
|
+
end #Pool
|
114
|
+
|
115
|
+
end #NeverBlock
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
2
|
+
# Copyright:: Copyright (c) 2009 eSpace, Inc.
|
3
|
+
# License:: Distributes under the same terms as Ruby
|
4
|
+
|
5
|
+
require 'thread'
|
6
|
+
require File.expand_path(File.dirname(__FILE__)+'/io')
|
7
|
+
|
8
|
+
class File < IO
|
9
|
+
|
10
|
+
def self.neverblock(*methods)
|
11
|
+
methods.each do |method|
|
12
|
+
class_eval %{
|
13
|
+
def #{method}(*args)
|
14
|
+
return rb_#{method}(*args) unless NB.neverblocking?
|
15
|
+
NB.defer(self, :#{method}, args)
|
16
|
+
end
|
17
|
+
}
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
neverblock :syswrite, :sysread, :write, :read, :readline,
|
22
|
+
:readlines, :readchar, :gets, :getc, :print
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,219 @@
|
|
1
|
+
require 'fcntl'
|
2
|
+
|
3
|
+
# This is an extention to the Ruby IO class that makes it compatable with
|
4
|
+
# NeverBlocks event loop to avoid blocking IO calls. That's done by delegating
|
5
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
6
|
+
# Copyright:: Copyright (c) 2009 eSpace, Inc.
|
7
|
+
# License:: Distributes under the same terms as Ruby
|
8
|
+
|
9
|
+
$:.unshift
|
10
|
+
|
11
|
+
require File.expand_path(File.dirname(__FILE__)+'/../../neverblock')
|
12
|
+
|
13
|
+
class IO
|
14
|
+
|
15
|
+
NB_BUFFER_LENGTH = 128*1024
|
16
|
+
|
17
|
+
alias_method :rb_sysread, :sysread
|
18
|
+
alias_method :rb_syswrite, :syswrite
|
19
|
+
alias_method :rb_read, :read
|
20
|
+
alias_method :rb_write, :write
|
21
|
+
alias_method :rb_gets, :gets
|
22
|
+
alias_method :rb_getc, :getc
|
23
|
+
alias_method :rb_readchar, :readchar
|
24
|
+
alias_method :rb_readline, :readline
|
25
|
+
alias_method :rb_readlines, :readlines
|
26
|
+
alias_method :rb_print, :print
|
27
|
+
|
28
|
+
# This method is the delegation method which reads using read_nonblock()
|
29
|
+
# and registers the IO call with event loop if the call blocks. The value
|
30
|
+
# @immediate_result is used to get the value that method got before it was blocked.
|
31
|
+
|
32
|
+
def read_neverblock(*args)
|
33
|
+
res = ""
|
34
|
+
begin
|
35
|
+
old_flags = get_flags
|
36
|
+
res << read_nonblock(*args)
|
37
|
+
set_flags(old_flags)
|
38
|
+
rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::EINTR
|
39
|
+
set_flags(old_flags)
|
40
|
+
NB.wait(:read, self)
|
41
|
+
retry
|
42
|
+
end
|
43
|
+
res
|
44
|
+
end
|
45
|
+
|
46
|
+
# The is the main reading method that all other methods use.
|
47
|
+
# If the mode is set to neverblock it uses the delegation method.
|
48
|
+
# Otherwise it uses the original ruby read method.
|
49
|
+
|
50
|
+
def sysread(length)
|
51
|
+
self.neverblock? ? read_neverblock(length) : rb_sysread(length)
|
52
|
+
end
|
53
|
+
|
54
|
+
def read(length=nil, sbuffer=nil)
|
55
|
+
return rb_read(length, sbuffer) if self.file?
|
56
|
+
return '' if length == 0
|
57
|
+
if sbuffer.nil?
|
58
|
+
sbuffer = ''
|
59
|
+
else
|
60
|
+
sbuffer = sbuffer.to_str
|
61
|
+
sbuffer.delete!(sbuffer)
|
62
|
+
end
|
63
|
+
if length.nil?
|
64
|
+
# we need to read till end of stream
|
65
|
+
loop do
|
66
|
+
begin
|
67
|
+
sbuffer << sysread(NB_BUFFER_LENGTH)
|
68
|
+
rescue EOFError
|
69
|
+
break
|
70
|
+
end
|
71
|
+
end
|
72
|
+
return sbuffer
|
73
|
+
else # length != nil
|
74
|
+
if self.buffer.length >= length
|
75
|
+
sbuffer << self.buffer.slice!(0, length)
|
76
|
+
return sbuffer
|
77
|
+
elsif self.buffer.length > 0
|
78
|
+
sbuffer << self.buffer
|
79
|
+
end
|
80
|
+
self.buffer = ''
|
81
|
+
remaining_length = length - sbuffer.length
|
82
|
+
while sbuffer.length < length && remaining_length > 0
|
83
|
+
begin
|
84
|
+
sbuffer << sysread(NB_BUFFER_LENGTH < remaining_length ? remaining_length : NB_BUFFER_LENGTH)
|
85
|
+
remaining_length = remaining_length - sbuffer.length
|
86
|
+
rescue EOFError
|
87
|
+
break
|
88
|
+
end #begin
|
89
|
+
end #while
|
90
|
+
end #if length
|
91
|
+
return nil if sbuffer.length.zero? && length > 0
|
92
|
+
return sbuffer if sbuffer.length <= length
|
93
|
+
self.buffer << sbuffer.slice!(length, sbuffer.length-1)
|
94
|
+
return sbuffer
|
95
|
+
end
|
96
|
+
|
97
|
+
def readpartial(length=nil,sbuffer=nil)
|
98
|
+
raise ArgumentError if !length.nil? && length < 0
|
99
|
+
if sbuffer.nil?
|
100
|
+
sbuffer = ''
|
101
|
+
else
|
102
|
+
sbuffer = sbuffer.to_str
|
103
|
+
sbuffer.delete!(sbuffer)
|
104
|
+
end
|
105
|
+
|
106
|
+
if self.buffer.length >= length
|
107
|
+
sbuffer << self.buffer.slice!(0, length)
|
108
|
+
elsif self.buffer.length > 0
|
109
|
+
sbuffer << self.buffer.slice!(0, self.buffer.length-1)
|
110
|
+
else
|
111
|
+
sbuffer << rb_sysread(length)
|
112
|
+
end
|
113
|
+
return sbuffer
|
114
|
+
end
|
115
|
+
|
116
|
+
def write_neverblock(data)
|
117
|
+
written = 0
|
118
|
+
begin
|
119
|
+
old_flags = get_flags
|
120
|
+
written = written + write_nonblock(data[written,data.length])
|
121
|
+
set_flags(old_flags)
|
122
|
+
raise Errno::EAGAIN if written < data.length
|
123
|
+
rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::EINTR
|
124
|
+
set_flags(old_flags)
|
125
|
+
NB.wait(:write, self)
|
126
|
+
retry
|
127
|
+
end
|
128
|
+
written
|
129
|
+
end
|
130
|
+
|
131
|
+
def syswrite(*args)
|
132
|
+
return rb_syswrite(*args) unless self.neverblock?
|
133
|
+
write_neverblock(*args)
|
134
|
+
end
|
135
|
+
|
136
|
+
def write(data)
|
137
|
+
return 0 if data.to_s.empty?
|
138
|
+
return rb_write(data) if self.file?
|
139
|
+
syswrite(data)
|
140
|
+
end
|
141
|
+
|
142
|
+
def gets(sep=$/)
|
143
|
+
return rb_gets(sep) if self.file?
|
144
|
+
res = ""
|
145
|
+
sep = "\n\n" if sep == ""
|
146
|
+
sep = $/ if sep.nil?
|
147
|
+
while res.index(sep).nil?
|
148
|
+
break if (c = read(1)).nil?
|
149
|
+
res << c
|
150
|
+
end
|
151
|
+
$_ = res
|
152
|
+
res
|
153
|
+
end
|
154
|
+
|
155
|
+
def readlines(sep=$/)
|
156
|
+
return rb_readlines(sep) if self.file?
|
157
|
+
res = []
|
158
|
+
begin
|
159
|
+
loop{res << readline(sep)}
|
160
|
+
rescue EOFError
|
161
|
+
end
|
162
|
+
res
|
163
|
+
end
|
164
|
+
|
165
|
+
def readchar
|
166
|
+
return rb_readchar if self.file?
|
167
|
+
ch = read(1)
|
168
|
+
raise EOFError if ch.nil?
|
169
|
+
ch
|
170
|
+
end
|
171
|
+
|
172
|
+
def getc
|
173
|
+
return rb_getc if self.file?
|
174
|
+
begin
|
175
|
+
res = readchar
|
176
|
+
rescue EOFError
|
177
|
+
res = nil
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
def readline(sep = $/)
|
182
|
+
return rb_readline(sep) if self.file?
|
183
|
+
res = gets(sep)
|
184
|
+
raise EOFError if res == nil
|
185
|
+
res
|
186
|
+
end
|
187
|
+
|
188
|
+
def print(*args)
|
189
|
+
return rb_print if self.file?
|
190
|
+
args.each{|element|syswrite(element)}
|
191
|
+
end
|
192
|
+
|
193
|
+
protected
|
194
|
+
|
195
|
+
def get_flags
|
196
|
+
self.fcntl(Fcntl::F_GETFL, 0)
|
197
|
+
end
|
198
|
+
|
199
|
+
def set_flags(flags)
|
200
|
+
self.fcntl(Fcntl::F_SETFL, flags)
|
201
|
+
end
|
202
|
+
|
203
|
+
def buffer
|
204
|
+
@buffer ||= ""
|
205
|
+
end
|
206
|
+
|
207
|
+
def buffer=(value)
|
208
|
+
@buffer = value
|
209
|
+
end
|
210
|
+
|
211
|
+
def file?
|
212
|
+
@file ||= self.stat.file?
|
213
|
+
end
|
214
|
+
|
215
|
+
def neverblock?
|
216
|
+
!file? && NB.neverblocking?
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# Author:: Mohammad A. Ali (mailto:oldmoe@gmail.com)
|
2
|
+
# Copyright:: Copyright (c) 2009 eSpace, Inc.
|
3
|
+
# License:: Distributes under the same terms as Ruby
|
4
|
+
|
5
|
+
require 'socket'
|
6
|
+
require 'fcntl'
|
7
|
+
require File.expand_path(File.dirname(__FILE__)+'/io')
|
8
|
+
|
9
|
+
class BasicSocket < IO
|
10
|
+
|
11
|
+
@@getaddress_method = IPSocket.method(:getaddress)
|
12
|
+
def self.getaddress(*args)
|
13
|
+
@@getaddress_method.call(*args)
|
14
|
+
end
|
15
|
+
|
16
|
+
alias_method :recv_blocking, :recv
|
17
|
+
|
18
|
+
def recv_neverblock(*args)
|
19
|
+
res = ""
|
20
|
+
begin
|
21
|
+
old_flags = self.fcntl(Fcntl::F_GETFL, 0)
|
22
|
+
res << recv_nonblock(*args)
|
23
|
+
self.fcntl(Fcntl::F_SETFL, old_flags)
|
24
|
+
rescue Errno::EWOULDBLOCK, Errno::EAGAIN, Errno::EINTR
|
25
|
+
self.fcntl(Fcntl::F_SETFL, old_flags)
|
26
|
+
NB.wait(:read, self)
|
27
|
+
retry
|
28
|
+
end
|
29
|
+
res
|
30
|
+
end
|
31
|
+
|
32
|
+
def recv(*args)
|
33
|
+
if NB.neverblocking?
|
34
|
+
recv_neverblock(*args)
|
35
|
+
else
|
36
|
+
recv_blocking(*args)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
class Socket < BasicSocket
|
43
|
+
|
44
|
+
alias_method :connect_blocking, :connect
|
45
|
+
|
46
|
+
def connect_neverblock(server_sockaddr)
|
47
|
+
begin
|
48
|
+
connect_nonblock(server_sockaddr)
|
49
|
+
rescue Errno::EINPROGRESS, Errno::EINTR, Errno::EALREADY, Errno::EWOULDBLOCK
|
50
|
+
NB.wait(:write, self)
|
51
|
+
retry
|
52
|
+
rescue Errno::EISCONN
|
53
|
+
# do nothing, we are good
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def connect(server_sockaddr)
|
58
|
+
if NB.neverblocking?
|
59
|
+
connect_neverblock(server_sockaddr)
|
60
|
+
else
|
61
|
+
connect_blocking(server_sockaddr)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
Object.send(:remove_const, :TCPSocket)
|
68
|
+
|
69
|
+
class TCPSocket < Socket
|
70
|
+
def initialize(*args)
|
71
|
+
super(AF_INET, SOCK_STREAM, 0)
|
72
|
+
self.connect(Socket.sockaddr_in(*(args.reverse)))
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
data/lib/system.rb
ADDED
data/neverblock.gemspec
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = "neverblock"
|
3
|
+
s.version = "1.0"
|
4
|
+
s.date = "2009-07-16"
|
5
|
+
s.summary = "Utilities for non-blocking stack components"
|
6
|
+
s.email = "oldmoe@gmail.com"
|
7
|
+
s.homepage = "http://github.com/oldmoe/neverblock"
|
8
|
+
s.description = "NeverBlock is a collection of classes and modules that help you write evented non-blocking applications in a seemingly blocking mannner."
|
9
|
+
s.has_rdoc = true
|
10
|
+
s.authors = ["Muhammad A. Ali", "Ahmed Sobhi", "Osama Brekaa"]
|
11
|
+
s.files = [
|
12
|
+
"neverblock.gemspec",
|
13
|
+
"README",
|
14
|
+
"lib/neverblock/core/reactor.rb",
|
15
|
+
"lib/neverblock/core/fiber.rb",
|
16
|
+
"lib/neverblock/core/pool.rb",
|
17
|
+
"lib/neverblock/core/system/system.rb",
|
18
|
+
"lib/neverblock/core/system/timeout.rb",
|
19
|
+
"lib/neverblock/io/db/pool.rb",
|
20
|
+
"lib/neverblock/io/db/drivers/mysql.rb",
|
21
|
+
"lib/neverblock/io/db/drivers/postgres.rb",
|
22
|
+
"lib/neverblock/io/db/connection.rb",
|
23
|
+
"lib/neverblock/io/file.rb",
|
24
|
+
"lib/neverblock/io/socket.rb",
|
25
|
+
"lib/neverblock/io/io.rb",
|
26
|
+
"lib/system.rb",
|
27
|
+
"lib/neverblock.rb",
|
28
|
+
"lib/never_block.rb",
|
29
|
+
"lib/neverblock_io.rb"
|
30
|
+
|
31
|
+
]
|
32
|
+
s.rdoc_options = ["--main", "README"]
|
33
|
+
s.extra_rdoc_files = ["README"]
|
34
|
+
s.add_dependency('reactor', '>= 0.2.3')
|
35
|
+
end
|
36
|
+
|
37
|
+
|
metadata
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: michaelyta-neverblock
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: "1.0"
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Muhammad A. Ali
|
8
|
+
- Ahmed Sobhi
|
9
|
+
- Osama Brekaa
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
|
14
|
+
date: 2009-07-16 00:00:00 -07:00
|
15
|
+
default_executable:
|
16
|
+
dependencies:
|
17
|
+
- !ruby/object:Gem::Dependency
|
18
|
+
name: reactor
|
19
|
+
type: :runtime
|
20
|
+
version_requirement:
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
22
|
+
requirements:
|
23
|
+
- - ">="
|
24
|
+
- !ruby/object:Gem::Version
|
25
|
+
version: 0.2.3
|
26
|
+
version:
|
27
|
+
description: NeverBlock is a collection of classes and modules that help you write evented non-blocking applications in a seemingly blocking mannner.
|
28
|
+
email: oldmoe@gmail.com
|
29
|
+
executables: []
|
30
|
+
|
31
|
+
extensions: []
|
32
|
+
|
33
|
+
extra_rdoc_files:
|
34
|
+
- README
|
35
|
+
files:
|
36
|
+
- neverblock.gemspec
|
37
|
+
- README
|
38
|
+
- lib/neverblock/core/reactor.rb
|
39
|
+
- lib/neverblock/core/fiber.rb
|
40
|
+
- lib/neverblock/core/pool.rb
|
41
|
+
- lib/neverblock/core/system/system.rb
|
42
|
+
- lib/neverblock/core/system/timeout.rb
|
43
|
+
- lib/neverblock/io/db/pool.rb
|
44
|
+
- lib/neverblock/io/db/drivers/mysql.rb
|
45
|
+
- lib/neverblock/io/db/drivers/postgres.rb
|
46
|
+
- lib/neverblock/io/db/connection.rb
|
47
|
+
- lib/neverblock/io/file.rb
|
48
|
+
- lib/neverblock/io/socket.rb
|
49
|
+
- lib/neverblock/io/io.rb
|
50
|
+
- lib/system.rb
|
51
|
+
- lib/neverblock.rb
|
52
|
+
- lib/never_block.rb
|
53
|
+
- lib/neverblock_io.rb
|
54
|
+
has_rdoc: true
|
55
|
+
homepage: http://github.com/oldmoe/neverblock
|
56
|
+
post_install_message:
|
57
|
+
rdoc_options:
|
58
|
+
- --main
|
59
|
+
- README
|
60
|
+
require_paths:
|
61
|
+
- lib
|
62
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: "0"
|
67
|
+
version:
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: "0"
|
73
|
+
version:
|
74
|
+
requirements: []
|
75
|
+
|
76
|
+
rubyforge_project:
|
77
|
+
rubygems_version: 1.2.0
|
78
|
+
signing_key:
|
79
|
+
specification_version: 2
|
80
|
+
summary: Utilities for non-blocking stack components
|
81
|
+
test_files: []
|
82
|
+
|