polyphony 0.43.5 → 0.43.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +45 -0
- data/Gemfile.lock +1 -1
- data/README.md +21 -4
- data/TODO.md +0 -7
- data/bin/stress.rb +28 -0
- data/docs/_includes/head.html +40 -0
- data/docs/_includes/title.html +1 -0
- data/docs/_user-guide/web-server.md +11 -11
- data/docs/getting-started/overview.md +2 -2
- data/docs/index.md +3 -1
- data/docs/polyphony-logo.png +0 -0
- data/examples/core/xx-channels.rb +4 -2
- data/examples/core/xx-using-a-mutex.rb +2 -1
- data/examples/io/xx-happy-eyeballs.rb +21 -22
- data/examples/io/xx-zip.rb +19 -0
- data/examples/performance/fiber_transfer.rb +47 -0
- data/examples/xx-spin.rb +32 -0
- data/ext/polyphony/agent.h +41 -0
- data/ext/polyphony/event.c +86 -0
- data/ext/polyphony/fiber.c +0 -5
- data/ext/polyphony/libev_agent.c +277 -135
- data/ext/polyphony/polyphony.c +2 -2
- data/ext/polyphony/polyphony.h +14 -21
- data/ext/polyphony/polyphony_ext.c +4 -2
- data/ext/polyphony/queue.c +208 -0
- data/ext/polyphony/ring_buffer.c +0 -24
- data/ext/polyphony/thread.c +42 -31
- data/lib/polyphony.rb +6 -7
- data/lib/polyphony/core/channel.rb +3 -34
- data/lib/polyphony/core/resource_pool.rb +13 -75
- data/lib/polyphony/core/sync.rb +12 -9
- data/lib/polyphony/extensions/fiber.rb +8 -8
- data/lib/polyphony/extensions/openssl.rb +8 -0
- data/lib/polyphony/extensions/socket.rb +11 -9
- data/lib/polyphony/extensions/thread.rb +1 -1
- data/lib/polyphony/net.rb +2 -1
- data/lib/polyphony/version.rb +1 -1
- data/test/helper.rb +2 -2
- data/test/test_agent.rb +2 -2
- data/test/test_event.rb +12 -0
- data/test/test_fiber.rb +1 -1
- data/test/test_io.rb +14 -0
- data/test/test_queue.rb +33 -0
- data/test/test_resource_pool.rb +24 -58
- data/test/test_trace.rb +18 -17
- metadata +12 -5
- data/ext/polyphony/libev_queue.c +0 -288
- data/lib/polyphony/event.rb +0 -27
data/lib/polyphony.rb
CHANGED
@@ -4,9 +4,6 @@ require 'fiber'
|
|
4
4
|
require_relative './polyphony_ext'
|
5
5
|
|
6
6
|
module Polyphony
|
7
|
-
# Map Queue to Libev queue implementation
|
8
|
-
Queue = LibevQueue
|
9
|
-
|
10
7
|
# replace core Queue class with our own
|
11
8
|
verbose = $VERBOSE
|
12
9
|
$VERBOSE = nil
|
@@ -20,13 +17,12 @@ require_relative './polyphony/extensions/fiber'
|
|
20
17
|
require_relative './polyphony/extensions/io'
|
21
18
|
|
22
19
|
Thread.current.setup_fiber_scheduling
|
23
|
-
Thread.current.agent = Polyphony::
|
20
|
+
Thread.current.agent = Polyphony::Agent.new
|
24
21
|
|
25
22
|
require_relative './polyphony/core/global_api'
|
26
23
|
require_relative './polyphony/core/resource_pool'
|
27
24
|
require_relative './polyphony/net'
|
28
25
|
require_relative './polyphony/adapters/process'
|
29
|
-
require_relative './polyphony/event'
|
30
26
|
|
31
27
|
# Main Polyphony API
|
32
28
|
module Polyphony
|
@@ -61,7 +57,7 @@ module Polyphony
|
|
61
57
|
rescue SystemExit
|
62
58
|
# fall through to ensure
|
63
59
|
rescue Exception => e
|
64
|
-
e.full_message
|
60
|
+
warn e.full_message
|
65
61
|
exit!
|
66
62
|
ensure
|
67
63
|
exit_forked_process
|
@@ -102,7 +98,10 @@ module Polyphony
|
|
102
98
|
|
103
99
|
def install_terminating_signal_handlers
|
104
100
|
trap('SIGTERM', SystemExit)
|
105
|
-
|
101
|
+
orig_trap('SIGINT') do
|
102
|
+
orig_trap('SIGINT') { exit! }
|
103
|
+
Thread.current.break_out_of_ev_loop(Thread.main.main_fiber, Interrupt.new)
|
104
|
+
end
|
106
105
|
end
|
107
106
|
|
108
107
|
def terminate_threads
|
@@ -5,42 +5,11 @@ require_relative './exceptions'
|
|
5
5
|
module Polyphony
|
6
6
|
# Implements a unidirectional communication channel along the lines of Go
|
7
7
|
# (buffered) channels.
|
8
|
-
class Channel
|
9
|
-
|
10
|
-
@payload_queue = []
|
11
|
-
@waiting_queue = []
|
12
|
-
end
|
8
|
+
class Channel < Polyphony::Queue
|
9
|
+
alias_method :receive, :shift
|
13
10
|
|
14
11
|
def close
|
15
|
-
|
16
|
-
@waiting_queue.slice(0..-1).each { |f| f.schedule(stop) }
|
17
|
-
end
|
18
|
-
|
19
|
-
def <<(value)
|
20
|
-
if @waiting_queue.empty?
|
21
|
-
@payload_queue << value
|
22
|
-
else
|
23
|
-
@waiting_queue.shift&.schedule(value)
|
24
|
-
end
|
25
|
-
snooze
|
26
|
-
end
|
27
|
-
|
28
|
-
def receive
|
29
|
-
Thread.current.agent.ref
|
30
|
-
if @payload_queue.empty?
|
31
|
-
@waiting_queue << Fiber.current
|
32
|
-
suspend
|
33
|
-
else
|
34
|
-
receive_from_queue
|
35
|
-
end
|
36
|
-
ensure
|
37
|
-
Thread.current.agent.unref
|
38
|
-
end
|
39
|
-
|
40
|
-
def receive_from_queue
|
41
|
-
payload = @payload_queue.shift
|
42
|
-
snooze
|
43
|
-
payload
|
12
|
+
flush_waiters(Polyphony::MoveOn.new)
|
44
13
|
end
|
45
14
|
end
|
46
15
|
end
|
@@ -10,13 +10,10 @@ module Polyphony
|
|
10
10
|
# @param &block [Proc] allocator block
|
11
11
|
def initialize(opts, &block)
|
12
12
|
@allocator = block
|
13
|
-
|
14
|
-
@stock = []
|
15
|
-
@queue = []
|
16
|
-
@acquired_resources = {}
|
17
|
-
|
18
13
|
@limit = opts[:limit] || 4
|
19
14
|
@size = 0
|
15
|
+
@stock = Polyphony::Queue.new
|
16
|
+
@acquired_resources = {}
|
20
17
|
end
|
21
18
|
|
22
19
|
def available
|
@@ -25,58 +22,17 @@ module Polyphony
|
|
25
22
|
|
26
23
|
def acquire
|
27
24
|
fiber = Fiber.current
|
28
|
-
if @acquired_resources[fiber]
|
29
|
-
yield @acquired_resources[fiber]
|
30
|
-
else
|
31
|
-
begin
|
32
|
-
Thread.current.agent.ref
|
33
|
-
resource = wait_for_resource
|
34
|
-
return unless resource
|
35
|
-
|
36
|
-
@acquired_resources[fiber] = resource
|
37
|
-
yield resource
|
38
|
-
ensure
|
39
|
-
@acquired_resources[fiber] = nil
|
40
|
-
Thread.current.agent.unref
|
41
|
-
release(resource) if resource
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
def wait_for_resource
|
47
|
-
fiber = Fiber.current
|
48
|
-
@queue << fiber
|
49
|
-
ready_resource = from_stock
|
50
|
-
return ready_resource if ready_resource
|
25
|
+
return @acquired_resources[fiber] if @acquired_resources[fiber]
|
51
26
|
|
52
|
-
|
27
|
+
add_to_stock if (@stock.empty? || @stock.pending?) && @size < @limit
|
28
|
+
resource = @stock.shift
|
29
|
+
@acquired_resources[fiber] = resource
|
30
|
+
yield resource
|
53
31
|
ensure
|
54
|
-
@
|
55
|
-
|
56
|
-
|
57
|
-
def release(resource)
|
58
|
-
if resource.__discarded__
|
59
|
-
@size -= 1
|
60
|
-
elsif resource
|
61
|
-
return_to_stock(resource)
|
62
|
-
dequeue
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def dequeue
|
67
|
-
return if @queue.empty? || @stock.empty?
|
68
|
-
|
69
|
-
@queue.shift.schedule(@stock.shift)
|
32
|
+
@acquired_resources.delete(fiber)
|
33
|
+
@stock.push resource if resource
|
70
34
|
end
|
71
|
-
|
72
|
-
def return_to_stock(resource)
|
73
|
-
@stock << resource
|
74
|
-
end
|
75
|
-
|
76
|
-
def from_stock
|
77
|
-
@stock.shift || (@size < @limit && allocate)
|
78
|
-
end
|
79
|
-
|
35
|
+
|
80
36
|
def method_missing(sym, *args, &block)
|
81
37
|
acquire { |r| r.send(sym, *args, &block) }
|
82
38
|
end
|
@@ -85,33 +41,15 @@ module Polyphony
|
|
85
41
|
true
|
86
42
|
end
|
87
43
|
|
88
|
-
# Extension to allow discarding of resources
|
89
|
-
module ResourceExtensions
|
90
|
-
def __discarded__
|
91
|
-
@__discarded__
|
92
|
-
end
|
93
|
-
|
94
|
-
def __discard__
|
95
|
-
@__discarded__ = true
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
44
|
# Allocates a resource
|
100
45
|
# @return [any] allocated resource
|
101
|
-
def
|
102
|
-
@size += 1
|
103
|
-
@allocator.().tap { |r| r.extend ResourceExtensions }
|
104
|
-
end
|
105
|
-
|
106
|
-
def <<(resource)
|
46
|
+
def add_to_stock
|
107
47
|
@size += 1
|
108
|
-
|
109
|
-
@stock << resource
|
110
|
-
dequeue
|
48
|
+
@stock << @allocator.call
|
111
49
|
end
|
112
50
|
|
113
51
|
def preheat!
|
114
|
-
|
52
|
+
add_to_stock while @size < @limit
|
115
53
|
end
|
116
54
|
end
|
117
55
|
end
|
data/lib/polyphony/core/sync.rb
CHANGED
@@ -4,18 +4,21 @@ module Polyphony
|
|
4
4
|
# Implements mutex lock for synchronizing access to a shared resource
|
5
5
|
class Mutex
|
6
6
|
def initialize
|
7
|
-
@
|
7
|
+
@store = Queue.new
|
8
|
+
@store << :token
|
8
9
|
end
|
9
10
|
|
10
11
|
def synchronize
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
12
|
+
return yield if @holding_fiber == Fiber.current
|
13
|
+
|
14
|
+
begin
|
15
|
+
token = @store.shift
|
16
|
+
@holding_fiber = Fiber.current
|
17
|
+
yield
|
18
|
+
ensure
|
19
|
+
@holding_fiber = nil
|
20
|
+
@store << token if token
|
21
|
+
end
|
19
22
|
end
|
20
23
|
end
|
21
24
|
end
|
@@ -221,14 +221,14 @@ module Polyphony
|
|
221
221
|
def await_all_children
|
222
222
|
return unless @children && !@children.empty?
|
223
223
|
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
224
|
+
@results = @children.dup
|
225
|
+
@on_child_done = proc do |c, r|
|
226
|
+
@results[c] = r
|
227
|
+
self.schedule if @children.empty?
|
228
|
+
end
|
229
|
+
suspend
|
230
|
+
@on_child_done = nil
|
231
|
+
@results.values
|
232
232
|
end
|
233
233
|
|
234
234
|
def shutdown_all_children
|
@@ -5,6 +5,12 @@ require_relative './socket'
|
|
5
5
|
|
6
6
|
# Open ssl socket helper methods (to make it compatible with Socket API)
|
7
7
|
class ::OpenSSL::SSL::SSLSocket
|
8
|
+
alias_method :orig_initialize, :initialize
|
9
|
+
def initialize(socket, context = nil)
|
10
|
+
socket = socket.respond_to?(:io) ? socket.io || socket : socket
|
11
|
+
context ? orig_initialize(socket, context) : orig_initialize(socket)
|
12
|
+
end
|
13
|
+
|
8
14
|
def dont_linger
|
9
15
|
io.dont_linger
|
10
16
|
end
|
@@ -35,6 +41,7 @@ class ::OpenSSL::SSL::SSLSocket
|
|
35
41
|
loop do
|
36
42
|
case (result = read_nonblock(maxlen, buf, exception: false))
|
37
43
|
when :wait_readable then Thread.current.agent.wait_io(io, false)
|
44
|
+
when :wait_writable then Thread.current.agent.wait_io(io, true)
|
38
45
|
else return result
|
39
46
|
end
|
40
47
|
end
|
@@ -44,6 +51,7 @@ class ::OpenSSL::SSL::SSLSocket
|
|
44
51
|
def syswrite(buf)
|
45
52
|
loop do
|
46
53
|
case (result = write_nonblock(buf, exception: false))
|
54
|
+
when :wait_readable then Thread.current.agent.wait_io(io, false)
|
47
55
|
when :wait_writable then Thread.current.agent.wait_io(io, true)
|
48
56
|
else
|
49
57
|
return result
|
@@ -24,15 +24,7 @@ class ::Socket
|
|
24
24
|
NO_EXCEPTION = { exception: false }.freeze
|
25
25
|
|
26
26
|
def connect(remotesockaddr)
|
27
|
-
|
28
|
-
result = connect_nonblock(remotesockaddr, **NO_EXCEPTION)
|
29
|
-
case result
|
30
|
-
when 0 then return
|
31
|
-
when :wait_writable then Thread.current.agent.wait_io(self, true)
|
32
|
-
else
|
33
|
-
raise IOError
|
34
|
-
end
|
35
|
-
end
|
27
|
+
Thread.current.agent.connect(self, remotesockaddr.ip_address, remotesockaddr.ip_port)
|
36
28
|
end
|
37
29
|
|
38
30
|
def recv(maxlen, flags = 0, outbuf = nil)
|
@@ -75,6 +67,10 @@ class ::Socket
|
|
75
67
|
setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
|
76
68
|
end
|
77
69
|
|
70
|
+
def reuse_port
|
71
|
+
setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
|
72
|
+
end
|
73
|
+
|
78
74
|
class << self
|
79
75
|
alias_method :orig_getaddrinfo, :getaddrinfo
|
80
76
|
def getaddrinfo(*args)
|
@@ -87,6 +83,8 @@ end
|
|
87
83
|
class ::TCPSocket
|
88
84
|
NO_EXCEPTION = { exception: false }.freeze
|
89
85
|
|
86
|
+
attr_reader :io
|
87
|
+
|
90
88
|
def initialize(remote_host, remote_port, local_host = nil, local_port = nil)
|
91
89
|
@io = Socket.new Socket::AF_INET, Socket::SOCK_STREAM
|
92
90
|
if local_host && local_port
|
@@ -126,6 +124,10 @@ class ::TCPSocket
|
|
126
124
|
def reuse_addr
|
127
125
|
setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, 1)
|
128
126
|
end
|
127
|
+
|
128
|
+
def reuse_port
|
129
|
+
setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
|
130
|
+
end
|
129
131
|
end
|
130
132
|
|
131
133
|
# Override stock TCPServer code by encapsulating a Socket instance.
|
@@ -18,7 +18,7 @@ class ::Thread
|
|
18
18
|
def execute
|
19
19
|
# agent must be created in the context of the new thread, therefore it
|
20
20
|
# cannot be created in Thread#initialize
|
21
|
-
@agent = Polyphony::
|
21
|
+
@agent = Polyphony::Agent.new
|
22
22
|
setup
|
23
23
|
@ready = true
|
24
24
|
result = @block.(*@args)
|
data/lib/polyphony/net.rb
CHANGED
@@ -35,9 +35,10 @@ module Polyphony
|
|
35
35
|
::Socket.new(:INET, :STREAM).tap do |s|
|
36
36
|
s.reuse_addr if opts[:reuse_addr]
|
37
37
|
s.dont_linger if opts[:dont_linger]
|
38
|
+
s.reuse_port if opts[:reuse_port]
|
38
39
|
addr = ::Socket.sockaddr_in(port, host)
|
39
40
|
s.bind(addr)
|
40
|
-
s.listen(
|
41
|
+
s.listen(opts[:backlog] || Socket::SOMAXCONN)
|
41
42
|
end
|
42
43
|
end
|
43
44
|
|
data/lib/polyphony/version.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -31,8 +31,8 @@ class MiniTest::Test
|
|
31
31
|
end
|
32
32
|
Fiber.current.setup_main_fiber
|
33
33
|
Fiber.current.instance_variable_set(:@auto_watcher, nil)
|
34
|
-
Thread.current.agent = Polyphony::
|
35
|
-
sleep 0
|
34
|
+
Thread.current.agent = Polyphony::Agent.new
|
35
|
+
sleep 0 # apparently this helps with timer accuracy
|
36
36
|
end
|
37
37
|
|
38
38
|
def teardown
|
data/test/test_agent.rb
CHANGED
@@ -6,7 +6,7 @@ class AgentTest < MiniTest::Test
|
|
6
6
|
def setup
|
7
7
|
super
|
8
8
|
@prev_agent = Thread.current.agent
|
9
|
-
@agent = Polyphony::
|
9
|
+
@agent = Polyphony::Agent.new
|
10
10
|
Thread.current.agent = @agent
|
11
11
|
end
|
12
12
|
|
@@ -97,7 +97,7 @@ class AgentTest < MiniTest::Test
|
|
97
97
|
o.close
|
98
98
|
|
99
99
|
# read_loop will snooze after every read
|
100
|
-
|
100
|
+
6.times { snooze }
|
101
101
|
|
102
102
|
assert_equal [:ready, 'foo', 'bar', :done], buf
|
103
103
|
end
|
data/test/test_event.rb
CHANGED
@@ -34,6 +34,7 @@ class EventTest < MiniTest::Test
|
|
34
34
|
}
|
35
35
|
}
|
36
36
|
snooze
|
37
|
+
|
37
38
|
t = Thread.new do
|
38
39
|
orig_sleep 0.001
|
39
40
|
3.times { a.signal }
|
@@ -45,4 +46,15 @@ class EventTest < MiniTest::Test
|
|
45
46
|
t&.kill
|
46
47
|
t&.join
|
47
48
|
end
|
49
|
+
|
50
|
+
def test_exception_while_waiting_for_event
|
51
|
+
e = Polyphony::Event.new
|
52
|
+
|
53
|
+
f = spin { e.await }
|
54
|
+
g = spin { f.raise 'foo' }
|
55
|
+
|
56
|
+
assert_raises(RuntimeError) do
|
57
|
+
f.await
|
58
|
+
end
|
59
|
+
end
|
48
60
|
end
|
data/test/test_fiber.rb
CHANGED
data/test/test_io.rb
CHANGED
@@ -91,6 +91,20 @@ class IOTest < MiniTest::Test
|
|
91
91
|
|
92
92
|
assert_raises(EOFError) { i.readpartial(1) }
|
93
93
|
end
|
94
|
+
|
95
|
+
# see https://github.com/digital-fabric/polyphony/issues/30
|
96
|
+
def test_reopened_tempfile
|
97
|
+
file = Tempfile.new
|
98
|
+
file << 'hello: world'
|
99
|
+
file.close
|
100
|
+
|
101
|
+
buf = nil
|
102
|
+
File.open(file, 'r:bom|utf-8') do |f|
|
103
|
+
buf = f.read(16384)
|
104
|
+
end
|
105
|
+
|
106
|
+
assert_equal 'hello: world', buf
|
107
|
+
end
|
94
108
|
end
|
95
109
|
|
96
110
|
class IOClassMethodsTest < MiniTest::Test
|