polyphony 0.43.4 → 0.43.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +1 -1
- data/CHANGELOG.md +45 -0
- data/Gemfile.lock +1 -1
- data/README.md +21 -4
- data/TODO.md +1 -6
- data/bin/stress.rb +28 -0
- data/docs/_includes/head.html +40 -0
- data/docs/_includes/title.html +1 -0
- data/docs/_user-guide/web-server.md +11 -11
- data/docs/getting-started/overview.md +2 -2
- data/docs/index.md +4 -3
- data/docs/main-concepts/design-principles.md +23 -34
- data/docs/main-concepts/fiber-scheduling.md +1 -1
- data/docs/polyphony-logo.png +0 -0
- data/examples/core/xx-channels.rb +4 -2
- data/examples/core/xx-using-a-mutex.rb +2 -1
- data/examples/io/xx-happy-eyeballs.rb +21 -22
- data/examples/io/xx-zip.rb +19 -0
- data/examples/performance/fiber_transfer.rb +47 -0
- data/examples/performance/messaging.rb +29 -0
- data/examples/performance/multi_snooze.rb +11 -9
- data/examples/xx-spin.rb +32 -0
- data/ext/polyphony/agent.h +39 -0
- data/ext/polyphony/event.c +86 -0
- data/ext/polyphony/fiber.c +0 -5
- data/ext/polyphony/libev_agent.c +231 -79
- data/ext/polyphony/polyphony.c +2 -2
- data/ext/polyphony/polyphony.h +19 -16
- data/ext/polyphony/polyphony_ext.c +4 -2
- data/ext/polyphony/queue.c +194 -0
- data/ext/polyphony/ring_buffer.c +96 -0
- data/ext/polyphony/ring_buffer.h +28 -0
- data/ext/polyphony/thread.c +48 -31
- data/lib/polyphony.rb +5 -6
- data/lib/polyphony/core/channel.rb +3 -34
- data/lib/polyphony/core/resource_pool.rb +13 -75
- data/lib/polyphony/core/sync.rb +12 -9
- data/lib/polyphony/core/thread_pool.rb +1 -1
- data/lib/polyphony/extensions/core.rb +9 -0
- data/lib/polyphony/extensions/fiber.rb +9 -2
- data/lib/polyphony/extensions/io.rb +16 -15
- data/lib/polyphony/extensions/openssl.rb +8 -0
- data/lib/polyphony/extensions/socket.rb +13 -9
- data/lib/polyphony/extensions/thread.rb +1 -1
- data/lib/polyphony/version.rb +1 -1
- data/test/helper.rb +2 -2
- data/test/q.rb +24 -0
- data/test/test_agent.rb +2 -2
- data/test/test_event.rb +12 -0
- data/test/test_global_api.rb +2 -2
- data/test/test_io.rb +24 -2
- data/test/test_queue.rb +59 -1
- data/test/test_resource_pool.rb +0 -43
- data/test/test_trace.rb +18 -17
- metadata +16 -5
- data/ext/polyphony/libev_queue.c +0 -217
- data/lib/polyphony/event.rb +0 -27
data/lib/polyphony.rb
CHANGED
@@ -4,9 +4,6 @@ require 'fiber'
|
|
4
4
|
require_relative './polyphony_ext'
|
5
5
|
|
6
6
|
module Polyphony
|
7
|
-
# Map Queue to Libev queue implementation
|
8
|
-
Queue = LibevQueue
|
9
|
-
|
10
7
|
# replace core Queue class with our own
|
11
8
|
verbose = $VERBOSE
|
12
9
|
$VERBOSE = nil
|
@@ -20,13 +17,12 @@ require_relative './polyphony/extensions/fiber'
|
|
20
17
|
require_relative './polyphony/extensions/io'
|
21
18
|
|
22
19
|
Thread.current.setup_fiber_scheduling
|
23
|
-
Thread.current.agent = Polyphony::
|
20
|
+
Thread.current.agent = Polyphony::Agent.new
|
24
21
|
|
25
22
|
require_relative './polyphony/core/global_api'
|
26
23
|
require_relative './polyphony/core/resource_pool'
|
27
24
|
require_relative './polyphony/net'
|
28
25
|
require_relative './polyphony/adapters/process'
|
29
|
-
require_relative './polyphony/event'
|
30
26
|
|
31
27
|
# Main Polyphony API
|
32
28
|
module Polyphony
|
@@ -102,7 +98,10 @@ module Polyphony
|
|
102
98
|
|
103
99
|
def install_terminating_signal_handlers
|
104
100
|
trap('SIGTERM', SystemExit)
|
105
|
-
|
101
|
+
orig_trap('SIGINT') do
|
102
|
+
orig_trap('SIGINT') { exit! }
|
103
|
+
Thread.current.break_out_of_ev_loop(Thread.main.main_fiber, Interrupt.new)
|
104
|
+
end
|
106
105
|
end
|
107
106
|
|
108
107
|
def terminate_threads
|
@@ -5,42 +5,11 @@ require_relative './exceptions'
|
|
5
5
|
module Polyphony
|
6
6
|
# Implements a unidirectional communication channel along the lines of Go
|
7
7
|
# (buffered) channels.
|
8
|
-
class Channel
|
9
|
-
|
10
|
-
@payload_queue = []
|
11
|
-
@waiting_queue = []
|
12
|
-
end
|
8
|
+
class Channel < Polyphony::Queue
|
9
|
+
alias_method :receive, :shift
|
13
10
|
|
14
11
|
def close
|
15
|
-
|
16
|
-
@waiting_queue.slice(0..-1).each { |f| f.schedule(stop) }
|
17
|
-
end
|
18
|
-
|
19
|
-
def <<(value)
|
20
|
-
if @waiting_queue.empty?
|
21
|
-
@payload_queue << value
|
22
|
-
else
|
23
|
-
@waiting_queue.shift&.schedule(value)
|
24
|
-
end
|
25
|
-
snooze
|
26
|
-
end
|
27
|
-
|
28
|
-
def receive
|
29
|
-
Thread.current.agent.ref
|
30
|
-
if @payload_queue.empty?
|
31
|
-
@waiting_queue << Fiber.current
|
32
|
-
suspend
|
33
|
-
else
|
34
|
-
receive_from_queue
|
35
|
-
end
|
36
|
-
ensure
|
37
|
-
Thread.current.agent.unref
|
38
|
-
end
|
39
|
-
|
40
|
-
def receive_from_queue
|
41
|
-
payload = @payload_queue.shift
|
42
|
-
snooze
|
43
|
-
payload
|
12
|
+
flush_waiters(Polyphony::MoveOn.new)
|
44
13
|
end
|
45
14
|
end
|
46
15
|
end
|
@@ -10,13 +10,10 @@ module Polyphony
|
|
10
10
|
# @param &block [Proc] allocator block
|
11
11
|
def initialize(opts, &block)
|
12
12
|
@allocator = block
|
13
|
-
|
14
|
-
@stock = []
|
15
|
-
@queue = []
|
16
|
-
@acquired_resources = {}
|
17
|
-
|
18
13
|
@limit = opts[:limit] || 4
|
19
14
|
@size = 0
|
15
|
+
@stock = Polyphony::Queue.new
|
16
|
+
@acquired_resources = {}
|
20
17
|
end
|
21
18
|
|
22
19
|
def available
|
@@ -25,58 +22,17 @@ module Polyphony
|
|
25
22
|
|
26
23
|
def acquire
|
27
24
|
fiber = Fiber.current
|
28
|
-
if @acquired_resources[fiber]
|
29
|
-
yield @acquired_resources[fiber]
|
30
|
-
else
|
31
|
-
begin
|
32
|
-
Thread.current.agent.ref
|
33
|
-
resource = wait_for_resource
|
34
|
-
return unless resource
|
35
|
-
|
36
|
-
@acquired_resources[fiber] = resource
|
37
|
-
yield resource
|
38
|
-
ensure
|
39
|
-
@acquired_resources[fiber] = nil
|
40
|
-
Thread.current.agent.unref
|
41
|
-
release(resource) if resource
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
def wait_for_resource
|
47
|
-
fiber = Fiber.current
|
48
|
-
@queue << fiber
|
49
|
-
ready_resource = from_stock
|
50
|
-
return ready_resource if ready_resource
|
25
|
+
return @acquired_resources[fiber] if @acquired_resources[fiber]
|
51
26
|
|
52
|
-
|
27
|
+
add_to_stock if @size < @limit && @stock.empty?
|
28
|
+
resource = @stock.shift
|
29
|
+
@acquired_resources[fiber] = resource
|
30
|
+
yield resource
|
53
31
|
ensure
|
54
|
-
@
|
55
|
-
|
56
|
-
|
57
|
-
def release(resource)
|
58
|
-
if resource.__discarded__
|
59
|
-
@size -= 1
|
60
|
-
elsif resource
|
61
|
-
return_to_stock(resource)
|
62
|
-
dequeue
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def dequeue
|
67
|
-
return if @queue.empty? || @stock.empty?
|
68
|
-
|
69
|
-
@queue.shift.schedule(@stock.shift)
|
32
|
+
@acquired_resources.delete(fiber)
|
33
|
+
@stock.push resource if resource
|
70
34
|
end
|
71
|
-
|
72
|
-
def return_to_stock(resource)
|
73
|
-
@stock << resource
|
74
|
-
end
|
75
|
-
|
76
|
-
def from_stock
|
77
|
-
@stock.shift || (@size < @limit && allocate)
|
78
|
-
end
|
79
|
-
|
35
|
+
|
80
36
|
def method_missing(sym, *args, &block)
|
81
37
|
acquire { |r| r.send(sym, *args, &block) }
|
82
38
|
end
|
@@ -85,33 +41,15 @@ module Polyphony
|
|
85
41
|
true
|
86
42
|
end
|
87
43
|
|
88
|
-
# Extension to allow discarding of resources
|
89
|
-
module ResourceExtensions
|
90
|
-
def __discarded__
|
91
|
-
@__discarded__
|
92
|
-
end
|
93
|
-
|
94
|
-
def __discard__
|
95
|
-
@__discarded__ = true
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
44
|
# Allocates a resource
|
100
45
|
# @return [any] allocated resource
|
101
|
-
def
|
102
|
-
@size += 1
|
103
|
-
@allocator.().tap { |r| r.extend ResourceExtensions }
|
104
|
-
end
|
105
|
-
|
106
|
-
def <<(resource)
|
46
|
+
def add_to_stock
|
107
47
|
@size += 1
|
108
|
-
|
109
|
-
@stock << resource
|
110
|
-
dequeue
|
48
|
+
@stock << @allocator.call
|
111
49
|
end
|
112
50
|
|
113
51
|
def preheat!
|
114
|
-
|
52
|
+
add_to_stock while @size < @limit
|
115
53
|
end
|
116
54
|
end
|
117
55
|
end
|
data/lib/polyphony/core/sync.rb
CHANGED
@@ -4,18 +4,21 @@ module Polyphony
|
|
4
4
|
# Implements mutex lock for synchronizing access to a shared resource
|
5
5
|
class Mutex
|
6
6
|
def initialize
|
7
|
-
@
|
7
|
+
@store = Queue.new
|
8
|
+
@store << :token
|
8
9
|
end
|
9
10
|
|
10
11
|
def synchronize
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
12
|
+
return yield if @holding_fiber == Fiber.current
|
13
|
+
|
14
|
+
begin
|
15
|
+
token = @store.shift
|
16
|
+
@holding_fiber = Fiber.current
|
17
|
+
yield
|
18
|
+
ensure
|
19
|
+
@holding_fiber = nil
|
20
|
+
@store << token if token
|
21
|
+
end
|
19
22
|
end
|
20
23
|
end
|
21
24
|
end
|
@@ -107,6 +107,15 @@ module ::Kernel
|
|
107
107
|
$stdin.gets
|
108
108
|
end
|
109
109
|
|
110
|
+
alias_method :orig_p, :p
|
111
|
+
def p(*args)
|
112
|
+
strs = args.inject([]) do |m, a|
|
113
|
+
m << a.inspect << "\n"
|
114
|
+
end
|
115
|
+
STDOUT.write *strs
|
116
|
+
args.size == 1 ? args.first : args
|
117
|
+
end
|
118
|
+
|
110
119
|
alias_method :orig_system, :system
|
111
120
|
def system(*args)
|
112
121
|
Open3.popen2(*args) do |i, o, _t|
|
@@ -189,7 +189,7 @@ module Polyphony
|
|
189
189
|
end
|
190
190
|
|
191
191
|
def receive_pending
|
192
|
-
@mailbox.
|
192
|
+
@mailbox.shift_all
|
193
193
|
end
|
194
194
|
end
|
195
195
|
|
@@ -221,7 +221,14 @@ module Polyphony
|
|
221
221
|
def await_all_children
|
222
222
|
return unless @children && !@children.empty?
|
223
223
|
|
224
|
-
|
224
|
+
@results = @children.dup
|
225
|
+
@on_child_done = proc do |c, r|
|
226
|
+
@results[c] = r
|
227
|
+
self.schedule if @children.empty?
|
228
|
+
end
|
229
|
+
suspend
|
230
|
+
@on_child_done = nil
|
231
|
+
@results.values
|
225
232
|
end
|
226
233
|
|
227
234
|
def shutdown_all_children
|
@@ -108,19 +108,23 @@ class ::IO
|
|
108
108
|
end
|
109
109
|
|
110
110
|
alias_method :orig_readpartial, :read
|
111
|
-
def readpartial(len)
|
111
|
+
def readpartial(len, str = nil)
|
112
112
|
@read_buffer ||= +''
|
113
113
|
result = Thread.current.agent.read(self, @read_buffer, len, false)
|
114
114
|
raise EOFError unless result
|
115
115
|
|
116
|
-
|
116
|
+
if str
|
117
|
+
str << @read_buffer
|
118
|
+
else
|
119
|
+
str = @read_buffer
|
120
|
+
end
|
117
121
|
@read_buffer = +''
|
118
|
-
|
122
|
+
str
|
119
123
|
end
|
120
124
|
|
121
125
|
alias_method :orig_write, :write
|
122
|
-
def write(str)
|
123
|
-
Thread.current.agent.write(self, str)
|
126
|
+
def write(str, *args)
|
127
|
+
Thread.current.agent.write(self, str, *args)
|
124
128
|
end
|
125
129
|
|
126
130
|
alias_method :orig_write_chevron, :<<
|
@@ -166,16 +170,13 @@ class ::IO
|
|
166
170
|
return
|
167
171
|
end
|
168
172
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
str << a
|
175
|
-
str << "\n" unless a =~ /\n$/
|
176
|
-
end
|
173
|
+
strs = args.inject([]) do |m, a|
|
174
|
+
a = a.to_s
|
175
|
+
m << a
|
176
|
+
m << "\n" unless a =~ /\n$/
|
177
|
+
m
|
177
178
|
end
|
178
|
-
write
|
179
|
+
write *strs
|
179
180
|
nil
|
180
181
|
end
|
181
182
|
|
@@ -193,7 +194,7 @@ class ::IO
|
|
193
194
|
|
194
195
|
alias_method :orig_write_nonblock, :write_nonblock
|
195
196
|
def write_nonblock(string, _options = {})
|
196
|
-
write(string
|
197
|
+
write(string)
|
197
198
|
end
|
198
199
|
|
199
200
|
alias_method :orig_read_nonblock, :read_nonblock
|
@@ -5,6 +5,12 @@ require_relative './socket'
|
|
5
5
|
|
6
6
|
# Open ssl socket helper methods (to make it compatible with Socket API)
|
7
7
|
class ::OpenSSL::SSL::SSLSocket
|
8
|
+
alias_method :orig_initialize, :initialize
|
9
|
+
def initialize(socket, context = nil)
|
10
|
+
socket = socket.respond_to?(:io) ? socket.io || socket : socket
|
11
|
+
context ? orig_initialize(socket, context) : orig_initialize(socket)
|
12
|
+
end
|
13
|
+
|
8
14
|
def dont_linger
|
9
15
|
io.dont_linger
|
10
16
|
end
|
@@ -35,6 +41,7 @@ class ::OpenSSL::SSL::SSLSocket
|
|
35
41
|
loop do
|
36
42
|
case (result = read_nonblock(maxlen, buf, exception: false))
|
37
43
|
when :wait_readable then Thread.current.agent.wait_io(io, false)
|
44
|
+
when :wait_writable then Thread.current.agent.wait_io(io, true)
|
38
45
|
else return result
|
39
46
|
end
|
40
47
|
end
|
@@ -44,6 +51,7 @@ class ::OpenSSL::SSL::SSLSocket
|
|
44
51
|
def syswrite(buf)
|
45
52
|
loop do
|
46
53
|
case (result = write_nonblock(buf, exception: false))
|
54
|
+
when :wait_readable then Thread.current.agent.wait_io(io, false)
|
47
55
|
when :wait_writable then Thread.current.agent.wait_io(io, true)
|
48
56
|
else
|
49
57
|
return result
|
@@ -5,6 +5,16 @@ require 'socket'
|
|
5
5
|
require_relative './io'
|
6
6
|
require_relative '../core/thread_pool'
|
7
7
|
|
8
|
+
class ::BasicSocket
|
9
|
+
def write_nonblock(string, _options = {})
|
10
|
+
write(string)
|
11
|
+
end
|
12
|
+
|
13
|
+
def read_nonblock(maxlen, str = nil, _options = {})
|
14
|
+
readpartial(maxlen, str)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
8
18
|
# Socket overrides (eventually rewritten in C)
|
9
19
|
class ::Socket
|
10
20
|
def accept
|
@@ -14,15 +24,7 @@ class ::Socket
|
|
14
24
|
NO_EXCEPTION = { exception: false }.freeze
|
15
25
|
|
16
26
|
def connect(remotesockaddr)
|
17
|
-
|
18
|
-
result = connect_nonblock(remotesockaddr, **NO_EXCEPTION)
|
19
|
-
case result
|
20
|
-
when 0 then return
|
21
|
-
when :wait_writable then Thread.current.agent.wait_io(self, true)
|
22
|
-
else
|
23
|
-
raise IOError
|
24
|
-
end
|
25
|
-
end
|
27
|
+
Thread.current.agent.connect(self, remotesockaddr.ip_address, remotesockaddr.ip_port)
|
26
28
|
end
|
27
29
|
|
28
30
|
def recv(maxlen, flags = 0, outbuf = nil)
|
@@ -77,6 +79,8 @@ end
|
|
77
79
|
class ::TCPSocket
|
78
80
|
NO_EXCEPTION = { exception: false }.freeze
|
79
81
|
|
82
|
+
attr_reader :io
|
83
|
+
|
80
84
|
def initialize(remote_host, remote_port, local_host = nil, local_port = nil)
|
81
85
|
@io = Socket.new Socket::AF_INET, Socket::SOCK_STREAM
|
82
86
|
if local_host && local_port
|
@@ -18,7 +18,7 @@ class ::Thread
|
|
18
18
|
def execute
|
19
19
|
# agent must be created in the context of the new thread, therefore it
|
20
20
|
# cannot be created in Thread#initialize
|
21
|
-
@agent = Polyphony::
|
21
|
+
@agent = Polyphony::Agent.new
|
22
22
|
setup
|
23
23
|
@ready = true
|
24
24
|
result = @block.(*@args)
|
data/lib/polyphony/version.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -31,8 +31,8 @@ class MiniTest::Test
|
|
31
31
|
end
|
32
32
|
Fiber.current.setup_main_fiber
|
33
33
|
Fiber.current.instance_variable_set(:@auto_watcher, nil)
|
34
|
-
Thread.current.agent = Polyphony::
|
35
|
-
sleep 0
|
34
|
+
Thread.current.agent = Polyphony::Agent.new
|
35
|
+
sleep 0 # apparently this helps with timer accuracy
|
36
36
|
end
|
37
37
|
|
38
38
|
def teardown
|
data/test/q.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'fiber'
|
5
|
+
require_relative '../lib/polyphony_ext'
|
6
|
+
|
7
|
+
queue = Polyphony::LibevQueue.new
|
8
|
+
|
9
|
+
queue.push :a
|
10
|
+
queue.push :b
|
11
|
+
queue.push :c
|
12
|
+
p [queue.shift_no_wait]
|
13
|
+
queue.push :d
|
14
|
+
p [queue.shift_no_wait]
|
15
|
+
p [queue.shift_no_wait]
|
16
|
+
p [queue.shift_no_wait]
|
17
|
+
p [queue.shift_no_wait]
|
18
|
+
|
19
|
+
queue.unshift :e
|
20
|
+
p [queue.shift_no_wait]
|
21
|
+
|
22
|
+
queue.push :f
|
23
|
+
queue.push :g
|
24
|
+
p [queue.shift_no_wait]
|