polyphony 0.99 → 0.99.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/.github/FUNDING.yml +1 -1
  3. data/.rubocop.yml +3 -3
  4. data/.yardopts +30 -0
  5. data/CHANGELOG.md +4 -0
  6. data/LICENSE +1 -1
  7. data/README.md +63 -29
  8. data/Rakefile +1 -5
  9. data/TODO.md +0 -4
  10. data/docs/{main-concepts/concurrency.md → concurrency.md} +2 -9
  11. data/docs/{main-concepts/design-principles.md → design-principles.md} +3 -9
  12. data/docs/{main-concepts/exception-handling.md → exception-handling.md} +2 -9
  13. data/docs/{main-concepts/extending.md → extending.md} +2 -9
  14. data/docs/faq.md +3 -16
  15. data/docs/{main-concepts/fiber-scheduling.md → fiber-scheduling.md} +1 -9
  16. data/docs/link_rewriter.rb +16 -0
  17. data/docs/{getting-started/overview.md → overview.md} +1 -30
  18. data/docs/{getting-started/tutorial.md → tutorial.md} +3 -28
  19. data/docs/{_posts/2020-07-26-polyphony-0.44.md → whats-new.md} +3 -1
  20. data/examples/adapters/redis_client.rb +3 -2
  21. data/examples/io/echo_server.rb +1 -1
  22. data/examples/io/echo_server_plain_ruby.rb +26 -0
  23. data/ext/polyphony/backend_io_uring.c +154 -9
  24. data/ext/polyphony/backend_io_uring_context.c +21 -12
  25. data/ext/polyphony/backend_io_uring_context.h +12 -7
  26. data/ext/polyphony/backend_libev.c +1 -1
  27. data/ext/polyphony/extconf.rb +24 -8
  28. data/ext/polyphony/fiber.c +79 -2
  29. data/ext/polyphony/io_extensions.c +53 -0
  30. data/ext/polyphony/pipe.c +42 -2
  31. data/ext/polyphony/polyphony.c +345 -31
  32. data/ext/polyphony/polyphony.h +9 -2
  33. data/ext/polyphony/queue.c +181 -0
  34. data/ext/polyphony/ring_buffer.c +0 -1
  35. data/ext/polyphony/runqueue.c +8 -1
  36. data/ext/polyphony/runqueue_ring_buffer.c +13 -0
  37. data/ext/polyphony/runqueue_ring_buffer.h +2 -1
  38. data/ext/polyphony/socket_extensions.c +6 -0
  39. data/ext/polyphony/thread.c +34 -2
  40. data/lib/polyphony/adapters/process.rb +11 -1
  41. data/lib/polyphony/adapters/sequel.rb +1 -1
  42. data/lib/polyphony/core/channel.rb +2 -0
  43. data/lib/polyphony/core/debug.rb +1 -1
  44. data/lib/polyphony/core/global_api.rb +25 -24
  45. data/lib/polyphony/core/resource_pool.rb +7 -6
  46. data/lib/polyphony/core/sync.rb +2 -2
  47. data/lib/polyphony/core/thread_pool.rb +3 -3
  48. data/lib/polyphony/core/timer.rb +8 -8
  49. data/lib/polyphony/extensions/exception.rb +2 -0
  50. data/lib/polyphony/extensions/fiber.rb +15 -13
  51. data/lib/polyphony/extensions/io.rb +127 -5
  52. data/lib/polyphony/extensions/kernel.rb +20 -2
  53. data/lib/polyphony/extensions/openssl.rb +100 -11
  54. data/lib/polyphony/extensions/pipe.rb +103 -7
  55. data/lib/polyphony/extensions/process.rb +13 -1
  56. data/lib/polyphony/extensions/socket.rb +93 -27
  57. data/lib/polyphony/extensions/thread.rb +9 -1
  58. data/lib/polyphony/extensions/timeout.rb +1 -1
  59. data/lib/polyphony/version.rb +2 -1
  60. data/lib/polyphony.rb +27 -7
  61. data/polyphony.gemspec +1 -8
  62. data/test/stress.rb +1 -1
  63. data/test/test_global_api.rb +45 -7
  64. data/test/test_socket.rb +96 -0
  65. data/test/test_timer.rb +5 -5
  66. metadata +17 -40
  67. data/docs/_config.yml +0 -64
  68. data/docs/_includes/head.html +0 -40
  69. data/docs/_includes/title.html +0 -1
  70. data/docs/_sass/custom/custom.scss +0 -10
  71. data/docs/_sass/overrides.scss +0 -0
  72. data/docs/api-reference/exception.md +0 -31
  73. data/docs/api-reference/fiber.md +0 -425
  74. data/docs/api-reference/index.md +0 -9
  75. data/docs/api-reference/io.md +0 -36
  76. data/docs/api-reference/object.md +0 -99
  77. data/docs/api-reference/polyphony-baseexception.md +0 -33
  78. data/docs/api-reference/polyphony-cancel.md +0 -26
  79. data/docs/api-reference/polyphony-moveon.md +0 -24
  80. data/docs/api-reference/polyphony-net.md +0 -20
  81. data/docs/api-reference/polyphony-process.md +0 -28
  82. data/docs/api-reference/polyphony-resourcepool.md +0 -59
  83. data/docs/api-reference/polyphony-restart.md +0 -18
  84. data/docs/api-reference/polyphony-terminate.md +0 -18
  85. data/docs/api-reference/polyphony-threadpool.md +0 -67
  86. data/docs/api-reference/polyphony-throttler.md +0 -77
  87. data/docs/api-reference/polyphony.md +0 -36
  88. data/docs/api-reference/thread.md +0 -88
  89. data/docs/favicon.ico +0 -0
  90. data/docs/getting-started/index.md +0 -10
  91. data/docs/getting-started/installing.md +0 -34
  92. /data/{docs/assets/img → assets}/echo-fibers.svg +0 -0
  93. /data/{docs → assets}/polyphony-logo.png +0 -0
  94. /data/{docs/assets/img → assets}/sleeping-fiber.svg +0 -0
@@ -5,30 +5,40 @@ require_relative './socket'
5
5
 
6
6
  # OpenSSL socket helper methods (to make it compatible with Socket API) and overrides
7
7
  class ::OpenSSL::SSL::SSLSocket
8
+ # @!visibility private
8
9
  def __read_method__
9
10
  :readpartial
10
11
  end
11
12
 
13
+ # @!visibility private
12
14
  alias_method :orig_initialize, :initialize
13
15
 
14
- # TODO: add docs to all methods in this file
16
+ # Initializese a new SSL socket
17
+ #
18
+ # @param socket [TCPSocket] socket to wrap
19
+ # @param context [OpenSSL::SSL::SSLContext] optional SSL context
20
+ # @return [void]
15
21
  def initialize(socket, context = nil)
16
22
  socket = socket.respond_to?(:io) ? socket.io || socket : socket
17
23
  context ? orig_initialize(socket, context) : orig_initialize(socket)
18
24
  end
19
25
 
26
+ # Sets DONT_LINGER option
20
27
  def dont_linger
21
28
  io.dont_linger
22
29
  end
23
30
 
31
+ # Sets NO_DELAY option
24
32
  def no_delay
25
33
  io.no_delay
26
34
  end
27
35
 
36
+ # Sets REUSE_ADDR option
28
37
  def reuse_addr
29
38
  io.reuse_addr
30
39
  end
31
40
 
41
+ # @!visibility private
32
42
  def fill_rbuff
33
43
  data = self.sysread(BLOCK_SIZE)
34
44
  if data
@@ -38,7 +48,10 @@ class ::OpenSSL::SSL::SSLSocket
38
48
  end
39
49
  end
40
50
 
51
+ # @!visibility private
41
52
  alias_method :orig_sysread, :sysread
53
+
54
+ # @!visibility private
42
55
  def sysread(maxlen, buf = +'')
43
56
  # ensure socket is non blocking
44
57
  Polyphony.backend_verify_blocking_mode(io, false)
@@ -51,7 +64,10 @@ class ::OpenSSL::SSL::SSLSocket
51
64
  end
52
65
  end
53
66
 
67
+ # @!visibility private
54
68
  alias_method :orig_syswrite, :syswrite
69
+
70
+ # @!visibility private
55
71
  def syswrite(buf)
56
72
  # ensure socket is non blocking
57
73
  Polyphony.backend_verify_blocking_mode(io, false)
@@ -65,16 +81,35 @@ class ::OpenSSL::SSL::SSLSocket
65
81
  end
66
82
  end
67
83
 
84
+ # @!visibility private
68
85
  def flush
69
- # osync = @sync
70
- # @sync = true
71
- # do_write ""
72
- # return self
73
- # ensure
74
- # @sync = osync
86
+ # warn "SSLSocket#flush is not yet implemented in Polyphony"
75
87
  end
76
88
 
89
+ # @!visibility private
77
90
  alias_method :orig_read, :read
91
+
92
+ # call-seq:
93
+ # socket.read -> string
94
+ # socket.read(maxlen) -> string
95
+ # socket.read(maxlen, buf) -> buf
96
+ # socket.read(maxlen, buf, buf_pos) -> buf
97
+ #
98
+ # Reads from the socket. If `maxlen` is given, reads up to `maxlen` bytes from
99
+ # the socket, otherwise reads to `EOF`. If `buf` is given, it is used as the
100
+ # buffer to read into, otherwise a new string is allocated. If `buf_pos` is
101
+ # given, reads into the given offset (in bytes) in the given buffer. If the
102
+ # given buffer offset is negative, it is calculated from the current end of
103
+ # the buffer (`-1` means the read data will be appended to the end of the
104
+ # buffer).
105
+ #
106
+ # If no bytes are available and `EOF` is not hit, this method will block until
107
+ # the socket is ready to read from.
108
+ #
109
+ # @param maxlen [Integer, nil] maximum bytes to read from socket
110
+ # @param buf [String, nil] buffer to read into
111
+ # @param buf_pos [Number] buffer position to read into
112
+ # @return [String] buffer used for reading
78
113
  def read(maxlen = nil, buf = nil, buf_pos = 0)
79
114
  return readpartial(maxlen, buf, buf_pos) if buf
80
115
 
@@ -88,13 +123,35 @@ class ::OpenSSL::SSL::SSLSocket
88
123
  buf
89
124
  end
90
125
 
91
- def readpartial(maxlen, buf = +'', buffer_pos = 0, raise_on_eof = true)
92
- if buffer_pos != 0
126
+ # call-seq:
127
+ # socket.readpartial(maxlen) -> string
128
+ # socket.readpartial(maxlen, buf) -> buf
129
+ # socket.readpartial(maxlen, buf, buf_pos) -> buf
130
+ # socket.readpartial(maxlen, buf, buf_pos, raise_on_eof) -> buf
131
+ #
132
+ # Reads up to `maxlen` from the socket. If `buf` is given, it is used as the
133
+ # buffer to read into, otherwise a new string is allocated. If `buf_pos` is
134
+ # given, reads into the given offset (in bytes) in the given buffer. If the
135
+ # given buffer offset is negative, it is calculated from the current end of
136
+ # the buffer (`-1` means the read data will be appended to the end of the
137
+ # buffer). If `raise_on_eof` is `true` (the default,) an `EOFError` will be
138
+ # raised on `EOF`, otherwise `nil` will be returned.
139
+ #
140
+ # If no bytes are available and `EOF` is not hit, this method will block until
141
+ # the socket is ready to read from.
142
+ #
143
+ # @param maxlen [Integer, nil] maximum bytes to read from socket
144
+ # @param buf [String, nil] buffer to read into
145
+ # @param buf_pos [Number] buffer position to read into
146
+ # @param raise_on_eof [bool] whether to raise an exception on `EOF`
147
+ # @return [String, nil] buffer used for reading or nil on `EOF`
148
+ def readpartial(maxlen, buf = +'', buf_pos = 0, raise_on_eof = true)
149
+ if buf_pos != 0
93
150
  if (result = sysread(maxlen, +''))
94
- if buffer_pos == -1
151
+ if buf_pos == -1
95
152
  result = buf + result
96
153
  else
97
- result = buf[0...buffer_pos] + result
154
+ result = buf[0...buf_pos] + result
98
155
  end
99
156
  end
100
157
  else
@@ -105,6 +162,18 @@ class ::OpenSSL::SSL::SSLSocket
105
162
  result
106
163
  end
107
164
 
165
+ # call-seq:
166
+ # socket.recv_loop { |data| ... }
167
+ # socket.recv_loop(maxlen) { |data| ... }
168
+ # socket.read_loop { |data| ... }
169
+ # socket.read_loop(maxlen) { |data| ... }
170
+ #
171
+ # Receives up to `maxlen` bytes at a time in an infinite loop. Read buffers
172
+ # will be passed to the given block.
173
+ #
174
+ # @param maxlen [Integer] maximum bytes to receive
175
+ # @yield [String] handler block
176
+ # @return [void]
108
177
  def read_loop(maxlen = 8192)
109
178
  while (data = sysread(maxlen))
110
179
  yield data
@@ -112,7 +181,10 @@ class ::OpenSSL::SSL::SSLSocket
112
181
  end
113
182
  alias_method :recv_loop, :read_loop
114
183
 
184
+ # @!visibility private
115
185
  alias_method :orig_peeraddr, :peeraddr
186
+
187
+ # @!visibility private
116
188
  def peeraddr(_ = nil)
117
189
  orig_peeraddr
118
190
  end
@@ -122,7 +194,12 @@ end
122
194
  class ::OpenSSL::SSL::SSLServer
123
195
  attr_reader :ctx
124
196
 
197
+ # @!visibility private
125
198
  alias_method :orig_accept, :accept
199
+
200
+ # Accepts a new connection and performs SSL handshake.
201
+ #
202
+ # @return [OpenSSL::SSL::SSLSocket] accepted SSL connection
126
203
  def accept
127
204
  # when @ctx.servername_cb is set, we use a worker thread to run the
128
205
  # ssl.accept call. We need to do this because:
@@ -165,6 +242,7 @@ class ::OpenSSL::SSL::SSLServer
165
242
  end
166
243
  end
167
244
 
245
+ # @!visibility private
168
246
  def start_accept_worker_thread
169
247
  fiber = Fiber.current
170
248
  @accept_worker_thread = Thread.new do
@@ -187,16 +265,27 @@ class ::OpenSSL::SSL::SSLServer
187
265
  @accept_worker_fiber = receive
188
266
  end
189
267
 
268
+ # @!visibility private
190
269
  def use_accept_worker_thread?
191
270
  !!@ctx.servername_cb
192
271
  end
193
272
 
273
+ # @!visibility private
194
274
  alias_method :orig_close, :close
275
+
276
+ # @!visibility private
195
277
  def close
196
278
  @accept_worker_thread&.kill
197
279
  orig_close
198
280
  end
199
281
 
282
+ # call-seq:
283
+ # socket.accept_loop { |conn| ... }
284
+ #
285
+ # Accepts incoming connections in an infinite loop.
286
+ #
287
+ # @yield [OpenSSL::SSL::SSLSocket] block receiving accepted sockets
288
+ # @return [void]
200
289
  def accept_loop(ignore_errors = true)
201
290
  loop do
202
291
  yield accept
@@ -1,20 +1,32 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Pipe instance methods
3
+ # A Pipe instance represents a UNIX pipe that can be read and written to. This
4
+ # API is an alternative to the `IO.pipe` API, that returns two separate fds, one
5
+ # for reading and one for writing. Instead, `Polyphony::Pipe` encapsulates the
6
+ # two fds in a single object, providing methods that enable us to treat the pipe
7
+ # as a normal IO object.
4
8
  class Polyphony::Pipe
9
+ # @!visibility private
5
10
  def __read_method__
6
11
  :backend_read
7
12
  end
8
13
 
14
+ # @!visibility private
9
15
  def __write_method__
10
16
  :backend_write
11
17
  end
12
18
 
19
+ # Reads a single byte from the pipe.
20
+ #
21
+ # @return [Integer, nil] byte value
13
22
  def getbyte
14
23
  char = getc
15
24
  char ? char.getbyte(0) : nil
16
25
  end
17
26
 
27
+ # Reads a single character from the pipe.
28
+ #
29
+ # @return |String, nil] read character
18
30
  def getc
19
31
  return @read_buffer.slice!(0) if @read_buffer && !@read_buffer.empty?
20
32
 
@@ -25,6 +37,12 @@ class Polyphony::Pipe
25
37
  nil
26
38
  end
27
39
 
40
+ # Reads from the pipe.
41
+ #
42
+ # @param len [Integer, nil] maximum bytes to read
43
+ # @param buf [String, nil] buffer to read into
44
+ # @param buf_pos [Integer] buffer position to read into
45
+ # @return [String] read data
28
46
  def read(len = nil, buf = nil, buf_pos = 0)
29
47
  if buf
30
48
  return Polyphony.backend_read(self, buf, len, true, buf_pos)
@@ -39,22 +57,48 @@ class Polyphony::Pipe
39
57
  already_read
40
58
  end
41
59
 
42
- def readpartial(len, str = +'', buffer_pos = 0, raise_on_eof = true)
43
- result = Polyphony.backend_read(self, str, len, false, buffer_pos)
60
+ # Reads from the pipe.
61
+ #
62
+ # @param len [Integer, nil] maximum bytes to read
63
+ # @param buf [String, nil] buffer to read into
64
+ # @param buf_pos [Integer] buffer position to read into
65
+ # @param raise_on_eof [boolean] whether to raise an error if EOF is detected
66
+ # @return [String] read data
67
+ def readpartial(len, buf = +'', buf_pos = 0, raise_on_eof = true)
68
+ result = Polyphony.backend_read(self, buf, len, false, buf_pos)
44
69
  raise EOFError if !result && raise_on_eof
45
70
 
46
71
  result
47
72
  end
48
73
 
49
- def write(str, *args)
50
- Polyphony.backend_write(self, str, *args)
74
+ # Writes to the pipe.
75
+
76
+ # @param buf [String] data to write
77
+ # @param args [any] further arguments to pass to Polyphony.backend_write
78
+ # @return [Integer] bytes written
79
+ def write(buf, *args)
80
+ Polyphony.backend_write(self, buf, *args)
51
81
  end
52
82
 
53
- def <<(str)
54
- Polyphony.backend_write(self, str)
83
+ # Writes to the pipe.
84
+
85
+ # @param buf [String] data to write
86
+ # @return [Integer] bytes written
87
+ def <<(buf)
88
+ Polyphony.backend_write(self, buf)
55
89
  self
56
90
  end
57
91
 
92
+ # call-seq:
93
+ # pipe.gets(limit, chomp)
94
+ # pipe.gets(separator, limit, chomp)
95
+ #
96
+ # Reads a single line from the pipe.
97
+ #
98
+ # @param sep [String] line separator
99
+ # @param _limit [Integer, nil] line length limit
100
+ # @param _chomp [boolean, nil] whether to chomp the read line
101
+ # @return [String, nil] read line
58
102
  def gets(sep = $/, _limit = nil, _chomp: nil)
59
103
  if sep.is_a?(Integer)
60
104
  sep = $/
@@ -84,9 +128,14 @@ class Polyphony::Pipe
84
128
  # def putc(obj)
85
129
  # end
86
130
 
131
+ # @!visibility private
87
132
  LINEFEED = "\n"
133
+ # @!visibility private
88
134
  LINEFEED_RE = /\n$/.freeze
89
135
 
136
+ # Writes a line with line feed to the pipe.
137
+ #
138
+ # @param args [Array] zero or more lines
90
139
  def puts(*args)
91
140
  if args.empty?
92
141
  write LINEFEED
@@ -121,22 +170,55 @@ class Polyphony::Pipe
121
170
  # def readlines(sep = $/, limit = nil, chomp: nil)
122
171
  # end
123
172
 
173
+ # @!visibility private
124
174
  def write_nonblock(string, _options = {})
125
175
  write(string)
126
176
  end
127
177
 
178
+ # @!visibility private
128
179
  def read_nonblock(maxlen, buf = nil, _options = nil)
129
180
  buf ? readpartial(maxlen, buf) : readpartial(maxlen)
130
181
  end
131
182
 
183
+ # Runs a read loop.
184
+ #
185
+ # @param maxlen [Integer] maximum bytes to read
186
+ # @yield [String] read block
187
+ # @return [void]
132
188
  def read_loop(maxlen = 8192, &block)
133
189
  Polyphony.backend_read_loop(self, maxlen, &block)
134
190
  end
135
191
 
192
+ # call-seq:
193
+ # pipe.feed_loop(receiver, method)
194
+ # pipe.feed_loop(receiver, method) { |result| ... }
195
+ #
196
+ # Receives data from the pipe in an infinite loop, passing the data to the
197
+ # given receiver using the given method. If a block is given, the result of
198
+ # the method call to the receiver is passed to the block.
199
+ #
200
+ # This method can be used to feed data into parser objects. The following
201
+ # example shows how to feed data from a pipe directly into a MessagePack
202
+ # unpacker:
203
+ #
204
+ # unpacker = MessagePack::Unpacker.new
205
+ # buffer = []
206
+ # reader = spin do
207
+ # pipe.feed_loop(unpacker, :feed_each) { |msg| handle_msg(msg) }
208
+ # end
209
+ #
210
+ # @param receiver [any] receiver object
211
+ # @param method [Symbol] method to call
212
+ # @yield [any] block to handle result of method call to receiver
213
+ # @return [void]
136
214
  def feed_loop(receiver, method = :call, &block)
137
215
  Polyphony.backend_feed_loop(self, receiver, method, &block)
138
216
  end
139
217
 
218
+ # Waits for pipe to become readable.
219
+ #
220
+ # @param timeout [Number, nil] optional timeout in seconds
221
+ # @return [Polyphony::Pipe] self
140
222
  def wait_readable(timeout = nil)
141
223
  if timeout
142
224
  move_on_after(timeout) do
@@ -149,6 +231,10 @@ class Polyphony::Pipe
149
231
  end
150
232
  end
151
233
 
234
+ # Waits for pipe to become writeable.
235
+ #
236
+ # @param timeout [Number, nil] optional timeout in seconds
237
+ # @return [Polyphony::Pipe] self
152
238
  def wait_writable(timeout = nil)
153
239
  if timeout
154
240
  move_on_after(timeout) do
@@ -161,11 +247,21 @@ class Polyphony::Pipe
161
247
  end
162
248
  end
163
249
 
250
+ # Splices to the pipe from the given source.
251
+ #
252
+ # @param src [IO] source to splice from
253
+ # @param maxlen [Integer] maximum bytes to splice
254
+ # @return [Integer] bytes spliced
164
255
  def splice_from(src, maxlen)
165
256
  Polyphony.backend_splice(src, self, maxlen)
166
257
  end
167
258
 
168
259
  if RUBY_PLATFORM =~ /linux/
260
+ # Tees to the pipe from the given source.
261
+ #
262
+ # @param src [IO] source to tee from
263
+ # @param maxlen [Integer] maximum bytes to tee
264
+ # @return [Integer] bytes teed
169
265
  def tee_from(src, maxlen)
170
266
  Polyphony.backend_tee(src, self, maxlen)
171
267
  end
@@ -1,16 +1,28 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # Overrides for Process
3
+ # Overrides for Process module
4
4
  module ::Process
5
5
  class << self
6
+ # @!visibility private
6
7
  alias_method :orig_detach, :detach
8
+
9
+ # Detaches the given pid and returns a fiber waiting on it.
10
+ #
11
+ # @param pid [Integer] child pid
12
+ # @return [Fiber] new fiber waiting on pid
7
13
  def detach(pid)
8
14
  fiber = spin { Polyphony.backend_waitpid(pid) }
9
15
  fiber.define_singleton_method(:pid) { pid }
10
16
  fiber
11
17
  end
12
18
 
19
+ # @!visibility private
13
20
  alias_method :orig_daemon, :daemon
21
+
22
+ # Starts a daemon with the given arguments.
23
+ #
24
+ # @param args [any] arguments to pass to daemon
25
+ # @return [Integer] daemon pid
14
26
  def daemon(*args)
15
27
  orig_daemon(*args)
16
28
  Polyphony.original_pid = Process.pid