polyphony 0.40 → 0.41

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +11 -2
  3. data/.gitignore +2 -2
  4. data/.rubocop.yml +30 -0
  5. data/CHANGELOG.md +6 -2
  6. data/Gemfile.lock +9 -6
  7. data/Rakefile +2 -2
  8. data/TODO.md +18 -97
  9. data/docs/_includes/head.html +40 -0
  10. data/docs/_includes/nav.html +5 -5
  11. data/docs/api-reference/fiber.md +2 -2
  12. data/docs/main-concepts/design-principles.md +67 -9
  13. data/docs/main-concepts/extending.md +1 -1
  14. data/examples/core/xx-agent.rb +102 -0
  15. data/examples/core/xx-sleeping.rb +14 -6
  16. data/examples/io/xx-irb.rb +1 -1
  17. data/examples/performance/thread-vs-fiber/polyphony_mt_server.rb +7 -6
  18. data/examples/performance/thread-vs-fiber/polyphony_server.rb +14 -25
  19. data/ext/{gyro → polyphony}/extconf.rb +2 -2
  20. data/ext/{gyro → polyphony}/fiber.c +15 -19
  21. data/ext/{gyro → polyphony}/libev.c +0 -0
  22. data/ext/{gyro → polyphony}/libev.h +0 -0
  23. data/ext/polyphony/libev_agent.c +503 -0
  24. data/ext/polyphony/libev_queue.c +214 -0
  25. data/ext/{gyro/gyro.c → polyphony/polyphony.c} +16 -25
  26. data/ext/polyphony/polyphony.h +90 -0
  27. data/ext/polyphony/polyphony_ext.c +23 -0
  28. data/ext/{gyro → polyphony}/socket.c +14 -14
  29. data/ext/{gyro → polyphony}/thread.c +32 -115
  30. data/ext/{gyro → polyphony}/tracing.c +1 -1
  31. data/lib/polyphony.rb +16 -12
  32. data/lib/polyphony/adapters/irb.rb +1 -1
  33. data/lib/polyphony/adapters/postgres.rb +6 -5
  34. data/lib/polyphony/adapters/process.rb +5 -5
  35. data/lib/polyphony/adapters/trace.rb +28 -28
  36. data/lib/polyphony/core/channel.rb +3 -3
  37. data/lib/polyphony/core/exceptions.rb +1 -1
  38. data/lib/polyphony/core/global_api.rb +11 -9
  39. data/lib/polyphony/core/resource_pool.rb +3 -3
  40. data/lib/polyphony/core/sync.rb +2 -2
  41. data/lib/polyphony/core/thread_pool.rb +6 -6
  42. data/lib/polyphony/core/throttler.rb +13 -6
  43. data/lib/polyphony/event.rb +27 -0
  44. data/lib/polyphony/extensions/core.rb +20 -11
  45. data/lib/polyphony/extensions/fiber.rb +4 -4
  46. data/lib/polyphony/extensions/io.rb +56 -26
  47. data/lib/polyphony/extensions/openssl.rb +4 -8
  48. data/lib/polyphony/extensions/socket.rb +27 -9
  49. data/lib/polyphony/extensions/thread.rb +16 -9
  50. data/lib/polyphony/net.rb +9 -9
  51. data/lib/polyphony/version.rb +1 -1
  52. data/polyphony.gemspec +2 -2
  53. data/test/helper.rb +12 -1
  54. data/test/test_agent.rb +77 -0
  55. data/test/{test_async.rb → test_event.rb} +13 -7
  56. data/test/test_ext.rb +25 -4
  57. data/test/test_fiber.rb +19 -10
  58. data/test/test_global_api.rb +4 -4
  59. data/test/test_io.rb +46 -24
  60. data/test/test_queue.rb +74 -0
  61. data/test/test_signal.rb +3 -40
  62. data/test/test_socket.rb +33 -0
  63. data/test/test_thread.rb +37 -16
  64. data/test/test_trace.rb +6 -5
  65. metadata +24 -24
  66. data/ext/gyro/async.c +0 -132
  67. data/ext/gyro/child.c +0 -108
  68. data/ext/gyro/gyro.h +0 -158
  69. data/ext/gyro/gyro_ext.c +0 -33
  70. data/ext/gyro/io.c +0 -457
  71. data/ext/gyro/queue.c +0 -146
  72. data/ext/gyro/selector.c +0 -205
  73. data/ext/gyro/signal.c +0 -99
  74. data/ext/gyro/timer.c +0 -115
  75. data/test/test_timer.rb +0 -56
@@ -26,7 +26,7 @@ module Polyphony
26
26
  end
27
27
 
28
28
  def receive
29
- Gyro.ref
29
+ Polyphony.ref
30
30
  if @payload_queue.empty?
31
31
  @waiting_queue << Fiber.current
32
32
  suspend
@@ -34,7 +34,7 @@ module Polyphony
34
34
  receive_from_queue
35
35
  end
36
36
  ensure
37
- Gyro.unref
37
+ Polyphony.unref
38
38
  end
39
39
 
40
40
  def receive_from_queue
@@ -43,4 +43,4 @@ module Polyphony
43
43
  payload
44
44
  end
45
45
  end
46
- end
46
+ end
@@ -33,4 +33,4 @@ module Polyphony
33
33
 
34
34
  # Restart is used to restart a fiber
35
35
  class Restart < BaseException; end
36
- end
36
+ end
@@ -45,13 +45,16 @@ module Polyphony
45
45
  end
46
46
 
47
47
  def every(interval)
48
- timer = Gyro::Timer.new(interval, interval)
48
+ next_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + interval
49
49
  loop do
50
- timer.await
50
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
51
+ Thread.current.agent.sleep(next_time - now)
51
52
  yield
53
+ loop do
54
+ next_time += interval
55
+ break if next_time > now
56
+ end
52
57
  end
53
- ensure
54
- timer.stop
55
58
  end
56
59
 
57
60
  def move_on_after(interval, with_value: nil, &block)
@@ -93,8 +96,7 @@ module Polyphony
93
96
  def sleep(duration = nil)
94
97
  return sleep_forever unless duration
95
98
 
96
- timer = Gyro::Timer.new(duration, 0)
97
- timer.await
99
+ Thread.current.agent.sleep duration
98
100
  end
99
101
 
100
102
  def sleep_forever
@@ -107,14 +109,14 @@ module Polyphony
107
109
  def throttled_loop(rate, count: nil, &block)
108
110
  throttler = Polyphony::Throttler.new(rate)
109
111
  if count
110
- count.times { throttler.(&block) }
112
+ count.times { |_i| throttler.(&block) }
111
113
  else
112
114
  loop { throttler.(&block) }
113
115
  end
114
116
  ensure
115
- throttler.stop
117
+ throttler&.stop
116
118
  end
117
119
  end
118
120
  end
119
121
 
120
- Object.include Polyphony::GlobalAPI
122
+ Object.include Polyphony::GlobalAPI
@@ -23,13 +23,13 @@ module Polyphony
23
23
  end
24
24
 
25
25
  def acquire
26
- Gyro.ref
26
+ Polyphony.ref
27
27
  resource = wait_for_resource
28
28
  return unless resource
29
29
 
30
30
  yield resource
31
31
  ensure
32
- Gyro.unref
32
+ Polyphony.unref
33
33
  release(resource) if resource
34
34
  end
35
35
 
@@ -104,4 +104,4 @@ module Polyphony
104
104
  (@limit - @size).times { @stock << allocate }
105
105
  end
106
106
  end
107
- end
107
+ end
@@ -4,7 +4,7 @@ module Polyphony
4
4
  # Implements mutex lock for synchronizing access to a shared resource
5
5
  class Mutex
6
6
  def initialize
7
- @waiting_fibers = Gyro::Queue.new
7
+ @waiting_fibers = Polyphony::Queue.new
8
8
  end
9
9
 
10
10
  def synchronize
@@ -18,4 +18,4 @@ module Polyphony
18
18
  snooze
19
19
  end
20
20
  end
21
- end
21
+ end
@@ -8,29 +8,29 @@ module Polyphony
8
8
  attr_reader :size
9
9
 
10
10
  def self.process(&block)
11
- @default_pool ||= self.new
11
+ @default_pool ||= new
12
12
  @default_pool.process(&block)
13
13
  end
14
14
 
15
15
  def self.reset
16
16
  return unless @default_pool
17
-
17
+
18
18
  @default_pool.stop
19
19
  @default_pool = nil
20
20
  end
21
21
 
22
22
  def initialize(size = Etc.nprocessors)
23
23
  @size = size
24
- @task_queue = Gyro::Queue.new
24
+ @task_queue = Polyphony::Queue.new
25
25
  @threads = (1..@size).map { Thread.new { thread_loop } }
26
26
  end
27
27
 
28
28
  def process(&block)
29
29
  setup unless @task_queue
30
30
 
31
- async = Fiber.current.auto_async
32
- @task_queue << [block, async]
33
- async.await
31
+ watcher = Fiber.current.auto_watcher
32
+ @task_queue << [block, watcher]
33
+ watcher.await
34
34
  end
35
35
 
36
36
  def cast(&block)
@@ -6,17 +6,24 @@ module Polyphony
6
6
  def initialize(rate)
7
7
  @rate = rate_from_argument(rate)
8
8
  @min_dt = 1.0 / @rate
9
+ @next_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
9
10
  end
10
11
 
11
- def call(&block)
12
- @timer ||= Gyro::Timer.new(0, @min_dt)
13
- @timer.await
14
- block.call(self)
12
+ def call
13
+ now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
14
+ delta = @next_time - now
15
+ Thread.current.agent.sleep(delta) if delta > 0
16
+ yield self
17
+
18
+ loop do
19
+ @next_time += @min_dt
20
+ break if @next_time > now
21
+ end
15
22
  end
16
23
  alias_method :process, :call
17
24
 
18
25
  def stop
19
- @timer&.stop
26
+ @stop = true
20
27
  end
21
28
 
22
29
  private
@@ -31,4 +38,4 @@ module Polyphony
31
38
  raise "Invalid rate argument #{arg.inspect}"
32
39
  end
33
40
  end
34
- end
41
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Polyphony
4
+ # Event watcher for thread-safe synchronisation
5
+ class Event
6
+ def initialize
7
+ @i, @o = IO.pipe
8
+ end
9
+
10
+ def await
11
+ Thread.current.agent.read(@i, +'', 8192, false)
12
+ raise @value if @value.is_a?(Exception)
13
+
14
+ @value
15
+ end
16
+
17
+ def await_no_raise
18
+ Thread.current.agent.read(@i, +'', 8192, false)
19
+ @value
20
+ end
21
+
22
+ def signal(value = nil)
23
+ @value = value
24
+ Thread.current.agent.write(@o, '1')
25
+ end
26
+ end
27
+ end
@@ -52,10 +52,13 @@ end
52
52
 
53
53
  # Overrides for Process
54
54
  module ::Process
55
- def self.detach(pid)
56
- fiber = spin { Gyro::Child.new(pid).await }
57
- fiber.define_singleton_method(:pid) { pid }
58
- fiber
55
+ class << self
56
+ alias_method :orig_detach, :detach
57
+ def detach(pid)
58
+ fiber = spin { Thread.current.agent.waitpid(pid) }
59
+ fiber.define_singleton_method(:pid) { pid }
60
+ fiber
61
+ end
59
62
  end
60
63
  end
61
64
 
@@ -67,10 +70,9 @@ module ::Kernel
67
70
  def `(cmd)
68
71
  Open3.popen3(cmd) do |i, o, e, _t|
69
72
  i.close
70
- while (l = e.readpartial(8192))
71
- $stderr << l
72
- end
73
- o.read
73
+ err = e.read
74
+ $stderr << err if err
75
+ o.read || ''
74
76
  end
75
77
  end
76
78
 
@@ -104,14 +106,21 @@ module ::Kernel
104
106
  def system(*args)
105
107
  Open3.popen2(*args) do |i, o, _t|
106
108
  i.close
107
- while (l = o.readpartial(8192))
108
- $stdout << l
109
- end
109
+ pipe_to_eof(o, $stdout)
110
110
  end
111
111
  true
112
112
  rescue SystemCallError
113
113
  nil
114
114
  end
115
+
116
+ def pipe_to_eof(src, dest)
117
+ loop do
118
+ data = src.readpartial(8192)
119
+ dest << data
120
+ rescue EOFError
121
+ break
122
+ end
123
+ end
115
124
  end
116
125
 
117
126
  # Override Timeout to use cancel scope
@@ -238,7 +238,7 @@ module Polyphony
238
238
  @parent = parent
239
239
  @caller = caller
240
240
  @block = block
241
- @mailbox = Gyro::Queue.new
241
+ @mailbox = Polyphony::Queue.new
242
242
  __fiber_trace__(:fiber_create, self)
243
243
  schedule
244
244
  end
@@ -268,7 +268,7 @@ module Polyphony
268
268
  # allows the fiber to be scheduled and to receive messages.
269
269
  def setup_raw
270
270
  @thread = Thread.current
271
- @mailbox = Gyro::Queue.new
271
+ @mailbox = Polyphony::Queue.new
272
272
  end
273
273
 
274
274
  def setup_main_fiber
@@ -277,11 +277,11 @@ module Polyphony
277
277
  @thread = Thread.current
278
278
  @running = true
279
279
  @children&.clear
280
- @mailbox = Gyro::Queue.new
280
+ @mailbox = Polyphony::Queue.new
281
281
  end
282
282
 
283
283
  def restart_self(first_value)
284
- @mailbox = Gyro::Queue.new
284
+ @mailbox = Polyphony::Queue.new
285
285
  @when_done_procs = nil
286
286
  @waiting_fibers = nil
287
287
  run(first_value)
@@ -2,7 +2,7 @@
2
2
 
3
3
  require 'open3'
4
4
 
5
- # IO overrides
5
+ # IO class method patches
6
6
  class ::IO
7
7
  class << self
8
8
  alias_method :orig_binread, :binread
@@ -72,7 +72,10 @@ class ::IO
72
72
  Open3.popen2(cmd) { |_i, o, _t| yield o }
73
73
  end
74
74
  end
75
+ end
75
76
 
77
+ # IO instance method patches
78
+ class ::IO
76
79
  # def each(sep = $/, limit = nil, chomp: nil)
77
80
  # sep, limit = $/, sep if sep.is_a?(Integer)
78
81
  # end
@@ -93,6 +96,39 @@ class ::IO
93
96
  # def getc
94
97
  # end
95
98
 
99
+ alias_method :orig_read, :read
100
+ def read(len = 1 << 30)
101
+ @read_buffer ||= +''
102
+ result = Thread.current.agent.read(self, @read_buffer, len, true)
103
+ return nil unless result
104
+
105
+ already_read = @read_buffer
106
+ @read_buffer = +''
107
+ already_read
108
+ end
109
+
110
+ alias_method :orig_readpartial, :read
111
+ def readpartial(len)
112
+ @read_buffer ||= +''
113
+ result = Thread.current.agent.read(self, @read_buffer, len, false)
114
+ raise EOFError unless result
115
+
116
+ already_read = @read_buffer
117
+ @read_buffer = +''
118
+ already_read
119
+ end
120
+
121
+ alias_method :orig_write, :write
122
+ def write(str)
123
+ Thread.current.agent.write(self, str)
124
+ end
125
+
126
+ alias_method :orig_write_chevron, :<<
127
+ def <<(str)
128
+ Thread.current.agent.write(self, str)
129
+ self
130
+ end
131
+
96
132
  alias_method :orig_gets, :gets
97
133
  def gets(sep = $/, _limit = nil, _chomp: nil)
98
134
  if sep.is_a?(Integer)
@@ -101,23 +137,17 @@ class ::IO
101
137
  end
102
138
  sep_size = sep.bytesize
103
139
 
104
- @gets_buffer ||= +''
140
+ @read_buffer ||= +''
105
141
 
106
142
  loop do
107
- idx = @gets_buffer.index(sep)
108
- return @gets_buffer.slice!(0, idx + sep_size) if idx
109
-
110
- if (data = readpartial(8192))
111
- @gets_buffer << data
112
- else
113
- return nil if @gets_buffer.empty?
143
+ idx = @read_buffer.index(sep)
144
+ return @read_buffer.slice!(0, idx + sep_size) if idx
114
145
 
115
- line = @gets_buffer.freeze
116
- @gets_buffer = +''
117
- return line
118
- end
146
+ data = readpartial(8192)
147
+ @read_buffer << data
148
+ rescue EOFError
149
+ return nil
119
150
  end
120
- # orig_gets(sep, limit, chomp: chomp)
121
151
  end
122
152
 
123
153
  # def print(*args)
@@ -171,16 +201,16 @@ class ::IO
171
201
  buf ? readpartial(maxlen, buf) : readpartial(maxlen)
172
202
  end
173
203
 
174
- alias_method :orig_read, :read
175
- def read(length = nil, outbuf = nil)
176
- if length
177
- return outbuf ? readpartial(length) : readpartial(length, outbuf)
178
- end
179
-
180
- until eof?
181
- outbuf ||= +''
182
- outbuf << readpartial(8192)
183
- end
184
- outbuf
185
- end
204
+ # alias_method :orig_read, :read
205
+ # def read(length = nil, outbuf = nil)
206
+ # if length
207
+ # return outbuf ? readpartial(length) : readpartial(length, outbuf)
208
+ # end
209
+
210
+ # until eof?
211
+ # outbuf ||= +''
212
+ # outbuf << readpartial(8192)
213
+ # end
214
+ # outbuf
215
+ # end
186
216
  end
@@ -19,12 +19,10 @@ class ::OpenSSL::SSL::SSLSocket
19
19
  end
20
20
 
21
21
  def sysread(maxlen, buf)
22
- read_watcher = nil
23
- write_watcher = nil
24
22
  loop do
25
23
  case (result = read_nonblock(maxlen, buf, exception: false))
26
- when :wait_readable then (read_watcher ||= io.read_watcher).await
27
- when :wait_writable then (write_watcher ||= io.write_watcher).await
24
+ when :wait_readable then Thread.current.agent.wait_io(io, false)
25
+ when :wait_writable then Thread.current.agent.wait_io(io, true)
28
26
  else result
29
27
  end
30
28
  end
@@ -40,12 +38,10 @@ class ::OpenSSL::SSL::SSLSocket
40
38
  end
41
39
 
42
40
  def syswrite(buf)
43
- read_watcher = nil
44
- write_watcher = nil
45
41
  loop do
46
42
  case (result = write_nonblock(buf, exception: false))
47
- when :wait_readable then (read_watcher ||= io.read_watcher).await
48
- when :wait_writable then (write_watcher ||= io.write_watcher).await
43
+ when :wait_readable then Thread.current.agent.wait_io(io, false)
44
+ when :wait_writable then Thread.current.agent.wait_io(io, true)
49
45
  else result
50
46
  end
51
47
  end