polyphony 0.43.5 → 0.43.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +45 -0
  3. data/Gemfile.lock +1 -1
  4. data/README.md +21 -4
  5. data/TODO.md +0 -7
  6. data/bin/stress.rb +28 -0
  7. data/docs/_includes/head.html +40 -0
  8. data/docs/_includes/title.html +1 -0
  9. data/docs/_user-guide/web-server.md +11 -11
  10. data/docs/getting-started/overview.md +2 -2
  11. data/docs/index.md +3 -1
  12. data/docs/polyphony-logo.png +0 -0
  13. data/examples/core/xx-channels.rb +4 -2
  14. data/examples/core/xx-using-a-mutex.rb +2 -1
  15. data/examples/io/xx-happy-eyeballs.rb +21 -22
  16. data/examples/io/xx-zip.rb +19 -0
  17. data/examples/performance/fiber_transfer.rb +47 -0
  18. data/examples/xx-spin.rb +32 -0
  19. data/ext/polyphony/agent.h +41 -0
  20. data/ext/polyphony/event.c +86 -0
  21. data/ext/polyphony/fiber.c +0 -5
  22. data/ext/polyphony/libev_agent.c +277 -135
  23. data/ext/polyphony/polyphony.c +2 -2
  24. data/ext/polyphony/polyphony.h +14 -21
  25. data/ext/polyphony/polyphony_ext.c +4 -2
  26. data/ext/polyphony/queue.c +208 -0
  27. data/ext/polyphony/ring_buffer.c +0 -24
  28. data/ext/polyphony/thread.c +42 -31
  29. data/lib/polyphony.rb +6 -7
  30. data/lib/polyphony/core/channel.rb +3 -34
  31. data/lib/polyphony/core/resource_pool.rb +13 -75
  32. data/lib/polyphony/core/sync.rb +12 -9
  33. data/lib/polyphony/extensions/fiber.rb +8 -8
  34. data/lib/polyphony/extensions/openssl.rb +8 -0
  35. data/lib/polyphony/extensions/socket.rb +11 -9
  36. data/lib/polyphony/extensions/thread.rb +1 -1
  37. data/lib/polyphony/net.rb +2 -1
  38. data/lib/polyphony/version.rb +1 -1
  39. data/test/helper.rb +2 -2
  40. data/test/test_agent.rb +2 -2
  41. data/test/test_event.rb +12 -0
  42. data/test/test_fiber.rb +1 -1
  43. data/test/test_io.rb +14 -0
  44. data/test/test_queue.rb +33 -0
  45. data/test/test_resource_pool.rb +24 -58
  46. data/test/test_trace.rb +18 -17
  47. metadata +12 -5
  48. data/ext/polyphony/libev_queue.c +0 -288
  49. data/lib/polyphony/event.rb +0 -27
@@ -4,9 +4,6 @@ require 'fiber'
4
4
  require_relative './polyphony_ext'
5
5
 
6
6
  module Polyphony
7
- # Map Queue to Libev queue implementation
8
- Queue = LibevQueue
9
-
10
7
  # replace core Queue class with our own
11
8
  verbose = $VERBOSE
12
9
  $VERBOSE = nil
@@ -20,13 +17,12 @@ require_relative './polyphony/extensions/fiber'
20
17
  require_relative './polyphony/extensions/io'
21
18
 
22
19
  Thread.current.setup_fiber_scheduling
23
- Thread.current.agent = Polyphony::LibevAgent.new
20
+ Thread.current.agent = Polyphony::Agent.new
24
21
 
25
22
  require_relative './polyphony/core/global_api'
26
23
  require_relative './polyphony/core/resource_pool'
27
24
  require_relative './polyphony/net'
28
25
  require_relative './polyphony/adapters/process'
29
- require_relative './polyphony/event'
30
26
 
31
27
  # Main Polyphony API
32
28
  module Polyphony
@@ -61,7 +57,7 @@ module Polyphony
61
57
  rescue SystemExit
62
58
  # fall through to ensure
63
59
  rescue Exception => e
64
- e.full_message
60
+ warn e.full_message
65
61
  exit!
66
62
  ensure
67
63
  exit_forked_process
@@ -102,7 +98,10 @@ module Polyphony
102
98
 
103
99
  def install_terminating_signal_handlers
104
100
  trap('SIGTERM', SystemExit)
105
- trap('SIGINT', Interrupt)
101
+ orig_trap('SIGINT') do
102
+ orig_trap('SIGINT') { exit! }
103
+ Thread.current.break_out_of_ev_loop(Thread.main.main_fiber, Interrupt.new)
104
+ end
106
105
  end
107
106
 
108
107
  def terminate_threads
@@ -5,42 +5,11 @@ require_relative './exceptions'
5
5
  module Polyphony
6
6
  # Implements a unidirectional communication channel along the lines of Go
7
7
  # (buffered) channels.
8
- class Channel
9
- def initialize
10
- @payload_queue = []
11
- @waiting_queue = []
12
- end
8
+ class Channel < Polyphony::Queue
9
+ alias_method :receive, :shift
13
10
 
14
11
  def close
15
- stop = Polyphony::MoveOn.new
16
- @waiting_queue.slice(0..-1).each { |f| f.schedule(stop) }
17
- end
18
-
19
- def <<(value)
20
- if @waiting_queue.empty?
21
- @payload_queue << value
22
- else
23
- @waiting_queue.shift&.schedule(value)
24
- end
25
- snooze
26
- end
27
-
28
- def receive
29
- Thread.current.agent.ref
30
- if @payload_queue.empty?
31
- @waiting_queue << Fiber.current
32
- suspend
33
- else
34
- receive_from_queue
35
- end
36
- ensure
37
- Thread.current.agent.unref
38
- end
39
-
40
- def receive_from_queue
41
- payload = @payload_queue.shift
42
- snooze
43
- payload
12
+ flush_waiters(Polyphony::MoveOn.new)
44
13
  end
45
14
  end
46
15
  end
@@ -10,13 +10,10 @@ module Polyphony
10
10
  # @param &block [Proc] allocator block
11
11
  def initialize(opts, &block)
12
12
  @allocator = block
13
-
14
- @stock = []
15
- @queue = []
16
- @acquired_resources = {}
17
-
18
13
  @limit = opts[:limit] || 4
19
14
  @size = 0
15
+ @stock = Polyphony::Queue.new
16
+ @acquired_resources = {}
20
17
  end
21
18
 
22
19
  def available
@@ -25,58 +22,17 @@ module Polyphony
25
22
 
26
23
  def acquire
27
24
  fiber = Fiber.current
28
- if @acquired_resources[fiber]
29
- yield @acquired_resources[fiber]
30
- else
31
- begin
32
- Thread.current.agent.ref
33
- resource = wait_for_resource
34
- return unless resource
35
-
36
- @acquired_resources[fiber] = resource
37
- yield resource
38
- ensure
39
- @acquired_resources[fiber] = nil
40
- Thread.current.agent.unref
41
- release(resource) if resource
42
- end
43
- end
44
- end
45
-
46
- def wait_for_resource
47
- fiber = Fiber.current
48
- @queue << fiber
49
- ready_resource = from_stock
50
- return ready_resource if ready_resource
25
+ return @acquired_resources[fiber] if @acquired_resources[fiber]
51
26
 
52
- suspend
27
+ add_to_stock if (@stock.empty? || @stock.pending?) && @size < @limit
28
+ resource = @stock.shift
29
+ @acquired_resources[fiber] = resource
30
+ yield resource
53
31
  ensure
54
- @queue.delete(fiber)
55
- end
56
-
57
- def release(resource)
58
- if resource.__discarded__
59
- @size -= 1
60
- elsif resource
61
- return_to_stock(resource)
62
- dequeue
63
- end
64
- end
65
-
66
- def dequeue
67
- return if @queue.empty? || @stock.empty?
68
-
69
- @queue.shift.schedule(@stock.shift)
32
+ @acquired_resources.delete(fiber)
33
+ @stock.push resource if resource
70
34
  end
71
-
72
- def return_to_stock(resource)
73
- @stock << resource
74
- end
75
-
76
- def from_stock
77
- @stock.shift || (@size < @limit && allocate)
78
- end
79
-
35
+
80
36
  def method_missing(sym, *args, &block)
81
37
  acquire { |r| r.send(sym, *args, &block) }
82
38
  end
@@ -85,33 +41,15 @@ module Polyphony
85
41
  true
86
42
  end
87
43
 
88
- # Extension to allow discarding of resources
89
- module ResourceExtensions
90
- def __discarded__
91
- @__discarded__
92
- end
93
-
94
- def __discard__
95
- @__discarded__ = true
96
- end
97
- end
98
-
99
44
  # Allocates a resource
100
45
  # @return [any] allocated resource
101
- def allocate
102
- @size += 1
103
- @allocator.().tap { |r| r.extend ResourceExtensions }
104
- end
105
-
106
- def <<(resource)
46
+ def add_to_stock
107
47
  @size += 1
108
- resource.extend ResourceExtensions
109
- @stock << resource
110
- dequeue
48
+ @stock << @allocator.call
111
49
  end
112
50
 
113
51
  def preheat!
114
- (@limit - @size).times { @stock << allocate }
52
+ add_to_stock while @size < @limit
115
53
  end
116
54
  end
117
55
  end
@@ -4,18 +4,21 @@ module Polyphony
4
4
  # Implements mutex lock for synchronizing access to a shared resource
5
5
  class Mutex
6
6
  def initialize
7
- @waiting_fibers = Polyphony::Queue.new
7
+ @store = Queue.new
8
+ @store << :token
8
9
  end
9
10
 
10
11
  def synchronize
11
- fiber = Fiber.current
12
- @waiting_fibers << fiber
13
- suspend if @waiting_fibers.size > 1
14
- yield
15
- ensure
16
- @waiting_fibers.delete(fiber)
17
- @waiting_fibers.first&.schedule
18
- snooze
12
+ return yield if @holding_fiber == Fiber.current
13
+
14
+ begin
15
+ token = @store.shift
16
+ @holding_fiber = Fiber.current
17
+ yield
18
+ ensure
19
+ @holding_fiber = nil
20
+ @store << token if token
21
+ end
19
22
  end
20
23
  end
21
24
  end
@@ -221,14 +221,14 @@ module Polyphony
221
221
  def await_all_children
222
222
  return unless @children && !@children.empty?
223
223
 
224
- # @results = @children.dup
225
- # @on_child_done = proc do |c, r|
226
- # @results[c] = r
227
- # self.schedule if @children.empty?
228
- # end
229
- # suspend
230
- # @results.values
231
- Fiber.await(*@children.keys)
224
+ @results = @children.dup
225
+ @on_child_done = proc do |c, r|
226
+ @results[c] = r
227
+ self.schedule if @children.empty?
228
+ end
229
+ suspend
230
+ @on_child_done = nil
231
+ @results.values
232
232
  end
233
233
 
234
234
  def shutdown_all_children
@@ -5,6 +5,12 @@ require_relative './socket'
5
5
 
6
6
  # Open ssl socket helper methods (to make it compatible with Socket API)
7
7
  class ::OpenSSL::SSL::SSLSocket
8
+ alias_method :orig_initialize, :initialize
9
+ def initialize(socket, context = nil)
10
+ socket = socket.respond_to?(:io) ? socket.io || socket : socket
11
+ context ? orig_initialize(socket, context) : orig_initialize(socket)
12
+ end
13
+
8
14
  def dont_linger
9
15
  io.dont_linger
10
16
  end
@@ -35,6 +41,7 @@ class ::OpenSSL::SSL::SSLSocket
35
41
  loop do
36
42
  case (result = read_nonblock(maxlen, buf, exception: false))
37
43
  when :wait_readable then Thread.current.agent.wait_io(io, false)
44
+ when :wait_writable then Thread.current.agent.wait_io(io, true)
38
45
  else return result
39
46
  end
40
47
  end
@@ -44,6 +51,7 @@ class ::OpenSSL::SSL::SSLSocket
44
51
  def syswrite(buf)
45
52
  loop do
46
53
  case (result = write_nonblock(buf, exception: false))
54
+ when :wait_readable then Thread.current.agent.wait_io(io, false)
47
55
  when :wait_writable then Thread.current.agent.wait_io(io, true)
48
56
  else
49
57
  return result
@@ -24,15 +24,7 @@ class ::Socket
24
24
  NO_EXCEPTION = { exception: false }.freeze
25
25
 
26
26
  def connect(remotesockaddr)
27
- loop do
28
- result = connect_nonblock(remotesockaddr, **NO_EXCEPTION)
29
- case result
30
- when 0 then return
31
- when :wait_writable then Thread.current.agent.wait_io(self, true)
32
- else
33
- raise IOError
34
- end
35
- end
27
+ Thread.current.agent.connect(self, remotesockaddr.ip_address, remotesockaddr.ip_port)
36
28
  end
37
29
 
38
30
  def recv(maxlen, flags = 0, outbuf = nil)
@@ -75,6 +67,10 @@ class ::Socket
75
67
  setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
76
68
  end
77
69
 
70
+ def reuse_port
71
+ setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
72
+ end
73
+
78
74
  class << self
79
75
  alias_method :orig_getaddrinfo, :getaddrinfo
80
76
  def getaddrinfo(*args)
@@ -87,6 +83,8 @@ end
87
83
  class ::TCPSocket
88
84
  NO_EXCEPTION = { exception: false }.freeze
89
85
 
86
+ attr_reader :io
87
+
90
88
  def initialize(remote_host, remote_port, local_host = nil, local_port = nil)
91
89
  @io = Socket.new Socket::AF_INET, Socket::SOCK_STREAM
92
90
  if local_host && local_port
@@ -126,6 +124,10 @@ class ::TCPSocket
126
124
  def reuse_addr
127
125
  setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, 1)
128
126
  end
127
+
128
+ def reuse_port
129
+ setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
130
+ end
129
131
  end
130
132
 
131
133
  # Override stock TCPServer code by encapsulating a Socket instance.
@@ -18,7 +18,7 @@ class ::Thread
18
18
  def execute
19
19
  # agent must be created in the context of the new thread, therefore it
20
20
  # cannot be created in Thread#initialize
21
- @agent = Polyphony::LibevAgent.new
21
+ @agent = Polyphony::Agent.new
22
22
  setup
23
23
  @ready = true
24
24
  result = @block.(*@args)
@@ -35,9 +35,10 @@ module Polyphony
35
35
  ::Socket.new(:INET, :STREAM).tap do |s|
36
36
  s.reuse_addr if opts[:reuse_addr]
37
37
  s.dont_linger if opts[:dont_linger]
38
+ s.reuse_port if opts[:reuse_port]
38
39
  addr = ::Socket.sockaddr_in(port, host)
39
40
  s.bind(addr)
40
- s.listen(0)
41
+ s.listen(opts[:backlog] || Socket::SOMAXCONN)
41
42
  end
42
43
  end
43
44
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Polyphony
4
- VERSION = '0.43.5'
4
+ VERSION = '0.43.11'
5
5
  end
@@ -31,8 +31,8 @@ class MiniTest::Test
31
31
  end
32
32
  Fiber.current.setup_main_fiber
33
33
  Fiber.current.instance_variable_set(:@auto_watcher, nil)
34
- Thread.current.agent = Polyphony::LibevAgent.new
35
- sleep 0
34
+ Thread.current.agent = Polyphony::Agent.new
35
+ sleep 0 # apparently this helps with timer accuracy
36
36
  end
37
37
 
38
38
  def teardown
@@ -6,7 +6,7 @@ class AgentTest < MiniTest::Test
6
6
  def setup
7
7
  super
8
8
  @prev_agent = Thread.current.agent
9
- @agent = Polyphony::LibevAgent.new
9
+ @agent = Polyphony::Agent.new
10
10
  Thread.current.agent = @agent
11
11
  end
12
12
 
@@ -97,7 +97,7 @@ class AgentTest < MiniTest::Test
97
97
  o.close
98
98
 
99
99
  # read_loop will snooze after every read
100
- 4.times { snooze }
100
+ 6.times { snooze }
101
101
 
102
102
  assert_equal [:ready, 'foo', 'bar', :done], buf
103
103
  end
@@ -34,6 +34,7 @@ class EventTest < MiniTest::Test
34
34
  }
35
35
  }
36
36
  snooze
37
+
37
38
  t = Thread.new do
38
39
  orig_sleep 0.001
39
40
  3.times { a.signal }
@@ -45,4 +46,15 @@ class EventTest < MiniTest::Test
45
46
  t&.kill
46
47
  t&.join
47
48
  end
49
+
50
+ def test_exception_while_waiting_for_event
51
+ e = Polyphony::Event.new
52
+
53
+ f = spin { e.await }
54
+ g = spin { f.raise 'foo' }
55
+
56
+ assert_raises(RuntimeError) do
57
+ f.await
58
+ end
59
+ end
48
60
  end
@@ -692,7 +692,7 @@ class FiberTest < MiniTest::Test
692
692
 
693
693
  f.schedule
694
694
  f << 'bar'
695
- snooze
695
+ 2.times { snooze }
696
696
  assert_equal ['bar'], buffer
697
697
  end
698
698
  end
@@ -91,6 +91,20 @@ class IOTest < MiniTest::Test
91
91
 
92
92
  assert_raises(EOFError) { i.readpartial(1) }
93
93
  end
94
+
95
+ # see https://github.com/digital-fabric/polyphony/issues/30
96
+ def test_reopened_tempfile
97
+ file = Tempfile.new
98
+ file << 'hello: world'
99
+ file.close
100
+
101
+ buf = nil
102
+ File.open(file, 'r:bom|utf-8') do |f|
103
+ buf = f.read(16384)
104
+ end
105
+
106
+ assert_equal 'hello: world', buf
107
+ end
94
108
  end
95
109
 
96
110
  class IOClassMethodsTest < MiniTest::Test