polyphony 0.69 → 0.70

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1074fc28d6110cf489e8dce4fcc9c9a2430b557b57a23d865d4ac35c2bb14847
-  data.tar.gz: 4d0de60084fdc5eec62c25306e098a19a84e7d859547397eff70e673be2fab9b
+  metadata.gz: 65be64df5933c24a4afb38b24c139213dcc71a7fbde7689c46c862d7bb04eef3
+  data.tar.gz: 3feec6f48ea17b15468790c91a790438d163b1d905d5ef6c7a2ffd6d3e47bac8
 SHA512:
-  metadata.gz: f2aa641bceb29837bedc2b8120109b05dac20afa32923b6e20bca9a3d9d6b54886ed1410915f24497a839866ff8a660573a847868747ff1c91aa4e0ba87b45c2
-  data.tar.gz: b49b24d6ea81f8c1a95402c9024102e0f999a079566af7667ad0ae5aab9d5fef4b934ae9711e447e7a0364d533ffe9071f41ce8309be425913606f3dad1e0860
+  metadata.gz: 3417d863dcd7e99c3123fbc81923627a77c6a5b45ed20d6e84148eabb945ecb38f429873be01fc53d5c35f36a225045c42e51b2c36f543fb3539ebc7d91f8bcf
+  data.tar.gz: be39ff1ec9616998fc2a5df0bd9af368687e3424cf4b97776466e803624953440bbf75fc166cd273f8f3d4502eefee4a13283c4129c3ed02113cc2015d314428

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.70 2021-08-19
+
+- New implementation for `#supervise`
+- Reset backend state and runqueue on fork
+
 ## 0.69 2021-08-16
 
 - Rename `#__polyphony_read_method__` to `#__parser_read_method__`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.69)
+    polyphony (0.70)
 
 GEM
   remote: https://rubygems.org/

data/TODO.md

@@ -26,31 +26,8 @@
 
 - Add support for `close` to io_uring backend
 
-- Graceful shutdown again:
-  - What happens to children when doing a graceful shutdown?
-  - What are the implications of passing graceful shutdown flag to children?
-  - What about errors while doing a graceful shutdown?
-  - What about graceful restarts?
-  - Some interesting discussions:
-    - https://trio.discourse.group/search?q=graceful%20shutdown
-    - https://github.com/python-trio/trio/issues/147
-    - https://github.com/python-trio/trio/issues/143
-    - https://trio.discourse.group/t/graceful-shutdown/93/2
-    - https://250bpm.com/blog:146/
-    - https://www.rodrigoaraujo.me/posts/golang-pattern-graceful-shutdown-of-concurrent-events/
-    - https://github.com/tj/go-gracefully
-  - `Fiber#finalize_children` should pass graceful shutdown flag to children
-  - A good use case is an HTTP server that on graceful shutdown:
-    - stops listening
-    - waits for all ongoing requests to finish, optionally with a timeout
-
 ## Roadmap for Polyphony 1.0
 
-- check integration with rb-inotify
-
-- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
-  inconsistent behaviour (see supervisor example).
-
 - Add test that mimics the original design for Monocrono:
   - 256 fibers each waiting for a message
   - When message received do some blocking work using a `ThreadPool`

data/ext/polyphony/backend_common.c

@@ -31,6 +31,24 @@ inline void backend_base_mark(struct Backend_base *base) {
   runqueue_mark(&base->parked_runqueue);
 }
 
+void backend_base_reset(struct Backend_base *base) {
+  runqueue_finalize(&base->runqueue);
+  runqueue_finalize(&base->parked_runqueue);
+  
+  runqueue_initialize(&base->runqueue);
+  runqueue_initialize(&base->parked_runqueue);
+
+  base->currently_polling = 0;
+  base->op_count = 0;
+  base->switch_count = 0;
+  base->poll_count = 0;
+  base->pending_count = 0;
+  base->idle_gc_period = 0;
+  base->idle_gc_last_time = 0;
+  base->idle_proc = Qnil;
+  base->trace_proc = Qnil;  
+}
+
 const unsigned int ANTI_STARVE_SWITCH_COUNT_THRESHOLD = 64;
 
 inline void conditional_nonblocking_poll(VALUE backend, struct Backend_base *base, VALUE current, VALUE next) {

data/ext/polyphony/backend_common.h

@@ -32,6 +32,7 @@ struct Backend_base {
 void backend_base_initialize(struct Backend_base *base);
 void backend_base_finalize(struct Backend_base *base);
 void backend_base_mark(struct Backend_base *base);
+void backend_base_reset(struct Backend_base *base);
 VALUE backend_base_switch_fiber(VALUE backend, struct Backend_base *base);
 void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_base *base, VALUE fiber, VALUE value, int prioritize);
 void backend_base_park_fiber(struct Backend_base *base, VALUE fiber);

data/ext/polyphony/backend_io_uring.c

@@ -106,9 +106,7 @@ VALUE Backend_post_fork(VALUE self) {
   io_uring_queue_exit(&backend->ring);
   io_uring_queue_init(backend->prepared_limit, &backend->ring, 0);
   context_store_free(&backend->store);
-  backend->base.currently_polling = 0;
-  backend->base.pending_count = 0;
-  backend->pending_sqes = 0;
+  backend_base_reset(&backend->base);
 
   return self;
 }

data/ext/polyphony/backend_libev.c

@@ -157,6 +157,8 @@ VALUE Backend_post_fork(VALUE self) {
   ev_loop_destroy(backend->ev_loop);
   backend->ev_loop = EV_DEFAULT;
 
+  backend_base_reset(&backend->base);
+
   return self;
 }
 

data/lib/polyphony/extensions/fiber.rb

@@ -27,6 +27,7 @@ module Polyphony
       end
 
       fiber = parent.spin(@tag, @caller, &@block)
+      @monitors&.each_key { |f| fiber.monitor(f) }
       fiber.schedule(value) unless value.nil?
       fiber
     end
@@ -80,9 +81,7 @@ module Polyphony
   # Fiber supervision
   module FiberSupervision
     def supervise(*fibers, **opts, &block)
-      block ||= opts[:on_done] ||
-                (opts[:on_error] && supervise_on_error_proc(opts[:on_error]))
-      raise "No block given" unless block
+      block ||= supervise_opts_to_block(opts)
 
       fibers.each do |f|
         f.attach_to(self) unless f.parent == self
@@ -97,8 +96,19 @@ module Polyphony
       end
     end
 
-    def supervise_on_error_proc(on_error)
-      ->(f, r) { opts[:on_error].(f, r) if r.is_a?(Exception) }
+    def supervise_opts_to_block(opts)
+      block = opts[:on_done] || opts[:on_error]
+      return nil unless block || opts[:restart]
+
+      error_only = !!opts[:on_error]
+      restart_always = opts[:restart] == :always
+      restart_on_error = opts[:restart] == :on_error
+
+      ->(f, r) do
+        is_error = r.is_a?(Exception)
+        block.(f, r) if block && (!error_only || is_error)
+        f.restart if restart_always || (restart_on_error && is_error)
+      end
     end
   end
 
@@ -225,13 +235,6 @@ module Polyphony
         c.terminate(graceful)
         c.await
       end
-      reap_dead_children
-    end
-
-    def reap_dead_children
-      return unless @children
-
-      @children.reject! { |f| f.dead? }
     end
 
     def detach
@@ -315,6 +318,7 @@ module Polyphony
       inform_monitors(result, uncaught_exception)
       @running = false
     ensure
+      @parent&.remove_child(self)
       # Prevent fiber from being resumed after terminating
       @thread.fiber_unschedule(self)
       Thread.current.switch_fiber

data/lib/polyphony/net.rb

@@ -67,7 +67,6 @@ module Polyphony
       def setup_alpn(context, protocols)
         context.alpn_protocols = protocols
         context.alpn_select_cb = lambda do |peer_protocols|
-          p alpn_select_cb: peer_protocols
           (protocols & peer_protocols).first
         end
       end

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.69'
+  VERSION = '0.70'
 end

data/test/test_fiber.rb

@@ -67,8 +67,6 @@ class FiberTest < MiniTest::Test
     }
     Fiber.await(f2, f3)
     assert_equal [:foo, :bar, :baz], buffer
-    assert_equal [f1], Fiber.current.children
-    Fiber.current.reap_dead_children
     assert_equal [], Fiber.current.children
   end
 
@@ -93,8 +91,6 @@ class FiberTest < MiniTest::Test
     f1.stop
 
     snooze
-    assert_equal [f1, f2, f3], Fiber.current.children
-    Fiber.current.reap_dead_children
     assert_equal [], Fiber.current.children
   end
 
@@ -602,7 +598,6 @@ class FiberTest < MiniTest::Test
 
     f.stop
     snooze
-    Fiber.current.reap_dead_children
     assert_equal [], Fiber.current.children
   end
 

data/test/test_global_api.rb

@@ -375,7 +375,6 @@ class SpinScopeTest < MiniTest::Test
       buffer << e.message
     end
     10.times { snooze }
-    Fiber.current.reap_dead_children
     assert_equal 0, Fiber.current.children.size
     assert_equal ['foobar'], buffer
   end

data/test/test_process_supervision.rb

@@ -6,7 +6,7 @@ class ProcessSupervisionTest < MiniTest::Test
   def test_process_supervisor_with_block
     i, o = IO.pipe
 
-    f = spin do
+    watcher = spin do
       Polyphony.watch_process do
         i.close
         sleep 5
@@ -14,31 +14,60 @@ class ProcessSupervisionTest < MiniTest::Test
         o << 'foo'
         o.close
       end
-      supervise(on_error: :restart)
     end
 
+    supervisor = spin { supervise(watcher, restart: :always) }
+
     sleep 0.05
-    f.terminate
-    f.await
+    supervisor.terminate
+    supervisor.await
 
     o.close
     msg = i.read
-    i.close
     assert_equal 'foo', msg
   end
 
+  def test_process_supervisor_restart_with_block
+    i1, o1 = IO.pipe
+    i2, o2 = IO.pipe
+
+    count = 0
+    watcher = spin do
+      count += 1
+      Polyphony.watch_process do
+        i1.gets
+        o2.puts count
+      end
+    end
+
+    supervisor = spin { supervise(watcher, restart: :always) }
+
+    o1.puts
+    l = i2.gets
+    assert_equal "1\n", l
+
+    o1.puts
+    l = i2.gets
+    assert_equal "2\n", l
+
+    o1.puts
+    l = i2.gets
+    assert_equal "3\n", l
+  end
+
   def test_process_supervisor_with_cmd
     fn = '/tmp/test_process_supervisor_with_cmd'
     FileUtils.rm(fn) rescue nil
 
-    f = spin do
+    watcher = spin do
       Polyphony.watch_process("echo foo >> #{fn}")
-      supervise(on_error: :restart)
     end
 
+    supervisor = spin { supervise(watcher) }
+
     sleep 0.05
-    f.terminate
-    f.await
+    supervisor.terminate
+    supervisor.await
 
     assert_equal "foo\n", IO.read(fn)
 

data/test/test_supervise.rb

@@ -3,12 +3,6 @@
 require_relative 'helper'
 
 class SuperviseTest < MiniTest::Test
-  def test_supervise_with_no_arguments
-    assert_raises(RuntimeError) do
-      supervise
-    end
-  end
-  
   def test_supervise_with_block
     buffer = []
     f1 = spin(:f1) { receive }
@@ -26,79 +20,168 @@ class SuperviseTest < MiniTest::Test
     assert_equal [[f1, 'foo'], [f2, 'bar']], buffer
   end
 
-#   def test_supervise_with_restart
-#     watcher = spin { receive }
-#     parent = spin { supervise(restart: true, watcher: watcher) }
-#     snooze
-
-#     buffer = []
-#     f1 = parent.spin do
-#       buffer << 'f1'
-#     end
-
-#     f1.await
-#     assert_equal ['f1'], buffer
-#     watcher.await
-#     assert_equal ['f1', 'f1'], buffer
-#   end
-
-#   def test_supervise_with_restart_on_error
-#     parent = spin { supervise(restart: true) }
-#     snooze
-
-#     buffer = []
-#     f1 = parent.spin do
-#       buffer << 'f1'
-#       buffer << receive
-#     end
-
-#     snooze
-#     assert_equal ['f1'], buffer
-
-#     f1.raise 'foo'
-
-#     3.times { snooze }
-
-#     assert_equal ['f1', 'f1'], buffer
-#     assert_equal :dead, f1.state
-
-#     # f1 should have been restarted by supervisor
-#     f1 = parent.children.first
-#     assert_kind_of Fiber, f1
-
-#     f1 << 'foo'
-#     f1.await
-
-#     assert_equal ['f1', 'f1', 'foo'], buffer
-#   end
-
-#   def test_supervisor_termination
-#     f = nil
-#     p = spin do
-#       f = spin { sleep 1 }
-#       supervise
-#     end
-#     sleep 0.01
-
-#     p.terminate
-#     p.await
-
-#     assert :dead, f.state
-#     assert :dead, p.state
-#   end
-
-#   def test_supervisor_termination_with_restart
-#     f = nil
-#     p = spin do
-#       f = spin { sleep 1 }
-#       supervise(restart: true)
-#     end
-#     sleep 0.01
-
-#     p.terminate
-#     p.await
-
-#     assert :dead, f.state
-#     assert :dead, p.state
-#   end
+  def test_supervise_with_on_done
+    buffer = []
+    f1 = spin(:f1) { receive }
+    f2 = spin(:f2) { receive }
+    supervisor = spin(:supervisor) do
+      supervise(f1, f2, on_done: ->(*args) { buffer << args })
+    end
+
+    snooze
+    f1 << 'foo'
+    f1.await
+    10.times { snooze }
+    assert_equal [[f1, 'foo']], buffer
+
+    f2 << 'bar'
+    f2.await
+    assert_equal [[f1, 'foo'], [f2, 'bar']], buffer
+  end
+
+  def test_supervise_with_on_error
+    buffer = []
+    f1 = spin(:f1) { receive }
+    f2 = spin(:f2) { receive }
+    supervisor = spin(:supervisor) do
+      supervise(f1, f2, on_error: ->(*args) { buffer << args })
+    end
+
+    snooze
+    f1 << 'foo'
+    f1.await
+    10.times { snooze }
+    assert_equal [], buffer
+
+    e = RuntimeError.new('blah')
+    f2.raise(e)
+    3.times { snooze }
+    assert_equal [[f2, e]], buffer
+  end
+
+  def test_supervise_with_manual_restart
+    buffer = []
+    f1 = spin(:f1) { receive }
+    supervisor = spin(:supervisor) do
+      supervise(f1) do |f, r|
+        buffer << [f, r]
+        f.restart
+      end
+    end
+
+    snooze
+    f1 << 'foo'
+    f1.await
+    snooze
+    assert_equal [[f1, 'foo']], buffer
+
+    10.times { snooze }
+
+    assert_equal 1, supervisor.children.size
+    f2 = supervisor.children.first
+    assert f1 != f2
+    assert_equal :f1, f2.tag
+    assert_equal supervisor, f2.parent
+
+    e = RuntimeError.new('bar')
+    f2.raise(e)
+    f2.await rescue nil
+    3.times { snooze }
+    assert_equal [[f1, 'foo'], [f2, e]], buffer
+
+    assert_equal 1, supervisor.children.size
+    f3 = supervisor.children.first
+    assert f2 != f3
+    assert f1 != f3
+    assert_equal :f1, f3.tag
+    assert_equal supervisor, f3.parent
+  end
+
+  def test_supervise_with_restart_always
+    buffer = []
+    f1 = spin(:f1) do
+      buffer << receive
+    rescue => e
+      buffer << e
+      e
+    end
+    supervisor = spin(:supervisor) { supervise(f1, restart: :always) }
+
+    snooze
+    f1 << 'foo'
+    f1.await
+    snooze
+    assert_equal ['foo'], buffer
+
+    10.times { snooze }
+
+    assert_equal 1, supervisor.children.size
+    f2 = supervisor.children.first
+    assert f1 != f2
+    assert_equal :f1, f2.tag
+    assert_equal supervisor, f2.parent
+
+    e = RuntimeError.new('bar')
+    f2.raise(e)
+    f2.await rescue nil
+    3.times { snooze }
+    assert_equal ['foo', e], buffer
+
+    assert_equal 1, supervisor.children.size
+    f3 = supervisor.children.first
+    assert f2 != f3
+    assert f1 != f3
+    assert_equal :f1, f3.tag
+    assert_equal supervisor, f3.parent
+  end
+
+  def test_supervise_with_restart_on_error
+    buffer = []
+    f1 = spin(:f1) do
+      buffer << receive
+    rescue => e
+      buffer << e
+      e
+    end
+    supervisor = spin(:supervisor) { supervise(f1, restart: :on_error) }
+
+    snooze
+    e = RuntimeError.new('bar')
+    f1.raise(e)
+    f1.await rescue nil
+    snooze
+    assert_equal [e], buffer
+
+    10.times { snooze }
+
+    assert_equal 1, supervisor.children.size
+    f2 = supervisor.children.first
+    assert f1 != f2
+    assert_equal :f1, f2.tag
+    assert_equal supervisor, f2.parent
+
+    f2 << 'foo'
+    f2.await rescue nil
+    3.times { snooze }
+    assert_equal [e, 'foo'], buffer
+
+    assert_equal 0, supervisor.children.size
+  end
+
+  def test_supervise_terminate
+    buffer = []
+    f1 = spin(:f1) do
+      buffer << receive
+    rescue => e
+      buffer << e
+      e
+    end
+    supervisor = spin(:supervisor) { supervise(f1, restart: :on_error) }
+
+    sleep 0.05
+    supervisor.terminate
+    supervisor.await
+
+    assert_equal [], buffer
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: '0.69'
+  version: '0.70'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-16 00:00:00.000000000 Z
+date: 2021-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler