polyphony 0.33 → 0.34

data/examples/core/xx-fork-spin.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+Exception.__disable_sanitized_backtrace__ = true
+
+puts "Parent pid: #{Process.pid}"
+
+i, o = IO.pipe
+
+pid = Polyphony.fork do
+  puts "Child pid: #{Process.pid}"
+  i.close
+  spin do
+    spin do
+      p :sleep
+      sleep 1
+    rescue ::Interrupt => e
+      p 1
+      # the signal should be raised only in the main fiber
+      o.puts "1-interrupt"
+    end.await
+  rescue Polyphony::Terminate
+    puts "terminate!"
+  end.await
+rescue ::Interrupt => e
+  p 2
+  o.puts "3-interrupt"
+ensure
+  p 3
+  o.close
+end
+sleep 0.2
+o.close
+watcher = Gyro::Child.new(pid)
+Process.kill('INT', pid)
+watcher.await
+buffer = i.read
+
+puts '*' * 40
+p buffer

data/examples/core/xx-spin-fork.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+Exception.__disable_sanitized_backtrace__ = true
+
+puts "Parent pid: #{Process.pid}"
+
+def start_worker
+  Polyphony.fork do
+    p :sleep
+    sleep 5
+    p :done_sleeping
+  ensure
+    p :start_worker_fork_ensure
+  end
+end
+
+f = spin do
+  Polyphony::ProcessSupervisor.supervise do
+    spin do
+      spin do
+        p :sleep
+        sleep 5
+        p :done_sleeping
+      end.await
+    end.await
+  ensure
+    p :start_worker_fork_ensure
+  end
+  # spin do
+  #   pid = start_worker
+  #   p [:before_child_await, pid]
+  #   Gyro::Child.new(pid).await
+  #   p :after_child_await
+  # ensure
+  #   puts "child done"
+  # end
+  # supervise
+# ensure
+#   puts "kill child"
+#   Process.kill('TERM', pid) rescue nil
+end
+
+sleep 1
+puts "terminate worker"
+f.terminate
+f.await

data/examples/core/xx-supervise-process.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+Exception.__disable_sanitized_backtrace__ = true
+
+supervisor = spin do
+  puts "parent pid #{Process.pid}"
+
+  Polyphony::ProcessSupervisor.supervise do
+    puts "child pid #{Process.pid}"
+    puts "go to sleep"
+    sleep 5
+  rescue Interrupt
+    puts "child got INT"
+  rescue SystemExit
+    puts "child got TERM"
+  ensure
+    puts "done sleeping"
+  end
+end
+
+begin
+  supervisor.await
+rescue Interrupt
+  exit!
+  # supervisor.terminate
+  # supervisor.await
+end

data/ext/gyro/gyro.h

@@ -41,6 +41,7 @@ struct ev_loop *Gyro_Selector_ev_loop(VALUE selector);
 ev_tstamp Gyro_Selector_now(VALUE selector);
 struct ev_loop *Gyro_Selector_current_thread_ev_loop();
 long Gyro_Selector_pending_count(VALUE self);
+VALUE Gyro_Selector_post_fork(VALUE self);
 
 VALUE Thread_current_event_selector();
 VALUE Thread_ref(VALUE thread);

data/ext/gyro/selector.c

@@ -129,6 +129,14 @@ VALUE Gyro_Selector_stop(VALUE self) {
   return Qnil;
 }
 
+VALUE Gyro_Selector_post_fork(VALUE self) {
+  struct Gyro_Selector *selector;
+  GetGyro_Selector(self, selector);
+
+  ev_loop_fork(selector->ev_loop);
+  return self;
+}
+
 VALUE Gyro_Selector_break_out_of_ev_loop(VALUE self) {
   struct Gyro_Selector *selector;
   GetGyro_Selector(self, selector);

data/ext/gyro/thread.c

@@ -107,6 +107,7 @@ VALUE Thread_schedule_fiber(VALUE self, VALUE fiber, VALUE value) {
   if (rb_fiber_alive_p(fiber) != Qtrue) {
     return self;
   }
+
   FIBER_TRACE(3, SYM_fiber_schedule, fiber, value);
   // if fiber is already scheduled, just set the scheduled value, then return
   rb_ivar_set(fiber, ID_runnable_value, value);
@@ -185,6 +186,8 @@ VALUE Thread_switch_fiber(VALUE self) {
     int ref_count = Thread_fiber_ref_count(self);
     if (next_fiber != Qnil) {
       if (ref_count > 0) {
+        // this mechanism prevents event starvation in case the run queue never
+        // empties
         Gyro_Selector_run_no_wait(selector, current_fiber, RARRAY_LEN(queue));
       }
       break;
@@ -218,8 +221,9 @@ VALUE Thread_reset_fiber_scheduling(VALUE self) {
 }
 
 VALUE Thread_post_fork(VALUE self) {
-  ev_loop_fork(EV_DEFAULT);
-  Thread_setup_fiber_scheduling(self);
+  VALUE selector = rb_ivar_get(self, ID_ivar_event_selector);
+  Gyro_Selector_post_fork(selector);
+
   return self;
 }
 
@@ -259,6 +263,8 @@ void Init_Thread() {
   rb_define_method(rb_cThread, "fiber_ref", Thread_ref, 0);
   rb_define_method(rb_cThread, "fiber_unref", Thread_unref, 0);
 
+  rb_define_method(rb_cThread, "post_fork", Thread_post_fork, 0);
+
   rb_define_method(rb_cThread, "setup_fiber_scheduling", Thread_setup_fiber_scheduling, 0);
   rb_define_method(rb_cThread, "stop_event_selector", Thread_stop_event_selector, 0);
   rb_define_method(rb_cThread, "reset_fiber_scheduling", Thread_reset_fiber_scheduling, 0);

data/lib/polyphony.rb

@@ -31,6 +31,7 @@ module Polyphony
   auto_import(
     Channel:      './polyphony/core/channel',
     FS:           './polyphony/adapters/fs',
+    Process:      './polyphony/adapters/process',
     ResourcePool: './polyphony/core/resource_pool',
     Sync:         './polyphony/core/sync',
     ThreadPool:   './polyphony/core/thread_pool',
@@ -51,27 +52,73 @@ module Polyphony
     end
 
     def fork(&block)
-      pid = Kernel.fork do
-        Gyro.post_fork
-        Fiber.current.setup_main_fiber
-        block.()
-      ensure
-        Fiber.current.terminate_all_children
-        Fiber.current.await_all_children
+      Kernel.fork do
+        # Since the fiber doing the fork will become the main fiber of the
+        # forked process, we leave it behind by transferring to a new fiber
+        # created in the context of the forked process, which rescues *all*
+        # exceptions, including Interrupt and SystemExit.
+        spin_forked_block(&block).transfer
       end
-      pid
     end
-  end
-end
 
-# install signal handlers
+    def spin_forked_block(&block)
+      Fiber.new do
+        run_forked_block(&block)
+        exit_forked_process
+      rescue ::SystemExit
+        exit_forked_process
+      rescue Exception => e
+        e.full_message
+        exit!
+      end
+    end
+
+    def run_forked_block(&block)
+      # A race condition can arise if a TERM or INT signal is received before
+      # the forked process has finished initializing. To prevent this we restore
+      # the default signal handlers, and then reinstall the custom Polyphony
+      # handlers just before running the given block.
+      trap('SIGTERM', 'DEFAULT')
+      trap('SIGINT', 'DEFAULT')
+
+      Thread.current.post_fork
+      Thread.current.setup
+      Fiber.current.setup_main_fiber
+
+      install_terminating_signal_handlers
+
+      block.()
+    end
+
+    def exit_forked_process
+      Fiber.current.shutdown_all_children
+      # Since fork could be called from any fiber, we explicitly call exit here.
+      # Otherwise, the fiber might want to pass execution to another fiber that
+      # previously transferred execution to the forking fiber, but doesn't exist
+      # anymore...
+      #
+      # The call to exit will invoke the at_exit handler we use to terminate the
+      # (forked) main fiber's child fibers.
+      exit
+    end
 
-def install_terminating_signal_handler(signal, exception_class)
-  trap(signal) do
-    exception = exception_class.new
-    Thread.current.break_out_of_ev_loop(Thread.current.main_fiber, exception)
+    def watch_process(cmd = nil, &block)
+      Polyphony::Process.watch(cmd, &block)
+    end
+
+    def emit_signal_exception(exception, fiber = Thread.main.main_fiber)
+      Thread.current.break_out_of_ev_loop(fiber, exception)
+    end
+
+    def install_terminating_signal_handler(signal, exception_class)
+      trap(signal) { emit_signal_exception(exception_class.new) }
+    end
+
+    def install_terminating_signal_handlers
+      install_terminating_signal_handler('SIGTERM', ::SystemExit)
+      install_terminating_signal_handler('SIGINT', ::Interrupt)
+    end
   end
 end
 
-install_terminating_signal_handler('SIGTERM', SystemExit)
-install_terminating_signal_handler('SIGINT', Interrupt)
+Polyphony.install_terminating_signal_handlers

data/lib/polyphony/adapters/process.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+export :watch
+
+def watch(cmd = nil, &block)
+  terminated = nil
+  pid = cmd ? Kernel.spawn(cmd) : Polyphony.fork(&block)
+  watcher = Gyro::Child.new(pid)
+  watcher.await
+  terminated = true
+ensure
+  kill_process(pid) unless terminated || pid.nil?
+end
+
+def kill_process(pid)
+  cancel_after(5) do
+    kill_and_await('TERM', pid)
+  end
+rescue Polyphony::Cancel
+  kill_and_await(-9, pid)
+end
+
+def kill_and_await(sig, pid)
+  Process.kill(sig, pid)
+  Gyro::Child.new(pid).await
+rescue SystemCallError
+  # ignore
+  puts 'SystemCallError in kill_and_await'
+end

data/lib/polyphony/adapters/trace.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-export :new, :analyze
+export :new, :analyze, :STOCK_EVENTS
 
 require 'polyphony'
 
@@ -14,9 +14,11 @@ end
 
 def trace_record(trp, start_stamp)
   stamp = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start_stamp
-  { stamp: stamp, self: trp.self, binding: trp.binding, event: trp.event,
-    fiber: tp_fiber(trp), lineno: trp.lineno, method_id: trp.method_id,
-    file: trp.path, parameters: tp_params(trp),
+
+  { stamp: stamp, event: trp.event, location: "#{trp.path}:#{trp.lineno}",
+    self: trp.self, binding: trp.binding, fiber: tp_fiber(trp),
+    lineno: trp.lineno, method_id: trp.method_id,
+    path: trp.path, parameters: tp_params(trp),
     return_value: tp_return_value(trp), schedule_value: tp_schedule_value(trp),
     exception: tp_raised_exception(trp) }
 end

data/lib/polyphony/core/exceptions.rb

@@ -12,8 +12,13 @@ class BaseException < ::Exception
   attr_reader :value
 
   def initialize(value = nil)
+    @caller_backtrace = caller
     @value = value
   end
+
+  def backtrace
+    sanitize(@caller_backtrace)
+  end
 end
 
 # MoveOn is used to interrupt a long-running blocking operation, while

data/lib/polyphony/core/global_api.rb

@@ -23,6 +23,10 @@ module API
       sleep interval
       fiber.schedule Exceptions::Cancel.new
     end
+    block ? cancel_after_wrap_block(canceller, &block) : canceller
+  end
+
+  def cancel_after_wrap_block(canceller, &block)
     block.call
   ensure
     canceller.stop
@@ -54,6 +58,17 @@ module API
 
   def move_on_after(interval, with_value: nil, &block)
     fiber = ::Fiber.current
+    unless block
+      return spin do
+        sleep interval
+        fiber.schedule with_value
+      end
+    end
+
+    move_on_after_with_block(fiber, interval, with_value, &block)
+  end
+
+  def move_on_after_with_block(fiber, interval, with_value, &block)
     canceller = spin do
       sleep interval
       fiber.schedule Exceptions::MoveOn.new(with_value)

data/lib/polyphony/extensions/fiber.rb

@@ -29,10 +29,15 @@ module FiberControl
 
   def restart(value = nil)
     raise "Can''t restart main fiber" if @main
-    return parent.spin(&@block).tap { |f| f.schedule(value) } unless @running
 
-    schedule Exceptions::Restart.new(value)
-    self
+    if @running
+      schedule Exceptions::Restart.new(value)
+      return self
+    end
+
+    parent.spin(@tag, @caller, &@block).tap do |f|
+      f.schedule(value) unless value.nil?
+    end
   end
   alias_method :reset, :restart
 
@@ -61,32 +66,37 @@ module FiberControl
     else RuntimeError.new
     end
   end
+end
 
-  def supervise(on_error: nil, &block)
-    @on_child_done = proc { schedule }
-    loop { supervise_perform(on_error, &block) }
+# Fiber supervision
+module FiberSupervision
+  def supervise(opts = {})
+    @counter = 0
+    @on_child_done = proc do |fiber, result|
+      self << fiber unless result.is_a?(Exception)
+    end
+    loop { supervise_perform(opts) }
+  ensure
+    @on_child_done = nil
   end
 
-  def supervise_perform(policy, &block)
-    suspend
-  rescue Polyphony::Restart
+  def supervise_perform(opts)
+    fiber = receive
+    restart_fiber(fiber, opts) if fiber
+  rescue Exceptions::Restart
     restart_all_children
   rescue Exception => e
-    case e.source_fiber
-    when nil, self
-      Kernel.raise e
-    else
-      handle_supervisor_exception(e, e.source_fiber, policy, &block)
-    end
-  end
+    Kernel.raise e if e.source_fiber.nil? || e.source_fiber == self
 
-  def handle_supervisor_exception(error, fiber, policy, &block)
-    return block.call(fiber, error) if block
+    restart_fiber(e.source_fiber, opts)
+  end
 
-    case policy
-    when :restart
+  def restart_fiber(fiber, opts)
+    opts[:watcher]&.send [:restart, fiber]
+    case opts[:restart]
+    when true
       fiber.restart
-    when :restart_all
+    when :one_for_all
       @children.keys.each(&:restart)
     end
   end
@@ -133,9 +143,9 @@ module FiberControlClassMethods
   def await_select_cleanup(state)
     return if state[:pending].empty?
 
-    move_on = Exceptions::MoveOn.new
+    terminate = Exceptions::Terminate.new
     state[:cleanup] = true
-    state[:pending].each_key { |f| f.schedule(move_on) }
+    state[:pending].each_key { |f| f.schedule(terminate) }
     suspend
   end
 
@@ -170,7 +180,6 @@ end
 module FiberMessaging
   def <<(value)
     @mailbox << value
-    snooze
   end
   alias_method :send, :<<
 
@@ -196,9 +205,9 @@ module ChildFiberControl
     f
   end
 
-  def child_done(child_fiber)
+  def child_done(child_fiber, result)
     @children.delete(child_fiber)
-    @on_child_done&.(child_fiber)
+    @on_child_done&.(child_fiber, result)
   end
 
   def terminate_all_children
@@ -220,16 +229,8 @@ module ChildFiberControl
   end
 end
 
-# Fiber extensions
-class ::Fiber
-  prepend FiberControl
-  include FiberMessaging
-  include ChildFiberControl
-
-  extend FiberControlClassMethods
-
-  attr_accessor :tag, :thread, :parent
-
+# Fiber life cycle methods
+module FiberLifeCycle
   def prepare(tag, block, caller, parent)
     @thread = Thread.current
     @tag = tag
@@ -241,15 +242,6 @@ class ::Fiber
     schedule
   end
 
-  def setup_main_fiber
-    @main = true
-    @tag = :main
-    @thread = Thread.current
-    @running = true
-    @children&.clear
-    @mailbox = Gyro::Queue.new
-  end
-
   def run(first_value)
     setup first_value
     result = @block.(first_value)
@@ -263,17 +255,37 @@ class ::Fiber
     finalize e, true
   end
 
-  def restart_self(first_value)
-    @mailbox = Gyro::Queue.new
-    run(first_value)
-  end
-
   def setup(first_value)
     Kernel.raise first_value if first_value.is_a?(Exception)
 
     @running = true
   end
 
+  # Performs setup for a "raw" Fiber created using Fiber.new. Note that this
+  # fiber is an orphan fiber (has no parent), since we cannot control how the
+  # fiber terminates after it has already been created. Calling #setup_raw
+  # allows the fiber to be scheduled and to receive messages.
+  def setup_raw
+    @thread = Thread.current
+    @mailbox = Gyro::Queue.new
+  end
+
+  def setup_main_fiber
+    @main = true
+    @tag = :main
+    @thread = Thread.current
+    @running = true
+    @children&.clear
+    @mailbox = Gyro::Queue.new
+  end
+
+  def restart_self(first_value)
+    @mailbox = Gyro::Queue.new
+    @when_done_procs = nil
+    @waiting_fibers = nil
+    run(first_value)
+  end
+
   def finalize(result, uncaught_exception = false)
     result, uncaught_exception = finalize_children(result, uncaught_exception)
     __fiber_trace__(:fiber_terminate, self, result)
@@ -298,29 +310,40 @@ class ::Fiber
   end
 
   def inform_dependants(result, uncaught_exception)
-    @parent.child_done(self)
+    @parent&.child_done(self, result)
     @when_done_procs&.each { |p| p.(result) }
-    has_waiting_fibers = nil
     @waiting_fibers&.each_key do |f|
-      has_waiting_fibers = true
       f.schedule(result)
     end
-    return unless uncaught_exception && !has_waiting_fibers
+    return unless uncaught_exception && !@waiting_fibers
 
     # propagate unaught exception to parent
-    @parent.schedule(result)
+    @parent&.schedule(result)
   end
 
+  def when_done(&block)
+    @when_done_procs ||= []
+    @when_done_procs << block
+  end
+end
+
+# Fiber extensions
+class ::Fiber
+  prepend FiberControl
+  include FiberSupervision
+  include FiberMessaging
+  include ChildFiberControl
+  include FiberLifeCycle
+
+  extend FiberControlClassMethods
+
+  attr_accessor :tag, :thread, :parent
   attr_reader :result
 
   def running?
     @running
   end
 
-  def when_done(&block)
-    (@when_done_procs ||= []) << block
-  end
-
   def inspect
     "#<Fiber:#{object_id} #{location} (#{state})>"
   end
@@ -346,7 +369,14 @@ end
 
 Fiber.current.setup_main_fiber
 
+# This at_exit handler is needed only when the original process exits. Due to
+# the behaviour of fibers on fork (and especially on exit from forked
+# processes,) we use a separate mechanism to terminate fibers in forked
+# processes (see Polyphony.fork).
+orig_pid = Process.pid
 at_exit do
+  next unless orig_pid == Process.pid
+
   Fiber.current.terminate_all_children
   Fiber.current.await_all_children
 end