polyphony 0.70 → 0.73.1

data/ext/polyphony/extconf.rb

@@ -3,32 +3,43 @@
 require 'rubygems'
 require 'mkmf'
 
-use_liburing = false
-use_pidfd_open = false
-force_use_libev = ENV['POLYPHONY_USE_LIBEV'] != nil
-linux = RUBY_PLATFORM =~ /linux/
-
-if linux && `uname -sr` =~ /Linux 5\.(\d+)/
-  kernel_minor_version = $1.to_i
-  use_liburing = !force_use_libev && kernel_minor_version >= 6
-  use_pidfd_open = kernel_minor_version >= 3
+
+KERNEL_INFO_RE = /Linux (\d)\.(\d+)\.(?:\d+)\-(?:\d+\-)?(\w+)/
+def get_config
+  config = { linux: !!(RUBY_PLATFORM =~ /linux/) }
+  return config if !config[:linux]
+
+  kernel_info = `uname -sr`
+  m = kernel_info.match(KERNEL_INFO_RE)
+  raise "Could not parse Linux kernel information (#{kernel_info.inspect})" if !m
+
+  version, major_revision, distribution = m[1].to_i, m[2].to_i, m[3]
+  config[:pidfd_open] = (version == 5) && (major_revision >= 3)
+
+  force_libev = ENV['POLYPHONY_USE_LIBEV'] != nil
+  config[:io_uring] = !force_libev &&
+    (version == 5) && (major_revision >= 6) && (distribution != 'linuxkit')
+  config
 end
 
-$defs << '-DPOLYPHONY_USE_PIDFD_OPEN' if use_pidfd_open
-if use_liburing
+config = get_config
+puts "Building Polyphony... (#{config.inspect})"
+
+$defs << '-DPOLYPHONY_USE_PIDFD_OPEN' if config[:pidfd_open]
+if config[:io_uring]
   $defs << "-DPOLYPHONY_BACKEND_LIBURING"
   $defs << "-DPOLYPHONY_UNSET_NONBLOCK" if RUBY_VERSION =~ /^3/
   $CFLAGS << " -Wno-pointer-arith"
 else
   $defs << "-DPOLYPHONY_BACKEND_LIBEV"
-  $defs << "-DPOLYPHONY_LINUX" if linux
+  $defs << "-DPOLYPHONY_LINUX" if config[:linux]
   $defs << '-DEV_USE_LINUXAIO'     if have_header('linux/aio_abi.h')
   $defs << '-DEV_USE_SELECT'       if have_header('sys/select.h')
   $defs << '-DEV_USE_POLL'         if have_type('port_event_t', 'poll.h')
   $defs << '-DEV_USE_EPOLL'        if have_header('sys/epoll.h')
   $defs << '-DEV_USE_KQUEUE'       if have_header('sys/event.h') && have_header('sys/queue.h')
   $defs << '-DEV_USE_PORT'         if have_type('port_event_t', 'port.h')
-  $defs << '-DHAVE_SYS_RESOURCE_H' if have_header('sys/resource.h')  
+  $defs << '-DHAVE_SYS_RESOURCE_H' if have_header('sys/resource.h')
 
   $CFLAGS << " -Wno-comment"
   $CFLAGS << " -Wno-unused-result"
@@ -40,6 +51,7 @@ $defs << '-DPOLYPHONY_PLAYGROUND' if ENV['POLYPHONY_PLAYGROUND']
 
 CONFIG['optflags'] << ' -fno-strict-aliasing' unless RUBY_PLATFORM =~ /mswin/
 
+have_func('rb_fiber_transfer', 'ruby.h')
 
 dir_config 'polyphony_ext'
 create_makefile 'polyphony_ext'

data/ext/polyphony/polyphony.h

@@ -26,7 +26,11 @@
 #define RAISE_IF_NOT_NIL(ret) if (ret != Qnil) { RAISE_EXCEPTION(ret); }
 
 // Fiber#transfer
-#define FIBER_TRANSFER(fiber, value) rb_funcall(fiber, ID_transfer, 1, value)
+#if HAVE_RB_FIBER_TRANSFER
+  #define FIBER_TRANSFER(fiber, value) rb_fiber_transfer(fiber, 1, &value)
+#else
+  #define FIBER_TRANSFER(fiber, value) rb_funcall(fiber, ID_transfer, 1, value)
+#endif
 
 #define BACKEND() (rb_ivar_get(rb_thread_current(), ID_ivar_backend))
 

data/ext/polyphony/queue.c

@@ -162,12 +162,12 @@ VALUE Queue_cap(VALUE self, VALUE cap) {
   Queue_t *queue;
   GetQueue(self, queue);
   queue->capacity = new_capacity;
-  
+
   if (queue->capacity)
     queue_schedule_blocked_fibers_to_capacity(queue);
   else
     queue_schedule_all_blocked_fibers(&queue->push_queue);
-  
+
   return self;
 }
 

data/ext/polyphony/runqueue_ring_buffer.c

@@ -61,8 +61,9 @@ inline void runqueue_ring_buffer_push(runqueue_ring_buffer *buffer, VALUE fiber,
 
 inline void runqueue_ring_buffer_mark(runqueue_ring_buffer *buffer) {
   for (unsigned int i = 0; i < buffer->count; i++) {
-    rb_gc_mark(buffer->entries[(buffer->head + i) % buffer->size].fiber);
-    rb_gc_mark(buffer->entries[(buffer->head + i) % buffer->size].value);
+    runqueue_entry entry = buffer->entries[(buffer->head + i) % buffer->size];
+    rb_gc_mark(entry.fiber);
+    rb_gc_mark(entry.value);
   }
 }
 

data/ext/polyphony/thread.c

@@ -40,7 +40,7 @@ VALUE Thread_fiber_schedule_and_wakeup(VALUE self, VALUE fiber, VALUE resume_obj
   }
 
   if (Backend_wakeup(rb_ivar_get(self, ID_ivar_backend)) == Qnil) {
-    // we're not inside the ev_loop, so we just do a switchpoint
+    // we're not inside Backend_poll, so we just do a switchpoint
     Thread_switch_fiber(self);
   }
 

data/lib/polyphony/adapters/irb.rb

@@ -3,26 +3,36 @@
 require 'polyphony'
 
 if Object.constants.include?(:Reline)
+  puts "reline"
   class Reline::ANSI
     def self.select(read_ios = [], write_ios = [], error_ios = [], timeout = nil)
-      p [:select, read_ios]
+      # p [:select, read_ios, timeout]
+      # puts caller.join("\n")
       raise if read_ios.size > 1
       raise if write_ios.size > 0
       raise if error_ios.size > 0
 
+      # p 1
       fiber = Fiber.current
       timer = spin do
         sleep timeout
         fiber.cancel
       end
+      # p 2
       read_ios.each do |io|
+        # p wait: io
         Polyphony.backend_wait_io(io, false)
+        # p :done_wait
         return [io]
       end
+      # p 3
     rescue Polyphony::Cancel
+      # p :cancel
       return nil
     ensure
+      # p :ensure
       timer.stop
+      # p :ensure_done      
     end
   end
 else

data/lib/polyphony/core/global_api.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require_relative '../extensions/core'
-require_relative '../extensions/fiber'
-require_relative './exceptions'
 require_relative './throttler'
 
 module Polyphony
@@ -73,7 +70,7 @@ module Polyphony
 
     def spin_scope
       raise unless block_given?
-    
+
       spin do
         result = yield
         Fiber.current.await_all_children
@@ -122,8 +119,8 @@ module Polyphony
       Fiber.current.receive_all_pending
     end
 
-    def supervise(*args, &block)
-      Fiber.current.supervise(*args, &block)
+    def supervise(*args, **opts, &block)
+      Fiber.current.supervise(*args, **opts, &block)
     end
 
     def sleep(duration = nil)

data/lib/polyphony/core/timer.rb

@@ -44,7 +44,7 @@ module Polyphony
     ensure
       @timeouts.delete(fiber)
     end
-  
+
     def cancel_after(interval, with_exception: Polyphony::Cancel)
       fiber = Fiber.current
       @timeouts[fiber] = {
@@ -74,7 +74,7 @@ module Polyphony
     def reset
       record = @timeouts[Fiber.current]
       return unless record
-  
+
       record[:target_stamp] = now + record[:interval]
     end
 

data/lib/polyphony/debugger.rb

@@ -9,11 +9,11 @@ module Polyphony
     :return,
     :b_call,
     :b_return
-  ]    
+  ]
 
   def self.start_debug_server(socket_path)
     server = DebugServer.new(socket_path)
-    controller = DebugController.new(server)      
+    controller = DebugController.new(server)
     trace = TracePoint.new(*TP_EVENTS) { |tp| controller.handle_tp(trace, tp) }
     trace.enable
 
@@ -124,7 +124,7 @@ module Polyphony
         h
       end
     end
-    
+
     def cmd_step(cmd)
       tp = nil
       fiber = nil

data/lib/polyphony/extensions/exception.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Exeption overrides
+class ::Exception
+  class << self
+    attr_accessor :__disable_sanitized_backtrace__
+  end
+
+  attr_accessor :source_fiber, :raising_fiber
+
+  alias_method :orig_initialize, :initialize
+  def initialize(*args)
+    @raising_fiber = Fiber.current
+    orig_initialize(*args)
+  end
+
+  alias_method :orig_backtrace, :backtrace
+  def backtrace
+    unless @backtrace_called
+      @backtrace_called = true
+      return orig_backtrace
+    end
+
+    sanitized_backtrace
+  end
+
+  def sanitized_backtrace
+    return sanitize(orig_backtrace) unless @raising_fiber
+
+    backtrace = orig_backtrace || []
+    sanitize(backtrace + @raising_fiber.caller)
+  end
+
+  POLYPHONY_DIR = File.expand_path(File.join(__dir__, '..'))
+
+  def sanitize(backtrace)
+    return backtrace if ::Exception.__disable_sanitized_backtrace__
+
+    backtrace.reject { |l| l[POLYPHONY_DIR] }
+  end
+
+  def invoke
+    Kernel.raise(self)
+  end
+end

data/lib/polyphony/extensions/fiber.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'fiber'
-
 require_relative '../core/exceptions'
 
 module Polyphony
@@ -19,7 +17,7 @@ module Polyphony
     alias_method :stop, :interrupt
 
     def restart(value = nil)
-      raise "Can''t restart main fiber" if @main
+      raise "Can't restart main fiber" if @main
 
       if @running
         schedule Polyphony::Restart.new(value)
@@ -83,6 +81,8 @@ module Polyphony
     def supervise(*fibers, **opts, &block)
       block ||= supervise_opts_to_block(opts)
 
+      @supervise_mode = true
+      fibers = children if fibers.empty?
       fibers.each do |f|
         f.attach_to(self) unless f.parent == self
         f.monitor(self)
@@ -94,15 +94,18 @@ module Polyphony
         (fiber, result) = mailbox.shift
         block&.call(fiber, result)
       end
+    ensure
+      @supervise_mode = false
     end
 
     def supervise_opts_to_block(opts)
       block = opts[:on_done] || opts[:on_error]
-      return nil unless block || opts[:restart]
+      restart = opts[:restart]
+      return nil unless block || restart
 
       error_only = !!opts[:on_error]
-      restart_always = opts[:restart] == :always
-      restart_on_error = opts[:restart] == :on_error
+      restart_always = (restart == :always) || (restart == true)
+      restart_on_error = restart == :on_error
 
       ->(f, r) do
         is_error = r.is_a?(Exception)
@@ -114,6 +117,14 @@ module Polyphony
 
   # Class methods for controlling fibers (namely await and select)
   module FiberControlClassMethods
+    # call-seq:
+    #   Fiber.await(*fibers) -> [*results]
+    #   Fiber.join(*fibers) -> [*results]
+    #
+    # Waits for all given fibers to terminate, then returns the respective
+    # return values for all terminated fibers. If any of the awaited fibers
+    # terminates with an uncaught exception, `Fiber.await` will await all the
+    # other fibers to terminate, then reraise the exception.
     def await(*fibers)
       return [] if fibers.empty?
 
@@ -149,7 +160,7 @@ module Polyphony
 
     def select(*fibers)
       return nil if fibers.empty?
-  
+
       current_fiber = self.current
       mailbox = current_fiber.monitor_mailbox
       fibers.each do |f|
@@ -163,7 +174,7 @@ module Polyphony
       while true
         (fiber, result) = mailbox.shift
         next unless fibers.include?(fiber)
-  
+
         fibers.each { |f| f.unmonitor(current_fiber) }
         if result.is_a?(Exception)
           raise result
@@ -197,6 +208,7 @@ module Polyphony
 
     def add_child(child_fiber)
       (@children ||= {})[child_fiber] = true
+      child_fiber.monitor(self) if @supervise_mode
     end
 
     def remove_child(child_fiber)
@@ -207,6 +219,7 @@ module Polyphony
       f = Fiber.new { |v| f.run(v) }
       f.prepare(tag, block, orig_caller, self)
       (@children ||= {})[f] = true
+      f.monitor(self) if @supervise_mode
       f
     end
 
@@ -237,10 +250,15 @@ module Polyphony
       end
     end
 
+    def attach_all_children_to(fiber)
+      @children&.keys.each { |c| c.attach_to(fiber) }
+    end
+
     def detach
       @parent.remove_child(self)
       @parent = @thread.main_fiber
       @parent.add_child(self)
+      self
     end
 
     def attach_to(fiber)
@@ -328,7 +346,7 @@ module Polyphony
     # the children are shut down, it is returned along with the uncaught_exception
     # flag set. Otherwise, it returns the given arguments.
     def finalize_children(result, uncaught_exception)
-      shutdown_all_children
+      shutdown_all_children(graceful_shutdown?)
       [result, uncaught_exception]
     rescue Exception => e
       [e, true]

data/lib/polyphony/extensions/io.rb

@@ -103,7 +103,7 @@ class ::IO
   alias_method :orig_getc, :getc
   def getc
     return @read_buffer.slice!(0) if @read_buffer && !@read_buffer.empty?
-    
+
     @read_buffer ||= +''
     Polyphony.backend_read(self, @read_buffer, 8192, false, -1)
     return @read_buffer.slice!(0) if !@read_buffer.empty?
@@ -116,7 +116,7 @@ class ::IO
     if buf
       return Polyphony.backend_read(self, buf, len, true, buf_pos)
     end
-    
+
     @read_buffer ||= +''
     result = Polyphony.backend_read(self, @read_buffer, len, true, -1)
     return nil unless result

data/lib/polyphony/extensions/{core.rb → kernel.rb}

@@ -1,73 +1,7 @@
 # frozen_string_literal: true
 
-require 'fiber'
-require 'timeout'
 require 'open3'
 
-require_relative '../core/exceptions'
-
-# Exeption overrides
-class ::Exception
-  class << self
-    attr_accessor :__disable_sanitized_backtrace__
-  end
-
-  attr_accessor :source_fiber, :raising_fiber
-
-  alias_method :orig_initialize, :initialize
-  def initialize(*args)
-    @raising_fiber = Fiber.current
-    orig_initialize(*args)
-  end
-
-  alias_method :orig_backtrace, :backtrace
-  def backtrace
-    unless @backtrace_called
-      @backtrace_called = true
-      return orig_backtrace
-    end
-
-    sanitized_backtrace
-  end
-
-  def sanitized_backtrace
-    return sanitize(orig_backtrace) unless @raising_fiber
-
-    backtrace = orig_backtrace || []
-    sanitize(backtrace + @raising_fiber.caller)
-  end
-
-  POLYPHONY_DIR = File.expand_path(File.join(__dir__, '..'))
-
-  def sanitize(backtrace)
-    return backtrace if ::Exception.__disable_sanitized_backtrace__
-
-    backtrace.reject { |l| l[POLYPHONY_DIR] }
-  end
-
-  def invoke
-    Kernel.raise(self)
-  end
-end
-
-# Overrides for Process
-module ::Process
-  class << self
-    alias_method :orig_detach, :detach
-    def detach(pid)
-      fiber = spin { Polyphony.backend_waitpid(pid) }
-      fiber.define_singleton_method(:pid) { pid }
-      fiber
-    end
-
-    alias_method :orig_daemon, :daemon
-    def daemon(*args)
-      orig_daemon(*args)
-      Polyphony.original_pid = Process.pid
-    end
-  end
-end
-
 # Kernel extensions (methods available to all objects / call sites)
 module ::Kernel
   alias_method :orig_sleep, :sleep
@@ -159,10 +93,3 @@ module ::Kernel
     end
   end
 end
-
-# Override Timeout to use cancel scope
-module ::Timeout
-  def self.timeout(sec, klass = Timeout::Error, message = 'execution expired', &block)
-    cancel_after(sec, with_exception: [klass, message], &block)
-  end
-end

data/lib/polyphony/extensions/openssl.rb

@@ -130,24 +130,32 @@ class ::OpenSSL::SSL::SSLServer
       end
     end
 
+    # STDOUT.puts 'SSLServer#accept'
     sock, = @svr.accept
+    # STDOUT.puts "- raw sock: #{sock.inspect}"
     begin
       ssl = OpenSSL::SSL::SSLSocket.new(sock, @ctx)
+      # STDOUT.puts "- ssl sock: #{ssl.inspect}"
       ssl.sync_close = true
       if @use_accept_worker
+        # STDOUT.puts "- send to accept worker"
         @accept_worker_fiber << [ssl, Fiber.current]
-        receive
+        # STDOUT.puts "- wait for accept worker"
+        r = receive
+        # STDOUT.puts "- got reply from accept worker: #{r.inspect}"
+        r.invoke if r.is_a?(Exception)
       else
         ssl.accept
       end
       ssl
-    rescue Exception => ex
+    rescue Exception => e
+      # STDOUT.puts "- accept exception: #{e.inspect}"
       if ssl
         ssl.close
       else
         sock.close
       end
-      raise ex
+      raise e
     end
   end
 
@@ -156,11 +164,18 @@ class ::OpenSSL::SSL::SSLServer
     @accept_worker_thread = Thread.new do
       fiber << Fiber.current
       loop do
+        # STDOUT.puts "- accept_worker wait for work"
         socket, peer = receive
+        # STDOUT.puts "- accept_worker got socket from peer #{peer.inspect}"
         socket.accept
+        # STDOUT.puts "- accept_worker accept returned"
         peer << socket
-      rescue => e
-        peer.schedule(e) if fiber
+        # STDOUT.puts "- accept_worker sent socket back to peer"
+      rescue Polyphony::BaseException
+        raise
+      rescue Exception => e
+        # STDOUT.puts "- accept_worker error: #{e}"
+        peer << e if peer
       end
     end
     @accept_worker_fiber = receive

data/lib/polyphony/extensions/process.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Overrides for Process
+module ::Process
+  class << self
+    alias_method :orig_detach, :detach
+    def detach(pid)
+      fiber = spin { Polyphony.backend_waitpid(pid) }
+      fiber.define_singleton_method(:pid) { pid }
+      fiber
+    end
+
+    alias_method :orig_daemon, :daemon
+    def daemon(*args)
+      orig_daemon(*args)
+      Polyphony.original_pid = Process.pid
+    end
+  end
+end

data/lib/polyphony/extensions/socket.rb

@@ -32,7 +32,7 @@ class ::Socket
   def read(maxlen = nil, buf = nil, buf_pos = 0)
     return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
     return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
-    
+
     buf = +''
     len = buf.bytesize
     while true
@@ -168,7 +168,7 @@ class ::TCPSocket
   def read(maxlen = nil, buf = nil, buf_pos = 0)
     return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
     return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
-    
+
     buf = +''
     len = buf.bytesize
     while true
@@ -232,7 +232,6 @@ class ::TCPServer
   alias_method :orig_accept, :accept
   def accept
     Polyphony.backend_accept(@io, TCPSocket)
-    # @io.accept
   end
 
   def accept_loop(&block)
@@ -261,7 +260,7 @@ class ::UNIXSocket
   def read(maxlen = nil, buf = nil, buf_pos = 0)
     return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
     return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
-    
+
     buf = +''
     len = buf.bytesize
     while true

data/lib/polyphony/extensions/timeout.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+# Override Timeout to use cancel scope
+module ::Timeout
+  def self.timeout(sec, klass = Timeout::Error, message = 'execution expired', &block)
+    cancel_after(sec, with_exception: [klass, message], &block)
+  end
+end

data/lib/polyphony/extensions.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative './extensions/exception'
+require_relative './extensions/fiber'
+require_relative './extensions/io'
+require_relative './extensions/kernel'
+require_relative './extensions/process'
+require_relative './extensions/thread'
+require_relative './extensions/timeout'

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.70'
+  VERSION = '0.73.1'
 end

data/lib/polyphony.rb

@@ -2,15 +2,12 @@
 
 require 'fiber'
 require_relative './polyphony_ext'
-
-require_relative './polyphony/extensions/core'
-require_relative './polyphony/extensions/thread'
-require_relative './polyphony/extensions/fiber'
-require_relative './polyphony/extensions/io'
+require_relative './polyphony/extensions'
 
 Thread.current.setup_fiber_scheduling
 Thread.current.backend = Polyphony::Backend.new
 
+require_relative './polyphony/core/exceptions'
 require_relative './polyphony/core/global_api'
 require_relative './polyphony/core/resource_pool'
 require_relative './polyphony/core/sync'

data/polyphony.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency  'simplecov',            '0.17.1'
   s.add_development_dependency  'rubocop',              '0.85.1'
   s.add_development_dependency  'pry',                  '0.13.1'
-  
+
   s.add_development_dependency  'msgpack',              '1.4.2'
   s.add_development_dependency  'httparty',             '0.17.1'
   s.add_development_dependency  'localhost',            '~>1.1.4'

data/test/coverage.rb

@@ -24,10 +24,10 @@ module Coverage
       @result = {}
       trace = TracePoint.new(:line) do |tp|
         next if tp.path =~ /\(/
-      
+
         absolute = File.expand_path(tp.path)
         next unless LIB_FILES.include?(absolute)# =~ /^#{LIB_DIR}/
-        
+
         @result[absolute] ||= relevant_lines_for_filename(absolute)
         @result[absolute][tp.lineno - 1] = 1
       end