polyphony 1.5 → 1.6

data/lib/polyphony/extensions/process.rb

@@ -2,6 +2,81 @@
 
 # Overrides for Process module
 module ::Process
+  module StatusExtensions
+    def coredump?
+      @status ? nil : super
+    end
+
+    def exited?
+      @status ? WIFEXITED(@status[1]) : super
+    end
+
+    def exitstatus
+      @status ? WEXITSTATUS(@status[1]) : super
+    end
+
+    def inspect
+      @status ? "#<Process::Status: pid #{@status[0]} exit #{@status[1]}>" : super
+    end
+
+    def pid
+      @status ? @status[0] : super
+    end
+
+    def signaled?
+      @status ? WIFSIGNALED(@status[1]) : super
+    end
+
+    def stopped?
+      @status ? WIFSTOPPED(@status[1]) : super
+    end
+
+    def stopsig
+      @status ? WIFSTOPPED(@status[1]) && WEXITSTATUS(@status[1]) : super
+    end
+
+    def success?
+      @status ? @status[1] == 0 : super
+    end
+
+    def termsig
+      @status ? WIFSIGNALED(@status[1]) && WTERMSIG(@status[1]) : super
+    end
+
+    private
+
+    # The following helper methods are translated from the C source:
+    # https://github.com/ruby/ruby/blob/v3_2_0/process.c
+
+    def WIFEXITED(w)
+      (w & 0xff) == 0
+    end
+
+    def WEXITSTATUS(w)
+      (w >> 8) & 0xff
+    end
+
+    def WIFSIGNALED(w)
+      (w & 0x7f) > 0 && ((w & 0x7f) < 0x7f)
+    end
+
+    def WIFSTOPPED(w)
+      (w & 0xff) == 0x7f
+    end
+
+    def WTERMSIG(w)
+      w & 0x7f
+    end
+  end
+
+  class Status
+    prepend StatusExtensions
+
+    def self.from_status_array(arr)
+      allocate.tap { |s| s.instance_variable_set(:@status, arr) }
+    end
+  end
+
   class << self
     # @!visibility private
     alias_method :orig_detach, :detach
@@ -11,7 +86,7 @@ module ::Process
     # @param pid [Integer] child pid
     # @return [Fiber] new fiber waiting on pid
     def detach(pid)
-      fiber = spin { Polyphony.backend_waitpid(pid) }
+      fiber = spin { ::Process::Status.from_status_array(Polyphony.backend_waitpid(pid)) }
       fiber.define_singleton_method(:pid) { pid }
       fiber
     end

data/lib/polyphony/extensions/thread.rb

@@ -14,7 +14,6 @@ class ::Thread
   # @param args [Array] arguments to pass to thread block
   def initialize(*args, &block)
     @join_wait_queue = []
-    @finalization_mutex = Mutex.new
     @args = args
     @block = block
     orig_initialize { execute }
@@ -41,16 +40,7 @@ class ::Thread
   # @param timeout [Number] timeout interval
   # @return [any] thread's return value
   def join(timeout = nil)
-    watcher = Fiber.current.auto_watcher
-
-    @finalization_mutex.synchronize do
-      if @terminated
-        @result.is_a?(Exception) ? (raise @result) : (return @result)
-      else
-        @join_wait_queue << watcher
-      end
-    end
-    timeout ? move_on_after(timeout) { watcher.await } : watcher.await
+    timeout ? move_on_after(timeout) { await_done } : await_done
   end
   alias_method :await, :join
 
@@ -62,13 +52,14 @@ class ::Thread
   #
   # @param error [Exception, Class, nil] exception spec
   def raise(error = nil)
-    Thread.pass until @main_fiber
-    error = RuntimeError.new if error.nil?
-    error = RuntimeError.new(error) if error.is_a?(String)
-    error = error.new if error.is_a?(Class)
-
     sleep 0.0001 until @ready
-    main_fiber&.raise(error)
+
+    error = Exception.instantiate(error)
+    if Thread.current == self
+      Kernel.raise(error)
+    else
+      @main_fiber&.raise(error)
+    end
   end
 
   # @!visibility private
@@ -77,11 +68,7 @@ class ::Thread
   # Terminates the thread.
   #
   # @return [Thread] self
-  def kill
-    return self if @terminated
-
-    raise Polyphony::Terminate
-  end
+  alias_method :kill, :kill_safe
 
   # @!visibility private
   alias_method :orig_inspect, :inspect
@@ -128,6 +115,11 @@ class ::Thread
     backend.idle_proc = block
   end
 
+  def value
+    join
+    @result.is_a?(Exception) ? raise(@result) : @result
+  end
+
   private
 
   # Runs the thread's block, handling any uncaught exceptions.
@@ -159,13 +151,13 @@ class ::Thread
   #
   # @param result [any] thread's return value
   def finalize(result)
+    # We need to make sure the fiber is not on the runqueue. This, in order to
+    # prevent a race condition between #finalize and #kill.
+    fiber_unschedule(Fiber.current)
     Fiber.current.shutdown_all_children if !Fiber.current.children.empty?
 
-    @finalization_mutex.synchronize do
-      @terminated = true
-      @result = result
-      signal_waiters(result)
-    end
+    @result = result
+    mark_as_done(result)
     @backend&.finalize
   end
 

data/lib/polyphony/version.rb

@@ -2,5 +2,5 @@
 
 module Polyphony
   # @!visibility private
-  VERSION = '1.5'
+  VERSION = '1.6'
 end

data/lib/polyphony.rb

@@ -14,6 +14,7 @@ require_relative './polyphony/core/sync'
 require_relative './polyphony/core/timer'
 require_relative './polyphony/net'
 require_relative './polyphony/adapters/process'
+require_relative './polyphony/adapters/open3'
 
 # Polyphony API
 module Polyphony
@@ -119,10 +120,10 @@ module Polyphony
       # processes,) we use a separate mechanism to terminate fibers in forked
       # processes (see Polyphony.fork).
       at_exit do
-        next unless @original_pid == ::Process.pid
-
-        terminate_threads
-        Fiber.current.shutdown_all_children
+        if @original_pid == ::Process.pid
+          terminate_threads
+          Fiber.current.shutdown_all_children
+        end
       end
     end
   end
@@ -131,12 +132,17 @@ module Polyphony
   verbose = $VERBOSE
   $VERBOSE = nil
   Object.const_set(:Queue, Polyphony::Queue)
+  Thread.const_set(:Queue, Polyphony::Queue)
+
   Object.const_set(:Mutex, Polyphony::Mutex)
+  Thread.const_set(:Mutex, Polyphony::Mutex)
 
   require 'monitor'
-  Object.const_set(:Monitor, Polyphony::Mutex)
+  Object.const_set(:Monitor, Polyphony::Monitor)
 
   Object.const_set(:ConditionVariable, Polyphony::ConditionVariable)
+  Thread.const_set(:ConditionVariable, Polyphony::ConditionVariable)
+
   $VERBOSE = verbose
 
   install_terminating_signal_handlers

data/test/helper.rb

@@ -17,6 +17,14 @@ require 'minitest/autorun'
 IS_LINUX = RUBY_PLATFORM =~ /linux/
 
 module ::Kernel
+  def debug(**h)
+    k, v = h.first
+    h.delete(k)
+
+    rest = h.inject(+'') { |s, (k, v)| s << "  #{k}: #{v.inspect}\n" }
+    STDOUT.orig_write("#{k}=>#{v} #{caller[0]}\n#{rest}")
+  end
+
   def trace(*args)
     STDOUT.orig_write(format_trace(args))
   end
@@ -41,20 +49,24 @@ end
 class MiniTest::Test
   def setup
     # trace "* setup #{self.name}"
-    Fiber.current.setup_main_fiber
+    @__stamp = Time.now
     Thread.current.backend.finalize
     Thread.current.backend = Polyphony::Backend.new
-    sleep 0.001
-    @__stamp = Time.now
+    Fiber.current.setup_main_fiber
+    Fiber.current.instance_variable_set(:@auto_watcher, nil)
+    sleep 0.0001
   end
 
   def teardown
-    # trace "* teardown #{self.name} (#{Time.now - @__stamp}s)"
+    Polyphony::ThreadPool.reset
+
+    # trace "* teardown #{self.name} (#{@__stamp ? (Time.now - @__stamp) : '?'}s)"
     Fiber.current.shutdown_all_children
     if Fiber.current.children.size > 0
       puts "Children left after #{self.name}: #{Fiber.current.children.inspect}"
       exit!
     end
+    # trace "* teardown done"
   rescue => e
     puts e
     puts e.backtrace.join("\n")
@@ -79,6 +91,36 @@ module Minitest::Assertions
     msg = message(msg) { "Expected #{mu_pp(act)} to be in range #{mu_pp(exp_range)}" }
     assert exp_range.include?(act), msg
   end
+
+  def assert_join_threads(threads, message = nil)
+    errs = []
+    values = []
+    while th = threads.shift
+      begin
+        values << th.value
+      rescue Exception
+        errs << [th, $!]
+        th = nil
+      end
+    end
+    values
+  ensure
+    if th&.alive?
+      th.raise(Timeout::Error.new)
+      th.join rescue errs << [th, $!]
+    end
+    if !errs.empty?
+      msg = "exceptions on #{errs.length} threads:\n" +
+        errs.map {|t, err|
+        "#{t.inspect}:\n" +
+          (err.respond_to?(:full_message) ? err.full_message(highlight: false, order: :top) : err.message)
+      }.join("\n---\n")
+      if message
+        msg = "#{message}\n#{msg}"
+      end
+      raise MiniTest::Assertion, msg
+    end
+  end
 end
 
 puts "Polyphony backend: #{Thread.current.backend.kind}"

data/test/open3/envutil.rb

@@ -0,0 +1,380 @@
+# Adapted from https://github.com/ruby/open3/blob/master/test/lib/envutil.rb
+
+# -*- coding: us-ascii -*-
+# frozen_string_literal: true
+require_relative "find_executable"
+begin
+  require 'rbconfig'
+rescue LoadError
+end
+begin
+  require "rbconfig/sizeof"
+rescue LoadError
+end
+
+module EnvUtil
+  def rubybin
+    if ruby = ENV["RUBY"]
+      return ruby
+    end
+    ruby = "ruby"
+    exeext = RbConfig::CONFIG["EXEEXT"]
+    rubyexe = (ruby + exeext if exeext and !exeext.empty?)
+    3.times do
+      if File.exist? ruby and File.executable? ruby and !File.directory? ruby
+        return File.expand_path(ruby)
+      end
+      if rubyexe and File.exist? rubyexe and File.executable? rubyexe
+        return File.expand_path(rubyexe)
+      end
+      ruby = File.join("..", ruby)
+    end
+    if defined?(RbConfig.ruby)
+      RbConfig.ruby
+    else
+      "ruby"
+    end
+  end
+  module_function :rubybin
+
+  LANG_ENVS = %w"LANG LC_ALL LC_CTYPE"
+
+  DEFAULT_SIGNALS = Signal.list
+  DEFAULT_SIGNALS.delete("TERM") if /mswin|mingw/ =~ RUBY_PLATFORM
+
+  RUBYLIB = ENV["RUBYLIB"]
+
+  class << self
+    attr_accessor :timeout_scale
+    attr_reader :original_internal_encoding, :original_external_encoding,
+                :original_verbose, :original_warning
+
+    def capture_global_values
+      @original_internal_encoding = Encoding.default_internal
+      @original_external_encoding = Encoding.default_external
+      @original_verbose = $VERBOSE
+      @original_warning = defined?(Warning.[]) ? %i[deprecated experimental].to_h {|i| [i, Warning[i]]} : nil
+    end
+  end
+
+  def apply_timeout_scale(t)
+    if scale = EnvUtil.timeout_scale
+      t * scale
+    else
+      t
+    end
+  end
+  module_function :apply_timeout_scale
+
+  def timeout(sec, klass = nil, message = nil, &blk)
+    return yield(sec) if sec == nil or sec.zero?
+    sec = apply_timeout_scale(sec)
+    Timeout.timeout(sec, klass, message, &blk)
+  end
+  module_function :timeout
+
+  def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1)
+    reprieve = apply_timeout_scale(reprieve) if reprieve
+
+    signals = Array(signal).select do |sig|
+      DEFAULT_SIGNALS[sig.to_s] or
+        DEFAULT_SIGNALS[Signal.signame(sig)] rescue false
+    end
+    signals |= [:ABRT, :KILL]
+    case pgroup
+    when 0, true
+      pgroup = -pid
+    when nil, false
+      pgroup = pid
+    end
+
+    lldb = true if /darwin/ =~ RUBY_PLATFORM
+
+    while signal = signals.shift
+
+      if lldb and [:ABRT, :KILL].include?(signal)
+        lldb = false
+        # sudo -n: --non-interactive
+        # lldb -p: attach
+        #      -o: run command
+        system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit])
+        true
+      end
+
+      begin
+        Process.kill signal, pgroup
+      rescue Errno::EINVAL
+        next
+      rescue Errno::ESRCH
+        break
+      end
+      if signals.empty? or !reprieve
+        Process.wait(pid)
+      else
+        begin
+          Timeout.timeout(reprieve) {Process.wait(pid)}
+        rescue Timeout::Error
+        else
+          break
+        end
+      end
+    end
+    $?
+  end
+  module_function :terminate
+
+  def invoke_ruby(args, stdin_data = "", capture_stdout = false, capture_stderr = false,
+                  encoding: nil, timeout: 10, reprieve: 1, timeout_error: Timeout::Error,
+                  stdout_filter: nil, stderr_filter: nil, ios: nil,
+                  signal: :TERM,
+                  rubybin: EnvUtil.rubybin, precommand: nil,
+                  **opt)
+    timeout = apply_timeout_scale(timeout)
+
+    in_c, in_p = IO.pipe
+    out_p, out_c = IO.pipe if capture_stdout
+    err_p, err_c = IO.pipe if capture_stderr && capture_stderr != :merge_to_stdout
+    opt[:in] = in_c
+    opt[:out] = out_c if capture_stdout
+    opt[:err] = capture_stderr == :merge_to_stdout ? out_c : err_c if capture_stderr
+    if encoding
+      out_p.set_encoding(encoding) if out_p
+      err_p.set_encoding(encoding) if err_p
+    end
+    ios.each {|i, o = i|opt[i] = o} if ios
+
+    c = "C"
+    child_env = {}
+    LANG_ENVS.each {|lc| child_env[lc] = c}
+    if Array === args and Hash === args.first
+      child_env.update(args.shift)
+    end
+    if RUBYLIB and lib = child_env["RUBYLIB"]
+      child_env["RUBYLIB"] = [lib, RUBYLIB].join(File::PATH_SEPARATOR)
+    end
+
+    # remain env
+    %w(ASAN_OPTIONS RUBY_ON_BUG).each{|name|
+      child_env[name] = ENV[name] if ENV[name]
+    }
+
+    args = [args] if args.kind_of?(String)
+    pid = spawn(child_env, *precommand, rubybin, *args, opt)
+    in_c.close
+    out_c&.close
+    out_c = nil
+    err_c&.close
+    err_c = nil
+    if block_given?
+      return yield in_p, out_p, err_p, pid
+    else
+      th_stdout = spin { out_p.read } if capture_stdout
+      th_stderr = spin { err_p.read } if capture_stderr && capture_stderr != :merge_to_stdout
+      in_p.write stdin_data.to_str unless stdin_data.empty?
+      in_p.close
+      if (!th_stdout || th_stdout.join(timeout)) && (!th_stderr || th_stderr.join(timeout))
+        timeout_error = nil
+      else
+        status = terminate(pid, signal, opt[:pgroup], reprieve)
+        terminated = Time.now
+      end
+      stdout = th_stdout.value if capture_stdout
+      stderr = th_stderr.value if capture_stderr && capture_stderr != :merge_to_stdout
+      out_p.close if capture_stdout
+      err_p.close if capture_stderr && capture_stderr != :merge_to_stdout
+      status ||= Process.wait2(pid)[1]
+      stdout = stdout_filter.call(stdout) if stdout_filter
+      stderr = stderr_filter.call(stderr) if stderr_filter
+      if timeout_error
+        bt = caller_locations
+        msg = "execution of #{bt.shift.label} expired timeout (#{timeout} sec)"
+        msg = failure_description(status, terminated, msg, [stdout, stderr].join("\n"))
+        raise timeout_error, msg, bt.map(&:to_s)
+      end
+      return stdout, stderr, status
+    end
+  ensure
+    [th_stdout, th_stderr].each do |th|
+      th.kill if th
+    end
+    [in_c, in_p, out_c, out_p, err_c, err_p].each do |io|
+      io&.close
+    end
+    [th_stdout, th_stderr].each do |th|
+      th.join if th
+    end
+  end
+  module_function :invoke_ruby
+
+  def verbose_warning
+    class << (stderr = "".dup)
+      alias write concat
+      def flush; end
+    end
+    stderr, $stderr = $stderr, stderr
+    $VERBOSE = true
+    yield stderr
+    return $stderr
+  ensure
+    stderr, $stderr = $stderr, stderr
+    $VERBOSE = EnvUtil.original_verbose
+    EnvUtil.original_warning&.each {|i, v| Warning[i] = v}
+  end
+  module_function :verbose_warning
+
+  def default_warning
+    $VERBOSE = false
+    yield
+  ensure
+    $VERBOSE = EnvUtil.original_verbose
+  end
+  module_function :default_warning
+
+  def suppress_warning
+    $VERBOSE = nil
+    yield
+  ensure
+    $VERBOSE = EnvUtil.original_verbose
+  end
+  module_function :suppress_warning
+
+  def under_gc_stress(stress = true)
+    stress, GC.stress = GC.stress, stress
+    yield
+  ensure
+    GC.stress = stress
+  end
+  module_function :under_gc_stress
+
+  def with_default_external(enc)
+    suppress_warning { Encoding.default_external = enc }
+    yield
+  ensure
+    suppress_warning { Encoding.default_external = EnvUtil.original_external_encoding }
+  end
+  module_function :with_default_external
+
+  def with_default_internal(enc)
+    suppress_warning { Encoding.default_internal = enc }
+    yield
+  ensure
+    suppress_warning { Encoding.default_internal = EnvUtil.original_internal_encoding }
+  end
+  module_function :with_default_internal
+
+  def labeled_module(name, &block)
+    Module.new do
+      singleton_class.class_eval {
+        define_method(:to_s) {name}
+        alias inspect to_s
+        alias name to_s
+      }
+      class_eval(&block) if block
+    end
+  end
+  module_function :labeled_module
+
+  def labeled_class(name, superclass = Object, &block)
+    Class.new(superclass) do
+      singleton_class.class_eval {
+        define_method(:to_s) {name}
+        alias inspect to_s
+        alias name to_s
+      }
+      class_eval(&block) if block
+    end
+  end
+  module_function :labeled_class
+
+  if /darwin/ =~ RUBY_PLATFORM
+    DIAGNOSTIC_REPORTS_PATH = File.expand_path("~/Library/Logs/DiagnosticReports")
+    DIAGNOSTIC_REPORTS_TIMEFORMAT = '%Y-%m-%d-%H%M%S'
+    @ruby_install_name = RbConfig::CONFIG['RUBY_INSTALL_NAME']
+
+    def self.diagnostic_reports(signame, pid, now)
+      return unless %w[ABRT QUIT SEGV ILL TRAP].include?(signame)
+      cmd = File.basename(rubybin)
+      cmd = @ruby_install_name if "ruby-runner#{RbConfig::CONFIG["EXEEXT"]}" == cmd
+      path = DIAGNOSTIC_REPORTS_PATH
+      timeformat = DIAGNOSTIC_REPORTS_TIMEFORMAT
+      pat = "#{path}/#{cmd}_#{now.strftime(timeformat)}[-_]*.{crash,ips}"
+      first = true
+      30.times do
+        first ? (first = false) : sleep(0.1)
+        Dir.glob(pat) do |name|
+          log = File.read(name) rescue next
+          case name
+          when /\.crash\z/
+            if /\AProcess:\s+#{cmd} \[#{pid}\]$/ =~ log
+              File.unlink(name)
+              File.unlink("#{path}/.#{File.basename(name)}.plist") rescue nil
+              return log
+            end
+          when /\.ips\z/
+            if /^ *"pid" *: *#{pid},/ =~ log
+              File.unlink(name)
+              return log
+            end
+          end
+        end
+      end
+      nil
+    end
+  else
+    def self.diagnostic_reports(signame, pid, now)
+    end
+  end
+
+  def self.failure_description(status, now, message = "", out = "")
+    pid = status.pid
+    if signo = status.termsig
+      signame = Signal.signame(signo)
+      sigdesc = "signal #{signo}"
+    end
+    log = diagnostic_reports(signame, pid, now)
+    if signame
+      sigdesc = "SIG#{signame} (#{sigdesc})"
+    end
+    if status.coredump?
+      sigdesc = "#{sigdesc} (core dumped)"
+    end
+    full_message = ''.dup
+    message = message.call if Proc === message
+    if message and !message.empty?
+      full_message << message << "\n"
+    end
+    full_message << "pid #{pid}"
+    full_message << " exit #{status.exitstatus}" if status.exited?
+    full_message << " killed by #{sigdesc}" if sigdesc
+    if out and !out.empty?
+      full_message << "\n" << out.b.gsub(/^/, '| ')
+      full_message.sub!(/(?<!\n)\z/, "\n")
+    end
+    if log
+      full_message << "Diagnostic reports:\n" << log.b.gsub(/^/, '| ')
+    end
+    full_message
+  end
+
+  def self.gc_stress_to_class?
+    unless defined?(@gc_stress_to_class)
+      _, _, status = invoke_ruby(["-e""exit GC.respond_to?(:add_stress_to_class)"])
+      @gc_stress_to_class = status.success?
+    end
+    @gc_stress_to_class
+  end
+end
+
+if defined?(RbConfig)
+  module RbConfig
+    @ruby = EnvUtil.rubybin
+    class << self
+      undef ruby if method_defined?(:ruby)
+      attr_reader :ruby
+    end
+    dir = File.dirname(ruby)
+    CONFIG['bindir'] = dir
+  end
+end
+
+EnvUtil.capture_global_values

data/test/open3/find_executable.rb

@@ -0,0 +1,24 @@
+# Adapted from https://github.com/ruby/open3/blob/master/test/lib/find_executable.rb
+
+# frozen_string_literal: true
+require "rbconfig"
+
+module EnvUtil
+  def find_executable(cmd, *args)
+    exts = RbConfig::CONFIG["EXECUTABLE_EXTS"].split | [RbConfig::CONFIG["EXEEXT"]]
+    ENV["PATH"].split(File::PATH_SEPARATOR).each do |path|
+      next if path.empty?
+      path = File.join(path, cmd)
+      exts.each do |ext|
+        cmdline = [path + ext, *args]
+        begin
+          return cmdline if yield(IO.popen(cmdline, "r", err: [:child, :out], &:read))
+        rescue
+          next
+        end
+      end
+    end
+    nil
+  end
+  module_function :find_executable
+end