polyphony 0.43.11 → 0.45.4

data/lib/polyphony/adapters/fs.rb

@@ -2,7 +2,7 @@
 
 require 'fileutils'
 
-require_relative './core/thread_pool'
+require_relative '../core/thread_pool'
 
 ::File.singleton_class.instance_eval do
   alias_method :orig_stat, :stat

data/lib/polyphony/adapters/irb.rb

@@ -16,7 +16,7 @@ if Object.constants.include?(:Reline)
         fiber.cancel
       end
       read_ios.each do |io|
-        Thread.current.agent.wait_io(io, false)
+        Thread.current.backend.wait_io(io, false)
         return [io]
       end
     rescue Polyphony::Cancel
@@ -26,22 +26,7 @@ if Object.constants.include?(:Reline)
     end
   end
 else
-  # readline blocks the current thread, so we offload it to the blocking-ops
-  # thread pool. That way, the reactor loop can keep running while waiting for
-  # readline to return
-  module ::Readline
-    alias_method :orig_readline, :readline
-
-    Workers = Polyphony::ThreadPool.new
-
-    def readline(*args)
-      p :readline
-      # caller.each do |l|
-      #   STDOUT.orig_puts l
-      # end
-      Workers.process { orig_readline(*args) }
-    end
-  end
+  require_relative './readline'
 
   # RubyLex patches
   class ::RubyLex

data/lib/polyphony/adapters/mysql2.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative '../../polyphony'
+require 'mysql2/client'
+
+# Mysql2::Client overrides
+Mysql2::Client.prepend(Module.new do
+  def initialize(config)
+    config[:async] = true
+    super
+    @io = ::IO.for_fd(socket)
+  end
+
+  def query(sql, **options)
+    super
+    Thread.current.backend.wait_io(@io, false)
+    async_result
+  end
+end)

data/lib/polyphony/adapters/postgres.rb

@@ -15,8 +15,8 @@ module ::PG
       res = conn.connect_poll
       case res
       when PGRES_POLLING_FAILED   then raise Error, conn.error_message
-      when PGRES_POLLING_READING  then Thread.current.agent.wait_io(socket_io, false)
-      when PGRES_POLLING_WRITING  then Thread.current.agent.wait_io(socket_io, true)
+      when PGRES_POLLING_READING  then Thread.current.backend.wait_io(socket_io, false)
+      when PGRES_POLLING_WRITING  then Thread.current.backend.wait_io(socket_io, true)
       when PGRES_POLLING_OK       then return conn.setnonblocking(true)
       end
     end
@@ -42,7 +42,7 @@ class ::PG::Connection
 
   def get_result(&block)
     while is_busy
-      Thread.current.agent.wait_io(socket_io, false)
+      Thread.current.backend.wait_io(socket_io, false)
       consume_input
     end
     orig_get_result(&block)
@@ -59,7 +59,7 @@ class ::PG::Connection
 
   def block(_timeout = 0)
     while is_busy
-      Thread.current.agent.wait_io(socket_io, false)
+      Thread.current.backend.wait_io(socket_io, false)
       consume_input
     end
   end
@@ -97,7 +97,7 @@ class ::PG::Connection
     return move_on_after(timeout) { wait_for_notify(&block) } if timeout
 
     loop do
-      Thread.current.agent.wait_io(socket_io, false)
+      Thread.current.backend.wait_io(socket_io, false)
       consume_input
       notice = notifies
       next unless notice

data/lib/polyphony/adapters/process.rb

@@ -7,7 +7,7 @@ module Polyphony
       def watch(cmd = nil, &block)
         terminated = nil
         pid = cmd ? Kernel.spawn(cmd) : Polyphony.fork(&block)
-        Thread.current.agent.waitpid(pid)
+        Thread.current.backend.waitpid(pid)
         terminated = true
       ensure
         kill_process(pid) unless terminated || pid.nil?
@@ -23,10 +23,7 @@ module Polyphony
 
       def kill_and_await(sig, pid)
         ::Process.kill(sig, pid)
-        Thread.current.agent.waitpid(pid)
-      rescue SystemCallError
-        # ignore
-        puts 'SystemCallError in kill_and_await'
+        Thread.current.backend.waitpid(pid)
       end
     end
   end

data/lib/polyphony/adapters/readline.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'polyphony'
+require 'readline'
+
+# readline blocks the current thread, so we offload it to the blocking-ops
+# thread pool. That way, the reactor loop can keep running while waiting for
+# readline to return
+module ::Readline
+  alias_method :orig_readline, :readline
+
+  Worker = Polyphony::ThreadPool.new(1)
+
+  def readline(*args)
+    Worker.process { orig_readline(*args) }
+  end
+end

data/lib/polyphony/adapters/redis.rb

@@ -56,7 +56,7 @@ class Polyphony::RedisDriver
     reply = @reader.gets
     return reply if reply
 
-    while (data = @connection.readpartial(8192))
+    @connection.read_loop do |data|
       @reader.feed(data)
       reply = @reader.gets
       return reply unless reply == false

data/lib/polyphony/adapters/sequel.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../../polyphony'
+require 'sequel'
+
+module Polyphony
+  # Sequel ConnectionPool that delegates to Polyphony::ResourcePool.
+  class FiberConnectionPool < Sequel::ConnectionPool
+    def initialize(db, opts = OPTS)
+      super
+      max_size = Integer(opts[:max_connections] || 4)
+      @pool = Polyphony::ResourcePool.new(limit: max_size) { make_new(:default) }
+    end
+
+    def hold(_server = nil)
+      @pool.acquire do |conn|
+        yield conn
+      rescue Polyphony::BaseException
+        # The connection may be in an unrecoverable state if interrupted,
+        # discard the connection from the pool so it isn't reused.
+        @pool.discard!
+        raise
+      end
+    end
+
+    def size
+      @pool.size
+    end
+
+    def max_size
+      @pool.limit
+    end
+
+    def preconnect(_concurrent = false)
+      @pool.preheat!
+    end
+  end
+
+  # Override Sequel::Database to use FiberConnectionPool by default.
+  Sequel::Database.prepend(Module.new do
+    def connection_pool_default_options
+      { pool_class: FiberConnectionPool }
+    end
+  end)
+end

data/lib/polyphony/core/exceptions.rb

@@ -33,4 +33,15 @@ module Polyphony
 
   # Restart is used to restart a fiber
   class Restart < BaseException; end
+
+  # Interjection is used to run arbitrary code on arbitrary fibers at any point
+  class Interjection < BaseException
+    def initialize(proc)
+      @proc = proc
+    end
+
+    def invoke
+      @proc.call
+    end
+  end
 end

data/lib/polyphony/core/global_api.rb

@@ -19,13 +19,18 @@ module Polyphony
       fiber = ::Fiber.current
       canceller = spin do
         sleep interval
-        exception = with_exception.is_a?(Class) ?
-          with_exception.new : RuntimeError.new(with_exception)
+        exception = cancel_exception(with_exception)
         fiber.schedule exception
       end
       block ? cancel_after_wrap_block(canceller, &block) : canceller
     end
 
+    def cancel_exception(exception)
+      return exception.new if exception.is_a?(Class)
+
+      RuntimeError.new(exception)
+    end
+
     def cancel_after_wrap_block(canceller, &block)
       block.call
     ensure
@@ -50,7 +55,7 @@ module Polyphony
       next_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + interval
       loop do
         now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        Thread.current.agent.sleep(next_time - now)
+        Thread.current.backend.sleep(next_time - now)
         yield
         loop do
           next_time += interval
@@ -87,8 +92,8 @@ module Polyphony
       Fiber.current.receive
     end
 
-    def receive_pending
-      Fiber.current.receive_pending
+    def receive_all_pending
+      Fiber.current.receive_all_pending
     end
 
     def supervise(*args, &block)
@@ -98,20 +103,20 @@ module Polyphony
     def sleep(duration = nil)
       return sleep_forever unless duration
 
-      Thread.current.agent.sleep duration
+      Thread.current.backend.sleep duration
     end
 
     def sleep_forever
-      Thread.current.agent.ref
+      Thread.current.backend.ref
       loop { sleep 60 }
     ensure
-      Thread.current.agent.unref
+      Thread.current.backend.unref
     end
 
-    def throttled_loop(rate, count: nil, &block)
-      throttler = Polyphony::Throttler.new(rate)
-      if count
-        count.times { |_i| throttler.(&block) }
+    def throttled_loop(rate = nil, **opts, &block)
+      throttler = Polyphony::Throttler.new(rate || opts)
+      if opts[:count]
+        opts[:count].times { |_i| throttler.(&block) }
       else
         loop { throttler.(&block) }
       end

data/lib/polyphony/core/resource_pool.rb

@@ -20,19 +20,25 @@ module Polyphony
       @stock.size
     end
 
-    def acquire
+    def acquire(&block)
       fiber = Fiber.current
-      return @acquired_resources[fiber] if @acquired_resources[fiber]
+      return yield @acquired_resources[fiber] if @acquired_resources[fiber]
 
-      add_to_stock if (@stock.empty? || @stock.pending?) && @size < @limit 
+      acquire_from_stock(fiber, &block)
+    end
+
+    def acquire_from_stock(fiber)
+      add_to_stock if (@stock.empty? || @stock.pending?) && @size < @limit
       resource = @stock.shift
       @acquired_resources[fiber] = resource
       yield resource
     ensure
-      @acquired_resources.delete(fiber)
-      @stock.push resource if resource
+      if resource && @acquired_resources[fiber] == resource
+        @acquired_resources.delete(fiber)
+        @stock.push resource
+      end
     end
-        
+
     def method_missing(sym, *args, &block)
       acquire { |r| r.send(sym, *args, &block) }
     end
@@ -45,7 +51,14 @@ module Polyphony
     # @return [any] allocated resource
     def add_to_stock
       @size += 1
-      @stock << @allocator.call
+      resource = @allocator.call
+      @stock << resource
+    end
+
+    # Discards the currently-acquired resource
+    # instead of returning it to the pool when done.
+    def discard!
+      @size -= 1 if @acquired_resources.delete(Fiber.current)
     end
 
     def preheat!

data/lib/polyphony/core/sync.rb

@@ -8,16 +8,54 @@ module Polyphony
       @store << :token
     end
 
-    def synchronize
+    def synchronize(&block)
       return yield if @holding_fiber == Fiber.current
 
-      begin
-        token = @store.shift
-        @holding_fiber = Fiber.current
-        yield
-      ensure
-        @holding_fiber = nil
-        @store << token if token
+      synchronize_not_holding(&block)
+    end
+
+    def synchronize_not_holding
+      @token = @store.shift
+      @holding_fiber = Fiber.current
+      yield
+    ensure
+      @holding_fiber = nil
+      @store << @token if @token
+    end
+
+    def conditional_release
+      @store << @token
+      @token = nil
+      @holding_fiber = nil
+    end
+
+    def conditional_reacquire
+      @token = @store.shift
+      @holding_fiber = Fiber.current
+    end
+  end
+
+  # Implements a fiber-aware ConditionVariable
+  class ConditionVariable
+    def initialize
+      @queue = Polyphony::Queue.new
+    end
+
+    def wait(mutex, _timeout = nil)
+      mutex.conditional_release
+      @queue << Fiber.current
+      Thread.current.backend.wait_event(true)
+      mutex.conditional_reacquire
+    end
+
+    def signal
+      fiber = @queue.shift
+      fiber.schedule
+    end
+
+    def broadcast
+      while (fiber = @queue.shift)
+        fiber.schedule
       end
     end
   end

data/lib/polyphony/core/throttler.rb

@@ -12,7 +12,7 @@ module Polyphony
     def call
       now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       delta = @next_time - now
-      Thread.current.agent.sleep(delta) if delta > 0
+      Thread.current.backend.sleep(delta) if delta > 0
       yield self
 
       loop do

data/lib/polyphony/extensions/core.rb

@@ -46,6 +46,10 @@ class ::Exception
 
     backtrace.reject { |l| l[POLYPHONY_DIR] }
   end
+
+  def invoke
+    Kernel.raise(self)
+  end
 end
 
 # Overrides for Process
@@ -53,7 +57,7 @@ module ::Process
   class << self
     alias_method :orig_detach, :detach
     def detach(pid)
-      fiber = spin { Thread.current.agent.waitpid(pid) }
+      fiber = spin { Thread.current.backend.waitpid(pid) }
       fiber.define_singleton_method(:pid) { pid }
       fiber
     end
@@ -112,52 +116,48 @@ module ::Kernel
     strs = args.inject([]) do |m, a|
       m << a.inspect << "\n"
     end
-    STDOUT.write *strs
+    STDOUT.write(*strs)
     args.size == 1 ? args.first : args
   end
 
   alias_method :orig_system, :system
   def system(*args)
-    Open3.popen2(*args) do |i, o, _t|
-      i.close
-      pipe_to_eof(o, $stdout)
+    Kernel.system(*args)
+  end
+
+  class << self
+    alias_method :orig_system, :system
+    def system(*args)
+      waiter = nil
+      Open3.popen2(*args) do |i, o, t|
+        waiter = t
+        i.close
+        pipe_to_eof(o, $stdout)
+      end
+      waiter.await.last == 0
+    rescue SystemCallError
+      nil
     end
-    true
-  rescue SystemCallError
-    nil
   end
 
   def pipe_to_eof(src, dest)
-    loop do
-      data = src.readpartial(8192)
-      dest << data
-    rescue EOFError
-      break
-    end
+    src.read_loop { |data| dest << data }
   end
 
   alias_method :orig_trap, :trap
   def trap(sig, command = nil, &block)
     return orig_trap(sig, command) if command.is_a? String
-      
-    block = command if command.respond_to?(:call) && !block
-    exception = command.is_a?(Class) && command.new
+
+    block = command if !block && command.respond_to?(:call)
 
     # The signal trap can be invoked at any time, including while the system
-    # agent is blocking while polling for events. In order to deal with this
-    # correctly, we spin a fiber that will run the signal handler code, then
-    # call break_out_of_ev_loop, which will put the fiber at the front of the
-    # run queue, then wake up the system agent.
-    #
-    # If the command argument is an exception class however, it will be raised
-    # directly in the context of the main fiber.
+    # backend is blocking while polling for events. In order to deal with this
+    # correctly, we run the signal handler code in an out-of-band, priority
+    # scheduled fiber, that will pass any uncaught exception (including
+    # SystemExit and Interrupt) to the main thread's main fiber. See also
+    # `Fiber#schedule_priority_oob_fiber`.
     orig_trap(sig) do
-      if exception
-        Thread.current.break_out_of_ev_loop(Thread.main.main_fiber, exception)
-      else
-        fiber = spin { snooze; block.call }
-        Thread.current.break_out_of_ev_loop(fiber, nil)
-      end
+      Fiber.schedule_priority_oob_fiber(&block)
     end
   end
 end