polyphony 0.43.8

data/lib/polyphony/core/resource_pool.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Polyphony
+  # Implements a limited resource pool
+  class ResourcePool
+    attr_reader :limit, :size
+
+    # Initializes a new resource pool
+    # @param opts [Hash] options
+    # @param &block [Proc] allocator block
+    def initialize(opts, &block)
+      @allocator = block
+
+      @stock = []
+      @queue = []
+      @acquired_resources = {}
+
+      @limit = opts[:limit] || 4
+      @size = 0
+    end
+
+    def available
+      @stock.size
+    end
+
+    def acquire
+      fiber = Fiber.current
+      if @acquired_resources[fiber]
+        yield @acquired_resources[fiber]
+      else
+        begin
+          Thread.current.agent.ref
+          resource = wait_for_resource
+          return unless resource
+          
+          @acquired_resources[fiber] = resource
+          yield resource
+        ensure
+          @acquired_resources.delete fiber
+          Thread.current.agent.unref
+          release(resource) if resource
+        end
+      end
+    end
+        
+    def wait_for_resource
+      fiber = Fiber.current
+      @queue << fiber
+      ready_resource = from_stock
+      return ready_resource if ready_resource
+
+      suspend
+    ensure
+      @queue.delete(fiber)
+    end
+
+    def release(resource)
+      if resource.__discarded__
+        @size -= 1
+      elsif resource
+        return_to_stock(resource)
+        dequeue
+      end
+    end
+
+    def dequeue
+      return if @queue.empty? || @stock.empty?
+
+      @queue.shift.schedule(@stock.shift)
+    end
+
+    def return_to_stock(resource)
+      @stock << resource
+    end
+
+    def from_stock
+      @stock.shift || (@size < @limit && allocate)
+    end
+
+    def method_missing(sym, *args, &block)
+      acquire { |r| r.send(sym, *args, &block) }
+    end
+
+    def respond_to_missing?(*_args)
+      true
+    end
+
+    # Extension to allow discarding of resources
+    module ResourceExtensions
+      def __discarded__
+        @__discarded__
+      end
+
+      def __discard__
+        @__discarded__ = true
+      end
+    end
+
+    # Allocates a resource
+    # @return [any] allocated resource
+    def allocate
+      @size += 1
+      @allocator.().tap { |r| r.extend ResourceExtensions }
+    end
+
+    def <<(resource)
+      @size += 1
+      resource.extend ResourceExtensions
+      @stock << resource
+      dequeue
+    end
+
+    def preheat!
+      (@limit - @size).times { @stock << allocate }
+    end
+  end
+end

data/lib/polyphony/core/sync.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Polyphony
+  # Implements mutex lock for synchronizing access to a shared resource
+  class Mutex
+    def initialize
+      @waiting_fibers = Polyphony::Queue.new
+    end
+
+    def synchronize
+      fiber = Fiber.current
+      @waiting_fibers << fiber
+      suspend if @waiting_fibers.size > 1
+      yield
+    ensure
+      @waiting_fibers.delete(fiber)
+      @waiting_fibers.first&.schedule
+      snooze
+    end
+  end
+end

data/lib/polyphony/core/thread_pool.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'etc'
+
+module Polyphony
+  # Implements a pool of threads
+  class ThreadPool
+    attr_reader :size
+
+    def self.process(&block)
+      @default_pool ||= new
+      @default_pool.process(&block)
+    end
+
+    def self.reset
+      return unless @default_pool
+
+      @default_pool.stop
+      @default_pool = nil
+    end
+
+    def initialize(size = Etc.nprocessors)
+      @size = size
+      @task_queue = Polyphony::Queue.new
+      @threads = (1..@size).map { Thread.new { thread_loop } }
+    end
+
+    def process(&block)
+      setup unless @task_queue
+
+      watcher = Fiber.current.auto_watcher
+      @task_queue << [block, watcher]
+      watcher.await
+    end
+
+    def cast(&block)
+      setup unless @task_queue
+
+      @task_queue << [block, nil]
+      self
+    end
+
+    def busy?
+      !@task_queue.empty?
+    end
+
+    def thread_loop
+      loop { run_queued_task }
+    end
+
+    def run_queued_task
+      (block, watcher) = @task_queue.shift
+      result = block.()
+      watcher&.signal(result)
+    rescue Exception => e
+      watcher ? watcher.signal(e) : raise(e)
+    end
+
+    def stop
+      @threads.each(&:kill)
+      @threads.each(&:join)
+    end
+  end
+end

data/lib/polyphony/core/throttler.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Polyphony
+  # Implements general-purpose throttling
+  class Throttler
+    def initialize(rate)
+      @rate = rate_from_argument(rate)
+      @min_dt = 1.0 / @rate
+      @next_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+
+    def call
+      now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      delta = @next_time - now
+      Thread.current.agent.sleep(delta) if delta > 0
+      yield self
+
+      loop do
+        @next_time += @min_dt
+        break if @next_time > now
+      end
+    end
+    alias_method :process, :call
+
+    def stop
+      @stop = true
+    end
+
+    private
+
+    def rate_from_argument(arg)
+      return arg if arg.is_a?(Numeric)
+
+      if arg.is_a?(Hash)
+        return 1.0 / arg[:interval] if arg[:interval]
+        return arg[:rate] if arg[:rate]
+      end
+      raise "Invalid rate argument #{arg.inspect}"
+    end
+  end
+end

data/lib/polyphony/event.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Polyphony
+  # Event watcher for thread-safe synchronisation
+  class Event
+    def await
+      @fiber = Fiber.current
+      Thread.current.agent.wait_event(true)
+    end
+
+    def signal(value = nil)
+      @fiber&.schedule(value)
+    ensure
+      @fiber = nil
+    end
+  end
+end

data/lib/polyphony/extensions/core.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'fiber'
+require 'timeout'
+require 'open3'
+
+require_relative '../core/exceptions'
+
+# Exeption overrides
+class ::Exception
+  class << self
+    attr_accessor :__disable_sanitized_backtrace__
+  end
+
+  attr_accessor :source_fiber
+
+  alias_method :orig_initialize, :initialize
+  def initialize(*args)
+    @__raising_fiber__ = Fiber.current
+    orig_initialize(*args)
+  end
+
+  alias_method :orig_backtrace, :backtrace
+  def backtrace
+    unless @first_backtrace_call
+      @first_backtrace_call = true
+      return orig_backtrace
+    end
+
+    sanitized_backtrace
+  end
+
+  def sanitized_backtrace
+    if @__raising_fiber__
+      backtrace = orig_backtrace || []
+      sanitize(backtrace + @__raising_fiber__.caller)
+    else
+      sanitize(orig_backtrace)
+    end
+  end
+
+  POLYPHONY_DIR = File.expand_path(File.join(__dir__, '..'))
+
+  def sanitize(backtrace)
+    return backtrace if ::Exception.__disable_sanitized_backtrace__
+
+    backtrace.reject { |l| l[POLYPHONY_DIR] }
+  end
+end
+
+# Overrides for Process
+module ::Process
+  class << self
+    alias_method :orig_detach, :detach
+    def detach(pid)
+      fiber = spin { Thread.current.agent.waitpid(pid) }
+      fiber.define_singleton_method(:pid) { pid }
+      fiber
+    end
+
+    alias_method :orig_daemon, :daemon
+    def daemon(*args)
+      orig_daemon(*args)
+      Polyphony.original_pid = Process.pid
+    end
+  end
+end
+
+# Kernel extensions (methods available to all objects / call sites)
+module ::Kernel
+  alias_method :orig_sleep, :sleep
+
+  alias_method :orig_backtick, :`
+  def `(cmd)
+    Open3.popen3(cmd) do |i, o, e, _t|
+      i.close
+      err = e.read
+      $stderr << err if err
+      o.read || ''
+    end
+  end
+
+  ARGV_GETS_LOOP = proc do |calling_fiber|
+    while (fn = ARGV.shift)
+      File.open(fn, 'r') do |f|
+        while (line = f.gets)
+          calling_fiber = calling_fiber.transfer(line)
+        end
+      end
+    end
+    nil
+  rescue Exception => e
+    calling_fiber.transfer(e)
+  end
+
+  alias_method :orig_gets, :gets
+  def gets(*_args)
+    if !ARGV.empty? || @gets_fiber
+      @gets_fiber ||= Fiber.new(&ARGV_GETS_LOOP)
+      @gets_fiber.thread = Thread.current
+      result = @gets_fiber.alive? && @gets_fiber.safe_transfer(Fiber.current)
+      return result if result
+
+      @gets_fiber = nil
+    end
+
+    $stdin.gets
+  end
+
+  alias_method :orig_p, :p
+  def p(*args)
+    strs = args.inject([]) do |m, a|
+      m << a.inspect << "\n"
+    end
+    STDOUT.write *strs
+    args.size == 1 ? args.first : args
+  end
+
+  alias_method :orig_system, :system
+  def system(*args)
+    Open3.popen2(*args) do |i, o, _t|
+      i.close
+      pipe_to_eof(o, $stdout)
+    end
+    true
+  rescue SystemCallError
+    nil
+  end
+
+  def pipe_to_eof(src, dest)
+    loop do
+      data = src.readpartial(8192)
+      dest << data
+    rescue EOFError
+      break
+    end
+  end
+
+  alias_method :orig_trap, :trap
+  def trap(sig, command = nil, &block)
+    return orig_trap(sig, command) if command.is_a? String
+      
+    block = command if command.respond_to?(:call) && !block
+    exception = command.is_a?(Class) && command.new
+
+    # The signal trap can be invoked at any time, including while the system
+    # agent is blocking while polling for events. In order to deal with this
+    # correctly, we spin a fiber that will run the signal handler code, then
+    # call break_out_of_ev_loop, which will put the fiber at the front of the
+    # run queue, then wake up the system agent.
+    #
+    # If the command argument is an exception class however, it will be raised
+    # directly in the context of the main fiber.
+    orig_trap(sig) do
+      if exception
+        Thread.current.break_out_of_ev_loop(Thread.main.main_fiber, exception)
+      else
+        fiber = spin { snooze; block.call }
+        Thread.current.break_out_of_ev_loop(fiber, nil)
+      end
+    end
+  end
+end
+
+# Override Timeout to use cancel scope
+module ::Timeout
+  def self.timeout(sec, klass = nil, message = nil, &block)
+    cancel_after(sec, &block)
+  rescue Polyphony::Cancel => e
+    error = klass ? klass.new(message) : ::Timeout::Error.new
+    error.set_backtrace(e.backtrace)
+    raise error
+  end
+end

data/lib/polyphony/extensions/fiber.rb

@@ -0,0 +1,379 @@
+# frozen_string_literal: true
+
+require 'fiber'
+
+require_relative '../core/exceptions'
+
+module Polyphony
+  # Fiber control API
+  module FiberControl
+    def await
+      if @running == false
+        return @result.is_a?(Exception) ? (Kernel.raise @result) : @result
+      end
+
+      fiber = Fiber.current
+      @waiting_fibers ||= {}
+      @waiting_fibers[fiber] = true
+      suspend
+    ensure
+      @waiting_fibers&.delete(fiber)
+    end
+    alias_method :join, :await
+
+    def interrupt(value = nil)
+      return if @running == false
+
+      schedule Polyphony::MoveOn.new(value)
+    end
+    alias_method :stop, :interrupt
+
+    def restart(value = nil)
+      raise "Can''t restart main fiber" if @main
+
+      if @running
+        schedule Polyphony::Restart.new(value)
+        return self
+      end
+
+      parent.spin(@tag, @caller, &@block).tap do |f|
+        f.schedule(value) unless value.nil?
+      end
+    end
+    alias_method :reset, :restart
+
+    def cancel
+      return if @running == false
+
+      schedule Polyphony::Cancel.new
+    end
+
+    def terminate
+      return if @running == false
+
+      schedule Polyphony::Terminate.new
+    end
+
+    def raise(*args)
+      error = error_from_raise_args(args)
+      schedule(error)
+    end
+
+    def error_from_raise_args(args)
+      case (arg = args.shift)
+      when String then RuntimeError.new(arg)
+      when Class  then arg.new(args.shift)
+      when Exception then arg
+      else RuntimeError.new
+      end
+    end
+  end
+
+  # Fiber supervision
+  module FiberSupervision
+    def supervise(opts = {})
+      @counter = 0
+      @on_child_done = proc do |fiber, result|
+        self << fiber unless result.is_a?(Exception)
+      end
+      loop { supervise_perform(opts) }
+    ensure
+      @on_child_done = nil
+    end
+
+    def supervise_perform(opts)
+      fiber = receive
+      restart_fiber(fiber, opts) if fiber
+    rescue Polyphony::Restart
+      restart_all_children
+    rescue Exception => e
+      Kernel.raise e if e.source_fiber.nil? || e.source_fiber == self
+
+      restart_fiber(e.source_fiber, opts)
+    end
+
+    def restart_fiber(fiber, opts)
+      opts[:watcher]&.send [:restart, fiber]
+      case opts[:restart]
+      when true
+        fiber.restart
+      when :one_for_all
+        @children.keys.each(&:restart)
+      end
+    end
+  end
+
+  # Class methods for controlling fibers (namely await and select)
+  module FiberControlClassMethods
+    def await(*fibers)
+      return [] if fibers.empty?
+
+      state = setup_await_select_state(fibers)
+      await_setup_monitoring(fibers, state)
+      suspend
+      fibers.map(&:result)
+    ensure
+      await_select_cleanup(state)
+    end
+    alias_method :join, :await
+
+    def setup_await_select_state(fibers)
+      {
+        awaiter: Fiber.current,
+        pending: fibers.each_with_object({}) { |f, h| h[f] = true }
+      }
+    end
+
+    def await_setup_monitoring(fibers, state)
+      fibers.each do |f|
+        f.when_done { |r| await_fiber_done(f, r, state) }
+      end
+    end
+
+    def await_fiber_done(fiber, result, state)
+      state[:pending].delete(fiber)
+
+      if state[:cleanup]
+        state[:awaiter].schedule if state[:pending].empty?
+      elsif !state[:done] && (result.is_a?(Exception) || state[:pending].empty?)
+        state[:awaiter].schedule(result)
+        state[:done] = true
+      end
+    end
+
+    def await_select_cleanup(state)
+      return if state[:pending].empty?
+
+      terminate = Polyphony::Terminate.new
+      state[:cleanup] = true
+      state[:pending].each_key { |f| f.schedule(terminate) }
+      suspend
+    end
+
+    def select(*fibers)
+      state = setup_await_select_state(fibers)
+      select_setup_monitoring(fibers, state)
+      suspend
+    ensure
+      await_select_cleanup(state)
+    end
+
+    def select_setup_monitoring(fibers, state)
+      fibers.each do |f|
+        f.when_done { |r| select_fiber_done(f, r, state) }
+      end
+    end
+
+    def select_fiber_done(fiber, result, state)
+      state[:pending].delete(fiber)
+      if state[:cleanup]
+        # in cleanup mode the selector is resumed if no more pending fibers
+        state[:awaiter].schedule if state[:pending].empty?
+      elsif !state[:selected]
+        # first fiber to complete, we schedule the result
+        state[:awaiter].schedule([fiber, result])
+        state[:selected] = true
+      end
+    end
+  end
+
+  # Messaging functionality
+  module FiberMessaging
+    def <<(value)
+      @mailbox << value
+    end
+    alias_method :send, :<<
+
+    def receive
+      @mailbox.shift
+    end
+
+    def receive_pending
+      @mailbox.shift_all
+    end
+  end
+
+  # Methods for controlling child fibers
+  module ChildFiberControl
+    def children
+      (@children ||= {}).keys
+    end
+
+    def spin(tag = nil, orig_caller = Kernel.caller, &block)
+      f = Fiber.new { |v| f.run(v) }
+      f.prepare(tag, block, orig_caller, self)
+      (@children ||= {})[f] = true
+      f
+    end
+
+    def child_done(child_fiber, result)
+      @children.delete(child_fiber)
+      @on_child_done&.(child_fiber, result)
+    end
+
+    def terminate_all_children
+      return unless @children
+
+      e = Polyphony::Terminate.new
+      @children.each_key { |c| c.raise e }
+    end
+
+    def await_all_children
+      return unless @children && !@children.empty?
+
+      @results = @children.dup
+      @on_child_done = proc do |c, r|
+        @results[c] = r
+        self.schedule if @children.empty?
+      end
+      suspend
+      @on_child_done = nil
+      @results.values
+    end
+
+    def shutdown_all_children
+      terminate_all_children
+      await_all_children
+    end
+  end
+
+  # Fiber life cycle methods
+  module FiberLifeCycle
+    def prepare(tag, block, caller, parent)
+      @thread = Thread.current
+      @tag = tag
+      @parent = parent
+      @caller = caller
+      @block = block
+      @mailbox = Polyphony::Queue.new
+      __fiber_trace__(:fiber_create, self)
+      schedule
+    end
+
+    def run(first_value)
+      setup first_value
+      result = @block.(first_value)
+      finalize result
+    rescue Polyphony::Restart => e
+      restart_self(e.value)
+    rescue Polyphony::MoveOn, Polyphony::Terminate => e
+      finalize e.value
+    rescue Exception => e
+      e.source_fiber = self
+      finalize e, true
+    end
+
+    def setup(first_value)
+      Kernel.raise first_value if first_value.is_a?(Exception)
+
+      @running = true
+    end
+
+    # Performs setup for a "raw" Fiber created using Fiber.new. Note that this
+    # fiber is an orphan fiber (has no parent), since we cannot control how the
+    # fiber terminates after it has already been created. Calling #setup_raw
+    # allows the fiber to be scheduled and to receive messages.
+    def setup_raw
+      @thread = Thread.current
+      @mailbox = Polyphony::Queue.new
+    end
+
+    def setup_main_fiber
+      @main = true
+      @tag = :main
+      @thread = Thread.current
+      @running = true
+      @children&.clear
+      @mailbox = Polyphony::Queue.new
+    end
+
+    def restart_self(first_value)
+      @mailbox = Polyphony::Queue.new
+      @when_done_procs = nil
+      @waiting_fibers = nil
+      run(first_value)
+    end
+
+    def finalize(result, uncaught_exception = false)
+      result, uncaught_exception = finalize_children(result, uncaught_exception)
+      __fiber_trace__(:fiber_terminate, self, result)
+      @result = result
+      @running = false
+      inform_dependants(result, uncaught_exception)
+    ensure
+      Thread.current.switch_fiber
+    end
+
+    # Shuts down all children of the current fiber. If any exception occurs while
+    # the children are shut down, it is returned along with the uncaught_exception
+    # flag set. Otherwise, it returns the given arguments.
+    def finalize_children(result, uncaught_exception)
+      begin
+        shutdown_all_children
+      rescue Exception => e
+        result = e
+        uncaught_exception = true
+      end
+      [result, uncaught_exception]
+    end
+
+    def inform_dependants(result, uncaught_exception)
+      @parent&.child_done(self, result)
+      @when_done_procs&.each { |p| p.(result) }
+      @waiting_fibers&.each_key do |f|
+        f.schedule(result)
+      end
+      return unless uncaught_exception && !@waiting_fibers
+
+      # propagate unaught exception to parent
+      @parent&.schedule(result)
+    end
+
+    def when_done(&block)
+      @when_done_procs ||= []
+      @when_done_procs << block
+    end
+  end
+end
+
+# Fiber extensions
+class ::Fiber
+  prepend Polyphony::FiberControl
+  include Polyphony::FiberSupervision
+  include Polyphony::FiberMessaging
+  include Polyphony::ChildFiberControl
+  include Polyphony::FiberLifeCycle
+
+  extend Polyphony::FiberControlClassMethods
+
+  attr_accessor :tag, :thread, :parent
+  attr_reader :result
+
+  def running?
+    @running
+  end
+
+  def inspect
+    "#<Fiber:#{object_id} #{location} (#{state})>"
+  end
+  alias_method :to_s, :inspect
+
+  def location
+    @caller ? @caller[0] : '(root)'
+  end
+
+  def caller
+    spin_caller = @caller || []
+    if @parent
+      spin_caller + @parent.caller
+    else
+      spin_caller
+    end
+  end
+
+  def main?
+    @main
+  end
+end
+
+Fiber.current.setup_main_fiber