polyphony 0.36 → 0.42

data/ext/polyphony/thread.c

@@ -0,0 +1,206 @@
+#include "polyphony.h"
+
+ID ID_deactivate_all_watchers_post_fork;
+ID ID_empty;
+ID ID_ivar_agent;
+ID ID_ivar_join_wait_queue;
+ID ID_ivar_main_fiber;
+ID ID_ivar_result;
+ID ID_ivar_terminated;
+ID ID_pop;
+ID ID_push;
+ID ID_run_queue;
+ID ID_runnable_next;
+ID ID_stop;
+
+static VALUE Thread_setup_fiber_scheduling(VALUE self) {
+  VALUE queue;
+  
+  rb_ivar_set(self, ID_ivar_main_fiber, rb_fiber_current());
+  queue = rb_ary_new();
+  rb_ivar_set(self, ID_run_queue, queue);
+
+  return self;
+}
+
+int Thread_fiber_ref_count(VALUE self) {
+  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+  return NUM2INT(LibevAgent_ref_count(agent));
+}
+
+inline void Thread_fiber_reset_ref_count(VALUE self) {
+  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+  LibevAgent_reset_ref_count(agent);
+}
+
+static VALUE SYM_scheduled_fibers;
+static VALUE SYM_pending_watchers;
+
+static VALUE Thread_fiber_scheduling_stats(VALUE self) {
+  VALUE agent = rb_ivar_get(self,ID_ivar_agent);
+  VALUE stats = rb_hash_new();
+  VALUE queue = rb_ivar_get(self, ID_run_queue);
+  long pending_count;
+
+  long scheduled_count = RARRAY_LEN(queue);
+  rb_hash_aset(stats, SYM_scheduled_fibers, INT2NUM(scheduled_count));
+
+  pending_count = LibevAgent_pending_count(agent);
+  rb_hash_aset(stats, SYM_pending_watchers, INT2NUM(pending_count));
+
+  return stats;
+}
+
+VALUE Thread_schedule_fiber(VALUE self, VALUE fiber, VALUE value) {
+  VALUE queue;
+
+  if (rb_fiber_alive_p(fiber) != Qtrue) return self;
+
+  FIBER_TRACE(3, SYM_fiber_schedule, fiber, value);
+  // if fiber is already scheduled, just set the scheduled value, then return
+  rb_ivar_set(fiber, ID_runnable_value, value);
+  if (rb_ivar_get(fiber, ID_runnable) != Qnil) {
+    return self;
+  }
+
+  queue = rb_ivar_get(self, ID_run_queue);
+  rb_ary_push(queue, fiber);
+  rb_ivar_set(fiber, ID_runnable, Qtrue);
+
+  if (rb_thread_current() != self) {
+    // if the fiber scheduling is done across threads, we need to make sure the
+    // target thread is woken up in case it is in the middle of running its
+    // event selector. Otherwise it's gonna be stuck waiting for an event to
+    // happen, not knowing that it there's already a fiber ready to run in its
+    // run queue.
+    VALUE agent = rb_ivar_get(self,ID_ivar_agent);
+    LibevAgent_break(agent);
+  }
+  return self;
+}
+
+VALUE Thread_schedule_fiber_with_priority(VALUE self, VALUE fiber, VALUE value) {
+  VALUE queue;
+
+  if (rb_fiber_alive_p(fiber) != Qtrue) return self;
+
+  FIBER_TRACE(3, SYM_fiber_schedule, fiber, value);
+  rb_ivar_set(fiber, ID_runnable_value, value);
+
+  queue = rb_ivar_get(self, ID_run_queue);
+
+  // if fiber is already scheduled, remove it from the run queue
+  if (rb_ivar_get(fiber, ID_runnable) != Qnil) {
+    rb_ary_delete(queue, fiber);
+  } else {
+    rb_ivar_set(fiber, ID_runnable, Qtrue);
+  }
+
+  // the fiber is given priority by putting it at the front of the run queue
+  rb_ary_unshift(queue, fiber);
+
+  if (rb_thread_current() != self) {
+    // if the fiber scheduling is done across threads, we need to make sure the
+    // target thread is woken up in case it is in the middle of running its
+    // event loop. Otherwise it's gonna be stuck waiting for an event to
+    // happen, not knowing that it there's already a fiber ready to run in its
+    // run queue.
+    VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+    LibevAgent_break(agent);
+  }
+  return self;
+}
+
+VALUE Thread_switch_fiber(VALUE self) {
+  VALUE current_fiber = rb_fiber_current();
+  VALUE queue = rb_ivar_get(self, ID_run_queue);
+  VALUE next_fiber;
+  VALUE value;
+  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+  int ref_count;
+
+  if (__tracing_enabled__) {
+    if (rb_ivar_get(current_fiber, ID_ivar_running) != Qfalse) {
+      rb_funcall(rb_cObject, ID_fiber_trace, 2, SYM_fiber_switchpoint, current_fiber);
+    }
+  }
+
+  ref_count = LibevAgent_ref_count(agent);
+  while (1) {
+    next_fiber = rb_ary_shift(queue);
+    if (next_fiber != Qnil) {
+      if (ref_count > 0) {
+        // this mechanism prevents event starvation in case the run queue never
+        // empties
+        LibevAgent_poll(agent, Qtrue, current_fiber, queue);
+      }
+      break;
+    }
+    if (ref_count == 0) break;
+
+    LibevAgent_poll(agent, Qnil, current_fiber, queue);
+  }
+
+  if (next_fiber == Qnil) return Qnil;
+
+  // run next fiber
+  value = rb_ivar_get(next_fiber, ID_runnable_value);
+  FIBER_TRACE(3, SYM_fiber_run, next_fiber, value);
+
+  rb_ivar_set(next_fiber, ID_runnable, Qnil);
+  RB_GC_GUARD(next_fiber);
+  RB_GC_GUARD(value);
+  return (next_fiber == current_fiber) ? 
+    value : rb_funcall(next_fiber, ID_transfer, 1, value);
+}
+
+VALUE Thread_reset_fiber_scheduling(VALUE self) {
+  VALUE queue = rb_ivar_get(self, ID_run_queue);
+  rb_ary_clear(queue);
+  Thread_fiber_reset_ref_count(self);
+  return self;
+}
+
+VALUE Thread_fiber_break_out_of_ev_loop(VALUE self, VALUE fiber, VALUE resume_obj) {
+  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+  if (fiber != Qnil) {
+    Thread_schedule_fiber_with_priority(self, fiber, resume_obj);
+  }
+
+  if (LibevAgent_break(agent) == Qnil) {
+    // we're not inside the ev_loop, so we just do a switchpoint
+    Thread_switch_fiber(self);
+  }
+
+  return self;
+}
+
+void Init_Thread() {
+  rb_define_method(rb_cThread, "setup_fiber_scheduling", Thread_setup_fiber_scheduling, 0);
+  rb_define_method(rb_cThread, "reset_fiber_scheduling", Thread_reset_fiber_scheduling, 0);
+  rb_define_method(rb_cThread, "fiber_scheduling_stats", Thread_fiber_scheduling_stats, 0);
+  rb_define_method(rb_cThread, "break_out_of_ev_loop", Thread_fiber_break_out_of_ev_loop, 2);
+
+  rb_define_method(rb_cThread, "schedule_fiber", Thread_schedule_fiber, 2);
+  rb_define_method(rb_cThread, "schedule_fiber_with_priority",
+    Thread_schedule_fiber_with_priority, 2);
+  rb_define_method(rb_cThread, "switch_fiber", Thread_switch_fiber, 0);
+
+  ID_deactivate_all_watchers_post_fork = rb_intern("deactivate_all_watchers_post_fork");
+  ID_empty                    = rb_intern("empty?");
+  ID_ivar_agent               = rb_intern("@agent");
+  ID_ivar_join_wait_queue     = rb_intern("@join_wait_queue");
+  ID_ivar_main_fiber          = rb_intern("@main_fiber");
+  ID_ivar_result              = rb_intern("@result");
+  ID_ivar_terminated          = rb_intern("@terminated");
+  ID_pop                      = rb_intern("pop");
+  ID_push                     = rb_intern("push");
+  ID_run_queue                = rb_intern("run_queue");
+  ID_runnable_next            = rb_intern("runnable_next");
+  ID_stop                     = rb_intern("stop");
+
+  SYM_scheduled_fibers = ID2SYM(rb_intern("scheduled_fibers"));
+  SYM_pending_watchers = ID2SYM(rb_intern("pending_watchers"));
+  rb_global_variable(&SYM_scheduled_fibers);
+  rb_global_variable(&SYM_pending_watchers);
+}

data/ext/{gyro → polyphony}/tracing.c

@@ -1,4 +1,4 @@
-#include "gyro.h"
+#include "polyphony.h"
 
 int __tracing_enabled__ = 0;
 

data/lib/polyphony.rb

@@ -1,62 +1,50 @@
 # frozen_string_literal: true
 
-require 'modulation/gem'
+require 'fiber'
+require_relative './polyphony_ext'
 
-export_default :Polyphony
+module Polyphony
+  # Map Queue to Libev queue implementation
+  Queue = LibevQueue
+end
 
-require 'fiber'
-require_relative './gyro_ext'
+require_relative './polyphony/extensions/core'
+require_relative './polyphony/extensions/thread'
+require_relative './polyphony/extensions/fiber'
+require_relative './polyphony/extensions/io'
 
-Thread.event_selector = Gyro::Selector
 Thread.current.setup_fiber_scheduling
+Thread.current.agent = Polyphony::LibevAgent.new
 
-import './polyphony/extensions/core'
-import './polyphony/extensions/thread'
-import './polyphony/extensions/fiber'
-import './polyphony/extensions/io'
+require_relative './polyphony/core/global_api'
+require_relative './polyphony/core/resource_pool'
+require_relative './polyphony/net'
+require_relative './polyphony/adapters/process'
+require_relative './polyphony/event'
 
 # Main Polyphony API
 module Polyphony
-  GlobalAPI = import './polyphony/core/global_api'
-  ::Object.include GlobalAPI
-
-  exceptions = import './polyphony/core/exceptions'
-  Cancel    = exceptions::Cancel
-  MoveOn    = exceptions::MoveOn
-  Restart   = exceptions::Restart
-  Terminate = exceptions::Terminate
-
-  Net = import './polyphony/net'
-
-  auto_import(
-    Channel:      './polyphony/core/channel',
-    FS:           './polyphony/adapters/fs',
-    Process:      './polyphony/adapters/process',
-    ResourcePool: './polyphony/core/resource_pool',
-    Sync:         './polyphony/core/sync',
-    ThreadPool:   './polyphony/core/thread_pool',
-    Throttler:    './polyphony/core/throttler',
-    Trace:        './polyphony/adapters/trace'
-  )
-
   class << self
     def wait_for_signal(sig)
+      raise "should be reimplemented"
+
       fiber = Fiber.current
-      Gyro.ref
+      # Polyphony.ref
       old_trap = trap(sig) do
-        Gyro.unref
+        # Polyphony.unref
         fiber.schedule(sig)
         trap(sig, old_trap)
       end
       suspend
+
     end
 
     def fork(&block)
       Kernel.fork do
-        # Since the fiber doing the fork will become the main fiber of the
-        # forked process, we leave it behind by transferring to a new fiber
-        # created in the context of the forked process, which rescues *all*
-        # exceptions, including Interrupt and SystemExit.
+        # # Since the fiber doing the fork will become the main fiber of the
+        # # forked process, we leave it behind by transferring to a new fiber
+        # # created in the context of the forked process, which rescues *all*
+        # # exceptions, including Interrupt and SystemExit.
         spin_forked_block(&block).transfer
       end
     end
@@ -64,12 +52,13 @@ module Polyphony
     def spin_forked_block(&block)
       Fiber.new do
         run_forked_block(&block)
-        exit_forked_process
-      rescue ::SystemExit
-        exit_forked_process
+      rescue SystemExit
+        # fall through to ensure
       rescue Exception => e
         e.full_message
         exit!
+      ensure
+        exit_forked_process
       end
     end
 
@@ -81,9 +70,9 @@ module Polyphony
       trap('SIGTERM', 'DEFAULT')
       trap('SIGINT', 'DEFAULT')
 
-      Thread.current.post_fork
       Thread.current.setup
       Fiber.current.setup_main_fiber
+      Thread.current.agent.post_fork
 
       install_terminating_signal_handlers
 
@@ -91,14 +80,13 @@ module Polyphony
     end
 
     def exit_forked_process
+      terminate_threads
       Fiber.current.shutdown_all_children
+
       # Since fork could be called from any fiber, we explicitly call exit here.
       # Otherwise, the fiber might want to pass execution to another fiber that
       # previously transferred execution to the forking fiber, but doesn't exist
       # anymore...
-      #
-      # The call to exit will invoke the at_exit handler we use to terminate the
-      # (forked) main fiber's child fibers.
       exit
     end
 
@@ -118,6 +106,14 @@ module Polyphony
       install_terminating_signal_handler('SIGTERM', ::SystemExit)
       install_terminating_signal_handler('SIGINT', ::Interrupt)
     end
+
+    def terminate_threads
+      threads = Thread.list - [Thread.current]
+      return if threads.empty?
+
+      threads.each(&:kill)
+      threads.each(&:join)
+    end
   end
 end
 

data/lib/polyphony/adapters/fs.rb

@@ -1,11 +1,8 @@
 # frozen_string_literal: true
 
-export  :stat,
-        :read
-
 require 'fileutils'
 
-ThreadPool = import('./core/thread_pool')
+require_relative './core/thread_pool'
 
 ::File.singleton_class.instance_eval do
   alias_method :orig_stat, :stat

data/lib/polyphony/adapters/irb.rb

@@ -16,7 +16,7 @@ if Object.constants.include?(:Reline)
         fiber.cancel
       end
       read_ios.each do |io|
-        io.read_watcher.await
+        Thread.current.agent.wait_io(io, false)
         return [io]
       end
     rescue Polyphony::Cancel

data/lib/polyphony/adapters/postgres.rb

@@ -10,12 +10,13 @@ module ::PG
   end
 
   def self.connect_async(conn)
+    socket_io = conn.socket_io
     loop do
       res = conn.connect_poll
       case res
       when PGRES_POLLING_FAILED   then raise Error, conn.error_message
-      when PGRES_POLLING_READING  then conn.socket_io.read_watcher.await
-      when PGRES_POLLING_WRITING  then conn.socket_io.write_watcher.await
+      when PGRES_POLLING_READING  then Thread.current.agent.wait_io(socket_io, false)
+      when PGRES_POLLING_WRITING  then Thread.current.agent.wait_io(socket_io, true)
       when PGRES_POLLING_OK       then return conn.setnonblocking(true)
       end
     end
@@ -41,7 +42,7 @@ class ::PG::Connection
 
   def get_result(&block)
     while is_busy
-      socket_io.read_watcher.await
+      Thread.current.agent.wait_io(socket_io, false)
       consume_input
     end
     orig_get_result(&block)
@@ -58,7 +59,7 @@ class ::PG::Connection
 
   def block(_timeout = 0)
     while is_busy
-      socket_io.read_watcher.await
+      Thread.current.agent.wait_io(socket_io, false)
       consume_input
     end
   end
@@ -96,7 +97,7 @@ class ::PG::Connection
     return move_on_after(timeout) { wait_for_notify(&block) } if timeout
 
     loop do
-      socket_io.read_watcher.await
+      Thread.current.agent.wait_io(socket_io, false)
       consume_input
       notice = notifies
       next unless notice

data/lib/polyphony/adapters/process.rb

@@ -1,29 +1,33 @@
 # frozen_string_literal: true
 
-export :watch
+module Polyphony
+  # Process patches
+  module Process
+    class << self
+      def watch(cmd = nil, &block)
+        terminated = nil
+        pid = cmd ? Kernel.spawn(cmd) : Polyphony.fork(&block)
+        Thread.current.agent.waitpid(pid)
+        terminated = true
+      ensure
+        kill_process(pid) unless terminated || pid.nil?
+      end
 
-def watch(cmd = nil, &block)
-  terminated = nil
-  pid = cmd ? Kernel.spawn(cmd) : Polyphony.fork(&block)
-  watcher = Gyro::Child.new(pid)
-  watcher.await
-  terminated = true
-ensure
-  kill_process(pid) unless terminated || pid.nil?
-end
+      def kill_process(pid)
+        cancel_after(5) do
+          kill_and_await('TERM', pid)
+        end
+      rescue Polyphony::Cancel
+        kill_and_await(-9, pid)
+      end
 
-def kill_process(pid)
-  cancel_after(5) do
-    kill_and_await('TERM', pid)
+      def kill_and_await(sig, pid)
+        ::Process.kill(sig, pid)
+        Thread.current.agent.waitpid(pid)
+      rescue SystemCallError
+        # ignore
+        puts 'SystemCallError in kill_and_await'
+      end
+    end
   end
-rescue Polyphony::Cancel
-  kill_and_await(-9, pid)
-end
-
-def kill_and_await(sig, pid)
-  Process.kill(sig, pid)
-  Gyro::Child.new(pid).await
-rescue SystemCallError
-  # ignore
-  puts 'SystemCallError in kill_and_await'
 end