polyphony 0.43.9 → 0.45.1

data/ext/polyphony/polyphony.c

@@ -7,8 +7,8 @@ ID ID_caller;
 ID ID_clear;
 ID ID_each;
 ID ID_inspect;
+ID ID_invoke;
 ID ID_new;
-ID ID_raise;
 ID ID_ivar_running;
 ID ID_ivar_thread;
 ID ID_runnable;
@@ -21,6 +21,8 @@ ID ID_R;
 ID ID_W;
 ID ID_RW;
 
+backend_interface_t backend_interface;
+
 VALUE Polyphony_snooze(VALUE self) {
   VALUE ret;
   VALUE fiber = rb_fiber_current();
@@ -58,10 +60,10 @@ void Init_Polyphony() {
   ID_clear          = rb_intern("clear");
   ID_each           = rb_intern("each");
   ID_inspect        = rb_intern("inspect");
+  ID_invoke         = rb_intern("invoke");
   ID_ivar_running   = rb_intern("@running");
   ID_ivar_thread    = rb_intern("@thread");
   ID_new            = rb_intern("new");
-  ID_raise          = rb_intern("raise");
   ID_runnable       = rb_intern("runnable");
   ID_runnable_value = rb_intern("runnable_value");
   ID_signal         = rb_intern("signal");

data/ext/polyphony/polyphony.h

@@ -4,20 +4,29 @@
 #include "ruby.h"
 #include "ruby/io.h"
 #include "libev.h"
+#include "backend.h"
 
 // debugging
 #define OBJ_ID(obj) (NUM2LONG(rb_funcall(obj, rb_intern("object_id"), 0)))
-#define INSPECT(obj) { VALUE s = rb_funcall(obj, rb_intern("inspect"), 0); printf("%s\n", StringValueCStr(s));}
-#define FIBER_TRACE(...) if (__tracing_enabled__) { \
-  rb_funcall(rb_cObject, ID_fiber_trace, __VA_ARGS__); \
+#define INSPECT(str, obj) { printf(str); VALUE s = rb_funcall(obj, rb_intern("inspect"), 0); printf("%s\n", StringValueCStr(s)); }
+#define TRACE_CALLER() { VALUE c = rb_funcall(rb_mKernel, rb_intern("caller"), 0); INSPECT("caller: ", c); }
+
+// tracing
+#define TRACE(...)  rb_funcall(rb_cObject, ID_fiber_trace, __VA_ARGS__)
+#define COND_TRACE(...) if (__tracing_enabled__) { \
+  TRACE(__VA_ARGS__); \
 }
 
 #define TEST_EXCEPTION(ret) (RTEST(rb_obj_is_kind_of(ret, rb_eException)))
 
+#define RAISE_EXCEPTION(e) rb_funcall(e, ID_invoke, 0);
 #define TEST_RESUME_EXCEPTION(ret) if (RTEST(rb_obj_is_kind_of(ret, rb_eException))) { \
-  return rb_funcall(rb_mKernel, ID_raise, 1, ret); \
+  return RAISE_EXCEPTION(ret); \
 }
 
+extern backend_interface_t backend_interface;
+#define __BACKEND__ (backend_interface)
+
 extern VALUE mPolyphony;
 extern VALUE cQueue;
 extern VALUE cEvent;
@@ -28,7 +37,8 @@ extern ID ID_clear;
 extern ID ID_each;
 extern ID ID_fiber_trace;
 extern ID ID_inspect;
-extern ID ID_ivar_agent;
+extern ID ID_invoke;
+extern ID ID_ivar_backend;
 extern ID ID_ivar_running;
 extern ID ID_ivar_thread;
 extern ID ID_new;
@@ -56,26 +66,9 @@ enum {
   FIBER_STATE_SCHEDULED     = 2
 };
 
-// watcher flags
-enum {
-  // a watcher's active field will be set to this after fork
-  GYRO_WATCHER_POST_FORK = 0xFF
-};
-
 VALUE Fiber_auto_watcher(VALUE self);
 void Fiber_make_runnable(VALUE fiber, VALUE value);
 
-VALUE LibevAgent_poll(VALUE self, VALUE nowait, VALUE current_fiber, VALUE queue);
-VALUE LibevAgent_break(VALUE self);
-VALUE LibevAgent_pending_count(VALUE self);
-VALUE LibevAgent_wait_io(VALUE self, VALUE io, VALUE write);
-
-VALUE LibevAgent_ref(VALUE self);
-VALUE LibevAgent_unref(VALUE self);
-int LibevAgent_ref_count(VALUE self);
-void LibevAgent_reset_ref_count(VALUE self);
-VALUE LibevAgent_wait_event(VALUE self, VALUE raise);
-
 VALUE Queue_push(VALUE self, VALUE value);
 VALUE Queue_unshift(VALUE self, VALUE value);
 VALUE Queue_shift(VALUE self);
@@ -85,13 +78,7 @@ VALUE Queue_delete(VALUE self, VALUE value);
 long Queue_len(VALUE self);
 void Queue_trace(VALUE self);
 
-VALUE Polyphony_snooze(VALUE self);
-
 VALUE Thread_schedule_fiber(VALUE thread, VALUE fiber, VALUE value);
 VALUE Thread_switch_fiber(VALUE thread);
 
-int io_setstrbuf(VALUE *str, long len);
-void io_set_read_length(VALUE str, long n, int shrinkable);
-VALUE io_enc_str(VALUE str, rb_io_t *fptr);
-
 #endif /* POLYPHONY_H */

data/ext/polyphony/polyphony_ext.c

@@ -2,7 +2,7 @@
 
 void Init_Fiber();
 void Init_Polyphony();
-void Init_LibevAgent();
+void Init_LibevBackend();
 void Init_Queue();
 void Init_Event();
 void Init_Thread();
@@ -12,12 +12,11 @@ void Init_polyphony_ext() {
   ev_set_allocator(xrealloc);
 
   Init_Polyphony();
-  Init_LibevAgent();
+
+  Init_LibevBackend();
   Init_Queue();
   Init_Event();
-
   Init_Fiber();
   Init_Thread();
-
   Init_Tracing();
 }

data/ext/polyphony/queue.c

@@ -54,6 +54,7 @@ static VALUE Queue_initialize(VALUE self) {
 VALUE Queue_push(VALUE self, VALUE value) {
   Queue_t *queue;
   GetQueue(self, queue);
+
   if (queue->shift_queue.count > 0) {
     VALUE fiber = ring_buffer_shift(&queue->shift_queue);
     if (fiber != Qnil) Fiber_make_runnable(fiber, Qnil);
@@ -77,21 +78,25 @@ VALUE Queue_shift(VALUE self) {
   Queue_t *queue;
   GetQueue(self, queue);
 
-  if (queue->values.count == 0) {
-    VALUE agent = rb_ivar_get(rb_thread_current(), ID_ivar_agent);
-    VALUE fiber = rb_fiber_current();
-    VALUE switchpoint_result = Qnil;
+  VALUE fiber = rb_fiber_current();
+  VALUE thread = rb_thread_current();
+  VALUE backend = rb_ivar_get(thread, ID_ivar_backend);
+
+  while (1) {
     ring_buffer_push(&queue->shift_queue, fiber);
-    switchpoint_result = LibevAgent_wait_event(agent, Qnil);
-    if (RTEST(rb_obj_is_kind_of(switchpoint_result, rb_eException))) {
-      ring_buffer_delete(&queue->shift_queue, fiber);
-      return rb_funcall(rb_mKernel, ID_raise, 1, switchpoint_result);
-    }
-    RB_GC_GUARD(agent);
+    if (queue->values.count > 0) Fiber_make_runnable(fiber, Qnil);
+
+    VALUE switchpoint_result = __BACKEND__.wait_event(backend, Qnil);
+    ring_buffer_delete(&queue->shift_queue, fiber);
+
+    TEST_RESUME_EXCEPTION(switchpoint_result);
     RB_GC_GUARD(switchpoint_result);
+
+    if (queue->values.count > 0)
+      return ring_buffer_shift(&queue->values);
   }
 
-  return ring_buffer_shift(&queue->values);
+  return Qnil;
 }
 
 VALUE Queue_shift_no_wait(VALUE self) {
@@ -146,7 +151,7 @@ VALUE Queue_flush_waiters(VALUE self, VALUE value) {
   while(1) {
     VALUE fiber = ring_buffer_shift(&queue->shift_queue);
     if (fiber == Qnil) return self;
-    
+
     Fiber_make_runnable(fiber, value);
   }
 }
@@ -158,6 +163,13 @@ VALUE Queue_empty_p(VALUE self) {
   return (queue->values.count == 0) ? Qtrue : Qfalse;
 }
 
+VALUE Queue_pending_p(VALUE self) {
+  Queue_t *queue;
+  GetQueue(self, queue);
+
+  return (queue->shift_queue.count > 0) ? Qtrue : Qfalse;
+}
+
 VALUE Queue_size_m(VALUE self) {
   Queue_t *queue;
   GetQueue(self, queue);
@@ -165,6 +177,13 @@ VALUE Queue_size_m(VALUE self) {
   return INT2NUM(queue->values.count);
 }
 
+void Queue_trace(VALUE self) {
+  Queue_t *queue;
+  GetQueue(self, queue);
+
+  printf("run queue size: %d count: %d\n", queue->values.size, queue->values.count);
+}
+
 void Init_Queue() {
   cQueue = rb_define_class_under(mPolyphony, "Queue", rb_cData);
   rb_define_alloc_func(cQueue, Queue_allocate);
@@ -183,5 +202,6 @@ void Init_Queue() {
   rb_define_method(cQueue, "shift_all", Queue_shift_all, 0);
   rb_define_method(cQueue, "flush_waiters", Queue_flush_waiters, 1);
   rb_define_method(cQueue, "empty?", Queue_empty_p, 0);
+  rb_define_method(cQueue, "pending?", Queue_pending_p, 0);
   rb_define_method(cQueue, "size", Queue_size_m, 0);
 }

data/ext/polyphony/ring_buffer.c

@@ -24,7 +24,6 @@ VALUE ring_buffer_shift(ring_buffer *buffer) {
   value = buffer->entries[buffer->head];
   buffer->head = (buffer->head + 1) % buffer->size;
   buffer->count--;
-  // INSPECT(value);
   return value;
 }
 

data/ext/polyphony/thread.c

@@ -1,7 +1,7 @@
 #include "polyphony.h"
 
 ID ID_deactivate_all_watchers_post_fork;
-ID ID_ivar_agent;
+ID ID_ivar_backend;
 ID ID_ivar_join_wait_queue;
 ID ID_ivar_main_fiber;
 ID ID_ivar_result;
@@ -12,7 +12,7 @@ ID ID_stop;
 
 static VALUE Thread_setup_fiber_scheduling(VALUE self) {
   VALUE queue = rb_funcall(cQueue, ID_new, 0);
-  
+
   rb_ivar_set(self, ID_ivar_main_fiber, rb_fiber_current());
   rb_ivar_set(self, ID_run_queue, queue);
 
@@ -20,20 +20,20 @@ static VALUE Thread_setup_fiber_scheduling(VALUE self) {
 }
 
 int Thread_fiber_ref_count(VALUE self) {
-  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
-  return NUM2INT(LibevAgent_ref_count(agent));
+  VALUE backend = rb_ivar_get(self, ID_ivar_backend);
+  return NUM2INT(__BACKEND__.ref_count(backend));
 }
 
 inline void Thread_fiber_reset_ref_count(VALUE self) {
-  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
-  LibevAgent_reset_ref_count(agent);
+  VALUE backend = rb_ivar_get(self, ID_ivar_backend);
+  __BACKEND__.reset_ref_count(backend);
 }
 
 static VALUE SYM_scheduled_fibers;
 static VALUE SYM_pending_watchers;
 
 static VALUE Thread_fiber_scheduling_stats(VALUE self) {
-  VALUE agent = rb_ivar_get(self,ID_ivar_agent);
+  VALUE backend = rb_ivar_get(self,ID_ivar_backend);
   VALUE stats = rb_hash_new();
   VALUE queue = rb_ivar_get(self, ID_run_queue);
   long pending_count;
@@ -41,7 +41,7 @@ static VALUE Thread_fiber_scheduling_stats(VALUE self) {
   long scheduled_count = RARRAY_LEN(queue);
   rb_hash_aset(stats, SYM_scheduled_fibers, INT2NUM(scheduled_count));
 
-  pending_count = LibevAgent_pending_count(agent);
+  pending_count = __BACKEND__.pending_count(backend);
   rb_hash_aset(stats, SYM_pending_watchers, INT2NUM(pending_count));
 
   return stats;
@@ -52,23 +52,34 @@ VALUE Thread_schedule_fiber(VALUE self, VALUE fiber, VALUE value) {
 
   if (rb_fiber_alive_p(fiber) != Qtrue) return self;
 
-  FIBER_TRACE(3, SYM_fiber_schedule, fiber, value);
+  int already_runnable = rb_ivar_get(fiber, ID_runnable) != Qnil;
+
+  if (already_runnable) {
+    VALUE current_runnable_value = rb_ivar_get(fiber, ID_runnable_value);
+
+    // If the fiber is already runnable and the runnable value is an exception,
+    // we don't update the value, in order to prevent a race condition where
+    // exceptions will be lost (see issue #33)
+    if (TEST_EXCEPTION(current_runnable_value)) return self;
+  }
+
   rb_ivar_set(fiber, ID_runnable_value, value);
-  // if fiber is already scheduled, just set the scheduled value, then return
-  if (rb_ivar_get(fiber, ID_runnable) != Qnil) return self;
+  COND_TRACE(3, SYM_fiber_schedule, fiber, value);
 
-  queue = rb_ivar_get(self, ID_run_queue);
-  Queue_push(queue, fiber);
-  rb_ivar_set(fiber, ID_runnable, Qtrue);
+  if (!already_runnable) {
+    queue = rb_ivar_get(self, ID_run_queue);
+    Queue_push(queue, fiber);
+    rb_ivar_set(fiber, ID_runnable, Qtrue);
 
-  if (rb_thread_current() != self) {
-    // if the fiber scheduling is done across threads, we need to make sure the
-    // target thread is woken up in case it is in the middle of running its
-    // event selector. Otherwise it's gonna be stuck waiting for an event to
-    // happen, not knowing that it there's already a fiber ready to run in its
-    // run queue.
-    VALUE agent = rb_ivar_get(self,ID_ivar_agent);
-    LibevAgent_break(agent);
+    if (rb_thread_current() != self) {
+      // If the fiber scheduling is done across threads, we need to make sure the
+      // target thread is woken up in case it is in the middle of running its
+      // event selector. Otherwise it's gonna be stuck waiting for an event to
+      // happen, not knowing that it there's already a fiber ready to run in its
+      // run queue.
+      VALUE backend = rb_ivar_get(self,ID_ivar_backend);
+      __BACKEND__.wakeup(backend);
+    }
   }
   return self;
 }
@@ -78,7 +89,7 @@ VALUE Thread_schedule_fiber_with_priority(VALUE self, VALUE fiber, VALUE value)
 
   if (rb_fiber_alive_p(fiber) != Qtrue) return self;
 
-  FIBER_TRACE(3, SYM_fiber_schedule, fiber, value);
+  COND_TRACE(3, SYM_fiber_schedule, fiber, value);
   rb_ivar_set(fiber, ID_runnable_value, value);
 
   queue = rb_ivar_get(self, ID_run_queue);
@@ -99,8 +110,8 @@ VALUE Thread_schedule_fiber_with_priority(VALUE self, VALUE fiber, VALUE value)
     // event loop. Otherwise it's gonna be stuck waiting for an event to
     // happen, not knowing that it there's already a fiber ready to run in its
     // run queue.
-    VALUE agent = rb_ivar_get(self, ID_ivar_agent);
-    LibevAgent_break(agent);
+    VALUE backend = rb_ivar_get(self, ID_ivar_backend);
+    __BACKEND__.wakeup(backend);
   }
   return self;
 }
@@ -110,43 +121,39 @@ VALUE Thread_switch_fiber(VALUE self) {
   VALUE queue = rb_ivar_get(self, ID_run_queue);
   VALUE next_fiber;
   VALUE value;
-  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+  VALUE backend = rb_ivar_get(self, ID_ivar_backend);
   int ref_count;
-  int agent_was_polled = 0;1;
+  int backend_was_polled = 0;
 
-  if (__tracing_enabled__) {
-    if (rb_ivar_get(current_fiber, ID_ivar_running) != Qfalse) {
-      rb_funcall(rb_cObject, ID_fiber_trace, 2, SYM_fiber_switchpoint, current_fiber);
-    }
-  }
+  if (__tracing_enabled__ && (rb_ivar_get(current_fiber, ID_ivar_running) != Qfalse))
+    TRACE(2, SYM_fiber_switchpoint, current_fiber);
 
-  ref_count = LibevAgent_ref_count(agent);
+  ref_count = __BACKEND__.ref_count(backend);
   while (1) {
     next_fiber = Queue_shift_no_wait(queue);
     if (next_fiber != Qnil) {
-      if (agent_was_polled == 0 && ref_count > 0) {
-        // this mechanism prevents event starvation in case the run queue never
-        // empties
-        LibevAgent_poll(agent, Qtrue, current_fiber, queue);
+      if (backend_was_polled == 0 && ref_count > 0) {
+        // this prevents event starvation in case the run queue never empties
+        __BACKEND__.poll(backend, Qtrue, current_fiber, queue);
       }
       break;
     }
     if (ref_count == 0) break;
 
-    LibevAgent_poll(agent, Qnil, current_fiber, queue);
-    agent_was_polled = 1;
+    __BACKEND__.poll(backend, Qnil, current_fiber, queue);
+    backend_was_polled = 1;
   }
 
   if (next_fiber == Qnil) return Qnil;
 
   // run next fiber
   value = rb_ivar_get(next_fiber, ID_runnable_value);
-  FIBER_TRACE(3, SYM_fiber_run, next_fiber, value);
+  COND_TRACE(3, SYM_fiber_run, next_fiber, value);
 
   rb_ivar_set(next_fiber, ID_runnable, Qnil);
   RB_GC_GUARD(next_fiber);
   RB_GC_GUARD(value);
-  return (next_fiber == current_fiber) ? 
+  return (next_fiber == current_fiber) ?
     value : rb_funcall(next_fiber, ID_transfer, 1, value);
 }
 
@@ -164,12 +171,12 @@ VALUE Thread_reset_fiber_scheduling(VALUE self) {
 }
 
 VALUE Thread_fiber_break_out_of_ev_loop(VALUE self, VALUE fiber, VALUE resume_obj) {
-  VALUE agent = rb_ivar_get(self, ID_ivar_agent);
+  VALUE backend = rb_ivar_get(self, ID_ivar_backend);
   if (fiber != Qnil) {
     Thread_schedule_fiber_with_priority(self, fiber, resume_obj);
   }
 
-  if (LibevAgent_break(agent) == Qnil) {
+  if (__BACKEND__.wakeup(backend) == Qnil) {
     // we're not inside the ev_loop, so we just do a switchpoint
     Thread_switch_fiber(self);
   }
@@ -177,6 +184,11 @@ VALUE Thread_fiber_break_out_of_ev_loop(VALUE self, VALUE fiber, VALUE resume_ob
   return self;
 }
 
+VALUE Thread_debug(VALUE self) {
+  rb_ivar_set(self, rb_intern("@__debug__"), Qtrue);
+  return self;
+}
+
 void Init_Thread() {
   rb_define_method(rb_cThread, "setup_fiber_scheduling", Thread_setup_fiber_scheduling, 0);
   rb_define_method(rb_cThread, "reset_fiber_scheduling", Thread_reset_fiber_scheduling, 0);
@@ -189,8 +201,10 @@ void Init_Thread() {
   rb_define_method(rb_cThread, "switch_fiber", Thread_switch_fiber, 0);
   rb_define_method(rb_cThread, "run_queue_trace", Thread_run_queue_trace, 0);
 
+  rb_define_method(rb_cThread, "debug!", Thread_debug, 0);
+
   ID_deactivate_all_watchers_post_fork = rb_intern("deactivate_all_watchers_post_fork");
-  ID_ivar_agent               = rb_intern("@agent");
+  ID_ivar_backend               = rb_intern("@backend");
   ID_ivar_join_wait_queue     = rb_intern("@join_wait_queue");
   ID_ivar_main_fiber          = rb_intern("@main_fiber");
   ID_ivar_result              = rb_intern("@result");

data/lib/polyphony.rb

@@ -3,50 +3,36 @@
 require 'fiber'
 require_relative './polyphony_ext'
 
-module Polyphony
-  # replace core Queue class with our own
-  verbose = $VERBOSE
-  $VERBOSE = nil
-  Object.const_set(:Queue, Polyphony::Queue)
-  $VERBOSE = verbose
-end
-
 require_relative './polyphony/extensions/core'
 require_relative './polyphony/extensions/thread'
 require_relative './polyphony/extensions/fiber'
 require_relative './polyphony/extensions/io'
 
 Thread.current.setup_fiber_scheduling
-Thread.current.agent = Polyphony::LibevAgent.new
+Thread.current.backend = Polyphony::Backend.new
 
 require_relative './polyphony/core/global_api'
 require_relative './polyphony/core/resource_pool'
+require_relative './polyphony/core/sync'
 require_relative './polyphony/net'
 require_relative './polyphony/adapters/process'
 
-# Main Polyphony API
+# Polyphony API
 module Polyphony
   class << self
-    def wait_for_signal(sig)
-      raise "should be reimplemented"
-
-      fiber = Fiber.current
-      # Polyphony.ref
-      old_trap = trap(sig) do
-        # Polyphony.unref
-        fiber.schedule(sig)
-        trap(sig, old_trap)
-      end
-      suspend
-
-    end
-
     def fork(&block)
       Kernel.fork do
-        # # Since the fiber doing the fork will become the main fiber of the
-        # # forked process, we leave it behind by transferring to a new fiber
-        # # created in the context of the forked process, which rescues *all*
-        # # exceptions, including Interrupt and SystemExit.
+        # A race condition can arise if a TERM or INT signal is received before
+        # the forked process has finished initializing. To prevent this we restore
+        # the default signal handlers, and then reinstall the custom Polyphony
+        # handlers just before running the given block.
+        trap('SIGTERM', 'DEFAULT')
+        trap('SIGINT', 'DEFAULT')
+
+        # Since the fiber doing the fork will become the main fiber of the
+        # forked process, we leave it behind by transferring to a new fiber
+        # created in the context of the forked process, which rescues *all*
+        # exceptions, including Interrupt and SystemExit.
         spin_forked_block(&block).transfer
       end
     end
@@ -57,7 +43,7 @@ module Polyphony
       rescue SystemExit
         # fall through to ensure
       rescue Exception => e
-        e.full_message
+        warn e.full_message
         exit!
       ensure
         exit_forked_process
@@ -65,16 +51,9 @@ module Polyphony
     end
 
     def run_forked_block(&block)
-      # A race condition can arise if a TERM or INT signal is received before
-      # the forked process has finished initializing. To prevent this we restore
-      # the default signal handlers, and then reinstall the custom Polyphony
-      # handlers just before running the given block.
-      trap('SIGTERM', 'DEFAULT')
-      trap('SIGINT', 'DEFAULT')
-
       Thread.current.setup
       Fiber.current.setup_main_fiber
-      Thread.current.agent.post_fork
+      Thread.current.backend.post_fork
 
       install_terminating_signal_handlers
 
@@ -123,12 +102,20 @@ module Polyphony
       # processes (see Polyphony.fork).
       at_exit do
         next unless @original_pid == ::Process.pid
-      
+
         Polyphony.terminate_threads
         Fiber.current.shutdown_all_children
       end
     end
   end
+
+  # replace core Queue class with our own
+  verbose = $VERBOSE
+  $VERBOSE = nil
+  Object.const_set(:Queue, Polyphony::Queue)
+  Object.const_set(:Mutex, Polyphony::Mutex)
+  Object.const_set(:ConditionVariable, Polyphony::ConditionVariable)
+  $VERBOSE = verbose
 end
 
 Polyphony.install_terminating_signal_handlers