polyphony 0.75 → 0.79

data/ext/polyphony/queue.c

@@ -2,6 +2,7 @@
 #include "ring_buffer.h"
 
 typedef struct queue {
+  unsigned int closed;
   ring_buffer values;
   ring_buffer shift_queue;
   ring_buffer push_queue;
@@ -9,6 +10,8 @@ typedef struct queue {
 } Queue_t;
 
 VALUE cQueue = Qnil;
+VALUE cClosedQueueError = Qnil;
+VALUE cThreadError = Qnil;
 
 static void Queue_mark(void *ptr) {
   Queue_t *queue = ptr;
@@ -49,6 +52,7 @@ static VALUE Queue_initialize(int argc, VALUE *argv, VALUE self) {
   Queue_t *queue;
   GetQueue(self, queue);
 
+  queue->closed = 0;
   ring_buffer_init(&queue->values);
   ring_buffer_init(&queue->shift_queue);
   ring_buffer_init(&queue->push_queue);
@@ -99,6 +103,9 @@ VALUE Queue_push(VALUE self, VALUE value) {
   Queue_t *queue;
   GetQueue(self, queue);
 
+  if (queue->closed)
+    rb_raise(cClosedQueueError, "queue closed");
+
   if (queue->capacity) capped_queue_block_push(queue);
 
   queue_schedule_first_blocked_fiber(&queue->shift_queue);
@@ -111,6 +118,9 @@ VALUE Queue_unshift(VALUE self, VALUE value) {
   Queue_t *queue;
   GetQueue(self, queue);
 
+  if (queue->closed)
+    rb_raise(cClosedQueueError, "queue closed");
+
   if (queue->capacity) capped_queue_block_push(queue);
 
   queue_schedule_first_blocked_fiber(&queue->shift_queue);
@@ -119,14 +129,25 @@ VALUE Queue_unshift(VALUE self, VALUE value) {
   return self;
 }
 
-VALUE Queue_shift(VALUE self) {
-  Queue_t *queue;
+VALUE Queue_shift_nonblock(Queue_t *queue) {
+  if (queue->values.count) {
+    VALUE value = ring_buffer_shift(&queue->values);
+    if ((queue->capacity) && (queue->capacity > queue->values.count))
+      queue_schedule_first_blocked_fiber(&queue->push_queue);
+    RB_GC_GUARD(value);
+    return value;
+  }
+  rb_raise(cThreadError, "queue empty");
+}
+
+VALUE Queue_shift_block(Queue_t *queue) {
   VALUE fiber = rb_fiber_current();
   VALUE thread = rb_thread_current();
   VALUE backend = rb_ivar_get(thread, ID_ivar_backend);
   VALUE value;
 
-  GetQueue(self, queue);
+  if (queue->closed && !queue->values.count)
+    rb_raise(cClosedQueueError, "queue closed");
 
   while (1) {
     VALUE switchpoint_result;
@@ -140,6 +161,7 @@ VALUE Queue_shift(VALUE self) {
     RAISE_IF_EXCEPTION(switchpoint_result);
     RB_GC_GUARD(switchpoint_result);
     if (queue->values.count) break;
+    if (queue->closed) return Qnil;
   }
   value = ring_buffer_shift(&queue->values);
   if ((queue->capacity) && (queue->capacity > queue->values.count))
@@ -148,6 +170,16 @@ VALUE Queue_shift(VALUE self) {
   return value;
 }
 
+VALUE Queue_shift(int argc,VALUE *argv, VALUE self) {
+  int nonblock = argc && RTEST(argv[0]);
+  Queue_t *queue;
+  GetQueue(self, queue);
+
+  return nonblock ?
+    Queue_shift_nonblock(queue) :
+    Queue_shift_block(queue);
+}
+
 VALUE Queue_delete(VALUE self, VALUE value) {
   Queue_t *queue;
   GetQueue(self, queue);
@@ -244,6 +276,13 @@ VALUE Queue_pending_p(VALUE self) {
   return (queue->shift_queue.count) ? Qtrue : Qfalse;
 }
 
+VALUE Queue_num_waiting(VALUE self) {
+  Queue_t *queue;
+  GetQueue(self, queue);
+
+  return INT2NUM(queue->shift_queue.count);
+}
+
 VALUE Queue_size_m(VALUE self) {
   Queue_t *queue;
   GetQueue(self, queue);
@@ -251,20 +290,54 @@ VALUE Queue_size_m(VALUE self) {
   return INT2NUM(queue->values.count);
 }
 
+VALUE Queue_closed_p(VALUE self) {
+  Queue_t *queue;
+  GetQueue(self, queue);
+
+  return (queue->closed) ? Qtrue : Qfalse;
+}
+
+VALUE Queue_close(VALUE self) {
+  Queue_t *queue;
+  GetQueue(self, queue);
+
+  if (queue->closed) goto end;
+  queue->closed = 1;
+
+  // release all fibers waiting on `#shift`
+  while (queue->shift_queue.count) {
+    VALUE fiber = ring_buffer_shift(&queue->shift_queue);
+    if (fiber == Qnil) break;
+    Fiber_make_runnable(fiber, Qnil);
+  }
+
+end:
+  return self;
+}
+
 void Init_Queue() {
+  cClosedQueueError = rb_const_get(rb_cObject, rb_intern("ClosedQueueError"));
+  cThreadError = rb_const_get(rb_cObject, rb_intern("ThreadError"));
+
   cQueue = rb_define_class_under(mPolyphony, "Queue", rb_cObject);
   rb_define_alloc_func(cQueue, Queue_allocate);
 
   rb_define_method(cQueue, "initialize", Queue_initialize, -1);
   rb_define_method(cQueue, "push", Queue_push, 1);
   rb_define_method(cQueue, "<<", Queue_push, 1);
+  rb_define_method(cQueue, "enq", Queue_push, 1);
   rb_define_method(cQueue, "unshift", Queue_unshift, 1);
 
-  rb_define_method(cQueue, "shift", Queue_shift, 0);
-  rb_define_method(cQueue, "pop", Queue_shift, 0);
+  rb_define_method(cQueue, "shift", Queue_shift, -1);
+  rb_define_method(cQueue, "pop", Queue_shift, -1);
+  rb_define_method(cQueue, "deq", Queue_shift, -1);
+
   rb_define_method(cQueue, "delete", Queue_delete, 1);
   rb_define_method(cQueue, "clear", Queue_clear, 0);
 
+  rb_define_method(cQueue, "size", Queue_size_m, 0);
+  rb_define_method(cQueue, "length", Queue_size_m, 0);
+
   rb_define_method(cQueue, "cap", Queue_cap, 1);
   rb_define_method(cQueue, "capped?", Queue_capped_p, 0);
 
@@ -273,5 +346,8 @@ void Init_Queue() {
   rb_define_method(cQueue, "flush_waiters", Queue_flush_waiters, 1);
   rb_define_method(cQueue, "empty?", Queue_empty_p, 0);
   rb_define_method(cQueue, "pending?", Queue_pending_p, 0);
-  rb_define_method(cQueue, "size", Queue_size_m, 0);
+  rb_define_method(cQueue, "num_waiting", Queue_num_waiting, 0);
+
+  rb_define_method(cQueue, "closed?", Queue_closed_p, 0);
+  rb_define_method(cQueue, "close", Queue_close, 0);
 }

data/ext/polyphony/thread.c

@@ -23,11 +23,15 @@ VALUE Thread_fiber_unschedule(VALUE self, VALUE fiber) {
 }
 
 inline void Thread_schedule_fiber(VALUE self, VALUE fiber, VALUE value) {
-  schedule_fiber(self, fiber, value, 0);
+  Backend_schedule_fiber(self, rb_ivar_get(self, ID_ivar_backend), fiber, value, 0);
+
+  // schedule_fiber(self, fiber, value, 0);
 }
 
 inline void Thread_schedule_fiber_with_priority(VALUE self, VALUE fiber, VALUE value) {
-  schedule_fiber(self, fiber, value, 1);
+  Backend_schedule_fiber(self, rb_ivar_get(self, ID_ivar_backend), fiber, value, 1);
+
+  // schedule_fiber(self, fiber, value, 1);
 }
 
 VALUE Thread_switch_fiber(VALUE self) {

data/ext/test_eintr.c

@@ -0,0 +1,50 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <signal.h>
+#include <poll.h>
+#include "./liburing/liburing.h"
+
+void sig_handler(int sig) {
+  printf("handle signal %d!\n", sig);
+}
+
+int main(int argc, char *argv[])
+{
+  int pid = getpid();
+  int child_pid = fork();
+  if (!child_pid) {
+    sleep(1);
+    kill(pid, SIGINT);
+    sleep(1);
+    kill(pid, SIGINT);
+  }
+  else {
+    struct sigaction sa;
+
+    sa.sa_handler = sig_handler;
+    sa.sa_flags = SA_SIGINFO | SA_ONSTACK;//0;
+    sigemptyset(&sa.sa_mask);
+    sigaction(SIGINT, &sa, NULL);
+
+    printf("pid: %d\n", pid);
+
+    struct io_uring ring;
+    int ret = io_uring_queue_init(16, &ring, 0);
+    printf("io_uring_queue_init: %d\n", ret);
+
+    struct io_uring_sqe *sqe = io_uring_get_sqe(&ring);
+    io_uring_prep_poll_add(sqe, STDIN_FILENO, POLLIN);
+    ret = io_uring_submit(&ring);
+    printf("io_uring_submit: %d\n", ret);
+
+    struct io_uring_cqe *cqe;
+
+wait_cqe:
+    ret = io_uring_wait_cqe(&ring, &cqe);
+    printf("io_uring_wait_cqe: %d\n", ret);
+    if (ret == -EINTR) goto wait_cqe;
+
+    printf("done\n");
+    return 0;
+  }
+}

data/lib/polyphony/core/debug.rb

@@ -6,8 +6,154 @@ module ::Kernel
   def format_trace(args)
     if args.size > 1 && args.first.is_a?(String)
       format("%s: %p\n", args.shift, args.size == 1 ? args.first : args)
+    elsif args.size == 1 && args.first.is_a?(String)
+      "#{args.first}\n"
     else
       format("%p\n", args.size == 1 ? args.first : args)
     end
   end
 end
+
+module Polyphony
+  module Trace
+    class << self
+      def start_event_firehose(io = nil, &block)
+        Thread.backend.trace_proc = firehose_proc(io, block)
+      end
+
+      private
+
+      def firehose_proc(io, block)
+        if io
+          ->(*e) { io.orig_write("#{trace_event_info(e).inspect}\n") }
+        elsif block
+          ->(*e) { block.(trace_event_info(e)) }
+        else
+          raise "Please provide an io or a block"
+        end
+      end
+
+      def trace_event_info(e)
+        {
+          stamp: format_current_time,
+          event: e[0]
+        }.merge(
+          send(:"event_props_#{e[0]}", e)
+        )
+      end
+      
+      def format_trace_event_message(e)
+        props = send(:"event_props_#{e[0]}", e).merge(
+          timestamp: format_current_time,
+          event: e[0]
+        )
+        # templ = send(:"event_format_#{e[0]}", e)
+
+        # msg = format("%<timestamp>s #{templ}\n", **props)
+      end
+
+      def format_current_time
+        Time.now.strftime('%Y-%m-%d %H:%M:%S')
+      end
+
+      def generic_event_format
+        '%<event>-12.12s'
+      end
+
+      def fiber_event_format
+        "#{generic_event_format} %<fiber>-44.44s"
+      end
+
+      def event_props_enter_poll(e)
+        {}
+      end
+
+      def event_format_enter_poll(e)
+        generic_event_format
+      end
+
+      def event_props_leave_poll(e)
+        {}
+      end
+
+      def event_format_leave_poll(e)
+        generic_event_format
+      end
+
+      def event_props_schedule(e)
+        {
+          fiber: e[1],
+          value: e[2],
+          caller: e[4],
+          source_fiber: Fiber.current
+        }
+      end
+
+      def event_format_schedule(e)
+        "#{fiber_event_format} %<value>-24.24p %<caller>-120.120s <= %<origin_fiber>s"
+      end
+
+      def event_props_unblock(e)
+        {
+          fiber: e[1],
+          value: e[2],
+          caller: e[3],
+        }
+      end
+
+      def event_format_unblock(e)
+        "#{fiber_event_format} %<value>-24.24p %<caller>-120.120s"
+      end
+
+      def event_props_terminate(e)
+        {
+          fiber: e[1],
+          value: e[2],
+        }
+      end
+
+      def event_format_terminate(e)
+        "#{fiber_event_format} %<value>-24.24p"
+      end
+
+      def event_props_block(e)
+        {
+          fiber: e[1],
+          caller: e[2]
+        }
+      end
+
+      def event_format_block(e)
+        "#{fiber_event_format} #{' ' * 24} %<caller>-120.120s"
+      end
+
+      def event_props_spin(e)
+        {
+          fiber: e[1],
+          caller: e[2],
+          source_fiber: Fiber.current
+        }
+      end
+
+      def event_format_spin(e)
+        "#{fiber_event_format} #{' ' * 24} %<caller>-120.120s <= %<origin_fiber>s"
+      end
+
+      def fibe_repr(fiber)
+        format("%-6x %-20.20s %-10.10s", fiber.object_id, fiber.tag, "(#{fiber.state})")
+      end
+
+      def fiber_compact_repr(fiber)
+        if fiber.tag
+          format("%-6x %-.20s %-.10s", fiber.object_id, fiber.tag, "(#{fiber.state})")
+        else
+          format("%-6x %-.10s", fiber.object_id, "(#{fiber.state})")
+        end
+      end
+
+      def caller_repr(c)
+       c.map { |i| i.gsub('/home/sharon/repo/polyphony/lib/polyphony', '') }.join('  ')
+      end
+    end
+  end
+end

data/lib/polyphony/extensions/fiber.rb

@@ -248,21 +248,28 @@ module Polyphony
     # also be scheduled with priority. This method is mainly used trapping
     # signals (see also the patched `Kernel#trap`)
     def schedule_priority_oob_fiber(&block)
-      f = Fiber.new do
+      oob_fiber = Fiber.new do
         Fiber.current.setup_raw
+        Thread.backend.trace(:unblock, oob_fiber, nil, @caller)
         result = block.call
       rescue Exception => e
         Thread.current.schedule_and_wakeup(Thread.main.main_fiber, e)
         result = e
       ensure
-        Thread.backend.trace(:fiber_terminate, f, result)
+        Thread.backend.trace(:terminate, Fiber.current, result)
         suspend
       end
-      f.oob = true
+      prepare_oob_fiber(oob_fiber, block)
+      Thread.backend.trace(:spin, oob_fiber, caller)
+      oob_fiber.schedule_with_priority(nil)
+    end
+
+    def prepare_oob_fiber(fiber, block)
+      fiber.oob = true
+      fiber.tag = :oob
+      fiber.thread = Thread.current
       location = block.source_location
-      f.set_caller(["#{location.join(':')}"])
-      Thread.backend.trace(:fiber_create, f)
-      Thread.current.schedule_and_wakeup(f, nil)
+      fiber.set_caller(["#{location.join(':')}"])
     end
   end
 
@@ -349,12 +356,13 @@ module Polyphony
       @parent = parent
       @caller = caller
       @block = block
-      Thread.backend.trace(:fiber_create, self)
+      Thread.backend.trace(:spin, self, Kernel.caller[1..-1])
       schedule
     end
 
     def run(first_value)
       setup first_value
+      Thread.backend.trace(:unblock, self, first_value, @caller)
       result = @block.(first_value)
       finalize result
     rescue Polyphony::Restart => e
@@ -396,7 +404,7 @@ module Polyphony
 
     def finalize(result, uncaught_exception = false)
       result, uncaught_exception = finalize_children(result, uncaught_exception)
-      Thread.backend.trace(:fiber_terminate, self, result)
+      Thread.backend.trace(:terminate, self, result)
       @result = result
 
       inform_monitors(result, uncaught_exception)

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.75'
+  VERSION = '0.79'
 end

data/test/helper.rb

@@ -46,10 +46,11 @@ class MiniTest::Test
     Thread.current.backend.finalize
     Thread.current.backend = Polyphony::Backend.new
     sleep 0.001
+    @__stamp = Time.now
   end
 
   def teardown
-    # trace "* teardown #{self.name}"
+    # trace "* teardown #{self.name} (#{Time.now - @__stamp}s)"
     Fiber.current.shutdown_all_children
     if Fiber.current.children.size > 0
       puts "Children left after #{self.name}: #{Fiber.current.children.inspect}"

data/test/stress.rb

@@ -3,7 +3,7 @@
 count = ARGV[0] ? ARGV[0].to_i : 100
 test_name = ARGV[1]
 
-$test_cmd = +'ruby test/run.rb  --name test_receive_cross_thread_exception'
+$test_cmd = +'ruby test/run.rb  --name test_cross_thread_send_receive'
 if test_name
   $test_cmd << " --name #{test_name}"
 end

data/test/test_fiber.rb

@@ -923,18 +923,20 @@ class MailboxTest < MiniTest::Test
   def test_cross_thread_send_receive
     ping_receive_buffer = []
     pong_receive_buffer = []
+    master = Fiber.current
 
     pong = Thread.new do
-      sleep 0.05
-      loop do
+      master << :pong_ready
+      3.times do
         peer, data = receive
         pong_receive_buffer << data
         peer << 'pong'
       end
     end
 
+    assert_equal :pong_ready, receive
+
     ping = Thread.new do
-      sleep 0.05
       3.times do
         pong << [Fiber.current, 'ping']
         data = receive
@@ -943,7 +945,7 @@ class MailboxTest < MiniTest::Test
     end
 
     ping.join
-    pong.kill
+    pong.join
     ping = pong = nil
 
     assert_equal %w{pong pong pong}, ping_receive_buffer

data/test/test_io.rb

@@ -309,13 +309,13 @@ class IOClassMethodsTest < MiniTest::Test
     assert_equal BIN_DATA, s
   end
 
-  def test_foreach
-    skip 'IO.foreach is not yet implemented'
-    lines = []
-    IO.foreach(__FILE__) { |l| lines << l }
-    assert_equal "# frozen_string_literal: true\n", lines[0]
-    assert_equal "end\n", lines[-1]
-  end
+  # def test_foreach
+  #   skip 'IO.foreach is not yet implemented'
+  #   lines = []
+  #   IO.foreach(__FILE__) { |l| lines << l }
+  #   assert_equal "# frozen_string_literal: true\n", lines[0]
+  #   assert_equal "end\n", lines[-1]
+  # end
 
   def test_read_class_method
     s = IO.read(__FILE__)

data/test/test_queue.rb

@@ -21,6 +21,44 @@ class QueueTest < MiniTest::Test
     assert_equal [1, 2, 3, 4], buf
   end
 
+  def test_chained_push
+    @queue << 5 << 6 << 7
+
+    buf = []
+    3.times { buf << @queue.shift }
+    assert_equal [5, 6, 7], buf
+  end
+
+  def test_push_aliases
+    @queue.push 1
+    @queue << 2
+    @queue.enq 3
+
+    buf = []
+    3.times { buf << @queue.shift }
+    assert_equal [1, 2, 3], buf
+  end
+
+  def test_pop_aliases
+    @queue << 1 << 2 << 3
+
+    assert_equal 1, @queue.pop
+    assert_equal 2, @queue.deq
+    assert_equal 3, @queue.shift
+
+    @queue << 1 << 2 << 3
+
+    assert_equal 1, @queue.pop(false)
+    assert_equal 2, @queue.deq(false)
+    assert_equal 3, @queue.shift(false)
+  end
+
+  def test_nonblocking_pop
+    assert_raises(ThreadError) { @queue.pop(true) }
+    assert_raises(ThreadError) { @queue.deq(true) }
+    assert_raises(ThreadError) { @queue.shift(true) }
+  end
+
   def test_unshift
     @queue.push 1
     @queue.push 2
@@ -112,22 +150,86 @@ class QueueTest < MiniTest::Test
 
   def test_queue_size
     assert_equal 0, @queue.size
+    assert_equal 0, @queue.length
 
     @queue.push 1
 
     assert_equal 1, @queue.size
+    assert_equal 1, @queue.length
 
     @queue.push 2
 
     assert_equal 2, @queue.size
+    assert_equal 2, @queue.length
 
     @queue.shift
 
     assert_equal 1, @queue.size
+    assert_equal 1, @queue.length
 
     @queue.shift
 
     assert_equal 0, @queue.size
+    assert_equal 0, @queue.length
+  end
+
+  def test_pending?
+    assert_equal false, @queue.pending?
+
+    buf = []
+    f = spin { buf << @queue.shift }
+    snooze
+    assert_equal true, @queue.pending?
+
+    @queue << 42
+    f.await
+    assert_equal [42], buf
+    assert_equal false, @queue.pending?
+  end
+
+  def test_num_waiting
+    assert_equal 0, @queue.num_waiting
+
+    f1 = spin { @queue.shift }
+    snooze # allow fiber to start
+    assert_equal 1, @queue.num_waiting
+
+    f2 = spin { @queue.shift }
+    snooze # allow fiber to start
+    assert_equal 2, @queue.num_waiting
+
+    @queue << 1
+    f1.await
+    assert_equal 1, @queue.num_waiting
+
+    @queue << 2
+    f2.await
+    assert_equal 0, @queue.num_waiting
+  end
+
+  def test_closed_queue
+    assert_equal false, @queue.closed?
+
+    buf = []
+    f = spin { buf << @queue.shift }
+    snooze # allow fiber to start
+
+    @queue.close
+    assert_equal true, @queue.closed?
+    cancel_after(1) { f.await }
+    assert_equal [nil], buf
+
+    assert_raises(ClosedQueueError) { @queue << 1 }
+    assert_raises(ClosedQueueError) { @queue.deq }
+    assert_raises(ThreadError) { @queue.pop(true) }
+
+    # test deq on closed non-empty queue
+    @queue = Polyphony::Queue.new
+    @queue << 42 << 43
+    @queue.close
+
+    assert_equal 42, @queue.deq(false)
+    assert_equal 43, @queue.deq(true)
   end
 end
 
@@ -246,4 +348,4 @@ class CappedQueueTest < MiniTest::Test
     a.join
     assert_equal [1, 2, 3, :d5, 4, :d8, 5], buffer
   end
-end
+end