polyphony 0.66 → 0.67

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8e2aa3d011286dda93222164cbf66f1ec4c18b9c212ee7a70dbebd4bc523943
-  data.tar.gz: 99b5fb95c682dfad8df33f07d98bfab7f2109b9d4ec0e24720869c2d331f23fb
+  metadata.gz: 6b8553a2f84e79520ce53412ee6891873c9a9d25e0789e82b26a7153d3d7ae67
+  data.tar.gz: 6889ef71f11d79d6fbb82adc77447f9b3526a29858f2aa3c9e51ee9098add2c7
 SHA512:
-  metadata.gz: 27c65b6e340d1aad41d38b4b322b98d282f96fe6d136c897ab8de14651b2da3cfc392650600d63983f76b5fb1dc70e4e49116efb289a790a509bec63caa9b9e4
-  data.tar.gz: 94e62ac7fbcb6d085af7475d987eca45be191aaf7654c3bacb07c4f7793d450281d3da1b08b12636a49398e97c1043c3a2cd628018925bdc082e02a0859743bc
+  metadata.gz: 91814f767f74c673681a2215f82a302f3aab5ced6ae7197720e474b8c1ad6846eb10001580c1f11d5b62bd0ed7563422fd71f0fb5728eb0b3fcb4ebd36d2038c
+  data.tar.gz: 74e1c7ed34b4f212ca1ffd5881da3e9f689c3934d0f3680da2396d7444b5a214a610ccc779c1ef59713116f0a8240491dc60f32b5d5e30de5092ddc41b8c9f18

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.67 2021-08-06
+
+- Improve fiber monitoring
+- Add fiber parking (a parked fiber is prevented from running). This is in
+  preparation for the upcoming work on an integrated debugger.
+
 ## 0.66 2021-08-01
 
 - Fix all splicing APIs on non-linux OSes (#63)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.66)
+    polyphony (0.67)
 
 GEM
   remote: https://rubygems.org/

data/TODO.md

@@ -1,3 +1,13 @@
+- io_uring backend:
+  - if `io_uring_get_sqe` returns null, call `io_uring_submit`, (snooze fiber)?
+    and try again
+
+- Tracing:
+  - Emit events on I/O ops, e.g.:
+    - [:op_read_submit, id, io, len]
+    - [:op_read_complete, id, io, len, buffer]
+  - Prevent tracing while an event is being emitted (to allow the trace proc to perform I/O)
+
 - Add support for IPv6:
   https://www.reddit.com/r/ruby/comments/lyen23/understanding_ipv6_and_why_its_important_to_you/
 
@@ -14,47 +24,8 @@
   - `IO#gets_loop`, `Socket#gets_loop`, `OpenSSL::Socket#gets_loop` (medium effort)
   - `Fiber#receive_loop` (very little effort, should be implemented in C)
 
-
-- Add `Backend#splice`, `Backend#splice_to_eof` for implementing stuff like proxying:
-
-  ```ruby
-  def two_way_proxy(socket1, socket2)
-    backend = Thread.current.backend
-    f1 = spin { backend.splice_to_eof(socket1, socket2) }
-    f2 = spin { backend.splice_to_eof(socket2, socket1) }
-    Fiber.await(f1, f2)
-  end
-  ```
-
 - Add support for `close` to io_uring backend
 
-- Add support for submission of multiple requests to io_uring backend:
-
-  ```ruby
-  Thread.current.backend.submit(
-    [:send, sock, chunk_header(len)],
-    [:splice, file, sock, len]
-  )
-  ```
-
-  Full example (for writing chunks from a file to an HTTP response):
-
-  ```ruby
-  def serve_io(io)
-    i, o = IO.pipe
-    backend = Thread.current.backend
-    while true
-      len = o.splice(io, 8192)
-      break if len == 0
-      
-      backend.submit(
-        [:write, sock, chunk_header(len)],
-        [:splice, i, sock, len]
-      )
-    end
-  end
-  ```
-
 - Graceful shutdown again:
   - What happens to children when doing a graceful shutdown?
   - What are the implications of passing graceful shutdown flag to children?
@@ -92,7 +63,6 @@
 
 -----------------------------------------------------
 
-- Add `Backend#splice(in, out, nbytes)` API
 - Adapter for io/console (what does `IO#raw` do?)
 - Adapter for Pry and IRB (Which fixes #5 and #6)
 - allow backend selection at runtime

data/bin/pdbg

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+UNIX_SOCKET_PATH = '/tmp/pdbg.sock'
+
+cmd = ARGV.join(' ')
+injected_lib_path = File.expand_path('../lib/polyphony/debugger/server_inject.rb', __dir__)
+p cmd
+pid = fork { exec("env POLYPHONY_DEBUG_SOCKET_PATH=#{UNIX_SOCKET_PATH} ruby #{cmd}") }
+puts "Started debugged process (#{pid})"
+
+sleep 3
+socket = UNIXSocket.new(UNIX_SOCKET_PATH)
+socket.puts 'pdbg'
+response = socket.gets
+if response.chomp == 'pdbg'
+  puts 'Connected to process'
+end
+loop do
+  status = socket.gets
+  puts status
+
+  STDOUT << "> "
+  cmd = STDIN.gets
+  puts '-' * 40
+  socket.puts cmd
+end

data/examples/core/await.rb

@@ -2,7 +2,11 @@
 
 require 'bundler/setup'
 require 'polyphony'
+require 'polyphony/extensions/debug'
 
+Exception.__disable_sanitized_backtrace__ = true
+
+puts '----- start await example ------'
 sleeper = spin do
   puts 'going to sleep'
   sleep 1
@@ -17,4 +21,8 @@ waiter = spin do
   puts 'done waiting'
 end
 
-waiter.await
+trace :before_await
+
+sleep 2
+waiter.await
+trace :after_await

data/ext/polyphony/backend_common.c

@@ -7,6 +7,7 @@
 
 inline void backend_base_initialize(struct Backend_base *base) {
   runqueue_initialize(&base->runqueue);
+  runqueue_initialize(&base->parked_runqueue);
   base->currently_polling = 0;
   base->op_count = 0;
   base->switch_count = 0;
@@ -20,12 +21,14 @@ inline void backend_base_initialize(struct Backend_base *base) {
 
 inline void backend_base_finalize(struct Backend_base *base) {
   runqueue_finalize(&base->runqueue);
+  runqueue_finalize(&base->parked_runqueue);
 }
 
 inline void backend_base_mark(struct Backend_base *base) {
   if (base->idle_proc != Qnil) rb_gc_mark(base->idle_proc);
   if (base->trace_proc != Qnil) rb_gc_mark(base->trace_proc);
   runqueue_mark(&base->runqueue);
+  runqueue_mark(&base->parked_runqueue);
 }
 
 const unsigned int ANTI_STARVE_SWITCH_COUNT_THRESHOLD = 64;
@@ -91,7 +94,10 @@ void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_bas
 
   COND_TRACE(base, 4, SYM_fiber_schedule, fiber, value, prioritize ? Qtrue : Qfalse);
 
-  (prioritize ? runqueue_unshift : runqueue_push)(&base->runqueue, fiber, value, already_runnable);
+  runqueue_t *runqueue = rb_ivar_get(fiber, ID_ivar_parked) == Qtrue ? 
+    &base->parked_runqueue : &base->runqueue;
+
+  (prioritize ? runqueue_unshift : runqueue_push)(runqueue, fiber, value, already_runnable);
   if (!already_runnable) {
     rb_ivar_set(fiber, ID_ivar_runnable, Qtrue);
     if (rb_thread_current() != thread) {
@@ -105,6 +111,13 @@ void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_bas
   }
 }
 
+inline void backend_base_park_fiber(struct Backend_base *base, VALUE fiber) {
+  runqueue_migrate(&base->runqueue, &base->parked_runqueue, fiber);
+}
+
+inline void backend_base_unpark_fiber(struct Backend_base *base, VALUE fiber) {
+  runqueue_migrate(&base->parked_runqueue, &base->runqueue, fiber);
+}
 
 inline void backend_trace(struct Backend_base *base, int argc, VALUE *argv) {
   if (base->trace_proc == Qnil) return;

data/ext/polyphony/backend_common.h

@@ -17,6 +17,7 @@ struct backend_stats {
 
 struct Backend_base {
   runqueue_t runqueue;
+  runqueue_t parked_runqueue;
   unsigned int currently_polling;
   unsigned int op_count;
   unsigned int switch_count;
@@ -33,6 +34,8 @@ void backend_base_finalize(struct Backend_base *base);
 void backend_base_mark(struct Backend_base *base);
 VALUE backend_base_switch_fiber(VALUE backend, struct Backend_base *base);
 void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_base *base, VALUE fiber, VALUE value, int prioritize);
+void backend_base_park_fiber(struct Backend_base *base, VALUE fiber);
+void backend_base_unpark_fiber(struct Backend_base *base, VALUE fiber);
 void backend_trace(struct Backend_base *base, int argc, VALUE *argv);
 struct backend_stats backend_base_stats(struct Backend_base *base);
 
@@ -104,7 +107,6 @@ VALUE Backend_sendv(VALUE self, VALUE io, VALUE ary, VALUE flags);
 VALUE Backend_stats(VALUE self);
 void backend_run_idle_tasks(struct Backend_base *base);
 void io_verify_blocking_mode(rb_io_t *fptr, VALUE io, VALUE blocking);
-
 void backend_setup_stats_symbols();
 
 #endif /* BACKEND_COMMON_H */

data/ext/polyphony/backend_io_uring.c

@@ -229,7 +229,7 @@ inline void Backend_unschedule_fiber(VALUE self, VALUE fiber) {
   Backend_t *backend;
   GetBackend(self, backend);
 
-  runqueue_delete(&backend->base.runqueue, fiber);  
+  runqueue_delete(&backend->base.runqueue, fiber);
 }
 
 inline VALUE Backend_switch_fiber(VALUE self) {
@@ -1474,6 +1474,20 @@ VALUE Backend_trace_proc_set(VALUE self, VALUE block) {
   return self;
 }
 
+void Backend_park_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend_base_park_fiber(&backend->base, fiber);
+}
+
+void Backend_unpark_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend_base_unpark_fiber(&backend->base, fiber);
+}
+
 void Init_Backend() {
   VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cObject);
   rb_define_alloc_func(cBackend, Backend_allocate);

data/ext/polyphony/backend_libev.c

@@ -1553,6 +1553,20 @@ VALUE Backend_trace_proc_set(VALUE self, VALUE block) {
   return self;
 }
 
+void Backend_park_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend_base_park_fiber(&backend->base, fiber);
+}
+
+void Backend_unpark_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend_base_unpark_fiber(&backend->base, fiber);
+}
+
 void Init_Backend() {
   ev_set_allocator(xrealloc);
 

data/ext/polyphony/fiber.c

@@ -114,6 +114,22 @@ VALUE Fiber_receive_all_pending(VALUE self) {
   return (mailbox == Qnil) ? rb_ary_new() : Queue_shift_all(mailbox);
 }
 
+VALUE Fiber_park(VALUE self) {
+  rb_ivar_set(self, ID_ivar_parked, Qtrue);
+  Backend_park_fiber(BACKEND(), self);
+  return self;
+}
+
+VALUE Fiber_unpark(VALUE self) {
+  rb_ivar_set(self, ID_ivar_parked, Qnil);
+  Backend_unpark_fiber(BACKEND(), self);
+  return self;
+}
+
+VALUE Fiber_parked_p(VALUE self) {
+  return rb_ivar_get(self, ID_ivar_parked);
+}
+
 void Init_Fiber() {
   VALUE cFiber = rb_const_get(rb_cObject, rb_intern("Fiber"));
   rb_define_method(cFiber, "safe_transfer", Fiber_safe_transfer, -1);
@@ -128,6 +144,10 @@ void Init_Fiber() {
   rb_define_method(cFiber, "receive_all_pending", Fiber_receive_all_pending, 0);
   rb_define_method(cFiber, "mailbox", Fiber_mailbox, 0);
 
+  rb_define_method(cFiber, "__park__", Fiber_park, 0);
+  rb_define_method(cFiber, "__unpark__", Fiber_unpark, 0);
+  rb_define_method(cFiber, "__parked__?", Fiber_parked_p, 0);
+
   SYM_dead = ID2SYM(rb_intern("dead"));
   SYM_running = ID2SYM(rb_intern("running"));
   SYM_runnable = ID2SYM(rb_intern("runnable"));

data/ext/polyphony/polyphony.c

@@ -12,6 +12,7 @@ ID ID_invoke;
 ID ID_new;
 ID ID_ivar_blocking_mode;
 ID ID_ivar_io;
+ID ID_ivar_parked;
 ID ID_ivar_runnable;
 ID ID_ivar_running;
 ID ID_ivar_thread;
@@ -160,6 +161,7 @@ void Init_Polyphony() {
   ID_invoke             = rb_intern("invoke");
   ID_ivar_blocking_mode = rb_intern("@blocking_mode");
   ID_ivar_io            = rb_intern("@io");
+  ID_ivar_parked        = rb_intern("@parked");
   ID_ivar_runnable      = rb_intern("@runnable");
   ID_ivar_running       = rb_intern("@running");
   ID_ivar_thread        = rb_intern("@thread");

data/ext/polyphony/polyphony.h

@@ -44,6 +44,7 @@ extern ID ID_invoke;
 extern ID ID_ivar_backend;
 extern ID ID_ivar_blocking_mode;
 extern ID ID_ivar_io;
+extern ID ID_ivar_parked;
 extern ID ID_ivar_runnable;
 extern ID ID_ivar_running;
 extern ID ID_ivar_thread;
@@ -115,9 +116,11 @@ VALUE Backend_run_idle_tasks(VALUE self);
 VALUE Backend_switch_fiber(VALUE self);
 void Backend_schedule_fiber(VALUE thread, VALUE self, VALUE fiber, VALUE value, int prioritize);
 void Backend_unschedule_fiber(VALUE self, VALUE fiber);
+void Backend_park_fiber(VALUE self, VALUE fiber);
+void Backend_unpark_fiber(VALUE self, VALUE fiber);
 
-VALUE Thread_schedule_fiber(VALUE thread, VALUE fiber, VALUE value);
-VALUE Thread_schedule_fiber_with_priority(VALUE thread, VALUE fiber, VALUE value);
+void Thread_schedule_fiber(VALUE thread, VALUE fiber, VALUE value);
+void Thread_schedule_fiber_with_priority(VALUE thread, VALUE fiber, VALUE value);
 VALUE Thread_switch_fiber(VALUE thread);
 
 VALUE Polyphony_snooze(VALUE self);

data/ext/polyphony/runqueue.c

@@ -40,6 +40,10 @@ inline int runqueue_index_of(runqueue_t *runqueue, VALUE fiber) {
   return runqueue_ring_buffer_index_of(&runqueue->entries, fiber);
 }
 
+inline void runqueue_migrate(runqueue_t *src, runqueue_t *dest, VALUE fiber) {
+  runqueue_ring_buffer_migrate(&src->entries, &dest->entries, fiber);
+}
+
 inline void runqueue_clear(runqueue_t *runqueue) {
   runqueue_ring_buffer_clear(&runqueue->entries);
 }

data/ext/polyphony/runqueue.h

@@ -18,6 +18,7 @@ void runqueue_unshift(runqueue_t *runqueue, VALUE fiber, VALUE value, int resche
 runqueue_entry runqueue_shift(runqueue_t *runqueue);
 void runqueue_delete(runqueue_t *runqueue, VALUE fiber);
 int runqueue_index_of(runqueue_t *runqueue, VALUE fiber);
+void runqueue_migrate(runqueue_t *src, runqueue_t *dest, VALUE fiber);
 void runqueue_clear(runqueue_t *runqueue);
 unsigned int runqueue_size(runqueue_t *runqueue);
 unsigned int runqueue_len(runqueue_t *runqueue);

data/ext/polyphony/runqueue_ring_buffer.c

@@ -1,7 +1,7 @@
 #include "polyphony.h"
 #include "runqueue_ring_buffer.h"
 
-void runqueue_ring_buffer_init(runqueue_ring_buffer *buffer) {
+inline void runqueue_ring_buffer_init(runqueue_ring_buffer *buffer) {
   buffer->size = 1;
   buffer->count = 0;
   buffer->entries = malloc(buffer->size * sizeof(runqueue_entry));
@@ -9,17 +9,21 @@ void runqueue_ring_buffer_init(runqueue_ring_buffer *buffer) {
   buffer->tail = 0;
 }
 
-void runqueue_ring_buffer_free(runqueue_ring_buffer *buffer) {
+inline void runqueue_ring_buffer_free(runqueue_ring_buffer *buffer) {
   free(buffer->entries);
 }
 
-int runqueue_ring_buffer_empty_p(runqueue_ring_buffer *buffer) {
+inline int runqueue_ring_buffer_empty_p(runqueue_ring_buffer *buffer) {
   return buffer->count == 0;
 }
 
+inline void runqueue_ring_buffer_clear(runqueue_ring_buffer *buffer) {
+  buffer->count = buffer->head = buffer->tail = 0;
+}
+
 static runqueue_entry nil_runqueue_entry = {(Qnil), (Qnil)};
 
-runqueue_entry runqueue_ring_buffer_shift(runqueue_ring_buffer *buffer) {
+inline runqueue_entry runqueue_ring_buffer_shift(runqueue_ring_buffer *buffer) {
   if (buffer->count == 0) return nil_runqueue_entry;
 
   runqueue_entry value = buffer->entries[buffer->head];
@@ -28,7 +32,7 @@ runqueue_entry runqueue_ring_buffer_shift(runqueue_ring_buffer *buffer) {
   return value;
 }
 
-void runqueue_ring_buffer_resize(runqueue_ring_buffer *buffer) {
+inline void runqueue_ring_buffer_resize(runqueue_ring_buffer *buffer) {
   unsigned int old_size = buffer->size;
   buffer->size = old_size == 1 ? 4 : old_size * 2;
   buffer->entries = realloc(buffer->entries, buffer->size * sizeof(runqueue_entry));
@@ -37,7 +41,7 @@ void runqueue_ring_buffer_resize(runqueue_ring_buffer *buffer) {
   buffer->tail = buffer->head + buffer->count;
 }
 
-void runqueue_ring_buffer_unshift(runqueue_ring_buffer *buffer, VALUE fiber, VALUE value) {
+inline void runqueue_ring_buffer_unshift(runqueue_ring_buffer *buffer, VALUE fiber, VALUE value) {
   if (buffer->count == buffer->size) runqueue_ring_buffer_resize(buffer);
 
   buffer->head = (buffer->head - 1) % buffer->size;
@@ -46,7 +50,7 @@ void runqueue_ring_buffer_unshift(runqueue_ring_buffer *buffer, VALUE fiber, VAL
   buffer->count++;
 }
 
-void runqueue_ring_buffer_push(runqueue_ring_buffer *buffer, VALUE fiber, VALUE value) {
+inline void runqueue_ring_buffer_push(runqueue_ring_buffer *buffer, VALUE fiber, VALUE value) {
   if (buffer->count == buffer->size) runqueue_ring_buffer_resize(buffer);
 
   buffer->entries[buffer->tail].fiber = fiber;
@@ -55,14 +59,14 @@ void runqueue_ring_buffer_push(runqueue_ring_buffer *buffer, VALUE fiber, VALUE
   buffer->count++;
 }
 
-void runqueue_ring_buffer_mark(runqueue_ring_buffer *buffer) {
+inline void runqueue_ring_buffer_mark(runqueue_ring_buffer *buffer) {
   for (unsigned int i = 0; i < buffer->count; i++) {
     rb_gc_mark(buffer->entries[(buffer->head + i) % buffer->size].fiber);
     rb_gc_mark(buffer->entries[(buffer->head + i) % buffer->size].value);
   }
 }
 
-void runqueue_ring_buffer_delete_at(runqueue_ring_buffer *buffer, unsigned int idx) {
+inline void runqueue_ring_buffer_delete_at(runqueue_ring_buffer *buffer, unsigned int idx) {
   for (unsigned int idx2 = idx; idx2 != buffer->tail; idx2 = (idx2 + 1) % buffer->size) {
     buffer->entries[idx2] = buffer->entries[(idx2 + 1) % buffer->size];
   }
@@ -70,7 +74,7 @@ void runqueue_ring_buffer_delete_at(runqueue_ring_buffer *buffer, unsigned int i
   buffer->tail = (buffer->tail - 1) % buffer->size;
 }
 
-void runqueue_ring_buffer_delete(runqueue_ring_buffer *buffer, VALUE fiber) {
+inline void runqueue_ring_buffer_delete(runqueue_ring_buffer *buffer, VALUE fiber) {
   for (unsigned int i = 0; i < buffer->count; i++) {
     unsigned int idx = (buffer->head + i) % buffer->size;
     if (buffer->entries[idx].fiber == fiber) {
@@ -80,7 +84,7 @@ void runqueue_ring_buffer_delete(runqueue_ring_buffer *buffer, VALUE fiber) {
   }
 }
 
-int runqueue_ring_buffer_index_of(runqueue_ring_buffer *buffer, VALUE fiber) {
+inline int runqueue_ring_buffer_index_of(runqueue_ring_buffer *buffer, VALUE fiber) {
   for (unsigned int i = 0; i < buffer->count; i++) {
     unsigned int idx = (buffer->head + i) % buffer->size;
     if (buffer->entries[idx].fiber == fiber)
@@ -89,6 +93,13 @@ int runqueue_ring_buffer_index_of(runqueue_ring_buffer *buffer, VALUE fiber) {
   return -1;
 }
 
-void runqueue_ring_buffer_clear(runqueue_ring_buffer *buffer) {
-  buffer->count = buffer->head = buffer->tail = 0;
-}
+inline void runqueue_ring_buffer_migrate(runqueue_ring_buffer *src, runqueue_ring_buffer *dest, VALUE fiber) {
+  for (unsigned int i = 0; i < src->count; i++) {
+    unsigned int idx = (src->head + i) % src->size;
+    if (src->entries[idx].fiber == fiber) {
+      runqueue_ring_buffer_push(dest, src->entries[idx].fiber, src->entries[idx].value);
+      runqueue_ring_buffer_delete_at(src, idx);
+      return;
+    }
+  }
+}

data/ext/polyphony/runqueue_ring_buffer.h

@@ -29,4 +29,6 @@ void runqueue_ring_buffer_push(runqueue_ring_buffer *buffer, VALUE fiber, VALUE
 void runqueue_ring_buffer_delete(runqueue_ring_buffer *buffer, VALUE fiber);
 int runqueue_ring_buffer_index_of(runqueue_ring_buffer *buffer, VALUE fiber);
 
+void runqueue_ring_buffer_migrate(runqueue_ring_buffer *src, runqueue_ring_buffer *dest, VALUE fiber);
+
 #endif /* RUNQUEUE_RING_BUFFER_H */

data/ext/polyphony/thread.c

@@ -22,14 +22,12 @@ VALUE Thread_fiber_unschedule(VALUE self, VALUE fiber) {
   return self;
 }
 
-VALUE Thread_schedule_fiber(VALUE self, VALUE fiber, VALUE value) {
+inline void Thread_schedule_fiber(VALUE self, VALUE fiber, VALUE value) {
   schedule_fiber(self, fiber, value, 0);
-  return self;
 }
 
-VALUE Thread_schedule_fiber_with_priority(VALUE self, VALUE fiber, VALUE value) {
+inline void Thread_schedule_fiber_with_priority(VALUE self, VALUE fiber, VALUE value) {
   schedule_fiber(self, fiber, value, 1);
-  return self;
 }
 
 VALUE Thread_switch_fiber(VALUE self) {
@@ -61,10 +59,6 @@ VALUE Thread_class_backend(VALUE _self) {
 void Init_Thread() {
   rb_define_method(rb_cThread, "setup_fiber_scheduling", Thread_setup_fiber_scheduling, 0);
   rb_define_method(rb_cThread, "schedule_and_wakeup", Thread_fiber_schedule_and_wakeup, 2);
-
-  rb_define_method(rb_cThread, "schedule_fiber", Thread_schedule_fiber, 2);
-  rb_define_method(rb_cThread, "schedule_fiber_with_priority",
-    Thread_schedule_fiber_with_priority, 2);
   rb_define_method(rb_cThread, "switch_fiber", Thread_switch_fiber, 0);
   rb_define_method(rb_cThread, "fiber_unschedule", Thread_fiber_unschedule, 1);
 

data/lib/polyphony.rb

@@ -121,3 +121,9 @@ end
 
 Polyphony.install_terminating_signal_handlers
 Polyphony.install_at_exit_handler
+
+if (debug_socket_path = ENV['POLYPHONY_DEBUG_SOCKET_PATH'])
+  puts "Starting debug server on #{debug_socket_path}"
+  require 'polyphony/debugger/server'
+  Polyphony::DebugServer.start(debug_socket_path)
+end

data/lib/polyphony/debugger/server.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'polyphony/extensions/debug'
+
+module Polyphony
+  class DebugServer
+    TP_EVENTS = [
+      :line,
+      :call,
+      :return,
+      :b_call,
+      :b_return
+    ]
+    
+    
+    def self.start(socket_path)
+      server = self.new(socket_path)
+      server.start
+    
+      trace = TracePoint.new(*TP_EVENTS) { |tp| server.handle_tp(trace, tp) }
+      trace.enable
+
+      at_exit do
+        puts "program terminated"
+        trace.disable
+        server.stop
+      end
+    end
+
+    def initialize(socket_path)
+      @socket_path = socket_path
+      @fiber = Fiber.current
+      @controller = spin { control_loop }
+      puts "@fiber: #{@fiber.inspect}"
+    end
+
+    def start
+      fiber = Fiber.current
+      @server = spin(:pdbg_server) do
+        puts("Listening on #{@socket_path}")
+        FileUtils.rm(@socket_path) if File.exists?(@socket_path)
+        socket = UNIXServer.new(@socket_path)
+        fiber << :ready
+        id = 0
+        socket.accept_loop do |client|
+          puts "accepted connection"
+          handle_client(client)
+        end
+      end
+      receive
+    end
+
+    def stop
+      @server.terminate
+      @controller.terminate
+    end
+
+    POLYPHONY_LIB_DIR = File.expand_path('../..', __dir__)
+    def handle_client(client)
+      @client = client
+      puts "trace enabled"
+    end
+
+    def control_loop
+      @cmd = :step
+      loop do
+        case @cmd
+        when :step
+          step
+        end
+      end
+    end
+
+    def step
+      tp = nil
+      fiber = nil
+      while true
+        event = receive
+        fiber = event[:fiber]
+        if fiber == @fiber && event[:kind] == :line && event[:path] !~ /#{POLYPHONY_LIB_DIR}/
+          interact_with_client(event)
+          fiber << :ok
+          fiber.__unpark__
+          return
+        end
+          
+        fiber << :ok
+        fiber.__unpark__
+      end
+    rescue => e
+      puts "Uncaught error: #{e.inspect}"
+      @trace&.disable
+      @client = nil
+    end
+
+    def interact_with_client(event)
+      @client.puts event.inspect
+      result = @client.gets&.chomp
+    end
+
+    def handle_tp(trace, tp)
+      return if @in_handle_tp
+
+      process_tp(trace, tp)
+    end
+
+    def process_tp(trace, tp)
+      @in_handle_tp = true
+      if !@client
+        wait_for_client
+      end
+
+      puts "- #{tp.event} #{tp.path}:#{tp.lineno}"
+
+      fiber = Fiber.current
+      fiber.__park__
+
+      @controller << {
+        fiber: fiber,
+        kind: tp.event,
+        path: tp.path,
+        lineno: tp.lineno
+      }
+      receive
+    ensure
+      @in_handle_tp = nil
+    end
+
+    def wait_for_client
+      puts "wait_for_client"
+      sleep 0.1 until @client
+      puts "  got client!"
+      msg = @client.gets
+      @client.puts msg
+    end
+  end
+end

data/lib/polyphony/extensions/debug.rb

@@ -10,4 +10,4 @@ module ::Kernel
       format("%p\n", args.size == 1 ? args.first : args)
     end
   end
-end
+end

data/lib/polyphony/extensions/fiber.rb

@@ -7,6 +7,10 @@ require_relative '../core/exceptions'
 module Polyphony
   # Fiber control API
   module FiberControl
+    def monitor_mailbox
+      @monitor_mailbox ||= Polyphony::Queue.new
+    end
+
     def interrupt(value = nil)
       return if @running == false
 
@@ -124,23 +128,24 @@ module Polyphony
     def await(*fibers)
       return [] if fibers.empty?
 
-      Fiber.current.message_on_child_termination = true
+      current_fiber = self.current
+      mailbox = current_fiber.monitor_mailbox
       results = {}
       fibers.each do |f|
         results[f] = nil
         if f.dead?
           # fiber already terminated, so queue message
-          Fiber.current.send [f, f.result]
+          mailbox << [f, f.result]
         else
-          f.monitor
+          f.monitor(current_fiber)
         end
       end
       exception = nil
       while !fibers.empty?
-        (fiber, result) = receive
+        (fiber, result) = mailbox.shift
         next unless fibers.include?(fiber)
-
         fibers.delete(fiber)
+        current_fiber.remove_child(fiber) if fiber.parent == current_fiber
         if result.is_a?(Exception)
           exception ||= result
           fibers.each { |f| f.terminate }
@@ -148,16 +153,16 @@ module Polyphony
           results[fiber] = result
         end
       end
-      results.values
-    ensure
-      Fiber.current.message_on_child_termination = false
       raise exception if exception
+      results.values
     end
     alias_method :join, :await
 
     def select(*fibers)
       return nil if fibers.empty?
   
+      current_fiber = self.current
+      mailbox = current_fiber.monitor_mailbox
       fibers.each do |f|
         if f.dead?
           result = f.result
@@ -165,21 +170,18 @@ module Polyphony
         end
       end
 
-      Fiber.current.message_on_child_termination = true
-      fibers.each { |f| f.monitor }
+      fibers.each { |f| f.monitor(current_fiber) }
       while true
-        (fiber, result) = receive
+        (fiber, result) = mailbox.shift
         next unless fibers.include?(fiber)
   
-        fibers.each { |f| f.unmonitor }
+        fibers.each { |f| f.unmonitor(current_fiber) }
         if result.is_a?(Exception)
           raise result
         else
           return [fiber, result]
         end
       end
-    ensure
-      Fiber.current.message_on_child_termination = false
     end
 
     # Creates and schedules with priority an out-of-band fiber that runs the
@@ -219,14 +221,6 @@ module Polyphony
       f
     end
 
-    def child_done(child_fiber, result)
-      @children.delete(child_fiber)
-
-      if result.is_a?(Exception) && !@message_on_child_termination
-        schedule_with_priority(result)
-      end
-    end
-
     def terminate_all_children(graceful = false)
       return unless @children
 
@@ -240,16 +234,25 @@ module Polyphony
     def await_all_children
       return unless @children && !@children.empty?
 
-      Fiber.await(*@children.keys)
+      Fiber.await(*@children.keys.reject { |c| c.dead? })
     end
 
     def shutdown_all_children(graceful = false)
       return unless @children
 
       @children.keys.each do |c|
+        next if c.dead?
+
         c.terminate(graceful)
         c.await
       end
+      reap_dead_children
+    end
+
+    def reap_dead_children
+      return unless @children
+
+      @children.reject! { |f| f.dead? }
     end
 
     def detach
@@ -323,7 +326,7 @@ module Polyphony
       Thread.backend.trace(:fiber_terminate, self, result)
       @result = result
 
-      inform_dependants(result, uncaught_exception)
+      inform_monitors(result, uncaught_exception)
       @running = false
     ensure
       # Prevent fiber from being resumed after terminating
@@ -341,24 +344,28 @@ module Polyphony
       [e, true]
     end
 
-    def inform_dependants(result, uncaught_exception)
+    def inform_monitors(result, uncaught_exception)
       if @monitors
         msg = [self, result]
-        @monitors.each { |f| f << msg }
+        @monitors.each_key { |f| f.monitor_mailbox << msg }
       end
 
-      @parent&.child_done(self, result)
+      if uncaught_exception && @parent
+        parent_is_monitor = @monitors&.has_key?(@parent)
+        @parent.schedule_with_priority(result) unless parent_is_monitor
+      end
     end
 
-    attr_accessor :message_on_child_termination 
+    def monitor(fiber)
+      (@monitors ||= {})[fiber] = true
+    end
 
-    def monitor
-      @monitors ||= []
-      @monitors << Fiber.current
+    def unmonitor(fiber)
+      (@monitors ||= []).delete(fiber)
     end
 
-    def unmonitor
-      @monitors.delete(Fiber.current) if @monitors
+    def monitors
+      @monitors&.keys || []
     end
 
     def dead?

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.66'
+  VERSION = '0.67'
 end

data/test/helper.rb

@@ -46,10 +46,6 @@ end
 class MiniTest::Test
   def setup
     # trace "* setup #{self.name}"
-    if Fiber.current.children.size > 0
-      puts "Children left: #{Fiber.current.children.inspect}"
-      exit!
-    end
     Fiber.current.setup_main_fiber
     Fiber.current.instance_variable_set(:@auto_watcher, nil)
     Thread.current.backend.finalize
@@ -60,6 +56,10 @@ class MiniTest::Test
   def teardown
     # trace "* teardown #{self.name}"
     Fiber.current.shutdown_all_children
+    if Fiber.current.children.size > 0
+      puts "Children left after #{self.name}: #{Fiber.current.children.inspect}"
+      exit!
+    end
     Fiber.current.instance_variable_set(:@auto_watcher, nil)
   rescue => e
     puts e

data/test/stress.rb

@@ -1,13 +1,17 @@
 # frozen_string_literal: true
 
 count = ARGV[0] ? ARGV[0].to_i : 100
+test_name = ARGV[1]
 
-TEST_CMD = 'ruby test/run.rb'
+$test_cmd = +'ruby test/run.rb'
+if test_name
+  $test_cmd << " --name #{test_name}"
+end
 
 def run_test(count)
   puts "#{count}: running tests..."
   # sleep 1
-  system(TEST_CMD)
+  system($test_cmd)
   puts
 
   return if $?.exitstatus == 0

data/test/test_fiber.rb

@@ -46,7 +46,7 @@ class FiberTest < MiniTest::Test
   def test_await_dead_children
     f1 = spin { :foo }
     f2 = spin { :bar }
-    2.times { snooze }
+    4.times { snooze }
     
     assert_equal [:foo, :bar], Fiber.await(f1, f2)
   end
@@ -67,10 +67,12 @@ class FiberTest < MiniTest::Test
     }
     Fiber.await(f2, f3)
     assert_equal [:foo, :bar, :baz], buffer
-    assert_equal 0, Fiber.current.children.size
+    assert_equal [f1], Fiber.current.children
+    Fiber.current.reap_dead_children
+    assert_equal [], Fiber.current.children
   end
 
-  def test_await_from_multiple_fibers_with_interruption=
+  def test_await_from_multiple_fibers_with_interruption
     buffer = []
     f1 = spin {
       sleep 0.02
@@ -91,6 +93,8 @@ class FiberTest < MiniTest::Test
     f1.stop
 
     snooze
+    assert_equal [f1, f2, f3], Fiber.current.children
+    Fiber.current.reap_dead_children
     assert_equal [], Fiber.current.children
   end
 
@@ -563,10 +567,10 @@ class FiberTest < MiniTest::Test
     end
 
     snooze
-    child.monitor
+    child.monitor(Fiber.current)
     spin { child << :foo }
 
-    msg = receive
+    msg = Fiber.current.monitor_mailbox.shift
     assert_equal [child, :foo], msg
   end
 
@@ -578,14 +582,14 @@ class FiberTest < MiniTest::Test
     end
 
     snooze
-    child.monitor
+    child.monitor(Fiber.current)
     spin { child << :foo }
     snooze
 
-    child.unmonitor
+    child.unmonitor(Fiber.current)
 
-    Fiber.current << :bar
-    msg = receive
+    Fiber.current.monitor_mailbox << :bar
+    msg = Fiber.current.monitor_mailbox.shift
     assert_equal :bar, msg
   end
 
@@ -598,6 +602,7 @@ class FiberTest < MiniTest::Test
 
     f.stop
     snooze
+    Fiber.current.reap_dead_children
     assert_equal [], Fiber.current.children
   end
 
@@ -1218,4 +1223,23 @@ class GracefulTerminationTest < MiniTest::Test
 
     assert_equal [1, 2], buffer
   end
+end
+
+class DebugTest < MiniTest::Test
+  def test_parking
+    buf = []
+    f = spin do
+      3.times { |i| snooze; buf << i }
+    end
+    assert_nil f.__parked__?
+    f.__park__
+    assert_equal true, f.__parked__?
+    10.times { snooze }
+    assert_equal [], buf
+    
+    f.__unpark__
+    assert_nil f.__parked__?
+    10.times { snooze }
+    assert_equal [0, 1, 2], buf
+  end
 end

data/test/test_global_api.rb

@@ -375,6 +375,7 @@ class SpinScopeTest < MiniTest::Test
       buffer << e.message
     end
     10.times { snooze }
+    Fiber.current.reap_dead_children
     assert_equal 0, Fiber.current.children.size
     assert_equal ['foobar'], buffer
   end

data/test/test_thread_pool.rb

@@ -70,7 +70,7 @@ class ThreadPoolTest < MiniTest::Test
     
     sleep 0.15 # allow time for threads to spawn
     assert_equal @pool.size, threads.uniq.size
-    assert_equal (0..9).to_a, buffer.sort
+    assert_equal (0..9).to_a, buffer.sort if IS_LINUX
   end
 
   def test_busy?

data/test/test_throttler.rb

@@ -10,7 +10,7 @@ class ThrottlerTest < MiniTest::Test
     f = spin { loop { t.process { buffer << 1 } } }
     sleep 0.2
     f.stop
-    assert_in_range 1..3, buffer.size
+    assert_in_range 1..4, buffer.size
   ensure
     t.stop
   end
@@ -23,7 +23,7 @@ class ThrottlerTest < MiniTest::Test
     end
     sleep 0.25
     f.stop
-    assert_in_range 2..6, buffer.size
+    assert_in_range 2..7, buffer.size
   ensure
     t.stop
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: '0.66'
+  version: '0.67'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-01 00:00:00.000000000 Z
+date: 2021-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -156,6 +156,7 @@ files:
 - README.md
 - Rakefile
 - TODO.md
+- bin/pdbg
 - bin/polyphony-debug
 - bin/stress.rb
 - bin/test
@@ -356,6 +357,7 @@ files:
 - lib/polyphony/core/thread_pool.rb
 - lib/polyphony/core/throttler.rb
 - lib/polyphony/core/timer.rb
+- lib/polyphony/debugger/server.rb
 - lib/polyphony/extensions/core.rb
 - lib/polyphony/extensions/debug.rb
 - lib/polyphony/extensions/fiber.rb