polyphony 0.57.0 → 0.60

data/ext/polyphony/backend_io_uring_context.c

@@ -25,6 +25,8 @@ void context_store_initialize(op_context_store_t *store) {
   store->last_id = 0;
   store->available = NULL;
   store->taken = NULL;
+  store->available_count = 0;
+  store->taken_count = 0;
 }
 
 inline op_context_t *context_store_acquire(op_context_store_t *store, enum op_type type) {
@@ -32,13 +34,12 @@ inline op_context_t *context_store_acquire(op_context_store_t *store, enum op_ty
   if (ctx) {
     if (ctx->next) ctx->next->prev = NULL;
     store->available = ctx->next;
+    store->available_count--;
   }
   else {
     ctx = malloc(sizeof(op_context_t));
   }
   ctx->id = (++store->last_id);
-  // printf("acquire %d (%s)\n", ctx->id, op_type_to_str(type));
-  
   ctx->prev = NULL;
   ctx->next = store->taken;
   if (store->taken) store->taken->prev = ctx;
@@ -50,18 +51,25 @@ inline op_context_t *context_store_acquire(op_context_store_t *store, enum op_ty
   ctx->ref_count = 2;
   ctx->result = 0;
 
+  store->taken_count++;
+
+  // printf("acquire %p %d (%s, ref_count: %d) taken: %d\n", ctx, ctx->id, op_type_to_str(type), ctx->ref_count, store->taken_count);
+
   return ctx;
 }
 
 // returns true if ctx was released
 inline int context_store_release(op_context_store_t *store, op_context_t *ctx) {
-  // printf("release %d (%s, ref_count: %d)\n", ctx->id, op_type_to_str(ctx->type), ctx->ref_count);
+  // printf("release %p %d (%s, ref_count: %d)\n", ctx, ctx->id, op_type_to_str(ctx->type), ctx->ref_count);
 
   assert(ctx->ref_count);
   
   ctx->ref_count--;
   if (ctx->ref_count) return 0;
 
+  store->taken_count--;
+  store->available_count++;
+
   if (ctx->next) ctx->next->prev = ctx->prev;
   if (ctx->prev) ctx->prev->next = ctx->next;
   if (store->taken == ctx) store->taken = ctx->next;

data/ext/polyphony/backend_io_uring_context.h

@@ -22,7 +22,7 @@ typedef struct op_context {
   struct op_context *prev;
   struct op_context *next;
   enum op_type      type: 16;
-  unsigned int      ref_count : 16;         
+  unsigned int      ref_count : 16;       
   int               id;
   int               result;
   VALUE             fiber;
@@ -31,8 +31,10 @@ typedef struct op_context {
 
 typedef struct op_context_store {
   int           last_id;
-  op_context_t *available;
-  op_context_t *taken;
+  op_context_t  *available;
+  op_context_t  *taken;
+  int           available_count;
+  int           taken_count;
 } op_context_store_t;
 
 const char *op_type_to_str(enum op_type type);

data/ext/polyphony/backend_libev.c

@@ -73,13 +73,23 @@ typedef struct Backend_t {
   struct ev_async break_async;
 } Backend_t;
 
+static void Backend_mark(void *ptr) {
+  Backend_t *backend = ptr;
+  backend_base_mark(&backend->base);
+}
+
+static void Backend_free(void *ptr) {
+  Backend_t *backend = ptr;
+  backend_base_finalize(&backend->base);
+}
+
 static size_t Backend_size(const void *ptr) {
   return sizeof(Backend_t);
 }
 
 static const rb_data_type_t Backend_type = {
     "LibevBackend",
-    {0, 0, Backend_size,},
+    {Backend_mark, Backend_free, Backend_size,},
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };
 
@@ -110,6 +120,8 @@ static VALUE Backend_initialize(VALUE self) {
   Backend_t *backend;
   
   GetBackend(self, backend);
+
+  backend_base_initialize(&backend->base);
   backend->ev_loop = libev_new_loop();
 
   // start async watcher used for breaking a poll op (from another thread)
@@ -119,11 +131,6 @@ static VALUE Backend_initialize(VALUE self) {
   // block when no other watcher is active
   ev_unref(backend->ev_loop);
 
-  backend->base.currently_polling = 0;
-  backend->base.pending_count = 0;
-  backend->base.idle_gc_period = 0;
-  backend->base.idle_gc_last_time = 0;
-
   return Qnil;
 }
 
@@ -153,24 +160,38 @@ VALUE Backend_post_fork(VALUE self) {
   return self;
 }
 
-inline unsigned int Backend_pending_count(VALUE self) {
+inline VALUE Backend_poll(VALUE self, VALUE blocking) {
   Backend_t *backend;
   GetBackend(self, backend);
 
-  return backend->base.pending_count;
+  COND_TRACE(&backend->base, 2, SYM_fiber_event_poll_enter, rb_fiber_current());
+  backend->base.currently_polling = 1;
+  ev_run(backend->ev_loop, blocking == Qtrue ? EVRUN_ONCE : EVRUN_NOWAIT);
+  backend->base.currently_polling = 0;
+  COND_TRACE(&backend->base, 2, SYM_fiber_event_poll_leave, rb_fiber_current());
+
+  return self;
 }
 
-VALUE Backend_poll(VALUE self, VALUE nowait, VALUE current_fiber, VALUE runqueue) {
+inline void Backend_schedule_fiber(VALUE thread, VALUE self, VALUE fiber, VALUE value, int prioritize) {
   Backend_t *backend;
   GetBackend(self, backend);
 
-  COND_TRACE(2, SYM_fiber_event_poll_enter, current_fiber);
-  backend->base.currently_polling = 1;
-  ev_run(backend->ev_loop, nowait == Qtrue ? EVRUN_NOWAIT : EVRUN_ONCE);
-  backend->base.currently_polling = 0;
-  COND_TRACE(2, SYM_fiber_event_poll_leave, current_fiber);
+  backend_base_schedule_fiber(thread, self, &backend->base, fiber, value, prioritize);
+}
 
-  return self;
+inline void Backend_unschedule_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  runqueue_delete(&backend->base.runqueue, fiber);  
+}
+
+inline VALUE Backend_switch_fiber(VALUE self) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  return backend_base_switch_fiber(self, &backend->base);
 }
 
 VALUE Backend_wakeup(VALUE self) {
@@ -190,6 +211,16 @@ VALUE Backend_wakeup(VALUE self) {
   return Qnil;
 }
 
+inline struct backend_stats Backend_stats(VALUE self) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  return (struct backend_stats){
+    .scheduled_fibers = runqueue_len(&backend->base.runqueue),
+    .pending_ops = backend->base.pending_count
+  };
+}
+
 struct libev_io {
   struct ev_io io;
   VALUE fiber;
@@ -229,14 +260,20 @@ VALUE libev_wait_fd(Backend_t *backend, int fd, int events, int raise_exception)
   return switchpoint_result;
 }
 
-VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof) {
+VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof, VALUE pos) {
   Backend_t *backend;
   struct libev_io watcher;
   rb_io_t *fptr;
   long dynamic_len = length == Qnil;
   long len = dynamic_len ? 4096 : NUM2INT(length);
-  int shrinkable = io_setstrbuf(&str, len);
-  char *buf = RSTRING_PTR(str);
+  long buf_pos = NUM2INT(pos);
+  if (str != Qnil) {
+    int current_len = RSTRING_LEN(str);
+    if (buf_pos < 0 || buf_pos > current_len) buf_pos = current_len;
+  }
+  else buf_pos = 0;
+  int shrinkable = io_setstrbuf(&str, buf_pos + len);
+  char *buf = RSTRING_PTR(str) + buf_pos;
   long total = 0;
   VALUE switchpoint_result = Qnil;
   int read_to_eof = RTEST(to_eof);
@@ -272,9 +309,9 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof)
       if (total == len) {
         if (!dynamic_len) break;
 
-        rb_str_resize(str, total);
+        rb_str_resize(str, buf_pos + total);
         rb_str_modify_expand(str, len);
-        buf = RSTRING_PTR(str) + total;
+        buf = RSTRING_PTR(str) + buf_pos + total;
         shrinkable = 0;
         len += len;
       }
@@ -282,7 +319,7 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof)
     }
   }
 
-  io_set_read_length(str, total, shrinkable);
+  io_set_read_length(str, buf_pos + total, shrinkable);
   io_enc_str(str, fptr);
 
   if (total == 0) return Qnil;
@@ -295,17 +332,17 @@ error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
 
-VALUE Backend_recv(VALUE self, VALUE io, VALUE str, VALUE length) {
-  return Backend_read(self, io, str, length, Qnil);
+VALUE Backend_recv(VALUE self, VALUE io, VALUE str, VALUE length, VALUE pos) {
+  return Backend_read(self, io, str, length, Qnil, pos);
 }
 
-VALUE Backend_read_loop(VALUE self, VALUE io) {
+VALUE Backend_read_loop(VALUE self, VALUE io, VALUE maxlen) {
   Backend_t *backend;
   struct libev_io watcher;
   rb_io_t *fptr;
   VALUE str;
   long total;
-  long len = 8192;
+  long len = NUM2INT(maxlen);
   int shrinkable;
   char *buf;
   VALUE switchpoint_result = Qnil;
@@ -1290,6 +1327,13 @@ VALUE Backend_idle_gc_period_set(VALUE self, VALUE period) {
   return self;
 }
 
+VALUE Backend_idle_proc_set(VALUE self, VALUE block) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+  backend->base.idle_proc = block;
+  return self;
+}
+
 inline VALUE Backend_run_idle_tasks(VALUE self) {
   Backend_t *backend;
   GetBackend(self, backend);
@@ -1433,6 +1477,21 @@ error:
   return RAISE_EXCEPTION(result);
 }
 
+VALUE Backend_trace(int argc, VALUE *argv, VALUE self) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+  backend_trace(&backend->base, argc, argv);
+  return self;
+}
+
+VALUE Backend_trace_proc_set(VALUE self, VALUE block) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend->base.trace_proc = block;
+  return self;
+}
+
 void Init_Backend() {
   ev_set_allocator(xrealloc);
 
@@ -1442,22 +1501,25 @@ void Init_Backend() {
   rb_define_method(cBackend, "initialize", Backend_initialize, 0);
   rb_define_method(cBackend, "finalize", Backend_finalize, 0);
   rb_define_method(cBackend, "post_fork", Backend_post_fork, 0);
+  rb_define_method(cBackend, "trace", Backend_trace, -1);
+  rb_define_method(cBackend, "trace_proc=", Backend_trace_proc_set, 1);
 
-  rb_define_method(cBackend, "poll", Backend_poll, 3);
+  rb_define_method(cBackend, "poll", Backend_poll, 1);
   rb_define_method(cBackend, "break", Backend_wakeup, 0);
   rb_define_method(cBackend, "kind", Backend_kind, 0);
   rb_define_method(cBackend, "chain", Backend_chain, -1);
   rb_define_method(cBackend, "idle_gc_period=", Backend_idle_gc_period_set, 1);
+  rb_define_method(cBackend, "idle_proc=", Backend_idle_proc_set, 1);
   rb_define_method(cBackend, "splice_chunks", Backend_splice_chunks, 7);
 
   rb_define_method(cBackend, "accept", Backend_accept, 2);
   rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
   rb_define_method(cBackend, "connect", Backend_connect, 3);
   rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
-  rb_define_method(cBackend, "read", Backend_read, 4);
-  rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
-  rb_define_method(cBackend, "recv", Backend_recv, 3);
-  rb_define_method(cBackend, "recv_loop", Backend_read_loop, 1);
+  rb_define_method(cBackend, "read", Backend_read, 5);
+  rb_define_method(cBackend, "read_loop", Backend_read_loop, 2);
+  rb_define_method(cBackend, "recv", Backend_recv, 4);
+  rb_define_method(cBackend, "recv_loop", Backend_read_loop, 2);
   rb_define_method(cBackend, "recv_feed_loop", Backend_feed_loop, 3);
   rb_define_method(cBackend, "send", Backend_send, 3);
   rb_define_method(cBackend, "sendv", Backend_sendv, 3);

data/ext/polyphony/extconf.rb

@@ -8,8 +8,8 @@ use_pidfd_open = false
 force_use_libev = ENV['POLYPHONY_USE_LIBEV'] != nil
 linux = RUBY_PLATFORM =~ /linux/
 
-if linux && `uname -sr` =~ /Linux 5\.([\d+])/
-  kernel_minor_version = $1.gsub('.', '').to_i
+if linux && `uname -sr` =~ /Linux 5\.(\d+)/
+  kernel_minor_version = $1.to_i
   use_liburing = !force_use_libev && kernel_minor_version >= 6
   use_pidfd_open = kernel_minor_version >= 3
 end

data/ext/polyphony/fiber.c

@@ -1,6 +1,5 @@
 #include "polyphony.h"
 
-ID ID_fiber_trace;
 ID ID_ivar_auto_watcher;
 ID ID_ivar_mailbox;
 ID ID_ivar_result;
@@ -82,34 +81,6 @@ static VALUE Fiber_state(VALUE self) {
   return SYM_waiting;
 }
 
-VALUE Fiber_await(VALUE self) {
-  VALUE result;
-
-  // we compare with false, since a fiber that has not yet started will have
-  // @running set to nil
-  if (rb_ivar_get(self, ID_ivar_running) == Qfalse) {
-    result = rb_ivar_get(self, ID_ivar_result);
-    RAISE_IF_EXCEPTION(result);
-    return result;
-  }
-
-  VALUE fiber = rb_fiber_current();
-  VALUE waiting_fibers = rb_ivar_get(self, ID_ivar_waiting_fibers);
-  if (waiting_fibers == Qnil) {
-    waiting_fibers = rb_hash_new();
-    rb_ivar_set(self, ID_ivar_waiting_fibers, waiting_fibers);
-  }
-  rb_hash_aset(waiting_fibers, fiber, Qtrue);
-
-  VALUE backend = rb_ivar_get(rb_thread_current(), ID_ivar_backend);
-  result = Backend_wait_event(backend, Qnil);
-
-  rb_hash_delete(waiting_fibers, fiber);
-  RAISE_IF_EXCEPTION(result);
-  RB_GC_GUARD(result);
-  return result;
-}
-
 VALUE Fiber_send(VALUE self, VALUE value) {
   VALUE mailbox = rb_ivar_get(self, ID_ivar_mailbox);
   if (mailbox == Qnil) {
@@ -126,7 +97,7 @@ VALUE Fiber_receive(VALUE self) {
     mailbox = rb_funcall(cQueue, ID_new, 0);
     rb_ivar_set(self, ID_ivar_mailbox, mailbox);
   }
-  return Queue_shift(mailbox);  
+  return Queue_shift(mailbox);
 }
 
 VALUE Fiber_mailbox(VALUE self) {
@@ -151,9 +122,6 @@ void Init_Fiber() {
   rb_define_method(cFiber, "state", Fiber_state, 0);
   rb_define_method(cFiber, "auto_watcher", Fiber_auto_watcher, 0);
 
-  rb_define_method(cFiber, "await", Fiber_await, 0);
-  rb_define_method(cFiber, "join", Fiber_await, 0);
-
   rb_define_method(cFiber, "<<", Fiber_send, 1);
   rb_define_method(cFiber, "send", Fiber_send, 1);
   rb_define_method(cFiber, "receive", Fiber_receive, 0);
@@ -169,7 +137,6 @@ void Init_Fiber() {
   rb_global_variable(&SYM_runnable);
   rb_global_variable(&SYM_waiting);
 
-  ID_fiber_trace              = rb_intern("__fiber_trace__");
   ID_ivar_auto_watcher        = rb_intern("@auto_watcher");
   ID_ivar_mailbox             = rb_intern("@mailbox");
   ID_ivar_result              = rb_intern("@result");

data/ext/polyphony/polyphony.c

@@ -42,11 +42,6 @@ static VALUE Polyphony_suspend(VALUE self) {
   return ret;
 }
 
-VALUE Polyphony_trace(VALUE self, VALUE enabled) {
-  __tracing_enabled__ = RTEST(enabled) ? 1 : 0;
-  return Qnil;
-}
-
 VALUE Polyphony_backend_accept(VALUE self, VALUE server_socket, VALUE socket_class) {
   return Backend_accept(BACKEND(), server_socket, socket_class);
 }
@@ -63,20 +58,20 @@ VALUE Polyphony_backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE me
   return Backend_feed_loop(BACKEND(), io, receiver, method);
 }
 
-VALUE Polyphony_backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof) {
-  return Backend_read(BACKEND(), io, str, length, to_eof);
+VALUE Polyphony_backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof, VALUE pos) {
+  return Backend_read(BACKEND(), io, str, length, to_eof, pos);
 }
 
-VALUE Polyphony_backend_read_loop(VALUE self, VALUE io) {
-  return Backend_read_loop(BACKEND(), io);
+VALUE Polyphony_backend_read_loop(VALUE self, VALUE io, VALUE maxlen) {
+  return Backend_read_loop(BACKEND(), io, maxlen);
 }
 
-VALUE Polyphony_backend_recv(VALUE self, VALUE io, VALUE str, VALUE length) {
-  return Backend_recv(BACKEND(), io, str, length);
+VALUE Polyphony_backend_recv(VALUE self, VALUE io, VALUE str, VALUE length, VALUE pos) {
+  return Backend_recv(BACKEND(), io, str, length, pos);
 }
 
-VALUE Polyphony_backend_recv_loop(VALUE self, VALUE io) {
-  return Backend_recv_loop(BACKEND(), io);
+VALUE Polyphony_backend_recv_loop(VALUE self, VALUE io, VALUE maxlen) {
+  return Backend_recv_loop(BACKEND(), io, maxlen);
 }
 
 VALUE Polyphony_backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
@@ -130,17 +125,15 @@ VALUE Polyphony_backend_write(int argc, VALUE *argv, VALUE self) {
 void Init_Polyphony() {
   mPolyphony = rb_define_module("Polyphony");
 
-  rb_define_singleton_method(mPolyphony, "trace", Polyphony_trace, 1);
-
   // backend methods
   rb_define_singleton_method(mPolyphony, "backend_accept", Polyphony_backend_accept, 2);
   rb_define_singleton_method(mPolyphony, "backend_accept_loop", Polyphony_backend_accept_loop, 2);
   rb_define_singleton_method(mPolyphony, "backend_connect", Polyphony_backend_connect, 3);
   rb_define_singleton_method(mPolyphony, "backend_feed_loop", Polyphony_backend_feed_loop, 3);
-  rb_define_singleton_method(mPolyphony, "backend_read", Polyphony_backend_read, 4);
-  rb_define_singleton_method(mPolyphony, "backend_read_loop", Polyphony_backend_read_loop, 1);
-  rb_define_singleton_method(mPolyphony, "backend_recv", Polyphony_backend_recv, 3);
-  rb_define_singleton_method(mPolyphony, "backend_recv_loop", Polyphony_backend_recv_loop, 1);
+  rb_define_singleton_method(mPolyphony, "backend_read", Polyphony_backend_read, 5);
+  rb_define_singleton_method(mPolyphony, "backend_read_loop", Polyphony_backend_read_loop, 2);
+  rb_define_singleton_method(mPolyphony, "backend_recv", Polyphony_backend_recv, 4);
+  rb_define_singleton_method(mPolyphony, "backend_recv_loop", Polyphony_backend_recv_loop, 2);
   rb_define_singleton_method(mPolyphony, "backend_recv_feed_loop", Polyphony_backend_recv_feed_loop, 3);
   rb_define_singleton_method(mPolyphony, "backend_send", Polyphony_backend_send, 3);
   rb_define_singleton_method(mPolyphony, "backend_sendv", Polyphony_backend_sendv, 3);