polyphony 0.60 → 0.64

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 515e9a5686bb0eedb02ad626e491b3e8acb350a765a468029e0b5357673f443c
-  data.tar.gz: 94fd7eaedd37c01f1ebc33ba78ffb00d3e8fc42f4a0266bf63ba8eedb83d008c
+  metadata.gz: 5e3938b66b15caff258b95c7f1421f71bc0bb4f7b94dd9db4a3ad0614fd35ded
+  data.tar.gz: 2198ea0483f959491372074007d5c32416de69ebcf73a8aca51832ea6036ea87
 SHA512:
-  metadata.gz: a546bcf43f556dc7d6bbc3c04b9448141a539e91d2d893441a161eecf9246ff516a54cf94cae476ef9358470e5475c98577c807fd1cef1f712f342337d3c8cc8
-  data.tar.gz: e0bb07cc0028c3205f3c2d87a59699e35e3d4d0e797eff63258892475548086125ed40e63152c56253a1c596b680eacc7080bf08f7d152576c34dc57df0206ab
+  metadata.gz: 318b3b2549a6d7d3c2bb1b8396adcf95109ef17f6235e25bcf1540fd3c5777d6f2666858d5f539cf17dd0c2102553eb853ba3b38b96d9476d2727a7ddfb0ed70
+  data.tar.gz: 025f5037c922531235bcf56076ca6986cb1d983b37eceda8e1585fa90e23f67c0abe17a81b0329851e80c050e9ccce8ba53f7a0d95565e6568546dbb923f9a25

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 0.64 2021-07-26
+
+- Add optional raise_on_eof argument to `#readpartial`
+
+## 0.63 2021-07-26
+
+- Add support for specifying buf and buf_pos in `IO#read`
+- Fix `Socket#read` to work and conform to `IO#read` interface
+
+## 0.62 2021-07-21
+
+- Add `runqueue_size` to backend stats
+
+## 0.61 2021-07-20
+
+- Add more statistics, move stats to `Backend#stats`
+
 ## 0.60 2021-07-15
 
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.59.2)
+    polyphony (0.64)
 
 GEM
   remote: https://rubygems.org/

data/Rakefile

@@ -9,8 +9,8 @@ Rake::ExtensionTask.new("polyphony_ext") do |ext|
 end
 
 task :recompile => [:clean, :compile]
-
 task :default => [:compile, :test]
+
 task :test do
   exec 'ruby test/run.rb'
 end

data/examples/io/echo_server.rb

@@ -3,16 +3,25 @@
 require 'bundler/setup'
 require 'polyphony'
 
+spin_loop(interval: 5) { p Thread.backend.stats }
+
 server = TCPServer.open('127.0.0.1', 1234)
 puts "Pid: #{Process.pid}"
 puts 'Echoing on port 1234...'
-while (client = server.accept)
-  spin do
-    while (data = client.gets)
-      # client.send("you said: #{data.chomp}!\n", 0)
-      client.write('you said: ', data.chomp, "!\n")
+begin
+  while (client = server.accept)
+    spin do
+      while (data = client.gets)
+        # client.send("you said: #{data.chomp}!\n", 0)
+        client.write('you said: ', data.chomp, "!\n")
+      end
+    rescue Errno::ECONNRESET
+      'Connection reset...'
+    ensure
+      client.shutdown
+      client.close
     end
-  rescue Errno::ECONNRESET
-    'Connection reset...'
   end
+ensure
+  server.close
 end

data/ext/polyphony/backend_common.c

@@ -8,6 +8,9 @@
 inline void backend_base_initialize(struct Backend_base *base) {
   runqueue_initialize(&base->runqueue);
   base->currently_polling = 0;
+  base->op_count = 0;
+  base->switch_count = 0;
+  base->poll_count = 0;
   base->pending_count = 0;
   base->idle_gc_period = 0;
   base->idle_gc_last_time = 0;
@@ -25,8 +28,10 @@ inline void backend_base_mark(struct Backend_base *base) {
   runqueue_mark(&base->runqueue);
 }
 
+const unsigned int ANTI_STARVE_SWITCH_COUNT_THRESHOLD = 64;
+
 inline void conditional_nonblocking_poll(VALUE backend, struct Backend_base *base, VALUE current, VALUE next) {
-  if (runqueue_should_poll_nonblocking(&base->runqueue) || next == current) 
+  if ((base->switch_count % ANTI_STARVE_SWITCH_COUNT_THRESHOLD) == 0 || next == current) 
     Backend_poll(backend, Qnil);
 }
 
@@ -36,7 +41,8 @@ VALUE backend_base_switch_fiber(VALUE backend, struct Backend_base *base) {
   unsigned int pending_ops_count = base->pending_count;
   unsigned int backend_was_polled = 0;
   unsigned int idle_tasks_run_count = 0;
-
+  
+  base->switch_count++;
   COND_TRACE(base, 2, SYM_fiber_switchpoint, current_fiber);
 
   while (1) {
@@ -289,3 +295,61 @@ inline void backend_run_idle_tasks(struct Backend_base *base) {
   rb_gc_start();
   rb_gc_disable();
 }
+
+inline struct backend_stats backend_base_stats(struct Backend_base *base) {
+  struct backend_stats stats = {
+    .runqueue_size = runqueue_size(&base->runqueue),
+    .runqueue_length = runqueue_len(&base->runqueue),
+    .runqueue_max_length = runqueue_max_len(&base->runqueue),
+    .op_count = base->op_count,
+    .switch_count = base->switch_count,
+    .poll_count = base->poll_count,
+    .pending_ops = base->pending_count
+  };
+
+  base->op_count = 0;
+  base->switch_count = 0;
+  base->poll_count = 0;
+  return stats;
+}
+
+VALUE SYM_runqueue_size;
+VALUE SYM_runqueue_length;
+VALUE SYM_runqueue_max_length;
+VALUE SYM_op_count;
+VALUE SYM_switch_count;
+VALUE SYM_poll_count;
+VALUE SYM_pending_ops;
+
+VALUE Backend_stats(VALUE self) {
+  struct backend_stats backend_stats = backend_get_stats(self);
+
+  VALUE stats = rb_hash_new();
+  rb_hash_aset(stats, SYM_runqueue_size, INT2NUM(backend_stats.runqueue_size));
+  rb_hash_aset(stats, SYM_runqueue_length, INT2NUM(backend_stats.runqueue_length));
+  rb_hash_aset(stats, SYM_runqueue_max_length, INT2NUM(backend_stats.runqueue_max_length));
+  rb_hash_aset(stats, SYM_op_count, INT2NUM(backend_stats.op_count));
+  rb_hash_aset(stats, SYM_switch_count, INT2NUM(backend_stats.switch_count));
+  rb_hash_aset(stats, SYM_poll_count, INT2NUM(backend_stats.poll_count));
+  rb_hash_aset(stats, SYM_pending_ops, INT2NUM(backend_stats.pending_ops));
+  RB_GC_GUARD(stats);
+  return stats;
+}
+
+void backend_setup_stats_symbols() {
+  SYM_runqueue_size       = ID2SYM(rb_intern("runqueue_size"));
+  SYM_runqueue_length     = ID2SYM(rb_intern("runqueue_length"));
+  SYM_runqueue_max_length = ID2SYM(rb_intern("runqueue_max_length"));
+  SYM_op_count            = ID2SYM(rb_intern("op_count"));
+  SYM_switch_count        = ID2SYM(rb_intern("switch_count"));
+  SYM_poll_count          = ID2SYM(rb_intern("poll_count"));
+  SYM_pending_ops         = ID2SYM(rb_intern("pending_ops"));
+  
+  rb_global_variable(&SYM_runqueue_size);
+  rb_global_variable(&SYM_runqueue_length);
+  rb_global_variable(&SYM_runqueue_max_length);
+  rb_global_variable(&SYM_op_count);
+  rb_global_variable(&SYM_switch_count);
+  rb_global_variable(&SYM_poll_count);
+  rb_global_variable(&SYM_pending_ops);
+}

data/ext/polyphony/backend_common.h

@@ -6,13 +6,21 @@
 #include "runqueue.h"
 
 struct backend_stats {
-  int scheduled_fibers;
-  int pending_ops;
+  unsigned int runqueue_size;
+  unsigned int runqueue_length;
+  unsigned int runqueue_max_length;
+  unsigned int op_count;
+  unsigned int switch_count;
+  unsigned int poll_count;
+  unsigned int pending_ops;
 };
 
 struct Backend_base {
   runqueue_t runqueue;
   unsigned int currently_polling;
+  unsigned int op_count;
+  unsigned int switch_count;
+  unsigned int poll_count;
   unsigned int pending_count;
   double idle_gc_period;
   double idle_gc_last_time;
@@ -26,6 +34,7 @@ void backend_base_mark(struct Backend_base *base);
 VALUE backend_base_switch_fiber(VALUE backend, struct Backend_base *base);
 void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_base *base, VALUE fiber, VALUE value, int prioritize);
 void backend_trace(struct Backend_base *base, int argc, VALUE *argv);
+struct backend_stats backend_base_stats(struct Backend_base *base);
 
 // tracing
 #define SHOULD_TRACE(base) ((base)->trace_proc != Qnil)
@@ -59,6 +68,7 @@ VALUE io_enc_str(VALUE str, rb_io_t *fptr);
 //////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////
 
+struct backend_stats backend_get_stats(VALUE self);
 VALUE backend_await(struct Backend_base *backend);
 VALUE backend_snooze();
 
@@ -91,7 +101,10 @@ VALUE backend_timeout_exception(VALUE exception);
 VALUE Backend_timeout_ensure_safe(VALUE arg);
 VALUE Backend_timeout_ensure_safe(VALUE arg);
 VALUE Backend_sendv(VALUE self, VALUE io, VALUE ary, VALUE flags);
+VALUE Backend_stats(VALUE self);
 void backend_run_idle_tasks(struct Backend_base *base);
 void io_verify_blocking_mode(rb_io_t *fptr, VALUE io, VALUE blocking);
 
+void backend_setup_stats_symbols();
+
 #endif /* BACKEND_COMMON_H */

data/ext/polyphony/backend_io_uring.c

@@ -191,6 +191,8 @@ inline VALUE Backend_poll(VALUE self, VALUE blocking) {
   Backend_t *backend;
   GetBackend(self, backend);
 
+  backend->base.poll_count++;
+
   if (!is_blocking && backend->pending_sqes) {
     backend->pending_sqes = 0;
     io_uring_submit(&backend->ring);
@@ -234,14 +236,11 @@ inline VALUE Backend_switch_fiber(VALUE self) {
   return backend_base_switch_fiber(self, &backend->base);
 }
 
-inline struct backend_stats Backend_stats(VALUE self) {
+inline struct backend_stats backend_get_stats(VALUE self) {
   Backend_t *backend;
   GetBackend(self, backend);
 
-  return (struct backend_stats){
-    .scheduled_fibers = runqueue_len(&backend->base.runqueue),
-    .pending_ops = backend->base.pending_count
-  };
+  return backend_base_stats(&backend->base);
 }
 
 VALUE Backend_wakeup(VALUE self) {
@@ -279,6 +278,7 @@ int io_uring_backend_defer_submit_and_await(
 {
   VALUE switchpoint_result = Qnil;
 
+  backend->base.op_count++;
   if (sqe) {
     io_uring_sqe_set_data(sqe, ctx);
     io_uring_sqe_set_flags(sqe, IOSQE_ASYNC);
@@ -1043,6 +1043,7 @@ VALUE Backend_timeout(int argc, VALUE *argv, VALUE self) {
   io_uring_prep_timeout(sqe, &ts, 0, 0);
   io_uring_sqe_set_data(sqe, ctx);
   io_uring_backend_defer_submit(backend);
+  backend->base.op_count++;
 
   struct Backend_timeout_ctx timeout_ctx = {backend, ctx};
   result = rb_ensure(Backend_timeout_ensure_safe, Qnil, Backend_timeout_ensure, (VALUE)&timeout_ctx);
@@ -1210,6 +1211,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
     sqe_count++;
   }
 
+  backend->base.op_count += sqe_count;
   ctx->ref_count = sqe_count + 1;
   io_uring_backend_defer_submit(backend);
   resume_value = backend_await((struct Backend_base *)backend);
@@ -1346,6 +1348,7 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
   if (prefix != Qnil) {
     splice_chunks_get_sqe(backend, &ctx, &sqe, OP_WRITE);
     splice_chunks_prep_write(ctx, sqe, dest_fptr->fd, prefix);
+    backend->base.op_count++;
   }
 
   while (1) {
@@ -1355,7 +1358,8 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
 
     splice_chunks_get_sqe(backend, &ctx, &sqe, OP_SPLICE);
     splice_chunks_prep_splice(ctx, sqe, src_fptr->fd, pipefd[1], maxlen);
-    
+    backend->base.op_count++;
+
     SPLICE_CHUNKS_AWAIT_OPS(backend, &ctx, &chunk_len, &switchpoint_result);
     if (chunk_len == 0) break;
     
@@ -1367,15 +1371,18 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
       chunk_prefix_str = (TYPE(chunk_prefix) == T_STRING) ? chunk_prefix : rb_funcall(chunk_prefix, ID_call, 1, chunk_len_value);
       splice_chunks_get_sqe(backend, &ctx, &sqe, OP_WRITE);
       splice_chunks_prep_write(ctx, sqe, dest_fptr->fd, chunk_prefix_str);
+      backend->base.op_count++;
     }
 
     splice_chunks_get_sqe(backend, &ctx, &sqe, OP_SPLICE);
     splice_chunks_prep_splice(ctx, sqe, pipefd[0], dest_fptr->fd, chunk_len);
+    backend->base.op_count++;
 
     if (chunk_postfix != Qnil) {
       chunk_postfix_str = (TYPE(chunk_postfix) == T_STRING) ? chunk_postfix : rb_funcall(chunk_postfix, ID_call, 1, chunk_len_value);
       splice_chunks_get_sqe(backend, &ctx, &sqe, OP_WRITE);
       splice_chunks_prep_write(ctx, sqe, dest_fptr->fd, chunk_postfix_str);
+      backend->base.op_count++;
     }
 
     RB_GC_GUARD(chunk_prefix_str);
@@ -1385,6 +1392,7 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
   if (postfix != Qnil) {
     splice_chunks_get_sqe(backend, &ctx, &sqe, OP_WRITE);
     splice_chunks_prep_write(ctx, sqe, dest_fptr->fd, postfix);
+    backend->base.op_count++;
   }
   if (ctx) {
     SPLICE_CHUNKS_AWAIT_OPS(backend, &ctx, 0, &switchpoint_result);
@@ -1430,6 +1438,7 @@ void Init_Backend() {
   rb_define_method(cBackend, "post_fork", Backend_post_fork, 0);
   rb_define_method(cBackend, "trace", Backend_trace, -1);
   rb_define_method(cBackend, "trace_proc=", Backend_trace_proc_set, 1);
+  rb_define_method(cBackend, "stats", Backend_stats, 0);
 
   rb_define_method(cBackend, "poll", Backend_poll, 1);
   rb_define_method(cBackend, "break", Backend_wakeup, 0);
@@ -1469,6 +1478,8 @@ void Init_Backend() {
   SYM_send = ID2SYM(rb_intern("send"));
   SYM_splice = ID2SYM(rb_intern("splice"));
   SYM_write = ID2SYM(rb_intern("write"));
+
+  backend_setup_stats_symbols();
 }
 
 #endif // POLYPHONY_BACKEND_LIBURING

data/ext/polyphony/backend_libev.c

@@ -164,6 +164,8 @@ inline VALUE Backend_poll(VALUE self, VALUE blocking) {
   Backend_t *backend;
   GetBackend(self, backend);
 
+  backend->base.poll_count++;
+
   COND_TRACE(&backend->base, 2, SYM_fiber_event_poll_enter, rb_fiber_current());
   backend->base.currently_polling = 1;
   ev_run(backend->ev_loop, blocking == Qtrue ? EVRUN_ONCE : EVRUN_NOWAIT);
@@ -211,14 +213,11 @@ VALUE Backend_wakeup(VALUE self) {
   return Qnil;
 }
 
-inline struct backend_stats Backend_stats(VALUE self) {
+inline struct backend_stats backend_get_stats(VALUE self) {
   Backend_t *backend;
   GetBackend(self, backend);
 
-  return (struct backend_stats){
-    .scheduled_fibers = runqueue_len(&backend->base.runqueue),
-    .pending_ops = backend->base.pending_count
-  };
+  return backend_base_stats(&backend->base);
 }
 
 struct libev_io {
@@ -289,6 +288,7 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof,
   OBJ_TAINT(str);
 
   while (1) {
+    backend->base.op_count++;
     ssize_t n = read(fptr->fd, buf, len - total);
     if (n < 0) {
       int e = errno;
@@ -359,6 +359,7 @@ VALUE Backend_read_loop(VALUE self, VALUE io, VALUE maxlen) {
   watcher.fiber = Qnil;
 
   while (1) {
+    backend->base.op_count++;
     ssize_t n = read(fptr->fd, buf, len);
     if (n < 0) {
       int e = errno;
@@ -411,6 +412,7 @@ VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
   watcher.fiber = Qnil;
 
   while (1) {
+    backend->base.op_count++;
     ssize_t n = read(fptr->fd, buf, len);
     if (n < 0) {
       int e = errno;
@@ -458,6 +460,7 @@ VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
   watcher.fiber = Qnil;
 
   while (left > 0) {
+    backend->base.op_count++;
     ssize_t n = write(fptr->fd, buf, left);
     if (n < 0) {
       int e = errno;
@@ -517,6 +520,7 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
   iov_ptr = iov;
 
   while (1) {
+    backend->base.op_count++;
     ssize_t n = writev(fptr->fd, iov_ptr, iov_count);
     if (n < 0) {
       int e = errno;
@@ -588,6 +592,7 @@ VALUE Backend_accept(VALUE self, VALUE server_socket, VALUE socket_class) {
   io_verify_blocking_mode(fptr, server_socket, Qfalse);
   watcher.fiber = Qnil;
   while (1) {
+    backend->base.op_count++;
     fd = accept(fptr->fd, &addr, &len);
     if (fd < 0) {
       int e = errno;
@@ -646,6 +651,7 @@ VALUE Backend_accept_loop(VALUE self, VALUE server_socket, VALUE socket_class) {
   watcher.fiber = Qnil;
 
   while (1) {
+    backend->base.op_count++;
     fd = accept(fptr->fd, &addr, &len);
     if (fd < 0) {
       int e = errno;
@@ -705,6 +711,7 @@ VALUE Backend_connect(VALUE self, VALUE sock, VALUE host, VALUE port) {
   addr.sin_addr.s_addr = inet_addr(host_buf);
   addr.sin_port = htons(NUM2INT(port));
 
+  backend->base.op_count++;
   int result = connect(fptr->fd, (struct sockaddr *)&addr, sizeof(addr));
   if (result < 0) {
     int e = errno;
@@ -745,6 +752,7 @@ VALUE Backend_send(VALUE self, VALUE io, VALUE str, VALUE flags) {
   watcher.fiber = Qnil;
 
   while (left > 0) {
+    backend->base.op_count++;
     ssize_t n = send(fptr->fd, buf, left, flags_int);
     if (n < 0) {
       int e = errno;
@@ -852,6 +860,7 @@ VALUE Backend_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
 
   watcher.ctx.fiber = Qnil;
   while (1) {
+    backend->base.op_count++;
     len = splice(src_fptr->fd, 0, dest_fptr->fd, 0, NUM2INT(maxlen), 0);
     if (len < 0) {
       int e = errno;
@@ -903,6 +912,7 @@ VALUE Backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
 
   watcher.ctx.fiber = Qnil;
   while (1) {
+    backend->base.op_count++;
     len = splice(src_fptr->fd, 0, dest_fptr->fd, 0, NUM2INT(maxlen), 0);
     if (len < 0) {
       int e = errno;
@@ -962,6 +972,7 @@ VALUE Backend_fake_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
   watcher.fiber = Qnil;
   
   while (1) {
+    backend->base.op_count++;
     ssize_t n = read(src_fptr->fd, buf, len);
     if (n < 0) {
       int e = errno;
@@ -977,6 +988,7 @@ VALUE Backend_fake_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
   }
 
   while (left > 0) {
+    backend->base.op_count++;
     ssize_t n = write(dest_fptr->fd, buf, left);
     if (n < 0) {
       int e = errno;
@@ -1037,6 +1049,7 @@ VALUE Backend_fake_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen
   while (1) { 
     char *ptr = buf;
     while (1) {
+      backend->base.op_count++;
       ssize_t n = read(src_fptr->fd, ptr, len);
       if (n < 0) {
         int e = errno;
@@ -1054,6 +1067,7 @@ VALUE Backend_fake_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen
     }
 
     while (left > 0) {
+      backend->base.op_count++;
       ssize_t n = write(dest_fptr->fd, ptr, left);
       if (n < 0) {
         int e = errno;
@@ -1093,6 +1107,7 @@ VALUE Backend_wait_io(VALUE self, VALUE io, VALUE write) {
   GetBackend(self, backend);
   GetOpenFile(io, fptr);
 
+  backend->base.op_count++;
   return libev_wait_fd(backend, fptr->fd, events, 1);
 }
 
@@ -1116,6 +1131,7 @@ VALUE Backend_sleep(VALUE self, VALUE duration) {
   watcher.fiber = rb_fiber_current();
   ev_timer_init(&watcher.timer, Backend_timer_callback, NUM2DBL(duration), 0.);
   ev_timer_start(backend->ev_loop, &watcher.timer);
+  backend->base.op_count++;
 
   switchpoint_result = backend_await((struct Backend_base *)backend);
 
@@ -1145,6 +1161,7 @@ noreturn VALUE Backend_timer_loop(VALUE self, VALUE interval) {
     VALUE switchpoint_result = Qnil;    
     ev_timer_init(&watcher.timer, Backend_timer_callback, sleep_duration, 0.);
     ev_timer_start(backend->ev_loop, &watcher.timer);
+    backend->base.op_count++;
     switchpoint_result = backend_await((struct Backend_base *)backend);
     ev_timer_stop(backend->ev_loop, &watcher.timer);
     RAISE_IF_EXCEPTION(switchpoint_result);
@@ -1196,6 +1213,7 @@ VALUE Backend_timeout(int argc,VALUE *argv, VALUE self) {
   watcher.resume_value = timeout;
   ev_timer_init(&watcher.timer, Backend_timeout_callback, NUM2DBL(duration), 0.);
   ev_timer_start(backend->ev_loop, &watcher.timer);
+  backend->base.op_count++;
 
   struct Backend_timeout_ctx timeout_ctx = {backend, &watcher};
   result = rb_ensure(Backend_timeout_ensure_safe, Qnil, Backend_timeout_ensure, (VALUE)&timeout_ctx);
@@ -1218,6 +1236,7 @@ VALUE Backend_waitpid(VALUE self, VALUE pid) {
   if (fd >= 0) {
     Backend_t *backend;
     GetBackend(self, backend);
+    backend->base.op_count++;
 
     VALUE resume_value = libev_wait_fd(backend, fd, EV_READ, 0);
     close(fd);
@@ -1261,6 +1280,7 @@ VALUE Backend_waitpid(VALUE self, VALUE pid) {
   watcher.fiber = rb_fiber_current();
   ev_child_init(&watcher.child, Backend_child_callback, NUM2INT(pid), 0);
   ev_child_start(backend->ev_loop, &watcher.child);
+  backend->base.op_count++;
 
   switchpoint_result = backend_await((struct Backend_base *)backend);
 
@@ -1283,6 +1303,7 @@ VALUE Backend_wait_event(VALUE self, VALUE raise) {
 
   ev_async_init(&async, Backend_async_callback);
   ev_async_start(backend->ev_loop, &async);
+  backend->base.op_count++;
 
   switchpoint_result = backend_await((struct Backend_base *)backend);
 
@@ -1346,6 +1367,7 @@ inline int splice_chunks_write(Backend_t *backend, int fd, VALUE str, struct lib
   int len = RSTRING_LEN(str);
   int left = len;
   while (left > 0) {
+    backend->base.op_count++;
     ssize_t n = write(fd, buf, left);
     if (n < 0) {
       int err = errno;
@@ -1406,6 +1428,7 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
     int chunk_len;
     // splice to pipe
     while (1) {
+      backend->base.op_count++;
       chunk_len = splice(src_fptr->fd, 0, pipefd[1], 0, maxlen, 0);
       if (chunk_len < 0) {
         err = errno;
@@ -1431,6 +1454,7 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
 
     int left = chunk_len;
     while (1) {
+      backend->base.op_count++;
       int n = splice(pipefd[0], 0, dest_fptr->fd, 0, left, 0);
       if (n < 0) {
         err = errno;
@@ -1503,6 +1527,7 @@ void Init_Backend() {
   rb_define_method(cBackend, "post_fork", Backend_post_fork, 0);
   rb_define_method(cBackend, "trace", Backend_trace, -1);
   rb_define_method(cBackend, "trace_proc=", Backend_trace_proc_set, 1);
+  rb_define_method(cBackend, "stats", Backend_stats, 0);
 
   rb_define_method(cBackend, "poll", Backend_poll, 1);
   rb_define_method(cBackend, "break", Backend_wakeup, 0);
@@ -1545,6 +1570,8 @@ void Init_Backend() {
   SYM_send = ID2SYM(rb_intern("send"));
   SYM_splice = ID2SYM(rb_intern("splice"));
   SYM_write = ID2SYM(rb_intern("write"));
+
+  backend_setup_stats_symbols();
 }
 
 #endif // POLYPHONY_BACKEND_LIBEV

data/ext/polyphony/polyphony.h

@@ -114,7 +114,6 @@ VALUE Backend_wakeup(VALUE self);
 VALUE Backend_run_idle_tasks(VALUE self);
 VALUE Backend_switch_fiber(VALUE self);
 void Backend_schedule_fiber(VALUE thread, VALUE self, VALUE fiber, VALUE value, int prioritize);
-struct backend_stats Backend_stats(VALUE self);
 void Backend_unschedule_fiber(VALUE self, VALUE fiber);
 
 VALUE Thread_schedule_fiber(VALUE thread, VALUE fiber, VALUE value);

data/ext/polyphony/runqueue.c

@@ -4,7 +4,6 @@
 inline void runqueue_initialize(runqueue_t *runqueue) {
   runqueue_ring_buffer_init(&runqueue->entries);
   runqueue->high_watermark = 0;
-  runqueue->switch_count = 0;
 }
 
 inline void runqueue_finalize(runqueue_t *runqueue) {
@@ -30,12 +29,7 @@ inline void runqueue_unshift(runqueue_t *runqueue, VALUE fiber, VALUE value, int
 }
 
 inline runqueue_entry runqueue_shift(runqueue_t *runqueue) {
-  runqueue_entry entry = runqueue_ring_buffer_shift(&runqueue->entries);
-  if (entry.fiber == Qnil)
-    runqueue->high_watermark = 0;
-  else
-    runqueue->switch_count += 1;
-  return entry;
+  return runqueue_ring_buffer_shift(&runqueue->entries);
 }
 
 inline void runqueue_delete(runqueue_t *runqueue, VALUE fiber) {
@@ -50,19 +44,20 @@ inline void runqueue_clear(runqueue_t *runqueue) {
   runqueue_ring_buffer_clear(&runqueue->entries);
 }
 
+inline long runqueue_size(runqueue_t *runqueue) {
+  return runqueue->entries.size;
+}
+
 inline long runqueue_len(runqueue_t *runqueue) {
   return runqueue->entries.count;
 }
 
-inline int runqueue_empty_p(runqueue_t *runqueue) {
-  return (runqueue->entries.count == 0);
+inline long runqueue_max_len(runqueue_t *runqueue) {
+  unsigned int max_len = runqueue->high_watermark;
+  runqueue->high_watermark = 0;
+  return max_len;
 }
 
-static const unsigned int ANTI_STARVE_SWITCH_COUNT_THRESHOLD = 64;
-
-inline int runqueue_should_poll_nonblocking(runqueue_t *runqueue) {
-  if (runqueue->switch_count < ANTI_STARVE_SWITCH_COUNT_THRESHOLD) return 0;
-
-  runqueue->switch_count = 0;
-  return 1;
+inline int runqueue_empty_p(runqueue_t *runqueue) {
+  return (runqueue->entries.count == 0);
 }

data/ext/polyphony/runqueue.h

@@ -7,7 +7,6 @@
 typedef struct runqueue {
   runqueue_ring_buffer entries;
   unsigned int high_watermark;
-  unsigned int switch_count;
 } runqueue_t;
 
 void runqueue_initialize(runqueue_t *runqueue);
@@ -20,8 +19,9 @@ runqueue_entry runqueue_shift(runqueue_t *runqueue);
 void runqueue_delete(runqueue_t *runqueue, VALUE fiber);
 int runqueue_index_of(runqueue_t *runqueue, VALUE fiber);
 void runqueue_clear(runqueue_t *runqueue);
+long runqueue_size(runqueue_t *runqueue);
 long runqueue_len(runqueue_t *runqueue);
+long runqueue_max_len(runqueue_t *runqueue);
 int runqueue_empty_p(runqueue_t *runqueue);
-int runqueue_should_poll_nonblocking(runqueue_t *runqueue);
 
 #endif /* RUNQUEUE_H */

data/ext/polyphony/thread.c

@@ -13,18 +13,6 @@ static VALUE Thread_setup_fiber_scheduling(VALUE self) {
   return self;
 }
 
-static VALUE SYM_scheduled_fibers;
-static VALUE SYM_pending_watchers;
-
-static VALUE Thread_fiber_scheduling_stats(VALUE self) {
-  struct backend_stats backend_stats = Backend_stats(rb_ivar_get(self, ID_ivar_backend));
-
-  VALUE stats = rb_hash_new();
-  rb_hash_aset(stats, SYM_scheduled_fibers, INT2NUM(backend_stats.scheduled_fibers));
-  rb_hash_aset(stats, SYM_pending_watchers, INT2NUM(backend_stats.pending_ops));
-  return stats;
-}
-
 inline void schedule_fiber(VALUE self, VALUE fiber, VALUE value, int prioritize) {
   Backend_schedule_fiber(self, rb_ivar_get(self, ID_ivar_backend), fiber, value, prioritize);
 }
@@ -72,7 +60,6 @@ VALUE Thread_class_backend(VALUE _self) {
 
 void Init_Thread() {
   rb_define_method(rb_cThread, "setup_fiber_scheduling", Thread_setup_fiber_scheduling, 0);
-  rb_define_method(rb_cThread, "fiber_scheduling_stats", Thread_fiber_scheduling_stats, 0);
   rb_define_method(rb_cThread, "schedule_and_wakeup", Thread_fiber_schedule_and_wakeup, 2);
 
   rb_define_method(rb_cThread, "schedule_fiber", Thread_schedule_fiber, 2);
@@ -91,9 +78,4 @@ void Init_Thread() {
   ID_ivar_main_fiber                    = rb_intern("@main_fiber");
   ID_ivar_terminated                    = rb_intern("@terminated");
   ID_stop                               = rb_intern("stop");
-
-  SYM_scheduled_fibers = ID2SYM(rb_intern("scheduled_fibers"));
-  SYM_pending_watchers = ID2SYM(rb_intern("pending_watchers"));
-  rb_global_variable(&SYM_scheduled_fibers);
-  rb_global_variable(&SYM_pending_watchers);
 }

data/lib/polyphony/extensions/io.rb

@@ -108,7 +108,11 @@ class ::IO
   end
 
   alias_method :orig_read, :read
-  def read(len = nil)
+  def read(len = nil, buf = nil, buf_pos = 0)
+    if buf
+      return Polyphony.backend_read(self, buf, len, true, buf_pos)
+    end
+    
     @read_buffer ||= +''
     result = Polyphony.backend_read(self, @read_buffer, len, true, -1)
     return nil unless result
@@ -119,9 +123,9 @@ class ::IO
   end
 
   alias_method :orig_readpartial, :read
-  def readpartial(len, str = +'', buffer_pos = 0)
+  def readpartial(len, str = +'', buffer_pos = 0, raise_on_eof = true)
     result = Polyphony.backend_read(self, str, len, false, buffer_pos)
-    raise EOFError unless result
+    raise EOFError if !result && raise_on_eof
 
     result
   end

data/lib/polyphony/extensions/openssl.rb

@@ -64,7 +64,21 @@ class ::OpenSSL::SSL::SSLSocket
     # @sync = osync
   end
 
-  def readpartial(maxlen, buf = +'', buffer_pos = 0)
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return readpartial(maxlen, buf, buf_pos) if buf
+
+    buf = +''
+    return readpartial(maxlen, buf) if maxlen
+
+    while true
+      readpartial(4096, buf, -1)
+    end
+  rescue EOFError
+    buf
+  end
+
+  def readpartial(maxlen, buf = +'', buffer_pos = 0, raise_on_eof = true)
     if buffer_pos != 0
       if (result = sysread(maxlen, +''))
         if buffer_pos == -1
@@ -76,7 +90,9 @@ class ::OpenSSL::SSL::SSLSocket
     else
       result = sysread(maxlen, buf)
     end
-    result || (raise EOFError)
+
+    raise EOFError if !result && raise_on_eof
+    result
   end
 
   def read_loop(maxlen = 8192)

data/lib/polyphony/extensions/socket.rb

@@ -22,6 +22,23 @@ class ::Socket
     Polyphony.backend_connect(self, addr.ip_address, addr.ip_port)
   end
 
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
+    return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
+    
+    buf = +''
+    len = buf.bytesize
+    while true
+      Polyphony.backend_recv(self, buf, maxlen || 4096, -1)
+      new_len = buf.bytesize
+      break if new_len == len
+
+      len = new_len
+    end
+    buf
+  end
+
   def recv(maxlen, flags = 0, outbuf = nil)
     Polyphony.backend_recv(self, outbuf || +'', maxlen, 0)
   end
@@ -60,8 +77,9 @@ class ::Socket
   #   Polyphony.backend_send(self, mesg, 0)
   # end
 
-  def readpartial(maxlen, str = +'', buffer_pos = 0)
-    Polyphony.backend_recv(self, str, maxlen, buffer_pos)
+  def readpartial(maxlen, str = +'', buffer_pos = 0, raise_on_eof = true)
+    result = Polyphony.backend_recv(self, str, maxlen, buffer_pos)
+    raise EOFError if !result && raise_on_eof
   end
 
   ZERO_LINGER = [0, 0].pack('ii').freeze
@@ -140,6 +158,23 @@ class ::TCPSocket
     setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
   end
 
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
+    return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
+    
+    buf = +''
+    len = buf.bytesize
+    while true
+      Polyphony.backend_recv(self, buf, maxlen || 4096, -1)
+      new_len = buf.bytesize
+      break if new_len == len
+
+      len = new_len
+    end
+    buf
+  end
+
   def recv(maxlen, flags = 0, outbuf = nil)
     Polyphony.backend_recv(self, outbuf || +'', maxlen, 0)
   end
@@ -165,11 +200,10 @@ class ::TCPSocket
   #   Polyphony.backend_send(self, mesg, 0)
   # end
 
-  def readpartial(maxlen, str = +'', buffer_pos = 0)
+  def readpartial(maxlen, str = +'', buffer_pos = 0, raise_on_eof)
     result = Polyphony.backend_recv(self, str, maxlen, buffer_pos)
-    raise EOFError unless result
-
-    str
+    raise EOFError if !result && raise_on_eof
+    result
   end
 
   def read_nonblock(len, str = nil, exception: true)
@@ -217,6 +251,23 @@ class ::UNIXServer
 end
 
 class ::UNIXSocket
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
+    return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
+    
+    buf = +''
+    len = buf.bytesize
+    while true
+      Polyphony.backend_recv(self, buf, maxlen || 4096, -1)
+      new_len = buf.bytesize
+      break if new_len == len
+
+      len = new_len
+    end
+    buf
+  end
+
   def recv(maxlen, flags = 0, outbuf = nil)
     Polyphony.backend_recv(self, outbuf || +'', maxlen, 0)
   end
@@ -242,11 +293,10 @@ class ::UNIXSocket
     Polyphony.backend_send(self, mesg, 0)
   end
 
-  def readpartial(maxlen, str = +'', buffer_pos = 0)
+  def readpartial(maxlen, str = +'', buffer_pos = 0, raise_on_eof)
     result = Polyphony.backend_recv(self, str, maxlen, buffer_pos)
-    raise EOFError unless result
-
-    str
+    raise EOFError if !result && raise_on_eof
+    result
   end
 
   def read_nonblock(len, str = nil, exception: true)

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.60'
+  VERSION = '0.64'
 end

data/test/test_backend.rb

@@ -26,7 +26,7 @@ class BackendTest < MiniTest::Test
       @backend.sleep 0.01
       count += 1
     }.await
-    assert_in_delta 0.03, Time.now - t0, 0.01
+    assert_in_range 0.02..0.04, Time.now - t0
     assert_equal 3, count
   end
 
@@ -98,6 +98,32 @@ class BackendTest < MiniTest::Test
     assert_equal return_value, buf
   end
 
+  def test_read_concat_big
+    i, o = IO.pipe
+
+    body = " " * 4000
+
+    data = "post /?q=time&blah=blah HTTP/1\r\nHost: dev.realiteq.net\r\n\r\n" +
+           "get /?q=time HTTP/1.1\r\nContent-Length: #{body.bytesize}\r\n\r\n#{body}" +
+           "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
+
+    o << data
+    o.close
+    
+    buf = +''
+
+    @backend.read(i, buf, 4096, false, -1)
+    assert_equal 4096, buf.bytesize
+    
+    @backend.read(i, buf, 1, false, -1)
+    assert_equal 4097, buf.bytesize
+
+    @backend.read(i, buf, 4096, false, -1)
+
+    assert_equal data.bytesize, buf.bytesize
+    assert_equal data, buf
+  end
+
   def test_waitpid
     pid = fork do
       @backend.post_fork

data/test/test_io.rb

@@ -73,6 +73,26 @@ class IOTest < MiniTest::Test
     assert_equal [:wait_readable, 'foo'], results
   end
 
+  def test_read
+    i, o = IO.pipe
+
+    o << 'hi'
+    assert_equal 'hi', i.read(2)
+
+    o << 'foobarbaz'
+    assert_equal 'foo', i.read(3)
+    assert_equal 'bar', i.read(3)
+
+    buf = +'abc'
+    assert_equal 'baz', i.read(3, buf)
+    assert_equal 'baz', buf
+
+    buf = +'def'
+    o << 'foobar'
+    assert_equal 'deffoobar', i.read(6, buf, -1)
+    assert_equal 'deffoobar', buf
+  end
+
   def test_readpartial
     i, o = IO.pipe
 

data/test/test_socket.rb

@@ -12,7 +12,6 @@ class SocketTest < MiniTest::Test
   def test_tcp
     port = rand(1234..5678)
     server = TCPServer.new('127.0.0.1', port)
-
     server_fiber = spin do
       while (socket = server.accept)
         spin do
@@ -34,6 +33,45 @@ class SocketTest < MiniTest::Test
     server&.close
   end
 
+  def test_read
+    port = rand(1234..5678)
+    server = TCPServer.new('127.0.0.1', port)
+    server_fiber = spin do
+      while (socket = server.accept)
+        spin do
+          while (data = socket.read(8192))
+            socket << data
+          end
+        end
+      end
+    end
+
+    snooze
+    client = TCPSocket.new('127.0.0.1', port)
+
+    client << 'hi'
+    assert_equal 'hi', client.read(2)
+
+    client << 'foobarbaz'
+    assert_equal 'foo', client.read(3)
+    assert_equal 'bar', client.read(3)
+
+    buf = +'abc'
+    assert_equal 'baz', client.read(3, buf)
+    assert_equal 'baz', buf
+
+    buf = +'def'
+    client << 'foobar'
+    assert_equal 'deffoobar', client.read(6, buf, -1)
+    assert_equal 'deffoobar', buf
+
+    client.close
+  ensure
+    server_fiber&.stop
+    server_fiber&.await
+    server&.close
+  end
+
   # sending multiple strings at once
   def test_sendv
     port = rand(1234..5678)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: '0.60'
+  version: '0.64'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-15 00:00:00.000000000 Z
+date: 2021-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -419,7 +419,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Fine grained concurrency for Ruby