polyphony 0.62 → 0.66

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2cea172818871812ebbafcbee0a4b98d130a351f904d7732a02400ef363eac62
-  data.tar.gz: f4ef85515436a463d3732f04bbef457426f267fac01f5c5c28619e42334cd745
+  metadata.gz: b8e2aa3d011286dda93222164cbf66f1ec4c18b9c212ee7a70dbebd4bc523943
+  data.tar.gz: 99b5fb95c682dfad8df33f07d98bfab7f2109b9d4ec0e24720869c2d331f23fb
 SHA512:
-  metadata.gz: 489b813f1bb2d7d97f87a60024b0665761571c4227e4fab6c733dcdb62b6ee98760473202c9154eb39a4b544663f759e4da6e70603d483e9f2ec2a64363cd4e1
-  data.tar.gz: 05b232d7d67120e0983e60855c928beac6e91271fafc87586da7cc7543128fe326be04d126154ac7e0b0c0b249bf41ca90ab902f79f4a5fbf00cad498061f7e7
+  metadata.gz: 27c65b6e340d1aad41d38b4b322b98d282f96fe6d136c897ab8de14651b2da3cfc392650600d63983f76b5fb1dc70e4e49116efb289a790a509bec63caa9b9e4
+  data.tar.gz: 94e62ac7fbcb6d085af7475d987eca45be191aaf7654c3bacb07c4f7793d450281d3da1b08b12636a49398e97c1043c3a2cd628018925bdc082e02a0859743bc

data/.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-latest, ubuntu-18.04, macos-10.15]
         ruby: [2.6, 2.7, 3.0]
 
     name: >-

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## 0.66 2021-08-01
+
+- Fix all splicing APIs on non-linux OSes (#63)
+- Add GC marking of buffers when cancelling read/write ops in io_uring backend 
+
+## 0.65 2021-07-29
+
+- Add `#__polyphony_read_method__` method for read method detection
+
+## 0.64 2021-07-26
+
+- Add optional raise_on_eof argument to `#readpartial`
+
+## 0.63 2021-07-26
+
+- Add support for specifying buf and buf_pos in `IO#read`
+- Fix `Socket#read` to work and conform to `IO#read` interface
+
 ## 0.62 2021-07-21
 
 - Add `runqueue_size` to backend stats

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.62)
+    polyphony (0.66)
 
 GEM
   remote: https://rubygems.org/

data/Rakefile

@@ -9,8 +9,8 @@ Rake::ExtensionTask.new("polyphony_ext") do |ext|
 end
 
 task :recompile => [:clean, :compile]
-
 task :default => [:compile, :test]
+
 task :test do
   exec 'ruby test/run.rb'
 end

data/ext/polyphony/backend_io_uring.c

@@ -25,6 +25,8 @@ VALUE SYM_send;
 VALUE SYM_splice;
 VALUE SYM_write;
 
+VALUE eArgumentError;
+
 #ifdef POLYPHONY_UNSET_NONBLOCK
 #define io_unset_nonblock(fptr, io) io_verify_blocking_mode(fptr, io, Qtrue)
 #else
@@ -45,6 +47,7 @@ typedef struct Backend_t {
 static void Backend_mark(void *ptr) {
   Backend_t *backend = ptr;
   backend_base_mark(&backend->base);
+  context_store_mark_taken_buffers(&backend->store);
 }
 
 static void Backend_free(void *ptr) {
@@ -349,8 +352,11 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof,
 
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -410,8 +416,11 @@ VALUE Backend_read_loop(VALUE self, VALUE io, VALUE maxlen) {
     
     ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -457,8 +466,11 @@ VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
     
     ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -500,8 +512,11 @@ VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
     
     if (result < 0)
@@ -549,12 +564,10 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    if (TEST_EXCEPTION(resume_value)) {
-      free(iov);
-      RAISE_EXCEPTION(resume_value);
-    }
     if (!completed) {
       free(iov);
+      context_attach_buffers(ctx, argc, argv);
+      RAISE_IF_EXCEPTION(resume_value);
       return resume_value;
     }
     RB_GC_GUARD(resume_value);
@@ -588,8 +601,7 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
 
 VALUE Backend_write_m(int argc, VALUE *argv, VALUE self) {
   if (argc < 2)
-    // TODO: raise ArgumentError
-    rb_raise(rb_eRuntimeError, "(wrong number of arguments (expected 2 or more))");
+    rb_raise(eArgumentError, "(wrong number of arguments (expected 2 or more))");
 
   return (argc == 2) ?
     Backend_write(self, argv[0], argv[1]) :
@@ -628,8 +640,11 @@ VALUE Backend_recv(VALUE self, VALUE io, VALUE str, VALUE length, VALUE pos) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -675,8 +690,11 @@ VALUE Backend_recv_loop(VALUE self, VALUE io, VALUE maxlen) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -721,8 +739,11 @@ VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method)
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -764,8 +785,11 @@ VALUE Backend_send(VALUE self, VALUE io, VALUE str, VALUE flags) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -1165,6 +1189,24 @@ struct io_uring_sqe *Backend_chain_prepare_splice(Backend_t *backend, VALUE src,
   return sqe;
 }
 
+void Backend_chain_ctx_attach_buffers(op_context_t *ctx, int argc, VALUE *argv) {
+  int count = 0;
+  if (argc > 1) ctx->buffers = malloc(sizeof(VALUE) * (argc - 1));
+
+  for (int i = 0; i < argc; i++) {
+    VALUE op = argv[i];
+    VALUE op_type = RARRAY_AREF(op, 0);
+
+    if (op_type == SYM_write || op_type == SYM_send) {
+      if (!count) ctx->buffer0 = RARRAY_AREF(op, 2);
+      else        ctx->buffers[count - 1] = RARRAY_AREF(op, 2);
+      count++;
+    }
+  }
+  ctx->buffer_count = count;
+}
+
+
 VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
   VALUE resume_value = Qnil;
   unsigned int sqe_count = 0;
@@ -1218,6 +1260,8 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
   int result = ctx->result;
   int completed = context_store_release(&backend->store, ctx);
   if (!completed) {
+    Backend_chain_ctx_attach_buffers(ctx, argc, argv);
+    
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
     struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
@@ -1409,6 +1453,7 @@ syscallerror:
   if (pipefd[1] != -1) close(pipefd[1]);
   rb_syserr_fail(err, strerror(err));
 error:
+  context_attach_buffers_v(ctx, 4, prefix, postfix, chunk_prefix, chunk_postfix);
   if (pipefd[0] != -1) close(pipefd[0]);
   if (pipefd[1] != -1) close(pipefd[1]);
   return RAISE_EXCEPTION(switchpoint_result);
@@ -1469,7 +1514,6 @@ void Init_Backend() {
   rb_define_method(cBackend, "waitpid", Backend_waitpid, 1);
   rb_define_method(cBackend, "write", Backend_write_m, -1);
 
-
   #ifdef POLYPHONY_UNSET_NONBLOCK
   ID_ivar_is_nonblocking = rb_intern("@is_nonblocking");
   #endif
@@ -1480,6 +1524,8 @@ void Init_Backend() {
   SYM_write = ID2SYM(rb_intern("write"));
 
   backend_setup_stats_symbols();
+
+  eArgumentError = rb_const_get(rb_cObject, rb_intern("ArgumentError"));
 }
 
 #endif // POLYPHONY_BACKEND_LIBURING

data/ext/polyphony/backend_io_uring_context.c

@@ -50,6 +50,7 @@ inline op_context_t *context_store_acquire(op_context_store_t *store, enum op_ty
   ctx->resume_value = Qnil;
   ctx->ref_count = 2;
   ctx->result = 0;
+  ctx->buffer_count = 0;
 
   store->taken_count++;
 
@@ -67,6 +68,8 @@ inline int context_store_release(op_context_store_t *store, op_context_t *ctx) {
   ctx->ref_count--;
   if (ctx->ref_count) return 0;
 
+  if (ctx->buffer_count > 1) free(ctx->buffers);
+
   store->taken_count--;
   store->available_count++;
 
@@ -93,3 +96,42 @@ void context_store_free(op_context_store_t *store) {
     store->taken = next;
   }
 }
+
+inline void context_store_mark_taken_buffers(op_context_store_t *store) {
+  op_context_t *ctx = store->taken;
+  while (ctx) {
+    for (unsigned int i = 0; i < ctx->buffer_count; i++)
+      rb_gc_mark(i == 0 ? ctx->buffer0 : ctx->buffers[i - 1]);
+    ctx = ctx->next;
+  }
+}
+
+inline void context_attach_buffers(op_context_t *ctx, unsigned int count, VALUE *buffers) {
+  // attaching buffers to the context is done in order to ensure that any GC
+  // pass done before the context is released will mark those buffers, even if
+  // the fiber has already been resumed and the buffers are not in use anymore.
+  // This is done in order to prevent a possible race condition where on the
+  // kernel side the buffers are still in use, but in userspace they have
+  // effectively been freed after a GC pass.
+  ctx->buffer_count = count;
+  if (count > 1)
+    ctx->buffers = malloc(sizeof(VALUE) * (count - 1));
+  for (unsigned int i = 0; i < count; i++)
+    if (!i) ctx->buffer0 = buffers[0];
+    else    ctx->buffers[i - 1] = buffers[i];
+}
+
+inline void context_attach_buffers_v(op_context_t *ctx, unsigned int count, ...) {
+  va_list values;
+
+  va_start(values, count);
+
+  ctx->buffer_count = count;
+  if (count > 1)
+    ctx->buffers = malloc(sizeof(VALUE) * (count - 1));
+  for (unsigned int i = 0; i < count; i++)
+    if (!i) ctx->buffer0 = va_arg(values, VALUE);
+    else    ctx->buffers[i - 1] = va_arg(values, VALUE);
+
+  va_end(values);
+}

data/ext/polyphony/backend_io_uring_context.h

@@ -27,6 +27,9 @@ typedef struct op_context {
   int               result;
   VALUE             fiber;
   VALUE             resume_value;
+  unsigned int      buffer_count;
+  VALUE             buffer0;
+  VALUE             *buffers;
 } op_context_t;
 
 typedef struct op_context_store {
@@ -43,14 +46,8 @@ void context_store_initialize(op_context_store_t *store);
 op_context_t *context_store_acquire(op_context_store_t *store, enum op_type type);
 int context_store_release(op_context_store_t *store, op_context_t *ctx);
 void context_store_free(op_context_store_t *store);
-
-inline unsigned int OP_CONTEXT_RELEASE(op_context_store_t *store, op_context_t *ctx) {
-  int completed = !ctx->ref_count;
-  if (ctx->ref_count)
-    ctx->ref_count -= 1;
-  else
-    context_store_release(store, ctx);
-  return completed;
-}
+void context_store_mark_taken_buffers(op_context_store_t *store);
+void context_attach_buffers(op_context_t *ctx, unsigned int count, VALUE *buffers);
+void context_attach_buffers_v(op_context_t *ctx, unsigned int count, ...);
 
 #endif /* BACKEND_IO_URING_CONTEXT_H */

data/ext/polyphony/backend_libev.c

@@ -941,9 +941,8 @@ VALUE Backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
 error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
-#endif
-
-VALUE Backend_fake_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
+#else
+VALUE Backend_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
   Backend_t *backend;
   struct libev_io watcher;
   VALUE switchpoint_result = Qnil;
@@ -1018,7 +1017,7 @@ error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
 
-VALUE Backend_fake_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
+VALUE Backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
   Backend_t *backend;
   struct libev_io watcher;
   VALUE switchpoint_result = Qnil;
@@ -1097,6 +1096,7 @@ done:
 error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
+#endif
 
 VALUE Backend_wait_io(VALUE self, VALUE io, VALUE write) {
   Backend_t *backend;
@@ -1384,6 +1384,64 @@ inline int splice_chunks_write(Backend_t *backend, int fd, VALUE str, struct lib
   return 0;
 }
 
+static inline int splice_chunks_splice(Backend_t *backend, int src_fd, int dest_fd, int maxlen, 
+  struct libev_rw_io *watcher, VALUE *result, int *chunk_len) {
+#ifdef POLYPHONY_LINUX
+  backend->base.op_count++;
+  while (1) {
+    *chunk_len = splice(src_fd, 0, dest_fd, 0, maxlen, 0);
+    if (*chunk_len >= 0) return 0;
+    
+    int err = errno;
+    if (err != EWOULDBLOCK && err != EAGAIN) return err;
+
+    *result = libev_wait_rw_fd_with_watcher(backend, src_fd, dest_fd, watcher);
+    if (TEST_EXCEPTION(*result)) return -1;
+  }
+#else
+  char *buf = malloc(maxlen);
+  int ret;
+
+  backend->base.op_count++;
+  while (1) {
+    *chunk_len = read(src_fd, buf, maxlen);
+    if (*chunk_len >= 0) break;
+
+    ret = errno;
+    if ((ret != EWOULDBLOCK && ret != EAGAIN)) goto done;
+
+    *result = libev_wait_rw_fd_with_watcher(backend, src_fd, -1, watcher);
+    if (TEST_EXCEPTION(*result)) goto exception;
+  }
+
+  backend->base.op_count++;
+  char *ptr = buf;
+  int left = *chunk_len;
+  while (left > 0) {
+    ssize_t n = write(dest_fd, ptr, left);
+    if (n < 0) {
+      ret = errno;
+      if ((ret != EWOULDBLOCK && ret != EAGAIN)) goto done;
+
+      *result = libev_wait_rw_fd_with_watcher(backend, -1, dest_fd, watcher);
+
+      if (TEST_EXCEPTION(*result)) goto exception;
+    }
+    else {
+      ptr += n;
+      left -= n;
+    }
+  }
+  ret = 0;
+  goto done;
+exception:
+  ret = -1;
+done:
+  free(buf);
+  return ret;
+#endif
+}
+
 VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VALUE postfix, VALUE chunk_prefix, VALUE chunk_postfix, VALUE chunk_size) {
   Backend_t *backend;
   GetBackend(self, backend);
@@ -1421,26 +1479,13 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
   fcntl(pipefd[1], F_SETFL, O_NONBLOCK);
 
   if (prefix != Qnil) {
-    int err = splice_chunks_write(backend, dest_fptr->fd, prefix, &watcher, &result);
+    err = splice_chunks_write(backend, dest_fptr->fd, prefix, &watcher, &result);
     if (err == -1) goto error; else if (err) goto syscallerror;
   }
   while (1) {
-    int chunk_len;
-    // splice to pipe
-    while (1) {
-      backend->base.op_count++;
-      chunk_len = splice(src_fptr->fd, 0, pipefd[1], 0, maxlen, 0);
-      if (chunk_len < 0) {
-        err = errno;
-        if (err != EWOULDBLOCK && err != EAGAIN) goto syscallerror;
-
-        result = libev_wait_rw_fd_with_watcher(backend, src_fptr->fd, pipefd[1], &watcher);
-        if (TEST_EXCEPTION(result)) goto error;
-      }
-      else {
-        break;
-      }
-    }
+    int chunk_len = 0;
+    err = splice_chunks_splice(backend, src_fptr->fd, pipefd[1], maxlen, &watcher, &result, &chunk_len);
+    if (err == -1) goto error; else if (err) goto syscallerror;
     if (chunk_len == 0) break;
     
     total += chunk_len;
@@ -1453,20 +1498,12 @@ VALUE Backend_splice_chunks(VALUE self, VALUE src, VALUE dest, VALUE prefix, VAL
     }
 
     int left = chunk_len;
-    while (1) {
-      backend->base.op_count++;
-      int n = splice(pipefd[0], 0, dest_fptr->fd, 0, left, 0);
-      if (n < 0) {
-        err = errno;
-        if (err != EWOULDBLOCK && err != EAGAIN) goto syscallerror;
+    while (left > 0) {
+      int len;
+      err = splice_chunks_splice(backend, pipefd[0], dest_fptr->fd, left, &watcher, &result, &len);
+      if (err == -1) goto error; else if (err) goto syscallerror;
 
-        result = libev_wait_rw_fd_with_watcher(backend, pipefd[0], dest_fptr->fd, &watcher);
-        if (TEST_EXCEPTION(result)) goto error;
-      }
-      else {
-        left -= n;
-        if (left == 0) break;
-      }
+      left -= len;
     }
 
     if (chunk_postfix != Qnil) {
@@ -1550,13 +1587,8 @@ void Init_Backend() {
   rb_define_method(cBackend, "sendv", Backend_sendv, 3);
   rb_define_method(cBackend, "sleep", Backend_sleep, 1);
 
-  #ifdef POLYPHONY_LINUX
   rb_define_method(cBackend, "splice", Backend_splice, 3);
   rb_define_method(cBackend, "splice_to_eof", Backend_splice_to_eof, 3);
-  #else
-  rb_define_method(cBackend, "splice", Backend_fake_splice, 3);
-  rb_define_method(cBackend, "splice_to_eof", Backend_fake_splice_to_eof, 3);
-  #endif
 
   rb_define_method(cBackend, "timeout", Backend_timeout, -1);
   rb_define_method(cBackend, "timer_loop", Backend_timer_loop, 1);

data/ext/polyphony/queue.c

@@ -78,7 +78,7 @@ inline void queue_schedule_blocked_fibers_to_capacity(Queue_t *queue) {
   }
 }
 
-inline void capped_queue_block_push(Queue_t *queue) {
+static inline void capped_queue_block_push(Queue_t *queue) {
   VALUE fiber = rb_fiber_current();
   VALUE backend = rb_ivar_get(rb_thread_current(), ID_ivar_backend);
   VALUE switchpoint_result;

data/ext/polyphony/runqueue.c

@@ -44,15 +44,15 @@ inline void runqueue_clear(runqueue_t *runqueue) {
   runqueue_ring_buffer_clear(&runqueue->entries);
 }
 
-inline long runqueue_size(runqueue_t *runqueue) {
+inline unsigned int runqueue_size(runqueue_t *runqueue) {
   return runqueue->entries.size;
 }
 
-inline long runqueue_len(runqueue_t *runqueue) {
+inline unsigned int runqueue_len(runqueue_t *runqueue) {
   return runqueue->entries.count;
 }
 
-inline long runqueue_max_len(runqueue_t *runqueue) {
+inline unsigned int runqueue_max_len(runqueue_t *runqueue) {
   unsigned int max_len = runqueue->high_watermark;
   runqueue->high_watermark = 0;
   return max_len;

data/ext/polyphony/runqueue.h

@@ -19,9 +19,9 @@ runqueue_entry runqueue_shift(runqueue_t *runqueue);
 void runqueue_delete(runqueue_t *runqueue, VALUE fiber);
 int runqueue_index_of(runqueue_t *runqueue, VALUE fiber);
 void runqueue_clear(runqueue_t *runqueue);
-long runqueue_size(runqueue_t *runqueue);
-long runqueue_len(runqueue_t *runqueue);
-long runqueue_max_len(runqueue_t *runqueue);
+unsigned int runqueue_size(runqueue_t *runqueue);
+unsigned int runqueue_len(runqueue_t *runqueue);
+unsigned int runqueue_max_len(runqueue_t *runqueue);
 int runqueue_empty_p(runqueue_t *runqueue);
 
 #endif /* RUNQUEUE_H */

data/lib/polyphony/extensions/io.rb

@@ -76,6 +76,10 @@ end
 
 # IO instance method patches
 class ::IO
+  def __polyphony_read_method__
+    :backend_read
+  end
+
   # def each(sep = $/, limit = nil, chomp: nil)
   #   sep, limit = $/, sep if sep.is_a?(Integer)
   # end
@@ -108,7 +112,11 @@ class ::IO
   end
 
   alias_method :orig_read, :read
-  def read(len = nil)
+  def read(len = nil, buf = nil, buf_pos = 0)
+    if buf
+      return Polyphony.backend_read(self, buf, len, true, buf_pos)
+    end
+    
     @read_buffer ||= +''
     result = Polyphony.backend_read(self, @read_buffer, len, true, -1)
     return nil unless result
@@ -119,9 +127,9 @@ class ::IO
   end
 
   alias_method :orig_readpartial, :read
-  def readpartial(len, str = +'', buffer_pos = 0)
+  def readpartial(len, str = +'', buffer_pos = 0, raise_on_eof = true)
     result = Polyphony.backend_read(self, str, len, false, buffer_pos)
-    raise EOFError unless result
+    raise EOFError if !result && raise_on_eof
 
     result
   end

data/lib/polyphony/extensions/openssl.rb

@@ -5,6 +5,10 @@ require_relative './socket'
 
 # OpenSSL socket helper methods (to make it compatible with Socket API) and overrides
 class ::OpenSSL::SSL::SSLSocket
+  def __polyphony_read_method__
+    :readpartial
+  end
+
   alias_method :orig_initialize, :initialize
   def initialize(socket, context = nil)
     socket = socket.respond_to?(:io) ? socket.io || socket : socket
@@ -64,7 +68,21 @@ class ::OpenSSL::SSL::SSLSocket
     # @sync = osync
   end
 
-  def readpartial(maxlen, buf = +'', buffer_pos = 0)
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return readpartial(maxlen, buf, buf_pos) if buf
+
+    buf = +''
+    return readpartial(maxlen, buf) if maxlen
+
+    while true
+      readpartial(4096, buf, -1)
+    end
+  rescue EOFError
+    buf
+  end
+
+  def readpartial(maxlen, buf = +'', buffer_pos = 0, raise_on_eof = true)
     if buffer_pos != 0
       if (result = sysread(maxlen, +''))
         if buffer_pos == -1
@@ -76,7 +94,9 @@ class ::OpenSSL::SSL::SSLSocket
     else
       result = sysread(maxlen, buf)
     end
-    result || (raise EOFError)
+
+    raise EOFError if !result && raise_on_eof
+    result
   end
 
   def read_loop(maxlen = 8192)

data/lib/polyphony/extensions/socket.rb

@@ -5,6 +5,12 @@ require 'socket'
 require_relative './io'
 require_relative '../core/thread_pool'
 
+class BasicSocket
+  def __polyphony_read_method__
+    :backend_recv
+  end
+end
+
 # Socket overrides (eventually rewritten in C)
 class ::Socket
   def accept
@@ -22,6 +28,23 @@ class ::Socket
     Polyphony.backend_connect(self, addr.ip_address, addr.ip_port)
   end
 
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
+    return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
+    
+    buf = +''
+    len = buf.bytesize
+    while true
+      Polyphony.backend_recv(self, buf, maxlen || 4096, -1)
+      new_len = buf.bytesize
+      break if new_len == len
+
+      len = new_len
+    end
+    buf
+  end
+
   def recv(maxlen, flags = 0, outbuf = nil)
     Polyphony.backend_recv(self, outbuf || +'', maxlen, 0)
   end
@@ -60,8 +83,9 @@ class ::Socket
   #   Polyphony.backend_send(self, mesg, 0)
   # end
 
-  def readpartial(maxlen, str = +'', buffer_pos = 0)
-    Polyphony.backend_recv(self, str, maxlen, buffer_pos)
+  def readpartial(maxlen, str = +'', buffer_pos = 0, raise_on_eof = true)
+    result = Polyphony.backend_recv(self, str, maxlen, buffer_pos)
+    raise EOFError if !result && raise_on_eof
   end
 
   ZERO_LINGER = [0, 0].pack('ii').freeze
@@ -140,6 +164,23 @@ class ::TCPSocket
     setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
   end
 
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
+    return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
+    
+    buf = +''
+    len = buf.bytesize
+    while true
+      Polyphony.backend_recv(self, buf, maxlen || 4096, -1)
+      new_len = buf.bytesize
+      break if new_len == len
+
+      len = new_len
+    end
+    buf
+  end
+
   def recv(maxlen, flags = 0, outbuf = nil)
     Polyphony.backend_recv(self, outbuf || +'', maxlen, 0)
   end
@@ -165,11 +206,10 @@ class ::TCPSocket
   #   Polyphony.backend_send(self, mesg, 0)
   # end
 
-  def readpartial(maxlen, str = +'', buffer_pos = 0)
+  def readpartial(maxlen, str = +'', buffer_pos = 0, raise_on_eof)
     result = Polyphony.backend_recv(self, str, maxlen, buffer_pos)
-    raise EOFError unless result
-
-    str
+    raise EOFError if !result && raise_on_eof
+    result
   end
 
   def read_nonblock(len, str = nil, exception: true)
@@ -217,6 +257,23 @@ class ::UNIXServer
 end
 
 class ::UNIXSocket
+  alias_method :orig_read, :read
+  def read(maxlen = nil, buf = nil, buf_pos = 0)
+    return Polyphony.backend_recv(self, buf, maxlen, buf_pos) if buf
+    return Polyphony.backend_recv(self, buf || +'', maxlen, 0) if maxlen
+    
+    buf = +''
+    len = buf.bytesize
+    while true
+      Polyphony.backend_recv(self, buf, maxlen || 4096, -1)
+      new_len = buf.bytesize
+      break if new_len == len
+
+      len = new_len
+    end
+    buf
+  end
+
   def recv(maxlen, flags = 0, outbuf = nil)
     Polyphony.backend_recv(self, outbuf || +'', maxlen, 0)
   end
@@ -242,11 +299,10 @@ class ::UNIXSocket
     Polyphony.backend_send(self, mesg, 0)
   end
 
-  def readpartial(maxlen, str = +'', buffer_pos = 0)
+  def readpartial(maxlen, str = +'', buffer_pos = 0, raise_on_eof)
     result = Polyphony.backend_recv(self, str, maxlen, buffer_pos)
-    raise EOFError unless result
-
-    str
+    raise EOFError if !result && raise_on_eof
+    result
   end
 
   def read_nonblock(len, str = nil, exception: true)

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.62'
+  VERSION = '0.66'
 end

data/test/helper.rb

@@ -15,6 +15,8 @@ require 'minitest/reporters'
 
 ::Exception.__disable_sanitized_backtrace__ = true
 
+IS_LINUX = RUBY_PLATFORM =~ /linux/
+
 # Minitest::Reporters.use! [
 #   Minitest::Reporters::SpecReporter.new
 # ]

data/test/test_backend.rb

@@ -26,7 +26,7 @@ class BackendTest < MiniTest::Test
       @backend.sleep 0.01
       count += 1
     }.await
-    assert_in_range 0.02..0.04, Time.now - t0
+    assert_in_range 0.02..0.06, Time.now - t0 if IS_LINUX
     assert_equal 3, count
   end
 
@@ -98,6 +98,32 @@ class BackendTest < MiniTest::Test
     assert_equal return_value, buf
   end
 
+  def test_read_concat_big
+    i, o = IO.pipe
+
+    body = " " * 4000
+
+    data = "post /?q=time&blah=blah HTTP/1\r\nHost: dev.realiteq.net\r\n\r\n" +
+           "get /?q=time HTTP/1.1\r\nContent-Length: #{body.bytesize}\r\n\r\n#{body}" +
+           "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
+
+    o << data
+    o.close
+    
+    buf = +''
+
+    @backend.read(i, buf, 4096, false, -1)
+    assert_equal 4096, buf.bytesize
+    
+    @backend.read(i, buf, 1, false, -1)
+    assert_equal 4097, buf.bytesize
+
+    @backend.read(i, buf, 4096, false, -1)
+
+    assert_equal data.bytesize, buf.bytesize
+    assert_equal data, buf
+  end
+
   def test_waitpid
     pid = fork do
       @backend.post_fork
@@ -217,6 +243,8 @@ class BackendTest < MiniTest::Test
   end
 
   def test_timer_loop
+    skip unless IS_LINUX
+
     i = 0
     f = spin do
       @backend.timer_loop(0.01) { i += 1 }
@@ -231,6 +259,8 @@ class BackendTest < MiniTest::Test
   end
 
   def test_timeout
+    skip unless IS_LINUX
+
     buffer = []
     assert_raises(Polyphony::TimeoutException) do
       @backend.timeout(0.01, Polyphony::TimeoutException) do
@@ -262,6 +292,8 @@ class BackendTest < MiniTest::Test
   end
 
   def test_nested_timeout
+    skip unless IS_LINUX
+
     buffer = []
     assert_raises(MyTimeoutException) do
       @backend.timeout(0.01, MyTimeoutException) do
@@ -321,8 +353,8 @@ class BackendTest < MiniTest::Test
 
 
   def test_splice_chunks
-    body = 'abcd' * 250
-    chunk_size = 750
+    body = 'abcd' * 4
+    chunk_size = 12
 
     buf = +''
     r, w = IO.pipe
@@ -347,7 +379,7 @@ class BackendTest < MiniTest::Test
     w.close
     reader.await
 
-    expected = "Content-Type: foo\r\n\r\n#{750.to_s(16)}\r\n#{body[0..749]}\r\n#{250.to_s(16)}\r\n#{body[750..999]}\r\n0\r\n\r\n"
+    expected = "Content-Type: foo\r\n\r\n#{12.to_s(16)}\r\n#{body[0..11]}\r\n#{4.to_s(16)}\r\n#{body[12..15]}\r\n0\r\n\r\n"
     assert_equal expected, buf
   ensure
     o.close
@@ -368,6 +400,9 @@ class BackendTest < MiniTest::Test
     assert_equal count, GC.count
     sleep 0.05
     assert_equal count, GC.count
+
+    return unless IS_LINUX
+
     # The idle tasks are ran at most once per fiber switch, before the backend
     # is polled. Therefore, the second sleep will not have triggered a GC, since
     # only 0.05s have passed since the gc period was set.

data/test/test_global_api.rb

@@ -137,7 +137,7 @@ class MoveOnAfterTest < MiniTest::Test
     t1 = Time.now
 
     assert_nil v
-    assert_in_range 0.014..0.02, t1 - t0
+    assert_in_range 0.014..0.02, t1 - t0 if IS_LINUX
   end
 
   def test_move_on_after_without_block
@@ -152,6 +152,8 @@ class MoveOnAfterTest < MiniTest::Test
   end
 
   def test_nested_move_on_after
+    skip unless IS_LINUX
+
     t0 = Time.now
     o = move_on_after(0.01, with_value: 1) do
       move_on_after(0.02, with_value: 2) do
@@ -210,7 +212,7 @@ class CancelAfterTest < MiniTest::Test
       sleep 0.007
     end
     t1 = Time.now
-    assert_in_range 0.014..0.024, t1 - t0
+    assert_in_range 0.014..0.024, t1 - t0 if IS_LINUX
   end
 
   class CustomException < Exception
@@ -399,7 +401,7 @@ class ThrottledLoopTest < MiniTest::Test
     end
     f.await
     t1 = Time.now
-    assert_in_range 0.075..0.15, t1 - t0
+    assert_in_range 0.075..0.15, t1 - t0 if IS_LINUX
     assert_equal [1, 2, 3, 4, 5], buffer    
   end
 end
@@ -415,6 +417,8 @@ class GlobalAPIEtcTest < MiniTest::Test
   end
 
   def test_every
+    skip unless IS_LINUX
+
     buffer = []
     t0 = Time.now
     f = spin do
@@ -429,7 +433,7 @@ class GlobalAPIEtcTest < MiniTest::Test
     t0 = Time.now
     sleep 0.1
     elapsed = Time.now - t0
-    assert (0.05..0.15).include? elapsed
+    assert (0.05..0.15).include? elapsed if IS_LINUX
 
     f = spin { sleep }
     snooze

data/test/test_io.rb

@@ -73,6 +73,26 @@ class IOTest < MiniTest::Test
     assert_equal [:wait_readable, 'foo'], results
   end
 
+  def test_read
+    i, o = IO.pipe
+
+    o << 'hi'
+    assert_equal 'hi', i.read(2)
+
+    o << 'foobarbaz'
+    assert_equal 'foo', i.read(3)
+    assert_equal 'bar', i.read(3)
+
+    buf = +'abc'
+    assert_equal 'baz', i.read(3, buf)
+    assert_equal 'baz', buf
+
+    buf = +'def'
+    o << 'foobar'
+    assert_equal 'deffoobar', i.read(6, buf, -1)
+    assert_equal 'deffoobar', buf
+  end
+
   def test_readpartial
     i, o = IO.pipe
 
@@ -331,6 +351,8 @@ class IOClassMethodsTest < MiniTest::Test
   end
 
   def test_popen
+    skip unless IS_LINUX
+
     counter = 0
     timer = spin { throttled_loop(200) { counter += 1 } }
 

data/test/test_socket.rb

@@ -12,7 +12,6 @@ class SocketTest < MiniTest::Test
   def test_tcp
     port = rand(1234..5678)
     server = TCPServer.new('127.0.0.1', port)
-
     server_fiber = spin do
       while (socket = server.accept)
         spin do
@@ -34,6 +33,45 @@ class SocketTest < MiniTest::Test
     server&.close
   end
 
+  def test_read
+    port = rand(1234..5678)
+    server = TCPServer.new('127.0.0.1', port)
+    server_fiber = spin do
+      while (socket = server.accept)
+        spin do
+          while (data = socket.read(8192))
+            socket << data
+          end
+        end
+      end
+    end
+
+    snooze
+    client = TCPSocket.new('127.0.0.1', port)
+
+    client << 'hi'
+    assert_equal 'hi', client.read(2)
+
+    client << 'foobarbaz'
+    assert_equal 'foo', client.read(3)
+    assert_equal 'bar', client.read(3)
+
+    buf = +'abc'
+    assert_equal 'baz', client.read(3, buf)
+    assert_equal 'baz', buf
+
+    buf = +'def'
+    client << 'foobar'
+    assert_equal 'deffoobar', client.read(6, buf, -1)
+    assert_equal 'deffoobar', buf
+
+    client.close
+  ensure
+    server_fiber&.stop
+    server_fiber&.await
+    server&.close
+  end
+
   # sending multiple strings at once
   def test_sendv
     port = rand(1234..5678)
@@ -120,19 +158,22 @@ class SocketTest < MiniTest::Test
   end
 end
 
-class HTTPClientTest < MiniTest::Test
-  require 'json'
+if IS_LINUX
+  class HTTPClientTest < MiniTest::Test
 
-  def test_http
-    res = HTTParty.get('http://ipinfo.io/')
+    require 'json'
 
-    response = JSON.load(res.body)
-    assert_equal 'https://ipinfo.io/missingauth', response['readme']
-  end
+    def test_http
+      res = HTTParty.get('http://ipinfo.io/')
 
-  def test_https
-    res = HTTParty.get('https://ipinfo.io/')
-    response = JSON.load(res.body)
-    assert_equal 'https://ipinfo.io/missingauth', response['readme']
+      response = JSON.load(res.body)
+      assert_equal 'https://ipinfo.io/missingauth', response['readme']
+    end
+
+    def test_https
+      res = HTTParty.get('https://ipinfo.io/')
+      response = JSON.load(res.body)
+      assert_equal 'https://ipinfo.io/missingauth', response['readme']
+    end
   end
 end

data/test/test_thread.rb

@@ -180,6 +180,9 @@ class ThreadTest < MiniTest::Test
     assert_equal count, GC.count
     sleep 0.05
     assert_equal count, GC.count
+    
+    return unless IS_LINUX
+
     # The idle tasks are ran at most once per fiber switch, before the backend
     # is polled. Therefore, the second sleep will not have triggered a GC, since
     # only 0.05s have passed since the gc period was set.

data/test/test_timer.rb

@@ -19,7 +19,7 @@ class TimerMoveOnAfterTest < MiniTest::Test
     end
     t1 = Time.now
 
-    assert_in_range 0.1..0.15, t1 - t0
+    assert_in_range 0.1..0.15, t1 - t0 if IS_LINUX
     assert_nil v
   end
 
@@ -31,11 +31,13 @@ class TimerMoveOnAfterTest < MiniTest::Test
     end
     t1 = Time.now
 
-    assert_in_range 0.01..0.05, t1 - t0
+    assert_in_range 0.01..0.05, t1 - t0 if IS_LINUX
     assert_equal :bar, v
   end
 
   def test_timer_move_on_after_with_reset
+    skip unless IS_LINUX
+
     t0 = Time.now
     v = @timer.move_on_after(0.01, with_value: :moved_on) do
       sleep 0.007
@@ -71,7 +73,7 @@ class TimerCancelAfterTest < MiniTest::Test
       end
     end
     t1 = Time.now
-    assert_in_range 0.01..0.03, t1 - t0
+    assert_in_range 0.01..0.03, t1 - t0 if IS_LINUX
   end
 
   def test_timer_cancel_after_with_reset

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: '0.62'
+  version: '0.66'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-21 00:00:00.000000000 Z
+date: 2021-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler