polyphony 0.51.0 → 0.52.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 72f11d867863c51f1fc1ab3ec85fe575cca9ee53a5047f4d3f7b76cea3b650b7
-  data.tar.gz: c29354b2de2f207185fc06fd86421b53d328aaf67d611109a6f95e9130ac5967
+  metadata.gz: 2056b02e3b364911ab627ffc49a7251ad2eed68ba6b5b61d380c599127c181f6
+  data.tar.gz: 440f57d240b90a255b470f405980b5fa2f806ba3ff2640f31d78b6403de7ce0e
 SHA512:
-  metadata.gz: 93351a3c4007145ff317257f82b764708b445e94b848d31f783f4d72071b097a8309322c4bedfdb7f321b0613ef436a314ad4e650928d0855143d3a5b98d9c13
-  data.tar.gz: 9119039548264867fa012bcbb90999f4ad82170c28539d43b0145687a761d508630da538ac87b75f9bf32f6b33cb0e13c22f50143248b85cc90289d5d3451d3f
+  metadata.gz: b5bf34a4ea7addf2b71c2e0acc75041d95f5d76ae425061bd5475e6523111a05cf969b0109ec90874f7f1a1929111182ed193b084b3388be060886145dd20567
+  data.tar.gz: 6165128a0393664c97baa494119e453db177b7d0a12c5a5ade0ab5bb8e77a4143f6156b9cb594ef2edbbe9e0ac3bb85a2c846da8e925c3a6ec1c294120c18bf2

data/.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        ruby: [2.6, 2.7]
+        ruby: [2.6, 2.7, 3.0]
 
     name: >-
       ${{matrix.os}}, ${{matrix.ruby}}

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.52.0
+
+- Polyphony is now compatible with Ruby 3.0
+- Add `Backend#sendv` method for sending multiple strings
+- Accept flags argument in `Backend#send` (#48)
+- Fix io_uring backend on Ruby 3.0 (#47)
+- Implement C-based public backend API: `Polyphony.backend_XXXX` methods
+- libev backend: Use` pidfd_open` for Linux 5.3+, otherwise use a libev child watcher
+- Use `:call` as default method in `#feed_loop`
+
 ## 0.51.0
 
 - Implement `IO#feed_loop`, `Socket#feed_loop`

data/Gemfile.lock

@@ -1,72 +1,27 @@
 PATH
   remote: .
   specs:
-    polyphony (0.51.0)
+    polyphony (0.52.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
-      public_suffix (>= 2.0.2, < 5.0)
     ansi (1.5.0)
     ast (2.4.0)
     builder (3.2.4)
     coderay (1.1.3)
-    colorator (1.1.0)
-    concurrent-ruby (1.1.6)
     docile (1.3.2)
-    em-websocket (0.5.1)
-      eventmachine (>= 0.12.9)
-      http_parser.rb (~> 0.6.0)
-    eventmachine (1.2.7)
-    ffi (1.12.1)
-    forwardable-extended (2.6.0)
     hiredis (0.6.3)
     http_parser.rb (0.6.0)
     httparty (0.17.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    i18n (0.9.5)
-      concurrent-ruby (~> 1.0)
-    jekyll (3.8.6)
-      addressable (~> 2.4)
-      colorator (~> 1.0)
-      em-websocket (~> 0.5)
-      i18n (~> 0.7)
-      jekyll-sass-converter (~> 1.0)
-      jekyll-watch (~> 2.0)
-      kramdown (~> 1.14)
-      liquid (~> 4.0)
-      mercenary (~> 0.3.3)
-      pathutil (~> 0.9)
-      rouge (>= 1.7, < 4)
-      safe_yaml (~> 1.0)
-    jekyll-remote-theme (0.4.1)
-      addressable (~> 2.0)
-      jekyll (>= 3.5, < 5.0)
-      rubyzip (>= 1.3.0)
-    jekyll-sass-converter (1.5.2)
-      sass (~> 3.4)
-    jekyll-seo-tag (2.6.1)
-      jekyll (>= 3.3, < 5.0)
-    jekyll-watch (2.2.1)
-      listen (~> 3.0)
     json (2.3.0)
-    just-the-docs (0.3.0)
-      jekyll (>= 3.8.5)
-      jekyll-seo-tag (~> 2.0)
-      rake (>= 12.3.1, < 13.1.0)
-    kramdown (1.17.0)
-    liquid (4.0.3)
-    listen (3.2.1)
-      rb-fsevent (~> 0.10, >= 0.10.3)
-      rb-inotify (~> 0.9, >= 0.9.10)
-    mercenary (0.3.6)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
-    minitest (5.13.0)
+    minitest (5.14.4)
     minitest-reporters (1.4.2)
       ansi
       builder
@@ -78,25 +33,18 @@ GEM
     parallel (1.19.1)
     parser (2.7.0.2)
       ast (~> 2.4.0)
-    pathutil (0.16.2)
-      forwardable-extended (~> 2.6)
     pg (1.1.4)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.3)
     rack (2.2.3)
     rainbow (3.0.0)
-    rake (12.3.3)
+    rake (13.0.3)
     rake-compiler (1.1.1)
       rake
-    rb-fsevent (0.10.3)
-    rb-inotify (0.10.1)
-      ffi (~> 1.0)
     redis (4.1.0)
     regexp_parser (1.7.1)
     rexml (3.2.4)
-    rouge (3.15.0)
     rubocop (0.85.1)
       parallel (~> 1.10)
       parser (>= 2.7.0.1)
@@ -109,13 +57,6 @@ GEM
     rubocop-ast (0.0.3)
       parser (>= 2.7.0.1)
     ruby-progressbar (1.10.1)
-    rubyzip (2.0.0)
-    safe_yaml (1.0.5)
-    sass (3.7.4)
-      sass-listen (~> 4.0.0)
-    sass-listen (4.0.0)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
     sequel (5.34.0)
     simplecov (0.17.1)
       docile (~> 1.1)
@@ -131,11 +72,7 @@ DEPENDENCIES
   hiredis (= 0.6.3)
   http_parser.rb (~> 0.6.0)
   httparty (= 0.17.1)
-  jekyll (~> 3.8.6)
-  jekyll-remote-theme (~> 0.4.1)
-  jekyll-seo-tag (~> 2.6.1)
-  just-the-docs (~> 0.3.0)
-  minitest (= 5.13.0)
+  minitest (= 5.14.4)
   minitest-reporters (= 1.4.2)
   msgpack (= 1.4.2)
   mysql2 (= 0.5.3)
@@ -150,4 +87,4 @@ DEPENDENCIES
   simplecov (= 0.17.1)
 
 BUNDLED WITH
-   2.1.4
+   2.2.3

data/TODO.md

@@ -1,9 +1,12 @@
+- Implement io_uring Backend_send with variable arity.
+- Implement a buffer store for use in:
+  - io_uring Backend_send_m
+  - io_uring Backend_writev (for iov)
+  - libvev Backend_writev (for iov)
+
 - Check segfault when resetting a `cancel_after` timeout lots of times at very high rate
 - Check why `throttled_loop` inside of `move_on_after` fails to stop
 
-- Commented out `io_uring_sqe_set_flags(sqe, IOSQE_ASYNC);` in `io_uring_backend_defer_submit_and_await`:
-  - This flag should be set for I/O ops, not for other stuff
-
 - Override stock `::SizedQueue` impl with Queue with capacity
 
 - Add support for `break` and `StopIteration` in all loops (with tests)

data/examples/core/forking.rb

@@ -5,7 +5,7 @@ require 'polyphony'
 
 puts "parent pid: #{Process.pid}"
 
-pid = Polyphony.fork do
+pid = fork do
   puts "child pid: #{Process.pid}"
 
   spin do
@@ -20,5 +20,5 @@ end
 puts "got child pid #{pid}"
 
 puts 'parent waiting for child'
-Thread.current.backend.waitpid(pid)
+Polyphony.backend_waitpid(pid)
 puts 'parent done waiting'

data/ext/polyphony/backend_common.h

@@ -3,6 +3,8 @@
 #include "ruby.h"
 #include "ruby/io.h"
 
+
+#ifdef POLYPHONY_USE_PIDFD_OPEN
 #ifndef __NR_pidfd_open
 #define __NR_pidfd_open 434   /* System call # on most architectures */
 #endif
@@ -10,6 +12,7 @@
 static int pidfd_open(pid_t pid, unsigned int flags) {
   return syscall(__NR_pidfd_open, pid, flags);
 }
+#endif
 
 //////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////
@@ -23,11 +26,11 @@ struct io_internal_read_struct {
 
 #define StringValue(v) rb_string_value(&(v))
 
-inline int io_setstrbuf(VALUE *str, long len) {
+int io_setstrbuf(VALUE *str, long len) {
   #ifdef _WIN32
     len = (len + 1) & ~1L;	/* round up for wide char */
   #endif
-  if (NIL_P(*str)) {
+  if (*str == Qnil) {
     *str = rb_str_new(0, len);
     return 1;
   }
@@ -52,7 +55,7 @@ inline void io_shrink_read_string(VALUE str, long n) {
   }
 }
 
-inline void io_set_read_length(VALUE str, long n, int shrinkable) {
+void io_set_read_length(VALUE str, long n, int shrinkable) {
   if (RSTRING_LEN(str) != n) {
     rb_str_modify(str);
     rb_str_set_len(str, n);
@@ -67,7 +70,7 @@ inline rb_encoding* io_read_encoding(rb_io_t *fptr) {
     return rb_default_external_encoding();
 }
 
-inline VALUE io_enc_str(VALUE str, rb_io_t *fptr) {
+VALUE io_enc_str(VALUE str, rb_io_t *fptr) {
     OBJ_TAINT(str);
     rb_enc_associate(str, io_read_encoding(fptr));
     return str;
@@ -135,9 +138,9 @@ inline double current_time() {
 }
 
 inline VALUE backend_timeout_exception(VALUE exception) {
-  if (RTEST(rb_obj_is_kind_of(exception, rb_cArray)))
+  if (rb_obj_is_kind_of(exception, rb_cArray) == Qtrue)
     return rb_funcall(rb_ary_entry(exception, 0), ID_new, 1, rb_ary_entry(exception, 1));
-  else if (RTEST(rb_obj_is_kind_of(exception, rb_cClass)))
+  else if (rb_obj_is_kind_of(exception, rb_cClass) == Qtrue)
     return rb_funcall(exception, ID_new, 0);
   else
     return rb_funcall(rb_eRuntimeError, ID_new, 1, exception);
@@ -154,3 +157,23 @@ VALUE Backend_timeout_rescue(VALUE arg, VALUE exception) {
 VALUE Backend_timeout_ensure_safe(VALUE arg) {
   return rb_rescue2(Backend_timeout_safe, Qnil, Backend_timeout_rescue, Qnil, rb_eException, (VALUE)0);
 }
+
+static VALUE empty_string = Qnil;
+
+VALUE Backend_sendv(VALUE self, VALUE io, VALUE ary, VALUE flags) {
+  switch (RARRAY_LEN(ary)) {
+  case 0:
+    return Qnil;
+  case 1:
+    return Backend_send(self, io, RARRAY_AREF(ary, 0), flags);
+  default:
+    if (empty_string == Qnil) {
+      empty_string = rb_str_new_literal("");
+      rb_global_variable(&empty_string);
+    }
+    VALUE joined = rb_ary_join(ary, empty_string);
+    VALUE result = Backend_send(self, io, joined, flags);
+    RB_GC_GUARD(joined);
+    return result;
+  }
+}

data/ext/polyphony/backend_io_uring.c

@@ -16,11 +16,36 @@
 
 #include "polyphony.h"
 #include "../liburing/liburing.h"
-#include "ruby/thread.h"
 #include "backend_io_uring_context.h"
+#include "ruby/thread.h"
+#include "ruby/io.h"
 
 VALUE SYM_io_uring;
 
+#ifdef POLYPHONY_UNSET_NONBLOCK
+ID ID_ivar_is_nonblocking;
+
+// One of the changes introduced in Ruby 3.0 as part of the work on the
+// FiberScheduler interface is that all created sockets are marked as
+// non-blocking. This prevents the io_uring backend from working correctly,
+// since it will return an EAGAIN error just like a normal syscall. So here
+// instead of setting O_NONBLOCK (which is required for the libev backend), we
+// unset it.
+inline void io_unset_nonblock(rb_io_t *fptr, VALUE io) {
+  VALUE is_nonblocking = rb_ivar_get(io, ID_ivar_is_nonblocking);
+  if (is_nonblocking == Qfalse) return;
+
+  rb_ivar_set(io, ID_ivar_is_nonblocking, Qfalse);
+
+  int oflags = fcntl(fptr->fd, F_GETFL);
+  if ((oflags == -1) && (oflags & O_NONBLOCK)) return;
+  oflags &= !O_NONBLOCK;
+  fcntl(fptr->fd, F_SETFL, oflags);
+}
+#else
+#define io_unset_nonblock(fptr, io)
+#endif
+
 typedef struct Backend_t {
   // common fields
   unsigned int        currently_polling;
@@ -300,6 +325,7 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof)
   if (underlying_io != Qnil) io = underlying_io;
   GetOpenFile(io, fptr);
   rb_io_check_byte_readable(fptr);
+  io_unset_nonblock(fptr, io);
   rectify_io_file_pos(fptr);
   OBJ_TAINT(str);
 
@@ -360,6 +386,7 @@ VALUE Backend_read_loop(VALUE self, VALUE io) {
   if (underlying_io != Qnil) io = underlying_io;
   GetOpenFile(io, fptr);
   rb_io_check_byte_readable(fptr);
+  io_unset_nonblock(fptr, io);
   rectify_io_file_pos(fptr);
 
   while (1) {
@@ -406,6 +433,7 @@ VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
   if (underlying_io != Qnil) io = underlying_io;
   GetOpenFile(io, fptr);
   rb_io_check_byte_readable(fptr);
+  io_unset_nonblock(fptr, io);
   rectify_io_file_pos(fptr);
 
   while (1) {
@@ -445,6 +473,7 @@ VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
   GetBackend(self, backend);
   io = rb_io_get_write_io(io);
   GetOpenFile(io, fptr);
+  io_unset_nonblock(fptr, io);
 
   char *buf = StringValuePtr(str);
   long len = RSTRING_LEN(str);
@@ -488,6 +517,7 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
   GetBackend(self, backend);
   io = rb_io_get_write_io(io);
   GetOpenFile(io, fptr);
+  io_unset_nonblock(fptr, io);
 
   iov = malloc(iov_count * sizeof(struct iovec));
   for (int i = 0; i < argc; i++) {
@@ -567,6 +597,7 @@ VALUE Backend_recv(VALUE self, VALUE io, VALUE str, VALUE length) {
   if (underlying_io != Qnil) io = underlying_io;
   GetOpenFile(io, fptr);
   rb_io_check_byte_readable(fptr);
+  io_unset_nonblock(fptr, io);
   rectify_io_file_pos(fptr);
   OBJ_TAINT(str);
 
@@ -614,6 +645,7 @@ VALUE Backend_recv_loop(VALUE self, VALUE io) {
   if (underlying_io != Qnil) io = underlying_io;
   GetOpenFile(io, fptr);
   rb_io_check_byte_readable(fptr);
+  io_unset_nonblock(fptr, io);
   rectify_io_file_pos(fptr);
 
   while (1) {
@@ -659,6 +691,7 @@ VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method)
   if (underlying_io != Qnil) io = underlying_io;
   GetOpenFile(io, fptr);
   rb_io_check_byte_readable(fptr);
+  io_unset_nonblock(fptr, io);
   rectify_io_file_pos(fptr);
 
   while (1) {
@@ -687,7 +720,7 @@ VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method)
   return io;
 }
 
-VALUE Backend_send(VALUE self, VALUE io, VALUE str) {
+VALUE Backend_send(VALUE self, VALUE io, VALUE str, VALUE flags) {
   Backend_t *backend;
   rb_io_t *fptr;
   VALUE underlying_io;
@@ -697,16 +730,18 @@ VALUE Backend_send(VALUE self, VALUE io, VALUE str) {
   GetBackend(self, backend);
   io = rb_io_get_write_io(io);
   GetOpenFile(io, fptr);
+  io_unset_nonblock(fptr, io);
 
   char *buf = StringValuePtr(str);
   long len = RSTRING_LEN(str);
   long left = len;
+  int flags_int = NUM2INT(flags);
 
   while (left > 0) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_SEND);
     struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
-    io_uring_prep_send(sqe, fptr->fd, buf, left, 0);
+    io_uring_prep_send(sqe, fptr->fd, buf, left, flags_int);
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     OP_CONTEXT_RELEASE(&backend->store, ctx);
@@ -734,6 +769,8 @@ VALUE io_uring_backend_accept(Backend_t *backend, VALUE server_socket, VALUE soc
   if (underlying_sock != Qnil) server_socket = underlying_sock;
 
   GetOpenFile(server_socket, fptr);
+  io_unset_nonblock(fptr, server_socket);
+
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_ACCEPT);
@@ -797,6 +834,7 @@ VALUE Backend_connect(VALUE self, VALUE sock, VALUE host, VALUE port) {
 
   GetBackend(self, backend);
   GetOpenFile(sock, fptr);
+  io_unset_nonblock(fptr, sock);
 
   addr.sin_family = AF_INET;
   addr.sin_addr.s_addr = inet_addr(host_buf);
@@ -823,6 +861,7 @@ VALUE Backend_wait_io(VALUE self, VALUE io, VALUE write) {
   if (underlying_io != Qnil) io = underlying_io;
   GetBackend(self, backend);
   GetOpenFile(io, fptr);
+  io_unset_nonblock(fptr, io);
 
   VALUE resume_value = io_uring_backend_wait_fd(backend, fptr->fd, RTEST(write));
   RAISE_IF_EXCEPTION(resume_value);
@@ -997,7 +1036,7 @@ VALUE Backend_kind(VALUE self) {
 }
 
 void Init_Backend() {
-  VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cData);
+  VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cObject);
   rb_define_alloc_func(cBackend, Backend_allocate);
 
   rb_define_method(cBackend, "initialize", Backend_initialize, 0);
@@ -1014,7 +1053,8 @@ void Init_Backend() {
   rb_define_method(cBackend, "recv", Backend_recv, 3);
   rb_define_method(cBackend, "recv_loop", Backend_recv_loop, 1);
   rb_define_method(cBackend, "recv_feed_loop", Backend_recv_feed_loop, 3);
-  rb_define_method(cBackend, "send", Backend_send, 2);
+  rb_define_method(cBackend, "send", Backend_send, 3);
+  rb_define_method(cBackend, "sendv", Backend_sendv, 3);
   rb_define_method(cBackend, "accept", Backend_accept, 2);
   rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
   rb_define_method(cBackend, "connect", Backend_connect, 3);
@@ -1027,6 +1067,10 @@ void Init_Backend() {
 
   rb_define_method(cBackend, "kind", Backend_kind, 0);
 
+  #ifdef POLYPHONY_UNSET_NONBLOCK
+  ID_ivar_is_nonblocking = rb_intern("@is_nonblocking");
+  #endif
+
   SYM_io_uring = ID2SYM(rb_intern("io_uring"));
 }