polyphony 0.53.2 → 0.58

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d656c0aad868a468632949355b617821b6a66aa4da69a994ea7afbc4af46e97b
-  data.tar.gz: edc45b184b0b50940925a1a1db56b0106ded10906ca181513e38450bc320e828
+  metadata.gz: e8002b8c0e03afa8e915b5ed67d64c1fc288a5fa220dfb2c32d3966a78046eee
+  data.tar.gz: c3a46eeae1f048f4adee9d3130f5dd3593936e843c1874d928f742d1fa83053f
 SHA512:
-  metadata.gz: f346672c82eee4055ec6c8f7fc1103850fd622a031ebf1c096e9ab130e3aa0b37c559d2b2f953f6b1bd721e4421199800fc0de54993a0c802ddadfa873f35030
-  data.tar.gz: 8273d691c2ed19ca04207380ea99392c7f65fefe6600c4a9bf9cc20a802f41e34b471e3af8d5adccf29c783c2f0c17610896fae2c183ac26b601c34dc27a5ce2
+  metadata.gz: e6681155761daee35b73c9674e69aa505786e576814db0c909028cb576d8609f99bccc79f80b7d06d90c7658747bc2589c9837d3bdb3419782147e573d39f278
+  data.tar.gz: 90197a8db54405ca37492a54a20adf00c85f142af606cad26ad823161825ccf6bd9a414adcbbf3e27eaf93eea1d94a7cbc50a2fb5982ee4ce48969d4378400cd

data/.github/workflows/test.yml

@@ -26,6 +26,6 @@ jobs:
     - name: Show Linux kernel version
       run: uname -r
     - name: Compile C-extension
-      run: bundle exec rake compile
+      run: POLYPHONY_USE_LIBEV=1 bundle exec rake compile
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -56,4 +56,6 @@ lib/*.bundle
 lib/*.so
 
 _site
-.sass-cache
+.sass-cache
+
+log

data/CHANGELOG.md

@@ -1,12 +1,43 @@
-## 0.53.2
+## 0.58 2021-06-25
+
+- Implement `Thread#idle_gc_period`, `#on_idle` (#56)
+- Implement `Backend#idle_block=` (#56)
+
+## 0.57.0 2021-06-23
+
+- Implement `Backend#splice_chunks` method for both libev and io_uring backends
+- Improve waiting for readiness in libev `Backend#splice`, `#splice_to_eof`
+- Enable splice op in libev `Backend#chain` for non-Linux OS
+
+## 0.56.0 2021-06-22
+
+- Implement fake `Backend#splice`, `Backend#splice_to_eof` methods for non-Linux
+  OS
+
+## 0.55.0 2021-06-17
+
+- Finish io_uring implementation of Backend#chain
+- Reimplement io_uring op_context acquire/release algorithm (using ref count) 
+- Fix #gets on sockets
+- Redesign event anti-starvation mechanism
+
+## 0.54.0 2021-06-14
+
+- Implement Mutex#owned?, #locked? (#50)
+- Fix arity for SSLSocket#peeraddr (#55)
+- Add missing SSLServer#accept_loop method (#53)
+- Fix SSLSocket buffering behaviour
+- Add recv_loop alias for SSLSocket (#54)
+
+## 0.53.2 2021-05-10
 
 - Remove `splice` methods on libev backend on non-Linux OS (#43)
 
-## 0.53.0
+## 0.53.0 2021-04-23
 
 - Implement `Backend#splice`, `Backend#splice_to_eof`, along with `IO#splice`, `IO#splice_to_eof`
 
-## 0.52.0
+## 0.52.0 2021-02-28
 
 - Polyphony is now compatible with Ruby 3.0
 - Add `Backend#sendv` method for sending multiple strings
@@ -16,19 +47,19 @@
 - libev backend: Use` pidfd_open` for Linux 5.3+, otherwise use a libev child watcher
 - Use `:call` as default method in `#feed_loop`
 
-## 0.51.0
+## 0.51.0 2021-02-02
 
 - Implement `IO#feed_loop`, `Socket#feed_loop`
 - Fix error handling in `Process.kill_and_await`
 
-## 0.50.1
+## 0.50.1 2021-01-31
 
 - Set `IOSQE_ASYNC` flag in io_uring backend
 - Fix error handling in `Backend#waitpid`
 - Reimplement libev backend's `#waitpid` by using pidfd_open (in similar manner
   to the io_uring backend)
 
-## 0.50.0
+## 0.50.0 2021-01-28
 
 - Use `Process::CLOCK_MONOTONIC` in Timer
 - Add `Timer#sleep`, `Timer#after`, `Timer#every`
@@ -36,50 +67,50 @@
 - Add `Thread#fiber_index_of` method
 - Use `Backend#wait_event` in `Fiber#await`
 
-## 0.49.2
+## 0.49.2 2021-01-19
 
 - Fix hang with 100s or more child fibers when terminating
 - Fix double pending_count increment in io_uring backend
 
-## 0.49.1
+## 0.49.1 2021-01-13
 
 - Use `TCPSocket` instead of `Socket` in `Net.tcp_connect`
 - Catch `Errno::ERSCH` in `Process.kill_and_await`
 - Set io_uring queue size to 2048
 
-## 0.49.0
+## 0.49.0 2021-01-11
 
 - Implement `Polyphony::Timer` for performant timeouts
 
-## 0.48.0
+## 0.48.0 2021-01-05
 
 - Implement graceful shutdown
 - Add support for `break` / `StopIteration` in `spin_loop`
 - Fix `IO#gets`, `IO#readpartial`
 
-## 0.47.5.1
+## 0.47.5.1 2020-11-20
 
 - Add missing `Socket#accept_loop` method
 
-## 0.47.5
+## 0.47.5 2020-11-20
 
 - Add `socket_class` argument to `Backend#accept`, `Backend#accept_loop`
 - Fix `#supervise` to stop when all children fibers are done
 
-## 0.47.4
+## 0.47.4 2020-11-14
 
 - Add support for Unix sockets
 
-## 0.47.3
+## 0.47.3 2020-11-12
 
 - Enable I/O in signal handlers (#45)
 - Accept `:interval` argument in `#spin_loop`
 
-## 0.47.2
+## 0.47.2 2020-11-10
 
 - Fix API compatibility between TCPSocket and IO
 
-## 0.47.0
+## 0.47.0 2020-11-10
 
 - Implement `#spin_scope` used for creating blocking fiber scopes 
 - Reimplement `move_on_after`, `cancel_after`, `Timeout.timeout` using
@@ -87,18 +118,18 @@
 - Implement `Backend#timeout` API
 - Implemented capped queues
 
-## 0.46.1
+## 0.46.1 2020-11-04
 
 - Add `TCPServer#accept_loop`, `OpenSSL::SSL::SSLSocket#accept_loop` method
 - Fix compilation error on MacOS (#43)
 - Fix backtrace for `Timeout.timeout`
 - Add `Backend#timer_loop`
 
-## 0.46.0
+## 0.46.0 2020-10-08
 
 - Implement [io_uring backend](https://github.com/digital-fabric/polyphony/pull/44)
 
-## 0.45.5
+## 0.45.5 2020-10-04
 
 - Fix compilation error (#43)
 - Add support for resetting move_on_after, cancel_after timeouts
@@ -107,22 +138,22 @@
 - Schedule parent with priority on uncaught exception
 - Fix race condition in `Mutex#synchronize` (#41)
 
-## 0.45.4
+## 0.45.4 2020-09-06
 
 - Improve signal trapping mechanism
 
-## 0.45.3
+## 0.45.3 2020-09-02
 
 - Don't swallow error in `Process#kill_and_await`
 - Add `Fiber#mailbox` attribute reader
 - Fix bug in `Fiber.await`
 - Implement `IO#getc`, `IO#getbyte`
 
-## 0.45.2
+## 0.45.2 2020-08-03
 
 - Rewrite `Fiber#<<`, `Fiber#await`, `Fiber#receive` in C
 
-## 0.45.1
+## 0.45.1 2020-08-01
 
 - Fix Net::HTTP compatibility
 - Fix fs adapter
@@ -132,7 +163,7 @@
 - Cleanup code
 - Improve support for Ruby 3 keyword args
 
-## 0.45.0
+## 0.45.0 2020-07-29
 
 - Cleanup code
 - Rename `Agent` to `Backend`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.53.2)
+    polyphony (0.58)
 
 GEM
   remote: https://rubygems.org/
@@ -17,6 +17,7 @@ GEM
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     json (2.3.0)
+    localhost (1.1.8)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
@@ -72,6 +73,7 @@ DEPENDENCIES
   hiredis (= 0.6.3)
   http_parser.rb (~> 0.6.0)
   httparty (= 0.17.1)
+  localhost (~> 1.1.4)
   minitest (= 5.14.4)
   minitest-reporters (= 1.4.2)
   msgpack (= 1.4.2)

data/TODO.md

@@ -10,9 +10,6 @@
 
 - Add support for `break` and `StopIteration` in all loops (with tests)
 
-- Change `IO#gets` to use `String#split` to cut into lines, much faster (see
-  examples/performance/line_splitting.rb)
-
 - More tight loops
   - `IO#gets_loop`, `Socket#gets_loop`, `OpenSSL::Socket#gets_loop` (medium effort)
   - `Fiber#receive_loop` (very little effort, should be implemented in C)

data/examples/core/idle_gc.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+require 'polyphony'
+
+GC.disable
+
+p count: GC.count
+snooze
+p count_after_snooze: GC.count
+sleep 0.1
+p count_after_sleep: GC.count
+
+Thread.current.backend.idle_gc_period = 60
+
+p count: GC.count
+snooze
+p count_after_snooze: GC.count
+sleep 0.1
+p count_after_sleep: GC.count

data/examples/core/queue.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+queue = Queue.new
+
+4.times { |i|
+  spin_loop {
+    job = queue.pop
+    puts("worker %d job %s" % [i, job.inspect])
+  }
+}
+
+(1..10).each do |i|
+  queue << "job#{i}"
+end
+
+sleep 0.1 until queue.empty?

data/examples/io/https_server.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'localhost/authority'
+
+authority = Localhost::Authority.fetch
+opts = {
+  reuse_addr:     true,
+  dont_linger:    true,
+  secure_context: authority.server_context
+}
+
+server = Polyphony::Net.tcp_listen('localhost', 1234, opts)
+
+puts 'Serving HTTPS on port 1234'
+
+spin_loop(interval: 1) { STDOUT << '.' }
+
+# server.accept_loop do |socket|
+while (socket = server.accept)
+  spin do
+    while (data = socket.gets("\n", 8192))
+      if data.chomp.empty?
+        socket << "HTTP/1.1 200 OK\nConnection: close\nContent-Length: 4\n\nfoo\n"
+        break
+      end
+    end
+  end
+end

data/examples/io/pipe.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+i, o = IO.pipe
+f = spin { p i.read }
+
+o << 'hello'
+o.close
+f.await

data/examples/io/splice_chunks.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+i, o = IO.pipe
+
+f = spin do
+  i.read_loop { |data| STDOUT << data }
+end
+
+result = nil
+# File.open(__FILE__, 'r') do |f|
+File.open('../tipi/log', 'r') do |f|
+  result = Thread.current.backend.splice_chunks(
+    f,
+    o,
+    "Content-Type: ruby\n\n",
+    "0\r\n\r\n",
+    ->(len) { "#{len.to_s(16)}\r\n" },
+    "\r\n",
+    16384
+  )
+end
+
+
+o.close
+f.await
+p result: result

data/examples/io/stdio.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+puts 'Please enter your name:'
+name = gets.chomp
+puts "Hello, #{name}!"

data/ext/polyphony/backend_common.c

@@ -0,0 +1,197 @@
+#include <time.h>
+#include <fcntl.h>
+#include "ruby.h"
+#include "ruby/io.h"
+#include "polyphony.h"
+#include "backend_common.h"
+
+inline void initialize_backend_base(struct Backend_base *base) {
+  base->currently_polling = 0;
+  base->pending_count = 0;
+  base->idle_gc_period = 0;
+  base->idle_gc_last_time = 0;
+  base->idle_block = Qnil;
+}
+
+#ifdef POLYPHONY_USE_PIDFD_OPEN
+#ifndef __NR_pidfd_open
+#define __NR_pidfd_open 434   /* System call # on most architectures */
+#endif
+
+inline int pidfd_open(pid_t pid, unsigned int flags) {
+  return syscall(__NR_pidfd_open, pid, flags);
+}
+#endif
+
+//////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////
+// the following is copied verbatim from the Ruby source code (io.c)
+
+inline int io_setstrbuf(VALUE *str, long len) {
+  #ifdef _WIN32
+    len = (len + 1) & ~1L;	/* round up for wide char */
+  #endif
+  if (*str == Qnil) {
+    *str = rb_str_new(0, len);
+    return 1;
+  }
+  else {
+    VALUE s = StringValue(*str);
+    long clen = RSTRING_LEN(s);
+    if (clen >= len) {
+      rb_str_modify(s);
+      return 0;
+    }
+    len -= clen;
+  }
+  rb_str_modify_expand(*str, len);
+  return 0;
+}
+
+#define MAX_REALLOC_GAP 4096
+
+inline void io_shrink_read_string(VALUE str, long n) {
+  if (rb_str_capacity(str) - n > MAX_REALLOC_GAP) {
+    rb_str_resize(str, n);
+  }
+}
+
+inline void io_set_read_length(VALUE str, long n, int shrinkable) {
+  if (RSTRING_LEN(str) != n) {
+    rb_str_modify(str);
+    rb_str_set_len(str, n);
+    if (shrinkable) io_shrink_read_string(str, n);
+  }
+}
+
+inline rb_encoding* io_read_encoding(rb_io_t *fptr) {
+  if (fptr->encs.enc) {
+	  return fptr->encs.enc;
+  }
+  return rb_default_external_encoding();
+}
+
+inline VALUE io_enc_str(VALUE str, rb_io_t *fptr) {
+  OBJ_TAINT(str);
+  rb_enc_associate(str, io_read_encoding(fptr));
+  return str;
+}
+
+//////////////////////////////////////////////////////////////////////
+//////////////////////////////////////////////////////////////////////
+
+VALUE backend_await(struct Backend_base *backend) {
+  VALUE ret;
+  backend->pending_count++;
+  ret = Thread_switch_fiber(rb_thread_current());
+  backend->pending_count--;
+  RB_GC_GUARD(ret);
+  return ret;
+}
+
+VALUE backend_snooze() {
+  Fiber_make_runnable(rb_fiber_current(), Qnil);
+  VALUE ret = Thread_switch_fiber(rb_thread_current());
+  return ret;
+}
+
+inline void rectify_io_file_pos(rb_io_t *fptr) {
+  // Apparently after reopening a closed file, the file position is not reset,
+  // which causes the read to fail. Fortunately we can use fptr->rbuf.len to
+  // find out if that's the case.
+  // See: https://github.com/digital-fabric/polyphony/issues/30
+  if (fptr->rbuf.len > 0) {
+    lseek(fptr->fd, -fptr->rbuf.len, SEEK_CUR);
+    fptr->rbuf.len = 0;
+  }
+}
+
+inline double current_time() {
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  long long ns = ts.tv_sec;
+  ns = ns * 1e9 + ts.tv_nsec;
+  double t = ns;
+  return t / 1e9;
+}
+
+inline VALUE backend_timeout_exception(VALUE exception) {
+  if (rb_obj_is_kind_of(exception, rb_cArray) == Qtrue)
+    return rb_funcall(rb_ary_entry(exception, 0), ID_new, 1, rb_ary_entry(exception, 1));
+  else if (rb_obj_is_kind_of(exception, rb_cClass) == Qtrue)
+    return rb_funcall(exception, ID_new, 0);
+  else
+    return rb_funcall(rb_eRuntimeError, ID_new, 1, exception);
+}
+
+VALUE Backend_timeout_safe(VALUE arg) {
+  return rb_yield(arg);
+}
+
+VALUE Backend_timeout_rescue(VALUE arg, VALUE exception) {
+  return exception;
+}
+
+VALUE Backend_timeout_ensure_safe(VALUE arg) {
+  return rb_rescue2(Backend_timeout_safe, Qnil, Backend_timeout_rescue, Qnil, rb_eException, (VALUE)0);
+}
+
+static VALUE empty_string = Qnil;
+
+VALUE Backend_sendv(VALUE self, VALUE io, VALUE ary, VALUE flags) {
+  switch (RARRAY_LEN(ary)) {
+  case 0:
+    return Qnil;
+  case 1:
+    return Backend_send(self, io, RARRAY_AREF(ary, 0), flags);
+  default:
+    if (empty_string == Qnil) {
+      empty_string = rb_str_new_literal("");
+      rb_global_variable(&empty_string);
+    }
+    VALUE joined = rb_ary_join(ary, empty_string);
+    VALUE result = Backend_send(self, io, joined, flags);
+    RB_GC_GUARD(joined);
+    return result;
+  }
+}
+
+inline void io_verify_blocking_mode(rb_io_t *fptr, VALUE io, VALUE blocking) {
+  VALUE blocking_mode = rb_ivar_get(io, ID_ivar_blocking_mode);
+  if (blocking == blocking_mode) return;
+
+  rb_ivar_set(io, ID_ivar_blocking_mode, blocking);
+
+#ifdef _WIN32
+  if (blocking != Qtrue)
+    rb_w32_set_nonblock(fptr->fd);
+#elif defined(F_GETFL)
+  int flags = fcntl(fptr->fd, F_GETFL);
+  if (flags == -1) return;
+  int is_nonblocking = flags & O_NONBLOCK;
+  
+  if (blocking == Qtrue) {
+    if (!is_nonblocking) return;
+    flags &= ~O_NONBLOCK;
+  } else {
+    if (is_nonblocking) return;
+    flags |= O_NONBLOCK;
+  }
+  fcntl(fptr->fd, F_SETFL, flags);
+#endif
+}
+
+inline void backend_run_idle_tasks(struct Backend_base *base) {
+  if (base->idle_block != Qnil)
+    rb_funcall(base->idle_block, ID_call, 0);
+
+  if (base->idle_gc_period == 0) return;
+
+  double now = current_time();
+  if (now - base->idle_gc_last_time < base->idle_gc_period) return;
+
+  base->idle_gc_last_time = now;
+  rb_gc_enable();
+  rb_gc_start();
+  rb_gc_disable();
+}