polyphony 0.47.1 → 0.47.5.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.47.1)
+    polyphony (0.47.5.1)
 
 GEM
   remote: https://rubygems.org/

data/TODO.md

@@ -1,12 +1,41 @@
+- Graceful shutdown again:
+
+  - Fiber API:
+    - `Fiber#terminate(graceul)` - with a graceful flag
+    - `Fiber#terminate_all_children(graceful)` - with a graceful flag
+    - `Fiber#shutdown_all_children(graceful)` - with a graceful flag
+
+  - Set graceful termination in `@graceful_shutdown`
+  - Add `Fiber#graceful_shutdown?` method
+    - Returns `@graceful_shutdown`
+
+  And then we have:
+
+  ```ruby
+  spin do
+    loop { do_some_stuff }
+  ensure
+    shutdown_gracefully if Fiber.current.graceful_shutdown?
+  end
+  ```
+
+  - `Fiber#finalize_children` should pass graceful shutdown flag to children
+
+- More tight loops
+  - IO#gets_loop, Socket#gets_loop, OpenSSL::Socket#gets_loop (medium effort)
+  - Fiber#receive_loop (very little effort, should be implemented in C)
+
 ## Roadmap for Polyphony 1.0
 
-- Check why worker-thread example doesn't work.
+- check integration with rb-inotify
+
+- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
+  inconsistent behaviour (see supervisor example).
+
 - Add test that mimics the original design for Monocrono:
   - 256 fibers each waiting for a message
   - When message received do some blocking work using a `ThreadPool`
   - Send messages, collect responses, check for correctness
-- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
-  inconsistent behaviour (see supervisor example).
 
 - io_uring
   - Use playground.c to find out why we when submitting and waiting for

data/examples/core/supervisor.rb

@@ -9,9 +9,9 @@ def my_sleep(t)
   puts "#{t} done"
 end
 
-spin { my_sleep(1) }
-spin { my_sleep(2) }
-spin { my_sleep(3) }
+spin { my_sleep(0.1) }
+spin { my_sleep(0.2) }
+spin { my_sleep(0.3) }
 spin { puts "fiber count: #{Fiber.current.children.count}" }
 snooze
 

data/examples/core/worker-thread.rb

@@ -13,11 +13,9 @@ end
 $worker = Thread.new do
   Fiber.current.tag = :worker
   loop do
-    client, block = receive
+    (client, block) = receive
     do_work(client, &block)
   end
-rescue Exception => e
-  p e
 end
 
 def process(&block)
@@ -27,4 +25,5 @@ end
 
 sleep 0.1
 
-p process { 1 + 1 }
+p process { 1 + 1 }
+p process { 42 ** 2 }

data/examples/io/unix_socket.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'fileutils'
+
+unix_path = '/tmp/polyphony-unix-socket'
+
+FileUtils.rm unix_path rescue nil
+server = UNIXServer.new(unix_path)
+spin do
+  server.accept_loop do |socket|
+    p [:accept, socket]
+    spin do
+      while (line = socket.gets)
+        socket.puts line
+      end
+    end
+  end
+end
+
+snooze
+client = UNIXSocket.new('/tmp/polyphony-unix-socket')
+p [:connected, client]
+client.puts 'hello!'
+p client.gets

data/ext/polyphony/backend_io_uring.c

@@ -27,7 +27,6 @@ static int pidfd_open(pid_t pid, unsigned int flags) {
   return syscall(__NR_pidfd_open, pid, flags);
 }
 
-VALUE cTCPSocket;
 VALUE SYM_io_uring;
 
 typedef struct Backend_t {
@@ -669,15 +668,15 @@ VALUE Backend_send(VALUE self, VALUE io, VALUE str) {
   return INT2NUM(len);
 }
 
-VALUE io_uring_backend_accept(Backend_t *backend, VALUE sock, int loop) {
+VALUE io_uring_backend_accept(Backend_t *backend, VALUE server_socket, VALUE socket_class, int loop) {
   rb_io_t *fptr;
   struct sockaddr addr;
   socklen_t len = (socklen_t)sizeof addr;
-  VALUE underlying_sock = rb_ivar_get(sock, ID_ivar_io);
   VALUE socket = Qnil;
-  if (underlying_sock != Qnil) sock = underlying_sock;
+  VALUE underlying_sock = rb_ivar_get(server_socket, ID_ivar_io);
+  if (underlying_sock != Qnil) server_socket = underlying_sock;
 
-  GetOpenFile(sock, fptr);
+  GetOpenFile(server_socket, fptr);
   while (1) {
     VALUE resume_value = Qnil;
     op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_ACCEPT);
@@ -695,7 +694,7 @@ VALUE io_uring_backend_accept(Backend_t *backend, VALUE sock, int loop) {
     else {
       rb_io_t *fp;
 
-      socket = rb_obj_alloc(cTCPSocket);
+      socket = rb_obj_alloc(socket_class);
       MakeOpenFile(socket, fp);
       rb_update_max_fd(fd);
       fp->fd = fd;
@@ -718,16 +717,16 @@ VALUE io_uring_backend_accept(Backend_t *backend, VALUE sock, int loop) {
   return Qnil;
 }
 
-VALUE Backend_accept(VALUE self, VALUE sock) {
+VALUE Backend_accept(VALUE self, VALUE server_socket, VALUE socket_class) {
   Backend_t *backend;
   GetBackend(self, backend);
-  return io_uring_backend_accept(backend, sock, 0);
+  return io_uring_backend_accept(backend, server_socket, socket_class, 0);
 }
 
-VALUE Backend_accept_loop(VALUE self, VALUE sock) {
+VALUE Backend_accept_loop(VALUE self, VALUE server_socket, VALUE socket_class) {
   Backend_t *backend;
   GetBackend(self, backend);
-  io_uring_backend_accept(backend, sock, 1);
+  io_uring_backend_accept(backend, server_socket, socket_class, 1);
   return self;
 }
 
@@ -945,9 +944,6 @@ VALUE Backend_kind(VALUE self) {
 }
 
 void Init_Backend() {
-  rb_require("socket");
-  cTCPSocket = rb_const_get(rb_cObject, rb_intern("TCPSocket"));
-
   VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cData);
   rb_define_alloc_func(cBackend, Backend_allocate);
 
@@ -968,8 +964,8 @@ void Init_Backend() {
   rb_define_method(cBackend, "recv", Backend_recv, 3);
   rb_define_method(cBackend, "recv_loop", Backend_recv_loop, 1);
   rb_define_method(cBackend, "send", Backend_send, 2);
-  rb_define_method(cBackend, "accept", Backend_accept, 1);
-  rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 1);
+  rb_define_method(cBackend, "accept", Backend_accept, 2);
+  rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
   rb_define_method(cBackend, "connect", Backend_connect, 3);
   rb_define_method(cBackend, "wait_io", Backend_wait_io, 2);
   rb_define_method(cBackend, "sleep", Backend_sleep, 1);

data/ext/polyphony/backend_libev.c

@@ -13,7 +13,6 @@
 #include "../libev/ev.h"
 #include "ruby/io.h"
 
-VALUE cTCPSocket;
 VALUE SYM_libev;
 
 ID ID_ivar_is_nonblocking;
@@ -494,7 +493,7 @@ VALUE Backend_write_m(int argc, VALUE *argv, VALUE self) {
     Backend_writev(self, argv[0], argc - 1, argv + 1);
 }
 
-VALUE Backend_accept(VALUE self, VALUE sock) {
+VALUE Backend_accept(VALUE self, VALUE server_socket, VALUE socket_class) {
   Backend_t *backend;
   struct libev_io watcher;
   rb_io_t *fptr;
@@ -502,12 +501,12 @@ VALUE Backend_accept(VALUE self, VALUE sock) {
   struct sockaddr addr;
   socklen_t len = (socklen_t)sizeof addr;
   VALUE switchpoint_result = Qnil;
-  VALUE underlying_sock = rb_ivar_get(sock, ID_ivar_io);
-  if (underlying_sock != Qnil) sock = underlying_sock;
+  VALUE underlying_sock = rb_ivar_get(server_socket, ID_ivar_io);
+  if (underlying_sock != Qnil) server_socket = underlying_sock;
 
   GetBackend(self, backend);
-  GetOpenFile(sock, fptr);
-  io_set_nonblock(fptr, sock);
+  GetOpenFile(server_socket, fptr);
+  io_set_nonblock(fptr, server_socket);
   watcher.fiber = Qnil;
   while (1) {
     fd = accept(fptr->fd, &addr, &len);
@@ -529,7 +528,7 @@ VALUE Backend_accept(VALUE self, VALUE sock) {
         goto error;
       }
 
-      socket = rb_obj_alloc(cTCPSocket);
+      socket = rb_obj_alloc(socket_class);
       MakeOpenFile(socket, fp);
       rb_update_max_fd(fd);
       fp->fd = fd;
@@ -550,7 +549,7 @@ error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
 
-VALUE Backend_accept_loop(VALUE self, VALUE sock) {
+VALUE Backend_accept_loop(VALUE self, VALUE server_socket, VALUE socket_class) {
   Backend_t *backend;
   struct libev_io watcher;
   rb_io_t *fptr;
@@ -559,12 +558,12 @@ VALUE Backend_accept_loop(VALUE self, VALUE sock) {
   socklen_t len = (socklen_t)sizeof addr;
   VALUE switchpoint_result = Qnil;
   VALUE socket = Qnil;
-  VALUE underlying_sock = rb_ivar_get(sock, ID_ivar_io);
-  if (underlying_sock != Qnil) sock = underlying_sock;
+  VALUE underlying_sock = rb_ivar_get(server_socket, ID_ivar_io);
+  if (underlying_sock != Qnil) server_socket = underlying_sock;
 
   GetBackend(self, backend);
-  GetOpenFile(sock, fptr);
-  io_set_nonblock(fptr, sock);
+  GetOpenFile(server_socket, fptr);
+  io_set_nonblock(fptr, server_socket);
   watcher.fiber = Qnil;
 
   while (1) {
@@ -586,7 +585,7 @@ VALUE Backend_accept_loop(VALUE self, VALUE sock) {
         goto error;
       }
 
-      socket = rb_obj_alloc(cTCPSocket);
+      socket = rb_obj_alloc(socket_class);
       MakeOpenFile(socket, fp);
       rb_update_max_fd(fd);
       fp->fd = fd;
@@ -849,9 +848,6 @@ VALUE Backend_kind(VALUE self) {
 void Init_Backend() {
   ev_set_allocator(xrealloc);
 
-  rb_require("socket");
-  cTCPSocket = rb_const_get(rb_cObject, rb_intern("TCPSocket"));
-
   VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cData);
   rb_define_alloc_func(cBackend, Backend_allocate);
 
@@ -869,8 +865,8 @@ void Init_Backend() {
   rb_define_method(cBackend, "read", Backend_read, 4);
   rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
   rb_define_method(cBackend, "write", Backend_write_m, -1);
-  rb_define_method(cBackend, "accept", Backend_accept, 1);
-  rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 1);
+  rb_define_method(cBackend, "accept", Backend_accept, 2);
+  rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
   rb_define_method(cBackend, "connect", Backend_connect, 3);
   rb_define_method(cBackend, "recv", Backend_recv, 3);
   rb_define_method(cBackend, "recv_loop", Backend_read_loop, 1);

data/lib/polyphony/core/global_api.rb

@@ -53,10 +53,10 @@ module Polyphony
       Fiber.current.spin(tag, caller, &block)
     end
 
-    def spin_loop(tag = nil, rate: nil, &block)
-      if rate
+    def spin_loop(tag = nil, rate: nil, interval: nil, &block)
+      if rate || interval
         Fiber.current.spin(tag, caller) do
-          throttled_loop(rate, &block)
+          throttled_loop(rate: rate, interval: interval, &block)
         end
       else
         Fiber.current.spin(tag, caller) { loop(&block) }
@@ -73,17 +73,8 @@ module Polyphony
       end.await
     end
 
-    def every(interval)
-      next_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + interval
-      loop do
-        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        Thread.current.backend.sleep(next_time - now)
-        yield
-        loop do
-          next_time += interval
-          break if next_time > now
-        end
-      end
+    def every(interval, &block)
+      Thread.current.backend.timer_loop(interval, &block)
     end
 
     def move_on_after(interval, with_value: nil, &block)

data/lib/polyphony/core/resource_pool.rb

@@ -58,18 +58,7 @@ module Polyphony
     # Discards the currently-acquired resource
     # instead of returning it to the pool when done.
     def discard!
-      if block_given?
-        @size.times do
-          acquire do |r|
-            next if yield(r)
-
-            @size -= 1
-            @acquired_resources.delete(Fiber.current)
-          end
-        end
-      else
-        @size -= 1 if @acquired_resources.delete(Fiber.current)
-      end
+      @size -= 1 if @acquired_resources.delete(Fiber.current)
     end
 
     def preheat!

data/lib/polyphony/extensions/fiber.rb

@@ -67,19 +67,29 @@ module Polyphony
         self << fiber unless result.is_a?(Exception)
       end
       loop { supervise_perform(opts) }
+    rescue Polyphony::MoveOn
+      # generated in #supervise_perform to stop supervisor
     ensure
       @on_child_done = nil
     end
 
     def supervise_perform(opts)
       fiber = receive
-      restart_fiber(fiber, opts) if fiber
+      if fiber && opts[:restart]
+        restart_fiber(fiber, opts)
+      elsif Fiber.current.children.empty?
+        Fiber.current.stop
+      end
     rescue Polyphony::Restart
       restart_all_children
     rescue Exception => e
       Kernel.raise e if e.source_fiber.nil? || e.source_fiber == self
 
-      restart_fiber(e.source_fiber, opts)
+      if opts[:restart]
+        restart_fiber(e.source_fiber, opts)
+      elsif Fiber.current.children.empty?
+        Fiber.current.stop
+      end
     end
 
     def restart_fiber(fiber, opts)
@@ -173,6 +183,7 @@ module Polyphony
     # signals (see also the patched `Kernel#trap`)
     def schedule_priority_oob_fiber(&block)
       f = Fiber.new do
+        Fiber.current.setup_raw
         block.call
       rescue Exception => e
         Thread.current.schedule_and_wakeup(Thread.main.main_fiber, e)
@@ -262,6 +273,7 @@ module Polyphony
     # allows the fiber to be scheduled and to receive messages.
     def setup_raw
       @thread = Thread.current
+      @running = true
     end
 
     def setup_main_fiber

data/lib/polyphony/extensions/socket.rb

@@ -8,7 +8,11 @@ require_relative '../core/thread_pool'
 # Socket overrides (eventually rewritten in C)
 class ::Socket
   def accept
-    Thread.current.backend.accept(self)
+    Thread.current.backend.accept(self, TCPSocket)
+  end
+
+  def accept_loop(&block)
+    Thread.current.backend.accept_loop(self, TCPSocket, &block)
   end
 
   NO_EXCEPTION = { exception: false }.freeze
@@ -19,17 +23,7 @@ class ::Socket
   end
 
   def recv(maxlen, flags = 0, outbuf = nil)
-    Thread.current.backend.recv(self, buf || +'', maxlen)
-    # outbuf ||= +''
-    # loop do
-    #   result = recv_nonblock(maxlen, flags, outbuf, **NO_EXCEPTION)
-    #   case result
-    #   when nil then raise IOError
-    #   when :wait_readable then Thread.current.backend.wait_io(self, false)
-    #   else
-    #     return result
-    #   end
-    # end
+    Thread.current.backend.recv(self, outbuf || +'', maxlen)
   end
 
   def recv_loop(&block)
@@ -144,19 +138,24 @@ class ::TCPSocket
   end
 
   def recv(maxlen, flags = 0, outbuf = nil)
-    Thread.current.backend.recv(self, buf || +'', maxlen)
+    Thread.current.backend.recv(self, outbuf || +'', maxlen)
   end
 
   def recv_loop(&block)
     Thread.current.backend.recv_loop(self, &block)
   end
+  alias_method :read_loop, :recv_loop
 
   def send(mesg, flags = 0)
     Thread.current.backend.send(self, mesg)
   end
 
-  def write(str)
-    Thread.current.backend.send(self, str)
+  def write(str, *args)
+    if args.empty?
+      Thread.current.backend.send(self, str)
+    else
+      Thread.current.backend.send(self, str + args.join)
+    end
   end
   alias_method :<<, :write
 
@@ -193,11 +192,12 @@ class ::TCPServer
 
   alias_method :orig_accept, :accept
   def accept
-    @io.accept
+    Thread.current.backend.accept(@io, TCPSocket)
+    # @io.accept
   end
 
   def accept_loop(&block)
-    Thread.current.backend.accept_loop(@io, &block)
+    Thread.current.backend.accept_loop(@io, TCPSocket, &block)
   end
 
   alias_method :orig_close, :close
@@ -205,3 +205,60 @@ class ::TCPServer
     @io.close
   end
 end
+
+class ::UNIXServer
+  alias_method :orig_accept, :accept
+ def accept
+    Thread.current.backend.accept(self, UNIXSocket)
+  end
+
+  def accept_loop(&block)
+    Thread.current.backend.accept_loop(self, UNIXSocket, &block)
+  end
+end
+
+class ::UNIXSocket
+  def recv(maxlen, flags = 0, outbuf = nil)
+    Thread.current.backend.recv(self, outbuf || +'', maxlen)
+  end
+
+  def recv_loop(&block)
+    Thread.current.backend.recv_loop(self, &block)
+  end
+  alias_method :read_loop, :recv_loop
+
+  def send(mesg, flags = 0)
+    Thread.current.backend.send(self, mesg)
+  end
+
+  def write(str, *args)
+    if args.empty?
+      Thread.current.backend.send(self, str)
+    else
+      Thread.current.backend.send(self, str + args.join)
+    end
+  end
+  alias_method :<<, :write
+
+  def readpartial(maxlen, str = nil)
+    @read_buffer ||= +''
+    result = Thread.current.backend.recv(self, @read_buffer, maxlen)
+    raise EOFError unless result
+
+    if str
+      str << @read_buffer
+    else
+      str = @read_buffer
+    end
+    @read_buffer = +''
+    str
+  end
+
+  def read_nonblock(len, str = nil, exception: true)
+    @io.read_nonblock(len, str, exception: exception)
+  end
+
+  def write_nonblock(buf, exception: true)
+    @io.write_nonblock(buf, exception: exception)
+  end
+end