polyphony 0.50.1 → 0.51.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 126431d6f1b6c547d57dd99834c5a2371b4f2b3399df5556f55fdf7ff8465702
-  data.tar.gz: bf4e8d8cbde0a0d25930996f21ca723ffb68c065688ea2423f96b2a1e8946a80
+  metadata.gz: 72f11d867863c51f1fc1ab3ec85fe575cca9ee53a5047f4d3f7b76cea3b650b7
+  data.tar.gz: c29354b2de2f207185fc06fd86421b53d328aaf67d611109a6f95e9130ac5967
 SHA512:
-  metadata.gz: 9b6cb2b64ac16e70941195aa17c019abdb026bac2d440550841f839bae96d248b87609979e6e9d7c57f9905845209714d90f74caa4488d8ba66afa96c35aa425
-  data.tar.gz: '093360cb9c3f5e81d7d23f1e707bf6399608475822b56d0171438748e65752adb2784b1ab36ce850a26840c615f73ce1fcc74ca842018361019269f7d3c31aa9'
+  metadata.gz: 93351a3c4007145ff317257f82b764708b445e94b848d31f783f4d72071b097a8309322c4bedfdb7f321b0613ef436a314ad4e650928d0855143d3a5b98d9c13
+  data.tar.gz: 9119039548264867fa012bcbb90999f4ad82170c28539d43b0145687a761d508630da538ac87b75f9bf32f6b33cb0e13c22f50143248b85cc90289d5d3451d3f

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.51.0
+
+- Implement `IO#feed_loop`, `Socket#feed_loop`
+- Fix error handling in `Process.kill_and_await`
+
 ## 0.50.1
 
 - Set `IOSQE_ASYNC` flag in io_uring backend

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.50.1)
+    polyphony (0.51.0)
 
 GEM
   remote: https://rubygems.org/
@@ -72,6 +72,7 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
+    msgpack (1.4.2)
     multi_xml (0.6.0)
     mysql2 (0.5.3)
     parallel (1.19.1)
@@ -136,6 +137,7 @@ DEPENDENCIES
   just-the-docs (~> 0.3.0)
   minitest (= 5.13.0)
   minitest-reporters (= 1.4.2)
+  msgpack (= 1.4.2)
   mysql2 (= 0.5.3)
   pg (= 1.1.4)
   polyphony!

data/ext/polyphony/backend_common.h

@@ -107,6 +107,13 @@ inline VALUE backend_snooze() {
   READ_LOOP_PREPARE_STR(); \
 }
 
+#define READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id) { \
+  io_set_read_length(str, total, shrinkable); \
+  io_enc_str(str, fptr); \
+  rb_funcall_passing_block(receiver, method_id, 1, &str); \
+  READ_LOOP_PREPARE_STR(); \
+}
+
 inline void rectify_io_file_pos(rb_io_t *fptr) {
   // Apparently after reopening a closed file, the file position is not reset,
   // which causes the read to fail. Fortunately we can use fptr->rbuf.len to

data/ext/polyphony/backend_io_uring.c

@@ -389,6 +389,52 @@ VALUE Backend_read_loop(VALUE self, VALUE io) {
   return io;
 }
 
+VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
+  Backend_t *backend;
+  rb_io_t *fptr;
+  VALUE str;
+  long total;
+  long len = 8192;
+  int shrinkable;
+  char *buf;
+  VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
+  ID method_id = SYM2ID(method);
+
+  READ_LOOP_PREPARE_STR();
+
+  GetBackend(self, backend);
+  if (underlying_io != Qnil) io = underlying_io;
+  GetOpenFile(io, fptr);
+  rb_io_check_byte_readable(fptr);
+  rectify_io_file_pos(fptr);
+
+  while (1) {
+    VALUE resume_value = Qnil;
+    op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_READ);
+    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    io_uring_prep_read(sqe, fptr->fd, buf, len, -1);
+    
+    ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
+    OP_CONTEXT_RELEASE(&backend->store, ctx);
+    RAISE_IF_EXCEPTION(resume_value);
+    if (!ctx->completed) return resume_value;
+    RB_GC_GUARD(resume_value);
+
+    if (result < 0)
+      rb_syserr_fail(-result, strerror(-result));
+    else if (!result)
+      break; // EOF
+    else {
+      total = result;
+      READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
+    }
+  }
+
+  RB_GC_GUARD(str);
+
+  return io;
+}
+
 VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
   Backend_t *backend;
   rb_io_t *fptr;
@@ -596,6 +642,51 @@ VALUE Backend_recv_loop(VALUE self, VALUE io) {
   return io;
 }
 
+VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
+  Backend_t *backend;
+  rb_io_t *fptr;
+  VALUE str;
+  long total;
+  long len = 8192;
+  int shrinkable;
+  char *buf;
+  VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
+  ID method_id = SYM2ID(method);
+
+  READ_LOOP_PREPARE_STR();
+
+  GetBackend(self, backend);
+  if (underlying_io != Qnil) io = underlying_io;
+  GetOpenFile(io, fptr);
+  rb_io_check_byte_readable(fptr);
+  rectify_io_file_pos(fptr);
+
+  while (1) {
+    VALUE resume_value = Qnil;
+    op_context_t *ctx = OP_CONTEXT_ACQUIRE(&backend->store, OP_RECV);
+    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    io_uring_prep_recv(sqe, fptr->fd, buf, len, 0);
+    
+    int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
+    OP_CONTEXT_RELEASE(&backend->store, ctx);
+    RAISE_IF_EXCEPTION(resume_value);
+    if (!ctx->completed) return resume_value;
+    RB_GC_GUARD(resume_value);
+
+    if (result < 0)
+      rb_syserr_fail(-result, strerror(-result));
+    else if (!result)
+      break; // EOF
+    else {
+      total = result;
+      READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
+    }
+  }
+
+  RB_GC_GUARD(str);
+  return io;
+}
+
 VALUE Backend_send(VALUE self, VALUE io, VALUE str) {
   Backend_t *backend;
   rb_io_t *fptr;
@@ -918,9 +1009,11 @@ void Init_Backend() {
 
   rb_define_method(cBackend, "read", Backend_read, 4);
   rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
+  rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
   rb_define_method(cBackend, "write", Backend_write_m, -1);
   rb_define_method(cBackend, "recv", Backend_recv, 3);
   rb_define_method(cBackend, "recv_loop", Backend_recv_loop, 1);
+  rb_define_method(cBackend, "recv_feed_loop", Backend_recv_feed_loop, 3);
   rb_define_method(cBackend, "send", Backend_send, 2);
   rb_define_method(cBackend, "accept", Backend_accept, 2);
   rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);

data/ext/polyphony/backend_libev.c

@@ -334,6 +334,58 @@ error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
 
+VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
+  Backend_t *backend;
+  struct libev_io watcher;
+  rb_io_t *fptr;
+  VALUE str;
+  long total;
+  long len = 8192;
+  int shrinkable;
+  char *buf;
+  VALUE switchpoint_result = Qnil;
+  VALUE underlying_io = rb_ivar_get(io, ID_ivar_io);
+  ID method_id = SYM2ID(method);
+
+  READ_LOOP_PREPARE_STR();
+
+  GetBackend(self, backend);
+  if (underlying_io != Qnil) io = underlying_io;
+  GetOpenFile(io, fptr);
+  rb_io_check_byte_readable(fptr);
+  io_set_nonblock(fptr, io);
+  rectify_io_file_pos(fptr);
+  watcher.fiber = Qnil;
+
+  while (1) {
+    ssize_t n = read(fptr->fd, buf, len);
+    if (n < 0) {
+      int e = errno;
+      if ((e != EWOULDBLOCK && e != EAGAIN)) rb_syserr_fail(e, strerror(e));
+
+      switchpoint_result = libev_wait_fd_with_watcher(backend, fptr->fd, &watcher, EV_READ);
+      if (TEST_EXCEPTION(switchpoint_result)) goto error;
+    }
+    else {
+      switchpoint_result = backend_snooze();
+
+      if (TEST_EXCEPTION(switchpoint_result)) goto error;
+
+      if (n == 0) break; // EOF
+      total = n;
+      READ_LOOP_PASS_STR_TO_RECEIVER(receiver, method_id);
+    }
+  }
+
+  RB_GC_GUARD(str);
+  RB_GC_GUARD(watcher.fiber);
+  RB_GC_GUARD(switchpoint_result);
+
+  return io;
+error:
+  return RAISE_EXCEPTION(switchpoint_result);
+}
+
 VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
   Backend_t *backend;
   struct libev_io watcher;
@@ -811,12 +863,14 @@ void Init_Backend() {
 
   rb_define_method(cBackend, "read", Backend_read, 4);
   rb_define_method(cBackend, "read_loop", Backend_read_loop, 1);
+  rb_define_method(cBackend, "feed_loop", Backend_feed_loop, 3);
   rb_define_method(cBackend, "write", Backend_write_m, -1);
   rb_define_method(cBackend, "accept", Backend_accept, 2);
   rb_define_method(cBackend, "accept_loop", Backend_accept_loop, 2);
   rb_define_method(cBackend, "connect", Backend_connect, 3);
   rb_define_method(cBackend, "recv", Backend_recv, 3);
   rb_define_method(cBackend, "recv_loop", Backend_read_loop, 1);
+  rb_define_method(cBackend, "recv_feed_loop", Backend_feed_loop, 3);
   rb_define_method(cBackend, "send", Backend_write, 2);
   rb_define_method(cBackend, "wait_io", Backend_wait_io, 2);
   rb_define_method(cBackend, "sleep", Backend_sleep, 1);

data/lib/polyphony/adapters/process.rb

@@ -24,8 +24,8 @@ module Polyphony
       def kill_and_await(sig, pid)
         ::Process.kill(sig, pid)
         Thread.current.backend.waitpid(pid)
-      rescue Errno::ERSCH
-        # ignore
+      rescue Errno::ESRCH
+        # process doesn't exist
       end
     end
   end

data/lib/polyphony/extensions/io.rb

@@ -220,6 +220,10 @@ class ::IO
     Thread.current.backend.read_loop(self, &block)
   end
 
+  def feed_loop(receiver, method, &block)
+    Thread.current.backend.feed_loop(self, receiver, method, &block)
+  end
+
   # alias_method :orig_read, :read
   # def read(length = nil, outbuf = nil)
   #   if length

data/lib/polyphony/extensions/socket.rb

@@ -31,6 +31,10 @@ class ::Socket
   end
   alias_method :read_loop, :recv_loop
 
+  def feed_loop(receiver, method, &block)
+    Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
+  end
+
   def recvfrom(maxlen, flags = 0)
     @read_buffer ||= +''
     while true
@@ -146,6 +150,10 @@ class ::TCPSocket
   end
   alias_method :read_loop, :recv_loop
 
+  def feed_loop(receiver, method, &block)
+    Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
+  end
+
   def send(mesg, flags = 0)
     Thread.current.backend.send(self, mesg)
   end
@@ -227,6 +235,10 @@ class ::UNIXSocket
   end
   alias_method :read_loop, :recv_loop
 
+  def feed_loop(receiver, method, &block)
+    Thread.current.backend.recv_feed_loop(self, receiver, method, &block)
+  end
+
   def send(mesg, flags = 0)
     Thread.current.backend.send(self, mesg)
   end

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.50.1'
+  VERSION = '0.51.0'
 end

data/polyphony.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency  'rubocop',              '0.85.1'
   s.add_development_dependency  'pry',                  '0.13.1'
   
+  s.add_development_dependency  'msgpack',              '1.4.2'
   s.add_development_dependency  'pg',                   '1.1.4'
   s.add_development_dependency  'redis',                '4.1.0'
   s.add_development_dependency  'hiredis',              '0.6.3'

data/test/test_io.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'helper'
+require 'msgpack'
 
 class IOTest < MiniTest::Test
   def setup
@@ -110,7 +111,7 @@ class IOTest < MiniTest::Test
     assert_equal [], buf
     
     o << "ulous\n"
-    15.times { snooze }
+    sleep 0.01
     assert_equal ["fabulous\n"], buf
 
     o.close
@@ -173,6 +174,57 @@ class IOTest < MiniTest::Test
 
     assert_equal 'hello: world', buf
   end
+
+  def test_feed_loop_with_block
+    i, o = IO.pipe
+    unpacker = MessagePack::Unpacker.new
+    buffer = []
+    reader = spin do
+      i.feed_loop(unpacker, :feed_each) { |msg| buffer << msg }
+    end
+    o << 'foo'.to_msgpack
+    sleep 0.01
+    assert_equal ['foo'], buffer
+
+    o << 'bar'.to_msgpack
+    sleep 0.01
+    assert_equal ['foo', 'bar'], buffer
+
+    o << 'baz'.to_msgpack
+    sleep 0.01
+    assert_equal ['foo', 'bar', 'baz'], buffer
+  end
+
+  class Receiver
+    attr_reader :buffer
+
+    def initialize
+      @buffer = []
+    end
+
+    def recv(obj)
+      @buffer << obj
+    end
+  end
+
+  def test_feed_loop_without_block
+    i, o = IO.pipe
+    receiver = Receiver.new
+    reader = spin do
+      i.feed_loop(receiver, :recv)
+    end
+    o << 'foo'
+    sleep 0.01
+    assert_equal ['foo'], receiver.buffer
+
+    o << 'bar'
+    sleep 0.01
+    assert_equal ['foo', 'bar'], receiver.buffer
+
+    o << 'baz'
+    sleep 0.01
+    assert_equal ['foo', 'bar', 'baz'], receiver.buffer
+  end
 end
 
 class IOClassMethodsTest < MiniTest::Test

data/test/test_socket.rb

@@ -2,6 +2,7 @@
 
 require_relative 'helper'
 require 'fileutils'
+require 'msgpack'
 
 class SocketTest < MiniTest::Test
   def setup
@@ -33,6 +34,39 @@ class SocketTest < MiniTest::Test
     server&.close
   end
 
+  def test_feed_loop
+    port = rand(1234..5678)
+    server = TCPServer.new('127.0.0.1', port)
+
+    server_fiber = spin do
+      reader = MessagePack::Unpacker.new
+      while (socket = server.accept)
+        spin do
+          socket.feed_loop(reader, :feed_each) do |msg|
+            msg = { 'result' => msg['x'] + msg['y'] }
+            socket << msg.to_msgpack
+          end
+        end
+      end
+    end
+
+    snooze
+    client = TCPSocket.new('127.0.0.1', port)
+    reader = MessagePack::Unpacker.new
+    client << { 'x' => 13, 'y' => 14 }.to_msgpack
+    result = nil
+    client.feed_loop(reader, :feed_each) do |msg|
+      result = msg
+      break
+    end
+    assert_equal({ 'result' => 27}, result)
+    client.close
+  ensure
+    server_fiber&.stop
+    server_fiber&.await
+    server&.close
+  end
+
   def test_unix_socket
     path = '/tmp/test_unix_socket'
     FileUtils.rm(path) rescue nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: 0.50.1
+  version: 0.51.0
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-31 00:00:00.000000000 Z
+date: 2021-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -94,6 +94,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.13.1
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.2
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement