polyphony 0.64 → 0.68

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e3938b66b15caff258b95c7f1421f71bc0bb4f7b94dd9db4a3ad0614fd35ded
-  data.tar.gz: 2198ea0483f959491372074007d5c32416de69ebcf73a8aca51832ea6036ea87
+  metadata.gz: d23a9c0f60530fdee1d367f34359e4f694bd4a42ba3c0112e61433df05c29a81
+  data.tar.gz: bf75e3f1a633f5c94a43fd45a2de2545d87b1e751753980abb8e1d3eddd4eade
 SHA512:
-  metadata.gz: 318b3b2549a6d7d3c2bb1b8396adcf95109ef17f6235e25bcf1540fd3c5777d6f2666858d5f539cf17dd0c2102553eb853ba3b38b96d9476d2727a7ddfb0ed70
-  data.tar.gz: 025f5037c922531235bcf56076ca6986cb1d983b37eceda8e1585fa90e23f67c0abe17a81b0329851e80c050e9ccce8ba53f7a0d95565e6568546dbb923f9a25
+  metadata.gz: b810c7ddbd383039e5f34fe76dbea089807417d4415e79a952f6d6a46538cf99c55ae7a34f3eadf61d22af23ac29a0910c62ea5c97dff6e86b94083d98444edf
+  data.tar.gz: '063459edd8f669a024e0b7b8f1daa47983db4cf9ac7f3ca81a69359de92eace4831d843e9b74645db08a50c26d83a2878a5fdb6b125adc260afd13a3135e710c'

data/.github/workflows/test.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-latest, ubuntu-18.04, macos-10.15]
         ruby: [2.6, 2.7, 3.0]
 
     name: >-

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.68 2021-08-13
+
+- Fix missing default value in socket classes' `#readpartial`
+- Fix linking of operations in `Backend#chain` (io_uring version)
+- Rename `Fiber#attach` to `Fiber#attach_to`
+- Expose original `SSLServer#accept`
+
+## 0.67 2021-08-06
+
+- Improve fiber monitoring
+- Add fiber parking (a parked fiber is prevented from running). This is in
+  preparation for the upcoming work on an integrated debugger.
+
+## 0.66 2021-08-01
+
+- Fix all splicing APIs on non-linux OSes (#63)
+- Add GC marking of buffers when cancelling read/write ops in io_uring backend 
+
+## 0.65 2021-07-29
+
+- Add `#__polyphony_read_method__` method for read method detection
+
 ## 0.64 2021-07-26
 
 - Add optional raise_on_eof argument to `#readpartial`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.64)
+    polyphony (0.68)
 
 GEM
   remote: https://rubygems.org/

data/TODO.md

@@ -1,3 +1,13 @@
+- io_uring backend:
+  - if `io_uring_get_sqe` returns null, call `io_uring_submit`, (snooze fiber)?
+    and try again
+
+- Tracing:
+  - Emit events on I/O ops, e.g.:
+    - [:op_read_submit, id, io, len]
+    - [:op_read_complete, id, io, len, buffer]
+  - Prevent tracing while an event is being emitted (to allow the trace proc to perform I/O)
+
 - Add support for IPv6:
   https://www.reddit.com/r/ruby/comments/lyen23/understanding_ipv6_and_why_its_important_to_you/
 
@@ -14,47 +24,8 @@
   - `IO#gets_loop`, `Socket#gets_loop`, `OpenSSL::Socket#gets_loop` (medium effort)
   - `Fiber#receive_loop` (very little effort, should be implemented in C)
 
-
-- Add `Backend#splice`, `Backend#splice_to_eof` for implementing stuff like proxying:
-
-  ```ruby
-  def two_way_proxy(socket1, socket2)
-    backend = Thread.current.backend
-    f1 = spin { backend.splice_to_eof(socket1, socket2) }
-    f2 = spin { backend.splice_to_eof(socket2, socket1) }
-    Fiber.await(f1, f2)
-  end
-  ```
-
 - Add support for `close` to io_uring backend
 
-- Add support for submission of multiple requests to io_uring backend:
-
-  ```ruby
-  Thread.current.backend.submit(
-    [:send, sock, chunk_header(len)],
-    [:splice, file, sock, len]
-  )
-  ```
-
-  Full example (for writing chunks from a file to an HTTP response):
-
-  ```ruby
-  def serve_io(io)
-    i, o = IO.pipe
-    backend = Thread.current.backend
-    while true
-      len = o.splice(io, 8192)
-      break if len == 0
-      
-      backend.submit(
-        [:write, sock, chunk_header(len)],
-        [:splice, i, sock, len]
-      )
-    end
-  end
-  ```
-
 - Graceful shutdown again:
   - What happens to children when doing a graceful shutdown?
   - What are the implications of passing graceful shutdown flag to children?
@@ -92,7 +63,6 @@
 
 -----------------------------------------------------
 
-- Add `Backend#splice(in, out, nbytes)` API
 - Adapter for io/console (what does `IO#raw` do?)
 - Adapter for Pry and IRB (Which fixes #5 and #6)
 - allow backend selection at runtime

data/bin/pdbg

@@ -0,0 +1,112 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+UNIX_SOCKET_PATH = '/tmp/pdbg.sock'
+
+cmd = ARGV.join(' ')
+injected_lib_path = File.expand_path('../lib/polyphony/debugger/server_inject.rb', __dir__)
+pid = fork { exec("env POLYPHONY_DEBUG_SOCKET_PATH=#{UNIX_SOCKET_PATH} ruby #{cmd}") }
+puts "Started debugged process (#{pid})"
+
+socket = nil
+while !socket
+  socket = UNIXSocket.new(UNIX_SOCKET_PATH) rescue nil
+end
+
+def parse_command(cmd)
+  case cmd
+  when /^(step|s)$/
+    { cmd: :step }
+  when /^(state|st)$/
+    { cmd: :state }
+  when /^(help|h)$/
+    { cmd: :help }
+  when /^(list|l)$/
+    { cmd: :list }
+  else
+    nil
+  end
+end
+
+def display_info(info)
+  info = eval(info)
+  case (info && info[:kind])
+  when :listing
+    print_listing(info)
+  when :state
+    print_state(info)
+  else
+    p info
+  end
+rescue SyntaxError
+  puts "Failed to eval:"
+  p info
+end
+
+FILE_LINES_CACHE = {}
+
+def self.get_snippet(path, lineno)
+  lines = FILE_LINES_CACHE[path] ||= IO.read(path).lines
+  start_idx = lineno - 5
+  stop_idx = lineno + 3
+  stop_idx = lines.size - 1 if stop_idx >= lines.size
+  start_idx = 0 if start_idx < 0
+  (start_idx..stop_idx).map { |idx| [idx + 1, lines[idx]]}
+end
+
+def print_snippet(info, snippet, cur_line)
+  places = FILE_LINES_CACHE[info[:path]].size.to_s.size
+  snippet.each do |(lineno, line)|
+    is_cur = lineno == cur_line
+    formatted = format("%s% #{places}d %s", is_cur ? '=> ' : '   ', lineno, line)
+    puts formatted
+  end
+end
+
+def print_listing(info)
+  snippet = get_snippet(info[:path], info[:lineno])
+  puts "Fiber: #{info[:fiber]} Location: #{info[:path]}:#{info[:lineno]}"
+  puts
+  print_snippet(info, snippet, info[:lineno])
+  puts
+end
+
+def print_help
+  puts
+  puts "Here's some help..."
+  puts
+end
+
+def print_state(info)
+  p info
+end
+
+def get_user_cmd
+  while true
+    STDOUT << "(pdbg) "
+    cmd = parse_command(STDIN.gets)
+    next unless cmd
+    
+    if cmd[:cmd] == :help
+      print_help
+    else
+      return cmd if cmd
+    end
+  end
+end
+
+socket.puts 'pdbg'
+response = socket.gets
+if response.chomp == 'pdbg'
+  puts 'Connected to process'
+end
+loop do
+  info = socket.gets.chomp
+  display_info(info)
+
+  cmd = get_user_cmd
+  socket.puts cmd.inspect
+end

data/examples/core/await.rb

@@ -2,7 +2,11 @@
 
 require 'bundler/setup'
 require 'polyphony'
+require 'polyphony/extensions/debug'
 
+Exception.__disable_sanitized_backtrace__ = true
+
+puts '----- start await example ------'
 sleeper = spin do
   puts 'going to sleep'
   sleep 1
@@ -17,4 +21,8 @@ waiter = spin do
   puts 'done waiting'
 end
 
-waiter.await
+trace :before_await
+
+sleep 2
+waiter.await
+trace :after_await

data/ext/polyphony/backend_common.c

@@ -7,6 +7,7 @@
 
 inline void backend_base_initialize(struct Backend_base *base) {
   runqueue_initialize(&base->runqueue);
+  runqueue_initialize(&base->parked_runqueue);
   base->currently_polling = 0;
   base->op_count = 0;
   base->switch_count = 0;
@@ -20,12 +21,14 @@ inline void backend_base_initialize(struct Backend_base *base) {
 
 inline void backend_base_finalize(struct Backend_base *base) {
   runqueue_finalize(&base->runqueue);
+  runqueue_finalize(&base->parked_runqueue);
 }
 
 inline void backend_base_mark(struct Backend_base *base) {
   if (base->idle_proc != Qnil) rb_gc_mark(base->idle_proc);
   if (base->trace_proc != Qnil) rb_gc_mark(base->trace_proc);
   runqueue_mark(&base->runqueue);
+  runqueue_mark(&base->parked_runqueue);
 }
 
 const unsigned int ANTI_STARVE_SWITCH_COUNT_THRESHOLD = 64;
@@ -91,7 +94,10 @@ void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_bas
 
   COND_TRACE(base, 4, SYM_fiber_schedule, fiber, value, prioritize ? Qtrue : Qfalse);
 
-  (prioritize ? runqueue_unshift : runqueue_push)(&base->runqueue, fiber, value, already_runnable);
+  runqueue_t *runqueue = rb_ivar_get(fiber, ID_ivar_parked) == Qtrue ? 
+    &base->parked_runqueue : &base->runqueue;
+
+  (prioritize ? runqueue_unshift : runqueue_push)(runqueue, fiber, value, already_runnable);
   if (!already_runnable) {
     rb_ivar_set(fiber, ID_ivar_runnable, Qtrue);
     if (rb_thread_current() != thread) {
@@ -105,6 +111,13 @@ void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_bas
   }
 }
 
+inline void backend_base_park_fiber(struct Backend_base *base, VALUE fiber) {
+  runqueue_migrate(&base->runqueue, &base->parked_runqueue, fiber);
+}
+
+inline void backend_base_unpark_fiber(struct Backend_base *base, VALUE fiber) {
+  runqueue_migrate(&base->parked_runqueue, &base->runqueue, fiber);
+}
 
 inline void backend_trace(struct Backend_base *base, int argc, VALUE *argv) {
   if (base->trace_proc == Qnil) return;

data/ext/polyphony/backend_common.h

@@ -17,6 +17,7 @@ struct backend_stats {
 
 struct Backend_base {
   runqueue_t runqueue;
+  runqueue_t parked_runqueue;
   unsigned int currently_polling;
   unsigned int op_count;
   unsigned int switch_count;
@@ -33,6 +34,8 @@ void backend_base_finalize(struct Backend_base *base);
 void backend_base_mark(struct Backend_base *base);
 VALUE backend_base_switch_fiber(VALUE backend, struct Backend_base *base);
 void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_base *base, VALUE fiber, VALUE value, int prioritize);
+void backend_base_park_fiber(struct Backend_base *base, VALUE fiber);
+void backend_base_unpark_fiber(struct Backend_base *base, VALUE fiber);
 void backend_trace(struct Backend_base *base, int argc, VALUE *argv);
 struct backend_stats backend_base_stats(struct Backend_base *base);
 
@@ -104,7 +107,6 @@ VALUE Backend_sendv(VALUE self, VALUE io, VALUE ary, VALUE flags);
 VALUE Backend_stats(VALUE self);
 void backend_run_idle_tasks(struct Backend_base *base);
 void io_verify_blocking_mode(rb_io_t *fptr, VALUE io, VALUE blocking);
-
 void backend_setup_stats_symbols();
 
 #endif /* BACKEND_COMMON_H */

data/ext/polyphony/backend_io_uring.c

@@ -25,6 +25,8 @@ VALUE SYM_send;
 VALUE SYM_splice;
 VALUE SYM_write;
 
+VALUE eArgumentError;
+
 #ifdef POLYPHONY_UNSET_NONBLOCK
 #define io_unset_nonblock(fptr, io) io_verify_blocking_mode(fptr, io, Qtrue)
 #else
@@ -45,6 +47,7 @@ typedef struct Backend_t {
 static void Backend_mark(void *ptr) {
   Backend_t *backend = ptr;
   backend_base_mark(&backend->base);
+  context_store_mark_taken_buffers(&backend->store);
 }
 
 static void Backend_free(void *ptr) {
@@ -226,7 +229,7 @@ inline void Backend_unschedule_fiber(VALUE self, VALUE fiber) {
   Backend_t *backend;
   GetBackend(self, backend);
 
-  runqueue_delete(&backend->base.runqueue, fiber);  
+  runqueue_delete(&backend->base.runqueue, fiber);
 }
 
 inline VALUE Backend_switch_fiber(VALUE self) {
@@ -349,8 +352,11 @@ VALUE Backend_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof,
 
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -410,8 +416,11 @@ VALUE Backend_read_loop(VALUE self, VALUE io, VALUE maxlen) {
     
     ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -457,8 +466,11 @@ VALUE Backend_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method) {
     
     ssize_t result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -500,8 +512,11 @@ VALUE Backend_write(VALUE self, VALUE io, VALUE str) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
     
     if (result < 0)
@@ -549,12 +564,10 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    if (TEST_EXCEPTION(resume_value)) {
-      free(iov);
-      RAISE_EXCEPTION(resume_value);
-    }
     if (!completed) {
       free(iov);
+      context_attach_buffers(ctx, argc, argv);
+      RAISE_IF_EXCEPTION(resume_value);
       return resume_value;
     }
     RB_GC_GUARD(resume_value);
@@ -588,8 +601,7 @@ VALUE Backend_writev(VALUE self, VALUE io, int argc, VALUE *argv) {
 
 VALUE Backend_write_m(int argc, VALUE *argv, VALUE self) {
   if (argc < 2)
-    // TODO: raise ArgumentError
-    rb_raise(rb_eRuntimeError, "(wrong number of arguments (expected 2 or more))");
+    rb_raise(eArgumentError, "(wrong number of arguments (expected 2 or more))");
 
   return (argc == 2) ?
     Backend_write(self, argv[0], argv[1]) :
@@ -628,8 +640,11 @@ VALUE Backend_recv(VALUE self, VALUE io, VALUE str, VALUE length, VALUE pos) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -675,8 +690,11 @@ VALUE Backend_recv_loop(VALUE self, VALUE io, VALUE maxlen) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -721,8 +739,11 @@ VALUE Backend_recv_feed_loop(VALUE self, VALUE io, VALUE receiver, VALUE method)
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -764,8 +785,11 @@ VALUE Backend_send(VALUE self, VALUE io, VALUE str, VALUE flags) {
     
     int result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
     int completed = context_store_release(&backend->store, ctx);
-    RAISE_IF_EXCEPTION(resume_value);
-    if (!completed) return resume_value;
+    if (!completed) {
+      context_attach_buffers(ctx, 1, &str);
+      RAISE_IF_EXCEPTION(resume_value);
+      return resume_value;
+    }
     RB_GC_GUARD(resume_value);
 
     if (result < 0)
@@ -1165,6 +1189,24 @@ struct io_uring_sqe *Backend_chain_prepare_splice(Backend_t *backend, VALUE src,
   return sqe;
 }
 
+void Backend_chain_ctx_attach_buffers(op_context_t *ctx, int argc, VALUE *argv) {
+  int count = 0;
+  if (argc > 1) ctx->buffers = malloc(sizeof(VALUE) * (argc - 1));
+
+  for (int i = 0; i < argc; i++) {
+    VALUE op = argv[i];
+    VALUE op_type = RARRAY_AREF(op, 0);
+
+    if (op_type == SYM_write || op_type == SYM_send) {
+      if (!count) ctx->buffer0 = RARRAY_AREF(op, 2);
+      else        ctx->buffers[count - 1] = RARRAY_AREF(op, 2);
+      count++;
+    }
+  }
+  ctx->buffer_count = count;
+}
+
+
 VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
   VALUE resume_value = Qnil;
   unsigned int sqe_count = 0;
@@ -1206,7 +1248,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
     }
     
     io_uring_sqe_set_data(last_sqe, ctx);
-    unsigned int flags = (i == argc - 1) ? IOSQE_ASYNC : IOSQE_ASYNC & IOSQE_IO_LINK;
+    unsigned int flags = (i == (argc - 1)) ? IOSQE_ASYNC : IOSQE_ASYNC | IOSQE_IO_LINK;
     io_uring_sqe_set_flags(last_sqe, flags);
     sqe_count++;
   }
@@ -1218,6 +1260,8 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
   int result = ctx->result;
   int completed = context_store_release(&backend->store, ctx);
   if (!completed) {
+    Backend_chain_ctx_attach_buffers(ctx, argc, argv);
+    
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
     struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
@@ -1409,6 +1453,7 @@ syscallerror:
   if (pipefd[1] != -1) close(pipefd[1]);
   rb_syserr_fail(err, strerror(err));
 error:
+  context_attach_buffers_v(ctx, 4, prefix, postfix, chunk_prefix, chunk_postfix);
   if (pipefd[0] != -1) close(pipefd[0]);
   if (pipefd[1] != -1) close(pipefd[1]);
   return RAISE_EXCEPTION(switchpoint_result);
@@ -1429,6 +1474,20 @@ VALUE Backend_trace_proc_set(VALUE self, VALUE block) {
   return self;
 }
 
+void Backend_park_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend_base_park_fiber(&backend->base, fiber);
+}
+
+void Backend_unpark_fiber(VALUE self, VALUE fiber) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  backend_base_unpark_fiber(&backend->base, fiber);
+}
+
 void Init_Backend() {
   VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cObject);
   rb_define_alloc_func(cBackend, Backend_allocate);
@@ -1469,7 +1528,6 @@ void Init_Backend() {
   rb_define_method(cBackend, "waitpid", Backend_waitpid, 1);
   rb_define_method(cBackend, "write", Backend_write_m, -1);
 
-
   #ifdef POLYPHONY_UNSET_NONBLOCK
   ID_ivar_is_nonblocking = rb_intern("@is_nonblocking");
   #endif
@@ -1480,6 +1538,8 @@ void Init_Backend() {
   SYM_write = ID2SYM(rb_intern("write"));
 
   backend_setup_stats_symbols();
+
+  eArgumentError = rb_const_get(rb_cObject, rb_intern("ArgumentError"));
 }
 
 #endif // POLYPHONY_BACKEND_LIBURING