polyphony 0.94 → 0.95

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02f56fa13c8397d86a0ceae20dd253461809397e8ad9eb5827a04650f3dc5020
-  data.tar.gz: f956d6e226e4a2b4239dc367abe9b4c1de89173df1b9968748f0da730b88410d
+  metadata.gz: 554854ea3a7b1bc8dd555b24283e42019bb7bfb91953cd0b3e6ac49c9c59198b
+  data.tar.gz: 59a88ec85837e68ed50594ee892c814dee9e15ba19b4392d50436fb0b51b1d82
 SHA512:
-  metadata.gz: 262b6272e6f2cdc86c09eca16b60e31713139ef37b7203c77e858d0c991a5754ee842311caac9ee76e09ce71a1f473e18f102f0ce460e903f31acb0317b6a0b5
-  data.tar.gz: d628e38d39ee4bbbeb1f5f9e7b5ba9d2e406911be1cb9af809a310f9cdb0315beb417e41462b68ed58e23e2c94435a5702192fd5946d5c51a9d4ae392ee50872
+  metadata.gz: 3637e9af046327dd642bb555224e4dd36169824a698b1176f57ebeb309e61d1ac6c50d2423c8ab2ac72abf9d8e9184f37ebf9ca8343032d7e40b3244d99f8ca7
+  data.tar.gz: 27fca71574396f73e783630691cbda094488ab19128c884ca2fdafd570b0df920f083e46750455a656c0f26c9acc6f11783634cb7d43b23f2689f000ee0c67ea

data/.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        ruby: ['2.7', '3.0', '3.1', 'head']
+        ruby: ['3.0', '3.1', '3.2']
 
     name: >-
       ${{matrix.os}}, ${{matrix.ruby}}
@@ -20,7 +20,7 @@ jobs:
 
     steps:
     - name: Setup machine
-      uses: actions/checkout@v1
+      uses: actions/checkout@v3
     - name: Setup Ruby
       uses: ruby/setup-ruby@v1
       with:

data/.gitignore

@@ -42,9 +42,9 @@ build-iPhoneSimulator/
 
 # for a library or gem, you might want to ignore these files since the code is
 # intended to run in multiple environments; otherwise, check them in:
-# Gemfile.lock
-# .ruby-version
-# .ruby-gemset
+Gemfile.lock
+.ruby-version
+.ruby-gemset
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.95 2023-01-16
+
+- Remove support for Ruby 2.7
+- Add support for explicit Enumerator (external enumeration) (#93)
+- Fix `Socket#readpartial`
+- Fix `receive_all_pending` docs (#92)
+- Improve linux kernel version detection
+- Update liburing
+- Always reset SQE user data in io_uring backend
+
+## 0.94 2022-10-03
+
+- Fix linux kernel version detection (#89)
+
 ## 0.93 2022-04-05
 
 - Add support for IO::Buffer in Ruby 3.1 (#80)

data/docs/api-reference/fiber.md

@@ -212,7 +212,7 @@ spin { Fiber.current.parent << 'hello from child' }
 message = receive #=> 'hello from child'
 ```
 
-### #receive_pending → [*object]
+### #receive_all_pending → [*object]
 
 Returns all messages currently in the mailbox, emptying the mailbox. This method
 does not block if no the mailbox is already empty. This method may be used to
@@ -225,7 +225,7 @@ worker = spin do
     handle_job(job)
   end
 rescue Polyphony::Terminate => e
-  receive_pending.each { |job| handle_job(job) }
+  receive_all_pending.each { |job| handle_job(job) }
 end
 ```
 

data/docs/api-reference/object.md

@@ -63,9 +63,9 @@ result #=> 'bar'
 
 Shortcut for `Fiber.current.receive`
 
-### #receive_pending → [*object]
+### #receive_all_pending → [*object]
 
-Shortcut for `Fiber.current.receive_pending`
+Shortcut for `Fiber.current.receive_all_pending`
 
 ### #sleep(duration = nil) → fiber
 
@@ -96,4 +96,4 @@ returns. Otherwise, the loop is infinite (unless an exception is raised).
 ```ruby
 # twice a second
 throttled_loop(2) { puts 'hello world' }
-```
+```

data/docs/main-concepts/exception-handling.md

@@ -229,7 +229,7 @@ def do_work
   end
 rescue Polyphony::Terminate
   # We still need to handle any pending request
-  receive_pending.each { handle_req(req) }
+  receive_all_pending.each { handle_req(req) }
 end
 
 # on the main fiber
@@ -288,4 +288,4 @@ loop.
 To ensure proper thread termination, including the termination of all the
 thread's fibers, Polyphony patches the `Thread#kill` and `Thread#raise` methods
 to schedule the thread's main fiber with the corresponding exceptions, thus
-ensuring an orderly termination or exception handling.
+ensuring an orderly termination or exception handling.

data/ext/polyphony/backend_common.c

@@ -280,7 +280,7 @@ inline void rectify_io_file_pos(rb_io_t *fptr) {
   }
 }
 
-inline double current_time() {
+inline double current_time(void) {
   struct timespec ts;
   double t;
   uint64_t ns;
@@ -292,7 +292,7 @@ inline double current_time() {
   return t / 1e9;
 }
 
-inline uint64_t current_time_ns() {
+inline uint64_t current_time_ns(void) {
   struct timespec ts;
   uint64_t ns;
 
@@ -439,7 +439,7 @@ VALUE Backend_verify_blocking_mode(VALUE self, VALUE io, VALUE blocking) {
   return self;
 }
 
-void backend_setup_stats_symbols() {
+void backend_setup_stats_symbols(void) {
   SYM_runqueue_size       = ID2SYM(rb_intern("runqueue_size"));
   SYM_runqueue_length     = ID2SYM(rb_intern("runqueue_length"));
   SYM_runqueue_max_length = ID2SYM(rb_intern("runqueue_max_length"));

data/ext/polyphony/backend_io_uring.c

@@ -286,6 +286,7 @@ VALUE Backend_wakeup(VALUE self) {
     // NOP which would cause the io_uring_enter syscall to return
     struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
     io_uring_prep_nop(sqe);
+    io_uring_sqe_set_data(sqe, NULL);
     io_uring_backend_immediate_submit(backend);
 
     return Qtrue;
@@ -304,10 +305,7 @@ int io_uring_backend_defer_submit_and_await(
   VALUE switchpoint_result = Qnil;
 
   backend->base.op_count++;
-  if (sqe) {
-    io_uring_sqe_set_data(sqe, ctx);
-    io_uring_sqe_set_flags(sqe, IOSQE_ASYNC);
-  }
+  if (sqe) io_uring_sqe_set_data(sqe, ctx);
   io_uring_backend_defer_submit(backend);
 
   switchpoint_result = backend_await((struct Backend_base *)backend);
@@ -318,7 +316,8 @@ int io_uring_backend_defer_submit_and_await(
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
     sqe = io_uring_backend_get_sqe(backend);
-    io_uring_prep_cancel(sqe, (__u64)ctx, 0);
+    io_uring_prep_cancel(sqe, ctx, 0);
+    io_uring_sqe_set_data(sqe, NULL);
     io_uring_backend_immediate_submit(backend);
   }
 
@@ -931,7 +930,6 @@ static inline op_context_t *prepare_double_splice_ctx(Backend_t *backend, int sr
   struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
   io_uring_prep_splice(sqe, src_fd, -1, dest_fd, -1, DOUBLE_SPLICE_MAXLEN, 0);
   io_uring_sqe_set_data(sqe, ctx);
-  io_uring_sqe_set_flags(sqe, IOSQE_ASYNC);
   backend->base.op_count += 1;
   backend->pending_sqes += 1;
 
@@ -941,7 +939,8 @@ static inline op_context_t *prepare_double_splice_ctx(Backend_t *backend, int sr
 static inline void io_uring_backend_cancel(Backend_t *backend, op_context_t *ctx) {
   struct io_uring_sqe *sqe = io_uring_backend_get_sqe(backend);
   ctx->result = -ECANCELED;
-  io_uring_prep_cancel(sqe, (__u64)ctx, 0);
+  io_uring_prep_cancel(sqe, ctx, 0);
+  io_uring_sqe_set_data(sqe, NULL);
 }
 
 VALUE double_splice_safe(struct double_splice_ctx *ctx) {
@@ -1212,7 +1211,8 @@ VALUE Backend_timeout_ensure(VALUE arg) {
     timeout_ctx->ctx->result = -ECANCELED;
     // op was not completed, so we need to cancel it
     sqe = io_uring_get_sqe(&timeout_ctx->backend->ring);
-    io_uring_prep_cancel(sqe, (__u64)timeout_ctx->ctx, 0);
+    io_uring_prep_cancel(sqe, timeout_ctx->ctx, 0);
+    io_uring_sqe_set_data(sqe, NULL);
     io_uring_backend_immediate_submit(timeout_ctx->backend);
   }
   context_store_release(&timeout_ctx->backend->store, timeout_ctx->ctx);
@@ -1330,7 +1330,8 @@ VALUE Backend_wait_event(VALUE self, VALUE raise) {
     // last fiber to use the eventfd, so we cancel the ongoing poll
     struct io_uring_sqe *sqe;
     sqe = io_uring_backend_get_sqe(backend);
-    io_uring_prep_cancel(sqe, (__u64)backend->event_fd_ctx, 0);
+    io_uring_prep_cancel(sqe, backend->event_fd_ctx, 0);
+    io_uring_sqe_set_data(sqe, NULL);
     io_uring_backend_immediate_submit(backend);
     backend->event_fd_ctx = NULL;
   }
@@ -1429,14 +1430,13 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
 
       if (sqe_count) {
         struct io_uring_sqe *sqe;
-        
         io_uring_sqe_set_data(last_sqe, ctx);
-        io_uring_sqe_set_flags(last_sqe, IOSQE_ASYNC);
 
         ctx->ref_count = sqe_count;
         ctx->result = -ECANCELED;
         sqe = io_uring_backend_get_sqe(backend);
-        io_uring_prep_cancel(sqe, (__u64)ctx, 0);
+        io_uring_prep_cancel(sqe, ctx, 0);
+        io_uring_sqe_set_data(sqe, NULL);
         io_uring_backend_immediate_submit(backend);
       }
       else {
@@ -1447,7 +1447,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
     }
 
     io_uring_sqe_set_data(last_sqe, ctx);
-    flags = (i == (argc - 1)) ? IOSQE_ASYNC : IOSQE_ASYNC | IOSQE_IO_LINK;
+    flags = (i == (argc - 1)) ? 0 : IOSQE_IO_LINK;
     io_uring_sqe_set_flags(last_sqe, flags);
     sqe_count++;
   }
@@ -1466,7 +1466,8 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
     sqe = io_uring_backend_get_sqe(backend);
-    io_uring_prep_cancel(sqe, (__u64)ctx, 0);
+    io_uring_prep_cancel(sqe, ctx, 0);
+    io_uring_sqe_set_data(sqe, NULL);
     io_uring_backend_immediate_submit(backend);
     RAISE_IF_EXCEPTION(resume_value);
     return resume_value;
@@ -1532,7 +1533,8 @@ static inline void splice_chunks_cancel(Backend_t *backend, op_context_t *ctx) {
   
   ctx->result = -ECANCELED;
   sqe = io_uring_backend_get_sqe(backend);
-  io_uring_prep_cancel(sqe, (__u64)ctx, 0);
+  io_uring_prep_cancel(sqe, ctx, 0);
+  io_uring_sqe_set_data(sqe, NULL);
   io_uring_backend_immediate_submit(backend);
 }
 
@@ -1699,7 +1701,7 @@ void Backend_unpark_fiber(VALUE self, VALUE fiber) {
   backend_base_unpark_fiber(&backend->base, fiber);
 }
 
-void Init_Backend() {
+void Init_Backend(void) {
   VALUE cBackend = rb_define_class_under(mPolyphony, "Backend", rb_cObject);
   rb_define_alloc_func(cBackend, Backend_allocate);
 

data/ext/polyphony/event.c

@@ -79,7 +79,7 @@ VALUE Event_await(VALUE self) {
   return switchpoint_result;
 }
 
-void Init_Event() {
+void Init_Event(void) {
   cEvent = rb_define_class_under(mPolyphony, "Event", rb_cObject);
   rb_define_alloc_func(cEvent, Event_allocate);
 

data/ext/polyphony/extconf.rb

@@ -15,11 +15,13 @@ def get_config
   raise "Could not parse Linux kernel information (#{kernel_info.inspect})" if !m
 
   version, major_revision, distribution = m[1].to_i, m[2].to_i, m[4]
-  config[:pidfd_open] = (version == 5) && (major_revision >= 3)
+
+  combined_version = version.to_i * 100 + major_revision.to_i
+
+  config[:pidfd_open] = combined_version > 503
 
   force_libev = ENV['POLYPHONY_LIBEV'] != nil
-  config[:io_uring] = !force_libev &&
-    (version == 5) && (major_revision >= 6) && (distribution != 'linuxkit')
+  config[:io_uring] = !force_libev && (combined_version >= 506) && (distribution != 'linuxkit')
   config
 end
 

data/ext/polyphony/fiber.c

@@ -38,24 +38,16 @@ inline VALUE Fiber_auto_watcher(VALUE self) {
   return watcher;
 }
 
-void Fiber_make_runnable(VALUE fiber, VALUE value) {
+inline void Fiber_make_runnable(VALUE fiber, VALUE value) {
   VALUE thread = rb_ivar_get(fiber, ID_ivar_thread);
-  if (thread == Qnil) {
-    rb_raise(rb_eRuntimeError, "No thread set for fiber");
-    // rb_warn("No thread set for fiber");
-    return;
-  }
+  if (thread == Qnil) rb_raise(rb_eRuntimeError, "No thread set for fiber");
 
   Thread_schedule_fiber(thread, fiber, value);
 }
 
-void Fiber_make_runnable_with_priority(VALUE fiber, VALUE value) {
+inline void Fiber_make_runnable_with_priority(VALUE fiber, VALUE value) {
   VALUE thread = rb_ivar_get(fiber, ID_ivar_thread);
-  if (thread == Qnil) {
-    rb_raise(rb_eRuntimeError, "No thread set for fiber");
-    // rb_warn("No thread set for fiber");
-    return;
-  }
+  if (thread == Qnil) rb_raise(rb_eRuntimeError, "No thread set for fiber");
 
   Thread_schedule_fiber_with_priority(thread, fiber, value);
 }
@@ -130,7 +122,7 @@ VALUE Fiber_parked_p(VALUE self) {
   return rb_ivar_get(self, ID_ivar_parked);
 }
 
-void Init_Fiber() {
+void Init_Fiber(void) {
   VALUE cFiber = rb_const_get(rb_cObject, rb_intern("Fiber"));
   rb_define_method(cFiber, "safe_transfer", Fiber_safe_transfer, -1);
   rb_define_method(cFiber, "schedule", Fiber_schedule, -1);

data/ext/polyphony/io_extensions.c

@@ -631,7 +631,7 @@ VALUE IO_http1_splice_chunked(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
   return self;
 }
 
-void Init_IOExtensions() {
+void Init_IOExtensions(void) {
   rb_define_singleton_method(rb_cIO, "gzip", IO_gzip, -1);
   rb_define_singleton_method(rb_cIO, "gunzip", IO_gunzip, -1);
   rb_define_singleton_method(rb_cIO, "deflate", IO_deflate, 2);

data/ext/polyphony/pipe.c

@@ -96,7 +96,7 @@ VALUE Pipe_fds(VALUE self) {
   return rb_ary_new_from_args(2, INT2FIX(pipe->fds[0]), INT2FIX(pipe->fds[1]));
 }
 
-void Init_Pipe() {
+void Init_Pipe(void) {
   cPipe = rb_define_class_under(mPolyphony, "Pipe", rb_cObject);
   cClosedPipeError = rb_define_class_under(cPipe, "ClosedPipeError", rb_eRuntimeError);
 

data/ext/polyphony/polyphony.c

@@ -170,7 +170,7 @@ VALUE Polyphony_raw_buffer_size(VALUE self, VALUE buffer) {
 //   return Backend_close(BACKEND(), io);
 // }
 
-void Init_Polyphony() {
+void Init_Polyphony(void) {
   mPolyphony = rb_define_module("Polyphony");
 
   // backend methods

data/ext/polyphony/polyphony_ext.c

@@ -15,7 +15,7 @@ void Init_SocketExtensions();
 extern void playground();
 #endif
 
-void Init_polyphony_ext() {
+void Init_polyphony_ext(void) {
   Init_Polyphony();
 
   Init_Backend();

data/ext/polyphony/queue.c

@@ -315,7 +315,7 @@ end:
   return self;
 }
 
-void Init_Queue() {
+void Init_Queue(void) {
   cClosedQueueError = rb_const_get(rb_cObject, rb_intern("ClosedQueueError"));
   cThreadError = rb_const_get(rb_cObject, rb_intern("ThreadError"));
 

data/ext/polyphony/ring_buffer.c

@@ -10,6 +10,7 @@ void ring_buffer_init(ring_buffer *buffer) {
 }
 
 void ring_buffer_free(ring_buffer *buffer) {
+  // printf("ring_buffer_free ring_buffer: %p entries: %p\n", buffer, buffer->entries);
   free(buffer->entries);
 }
 

data/ext/polyphony/socket_extensions.c

@@ -16,7 +16,7 @@ VALUE Socket_double_chevron(VALUE self, VALUE msg) {
   return self;
 }
 
-void Init_SocketExtensions() {
+void Init_SocketExtensions(void) {
   VALUE cSocket;
   VALUE cTCPSocket;
 

data/ext/polyphony/thread.c

@@ -60,7 +60,7 @@ VALUE Thread_class_backend(VALUE _self) {
   return rb_ivar_get(rb_thread_current(), ID_ivar_backend);
 }
 
-void Init_Thread() {
+void Init_Thread(void) {
   rb_define_method(rb_cThread, "setup_fiber_scheduling", Thread_setup_fiber_scheduling, 0);
   rb_define_method(rb_cThread, "schedule_and_wakeup", Thread_fiber_schedule_and_wakeup, 2);
   rb_define_method(rb_cThread, "switch_fiber", Thread_switch_fiber, 0);

data/lib/polyphony/extensions/enumerator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# Extensions to the Enumerator class
+class ::Enumerator
+  alias_method :orig_next, :next
+  def next
+    Fiber.current.thread ||= Thread.current
+    orig_next
+  end
+
+  alias_method :orig_each, :each
+  def each(*a, &b)
+    Fiber.current.thread ||= Thread.current
+    orig_each(*a, &b)
+  end
+end

data/lib/polyphony/extensions/socket.rb

@@ -194,6 +194,8 @@ class ::Socket
   def readpartial(maxlen, buf = +'', buf_pos = 0, raise_on_eof = true)
     result = Polyphony.backend_recv(self, buf, maxlen, buf_pos)
     raise EOFError if !result && raise_on_eof
+
+    result
   end
 
   ZERO_LINGER = [0, 0].pack('ii').freeze

data/lib/polyphony/extensions.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative './extensions/enumerator'
 require_relative './extensions/exception'
 require_relative './extensions/fiber'
 require_relative './extensions/io'

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.94'
+  VERSION = '0.95'
 end

data/polyphony.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.extra_rdoc_files = ["README.md"]
   s.extensions = ["ext/polyphony/extconf.rb"]
   s.require_paths = ["lib"]
-  s.required_ruby_version = '>= 2.6'
+  s.required_ruby_version = '>= 3.0'
 
   s.add_development_dependency  'rake-compiler',        '1.1.1'
   s.add_development_dependency  'minitest',             '5.14.4'
@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency  'pry',                  '0.13.1'
 
   s.add_development_dependency  'msgpack',              '1.4.2'
-  s.add_development_dependency  'httparty',             '0.17.1'
+  s.add_development_dependency  'httparty',             '0.21.0'
   s.add_development_dependency  'localhost',            '~>1.1.4'
 
   # s.add_development_dependency  'jekyll',               '~>3.8.6'

data/test/test_backend.rb

@@ -144,8 +144,12 @@ class BackendTest < MiniTest::Test
       buf << :done
     end
 
-    # writing always causes snoozing
+    # Caution: we want to read in two chunks, that a race condition lurks here:
+    # writing always causes snoozing, but that might not be enough for the
+    # reader fiber to perform the read before the second write, so for the test
+    # to work consistently we add a little sleep between the two writes.
     o << 'foo'
+    sleep 0.03
     o << 'bar'
     o.close
 

data/test/test_enumerator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+
+class EnumeratorTest < MiniTest::Test
+  def test_each_enumerator
+    o = [1, 2, 3]
+    e = o.each
+
+    r = []
+    loop do
+      r << e.next
+    rescue StopIteration
+      break
+    end
+
+    assert_equal o, r
+  end
+
+  def test_custom_io_enumerator
+    i, o = IO.pipe
+
+    spin do
+      10.times { o.puts 'foo' }
+      o.close
+    end
+
+    e_fiber = nil
+    e = Enumerator.new do |y|
+      e_fiber ||= Fiber.current
+      while (l = i.gets)
+        y << l
+      end
+    end
+
+    r = []
+    loop do
+      r << e.next
+    rescue StopIteration
+      break
+    end
+
+    assert_equal ["foo\n"] * 10, r
+    assert Fiber.current != e_fiber
+  end
+end