polyphony 0.83 → 0.84

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1e0c71b9d91e674e4ce004da08cdf6535f56e48b063b63538685eb781654df6
-  data.tar.gz: 58bf12edd0751ecc552ee01b6eb7042585061f7800f0dfa6fdc0990b87349982
+  metadata.gz: ce9742f5b50581c5569df4ad3e2232c7c32392a92e7ab23960373c8a4e3b0de4
+  data.tar.gz: a695f3191cc7cefb7d720565c3564b24784beb836528342e8d4fd9e8d0231dda
 SHA512:
-  metadata.gz: c35e1c2756556db5bf11012b95c7e729d7ad43f52802c61fa5467d44a6460a2f30f840bc945347e83ed24d35e559b0ca4ac9d5657c9d0c805ffb44774765f867
-  data.tar.gz: 37c34a7c2cc0b3443c0f33908f40156c223eae233ca08f564a606a7bec5c8c68189b973e6fa40fd1534bb3c9f2f3985b52fcdd33c7ea9069ba21e32e7ebc1760
+  metadata.gz: 041cb949e600ef328ed6d1c92badcaa9ef502e4776043d6a7786a811d5935de450f933b8eff4dd6100db6e55ff19878186bd9ef6bb840ac1cdcdb8c53341f51c
+  data.tar.gz: fc0542606c3ee5ab044cc5e8175a60fb3fa6a8466422643ff1446fb0271a6db09236b5c0dbc7a632fa47853ae0ac3a2007fa4644a85283b4225f453f6155dbce

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/liburing"]
+	path = vendor/liburing
+	url = https://github.com/axboe/liburing

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.84 2022-03-11
+
+- Implement `IO.tee` (#82)
+- Implement `IO#tee_from` (#81)
+- Bundle liburing as git submodule
+
 ## 0.83 2022-03-10
 
 - Implement `Polyphony::Pipe` class, `Polyphony.pipe` method (#83)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.83)
+    polyphony (0.84)
 
 GEM
   remote: https://rubygems.org/

data/Rakefile

@@ -24,3 +24,17 @@ task :docs do
 end
 
 CLEAN.include "**/*.o", "**/*.so", "**/*.so.*", "**/*.a", "**/*.bundle", "**/*.jar", "pkg", "tmp"
+
+task :release do
+  require_relative './lib/polyphony/version'
+  version = Polyphony::VERSION
+  
+  puts 'Building polyphony...'
+  `gem build polyphony.gemspec`
+
+  puts "Pushing polyphony #{version}..."
+  `gem push extralite-#{version}.gem`
+
+  puts "Cleaning up..."
+  `rm *.gem`
+end

data/ext/polyphony/backend_io_uring.c

@@ -14,7 +14,7 @@
 #include <errno.h>
 
 #include "polyphony.h"
-#include "../liburing/liburing.h"
+#include "liburing.h"
 #include "backend_io_uring_context.h"
 #include "ruby/thread.h"
 #include "ruby/io.h"
@@ -304,7 +304,7 @@ int io_uring_backend_defer_submit_and_await(
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
     sqe = io_uring_get_sqe(&backend->ring);
-    io_uring_prep_cancel(sqe, ctx, 0);
+    io_uring_prep_cancel(sqe, (__u64)ctx, 0);
     backend->pending_sqes = 0;
     io_uring_submit(&backend->ring);
   }
@@ -971,6 +971,41 @@ VALUE Backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE chunksize)
   return io_uring_backend_splice(backend, src, dest, chunksize, 1);
 }
 
+VALUE Backend_tee(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
+  Backend_t *backend;
+  GetBackend(self, backend);
+
+  int src_fd;
+  int dest_fd;
+  rb_io_t *src_fptr;
+  rb_io_t *dest_fptr;
+  VALUE resume_value = Qnil;
+
+  src_fd = fd_from_io(src, &src_fptr, 0, 0);
+  dest_fd = fd_from_io(dest, &dest_fptr, 1, 0);
+
+  while (1) {
+    op_context_t *ctx = context_store_acquire(&backend->store, OP_SPLICE);
+    struct io_uring_sqe *sqe = io_uring_get_sqe(&backend->ring);
+    int result;
+    int completed;
+
+    io_uring_prep_tee(sqe, src_fd, dest_fd, NUM2INT(maxlen), 0);
+
+    result = io_uring_backend_defer_submit_and_await(backend, sqe, ctx, &resume_value);
+    completed = context_store_release(&backend->store, ctx);
+    RAISE_IF_EXCEPTION(resume_value);
+    if (!completed) return resume_value;
+
+    if (result < 0)
+      rb_syserr_fail(-result, strerror(-result));
+
+    return INT2NUM(result);
+  }
+
+  RB_GC_GUARD(resume_value);
+}
+
 VALUE Backend_connect(VALUE self, VALUE sock, VALUE host, VALUE port) {
   Backend_t *backend;
   int fd;
@@ -1125,7 +1160,7 @@ VALUE Backend_timeout_ensure(VALUE arg) {
     timeout_ctx->ctx->result = -ECANCELED;
     // op was not completed, so we need to cancel it
     sqe = io_uring_get_sqe(&timeout_ctx->backend->ring);
-    io_uring_prep_cancel(sqe, timeout_ctx->ctx, 0);
+    io_uring_prep_cancel(sqe, (__u64)timeout_ctx->ctx, 0);
     timeout_ctx->backend->pending_sqes = 0;
     io_uring_submit(&timeout_ctx->backend->ring);
   }
@@ -1318,7 +1353,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
         ctx->ref_count = sqe_count;
         ctx->result = -ECANCELED;
         sqe = io_uring_get_sqe(&backend->ring);
-        io_uring_prep_cancel(sqe, ctx, 0);
+        io_uring_prep_cancel(sqe, (__u64)ctx, 0);
         backend->pending_sqes = 0;
         io_uring_submit(&backend->ring);
       }
@@ -1349,7 +1384,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
     // op was not completed (an exception was raised), so we need to cancel it
     ctx->result = -ECANCELED;
     sqe = io_uring_get_sqe(&backend->ring);
-    io_uring_prep_cancel(sqe, ctx, 0);
+    io_uring_prep_cancel(sqe, (__u64)ctx, 0);
     backend->pending_sqes = 0;
     io_uring_submit(&backend->ring);
     RAISE_IF_EXCEPTION(resume_value);
@@ -1416,7 +1451,7 @@ static inline void splice_chunks_cancel(Backend_t *backend, op_context_t *ctx) {
   
   ctx->result = -ECANCELED;
   sqe = io_uring_get_sqe(&backend->ring);
-  io_uring_prep_cancel(sqe, ctx, 0);
+  io_uring_prep_cancel(sqe, (__u64)ctx, 0);
   backend->pending_sqes = 0;
   io_uring_submit(&backend->ring);
 }
@@ -1619,6 +1654,7 @@ void Init_Backend() {
   rb_define_method(cBackend, "sleep", Backend_sleep, 1);
   rb_define_method(cBackend, "splice", Backend_splice, 3);
   rb_define_method(cBackend, "splice_to_eof", Backend_splice_to_eof, 3);
+  rb_define_method(cBackend, "tee", Backend_tee, 3);
   rb_define_method(cBackend, "timeout", Backend_timeout, -1);
   rb_define_method(cBackend, "timer_loop", Backend_timer_loop, 1);
   rb_define_method(cBackend, "wait_event", Backend_wait_event, 1);

data/ext/polyphony/backend_libev.c

@@ -940,7 +940,52 @@ VALUE Backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
 error:
   return RAISE_EXCEPTION(switchpoint_result);
 }
+
+VALUE Backend_tee(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
+  Backend_t *backend;
+  struct libev_rw_io watcher;
+  VALUE switchpoint_result = Qnil;
+  int src_fd;
+  int dest_fd;
+  rb_io_t *src_fptr;
+  rb_io_t *dest_fptr;
+  int len;
+
+  GetBackend(self, backend);
+  src_fd = fd_from_io(src, &src_fptr, 0, 0);
+  dest_fd = fd_from_io(dest, &dest_fptr, 1, 0);
+  watcher.ctx.fiber = Qnil;
+
+  while (1) {
+    backend->base.op_count++;
+    len = tee(src_fd, dest_fd, NUM2INT(maxlen), 0);
+    if (len < 0) {
+      int e = errno;
+      if ((e != EWOULDBLOCK && e != EAGAIN)) rb_syserr_fail(e, strerror(e));
+
+      switchpoint_result = libev_wait_rw_fd_with_watcher(backend, src_fd, dest_fd, &watcher);
+      if (TEST_EXCEPTION(switchpoint_result)) goto error;
+    }
+    else {
+      break;
+    }
+  }
+
+  if (watcher.ctx.fiber == Qnil) {
+    switchpoint_result = backend_snooze(&backend->base);
+    if (TEST_EXCEPTION(switchpoint_result)) goto error;
+  }
+
+  RB_GC_GUARD(watcher.ctx.fiber);
+  RB_GC_GUARD(switchpoint_result);
+
+  return INT2NUM(len);
+error:
+  return RAISE_EXCEPTION(switchpoint_result);
+}
+
 #else
+
 VALUE Backend_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen) {
   Backend_t *backend;
   struct libev_io watcher;

data/ext/polyphony/extconf.rb

@@ -28,6 +28,22 @@ puts "Building Polyphony... (#{config.inspect})"
 
 require_relative 'zlib_conf'
 
+if config[:io_uring]
+  liburing_path = File.expand_path('../../vendor/liburing', __dir__)
+  FileUtils.cd liburing_path do
+    system('./configure', exception: true)
+    FileUtils.cd File.join(liburing_path, 'src') do
+      system('make', 'liburing.a', exception: true)
+    end
+  end
+
+  if !find_header 'liburing.h', File.expand_path('../../vendor/liburing/src/include', __dir__)
+    raise "Couldn't find liburing.h"
+  end
+
+  $LDFLAGS << " -L#{File.expand_path('../../vendor/liburing/src', __dir__)} -l uring"
+end
+
 $defs << '-DPOLYPHONY_USE_PIDFD_OPEN' if config[:pidfd_open]
 if config[:io_uring]
   $defs << "-DPOLYPHONY_BACKEND_LIBURING"
@@ -56,4 +72,5 @@ CONFIG['optflags'] << ' -fno-strict-aliasing' unless RUBY_PLATFORM =~ /mswin/
 
 have_func('rb_fiber_transfer', 'ruby.h')
 
+
 create_makefile 'polyphony_ext'

data/ext/polyphony/polyphony.c

@@ -94,6 +94,10 @@ VALUE Polyphony_backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE c
   return Backend_splice_to_eof(BACKEND(), src, dest, chunksize);
 }
 
+VALUE Polyphony_backend_tee(VALUE self, VALUE src, VALUE dest, VALUE chunksize) {
+  return Backend_tee(BACKEND(), src, dest, chunksize);
+}
+
 VALUE Polyphony_backend_timeout(int argc,VALUE *argv, VALUE self) {
   return Backend_timeout(argc, argv, BACKEND());
 }
@@ -186,6 +190,7 @@ void Init_Polyphony() {
   rb_define_singleton_method(mPolyphony, "backend_sleep", Polyphony_backend_sleep, 1);
   rb_define_singleton_method(mPolyphony, "backend_splice", Polyphony_backend_splice, 3);
   rb_define_singleton_method(mPolyphony, "backend_splice_to_eof", Polyphony_backend_splice_to_eof, 3);
+  rb_define_singleton_method(mPolyphony, "backend_tee", Polyphony_backend_tee, 3);
   rb_define_singleton_method(mPolyphony, "backend_timeout", Polyphony_backend_timeout, -1);
   rb_define_singleton_method(mPolyphony, "backend_timer_loop", Polyphony_backend_timer_loop, 1);
   rb_define_singleton_method(mPolyphony, "backend_wait_event", Polyphony_backend_wait_event, 1);

data/ext/polyphony/polyphony.h

@@ -112,6 +112,7 @@ VALUE Backend_sendv(VALUE self, VALUE io, VALUE ary, VALUE flags);
 VALUE Backend_sleep(VALUE self, VALUE duration);
 VALUE Backend_splice(VALUE self, VALUE src, VALUE dest, VALUE maxlen);
 VALUE Backend_splice_to_eof(VALUE self, VALUE src, VALUE dest, VALUE chunksize);
+VALUE Backend_tee(VALUE self, VALUE src, VALUE dest, VALUE maxlen);
 VALUE Backend_timeout(int argc,VALUE *argv, VALUE self);
 VALUE Backend_timer_loop(VALUE self, VALUE interval);
 VALUE Backend_wait_event(VALUE self, VALUE raise);

data/lib/polyphony/extensions/io.rb

@@ -81,6 +81,12 @@ class ::IO
     def splice_to_eof(src, dest, chunk_size = 8192)
       Polyphony.backend_splice_to_eof(src, dest, chunk_size)
     end
+
+    if RUBY_PLATFORM =~ /linux/
+      def tee(src, dest, maxlen)
+        Polyphony.backend_tee(src, dest, maxlen)
+      end
+    end
   end
 end
 
@@ -272,4 +278,8 @@ class ::IO
   def splice_to_eof_from(src, chunksize = 8192)
     Polyphony.backend_splice_to_eof(src, self, chunksize)
   end
+
+  def tee_from(src, maxlen)
+    Polyphony.backend_tee(src, self, maxlen)
+  end
 end

data/lib/polyphony/extensions/pipe.rb

@@ -164,4 +164,8 @@ class Polyphony::Pipe
   def splice_to_eof_from(src, chunksize = 8192)
     Polyphony.backend_splice_to_eof(src, self, chunksize)
   end
+
+  def tee_from(src, maxlen)
+    Polyphony.backend_tee(src, self, maxlen)
+  end
 end

data/lib/polyphony/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyphony
-  VERSION = '0.83'
+  VERSION = '0.84'
 end

data/test/test_io.rb

@@ -367,6 +367,65 @@ class IOTest < MiniTest::Test
       f.await
     end
   end
+
+  def test_tee_from
+    skip "tested only on Linux" unless RUBY_PLATFORM =~ /linux/
+    
+    src = Polyphony.pipe
+    dest1 = Polyphony.pipe
+    dest2 = Polyphony.pipe
+
+    len1 = len2 = nil
+
+    spin {
+      len1 = dest1.tee_from(src, 1000)
+      dest1.close
+      len2 = IO.splice(src, dest2, 1000)
+      dest2.close
+    }
+
+    src << 'foobar'
+    src.close
+    result1 = dest1.read
+    result2 = dest2.read
+
+    assert_equal 'foobar', result1
+    assert_equal 6, len1
+
+    assert_equal 'foobar', result2
+    assert_equal 6, len2
+  end
+
+  def test_tee_class_method
+    skip "tested only on Linux" unless RUBY_PLATFORM =~ /linux/
+    
+    src = Polyphony.pipe
+    dest1 = Polyphony.pipe
+    dest2 = Polyphony.pipe
+
+    len1 = len2 = nil
+
+    spin {
+      len1 = IO.tee(src, dest1, 1000)
+      dest1.close
+      len2 = IO.splice(src, dest2, 1000)
+      dest2.close
+    }
+
+    src << 'foobar'
+    src.close
+    result1 = dest1.read
+    result2 = dest2.read
+
+    assert_equal 'foobar', result1
+    assert_equal 6, len1
+
+    assert_equal 'foobar', result2
+    assert_equal 6, len2
+  end
+
+
+
 end
 
 class IOWithRawBufferTest < MiniTest::Test
@@ -643,7 +702,7 @@ class IOExtensionsTest < MiniTest::Test
     gz = Zlib::GzipReader.new(dest)
     data = gz.read
     assert_equal IO.read(__FILE__), data
-    assert_in_range (now-1)..(now+1), gz.mtime
+    assert_in_range (now-2)..(now+1), gz.mtime
     assert_nil gz.orig_name
     assert_nil gz.comment
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polyphony
 version: !ruby/object:Gem::Version
-  version: '0.83'
+  version: '0.84'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-10 00:00:00.000000000 Z
+date: 2022-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -315,16 +315,6 @@ files:
 - ext/libev/ev_win32.c
 - ext/libev/ev_wrap.h
 - ext/libev/test_libev_win32.c
-- ext/liburing/liburing.h
-- ext/liburing/liburing/README.md
-- ext/liburing/liburing/barrier.h
-- ext/liburing/liburing/compat.h
-- ext/liburing/liburing/io_uring.h
-- ext/liburing/queue.c
-- ext/liburing/register.c
-- ext/liburing/setup.c
-- ext/liburing/syscall.c
-- ext/liburing/syscall.h
 - ext/polyphony/backend_common.c
 - ext/polyphony/backend_common.h
 - ext/polyphony/backend_io_uring.c
@@ -337,7 +327,6 @@ files:
 - ext/polyphony/io_extensions.c
 - ext/polyphony/libev.c
 - ext/polyphony/libev.h
-- ext/polyphony/liburing.c
 - ext/polyphony/pipe.c
 - ext/polyphony/playground.c
 - ext/polyphony/polyphony.c

data/ext/liburing/liburing/README.md

@@ -1,4 +0,0 @@
-## Updating the liburing source code
-
-- copy liburing/src/**/* into ext/liburing
-- move ext/liburing/include/**/* to ext/liburing/

data/ext/liburing/liburing/barrier.h

@@ -1,73 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-#ifndef LIBURING_BARRIER_H
-#define LIBURING_BARRIER_H
-
-/*
-From the kernel documentation file refcount-vs-atomic.rst:
-
-A RELEASE memory ordering guarantees that all prior loads and
-stores (all po-earlier instructions) on the same CPU are completed
-before the operation. It also guarantees that all po-earlier
-stores on the same CPU and all propagated stores from other CPUs
-must propagate to all other CPUs before the release operation
-(A-cumulative property). This is implemented using
-:c:func:`smp_store_release`.
-
-An ACQUIRE memory ordering guarantees that all post loads and
-stores (all po-later instructions) on the same CPU are
-completed after the acquire operation. It also guarantees that all
-po-later stores on the same CPU must propagate to all other CPUs
-after the acquire operation executes. This is implemented using
-:c:func:`smp_acquire__after_ctrl_dep`.
-*/
-
-#ifdef __cplusplus
-#include <atomic>
-
-template <typename T>
-static inline void IO_URING_WRITE_ONCE(T &var, T val)
-{
-	std::atomic_store_explicit(reinterpret_cast<std::atomic<T> *>(&var),
-				   val, std::memory_order_relaxed);
-}
-template <typename T>
-static inline T IO_URING_READ_ONCE(const T &var)
-{
-	return std::atomic_load_explicit(
-		reinterpret_cast<const std::atomic<T> *>(&var),
-		std::memory_order_relaxed);
-}
-
-template <typename T>
-static inline void io_uring_smp_store_release(T *p, T v)
-{
-	std::atomic_store_explicit(reinterpret_cast<std::atomic<T> *>(p), v,
-				   std::memory_order_release);
-}
-
-template <typename T>
-static inline T io_uring_smp_load_acquire(const T *p)
-{
-	return std::atomic_load_explicit(
-		reinterpret_cast<const std::atomic<T> *>(p),
-		std::memory_order_acquire);
-}
-#else
-#include <stdatomic.h>
-
-#define IO_URING_WRITE_ONCE(var, val)				\
-	atomic_store_explicit((_Atomic typeof(var) *)&(var),	\
-			      (val), memory_order_relaxed)
-#define IO_URING_READ_ONCE(var)					\
-	atomic_load_explicit((_Atomic typeof(var) *)&(var),	\
-			     memory_order_relaxed)
-
-#define io_uring_smp_store_release(p, v)			\
-	atomic_store_explicit((_Atomic typeof(*(p)) *)(p), (v), \
-			      memory_order_release)
-#define io_uring_smp_load_acquire(p)				\
-	atomic_load_explicit((_Atomic typeof(*(p)) *)(p),	\
-			     memory_order_acquire)
-#endif
-
-#endif /* defined(LIBURING_BARRIER_H) */

data/ext/liburing/liburing/compat.h

@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-#ifndef LIBURING_COMPAT_H
-#define LIBURING_COMPAT_H
-
-#include <linux/time_types.h>
-
-#include <inttypes.h>
-
-struct open_how {
-	uint64_t	flags;
-	uint64_t	mode;
-	uint64_t	resolve;
-};
-
-#endif