sleepy_penguin 3.4.1 → 3.5.0

data/ext/sleepy_penguin/splice.c

@@ -0,0 +1,174 @@
+#include "sleepy_penguin.h"
+#include "sp_copy.h"
+#ifdef HAVE_SPLICE
+#include <errno.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+static VALUE sym_EAGAIN;
+
+#ifndef F_LINUX_SPECIFIC_BASE
+#  define F_LINUX_SPECIFIC_BASE 1024
+#endif
+
+#ifndef F_GETPIPE_SZ
+#  define F_SETPIPE_SZ    (F_LINUX_SPECIFIC_BASE + 7)
+#  define F_GETPIPE_SZ    (F_LINUX_SPECIFIC_BASE + 8)
+#endif
+
+static int check_fileno(VALUE io)
+{
+	int saved_errno = errno;
+	int fd = rb_sp_fileno(io);
+	errno = saved_errno;
+	return fd;
+}
+
+static void *nogvl_splice(void *ptr)
+{
+	struct copy_args *a = ptr;
+
+	return (void *)splice(a->fd_in, a->off_in, a->fd_out, a->off_out,
+	                     a->len, a->flags);
+}
+
+/* :nodoc: */
+static VALUE my_splice(VALUE mod, VALUE io_in, VALUE off_in,
+			VALUE io_out, VALUE off_out,
+			VALUE len, VALUE flags)
+{
+	off_t i = 0, o = 0;
+	struct copy_args a;
+	ssize_t bytes;
+
+	a.off_in = NIL_P(off_in) ? NULL : (i = NUM2OFFT(off_in), &i);
+	a.off_out = NIL_P(off_out) ? NULL : (o = NUM2OFFT(off_out), &o);
+	a.len = NUM2SIZET(len);
+	a.flags = NUM2UINT(flags);
+
+	for (;;) {
+		a.fd_in = check_fileno(io_in);
+		a.fd_out = check_fileno(io_out);
+		bytes = (ssize_t)IO_RUN(nogvl_splice, &a);
+		if (bytes == 0) return Qnil;
+		if (bytes < 0) {
+			switch (errno) {
+			case EINTR: continue;
+			case EAGAIN: return sym_EAGAIN;
+			default: rb_sys_fail("splice");
+			}
+		}
+		return SSIZET2NUM(bytes);
+	}
+}
+
+struct tee_args {
+	int fd_in;
+	int fd_out;
+	size_t len;
+	unsigned flags;
+};
+
+/* runs without GVL */
+static void *nogvl_tee(void *ptr)
+{
+	struct tee_args *a = ptr;
+
+	return (void *)tee(a->fd_in, a->fd_out, a->len, a->flags);
+}
+
+/* :nodoc: */
+static VALUE my_tee(VALUE mod, VALUE io_in, VALUE io_out,
+			VALUE len, VALUE flags)
+{
+	struct tee_args a;
+	ssize_t bytes;
+
+	a.len = (size_t)NUM2SIZET(len);
+	a.flags = NUM2UINT(flags);
+
+	for (;;) {
+		a.fd_in = check_fileno(io_in);
+		a.fd_out = check_fileno(io_out);
+		bytes = (ssize_t)IO_RUN(nogvl_tee, &a);
+		if (bytes == 0) return Qnil;
+		if (bytes < 0) {
+			switch (errno) {
+			case EINTR: continue;
+			case EAGAIN: return sym_EAGAIN;
+			default: rb_sys_fail("tee");
+			}
+		}
+		return SSIZET2NUM(bytes);
+	}
+}
+
+void sleepy_penguin_init_splice(void)
+{
+	VALUE mod = rb_define_module("SleepyPenguin");
+	rb_define_singleton_method(mod, "__splice", my_splice, 6);
+	rb_define_singleton_method(mod, "__tee", my_tee, 4);
+
+	/*
+	 * Attempt to move pages instead of copying.  This is only a hint
+	 * and support for it was removed in Linux 2.6.21.  It will be
+         * re-added for FUSE filesystems only in Linux 2.6.35.
+	 */
+	rb_define_const(mod, "F_MOVE", UINT2NUM(SPLICE_F_MOVE));
+
+	/*
+	 * Do not block on pipe I/O.  This flag only affects the pipe(s)
+	 * being spliced from/to and has no effect on the non-pipe
+	 * descriptor (which requires non-blocking operation to be set
+	 * explicitly).
+	 *
+	 * The non-blocking flag (O_NONBLOCK) on the pipe descriptors
+	 * themselves are ignored by this family of functions, and
+	 * using this flag is the only way to get non-blocking operation
+	 * out of them.
+	 *
+	 * It is highly recommended this flag be set
+         * (or SleepyPenguin.trysplice used)
+	 * whenever splicing from a socket into a pipe unless there is
+	 * another (native) thread or process doing a blocking read on that
+	 * pipe.  Otherwise it is possible to block a single-threaded process
+	 * if the socket buffers are larger than the pipe buffers.
+	 */
+	rb_define_const(mod, "F_NONBLOCK", UINT2NUM(SPLICE_F_NONBLOCK));
+
+	/*
+	 * Indicate that there may be more data coming into the outbound
+	 * descriptor.  This can allow the kernel to avoid sending partial
+	 * frames from sockets.  Currently only used with splice.
+	 */
+	rb_define_const(mod, "F_MORE", UINT2NUM(SPLICE_F_MORE));
+
+	/*
+	 * fcntl() command constant used to return the size of a pipe.
+	 * This constant is only defined when running Linux 2.6.35
+	 * or later.
+	 *
+	 *	require 'fcntl'
+	 *	r, w = IO.pipe
+	 *	r.fcntl(SleepyPenguin::F_GETPIPE_SZ) => Integer
+	 */
+	rb_define_const(mod, "F_GETPIPE_SZ", UINT2NUM(F_GETPIPE_SZ));
+
+	/*
+	 * fcntl() command constant used to set the size of a pipe.
+	 * This constant is only defined when running Linux 2.6.35
+	 * or later.
+	 *
+	 * call-seq:
+	 *
+	 *	require 'fcntl'
+	 *	r, w = IO.pipe
+	 *	r.fcntl(SleepyPenguin::F_SETPIPE_SZ, 131072)
+	 */
+	rb_define_const(mod, "F_SETPIPE_SZ", UINT2NUM(F_SETPIPE_SZ));
+
+	sym_EAGAIN = ID2SYM(rb_intern("EAGAIN"));
+}
+#endif

data/ext/sleepy_penguin/timerfd.c

@@ -31,10 +31,8 @@ static VALUE s_new(int argc, VALUE *argv, VALUE klass)
 
 	fd = timerfd_create(clockid, flags);
 	if (fd < 0) {
-		if (errno == EMFILE || errno == ENFILE || errno == ENOMEM) {
-			rb_gc();
+		if (rb_sp_gc_for_fd(errno))
 			fd = timerfd_create(clockid, flags);
-		}
 		if (fd < 0)
 			rb_sys_fail("timerfd_create");
 	}
@@ -122,8 +120,6 @@ static VALUE expirations(int argc, VALUE *argv, VALUE self)
 	rb_scan_args(argc, argv, "01", &nonblock);
 	if (RTEST(nonblock))
 		rb_sp_set_nonblock(fd);
-	else
-		blocking_io_prepare(fd);
 retry:
 	r = (ssize_t)rb_sp_fd_region(tfd_read, &buf, fd);
 	if (r < 0) {

data/ext/sleepy_penguin/util.c

@@ -118,6 +118,9 @@ int rb_sp_fileno(VALUE io)
 {
 	rb_io_t *fptr;
 
+	if (RB_TYPE_P(io, T_FIXNUM))
+		return FIX2INT(io);
+
 	io = rb_io_get_io(io);
 	GetOpenFile(io, fptr);
 	return FPTR_TO_FD(fptr);
@@ -153,3 +156,12 @@ int rb_sp_wait(rb_sp_waitfn waiter, VALUE obj, int *fd)
 	*fd = rb_sp_fileno(obj);
 	return rc;
 }
+
+int rb_sp_gc_for_fd(int err)
+{
+	if (err == EMFILE || err == ENFILE || err == ENOMEM) {
+		rb_gc();
+		return 1;
+	}
+	return 0;
+}

data/lib/sleepy_penguin.rb

@@ -15,3 +15,31 @@ if defined?(SleepyPenguin::Inotify) &&
     # :startdoc
   end
 end
+
+module SleepyPenguin
+  require_relative 'sleepy_penguin/splice' if respond_to?(:__splice)
+  require_relative 'sleepy_penguin/cfr' if respond_to?(:__cfr)
+  require_relative 'sleepy_penguin/epoll' if const_defined?(:Epoll)
+  require_relative 'sleepy_penguin/kqueue' if const_defined?(:Kqueue)
+
+  # Copies +len+ bytes from +src+ to +dst+, where +src+ refers to
+  # an open, mmap(2)-able File and +dst+ refers to a Socket.
+  # An optional +offset+ keyword may be specified for the +src+ File.
+  # Using +offset+ will not adjust the offset of the underlying file
+  # handle itself; in other words: this allows concurrent threads to
+  # use linux_sendfile to write data from one open file to multiple
+  # sockets.
+  #
+  # Returns the number of bytes written on success, or :wait_writable
+  # if the +dst+ Socket is non-blocking and the operation would block.
+  # A return value of zero bytes indicates EOF is reached on the +src+
+  # file.
+  #
+  # Newer OSes may be more flexible in whether or not +dst+ or +src+
+  # is a regular file or socket, respectively.
+  #
+  # This method was added in sleepy_penguin 3.5.0.
+  def self.linux_sendfile(dst, src, len, offset: nil)
+    __lsf(dst, src, offset, len)
+  end
+end

data/lib/sleepy_penguin/cfr.rb

@@ -0,0 +1,29 @@
+module SleepyPenguin
+
+  # call-seq:
+  #    SleepyPenguin.copy_file_range(src, dst, len[, keywords]) => # Integer
+  #
+  # Performs and in-kernel copy of +len+ bytes from +src+ to +dst+,
+  # where +src+ and +dst+ are regular files on the same filesystem.
+  # Returns the number of bytes copied, which may be less than
+  # requested.
+  #
+  # +flags+ is currently unused, but may be specified in the future.
+  #
+  # Keywords:
+  #
+  # :off_in and :off_out if non-nil may be used to specify an Integer
+  # offset for each respective descriptor.  If specified, the file
+  # offsets of each file description will not be moved, providing
+  # pread(2)/pwrite(2)-like semantics.
+  #
+  # See copy_file_range(2) manpage for full documentation:
+  # http://man7.org/linux/man-pages/man2/copy_file_range.2.html
+  #
+  # This method only works in Linux 4.5+ with sleepy_penguin 3.5.0+,
+  # and may require up-to-date kernel headers for non-x86/x86-64 systems.
+  def self.copy_file_range(io_in, io_out, len, flags = 0,
+                           off_in: nil, off_out: nil)
+    __cfr(io_in, off_in, io_out, off_out, len, flags)
+  end
+end

data/lib/sleepy_penguin/epoll.rb

@@ -45,13 +45,16 @@ class SleepyPenguin::Epoll
     end
   end
 
-  # Calls epoll_wait(2) and yields Integer +events+ and IO objects watched
+  # Calls epoll_wait(2) and yields Integer +events+ and +IO+ objects watched
   # for.  +maxevents+ is the maximum number of events to process at once,
   # lower numbers may prevent starvation when used by epoll_wait in multiple
   # threads.  Larger +maxevents+ reduces syscall overhead for
   # single-threaded applications. +maxevents+ defaults to 64 events.
   # +timeout+ is specified in milliseconds, +nil+
   # (the default) meaning it will block and wait indefinitely.
+  #
+  # As of sleepy_penguin 3.5.0+, it is possible to nest
+  # #wait calls within the same thread.
   def wait(maxevents = 64, timeout = nil)
     # snapshot the marks so we do can sit this thread on epoll_wait while other
     # threads may call epoll_ctl.  People say RCU is a poor man's GC, but our
@@ -62,7 +65,7 @@ class SleepyPenguin::Epoll
     end
 
     # we keep a snapshot of @marks around in case another thread closes
-    # the IO while it is being transferred to userspace.  We release mtx
+    # the io while it is being transferred to userspace.  We release mtx
     # so another thread may add events to us while we're sleeping.
     @io.epoll_wait(maxevents, timeout) { |events, io| yield(events, io) }
   ensure
@@ -87,7 +90,7 @@ class SleepyPenguin::Epoll
   # call-seq:
   #     ep.del(io) -> 0
   #
-  # Disables an IO object from being watched.
+  # Disables an +IO+ object from being watched.
   def del(io)
     fd = io.to_io.fileno
     @mtx.synchronize do
@@ -125,7 +128,7 @@ class SleepyPenguin::Epoll
   # call-seq:
   #     epoll.mod(io, flags) -> 0
   #
-  # Changes the watch for an existing IO object based on +events+.
+  # Changes the watch for an existing +IO+ object based on +events+.
   # Returns zero on success, will raise SystemError on failure.
   def mod(io, events)
     events = __event_flags(events)
@@ -166,7 +169,7 @@ class SleepyPenguin::Epoll
         begin
           @io.epoll_ctl(CTL_MOD, io, events)
         rescue Errno::ENOENT
-          warn "epoll event cache failed (mod -> add)"
+          warn "epoll event cache failed (mod -> add)\n"
           @io.epoll_ctl(CTL_ADD, io, events)
           @marks[fd] = io
         end
@@ -174,7 +177,7 @@ class SleepyPenguin::Epoll
         begin
           @io.epoll_ctl(CTL_ADD, io, events)
         rescue Errno::EEXIST
-          warn "epoll event cache failed (add -> mod)"
+          warn "epoll event cache failed (add -> mod)\n"
           @io.epoll_ctl(CTL_MOD, io, events)
         end
         @marks[fd] = io
@@ -214,8 +217,8 @@ class SleepyPenguin::Epoll
   # call-seq:
   #     ep.io_for(io) -> object
   #
-  # Returns the given IO object currently being watched for.  Different
-  # IO objects may internally refer to the same process file descriptor.
+  # Returns the given +IO+ object currently being watched for.  Different
+  # +IO+ objects may internally refer to the same process file descriptor.
   # Mostly used for debugging.
   def io_for(io)
     fd = __fileno(io)
@@ -244,9 +247,9 @@ class SleepyPenguin::Epoll
   # call-seq:
   #     epoll.include?(io) -> true or false
   #
-  # Returns whether or not a given IO is watched and prevented from being
+  # Returns whether or not a given +IO+ is watched and prevented from being
   # garbage-collected by the current Epoll object.  This may include
-  # closed IO objects.
+  # closed +IO+ objects.
   def include?(io)
     fd = __fileno(io)
     @mtx.synchronize do

data/lib/sleepy_penguin/kqueue.rb

@@ -1,7 +1,8 @@
 require 'thread'
+require_relative 'kevent'
 
-# The high-level Kqueue interface.  This provides fork-safety under Ruby 1.9
-# and later (but not Ruby 1.8).
+# The high-level Kqueue interface.  This provides fork-safety; as
+# underlying kqueue descriptors are closed by the OS upon fork.
 # This also provides memory protection from bugs due to not storing an
 # external reference to an object, but still requires the user to store
 # their own object references.
@@ -33,10 +34,6 @@ class SleepyPenguin::Kqueue
 
   def __kq_check # :nodoc:
     return if @pid == $$ || @io.closed?
-    unless @io.respond_to?(:autoclose=)
-      raise RuntimeError,
-       "Kqueue is not safe to use without IO#autoclose=, upgrade to Ruby 1.9+"
-    end
 
     # kqueue has (strange) close-on-fork behavior
     objects = @copies.values
@@ -55,6 +52,9 @@ class SleepyPenguin::Kqueue
   # Ruby GC, otherwise ObjectSpace._id2ref may return invalid objects.
   # Unlike the low-level Kqueue::IO#kevent, the block given yields only
   # a single Kevent struct, not a 6-element array.
+  #
+  # As of sleepy_penguin 3.5.0+, it is possible to nest #kevent
+  # calls within the same thread.
   def kevent(changelist = nil, *args)
     @mtx.synchronize { __kq_check }
     if changelist

data/lib/sleepy_penguin/sp.rb

@@ -1,4 +1,4 @@
 # :stopdoc:
-require "sleepy_penguin"
+require_relative '../sleepy_penguin'
 Object.const_set(:SP, SleepyPenguin)
 # :startdoc:

data/lib/sleepy_penguin/splice.rb

@@ -0,0 +1,125 @@
+# -*- encoding: binary -*-
+
+module SleepyPenguin
+  # call-seq:
+  #    SleepyPenguin.splice(io_in, io_out, len[, flags [, keywords]) => Integer
+  #
+  # Splice +len+ bytes from/to a pipe.  Either +io_in+ or +io_out+
+  # MUST be a pipe.  +io_in+ and +io_out+ may BOTH be pipes as of
+  # Linux 2.6.31 or later.
+  #
+  # +flags+ defaults to zero if unspecified.
+  # It may be an Integer bitmask, a Symbol, or Array of Symbols
+  #
+  # The integer bitmask may any combination of:
+  #
+  # * SleepyPenguin::F_MOVE - attempt to move pages instead of copying
+  #
+  # * SleepyPenguin::F_NONBLOCK - do not block on pipe I/O (only)
+  #
+  # * SleepyPenguin::F_MORE - indicates more data will be sent soon
+  #
+  # Symbols may be used as well to specify a single flag:
+  #
+  # * :move        - corresponds to F_MOVE
+  # * :nonblock    - corresponds to F_NONBLOCK
+  # * :more        - corresponds to F_MORE
+  #
+  # Or, an array of any combination of the above symbols.
+  #
+  # Keywords:
+  #
+  # :off_in and :off_out if non-nil may be used to
+  #  specify an offset for the respective non-pipe file descriptor.
+  #
+  # :exception defaults to +true+.  Setting it to +false+
+  # will return :EAGAIN symbol instead of raising Errno::EAGAIN.
+  # This will also return +nil+ instead of raising EOFError
+  # when +io_in+ is at the end.
+  #
+  # Raises EOFError when +io_in+ has reached end of file.
+  # Raises Errno::EAGAIN if the SleepyPenguin::F_NONBLOCK flag is set
+  # and the pipe has no data to read from or space to write to.  May
+  # also raise Errno::EAGAIN if the non-pipe descriptor has no data
+  # to read from or space to write to.
+  #
+  # As splice never exposes buffers to userspace, it will not take
+  # into account userspace buffering done by Ruby or stdio.  It is
+  # also not subject to encoding/decoding filters under Ruby 1.9+.
+  #
+  # Consider using `exception: false` if +io_out+ is a pipe or if you
+  # are using non-blocking I/O on both descriptors as it avoids the
+  # cost of raising common Errno::EAGAIN exceptions.
+  #
+  # See manpage for full documentation:
+  # http://man7.org/linux/man-pages/man2/splice.2.html
+  #
+  # Support for this exists in sleepy_penguin 3.5.0+
+  def self.splice(io_in, io_out, len, flags = 0,
+                  off_in: nil, off_out: nil, exception: true)
+    flags = __map_splice_flags(flags)
+    ret = __splice(io_in, off_in, io_out, off_out, len, flags)
+    exception ? __map_exc(ret) : ret
+  end
+
+  # call-seq:
+  #   SleepyPenguin.tee(io_in, io_out, len[, flags[, keywords]) => Integer
+  #
+  # Copies up to +len+ bytes of data from +io_in+ to +io_out+.  +io_in+
+  # and +io_out+ must both refer to pipe descriptors.  +io_in+ and +io_out+
+  # may not be endpoints of the same pipe.
+  #
+  # +flags+ may be zero (the default) or a combination of:
+  # * SleepyPenguin::F_NONBLOCK
+  #
+  # As a shortcut, the `:nonblock` symbol may be used instead.
+  #
+  # Other splice-related flags are currently unimplemented in the
+  # kernel and have no effect.
+  #
+  # Returns the number of bytes duplicated if successful.
+  # Raises EOFError when +io_in+ is closed and emptied.
+  # Raises Errno::EAGAIN when +io_in+ is empty and/or +io_out+ is full
+  # and +flags+ specifies non-blocking operation
+  #
+  # Keywords:
+  #
+  # :exception defaults to +true+.  Setting it to +false+
+  # will return :EAGAIN symbol instead of raising Errno::EAGAIN.
+  # This will also return +nil+ instead of raising EOFError
+  # when +io_in+ is at the end.
+  #
+  # Consider using `exception: false` if +io_out+ is a pipe or if you
+  # are using non-blocking I/O on both descriptors as it avoids the
+  # cost of raising common Errno::EAGAIN exceptions.
+  #
+  # See manpage for full documentation:
+  # http://man7.org/linux/man-pages/man2/tee.2.html
+  #
+  # Support for this exists in sleepy_penguin 3.5.0+
+  def self.tee(io_in, io_out, len, flags = 0, exception: true)
+    flags = __map_splice_flags(flags)
+    ret = __tee(io_in, io_out, len, flags)
+    exception ? __map_exc(ret) : ret
+  end if respond_to?(:__tee)
+
+  @__splice_f_map = { # :nodoc:
+    :nonblock => F_NONBLOCK,
+    :more => F_MORE,
+    :move => F_MOVE
+  }
+
+  def self.__map_splice_flags(flags) # :nodoc:
+    onef = @__splice_f_map[flags] and return onef
+    flags.respond_to?(:inject) ?
+        flags.inject(0) { |fl, sym| fl |= @__splice_f_map[sym] } : flags
+  end
+
+  def self.__map_exc(ret) # :nodoc:
+    case ret
+    when :EAGAIN then raise Errno::EAGAIN, 'Resource temporarily unavailable'
+    when nil then raise EOFError, 'end of file reached'
+    end
+    ret
+  end
+end