io_splice 0.1.0

data/Rakefile

@@ -0,0 +1,156 @@
+# -*- encoding: binary -*-
+
+# most tasks are in the GNUmakefile which offers better parallelism
+
+def tags
+  timefmt = '%Y-%m-%dT%H:%M:%SZ'
+  @tags ||= `git tag -l`.split(/\n/).map do |tag|
+    if %r{\Av[\d\.]+\z} =~ tag
+      header, subject, body = `git cat-file tag #{tag}`.split(/\n\n/, 3)
+      header = header.split(/\n/)
+      tagger = header.grep(/\Atagger /).first
+      body ||= "initial"
+      {
+        :time => Time.at(tagger.split(/ /)[-2].to_i).utc.strftime(timefmt),
+        :tagger_name => %r{^tagger ([^<]+)}.match(tagger)[1].strip,
+        :tagger_email => %r{<([^>]+)>}.match(tagger)[1].strip,
+        :id => `git rev-parse refs/tags/#{tag}`.chomp!,
+        :tag => tag,
+        :subject => subject,
+        :body => body,
+      }
+    end
+  end.compact.sort { |a,b| b[:time] <=> a[:time] }
+end
+
+cgit_url = "http://git.bogomips.org/cgit/ruby_io_splice.git"
+git_url = ENV['GIT_URL'] || 'git://git.bogomips.org/ruby_io_splice.git'
+web_url = "http://bogomips.org/ruby_io_splice/"
+
+desc 'prints news as an Atom feed'
+task :news_atom do
+  require 'nokogiri'
+  new_tags = tags[0,10]
+  puts(Nokogiri::XML::Builder.new do
+    feed :xmlns => "http://www.w3.org/2005/Atom" do
+      id! "#{web_url}NEWS.atom.xml"
+      title "Ruby io_splice news"
+      subtitle "splice and tee Linux syscalls from Ruby"
+      link! :rel => "alternate", :type => "text/html",
+            :href => "#{web_url}NEWS.html"
+      updated(new_tags.empty? ? "1970-01-01T00:00:00Z" : new_tags.first[:time])
+      new_tags.each do |tag|
+        entry do
+          title tag[:subject]
+          updated tag[:time]
+          published tag[:time]
+          author {
+            name tag[:tagger_name]
+            email tag[:tagger_email]
+          }
+          url = "#{cgit_url}/tag/?id=#{tag[:tag]}"
+          link! :rel => "alternate", :type => "text/html", :href =>url
+          id! url
+          message_only = tag[:body].split(/\n.+\(\d+\):\n {6}/s).first.strip
+          content({:type =>:text}, message_only)
+          content(:type =>:xhtml) { pre tag[:body] }
+        end
+      end
+    end
+  end.to_xml)
+end
+
+desc 'prints RDoc-formatted news'
+task :news_rdoc do
+  tags.each do |tag|
+    time = tag[:time].tr!('T', ' ').gsub!(/:\d\dZ/, ' UTC')
+    puts "=== #{tag[:tag].sub(/^v/, '')} / #{time}"
+    puts ""
+
+    body = tag[:body]
+    puts tag[:body].gsub(/^/sm, "  ").gsub(/[ \t]+$/sm, "")
+    puts ""
+  end
+end
+
+desc "print release changelog for Rubyforge"
+task :release_changes do
+  version = ENV['VERSION'] or abort "VERSION= needed"
+  version = "v#{version}"
+  vtags = tags.map { |tag| tag[:tag] =~ /\Av/ and tag[:tag] }.sort
+  prev = vtags[vtags.index(version) - 1]
+  if prev
+    system('git', 'diff', '--stat', prev, version) or abort $?
+    puts ""
+    system('git', 'log', "#{prev}..#{version}") or abort $?
+  else
+    system('git', 'log', version) or abort $?
+  end
+end
+
+desc "print release notes for Rubyforge"
+task :release_notes do
+  spec = Gem::Specification.load('io_splice.gemspec')
+  puts spec.description.strip
+  puts ""
+  puts "* #{spec.homepage}"
+  puts "* #{spec.email}"
+  puts "* #{git_url}"
+
+  _, _, body = `git cat-file tag v#{spec.version}`.split(/\n\n/, 3)
+  print "\nChanges:\n\n"
+  puts body
+end
+
+desc "read news article from STDIN and post to rubyforge"
+task :publish_news do
+  require 'rubyforge'
+  IO.select([STDIN], nil, nil, 1) or abort "E: news must be read from stdin"
+  msg = STDIN.readlines
+  subject = msg.shift
+  blank = msg.shift
+  blank == "\n" or abort "no newline after subject!"
+  subject.strip!
+  body = msg.join("").strip!
+
+  rf = RubyForge.new.configure
+  rf.login
+  rf.post_news('qrp', subject, body)
+end
+
+desc "post to RAA"
+task :raa_update do
+  require 'net/http'
+  require 'net/netrc'
+  rc = Net::Netrc.locate('io_splice-raa') or abort "~/.netrc not found"
+  password = rc.password
+
+  s = Gem::Specification.load('io_splice.gemspec')
+  desc = [ s.description.strip ]
+  desc << ""
+  desc << "* #{s.email}"
+  desc << "* #{git_url}"
+  desc << "* #{cgit_url}"
+  desc = desc.join("\n")
+  uri = URI.parse('http://raa.ruby-lang.org/regist.rhtml')
+  form = {
+    :name => s.name,
+    :short_description => s.summary,
+    :version => s.version.to_s,
+    :status => 'experimental',
+    :owner => s.authors.first,
+    :email => s.email,
+    :category_major => 'Library',
+    :category_minor => 'System',
+    :url => s.homepage,
+    :download => 'http://rubyforge.org/frs/?group_id=5626',
+    :license => 'LGPL', # LGPLv3, actually, but RAA is ancient...
+    :description_style => 'Plain',
+    :description => desc,
+    :pass => password,
+    :submit => 'Update',
+  }
+  res = Net::HTTP.post_form(uri, form)
+  p res
+  puts res.body
+end

data/examples/splice-cp.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# -*- encoding: binary -*-
+
+# Example of using IO.splice to copy a file
+# This can be significantly faster than IO.copy_stream as data
+# is never copied into userspace.
+
+require 'io/splice'
+
+usage = "#$0 SOURCE DEST"
+source = ARGV.shift or abort usage
+dest = ARGV.shift or abort usage
+
+source = File.open(source, 'rb')
+dest = File.open(dest, 'wb')
+source_fd, dest_fd = source.fileno, dest.fileno
+
+# We use a pipe as a ring buffer in kernel space.
+# pipes may store up to IO::Splice::PIPE_CAPA bytes
+pipe = IO.pipe
+rfd, wfd = pipe.map { |io| io.fileno }
+
+begin
+  nread = begin
+    # first pull as many bytes as possible into the pipe
+    IO.splice(source_fd, nil, wfd, nil, IO::Splice::PIPE_CAPA, 0)
+  rescue EOFError
+    break
+  end
+
+  # now move the contents of the pipe buffer into the destination file
+  # the copied data never enters userspace
+  nwritten = IO.splice(rfd, nil, dest_fd, nil, nread, 0)
+
+  nwritten == nread or
+    abort "short splice to destination file: #{nwritten} != #{nread}"
+end while true

data/examples/splice-tee.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# -*- encoding: binary -*-
+
+# An example of using IO.tee, this is a limited version of the standard
+# "tee" utility that requires stdin and stdout to both be pipes.
+require 'io/splice'
+
+usage = "filter_prog1 | #$0 DEST | filter_prog2"
+dest = ARGV.shift or abort usage
+$stdin.stat.pipe? or abort "stdin must be a pipe"
+$stdout.stat.pipe? or abort "stdout must be a pipe"
+
+dest = File.open(dest, 'wb')
+out_fd = dest.fileno
+
+stdin_fd = $stdin.fileno
+stdout_fd = $stdout.fileno
+
+begin
+  nread = begin
+    # "copy" data from stdin to stdout, without consuming stdin
+    IO.tee(stdin_fd, stdout_fd, IO::Splice::PIPE_CAPA, 0)
+  rescue EOFError
+    break
+  end
+
+  # sends data to the file, consumes stdin
+  nwritten = IO.splice(stdin_fd, nil, out_fd, nil, nread, 0)
+
+  nwritten == nread or
+    abort "short splice to file: #{nwritten} != #{nread}"
+end while true

data/ext/io_splice/extconf.rb

@@ -0,0 +1,9 @@
+require 'mkmf'
+$CPPFLAGS << ' -D_GNU_SOURCE=1'
+
+have_func('splice', %w(fcntl.h)) or abort 'splice(2) not defined'
+have_func('tee', %w(fcntl.h)) or abort 'tee(2) not defined'
+have_func('rb_thread_blocking_region')
+
+dir_config('io_splice')
+create_makefile('io_splice_ext')

data/ext/io_splice/io_splice_ext.c

@@ -0,0 +1,319 @@
+#include "ruby.h"
+#ifdef HAVE_RUBY_IO_H
+#  include "ruby/io.h"
+#else
+#  include "rubyio.h"
+#endif
+#include <fcntl.h>
+#include <assert.h>
+#include <sys/uio.h>
+#include <limits.h>
+#include <alloca.h>
+
+#ifdef HAVE_RB_THREAD_BLOCKING_REGION
+#  define RUBY_1_8_TRAP_BEG for(;0;)
+#  define RUBY_1_8_TRAP_END for(;0;)
+#else
+/* partial emulation of the 1.9 rb_thread_blocking_region under 1.8 */
+#  include <rubysig.h>
+#  define RUBY_UBF_IO ((rb_unblock_function_t *)-1)
+typedef void rb_unblock_function_t(void *);
+typedef VALUE rb_blocking_function_t(void *);
+static VALUE
+rb_thread_blocking_region(
+	rb_blocking_function_t *fn, void *data1,
+	rb_unblock_function_t *ubf, void *data2)
+{
+	VALUE rv;
+
+	assert(RUBY_UBF_IO == ubf && "RUBY_UBF_IO required for emulation");
+
+	TRAP_BEG;
+	rv = fn(data1);
+	TRAP_END;
+
+	return rv;
+}
+#  define RUBY_1_8_TRAP_BEG TRAP_BEG
+#  define RUBY_1_8_TRAP_END TRAP_END
+#endif /* ! HAVE_RB_THREAD_BLOCKING_REGION */
+
+#ifndef RSTRING_PTR
+#  define RSTRING_PTR(s) (RSTRING(s)->ptr)
+#endif
+#ifndef RSTRING_LEN
+#  define RSTRING_LEN(s) (RSTRING(s)->len)
+#endif
+#ifndef RARRAY_PTR
+#  define RARRAY_PTR(s) (RARRAY(s)->ptr)
+#endif
+#ifndef RARRAY_LEN
+#  define RARRAY_LEN(s) (RARRAY(s)->len)
+#endif
+
+/*
+ * Releases GVL only iff blocking I/O is used.
+ * We'll trust programmers who use non-blocking I/O explicitly to
+ * want the fastest possible performance without resorting to threads,
+ * so releasing and them immediately reacquiring the GVL would be
+ * a waste of time.
+ */
+static VALUE nb_io_run(rb_blocking_function_t *fn, void *data, unsigned flags)
+{
+	if (flags & SPLICE_F_NONBLOCK)
+		return fn(data);
+	return rb_thread_blocking_region(fn, data, RUBY_UBF_IO, 0);
+}
+
+struct splice_args {
+	int fd_in;
+	off_t *off_in;
+	int fd_out;
+	off_t *off_out;
+	size_t len;
+	unsigned flags;
+};
+
+static VALUE nogvl_splice(void *ptr)
+{
+	struct splice_args *a = ptr;
+	long n;
+
+	/*
+	 * it's still possible to block because the SPLICE_F_NONBLOCK flag
+	 * only affects the pipe descriptor, not the non-pipe descriptor.
+	 * So use TRAP_BEG/TRAP_END (only) to make Ruby 1.8 happy.  We also
+	 * don't want the TRAP_BEG/TRAP_END compatibility layer in 1.9,
+	 * so we use the 1.8-only versions
+	 */
+	RUBY_1_8_TRAP_BEG;
+	n = splice(a->fd_in, a->off_in, a->fd_out, a->off_out,
+	           a->len, a->flags);
+	RUBY_1_8_TRAP_END;
+
+	return (VALUE)n;
+}
+
+/*
+ * call-seq:
+ *    IO.splice(fd_in, off_in, fd_out, off_out, len, flags) => integer
+ *
+ * Splice +len+ bytes from/to a pipe.  Either +fd_in+ or +fd_out+
+ * MUST be a pipe.  +fd_in+ and +fd_out+ may BOTH be pipes as of
+ * Linux 2.6.31 or later.
+ *
+ * +off_in+ and +off_out+ if non-nil may be used to
+ * specify an offset for the non-pipe file descriptor.
+ *
+ * +flags+ may be a bitmask of the following flags:
+ *
+ * IO::Splice::F_MOVE, IO::Splice::F_NONBLOCK, IO::Splice::F_MORE
+ *
+ * Returns the number of bytes spliced.
+ * Raises EOFError when +fd_in+ has reached end of file.
+ * Raises Errno::EAGAIN if the IO::Splice::F_NONBLOCK flag is set
+ * and the pipe has no data to read from or space to write to.  May
+ * also raise Errno::EAGAIN if the non-pipe descriptor has no data
+ * to read from or space to write to.
+ *
+ *     rd, wr = (pipe = IO.pipe).map { |io| io.fileno }
+ *     src_io, dst_io = File.open("/path/to/src"), File.open("/path/to/dst")
+ *     src, dst = src_io.fileno, dst_io.fileno
+ *
+ *     nr = IO.splice(src, nil, wr, nil, IO::Splice::PIPE_CAPA, 0)
+ *     IO.splice(rd, nil, dst, nil, nr, 0)
+ *
+ * As splice never exposes buffers to userspace, it will not take
+ * into account userspace buffering done by Ruby or stdio.  It is
+ * also not subject to encoding/decoding filters under Ruby 1.9.
+ *
+ * See manpage for full documentation:
+ * http://kernel.org/doc/man-pages/online/pages/man2/splice.2.html
+ */
+static VALUE my_splice(VALUE self,
+	VALUE fd_in, VALUE off_in,
+	VALUE fd_out, VALUE off_out,
+	VALUE len, VALUE flags)
+{
+	off_t i, o;
+	long n;
+	struct splice_args a = {
+		.off_in = NIL_P(off_in) ? NULL : (i = NUM2OFFT(off_in), &i),
+		.off_out = NIL_P(off_out) ? NULL : (o = NUM2OFFT(off_out), &o),
+		.fd_in = NUM2INT(fd_in),
+		.fd_out = NUM2INT(fd_out),
+		.len = (size_t)NUM2ULONG(len),
+		.flags = NUM2UINT(flags),
+	};
+
+	n = (long)nb_io_run(nogvl_splice, &a, a.flags);
+	if (n == 0)
+		rb_eof_error();
+	if (n < 0)
+		rb_sys_fail("splice");
+	return LONG2NUM(n);
+}
+
+struct tee_args {
+	int fd_in;
+	int fd_out;
+	size_t len;
+	unsigned flags;
+};
+
+/* runs without GVL */
+static VALUE nogvl_tee(void *ptr)
+{
+	struct tee_args *a = ptr;
+
+	return (VALUE)tee(a->fd_in, a->fd_out, a->len, a->flags);
+}
+
+/*
+ * call-seq:
+ *   IO.tee(fd_in, fd_out, len, flags) => integer
+ *
+ * Copies up to +len+ bytes of data from +fd_in+ to +fd_out+.  +fd_in+
+ * and +fd_out+ must both refer to pipe descriptors.  +fd_in+ and +fd_out+
+ * may not be endpoints of the same pipe.
+ *
+ * +flags+ may be zero or IO::Splice::F_NONBLOCK
+ * Other IO::Splice flags are currently unimplemented or have no effect.
+ *
+ * Returns the number of bytes duplicated if successful.
+ * Raises EOFError when +fd_in+ is closed and emptied.
+ * Raises Errno::EAGAIN when +fd_in+ is empty and/or +fd_out+ is full
+ * and +flags+ contains IO::Splice::F_NONBLOCK
+ *
+ * See manpage for full documentation:
+ * http://kernel.org/doc/man-pages/online/pages/man2/tee.2.html
+ */
+static VALUE my_tee(VALUE self,
+	VALUE fd_in, VALUE fd_out,
+	VALUE len, VALUE flags)
+{
+	long n;
+	struct tee_args a = {
+		.fd_in = NUM2INT(fd_in),
+		.fd_out = NUM2INT(fd_out),
+		.len = (size_t)NUM2ULONG(len),
+		.flags = NUM2UINT(flags),
+	};
+
+	n = (long)nb_io_run(nogvl_tee, &a, a.flags);
+	if (n == 0)
+		rb_eof_error();
+	if (n < 0)
+		rb_sys_fail("tee");
+
+	return LONG2NUM(n);
+}
+
+struct vmsplice_args {
+	int fd;
+        struct iovec *iov;
+	unsigned long nr_segs;
+	unsigned flags;
+};
+
+static VALUE nogvl_vmsplice(void *ptr)
+{
+	struct vmsplice_args *a = ptr;
+
+	return (VALUE)vmsplice(a->fd, a->iov, a->nr_segs, a->flags);
+}
+
+/*
+ * call-seq:
+ *   IO.vmsplice(fd, string_array, flags) => integer
+ *
+ * Transfers an array of strings into the pipe descriptor given by fd.
+ * +fd+ must be the writable end of a pipe.
+ *
+ * This may allow the kernel to avoid data copies in some cases.
+ * but is (probably) of limited usefulness in Ruby.
+ *
+ * See manpage for full documentation:
+ * http://kernel.org/doc/man-pages/online/pages/man2/vmsplice.2.html
+ */
+static VALUE my_vmsplice(VALUE self, VALUE fd, VALUE data, VALUE flags)
+{
+	long n;
+	struct vmsplice_args a;
+	struct iovec *tmp;
+	VALUE *ary;
+
+	switch (TYPE(data)) {
+	case T_ARRAY:
+		ary = RARRAY_PTR(data);
+		a.nr_segs = RARRAY_LEN(data);
+
+		if (a.nr_segs > IOV_MAX)
+			rb_raise(rb_eArgError, "array larger than IOV_MAX");
+
+		a.iov = tmp = alloca(sizeof(struct iovec) * a.nr_segs);
+
+		for (n = (long)a.nr_segs; --n >= 0; tmp++, ary++) {
+			if (TYPE(*ary) != T_STRING)
+				rb_raise(rb_eArgError,
+				         "must be an array of strings");
+			tmp->iov_base = RSTRING_PTR(*ary);
+			tmp->iov_len = RSTRING_LEN(*ary);
+		}
+		break;
+	default:
+		rb_raise(rb_eArgError, "must be an array of strings");
+	}
+
+	a.fd = NUM2INT(fd);
+	a.flags = NUM2UINT(flags);
+
+	n = (long)nb_io_run(nogvl_vmsplice, &a, a.flags);
+	if (n < 0)
+		rb_sys_fail("vmsplice");
+
+	return LONG2NUM(n);
+}
+
+void Init_io_splice_ext(void)
+{
+	VALUE cSplice = rb_define_class_under(rb_cIO, "Splice", rb_cObject);
+
+	rb_define_singleton_method(rb_cIO, "splice", my_splice, 6);
+	rb_define_singleton_method(rb_cIO, "tee", my_tee, 4);
+	rb_define_singleton_method(rb_cIO, "vmsplice", my_vmsplice, 3);
+
+	/*
+	 * Attempt to move pages instead of copying.  This is only a hint
+	 * and support for it was removed in Linux 2.6.21 and has not been
+	 * readded as of 2.6.30.
+	 */
+	rb_define_const(cSplice, "F_MOVE", UINT2NUM(SPLICE_F_MOVE));
+
+	/*
+	 * Do not block on I/O.  This flag only affects the pipe(s) being
+	 * spliced from/to and has no effect on the non-pipe descriptor
+	 * (which requires non-blocking operation to be set explicitly).
+	 */
+	rb_define_const(cSplice, "F_NONBLOCK", UINT2NUM(SPLICE_F_NONBLOCK));
+
+	/*
+	 * Indicate that there may be more data coming into the outbound
+	 * descriptor.  This can allow the kernel to avoid sending partial
+	 * frames from sockets.  Currently only used with splice.
+	 */
+	rb_define_const(cSplice, "F_MORE", UINT2NUM(SPLICE_F_MORE));
+
+	/*
+	 * Only usable by vmsplice.  This flag probably not useful in the
+	 * context of Ruby applications which cannot control alignment.
+	 */
+	rb_define_const(cSplice, "F_GIFT", UINT2NUM(SPLICE_F_GIFT));
+
+	/*
+	 * The maximum size of an atomic write to a pipe
+	 * POSIX requires this to be at least 512 bytes.
+	 * Under Linux, this is 4096 bytes.
+	 */
+	rb_define_const(cSplice, "PIPE_BUF", UINT2NUM(PIPE_BUF));
+}