io-extra 1.2.1 → 1.2.2

data/CHANGES

@@ -1,3 +1,9 @@
+== 1.2.2 - 1-May-2010
+* Added the IO.writev method which allows you to write arrays without
+  requiring an Array#join call, i.e. "gather write". Thanks go to Eric
+  Wong for supplying the code to support this.
+* Some Rakefile updates.
+
 == 1.2.1 - 16-Jan-2010
 * Some README updates
 * Updates to the gemspec.

data/README

@@ -60,15 +60,18 @@
   documentation that was generated by RDoc (if you did a gem install).
 
 = Known Issues
-  None that I'm aware of (beyond those documented above). Please file any bug
-  reports on the project page at http://www.rubyforge.org/projects/shards.
+  The IO.writev tests fail on Solaris. We are not sure why yet.
+
+  Please file any bug reports on the project page at
+  http://www.rubyforge.org/projects/shards.
 
 = Future Plans
   * I may add the File::O_DIRECT open constant on platforms that support it.
   * Switch from C extension to FFI.
 
 = Acknowledgements
-  Eric Wong for some great work on Linux compatibility and other fixes.
+  Eric Wong for some great work on Linux compatibility and other fixes, as
+  well as the code for the IO.writev method.
    
 = License
   Artistic 2.0

data/Rakefile

@@ -19,6 +19,7 @@ task :clean do
     build_file = File.join(Dir.pwd, 'io', 'extra.' + Config::CONFIG['DLEXT'])
     File.delete(build_file) if File.exists?(build_file)
   end
+  Dir['*.gem'].each{ |f| File.delete(f) }
 end
 
 desc "Build the io-extra library (but don't install it)"
@@ -40,14 +41,14 @@ task :install => [:build] do
 end
 
 namespace :gem do
-  desc 'Build the io-extra gem'
-  task :build do
+  desc 'Create the io-extra gem'
+  task :create do
     spec = eval(IO.read('io-extra.gemspec'))
     Gem::Builder.new(spec).build
   end
 
   desc "Install the io-extra library as a gem"
-  task :install => [:build] do
+  task :install => [:create] do
     file = Dir["io-extra*.gem"].last
     sh "gem install #{file}"
   end

data/ext/extconf.rb

@@ -6,12 +6,15 @@ dir_config('io')
 
 have_header('stdint.h')
 have_header('sys/resource.h')
+have_header('sys/uio.h')
 have_func('closefrom')
 have_func('fdwalk')
 have_func('directio')
 have_func('pread')
 have_func('pwrite')
+have_func('writev')
 have_func('rb_str_set_len', 'ruby.h')
+have_func('rb_thread_blocking_region')
 
 case Config::CONFIG['host_os']
 when /darwin/i

data/ext/io/extra.c

@@ -14,6 +14,49 @@
 #include <stdint.h>
 #endif
 
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+
+#ifdef HAVE_LIMITS_H
+#include <limits.h>
+#endif
+
+#if !defined(IOV_MAX)
+#if defined(_SC_IOV_MAX)
+#define IOV_MAX (sysconf(_SC_IOV_MAX))
+#else
+/* assume infinity, or let the syscall return with error ... */
+#define IOV_MAX INT_MAX
+#endif
+#endif
+
+#ifndef HAVE_RB_THREAD_BLOCKING_REGION
+/*
+ * partial emulation of the 1.9 rb_thread_blocking_region under 1.8,
+ * this is enough to ensure signals are processed safely when doing I/O
+ * to a slow device, but doesn't actually ensure threads can be
+ * scheduled fairly in 1.8
+ */
+#include <rubysig.h>
+#define RUBY_UBF_IO ((rb_unblock_function_t *)-1)
+typedef void rb_unblock_function_t(void *);
+typedef VALUE rb_blocking_function_t(void *);
+static VALUE
+rb_thread_blocking_region(
+   rb_blocking_function_t *fn, void *data1,
+   rb_unblock_function_t *ubf, void *data2)
+{
+   VALUE rv;
+
+   TRAP_BEG;
+   rv = fn(data1);
+   TRAP_END;
+
+   return rv;
+}
+#endif
+
 #ifndef RSTRING_PTR
 #define RSTRING_PTR(v) (RSTRING(v)->ptr)
 #define RSTRING_LEN(v) (RSTRING(v)->len)
@@ -287,7 +330,7 @@ static VALUE s_io_pread(VALUE klass, VALUE v_fd, VALUE v_nbyte, VALUE v_offset){
 
    if (nread < 0)
       rb_sys_fail("pread");
-   if (nread != nbyte)
+   if ((size_t)nread != nbyte)
       rb_str_set_len(str, nread);
 
    return str;
@@ -345,6 +388,121 @@ static VALUE s_io_pwrite(VALUE klass, VALUE v_fd, VALUE v_buf, VALUE v_offset){
 }
 #endif
 
+/* this can't be a function since we use alloca() */
+#define ARY2IOVEC(iov,iovcnt,expect,ary) \
+   do { \
+      VALUE *cur; \
+      struct iovec *tmp; \
+      long n; \
+      if (TYPE(ary) != T_ARRAY) \
+         rb_raise(rb_eArgError, "must be an array of strings"); \
+      cur = RARRAY_PTR(ary); \
+      n = RARRAY_LEN(ary); \
+      if (n > IOV_MAX) \
+         rb_raise(rb_eArgError, "array is larger than IOV_MAX"); \
+      iov = tmp = alloca(sizeof(struct iovec) * n); \
+      expect = 0; \
+      iovcnt = (int)n; \
+      for (; --n >= 0; tmp++, cur++) { \
+         if (TYPE(*cur) != T_STRING) \
+            rb_raise(rb_eArgError, "must be an array of strings"); \
+         tmp->iov_base = RSTRING_PTR(*cur); \
+         tmp->iov_len = RSTRING_LEN(*cur); \
+         expect += tmp->iov_len; \
+      } \
+   } while (0)
+
+#if defined(HAVE_WRITEV)
+struct writev_args {
+   int fd;
+   struct iovec *iov;
+   int iovcnt;
+};
+
+static VALUE nogvl_writev(void *ptr)
+{
+   struct writev_args *args = ptr;
+
+   return (VALUE)writev(args->fd, args->iov, args->iovcnt);
+}
+
+/*
+ * IO.writev(fd, %w(hello world))
+ *
+ * This method writes the contents of an array of strings to the given +fd+.
+ * It can be useful to avoid generating a temporary string via Array#join
+ * when writing out large arrays of strings.
+ *
+ * The given array should have fewer elements than the IO::IOV_MAX constant.
+ *
+ * Returns the number of bytes written.
+ */
+static VALUE s_io_writev(VALUE klass, VALUE fd, VALUE ary) {
+   ssize_t result = 0;
+   ssize_t left;
+   struct writev_args args;
+
+   args.fd = NUM2INT(fd);
+   ARY2IOVEC(args.iov, args.iovcnt, left, ary);
+
+   for(;;) {
+      ssize_t w = (ssize_t)rb_thread_blocking_region(nogvl_writev, &args,
+                                                     RUBY_UBF_IO, 0);
+
+      if(w == -1) {
+         if (rb_io_wait_writable(args.fd)) {
+            continue;
+         } else {
+            if (result > 0) {
+               /*
+                * unlikely to hit this case, return the already written bytes,
+                * we'll let the next write (or close) fail instead
+                */
+               break;
+            }
+            rb_sys_fail("writev");
+         }
+      }
+
+      result += w;
+      if(w == left) {
+         break;
+      } else { /* partial write, this can get tricky */
+         int i;
+         struct iovec *new_iov = args.iov;
+
+         left -= w;
+
+         /* skip over iovecs we've already written completely */
+         for (i = 0; i < args.iovcnt; i++, new_iov++) {
+            if (w == 0)
+               break;
+
+            /*
+             * partially written iov,
+             * modify and retry with current iovec in front
+             */
+            if (new_iov->iov_len > (size_t)w) {
+               VALUE base = (VALUE)new_iov->iov_base;
+
+               new_iov->iov_len -= w;
+               new_iov->iov_base = (void *)(base + w);
+               break;
+            }
+
+            w -= new_iov->iov_len;
+         }
+
+         /* retry without the already-written iovecs */
+         args.iovcnt -= i;
+         args.iov = new_iov;
+      }
+   }
+
+   return LONG2NUM(result);
+}
+#endif
+
 /* Adds the IO.closefrom, IO.fdwalk class methods, as well as the IO#directio
  * and IO#directio? instance methods (if supported on your platform).
  */
@@ -371,6 +529,8 @@ void Init_extra(){
    rb_define_const(rb_cIO, "DIRECT", UINT2NUM(O_DIRECT));
 #endif
 
+   rb_define_const(rb_cIO, "IOV_MAX", LONG2NUM(IOV_MAX));
+
 #ifdef HAVE_PREAD
    rb_define_singleton_method(rb_cIO, "pread", s_io_pread, 3);
    rb_define_singleton_method(rb_cIO, "pread_ptr", s_io_pread_ptr, 3);
@@ -380,6 +540,10 @@ void Init_extra(){
    rb_define_singleton_method(rb_cIO, "pwrite", s_io_pwrite, 3);
 #endif
 
-   /* 1.2.1: The version of this library. This a string. */
-   rb_define_const(rb_cIO, "EXTRA_VERSION", rb_str_new2("1.2.1"));
+#ifdef HAVE_WRITEV
+   rb_define_singleton_method(rb_cIO, "writev", s_io_writev, 2);
+#endif
+
+   /* 1.2.2: The version of this library. This a string. */
+   rb_define_const(rb_cIO, "EXTRA_VERSION", rb_str_new2("1.2.2"));
 }

data/io-extra.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
   end
    
   spec.name       = 'io-extra'
-  spec.version    = '1.2.1'
+  spec.version    = '1.2.2'
   spec.author     = 'Daniel J. Berger'
   spec.license    = 'Artistic 2.0'
   spec.email      = 'djberg96@gmail.com'

data/test/test_io_extra.rb

@@ -9,103 +9,147 @@ gem 'test-unit'
 
 require 'test/unit'
 require 'rbconfig'
+require 'io/nonblock'
 require 'io/extra'
 
 class TC_IO_Extra < Test::Unit::TestCase
-   def setup
-      @file = 'delete_this.txt'
-      @fh = File.open(@file, 'w+')
-      @fh.puts "The quick brown fox jumped over the lazy dog's back"
-   end
+  def setup
+    @file = 'delete_this.txt'
+    @fh = File.open(@file, 'w+')
+    @fh.puts "The quick brown fox jumped over the lazy dog's back"
+  end
      
-   def test_version
-      assert_equal('1.2.1', IO::EXTRA_VERSION)
-   end
-
-   def test_direct_constant
-      omit_unless(Config::CONFIG['host_os'] =~ /linux/i, 'Linux-only')
-      assert_equal(040000, IO::DIRECT)
-      assert_equal(040000, File::DIRECT)
-   end
-
-   def test_open_direct
-      omit_unless(Config::CONFIG['host_os'] =~ /linux/i, 'Linux-only')
-      assert_nothing_raised do
-         fh = File.open(@fh.path, IO::RDWR|IO::DIRECT)
-         fh.close
-      end
-   end
-
-   def test_directio
-      omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
-      assert_respond_to(@fh, :directio?)
-      assert_nothing_raised{ @fh.directio? }
-   end
-
-   def test_directio_set
-      omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
-      assert_respond_to(@fh, :directio=)
-      assert_raises(StandardError){ @fh.directio = 99 }
-      assert_nothing_raised{ @fh.directio = IO::DIRECTIO_ON }
-   end
-
-   def test_constants
-      omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
-      assert_not_nil(IO::DIRECTIO_ON)
-      assert_not_nil(IO::DIRECTIO_OFF)
-   end
-
-   def test_fdwalk
-      omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
-      assert_respond_to(IO, :fdwalk)
-      assert_nothing_raised{ IO.fdwalk(0){ } }
-   end
-
-   def test_closefrom
-      assert_respond_to(IO, :closefrom)
-      assert_nothing_raised{ IO.closefrom(3) }
-   end
-
-   def test_pread
-      @fh.close rescue nil
-      @fh = File.open(@file)
-      assert_respond_to(IO, :pread)
-      assert_equal("quick", IO.pread(@fh.fileno, 5, 4))
-   end
-
-   def test_pread_binary
-      @fh.close rescue nil
-      @fh = File.open(@file, "ab")
-      @fh.binmode
-      size = @fh.stat.size
-      assert_nothing_raised { @fh.syswrite("FOO\0HELLO") }
-      @fh.close rescue nil
-      @fh = File.open(@file)
-      assert_equal("O\0H", IO.pread(@fh.fileno, 3, size + 2))
-   end
-
-   def test_pread_ptr
-      @fh.close rescue nil
-      @fh = File.open(@file)
-      assert_respond_to(IO, :pread_ptr)
-      assert_kind_of(Integer, IO.pread_ptr(@fh.fileno, 5, 4))
-   end
-
-   def test_pread_last
-      @fh.close rescue nil
-      @fh = File.open(@file)
-      size = @fh.stat.size
-      assert_equal("ck\n", IO.pread(@fh.fileno, 5, size - 3))
-   end
-
-   def test_pwrite
-      assert_respond_to(IO, :pwrite)
-      assert_nothing_raised{ IO.pwrite(@fh.fileno, "HAL", 0) }
-   end
-  
-   def teardown
-      @fh.close rescue nil
-      @fh = nil
-      File.delete(@file) if File.exists?(@file)
-   end
+  def test_version
+    assert_equal('1.2.2', IO::EXTRA_VERSION)
+  end
+
+  def test_direct_constant
+    omit_unless(Config::CONFIG['host_os'] =~ /linux/i, 'Linux-only')
+    assert_equal(040000, IO::DIRECT)
+    assert_equal(040000, File::DIRECT)
+  end
+
+  def test_open_direct
+    omit_unless(Config::CONFIG['host_os'] =~ /linux/i, 'Linux-only')
+    assert_nothing_raised do
+      fh = File.open(@fh.path, IO::RDWR|IO::DIRECT)
+      fh.close
+    end
+  end
+
+  def test_directio
+    omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
+    assert_respond_to(@fh, :directio?)
+    assert_nothing_raised{ @fh.directio? }
+  end
+
+  def test_directio_set
+    omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
+    assert_respond_to(@fh, :directio=)
+    assert_raises(StandardError){ @fh.directio = 99 }
+    assert_nothing_raised{ @fh.directio = IO::DIRECTIO_ON }
+  end
+
+  def test_constants
+    omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
+    assert_not_nil(IO::DIRECTIO_ON)
+    assert_not_nil(IO::DIRECTIO_OFF)
+  end
+
+  def test_IOV_MAX_constant
+    assert_kind_of(Integer, IO::IOV_MAX)
+  end
+
+  def test_fdwalk
+    omit_if(Config::CONFIG['host_os'] =~ /darwin/i, 'unsupported')
+    assert_respond_to(IO, :fdwalk)
+    assert_nothing_raised{ IO.fdwalk(0){ } }
+  end
+
+  def test_closefrom
+    assert_respond_to(IO, :closefrom)
+    assert_nothing_raised{ IO.closefrom(3) }
+  end
+
+  def test_pread
+    @fh.close rescue nil
+    @fh = File.open(@file)
+    assert_respond_to(IO, :pread)
+    assert_equal("quick", IO.pread(@fh.fileno, 5, 4))
+  end
+
+  def test_pread_binary
+    @fh.close rescue nil
+    @fh = File.open(@file, "ab")
+    @fh.binmode
+    size = @fh.stat.size
+    assert_nothing_raised { @fh.syswrite("FOO\0HELLO") }
+    @fh.close rescue nil
+    @fh = File.open(@file)
+    assert_equal("O\0H", IO.pread(@fh.fileno, 3, size + 2))
+  end
+
+  def test_pread_ptr
+    @fh.close rescue nil
+    @fh = File.open(@file)
+    assert_respond_to(IO, :pread_ptr)
+    assert_kind_of(Integer, IO.pread_ptr(@fh.fileno, 5, 4))
+  end
+
+  def test_pread_last
+    @fh.close rescue nil
+    @fh = File.open(@file)
+    size = @fh.stat.size
+    assert_equal("ck\n", IO.pread(@fh.fileno, 5, size - 3))
+  end
+
+  def test_pwrite
+    assert_respond_to(IO, :pwrite)
+    assert_nothing_raised{ IO.pwrite(@fh.fileno, "HAL", 0) }
+  end
+
+  def test_writev
+    assert_respond_to(IO, :writev)
+    assert_equal(10, IO.writev(@fh.fileno, %w(hello world)))
+  end
+
+  def test_writev_retry
+    empty = ""
+    if empty.respond_to?(:force_encoding)
+      empty.force_encoding(Encoding::BINARY)
+    end
+
+    # bs * count should be > PIPE_BUF
+    [ true, false ].each do |nonblock|
+       [ [ 512, 512 ], [ 131073, 3 ], [ 4098, 64 ] ].each do |(bs,count)|
+          rd, wr = IO.pipe
+          wr.nonblock = nonblock
+          buf = File.open("/dev/urandom", "rb") { |fp| fp.sysread(bs) }
+          vec = (1..count).map { buf }
+          pid = fork do
+             wr.close
+             tmp = []
+             sleep 0.1
+             begin
+                tmp << rd.readpartial(8192, buf)
+             rescue EOFError
+                break
+             end while true
+             ok = (vec.join(empty) == tmp.join(empty))
+             exit! ok
+          end
+          assert_nothing_raised { rd.close }
+          assert_equal(bs * count, IO.writev(wr.fileno, vec))
+          assert_nothing_raised { wr.close }
+          _, status = Process.waitpid2(pid)
+          assert status.success?
+       end
+    end
+  end
+
+  def teardown
+    @fh.close rescue nil
+    @fh = nil
+    File.delete(@file) if File.exists?(@file)
+  end
 end

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: io-extra
 version: !ruby/object:Gem::Version 
-  version: 1.2.1
+  prerelease: false
+  segments: 
+  - 1
+  - 2
+  - 2
+  version: 1.2.2
 platform: ruby
 authors: 
 - Daniel J. Berger
@@ -9,19 +14,23 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-16 00:00:00 -07:00
+date: 2010-05-01 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: test-unit
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 0
+        - 3
         version: 2.0.3
-    version: 
+  type: :development
+  version_requirements: *id001
 description: "    Adds the IO.closefrom, IO.fdwalk, IO.pread, IO.pread_ptr, and IO.pwrite\n    class methods as well as the IO#directio and IO#directio? instance\n    methods (for those platforms that support them).\n"
 email: djberg96@gmail.com
 executables: []
@@ -34,16 +43,16 @@ extra_rdoc_files:
 - MANIFEST
 - ext/io/extra.c
 files: 
-- MANIFEST
-- examples/example_io_extra.rb
-- README
 - CHANGES
+- doc/io_extra.txt
+- examples/example_io_extra.rb
+- ext/extconf.rb
+- ext/io/extra.c
 - io-extra.gemspec
+- MANIFEST
 - Rakefile
+- README
 - test/test_io_extra.rb
-- doc/io_extra.txt
-- ext/io/extra.c
-- ext/extconf.rb
 has_rdoc: true
 homepage: http://www.rubyforge.org/projects/shards
 licenses: 
@@ -57,18 +66,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 6
       version: 1.8.6
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: shards
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Adds extra methods to the IO class.