unix_utils 0.0.2 → 0.0.4

data/Gemfile

@@ -5,3 +5,5 @@ gemspec
 # development dependencies
 gem 'minitest'
 gem 'minitest-reporters'
+gem 'yard'
+# gem 'ruby-prof'

data/History.txt

@@ -1,3 +1,20 @@
+== 0.0.4 / 2012-03-19
+
+* Bug fixes
+
+  * NoMethodError when trying to print $stderr - thanks @leomao10 !
+
+== 0.0.3 / 2012-03-19
+
+* Enhancements
+
+  * Uses https://github.com/rtomayko/posix-spawn to make things faster - suggested by @jjb in https://github.com/seamusabshere/unix_utils/issues/1
+  * Depends less on chdir to work properly - expands file paths immediately
+
+* Bug fixes
+
+  * Correctly use pipes (I hope) - imitate POSIX::Spawn::Child's use of IO.select
+
 == 0.0.2 / 2012-02-16
 
 * Bug fixes

data/README.markdown

@@ -4,7 +4,27 @@ Like FileUtils, but provides zip, unzip, bzip2, bunzip2, tar, untar, sed, du, md
 
 Works in MRI 1.8.7+, MRI 1.9.2+, and JRuby 1.6.7+
 
-## What to expect
+## Where it's used
+
+* [Brighter Planet CM1 Impact Estimate web service](http://impact.brighterplanet.com)
+* [Brighter Planet Reference Data web service](http://data.brighterplanet.com)
+* Extracted from [`remote_table`](https://github.com/seamusabshere/remote_table)
+
+## Philosophy
+
+Use a subprocess to perform a big task and then get out of memory.
+
+<table>
+  <tr>
+    <td rowspan="2"><img src="https://raw.github.com/seamusabshere/unix_utils/master/unix-philosophy-cover.png" alt="cover of the Unix Philosophy book" /></td>
+    <td><img src="https://raw.github.com/seamusabshere/unix_utils/master/unix-philosophy-quote-pg1.png" alt="Tenet 2: Make Each Program Do One Thing Well. The best programs, like Cousteau's lake fly, does but one task in its life and does it well. The program is loaded into memory, accomplishes its function, and then gets out ot the way to allow" /></td>
+  </tr>
+  <tr>
+    <td><img src="https://raw.github.com/seamusabshere/unix_utils/master/unix-philosophy-quote-pg2.png" alt="the next single-minded program to begin. This sounds simple, yet it may surprise you how many software developers have difficulty sticking to this singular goal." /></td>
+  </tr>
+</table>
+
+## Rules (what you can expect)
 
 For commands like zip, untar, sed, head, cut, dos2unix, etc.:
 
@@ -17,10 +37,6 @@ For commands like du, md5sum, shasum, etc.:
 1. Just returns the good stuff (the checksum, for example, not the filename that is listed after it in the standard command output)
 2. Never touches the input
 
-## Philosophy
-
-Use a subprocess to perform a big task and then get out of memory.
-
 ## But I can just spawn these myself
 
 This lib was created to ease the pain of remembering command options for Gentoo, deciding which spawning method to use, possibly handling pipes...
@@ -39,6 +55,8 @@ is replaced safely with
 
 ## But I can just use `Digest::SHA256`
 
+(Note: [Balazs Kutil](https://github.com/bkutil) pointed out [this is a bad example](https://gist.github.com/1950707)... I will replace it soon)
+
 This will load an entire file into memory before it can be processed...
 
     require 'digest'
@@ -60,13 +78,9 @@ You get the same low memory footprint with
 
 ## Compatibility
 
-Uses `open3` because it's in the Ruby stdlib and is consistent across MRI and JRuby.
-
-## Where it's used
+Now using [`posix-spawn`](https://github.com/rtomayko/posix-spawn) for speed. Thanks for the suggestion [jjb](https://github.com/jjb)!
 
-* [Brighter Planet Reference Data web service](http://data.brighterplanet.com)
-* [Brighter Planet Emission Estimate web service](http://impact.brighterplanet.com) aka CM1
-* [`remote_table` library](https://github.com/seamusabshere/remote_table)
+Previously used `open3` because it's in the Ruby stdlib and is consistent across MRI and JRuby.
 
 ## Authors
 

data/Rakefile

@@ -10,3 +10,6 @@ Rake::TestTask.new(:test) do |test|
 end
 
 task :default => :test
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/lib/unix_utils/version.rb

@@ -1,3 +1,3 @@
 module UnixUtils
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

data/lib/unix_utils.rb

@@ -1,12 +1,15 @@
 require 'fileutils'
 require 'tmpdir'
 require 'uri'
-require 'open3'
+require 'stringio'
+require 'posix/spawn'
 
 require "unix_utils/version"
 
 module UnixUtils
 
+  BUFSIZE = 4_194_304
+
   def self.curl(url, form_data = nil)
     outfile = tmp_path url
     if url.start_with?('/') or url.start_with?('file://')
@@ -26,15 +29,16 @@ module UnixUtils
 
   #--
   # most platforms
-  # $ openssl dgst -sha256 .bash_profile 
+  # $ openssl dgst -sha256 .bash_profile
   # SHA256(.bash_profile)= ae12206aaa35dc96273ed421f4e85ca26a1707455e3cc9f054c7f5e2e9c53df6
   # ubuntu 11.04
-  # $ shasum -a 256 --portable .mysql_history 
+  # $ shasum -a 256 --portable .mysql_history
   # 856aa27deb0b80b41031c2ddf722af28ba2a8c4999ff9cf2d45f33bc67d992ba ?.mysql_history
   # fedora 7
-  # $ sha256sum --binary .bash_profile 
+  # $ sha256sum --binary .bash_profile
   # 01b1210962b3d1e5e1ccba26f93d98efbb7b315b463f9f6bdb40ab496728d886 *.bash_profile
   def self.shasum(infile, algorithm)
+    infile = ::File.expand_path infile
     if available?('shasum')
       argv = ['shasum', '--binary', '-a', algorithm.to_s, infile]
       stdout = spawn argv
@@ -45,15 +49,16 @@ module UnixUtils
       stdout.strip.split(' ').last
     end
   end
-  
+
   #--
   # os x 10.6.8; most platforms
-  # $ openssl dgst -md5 .bashrc 
+  # $ openssl dgst -md5 .bashrc
   # MD5(.bashrc)= 88f464fb6d1d6fe9141135248bf7b265
   # ubuntu 11.04; fedora 7; gentoo
-  # $ md5sum --binary .mysql_history 
-  # 8d01e54ab8142d6786850e22d55a1b6c *.mysql_history  
+  # $ md5sum --binary .mysql_history
+  # 8d01e54ab8142d6786850e22d55a1b6c *.mysql_history
   def self.md5sum(infile)
+    infile = ::File.expand_path infile
     if available?('md5sum')
       argv = ['md5sum', '--binary', infile]
       stdout = spawn argv
@@ -64,22 +69,25 @@ module UnixUtils
       stdout.strip.split(' ').last
     end
   end
-    
+
   def self.du(srcdir)
+    srcdir = ::File.expand_path srcdir
     argv = ['du', srcdir]
     stdout = spawn argv
     stdout.strip.split(/\s+/).first.to_i
   end
-  
+
   def self.wc(infile)
+    infile = ::File.expand_path infile
     argv = ['wc', infile]
     stdout = spawn argv
     stdout.strip.split(/\s+/)[0..2].map { |s| s.to_i }
   end
 
   # --
-  
+
   def self.unzip(infile)
+    infile = ::File.expand_path infile
     destdir = tmp_path infile
     ::FileUtils.mkdir destdir
     argv = ['unzip', '-qq', '-n', infile, '-d', destdir]
@@ -88,6 +96,7 @@ module UnixUtils
   end
 
   def self.untar(infile)
+    infile = ::File.expand_path infile
     destdir = tmp_path infile
     ::FileUtils.mkdir destdir
     argv = ['tar', '-xf', infile, '-C', destdir]
@@ -96,6 +105,7 @@ module UnixUtils
   end
 
   def self.gunzip(infile)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     argv = ['gunzip', '--stdout', infile]
     spawn argv, :write_to => outfile
@@ -103,8 +113,9 @@ module UnixUtils
   end
 
   def self.bunzip2(infile)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
-    argv = ['bunzip2', '--stdout', infile,]
+    argv = ['bunzip2', '--stdout', infile]
     spawn argv, :write_to => outfile
     outfile
   end
@@ -112,13 +123,15 @@ module UnixUtils
   # --
 
   def self.bzip2(infile)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile, '.bz2'
     argv = ['bzip2', '--keep', '--stdout', infile]
     spawn argv, :write_to => outfile
     outfile
   end
-  
+
   def self.tar(srcdir)
+    srcdir = ::File.expand_path srcdir
     outfile = tmp_path srcdir, '.tar'
     argv = ['tar', '-cf', outfile, '-C', srcdir, '.']
     spawn argv
@@ -126,46 +139,52 @@ module UnixUtils
   end
 
   def self.zip(srcdir)
+    srcdir = ::File.expand_path srcdir
     outfile = tmp_path srcdir, '.zip'
     argv = ['zip', '-rq', outfile, '.']
     spawn argv, :chdir => srcdir
     outfile
   end
-  
+
   def self.gzip(infile)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile, '.gz'
     argv = ['gzip', '--stdout', infile]
     spawn argv, :write_to => outfile
     outfile
   end
-  
+
   # --
-  
+
   def self.awk(infile, *expr)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     bin = available?('gawk') ? 'gawk' : 'awk'
     argv = [bin, expr, infile].flatten
     spawn argv, :write_to => outfile
     outfile
   end
-  
+
   # Yes, this is a very limited use of perl.
   def self.perl(infile, *expr)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     argv = [ 'perl', expr.map { |e| ['-pe', e] } ].flatten
     spawn argv, :read_from => infile, :write_to => outfile
     outfile
   end
-  
+
   def self.unix2dos(infile)
+    infile = ::File.expand_path infile
     if available?('gawk') or available?('awk')
       awk infile, '{ sub(/\r/, ""); printf("%s\r\n", $0) }'
     else
       perl infile, 's/\r\n|\n|\r/\r\n/g'
     end
   end
-  
+
   def self.dos2unix(infile)
+    infile = ::File.expand_path infile
     if available?('gawk') or available?('awk')
       awk infile, '{ sub(/\r/, ""); printf("%s\n", $0) }'
     else
@@ -174,6 +193,7 @@ module UnixUtils
   end
 
   def self.sed(infile, *expr)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     bin = available?('gsed') ? 'gsed' : 'sed'
     argv = [ bin, expr.map { |e| ['-e', e] } ].flatten
@@ -182,13 +202,15 @@ module UnixUtils
   end
 
   def self.tail(infile, lines)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     argv = ['tail', '-n', lines.to_s, infile]
     spawn argv, :write_to => outfile
     outfile
   end
-  
+
   def self.head(infile, lines)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     argv = ['head', '-n', lines.to_s, infile]
     spawn argv, :write_to => outfile
@@ -197,6 +219,7 @@ module UnixUtils
 
   # specify character_positions as a string like "3-5" or "3,9-10"
   def self.cut(infile, character_positions)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     argv = ['cut', '-c', character_positions, infile]
     spawn argv, :write_to => outfile
@@ -204,12 +227,13 @@ module UnixUtils
   end
 
   def self.iconv(infile, to, from)
+    infile = ::File.expand_path infile
     outfile = tmp_path infile
     argv = ['iconv', '-t', to, '-f', from, infile]
     spawn argv, :write_to => outfile
     outfile
   end
-  
+
   def self.available?(bin) # :nodoc:
     bin = bin.to_s
     return @@available_query[bin] if defined?(@@available_query) and @@available_query.is_a?(::Hash) and @@available_query.has_key?(bin)
@@ -226,51 +250,76 @@ module UnixUtils
   end
 
   def self.spawn(argv, options = {}) # :nodoc:
-    if options[:chdir]
-      old_pwd = ::Dir.pwd
-      ::Dir.chdir options[:chdir]
+    options = options.dup
+
+    input = if (read_from = options.delete(:read_from))
+      ::File.open(read_from, 'r')
     end
+
+    output = if (write_to = options.delete(:write_to))
+      output_redirected = true
+      ::File.open(write_to, 'wb')
+    else
+      output_redirected = false
+      ::StringIO.new
+    end
+
+    error = ::StringIO.new
+
+    pid, stdin, stdout, stderr = ::POSIX::Spawn.popen4(*(argv+[options]))
     
-    whole_stdout = nil
-    whole_stderr = nil
-
-    ::Open3.popen3(*argv) do |stdin, stdout, stderr|
-      # deal with STDIN
-      if options[:read_from]
-        ::File.open(options[:read_from], 'r') do |in_f|
-          while chunk = in_f.read(4_194_304)
-            stdin.write chunk
-          end
+    # lifted from posix-spawn
+    # https://github.com/rtomayko/posix-spawn/blob/master/lib/posix/spawn/child.rb
+    readers = [stdout, stderr]
+    writers = if input
+      [stdin]
+    else
+      stdin.close
+      []
+    end
+    while readers.any? or writers.any?
+      ready = ::IO.select(readers, writers, readers + writers)
+      # write to stdin stream
+      ready[1].each do |fd|
+        begin
+          boom = nil
+          size = fd.write_nonblock(input.read(BUFSIZE))
+        rescue ::Errno::EPIPE => boom
+        rescue ::Errno::EAGAIN, ::Errno::EINTR
+        end
+        if boom || size < BUFSIZE
+          stdin.close
+          input.close
+          writers.delete(stdin)
         end
       end
-      stdin.close
-
-      # deal with STDOUT
-      if options[:write_to]
-        ::File.open(options[:write_to], 'wb') do |out_f|
-          while chunk = stdout.read(4_194_304)
-            out_f.write chunk
-          end
+      # read from stdout and stderr streams
+      ready[0].each do |fd|
+        buf = (fd == stdout) ? output : error
+        begin
+          buf << fd.readpartial(BUFSIZE)
+        rescue ::Errno::EAGAIN, ::Errno::EINTR
+        rescue ::EOFError
+          readers.delete(fd)
+          fd.close
         end
-        whole_stdout = "Redirected to #{options[:write_to]}"
-      else
-        whole_stdout = stdout.read
       end
-      
-      # deal with STDERR
-      whole_stderr = stderr.read
     end
+    # thanks @tmm1 and @rtomayko for showing how it's done!
+
+    ::Process.waitpid pid
 
-    unless whole_stderr.empty?
+    error.rewind
+    unless (whole_error = error.read).empty?
       $stderr.puts "[unix_utils] `#{argv.join(' ')}` STDERR:"
-      $stderr.puts whole_stderr
+      $stderr.puts whole_error
     end
 
-    whole_stdout
-
-  ensure
-    if options[:chdir]
-      ::Dir.chdir old_pwd
+    unless output_redirected
+      output.rewind
+      output.read
     end
+  ensure
+    [stdin, stdout, stderr, input, output, error].each { |io| io.close if io and not io.closed? }
   end
 end

data/test/helper.rb

@@ -11,6 +11,7 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'unix_utils'
 
+require 'stringio'
 require 'fileutils'
 require 'tmpdir'
 require 'tempfile'
@@ -36,7 +37,7 @@ module TestHelper
       UnixUtils.du(infile_or_srcdir).must_equal size
     end
   end
-  
+
   def assert_unpack_dir(method_id, infile)
     destdir = UnixUtils.send method_id, infile
     File.directory?(destdir).must_equal true
@@ -44,7 +45,7 @@ module TestHelper
     File.dirname(destdir).start_with?(Dir.tmpdir).must_equal true
     safe_delete destdir
   end
-  
+
   def assert_unpack_file(method_id, infile)
     outfile = UnixUtils.send method_id, infile
     File.file?(outfile).must_equal true
@@ -52,7 +53,7 @@ module TestHelper
     File.dirname(outfile).start_with?(Dir.tmpdir).must_equal true
     safe_delete outfile
   end
-    
+
   def assert_pack(method_id, infile)
     outfile = UnixUtils.send method_id, infile
     File.file?(outfile).must_equal true
@@ -60,7 +61,7 @@ module TestHelper
     File.dirname(outfile).start_with?(Dir.tmpdir).must_equal true
     safe_delete outfile
   end
-  
+
   def safe_delete(path)
     path = File.expand_path path
     raise "Refusing to rm -rf #{path} because it's not in #{Dir.tmpdir}" unless File.dirname(path).start_with?(Dir.tmpdir)

data/test/test_unix_utils.rb

@@ -6,11 +6,26 @@ describe UnixUtils do
     @old_pwd = Dir.pwd
     Dir.chdir File.expand_path('../target', __FILE__)
   end
-  
+
   after do
     Dir.chdir @old_pwd
   end
-  
+
+  describe "errors" do
+    it "gets printed to stderr" do
+      begin
+        old_stderr = $stderr
+        capture = StringIO.new
+        $stderr = capture
+        UnixUtils.unzip(__FILE__)
+        capture.rewind
+        capture.read.must_match %r{End-of-central-directory signature not found}
+      ensure
+        $stderr = old_stderr
+      end
+    end
+  end
+
   describe :curl do
     it "downloads to a temp file" do
       outfile = UnixUtils.curl('http://brighterplanet.com')
@@ -18,7 +33,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :shasum do
     it "checksums a file with SHA-1" do
       UnixUtils.shasum('directory.zip', 1).must_equal 'c0abb36c923ed7bf87ebb8d7097cb8e264e528d2'
@@ -26,8 +41,14 @@ describe UnixUtils do
     it "checksums a file with SHA-256" do
       UnixUtils.shasum('directory.zip', 256).must_equal '661af2b7b0993088263228b071b649a88d82a6a655562162c32307d1e127f27a'
     end
+    it "works just as well with absolute paths" do
+      target_path = File.join(Dir.pwd, 'directory.zip')
+      Dir.chdir('/') do
+        UnixUtils.shasum(target_path, 256).must_equal '661af2b7b0993088263228b071b649a88d82a6a655562162c32307d1e127f27a'
+      end
+    end
   end
-  
+
   describe :md5sum do
     it "checksums a file" do
       UnixUtils.md5sum('directory.zip').must_equal 'd6e15da798ae19551da6c49ec09afaef'
@@ -39,7 +60,7 @@ describe UnixUtils do
       UnixUtils.du('directory').must_equal 16
     end
   end
-  
+
   describe :unzip do
     before do
       @infile = 'directory.zip'
@@ -55,7 +76,7 @@ describe UnixUtils do
       assert_does_not_touch :unzip, @infile
     end
   end
-  
+
   describe :untar do
     before do
       @infile = 'directory.tar'
@@ -71,7 +92,7 @@ describe UnixUtils do
       assert_does_not_touch :untar, @infile
     end
   end
-  
+
   describe :bunzip2 do
     before do
       @infile = 'file.bz2'
@@ -87,7 +108,7 @@ describe UnixUtils do
       assert_does_not_touch :bunzip2, @infile
     end
   end
-  
+
   describe :gunzip do
     before do
       @infile = 'file.gz'
@@ -103,7 +124,7 @@ describe UnixUtils do
       assert_does_not_touch :gunzip, @infile
     end
   end
-  
+
   describe :bzip2 do
     before do
       @infile = 'directory.tar'
@@ -120,7 +141,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :gzip do
     before do
       @infile = 'directory.tar'
@@ -137,7 +158,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :zip do
     before do
       @srcdir = 'directory'
@@ -154,7 +175,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :tar do
     before do
       @srcdir = 'directory'
@@ -171,7 +192,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :perl do
     before do
       @f = Tempfile.new('perl.txt')
@@ -196,7 +217,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :awk do
     before do
       @f = Tempfile.new('awk.txt')
@@ -221,7 +242,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :unix2dos do
     before do
       @f = Tempfile.new('unix2dos.txt')
@@ -239,7 +260,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :dos2unix do
     before do
       @f = Tempfile.new('dos2unix.txt')
@@ -257,7 +278,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :wc do
     before do
       @f = Tempfile.new('wc.txt')
@@ -297,7 +318,7 @@ describe UnixUtils do
       File.extname(outfile).must_equal File.extname(@infile)
       safe_delete outfile
     end
-    
+
   end
 
   describe :tail do
@@ -322,7 +343,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :head do
     before do
       @a2z = ('a'..'z').to_a
@@ -340,7 +361,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :cut do
     before do
       @a2z = ('a'..'z').to_a
@@ -367,7 +388,7 @@ describe UnixUtils do
       safe_delete outfile
     end
   end
-  
+
   describe :iconv do
     it 'converts files from utf-8 to latin1' do
       outfile = UnixUtils.iconv('utf8.txt', 'ISO-8859-1', 'UTF-8')

data/unix_utils.gemspec

@@ -15,4 +15,6 @@ Gem::Specification.new do |gem|
   gem.name          = "unix_utils"
   gem.require_paths = ["lib"]
   gem.version       = UnixUtils::VERSION
+
+  gem.add_runtime_dependency 'posix-spawn'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: unix_utils
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,19 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-17 00:00:00.000000000 Z
-dependencies: []
+date: 2012-03-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: posix-spawn
+  requirement: &2154399440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2154399440
 description: Like FileUtils, but provides zip, unzip, bzip2, bunzip2, tar, untar,
   sed, du, md5sum, shasum, cut, head, tail, wc, unix2dos, dos2unix, iconv, curl, perl,
   etc.
@@ -41,6 +52,9 @@ files:
 - test/target/iso-8859-1.txt
 - test/target/utf8.txt
 - test/test_unix_utils.rb
+- unix-philosophy-cover.png
+- unix-philosophy-quote-pg1.png
+- unix-philosophy-quote-pg2.png
 - unix_utils.gemspec
 homepage: https://github.com/seamusabshere/unix_utils
 licenses: []