hadupils 0.6.1 → 0.7.2

data/CHANGELOG.md

@@ -78,3 +78,21 @@
   was hanging reading from stderr, which isn't strictly necessary anyway.
   Now it no hangy.
 
+### 0.6.2
+* Stopped the hadupils cleanup command from outputting 'Removing...' to STDOUT when it
+  doesn't have anything to actually remove.
+
+### 0.7.0
+* Now using Process.spawn instead of Kernel.system, the ruby way for 1.9+
+* Fixed subcommand process deadlocking when any IO pipe buffer became full (Ruby 1.9+) via
+  threads and process waiting, the ruby way or a ruby way at least ;)
+* $HADUPILS_TMP_TTL now defaults to 1209600 (last 2 weeks)
+* Passing unit tests
+
+### 0.7.1
+* Cleanup command now outputs additional context, candidate tmpdir, to help identify the
+  runtime state for long running cleanups
+* Now ignoring the 'vendor' directory in .gitignore
+
+### 0.7.2
+* $HADUPILS_TMP_TTL default reverts back to 86400 (last 24 hours)

data/lib/hadupils/commands.rb

@@ -234,7 +234,7 @@ module Hadupils::Commands
             # Now want the user to see the Runner's shell STDOUT
             Shell.silence_stdout = false
 
-            puts 'Removing...'
+            puts 'Removing...' unless rm_array.empty?
             rm_array.each do |dir|
               rm_stdout, rm_exitstatus = Hadupils::Commands::RmFile.run ['-r', dir]
               rm_exitstatuses << rm_exitstatus
@@ -248,6 +248,7 @@ module Hadupils::Commands
     end
 
     def has_expired?(dir_candidate, ttl)
+      puts "Checking directory candidate: #{dir_candidate}"
       stdout, exitstatus = Hadupils::Commands::Hadoop.run ['fs', '-count', dir_candidate]
       expired_exitstatuses << exitstatus
       if successful? exitstatus

data/lib/hadupils/extensions.rb

@@ -7,22 +7,19 @@ module Hadupils::Extensions
   module Runners
     module Shell
       def self.command(*command_list)
-        opts      = {}
-        stdout    = nil
-        stderr    = nil
-        status    = nil
+        opts = {}
 
         begin
           if RUBY_VERSION < '1.9'
             Open3.popen3(*command_list) do |i, o, e|
-              stdout = o.read
-              stderr = e.read
+              @stdout = o.read
+              @stderr = e.read
             end
-            status = $?
-            $stdout.puts stdout unless stdout.nil? || stdout.empty? || Shell.silence_stdout?
-            $stderr.puts stderr unless stderr.nil? || stderr.empty?
-            stdout = nil unless capture_stdout?
-            stderr = nil unless capture_stderr?
+            @status = $?
+            $stdout.puts @stdout unless @stdout.nil? || @stdout.empty? || Shell.silence_stdout?
+            $stderr.puts @stderr unless @stderr.nil? || @stderr.empty?
+            @stdout = nil unless capture_stdout?
+            @stderr = nil unless capture_stderr?
           else
             stdout_rd, stdout_wr  = IO.pipe     if capture_stdout?
             stderr_rd, stderr_wr  = IO.pipe     if capture_stderr?
@@ -30,25 +27,38 @@ module Hadupils::Extensions
             opts[:err]            = stderr_wr   if capture_stderr?
 
             # NOTE: eval prevents Ruby 1.8.7 from throwing a syntax error on Ruby 1.9+ syntax
-            result = eval 'Kernel.system(*command_list, opts)'
-            status = result ? $? : nil
-            if capture_stdout?
-              stdout_wr.close
-              stdout = stdout_rd.read
-              stdout_rd.close
-              $stdout.puts stdout unless stdout.nil? || stdout.empty? || Shell.silence_stdout?
+            pid = eval 'Process.spawn(*command_list, opts)'
+            @status = :waiting unless pid.nil?
+
+            Thread.new do
+              Process.wait pid
+              @status = $?
             end
-            if capture_stderr?
-              stderr_wr.close
-              stderr = stderr_rd.read
-              stderr_rd.close
-              $stderr.puts stderr unless stderr.nil? || stderr.empty?
+
+            Thread.new do
+              if capture_stdout?
+                stdout_wr.close
+                @stdout = stdout_rd.read
+                stdout_rd.close
+                $stdout.puts @stdout unless @stdout.nil? || @stdout.empty? || Shell.silence_stdout?
+              end
             end
+
+            Thread.new do
+              if capture_stderr?
+                stderr_wr.close
+                @stderr = stderr_rd.read
+                stderr_rd.close
+                $stderr.puts @stderr unless @stderr.nil? || @stderr.empty?
+              end
+            end
+            sleep 0.1 while @status == :waiting
           end
-          [stdout, stderr, status]
+
+          [@stdout, @stderr, @status]
         rescue Errno::ENOENT => e
           $stderr.puts e
-          [stdout, stderr, nil]
+          [@stdout, @stderr, nil]
         end
       end
 
@@ -86,7 +96,7 @@ module Hadupils::Extensions
       end
 
       def self.tmp_ttl
-        @tmp_ttl ||= (ENV['HADUPILS_TMP_TTL'] || '86400').to_i
+        @tmp_ttl ||= (ENV['HADUPILS_TMP_TTL'] || '1209600').to_i
       end
 
       def self.tmp_path
@@ -174,7 +184,7 @@ module Hadupils::Extensions
         @file_handler || ::Tempfile
       end
 
-      # Sets up a wrapped file, using the class' file_handler, 
+      # Sets up a wrapped file, using the class' file_handler,
       def initialize
         @file = self.class.file_handler.new('hadupils-hiverc')
       end

data/test/unit/commands_test.rb

@@ -292,7 +292,7 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
         end
 
         should 'produce a valid set of parameters and hivercs' do
-          Kernel.stubs(:system).with() do |*args|
+          Process.stubs(:spawn).with() do |*args|
             args[0] == {'HIVE_AUX_JARS_PATH' => @hive_aux_jars_path_val} &&
             args[1] == @hive_prog &&
             args[2] == '-i' &&
@@ -464,7 +464,7 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
        run_common_cleanup_assertions_with(tmp_path, tmpdir1, tmpdir2)
        instance = @klass.new([])
        assert_equal [nil, 0], instance.run
-       assert_equal 86400, instance.tmp_ttl
+       assert_equal 1209600, instance.tmp_ttl
        assert_equal '/tmp', instance.tmp_path
      end
 
@@ -476,7 +476,7 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
        run_common_cleanup_assertions_with(tmp_path, tmpdir1, tmpdir2)
        instance = @klass.new([tmp_path])
        assert_equal [nil, 0], instance.run
-       assert_equal 86400, instance.tmp_ttl
+       assert_equal 1209600, instance.tmp_ttl
        assert_equal tmp_path, instance.tmp_path
      end
 
@@ -504,8 +504,8 @@ class Hadupils::CommandsTest < Test::Unit::TestCase
   def run_common_cleanup_assertions_with(tmp_path, tmpdir1, tmpdir2)
     ls_stdout =
       "Found 2 items\n" +
-      "drwx------   - willdrew supergroup          0 2013-10-24 16:23 #{tmpdir1}\n" +
-      "drwx------   - willdrew supergroup          0 2013-10-24 16:23 #{tmpdir2}\n"
+      "drwx------   - someuser somegroup          0 2013-10-24 16:23 #{tmpdir1}\n" +
+      "drwx------   - someuser somegroup          0 2013-10-24 16:23 #{tmpdir2}\n"
     count_stdout1 = "           1            0                  0 hdfs://localhost:9000#{tmpdir1}\n"
     count_stdout2 = "           1            1                  0 hdfs://localhost:9000#{tmpdir2}\n"
     Hadupils::Runners::Hadoop.expects(:run).with(['fs', '-ls', tmp_path]).returns([ls_stdout, 0])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hadupils
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-28 00:00:00.000000000 Z
+date: 2014-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuid
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.5
+        version: 1.6.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.3.5
+        version: 1.6.2
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -98,31 +98,31 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/hadupils/search.rb
 - lib/hadupils/commands.rb
-- lib/hadupils/runners.rb
+- lib/hadupils/search.rb
 - lib/hadupils/extensions/hive.rb
-- lib/hadupils/helpers.rb
 - lib/hadupils/commands/options.rb
 - lib/hadupils/extensions.rb
+- lib/hadupils/util.rb
 - lib/hadupils/hacks.rb
+- lib/hadupils/runners.rb
+- lib/hadupils/helpers.rb
 - lib/hadupils/assets.rb
-- lib/hadupils/util.rb
 - lib/hadupils.rb
-- test/unit/assets_test.rb
-- test/unit/commands_test.rb
+- test/hadupil_test_setup.rb
 - test/unit/runners_test.rb
-- test/unit/search_test.rb
 - test/unit/extensions/base_test.rb
 - test/unit/extensions/hive_test.rb
-- test/hadupil_test_setup.rb
+- test/unit/assets_test.rb
+- test/unit/search_test.rb
+- test/unit/commands_test.rb
 - bin/hadupils
+- CHANGELOG.md
+- Gemfile
 - Rakefile.rb
-- Gemfile.lock
 - README.md
-- Gemfile
+- Gemfile.lock
 - LICENSE
-- CHANGELOG.md
 homepage: http://github.com/ethanrowe/hadupils
 licenses:
 - MIT
@@ -144,7 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Provides utilities for dynamic hadoop client environment configuration